Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
37bdc20ca9 | ||
|
|
37b31940fa | ||
|
|
21ba73ff39 | ||
|
|
7a47f936da | ||
|
|
af26c2fd49 | ||
|
|
43e70514d1 | ||
|
|
38f9f8acc3 | ||
|
|
b9636ce88c | ||
|
|
0e6ec0647f | ||
|
|
a3d5c16318 | ||
|
|
bc14f29db7 | ||
|
|
1659b58c5c | ||
|
|
3cfa81419b | ||
|
|
f95f66d76a | ||
|
|
2cecfcf0ac | ||
|
|
a6f6697c50 | ||
|
|
6e9d8f81fd | ||
|
|
454ac12e1d | ||
|
|
27b4895795 | ||
|
|
e9996996e2 | ||
|
|
762669758b | ||
|
|
1ff2ddce05 |
@@ -16,9 +16,19 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
# Timeouts are the measured job time plus at least 50%, and never under 20 minutes for a browser job,
|
||||
# because runner speed varies. Before sharding (main 2cecfcf, one worker): frontend coverage 32m10s,
|
||||
# page workflows firefox 18m59s, webkit 19m26s. First sharded run (PR #274 run 39712909047, 3 workers,
|
||||
# no npm cache, where npm ci stalled for 5m in six of eight jobs): coverage shards 11m19s/10m47s/10m49s
|
||||
# (tests 5m20s/4m42s/4m46s), merge 5m52s (report and check 4s), firefox shards 7m53s/9m27s (tests
|
||||
# 4m41s/3m29s), webkit shards 12m25s+ (cancelled at the old 12-minute limit after its 267 tests passed in
|
||||
# 6m29s) and 4m53s (tests 3m08s). With the npm cache hit (run 39725176320 at cbc8a61, npm ci 0-2s):
|
||||
# coverage shards 7m34s/6m04s/5m22s (tests 6m38s/4m53s/4m04s), merge 54s, firefox 7m57s/5m21s (tests
|
||||
# 7m08s/4m34s), webkit 8m16s/5m03s (tests 7m26s/3m58s); a cache miss adds up to 5m to npm ci, which
|
||||
# the 20-minute limits absorb.
|
||||
jobs:
|
||||
frontend-coverage:
|
||||
name: frontend coverage (chromium and touch)
|
||||
name: frontend coverage (shard ${{ matrix.shard }}/3)
|
||||
runs-on: ubuntu-latest
|
||||
# Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the
|
||||
# same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json;
|
||||
@@ -26,12 +36,15 @@ jobs:
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble
|
||||
options: --user 1001
|
||||
# This job runs every page workflow twice (chromium, then touch) on one worker with V8 coverage on, so it
|
||||
# takes about twice a single-browser job. Measured 2026-10-20: whole job 18m26s-24m57s (main 5ded3cf
|
||||
# 22m15s; PR head 6359999 was cancelled at the old 25-minute limit while still passing), coverage step
|
||||
# alone 20m55s-22m17s+, against ~12-15 minutes for the firefox/webkit jobs. 40 minutes leaves headroom
|
||||
# for suite growth while still stopping a hung run; re-measure if a run passes 30 minutes.
|
||||
timeout-minutes: 40
|
||||
timeout-minutes: 20
|
||||
# Public-repo ubuntu-latest runners have 4 vCPUs: three Playwright workers, each with its own test
|
||||
# server, leave one CPU for the servers and the runner.
|
||||
env:
|
||||
PM_TEST_WORKERS: 3
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: [1, 2, 3]
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
# The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache
|
||||
@@ -50,24 +63,107 @@ jobs:
|
||||
"$venv/bin/python" -m pip install --no-cache-dir -r requirements-dev.txt
|
||||
echo "$venv/bin" >> "$GITHUB_PATH"
|
||||
- run: python --version && node --version && node --version | grep -q '^v24\.'
|
||||
# npm's download cache, keyed on the lockfile; inside the container ~ is the job's HOME (/github/home),
|
||||
# which npm also uses. node_modules itself is still built fresh by npm ci.
|
||||
- name: Cache npm downloads
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/.npm
|
||||
key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }}
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
- run: npm run --prefix tools/vendor check
|
||||
- run: npm run --prefix tools/vendor test
|
||||
- name: Frontend unit, viewer, workflow, and coverage checks
|
||||
run: npm run --prefix tools/vendor test:coverage
|
||||
- name: Upload frontend coverage and Chromium traces
|
||||
# Shard 1 also runs the node units and the viewer suite, so the vendor pins and the tooling tests
|
||||
# run on the lightest shard instead.
|
||||
- if: matrix.shard == 3
|
||||
run: npm run --prefix tools/vendor check
|
||||
- if: matrix.shard == 3
|
||||
run: npm run --prefix tools/vendor test
|
||||
- name: Page workflows with coverage, shard ${{ matrix.shard }}/3
|
||||
run: npm run --prefix tools/vendor test:coverage:shard -- ${{ matrix.shard }}/3
|
||||
- name: Upload raw coverage
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: coverage-raw-${{ matrix.shard }}
|
||||
path: coverage/js/raw/
|
||||
if-no-files-found: error
|
||||
retention-days: 3
|
||||
- name: Upload Chromium and touch traces
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: frontend-traces-shard-${{ matrix.shard }}
|
||||
path: tools/vendor/test-results/
|
||||
retention-days: 14
|
||||
|
||||
frontend-coverage-merge:
|
||||
name: frontend coverage (merge)
|
||||
needs: frontend-coverage
|
||||
runs-on: ubuntu-latest
|
||||
# Same image as the shards: the raw files name sources by the container's workspace path.
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble
|
||||
options: --user 1001
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- run: node --version && node --version | grep -q '^v24\.'
|
||||
# npm's download cache, keyed on the lockfile; inside the container ~ is the job's HOME (/github/home),
|
||||
# which npm also uses. node_modules itself is still built fresh by npm ci.
|
||||
- name: Cache npm downloads
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/.npm
|
||||
key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }}
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
# Each artifact lands in its own coverage-raw-N directory, so the next step can count them.
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: coverage-raw-*
|
||||
path: ${{ runner.temp }}/coverage-raw
|
||||
# coverage:merge only checks that raw files exist; merging two of three shards would report
|
||||
# partial coverage, so a missing or empty shard artifact fails here.
|
||||
- name: Verify every shard's raw coverage arrived
|
||||
shell: bash
|
||||
env:
|
||||
STAGED_RAW: ${{ runner.temp }}/coverage-raw
|
||||
EXPECTED_SHARDS: 3
|
||||
run: |
|
||||
found=0
|
||||
for shard in $(seq 1 "$EXPECTED_SHARDS"); do
|
||||
directory="$STAGED_RAW/coverage-raw-$shard"
|
||||
count=0
|
||||
if [ -d "$directory" ]; then count=$(find "$directory" -maxdepth 1 -type f -name '*.json' | wc -l); fi
|
||||
echo "coverage-raw-$shard: $count raw files"
|
||||
if [ "$count" -gt 0 ]; then found=$((found + 1)); fi
|
||||
done
|
||||
echo "shard artifacts with raw coverage: $found of $EXPECTED_SHARDS"
|
||||
if [ "$found" -ne "$EXPECTED_SHARDS" ]; then
|
||||
echo "::error::Expected raw coverage from $EXPECTED_SHARDS shards, got $found of $EXPECTED_SHARDS"
|
||||
exit 1
|
||||
fi
|
||||
staged=$(find "$STAGED_RAW" -mindepth 2 -maxdepth 2 -type f -name '*.json' | wc -l)
|
||||
mkdir -p coverage/js/raw
|
||||
# cp -n never overwrites; a clash shows up as fewer merged files than staged ones.
|
||||
find "$STAGED_RAW" -mindepth 2 -maxdepth 2 -type f -name '*.json' -exec cp -n -t coverage/js/raw {} + || true
|
||||
merged=$(find coverage/js/raw -maxdepth 1 -type f -name '*.json' | wc -l)
|
||||
if [ "$merged" -ne "$staged" ]; then
|
||||
echo "::error::Shards wrote clashing raw file names: $staged staged, $merged merged"
|
||||
exit 1
|
||||
fi
|
||||
echo "merged $merged raw files"
|
||||
- name: Frontend coverage report and baseline check
|
||||
run: npm run --prefix tools/vendor coverage:merge
|
||||
- name: Upload frontend coverage report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: frontend-test-reports
|
||||
path: |
|
||||
tools/vendor/test-results/
|
||||
coverage/js/
|
||||
!coverage/js/raw/**
|
||||
retention-days: 14
|
||||
|
||||
browser-workflows:
|
||||
name: page workflows (${{ matrix.browser }})
|
||||
name: page workflows (${{ matrix.browser }}, shard ${{ matrix.shard }}/2)
|
||||
runs-on: ubuntu-latest
|
||||
# Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the
|
||||
# same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json;
|
||||
@@ -75,11 +171,16 @@ jobs:
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble
|
||||
options: --user 1001
|
||||
timeout-minutes: 25
|
||||
timeout-minutes: 20
|
||||
# Public-repo ubuntu-latest runners have 4 vCPUs: three Playwright workers, each with its own test
|
||||
# server, leave one CPU for the servers and the runner.
|
||||
env:
|
||||
PM_TEST_WORKERS: 3
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
browser: [firefox, webkit]
|
||||
shard: [1, 2]
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
# The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache
|
||||
@@ -98,22 +199,29 @@ jobs:
|
||||
"$venv/bin/python" -m pip install --no-cache-dir -r requirements-dev.txt
|
||||
echo "$venv/bin" >> "$GITHUB_PATH"
|
||||
- run: python --version && node --version && node --version | grep -q '^v24\.'
|
||||
# npm's download cache, keyed on the lockfile; inside the container ~ is the job's HOME (/github/home),
|
||||
# which npm also uses. node_modules itself is still built fresh by npm ci.
|
||||
- name: Cache npm downloads
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/.npm
|
||||
key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }}
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
- name: Page workflows in ${{ matrix.browser }}
|
||||
run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }}
|
||||
- name: Page workflows in ${{ matrix.browser }}, shard ${{ matrix.shard }}/2
|
||||
run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }} --shard ${{ matrix.shard }}/2
|
||||
- name: Upload browser traces
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: browser-traces-${{ matrix.browser }}
|
||||
name: browser-traces-${{ matrix.browser }}-shard-${{ matrix.shard }}
|
||||
path: tools/vendor/test-results/
|
||||
retention-days: 14
|
||||
|
||||
# Branch protection requires the exact context "viewerjs". This job keeps that
|
||||
# name and fails unless every frontend job above succeeded, including when one
|
||||
# was cancelled or skipped.
|
||||
# name and fails unless every frontend job above succeeded (every shard, the
|
||||
# coverage merge and every browser shard), including when one was cancelled or skipped.
|
||||
viewerjs:
|
||||
needs: [frontend-coverage, browser-workflows]
|
||||
needs: [frontend-coverage, frontend-coverage-merge, browser-workflows]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
@@ -121,8 +229,10 @@ jobs:
|
||||
- name: Require every frontend job to succeed
|
||||
env:
|
||||
COVERAGE_RESULT: ${{ needs.frontend-coverage.result }}
|
||||
MERGE_RESULT: ${{ needs.frontend-coverage-merge.result }}
|
||||
WORKFLOWS_RESULT: ${{ needs.browser-workflows.result }}
|
||||
run: |
|
||||
echo "frontend-coverage: $COVERAGE_RESULT"
|
||||
echo "frontend-coverage-merge: $MERGE_RESULT"
|
||||
echo "browser-workflows: $WORKFLOWS_RESULT"
|
||||
test "$COVERAGE_RESULT" = success && test "$WORKFLOWS_RESULT" = success
|
||||
test "$COVERAGE_RESULT" = success && test "$MERGE_RESULT" = success && test "$WORKFLOWS_RESULT" = success
|
||||
|
||||
+503
-40
@@ -39,9 +39,44 @@ npm run --prefix tools/vendor test:coverage
|
||||
npm run --prefix tools/vendor test:workflows -- --project firefox --project webkit
|
||||
```
|
||||
|
||||
`PM_TEST_PYTHON` selects the Python executable; `PM_TEST_PORT` selects the local
|
||||
port (default 8765). Set these environment variables when Python is outside PATH
|
||||
or the port is occupied. Test servers are never reused. On unsupported Linux
|
||||
`PM_TEST_PYTHON` selects the Python executable; set it when Python is outside
|
||||
PATH. Every Playwright worker starts its own `tests/browser/server.py --port 0`,
|
||||
which binds a port the OS picks and prints it as `PM_TEST_SERVER_PORT=<n>`, with
|
||||
its own temp root and database (`tests/browser/fixtures.mjs`), so
|
||||
parallel workers and concurrent runs never share state. `PM_TEST_WORKERS` sets the
|
||||
worker count (default 1 under `CI`, otherwise a quarter of the CPUs, at most 8).
|
||||
Test servers are never reused across runs. The launcher holds each server's
|
||||
stdin open and the server (`--exit-on-stdin-eof`) shuts down cleanly, removing its
|
||||
temp root, when that pipe closes, so a worker killed without teardown leaves no
|
||||
server behind. A test that fails gets the stderr its server wrote during the test
|
||||
attached to the report as `server-stderr` (an aiohttp traceback behind a 500).
|
||||
|
||||
The coverage run can be split the way CI splits it, and each shard keeps the raw
|
||||
V8 files it writes under `coverage/js/raw`:
|
||||
|
||||
```sh
|
||||
PM_TEST_WORKERS=8 npm run --prefix tools/vendor test:coverage # whole run, one report
|
||||
npm run --prefix tools/vendor test:coverage:shard -- 1/3 # node units, viewer suite, workflows 1/3
|
||||
npm run --prefix tools/vendor test:coverage:shard -- 2/3 # workflows 2/3 (likewise 3/3)
|
||||
npm run --prefix tools/vendor coverage:merge # report and baseline check of every raw file
|
||||
npm run --prefix tools/vendor test:workflows -- --project webkit --shard 2/2
|
||||
```
|
||||
|
||||
`coverage:merge` refuses an empty raw directory; it does not count shards, so run
|
||||
every shard first (CI's merge job checks all three arrived). Measured on a 32-CPU
|
||||
machine in the pinned image (2026-10-21): `test:coverage` takes about 5 minutes
|
||||
with 8 workers and 24 minutes with 1; all four projects, 8 workers, each test 3
|
||||
times (6132 tests), 24 minutes. A CI-sized run (`docker run --cpus=4`,
|
||||
`PM_TEST_WORKERS=3`, all four projects once, 2044 tests) takes 25.5 minutes.
|
||||
|
||||
Specs must pass beside each other, not only alone. Wait for the page's own
|
||||
signal, never for time: a rendering frame (`requestAnimationFrame`) for layout
|
||||
work queued by a render or a scroll, a response or a visible state for data. A
|
||||
timer that "usually" covers a frame is a race on a loaded runner. Library
|
||||
virtualization specs settle with `libraryObservation.settled()` (two consecutive
|
||||
frames with no Library frame queued). To look for such races, repeat a spec under
|
||||
load (`--workers 8 --repeat-each 10`, and in the image with `--cpus=1` or `2`).
|
||||
On unsupported Linux
|
||||
distributions, browser system dependencies may need manual installation. For a
|
||||
quick browser run, use `test:workflows -- --project chromium`.
|
||||
|
||||
@@ -109,14 +144,44 @@ the baseline updater. Raise floors with `npm run --prefix tools/vendor
|
||||
coverage:update`; it reruns the suites and cannot lower an existing floor.
|
||||
Review baseline changes like code changes.
|
||||
|
||||
`npm run --prefix tools/vendor coverage:baseline-check` runs in well under a
|
||||
second without a browser and fails, naming the files, when a measured
|
||||
`web/js` file has no baseline entry or the baseline lists a file that no longer
|
||||
exists. It uses the gate's own `discoverSources`, and `npm test` runs the same
|
||||
check, so run it before pushing a new or deleted frontend file instead of
|
||||
waiting for the `frontend coverage` jobs.
|
||||
|
||||
The `viewerjs` check runs on every PR, including backend-only changes. It is an
|
||||
aggregate job that passes only when the `frontend coverage (chromium and touch)`
|
||||
job and both `page workflows (firefox)` and `page workflows (webkit)` jobs
|
||||
succeed; the browsers run on separate runners in parallel. GitHub artifacts
|
||||
named `frontend-test-reports` contain HTML coverage, a JSON summary, and the
|
||||
Chromium/touch failure screenshots/traces; `browser-traces-firefox` and
|
||||
`browser-traces-webkit` hold the other browsers' traces. Open traces with
|
||||
Playwright's `show-trace` tool.
|
||||
aggregate job that passes only when every frontend job succeeds: the three
|
||||
`frontend coverage (shard N/3)` jobs, `frontend coverage (merge)`, and the four
|
||||
`page workflows (firefox|webkit, shard N/2)` jobs. A failed, cancelled or skipped
|
||||
job fails it. Every browser job runs with `PM_TEST_WORKERS: 3` (the public
|
||||
runners have 4 vCPUs). Each coverage shard runs `test:coverage:shard -- N/3`
|
||||
(shard 1 also runs the node units and the viewer suite, shard 3 the vendor pins
|
||||
and tooling tests) and uploads its raw V8 files as `coverage-raw-N` (kept 3
|
||||
days). The merge job downloads them, fails unless all three arrived with raw
|
||||
files and without clashing names, then runs `coverage:merge` (report plus
|
||||
baseline check). `tests/vendor/ci-workflow.test.mjs` checks this shape. GitHub
|
||||
artifacts named `frontend-test-reports` contain the merged HTML coverage and
|
||||
JSON summary; `frontend-traces-shard-N` hold the Chromium/touch failure
|
||||
screenshots/traces and `browser-traces-<browser>-shard-N` the other browsers'
|
||||
traces. Open traces with Playwright's `show-trace` tool.
|
||||
|
||||
Every `npm ci` first restores npm's download cache (`~/.npm`), keyed on the vendor
|
||||
lockfile and the job's pinned image tag (`tests/vendor/ci-workflow.test.mjs` checks
|
||||
the key names the same tag as the job's `container.image`). Without it, `npm ci` stalled for about 300 s
|
||||
(npm's fetch timeout) in most of the eight concurrent jobs; with a hit it takes
|
||||
0 to 2 s. A lockfile change misses the cache once, so expect one slow run after it.
|
||||
Measured on PR #274 (run 39725176320 at cbc8a61, cache hit): the whole workflow
|
||||
took 8m38s, against about 33 minutes with one worker and no shards. Jobs: coverage
|
||||
shards 7m34s/6m04s/5m22s (tests 6m38s/4m53s/4m04s), merge 54s, firefox shards
|
||||
7m57s/5m21s (tests 7m08s/4m34s), webkit shards 8m16s/5m03s (tests 7m26s/3m58s).
|
||||
Browser jobs keep a 20-minute limit: the slowest job is about 8.5 minutes, a cache
|
||||
miss adds up to 5 to `npm ci`, and runner speed varies (the first sharded run was
|
||||
cancelled at a 12-minute limit).
|
||||
Playwright shards by test count, whole files at a time, so shard 1 carries the
|
||||
slow concept-comparison specs (`*-concept.spec.mjs`, alphabetically first) and
|
||||
runs longer; the remedy, if it starts to matter, is more shards, not reordering.
|
||||
|
||||
The real-host suite covers both Classic and Nodes 2.0 startup, switching both
|
||||
ways, actual database prompt selection, execution-text serialization, workflow
|
||||
@@ -267,7 +332,30 @@ becomes record format `limited` (no raw text, graph `limited`, roles and facts
|
||||
unaffected); other text chunks are skipped unread; byte-identical duplicates are
|
||||
accepted within a bounded read budget, while differing duplicates (and any duplicate
|
||||
role record over that budget) still fail. Other read bounds are unchanged: 8 MiB
|
||||
non-PNG header, 10,000 PNG chunks and a three-second ffprobe. Resolution under
|
||||
non-PNG header, 10,000 PNG chunks and a three-second ffprobe (per probe, at most two
|
||||
probes). Video containers are read through one shared carrier selector
|
||||
(`utils/video_metadata.select_video_carrier`, used by the inspector and by the admin
|
||||
scan): carriers are, in order, the per-key tags that current savers write (`prompt`,
|
||||
`workflow`, `parameters`, `comfyassets_prompt_pairs`, each `original-text`; Matroska
|
||||
upper-cased names fold on our side), then the old VideoHelperSuite `comment` JSON
|
||||
envelope, then `description` (object values are `reserialized-json`; envelope keys are
|
||||
matched exactly, as the legacy readers did, never folded). The first carrier holding a
|
||||
role key wins, otherwise the first holding a workflow, and carriers never merge. Two
|
||||
per-key tags folding to one key are a duplicate record: byte-identical ones are one
|
||||
record, differing ones fail the read. ffprobe is asked for exactly those six tags.
|
||||
Its JSON output escapes tag text (at most doubling it), so the stdout bounds are sized
|
||||
so that only the decoded per-record budgets above can fail or limit a record, and a
|
||||
video reads exactly as the identical PNG: the first probe allows twice the role and
|
||||
workflow budgets plus 64 KiB of envelope (12 MiB + 64 KiB); past that the inspector
|
||||
probes once more without the `workflow` tag within twice the role budget plus 64 KiB
|
||||
(4 MiB + 64 KiB) and the per-key workflow becomes `limited`, while an envelope over
|
||||
that bound still fails the read because it cannot be split. The admin scan reads the
|
||||
same carrier with the default resolver profile, so a per-key video links like the same
|
||||
graph in a PNG; the live monitor processes images only (unchanged, see the follow-up
|
||||
below) (`tests/test_utils_video_metadata.py`, `tests/test_api_admin.py`
|
||||
`TestScanReadsVideos`, `tests/test_api_prompt_inspector.py` `test_video_*`,
|
||||
`tests/test_inspection_response.py` `TestVideoProbeBoundsMatchPngBudgets`; the
|
||||
real-container round trips skip without ffmpeg). Resolution under
|
||||
`INSPECTION_PROFILE` bounds work, not graph size: 2,000 visits per candidate chain,
|
||||
20,000 in total, 64 link/JSON-container levels, 100,000 resolved characters, and a
|
||||
two 20,000-node ceilings: one over the top-level entries of the API prompt, and one
|
||||
@@ -361,8 +449,8 @@ of the approved concept's `docs/concepts/workspace/style.css`. Never edit the co
|
||||
it is only rewritten from the concept with `npm run --prefix tools/vendor sync:concept`.
|
||||
`npm run --prefix tools/vendor check` runs `sync-concept-css.mjs --check`, which fails
|
||||
on a missing copy or names the first differing byte. `workspace.html` loads, in order,
|
||||
the toolkit controls, Viewer.js, `concept.css`, the page stylesheets that their own
|
||||
PRs retire, and the production overlay `workspace.css` last.
|
||||
the toolkit controls, Viewer.js, `concept.css` and the production overlay `workspace.css`
|
||||
last; every page stylesheet that waited for its page PR is retired.
|
||||
`tests/vendor/concept-css.test.mjs` asserts that order, the drift check, and that no
|
||||
other page or canvas script loads workspace CSS.
|
||||
|
||||
@@ -373,19 +461,28 @@ D-number must be one of this spec's (#249): a comment that cites any other issue
|
||||
as `D5 (#245)`, is that issue's ruling and is not a marker, so write `#245 decision 5`
|
||||
for those. The lint test fails if `workspace.css` is missing. The match is by exact
|
||||
selector, so a narrower selector over a concept rule (for example
|
||||
`.ca-tools dialog.prompt-sender`) passes the lint. Such rules, and the page-stylesheet
|
||||
leftovers such as `library.css` `.row-actions`, name the PR that retires them, by
|
||||
convention.
|
||||
`.ca-tools dialog.prompt-sender`) passes the lint; by convention such rules carry a
|
||||
marker too.
|
||||
|
||||
The lint also runs over every page stylesheet that loads before the overlay and is
|
||||
still kept (`library.css`; test `kept page stylesheets mark their concept-selector
|
||||
overrides`). PR D deleted `images.css` and `inspector.css` (test `the retired shell,
|
||||
vocabulary, editor, Images and inspector stylesheets stay deleted`); their few
|
||||
production-only rules with a live consumer (inspector wrapping and the inert download,
|
||||
the narrow Images container, the Library and recorded-run prompt blocks, the Archive
|
||||
negative's `section.prompt-block`, the prompt sender) moved to the overlay under
|
||||
`#249 D3`, and `the overlay keeps no retired Images page or inspector rules` keeps the
|
||||
replaced card, filter, toolbar, preview, facts and inspector rules out.
|
||||
PR D deleted `images.css` and `inspector.css`, and PR F-b deleted `library.css`, the
|
||||
last page stylesheet (test `the retired shell, vocabulary, editor, Images, inspector and
|
||||
library stylesheets stay deleted`). `the overlay keeps no retired Images page or
|
||||
inspector rules` keeps the replaced card, filter, toolbar, preview, facts and inspector
|
||||
rules out, and `the overlay keeps no rule waiting for a page rebuild` keeps out the
|
||||
interim guards and the history page's private role colors. Every rule that stayed has
|
||||
a live consumer and a marker: the Library rows windowed (D3) and the shelf's states
|
||||
(D40), the history panel's wrap, archived-lane and coarse-target rules (D45), and under
|
||||
**D48** the production surfaces the concept never shows, drawn in the concept's tokens
|
||||
(`--green`, `--red`, `--ca-*`): recorded executions and the Recorded run dialog (#209,
|
||||
D26), `pre.prompt-text` for stored text in the frozen field provenance and the Archive
|
||||
negative confirmation, whose `section.prompt-block` (#235) keeps its rules too, and the
|
||||
acknowledged prompt sender (spec §2.2). Rules that repeated the concept
|
||||
(the `.library-grid` tracks and breakpoints, the history `.row-actions` flex row and
|
||||
button size, `summary` cursors, `h2` margins, the sender and Recorded run dialogs'
|
||||
colors and border) or had no effect (`min-inline-size` on block children, a
|
||||
`.recorded-run-dialog pre` limit, a 700px padding that a later rule overrode) were
|
||||
deleted; the workspace browser specs on Chromium and the concept specs on touch prove
|
||||
the pages unchanged.
|
||||
|
||||
**Comparison method.** Production must equal the concept except for the recorded
|
||||
deviations in the #249 design (spec §8, D-numbers). It is never compared against
|
||||
@@ -421,14 +518,255 @@ the first nav link and heading button.
|
||||
Recorded shell deviations:
|
||||
- **D4**: a "Theme" select with Dark, Light and System. System follows the ComfyUI
|
||||
palette, then the OS.
|
||||
- **D5**: no Activity button or Activity row until an activity center exists.
|
||||
- **D28**: no SAVED VIEWS or COLLECTIONS, and no Design review. The bottom nav is
|
||||
System plus the production Overview hub (its own outline home glyph, not the concept's
|
||||
`activity` pulse, and no count).
|
||||
- **D5**: retired by Activity (#249 Activity-1). The concept's Activity button and
|
||||
Activity row are shown; their differences are D52 to D54.
|
||||
- **D28**: no COLLECTIONS, and no Design review. The bottom nav is
|
||||
the concept's Activity and System, plus the production Overview hub (its own
|
||||
outline home glyph, not the concept's `activity` pulse, and no count).
|
||||
The connection reads "Your library, on your server" with a neutral muted dot.
|
||||
- **D49** (saved views, #249 D16): the SAVED VIEWS group label has no "PROPOSED"
|
||||
badge, because saved views are persisted (`/prompt_manager/saved-views`). Its rows are
|
||||
the concept's `links()` rows: the search icon, the view name and no count, linking
|
||||
`#/saved/<id>`. Opening one shows its page with the saved toolbar value and Refine
|
||||
filters, and the view's name as the h1, breadcrumb and title. A value its page no
|
||||
longer accepts (a removed Source or Folder) stays default; an unknown id opens Images.
|
||||
A toolbar change made while the sources are still loading keeps the saved Source and
|
||||
Folder until discovery settles. A saved view, like Favorites, never replaces the plain
|
||||
Images or Library return state. Writes (POST, PATCH) validate strictly; a state over
|
||||
4 KB is refused in the browser with "The view is too large to save. Shorten its
|
||||
search, folder or tags and try again." Startup and restore check only the table's
|
||||
structure and that each row's state is a JSON object of at most 4 KB, never the value
|
||||
rules, so a row saved under older rules (a missing or retired key, a sort no longer
|
||||
offered) never stops the database from opening: reads fill missing keys with their
|
||||
defaults, drop unknown keys, default invalid values and make the name displayable
|
||||
(`tests/test_saved_views.py`). A live row whose state is not a JSON object never
|
||||
blocks startup either: its id is logged as a warning, the row is left in place, the
|
||||
listing and rename skip it, and its `#/saved/<id>` link behaves like a missing view.
|
||||
Restore stays strict and rejects such a backup ("Saved views contains invalid values"). Only request validation is a 400; any other failure is
|
||||
a logged 500 with a generic message. The Favorites count re-read after starring is
|
||||
coalesced to one read 300 ms after the last toggle.
|
||||
`a saved view row matches the concept's` saves one view in both documents and
|
||||
compares the rows; `tests/browser/workspace-saved-views.spec.mjs` covers saving,
|
||||
reopening, the defaults that replace rejected values, and failed reads and saves.
|
||||
- **D50**: no "Needs review" row in SAVED VIEWS until the tag-review slice (maintainer
|
||||
ruling, 2026-10-21). The comparison maps only that row out.
|
||||
- **D51** (maintainer ruling, 2026-10-21, beyond the concept): SAVED VIEWS starts with
|
||||
the concept's Favorites row (star icon, name, count). Its count is the Favorites
|
||||
listing's own `limit=1` total (`/images/output?mode=workspace&favorites=1`), follows
|
||||
the nav-count rules (hidden until loaded, absent when its read fails, never a
|
||||
placeholder) and is re-read after every confirmed star or unstar, on any page.
|
||||
`#/favorites` is Images filtered to favorites, named "Favorites" (concept
|
||||
`renderImages`); `favorites=1` keeps only the selected roots' favorites, read per root
|
||||
from `image_favorites` and matched against the scan with the toggle's case folding,
|
||||
and a failed read is an error, never an unfiltered page. Unlike the concept, an
|
||||
unstarred card stays on Favorites (showing ☆) until the next load, so the star can be
|
||||
undone in place. Favorites neither restores nor replaces the Images return state.
|
||||
Save view there saves `favorites: true`, so the view reopens the favorites. An open
|
||||
saved view's heading leads with **Edit**: "Edit saved view" with "View name" and
|
||||
Delete… (danger), Cancel and Save. Save re-reads the list, retitles the page and
|
||||
toasts "Saved view renamed."; an empty name keeps the dialog open. Delete… swaps the
|
||||
same dialog to "Delete saved view" (Cancel / Delete; never `window.confirm`; its Cancel
|
||||
closes the dialog without deleting rather than returning to Edit); Delete
|
||||
removes the view, replaces the dead route with the view's plain page and toasts
|
||||
"Saved view deleted.". Failures toast "The view could not be renamed. Try again." or
|
||||
"The view could not be deleted. Try again."; a view another tab already deleted counts
|
||||
as deleted. `tests/browser/workspace-saved-views.spec.mjs` covers the row, count,
|
||||
route, star updates, Save view on Favorites, Edit, and the failure paths;
|
||||
`tests/test_image_favorites.py` covers `favorites=1`.
|
||||
- **D52** (Activity, #249 Activity-1): the top-bar `button#activity-open` and the
|
||||
sidebar's Activity row (the concept's `activity` pulse, first in `system-nav`) count
|
||||
this session's *running* jobs. The concept counts
|
||||
every sample job. `span.count#activity-count` is `hidden` at 0 (its text stays "0"),
|
||||
and the row has no `.nav-count` then. The Activity heading has no "Unified activity ·
|
||||
proposed" chip (prototype copy). At `max-width: 700px` the full-width `.top-actions`
|
||||
row may wrap (`flex-wrap: wrap`), so the button, Theme and ComfyUI stay inside a
|
||||
zoomed phone. The shell comparison hides the concept's count at runtime
|
||||
(`#activity-count { display: none }`), so the concept's own CSS lays out its button
|
||||
without it, and maps the row's count to none, the button's count to "0" and the
|
||||
phone row's `flexWrap` to `wrap`.
|
||||
- **D53**: while a job runs, the approved Activity mockup's progress pill sits before
|
||||
the Activity button on every page: `a#activity-pill.activity-pill`, "Scanning · 41% ·
|
||||
8,240 / 19,622" (the verb by job kind, the server's floored percentage, and the file
|
||||
counts once the total is known), a 70px mini bar, `href="#/activity/<id>"` and the same
|
||||
text as its `aria-label`. With no running job it is `hidden`, which the shell
|
||||
comparison maps as a hidden item before the button. At `max-width: 700px` it keeps the
|
||||
verb and percentage only; on coarse pointers it is 40px tall like the concept's
|
||||
buttons. Its rules are in `workspace.css` under D53.
|
||||
- **D54**: each `.activity-item` is a real job of this session (`GET
|
||||
/prompt_manager/jobs`), running first, then finished newest first. The concept's
|
||||
structure and CSS stay: `.panel-heading` > `h3` "✓ <title>" for a completed job and
|
||||
"○ <title>" otherwise, plus `span.chip`, then `p`, and for a running job `<progress
|
||||
max=100>` ("<title> progress") and `.row-actions`. The chip is green "Completed" or
|
||||
blue "Running", "Cancelled", "Failed" or "Ended". The line reads "Started HH:MM · K
|
||||
output folders · closing this view does not cancel it" while running, "Finished HH:MM
|
||||
· N added · M errors", "Cancelled HH:MM at a file boundary · N added · completed work
|
||||
is kept", "Stopped on an error at HH:MM · N added · M errors", or "This job ended when
|
||||
ComfyUI restarted." The running actions are "Cancel job" only (no "Simulate
|
||||
completion"); a failed cancel toasts "The job could not be cancelled. Try again.". With
|
||||
no job the section says "No jobs have run since ComfyUI started. Scans and other long
|
||||
operations appear here." `#/activity/<id>` is that job's own view (D56). When a job
|
||||
ends while its Cancel job has focus, focus moves to the item's title link.
|
||||
|
||||
The Activity store reads the list once, then follows each running job over
|
||||
`/prompt_manager/jobs/<id>/events`. A dropped stream re-reads the list after 1 s,
|
||||
doubling to at most 30 s; a received event resets the wait, and so does a good list
|
||||
read after failed ones or after any read that was not for a dropped stream. A running
|
||||
job the server no longer lists, or one whose server could not be read twice in a row,
|
||||
is shown as ended, so no pill stays; one failed read (a proxy's 502) changes nothing.
|
||||
A finished job the server no longer lists (it keeps 20) is dropped. Numbers are
|
||||
grouped "en-US", as the server's lines are. While the tab is visible the list is read again every 15 s (and whenever
|
||||
Activity opens), so a job started in another tab or on the legacy admin page appears
|
||||
without a reload. Toasts: "Scan started." after this tab's own start; "Scan finished
|
||||
· N added · M errors", "Scan cancelled · N added" or "Scan failed · N added · M
|
||||
errors" when a job this tab saw running ends (plural helper for "error").
|
||||
`tests/js/workspace-activity-store.test.js` covers the store, its backoff and the
|
||||
copy; `tests/browser/workspace-activity.spec.mjs` covers the list against the
|
||||
concept's own items (`renderActivity` and `startJob`), the pill and counts, a phone,
|
||||
reload and a second tab, Cancel job, a restart and an unreachable server, and the
|
||||
toasts, with fake jobs from `tests/browser/workspace_activity_fixture.py`
|
||||
(`/__test__/jobs/*`, including a simulated restart that drops the job streams).
|
||||
- **D55** (Scan for metadata, Activity-1 Task 3): Images → Import keeps Rescan as the
|
||||
quick refresh (D15) and then shows the concept's own "Scan an output folder → Review
|
||||
scan" row as "Scan for metadata" ("Discover and enrich existing files using the
|
||||
configured sources.", button "Review scan"). It opens the concept's `scan-review`,
|
||||
"Review metadata scan": no sample-plan `.demo-note` (prototype copy); Sources and
|
||||
Scope in a `dl.meta-grid` (the concept's maintenance plan uses `dl`; `scan-review`
|
||||
uses a `div`) reading "N output folders" (the discovered sources) and the concept's
|
||||
"New and unresolved files" (the D58 checkbox switches it to "All files"); the
|
||||
concept's sentence; then
|
||||
a backup `.notice` (the concept's System notice box): `h3` "Back up the database first
|
||||
(recommended)", "Downloads a copy of the prompts database before the scan changes any
|
||||
records." and "Back up now", which downloads the existing `/prompt_manager/backup`
|
||||
copy (as the legacy admin page does), reads "Backing up…" while aria-disabled, keeps
|
||||
the dialog open and toasts "Database backup downloaded." or "The database could not be
|
||||
backed up. Try again."; then Cancel / Start scan (primary; the concept's "Simulate
|
||||
scan"). Start scan closes the dialog, starts the session scan job (`POST
|
||||
/prompt_manager/jobs/scan`), toasts "Scan started." and opens `#/activity/<id>`; a
|
||||
failed start toasts "The scan could not start. Try again.". The scan's progress line
|
||||
now groups thousands ("Processing file 8,240 of 19,622"), as the mockup does.
|
||||
`tests/browser/activity-concept.spec.mjs` compares the dialog with the concept's
|
||||
`scan-review` and its box with the concept's System `.notice` (dark and light,
|
||||
Chromium and touch, with a CSS and a DOM mutation check); `images-concept.spec.mjs`
|
||||
maps the Import row; `workspace-images.spec.mjs` covers the dialog, the backup and
|
||||
Cancel.
|
||||
- **D56**: one job's view at `#/activity/<id>`, the approved mockup
|
||||
(`activity-scan.html`, "100% spot on"), which the concept does not have. The page
|
||||
heading names the job (h1, document title "<title> · Prompt Manager"; the breadcrumb
|
||||
stays "Activity"), its line is the D54 item line, and its actions are Download report
|
||||
(disabled until Activity-2) and, while the job runs, "Cancel scan" (`.danger`; "Cancel
|
||||
job" for another kind; a failed request toasts the D54 message). Then `section.surface`
|
||||
"Progress": `.panel-heading` (h2, `span.small` with the server's "Processing file N of
|
||||
M · about X min left", or after the end "N of M files processed", "Cancelled after N
|
||||
of M files", the error line, or the restart line), `<progress max=100>` ("Scan
|
||||
progress") and `dl.stat-row` tiles in order: Added to Library (`.added`, green),
|
||||
Already known, Needs review (`.warn`, warning), No usable metadata, Errors (`.bad`,
|
||||
red). Then `section.surface` "Results": `label.search-field` "Filter by file path…"
|
||||
(`aria-label` "Filter results", case-folded match on the root label and path), and
|
||||
`.outcomes` with `.outcome-list` (`button[aria-pressed]` label + count, in the
|
||||
mockup's order Added, Needs review, Errors, No usable metadata, Already known; the
|
||||
view opens on the first with files) beside `.outcome-files`: one `.file-row` (`.path`
|
||||
root-relative, `.why` the server's detail) per listed file. The detail wording is the
|
||||
builder's (approved with Task 1): "Added to the Library as “title”", "Linked to
|
||||
“title”", "Already linked to “title”", "Needs review: …", "Error: …", where the title
|
||||
is the prompt's Library title (its stored title, else its text's first line), not the
|
||||
mockup's sample "→ “title” · new positive + negative". Each log line is "HH:MM:SS
|
||||
<prefix> <root label>/<path> · <detail>" rather than the mockup's "[HH:MM:SS] + path →
|
||||
prompt #4812 (new)". The header's actions sit in the page heading's
|
||||
`.heading-actions`, where the mockup has a `.panel-heading > .row-actions`. An
|
||||
empty outcome shows "No files in this group."
|
||||
or "No listed file matches this filter.", and the note "Showing N of M" when fewer
|
||||
than the counter are shown (the server lists 200 per outcome), with "each row opens
|
||||
the image in the viewer" for Added and Already known. Then `details.term`, collapsed:
|
||||
"● Scan log — raw detail, click to show" and `pre[tabindex=0]` with the raw lines.
|
||||
The summary follows the store and carries `listed` (rows per outcome) and
|
||||
`log_length`; the rows and log are read from `GET /prompt_manager/jobs/<id>` once on
|
||||
opening, then at most once a second only while what it lists changes (or, with the
|
||||
log open, its length), so a job whose lists are capped and whose log is closed is not
|
||||
re-read; opening the log reads its new lines. When the job ends while Cancel scan has
|
||||
focus, focus moves to the job's heading. An unknown id, or a job the store's list no
|
||||
longer has, shows "This job is not in this session’s Activity. Jobs are kept until
|
||||
ComfyUI restarts." and "All activity"; a job lost with a restart keeps what was read.
|
||||
At `max-width: 700px` the tiles take two columns and the outcome list stacks above
|
||||
the files. Each Activity item's title is now a link to its view, in the title's
|
||||
colour. The rules are in `workspace.css` under D56. `activity-concept.spec.mjs`
|
||||
restates the mockup's structure and CSS values and checks them with mutations;
|
||||
`workspace-activity.spec.mjs` covers the counters, live updates, viewer rows, the
|
||||
filter, the 200-row cap, Cancel scan, the busy start and an unknown or lost job.
|
||||
- **D57**: rows under Added and Already known are `button.file-row`s that open the
|
||||
existing media viewer on that one file, by the identity Images opens it by
|
||||
(`root_index:path`). Each scan row records its root's listing index (`root_index`,
|
||||
the place in `_get_all_output_dirs`) beside its label. The row's source is that index
|
||||
in `/prompt_manager/gallery/subfolders?mode=workspace` and must still carry the row's
|
||||
label, so a reordered or replaced source never opens another root's file; the
|
||||
workspace listing's new `path` filter (one exact relative path, case-folded, at most
|
||||
1,024 characters, no traversal) returns that file. A row without an index, a changed
|
||||
source or a file the listing no longer has toasts "This file is no longer in Images.
|
||||
Rescan from Import and try again."; other rows are plain text, with no new file
|
||||
route. A start refused because another record-changing job runs toasts "Another job
|
||||
is running: <title>." with the title linking that job's view (the shared toast now
|
||||
accepts link parts). A toast with a link stays 20 s instead of 6.5 s, and any toast
|
||||
stays while the pointer or focus is on it.
|
||||
- **D58** (scan scope, Activity-2 Task 4): the concept's `scan-review` states a fixed
|
||||
scope, "New and unresolved files". Production keeps that value as the default and adds
|
||||
the concept's own System toggle after the `dl.meta-grid`: a `.setting-row` with the
|
||||
label "All files", the hint "Also read the files already linked to their prompt
|
||||
again." (the switch's `aria-describedby`) and the Toolkit switch (`label.ca-switch`, a
|
||||
checkbox with `role="switch"`, as the concept's `hydrateSwitches` renders it),
|
||||
unchecked. Checking it switches the Scope value to "All files"; Start scan sends
|
||||
`POST /prompt_manager/jobs/scan {scope: "new" | "all"}` (no body, an empty body or no
|
||||
`scope` is "new"; another value is a 400).
|
||||
- In "new" the scan skips, before reading any metadata, the files the Library already
|
||||
links with a resolved link and that are unchanged on disk: exactly one
|
||||
`generated_images` row for the file, exactly one distinct pair link on it
|
||||
(`image_prompt_pairs`, written only by resolved ingestion), and a stored
|
||||
`file_size` equal to the file's size in this walk. Both paths go through
|
||||
`normalize_image_path` (absolute, normalized, `os.path.normcase`); a row without
|
||||
`file_path` matches by its `image_path`.
|
||||
- Read again: files linked without a pair (the monitor's fallback and FIFO links, the
|
||||
legacy positive-only import), linked to more than one prompt or pair, not linked, a
|
||||
file whose size changed (one replaced at the same path, e.g. a reused ComfyUI
|
||||
counter name, is read and its conflict lands under Needs review), and a row with no
|
||||
stored size (unknown is not known). The scan now stores each file's size when it
|
||||
ingests it, so a file read once is skipped by the next "new" pass; the first pass
|
||||
after the upgrade fills the missing sizes, which reports those files as enriched
|
||||
(Added) instead of unchanged.
|
||||
- The known set is one streamed query that keeps only the scanned paths. If it cannot
|
||||
be read, the scan logs "Known files could not be read · reading every file" and
|
||||
reads every file.
|
||||
- A skipped file counts as processed and unchanged in the legacy counts (not as
|
||||
found) and under Already known, with the detail "Skipped · already linked" and the
|
||||
log line "= <root>/<path> · Skipped · already linked"; the log's first line names
|
||||
the scope ("Scan started · N output folders · new and unresolved files" or "· all
|
||||
files"). A legacy-page scan that attaches to a running "new" job therefore shows
|
||||
the skipped files in its `unchanged` count.
|
||||
- "All files" reads every file, as the scan did in Activity-1, and the legacy
|
||||
`/prompt_manager/scan` route always does when it starts the scan (its SSE payload
|
||||
keys are unchanged).
|
||||
- `activity-concept.spec.mjs` compares the switch with the concept's System → Sources
|
||||
toggle ("Watch for new outputs"), without its place-dependent top margin;
|
||||
`workspace-activity.spec.mjs` covers the Scope value, the description and the
|
||||
request both ways; `tests/test_scan_scope.py` covers the skip, the size rule, the
|
||||
replaced file, the re-read of unresolved files, "all", the legacy route, the
|
||||
fallback, the empty body, the 400 and Windows case folding.
|
||||
- **D59** (job report and log, Activity-2 Task 5): the job view's "Download report" is
|
||||
enabled. It fetches `GET /prompt_manager/jobs/<id>/report` through the api-client URL
|
||||
(so a proxy mount works) and saves a JSON attachment named
|
||||
`prompt-manager-<kind>-<YYYYMMDD-HHMM>.json` (server local time), then toasts "Job
|
||||
report downloaded." (or "The report could not be downloaded. Try again."). The report
|
||||
holds the summary, every counter, each outcome group's exact `count`, its listed
|
||||
`rows` (at most 200) and a `truncated` flag, the scope and the log; rows and log lines
|
||||
carry root-relative paths only, never an absolute path (an unknown id is a 404).
|
||||
A job lost with a restart has no report on the server, so its Download report is
|
||||
disabled; a view left while the report downloads still saves it but shows no toast.
|
||||
The scan log opens at its newest line every time it is opened and follows new lines
|
||||
only while the reader is at the bottom; a reader who scrolled up keeps the position
|
||||
while it stays open (checked on Chromium, Firefox and WebKit).
|
||||
`test_report_is_attachment_without_absolute_paths` and the browser tests `download
|
||||
report saves the job JSON` and `log keeps the reader's scroll position` cover it.
|
||||
- **D29**: no prototype copy. There is no "Interactive concept" badge and no
|
||||
"What changes & why" button. The footer reads "ComfyAssets · Local tools. Your
|
||||
library.", and the title has no " concept" suffix.
|
||||
library.", and the title has no " concept" suffix. Save view's dialog action reads
|
||||
"Save", not "Apply in prototype", and its toast drops the demo's word "Sample".
|
||||
- **D31**: at `max-width: 950px` the overlay caps the one-column shell at
|
||||
`minmax(0, 1fr)`, so extreme zoom cannot widen it to the sidebar's min-content.
|
||||
- **#245 D2**: no "Auto-tag images" on the Tags heading until the image-annotation
|
||||
@@ -455,7 +793,7 @@ zoom, the 2nd card opens the viewer titled "<title> · <filename>", Escape retur
|
||||
to it (production only: the concept re-renders its grid and drops focus), Info on the
|
||||
3rd card moves `.selected`, searches for "alpine" and for one image, and Load more with
|
||||
its toast. `mapImagesConcept` applies D3, D8 (the selection bar's real actions only),
|
||||
D16 (follow-up PR: no Save view), D9, D10, D14, D15, D17, D29, D32, D35 (singular counts) and D36 (categories
|
||||
D9, D10, D14, D15, D17, D29, D32, D35 (singular counts) and D36 (categories
|
||||
sorted alphabetically). Removed toolbar and filter-row items flow again from the
|
||||
concept's own boxes (`reflow`), checked against the concept page with those items
|
||||
removed at runtime. Masonry tops and heights are compared unrounded, within one layout
|
||||
@@ -511,7 +849,8 @@ Recorded deviations on this page (spec §8):
|
||||
footer, masonry tops may differ by sub-pixel snapping (at most one layout unit per
|
||||
card above in the column), and a List row's info takes its natural height, centred
|
||||
beside the 160×95 art.
|
||||
- **D5**, **D29**: no Activity button and no prototype copy (shell).
|
||||
- **D29**: no prototype copy (shell). The Activity button and pill are the shell's
|
||||
(D52, D53).
|
||||
- **D7**: the star is a real persisted favorite with "Added to favorites." /
|
||||
"Removed from favorites." and a rollback toast on failure.
|
||||
- **D8**: every Images card has the concept's select checkbox before the star; the
|
||||
@@ -561,7 +900,9 @@ Recorded deviations on this page (spec §8):
|
||||
files in M output folders." (the scan's `total` and `roots`) or "Rescan could not
|
||||
finish. Try again."; while it runs, a reopened Import shows an aria-disabled
|
||||
"Rescanning…".
|
||||
- **D16 (follow-up)**: no Save view in PR C; a follow-up PR adds real saved views.
|
||||
- **D16 (retired)**: Save view is the concept's own toolbar button, between sort and
|
||||
Filters, on Images and the Prompt library (see D49). It saves the visible toolbar value
|
||||
with the Refine filters; the comparisons no longer map it.
|
||||
- **D17**: the caption date is the newest result's day under Recently added (", YYYY"
|
||||
outside the current year), hidden under every other sort.
|
||||
- **D18**: the concept hover zoom, with no sticky zoom on hover-less (touch) devices.
|
||||
@@ -766,8 +1107,7 @@ Behaviors, in both documents: the title and Explore history open the prompt's hi
|
||||
(its `h1` is the title), the footer send carries the positive only (a bridge host page
|
||||
records the roles production's preview asks for), copy writes the full positive with
|
||||
"Prompt copied.", and the "neon" search and the Landscapes category leave the same cards
|
||||
and caption. `mapLibraryConcept` applies D14 (Oldest first appended), D16 (no Save
|
||||
view; the toolbar flows again, checked against the concept without it), D23 (the
|
||||
and caption. `mapLibraryConcept` applies D14 (Oldest first appended), D23 (the
|
||||
title button's `min-width: 0`; "− 1 negative variant", measured on the concept with the
|
||||
singular at runtime; the spoken "Rated N out of 5"; the two-line clamp, which the
|
||||
concept's samples exceed at 390px, checked against the concept with the clamp rule
|
||||
@@ -835,11 +1175,71 @@ Recorded deviations on this page:
|
||||
`negatives[0]`, its baseline: the first comparison reads "Baseline compared with
|
||||
<newest>" where the concept compares the baseline with itself. The history concept
|
||||
comparison (PR F-b, F5) maps this.
|
||||
- **D47** (#249 PR F-b): the shared gallery is the concept's `section.history-gallery`
|
||||
(region "Shared gallery") with `.panel-heading` (h2 "Shared gallery", `span.small`
|
||||
"<name> · same positive prompt") after the variants panel (and Compare), and
|
||||
recorded executions (#209) follow it, as the concept has no recorded section. The
|
||||
name is the lane's own name ("<title>", else "Negative variant N"), with " · Archived"
|
||||
for an archived pair, and follows a rename. Its cards are the Images media cards: the
|
||||
star wherever the read returns `favorite`, no checkbox or selection bar (Images only,
|
||||
#259), the image opens the Viewer. A saved variant with no images shows the concept's
|
||||
`.empty` "Saved variant · no runs yet." beside the grid (the concept puts it inside
|
||||
`.image-grid`, which only holds positioned cards here). The concept's "Shared gallery"
|
||||
heading is the gallery's only heading and names its region: the embedded collection's
|
||||
"Linked images" heading and "N loaded · N linked images" count are removed, since they
|
||||
repeated it (maintainer ruling, 2026-10-21). Two production pieces stay, which the
|
||||
concept's gallery does not show: the layout row (Grid/List/Masonry and Thumbnail size),
|
||||
because the legacy gallery let people change layout and density and nothing else on the
|
||||
page offers it, and the selected-image side panel, the one shared inspector the Images
|
||||
page uses (spec §3.3), so an image's details stay reachable without opening the viewer.
|
||||
The same ruling covers the page's two media dialogs, All images for this positive and
|
||||
Repeat run: as in the concept (app.js a1282-1304), each is named by its own dialog
|
||||
title, with no "Linked images" heading or "N loaded · N linked images" count, and a
|
||||
stranded focus returns to the dialog's close button (maintainer ruling, 2026-10-21).
|
||||
The Images page keeps its caption. With no saved variant the page shows every image
|
||||
of the positive under "All images for this positive, including unassigned images. …"
|
||||
and no empty copy (a state the concept never reaches).
|
||||
- **D48** (#249 PR F-b): production surfaces the concept never shows keep their own
|
||||
overlay rules, drawn in the concept's tokens: recorded executions and the Recorded run
|
||||
dialog (#209, D26), the Archive negative confirmation (#235) and the acknowledged
|
||||
prompt sender (spec §2.2). With `library.css` retired, these rules live in
|
||||
`workspace.css` (see "Concept shell and comparison method").
|
||||
|
||||
The run dot reads `GET /prompt_manager/library/{positive_id}/history/pairs/{pair_id}/newest`
|
||||
(see "Chronological Library history"), and each lane's "N runs" is the pair's
|
||||
`run_count` (see "Workspace data fields").
|
||||
|
||||
`tests/browser/history-concept.spec.mjs` (#249 PR F-b, F5) compares this page with the
|
||||
immutable concept's `#prompt/alpine` on the concept fixture, at 1600×1100 and 390×844 in
|
||||
both themes and motion settings, on Chromium and touch: the heading, the summary, the
|
||||
variants panel, the comparison (Compare negatives pressed) and the shared gallery's
|
||||
place and heading, all measured by `inventoryOf`'s `history` section with the concept's
|
||||
selectors in both documents, with "Clean architecture" selected (the middle lane in
|
||||
either order). The fixture links and records its images oldest first, so ids grow with
|
||||
time as real runs do, and counts one pair use per image, which gives the concept's
|
||||
"Quality baseline · 2 runs" and its run dot. `mapHistoryConcept` applies D24 (the four
|
||||
recorded runs give the concept's own "4 tracked runs", asserted on both), D45 (the
|
||||
visually hidden "3 of 3 saved negative variants" status, and its wrap rules applied to
|
||||
the concept at runtime), D35 and D46 (the concept's lanes reordered newest first with
|
||||
"1 run", laid out by the browser), D46's Baseline/Selected action names, #235's Show
|
||||
archived button (added to the concept at runtime) and D47 (the gallery's body is the
|
||||
Images collection, so its height is not compared). D25: each compared block mounts in a
|
||||
`display: contents` child of the concept's `.compare-item`, so the item carries no
|
||||
production class or margin. Behaviors are compared in both documents: a red dot changes
|
||||
the negative block and the gallery, Compare toggles `aria-pressed`, All images opens a
|
||||
dialog with the positive's four images, the run dot opens a dialog (D26: Recorded run
|
||||
where the concept shows Repeat run), neither dialog has a collection caption (D47), the
|
||||
heading send previews the selected pair, and
|
||||
← Prompt library navigates. `history comparison detects mutations` injects
|
||||
`.ca-tools .variant-dot { width: 20px }` and removes Compare negatives. The older
|
||||
`history-canonical-comparison.mjs` stays for `workspace-visual.spec.mjs`'s recorded
|
||||
executions (#209) checks; its production selectors are now the concept's.
|
||||
|
||||
```sh
|
||||
npm run --prefix tools/vendor test:workflows -- history-concept.spec.mjs \
|
||||
workspace-history.spec.mjs --project chromium --project touch --project firefox --project webkit
|
||||
```
|
||||
|
||||
## Workspace data fields for the concept pages (#249)
|
||||
|
||||
These fields are additive. The legacy `/prompt_manager/images/output` listing (no
|
||||
@@ -967,6 +1367,61 @@ concept fixture stars the concept's favorites (each other prompt's first image).
|
||||
restore. Old backups without the table restore, and backup plus restore keeps
|
||||
WAL-resident favorites and closes every handle (`tests/test_database_backup.py`).
|
||||
|
||||
Image annotations schema (#249, `database/annotation_schema.py`): the group
|
||||
`media_annotation_metadata` (one row: schema version, `image_index_revision`),
|
||||
`media_annotation_files` (one private locator per physical file,
|
||||
`normcase(join(realpath(root), relative))`, server-side only),
|
||||
`media_annotation_bytes` (one record per observed whole-file SHA-256 with the
|
||||
`size`/`mtime_ns` verified when those bytes were last seen and the set's
|
||||
`tag_revision`) and `media_image_tags` (the ordered tag set of one byte record,
|
||||
joined into the global `tags` registry). It installs like favorites: optional in
|
||||
old backups, required once current (`annotation_schema_error`, the single
|
||||
`require_library_state` flag), validated read-only before and after a SAVEPOINT
|
||||
install and on restore, never backfilled and never stat-ing media
|
||||
(`tests/test_annotation_schema.py`). Writers (`database/annotations.py`) always
|
||||
supply the UUID ids, run in `_pair_transaction` fenced by the storage epoch, and
|
||||
replace or union one byte record's set (`tag-conflict` / `tag-limit` with the
|
||||
current set, identical retries `replayed`); vocabulary rename, merge and delete
|
||||
carry `media_image_tags` through the one `_move_tag_links` loop, and tag rows,
|
||||
impact and command receipts report `image_count` / `images` /
|
||||
`images_changed` as files (`tests/test_annotations.py`,
|
||||
`tests/test_pair_tag_maintenance.py`, `tests/test_vocabulary.py`). Cheap reads
|
||||
(spec amendment S1) use a byte record only while the file's current
|
||||
`(size, mtime_ns)` equals its verified fingerprint; an in-place rewrite that
|
||||
preserves both keeps the old set matching until the file is next observed. The
|
||||
browser fixtures clear `media_image_tags`, `media_annotation_bytes` and
|
||||
`media_annotation_files` on reset and never the metadata row.
|
||||
|
||||
Capture store (#249, `utils/media_capture.py`, `tests/test_media_capture.py`):
|
||||
one explicit action captures one rooted media file into an owner-only
|
||||
`prompt-manager-captures-*` temp directory, streaming it in 1 MiB chunks while
|
||||
hashing. Identity is compared like with like: the opened descriptor against
|
||||
the path only on the API-stable facts (`size`, `mtime_ns`); the full identity
|
||||
(`size`, `mtime_ns`, `ctime_ns`, `dev`/`ino` where meaningful) between the
|
||||
`fstat` before and after the copy and between the path stats before and after
|
||||
(native Windows CI with Python 3.12/3.13 showed `stat` and `fstat` disagreeing
|
||||
on `st_ctime_ns`: change time versus creation time; 3.10/3.11 agree). Any
|
||||
disagreement is `CaptureChanged` and the copy is removed. Error messages carry
|
||||
`strerror` only, never a path. Aliases (a symlinked
|
||||
file or directory, even inside the root), escapes, non-media names and
|
||||
directories are `CaptureDenied`; the locator is
|
||||
`normcase(join(realpath(root), relative))`, so a symlinked root shares the
|
||||
listing's identity. Fixed limits: 256 MiB per file (stat and stream), 8
|
||||
completed captures, 512 MiB reserved, 2 capture/verify workers, 900 s idle,
|
||||
1800 s hard; admission is nonblocking (`CaptureCapacity`) and lifetimes are
|
||||
tested with an injected clock. An acquired capture outlives expiry and release
|
||||
until its last use ends. Header-only dimensions (4096 px per side,
|
||||
16,777,216 px) and video mark Generate unsupported; `verify` re-hashes the
|
||||
current file without a copy. Every handle closes before unlink (a patched
|
||||
`os.unlink` asserts it, plus `assert_closed`), and files are unlinked outside
|
||||
the store lock. Known limit: on Windows before Python 3.13 `fstat` reports the
|
||||
creation time, so a same-size rewrite that restores `mtime_ns` during the copy
|
||||
window completes there with a digest of the copied bytes; POSIX and Windows on
|
||||
3.13+ detect it through the descriptor's change time (native CI, 3.13 only).
|
||||
Vocabulary rename, merge and delete bump the `tag_revision` of every image set
|
||||
they change (and `image_index_revision`), so a Tag dialog opened before the
|
||||
change gets `tag-conflict` on Save (`tests/test_pair_tag_maintenance.py`).
|
||||
|
||||
Budgets: `test_filter_work_is_bounded` always runs. It asserts that a plain first
|
||||
page makes 1 link SELECT and a plain cursor page none, that every filter shape
|
||||
makes exactly 1, the covering-index plan, and that the per-file values are built
|
||||
@@ -1237,7 +1692,7 @@ bytes at the same path without changing the immutable receipt.
|
||||
The focused acceptance command retains all four projects:
|
||||
|
||||
```sh
|
||||
PM_TEST_PORT=8766 npm run --prefix tools/vendor test:workflows -- \
|
||||
npm run --prefix tools/vendor test:workflows -- \
|
||||
workspace-history.spec.mjs workspace-visual.spec.mjs \
|
||||
--project chromium --project touch --project firefox --project webkit
|
||||
npm run --prefix tools/vendor test:executions
|
||||
@@ -1271,12 +1726,20 @@ Initial production acceptance requires maintainer inspection of paired PNGs and
|
||||
geometry; never update snapshots automatically or mask content to bless a gap.
|
||||
Data-dependent text/images/counts and the clearly labeled saved/frozen/current
|
||||
distinctions are documented in the comparison helper. The Images card tag
|
||||
button (#249 D9) and shared role-block
|
||||
spacing/padding/glyph polish remain outside #209. The
|
||||
button (#249 D9) remains outside #209; every prompt block now renders the concept's
|
||||
`section.prompt-section` (D22), so the shared role-block spacing, padding and glyph
|
||||
gap is closed, and only the production-only surfaces above keep their own rules
|
||||
(D48). The
|
||||
responsive production Viewer keeps its toolbar in the media panel and prompt details readable rather than reproducing the concept's
|
||||
mobile toolbar collision. These are explicit remaining visual differences, not
|
||||
a whole-application or full-gallery/Viewer pixel-parity claim.
|
||||
|
||||
Follow-up: monitor video support. The live image monitor (`utils/image_monitor.py`,
|
||||
`GalleryConfig.SUPPORTED_EXTENSIONS`) processes images only; videos written by
|
||||
ComfyUI savers reach the Library through the admin scan, which reads the shared video
|
||||
carrier. Accepting `VIDEO_EXTENSIONS` in the monitor and routing them through
|
||||
`read_video_metadata` is a separate, issue-linked change.
|
||||
|
||||
Preserve raw logs, failed traces, paired PNGs, geometry, runtime versions and
|
||||
source hashes before later managed commands clear their output directories.
|
||||
Frontend baseline updates are additive through `coverage:update` only after
|
||||
@@ -1328,7 +1791,7 @@ acceptance; a baseline is added only after the maintainer accepts them.
|
||||
Focused commands:
|
||||
|
||||
```sh
|
||||
PM_TEST_PORT=8766 npm run --prefix tools/vendor test:workflows -- \
|
||||
npm run --prefix tools/vendor test:workflows -- \
|
||||
workspace-library-editor.spec.mjs library-editor-visual.spec.mjs \
|
||||
--project chromium --project touch --project firefox --project webkit
|
||||
npm run --prefix tools/vendor test:comfyui # library-sender.spec.mjs host regression
|
||||
@@ -1428,7 +1891,7 @@ mobile, the concept geometry/token/control comparison and its mutation check),
|
||||
|
||||
```sh
|
||||
node --test tests/js/workspace-vocabulary-state.test.js tests/js/workspace-state.test.js
|
||||
PM_TEST_PORT=8766 npm run --prefix tools/vendor test:workflows -- \
|
||||
npm run --prefix tools/vendor test:workflows -- \
|
||||
workspace-vocabulary.spec.mjs vocabulary-visual.spec.mjs focused-node.spec.mjs \
|
||||
--project chromium --project touch --project firefox --project webkit
|
||||
```
|
||||
|
||||
@@ -0,0 +1,290 @@
|
||||
"""Image annotation schema group (#249): file locators, byte records, image tags.
|
||||
|
||||
The group installs like ``image_favorites`` and ``saved_views``: optional in old
|
||||
backups, required once a database is current. It owns
|
||||
|
||||
- ``media_annotation_metadata``: one row, the group's schema version and the
|
||||
image index revision;
|
||||
- ``media_annotation_files``: one private canonical locator per physical file
|
||||
(``normcase(join(realpath(root), relative))``, server-side only);
|
||||
- ``media_annotation_bytes``: one record per observed whole-file SHA-256 of a
|
||||
file, with the size and ``mtime_ns`` verified when those bytes were last seen
|
||||
and the tag set's revision;
|
||||
- ``media_image_tags``: the ordered image tag set of one byte record, joined
|
||||
into the global ``tags`` registry so vocabulary maintenance carries image
|
||||
links like prompt and pair links.
|
||||
|
||||
Installation never backfills anything from prompt or pair tags and never
|
||||
touches the filesystem. Validation reads only: the catalog objects are compared
|
||||
after whitespace compaction, columns, foreign keys and indexes through the
|
||||
PRAGMAs, and persisted values in single bounded SQL passes.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
from typing import Optional
|
||||
|
||||
from .image_favorites import MAX_KEY_LENGTH
|
||||
|
||||
SCHEMA_VERSION = 1
|
||||
METADATA_TABLE = "media_annotation_metadata"
|
||||
FILES_TABLE = "media_annotation_files"
|
||||
BYTES_TABLE = "media_annotation_bytes"
|
||||
IMAGE_TAGS_TABLE = "media_image_tags"
|
||||
TABLES = (METADATA_TABLE, FILES_TABLE, BYTES_TABLE, IMAGE_TAGS_TABLE)
|
||||
MAX_LOCATOR_LENGTH = MAX_KEY_LENGTH
|
||||
_TIMESTAMP_BYTES = 64
|
||||
|
||||
_TABLE_DDL = {
|
||||
METADATA_TABLE: (
|
||||
f"CREATE TABLE {METADATA_TABLE} (singleton_id INTEGER PRIMARY KEY "
|
||||
"CHECK(singleton_id=1), schema_version INTEGER NOT NULL "
|
||||
f"CHECK(schema_version={SCHEMA_VERSION}), "
|
||||
"image_index_revision INTEGER NOT NULL DEFAULT 0 "
|
||||
"CHECK(image_index_revision>=0))"
|
||||
),
|
||||
FILES_TABLE: (
|
||||
f"CREATE TABLE {FILES_TABLE} (id TEXT PRIMARY KEY CHECK(length(id)=36), "
|
||||
"locator TEXT NOT NULL UNIQUE, created_at TEXT NOT NULL)"
|
||||
),
|
||||
BYTES_TABLE: (
|
||||
f"CREATE TABLE {BYTES_TABLE} (id TEXT PRIMARY KEY CHECK(length(id)=36), "
|
||||
f"file_id TEXT NOT NULL REFERENCES {FILES_TABLE}(id) ON DELETE CASCADE, "
|
||||
"sha256 TEXT NOT NULL CHECK(length(sha256)=64 AND sha256 NOT GLOB "
|
||||
"'*[^0-9a-f]*'), size INTEGER NOT NULL CHECK(size>=0), "
|
||||
"mtime_ns INTEGER NOT NULL, verified_at TEXT NOT NULL, "
|
||||
"first_seen_at TEXT NOT NULL, tag_revision INTEGER NOT NULL DEFAULT 0 "
|
||||
"CHECK(tag_revision>=0), UNIQUE(file_id, sha256))"
|
||||
),
|
||||
IMAGE_TAGS_TABLE: (
|
||||
f"CREATE TABLE {IMAGE_TAGS_TABLE} (byte_id TEXT NOT NULL REFERENCES "
|
||||
f"{BYTES_TABLE}(id) ON DELETE CASCADE, tag_id INTEGER NOT NULL "
|
||||
"REFERENCES tags(id) ON DELETE CASCADE, position INTEGER NOT NULL "
|
||||
"CHECK(position>=0), PRIMARY KEY(byte_id, tag_id))"
|
||||
),
|
||||
}
|
||||
_INDEX_DDL = {
|
||||
"idx_media_annotation_bytes_fingerprint": (
|
||||
"CREATE INDEX idx_media_annotation_bytes_fingerprint ON "
|
||||
f"{BYTES_TABLE}(file_id, size, mtime_ns)"
|
||||
),
|
||||
"idx_media_image_tags_tag": (
|
||||
f"CREATE INDEX idx_media_image_tags_tag ON {IMAGE_TAGS_TABLE}(tag_id)"
|
||||
),
|
||||
"idx_media_image_tags_position": (
|
||||
"CREATE UNIQUE INDEX idx_media_image_tags_position ON "
|
||||
f"{IMAGE_TAGS_TABLE}(byte_id, position)"
|
||||
),
|
||||
}
|
||||
# SQLite names the indexes behind TEXT primary keys and UNIQUE constraints
|
||||
# ``sqlite_autoindex_<table>_<n>``; their catalog rows carry no SQL.
|
||||
_AUTOINDEXES = {FILES_TABLE: 2, BYTES_TABLE: 2, IMAGE_TAGS_TABLE: 1}
|
||||
# (name, declared type, not null, default, primary key position)
|
||||
_COLUMNS = {
|
||||
METADATA_TABLE: (
|
||||
("singleton_id", "INTEGER", 0, None, 1),
|
||||
("schema_version", "INTEGER", 1, None, 0),
|
||||
("image_index_revision", "INTEGER", 1, "0", 0),
|
||||
),
|
||||
FILES_TABLE: (
|
||||
("id", "TEXT", 0, None, 1),
|
||||
("locator", "TEXT", 1, None, 0),
|
||||
("created_at", "TEXT", 1, None, 0),
|
||||
),
|
||||
BYTES_TABLE: (
|
||||
("id", "TEXT", 0, None, 1),
|
||||
("file_id", "TEXT", 1, None, 0),
|
||||
("sha256", "TEXT", 1, None, 0),
|
||||
("size", "INTEGER", 1, None, 0),
|
||||
("mtime_ns", "INTEGER", 1, None, 0),
|
||||
("verified_at", "TEXT", 1, None, 0),
|
||||
("first_seen_at", "TEXT", 1, None, 0),
|
||||
("tag_revision", "INTEGER", 1, "0", 0),
|
||||
),
|
||||
IMAGE_TAGS_TABLE: (
|
||||
("byte_id", "TEXT", 1, None, 1),
|
||||
("tag_id", "INTEGER", 1, None, 2),
|
||||
("position", "INTEGER", 1, None, 0),
|
||||
),
|
||||
}
|
||||
# (column, parent table, parent column, on delete)
|
||||
_FOREIGN_KEYS = {
|
||||
METADATA_TABLE: set(),
|
||||
FILES_TABLE: set(),
|
||||
BYTES_TABLE: {("file_id", FILES_TABLE, "id", "CASCADE")},
|
||||
IMAGE_TAGS_TABLE: {
|
||||
("byte_id", BYTES_TABLE, "id", "CASCADE"),
|
||||
("tag_id", "tags", "id", "CASCADE"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _compact(sql: str) -> str:
|
||||
return re.sub(r"\s+", "", sql).lower()
|
||||
|
||||
|
||||
def _uuid(column: str) -> str:
|
||||
"""SQL: *column* is a lowercase hyphenated 36-character UUID string."""
|
||||
return (
|
||||
f"typeof({column}) = 'text' AND length({column}) = 36 "
|
||||
f"AND substr({column}, 9, 1) = '-' AND substr({column}, 14, 1) = '-' "
|
||||
f"AND substr({column}, 19, 1) = '-' AND substr({column}, 24, 1) = '-' "
|
||||
f"AND length(replace({column}, '-', '')) = 32 "
|
||||
f"AND replace({column}, '-', '') NOT GLOB '*[^0-9a-f]*'"
|
||||
)
|
||||
|
||||
|
||||
def _timestamp(column: str) -> str:
|
||||
return (
|
||||
f"typeof({column}) = 'text' "
|
||||
f"AND length(CAST({column} AS BLOB)) <= {_TIMESTAMP_BYTES}"
|
||||
)
|
||||
|
||||
|
||||
def _non_negative(column: str) -> str:
|
||||
return f"typeof({column}) = 'integer' AND {column} >= 0"
|
||||
|
||||
|
||||
# Every row of a table must satisfy its predicate; one pass each, LIMIT 1.
|
||||
_ROW_PREDICATES = {
|
||||
FILES_TABLE: " AND ".join(
|
||||
(
|
||||
_uuid("id"),
|
||||
"typeof(locator) = 'text'",
|
||||
# UTF-8 is at most 4 bytes per character; the decoded length of
|
||||
# the few candidates above the byte bound is checked in Python.
|
||||
f"length(CAST(locator AS BLOB)) <= {MAX_LOCATOR_LENGTH * 4}",
|
||||
_timestamp("created_at"),
|
||||
)
|
||||
),
|
||||
BYTES_TABLE: " AND ".join(
|
||||
(
|
||||
_uuid("id"),
|
||||
_uuid("file_id"),
|
||||
"typeof(sha256) = 'text' AND length(sha256) = 64 "
|
||||
"AND sha256 NOT GLOB '*[^0-9a-f]*'",
|
||||
_non_negative("size"),
|
||||
"typeof(mtime_ns) = 'integer'",
|
||||
_timestamp("verified_at"),
|
||||
_timestamp("first_seen_at"),
|
||||
_non_negative("tag_revision"),
|
||||
)
|
||||
),
|
||||
IMAGE_TAGS_TABLE: " AND ".join(
|
||||
(_uuid("byte_id"), "typeof(tag_id) = 'integer'", _non_negative("position"))
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _expected_catalog() -> set:
|
||||
objects = {("table", name, _compact(sql)) for name, sql in _TABLE_DDL.items()}
|
||||
objects |= {("index", name, _compact(sql)) for name, sql in _INDEX_DDL.items()}
|
||||
objects |= {
|
||||
("index", f"sqlite_autoindex_{table}_{n}", "")
|
||||
for table, count in _AUTOINDEXES.items()
|
||||
for n in range(1, count + 1)
|
||||
}
|
||||
return objects
|
||||
|
||||
|
||||
def _catalog(conn: sqlite3.Connection) -> list:
|
||||
names = TABLES + tuple(_INDEX_DDL)
|
||||
marks = ",".join("?" * len(names))
|
||||
table_marks = ",".join("?" * len(TABLES))
|
||||
return conn.execute( # only "?" placeholders are interpolated
|
||||
"SELECT type,name,sql FROM sqlite_master "
|
||||
f"WHERE name IN ({marks}) OR tbl_name IN ({table_marks})", # nosec B608
|
||||
names + TABLES,
|
||||
).fetchall()
|
||||
|
||||
|
||||
def _structure_error(conn: sqlite3.Connection) -> Optional[str]:
|
||||
for table, expected in _COLUMNS.items():
|
||||
columns = tuple(
|
||||
(row[1], row[2].upper(), row[3], row[4], row[5])
|
||||
for row in conn.execute(f"PRAGMA table_info({table})")
|
||||
)
|
||||
if columns != expected:
|
||||
return f"Image annotations table {table} has invalid columns"
|
||||
foreign_keys = {
|
||||
(row[3], row[2], row[4], row[6])
|
||||
for row in conn.execute(f"PRAGMA foreign_key_list({table})")
|
||||
}
|
||||
if foreign_keys != _FOREIGN_KEYS[table]:
|
||||
return f"Image annotations table {table} has invalid foreign keys"
|
||||
if conn.execute(f"PRAGMA foreign_key_check({table})").fetchone():
|
||||
return f"Image annotations table {table} contains foreign key violations"
|
||||
return None
|
||||
|
||||
|
||||
def _value_error(conn: sqlite3.Connection) -> Optional[str]:
|
||||
# Only this module's constants are interpolated into the SQL below.
|
||||
rows = conn.execute(
|
||||
"SELECT COUNT(*), MIN(typeof(singleton_id) = 'integer' " # nosec B608
|
||||
"AND singleton_id = 1 AND typeof(schema_version) = 'integer' "
|
||||
f"AND schema_version = {SCHEMA_VERSION} "
|
||||
f"AND {_non_negative('image_index_revision')}) FROM {METADATA_TABLE}"
|
||||
).fetchone()
|
||||
if rows[0] != 1 or rows[1] != 1:
|
||||
return "Image annotations metadata is invalid"
|
||||
for table, predicate in _ROW_PREDICATES.items():
|
||||
if conn.execute(
|
||||
f"SELECT 1 FROM {table} WHERE NOT ({predicate}) LIMIT 1" # nosec B608
|
||||
).fetchone():
|
||||
return f"Image annotations table {table} contains invalid values"
|
||||
# length() stops at an embedded NUL, so the SQL pass bounds bytes and the
|
||||
# few candidates over the character bound are measured decoded here.
|
||||
for (raw,) in conn.execute(
|
||||
f"SELECT CAST(locator AS BLOB) FROM {FILES_TABLE} " # nosec B608
|
||||
f"WHERE length(CAST(locator AS BLOB)) > {MAX_LOCATOR_LENGTH}"
|
||||
):
|
||||
if len(bytes(raw).decode("utf-8", "replace")) > MAX_LOCATOR_LENGTH:
|
||||
return f"Image annotations table {FILES_TABLE} contains invalid values"
|
||||
return None
|
||||
|
||||
|
||||
def annotation_schema_error(
|
||||
conn: sqlite3.Connection, *, allow_missing: bool = True
|
||||
) -> Optional[str]:
|
||||
"""Reject partial or unsupported annotation schemas and invalid rows.
|
||||
|
||||
Reads only: nothing is ever repaired. A database without any of the
|
||||
group's objects is acceptable for backups that predate it unless
|
||||
``allow_missing`` is false.
|
||||
"""
|
||||
objects = _catalog(conn)
|
||||
if not objects:
|
||||
return None if allow_missing else "Missing image annotation schema"
|
||||
present = {name for kind, name, _sql in objects if kind == "table"}
|
||||
if present != set(TABLES):
|
||||
return "Incomplete image annotation schema"
|
||||
actual = {(kind, name, _compact(sql or "")) for kind, name, sql in objects}
|
||||
if actual != _expected_catalog():
|
||||
return "Image annotations has an invalid schema definition"
|
||||
return _structure_error(conn) or _value_error(conn)
|
||||
|
||||
|
||||
def migrate_annotation_state(conn: sqlite3.Connection) -> None:
|
||||
"""Install the group atomically; validate before and after creating it."""
|
||||
conn.execute("SAVEPOINT media_annotation_schema")
|
||||
try:
|
||||
error = annotation_schema_error(conn)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
for sql in _TABLE_DDL.values():
|
||||
conn.execute(sql.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
|
||||
for sql in _INDEX_DDL.values():
|
||||
conn.execute(sql.replace("INDEX", "INDEX IF NOT EXISTS", 1))
|
||||
conn.execute(
|
||||
f"INSERT OR IGNORE INTO {METADATA_TABLE} "
|
||||
"(singleton_id, schema_version, image_index_revision) VALUES (1, ?, 0)",
|
||||
(SCHEMA_VERSION,),
|
||||
)
|
||||
error = annotation_schema_error(conn, allow_missing=False)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute("RELEASE media_annotation_schema")
|
||||
except BaseException:
|
||||
conn.execute("ROLLBACK TO media_annotation_schema")
|
||||
conn.execute("RELEASE media_annotation_schema")
|
||||
raise
|
||||
@@ -0,0 +1,457 @@
|
||||
"""Per-image tag sets (#249): byte-exact storage in the shared vocabulary.
|
||||
|
||||
A tag set belongs to one byte record (``media_annotation_bytes``): the
|
||||
whole-file SHA-256 of one physical file, owned by the file's private locator
|
||||
(``media_annotation_files``). Callers pass verified facts (digest, size,
|
||||
``mtime_ns``); nothing here reads the filesystem or resolves an owner through
|
||||
prompt links, so tagging one image never changes another image, a prompt or a
|
||||
pair.
|
||||
|
||||
Commands run in one ``_pair_transaction`` fenced by the storage epoch, like the
|
||||
native Library and vocabulary commands. A save is a full replacement guarded
|
||||
by the set's ``tag_revision``; the identical set sent again (a lost-response
|
||||
retry, or the current set at its own revision) answers ``replayed: true`` and
|
||||
writes nothing. ``add_image_tags`` unions into the existing set, keeping the
|
||||
stored spelling. Names enter the global ``tags`` registry exactly as spelled,
|
||||
so vocabulary rename, merge and delete carry image links with prompt and pair
|
||||
links (``PromptDatabase._move_tag_links``).
|
||||
|
||||
Cheap reads (spec amendment S1) treat a byte record as *current* only while
|
||||
the file's present ``(size, mtime_ns)`` equals the record's verified
|
||||
fingerprint; a file without a current record shows no image tags.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
from contextlib import contextmanager
|
||||
from typing import Iterator
|
||||
from uuid import uuid4
|
||||
|
||||
from .annotation_schema import (
|
||||
BYTES_TABLE,
|
||||
FILES_TABLE,
|
||||
IMAGE_TAGS_TABLE,
|
||||
MAX_LOCATOR_LENGTH,
|
||||
METADATA_TABLE,
|
||||
)
|
||||
from .library_commands import _UUID, LibraryCommandError, _validate_epoch
|
||||
from .models import utc_now_iso
|
||||
from .vocabulary import validate_vocabulary_tag_name
|
||||
|
||||
MAX_IMAGE_TAGS = 100
|
||||
_LOOKUP_CHUNK = 500
|
||||
_DIGEST = re.compile(r"[0-9a-f]{64}")
|
||||
_MAX_INT = 2**63 - 1
|
||||
_MIN_INT = -(2**63)
|
||||
# Only this module's table constants are interpolated into the SQL below.
|
||||
_OWNER = f"SELECT id FROM {FILES_TABLE} WHERE locator=?" # nosec B608
|
||||
_INSERT_OWNER = (
|
||||
f"INSERT INTO {FILES_TABLE}(id,locator,created_at) VALUES(?,?,?)" # nosec B608
|
||||
)
|
||||
_BYTE = (
|
||||
f"SELECT id,tag_revision,size,mtime_ns FROM {BYTES_TABLE} " # nosec B608
|
||||
"WHERE file_id=? AND sha256=?"
|
||||
)
|
||||
_INSERT_BYTE = (
|
||||
f"INSERT INTO {BYTES_TABLE}(id,file_id,sha256,size,mtime_ns," # nosec B608
|
||||
"verified_at,first_seen_at,tag_revision) VALUES(?,?,?,?,?,?,?,0)"
|
||||
)
|
||||
_REFRESH_BYTE = (
|
||||
f"UPDATE {BYTES_TABLE} SET size=?,mtime_ns=?,verified_at=? WHERE id=?" # nosec B608
|
||||
)
|
||||
_REFRESH_BY_LOCATOR = (
|
||||
f"UPDATE {BYTES_TABLE} SET size=?,mtime_ns=?,verified_at=? " # nosec B608
|
||||
f"WHERE sha256=? AND file_id=({_OWNER})"
|
||||
)
|
||||
_BYTE_TAGS = (
|
||||
f"SELECT t.name FROM {IMAGE_TAGS_TABLE} mit " # nosec B608
|
||||
"JOIN tags t ON t.id=mit.tag_id WHERE mit.byte_id=? ORDER BY mit.position"
|
||||
)
|
||||
_CLEAR_TAGS = f"DELETE FROM {IMAGE_TAGS_TABLE} WHERE byte_id=?" # nosec B608
|
||||
_INSERT_TAG = (
|
||||
f"INSERT INTO {IMAGE_TAGS_TABLE}(byte_id,tag_id,position) " # nosec B608
|
||||
"VALUES(?,?,?)"
|
||||
)
|
||||
_BUMP_REVISION = (
|
||||
f"UPDATE {BYTES_TABLE} SET tag_revision=tag_revision+1 WHERE id=?" # nosec B608
|
||||
)
|
||||
_REVISION = f"SELECT tag_revision FROM {BYTES_TABLE} WHERE id=?" # nosec B608
|
||||
_BUMP_INDEX = (
|
||||
f"UPDATE {METADATA_TABLE} SET " # nosec B608
|
||||
"image_index_revision=image_index_revision+1 WHERE singleton_id=1"
|
||||
)
|
||||
_READ = (
|
||||
"SELECT b.id,b.tag_revision," # nosec B608
|
||||
f"(SELECT COUNT(*) FROM {BYTES_TABLE} o WHERE o.file_id=f.id AND o.sha256!=?) "
|
||||
f"FROM {FILES_TABLE} f LEFT JOIN {BYTES_TABLE} b "
|
||||
"ON b.file_id=f.id AND b.sha256=? WHERE f.locator=?"
|
||||
)
|
||||
_FINGERPRINTS = (
|
||||
"SELECT f.locator,b.id,b.size,b.mtime_ns,b.verified_at " # nosec B608
|
||||
f"FROM {FILES_TABLE} f JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
||||
"WHERE f.locator IN ({marks})"
|
||||
)
|
||||
_TAGS_OF = (
|
||||
f"SELECT mit.byte_id,t.name FROM {IMAGE_TAGS_TABLE} mit " # nosec B608
|
||||
"JOIN tags t ON t.id=mit.tag_id WHERE mit.byte_id IN ({marks}) "
|
||||
"ORDER BY mit.byte_id,mit.position"
|
||||
)
|
||||
_RECORDS = (
|
||||
"SELECT f.locator,b.id,b.size,b.mtime_ns,t.name " # nosec B608
|
||||
f"FROM {FILES_TABLE} f JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
||||
f"LEFT JOIN {IMAGE_TAGS_TABLE} mit ON mit.byte_id=b.id "
|
||||
"LEFT JOIN tags t ON t.id=mit.tag_id "
|
||||
"ORDER BY f.locator,b.first_seen_at,b.id,mit.position"
|
||||
)
|
||||
|
||||
|
||||
class AnnotationCommandError(LibraryCommandError):
|
||||
"""An image tag command lost a race or hit the set's bound.
|
||||
|
||||
``code`` is ``tag-conflict`` (the set changed since it was read) or
|
||||
``tag-limit`` (the union would exceed ``MAX_IMAGE_TAGS``); ``current`` is
|
||||
the stored set ``{sha256, tags, tag_revision}`` read inside the
|
||||
transaction, so the caller can show it.
|
||||
"""
|
||||
|
||||
def __init__(self, code: str, epoch: str, current: dict | None = None):
|
||||
super().__init__(code, epoch)
|
||||
self.current = current
|
||||
|
||||
|
||||
def normalize_image_tags(names: object) -> list[str]:
|
||||
"""Trim and validate each name, fold case-insensitive repeats, cap at 100.
|
||||
|
||||
The first occurrence keeps its spelling and position. Raises ``ValueError``
|
||||
for anything but a list of valid vocabulary names or for more than
|
||||
``MAX_IMAGE_TAGS`` distinct names.
|
||||
"""
|
||||
if not isinstance(names, list):
|
||||
raise ValueError("tags must be a list of strings")
|
||||
result: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for value in names:
|
||||
name = validate_vocabulary_tag_name(value)
|
||||
key = name.casefold()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
result.append(name)
|
||||
if len(result) > MAX_IMAGE_TAGS:
|
||||
raise ValueError(f"An image can carry at most {MAX_IMAGE_TAGS} tags")
|
||||
return result
|
||||
|
||||
|
||||
def _validate_locator(locator: object) -> None:
|
||||
if not isinstance(locator, str) or not 0 < len(locator) <= MAX_LOCATOR_LENGTH:
|
||||
raise ValueError(
|
||||
f"locator must be a nonblank path of at most {MAX_LOCATOR_LENGTH} chars"
|
||||
)
|
||||
|
||||
|
||||
def _validate_digest(sha256: object) -> None:
|
||||
if not isinstance(sha256, str) or not _DIGEST.fullmatch(sha256):
|
||||
raise ValueError("sha256 must be 64 lowercase hexadecimal characters")
|
||||
|
||||
|
||||
def _validate_int(value: object, name: str, minimum: int) -> None:
|
||||
if type(value) is not int or not minimum <= value <= _MAX_INT:
|
||||
raise ValueError(f"{name} must be an integer not below {minimum}")
|
||||
|
||||
|
||||
def _validate_target(locator: object, sha256: object, size: object, mtime_ns: object):
|
||||
_validate_locator(locator)
|
||||
_validate_digest(sha256)
|
||||
_validate_int(size, "size", 0)
|
||||
_validate_int(mtime_ns, "mtime_ns", _MIN_INT)
|
||||
|
||||
|
||||
def _validate_fingerprint(locator: object, fingerprint: object) -> None:
|
||||
_validate_locator(locator)
|
||||
if (
|
||||
not isinstance(fingerprint, tuple)
|
||||
or len(fingerprint) != 2
|
||||
or any(type(value) is not int for value in fingerprint)
|
||||
):
|
||||
raise ValueError("stats must map locators to (size, mtime_ns) integer pairs")
|
||||
|
||||
|
||||
def _new_id() -> str:
|
||||
"""A canonical UUID; the schema's CHECK lets NULL through, this does not."""
|
||||
value = str(uuid4())
|
||||
if not _UUID.fullmatch(value):
|
||||
raise ValueError("Annotation ids must be canonical UUIDs")
|
||||
return value
|
||||
|
||||
|
||||
def _receipt(epoch: str, sha256: str, tags: list, revision: int, replayed: bool):
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"sha256": sha256,
|
||||
"tags": list(tags),
|
||||
"tag_revision": revision,
|
||||
"replayed": replayed,
|
||||
}
|
||||
|
||||
|
||||
class AnnotationOperations:
|
||||
"""PromptDatabase mixin: image tag sets, fingerprints and cheap reads.
|
||||
|
||||
``_`` helpers take the open ``_pair_transaction`` connection.
|
||||
"""
|
||||
|
||||
@contextmanager
|
||||
def _annotation_read(self) -> Iterator[sqlite3.Connection]:
|
||||
"""One deferred read transaction, so paired SELECTs see one snapshot."""
|
||||
conn = self.model.get_connection()
|
||||
if conn.in_transaction:
|
||||
raise sqlite3.ProgrammingError(
|
||||
"Annotation reads require their own snapshot"
|
||||
)
|
||||
with conn:
|
||||
conn.execute("BEGIN")
|
||||
yield conn
|
||||
|
||||
@staticmethod
|
||||
def _annotation_owner(conn: sqlite3.Connection, locator: str, now: str) -> str:
|
||||
row = conn.execute(_OWNER, (locator,)).fetchone()
|
||||
if row is not None:
|
||||
return row[0]
|
||||
file_id = _new_id()
|
||||
conn.execute(_INSERT_OWNER, (file_id, locator, now))
|
||||
return file_id
|
||||
|
||||
@staticmethod
|
||||
def _annotation_byte(
|
||||
conn: sqlite3.Connection,
|
||||
file_id: str,
|
||||
sha256: str,
|
||||
size: int,
|
||||
mtime_ns: int,
|
||||
now: str,
|
||||
) -> tuple[str, int, bool, bool]:
|
||||
"""The byte record for these bytes, refreshed or created.
|
||||
|
||||
Returns ``(id, tag_revision, created, refingerprinted)``; the last is
|
||||
True when an existing record's verified ``(size, mtime_ns)`` changed,
|
||||
which moves its currency for cheap reads (the caller bumps the index
|
||||
revision when nothing else does).
|
||||
"""
|
||||
row = conn.execute(_BYTE, (file_id, sha256)).fetchone()
|
||||
if row is None:
|
||||
byte_id = _new_id()
|
||||
conn.execute(
|
||||
_INSERT_BYTE, (byte_id, file_id, sha256, size, mtime_ns, now, now)
|
||||
)
|
||||
return byte_id, 0, True, False
|
||||
conn.execute(_REFRESH_BYTE, (size, mtime_ns, now, row[0]))
|
||||
return row[0], row[1], False, (row[2], row[3]) != (size, mtime_ns)
|
||||
|
||||
@staticmethod
|
||||
def _byte_tags(conn: sqlite3.Connection, byte_id: str) -> list[str]:
|
||||
return [row[0] for row in conn.execute(_BYTE_TAGS, (byte_id,))]
|
||||
|
||||
def _write_image_tags(
|
||||
self, conn: sqlite3.Connection, byte_id: str, tags: list[str]
|
||||
) -> int:
|
||||
"""Rewrite the junction in order, bump both revisions; return the new one."""
|
||||
tag_ids = self._ensure_tags(conn, tags)
|
||||
conn.execute(_CLEAR_TAGS, (byte_id,))
|
||||
conn.executemany(
|
||||
_INSERT_TAG,
|
||||
[(byte_id, tag_ids[name], position) for position, name in enumerate(tags)],
|
||||
)
|
||||
conn.execute(_BUMP_REVISION, (byte_id,))
|
||||
conn.execute(_BUMP_INDEX)
|
||||
return conn.execute(_REVISION, (byte_id,)).fetchone()[0]
|
||||
|
||||
def _current_set(self, conn, sha256, byte_id, revision) -> dict:
|
||||
return {
|
||||
"sha256": sha256,
|
||||
"tags": self._byte_tags(conn, byte_id),
|
||||
"tag_revision": revision,
|
||||
}
|
||||
|
||||
def read_image_tags(self, locator: str, sha256: str) -> dict:
|
||||
"""The stored set of these exact bytes, or an empty set at revision 0.
|
||||
|
||||
``prior_count`` is the number of other byte records at the locator
|
||||
(earlier or later bytes of the same file).
|
||||
"""
|
||||
_validate_locator(locator)
|
||||
_validate_digest(sha256)
|
||||
with self._annotation_read() as conn:
|
||||
row = conn.execute(_READ, (sha256, sha256, locator)).fetchone()
|
||||
if row is None:
|
||||
return {"tags": [], "tag_revision": 0, "prior_count": 0}
|
||||
tags = self._byte_tags(conn, row[0]) if row[0] is not None else []
|
||||
return {"tags": tags, "tag_revision": row[1] or 0, "prior_count": row[2]}
|
||||
|
||||
def replace_image_tags(
|
||||
self,
|
||||
*,
|
||||
locator: str,
|
||||
sha256: str,
|
||||
size: int,
|
||||
mtime_ns: int,
|
||||
expected_epoch: str,
|
||||
expected_tag_revision: int,
|
||||
tags: list,
|
||||
) -> dict:
|
||||
"""Replace the set of these exact bytes, guarded by epoch and revision.
|
||||
|
||||
Creates the owner and byte record when the bytes are new (an explicit
|
||||
empty set is a real save). The identical set at the current revision,
|
||||
or one behind it (a lost-response retry), is ``replayed`` and writes
|
||||
nothing; any other revision mismatch is ``tag-conflict`` with the
|
||||
current set and no writes.
|
||||
"""
|
||||
_validate_target(locator, sha256, size, mtime_ns)
|
||||
_validate_epoch(expected_epoch)
|
||||
_validate_int(expected_tag_revision, "expected_tag_revision", 0)
|
||||
tags = normalize_image_tags(tags)
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
now = utc_now_iso()
|
||||
file_id = self._annotation_owner(conn, locator, now)
|
||||
byte_id, revision, created, refingerprinted = self._annotation_byte(
|
||||
conn, file_id, sha256, size, mtime_ns, now
|
||||
)
|
||||
current = self._byte_tags(conn, byte_id)
|
||||
if (
|
||||
not created
|
||||
and current == tags
|
||||
and expected_tag_revision in (revision, revision - 1)
|
||||
):
|
||||
if refingerprinted:
|
||||
conn.execute(_BUMP_INDEX)
|
||||
return _receipt(epoch, sha256, current, revision, True)
|
||||
if expected_tag_revision != revision:
|
||||
raise AnnotationCommandError(
|
||||
"tag-conflict",
|
||||
epoch,
|
||||
{"sha256": sha256, "tags": current, "tag_revision": revision},
|
||||
)
|
||||
revision = self._write_image_tags(conn, byte_id, tags)
|
||||
return _receipt(epoch, sha256, tags, revision, False)
|
||||
|
||||
def add_image_tags(
|
||||
self,
|
||||
*,
|
||||
locator: str,
|
||||
sha256: str,
|
||||
size: int,
|
||||
mtime_ns: int,
|
||||
expected_epoch: str,
|
||||
tags: list,
|
||||
) -> dict:
|
||||
"""Union ``tags`` into the set of these exact bytes, keeping stored spelling.
|
||||
|
||||
``added`` counts the names that were new (case-insensitively). A union
|
||||
over ``MAX_IMAGE_TAGS`` is ``tag-limit`` with the current set and no
|
||||
writes; a union that adds nothing to an existing record is ``replayed``.
|
||||
"""
|
||||
_validate_target(locator, sha256, size, mtime_ns)
|
||||
_validate_epoch(expected_epoch)
|
||||
tags = normalize_image_tags(tags)
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
now = utc_now_iso()
|
||||
file_id = self._annotation_owner(conn, locator, now)
|
||||
byte_id, revision, created, refingerprinted = self._annotation_byte(
|
||||
conn, file_id, sha256, size, mtime_ns, now
|
||||
)
|
||||
current = self._byte_tags(conn, byte_id)
|
||||
present = {name.casefold() for name in current}
|
||||
added = [name for name in tags if name.casefold() not in present]
|
||||
merged = current + added
|
||||
if len(merged) > MAX_IMAGE_TAGS:
|
||||
raise AnnotationCommandError(
|
||||
"tag-limit",
|
||||
epoch,
|
||||
{"sha256": sha256, "tags": current, "tag_revision": revision},
|
||||
)
|
||||
if not created and not added:
|
||||
if refingerprinted:
|
||||
conn.execute(_BUMP_INDEX)
|
||||
return {**_receipt(epoch, sha256, current, revision, True), "added": 0}
|
||||
revision = self._write_image_tags(conn, byte_id, merged)
|
||||
return {
|
||||
**_receipt(epoch, sha256, merged, revision, False),
|
||||
"added": len(added),
|
||||
}
|
||||
|
||||
def refresh_image_fingerprint(
|
||||
self, locator: str, sha256: str, size: int, mtime_ns: int
|
||||
) -> bool:
|
||||
"""Record that these exact bytes were just seen with this stat.
|
||||
|
||||
True when a byte record was refreshed; nothing is created and no tag
|
||||
or tag revision changes. The image index revision moves because the
|
||||
record's currency for cheap reads may have changed.
|
||||
"""
|
||||
_validate_target(locator, sha256, size, mtime_ns)
|
||||
with self.model.get_connection() as conn:
|
||||
cursor = conn.execute(
|
||||
_REFRESH_BY_LOCATOR, (size, mtime_ns, utc_now_iso(), sha256, locator)
|
||||
)
|
||||
if cursor.rowcount > 0:
|
||||
conn.execute(_BUMP_INDEX)
|
||||
return cursor.rowcount > 0
|
||||
|
||||
def current_image_tags(self, stats: dict) -> dict:
|
||||
"""Image tags of the files whose current ``(size, mtime_ns)`` is verified.
|
||||
|
||||
``stats`` maps locators to their present fingerprint. A locator is
|
||||
answered (possibly with an empty set) only when a byte record carries
|
||||
exactly that verified fingerprint; the most recently verified record
|
||||
wins when several do. Lookups go in chunks of 500 locators.
|
||||
"""
|
||||
if not isinstance(stats, dict):
|
||||
raise ValueError(
|
||||
"stats must map locators to (size, mtime_ns) integer pairs"
|
||||
)
|
||||
for locator, fingerprint in stats.items():
|
||||
_validate_fingerprint(locator, fingerprint)
|
||||
locators = list(stats)
|
||||
chosen: dict[str, tuple[tuple[str, str], str]] = {}
|
||||
result: dict[str, list[str]] = {}
|
||||
with self._annotation_read() as conn:
|
||||
for start in range(0, len(locators), _LOOKUP_CHUNK):
|
||||
chunk = locators[start : start + _LOOKUP_CHUNK]
|
||||
marks = ",".join("?" * len(chunk))
|
||||
for locator, byte_id, size, mtime_ns, verified_at in conn.execute(
|
||||
_FINGERPRINTS.format(marks=marks), chunk
|
||||
):
|
||||
if (size, mtime_ns) != stats[locator]:
|
||||
continue
|
||||
key = (verified_at, byte_id)
|
||||
if locator not in chosen or key > chosen[locator][0]:
|
||||
chosen[locator] = (key, byte_id)
|
||||
owners = {byte_id: locator for locator, (_key, byte_id) in chosen.items()}
|
||||
result = {locator: [] for locator in chosen}
|
||||
byte_ids = list(owners)
|
||||
for start in range(0, len(byte_ids), _LOOKUP_CHUNK):
|
||||
chunk = byte_ids[start : start + _LOOKUP_CHUNK]
|
||||
marks = ",".join("?" * len(chunk))
|
||||
for byte_id, name in conn.execute(_TAGS_OF.format(marks=marks), chunk):
|
||||
result[owners[byte_id]].append(name)
|
||||
return result
|
||||
|
||||
def iter_image_tag_records(self) -> Iterator[tuple[str, int, int, list[str]]]:
|
||||
"""Stream every byte record as ``(locator, size, mtime_ns, tags)``.
|
||||
|
||||
One SELECT ordered by locator, then by the bytes' first sighting, with
|
||||
each set's names in position order.
|
||||
"""
|
||||
conn = self.model.get_connection()
|
||||
current: tuple | None = None
|
||||
for locator, byte_id, size, mtime_ns, name in conn.execute(_RECORDS):
|
||||
if current is None or current[0] != byte_id:
|
||||
if current is not None:
|
||||
yield current[1]
|
||||
current = (byte_id, (locator, size, mtime_ns, []))
|
||||
if name is not None:
|
||||
current[1][3].append(name)
|
||||
if current is not None:
|
||||
yield current[1]
|
||||
@@ -137,3 +137,16 @@ class ImageFavoriteOperations:
|
||||
)
|
||||
found.update(path for (path,) in rows)
|
||||
return found
|
||||
|
||||
def get_root_favorites(self, root_key: str) -> set[str]:
|
||||
"""Every relative key favorited under *root_key* (a primary-key range).
|
||||
|
||||
The Favorites listing (``favorites=1``) intersects these with the
|
||||
root's scan, so its size follows the root's favorites, not its files.
|
||||
"""
|
||||
conn = self.model.get_connection()
|
||||
rows = conn.execute(
|
||||
"SELECT relative_path FROM image_favorites WHERE root_key=?",
|
||||
(root_key,),
|
||||
)
|
||||
return {path for (path,) in rows}
|
||||
|
||||
+17
-2
@@ -17,6 +17,8 @@ from typing import Optional, Set, Tuple
|
||||
|
||||
from .library_schema import library_schema_error, migrate_library_state
|
||||
from .image_favorites import image_favorites_schema_error, migrate_image_favorites
|
||||
from .annotation_schema import annotation_schema_error, migrate_annotation_state
|
||||
from .saved_views import migrate_saved_views, saved_views_schema_error
|
||||
from .execution_history_schema import (
|
||||
history_schema_error,
|
||||
migrate_execution_history,
|
||||
@@ -300,6 +302,8 @@ class PromptModel:
|
||||
migrate_execution_history(conn)
|
||||
migrate_library_state(conn)
|
||||
migrate_image_favorites(conn)
|
||||
migrate_saved_views(conn)
|
||||
migrate_annotation_state(conn)
|
||||
|
||||
def _migrate_prompt_pairs(self, conn: sqlite3.Connection) -> None:
|
||||
conn.execute("SAVEPOINT prompt_pair_schema")
|
||||
@@ -1019,8 +1023,9 @@ class PromptModel:
|
||||
Args:
|
||||
path: File to inspect
|
||||
require_library_state: Also require the tables every current
|
||||
database has (Library state and image favorites), as a
|
||||
migrated restore stage must; old backups may lack them.
|
||||
database has (Library state, image favorites, saved views
|
||||
and image annotations), as a migrated restore stage must;
|
||||
old backups may lack them.
|
||||
|
||||
Returns:
|
||||
(ok, reason): reason is "ok" when the file is acceptable, otherwise
|
||||
@@ -1080,6 +1085,16 @@ class PromptModel:
|
||||
)
|
||||
if favorites_error:
|
||||
return False, favorites_error
|
||||
views_error = saved_views_schema_error(
|
||||
conn, allow_missing=not require_library_state
|
||||
)
|
||||
if views_error:
|
||||
return False, views_error
|
||||
annotation_error = annotation_schema_error(
|
||||
conn, allow_missing=not require_library_state
|
||||
)
|
||||
if annotation_error:
|
||||
return False, annotation_error
|
||||
return True, "ok"
|
||||
except sqlite3.Error as e:
|
||||
return False, f"Invalid SQLite database: {e}"
|
||||
|
||||
+99
-6
@@ -12,6 +12,8 @@ from .models import PromptModel, normalize_image_path, utc_now_iso
|
||||
from .pairs import PairDependencyError, PairOperations
|
||||
from .library import LibraryOperations, titled
|
||||
from .image_favorites import ImageFavoriteOperations
|
||||
from .annotations import AnnotationOperations
|
||||
from .saved_views import SavedViewOperations
|
||||
from .library_commands import LibraryCommandOperations
|
||||
from .vocabulary import VocabularyOperations
|
||||
from .execution_history import ExecutionHistoryOperations
|
||||
@@ -180,6 +182,21 @@ AND p.id NOT IN (SELECT prompt_id FROM live_hint_prompts)
|
||||
"""
|
||||
|
||||
|
||||
# The scan's known set (#249 Activity-2): every image row, with its stored file
|
||||
# size and whether its pair link is resolved: exactly one distinct pair, the rule
|
||||
# _LINK_SUMMARY_SQL and get_image_prompt_info use. A file is known and resolved
|
||||
# when it has exactly one image row, that row is resolved, and the row's stored
|
||||
# size equals the file's size now; several rows (one file linked to more than
|
||||
# one prompt) are ambiguous, and an unknown (NULL) stored size is not known. The
|
||||
# rows are streamed in fetches of KNOWN_PATHS_FETCH_SIZE and only the scanned
|
||||
# paths are kept, so memory follows the scan (at most MAX_SCAN_FILES paths), not
|
||||
# the table.
|
||||
_KNOWN_PATHS_SQL = """SELECT gi.file_path, gi.image_path, gi.file_size,
|
||||
(SELECT COUNT(DISTINCT ip.pair_id) FROM image_prompt_pairs ip
|
||||
WHERE ip.image_id = gi.id) = 1
|
||||
FROM generated_images gi"""
|
||||
KNOWN_PATHS_FETCH_SIZE = 1000
|
||||
|
||||
# One SELECT reads the link summaries of at most this many paths: three bound
|
||||
# parameters per path stays under SQLite's historical 999-variable limit.
|
||||
LINK_SUMMARY_BATCH = 200
|
||||
@@ -306,6 +323,8 @@ class PromptDatabase(
|
||||
ExecutionOutputOperations,
|
||||
HistoryReadOperations,
|
||||
ImageFavoriteOperations,
|
||||
AnnotationOperations,
|
||||
SavedViewOperations,
|
||||
):
|
||||
"""Database operations class for managing prompts."""
|
||||
|
||||
@@ -1067,16 +1086,39 @@ class PromptDatabase(
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _bump_image_tag_bytes(conn: sqlite3.Connection, tag_id: int) -> int:
|
||||
"""Move the tag revision of every image set that carries *tag_id*.
|
||||
|
||||
Vocabulary rename, merge and delete change what an image's set reads
|
||||
as, so a Tag dialog opened before the change must see ``tag-conflict``
|
||||
on Save instead of recreating the old name. Bumps the image index
|
||||
revision once when any set was affected; returns that count.
|
||||
"""
|
||||
cursor = conn.execute(
|
||||
"UPDATE media_annotation_bytes SET tag_revision = tag_revision + 1 "
|
||||
"WHERE id IN (SELECT byte_id FROM media_image_tags WHERE tag_id = ?)",
|
||||
(tag_id,),
|
||||
)
|
||||
if cursor.rowcount > 0:
|
||||
conn.execute(
|
||||
"UPDATE media_annotation_metadata SET "
|
||||
"image_index_revision = image_index_revision + 1 WHERE singleton_id = 1"
|
||||
)
|
||||
return cursor.rowcount
|
||||
|
||||
@classmethod
|
||||
def _move_tag_links(
|
||||
conn: sqlite3.Connection, source_id: int, target_id: int
|
||||
cls, conn: sqlite3.Connection, source_id: int, target_id: int
|
||||
) -> None:
|
||||
"""Repoint component and pair links from source to target.
|
||||
"""Repoint component, pair and image links from source to target.
|
||||
|
||||
Owners that already carry the target keep that one link; the source's
|
||||
leftover duplicates are removed. The source registry row is left for the
|
||||
caller to delete.
|
||||
leftover duplicates are removed (an image set keeps the target's
|
||||
position). The source registry row is left for the caller to delete.
|
||||
Image sets that carried the source get a new tag revision first.
|
||||
"""
|
||||
for junction in ("prompt_tags", "prompt_pair_tags"):
|
||||
cls._bump_image_tag_bytes(conn, source_id)
|
||||
for junction in ("prompt_tags", "prompt_pair_tags", "media_image_tags"):
|
||||
conn.execute(
|
||||
f"UPDATE OR IGNORE {junction} SET tag_id = ? WHERE tag_id = ?",
|
||||
(target_id, source_id),
|
||||
@@ -1129,6 +1171,7 @@ class PromptDatabase(
|
||||
conn.execute("DELETE FROM tags WHERE id = ?", (old_tag_id,))
|
||||
else:
|
||||
# Simple rename
|
||||
self._bump_image_tag_bytes(conn, old_tag_id)
|
||||
conn.execute(
|
||||
"UPDATE tags SET name = ? WHERE id = ?", (new_name, old_tag_id)
|
||||
)
|
||||
@@ -1170,6 +1213,7 @@ class PromptDatabase(
|
||||
affected = cursor.fetchone()["c"]
|
||||
|
||||
# CASCADE delete handles prompt_tags entries
|
||||
self._bump_image_tag_bytes(conn, tag_id)
|
||||
conn.execute("DELETE FROM tags WHERE id = ?", (tag_id,))
|
||||
conn.commit()
|
||||
|
||||
@@ -1222,7 +1266,10 @@ class PromptDatabase(
|
||||
pair_count = conn.execute(
|
||||
"SELECT COUNT(*) FROM prompt_pair_tags WHERE tag_id = ?", (src_id,)
|
||||
).fetchone()[0]
|
||||
if src_count or pair_count:
|
||||
image_count = conn.execute(
|
||||
"SELECT COUNT(*) FROM media_image_tags WHERE tag_id = ?", (src_id,)
|
||||
).fetchone()[0]
|
||||
if src_count or pair_count or image_count:
|
||||
self._move_tag_links(conn, src_id, target_id)
|
||||
tags_merged += 1
|
||||
affected += src_count
|
||||
@@ -2278,6 +2325,52 @@ class PromptDatabase(
|
||||
"parameters": row[8] if len(row) > 8 else None,
|
||||
}
|
||||
|
||||
def get_known_resolved_paths(self, files: Dict[str, Optional[int]]) -> Set[str]:
|
||||
"""The paths in ``files`` the Library already links with a resolved link.
|
||||
|
||||
``files`` maps each scanned path to its size on disk (None when it could
|
||||
not be read). A path qualifies when exactly one ``generated_images`` row
|
||||
claims the file, that row has exactly one pair link
|
||||
(``image_prompt_pairs``, written only by resolved ingestion), and its
|
||||
stored ``file_size`` is not NULL and equals the given size, so a file
|
||||
replaced at the same path is read again. Files linked without a pair
|
||||
(the monitor's fallback and FIFO links, the legacy positive-only
|
||||
import), linked to several prompts or pairs, or not linked at all do
|
||||
not qualify. Both sides go through ``normalize_image_path`` (absolute,
|
||||
normalized, ``os.path.normcase``), so a Windows row stored in another
|
||||
case or spelling still matches. One query, streamed; returns the
|
||||
matching keys of ``files``.
|
||||
"""
|
||||
wanted: Dict[str, List[Tuple[str, Optional[int]]]] = {}
|
||||
for path, size in files.items():
|
||||
key = normalize_image_path(str(path))
|
||||
wanted.setdefault(key, []).append((str(path), size))
|
||||
if not wanted:
|
||||
return set()
|
||||
rows_seen: Dict[str, int] = {}
|
||||
stored_sizes: Dict[str, int] = {}
|
||||
with self.model.get_connection() as conn:
|
||||
cursor = conn.execute(_KNOWN_PATHS_SQL)
|
||||
while True:
|
||||
rows = cursor.fetchmany(KNOWN_PATHS_FETCH_SIZE)
|
||||
if not rows:
|
||||
break
|
||||
for file_path, image_path, file_size, is_resolved in rows:
|
||||
# image_path is NOT NULL; file_path is NULL only on old rows.
|
||||
key = normalize_image_path(file_path or image_path)
|
||||
if key not in wanted:
|
||||
continue
|
||||
rows_seen[key] = rows_seen.get(key, 0) + 1
|
||||
if is_resolved and file_size is not None:
|
||||
stored_sizes[key] = file_size
|
||||
return {
|
||||
path
|
||||
for key, stored_size in stored_sizes.items()
|
||||
if rows_seen[key] == 1
|
||||
for path, size in wanted[key]
|
||||
if size is not None and size == stored_size
|
||||
}
|
||||
|
||||
def get_image_link_summaries(
|
||||
self, paths: List[str], allow_filename_fallback: bool = True
|
||||
) -> Dict[str, Dict[str, Any]]:
|
||||
|
||||
@@ -0,0 +1,334 @@
|
||||
"""Persisted saved views (D16): a named Images or Library toolbar state.
|
||||
|
||||
A view's state holds exactly its page's toolbar values and its Refine-this-view
|
||||
refinements, whitelisted per page and stored as compact JSON of at most 4 KB.
|
||||
|
||||
Writes (the API's POST and PATCH) validate strictly. Startup and restore check
|
||||
only the structure (the table, its columns and each row's types, sizes and JSON
|
||||
object), never the value rules: a row saved under older rules must not stop the
|
||||
database from opening. Startup is more tolerant still: a row whose state is not
|
||||
a JSON object is logged by id, left in place and skipped by every read, while
|
||||
restore rejects such a backup. Reads normalize instead, so a stale row opens with its
|
||||
usable values and defaults for the rest. Optional in old backups, required once
|
||||
a database is current (see ``library_schema.py``).
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import sqlite3
|
||||
from typing import Optional
|
||||
|
||||
# The models logger's parent ("prompt_manager.database.models" logs there too).
|
||||
_LOGGER = logging.getLogger("prompt_manager.database.saved_views")
|
||||
|
||||
TABLE = "saved_views"
|
||||
PAGES = ("images", "library")
|
||||
# The schema check's literal SQL repeats the name and state caps (names in
|
||||
# bytes, at most 4 per character); keep them in step.
|
||||
MAX_NAME_LENGTH = 100
|
||||
MAX_STATE_BYTES = 4096
|
||||
MAX_TAGS = 20
|
||||
MAX_TAG_LENGTH = 50
|
||||
_IMAGE_SORTS = ("newest", "oldest", "name", "name-desc", "rating")
|
||||
_LIBRARY_SORTS = ("newest", "oldest", "name", "rating")
|
||||
_MEDIA = ("all", "image", "video")
|
||||
_CONTROL = re.compile(r"[\x00-\x1f\x7f]")
|
||||
_SOURCE = re.compile(r"all|0|[1-9][0-9]{0,8}")
|
||||
FALLBACK_NAME = "Saved view"
|
||||
|
||||
|
||||
class SavedViewInvalid(ValueError):
|
||||
"""A request's page, name or state breaks the write rules (a 400)."""
|
||||
|
||||
|
||||
# (kind, limit-or-choices, default) per state key; "text" values are
|
||||
# length-capped. The defaults match the client's (saved-views.mjs FIELDS).
|
||||
_STATE_FIELDS = {
|
||||
"images": {
|
||||
"q": ("text", 500, ""),
|
||||
"category": ("text", 100, ""),
|
||||
"sort": ("choice", _IMAGE_SORTS, "newest"),
|
||||
"source": ("source", None, "all"),
|
||||
"folder": ("text", 1024, ""),
|
||||
"media": ("choice", _MEDIA, "all"),
|
||||
"tags": ("tags", None, ()),
|
||||
# The Favorites page's filter (#249 D51): a view saved there reopens it.
|
||||
"favorites": ("bool", None, False),
|
||||
},
|
||||
"library": {
|
||||
"q": ("text", 500, ""),
|
||||
"category": ("text", 100, ""),
|
||||
"sort": ("choice", _LIBRARY_SORTS, "newest"),
|
||||
"tags": ("tags", None, ()),
|
||||
},
|
||||
}
|
||||
|
||||
_DDL = (
|
||||
f"CREATE TABLE {TABLE} (id INTEGER PRIMARY KEY AUTOINCREMENT, "
|
||||
"page TEXT NOT NULL CHECK(page IN ('images','library')), "
|
||||
"name TEXT NOT NULL, state TEXT NOT NULL, position INTEGER NOT NULL, "
|
||||
"created_at TEXT NOT NULL, updated_at TEXT NOT NULL)"
|
||||
)
|
||||
_COLUMNS = (
|
||||
("id", "INTEGER", 0, 1),
|
||||
("page", "TEXT", 1, 0),
|
||||
("name", "TEXT", 1, 0),
|
||||
("state", "TEXT", 1, 0),
|
||||
("position", "INTEGER", 1, 0),
|
||||
("created_at", "TEXT", 1, 0),
|
||||
("updated_at", "TEXT", 1, 0),
|
||||
)
|
||||
_SELECT = "SELECT id, page, name, state FROM saved_views"
|
||||
|
||||
|
||||
def _compact(sql: str) -> str:
|
||||
return re.sub(r"\s+", "", sql).lower()
|
||||
|
||||
|
||||
def validate_name(name: object) -> str:
|
||||
"""The trimmed name, 1 to 100 characters without control characters."""
|
||||
if not isinstance(name, str):
|
||||
raise SavedViewInvalid("Name must be a string")
|
||||
trimmed = name.strip()
|
||||
if not 1 <= len(trimmed) <= MAX_NAME_LENGTH:
|
||||
raise SavedViewInvalid(f"Name must be 1 to {MAX_NAME_LENGTH} characters")
|
||||
if _CONTROL.search(trimmed):
|
||||
raise SavedViewInvalid("Name contains control characters")
|
||||
return trimmed
|
||||
|
||||
|
||||
def _validate_value(key: str, kind: str, limit, value: object) -> None:
|
||||
"""Raise ``SavedViewInvalid`` when *value* breaks *key*'s write rule."""
|
||||
if kind == "tags":
|
||||
if not isinstance(value, list) or len(value) > MAX_TAGS:
|
||||
raise SavedViewInvalid(
|
||||
f"State '{key}' must be a list of at most {MAX_TAGS}"
|
||||
)
|
||||
for tag in value:
|
||||
if (
|
||||
not isinstance(tag, str)
|
||||
or not tag.strip()
|
||||
or len(tag) > MAX_TAG_LENGTH
|
||||
or "," in tag
|
||||
or _CONTROL.search(tag)
|
||||
):
|
||||
raise SavedViewInvalid(f"State '{key}' holds an invalid tag")
|
||||
return
|
||||
if kind == "bool":
|
||||
if not isinstance(value, bool):
|
||||
raise SavedViewInvalid(f"State '{key}' must be true or false")
|
||||
return
|
||||
if not isinstance(value, str):
|
||||
raise SavedViewInvalid(f"State '{key}' must be a string")
|
||||
if kind == "choice" and value not in limit:
|
||||
raise SavedViewInvalid(f"State '{key}' is not a supported value")
|
||||
if kind == "source" and not _SOURCE.fullmatch(value):
|
||||
raise SavedViewInvalid(f"State '{key}' must be 'all' or a source index")
|
||||
if kind == "text" and (len(value) > limit or _CONTROL.search(value)):
|
||||
raise SavedViewInvalid(f"State '{key}' is too long or has control characters")
|
||||
|
||||
|
||||
def encode_state(page: object, state: object) -> str:
|
||||
"""Validate *state* for *page* and return its compact JSON (at most 4 KB)."""
|
||||
if page not in PAGES:
|
||||
raise SavedViewInvalid("Page must be 'images' or 'library'")
|
||||
fields = _STATE_FIELDS[page]
|
||||
if not isinstance(state, dict):
|
||||
raise SavedViewInvalid("State must be an object")
|
||||
if set(state) != set(fields):
|
||||
raise SavedViewInvalid(
|
||||
f"State for {page} must contain exactly " + ", ".join(sorted(fields))
|
||||
)
|
||||
for key, (kind, limit, _default) in fields.items():
|
||||
_validate_value(key, kind, limit, state[key])
|
||||
encoded = json.dumps(
|
||||
{key: state[key] for key in fields}, ensure_ascii=False, separators=(",", ":")
|
||||
)
|
||||
if len(encoded.encode("utf-8")) > MAX_STATE_BYTES:
|
||||
raise SavedViewInvalid("State must encode to at most 4 KB")
|
||||
return encoded
|
||||
|
||||
|
||||
def _decode_object(raw: object) -> Optional[dict]:
|
||||
"""The JSON object stored in *raw*, or None when it holds no object."""
|
||||
if isinstance(raw, dict):
|
||||
return raw
|
||||
if not isinstance(raw, str):
|
||||
return None
|
||||
try:
|
||||
decoded = json.loads(raw)
|
||||
except (ValueError, RecursionError):
|
||||
return None
|
||||
return decoded if isinstance(decoded, dict) else None
|
||||
|
||||
|
||||
def normalize_state(page: str, raw: object) -> dict:
|
||||
"""The page's full state from a stored row, never raising.
|
||||
|
||||
Every key of the page's current whitelist is present: a stored value that
|
||||
passes today's write rule is kept, anything else (missing, invalid, or a
|
||||
row that is not a JSON object) takes the key's default, and keys the
|
||||
whitelist no longer has are dropped. An unknown page has no keys.
|
||||
"""
|
||||
fields = _STATE_FIELDS.get(page, {})
|
||||
stored = _decode_object(raw) or {}
|
||||
state = {}
|
||||
for key, (kind, limit, default) in fields.items():
|
||||
value = stored.get(key, default)
|
||||
try:
|
||||
_validate_value(key, kind, limit, value)
|
||||
except SavedViewInvalid:
|
||||
value = default
|
||||
state[key] = list(value) if kind == "tags" else value
|
||||
return state
|
||||
|
||||
|
||||
def normalize_name(name: object) -> str:
|
||||
"""A stored name made displayable: trimmed, capped, never empty."""
|
||||
text = _CONTROL.sub("", name) if isinstance(name, str) else ""
|
||||
trimmed = text.strip()[:MAX_NAME_LENGTH].strip()
|
||||
return trimmed or FALLBACK_NAME
|
||||
|
||||
|
||||
def saved_views_schema_error(
|
||||
conn: sqlite3.Connection,
|
||||
*,
|
||||
allow_missing: bool = True,
|
||||
check_states: bool = True,
|
||||
) -> Optional[str]:
|
||||
"""Reject unsupported saved-views schemas and structurally invalid rows.
|
||||
|
||||
Reads only. Checks the table, its columns, each row's types and sizes and
|
||||
that each state is a JSON object; never the value rules, which change
|
||||
between versions and are normalized on read instead. A missing table is
|
||||
acceptable for backups that predate it unless ``allow_missing`` is false.
|
||||
``check_states=False`` (live startup) skips the JSON-object check; such
|
||||
rows are reported by ``unusable_view_ids`` and skipped on read.
|
||||
"""
|
||||
objects = conn.execute(
|
||||
"SELECT type,name,sql FROM sqlite_master WHERE name=? "
|
||||
"OR (type IN ('index','trigger') AND tbl_name=?)",
|
||||
(TABLE, TABLE),
|
||||
).fetchall()
|
||||
if not objects:
|
||||
return None if allow_missing else "Missing saved views schema"
|
||||
tables = [sql for kind, _name, sql in objects if kind == "table"]
|
||||
if len(tables) != 1 or len(objects) != 1:
|
||||
return "Saved views name is reserved by an unsupported schema object"
|
||||
if _compact(tables[0] or "") != _compact(_DDL):
|
||||
return "Saved views has an invalid table definition"
|
||||
columns = tuple(
|
||||
(row[1], row[2].upper(), row[3], row[5])
|
||||
for row in conn.execute(f"PRAGMA table_info({TABLE})")
|
||||
)
|
||||
if columns != _COLUMNS:
|
||||
return "Saved views has invalid columns"
|
||||
# Literal SQL: the table name and limits are constants of this module.
|
||||
if conn.execute(
|
||||
"SELECT 1 FROM saved_views WHERE typeof(page) != 'text' "
|
||||
"OR page NOT IN ('images','library') "
|
||||
"OR typeof(name) != 'text' OR typeof(state) != 'text' "
|
||||
"OR typeof(position) != 'integer' OR typeof(created_at) != 'text' "
|
||||
"OR typeof(updated_at) != 'text' "
|
||||
"OR length(CAST(name AS BLOB)) > 400 "
|
||||
"OR length(CAST(state AS BLOB)) > 4096 "
|
||||
"OR length(CAST(created_at AS BLOB)) > 64 "
|
||||
"OR length(CAST(updated_at AS BLOB)) > 64 LIMIT 1"
|
||||
).fetchone():
|
||||
return "Saved views contains invalid values"
|
||||
if check_states and unusable_view_ids(conn):
|
||||
return "Saved views contains invalid values"
|
||||
return None
|
||||
|
||||
|
||||
def unusable_view_ids(conn: sqlite3.Connection) -> list[int]:
|
||||
"""Ids of rows whose state is not a JSON object (never listed)."""
|
||||
return [
|
||||
view_id
|
||||
for view_id, state in conn.execute("SELECT id, state FROM saved_views")
|
||||
if _decode_object(state) is None
|
||||
]
|
||||
|
||||
|
||||
def migrate_saved_views(conn: sqlite3.Connection) -> None:
|
||||
"""Install the table atomically; validate before and after creating it."""
|
||||
conn.execute("SAVEPOINT saved_views_schema")
|
||||
try:
|
||||
error = saved_views_schema_error(conn, check_states=False)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute(_DDL.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
|
||||
error = saved_views_schema_error(conn, allow_missing=False, check_states=False)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
# A corrupt state never stops the live database from opening: the row
|
||||
# stays (no data is destroyed) and reads skip it. Only its id is logged.
|
||||
for view_id in unusable_view_ids(conn):
|
||||
_LOGGER.warning(
|
||||
"Saved view id %d has an unreadable state and is hidden", view_id
|
||||
)
|
||||
conn.execute("RELEASE saved_views_schema")
|
||||
except BaseException:
|
||||
conn.execute("ROLLBACK TO saved_views_schema")
|
||||
conn.execute("RELEASE saved_views_schema")
|
||||
raise
|
||||
|
||||
|
||||
def _view(row) -> dict:
|
||||
view_id, page, name, state = row
|
||||
return {
|
||||
"id": view_id,
|
||||
"page": page,
|
||||
"name": normalize_name(name),
|
||||
"state": normalize_state(page, state),
|
||||
}
|
||||
|
||||
|
||||
class SavedViewOperations:
|
||||
"""PromptDatabase mixin: list, create, rename and delete saved views."""
|
||||
|
||||
def list_saved_views(self) -> list[dict]:
|
||||
"""Every view, in list order (position, then creation)."""
|
||||
rows = self.model.get_connection().execute(_SELECT + " ORDER BY position, id")
|
||||
# A row whose state is no JSON object (logged at startup) is never listed.
|
||||
return [_view(row) for row in rows if _decode_object(row[3]) is not None]
|
||||
|
||||
def create_saved_view(self, page: object, name: object, state: object) -> dict:
|
||||
"""Validate and append one view at the end of the list."""
|
||||
from .models import utc_now_iso # models imports this module's schema
|
||||
|
||||
encoded = encode_state(page, state)
|
||||
trimmed = validate_name(name)
|
||||
now = utc_now_iso()
|
||||
with self.model.get_connection() as conn:
|
||||
cursor = conn.execute(
|
||||
"INSERT INTO saved_views (page, name, state, position, "
|
||||
"created_at, updated_at) SELECT ?, ?, ?, "
|
||||
"COALESCE(MAX(position), -1) + 1, ?, ? FROM saved_views",
|
||||
(page, trimmed, encoded, now, now),
|
||||
)
|
||||
row = conn.execute(
|
||||
_SELECT + " WHERE id = ?", (cursor.lastrowid,)
|
||||
).fetchone()
|
||||
return _view(row)
|
||||
|
||||
def rename_saved_view(self, view_id: int, name: object) -> Optional[dict]:
|
||||
"""The renamed view, or None when no view has *view_id*."""
|
||||
from .models import utc_now_iso
|
||||
|
||||
trimmed = validate_name(name)
|
||||
with self.model.get_connection() as conn:
|
||||
conn.execute(
|
||||
"UPDATE saved_views SET name = ?, updated_at = ? WHERE id = ?",
|
||||
(trimmed, utc_now_iso(), view_id),
|
||||
)
|
||||
row = conn.execute(_SELECT + " WHERE id = ?", (view_id,)).fetchone()
|
||||
# A hidden (unreadable) view is not found, as in the listing.
|
||||
return _view(row) if row and _decode_object(row[3]) is not None else None
|
||||
|
||||
def delete_saved_view(self, view_id: int) -> bool:
|
||||
"""True when a view was removed."""
|
||||
with self.model.get_connection() as conn:
|
||||
cursor = conn.execute("DELETE FROM saved_views WHERE id = ?", (view_id,))
|
||||
return cursor.rowcount > 0
|
||||
+30
-4
@@ -3,6 +3,9 @@
|
||||
A count is the number of eligible Library parents that carry the value
|
||||
directly or through one active saved pair, which is exactly what
|
||||
``list_library(tags=[name])`` / ``list_library(category=value)`` return.
|
||||
A tag row's ``image_count`` is the number of files (private locators) with any
|
||||
byte record carrying the tag (#249), so a file counts once however many of its
|
||||
byte versions carry it, and equal bytes at two paths count twice.
|
||||
|
||||
Tag commands run in one ``_pair_transaction``: the storage epoch is checked
|
||||
first, then the exact ``{id, name}`` identity, and the receipt is read on the
|
||||
@@ -26,6 +29,11 @@ except ImportError:
|
||||
from utils.validators import validate_category, validate_tags
|
||||
|
||||
_CATEGORY_DOMAIN = "prompt-manager-vocabulary-category:v1\n"
|
||||
# Files carrying a tag through any of their byte records (#249 image tags).
|
||||
_IMAGE_FILES = (
|
||||
"SELECT mit.tag_id,COUNT(DISTINCT b.file_id) FROM media_image_tags mit "
|
||||
"JOIN media_annotation_bytes b ON b.id=mit.byte_id"
|
||||
)
|
||||
|
||||
|
||||
def _tag_rows(page: str) -> str:
|
||||
@@ -40,9 +48,13 @@ def _tag_rows(page: str) -> str:
|
||||
"WHERE pt.tag_id IN (SELECT id FROM page) AND " + _ELIGIBLE + " "
|
||||
"UNION SELECT ppt.tag_id,pp.positive_prompt_id FROM prompt_pairs pp "
|
||||
"JOIN prompt_pair_tags ppt ON ppt.pair_id=pp.id "
|
||||
"WHERE pp.archived=0 AND ppt.tag_id IN (SELECT id FROM page)) "
|
||||
"SELECT page.id,page.name,COUNT(DISTINCT owners.pid) AS prompt_count "
|
||||
"WHERE pp.archived=0 AND ppt.tag_id IN (SELECT id FROM page)),"
|
||||
"images(tag_id,n) AS (" + _IMAGE_FILES + " "
|
||||
"WHERE mit.tag_id IN (SELECT id FROM page) GROUP BY mit.tag_id) "
|
||||
"SELECT page.id,page.name,COUNT(DISTINCT owners.pid) AS prompt_count,"
|
||||
"COALESCE(MAX(images.n),0) AS image_count "
|
||||
"FROM page LEFT JOIN owners ON owners.tag_id=page.id "
|
||||
"LEFT JOIN images ON images.tag_id=page.id "
|
||||
"GROUP BY page.id ORDER BY page.id"
|
||||
)
|
||||
|
||||
@@ -61,7 +73,10 @@ _TAG_IMPACT = (
|
||||
"AS negative_variants,"
|
||||
"(SELECT COUNT(*) FROM prompt_pairs pp JOIN prompt_pair_tags ppt "
|
||||
"ON ppt.pair_id=pp.id WHERE ppt.tag_id=:tag AND pp.archived!=0) "
|
||||
"AS archived_negative_variants"
|
||||
"AS archived_negative_variants,"
|
||||
"(SELECT COUNT(DISTINCT b.file_id) FROM media_image_tags mit "
|
||||
"JOIN media_annotation_bytes b ON b.id=mit.byte_id WHERE mit.tag_id=:tag) "
|
||||
"AS images"
|
||||
)
|
||||
_MAX_ID = 2**63 - 1
|
||||
|
||||
@@ -205,7 +220,16 @@ class VocabularyOperations:
|
||||
"SELECT COUNT(DISTINCT pair_id) FROM prompt_pair_tags WHERE tag_id=?",
|
||||
(tag_id,),
|
||||
).fetchone()[0]
|
||||
return {"components_changed": components, "pairs_changed": pairs}
|
||||
images = conn.execute(
|
||||
"SELECT COUNT(DISTINCT b.file_id) FROM media_image_tags mit "
|
||||
"JOIN media_annotation_bytes b ON b.id=mit.byte_id WHERE mit.tag_id=?",
|
||||
(tag_id,),
|
||||
).fetchone()[0]
|
||||
return {
|
||||
"components_changed": components,
|
||||
"pairs_changed": pairs,
|
||||
"images_changed": images,
|
||||
}
|
||||
|
||||
def create_vocabulary_tag(self, name: str, *, expected_epoch: str) -> dict:
|
||||
"""Insert an unused registry row, or reuse the exact existing name."""
|
||||
@@ -250,6 +274,7 @@ class VocabularyOperations:
|
||||
"SELECT id FROM tags WHERE name=?", (new_name,)
|
||||
).fetchone()
|
||||
if target is None:
|
||||
self._bump_image_tag_bytes(conn, tag_id)
|
||||
conn.execute("UPDATE tags SET name=? WHERE id=?", (new_name, tag_id))
|
||||
result_id = tag_id
|
||||
else:
|
||||
@@ -275,6 +300,7 @@ class VocabularyOperations:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
self._vocabulary_identity(conn, epoch, tag_id, name)
|
||||
counts = self._tag_owner_counts(conn, tag_id)
|
||||
self._bump_image_tag_bytes(conn, tag_id)
|
||||
conn.execute("DELETE FROM tags WHERE id=?", (tag_id,))
|
||||
return {"library_epoch": epoch, "deleted": True, **counts}
|
||||
|
||||
|
||||
@@ -29,6 +29,12 @@ Each role has `status` (`resolved`, `absent`, `unresolved`, or `ambiguous`), `te
|
||||
(string only when resolved, otherwise null), and `source`. Inspection does not
|
||||
index files, alter saved prompts, run scans, or decode image pixels. PNG text and
|
||||
video probe output are bounded; unsupported or malformed evidence remains
|
||||
unresolved rather than becoming sendable placeholder text.
|
||||
unresolved rather than becoming sendable placeholder text. Videos are read from
|
||||
one container carrier: the per-key tags written by core SaveVideo/SaveWEBM and
|
||||
current VideoHelperSuite (`prompt`, `workflow`, `parameters`,
|
||||
`comfyassets_prompt_pairs`), else the old VideoHelperSuite `comment` envelope,
|
||||
else `description`. The admin scan reads the same carrier, so Images and
|
||||
ingestion agree on a video's prompts (see TESTING.md for the order and budgets;
|
||||
the live monitor processes images only).
|
||||
|
||||
Python route changes require restarting ComfyUI. Browser assets require a refresh.
|
||||
|
||||
+13
-2
@@ -6,6 +6,7 @@ Split into domain-specific mixins for maintainability:
|
||||
- AdminRoutesMixin: duplicates, stats, settings, diagnostics, maintenance, backup, scan
|
||||
- LoggingRoutesMixin: log management endpoints
|
||||
- AutotagRoutesMixin: auto-tagging model management and tagging
|
||||
- JobRoutesMixin: the session job registry behind Activity (py/api/jobs.py)
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
@@ -25,9 +26,12 @@ from .prompts import PromptRoutesMixin
|
||||
from .pairs import PairRoutesMixin
|
||||
from .library import LibraryRoutesMixin
|
||||
from .vocabulary import VocabularyRoutesMixin
|
||||
from .saved_views import SavedViewRoutesMixin
|
||||
from .history import HistoryRoutesMixin
|
||||
from .images import ImageRoutesMixin
|
||||
from .admin import AdminRoutesMixin
|
||||
from .job_routes import JobRoutesMixin
|
||||
from .jobs import JobRegistry
|
||||
from .logging_routes import LoggingRoutesMixin
|
||||
from .autotag_routes import AutotagRoutesMixin
|
||||
from .lora_integration import LoraIntegrationMixin
|
||||
@@ -248,10 +252,12 @@ class PromptManagerAPI(
|
||||
PairRoutesMixin,
|
||||
LibraryRoutesMixin,
|
||||
VocabularyRoutesMixin,
|
||||
SavedViewRoutesMixin,
|
||||
HistoryRoutesMixin,
|
||||
PromptRoutesMixin,
|
||||
ImageRoutesMixin,
|
||||
AdminRoutesMixin,
|
||||
JobRoutesMixin,
|
||||
LoggingRoutesMixin,
|
||||
AutotagRoutesMixin,
|
||||
LoraIntegrationMixin,
|
||||
@@ -297,9 +303,12 @@ class PromptManagerAPI(
|
||||
# _acquire_long_job(). Created lazily so no lock binds to an event
|
||||
# loop before the server's loop exists.
|
||||
self._long_jobs: dict = {}
|
||||
# The current output scan (py/api/scan_job.ScanJob), if one was started.
|
||||
# This session's jobs (py/api/jobs.py): the metadata scan, and later
|
||||
# every other long operation the Activity page tracks.
|
||||
self._jobs = JobRegistry(self.logger)
|
||||
# The latest metadata scan job (also in self._jobs), if one was started.
|
||||
self._scan_job = None
|
||||
# The current duplicate scan (also a ScanJob), if one was started.
|
||||
# The current duplicate scan (a py/api/jobs.Job outside the registry).
|
||||
self._duplicate_job = None
|
||||
|
||||
# Run cleanup on initialization to remove any existing duplicates
|
||||
@@ -432,10 +441,12 @@ class PromptManagerAPI(
|
||||
self._register_pair_routes(routes)
|
||||
self._register_library_routes(routes)
|
||||
self._register_vocabulary_routes(routes)
|
||||
self._register_saved_view_routes(routes)
|
||||
self._register_history_routes(routes)
|
||||
self._register_prompt_routes(routes)
|
||||
self._register_image_routes(routes)
|
||||
self._register_admin_routes(routes)
|
||||
self._register_job_routes(routes)
|
||||
self._register_logging_routes(routes)
|
||||
self._register_autotag_routes(routes)
|
||||
self._register_lora_routes(routes)
|
||||
|
||||
+305
-53
@@ -6,16 +6,25 @@ from contextlib import closing
|
||||
import datetime
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .prompts import thumbnail_url_for
|
||||
from .scan_job import ScanJob
|
||||
from .jobs import (
|
||||
DEFAULT_SCAN_SCOPE,
|
||||
SCAN_GROUP_NAMES,
|
||||
SCAN_GROUPS,
|
||||
SCAN_SCOPES,
|
||||
Job,
|
||||
JobBusy,
|
||||
)
|
||||
|
||||
try:
|
||||
from ...utils.parallel import map_parallel
|
||||
@@ -47,8 +56,10 @@ except ImportError:
|
||||
|
||||
|
||||
try:
|
||||
from ...database.library import display_title
|
||||
from ...utils.hashing import generate_prompt_hash
|
||||
except ImportError:
|
||||
from database.library import display_title
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
IMAGE_EXTENSIONS = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tiff")
|
||||
@@ -138,6 +149,127 @@ def _sse(payload):
|
||||
return f"data: {json.dumps(payload)}\n\n"
|
||||
|
||||
|
||||
SCAN_JOB_KIND = "scan"
|
||||
SCAN_JOB_TITLE = "Scan for metadata"
|
||||
# The legacy final event type of a scan and the job state it ends in.
|
||||
SCAN_FINAL_STATES = {
|
||||
"complete": "finished",
|
||||
"cancelled": "cancelled",
|
||||
"error": "failed",
|
||||
}
|
||||
# The scope as the scan log's first line names it (the review dialog's words).
|
||||
SCAN_SCOPE_LOG = {"new": "new and unresolved files", "all": "all files"}
|
||||
MIN_BATCHES_FOR_ESTIMATE = 2 # "about X min left" waits for this many batches
|
||||
# Log line prefix per outcome group (spec: + added, = known, ? review, ! error,
|
||||
# - no usable metadata).
|
||||
SCAN_LOG_PREFIXES = {
|
||||
"added": "+",
|
||||
"known": "=",
|
||||
"review": "?",
|
||||
"errors": "!",
|
||||
"no_metadata": "-",
|
||||
}
|
||||
# What an outcome row says about a file: where it went (with the prompt's
|
||||
# title) or why it was skipped. "error" is a file whose ingestion raised;
|
||||
# None is a file without prompt metadata.
|
||||
SCAN_TITLED_DETAILS = {
|
||||
"added": "Added to the Library as “{title}”",
|
||||
"linked": "Linked to “{title}”",
|
||||
"enriched": "Metadata added to “{title}”",
|
||||
"unchanged": "Already linked to “{title}”",
|
||||
}
|
||||
SCAN_DETAILS = {
|
||||
"added": "Added to the Library",
|
||||
"linked": "Linked to an existing prompt",
|
||||
"enriched": "Metadata added to its prompt",
|
||||
"unchanged": "Already linked to its prompt",
|
||||
"skipped": "Skipped · already linked",
|
||||
"ambiguous": "Needs review: more than one prompt could match",
|
||||
"unresolved": "Needs review: the prompt could not be traced in the workflow",
|
||||
"conflict": "Needs review: already linked to a different prompt",
|
||||
"failed": "Error: the prompt could not be stored",
|
||||
"error": "Error: the file could not be processed (see the server log)",
|
||||
"found": "Metadata found, but no prompt text to store",
|
||||
None: "No prompt metadata in the file",
|
||||
}
|
||||
|
||||
|
||||
def _plural(count, word):
|
||||
return f"{count} {word}" if count == 1 else f"{count} {word}s"
|
||||
|
||||
|
||||
def _scan_status_line(done, total, batches, elapsed):
|
||||
""" "Processing file N of M", with an estimate once enough batches ran."""
|
||||
line = f"Processing file {done:,} of {total:,}"
|
||||
if batches < MIN_BATCHES_FOR_ESTIMATE or done <= 0 or done >= total:
|
||||
return line
|
||||
minutes = max(1, math.ceil(elapsed / done * (total - done) / 60))
|
||||
return f"{line} · about {minutes} min left"
|
||||
|
||||
|
||||
def _file_size(path):
|
||||
"""``path``'s size on disk, or None when it cannot be read."""
|
||||
try:
|
||||
return os.path.getsize(path)
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def _with_file_size(canonical, media_file):
|
||||
"""Record the file's size in ``file_info`` unless the metadata carries one.
|
||||
|
||||
Stored as ``generated_images.file_size``; the "new" scan scope skips a known
|
||||
file only while its size is unchanged.
|
||||
"""
|
||||
info = canonical.get("file_info")
|
||||
info = dict(info) if isinstance(info, dict) else {}
|
||||
if info.get("size") is None:
|
||||
size = _file_size(media_file)
|
||||
if size is not None:
|
||||
info["size"] = size
|
||||
if info:
|
||||
canonical["file_info"] = info
|
||||
|
||||
|
||||
def _scan_roots(output_dirs):
|
||||
"""(absolute root, public label, listing index), the most specific first.
|
||||
|
||||
The index is the root's place in ``output_dirs`` (``_get_all_output_dirs``),
|
||||
the ``root_index`` the workspace listing names it by.
|
||||
"""
|
||||
public_path, _ = _public_helpers()
|
||||
roots = [
|
||||
(os.path.abspath(str(d)), public_path(d), index)
|
||||
for index, d in enumerate(output_dirs)
|
||||
]
|
||||
return sorted(roots, key=lambda root: len(root[0]), reverse=True)
|
||||
|
||||
|
||||
def _scan_location(media_file, roots):
|
||||
"""``media_file`` relative to its output root, that root's label and index."""
|
||||
full = os.path.abspath(str(media_file))
|
||||
folded = os.path.normcase(full)
|
||||
for root, label, index in roots:
|
||||
base = os.path.normcase(root).rstrip("\\/") + os.sep
|
||||
if folded.startswith(base):
|
||||
return Path(os.path.relpath(full, root)).as_posix(), label, index
|
||||
return Path(full).name, "", None
|
||||
|
||||
|
||||
def _scan_outcome(media_file, roots, status, title=None):
|
||||
"""The (group, row, log line) one scanned file records in its job."""
|
||||
if status in SCAN_GROUPS:
|
||||
group = SCAN_GROUPS[status]
|
||||
else:
|
||||
group = "errors" if status == "error" else "no_metadata"
|
||||
template = SCAN_TITLED_DETAILS.get(status) if title else None
|
||||
detail = template.format(title=title) if template else SCAN_DETAILS[status]
|
||||
path, label, index = _scan_location(media_file, roots)
|
||||
row = {"path": path, "root": label, "root_index": index, "detail": detail}
|
||||
where = f"{label}/{path}" if label else path
|
||||
return group, row, f"{SCAN_LOG_PREFIXES[group]} {where} · {detail}"
|
||||
|
||||
|
||||
def _inspect_restore_upload(path):
|
||||
"""Checks on a restore upload beyond PromptModel.verify_database_file.
|
||||
|
||||
@@ -437,7 +569,7 @@ class AdminRoutesMixin:
|
||||
guard = self._acquire_long_job(self.LONG_JOB_DUPLICATES)
|
||||
if guard is None:
|
||||
return self._long_job_busy_response("Duplicate scan or delete")
|
||||
job = ScanJob()
|
||||
job = Job("duplicates", "Duplicate scan")
|
||||
self._duplicate_job = job
|
||||
job.task = asyncio.ensure_future(self._run_duplicate_job(job, guard))
|
||||
return await self._stream_scan_job(request, job)
|
||||
@@ -1319,14 +1451,18 @@ class AdminRoutesMixin:
|
||||
async def scan_images(self, request):
|
||||
"""Scan ComfyUI output images for prompt metadata and add them to the database.
|
||||
|
||||
The scan runs as a background task that outlives this request, so a
|
||||
closed tab never leaves a half-finished scan. Progress streams as
|
||||
server-sent events; while a scan is running, a new request attaches
|
||||
to it instead of starting another one.
|
||||
The scan runs as the registry's ``scan`` job, which outlives this
|
||||
request, so a closed tab never leaves a half-finished scan. Progress
|
||||
streams as the legacy server-sent events; while a scan is running
|
||||
(started here or from the workspace), a new request attaches to it
|
||||
instead of starting another one.
|
||||
"""
|
||||
job = self._scan_job
|
||||
if job is None or job.finished:
|
||||
job = self._start_scan_job()
|
||||
job = self._jobs.active(SCAN_JOB_KIND)
|
||||
if job is None:
|
||||
try:
|
||||
job = self._start_scan_job("all")
|
||||
except JobBusy as busy:
|
||||
return self._long_job_busy_response(busy.running.title)
|
||||
return await self._stream_scan_job(request, job)
|
||||
|
||||
async def cancel_scan(self, request):
|
||||
@@ -1348,19 +1484,37 @@ class AdminRoutesMixin:
|
||||
}
|
||||
)
|
||||
|
||||
def _start_scan_job(self):
|
||||
"""Create the job record synchronously, then run the scan as a task."""
|
||||
job = ScanJob()
|
||||
def _start_scan_job(self, files=DEFAULT_SCAN_SCOPE):
|
||||
"""Register the scan job synchronously; its runner starts on the loop.
|
||||
|
||||
``files`` is the scope (``SCAN_SCOPES``). Raises JobBusy while another
|
||||
record-changing job runs.
|
||||
"""
|
||||
if files not in SCAN_SCOPES:
|
||||
raise ValueError(f"Unknown scan scope: {files!r}")
|
||||
job = self._jobs.start(
|
||||
SCAN_JOB_KIND,
|
||||
SCAN_JOB_TITLE,
|
||||
record_changing=True,
|
||||
scope={"files": files},
|
||||
runner=self._run_scan_job,
|
||||
groups=SCAN_GROUP_NAMES,
|
||||
)
|
||||
self._scan_job = job
|
||||
job.task = asyncio.ensure_future(self._run_scan_job(job))
|
||||
return job
|
||||
|
||||
async def _run_scan_job(self, job):
|
||||
try:
|
||||
async for payload in self._scan_images_events():
|
||||
job.publish(payload)
|
||||
finally:
|
||||
job.finish()
|
||||
final = None
|
||||
async for payload in self._scan_images_events(job):
|
||||
job.publish(payload)
|
||||
final = payload.get("type")
|
||||
state = SCAN_FINAL_STATES.get(final, "finished")
|
||||
if state == "failed":
|
||||
job.log_line(f"Scan failed · {job.status_line}")
|
||||
else:
|
||||
verb = "cancelled" if state == "cancelled" else "finished"
|
||||
job.log_line(f"Scan {verb} · {_plural(job.processed, 'file')} processed")
|
||||
job.finish(state)
|
||||
|
||||
async def _stream_scan_job(self, request, job):
|
||||
"""Stream ``job`` as SSE; stop quietly if the client goes away."""
|
||||
@@ -1386,21 +1540,31 @@ class AdminRoutesMixin:
|
||||
)
|
||||
return response
|
||||
|
||||
async def _scan_images_events(self):
|
||||
"""Yield progress payloads while scanning every configured output directory."""
|
||||
async def _scan_images_events(self, job):
|
||||
"""Yield progress payloads while scanning every configured output directory.
|
||||
|
||||
The payloads are the legacy SSE events, unchanged; ``job`` also gets
|
||||
its status line, totals and one outcome per processed file.
|
||||
"""
|
||||
try:
|
||||
self.logger.info("Starting image scan operation")
|
||||
output_dirs = self._get_all_output_dirs()
|
||||
if not output_dirs:
|
||||
self.logger.error("No output directories found")
|
||||
yield {
|
||||
"type": "error",
|
||||
"message": "No output directories found. "
|
||||
"Configure scan directories in Settings.",
|
||||
}
|
||||
message = (
|
||||
"No output directories found. "
|
||||
"Configure scan directories in Settings."
|
||||
)
|
||||
job.update(status_line=message)
|
||||
yield {"type": "error", "message": message}
|
||||
return
|
||||
|
||||
dir_names = [self._public_path(d) for d in output_dirs]
|
||||
folders = _plural(len(output_dirs), "output folder")
|
||||
job.scope["output_folders"] = len(output_dirs)
|
||||
scope = SCAN_SCOPE_LOG[job.scope.get("files", "all")]
|
||||
job.log_line(f"Scan started · {folders} · {scope}")
|
||||
job.update(status_line=f"Looking for media files in {folders}")
|
||||
yield self._scan_progress(
|
||||
0, f"Scanning {len(output_dirs)} directory(ies) for media files..."
|
||||
)
|
||||
@@ -1409,11 +1573,21 @@ class AdminRoutesMixin:
|
||||
self._collect_output_media_sync, output_dirs
|
||||
)
|
||||
counts = dict.fromkeys(SCAN_COUNT_KEYS, 0)
|
||||
job.update(
|
||||
total=len(media_files),
|
||||
status_line=_scan_status_line(0, len(media_files), 0, 0),
|
||||
)
|
||||
if media_files:
|
||||
yield self._scan_progress(
|
||||
5, f"Found {len(media_files)} media files to process...", counts
|
||||
)
|
||||
async for event in self._scan_batches(media_files, counts):
|
||||
roots = _scan_roots(output_dirs)
|
||||
known = set()
|
||||
if media_files and job.scope.get("files") != "all":
|
||||
known = await self._known_scan_files(job, media_files)
|
||||
async for event in self._scan_batches(
|
||||
media_files, counts, job, roots, known
|
||||
):
|
||||
yield event
|
||||
|
||||
self.logger.info(
|
||||
@@ -1421,37 +1595,78 @@ class AdminRoutesMixin:
|
||||
f"found={counts['found']}, new_prompts_added={counts['added']}, "
|
||||
f"images_linked_to_existing={counts['linked']}"
|
||||
)
|
||||
job = self._scan_job
|
||||
final_type = "cancelled" if job and job.cancelled else "complete"
|
||||
final_type = "cancelled" if job.cancelled else "complete"
|
||||
if final_type == "complete":
|
||||
job.update(progress=100)
|
||||
yield {"type": final_type, **counts, "directories": dir_names}
|
||||
|
||||
except Exception:
|
||||
self.logger.exception("Scan error")
|
||||
job.update(status_line="The scan stopped on an internal error.")
|
||||
yield {
|
||||
"type": "error",
|
||||
"message": "An internal error occurred. Check server logs for details.",
|
||||
}
|
||||
|
||||
async def _scan_batches(self, media_files, counts):
|
||||
async def _known_scan_files(self, job, media_files):
|
||||
"""The "new" scope's skip set; a failed read falls back to reading all."""
|
||||
try:
|
||||
return await self._run_in_executor(self._known_scan_files_sync, media_files)
|
||||
except Exception:
|
||||
self.logger.warning(
|
||||
"Known files could not be read; the scan reads every file",
|
||||
exc_info=True,
|
||||
)
|
||||
job.log_line("Known files could not be read · reading every file")
|
||||
return set()
|
||||
|
||||
def _known_scan_files_sync(self, media_files):
|
||||
"""Stat each scanned file and ask the database which are known (blocking)."""
|
||||
sizes = {str(media_file): _file_size(media_file) for media_file in media_files}
|
||||
return self.db.get_known_resolved_paths(sizes)
|
||||
|
||||
async def _scan_batches(self, media_files, counts, job, roots=(), known=()):
|
||||
"""Process ``media_files`` in executor batches, updating ``counts`` in place.
|
||||
|
||||
Yields one progress event per batch.
|
||||
Records each file's outcome in ``job`` and yields one progress event
|
||||
per batch. Files in ``known`` (path strings) are not read: they count
|
||||
as processed and unchanged and are recorded as skipped.
|
||||
"""
|
||||
total = len(media_files)
|
||||
started = time.monotonic()
|
||||
batches = 0
|
||||
for batch_start in range(0, total, SCAN_BATCH_SIZE):
|
||||
if self._scan_job and self._scan_job.cancelled:
|
||||
if job.cancelled:
|
||||
return
|
||||
batch = media_files[batch_start : batch_start + SCAN_BATCH_SIZE]
|
||||
batch_results = await self._run_in_executor(
|
||||
self._extract_batch_metadata_sync, batch
|
||||
)
|
||||
batch_counts = await self._run_in_executor(
|
||||
self._ingest_scan_batch_sync, batch_results
|
||||
)
|
||||
for key in counts:
|
||||
counts[key] += batch_counts[key]
|
||||
skipped = [f for f in batch if str(f) in known]
|
||||
to_read = [f for f in batch if str(f) not in known]
|
||||
outcomes = [_scan_outcome(f, roots, "skipped") for f in skipped]
|
||||
counts["processed"] += len(skipped)
|
||||
counts["unchanged"] += len(skipped)
|
||||
if to_read:
|
||||
batch_results = await self._run_in_executor(
|
||||
self._extract_batch_metadata_sync, to_read
|
||||
)
|
||||
batch_counts = await self._run_in_executor(
|
||||
self._ingest_scan_batch_sync,
|
||||
batch_results,
|
||||
roots=roots,
|
||||
outcomes=outcomes,
|
||||
)
|
||||
for key in counts:
|
||||
counts[key] += batch_counts[key]
|
||||
for group, row, log_line in outcomes:
|
||||
job.record(group, row, log_line)
|
||||
|
||||
batches += 1
|
||||
done = min(batch_start + len(batch), total)
|
||||
job.update(
|
||||
processed=counts["processed"],
|
||||
status_line=_scan_status_line(
|
||||
done, total, batches, time.monotonic() - started
|
||||
),
|
||||
)
|
||||
yield self._scan_progress(
|
||||
int(done / total * 100), f"Processing file {done}/{total}...", counts
|
||||
)
|
||||
@@ -1490,9 +1705,19 @@ class AdminRoutesMixin:
|
||||
|
||||
return map_parallel(extract, file_batch, PromptManagerConfig.WORKER_THREADS)
|
||||
|
||||
def _ingest_scan_batch_sync(self, batch_results):
|
||||
"""Store the prompts found in a metadata batch (blocking DB work)."""
|
||||
def _ingest_scan_batch_sync(self, batch_results, roots=(), outcomes=None):
|
||||
"""Store the prompts found in a metadata batch (blocking DB work).
|
||||
|
||||
Returns the legacy counts. When ``outcomes`` is a list, every processed
|
||||
file appends its ``(group, row, log_line)`` for the scan job, with the
|
||||
path relative to its root in ``roots`` (see ``_scan_roots``).
|
||||
"""
|
||||
counts = dict.fromkeys(SCAN_COUNT_KEYS, 0)
|
||||
|
||||
def note(media_file, status, title=None):
|
||||
if outcomes is not None:
|
||||
outcomes.append(_scan_outcome(media_file, roots, status, title))
|
||||
|
||||
for media_file, metadata in batch_results:
|
||||
if self._scan_job and self._scan_job.cancelled:
|
||||
break
|
||||
@@ -1502,25 +1727,21 @@ class AdminRoutesMixin:
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error processing {media_file.name}: {e}")
|
||||
counts["failed"] += 1
|
||||
note(media_file, "error")
|
||||
continue
|
||||
if outcome is None:
|
||||
note(media_file, None)
|
||||
continue
|
||||
counts["found"] += 1
|
||||
status = outcome["status"] if isinstance(outcome, dict) else outcome
|
||||
if status in (
|
||||
"added",
|
||||
"linked",
|
||||
"enriched",
|
||||
"unchanged",
|
||||
"ambiguous",
|
||||
"unresolved",
|
||||
"conflict",
|
||||
"failed",
|
||||
):
|
||||
if status in SCAN_GROUPS:
|
||||
counts[status] += 1
|
||||
if isinstance(outcome, dict):
|
||||
counts["pairs_added"] += outcome.get("pairs_added", 0)
|
||||
counts["pair_links_added"] += outcome.get("links_added", 0)
|
||||
note(media_file, status, outcome.get("title"))
|
||||
else:
|
||||
note(media_file, status)
|
||||
return counts
|
||||
|
||||
@staticmethod
|
||||
@@ -1539,7 +1760,13 @@ class AdminRoutesMixin:
|
||||
return positive.strip() or None
|
||||
|
||||
def _ingest_scanned_file(self, media_file, metadata, detailed=False):
|
||||
"""Enrich exact pairs; retain legacy positive-only import for absent pairs."""
|
||||
"""Enrich exact pairs; retain legacy positive-only import for absent pairs.
|
||||
|
||||
Returns the file's scan status, or None without prompt metadata. With
|
||||
``detailed``, a stored or linked file returns a dict instead: its
|
||||
``status``, the prompt's display ``title`` and, for a resolved pair,
|
||||
the pair counts.
|
||||
"""
|
||||
if not metadata:
|
||||
return None
|
||||
canonical = dict(metadata)
|
||||
@@ -1552,12 +1779,18 @@ class AdminRoutesMixin:
|
||||
pass
|
||||
if not canonical.get("prompt") and not canonical.get("parameters"):
|
||||
return None
|
||||
_with_file_size(canonical, media_file)
|
||||
resolution = resolve_prompt_pair(canonical)
|
||||
if resolution["state"] == "resolved":
|
||||
result = ingest_resolved_pair(
|
||||
self.db, str(media_file), canonical, resolution
|
||||
)
|
||||
return result if detailed else result["status"]
|
||||
if not detailed:
|
||||
return result["status"]
|
||||
return {
|
||||
**result,
|
||||
"title": self._scan_title(media_file, resolution["positive"]["text"]),
|
||||
}
|
||||
if canonical.get("prompt") and resolution["state"] in (
|
||||
"ambiguous",
|
||||
"unresolved",
|
||||
@@ -1585,6 +1818,25 @@ class AdminRoutesMixin:
|
||||
return "found"
|
||||
prompt_text = prompt_text.strip()
|
||||
|
||||
status = self._store_scanned_prompt(media_file, canonical, prompt_text)
|
||||
if not detailed or status not in SCAN_TITLED_DETAILS:
|
||||
return status
|
||||
return {"status": status, "title": self._scan_title(media_file, prompt_text)}
|
||||
|
||||
def _scan_title(self, media_file, text):
|
||||
"""The linked prompt's Library title (its stored title, else ``text``'s
|
||||
first line), as the Library and the inspector name it."""
|
||||
try:
|
||||
path = str(media_file)
|
||||
summary = self.db.get_image_link_summaries([path], False).get(path)
|
||||
if summary and summary.get("title"):
|
||||
return summary["title"]
|
||||
except Exception:
|
||||
self.logger.warning("Scan row title could not be read; using its text")
|
||||
return display_title((), text)
|
||||
|
||||
def _store_scanned_prompt(self, media_file, canonical, prompt_text):
|
||||
"""Link ``media_file`` to ``prompt_text``'s prompt, saving it when new."""
|
||||
prompt_hash = generate_prompt_hash(prompt_text)
|
||||
existing = self.db.get_prompt_by_hash(prompt_hash)
|
||||
if existing:
|
||||
|
||||
+49
-4
@@ -696,19 +696,25 @@ def _workspace_filters(query: Mapping) -> dict:
|
||||
``q`` (at most 500 characters) matches the relative path, case-folded, and
|
||||
the linked positive's text, title and tag names. ``category`` is exact,
|
||||
``tags`` is a comma list that must all be present, ``sort=rating`` ranks by
|
||||
the linked positive. Raises ValueError for any unknown or oversized value.
|
||||
the linked positive, and ``favorites=1`` keeps only favorites (the
|
||||
Favorites page, #249 D51). Raises ValueError for any unknown or oversized
|
||||
value.
|
||||
"""
|
||||
text = query.get("q", "").strip()
|
||||
folded = text.casefold()
|
||||
media = query.get("media", "all")
|
||||
order = query.get("sort", "newest")
|
||||
folder = query.get("subfolder", "").strip().replace("\\", "/")
|
||||
# ``path``: one exact relative path (Activity opens a scan row by it, D57).
|
||||
exact = query.get("path", "").strip().replace("\\", "/")
|
||||
category = query.get("category", "").strip() or None
|
||||
tags = sorted(
|
||||
{tag.strip() for tag in query.get("tags", "").split(",") if tag.strip()}
|
||||
)
|
||||
favorites = query.get("favorites", "0")
|
||||
if (
|
||||
len(text) > 500
|
||||
favorites not in ("0", "1")
|
||||
or len(text) > 500
|
||||
or len(folded) > 500
|
||||
or media not in ("all", "image", "video")
|
||||
or order not in WORKSPACE_ORDERS
|
||||
@@ -719,14 +725,18 @@ def _workspace_filters(query: Mapping) -> dict:
|
||||
raise ValueError("Invalid gallery filters")
|
||||
if folder and (_has_traversal(folder) or len(folder) > 1024):
|
||||
raise ValueError("Invalid gallery folder")
|
||||
if exact and (_has_traversal(exact) or len(exact) > 1024):
|
||||
raise ValueError("Invalid gallery path")
|
||||
return {
|
||||
"text": text,
|
||||
"query": folded,
|
||||
"media": media,
|
||||
"order": order,
|
||||
"folder": folder,
|
||||
"path": exact.casefold(),
|
||||
"category": category,
|
||||
"tags": tags,
|
||||
"favorites": favorites == "1",
|
||||
}
|
||||
|
||||
|
||||
@@ -736,7 +746,12 @@ _COUNT_OMITTED = object()
|
||||
|
||||
|
||||
def _filtered_page(
|
||||
entries: list, start: int, size: int, filters: dict, links: dict | None
|
||||
entries: list,
|
||||
start: int,
|
||||
size: int,
|
||||
filters: dict,
|
||||
links: dict | None,
|
||||
favorites: frozenset | None = None,
|
||||
) -> tuple[int, list, int | None]:
|
||||
"""Count the *entries* the filters admit and take *size* from *start* on.
|
||||
|
||||
@@ -745,19 +760,25 @@ def _filtered_page(
|
||||
category or tag filter admits only those, and ``q`` admits a filename match
|
||||
or a linked-positive hit. Returns the total, the page and the number of
|
||||
distinct linked positives among all admitted entries (None without links).
|
||||
*favorites*, when given, admits only those listed paths (``favorites=1``).
|
||||
"""
|
||||
query, media, folder = filters["query"], filters["media"], filters["folder"]
|
||||
members_only = filters["category"] is not None or bool(filters["tags"])
|
||||
native_folder = folder.replace("/", os.sep)
|
||||
nested = native_folder + os.sep
|
||||
total, page, positives = 0, [], set()
|
||||
exact = filters.get("path", "")
|
||||
for position, entry in enumerate(entries):
|
||||
if exact and entry.folded != exact:
|
||||
continue
|
||||
if folder and not (
|
||||
entry.folder == native_folder or entry.folder.startswith(nested)
|
||||
):
|
||||
continue
|
||||
if media != "all" and (media == "video") != entry.video:
|
||||
continue
|
||||
if favorites is not None and entry.path not in favorites:
|
||||
continue
|
||||
match = None if links is None else links.get(entry.normalized)
|
||||
if members_only and match is None:
|
||||
continue
|
||||
@@ -1142,10 +1163,12 @@ class ImageRoutesMixin:
|
||||
filters["media"],
|
||||
filters["order"],
|
||||
filters["folder"],
|
||||
filters["path"],
|
||||
root_index,
|
||||
[str(root) for root in output_dirs],
|
||||
filters["category"],
|
||||
filters["tags"],
|
||||
filters["favorites"],
|
||||
]
|
||||
).encode()
|
||||
).hexdigest()
|
||||
@@ -1333,8 +1356,11 @@ class ImageRoutesMixin:
|
||||
start = len(entries) - bisect_left(ascending, after, key=key)
|
||||
else:
|
||||
start = bisect_right(entries, after, key=key)
|
||||
favorites = (
|
||||
self._workspace_favorite_paths(listings) if filters["favorites"] else None
|
||||
)
|
||||
total, page, positive_count = _filtered_page(
|
||||
entries, start, limit + 1, filters, links
|
||||
entries, start, limit + 1, filters, links, favorites
|
||||
)
|
||||
if not needed and after is not None:
|
||||
positive_count = _COUNT_OMITTED
|
||||
@@ -2347,6 +2373,25 @@ class ImageRoutesMixin:
|
||||
self._favorite_listing_cache[cache_key] = (files, listing)
|
||||
return listing
|
||||
|
||||
def _workspace_favorite_paths(self, listings: list[tuple]) -> frozenset:
|
||||
"""The listed paths of every favorite in the selected roots (blocking).
|
||||
|
||||
One primary-key read per root (get_root_favorites), matched against the
|
||||
root's cached ``{relative key: listing spelling}``, so the work follows
|
||||
the favorites rather than the files, and a key folds case exactly as
|
||||
the toggle stored it (Windows). A favorite whose file is gone matches
|
||||
nothing. A failed read raises: an unfiltered page must never pass for
|
||||
the favorites.
|
||||
"""
|
||||
paths = set()
|
||||
for _index, root, files in listings:
|
||||
listing = self._favorite_listing(root, files)
|
||||
for key in self.db.get_root_favorites(_favorite_root_key(root)):
|
||||
listed = listing.get(key)
|
||||
if listed is not None:
|
||||
paths.add(root / listed)
|
||||
return frozenset(paths)
|
||||
|
||||
def _workspace_favorites(self, page: list[tuple]) -> set[tuple[int, str]]:
|
||||
"""``(root index, listing path)`` of the favorites among *page* entries."""
|
||||
by_root: dict[tuple[int, Path], dict[str, str]] = {}
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
"""Session job routes (Activity): list, inspect, follow and cancel jobs.
|
||||
|
||||
The registry itself is ``py/api/jobs.py``; the scan job's runner is in
|
||||
``py/api/admin.py``. Every response leaves out absolute server paths: job
|
||||
rows already carry root-relative paths and public root labels.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .jobs import DEFAULT_SCAN_SCOPE, SCAN_SCOPES, JobBusy
|
||||
|
||||
|
||||
def _not_found():
|
||||
return web.json_response({"success": False, "error": "Job not found"}, status=404)
|
||||
|
||||
|
||||
def _bad_request(message):
|
||||
return web.json_response({"success": False, "error": message}, status=400)
|
||||
|
||||
|
||||
async def _scan_scope(request):
|
||||
"""The requested scan scope ("new" or "all"), or None when the body is invalid.
|
||||
|
||||
An empty body (also a chunked one with no bytes), or one without
|
||||
``scope``, asks for the default.
|
||||
"""
|
||||
if not request.can_read_body:
|
||||
return DEFAULT_SCAN_SCOPE
|
||||
text = await request.text()
|
||||
if not text.strip(): # e.g. a chunked POST that sent no bytes
|
||||
return DEFAULT_SCAN_SCOPE
|
||||
try:
|
||||
body = json.loads(text)
|
||||
except ValueError:
|
||||
return None
|
||||
if not isinstance(body, dict):
|
||||
return None
|
||||
scope = body.get("scope", DEFAULT_SCAN_SCOPE)
|
||||
return scope if scope in SCAN_SCOPES else None
|
||||
|
||||
|
||||
def _summary_event(job):
|
||||
return f"data: {json.dumps(job.summary())}\n\n".encode("utf-8")
|
||||
|
||||
|
||||
class JobRoutesMixin:
|
||||
"""Mixin providing the /prompt_manager/jobs endpoints."""
|
||||
|
||||
def _register_job_routes(self, routes):
|
||||
@routes.get("/prompt_manager/jobs")
|
||||
async def list_jobs_route(request):
|
||||
return await self.list_jobs(request)
|
||||
|
||||
@routes.post("/prompt_manager/jobs/scan")
|
||||
async def start_scan_job_route(request):
|
||||
return await self.start_scan_job(request)
|
||||
|
||||
@routes.get("/prompt_manager/jobs/{job_id}")
|
||||
async def job_detail_route(request):
|
||||
return await self.job_detail(request)
|
||||
|
||||
@routes.get("/prompt_manager/jobs/{job_id}/report")
|
||||
async def job_report_route(request):
|
||||
return await self.job_report(request)
|
||||
|
||||
@routes.get("/prompt_manager/jobs/{job_id}/events")
|
||||
async def job_events_route(request):
|
||||
return await self.job_events(request)
|
||||
|
||||
@routes.post("/prompt_manager/jobs/{job_id}/cancel")
|
||||
async def cancel_job_route(request):
|
||||
return await self.cancel_job(request)
|
||||
|
||||
async def list_jobs(self, request):
|
||||
"""This session's jobs: running first, then finished, newest first."""
|
||||
jobs = [job.summary() for job in self._jobs.list()]
|
||||
return web.json_response({"success": True, "jobs": jobs})
|
||||
|
||||
async def job_detail(self, request):
|
||||
job = self._jobs.get(request.match_info["job_id"])
|
||||
if job is None:
|
||||
return _not_found()
|
||||
return web.json_response({"success": True, "job": job.detail()})
|
||||
|
||||
async def job_report(self, request):
|
||||
"""The job's whole record as a JSON attachment (no absolute paths)."""
|
||||
job = self._jobs.get(request.match_info["job_id"])
|
||||
if job is None:
|
||||
return _not_found()
|
||||
body = json.dumps(job.report(), indent=2)
|
||||
disposition = f'attachment; filename="{job.report_filename()}"'
|
||||
return web.Response(
|
||||
text=body,
|
||||
content_type="application/json",
|
||||
headers={
|
||||
"Content-Disposition": disposition,
|
||||
"Cache-Control": "no-store",
|
||||
},
|
||||
)
|
||||
|
||||
async def cancel_job(self, request):
|
||||
"""Stop a job at its next file boundary; a finished job is a no-op."""
|
||||
job_id = request.match_info["job_id"]
|
||||
if self._jobs.get(job_id) is None:
|
||||
return _not_found()
|
||||
requested = self._jobs.cancel(job_id)
|
||||
return web.json_response({"success": True, "cancellation_requested": requested})
|
||||
|
||||
async def start_scan_job(self, request):
|
||||
"""Start the metadata scan as a job (202), or 409 naming the busy job.
|
||||
|
||||
The body's ``scope`` is "new" (the default: new and unresolved files)
|
||||
or "all"; anything else is a 400.
|
||||
"""
|
||||
files = await _scan_scope(request)
|
||||
if files is None:
|
||||
return _bad_request('The scan scope must be "new" or "all"')
|
||||
try:
|
||||
job = self._start_scan_job(files)
|
||||
except JobBusy as busy:
|
||||
running = busy.running
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": "busy",
|
||||
"error": str(busy),
|
||||
"running": {"id": running.id, "title": running.title},
|
||||
},
|
||||
status=409,
|
||||
)
|
||||
return web.json_response({"success": True, "job": job.summary()}, status=202)
|
||||
|
||||
async def job_events(self, request):
|
||||
"""Stream the job's summary as SSE: now, after each event, and at the end."""
|
||||
job = self._jobs.get(request.match_info["job_id"])
|
||||
if job is None:
|
||||
return _not_found()
|
||||
response = web.StreamResponse(
|
||||
status=200,
|
||||
reason="OK",
|
||||
headers={
|
||||
"Content-Type": "text/event-stream",
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
},
|
||||
)
|
||||
await response.prepare(request)
|
||||
try:
|
||||
await response.write(_summary_event(job))
|
||||
async with contextlib.aclosing(job.events()) as events:
|
||||
async for _payload in events:
|
||||
await response.write(_summary_event(job))
|
||||
# The last event is published before the job takes its final state.
|
||||
await response.write(_summary_event(job))
|
||||
await response.write_eof()
|
||||
except Exception as e: # the socket closed under us; the job goes on
|
||||
self.logger.info(f"Job progress client disconnected ({e})")
|
||||
return response
|
||||
+359
@@ -0,0 +1,359 @@
|
||||
"""The session job registry: every long operation as one tracked job.
|
||||
|
||||
A job runs as an asyncio task that outlives the HTTP request that started it,
|
||||
so an accidental tab close never abandons half-finished work. Each job keeps
|
||||
its counters, a capped list of file outcomes per outcome group and a capped
|
||||
log, and fans its progress out to SSE subscribers: the request that started
|
||||
it, a reconnecting page, a second tab. A subscriber receives the latest event
|
||||
as a snapshot and then every event published after it, until the job ends.
|
||||
|
||||
Jobs live only in server memory and are gone after a restart. The registry
|
||||
keeps the running jobs and this session's last ``MAX_FINISHED_JOBS`` finished
|
||||
ones. Record-changing jobs (the metadata scan, and later maintenance, cleanup
|
||||
and restore) never run together, and one job of a kind runs at a time.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import itertools
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import PurePosixPath, PureWindowsPath
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
# Start and finish order: wall-clock times can tie (coarse Windows clocks).
|
||||
_sequence = itertools.count()
|
||||
|
||||
JOB_STATES = ("running", "finished", "cancelled", "failed")
|
||||
FINAL_STATES = JOB_STATES[1:]
|
||||
MAX_OUTCOME_ROWS = 200 # rows listed per outcome group; counters stay exact
|
||||
MAX_LOG_LINES = 2000
|
||||
LOG_CAP_LINE = "[Log capped at 2,000 lines]"
|
||||
MAX_FINISHED_JOBS = 20 # finished jobs this session keeps
|
||||
|
||||
# The metadata scan's outcome groups, in counter-tile order of the spec's
|
||||
# mapping, and the per-file scan status each group collects. Files without
|
||||
# usable metadata have no status and go to "no_metadata".
|
||||
# Which files a scan reads (#249 Activity-2): "new" skips the files the Library
|
||||
# already links with a resolved link (get_known_resolved_paths) before reading
|
||||
# them, so new, unlinked, unresolved and ambiguous files are read; "all" reads
|
||||
# every file again. The workspace job defaults to "new"; legacy /scan is "all".
|
||||
SCAN_SCOPES = ("new", "all")
|
||||
DEFAULT_SCAN_SCOPE = "new"
|
||||
SCAN_GROUP_NAMES = ("added", "known", "review", "no_metadata", "errors")
|
||||
SCAN_GROUPS = {
|
||||
"added": "added",
|
||||
"linked": "added",
|
||||
"enriched": "added",
|
||||
"unchanged": "known",
|
||||
"skipped": "known", # "new" scope: known and resolved, not read again
|
||||
"ambiguous": "review",
|
||||
"conflict": "review",
|
||||
"unresolved": "review",
|
||||
"failed": "errors",
|
||||
}
|
||||
|
||||
|
||||
def _relative_path(path) -> str:
|
||||
"""``path`` as a root-relative POSIX path; an absolute one keeps its name."""
|
||||
text = str(path or "")
|
||||
if (
|
||||
PurePosixPath(text).is_absolute()
|
||||
or PureWindowsPath(text).is_absolute()
|
||||
or PureWindowsPath(text).drive
|
||||
):
|
||||
return re.split(r"[\\/]", text)[-1]
|
||||
return text.replace("\\", "/")
|
||||
|
||||
|
||||
class JobBusy(Exception):
|
||||
"""A job could not start because ``running`` holds its slot."""
|
||||
|
||||
def __init__(self, running):
|
||||
super().__init__(f"{running.title} is already running")
|
||||
self.running = running
|
||||
|
||||
|
||||
class Job:
|
||||
"""One tracked operation: state, counters, outcomes, log and subscribers."""
|
||||
|
||||
def __init__(
|
||||
self, kind, title, *, record_changing=False, scope=None, groups=(), clock=None
|
||||
):
|
||||
self._clock = clock or time.time
|
||||
self.id = uuid.uuid4().hex
|
||||
self.kind = kind
|
||||
self.title = title
|
||||
self.record_changing = record_changing
|
||||
self.scope = dict(scope or {})
|
||||
self.state = "running"
|
||||
self.started_at = self._clock()
|
||||
self._started_seq = next(_sequence)
|
||||
self._finished_seq = None
|
||||
self.finished_at = None
|
||||
self.progress = 0
|
||||
self.status_line = ""
|
||||
self.processed = 0
|
||||
self.total = 0
|
||||
self.counters = dict.fromkeys(groups, 0)
|
||||
self.outcomes = {group: [] for group in groups}
|
||||
self.log = []
|
||||
self.cancelled = False
|
||||
self.last_event = None
|
||||
self.task = None
|
||||
self._subscribers = set()
|
||||
|
||||
@property
|
||||
def running(self):
|
||||
return self.state == "running"
|
||||
|
||||
@property
|
||||
def finished(self):
|
||||
return not self.running
|
||||
|
||||
@property
|
||||
def subscriber_count(self):
|
||||
return len(self._subscribers)
|
||||
|
||||
# ── progress ──────────────────────────────────────────────────────
|
||||
|
||||
def update(self, *, progress=None, status_line=None, processed=None, total=None):
|
||||
"""Set the summary's progress fields; ``None`` leaves one unchanged."""
|
||||
if progress is not None:
|
||||
self.progress = max(0, min(100, int(progress)))
|
||||
if status_line is not None:
|
||||
self.status_line = status_line
|
||||
if processed is not None:
|
||||
self.processed = processed
|
||||
if total is not None:
|
||||
self.total = total
|
||||
|
||||
def publish(self, payload):
|
||||
"""Record ``payload`` as the latest event and hand it to every subscriber."""
|
||||
if isinstance(payload.get("progress"), (int, float)):
|
||||
self.update(progress=payload["progress"])
|
||||
self.last_event = payload
|
||||
for queue in list(self._subscribers):
|
||||
queue.put_nowait(payload)
|
||||
|
||||
def log_line(self, text):
|
||||
"""Append a timestamped log line, up to MAX_LOG_LINES and one cap line."""
|
||||
if len(self.log) > MAX_LOG_LINES:
|
||||
return
|
||||
if len(self.log) == MAX_LOG_LINES:
|
||||
self.log.append(LOG_CAP_LINE)
|
||||
return
|
||||
stamp = time.strftime("%H:%M:%S", time.localtime(self._clock()))
|
||||
self.log.append(f"{stamp} {text}")
|
||||
|
||||
def record(self, group, row, log_line):
|
||||
"""Count one file outcome in ``group``, list its row and log it.
|
||||
|
||||
``row`` is ``{path, root, root_index, detail}``: the path relative to
|
||||
its output root, the root's public label and its listing index (the
|
||||
identity Images opens it by; None when unknown). An absolute path is
|
||||
cut to its file name, so a row never reveals the server's layout.
|
||||
"""
|
||||
if group not in self.counters:
|
||||
raise ValueError(f"Unknown outcome group {group!r} for {self.kind}")
|
||||
self.counters[group] += 1
|
||||
raw = str(row.get("path") or "")
|
||||
path = _relative_path(raw)
|
||||
if raw and raw != path:
|
||||
log_line = log_line.replace(raw, path)
|
||||
rows = self.outcomes[group]
|
||||
index = row.get("root_index")
|
||||
if isinstance(index, bool) or not isinstance(index, int) or index < 0:
|
||||
index = None
|
||||
if len(rows) < MAX_OUTCOME_ROWS:
|
||||
rows.append(
|
||||
{
|
||||
"path": path,
|
||||
"root": str(row.get("root") or ""),
|
||||
"root_index": index,
|
||||
"detail": str(row.get("detail") or ""),
|
||||
}
|
||||
)
|
||||
self.log_line(log_line)
|
||||
|
||||
def finish(self, state="finished"):
|
||||
"""End the job in ``state`` and release every subscriber.
|
||||
|
||||
Each subscriber still gets the events queued before the release. A
|
||||
second finish is a no-op, so the first final state stands.
|
||||
"""
|
||||
if state not in FINAL_STATES:
|
||||
raise ValueError(f"{state!r} is not a final job state")
|
||||
if self.finished:
|
||||
return
|
||||
self.state = state
|
||||
self.finished_at = self._clock()
|
||||
self._finished_seq = next(_sequence)
|
||||
for queue in list(self._subscribers):
|
||||
queue.put_nowait(None)
|
||||
|
||||
async def events(self):
|
||||
"""Yield the latest event, then each later one, until the job finishes.
|
||||
|
||||
Consume it under ``contextlib.aclosing`` so a consumer that stops early
|
||||
(a closed socket) drops its queue at once instead of at garbage collection.
|
||||
"""
|
||||
queue = asyncio.Queue()
|
||||
self._subscribers.add(queue)
|
||||
try:
|
||||
if self.last_event is not None:
|
||||
yield self.last_event
|
||||
if self.finished:
|
||||
return
|
||||
while True:
|
||||
payload = await queue.get()
|
||||
if payload is None:
|
||||
return
|
||||
yield payload
|
||||
finally:
|
||||
self._subscribers.discard(queue)
|
||||
|
||||
# ── views ─────────────────────────────────────────────────────────
|
||||
|
||||
def summary(self):
|
||||
"""The job as the Activity list and pill show it."""
|
||||
return {
|
||||
"id": self.id,
|
||||
"kind": self.kind,
|
||||
"title": self.title,
|
||||
"state": self.state,
|
||||
"started_at": self.started_at,
|
||||
"finished_at": self.finished_at,
|
||||
"progress": self.progress,
|
||||
"status_line": self.status_line,
|
||||
"processed": self.processed,
|
||||
"total": self.total,
|
||||
"counters": dict(self.counters),
|
||||
"scope": dict(self.scope),
|
||||
# What detail() lists, so a view re-reads it only when it changed.
|
||||
"listed": {group: len(rows) for group, rows in self.outcomes.items()},
|
||||
"log_length": len(self.log),
|
||||
}
|
||||
|
||||
def detail(self):
|
||||
"""The summary plus the listed outcome rows and the log."""
|
||||
return {
|
||||
**self.summary(),
|
||||
"outcomes": {
|
||||
group: [dict(row) for row in rows]
|
||||
for group, rows in self.outcomes.items()
|
||||
},
|
||||
"log": list(self.log),
|
||||
}
|
||||
|
||||
def report(self):
|
||||
"""The downloadable report: the summary, every outcome group and the log.
|
||||
|
||||
Each group carries its exact count, its listed rows (at most
|
||||
MAX_OUTCOME_ROWS) and whether the list is cut short. Rows and log lines
|
||||
already hold root-relative paths only.
|
||||
"""
|
||||
summary = self.summary()
|
||||
summary.pop("listed", None)
|
||||
summary.pop("log_length", None)
|
||||
return {
|
||||
"report_version": 1,
|
||||
"generated_at": self._clock(),
|
||||
"job": summary,
|
||||
"outcomes": {
|
||||
group: {
|
||||
"count": self.counters.get(group, len(rows)),
|
||||
"rows": [dict(row) for row in rows],
|
||||
"truncated": self.counters.get(group, len(rows)) > len(rows),
|
||||
}
|
||||
for group, rows in self.outcomes.items()
|
||||
},
|
||||
"log": list(self.log),
|
||||
}
|
||||
|
||||
def report_filename(self):
|
||||
"""The download name: prompt-manager-KIND-YYYYMMDD-HHMM.json (local time)."""
|
||||
stamp = time.strftime("%Y%m%d-%H%M", time.localtime(self._clock()))
|
||||
return f"prompt-manager-{self.kind}-{stamp}.json"
|
||||
|
||||
|
||||
class JobRegistry:
|
||||
"""Every job of this server session, running and recently finished."""
|
||||
|
||||
def __init__(self, logger_=None):
|
||||
self._logger = logger_ or logger
|
||||
self._jobs = {}
|
||||
|
||||
def start(self, kind, title, *, record_changing, scope, runner, groups=()) -> Job:
|
||||
"""Create a job and run ``runner(job)`` as a task.
|
||||
|
||||
Raises JobBusy, naming the job in the way, when a job of ``kind`` is
|
||||
running or when ``record_changing`` and another record-changing job
|
||||
runs. Must be called from the event loop.
|
||||
"""
|
||||
same_kind = self.active(kind)
|
||||
if same_kind is not None:
|
||||
raise JobBusy(same_kind)
|
||||
if record_changing:
|
||||
for job in self._jobs.values():
|
||||
if job.running and job.record_changing:
|
||||
raise JobBusy(job)
|
||||
self._prune()
|
||||
job = Job(
|
||||
kind, title, record_changing=record_changing, scope=scope, groups=groups
|
||||
)
|
||||
self._jobs[job.id] = job
|
||||
job.task = asyncio.ensure_future(self._run(job, runner))
|
||||
return job
|
||||
|
||||
async def _run(self, job, runner):
|
||||
try:
|
||||
await runner(job)
|
||||
except asyncio.CancelledError:
|
||||
job.finish("cancelled")
|
||||
raise
|
||||
except Exception:
|
||||
self._logger.exception(f"Job {job.kind} failed")
|
||||
job.finish("failed")
|
||||
finally:
|
||||
if job.running:
|
||||
job.finish("cancelled" if job.cancelled else "finished")
|
||||
|
||||
def _prune(self):
|
||||
"""Drop the oldest finished jobs so a new one keeps the cap."""
|
||||
finished = [job for job in self._jobs.values() if job.finished]
|
||||
finished.sort(key=lambda job: job._finished_seq)
|
||||
for job in finished[: max(0, len(finished) - MAX_FINISHED_JOBS + 1)]:
|
||||
del self._jobs[job.id]
|
||||
|
||||
def get(self, job_id):
|
||||
return self._jobs.get(job_id)
|
||||
|
||||
def active(self, kind):
|
||||
"""The running job of ``kind``, or None."""
|
||||
for job in self._jobs.values():
|
||||
if job.kind == kind and job.running:
|
||||
return job
|
||||
return None
|
||||
|
||||
def list(self):
|
||||
"""Running jobs first, then the last finished ones, newest first."""
|
||||
jobs = list(self._jobs.values())
|
||||
running = [job for job in jobs if job.running]
|
||||
finished = [job for job in jobs if job.finished]
|
||||
running.sort(key=lambda job: job._started_seq, reverse=True)
|
||||
finished.sort(key=lambda job: job._finished_seq, reverse=True)
|
||||
return running + finished[:MAX_FINISHED_JOBS]
|
||||
|
||||
def cancel(self, job_id) -> bool:
|
||||
"""Ask a running job to stop at its next safe boundary.
|
||||
|
||||
Returns whether a cancellation was requested: False for a finished or
|
||||
unknown job, so a cancel after the end is a no-op.
|
||||
"""
|
||||
job = self._jobs.get(job_id)
|
||||
if job is None or job.finished:
|
||||
return False
|
||||
job.cancelled = True
|
||||
return True
|
||||
@@ -0,0 +1,115 @@
|
||||
"""Saved views (D16): list, create, rename and delete named toolbar states.
|
||||
|
||||
The page, the name and the whitelisted per-page state are validated here
|
||||
(through ``database/saved_views.py``) before anything is written. Only that
|
||||
request validation (``SavedViewInvalid``) is a 400 carrying its message; any
|
||||
other failure, including a ``ValueError`` raised by the database layer, is a
|
||||
logged 500 with a generic message.
|
||||
"""
|
||||
|
||||
from functools import wraps
|
||||
from typing import Callable
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
try:
|
||||
from ...database.saved_views import SavedViewInvalid, encode_state, validate_name
|
||||
except ImportError:
|
||||
from database.saved_views import SavedViewInvalid, encode_state, validate_name
|
||||
|
||||
_CREATE_FIELDS = frozenset({"page", "name", "state"})
|
||||
_RENAME_FIELDS = frozenset({"name"})
|
||||
_MAX_ID = 2**63 - 1
|
||||
|
||||
|
||||
class SavedViewNotFound(LookupError):
|
||||
pass
|
||||
|
||||
|
||||
def saved_view_response(handler: Callable) -> Callable:
|
||||
"""``{success:false,error}`` with 400, 404 or 500 on every failure."""
|
||||
|
||||
@wraps(handler)
|
||||
async def wrapped(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
return await handler(self, request)
|
||||
except SavedViewNotFound:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Saved view not found"}, status=404
|
||||
)
|
||||
except SavedViewInvalid as exc:
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=400)
|
||||
except Exception:
|
||||
self.logger.exception("Saved view operation failed")
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Saved view operation failed"}, status=500
|
||||
)
|
||||
|
||||
return wrapped
|
||||
|
||||
|
||||
async def _body(request: web.Request, fields: frozenset) -> dict:
|
||||
"""A JSON object whose keys are exactly ``fields``."""
|
||||
try:
|
||||
body = await request.json()
|
||||
except (ValueError, UnicodeDecodeError): # JSONDecodeError is a ValueError
|
||||
raise SavedViewInvalid("Expected a JSON object") from None
|
||||
if not isinstance(body, dict):
|
||||
raise SavedViewInvalid("Expected a JSON object")
|
||||
if set(body) != fields:
|
||||
raise SavedViewInvalid(
|
||||
"Request must contain exactly " + ", ".join(sorted(fields))
|
||||
)
|
||||
return body
|
||||
|
||||
|
||||
def _view_id(request: web.Request) -> int:
|
||||
raw = request.match_info["view_id"]
|
||||
if not raw.isascii() or not raw.isdigit():
|
||||
raise SavedViewInvalid("Invalid saved view ID")
|
||||
value = int(raw)
|
||||
if not 1 <= value <= _MAX_ID:
|
||||
raise SavedViewInvalid("Invalid saved view ID")
|
||||
return value
|
||||
|
||||
|
||||
class SavedViewRoutesMixin:
|
||||
def _register_saved_view_routes(self, routes: web.RouteTableDef) -> None:
|
||||
routes.get("/prompt_manager/saved-views")(self.list_saved_views)
|
||||
routes.post("/prompt_manager/saved-views")(self.create_saved_view)
|
||||
routes.patch("/prompt_manager/saved-views/{view_id}")(self.rename_saved_view)
|
||||
routes.delete("/prompt_manager/saved-views/{view_id}")(self.delete_saved_view)
|
||||
|
||||
@saved_view_response
|
||||
async def list_saved_views(self, request: web.Request) -> web.Response:
|
||||
views = await self._run_in_executor(self.db.list_saved_views)
|
||||
return web.json_response({"success": True, "views": views})
|
||||
|
||||
@saved_view_response
|
||||
async def create_saved_view(self, request: web.Request) -> web.Response:
|
||||
body = await _body(request, _CREATE_FIELDS)
|
||||
encode_state(body["page"], body["state"])
|
||||
validate_name(body["name"])
|
||||
view = await self._run_in_executor(
|
||||
self.db.create_saved_view, body["page"], body["name"], body["state"]
|
||||
)
|
||||
return web.json_response({"success": True, "view": view}, status=201)
|
||||
|
||||
@saved_view_response
|
||||
async def rename_saved_view(self, request: web.Request) -> web.Response:
|
||||
view_id = _view_id(request)
|
||||
body = await _body(request, _RENAME_FIELDS)
|
||||
validate_name(body["name"])
|
||||
view = await self._run_in_executor(
|
||||
self.db.rename_saved_view, view_id, body["name"]
|
||||
)
|
||||
if view is None:
|
||||
raise SavedViewNotFound
|
||||
return web.json_response({"success": True, "view": view})
|
||||
|
||||
@saved_view_response
|
||||
async def delete_saved_view(self, request: web.Request) -> web.Response:
|
||||
view_id = _view_id(request)
|
||||
if not await self._run_in_executor(self.db.delete_saved_view, view_id):
|
||||
raise SavedViewNotFound
|
||||
return web.json_response({"success": True, "id": view_id})
|
||||
@@ -1,62 +0,0 @@
|
||||
"""Progress fan-out for the background output scan.
|
||||
|
||||
The scan runs as an asyncio task that outlives the HTTP request that started
|
||||
it, so an accidental tab close does not abandon a half-finished scan. Every
|
||||
SSE subscriber (the request that started the scan, a reconnecting page, a
|
||||
second tab) receives the latest event as a snapshot and then every event
|
||||
published after it, until the job finishes.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
|
||||
class ScanJob:
|
||||
"""One scan run: its latest event, completion flag and live subscribers."""
|
||||
|
||||
def __init__(self):
|
||||
self.last_event = None
|
||||
self.finished = False
|
||||
self.cancelled = False
|
||||
self.task = None
|
||||
self._subscribers = set()
|
||||
|
||||
@property
|
||||
def running(self):
|
||||
return not self.finished
|
||||
|
||||
@property
|
||||
def subscriber_count(self):
|
||||
return len(self._subscribers)
|
||||
|
||||
def publish(self, payload):
|
||||
"""Record ``payload`` as the latest event and hand it to every subscriber."""
|
||||
self.last_event = payload
|
||||
for queue in list(self._subscribers):
|
||||
queue.put_nowait(payload)
|
||||
|
||||
def finish(self):
|
||||
"""Mark the job done and release every subscriber after its queued events."""
|
||||
self.finished = True
|
||||
for queue in list(self._subscribers):
|
||||
queue.put_nowait(None)
|
||||
|
||||
async def events(self):
|
||||
"""Yield the latest event, then each later one, until the job finishes.
|
||||
|
||||
Consume it under ``contextlib.aclosing`` so a consumer that stops early
|
||||
(a closed socket) drops its queue at once instead of at garbage collection.
|
||||
"""
|
||||
queue = asyncio.Queue()
|
||||
self._subscribers.add(queue)
|
||||
try:
|
||||
if self.last_event is not None:
|
||||
yield self.last_event
|
||||
if self.finished:
|
||||
return
|
||||
while True:
|
||||
payload = await queue.get()
|
||||
if payload is None:
|
||||
return
|
||||
yield payload
|
||||
finally:
|
||||
self._subscribers.discard(queue)
|
||||
@@ -0,0 +1,296 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { THEMES, seedConcept, openConcept, openProduction, diffInventory, mutate } from './concept-inventory.mjs';
|
||||
|
||||
// Activity-1 Task 3. The review dialog is compared with the concept's own `scan-review` ("Review metadata scan",
|
||||
// app.js 1428–1432) and its backup box with the concept's System `.notice`; the differences are D55. The job view
|
||||
// has no concept counterpart: it is compared with the approved mockup `activity-scan.html` (a local, ignored file),
|
||||
// whose structure and CSS values are restated below (D56). Each comparison is shown to notice a mutation.
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
page.conceptErrors = [];
|
||||
page.on('pageerror', error => page.conceptErrors.push(error.message));
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, request, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
expect(page.conceptErrors).toEqual([]);
|
||||
await request.post('/__test__/jobs/restart');
|
||||
});
|
||||
|
||||
/** The open review dialog: its body's element sequence, the meta grid, the sentence, the notice and the actions. */
|
||||
function measureReview() {
|
||||
const dialog = document.querySelector('dialog#modal[open]');
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
const name = node => [node.localName, ...[...node.classList].sort()].join('.');
|
||||
const grid = dialog.querySelector('#modal-body > .meta-grid');
|
||||
const sentence = dialog.querySelector('#modal-body > p.small');
|
||||
const actions = dialog.querySelector('#modal-body > .dialog-actions');
|
||||
return {
|
||||
title: dialog.querySelector('#modal-title').textContent,
|
||||
body: [...dialog.querySelector('#modal-body').children].map(name),
|
||||
grid: {
|
||||
tag: grid.localName,
|
||||
...css(grid, 'display', 'gridTemplateColumns', 'columnGap', 'rowGap', 'marginTop', 'marginBottom', 'fontSize'),
|
||||
entries: [...grid.children].map(entry => ({
|
||||
tag: entry.localName, dt: entry.querySelector('dt').textContent, dd: entry.querySelector('dd').textContent,
|
||||
dtStyle: css(entry.querySelector('dt'), 'color', 'fontSize'),
|
||||
ddStyle: css(entry.querySelector('dd'), 'marginTop', 'marginLeft', 'fontSize', 'color'),
|
||||
})),
|
||||
},
|
||||
sentence: { text: sentence.textContent, ...css(sentence, 'fontSize', 'color', 'lineHeight', 'marginTop') },
|
||||
actions: {
|
||||
...css(actions, 'display', 'justifyContent', 'columnGap', 'marginTop'),
|
||||
buttons: [...actions.children].map(button => ({ text: button.textContent, className: button.className,
|
||||
...css(button, 'backgroundColor', 'color', 'fontSize', 'paddingTop', 'paddingLeft', 'borderTopLeftRadius') })),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** A `.notice` box: its frame and its text and actions columns, without the copy. */
|
||||
function measureNotice(selector) {
|
||||
const notice = document.querySelector(selector);
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
return {
|
||||
frame: css(notice, 'display', 'alignItems', 'justifyContent', 'columnGap', 'borderTopWidth', 'borderTopStyle',
|
||||
'borderTopColor', 'backgroundColor', 'borderTopLeftRadius', 'paddingTop', 'paddingLeft'),
|
||||
columns: [...notice.children].map(node => node.localName + (node.className ? `.${node.className}` : '')),
|
||||
text: css(notice.querySelector(':scope > div > p'), 'fontSize', 'color', 'marginTop'),
|
||||
actions: css(notice.querySelector(':scope > .row-actions'), 'display', 'flexShrink'),
|
||||
};
|
||||
}
|
||||
|
||||
/** A concept `.setting-row` switch: its frame, its label and hint, and its checkbox (not its place-dependent margin). */
|
||||
function measureToggle(selector) {
|
||||
const row = document.querySelector(selector);
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
return {
|
||||
children: [...row.children].map(node => [node.localName, ...node.classList].join('.')),
|
||||
frame: css(row, 'display', 'alignItems', 'justifyContent', 'columnGap', 'paddingTop', 'paddingBottom',
|
||||
'borderTopWidth', 'borderTopStyle', 'borderTopColor'),
|
||||
label: css(row.querySelector('label'), 'fontSize', 'fontWeight', 'color'),
|
||||
hint: css(row.querySelector('p'), 'fontSize', 'color', 'marginTop'),
|
||||
box: css(row.querySelector('input[type="checkbox"]'), 'accentColor', 'width', 'height', 'appearance',
|
||||
'backgroundColor', 'borderTopLeftRadius'),
|
||||
role: row.querySelector('input').getAttribute('role'),
|
||||
labelled: row.querySelector('label').htmlFor === row.querySelector('input').id,
|
||||
};
|
||||
}
|
||||
|
||||
/** D55 and D58: the concept's dialog with the production differences applied. */
|
||||
function mapReviewConcept(reference) {
|
||||
return {
|
||||
...reference,
|
||||
// No sample-plan note (prototype copy); the D58 "All files" toggle follows the grid; the backup box sits
|
||||
// before the actions.
|
||||
body: [...reference.body.filter(item => item !== 'p.demo-note').slice(0, -1), 'div.notice', reference.body.at(-1)]
|
||||
.flatMap(item => item === 'div.meta-grid' ? ['dl.meta-grid', 'div.setting-row'] : [item]),
|
||||
// A `dl`, as in the concept's maintenance plan; real sources. The scope keeps the concept's
|
||||
// "New and unresolved files" until "All files" is checked.
|
||||
grid: { ...reference.grid, tag: 'dl', entries: reference.grid.entries.map(entry => ({ ...entry,
|
||||
dd: { Sources: '2 output folders', Scope: 'New and unresolved files' }[entry.dt] })) },
|
||||
actions: { ...reference.actions, buttons: reference.actions.buttons.map(button => button.text === 'Simulate scan'
|
||||
? { ...button, text: 'Start scan' } : button) },
|
||||
};
|
||||
}
|
||||
|
||||
async function conceptReview(page, theme) {
|
||||
await openConcept(page, 'images', theme);
|
||||
await page.locator('#main .heading-actions').getByRole('button', { name: 'Import', exact: true }).click();
|
||||
await page.locator('dialog#modal[open] .operation').filter({ hasText: 'Scan an output folder' })
|
||||
.getByRole('button', { name: 'Review scan', exact: true }).click();
|
||||
await expect(page.locator('dialog#modal[open] #modal-title')).toHaveText('Review metadata scan');
|
||||
await page.mouse.move(0, 0);
|
||||
const review = await page.evaluate(measureReview);
|
||||
await page.keyboard.press('Escape');
|
||||
// The concept's System → Sources notice ("Bring older images into your prompt history").
|
||||
await page.goto(`/__test__/canonical/workspace#system`);
|
||||
await page.getByRole('combobox', { name: 'Theme', exact: true }).selectOption(theme);
|
||||
await page.locator('#main').getByRole('button', { name: 'Manage sources', exact: true }).click();
|
||||
await expect(page.locator('#main .notice')).toBeVisible();
|
||||
return { review, notice: await page.evaluate(measureNotice, '#main .notice'),
|
||||
toggle: await page.evaluate(measureToggle, '#main .setting-row:has(input[type="checkbox"])') };
|
||||
}
|
||||
|
||||
async function productionReview(page, theme) {
|
||||
await openProduction(page, 'images', theme);
|
||||
await page.locator('#workspace-main .heading-actions').getByRole('button', { name: 'Import', exact: true }).click();
|
||||
await page.locator('dialog#modal[open] .operation').filter({ hasText: 'Scan for metadata' })
|
||||
.getByRole('button', { name: 'Review scan', exact: true }).click();
|
||||
await expect(page.locator('dialog#modal[open] #modal-title')).toHaveText('Review metadata scan');
|
||||
// The swapped dialog can open under the pointer that pressed Review scan; measure it at rest.
|
||||
await page.mouse.move(0, 0);
|
||||
const review = await page.evaluate(measureReview);
|
||||
const notice = await page.evaluate(measureNotice, 'dialog#modal[open] .notice');
|
||||
// The concept's switch is on; compare ours on too, then turn it off again (the default).
|
||||
const allFiles = page.getByRole('switch', { name: 'All files', exact: true });
|
||||
await allFiles.check();
|
||||
await page.mouse.move(0, 0);
|
||||
const toggle = await page.evaluate(measureToggle, 'dialog#modal[open] .setting-row');
|
||||
await allFiles.uncheck();
|
||||
await page.mouse.move(0, 0);
|
||||
return { review, notice, toggle };
|
||||
}
|
||||
|
||||
for (const theme of THEMES) test(`review dialog matches the concept scan-review ${theme}`, async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
await seedConcept(request);
|
||||
const concept = await conceptReview(page, theme);
|
||||
expect(concept.review.body).toEqual(['p.demo-note', 'div.meta-grid', 'p.small', 'div.dialog-actions']);
|
||||
const production = await productionReview(page, theme);
|
||||
expect(diffInventory(mapReviewConcept(concept.review), production.review)).toEqual([]);
|
||||
expect(diffInventory(concept.notice, production.notice)).toEqual([]);
|
||||
expect(diffInventory(concept.toggle, production.toggle)).toEqual([]);
|
||||
await expect(page.locator('dialog#modal[open] .setting-row > div > label')).toHaveText('All files');
|
||||
await expect(page.locator('dialog#modal[open] .setting-row input')).not.toBeChecked();
|
||||
await expect(page.getByRole('switch', { name: 'All files', exact: true })).toBeVisible();
|
||||
await expect(page.locator('dialog#modal[open] .notice h3')).toHaveText('Back up the database first (recommended)');
|
||||
await expect(page.locator('dialog#modal[open] .notice .row-actions button')).toHaveText(['Back up now']);
|
||||
|
||||
// Mutation check: one CSS change and one removed element are both noticed, then undone to an exact match again.
|
||||
const undoCss = await mutate(page, { css: '#modal .meta-grid { gap: 3px; } #modal .notice { padding: 2px; } #modal .setting-row { gap: 3px; }' });
|
||||
const cssMutated = await page.evaluate(measureReview);
|
||||
const noticeMutated = await page.evaluate(measureNotice, 'dialog#modal[open] .notice');
|
||||
await page.getByRole('switch', { name: 'All files', exact: true }).check();
|
||||
await page.mouse.move(0, 0);
|
||||
const toggleMutated = await page.evaluate(measureToggle, 'dialog#modal[open] .setting-row');
|
||||
await page.getByRole('switch', { name: 'All files', exact: true }).uncheck();
|
||||
await page.mouse.move(0, 0);
|
||||
expect(diffInventory(concept.toggle, toggleMutated).map(item => item.path)).toEqual(['frame.columnGap']);
|
||||
expect(diffInventory(mapReviewConcept(concept.review), cssMutated).map(item => item.path)).toEqual(expect.arrayContaining(['grid.columnGap', 'grid.rowGap']));
|
||||
expect(diffInventory(concept.notice, noticeMutated).map(item => item.path)).toEqual(['frame.paddingTop', 'frame.paddingLeft']);
|
||||
await undoCss();
|
||||
const undoDom = await mutate(page, { remove: ['#modal-body > p.small'] });
|
||||
const domMutated = await page.evaluate(() => [...document.querySelectorAll('#modal-body > *')].map(node => node.localName));
|
||||
expect(domMutated).toEqual(['dl', 'div', 'div', 'div']);
|
||||
await undoDom();
|
||||
expect(diffInventory(mapReviewConcept(concept.review), await page.evaluate(measureReview))).toEqual([]);
|
||||
});
|
||||
|
||||
// The approved mockup's job view, restated: structure first, then the values of its CSS rules.
|
||||
const MOCKUP_TREE = ['div.heading', 'section.surface', 'section.surface', 'details.term'];
|
||||
const MOCKUP_PROGRESS = ['div.panel-heading(h2,span.small)', 'progress', 'dl.stat-row(div.added.stat,div.stat,div.stat.warn,div.stat,div.bad.stat)'];
|
||||
const MOCKUP_RESULTS = ['div.panel-heading(h2,label.search-field)', 'div.outcomes(div.outcome-list,div.outcome-files)'];
|
||||
const MOCKUP_STYLES = {
|
||||
statRow: { display: 'grid', columnGap: '10px', columns: 5, marginTop: '16px', marginBottom: '16px' },
|
||||
stat: { borderTopWidth: '1px', borderTopLeftRadius: '7px', paddingTop: '12px', paddingLeft: '14px' },
|
||||
dt: { fontSize: '11px' },
|
||||
dd: { fontSize: '20px', fontWeight: '600', marginTop: '6px', marginLeft: '0px' },
|
||||
outcomes: { display: 'grid', columnGap: '16px', first: '220px', marginTop: '6px' },
|
||||
outcomeButton: { display: 'flex', justifyContent: 'space-between', marginBottom: '6px', textAlign: 'left' },
|
||||
fileRow: { display: 'grid', columnGap: '12px', paddingTop: '9px', paddingLeft: '4px', borderBottomWidth: '1px', fontSize: '12px' },
|
||||
path: { monospace: true, overflowWrap: 'anywhere' },
|
||||
search: { maxWidth: '320px' },
|
||||
term: { marginTop: '16px', borderTopWidth: '1px', borderTopLeftRadius: '7px', overflow: 'hidden' },
|
||||
summary: { paddingTop: '11px', paddingLeft: '14px', fontSize: '12px', cursor: 'pointer' },
|
||||
pre: { marginTop: '0px', paddingTop: '12px', paddingLeft: '14px', maxHeight: '180px', fontSize: '11px', lineHeight: '18.7px', overflow: 'auto' },
|
||||
};
|
||||
|
||||
function measureJobView() {
|
||||
const main = document.querySelector('#workspace-main');
|
||||
const css = (node, ...keys) => Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]]));
|
||||
const name = node => [node.localName, ...[...node.classList].sort()].join('.');
|
||||
// The rows and sections the mockup nests are listed with their children.
|
||||
const shape = node => node.matches('.panel-heading, .outcomes, dl.stat-row')
|
||||
? `${name(node)}(${[...node.children].map(name).join(',')})` : name(node);
|
||||
const [progress, results] = main.querySelectorAll(':scope > section.surface');
|
||||
const statRow = progress.querySelector('dl.stat-row');
|
||||
const stat = statRow.querySelector('.stat');
|
||||
const outcomes = results.querySelector('.outcomes');
|
||||
const button = outcomes.querySelector('.outcome-list > button');
|
||||
const row = outcomes.querySelector('.file-row');
|
||||
const term = main.querySelector(':scope > details.term');
|
||||
const pre = term.querySelector('pre');
|
||||
const columns = getComputedStyle(statRow).gridTemplateColumns.split(' ');
|
||||
return {
|
||||
tree: [...main.children].map(name),
|
||||
progress: [...progress.children].map(shape),
|
||||
results: [...results.children].map(shape),
|
||||
h2: [...main.querySelectorAll(':scope > section.surface > .panel-heading > h2')].map(node => node.textContent),
|
||||
summary: term.querySelector('summary').textContent,
|
||||
styles: {
|
||||
statRow: { ...css(statRow, 'display', 'columnGap', 'marginTop', 'marginBottom'), columns: columns.length },
|
||||
stat: css(stat, 'borderTopWidth', 'borderTopLeftRadius', 'paddingTop', 'paddingLeft'),
|
||||
dt: css(stat.querySelector('dt'), 'fontSize'),
|
||||
dd: css(stat.querySelector('dd'), 'fontSize', 'fontWeight', 'marginTop', 'marginLeft'),
|
||||
outcomes: { ...css(outcomes, 'display', 'columnGap', 'marginTop'), first: getComputedStyle(outcomes).gridTemplateColumns.split(' ')[0] },
|
||||
outcomeButton: css(button, 'display', 'justifyContent', 'marginBottom', 'textAlign'),
|
||||
fileRow: css(row, 'display', 'columnGap', 'paddingTop', 'paddingLeft', 'borderBottomWidth', 'fontSize'),
|
||||
path: { monospace: /monospace/.test(getComputedStyle(row.querySelector('.path')).fontFamily),
|
||||
overflowWrap: getComputedStyle(row.querySelector('.path')).overflowWrap },
|
||||
search: css(results.querySelector('.search-field'), 'maxWidth'),
|
||||
term: css(term, 'marginTop', 'borderTopWidth', 'borderTopLeftRadius', 'overflow'),
|
||||
summary: css(term.querySelector('summary'), 'paddingTop', 'paddingLeft', 'fontSize', 'cursor'),
|
||||
// WebKit reports the 1.7 line height of 11px as 18.700001px; compare it to the hundredth of a pixel.
|
||||
pre: { ...css(pre, 'marginTop', 'paddingTop', 'paddingLeft', 'maxHeight', 'fontSize', 'overflow'),
|
||||
lineHeight: `${Math.round(parseFloat(getComputedStyle(pre).lineHeight) * 100) / 100}px` },
|
||||
},
|
||||
// Mockup colours: the pressed outcome takes the accent border on the soft blue, the tiles their tone.
|
||||
pressed: (() => {
|
||||
const probe = document.createElement('span');
|
||||
main.append(probe);
|
||||
const tone = name => { probe.style.color = `var(${name})`; return getComputedStyle(probe).color; };
|
||||
const values = { accent: tone('--ca-accent'), soft: tone('--soft-blue'), green: tone('--green'), red: tone('--red'), warning: tone('--warning') };
|
||||
probe.remove();
|
||||
const on = outcomes.querySelector('.outcome-list > button[aria-pressed="true"]');
|
||||
const dd = selector => getComputedStyle(statRow.querySelector(`${selector} dd`)).color;
|
||||
return { border: getComputedStyle(on).borderTopColor === values.accent, background: getComputedStyle(on).backgroundColor === values.soft,
|
||||
added: dd('.stat.added') === values.green, bad: dd('.stat.bad') === values.red, warn: dd('.stat.warn') === values.warning };
|
||||
})(),
|
||||
};
|
||||
}
|
||||
|
||||
const EXPECTED_JOB_VIEW = {
|
||||
tree: MOCKUP_TREE, progress: MOCKUP_PROGRESS, results: MOCKUP_RESULTS, h2: ['Progress', 'Results'],
|
||||
summary: '● Scan log — raw detail, click to show', styles: MOCKUP_STYLES,
|
||||
pressed: { border: true, background: true, added: true, bad: true, warn: true },
|
||||
};
|
||||
|
||||
async function openRunningJob(page, request, theme, viewport = { width: 1600, height: 1100 }) {
|
||||
await page.setViewportSize(viewport);
|
||||
const started = await request.post('/__test__/jobs/start', { data: { total: 19622 } });
|
||||
await expect(started).toBeOK();
|
||||
const { job } = await started.json();
|
||||
await expect(await request.post(`/__test__/jobs/${job.id}/step`, { data: { processed: 8240,
|
||||
counters: { known: 6511, review: 37, no_metadata: 254, errors: 18 },
|
||||
records: [{ group: 'added', path: 'CCC/mabes9sans-dataset-v1/IMG_GEN_00369_.png', detail: 'Added to the Library as “Mabes portrait”' }] } })).toBeOK();
|
||||
await openProduction(page, `activity/${job.id}`, theme);
|
||||
await expect(page.locator('#workspace-main .file-row')).toHaveCount(1);
|
||||
return job;
|
||||
}
|
||||
|
||||
for (const theme of THEMES) test(`job view follows the approved mockup ${theme}`, async ({ page, request }) => {
|
||||
await seedConcept(request);
|
||||
await openRunningJob(page, request, theme);
|
||||
expect(await page.evaluate(measureJobView)).toEqual(EXPECTED_JOB_VIEW);
|
||||
|
||||
// Mutation check: a CSS change (tile gap) and a removed element (the outcome list) are both noticed.
|
||||
const undoCss = await mutate(page, { css: '#workspace-main .stat-row { gap: 3px; }' });
|
||||
expect(diffInventory(EXPECTED_JOB_VIEW, await page.evaluate(measureJobView)).map(item => item.path)).toEqual(['styles.statRow.columnGap']);
|
||||
await undoCss();
|
||||
const undoDom = await mutate(page, { remove: ['#workspace-main details.term'] });
|
||||
expect(await page.evaluate(() => [...document.querySelector('#workspace-main').children].map(node => node.localName)))
|
||||
.toEqual(['div', 'section', 'section']);
|
||||
await undoDom();
|
||||
expect(await page.evaluate(measureJobView)).toEqual(EXPECTED_JOB_VIEW);
|
||||
});
|
||||
|
||||
test('job view fits a phone without page scroll', async ({ page, request }) => {
|
||||
await seedConcept(request);
|
||||
await openRunningJob(page, request, 'dark', { width: 390, height: 844 });
|
||||
const layout = await page.evaluate(() => {
|
||||
const main = document.querySelector('#workspace-main');
|
||||
const columns = node => getComputedStyle(node).gridTemplateColumns.split(' ').length;
|
||||
return {
|
||||
scroll: document.documentElement.scrollWidth <= document.documentElement.clientWidth,
|
||||
tiles: columns(main.querySelector('dl.stat-row')),
|
||||
outcomes: columns(main.querySelector('.outcomes')),
|
||||
inside: [...main.querySelectorAll('.stat, .outcome-list button, .file-row, .search-field')]
|
||||
.every(node => node.getBoundingClientRect().right <= main.getBoundingClientRect().right + 0.5),
|
||||
};
|
||||
});
|
||||
expect(layout).toEqual({ scroll: true, tiles: 2, outcomes: 1, inside: true });
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
|
||||
test.beforeEach(async ({page, browserName}) => {
|
||||
@@ -57,7 +57,8 @@ test('Classic keeps immediate selection, search query and metadata serialization
|
||||
});
|
||||
|
||||
test('Classic detaches native text synchronization on removal',async({page})=>{
|
||||
await page.waitForTimeout(60);
|
||||
// The page installs the text sync (serializeValue and its listeners) in a 50 ms timer; wait for it, not for time.
|
||||
await expect.poll(()=>page.evaluate(()=>typeof node.widgets.find(item=>item.name==='text').serializeValue)).toBe('function');
|
||||
const values=await page.evaluate(()=>{
|
||||
const widget=node.widgets.find(item=>item.name==='text');
|
||||
widget.inputEl.value='Before removal';widget.inputEl.dispatchEvent(new Event('input'));
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// same concept selectors in the immutable concept (/__test__/canonical/workspace) and in production, which now renders
|
||||
// the concept's classes. The concept is never rewritten: each page's map<Page>Concept(reference) applies only the
|
||||
// recorded spec-§8 deviations, each commented with its D-number, and diffInventory must then report nothing.
|
||||
import {expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {expect} from './fixtures.mjs';
|
||||
export {diffInventory} from './library-editor-canonical-comparison.mjs';
|
||||
|
||||
export const CONCEPT_URL='/__test__/canonical/workspace';
|
||||
@@ -97,6 +97,8 @@ export async function openProduction(page,route,theme,motion='no-preference') {
|
||||
await selectTheme(page,theme);
|
||||
await expect(page.locator('#workspace-main #page-title')).toBeVisible();
|
||||
await expect(page.locator('#primary-nav .nav-count')).toHaveCount(3);
|
||||
// SAVED VIEWS' Favorites count (D51), the concept's starred images in the concept fixture.
|
||||
await expect(page.locator('#saved-nav a[data-route="favorites"] .nav-count')).toHaveCount(1);
|
||||
}
|
||||
|
||||
/** Computed outline of the focused element, after keyboard focus so `:focus-visible` applies. */
|
||||
@@ -152,7 +154,8 @@ export async function mutate(page,{css=null,remove=[]}={}) {
|
||||
* history, noPageScroll}. A section is null when its element is absent. The Images sections (toolbar … panel, #249 C7)
|
||||
* measure the concept's Images markup; `cards` holds the first six; `bulkbar` is the selection bar's place and, while
|
||||
* anything is checked, its `.bulkbar` row. `library` (#249 E3) holds the Prompt library's toolbar,
|
||||
* caption, grid, first six cards and whatever follows the grid; history stays null until PR F.
|
||||
* caption, grid, first six cards and whatever follows the grid. `history` (#249 F5) holds the Prompt history's summary,
|
||||
* negative variants panel, comparison (while Compare negatives is pressed) and the shared gallery's heading.
|
||||
*/
|
||||
export function inventoryOf(page,scope='main') {
|
||||
return page.evaluate(measureInventory,scope);
|
||||
@@ -441,6 +444,85 @@ export function measureInventory(scope) {
|
||||
grid:{columns:style.gridTemplateColumns.split(/\s+/).length,...css(grid,'columnGap','rowGap'),...relative(grid,main),cardCount:cards.length},
|
||||
cards:cards.slice(0,6).map(card=>libraryCardOf(card,grid)),trailing};
|
||||
}
|
||||
// ---- Prompt history (#249 F5): renderHistory a587–622, style.css 1059–1179 and 1546–1567. The concept's main holds
|
||||
// the summary, the variants panel, the comparison and the gallery; production's sit in `section.library-history`, so
|
||||
// each is found by descendant and placed relative to `main` (vertical rhythm) or to its own section.
|
||||
const spoken=node=>node.classList.contains('ca-visually-hidden');
|
||||
const outline=node=>css(node,'outlineStyle','outlineWidth','outlineColor','outlineOffset');
|
||||
const pseudo=(node,which)=>{const style=getComputedStyle(node,which);return Object.fromEntries(['content','position','top','left','width','height',
|
||||
'borderLeftWidth','borderBottomWidth','borderLeftColor','borderBottomColor','borderBottomLeftRadius','backgroundColor','color'].map(key=>[key,style[key]]));};
|
||||
// What a section shows, in order: each shown child as `tag.classes`, visually hidden statuses apart (`spoken`).
|
||||
const structureOf=node=>({children:boxedChildren(node).filter(child=>shown(child)&&!spoken(child)).map(child=>`${child.tagName.toLowerCase()}.${classes(child)}`),
|
||||
spoken:[...node.querySelectorAll('.ca-visually-hidden')].filter(shown).map(child=>({role:child.getAttribute('role'),text:child.textContent}))});
|
||||
function promptSectionOf(section,origin) {
|
||||
const heading=section.querySelector(':scope > .prompt-heading'),label=heading?.querySelector('.role-label');
|
||||
const actions=heading?.querySelector('.prompt-actions'),text=section.querySelector(':scope > .prompt-text');
|
||||
return {className:classes(section),...relative(section,origin),...edges(section),
|
||||
label:when(label,()=>({text:label.textContent,...font(label)})),
|
||||
actions:when(actions,()=>({...layout(actions),...relative(actions,section),buttons:[...actions.children].map(node=>controlOf(node,actions))})),
|
||||
text:when(text,()=>({text:text.textContent,...font(text),...edges(text),...relative(text,section),
|
||||
lines:Math.round(rect(text).height/parseFloat(getComputedStyle(text).lineHeight))}))};
|
||||
}
|
||||
function panelHeadingOf(heading,origin) {
|
||||
const h2=heading.querySelector('h2'),note=heading.querySelector('.small'),actions=heading.querySelector(':scope > .row-actions');
|
||||
return {...layout(heading),...edges(heading),...relative(heading,origin),
|
||||
h2:when(h2,()=>({text:h2.textContent,...font(h2),...edges(h2),...relative(h2,heading)})),
|
||||
note:when(note,()=>({tag:note.tagName.toLowerCase(),text:note.textContent,...font(note),...edges(note),...relative(note,heading)})),
|
||||
actions:when(actions,()=>({...layout(actions),...relative(actions,heading),buttons:[...actions.children].filter(shown).map(node=>controlOf(node,actions))}))};
|
||||
}
|
||||
function laneOf(lane,timeline) {
|
||||
const dot=lane.querySelector(':scope > .variant-dot'),label=lane.querySelector(':scope > span'),run=lane.querySelector(':scope > .run-dot');
|
||||
return {className:classes(lane),offset:lane.style.getPropertyValue('--lane-offset'),...relative(lane,timeline),...edges(lane),...css(lane,'minWidth','position'),
|
||||
before:pseudo(lane,'::before'),
|
||||
dot:when(dot,()=>({...controlOf(dot,lane),...outline(dot),...css(dot,'position','top','left','width','height','color')})),
|
||||
label:when(label,()=>({text:label.textContent,...font(label),...relative(label,lane),...css(label,'position','top','left')})),
|
||||
run:shown(run)?{...controlOf(run,lane),...css(run,'position','top','left','width','height')}:null};
|
||||
}
|
||||
function timelineOf(timeline,origin) {
|
||||
const positive=timeline.querySelector(':scope > .positive-lane'),span=positive?.querySelector('span');
|
||||
return {...edges(timeline),...css(timeline,'overflowX','position'),...relative(timeline,origin),
|
||||
positive:when(positive,()=>({...edges(positive),...css(positive,'height','minWidth'),...relative(positive,timeline),before:pseudo(positive,'::before'),
|
||||
span:when(span,()=>({text:span.textContent,...font(span),...relative(span,positive)}))})),
|
||||
lanes:[...timeline.querySelectorAll(':scope > .variant-lane')].map(lane=>laneOf(lane,timeline)),
|
||||
structure:structureOf(timeline)};
|
||||
}
|
||||
function variantsOf(panel) {
|
||||
const heading=panel.querySelector(':scope > .panel-heading'),timeline=panel.querySelector(':scope > .timeline'),lower=panel.querySelector(':scope > .timeline-lower');
|
||||
return {className:classes(panel),...edges(panel),...relative(panel,main),structure:structureOf(panel),
|
||||
heading:when(heading,()=>panelHeadingOf(heading,panel)),timeline:when(timeline,()=>timelineOf(timeline,panel)),
|
||||
lower:when(lower,()=>({...layout(lower),...edges(lower),...relative(lower,panel),
|
||||
items:[...lower.children].filter(shown).map(node=>({...controlOf(node,lower),...font(node)}))}))};
|
||||
}
|
||||
function compareOf(section) {
|
||||
const h2=section.querySelector(':scope > h2'),intro=section.querySelector(':scope > p:not(.small)'),grid=section.querySelector(':scope > .compare-grid');
|
||||
return {...edges(section),...relative(section,main),structure:structureOf(section),
|
||||
h2:when(h2,()=>({text:h2.textContent,...font(h2),...edges(h2)})),
|
||||
intro:when(intro,()=>({text:intro.textContent,...font(intro),...edges(intro),...relative(intro,section)})),
|
||||
grid:when(grid,()=>({columns:getComputedStyle(grid).gridTemplateColumns.split(/\s+/).length,...css(grid,'columnGap','rowGap'),...relative(grid,section),
|
||||
items:[...grid.children].map(item=>({className:classes(item),...edges(item),...relative(item,grid),
|
||||
sections:[...item.querySelectorAll('.prompt-section')].map(node=>promptSectionOf(node,item))}))})),
|
||||
notes:[...section.querySelectorAll(':scope > p.small')].map(note=>({text:note.textContent,...font(note),...edges(note),...relative(note,section)})),
|
||||
added:[...section.querySelectorAll(':scope > .diff-added')].map(chip=>({text:chip.textContent,...font(chip),...edges(chip),...css(chip,'display'),...relative(chip,section)}))};
|
||||
}
|
||||
function historyOf() {
|
||||
const panel=main?.querySelector('section.timeline-panel');
|
||||
if(!panel)return null;
|
||||
const summary=main.querySelector('article.surface'),gallery=main.querySelector('section.history-gallery');
|
||||
const compare=[...main.querySelectorAll('section.surface:not(.timeline-panel)')].find(shown);
|
||||
const chip=summary?.querySelector(':scope > .chip'),meta=summary?.querySelector(':scope > .card-meta');
|
||||
const galleryHeading=gallery?.querySelector(':scope > .panel-heading');
|
||||
return {
|
||||
summary:when(summary,()=>({...edges(summary),...relative(summary,main),structure:structureOf(summary),
|
||||
chip:when(chip,()=>({text:chip.textContent,...font(chip),...edges(chip),...relative(chip,summary)})),
|
||||
sections:[...summary.querySelectorAll('.prompt-section')].map(node=>promptSectionOf(node,summary)),
|
||||
meta:when(meta,()=>({...layout(meta),...edges(meta),...font(meta),...relative(meta,summary),
|
||||
items:[...meta.children].filter(shown).map(node=>({text:node.textContent,...font(node),...relative(node,meta)}))}))})),
|
||||
variants:variantsOf(panel),
|
||||
compare:when(compare,compareOf),
|
||||
gallery:when(gallery,()=>({...edges(gallery),...relative(gallery,main),
|
||||
heading:when(galleryHeading,()=>panelHeadingOf(galleryHeading,gallery))})),
|
||||
};
|
||||
}
|
||||
return {chrome:chromeOf(),heading:headingOf(),
|
||||
toolbar:when(main?.querySelector(':scope > search.toolbar'),rowOf),
|
||||
filterRow:when(main?.querySelector(':scope > .filter-row'),rowOf),
|
||||
@@ -450,7 +532,7 @@ export function measureInventory(scope) {
|
||||
cards:grid?[...grid.querySelectorAll(':scope > .media-card')].slice(0,6).map(cardOf):null,
|
||||
continuation:when(continuation,continuationOf),
|
||||
panel:when(gallery,panelOf),
|
||||
library:libraryOf(),history:null,noPageScroll:document.documentElement.scrollWidth<=innerWidth+1};
|
||||
library:libraryOf(),history:historyOf(),noPageScroll:document.documentElement.scrollWidth<=innerWidth+1};
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
// Shared `test` for the page workflows: every Playwright worker owns one
|
||||
// tests/browser/server.py process on a port the OS chose for it, with its own temp root
|
||||
// and database. `baseURL` (and so `page`, `request` and `browser.newContext`)
|
||||
// points at that worker's server. A server that dies fails the running test
|
||||
// with its stderr tail, and the next test in the worker gets a fresh server.
|
||||
// A test that fails gets the stderr its server wrote during it attached as
|
||||
// `server-stderr`.
|
||||
import { test as base, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { attachServerStderr, createServerSupervisor, startBrowserServer } from '../../tools/vendor/start-browser-server.mjs';
|
||||
|
||||
const SERVER_FIXTURE_TIMEOUT_MS = 90_000;
|
||||
|
||||
export const test = base.extend({
|
||||
testServer: [async ({}, use) => {
|
||||
const supervisor = createServerSupervisor({ start: () => startBrowserServer() });
|
||||
try {
|
||||
await use(supervisor);
|
||||
} finally {
|
||||
await supervisor.stop();
|
||||
}
|
||||
}, { scope: 'worker', timeout: SERVER_FIXTURE_TIMEOUT_MS }],
|
||||
workerServer: [async ({ testServer }, use, testInfo) => {
|
||||
const server = await testServer.ensure();
|
||||
const mark = server.stderrMark();
|
||||
await use(server);
|
||||
await attachServerStderr(testInfo, server, mark);
|
||||
const crash = testServer.crash(server);
|
||||
if (crash) throw crash;
|
||||
}, { auto: true, timeout: SERVER_FIXTURE_TIMEOUT_MS }],
|
||||
baseURL: async ({ workerServer }, use) => use(workerServer.url),
|
||||
});
|
||||
|
||||
export { expect };
|
||||
@@ -1,5 +1,5 @@
|
||||
import {readFileSync} from 'node:fs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
|
||||
for (const variant of ['Focused', 'Split']) {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept, openConcept, openProduction, SIZES } from './concept-inventory.mjs';
|
||||
// D10/D20: the heading is the linked positive's title (its confirmed first line here), never the pair's
|
||||
// "Inspector recipe", which names the negative variant.
|
||||
@@ -804,6 +804,7 @@ test('slideshow preference failures leave native playback usable',async({page,re
|
||||
});
|
||||
|
||||
test('full inspector attaches matched theme and mobile evidence with reachable content',async({page,request},testInfo)=>{
|
||||
test.setTimeout(120_000); // 32 evidence captures across four viewports, like the concept comparisons
|
||||
await expect(await request.post('/__test__/reset',{data:{full_inspector:true,include_video:true}})).toBeOK();
|
||||
await page.goto('/prompt_manager/workspace#/images');
|
||||
const measurements=[];
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { writeFile } from 'node:fs/promises';
|
||||
import { expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { expect } from './fixtures.mjs';
|
||||
|
||||
// Content, row counts, and image bytes come from independent synthetic datasets.
|
||||
// Saved choices and actual executions are separate production sections. The concept's
|
||||
@@ -14,9 +14,10 @@ export const canonicalDifferences = {
|
||||
'Recorded run exposes exact frozen evidence; Repeat run is illustrative',
|
||||
'Current-file candidates disclose unknown identity and context membership',
|
||||
],
|
||||
// Closed by #249: the gallery's checkbox is Images-only (#259) and the role blocks are the concept's (PR D, F); the
|
||||
// exact history comparison is tests/browser/history-concept.spec.mjs.
|
||||
remainingApplicationPolish: [
|
||||
'Gallery card select checkbox (#249 D8 follow-up) and tag button (#249 D9, image-annotations slice) remain outside #209',
|
||||
'Shared role-block spacing, padding and glyph styling remain outside #209; exact role order/actions and evidence disclosure are covered',
|
||||
'Gallery card tag button (#249 D9, image-annotations slice) remains outside #209',
|
||||
'Production keeps mobile toolbar inside media panel and readable scrolling details; the concept toolbar collision is not reproduced',
|
||||
],
|
||||
};
|
||||
@@ -33,29 +34,27 @@ export async function captureCanonicalState(page, testInfo, name, selector) {
|
||||
await testInfo.attach(name, { path, contentType: 'image/png' });
|
||||
}
|
||||
|
||||
const CONCEPT_HISTORY_SELECTORS = Object.freeze({
|
||||
surface: 'article.surface', timeline: '.timeline', positive: '.positive-lane', branch: '.variant-lane',
|
||||
label: '.variant-lane > span', dot: '.variant-dot[aria-pressed="true"]', role: 'article.surface .prompt-section',
|
||||
positiveLabel: '.positive-lane span',
|
||||
prompt: 'article.surface .prompt-text', gallery: '.history-gallery',
|
||||
heading: '.timeline-panel h2', intro: '.timeline-panel .panel-heading .small',
|
||||
footer: '.timeline-lower', all: '.timeline-lower button',
|
||||
repeat: '.timeline .run-dot:not([hidden])',
|
||||
});
|
||||
const RECORDED_HISTORY_SELECTORS = Object.freeze({
|
||||
recordedPositiveLabel: '.history-positive-lane span', recordedBranch: '.history-context-branch',
|
||||
recordedHeading: '.recorded-history h2', recordedTimeline: '.history-timeline', recordedCanvas: '.history-canvas',
|
||||
recordedPositive: '.history-positive-lane', recordedLabel: '.history-context-label',
|
||||
recordedAnchor: '.history-context-anchor', recordedRepeat: '.history-run-marker:not(.history-context-anchor)',
|
||||
recordedMarker: '.history-run-marker', recordedColumn: '.history-run-column',
|
||||
});
|
||||
|
||||
export async function historyGeometry(page, production) {
|
||||
const selectors = production ? {
|
||||
surface: '.library-history article.surface', timeline: '.library-history .timeline', positive: '.library-history .positive-lane',
|
||||
branch: '.library-history .variant-lane', label: '.library-history .variant-lane > span', dot: '.library-history .variant-dot[aria-pressed="true"]',
|
||||
positiveLabel: '.library-history .positive-lane span',
|
||||
role: '[aria-label="Saved prompt texts"] .prompt-section',
|
||||
prompt: '[aria-label="Saved prompt texts"] .prompt-text', gallery: '.library-shared-gallery',
|
||||
heading: '.library-history .timeline-panel h2', intro: '.library-history .timeline-panel .panel-heading .small',
|
||||
footer: '.library-history .timeline-lower', all: '.library-history .timeline-lower button',
|
||||
recordedPositiveLabel: '.history-positive-lane span', recordedBranch: '.history-context-branch',
|
||||
recordedHeading: '.recorded-history h2', recordedTimeline: '.history-timeline', recordedCanvas: '.history-canvas',
|
||||
recordedPositive: '.history-positive-lane', recordedLabel: '.history-context-label',
|
||||
recordedAnchor: '.history-context-anchor', recordedRepeat: '.history-run-marker:not(.history-context-anchor)',
|
||||
recordedMarker: '.history-run-marker', recordedColumn: '.history-run-column',
|
||||
} : {
|
||||
surface: '.surface', timeline: '.timeline', positive: '.positive-lane', branch: '.variant-lane',
|
||||
label: '.variant-lane span', dot: '.variant-dot[aria-pressed="true"]', role: '.surface .prompt-section',
|
||||
positiveLabel: '.positive-lane span',
|
||||
prompt: '.surface .prompt-text', gallery: '.history-gallery',
|
||||
heading: '.timeline-panel h2', intro: '.timeline-panel .panel-heading .small',
|
||||
footer: '.timeline-lower', all: '.timeline-lower button',
|
||||
repeat: '.timeline .run-dot',
|
||||
};
|
||||
// Production renders the concept's history classes (#249 PR F), so both documents use the concept's selectors;
|
||||
// production adds the recorded executions (#209), which the concept does not have.
|
||||
const selectors = production ? { ...CONCEPT_HISTORY_SELECTORS, ...RECORDED_HISTORY_SELECTORS } : CONCEPT_HISTORY_SELECTORS;
|
||||
return page.evaluate(selectors => ({ pointerCoarse: matchMedia('(pointer: coarse)').matches,
|
||||
...Object.fromEntries(Object.entries(selectors).map(([key, selector]) => {
|
||||
const nodes = [...document.querySelectorAll(selector)];
|
||||
|
||||
@@ -0,0 +1,294 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,diffInventory,mutate,stripExact} from './concept-inventory.mjs';
|
||||
|
||||
// Task F5 of #249: the production Prompt history (#/library/<id>) against the immutable concept's (#prompt/alpine,
|
||||
// app.js renderHistory a587–622, the dialogs a1282–1304, style.css 1059–1179 and 1546–1567), on the concept-shaped real
|
||||
// fixture: Alpine observatory with its three saved negatives, four images linked to their pairs, four recorded runs and
|
||||
// real pair run counts (2, 1, 1). Heading, summary, the negative variants panel, the comparison and the shared gallery's
|
||||
// heading are compared at 1600x1100 and 390x844 in both themes and motion settings, with the page's behaviors in both
|
||||
// documents. Both documents are measured with "Clean architecture" selected: it is the middle lane in either order
|
||||
// (D46). The topbar and sidebar are the shell spec's, the gallery's cards the Images spec's (D47). Paired captures are
|
||||
// artifacts only; there are no PNG baselines.
|
||||
test.use({locale:'en-US',timezoneId:'UTC'});
|
||||
|
||||
const TITLE='Alpine observatory';
|
||||
const SELECTED='Clean architecture';
|
||||
const LANES=3;
|
||||
const SHOW_ARCHIVED='Show archived (0)';
|
||||
const SECTIONS=['heading','history','noPageScroll'];
|
||||
|
||||
/**
|
||||
* Recorded history deviations only (spec §8, TESTING.md D24–D26, D35, D45–D47); everything else must equal production.
|
||||
* `reference.oracle` is the concept page itself, measured once its lanes list newest first with the singular for one
|
||||
* (D46, D35) and once the approved Show archived button (#235) follows + Add negative: the browser lays out what moves.
|
||||
*/
|
||||
export function mapHistoryConcept(reference) {
|
||||
const {history}=reference.oracle;
|
||||
return {...reference,history:{...history,
|
||||
// D24: "N tracked runs" is the recorded window's `recorded_run_count`; the fixture records one run per image, so
|
||||
// the concept's "4 tracked runs" holds by construction (asserted on both documents below).
|
||||
summary:history.summary,
|
||||
variants:{...history.variants,
|
||||
// D45: the bounded "N of M saved negative variants" count stays a visually hidden status.
|
||||
structure:{...history.variants.structure,spoken:[{role:'status',text:`${LANES} of ${LANES} saved negative variants`}]}},
|
||||
compare:history.compare&&{...history.compare,grid:{...history.compare.grid,items:history.compare.grid.items.map((item,index)=>({...item,
|
||||
// D46: the two compared blocks are named Baseline and Selected, so no action name repeats on the page.
|
||||
sections:item.sections.map(section=>({...section,actions:{...section.actions,buttons:section.actions.buttons.map(button=>({...button,
|
||||
label:button.label.replace(/^(Copy|Send) negative/,`$1 ${COMPARED[index]} negative`)}))}}))}))}},
|
||||
// D47: the gallery's body is the Images collection (layout row, cards, side panel), so only its place, frame and
|
||||
// heading are compared here; the cards are the Images comparison's.
|
||||
gallery:history.gallery&&withoutHeight(history.gallery),
|
||||
}};
|
||||
}
|
||||
const COMPARED=['baseline','selected'];
|
||||
const withoutHeight=({height,...gallery})=>gallery;
|
||||
|
||||
function compare(reference,actual) {
|
||||
const only=inventory=>Object.fromEntries(SECTIONS.map(key=>[key,key==='history'&&inventory.history?.gallery
|
||||
?{...inventory.history,gallery:withoutHeight(inventory.history.gallery)}:inventory[key]]));
|
||||
return diffInventory(stripExact(only(mapHistoryConcept(reference))),stripExact(only(actual)));
|
||||
}
|
||||
|
||||
/** The comparison as a gate: throws, naming every differing path, unless production equals the mapped concept. */
|
||||
function assertHistoryMatch(reference,actual) {
|
||||
const diff=compare(reference,actual);
|
||||
if(diff.length)throw new Error(`the Prompt history differs from the concept at ${diff.map(item=>item.path).join(', ')}`);
|
||||
}
|
||||
|
||||
// Two frames, then every running transition finished.
|
||||
const settle=page=>page.evaluate(async()=>{
|
||||
await new Promise(resolve=>requestAnimationFrame(()=>requestAnimationFrame(resolve)));
|
||||
await Promise.race([Promise.all(document.getAnimations().map(animation=>animation.finished.catch(()=>{}))),
|
||||
new Promise(resolve=>setTimeout(resolve,2000))]);
|
||||
});
|
||||
const decoded=page=>page.evaluate(async()=>{
|
||||
await document.fonts.ready;
|
||||
await Promise.all([...document.querySelectorAll('.history-gallery img')].map(img=>{img.loading='eager';return img.decode().catch(()=>{});}));
|
||||
});
|
||||
|
||||
const dot=(page,name)=>page.getByRole('button',{name:`Select negative ${name}`,exact:true});
|
||||
const compareButton=page=>page.getByRole('button',{name:'Compare negatives',exact:true});
|
||||
const galleryCards=page=>page.locator('.history-gallery .media-card');
|
||||
|
||||
/** Press a variant dot and wait until the summary and the gallery follow it. */
|
||||
async function selectVariant(page,name) {
|
||||
await dot(page,name).click();
|
||||
await expect(dot(page,name)).toHaveAttribute('aria-pressed','true');
|
||||
await expect(page.locator('.history-gallery .panel-heading .small')).toHaveText(`${name} · same positive prompt`);
|
||||
await expect(galleryCards(page).first()).toBeVisible();
|
||||
await page.mouse.move(0,0);await decoded(page);await settle(page);
|
||||
}
|
||||
async function setCompare(page,pressed) {
|
||||
const button=compareButton(page);
|
||||
if(await button.getAttribute('aria-pressed')!==String(pressed))await button.click();
|
||||
await expect(compareButton(page)).toHaveAttribute('aria-pressed',String(pressed));
|
||||
await page.mouse.move(0,0);await settle(page);
|
||||
}
|
||||
|
||||
async function openConceptHistory(page,theme,motion) {
|
||||
await openConcept(page,'prompt/alpine',theme,motion);
|
||||
await expect(page.locator('.timeline .variant-lane')).toHaveCount(LANES);
|
||||
await selectVariant(page,SELECTED);
|
||||
}
|
||||
async function openProductionHistory(page,theme,motion,positiveId) {
|
||||
await openProduction(page,`library/${positiveId}`,theme,motion);
|
||||
await expect(page.locator('.timeline .variant-lane')).toHaveCount(LANES);
|
||||
await expect(page.locator('article.surface .card-meta > span').first()).toBeVisible();
|
||||
await selectVariant(page,SELECTED);
|
||||
}
|
||||
|
||||
// D45: the recorded production rules that lay the panel out (workspace.css): the panel heading and the legend row wrap,
|
||||
// All images wraps its label, and a lane's text wraps before a shown repeat-run dot. Applied to the concept so the browser
|
||||
// lays out what follows; at 1600px nothing wraps.
|
||||
const D45_LAYOUT=`.timeline-panel > .panel-heading, .timeline-lower { flex-wrap: wrap; }
|
||||
.variant-lane:has(> .run-dot:not([hidden])) > span { max-inline-size: 139px; overflow-wrap: anywhere; }
|
||||
.ca-tools .timeline-lower > button { max-inline-size: 100%; white-space: normal; }`;
|
||||
|
||||
/**
|
||||
* The concept laid out as production lists it: lanes newest first with their offsets (D46), "1 run" in the singular
|
||||
* (D35), the approved Show archived button after + Add negative (#235) and D45's layout rules. Returns the undo.
|
||||
*/
|
||||
async function conceptAsRecorded(page) {
|
||||
const restoreLayout=await mutate(page,{css:D45_LAYOUT});
|
||||
await page.evaluate(label=>{
|
||||
const timeline=document.querySelector('.timeline'),lanes=[...timeline.querySelectorAll(':scope > .variant-lane')];
|
||||
const texts=lanes.map(lane=>[lane.querySelector(':scope > span'),lane.querySelector(':scope > span').textContent]);
|
||||
lanes.reverse().forEach((lane,index)=>{lane.style.setProperty('--lane-offset',`${38+index*47}px`);timeline.append(lane);});
|
||||
for(const [span,text] of texts)span.textContent=text.replace(/(^|\D)1 runs$/,(match,before)=>`${before}1 run`);
|
||||
const add=[...document.querySelectorAll('.timeline-panel .row-actions > button')].find(node=>node.textContent==='+ Add negative');
|
||||
const archived=add.cloneNode(false);archived.removeAttribute('data-action');archived.removeAttribute('data-prompt');
|
||||
archived.setAttribute('aria-pressed','false');archived.textContent=label;add.after(archived);
|
||||
window.__historyOracle={timeline,order:[...lanes].reverse(),texts,archived};
|
||||
},SHOW_ARCHIVED);
|
||||
await settle(page);
|
||||
return ()=>page.evaluate(()=>{
|
||||
const {timeline,order,texts,archived}=window.__historyOracle;
|
||||
order.forEach((lane,index)=>{lane.style.setProperty('--lane-offset',`${38+index*47}px`);timeline.append(lane);});
|
||||
for(const [span,text] of texts)span.textContent=text;
|
||||
archived.remove();delete window.__historyOracle;
|
||||
}).then(restoreLayout);
|
||||
}
|
||||
|
||||
/** The concept inventory, with `oracle`: the same page laid out as recorded (`conceptAsRecorded`). */
|
||||
async function conceptInventory(page) {
|
||||
const reference=await inventoryOf(page,'#main');
|
||||
const restore=await conceptAsRecorded(page);
|
||||
const oracle=await inventoryOf(page,'#main');
|
||||
await restore();await settle(page);
|
||||
return {...reference,oracle};
|
||||
}
|
||||
|
||||
const negativeText=page=>page.locator('article.surface .prompt-section.negative .prompt-text').textContent();
|
||||
const galleryOf=async page=>({context:await page.locator('.history-gallery .panel-heading .small').textContent(),
|
||||
cards:await galleryCards(page).locator('.image-open').evaluateAll(nodes=>nodes.map(node=>node.getAttribute('aria-label')))});
|
||||
|
||||
/** A dialog the page opens: its accessible name, how many media cards it shows once loaded and its captions; then closed. */
|
||||
async function dialogOf(page,trigger,{name,cards}) {
|
||||
await trigger.click();
|
||||
const dialog=page.getByRole('dialog',{name,exact:true});
|
||||
await expect(dialog).toBeVisible();
|
||||
if(cards)await expect(dialog.locator('.media-card')).toHaveCount(cards);
|
||||
// D47: neither document's media dialog carries a collection caption ("Linked images", "N loaded · …").
|
||||
const captions=await dialog.locator('.section-caption').count();
|
||||
const result=cards?{name,cards:await dialog.locator('.media-card').count(),captions}:{name,captions};
|
||||
await page.keyboard.press('Escape');await expect(dialog).toHaveCount(0);
|
||||
return result;
|
||||
}
|
||||
|
||||
/** The behaviors Task F5 lists, with identical selectors in both documents. */
|
||||
async function behaviorsOf(page,{sent,runDialog}) {
|
||||
await selectVariant(page,'Quality baseline');
|
||||
const before={negative:await negativeText(page),gallery:await galleryOf(page)};
|
||||
await selectVariant(page,SELECTED);
|
||||
const after={negative:await negativeText(page),gallery:await galleryOf(page)};
|
||||
const pressed=[await compareButton(page).getAttribute('aria-pressed')];
|
||||
await setCompare(page,true);pressed.push(await compareButton(page).getAttribute('aria-pressed'));
|
||||
const compared=await page.locator('section.surface:not(.timeline-panel) > h2').textContent();
|
||||
await setCompare(page,false);pressed.push(await compareButton(page).getAttribute('aria-pressed'));
|
||||
const all=await dialogOf(page,page.getByRole('button',{name:'All images for this positive',exact:true}),{name:'All images for this positive',cards:4});
|
||||
const run=await dialogOf(page,page.getByRole('button',{name:'Show repeat run for Quality baseline',exact:true}),{name:runDialog});
|
||||
await page.locator('.heading-actions').getByRole('button',{name:`Send prompt pair to ComfyUI: ${TITLE}`,exact:true}).click();
|
||||
const send=await sent(page);
|
||||
await page.getByRole('button',{name:'← Prompt library',exact:true}).click();
|
||||
await expect(page.locator('h1#page-title')).toHaveText('Prompt library');
|
||||
return {variant:{before,after},pressed,compared,all,run,send,back:await page.locator('h1#page-title').textContent()};
|
||||
}
|
||||
|
||||
/** The concept's demo sender: the roles its preview lists, with their text. */
|
||||
async function conceptSent(page) {
|
||||
const modal=page.locator('dialog#modal[open]');
|
||||
await expect(modal.locator('#modal-title')).toHaveText('Send to ComfyUI');
|
||||
const roles=await modal.locator('.send-preview').evaluateAll(nodes=>Object.fromEntries(nodes.map(node=>
|
||||
[node.querySelector('h3').textContent.startsWith('+')?'positive':'negative',node.querySelector('p').textContent])));
|
||||
await page.keyboard.press('Escape');await expect(modal).toHaveCount(0);
|
||||
return roles;
|
||||
}
|
||||
/** Production's acknowledged sender against a bridge host page: the roles its preview asked the host to change. */
|
||||
function productionSent(host) {
|
||||
return async page=>{
|
||||
const sender=page.getByRole('dialog',{name:'Send to ComfyUI',exact:true});
|
||||
await sender.getByRole('combobox',{name:'ComfyUI window',exact:true}).selectOption({label:'Concept ComfyUI'});
|
||||
await sender.getByRole('combobox',{name:'Destination node',exact:true}).selectOption({label:'Concept destination'});
|
||||
await sender.getByRole('button',{name:'Preview changes',exact:true}).click();
|
||||
await expect.poll(()=>host.evaluate(()=>window.previewed.length)).toBe(1);
|
||||
const roles=await host.evaluate(()=>window.previewed.pop());
|
||||
await page.keyboard.press('Escape');await expect(sender).toHaveCount(0);
|
||||
return roles;
|
||||
};
|
||||
}
|
||||
async function openHost(context) {
|
||||
const host=await context.newPage();await host.goto('/prompt_manager/workspace#/overview');
|
||||
await host.evaluate(async()=>{
|
||||
const {createBridgeHost}=await import('/prompt_manager/js/shared/toolkit/prompt-bridge.js');
|
||||
const target={id:'pair-1',label:'Concept destination',type:'PromptManagerPair',roles:['positive','negative']};
|
||||
window.previewed=[];
|
||||
window.conceptHost=createBridgeHost({origin:location.origin,namespace:'/',label:'Concept ComfyUI',adapter:{
|
||||
listTargets:()=>[target],
|
||||
preview:(_id,roles)=>{window.previewed.push(roles);return {preview:{target,fields:Object.entries(roles).map(([role,after])=>({role,before:`Previous ${role}`,after})),
|
||||
composition:{prepend:'',append:'',prependConnected:false,appendConnected:false,before:'Previous positive',after:roles.positive??'Previous positive'}},snapshot:roles};},
|
||||
apply:roles=>({targetId:target.id,values:roles}),
|
||||
}});
|
||||
});
|
||||
return host;
|
||||
}
|
||||
|
||||
test.beforeEach(async({page,request},testInfo)=>{
|
||||
testInfo.manifest=await seedConcept(request);
|
||||
await page.addInitScript(()=>{window.copied=[];Object.defineProperty(navigator,'clipboard',{configurable:true,value:{writeText:async text=>{window.copied.push(text);}}});});
|
||||
});
|
||||
|
||||
for(const [width,height] of SIZES)for(const theme of THEMES)test(`history geometry, tokens, controls and behaviors match the concept ${width}x${height} ${theme}`,async({page,context},testInfo)=>{
|
||||
test.setTimeout(240_000);
|
||||
await page.setViewportSize({width,height});
|
||||
const positiveId=testInfo.manifest.prompts[0].positive_id;
|
||||
const host=await openHost(context);
|
||||
try {
|
||||
const result={};
|
||||
for(const motion of MOTIONS) {
|
||||
const capture=motion==='no-preference';
|
||||
await openConceptHistory(page,theme,motion);
|
||||
const reference=await conceptInventory(page);
|
||||
await setCompare(page,true);
|
||||
const referenceCompared=await conceptInventory(page);
|
||||
if(capture)await captureCanonicalState(page,testInfo,`concept-history-${width}-${theme}`);
|
||||
const referenceBehaviors=capture?await behaviorsOf(page,{sent:conceptSent,runDialog:'Repeat run'}):null;
|
||||
|
||||
await openProductionHistory(page,theme,motion,positiveId);
|
||||
const actual=await inventoryOf(page,'#workspace-main');
|
||||
await setCompare(page,true);
|
||||
const actualCompared=await inventoryOf(page,'#workspace-main');
|
||||
if(capture)await captureCanonicalState(page,testInfo,`production-history-${width}-${theme}`);
|
||||
const actualBehaviors=capture?await behaviorsOf(page,{sent:productionSent(host),runDialog:'Recorded run'}):null;
|
||||
result[motion]={reference,actual,diff:compare(reference,actual),referenceCompared,actualCompared,
|
||||
compareDiff:compare(referenceCompared,actualCompared),referenceBehaviors,actualBehaviors};
|
||||
}
|
||||
const path=testInfo.outputPath('history-geometry.json');await writeFile(path,JSON.stringify(result,null,2));
|
||||
await testInfo.attach('history-geometry',{path,contentType:'application/json'});
|
||||
for(const [motion,{diff,compareDiff,actual,referenceCompared}] of Object.entries(result)) {
|
||||
// The concept itself is what this comparison assumes (style.css 709–722, 1059–1179, 1426, 1546).
|
||||
const {summary,variants,compare:compared,gallery}=referenceCompared.history;
|
||||
expect.soft(summary,`${motion} concept surface`).toMatchObject({paddingTop:'22px',paddingLeft:'22px',borderTopLeftRadius:'7px',borderTopWidth:'1px'});
|
||||
// D24: the fixture's four recorded runs give the concept's own "4 tracked runs"; production reads them.
|
||||
expect.soft(summary.meta.items.map(item=>item.text),`${motion} concept counts`).toEqual(['4 tracked runs','4 images','3 negative variants']);
|
||||
expect.soft(actual.history.summary.meta.items.map(item=>item.text),`${motion} tracked runs`).toEqual(['4 tracked runs','4 images','3 negative variants']);
|
||||
expect.soft(variants.timeline,`${motion} concept timeline`).toMatchObject({paddingTop:'18px',paddingLeft:'20px',borderTopLeftRadius:'6px'});
|
||||
variants.timeline.lanes.forEach((lane,index)=>{
|
||||
expect.soft(lane,`${motion} concept lane ${index}`).toMatchObject({offset:`${38+index*47}px`,marginLeft:`${38+index*47}px`,height:57});
|
||||
expect.soft(lane.dot,`${motion} concept dot ${index}`).toMatchObject({width:'29px',height:'29px',text:'−'});
|
||||
});
|
||||
expect.soft(variants.timeline.lanes.map(lane=>lane.run&&{width:lane.run.width,left:lane.run.left}),`${motion} concept run dots`).toEqual([{width:'12px',left:'190px'},null,null]);
|
||||
expect.soft(compared.grid,`${motion} concept compare grid`).toMatchObject({columns:width>700?2:1,columnGap:'15px'});
|
||||
expect.soft(compared.added.map(chip=>chip.text),`${motion} concept added terms`).toEqual(['+ distorted architecture','+ watermark','+ text']);
|
||||
expect.soft(compared.added[0],`${motion} concept diff chip`).toMatchObject({fontSize:'11px',paddingTop:'3px',paddingLeft:'7px',borderTopLeftRadius:'4px',display:'inline-block'});
|
||||
expect.soft(gallery,`${motion} concept gallery`).toMatchObject({marginTop:'24px',heading:{h2:{text:'Shared gallery',fontSize:'14px'},note:{text:`${SELECTED} · same positive prompt`}}});
|
||||
expect.soft(diff,`${motion} differences after the recorded deviations`).toEqual([]);
|
||||
expect.soft(compareDiff,`${motion} differences with Compare negatives pressed`).toEqual([]);
|
||||
expect.soft(actual.noPageScroll,`${motion} no page-level horizontal scroll`).toBe(true);
|
||||
}
|
||||
const {referenceBehaviors:concept,actualBehaviors:production}=result['no-preference'];
|
||||
expect.soft(production,'behaviors').toEqual({...concept,run:{...concept.run,name:'Recorded run'}});
|
||||
} finally {await host.evaluate(()=>window.conceptHost.dispose());await host.close();}
|
||||
});
|
||||
|
||||
// The comparison is only worth something if it notices a change (spec §7 mutation check): one CSS mutation (the variant
|
||||
// dot's width) and one DOM mutation (Compare negatives removed), each applied after an exact match.
|
||||
test('history comparison detects mutations',async({page},testInfo)=>{
|
||||
await page.setViewportSize({width:1600,height:1100});
|
||||
await openConceptHistory(page,'dark','no-preference');
|
||||
const reference=await conceptInventory(page);
|
||||
await openProductionHistory(page,'dark','no-preference',testInfo.manifest.prompts[0].positive_id);
|
||||
const measure=()=>inventoryOf(page,'#workspace-main');
|
||||
assertHistoryMatch(reference,await measure());
|
||||
|
||||
const restoreCss=await mutate(page,{css:'.ca-tools .variant-dot { width: 20px }'});
|
||||
const cssMutated=await measure();
|
||||
expect(()=>assertHistoryMatch(reference,cssMutated)).toThrow(/history\.variants\.timeline\.lanes\.0\.dot\.width/);
|
||||
await restoreCss();await settle(page);
|
||||
assertHistoryMatch(reference,await measure());
|
||||
|
||||
await compareButton(page).evaluate(node=>node.remove());
|
||||
const domMutated=await measure();
|
||||
expect(()=>assertHistoryMatch(reference,domMutated)).toThrow(/history\.variants\.heading\.actions\.buttons\.0\.text/);
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,diffInventory,mutate,stripExact,countCopy,reflow,withoutItems} from './concept-inventory.mjs';
|
||||
|
||||
@@ -7,7 +7,7 @@ import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,
|
||||
// images of the concept's six prompts, its seeds, sizes and favorites). Heading, toolbar, filter row, caption, grid,
|
||||
// the first six cards, the continuation and the panel box are compared at 1600x1100 and 390x844 in both themes, and the
|
||||
// page's behaviors under normal and reduced motion; grid, cards and panel also at 1200x900 and 1800x1100 (2 and 4
|
||||
// columns, the 275px and 310px panel). The topbar and sidebar (D5, D28, D29) are the shell spec's.
|
||||
// columns, the 275px and 310px panel). The topbar and sidebar (D28, D29, D52, D53) are the shell spec's.
|
||||
// Paired captures are artifacts only; there are no PNG baselines.
|
||||
test.use({locale:'en-US',timezoneId:'UTC'});
|
||||
|
||||
@@ -85,8 +85,8 @@ const nocase=(a,b)=>{const x=a.toLowerCase(),y=b.toLowerCase();return x<y?-1:x>y
|
||||
|
||||
export function mapImagesConcept(reference,{manifest,now=new Date()}) {
|
||||
const paths=new Map(manifest.images.map(image=>[image.filename,image.path]));
|
||||
const toolbar=withoutItems(reference.toolbar,item=>item.text==='Save view');
|
||||
// D16 (follow-up PR): no Save view until the saved-views store ships. D14: the sort appends production's two sorts.
|
||||
// D14: the sort appends production's two sorts. Save view is the concept's own (saved views, #249 D16 retired).
|
||||
const toolbar={...reference.toolbar};
|
||||
toolbar.items=toolbar.items.map(item=>item.id==='sort'?{...item,options:[...item.options,'Oldest first','Name Z–A']}:item);
|
||||
// D36: the category list is the Library's real list, sorted alphabetically (database/library.py `ORDER BY category
|
||||
// COLLATE NOCASE, category`); the concept lists its samples in prompt order (app.js 348). Only the order is mapped.
|
||||
@@ -149,12 +149,15 @@ function compareLoadingRow(reference,actual,count) {
|
||||
return diffInventory(unplaced(mapContinuation(reference,{loaded,total})),unplaced(actual));
|
||||
}
|
||||
|
||||
/** D15: the Import dialog's two real operations, with the ruled intro and the Rescan wording. */
|
||||
/**
|
||||
* D15: the Import dialog's real operations, with the ruled intro and the Rescan wording. D55: the concept's own "Scan an
|
||||
* output folder → Review scan" row follows Rescan as "Scan for metadata", opening the concept's "Review metadata scan".
|
||||
*/
|
||||
function mapImportConcept(reference) {
|
||||
return {...reference,
|
||||
intro:{...reference.intro,text:'Image inspection and output scanning have different effects. Choose the workflow you want.'},
|
||||
rows:reference.rows.filter(row=>row.title!=='Import portable recipes').map(row=>row.title==='Scan an output folder'
|
||||
?{...row,description:'Find files added or removed in the output folders.',buttons:['Rescan']}:row)};
|
||||
rows:reference.rows.filter(row=>row.title!=='Import portable recipes').flatMap(row=>row.title==='Scan an output folder'
|
||||
?[{...row,description:'Find files added or removed in the output folders.',buttons:['Rescan']},{...row,title:'Scan for metadata'}]:[row])};
|
||||
}
|
||||
|
||||
const SECTIONS=['heading','toolbar','filterRow','bulkbar','caption','grid','cards','continuation','panel','noPageScroll'];
|
||||
@@ -329,9 +332,9 @@ for(const [width,height] of SIZES)for(const theme of THEMES)test(`images page ge
|
||||
const conceptLoad=await openConceptLoaded(page,theme,motion);
|
||||
const reference=await inventoryOf(page,'#main');
|
||||
if(capture)await captureCanonicalState(page,testInfo,`concept-images-${width}-${theme}`);
|
||||
// The browser as the re-layout's oracle: the concept page itself without Save view, then without the sample tag.
|
||||
// The browser as the re-layout's oracle: the concept page itself without the sample tag.
|
||||
const removed={};
|
||||
for(const [key,selector] of [['toolbar','.toolbar > [data-action="save-view"]'],['filterRow','.filter-row > .sample-tag']]) {
|
||||
for(const [key,selector] of [['filterRow','.filter-row > .sample-tag']]) {
|
||||
const restore=await mutate(page,{remove:[selector]});
|
||||
removed[key]=(await inventoryOf(page,'#main'))[key];
|
||||
await restore();
|
||||
@@ -385,12 +388,10 @@ for(const [width,height] of SIZES)for(const theme of THEMES)test(`images page ge
|
||||
expect.soft(reference.cards[0].info.p.text,`${motion} concept subtitle`).toBe(`${first.width} × ${first.height} · PNG · Seed ${first.seed}`);
|
||||
expect.soft(reference.cards[0].info.h3.fontSize,`${motion} concept h3 size`).toBe('11px');
|
||||
expect.soft(referenceImport.rows.map(row=>row.title),`${motion} concept Import rows`).toEqual(['Inspect a local image','Scan an output folder','Import portable recipes']);
|
||||
// The row model reproduces the concept's own rows, and the concept page itself once Save view (D16) or the
|
||||
// sample tag (D29) is removed from it at runtime: the browser is the oracle for the remainders the mapping uses.
|
||||
// The row model reproduces the concept's own rows, and the concept page itself once the sample tag (D29) is
|
||||
// removed from it at runtime: the browser is the oracle for the remainders the mapping uses.
|
||||
expect.soft(diffInventory(stripExact(reflow(reference.toolbar,reference.toolbar.items)),stripExact(reference.toolbar)),`${motion} toolbar model`).toEqual([]);
|
||||
expect.soft(diffInventory(stripExact(reflow(reference.filterRow,reference.filterRow.items)),stripExact(reference.filterRow)),`${motion} filter row model`).toEqual([]);
|
||||
expect.soft(diffInventory(stripExact(withoutItems(reference.toolbar,item=>item.text==='Save view')),stripExact(removed.toolbar)),
|
||||
`${motion} toolbar without Save view`).toEqual([]);
|
||||
expect.soft(diffInventory(stripExact(withoutItems(reference.filterRow,item=>String(item.className).includes('sample-tag'))),stripExact(removed.filterRow)),
|
||||
`${motion} filter row without the sample tag`).toEqual([]);
|
||||
expect.soft(reference.toolbar.items.find(item=>item.id==='category-filter').options,`${motion} concept categories`).toEqual(CONCEPT_CATEGORIES);
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {SIZES,THEMES,seedConcept,openConcept,openProduction,inspectorOf,diffInventory,mutate,countCopy} from './concept-inventory.mjs';
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,diffInventory,mutate,stripExact,countCopy,reflow,withoutItems} from './concept-inventory.mjs';
|
||||
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,diffInventory,mutate,stripExact,countCopy,reflow} from './concept-inventory.mjs';
|
||||
|
||||
// Task E3 of #249: the production Prompt library shelf against the immutable concept (app.js renderLibrary a572–586,
|
||||
// style.css s648–708), on the concept-shaped real fixture (the concept's six prompts with their negatives, categories,
|
||||
@@ -18,8 +18,8 @@ const CONCEPT_PROMPTS=6;
|
||||
const nocase=(a,b)=>{const x=a.toLowerCase(),y=b.toLowerCase();return x<y?-1:x>y?1:a<b?-1:a>b?1:0;};
|
||||
export function mapLibraryConcept(reference) {
|
||||
const library=reference.library;
|
||||
// D16 (follow-up PR): no Save view until the saved-views store ships; the remaining items flow again (`reflow`).
|
||||
const toolbar=withoutItems(library.toolbar,item=>item.text==='Save view');
|
||||
// Save view is the concept's own (saved views, #249 D16 retired).
|
||||
const toolbar={...library.toolbar};
|
||||
toolbar.items=toolbar.items.map(item=>{
|
||||
// D14: the Library appends its Oldest first after the concept's three sorts.
|
||||
if(item.id==='sort')return {...item,options:[...item.options,'Oldest first']};
|
||||
@@ -158,6 +158,10 @@ async function filteredOf(page,apply,undo) {
|
||||
await apply();
|
||||
await expect.poll(()=>captionOf(page)).not.toBe(before);
|
||||
await expect.poll(()=>page.locator(CARDS).count()).toBeLessThan(CONCEPT_PROMPTS);
|
||||
// Production empties the shelf when the filter applies, writes the answer's caption at once and places its cards
|
||||
// in the next animation frame (one paint for a viewer), so the empty shelf already counts as "fewer": read the
|
||||
// titles only after that frame, as the Images and Inspector comparisons do.
|
||||
await settle(page);
|
||||
const result={caption:await captionOf(page),titles:await titlesOf(page)};
|
||||
await undo();
|
||||
await expect.poll(()=>captionOf(page)).toBe(before);
|
||||
@@ -251,11 +255,7 @@ for(const [width,height] of SIZES)for(const theme of THEMES)test(`library geomet
|
||||
await openConceptLibrary(page,theme,motion);
|
||||
const reference=await conceptInventory(page);
|
||||
if(capture)await captureCanonicalState(page,testInfo,`concept-library-${width}-${theme}`);
|
||||
// The browser as the re-layout's oracle: the concept page itself without Save view (D16).
|
||||
const restore=await mutate(page,{remove:['.toolbar > [data-action="save-view"]']});
|
||||
const removed=(await inventoryOf(page,'#main')).library.toolbar;
|
||||
await restore();
|
||||
// ...and with the D23 clamp applied to its cards.
|
||||
// The browser as the re-layout's oracle: the concept page itself with the D23 clamp applied to its cards.
|
||||
const restoreClamp=await mutate(page,{css:CLAMP_CSS});
|
||||
const clamped=(await inventoryOf(page,'#main')).library;
|
||||
await restoreClamp();
|
||||
@@ -269,13 +269,13 @@ for(const [width,height] of SIZES)for(const theme of THEMES)test(`library geomet
|
||||
const actual=await inventoryOf(page,'#workspace-main');
|
||||
if(capture)await captureCanonicalState(page,testInfo,`production-library-${width}-${theme}`);
|
||||
const actualBehaviors=capture?await behaviorsOf(page,{back:()=>page.getByRole('button',{name:'← Prompt library',exact:true}).click(),sent:productionSent(host)}):null;
|
||||
result[motion]={reference,removed,sorted,clamped,actual,diff:compare(reference,actual),referenceBehaviors,actualBehaviors};
|
||||
result[motion]={reference,sorted,clamped,actual,diff:compare(reference,actual),referenceBehaviors,actualBehaviors};
|
||||
}
|
||||
const path=testInfo.outputPath('library-geometry.json');await writeFile(path,JSON.stringify(result,null,2));
|
||||
await testInfo.attach('library-geometry',{path,contentType:'application/json'});
|
||||
|
||||
for(const [motion,{reference,removed,sorted,clamped,actual,diff,referenceBehaviors:concept,actualBehaviors:production}] of Object.entries(result)) {
|
||||
const {grid,cards,toolbar}=reference.library,card=cards[0];
|
||||
for(const [motion,{reference,sorted,clamped,actual,diff,referenceBehaviors:concept,actualBehaviors:production}] of Object.entries(result)) {
|
||||
const {grid,cards}=reference.library,card=cards[0];
|
||||
// The concept itself is what this comparison assumes (style.css s648–708, 1424–1428, 1475–1483).
|
||||
expect.soft(grid,`${motion} concept grid`).toMatchObject({columns:narrow?1:2,columnGap:'18px',rowGap:'18px',cardCount:CONCEPT_PROMPTS});
|
||||
expect.soft(card,`${motion} concept card`).toMatchObject({borderTopLeftRadius:'8px',borderTopWidth:'1px',overflow:'hidden'});
|
||||
@@ -291,7 +291,6 @@ for(const [width,height] of SIZES)for(const theme of THEMES)test(`library geomet
|
||||
expect.soft(card.meta.items.map(item=>item.text),`${motion} concept meta text`).toEqual(['− 3 negative variants','4 runs','4 images']);
|
||||
expect.soft(card.footer,`${motion} concept footer`).toMatchObject({paddingTop:'11px',paddingRight:'18px',paddingBottom:'11px',paddingLeft:'18px',borderTopWidth:'1px',borderTopStyle:'solid'});
|
||||
expect.soft(card.footer.facts.text,`${motion} concept footer facts`).toBe('★★★★★ · Used today');
|
||||
expect.soft(diffInventory(stripExact(withoutItems(toolbar,item=>item.text==='Save view')),stripExact(removed)),`${motion} toolbar without Save view`).toEqual([]);
|
||||
expect.soft(diffInventory(stripExact(cards.map(item=>sortedTags(item.tags))),stripExact(sorted)),`${motion} tags in alphabetical order`).toEqual([]);
|
||||
expect.soft(diffInventory(stripExact(clampedCards(reference.library)),stripExact({grid:clamped.grid,cards:clamped.cards})),`${motion} clamped cards`).toEqual([]);
|
||||
expect.soft(diff,`${motion} differences after the recorded deviations`).toEqual([]);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
|
||||
// Artifact-only captures of the editor states that have no concept counterpart (archive confirmation, Show archived,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {readFileSync} from 'node:fs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
|
||||
const pair={id:4,positive_text:'Library positive',negative_text:'Library negative',category:'Portrait',tags:['studio']};
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
|
||||
const moduleUrl = '/prompt_manager/js/shared/toolkit/prompt-bridge.js';
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
|
||||
const workspace='/prompt_manager/workspace#/images';
|
||||
const details=page=>page.getByRole('complementary',{name:'Selected image details',exact:true});
|
||||
|
||||
+102
-6
@@ -1,19 +1,24 @@
|
||||
"""Serve production pages and API routes with disposable state for browser tests."""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import types
|
||||
|
||||
from aiohttp import web
|
||||
from aiohttp.web_runner import GracefulExit
|
||||
from PIL import Image
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
|
||||
from tests.browser.workspace_activity_fixture import register_activity_commands
|
||||
from tests.browser.workspace_concept_fixture import (
|
||||
MAX_EXTRA_IMAGES as CONCEPT_MAX_EXTRA_IMAGES,
|
||||
MAX_SEEDLESS_IMAGES as CONCEPT_MAX_SEEDLESS_IMAGES,
|
||||
@@ -159,6 +164,7 @@ def create_app(root: Path) -> web.Application:
|
||||
library_editor_commands = register_library_editor_commands(routes, api, root)
|
||||
register_vocabulary_commands(routes, api, root)
|
||||
reveal_stub = register_reveal_stub(routes, api, root)
|
||||
register_activity_commands(app, routes, api)
|
||||
|
||||
async def canonical_asset(request: web.Request) -> web.Response:
|
||||
filename, content_type = CANONICAL_ASSETS[request.path]
|
||||
@@ -181,7 +187,11 @@ def create_app(root: Path) -> web.Application:
|
||||
) -> None:
|
||||
with api.db.model.get_connection() as conn:
|
||||
for table in (
|
||||
"saved_views",
|
||||
"image_favorites",
|
||||
"media_image_tags",
|
||||
"media_annotation_bytes",
|
||||
"media_annotation_files",
|
||||
"image_prompt_pairs",
|
||||
"prompt_pair_tags",
|
||||
"prompt_pairs",
|
||||
@@ -469,6 +479,29 @@ def create_app(root: Path) -> web.Application:
|
||||
)
|
||||
return web.json_response({"success": True, **manifest})
|
||||
|
||||
@routes.post("/__test__/saved-views/raw")
|
||||
async def insert_raw_saved_view(request: web.Request) -> web.Response:
|
||||
"""Store a view row as an older build wrote it, bypassing the API's rules."""
|
||||
data = await request.json()
|
||||
if (
|
||||
not isinstance(data, dict)
|
||||
or set(data) != {"page", "name", "state"}
|
||||
or not all(isinstance(data[key], str) for key in data)
|
||||
):
|
||||
raise web.HTTPBadRequest()
|
||||
|
||||
def insert() -> int:
|
||||
with api.db.model.get_connection() as conn:
|
||||
return conn.execute(
|
||||
"INSERT INTO saved_views (page, name, state, position, "
|
||||
"created_at, updated_at) VALUES (?, ?, ?, 0, 'x', 'x')",
|
||||
(data["page"], data["name"], data["state"]),
|
||||
).lastrowid
|
||||
|
||||
return web.json_response(
|
||||
{"success": True, "id": await api._run_in_executor(insert)}
|
||||
)
|
||||
|
||||
@routes.post("/__test__/add-nested-image")
|
||||
async def add_nested_image(request: web.Request) -> web.Response:
|
||||
destination = root / "output" / "project" / "day" / "new-output.png"
|
||||
@@ -489,14 +522,77 @@ def create_app(root: Path) -> web.Application:
|
||||
return app
|
||||
|
||||
|
||||
def main() -> None:
|
||||
HOST = "127.0.0.1"
|
||||
PORT_ANNOUNCEMENT = "PM_TEST_SERVER_PORT="
|
||||
|
||||
|
||||
def bind_listener(port: int) -> socket.socket:
|
||||
"""Bind the server's socket before the app is built.
|
||||
|
||||
``port`` 0 lets the OS choose a free port, which this process then holds,
|
||||
so parallel launchers never race for one between choosing and binding.
|
||||
"""
|
||||
listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
try:
|
||||
if port and os.name != "nt":
|
||||
# aiohttp's own default on POSIX; on Windows it would let a
|
||||
# second process share the port.
|
||||
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
listener.bind((HOST, port))
|
||||
listener.listen(128)
|
||||
except OSError:
|
||||
listener.close()
|
||||
raise
|
||||
return listener
|
||||
|
||||
|
||||
def _graceful_exit() -> None:
|
||||
raise GracefulExit()
|
||||
|
||||
|
||||
def exit_when_stdin_closes(loop: asyncio.AbstractEventLoop) -> threading.Thread:
|
||||
"""Stop the app gracefully once stdin reaches end of file.
|
||||
|
||||
The launcher keeps the write end of this process's stdin pipe, and the OS
|
||||
closes it however the launcher dies, SIGKILL included. GracefulExit runs
|
||||
aiohttp's cleanup (closing the database) and lets the temp root be removed,
|
||||
on every OS, where a signal from a dead parent would not.
|
||||
"""
|
||||
|
||||
def watch() -> None:
|
||||
while sys.stdin.buffer.read(65536):
|
||||
pass
|
||||
loop.call_soon_threadsafe(_graceful_exit)
|
||||
|
||||
thread = threading.Thread(target=watch, name="stdin-eof", daemon=True)
|
||||
thread.start()
|
||||
return thread
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--port", type=int, default=8765)
|
||||
args = parser.parse_args()
|
||||
parser.add_argument(
|
||||
"--port", type=int, default=8765, help="0 binds an OS-chosen free port"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--exit-on-stdin-eof",
|
||||
action="store_true",
|
||||
help="shut down when stdin closes (the launcher's parent-death guard)",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
listener = bind_listener(args.port)
|
||||
# One machine-readable line tells the launcher which port this process holds.
|
||||
print(f"{PORT_ANNOUNCEMENT}{listener.getsockname()[1]}", flush=True)
|
||||
loop = asyncio.new_event_loop()
|
||||
if args.exit_on_stdin_eof:
|
||||
exit_when_stdin_closes(loop)
|
||||
original_cwd = os.getcwd()
|
||||
with tempfile.TemporaryDirectory(prefix="pm-browser-") as directory:
|
||||
web.run_app(
|
||||
create_app(Path(directory).resolve()), host="127.0.0.1", port=args.port
|
||||
)
|
||||
try:
|
||||
web.run_app(create_app(Path(directory).resolve()), sock=listener, loop=loop)
|
||||
finally:
|
||||
# create_app works inside the root; Windows cannot remove a process's cwd.
|
||||
os.chdir(original_cwd)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,diffInventory,focusRingOf,hoverStateOf,mutate} from './concept-inventory.mjs';
|
||||
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept as openConceptPage,openProduction,inventoryOf,diffInventory,focusRingOf,hoverStateOf,mutate} from './concept-inventory.mjs';
|
||||
|
||||
// Task A6 of #249: the production shell (sidebar, topbar, heading, footer) against the immutable concept, with the
|
||||
// concept-shaped real fixture so the sidebar counts are the concept's own (24 images, 6 prompts, 15 tags). Both themes,
|
||||
@@ -16,7 +16,16 @@ const FULL_HEADING=new Set(['tags']);
|
||||
|
||||
const rowNamed=(nav,text)=>nav.rows.find(row=>row.text===text);
|
||||
|
||||
const removedItem=item=>String(item.className).includes('concept-badge')||(item.tag==='button'&&item.text.startsWith('Activity'));
|
||||
const removedItem=item=>String(item.className).includes('concept-badge');
|
||||
|
||||
/**
|
||||
* The concept with its Activity count hidden (D52): production counts running jobs only and hides the count at 0, as
|
||||
* the comparison fixture has none. The concept's own CSS then lays out its button without the count.
|
||||
*/
|
||||
async function openConcept(page,route,theme,motion) {
|
||||
await openConceptPage(page,route,theme,motion);
|
||||
await mutate(page,{css:'#activity-count { display: none }'});
|
||||
}
|
||||
|
||||
/** Left offsets of flex items packed along a row of `width` by its `justify-content` and `gap`. */
|
||||
function packedLefts(items,width,justify,gap) {
|
||||
@@ -37,10 +46,12 @@ function remainingTopActions(actions) {
|
||||
const gap=parseFloat(actions.columnGap);
|
||||
// A full-width row (<=700px) keeps its width; otherwise it shrinks to its remaining items.
|
||||
const width=actions.fullWidth?actions.width:shown.reduce((sum,item)=>sum+item.width,0)+gap*(shown.length-1);
|
||||
// At <=700px the concept spreads three items with `space-between`, so its Theme select sits in the middle; with
|
||||
// Activity removed (D5) the same rule puts Theme at the row's start. Offsets come from the concept's own sizes.
|
||||
// At <=700px the concept spreads its shown items with `space-between`; the hidden pill (D53) takes no place.
|
||||
// Offsets come from the concept's own sizes.
|
||||
const lefts=new Map(shown.map((item,index)=>[item,packedLefts(shown,width,actions.justifyContent,gap)[index]]));
|
||||
return {...actions,height,width,
|
||||
// D52: the full-width row (<=700px) wraps its items when a zoomed phone is narrower than they are.
|
||||
flexWrap:actions.fullWidth?'wrap':actions.flexWrap,
|
||||
items:kept.map(item=>item.shown?{...item,top:Math.round((height-item.height)/2),left:lefts.get(item)}:item)};
|
||||
}
|
||||
|
||||
@@ -53,20 +64,32 @@ function remainingFooter(footer,spans) {
|
||||
}
|
||||
|
||||
// D28 ruling (final review A, Minor 1): Overview gets a distinct outline home glyph. The concept has no home icon, and
|
||||
// its `activity` pulse stays reserved for the Activity row D5 omits, so the two entries never share one glyph.
|
||||
// its `activity` pulse belongs to the Activity row, so the two entries never share one glyph.
|
||||
const OVERVIEW_GLYPH='M3 11 12 4l9 7M5 10v10h14V10M10 20v-6h4v6';
|
||||
|
||||
/**
|
||||
* The concept's SAVED VIEWS rows (app.js a331–335) as production lists them: Favorites with its count (D51), then the
|
||||
* saved views. D50: no "Needs review" row until the tag-review slice (maintainer ruling 2026-10-21).
|
||||
*/
|
||||
function savedNav(nav) {
|
||||
const rows=nav.rows.filter(row=>row.text!=='Needs review');
|
||||
return {...nav,rows:rows.map((row,index)=>index===rows.length-1?{...row,gapAfter:null}:row)};
|
||||
}
|
||||
|
||||
/** Recorded shell deviations only (spec §8). Everything else in the concept inventory must equal production. */
|
||||
function mapShellConcept(reference) {
|
||||
const {chrome}=reference;
|
||||
const navs=chrome.navs
|
||||
// D28: SAVED VIEWS and COLLECTIONS are proposals with no persisted store, so their navs are not rendered.
|
||||
.filter(nav=>nav.id!=='saved-nav'&&nav.id!=='collections-nav')
|
||||
// D28: COLLECTIONS is a proposal with no persisted store, so its nav is not rendered.
|
||||
.filter(nav=>nav.id!=='collections-nav')
|
||||
.map(nav=>{
|
||||
if(nav.id==='saved-nav')return savedNav(nav);
|
||||
if(nav.id!=='system-nav')return nav;
|
||||
const system=rowNamed(nav,'System');
|
||||
return {...nav,rows:[
|
||||
// D5: no Activity row until an activity center exists. D28: no Design review (prototype page).
|
||||
// D52: the Activity row counts running jobs only, so with none it has no count.
|
||||
{...rowNamed(nav,'Activity'),count:null},
|
||||
// D28: no Design review (prototype page).
|
||||
system,
|
||||
// D28: the production Overview hub follows System, as a concept nav row without a count. Its icon is
|
||||
// the ruling's own outline home glyph (OVERVIEW_GLYPH), never the concept's `activity` pulse.
|
||||
@@ -76,17 +99,20 @@ function mapShellConcept(reference) {
|
||||
return {...reference,chrome:{...chrome,
|
||||
// D29: no prototype suffix in the document title.
|
||||
title:chrome.title.replace(/ · Prompt Manager concept$/,' · Prompt Manager'),
|
||||
// D28: only the WORKSPACE group label remains.
|
||||
labels:chrome.labels.filter(label=>label.text==='WORKSPACE'),
|
||||
// D28: no COLLECTIONS label. D49: SAVED VIEWS without its "PROPOSED" badge.
|
||||
labels:chrome.labels.filter(label=>label.text!=='COLLECTIONS PROPOSED')
|
||||
.map(label=>label.text==='SAVED VIEWS PROPOSED'?{...label,text:'SAVED VIEWS'}:label),
|
||||
navs,
|
||||
connection:{...chrome.connection,
|
||||
// D28: truthful copy instead of "Demo connection · Classic / Nodes 2.0".
|
||||
small:'Your library, on your server',
|
||||
// D28: no fake connection state, so the dot is the neutral muted token (the concept's label color).
|
||||
dot:{...chrome.connection.dot,backgroundColor:chrome.labels[0].color}},
|
||||
// D29: no "Interactive concept" badge. D5: no Activity button. The row's size and the remaining items'
|
||||
// centered offsets follow from the concept's own item sizes.
|
||||
topActions:remainingTopActions(chrome.topActions),
|
||||
// D29: no "Interactive concept" badge. D53: the progress pill before the Activity button, hidden with no running
|
||||
// job. D52: the button's count is hidden at 0. The row's size and the remaining items' centered offsets follow
|
||||
// from the concept's own item sizes.
|
||||
topActions:remainingTopActions({...chrome.topActions,items:chrome.topActions.items.flatMap(item=>
|
||||
item.tag==='button'&&item.text==='Activity'?[{tag:'a',className:'activity-pill',shown:false},{...item,count:'0'}]:[item])}),
|
||||
footer:(()=>{
|
||||
// D29: production footer copy (one line, as the concept's own copy is), and no "What changes & why"
|
||||
// (a prototype review page); the footer then holds only its span.
|
||||
@@ -199,6 +225,30 @@ for(const [width,height] of SIZES)for(const theme of THEMES)test(`shell geometry
|
||||
}
|
||||
});
|
||||
|
||||
// Saved views (#249 D16): a view saved from the Images toolbar adds the concept's own saved row (search icon, name, no
|
||||
// count) to SAVED VIEWS. The concept's demo action reads "Apply in prototype"; production's is "Save" (D29).
|
||||
for(const [width,height] of SIZES)test(`a saved view row matches the concept's ${width}x${height}`,async({page})=>{
|
||||
await page.setViewportSize({width,height});
|
||||
const save=async(action,mainSelector)=>{
|
||||
await page.locator(`${mainSelector} .toolbar`).getByRole('button',{name:'Save view',exact:true}).click();
|
||||
const dialog=page.getByRole('dialog',{name:'Save this view',exact:true});
|
||||
await expect(dialog.getByRole('textbox',{name:'View name',exact:true})).toHaveValue('My workspace view');
|
||||
await dialog.getByRole('textbox',{name:'View name',exact:true}).fill('Golden hour');
|
||||
await dialog.getByRole('button',{name:action,exact:true}).click();
|
||||
await expect(page.locator('#saved-nav').getByRole('link',{name:'Golden hour',exact:true})).toHaveCount(1);
|
||||
await page.mouse.move(0,0);
|
||||
return inventoryOf(page,mainSelector);
|
||||
};
|
||||
await openConcept(page,'images','dark');
|
||||
const reference=await save('Apply in prototype','#main');
|
||||
expect(reference.chrome.navs.find(nav=>nav.id==='saved-nav').rows.map(row=>row.text)).toEqual(['Favorites','Needs review','Golden hour']);
|
||||
await openProduction(page,'images','dark');
|
||||
const actual=await save('Save','#workspace-main');
|
||||
expect(diffInventory(mapShellConcept(reference).chrome.navs,actual.chrome.navs)).toEqual([]);
|
||||
expect(actual.chrome.navs.find(nav=>nav.id==='saved-nav').rows.map(row=>[row.text,row.count]))
|
||||
.toEqual([['Favorites',reference.chrome.navs.find(nav=>nav.id==='saved-nav').rows[0].count],['Golden hour',null]]);
|
||||
});
|
||||
|
||||
test('the inventory reports a missing shell element by its path instead of failing',async({page})=>{
|
||||
await page.setViewportSize({width:1600,height:1100});
|
||||
await openConcept(page,'tags','dark');await page.mouse.move(0,0);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
|
||||
// Artifact-only captures of Tags & categories states that have no concept counterpart: the Manage categories
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
// Each Playwright worker owns its server: `request`, `page` and manual
|
||||
// `browser.newContext()` all reach it through `baseURL`, and another server
|
||||
// process never sees its database.
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { startBrowserServer } from '../../tools/vendor/start-browser-server.mjs';
|
||||
|
||||
const search = text => `/prompt_manager/search?text=${encodeURIComponent(text)}`;
|
||||
|
||||
test('every fixture reaches this worker\'s own server and another server keeps its own database', async ({ request, page, browser, baseURL, workerServer }, testInfo) => {
|
||||
expect(baseURL).toBe(workerServer.url);
|
||||
await expect(await request.post('/__test__/reset')).toBeOK();
|
||||
const marker = `worker ${testInfo.workerIndex} isolation ${testInfo.project.name} ${testInfo.repeatEachIndex}`;
|
||||
await expect(await request.post('/prompt_manager/save', { data: { text: marker } })).toBeOK();
|
||||
expect((await (await request.get(search(marker))).json()).count).toBe(1);
|
||||
|
||||
const viaPage = await page.request.get(search(marker));
|
||||
expect(viaPage.url().startsWith(`${baseURL}/`)).toBe(true);
|
||||
expect((await viaPage.json()).count).toBe(1);
|
||||
const context = await browser.newContext();
|
||||
try {
|
||||
const response = await (await context.newPage()).goto(search(marker));
|
||||
expect(response.url().startsWith(`${baseURL}/`)).toBe(true);
|
||||
expect((await response.json()).count).toBe(1);
|
||||
} finally {
|
||||
await context.close();
|
||||
}
|
||||
|
||||
const other = await startBrowserServer();
|
||||
try {
|
||||
expect(other.url).not.toBe(baseURL);
|
||||
expect((await (await fetch(`${other.url}${search(marker)}`)).json()).count).toBe(0);
|
||||
} finally {
|
||||
await other.stop();
|
||||
}
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
|
||||
test.beforeEach(async ({ request, page, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
@@ -64,11 +64,16 @@ test('actual Tags page reloads the selected filter and opens its image viewer',
|
||||
await page.goto('/prompt_manager/admin#/tags');
|
||||
await page.locator('[data-tag-name="cats"]').click();
|
||||
await expect(page.locator('#tagPromptsGrid')).toContainText('Cat portrait');
|
||||
const dogs = page.waitForResponse(response => response.url().includes('/tags/dogs/prompts?'));
|
||||
await page.locator('[data-tag-name="dogs"]').click();
|
||||
const [shown] = (await (await dogs).json()).prompts[0].images;
|
||||
await expect(page.locator('#tagPromptsGrid')).toContainText('Dog landscape');
|
||||
await expect(page.locator('#tagPromptsGrid')).not.toContainText('Cat portrait');
|
||||
await page.locator('#tagPromptsGrid img').first().click();
|
||||
await expect(page.locator('.viewer-canvas img')).toBeVisible();
|
||||
// The viewer's title names the shown image through the page's title callback. ViewerJS renders it only once the
|
||||
// image is viewed, so closing at "visible" alone left the callback unrun on slower hosts (CI coverage missed it).
|
||||
await expect(page.locator('.viewer-title')).toHaveText(shown.filename);
|
||||
await page.locator('.viewer-container').getByRole('button', { name: 'Close', exact: true }).click();
|
||||
await expect(page.locator('.viewer-container')).toHaveCount(0);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,801 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept, openConcept, openProduction } from './concept-inventory.mjs';
|
||||
|
||||
// Activity (#249 Activity-1): the concept's Activity page (app.js renderActivity 732–738), its top-bar button
|
||||
// (index.html 51–52) and sidebar row (app.js ~341), with the recorded D52 (counts of running jobs), D53 (the progress
|
||||
// pill) and D54 (real job items). Fake jobs come from tests/browser/workspace_activity_fixture.py.
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
const main = page => page.locator('#workspace-main');
|
||||
const pill = page => page.locator('.top-actions #activity-pill');
|
||||
const activityButton = page => page.locator('.top-actions button#activity-open');
|
||||
const topCount = page => page.locator('#activity-count');
|
||||
const navRow = page => page.locator('#system-nav a[data-route="activity"]');
|
||||
const toast = page => page.locator('#workspace-toast');
|
||||
const items = page => main(page).locator('section.surface > .activity-item');
|
||||
const LOST = 'This job ended when ComfyUI restarted.';
|
||||
// The job view (Task 3, D56): the page heading, then the Progress and Results surfaces and the Scan log.
|
||||
const heading = page => main(page).locator(':scope > .heading');
|
||||
const progressSection = page => main(page).getByRole('region', { name: 'Progress', exact: true });
|
||||
const resultsSection = page => main(page).getByRole('region', { name: 'Results', exact: true });
|
||||
const tiles = page => progressSection(page).locator('dl.stat-row > div.stat');
|
||||
const outcomeButtons = page => resultsSection(page).locator('.outcome-list > button');
|
||||
const fileRows = page => resultsSection(page).locator('.outcome-files > .file-row');
|
||||
const resultsNote = page => resultsSection(page).locator('.outcome-files > p.small');
|
||||
const filter = page => resultsSection(page).getByRole('searchbox', { name: 'Filter results', exact: true });
|
||||
const scanLog = page => main(page).locator('details.term');
|
||||
const TILE_LABELS = ['Added to Library', 'Already known', 'Needs review', 'No usable metadata', 'Errors'];
|
||||
const VIEWER_HINT = 'each row opens the image in the viewer';
|
||||
|
||||
async function tileValues(page) {
|
||||
return tiles(page).evaluateAll(nodes => nodes.map(node => [node.className, node.querySelector('dt').textContent,
|
||||
node.querySelector('dd').textContent]));
|
||||
}
|
||||
|
||||
async function openJob(page, id) {
|
||||
await page.goto(`${workspace}#/activity/${id}`);
|
||||
await expect(progressSection(page)).toBeVisible();
|
||||
}
|
||||
|
||||
async function startJob(request, data = {}) {
|
||||
const response = await request.post('/__test__/jobs/start', { data: { total: 19622, ...data } });
|
||||
await expect(response).toBeOK();
|
||||
return (await response.json()).job;
|
||||
}
|
||||
|
||||
async function step(request, id, data) {
|
||||
await expect(await request.post(`/__test__/jobs/${id}/step`, { data })).toBeOK();
|
||||
}
|
||||
|
||||
async function finish(request, id, state = 'finished') {
|
||||
const response = await request.post(`/__test__/jobs/${id}/finish`, { data: { state } });
|
||||
await expect(response).toBeOK();
|
||||
return (await response.json()).job;
|
||||
}
|
||||
|
||||
async function listed(request) {
|
||||
const response = await request.get('/prompt_manager/jobs');
|
||||
await expect(response).toBeOK();
|
||||
return (await response.json()).jobs;
|
||||
}
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
await expect(await request.post('/__test__/reset')).toBeOK();
|
||||
page.activityErrors = [];
|
||||
page.on('pageerror', error => page.activityErrors.push(error.message));
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, request, browserName }) => {
|
||||
// The store's own reads can meet a route handler at teardown; that must not fail the test it outlived.
|
||||
await page.unrouteAll({ behavior: 'ignoreErrors' });
|
||||
await saveCoverage(page, browserName);
|
||||
expect(page.activityErrors).toEqual([]);
|
||||
await request.post('/__test__/jobs/restart');
|
||||
});
|
||||
|
||||
/** The concept's item structure and the computed styles its CSS gives it, measured the same way in both documents. */
|
||||
function measureItems(selector) {
|
||||
return [...document.querySelectorAll(selector)].map(item => {
|
||||
const css = (node, ...keys) => node && Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]]));
|
||||
const tree = node => ({ tag: node.localName, className: [...node.classList].sort().join(' '), children: [...node.children].map(tree) });
|
||||
const progress = item.querySelector('progress');
|
||||
return {
|
||||
tree: tree(item),
|
||||
title: item.querySelector('h3').textContent,
|
||||
chip: item.querySelector('.chip').textContent,
|
||||
buttons: [...item.querySelectorAll('.row-actions button')].map(button => button.textContent),
|
||||
styles: {
|
||||
item: css(item, 'paddingTop', 'paddingBottom', 'borderBottomWidth', 'borderBottomStyle', 'borderBottomColor'),
|
||||
heading: css(item.querySelector('.panel-heading'), 'display', 'justifyContent', 'alignItems', 'columnGap'),
|
||||
h3: css(item.querySelector('h3'), 'fontSize', 'fontWeight', 'color', 'marginTop', 'marginBottom'),
|
||||
chip: css(item.querySelector('.chip'), 'fontSize', 'color', 'backgroundColor', 'paddingTop', 'paddingLeft', 'borderTopLeftRadius'),
|
||||
p: css(item.querySelector('p'), 'fontSize', 'color', 'marginTop', 'lineHeight'),
|
||||
progress: progress && { ...css(progress, 'height', 'marginTop', 'marginBottom', 'accentColor'),
|
||||
fills: Math.abs(progress.getBoundingClientRect().width - item.getBoundingClientRect().width) < 1,
|
||||
max: progress.max, label: Boolean(progress.getAttribute('aria-label')) },
|
||||
actions: css(item.querySelector('.row-actions'), 'display', 'columnGap', 'marginTop'),
|
||||
button: css(item.querySelector('.row-actions button'), 'fontSize', 'paddingTop', 'paddingLeft', 'minHeight', 'borderTopLeftRadius'),
|
||||
},
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/** D56: an item tree whose `h3` holds the link to the job's view. */
|
||||
function withTitleLink(tree) {
|
||||
const [heading, ...rest] = tree.children;
|
||||
const link = { tag: 'a', className: '', children: [] };
|
||||
return { ...tree, children: [{ ...heading, children: heading.children.map(node => node.tag === 'h3' ? { ...node, children: [link] } : node) }, ...rest] };
|
||||
}
|
||||
|
||||
function measurePage(mainSelector) {
|
||||
const root = document.querySelector(mainSelector);
|
||||
const section = root.querySelector(':scope > section.surface');
|
||||
return {
|
||||
h1: root.querySelector('.heading h1').textContent,
|
||||
description: root.querySelector('.heading p.muted').textContent,
|
||||
actions: [...root.querySelector('.heading .heading-actions').children].map(node => node.textContent),
|
||||
h2: section.querySelector(':scope > h2').textContent,
|
||||
surface: Object.fromEntries(['paddingTop', 'borderTopWidth', 'backgroundColor', 'borderTopLeftRadius']
|
||||
.map(key => [key, getComputedStyle(section)[key]])),
|
||||
};
|
||||
}
|
||||
|
||||
for (const theme of ['dark', 'light']) test(`activity page matches the concept list markup ${theme}`, async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
await seedConcept(request);
|
||||
// The concept: its finished sample, then a running job started from System (startJob, app.js 940–952).
|
||||
await openConcept(page, 'activity', theme);
|
||||
const conceptPage = await page.evaluate(measurePage, '#main');
|
||||
const [conceptDone] = await page.evaluate(measureItems, '#main .activity-item');
|
||||
await page.goto('/__test__/canonical/workspace#system');
|
||||
await page.getByRole('combobox', { name: 'Theme', exact: true }).selectOption(theme);
|
||||
await page.getByRole('button', { name: 'Review thumbnails', exact: true }).click();
|
||||
await expect(page.locator('#main .activity-item progress')).toHaveCount(1);
|
||||
const [conceptRunning] = await page.evaluate(measureItems, '#main .activity-item');
|
||||
|
||||
const done = await startJob(request);
|
||||
await step(request, done.id, { processed: 19622, counters: { added: 1420, errors: 18 } });
|
||||
await finish(request, done.id);
|
||||
const running = await startJob(request);
|
||||
await step(request, running.id, { processed: 8240 });
|
||||
await openProduction(page, 'activity', theme);
|
||||
await expect(items(page)).toHaveCount(2);
|
||||
const productionPage = await page.evaluate(measurePage, '#workspace-main');
|
||||
const [productionRunning, productionDone] = await page.evaluate(measureItems, '#workspace-main .activity-item');
|
||||
|
||||
// D52: no "Unified activity · proposed" chip (prototype copy); the rest of the heading and the section are the concept's.
|
||||
expect(conceptPage.actions).toEqual(['Unified activity · proposed']);
|
||||
expect(productionPage).toEqual({ ...conceptPage, actions: [] });
|
||||
// A completed job is the concept's completed item; D56: its title is a link to the job's view, in the title's colour.
|
||||
expect(conceptDone.title).toBe('✓ Library metadata enriched');
|
||||
expect(productionDone).toEqual({ ...conceptDone, tree: withTitleLink(conceptDone.tree), title: '✓ Scan for metadata' });
|
||||
const links = await items(page).locator('h3 > a').evaluateAll(nodes => nodes.map(node => [node.getAttribute('href'),
|
||||
getComputedStyle(node).color === getComputedStyle(node.parentElement).color, getComputedStyle(node).textDecorationLine]));
|
||||
expect(links).toEqual([[`#/activity/${running.id}`, true, 'none'], [`#/activity/${done.id}`, true, 'none']]);
|
||||
// D54: a running job's chip names its state ("Running", where the concept's sample reads "Needs attention"), and
|
||||
// its actions are Cancel job only (no "Simulate completion").
|
||||
expect(conceptRunning.buttons).toEqual(['Simulate completion', 'Cancel job']);
|
||||
const [, ...cancelOnly] = conceptRunning.tree.children.at(-1).children;
|
||||
const conceptActions = { ...conceptRunning.tree.children.at(-1), children: cancelOnly };
|
||||
expect(productionRunning).toEqual({ ...conceptRunning,
|
||||
tree: withTitleLink({ ...conceptRunning.tree, children: [...conceptRunning.tree.children.slice(0, -1), conceptActions] }),
|
||||
title: '○ Scan for metadata', chip: 'Running', buttons: ['Cancel job'] });
|
||||
await expect(items(page).nth(1).locator('p')).toHaveText(/^Finished \d\d:\d\d · 1,420 added · 18 errors$/);
|
||||
await expect(items(page).first().locator('p')).toHaveText(/^Started \d\d:\d\d · 2 output folders · closing this view does not cancel it$/);
|
||||
await expect(items(page).first().locator('progress')).toHaveAttribute('aria-label', 'Scan for metadata progress');
|
||||
expect(await items(page).first().locator('progress').evaluate(node => node.value)).toBe(41);
|
||||
});
|
||||
|
||||
test('pill shows while a job runs and opens its view', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
await page.goto(`${workspace}#/images`);
|
||||
// D52: no running job, so neither count shows; the button is the concept's.
|
||||
await expect(activityButton(page)).toHaveAccessibleName('Activity');
|
||||
await expect(topCount(page)).toBeHidden();
|
||||
await expect(navRow(page)).toHaveText('Activity');
|
||||
await expect(navRow(page).locator('.nav-count')).toHaveCount(0);
|
||||
await expect(pill(page)).toBeHidden();
|
||||
|
||||
const job = await startJob(request);
|
||||
// The list is read again on its own (15 s), or at once on a reload or when Activity opens.
|
||||
await page.reload();
|
||||
await expect(pill(page)).toBeVisible();
|
||||
await expect(pill(page)).toHaveText('Scanning · 0% · 0 / 19,622');
|
||||
await step(request, job.id, { processed: 8240 });
|
||||
const text = 'Scanning · 41% · 8,240 / 19,622';
|
||||
await expect(pill(page)).toHaveText(text);
|
||||
await expect(pill(page)).toHaveAttribute('aria-label', text);
|
||||
await expect(pill(page)).toHaveAttribute('href', `#/activity/${job.id}`);
|
||||
expect(await pill(page).locator('.activity-pill-bar i').evaluate(node => node.style.width)).toBe('41%');
|
||||
// The pill sits before the button, in the mockup's place, and both counts read 1.
|
||||
expect(await page.locator('.top-actions > *').evaluateAll(nodes => nodes.map(node => node.id || node.className)))
|
||||
.toEqual(['activity-pill', 'activity-open', 'theme-label', 'canvas-link']);
|
||||
await expect(topCount(page)).toBeVisible();
|
||||
await expect(topCount(page)).toHaveText('1');
|
||||
await expect(navRow(page).locator('.nav-count')).toHaveText('1');
|
||||
|
||||
// The pill opens the job's own view (Task 3, D56), named by the job.
|
||||
await pill(page).click();
|
||||
await expect(page).toHaveURL(new RegExp(`#/activity/${job.id}$`));
|
||||
await expect(main(page).getByRole('heading', { level: 1 })).toHaveText('Scan for metadata');
|
||||
await expect(page.locator('#workspace-breadcrumb')).toHaveText('Activity');
|
||||
await expect(page).toHaveTitle('Scan for metadata · Prompt Manager');
|
||||
await expect(navRow(page)).toHaveAttribute('aria-current', 'page');
|
||||
await expect(progressSection(page).locator('progress')).toHaveJSProperty('value', 41);
|
||||
|
||||
// The Activity button opens the list itself from any page.
|
||||
await page.locator('nav [data-route="library"]').click();
|
||||
await activityButton(page).click();
|
||||
await expect(page).toHaveURL(/#\/activity$/);
|
||||
await expect(items(page)).toHaveCount(1);
|
||||
|
||||
await finish(request, job.id);
|
||||
await expect(pill(page)).toBeHidden();
|
||||
await expect(topCount(page)).toBeHidden();
|
||||
await expect(navRow(page).locator('.nav-count')).toHaveCount(0);
|
||||
await expect(items(page).first().locator('.chip.green')).toHaveText('Completed');
|
||||
await expect(items(page).first().locator('progress, .row-actions')).toHaveCount(0);
|
||||
// A finished job's title still opens its view (D56).
|
||||
await items(page).first().locator('h3 > a').click();
|
||||
await expect(page).toHaveURL(new RegExp(`#/activity/${job.id}$`));
|
||||
await expect(heading(page).locator('p.muted')).toHaveText(/^Finished \d\d:\d\d · 0 added · 0 errors$/);
|
||||
});
|
||||
|
||||
test('a phone keeps the pill and its controls inside the topbar', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 390, height: 844 });
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 8240 });
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await expect(pill(page)).toHaveText('Scanning · 41% · 8,240 / 19,622');
|
||||
await expect(pill(page)).toHaveAttribute('aria-label', 'Scanning · 41% · 8,240 / 19,622');
|
||||
await expect(pill(page).locator('.activity-pill-files, .activity-pill-bar')).toHaveCount(2);
|
||||
await expect(pill(page).locator('.activity-pill-files')).toBeHidden();
|
||||
const layout = await page.locator('.top-actions').evaluate(row => ({
|
||||
scroll: document.documentElement.scrollWidth <= document.documentElement.clientWidth,
|
||||
inside: [...row.children].filter(node => node.getBoundingClientRect().width)
|
||||
.every(node => node.getBoundingClientRect().right <= row.getBoundingClientRect().right + 0.5
|
||||
&& node.getBoundingClientRect().left >= row.getBoundingClientRect().left - 0.5),
|
||||
}));
|
||||
expect(layout).toEqual({ scroll: true, inside: true });
|
||||
});
|
||||
|
||||
test('reload during a scan reattaches to the running job', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 1000 });
|
||||
await page.goto(`${workspace}#/activity`);
|
||||
await expect(items(page)).toHaveCount(1);
|
||||
await expect(pill(page)).toHaveText('Scanning · 5% · 1,000 / 19,622');
|
||||
await page.reload();
|
||||
await expect(pill(page)).toHaveText('Scanning · 5% · 1,000 / 19,622');
|
||||
// The reloaded page follows the same job live and started nothing.
|
||||
await step(request, job.id, { processed: 9811 });
|
||||
await expect(pill(page)).toHaveText('Scanning · 50% · 9,811 / 19,622');
|
||||
expect(await items(page).first().locator('progress').evaluate(node => node.value)).toBe(50);
|
||||
await expect(items(page)).toHaveCount(1);
|
||||
expect((await listed(request)).map(item => [item.id, item.state])).toEqual([[job.id, 'running']]);
|
||||
|
||||
// A second tab attaches to the same job and sees the same progress.
|
||||
const other = await page.context().newPage();
|
||||
await other.goto(`${workspace}#/activity`);
|
||||
await expect(pill(other)).toHaveText('Scanning · 50% · 9,811 / 19,622');
|
||||
await expect(other.locator('#workspace-main .activity-item')).toHaveCount(1);
|
||||
await other.close();
|
||||
expect((await listed(request)).length).toBe(1);
|
||||
|
||||
// Cancel job asks the server to stop at a file boundary; the job ends cancelled with its partial counts.
|
||||
await step(request, job.id, { counters: { added: 7 } });
|
||||
await items(page).first().getByRole('button', { name: 'Cancel job', exact: true }).focus();
|
||||
await page.keyboard.press('Enter');
|
||||
await expect(toast(page)).toHaveText('Scan cancelled · 7 added');
|
||||
// The removed Cancel job hands its focus to the item's title link (review M4).
|
||||
await expect(items(page).first().locator('h3 > a')).toBeFocused();
|
||||
await expect(items(page).first().locator('.chip.blue')).toHaveText('Cancelled');
|
||||
await expect(items(page).first().locator('p')).toHaveText(/^Cancelled \d\d:\d\d at a file boundary · 7 added · completed work is kept$/);
|
||||
await expect(pill(page)).toBeHidden();
|
||||
});
|
||||
|
||||
test('lost server clears the pill and marks jobs ended', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 4000 });
|
||||
await page.goto(`${workspace}#/activity`);
|
||||
await expect(pill(page)).toBeVisible();
|
||||
await expect(topCount(page)).toHaveText('1');
|
||||
const reads = [];
|
||||
page.on('request', item => { if (new URL(item.url()).pathname === '/prompt_manager/jobs') reads.push(Date.now()); });
|
||||
// ComfyUI restarts: the stream drops without a final event and the new registry is empty.
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
await expect(pill(page)).toBeHidden();
|
||||
await expect(topCount(page)).toBeHidden();
|
||||
await expect(items(page).first().locator('.chip.blue')).toHaveText('Ended');
|
||||
await expect(items(page).first().locator('p')).toHaveText(LOST);
|
||||
await expect(items(page).first().locator('progress, .row-actions')).toHaveCount(0);
|
||||
await expect(navRow(page).locator('.nav-count')).toHaveCount(0);
|
||||
// One re-read found the restart; there is no retry storm and no finish toast for a job that only vanished.
|
||||
await page.waitForTimeout(2500);
|
||||
expect(reads.length).toBeLessThanOrEqual(2);
|
||||
await expect(toast(page)).toHaveText('');
|
||||
|
||||
// A server that cannot be read at all also ends the running job, then is read again with a growing backoff.
|
||||
const second = await startJob(request);
|
||||
await page.reload();
|
||||
await expect(pill(page)).toBeVisible();
|
||||
const failed = [];
|
||||
await page.route('**/prompt_manager/jobs**', route => { failed.push(Date.now()); return route.abort(); });
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
await expect(pill(page)).toBeHidden();
|
||||
await expect(items(page).filter({ hasText: LOST })).toHaveCount(1);
|
||||
await page.waitForTimeout(3500);
|
||||
// 1 s, then 2 s: at most two re-reads in 3.5 s.
|
||||
expect(failed.length).toBeLessThanOrEqual(2);
|
||||
await page.unroute('**/prompt_manager/jobs**');
|
||||
expect(second.id).not.toBe(job.id);
|
||||
});
|
||||
|
||||
test('start and finish toasts', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await expect(pill(page)).toBeVisible();
|
||||
await step(request, job.id, { processed: 19622, counters: { added: 1420, errors: 18 } });
|
||||
await finish(request, job.id);
|
||||
await expect(toast(page)).toHaveText('Scan finished · 1,420 added · 18 errors');
|
||||
await expect(pill(page)).toBeHidden();
|
||||
|
||||
const single = await startJob(request);
|
||||
await page.reload();
|
||||
await expect(pill(page)).toBeVisible();
|
||||
await step(request, single.id, { processed: 19622, counters: { added: 1, errors: 1 } });
|
||||
await finish(request, single.id, 'failed');
|
||||
await expect(toast(page)).toHaveText('Scan failed · 1 added · 1 error');
|
||||
|
||||
// The store's own start: the real metadata scan over this server's outputs, announced, followed and finished.
|
||||
await page.goto(`${workspace}#/activity`);
|
||||
const started = await page.evaluate(async () => {
|
||||
const { createActivityStore } = await import('/prompt_manager/js/workspace/activity-store.mjs');
|
||||
const controller = new AbortController();
|
||||
const store = createActivityStore({ api: window.ApiClient.createApiClient(), pollMs: 0,
|
||||
scope: { signal: controller.signal, cleanup: () => {} } });
|
||||
const result = await store.start('scan');
|
||||
// Read before the busy attempt: its list read may already see the small scan finished and toast that.
|
||||
const toast = document.getElementById('workspace-toast').textContent;
|
||||
const busy = await store.start('scan').catch(error => ({ error: error.message }));
|
||||
return { id: result.job.id, title: result.job.title, toast, busy: busy.busy?.id ?? null, message: busy.message ?? null };
|
||||
});
|
||||
expect(started.title).toBe('Scan for metadata');
|
||||
expect(started.toast).toBe('Scan started.');
|
||||
// A second start while the scan runs is refused and names it (unless the small fixture scan already ended).
|
||||
if (started.busy !== null) expect(started.busy).toBe(started.id);
|
||||
await expect(toast(page)).toHaveText(/^Scan (started\.|finished · \d+ added · \d+ errors?)$/);
|
||||
await expect.poll(async () => (await listed(request)).find(item => item.id === started.id)?.state, { timeout: 15000 }).toBe('finished');
|
||||
});
|
||||
|
||||
// ── Task 3: Scan for metadata, its review dialog and the job view ─────────────────────────────
|
||||
|
||||
async function startFromImport(page) {
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await page.getByRole('button', { name: 'Import', exact: true }).click();
|
||||
const importDialog = page.getByRole('dialog', { name: 'Import & inspect', exact: true });
|
||||
const row = importDialog.locator('.operation').filter({ hasText: 'Scan for metadata' });
|
||||
await row.getByRole('button', { name: 'Review scan', exact: true }).click();
|
||||
const review = page.getByRole('dialog', { name: 'Review metadata scan', exact: true });
|
||||
await expect(review).toBeVisible();
|
||||
await review.getByRole('button', { name: 'Start scan', exact: true }).click();
|
||||
await expect(review).toHaveCount(0);
|
||||
}
|
||||
|
||||
test('start scan creates a job, toasts, and opens its view', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const posted = [];
|
||||
page.on('request', item => { if (item.method() === 'POST' && new URL(item.url()).pathname === '/prompt_manager/jobs/scan') posted.push(item); });
|
||||
await startFromImport(page);
|
||||
await expect(page).toHaveURL(/#\/activity\/[0-9a-f]{32}$/);
|
||||
const id = page.url().split('/').at(-1);
|
||||
expect(posted).toHaveLength(1);
|
||||
// The fixture scan is small and may already have finished; the start toast came first either way.
|
||||
await expect(toast(page)).toHaveText(/^Scan (started\.|finished · \d+ added · \d+ errors?)$/);
|
||||
await expect(heading(page).locator('h1')).toHaveText('Scan for metadata');
|
||||
await expect(heading(page).locator('h1')).toBeFocused();
|
||||
expect((await listed(request)).map(item => item.id)).toEqual([id]);
|
||||
await expect.poll(async () => (await listed(request))[0].state, { timeout: 15000 }).toBe('finished');
|
||||
await expect(heading(page).locator('p.muted')).toHaveText(/^Finished \d\d:\d\d · \d+ added · \d+ errors?$/);
|
||||
// The finished scan's tiles add up to the files it processed, and its log is there to open.
|
||||
const job = (await listed(request))[0];
|
||||
const values = await tileValues(page);
|
||||
expect(values.map(([, label]) => label)).toEqual(TILE_LABELS);
|
||||
expect(values.reduce((sum, [, , value]) => sum + Number(value.replaceAll(',', '')), 0)).toBe(job.processed);
|
||||
// Opening the log reads its last lines (review M12).
|
||||
await scanLog(page).locator('summary').click();
|
||||
await expect(scanLog(page).locator('pre')).toContainText('Scan finished');
|
||||
});
|
||||
|
||||
// ── Task 4 (Activity-2): scan scope, new and unresolved files by default ─────────────────────
|
||||
|
||||
test('all files checkbox changes the scope line and the request', async ({ page }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const bodies = [];
|
||||
page.on('request', item => {
|
||||
if (item.method() === 'POST' && new URL(item.url()).pathname === '/prompt_manager/jobs/scan') bodies.push(item.postDataJSON());
|
||||
});
|
||||
const review = page.getByRole('dialog', { name: 'Review metadata scan', exact: true });
|
||||
const scopeValue = review.locator('.meta-grid > div').filter({ has: page.locator('dt', { hasText: 'Scope' }) }).locator('dd');
|
||||
// The concept's switch: a checkbox with role=switch.
|
||||
const allFiles = review.getByRole('switch', { name: 'All files', exact: true });
|
||||
const openReview = async () => {
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await page.getByRole('button', { name: 'Import', exact: true }).click();
|
||||
await page.getByRole('dialog', { name: 'Import & inspect', exact: true }).locator('.operation')
|
||||
.filter({ hasText: 'Scan for metadata' }).getByRole('button', { name: 'Review scan', exact: true }).click();
|
||||
await expect(review).toBeVisible();
|
||||
};
|
||||
|
||||
// The default: the concept's "New and unresolved files", the box unchecked; the request asks for "new".
|
||||
await openReview();
|
||||
await expect(scopeValue).toHaveText('New and unresolved files');
|
||||
await expect(allFiles).not.toBeChecked();
|
||||
await expect(review.locator('.setting-row p')).toHaveText('Also read the files already linked to their prompt again.');
|
||||
await expect(allFiles).toHaveAccessibleDescription('Also read the files already linked to their prompt again.');
|
||||
// Checking and unchecking switches the Scope value both ways.
|
||||
await allFiles.check();
|
||||
await expect(scopeValue).toHaveText('All files');
|
||||
await allFiles.uncheck();
|
||||
await expect(scopeValue).toHaveText('New and unresolved files');
|
||||
await review.getByRole('button', { name: 'Start scan', exact: true }).click();
|
||||
await expect(page).toHaveURL(/#\/activity\/[0-9a-f]{32}$/);
|
||||
await expect.poll(() => bodies.length).toBe(1);
|
||||
expect(bodies[0]).toEqual({ scope: 'new' });
|
||||
await expect(heading(page).locator('h1')).toHaveText('Scan for metadata');
|
||||
// Wait for the first scan to end so the second is not refused as busy.
|
||||
await scanLog(page).locator('summary').click();
|
||||
await expect(scanLog(page).locator('pre')).toContainText('Scan finished', { timeout: 15000 });
|
||||
await expect(scanLog(page).locator('pre')).toContainText('new and unresolved files');
|
||||
|
||||
// Checked: "All files", and the request asks for "all".
|
||||
await openReview();
|
||||
await allFiles.check();
|
||||
await expect(scopeValue).toHaveText('All files');
|
||||
await review.getByRole('button', { name: 'Start scan', exact: true }).click();
|
||||
await expect.poll(() => bodies.length).toBe(2);
|
||||
expect(bodies[1]).toEqual({ scope: 'all' });
|
||||
await scanLog(page).locator('summary').click();
|
||||
await expect(scanLog(page).locator('pre')).toContainText('Scan started · ', { timeout: 15000 });
|
||||
await expect(scanLog(page).locator('pre')).toContainText('all files');
|
||||
});
|
||||
|
||||
test('job view counters follow the mapping and update live', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 8240, counters: { added: 1420, known: 6511, review: 37, no_metadata: 254, errors: 18 } });
|
||||
await openJob(page, job.id);
|
||||
// The header: the title, the detail line, then Download report (enabled in Task 5) and Cancel scan (danger).
|
||||
await expect(heading(page).locator('h1')).toHaveText('Scan for metadata');
|
||||
await expect(heading(page).locator('p.muted')).toHaveText(/^Started \d\d:\d\d · 2 output folders · closing this view does not cancel it$/);
|
||||
await expect(heading(page).locator('.heading-actions > button')).toHaveText(['Download report', 'Cancel scan']);
|
||||
await expect(heading(page).getByRole('button', { name: 'Download report', exact: true })).toBeEnabled();
|
||||
await expect(heading(page).getByRole('button', { name: 'Cancel scan', exact: true })).toHaveClass(/danger/);
|
||||
// Progress: the bar, the server's line and the five tiles in order and colour.
|
||||
await expect(progressSection(page).locator('.panel-heading > span.small')).toHaveText('Processing file 8,240 of 19,622');
|
||||
await expect(progressSection(page).locator('progress')).toHaveAttribute('aria-label', 'Scan progress');
|
||||
await expect(progressSection(page).locator('progress')).toHaveJSProperty('value', 41);
|
||||
expect(await tileValues(page)).toEqual([
|
||||
['stat added', 'Added to Library', '1,420'], ['stat', 'Already known', '6,511'], ['stat warn', 'Needs review', '37'],
|
||||
['stat', 'No usable metadata', '254'], ['stat bad', 'Errors', '18'],
|
||||
]);
|
||||
const colours = await tiles(page).evaluateAll(nodes => nodes.map(node => getComputedStyle(node.querySelector('dd')).color));
|
||||
const expected = await page.evaluate(() => ['--green', '--ca-text', '--warning', '--ca-text', '--red'].map(name => {
|
||||
const probe = document.createElement('span');
|
||||
probe.style.color = `var(${name})`;
|
||||
document.querySelector('#workspace-main').append(probe);
|
||||
const colour = getComputedStyle(probe).color;
|
||||
probe.remove();
|
||||
return colour;
|
||||
}));
|
||||
expect([colours[0], colours[2], colours[4]]).toEqual([expected[0], expected[2], expected[4]]);
|
||||
expect(colours[1]).toBe(colours[3]);
|
||||
// The outcome list follows the mockup's order with the same counts.
|
||||
await expect(outcomeButtons(page)).toHaveText(['Added to Library1,420', 'Needs review37', 'Errors18', 'No usable metadata254', 'Already known6,511']);
|
||||
|
||||
// Live: the next step updates the bar, the line and the tiles without a reload.
|
||||
await step(request, job.id, { processed: 9811, counters: { added: 1500, errors: 19 }, records: [
|
||||
{ group: 'added', path: 'live/new.png', detail: 'Added to the Library as “Live”' },
|
||||
] });
|
||||
await expect(progressSection(page).locator('.panel-heading > span.small')).toHaveText('Processing file 9,811 of 19,622');
|
||||
await expect(progressSection(page).locator('progress')).toHaveJSProperty('value', 50);
|
||||
await expect(tiles(page).first().locator('dd')).toHaveText('1,500');
|
||||
await expect(tiles(page).last().locator('dd')).toHaveText('19');
|
||||
await expect(fileRows(page)).toHaveCount(1);
|
||||
// A focused row keeps its focus while later reads append rows.
|
||||
await fileRows(page).first().focus();
|
||||
await step(request, job.id, { records: [{ group: 'added', path: 'live/next.png', detail: 'Added' }] });
|
||||
await expect(fileRows(page)).toHaveCount(2);
|
||||
await expect(fileRows(page).first()).toBeFocused();
|
||||
// The log is collapsed by default and holds the raw timestamped lines.
|
||||
await expect(scanLog(page)).not.toHaveAttribute('open', '');
|
||||
await expect(scanLog(page).locator('summary')).toContainText('Scan log');
|
||||
await scanLog(page).locator('summary').click();
|
||||
await expect(scanLog(page).locator('pre')).toHaveAttribute('tabindex', '0');
|
||||
await expect(scanLog(page).locator('pre')).toHaveText(/^\d\d:\d\d:\d\d \+ live\/new\.png · Added to the Library as “Live”\n\d\d:\d\d:\d\d \+ live\/next\.png · Added$/);
|
||||
|
||||
// Finished: the header says so, Cancel scan is gone and the counters stay.
|
||||
await finish(request, job.id);
|
||||
await expect(heading(page).locator('p.muted')).toHaveText(/^Finished \d\d:\d\d · 1,501 added · 19 errors$/);
|
||||
await expect(heading(page).locator('.heading-actions > button')).toHaveText(['Download report']);
|
||||
await expect(tiles(page).first().locator('dd')).toHaveText('1,501');
|
||||
});
|
||||
|
||||
test('added rows open the viewer, error rows are plain', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
// The fixture's two listed files share a name in the output and input roots; the row's root label tells them apart.
|
||||
await step(request, job.id, { processed: 3, records: [
|
||||
{ group: 'added', path: 'shared/same.png', root: 'output', detail: 'Added to the Library as “Cat portrait”' },
|
||||
{ group: 'known', path: 'shared/same.png', root: 'input', detail: 'Already linked to “Dog portrait”' },
|
||||
{ group: 'errors', path: 'broken/truncated.png', root: 'output', detail: 'Error: the file could not be processed (see the server log)' },
|
||||
] });
|
||||
await openJob(page, job.id);
|
||||
await expect(outcomeButtons(page).first()).toHaveAttribute('aria-pressed', 'true');
|
||||
await expect(fileRows(page)).toHaveCount(1);
|
||||
const added = fileRows(page).first();
|
||||
expect(await added.evaluate(node => node.localName)).toBe('button');
|
||||
await expect(added.locator('.path')).toHaveText('shared/same.png');
|
||||
await expect(added.locator('.why')).toHaveText('Added to the Library as “Cat portrait”');
|
||||
await expect(resultsNote(page)).toHaveText('Each row opens the image in the viewer.');
|
||||
await added.click();
|
||||
const viewer = page.locator('dialog#viewer-dialog[open]');
|
||||
await expect(viewer).toBeVisible();
|
||||
await expect(viewer.locator('#viewer-title')).toContainText('same.png');
|
||||
await expect(viewer.locator('#viewer-stage img').first()).toHaveAttribute('src', /[?&]root=0(&|$)/);
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(viewer).toHaveCount(0);
|
||||
await expect(added).toBeFocused();
|
||||
|
||||
// Already known opens the input root's file.
|
||||
await outcomeButtons(page).filter({ hasText: 'Already known' }).click();
|
||||
await expect(outcomeButtons(page).filter({ hasText: 'Already known' })).toHaveAttribute('aria-pressed', 'true');
|
||||
await fileRows(page).first().click();
|
||||
await expect(page.locator('dialog#viewer-dialog[open] #viewer-stage img').first()).toHaveAttribute('src', /[?&]root=1(&|$)/);
|
||||
await page.keyboard.press('Escape');
|
||||
|
||||
// Errors: plain text rows, no viewer hint.
|
||||
await outcomeButtons(page).filter({ hasText: 'Errors' }).click();
|
||||
await expect(fileRows(page)).toHaveCount(1);
|
||||
expect(await fileRows(page).first().evaluate(node => node.localName)).toBe('div');
|
||||
await expect(fileRows(page).first().locator('.path')).toHaveText('broken/truncated.png');
|
||||
await expect(resultsNote(page)).toHaveCount(0);
|
||||
// An empty group says so.
|
||||
await outcomeButtons(page).filter({ hasText: 'Needs review' }).click();
|
||||
await expect(resultsSection(page).locator('.outcome-files > p.empty-state')).toHaveText('No files in this group.');
|
||||
|
||||
// A listed row whose file has left the listing toasts instead of opening an empty viewer.
|
||||
await step(request, job.id, { records: [{ group: 'added', path: 'gone/missing.png', root: 'output', detail: 'Added' }] });
|
||||
await outcomeButtons(page).first().click();
|
||||
await expect(fileRows(page)).toHaveCount(2);
|
||||
await fileRows(page).nth(1).click();
|
||||
await expect(toast(page)).toHaveText('This file is no longer in Images. Rescan from Import and try again.');
|
||||
await expect(page.locator('dialog#viewer-dialog')).toHaveCount(0);
|
||||
// A row opens only its own root's file: a root whose label no longer matches (a replaced source) never opens the
|
||||
// other root's same path, and a row without a root index opens nothing (review M8, M9).
|
||||
const lookups = [];
|
||||
page.on('request', item => { const url = new URL(item.url()); if (url.pathname === '/prompt_manager/images/output' && url.searchParams.get('path')) lookups.push(url.search); });
|
||||
await step(request, job.id, { records: [
|
||||
{ group: 'added', path: 'shared/same.png', root: 'output', root_index: 1, detail: 'Added' },
|
||||
{ group: 'added', path: 'shared/same.png', root: 'output', root_index: null, detail: 'Added' },
|
||||
] });
|
||||
await expect(fileRows(page)).toHaveCount(4);
|
||||
for (const index of [2, 3]) {
|
||||
await fileRows(page).nth(index).click();
|
||||
await expect(toast(page)).toHaveText('This file is no longer in Images. Rescan from Import and try again.');
|
||||
await expect(page.locator('dialog#viewer-dialog')).toHaveCount(0);
|
||||
}
|
||||
expect(lookups).toEqual([]);
|
||||
// The lookup is the exact identity, not a name search.
|
||||
await fileRows(page).first().click();
|
||||
await expect(page.locator('dialog#viewer-dialog[open]')).toBeVisible();
|
||||
expect(lookups).toEqual(['?mode=workspace&root=0&path=shared%2Fsame.png&limit=5']);
|
||||
await page.keyboard.press('Escape');
|
||||
});
|
||||
|
||||
test('filter narrows the selected outcome', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 4, records: [
|
||||
{ group: 'added', path: 'CCC/portrait_0001.png', detail: 'Added' },
|
||||
{ group: 'added', path: 'CCC/portrait_0002.png', detail: 'Added' },
|
||||
{ group: 'added', path: 'forest/pines_0008.png', detail: 'Added' },
|
||||
{ group: 'errors', path: 'CCC/broken.png', detail: 'Error' },
|
||||
] });
|
||||
await openJob(page, job.id);
|
||||
await expect(fileRows(page)).toHaveCount(3);
|
||||
await expect(filter(page)).toHaveAttribute('placeholder', 'Filter by file path…');
|
||||
await filter(page).fill('ccc/');
|
||||
await expect(fileRows(page).locator('.path')).toHaveText(['CCC/portrait_0001.png', 'CCC/portrait_0002.png']);
|
||||
await expect(resultsNote(page)).toHaveText(`Showing 2 of 3 · ${VIEWER_HINT}`);
|
||||
// The filter stays while another outcome is chosen.
|
||||
await outcomeButtons(page).filter({ hasText: 'Errors' }).click();
|
||||
await expect(fileRows(page).locator('.path')).toHaveText(['CCC/broken.png']);
|
||||
await filter(page).fill('pines');
|
||||
await expect(fileRows(page)).toHaveCount(0);
|
||||
await expect(resultsSection(page).locator('.outcome-files > p.empty-state')).toHaveText('No listed file matches this filter.');
|
||||
await outcomeButtons(page).first().click();
|
||||
await expect(fileRows(page).locator('.path')).toHaveText(['forest/pines_0008.png']);
|
||||
await filter(page).fill('');
|
||||
await expect(fileRows(page)).toHaveCount(3);
|
||||
});
|
||||
|
||||
test('job view renders 200-row cap note', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 260, bulk: { group: 'added', count: 250 }, records: [
|
||||
...Array.from({ length: 10 }, (_, index) => ({ group: 'no_metadata', path: `misc/plain_${index}.png`, detail: 'No prompt metadata in the file' })),
|
||||
] });
|
||||
// Every detail read of this job: in flight, and whether one has returned the full listing.
|
||||
const detailUrl = `/prompt_manager/jobs/${job.id}`;
|
||||
const isDetail = item => new URL(item.url()).pathname === detailUrl;
|
||||
const detailReads = { inFlight: 0, full: false };
|
||||
page.on('request', item => { if (isDetail(item)) detailReads.inFlight += 1; });
|
||||
page.on('requestfailed', item => { if (isDetail(item)) detailReads.inFlight -= 1; });
|
||||
page.on('response', async response => {
|
||||
if (!isDetail(response)) return;
|
||||
const body = await response.json().catch(() => null);
|
||||
const outcomes = body?.job?.outcomes ?? {};
|
||||
if (outcomes.added?.length === 200 && outcomes.no_metadata?.length === 10) detailReads.full = true;
|
||||
detailReads.inFlight -= 1;
|
||||
});
|
||||
await openJob(page, job.id);
|
||||
await expect(tiles(page).first().locator('dd')).toHaveText('250');
|
||||
await expect(fileRows(page)).toHaveCount(200);
|
||||
await expect(resultsNote(page)).toHaveText(`Showing 200 of 250 · ${VIEWER_HINT}`);
|
||||
await outcomeButtons(page).filter({ hasText: 'No usable metadata' }).click();
|
||||
await expect(fileRows(page)).toHaveCount(10);
|
||||
await expect(resultsNote(page)).toHaveCount(0);
|
||||
// Every listed row is in the page and it stays responsive: a filter answers at once.
|
||||
await outcomeButtons(page).first().click();
|
||||
await filter(page).fill('file_00199');
|
||||
await expect(fileRows(page)).toHaveCount(1);
|
||||
await expect(resultsNote(page)).toHaveText(`Showing 1 of 250 · ${VIEWER_HINT}`);
|
||||
|
||||
// Review M12: with the list capped and the log closed, more files only update the counters; the detail is read
|
||||
// again only when the log opens with new lines.
|
||||
// The reads from opening the view have settled: one returned the full listing and none is in flight. The view
|
||||
// re-reads only while what the summary lists differs from the last read, so no further read is due.
|
||||
await expect.poll(() => detailReads.full && detailReads.inFlight === 0).toBe(true);
|
||||
const reads = [];
|
||||
page.on('request', item => { if (new URL(item.url()).pathname === `/prompt_manager/jobs/${job.id}`) reads.push(Date.now()); });
|
||||
await step(request, job.id, { processed: 300, bulk: { group: 'added', count: 40 } });
|
||||
await expect(tiles(page).first().locator('dd')).toHaveText('290');
|
||||
await page.waitForTimeout(1500);
|
||||
expect(reads).toEqual([]);
|
||||
await scanLog(page).locator('summary').click();
|
||||
await expect.poll(() => reads.length).toBe(1);
|
||||
await expect(scanLog(page).locator('pre')).toContainText('bulk/file_00039.png');
|
||||
});
|
||||
|
||||
test('cancel scan ends cancelled with partial counts', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 9, counters: { added: 7, errors: 2 } });
|
||||
await openJob(page, job.id);
|
||||
// From the keyboard: when the job ends and Cancel scan goes away, focus moves to the job's heading (review M4).
|
||||
await heading(page).getByRole('button', { name: 'Cancel scan', exact: true }).focus();
|
||||
await page.keyboard.press('Enter');
|
||||
await expect(toast(page)).toHaveText('Scan cancelled · 7 added');
|
||||
await expect(heading(page).locator('h1')).toBeFocused();
|
||||
await expect(heading(page).locator('p.muted')).toHaveText(/^Cancelled \d\d:\d\d at a file boundary · 7 added · completed work is kept$/);
|
||||
await expect(heading(page).getByRole('button', { name: 'Cancel scan', exact: true })).toHaveCount(0);
|
||||
await expect(progressSection(page).locator('.panel-heading > span.small')).toHaveText('Cancelled after 9 of 19,622 files');
|
||||
expect((await tileValues(page)).map(([, , value]) => value)).toEqual(['7', '0', '0', '0', '2']);
|
||||
expect((await listed(request))[0].state).toBe('cancelled');
|
||||
|
||||
// A failed cancel request keeps the button and says so.
|
||||
const second = await startJob(request);
|
||||
await openJob(page, second.id);
|
||||
await page.route('**/cancel', route => route.fulfill({ status: 500, json: { success: false, error: 'nope' } }));
|
||||
await heading(page).getByRole('button', { name: 'Cancel scan', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('The job could not be cancelled. Try again.');
|
||||
await expect(heading(page).getByRole('button', { name: 'Cancel scan', exact: true })).not.toHaveAttribute('aria-disabled', 'true');
|
||||
});
|
||||
|
||||
test('busy start names the running job', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const running = await startJob(request, { title: 'Database maintenance', kind: 'maintenance' });
|
||||
await startFromImport(page);
|
||||
await expect(toast(page)).toHaveText('Another job is running: Database maintenance.');
|
||||
await expect(toast(page).getByRole('link', { name: 'Database maintenance', exact: true })).toHaveAttribute('href', `#/activity/${running.id}`);
|
||||
await expect(page).toHaveURL(/#\/images$/);
|
||||
expect((await listed(request)).map(item => item.id)).toEqual([running.id]);
|
||||
// Review M14: a toast with a link outlasts the plain 6.5 s, so the link stays reachable.
|
||||
await page.waitForTimeout(7000);
|
||||
await expect(toast(page).getByRole('link', { name: 'Database maintenance', exact: true })).toBeVisible();
|
||||
await toast(page).getByRole('link', { name: 'Database maintenance', exact: true }).click();
|
||||
await expect(heading(page).locator('h1')).toHaveText('Database maintenance');
|
||||
// A job of another kind cancels as "Cancel job".
|
||||
await expect(heading(page).locator('.heading-actions > button')).toHaveText(['Download report', 'Cancel job']);
|
||||
});
|
||||
|
||||
test('an unknown or restarted job view says so', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
await page.goto(`${workspace}#/activity/${'0'.repeat(32)}`);
|
||||
await expect(main(page).locator('section.surface p.empty-state')).toHaveText('This job is not in this session’s Activity. Jobs are kept until ComfyUI restarts.');
|
||||
await expect(main(page).getByRole('link', { name: 'All activity', exact: true })).toHaveAttribute('href', '#/activity');
|
||||
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 5, records: [{ group: 'added', path: 'kept/row.png', detail: 'Added' }] });
|
||||
await openJob(page, job.id);
|
||||
await expect(fileRows(page)).toHaveCount(1);
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
await expect(heading(page).locator('p.muted')).toHaveText(LOST);
|
||||
await expect(heading(page).getByRole('button', { name: 'Cancel scan', exact: true })).toHaveCount(0);
|
||||
// A lost job has no report on the server (review M9): Download report is disabled rather than failing.
|
||||
await expect(heading(page).getByRole('button', { name: 'Download report', exact: true })).toBeDisabled();
|
||||
// What was read before the restart stays on the page.
|
||||
await expect(fileRows(page)).toHaveCount(1);
|
||||
await expect(pill(page)).toBeHidden();
|
||||
});
|
||||
|
||||
// Review M3: a finished job the server no longer lists is gone from this tab too, and its open view says so.
|
||||
test('a finished job the server pruned leaves its open view honest', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 1, records: [{ group: 'added', path: 'kept/row.png', detail: 'Added' }] });
|
||||
await finish(request, job.id);
|
||||
await openJob(page, job.id);
|
||||
await expect(fileRows(page)).toHaveCount(1);
|
||||
await filter(page).focus();
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
// The next periodic list read (15 s) no longer has the finished job.
|
||||
test.slow();
|
||||
await expect(main(page).locator('section.surface p.empty-state')).toHaveText('This job is not in this session’s Activity. Jobs are kept until ComfyUI restarts.', { timeout: 20000 });
|
||||
await expect(progressSection(page)).toHaveCount(0);
|
||||
await expect(heading(page).locator('h1')).toHaveText('Activity');
|
||||
await expect(heading(page).locator('h1')).toBeFocused();
|
||||
await expect(page).toHaveTitle('Activity · Prompt Manager');
|
||||
});
|
||||
|
||||
// Activity-2 Task 5 (D59): Download report and the log's scroll position.
|
||||
test('download report saves the job JSON', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 2, records: [
|
||||
{ group: 'added', path: 'sub/a.png', root: 'output', detail: 'Added' },
|
||||
{ group: 'errors', path: 'broken/b.png', root: 'output', detail: 'Error' },
|
||||
] });
|
||||
await finish(request, job.id);
|
||||
await openJob(page, job.id);
|
||||
const button = heading(page).getByRole('button', { name: 'Download report', exact: true });
|
||||
await expect(button).toBeEnabled();
|
||||
const [download] = await Promise.all([page.waitForEvent('download'), button.click()]);
|
||||
expect(download.suggestedFilename()).toMatch(/^prompt-manager-scan-\d{8}-\d{4}\.json$/);
|
||||
const path = await download.path();
|
||||
const report = JSON.parse(await (await import('node:fs/promises')).readFile(path, 'utf8'));
|
||||
expect(report.job.id).toBe(job.id);
|
||||
expect(report.outcomes.added.rows.map(row => row.path)).toEqual(['sub/a.png']);
|
||||
expect(report.outcomes.errors.truncated).toBe(false);
|
||||
expect(report.log.length).toBeGreaterThan(0);
|
||||
await expect(toast(page)).toHaveText('Job report downloaded.');
|
||||
});
|
||||
|
||||
test('log keeps the reader\'s scroll position', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
const more = (from, count) => Array.from({ length: count }, (_, index) => ({ group: 'known', path: `log/f_${from + index}.png`,
|
||||
root: 'output', detail: 'Known' }));
|
||||
await step(request, job.id, { processed: 60, records: more(0, 60) });
|
||||
await openJob(page, job.id);
|
||||
await scanLog(page).locator('summary').click();
|
||||
const pre = scanLog(page).locator('pre');
|
||||
await expect(pre).toContainText('log/f_59.png');
|
||||
const metrics = () => pre.evaluate(node => ({ top: node.scrollTop, max: node.scrollHeight - node.clientHeight }));
|
||||
// Opening starts at the newest line.
|
||||
await expect.poll(async () => { const m = await metrics(); return m.max > 0 && m.top >= m.max - 2; }).toBe(true);
|
||||
// Pinned to the bottom: new lines are followed.
|
||||
await step(request, job.id, { processed: 90, records: more(60, 30) });
|
||||
await expect(pre).toContainText('log/f_89.png');
|
||||
await expect.poll(async () => { const m = await metrics(); return m.top >= m.max - 2; }).toBe(true);
|
||||
// Scrolled up: the position stays while more lines arrive.
|
||||
await pre.evaluate(node => { node.scrollTop = 40; });
|
||||
await step(request, job.id, { processed: 120, records: more(90, 30) });
|
||||
await expect(pre).toContainText('log/f_119.png');
|
||||
expect((await metrics()).top).toBe(40);
|
||||
// Closed and opened again (review M6): every open starts at the newest line, also after lines arrived meanwhile.
|
||||
await scanLog(page).locator('summary').click();
|
||||
await expect(scanLog(page)).not.toHaveAttribute('open', '');
|
||||
await step(request, job.id, { processed: 150, records: more(120, 30) });
|
||||
await scanLog(page).locator('summary').click();
|
||||
await expect(pre).toContainText('log/f_149.png');
|
||||
await expect.poll(async () => { const m = await metrics(); return m.max > 0 && m.top >= m.max - 2; }).toBe(true);
|
||||
});
|
||||
|
||||
test('a failed report download says so and leaves the button usable', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await finish(request, job.id);
|
||||
await openJob(page, job.id);
|
||||
await page.route(`**/prompt_manager/jobs/${job.id}/report`, route => route.fulfill({ status: 500, body: 'no' }));
|
||||
const button = heading(page).getByRole('button', { name: 'Download report', exact: true });
|
||||
await button.click();
|
||||
await expect(toast(page)).toHaveText('The report could not be downloaded. Try again.');
|
||||
await expect(button).not.toHaveAttribute('aria-disabled', 'true');
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {readFile,writeFile} from 'node:fs/promises';
|
||||
import {createHash} from 'node:crypto';
|
||||
|
||||
@@ -381,7 +381,7 @@ for(const [width,height] of [[1600,1100],[390,844]])for(const theme of ['dark','
|
||||
await recorded(page).getByRole('button',{name:'View selected run',exact:true}).click();await expect(dialog(page)).toContainText(oldRun);await expect(dialog(page).getByRole('button',{name:'Effective',exact:true})).toHaveAttribute('aria-pressed','true');
|
||||
await reviewCapture(page,testInfo,`outside-selected-dialog-${width}-${theme}`,dialog(page));
|
||||
const hashes={};
|
||||
for(const path of ['docs/concepts/workspace/index.html','docs/concepts/workspace/app.js','docs/concepts/workspace/style.css','database/history_reads.py','py/api/history.py','web/js/workspace/history-state.mjs','web/js/workspace/history-media.mjs','web/js/workspace/history.mjs','web/js/workspace/recorded-run.mjs','web/js/workspace/library.css'])hashes[path]=createHash('sha256').update(await readFile(new URL(`../../${path}`,import.meta.url))).digest('hex');
|
||||
for(const path of ['docs/concepts/workspace/index.html','docs/concepts/workspace/app.js','docs/concepts/workspace/style.css','database/history_reads.py','py/api/history.py','web/js/workspace/history-state.mjs','web/js/workspace/history-media.mjs','web/js/workspace/history.mjs','web/js/workspace/recorded-run.mjs','web/js/workspace/workspace.css'])hashes[path]=createHash('sha256').update(await readFile(new URL(`../../${path}`,import.meta.url))).digest('hex');
|
||||
expect(hashes['docs/concepts/workspace/index.html']).toBe('d84712c7284bb232919a1ed9b154575c8ac574f63b2404a4ac4bc06cbf323c03');expect(hashes['docs/concepts/workspace/app.js']).toBe('e33e7652da12a68cd3fe31e94d77d680aa439b6434ceee94f21cc4ce8883263e');expect(hashes['docs/concepts/workspace/style.css']).toBe('2c479ca1a2ddbbd14a550eae27dd731c326ccdb91fe9ddc87e2b81cd5298fce8');
|
||||
const manifest=testInfo.outputPath('final-fix-visual-source-and-scroll.json');await writeFile(manifest,JSON.stringify({width,height,theme,oldRun,before,after,hashes},null,2));
|
||||
await testInfo.attach('final-fix-visual-source-and-scroll.json',{path:manifest,contentType:'application/json'});
|
||||
@@ -832,6 +832,11 @@ test('run dot appears only for pairs with more than one run and opens its run',a
|
||||
await clean.click();const repeat=modalDialog(page,'Repeat run');await expect(repeat).toBeVisible();
|
||||
await expect(repeat.locator('#modal-body > p.small')).toHaveText('Same positive and negative, another execution.');
|
||||
await expect(repeat.locator('#modal-body > .spaced-15 article[data-image-key]')).toHaveCount(1);
|
||||
// D47: as in the concept (a1291-1304), the dialog title names the image; no collection heading or count repeats it.
|
||||
await expect(repeat.locator('.section-caption')).toHaveCount(0);
|
||||
await expect(repeat.getByRole('heading',{name:'Linked images',exact:true})).toHaveCount(0);
|
||||
await expect(repeat.getByText(/ loaded · /)).toHaveCount(0);
|
||||
expect(await repeat.evaluate(node=>[...node.querySelectorAll('h1, h2')].filter(item=>!item.closest('aside')).map(item=>item.textContent))).toEqual(['Repeat run']);
|
||||
expect(reads.at(-1).get('pair_id')).toBe(String(f.clean_pair));expect(reads.at(-1).get('limit')).toBe('1');
|
||||
await page.keyboard.press('Escape');await expect(repeat).toHaveCount(0);await expect(clean).toBeFocused();
|
||||
});
|
||||
@@ -959,3 +964,68 @@ test('dot names never repeat for trimmed titles or titles that read like a fallb
|
||||
expect(new Set(names).size).toBe(names.length);
|
||||
expect(names).toEqual(['Select negative variant 1','Select negative variant 2','Select negative variant 3']);
|
||||
});
|
||||
// #249 F4: the concept's shared gallery (app.js a616-621, style.css 1546-1548) of PR C media cards (C3 activation).
|
||||
test('shared gallery uses concept cards, image opens the viewer, empty copy',async({page,request})=>{
|
||||
const f=page.fixture;await open(page,f.canonical_id);
|
||||
const section=gallery(page),heading=section.locator(':scope > .panel-heading');
|
||||
expect(await section.evaluate(node=>`${node.tagName.toLowerCase()}.${node.className}`)).toBe('section.history-gallery');
|
||||
await expect(heading.locator(':scope > h2')).toHaveText('Shared gallery');
|
||||
// The page opens on the newest lane, a saved variant with no runs: the concept's empty copy.
|
||||
await expect(heading.locator(':scope > span.small')).toHaveText('No negative · same positive prompt');
|
||||
await expect(section.locator('.empty')).toHaveText('Saved variant · no runs yet.');await expect(section.locator('.empty')).toBeVisible();
|
||||
await expect(section.locator('article[data-image-key]')).toHaveCount(0);
|
||||
// D47 (maintainer ruling 2026-10-21): the concept's "Shared gallery" heading is the gallery's only heading, so the
|
||||
// collection's "Linked images" heading and its "N loaded · N linked images" count are gone; the layout row and the
|
||||
// selected-image side panel stay.
|
||||
await expect(section.locator('.section-caption')).toHaveCount(0);
|
||||
await expect(section.getByRole('heading',{name:'Linked images',exact:true})).toHaveCount(0);
|
||||
await expect(section.getByText(/ loaded · /)).toHaveCount(0);
|
||||
// Outside the side panel (whose own heading names the selected image), the gallery has one section heading.
|
||||
expect(await section.evaluate(node=>[...node.querySelectorAll('h1, h2')].filter(item=>!item.closest('aside')).map(item=>item.textContent))).toEqual(['Shared gallery']);
|
||||
await expect(section.getByRole('group',{name:'Image layout',exact:true})).toBeVisible();
|
||||
await expect(section.getByRole('combobox',{name:'Thumbnail size',exact:true})).toBeVisible();
|
||||
await expect(section.getByRole('complementary',{name:'Selected image details',exact:true})).toBeVisible();
|
||||
// As in the concept, the gallery follows the variants panel; recorded executions (#209) stay below it.
|
||||
const order=await page.locator('.library-history').evaluate(node=>[...node.querySelectorAll('section[aria-label]')].map(item=>item.getAttribute('aria-label')).filter(name=>['Saved negative variants','Shared gallery','Recorded executions'].includes(name)));
|
||||
expect(order).toEqual(['Saved negative variants','Shared gallery','Recorded executions']);
|
||||
const dot=variantsPanel(page).getByRole('button',{name:'Select negative Quality baseline',exact:true});await dot.click();await expect(dot).toHaveAttribute('aria-pressed','true');
|
||||
await expect(heading.locator(':scope > span.small')).toHaveText('Quality baseline · same positive prompt');
|
||||
const cards=section.locator('article.media-card[data-image-key]');await expect(cards).toHaveCount(2);
|
||||
await expect(section.locator('.empty')).toBeHidden();
|
||||
const details=await(await request.get(`/prompt_manager/library/${f.canonical_id}`)).json();
|
||||
const pair=details.variants.find(variant=>variant.title==='Quality baseline').id;
|
||||
const listing=await(await request.get(`/prompt_manager/library/${f.canonical_id}/images?pair_id=${pair}`)).json();
|
||||
const first=cards.first(),filename=(await first.locator('.image-open').getAttribute('aria-label')).replace(/^Open /,'');
|
||||
const image=listing.images.find(item=>item.filename===filename);expect(typeof image.favorite).toBe('boolean');
|
||||
// The concept card: art (open button + overlays), then the info row; the star where the read returns favorite and
|
||||
// no checkbox (the selection bar stays Images-only, #259).
|
||||
expect(await first.evaluate(node=>[...node.children].map(child=>child.className))).toEqual(['media-art','media-info']);
|
||||
expect(await first.locator(':scope > .media-art').evaluate(node=>[...node.children].map(child=>`${child.tagName.toLowerCase()}.${child.className}`))).toEqual(['button.image-open','div.image-overlays']);
|
||||
await expect(first.locator('.image-overlays input[type="checkbox"]')).toHaveCount(0);
|
||||
const star=first.locator('.image-overlays > button.star-button');
|
||||
await expect(star).toHaveAccessibleName(`Favorite ${filename}`);await expect(star).toHaveAttribute('aria-pressed',String(image.favorite));
|
||||
await expect(star).toHaveText(image.favorite?'★':'☆');
|
||||
await expect(first.locator('.media-info h3')).toHaveCount(1);
|
||||
await expect(first.locator('.image-card-actions > button.icon-btn')).toHaveAccessibleName(`Inspect ${filename}`);
|
||||
// C3 activation: the image opens the viewer and Escape returns focus to it.
|
||||
const openButton=first.locator('.image-open');await openButton.click();
|
||||
const viewer=page.locator('dialog#viewer-dialog');await expect(viewer).toBeVisible();
|
||||
await page.keyboard.press('Escape');await expect(viewer).toBeHidden();await expect(openButton).toBeFocused();
|
||||
await expect(section.locator(':scope > .panel-heading')).toHaveCSS('justify-content','space-between');
|
||||
await expect(section).toHaveCSS('margin-top','24px');
|
||||
// D47 (maintainer ruling 2026-10-21) covers the history page's dialogs too: the All images dialog is named by its
|
||||
// own title, with no "Linked images" heading or "N loaded · N linked images" count, as the concept's (a1282-1290).
|
||||
await variantsPanel(page).getByRole('button',{name:'All images for this positive',exact:true}).click();
|
||||
const all=page.getByRole('dialog',{name:'All images for this positive',exact:true});
|
||||
await expect(all.locator('article[data-image-key]').first()).toBeVisible();
|
||||
await expect(all.locator('.section-caption')).toHaveCount(0);
|
||||
await expect(all.getByRole('heading',{name:'Linked images',exact:true})).toHaveCount(0);
|
||||
await expect(all.getByText(/ loaded · /)).toHaveCount(0);
|
||||
expect(await all.evaluate(node=>[...node.querySelectorAll('h1, h2')].filter(item=>!item.closest('aside')).map(item=>item.textContent))).toEqual(['All images for this positive']);
|
||||
await page.keyboard.press('Escape');await expect(all).toHaveCount(0);
|
||||
// At 390px the heading stays inside the gallery and the page never scrolls sideways.
|
||||
await page.setViewportSize({width:390,height:844});
|
||||
await expect.poll(()=>page.evaluate(()=>document.documentElement.scrollWidth-document.documentElement.clientWidth)).toBeLessThanOrEqual(1);
|
||||
const [box,text]=await Promise.all([section.boundingBox(),heading.locator(':scope > span.small').boundingBox()]);
|
||||
expect(text.x+text.width).toBeLessThanOrEqual(box.x+box.width+0.5);
|
||||
});
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept, openConcept, openProduction } from './concept-inventory.mjs';
|
||||
|
||||
const workspace = '/prompt_manager/workspace#/images';
|
||||
@@ -174,7 +174,9 @@ for (const embedded of [false, true]) {
|
||||
if (close !== 'parent abort') {
|
||||
await expect(preview).toBeFocused();
|
||||
await expect.poll(() => page.evaluate(() => window.continuationFixture.reads)).toEqual([null, 'next-0', 'next-1']);
|
||||
await expect(caption(page)).toContainText(embedded ? '80 loaded' : '80 of 80');
|
||||
// An embedded collection has no caption (D47); its loaded cards show the continuation instead.
|
||||
if (embedded) { await expect(cards(page)).toHaveCount(80); await expect(caption(page)).toHaveCount(0); }
|
||||
else await expect(caption(page)).toContainText('80 of 80');
|
||||
}
|
||||
await page.evaluate(() => window.continuationFixture.scope.dispose());
|
||||
await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))));
|
||||
@@ -806,7 +808,9 @@ test('images toolbar and filter row match the concept controls', async ({ page,
|
||||
await expect(page.getByRole('button', { name: 'Refresh', exact: true })).toHaveCount(0);
|
||||
await expect(page.getByRole('link', { name: 'Current gallery' })).toHaveCount(0);
|
||||
await expect(main.locator('h2', { hasText: /^All images$/ })).toHaveCount(0);
|
||||
await expect(main.locator('search.toolbar > *')).toHaveCount(4);
|
||||
// app.js toolbar() a347–360: search, category, sort, Save view, Filters.
|
||||
await expect(main.locator('search.toolbar > *')).toHaveCount(5);
|
||||
await expect(main.locator('search.toolbar > button')).toHaveText(['Save view', 'Filters']);
|
||||
await expect(sortBy(page).locator('option')).toHaveText(['Recently added', 'Name A–Z', 'Highest rated', 'Oldest first', 'Name Z–A']);
|
||||
const row = main.locator('.filter-row');
|
||||
await expect(row.locator(':scope > *')).toHaveCount(5);
|
||||
@@ -1483,7 +1487,50 @@ test('a star press leaves the continuation status, count and caption date untouc
|
||||
expect(await page.evaluate(() => window.__statusMutations)).toEqual([]);
|
||||
});
|
||||
|
||||
// Opened before the source and folder discovery answered (a slow server), Refine lists them once it does.
|
||||
test('Refine opened before discovery settles lists the discovered sources and folders', async ({ page }) => {
|
||||
let release, started;
|
||||
const held = new Promise(resolve => { release = resolve; }), hit = new Promise(resolve => { started = resolve; });
|
||||
await page.route('**/prompt_manager/gallery/subfolders?*', async route => { started(); await held; await route.continue(); }, { times: 1 });
|
||||
await page.goto(workspace);
|
||||
await expect(cards(page)).toHaveCount(2);
|
||||
await hit;
|
||||
await filtersButton(page).click();
|
||||
const source = refineDialog(page).getByLabel('Source', { exact: true });
|
||||
const folder = refineDialog(page).getByLabel('Folder', { exact: true });
|
||||
const values = select => select.locator('option').evaluateAll(options => options.map(option => option.value));
|
||||
expect(await values(source)).toEqual(['all']);
|
||||
expect(await values(folder)).toEqual(['']);
|
||||
release();
|
||||
await expect.poll(() => values(source)).toEqual(['all', '0', '1']);
|
||||
expect(await values(folder)).toEqual(['', 'shared']);
|
||||
await expect(source).toHaveValue('all');
|
||||
await source.selectOption('1');
|
||||
await refineDialog(page).getByRole('button', { name: 'Apply filters', exact: true }).click();
|
||||
await expect(cards(page)).toHaveCount(1);
|
||||
await expect(cards(page)).toHaveAttribute('data-image-key', '1:shared/same.png');
|
||||
});
|
||||
|
||||
// #249 final review m1: as in the old field row, a Source offers only its own folders.
|
||||
test('a search reload keeps the Filters label node, so a click landing during the reload is not lost', async ({ page, request }) => {
|
||||
// WebKit drops a click whose mousedown hit a text node that was replaced before mouseup; the reload reports the
|
||||
// (unchanged) active count, so the label must keep its node.
|
||||
await expect(await request.post('/__test__/reset', { data: { extra_images: 80 } })).toBeOK();
|
||||
await page.goto(workspace);
|
||||
const button = filtersButton(page);
|
||||
await button.click();await expect(refineDialog(page).getByLabel('Folder', { exact: true }).locator('option[value="bulk"]')).toHaveCount(1);await page.keyboard.press('Escape');
|
||||
await refine(page, { Source: '0', Folder: 'bulk' });
|
||||
await expect(button).toHaveText('Filters · 2');
|
||||
const label = await button.evaluateHandle(node => node.firstChild);
|
||||
const reloaded = page.waitForResponse(response => {
|
||||
const url = new URL(response.url());return imagePages(url) && url.searchParams.get('q') === 'png';
|
||||
});
|
||||
await search(page).fill('png');
|
||||
await reloaded;
|
||||
await expect(button).toHaveText('Filters · 2');
|
||||
expect(await button.evaluate((node, text) => node.firstChild === text && text.isConnected, label)).toBe(true);
|
||||
});
|
||||
|
||||
test('Refine offers only the chosen source folders', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/add-nested-image')).toBeOK();
|
||||
await page.goto(workspace);
|
||||
@@ -1917,7 +1964,7 @@ test('Set category without a linked image posts nothing and opens no dialog', as
|
||||
await page.route(categoryRoute, route => { posted += 1; return route.continue(); });
|
||||
await checkbox(cards(page).last(), 'same.png').check();
|
||||
await categoryButton(page).click();
|
||||
await expect(toast(page)).toHaveText('None of the selected images has a linked prompt.');
|
||||
await expect(toast(page)).toHaveText("None of the selected images is linked to a saved prompt, so there is no prompt category to set.");
|
||||
await expect(page.locator('dialog[open]')).toHaveCount(0);
|
||||
expect(posted).toBe(0);
|
||||
await expect(bulkbar(page).locator('.bulkbar > strong')).toHaveText('1 selected');
|
||||
@@ -2009,3 +2056,50 @@ test('Set category limits the field to the server length, shows the server count
|
||||
await expect(bulkbar(page)).toBeEmpty();
|
||||
expect(answers).toHaveLength(0);
|
||||
});
|
||||
|
||||
// Activity-1 Task 3 (D55): Import → "Scan for metadata" → Review scan opens the concept's "Review metadata scan".
|
||||
const scanReview = page => page.getByRole('dialog', { name: 'Review metadata scan', exact: true });
|
||||
async function openScanReview(page) {
|
||||
await page.getByRole('button', { name: 'Import', exact: true }).click();
|
||||
const row = importDialog(page).locator('.operation').filter({ has: page.getByRole('heading', { name: 'Scan for metadata', exact: true }) });
|
||||
await row.getByRole('button', { name: 'Review scan', exact: true }).click();
|
||||
await expect(scanReview(page)).toBeVisible();
|
||||
}
|
||||
|
||||
test('scan for metadata opens the review dialog from Import', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
await page.goto(workspace);
|
||||
await expect(cards(page)).toHaveCount(2);
|
||||
// Rescan stays the quick refresh; the metadata scan is its own row under it.
|
||||
await page.getByRole('button', { name: 'Import', exact: true }).click();
|
||||
await expect(importDialog(page).locator('.operation h3')).toHaveText(['Inspect a local image', 'Scan an output folder', 'Scan for metadata']);
|
||||
await expect(importDialog(page).locator('.operation').nth(2).locator('p')).toHaveText('Discover and enrich existing files using the configured sources.');
|
||||
await page.keyboard.press('Escape');
|
||||
await openScanReview(page);
|
||||
const dialog = scanReview(page);
|
||||
await expect(dialog.locator('dl.meta-grid > div > dt')).toHaveText(['Sources', 'Scope']);
|
||||
await expect(dialog.locator('dl.meta-grid > div > dd')).toHaveText(['2 output folders', 'New and unresolved files']);
|
||||
await expect(dialog.locator('#modal-body > p.small')).toHaveText('Enrich positive / negative associations, reuse canonical text and report unchanged files. Do not inflate run counts or recreate existing thumbnails.');
|
||||
await expect(dialog.locator('.notice h3')).toHaveText('Back up the database first (recommended)');
|
||||
await expect(dialog.locator('.dialog-actions button')).toHaveText(['Cancel', 'Start scan']);
|
||||
await expect(dialog.getByRole('button', { name: 'Start scan', exact: true })).toHaveClass(/ca-primary/);
|
||||
|
||||
// Back up now downloads the existing backup, toasts, and keeps the dialog open for the scan.
|
||||
const download = page.waitForEvent('download');
|
||||
await dialog.getByRole('button', { name: 'Back up now', exact: true }).click();
|
||||
expect((await download).suggestedFilename()).toMatch(/^prompts_backup_\d{8}_\d{6}\.db$/);
|
||||
await expect(toast(page)).toHaveText('Database backup downloaded.');
|
||||
await expect(dialog).toBeVisible();
|
||||
// A failed backup says so and downloads nothing.
|
||||
await page.route('**/prompt_manager/backup', route => route.fulfill({ status: 500, json: { success: false, error: 'disk full' } }));
|
||||
await dialog.getByRole('button', { name: 'Back up now', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('The database could not be backed up. Try again.');
|
||||
|
||||
// Cancel closes it and starts nothing.
|
||||
await dialog.getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(scanReview(page)).toHaveCount(0);
|
||||
await expect(page.getByRole('button', { name: 'Import', exact: true })).toBeFocused();
|
||||
const jobs = await (await request.get('/prompt_manager/jobs')).json();
|
||||
expect(jobs.jobs).toEqual([]);
|
||||
});
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {inventoryOf,mapConcept,diffInventory} from './library-editor-canonical-comparison.mjs';
|
||||
|
||||
@@ -708,7 +708,8 @@ test('archive hides a pair, Show archived finds it and Restore keeps it selected
|
||||
// #249 D8 ruling (a): the Library-embedded gallery has no select checkbox and no selection bar.
|
||||
await expect(page.getByRole('region',{name:'Shared gallery',exact:true}).locator('.image-overlays input[type="checkbox"]')).toHaveCount(0);
|
||||
await expect(page.locator('#bulkbar, .bulkbar')).toHaveCount(0);
|
||||
await expect(page.getByRole('region',{name:'Shared gallery',exact:true}).getByText('1 loaded · 1 linked image',{exact:true})).toBeVisible();
|
||||
// D47: the archived pair's gallery is named by its "Shared gallery" heading, with no separate loaded count.
|
||||
await expect(page.getByRole('region',{name:'Shared gallery',exact:true}).locator('.panel-heading .small')).toHaveText(/ · Archived · same positive prompt$/);
|
||||
expect(galleryReads.at(-1).get('archive')).toBe('archived');expect(galleryReads.at(-1).get('pair_id')).toBe(String(f.pair_id));
|
||||
await expect(headingActions(page).getByRole('button',{name:/^Send prompt pair to ComfyUI/})).toBeEnabled();
|
||||
const writes=page.writes.length;
|
||||
@@ -898,7 +899,7 @@ test('long and error states retain footer focus and reflow',async({page})=>{
|
||||
|
||||
// Task A2: the editor dialogs keep matching the concept on the concept base stylesheet, with library-editor.css retired.
|
||||
const CONCEPT_BASE=['js/shared/toolkit/controls.css','lib/viewerjs/viewer.min.css','js/workspace/concept.css',
|
||||
'js/workspace/library.css','js/workspace/workspace.css'];
|
||||
'js/workspace/workspace.css'];
|
||||
for(const [width,height,theme] of [[1600,1100,'dark'],[390,844,'light']])test(`concept base keeps the tags screen and editor dialogs equal to the concept ${width} ${theme}`,async({page})=>{
|
||||
await page.setViewportSize({width,height});const f=page.fixture;const references={};
|
||||
for(const mode of Object.keys(conceptModes)) {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
|
||||
const route='/prompt_manager/workspace#/library';
|
||||
const cards=page=>page.locator('article[data-prompt-id]');
|
||||
@@ -25,6 +25,8 @@ async function loadShelfPages(page,more,pages) {
|
||||
await more.evaluate(node=>node.click());await response;await expect(more).toHaveAttribute('aria-disabled','false');
|
||||
}
|
||||
}
|
||||
/** Resolves in the rendering frame after every frame callback the page had already queued (callbacks run in request order). */
|
||||
const nextFrame=page=>page.evaluate(()=>new Promise(resolve=>requestAnimationFrame(()=>resolve())));
|
||||
async function openHistory(page,text='Cat portrait') {
|
||||
await cards(page).filter({has:page.getByText(text,{exact:true})}).getByRole('button',{name:'Explore history',exact:true}).click();
|
||||
await expect(history(page)).toBeVisible();
|
||||
@@ -103,7 +105,8 @@ test('library infinite loading keeps unique parents and keyboard-accessible hist
|
||||
const appended=await (await nextPage).json();expect(appended.items.length).toBeGreaterThan(0);hasMore=appended.has_more;
|
||||
if(hasMore)await expect(more).toHaveAttribute('aria-disabled','false');
|
||||
}
|
||||
await expect(more).toBeHidden();expect(await cards(page).count()).toBeLessThan(80);
|
||||
// The last page's render queued the window update for the next frame; count after that frame, not before it.
|
||||
await expect(more).toBeHidden();await nextFrame(page);expect(await cards(page).count()).toBeLessThan(80);
|
||||
const ids=await cards(page).evaluateAll(nodes=>nodes.map(node=>node.dataset.promptId));expect(new Set(ids).size).toBe(ids.length);
|
||||
await expect(page.getByRole('button',{name:/next page|previous page/i})).toHaveCount(0);
|
||||
const firstId=await cards(page).first().getAttribute('data-prompt-id');
|
||||
@@ -151,19 +154,19 @@ test('failed library reads retry while late search responses cannot overwrite ne
|
||||
|
||||
test('library toolbar matches the concept and tags live in Refine this view',async({page})=>{
|
||||
await page.goto(route);await expect(cards(page)).toHaveCount(3);
|
||||
// app.js toolbar() a347–360: search.toolbar > label.search-field (input#search), select#category-filter, select#sort, Filters.
|
||||
// app.js toolbar() a347–360: search.toolbar > label.search-field (input#search), select#category-filter, select#sort, Save view, Filters.
|
||||
const toolbar=page.locator('#workspace-main search.toolbar');await expect(toolbar).toHaveCount(1);
|
||||
expect(await toolbar.evaluate(node=>[...node.children].map(child=>child.tagName.toLowerCase()+(child.id?`#${child.id}`:'')+(child.className?`.${child.className.split(' ').filter(name=>!name.startsWith('ca-')).join('.')}`:'')))).toEqual(['label.search-field','select#category-filter','select#sort','button']);
|
||||
expect(await toolbar.evaluate(node=>[...node.children].map(child=>child.tagName.toLowerCase()+(child.id?`#${child.id}`:'')+(child.className?`.${child.className.split(' ').filter(name=>!name.startsWith('ca-')).join('.')}`:'')))).toEqual(['label.search-field','select#category-filter','select#sort','button','button']);
|
||||
await expect(toolbar.locator('#search')).toHaveAttribute('placeholder','Search prompts, tags, filenames…');
|
||||
await expect(toolbar.locator('#sort option')).toHaveText(['Recently added','Name A–Z','Highest rated','Oldest first']);
|
||||
await expect(toolbar.getByRole('button')).toHaveText(['Filters']);
|
||||
await expect(toolbar.getByRole('button')).toHaveText(['Save view','Filters']);
|
||||
await expect(page.getByRole('button',{name:'Refresh library',exact:true})).toHaveCount(0);
|
||||
await expect(page.getByRole('textbox',{name:'Tags',exact:true})).toHaveCount(0);
|
||||
await expect(page.locator('.library-shelf > search.toolbar:first-child')).toHaveCount(1);
|
||||
// Tags apply from the dialog only; Clear clears them and the toolbar category, as the concept's does.
|
||||
await page.getByRole('combobox',{name:'Filter by category',exact:true}).selectOption('Portraits');await applyTags(page,'studio');await expect(cards(page)).toHaveCount(1);
|
||||
// D44: the active Tags filter shows on the button; the category is the toolbar's own and is not counted.
|
||||
await expect(toolbar.getByRole('button')).toHaveText(['Filters · 1']);
|
||||
await expect(toolbar.getByRole('button')).toHaveText(['Save view','Filters · 1']);
|
||||
await toolbar.getByRole('button',{name:'Filters · 1',exact:true}).click();
|
||||
await expect(refine(page).locator('.field label')).toHaveText(['Tags']);await expect(refine(page).getByRole('textbox',{name:'Tags',exact:true})).toHaveValue('studio');
|
||||
await refine(page).getByRole('button',{name:'Clear filters',exact:true}).click();await expect(refine(page)).toHaveCount(0);
|
||||
@@ -376,14 +379,18 @@ async function observeLibrary(page, hold=false) {
|
||||
holdFrames(){holdingFrames=true;},
|
||||
releaseFrames(){holdingFrames=false;const callbacks=[...heldFrames.values()];heldFrames.clear();for(const callback of callbacks)callback();},
|
||||
equal(){for(const record of records){const entries=[...record.latest.values()].filter(entry=>record.targets.has(entry.target));if(entries.length)record.callback(entries,record.native);}},
|
||||
// Resolves once two consecutive rendering frames find no Library frame queued. Each check runs inside a native
|
||||
// frame callback, so a frame the page queues later in that update (a scroll event, a ResizeObserver delivery) is
|
||||
// still pending there. A timer is not a frame: under CPU load a 60 ms wait could end before the scroll's frame.
|
||||
// A frame that never drains (a rAF loop, or settling while frames are held) fails after `limit` frames with the stats.
|
||||
settled(limit=600){return new Promise((resolve,reject)=>{let quiet=0,frames=0;const check=()=>{frames++;quiet=pending.size?0:quiet+1;
|
||||
if(quiet>=2)resolve();else if(frames>=limit)reject(new Error(`Library never settled within ${limit} frames: ${JSON.stringify(window.libraryObservation.stats())}`));else raf(check);};raf(check);});},
|
||||
stats(){return {requested,pending:pending.size,heldFrames:heldFrames.size,measurements:records.reduce((sum,record)=>sum+record.entries.size,0),mutations,unmounted,targets:records.reduce((sum,record)=>sum+record.targets.size,0)};},
|
||||
};
|
||||
},{hold});
|
||||
}
|
||||
async function settleLibrary(page) {
|
||||
await expect.poll(async()=>page.evaluate(()=>window.libraryObservation.stats().pending)).toBe(0);
|
||||
await page.evaluate(()=>new Promise(resolve=>setTimeout(resolve,60)));
|
||||
await expect.poll(async()=>page.evaluate(()=>window.libraryObservation.stats().pending)).toBe(0);
|
||||
await page.evaluate(()=>window.libraryObservation.settled());
|
||||
}
|
||||
async function firstVisibleParent(page) {
|
||||
return cards(page).evaluateAll(nodes=>{
|
||||
@@ -563,7 +570,7 @@ test('saved choices retain curved staggered lanes and the full negative immediat
|
||||
const values=await page.locator('.library-history .variant-lane').evaluateAll(nodes=>nodes.map(node=>({offset:parseFloat(getComputedStyle(node).marginInlineStart),radius:getComputedStyle(node,'::before').borderBottomLeftRadius})));
|
||||
expect(values).toEqual([{offset:38,radius:'22px'},{offset:85,radius:'22px'},{offset:132,radius:'22px'},{offset:179,radius:'22px'}]);
|
||||
const order=await page.locator('.library-history').evaluate(node=>[...node.querySelectorAll('section[aria-label]')].filter(section=>['Saved negative variants','Recorded executions','Shared gallery'].includes(section.getAttribute('aria-label'))).map(section=>section.getAttribute('aria-label')));
|
||||
expect(order).toEqual(['Saved negative variants','Recorded executions','Shared gallery']);
|
||||
expect(order).toEqual(['Saved negative variants','Shared gallery','Recorded executions']);
|
||||
const rows=await(await request.get('/prompt_manager/library')).json();const cat=rows.items.find(item=>item.text==='Cat portrait');
|
||||
const details=await(await request.get(`/prompt_manager/library/${cat.id}`)).json();
|
||||
const selected=details.variants[1].negative_text;
|
||||
@@ -683,6 +690,17 @@ test('title and Explore history are buttons that open history',async({page,reque
|
||||
await page.keyboard.press('ArrowLeft');await expect(cards(page).nth(1).locator('.card-title > button')).toBeFocused();
|
||||
});
|
||||
|
||||
test('settling fails with the observation stats when a Library frame stays queued',async({page,request})=>{
|
||||
await request.post('/__test__/reset',{data:{native_library:true,extra_prompts:200}});
|
||||
await observeLibrary(page);await page.addInitScript(()=>{window.IntersectionObserver=undefined;});
|
||||
await page.goto(route);await expect(cards(page).first()).toBeVisible();await settleLibrary(page);
|
||||
await page.evaluate(()=>{window.libraryObservation.holdFrames();dispatchEvent(new Event('scroll'));});
|
||||
await expect.poll(()=>page.evaluate(()=>window.libraryObservation.stats().heldFrames)).toBeGreaterThan(0);
|
||||
const failure=await page.evaluate(()=>window.libraryObservation.settled(30).then(()=>'settled',error=>error.message));
|
||||
expect(failure).toMatch(/^Library never settled within 30 frames: \{.*"heldFrames":[1-9]/);
|
||||
await page.evaluate(()=>window.libraryObservation.releaseFrames());await settleLibrary(page);
|
||||
});
|
||||
|
||||
test('virtualized rows keep measured heights with the clamp',async({page,request,isMobile})=>{
|
||||
await request.post('/__test__/reset',{data:{native_library:true,extra_prompts:200}});
|
||||
await observeLibrary(page);await page.addInitScript(()=>{window.IntersectionObserver=undefined;});
|
||||
|
||||
@@ -0,0 +1,546 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
|
||||
// Saved views (#249 D16, maintainer ruling 2026-10-21): the toolbar's Save view (concept app.js a359, a1330–1346),
|
||||
// the sidebar SAVED VIEWS rows (a331–335) and `#/saved/<id>` (a1158–1162), on Images and the Prompt library.
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
const API = '/prompt_manager/saved-views';
|
||||
const main = page => page.locator('#workspace-main');
|
||||
const toolbar = page => main(page).locator('search.toolbar');
|
||||
const search = page => page.getByRole('searchbox', { name: 'Search workspace', exact: true });
|
||||
const sortBy = page => page.getByRole('combobox', { name: 'Sort results', exact: true });
|
||||
const filtersButton = page => toolbar(page).getByRole('button', { name: /^Filters(?: · \d+)?$/ });
|
||||
const imageCards = page => page.locator('article[data-image-key]');
|
||||
const promptCards = page => page.locator('article[data-prompt-id]');
|
||||
const savedNav = page => page.getByRole('navigation', { name: 'Saved views', exact: true });
|
||||
// The saved views' own rows, after the Favorites row (D51).
|
||||
const viewRows = page => savedNav(page).locator('a[data-route^="saved/"]');
|
||||
const favoritesRow = page => savedNav(page).locator('a[data-route="favorites"]');
|
||||
const favoritesCount = page => favoritesRow(page).locator('.nav-count');
|
||||
const editDialog = page => page.getByRole('dialog', { name: 'Edit saved view', exact: true });
|
||||
const deleteDialog = page => page.getByRole('dialog', { name: 'Delete saved view', exact: true });
|
||||
const heading = page => main(page).getByRole('heading', { level: 1 });
|
||||
const saveDialog = page => page.getByRole('dialog', { name: 'Save this view', exact: true });
|
||||
const refineDialog = page => page.getByRole('dialog', { name: 'Refine this view', exact: true });
|
||||
const toast = page => page.locator('#workspace-toast');
|
||||
const IMAGE_DEFAULTS = { q: '', category: '', sort: 'newest', source: 'all', folder: '', media: 'all', tags: [], favorites: false };
|
||||
|
||||
async function refine(page, values) {
|
||||
await filtersButton(page).click();
|
||||
for (const [label, value] of Object.entries(values)) {
|
||||
const control = refineDialog(page).getByLabel(label, { exact: true });
|
||||
if (label === 'Tags') await control.fill(value);
|
||||
else await control.selectOption(value);
|
||||
}
|
||||
await refineDialog(page).getByRole('button', { name: 'Apply filters', exact: true }).click();
|
||||
await expect(refineDialog(page)).toHaveCount(0);
|
||||
}
|
||||
|
||||
async function listed(request) {
|
||||
const response = await request.get(API);
|
||||
await expect(response).toBeOK();
|
||||
return (await response.json()).views;
|
||||
}
|
||||
|
||||
async function create(request, data) {
|
||||
const response = await request.post(API, { data });
|
||||
expect(response.status()).toBe(201);
|
||||
return (await response.json()).view;
|
||||
}
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await expect(await request.post('/__test__/reset')).toBeOK();
|
||||
page.savedErrors = [];
|
||||
page.on('pageerror', error => page.savedErrors.push(error.message));
|
||||
// D51: deleting asks inside the dialog; no browser dialog (window.confirm) may ever open.
|
||||
page.on('dialog', dialog => { page.savedErrors.push(`browser ${dialog.type()}`); dialog.dismiss().catch(() => {}); });
|
||||
// No listing read may fail on a saved view's values (an unknown Source would be a 400).
|
||||
page.on('response', response => {
|
||||
if (response.status() >= 400 && new URL(response.url()).pathname === '/prompt_manager/images/output') {
|
||||
page.savedErrors.push(`${response.status()} ${response.url()}`);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
expect(page.savedErrors).toEqual([]);
|
||||
});
|
||||
|
||||
test('Save view saves the Images toolbar and Refine filters, and its sidebar row reopens them', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await expect(imageCards(page)).toHaveCount(2);
|
||||
// The concept's sidebar group: SAVED VIEWS without its PROPOSED badge (D49), empty until a view is saved.
|
||||
await expect(page.locator('aside.sidebar .sidebar-label')).toHaveText(['WORKSPACE', 'SAVED VIEWS']);
|
||||
await expect(viewRows(page)).toHaveCount(0);
|
||||
|
||||
await search(page).fill('dogs');
|
||||
await expect(imageCards(page)).toHaveCount(1);
|
||||
await sortBy(page).selectOption('name');
|
||||
await refine(page, { Source: '1', Media: 'image' });
|
||||
await expect(filtersButton(page)).toHaveText('Filters · 2');
|
||||
const saveButton = toolbar(page).getByRole('button', { name: 'Save view', exact: true });
|
||||
await saveButton.click();
|
||||
// The concept's simpleInput: "View name", the search's default name, Cancel and the primary action.
|
||||
const dialog = saveDialog(page);
|
||||
await expect(dialog.getByRole('textbox', { name: 'View name', exact: true })).toHaveValue('Search: dogs');
|
||||
await expect(dialog.getByRole('textbox', { name: 'View name', exact: true })).toBeFocused();
|
||||
await expect(dialog.locator('.dialog-actions button')).toHaveText(['Cancel', 'Save']);
|
||||
await dialog.getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(dialog).toHaveCount(0);
|
||||
await expect(saveButton).toBeFocused();
|
||||
expect(await listed(request)).toEqual([]);
|
||||
|
||||
await saveButton.click();
|
||||
await dialog.getByRole('textbox', { name: 'View name', exact: true }).fill('Dog shots');
|
||||
await dialog.getByRole('button', { name: 'Save', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('Saved view added to the sidebar.');
|
||||
const [view] = await listed(request);
|
||||
expect(view).toMatchObject({ page: 'images', name: 'Dog shots',
|
||||
state: { ...IMAGE_DEFAULTS, q: 'dogs', sort: 'name', source: '1', media: 'image' } });
|
||||
// The concept's links() row: the search icon, then the name, and no count.
|
||||
const row = savedNav(page).getByRole('link', { name: 'Dog shots', exact: true });
|
||||
await expect(row).toHaveAttribute('href', `#/saved/${view.id}`);
|
||||
expect(await row.evaluate(link => [...link.childNodes].map(node => node.nodeType === Node.TEXT_NODE ? `text:${node.textContent}`
|
||||
: `${node.localName}.${node.getAttribute('class')}`))).toEqual(['svg.icon', 'text:Dog shots']);
|
||||
await expect(row).not.toHaveAttribute('aria-current', 'page');
|
||||
|
||||
// Another page, then the saved row: its page opens with every saved value and the Filters count (D44).
|
||||
await page.getByRole('navigation', { name: 'Workspace', exact: true }).getByRole('link', { name: /^Prompt library/ }).click();
|
||||
await expect(promptCards(page).first()).toBeVisible();
|
||||
await row.click();
|
||||
await expect(page).toHaveURL(new RegExp(`#/saved/${view.id}$`));
|
||||
await expect(main(page).getByRole('heading', { level: 1 })).toHaveText('Dog shots');
|
||||
await expect(main(page).getByRole('heading', { level: 1 })).toBeFocused();
|
||||
await expect(page.locator('#workspace-breadcrumb')).toHaveText('Dog shots');
|
||||
await expect(page).toHaveTitle('Dog shots · Prompt Manager');
|
||||
await expect(row).toHaveAttribute('aria-current', 'page');
|
||||
await expect(page.locator('aside.sidebar nav a[aria-current]')).toHaveCount(1);
|
||||
await expect(search(page)).toHaveValue('dogs');
|
||||
await expect(sortBy(page)).toHaveValue('name');
|
||||
await expect(filtersButton(page)).toHaveText('Filters · 2');
|
||||
await expect(imageCards(page)).toHaveCount(1);
|
||||
await expect(imageCards(page)).toHaveAttribute('data-image-key', '1:shared/same.png');
|
||||
await filtersButton(page).click();
|
||||
await expect(refineDialog(page).getByLabel('Source', { exact: true })).toHaveValue('1');
|
||||
await expect(refineDialog(page).getByLabel('Media', { exact: true })).toHaveValue('image');
|
||||
await page.keyboard.press('Escape');
|
||||
|
||||
// A reload opens the same view from its link.
|
||||
await page.reload();
|
||||
await expect(main(page).getByRole('heading', { level: 1 })).toHaveText('Dog shots');
|
||||
await expect(imageCards(page)).toHaveCount(1);
|
||||
await expect(savedNav(page).getByRole('link', { name: 'Dog shots', exact: true })).toHaveAttribute('aria-current', 'page');
|
||||
});
|
||||
|
||||
test('an empty name is Saved view, and a Library view reopens the shelf filters', async ({ page, request }) => {
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await expect(imageCards(page)).toHaveCount(2);
|
||||
await toolbar(page).getByRole('button', { name: 'Save view', exact: true }).click();
|
||||
const name = saveDialog(page).getByRole('textbox', { name: 'View name', exact: true });
|
||||
await expect(name).toHaveValue('My workspace view');
|
||||
await name.fill(' ');
|
||||
await saveDialog(page).getByRole('button', { name: 'Save', exact: true }).click();
|
||||
await expect(viewRows(page)).toHaveText(['Saved view']);
|
||||
expect((await listed(request))[0]).toMatchObject({ page: 'images', name: 'Saved view', state: IMAGE_DEFAULTS });
|
||||
|
||||
await page.goto(`${workspace}#/library`);
|
||||
await expect(promptCards(page)).toHaveCount(2);
|
||||
await search(page).fill('dog');
|
||||
await expect(promptCards(page)).toHaveCount(1);
|
||||
await refine(page, { Tags: 'dogs' });
|
||||
await expect(filtersButton(page)).toHaveText('Filters · 1');
|
||||
await toolbar(page).getByRole('button', { name: 'Save view', exact: true }).click();
|
||||
await expect(name).toHaveValue('Search: dog');
|
||||
await saveDialog(page).getByRole('button', { name: 'Save', exact: true }).click();
|
||||
await expect(viewRows(page)).toHaveText(['Saved view', 'Search: dog']);
|
||||
const library = (await listed(request))[1];
|
||||
expect(library).toMatchObject({ page: 'library', name: 'Search: dog', state: { q: 'dog', category: '', sort: 'newest', tags: ['dogs'] } });
|
||||
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await expect(imageCards(page)).toHaveCount(2);
|
||||
await savedNav(page).getByRole('link', { name: 'Search: dog', exact: true }).click();
|
||||
await expect(main(page).getByRole('heading', { level: 1 })).toHaveText('Search: dog');
|
||||
await expect(search(page)).toHaveValue('dog');
|
||||
await expect(filtersButton(page)).toHaveText('Filters · 1');
|
||||
await expect(promptCards(page)).toHaveCount(1);
|
||||
// Saved views open the page; one saved view to another re-renders with the other's state.
|
||||
await savedNav(page).getByRole('link', { name: 'Saved view', exact: true }).click();
|
||||
await expect(main(page).getByRole('heading', { level: 1 })).toHaveText('Saved view');
|
||||
await expect(search(page)).toHaveValue('');
|
||||
await expect(imageCards(page)).toHaveCount(2);
|
||||
});
|
||||
|
||||
test('a saved value the page no longer accepts stays default while the rest applies', async ({ page, request }) => {
|
||||
const view = await create(request, { page: 'images', name: 'Gone source',
|
||||
state: { ...IMAGE_DEFAULTS, q: 'cat', sort: 'name-desc', source: '7', folder: 'missing/folder' } });
|
||||
await page.goto(`${workspace}#/saved/${view.id}`);
|
||||
await expect(main(page).getByRole('heading', { level: 1 })).toHaveText('Gone source');
|
||||
await expect(imageCards(page)).toHaveCount(1);
|
||||
await expect(imageCards(page)).toHaveAttribute('data-image-key', '0:shared/same.png');
|
||||
await expect(search(page)).toHaveValue('cat');
|
||||
await expect(sortBy(page)).toHaveValue('name-desc');
|
||||
// The missing Source and Folder are dropped, so no refinement is active.
|
||||
await expect(filtersButton(page)).toHaveText('Filters');
|
||||
await expect(main(page).locator('.filter-row > span.chip')).toHaveText('All output folders');
|
||||
|
||||
// A Source that does exist, with a folder it lists, applies.
|
||||
const kept = await create(request, { page: 'images', name: 'Input shared',
|
||||
state: { ...IMAGE_DEFAULTS, source: '1', folder: 'shared' } });
|
||||
await page.goto(`${workspace}#/saved/${kept.id}`);
|
||||
await expect(main(page).getByRole('heading', { level: 1 })).toHaveText('Input shared');
|
||||
await expect(filtersButton(page)).toHaveText('Filters · 2');
|
||||
await expect(imageCards(page)).toHaveCount(1);
|
||||
await expect(imageCards(page)).toHaveAttribute('data-image-key', '1:shared/same.png');
|
||||
});
|
||||
|
||||
test('a view stored under older rules still lists and opens with its usable values (#249 I1)', async ({ page, request }) => {
|
||||
// No favorites key, a retired key and a sort that is no longer offered, as an older build stored it.
|
||||
const raw = async data => {
|
||||
const response = await request.post('/__test__/saved-views/raw', { data });
|
||||
await expect(response).toBeOK();
|
||||
return (await response.json()).id;
|
||||
};
|
||||
const imagesId = await raw({ page: 'images', name: 'Old cats', state: JSON.stringify({ q: 'cat', sort: 'shuffle', layout: 'grid' }) });
|
||||
await raw({ page: 'library', name: '', state: JSON.stringify({ q: 'dog', sort: 'name-desc' }) });
|
||||
expect(await listed(request)).toEqual([
|
||||
{ id: imagesId, page: 'images', name: 'Old cats', state: { ...IMAGE_DEFAULTS, q: 'cat' } },
|
||||
expect.objectContaining({ page: 'library', name: 'Saved view', state: { q: 'dog', category: '', sort: 'newest', tags: [] } }),
|
||||
]);
|
||||
// A state that is no JSON object is never listed, and its link behaves like a missing view.
|
||||
const corruptId = await raw({ page: 'images', name: 'Corrupt', state: '{not json' });
|
||||
expect((await listed(request)).map(view => view.id)).not.toContain(corruptId);
|
||||
await page.goto(`${workspace}#/saved/${corruptId}`);
|
||||
await expect(page).toHaveURL(/#\/images$/);
|
||||
await expect(heading(page)).toHaveText('Images');
|
||||
await expect(viewRows(page)).toHaveText(['Old cats', 'Saved view']);
|
||||
|
||||
await page.goto(`${workspace}#/saved/${imagesId}`);
|
||||
await expect(heading(page)).toHaveText('Old cats');
|
||||
await expect(viewRows(page)).toHaveText(['Old cats', 'Saved view']);
|
||||
await expect(search(page)).toHaveValue('cat');
|
||||
await expect(sortBy(page)).toHaveValue('newest');
|
||||
await expect(imageCards(page)).toHaveCount(1);
|
||||
await savedNav(page).getByRole('link', { name: 'Saved view', exact: true }).click();
|
||||
await expect(heading(page)).toHaveText('Saved view');
|
||||
await expect(search(page)).toHaveValue('dog');
|
||||
await expect(promptCards(page)).toHaveCount(1);
|
||||
});
|
||||
|
||||
test('leaving a saved view returns Images and the Library to their own last state (#249 M2)', async ({ page, request }) => {
|
||||
const dogs = await create(request, { page: 'images', name: 'Dogs', state: { ...IMAGE_DEFAULTS, q: 'dogs' } });
|
||||
const cats = await create(request, { page: 'library', name: 'Cats', state: { q: 'cat', category: '', sort: 'newest', tags: [] } });
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await search(page).fill('cat');
|
||||
await expect(imageCards(page)).toHaveCount(1);
|
||||
await viewRows(page).filter({ hasText: 'Dogs' }).click();
|
||||
await expect(heading(page)).toHaveText('Dogs');
|
||||
await expect(search(page)).toHaveValue('dogs');
|
||||
await page.locator('#primary-nav a[data-route="images"]').click();
|
||||
await expect(heading(page)).toHaveText('Images');
|
||||
await expect(search(page)).toHaveValue('cat');
|
||||
await expect(imageCards(page)).toHaveCount(1);
|
||||
|
||||
await page.locator('#primary-nav a[data-route="library"]').click();
|
||||
await expect(promptCards(page)).toHaveCount(2);
|
||||
await search(page).fill('dog');
|
||||
await expect(promptCards(page)).toHaveCount(1);
|
||||
await page.goto(`${workspace}#/saved/${cats.id}`);
|
||||
await expect(heading(page)).toHaveText('Cats');
|
||||
await expect(search(page)).toHaveValue('cat');
|
||||
await page.locator('#primary-nav a[data-route="library"]').click();
|
||||
await expect(heading(page)).toHaveText('Prompt library');
|
||||
await expect(search(page)).toHaveValue('dog');
|
||||
await expect(promptCards(page)).toHaveCount(1);
|
||||
expect(dogs.id).not.toBe(cats.id);
|
||||
});
|
||||
|
||||
test('a toolbar change while sources load keeps the saved Source and Folder (#249 M3)', async ({ page, request }) => {
|
||||
const view = await create(request, { page: 'images', name: 'Input shared', state: { ...IMAGE_DEFAULTS, source: '1', folder: 'shared' } });
|
||||
let release;
|
||||
const held = new Promise(resolve => { release = resolve; });
|
||||
await page.route('**/prompt_manager/gallery/subfolders?**', async route => { await held; await route.fallback(); });
|
||||
await page.goto(`${workspace}#/saved/${view.id}`);
|
||||
await expect(heading(page)).toHaveText('Input shared');
|
||||
await sortBy(page).selectOption('name');
|
||||
await expect(imageCards(page)).toHaveCount(0);
|
||||
release();
|
||||
await expect(filtersButton(page)).toHaveText('Filters · 2');
|
||||
await expect(sortBy(page)).toHaveValue('name');
|
||||
await expect(imageCards(page)).toHaveCount(1);
|
||||
await expect(imageCards(page)).toHaveAttribute('data-image-key', '1:shared/same.png');
|
||||
});
|
||||
|
||||
test('a saved view left while its list or rename is pending leaves the next page alone', async ({ page, request }) => {
|
||||
const view = await create(request, { page: 'library', name: 'Cats', state: { q: 'cat', category: '', sort: 'newest', tags: [] } });
|
||||
let releaseList = () => {};
|
||||
const holdList = async () => {
|
||||
const held = new Promise(resolve => { releaseList = resolve; });
|
||||
await page.unroute(`**${API}`);
|
||||
await page.route(`**${API}`, async route => { if (route.request().method() === 'GET') await held; await route.fallback(); });
|
||||
};
|
||||
// Leaving while the shell's first read is pending: the view never mounts over the page left to.
|
||||
await holdList();
|
||||
await page.goto(`${workspace}#/saved/${view.id}`);
|
||||
await page.evaluate(() => { window.location.hash = '#/tags'; });
|
||||
await expect(heading(page)).toHaveText('Tags & categories');
|
||||
releaseList();
|
||||
await expect(viewRows(page)).toHaveText(['Cats']);
|
||||
await expect(heading(page)).toHaveText('Tags & categories');
|
||||
// An unknown id re-reads the list once; leaving during that read keeps the page left to.
|
||||
await holdList();
|
||||
await page.evaluate(() => { window.location.hash = '#/saved/999'; });
|
||||
await page.evaluate(() => { window.location.hash = '#/library'; });
|
||||
await expect(heading(page)).toHaveText('Prompt library');
|
||||
releaseList();
|
||||
await expect(viewRows(page)).toHaveText(['Cats']);
|
||||
await expect(page).toHaveURL(/#\/library$/);
|
||||
await expect(heading(page)).toHaveText('Prompt library');
|
||||
await page.unroute(`**${API}`);
|
||||
|
||||
// A rename that lands after the view was left does not retitle the next page.
|
||||
let releaseRename;
|
||||
const renameHeld = new Promise(resolve => { releaseRename = resolve; });
|
||||
await page.route(`**${API}/${view.id}`, async route => { await renameHeld; await route.fallback(); });
|
||||
await page.goto(`${workspace}#/saved/${view.id}`);
|
||||
await expect(heading(page)).toHaveText('Cats');
|
||||
await main(page).locator('.heading-actions > button').first().click();
|
||||
await editDialog(page).getByRole('textbox', { name: 'View name', exact: true }).fill('Kittens');
|
||||
await editDialog(page).getByRole('button', { name: 'Save', exact: true }).click();
|
||||
await page.locator('#primary-nav a[data-route="images"]').click();
|
||||
await expect(heading(page)).toHaveText('Images');
|
||||
releaseRename();
|
||||
await expect(viewRows(page)).toHaveText(['Kittens']);
|
||||
await expect(heading(page)).toHaveText('Images');
|
||||
await expect(page).toHaveTitle('Images · Prompt Manager');
|
||||
});
|
||||
|
||||
test('an unknown saved view opens Images, and a failed listing offers Retry', async ({ page, request }) => {
|
||||
await page.goto(`${workspace}#/saved/999`);
|
||||
await expect(page).toHaveURL(/#\/images$/);
|
||||
await expect(main(page).getByRole('heading', { level: 1 })).toHaveText('Images');
|
||||
|
||||
const view = await create(request, { page: 'library', name: 'Cats', state: { q: 'cat', category: '', sort: 'newest', tags: [] } });
|
||||
let failing = true;
|
||||
await page.route(`**${API}`, route => failing && route.request().method() === 'GET'
|
||||
? route.fulfill({ status: 500, contentType: 'application/json', body: '{"success":false,"error":"down"}' })
|
||||
: route.fallback());
|
||||
await page.goto(`${workspace}#/saved/${view.id}`);
|
||||
await expect(main(page).getByRole('alert')).toHaveText('Saved views could not be loaded. Your saved data has not changed.');
|
||||
await expect(viewRows(page)).toHaveCount(0);
|
||||
failing = false;
|
||||
await main(page).getByRole('button', { name: 'Retry', exact: true }).click();
|
||||
await expect(main(page).getByRole('heading', { level: 1 })).toHaveText('Cats');
|
||||
await expect(search(page)).toHaveValue('cat');
|
||||
await expect(promptCards(page)).toHaveCount(1);
|
||||
await expect(viewRows(page)).toHaveText(['Cats']);
|
||||
});
|
||||
|
||||
test('a failed save says so and adds nothing', async ({ page, request }) => {
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await expect(imageCards(page)).toHaveCount(2);
|
||||
await page.route(`**${API}`, route => route.request().method() === 'POST'
|
||||
? route.fulfill({ status: 500, contentType: 'application/json', body: '{"success":false,"error":"Saved view operation failed"}' })
|
||||
: route.fallback());
|
||||
await toolbar(page).getByRole('button', { name: 'Save view', exact: true }).click();
|
||||
await saveDialog(page).getByRole('button', { name: 'Save', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('The view could not be saved. Try again.');
|
||||
await expect(viewRows(page)).toHaveCount(0);
|
||||
expect(await listed(request)).toEqual([]);
|
||||
});
|
||||
|
||||
async function star(page, key) {
|
||||
await page.locator(`article[data-image-key="${key}"] .star-button`).click();
|
||||
}
|
||||
|
||||
test('Favorites is the first SAVED VIEWS row, with a real count that follows every star', async ({ page }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
let release;
|
||||
const held = new Promise(resolve => { release = resolve; });
|
||||
// Hidden until its read settles: never a placeholder "0".
|
||||
await page.route(url => url.pathname.endsWith('/prompt_manager/images/output') && url.searchParams.get('favorites') === '1'
|
||||
&& url.searchParams.get('limit') === '1', async route => { await held; await route.fallback(); }, { times: 1 });
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await expect(imageCards(page)).toHaveCount(2);
|
||||
await expect(favoritesRow(page)).toHaveAttribute('href', '#/favorites');
|
||||
await expect(favoritesCount(page)).toHaveCount(0);
|
||||
release();
|
||||
// The concept's row (app.js a332): the star icon, "Favorites", then the count.
|
||||
await expect(favoritesCount(page)).toHaveText('0');
|
||||
expect(await favoritesRow(page).evaluate(link => [...link.childNodes].map(node => node.nodeType === Node.TEXT_NODE
|
||||
? `text:${node.textContent}` : `${node.localName}.${node.getAttribute('class')}`)))
|
||||
.toEqual(['svg.icon', 'text:Favorites', 'span.nav-count']);
|
||||
expect(await savedNav(page).locator('a').evaluateAll(links => links.map(link => link.dataset.route))).toEqual(['favorites']);
|
||||
|
||||
await star(page, '1:shared/same.png');
|
||||
await expect(toast(page)).toHaveText('Added to favorites.');
|
||||
await expect(favoritesCount(page)).toHaveText('1');
|
||||
|
||||
// #/favorites: Images filtered to favorites, named Favorites (concept renderImages a417).
|
||||
const listings = [];
|
||||
page.on('request', item => {
|
||||
const url = new URL(item.url());
|
||||
if (url.pathname.endsWith('/prompt_manager/images/output') && url.searchParams.get('limit') !== '1') listings.push(url.searchParams);
|
||||
});
|
||||
await favoritesRow(page).click();
|
||||
await expect(page).toHaveURL(/#\/favorites$/);
|
||||
await expect(heading(page)).toHaveText('Favorites');
|
||||
await expect(heading(page)).toBeFocused();
|
||||
await expect(page.locator('#workspace-breadcrumb')).toHaveText('Favorites');
|
||||
await expect(page).toHaveTitle('Favorites · Prompt Manager');
|
||||
await expect(favoritesRow(page)).toHaveAttribute('aria-current', 'page');
|
||||
await expect(page.locator('aside.sidebar nav a[aria-current]')).toHaveCount(1);
|
||||
await expect(imageCards(page)).toHaveCount(1);
|
||||
await expect(imageCards(page)).toHaveAttribute('data-image-key', '1:shared/same.png');
|
||||
await expect(main(page).locator('#result-count')).toHaveText(/^1 of 1 image/);
|
||||
expect(listings.at(-1).get('favorites')).toBe('1');
|
||||
|
||||
// Unstarring on Favorites keeps the card (with ☆) so it can be undone in place; the count follows.
|
||||
await star(page, '1:shared/same.png');
|
||||
await expect(toast(page)).toHaveText('Removed from favorites.');
|
||||
await expect(imageCards(page).locator('.star-button')).toHaveAttribute('aria-pressed', 'false');
|
||||
await expect(favoritesCount(page)).toHaveText('0');
|
||||
await star(page, '1:shared/same.png');
|
||||
await expect(favoritesCount(page)).toHaveText('1');
|
||||
|
||||
// Favorites never replaces the Images return state: Images still lists everything.
|
||||
await page.getByRole('navigation', { name: 'Workspace', exact: true }).getByRole('link', { name: /^Images/ }).click();
|
||||
await expect(heading(page)).toHaveText('Images');
|
||||
await expect(imageCards(page)).toHaveCount(2);
|
||||
await page.goto(`${workspace}#/favorites`);
|
||||
await expect(heading(page)).toHaveText('Favorites');
|
||||
await expect(imageCards(page)).toHaveCount(1);
|
||||
});
|
||||
|
||||
test('a failed Favorites count shows no badge', async ({ page }) => {
|
||||
await page.route(url => url.pathname.endsWith('/prompt_manager/images/output') && url.searchParams.get('favorites') === '1'
|
||||
&& url.searchParams.get('limit') === '1', route => route.fulfill({ status: 503, json: { success: false, error: 'Unavailable' } }));
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await expect(page.locator('#primary-nav a[data-route="images"] .nav-count')).toHaveText('2');
|
||||
await expect(favoritesRow(page)).toBeVisible();
|
||||
await expect(favoritesCount(page)).toHaveCount(0);
|
||||
// The refused count read is this test's own; nothing else failed.
|
||||
page.savedErrors = page.savedErrors.filter(error => !/^503 .*favorites=1/.test(error));
|
||||
});
|
||||
|
||||
test('Save view on Favorites reopens the favorites', async ({ page, request }) => {
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await expect(imageCards(page)).toHaveCount(2);
|
||||
await star(page, '0:shared/same.png');
|
||||
await expect(favoritesCount(page)).toHaveText('1');
|
||||
await page.goto(`${workspace}#/favorites`);
|
||||
await expect(imageCards(page)).toHaveCount(1);
|
||||
await toolbar(page).getByRole('button', { name: 'Save view', exact: true }).click();
|
||||
await saveDialog(page).getByRole('textbox', { name: 'View name', exact: true }).fill('Best');
|
||||
await saveDialog(page).getByRole('button', { name: 'Save', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('Saved view added to the sidebar.');
|
||||
const [view] = await listed(request);
|
||||
expect(view).toMatchObject({ page: 'images', name: 'Best', state: { ...IMAGE_DEFAULTS, favorites: true } });
|
||||
await expect(viewRows(page)).toHaveText(['Best']);
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await expect(imageCards(page)).toHaveCount(2);
|
||||
await viewRows(page).click();
|
||||
await expect(heading(page)).toHaveText('Best');
|
||||
await expect(imageCards(page)).toHaveCount(1);
|
||||
await expect(imageCards(page)).toHaveAttribute('data-image-key', '0:shared/same.png');
|
||||
});
|
||||
|
||||
test('Edit renames an open view, and deletes it only after confirming inside the dialog', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const view = await create(request, { page: 'images', name: 'Dogs', state: { ...IMAGE_DEFAULTS, q: 'dogs' } });
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await expect(imageCards(page)).toHaveCount(2);
|
||||
// Plain pages have no Edit.
|
||||
await expect(main(page).locator('.heading-actions').getByRole('button', { name: 'Edit', exact: true })).toHaveCount(0);
|
||||
await viewRows(page).click();
|
||||
await expect(heading(page)).toHaveText('Dogs');
|
||||
// D51: Edit leads the heading's actions of an open view.
|
||||
const edit = main(page).locator('.heading-actions > button').first();
|
||||
await expect(edit).toHaveText('Edit');
|
||||
await edit.click();
|
||||
const dialog = editDialog(page);
|
||||
const name = dialog.getByRole('textbox', { name: 'View name', exact: true });
|
||||
await expect(name).toHaveValue('Dogs');
|
||||
await expect(name).toBeFocused();
|
||||
await expect(dialog.locator('.dialog-actions button')).toHaveText(['Delete…', 'Cancel', 'Save']);
|
||||
await expect(dialog.getByRole('button', { name: 'Delete…', exact: true })).toHaveClass(/\bdanger\b/);
|
||||
await dialog.getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(dialog).toHaveCount(0);
|
||||
await expect(edit).toBeFocused();
|
||||
|
||||
// An empty name keeps the dialog open on its field.
|
||||
await edit.click();
|
||||
await name.fill(' ');
|
||||
await dialog.getByRole('button', { name: 'Save', exact: true }).click();
|
||||
await expect(name).toBeFocused();
|
||||
await name.fill('Good dogs');
|
||||
await dialog.getByRole('button', { name: 'Save', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('Saved view renamed.');
|
||||
await expect(viewRows(page)).toHaveText(['Good dogs']);
|
||||
await expect(heading(page)).toHaveText('Good dogs');
|
||||
await expect(page.locator('#workspace-breadcrumb')).toHaveText('Good dogs');
|
||||
await expect(page).toHaveTitle('Good dogs · Prompt Manager');
|
||||
await expect(edit).toBeFocused();
|
||||
expect(await listed(request)).toEqual([{ ...view, name: 'Good dogs' }]);
|
||||
// The rename kept the view's page state.
|
||||
await expect(search(page)).toHaveValue('dogs');
|
||||
await expect(imageCards(page)).toHaveCount(1);
|
||||
|
||||
// Delete… asks first; its Cancel deletes nothing.
|
||||
await edit.click();
|
||||
await dialog.getByRole('button', { name: 'Delete…', exact: true }).click();
|
||||
const confirm = deleteDialog(page);
|
||||
await expect(confirm).toContainText('Delete “Good dogs”? Only the saved view is removed; its images and prompts stay.');
|
||||
await expect(confirm.locator('.dialog-actions button')).toHaveText(['Cancel', 'Delete']);
|
||||
await expect(confirm.getByRole('button', { name: 'Cancel', exact: true })).toBeFocused();
|
||||
await confirm.getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(confirm).toHaveCount(0);
|
||||
expect(await listed(request)).toHaveLength(1);
|
||||
|
||||
await edit.click();
|
||||
await dialog.getByRole('button', { name: 'Delete…', exact: true }).click();
|
||||
await confirm.getByRole('button', { name: 'Delete', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('Saved view deleted.');
|
||||
await expect(page).toHaveURL(/#\/images$/);
|
||||
await expect(heading(page)).toHaveText('Images');
|
||||
await expect(viewRows(page)).toHaveCount(0);
|
||||
expect(await listed(request)).toEqual([]);
|
||||
// The dead link left no history entry.
|
||||
await page.goBack();
|
||||
await expect(page).not.toHaveURL(new RegExp(`#/saved/${view.id}$`));
|
||||
});
|
||||
|
||||
test('a deleted Library view returns to the Library, and failed edits say so', async ({ page, request }) => {
|
||||
const view = await create(request, { page: 'library', name: 'Cats', state: { q: 'cat', category: '', sort: 'newest', tags: [] } });
|
||||
await page.goto(`${workspace}#/saved/${view.id}`);
|
||||
await expect(heading(page)).toHaveText('Cats');
|
||||
const edit = main(page).locator('.heading-actions > button').first();
|
||||
await expect(edit).toHaveText('Edit');
|
||||
let failing = true;
|
||||
await page.route(`**${API}/${view.id}`, route => failing
|
||||
? route.fulfill({ status: 500, contentType: 'application/json', body: '{"success":false,"error":"Saved view operation failed"}' })
|
||||
: route.fallback());
|
||||
await edit.click();
|
||||
await editDialog(page).getByRole('textbox', { name: 'View name', exact: true }).fill('Kittens');
|
||||
await editDialog(page).getByRole('button', { name: 'Save', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('The view could not be renamed. Try again.');
|
||||
await expect(heading(page)).toHaveText('Cats');
|
||||
await expect(viewRows(page)).toHaveText(['Cats']);
|
||||
await edit.click();
|
||||
await editDialog(page).getByRole('button', { name: 'Delete…', exact: true }).click();
|
||||
await deleteDialog(page).getByRole('button', { name: 'Delete', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('The view could not be deleted. Try again.');
|
||||
await expect(page).toHaveURL(new RegExp(`#/saved/${view.id}$`));
|
||||
expect(await listed(request)).toHaveLength(1);
|
||||
|
||||
failing = false;
|
||||
await edit.click();
|
||||
await editDialog(page).getByRole('button', { name: 'Delete…', exact: true }).click();
|
||||
await deleteDialog(page).getByRole('button', { name: 'Delete', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('Saved view deleted.');
|
||||
await expect(page).toHaveURL(/#\/library$/);
|
||||
await expect(heading(page)).toHaveText('Prompt library');
|
||||
await expect(viewRows(page)).toHaveCount(0);
|
||||
});
|
||||
@@ -3,12 +3,12 @@ import { readFile } from 'node:fs/promises';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { captureCanonicalState, historyGeometry, compareCanonicalHistory, nativeDialogState, exerciseCurrentPairSend, assertRecordedHistory } from './history-canonical-comparison.mjs';
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
|
||||
test.use({ locale: 'en-US', timezoneId: 'UTC', reducedMotion: 'reduce' });
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
const primary = ['images', 'library', 'tags', 'inspect'];
|
||||
const tools = ['system', 'overview'];
|
||||
const tools = ['activity', 'system', 'overview'];
|
||||
|
||||
async function noOverflow(page) {
|
||||
expect(await page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth)).toBeLessThanOrEqual(1);
|
||||
@@ -106,12 +106,12 @@ for (const [width, height] of [[1600, 1100], [390, 844]]) for (const theme of ['
|
||||
// As in the concept, the selected pair's send sits in the history heading, not among the saved texts.
|
||||
await expect(roles.getByRole('button', { name: 'Send prompt pair', exact: true })).toHaveCount(0);
|
||||
await expect(page.locator('#workspace-main > .heading button.send')).toBeEnabled();
|
||||
await expect(page.locator('.library-shared-gallery article[data-image-key]')).toHaveCount(2);
|
||||
await page.locator('.library-shared-gallery .image-card-actions button').first().click();
|
||||
await expect(page.locator('.library-history .history-gallery article[data-image-key]')).toHaveCount(2);
|
||||
await page.locator('.library-history .history-gallery .image-card-actions button').first().click();
|
||||
await expect(page.getByRole('complementary', { name: 'Selected image details', exact: true })).toContainText('canonical-alt.png');
|
||||
await page.locator('.library-shared-gallery .media-card.selected .image-open').click();
|
||||
await page.locator('.library-history .history-gallery .media-card.selected .image-open').click();
|
||||
await expect(page.locator('dialog#viewer-dialog')).toBeVisible();await page.keyboard.press('Escape');
|
||||
for (const [name, selector] of [['saved', '.library-history article.surface'], ['branches', '.library-history .timeline-panel'], ['gallery', '.library-shared-gallery'], ['chronology', '.recorded-history']]) {
|
||||
for (const [name, selector] of [['saved', '.library-history article.surface'], ['branches', '.library-history .timeline-panel'], ['gallery', '.library-history .history-gallery'], ['chronology', '.recorded-history']]) {
|
||||
await captureCanonicalState(page, testInfo, `production-${name}-${width}-${theme}`, selector);
|
||||
}
|
||||
const run = page.locator(`button[data-run-id="${fixture.canonical_run}"][data-anchor-id="10"]`);
|
||||
@@ -556,7 +556,7 @@ test('approved icons are decorative safe SVGs and reject unknown vocabulary', as
|
||||
await page.goto(`${workspace}#/images`);
|
||||
const icons = await page.evaluate(async () => {
|
||||
const { createWorkspaceIcon } = await import('/prompt_manager/js/workspace/icons.mjs');
|
||||
const names = ['images', 'library', 'tags', 'upload', 'overview', 'settings', 'grid', 'list', 'masonry', 'search', 'info', 'copy', 'branch', 'star', 'tagImage', 'folder', 'history', 'review'];
|
||||
const names = ['images', 'library', 'tags', 'upload', 'overview', 'activity', 'settings', 'grid', 'list', 'masonry', 'search', 'info', 'copy', 'branch', 'star', 'tagImage', 'folder', 'history', 'review'];
|
||||
const otherDocument = document.implementation.createHTMLDocument();
|
||||
const rendered = names.map(name => {
|
||||
const icon = createWorkspaceIcon(name, otherDocument);
|
||||
@@ -575,7 +575,8 @@ test('approved icons are decorative safe SVGs and reject unknown vocabulary', as
|
||||
expect(icon).toMatchObject({ svg: true, className: 'icon', owner: true, viewBox: '0 0 24 24', hidden: 'true', focusable: 'false', fill: 'none', stroke: 'currentColor', width: '1.65', children: 1 });
|
||||
expect(icon.path.length).toBeGreaterThan(10);
|
||||
}
|
||||
// D28 ruling: Overview's own outline home glyph, not the concept's `activity` pulse ('M3 12h4l3-8 4 16 3-8h4').
|
||||
// D28 ruling: Overview's own outline home glyph; the concept's `activity` pulse is the Activity row's.
|
||||
expect(icons.rendered.find(icon => icon.name === 'activity').path).toBe('M3 12h4l3-8 4 16 3-8h4');
|
||||
expect(icons.rendered.find(icon => icon.name === 'overview').path).toBe('M3 11 12 4l9 7M5 10v10h14V10M10 20v-6h4v6');
|
||||
expect(new Set(icons.rendered.map(icon => icon.path)).size).toBe(icons.rendered.length);
|
||||
});
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {pageInventoryOf,inventoryOf,diffInventory,mapVocabularyConcept,mapVocabularyDialog} from './vocabulary-canonical-comparison.mjs';
|
||||
|
||||
@@ -95,11 +95,11 @@ test('a tag Browse shows Filters · 1 until its filter is cleared',async({page})
|
||||
await page.goto('/prompt_manager/workspace#/library');
|
||||
const shelf=page.locator('article[data-prompt-id]');await expect(shelf.first()).toBeVisible();const total=await shelf.count();
|
||||
const toolbar=page.locator('#workspace-main search.toolbar');
|
||||
await expect(toolbar.getByRole('button')).toHaveText(['Filters']);
|
||||
await expect(toolbar.getByRole('button')).toHaveText(['Save view','Filters']);
|
||||
await page.goto(route);
|
||||
const row=tagRow(page,'cinematic');await row.getByRole('button',{name:'Browse',exact:true}).click();
|
||||
await expectLibrary(page,{tags:'cinematic',count:2});
|
||||
await expect(toolbar.getByRole('button')).toHaveText(['Filters · 1']);
|
||||
await expect(toolbar.getByRole('button')).toHaveText(['Save view','Filters · 1']);
|
||||
await toolbar.getByRole('button',{name:'Filters · 1',exact:true}).click();
|
||||
const refine=page.getByRole('dialog',{name:'Refine this view',exact:true});
|
||||
await refine.getByRole('button',{name:'Clear filters',exact:true}).click();await expect(refine).toHaveCount(0);
|
||||
@@ -109,7 +109,7 @@ test('a tag Browse shows Filters · 1 until its filter is cleared',async({page})
|
||||
const nav=page.getByRole('navigation',{name:'Workspace',exact:true});
|
||||
await nav.getByRole('link',{name:/^Tags & categories(?:\s*[\d,]+)?$/}).click();await expect(main(page).getByRole('heading',{level:1})).toHaveText('Tags & categories');
|
||||
await nav.getByRole('link',{name:/^Prompt library(?:\s*[\d,]+)?$/}).click();
|
||||
await expect(shelf).toHaveCount(total);await expect(toolbar.getByRole('button')).toHaveText(['Filters']);
|
||||
await expect(shelf).toHaveCount(total);await expect(toolbar.getByRole('button')).toHaveText(['Save view','Filters']);
|
||||
});
|
||||
|
||||
test('category Browse sets only the category constraint',async({page})=>{
|
||||
@@ -825,7 +825,7 @@ test('mutation check: the comparison reports a changed surface padding and a cha
|
||||
// Task A2: the workspace loads the verbatim concept stylesheet as its base and one overlay last; the screen and its
|
||||
// dialogs must stay equal to the concept with the retired vocabulary and editor sheets gone.
|
||||
const CONCEPT_BASE=['js/shared/toolkit/controls.css','lib/viewerjs/viewer.min.css','js/workspace/concept.css',
|
||||
'js/workspace/library.css','js/workspace/workspace.css'];
|
||||
'js/workspace/workspace.css'];
|
||||
const loadedSheets=page=>page.evaluate(()=>[...document.styleSheets].filter(sheet=>sheet.href).map(sheet=>new URL(sheet.href).pathname.replace(/^\/prompt_manager\//,'')));
|
||||
for(const [width,height,theme] of [[1600,1100,'dark'],[390,844,'light']])test(`concept base keeps the tags screen and editor dialogs equal to the concept ${width} ${theme}`,async({page})=>{
|
||||
await page.setViewportSize({width,height});
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
const statistic = (page, label) => page.locator('#workspace-main dt')
|
||||
@@ -18,6 +18,10 @@ const routes = [
|
||||
['tags', 'Tags & categories', null],
|
||||
['inspect', 'Inspect a file', '/prompt_manager/gallery.html'],
|
||||
['system', 'System', '/prompt_manager/admin'],
|
||||
// #249 D51: SAVED VIEWS' Favorites, Images filtered to favorites.
|
||||
['favorites', 'Favorites', null],
|
||||
// #249 Activity-1: the concept's Activity page in the tools-and-system group.
|
||||
['activity', 'Activity', null],
|
||||
];
|
||||
|
||||
async function countThemeWrites(page) {
|
||||
@@ -44,6 +48,8 @@ test.beforeEach(async ({ page, request, browserName }) => {
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, browserName }) => {
|
||||
// A route handler still running at teardown must not fail the test it outlived.
|
||||
await page.unrouteAll({ behavior: 'ignoreErrors' });
|
||||
await saveCoverage(page, browserName);
|
||||
expect(page.workspaceErrors).toEqual([]);
|
||||
});
|
||||
@@ -174,7 +180,8 @@ test('theme System follows ComfyUI and persists an explicit light or dark prefer
|
||||
await expect(theme).toHaveAttribute('data-ca-theme', 'light');
|
||||
});
|
||||
|
||||
// Task A4 (D4, D5): the concept topbar (index.html 45-63, style.css 225-277) without the prototype badge or Activity.
|
||||
// Task A4 (D4, D29): the concept topbar (index.html 45-63, style.css 225-277) without the prototype badge; Activity
|
||||
// (D52, D53) shows its button, and its pill only while a job runs.
|
||||
test('topbar matches the concept actions', async ({ page }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
await page.goto(`${workspace}#/images`);
|
||||
@@ -202,7 +209,7 @@ test('topbar matches the concept actions', async ({ page }) => {
|
||||
expect(shape).toEqual({
|
||||
children: ['div.breadcrumbs', 'div.top-actions'],
|
||||
crumbs: ['ComfyAssets', 'span:/', 'span#workspace-breadcrumb:Images'],
|
||||
actions: ['label.theme-label', 'a.canvas-link'],
|
||||
actions: ['a.activity-pill', 'button.', 'label.theme-label', 'a.canvas-link'],
|
||||
label: 'Theme', labelFor: 'theme',
|
||||
options: [['dark', 'Dark'], ['light', 'Light'], ['system', 'System']],
|
||||
arrow: [['↗', 'true']],
|
||||
@@ -216,11 +223,12 @@ test('topbar matches the concept actions', async ({ page }) => {
|
||||
await expect(link).toHaveAttribute('href', '/');
|
||||
expect(await link.evaluate(node => node.href === `${location.origin}/`)).toBe(true);
|
||||
await expect(page.getByRole('combobox', { name: 'Appearance' })).toHaveCount(0);
|
||||
await expect(page.getByRole('button', { name: /^Activity/ })).toHaveCount(0);
|
||||
await expect(page.locator('.concept-badge, #activity-open')).toHaveCount(0);
|
||||
await expect(page.getByRole('button', { name: 'Activity', exact: true })).toHaveCount(1);
|
||||
await expect(page.locator('#activity-pill')).toBeHidden();
|
||||
await expect(page.locator('.concept-badge')).toHaveCount(0);
|
||||
await page.locator('nav [data-route="library"]').click();
|
||||
await expect(page.locator('#workspace-breadcrumb')).toHaveText('Prompt library');
|
||||
await expect(topbar.locator('.top-actions > *')).toHaveCount(2);
|
||||
await expect(topbar.locator('.top-actions > *')).toHaveCount(4);
|
||||
});
|
||||
|
||||
// Concept style.css 1375-1391: at <=700px the breadcrumbs hide, the actions spread across the bar and the Theme
|
||||
@@ -240,13 +248,18 @@ test('mobile topbar hides breadcrumbs and spreads actions', async ({ page }) =>
|
||||
expected: bar.width - 28,
|
||||
label: getComputedStyle(node.querySelector('.theme-label')).fontSize,
|
||||
select: getComputedStyle(node.querySelector('#theme')).fontSize,
|
||||
labelLeft: box(node.querySelector('.theme-label')).left - bar.left,
|
||||
activityLeft: box(node.querySelector('#activity-open')).left - bar.left,
|
||||
actionsGap: [box(node.querySelector('#activity-open')).right, box(node.querySelector('.theme-label')).left,
|
||||
box(node.querySelector('.theme-label')).right, box(node.querySelector('.canvas-link')).left],
|
||||
linkRight: bar.right - box(node.querySelector('.canvas-link')).right,
|
||||
};
|
||||
});
|
||||
expect(layout).toMatchObject({ padding: '14px', actions: { justify: 'space-between', gap: '8px' }, label: '0px', select: '11px' });
|
||||
expect(Math.abs(layout.actions.width - layout.expected)).toBeLessThanOrEqual(1);
|
||||
expect(Math.abs(layout.labelLeft - 14)).toBeLessThanOrEqual(1);
|
||||
// The concept's three items spread with space-between: Activity first, Theme between equal gaps, ComfyUI last.
|
||||
expect(Math.abs(layout.activityLeft - 14)).toBeLessThanOrEqual(1);
|
||||
const [activityRight, themeLeft, themeRight, linkLeft] = layout.actionsGap;
|
||||
expect(Math.abs((themeLeft - activityRight) - (linkLeft - themeRight))).toBeLessThanOrEqual(1);
|
||||
expect(Math.abs(layout.linkRight - 14)).toBeLessThanOrEqual(1);
|
||||
await expect(page.getByRole('link', { name: 'ComfyUI', exact: true })).toBeVisible();
|
||||
await theme.selectOption('light');
|
||||
@@ -427,10 +440,13 @@ test('sidebar matches the concept groups and shows real counts', async ({ page,
|
||||
await expect(page.locator('body > .app-shell > .workspace > #workspace-main')).toHaveCount(1);
|
||||
await expect(page.locator('.workspace-shell, .workspace-body, .sidebar-footer, .connection-dot, .workspace-icon')).toHaveCount(0);
|
||||
const sidebar = page.locator('aside.sidebar');
|
||||
expect(await childShape(sidebar)).toEqual(['a.brand', 'div.sidebar-label', 'nav#primary-nav', 'div.sidebar-bottom']);
|
||||
expect(await childShape(sidebar)).toEqual(['a.brand', 'div.sidebar-label', 'nav#primary-nav', 'div.sidebar-label',
|
||||
'nav#saved-nav', 'div.sidebar-bottom']);
|
||||
await expect(sidebar.locator('.brand')).toHaveAttribute('href', '#/images');
|
||||
await expect(sidebar.locator('.sidebar-label')).toHaveText(['WORKSPACE']);
|
||||
await expect(page.locator('#saved-nav, #collections-nav, .new-label')).toHaveCount(0);
|
||||
// D49: SAVED VIEWS without the concept's PROPOSED badge; D28: still no COLLECTIONS.
|
||||
await expect(sidebar.locator('.sidebar-label')).toHaveText(['WORKSPACE', 'SAVED VIEWS']);
|
||||
await expect(page.locator('nav#saved-nav')).toHaveAttribute('aria-label', 'Saved views');
|
||||
await expect(page.locator('#collections-nav, .new-label')).toHaveCount(0);
|
||||
const primary = page.locator('nav#primary-nav');
|
||||
await expect(primary).toHaveAttribute('aria-label', 'Workspace');
|
||||
for (const [id] of NAV_TOTALS) await expect(navCount(page, id)).toHaveText(totals[id].toLocaleString('en-US'));
|
||||
@@ -457,7 +473,7 @@ test('sidebar matches the concept groups and shows real counts', async ({ page,
|
||||
// At phone width the concept wraps the links and hides counts and the connection (concept 950px block).
|
||||
await page.setViewportSize({ width: 390, height: 844 });
|
||||
await expect(navCount(page, 'library')).toBeHidden();
|
||||
await expect(page.locator('.sidebar-label, .connection')).toHaveCount(2);
|
||||
await expect(page.locator('.sidebar-label, .connection')).toHaveCount(3);
|
||||
for (const hidden of await page.locator('.sidebar-label, .connection').all()) await expect(hidden).toBeHidden();
|
||||
await expect(primary.locator('a[href="#/inspect"]')).toBeVisible();
|
||||
});
|
||||
@@ -501,18 +517,18 @@ test('a slow Images count does not delay the Library and Tags badges', async ({
|
||||
await expect(navCount(page, 'images')).toHaveText(String(totals.images));
|
||||
});
|
||||
|
||||
test('sidebar bottom holds System and Overview with a neutral connection', async ({ page }) => {
|
||||
test('sidebar bottom holds Activity, System and Overview with a neutral connection', async ({ page }) => {
|
||||
await page.goto(`${workspace}#/system`);
|
||||
const bottom = page.locator('aside.sidebar > .sidebar-bottom');
|
||||
expect(await childShape(bottom)).toEqual(['nav#system-nav', 'div.connection']);
|
||||
const system = page.locator('nav#system-nav');
|
||||
await expect(system).toHaveAttribute('aria-label', 'Tools and system');
|
||||
expect(await system.locator(':scope > a').evaluateAll(nodes => nodes.map(node => [node.getAttribute('href'), node.textContent]))).toEqual([
|
||||
['#/system', 'System'], ['#/overview', 'Overview'],
|
||||
['#/activity', 'Activity'], ['#/system', 'System'], ['#/overview', 'Overview'],
|
||||
]);
|
||||
await expect(system.locator('.nav-count')).toHaveCount(0);
|
||||
await expect(system.locator('a[href="#/system"]')).toHaveAttribute('aria-current', 'page');
|
||||
await expect(system.locator('svg.icon')).toHaveCount(2);
|
||||
await expect(system.locator('svg.icon')).toHaveCount(3);
|
||||
const connection = bottom.locator('.connection');
|
||||
expect(await childShape(connection)).toEqual(['i', 'div']);
|
||||
await expect(connection.locator(':scope > div')).toHaveText('ComfyUI workspaceYour library, on your server');
|
||||
@@ -674,8 +690,14 @@ async function serveUnderProxyMount(page) {
|
||||
await page.route(url => url.pathname.startsWith(`${PROXY_MOUNT}/`), async route => {
|
||||
const url = new URL(route.request().url());
|
||||
url.pathname = url.pathname.slice(PROXY_MOUNT.length);
|
||||
const response = await route.fetch({ url: url.href });
|
||||
await route.fulfill({ response });
|
||||
// A request still in flight when the test ends (a thumbnail the last route started) meets a page that is
|
||||
// closing: its fetched response is disposed under it. Only that teardown error is ignored.
|
||||
try {
|
||||
const response = await route.fetch({ url: url.href });
|
||||
await route.fulfill({ response });
|
||||
} catch (error) {
|
||||
if (!/has been disposed|has been closed|Target page, context or browser has been closed/.test(String(error?.message))) throw error;
|
||||
}
|
||||
});
|
||||
return escaped;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
"""Fake Activity jobs for the browser tests (#249 Activity-1).
|
||||
|
||||
A fake job is a real registry job whose runner waits for commands instead of
|
||||
reading files, so a test sets its progress and counters exactly:
|
||||
|
||||
- ``POST /__test__/jobs/start`` starts one (``kind``, ``title``, ``total``,
|
||||
``output_folders``) and returns its summary, or 409 while one runs;
|
||||
- ``POST /__test__/jobs/{id}/step`` records ``records`` (``{group, path,
|
||||
root, root_index, detail}`` file outcomes, counted, listed and logged as the scan
|
||||
records them) and ``bulk`` (``{group, count, root}``: ``count`` numbered
|
||||
rows), sets ``processed`` (and ``total``) and ``counters``, then publishes
|
||||
the progress to every subscriber;
|
||||
- ``POST /__test__/jobs/{id}/finish`` ends it as ``finished``, ``cancelled``
|
||||
or ``failed``; a cancel through the real route ends it ``cancelled``;
|
||||
- ``POST /__test__/jobs/restart`` acts out a ComfyUI restart: the open job
|
||||
streams drop without a final event and the registry starts empty.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
EVENTS_PATH = re.compile(r"^/prompt_manager/jobs/[0-9a-f]{32}/events$")
|
||||
FINAL_STATES = {"finished", "cancelled", "failed"}
|
||||
POLL_SECONDS = 0.02
|
||||
|
||||
|
||||
PREFIXES = {"added": "+", "known": "=", "review": "?", "errors": "!"}
|
||||
PREFIXES["no_metadata"] = "-"
|
||||
ROOT_INDEXES = {"output": 0, "input": 1}
|
||||
|
||||
|
||||
def outcome_records(data):
|
||||
"""The step's file outcomes: its ``records``, then its ``bulk`` rows."""
|
||||
records = data.get("records", [])
|
||||
bulk = data.get("bulk")
|
||||
if not isinstance(records, list):
|
||||
raise web.HTTPBadRequest()
|
||||
rows = []
|
||||
for record in records:
|
||||
if not isinstance(record, dict) or record.get("group") not in PREFIXES:
|
||||
raise web.HTTPBadRequest()
|
||||
root = str(record.get("root", "output"))
|
||||
rows.append(
|
||||
{
|
||||
"group": record["group"],
|
||||
"path": str(record.get("path", "")),
|
||||
"root": root,
|
||||
# The fixture's listing order: output is root 0, input root 1.
|
||||
"root_index": record.get("root_index", ROOT_INDEXES.get(root)),
|
||||
"detail": str(record.get("detail", "")),
|
||||
}
|
||||
)
|
||||
if bulk is not None:
|
||||
if not isinstance(bulk, dict) or bulk.get("group") not in PREFIXES:
|
||||
raise web.HTTPBadRequest()
|
||||
total = bulk.get("count")
|
||||
if isinstance(total, bool) or not isinstance(total, int) or total < 0:
|
||||
raise web.HTTPBadRequest()
|
||||
rows.extend(
|
||||
{
|
||||
"group": bulk["group"],
|
||||
"path": f"bulk/file_{index:05d}.png",
|
||||
"root": str(bulk.get("root", "output")),
|
||||
"root_index": 0,
|
||||
"detail": f"Bulk row {index}",
|
||||
}
|
||||
for index in range(total)
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def register_activity_commands(app, routes, api):
|
||||
from py.api.jobs import SCAN_GROUP_NAMES, JobBusy, JobRegistry
|
||||
|
||||
streams = set()
|
||||
endings = {}
|
||||
|
||||
@web.middleware
|
||||
async def track_streams(request, handler):
|
||||
if not EVENTS_PATH.match(request.path):
|
||||
return await handler(request)
|
||||
transport = request.transport
|
||||
streams.add(transport)
|
||||
try:
|
||||
return await handler(request)
|
||||
finally:
|
||||
streams.discard(transport)
|
||||
|
||||
app.middlewares.append(track_streams)
|
||||
|
||||
def fake_runner(ending):
|
||||
async def run(job):
|
||||
while not ending.done() and not job.cancelled:
|
||||
await asyncio.sleep(POLL_SECONDS)
|
||||
state = ending.result() if ending.done() else "cancelled"
|
||||
job.publish({"type": "complete", "progress": job.progress})
|
||||
job.finish(state)
|
||||
|
||||
return run
|
||||
|
||||
def job_or_404(request):
|
||||
job = api._jobs.get(request.match_info["job_id"])
|
||||
if job is None or job.id not in endings:
|
||||
raise web.HTTPNotFound()
|
||||
return job
|
||||
|
||||
async def body(request):
|
||||
data = await request.json() if request.can_read_body else {}
|
||||
if not isinstance(data, dict):
|
||||
raise web.HTTPBadRequest()
|
||||
return data
|
||||
|
||||
def count(value, default=0):
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
||||
raise web.HTTPBadRequest()
|
||||
return value
|
||||
|
||||
@routes.post("/__test__/jobs/start")
|
||||
async def start_job(request):
|
||||
data = await body(request)
|
||||
ending = asyncio.get_running_loop().create_future()
|
||||
try:
|
||||
job = api._jobs.start(
|
||||
str(data.get("kind", "scan")),
|
||||
str(data.get("title", "Scan for metadata")),
|
||||
record_changing=True,
|
||||
scope={
|
||||
"files": str(data.get("files", "new")),
|
||||
"output_folders": count(data.get("output_folders"), 2),
|
||||
},
|
||||
runner=fake_runner(ending),
|
||||
groups=SCAN_GROUP_NAMES,
|
||||
)
|
||||
except JobBusy:
|
||||
raise web.HTTPConflict()
|
||||
job.update(total=count(data.get("total")))
|
||||
endings[job.id] = ending
|
||||
return web.json_response({"success": True, "job": job.summary()})
|
||||
|
||||
@routes.post("/__test__/jobs/{job_id}/step")
|
||||
async def step_job(request):
|
||||
job = job_or_404(request)
|
||||
data = await body(request)
|
||||
total = count(data.get("total"), job.total)
|
||||
processed = count(data.get("processed"), job.processed)
|
||||
counters = data.get("counters", {})
|
||||
if not isinstance(counters, dict) or set(counters) - set(job.counters):
|
||||
raise web.HTTPBadRequest()
|
||||
for record in outcome_records(data):
|
||||
group = record["group"]
|
||||
path = record["path"]
|
||||
detail = record["detail"]
|
||||
job.record(group, record, f"{PREFIXES[group]} {path} · {detail}")
|
||||
for key, value in counters.items():
|
||||
job.counters[key] = count(value)
|
||||
progress = processed * 100 // total if total else 0
|
||||
job.update(
|
||||
progress=progress,
|
||||
processed=processed,
|
||||
total=total,
|
||||
status_line=f"Processing file {processed:,} of {total:,}",
|
||||
)
|
||||
job.publish({"type": "progress", "progress": progress})
|
||||
return web.json_response({"success": True, "job": job.summary()})
|
||||
|
||||
@routes.post("/__test__/jobs/{job_id}/finish")
|
||||
async def finish_job(request):
|
||||
job = job_or_404(request)
|
||||
state = (await body(request)).get("state", "finished")
|
||||
if state not in FINAL_STATES:
|
||||
raise web.HTTPBadRequest()
|
||||
ending = endings[job.id]
|
||||
if not ending.done():
|
||||
ending.set_result(state)
|
||||
while job.running:
|
||||
await asyncio.sleep(POLL_SECONDS)
|
||||
return web.json_response({"success": True, "job": job.summary()})
|
||||
|
||||
@routes.post("/__test__/jobs/restart")
|
||||
async def restart(request):
|
||||
old = api._jobs
|
||||
api._jobs = JobRegistry(api.logger)
|
||||
endings.clear()
|
||||
for transport in list(streams):
|
||||
transport.close()
|
||||
tasks = [job.task for job in old.list() if job.task and not job.task.done()]
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
return web.json_response({"success": True})
|
||||
@@ -168,7 +168,11 @@ _TABLES = (
|
||||
"execution_contexts",
|
||||
"execution_prompt_snapshots",
|
||||
"execution_context_texts",
|
||||
"saved_views",
|
||||
"image_favorites",
|
||||
"media_image_tags",
|
||||
"media_annotation_bytes",
|
||||
"media_annotation_files",
|
||||
"image_prompt_pairs",
|
||||
"prompt_pair_tags",
|
||||
"prompt_pairs",
|
||||
@@ -391,55 +395,66 @@ def seed_workspace_concept(
|
||||
rounds = math.ceil(total / len(CONCEPT_PROMPTS))
|
||||
now = time.time()
|
||||
images, runs = [], []
|
||||
for round_index in range(rounds):
|
||||
for prompt_index, prompt in enumerate(CONCEPT_PROMPTS):
|
||||
index = round_index * len(CONCEPT_PROMPTS) + prompt_index
|
||||
if index >= total:
|
||||
break
|
||||
name = concept_filename(prompt, round_index)
|
||||
path = folder / name
|
||||
facts = write_concept_png(path, prompt_index, round_index)
|
||||
variant, negative, seed = facts["variant"], facts["negative"], facts["seed"]
|
||||
# The concept lists images newest first in this order.
|
||||
stamp = now - 60 - index * 60
|
||||
os.utime(path, (stamp, stamp))
|
||||
owner = prompts[prompt_index]
|
||||
image_id = db.link_image_to_prompt(owner["positive_id"], str(path))
|
||||
pair_id = owner["pair_ids"][variant]
|
||||
db.link_image_to_pair(pair_id, image_id, {"source": "concept fixture"})
|
||||
if concept_starred(prompt_index, round_index):
|
||||
db.set_image_favorite(
|
||||
favorite_root, _favorite_relative_key(f"concepts/{name}")
|
||||
)
|
||||
context = RecipeContext(
|
||||
"3", _snapshot(prompt.positive), _snapshot(negative), "inferred"
|
||||
slots = [
|
||||
(round_index, prompt_index, prompt)
|
||||
for round_index in range(rounds)
|
||||
for prompt_index, prompt in enumerate(CONCEPT_PROMPTS)
|
||||
if round_index * len(CONCEPT_PROMPTS) + prompt_index < total
|
||||
]
|
||||
# Linked and recorded oldest first, as real runs arrive, so link and run ids
|
||||
# grow with time and every newest-first read (by id or by mtime) lists the
|
||||
# concept's order (app.js:180-239). The manifest keeps the concept's order.
|
||||
for round_index, prompt_index, prompt in reversed(slots):
|
||||
index = round_index * len(CONCEPT_PROMPTS) + prompt_index
|
||||
name = concept_filename(prompt, round_index)
|
||||
path = folder / name
|
||||
facts = write_concept_png(path, prompt_index, round_index)
|
||||
variant, negative, seed = facts["variant"], facts["negative"], facts["seed"]
|
||||
# The concept lists images newest first in this order.
|
||||
stamp = now - 60 - index * 60
|
||||
os.utime(path, (stamp, stamp))
|
||||
owner = prompts[prompt_index]
|
||||
image_id = db.link_image_to_prompt(owner["positive_id"], str(path))
|
||||
pair_id = owner["pair_ids"][variant]
|
||||
db.link_image_to_pair(pair_id, image_id, {"source": "concept fixture"})
|
||||
# Real pair counting: each image is one run of its pair, as the
|
||||
# concept's "<name> · N runs" counts its images (app.js a598, D26).
|
||||
db.record_prompt_pair_use(pair_id)
|
||||
if concept_starred(prompt_index, round_index):
|
||||
db.set_image_favorite(
|
||||
favorite_root, _favorite_relative_key(f"concepts/{name}")
|
||||
)
|
||||
reference = OutputReference(
|
||||
"4", 0, "output", "concepts", name, "posix", roots.output_key
|
||||
)
|
||||
runs.append(
|
||||
_record_run(
|
||||
api,
|
||||
epoch=epoch,
|
||||
item=index,
|
||||
context=context,
|
||||
output=reference,
|
||||
observed_at=int(stamp * 1000),
|
||||
)
|
||||
)
|
||||
images.append(
|
||||
{
|
||||
"filename": name,
|
||||
"image_id": image_id,
|
||||
"pair_id": pair_id,
|
||||
"prompt": prompt.id,
|
||||
"variant": variant,
|
||||
"seed": seed,
|
||||
"path": f"concepts/{name}",
|
||||
"width": facts["width"],
|
||||
"height": facts["height"],
|
||||
}
|
||||
context = RecipeContext(
|
||||
"3", _snapshot(prompt.positive), _snapshot(negative), "inferred"
|
||||
)
|
||||
reference = OutputReference(
|
||||
"4", 0, "output", "concepts", name, "posix", roots.output_key
|
||||
)
|
||||
runs.append(
|
||||
_record_run(
|
||||
api,
|
||||
epoch=epoch,
|
||||
item=index,
|
||||
context=context,
|
||||
output=reference,
|
||||
observed_at=int(stamp * 1000),
|
||||
)
|
||||
)
|
||||
images.append(
|
||||
{
|
||||
"filename": name,
|
||||
"image_id": image_id,
|
||||
"pair_id": pair_id,
|
||||
"prompt": prompt.id,
|
||||
"variant": variant,
|
||||
"seed": seed,
|
||||
"path": f"concepts/{name}",
|
||||
"width": facts["width"],
|
||||
"height": facts["height"],
|
||||
}
|
||||
)
|
||||
images.reverse()
|
||||
runs.reverse()
|
||||
seedless = []
|
||||
for number in range(seedless_images):
|
||||
name = f"seedless_{number + 1:05}.png"
|
||||
|
||||
@@ -26,7 +26,11 @@ EDITOR_METADATA = {
|
||||
"notes": "Keep the soft light.",
|
||||
}
|
||||
_TABLES = (
|
||||
"saved_views",
|
||||
"image_favorites",
|
||||
"media_image_tags",
|
||||
"media_annotation_bytes",
|
||||
"media_annotation_files",
|
||||
"image_prompt_pairs",
|
||||
"prompt_pair_tags",
|
||||
"prompt_pairs",
|
||||
|
||||
@@ -112,7 +112,11 @@ def seed_workspace_visual(api: PromptManagerAPI, root: Path) -> None:
|
||||
"""Replace only the owned browser harness records and media roots."""
|
||||
with api.db.model.get_connection() as conn:
|
||||
for table in (
|
||||
"saved_views",
|
||||
"image_favorites",
|
||||
"media_image_tags",
|
||||
"media_annotation_bytes",
|
||||
"media_annotation_files",
|
||||
"image_prompt_pairs",
|
||||
"prompt_pair_tags",
|
||||
"prompt_pairs",
|
||||
|
||||
@@ -79,7 +79,11 @@ UNUSED_TAG = "unused_vocabulary"
|
||||
MAX_EXTRA_TAGS = 300
|
||||
MAX_EXTRA_CATEGORIES = 200
|
||||
_TABLES = (
|
||||
"saved_views",
|
||||
"image_favorites",
|
||||
"media_image_tags",
|
||||
"media_annotation_bytes",
|
||||
"media_annotation_files",
|
||||
"image_prompt_pairs",
|
||||
"prompt_pair_tags",
|
||||
"prompt_pairs",
|
||||
|
||||
@@ -224,6 +224,22 @@ test("facts use resolution.sampler_id for detailer-stage nodes", () => {
|
||||
assert.ok(Object.values(factsFor(wide, { resolution: { state: "ambiguous", sampler_id: "d" } })).every(fact => fact.status === "unresolved"), "the stage walk is bounded by MAX_VISITS");
|
||||
});
|
||||
|
||||
test("a shared pair resolved across sampler branches merges facts per field (#249)", () => {
|
||||
// The server resolves identical branches with no sampler named: settings the
|
||||
// branches agree on are known, those they differ on (a LoRA per epoch) are ambiguous.
|
||||
const shared = { resolution: { state: "resolved", sampler_id: null, reason: "Every sampler branch uses the same prompt pair" } };
|
||||
const grid = factGraph();
|
||||
grid.lora = { class_type: "CheckpointLoaderSimple", inputs: { ckpt_name: "epoch_10.safetensors" } };
|
||||
grid.other = structuredClone(grid.s); grid.other.inputs.model = ["lora", 0];
|
||||
assert.deepEqual(statusValues(factsFor(grid, shared)), {
|
||||
seed: ["known", "9007199254740993"], steps: ["known", "28"], cfg: ["known", "6.5000000000000000001"],
|
||||
sampler: ["known", "dpmpp_2m"], scheduler: ["known", "karras"], model: ["ambiguous", null],
|
||||
});
|
||||
assert.ok(Object.values(factsFor(grid, shared)).every(fact => fact.sampler_id === null));
|
||||
// One listed sampler keeps reading that sampler alone.
|
||||
assert.equal(factsFor(factGraph(), shared).model.value, "model.safetensors");
|
||||
});
|
||||
|
||||
test("ambiguous results without a sampler_id merge facts across every listed stage", () => {
|
||||
const ambiguous = { resolution: { state: "ambiguous", sampler_id: null, reason: "Multiple sampler branches" } };
|
||||
// f01b's shape: an upstream sampler and a detailer whose settings partly differ.
|
||||
|
||||
@@ -30,7 +30,7 @@ test("facts guider providers are guiders in the semantics table", () => {
|
||||
});
|
||||
|
||||
test("the facts tables are not part of the legacy page contract", () => {
|
||||
assert.deepStrictEqual(Object.keys(metadata).filter(key => !key.startsWith("_")).sort(), ["PLACEHOLDERS", "extractGenerationFacts", "extractGenerationParams"]);
|
||||
assert.deepStrictEqual(Object.keys(metadata).filter(key => !key.startsWith("_")).sort(), ["PLACEHOLDERS", "extractGenerationFacts", "extractGenerationParams", "legacyNegativeSource"]);
|
||||
assert.ok(Object.isFrozen(metadata._factsTables));
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,371 @@
|
||||
// Activity (#249 Activity-1): the session job store behind the top-bar button and pill and the Activity page.
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
|
||||
const source = import('../../web/js/workspace/activity-store.mjs');
|
||||
|
||||
const ID_A = 'a'.repeat(32);
|
||||
const ID_B = 'b'.repeat(32);
|
||||
const ID_C = 'c'.repeat(32);
|
||||
|
||||
function summary(overrides = {}) {
|
||||
return { id: ID_A, kind: 'scan', title: 'Scan for metadata', state: 'running', started_at: 1000, finished_at: null,
|
||||
progress: 0, status_line: '', processed: 0, total: 0,
|
||||
counters: { added: 0, known: 0, review: 0, no_metadata: 0, errors: 0 }, scope: { files: 'new', output_folders: 2 },
|
||||
...overrides };
|
||||
}
|
||||
|
||||
function apiError(status, message, code) {
|
||||
return Object.assign(new Error(message), { status, code });
|
||||
}
|
||||
|
||||
/** A fake world: the API's answers, every opened stream, the pending timers and the toasts. */
|
||||
function harness() {
|
||||
const world = { list: [], listError: null, posts: [], postResult: null, postError: null, streams: [], timers: [],
|
||||
toasts: [], gets: 0 };
|
||||
const cleanups = [];
|
||||
const controller = new AbortController();
|
||||
const scope = { signal: controller.signal, cleanup: fn => cleanups.push(fn) };
|
||||
const api = {
|
||||
async get(url, options) {
|
||||
assert.equal(url, '/prompt_manager/jobs');
|
||||
assert.equal(options.cache, 'no-store');
|
||||
world.gets += 1;
|
||||
if (world.listError) throw world.listError;
|
||||
return { success: true, jobs: world.list };
|
||||
},
|
||||
async post(url, body) {
|
||||
world.posts.push([url, body]);
|
||||
if (world.postError) throw world.postError;
|
||||
return world.postResult;
|
||||
},
|
||||
};
|
||||
const connect = url => {
|
||||
const stream = { url, closed: false, onmessage: null, onerror: null, close() { this.closed = true; } };
|
||||
world.streams.push(stream);
|
||||
return stream;
|
||||
};
|
||||
const timers = {
|
||||
setTimeout(fn, delay) { const timer = { fn, delay, cleared: false }; world.timers.push(timer); return timer; },
|
||||
clearTimeout(timer) { timer.cleared = true; },
|
||||
};
|
||||
world.dispose = () => { controller.abort(); cleanups.forEach(fn => fn()); };
|
||||
world.fire = async () => {
|
||||
const timer = world.timers.find(item => !item.cleared && !item.fired);
|
||||
timer.fired = true;
|
||||
timer.fn();
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
return timer.delay;
|
||||
};
|
||||
world.open = () => world.streams.filter(stream => !stream.closed);
|
||||
return { world, api, scope, connect, timers, pollMs: 0, notify: message => world.toasts.push(message) };
|
||||
}
|
||||
|
||||
const send = (stream, job) => stream.onmessage({ data: JSON.stringify(job) });
|
||||
|
||||
test('store merges list and SSE updates and caps backoff', async () => {
|
||||
const { createActivityStore, BACKOFF_MAX_MS, LOST_DETAIL } = await source;
|
||||
const { world, ...deps } = harness();
|
||||
world.list = [summary(), summary({ id: ID_B, state: 'finished', finished_at: 900, counters: { added: 3, errors: 0 } })];
|
||||
const store = createActivityStore(deps);
|
||||
let changes = 0;
|
||||
store.subscribe(() => { changes += 1; });
|
||||
await store.load();
|
||||
assert.equal(store.status, 'ready');
|
||||
assert.deepEqual(store.jobs().map(job => [job.id, job.state]), [[ID_A, 'running'], [ID_B, 'finished']]);
|
||||
assert.deepEqual(store.running().map(job => job.id), [ID_A]);
|
||||
// One stream per running job only; a finished job is not followed.
|
||||
assert.deepEqual(world.streams.map(stream => stream.url), [`/prompt_manager/jobs/${ID_A}/events`]);
|
||||
assert.ok(changes >= 1);
|
||||
|
||||
// An SSE summary replaces the listed one; garbage and another job's summary are ignored.
|
||||
const [stream] = world.streams;
|
||||
send(stream, summary({ progress: 42, processed: 8240, total: 19622 }));
|
||||
stream.onmessage({ data: 'not json' });
|
||||
send(stream, summary({ id: ID_B, progress: 99 }));
|
||||
assert.equal(store.find(ID_A).progress, 42);
|
||||
assert.equal(store.find(ID_B).state, 'finished');
|
||||
assert.deepEqual(world.toasts, []);
|
||||
|
||||
// The end of the job: the finish toast, and the stream is closed.
|
||||
send(stream, summary({ state: 'finished', progress: 100, finished_at: 2000, counters: { added: 1420, errors: 18 } }));
|
||||
assert.deepEqual(world.toasts, ['Scan finished · 1,420 added · 18 errors']);
|
||||
assert.equal(stream.closed, true);
|
||||
assert.deepEqual(store.running(), []);
|
||||
|
||||
// A new running job from another tab, then the server goes away: every re-read waits longer, up to 30 s.
|
||||
world.list = [summary({ id: ID_C, state: 'running' })];
|
||||
await store.load();
|
||||
const second = world.open()[0];
|
||||
assert.equal(second.url, `/prompt_manager/jobs/${ID_C}/events`);
|
||||
world.listError = new Error('connection refused');
|
||||
second.onerror();
|
||||
assert.equal(second.closed, true);
|
||||
const delays = [];
|
||||
for (let index = 0; index < 8; index += 1) delays.push(await world.fire());
|
||||
assert.deepEqual(delays, [1000, 2000, 4000, 8000, 16000, 30000, 30000, 30000]);
|
||||
assert.ok(delays.every(delay => delay <= BACKOFF_MAX_MS));
|
||||
// While the server cannot be read the running job shows as ended, so no pill stays.
|
||||
assert.equal(store.status, 'error');
|
||||
assert.equal(store.find(ID_C).state, 'lost');
|
||||
assert.deepEqual(store.running(), []);
|
||||
assert.equal(typeof LOST_DETAIL, 'string');
|
||||
|
||||
// The server is back without the job (a restart): it stays ended; finished jobs the server no longer lists are
|
||||
// dropped, so the list never outgrows the server's (review M3).
|
||||
world.listError = null;
|
||||
world.list = [];
|
||||
await world.fire();
|
||||
assert.equal(store.status, 'ready');
|
||||
assert.deepEqual(store.jobs().map(job => [job.id, job.state]), [[ID_C, 'lost']]);
|
||||
assert.equal(world.timers.filter(timer => !timer.cleared && !timer.fired).length, 0, 'no retry after a good read');
|
||||
assert.equal(world.toasts.length, 1, 'an ended job has no finish toast');
|
||||
world.dispose();
|
||||
});
|
||||
|
||||
test('a dropped stream of a still running job reattaches after the backoff, then resets it', async () => {
|
||||
const { createActivityStore } = await source;
|
||||
const { world, ...deps } = harness();
|
||||
world.list = [summary()];
|
||||
const store = createActivityStore(deps);
|
||||
await store.load();
|
||||
world.streams[0].onerror();
|
||||
assert.equal(await world.fire(), 1000);
|
||||
const again = world.open();
|
||||
assert.equal(again.length, 1);
|
||||
assert.equal(store.find(ID_A).state, 'running');
|
||||
again[0].onerror();
|
||||
assert.equal(await world.fire(), 2000);
|
||||
send(world.open()[0], summary({ progress: 5 }));
|
||||
world.open()[0].onerror();
|
||||
assert.equal(await world.fire(), 1000, 'a received message resets the backoff');
|
||||
// A second drop while a re-read is already scheduled does not schedule another.
|
||||
world.open()[0].onerror();
|
||||
store.load();
|
||||
assert.equal(world.timers.filter(timer => !timer.cleared && !timer.fired).length, 1);
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
world.dispose();
|
||||
assert.ok(world.streams.every(stream => stream.closed), 'dispose closes every stream');
|
||||
assert.ok(world.timers.every(timer => timer.cleared || timer.fired), 'dispose clears the pending re-read');
|
||||
});
|
||||
|
||||
test('start returns busy with the running job', async () => {
|
||||
const { createActivityStore } = await source;
|
||||
const { world, ...deps } = harness();
|
||||
const store = createActivityStore(deps);
|
||||
world.postResult = { success: true, job: summary() };
|
||||
const started = await store.start('scan');
|
||||
assert.equal(started.job.id, ID_A);
|
||||
assert.deepEqual(world.posts, [['/prompt_manager/jobs/scan', {}]]);
|
||||
assert.deepEqual(world.toasts, ['Scan started.']);
|
||||
assert.equal(world.open().length, 1, 'the started job is followed');
|
||||
|
||||
world.postError = apiError(409, 'Scan for metadata is already running', 'busy');
|
||||
world.list = [summary()];
|
||||
const busy = await store.start('scan');
|
||||
assert.equal(busy.busy.id, ID_A);
|
||||
assert.equal(busy.busy.title, 'Scan for metadata');
|
||||
assert.equal(busy.message, 'Scan for metadata is already running');
|
||||
assert.equal(world.toasts.length, 1, 'a refused start has no start toast');
|
||||
|
||||
world.postError = apiError(500, 'boom', null);
|
||||
await assert.rejects(store.start('scan'), /boom/);
|
||||
world.postError = null;
|
||||
world.postResult = { success: true, job: { id: 'nope' } };
|
||||
await assert.rejects(store.start('scan'), /Invalid job/);
|
||||
|
||||
world.postResult = { success: true, cancellation_requested: true };
|
||||
assert.equal(await store.cancel(ID_A), true);
|
||||
assert.deepEqual(world.posts.at(-1), [`/prompt_manager/jobs/${ID_A}/cancel`, {}]);
|
||||
world.postResult = { success: true, cancellation_requested: false };
|
||||
assert.equal(await store.cancel(ID_A), false);
|
||||
world.dispose();
|
||||
});
|
||||
|
||||
test('a list that is not a job list counts as a failed read', async () => {
|
||||
const { createActivityStore } = await source;
|
||||
const { world, api, ...deps } = harness();
|
||||
const store = createActivityStore({ ...deps, api: { ...api, get: async () => ({ success: true }) } });
|
||||
await store.load();
|
||||
assert.equal(store.status, 'error');
|
||||
assert.equal(world.timers.length, 1);
|
||||
world.dispose();
|
||||
await store.load();
|
||||
assert.equal(store.status, 'error', 'a disposed store reads nothing');
|
||||
});
|
||||
|
||||
test('a visible tab reads the list again every 15 s; a hidden one skips its turn', async () => {
|
||||
const { createActivityStore, POLL_MS } = await source;
|
||||
const { world, ...deps } = harness();
|
||||
let visible = true;
|
||||
const store = createActivityStore({ ...deps, pollMs: POLL_MS, isVisible: () => visible });
|
||||
await store.load();
|
||||
assert.equal(world.gets, 1);
|
||||
// A job started in another tab appears on the next read.
|
||||
world.list = [summary()];
|
||||
assert.equal(await world.fire(), 15000);
|
||||
assert.equal(world.gets, 2);
|
||||
assert.deepEqual(store.running().map(job => job.id), [ID_A]);
|
||||
visible = false;
|
||||
assert.equal(await world.fire(), 15000);
|
||||
assert.equal(world.gets, 2, 'a hidden tab does not read');
|
||||
visible = true;
|
||||
await world.fire();
|
||||
assert.equal(world.gets, 3);
|
||||
// A manual read replaces the pending poll rather than adding one.
|
||||
await store.load();
|
||||
assert.equal(world.timers.filter(timer => !timer.cleared && !timer.fired).length, 1);
|
||||
world.dispose();
|
||||
assert.ok(world.timers.every(timer => timer.cleared || timer.fired));
|
||||
});
|
||||
|
||||
test('a late or older answer never brings an ended job back, so it is announced once', async () => {
|
||||
const { createActivityStore } = await source;
|
||||
const { world, api, ...deps } = harness();
|
||||
const answers = [];
|
||||
const slowApi = { ...api, get: () => new Promise(resolve => answers.push(resolve)) };
|
||||
const store = createActivityStore({ ...deps, api: slowApi });
|
||||
const first = store.load();
|
||||
const second = store.load();
|
||||
// The newer read answers first; the older one, answering later with stale data, is dropped.
|
||||
answers[1]({ success: true, jobs: [summary()] });
|
||||
await second;
|
||||
answers[0]({ success: true, jobs: [] });
|
||||
await first;
|
||||
assert.deepEqual(store.jobs().map(job => job.state), ['running']);
|
||||
send(world.open()[0], summary({ state: 'finished', counters: { added: 2, errors: 0 } }));
|
||||
assert.deepEqual(world.toasts, ['Scan finished · 2 added · 0 errors']);
|
||||
// A list read sent before the end still says running: the job stays finished and no stream reopens.
|
||||
const third = store.load();
|
||||
answers[2]({ success: true, jobs: [summary()] });
|
||||
await third;
|
||||
assert.equal(store.find(ID_A).state, 'finished');
|
||||
assert.equal(world.open().length, 0);
|
||||
assert.equal(world.toasts.length, 1);
|
||||
// A stale failure is dropped too.
|
||||
const failing = store.load();
|
||||
const newest = store.load();
|
||||
answers[4]({ success: true, jobs: [] });
|
||||
await newest;
|
||||
answers[3](Promise.reject(new Error('late failure')));
|
||||
await failing;
|
||||
assert.equal(store.status, 'ready');
|
||||
world.dispose();
|
||||
});
|
||||
|
||||
test('summaries are validated and the copy follows the spec', async () => {
|
||||
const { parseJob, finishToast, startToast, pillParts, outcomeLine, backoffDelay } = await source;
|
||||
for (const raw of [null, [], { id: 'x', state: 'running' }, summary({ state: 'lost' }), summary({ id: ID_A.toUpperCase() })]) {
|
||||
assert.equal(parseJob(raw), null);
|
||||
}
|
||||
const loose = parseJob({ id: ID_A, state: 'running', progress: 140.4, processed: -1, total: 1.5, counters: { added: 'x', errors: 2 },
|
||||
title: ' ', started_at: 'x', scope: [] });
|
||||
assert.deepEqual(loose, { id: ID_A, kind: '', title: 'Background job', state: 'running', started_at: null, finished_at: null,
|
||||
progress: 100, status_line: '', processed: 0, total: 0, counters: { added: 0, errors: 2 }, scope: {}, listed: {}, log_length: 0 });
|
||||
assert.equal(parseJob(summary({ progress: -3 })).progress, 0);
|
||||
|
||||
const scan = parseJob(summary({ progress: 42, processed: 8240, total: 19622, counters: { added: 1, errors: 1 } }));
|
||||
assert.deepEqual(pillParts(scan), { label: 'Scanning · 42%', files: ' · 8,240 / 19,622', text: 'Scanning · 42% · 8,240 / 19,622' });
|
||||
assert.equal(pillParts({ ...scan, total: 0 }).text, 'Scanning · 42%');
|
||||
assert.equal(pillParts({ ...scan, kind: 'thumbnails', title: 'Thumbnails' }).text, 'Thumbnails · 42% · 8,240 / 19,622');
|
||||
assert.equal(finishToast({ ...scan, state: 'finished' }), 'Scan finished · 1 added · 1 error');
|
||||
assert.equal(finishToast({ ...scan, state: 'cancelled' }), 'Scan cancelled · 1 added');
|
||||
assert.equal(finishToast({ ...scan, state: 'failed' }), 'Scan failed · 1 added · 1 error');
|
||||
assert.equal(finishToast({ ...scan, counters: {}, state: 'finished' }), 'Scan finished · 0 added · 0 errors');
|
||||
const other = { ...scan, kind: 'backup', title: 'Backup' };
|
||||
assert.equal(finishToast({ ...other, state: 'finished' }), 'Backup finished.');
|
||||
assert.equal(finishToast({ ...other, state: 'cancelled' }), 'Backup was cancelled.');
|
||||
assert.equal(finishToast({ ...other, state: 'failed' }), 'Backup failed.');
|
||||
assert.equal(startToast(scan), 'Scan started.');
|
||||
assert.equal(startToast(other), 'Backup started.');
|
||||
assert.equal(outcomeLine(scan), '1 added · 1 error');
|
||||
assert.equal(outcomeLine(other), '');
|
||||
assert.deepEqual([-1, 0, 1, 4, 5, 6, 1e9, 0.5].map(backoffDelay), [1000, 1000, 2000, 16000, 30000, 30000, 30000, 1000]);
|
||||
});
|
||||
|
||||
test('pagehide stops polling, closes streams and aborts the in-flight read; pageshow resumes', async () => {
|
||||
const { createActivityStore } = await source;
|
||||
const { world, api, ...deps } = harness();
|
||||
const lifecycle = new EventTarget();
|
||||
const signals = [];
|
||||
let answer = null;
|
||||
const pendingApi = { ...api, get: (url, options) => {
|
||||
signals.push(options.signal);
|
||||
world.gets += 1;
|
||||
if (world.gets === 1) return Promise.resolve({ success: true, jobs: [summary()] });
|
||||
return new Promise((resolve, reject) => {
|
||||
answer = resolve;
|
||||
options.signal.addEventListener('abort', () => reject(Object.assign(new Error('aborted'), { name: 'AbortError' })));
|
||||
});
|
||||
} };
|
||||
const store = createActivityStore({ ...deps, api: pendingApi, pollMs: 15000, lifecycle });
|
||||
await store.load();
|
||||
assert.equal(world.open().length, 1);
|
||||
const poll = world.timers.find(timer => !timer.cleared && timer.delay === 15000);
|
||||
assert.ok(poll, 'a visible tab polls');
|
||||
// A read in flight when the page goes away (a closed tab, a test teardown) is aborted, nothing is scheduled again.
|
||||
const read = store.load();
|
||||
lifecycle.dispatchEvent(new Event('pagehide'));
|
||||
await read;
|
||||
assert.equal(signals.at(-1).aborted, true);
|
||||
assert.equal(world.open().length, 0);
|
||||
assert.ok(world.timers.every(timer => timer.cleared || timer.fired));
|
||||
assert.equal(store.find(ID_A).state, 'running', 'a hidden page marks nothing lost');
|
||||
await store.load();
|
||||
assert.equal(world.gets, 2, 'a suspended store reads nothing');
|
||||
// Back from the cache, it reads the list again and follows the job.
|
||||
const resumed = Object.assign(new Event('pageshow'), { persisted: true });
|
||||
lifecycle.dispatchEvent(resumed);
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
assert.equal(world.gets, 3);
|
||||
answer({ success: true, jobs: [summary()] });
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
assert.equal(world.open().length, 1);
|
||||
world.dispose();
|
||||
assert.equal(signals.at(-1).aborted, true, 'disposing aborts the store reads');
|
||||
assert.equal(world.open().length, 0);
|
||||
});
|
||||
|
||||
test('one failed read is not a restart; two in a row are, and a good read resets the backoff', async () => {
|
||||
const { createActivityStore } = await source;
|
||||
const { world, ...deps } = harness();
|
||||
world.list = [summary(), summary({ id: ID_B, state: 'finished', finished_at: 900 })];
|
||||
const store = createActivityStore(deps);
|
||||
await store.load();
|
||||
// A single failed read (a proxy's 502) keeps the job running and its stream open (review M1).
|
||||
world.listError = apiError(502, 'Bad gateway', null);
|
||||
await store.load();
|
||||
assert.equal(store.status, 'error');
|
||||
assert.equal(store.find(ID_A).state, 'running');
|
||||
assert.equal(world.open().length, 1);
|
||||
assert.equal(await world.fire(), 1000);
|
||||
// The second failure in a row: the server is gone, so the running job ended with it.
|
||||
assert.equal(store.find(ID_A).state, 'lost');
|
||||
assert.equal(world.open().length, 0);
|
||||
for (let index = 0; index < 3; index += 1) await world.fire();
|
||||
// The server is back with a new running job: the backoff starts again at 1 s (review M2).
|
||||
world.listError = null;
|
||||
world.list = [summary({ id: ID_C })];
|
||||
await world.fire();
|
||||
assert.equal(store.status, 'ready');
|
||||
assert.deepEqual(store.jobs().map(job => [job.id, job.state]), [[ID_C, 'running'], [ID_A, 'lost']]);
|
||||
world.open()[0].onerror();
|
||||
assert.equal(await world.fire(), 1000);
|
||||
// A summary carries what its detail lists, for the job view's reads (review M12).
|
||||
send(world.open()[0], summary({ id: ID_C, listed: { added: 3, errors: -1 }, log_length: 7 }));
|
||||
assert.deepEqual([store.find(ID_C).listed, store.find(ID_C).log_length], [{ added: 3, errors: 0 }, 7]);
|
||||
world.dispose();
|
||||
});
|
||||
|
||||
test('numbers group the way the server writes them, whatever the browser locale', async () => {
|
||||
const { pillParts, finishToast, parseJob } = await source;
|
||||
const original = Number.prototype.toLocaleString;
|
||||
Number.prototype.toLocaleString = function (locale, ...rest) { return original.call(this, locale ?? 'de-DE', ...rest); };
|
||||
try {
|
||||
const job = parseJob(summary({ progress: 41, processed: 8240, total: 19622, counters: { added: 1420, errors: 18 } }));
|
||||
assert.equal(pillParts(job).text, 'Scanning · 41% · 8,240 / 19,622');
|
||||
assert.equal(finishToast({ ...job, state: 'finished' }), 'Scan finished · 1,420 added · 18 errors');
|
||||
} finally {
|
||||
Number.prototype.toLocaleString = original;
|
||||
}
|
||||
});
|
||||
@@ -33,7 +33,7 @@ test('the outcome toast counts the prompts and the skipped images', async () =>
|
||||
test('the dialog and failure copy is the plan\'s', async () => {
|
||||
const {CATEGORY_DIALOG, CATEGORY_NONE_LINKED, CATEGORY_FAILED} = await source;
|
||||
assert.deepEqual(CATEGORY_DIALOG, {title:'Categorize selected images', label:'Category', action:'Apply'});
|
||||
assert.equal(CATEGORY_NONE_LINKED, 'None of the selected images has a linked prompt.');
|
||||
assert.equal(CATEGORY_NONE_LINKED, "None of the selected images is linked to a saved prompt, so there is no prompt category to set.");
|
||||
assert.equal(CATEGORY_FAILED, 'Category could not be set. Try again.');
|
||||
assert.ok(Object.isFrozen(CATEGORY_DIALOG));
|
||||
});
|
||||
|
||||
@@ -128,6 +128,17 @@ test('all-source defaults are omitted and descending name sort remains supported
|
||||
await stream.reset({root:'all',media:'all',sort:'name-desc'});
|
||||
assert.equal(query.has('root'),false);assert.equal(query.has('media'),false);
|
||||
assert.equal(query.get('sort'),'name-desc');assert.equal(query.get('limit'),'40');
|
||||
assert.equal(query.has('favorites'),false);
|
||||
});
|
||||
|
||||
test('the Favorites page asks the listing for favorites only (#249 D51)', async () => {
|
||||
const {createImageStream}=await source;let query;
|
||||
const stream=createImageStream({api:{get:async url=>{query=new URL(url,'http://local').searchParams;return page([]);}}});
|
||||
await stream.reset({favorites:true,q:'fog'});
|
||||
assert.equal(query.get('favorites'),'1');assert.equal(query.get('q'),'fog');
|
||||
await stream.reset({favorites:false});
|
||||
assert.equal(query.has('favorites'),false);
|
||||
for(const favorites of ['1',1,'true'])await assert.rejects(stream.reset({favorites}),/Invalid favorites filter/);
|
||||
});
|
||||
|
||||
test('restored browsing state resumes its cursor and filters without an initial request', async () => {
|
||||
|
||||
@@ -538,6 +538,136 @@ test('a definite server prompt is kept even when the legacy parser reads it diff
|
||||
assert.equal(record.prompts.positive.text, 'a lighthouse at dusk, oil painting');
|
||||
});
|
||||
|
||||
// #249: a LoRA compare grid. Every sampler's negative is a ConditioningZeroOut of the
|
||||
// positive encode; the legacy parser follows the zero-out back to the positive text.
|
||||
function zeroedGridGraph(samplers = 3) {
|
||||
const graph = {
|
||||
1: { class_type: 'UNETLoader', inputs: { unet_name: 'base.safetensors', weight_dtype: 'default' } },
|
||||
2: { class_type: 'CLIPLoader', inputs: { clip_name: 'te.safetensors', type: 'stable_diffusion' } },
|
||||
4: { class_type: 'CLIPTextEncode', inputs: { text: 'a woman holding a coffee cup', clip: ['2', 0] } },
|
||||
5: { class_type: 'ConditioningZeroOut', inputs: { conditioning: ['4', 0] } },
|
||||
6: { class_type: 'EmptyLatentImage', inputs: { width: 512, height: 512, batch_size: 1 } },
|
||||
};
|
||||
for (let index = 0; index < samplers; index++) {
|
||||
graph[`1${index}`] = { class_type: 'LoraLoaderModelOnly', inputs: { lora_name: `epoch_${index}.safetensors`, strength_model: 1, model: ['1', 0] } };
|
||||
graph[`2${index}`] = { class_type: 'KSampler', inputs: { seed: 42, steps: 8, cfg: 1, sampler_name: 'euler', scheduler: 'simple', denoise: 1,
|
||||
model: [`1${index}`, 0], positive: ['4', 0], negative: ['5', 0], latent_image: ['6', 0] } };
|
||||
}
|
||||
return graph;
|
||||
}
|
||||
|
||||
function undecided(graph) {
|
||||
const entry = image(), data = envelope(entry);
|
||||
data.prompts = { positive: { status: 'ambiguous', text: null, source: null }, negative: { status: 'ambiguous', text: null, source: null } };
|
||||
data.metadata.prompt_graph = { status: 'parsed', value: graph };
|
||||
data.resolution = { state: 'ambiguous', sampler_id: null, reason: 'Multiple sampler branches' };
|
||||
return contract('toInspectionRecord')(data, entry, extractGenerationFacts);
|
||||
}
|
||||
|
||||
test('a zeroed negative is never filled with the positive text from the legacy parse (#249)', t => {
|
||||
const parser = require('../../web/js/comfy-metadata.js');
|
||||
globalThis.ComfyMetadata = parser;
|
||||
t.after(() => { delete globalThis.ComfyMetadata; });
|
||||
// The legacy pages keep their reading: the zero-out passes the positive through.
|
||||
assert.equal(parser.extractGenerationParams({ prompt: zeroedGridGraph() }).negativePrompt, 'a woman holding a coffee cup');
|
||||
const record = undecided(zeroedGridGraph());
|
||||
assert.deepEqual(record.prompts.positive, { status: 'resolved', text: 'a woman holding a coffee cup', source: 'graph-inferred' });
|
||||
assert.deepEqual(record.prompts.negative, { status: 'resolved', text: '', source: 'zeroed-conditioning' });
|
||||
// Behind an API-graph Reroute the legacy parse reads no negative, so there is nothing
|
||||
// to correct: the server's answer stays.
|
||||
const rerouted = zeroedGridGraph(1);
|
||||
rerouted[7] = { class_type: 'Reroute', inputs: { value: ['5', 0] } };
|
||||
rerouted[20].inputs.negative = ['7', 0];
|
||||
assert.equal(parser.legacyNegativeSource({ prompt: rerouted }), null);
|
||||
assert.deepEqual(undecided(rerouted).prompts.negative, { status: 'ambiguous', text: null, source: null });
|
||||
});
|
||||
|
||||
test('the zero-out check follows the sampler the legacy parse read (#249)', t => {
|
||||
const parser = require('../../web/js/comfy-metadata.js');
|
||||
globalThis.ComfyMetadata = parser;
|
||||
t.after(() => { delete globalThis.ComfyMetadata; });
|
||||
const zeroed = { status: 'resolved', text: '', source: 'zeroed-conditioning' };
|
||||
// The legacy parse reads the first sampler (20): its negative is the zero-out.
|
||||
const mixed = zeroedGridGraph(2);
|
||||
mixed[8] = { class_type: 'CLIPTextEncode', inputs: { text: 'blurry', clip: ['2', 0] } };
|
||||
mixed[21].inputs.negative = ['8', 0];
|
||||
assert.deepEqual(undecided(mixed).prompts.negative, zeroed);
|
||||
// Reversed: the sampler it reads encodes "blurry", which it showed before and still does.
|
||||
const reversed = zeroedGridGraph(2);
|
||||
reversed[8] = { class_type: 'CLIPTextEncode', inputs: { text: 'blurry', clip: ['2', 0] } };
|
||||
reversed[20].inputs.negative = ['8', 0];
|
||||
assert.equal(parser.extractGenerationParams({ prompt: reversed }).negativePrompt, 'blurry');
|
||||
assert.deepEqual(undecided(reversed).prompts.negative, { status: 'resolved', text: 'blurry', source: 'graph-inferred' });
|
||||
// The positive fills from the legacy parse in both cases.
|
||||
assert.equal(undecided(mixed).prompts.positive.text, 'a woman holding a coffee cup');
|
||||
assert.equal(undecided(reversed).prompts.positive.text, 'a woman holding a coffee cup');
|
||||
});
|
||||
|
||||
test('a zero-out mixed into the read negative leaves it as the server left it (#249)', t => {
|
||||
globalThis.ComfyMetadata = require('../../web/js/comfy-metadata.js');
|
||||
t.after(() => { delete globalThis.ComfyMetadata; });
|
||||
// A zero-out combined with another conditioning is not an empty negative.
|
||||
const combined = zeroedGridGraph(1);
|
||||
combined[8] = { class_type: 'CLIPTextEncode', inputs: { text: 'blurry', clip: ['2', 0] } };
|
||||
combined[9] = { class_type: 'ConditioningCombine', inputs: { conditioning_1: ['5', 0], conditioning_2: ['8', 0] } };
|
||||
combined[20].inputs.negative = ['9', 0];
|
||||
assert.deepEqual(undecided(combined).prompts.negative, { status: 'ambiguous', text: null, source: null });
|
||||
});
|
||||
|
||||
function workflowOnly(workflow) {
|
||||
const entry = image(), data = envelope(entry);
|
||||
data.prompts = { positive: { status: 'unresolved', text: null, source: null }, negative: { status: 'unresolved', text: null, source: null } };
|
||||
Object.assign(data.metadata, { raw: { workflow: JSON.stringify(workflow) }, record_formats: { workflow: 'reserialized-json' },
|
||||
prompt_graph: { status: 'absent', value: null }, workflow_graph: { status: 'parsed', value: workflow } });
|
||||
return contract('toInspectionRecord')(data, entry, extractGenerationFacts);
|
||||
}
|
||||
|
||||
// A UI workflow: encode 4 -> zero-out 5 -> sampler 3's negative, encode 4 -> positive.
|
||||
function zeroedWorkflow({ zeroMode = 0, extraNodes = [], extraLinks = [], negativeLink = 3 } = {}) {
|
||||
return { nodes: [
|
||||
{ id: 4, type: 'CLIPTextEncode', inputs: [], outputs: [{ links: [1, 2] }], widgets_values: ['a woman holding a coffee cup'] },
|
||||
{ id: 5, type: 'ConditioningZeroOut', mode: zeroMode, inputs: [{ name: 'conditioning', type: 'CONDITIONING', link: 1 }], outputs: [{ links: [3] }] },
|
||||
{ id: 1, type: 'UNETLoader', inputs: [], outputs: [{ links: [4] }], widgets_values: ['base.safetensors', 'default'] },
|
||||
{ id: 3, type: 'KSampler', inputs: [{ name: 'model', link: 4 }, { name: 'positive', link: 2 }, { name: 'negative', link: negativeLink }], widgets_values: [42, 'fixed', 8, 1, 'euler', 'simple', 1] },
|
||||
...extraNodes,
|
||||
], links: [[1, 4, 0, 5, 0, 'CONDITIONING'], [2, 4, 0, 3, 1, 'CONDITIONING'], [3, 5, 0, 3, 2, 'CONDITIONING'], [4, 1, 0, 3, 0, 'MODEL'], ...extraLinks] };
|
||||
}
|
||||
|
||||
test('workflow-only files follow the legacy reading of their zero-out (#249)', t => {
|
||||
const parser = require('../../web/js/comfy-metadata.js');
|
||||
globalThis.ComfyMetadata = parser;
|
||||
t.after(() => { delete globalThis.ComfyMetadata; });
|
||||
// A connected zero-out on the read sampler's negative is the zeroed empty negative.
|
||||
const direct = zeroedWorkflow();
|
||||
assert.equal(parser.extractGenerationParams({ workflow: direct }).negativePrompt, 'a woman holding a coffee cup', 'the legacy parse would fill it');
|
||||
assert.deepEqual(workflowOnly(direct).prompts.negative, { status: 'resolved', text: '', source: 'zeroed-conditioning' });
|
||||
// A bypassed zero-out (mode 4) passes its conditioning through: the legacy reading is right.
|
||||
const bypassed = workflowOnly(zeroedWorkflow({ zeroMode: 4 }));
|
||||
assert.deepEqual(bypassed.prompts.negative, { status: 'resolved', text: 'a woman holding a coffee cup', source: 'graph-inferred' });
|
||||
// An unconnected zero-out beside a text negative changes nothing.
|
||||
const unconnected = workflowOnly(zeroedWorkflow({ negativeLink: 6,
|
||||
extraNodes: [{ id: 6, type: 'CLIPTextEncode', inputs: [], outputs: [{ links: [6] }], widgets_values: ['blurry'] }],
|
||||
extraLinks: [[6, 6, 0, 3, 2, 'CONDITIONING']] }));
|
||||
assert.deepEqual(unconnected.prompts.negative, { status: 'resolved', text: 'blurry', source: 'graph-inferred' });
|
||||
});
|
||||
|
||||
test('a negative read through a subgraph holding a zero-out is not filled (#249)', t => {
|
||||
const parser = require('../../web/js/comfy-metadata.js');
|
||||
globalThis.ComfyMetadata = parser;
|
||||
t.after(() => { delete globalThis.ComfyMetadata; });
|
||||
// The sampler's negative comes from a subgraph instance whose definition zeroes it.
|
||||
const definition = 'aaaaaaaa-0000-4000-8000-000000000001';
|
||||
const workflow = zeroedWorkflow({ negativeLink: 7,
|
||||
extraNodes: [{ id: 9, type: definition, inputs: [{ name: 'conditioning', type: 'CONDITIONING', link: 8 }], outputs: [{ links: [7] }] }],
|
||||
extraLinks: [[7, 9, 0, 3, 2, 'CONDITIONING'], [8, 4, 0, 9, 0, 'CONDITIONING']] });
|
||||
workflow.definitions = { subgraphs: [{ id: definition, nodes: [{ id: 1, type: 'ConditioningZeroOut', inputs: [], outputs: [] }], links: [] }] };
|
||||
assert.equal(parser.extractGenerationParams({ workflow }).negativePrompt, 'a woman holding a coffee cup', 'the legacy parse would fill it');
|
||||
assert.deepEqual(workflowOnly(workflow).prompts.negative, { status: 'unresolved', text: null, source: null });
|
||||
// The same instance of a definition without a zero-out keeps the legacy fill.
|
||||
workflow.definitions.subgraphs[0].nodes[0].type = 'Reroute';
|
||||
assert.equal(workflowOnly(workflow).prompts.negative.text, 'a woman holding a coffee cup');
|
||||
});
|
||||
|
||||
test('callers without the browser global pass the legacy parser explicitly', () => {
|
||||
const parser = require('../../web/js/comfy-metadata.js');
|
||||
const entry = image(), data = envelope(entry);
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
// Saved views (#249 D16): the state whitelist, the concept's default name, the listing parser and the shell's store.
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
|
||||
const source = import('../../web/js/workspace/saved-views.mjs');
|
||||
|
||||
const IMAGE_DEFAULTS = { q: '', category: '', sort: 'newest', source: 'all', folder: '', media: 'all', tags: [], favorites: false };
|
||||
const LIBRARY_DEFAULTS = { q: '', category: '', sort: 'newest', tags: [] };
|
||||
|
||||
test('a saved state always holds every page key and keeps only values its page accepts', async () => {
|
||||
const { normalizeViewState, SAVED_VIEW_PAGES } = await source;
|
||||
assert.deepEqual(SAVED_VIEW_PAGES, ['images', 'library']);
|
||||
assert.deepEqual(normalizeViewState('images', {}), IMAGE_DEFAULTS);
|
||||
assert.deepEqual(normalizeViewState('library', null), LIBRARY_DEFAULTS);
|
||||
const full = { q: 'castle', category: 'Portraits', sort: 'name-desc', source: '2', folder: 'a/b', media: 'video', tags: ['moody', 'fog'], favorites: true };
|
||||
assert.deepEqual(normalizeViewState('images', full), full);
|
||||
// D51: favorites is a boolean; anything else is the whole listing.
|
||||
for (const favorites of ['true', 1, null]) assert.equal(normalizeViewState('images', { ...full, favorites }).favorites, false);
|
||||
// Each rejected value falls back on its own; the rest of the view still applies.
|
||||
assert.deepEqual(normalizeViewState('images', { ...full, sort: 'bogus', source: '-1', media: 'audio', tags: ['a,b'], extra: 1 }),
|
||||
{ ...full, sort: 'newest', source: 'all', media: 'all', tags: [] });
|
||||
assert.deepEqual(normalizeViewState('images', { ...full, q: 'x'.repeat(501), folder: 7, tags: Array(21).fill('t') }),
|
||||
{ ...full, q: '', folder: '', tags: [] });
|
||||
assert.deepEqual(normalizeViewState('images', { ...full, category: 'bad\u0007', tags: [' '] }), { ...full, category: '', tags: [] });
|
||||
// Library has no source, folder or media, and no Name Z–A.
|
||||
assert.deepEqual(normalizeViewState('library', { ...full, sort: 'name-desc' }), { q: 'castle', category: 'Portraits', sort: 'newest', tags: ['moody', 'fog'] });
|
||||
assert.deepEqual(normalizeViewState('library', { sort: 'oldest' }), { ...LIBRARY_DEFAULTS, sort: 'oldest' });
|
||||
// The returned tags are a copy.
|
||||
const tags = ['one'];
|
||||
const state = normalizeViewState('library', { tags });
|
||||
state.tags.push('two');
|
||||
assert.deepEqual(tags, ['one']);
|
||||
assert.throws(() => normalizeViewState('tags', {}), TypeError);
|
||||
});
|
||||
|
||||
test('the default name is the concept search name, else My workspace view, within the name limit', async () => {
|
||||
const { defaultViewName } = await source;
|
||||
assert.equal(defaultViewName('neon'), 'Search: neon');
|
||||
assert.equal(defaultViewName(' neon city '), 'Search: neon city');
|
||||
assert.equal(defaultViewName(''), 'My workspace view');
|
||||
assert.equal(defaultViewName(' '), 'My workspace view');
|
||||
assert.equal(defaultViewName(undefined), 'My workspace view');
|
||||
assert.equal(defaultViewName('x'.repeat(500)).length, 100);
|
||||
});
|
||||
|
||||
test('the listing keeps usable views in order and normalizes their state', async () => {
|
||||
const { parseSavedViews, parseSavedView, savedViewHash } = await source;
|
||||
const views = parseSavedViews({ success: true, views: [
|
||||
{ id: 3, page: 'library', name: 'Neon', state: { q: 'neon', category: '', sort: 'name', tags: [] } },
|
||||
{ id: 0, page: 'images', name: 'Bad id', state: {} },
|
||||
{ id: 4, page: 'tags', name: 'Bad page', state: {} },
|
||||
{ id: 5, page: 'images', name: ' ', state: {} },
|
||||
{ id: 1, page: 'images', name: 'Videos', state: { media: 'video', source: '9' } },
|
||||
] });
|
||||
assert.deepEqual(views, [
|
||||
{ id: 3, page: 'library', name: 'Neon', state: { q: 'neon', category: '', sort: 'name', tags: [] } },
|
||||
{ id: 1, page: 'images', name: 'Videos', state: { ...IMAGE_DEFAULTS, media: 'video', source: '9' } },
|
||||
]);
|
||||
assert.equal(parseSavedView(null), null);
|
||||
assert.throws(() => parseSavedViews({ success: true }), TypeError);
|
||||
assert.equal(savedViewHash(12), '#/saved/12');
|
||||
});
|
||||
|
||||
test('page filters and saved states convert both ways', async () => {
|
||||
const { imagesFilters, imagesViewState, libraryFilters, libraryViewState } = await source;
|
||||
const filters = { q: 'castle', category: 'Portraits', sort: 'rating', root: '1', subfolder: 'x', media: 'image', tags: ['fog'], favorites: true };
|
||||
const state = imagesViewState(filters);
|
||||
assert.deepEqual(state, { q: 'castle', category: 'Portraits', sort: 'rating', source: '1', folder: 'x', media: 'image', tags: ['fog'], favorites: true });
|
||||
assert.deepEqual(imagesFilters(state), filters);
|
||||
assert.deepEqual(imagesFilters({}), { q: '', category: '', sort: 'newest', root: 'all', subfolder: '', media: 'all', tags: [], favorites: false });
|
||||
assert.equal(imagesViewState({ ...filters, favorites: undefined }).favorites, false);
|
||||
const shelf = { text: 'neon', category: 'Cities', sort: 'oldest', tags: ['night'] };
|
||||
assert.deepEqual(libraryViewState(shelf), { q: 'neon', category: 'Cities', sort: 'oldest', tags: ['night'] });
|
||||
assert.deepEqual(libraryFilters(libraryViewState(shelf)), shelf);
|
||||
});
|
||||
|
||||
function fakeApi() {
|
||||
const calls = [];
|
||||
const pending = [];
|
||||
const api = {
|
||||
calls,
|
||||
get(url, options) {
|
||||
calls.push(['GET', url, options.cache]);
|
||||
return new Promise((resolve, reject) => pending.push({ resolve, reject, signal: options.signal }));
|
||||
},
|
||||
post(url, body) {
|
||||
calls.push(['POST', url, body]);
|
||||
return Promise.resolve({ success: true, view: { id: 9, ...body } });
|
||||
},
|
||||
patch(url, body) {
|
||||
calls.push(['PATCH', url, body]);
|
||||
return api.patchResult ?? Promise.resolve({ success: true, view: { id: 9, page: 'images', name: body.name, state: {} } });
|
||||
},
|
||||
del(url) {
|
||||
calls.push(['DELETE', url]);
|
||||
return api.deleteResult ?? Promise.resolve({ success: true, id: 9 });
|
||||
},
|
||||
};
|
||||
return { api, pending };
|
||||
}
|
||||
|
||||
const flush = () => new Promise(resolve => setImmediate(resolve));
|
||||
|
||||
test('the store reports loads, keeps the last views after a failed read, and a newer read supersedes', async () => {
|
||||
const { createSavedViews, SAVED_VIEWS_URL } = await source;
|
||||
const { api, pending } = fakeApi();
|
||||
const controller = new AbortController();
|
||||
const store = createSavedViews(api, controller.signal);
|
||||
const seen = [];
|
||||
store.subscribe(next => seen.push([next.status, next.views.map(view => view.id)]));
|
||||
store.load();
|
||||
const settled = store.settled();
|
||||
pending[0].resolve({ success: true, views: [{ id: 2, page: 'images', name: 'A', state: {} }] });
|
||||
assert.equal((await settled).status, 'ready');
|
||||
assert.deepEqual(store.find(2).state, IMAGE_DEFAULTS);
|
||||
assert.equal(store.find(3), null);
|
||||
assert.deepEqual(api.calls[0], ['GET', SAVED_VIEWS_URL, 'no-store']);
|
||||
|
||||
store.load();
|
||||
pending[1].reject(new Error('offline'));
|
||||
await flush();
|
||||
assert.equal(store.status, 'error');
|
||||
assert.deepEqual(store.views.map(view => view.id), [2]);
|
||||
|
||||
// A second read aborts the first; only the newer one lands.
|
||||
store.load();
|
||||
store.load();
|
||||
assert.equal(pending[2].signal.aborted, true);
|
||||
pending[2].resolve({ success: true, views: [] });
|
||||
pending[3].resolve({ success: true, views: [{ id: 5, page: 'library', name: 'B', state: {} }] });
|
||||
await flush();
|
||||
assert.deepEqual(store.views.map(view => view.id), [5]);
|
||||
assert.deepEqual(seen, [['loading', []], ['ready', [2]], ['loading', [2]], ['error', [2]], ['loading', [2]], ['loading', [2]], ['ready', [5]]]);
|
||||
controller.abort();
|
||||
});
|
||||
|
||||
test('creating a view posts the full state and re-reads the list', async () => {
|
||||
const { createSavedViews, SAVED_VIEWS_URL } = await source;
|
||||
const { api, pending } = fakeApi();
|
||||
const store = createSavedViews(api);
|
||||
const created = store.create('library', 'Neon', { q: 'neon' });
|
||||
await flush();
|
||||
assert.deepEqual(api.calls[0], ['POST', SAVED_VIEWS_URL, { page: 'library', name: 'Neon', state: { ...LIBRARY_DEFAULTS, q: 'neon' } }]);
|
||||
pending[0].resolve({ success: true, views: [{ id: 9, page: 'library', name: 'Neon', state: { q: 'neon' } }] });
|
||||
assert.deepEqual(await created, { id: 9, page: 'library', name: 'Neon', state: { ...LIBRARY_DEFAULTS, q: 'neon' } });
|
||||
assert.equal(store.find(9).name, 'Neon');
|
||||
const broken = createSavedViews({ ...api, post: () => Promise.resolve({ success: true, view: null }) });
|
||||
await assert.rejects(broken.create('images', 'X', {}), TypeError);
|
||||
});
|
||||
|
||||
test('renaming and deleting a view re-read the list, whatever the outcome', async () => {
|
||||
const { createSavedViews, SAVED_VIEWS_URL } = await source;
|
||||
const { api, pending } = fakeApi();
|
||||
const store = createSavedViews(api);
|
||||
const renamed = store.rename(9, 'Dusk');
|
||||
await flush();
|
||||
assert.deepEqual(api.calls[0], ['PATCH', `${SAVED_VIEWS_URL}/9`, { name: 'Dusk' }]);
|
||||
assert.deepEqual(api.calls[1], ['GET', SAVED_VIEWS_URL, 'no-store']);
|
||||
pending[0].resolve({ success: true, views: [{ id: 9, page: 'images', name: 'Dusk', state: {} }] });
|
||||
assert.deepEqual(await renamed, { id: 9, page: 'images', name: 'Dusk', state: IMAGE_DEFAULTS });
|
||||
assert.equal(store.find(9).name, 'Dusk');
|
||||
|
||||
const removed = store.remove(9);
|
||||
await flush();
|
||||
assert.deepEqual(api.calls[2], ['DELETE', `${SAVED_VIEWS_URL}/9`]);
|
||||
pending[1].resolve({ success: true, views: [] });
|
||||
assert.equal(await removed, 9);
|
||||
assert.equal(store.find(9), null);
|
||||
|
||||
// A view another tab already deleted is the outcome asked for; any other failure rejects, after a re-read.
|
||||
api.deleteResult = Promise.reject(Object.assign(new Error('Not found'), { status: 404 }));
|
||||
const gone = store.remove(4);
|
||||
await flush();
|
||||
pending[2].resolve({ success: true, views: [] });
|
||||
assert.equal(await gone, 4);
|
||||
api.deleteResult = Promise.reject(Object.assign(new Error('Locked'), { status: 500 }));
|
||||
const failed = store.remove(4);
|
||||
await flush();
|
||||
pending[3].resolve({ success: true, views: [] });
|
||||
await assert.rejects(failed, /Locked/);
|
||||
api.patchResult = Promise.resolve({ success: true, view: null });
|
||||
const invalid = store.rename(4, 'X');
|
||||
await flush();
|
||||
pending[4].resolve({ success: true, views: [] });
|
||||
await assert.rejects(invalid, TypeError);
|
||||
assert.equal(pending.length, 5);
|
||||
assert.throws(() => store.rename(0, 'X'), TypeError);
|
||||
assert.throws(() => store.remove('9'), TypeError);
|
||||
});
|
||||
|
||||
test('a state over the API 4 KB cap is refused before anything is sent (#249 M5)', async () => {
|
||||
const { createSavedViews, stateFits, SavedViewTooLarge, MAX_STATE_BYTES } = await source;
|
||||
assert.equal(MAX_STATE_BYTES, 4096);
|
||||
assert.equal(stateFits(IMAGE_DEFAULTS), true);
|
||||
// Each value is within its own limit; their multibyte UTF-8 together passes 4 KB.
|
||||
const big = { ...IMAGE_DEFAULTS, q: 'é'.repeat(500), folder: 'é'.repeat(1024), tags: Array.from({ length: 20 }, () => 'é'.repeat(50)) };
|
||||
assert.equal(stateFits(big), false);
|
||||
const { api } = fakeApi();
|
||||
const store = createSavedViews(api);
|
||||
await assert.rejects(store.create('images', 'Big', big), error => error instanceof SavedViewTooLarge && error instanceof RangeError);
|
||||
assert.deepEqual(api.calls, []);
|
||||
});
|
||||
@@ -4,10 +4,10 @@ const source = import('../../web/js/workspace/state.mjs');
|
||||
|
||||
test('workspace routes resolve only supported deep links', async () => {
|
||||
const { resolveRoute } = await source;
|
||||
for (const route of ['overview', 'images', 'library', 'tags', 'inspect', 'system']) {
|
||||
for (const route of ['overview', 'images', 'library', 'tags', 'inspect', 'system', 'activity']) {
|
||||
assert.equal(resolveRoute(`#/${route}`).id, route);
|
||||
}
|
||||
for (const hash of ['', '#', '#/unknown', '#/images/other', '#/<script>']) {
|
||||
for (const hash of ['', '#', '#/unknown', '#/images/other', '#/<script>', '#/activity/abc', `#/activity/${'A'.repeat(32)}`]) {
|
||||
assert.equal(resolveRoute(hash).id, 'images');
|
||||
}
|
||||
});
|
||||
@@ -23,10 +23,33 @@ test('tags is a native route titled Tags; only unbuilt routes hand off to legacy
|
||||
assert.equal(routeTitle(route), route.label);
|
||||
}
|
||||
assert.equal(routeTitle(resolveRoute('#/library/7')), 'Prompt library');
|
||||
for (const id of ['images', 'library', 'tags', 'overview']) assert.equal(isHandoff(resolveRoute(`#/${id}`)), false, id);
|
||||
for (const id of ['images', 'library', 'tags', 'overview', 'activity']) assert.equal(isHandoff(resolveRoute(`#/${id}`)), false, id);
|
||||
for (const id of ['inspect', 'system']) assert.equal(isHandoff(resolveRoute(`#/${id}`)), true, id);
|
||||
});
|
||||
|
||||
test('a job of the Activity page is its own deep link', async () => {
|
||||
const { resolveRoute, routeTitle, isHandoff } = await source;
|
||||
const id = '0123456789abcdef'.repeat(2);
|
||||
const route = resolveRoute(`#/activity/${id}`);
|
||||
assert.deepEqual([route.id, route.jobId, route.hash, routeTitle(route), isHandoff(route)],
|
||||
['activity', id, `#/activity/${id}`, 'Activity', false]);
|
||||
assert.equal(resolveRoute('#/activity').jobId, undefined);
|
||||
assert.equal(route.href, undefined, 'Activity has no legacy card on Overview');
|
||||
// The router renders again when only the job changes, and not for the same job.
|
||||
const { mountRouter } = await source;
|
||||
const other = 'f'.repeat(32);
|
||||
const win = new EventTarget();
|
||||
win.location = { hash: `#/activity/${id}` };
|
||||
win.history = { state: null, replaceState(state, unused, hash) { win.location.hash = hash; } };
|
||||
const seen = [];
|
||||
const dispose = mountRouter(win, next => seen.push([next.id, next.jobId]));
|
||||
win.dispatchEvent(new Event('hashchange'));
|
||||
win.location.hash = `#/activity/${other}`; win.dispatchEvent(new Event('hashchange'));
|
||||
win.location.hash = '#/activity'; win.dispatchEvent(new Event('hashchange'));
|
||||
assert.deepEqual(seen, [['activity', id], ['activity', other], ['activity', undefined]]);
|
||||
dispose();
|
||||
});
|
||||
|
||||
test('statistics preserve zero and reject incomplete or invalid counts', async () => {
|
||||
const { summarizeStats } = await source;
|
||||
assert.deepEqual(summarizeStats({total_prompts:0, total_images:12, total_tags:3}), [
|
||||
@@ -79,18 +102,19 @@ test('positive library deep links preserve the selected identity without duplica
|
||||
|
||||
test('nav counts reject negative, non-integer and missing totals', async () => {
|
||||
const { summarizeNavCounts } = await source;
|
||||
const counts = summarizeNavCounts({images: {total: 24}, library: {total: 0}, tags: {total: 7}});
|
||||
const counts = summarizeNavCounts({images: {total: 24}, favorites: {total: 3}, library: {total: 0}, tags: {total: 7}});
|
||||
assert.ok(counts instanceof Map);
|
||||
assert.deepEqual([...counts], [['images', 24], ['library', 0], ['tags', 7]]);
|
||||
assert.deepEqual([...counts], [['images', 24], ['favorites', 3], ['library', 0], ['tags', 7]]);
|
||||
assert.equal(counts.has('inspect'), false, 'Inspect a file has no count');
|
||||
for (const invalid of [undefined, null, {}, {total: -1}, {total: 1.5}, {total: '3'}, {total: NaN},
|
||||
{total: Infinity}, {total: Number.MAX_SAFE_INTEGER + 1}, {total: null}, [], 'x']) {
|
||||
const result = summarizeNavCounts({images: invalid, library: {total: 2}, tags: invalid});
|
||||
const result = summarizeNavCounts({images: invalid, favorites: invalid, library: {total: 2}, tags: invalid});
|
||||
assert.equal(result.get('images'), null, JSON.stringify(invalid));
|
||||
assert.equal(result.get('favorites'), null);
|
||||
assert.equal(result.get('library'), 2);
|
||||
assert.equal(result.get('tags'), null);
|
||||
}
|
||||
assert.deepEqual([...summarizeNavCounts()], [['images', null], ['library', null], ['tags', null]]);
|
||||
assert.deepEqual([...summarizeNavCounts()], [['images', null], ['favorites', null], ['library', null], ['tags', null]]);
|
||||
});
|
||||
|
||||
test('nav counts settle independently so a slow images read never delays the others', async () => {
|
||||
@@ -98,6 +122,7 @@ test('nav counts settle independently so a slow images read never delays the oth
|
||||
let releaseImages;
|
||||
const images = new Promise(resolve => { releaseImages = resolve; });
|
||||
const api = {get: async url => {
|
||||
if (url.includes('favorites=1')) return {success: true, total: 2};
|
||||
if (url.startsWith('/prompt_manager/images')) return images;
|
||||
if (url.startsWith('/prompt_manager/library')) return {success: true, total: 3};
|
||||
throw new Error('Unavailable');
|
||||
@@ -105,14 +130,14 @@ test('nav counts settle independently so a slow images read never delays the oth
|
||||
const settled = [];
|
||||
const done = loadNavCounts(api, new AbortController().signal, (id, count) => settled.push([id, count]));
|
||||
await new Promise(resolve => setTimeout(resolve, 0));
|
||||
assert.deepEqual(settled.sort(), [['library', 3], ['tags', null]]);
|
||||
assert.deepEqual(settled.sort(), [['favorites', 2], ['library', 3], ['tags', null]]);
|
||||
releaseImages({success: true, total: 40});
|
||||
const counts = await done;
|
||||
assert.deepEqual(settled.at(-1), ['images', 40]);
|
||||
assert.deepEqual([...counts], [['images', 40], ['library', 3], ['tags', null]]);
|
||||
assert.deepEqual([...counts], [['images', 40], ['favorites', 2], ['library', 3], ['tags', null]]);
|
||||
});
|
||||
|
||||
test('nav counts issue three bounded requests and drop only the failed count', async () => {
|
||||
test('nav counts issue four bounded requests and drop only the failed count', async () => {
|
||||
const { loadNavCounts } = await source;
|
||||
const calls = [];
|
||||
const signal = new AbortController().signal;
|
||||
@@ -123,15 +148,42 @@ test('nav counts issue three bounded requests and drop only the failed count', a
|
||||
return {success: true, total: 12};
|
||||
}};
|
||||
const counts = await loadNavCounts(api, signal);
|
||||
assert.deepEqual([...counts], [['images', 12], ['library', null], ['tags', 5]]);
|
||||
assert.deepEqual([...counts], [['images', 12], ['favorites', 12], ['library', null], ['tags', 5]]);
|
||||
const urls = calls.map(([url]) => new URL(url, 'http://host'));
|
||||
assert.deepEqual(urls.map(url => url.pathname),
|
||||
['/prompt_manager/images/output', '/prompt_manager/library', '/prompt_manager/vocabulary/tags']);
|
||||
['/prompt_manager/images/output', '/prompt_manager/images/output', '/prompt_manager/library', '/prompt_manager/vocabulary/tags']);
|
||||
for (const url of urls) assert.equal(url.searchParams.get('limit'), '1');
|
||||
assert.equal(urls[0].searchParams.get('mode'), 'workspace');
|
||||
assert.equal(urls[0].searchParams.has('favorites'), false);
|
||||
// D51: the Favorites count is the Favorites page's own listing.
|
||||
assert.equal(urls[1].searchParams.get('mode'), 'workspace');
|
||||
assert.equal(urls[1].searchParams.get('favorites'), '1');
|
||||
assert.ok(calls.every(([, used]) => used === signal));
|
||||
});
|
||||
|
||||
test('a favorite change re-reads only the Favorites count', async () => {
|
||||
const { loadNavCounts, FAVORITE_WRITE_COUNTS } = await source;
|
||||
assert.deepEqual([...FAVORITE_WRITE_COUNTS], ['favorites']);
|
||||
assert.ok(Object.isFrozen(FAVORITE_WRITE_COUNTS));
|
||||
const calls = [];
|
||||
const api = {get: async url => { calls.push(url); return {success: true, total: 0}; }};
|
||||
const counts = await loadNavCounts(api, new AbortController().signal, () => {}, FAVORITE_WRITE_COUNTS);
|
||||
assert.deepEqual(calls, ['/prompt_manager/images/output?mode=workspace&favorites=1&limit=1']);
|
||||
// A real zero is a count, unlike an outstanding read.
|
||||
assert.deepEqual([...counts], [['favorites', 0]]);
|
||||
});
|
||||
|
||||
test('the Favorites route is native and resolves by its hash', async () => {
|
||||
const { resolveRoute, isHandoff, routeTitle, routes } = await source;
|
||||
const route = resolveRoute('#/favorites');
|
||||
assert.equal(route.id, 'favorites');
|
||||
assert.equal(route.label, 'Favorites');
|
||||
assert.equal(routeTitle(route), 'Favorites');
|
||||
assert.equal(isHandoff(route), false);
|
||||
// No href: Overview lists no legacy card for it.
|
||||
assert.equal(routes.find(item => item.id === 'favorites').href, undefined);
|
||||
});
|
||||
|
||||
test('a Library or vocabulary write re-reads only the Library and Tags counts', async () => {
|
||||
const { loadNavCounts, LIBRARY_WRITE_COUNTS } = await source;
|
||||
assert.deepEqual([...LIBRARY_WRITE_COUNTS], ['library', 'tags']);
|
||||
@@ -160,3 +212,24 @@ test('navigating to the current route re-renders it, as the concept go() does',
|
||||
win.dispatchEvent(new Event('hashchange'));assert.deepEqual(seen.at(-1),['library',13]);
|
||||
dispose();navigateTo('#/library/13',win);assert.equal(seen.length,3);
|
||||
});
|
||||
|
||||
// Saved views (#249 D16): `#/saved/<id>` opens one persisted view; the shell resolves it to its page.
|
||||
test('saved-view links resolve to their id and re-render only when the id changes', async () => {
|
||||
const {resolveRoute,mountRouter,isHandoff,routes}=await source;
|
||||
const route=resolveRoute('#/saved/7');
|
||||
assert.deepEqual([route.id,route.savedId,route.hash],['saved',7,'#/saved/7']);
|
||||
assert.equal(isHandoff(route),false);
|
||||
assert.equal(routes.some(item=>item.id==='saved'),false,'saved views are not an Overview card or a WORKSPACE link');
|
||||
for(const hash of ['#/saved','#/saved/0','#/saved/-2','#/saved/1.5','#/saved/9007199254740992','#/saved/3/x','#/saved/abc']) {
|
||||
assert.equal(resolveRoute(hash).id,'images',hash);
|
||||
}
|
||||
const win=new EventTarget();win.location={hash:'#/saved/3'};
|
||||
win.history={state:null,replaceState(state,unused,hash){win.location.hash=hash;}};
|
||||
const seen=[];const dispose=mountRouter(win,next=>seen.push([next.id,next.savedId]));
|
||||
assert.equal(win.location.hash,'#/saved/3');
|
||||
win.dispatchEvent(new Event('hashchange'));
|
||||
win.location.hash='#/saved/4';win.dispatchEvent(new Event('hashchange'));
|
||||
win.location.hash='#/images';win.dispatchEvent(new Event('hashchange'));
|
||||
assert.deepEqual(seen,[['saved',3],['saved',4],['images',undefined]]);
|
||||
dispose();
|
||||
});
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
// The shell's shared toast: plain messages clear after 6.5 s; a message with a link stays 20 s (D57), and any toast
|
||||
// stays while the pointer or keyboard focus rests on it.
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
|
||||
const source = import('../../web/js/workspace/toast.mjs');
|
||||
|
||||
class FakeToast extends EventTarget {
|
||||
constructor() { super(); this.held = false; this.nodes = []; this.text = ''; }
|
||||
get textContent() { return this.text; }
|
||||
set textContent(value) { this.text = value; this.nodes = []; }
|
||||
replaceChildren(...nodes) { this.nodes = nodes; this.text = nodes.map(node => node.textContent).join(''); }
|
||||
matches(selector) { assert.equal(selector, ':hover, :focus-within'); return this.held; }
|
||||
}
|
||||
|
||||
function install() {
|
||||
const toast = new FakeToast();
|
||||
const previous = global.document;
|
||||
global.document = {
|
||||
getElementById: id => id === 'workspace-toast' ? toast : null,
|
||||
createTextNode: text => ({ textContent: text }),
|
||||
createElement: tag => ({ tag, href: '', textContent: '' }),
|
||||
};
|
||||
return { toast, restore: () => { global.document = previous; } };
|
||||
}
|
||||
|
||||
test('a plain toast clears after 6.5 s and a linked one after 20 s', async t => {
|
||||
const { showWorkspaceToast, TOAST_MS, LINK_TOAST_MS } = await source;
|
||||
t.mock.timers.enable({ apis: ['setTimeout'] });
|
||||
const { toast, restore } = install();
|
||||
try {
|
||||
showWorkspaceToast('Scan started.');
|
||||
assert.equal(toast.textContent, 'Scan started.');
|
||||
t.mock.timers.tick(TOAST_MS);
|
||||
assert.equal(toast.textContent, '');
|
||||
|
||||
showWorkspaceToast(['Another job is running: ', { text: 'Scan', href: '#/activity/x' }, '.']);
|
||||
assert.equal(toast.textContent, 'Another job is running: Scan.');
|
||||
assert.deepEqual(toast.nodes[1], { tag: 'a', href: '#/activity/x', textContent: 'Scan' });
|
||||
t.mock.timers.tick(TOAST_MS);
|
||||
assert.equal(toast.textContent, 'Another job is running: Scan.');
|
||||
t.mock.timers.tick(LINK_TOAST_MS - TOAST_MS);
|
||||
assert.equal(toast.textContent, '');
|
||||
} finally {
|
||||
restore();
|
||||
}
|
||||
});
|
||||
|
||||
test('a toast under the pointer or focus waits until they leave', async t => {
|
||||
const { showWorkspaceToast, TOAST_MS } = await source;
|
||||
t.mock.timers.enable({ apis: ['setTimeout'] });
|
||||
const { toast, restore } = install();
|
||||
try {
|
||||
showWorkspaceToast('Prompt copied.');
|
||||
toast.held = true;
|
||||
t.mock.timers.tick(TOAST_MS);
|
||||
assert.equal(toast.textContent, 'Prompt copied.', 'held: it stays');
|
||||
// Focus moving inside the toast keeps it; leaving it starts a fresh 6.5 s.
|
||||
toast.dispatchEvent(new Event('focusout'));
|
||||
t.mock.timers.tick(0);
|
||||
toast.dispatchEvent(new Event('pointerleave'));
|
||||
assert.equal(toast.textContent, 'Prompt copied.');
|
||||
toast.held = false;
|
||||
toast.dispatchEvent(new Event('focusout'));
|
||||
t.mock.timers.tick(0);
|
||||
t.mock.timers.tick(TOAST_MS - 1);
|
||||
assert.equal(toast.textContent, 'Prompt copied.');
|
||||
t.mock.timers.tick(1);
|
||||
assert.equal(toast.textContent, '');
|
||||
|
||||
// A newer toast replaces a held one and stops waiting for the old one's pointer.
|
||||
showWorkspaceToast('First.');
|
||||
toast.held = true;
|
||||
t.mock.timers.tick(TOAST_MS);
|
||||
showWorkspaceToast('Second.');
|
||||
toast.held = false;
|
||||
toast.dispatchEvent(new Event('pointerleave'));
|
||||
t.mock.timers.tick(TOAST_MS - 1);
|
||||
assert.equal(toast.textContent, 'Second.');
|
||||
t.mock.timers.tick(1);
|
||||
assert.equal(toast.textContent, '');
|
||||
} finally {
|
||||
restore();
|
||||
}
|
||||
});
|
||||
|
||||
test('a page without the toast shows nothing and does not fail', async () => {
|
||||
const { showWorkspaceToast } = await source;
|
||||
const previous = global.document;
|
||||
global.document = { getElementById: () => null };
|
||||
try {
|
||||
assert.doesNotThrow(() => showWorkspaceToast('Nothing.'));
|
||||
} finally {
|
||||
global.document = previous;
|
||||
}
|
||||
});
|
||||
@@ -116,19 +116,21 @@ test('toolbar emits concept markup and order', async () => {
|
||||
dom.body.append(container);
|
||||
const changes = [];
|
||||
const filters = [];
|
||||
const saves = [];
|
||||
const toolbar = mountToolbar(container, {
|
||||
scope, categories: ['Portraits', 'Landscapes'],
|
||||
sorts: [['newest', 'Recently added'], ['name', 'Name A–Z'], ['rating', 'Highest rated']],
|
||||
value: { q: 'castle', category: 'Landscapes', sort: 'name' },
|
||||
onChange: value => changes.push(value), onFilters: trigger => filters.push(trigger),
|
||||
onSaveView: trigger => saves.push(trigger),
|
||||
});
|
||||
const root = container.children[0];
|
||||
assert.equal(toolbar.element, root);
|
||||
assert.equal(root.tagName, 'SEARCH');
|
||||
assert.equal(root.className, 'toolbar');
|
||||
assert.deepEqual(tags(root), ['label', 'select', 'select', 'button']);
|
||||
assert.deepEqual(tags(root), ['label', 'select', 'select', 'button', 'button']);
|
||||
|
||||
const [label, category, sort, filtersButton] = root.children;
|
||||
const [label, category, sort, saveButton, filtersButton] = root.children;
|
||||
assert.equal(label.className, 'search-field');
|
||||
assert.equal(label.htmlFor, 'search');
|
||||
assert.deepEqual(tags(label), ['svg', 'input']);
|
||||
@@ -151,8 +153,12 @@ test('toolbar emits concept markup and order', async () => {
|
||||
assert.equal(sort.getAttribute('aria-label'), 'Sort results');
|
||||
assert.equal(sort.value, 'name');
|
||||
|
||||
// D16: no Save view until saved views are persisted.
|
||||
assert.equal(button(root, 'Save view'), undefined);
|
||||
// The concept's Save view, between sort and Filters (a359), reports its own button as the trigger.
|
||||
assert.equal(button(root, 'Save view'), saveButton);
|
||||
assert.equal(saveButton.type, 'button');
|
||||
saveButton.click();
|
||||
assert.deepEqual(saves, [saveButton]);
|
||||
assert.deepEqual(filters, []);
|
||||
assert.equal(filtersButton.textContent, 'Filters');
|
||||
assert.equal(filtersButton.type, 'button');
|
||||
filtersButton.click();
|
||||
@@ -184,6 +190,9 @@ test('toolbar emits concept markup and order', async () => {
|
||||
toolbar.setValue({ sort: 'bogus' });
|
||||
assert.equal(sort.value, 'newest');
|
||||
assert.throws(() => mountToolbar(container, { categories: [], sorts: [['newest', 'Recently added']],
|
||||
onChange() {}, onFilters() {}, onSaveView() {} }), TypeError);
|
||||
// Every toolbar page offers Save view, so the callback is required.
|
||||
assert.throws(() => mountToolbar(container, { scope, categories: [], sorts: [['newest', 'Recently added']],
|
||||
onChange() {}, onFilters() {} }), TypeError);
|
||||
} finally {
|
||||
scope.dispose();
|
||||
@@ -200,7 +209,7 @@ test('toolbar search uses a debounced onChange that stops with its scope', async
|
||||
const container = new dom.Element('div');
|
||||
const changes = [];
|
||||
mountToolbar(container, { scope, categories: [], sorts: [['newest', 'Recently added']],
|
||||
value: { q: '', category: '', sort: 'newest' }, onChange: value => changes.push(value), onFilters() {} });
|
||||
value: { q: '', category: '', sort: 'newest' }, onChange: value => changes.push(value), onFilters() {}, onSaveView() {} });
|
||||
const input = container.children[0].children[0].children[1];
|
||||
assert.equal(input.id, 'search');
|
||||
for (const text of ['c', 'ca', 'cat']) {
|
||||
@@ -244,7 +253,7 @@ test('sort lists concept options first', async () => {
|
||||
try {
|
||||
const container = new dom.Element('div');
|
||||
mountToolbar(container, { scope, categories: [], sorts: given, value: { q: '', category: '', sort: 'oldest' },
|
||||
onChange() {}, onFilters() {} });
|
||||
onChange() {}, onFilters() {}, onSaveView() {} });
|
||||
const sort = container.children[0].children[2];
|
||||
assert.deepEqual(optionPairs(sort).map(([, label]) => label),
|
||||
['Recently added', 'Name A–Z', 'Highest rated', 'Oldest first', 'Filename Z–A']);
|
||||
@@ -333,6 +342,56 @@ test('refine dialog round-trips values and clear resets them', async () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('Refine lists options discovered after it opened and keeps the wanted values', async () => {
|
||||
const { openRefineDialog, openImportDialog, closeWorkspaceDialog } = await dialogsModule;
|
||||
const dom = fakeDOM();
|
||||
const scope = (await coreModule).createScope();
|
||||
try {
|
||||
let bySource = {};
|
||||
const optionsFor = source => [['', 'All output folders'], ...(bySource[source] ?? []).map(path => [path, path])];
|
||||
const fields = [{ key: 'source', options: [['all', 'All sources']] }, { key: 'folder', options: optionsFor('all'), optionsFor },
|
||||
REFINE_FIELDS[2]];
|
||||
const applied = [];
|
||||
const refine = openRefineDialog({ scope, fields, values: { source: '1', folder: 'temp-only', media: 'video' },
|
||||
onApply: value => applied.push(value), onClear() {} });
|
||||
const [source, folder, media] = byId(dom, 'modal-body').children.filter(child => child.className === 'field')
|
||||
.map(field => field.children[1]);
|
||||
// Opened before discovery: only the cleared values.
|
||||
assert.deepEqual(optionPairs(source), [['all', 'All sources']]);
|
||||
assert.equal(source.value, 'all');
|
||||
assert.equal(folder.value, '');
|
||||
bySource = { all: ['portraits', 'temp-only'], 0: ['portraits'], 1: ['temp-only'] };
|
||||
refine.setOptions('source', [['all', 'All sources'], ['0', 'output'], ['1', 'archive']]);
|
||||
// The values it opened with, now offered, and the chosen Source's folders.
|
||||
assert.deepEqual(optionPairs(source).map(([value]) => value), ['all', '0', '1']);
|
||||
assert.equal(source.value, '1');
|
||||
assert.deepEqual(optionPairs(folder), [['', 'All output folders'], ['temp-only', 'temp-only']]);
|
||||
assert.equal(folder.value, 'temp-only');
|
||||
// A choice made since wins over the opening value; tags and unknown keys are ignored.
|
||||
source.value = '0';
|
||||
source.dispatchEvent(new Event('change'));
|
||||
refine.setOptions('source', [['all', 'All sources'], ['0', 'output'], ['1', 'archive']]);
|
||||
refine.setOptions('tags', [['x', 'x']]);
|
||||
assert.equal(source.value, '0');
|
||||
assert.deepEqual(optionPairs(folder).map(([value]) => value), ['', 'portraits']);
|
||||
assert.equal(media.value, 'video');
|
||||
button(byId(dom, 'modal-body'), 'Apply filters').click();
|
||||
assert.deepEqual(applied, [{ source: '0', folder: '', media: 'video' }]);
|
||||
// Closed: a late discovery leaves the shell alone.
|
||||
refine.setOptions('source', [['all', 'All sources']]);
|
||||
assert.equal(source.options.length, 3);
|
||||
// Replaced by another dialog: the same.
|
||||
const again = openRefineDialog({ scope, fields, values: {}, onApply() {}, onClear() {} });
|
||||
openImportDialog({ scope, onInspect() {}, onRescan() {} });
|
||||
again.setOptions('source', [['all', 'All sources'], ['9', 'late']]);
|
||||
assert.equal(byId(dom, 'modal-title').textContent, 'Import & inspect');
|
||||
} finally {
|
||||
closeWorkspaceDialog();
|
||||
scope.dispose();
|
||||
dom.restore();
|
||||
}
|
||||
});
|
||||
|
||||
test('the Folder list follows the chosen Source', async () => {
|
||||
const { openRefineDialog, closeWorkspaceDialog } = await dialogsModule;
|
||||
const dom = fakeDOM();
|
||||
@@ -469,6 +528,40 @@ test('copy fallback selects the text', async () => {
|
||||
}
|
||||
});
|
||||
|
||||
// The concept's Save view dialog (a1330–1346 via simpleInput a1138): an empty name falls back to "Saved view".
|
||||
test('an input dialog with a fallback applies it for an empty value', async () => {
|
||||
const { openInputDialog, closeWorkspaceDialog } = await dialogsModule;
|
||||
const dom = fakeDOM();
|
||||
const scope = (await coreModule).createScope();
|
||||
try {
|
||||
const applied = [];
|
||||
const open = value => openInputDialog({ scope, title: 'Save this view', label: 'View name', action: 'Save',
|
||||
value, maxLength: 100, fallback: 'Saved view', onApply: name => applied.push(name) });
|
||||
open('My workspace view');
|
||||
assert.equal(byId(dom, 'modal-title').textContent, 'Save this view');
|
||||
const body = byId(dom, 'modal-body');
|
||||
const [field, actions] = body.children;
|
||||
const [label, input] = field.children;
|
||||
assert.equal(label.textContent, 'View name');
|
||||
assert.equal(input.id, 'action-input');
|
||||
assert.equal(input.value, 'My workspace view');
|
||||
assert.equal(input.maxLength, 100);
|
||||
assert.deepEqual(actions.children.map(node => node.textContent), ['Cancel', 'Save']);
|
||||
input.value = ' ';
|
||||
actions.children[1].click();
|
||||
assert.deepEqual(applied, ['Saved view']);
|
||||
open('');
|
||||
byId(dom, 'action-input').value = ' Castles ';
|
||||
byId(dom, 'modal-body').children[1].children[1].click();
|
||||
assert.deepEqual(applied, ['Saved view', 'Castles']);
|
||||
assert.throws(() => openInputDialog({ scope, title: 'T', label: 'L', action: 'A', fallback: '', onApply() {} }), TypeError);
|
||||
} finally {
|
||||
closeWorkspaceDialog();
|
||||
scope.dispose();
|
||||
dom.restore();
|
||||
}
|
||||
});
|
||||
|
||||
test('dialogs restore focus to the trigger on Escape', async () => {
|
||||
const { openRefineDialog, openImportDialog, openCopyFallback, closeWorkspaceDialog } = await dialogsModule;
|
||||
const dom = fakeDOM();
|
||||
@@ -662,8 +755,8 @@ test('the Filters button shows the active refinement count', async () => {
|
||||
try {
|
||||
const container = new dom.Element('div');
|
||||
const toolbar = mountToolbar(container, { scope, categories: [], sorts: [['newest', 'Recently added']],
|
||||
activeFilters: 2, onChange() {}, onFilters() {} });
|
||||
const filtersButton = container.children[0].children[3];
|
||||
activeFilters: 2, onChange() {}, onFilters() {}, onSaveView() {} });
|
||||
const filtersButton = container.children[0].children[4];
|
||||
// The visible text is the accessible name, so the count is spoken too.
|
||||
assert.equal(filtersButton.textContent, 'Filters · 2');
|
||||
toolbar.setActiveFilters(1);
|
||||
@@ -672,8 +765,22 @@ test('the Filters button shows the active refinement count', async () => {
|
||||
assert.equal(filtersButton.textContent, 'Filters');
|
||||
for (const bad of [-1, 1.5, '1', null, Number.NaN]) assert.throws(() => toolbar.setActiveFilters(bad), TypeError);
|
||||
assert.equal(filtersButton.textContent, 'Filters');
|
||||
// An unchanged count leaves the label's text node alone: every listing reload reports the count, and WebKit
|
||||
// drops a click whose mousedown hit a text node that was replaced before mouseup.
|
||||
let writes = 0;
|
||||
const proto = Object.getPrototypeOf(filtersButton);
|
||||
const descriptor = Object.getOwnPropertyDescriptor(proto, 'textContent');
|
||||
Object.defineProperty(filtersButton, 'textContent', {
|
||||
get() { return descriptor.get.call(this); },
|
||||
set(text) { writes += 1; descriptor.set.call(this, text); },
|
||||
configurable: true,
|
||||
});
|
||||
toolbar.setActiveFilters(0);
|
||||
assert.equal(writes, 0);
|
||||
toolbar.setActiveFilters(3);
|
||||
assert.deepEqual([writes, filtersButton.textContent], [1, 'Filters · 3']);
|
||||
assert.throws(() => mountToolbar(new dom.Element('div'), { scope, categories: [], sorts: [['newest', 'Recently added']],
|
||||
activeFilters: -1, onChange() {}, onFilters() {} }), TypeError);
|
||||
activeFilters: -1, onChange() {}, onFilters() {}, onSaveView() {} }), TypeError);
|
||||
} finally {
|
||||
scope.dispose();
|
||||
dom.restore();
|
||||
|
||||
@@ -507,10 +507,49 @@ test('category preview text states the real impact and whether it renames or con
|
||||
'“Landscapes” is used by 2 other saved prompts. Apply renames it to “Scenery”.');
|
||||
});
|
||||
|
||||
test('accepts image_count and names images in the delete text', async () => {
|
||||
const {createVocabularyStream, createVocabularyCommand, readTagImpact, deleteTagText} = await helpers;
|
||||
const rows = [{...tag(1, 'mist', 2), image_count: 3}, tag(2, 'dusk', 0)];
|
||||
const stream = createVocabularyStream({api: queued([tagPage(rows)]).api, kind: 'tags'});
|
||||
assert.equal(await stream.loadMore(), 2);
|
||||
assert.deepEqual(stream.state.items.map(row => row.image_count), [3, undefined]);
|
||||
for (const bad of [{...tag(1), image_count: -1}, {...tag(1), image_count: '3'}, {...tag(1), image_count: 1.5}]) {
|
||||
const rejected = createVocabularyStream({api: queued([tagPage([bad])]).api, kind: 'tags'});
|
||||
await rejected.loadMore();
|
||||
assert.ok(rejected.state.error instanceof TypeError, JSON.stringify(bad));
|
||||
assert.deepEqual(rejected.state.items, []);
|
||||
}
|
||||
const withImages = {...impact, images: 2};
|
||||
const api = {get: async () => ({success: true, library_epoch: EPOCH, impact: withImages})};
|
||||
assert.deepEqual((await readTagImpact(api, {id: 4, name: 'mist'})).impact, withImages);
|
||||
await assert.rejects(readTagImpact({get: async () => ({success: true, library_epoch: EPOCH, impact: {...impact, images: -2}})},
|
||||
{id: 4, name: 'mist'}), TypeError);
|
||||
const command = createVocabularyCommand({api: {
|
||||
patch: async () => renameReceipt({tag: {...tag(1, 'dusk', 2), image_count: 1}, images_changed: 1}),
|
||||
post: async () => ({success: true, library_epoch: EPOCH, deleted: true, components_changed: 0, pairs_changed: 0, images_changed: 2}),
|
||||
}});
|
||||
await command.renameTag({id: 1, name: 'dawn', prompt_count: 2}, 'dusk', EPOCH);
|
||||
assert.equal(command.state.phase, 'done');
|
||||
assert.equal(command.state.receipt.images_changed, 1);
|
||||
command.reset();
|
||||
await command.deleteTag({id: 1, name: 'dusk'}, EPOCH);
|
||||
assert.equal(command.state.phase, 'done');
|
||||
const malformed = createVocabularyCommand({api: {post: async () => ({success: true, library_epoch: EPOCH, deleted: true,
|
||||
components_changed: 0, pairs_changed: 0, images_changed: 'two'})}});
|
||||
await malformed.deleteTag({id: 1, name: 'dusk'}, EPOCH);
|
||||
assert.equal(malformed.state.phase, 'unknown-outcome');
|
||||
assert.equal(deleteTagText('mist', withImages),
|
||||
'Deleting “mist” removes the name and its links from 3 positive prompts, 3 negative variants (1 archived), 1 other saved prompt and 2 images. The prompts, negative variants, images and history themselves are kept.');
|
||||
assert.equal(deleteTagText('mist', {...zero, images: 1}),
|
||||
'Deleting “mist” removes the name and its links from 1 image. The prompts, negative variants, images and history themselves are kept.');
|
||||
assert.equal(deleteTagText('unused', {...zero, images: 0}),
|
||||
'“unused” is not linked to any saved prompt or image. Deleting it removes only the name.');
|
||||
});
|
||||
|
||||
test('delete text names the tag, its real links and what stays unchanged', async () => {
|
||||
const {deleteTagText} = await helpers;
|
||||
assert.equal(deleteTagText('mist', impact),
|
||||
'Deleting “mist” removes the name and its links from 3 positive prompts, 3 negative variants (1 archived) and 1 other saved prompt. The prompts, negative variants, images and history themselves are kept.');
|
||||
assert.equal(deleteTagText('unused', zero),
|
||||
'“unused” is not linked to any saved prompt. Deleting it removes only the name.');
|
||||
'“unused” is not linked to any saved prompt or image. Deleting it removes only the name.');
|
||||
});
|
||||
|
||||
@@ -0,0 +1,602 @@
|
||||
"""Image annotation schema group (#249): install, validation and restore safety."""
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
try:
|
||||
from tests.open_handles import assert_closed
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from open_handles import assert_closed
|
||||
|
||||
from database.models import PromptModel
|
||||
|
||||
try: # guarded so a missing module fails assertions, not test loading
|
||||
from database.annotation_schema import (
|
||||
annotation_schema_error,
|
||||
migrate_annotation_state,
|
||||
)
|
||||
except ImportError:
|
||||
annotation_schema_error = None
|
||||
migrate_annotation_state = None
|
||||
|
||||
METADATA = "media_annotation_metadata"
|
||||
FILES = "media_annotation_files"
|
||||
BYTES = "media_annotation_bytes"
|
||||
IMAGE_TAGS = "media_image_tags"
|
||||
TABLES = (METADATA, FILES, BYTES, IMAGE_TAGS)
|
||||
# The exact DDL the plan pins; the module must produce precisely these objects.
|
||||
DDL = {
|
||||
METADATA: (
|
||||
f"CREATE TABLE {METADATA}(singleton_id INTEGER PRIMARY KEY "
|
||||
"CHECK(singleton_id=1), schema_version INTEGER NOT NULL "
|
||||
"CHECK(schema_version=1), image_index_revision INTEGER NOT NULL DEFAULT 0 "
|
||||
"CHECK(image_index_revision>=0))"
|
||||
),
|
||||
FILES: (
|
||||
f"CREATE TABLE {FILES}(id TEXT PRIMARY KEY CHECK(length(id)=36), "
|
||||
"locator TEXT NOT NULL UNIQUE, created_at TEXT NOT NULL)"
|
||||
),
|
||||
BYTES: (
|
||||
f"CREATE TABLE {BYTES}(id TEXT PRIMARY KEY CHECK(length(id)=36), "
|
||||
f"file_id TEXT NOT NULL REFERENCES {FILES}(id) ON DELETE CASCADE, "
|
||||
"sha256 TEXT NOT NULL CHECK(length(sha256)=64 AND sha256 NOT GLOB "
|
||||
"'*[^0-9a-f]*'), size INTEGER NOT NULL CHECK(size>=0), "
|
||||
"mtime_ns INTEGER NOT NULL, verified_at TEXT NOT NULL, "
|
||||
"first_seen_at TEXT NOT NULL, tag_revision INTEGER NOT NULL DEFAULT 0 "
|
||||
"CHECK(tag_revision>=0), UNIQUE(file_id, sha256))"
|
||||
),
|
||||
IMAGE_TAGS: (
|
||||
f"CREATE TABLE {IMAGE_TAGS}(byte_id TEXT NOT NULL REFERENCES {BYTES}(id) "
|
||||
"ON DELETE CASCADE, tag_id INTEGER NOT NULL REFERENCES tags(id) "
|
||||
"ON DELETE CASCADE, position INTEGER NOT NULL CHECK(position>=0), "
|
||||
"PRIMARY KEY(byte_id, tag_id))"
|
||||
),
|
||||
}
|
||||
INDEXES = {
|
||||
"idx_media_annotation_bytes_fingerprint": (
|
||||
f"CREATE INDEX idx_media_annotation_bytes_fingerprint ON {BYTES}"
|
||||
"(file_id, size, mtime_ns)"
|
||||
),
|
||||
"idx_media_image_tags_tag": (
|
||||
f"CREATE INDEX idx_media_image_tags_tag ON {IMAGE_TAGS}(tag_id)"
|
||||
),
|
||||
"idx_media_image_tags_position": (
|
||||
f"CREATE UNIQUE INDEX idx_media_image_tags_position ON {IMAGE_TAGS}"
|
||||
"(byte_id, position)"
|
||||
),
|
||||
}
|
||||
AUTOINDEXES = {FILES: 2, BYTES: 2, IMAGE_TAGS: 1}
|
||||
LEGACY_TABLES = (
|
||||
"prompts",
|
||||
"generated_images",
|
||||
"tags",
|
||||
"prompt_tags",
|
||||
"prompt_pair_tags",
|
||||
"image_favorites",
|
||||
)
|
||||
FILE_ID = "0f8e4b2a-1c3d-4e5f-8a9b-0c1d2e3f4a5b"
|
||||
BYTE_ID = "6a1b2c3d-4e5f-4a6b-8c9d-0e1f2a3b4c5d"
|
||||
OTHER_ID = "7b2c3d4e-5f6a-4b7c-9d0e-1f2a3b4c5d6e"
|
||||
DIGEST = hashlib.sha256(b"img").hexdigest()
|
||||
NOW = "2026-10-22T00:00:00.000000+00:00"
|
||||
MAX_LOCATOR = 4096
|
||||
|
||||
|
||||
def _compact(sql):
|
||||
return re.sub(r"\s+", "", sql or "").lower()
|
||||
|
||||
|
||||
def expected_catalog():
|
||||
objects = {("table", name, _compact(sql)) for name, sql in DDL.items()}
|
||||
objects |= {("index", name, _compact(sql)) for name, sql in INDEXES.items()}
|
||||
objects |= {
|
||||
("index", f"sqlite_autoindex_{table}_{n}", "")
|
||||
for table, count in AUTOINDEXES.items()
|
||||
for n in range(1, count + 1)
|
||||
}
|
||||
return objects
|
||||
|
||||
|
||||
def catalog(conn):
|
||||
names = tuple(TABLES) + tuple(INDEXES)
|
||||
marks = ",".join("?" * len(names))
|
||||
table_marks = ",".join("?" * len(TABLES))
|
||||
return {
|
||||
(kind, name, _compact(sql))
|
||||
for kind, name, sql in conn.execute(
|
||||
"SELECT type,name,sql FROM sqlite_master "
|
||||
f"WHERE name IN ({marks}) OR tbl_name IN ({table_marks})", # nosec B608
|
||||
names + tuple(TABLES),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def insert_group_rows(conn, tag_name="img-only", locator="/out/img.png"):
|
||||
"""One file, one byte record and one image tag carrying *tag_name*."""
|
||||
tag_id = conn.execute("INSERT INTO tags(name) VALUES(?)", (tag_name,)).lastrowid
|
||||
conn.execute(f"INSERT INTO {FILES} VALUES(?,?,?)", (FILE_ID, locator, NOW))
|
||||
conn.execute(
|
||||
f"INSERT INTO {BYTES} VALUES(?,?,?,?,?,?,?,?)",
|
||||
(BYTE_ID, FILE_ID, DIGEST, 3, 1_700_000_000_000_000_000, NOW, NOW, 1),
|
||||
)
|
||||
conn.execute(f"INSERT INTO {IMAGE_TAGS} VALUES(?,?,0)", (BYTE_ID, tag_id))
|
||||
return tag_id
|
||||
|
||||
|
||||
def rows(conn, sql, *params):
|
||||
"""Plain tuples whatever row factory *conn* uses."""
|
||||
return [tuple(row) for row in conn.execute(sql, params)]
|
||||
|
||||
|
||||
def image_tags(conn):
|
||||
return rows(
|
||||
conn,
|
||||
f"SELECT t.name, it.position FROM {IMAGE_TAGS} it "
|
||||
"JOIN tags t ON t.id = it.tag_id ORDER BY it.position",
|
||||
)
|
||||
|
||||
|
||||
def metadata(conn):
|
||||
return rows(conn, f"SELECT schema_version,image_index_revision FROM {METADATA}")
|
||||
|
||||
|
||||
def table_rows(conn, tables):
|
||||
found = {}
|
||||
for table in tables:
|
||||
present = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)
|
||||
).fetchone()
|
||||
if present:
|
||||
found[table] = rows(conn, f"SELECT * FROM {table}") # nosec B608
|
||||
return found
|
||||
|
||||
|
||||
def populate_legacy(conn):
|
||||
conn.execute(
|
||||
"INSERT INTO prompts(id,text,hash,category,notes,run_count) VALUES"
|
||||
"(7,'positive','p','keep','notes',12)"
|
||||
)
|
||||
conn.execute("INSERT INTO prompts(id,text,hash) VALUES(9,'negative','n')")
|
||||
conn.execute(
|
||||
"INSERT INTO generated_images(id,prompt_id,image_path,filename,file_path) "
|
||||
"VALUES(4,7,'/tmp/image.png','image.png','/tmp/image.png')"
|
||||
)
|
||||
conn.execute("INSERT INTO tags(id,name) VALUES(3,'keep-tag')")
|
||||
conn.execute("INSERT INTO prompt_tags VALUES(7,3)")
|
||||
conn.execute(
|
||||
"INSERT INTO prompt_pairs(id,positive_prompt_id,negative_prompt_id) "
|
||||
"VALUES(6,7,9)"
|
||||
)
|
||||
conn.execute("INSERT INTO prompt_pair_tags VALUES(6,3)")
|
||||
conn.execute("INSERT INTO image_favorites VALUES('root','a.png',?)", (NOW,))
|
||||
|
||||
|
||||
class AnnotationSchemaTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.addCleanup(assert_closed, self, self.temp.name)
|
||||
self.addCleanup(PromptModel.reset_schema_cache)
|
||||
self.models = []
|
||||
self.addCleanup(self.close_models)
|
||||
self.path = str(Path(self.temp.name) / "live.db")
|
||||
|
||||
def close_models(self):
|
||||
for model in self.models:
|
||||
model.close_all()
|
||||
|
||||
def model(self, path=None):
|
||||
model = PromptModel(path or self.path)
|
||||
self.models.append(model)
|
||||
return model
|
||||
|
||||
def require_module(self):
|
||||
self.assertIsNotNone(
|
||||
annotation_schema_error, "database.annotation_schema is not implemented"
|
||||
)
|
||||
|
||||
def drop_group(self, path):
|
||||
"""Remove the group from a closed database file (an old backup)."""
|
||||
with closing(sqlite3.connect(path)) as raw, raw:
|
||||
raw.execute("PRAGMA foreign_keys=OFF")
|
||||
for table in reversed(TABLES):
|
||||
raw.execute(f"DROP TABLE {table}")
|
||||
|
||||
def test_fresh_install_and_reopen_is_idempotent(self):
|
||||
self.require_module()
|
||||
model = self.model()
|
||||
with model.get_connection() as conn:
|
||||
self.assertEqual(catalog(conn), expected_catalog())
|
||||
self.assertIsNone(annotation_schema_error(conn, allow_missing=False))
|
||||
self.assertEqual(metadata(conn), [(1, 0)])
|
||||
insert_group_rows(conn)
|
||||
self.assertEqual(
|
||||
PromptModel.verify_database_file(self.path, require_library_state=True),
|
||||
(True, "ok"),
|
||||
)
|
||||
model.close_all()
|
||||
for _ in range(2):
|
||||
PromptModel.reset_schema_cache()
|
||||
model = self.model()
|
||||
with model.get_connection() as conn:
|
||||
self.assertEqual(catalog(conn), expected_catalog())
|
||||
self.assertIsNone(annotation_schema_error(conn, allow_missing=False))
|
||||
self.assertEqual(metadata(conn), [(1, 0)])
|
||||
self.assertEqual(image_tags(conn), [("img-only", 0)])
|
||||
# Running the migration again on a current database changes nothing.
|
||||
migrate_annotation_state(conn)
|
||||
self.assertEqual(metadata(conn), [(1, 0)])
|
||||
self.assertEqual(image_tags(conn), [("img-only", 0)])
|
||||
model.close_all()
|
||||
# A tag delete and a file delete cascade into the junction and bytes.
|
||||
PromptModel.reset_schema_cache()
|
||||
model = self.model()
|
||||
with model.get_connection() as conn:
|
||||
conn.execute("DELETE FROM tags WHERE name='img-only'")
|
||||
self.assertEqual(image_tags(conn), [])
|
||||
conn.execute(f"DELETE FROM {FILES}")
|
||||
self.assertEqual(rows(conn, f"SELECT * FROM {BYTES}"), [])
|
||||
self.assertIsNone(annotation_schema_error(conn, allow_missing=False))
|
||||
|
||||
def test_old_database_without_group_migrates_without_touching_legacy_rows(self):
|
||||
self.require_module()
|
||||
model = self.model()
|
||||
with model.get_connection() as conn:
|
||||
populate_legacy(conn)
|
||||
model.close_all()
|
||||
self.drop_group(self.path)
|
||||
with closing(sqlite3.connect(self.path)) as raw:
|
||||
self.assertIsNone(annotation_schema_error(raw))
|
||||
self.assertEqual(
|
||||
annotation_schema_error(raw, allow_missing=False),
|
||||
"Missing image annotation schema",
|
||||
)
|
||||
legacy_rows_before = table_rows(raw, LEGACY_TABLES)
|
||||
self.assertTrue(PromptModel.verify_database_file(self.path)[0])
|
||||
self.assertFalse(
|
||||
PromptModel.verify_database_file(self.path, require_library_state=True)[0]
|
||||
)
|
||||
PromptModel.reset_schema_cache()
|
||||
|
||||
model = self.model()
|
||||
|
||||
with model.get_connection() as conn:
|
||||
self.assertEqual(catalog(conn), expected_catalog())
|
||||
self.assertIsNone(annotation_schema_error(conn, allow_missing=False))
|
||||
self.assertEqual(metadata(conn), [(1, 0)])
|
||||
legacy_rows_after = table_rows(conn, LEGACY_TABLES)
|
||||
# No backfill: existing prompt tags never become image tags.
|
||||
counts = [
|
||||
conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]
|
||||
for table in (FILES, BYTES, IMAGE_TAGS)
|
||||
]
|
||||
self.assertEqual(counts, [0, 0, 0])
|
||||
self.assertEqual(legacy_rows_after, legacy_rows_before)
|
||||
self.assertEqual(
|
||||
PromptModel.verify_database_file(self.path, require_library_state=True),
|
||||
(True, "ok"),
|
||||
)
|
||||
|
||||
def tampered(self, name, statements, *, checks_off=False):
|
||||
"""A closed current database whose group was altered after the fact."""
|
||||
path = str(Path(self.temp.name) / f"{name.replace(' ', '-')}.db")
|
||||
model = self.model(path)
|
||||
with model.get_connection() as conn:
|
||||
populate_legacy(conn)
|
||||
insert_group_rows(conn)
|
||||
model.close_all()
|
||||
with closing(sqlite3.connect(path)) as raw, raw:
|
||||
raw.execute("PRAGMA foreign_keys=OFF")
|
||||
if checks_off:
|
||||
raw.execute("PRAGMA ignore_check_constraints=ON")
|
||||
for statement in statements:
|
||||
if isinstance(statement, tuple):
|
||||
raw.execute(*statement)
|
||||
else:
|
||||
raw.execute(statement)
|
||||
return path
|
||||
|
||||
def assert_rejected_unrepaired(self, path):
|
||||
with closing(sqlite3.connect(path)) as raw:
|
||||
schema_before = raw.execute(
|
||||
"SELECT type,name,tbl_name,sql FROM sqlite_master ORDER BY name"
|
||||
).fetchall()
|
||||
rows_before = table_rows(raw, TABLES)
|
||||
for allow_missing in (False, True):
|
||||
error = annotation_schema_error(raw, allow_missing=allow_missing)
|
||||
self.assertIsNotNone(error)
|
||||
self.assertIn("image annotation", error.lower())
|
||||
self.assertFalse(PromptModel.verify_database_file(path)[0])
|
||||
self.assertFalse(
|
||||
PromptModel.verify_database_file(path, require_library_state=True)[0]
|
||||
)
|
||||
PromptModel.reset_schema_cache()
|
||||
with self.assertRaises(sqlite3.DatabaseError):
|
||||
self.model(path)
|
||||
PromptModel.reset_schema_cache()
|
||||
with closing(sqlite3.connect(path)) as raw:
|
||||
self.assertEqual(
|
||||
raw.execute(
|
||||
"SELECT type,name,tbl_name,sql FROM sqlite_master ORDER BY name"
|
||||
).fetchall(),
|
||||
schema_before,
|
||||
)
|
||||
self.assertEqual(table_rows(raw, TABLES), rows_before)
|
||||
|
||||
def test_partial_or_malformed_group_is_rejected_without_repair(self):
|
||||
self.require_module()
|
||||
cases = {
|
||||
"missing metadata": (f"DROP TABLE {METADATA}",),
|
||||
"missing files": (f"DROP TABLE {FILES}",),
|
||||
"missing bytes": (f"DROP TABLE {BYTES}",),
|
||||
"missing image tags": (f"DROP TABLE {IMAGE_TAGS}",),
|
||||
"reserved view": (
|
||||
f"DROP TABLE {FILES}",
|
||||
f"CREATE VIEW {FILES} AS SELECT 'x' AS id",
|
||||
),
|
||||
"extra trigger": (
|
||||
f"CREATE TRIGGER media_annotation_trigger AFTER INSERT ON {FILES} "
|
||||
"BEGIN SELECT 1; END",
|
||||
),
|
||||
"extra index": (
|
||||
f"CREATE INDEX idx_media_annotation_extra ON {FILES}(created_at)",
|
||||
),
|
||||
"extra column": (
|
||||
f"DROP TABLE {IMAGE_TAGS}",
|
||||
DDL[IMAGE_TAGS].replace(
|
||||
"position INTEGER", "note TEXT, position INTEGER"
|
||||
),
|
||||
),
|
||||
"wrong primary key": (
|
||||
f"DROP TABLE {FILES}",
|
||||
DDL[FILES].replace("id TEXT PRIMARY KEY", "id TEXT"),
|
||||
),
|
||||
"wrong foreign key": (
|
||||
f"DROP TABLE {BYTES}",
|
||||
DDL[BYTES].replace("ON DELETE CASCADE", "ON DELETE RESTRICT"),
|
||||
INDEXES["idx_media_annotation_bytes_fingerprint"],
|
||||
),
|
||||
"wrong check": (
|
||||
f"DROP TABLE {METADATA}",
|
||||
DDL[METADATA].replace("schema_version=1", "schema_version=2"),
|
||||
f"INSERT INTO {METADATA} VALUES(1,2,0)",
|
||||
),
|
||||
"missing unique": (
|
||||
f"DROP TABLE {FILES}",
|
||||
DDL[FILES].replace("NOT NULL UNIQUE", "NOT NULL"),
|
||||
),
|
||||
"missing index": ("DROP INDEX idx_media_image_tags_tag",),
|
||||
"non-unique position index": (
|
||||
"DROP INDEX idx_media_image_tags_position",
|
||||
INDEXES["idx_media_image_tags_position"].replace("UNIQUE ", ""),
|
||||
),
|
||||
"wrong index columns": (
|
||||
"DROP INDEX idx_media_annotation_bytes_fingerprint",
|
||||
INDEXES["idx_media_annotation_bytes_fingerprint"].replace(
|
||||
", mtime_ns", ""
|
||||
),
|
||||
),
|
||||
"orphan tag": (f"INSERT INTO {IMAGE_TAGS} VALUES('{BYTE_ID}',404,1)",),
|
||||
"orphan byte": (
|
||||
(
|
||||
f"INSERT INTO {BYTES} VALUES(?,?,?,1,1,?,?,0)",
|
||||
(
|
||||
OTHER_ID,
|
||||
"9" * 8 + "-0000-4000-8000-" + "0" * 12,
|
||||
DIGEST,
|
||||
NOW,
|
||||
NOW,
|
||||
),
|
||||
),
|
||||
),
|
||||
"bad file uuid": (
|
||||
(f"UPDATE {FILES} SET id=?", ("x" * 36,)),
|
||||
(f"UPDATE {BYTES} SET file_id=?", ("x" * 36,)),
|
||||
),
|
||||
"uppercase byte uuid": (
|
||||
f"UPDATE {BYTES} SET id=upper(id)",
|
||||
f"UPDATE {IMAGE_TAGS} SET byte_id=upper(byte_id)",
|
||||
),
|
||||
"null file id": ((f"INSERT INTO {FILES} VALUES(NULL,?,?)", ("/n", NOW)),),
|
||||
"blob ids": (
|
||||
f"UPDATE {FILES} SET id=CAST(id AS BLOB)",
|
||||
f"UPDATE {BYTES} SET file_id=CAST(file_id AS BLOB)",
|
||||
),
|
||||
"no metadata row": (f"DELETE FROM {METADATA}",),
|
||||
"long created_at": ((f"UPDATE {FILES} SET created_at=?", ("x" * 65,)),),
|
||||
"blob verified_at": (f"UPDATE {BYTES} SET verified_at=X'00'",),
|
||||
"text mtime": (f"UPDATE {BYTES} SET mtime_ns='soon'",),
|
||||
"duplicate position": (
|
||||
"DROP INDEX idx_media_image_tags_position",
|
||||
"INSERT INTO tags(id,name) VALUES(77,'dup')",
|
||||
f"INSERT INTO {IMAGE_TAGS} VALUES('{BYTE_ID}',77,0)",
|
||||
),
|
||||
"long locator": (
|
||||
(f"UPDATE {FILES} SET locator=?", ("x" * (MAX_LOCATOR + 1),)),
|
||||
),
|
||||
"multibyte long locator": (
|
||||
(f"UPDATE {FILES} SET locator=?", ("é" * (MAX_LOCATOR + 1),)),
|
||||
),
|
||||
"nul locator bypass": (
|
||||
(f"UPDATE {FILES} SET locator=?", ("x\0" + "y" * MAX_LOCATOR,)),
|
||||
),
|
||||
}
|
||||
for label, statements in cases.items():
|
||||
with self.subTest(label):
|
||||
self.assert_rejected_unrepaired(self.tampered(label, statements))
|
||||
unchecked = {
|
||||
"two metadata rows": (f"INSERT INTO {METADATA} VALUES(2,1,0)",),
|
||||
"wrong schema version": (f"UPDATE {METADATA} SET schema_version=2",),
|
||||
"negative index revision": (
|
||||
f"UPDATE {METADATA} SET image_index_revision=-1",
|
||||
),
|
||||
"bad digest": (f"UPDATE {BYTES} SET sha256=upper(sha256)",),
|
||||
"short digest": (f"UPDATE {BYTES} SET sha256=substr(sha256,1,63)",),
|
||||
"negative size": (f"UPDATE {BYTES} SET size=-1",),
|
||||
"negative tag revision": (f"UPDATE {BYTES} SET tag_revision=-1",),
|
||||
"negative position": (f"UPDATE {IMAGE_TAGS} SET position=-1",),
|
||||
}
|
||||
for label, statements in unchecked.items():
|
||||
with self.subTest(label):
|
||||
path = self.tampered(label, statements, checks_off=True)
|
||||
self.assert_rejected_unrepaired(path)
|
||||
accepted = {
|
||||
"locator at bound": ("x" * MAX_LOCATOR,),
|
||||
"multibyte locator at bound": ("é" * MAX_LOCATOR,),
|
||||
}
|
||||
for label, (locator,) in accepted.items():
|
||||
with self.subTest(label):
|
||||
path = self.tampered(
|
||||
label, ((f"UPDATE {FILES} SET locator=?", (locator,)),)
|
||||
)
|
||||
with closing(sqlite3.connect(path)) as raw:
|
||||
self.assertIsNone(annotation_schema_error(raw, allow_missing=False))
|
||||
self.assertEqual(
|
||||
PromptModel.verify_database_file(path, require_library_state=True),
|
||||
(True, "ok"),
|
||||
)
|
||||
|
||||
def test_install_failure_rolls_back_and_retries(self):
|
||||
self.require_module()
|
||||
with patch(
|
||||
"database.annotation_schema.annotation_schema_error",
|
||||
side_effect=[None, "injected final validation failure"],
|
||||
):
|
||||
with self.assertRaisesRegex(sqlite3.DatabaseError, "injected"):
|
||||
self.model()
|
||||
with closing(sqlite3.connect(self.path)) as raw:
|
||||
self.assertEqual(catalog(raw), set())
|
||||
self.assertIsNotNone(
|
||||
raw.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE name='saved_views'"
|
||||
).fetchone()
|
||||
)
|
||||
PromptModel.reset_schema_cache()
|
||||
model = self.model()
|
||||
self.assertEqual(
|
||||
PromptModel.verify_database_file(model.db_path, require_library_state=True),
|
||||
(True, "ok"),
|
||||
)
|
||||
with model.get_connection() as conn:
|
||||
self.assertEqual(metadata(conn), [(1, 0)])
|
||||
|
||||
def test_wal_backup_restore_round_trips_group(self):
|
||||
self.require_module()
|
||||
source_path = str(Path(self.temp.name) / "source.db")
|
||||
source = self.model(source_path)
|
||||
with source.get_connection() as conn:
|
||||
conn.execute("PRAGMA wal_autocheckpoint=0")
|
||||
conn.execute("INSERT INTO prompts(id,text,hash) VALUES(7,'p','p')")
|
||||
insert_group_rows(conn)
|
||||
conn.commit()
|
||||
source_files = rows(conn, f"SELECT * FROM {FILES}")
|
||||
source_bytes = rows(conn, f"SELECT * FROM {BYTES}")
|
||||
self.assertGreater(Path(source_path + "-wal").stat().st_size, 0)
|
||||
live = self.model()
|
||||
with live.get_connection() as conn:
|
||||
conn.execute("INSERT INTO prompts(id,text,hash) VALUES(1,'live','live')")
|
||||
insert_group_rows(conn, tag_name="live-only", locator="/out/live.png")
|
||||
source_before = Path(source_path).read_bytes()
|
||||
self.assertEqual(
|
||||
PromptModel.verify_database_file(source_path, require_library_state=True),
|
||||
(True, "ok"),
|
||||
)
|
||||
# The rows still sit in the WAL; a backup file must carry them too.
|
||||
backup_file = str(Path(self.temp.name) / "backup.db")
|
||||
self.assertTrue(source.backup_database(backup_file))
|
||||
with closing(sqlite3.connect(backup_file)) as raw:
|
||||
self.assertEqual(image_tags(raw), [("img-only", 0)])
|
||||
self.assertIsNone(annotation_schema_error(raw, allow_missing=False))
|
||||
|
||||
safety = live.restore_from_file(source_path)
|
||||
|
||||
self.assertTrue(Path(safety).is_file())
|
||||
self.assertEqual(Path(source_path).read_bytes(), source_before)
|
||||
with live.get_connection() as conn:
|
||||
restored_image_tags = image_tags(conn)
|
||||
self.assertEqual(rows(conn, f"SELECT * FROM {FILES}"), source_files)
|
||||
self.assertEqual(rows(conn, f"SELECT * FROM {BYTES}"), source_bytes)
|
||||
self.assertEqual(metadata(conn), [(1, 0)])
|
||||
self.assertIsNone(annotation_schema_error(conn, allow_missing=False))
|
||||
self.assertEqual(rows(conn, "SELECT id FROM prompts"), [(7,)])
|
||||
self.assertEqual(restored_image_tags, [("img-only", 0)])
|
||||
with closing(sqlite3.connect(safety)) as raw:
|
||||
self.assertEqual(image_tags(raw), [("live-only", 0)])
|
||||
self.assertEqual(raw.execute("SELECT id FROM prompts").fetchall(), [(1,)])
|
||||
# The same rows restore from the backup file as well.
|
||||
PromptModel.reset_schema_cache()
|
||||
live.restore_from_file(backup_file)
|
||||
with live.get_connection() as conn:
|
||||
self.assertEqual(image_tags(conn), [("img-only", 0)])
|
||||
PromptModel.close_all_instances()
|
||||
assert_closed(self, self.temp.name)
|
||||
|
||||
def test_staged_failure_preserves_source_live_safety_and_handles(self):
|
||||
self.require_module()
|
||||
live = self.model()
|
||||
with live.get_connection() as conn:
|
||||
conn.execute("INSERT INTO prompts(id,text,hash) VALUES(1,'live','live')")
|
||||
insert_group_rows(conn, tag_name="live-only")
|
||||
# A source with a malformed group is refused before any change.
|
||||
bad = self.tampered(
|
||||
"restore bad",
|
||||
(f"UPDATE {BYTES} SET sha256=upper(sha256)",),
|
||||
checks_off=True,
|
||||
)
|
||||
# A staging failure while installing the group must be refused too.
|
||||
old = str(Path(self.temp.name) / "old.db")
|
||||
self.model(old).close_all()
|
||||
self.drop_group(old)
|
||||
for source, patched in ((bad, False), (old, True)):
|
||||
with self.subTest(source=Path(source).name):
|
||||
before = Path(source).read_bytes()
|
||||
safety = sorted(Path(self.temp.name).glob("live.db.backup_*"))
|
||||
if patched:
|
||||
context = patch(
|
||||
"database.models.migrate_annotation_state",
|
||||
side_effect=sqlite3.OperationalError("staging failed"),
|
||||
)
|
||||
expected = sqlite3.OperationalError
|
||||
else:
|
||||
context = patch.object(PromptModel, "backup_database")
|
||||
expected = ValueError
|
||||
with context as hook, self.assertRaises(expected):
|
||||
live.restore_from_file(source)
|
||||
if not patched:
|
||||
hook.assert_not_called()
|
||||
self.assertEqual(Path(source).read_bytes(), before)
|
||||
self.assertEqual(
|
||||
sorted(Path(self.temp.name).glob("live.db.backup_*")), safety
|
||||
)
|
||||
assert_closed(self, source)
|
||||
with live.get_connection() as conn:
|
||||
self.assertEqual(image_tags(conn), [("live-only", 0)])
|
||||
self.assertEqual(metadata(conn), [(1, 0)])
|
||||
self.assertFalse(
|
||||
PromptModel.verify_database_file(old, require_library_state=True)[0]
|
||||
)
|
||||
self.assertTrue(PromptModel.verify_database_file(old)[0])
|
||||
|
||||
def test_verify_requires_group_only_with_require_library_state(self):
|
||||
self.require_module()
|
||||
self.model().close_all()
|
||||
self.assertEqual(
|
||||
PromptModel.verify_database_file(self.path, require_library_state=True),
|
||||
(True, "ok"),
|
||||
)
|
||||
self.drop_group(self.path)
|
||||
self.assertEqual(PromptModel.verify_database_file(self.path), (True, "ok"))
|
||||
self.assertEqual(
|
||||
PromptModel.verify_database_file(self.path, require_library_state=True),
|
||||
(False, "Missing image annotation schema"),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,598 @@
|
||||
"""Per-image tag sets (#249 A2): byte-exact storage in the shared vocabulary."""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
try:
|
||||
from tests.open_handles import assert_closed
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from open_handles import assert_closed
|
||||
|
||||
from database.annotation_schema import annotation_schema_error
|
||||
from database.library_commands import LibraryCommandError
|
||||
from database.operations import PromptDatabase
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
try: # guarded so a missing module fails assertions, not test loading
|
||||
from database import annotations as annotations_module
|
||||
from database.annotations import (
|
||||
MAX_IMAGE_TAGS,
|
||||
AnnotationCommandError,
|
||||
normalize_image_tags,
|
||||
)
|
||||
except ImportError:
|
||||
annotations_module = None
|
||||
MAX_IMAGE_TAGS = None
|
||||
AnnotationCommandError = None
|
||||
normalize_image_tags = None
|
||||
|
||||
UUID = re.compile(r"[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}")
|
||||
DIGEST_A = hashlib.sha256(b"bytes a").hexdigest()
|
||||
DIGEST_B = hashlib.sha256(b"bytes b").hexdigest()
|
||||
SIZE = 7
|
||||
MTIME = 1_700_000_000_123_456_789
|
||||
ANNOTATION_TABLES = (
|
||||
"media_annotation_files",
|
||||
"media_annotation_bytes",
|
||||
"media_image_tags",
|
||||
"media_annotation_metadata",
|
||||
)
|
||||
LEGACY_TABLES = (
|
||||
"tags",
|
||||
"prompt_tags",
|
||||
"prompt_pair_tags",
|
||||
"generated_images",
|
||||
"image_prompt_pairs",
|
||||
)
|
||||
|
||||
|
||||
class AnnotationTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.assertIsNotNone(
|
||||
annotations_module, "database.annotations is not implemented"
|
||||
)
|
||||
directory = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(directory.cleanup)
|
||||
self.addCleanup(assert_closed, self, directory.name)
|
||||
self.root = directory.name
|
||||
self.db = PromptDatabase(os.path.join(self.root, "annotations.db"))
|
||||
self.addCleanup(self.db.close_all)
|
||||
self.epoch = self.db.get_execution_epoch()
|
||||
|
||||
def locator(self, name: str) -> str:
|
||||
return os.path.normcase(os.path.join(os.path.realpath(self.root), name))
|
||||
|
||||
def replace(self, loc, digest=DIGEST_A, tags=(), *, revision=0, **over):
|
||||
options = {
|
||||
"locator": loc,
|
||||
"sha256": digest,
|
||||
"size": SIZE,
|
||||
"mtime_ns": MTIME,
|
||||
"expected_epoch": self.epoch,
|
||||
"expected_tag_revision": revision,
|
||||
"tags": list(tags) if isinstance(tags, tuple) else tags,
|
||||
}
|
||||
options.update(over)
|
||||
return self.db.replace_image_tags(**options)
|
||||
|
||||
def add(self, loc, digest=DIGEST_A, tags=(), **over):
|
||||
options = {
|
||||
"locator": loc,
|
||||
"sha256": digest,
|
||||
"size": SIZE,
|
||||
"mtime_ns": MTIME,
|
||||
"expected_epoch": self.epoch,
|
||||
"tags": list(tags) if isinstance(tags, tuple) else tags,
|
||||
}
|
||||
options.update(over)
|
||||
return self.db.add_image_tags(**options)
|
||||
|
||||
def read(self, loc, digest=DIGEST_A):
|
||||
return self.db.read_image_tags(loc, digest)
|
||||
|
||||
def rows(self, *tables):
|
||||
"""Every row of each table; ``verified_at`` is a fingerprint, not content."""
|
||||
conn = self.db.model.get_connection()
|
||||
result = []
|
||||
for table in tables:
|
||||
columns = [
|
||||
row[1]
|
||||
for row in conn.execute(f"PRAGMA table_info({table})")
|
||||
if row[1] != "verified_at"
|
||||
]
|
||||
result.append(
|
||||
[
|
||||
tuple(row)
|
||||
for row in conn.execute(
|
||||
f"SELECT {','.join(columns)} FROM {table} ORDER BY 1,2"
|
||||
)
|
||||
]
|
||||
)
|
||||
return result
|
||||
|
||||
def counts(self):
|
||||
conn = self.db.model.get_connection()
|
||||
return {
|
||||
table: conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]
|
||||
for table in ANNOTATION_TABLES
|
||||
}
|
||||
|
||||
def index_revision(self):
|
||||
return (
|
||||
self.db.model.get_connection()
|
||||
.execute("SELECT image_index_revision FROM media_annotation_metadata")
|
||||
.fetchone()[0]
|
||||
)
|
||||
|
||||
def schema_clean(self):
|
||||
self.assertIsNone(
|
||||
annotation_schema_error(self.db.model.get_connection(), allow_missing=False)
|
||||
)
|
||||
|
||||
|
||||
class NormalizeImageTagsTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.assertIsNotNone(
|
||||
normalize_image_tags, "database.annotations is not implemented"
|
||||
)
|
||||
|
||||
def test_case_insensitive_dedupe_keeps_first_spelling_and_position(self):
|
||||
self.assertEqual(
|
||||
normalize_image_tags([" Forest ", "mist", "forest", "MIST", "Rain"]),
|
||||
["Forest", "mist", "Rain"],
|
||||
)
|
||||
self.assertEqual(normalize_image_tags([]), [])
|
||||
self.assertEqual(MAX_IMAGE_TAGS, 100)
|
||||
|
||||
def test_limit_and_invalid_names_rejected(self):
|
||||
for bad in (
|
||||
None,
|
||||
"forest",
|
||||
{"a": 1},
|
||||
[1],
|
||||
[None],
|
||||
["a,b"],
|
||||
[""],
|
||||
[" "],
|
||||
["x" * 101],
|
||||
[f"t{i}" for i in range(MAX_IMAGE_TAGS + 1)],
|
||||
):
|
||||
with self.subTest(bad=bad), self.assertRaises(ValueError):
|
||||
normalize_image_tags(bad)
|
||||
# Duplicates fold before the cap is applied.
|
||||
many = [f"t{i}" for i in range(MAX_IMAGE_TAGS)]
|
||||
self.assertEqual(normalize_image_tags(many + ["T0", "T99"]), many)
|
||||
|
||||
|
||||
class ReplaceImageTagsTests(AnnotationTestCase):
|
||||
def test_first_save_creates_owner_and_byte_atomically_including_empty_set(self):
|
||||
loc = self.locator("a.png")
|
||||
receipt = self.replace(loc, tags=["Forest", "mist", "forest"])
|
||||
self.assertEqual(receipt["tags"], ["Forest", "mist"])
|
||||
self.assertEqual(receipt["tag_revision"], 1)
|
||||
self.assertEqual(
|
||||
receipt,
|
||||
{
|
||||
"library_epoch": self.epoch,
|
||||
"sha256": DIGEST_A,
|
||||
"tags": ["Forest", "mist"],
|
||||
"tag_revision": 1,
|
||||
"replayed": False,
|
||||
},
|
||||
)
|
||||
conn = self.db.model.get_connection()
|
||||
files = conn.execute("SELECT * FROM media_annotation_files").fetchall()
|
||||
self.assertEqual(len(files), 1)
|
||||
self.assertRegex(files[0]["id"], UUID)
|
||||
self.assertEqual(files[0]["locator"], loc)
|
||||
byte = conn.execute("SELECT * FROM media_annotation_bytes").fetchone()
|
||||
self.assertRegex(byte["id"], UUID)
|
||||
self.assertEqual(
|
||||
(byte["file_id"], byte["sha256"], byte["size"], byte["mtime_ns"]),
|
||||
(files[0]["id"], DIGEST_A, SIZE, MTIME),
|
||||
)
|
||||
self.assertEqual(byte["tag_revision"], 1)
|
||||
self.assertEqual(byte["verified_at"], byte["first_seen_at"])
|
||||
self.assertEqual(
|
||||
[
|
||||
tuple(row)
|
||||
for row in conn.execute(
|
||||
"SELECT t.name, mit.position FROM media_image_tags mit "
|
||||
"JOIN tags t ON t.id = mit.tag_id ORDER BY mit.position"
|
||||
)
|
||||
],
|
||||
[("Forest", 0), ("mist", 1)],
|
||||
)
|
||||
self.assertEqual(self.index_revision(), 1)
|
||||
# An explicit empty set is a real save: owner and byte record exist.
|
||||
empty = self.replace(self.locator("b.png"), DIGEST_B, [])
|
||||
self.assertEqual((empty["tags"], empty["tag_revision"]), ([], 1))
|
||||
self.assertEqual(self.counts()["media_annotation_files"], 2)
|
||||
self.assertEqual(self.counts()["media_annotation_bytes"], 2)
|
||||
self.assertEqual(
|
||||
self.db.read_image_tags(self.locator("b.png"), DIGEST_B),
|
||||
{"tags": [], "tag_revision": 1, "prior_count": 0},
|
||||
)
|
||||
self.assertEqual(self.index_revision(), 2)
|
||||
self.schema_clean()
|
||||
|
||||
def test_save_changes_only_target_byte_set(self):
|
||||
a, b, c = self.locator("a.png"), self.locator("b.png"), self.locator("c.png")
|
||||
prompt = self.db.save_prompt(
|
||||
"Shared forest",
|
||||
tags=["shared"],
|
||||
prompt_hash=generate_prompt_hash("Shared forest"),
|
||||
)
|
||||
pair = self.db.save_prompt_pair("Shared forest", "blurry", tags=["pairtag"])
|
||||
image_a = self.db.link_image_to_prompt(prompt, a)
|
||||
image_b = self.db.link_image_to_prompt(prompt, b)
|
||||
self.db.link_image_to_pair(pair, image_a, {"seed": 1})
|
||||
self.db.link_image_to_pair(pair, image_b, {"seed": 2})
|
||||
self.replace(b, DIGEST_A, ["sibling"])
|
||||
self.replace(c, DIGEST_A, ["copy"]) # equal bytes at another path
|
||||
before = self.rows(*LEGACY_TABLES)
|
||||
sibling_before = self.db.read_image_tags(b, DIGEST_A)
|
||||
copy_before = self.db.read_image_tags(c, DIGEST_A)
|
||||
|
||||
receipt = self.replace(a, DIGEST_A, ["only a"])
|
||||
|
||||
self.assertEqual((receipt["tags"], receipt["tag_revision"]), (["only a"], 1))
|
||||
after = self.rows(*LEGACY_TABLES)
|
||||
# The registry gained the new name; nothing else moved.
|
||||
self.assertEqual(after[1:], before[1:])
|
||||
self.assertEqual([row for row in after[0] if row[1] != "only a"], before[0])
|
||||
self.assertEqual(self.db.read_image_tags(b, DIGEST_A), sibling_before)
|
||||
self.assertEqual(self.db.read_image_tags(c, DIGEST_A), copy_before)
|
||||
self.assertEqual(self.db.get_prompt_by_id(prompt)["tags"], ["shared"])
|
||||
self.assertEqual(self.db.get_prompt_pair(pair)["tags"], ["pairtag"])
|
||||
|
||||
def test_changed_bytes_at_same_locator_start_empty_and_keep_prior(self):
|
||||
loc = self.locator("a.png")
|
||||
self.replace(loc, DIGEST_A, ["old"])
|
||||
self.assertEqual(
|
||||
self.db.read_image_tags(loc, DIGEST_B),
|
||||
{"tags": [], "tag_revision": 0, "prior_count": 1},
|
||||
)
|
||||
receipt = self.replace(loc, DIGEST_B, ["new"], mtime_ns=MTIME + 5)
|
||||
self.assertEqual((receipt["tags"], receipt["tag_revision"]), (["new"], 1))
|
||||
self.assertEqual(
|
||||
self.db.read_image_tags(loc, DIGEST_A),
|
||||
{"tags": ["old"], "tag_revision": 1, "prior_count": 1},
|
||||
)
|
||||
self.assertEqual(
|
||||
self.db.read_image_tags(loc, DIGEST_B),
|
||||
{"tags": ["new"], "tag_revision": 1, "prior_count": 1},
|
||||
)
|
||||
self.assertEqual(self.counts()["media_annotation_files"], 1)
|
||||
self.assertEqual(self.counts()["media_annotation_bytes"], 2)
|
||||
self.assertEqual(
|
||||
self.db.read_image_tags(self.locator("unseen.png"), DIGEST_A),
|
||||
{"tags": [], "tag_revision": 0, "prior_count": 0},
|
||||
)
|
||||
|
||||
def test_revision_conflict_returns_current_set(self):
|
||||
loc = self.locator("a.png")
|
||||
self.replace(loc, tags=["a"])
|
||||
before = self.rows(*ANNOTATION_TABLES, "tags")
|
||||
with self.assertRaises(AnnotationCommandError) as ctx:
|
||||
self.replace(loc, tags=["b"], revision=0)
|
||||
self.assertEqual(
|
||||
(ctx.exception.code, ctx.exception.current["tag_revision"]),
|
||||
("tag-conflict", 1),
|
||||
)
|
||||
self.assertIsInstance(ctx.exception, LibraryCommandError)
|
||||
self.assertEqual(ctx.exception.epoch, self.epoch)
|
||||
self.assertEqual(ctx.exception.current["tags"], ["a"])
|
||||
self.assertEqual(ctx.exception.current["sha256"], DIGEST_A)
|
||||
with self.assertRaises(AnnotationCommandError) as ahead:
|
||||
self.replace(loc, tags=["a"], revision=5)
|
||||
self.assertEqual(ahead.exception.code, "tag-conflict")
|
||||
self.assertEqual(self.rows(*ANNOTATION_TABLES, "tags"), before)
|
||||
self.assertFalse(self.db.model.get_connection().in_transaction)
|
||||
|
||||
def test_identical_replay_after_commit_is_success_replayed(self):
|
||||
loc = self.locator("a.png")
|
||||
first = self.replace(loc, tags=["a", "b"])
|
||||
self.assertEqual((first["tag_revision"], first["replayed"]), (1, False))
|
||||
before = self.rows(*ANNOTATION_TABLES)
|
||||
# The lost-response retry: same set, the revision it was sent against.
|
||||
retry = self.replace(loc, tags=["a", "b"], revision=0)
|
||||
self.assertEqual(retry, {**first, "replayed": True})
|
||||
# Saving the current set at the current revision writes nothing either.
|
||||
again = self.replace(loc, tags=["a", "b"], revision=1)
|
||||
self.assertEqual(again, {**first, "replayed": True})
|
||||
self.assertEqual(self.rows(*ANNOTATION_TABLES), before)
|
||||
# Order is part of the set.
|
||||
reordered = self.replace(loc, tags=["b", "a"], revision=1)
|
||||
self.assertEqual(
|
||||
(reordered["tags"], reordered["tag_revision"], reordered["replayed"]),
|
||||
(["b", "a"], 2, False),
|
||||
)
|
||||
self.assertEqual(self.index_revision(), 2)
|
||||
|
||||
def test_replay_with_new_fingerprint_moves_only_the_index_revision(self):
|
||||
loc = self.locator("a.png")
|
||||
first = self.replace(loc, tags=["a"])
|
||||
index = self.index_revision()
|
||||
same = self.replace(loc, tags=["a"], revision=1)
|
||||
self.assertEqual(same, {**first, "replayed": True})
|
||||
self.assertEqual(self.index_revision(), index) # identical: nothing moved
|
||||
touched = self.replace(loc, tags=["a"], revision=1, mtime_ns=MTIME + 5)
|
||||
self.assertEqual(touched, {**first, "replayed": True})
|
||||
self.assertEqual(self.index_revision(), index + 1) # currency changed
|
||||
self.assertEqual(
|
||||
self.db.current_image_tags({loc: (SIZE, MTIME + 5)}), {loc: ["a"]}
|
||||
)
|
||||
self.assertEqual(self.db.current_image_tags({loc: (SIZE, MTIME)}), {})
|
||||
self.assertEqual(self.read(loc)["tag_revision"], 1)
|
||||
|
||||
def test_epoch_checked_inside_transaction(self):
|
||||
loc = self.locator("a.png")
|
||||
seen = []
|
||||
original = PromptDatabase._library_epoch
|
||||
|
||||
def spy(db, conn, expected_epoch):
|
||||
seen.append(conn.in_transaction)
|
||||
return original(db, conn, expected_epoch)
|
||||
|
||||
with patch.object(PromptDatabase, "_library_epoch", spy):
|
||||
self.replace(loc, tags=["a"])
|
||||
self.add(loc, tags=["b"])
|
||||
self.assertEqual(seen, [True, True])
|
||||
stale = "11111111-2222-4333-8444-555555555555"
|
||||
for call in (
|
||||
lambda: self.replace(
|
||||
self.locator("x.png"), tags=["a"], expected_epoch=stale
|
||||
),
|
||||
lambda: self.add(self.locator("x.png"), tags=["a"], expected_epoch=stale),
|
||||
):
|
||||
with self.assertRaises(LibraryCommandError) as ctx:
|
||||
call()
|
||||
self.assertEqual(ctx.exception.code, "storage-changed")
|
||||
self.assertEqual(ctx.exception.epoch, self.epoch)
|
||||
self.assertEqual(self.counts()["media_annotation_files"], 1)
|
||||
for bad_epoch in (None, 7, "not-a-uuid"):
|
||||
with self.assertRaises(ValueError):
|
||||
self.replace(loc, tags=["a"], expected_epoch=bad_epoch)
|
||||
|
||||
def test_limit_and_invalid_names_rejected_without_partial_write(self):
|
||||
loc = self.locator("a.png")
|
||||
before = self.rows(*ANNOTATION_TABLES, "tags")
|
||||
for tags in (
|
||||
["ok", "a,b"],
|
||||
["ok", ""],
|
||||
["ok", 3],
|
||||
"ok",
|
||||
[f"t{i}" for i in range(101)],
|
||||
):
|
||||
with self.subTest(tags=tags), self.assertRaises(ValueError):
|
||||
self.replace(loc, tags=tags)
|
||||
with self.subTest(tags=tags, command="add"), self.assertRaises(ValueError):
|
||||
self.add(loc, tags=tags)
|
||||
for field, value in (
|
||||
("locator", ""),
|
||||
("locator", None),
|
||||
("locator", "x" * 4097),
|
||||
("sha256", DIGEST_A.upper()),
|
||||
("sha256", DIGEST_A[:-1]),
|
||||
("sha256", None),
|
||||
("size", -1),
|
||||
("size", True),
|
||||
("size", "7"),
|
||||
("mtime_ns", 1.5),
|
||||
("mtime_ns", None),
|
||||
("expected_tag_revision", -1),
|
||||
("expected_tag_revision", False),
|
||||
):
|
||||
with self.subTest(field=field, value=value), self.assertRaises(ValueError):
|
||||
self.replace(loc, tags=["ok"], **{field: value})
|
||||
self.assertEqual(self.rows(*ANNOTATION_TABLES, "tags"), before)
|
||||
self.assertEqual(self.index_revision(), 0)
|
||||
|
||||
def test_writers_supply_uuid_ids_and_refuse_to_write_without_one(self):
|
||||
loc = self.locator("a.png")
|
||||
before = self.rows(*ANNOTATION_TABLES)
|
||||
with patch.object(annotations_module, "uuid4", return_value=""):
|
||||
with self.assertRaises(ValueError):
|
||||
self.replace(loc, tags=["a"])
|
||||
self.assertEqual(self.rows(*ANNOTATION_TABLES), before)
|
||||
self.replace(loc, tags=["a"])
|
||||
self.add(self.locator("b.png"), DIGEST_B, ["b"])
|
||||
conn = self.db.model.get_connection()
|
||||
for table in ("media_annotation_files", "media_annotation_bytes"):
|
||||
for (value,) in conn.execute(f"SELECT id FROM {table}"):
|
||||
self.assertRegex(value, UUID)
|
||||
self.schema_clean()
|
||||
|
||||
def test_never_resolves_owner_through_prompt_lookup(self):
|
||||
loc = self.locator("a.png")
|
||||
prompt = self.db.save_prompt("Linked", tags=["prompt-tag"])
|
||||
self.db.link_image_to_prompt(prompt, loc)
|
||||
with patch.object(
|
||||
PromptDatabase,
|
||||
"get_prompt_id_for_image",
|
||||
side_effect=AssertionError("prompt lookup is not an owner"),
|
||||
):
|
||||
self.replace(loc, tags=["img"])
|
||||
self.add(loc, tags=["more"])
|
||||
self.assertEqual(
|
||||
self.db.read_image_tags(loc, DIGEST_A)["tags"], ["img", "more"]
|
||||
)
|
||||
self.assertEqual(
|
||||
self.db.current_image_tags({loc: (SIZE, MTIME)}),
|
||||
{loc: ["img", "more"]},
|
||||
)
|
||||
self.assertTrue(
|
||||
self.db.refresh_image_fingerprint(loc, DIGEST_A, SIZE, MTIME + 1)
|
||||
)
|
||||
list(self.db.iter_image_tag_records())
|
||||
self.assertEqual(self.db.get_prompt_by_id(prompt)["tags"], ["prompt-tag"])
|
||||
|
||||
|
||||
class AddImageTagsTests(AnnotationTestCase):
|
||||
def test_add_merges_without_removing_and_caps_at_100(self):
|
||||
loc = self.locator("a.png")
|
||||
self.replace(loc, tags=["Forest", "mist"])
|
||||
receipt = self.add(loc, tags=["forest", "Rain", "rain"])
|
||||
self.assertEqual(
|
||||
receipt,
|
||||
{
|
||||
"library_epoch": self.epoch,
|
||||
"sha256": DIGEST_A,
|
||||
"tags": ["Forest", "mist", "Rain"],
|
||||
"tag_revision": 2,
|
||||
"replayed": False,
|
||||
"added": 1,
|
||||
},
|
||||
)
|
||||
nothing = self.add(loc, tags=["MIST"])
|
||||
self.assertEqual(
|
||||
(nothing["tags"], nothing["tag_revision"], nothing["replayed"]),
|
||||
(["Forest", "mist", "Rain"], 2, True),
|
||||
)
|
||||
self.assertEqual(nothing["added"], 0)
|
||||
before = self.rows(*ANNOTATION_TABLES, "tags")
|
||||
with self.assertRaises(AnnotationCommandError) as ctx:
|
||||
self.add(loc, tags=[f"t{i}" for i in range(98)])
|
||||
self.assertEqual(ctx.exception.code, "tag-limit")
|
||||
self.assertEqual(ctx.exception.current["tag_revision"], 2)
|
||||
self.assertEqual(self.rows(*ANNOTATION_TABLES, "tags"), before)
|
||||
exact = self.add(loc, tags=[f"t{i}" for i in range(97)])
|
||||
self.assertEqual((len(exact["tags"]), exact["added"]), (100, 97))
|
||||
# An unseen file: add creates the owner and byte record like a save.
|
||||
fresh = self.add(self.locator("b.png"), DIGEST_B, ["one"])
|
||||
self.assertEqual(
|
||||
(fresh["tags"], fresh["tag_revision"], fresh["added"]), (["one"], 1, 1)
|
||||
)
|
||||
self.schema_clean()
|
||||
|
||||
|
||||
class CurrentImageTagsTests(AnnotationTestCase):
|
||||
def test_current_tags_require_matching_fingerprint(self):
|
||||
loc = self.locator("a.png")
|
||||
other = self.locator("b.png")
|
||||
self.replace(loc, tags=["a"])
|
||||
self.replace(other, DIGEST_B, [], mtime_ns=MTIME + 9)
|
||||
self.assertEqual(
|
||||
self.db.current_image_tags({loc: (SIZE, MTIME), other: (SIZE, MTIME + 9)}),
|
||||
{loc: ["a"], other: []},
|
||||
)
|
||||
self.assertEqual(self.db.current_image_tags({loc: (SIZE, MTIME + 1)}), {})
|
||||
self.assertEqual(self.db.current_image_tags({loc: (SIZE + 1, MTIME)}), {})
|
||||
self.assertEqual(self.db.current_image_tags({}), {})
|
||||
self.assertEqual(
|
||||
self.db.current_image_tags({self.locator("none.png"): (SIZE, MTIME)}), {}
|
||||
)
|
||||
# Refreshing the verified fingerprint makes the record current again.
|
||||
self.assertTrue(
|
||||
self.db.refresh_image_fingerprint(loc, DIGEST_A, SIZE, MTIME + 1)
|
||||
)
|
||||
self.assertFalse(self.db.refresh_image_fingerprint(loc, DIGEST_B, SIZE, MTIME))
|
||||
self.assertFalse(
|
||||
self.db.refresh_image_fingerprint(
|
||||
self.locator("x.png"), DIGEST_A, SIZE, MTIME
|
||||
)
|
||||
)
|
||||
self.assertEqual(
|
||||
self.db.current_image_tags({loc: (SIZE, MTIME + 1)}), {loc: ["a"]}
|
||||
)
|
||||
self.assertEqual(self.db.current_image_tags({loc: (SIZE, MTIME)}), {})
|
||||
self.assertEqual(
|
||||
self.db.read_image_tags(loc, DIGEST_A),
|
||||
{"tags": ["a"], "tag_revision": 1, "prior_count": 0},
|
||||
)
|
||||
for bad in (None, [], {loc: (SIZE,)}, {loc: (SIZE, 1.5)}, {3: (SIZE, MTIME)}):
|
||||
with self.subTest(bad=bad), self.assertRaises(ValueError):
|
||||
self.db.current_image_tags(bad)
|
||||
|
||||
def test_newest_verification_wins_on_equal_fingerprints(self):
|
||||
loc = self.locator("a.png")
|
||||
stamps = iter(f"2026-10-22T00:00:0{i}.000000+00:00" for i in range(10))
|
||||
with patch.object(annotations_module, "utc_now_iso", side_effect=stamps):
|
||||
self.replace(loc, DIGEST_A, ["old bytes"])
|
||||
self.replace(loc, DIGEST_B, ["new bytes"]) # same size and mtime
|
||||
self.assertEqual(
|
||||
self.db.current_image_tags({loc: (SIZE, MTIME)}), {loc: ["new bytes"]}
|
||||
)
|
||||
with patch.object(
|
||||
annotations_module,
|
||||
"utc_now_iso",
|
||||
return_value="2026-10-22T00:01:00.000000+00:00",
|
||||
):
|
||||
self.assertTrue(
|
||||
self.db.refresh_image_fingerprint(loc, DIGEST_A, SIZE, MTIME)
|
||||
)
|
||||
self.assertEqual(
|
||||
self.db.current_image_tags({loc: (SIZE, MTIME)}), {loc: ["old bytes"]}
|
||||
)
|
||||
|
||||
def test_lookup_is_chunked_and_records_stream_in_position_order(self):
|
||||
locators = [self.locator(f"{i:04}.png") for i in range(503)]
|
||||
self.replace(locators[0], tags=["z", "y"])
|
||||
self.replace(locators[502], DIGEST_B, ["x"], mtime_ns=MTIME + 2)
|
||||
self.replace(locators[250], DIGEST_B, [], mtime_ns=MTIME + 1)
|
||||
statements = []
|
||||
conn = self.db.model.get_connection()
|
||||
conn.set_trace_callback(statements.append)
|
||||
try:
|
||||
current = self.db.current_image_tags(
|
||||
{
|
||||
loc: (SIZE, MTIME + (2 if i == 502 else 0))
|
||||
for i, loc in enumerate(locators)
|
||||
}
|
||||
)
|
||||
finally:
|
||||
conn.set_trace_callback(None)
|
||||
self.assertEqual(current, {locators[0]: ["z", "y"], locators[502]: ["x"]})
|
||||
lookups = [s for s in statements if "media_annotation_files" in s]
|
||||
self.assertEqual(len(lookups), 2)
|
||||
self.assertEqual(
|
||||
list(self.db.iter_image_tag_records()),
|
||||
[
|
||||
(locators[0], SIZE, MTIME, ["z", "y"]),
|
||||
(locators[250], SIZE, MTIME + 1, []),
|
||||
(locators[502], SIZE, MTIME + 2, ["x"]),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class AnnotationIsolationTests(AnnotationTestCase):
|
||||
def test_cascade_from_vocabulary_delete_keeps_byte_records(self):
|
||||
loc = self.locator("a.png")
|
||||
self.replace(loc, tags=["gone", "kept"])
|
||||
self.db.delete_tag_all_prompts("gone")
|
||||
self.assertEqual(
|
||||
self.db.read_image_tags(loc, DIGEST_A),
|
||||
{
|
||||
"tags": ["kept"],
|
||||
"tag_revision": 2,
|
||||
"prior_count": 0,
|
||||
}, # moved by the delete
|
||||
)
|
||||
self.schema_clean()
|
||||
|
||||
def test_junction_failure_leaves_nothing_behind(self):
|
||||
loc = self.locator("a.png")
|
||||
self.replace(loc, tags=["a"])
|
||||
conn = self.db.model.get_connection()
|
||||
conn.execute(
|
||||
"CREATE TRIGGER reject_image_tag BEFORE INSERT ON media_image_tags "
|
||||
"BEGIN SELECT RAISE(ABORT,'image tag failure'); END"
|
||||
)
|
||||
conn.commit()
|
||||
before = self.rows(*ANNOTATION_TABLES, "tags")
|
||||
with self.assertRaises(sqlite3.IntegrityError):
|
||||
self.replace(loc, tags=["b"], revision=1)
|
||||
with self.assertRaises(sqlite3.IntegrityError):
|
||||
self.add(self.locator("new.png"), DIGEST_B, ["c"])
|
||||
self.assertFalse(conn.in_transaction)
|
||||
self.assertEqual(self.rows(*ANNOTATION_TABLES, "tags"), before)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+465
-1
@@ -1965,6 +1965,419 @@ class TestOutputScanIsABackgroundJob(AdminAPITestCase):
|
||||
self.assertEqual(data["last_event"]["type"], "error")
|
||||
|
||||
|
||||
class TestActivityJobs(AdminAPITestCase):
|
||||
"""The scan runs as a registry job; legacy routes keep their payloads."""
|
||||
|
||||
def _gate_collect(self):
|
||||
import threading
|
||||
|
||||
gate = threading.Event()
|
||||
calls = []
|
||||
original = self.api._collect_output_media_sync
|
||||
|
||||
def gated(output_dirs):
|
||||
calls.append(output_dirs)
|
||||
gate.wait(5)
|
||||
return original(output_dirs)
|
||||
|
||||
self.api._collect_output_media_sync = gated
|
||||
return gate, calls
|
||||
|
||||
async def _json(self, method, url):
|
||||
resp = await self.client.request(method, url)
|
||||
return resp.status, await resp.json()
|
||||
|
||||
async def test_legacy_scan_route_reuses_registry_job_and_payload_shape(self):
|
||||
from py.api.admin import SCAN_COUNT_KEYS
|
||||
|
||||
_write_png(self.output_dir / "gen.png", prompt_text="a red square")
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
gate, calls = self._gate_collect()
|
||||
|
||||
# The workspace starts the job; the legacy page then attaches to it.
|
||||
status, started = await self._json("POST", "/prompt_manager/jobs/scan")
|
||||
self.assertEqual(status, 202)
|
||||
self.assertTrue(started["success"])
|
||||
legacy = asyncio.ensure_future(
|
||||
self.client.request("POST", "/prompt_manager/scan", data="{}")
|
||||
)
|
||||
await asyncio.sleep(0.05)
|
||||
gate.set()
|
||||
body = await (await legacy).text()
|
||||
await asyncio.wait_for(self.api._scan_job.task, 10)
|
||||
|
||||
self.assertEqual(len(calls), 1)
|
||||
payloads = _sse_payloads(body)
|
||||
progress = [p for p in payloads if p["type"] == "progress"]
|
||||
self.assertTrue(progress)
|
||||
legacy_keys = set(SCAN_COUNT_KEYS) | {"type", "progress", "status", "processed"}
|
||||
for payload in progress:
|
||||
self.assertEqual(set(payload), legacy_keys)
|
||||
final = payloads[-1]
|
||||
self.assertEqual(final["type"], "complete")
|
||||
self.assertEqual(set(final), set(SCAN_COUNT_KEYS) | {"type", "directories"})
|
||||
self.assertEqual(final["added"], 1)
|
||||
self.assertEqual(final["directories"], ["output"])
|
||||
|
||||
status, listed = await self._json("GET", "/prompt_manager/jobs")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(len(listed["jobs"]), 1)
|
||||
job = listed["jobs"][0]
|
||||
self.assertEqual(job["id"], started["job"]["id"])
|
||||
self.assertEqual(job["id"], self.api._scan_job.id)
|
||||
self.assertEqual((job["kind"], job["state"]), ("scan", "finished"))
|
||||
self.assertEqual(job["counters"]["added"], 1)
|
||||
self.assertEqual(job["progress"], 100)
|
||||
self.assertEqual(job["scope"]["output_folders"], 1)
|
||||
|
||||
async def test_scan_records_outcomes_by_group(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
(self.output_dir / "sub").mkdir()
|
||||
names = {
|
||||
"sub/added.png": {"status": "added", "title": "A red fox"},
|
||||
"linked.png": {"status": "linked", "title": "A grey cat"},
|
||||
"unchanged.png": {"status": "unchanged", "title": "A brown dog"},
|
||||
"ambiguous.png": {"status": "ambiguous"},
|
||||
"failed.png": RuntimeError(f"{self.tmpdir}/secret boom"),
|
||||
"none.png": None,
|
||||
}
|
||||
for name in names:
|
||||
(self.output_dir / name).write_bytes(b"not an image")
|
||||
|
||||
def fake_ingest(media_file, metadata, detailed=False):
|
||||
key = Path(media_file).relative_to(self.output_dir).as_posix()
|
||||
outcome = names[key]
|
||||
if isinstance(outcome, Exception):
|
||||
raise outcome
|
||||
return outcome
|
||||
|
||||
self.api._ingest_scanned_file = fake_ingest
|
||||
with self.assertLogs("prompt_manager.api", level="ERROR"):
|
||||
status, started = await self._json("POST", "/prompt_manager/jobs/scan")
|
||||
self.assertEqual(status, 202)
|
||||
await asyncio.wait_for(self.api._scan_job.task, 10)
|
||||
|
||||
resp = await self.client.request(
|
||||
"GET", f"/prompt_manager/jobs/{started['job']['id']}"
|
||||
)
|
||||
text = await resp.text()
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertNotIn(self.tmpdir, text)
|
||||
job = json.loads(text)["job"]
|
||||
self.assertEqual(
|
||||
job["counters"],
|
||||
{"added": 2, "known": 1, "review": 1, "no_metadata": 1, "errors": 1},
|
||||
)
|
||||
outcomes = job["outcomes"]
|
||||
self.assertEqual(
|
||||
{k: len(v) for k, v in outcomes.items()},
|
||||
{"added": 2, "known": 1, "review": 1, "no_metadata": 1, "errors": 1},
|
||||
)
|
||||
added = {row["path"]: row for row in outcomes["added"]}
|
||||
self.assertEqual(set(added), {"sub/added.png", "linked.png"})
|
||||
self.assertIn("A red fox", added["sub/added.png"]["detail"])
|
||||
self.assertIn("A grey cat", added["linked.png"]["detail"])
|
||||
self.assertEqual(added["linked.png"]["root"], "output")
|
||||
self.assertIn("A brown dog", outcomes["known"][0]["detail"])
|
||||
self.assertIn("review", outcomes["review"][0]["detail"].lower())
|
||||
self.assertEqual(outcomes["errors"][0]["path"], "failed.png")
|
||||
self.assertTrue(outcomes["errors"][0]["detail"])
|
||||
self.assertEqual(outcomes["no_metadata"][0]["path"], "none.png")
|
||||
self.assertTrue(outcomes["no_metadata"][0]["detail"])
|
||||
prefixes = {line.split(" ", 2)[1] for line in job["log"][1:-1]}
|
||||
self.assertEqual(prefixes, {"+", "=", "?", "!", "-"})
|
||||
self.assertEqual(job["state"], "finished")
|
||||
self.assertTrue(job["status_line"].startswith("Processing file 6 of 6"))
|
||||
|
||||
legacy = self.api._scan_job.last_event
|
||||
self.assertEqual(
|
||||
(legacy["processed"], legacy["found"], legacy["failed"]), (6, 4, 1)
|
||||
)
|
||||
|
||||
async def test_report_is_attachment_without_absolute_paths(self):
|
||||
"""Task 5: the report downloads as JSON, whole and without server paths."""
|
||||
import re
|
||||
|
||||
from py.api.jobs import MAX_OUTCOME_ROWS
|
||||
|
||||
async def runner(job):
|
||||
absolute = f"{self.tmpdir}/output/deep/abs.png"
|
||||
job.record(
|
||||
"added",
|
||||
{"path": absolute, "root": "output", "root_index": 0, "detail": "ok"},
|
||||
f"+ {absolute} · ok",
|
||||
)
|
||||
for index in range(MAX_OUTCOME_ROWS + 5):
|
||||
job.record(
|
||||
"known",
|
||||
{"path": f"k/{index}.png", "root": "output", "detail": "x"},
|
||||
f"= k/{index}.png",
|
||||
)
|
||||
job.update(processed=MAX_OUTCOME_ROWS + 6, total=MAX_OUTCOME_ROWS + 6)
|
||||
|
||||
job = self.api._jobs.start(
|
||||
"scan",
|
||||
"Scan for metadata",
|
||||
record_changing=True,
|
||||
scope={"files": "new", "output_folders": 1},
|
||||
runner=runner,
|
||||
groups=("added", "known", "review", "no_metadata", "errors"),
|
||||
)
|
||||
await asyncio.wait_for(job.task, 10)
|
||||
|
||||
resp = await self.client.request("GET", f"/prompt_manager/jobs/{job.id}/report")
|
||||
text = await resp.text()
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertIn("application/json", resp.headers["Content-Type"])
|
||||
disposition = resp.headers["Content-Disposition"]
|
||||
self.assertRegex(
|
||||
disposition,
|
||||
r'^attachment; filename="prompt-manager-scan-\d{8}-\d{4}\.json"$',
|
||||
)
|
||||
self.assertNotIn(self.tmpdir, text)
|
||||
self.assertIsNone(
|
||||
re.search(r"[A-Za-z]:[\\/]|(?<![\w.])/(?:tmp|home|var)/", text)
|
||||
)
|
||||
report = json.loads(text)
|
||||
self.assertEqual(report["job"]["id"], job.id)
|
||||
self.assertEqual(report["job"]["scope"]["files"], "new")
|
||||
self.assertEqual(report["job"]["counters"]["known"], MAX_OUTCOME_ROWS + 5)
|
||||
added = report["outcomes"]["added"]
|
||||
self.assertEqual(
|
||||
(added["count"], len(added["rows"]), added["truncated"]), (1, 1, False)
|
||||
)
|
||||
self.assertEqual(added["rows"][0]["path"], "abs.png")
|
||||
known = report["outcomes"]["known"]
|
||||
self.assertEqual(
|
||||
(known["count"], len(known["rows"]), known["truncated"]),
|
||||
(MAX_OUTCOME_ROWS + 5, MAX_OUTCOME_ROWS, True),
|
||||
)
|
||||
self.assertEqual(report["outcomes"]["errors"]["rows"], [])
|
||||
self.assertFalse(report["outcomes"]["errors"]["truncated"])
|
||||
self.assertEqual(report["log"], job.detail()["log"])
|
||||
self.assertIn("generated_at", report)
|
||||
|
||||
resp = await self.client.request("GET", "/prompt_manager/jobs/nope/report")
|
||||
self.assertEqual(resp.status, 404)
|
||||
|
||||
async def test_cancel_after_first_batch_keeps_partial_counts(self):
|
||||
"""Review M7: the real scan stops at the batch boundary, keeping batch 1."""
|
||||
import threading
|
||||
from unittest import mock
|
||||
|
||||
from py.api import admin
|
||||
|
||||
_write_png(self.output_dir / "a.png", prompt_text="a red square")
|
||||
_write_png(self.output_dir / "b.png", prompt_text="a blue circle")
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
first_done, release = threading.Event(), threading.Event()
|
||||
original = self.api._ingest_scan_batch_sync
|
||||
|
||||
def gated(batch_results, roots=(), outcomes=None):
|
||||
counts = original(batch_results, roots=roots, outcomes=outcomes)
|
||||
first_done.set()
|
||||
release.wait(5)
|
||||
return counts
|
||||
|
||||
self.api._ingest_scan_batch_sync = gated
|
||||
with mock.patch.object(admin, "SCAN_BATCH_SIZE", 1):
|
||||
status, started = await self._json("POST", "/prompt_manager/jobs/scan")
|
||||
self.assertEqual(status, 202)
|
||||
job_id = started["job"]["id"]
|
||||
await asyncio.get_running_loop().run_in_executor(None, first_done.wait, 5)
|
||||
status, cancelled = await self._json(
|
||||
"POST", f"/prompt_manager/jobs/{job_id}/cancel"
|
||||
)
|
||||
self.assertEqual((status, cancelled["cancellation_requested"]), (200, True))
|
||||
release.set()
|
||||
await asyncio.wait_for(self.api._scan_job.task, 10)
|
||||
|
||||
status, body = await self._json("GET", f"/prompt_manager/jobs/{job_id}")
|
||||
job = body["job"]
|
||||
self.assertEqual(job["state"], "cancelled")
|
||||
self.assertEqual((job["processed"], job["total"]), (1, 2))
|
||||
self.assertEqual(sum(job["counters"].values()), job["processed"])
|
||||
self.assertEqual(job["counters"]["added"], 1)
|
||||
listed = [row["path"] for rows in job["outcomes"].values() for row in rows]
|
||||
self.assertEqual(len(listed), 1)
|
||||
self.assertIn(listed[0], {"a.png", "b.png"})
|
||||
self.assertIn("Scan cancelled · 1 file processed", job["log"][-1])
|
||||
# A cancel after the end is a no-op.
|
||||
status, again = await self._json(
|
||||
"POST", f"/prompt_manager/jobs/{job_id}/cancel"
|
||||
)
|
||||
self.assertEqual((status, again["cancellation_requested"]), (200, False))
|
||||
|
||||
async def test_scan_rows_carry_their_root_index(self):
|
||||
"""Review M9: two roots labelled "output" stay apart by listing index."""
|
||||
second = self.outside_dir / "output"
|
||||
second.mkdir(parents=True)
|
||||
os.environ[EXTRA_ROOTS_ENV] = str(second)
|
||||
_write_png(self.output_dir / "same.png", prompt_text="a red square")
|
||||
_write_png(second / "same.png", prompt_text="a blue circle")
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [
|
||||
self._root(self.output_dir),
|
||||
self._root(second),
|
||||
]
|
||||
output_dirs = [Path(d) for d in self.api._get_all_output_dirs()]
|
||||
self.assertEqual(len(output_dirs), 2)
|
||||
status, started = await self._json("POST", "/prompt_manager/jobs/scan")
|
||||
await asyncio.wait_for(self.api._scan_job.task, 10)
|
||||
status, body = await self._json(
|
||||
"GET", f"/prompt_manager/jobs/{started['job']['id']}"
|
||||
)
|
||||
rows = body["job"]["outcomes"]["added"]
|
||||
self.assertEqual(len(rows), 2)
|
||||
self.assertEqual({row["root"] for row in rows}, {"output"})
|
||||
by_index = {row["root_index"]: row for row in rows}
|
||||
self.assertEqual(set(by_index), {0, 1})
|
||||
for index, row in by_index.items():
|
||||
self.assertEqual(row["path"], "same.png")
|
||||
self.assertTrue((output_dirs[index] / row["path"]).is_file())
|
||||
texts = {
|
||||
index: (
|
||||
"red"
|
||||
if output_dirs[index] == Path(self._root(self.output_dir))
|
||||
else "blue"
|
||||
)
|
||||
for index in by_index
|
||||
}
|
||||
for index, row in by_index.items():
|
||||
self.assertIn(texts[index], row["detail"])
|
||||
|
||||
async def test_scan_row_names_the_prompt_by_its_stored_title(self):
|
||||
"""Review M10: a row names the prompt as the Library does."""
|
||||
_write_png(self.output_dir / "gen.png", prompt_text="a red square\nmore")
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
summaries = self.api.db.get_image_link_summaries
|
||||
|
||||
def titled(paths, allow_filename_fallback=True):
|
||||
found = summaries(paths, allow_filename_fallback)
|
||||
return {
|
||||
path: {**summary, "title": "Stored square title"}
|
||||
for path, summary in found.items()
|
||||
}
|
||||
|
||||
self.api.db.get_image_link_summaries = titled
|
||||
status, started = await self._json("POST", "/prompt_manager/jobs/scan")
|
||||
await asyncio.wait_for(self.api._scan_job.task, 10)
|
||||
status, body = await self._json(
|
||||
"GET", f"/prompt_manager/jobs/{started['job']['id']}"
|
||||
)
|
||||
(row,) = body["job"]["outcomes"]["added"]
|
||||
self.assertIn("“Stored square title”", row["detail"])
|
||||
|
||||
# Without a stored title (or an unreadable one): the text's first line.
|
||||
def broken(paths, allow_filename_fallback=True):
|
||||
raise RuntimeError("unreadable")
|
||||
|
||||
self.api.db.get_image_link_summaries = broken
|
||||
_write_png(self.output_dir / "next.png", prompt_text="a green cone\nmore")
|
||||
status, started = await self._json("POST", "/prompt_manager/jobs/scan")
|
||||
await asyncio.wait_for(self.api._scan_job.task, 10)
|
||||
status, body = await self._json(
|
||||
"GET", f"/prompt_manager/jobs/{started['job']['id']}"
|
||||
)
|
||||
details = {r["path"]: r["detail"] for r in body["job"]["outcomes"]["added"]}
|
||||
self.assertIn("“a green cone”", details["next.png"])
|
||||
|
||||
async def test_jobs_routes_404_and_409(self):
|
||||
_write_png(self.output_dir / "gen.png", prompt_text="a red square")
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
for method, url in (
|
||||
("GET", "/prompt_manager/jobs/nope"),
|
||||
("GET", "/prompt_manager/jobs/nope/events"),
|
||||
("POST", "/prompt_manager/jobs/nope/cancel"),
|
||||
):
|
||||
status, data = await self._json(method, url)
|
||||
self.assertEqual(status, 404, url)
|
||||
self.assertFalse(data["success"])
|
||||
|
||||
gate, calls = self._gate_collect()
|
||||
status, started = await self._json("POST", "/prompt_manager/jobs/scan")
|
||||
self.assertEqual(status, 202)
|
||||
job_id = started["job"]["id"]
|
||||
self.assertEqual(started["job"]["state"], "running")
|
||||
|
||||
status, busy = await self._json("POST", "/prompt_manager/jobs/scan")
|
||||
self.assertEqual(status, 409)
|
||||
self.assertEqual(
|
||||
busy["running"], {"id": job_id, "title": started["job"]["title"]}
|
||||
)
|
||||
self.assertEqual((busy["success"], busy["code"]), (False, "busy"))
|
||||
|
||||
watcher = asyncio.ensure_future(
|
||||
self.client.request("GET", f"/prompt_manager/jobs/{job_id}/events")
|
||||
)
|
||||
await asyncio.sleep(0.05)
|
||||
status, cancel = await self._json(
|
||||
"POST", f"/prompt_manager/jobs/{job_id}/cancel"
|
||||
)
|
||||
self.assertEqual((status, cancel["cancellation_requested"]), (200, True))
|
||||
gate.set()
|
||||
events = _sse_payloads(await (await watcher).text())
|
||||
await asyncio.wait_for(self.api._scan_job.task, 10)
|
||||
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.assertEqual(events[0]["id"], job_id)
|
||||
self.assertEqual(events[-1]["state"], "cancelled")
|
||||
status, cancel = await self._json(
|
||||
"POST", f"/prompt_manager/jobs/{job_id}/cancel"
|
||||
)
|
||||
self.assertEqual((status, cancel["cancellation_requested"]), (200, False))
|
||||
status, legacy = await self._json("GET", "/prompt_manager/scan/status")
|
||||
self.assertEqual(legacy["last_event"]["type"], "cancelled")
|
||||
|
||||
async def test_scan_without_directories_fails_the_job(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.outside_dir)]
|
||||
sys.modules.pop("folder_paths", None)
|
||||
self.api._find_comfyui_output_dir = lambda: None
|
||||
|
||||
status, started = await self._json("POST", "/prompt_manager/jobs/scan")
|
||||
await asyncio.wait_for(self.api._scan_job.task, 10)
|
||||
status, data = await self._json(
|
||||
"GET", f"/prompt_manager/jobs/{started['job']['id']}"
|
||||
)
|
||||
self.assertEqual(data["job"]["state"], "failed")
|
||||
self.assertIn("No output directories", data["job"]["status_line"])
|
||||
|
||||
async def test_legacy_scan_refused_while_another_record_job_runs(self):
|
||||
hold = asyncio.Event()
|
||||
|
||||
async def runner(job):
|
||||
await hold.wait()
|
||||
|
||||
other = self.api._jobs.start(
|
||||
"maintenance", "Maintenance", record_changing=True, scope={}, runner=runner
|
||||
)
|
||||
resp = await self.client.request("POST", "/prompt_manager/scan", data="{}")
|
||||
self.assertEqual(resp.status, 409)
|
||||
self.assertIn("Maintenance", (await resp.json())["error"])
|
||||
hold.set()
|
||||
await asyncio.wait_for(other.task, 5)
|
||||
|
||||
|
||||
class TestScanStatusLine(unittest.TestCase):
|
||||
|
||||
def test_estimate_appears_after_two_batches(self):
|
||||
from py.api.admin import _scan_status_line
|
||||
|
||||
self.assertEqual(
|
||||
_scan_status_line(50, 200, 1, 10.0), "Processing file 50 of 200"
|
||||
)
|
||||
self.assertEqual(
|
||||
_scan_status_line(100, 1000, 2, 30.0),
|
||||
"Processing file 100 of 1,000 · about 5 min left",
|
||||
)
|
||||
self.assertEqual(
|
||||
_scan_status_line(100, 120, 2, 1.0),
|
||||
"Processing file 100 of 120 · about 1 min left",
|
||||
)
|
||||
self.assertEqual(
|
||||
_scan_status_line(120, 120, 3, 9.0), "Processing file 120 of 120"
|
||||
)
|
||||
|
||||
|
||||
class TestWorkerThreadsSetting(AdminAPITestCase):
|
||||
"""The worker-thread count is exposed, validated, persisted and used by the scan."""
|
||||
|
||||
@@ -2063,7 +2476,8 @@ def _write_mp4(path, comment):
|
||||
|
||||
|
||||
class TestScanReadsVideos(AdminAPITestCase):
|
||||
"""Videos saved by ComfyUI carry the graph in their comment tag."""
|
||||
"""Videos carry the graph in per-key container tags (current savers) or in
|
||||
the old VideoHelperSuite comment envelope; the scan reads one carrier."""
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
@@ -2093,6 +2507,56 @@ class TestScanReadsVideos(AdminAPITestCase):
|
||||
images = self.api.db.get_prompt_images(prompts[0]["id"])
|
||||
self.assertTrue(images[0]["image_path"].endswith("clip.mp4"))
|
||||
|
||||
async def test_scan_links_per_key_tag_video_with_default_profile(self):
|
||||
"""A current saver's per-key tags scan exactly like the same graph as PNG."""
|
||||
from utils import video_metadata
|
||||
|
||||
_write_png(self.output_dir / "same.png", prompt_text="a cat")
|
||||
graph = {
|
||||
"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "a cat"}},
|
||||
"2": {"class_type": "CLIPTextEncode", "inputs": {"text": "bad"}},
|
||||
"3": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {"positive": ["1", 0], "negative": ["2", 0]},
|
||||
},
|
||||
}
|
||||
(self.output_dir / "clip.mkv").write_bytes(b"\x1aE\xdf\xa3 fixture")
|
||||
probed = {
|
||||
"format": {
|
||||
"tags": {
|
||||
"PROMPT": json.dumps(graph),
|
||||
"WORKFLOW": '{"nodes":[]}',
|
||||
"COMMENT": json.dumps({"prompt": "must not win"}),
|
||||
}
|
||||
}
|
||||
}
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
|
||||
with (
|
||||
patch.object(video_metadata.shutil, "which", return_value="/bin/ffprobe"),
|
||||
patch.object(
|
||||
video_metadata.subprocess,
|
||||
"run",
|
||||
return_value=MagicMock(returncode=0, stdout=json.dumps(probed)),
|
||||
) as run,
|
||||
self.assertNoLogs(level="ERROR"),
|
||||
):
|
||||
body = await (
|
||||
await self.client.request("POST", "/prompt_manager/scan")
|
||||
).text()
|
||||
|
||||
self.assertEqual(run.call_args.args[0][-1], str(self.output_dir / "clip.mkv"))
|
||||
self.assertIn('"found": 2', body)
|
||||
prompts = self.api.db.get_recent_prompts(limit=10)["prompts"]
|
||||
self.assertEqual(sorted(p["text"] for p in prompts), ["a cat", "bad"])
|
||||
positive = next(p for p in prompts if p["text"] == "a cat")
|
||||
images = self.api.db.get_prompt_images(positive["id"])
|
||||
self.assertEqual(
|
||||
sorted(os.path.basename(image["image_path"]) for image in images),
|
||||
["clip.mkv", "same.png"],
|
||||
)
|
||||
self.assertNotIn("must not win", json.dumps(prompts))
|
||||
|
||||
async def test_scan_without_ffprobe_skips_videos_quietly(self):
|
||||
from unittest.mock import patch
|
||||
from utils import video_metadata
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
"""Physical-file prompt inspection is bounded, conservative and read-only."""
|
||||
|
||||
import io
|
||||
import json
|
||||
from unittest.mock import patch
|
||||
import shutil
|
||||
import subprocess
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from PIL import Image, PngImagePlugin
|
||||
|
||||
@@ -15,6 +19,9 @@ except ImportError:
|
||||
from utils.pair_execution import EXECUTION_KEY, record_pair_execution
|
||||
from database.pair_ingestion import ingest_resolved_pair
|
||||
from utils.prompt_pair_resolver import resolve_prompt_pair
|
||||
from utils import video_metadata
|
||||
|
||||
HAVE_FFMPEG = bool(shutil.which("ffmpeg") and shutil.which("ffprobe"))
|
||||
|
||||
|
||||
def graph(positive="portrait", negative="blurry"):
|
||||
@@ -28,6 +35,54 @@ def graph(positive="portrait", negative="blurry"):
|
||||
}
|
||||
|
||||
|
||||
def write_video(path, metadata_tags=True, **tags):
|
||||
"""Tiny black clip carrying one container tag per keyword (needs ffmpeg).
|
||||
|
||||
MP4 keeps arbitrary keys only with ``use_metadata_tags`` (mdta atoms), the
|
||||
movflag the core SaveVideo node sets; without it ffmpeg writes the legacy
|
||||
ilst atoms that old VideoHelperSuite files carry. Matroska and WebM
|
||||
upper-case the key names.
|
||||
"""
|
||||
command = [
|
||||
"ffmpeg",
|
||||
"-v",
|
||||
"quiet",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"color=c=black:s=16x16:d=0.2",
|
||||
]
|
||||
if metadata_tags and path.suffix.lower() in (".mp4", ".mov", ".m4v"):
|
||||
command += ["-movflags", "use_metadata_tags"]
|
||||
for key, value in tags.items():
|
||||
command += ["-metadata", f"{key}={value}"]
|
||||
subprocess.run(command + ["-y", str(path)], check=True, timeout=60)
|
||||
return path
|
||||
|
||||
|
||||
class ProbeOutput(io.BytesIO):
|
||||
"""ffprobe stdout that records the bounded read size asked of it."""
|
||||
|
||||
requested_size = None
|
||||
|
||||
def read(self, size=-1):
|
||||
self.requested_size = size
|
||||
return super().read(size)
|
||||
|
||||
|
||||
def probe_process(payload, poll=0, returncode=0):
|
||||
"""A fake ffprobe whose stdout is ``payload`` bytes."""
|
||||
process = MagicMock()
|
||||
process.stdout = ProbeOutput(payload)
|
||||
process.poll.return_value = poll
|
||||
process.returncode = returncode
|
||||
return process
|
||||
|
||||
|
||||
def probe_payload(tags):
|
||||
return json.dumps({"format": {"tags": tags}}).encode()
|
||||
|
||||
|
||||
def png(path, **metadata):
|
||||
info = PngImagePlugin.PngInfo()
|
||||
for key, value in metadata.items():
|
||||
@@ -128,7 +183,12 @@ class TestPromptInspector(ImageAPITestCase):
|
||||
for expected in ("ambiguous", "unresolved"):
|
||||
source = graph()
|
||||
if expected == "ambiguous":
|
||||
source["4"] = source["3"]
|
||||
# Branches with different prompts (#249: identical ones resolve).
|
||||
source["5"] = {"class_type": "CLIPTextEncode", "inputs": {"text": "x"}}
|
||||
source["4"] = {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {"positive": ["5", 0], "negative": ["2", 0]},
|
||||
}
|
||||
else:
|
||||
source["1"]["class_type"] = "UnknownEncoder"
|
||||
png(
|
||||
@@ -307,7 +367,7 @@ class TestPromptInspector(ImageAPITestCase):
|
||||
async def test_corrupt_truncated_duplicate_and_expanding_metadata_fail_closed(self):
|
||||
import struct
|
||||
import zlib
|
||||
from utils.prompt_inspector import MAX_METADATA_BYTES
|
||||
from utils.prompt_inspector import VIDEO_REPROBE_BYTES
|
||||
|
||||
path = self.output_dir / "image.png"
|
||||
base = png(path, prompt=graph()).read_bytes()
|
||||
@@ -320,8 +380,8 @@ class TestPromptInspector(ImageAPITestCase):
|
||||
cases.append(bytes(broken))
|
||||
for kind, payload in (
|
||||
(b"tEXt", b"prompt\0{}"),
|
||||
(b"zTXt", b"prompt\0\0" + zlib.compress(b"x" * (MAX_METADATA_BYTES + 1))),
|
||||
(b"tEXt", b"prompt\0" + b"x" * (MAX_METADATA_BYTES + 1)),
|
||||
(b"zTXt", b"prompt\0\0" + zlib.compress(b"x" * (VIDEO_REPROBE_BYTES + 1))),
|
||||
(b"tEXt", b"prompt\0" + b"x" * (VIDEO_REPROBE_BYTES + 1)),
|
||||
(b"tEXt", b"bad-key-without-separator"),
|
||||
(b"zTXt", b"prompt\0\1bad-compression"),
|
||||
(b"iTXt", b"prompt\0\2\0\0\0text"),
|
||||
@@ -380,9 +440,7 @@ class TestPromptInspector(ImageAPITestCase):
|
||||
self.assertEqual(result["prompts"]["positive"]["status"], "absent")
|
||||
|
||||
async def test_video_inspection_uses_bounded_container_evidence(self):
|
||||
import io
|
||||
from unittest.mock import MagicMock
|
||||
from utils.prompt_inspector import MAX_METADATA_BYTES
|
||||
from utils.prompt_inspector import VIDEO_PROBE_BYTES, VIDEO_REPROBE_BYTES
|
||||
|
||||
(self.output_dir / "clip.mp4").write_bytes(b"video fixture")
|
||||
metadata = {"prompt": graph("Video portrait", "")}
|
||||
@@ -407,14 +465,25 @@ class TestPromptInspector(ImageAPITestCase):
|
||||
process.wait.assert_called()
|
||||
self.assertNotIn("shell", spawn.call_args.kwargs)
|
||||
self.assertIn(str(self.output_dir / "clip.mp4"), spawn.call_args.args[0])
|
||||
process.stdout = io.BytesIO(b"x" * (MAX_METADATA_BYTES + 1))
|
||||
# An envelope over the probe bound cannot be split: the role-only
|
||||
# re-probe overflows too and the read fails instead of guessing.
|
||||
processes = [
|
||||
probe_process(b"x" * (VIDEO_PROBE_BYTES + 1)),
|
||||
probe_process(b"x" * (VIDEO_REPROBE_BYTES + 1)),
|
||||
]
|
||||
with (
|
||||
patch("utils.prompt_inspector.ffprobe_path", return_value="/tool/ffprobe"),
|
||||
patch("utils.prompt_inspector.subprocess.Popen", return_value=process),
|
||||
patch(
|
||||
"utils.prompt_inspector.subprocess.Popen", side_effect=processes
|
||||
) as spawn,
|
||||
):
|
||||
_, result = await self.inspect("clip.mp4")
|
||||
self.assertEqual(spawn.call_count, 2)
|
||||
self.assertEqual(result["prompts"]["positive"]["status"], "unresolved")
|
||||
self.assertTrue(process.stdout.closed)
|
||||
self.assertEqual(result["metadata"]["status"], "unreadable")
|
||||
for process in processes:
|
||||
self.assertTrue(process.stdout.closed)
|
||||
process.wait.assert_called()
|
||||
|
||||
async def test_video_missing_probe_and_timeout_preserve_uncertainty(self):
|
||||
import io
|
||||
@@ -890,11 +959,9 @@ class TestPromptInspector(ImageAPITestCase):
|
||||
self.assertNotIn("\ufffd".encode(), body)
|
||||
self.assertLessEqual(len(body), 8 * 1024 * 1024)
|
||||
|
||||
async def video_inspect(self, comment=None, description=None):
|
||||
import io
|
||||
async def video_inspect(self, comment=None, description=None, **per_key):
|
||||
import threading
|
||||
from unittest.mock import MagicMock
|
||||
from utils.prompt_inspector import MAX_METADATA_BYTES
|
||||
from utils.prompt_inspector import VIDEO_PROBE_BYTES
|
||||
|
||||
path = self.output_dir / "clip.mp4"
|
||||
path.write_bytes(b"fixture")
|
||||
@@ -903,17 +970,8 @@ class TestPromptInspector(ImageAPITestCase):
|
||||
for key, value in (("comment", comment), ("description", description))
|
||||
if value is not None
|
||||
}
|
||||
payload = json.dumps({"format": {"tags": tags}}).encode()
|
||||
|
||||
class ProbeOutput(io.BytesIO):
|
||||
def read(self, size=-1):
|
||||
self.requested_size = size
|
||||
return super().read(size)
|
||||
|
||||
process = MagicMock()
|
||||
process.stdout = ProbeOutput(payload)
|
||||
process.poll.return_value = 0
|
||||
process.returncode = 0
|
||||
tags.update(per_key)
|
||||
process = probe_process(probe_payload(tags))
|
||||
timers = []
|
||||
timer_type = threading.Timer
|
||||
|
||||
@@ -932,11 +990,156 @@ class TestPromptInspector(ImageAPITestCase):
|
||||
response, result = await self.inspect("clip.mp4")
|
||||
self.assertEqual(spawn.call_count, 1)
|
||||
self.assertTrue(process.stdout.closed)
|
||||
self.assertEqual(process.stdout.requested_size, MAX_METADATA_BYTES + 1)
|
||||
self.assertEqual(process.stdout.requested_size, VIDEO_PROBE_BYTES + 1)
|
||||
self.assertEqual(len(timers), 1)
|
||||
self.assertTrue(timers[0].finished.is_set())
|
||||
self.assertEqual(
|
||||
spawn.call_args.args[0][spawn.call_args.args[0].index("-show_entries") + 1],
|
||||
"format_tags=prompt,workflow,parameters,"
|
||||
"comfyassets_prompt_pairs,comment,description",
|
||||
)
|
||||
return response, result
|
||||
|
||||
async def test_video_separate_prompt_workflow_tags_are_read(self):
|
||||
source = graph("per-key portrait", "per-key negative")
|
||||
workflow = '{"nodes":[{"id":"per-key"}],"cfg":6.5000000000000000001}'
|
||||
for tags in (
|
||||
{"prompt": json.dumps(source), "workflow": workflow},
|
||||
{"PROMPT": json.dumps(source), "WORKFLOW": workflow},
|
||||
):
|
||||
_, result = await self.video_inspect(
|
||||
json.dumps({"prompt": graph("comment must not win")}), **tags
|
||||
)
|
||||
self.assertEqual(
|
||||
result["prompts"]["positive"], self.side("per-key portrait")
|
||||
)
|
||||
self.assertEqual(
|
||||
result["prompts"]["negative"], self.side("per-key negative")
|
||||
)
|
||||
self.assertEqual(
|
||||
result["metadata"]["record_formats"],
|
||||
{"prompt": "original-text", "workflow": "original-text"},
|
||||
)
|
||||
self.assertEqual(result["metadata"]["raw"]["workflow"], workflow)
|
||||
self.assertEqual(
|
||||
result["metadata"]["workflow_graph"]["value"]["cfg"],
|
||||
"6.5000000000000000001",
|
||||
)
|
||||
self.assertNotIn("must not win", json.dumps(result))
|
||||
_, result = await self.video_inspect(
|
||||
**{"PROMPT": json.dumps(source), "prompt": json.dumps(graph("other"))}
|
||||
)
|
||||
self.assertEqual(result["prompts"]["positive"]["status"], "unresolved")
|
||||
self.assertEqual(result["metadata"]["status"], "unreadable")
|
||||
|
||||
@unittest.skipUnless(HAVE_FFMPEG, "ffmpeg/ffprobe not installed")
|
||||
async def test_video_per_key_and_comment_tags_read_from_real_containers(self):
|
||||
video_metadata.ffprobe_path.cache_clear()
|
||||
self.addCleanup(video_metadata.ffprobe_path.cache_clear)
|
||||
source = json.dumps(graph("real portrait", ""))
|
||||
workflow = '{"nodes":[{"id":"real"}]}'
|
||||
for name in ("per_key.mp4", "per_key.mkv", "per_key.webm"):
|
||||
write_video(self.output_dir / name, prompt=source, workflow=workflow)
|
||||
response, result = await self.inspect(name)
|
||||
self.assertEqual(response.status, 200, name)
|
||||
self.assertEqual(
|
||||
result["prompts"]["positive"], self.side("real portrait"), name
|
||||
)
|
||||
self.assertEqual(result["metadata"]["raw"]["workflow"], workflow, name)
|
||||
self.assertEqual(
|
||||
result["metadata"]["record_formats"],
|
||||
{"prompt": "original-text", "workflow": "original-text"},
|
||||
name,
|
||||
)
|
||||
# Old VideoHelperSuite envelope in the legacy ilst comment atom, unchanged.
|
||||
write_video(
|
||||
self.output_dir / "vhs.mp4",
|
||||
metadata_tags=False,
|
||||
comment=json.dumps({"prompt": json.loads(source), "workflow": {"v": 1}}),
|
||||
)
|
||||
_, result = await self.inspect("vhs.mp4")
|
||||
self.assertEqual(result["prompts"]["positive"], self.side("real portrait"))
|
||||
self.assertEqual(
|
||||
result["metadata"]["record_formats"],
|
||||
{"prompt": "reserialized-json", "workflow": "reserialized-json"},
|
||||
)
|
||||
self.assertEqual(result["metadata"]["workflow_graph"]["value"], {"v": 1})
|
||||
|
||||
async def test_video_oversize_workflow_reprobes_role_only_and_marks_limited(self):
|
||||
from utils.prompt_inspector import VIDEO_PROBE_BYTES, VIDEO_REPROBE_BYTES
|
||||
|
||||
path = self.output_dir / "clip.mp4"
|
||||
path.write_bytes(b"fixture")
|
||||
source = json.dumps(graph("bounded portrait", ""))
|
||||
processes = [
|
||||
probe_process(b"x" * (VIDEO_PROBE_BYTES + 1)),
|
||||
probe_process(probe_payload({"PROMPT": source})),
|
||||
]
|
||||
with (
|
||||
patch("utils.prompt_inspector.ffprobe_path", return_value="/tool/ffprobe"),
|
||||
patch(
|
||||
"utils.prompt_inspector.subprocess.Popen", side_effect=processes
|
||||
) as spawn,
|
||||
):
|
||||
response, result = await self.inspect("clip.mp4")
|
||||
self.assertEqual(response.status, 200)
|
||||
self.assertEqual(spawn.call_count, 2)
|
||||
first, second = (call.args[0] for call in spawn.call_args_list)
|
||||
self.assertEqual(
|
||||
first[first.index("-show_entries") + 1],
|
||||
"format_tags=prompt,workflow,parameters,"
|
||||
"comfyassets_prompt_pairs,comment,description",
|
||||
)
|
||||
self.assertEqual(
|
||||
second[second.index("-show_entries") + 1],
|
||||
"format_tags=prompt,parameters,comfyassets_prompt_pairs,"
|
||||
"comment,description",
|
||||
)
|
||||
self.assertEqual(processes[0].stdout.requested_size, VIDEO_PROBE_BYTES + 1)
|
||||
self.assertEqual(processes[1].stdout.requested_size, VIDEO_REPROBE_BYTES + 1)
|
||||
for process in processes:
|
||||
self.assertTrue(process.stdout.closed)
|
||||
process.wait.assert_called()
|
||||
self.assertEqual(result["prompts"]["positive"], self.side("bounded portrait"))
|
||||
self.assertEqual(result["metadata"]["status"], "available")
|
||||
self.assertEqual(
|
||||
result["metadata"]["record_formats"],
|
||||
{"prompt": "original-text", "workflow": "limited"},
|
||||
)
|
||||
self.assertNotIn("workflow", result["metadata"]["raw"])
|
||||
self.assertEqual(result["metadata"]["workflow_graph"]["status"], "limited")
|
||||
# The dropped per-key workflow belonged to an unselected carrier when
|
||||
# an envelope wins the re-probe: that envelope is whole, not limited.
|
||||
envelope = json.dumps({"prompt": json.loads(source), "workflow": {"v": 2}})
|
||||
processes = [
|
||||
probe_process(b"x" * (VIDEO_PROBE_BYTES + 1)),
|
||||
probe_process(probe_payload({"comment": envelope})),
|
||||
]
|
||||
with (
|
||||
patch("utils.prompt_inspector.ffprobe_path", return_value="/tool/ffprobe"),
|
||||
patch("utils.prompt_inspector.subprocess.Popen", side_effect=processes),
|
||||
):
|
||||
_, result = await self.inspect("clip.mp4")
|
||||
self.assertEqual(result["prompts"]["positive"], self.side("bounded portrait"))
|
||||
self.assertEqual(result["metadata"]["workflow_graph"]["value"], {"v": 2})
|
||||
self.assertEqual(
|
||||
result["metadata"]["record_formats"]["workflow"], "reserialized-json"
|
||||
)
|
||||
# A workflow-only per-key video over budget is limited, not absent.
|
||||
processes = [
|
||||
probe_process(b"x" * (VIDEO_PROBE_BYTES + 1)),
|
||||
probe_process(probe_payload({})),
|
||||
]
|
||||
with (
|
||||
patch("utils.prompt_inspector.ffprobe_path", return_value="/tool/ffprobe"),
|
||||
patch("utils.prompt_inspector.subprocess.Popen", side_effect=processes),
|
||||
):
|
||||
_, result = await self.inspect("clip.mp4")
|
||||
self.assertEqual(result["metadata"]["status"], "available")
|
||||
self.assertEqual(result["metadata"]["record_formats"], {"workflow": "limited"})
|
||||
self.assertEqual(result["metadata"]["workflow_graph"]["status"], "limited")
|
||||
self.assertEqual(result["prompts"]["positive"]["status"], "absent")
|
||||
|
||||
async def test_video_role_description_wins_over_workflow_only_comment(self):
|
||||
evidence = {}
|
||||
source = graph("chosen", "")
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
"""Vocabulary read routes page with signed, epoch-scoped cursors."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
@@ -184,6 +186,7 @@ class TestVocabularyAPI(ImageAPITestCase):
|
||||
"id": created["tag"]["id"],
|
||||
"name": "sunset",
|
||||
"prompt_count": 0,
|
||||
"image_count": 0,
|
||||
},
|
||||
},
|
||||
)
|
||||
@@ -202,6 +205,7 @@ class TestVocabularyAPI(ImageAPITestCase):
|
||||
"other_prompts": 0,
|
||||
"negative_variants": 1,
|
||||
"archived_negative_variants": 0,
|
||||
"images": 0,
|
||||
},
|
||||
},
|
||||
)
|
||||
@@ -221,9 +225,11 @@ class TestVocabularyAPI(ImageAPITestCase):
|
||||
"id": created["tag"]["id"],
|
||||
"name": "sunset",
|
||||
"prompt_count": 2,
|
||||
"image_count": 0,
|
||||
},
|
||||
"components_changed": 1,
|
||||
"pairs_changed": 1,
|
||||
"images_changed": 0,
|
||||
},
|
||||
)
|
||||
self.assertEqual(self.api.db.get_prompt_by_id(parent)["tags"], ["sunset"])
|
||||
@@ -242,6 +248,7 @@ class TestVocabularyAPI(ImageAPITestCase):
|
||||
"deleted": True,
|
||||
"components_changed": 1,
|
||||
"pairs_changed": 1,
|
||||
"images_changed": 0,
|
||||
},
|
||||
)
|
||||
self.assertEqual(self.api.db.get_all_tags(), [])
|
||||
@@ -253,6 +260,95 @@ class TestVocabularyAPI(ImageAPITestCase):
|
||||
self.assertEqual(response.status, 400, bad)
|
||||
self.assertNotIn("code", result)
|
||||
|
||||
def image(self, name, tags, digest="a", mtime_ns=1):
|
||||
"""Tag the bytes ``digest`` of the file ``name``; return its locator."""
|
||||
locator = os.path.normcase(str(self.output_dir / name))
|
||||
self.api.db.replace_image_tags(
|
||||
locator=locator,
|
||||
sha256=hashlib.sha256(digest.encode()).hexdigest(),
|
||||
size=1,
|
||||
mtime_ns=mtime_ns,
|
||||
expected_epoch=self.api.db.get_execution_epoch(),
|
||||
expected_tag_revision=0,
|
||||
tags=tags,
|
||||
)
|
||||
return locator
|
||||
|
||||
async def test_tag_rows_report_image_count_by_locator(self):
|
||||
self._save_prompt("Prompt only", tags=["prompt-only", "shared"])
|
||||
self.image("one.png", ["twice", "shared"], digest="old")
|
||||
self.image("one.png", ["twice"], digest="new", mtime_ns=2)
|
||||
self.image("two.png", ["shared"], digest="new")
|
||||
self.image("three.png", ["prior-only"], digest="first")
|
||||
self.image("three.png", [], digest="second", mtime_ns=2)
|
||||
response, page = await self.get("tags")
|
||||
self.assertEqual(response.status, 200, page)
|
||||
rows = {row["name"]: row for row in page["tags"]}
|
||||
self.assertEqual(
|
||||
{
|
||||
name: (row["prompt_count"], row["image_count"])
|
||||
for name, row in rows.items()
|
||||
},
|
||||
{
|
||||
"prompt-only": (1, 0),
|
||||
"shared": (1, 2),
|
||||
"twice": (0, 1),
|
||||
"prior-only": (0, 1),
|
||||
},
|
||||
)
|
||||
row = rows["shared"]
|
||||
self.assertEqual(row["image_count"], 2)
|
||||
self.assertEqual(set(row), {"id", "name", "prompt_count", "image_count"})
|
||||
|
||||
async def test_impact_and_delete_counts_include_images(self):
|
||||
epoch = self.api.db.get_execution_epoch()
|
||||
self._save_prompt("Parent", tags=["moody"])
|
||||
self.image("a.png", ["moody", "keep"])
|
||||
self.image("b.png", ["moody"])
|
||||
_, page = await self.get("tags")
|
||||
moody = next(row for row in page["tags"] if row["name"] == "moody")
|
||||
self.assertEqual(moody["image_count"], 2)
|
||||
response, impact = await self.get(f"tags/{moody['id']}/impact")
|
||||
self.assertEqual(response.status, 200, impact)
|
||||
self.assertEqual(
|
||||
impact["impact"],
|
||||
{
|
||||
"positive_prompts": 1,
|
||||
"other_prompts": 0,
|
||||
"negative_variants": 0,
|
||||
"archived_negative_variants": 0,
|
||||
"images": 2,
|
||||
},
|
||||
)
|
||||
response, deleted = await self.send(
|
||||
"POST",
|
||||
f"tags/{moody['id']}/delete",
|
||||
{"expected_epoch": epoch, "name": "moody"},
|
||||
)
|
||||
self.assertEqual(response.status, 200, deleted)
|
||||
self.assertEqual(
|
||||
deleted,
|
||||
{
|
||||
"success": True,
|
||||
"library_epoch": epoch,
|
||||
"deleted": True,
|
||||
"components_changed": 1,
|
||||
"pairs_changed": 0,
|
||||
"images_changed": 2,
|
||||
},
|
||||
)
|
||||
keep = next(
|
||||
row for row in (await self.get("tags"))[1]["tags"] if row["name"] == "keep"
|
||||
)
|
||||
response, renamed = await self.send(
|
||||
"PATCH",
|
||||
f"tags/{keep['id']}",
|
||||
{"expected_epoch": epoch, "name": "keep", "new_name": "kept"},
|
||||
)
|
||||
self.assertEqual(response.status, 200, renamed)
|
||||
self.assertEqual(renamed["images_changed"], 1)
|
||||
self.assertEqual(renamed["tag"]["image_count"], 1)
|
||||
|
||||
async def test_create_route_validates_names(self):
|
||||
epoch = self.api.db.get_execution_epoch()
|
||||
before = self.tag_rows()
|
||||
|
||||
@@ -190,6 +190,26 @@ class TestWorkspaceImages(ImageAPITestCase):
|
||||
self.assertEqual(data["images"], [])
|
||||
self.assertIsNone(data["next_cursor"])
|
||||
|
||||
async def test_path_filter_lists_only_that_exact_file(self):
|
||||
# Activity (D57): a scan row opens its file by root and exact relative path,
|
||||
# never by a substring that also matches the same name in other folders.
|
||||
self.media("ComfyUI_00001_.png")
|
||||
for day in range(3):
|
||||
self.media(f"2026-10-0{day + 1}/ComfyUI_00001_.png")
|
||||
self.media("2026-10-01/ComfyUI_00001_.png.bak.png")
|
||||
_, data = await self.get(path="ComfyUI_00001_.png")
|
||||
self.assertEqual([i["path"] for i in data["images"]], ["ComfyUI_00001_.png"])
|
||||
_, data = await self.get(path="2026-10-02/ComfyUI_00001_.png")
|
||||
self.assertEqual(
|
||||
[i["path"] for i in data["images"]], ["2026-10-02/ComfyUI_00001_.png"]
|
||||
)
|
||||
_, data = await self.get(path="missing.png")
|
||||
self.assertEqual(data["images"], [])
|
||||
for bad in ("../outside.png", "a" * 1025):
|
||||
with self.subTest(bad=bad[:12]):
|
||||
response, _ = await self.get(path=bad)
|
||||
self.assertEqual(response.status, 400)
|
||||
|
||||
async def test_cursor_cannot_be_reused_with_changed_filters_or_tampered(self):
|
||||
self.media("a.png")
|
||||
self.media("b.png")
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
"""The page-workflow test server ends with its launcher (#274 review M2).
|
||||
|
||||
A Playwright worker that is SIGKILLed never runs its fixture teardown, so the
|
||||
server it started must notice on its own: with ``--exit-on-stdin-eof`` it shuts
|
||||
down when the launcher's end of its stdin pipe closes, and still removes its
|
||||
temp root. This runs on every OS in CI, where Windows is the strict case.
|
||||
"""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
SERVER_TIMEOUT_S = 60
|
||||
|
||||
|
||||
class TestBrowserServerLifetime(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
|
||||
def start(self, *extra):
|
||||
env = {**os.environ, "TMPDIR": self.tmp.name}
|
||||
env.update(TEMP=self.tmp.name, TMP=self.tmp.name)
|
||||
process = subprocess.Popen(
|
||||
[sys.executable, "-m", "tests.browser.server", "--port", "0", *extra],
|
||||
cwd=REPO_ROOT,
|
||||
env=env,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
|
||||
def stop():
|
||||
if process.poll() is None:
|
||||
process.kill()
|
||||
process.wait(timeout=SERVER_TIMEOUT_S)
|
||||
# communicate() would flush a closed stdin, which Python < 3.13 refuses.
|
||||
for stream in (process.stdin, process.stdout, process.stderr):
|
||||
stream.close()
|
||||
|
||||
self.addCleanup(stop)
|
||||
line = process.stdout.readline().decode()
|
||||
self.assertTrue(line.startswith("PM_TEST_SERVER_PORT="), line)
|
||||
url = f"http://127.0.0.1:{int(line.split('=', 1)[1])}/prompt_manager/test"
|
||||
deadline = time.monotonic() + SERVER_TIMEOUT_S
|
||||
while True:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=1) as response: # nosec B310
|
||||
self.assertEqual(response.status, 200)
|
||||
return process
|
||||
except OSError:
|
||||
self.assertIsNone(process.poll(), "the server exited early")
|
||||
self.assertLess(time.monotonic(), deadline, "never ready")
|
||||
time.sleep(0.1)
|
||||
|
||||
def roots(self):
|
||||
return sorted(path.name for path in Path(self.tmp.name).glob("pm-browser-*"))
|
||||
|
||||
def test_server_exits_and_removes_its_temp_root_when_stdin_closes(self):
|
||||
process = self.start("--exit-on-stdin-eof")
|
||||
self.assertEqual(len(self.roots()), 1, "the server works in its own temp root")
|
||||
process.stdin.close()
|
||||
code = process.wait(timeout=SERVER_TIMEOUT_S)
|
||||
self.assertEqual(
|
||||
code, 0, process.stderr.read().decode(errors="replace")[-4000:]
|
||||
)
|
||||
self.assertEqual(self.roots(), [], "the temp root and its database are removed")
|
||||
|
||||
def test_server_keeps_running_with_stdin_closed_without_the_flag(self):
|
||||
process = self.start()
|
||||
process.stdin.close()
|
||||
with self.assertRaises(subprocess.TimeoutExpired):
|
||||
process.wait(timeout=1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -452,6 +452,44 @@ class TestRestoreFromFile(BackupTestCase):
|
||||
).fetchone()
|
||||
)
|
||||
|
||||
def test_restore_old_backup_without_annotations(self):
|
||||
tables = (
|
||||
"media_image_tags",
|
||||
"media_annotation_bytes",
|
||||
"media_annotation_files",
|
||||
"media_annotation_metadata",
|
||||
)
|
||||
src = self._make_source_db(["predates image annotations"])
|
||||
with closing(sqlite3.connect(src)) as conn:
|
||||
conn.execute("PRAGMA foreign_keys=OFF")
|
||||
for table in tables:
|
||||
conn.execute(f"DROP TABLE {table}")
|
||||
conn.commit()
|
||||
self.assertTrue(PromptModel.verify_database_file(src)[0])
|
||||
ok, reason = PromptModel.verify_database_file(src, require_library_state=True)
|
||||
self.assertFalse(ok)
|
||||
self.assertEqual(reason, "Missing image annotation schema")
|
||||
|
||||
self.db.model.restore_from_file(src)
|
||||
|
||||
self.assertEqual(_texts(self.path), ["predates image annotations"])
|
||||
with closing(sqlite3.connect(self.path)) as conn:
|
||||
self.assertEqual(
|
||||
conn.execute("SELECT * FROM media_annotation_metadata").fetchall(),
|
||||
[(1, 1, 0)],
|
||||
)
|
||||
ok, reason = PromptModel.verify_database_file(
|
||||
self.path, require_library_state=True
|
||||
)
|
||||
self.assertTrue(ok, reason)
|
||||
with closing(sqlite3.connect(src)) as conn: # the source is untouched
|
||||
self.assertEqual(
|
||||
conn.execute(
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE name LIKE 'media_%'"
|
||||
).fetchone()[0],
|
||||
0,
|
||||
)
|
||||
|
||||
def test_restore_preserves_favorites_and_closes_handles(self):
|
||||
self.db.set_image_favorite("output", "live.png")
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
|
||||
@@ -200,6 +200,104 @@ class TestIdempotentMigrations(MigrationTestCase):
|
||||
|
||||
self.assertEqual(self._columns("prompt_library_state"), ["prompt_id", "title"])
|
||||
|
||||
def test_legacy_database_gains_annotation_group(self):
|
||||
self._legacy_db([])
|
||||
|
||||
self._open()
|
||||
|
||||
for table in (
|
||||
"media_annotation_metadata",
|
||||
"media_annotation_files",
|
||||
"media_annotation_bytes",
|
||||
"media_image_tags",
|
||||
):
|
||||
self.assertIn(table, self._table_names())
|
||||
conn = sqlite3.connect(self.path)
|
||||
try:
|
||||
self.assertEqual(
|
||||
conn.execute("SELECT * FROM media_annotation_metadata").fetchall(),
|
||||
[(1, 1, 0)],
|
||||
)
|
||||
# No backfill: a legacy database starts with no image tags.
|
||||
self.assertEqual(
|
||||
conn.execute("SELECT COUNT(*) FROM media_image_tags").fetchone()[0], 0
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def test_rebuild_keeps_annotation_rows(self):
|
||||
# The prompts rebuild runs with foreign keys off; image tags that
|
||||
# reference the tags table must survive it untouched.
|
||||
file_id = "0f8e4b2a-1c3d-4e5f-8a9b-0c1d2e3f4a5b"
|
||||
byte_id = "6a1b2c3d-4e5f-4a6b-8c9d-0e1f2a3b4c5d"
|
||||
self._legacy_db(
|
||||
[
|
||||
"INSERT INTO prompts (text, hash, workflow_name)"
|
||||
" VALUES ('a', 'h1', 'wf')",
|
||||
"CREATE TABLE tags (id INTEGER PRIMARY KEY AUTOINCREMENT,"
|
||||
" name TEXT NOT NULL UNIQUE)",
|
||||
"INSERT INTO tags (id, name) VALUES (3, 'img-only')",
|
||||
"CREATE TABLE media_annotation_metadata (singleton_id INTEGER"
|
||||
" PRIMARY KEY CHECK(singleton_id=1), schema_version INTEGER NOT"
|
||||
" NULL CHECK(schema_version=1), image_index_revision INTEGER NOT"
|
||||
" NULL DEFAULT 0 CHECK(image_index_revision>=0))",
|
||||
"INSERT INTO media_annotation_metadata VALUES (1, 1, 4)",
|
||||
"CREATE TABLE media_annotation_files (id TEXT PRIMARY KEY"
|
||||
" CHECK(length(id)=36), locator TEXT NOT NULL UNIQUE,"
|
||||
" created_at TEXT NOT NULL)",
|
||||
f"INSERT INTO media_annotation_files VALUES ('{file_id}',"
|
||||
" '/out/a.png', '2026-10-22T00:00:00.000000+00:00')",
|
||||
"CREATE TABLE media_annotation_bytes (id TEXT PRIMARY KEY"
|
||||
" CHECK(length(id)=36), file_id TEXT NOT NULL REFERENCES"
|
||||
" media_annotation_files(id) ON DELETE CASCADE, sha256 TEXT NOT"
|
||||
" NULL CHECK(length(sha256)=64 AND sha256 NOT GLOB"
|
||||
" '*[^0-9a-f]*'), size INTEGER NOT NULL CHECK(size>=0), mtime_ns"
|
||||
" INTEGER NOT NULL, verified_at TEXT NOT NULL, first_seen_at TEXT"
|
||||
" NOT NULL, tag_revision INTEGER NOT NULL DEFAULT 0"
|
||||
" CHECK(tag_revision>=0), UNIQUE(file_id, sha256))",
|
||||
"CREATE INDEX idx_media_annotation_bytes_fingerprint ON"
|
||||
" media_annotation_bytes(file_id, size, mtime_ns)",
|
||||
f"INSERT INTO media_annotation_bytes VALUES ('{byte_id}',"
|
||||
f" '{file_id}', '{'a' * 64}', 3, 1700000000000000000,"
|
||||
" '2026-10-22T00:00:00.000000+00:00',"
|
||||
" '2026-10-22T00:00:00.000000+00:00', 2)",
|
||||
"CREATE TABLE media_image_tags (byte_id TEXT NOT NULL REFERENCES"
|
||||
" media_annotation_bytes(id) ON DELETE CASCADE, tag_id INTEGER"
|
||||
" NOT NULL REFERENCES tags(id) ON DELETE CASCADE, position"
|
||||
" INTEGER NOT NULL CHECK(position>=0), PRIMARY KEY(byte_id,"
|
||||
" tag_id))",
|
||||
"CREATE INDEX idx_media_image_tags_tag ON media_image_tags(tag_id)",
|
||||
"CREATE UNIQUE INDEX idx_media_image_tags_position ON"
|
||||
" media_image_tags(byte_id, position)",
|
||||
f"INSERT INTO media_image_tags VALUES ('{byte_id}', 3, 0)",
|
||||
]
|
||||
)
|
||||
|
||||
self._open()
|
||||
|
||||
self.assertNotIn("workflow_name", self._columns("prompts"))
|
||||
conn = sqlite3.connect(self.path)
|
||||
try:
|
||||
self.assertEqual(
|
||||
conn.execute("SELECT * FROM media_annotation_metadata").fetchall(),
|
||||
[(1, 1, 4)],
|
||||
)
|
||||
self.assertEqual(
|
||||
conn.execute(
|
||||
"SELECT t.name, it.position FROM media_image_tags it"
|
||||
" JOIN tags t ON t.id = it.tag_id"
|
||||
).fetchall(),
|
||||
[("img-only", 0)],
|
||||
)
|
||||
self.assertEqual(
|
||||
conn.execute(
|
||||
"SELECT tag_revision FROM media_annotation_bytes"
|
||||
).fetchall(),
|
||||
[(2,)],
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def test_migration_error_is_logged_and_raised(self):
|
||||
# The legacy table has no CHECK on rating; the rebuilt one does, so the
|
||||
# copy into prompts_new fails inside the migration.
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
"""Several sampler branches that share one prompt pair resolve to that pair (#249).
|
||||
|
||||
The shape is a LoRA epoch-compare grid: one sampler per LoRA epoch, every
|
||||
sampler fed by the same positive ``CLIPTextEncode`` and the same
|
||||
``ConditioningZeroOut`` negative, every decoded image labelled, batched and
|
||||
concatenated into one grid that a single ``SaveImage`` writes. Which branch
|
||||
made which tile does not matter to the prompt: every branch used the same pair.
|
||||
Branches that differ stay ambiguous.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
|
||||
from utils import prompt_pair_resolver as resolver
|
||||
from utils.prompt_inspector import inspect_prompt_evidence
|
||||
from utils.prompt_pair_resolver import resolve_prompt_pair
|
||||
|
||||
POSITIVE = "a woman holding a coffee cup, in a beanie, sitting by a window"
|
||||
OTHER = "a man reading a newspaper on a park bench"
|
||||
NEGATIVE = "blurry, low quality"
|
||||
# Each profile with the negative it can read: inspection reads the zero-out.
|
||||
PROFILES = {
|
||||
"default": (resolver.DEFAULT_PROFILE, NEGATIVE),
|
||||
"inspection": (resolver.INSPECTION_PROFILE, None),
|
||||
}
|
||||
|
||||
|
||||
def _grid(epochs: int = 4, positives: dict = None, negative: str = None) -> dict:
|
||||
"""A compare grid; ``positives`` maps an epoch to its own positive text.
|
||||
|
||||
The negative is a ``ConditioningZeroOut`` of the positive, or with
|
||||
``negative`` an encoded text (the default profile cannot read a zero-out).
|
||||
"""
|
||||
graph = {
|
||||
"1": {
|
||||
"class_type": "UNETLoader",
|
||||
"inputs": {"unet_name": "base.safetensors", "weight_dtype": "default"},
|
||||
},
|
||||
"2": {
|
||||
"class_type": "CLIPLoader",
|
||||
"inputs": {"clip_name": "te.safetensors", "type": "stable_diffusion"},
|
||||
},
|
||||
"3": {"class_type": "VAELoader", "inputs": {"vae_name": "vae.safetensors"}},
|
||||
"4": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": POSITIVE, "clip": ["2", 0]},
|
||||
},
|
||||
"5": (
|
||||
{"class_type": "ConditioningZeroOut", "inputs": {"conditioning": ["4", 0]}}
|
||||
if negative is None
|
||||
else {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": negative, "clip": ["2", 0]},
|
||||
}
|
||||
),
|
||||
"6": {
|
||||
"class_type": "EmptyLatentImage",
|
||||
"inputs": {"width": 512, "height": 512, "batch_size": 1},
|
||||
},
|
||||
}
|
||||
batch = {}
|
||||
for epoch in range(epochs):
|
||||
base = 10 + epoch * 10
|
||||
lora, sampler, decode, label = (str(base + offset) for offset in range(4))
|
||||
positive = "4"
|
||||
if positives and epoch in positives:
|
||||
positive = str(base + 5)
|
||||
graph[positive] = {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": positives[epoch], "clip": ["2", 0]},
|
||||
}
|
||||
graph[lora] = {
|
||||
"class_type": "LoraLoaderModelOnly",
|
||||
"inputs": {
|
||||
"lora_name": f"subject_epoch_{epoch:02d}.safetensors",
|
||||
"strength_model": 1.0,
|
||||
"model": ["1", 0],
|
||||
},
|
||||
}
|
||||
graph[sampler] = {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"seed": 42,
|
||||
"steps": 8,
|
||||
"cfg": 1.0,
|
||||
"sampler_name": "euler",
|
||||
"scheduler": "simple",
|
||||
"denoise": 1.0,
|
||||
"model": [lora, 0],
|
||||
"positive": [positive, 0],
|
||||
"negative": ["5", 0],
|
||||
"latent_image": ["6", 0],
|
||||
},
|
||||
}
|
||||
graph[decode] = {
|
||||
"class_type": "VAEDecode",
|
||||
"inputs": {"samples": [sampler, 0], "vae": ["3", 0]},
|
||||
}
|
||||
graph[label] = {
|
||||
"class_type": "AddLabel",
|
||||
"inputs": {"text": f"epoch {epoch}", "image": [decode, 0]},
|
||||
}
|
||||
batch[f"images.image{epoch}"] = [label, 0]
|
||||
graph["900"] = {"class_type": "BatchImagesNode", "inputs": batch}
|
||||
graph["901"] = {
|
||||
"class_type": "ImageConcatFromBatch",
|
||||
"inputs": {"num_columns": 2, "images": ["900", 0]},
|
||||
}
|
||||
graph["902"] = {
|
||||
"class_type": "SaveImage",
|
||||
"inputs": {"filename_prefix": "compare/grid", "images": ["901", 0]},
|
||||
}
|
||||
return graph
|
||||
|
||||
|
||||
def _texts(result: dict) -> tuple:
|
||||
return tuple(
|
||||
result[role]["text"] if isinstance(result[role], dict) else None
|
||||
for role in ("positive", "negative")
|
||||
)
|
||||
|
||||
|
||||
class TestIdenticalSamplerBranches(unittest.TestCase):
|
||||
def test_grid_of_identical_branches_resolves_to_the_shared_pair(self):
|
||||
for name, (profile, negative) in PROFILES.items():
|
||||
with self.subTest(profile=name):
|
||||
result = resolve_prompt_pair(
|
||||
{"prompt": _grid(negative=negative)}, profile=profile
|
||||
)
|
||||
self.assertEqual(result["state"], "resolved")
|
||||
self.assertEqual(_texts(result), (POSITIVE, negative or ""))
|
||||
# No one branch is named: every branch made this pair.
|
||||
self.assertIsNone(result["sampler_id"])
|
||||
self.assertEqual(result["reason"], resolver.IDENTICAL_BRANCHES_REASON)
|
||||
# Every branch stays listed as the evidence for the shared pair.
|
||||
self.assertEqual(
|
||||
[item["sampler_id"] for item in result["candidates"]],
|
||||
["11", "21", "31", "41"],
|
||||
)
|
||||
self.assertEqual(
|
||||
{_texts(item) for item in result["candidates"]},
|
||||
{(POSITIVE, negative or "")},
|
||||
)
|
||||
self.assertEqual(result["reason_code"], None)
|
||||
if negative is None:
|
||||
self.assertEqual(
|
||||
result["negative"]["provenance"]["text_source"],
|
||||
"zeroed-conditioning",
|
||||
)
|
||||
|
||||
def test_default_profile_keeps_an_unreadable_zero_out_grid_ambiguous(self):
|
||||
# Ingestion cannot read ConditioningZeroOut: every branch is unresolved,
|
||||
# so the grid stays ambiguous and the scan keeps its legacy fallback.
|
||||
result = resolve_prompt_pair({"prompt": _grid()})
|
||||
self.assertEqual(result["state"], "ambiguous")
|
||||
self.assertEqual(result["reason"], "Multiple sampler branches")
|
||||
self.assertEqual(
|
||||
{item["state"] for item in result["candidates"]}, {"unresolved"}
|
||||
)
|
||||
|
||||
def test_single_branch_grid_still_names_its_sampler(self):
|
||||
for name, (profile, negative) in PROFILES.items():
|
||||
with self.subTest(profile=name):
|
||||
result = resolve_prompt_pair(
|
||||
{"prompt": _grid(epochs=1, negative=negative)}, profile=profile
|
||||
)
|
||||
self.assertEqual(result["state"], "resolved")
|
||||
self.assertEqual(result["sampler_id"], "11")
|
||||
self.assertEqual(result["reason"], "")
|
||||
|
||||
def test_branches_with_different_prompts_stay_ambiguous(self):
|
||||
for name, (profile, negative) in PROFILES.items():
|
||||
with self.subTest(profile=name):
|
||||
source = _grid(positives={2: OTHER}, negative=negative)
|
||||
result = resolve_prompt_pair({"prompt": source}, profile=profile)
|
||||
self.assertEqual(result["state"], "ambiguous")
|
||||
self.assertEqual(result["reason"], "Multiple sampler branches")
|
||||
self.assertIsNone(result["positive"])
|
||||
self.assertEqual(
|
||||
sorted(_texts(item)[0] for item in result["candidates"]),
|
||||
sorted([POSITIVE] * 3 + [OTHER]),
|
||||
)
|
||||
|
||||
def test_a_branch_that_cannot_be_read_keeps_the_grid_ambiguous(self):
|
||||
for name, (profile, negative) in PROFILES.items():
|
||||
with self.subTest(profile=name):
|
||||
source = _grid(negative=negative)
|
||||
source["31"]["inputs"]["negative"] = ["800", 0]
|
||||
source["800"] = {"class_type": "MysteryConditioning", "inputs": {}}
|
||||
result = resolve_prompt_pair({"prompt": source}, profile=profile)
|
||||
self.assertEqual(result["state"], "ambiguous")
|
||||
self.assertEqual(result["reason"], "Multiple sampler branches")
|
||||
|
||||
def test_identical_unreadable_branches_are_not_collapsed(self):
|
||||
source = _grid(epochs=2)
|
||||
source["5"] = {"class_type": "MysteryConditioning", "inputs": {}}
|
||||
for name, (profile, _negative) in PROFILES.items():
|
||||
with self.subTest(profile=name):
|
||||
result = resolve_prompt_pair({"prompt": source}, profile=profile)
|
||||
self.assertEqual(result["state"], "ambiguous")
|
||||
|
||||
def test_same_text_with_a_different_recipe_stays_ambiguous(self):
|
||||
adapters = {
|
||||
("RecipeText", 0): resolver.OutputAdapter(
|
||||
"text", "conditioning", compose=True
|
||||
)
|
||||
}
|
||||
source = _grid(epochs=2, negative=NEGATIVE)
|
||||
source["21"]["inputs"]["positive"] = ["700", 0]
|
||||
source["700"] = {
|
||||
"class_type": "RecipeText",
|
||||
"inputs": {"prepend_text": "a woman", "text": "holding a coffee cup"},
|
||||
}
|
||||
alone = resolve_prompt_pair(
|
||||
{"prompt": source}, output_node_id="21", adapters=adapters
|
||||
)
|
||||
# Branch "10" reads the same effective text through a plain encoder.
|
||||
source["4"]["inputs"]["text"] = alone["positive"]["text"]
|
||||
result = resolve_prompt_pair({"prompt": source}, adapters=adapters)
|
||||
self.assertEqual(result["state"], "ambiguous")
|
||||
self.assertEqual(
|
||||
{_texts(item) for item in result["candidates"]},
|
||||
{(alone["positive"]["text"], NEGATIVE)},
|
||||
)
|
||||
self.assertEqual(
|
||||
{item["positive"]["storage_text"] for item in result["candidates"]},
|
||||
{alone["positive"]["text"], "holding a coffee cup"},
|
||||
)
|
||||
|
||||
def test_inspector_shows_the_positive_and_an_explicitly_empty_negative(self):
|
||||
evidence = inspect_prompt_evidence({"prompt": _grid()})
|
||||
prompts = evidence["prompts"]
|
||||
self.assertEqual(prompts["positive"]["status"], "resolved")
|
||||
self.assertEqual(prompts["positive"]["text"], POSITIVE)
|
||||
self.assertEqual(prompts["negative"]["status"], "resolved")
|
||||
self.assertEqual(prompts["negative"]["text"], "")
|
||||
self.assertEqual(prompts["negative"]["source"], "zeroed-conditioning")
|
||||
self.assertEqual(evidence["resolution"]["state"], "resolved")
|
||||
self.assertIsNone(evidence["resolution"]["sampler_id"])
|
||||
|
||||
def test_inspector_keeps_differing_grids_ambiguous(self):
|
||||
evidence = inspect_prompt_evidence({"prompt": _grid(positives={0: OTHER})})
|
||||
self.assertEqual(evidence["resolution"]["state"], "ambiguous")
|
||||
self.assertEqual(evidence["prompts"]["positive"]["status"], "ambiguous")
|
||||
self.assertNotIn(POSITIVE, str(evidence["prompts"]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -292,6 +292,134 @@ class TestImageFavoriteApi(ImageAPITestCase):
|
||||
self.assertEqual(data, {"success": False, "error": "Favorite update failed"})
|
||||
|
||||
|
||||
class TestWorkspaceFavoritesFilter(ImageAPITestCase):
|
||||
"""``favorites=1`` on the workspace listing: the Favorites page (#249 D51)."""
|
||||
|
||||
async def tearDownAsync(self):
|
||||
self.api.db.close_all()
|
||||
assert_closed(self, self._temp_db.name, self._output_tmp.name)
|
||||
await super().tearDownAsync()
|
||||
|
||||
async def page(self, status=200, **params):
|
||||
response = await self.client.get(
|
||||
"/prompt_manager/images/output", params={"mode": "workspace", **params}
|
||||
)
|
||||
data = await response.json()
|
||||
self.assertEqual(response.status, status, data)
|
||||
return data
|
||||
|
||||
async def star(self, path, root_index=0, method="PUT"):
|
||||
response = await self.client.request(
|
||||
method, FAVORITE, json={"root_index": root_index, "path": path}
|
||||
)
|
||||
self.assertEqual(response.status, 200)
|
||||
|
||||
def make(self, *names):
|
||||
for offset, name in enumerate(names):
|
||||
target = make_png(self.output_dir / name)
|
||||
os.utime(target, (1_700_000_000 + offset, 1_700_000_000 + offset))
|
||||
|
||||
async def test_lists_only_favorites_and_counts_them(self):
|
||||
self.make("a.png", "sub/b.png", "c.png", "sub/d.png")
|
||||
await self.star("a.png")
|
||||
await self.star("sub/d.png")
|
||||
data = await self.page(favorites="1")
|
||||
self.assertEqual([i["path"] for i in data["images"]], ["sub/d.png", "a.png"])
|
||||
self.assertEqual(data["total"], 2)
|
||||
self.assertTrue(all(i["favorite"] is True for i in data["images"]))
|
||||
# The nav count's read: limit=1 still reports every favorite's total.
|
||||
self.assertEqual((await self.page(favorites="1", limit="1"))["total"], 2)
|
||||
# Other filters intersect with it.
|
||||
sub = await self.page(favorites="1", subfolder="sub")
|
||||
self.assertEqual([i["path"] for i in sub["images"]], ["sub/d.png"])
|
||||
self.assertEqual((await self.page(favorites="1", q="a.png"))["total"], 1)
|
||||
self.assertEqual((await self.page(favorites="1", media="video"))["total"], 0)
|
||||
# favorites=0 (and no parameter) is the whole listing.
|
||||
self.assertEqual((await self.page(favorites="0"))["total"], 4)
|
||||
self.assertEqual((await self.page())["total"], 4)
|
||||
await self.star("a.png", method="DELETE")
|
||||
self.assertEqual((await self.page(favorites="1"))["total"], 1)
|
||||
|
||||
async def test_cursor_pages_walk_only_favorites_in_every_order(self):
|
||||
self.make("a.png", "b.png", "c.png", "d.png", "e.png")
|
||||
for name in ("a.png", "c.png", "e.png"):
|
||||
await self.star(name)
|
||||
expected = {
|
||||
"newest": ["e.png", "c.png", "a.png"],
|
||||
"oldest": ["a.png", "c.png", "e.png"],
|
||||
"name": ["a.png", "c.png", "e.png"],
|
||||
"name-desc": ["e.png", "c.png", "a.png"],
|
||||
"rating": ["e.png", "c.png", "a.png"],
|
||||
}
|
||||
for sort, names in expected.items():
|
||||
seen, cursor = [], None
|
||||
while True:
|
||||
params = {"favorites": "1", "sort": sort, "limit": "1"}
|
||||
if cursor:
|
||||
params["cursor"] = cursor
|
||||
data = await self.page(**params)
|
||||
self.assertEqual(data["total"], 3, sort)
|
||||
seen += [i["path"] for i in data["images"]]
|
||||
cursor = data["next_cursor"]
|
||||
if not data["has_more"]:
|
||||
break
|
||||
self.assertEqual(seen, names, sort)
|
||||
# A cursor is bound to its filters: a whole-listing cursor is refused.
|
||||
whole = await self.page(limit="1")
|
||||
await self.page(400, favorites="1", cursor=whole["next_cursor"])
|
||||
|
||||
async def test_favorites_follow_their_own_root(self):
|
||||
second = self.comfy_dir / "second"
|
||||
make_png(second / "a.png")
|
||||
make_png(self.output_dir / "a.png")
|
||||
self.api._get_all_output_dirs = lambda: [self.output_dir, second]
|
||||
await self.star("a.png", root_index=1)
|
||||
data = await self.page(favorites="1")
|
||||
self.assertEqual(
|
||||
[(i["root_index"], i["path"]) for i in data["images"]], [(1, "a.png")]
|
||||
)
|
||||
self.assertEqual((await self.page(favorites="1", root="0"))["total"], 0)
|
||||
self.assertEqual((await self.page(favorites="1", root="1"))["total"], 1)
|
||||
|
||||
async def test_windows_case_folded_favorite_is_listed(self):
|
||||
make_png(self.output_dir / "Sub" / "A.png")
|
||||
with patch("py.api.images._path_flavor", ntpath):
|
||||
await self.star("Sub/A.png")
|
||||
(self.output_dir / "Sub" / "A.png").rename(
|
||||
self.output_dir / "Sub" / "a.png"
|
||||
)
|
||||
(self.output_dir / "Sub").rename(self.output_dir / "sub")
|
||||
self.api.invalidate_gallery_cache()
|
||||
data = await self.page(favorites="1")
|
||||
self.assertEqual([i["path"] for i in data["images"]], ["sub/a.png"])
|
||||
|
||||
async def test_a_favorite_whose_file_is_gone_is_not_listed(self):
|
||||
self.make("a.png", "b.png")
|
||||
await self.star("a.png")
|
||||
await self.star("b.png")
|
||||
(self.output_dir / "b.png").unlink()
|
||||
self.api.invalidate_gallery_cache()
|
||||
data = await self.page(favorites="1")
|
||||
self.assertEqual([i["path"] for i in data["images"]], ["a.png"])
|
||||
self.assertEqual(data["total"], 1)
|
||||
|
||||
async def test_rejects_other_values(self):
|
||||
for value in ("true", "2", "", "yes"):
|
||||
data = await self.page(400, favorites=value)
|
||||
self.assertFalse(data["success"])
|
||||
|
||||
async def test_store_failure_is_an_error_not_an_unfiltered_page(self):
|
||||
self.make("a.png")
|
||||
|
||||
def broken(*_args, **_kwargs):
|
||||
raise sqlite3.OperationalError("database is locked")
|
||||
|
||||
self.api.db.get_root_favorites = broken
|
||||
data = await self.page(500, favorites="1")
|
||||
self.assertFalse(data["success"])
|
||||
self.assertNotIn("images", data)
|
||||
|
||||
|
||||
class TestImageFavoriteSchema(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmpdir = tempfile.mkdtemp()
|
||||
@@ -314,6 +442,10 @@ class TestImageFavoriteSchema(unittest.TestCase):
|
||||
db.get_image_favorites("output", ["a.png", "b.png"]), {"a.png"}
|
||||
)
|
||||
self.assertEqual(db.get_image_favorites("output", []), set())
|
||||
# One root's favorites, read through the primary key.
|
||||
self.assertEqual(db.get_root_favorites("output"), {"a.png"})
|
||||
self.assertEqual(db.get_root_favorites("other"), {"a.png"})
|
||||
self.assertEqual(db.get_root_favorites("missing"), set())
|
||||
many = [f"{n}.png" for n in range(2000)] + ["a.png"]
|
||||
self.assertEqual(db.get_image_favorites("output", many), {"a.png"})
|
||||
self.assertTrue(db.clear_image_favorite("output", "a.png"))
|
||||
|
||||
@@ -254,7 +254,9 @@ class TestOutcomeMessages(unittest.TestCase):
|
||||
def test_candidates_and_absence_keep_their_status_lines(self):
|
||||
graph = _runtime()
|
||||
graph["7"] = _encode("positive")
|
||||
graph["5"] = _ksampler()
|
||||
# A second sampler with its own prompt (#249: identical ones resolve).
|
||||
graph["8"] = _encode("other positive")
|
||||
graph["5"] = _ksampler("8")
|
||||
evidence = inspect_prompt_evidence({"prompt": graph})
|
||||
self.assertEqual(evidence["resolution"]["state"], "ambiguous")
|
||||
self.assertEqual(_messages(evidence), {"positive": None, "negative": None})
|
||||
|
||||
@@ -823,6 +823,88 @@ def _png_with_chunks(path, chunks):
|
||||
return path
|
||||
|
||||
|
||||
class TestVideoProbeBoundsMatchPngBudgets(unittest.TestCase):
|
||||
"""ffprobe escapes tag text; the stdout bounds must never gate a record
|
||||
that the identical PNG reads within its decoded budget (I3)."""
|
||||
|
||||
@staticmethod
|
||||
def _dense(size):
|
||||
# Quote-dense JSON text: every other byte escapes, so ffprobe's JSON
|
||||
# output is close to twice the decoded size.
|
||||
unit = '{"k":"v",' + "".join('"' for _ in range(6)) + "}"
|
||||
return (unit * (size // len(unit) + 1))[:size]
|
||||
|
||||
def _video_read(self, path, tags, bound):
|
||||
payload = json.dumps({"format": {"tags": tags}}).encode()
|
||||
self.assertGreater(len(payload), bound)
|
||||
process = MagicMock()
|
||||
process.stdout = io.BytesIO(payload)
|
||||
process.poll.return_value = 0
|
||||
process.returncode = 0
|
||||
with (
|
||||
patch("utils.prompt_inspector.ffprobe_path", return_value="/tool/ffprobe"),
|
||||
patch(
|
||||
"utils.prompt_inspector.subprocess.Popen", return_value=process
|
||||
) as spawn,
|
||||
):
|
||||
read = read_inspection_file(path)
|
||||
self.assertEqual(spawn.call_count, 1)
|
||||
self.assertTrue(process.stdout.closed)
|
||||
return read
|
||||
|
||||
def test_near_budget_records_read_the_same_as_png(self):
|
||||
from utils.prompt_inspector import (
|
||||
ROLE_METADATA_BYTES,
|
||||
VIDEO_PROBE_BYTES,
|
||||
VIDEO_REPROBE_BYTES,
|
||||
WORKFLOW_METADATA_BYTES,
|
||||
)
|
||||
|
||||
prompt = self._dense(ROLE_METADATA_BYTES - 1024)
|
||||
workflow = self._dense(WORKFLOW_METADATA_BYTES - 1024)
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
png = _png_with_chunks(
|
||||
Path(directory) / "same.png",
|
||||
[
|
||||
_text_chunk(b"tEXt", "prompt", prompt.encode("latin-1")),
|
||||
_text_chunk(b"tEXt", "workflow", workflow.encode("latin-1")),
|
||||
],
|
||||
)
|
||||
expected = read_inspection_file(png)
|
||||
self.assertEqual(
|
||||
expected.record_formats,
|
||||
{"prompt": "original-text", "workflow": "original-text"},
|
||||
)
|
||||
video = Path(directory) / "same.mp4"
|
||||
video.write_bytes(b"fixture")
|
||||
# Escaped output is over the old 6 MiB + 64 KiB stdout bound.
|
||||
read = self._video_read(
|
||||
video,
|
||||
{"prompt": prompt, "workflow": workflow},
|
||||
ROLE_METADATA_BYTES + WORKFLOW_METADATA_BYTES + 64 * 1024,
|
||||
)
|
||||
self.assertEqual(read.metadata, expected.metadata)
|
||||
self.assertEqual(read.record_formats, expected.record_formats)
|
||||
self.assertEqual(read.limited_records, frozenset())
|
||||
self.assertGreaterEqual(
|
||||
VIDEO_PROBE_BYTES,
|
||||
2 * (ROLE_METADATA_BYTES + WORKFLOW_METADATA_BYTES) + 64 * 1024,
|
||||
)
|
||||
self.assertGreaterEqual(
|
||||
VIDEO_REPROBE_BYTES, 2 * ROLE_METADATA_BYTES + 64 * 1024
|
||||
)
|
||||
# A role record near its budget alone escapes past 2 MiB and must
|
||||
# still read as the PNG does.
|
||||
png = _png_with_chunks(
|
||||
Path(directory) / "role.png",
|
||||
[_text_chunk(b"tEXt", "prompt", prompt.encode("latin-1"))],
|
||||
)
|
||||
expected = read_inspection_file(png)
|
||||
read = self._video_read(video, {"prompt": prompt}, ROLE_METADATA_BYTES)
|
||||
self.assertEqual(read.metadata, expected.metadata)
|
||||
self.assertEqual(read.record_formats, {"prompt": "original-text"})
|
||||
|
||||
|
||||
class TestPerRecordBudgets(unittest.TestCase):
|
||||
"""Role records keep the 2 MiB budget; a workflow has its own 4 MiB budget."""
|
||||
|
||||
|
||||
@@ -0,0 +1,436 @@
|
||||
"""Unit tests for py/api/jobs.py: the in-memory session job registry.
|
||||
|
||||
Fake runners only: no files, no database. Ordering is driven by events and
|
||||
awaited tasks, never by wall-clock bounds.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from py.api.jobs import ( # noqa: E402
|
||||
DEFAULT_SCAN_SCOPE,
|
||||
LOG_CAP_LINE,
|
||||
MAX_FINISHED_JOBS,
|
||||
MAX_LOG_LINES,
|
||||
MAX_OUTCOME_ROWS,
|
||||
SCAN_GROUP_NAMES,
|
||||
SCAN_GROUPS,
|
||||
SCAN_SCOPES,
|
||||
Job,
|
||||
JobBusy,
|
||||
JobRegistry,
|
||||
)
|
||||
|
||||
|
||||
async def _collect(job, stop_after=None):
|
||||
seen = []
|
||||
async with contextlib.aclosing(job.events()) as events:
|
||||
async for payload in events:
|
||||
seen.append(payload)
|
||||
if stop_after is not None and len(seen) >= stop_after:
|
||||
break
|
||||
return seen
|
||||
|
||||
|
||||
def _gated_runner(gate, publish=None):
|
||||
"""A runner that publishes ``publish`` (if any) and waits for ``gate``."""
|
||||
|
||||
async def runner(job):
|
||||
if publish is not None:
|
||||
job.publish(publish)
|
||||
await gate.wait()
|
||||
|
||||
return runner
|
||||
|
||||
|
||||
async def _noop(job):
|
||||
return None
|
||||
|
||||
|
||||
class TestJobFanOut(unittest.IsolatedAsyncioTestCase):
|
||||
"""The SSE fan-out the scan job always had, now on every Job."""
|
||||
|
||||
async def test_new_job_is_running_with_no_event(self):
|
||||
job = Job("scan", "Scan")
|
||||
self.assertTrue(job.running)
|
||||
self.assertIsNone(job.last_event)
|
||||
self.assertEqual(job.state, "running")
|
||||
self.assertEqual(len(job.id), 32)
|
||||
|
||||
async def test_subscriber_receives_events_until_finish(self):
|
||||
job = Job("scan", "Scan")
|
||||
consumer = asyncio.ensure_future(_collect(job))
|
||||
await asyncio.sleep(0)
|
||||
|
||||
job.publish({"type": "progress", "progress": 10})
|
||||
job.publish({"type": "complete"})
|
||||
job.finish("finished")
|
||||
|
||||
self.assertEqual(
|
||||
await asyncio.wait_for(consumer, 2),
|
||||
[{"type": "progress", "progress": 10}, {"type": "complete"}],
|
||||
)
|
||||
self.assertFalse(job.running)
|
||||
|
||||
async def test_late_subscriber_gets_latest_snapshot_then_live_events(self):
|
||||
job = Job("scan", "Scan")
|
||||
job.publish({"type": "progress", "progress": 10})
|
||||
job.publish({"type": "progress", "progress": 40})
|
||||
|
||||
consumer = asyncio.ensure_future(_collect(job))
|
||||
await asyncio.sleep(0)
|
||||
job.publish({"type": "complete"})
|
||||
job.finish()
|
||||
|
||||
self.assertEqual(
|
||||
await asyncio.wait_for(consumer, 2),
|
||||
[{"type": "progress", "progress": 40}, {"type": "complete"}],
|
||||
)
|
||||
|
||||
async def test_subscriber_after_finish_gets_only_the_final_event(self):
|
||||
job = Job("scan", "Scan")
|
||||
job.publish({"type": "progress", "progress": 99})
|
||||
job.publish({"type": "complete"})
|
||||
job.finish()
|
||||
|
||||
self.assertEqual(await _collect(job), [{"type": "complete"}])
|
||||
|
||||
async def test_departed_subscriber_is_forgotten(self):
|
||||
job = Job("scan", "Scan")
|
||||
job.publish({"type": "progress", "progress": 1})
|
||||
await _collect(job, stop_after=1)
|
||||
|
||||
self.assertEqual(job.subscriber_count, 0)
|
||||
job.publish({"type": "complete"}) # must not raise or queue anywhere
|
||||
job.finish()
|
||||
|
||||
async def test_second_subscriber_attaches_to_running_job(self):
|
||||
registry = JobRegistry()
|
||||
gate = asyncio.Event()
|
||||
job = registry.start(
|
||||
"scan",
|
||||
"Scan",
|
||||
record_changing=True,
|
||||
scope={},
|
||||
runner=_gated_runner(gate, {"type": "progress", "progress": 30}),
|
||||
)
|
||||
await asyncio.sleep(0) # the runner publishes its first event
|
||||
first = asyncio.ensure_future(_collect(job))
|
||||
second = asyncio.ensure_future(_collect(job))
|
||||
await asyncio.sleep(0)
|
||||
self.assertEqual(job.subscriber_count, 2)
|
||||
self.assertIs(registry.active("scan"), job)
|
||||
|
||||
job.publish({"type": "progress", "progress": 60})
|
||||
gate.set()
|
||||
await asyncio.wait_for(job.task, 2)
|
||||
|
||||
expected = [
|
||||
{"type": "progress", "progress": 30},
|
||||
{"type": "progress", "progress": 60},
|
||||
]
|
||||
self.assertEqual(await asyncio.wait_for(first, 2), expected)
|
||||
self.assertEqual(await asyncio.wait_for(second, 2), expected)
|
||||
self.assertEqual(job.summary()["progress"], 60)
|
||||
|
||||
async def test_finish_rejects_unknown_state_and_is_final(self):
|
||||
job = Job("scan", "Scan")
|
||||
with self.assertRaises(ValueError):
|
||||
job.finish("running")
|
||||
job.finish("cancelled")
|
||||
job.finish("finished") # a second finish keeps the first state
|
||||
self.assertEqual(job.state, "cancelled")
|
||||
self.assertIsNotNone(job.finished_at)
|
||||
|
||||
|
||||
class TestJobRegistry(unittest.IsolatedAsyncioTestCase):
|
||||
|
||||
async def test_start_lists_and_finishes_a_job(self):
|
||||
registry = JobRegistry()
|
||||
gate = asyncio.Event()
|
||||
job = registry.start(
|
||||
"scan",
|
||||
"Scan for metadata",
|
||||
record_changing=True,
|
||||
scope={"files": "new", "output_folders": 2},
|
||||
runner=_gated_runner(gate),
|
||||
)
|
||||
self.assertEqual(job.state, "running")
|
||||
self.assertIs(registry.get(job.id), job)
|
||||
self.assertIsNone(registry.get("missing"))
|
||||
summary = job.summary()
|
||||
self.assertEqual(
|
||||
set(summary),
|
||||
{
|
||||
"id",
|
||||
"kind",
|
||||
"title",
|
||||
"state",
|
||||
"started_at",
|
||||
"finished_at",
|
||||
"progress",
|
||||
"status_line",
|
||||
"processed",
|
||||
"total",
|
||||
"counters",
|
||||
"scope",
|
||||
"listed",
|
||||
"log_length",
|
||||
},
|
||||
)
|
||||
self.assertEqual(summary["scope"], {"files": "new", "output_folders": 2})
|
||||
# What the detail would list, so a view reads it only when that changed.
|
||||
self.assertEqual((summary["listed"], summary["log_length"]), ({}, 0))
|
||||
self.assertIsNone(summary["finished_at"])
|
||||
|
||||
gate.set()
|
||||
await asyncio.wait_for(job.task, 2)
|
||||
self.assertEqual(job.state, "finished")
|
||||
self.assertIsNone(registry.active("scan"))
|
||||
|
||||
# Running jobs list first, then finished ones newest first.
|
||||
older = registry.start(
|
||||
"thumbs", "Thumbnails", record_changing=False, scope={}, runner=_noop
|
||||
)
|
||||
await asyncio.wait_for(older.task, 2)
|
||||
hold = asyncio.Event()
|
||||
running = registry.start(
|
||||
"other",
|
||||
"Other",
|
||||
record_changing=False,
|
||||
scope={},
|
||||
runner=_gated_runner(hold),
|
||||
)
|
||||
listed = registry.list()
|
||||
self.assertEqual([j.id for j in listed], [running.id, older.id, job.id])
|
||||
|
||||
# Only the last MAX_FINISHED_JOBS finished jobs are kept.
|
||||
for index in range(MAX_FINISHED_JOBS + 5):
|
||||
done = registry.start(
|
||||
f"k{index}", "Fake", record_changing=False, scope={}, runner=_noop
|
||||
)
|
||||
await asyncio.wait_for(done.task, 2)
|
||||
listed = registry.list()
|
||||
finished = [j for j in listed if not j.running]
|
||||
self.assertEqual(len(finished), MAX_FINISHED_JOBS)
|
||||
self.assertEqual(finished[0].id, done.id)
|
||||
self.assertIsNone(registry.get(job.id))
|
||||
self.assertIs(listed[0], running)
|
||||
hold.set()
|
||||
await asyncio.wait_for(running.task, 2)
|
||||
|
||||
async def test_record_changing_jobs_are_exclusive(self):
|
||||
registry = JobRegistry()
|
||||
gate = asyncio.Event()
|
||||
first = registry.start(
|
||||
"scan",
|
||||
"Scan for metadata",
|
||||
record_changing=True,
|
||||
scope={},
|
||||
runner=_gated_runner(gate),
|
||||
)
|
||||
with self.assertRaises(JobBusy) as raised:
|
||||
registry.start(
|
||||
"maintenance",
|
||||
"Maintenance",
|
||||
record_changing=True,
|
||||
scope={},
|
||||
runner=_noop,
|
||||
)
|
||||
self.assertIs(raised.exception.running, first)
|
||||
self.assertIn("Scan for metadata", str(raised.exception))
|
||||
|
||||
reader = registry.start(
|
||||
"report", "Report", record_changing=False, scope={}, runner=_noop
|
||||
)
|
||||
await asyncio.wait_for(reader.task, 2)
|
||||
self.assertEqual(reader.state, "finished")
|
||||
|
||||
# One job per kind, even a read-only one.
|
||||
hold = asyncio.Event()
|
||||
registry.start(
|
||||
"report",
|
||||
"Report",
|
||||
record_changing=False,
|
||||
scope={},
|
||||
runner=_gated_runner(hold),
|
||||
)
|
||||
with self.assertRaises(JobBusy):
|
||||
registry.start(
|
||||
"report", "Report", record_changing=False, scope={}, runner=_noop
|
||||
)
|
||||
gate.set()
|
||||
hold.set()
|
||||
await asyncio.wait_for(first.task, 2)
|
||||
|
||||
async def test_cancel_is_idempotent_and_keeps_partial_counts(self):
|
||||
registry = JobRegistry()
|
||||
reached = asyncio.Event()
|
||||
proceed = asyncio.Event()
|
||||
|
||||
async def runner(job):
|
||||
for index in range(3):
|
||||
if job.cancelled:
|
||||
return
|
||||
job.record("added", {"path": f"a{index}.png", "root": "output"}, "+")
|
||||
if index == 0:
|
||||
reached.set()
|
||||
await proceed.wait()
|
||||
|
||||
job = registry.start(
|
||||
"scan",
|
||||
"Scan",
|
||||
record_changing=True,
|
||||
scope={},
|
||||
runner=runner,
|
||||
groups=SCAN_GROUP_NAMES,
|
||||
)
|
||||
await asyncio.wait_for(reached.wait(), 2)
|
||||
self.assertTrue(registry.cancel(job.id))
|
||||
self.assertTrue(registry.cancel(job.id)) # still running: still requested
|
||||
proceed.set()
|
||||
await asyncio.wait_for(job.task, 2)
|
||||
|
||||
self.assertEqual(job.state, "cancelled")
|
||||
self.assertEqual(job.counters["added"], 1) # the work done before the stop
|
||||
self.assertFalse(registry.cancel(job.id)) # finished: a no-op
|
||||
self.assertFalse(registry.cancel("missing"))
|
||||
self.assertEqual(job.state, "cancelled")
|
||||
|
||||
async def test_failing_runner_ends_failed_and_frees_the_slot(self):
|
||||
registry = JobRegistry()
|
||||
|
||||
async def runner(job):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
job = registry.start(
|
||||
"scan", "Scan", record_changing=True, scope={}, runner=runner
|
||||
)
|
||||
with self.assertLogs("py.api.jobs", level="ERROR"):
|
||||
await asyncio.wait_for(job.task, 2)
|
||||
self.assertEqual(job.state, "failed")
|
||||
self.assertIsNone(registry.active("scan"))
|
||||
|
||||
async def test_runner_that_finishes_its_own_job_keeps_its_state(self):
|
||||
registry = JobRegistry()
|
||||
|
||||
async def runner(job):
|
||||
job.finish("failed")
|
||||
|
||||
job = registry.start(
|
||||
"scan", "Scan", record_changing=True, scope={}, runner=runner
|
||||
)
|
||||
await asyncio.wait_for(job.task, 2)
|
||||
self.assertEqual(job.state, "failed")
|
||||
|
||||
|
||||
class TestJobOutcomesAndLog(unittest.TestCase):
|
||||
|
||||
def test_outcome_and_log_caps(self):
|
||||
job = Job("scan", "Scan", groups=SCAN_GROUP_NAMES)
|
||||
for index in range(MAX_OUTCOME_ROWS + 50):
|
||||
job.record("errors", {"path": f"f{index}.png", "root": "output"}, "!")
|
||||
self.assertEqual(len(job.outcomes["errors"]), MAX_OUTCOME_ROWS)
|
||||
self.assertEqual(job.counters["errors"], MAX_OUTCOME_ROWS + 50)
|
||||
self.assertEqual(job.outcomes["errors"][0]["path"], "f0.png")
|
||||
|
||||
job = Job("scan", "Scan")
|
||||
for index in range(MAX_LOG_LINES + 100):
|
||||
job.log_line(f"line {index}")
|
||||
self.assertEqual(len(job.log), MAX_LOG_LINES + 1)
|
||||
self.assertEqual(job.log[-1], LOG_CAP_LINE)
|
||||
self.assertTrue(job.log[0].endswith("line 0"))
|
||||
self.assertEqual(LOG_CAP_LINE, "[Log capped at 2,000 lines]")
|
||||
|
||||
def test_paths_are_root_relative(self):
|
||||
job = Job("scan", "Scan", groups=SCAN_GROUP_NAMES)
|
||||
absolute = os.path.abspath(os.path.join("outputs", "sub", "a.png"))
|
||||
job.record(
|
||||
"added",
|
||||
{"path": absolute, "root": "output", "detail": "x"},
|
||||
f"+ {absolute}",
|
||||
)
|
||||
job.record("known", {"path": "sub/b.png", "root": "output"}, "= sub/b.png")
|
||||
job.record("review", {"path": "C:\\out\\c.png", "root": "output"}, "?")
|
||||
rows = [row for group in job.outcomes.values() for row in group]
|
||||
for row in rows:
|
||||
self.assertFalse(os.path.isabs(row["path"]), row)
|
||||
self.assertNotIn(":", row["path"])
|
||||
self.assertEqual(job.outcomes["added"][0]["path"], "a.png")
|
||||
self.assertEqual(job.outcomes["known"][0]["path"], "sub/b.png")
|
||||
self.assertEqual(job.outcomes["review"][0]["path"], "c.png")
|
||||
self.assertEqual(job.outcomes["known"][0]["detail"], "")
|
||||
self.assertNotIn(absolute, "\n".join(job.log))
|
||||
|
||||
def test_record_rejects_unknown_group(self):
|
||||
job = Job("scan", "Scan", groups=SCAN_GROUP_NAMES)
|
||||
with self.assertRaises(ValueError):
|
||||
job.record("bogus", {"path": "a.png", "root": "output"}, "x")
|
||||
|
||||
def test_detail_and_progress(self):
|
||||
job = Job(
|
||||
"scan",
|
||||
"Scan",
|
||||
scope={"files": "new", "output_folders": 2},
|
||||
groups=SCAN_GROUP_NAMES,
|
||||
)
|
||||
job.update(progress=150, status_line="Processing file 3 of 4", processed=3)
|
||||
job.update(total=4)
|
||||
job.record("known", {"path": "a.png", "root": "output"}, "= a.png")
|
||||
detail = job.detail()
|
||||
self.assertEqual(detail["progress"], 100)
|
||||
self.assertEqual(detail["status_line"], "Processing file 3 of 4")
|
||||
self.assertEqual((detail["processed"], detail["total"]), (3, 4))
|
||||
self.assertEqual(detail["counters"]["known"], 1)
|
||||
self.assertEqual(detail["outcomes"]["known"][0]["path"], "a.png")
|
||||
self.assertEqual(len(detail["log"]), 1)
|
||||
detail["outcomes"]["known"].clear() # a copy, not the live list
|
||||
self.assertEqual(len(job.outcomes["known"]), 1)
|
||||
job.publish({"type": "progress", "progress": -5})
|
||||
self.assertEqual(job.summary()["progress"], 0)
|
||||
|
||||
def test_scan_groups_follow_the_counter_mapping(self):
|
||||
self.assertEqual(
|
||||
SCAN_GROUPS,
|
||||
{
|
||||
"added": "added",
|
||||
"linked": "added",
|
||||
"enriched": "added",
|
||||
"unchanged": "known",
|
||||
# Activity-2: a file the "new" scope skipped is Already known.
|
||||
"skipped": "known",
|
||||
"ambiguous": "review",
|
||||
"conflict": "review",
|
||||
"unresolved": "review",
|
||||
"failed": "errors",
|
||||
},
|
||||
)
|
||||
self.assertEqual((SCAN_SCOPES, DEFAULT_SCAN_SCOPE), (("new", "all"), "new"))
|
||||
self.assertEqual(
|
||||
SCAN_GROUP_NAMES, ("added", "known", "review", "no_metadata", "errors")
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
class TestListedSummary(unittest.TestCase):
|
||||
def test_summary_counts_listed_rows_and_log_lines(self):
|
||||
job = Job("scan", "Scan", groups=("added", "errors"))
|
||||
job.record(
|
||||
"added", {"path": "a.png", "root": "output", "root_index": 0}, "+ a.png"
|
||||
)
|
||||
job.record("errors", {"path": "b.png", "root_index": "x"}, "! b.png")
|
||||
summary = job.summary()
|
||||
self.assertEqual(summary["listed"], {"added": 1, "errors": 1})
|
||||
self.assertEqual(summary["log_length"], 2)
|
||||
rows = job.detail()["outcomes"]
|
||||
self.assertEqual(rows["added"][0]["root_index"], 0)
|
||||
# Only a whole non-negative number is a root index.
|
||||
self.assertIsNone(rows["errors"][0]["root_index"])
|
||||
@@ -0,0 +1,886 @@
|
||||
"""Bounded private capture store (#249 A3): exact bytes, hard limits, cleanup."""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import stat
|
||||
import struct
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
try:
|
||||
from tests.open_handles import assert_closed, open_handles_under
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from open_handles import assert_closed, open_handles_under
|
||||
|
||||
try: # guarded so a missing module fails assertions, not test loading
|
||||
from utils import media_capture as capture_module
|
||||
from utils.media_capture import (
|
||||
CAPTURE_EXTENSIONS,
|
||||
CaptureCapacity,
|
||||
CaptureChanged,
|
||||
CaptureDenied,
|
||||
CaptureStore,
|
||||
CaptureTooLarge,
|
||||
)
|
||||
except ImportError:
|
||||
capture_module = None
|
||||
CAPTURE_EXTENSIONS = None
|
||||
CaptureCapacity = CaptureChanged = CaptureDenied = CaptureTooLarge = None
|
||||
CaptureStore = None
|
||||
|
||||
EPOCH = "epoch-1"
|
||||
SCOPE = "a" * 64
|
||||
WAIT = 30.0 # generous ceiling for cross-thread handoffs, never a short clock
|
||||
|
||||
|
||||
def _chunk(kind: bytes, data: bytes) -> bytes:
|
||||
return (
|
||||
struct.pack(">I", len(data))
|
||||
+ kind
|
||||
+ data
|
||||
+ struct.pack(">I", zlib.crc32(kind + data))
|
||||
)
|
||||
|
||||
|
||||
def png_bytes(width: int, height: int, pixels: bytes = b"\x00" * 16) -> bytes:
|
||||
"""A structurally valid PNG whose header declares *width* x *height*."""
|
||||
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
return (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ _chunk(b"IHDR", ihdr)
|
||||
+ _chunk(b"IDAT", pixels)
|
||||
+ _chunk(b"IEND", b"")
|
||||
)
|
||||
|
||||
|
||||
class FakeClock:
|
||||
def __init__(self, now: float = 1_000.0):
|
||||
self.now = now
|
||||
|
||||
def __call__(self) -> float:
|
||||
return self.now
|
||||
|
||||
def advance(self, seconds: float) -> None:
|
||||
self.now += seconds
|
||||
|
||||
|
||||
class CaptureStoreTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.assertIsNotNone(capture_module, "utils.media_capture is not implemented")
|
||||
parent = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(parent.cleanup)
|
||||
self.addCleanup(assert_closed, self, parent.name)
|
||||
# The resolved path: macOS /var -> /private/var, Windows 8.3 names.
|
||||
self.parent = Path(os.path.realpath(parent.name))
|
||||
self.root = self.parent / "output"
|
||||
self.root.mkdir()
|
||||
self.clock = FakeClock()
|
||||
|
||||
def make_store(self, **options) -> "CaptureStore":
|
||||
options.setdefault("clock", self.clock)
|
||||
store = CaptureStore(self.parent / "captures", **options)
|
||||
self.addCleanup(store.close)
|
||||
return store
|
||||
|
||||
def write(self, relative: str, data: bytes) -> Path:
|
||||
path = self.root / relative
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(data)
|
||||
return path
|
||||
|
||||
def symlink(self, target, link):
|
||||
try:
|
||||
os.symlink(str(target), str(link))
|
||||
except (OSError, NotImplementedError) as error:
|
||||
self.skipTest(f"symlinks unavailable here: {error}")
|
||||
|
||||
def wait_for_change_time_after(self, ns: int) -> None:
|
||||
"""Block until a new file's ``st_ctime_ns`` passes *ns*.
|
||||
|
||||
File timestamps tick coarsely (Linux jiffies, Windows ~1-16 ms). Each
|
||||
probe is a fresh name: Windows reports the creation time (or, on
|
||||
Python 3.12+, the change time) and tunnels the creation time of a
|
||||
reused name for 15 s.
|
||||
"""
|
||||
deadline = time.monotonic() + WAIT
|
||||
for index in range(1_000_000):
|
||||
probe = self.parent / f"tick-{index}"
|
||||
probe.write_bytes(b"x")
|
||||
advanced = os.stat(probe).st_ctime_ns > ns
|
||||
probe.unlink()
|
||||
if advanced:
|
||||
return
|
||||
if time.monotonic() >= deadline:
|
||||
break
|
||||
time.sleep(0.001)
|
||||
self.fail("the filesystem clock did not advance") # pragma: no cover
|
||||
|
||||
def capture(self, store, relative="a.png", **options):
|
||||
options.setdefault("epoch", EPOCH)
|
||||
options.setdefault("scope", SCOPE)
|
||||
return store.capture(self.root, relative, **options)
|
||||
|
||||
def assert_empty(self, store):
|
||||
self.assertFalse(any(Path(store.directory).iterdir()))
|
||||
self.assertEqual(
|
||||
store.stats(), {"captures": 0, "reserved_bytes": 0, "workers": 0}
|
||||
)
|
||||
|
||||
# -- exact bytes and identity -------------------------------------------
|
||||
|
||||
def test_capture_hashes_and_copies_exact_bytes_owner_only(self):
|
||||
source_bytes = png_bytes(3, 2, os.urandom(40))
|
||||
source = self.write("a.png", source_bytes)
|
||||
store = self.make_store()
|
||||
|
||||
cap = self.capture(store, root_index=1)
|
||||
|
||||
self.assertEqual(Path(cap.path).read_bytes(), source_bytes)
|
||||
self.assertEqual(cap.sha256, hashlib.sha256(source_bytes).hexdigest())
|
||||
self.assertEqual(cap.mtime_ns, os.stat(source).st_mtime_ns)
|
||||
self.assertEqual(cap.size, len(source_bytes))
|
||||
self.assertEqual(
|
||||
cap.locator,
|
||||
os.path.normcase(os.path.join(os.path.realpath(self.root), "a.png")),
|
||||
)
|
||||
self.assertEqual((cap.root_index, cap.scope, cap.epoch), (1, SCOPE, EPOCH))
|
||||
self.assertEqual((cap.media_type, cap.width, cap.height), ("image", 3, 2))
|
||||
self.assertTrue(cap.inferable)
|
||||
self.assertIsNone(cap.inferable_reason)
|
||||
self.assertIsInstance(cap.token, str)
|
||||
self.assertGreaterEqual(len(cap.token), 32)
|
||||
self.assertNotIn(cap.token, str(cap.path))
|
||||
self.assertNotIn(str(self.root), str(cap.path))
|
||||
capture_path = Path(cap.path)
|
||||
self.assertEqual(capture_path.parent, Path(store.directory))
|
||||
self.assertEqual(capture_path.suffix, ".png")
|
||||
self.assertTrue(
|
||||
Path(store.directory).name.startswith("prompt-manager-captures-")
|
||||
)
|
||||
if os.name == "posix":
|
||||
self.assertEqual(stat.S_IMODE(os.stat(store.directory).st_mode) & 0o077, 0)
|
||||
self.assertEqual(stat.S_IMODE(os.stat(cap.path).st_mode) & 0o077, 0)
|
||||
self.assertIs(store.get(cap.token), cap)
|
||||
self.assertEqual(
|
||||
store.stats(),
|
||||
{"captures": 1, "reserved_bytes": len(source_bytes), "workers": 0},
|
||||
)
|
||||
self.assertTrue(store.release(cap.token))
|
||||
self.assertFalse(store.release(cap.token))
|
||||
self.assertIsNone(store.get(cap.token))
|
||||
self.assert_empty(store)
|
||||
assert_closed(self, self.parent)
|
||||
|
||||
def test_streams_in_bounded_chunks_never_the_whole_file(self):
|
||||
data = png_bytes(1, 1, os.urandom(5 * capture_module.CHUNK_SIZE + 7))
|
||||
self.write("big.png", data)
|
||||
store = self.make_store(max_file=len(data))
|
||||
sizes = []
|
||||
original = CaptureStore._read
|
||||
|
||||
def hook(store_, stream, size):
|
||||
sizes.append(size)
|
||||
return original(store_, stream, size)
|
||||
|
||||
with patch.object(CaptureStore, "_read", hook):
|
||||
cap = self.capture(store, "big.png")
|
||||
self.assertTrue(sizes)
|
||||
self.assertLessEqual(max(sizes), capture_module.CHUNK_SIZE)
|
||||
self.assertEqual(cap.sha256, hashlib.sha256(data).hexdigest())
|
||||
|
||||
def test_symlinked_file_or_root_escape_is_denied(self):
|
||||
outside = self.parent / "outside"
|
||||
outside.mkdir()
|
||||
(outside / "real.png").write_bytes(png_bytes(1, 1))
|
||||
self.write("a.png", png_bytes(1, 1))
|
||||
self.write("notes.txt", b"text")
|
||||
(self.root / "folder").mkdir()
|
||||
store = self.make_store()
|
||||
|
||||
denied = [
|
||||
"../outside/real.png",
|
||||
"a/../a.png",
|
||||
"./a.png",
|
||||
"a.png/",
|
||||
"",
|
||||
"/a.png",
|
||||
"C:/a.png",
|
||||
"a\\b.png",
|
||||
"a\x00.png",
|
||||
"notes.txt",
|
||||
"folder",
|
||||
"x" * 1021 + ".png",
|
||||
]
|
||||
for relative in denied:
|
||||
with self.subTest(relative=relative):
|
||||
with self.assertRaises(CaptureDenied):
|
||||
self.capture(store, relative)
|
||||
with self.assertRaises(CaptureDenied):
|
||||
self.capture(store, "a.png", root_index=True)
|
||||
with self.assertRaises(CaptureChanged):
|
||||
self.capture(store, "missing.png")
|
||||
self.assert_empty(store)
|
||||
|
||||
self.symlink(outside / "real.png", self.root / "link.png")
|
||||
self.symlink(outside, self.root / "linked-dir")
|
||||
with self.assertRaises(CaptureDenied):
|
||||
self.capture(store, "link.png")
|
||||
with self.assertRaises(CaptureDenied):
|
||||
self.capture(store, "linked-dir/real.png")
|
||||
self.assert_empty(store)
|
||||
# A symlink to a file inside the root is still an alias, never a locator.
|
||||
self.symlink(self.root / "a.png", self.root / "alias.png")
|
||||
with self.assertRaises(CaptureDenied):
|
||||
self.capture(store, "alias.png")
|
||||
self.assert_empty(store)
|
||||
|
||||
def test_locator_matches_listing_identity_for_symlinked_root(self):
|
||||
real_root = self.parent / "real-output"
|
||||
real_root.mkdir()
|
||||
(real_root / "sub").mkdir()
|
||||
source_bytes = png_bytes(2, 2)
|
||||
(real_root / "sub" / "a.png").write_bytes(source_bytes)
|
||||
link_root = self.parent / "link-output"
|
||||
self.symlink(real_root, link_root)
|
||||
store = self.make_store()
|
||||
|
||||
cap = store.capture(link_root, "sub/a.png", epoch=EPOCH, scope=SCOPE)
|
||||
|
||||
expected = os.path.normcase(
|
||||
os.path.join(os.path.realpath(link_root), "sub", "a.png")
|
||||
)
|
||||
self.assertEqual(cap.locator, expected)
|
||||
self.assertEqual(
|
||||
cap.locator, os.path.normcase(os.path.realpath(link_root / "sub/a.png"))
|
||||
)
|
||||
self.assertTrue(
|
||||
cap.locator.startswith(os.path.normcase(os.path.realpath(real_root)))
|
||||
)
|
||||
self.assertEqual(cap.sha256, hashlib.sha256(source_bytes).hexdigest())
|
||||
|
||||
# -- change detection ----------------------------------------------------
|
||||
|
||||
def test_change_during_copy_is_detected_and_temp_removed(self):
|
||||
source_bytes = png_bytes(1, 1, b"\x11" * 40)
|
||||
source = self.write("a.png", source_bytes)
|
||||
store = self.make_store(max_file=4096)
|
||||
original = CaptureStore._read
|
||||
|
||||
def appending(store_, stream, size):
|
||||
data = original(store_, stream, 16)
|
||||
if data and not getattr(appending, "done", False):
|
||||
appending.done = True
|
||||
with open(source, "ab") as handle:
|
||||
handle.write(b"\x22" * 8)
|
||||
return data
|
||||
|
||||
with patch.object(CaptureStore, "_read", appending):
|
||||
with self.assertRaises(CaptureChanged):
|
||||
store.capture(self.root, "a.png", epoch=EPOCH, scope=SCOPE)
|
||||
self.assert_empty(store)
|
||||
|
||||
# Same size, different bytes, mtime restored: only the change time
|
||||
# (POSIX ctime) can tell. Windows exposes no change time through stat,
|
||||
# so there the torn copy completes and its digest describes the copy.
|
||||
source.write_bytes(source_bytes)
|
||||
before = os.stat(source)
|
||||
self.wait_for_change_time_after(before.st_ctime_ns)
|
||||
rewritten = bytes(b ^ 0xFF for b in source_bytes)
|
||||
|
||||
def rewriting(store_, stream, size):
|
||||
data = original(store_, stream, 16)
|
||||
if data and not getattr(rewriting, "done", False):
|
||||
rewriting.done = True
|
||||
with open(source, "r+b") as handle:
|
||||
handle.write(rewritten)
|
||||
os.utime(source, ns=(before.st_atime_ns, before.st_mtime_ns))
|
||||
return data
|
||||
|
||||
# Only a change time can tell: POSIX always has one; Windows exposes
|
||||
# it from stat on Python 3.12+ (where the OS supports the fast path)
|
||||
# and never from fstat, so detection there is best effort. The test
|
||||
# asserts the outcome, not the platform: a detected change leaves no
|
||||
# temp file; an undetected one (Windows only) completes with a digest
|
||||
# that describes the copied bytes.
|
||||
with patch.object(CaptureStore, "_read", rewriting):
|
||||
try:
|
||||
cap = store.capture(self.root, "a.png", epoch=EPOCH, scope=SCOPE)
|
||||
except CaptureChanged:
|
||||
detected = True
|
||||
else:
|
||||
detected = False
|
||||
self.assertEqual(os.name, "nt", "only Windows may miss the rewrite")
|
||||
copied = Path(cap.path).read_bytes()
|
||||
self.assertEqual(cap.sha256, hashlib.sha256(copied).hexdigest())
|
||||
store.release(cap.token)
|
||||
if os.name == "posix":
|
||||
self.assertTrue(detected)
|
||||
self.assert_empty(store)
|
||||
self.assertEqual(os.stat(source).st_mtime_ns, before.st_mtime_ns)
|
||||
|
||||
if os.name == "posix":
|
||||
# The path now names another file (new inode) once the copy ends.
|
||||
source.write_bytes(source_bytes)
|
||||
|
||||
def replacing(store_, stream, size):
|
||||
data = original(store_, stream, size)
|
||||
if not data and not getattr(replacing, "done", False):
|
||||
replacing.done = True
|
||||
other = self.root / "other.png"
|
||||
other.write_bytes(source_bytes)
|
||||
os.replace(other, source)
|
||||
return data
|
||||
|
||||
with patch.object(CaptureStore, "_read", replacing):
|
||||
with self.assertRaises(CaptureChanged):
|
||||
store.capture(self.root, "a.png", epoch=EPOCH, scope=SCOPE)
|
||||
self.assert_empty(store)
|
||||
assert_closed(self, self.parent)
|
||||
|
||||
def test_size_cap_on_stat_and_stream(self):
|
||||
store = self.make_store(max_file=64)
|
||||
self.write("large.png", bytes(65))
|
||||
self.write("fits.png", bytes(64))
|
||||
|
||||
def never(store_, stream, size): # pragma: no cover - guard only
|
||||
raise AssertionError("an over-cap file must not be read")
|
||||
|
||||
with patch.object(CaptureStore, "_read", never):
|
||||
with self.assertRaises(CaptureTooLarge):
|
||||
self.capture(store, "large.png")
|
||||
self.assert_empty(store)
|
||||
|
||||
cap = self.capture(store, "fits.png")
|
||||
self.assertEqual(cap.size, 64)
|
||||
store.release(cap.token)
|
||||
|
||||
growing = self.write("growing.png", bytes(40))
|
||||
original = CaptureStore._read
|
||||
|
||||
def grow(store_, stream, size):
|
||||
data = original(store_, stream, 16)
|
||||
if data and not getattr(grow, "done", False):
|
||||
grow.done = True
|
||||
with open(growing, "ab") as handle:
|
||||
handle.write(bytes(40))
|
||||
return data
|
||||
|
||||
with patch.object(CaptureStore, "_read", grow):
|
||||
with self.assertRaises(CaptureTooLarge):
|
||||
self.capture(store, "growing.png")
|
||||
self.assert_empty(store)
|
||||
|
||||
# -- admission -----------------------------------------------------------
|
||||
|
||||
def test_count_byte_and_worker_admission_is_nonblocking(self):
|
||||
for name in ("a", "b", "c"):
|
||||
self.write(f"{name}.png", bytes(60))
|
||||
store = self.make_store(max_captures=2, max_bytes=1000)
|
||||
first = self.capture(store, "a.png")
|
||||
self.capture(store, "b.png")
|
||||
with self.assertRaises(CaptureCapacity):
|
||||
self.capture(store, "c.png")
|
||||
self.assertTrue(store.release(first.token))
|
||||
third = self.capture(store, "c.png")
|
||||
self.assertEqual(store.stats()["captures"], 2)
|
||||
store.invalidate_all()
|
||||
self.assertIsNone(store.get(third.token))
|
||||
self.assert_empty(store)
|
||||
|
||||
bytes_store = self.make_store(max_bytes=100)
|
||||
kept = self.capture(bytes_store, "a.png")
|
||||
with self.assertRaises(CaptureCapacity):
|
||||
self.capture(bytes_store, "b.png")
|
||||
self.assertEqual(bytes_store.stats()["reserved_bytes"], 60)
|
||||
bytes_store.release(kept.token)
|
||||
self.capture(bytes_store, "b.png")
|
||||
|
||||
workers = self.make_store(max_workers=1)
|
||||
entered, proceed = threading.Event(), threading.Event()
|
||||
original = CaptureStore._read
|
||||
results = {}
|
||||
|
||||
def blocking(store_, stream, size):
|
||||
if not entered.is_set():
|
||||
entered.set()
|
||||
self.assertTrue(proceed.wait(WAIT))
|
||||
return original(store_, stream, size)
|
||||
|
||||
def worker():
|
||||
try:
|
||||
results["capture"] = self.capture(workers, "a.png")
|
||||
except Exception as error: # pragma: no cover - reported below
|
||||
results["error"] = error
|
||||
|
||||
with patch.object(CaptureStore, "_read", blocking):
|
||||
thread = threading.Thread(target=worker)
|
||||
thread.start()
|
||||
try:
|
||||
self.assertTrue(entered.wait(WAIT))
|
||||
self.assertEqual(workers.stats()["workers"], 1)
|
||||
with self.assertRaises(CaptureCapacity):
|
||||
self.capture(workers, "b.png")
|
||||
finally:
|
||||
proceed.set()
|
||||
thread.join(WAIT)
|
||||
self.assertFalse(thread.is_alive())
|
||||
self.assertNotIn("error", results)
|
||||
self.assertEqual(workers.stats()["workers"], 0)
|
||||
self.assertEqual(results["capture"].size, 60)
|
||||
self.capture(workers, "b.png")
|
||||
|
||||
# -- lifetime ------------------------------------------------------------
|
||||
|
||||
def test_idle_and_hard_expiry_and_in_use_retention(self):
|
||||
self.write("a.png", png_bytes(1, 1))
|
||||
self.write("b.png", png_bytes(1, 1))
|
||||
store = self.make_store(idle=900, hard=1800)
|
||||
cap = self.capture(store, "a.png")
|
||||
|
||||
self.clock.advance(899)
|
||||
self.assertIs(store.get(cap.token), cap) # touches idle
|
||||
self.assertAlmostEqual(store.expires_in(cap.token), 900)
|
||||
self.clock.advance(899)
|
||||
self.assertIs(store.get(cap.token), cap)
|
||||
self.assertAlmostEqual(store.expires_in(cap.token), 2)
|
||||
self.clock.advance(2)
|
||||
self.assertIsNone(store.get(cap.token)) # hard limit despite touches
|
||||
self.assertIsNone(store.expires_in(cap.token))
|
||||
self.assertFalse(Path(cap.path).exists())
|
||||
self.assert_empty(store)
|
||||
|
||||
idle = self.capture(store, "a.png")
|
||||
self.clock.advance(900)
|
||||
store.sweep()
|
||||
self.assertIsNone(store.get(idle.token))
|
||||
self.assert_empty(store)
|
||||
|
||||
used = self.capture(store, "b.png")
|
||||
self.assertIs(store.acquire(used.token), used)
|
||||
self.assertIs(store.acquire(used.token), used)
|
||||
self.clock.advance(901)
|
||||
store.sweep()
|
||||
self.assertIs(store.get(used.token), used) # in use is not idle
|
||||
self.clock.advance(900)
|
||||
store.sweep()
|
||||
self.assertIsNone(store.get(used.token)) # expired for new requests
|
||||
self.assertIsNone(store.acquire(used.token))
|
||||
self.assertTrue(Path(used.path).exists()) # retained while in use
|
||||
self.assertEqual(store.stats()["captures"], 1)
|
||||
store.release_use(used.token)
|
||||
self.assertTrue(Path(used.path).exists())
|
||||
store.release_use(used.token)
|
||||
self.assertFalse(Path(used.path).exists())
|
||||
self.assert_empty(store)
|
||||
store.release_use(used.token) # unknown token: no error, nothing freed
|
||||
self.assertIsNone(store.acquire("unknown"))
|
||||
|
||||
def test_release_while_in_use_keeps_file_until_last_use_ends(self):
|
||||
self.write("a.png", png_bytes(1, 1))
|
||||
store = self.make_store()
|
||||
cap = self.capture(store, "a.png")
|
||||
self.assertIs(store.acquire(cap.token), cap)
|
||||
self.assertTrue(store.release(cap.token))
|
||||
self.assertIsNone(store.get(cap.token))
|
||||
self.assertTrue(Path(cap.path).exists())
|
||||
store.invalidate_all()
|
||||
self.assertTrue(Path(cap.path).exists())
|
||||
store.release_use(cap.token)
|
||||
self.assertFalse(Path(cap.path).exists())
|
||||
self.assert_empty(store)
|
||||
|
||||
def test_invalidate_all_and_close_remove_owned_directory_only(self):
|
||||
self.write("a.png", png_bytes(1, 1))
|
||||
self.write("b.png", png_bytes(1, 1))
|
||||
sibling = self.parent / "captures" / "sibling.txt"
|
||||
sibling.parent.mkdir()
|
||||
sibling.write_bytes(b"keep")
|
||||
store = CaptureStore(self.parent / "captures", clock=self.clock)
|
||||
try:
|
||||
a = self.capture(store, "a.png")
|
||||
b = self.capture(store, "b.png")
|
||||
self.assertEqual(Path(store.directory).parent, sibling.parent)
|
||||
store.invalidate_all()
|
||||
self.assertIsNone(store.get(a.token))
|
||||
self.assertIsNone(store.get(b.token))
|
||||
self.assert_empty(store)
|
||||
self.assertTrue(Path(store.directory).is_dir())
|
||||
c = self.capture(store, "a.png")
|
||||
self.assertTrue(Path(c.path).exists())
|
||||
finally:
|
||||
store.close()
|
||||
self.assertFalse(Path(store.directory).exists())
|
||||
self.assertEqual(sibling.read_bytes(), b"keep")
|
||||
self.assertTrue((self.root / "a.png").exists())
|
||||
store.close() # idempotent
|
||||
with self.assertRaises(RuntimeError):
|
||||
self.capture(store, "a.png")
|
||||
self.assertIsNone(store.get(c.token))
|
||||
assert_closed(self, self.parent)
|
||||
|
||||
def test_windows_handles_closed_before_unlink(self):
|
||||
source_bytes = png_bytes(1, 1, b"\x33" * 40)
|
||||
source = self.write("a.png", source_bytes)
|
||||
store = self.make_store()
|
||||
original_unlink = os.unlink
|
||||
unlinked = []
|
||||
|
||||
def checked_unlink(path, *args, **kwargs):
|
||||
self.assertEqual(open_handles_under(path), [])
|
||||
self.assertTrue(store._lock.acquire(blocking=False)) # no I/O under lock
|
||||
store._lock.release()
|
||||
unlinked.append(os.fspath(path))
|
||||
return original_unlink(path, *args, **kwargs)
|
||||
|
||||
with patch("os.unlink", checked_unlink):
|
||||
cap = self.capture(store, "a.png")
|
||||
self.assertEqual(open_handles_under(cap.path), [])
|
||||
self.assertEqual(open_handles_under(source), [])
|
||||
store.release(cap.token)
|
||||
original = CaptureStore._read
|
||||
|
||||
def appending(store_, stream, size):
|
||||
data = original(store_, stream, 16)
|
||||
if data and not getattr(appending, "done", False):
|
||||
appending.done = True
|
||||
with open(source, "ab") as handle:
|
||||
handle.write(b"\x44")
|
||||
return data
|
||||
|
||||
with patch.object(CaptureStore, "_read", appending):
|
||||
with self.assertRaises(CaptureChanged):
|
||||
self.capture(store, "a.png")
|
||||
self.assertEqual(open_handles_under(source), [])
|
||||
self.assertEqual(len(unlinked), 2)
|
||||
self.assertTrue(all(p.startswith(str(store.directory)) for p in unlinked))
|
||||
self.assert_empty(store)
|
||||
assert_closed(self, self.parent)
|
||||
|
||||
# -- inference readiness -------------------------------------------------
|
||||
|
||||
def test_dimension_limits_mark_generate_unsupported(self):
|
||||
cases = {
|
||||
"edge.png": (png_bytes(4096, 4096), True),
|
||||
"wide.png": (png_bytes(4097, 1), False),
|
||||
"tall.png": (png_bytes(1, 4097), False),
|
||||
}
|
||||
for name, (data, _) in cases.items():
|
||||
self.write(name, data)
|
||||
self.write("garbage.jpg", b"\xff\xd8not really a jpeg")
|
||||
store = self.make_store()
|
||||
|
||||
for name, (data, supported) in cases.items():
|
||||
with self.subTest(name=name):
|
||||
cap = self.capture(store, name)
|
||||
self.assertEqual(cap.inferable, supported)
|
||||
self.assertEqual((cap.width, cap.height) != (None, None), True)
|
||||
if supported:
|
||||
self.assertIsNone(cap.inferable_reason)
|
||||
else:
|
||||
self.assertIn("4096", cap.inferable_reason)
|
||||
store.release(cap.token)
|
||||
self.assertEqual(
|
||||
(capture_module.MAX_INFERENCE_SIDE, capture_module.MAX_INFERENCE_PIXELS),
|
||||
(4096, 16_777_216),
|
||||
)
|
||||
cap = self.capture(store, "garbage.jpg")
|
||||
self.assertEqual((cap.media_type, cap.width, cap.height), ("image", None, None))
|
||||
self.assertFalse(cap.inferable)
|
||||
self.assertIn("header", cap.inferable_reason)
|
||||
self.assertEqual(
|
||||
cap.sha256, hashlib.sha256(b"\xff\xd8not really a jpeg").hexdigest()
|
||||
)
|
||||
|
||||
def test_video_capture_is_not_inferable(self):
|
||||
data = b"\x00\x00\x00\x18ftypmp42" + os.urandom(32)
|
||||
self.write("clip.mp4", data)
|
||||
store = self.make_store()
|
||||
|
||||
def never(path): # pragma: no cover - guard only
|
||||
raise AssertionError("video captures are never probed")
|
||||
|
||||
with patch.object(capture_module, "read_inspection_file", never):
|
||||
cap = self.capture(store, "clip.mp4")
|
||||
self.assertEqual((cap.media_type, cap.width, cap.height), ("video", None, None))
|
||||
self.assertFalse(cap.inferable)
|
||||
self.assertEqual(
|
||||
cap.inferable_reason,
|
||||
"Auto-tagging is available for supported still images only.",
|
||||
)
|
||||
self.assertEqual(Path(cap.path).suffix, ".mp4")
|
||||
self.assertEqual(cap.sha256, hashlib.sha256(data).hexdigest())
|
||||
|
||||
# -- verification without a copy -------------------------------------
|
||||
|
||||
def test_verify_hashes_current_file_without_retaining_a_copy(self):
|
||||
source_bytes = png_bytes(1, 1, b"\x55" * 30)
|
||||
source = self.write("a.png", source_bytes)
|
||||
store = self.make_store(max_file=128, max_workers=1)
|
||||
|
||||
facts = store.verify(self.root, "a.png")
|
||||
|
||||
self.assertEqual(facts.sha256, hashlib.sha256(source_bytes).hexdigest())
|
||||
self.assertEqual(facts.size, len(source_bytes))
|
||||
self.assertEqual(facts.mtime_ns, os.stat(source).st_mtime_ns)
|
||||
self.assertEqual(
|
||||
facts.locator,
|
||||
os.path.normcase(os.path.join(os.path.realpath(self.root), "a.png")),
|
||||
)
|
||||
self.assert_empty(store)
|
||||
self.write("large.png", bytes(129))
|
||||
with self.assertRaises(CaptureTooLarge):
|
||||
store.verify(self.root, "large.png")
|
||||
with self.assertRaises(CaptureDenied):
|
||||
store.verify(self.root, "../a.png")
|
||||
self.assert_empty(store)
|
||||
|
||||
# -- arguments and failure paths ---------------------------------------
|
||||
|
||||
def test_constructor_and_arguments_are_validated(self):
|
||||
for options in (
|
||||
{"max_file": 0},
|
||||
{"max_captures": True},
|
||||
{"max_bytes": -1},
|
||||
{"max_workers": "2"},
|
||||
{"idle": 0},
|
||||
{"hard": -5},
|
||||
):
|
||||
with self.subTest(options=options):
|
||||
with self.assertRaises(ValueError):
|
||||
CaptureStore(self.parent / "captures", **options)
|
||||
self.write("a.png", png_bytes(1, 1))
|
||||
store = self.make_store()
|
||||
with self.assertRaises(ValueError):
|
||||
store.capture(self.root, "a.png", epoch="", scope=SCOPE)
|
||||
with self.assertRaises(ValueError):
|
||||
store.capture(self.root, "a.png", epoch=EPOCH, scope=None)
|
||||
with self.assertRaises(CaptureDenied):
|
||||
self.capture(store, root_index=-1)
|
||||
with self.assertRaises(CaptureDenied):
|
||||
store.capture(
|
||||
self.parent / "missing-root", "a.png", epoch=EPOCH, scope=SCOPE
|
||||
)
|
||||
self.assert_empty(store)
|
||||
|
||||
default = CaptureStore(clock=self.clock)
|
||||
try:
|
||||
self.assertEqual(
|
||||
Path(default.directory).parent, Path(tempfile.gettempdir())
|
||||
)
|
||||
self.assertTrue(Path(default.directory).is_dir())
|
||||
finally:
|
||||
default.close()
|
||||
self.assertFalse(Path(default.directory).exists())
|
||||
|
||||
def test_directory_target_and_unreadable_file(self):
|
||||
(self.root / "dir.png").mkdir()
|
||||
store = self.make_store()
|
||||
with self.assertRaises(CaptureDenied):
|
||||
self.capture(store, "dir.png")
|
||||
self.assert_empty(store)
|
||||
if os.name != "posix" or os.geteuid() == 0:
|
||||
return # read permission cannot be withdrawn here
|
||||
locked = self.write("locked.png", png_bytes(1, 1))
|
||||
os.chmod(locked, 0)
|
||||
try:
|
||||
with self.assertRaises(CaptureChanged):
|
||||
self.capture(store, "locked.png")
|
||||
finally:
|
||||
os.chmod(locked, 0o600)
|
||||
self.assert_empty(store)
|
||||
|
||||
def test_removal_failures_are_logged_and_never_touch_other_files(self):
|
||||
self.write("a.png", png_bytes(1, 1))
|
||||
store = CaptureStore(self.parent / "captures", clock=self.clock)
|
||||
stray = Path(store.directory) / "stray.bin"
|
||||
try:
|
||||
cap = self.capture(store, "a.png")
|
||||
self.assertIs(store.acquire(cap.token), cap)
|
||||
self.assertAlmostEqual(store.expires_in(cap.token), 1800) # in use
|
||||
store.release_use(cap.token)
|
||||
original_unlink = os.unlink
|
||||
|
||||
def failing(path, *args, **kwargs):
|
||||
if os.fspath(path) == cap.path:
|
||||
raise PermissionError("busy")
|
||||
return original_unlink(path, *args, **kwargs)
|
||||
|
||||
with patch("os.unlink", failing):
|
||||
with self.assertLogs("utils.media_capture", "WARNING") as logs:
|
||||
self.assertTrue(store.release(cap.token))
|
||||
self.assertIn("could not be removed", logs.output[0])
|
||||
self.assertIsNone(store.get(cap.token))
|
||||
self.assertTrue(Path(cap.path).exists())
|
||||
os.unlink(cap.path)
|
||||
stray.write_bytes(b"not ours")
|
||||
finally:
|
||||
with self.assertLogs("utils.media_capture", "WARNING") as logs:
|
||||
store.close()
|
||||
self.assertIn("could not be removed", logs.output[0])
|
||||
self.assertEqual(stray.read_bytes(), b"not ours")
|
||||
stray.unlink()
|
||||
os.rmdir(store.directory)
|
||||
assert_closed(self, self.parent)
|
||||
|
||||
# -- platform identity checks -------------------------------------------
|
||||
|
||||
def test_path_and_handle_stats_are_never_compared_across_apis(self):
|
||||
"""Windows (Python 3.12+) reports the change time from stat but the
|
||||
creation time from fstat, and may encode dev/ino differently; only
|
||||
size and mtime_ns may be compared across the two APIs."""
|
||||
source_bytes = png_bytes(2, 2, b"\x66" * 40)
|
||||
self.write("a.png", source_bytes)
|
||||
store = self.make_store()
|
||||
original_fstat = os.fstat
|
||||
|
||||
def shifted(fd):
|
||||
st = original_fstat(fd)
|
||||
return os.stat_result(
|
||||
(
|
||||
st.st_mode,
|
||||
st.st_ino + 7,
|
||||
st.st_dev + 7,
|
||||
st.st_nlink,
|
||||
st.st_uid,
|
||||
st.st_gid,
|
||||
st.st_size,
|
||||
st.st_atime,
|
||||
st.st_mtime,
|
||||
st.st_ctime + 1,
|
||||
),
|
||||
{
|
||||
"st_atime_ns": st.st_atime_ns,
|
||||
"st_mtime_ns": st.st_mtime_ns,
|
||||
"st_ctime_ns": st.st_ctime_ns + 1_000_000_000,
|
||||
},
|
||||
)
|
||||
|
||||
with patch.object(capture_module.os, "fstat", shifted):
|
||||
cap = self.capture(store, "a.png")
|
||||
facts = store.verify(self.root, "a.png")
|
||||
self.assertEqual(cap.sha256, hashlib.sha256(source_bytes).hexdigest())
|
||||
self.assertEqual(facts.sha256, cap.sha256)
|
||||
store.release(cap.token)
|
||||
self.assert_empty(store)
|
||||
|
||||
def test_change_time_moving_between_fstat_reads_is_detected(self):
|
||||
"""The Windows 3.13 / POSIX signal, forced on every platform: the
|
||||
descriptor's change time differs after the copy, size and mtime equal."""
|
||||
self.write("a.png", png_bytes(1, 1, b"\x77" * 40))
|
||||
store = self.make_store()
|
||||
original_fstat = os.fstat
|
||||
calls = []
|
||||
|
||||
def drifting(fd):
|
||||
st = original_fstat(fd)
|
||||
calls.append(fd)
|
||||
if len(calls) < 2:
|
||||
return st
|
||||
return os.stat_result(
|
||||
tuple(st)[:10],
|
||||
{
|
||||
"st_atime_ns": st.st_atime_ns,
|
||||
"st_mtime_ns": st.st_mtime_ns,
|
||||
"st_ctime_ns": st.st_ctime_ns + 1,
|
||||
},
|
||||
)
|
||||
|
||||
with patch.object(capture_module.os, "fstat", drifting):
|
||||
with self.assertRaises(CaptureChanged):
|
||||
self.capture(store, "a.png")
|
||||
self.assertEqual(len(calls), 2)
|
||||
self.assert_empty(store)
|
||||
|
||||
def test_fresh_file_captures_twice_on_every_platform(self):
|
||||
data = png_bytes(1, 1, os.urandom(3 * capture_module.CHUNK_SIZE))
|
||||
self.write("fresh.png", data)
|
||||
store = self.make_store()
|
||||
first = self.capture(store, "fresh.png")
|
||||
second = self.capture(store, "fresh.png")
|
||||
self.assertEqual(first.sha256, hashlib.sha256(data).hexdigest())
|
||||
self.assertEqual(second.sha256, first.sha256)
|
||||
self.assertNotEqual(first.token, second.token)
|
||||
self.assertEqual(store.stats()["captures"], 2)
|
||||
|
||||
def test_error_messages_never_name_server_paths(self):
|
||||
self.write("a.png", png_bytes(1, 1))
|
||||
self.write("dir.png/x", b"") # dir.png is a directory
|
||||
store = self.make_store(max_file=64)
|
||||
attempts = [
|
||||
lambda: self.capture(store, "missing.png"),
|
||||
lambda: self.capture(store, "a.png"), # over the 64-byte cap
|
||||
lambda: self.capture(store, "dir.png"),
|
||||
lambda: self.capture(store, "../a.png"),
|
||||
lambda: self.capture(store, "notes.txt"),
|
||||
lambda: store.verify(self.root, "missing.png"),
|
||||
]
|
||||
if os.name == "posix" and os.geteuid() != 0:
|
||||
locked = self.write("locked.png", bytes(10))
|
||||
os.chmod(locked, 0)
|
||||
self.addCleanup(os.chmod, locked, 0o600)
|
||||
attempts.append(lambda: self.capture(store, "locked.png"))
|
||||
for attempt in attempts:
|
||||
with self.assertRaises(capture_module.CaptureError) as caught:
|
||||
attempt()
|
||||
message = str(caught.exception)
|
||||
self.assertTrue(message)
|
||||
self.assertNotIn(str(self.root), message)
|
||||
self.assertNotIn(str(self.parent), message)
|
||||
self.assertNotIn(os.sep, message)
|
||||
self.assert_empty(store)
|
||||
|
||||
def test_close_during_a_capture_removes_the_directory_once_it_finishes(self):
|
||||
self.write("a.png", png_bytes(1, 1, bytes(64)))
|
||||
store = CaptureStore(self.parent / "captures", clock=self.clock)
|
||||
original = CaptureStore._read
|
||||
|
||||
def closing(store_, stream, size):
|
||||
data = original(store_, stream, 16)
|
||||
if data and not getattr(closing, "done", False):
|
||||
closing.done = True
|
||||
store.close()
|
||||
return data
|
||||
|
||||
with patch.object(CaptureStore, "_read", closing):
|
||||
with self.assertRaises(RuntimeError):
|
||||
self.capture(store, "a.png")
|
||||
self.assertFalse(Path(store.directory).exists())
|
||||
self.assertEqual(store.stats()["reserved_bytes"], 0)
|
||||
store.close()
|
||||
assert_closed(self, self.parent)
|
||||
|
||||
# -- constants ---------------------------------------------------------
|
||||
|
||||
def test_defaults_match_the_plan(self):
|
||||
store = self.make_store()
|
||||
self.assertEqual(
|
||||
(
|
||||
store.max_file,
|
||||
store.max_captures,
|
||||
store.max_bytes,
|
||||
store.max_workers,
|
||||
store.idle,
|
||||
store.hard,
|
||||
),
|
||||
(256 * 2**20, 8, 512 * 2**20, 2, 900, 1800),
|
||||
)
|
||||
from py.api.inspection_service import SERVABLE_EXTENSIONS
|
||||
|
||||
self.assertEqual(CAPTURE_EXTENSIONS, SERVABLE_EXTENSIONS)
|
||||
|
||||
def test_exceptions_carry_codes(self):
|
||||
self.assertEqual(CaptureCapacity.code, "review-capacity")
|
||||
self.assertEqual(CaptureChanged.code, "file-changed")
|
||||
self.assertEqual(CaptureTooLarge.code, "unsupported-media")
|
||||
self.assertEqual(CaptureDenied.code, "source-changed")
|
||||
for cls in (CaptureCapacity, CaptureChanged, CaptureTooLarge, CaptureDenied):
|
||||
self.assertTrue(issubclass(cls, capture_module.CaptureError))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -51,11 +51,34 @@ class TestPairImageLinking(unittest.TestCase):
|
||||
self.assertEqual(pairs[0]["image_count"], 1)
|
||||
self.assertEqual(pairs[0]["run_count"], 0)
|
||||
|
||||
def test_parallel_identical_branches_create_their_shared_pair(self):
|
||||
# #249: two samplers on one positive and one negative are one certain pair.
|
||||
meta = metadata()
|
||||
meta["prompt"]["4"] = {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {"positive": ["1", 0], "negative": ["2", 0]},
|
||||
}
|
||||
self.handler.link_image_to_prompt(self.path, {"id": self.pid}, meta)
|
||||
pairs = self.db.list_prompt_pairs()["pairs"]
|
||||
self.assertEqual(
|
||||
[(p["positive_text"], p["negative_text"]) for p in pairs], [("cat", "blur")]
|
||||
)
|
||||
self.assertEqual(pairs[0]["image_count"], 1)
|
||||
self.assertEqual(pairs[0]["run_count"], 0)
|
||||
|
||||
def test_stale_context_or_ambiguous_metadata_does_not_create_pair(self):
|
||||
meta = metadata(positive="dog")
|
||||
self.handler.link_image_to_prompt(self.path, {"id": self.pid}, meta)
|
||||
self.assertEqual(self.db.list_prompt_pairs()["total"], 0)
|
||||
meta = metadata()
|
||||
meta["prompt"]["4"] = meta["prompt"]["3"]
|
||||
# Two samplers with different positives (#249: identical ones resolve).
|
||||
meta["prompt"]["5"] = {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": "dog"},
|
||||
}
|
||||
meta["prompt"]["4"] = {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {"positive": ["5", 0], "negative": ["2", 0]},
|
||||
}
|
||||
self.handler.link_image_to_prompt(self.path + "2", {"id": self.pid}, meta)
|
||||
self.assertEqual(self.db.list_prompt_pairs()["total"], 0)
|
||||
|
||||
@@ -44,10 +44,15 @@ class TestPairScan(AdminAPITestCase):
|
||||
return [p["text"] for p in self.api.db.get_recent_prompts()["prompts"]]
|
||||
|
||||
def test_samplers_sharing_one_prompt_link_that_prompt(self):
|
||||
# #249: every branch resolves to one exact pair, so the pair is ingested.
|
||||
g = sampled_graph()
|
||||
g["5"] = {**g["3"]}
|
||||
self.assertEqual(self._scanned("multi.png", g), "added")
|
||||
self.assertEqual(self._texts(), ["cat"])
|
||||
self.assertEqual(sorted(self._texts()), ["blur", "cat"])
|
||||
pairs = self.api.db.list_prompt_pairs()["pairs"]
|
||||
self.assertEqual(
|
||||
[(p["positive_text"], p["negative_text"]) for p in pairs], [("cat", "blur")]
|
||||
)
|
||||
|
||||
def test_samplers_with_different_prompts_link_the_first_sampler_prompt(self):
|
||||
# The prompt the inspector and the legacy panels show for this file.
|
||||
@@ -115,9 +120,9 @@ class TestPairScan(AdminAPITestCase):
|
||||
self.assertEqual(self.api._ingest_scanned_file(path, meta), "unchanged")
|
||||
|
||||
async def test_cancel_finishes_at_safe_boundary_and_restart_is_idempotent(self):
|
||||
from py.api.scan_job import ScanJob
|
||||
from py.api.jobs import Job
|
||||
|
||||
job = ScanJob()
|
||||
job = Job("scan", "Scan")
|
||||
self.api._scan_job = job
|
||||
response = await self.client.post("/prompt_manager/scan/cancel")
|
||||
self.assertEqual(response.status, 200)
|
||||
@@ -128,7 +133,7 @@ class TestPairScan(AdminAPITestCase):
|
||||
self.assertEqual(counts["processed"], 0)
|
||||
self.assertEqual(self.api.db.list_prompt_pairs()["total"], 0)
|
||||
job.finish()
|
||||
self.api._scan_job = ScanJob()
|
||||
self.api._scan_job = Job("scan", "Scan")
|
||||
first = self.api._ingest_scan_batch_sync(
|
||||
[(self.output_dir / "cancel.png", {"prompt": json.dumps(graph())})]
|
||||
)
|
||||
|
||||
@@ -251,7 +251,19 @@ class PairSchemaTests(unittest.TestCase):
|
||||
[r[1] for r in conn.execute("PRAGMA table_info(prompt_library_state)")],
|
||||
["prompt_id", "title"],
|
||||
)
|
||||
self.assertEqual(
|
||||
[
|
||||
tuple(r)
|
||||
for r in conn.execute("SELECT * FROM media_annotation_metadata")
|
||||
],
|
||||
[(1, 1, 0)],
|
||||
)
|
||||
self.assertTrue(PromptModel.verify_database_file(live.db_path)[0])
|
||||
self.assertTrue(
|
||||
PromptModel.verify_database_file(live.db_path, require_library_state=True)[
|
||||
0
|
||||
]
|
||||
)
|
||||
|
||||
def test_restore_staged_verification_failure_does_not_replace_live_pages(self):
|
||||
live = self.model()
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Global tag maintenance preserves pair and standalone tag relationships."""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import sqlite3
|
||||
import tempfile
|
||||
@@ -10,6 +11,7 @@ try:
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from open_handles import assert_closed
|
||||
|
||||
from database.library_commands import LibraryCommandError
|
||||
from database.operations import PromptDatabase
|
||||
|
||||
|
||||
@@ -18,6 +20,7 @@ class TestPairTagMaintenance(unittest.TestCase):
|
||||
directory = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(directory.cleanup)
|
||||
self.addCleanup(assert_closed, self, directory.name)
|
||||
self.root = directory.name
|
||||
self.db = PromptDatabase(os.path.join(directory.name, "pair-tags.db"))
|
||||
self.addCleanup(self.db.close_all)
|
||||
|
||||
@@ -27,6 +30,36 @@ class TestPairTagMaintenance(unittest.TestCase):
|
||||
def tags(self, pair_id: int) -> list:
|
||||
return self.db.get_prompt_pair(pair_id)["tags"]
|
||||
|
||||
def image(self, name: str, tags: list, digest: str = "a", revision: int = 0):
|
||||
"""Tag the bytes ``digest`` of the file ``name``; return its locator."""
|
||||
locator = os.path.normcase(os.path.join(os.path.realpath(self.root), name))
|
||||
replace = getattr(self.db, "replace_image_tags", None)
|
||||
self.assertTrue(callable(replace), "replace_image_tags is missing")
|
||||
replace(
|
||||
locator=locator,
|
||||
sha256=hashlib.sha256(digest.encode()).hexdigest(),
|
||||
size=1,
|
||||
mtime_ns=1,
|
||||
expected_epoch=self.db.get_execution_epoch(),
|
||||
expected_tag_revision=revision,
|
||||
tags=tags,
|
||||
)
|
||||
return locator
|
||||
|
||||
def image_tags(self, locator: str, digest: str = "a") -> list:
|
||||
return self.db.read_image_tags(
|
||||
locator, hashlib.sha256(digest.encode()).hexdigest()
|
||||
)["tags"]
|
||||
|
||||
def image_rows(self) -> list:
|
||||
conn = self.db.model.get_connection()
|
||||
return [
|
||||
tuple(row)
|
||||
for row in conn.execute(
|
||||
"SELECT byte_id, tag_id, position FROM media_image_tags ORDER BY 1,3"
|
||||
)
|
||||
]
|
||||
|
||||
def test_rename_to_existing_tag_preserves_pair_links_and_deduplicates(self):
|
||||
first = self.pair("portrait", ["old", "new", "keep"])
|
||||
second = self.pair("landscape", ["old"])
|
||||
@@ -136,6 +169,207 @@ class TestPairTagMaintenance(unittest.TestCase):
|
||||
self.assertEqual(self.tags(other), ["source"])
|
||||
self.assertEqual(self.db.get_prompt_by_id(prompt)["tags"], ["source"])
|
||||
|
||||
def test_rename_and_merge_move_image_links_with_dedup(self):
|
||||
pair = self.pair("portrait", ["old", "new"])
|
||||
both = self.image("both.png", ["new", "old", "keep"])
|
||||
only_old = self.image("old.png", ["old"])
|
||||
result = self.db.rename_tag_all_prompts("old", "new")
|
||||
self.assertEqual(result["affected_count"], 0) # prompts only, as before
|
||||
self.assertEqual(self.tags(pair), ["new"])
|
||||
self.assertEqual(self.image_tags(both), ["new", "keep"])
|
||||
self.assertEqual(self.image_tags(only_old), ["new"])
|
||||
self.assertNotIn("old", self.db.get_all_tags())
|
||||
self.db.rename_tag_all_prompts("keep", "kept")
|
||||
self.assertEqual(self.image_tags(both), ["new", "kept"])
|
||||
merged = self.db.merge_tags(["kept", "new"], "final")
|
||||
self.assertEqual(merged["tags_merged"], 2)
|
||||
self.assertEqual(merged["affected_count"], 0)
|
||||
self.assertEqual(self.image_tags(both), ["final"])
|
||||
self.assertEqual(self.image_tags(only_old), ["final"])
|
||||
self.assertEqual(self.tags(pair), ["final"])
|
||||
epoch = self.db.get_execution_epoch()
|
||||
conn = self.db.model.get_connection()
|
||||
final = conn.execute("SELECT id FROM tags WHERE name='final'").fetchone()[0]
|
||||
native = self.db.rename_vocabulary_tag(
|
||||
final, expected_epoch=epoch, name="final", new_name="native"
|
||||
)
|
||||
self.assertEqual(native["images_changed"], 2)
|
||||
self.assertEqual(self.image_tags(both), ["native"])
|
||||
self.assertEqual(self.image_tags(only_old), ["native"])
|
||||
self.image("again.png", ["native", "merge-me"])
|
||||
source = conn.execute("SELECT id FROM tags WHERE name='merge-me'").fetchone()
|
||||
receipt = self.db.rename_vocabulary_tag(
|
||||
source[0], expected_epoch=epoch, name="merge-me", new_name="native"
|
||||
)
|
||||
self.assertTrue(receipt["merged"])
|
||||
self.assertEqual(receipt["images_changed"], 1)
|
||||
self.assertEqual(receipt["tag"]["image_count"], 3)
|
||||
self.assertEqual(len(self.image_rows()), 3)
|
||||
|
||||
def test_delete_cascades_image_links_and_keeps_legacy_counts(self):
|
||||
pair = self.pair("portrait", ["remove", "keep"])
|
||||
prompt = self.db.save_prompt("standalone", tags=["remove", "keep"])
|
||||
locator = self.image("a.png", ["keep", "remove"])
|
||||
result = self.db.delete_tag_all_prompts("remove")
|
||||
self.assertEqual(result["affected_count"], 1)
|
||||
self.assertEqual(self.tags(pair), ["keep"])
|
||||
self.assertEqual(self.db.get_prompt_by_id(prompt)["tags"], ["keep"])
|
||||
self.assertEqual(self.image_tags(locator), ["keep"])
|
||||
self.assertEqual(len(self.image_rows()), 1)
|
||||
epoch = self.db.get_execution_epoch()
|
||||
conn = self.db.model.get_connection()
|
||||
keep = conn.execute("SELECT id FROM tags WHERE name='keep'").fetchone()[0]
|
||||
impact = self.db.vocabulary_tag_impact(keep)["impact"]
|
||||
self.assertEqual(impact["images"], 1)
|
||||
deleted = self.db.delete_vocabulary_tag(keep, expected_epoch=epoch, name="keep")
|
||||
self.assertEqual(
|
||||
(
|
||||
deleted["components_changed"],
|
||||
deleted["pairs_changed"],
|
||||
deleted["images_changed"],
|
||||
),
|
||||
(1, 1, 1),
|
||||
)
|
||||
self.assertEqual(self.image_tags(locator), [])
|
||||
self.assertEqual(self.image_rows(), [])
|
||||
self.assertEqual(
|
||||
conn.execute("SELECT COUNT(*) FROM media_annotation_bytes").fetchone()[0], 1
|
||||
)
|
||||
|
||||
def index_revision(self) -> int:
|
||||
return (
|
||||
self.db.model.get_connection()
|
||||
.execute("SELECT image_index_revision FROM media_annotation_metadata")
|
||||
.fetchone()[0]
|
||||
)
|
||||
|
||||
def revision(self, locator: str, digest: str = "a") -> int:
|
||||
return self.db.read_image_tags(
|
||||
locator, hashlib.sha256(digest.encode()).hexdigest()
|
||||
)["tag_revision"]
|
||||
|
||||
def tag_id(self, name: str) -> int:
|
||||
conn = self.db.model.get_connection()
|
||||
return conn.execute("SELECT id FROM tags WHERE name=?", (name,)).fetchone()[0]
|
||||
|
||||
def test_vocabulary_maintenance_bumps_image_revisions_so_stale_saves_conflict(
|
||||
self,
|
||||
):
|
||||
forest = self.image("a.png", ["forest"])
|
||||
misty = self.image("b.png", ["mist"])
|
||||
self.pair("portrait", ["unlinked"])
|
||||
index = self.index_revision()
|
||||
epoch = self.db.get_execution_epoch()
|
||||
|
||||
# A rename of a tag no image carries moves nothing image-side.
|
||||
self.db.rename_vocabulary_tag(
|
||||
self.tag_id("unlinked"), expected_epoch=epoch, name="unlinked", new_name="u"
|
||||
)
|
||||
self.assertEqual(self.index_revision(), index)
|
||||
|
||||
self.db.rename_vocabulary_tag(
|
||||
self.tag_id("forest"), expected_epoch=epoch, name="forest", new_name="woods"
|
||||
)
|
||||
self.assertEqual(self.image_tags(forest), ["woods"])
|
||||
self.assertEqual((self.revision(forest), self.revision(misty)), (2, 1))
|
||||
self.assertEqual(self.index_revision(), index + 1)
|
||||
# The Tag dialog still open at revision 1 cannot recreate "forest".
|
||||
with self.assertRaises(LibraryCommandError) as caught:
|
||||
self.db.replace_image_tags(
|
||||
locator=forest,
|
||||
sha256=hashlib.sha256(b"a").hexdigest(),
|
||||
size=1,
|
||||
mtime_ns=1,
|
||||
expected_epoch=epoch,
|
||||
expected_tag_revision=1,
|
||||
tags=["forest", "mist"],
|
||||
)
|
||||
self.assertEqual(caught.exception.code, "tag-conflict")
|
||||
self.assertEqual(caught.exception.current["tags"], ["woods"])
|
||||
self.assertEqual(caught.exception.current["tag_revision"], 2)
|
||||
self.assertNotIn("forest", self.db.get_all_tags())
|
||||
|
||||
self.db.rename_tag_all_prompts("woods", "trees") # legacy rename
|
||||
self.assertEqual((self.revision(forest), self.revision(misty)), (3, 1))
|
||||
self.db.merge_tags(["trees"], "mist") # legacy merge: only the source side
|
||||
self.assertEqual(self.image_tags(forest), ["mist"])
|
||||
self.assertEqual((self.revision(forest), self.revision(misty)), (4, 1))
|
||||
self.db.rename_tag_all_prompts("mist", "fog") # both carry it now
|
||||
self.assertEqual((self.revision(forest), self.revision(misty)), (5, 2))
|
||||
self.db.delete_vocabulary_tag(
|
||||
self.tag_id("fog"), expected_epoch=epoch, name="fog"
|
||||
)
|
||||
self.assertEqual((self.revision(forest), self.revision(misty)), (6, 3))
|
||||
self.assertEqual((self.image_tags(forest), self.image_tags(misty)), ([], []))
|
||||
self.image("c.png", ["gone"])
|
||||
self.db.delete_tag_all_prompts("gone") # legacy delete
|
||||
self.assertEqual(self.revision(forest), 6)
|
||||
self.assertEqual(
|
||||
self.index_revision(), index + 7
|
||||
) # 6 maintenance ops + the c.png save
|
||||
|
||||
def test_image_only_and_prior_only_names_participate(self):
|
||||
only = self.image("only.png", ["image-only"])
|
||||
prior = self.image("prior.png", ["prior-only"], digest="old")
|
||||
self.image("prior.png", [], digest="new") # the file's bytes changed
|
||||
self.db.save_prompt("standalone", tags=["target"])
|
||||
merged = self.db.merge_tags(["image-only"], "target")
|
||||
self.assertEqual((merged["tags_merged"], merged["affected_count"]), (1, 0))
|
||||
self.assertEqual(self.image_tags(only), ["target"])
|
||||
self.assertEqual(
|
||||
self.db.rename_tag_all_prompts("prior-only", "renamed")["affected_count"], 0
|
||||
)
|
||||
self.assertEqual(self.image_tags(prior, "old"), ["renamed"])
|
||||
self.assertEqual(self.image_tags(prior, "new"), [])
|
||||
self.assertEqual(set(self.db.get_all_tags()), {"target", "renamed"})
|
||||
epoch = self.db.get_execution_epoch()
|
||||
conn = self.db.model.get_connection()
|
||||
renamed = conn.execute("SELECT id FROM tags WHERE name='renamed'").fetchone()[0]
|
||||
receipt = self.db.rename_vocabulary_tag(
|
||||
renamed, expected_epoch=epoch, name="renamed", new_name="target"
|
||||
)
|
||||
self.assertTrue(receipt["merged"])
|
||||
self.assertEqual(receipt["images_changed"], 1)
|
||||
self.assertEqual(self.image_tags(prior, "old"), ["target"])
|
||||
self.assertEqual(receipt["tag"]["image_count"], 2)
|
||||
|
||||
def test_image_junction_failure_rolls_back_prompt_pair_and_name_changes(self):
|
||||
pair = self.pair("portrait", ["source", "target"])
|
||||
prompt = self.db.save_prompt("standalone", tags=["source"])
|
||||
locator = self.image("a.png", ["source"])
|
||||
conn = self.db.model.get_connection()
|
||||
tables = ("tags", "prompt_tags", "prompt_pair_tags", "media_image_tags")
|
||||
before = [
|
||||
conn.execute(f"SELECT * FROM {table} ORDER BY 1,2").fetchall()
|
||||
for table in tables
|
||||
]
|
||||
source = conn.execute("SELECT id FROM tags WHERE name='source'").fetchone()[0]
|
||||
conn.execute(
|
||||
"CREATE TRIGGER reject_image_tag_change BEFORE UPDATE "
|
||||
"ON media_image_tags BEGIN SELECT RAISE(ABORT,'image tag failure'); END"
|
||||
)
|
||||
conn.commit()
|
||||
with self.assertRaises(sqlite3.IntegrityError):
|
||||
self.db.rename_vocabulary_tag(
|
||||
source,
|
||||
expected_epoch=self.db.get_execution_epoch(),
|
||||
name="source",
|
||||
new_name="target",
|
||||
)
|
||||
with self.assertRaises(sqlite3.IntegrityError):
|
||||
self.db.merge_tags(["source"], "target")
|
||||
with self.assertRaises(sqlite3.IntegrityError):
|
||||
self.db.rename_tag_all_prompts("source", "target")
|
||||
self.assertFalse(conn.in_transaction)
|
||||
after = [
|
||||
conn.execute(f"SELECT * FROM {table} ORDER BY 1,2").fetchall()
|
||||
for table in tables
|
||||
]
|
||||
self.assertEqual(after, before)
|
||||
self.assertEqual(self.tags(pair), ["source", "target"])
|
||||
self.assertEqual(self.db.get_prompt_by_id(prompt)["tags"], ["source"])
|
||||
self.assertEqual(self.image_tags(locator), ["source"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -54,13 +54,30 @@ class TestPromptPairResolver(unittest.TestCase):
|
||||
self.assertEqual(result["positive"]["text"], "portrait")
|
||||
self.assertIsNone(result["negative"])
|
||||
|
||||
def test_multiple_samplers_are_ambiguous_even_when_texts_match(self):
|
||||
def test_multiple_samplers_with_matching_texts_resolve_to_the_shared_pair(self):
|
||||
source = graph()
|
||||
source["4"] = {
|
||||
"class_type": "KSamplerAdvanced",
|
||||
"inputs": source["3"]["inputs"],
|
||||
}
|
||||
result = resolve_prompt_pair(source)
|
||||
self.assertEqual(result["state"], "resolved")
|
||||
self.assertEqual(
|
||||
(result["positive"]["text"], result["negative"]["text"]),
|
||||
("portrait", "blurry"),
|
||||
)
|
||||
self.assertIsNone(result["sampler_id"])
|
||||
self.assertEqual(result["reason"], resolver.IDENTICAL_BRANCHES_REASON)
|
||||
self.assertEqual(len(result["candidates"]), 2)
|
||||
|
||||
def test_multiple_samplers_with_different_texts_are_ambiguous(self):
|
||||
source = graph()
|
||||
source["5"] = {"class_type": "CLIPTextEncode", "inputs": {"text": "other"}}
|
||||
source["4"] = {
|
||||
"class_type": "KSamplerAdvanced",
|
||||
"inputs": {"positive": ["5", 0], "negative": ["2", 0]},
|
||||
}
|
||||
result = resolve_prompt_pair(source)
|
||||
self.assertEqual(result["state"], "ambiguous")
|
||||
self.assertEqual(len(result["candidates"]), 2)
|
||||
self.assertIsNone(result["positive"])
|
||||
@@ -2253,12 +2270,14 @@ class TestDetailerStages(unittest.TestCase):
|
||||
self.assertEqual(_sides(result), (parity.DETAILER_POS, parity.NEG))
|
||||
self.assertEqual(result["sampler_id"], "50")
|
||||
|
||||
def test_raw_and_detailed_saves_stay_ambiguous_without_an_anchor(self):
|
||||
def test_raw_and_detailed_saves_of_one_prompt_share_it_without_an_anchor(self):
|
||||
source = _detailed(same_prompt=True)
|
||||
source["14"] = {"class_type": "SaveImage", "inputs": {"images": ["8", 0]}}
|
||||
result = _inspect(source)
|
||||
self.assertEqual(result["state"], "ambiguous")
|
||||
self.assertEqual(result["reason"], "Multiple sampler branches")
|
||||
# #249: both saves used one pair, so it resolves with no sampler named.
|
||||
self.assertEqual(result["state"], "resolved")
|
||||
self.assertIsNone(result["sampler_id"])
|
||||
self.assertEqual(result["reason"], resolver.IDENTICAL_BRANCHES_REASON)
|
||||
self.assertEqual(
|
||||
sorted(item["sampler_id"] for item in result["candidates"]), ["3", "50"]
|
||||
)
|
||||
|
||||
@@ -0,0 +1,652 @@
|
||||
"""Saved views (D16): the persisted store, its restore schema and the API."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from test_api_images import ImageAPITestCase
|
||||
from open_handles import assert_closed
|
||||
except ImportError:
|
||||
from tests.test_api_images import ImageAPITestCase
|
||||
from tests.open_handles import assert_closed
|
||||
|
||||
from database.models import PromptModel
|
||||
from database.operations import PromptDatabase
|
||||
from database.saved_views import (
|
||||
MAX_STATE_BYTES,
|
||||
TABLE,
|
||||
SavedViewInvalid,
|
||||
encode_state,
|
||||
migrate_saved_views,
|
||||
normalize_state,
|
||||
saved_views_schema_error,
|
||||
validate_name,
|
||||
)
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
URL = "/prompt_manager/saved-views"
|
||||
|
||||
IMAGES_STATE = {
|
||||
"q": "cat",
|
||||
"category": "Animals",
|
||||
"sort": "newest",
|
||||
"source": "all",
|
||||
"folder": "",
|
||||
"media": "all",
|
||||
"tags": ["fluffy"],
|
||||
"favorites": False,
|
||||
}
|
||||
LIBRARY_STATE = {"q": "", "category": "", "sort": "rating", "tags": []}
|
||||
IMAGES_DEFAULTS = {
|
||||
"q": "",
|
||||
"category": "",
|
||||
"sort": "newest",
|
||||
"source": "all",
|
||||
"folder": "",
|
||||
"media": "all",
|
||||
"tags": [],
|
||||
"favorites": False,
|
||||
}
|
||||
LIBRARY_DEFAULTS = {"q": "", "category": "", "sort": "newest", "tags": []}
|
||||
# A row saved by an older build: no favorites key, a retired key, a sort
|
||||
# that is no longer offered and an over-long folder.
|
||||
OLD_IMAGES_STATE = json.dumps(
|
||||
{
|
||||
"q": "sunset",
|
||||
"category": "",
|
||||
"sort": "shuffle",
|
||||
"source": "all",
|
||||
"folder": "x" * 2000,
|
||||
"media": "image",
|
||||
"tags": ["golden"],
|
||||
"layout": "grid",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class TestSavedViewsApi(ImageAPITestCase):
|
||||
async def send(self, method, path="", body=None, **kwargs):
|
||||
if body is not None:
|
||||
kwargs["json"] = body
|
||||
response = await self.client.request(method, URL + path, **kwargs)
|
||||
return response, await response.json()
|
||||
|
||||
async def create(self, page="images", name="Cats", state=None):
|
||||
state = dict(
|
||||
IMAGES_STATE if page == "images" else LIBRARY_STATE, **(state or {})
|
||||
)
|
||||
response, data = await self.send(
|
||||
"POST", body={"page": page, "name": name, "state": state}
|
||||
)
|
||||
self.assertEqual(response.status, 201, data)
|
||||
return data["view"]
|
||||
|
||||
async def test_create_list_rename_delete_round_trip(self):
|
||||
response, data = await self.send("GET")
|
||||
self.assertEqual(response.status, 200)
|
||||
self.assertEqual(data, {"success": True, "views": []})
|
||||
|
||||
first = await self.create(name=" Cats ")
|
||||
self.assertEqual(
|
||||
first,
|
||||
{
|
||||
"id": first["id"],
|
||||
"page": "images",
|
||||
"name": "Cats",
|
||||
"state": IMAGES_STATE,
|
||||
},
|
||||
)
|
||||
second = await self.create(page="library", name="Best", state={})
|
||||
_, data = await self.send("GET")
|
||||
self.assertTrue(data["success"])
|
||||
self.assertEqual(data["views"], [first, second])
|
||||
|
||||
response, data = await self.send(
|
||||
"PATCH", f"/{first['id']}", {"name": " Kittens "}
|
||||
)
|
||||
self.assertEqual(response.status, 200)
|
||||
self.assertEqual(data, {"success": True, "view": {**first, "name": "Kittens"}})
|
||||
_, listed = await self.send("GET")
|
||||
# Renaming keeps the view's place in the list.
|
||||
self.assertEqual(
|
||||
[view["name"] for view in listed["views"]], ["Kittens", "Best"]
|
||||
)
|
||||
|
||||
response, data = await self.send("DELETE", f"/{first['id']}")
|
||||
self.assertEqual(response.status, 200)
|
||||
self.assertEqual(data, {"success": True, "id": first["id"]})
|
||||
_, listed = await self.send("GET")
|
||||
self.assertEqual(listed["views"], [second])
|
||||
|
||||
for method, body in (("PATCH", {"name": "Again"}), ("DELETE", None)):
|
||||
response, data = await self.send(method, f"/{first['id']}", body)
|
||||
self.assertEqual(response.status, 404)
|
||||
self.assertFalse(data["success"])
|
||||
self.assertIn("error", data)
|
||||
|
||||
async def test_views_are_ordered_by_position_and_ids_are_not_reused(self):
|
||||
views = [await self.create(name=f"View {n}") for n in range(3)]
|
||||
await self.send("DELETE", f"/{views[2]['id']}")
|
||||
later = await self.create(name="Later")
|
||||
self.assertGreater(later["id"], views[2]["id"])
|
||||
with self.api.db.model.get_connection() as conn:
|
||||
conn.execute(
|
||||
f"UPDATE {TABLE} SET position = -1 WHERE id = ?", (views[1]["id"],)
|
||||
)
|
||||
_, listed = await self.send("GET")
|
||||
self.assertEqual(
|
||||
[view["name"] for view in listed["views"]], ["View 1", "View 0", "Later"]
|
||||
)
|
||||
|
||||
async def test_create_rejects_invalid_bodies(self):
|
||||
good = {"page": "images", "name": "Cats", "state": IMAGES_STATE}
|
||||
cases = [
|
||||
["not an object"],
|
||||
{"page": "images", "name": "Cats"},
|
||||
{**good, "extra": 1},
|
||||
{**good, "page": "history"},
|
||||
{**good, "page": 1},
|
||||
{**good, "name": ""},
|
||||
{**good, "name": " "},
|
||||
{**good, "name": "x" * 101},
|
||||
{**good, "name": 7},
|
||||
{**good, "name": "bad\nname"},
|
||||
{**good, "state": "q=cat"},
|
||||
{**good, "state": None},
|
||||
{**good, "state": {**IMAGES_STATE, "unknown": ""}},
|
||||
{**good, "state": {k: v for k, v in IMAGES_STATE.items() if k != "q"}},
|
||||
{**good, "state": {**IMAGES_STATE, "q": 5}},
|
||||
{**good, "state": {**IMAGES_STATE, "q": "x" * 501}},
|
||||
{**good, "state": {**IMAGES_STATE, "category": "x" * 101}},
|
||||
{**good, "state": {**IMAGES_STATE, "sort": "random"}},
|
||||
{**good, "state": {**IMAGES_STATE, "media": "audio"}},
|
||||
{**good, "state": {**IMAGES_STATE, "source": "-1"}},
|
||||
{**good, "state": {**IMAGES_STATE, "source": 0}},
|
||||
{**good, "state": {**IMAGES_STATE, "folder": None}},
|
||||
{**good, "state": {**IMAGES_STATE, "folder": "x" * 1025}},
|
||||
{**good, "state": {**IMAGES_STATE, "tags": "cat"}},
|
||||
{**good, "state": {**IMAGES_STATE, "tags": [1]}},
|
||||
{**good, "state": {**IMAGES_STATE, "tags": [" "]}},
|
||||
{**good, "state": {**IMAGES_STATE, "tags": ["a,b"]}},
|
||||
{**good, "state": {**IMAGES_STATE, "tags": ["x" * 51]}},
|
||||
{**good, "state": {**IMAGES_STATE, "tags": [f"t{n}" for n in range(21)]}},
|
||||
# Favorites (the Favorites page's filter) is a JSON boolean.
|
||||
{**good, "state": {**IMAGES_STATE, "favorites": "true"}},
|
||||
{**good, "state": {**IMAGES_STATE, "favorites": 1}},
|
||||
{**good, "state": {**IMAGES_STATE, "favorites": None}},
|
||||
{
|
||||
**good,
|
||||
"state": {k: v for k, v in IMAGES_STATE.items() if k != "favorites"},
|
||||
},
|
||||
# Library has no source, folder or media, and its own sorts.
|
||||
{"page": "library", "name": "L", "state": IMAGES_STATE},
|
||||
{
|
||||
"page": "library",
|
||||
"name": "L",
|
||||
"state": {**LIBRARY_STATE, "sort": "name-desc"},
|
||||
},
|
||||
]
|
||||
for body in cases:
|
||||
with self.subTest(body=body):
|
||||
response, data = await self.send("POST", body=body)
|
||||
self.assertEqual(response.status, 400)
|
||||
self.assertFalse(data["success"])
|
||||
self.assertIsInstance(data["error"], str)
|
||||
response = await self.client.post(URL, data="{not json")
|
||||
self.assertEqual(response.status, 400)
|
||||
self.assertFalse((await response.json())["success"])
|
||||
_, listed = await self.send("GET")
|
||||
self.assertEqual(listed["views"], [])
|
||||
|
||||
async def test_encoded_state_is_capped(self):
|
||||
# Each value is within its own limit; together they pass 4 KB.
|
||||
state = {
|
||||
**IMAGES_STATE,
|
||||
"q": "é" * 500,
|
||||
"folder": "é" * 1024,
|
||||
"tags": ["é" * 50 for _ in range(20)],
|
||||
}
|
||||
encoded = json.dumps(state, ensure_ascii=False, separators=(",", ":"))
|
||||
self.assertGreater(len(encoded.encode("utf-8")), MAX_STATE_BYTES)
|
||||
response, data = await self.send(
|
||||
"POST", body={"page": "images", "name": "Big", "state": state}
|
||||
)
|
||||
self.assertEqual(response.status, 400)
|
||||
self.assertIn("4 KB", data["error"])
|
||||
|
||||
async def test_every_documented_value_is_accepted(self):
|
||||
for sort in ("newest", "oldest", "name", "name-desc", "rating"):
|
||||
for media in ("all", "image", "video"):
|
||||
await self.create(state={"sort": sort, "media": media, "source": "3"})
|
||||
favorite = await self.create(state={"favorites": True})
|
||||
self.assertIs(favorite["state"]["favorites"], True)
|
||||
for sort in ("newest", "oldest", "name", "rating"):
|
||||
await self.create(page="library", state={"sort": sort, "tags": ["a b"]})
|
||||
_, listed = await self.send("GET")
|
||||
self.assertEqual(len(listed["views"]), 20)
|
||||
|
||||
async def test_rename_and_delete_validate_id_and_body(self):
|
||||
view = await self.create()
|
||||
for path in ("/0", "/-1", "/abc", "/1.5", f"/{2**63}"):
|
||||
for method, body in (("PATCH", {"name": "X"}), ("DELETE", None)):
|
||||
with self.subTest(path=path, method=method):
|
||||
response, data = await self.send(method, path, body)
|
||||
self.assertEqual(response.status, 400)
|
||||
self.assertFalse(data["success"])
|
||||
for body in ({}, {"name": ""}, {"name": "X", "page": "library"}, ["X"]):
|
||||
with self.subTest(body=body):
|
||||
response, _ = await self.send("PATCH", f"/{view['id']}", body)
|
||||
self.assertEqual(response.status, 400)
|
||||
_, listed = await self.send("GET")
|
||||
self.assertEqual(listed["views"], [view])
|
||||
|
||||
async def test_database_failure_is_a_500_envelope(self):
|
||||
def broken():
|
||||
raise sqlite3.OperationalError("disk I/O error")
|
||||
|
||||
self.api.db.list_saved_views = broken
|
||||
with self.assertLogs("prompt_manager.api", level="ERROR"):
|
||||
response, data = await self.send("GET")
|
||||
self.assertEqual(response.status, 500)
|
||||
self.assertEqual(
|
||||
data, {"success": False, "error": "Saved view operation failed"}
|
||||
)
|
||||
|
||||
async def test_internal_value_errors_are_a_generic_500(self):
|
||||
# Only the request's own validation is a 400; a ValueError or
|
||||
# TypeError raised by the database layer is a server fault (M4).
|
||||
for error in (
|
||||
ValueError("secret parser detail"),
|
||||
TypeError("secret type detail"),
|
||||
json.JSONDecodeError("secret json detail", "{", 0),
|
||||
):
|
||||
|
||||
def broken(*_args, error=error):
|
||||
raise error
|
||||
|
||||
for method, path, body, attr in (
|
||||
("GET", "", None, "list_saved_views"),
|
||||
(
|
||||
"POST",
|
||||
"",
|
||||
{"page": "library", "name": "L", "state": LIBRARY_STATE},
|
||||
"create_saved_view",
|
||||
),
|
||||
("PATCH", "/1", {"name": "X"}, "rename_saved_view"),
|
||||
("DELETE", "/1", None, "delete_saved_view"),
|
||||
):
|
||||
with self.subTest(error=type(error).__name__, method=method):
|
||||
setattr(self.api.db, attr, broken)
|
||||
try:
|
||||
with self.assertLogs("prompt_manager.api", level="ERROR"):
|
||||
response, data = await self.send(method, path, body)
|
||||
finally:
|
||||
delattr(self.api.db, attr)
|
||||
self.assertEqual(response.status, 500)
|
||||
self.assertEqual(
|
||||
data,
|
||||
{"success": False, "error": "Saved view operation failed"},
|
||||
)
|
||||
|
||||
async def test_a_corrupt_state_is_skipped_by_the_listing(self):
|
||||
kept = await self.create(page="library", name="Kept", state={})
|
||||
with self.api.db.model.get_connection() as conn:
|
||||
conn.execute(
|
||||
f"INSERT INTO {TABLE} (page, name, state, position, created_at, "
|
||||
"updated_at) VALUES ('images', 'Broken', '{', 0, 'x', 'x')"
|
||||
)
|
||||
response, data = await self.send("GET")
|
||||
self.assertEqual(response.status, 200, data)
|
||||
self.assertEqual(data["views"], [kept])
|
||||
# Hidden from the listing, it is not found by rename either.
|
||||
broken_id = kept["id"] + 1
|
||||
response, _ = await self.send("PATCH", f"/{broken_id}", {"name": "X"})
|
||||
self.assertEqual(response.status, 404)
|
||||
response, _ = await self.send("DELETE", f"/{broken_id}")
|
||||
self.assertEqual(response.status, 200)
|
||||
|
||||
async def test_a_row_saved_under_older_rules_lists_normalized(self):
|
||||
stale = json.dumps(
|
||||
{
|
||||
"q": "cat",
|
||||
"sort": "random",
|
||||
"media": "video",
|
||||
"tags": ["ok", 7],
|
||||
"retired": True,
|
||||
}
|
||||
)
|
||||
with self.api.db.model.get_connection() as conn:
|
||||
conn.execute(
|
||||
f"INSERT INTO {TABLE} (page, name, state, position, created_at, "
|
||||
"updated_at) VALUES ('images', ' Old\tview ', ?, 0, 'x', 'x')",
|
||||
(stale,),
|
||||
)
|
||||
conn.execute(
|
||||
f"INSERT INTO {TABLE} (page, name, state, position, created_at, "
|
||||
"updated_at) VALUES ('library', '', '{}', 1, 'x', 'x')"
|
||||
)
|
||||
response, data = await self.send("GET")
|
||||
self.assertEqual(response.status, 200, data)
|
||||
images, library = data["views"]
|
||||
self.assertEqual(images["name"], "Oldview")
|
||||
self.assertEqual(
|
||||
images["state"],
|
||||
{**IMAGES_DEFAULTS, "q": "cat", "media": "video"},
|
||||
)
|
||||
self.assertEqual(library["name"], "Saved view")
|
||||
self.assertEqual(library["state"], LIBRARY_DEFAULTS)
|
||||
# A stale view still renames and deletes.
|
||||
response, data = await self.send(
|
||||
"PATCH", f"/{images['id']}", {"name": "Renamed"}
|
||||
)
|
||||
self.assertEqual(response.status, 200, data)
|
||||
self.assertEqual(data["view"]["state"], images["state"])
|
||||
|
||||
|
||||
class TestSavedViewsSchema(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmpdir = tempfile.mkdtemp()
|
||||
self.path = os.path.join(self.tmpdir, "prompts.db")
|
||||
self.db = PromptDatabase(self.path)
|
||||
self.addCleanup(self._cleanup)
|
||||
|
||||
def _cleanup(self):
|
||||
PromptModel.close_all_instances()
|
||||
assert_closed(self, self.tmpdir)
|
||||
for name in os.listdir(self.tmpdir):
|
||||
os.unlink(os.path.join(self.tmpdir, name))
|
||||
os.rmdir(self.tmpdir)
|
||||
|
||||
def _source(self, name="source.db", views=()):
|
||||
"""A closed database file holding a prompt and the given views."""
|
||||
path = os.path.join(self.tmpdir, name)
|
||||
src = PromptDatabase(path)
|
||||
try:
|
||||
src.save_prompt(text="kept", prompt_hash=generate_prompt_hash("kept"))
|
||||
for page, view_name, state in views:
|
||||
src.create_saved_view(page, view_name, state)
|
||||
finally:
|
||||
src.close_all()
|
||||
return path
|
||||
|
||||
def test_current_database_has_a_valid_table(self):
|
||||
conn = self.db.model.get_connection()
|
||||
self.assertIsNone(saved_views_schema_error(conn, allow_missing=False))
|
||||
ok, reason = PromptModel.verify_database_file(
|
||||
self.path, require_library_state=True
|
||||
)
|
||||
self.assertTrue(ok, reason)
|
||||
|
||||
def test_migration_is_idempotent(self):
|
||||
conn = self.db.model.get_connection()
|
||||
self.db.create_saved_view("images", "Cats", IMAGES_STATE)
|
||||
migrate_saved_views(conn)
|
||||
self.assertEqual(len(self.db.list_saved_views()), 1)
|
||||
|
||||
def test_backup_restore_round_trip_keeps_saved_views(self):
|
||||
self.db.create_saved_view("library", "Live", LIBRARY_STATE)
|
||||
src_path = self._source(
|
||||
views=[
|
||||
("images", "Cats", IMAGES_STATE),
|
||||
("library", "Best", LIBRARY_STATE),
|
||||
]
|
||||
)
|
||||
# The views may still sit in the WAL; the backup must carry them.
|
||||
backup = os.path.join(self.tmpdir, "backup.db")
|
||||
reopened = PromptDatabase(src_path)
|
||||
try:
|
||||
self.assertTrue(reopened.model.backup_database(backup))
|
||||
finally:
|
||||
reopened.close_all()
|
||||
|
||||
previous = self.db.model.restore_from_file(backup)
|
||||
|
||||
self.assertEqual(
|
||||
[
|
||||
(view["page"], view["name"], view["state"])
|
||||
for view in self.db.list_saved_views()
|
||||
],
|
||||
[("images", "Cats", IMAGES_STATE), ("library", "Best", LIBRARY_STATE)],
|
||||
)
|
||||
created = self.db.create_saved_view("images", "After", IMAGES_STATE)
|
||||
self.assertEqual(self.db.list_saved_views()[-1], created)
|
||||
with closing(sqlite3.connect(previous)) as conn:
|
||||
self.assertEqual(
|
||||
conn.execute(f"SELECT name FROM {TABLE}").fetchall(), [("Live",)]
|
||||
)
|
||||
|
||||
def test_restore_old_backup_without_saved_views(self):
|
||||
self.db.create_saved_view("images", "Live", IMAGES_STATE)
|
||||
src = self._source()
|
||||
with closing(sqlite3.connect(src)) as conn:
|
||||
conn.execute(f"DROP TABLE {TABLE}")
|
||||
conn.commit()
|
||||
self.assertTrue(PromptModel.verify_database_file(src)[0])
|
||||
ok, _ = PromptModel.verify_database_file(src, require_library_state=True)
|
||||
self.assertFalse(ok)
|
||||
|
||||
self.db.model.restore_from_file(src)
|
||||
|
||||
self.assertEqual(self.db.list_saved_views(), [])
|
||||
self.db.create_saved_view("library", "New", LIBRARY_STATE)
|
||||
self.assertEqual(len(self.db.list_saved_views()), 1)
|
||||
with closing(sqlite3.connect(src)) as conn: # the source is untouched
|
||||
self.assertIsNone(
|
||||
conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE name=?", (TABLE,)
|
||||
).fetchone()
|
||||
)
|
||||
|
||||
def _tampered(self, *statements, views=(("images", "Cats", IMAGES_STATE),)):
|
||||
src = self._source(
|
||||
name=f"tampered{len(os.listdir(self.tmpdir))}.db", views=views
|
||||
)
|
||||
with closing(sqlite3.connect(src)) as conn:
|
||||
for statement in statements:
|
||||
conn.execute(statement)
|
||||
conn.commit()
|
||||
return src
|
||||
|
||||
def test_restore_rejects_tampered_saved_views(self):
|
||||
cases = {
|
||||
"definition": (
|
||||
f"DROP TABLE {TABLE}",
|
||||
f"CREATE TABLE {TABLE} (id INTEGER PRIMARY KEY, page TEXT, "
|
||||
"name TEXT, state TEXT, position INTEGER, created_at TEXT, "
|
||||
"updated_at TEXT)",
|
||||
),
|
||||
"index": (f"CREATE INDEX saved_views_extra ON {TABLE}(name)",),
|
||||
"page": (
|
||||
"PRAGMA ignore_check_constraints = ON",
|
||||
f"UPDATE {TABLE} SET page = 'history'",
|
||||
),
|
||||
"long name": (f"UPDATE {TABLE} SET name = '{'x' * 401}'",),
|
||||
"blob name": (f"UPDATE {TABLE} SET name = CAST('Cats' AS BLOB)",),
|
||||
"state json": (f"UPDATE {TABLE} SET state = '{{'",),
|
||||
"state array": (f"UPDATE {TABLE} SET state = '[]'",),
|
||||
"state string": (f"UPDATE {TABLE} SET state = '\"q\"'",),
|
||||
"state size": (
|
||||
f"UPDATE {TABLE} SET state = '{'x' * (MAX_STATE_BYTES + 1)}'",
|
||||
),
|
||||
"position": (f"UPDATE {TABLE} SET position = 'first'",),
|
||||
"timestamp": (f"UPDATE {TABLE} SET created_at = X'00'",),
|
||||
}
|
||||
for label, statements in cases.items():
|
||||
with self.subTest(label=label):
|
||||
src = self._tampered(*statements)
|
||||
ok, reason = PromptModel.verify_database_file(src)
|
||||
self.assertFalse(ok)
|
||||
self.assertIn("Saved views", reason)
|
||||
with self.assertRaises(ValueError):
|
||||
self.db.model.restore_from_file(src)
|
||||
|
||||
def _insert_old_rows(self, path):
|
||||
"""Rows written under older value rules, straight into the file."""
|
||||
with closing(sqlite3.connect(path)) as conn:
|
||||
conn.execute(
|
||||
f"INSERT INTO {TABLE} (page, name, state, position, created_at, "
|
||||
"updated_at) VALUES ('images', 'Golden hour', ?, 5, 'x', 'x')",
|
||||
(OLD_IMAGES_STATE,),
|
||||
)
|
||||
conn.execute(
|
||||
f"INSERT INTO {TABLE} (page, name, state, position, created_at, "
|
||||
"updated_at) VALUES ('library', ?, ?, 6, 'x', 'x')",
|
||||
(" " + "n" * 120, json.dumps({"sort": "name-desc", "q": 4})),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def _assert_old_rows_normalized(self, views):
|
||||
self.assertEqual(
|
||||
[(view["page"], view["name"], view["state"]) for view in views],
|
||||
[
|
||||
(
|
||||
"images",
|
||||
"Golden hour",
|
||||
{
|
||||
**IMAGES_DEFAULTS,
|
||||
"q": "sunset",
|
||||
"media": "image",
|
||||
"tags": ["golden"],
|
||||
},
|
||||
),
|
||||
("library", "n" * 100, LIBRARY_DEFAULTS),
|
||||
],
|
||||
)
|
||||
|
||||
def test_a_database_with_rows_from_older_rules_still_opens(self):
|
||||
self.db.close_all()
|
||||
self._insert_old_rows(self.path)
|
||||
PromptModel.reset_schema_cache()
|
||||
|
||||
reopened = PromptDatabase(self.path)
|
||||
try:
|
||||
self._assert_old_rows_normalized(reopened.list_saved_views())
|
||||
finally:
|
||||
reopened.close_all()
|
||||
ok, reason = PromptModel.verify_database_file(
|
||||
self.path, require_library_state=True
|
||||
)
|
||||
self.assertTrue(ok, reason)
|
||||
|
||||
def test_restore_accepts_rows_from_older_rules(self):
|
||||
src = self._source()
|
||||
self._insert_old_rows(src)
|
||||
ok, reason = PromptModel.verify_database_file(src, require_library_state=True)
|
||||
self.assertTrue(ok, reason)
|
||||
|
||||
self.db.model.restore_from_file(src)
|
||||
|
||||
self._assert_old_rows_normalized(self.db.list_saved_views())
|
||||
|
||||
def _insert_corrupt_rows(self, path):
|
||||
"""A valid view between two rows whose state is no JSON object."""
|
||||
with closing(sqlite3.connect(path)) as conn:
|
||||
ids = []
|
||||
for name, state in (
|
||||
("Broken", "{not json"),
|
||||
("Kept", json.dumps(LIBRARY_STATE)),
|
||||
("Array", "[1, 2]"),
|
||||
):
|
||||
ids.append(
|
||||
conn.execute(
|
||||
f"INSERT INTO {TABLE} (page, name, state, position, "
|
||||
"created_at, updated_at) VALUES ('library', ?, ?, 0, "
|
||||
"'x', 'x')",
|
||||
(name, state),
|
||||
).lastrowid
|
||||
)
|
||||
conn.commit()
|
||||
return ids
|
||||
|
||||
def test_startup_tolerates_a_corrupt_state_and_list_skips_it(self):
|
||||
self.db.close_all()
|
||||
broken, kept, array = self._insert_corrupt_rows(self.path)
|
||||
PromptModel.reset_schema_cache()
|
||||
|
||||
with self.assertLogs("prompt_manager.database", level="WARNING") as logs:
|
||||
reopened = PromptDatabase(self.path)
|
||||
try:
|
||||
self.assertEqual(
|
||||
[(view["id"], view["name"]) for view in reopened.list_saved_views()],
|
||||
[(kept, "Kept")],
|
||||
)
|
||||
finally:
|
||||
reopened.close_all()
|
||||
output = "\n".join(logs.output)
|
||||
self.assertIn(f"id {broken}", output)
|
||||
self.assertIn(f"id {array}", output)
|
||||
self.assertNotIn("not json", output)
|
||||
# The rows stay in place, untouched.
|
||||
with closing(sqlite3.connect(self.path)) as conn:
|
||||
self.assertEqual(
|
||||
conn.execute(f"SELECT COUNT(*) FROM {TABLE}").fetchone()[0], 3
|
||||
)
|
||||
|
||||
def test_restore_rejects_a_backup_with_a_corrupt_state(self):
|
||||
src = self._source()
|
||||
self._insert_corrupt_rows(src)
|
||||
ok, reason = PromptModel.verify_database_file(src)
|
||||
self.assertFalse(ok)
|
||||
self.assertEqual(reason, "Saved views contains invalid values")
|
||||
with self.assertRaises(ValueError):
|
||||
self.db.model.restore_from_file(src)
|
||||
|
||||
def test_normalize_state_keeps_valid_values_and_defaults_the_rest(self):
|
||||
self.assertEqual(normalize_state("images", IMAGES_STATE), IMAGES_STATE)
|
||||
self.assertEqual(normalize_state("library", LIBRARY_STATE), LIBRARY_STATE)
|
||||
self.assertEqual(
|
||||
normalize_state("images", json.dumps(IMAGES_STATE)), IMAGES_STATE
|
||||
)
|
||||
for raw in (None, "{", "[]", '"q"', 7, ["q"], b"{}"):
|
||||
with self.subTest(raw=raw):
|
||||
self.assertEqual(normalize_state("library", raw), LIBRARY_DEFAULTS)
|
||||
self.assertEqual(
|
||||
normalize_state(
|
||||
"images",
|
||||
{
|
||||
"q": "bad\x00",
|
||||
"source": 3,
|
||||
"favorites": "true",
|
||||
"tags": [f"t{n}" for n in range(21)],
|
||||
"folder": "kept",
|
||||
},
|
||||
),
|
||||
{**IMAGES_DEFAULTS, "folder": "kept"},
|
||||
)
|
||||
self.assertEqual(normalize_state("history", {}), {})
|
||||
|
||||
def test_validation_errors_are_saved_view_invalid(self):
|
||||
for call in (
|
||||
lambda: validate_name(""),
|
||||
lambda: validate_name(3),
|
||||
lambda: encode_state("history", {}),
|
||||
lambda: encode_state("library", {"q": ""}),
|
||||
lambda: encode_state("library", {**LIBRARY_STATE, "sort": "x"}),
|
||||
):
|
||||
with self.assertRaises(SavedViewInvalid):
|
||||
call()
|
||||
self.assertTrue(issubclass(SavedViewInvalid, ValueError))
|
||||
|
||||
def test_migration_refuses_an_unsupported_table(self):
|
||||
path = os.path.join(self.tmpdir, "foreign.db")
|
||||
with closing(sqlite3.connect(path)) as conn:
|
||||
conn.execute(f"CREATE TABLE {TABLE} (id INTEGER PRIMARY KEY)")
|
||||
with self.assertRaises(sqlite3.DatabaseError):
|
||||
migrate_saved_views(conn)
|
||||
# The savepoint rolled back, so the foreign table is untouched.
|
||||
self.assertEqual(
|
||||
conn.execute(
|
||||
f"SELECT COUNT(*) FROM pragma_table_info('{TABLE}')"
|
||||
).fetchone()[0],
|
||||
1,
|
||||
)
|
||||
self.assertTrue(Path(path).exists())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,92 +0,0 @@
|
||||
"""Unit tests for py/api/scan_job.py: fan-out of scan progress to SSE subscribers."""
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from py.api.scan_job import ScanJob # noqa: E402
|
||||
|
||||
|
||||
async def _collect(job, stop_after=None):
|
||||
seen = []
|
||||
async with contextlib.aclosing(job.events()) as events:
|
||||
async for payload in events:
|
||||
seen.append(payload)
|
||||
if stop_after is not None and len(seen) >= stop_after:
|
||||
break
|
||||
return seen
|
||||
|
||||
|
||||
class TestScanJob(unittest.IsolatedAsyncioTestCase):
|
||||
|
||||
async def test_new_job_is_running_with_no_event(self):
|
||||
job = ScanJob()
|
||||
self.assertTrue(job.running)
|
||||
self.assertIsNone(job.last_event)
|
||||
|
||||
async def test_subscriber_receives_events_until_finish(self):
|
||||
job = ScanJob()
|
||||
consumer = asyncio.ensure_future(_collect(job))
|
||||
await asyncio.sleep(0)
|
||||
|
||||
job.publish({"type": "progress", "progress": 10})
|
||||
job.publish({"type": "complete"})
|
||||
job.finish()
|
||||
|
||||
self.assertEqual(
|
||||
await asyncio.wait_for(consumer, 2),
|
||||
[{"type": "progress", "progress": 10}, {"type": "complete"}],
|
||||
)
|
||||
self.assertFalse(job.running)
|
||||
|
||||
async def test_late_subscriber_gets_latest_snapshot_then_live_events(self):
|
||||
job = ScanJob()
|
||||
job.publish({"type": "progress", "progress": 10})
|
||||
job.publish({"type": "progress", "progress": 40})
|
||||
|
||||
consumer = asyncio.ensure_future(_collect(job))
|
||||
await asyncio.sleep(0)
|
||||
job.publish({"type": "complete"})
|
||||
job.finish()
|
||||
|
||||
self.assertEqual(
|
||||
await asyncio.wait_for(consumer, 2),
|
||||
[{"type": "progress", "progress": 40}, {"type": "complete"}],
|
||||
)
|
||||
|
||||
async def test_subscriber_after_finish_gets_only_the_final_event(self):
|
||||
job = ScanJob()
|
||||
job.publish({"type": "progress", "progress": 99})
|
||||
job.publish({"type": "complete"})
|
||||
job.finish()
|
||||
|
||||
self.assertEqual(await _collect(job), [{"type": "complete"}])
|
||||
|
||||
async def test_two_subscribers_both_see_every_event(self):
|
||||
job = ScanJob()
|
||||
first = asyncio.ensure_future(_collect(job))
|
||||
second = asyncio.ensure_future(_collect(job))
|
||||
await asyncio.sleep(0)
|
||||
|
||||
job.publish({"type": "progress", "progress": 1})
|
||||
job.finish()
|
||||
|
||||
self.assertEqual(await first, [{"type": "progress", "progress": 1}])
|
||||
self.assertEqual(await second, [{"type": "progress", "progress": 1}])
|
||||
|
||||
async def test_departed_subscriber_is_forgotten(self):
|
||||
job = ScanJob()
|
||||
job.publish({"type": "progress", "progress": 1})
|
||||
await _collect(job, stop_after=1)
|
||||
|
||||
self.assertEqual(job.subscriber_count, 0)
|
||||
job.publish({"type": "complete"}) # must not raise or queue anywhere
|
||||
job.finish()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,390 @@
|
||||
"""Scan scope (#249 Activity-2 Task 4): new and unresolved files by default.
|
||||
|
||||
A "Scan for metadata" job started without ``scope`` (or with ``"new"``) skips
|
||||
the files the Library already links with a resolved link, before their
|
||||
metadata is read, and counts them as Already known. ``"all"`` reads every file
|
||||
again, as the legacy ``/prompt_manager/scan`` route always does.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
try:
|
||||
from tests.test_api_admin import AdminAPITestCase, _sse_payloads, _write_png
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from test_api_admin import AdminAPITestCase, _sse_payloads, _write_png
|
||||
|
||||
from database import operations # noqa: E402
|
||||
from database.pair_ingestion import ingest_resolved_pair # noqa: E402
|
||||
from py.config import GalleryConfig # noqa: E402
|
||||
from utils.prompt_pair_resolver import resolve_prompt_pair # noqa: E402
|
||||
|
||||
SKIPPED_DETAIL = "Skipped · already linked"
|
||||
ON_DISK = object() # _link_resolved: store the file's size on disk
|
||||
|
||||
|
||||
def _graph(text):
|
||||
return {
|
||||
"1": {"class_type": "CLIPTextEncode", "inputs": {"text": text}},
|
||||
"2": {"class_type": "CLIPTextEncode", "inputs": {"text": "bad"}},
|
||||
"3": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {"positive": ["1", 0], "negative": ["2", 0]},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class ScanScopeTestCase(AdminAPITestCase):
|
||||
"""Helpers: link files the way the scan and the monitor do, spy on reads."""
|
||||
|
||||
def _link_resolved(self, path, text, size=ON_DISK):
|
||||
"""``path`` linked with a resolved pair link (what a resolved scan writes).
|
||||
|
||||
The row stores ``size`` as its file size: by default the file's size on
|
||||
disk (None when the file does not exist).
|
||||
"""
|
||||
if size is ON_DISK:
|
||||
size = os.path.getsize(path) if os.path.exists(path) else None
|
||||
metadata = {"prompt": _graph(text)}
|
||||
if size is not None:
|
||||
metadata["file_info"] = {"size": size}
|
||||
result = ingest_resolved_pair(
|
||||
self.api.db, str(path), metadata, resolve_prompt_pair(metadata)
|
||||
)
|
||||
self.assertEqual(result["status"], "added")
|
||||
|
||||
def _link_without_pair(self, path, text):
|
||||
"""``path`` linked to a prompt with no pair link (fallback / FIFO link)."""
|
||||
prompt_id = self.api.db.save_prompt(text, None, [], None, None)
|
||||
self.assertTrue(self.api.db.link_image_to_prompt(prompt_id, str(path)))
|
||||
return prompt_id
|
||||
|
||||
def _spy_reads(self):
|
||||
reads = []
|
||||
original = self.api._extract_comfyui_metadata
|
||||
|
||||
def spy(path):
|
||||
reads.append(os.path.basename(path))
|
||||
return original(path)
|
||||
|
||||
self.api._extract_comfyui_metadata = spy
|
||||
return reads
|
||||
|
||||
async def _scan(self, body=None):
|
||||
kwargs = {} if body is None else {"json": body}
|
||||
resp = await self.client.request("POST", "/prompt_manager/jobs/scan", **kwargs)
|
||||
data = await resp.json()
|
||||
self.assertEqual(resp.status, 202, data)
|
||||
await asyncio.wait_for(self.api._scan_job.task, 10)
|
||||
detail = await self.client.request(
|
||||
"GET", f"/prompt_manager/jobs/{data['job']['id']}"
|
||||
)
|
||||
return (await detail.json())["job"]
|
||||
|
||||
|
||||
class TestScanScope(ScanScopeTestCase):
|
||||
|
||||
async def test_new_scope_skips_known_resolved_files_without_reading_them(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
known = self.output_dir / "known.png"
|
||||
_write_png(known, prompt_text="a red square")
|
||||
_write_png(self.output_dir / "new.png", prompt_text="a blue circle")
|
||||
self._link_resolved(known, "a red square")
|
||||
reads = self._spy_reads()
|
||||
|
||||
job = await self._scan()
|
||||
|
||||
self.assertEqual(reads, ["new.png"])
|
||||
self.assertEqual(job["scope"]["files"], "new")
|
||||
self.assertEqual(job["counters"]["known"], 1)
|
||||
self.assertEqual(job["counters"]["added"], 1)
|
||||
self.assertEqual(job["processed"], 2)
|
||||
self.assertEqual(
|
||||
job["outcomes"]["known"],
|
||||
[
|
||||
{
|
||||
"path": "known.png",
|
||||
"root": "output",
|
||||
"root_index": 0,
|
||||
"detail": SKIPPED_DETAIL,
|
||||
}
|
||||
],
|
||||
)
|
||||
self.assertIn(f"= output/known.png · {SKIPPED_DETAIL}", "\n".join(job["log"]))
|
||||
self.assertTrue(
|
||||
job["log"][0].endswith(
|
||||
"Scan started · 1 output folder · new and unresolved files"
|
||||
)
|
||||
)
|
||||
legacy = self.api._scan_job.last_event
|
||||
self.assertEqual((legacy["processed"], legacy["unchanged"]), (2, 1))
|
||||
self.assertTrue(job["status_line"].startswith("Processing file 2 of 2"))
|
||||
|
||||
async def test_whole_known_batch_skips_the_executor_reads(self):
|
||||
from py.api import admin
|
||||
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
for name in ("a.png", "b.png"):
|
||||
_write_png(self.output_dir / name, prompt_text=f"prompt {name}")
|
||||
self._link_resolved(self.output_dir / name, f"prompt {name}")
|
||||
_write_png(self.output_dir / "c.png", prompt_text="prompt c")
|
||||
reads = self._spy_reads()
|
||||
|
||||
with patch.object(admin, "SCAN_BATCH_SIZE", 1):
|
||||
job = await self._scan({"scope": "new"})
|
||||
|
||||
self.assertEqual(reads, ["c.png"])
|
||||
self.assertEqual(job["counters"]["known"], 2)
|
||||
self.assertEqual(job["counters"]["added"], 1)
|
||||
|
||||
async def test_unresolved_files_are_rescanned_in_new_scope(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
names = ("fallback.png", "conflict.png", "unlinked.png")
|
||||
for name in names:
|
||||
_write_png(self.output_dir / name, prompt_text=f"text of {name}")
|
||||
# Linked without a resolved pair link (the monitor's fallback / FIFO link).
|
||||
self._link_without_pair(self.output_dir / "fallback.png", "text of fallback")
|
||||
# Resolved once, then also linked to a second prompt: ambiguous.
|
||||
conflict = self.output_dir / "conflict.png"
|
||||
self._link_resolved(conflict, "text of conflict.png")
|
||||
self._link_without_pair(conflict, "another prompt")
|
||||
reads = self._spy_reads()
|
||||
|
||||
job = await self._scan()
|
||||
|
||||
self.assertEqual(sorted(reads), sorted(names))
|
||||
self.assertNotIn(SKIPPED_DETAIL, json.dumps(job["outcomes"]))
|
||||
|
||||
async def test_all_scope_reads_everything(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
known = self.output_dir / "known.png"
|
||||
_write_png(known, prompt_text="a red square")
|
||||
self._link_resolved(known, "a red square")
|
||||
reads = self._spy_reads()
|
||||
|
||||
job = await self._scan({"scope": "all"})
|
||||
|
||||
self.assertEqual(reads, ["known.png"])
|
||||
self.assertEqual(job["scope"]["files"], "all")
|
||||
self.assertEqual(job["counters"]["known"], 1)
|
||||
self.assertNotEqual(job["outcomes"]["known"][0]["detail"], SKIPPED_DETAIL)
|
||||
self.assertTrue(
|
||||
job["log"][0].endswith("Scan started · 1 output folder · all files")
|
||||
)
|
||||
|
||||
async def test_legacy_scan_route_reads_everything(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
known = self.output_dir / "known.png"
|
||||
_write_png(known, prompt_text="a red square")
|
||||
self._link_resolved(known, "a red square")
|
||||
reads = self._spy_reads()
|
||||
|
||||
resp = await self.client.request("POST", "/prompt_manager/scan", data="{}")
|
||||
payloads = _sse_payloads(await resp.text())
|
||||
|
||||
self.assertEqual(reads, ["known.png"])
|
||||
self.assertEqual(self.api._scan_job.scope["files"], "all")
|
||||
self.assertEqual(payloads[-1]["type"], "complete")
|
||||
self.assertEqual(payloads[-1]["unchanged"], 1)
|
||||
|
||||
async def test_file_replaced_at_a_known_path_is_reread_and_needs_review(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
path = self.output_dir / "out_00012.png"
|
||||
_write_png(path, prompt_text="the first prompt")
|
||||
self._link_resolved(path, "the first prompt")
|
||||
before = os.path.getsize(path)
|
||||
# ComfyUI reused the name for a new image with another prompt.
|
||||
_write_png(path, prompt_text="a much longer second prompt for the same name")
|
||||
self.assertNotEqual(os.path.getsize(path), before)
|
||||
reads = self._spy_reads()
|
||||
|
||||
job = await self._scan()
|
||||
|
||||
self.assertEqual(reads, ["out_00012.png"])
|
||||
self.assertEqual(job["counters"]["review"], 1)
|
||||
self.assertEqual(job["counters"]["known"], 0)
|
||||
self.assertIn("different prompt", job["outcomes"]["review"][0]["detail"])
|
||||
|
||||
async def test_scanned_files_store_their_size_so_the_next_new_scan_skips(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
_write_png(self.output_dir / "fresh.png", prompt_text="a fresh prompt")
|
||||
first = await self._scan()
|
||||
self.assertEqual(first["counters"]["added"], 1)
|
||||
reads = self._spy_reads()
|
||||
|
||||
second = await self._scan()
|
||||
|
||||
self.assertEqual(reads, [])
|
||||
self.assertEqual(second["outcomes"]["known"][0]["detail"], SKIPPED_DETAIL)
|
||||
|
||||
async def test_stored_null_size_is_reread(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
path = self.output_dir / "old.png"
|
||||
_write_png(path, prompt_text="an old prompt")
|
||||
self._link_resolved(path, "an old prompt", size=None)
|
||||
reads = self._spy_reads()
|
||||
|
||||
job = await self._scan()
|
||||
|
||||
self.assertEqual(reads, ["old.png"])
|
||||
# The read stores the size it was missing, so the file is known next time.
|
||||
self.assertEqual(job["counters"]["added"], 1)
|
||||
self.assertEqual(
|
||||
self.api.db.get_known_resolved_paths({str(path): os.path.getsize(path)}),
|
||||
{str(path)},
|
||||
)
|
||||
|
||||
async def test_known_set_failure_falls_back_to_reading_every_file(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
known = self.output_dir / "known.png"
|
||||
_write_png(known, prompt_text="a red square")
|
||||
self._link_resolved(known, "a red square")
|
||||
reads = self._spy_reads()
|
||||
|
||||
def broken(files):
|
||||
raise sqlite3.OperationalError("database is locked")
|
||||
|
||||
self.api.db.get_known_resolved_paths = broken
|
||||
with self.assertLogs("prompt_manager.api", level="WARNING"):
|
||||
job = await self._scan()
|
||||
|
||||
self.assertEqual(job["state"], "finished")
|
||||
self.assertEqual(reads, ["known.png"])
|
||||
self.assertIn(
|
||||
"Known files could not be read · reading every file", "\n".join(job["log"])
|
||||
)
|
||||
|
||||
async def test_empty_read_body_asks_for_the_default_scope(self):
|
||||
"""A chunked POST with no bytes can read a body: still the default."""
|
||||
from py.api.job_routes import _scan_scope
|
||||
|
||||
class Request:
|
||||
can_read_body = True
|
||||
|
||||
def __init__(self, text):
|
||||
self._text = text
|
||||
|
||||
async def text(self):
|
||||
return self._text
|
||||
|
||||
self.assertEqual(await _scan_scope(Request("")), "new")
|
||||
self.assertEqual(await _scan_scope(Request(" \n")), "new")
|
||||
self.assertEqual(await _scan_scope(Request('{"scope": "all"}')), "all")
|
||||
self.assertIsNone(await _scan_scope(Request("{")))
|
||||
|
||||
async def test_invalid_scope_is_refused(self):
|
||||
for body in ({"scope": "everything"}, {"scope": 1}, ["new"]):
|
||||
with self.subTest(body=body):
|
||||
resp = await self.client.request(
|
||||
"POST", "/prompt_manager/jobs/scan", json=body
|
||||
)
|
||||
self.assertEqual(resp.status, 400)
|
||||
data = await resp.json()
|
||||
self.assertFalse(data["success"])
|
||||
resp = await self.client.request(
|
||||
"POST", "/prompt_manager/jobs/scan", data="not json"
|
||||
)
|
||||
self.assertEqual(resp.status, 400)
|
||||
self.assertIsNone(self.api._scan_job)
|
||||
|
||||
|
||||
class TestKnownResolvedPaths(ScanScopeTestCase):
|
||||
"""``PromptDatabase.get_known_resolved_paths``: one streamed query."""
|
||||
|
||||
async def test_returns_the_given_paths_that_are_known_and_resolved(self):
|
||||
known, fallback = self.output_dir / "known.png", self.output_dir / "fb.png"
|
||||
self._link_resolved(known, "a red square", size=10)
|
||||
self._link_without_pair(fallback, "a fallback")
|
||||
asked = {str(known): 10, str(fallback): 10, str(self.output_dir / "n.png"): 10}
|
||||
|
||||
self.assertEqual(self.api.db.get_known_resolved_paths(asked), {str(known)})
|
||||
self.assertEqual(self.api.db.get_known_resolved_paths({}), set())
|
||||
|
||||
async def test_size_must_match_and_a_stored_null_size_is_not_known(self):
|
||||
same, other = self.output_dir / "same.png", self.output_dir / "other.png"
|
||||
unknown = self.output_dir / "unknown.png"
|
||||
self._link_resolved(same, "same size", size=10)
|
||||
self._link_resolved(other, "other size", size=10)
|
||||
self._link_resolved(unknown, "no stored size", size=None)
|
||||
asked = {str(same): 10, str(other): 11, str(unknown): 10}
|
||||
|
||||
self.assertEqual(self.api.db.get_known_resolved_paths(asked), {str(same)})
|
||||
# A file the walk could not stat is not known either.
|
||||
self.assertEqual(self.api.db.get_known_resolved_paths({str(same): None}), set())
|
||||
|
||||
async def test_row_without_file_path_matches_through_image_path(self):
|
||||
known = self.output_dir / "legacy.png"
|
||||
self._link_resolved(known, "a legacy row", size=10)
|
||||
with sqlite3.connect(self.api.db.model.db_path) as conn:
|
||||
conn.execute("UPDATE generated_images SET file_path=NULL")
|
||||
conn.close()
|
||||
|
||||
found = self.api.db.get_known_resolved_paths({str(known): 10})
|
||||
self.assertEqual(found, {str(known)})
|
||||
|
||||
async def test_stored_spelling_is_normalized_like_the_scanned_path(self):
|
||||
known = self.output_dir / "sub" / "known.png"
|
||||
self._link_resolved(known, "a red square", size=10)
|
||||
spelled = os.path.join(str(self.output_dir), "sub", "..", "sub", "known.png")
|
||||
with sqlite3.connect(self.api.db.model.db_path) as conn:
|
||||
conn.execute("UPDATE generated_images SET file_path=?", (spelled,))
|
||||
conn.close()
|
||||
|
||||
found = self.api.db.get_known_resolved_paths({str(known): 10})
|
||||
self.assertEqual(found, {str(known)})
|
||||
|
||||
async def test_one_query_streamed_in_small_fetches(self):
|
||||
paths = [self.output_dir / f"{index}.png" for index in range(5)]
|
||||
for path in paths:
|
||||
self._link_resolved(path, f"prompt {path.name}", size=10)
|
||||
original = self.api.db.model.get_connection
|
||||
opened = []
|
||||
|
||||
def counting():
|
||||
opened.append(1)
|
||||
return original()
|
||||
|
||||
self.api.db.model.get_connection = counting
|
||||
with patch.object(operations, "KNOWN_PATHS_FETCH_SIZE", 2):
|
||||
found = self.api.db.get_known_resolved_paths({str(p): 10 for p in paths})
|
||||
self.assertEqual(found, {str(p) for p in paths})
|
||||
self.assertEqual(len(opened), 1)
|
||||
|
||||
async def test_windows_path_case_matches_known_set(self):
|
||||
stored = self.output_dir / "Sub" / "Known.PNG"
|
||||
self._link_resolved(stored, "a red square", size=10)
|
||||
# A row written before paths were case-folded keeps its own spelling.
|
||||
with sqlite3.connect(self.api.db.model.db_path) as conn:
|
||||
conn.execute(
|
||||
"UPDATE generated_images SET file_path=?", (str(stored).upper(),)
|
||||
)
|
||||
conn.close()
|
||||
asked = str(self.output_dir / "sub" / "known.png")
|
||||
|
||||
with patch("os.path.normcase", lambda path: path.lower()):
|
||||
self.assertEqual(self.api.db.get_known_resolved_paths({asked: 10}), {asked})
|
||||
if os.path.normcase("A") == "A": # a case-sensitive platform
|
||||
self.assertEqual(self.api.db.get_known_resolved_paths({asked: 10}), set())
|
||||
|
||||
|
||||
class TestSkippedOutcome(unittest.TestCase):
|
||||
|
||||
def test_skipped_file_is_already_known_with_its_own_detail(self):
|
||||
from py.api.admin import _scan_outcome
|
||||
|
||||
root = os.path.abspath("out")
|
||||
group, row, line = _scan_outcome(
|
||||
os.path.join(root, "a", "b.png"), [(root, "output", 0)], "skipped"
|
||||
)
|
||||
self.assertEqual(group, "known")
|
||||
self.assertEqual(row["detail"], SKIPPED_DETAIL)
|
||||
self.assertEqual(row["path"], "a/b.png")
|
||||
self.assertEqual(line, f"= output/a/b.png · {SKIPPED_DETAIL}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,8 +1,11 @@
|
||||
"""Tests for utils/video_metadata.py: prompt metadata from video containers.
|
||||
|
||||
ComfyUI's video savers store ``{"prompt": ..., "workflow": ...}`` as JSON in
|
||||
the container's ``comment`` tag. ffprobe reads it; when ffprobe is missing
|
||||
videos are skipped without noise.
|
||||
Current ComfyUI savers (core SaveVideo/SaveWEBM, VideoHelperSuite) write one
|
||||
container tag per key (``prompt``, ``workflow``, ``parameters``,
|
||||
``comfyassets_prompt_pairs``); older VideoHelperSuite wrote one ``comment``
|
||||
JSON envelope, with ``description`` as a fallback. One selector picks a single
|
||||
carrier for the inspector and the scan. ffprobe reads the tags; when ffprobe
|
||||
is missing videos are skipped without noise.
|
||||
"""
|
||||
|
||||
import json
|
||||
@@ -19,34 +22,44 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from utils import video_metadata # noqa: E402
|
||||
from utils.video_metadata import ( # noqa: E402
|
||||
VIDEO_EXTENSIONS,
|
||||
DuplicateVideoTagError,
|
||||
is_video_path,
|
||||
metadata_from_tags,
|
||||
read_video_metadata,
|
||||
select_video_carrier,
|
||||
)
|
||||
|
||||
HAVE_FFMPEG = bool(shutil.which("ffmpeg") and shutil.which("ffprobe"))
|
||||
GRAPH = json.dumps({"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "a cat"}}})
|
||||
|
||||
|
||||
def write_video(path, metadata_tags=True, **tags):
|
||||
"""Write a tiny black video carrying one container tag per keyword.
|
||||
|
||||
MP4 keeps arbitrary keys only with ``use_metadata_tags`` (mdta atoms), the
|
||||
movflag the core SaveVideo node sets; without it ffmpeg writes the legacy
|
||||
ilst atoms (``comment`` becomes ``©cmt``) that old VideoHelperSuite files
|
||||
carry. Matroska and WebM upper-case the key names.
|
||||
"""
|
||||
command = [
|
||||
"ffmpeg",
|
||||
"-v",
|
||||
"quiet",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"color=c=black:s=16x16:d=0.2",
|
||||
]
|
||||
if metadata_tags and str(path).lower().endswith((".mp4", ".mov", ".m4v")):
|
||||
command += ["-movflags", "use_metadata_tags"]
|
||||
for key, value in tags.items():
|
||||
command += ["-metadata", f"{key}={value}"]
|
||||
subprocess.run(command + ["-y", str(path)], check=True, timeout=60)
|
||||
|
||||
|
||||
def write_mp4(path, comment):
|
||||
"""Write a tiny black video carrying ``comment`` in its container tags."""
|
||||
subprocess.run(
|
||||
[
|
||||
"ffmpeg",
|
||||
"-v",
|
||||
"quiet",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"color=c=black:s=16x16:d=0.2",
|
||||
"-metadata",
|
||||
f"comment={comment}",
|
||||
"-y",
|
||||
str(path),
|
||||
],
|
||||
check=True,
|
||||
timeout=60,
|
||||
)
|
||||
"""Write a tiny black MP4 with ``comment`` in its legacy ilst ``©cmt`` atom."""
|
||||
write_video(path, metadata_tags=False, comment=comment)
|
||||
|
||||
|
||||
class TestMetadataFromTags(unittest.TestCase):
|
||||
@@ -69,6 +82,158 @@ class TestMetadataFromTags(unittest.TestCase):
|
||||
self.assertEqual(metadata_from_tags(tags), {"prompt": GRAPH})
|
||||
|
||||
|
||||
class TestSelectVideoCarrier(unittest.TestCase):
|
||||
"""Spec 2026-10-19 resolution parity section 8: one carrier, never merged."""
|
||||
|
||||
WORKFLOW = json.dumps({"nodes": [{"id": "per-key"}]})
|
||||
|
||||
def test_per_key_tags_win_over_comment(self):
|
||||
tags = {
|
||||
"prompt": GRAPH,
|
||||
"workflow": self.WORKFLOW,
|
||||
"comment": json.dumps({"prompt": json.dumps({"9": {}})}),
|
||||
}
|
||||
metadata, formats, carrier = select_video_carrier(tags)
|
||||
self.assertEqual(metadata, {"prompt": GRAPH, "workflow": self.WORKFLOW})
|
||||
self.assertEqual(
|
||||
formats, {"prompt": "original-text", "workflow": "original-text"}
|
||||
)
|
||||
self.assertIsNone(carrier)
|
||||
|
||||
def test_matroska_uppercase_keys_fold(self):
|
||||
tags = {"PROMPT": GRAPH, "Workflow": self.WORKFLOW, "ENCODER": "Lavf"}
|
||||
metadata, formats, carrier = select_video_carrier(tags)
|
||||
self.assertEqual(metadata, {"prompt": GRAPH, "workflow": self.WORKFLOW})
|
||||
self.assertEqual(
|
||||
formats, {"prompt": "original-text", "workflow": "original-text"}
|
||||
)
|
||||
self.assertIsNone(carrier)
|
||||
# Envelope keys are read exactly as legacy did: no fold, so a Matroska
|
||||
# upper-cased COMMENT is not an envelope.
|
||||
self.assertIsNone(
|
||||
select_video_carrier({"COMMENT": json.dumps({"prompt": GRAPH})})
|
||||
)
|
||||
|
||||
def test_folded_duplicate_keys_fail(self):
|
||||
for tags in (
|
||||
{"PROMPT": GRAPH, "prompt": json.dumps({"9": {}})},
|
||||
{"Workflow": self.WORKFLOW, "WORKFLOW": "{}", "prompt": GRAPH},
|
||||
):
|
||||
with self.assertRaises(DuplicateVideoTagError):
|
||||
select_video_carrier(tags)
|
||||
# Byte-identical duplicates are one record, as for PNG text chunks.
|
||||
metadata, _, _ = select_video_carrier({"PROMPT": GRAPH, "prompt": GRAPH})
|
||||
self.assertEqual(metadata, {"prompt": GRAPH})
|
||||
# Envelope keys never fold, so a foreign COMMENT note is not a duplicate.
|
||||
envelope = json.dumps({"prompt": GRAPH})
|
||||
metadata, _, carrier = select_video_carrier(
|
||||
{"comment": envelope, "COMMENT": "a free-text note"}
|
||||
)
|
||||
self.assertEqual(metadata, {"prompt": GRAPH})
|
||||
self.assertEqual(carrier, envelope)
|
||||
|
||||
def test_comment_envelope_still_read(self):
|
||||
envelope = json.dumps({"prompt": GRAPH, "workflow": {"nodes": []}})
|
||||
metadata, formats, carrier = select_video_carrier(
|
||||
{"comment": envelope, "encoder": "Lavf"}
|
||||
)
|
||||
self.assertEqual(metadata["prompt"], GRAPH)
|
||||
self.assertEqual(json.loads(metadata["workflow"]), {"nodes": []})
|
||||
self.assertEqual(
|
||||
formats, {"prompt": "original-text", "workflow": "reserialized-json"}
|
||||
)
|
||||
self.assertEqual(carrier, envelope)
|
||||
|
||||
def test_description_fallback(self):
|
||||
envelope = json.dumps({"prompt": GRAPH})
|
||||
tags = {"comment": "not json", "description": envelope}
|
||||
self.assertEqual(
|
||||
select_video_carrier(tags),
|
||||
({"prompt": GRAPH}, {"prompt": "original-text"}, envelope),
|
||||
)
|
||||
self.assertIsNone(select_video_carrier({"comment": "note", "encoder": "x"}))
|
||||
self.assertIsNone(select_video_carrier({"comment": "[1]"}))
|
||||
self.assertIsNone(select_video_carrier({}))
|
||||
self.assertIsNone(select_video_carrier(None))
|
||||
self.assertIsNone(select_video_carrier({"comment": 7}))
|
||||
|
||||
def test_role_carrier_beats_workflow_only_carrier(self):
|
||||
envelope = json.dumps({"prompt": GRAPH, "workflow": {"nodes": ["c"]}})
|
||||
metadata, formats, carrier = select_video_carrier(
|
||||
{"workflow": self.WORKFLOW, "comment": envelope}
|
||||
)
|
||||
self.assertEqual(metadata["prompt"], GRAPH)
|
||||
self.assertEqual(json.loads(metadata["workflow"]), {"nodes": ["c"]})
|
||||
self.assertEqual(carrier, envelope)
|
||||
for role in ("parameters", "comfyassets_prompt_pairs"):
|
||||
metadata, _, carrier = select_video_carrier(
|
||||
{"comment": json.dumps({"workflow": {}}), role: "x"}
|
||||
)
|
||||
self.assertEqual(metadata, {role: "x"})
|
||||
self.assertIsNone(carrier)
|
||||
# Without a role anywhere, the first workflow-only carrier wins.
|
||||
metadata, _, carrier = select_video_carrier(
|
||||
{"workflow": self.WORKFLOW, "comment": json.dumps({"workflow": {}})}
|
||||
)
|
||||
self.assertEqual(metadata, {"workflow": self.WORKFLOW})
|
||||
self.assertIsNone(carrier)
|
||||
|
||||
def test_carriers_never_merge(self):
|
||||
metadata, _, carrier = select_video_carrier(
|
||||
{
|
||||
"prompt": GRAPH,
|
||||
"comment": json.dumps({"workflow": {"nodes": ["comment"]}}),
|
||||
"description": json.dumps({"parameters": "Steps: 1"}),
|
||||
}
|
||||
)
|
||||
self.assertEqual(metadata, {"prompt": GRAPH})
|
||||
self.assertIsNone(carrier)
|
||||
metadata, _, carrier = select_video_carrier(
|
||||
{
|
||||
"workflow": self.WORKFLOW,
|
||||
"comment": json.dumps({"prompt": GRAPH}),
|
||||
"description": json.dumps({"parameters": "Steps: 1"}),
|
||||
}
|
||||
)
|
||||
self.assertEqual(metadata, {"prompt": GRAPH})
|
||||
self.assertEqual(carrier, json.dumps({"prompt": GRAPH}))
|
||||
|
||||
def test_per_key_values_are_original_text_and_envelope_objects_reserialized(
|
||||
self,
|
||||
):
|
||||
exact = '{"3":{"inputs":{"cfg":6.5000000000000000001}}}'
|
||||
metadata, formats, _ = select_video_carrier({"prompt": exact})
|
||||
self.assertEqual(metadata, {"prompt": exact})
|
||||
self.assertEqual(formats, {"prompt": "original-text"})
|
||||
envelope = json.dumps(
|
||||
{"prompt": {"3": {}}, "workflow": {"nodes": []}, "parameters": "Steps: 1"}
|
||||
)
|
||||
metadata, formats, carrier = select_video_carrier({"comment": envelope})
|
||||
self.assertEqual(json.loads(metadata["prompt"]), {"3": {}})
|
||||
self.assertEqual(json.loads(metadata["workflow"]), {"nodes": []})
|
||||
self.assertEqual(metadata["parameters"], "Steps: 1")
|
||||
self.assertEqual(
|
||||
formats,
|
||||
{
|
||||
"prompt": "reserialized-json",
|
||||
"workflow": "reserialized-json",
|
||||
"parameters": "original-text",
|
||||
},
|
||||
)
|
||||
self.assertEqual(carrier, envelope)
|
||||
|
||||
def test_null_role_markers_and_invalid_workflow_keep_their_shape(self):
|
||||
# The inspector refuses a null role marker rather than falling back.
|
||||
metadata, formats, _ = select_video_carrier(
|
||||
{"comment": json.dumps({"prompt": None, "workflow": [1]})}
|
||||
)
|
||||
self.assertEqual(metadata, {"prompt": None})
|
||||
self.assertEqual(formats, {"workflow": "invalid"})
|
||||
metadata, formats, _ = select_video_carrier({"prompt": 7, "workflow": 1})
|
||||
self.assertEqual(metadata, {"prompt": None})
|
||||
self.assertEqual(formats, {"workflow": "invalid"})
|
||||
|
||||
|
||||
class TestIsVideoPath(unittest.TestCase):
|
||||
|
||||
def test_extensions(self):
|
||||
@@ -125,6 +290,37 @@ class TestReadVideoMetadataWithoutFfprobe(unittest.TestCase):
|
||||
):
|
||||
self.assertEqual(read_video_metadata("clip.mp4"), {})
|
||||
|
||||
def test_read_video_metadata_returns_per_key_prompt(self):
|
||||
payload = {
|
||||
"format": {
|
||||
"tags": {
|
||||
"PROMPT": GRAPH,
|
||||
"WORKFLOW": '{"nodes":[]}',
|
||||
"COMFYASSETS_PROMPT_PAIRS": '{"version":1}',
|
||||
"COMMENT": json.dumps({"prompt": "must not win"}),
|
||||
"ENCODER": "Lavf",
|
||||
}
|
||||
}
|
||||
}
|
||||
with mock.patch.object(
|
||||
video_metadata.shutil, "which", return_value="/bin/ffprobe"
|
||||
):
|
||||
with self._run_returning(stdout=json.dumps(payload)):
|
||||
self.assertEqual(
|
||||
read_video_metadata("clip.mkv"),
|
||||
{
|
||||
"prompt": GRAPH,
|
||||
"workflow": '{"nodes":[]}',
|
||||
"comfyassets_prompt_pairs": '{"version":1}',
|
||||
},
|
||||
)
|
||||
duplicates = {"format": {"tags": {"PROMPT": GRAPH, "prompt": "{}"}}}
|
||||
with self._run_returning(stdout=json.dumps(duplicates)):
|
||||
self.assertEqual(read_video_metadata("clip.mkv"), {})
|
||||
markers = {"format": {"tags": {"comment": json.dumps({"prompt": None})}}}
|
||||
with self._run_returning(stdout=json.dumps(markers)):
|
||||
self.assertEqual(read_video_metadata("clip.mp4"), {})
|
||||
|
||||
def test_ffprobe_output_is_parsed(self):
|
||||
payload = {"format": {"tags": {"comment": json.dumps({"prompt": GRAPH})}}}
|
||||
with mock.patch.object(
|
||||
@@ -155,6 +351,39 @@ class TestReadVideoMetadataWithRealFfprobe(unittest.TestCase):
|
||||
self.assertEqual(meta["prompt"], GRAPH)
|
||||
self.assertEqual(json.loads(meta["workflow"]), {"v": 1})
|
||||
|
||||
def test_per_key_tags_round_trip_through_real_containers(self):
|
||||
for name in ("per_key.mp4", "per_key.mkv", "per_key.webm"):
|
||||
path = os.path.join(self.tmpdir, name)
|
||||
write_video(path, prompt=GRAPH, workflow='{"nodes":[]}')
|
||||
self.assertEqual(
|
||||
read_video_metadata(path),
|
||||
{"prompt": GRAPH, "workflow": '{"nodes":[]}'},
|
||||
name,
|
||||
)
|
||||
|
||||
def test_legacy_ilst_comment_atom_is_read(self):
|
||||
path = os.path.join(self.tmpdir, "vhs.mp4")
|
||||
write_mp4(path, json.dumps({"prompt": GRAPH}))
|
||||
probe = subprocess.run(
|
||||
[
|
||||
shutil.which("ffprobe"),
|
||||
"-v",
|
||||
"quiet",
|
||||
"-show_entries",
|
||||
"format_tags",
|
||||
"-of",
|
||||
"json",
|
||||
path,
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=60,
|
||||
)
|
||||
tags = json.loads(probe.stdout)["format"]["tags"]
|
||||
self.assertIn("comment", tags)
|
||||
self.assertEqual(read_video_metadata(path), {"prompt": GRAPH})
|
||||
|
||||
def test_video_without_a_comment_is_empty(self):
|
||||
path = os.path.join(self.tmpdir, "plain.mp4")
|
||||
write_mp4(path, "")
|
||||
|
||||
+124
-4
@@ -1,5 +1,7 @@
|
||||
"""Bounded vocabulary reads count exactly what Library Browse shows."""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import tempfile
|
||||
@@ -61,7 +63,48 @@ class TestVocabularyReads(unittest.TestCase):
|
||||
by_name = {row["name"]: row for row in page["tags"]}
|
||||
self.assertEqual(by_name["z"]["prompt_count"], 0)
|
||||
self.assertEqual(by_name["a"]["prompt_count"], 2)
|
||||
self.assertEqual(set(by_name["c"]), {"id", "name", "prompt_count"})
|
||||
self.assertEqual(
|
||||
set(by_name["c"]), {"id", "name", "prompt_count", "image_count"}
|
||||
)
|
||||
self.assertEqual({row["image_count"] for row in page["tags"]}, {0})
|
||||
|
||||
def image(self, name, tags, digest="a", mtime_ns=1):
|
||||
"""Tag the bytes ``digest`` of the file ``name``; return its locator."""
|
||||
locator = os.path.normcase(os.path.join(os.path.realpath(self.temp.name), name))
|
||||
self.read(
|
||||
"replace_image_tags",
|
||||
locator=locator,
|
||||
sha256=hashlib.sha256(digest.encode()).hexdigest(),
|
||||
size=1,
|
||||
mtime_ns=mtime_ns,
|
||||
expected_epoch=self.db.get_execution_epoch(),
|
||||
expected_tag_revision=0,
|
||||
tags=tags,
|
||||
)
|
||||
return locator
|
||||
|
||||
def test_tag_rows_report_image_count_by_locator(self):
|
||||
self.save("Prompt only", tags=["prompt-only"])
|
||||
self.image("one.png", ["twice", "shared"], digest="old")
|
||||
self.image(
|
||||
"one.png", ["twice"], digest="new", mtime_ns=2
|
||||
) # two bytes, one file
|
||||
self.image("two.png", ["shared"], digest="new") # equal digest, other locator
|
||||
self.image("three.png", ["prior-only"], digest="first")
|
||||
self.image("three.png", [], digest="second", mtime_ns=2)
|
||||
rows = {row["name"]: row for row in self.all_tags()}
|
||||
self.assertEqual(rows["twice"]["image_count"], 1)
|
||||
row = rows["shared"]
|
||||
self.assertEqual(row["image_count"], 2)
|
||||
self.assertEqual(rows["prior-only"]["image_count"], 1)
|
||||
self.assertEqual(rows["prompt-only"]["image_count"], 0)
|
||||
self.assertEqual(
|
||||
{name: r["prompt_count"] for name, r in rows.items()},
|
||||
{"prompt-only": 1, "twice": 0, "shared": 0, "prior-only": 0},
|
||||
)
|
||||
self.assertEqual(
|
||||
self.db.list_library(tags=["shared"])["total"], row["prompt_count"]
|
||||
)
|
||||
|
||||
def test_prompt_count_equals_library_browse_total_for_every_tag(self):
|
||||
self.save("Parent only", tags=["parent-only"])
|
||||
@@ -302,7 +345,12 @@ class TestVocabularyTagCommands(unittest.TestCase):
|
||||
self.assertTrue(created["created"])
|
||||
self.assertEqual(
|
||||
created["tag"],
|
||||
{"id": self.tag_id("sunset"), "name": "sunset", "prompt_count": 0},
|
||||
{
|
||||
"id": self.tag_id("sunset"),
|
||||
"name": "sunset",
|
||||
"prompt_count": 0,
|
||||
"image_count": 0,
|
||||
},
|
||||
)
|
||||
after = self.junction_rows()
|
||||
self.assertEqual(after[1:], before[1:])
|
||||
@@ -345,9 +393,15 @@ class TestVocabularyTagCommands(unittest.TestCase):
|
||||
{
|
||||
"library_epoch": self.epoch,
|
||||
"merged": False,
|
||||
"tag": {"id": tag_id, "name": "dramatic", "prompt_count": 2},
|
||||
"tag": {
|
||||
"id": tag_id,
|
||||
"name": "dramatic",
|
||||
"prompt_count": 2,
|
||||
"image_count": 0,
|
||||
},
|
||||
"components_changed": 2,
|
||||
"pairs_changed": 2,
|
||||
"images_changed": 0,
|
||||
},
|
||||
)
|
||||
self.assertIsNone(self.tag_id("moody"))
|
||||
@@ -397,7 +451,8 @@ class TestVocabularyTagCommands(unittest.TestCase):
|
||||
)
|
||||
self.assertFalse(result["merged"])
|
||||
self.assertEqual(
|
||||
result["tag"], {"id": tag_id, "name": "cat", "prompt_count": 1}
|
||||
result["tag"],
|
||||
{"id": tag_id, "name": "cat", "prompt_count": 1, "image_count": 0},
|
||||
)
|
||||
self.assertEqual(self.db.get_prompt_by_id(prompt)["tags"], ["cat"])
|
||||
|
||||
@@ -467,6 +522,7 @@ class TestVocabularyTagCommands(unittest.TestCase):
|
||||
"other_prompts": 1,
|
||||
"negative_variants": 1,
|
||||
"archived_negative_variants": 1,
|
||||
"images": 0,
|
||||
},
|
||||
},
|
||||
)
|
||||
@@ -482,6 +538,7 @@ class TestVocabularyTagCommands(unittest.TestCase):
|
||||
"deleted": True,
|
||||
"components_changed": 2,
|
||||
"pairs_changed": 2,
|
||||
"images_changed": 0,
|
||||
},
|
||||
)
|
||||
self.assertIsNone(self.tag_id("moody"))
|
||||
@@ -494,6 +551,69 @@ class TestVocabularyTagCommands(unittest.TestCase):
|
||||
)
|
||||
self.assertEqual(caught.exception.code, "vocabulary-conflict")
|
||||
|
||||
def image(self, name, tags, digest="a", mtime_ns=1):
|
||||
locator = os.path.normcase(os.path.join(os.path.realpath(self.temp.name), name))
|
||||
self.command(
|
||||
"replace_image_tags",
|
||||
locator=locator,
|
||||
sha256=hashlib.sha256(digest.encode()).hexdigest(),
|
||||
size=1,
|
||||
mtime_ns=mtime_ns,
|
||||
expected_epoch=self.epoch,
|
||||
expected_tag_revision=0,
|
||||
tags=tags,
|
||||
)
|
||||
return locator
|
||||
|
||||
def test_impact_and_delete_counts_include_images(self):
|
||||
self.owners()
|
||||
self.image("a.png", ["moody", "keep"])
|
||||
self.image("a.png", ["moody"], digest="prior", mtime_ns=2)
|
||||
self.image("b.png", ["moody"])
|
||||
tag_id = self.tag_id("moody")
|
||||
impact = self.command("vocabulary_tag_impact", tag_id)["impact"]
|
||||
self.assertEqual(impact["images"], 2)
|
||||
self.assertEqual(
|
||||
self.command("vocabulary_tag_impact", self.tag_id("keep"))["impact"],
|
||||
{
|
||||
"positive_prompts": 0,
|
||||
"other_prompts": 0,
|
||||
"negative_variants": 0,
|
||||
"archived_negative_variants": 0,
|
||||
"images": 1,
|
||||
},
|
||||
)
|
||||
result = self.command(
|
||||
"delete_vocabulary_tag", tag_id, expected_epoch=self.epoch, name="moody"
|
||||
)
|
||||
self.assertEqual(
|
||||
(
|
||||
result["components_changed"],
|
||||
result["pairs_changed"],
|
||||
result["images_changed"],
|
||||
),
|
||||
(2, 2, 2),
|
||||
)
|
||||
with self.db.model.get_connection() as conn:
|
||||
self.assertEqual(
|
||||
conn.execute("SELECT COUNT(*) FROM media_image_tags").fetchone()[0], 1
|
||||
)
|
||||
self.assertEqual(
|
||||
conn.execute("SELECT COUNT(*) FROM media_annotation_bytes").fetchone()[
|
||||
0
|
||||
],
|
||||
3,
|
||||
)
|
||||
renamed = self.command(
|
||||
"rename_vocabulary_tag",
|
||||
self.tag_id("keep"),
|
||||
expected_epoch=self.epoch,
|
||||
name="keep",
|
||||
new_name="kept",
|
||||
)
|
||||
self.assertEqual(renamed["images_changed"], 1)
|
||||
self.assertEqual(renamed["tag"]["image_count"], 1)
|
||||
|
||||
|
||||
_OLD_STAMP = "2000-01-01T00:00:00+00:00"
|
||||
_DML = re.compile(r"\s*(INSERT|UPDATE|DELETE|REPLACE)\b", re.IGNORECASE)
|
||||
|
||||
+234
@@ -0,0 +1,234 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createServer } from 'node:net';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { once } from 'node:events';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
import { startBrowserServer, createServerSupervisor, attachServerStderr } from '../../tools/vendor/start-browser-server.mjs';
|
||||
import { workflowWorkers } from '../../tools/vendor/test-workers.mjs';
|
||||
|
||||
const fakeServer = fileURLToPath(new URL('./fake-browser-server.mjs', import.meta.url));
|
||||
const fake = (...extra) => ({ command: process.execPath, args: port => [fakeServer, '--port', String(port), ...extra] });
|
||||
const REAL_SERVER_TIMEOUT_MS = 90_000;
|
||||
const launcher = fileURLToPath(new URL('../../tools/vendor/start-browser-server.mjs', import.meta.url));
|
||||
|
||||
/** Polls `check` until it is truthy, failing with `what` after `ms`. */
|
||||
async function waitFor(check, what, ms = 5_000) {
|
||||
const deadline = Date.now() + ms;
|
||||
while (!check()) {
|
||||
if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}`);
|
||||
await delay(20);
|
||||
}
|
||||
}
|
||||
|
||||
async function writeStderr(server, text) {
|
||||
await (await fetch(`${server.url}/__stderr?text=${encodeURIComponent(text)}`)).text();
|
||||
await waitFor(() => server.stderrTail().endsWith(text.trimEnd()), `stderr ${JSON.stringify(text)}`);
|
||||
}
|
||||
|
||||
/** A process that exited counts as gone even while a non-reaping PID 1 (CI's container) leaves it a zombie. */
|
||||
function alive(pid) {
|
||||
try { process.kill(pid, 0); } catch { return false; }
|
||||
try { return readFileSync(`/proc/${pid}/stat`, 'utf8').replace(/^.*\) /s, '')[0] !== 'Z'; } catch { return true; }
|
||||
}
|
||||
|
||||
/** Starts a server from a separate Node worker process, SIGKILLs that worker and returns the server's pid. */
|
||||
async function serverOfKilledWorker(options) {
|
||||
const script = `import { startBrowserServer } from ${JSON.stringify(pathToFileURL(launcher).href)};
|
||||
const server = await startBrowserServer(${options});
|
||||
console.log('SERVER_PID=' + server.pid);
|
||||
setInterval(() => {}, 1000);`;
|
||||
const worker = spawn(process.execPath, ['--input-type=module', '-e', script], { stdio: ['ignore', 'pipe', 'inherit'] });
|
||||
let out = '';
|
||||
worker.stdout.setEncoding('utf8');
|
||||
worker.stdout.on('data', chunk => { out += chunk; });
|
||||
await waitFor(() => /SERVER_PID=\d+\n/.test(out) || worker.exitCode !== null, 'the worker to start its server', REAL_SERVER_TIMEOUT_MS);
|
||||
const pid = Number(/SERVER_PID=(\d+)/.exec(out)?.[1]);
|
||||
assert.ok(pid > 0, `the worker started a server (output: ${out})`);
|
||||
assert.ok(alive(pid), 'the server runs while its worker lives');
|
||||
worker.kill('SIGKILL');
|
||||
await once(worker, 'exit');
|
||||
return pid;
|
||||
}
|
||||
|
||||
function listen(port = 0) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const server = createServer();
|
||||
server.once('error', reject);
|
||||
server.listen(port, '127.0.0.1', () => resolve(server));
|
||||
});
|
||||
}
|
||||
|
||||
async function releasedPort() {
|
||||
const probe = await listen();
|
||||
const { port } = probe.address();
|
||||
await new Promise(resolve => probe.close(resolve));
|
||||
return port;
|
||||
}
|
||||
|
||||
test('free port allocation never reuses a bound port', async t => {
|
||||
const held = await listen();
|
||||
t.after(() => held.close());
|
||||
const heldPort = held.address().port;
|
||||
const servers = await Promise.all(Array.from({ length: 6 }, () => startBrowserServer(fake())));
|
||||
t.after(() => Promise.all(servers.map(server => server.stop())));
|
||||
const ports = servers.map(server => server.port);
|
||||
for (const server of servers) {
|
||||
assert.ok(Number.isInteger(server.port) && server.port > 0 && server.port < 65536, `port ${server.port} is a TCP port`);
|
||||
assert.equal(server.url, `http://127.0.0.1:${server.port}`);
|
||||
assert.equal(await (await fetch(`${server.url}/prompt_manager/test`)).text(), 'ready');
|
||||
}
|
||||
assert.ok(!ports.includes(heldPort), 'a port another process holds is never handed out');
|
||||
assert.equal(new Set(ports).size, ports.length, 'concurrent servers never share a port');
|
||||
});
|
||||
|
||||
test('startBrowserServer asks the server to bind port 0 and reads the port it announces', async () => {
|
||||
const asked = [];
|
||||
const server = await startBrowserServer({ command: process.execPath, args: port => { asked.push(port); return fake().args(port); } });
|
||||
try {
|
||||
assert.deepEqual(asked, [0], 'no port is reserved and released before the server binds');
|
||||
assert.ok(server.port > 0);
|
||||
} finally {
|
||||
await server.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test('startBrowserServer serves the readiness page on a requested port and stops cleanly', async () => {
|
||||
const port = await releasedPort();
|
||||
const server = await startBrowserServer({ port, ...fake() });
|
||||
assert.equal(server.port, port);
|
||||
assert.equal(server.url, `http://127.0.0.1:${port}`);
|
||||
assert.equal(await (await fetch(`${server.url}/prompt_manager/test`)).text(), 'ready');
|
||||
await server.stop();
|
||||
await assert.rejects(fetch(`${server.url}/prompt_manager/test`));
|
||||
});
|
||||
|
||||
test('startBrowserServer reports a startup failure with the server\'s stderr tail', async () => {
|
||||
await assert.rejects(startBrowserServer(fake('--fail')), error => {
|
||||
assert.match(error.message, /exited with code 4 before it was ready/);
|
||||
assert.match(error.message, /last fake stderr line/);
|
||||
return true;
|
||||
});
|
||||
});
|
||||
|
||||
test('startBrowserServer gives up on a server that never announces a port or becomes ready', async () => {
|
||||
const silent = { command: process.execPath, args: () => ['-e', 'setInterval(() => {}, 1000)'] };
|
||||
await assert.rejects(startBrowserServer({ ...silent, timeoutMs: 500 }), /not ready .* within 500 ms/);
|
||||
});
|
||||
|
||||
test('worker server restarts after an unexpected exit', async t => {
|
||||
const supervisor = createServerSupervisor({ start: () => startBrowserServer(fake()) });
|
||||
t.after(() => supervisor.stop());
|
||||
const first = await supervisor.ensure();
|
||||
assert.equal(supervisor.crash(first), null, 'a running server reports no crash');
|
||||
await fetch(`${first.url}/__exit`);
|
||||
await first.exited;
|
||||
const crash = supervisor.crash(first);
|
||||
assert.match(crash.message, /test server on port \d+ exited unexpectedly \(code 3\)/);
|
||||
assert.match(crash.message, /crashed on purpose/);
|
||||
const second = await supervisor.ensure();
|
||||
assert.notEqual(second, first, 'the next test gets a fresh server');
|
||||
assert.equal(await (await fetch(`${second.url}/prompt_manager/test`)).text(), 'ready');
|
||||
assert.equal(await supervisor.ensure(), second, 'a healthy server is reused');
|
||||
});
|
||||
|
||||
test('supervisor reports a startup failure once and tries again on the next ensure', async t => {
|
||||
let attempts = 0;
|
||||
const supervisor = createServerSupervisor({
|
||||
start: async () => {
|
||||
attempts += 1;
|
||||
if (attempts === 1) throw new Error('ImportError: no module');
|
||||
return startBrowserServer(fake());
|
||||
},
|
||||
});
|
||||
t.after(() => supervisor.stop());
|
||||
await assert.rejects(supervisor.ensure(), /ImportError/);
|
||||
assert.equal(attempts, 1, 'a failed startup is not retried behind the caller\'s back');
|
||||
const server = await supervisor.ensure();
|
||||
assert.equal(attempts, 2);
|
||||
assert.equal(await (await fetch(`${server.url}/prompt_manager/test`)).text(), 'ready');
|
||||
});
|
||||
|
||||
test('the port announcement counts only once its line is complete', async t => {
|
||||
const server = await startBrowserServer({ ...fake('--split-announce'), timeoutMs: 5_000 });
|
||||
t.after(() => server.stop());
|
||||
assert.equal(await (await fetch(`${server.url}/prompt_manager/test`)).text(), 'ready');
|
||||
assert.equal(server.url, `http://127.0.0.1:${server.port}`);
|
||||
});
|
||||
|
||||
test('server stderr since a mark holds only what came after it, bounded to the newest output', async t => {
|
||||
const server = await startBrowserServer({ ...fake(), stderrLogChars: 64 });
|
||||
t.after(() => server.stop());
|
||||
await writeStderr(server, 'earlier test\n');
|
||||
const mark = server.stderrMark();
|
||||
assert.equal(server.stderrSince(mark), '');
|
||||
await writeStderr(server, 'Traceback: this test\n');
|
||||
assert.equal(server.stderrSince(mark), 'Traceback: this test\n');
|
||||
const flood = `${'x'.repeat(80)}newest\n`;
|
||||
await writeStderr(server, flood);
|
||||
const kept = server.stderrSince(mark);
|
||||
assert.match(kept, /^\[earlier server stderr dropped\]\n/);
|
||||
assert.ok(kept.endsWith('newest\n'), 'the newest output is kept');
|
||||
assert.ok(kept.length <= 64 + '[earlier server stderr dropped]\n'.length, `bounded, got ${kept.length} chars`);
|
||||
});
|
||||
|
||||
test('a failed test gets the server stderr written during it attached; a passing one does not', async t => {
|
||||
const server = await startBrowserServer(fake());
|
||||
t.after(() => server.stop());
|
||||
const attached = [];
|
||||
const info = status => ({ status, expectedStatus: 'passed', attach: async (name, options) => { attached.push({ name, ...options }); } });
|
||||
await writeStderr(server, 'before the test\n');
|
||||
const mark = server.stderrMark();
|
||||
await attachServerStderr(info('failed'), server, mark);
|
||||
assert.deepEqual(attached, [], 'nothing to attach when the server wrote nothing');
|
||||
await writeStderr(server, 'Traceback (most recent call last): boom\n');
|
||||
await attachServerStderr(info('passed'), server, mark);
|
||||
assert.deepEqual(attached, [], 'a passing test attaches nothing');
|
||||
await attachServerStderr(info('timedOut'), server, mark);
|
||||
assert.equal(attached.length, 1);
|
||||
assert.equal(attached[0].name, 'server-stderr');
|
||||
assert.equal(attached[0].contentType, 'text/plain');
|
||||
assert.equal(String(attached[0].body), 'Traceback (most recent call last): boom\n');
|
||||
});
|
||||
|
||||
test('a server exits with its worker even when the worker is SIGKILLed', async () => {
|
||||
const pid = await serverOfKilledWorker(`{ command: process.execPath, args: port => [${JSON.stringify(fakeServer)}, '--port', String(port), '--exit-on-stdin-eof'] }`);
|
||||
await waitFor(() => !alive(pid), `server ${pid} to exit after its worker died`);
|
||||
});
|
||||
|
||||
test('tests/browser/server.py exits with a SIGKILLed worker', { timeout: 2 * REAL_SERVER_TIMEOUT_MS }, async () => {
|
||||
const pid = await serverOfKilledWorker(`{ timeoutMs: ${REAL_SERVER_TIMEOUT_MS} }`);
|
||||
await waitFor(() => !alive(pid), `server.py ${pid} to exit after its worker died`, 15_000);
|
||||
});
|
||||
|
||||
test('tests/browser/server.py binds an OS-chosen port for --port 0 and still honours --port N', { timeout: 2 * REAL_SERVER_TIMEOUT_MS }, async t => {
|
||||
const held = await listen();
|
||||
t.after(() => held.close());
|
||||
const chosen = await startBrowserServer({ timeoutMs: REAL_SERVER_TIMEOUT_MS });
|
||||
try {
|
||||
assert.notEqual(chosen.port, held.address().port);
|
||||
assert.equal((await fetch(`${chosen.url}/prompt_manager/test`)).status, 200);
|
||||
} finally {
|
||||
await chosen.stop();
|
||||
}
|
||||
const port = await releasedPort();
|
||||
const fixed = await startBrowserServer({ port, timeoutMs: REAL_SERVER_TIMEOUT_MS });
|
||||
try {
|
||||
assert.equal(fixed.port, port);
|
||||
assert.equal((await fetch(`http://127.0.0.1:${port}/prompt_manager/test`)).status, 200);
|
||||
} finally {
|
||||
await fixed.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test('worker count follows PM_TEST_WORKERS, then one in CI, then a quarter of the CPUs up to eight', () => {
|
||||
assert.equal(workflowWorkers({ env: { PM_TEST_WORKERS: '3', CI: '1' }, cpus: 32 }), 3);
|
||||
assert.equal(workflowWorkers({ env: { CI: '1' }, cpus: 32 }), 1);
|
||||
assert.equal(workflowWorkers({ env: {}, cpus: 32 }), 8);
|
||||
assert.equal(workflowWorkers({ env: {}, cpus: 12 }), 3);
|
||||
assert.equal(workflowWorkers({ env: {}, cpus: 2 }), 1);
|
||||
assert.throws(() => workflowWorkers({ env: { PM_TEST_WORKERS: 'many' }, cpus: 8 }), /PM_TEST_WORKERS must be a positive integer/);
|
||||
assert.throws(() => workflowWorkers({ env: { PM_TEST_WORKERS: '0' }, cpus: 8 }), /PM_TEST_WORKERS must be a positive integer/);
|
||||
});
|
||||
Vendored
+223
@@ -0,0 +1,223 @@
|
||||
// Structural checks on .github/workflows/viewerjs-check.yml. The tooling has no YAML parser, so these
|
||||
// read the workflow's fixed two-space layout: jobs at two spaces, their keys at four, steps at six.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { mkdtemp, mkdir, readdir, rm, writeFile } from 'node:fs/promises';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
const workflow = readFileSync(new URL('../../.github/workflows/viewerjs-check.yml', import.meta.url), 'utf8');
|
||||
const PINNED_IMAGE = /^\s+image: mcr\.microsoft\.com\/playwright:v\d+\.\d+\.\d+-noble$/m;
|
||||
|
||||
function jobs(text) {
|
||||
const lines = text.split('\n');
|
||||
const start = lines.indexOf('jobs:');
|
||||
assert.notEqual(start, -1, 'workflow has a jobs section');
|
||||
const found = new Map();
|
||||
let current = null;
|
||||
for (const line of lines.slice(start + 1)) {
|
||||
const header = /^ {2}([\w-]+):\s*$/.exec(line);
|
||||
if (header) {
|
||||
current = header[1];
|
||||
found.set(current, []);
|
||||
} else if (current && (line.startsWith(' ') || line.trim() === '' || /^ {2}#/.test(line))) {
|
||||
found.get(current).push(line);
|
||||
} else if (current && /^\S/.test(line)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return new Map([...found].map(([id, body]) => [id, body.join('\n')]));
|
||||
}
|
||||
|
||||
function steps(jobText) {
|
||||
const parts = jobText.split(/^(?= {6}- )/m);
|
||||
return parts.slice(1);
|
||||
}
|
||||
|
||||
function runScript(stepText) {
|
||||
const lines = stepText.split('\n');
|
||||
const index = lines.findIndex(line => /^ {8}run: \|\s*$/.test(line));
|
||||
assert.notEqual(index, -1, 'step has a block run script');
|
||||
const body = [];
|
||||
for (const line of lines.slice(index + 1)) {
|
||||
if (line.trim() !== '' && !line.startsWith(' ')) break;
|
||||
body.push(line.slice(10));
|
||||
}
|
||||
return body.join('\n');
|
||||
}
|
||||
|
||||
function needsOf(jobText) {
|
||||
const match = /^ {4}needs: \[([^\]]*)\]\s*$/m.exec(jobText) ?? /^ {4}needs: ([\w-]+)\s*$/m.exec(jobText);
|
||||
assert.ok(match, 'job declares needs');
|
||||
return match[1].split(',').map(value => value.trim()).filter(Boolean);
|
||||
}
|
||||
|
||||
function matrixList(jobText, key) {
|
||||
const match = new RegExp(`^ {8}${key}: \\[([^\\]]*)\\]\\s*$`, 'm').exec(jobText);
|
||||
assert.ok(match, `job has a ${key} matrix`);
|
||||
return match[1].split(',').map(value => value.trim());
|
||||
}
|
||||
|
||||
const allJobs = jobs(workflow);
|
||||
const browserJobIds = [...allJobs].filter(([, body]) => /npm run --prefix tools\/vendor test:(workflows|coverage)/.test(body))
|
||||
.map(([id]) => id);
|
||||
const mergeJob = () => allJobs.get('frontend-coverage-merge');
|
||||
const verifyStep = () => steps(mergeJob()).find(step => /name: Verify every shard's raw coverage arrived/.test(step));
|
||||
|
||||
test('viewerjs keeps its name and needs every frontend job', () => {
|
||||
const viewerjs = allJobs.get('viewerjs');
|
||||
assert.ok(viewerjs, 'the required context is the job id viewerjs');
|
||||
assert.doesNotMatch(viewerjs, /^ {4}name:/m, 'no display name may replace the context "viewerjs"');
|
||||
const needs = needsOf(viewerjs);
|
||||
const others = [...allJobs.keys()].filter(id => id !== 'viewerjs');
|
||||
assert.deepEqual([...needs].sort(), [...others].sort(), 'viewerjs needs every other job in the workflow');
|
||||
assert.deepEqual([...needs].sort(), ['browser-workflows', 'frontend-coverage', 'frontend-coverage-merge']);
|
||||
assert.match(viewerjs, /^ {4}if: always\(\)$/m, 'viewerjs runs (and fails) when a needed job failed or was skipped');
|
||||
const script = runScript(steps(viewerjs).find(step => /run: \|/.test(step)));
|
||||
for (const id of needs) {
|
||||
const variable = new RegExp(`^ {10}([A-Z_]+): \\$\\{\\{ needs\\.${id}\\.result \\}\\}$`, 'm').exec(viewerjs);
|
||||
assert.ok(variable, `${id}'s result reaches the check`);
|
||||
assert.match(script, new RegExp(`test "\\$${variable[1]}" = success`), `${id} must be exactly success`);
|
||||
}
|
||||
});
|
||||
|
||||
test('viewerjs fails unless every needed job succeeded', () => {
|
||||
const viewerjs = allJobs.get('viewerjs');
|
||||
const script = runScript(steps(viewerjs).find(step => /run: \|/.test(step)));
|
||||
const variables = [...viewerjs.matchAll(/^ {10}([A-Z_]+): \$\{\{ needs\.[\w-]+\.result \}\}$/gm)].map(match => match[1]);
|
||||
const run = env => {
|
||||
try {
|
||||
execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', script], { env: { PATH: process.env.PATH, ...env }, stdio: 'pipe' });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
const allSuccess = Object.fromEntries(variables.map(name => [name, 'success']));
|
||||
assert.equal(run(allSuccess), true);
|
||||
for (const name of variables) {
|
||||
for (const result of ['failure', 'cancelled', 'skipped']) {
|
||||
assert.equal(run({ ...allSuccess, [name]: result }), false, `${name}=${result} must fail viewerjs`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('coverage runs as three shards that each upload their raw coverage', () => {
|
||||
const coverage = allJobs.get('frontend-coverage');
|
||||
assert.deepEqual(matrixList(coverage, 'shard'), ['1', '2', '3']);
|
||||
assert.match(coverage, /^ {4}name: frontend coverage \(shard \$\{\{ matrix\.shard \}\}\/3\)$/m);
|
||||
assert.match(coverage, /npm run --prefix tools\/vendor test:coverage:shard -- \$\{\{ matrix\.shard \}\}\/3/);
|
||||
const upload = steps(coverage).find(step => /name: coverage-raw-\$\{\{ matrix\.shard \}\}/.test(step));
|
||||
assert.ok(upload, 'each shard uploads coverage-raw-N');
|
||||
assert.match(upload, /^ {10}path: coverage\/js\/raw\/?$/m);
|
||||
assert.match(upload, /^ {10}retention-days: 3$/m);
|
||||
assert.match(upload, /^ {10}if-no-files-found: error$/m);
|
||||
});
|
||||
|
||||
test('merge job downloads every shard and runs the merge-only report and check', () => {
|
||||
const merge = mergeJob();
|
||||
assert.ok(merge, 'frontend-coverage-merge exists');
|
||||
assert.deepEqual(needsOf(merge), ['frontend-coverage']);
|
||||
assert.match(merge, /^ {4}name: frontend coverage \(merge\)$/m);
|
||||
const download = steps(merge).find(step => /uses: actions\/download-artifact@/.test(step));
|
||||
assert.ok(download, 'merge downloads the shard artifacts');
|
||||
assert.match(download, /^ {10}pattern: coverage-raw-\*$/m);
|
||||
assert.doesNotMatch(download, /merge-multiple: true/, 'each artifact keeps its own directory so the check can count them');
|
||||
assert.ok(verifyStep(), 'merge has an explicit artifact count check');
|
||||
assert.match(verifyStep(), new RegExp(`^ {10}EXPECTED_SHARDS: ${matrixList(allJobs.get('frontend-coverage'), 'shard').length}$`, 'm'),
|
||||
'the expected count matches the shard matrix');
|
||||
const order = steps(merge).map(step => (/name: (.*)/.exec(step) ?? [, step.trim().split('\n')[0]])[1]);
|
||||
const verifyAt = order.findIndex(name => /Verify every shard/.test(name));
|
||||
const mergeAt = steps(merge).findIndex(step => /npm run --prefix tools\/vendor coverage:merge/.test(step));
|
||||
assert.ok(verifyAt >= 0 && mergeAt > verifyAt, 'the count check runs before coverage:merge');
|
||||
const report = steps(merge).find(step => /name: frontend-test-reports$/m.test(step));
|
||||
assert.ok(report, 'merge uploads the HTML report as frontend-test-reports');
|
||||
assert.match(report, /!coverage\/js\/raw\/\*\*/);
|
||||
});
|
||||
|
||||
async function stagedShards(root, shards) {
|
||||
const staged = join(root, 'staged');
|
||||
for (const [shard, files] of shards) {
|
||||
const directory = join(staged, `coverage-raw-${shard}`);
|
||||
await mkdir(directory, { recursive: true });
|
||||
for (const file of files) await writeFile(join(directory, file), '{"result":[]}');
|
||||
}
|
||||
await mkdir(staged, { recursive: true });
|
||||
return staged;
|
||||
}
|
||||
|
||||
async function verify(t, shards) {
|
||||
const root = await mkdtemp(join(tmpdir(), 'pm-ci-merge-'));
|
||||
t.after(() => rm(root, { recursive: true, force: true }));
|
||||
const staged = await stagedShards(root, shards);
|
||||
try {
|
||||
execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', runScript(verifyStep())], {
|
||||
cwd: root, stdio: 'pipe',
|
||||
env: { PATH: process.env.PATH, STAGED_RAW: staged, EXPECTED_SHARDS: '3' },
|
||||
});
|
||||
} catch (error) {
|
||||
return { ok: false, output: `${error.stdout}${error.stderr}` };
|
||||
}
|
||||
return { ok: true, raw: (await readdir(join(root, 'coverage/js/raw'))).sort() };
|
||||
}
|
||||
|
||||
test('merge refuses when a shard\'s raw coverage is missing', async t => {
|
||||
const missing = await verify(t, [[1, ['a.json']], [3, ['c.json']]]);
|
||||
assert.equal(missing.ok, false);
|
||||
assert.match(missing.output, /2 of 3/);
|
||||
const empty = await verify(t, [[1, ['a.json']], [2, []], [3, ['c.json']]]);
|
||||
assert.equal(empty.ok, false, 'an artifact without raw files counts as missing');
|
||||
const collision = await verify(t, [[1, ['a.json']], [2, ['a.json']], [3, ['c.json']]]);
|
||||
assert.equal(collision.ok, false, 'two shards writing the same file name must not silently drop one');
|
||||
const complete = await verify(t, [[1, ['a.json', 'node.json']], [2, ['b.json']], [3, ['c.json']]]);
|
||||
assert.deepEqual(complete, { ok: true, raw: ['a.json', 'b.json', 'c.json', 'node.json'] });
|
||||
});
|
||||
|
||||
test('every browser job sets PM_TEST_WORKERS and runs in the pinned image', () => {
|
||||
assert.deepEqual([...browserJobIds].sort(), ['browser-workflows', 'frontend-coverage']);
|
||||
for (const id of browserJobIds) {
|
||||
const body = allJobs.get(id);
|
||||
assert.match(body, /^ {6}PM_TEST_WORKERS: 3$/m, `${id} sets PM_TEST_WORKERS for the whole job`);
|
||||
assert.match(body, PINNED_IMAGE, `${id} keeps the pinned Playwright image`);
|
||||
assert.match(body, /PIP_SHA256: [0-9a-f]{64}/, `${id} keeps the pinned pip bootstrap`);
|
||||
assert.match(body, /^ {4}timeout-minutes: \d+$/m);
|
||||
}
|
||||
assert.match(mergeJob(), PINNED_IMAGE, 'the merge sees the same workspace paths as the shards');
|
||||
});
|
||||
|
||||
test('page workflows run each browser in two shards', () => {
|
||||
const workflows = allJobs.get('browser-workflows');
|
||||
assert.deepEqual(matrixList(workflows, 'browser'), ['firefox', 'webkit']);
|
||||
assert.deepEqual(matrixList(workflows, 'shard'), ['1', '2']);
|
||||
assert.match(workflows, /test:workflows -- --project \$\{\{ matrix\.browser \}\} --shard \$\{\{ matrix\.shard \}\}\/2/);
|
||||
assert.match(workflows, /^ {6}fail-fast: false$/m);
|
||||
assert.match(workflows, /name: browser-traces-\$\{\{ matrix\.browser \}\}-shard-\$\{\{ matrix\.shard \}\}/);
|
||||
});
|
||||
|
||||
test('every npm ci restores the npm download cache keyed on the vendor lockfile first', () => {
|
||||
const installing = [...allJobs].filter(([, body]) => /npm ci --prefix tools\/vendor/.test(body));
|
||||
assert.deepEqual(installing.map(([id]) => id).sort(), ['browser-workflows', 'frontend-coverage', 'frontend-coverage-merge']);
|
||||
for (const [id, body] of installing) {
|
||||
const jobSteps = steps(body);
|
||||
const cacheAt = jobSteps.findIndex(step => /uses: actions\/cache@v\d+/.test(step));
|
||||
const installAt = jobSteps.findIndex(step => /npm ci --prefix tools\/vendor/.test(step));
|
||||
assert.ok(cacheAt >= 0 && cacheAt < installAt, `${id} restores the npm cache before npm ci`);
|
||||
assert.match(jobSteps[cacheAt], /^ {10}path: ~\/\.npm$/m, `${id} caches npm's download cache, not node_modules`);
|
||||
assert.match(jobSteps[cacheAt], /hashFiles\('tools\/vendor\/package-lock\.json'\)/, `${id} keys the cache on the lockfile`);
|
||||
assert.match(jobSteps[installAt], /npm ci --prefix tools\/vendor --ignore-scripts/, `${id} still installs without scripts`);
|
||||
// The key names the job's own pinned image, so an image bump starts a fresh cache.
|
||||
const image = /^ {6}image: mcr\.microsoft\.com\/playwright:(v\d+\.\d+\.\d+)-noble$/m.exec(body)?.[1];
|
||||
assert.ok(image, `${id} runs in the pinned Playwright image`);
|
||||
assert.match(jobSteps[cacheAt], new RegExp(`key: npm-\\$\\{\\{ runner\\.os \\}\\}-playwright-${image.replaceAll('.', '\\.')}-\\$\\{\\{ hashFiles`),
|
||||
`${id} keys the cache on its image ${image}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('browser job timeouts keep a real margin over measured runs', () => {
|
||||
for (const id of ['frontend-coverage', 'browser-workflows']) {
|
||||
const minutes = Number(/^ {4}timeout-minutes: (\d+)$/m.exec(allJobs.get(id))[1]);
|
||||
assert.ok(minutes >= 20, `${id} allows at least 20 minutes, since runner speed varies`);
|
||||
}
|
||||
});
|
||||
Vendored
+40
-11
@@ -73,11 +73,41 @@ test('overlay rules on concept selectors carry a decision marker', () => {
|
||||
assert.deepEqual(violations, []);
|
||||
});
|
||||
|
||||
// Each kept page stylesheet is linted: a page rule on a concept selector is an override like the overlay's.
|
||||
test('kept page stylesheets mark their concept-selector overrides', () => {
|
||||
for (const name of ['library.css']) {
|
||||
const { violations } = lintOverlay(readFileSync(concept, 'utf8'), readFileSync(join(root, 'web/js/workspace', name), 'utf8'));
|
||||
assert.deepEqual(violations, [], name);
|
||||
// #249 PR F-b Task 3: every page is rebuilt, so no rule waits for a page PR any more and the history page's
|
||||
// production-only rules live in the overlay in the concept's tokens (D48). A comment that opens with a page PR
|
||||
// ("PR F: ...", "PR E/F: ...") or defers to one ("until ... PR", "retired in PR F") is a waiting rule; a provenance
|
||||
// label inside a heading ("(#249 PR E)") is not.
|
||||
const WAITING_RULE = [
|
||||
/\/\*\s*PR [A-F](\/[A-F]|-[ab])?\b/i,
|
||||
/\buntil\b[^*]*\b(PR [A-F]?|rebuild)/i,
|
||||
/\bretired in PR\b/i,
|
||||
/\binterim page guards\b/i,
|
||||
];
|
||||
const waitingRules = css => WAITING_RULE.filter(pattern => pattern.test(css)).map(String);
|
||||
|
||||
test('the waiting-rule guard catches the retired comments and spares provenance labels', () => {
|
||||
// The exact comments this PR retired (workspace.css and library.css at 10a4847).
|
||||
for (const retired of [
|
||||
'/* Interim page guards: concept element rules (`.ca-tools dialog`, `pre`) now reach production dialogs and prompt text',
|
||||
' that keep their own design until their page PR rebuilds them in concept markup. */',
|
||||
'.ca-tools dialog.prompt-sender::backdrop { background: #000b; } /* PR D */',
|
||||
'.ca-tools dialog.recorded-run-dialog { padding: 16px; } /* PR F */',
|
||||
' .ca-tools .recorded-history .history-toolbar button { min-height: 44px; } /* PR F: concept coarse buttons are 40px */',
|
||||
'/* Until their pages rebuild (PR F history): the history page\'s and the recorded run\'s prompt blocks keep their',
|
||||
'/* PR E/F: the Archive dialog and the frozen field provenance keep `pre.prompt-text`, guarded against the concept `pre` rule. */',
|
||||
" keeps the concept's) and are retired in PR F (history). */",
|
||||
]) assert.notDeepEqual(waitingRules(retired), [], retired);
|
||||
for (const kept of [
|
||||
"/* Prompt library cards (#249 PR E): the concept's `article.prompt-card` (s653–708) with real prompts, titles and media. */",
|
||||
'/* History variants panel (#249 PR F). */',
|
||||
]) assert.deepEqual(waitingRules(kept), [], kept);
|
||||
});
|
||||
|
||||
test('the overlay keeps no rule waiting for a page rebuild', () => {
|
||||
const css = readFileSync(join(root, 'web/js/workspace/workspace.css'), 'utf8');
|
||||
assert.deepEqual(waitingRules(css), []);
|
||||
for (const retired of ['.library-chip', '--library-positive', '--library-negative', '--prompt-role']) {
|
||||
assert.equal(css.includes(retired), false, `${retired} is retired`);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -100,22 +130,21 @@ test('only workspace.html loads workspace CSS', () => {
|
||||
assert.ok(offenders.every(path => path === 'web/workspace.html'), `unexpected consumers: ${offenders.join(', ')}`);
|
||||
});
|
||||
|
||||
// Task A2 order: the concept's own order (toolkit, Viewer.js, concept) with the page stylesheets that their own PRs
|
||||
// retire, then the production overlay last.
|
||||
test('workspace.html loads controls, viewer, concept, page CSS and the overlay in order', () => {
|
||||
// Task A2 order: the concept's own order (toolkit, Viewer.js, concept), then the production overlay last; the page
|
||||
// stylesheets that waited for their PRs are all retired (PR F-b deleted the last, library.css).
|
||||
test('workspace.html loads controls, viewer, concept and the overlay in order', () => {
|
||||
const html = readFileSync(join(root, 'web/workspace.html'), 'utf8');
|
||||
const hrefs = [...html.matchAll(/<link[^>]+rel="stylesheet"[^>]+href="([^"]+)"/g)].map(match => match[1]);
|
||||
assert.deepEqual(hrefs, [
|
||||
'js/shared/toolkit/controls.css',
|
||||
'lib/viewerjs/viewer.min.css',
|
||||
'js/workspace/concept.css',
|
||||
'js/workspace/library.css',
|
||||
'js/workspace/workspace.css',
|
||||
]);
|
||||
});
|
||||
|
||||
test('the retired shell, vocabulary, editor, Images and inspector stylesheets stay deleted', () => {
|
||||
for (const name of ['style.css', 'vocabulary.css', 'library-editor.css', 'images.css', 'inspector.css']) {
|
||||
test('the retired shell, vocabulary, editor, Images, inspector and library stylesheets stay deleted', () => {
|
||||
for (const name of ['style.css', 'vocabulary.css', 'library-editor.css', 'images.css', 'inspector.css', 'library.css']) {
|
||||
assert.equal(existsSync(join(root, 'web/js/workspace', name)), false, `web/js/workspace/${name} must not come back`);
|
||||
}
|
||||
});
|
||||
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { baselineGaps, measuredSources, readBaseline } from '../../tools/vendor/coverage-baseline-check.mjs';
|
||||
|
||||
const floor = { lines: 80, branches: 80, functions: 80, statements: 80 };
|
||||
|
||||
test('baseline gaps name measured files without a baseline entry', () => {
|
||||
const gaps = baselineGaps(['a.js', 'nested/new.mjs'], { 'a.js': floor });
|
||||
assert.deepEqual(gaps.missing, ['nested/new.mjs']);
|
||||
assert.deepEqual(gaps.stale, []);
|
||||
});
|
||||
|
||||
test('baseline gaps name baseline entries whose file no longer exists', () => {
|
||||
const gaps = baselineGaps(['a.js'], { 'a.js': floor, 'gone.js': floor });
|
||||
assert.deepEqual(gaps.missing, []);
|
||||
assert.deepEqual(gaps.stale, ['gone.js']);
|
||||
});
|
||||
|
||||
test('baseline gaps are empty when files and baseline agree', () => {
|
||||
assert.deepEqual(baselineGaps(['a.js'], { 'a.js': floor }), { missing: [], stale: [] });
|
||||
});
|
||||
|
||||
test('the committed baseline covers exactly the measured frontend sources', () => {
|
||||
const gaps = baselineGaps(measuredSources(), readBaseline());
|
||||
assert.deepEqual(gaps.missing, [], `Missing baseline (run coverage:update): ${gaps.missing.join(', ')}`);
|
||||
assert.deepEqual(gaps.stale, [], `Stale baseline entries (file removed): ${gaps.stale.join(', ')}`);
|
||||
});
|
||||
|
||||
test('the real gate and this check select files through the same function', () => {
|
||||
const gate = readFileSync(new URL('../../tools/vendor/frontend-coverage.mjs', import.meta.url), 'utf8');
|
||||
assert.match(gate, /discoverSources\(sourceRoot\)/);
|
||||
});
|
||||
Vendored
+44
@@ -0,0 +1,44 @@
|
||||
// A stand-in for tests/browser/server.py: `--port N` (0 = an OS-chosen port)
|
||||
// serves the readiness page and announces the bound port on stdout as
|
||||
// `PM_TEST_SERVER_PORT=<n>`, `--fail` writes to stderr and exits before
|
||||
// listening, and GET /__exit makes the process die unexpectedly after writing a
|
||||
// last stderr line. GET /__stderr?text=T writes T to stderr before answering.
|
||||
// `--split-announce` writes the announcement in two chunks, the first ending
|
||||
// mid-number. `--exit-on-stdin-eof` exits when its stdin closes, as server.py does.
|
||||
import { createServer } from 'node:http';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const port = Number(args[args.indexOf('--port') + 1]);
|
||||
if (args.includes('--fail')) {
|
||||
process.stderr.write('fake server: refusing to start\nlast fake stderr line\n');
|
||||
process.exit(4);
|
||||
}
|
||||
const server = createServer((request, response) => {
|
||||
const url = new URL(request.url, 'http://fake');
|
||||
if (url.pathname === '/__stderr') {
|
||||
process.stderr.write(url.searchParams.get('text'));
|
||||
response.end('written');
|
||||
return;
|
||||
}
|
||||
if (request.url === '/__exit') {
|
||||
process.stderr.write('fake server: crashed on purpose\n');
|
||||
response.end('bye', () => process.exit(3));
|
||||
return;
|
||||
}
|
||||
response.end(request.url === '/prompt_manager/test' ? 'ready' : `pid ${process.pid}`);
|
||||
});
|
||||
server.listen(port, '127.0.0.1', () => {
|
||||
const line = `PM_TEST_SERVER_PORT=${server.address().port}\n`;
|
||||
if (!args.includes('--split-announce')) {
|
||||
process.stdout.write(`fake server: listening\n${line}`);
|
||||
return;
|
||||
}
|
||||
// The first chunk is a complete-looking number, one digit short of the real port.
|
||||
process.stdout.write(`fake server: listening\n${line.slice(0, -2)}`);
|
||||
setTimeout(() => process.stdout.write(line.slice(-2)), 300);
|
||||
});
|
||||
if (args.includes('--exit-on-stdin-eof')) {
|
||||
process.stdin.resume();
|
||||
process.stdin.once('end', () => process.exit(0));
|
||||
}
|
||||
for (const signal of ['SIGTERM', 'SIGINT']) process.once(signal, () => server.close(() => process.exit(0)));
|
||||
+93
@@ -0,0 +1,93 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { existsSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { plan, runCoverage } from '../../tools/vendor/frontend-coverage.mjs';
|
||||
|
||||
const steps = argv => plan(argv).map(({ step, ...options }) => (Object.keys(options).length ? { step, ...options } : step));
|
||||
|
||||
function tempPaths(t) {
|
||||
const root = mkdtempSync(join(tmpdir(), 'pm-coverage-cli-'));
|
||||
t.after(() => rmSync(root, { recursive: true, force: true }));
|
||||
const reportRoot = join(root, 'coverage/js');
|
||||
return { root, paths: { repositoryRoot: root, tools: join(root, 'tools'), reportRoot, raw: join(reportRoot, 'raw'), sourceRoot: join(root, 'web/js') } };
|
||||
}
|
||||
|
||||
function recorder() {
|
||||
const calls = [];
|
||||
const run = (args, env = {}) => { calls.push({ args, env }); };
|
||||
return { calls, run };
|
||||
}
|
||||
|
||||
test('no arguments plans the unchanged full run', () => {
|
||||
assert.deepEqual(steps([]), [
|
||||
'clean', 'node-units', 'viewer', { step: 'workflows', shard: null }, 'report', { step: 'check', updateBaseline: false },
|
||||
]);
|
||||
assert.deepEqual(steps(['--update-baseline']), [
|
||||
'clean', 'node-units', 'viewer', { step: 'workflows', shard: null }, 'report', { step: 'check', updateBaseline: true },
|
||||
]);
|
||||
});
|
||||
|
||||
test('--shard 2/3 plans only workflows with --shard 2/3 and no report', () => {
|
||||
assert.deepEqual(steps(['--shard', '2/3']), ['prepare', { step: 'workflows', shard: '2/3' }]);
|
||||
assert.deepEqual(steps(['--shard=3/3']), ['prepare', { step: 'workflows', shard: '3/3' }]);
|
||||
});
|
||||
|
||||
test('--shard 1/N also runs the node units and the viewer suite', () => {
|
||||
assert.deepEqual(steps(['--shard', '1/3']), ['prepare', 'node-units', 'viewer', { step: 'workflows', shard: '1/3' }]);
|
||||
assert.deepEqual(steps(['--shard', '1/1']), ['prepare', 'node-units', 'viewer', { step: 'workflows', shard: '1/1' }]);
|
||||
});
|
||||
|
||||
test('--merge-only plans the report and the check over existing raw coverage', () => {
|
||||
assert.deepEqual(steps(['--merge-only']), ['require-raw', 'report', { step: 'check', updateBaseline: false }]);
|
||||
assert.deepEqual(steps(['--merge-only', '--update-baseline']), ['require-raw', 'report', { step: 'check', updateBaseline: true }]);
|
||||
});
|
||||
|
||||
test('invalid shard and mode combinations are refused', () => {
|
||||
for (const argv of [['--shard'], ['--shard', '0/3'], ['--shard', '4/3'], ['--shard', '2'], ['--shard', 'a/b'], ['--shard', '1/0'], ['--shard', '1.5/3']]) {
|
||||
assert.throws(() => plan(argv), /--shard needs I\/N/, argv.join(' '));
|
||||
}
|
||||
assert.throws(() => plan(['--shard', '1/2', '--merge-only']), /either --shard or --merge-only/);
|
||||
assert.throws(() => plan(['--shard', '1/2', '--update-baseline']), /--update-baseline .* --merge-only or a full run/);
|
||||
assert.throws(() => plan(['--shard', '1/2', '--shard', '2/2']), /--shard once/);
|
||||
assert.throws(() => plan(['--report']), /Unknown argument: --report/);
|
||||
});
|
||||
|
||||
test('--merge-only refuses an empty raw dir', async t => {
|
||||
const { paths } = tempPaths(t);
|
||||
const { calls, run } = recorder();
|
||||
await assert.rejects(runCoverage({ argv: ['--merge-only'], run, paths }), /no raw coverage .*coverage\/js\/raw.* run the --shard steps first/i);
|
||||
mkdirSync(paths.raw, { recursive: true });
|
||||
await assert.rejects(runCoverage({ argv: ['--merge-only'], run, paths }), /no raw coverage/i);
|
||||
writeFileSync(join(paths.raw, 'notes.txt'), 'not coverage');
|
||||
await assert.rejects(runCoverage({ argv: ['--merge-only'], run, paths }), /no raw coverage/i);
|
||||
assert.deepEqual(calls, [], 'nothing is reported or checked over partial or empty coverage');
|
||||
});
|
||||
|
||||
test('a shard keeps raw coverage other shards wrote and passes its shard to Playwright', async t => {
|
||||
const { paths } = tempPaths(t);
|
||||
mkdirSync(paths.raw, { recursive: true });
|
||||
const earlier = join(paths.raw, 'browser-earlier-shard.json');
|
||||
writeFileSync(earlier, '{"result":[]}');
|
||||
const { calls, run } = recorder();
|
||||
await runCoverage({ argv: ['--shard', '2/3'], run, paths });
|
||||
assert.ok(existsSync(earlier), 'raw files from another shard survive');
|
||||
assert.equal(calls.length, 1, 'shard 2 runs only the workflows');
|
||||
const [{ args, env }] = calls;
|
||||
assert.deepEqual(args.slice(-6), ['--project', 'chromium', '--project', 'touch', '--shard', '2/3']);
|
||||
assert.equal(env.PM_JS_COVERAGE, paths.raw);
|
||||
});
|
||||
|
||||
test('shard 1 runs the node units into the shared raw dir, then the viewer suite, then its workflows', async t => {
|
||||
const { root, paths } = tempPaths(t);
|
||||
mkdirSync(join(root, 'tests/js'), { recursive: true });
|
||||
writeFileSync(join(root, 'tests/js/a.test.js'), '');
|
||||
const { calls, run } = recorder();
|
||||
await runCoverage({ argv: ['--shard', '1/3'], run, paths });
|
||||
assert.equal(calls.length, 3);
|
||||
assert.deepEqual(calls[0].args, ['--test', 'tests/js/a.test.js']);
|
||||
assert.equal(calls[0].env.NODE_V8_COVERAGE, paths.raw);
|
||||
assert.ok(calls[1].args.includes(join(paths.tools, 'playwright.config.mjs')));
|
||||
assert.deepEqual(calls[2].args.slice(-2), ['--shard', '1/3']);
|
||||
});
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
// Fast, browser-free check that frontend-coverage-baseline.json lists exactly the
|
||||
// files the coverage gate measures. CI's `frontend coverage` job reports a missing
|
||||
// entry only after ~20 minutes; this reports it in milliseconds.
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { repositoryRoot } from './vendor-viewerjs.mjs';
|
||||
import { discoverSources } from './frontend-coverage.mjs';
|
||||
|
||||
const baselinePath = join(dirname(fileURLToPath(import.meta.url)), 'frontend-coverage-baseline.json');
|
||||
|
||||
export function measuredSources() {
|
||||
return discoverSources(join(repositoryRoot, 'web/js'));
|
||||
}
|
||||
|
||||
export function readBaseline() {
|
||||
return JSON.parse(readFileSync(baselinePath, 'utf8'));
|
||||
}
|
||||
|
||||
export function baselineGaps(files, baseline) {
|
||||
const measured = new Set(files);
|
||||
return {
|
||||
missing: files.filter(file => !baseline[file]),
|
||||
stale: Object.keys(baseline).filter(file => !measured.has(file)).sort(),
|
||||
};
|
||||
}
|
||||
|
||||
function main() {
|
||||
const { missing, stale } = baselineGaps(measuredSources(), readBaseline());
|
||||
if (!missing.length && !stale.length) {
|
||||
console.log('Coverage baseline matches the measured frontend files.');
|
||||
return;
|
||||
}
|
||||
if (missing.length) console.error(`Missing baseline for:\n ${missing.join('\n ')}\nRun: npm run --prefix tools/vendor coverage:update`);
|
||||
if (stale.length) console.error(`Baseline lists files that no longer exist:\n ${stale.join('\n ')}\nRemove them from tools/vendor/frontend-coverage-baseline.json`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) main();
|
||||
+44
-14
@@ -155,12 +155,36 @@
|
||||
"functions": 98,
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/file-inspector.mjs": {
|
||||
"workspace/activity-job-view.mjs": {
|
||||
"lines": 97,
|
||||
"branches": 83,
|
||||
"functions": 98,
|
||||
"statements": 97
|
||||
},
|
||||
"workspace/activity-scan.mjs": {
|
||||
"lines": 95,
|
||||
"branches": 92,
|
||||
"functions": 88,
|
||||
"branches": 64,
|
||||
"functions": 98,
|
||||
"statements": 95
|
||||
},
|
||||
"workspace/activity-store.mjs": {
|
||||
"lines": 98,
|
||||
"branches": 95,
|
||||
"functions": 98,
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/activity.mjs": {
|
||||
"lines": 95,
|
||||
"branches": 87,
|
||||
"functions": 98,
|
||||
"statements": 95
|
||||
},
|
||||
"workspace/file-inspector.mjs": {
|
||||
"lines": 96,
|
||||
"branches": 92,
|
||||
"functions": 92,
|
||||
"statements": 96
|
||||
},
|
||||
"workspace/heading.mjs": {
|
||||
"lines": 98,
|
||||
"branches": 98,
|
||||
@@ -229,7 +253,7 @@
|
||||
},
|
||||
"workspace/index.mjs": {
|
||||
"lines": 98,
|
||||
"branches": 96,
|
||||
"branches": 97,
|
||||
"functions": 98,
|
||||
"statements": 98
|
||||
},
|
||||
@@ -264,10 +288,10 @@
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/library.mjs": {
|
||||
"lines": 95,
|
||||
"lines": 96,
|
||||
"branches": 85,
|
||||
"functions": 96,
|
||||
"statements": 95
|
||||
"statements": 96
|
||||
},
|
||||
"workspace/media-viewer.mjs": {
|
||||
"lines": 98,
|
||||
@@ -282,10 +306,10 @@
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/prompt-blocks.mjs": {
|
||||
"lines": 89,
|
||||
"branches": 89,
|
||||
"lines": 90,
|
||||
"branches": 92,
|
||||
"functions": 98,
|
||||
"statements": 89
|
||||
"statements": 90
|
||||
},
|
||||
"workspace/prompt-inspector.mjs": {
|
||||
"lines": 98,
|
||||
@@ -302,9 +326,15 @@
|
||||
"workspace/recorded-run.mjs": {
|
||||
"lines": 95,
|
||||
"branches": 83,
|
||||
"functions": 77,
|
||||
"functions": 80,
|
||||
"statements": 95
|
||||
},
|
||||
"workspace/saved-views.mjs": {
|
||||
"lines": 98,
|
||||
"branches": 95,
|
||||
"functions": 95,
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/state.mjs": {
|
||||
"lines": 98,
|
||||
"branches": 98,
|
||||
@@ -319,15 +349,15 @@
|
||||
},
|
||||
"workspace/toast.mjs": {
|
||||
"lines": 98,
|
||||
"branches": 64,
|
||||
"branches": 98,
|
||||
"functions": 98,
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/toolbar.mjs": {
|
||||
"lines": 94,
|
||||
"branches": 93,
|
||||
"lines": 96,
|
||||
"branches": 96,
|
||||
"functions": 98,
|
||||
"statements": 94
|
||||
"statements": 96
|
||||
},
|
||||
"workspace/topbar.mjs": {
|
||||
"lines": 98,
|
||||
|
||||
Vendored
+99
-20
@@ -1,4 +1,4 @@
|
||||
import { readFileSync, readdirSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { existsSync, readFileSync, readdirSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { dirname, join, relative } from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
@@ -7,33 +7,69 @@ import { repositoryRoot } from './vendor-viewerjs.mjs';
|
||||
import { discoverSources, checkCoverage } from '@comfyassets/toolkit/testing/coverage';
|
||||
export { discoverSources, checkCoverage };
|
||||
const metrics = ['lines', 'branches', 'functions', 'statements'];
|
||||
const SHARD = /^([1-9]\d*)\/([1-9]\d*)$/;
|
||||
const USAGE = 'usage: frontend-coverage.mjs [--update-baseline] | --shard I/N | --merge-only [--update-baseline]';
|
||||
|
||||
function run(args, env = {}) {
|
||||
function runNode(args, env = {}) {
|
||||
const result = spawnSync(process.execPath, args, { cwd: repositoryRoot, stdio: 'inherit',
|
||||
env: { ...process.env, ...env } });
|
||||
if (result.status !== 0) throw new Error(`Command failed (${result.status}): ${args.join(' ')}`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const tools = dirname(fileURLToPath(import.meta.url));
|
||||
function defaultPaths() {
|
||||
const reportRoot = join(repositoryRoot, 'coverage/js');
|
||||
const raw = join(reportRoot, 'raw');
|
||||
const sourceRoot = join(repositoryRoot, 'web/js');
|
||||
return { repositoryRoot, tools: dirname(fileURLToPath(import.meta.url)), reportRoot,
|
||||
raw: join(reportRoot, 'raw'), sourceRoot: join(repositoryRoot, 'web/js') };
|
||||
}
|
||||
|
||||
function parseShard(value) {
|
||||
const match = SHARD.exec(value ?? '');
|
||||
if (!match || Number(match[1]) > Number(match[2])) {
|
||||
throw new Error(`--shard needs I/N with 1 <= I <= N, got ${value ?? 'nothing'}\n${USAGE}`);
|
||||
}
|
||||
return { index: Number(match[1]), total: Number(match[2]), value };
|
||||
}
|
||||
|
||||
// Turn the command line into the ordered steps of one coverage run:
|
||||
// - no mode: the full run (clean, node units, viewer suite, chromium+touch
|
||||
// workflows, report, check), exactly as before shards existed;
|
||||
// - --shard I/N: one slice of the workflows into coverage/js/raw, without wiping
|
||||
// it or reporting; shard 1 also runs the node units and the viewer suite;
|
||||
// - --merge-only: report and check over raw coverage the shards already wrote.
|
||||
export function plan(argv) {
|
||||
let shard = null;
|
||||
let mergeOnly = false;
|
||||
let updateBaseline = false;
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === '--update-baseline') updateBaseline = true;
|
||||
else if (arg === '--merge-only') mergeOnly = true;
|
||||
else if (arg === '--shard' || arg.startsWith('--shard=')) {
|
||||
if (shard) throw new Error(`Pass --shard once\n${USAGE}`);
|
||||
shard = parseShard(arg === '--shard' ? argv[(index += 1)] : arg.slice('--shard='.length));
|
||||
} else throw new Error(`Unknown argument: ${arg}\n${USAGE}`);
|
||||
}
|
||||
if (shard && mergeOnly) throw new Error(`Pass either --shard or --merge-only\n${USAGE}`);
|
||||
if (shard && updateBaseline) throw new Error(`--update-baseline needs --merge-only or a full run, not a shard\n${USAGE}`);
|
||||
const check = { step: 'check', updateBaseline };
|
||||
if (mergeOnly) return [{ step: 'require-raw' }, { step: 'report' }, check];
|
||||
if (shard) {
|
||||
const first = shard.index === 1 ? [{ step: 'node-units' }, { step: 'viewer' }] : [];
|
||||
return [{ step: 'prepare' }, ...first, { step: 'workflows', shard: shard.value }];
|
||||
}
|
||||
return [{ step: 'clean' }, { step: 'node-units' }, { step: 'viewer' }, { step: 'workflows', shard: null },
|
||||
{ step: 'report' }, check];
|
||||
}
|
||||
|
||||
function rawCoverageFiles(raw) {
|
||||
return existsSync(raw) ? readdirSync(raw).filter(file => file.endsWith('.json')) : [];
|
||||
}
|
||||
|
||||
function checkSummary({ tools, reportRoot, sourceRoot }, updateBaseline) {
|
||||
const files = discoverSources(sourceRoot);
|
||||
rmSync(reportRoot, { recursive: true, force: true });
|
||||
mkdirSync(raw, { recursive: true });
|
||||
const tests = readdirSync(join(repositoryRoot, 'tests/js')).filter(file => file.endsWith('.test.js')).map(file => `tests/js/${file}`);
|
||||
run(['--test', ...tests], { NODE_V8_COVERAGE: raw });
|
||||
const playwright = join(tools, 'node_modules/@playwright/test/cli.js');
|
||||
run([playwright, 'test', '--config', join(tools, 'playwright.config.mjs')]);
|
||||
run([playwright, 'test', '--config', join(tools, 'playwright.workflows.config.mjs'),
|
||||
'--project', 'chromium', '--project', 'touch'], { PM_JS_COVERAGE: raw });
|
||||
run([join(tools, 'node_modules/c8/bin/c8.js'), 'report', '--all', '--src', sourceRoot,
|
||||
'--include', 'web/js/**/*.js', '--include', 'web/js/**/*.mjs', '--temp-directory', raw, '--reports-dir', reportRoot,
|
||||
'--reporter', 'text', '--reporter', 'json-summary', '--reporter', 'html']);
|
||||
const summary = JSON.parse(readFileSync(join(reportRoot, 'coverage-summary.json'), 'utf8'));
|
||||
const baselinePath = join(tools, 'frontend-coverage-baseline.json');
|
||||
if (process.argv.includes('--update-baseline')) {
|
||||
if (updateBaseline) {
|
||||
const previous = JSON.parse(readFileSync(baselinePath, 'utf8'));
|
||||
const failures = checkCoverage(summary, previous, sourceRoot, files.filter(file => previous[file]));
|
||||
if (failures.length) throw new Error(`Baseline updates cannot lower coverage:\n${failures.join('\n')}`);
|
||||
@@ -53,8 +89,51 @@ async function main() {
|
||||
const failures = checkCoverage(summary, baseline, sourceRoot, files);
|
||||
if (failures.length) throw new Error(`Frontend coverage regressed:\n${failures.join('\n')}`);
|
||||
}
|
||||
console.log(`Frontend coverage passed for ${files.length} files. Report: ${relative(repositoryRoot, reportRoot)}/index.html`);
|
||||
return files.length;
|
||||
}
|
||||
|
||||
export async function runCoverage({ argv = process.argv.slice(2), run = runNode, paths = defaultPaths() } = {}) {
|
||||
const { repositoryRoot: root, tools, reportRoot, raw, sourceRoot } = paths;
|
||||
const playwright = join(tools, 'node_modules/@playwright/test/cli.js');
|
||||
const actions = {
|
||||
clean() {
|
||||
rmSync(reportRoot, { recursive: true, force: true });
|
||||
mkdirSync(raw, { recursive: true });
|
||||
},
|
||||
prepare() {
|
||||
mkdirSync(raw, { recursive: true });
|
||||
},
|
||||
'require-raw'() {
|
||||
if (!rawCoverageFiles(raw).length) {
|
||||
throw new Error(`No raw coverage in ${relative(root, raw) || raw}: run the --shard steps first, `
|
||||
+ 'or put every shard\'s raw files there');
|
||||
}
|
||||
},
|
||||
'node-units'() {
|
||||
const tests = readdirSync(join(root, 'tests/js')).filter(file => file.endsWith('.test.js'))
|
||||
.map(file => `tests/js/${file}`);
|
||||
run(['--test', ...tests], { NODE_V8_COVERAGE: raw });
|
||||
},
|
||||
viewer() {
|
||||
run([playwright, 'test', '--config', join(tools, 'playwright.config.mjs')]);
|
||||
},
|
||||
workflows({ shard }) {
|
||||
run([playwright, 'test', '--config', join(tools, 'playwright.workflows.config.mjs'),
|
||||
'--project', 'chromium', '--project', 'touch', ...(shard ? ['--shard', shard] : [])], { PM_JS_COVERAGE: raw });
|
||||
},
|
||||
report() {
|
||||
run([join(tools, 'node_modules/c8/bin/c8.js'), 'report', '--all', '--src', sourceRoot,
|
||||
'--include', 'web/js/**/*.js', '--include', 'web/js/**/*.mjs', '--temp-directory', raw, '--reports-dir', reportRoot,
|
||||
'--reporter', 'text', '--reporter', 'json-summary', '--reporter', 'html']);
|
||||
},
|
||||
check({ updateBaseline }) {
|
||||
const count = checkSummary(paths, updateBaseline);
|
||||
console.log(`Frontend coverage passed for ${count} files. Report: ${relative(root, reportRoot)}/index.html`);
|
||||
},
|
||||
};
|
||||
for (const { step, ...options } of plan(argv)) await actions[step](options);
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch(error => { console.error(error.message); process.exitCode = 1; });
|
||||
runCoverage().catch(error => { console.error(error.message); process.exitCode = 1; });
|
||||
}
|
||||
|
||||
Vendored
+3
@@ -15,6 +15,9 @@
|
||||
"test:comfyui": "playwright test --config comfyui.config.mjs",
|
||||
"test:executions": "playwright test --config playwright.execution.config.mjs",
|
||||
"test:coverage": "node frontend-coverage.mjs",
|
||||
"test:coverage:shard": "node frontend-coverage.mjs --shard",
|
||||
"coverage:merge": "node frontend-coverage.mjs --merge-only",
|
||||
"coverage:baseline-check": "node coverage-baseline-check.mjs",
|
||||
"coverage:update": "node frontend-coverage.mjs --update-baseline",
|
||||
"test:compat": "playwright test --config pack-compat.config.mjs"
|
||||
},
|
||||
|
||||
+6
-11
@@ -1,23 +1,18 @@
|
||||
import { defineConfig, devices } from '@playwright/test';
|
||||
import { repositoryRoot } from './vendor-viewerjs.mjs';
|
||||
const port = Number(process.env.PM_TEST_PORT || 8765);
|
||||
import { workflowWorkers } from './test-workers.mjs';
|
||||
|
||||
// No global webServer: tests/browser/fixtures.mjs starts one server per worker
|
||||
// on an OS-chosen port. Files run in parallel; tests inside a file keep their order.
|
||||
export default defineConfig({
|
||||
testDir: '../../tests/browser',
|
||||
outputDir: './test-results/workflows',
|
||||
testMatch: '*.spec.mjs',
|
||||
workers: 1,
|
||||
workers: workflowWorkers(),
|
||||
fullyParallel: false,
|
||||
forbidOnly: Boolean(process.env.CI),
|
||||
retries: 0,
|
||||
reporter: 'list',
|
||||
use: { baseURL: `http://127.0.0.1:${port}`, trace: 'retain-on-failure', screenshot: 'only-on-failure' },
|
||||
webServer: {
|
||||
command: `node tools/vendor/start-browser-server.mjs --port ${port}`,
|
||||
cwd: repositoryRoot,
|
||||
url: `http://127.0.0.1:${port}/prompt_manager/test`,
|
||||
reuseExistingServer: false,
|
||||
gracefulShutdown: { signal: 'SIGTERM', timeout: 5000 },
|
||||
},
|
||||
use: { trace: 'retain-on-failure', screenshot: 'only-on-failure' },
|
||||
projects: [
|
||||
{ name: 'chromium', outputDir: './test-results/workflows/chromium', use: { browserName: 'chromium' } },
|
||||
{ name: 'firefox', outputDir: './test-results/workflows/firefox', use: { browserName: 'firefox' } },
|
||||
|
||||
+187
-6
@@ -1,8 +1,189 @@
|
||||
import { spawn } from 'node:child_process';
|
||||
const child = spawn(process.env.PM_TEST_PYTHON || 'python',
|
||||
['-m', 'tests.browser.server', ...process.argv.slice(2)], { stdio: 'inherit' });
|
||||
child.on('error', error => { console.error(error.message); process.exitCode = 1; });
|
||||
child.on('exit', code => { process.exitCode = code ?? 1; });
|
||||
for (const signal of ['SIGTERM', 'SIGINT']) {
|
||||
process.once(signal, () => child.kill(signal));
|
||||
import { resolve } from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
import { repositoryRoot } from './vendor-viewerjs.mjs';
|
||||
|
||||
const READY_PATH = '/prompt_manager/test';
|
||||
const STARTUP_TIMEOUT_MS = 60_000;
|
||||
const STOP_TIMEOUT_MS = 5_000;
|
||||
const POLL_INTERVAL_MS = 100;
|
||||
const PROBE_TIMEOUT_MS = 1_000;
|
||||
const STDERR_TAIL_BYTES = 8_192;
|
||||
// Server stderr kept for the failing test's attachment; the newest output wins.
|
||||
const STDERR_LOG_CHARS = 256 * 1024;
|
||||
const STDERR_DROPPED = '[earlier server stderr dropped]\n';
|
||||
const HOST = '127.0.0.1';
|
||||
// tests/browser/server.py prints this once it holds its listening socket. Only a
|
||||
// complete line counts: a chunk may end mid-number.
|
||||
const PORT_ANNOUNCEMENT = /^PM_TEST_SERVER_PORT=(\d+)\r?\n/m;
|
||||
|
||||
// The server exits when its stdin closes, so a worker that dies without teardown
|
||||
// (SIGKILL) leaves no server behind; the launcher holds the pipe's other end.
|
||||
const pythonServer = port => ['-m', 'tests.browser.server', '--port', String(port), '--exit-on-stdin-eof'];
|
||||
|
||||
function describeExit({ code, signal }) {
|
||||
return signal ? `signal ${signal}` : `code ${code}`;
|
||||
}
|
||||
|
||||
// Start one disposable tests/browser/server.py and wait until it serves the
|
||||
// readiness page. `port` 0 (the default) lets the server bind an OS-chosen port
|
||||
// itself and announce it on stdout, so nothing can take the port between
|
||||
// choosing and binding. Resolves to {port, url, exited, exitStatus,
|
||||
// pid, stderrTail(), stderrMark(), stderrSince(mark), stop()}; rejects with the
|
||||
// server's stderr tail when it exits or stays unready. stdin is a pipe this
|
||||
// process holds open for the server's life.
|
||||
export async function startBrowserServer({
|
||||
port = 0,
|
||||
command = process.env.PM_TEST_PYTHON || 'python',
|
||||
args = pythonServer,
|
||||
cwd = repositoryRoot,
|
||||
timeoutMs = STARTUP_TIMEOUT_MS,
|
||||
stderrLogChars = STDERR_LOG_CHARS,
|
||||
} = {}) {
|
||||
if (!Number.isInteger(port) || port < 0 || port > 65535) {
|
||||
throw new Error(`startBrowserServer needs a port from 0 to 65535, got ${port}`);
|
||||
}
|
||||
const child = spawn(command, args(port), { cwd, stdio: ['pipe', 'pipe', 'pipe'] });
|
||||
// Never written; closing it (or this process dying) tells the server to exit.
|
||||
child.stdin.on('error', () => {});
|
||||
let stderr = '';
|
||||
let stderrTotal = 0;
|
||||
child.stderr.setEncoding('utf8');
|
||||
child.stderr.on('data', chunk => {
|
||||
stderr = (stderr + chunk).slice(-Math.max(stderrLogChars, STDERR_TAIL_BYTES));
|
||||
stderrTotal += chunk.length;
|
||||
});
|
||||
let stdout = '';
|
||||
let announced = null;
|
||||
child.stdout.setEncoding('utf8');
|
||||
// Keep draining stdout for the server's whole life so it never blocks on a full pipe.
|
||||
child.stdout.on('data', chunk => {
|
||||
if (announced !== null) return;
|
||||
stdout = (stdout + chunk).slice(-STDERR_TAIL_BYTES);
|
||||
const match = PORT_ANNOUNCEMENT.exec(stdout);
|
||||
if (match) announced = Number(match[1]);
|
||||
});
|
||||
const killOnExit = () => child.kill('SIGKILL');
|
||||
process.once('exit', killOnExit);
|
||||
const server = {
|
||||
pid: child.pid,
|
||||
port: port || null,
|
||||
url: port ? `http://${HOST}:${port}` : null,
|
||||
exitStatus: null,
|
||||
stopping: false,
|
||||
stderrTail: () => stderr.slice(-STDERR_TAIL_BYTES).trimEnd(),
|
||||
// A position in the server's stderr stream, for stderrSince().
|
||||
stderrMark: () => stderrTotal,
|
||||
// What the server wrote after `mark`, at most stderrLogChars of the newest of it.
|
||||
stderrSince(mark) {
|
||||
const kept = Math.min(stderr.length, stderrLogChars);
|
||||
const keptFrom = stderrTotal - kept;
|
||||
const log = stderr.slice(stderr.length - kept);
|
||||
return mark >= keptFrom ? log.slice(mark - keptFrom) : STDERR_DROPPED + log;
|
||||
},
|
||||
exited: new Promise(done => {
|
||||
child.once('error', error => {
|
||||
stderr += `\n${error.message}`;
|
||||
server.exitStatus = { code: null, signal: null, error };
|
||||
done(server.exitStatus);
|
||||
});
|
||||
child.once('exit', (code, signal) => {
|
||||
server.exitStatus = { code, signal };
|
||||
done(server.exitStatus);
|
||||
});
|
||||
}),
|
||||
async stop() {
|
||||
server.stopping = true;
|
||||
child.stdin.destroy();
|
||||
if (server.exitStatus === null) {
|
||||
child.kill('SIGTERM');
|
||||
const stopped = await Promise.race([server.exited, delay(STOP_TIMEOUT_MS).then(() => false)]);
|
||||
if (!stopped) {
|
||||
child.kill('SIGKILL');
|
||||
await server.exited;
|
||||
}
|
||||
}
|
||||
process.removeListener('exit', killOnExit);
|
||||
},
|
||||
};
|
||||
server.exited.then(() => process.removeListener('exit', killOnExit));
|
||||
const failure = async reason => {
|
||||
await server.stop();
|
||||
const tail = server.stderrTail();
|
||||
return new Error(`test server on port ${server.port ?? '(not yet announced)'} ${reason}`
|
||||
+ (tail ? `\n--- server stderr tail ---\n${tail}` : ''));
|
||||
};
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
if (server.exitStatus !== null) {
|
||||
throw await failure(`exited with ${describeExit(server.exitStatus)} before it was ready`);
|
||||
}
|
||||
if (announced !== null && server.url === null) {
|
||||
server.port = announced;
|
||||
server.url = `http://${HOST}:${announced}`;
|
||||
}
|
||||
if (announced !== null && announced !== server.port) {
|
||||
throw await failure(`announced port ${announced} instead`);
|
||||
}
|
||||
if (server.url === null) {
|
||||
await delay(POLL_INTERVAL_MS);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const response = await fetch(`${server.url}${READY_PATH}`, { signal: AbortSignal.timeout(PROBE_TIMEOUT_MS) });
|
||||
await response.arrayBuffer();
|
||||
if (response.ok) return server;
|
||||
} catch {
|
||||
// Not listening yet.
|
||||
}
|
||||
await delay(POLL_INTERVAL_MS);
|
||||
}
|
||||
throw await failure(`was not ready at ${READY_PATH} within ${timeoutMs} ms`);
|
||||
}
|
||||
|
||||
// After a test: when it did not end as expected, attach what the server wrote to
|
||||
// stderr since `mark` (an aiohttp traceback names the cause a "got 500" hides).
|
||||
export async function attachServerStderr(testInfo, server, mark) {
|
||||
if (testInfo.status === testInfo.expectedStatus) return;
|
||||
const log = server.stderrSince(mark);
|
||||
if (log) await testInfo.attach('server-stderr', { body: log, contentType: 'text/plain' });
|
||||
}
|
||||
|
||||
// Keep one live server for a Playwright worker. ensure() returns the running
|
||||
// server, or starts a fresh one when the last one died (a failed start is
|
||||
// reported, not retried); crash(server) turns an unexpected exit into an error
|
||||
// carrying the stderr tail.
|
||||
export function createServerSupervisor({ start }) {
|
||||
let current = null;
|
||||
return {
|
||||
async ensure() {
|
||||
if (current && current.exitStatus === null) return current;
|
||||
current = null;
|
||||
current = await start();
|
||||
return current;
|
||||
},
|
||||
crash(server) {
|
||||
if (!server || server.stopping || server.exitStatus === null) return null;
|
||||
const tail = server.stderrTail();
|
||||
return new Error(`test server on port ${server.port} exited unexpectedly (${describeExit(server.exitStatus)})`
|
||||
+ (tail ? `\n--- server stderr tail ---\n${tail}` : ''));
|
||||
},
|
||||
async stop() {
|
||||
const server = current;
|
||||
current = null;
|
||||
await server?.stop();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// CLI: `node start-browser-server.mjs --port N` runs one server in the foreground.
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
|
||||
const child = spawn(process.env.PM_TEST_PYTHON || 'python',
|
||||
['-m', 'tests.browser.server', ...process.argv.slice(2)], { stdio: 'inherit' });
|
||||
child.on('error', error => { console.error(error.message); process.exitCode = 1; });
|
||||
child.on('exit', code => { process.exitCode = code ?? 1; });
|
||||
for (const signal of ['SIGTERM', 'SIGINT']) {
|
||||
process.once(signal, () => child.kill(signal));
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+18
@@ -0,0 +1,18 @@
|
||||
import { availableParallelism } from 'node:os';
|
||||
|
||||
const MAX_LOCAL_WORKERS = 8;
|
||||
const CPUS_PER_WORKER = 4;
|
||||
|
||||
// PM_TEST_WORKERS wins; CI without it keeps one worker, so CI only changes
|
||||
// through explicit workflow edits; locally a quarter of the CPUs, at most eight.
|
||||
export function workflowWorkers({ env = process.env, cpus = availableParallelism() } = {}) {
|
||||
if (env.PM_TEST_WORKERS !== undefined) {
|
||||
const workers = Number(env.PM_TEST_WORKERS);
|
||||
if (!Number.isInteger(workers) || workers < 1) {
|
||||
throw new Error(`PM_TEST_WORKERS must be a positive integer, got "${env.PM_TEST_WORKERS}"`);
|
||||
}
|
||||
return workers;
|
||||
}
|
||||
if (env.CI) return 1;
|
||||
return Math.max(1, Math.min(MAX_LOCAL_WORKERS, Math.floor(cpus / CPUS_PER_WORKER)));
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user