Compare commits
17
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
37bdc20ca9 | ||
|
|
37b31940fa | ||
|
|
21ba73ff39 | ||
|
|
7a47f936da | ||
|
|
af26c2fd49 | ||
|
|
43e70514d1 | ||
|
|
38f9f8acc3 | ||
|
|
b9636ce88c | ||
|
|
0e6ec0647f | ||
|
|
a3d5c16318 | ||
|
|
bc14f29db7 | ||
|
|
1659b58c5c | ||
|
|
3cfa81419b | ||
|
|
f95f66d76a | ||
|
|
2cecfcf0ac | ||
|
|
a6f6697c50 | ||
|
|
6e9d8f81fd |
@@ -16,9 +16,19 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
# Timeouts are the measured job time plus at least 50%, and never under 20 minutes for a browser job,
|
||||
# because runner speed varies. Before sharding (main 2cecfcf, one worker): frontend coverage 32m10s,
|
||||
# page workflows firefox 18m59s, webkit 19m26s. First sharded run (PR #274 run 39712909047, 3 workers,
|
||||
# no npm cache, where npm ci stalled for 5m in six of eight jobs): coverage shards 11m19s/10m47s/10m49s
|
||||
# (tests 5m20s/4m42s/4m46s), merge 5m52s (report and check 4s), firefox shards 7m53s/9m27s (tests
|
||||
# 4m41s/3m29s), webkit shards 12m25s+ (cancelled at the old 12-minute limit after its 267 tests passed in
|
||||
# 6m29s) and 4m53s (tests 3m08s). With the npm cache hit (run 39725176320 at cbc8a61, npm ci 0-2s):
|
||||
# coverage shards 7m34s/6m04s/5m22s (tests 6m38s/4m53s/4m04s), merge 54s, firefox 7m57s/5m21s (tests
|
||||
# 7m08s/4m34s), webkit 8m16s/5m03s (tests 7m26s/3m58s); a cache miss adds up to 5m to npm ci, which
|
||||
# the 20-minute limits absorb.
|
||||
jobs:
|
||||
frontend-coverage:
|
||||
name: frontend coverage (chromium and touch)
|
||||
name: frontend coverage (shard ${{ matrix.shard }}/3)
|
||||
runs-on: ubuntu-latest
|
||||
# Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the
|
||||
# same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json;
|
||||
@@ -26,12 +36,15 @@ jobs:
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble
|
||||
options: --user 1001
|
||||
# This job runs every page workflow twice (chromium, then touch) on one worker with V8 coverage on, so it
|
||||
# takes about twice a single-browser job. Measured 2026-10-20: whole job 18m26s-24m57s (main 5ded3cf
|
||||
# 22m15s; PR head 6359999 was cancelled at the old 25-minute limit while still passing), coverage step
|
||||
# alone 20m55s-22m17s+, against ~12-15 minutes for the firefox/webkit jobs. 40 minutes leaves headroom
|
||||
# for suite growth while still stopping a hung run; re-measure if a run passes 30 minutes.
|
||||
timeout-minutes: 40
|
||||
timeout-minutes: 20
|
||||
# Public-repo ubuntu-latest runners have 4 vCPUs: three Playwright workers, each with its own test
|
||||
# server, leave one CPU for the servers and the runner.
|
||||
env:
|
||||
PM_TEST_WORKERS: 3
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: [1, 2, 3]
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
# The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache
|
||||
@@ -50,24 +63,107 @@ jobs:
|
||||
"$venv/bin/python" -m pip install --no-cache-dir -r requirements-dev.txt
|
||||
echo "$venv/bin" >> "$GITHUB_PATH"
|
||||
- run: python --version && node --version && node --version | grep -q '^v24\.'
|
||||
# npm's download cache, keyed on the lockfile; inside the container ~ is the job's HOME (/github/home),
|
||||
# which npm also uses. node_modules itself is still built fresh by npm ci.
|
||||
- name: Cache npm downloads
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/.npm
|
||||
key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }}
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
- run: npm run --prefix tools/vendor check
|
||||
- run: npm run --prefix tools/vendor test
|
||||
- name: Frontend unit, viewer, workflow, and coverage checks
|
||||
run: npm run --prefix tools/vendor test:coverage
|
||||
- name: Upload frontend coverage and Chromium traces
|
||||
# Shard 1 also runs the node units and the viewer suite, so the vendor pins and the tooling tests
|
||||
# run on the lightest shard instead.
|
||||
- if: matrix.shard == 3
|
||||
run: npm run --prefix tools/vendor check
|
||||
- if: matrix.shard == 3
|
||||
run: npm run --prefix tools/vendor test
|
||||
- name: Page workflows with coverage, shard ${{ matrix.shard }}/3
|
||||
run: npm run --prefix tools/vendor test:coverage:shard -- ${{ matrix.shard }}/3
|
||||
- name: Upload raw coverage
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: coverage-raw-${{ matrix.shard }}
|
||||
path: coverage/js/raw/
|
||||
if-no-files-found: error
|
||||
retention-days: 3
|
||||
- name: Upload Chromium and touch traces
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: frontend-traces-shard-${{ matrix.shard }}
|
||||
path: tools/vendor/test-results/
|
||||
retention-days: 14
|
||||
|
||||
frontend-coverage-merge:
|
||||
name: frontend coverage (merge)
|
||||
needs: frontend-coverage
|
||||
runs-on: ubuntu-latest
|
||||
# Same image as the shards: the raw files name sources by the container's workspace path.
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble
|
||||
options: --user 1001
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- run: node --version && node --version | grep -q '^v24\.'
|
||||
# npm's download cache, keyed on the lockfile; inside the container ~ is the job's HOME (/github/home),
|
||||
# which npm also uses. node_modules itself is still built fresh by npm ci.
|
||||
- name: Cache npm downloads
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/.npm
|
||||
key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }}
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
# Each artifact lands in its own coverage-raw-N directory, so the next step can count them.
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: coverage-raw-*
|
||||
path: ${{ runner.temp }}/coverage-raw
|
||||
# coverage:merge only checks that raw files exist; merging two of three shards would report
|
||||
# partial coverage, so a missing or empty shard artifact fails here.
|
||||
- name: Verify every shard's raw coverage arrived
|
||||
shell: bash
|
||||
env:
|
||||
STAGED_RAW: ${{ runner.temp }}/coverage-raw
|
||||
EXPECTED_SHARDS: 3
|
||||
run: |
|
||||
found=0
|
||||
for shard in $(seq 1 "$EXPECTED_SHARDS"); do
|
||||
directory="$STAGED_RAW/coverage-raw-$shard"
|
||||
count=0
|
||||
if [ -d "$directory" ]; then count=$(find "$directory" -maxdepth 1 -type f -name '*.json' | wc -l); fi
|
||||
echo "coverage-raw-$shard: $count raw files"
|
||||
if [ "$count" -gt 0 ]; then found=$((found + 1)); fi
|
||||
done
|
||||
echo "shard artifacts with raw coverage: $found of $EXPECTED_SHARDS"
|
||||
if [ "$found" -ne "$EXPECTED_SHARDS" ]; then
|
||||
echo "::error::Expected raw coverage from $EXPECTED_SHARDS shards, got $found of $EXPECTED_SHARDS"
|
||||
exit 1
|
||||
fi
|
||||
staged=$(find "$STAGED_RAW" -mindepth 2 -maxdepth 2 -type f -name '*.json' | wc -l)
|
||||
mkdir -p coverage/js/raw
|
||||
# cp -n never overwrites; a clash shows up as fewer merged files than staged ones.
|
||||
find "$STAGED_RAW" -mindepth 2 -maxdepth 2 -type f -name '*.json' -exec cp -n -t coverage/js/raw {} + || true
|
||||
merged=$(find coverage/js/raw -maxdepth 1 -type f -name '*.json' | wc -l)
|
||||
if [ "$merged" -ne "$staged" ]; then
|
||||
echo "::error::Shards wrote clashing raw file names: $staged staged, $merged merged"
|
||||
exit 1
|
||||
fi
|
||||
echo "merged $merged raw files"
|
||||
- name: Frontend coverage report and baseline check
|
||||
run: npm run --prefix tools/vendor coverage:merge
|
||||
- name: Upload frontend coverage report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: frontend-test-reports
|
||||
path: |
|
||||
tools/vendor/test-results/
|
||||
coverage/js/
|
||||
!coverage/js/raw/**
|
||||
retention-days: 14
|
||||
|
||||
browser-workflows:
|
||||
name: page workflows (${{ matrix.browser }})
|
||||
name: page workflows (${{ matrix.browser }}, shard ${{ matrix.shard }}/2)
|
||||
runs-on: ubuntu-latest
|
||||
# Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the
|
||||
# same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json;
|
||||
@@ -75,11 +171,16 @@ jobs:
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble
|
||||
options: --user 1001
|
||||
timeout-minutes: 25
|
||||
timeout-minutes: 20
|
||||
# Public-repo ubuntu-latest runners have 4 vCPUs: three Playwright workers, each with its own test
|
||||
# server, leave one CPU for the servers and the runner.
|
||||
env:
|
||||
PM_TEST_WORKERS: 3
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
browser: [firefox, webkit]
|
||||
shard: [1, 2]
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
# The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache
|
||||
@@ -98,22 +199,29 @@ jobs:
|
||||
"$venv/bin/python" -m pip install --no-cache-dir -r requirements-dev.txt
|
||||
echo "$venv/bin" >> "$GITHUB_PATH"
|
||||
- run: python --version && node --version && node --version | grep -q '^v24\.'
|
||||
# npm's download cache, keyed on the lockfile; inside the container ~ is the job's HOME (/github/home),
|
||||
# which npm also uses. node_modules itself is still built fresh by npm ci.
|
||||
- name: Cache npm downloads
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/.npm
|
||||
key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }}
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
- name: Page workflows in ${{ matrix.browser }}
|
||||
run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }}
|
||||
- name: Page workflows in ${{ matrix.browser }}, shard ${{ matrix.shard }}/2
|
||||
run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }} --shard ${{ matrix.shard }}/2
|
||||
- name: Upload browser traces
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: browser-traces-${{ matrix.browser }}
|
||||
name: browser-traces-${{ matrix.browser }}-shard-${{ matrix.shard }}
|
||||
path: tools/vendor/test-results/
|
||||
retention-days: 14
|
||||
|
||||
# Branch protection requires the exact context "viewerjs". This job keeps that
|
||||
# name and fails unless every frontend job above succeeded, including when one
|
||||
# was cancelled or skipped.
|
||||
# name and fails unless every frontend job above succeeded (every shard, the
|
||||
# coverage merge and every browser shard), including when one was cancelled or skipped.
|
||||
viewerjs:
|
||||
needs: [frontend-coverage, browser-workflows]
|
||||
needs: [frontend-coverage, frontend-coverage-merge, browser-workflows]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
@@ -121,8 +229,10 @@ jobs:
|
||||
- name: Require every frontend job to succeed
|
||||
env:
|
||||
COVERAGE_RESULT: ${{ needs.frontend-coverage.result }}
|
||||
MERGE_RESULT: ${{ needs.frontend-coverage-merge.result }}
|
||||
WORKFLOWS_RESULT: ${{ needs.browser-workflows.result }}
|
||||
run: |
|
||||
echo "frontend-coverage: $COVERAGE_RESULT"
|
||||
echo "frontend-coverage-merge: $MERGE_RESULT"
|
||||
echo "browser-workflows: $WORKFLOWS_RESULT"
|
||||
test "$COVERAGE_RESULT" = success && test "$WORKFLOWS_RESULT" = success
|
||||
test "$COVERAGE_RESULT" = success && test "$MERGE_RESULT" = success && test "$WORKFLOWS_RESULT" = success
|
||||
|
||||
+353
-19
@@ -39,9 +39,44 @@ npm run --prefix tools/vendor test:coverage
|
||||
npm run --prefix tools/vendor test:workflows -- --project firefox --project webkit
|
||||
```
|
||||
|
||||
`PM_TEST_PYTHON` selects the Python executable; `PM_TEST_PORT` selects the local
|
||||
port (default 8765). Set these environment variables when Python is outside PATH
|
||||
or the port is occupied. Test servers are never reused. On unsupported Linux
|
||||
`PM_TEST_PYTHON` selects the Python executable; set it when Python is outside
|
||||
PATH. Every Playwright worker starts its own `tests/browser/server.py --port 0`,
|
||||
which binds a port the OS picks and prints it as `PM_TEST_SERVER_PORT=<n>`, with
|
||||
its own temp root and database (`tests/browser/fixtures.mjs`), so
|
||||
parallel workers and concurrent runs never share state. `PM_TEST_WORKERS` sets the
|
||||
worker count (default 1 under `CI`, otherwise a quarter of the CPUs, at most 8).
|
||||
Test servers are never reused across runs. The launcher holds each server's
|
||||
stdin open and the server (`--exit-on-stdin-eof`) shuts down cleanly, removing its
|
||||
temp root, when that pipe closes, so a worker killed without teardown leaves no
|
||||
server behind. A test that fails gets the stderr its server wrote during the test
|
||||
attached to the report as `server-stderr` (an aiohttp traceback behind a 500).
|
||||
|
||||
The coverage run can be split the way CI splits it, and each shard keeps the raw
|
||||
V8 files it writes under `coverage/js/raw`:
|
||||
|
||||
```sh
|
||||
PM_TEST_WORKERS=8 npm run --prefix tools/vendor test:coverage # whole run, one report
|
||||
npm run --prefix tools/vendor test:coverage:shard -- 1/3 # node units, viewer suite, workflows 1/3
|
||||
npm run --prefix tools/vendor test:coverage:shard -- 2/3 # workflows 2/3 (likewise 3/3)
|
||||
npm run --prefix tools/vendor coverage:merge # report and baseline check of every raw file
|
||||
npm run --prefix tools/vendor test:workflows -- --project webkit --shard 2/2
|
||||
```
|
||||
|
||||
`coverage:merge` refuses an empty raw directory; it does not count shards, so run
|
||||
every shard first (CI's merge job checks all three arrived). Measured on a 32-CPU
|
||||
machine in the pinned image (2026-10-21): `test:coverage` takes about 5 minutes
|
||||
with 8 workers and 24 minutes with 1; all four projects, 8 workers, each test 3
|
||||
times (6132 tests), 24 minutes. A CI-sized run (`docker run --cpus=4`,
|
||||
`PM_TEST_WORKERS=3`, all four projects once, 2044 tests) takes 25.5 minutes.
|
||||
|
||||
Specs must pass beside each other, not only alone. Wait for the page's own
|
||||
signal, never for time: a rendering frame (`requestAnimationFrame`) for layout
|
||||
work queued by a render or a scroll, a response or a visible state for data. A
|
||||
timer that "usually" covers a frame is a race on a loaded runner. Library
|
||||
virtualization specs settle with `libraryObservation.settled()` (two consecutive
|
||||
frames with no Library frame queued). To look for such races, repeat a spec under
|
||||
load (`--workers 8 --repeat-each 10`, and in the image with `--cpus=1` or `2`).
|
||||
On unsupported Linux
|
||||
distributions, browser system dependencies may need manual installation. For a
|
||||
quick browser run, use `test:workflows -- --project chromium`.
|
||||
|
||||
@@ -114,16 +149,39 @@ second without a browser and fails, naming the files, when a measured
|
||||
`web/js` file has no baseline entry or the baseline lists a file that no longer
|
||||
exists. It uses the gate's own `discoverSources`, and `npm test` runs the same
|
||||
check, so run it before pushing a new or deleted frontend file instead of
|
||||
waiting for the `frontend coverage` job.
|
||||
waiting for the `frontend coverage` jobs.
|
||||
|
||||
The `viewerjs` check runs on every PR, including backend-only changes. It is an
|
||||
aggregate job that passes only when the `frontend coverage (chromium and touch)`
|
||||
job and both `page workflows (firefox)` and `page workflows (webkit)` jobs
|
||||
succeed; the browsers run on separate runners in parallel. GitHub artifacts
|
||||
named `frontend-test-reports` contain HTML coverage, a JSON summary, and the
|
||||
Chromium/touch failure screenshots/traces; `browser-traces-firefox` and
|
||||
`browser-traces-webkit` hold the other browsers' traces. Open traces with
|
||||
Playwright's `show-trace` tool.
|
||||
aggregate job that passes only when every frontend job succeeds: the three
|
||||
`frontend coverage (shard N/3)` jobs, `frontend coverage (merge)`, and the four
|
||||
`page workflows (firefox|webkit, shard N/2)` jobs. A failed, cancelled or skipped
|
||||
job fails it. Every browser job runs with `PM_TEST_WORKERS: 3` (the public
|
||||
runners have 4 vCPUs). Each coverage shard runs `test:coverage:shard -- N/3`
|
||||
(shard 1 also runs the node units and the viewer suite, shard 3 the vendor pins
|
||||
and tooling tests) and uploads its raw V8 files as `coverage-raw-N` (kept 3
|
||||
days). The merge job downloads them, fails unless all three arrived with raw
|
||||
files and without clashing names, then runs `coverage:merge` (report plus
|
||||
baseline check). `tests/vendor/ci-workflow.test.mjs` checks this shape. GitHub
|
||||
artifacts named `frontend-test-reports` contain the merged HTML coverage and
|
||||
JSON summary; `frontend-traces-shard-N` hold the Chromium/touch failure
|
||||
screenshots/traces and `browser-traces-<browser>-shard-N` the other browsers'
|
||||
traces. Open traces with Playwright's `show-trace` tool.
|
||||
|
||||
Every `npm ci` first restores npm's download cache (`~/.npm`), keyed on the vendor
|
||||
lockfile and the job's pinned image tag (`tests/vendor/ci-workflow.test.mjs` checks
|
||||
the key names the same tag as the job's `container.image`). Without it, `npm ci` stalled for about 300 s
|
||||
(npm's fetch timeout) in most of the eight concurrent jobs; with a hit it takes
|
||||
0 to 2 s. A lockfile change misses the cache once, so expect one slow run after it.
|
||||
Measured on PR #274 (run 39725176320 at cbc8a61, cache hit): the whole workflow
|
||||
took 8m38s, against about 33 minutes with one worker and no shards. Jobs: coverage
|
||||
shards 7m34s/6m04s/5m22s (tests 6m38s/4m53s/4m04s), merge 54s, firefox shards
|
||||
7m57s/5m21s (tests 7m08s/4m34s), webkit shards 8m16s/5m03s (tests 7m26s/3m58s).
|
||||
Browser jobs keep a 20-minute limit: the slowest job is about 8.5 minutes, a cache
|
||||
miss adds up to 5 to `npm ci`, and runner speed varies (the first sharded run was
|
||||
cancelled at a 12-minute limit).
|
||||
Playwright shards by test count, whole files at a time, so shard 1 carries the
|
||||
slow concept-comparison specs (`*-concept.spec.mjs`, alphabetically first) and
|
||||
runs longer; the remedy, if it starts to matter, is more shards, not reordering.
|
||||
|
||||
The real-host suite covers both Classic and Nodes 2.0 startup, switching both
|
||||
ways, actual database prompt selection, execution-text serialization, workflow
|
||||
@@ -274,7 +332,30 @@ becomes record format `limited` (no raw text, graph `limited`, roles and facts
|
||||
unaffected); other text chunks are skipped unread; byte-identical duplicates are
|
||||
accepted within a bounded read budget, while differing duplicates (and any duplicate
|
||||
role record over that budget) still fail. Other read bounds are unchanged: 8 MiB
|
||||
non-PNG header, 10,000 PNG chunks and a three-second ffprobe. Resolution under
|
||||
non-PNG header, 10,000 PNG chunks and a three-second ffprobe (per probe, at most two
|
||||
probes). Video containers are read through one shared carrier selector
|
||||
(`utils/video_metadata.select_video_carrier`, used by the inspector and by the admin
|
||||
scan): carriers are, in order, the per-key tags that current savers write (`prompt`,
|
||||
`workflow`, `parameters`, `comfyassets_prompt_pairs`, each `original-text`; Matroska
|
||||
upper-cased names fold on our side), then the old VideoHelperSuite `comment` JSON
|
||||
envelope, then `description` (object values are `reserialized-json`; envelope keys are
|
||||
matched exactly, as the legacy readers did, never folded). The first carrier holding a
|
||||
role key wins, otherwise the first holding a workflow, and carriers never merge. Two
|
||||
per-key tags folding to one key are a duplicate record: byte-identical ones are one
|
||||
record, differing ones fail the read. ffprobe is asked for exactly those six tags.
|
||||
Its JSON output escapes tag text (at most doubling it), so the stdout bounds are sized
|
||||
so that only the decoded per-record budgets above can fail or limit a record, and a
|
||||
video reads exactly as the identical PNG: the first probe allows twice the role and
|
||||
workflow budgets plus 64 KiB of envelope (12 MiB + 64 KiB); past that the inspector
|
||||
probes once more without the `workflow` tag within twice the role budget plus 64 KiB
|
||||
(4 MiB + 64 KiB) and the per-key workflow becomes `limited`, while an envelope over
|
||||
that bound still fails the read because it cannot be split. The admin scan reads the
|
||||
same carrier with the default resolver profile, so a per-key video links like the same
|
||||
graph in a PNG; the live monitor processes images only (unchanged, see the follow-up
|
||||
below) (`tests/test_utils_video_metadata.py`, `tests/test_api_admin.py`
|
||||
`TestScanReadsVideos`, `tests/test_api_prompt_inspector.py` `test_video_*`,
|
||||
`tests/test_inspection_response.py` `TestVideoProbeBoundsMatchPngBudgets`; the
|
||||
real-container round trips skip without ffmpeg). Resolution under
|
||||
`INSPECTION_PROFILE` bounds work, not graph size: 2,000 visits per candidate chain,
|
||||
20,000 in total, 64 link/JSON-container levels, 100,000 resolved characters, and a
|
||||
two 20,000-node ceilings: one over the top-level entries of the API prompt, and one
|
||||
@@ -437,10 +518,11 @@ the first nav link and heading button.
|
||||
Recorded shell deviations:
|
||||
- **D4**: a "Theme" select with Dark, Light and System. System follows the ComfyUI
|
||||
palette, then the OS.
|
||||
- **D5**: no Activity button or Activity row until an activity center exists.
|
||||
- **D5**: retired by Activity (#249 Activity-1). The concept's Activity button and
|
||||
Activity row are shown; their differences are D52 to D54.
|
||||
- **D28**: no COLLECTIONS, and no Design review. The bottom nav is
|
||||
System plus the production Overview hub (its own outline home glyph, not the concept's
|
||||
`activity` pulse, and no count).
|
||||
the concept's Activity and System, plus the production Overview hub (its own
|
||||
outline home glyph, not the concept's `activity` pulse, and no count).
|
||||
The connection reads "Your library, on your server" with a neutral muted dot.
|
||||
- **D49** (saved views, #249 D16): the SAVED VIEWS group label has no "PROPOSED"
|
||||
badge, because saved views are persisted (`/prompt_manager/saved-views`). Its rows are
|
||||
@@ -491,6 +573,196 @@ Recorded shell deviations:
|
||||
as deleted. `tests/browser/workspace-saved-views.spec.mjs` covers the row, count,
|
||||
route, star updates, Save view on Favorites, Edit, and the failure paths;
|
||||
`tests/test_image_favorites.py` covers `favorites=1`.
|
||||
- **D52** (Activity, #249 Activity-1): the top-bar `button#activity-open` and the
|
||||
sidebar's Activity row (the concept's `activity` pulse, first in `system-nav`) count
|
||||
this session's *running* jobs. The concept counts
|
||||
every sample job. `span.count#activity-count` is `hidden` at 0 (its text stays "0"),
|
||||
and the row has no `.nav-count` then. The Activity heading has no "Unified activity ·
|
||||
proposed" chip (prototype copy). At `max-width: 700px` the full-width `.top-actions`
|
||||
row may wrap (`flex-wrap: wrap`), so the button, Theme and ComfyUI stay inside a
|
||||
zoomed phone. The shell comparison hides the concept's count at runtime
|
||||
(`#activity-count { display: none }`), so the concept's own CSS lays out its button
|
||||
without it, and maps the row's count to none, the button's count to "0" and the
|
||||
phone row's `flexWrap` to `wrap`.
|
||||
- **D53**: while a job runs, the approved Activity mockup's progress pill sits before
|
||||
the Activity button on every page: `a#activity-pill.activity-pill`, "Scanning · 41% ·
|
||||
8,240 / 19,622" (the verb by job kind, the server's floored percentage, and the file
|
||||
counts once the total is known), a 70px mini bar, `href="#/activity/<id>"` and the same
|
||||
text as its `aria-label`. With no running job it is `hidden`, which the shell
|
||||
comparison maps as a hidden item before the button. At `max-width: 700px` it keeps the
|
||||
verb and percentage only; on coarse pointers it is 40px tall like the concept's
|
||||
buttons. Its rules are in `workspace.css` under D53.
|
||||
- **D54**: each `.activity-item` is a real job of this session (`GET
|
||||
/prompt_manager/jobs`), running first, then finished newest first. The concept's
|
||||
structure and CSS stay: `.panel-heading` > `h3` "✓ <title>" for a completed job and
|
||||
"○ <title>" otherwise, plus `span.chip`, then `p`, and for a running job `<progress
|
||||
max=100>` ("<title> progress") and `.row-actions`. The chip is green "Completed" or
|
||||
blue "Running", "Cancelled", "Failed" or "Ended". The line reads "Started HH:MM · K
|
||||
output folders · closing this view does not cancel it" while running, "Finished HH:MM
|
||||
· N added · M errors", "Cancelled HH:MM at a file boundary · N added · completed work
|
||||
is kept", "Stopped on an error at HH:MM · N added · M errors", or "This job ended when
|
||||
ComfyUI restarted." The running actions are "Cancel job" only (no "Simulate
|
||||
completion"); a failed cancel toasts "The job could not be cancelled. Try again.". With
|
||||
no job the section says "No jobs have run since ComfyUI started. Scans and other long
|
||||
operations appear here." `#/activity/<id>` is that job's own view (D56). When a job
|
||||
ends while its Cancel job has focus, focus moves to the item's title link.
|
||||
|
||||
The Activity store reads the list once, then follows each running job over
|
||||
`/prompt_manager/jobs/<id>/events`. A dropped stream re-reads the list after 1 s,
|
||||
doubling to at most 30 s; a received event resets the wait, and so does a good list
|
||||
read after failed ones or after any read that was not for a dropped stream. A running
|
||||
job the server no longer lists, or one whose server could not be read twice in a row,
|
||||
is shown as ended, so no pill stays; one failed read (a proxy's 502) changes nothing.
|
||||
A finished job the server no longer lists (it keeps 20) is dropped. Numbers are
|
||||
grouped "en-US", as the server's lines are. While the tab is visible the list is read again every 15 s (and whenever
|
||||
Activity opens), so a job started in another tab or on the legacy admin page appears
|
||||
without a reload. Toasts: "Scan started." after this tab's own start; "Scan finished
|
||||
· N added · M errors", "Scan cancelled · N added" or "Scan failed · N added · M
|
||||
errors" when a job this tab saw running ends (plural helper for "error").
|
||||
`tests/js/workspace-activity-store.test.js` covers the store, its backoff and the
|
||||
copy; `tests/browser/workspace-activity.spec.mjs` covers the list against the
|
||||
concept's own items (`renderActivity` and `startJob`), the pill and counts, a phone,
|
||||
reload and a second tab, Cancel job, a restart and an unreachable server, and the
|
||||
toasts, with fake jobs from `tests/browser/workspace_activity_fixture.py`
|
||||
(`/__test__/jobs/*`, including a simulated restart that drops the job streams).
|
||||
- **D55** (Scan for metadata, Activity-1 Task 3): Images → Import keeps Rescan as the
|
||||
quick refresh (D15) and then shows the concept's own "Scan an output folder → Review
|
||||
scan" row as "Scan for metadata" ("Discover and enrich existing files using the
|
||||
configured sources.", button "Review scan"). It opens the concept's `scan-review`,
|
||||
"Review metadata scan": no sample-plan `.demo-note` (prototype copy); Sources and
|
||||
Scope in a `dl.meta-grid` (the concept's maintenance plan uses `dl`; `scan-review`
|
||||
uses a `div`) reading "N output folders" (the discovered sources) and the concept's
|
||||
"New and unresolved files" (the D58 checkbox switches it to "All files"); the
|
||||
concept's sentence; then
|
||||
a backup `.notice` (the concept's System notice box): `h3` "Back up the database first
|
||||
(recommended)", "Downloads a copy of the prompts database before the scan changes any
|
||||
records." and "Back up now", which downloads the existing `/prompt_manager/backup`
|
||||
copy (as the legacy admin page does), reads "Backing up…" while aria-disabled, keeps
|
||||
the dialog open and toasts "Database backup downloaded." or "The database could not be
|
||||
backed up. Try again."; then Cancel / Start scan (primary; the concept's "Simulate
|
||||
scan"). Start scan closes the dialog, starts the session scan job (`POST
|
||||
/prompt_manager/jobs/scan`), toasts "Scan started." and opens `#/activity/<id>`; a
|
||||
failed start toasts "The scan could not start. Try again.". The scan's progress line
|
||||
now groups thousands ("Processing file 8,240 of 19,622"), as the mockup does.
|
||||
`tests/browser/activity-concept.spec.mjs` compares the dialog with the concept's
|
||||
`scan-review` and its box with the concept's System `.notice` (dark and light,
|
||||
Chromium and touch, with a CSS and a DOM mutation check); `images-concept.spec.mjs`
|
||||
maps the Import row; `workspace-images.spec.mjs` covers the dialog, the backup and
|
||||
Cancel.
|
||||
- **D56**: one job's view at `#/activity/<id>`, the approved mockup
|
||||
(`activity-scan.html`, "100% spot on"), which the concept does not have. The page
|
||||
heading names the job (h1, document title "<title> · Prompt Manager"; the breadcrumb
|
||||
stays "Activity"), its line is the D54 item line, and its actions are Download report
|
||||
(disabled until Activity-2) and, while the job runs, "Cancel scan" (`.danger`; "Cancel
|
||||
job" for another kind; a failed request toasts the D54 message). Then `section.surface`
|
||||
"Progress": `.panel-heading` (h2, `span.small` with the server's "Processing file N of
|
||||
M · about X min left", or after the end "N of M files processed", "Cancelled after N
|
||||
of M files", the error line, or the restart line), `<progress max=100>` ("Scan
|
||||
progress") and `dl.stat-row` tiles in order: Added to Library (`.added`, green),
|
||||
Already known, Needs review (`.warn`, warning), No usable metadata, Errors (`.bad`,
|
||||
red). Then `section.surface` "Results": `label.search-field` "Filter by file path…"
|
||||
(`aria-label` "Filter results", case-folded match on the root label and path), and
|
||||
`.outcomes` with `.outcome-list` (`button[aria-pressed]` label + count, in the
|
||||
mockup's order Added, Needs review, Errors, No usable metadata, Already known; the
|
||||
view opens on the first with files) beside `.outcome-files`: one `.file-row` (`.path`
|
||||
root-relative, `.why` the server's detail) per listed file. The detail wording is the
|
||||
builder's (approved with Task 1): "Added to the Library as “title”", "Linked to
|
||||
“title”", "Already linked to “title”", "Needs review: …", "Error: …", where the title
|
||||
is the prompt's Library title (its stored title, else its text's first line), not the
|
||||
mockup's sample "→ “title” · new positive + negative". Each log line is "HH:MM:SS
|
||||
<prefix> <root label>/<path> · <detail>" rather than the mockup's "[HH:MM:SS] + path →
|
||||
prompt #4812 (new)". The header's actions sit in the page heading's
|
||||
`.heading-actions`, where the mockup has a `.panel-heading > .row-actions`. An
|
||||
empty outcome shows "No files in this group."
|
||||
or "No listed file matches this filter.", and the note "Showing N of M" when fewer
|
||||
than the counter are shown (the server lists 200 per outcome), with "each row opens
|
||||
the image in the viewer" for Added and Already known. Then `details.term`, collapsed:
|
||||
"● Scan log — raw detail, click to show" and `pre[tabindex=0]` with the raw lines.
|
||||
The summary follows the store and carries `listed` (rows per outcome) and
|
||||
`log_length`; the rows and log are read from `GET /prompt_manager/jobs/<id>` once on
|
||||
opening, then at most once a second only while what it lists changes (or, with the
|
||||
log open, its length), so a job whose lists are capped and whose log is closed is not
|
||||
re-read; opening the log reads its new lines. When the job ends while Cancel scan has
|
||||
focus, focus moves to the job's heading. An unknown id, or a job the store's list no
|
||||
longer has, shows "This job is not in this session’s Activity. Jobs are kept until
|
||||
ComfyUI restarts." and "All activity"; a job lost with a restart keeps what was read.
|
||||
At `max-width: 700px` the tiles take two columns and the outcome list stacks above
|
||||
the files. Each Activity item's title is now a link to its view, in the title's
|
||||
colour. The rules are in `workspace.css` under D56. `activity-concept.spec.mjs`
|
||||
restates the mockup's structure and CSS values and checks them with mutations;
|
||||
`workspace-activity.spec.mjs` covers the counters, live updates, viewer rows, the
|
||||
filter, the 200-row cap, Cancel scan, the busy start and an unknown or lost job.
|
||||
- **D57**: rows under Added and Already known are `button.file-row`s that open the
|
||||
existing media viewer on that one file, by the identity Images opens it by
|
||||
(`root_index:path`). Each scan row records its root's listing index (`root_index`,
|
||||
the place in `_get_all_output_dirs`) beside its label. The row's source is that index
|
||||
in `/prompt_manager/gallery/subfolders?mode=workspace` and must still carry the row's
|
||||
label, so a reordered or replaced source never opens another root's file; the
|
||||
workspace listing's new `path` filter (one exact relative path, case-folded, at most
|
||||
1,024 characters, no traversal) returns that file. A row without an index, a changed
|
||||
source or a file the listing no longer has toasts "This file is no longer in Images.
|
||||
Rescan from Import and try again."; other rows are plain text, with no new file
|
||||
route. A start refused because another record-changing job runs toasts "Another job
|
||||
is running: <title>." with the title linking that job's view (the shared toast now
|
||||
accepts link parts). A toast with a link stays 20 s instead of 6.5 s, and any toast
|
||||
stays while the pointer or focus is on it.
|
||||
- **D58** (scan scope, Activity-2 Task 4): the concept's `scan-review` states a fixed
|
||||
scope, "New and unresolved files". Production keeps that value as the default and adds
|
||||
the concept's own System toggle after the `dl.meta-grid`: a `.setting-row` with the
|
||||
label "All files", the hint "Also read the files already linked to their prompt
|
||||
again." (the switch's `aria-describedby`) and the Toolkit switch (`label.ca-switch`, a
|
||||
checkbox with `role="switch"`, as the concept's `hydrateSwitches` renders it),
|
||||
unchecked. Checking it switches the Scope value to "All files"; Start scan sends
|
||||
`POST /prompt_manager/jobs/scan {scope: "new" | "all"}` (no body, an empty body or no
|
||||
`scope` is "new"; another value is a 400).
|
||||
- In "new" the scan skips, before reading any metadata, the files the Library already
|
||||
links with a resolved link and that are unchanged on disk: exactly one
|
||||
`generated_images` row for the file, exactly one distinct pair link on it
|
||||
(`image_prompt_pairs`, written only by resolved ingestion), and a stored
|
||||
`file_size` equal to the file's size in this walk. Both paths go through
|
||||
`normalize_image_path` (absolute, normalized, `os.path.normcase`); a row without
|
||||
`file_path` matches by its `image_path`.
|
||||
- Read again: files linked without a pair (the monitor's fallback and FIFO links, the
|
||||
legacy positive-only import), linked to more than one prompt or pair, not linked, a
|
||||
file whose size changed (one replaced at the same path, e.g. a reused ComfyUI
|
||||
counter name, is read and its conflict lands under Needs review), and a row with no
|
||||
stored size (unknown is not known). The scan now stores each file's size when it
|
||||
ingests it, so a file read once is skipped by the next "new" pass; the first pass
|
||||
after the upgrade fills the missing sizes, which reports those files as enriched
|
||||
(Added) instead of unchanged.
|
||||
- The known set is one streamed query that keeps only the scanned paths. If it cannot
|
||||
be read, the scan logs "Known files could not be read · reading every file" and
|
||||
reads every file.
|
||||
- A skipped file counts as processed and unchanged in the legacy counts (not as
|
||||
found) and under Already known, with the detail "Skipped · already linked" and the
|
||||
log line "= <root>/<path> · Skipped · already linked"; the log's first line names
|
||||
the scope ("Scan started · N output folders · new and unresolved files" or "· all
|
||||
files"). A legacy-page scan that attaches to a running "new" job therefore shows
|
||||
the skipped files in its `unchanged` count.
|
||||
- "All files" reads every file, as the scan did in Activity-1, and the legacy
|
||||
`/prompt_manager/scan` route always does when it starts the scan (its SSE payload
|
||||
keys are unchanged).
|
||||
- `activity-concept.spec.mjs` compares the switch with the concept's System → Sources
|
||||
toggle ("Watch for new outputs"), without its place-dependent top margin;
|
||||
`workspace-activity.spec.mjs` covers the Scope value, the description and the
|
||||
request both ways; `tests/test_scan_scope.py` covers the skip, the size rule, the
|
||||
replaced file, the re-read of unresolved files, "all", the legacy route, the
|
||||
fallback, the empty body, the 400 and Windows case folding.
|
||||
- **D59** (job report and log, Activity-2 Task 5): the job view's "Download report" is
|
||||
enabled. It fetches `GET /prompt_manager/jobs/<id>/report` through the api-client URL
|
||||
(so a proxy mount works) and saves a JSON attachment named
|
||||
`prompt-manager-<kind>-<YYYYMMDD-HHMM>.json` (server local time), then toasts "Job
|
||||
report downloaded." (or "The report could not be downloaded. Try again."). The report
|
||||
holds the summary, every counter, each outcome group's exact `count`, its listed
|
||||
`rows` (at most 200) and a `truncated` flag, the scope and the log; rows and log lines
|
||||
carry root-relative paths only, never an absolute path (an unknown id is a 404).
|
||||
A job lost with a restart has no report on the server, so its Download report is
|
||||
disabled; a view left while the report downloads still saves it but shows no toast.
|
||||
The scan log opens at its newest line every time it is opened and follows new lines
|
||||
only while the reader is at the bottom; a reader who scrolled up keeps the position
|
||||
while it stays open (checked on Chromium, Firefox and WebKit).
|
||||
`test_report_is_attachment_without_absolute_paths` and the browser tests `download
|
||||
report saves the job JSON` and `log keeps the reader's scroll position` cover it.
|
||||
- **D29**: no prototype copy. There is no "Interactive concept" badge and no
|
||||
"What changes & why" button. The footer reads "ComfyAssets · Local tools. Your
|
||||
library.", and the title has no " concept" suffix. Save view's dialog action reads
|
||||
@@ -577,7 +849,8 @@ Recorded deviations on this page (spec §8):
|
||||
footer, masonry tops may differ by sub-pixel snapping (at most one layout unit per
|
||||
card above in the column), and a List row's info takes its natural height, centred
|
||||
beside the 160×95 art.
|
||||
- **D5**, **D29**: no Activity button and no prototype copy (shell).
|
||||
- **D29**: no prototype copy (shell). The Activity button and pill are the shell's
|
||||
(D52, D53).
|
||||
- **D7**: the star is a real persisted favorite with "Added to favorites." /
|
||||
"Removed from favorites." and a rollback toast on failure.
|
||||
- **D8**: every Images card has the concept's select checkbox before the star; the
|
||||
@@ -1094,6 +1367,61 @@ concept fixture stars the concept's favorites (each other prompt's first image).
|
||||
restore. Old backups without the table restore, and backup plus restore keeps
|
||||
WAL-resident favorites and closes every handle (`tests/test_database_backup.py`).
|
||||
|
||||
Image annotations schema (#249, `database/annotation_schema.py`): the group
|
||||
`media_annotation_metadata` (one row: schema version, `image_index_revision`),
|
||||
`media_annotation_files` (one private locator per physical file,
|
||||
`normcase(join(realpath(root), relative))`, server-side only),
|
||||
`media_annotation_bytes` (one record per observed whole-file SHA-256 with the
|
||||
`size`/`mtime_ns` verified when those bytes were last seen and the set's
|
||||
`tag_revision`) and `media_image_tags` (the ordered tag set of one byte record,
|
||||
joined into the global `tags` registry). It installs like favorites: optional in
|
||||
old backups, required once current (`annotation_schema_error`, the single
|
||||
`require_library_state` flag), validated read-only before and after a SAVEPOINT
|
||||
install and on restore, never backfilled and never stat-ing media
|
||||
(`tests/test_annotation_schema.py`). Writers (`database/annotations.py`) always
|
||||
supply the UUID ids, run in `_pair_transaction` fenced by the storage epoch, and
|
||||
replace or union one byte record's set (`tag-conflict` / `tag-limit` with the
|
||||
current set, identical retries `replayed`); vocabulary rename, merge and delete
|
||||
carry `media_image_tags` through the one `_move_tag_links` loop, and tag rows,
|
||||
impact and command receipts report `image_count` / `images` /
|
||||
`images_changed` as files (`tests/test_annotations.py`,
|
||||
`tests/test_pair_tag_maintenance.py`, `tests/test_vocabulary.py`). Cheap reads
|
||||
(spec amendment S1) use a byte record only while the file's current
|
||||
`(size, mtime_ns)` equals its verified fingerprint; an in-place rewrite that
|
||||
preserves both keeps the old set matching until the file is next observed. The
|
||||
browser fixtures clear `media_image_tags`, `media_annotation_bytes` and
|
||||
`media_annotation_files` on reset and never the metadata row.
|
||||
|
||||
Capture store (#249, `utils/media_capture.py`, `tests/test_media_capture.py`):
|
||||
one explicit action captures one rooted media file into an owner-only
|
||||
`prompt-manager-captures-*` temp directory, streaming it in 1 MiB chunks while
|
||||
hashing. Identity is compared like with like: the opened descriptor against
|
||||
the path only on the API-stable facts (`size`, `mtime_ns`); the full identity
|
||||
(`size`, `mtime_ns`, `ctime_ns`, `dev`/`ino` where meaningful) between the
|
||||
`fstat` before and after the copy and between the path stats before and after
|
||||
(native Windows CI with Python 3.12/3.13 showed `stat` and `fstat` disagreeing
|
||||
on `st_ctime_ns`: change time versus creation time; 3.10/3.11 agree). Any
|
||||
disagreement is `CaptureChanged` and the copy is removed. Error messages carry
|
||||
`strerror` only, never a path. Aliases (a symlinked
|
||||
file or directory, even inside the root), escapes, non-media names and
|
||||
directories are `CaptureDenied`; the locator is
|
||||
`normcase(join(realpath(root), relative))`, so a symlinked root shares the
|
||||
listing's identity. Fixed limits: 256 MiB per file (stat and stream), 8
|
||||
completed captures, 512 MiB reserved, 2 capture/verify workers, 900 s idle,
|
||||
1800 s hard; admission is nonblocking (`CaptureCapacity`) and lifetimes are
|
||||
tested with an injected clock. An acquired capture outlives expiry and release
|
||||
until its last use ends. Header-only dimensions (4096 px per side,
|
||||
16,777,216 px) and video mark Generate unsupported; `verify` re-hashes the
|
||||
current file without a copy. Every handle closes before unlink (a patched
|
||||
`os.unlink` asserts it, plus `assert_closed`), and files are unlinked outside
|
||||
the store lock. Known limit: on Windows before Python 3.13 `fstat` reports the
|
||||
creation time, so a same-size rewrite that restores `mtime_ns` during the copy
|
||||
window completes there with a digest of the copied bytes; POSIX and Windows on
|
||||
3.13+ detect it through the descriptor's change time (native CI, 3.13 only).
|
||||
Vocabulary rename, merge and delete bump the `tag_revision` of every image set
|
||||
they change (and `image_index_revision`), so a Tag dialog opened before the
|
||||
change gets `tag-conflict` on Save (`tests/test_pair_tag_maintenance.py`).
|
||||
|
||||
Budgets: `test_filter_work_is_bounded` always runs. It asserts that a plain first
|
||||
page makes 1 link SELECT and a plain cursor page none, that every filter shape
|
||||
makes exactly 1, the covering-index plan, and that the per-file values are built
|
||||
@@ -1364,7 +1692,7 @@ bytes at the same path without changing the immutable receipt.
|
||||
The focused acceptance command retains all four projects:
|
||||
|
||||
```sh
|
||||
PM_TEST_PORT=8766 npm run --prefix tools/vendor test:workflows -- \
|
||||
npm run --prefix tools/vendor test:workflows -- \
|
||||
workspace-history.spec.mjs workspace-visual.spec.mjs \
|
||||
--project chromium --project touch --project firefox --project webkit
|
||||
npm run --prefix tools/vendor test:executions
|
||||
@@ -1406,6 +1734,12 @@ responsive production Viewer keeps its toolbar in the media panel and prompt det
|
||||
mobile toolbar collision. These are explicit remaining visual differences, not
|
||||
a whole-application or full-gallery/Viewer pixel-parity claim.
|
||||
|
||||
Follow-up: monitor video support. The live image monitor (`utils/image_monitor.py`,
|
||||
`GalleryConfig.SUPPORTED_EXTENSIONS`) processes images only; videos written by
|
||||
ComfyUI savers reach the Library through the admin scan, which reads the shared video
|
||||
carrier. Accepting `VIDEO_EXTENSIONS` in the monitor and routing them through
|
||||
`read_video_metadata` is a separate, issue-linked change.
|
||||
|
||||
Preserve raw logs, failed traces, paired PNGs, geometry, runtime versions and
|
||||
source hashes before later managed commands clear their output directories.
|
||||
Frontend baseline updates are additive through `coverage:update` only after
|
||||
@@ -1457,7 +1791,7 @@ acceptance; a baseline is added only after the maintainer accepts them.
|
||||
Focused commands:
|
||||
|
||||
```sh
|
||||
PM_TEST_PORT=8766 npm run --prefix tools/vendor test:workflows -- \
|
||||
npm run --prefix tools/vendor test:workflows -- \
|
||||
workspace-library-editor.spec.mjs library-editor-visual.spec.mjs \
|
||||
--project chromium --project touch --project firefox --project webkit
|
||||
npm run --prefix tools/vendor test:comfyui # library-sender.spec.mjs host regression
|
||||
@@ -1557,7 +1891,7 @@ mobile, the concept geometry/token/control comparison and its mutation check),
|
||||
|
||||
```sh
|
||||
node --test tests/js/workspace-vocabulary-state.test.js tests/js/workspace-state.test.js
|
||||
PM_TEST_PORT=8766 npm run --prefix tools/vendor test:workflows -- \
|
||||
npm run --prefix tools/vendor test:workflows -- \
|
||||
workspace-vocabulary.spec.mjs vocabulary-visual.spec.mjs focused-node.spec.mjs \
|
||||
--project chromium --project touch --project firefox --project webkit
|
||||
```
|
||||
|
||||
@@ -0,0 +1,290 @@
|
||||
"""Image annotation schema group (#249): file locators, byte records, image tags.
|
||||
|
||||
The group installs like ``image_favorites`` and ``saved_views``: optional in old
|
||||
backups, required once a database is current. It owns
|
||||
|
||||
- ``media_annotation_metadata``: one row, the group's schema version and the
|
||||
image index revision;
|
||||
- ``media_annotation_files``: one private canonical locator per physical file
|
||||
(``normcase(join(realpath(root), relative))``, server-side only);
|
||||
- ``media_annotation_bytes``: one record per observed whole-file SHA-256 of a
|
||||
file, with the size and ``mtime_ns`` verified when those bytes were last seen
|
||||
and the tag set's revision;
|
||||
- ``media_image_tags``: the ordered image tag set of one byte record, joined
|
||||
into the global ``tags`` registry so vocabulary maintenance carries image
|
||||
links like prompt and pair links.
|
||||
|
||||
Installation never backfills anything from prompt or pair tags and never
|
||||
touches the filesystem. Validation reads only: the catalog objects are compared
|
||||
after whitespace compaction, columns, foreign keys and indexes through the
|
||||
PRAGMAs, and persisted values in single bounded SQL passes.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
from typing import Optional
|
||||
|
||||
from .image_favorites import MAX_KEY_LENGTH
|
||||
|
||||
SCHEMA_VERSION = 1
|
||||
METADATA_TABLE = "media_annotation_metadata"
|
||||
FILES_TABLE = "media_annotation_files"
|
||||
BYTES_TABLE = "media_annotation_bytes"
|
||||
IMAGE_TAGS_TABLE = "media_image_tags"
|
||||
TABLES = (METADATA_TABLE, FILES_TABLE, BYTES_TABLE, IMAGE_TAGS_TABLE)
|
||||
MAX_LOCATOR_LENGTH = MAX_KEY_LENGTH
|
||||
_TIMESTAMP_BYTES = 64
|
||||
|
||||
_TABLE_DDL = {
|
||||
METADATA_TABLE: (
|
||||
f"CREATE TABLE {METADATA_TABLE} (singleton_id INTEGER PRIMARY KEY "
|
||||
"CHECK(singleton_id=1), schema_version INTEGER NOT NULL "
|
||||
f"CHECK(schema_version={SCHEMA_VERSION}), "
|
||||
"image_index_revision INTEGER NOT NULL DEFAULT 0 "
|
||||
"CHECK(image_index_revision>=0))"
|
||||
),
|
||||
FILES_TABLE: (
|
||||
f"CREATE TABLE {FILES_TABLE} (id TEXT PRIMARY KEY CHECK(length(id)=36), "
|
||||
"locator TEXT NOT NULL UNIQUE, created_at TEXT NOT NULL)"
|
||||
),
|
||||
BYTES_TABLE: (
|
||||
f"CREATE TABLE {BYTES_TABLE} (id TEXT PRIMARY KEY CHECK(length(id)=36), "
|
||||
f"file_id TEXT NOT NULL REFERENCES {FILES_TABLE}(id) ON DELETE CASCADE, "
|
||||
"sha256 TEXT NOT NULL CHECK(length(sha256)=64 AND sha256 NOT GLOB "
|
||||
"'*[^0-9a-f]*'), size INTEGER NOT NULL CHECK(size>=0), "
|
||||
"mtime_ns INTEGER NOT NULL, verified_at TEXT NOT NULL, "
|
||||
"first_seen_at TEXT NOT NULL, tag_revision INTEGER NOT NULL DEFAULT 0 "
|
||||
"CHECK(tag_revision>=0), UNIQUE(file_id, sha256))"
|
||||
),
|
||||
IMAGE_TAGS_TABLE: (
|
||||
f"CREATE TABLE {IMAGE_TAGS_TABLE} (byte_id TEXT NOT NULL REFERENCES "
|
||||
f"{BYTES_TABLE}(id) ON DELETE CASCADE, tag_id INTEGER NOT NULL "
|
||||
"REFERENCES tags(id) ON DELETE CASCADE, position INTEGER NOT NULL "
|
||||
"CHECK(position>=0), PRIMARY KEY(byte_id, tag_id))"
|
||||
),
|
||||
}
|
||||
_INDEX_DDL = {
|
||||
"idx_media_annotation_bytes_fingerprint": (
|
||||
"CREATE INDEX idx_media_annotation_bytes_fingerprint ON "
|
||||
f"{BYTES_TABLE}(file_id, size, mtime_ns)"
|
||||
),
|
||||
"idx_media_image_tags_tag": (
|
||||
f"CREATE INDEX idx_media_image_tags_tag ON {IMAGE_TAGS_TABLE}(tag_id)"
|
||||
),
|
||||
"idx_media_image_tags_position": (
|
||||
"CREATE UNIQUE INDEX idx_media_image_tags_position ON "
|
||||
f"{IMAGE_TAGS_TABLE}(byte_id, position)"
|
||||
),
|
||||
}
|
||||
# SQLite names the indexes behind TEXT primary keys and UNIQUE constraints
|
||||
# ``sqlite_autoindex_<table>_<n>``; their catalog rows carry no SQL.
|
||||
_AUTOINDEXES = {FILES_TABLE: 2, BYTES_TABLE: 2, IMAGE_TAGS_TABLE: 1}
|
||||
# (name, declared type, not null, default, primary key position)
|
||||
_COLUMNS = {
|
||||
METADATA_TABLE: (
|
||||
("singleton_id", "INTEGER", 0, None, 1),
|
||||
("schema_version", "INTEGER", 1, None, 0),
|
||||
("image_index_revision", "INTEGER", 1, "0", 0),
|
||||
),
|
||||
FILES_TABLE: (
|
||||
("id", "TEXT", 0, None, 1),
|
||||
("locator", "TEXT", 1, None, 0),
|
||||
("created_at", "TEXT", 1, None, 0),
|
||||
),
|
||||
BYTES_TABLE: (
|
||||
("id", "TEXT", 0, None, 1),
|
||||
("file_id", "TEXT", 1, None, 0),
|
||||
("sha256", "TEXT", 1, None, 0),
|
||||
("size", "INTEGER", 1, None, 0),
|
||||
("mtime_ns", "INTEGER", 1, None, 0),
|
||||
("verified_at", "TEXT", 1, None, 0),
|
||||
("first_seen_at", "TEXT", 1, None, 0),
|
||||
("tag_revision", "INTEGER", 1, "0", 0),
|
||||
),
|
||||
IMAGE_TAGS_TABLE: (
|
||||
("byte_id", "TEXT", 1, None, 1),
|
||||
("tag_id", "INTEGER", 1, None, 2),
|
||||
("position", "INTEGER", 1, None, 0),
|
||||
),
|
||||
}
|
||||
# (column, parent table, parent column, on delete)
|
||||
_FOREIGN_KEYS = {
|
||||
METADATA_TABLE: set(),
|
||||
FILES_TABLE: set(),
|
||||
BYTES_TABLE: {("file_id", FILES_TABLE, "id", "CASCADE")},
|
||||
IMAGE_TAGS_TABLE: {
|
||||
("byte_id", BYTES_TABLE, "id", "CASCADE"),
|
||||
("tag_id", "tags", "id", "CASCADE"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _compact(sql: str) -> str:
|
||||
return re.sub(r"\s+", "", sql).lower()
|
||||
|
||||
|
||||
def _uuid(column: str) -> str:
|
||||
"""SQL: *column* is a lowercase hyphenated 36-character UUID string."""
|
||||
return (
|
||||
f"typeof({column}) = 'text' AND length({column}) = 36 "
|
||||
f"AND substr({column}, 9, 1) = '-' AND substr({column}, 14, 1) = '-' "
|
||||
f"AND substr({column}, 19, 1) = '-' AND substr({column}, 24, 1) = '-' "
|
||||
f"AND length(replace({column}, '-', '')) = 32 "
|
||||
f"AND replace({column}, '-', '') NOT GLOB '*[^0-9a-f]*'"
|
||||
)
|
||||
|
||||
|
||||
def _timestamp(column: str) -> str:
|
||||
return (
|
||||
f"typeof({column}) = 'text' "
|
||||
f"AND length(CAST({column} AS BLOB)) <= {_TIMESTAMP_BYTES}"
|
||||
)
|
||||
|
||||
|
||||
def _non_negative(column: str) -> str:
|
||||
return f"typeof({column}) = 'integer' AND {column} >= 0"
|
||||
|
||||
|
||||
# Every row of a table must satisfy its predicate; one pass each, LIMIT 1.
|
||||
_ROW_PREDICATES = {
|
||||
FILES_TABLE: " AND ".join(
|
||||
(
|
||||
_uuid("id"),
|
||||
"typeof(locator) = 'text'",
|
||||
# UTF-8 is at most 4 bytes per character; the decoded length of
|
||||
# the few candidates above the byte bound is checked in Python.
|
||||
f"length(CAST(locator AS BLOB)) <= {MAX_LOCATOR_LENGTH * 4}",
|
||||
_timestamp("created_at"),
|
||||
)
|
||||
),
|
||||
BYTES_TABLE: " AND ".join(
|
||||
(
|
||||
_uuid("id"),
|
||||
_uuid("file_id"),
|
||||
"typeof(sha256) = 'text' AND length(sha256) = 64 "
|
||||
"AND sha256 NOT GLOB '*[^0-9a-f]*'",
|
||||
_non_negative("size"),
|
||||
"typeof(mtime_ns) = 'integer'",
|
||||
_timestamp("verified_at"),
|
||||
_timestamp("first_seen_at"),
|
||||
_non_negative("tag_revision"),
|
||||
)
|
||||
),
|
||||
IMAGE_TAGS_TABLE: " AND ".join(
|
||||
(_uuid("byte_id"), "typeof(tag_id) = 'integer'", _non_negative("position"))
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _expected_catalog() -> set:
|
||||
objects = {("table", name, _compact(sql)) for name, sql in _TABLE_DDL.items()}
|
||||
objects |= {("index", name, _compact(sql)) for name, sql in _INDEX_DDL.items()}
|
||||
objects |= {
|
||||
("index", f"sqlite_autoindex_{table}_{n}", "")
|
||||
for table, count in _AUTOINDEXES.items()
|
||||
for n in range(1, count + 1)
|
||||
}
|
||||
return objects
|
||||
|
||||
|
||||
def _catalog(conn: sqlite3.Connection) -> list:
|
||||
names = TABLES + tuple(_INDEX_DDL)
|
||||
marks = ",".join("?" * len(names))
|
||||
table_marks = ",".join("?" * len(TABLES))
|
||||
return conn.execute( # only "?" placeholders are interpolated
|
||||
"SELECT type,name,sql FROM sqlite_master "
|
||||
f"WHERE name IN ({marks}) OR tbl_name IN ({table_marks})", # nosec B608
|
||||
names + TABLES,
|
||||
).fetchall()
|
||||
|
||||
|
||||
def _structure_error(conn: sqlite3.Connection) -> Optional[str]:
|
||||
for table, expected in _COLUMNS.items():
|
||||
columns = tuple(
|
||||
(row[1], row[2].upper(), row[3], row[4], row[5])
|
||||
for row in conn.execute(f"PRAGMA table_info({table})")
|
||||
)
|
||||
if columns != expected:
|
||||
return f"Image annotations table {table} has invalid columns"
|
||||
foreign_keys = {
|
||||
(row[3], row[2], row[4], row[6])
|
||||
for row in conn.execute(f"PRAGMA foreign_key_list({table})")
|
||||
}
|
||||
if foreign_keys != _FOREIGN_KEYS[table]:
|
||||
return f"Image annotations table {table} has invalid foreign keys"
|
||||
if conn.execute(f"PRAGMA foreign_key_check({table})").fetchone():
|
||||
return f"Image annotations table {table} contains foreign key violations"
|
||||
return None
|
||||
|
||||
|
||||
def _value_error(conn: sqlite3.Connection) -> Optional[str]:
|
||||
# Only this module's constants are interpolated into the SQL below.
|
||||
rows = conn.execute(
|
||||
"SELECT COUNT(*), MIN(typeof(singleton_id) = 'integer' " # nosec B608
|
||||
"AND singleton_id = 1 AND typeof(schema_version) = 'integer' "
|
||||
f"AND schema_version = {SCHEMA_VERSION} "
|
||||
f"AND {_non_negative('image_index_revision')}) FROM {METADATA_TABLE}"
|
||||
).fetchone()
|
||||
if rows[0] != 1 or rows[1] != 1:
|
||||
return "Image annotations metadata is invalid"
|
||||
for table, predicate in _ROW_PREDICATES.items():
|
||||
if conn.execute(
|
||||
f"SELECT 1 FROM {table} WHERE NOT ({predicate}) LIMIT 1" # nosec B608
|
||||
).fetchone():
|
||||
return f"Image annotations table {table} contains invalid values"
|
||||
# length() stops at an embedded NUL, so the SQL pass bounds bytes and the
|
||||
# few candidates over the character bound are measured decoded here.
|
||||
for (raw,) in conn.execute(
|
||||
f"SELECT CAST(locator AS BLOB) FROM {FILES_TABLE} " # nosec B608
|
||||
f"WHERE length(CAST(locator AS BLOB)) > {MAX_LOCATOR_LENGTH}"
|
||||
):
|
||||
if len(bytes(raw).decode("utf-8", "replace")) > MAX_LOCATOR_LENGTH:
|
||||
return f"Image annotations table {FILES_TABLE} contains invalid values"
|
||||
return None
|
||||
|
||||
|
||||
def annotation_schema_error(
|
||||
conn: sqlite3.Connection, *, allow_missing: bool = True
|
||||
) -> Optional[str]:
|
||||
"""Reject partial or unsupported annotation schemas and invalid rows.
|
||||
|
||||
Reads only: nothing is ever repaired. A database without any of the
|
||||
group's objects is acceptable for backups that predate it unless
|
||||
``allow_missing`` is false.
|
||||
"""
|
||||
objects = _catalog(conn)
|
||||
if not objects:
|
||||
return None if allow_missing else "Missing image annotation schema"
|
||||
present = {name for kind, name, _sql in objects if kind == "table"}
|
||||
if present != set(TABLES):
|
||||
return "Incomplete image annotation schema"
|
||||
actual = {(kind, name, _compact(sql or "")) for kind, name, sql in objects}
|
||||
if actual != _expected_catalog():
|
||||
return "Image annotations has an invalid schema definition"
|
||||
return _structure_error(conn) or _value_error(conn)
|
||||
|
||||
|
||||
def migrate_annotation_state(conn: sqlite3.Connection) -> None:
|
||||
"""Install the group atomically; validate before and after creating it."""
|
||||
conn.execute("SAVEPOINT media_annotation_schema")
|
||||
try:
|
||||
error = annotation_schema_error(conn)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
for sql in _TABLE_DDL.values():
|
||||
conn.execute(sql.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
|
||||
for sql in _INDEX_DDL.values():
|
||||
conn.execute(sql.replace("INDEX", "INDEX IF NOT EXISTS", 1))
|
||||
conn.execute(
|
||||
f"INSERT OR IGNORE INTO {METADATA_TABLE} "
|
||||
"(singleton_id, schema_version, image_index_revision) VALUES (1, ?, 0)",
|
||||
(SCHEMA_VERSION,),
|
||||
)
|
||||
error = annotation_schema_error(conn, allow_missing=False)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute("RELEASE media_annotation_schema")
|
||||
except BaseException:
|
||||
conn.execute("ROLLBACK TO media_annotation_schema")
|
||||
conn.execute("RELEASE media_annotation_schema")
|
||||
raise
|
||||
@@ -0,0 +1,457 @@
|
||||
"""Per-image tag sets (#249): byte-exact storage in the shared vocabulary.
|
||||
|
||||
A tag set belongs to one byte record (``media_annotation_bytes``): the
|
||||
whole-file SHA-256 of one physical file, owned by the file's private locator
|
||||
(``media_annotation_files``). Callers pass verified facts (digest, size,
|
||||
``mtime_ns``); nothing here reads the filesystem or resolves an owner through
|
||||
prompt links, so tagging one image never changes another image, a prompt or a
|
||||
pair.
|
||||
|
||||
Commands run in one ``_pair_transaction`` fenced by the storage epoch, like the
|
||||
native Library and vocabulary commands. A save is a full replacement guarded
|
||||
by the set's ``tag_revision``; the identical set sent again (a lost-response
|
||||
retry, or the current set at its own revision) answers ``replayed: true`` and
|
||||
writes nothing. ``add_image_tags`` unions into the existing set, keeping the
|
||||
stored spelling. Names enter the global ``tags`` registry exactly as spelled,
|
||||
so vocabulary rename, merge and delete carry image links with prompt and pair
|
||||
links (``PromptDatabase._move_tag_links``).
|
||||
|
||||
Cheap reads (spec amendment S1) treat a byte record as *current* only while
|
||||
the file's present ``(size, mtime_ns)`` equals the record's verified
|
||||
fingerprint; a file without a current record shows no image tags.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
from contextlib import contextmanager
|
||||
from typing import Iterator
|
||||
from uuid import uuid4
|
||||
|
||||
from .annotation_schema import (
|
||||
BYTES_TABLE,
|
||||
FILES_TABLE,
|
||||
IMAGE_TAGS_TABLE,
|
||||
MAX_LOCATOR_LENGTH,
|
||||
METADATA_TABLE,
|
||||
)
|
||||
from .library_commands import _UUID, LibraryCommandError, _validate_epoch
|
||||
from .models import utc_now_iso
|
||||
from .vocabulary import validate_vocabulary_tag_name
|
||||
|
||||
MAX_IMAGE_TAGS = 100
|
||||
_LOOKUP_CHUNK = 500
|
||||
_DIGEST = re.compile(r"[0-9a-f]{64}")
|
||||
_MAX_INT = 2**63 - 1
|
||||
_MIN_INT = -(2**63)
|
||||
# Only this module's table constants are interpolated into the SQL below.
|
||||
_OWNER = f"SELECT id FROM {FILES_TABLE} WHERE locator=?" # nosec B608
|
||||
_INSERT_OWNER = (
|
||||
f"INSERT INTO {FILES_TABLE}(id,locator,created_at) VALUES(?,?,?)" # nosec B608
|
||||
)
|
||||
_BYTE = (
|
||||
f"SELECT id,tag_revision,size,mtime_ns FROM {BYTES_TABLE} " # nosec B608
|
||||
"WHERE file_id=? AND sha256=?"
|
||||
)
|
||||
_INSERT_BYTE = (
|
||||
f"INSERT INTO {BYTES_TABLE}(id,file_id,sha256,size,mtime_ns," # nosec B608
|
||||
"verified_at,first_seen_at,tag_revision) VALUES(?,?,?,?,?,?,?,0)"
|
||||
)
|
||||
_REFRESH_BYTE = (
|
||||
f"UPDATE {BYTES_TABLE} SET size=?,mtime_ns=?,verified_at=? WHERE id=?" # nosec B608
|
||||
)
|
||||
_REFRESH_BY_LOCATOR = (
|
||||
f"UPDATE {BYTES_TABLE} SET size=?,mtime_ns=?,verified_at=? " # nosec B608
|
||||
f"WHERE sha256=? AND file_id=({_OWNER})"
|
||||
)
|
||||
_BYTE_TAGS = (
|
||||
f"SELECT t.name FROM {IMAGE_TAGS_TABLE} mit " # nosec B608
|
||||
"JOIN tags t ON t.id=mit.tag_id WHERE mit.byte_id=? ORDER BY mit.position"
|
||||
)
|
||||
_CLEAR_TAGS = f"DELETE FROM {IMAGE_TAGS_TABLE} WHERE byte_id=?" # nosec B608
|
||||
_INSERT_TAG = (
|
||||
f"INSERT INTO {IMAGE_TAGS_TABLE}(byte_id,tag_id,position) " # nosec B608
|
||||
"VALUES(?,?,?)"
|
||||
)
|
||||
_BUMP_REVISION = (
|
||||
f"UPDATE {BYTES_TABLE} SET tag_revision=tag_revision+1 WHERE id=?" # nosec B608
|
||||
)
|
||||
_REVISION = f"SELECT tag_revision FROM {BYTES_TABLE} WHERE id=?" # nosec B608
|
||||
_BUMP_INDEX = (
|
||||
f"UPDATE {METADATA_TABLE} SET " # nosec B608
|
||||
"image_index_revision=image_index_revision+1 WHERE singleton_id=1"
|
||||
)
|
||||
_READ = (
|
||||
"SELECT b.id,b.tag_revision," # nosec B608
|
||||
f"(SELECT COUNT(*) FROM {BYTES_TABLE} o WHERE o.file_id=f.id AND o.sha256!=?) "
|
||||
f"FROM {FILES_TABLE} f LEFT JOIN {BYTES_TABLE} b "
|
||||
"ON b.file_id=f.id AND b.sha256=? WHERE f.locator=?"
|
||||
)
|
||||
_FINGERPRINTS = (
|
||||
"SELECT f.locator,b.id,b.size,b.mtime_ns,b.verified_at " # nosec B608
|
||||
f"FROM {FILES_TABLE} f JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
||||
"WHERE f.locator IN ({marks})"
|
||||
)
|
||||
_TAGS_OF = (
|
||||
f"SELECT mit.byte_id,t.name FROM {IMAGE_TAGS_TABLE} mit " # nosec B608
|
||||
"JOIN tags t ON t.id=mit.tag_id WHERE mit.byte_id IN ({marks}) "
|
||||
"ORDER BY mit.byte_id,mit.position"
|
||||
)
|
||||
_RECORDS = (
|
||||
"SELECT f.locator,b.id,b.size,b.mtime_ns,t.name " # nosec B608
|
||||
f"FROM {FILES_TABLE} f JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
||||
f"LEFT JOIN {IMAGE_TAGS_TABLE} mit ON mit.byte_id=b.id "
|
||||
"LEFT JOIN tags t ON t.id=mit.tag_id "
|
||||
"ORDER BY f.locator,b.first_seen_at,b.id,mit.position"
|
||||
)
|
||||
|
||||
|
||||
class AnnotationCommandError(LibraryCommandError):
|
||||
"""An image tag command lost a race or hit the set's bound.
|
||||
|
||||
``code`` is ``tag-conflict`` (the set changed since it was read) or
|
||||
``tag-limit`` (the union would exceed ``MAX_IMAGE_TAGS``); ``current`` is
|
||||
the stored set ``{sha256, tags, tag_revision}`` read inside the
|
||||
transaction, so the caller can show it.
|
||||
"""
|
||||
|
||||
def __init__(self, code: str, epoch: str, current: dict | None = None):
|
||||
super().__init__(code, epoch)
|
||||
self.current = current
|
||||
|
||||
|
||||
def normalize_image_tags(names: object) -> list[str]:
|
||||
"""Trim and validate each name, fold case-insensitive repeats, cap at 100.
|
||||
|
||||
The first occurrence keeps its spelling and position. Raises ``ValueError``
|
||||
for anything but a list of valid vocabulary names or for more than
|
||||
``MAX_IMAGE_TAGS`` distinct names.
|
||||
"""
|
||||
if not isinstance(names, list):
|
||||
raise ValueError("tags must be a list of strings")
|
||||
result: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for value in names:
|
||||
name = validate_vocabulary_tag_name(value)
|
||||
key = name.casefold()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
result.append(name)
|
||||
if len(result) > MAX_IMAGE_TAGS:
|
||||
raise ValueError(f"An image can carry at most {MAX_IMAGE_TAGS} tags")
|
||||
return result
|
||||
|
||||
|
||||
def _validate_locator(locator: object) -> None:
|
||||
if not isinstance(locator, str) or not 0 < len(locator) <= MAX_LOCATOR_LENGTH:
|
||||
raise ValueError(
|
||||
f"locator must be a nonblank path of at most {MAX_LOCATOR_LENGTH} chars"
|
||||
)
|
||||
|
||||
|
||||
def _validate_digest(sha256: object) -> None:
|
||||
if not isinstance(sha256, str) or not _DIGEST.fullmatch(sha256):
|
||||
raise ValueError("sha256 must be 64 lowercase hexadecimal characters")
|
||||
|
||||
|
||||
def _validate_int(value: object, name: str, minimum: int) -> None:
|
||||
if type(value) is not int or not minimum <= value <= _MAX_INT:
|
||||
raise ValueError(f"{name} must be an integer not below {minimum}")
|
||||
|
||||
|
||||
def _validate_target(locator: object, sha256: object, size: object, mtime_ns: object):
|
||||
_validate_locator(locator)
|
||||
_validate_digest(sha256)
|
||||
_validate_int(size, "size", 0)
|
||||
_validate_int(mtime_ns, "mtime_ns", _MIN_INT)
|
||||
|
||||
|
||||
def _validate_fingerprint(locator: object, fingerprint: object) -> None:
|
||||
_validate_locator(locator)
|
||||
if (
|
||||
not isinstance(fingerprint, tuple)
|
||||
or len(fingerprint) != 2
|
||||
or any(type(value) is not int for value in fingerprint)
|
||||
):
|
||||
raise ValueError("stats must map locators to (size, mtime_ns) integer pairs")
|
||||
|
||||
|
||||
def _new_id() -> str:
|
||||
"""A canonical UUID; the schema's CHECK lets NULL through, this does not."""
|
||||
value = str(uuid4())
|
||||
if not _UUID.fullmatch(value):
|
||||
raise ValueError("Annotation ids must be canonical UUIDs")
|
||||
return value
|
||||
|
||||
|
||||
def _receipt(epoch: str, sha256: str, tags: list, revision: int, replayed: bool):
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"sha256": sha256,
|
||||
"tags": list(tags),
|
||||
"tag_revision": revision,
|
||||
"replayed": replayed,
|
||||
}
|
||||
|
||||
|
||||
class AnnotationOperations:
|
||||
"""PromptDatabase mixin: image tag sets, fingerprints and cheap reads.
|
||||
|
||||
``_`` helpers take the open ``_pair_transaction`` connection.
|
||||
"""
|
||||
|
||||
@contextmanager
|
||||
def _annotation_read(self) -> Iterator[sqlite3.Connection]:
|
||||
"""One deferred read transaction, so paired SELECTs see one snapshot."""
|
||||
conn = self.model.get_connection()
|
||||
if conn.in_transaction:
|
||||
raise sqlite3.ProgrammingError(
|
||||
"Annotation reads require their own snapshot"
|
||||
)
|
||||
with conn:
|
||||
conn.execute("BEGIN")
|
||||
yield conn
|
||||
|
||||
@staticmethod
|
||||
def _annotation_owner(conn: sqlite3.Connection, locator: str, now: str) -> str:
|
||||
row = conn.execute(_OWNER, (locator,)).fetchone()
|
||||
if row is not None:
|
||||
return row[0]
|
||||
file_id = _new_id()
|
||||
conn.execute(_INSERT_OWNER, (file_id, locator, now))
|
||||
return file_id
|
||||
|
||||
@staticmethod
|
||||
def _annotation_byte(
|
||||
conn: sqlite3.Connection,
|
||||
file_id: str,
|
||||
sha256: str,
|
||||
size: int,
|
||||
mtime_ns: int,
|
||||
now: str,
|
||||
) -> tuple[str, int, bool, bool]:
|
||||
"""The byte record for these bytes, refreshed or created.
|
||||
|
||||
Returns ``(id, tag_revision, created, refingerprinted)``; the last is
|
||||
True when an existing record's verified ``(size, mtime_ns)`` changed,
|
||||
which moves its currency for cheap reads (the caller bumps the index
|
||||
revision when nothing else does).
|
||||
"""
|
||||
row = conn.execute(_BYTE, (file_id, sha256)).fetchone()
|
||||
if row is None:
|
||||
byte_id = _new_id()
|
||||
conn.execute(
|
||||
_INSERT_BYTE, (byte_id, file_id, sha256, size, mtime_ns, now, now)
|
||||
)
|
||||
return byte_id, 0, True, False
|
||||
conn.execute(_REFRESH_BYTE, (size, mtime_ns, now, row[0]))
|
||||
return row[0], row[1], False, (row[2], row[3]) != (size, mtime_ns)
|
||||
|
||||
@staticmethod
|
||||
def _byte_tags(conn: sqlite3.Connection, byte_id: str) -> list[str]:
|
||||
return [row[0] for row in conn.execute(_BYTE_TAGS, (byte_id,))]
|
||||
|
||||
def _write_image_tags(
|
||||
self, conn: sqlite3.Connection, byte_id: str, tags: list[str]
|
||||
) -> int:
|
||||
"""Rewrite the junction in order, bump both revisions; return the new one."""
|
||||
tag_ids = self._ensure_tags(conn, tags)
|
||||
conn.execute(_CLEAR_TAGS, (byte_id,))
|
||||
conn.executemany(
|
||||
_INSERT_TAG,
|
||||
[(byte_id, tag_ids[name], position) for position, name in enumerate(tags)],
|
||||
)
|
||||
conn.execute(_BUMP_REVISION, (byte_id,))
|
||||
conn.execute(_BUMP_INDEX)
|
||||
return conn.execute(_REVISION, (byte_id,)).fetchone()[0]
|
||||
|
||||
def _current_set(self, conn, sha256, byte_id, revision) -> dict:
|
||||
return {
|
||||
"sha256": sha256,
|
||||
"tags": self._byte_tags(conn, byte_id),
|
||||
"tag_revision": revision,
|
||||
}
|
||||
|
||||
def read_image_tags(self, locator: str, sha256: str) -> dict:
|
||||
"""The stored set of these exact bytes, or an empty set at revision 0.
|
||||
|
||||
``prior_count`` is the number of other byte records at the locator
|
||||
(earlier or later bytes of the same file).
|
||||
"""
|
||||
_validate_locator(locator)
|
||||
_validate_digest(sha256)
|
||||
with self._annotation_read() as conn:
|
||||
row = conn.execute(_READ, (sha256, sha256, locator)).fetchone()
|
||||
if row is None:
|
||||
return {"tags": [], "tag_revision": 0, "prior_count": 0}
|
||||
tags = self._byte_tags(conn, row[0]) if row[0] is not None else []
|
||||
return {"tags": tags, "tag_revision": row[1] or 0, "prior_count": row[2]}
|
||||
|
||||
def replace_image_tags(
|
||||
self,
|
||||
*,
|
||||
locator: str,
|
||||
sha256: str,
|
||||
size: int,
|
||||
mtime_ns: int,
|
||||
expected_epoch: str,
|
||||
expected_tag_revision: int,
|
||||
tags: list,
|
||||
) -> dict:
|
||||
"""Replace the set of these exact bytes, guarded by epoch and revision.
|
||||
|
||||
Creates the owner and byte record when the bytes are new (an explicit
|
||||
empty set is a real save). The identical set at the current revision,
|
||||
or one behind it (a lost-response retry), is ``replayed`` and writes
|
||||
nothing; any other revision mismatch is ``tag-conflict`` with the
|
||||
current set and no writes.
|
||||
"""
|
||||
_validate_target(locator, sha256, size, mtime_ns)
|
||||
_validate_epoch(expected_epoch)
|
||||
_validate_int(expected_tag_revision, "expected_tag_revision", 0)
|
||||
tags = normalize_image_tags(tags)
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
now = utc_now_iso()
|
||||
file_id = self._annotation_owner(conn, locator, now)
|
||||
byte_id, revision, created, refingerprinted = self._annotation_byte(
|
||||
conn, file_id, sha256, size, mtime_ns, now
|
||||
)
|
||||
current = self._byte_tags(conn, byte_id)
|
||||
if (
|
||||
not created
|
||||
and current == tags
|
||||
and expected_tag_revision in (revision, revision - 1)
|
||||
):
|
||||
if refingerprinted:
|
||||
conn.execute(_BUMP_INDEX)
|
||||
return _receipt(epoch, sha256, current, revision, True)
|
||||
if expected_tag_revision != revision:
|
||||
raise AnnotationCommandError(
|
||||
"tag-conflict",
|
||||
epoch,
|
||||
{"sha256": sha256, "tags": current, "tag_revision": revision},
|
||||
)
|
||||
revision = self._write_image_tags(conn, byte_id, tags)
|
||||
return _receipt(epoch, sha256, tags, revision, False)
|
||||
|
||||
def add_image_tags(
|
||||
self,
|
||||
*,
|
||||
locator: str,
|
||||
sha256: str,
|
||||
size: int,
|
||||
mtime_ns: int,
|
||||
expected_epoch: str,
|
||||
tags: list,
|
||||
) -> dict:
|
||||
"""Union ``tags`` into the set of these exact bytes, keeping stored spelling.
|
||||
|
||||
``added`` counts the names that were new (case-insensitively). A union
|
||||
over ``MAX_IMAGE_TAGS`` is ``tag-limit`` with the current set and no
|
||||
writes; a union that adds nothing to an existing record is ``replayed``.
|
||||
"""
|
||||
_validate_target(locator, sha256, size, mtime_ns)
|
||||
_validate_epoch(expected_epoch)
|
||||
tags = normalize_image_tags(tags)
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
now = utc_now_iso()
|
||||
file_id = self._annotation_owner(conn, locator, now)
|
||||
byte_id, revision, created, refingerprinted = self._annotation_byte(
|
||||
conn, file_id, sha256, size, mtime_ns, now
|
||||
)
|
||||
current = self._byte_tags(conn, byte_id)
|
||||
present = {name.casefold() for name in current}
|
||||
added = [name for name in tags if name.casefold() not in present]
|
||||
merged = current + added
|
||||
if len(merged) > MAX_IMAGE_TAGS:
|
||||
raise AnnotationCommandError(
|
||||
"tag-limit",
|
||||
epoch,
|
||||
{"sha256": sha256, "tags": current, "tag_revision": revision},
|
||||
)
|
||||
if not created and not added:
|
||||
if refingerprinted:
|
||||
conn.execute(_BUMP_INDEX)
|
||||
return {**_receipt(epoch, sha256, current, revision, True), "added": 0}
|
||||
revision = self._write_image_tags(conn, byte_id, merged)
|
||||
return {
|
||||
**_receipt(epoch, sha256, merged, revision, False),
|
||||
"added": len(added),
|
||||
}
|
||||
|
||||
def refresh_image_fingerprint(
|
||||
self, locator: str, sha256: str, size: int, mtime_ns: int
|
||||
) -> bool:
|
||||
"""Record that these exact bytes were just seen with this stat.
|
||||
|
||||
True when a byte record was refreshed; nothing is created and no tag
|
||||
or tag revision changes. The image index revision moves because the
|
||||
record's currency for cheap reads may have changed.
|
||||
"""
|
||||
_validate_target(locator, sha256, size, mtime_ns)
|
||||
with self.model.get_connection() as conn:
|
||||
cursor = conn.execute(
|
||||
_REFRESH_BY_LOCATOR, (size, mtime_ns, utc_now_iso(), sha256, locator)
|
||||
)
|
||||
if cursor.rowcount > 0:
|
||||
conn.execute(_BUMP_INDEX)
|
||||
return cursor.rowcount > 0
|
||||
|
||||
def current_image_tags(self, stats: dict) -> dict:
|
||||
"""Image tags of the files whose current ``(size, mtime_ns)`` is verified.
|
||||
|
||||
``stats`` maps locators to their present fingerprint. A locator is
|
||||
answered (possibly with an empty set) only when a byte record carries
|
||||
exactly that verified fingerprint; the most recently verified record
|
||||
wins when several do. Lookups go in chunks of 500 locators.
|
||||
"""
|
||||
if not isinstance(stats, dict):
|
||||
raise ValueError(
|
||||
"stats must map locators to (size, mtime_ns) integer pairs"
|
||||
)
|
||||
for locator, fingerprint in stats.items():
|
||||
_validate_fingerprint(locator, fingerprint)
|
||||
locators = list(stats)
|
||||
chosen: dict[str, tuple[tuple[str, str], str]] = {}
|
||||
result: dict[str, list[str]] = {}
|
||||
with self._annotation_read() as conn:
|
||||
for start in range(0, len(locators), _LOOKUP_CHUNK):
|
||||
chunk = locators[start : start + _LOOKUP_CHUNK]
|
||||
marks = ",".join("?" * len(chunk))
|
||||
for locator, byte_id, size, mtime_ns, verified_at in conn.execute(
|
||||
_FINGERPRINTS.format(marks=marks), chunk
|
||||
):
|
||||
if (size, mtime_ns) != stats[locator]:
|
||||
continue
|
||||
key = (verified_at, byte_id)
|
||||
if locator not in chosen or key > chosen[locator][0]:
|
||||
chosen[locator] = (key, byte_id)
|
||||
owners = {byte_id: locator for locator, (_key, byte_id) in chosen.items()}
|
||||
result = {locator: [] for locator in chosen}
|
||||
byte_ids = list(owners)
|
||||
for start in range(0, len(byte_ids), _LOOKUP_CHUNK):
|
||||
chunk = byte_ids[start : start + _LOOKUP_CHUNK]
|
||||
marks = ",".join("?" * len(chunk))
|
||||
for byte_id, name in conn.execute(_TAGS_OF.format(marks=marks), chunk):
|
||||
result[owners[byte_id]].append(name)
|
||||
return result
|
||||
|
||||
def iter_image_tag_records(self) -> Iterator[tuple[str, int, int, list[str]]]:
|
||||
"""Stream every byte record as ``(locator, size, mtime_ns, tags)``.
|
||||
|
||||
One SELECT ordered by locator, then by the bytes' first sighting, with
|
||||
each set's names in position order.
|
||||
"""
|
||||
conn = self.model.get_connection()
|
||||
current: tuple | None = None
|
||||
for locator, byte_id, size, mtime_ns, name in conn.execute(_RECORDS):
|
||||
if current is None or current[0] != byte_id:
|
||||
if current is not None:
|
||||
yield current[1]
|
||||
current = (byte_id, (locator, size, mtime_ns, []))
|
||||
if name is not None:
|
||||
current[1][3].append(name)
|
||||
if current is not None:
|
||||
yield current[1]
|
||||
+10
-3
@@ -17,6 +17,7 @@ from typing import Optional, Set, Tuple
|
||||
|
||||
from .library_schema import library_schema_error, migrate_library_state
|
||||
from .image_favorites import image_favorites_schema_error, migrate_image_favorites
|
||||
from .annotation_schema import annotation_schema_error, migrate_annotation_state
|
||||
from .saved_views import migrate_saved_views, saved_views_schema_error
|
||||
from .execution_history_schema import (
|
||||
history_schema_error,
|
||||
@@ -302,6 +303,7 @@ class PromptModel:
|
||||
migrate_library_state(conn)
|
||||
migrate_image_favorites(conn)
|
||||
migrate_saved_views(conn)
|
||||
migrate_annotation_state(conn)
|
||||
|
||||
def _migrate_prompt_pairs(self, conn: sqlite3.Connection) -> None:
|
||||
conn.execute("SAVEPOINT prompt_pair_schema")
|
||||
@@ -1021,9 +1023,9 @@ class PromptModel:
|
||||
Args:
|
||||
path: File to inspect
|
||||
require_library_state: Also require the tables every current
|
||||
database has (Library state, image favorites and saved
|
||||
views), as a migrated restore stage must; old backups may
|
||||
lack them.
|
||||
database has (Library state, image favorites, saved views
|
||||
and image annotations), as a migrated restore stage must;
|
||||
old backups may lack them.
|
||||
|
||||
Returns:
|
||||
(ok, reason): reason is "ok" when the file is acceptable, otherwise
|
||||
@@ -1088,6 +1090,11 @@ class PromptModel:
|
||||
)
|
||||
if views_error:
|
||||
return False, views_error
|
||||
annotation_error = annotation_schema_error(
|
||||
conn, allow_missing=not require_library_state
|
||||
)
|
||||
if annotation_error:
|
||||
return False, annotation_error
|
||||
return True, "ok"
|
||||
except sqlite3.Error as e:
|
||||
return False, f"Invalid SQLite database: {e}"
|
||||
|
||||
+97
-6
@@ -12,6 +12,7 @@ from .models import PromptModel, normalize_image_path, utc_now_iso
|
||||
from .pairs import PairDependencyError, PairOperations
|
||||
from .library import LibraryOperations, titled
|
||||
from .image_favorites import ImageFavoriteOperations
|
||||
from .annotations import AnnotationOperations
|
||||
from .saved_views import SavedViewOperations
|
||||
from .library_commands import LibraryCommandOperations
|
||||
from .vocabulary import VocabularyOperations
|
||||
@@ -181,6 +182,21 @@ AND p.id NOT IN (SELECT prompt_id FROM live_hint_prompts)
|
||||
"""
|
||||
|
||||
|
||||
# The scan's known set (#249 Activity-2): every image row, with its stored file
|
||||
# size and whether its pair link is resolved: exactly one distinct pair, the rule
|
||||
# _LINK_SUMMARY_SQL and get_image_prompt_info use. A file is known and resolved
|
||||
# when it has exactly one image row, that row is resolved, and the row's stored
|
||||
# size equals the file's size now; several rows (one file linked to more than
|
||||
# one prompt) are ambiguous, and an unknown (NULL) stored size is not known. The
|
||||
# rows are streamed in fetches of KNOWN_PATHS_FETCH_SIZE and only the scanned
|
||||
# paths are kept, so memory follows the scan (at most MAX_SCAN_FILES paths), not
|
||||
# the table.
|
||||
_KNOWN_PATHS_SQL = """SELECT gi.file_path, gi.image_path, gi.file_size,
|
||||
(SELECT COUNT(DISTINCT ip.pair_id) FROM image_prompt_pairs ip
|
||||
WHERE ip.image_id = gi.id) = 1
|
||||
FROM generated_images gi"""
|
||||
KNOWN_PATHS_FETCH_SIZE = 1000
|
||||
|
||||
# One SELECT reads the link summaries of at most this many paths: three bound
|
||||
# parameters per path stays under SQLite's historical 999-variable limit.
|
||||
LINK_SUMMARY_BATCH = 200
|
||||
@@ -307,6 +323,7 @@ class PromptDatabase(
|
||||
ExecutionOutputOperations,
|
||||
HistoryReadOperations,
|
||||
ImageFavoriteOperations,
|
||||
AnnotationOperations,
|
||||
SavedViewOperations,
|
||||
):
|
||||
"""Database operations class for managing prompts."""
|
||||
@@ -1069,16 +1086,39 @@ class PromptDatabase(
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _bump_image_tag_bytes(conn: sqlite3.Connection, tag_id: int) -> int:
|
||||
"""Move the tag revision of every image set that carries *tag_id*.
|
||||
|
||||
Vocabulary rename, merge and delete change what an image's set reads
|
||||
as, so a Tag dialog opened before the change must see ``tag-conflict``
|
||||
on Save instead of recreating the old name. Bumps the image index
|
||||
revision once when any set was affected; returns that count.
|
||||
"""
|
||||
cursor = conn.execute(
|
||||
"UPDATE media_annotation_bytes SET tag_revision = tag_revision + 1 "
|
||||
"WHERE id IN (SELECT byte_id FROM media_image_tags WHERE tag_id = ?)",
|
||||
(tag_id,),
|
||||
)
|
||||
if cursor.rowcount > 0:
|
||||
conn.execute(
|
||||
"UPDATE media_annotation_metadata SET "
|
||||
"image_index_revision = image_index_revision + 1 WHERE singleton_id = 1"
|
||||
)
|
||||
return cursor.rowcount
|
||||
|
||||
@classmethod
|
||||
def _move_tag_links(
|
||||
conn: sqlite3.Connection, source_id: int, target_id: int
|
||||
cls, conn: sqlite3.Connection, source_id: int, target_id: int
|
||||
) -> None:
|
||||
"""Repoint component and pair links from source to target.
|
||||
"""Repoint component, pair and image links from source to target.
|
||||
|
||||
Owners that already carry the target keep that one link; the source's
|
||||
leftover duplicates are removed. The source registry row is left for the
|
||||
caller to delete.
|
||||
leftover duplicates are removed (an image set keeps the target's
|
||||
position). The source registry row is left for the caller to delete.
|
||||
Image sets that carried the source get a new tag revision first.
|
||||
"""
|
||||
for junction in ("prompt_tags", "prompt_pair_tags"):
|
||||
cls._bump_image_tag_bytes(conn, source_id)
|
||||
for junction in ("prompt_tags", "prompt_pair_tags", "media_image_tags"):
|
||||
conn.execute(
|
||||
f"UPDATE OR IGNORE {junction} SET tag_id = ? WHERE tag_id = ?",
|
||||
(target_id, source_id),
|
||||
@@ -1131,6 +1171,7 @@ class PromptDatabase(
|
||||
conn.execute("DELETE FROM tags WHERE id = ?", (old_tag_id,))
|
||||
else:
|
||||
# Simple rename
|
||||
self._bump_image_tag_bytes(conn, old_tag_id)
|
||||
conn.execute(
|
||||
"UPDATE tags SET name = ? WHERE id = ?", (new_name, old_tag_id)
|
||||
)
|
||||
@@ -1172,6 +1213,7 @@ class PromptDatabase(
|
||||
affected = cursor.fetchone()["c"]
|
||||
|
||||
# CASCADE delete handles prompt_tags entries
|
||||
self._bump_image_tag_bytes(conn, tag_id)
|
||||
conn.execute("DELETE FROM tags WHERE id = ?", (tag_id,))
|
||||
conn.commit()
|
||||
|
||||
@@ -1224,7 +1266,10 @@ class PromptDatabase(
|
||||
pair_count = conn.execute(
|
||||
"SELECT COUNT(*) FROM prompt_pair_tags WHERE tag_id = ?", (src_id,)
|
||||
).fetchone()[0]
|
||||
if src_count or pair_count:
|
||||
image_count = conn.execute(
|
||||
"SELECT COUNT(*) FROM media_image_tags WHERE tag_id = ?", (src_id,)
|
||||
).fetchone()[0]
|
||||
if src_count or pair_count or image_count:
|
||||
self._move_tag_links(conn, src_id, target_id)
|
||||
tags_merged += 1
|
||||
affected += src_count
|
||||
@@ -2280,6 +2325,52 @@ class PromptDatabase(
|
||||
"parameters": row[8] if len(row) > 8 else None,
|
||||
}
|
||||
|
||||
def get_known_resolved_paths(self, files: Dict[str, Optional[int]]) -> Set[str]:
|
||||
"""The paths in ``files`` the Library already links with a resolved link.
|
||||
|
||||
``files`` maps each scanned path to its size on disk (None when it could
|
||||
not be read). A path qualifies when exactly one ``generated_images`` row
|
||||
claims the file, that row has exactly one pair link
|
||||
(``image_prompt_pairs``, written only by resolved ingestion), and its
|
||||
stored ``file_size`` is not NULL and equals the given size, so a file
|
||||
replaced at the same path is read again. Files linked without a pair
|
||||
(the monitor's fallback and FIFO links, the legacy positive-only
|
||||
import), linked to several prompts or pairs, or not linked at all do
|
||||
not qualify. Both sides go through ``normalize_image_path`` (absolute,
|
||||
normalized, ``os.path.normcase``), so a Windows row stored in another
|
||||
case or spelling still matches. One query, streamed; returns the
|
||||
matching keys of ``files``.
|
||||
"""
|
||||
wanted: Dict[str, List[Tuple[str, Optional[int]]]] = {}
|
||||
for path, size in files.items():
|
||||
key = normalize_image_path(str(path))
|
||||
wanted.setdefault(key, []).append((str(path), size))
|
||||
if not wanted:
|
||||
return set()
|
||||
rows_seen: Dict[str, int] = {}
|
||||
stored_sizes: Dict[str, int] = {}
|
||||
with self.model.get_connection() as conn:
|
||||
cursor = conn.execute(_KNOWN_PATHS_SQL)
|
||||
while True:
|
||||
rows = cursor.fetchmany(KNOWN_PATHS_FETCH_SIZE)
|
||||
if not rows:
|
||||
break
|
||||
for file_path, image_path, file_size, is_resolved in rows:
|
||||
# image_path is NOT NULL; file_path is NULL only on old rows.
|
||||
key = normalize_image_path(file_path or image_path)
|
||||
if key not in wanted:
|
||||
continue
|
||||
rows_seen[key] = rows_seen.get(key, 0) + 1
|
||||
if is_resolved and file_size is not None:
|
||||
stored_sizes[key] = file_size
|
||||
return {
|
||||
path
|
||||
for key, stored_size in stored_sizes.items()
|
||||
if rows_seen[key] == 1
|
||||
for path, size in wanted[key]
|
||||
if size is not None and size == stored_size
|
||||
}
|
||||
|
||||
def get_image_link_summaries(
|
||||
self, paths: List[str], allow_filename_fallback: bool = True
|
||||
) -> Dict[str, Dict[str, Any]]:
|
||||
|
||||
+30
-4
@@ -3,6 +3,9 @@
|
||||
A count is the number of eligible Library parents that carry the value
|
||||
directly or through one active saved pair, which is exactly what
|
||||
``list_library(tags=[name])`` / ``list_library(category=value)`` return.
|
||||
A tag row's ``image_count`` is the number of files (private locators) with any
|
||||
byte record carrying the tag (#249), so a file counts once however many of its
|
||||
byte versions carry it, and equal bytes at two paths count twice.
|
||||
|
||||
Tag commands run in one ``_pair_transaction``: the storage epoch is checked
|
||||
first, then the exact ``{id, name}`` identity, and the receipt is read on the
|
||||
@@ -26,6 +29,11 @@ except ImportError:
|
||||
from utils.validators import validate_category, validate_tags
|
||||
|
||||
_CATEGORY_DOMAIN = "prompt-manager-vocabulary-category:v1\n"
|
||||
# Files carrying a tag through any of their byte records (#249 image tags).
|
||||
_IMAGE_FILES = (
|
||||
"SELECT mit.tag_id,COUNT(DISTINCT b.file_id) FROM media_image_tags mit "
|
||||
"JOIN media_annotation_bytes b ON b.id=mit.byte_id"
|
||||
)
|
||||
|
||||
|
||||
def _tag_rows(page: str) -> str:
|
||||
@@ -40,9 +48,13 @@ def _tag_rows(page: str) -> str:
|
||||
"WHERE pt.tag_id IN (SELECT id FROM page) AND " + _ELIGIBLE + " "
|
||||
"UNION SELECT ppt.tag_id,pp.positive_prompt_id FROM prompt_pairs pp "
|
||||
"JOIN prompt_pair_tags ppt ON ppt.pair_id=pp.id "
|
||||
"WHERE pp.archived=0 AND ppt.tag_id IN (SELECT id FROM page)) "
|
||||
"SELECT page.id,page.name,COUNT(DISTINCT owners.pid) AS prompt_count "
|
||||
"WHERE pp.archived=0 AND ppt.tag_id IN (SELECT id FROM page)),"
|
||||
"images(tag_id,n) AS (" + _IMAGE_FILES + " "
|
||||
"WHERE mit.tag_id IN (SELECT id FROM page) GROUP BY mit.tag_id) "
|
||||
"SELECT page.id,page.name,COUNT(DISTINCT owners.pid) AS prompt_count,"
|
||||
"COALESCE(MAX(images.n),0) AS image_count "
|
||||
"FROM page LEFT JOIN owners ON owners.tag_id=page.id "
|
||||
"LEFT JOIN images ON images.tag_id=page.id "
|
||||
"GROUP BY page.id ORDER BY page.id"
|
||||
)
|
||||
|
||||
@@ -61,7 +73,10 @@ _TAG_IMPACT = (
|
||||
"AS negative_variants,"
|
||||
"(SELECT COUNT(*) FROM prompt_pairs pp JOIN prompt_pair_tags ppt "
|
||||
"ON ppt.pair_id=pp.id WHERE ppt.tag_id=:tag AND pp.archived!=0) "
|
||||
"AS archived_negative_variants"
|
||||
"AS archived_negative_variants,"
|
||||
"(SELECT COUNT(DISTINCT b.file_id) FROM media_image_tags mit "
|
||||
"JOIN media_annotation_bytes b ON b.id=mit.byte_id WHERE mit.tag_id=:tag) "
|
||||
"AS images"
|
||||
)
|
||||
_MAX_ID = 2**63 - 1
|
||||
|
||||
@@ -205,7 +220,16 @@ class VocabularyOperations:
|
||||
"SELECT COUNT(DISTINCT pair_id) FROM prompt_pair_tags WHERE tag_id=?",
|
||||
(tag_id,),
|
||||
).fetchone()[0]
|
||||
return {"components_changed": components, "pairs_changed": pairs}
|
||||
images = conn.execute(
|
||||
"SELECT COUNT(DISTINCT b.file_id) FROM media_image_tags mit "
|
||||
"JOIN media_annotation_bytes b ON b.id=mit.byte_id WHERE mit.tag_id=?",
|
||||
(tag_id,),
|
||||
).fetchone()[0]
|
||||
return {
|
||||
"components_changed": components,
|
||||
"pairs_changed": pairs,
|
||||
"images_changed": images,
|
||||
}
|
||||
|
||||
def create_vocabulary_tag(self, name: str, *, expected_epoch: str) -> dict:
|
||||
"""Insert an unused registry row, or reuse the exact existing name."""
|
||||
@@ -250,6 +274,7 @@ class VocabularyOperations:
|
||||
"SELECT id FROM tags WHERE name=?", (new_name,)
|
||||
).fetchone()
|
||||
if target is None:
|
||||
self._bump_image_tag_bytes(conn, tag_id)
|
||||
conn.execute("UPDATE tags SET name=? WHERE id=?", (new_name, tag_id))
|
||||
result_id = tag_id
|
||||
else:
|
||||
@@ -275,6 +300,7 @@ class VocabularyOperations:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
self._vocabulary_identity(conn, epoch, tag_id, name)
|
||||
counts = self._tag_owner_counts(conn, tag_id)
|
||||
self._bump_image_tag_bytes(conn, tag_id)
|
||||
conn.execute("DELETE FROM tags WHERE id=?", (tag_id,))
|
||||
return {"library_epoch": epoch, "deleted": True, **counts}
|
||||
|
||||
|
||||
@@ -29,6 +29,12 @@ Each role has `status` (`resolved`, `absent`, `unresolved`, or `ambiguous`), `te
|
||||
(string only when resolved, otherwise null), and `source`. Inspection does not
|
||||
index files, alter saved prompts, run scans, or decode image pixels. PNG text and
|
||||
video probe output are bounded; unsupported or malformed evidence remains
|
||||
unresolved rather than becoming sendable placeholder text.
|
||||
unresolved rather than becoming sendable placeholder text. Videos are read from
|
||||
one container carrier: the per-key tags written by core SaveVideo/SaveWEBM and
|
||||
current VideoHelperSuite (`prompt`, `workflow`, `parameters`,
|
||||
`comfyassets_prompt_pairs`), else the old VideoHelperSuite `comment` envelope,
|
||||
else `description`. The admin scan reads the same carrier, so Images and
|
||||
ingestion agree on a video's prompts (see TESTING.md for the order and budgets;
|
||||
the live monitor processes images only).
|
||||
|
||||
Python route changes require restarting ComfyUI. Browser assets require a refresh.
|
||||
|
||||
+10
-2
@@ -6,6 +6,7 @@ Split into domain-specific mixins for maintainability:
|
||||
- AdminRoutesMixin: duplicates, stats, settings, diagnostics, maintenance, backup, scan
|
||||
- LoggingRoutesMixin: log management endpoints
|
||||
- AutotagRoutesMixin: auto-tagging model management and tagging
|
||||
- JobRoutesMixin: the session job registry behind Activity (py/api/jobs.py)
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
@@ -29,6 +30,8 @@ from .saved_views import SavedViewRoutesMixin
|
||||
from .history import HistoryRoutesMixin
|
||||
from .images import ImageRoutesMixin
|
||||
from .admin import AdminRoutesMixin
|
||||
from .job_routes import JobRoutesMixin
|
||||
from .jobs import JobRegistry
|
||||
from .logging_routes import LoggingRoutesMixin
|
||||
from .autotag_routes import AutotagRoutesMixin
|
||||
from .lora_integration import LoraIntegrationMixin
|
||||
@@ -254,6 +257,7 @@ class PromptManagerAPI(
|
||||
PromptRoutesMixin,
|
||||
ImageRoutesMixin,
|
||||
AdminRoutesMixin,
|
||||
JobRoutesMixin,
|
||||
LoggingRoutesMixin,
|
||||
AutotagRoutesMixin,
|
||||
LoraIntegrationMixin,
|
||||
@@ -299,9 +303,12 @@ class PromptManagerAPI(
|
||||
# _acquire_long_job(). Created lazily so no lock binds to an event
|
||||
# loop before the server's loop exists.
|
||||
self._long_jobs: dict = {}
|
||||
# The current output scan (py/api/scan_job.ScanJob), if one was started.
|
||||
# This session's jobs (py/api/jobs.py): the metadata scan, and later
|
||||
# every other long operation the Activity page tracks.
|
||||
self._jobs = JobRegistry(self.logger)
|
||||
# The latest metadata scan job (also in self._jobs), if one was started.
|
||||
self._scan_job = None
|
||||
# The current duplicate scan (also a ScanJob), if one was started.
|
||||
# The current duplicate scan (a py/api/jobs.Job outside the registry).
|
||||
self._duplicate_job = None
|
||||
|
||||
# Run cleanup on initialization to remove any existing duplicates
|
||||
@@ -439,6 +446,7 @@ class PromptManagerAPI(
|
||||
self._register_prompt_routes(routes)
|
||||
self._register_image_routes(routes)
|
||||
self._register_admin_routes(routes)
|
||||
self._register_job_routes(routes)
|
||||
self._register_logging_routes(routes)
|
||||
self._register_autotag_routes(routes)
|
||||
self._register_lora_routes(routes)
|
||||
|
||||
+305
-53
@@ -6,16 +6,25 @@ from contextlib import closing
|
||||
import datetime
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .prompts import thumbnail_url_for
|
||||
from .scan_job import ScanJob
|
||||
from .jobs import (
|
||||
DEFAULT_SCAN_SCOPE,
|
||||
SCAN_GROUP_NAMES,
|
||||
SCAN_GROUPS,
|
||||
SCAN_SCOPES,
|
||||
Job,
|
||||
JobBusy,
|
||||
)
|
||||
|
||||
try:
|
||||
from ...utils.parallel import map_parallel
|
||||
@@ -47,8 +56,10 @@ except ImportError:
|
||||
|
||||
|
||||
try:
|
||||
from ...database.library import display_title
|
||||
from ...utils.hashing import generate_prompt_hash
|
||||
except ImportError:
|
||||
from database.library import display_title
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
IMAGE_EXTENSIONS = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tiff")
|
||||
@@ -138,6 +149,127 @@ def _sse(payload):
|
||||
return f"data: {json.dumps(payload)}\n\n"
|
||||
|
||||
|
||||
SCAN_JOB_KIND = "scan"
|
||||
SCAN_JOB_TITLE = "Scan for metadata"
|
||||
# The legacy final event type of a scan and the job state it ends in.
|
||||
SCAN_FINAL_STATES = {
|
||||
"complete": "finished",
|
||||
"cancelled": "cancelled",
|
||||
"error": "failed",
|
||||
}
|
||||
# The scope as the scan log's first line names it (the review dialog's words).
|
||||
SCAN_SCOPE_LOG = {"new": "new and unresolved files", "all": "all files"}
|
||||
MIN_BATCHES_FOR_ESTIMATE = 2 # "about X min left" waits for this many batches
|
||||
# Log line prefix per outcome group (spec: + added, = known, ? review, ! error,
|
||||
# - no usable metadata).
|
||||
SCAN_LOG_PREFIXES = {
|
||||
"added": "+",
|
||||
"known": "=",
|
||||
"review": "?",
|
||||
"errors": "!",
|
||||
"no_metadata": "-",
|
||||
}
|
||||
# What an outcome row says about a file: where it went (with the prompt's
|
||||
# title) or why it was skipped. "error" is a file whose ingestion raised;
|
||||
# None is a file without prompt metadata.
|
||||
SCAN_TITLED_DETAILS = {
|
||||
"added": "Added to the Library as “{title}”",
|
||||
"linked": "Linked to “{title}”",
|
||||
"enriched": "Metadata added to “{title}”",
|
||||
"unchanged": "Already linked to “{title}”",
|
||||
}
|
||||
SCAN_DETAILS = {
|
||||
"added": "Added to the Library",
|
||||
"linked": "Linked to an existing prompt",
|
||||
"enriched": "Metadata added to its prompt",
|
||||
"unchanged": "Already linked to its prompt",
|
||||
"skipped": "Skipped · already linked",
|
||||
"ambiguous": "Needs review: more than one prompt could match",
|
||||
"unresolved": "Needs review: the prompt could not be traced in the workflow",
|
||||
"conflict": "Needs review: already linked to a different prompt",
|
||||
"failed": "Error: the prompt could not be stored",
|
||||
"error": "Error: the file could not be processed (see the server log)",
|
||||
"found": "Metadata found, but no prompt text to store",
|
||||
None: "No prompt metadata in the file",
|
||||
}
|
||||
|
||||
|
||||
def _plural(count, word):
|
||||
return f"{count} {word}" if count == 1 else f"{count} {word}s"
|
||||
|
||||
|
||||
def _scan_status_line(done, total, batches, elapsed):
|
||||
""" "Processing file N of M", with an estimate once enough batches ran."""
|
||||
line = f"Processing file {done:,} of {total:,}"
|
||||
if batches < MIN_BATCHES_FOR_ESTIMATE or done <= 0 or done >= total:
|
||||
return line
|
||||
minutes = max(1, math.ceil(elapsed / done * (total - done) / 60))
|
||||
return f"{line} · about {minutes} min left"
|
||||
|
||||
|
||||
def _file_size(path):
|
||||
"""``path``'s size on disk, or None when it cannot be read."""
|
||||
try:
|
||||
return os.path.getsize(path)
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def _with_file_size(canonical, media_file):
|
||||
"""Record the file's size in ``file_info`` unless the metadata carries one.
|
||||
|
||||
Stored as ``generated_images.file_size``; the "new" scan scope skips a known
|
||||
file only while its size is unchanged.
|
||||
"""
|
||||
info = canonical.get("file_info")
|
||||
info = dict(info) if isinstance(info, dict) else {}
|
||||
if info.get("size") is None:
|
||||
size = _file_size(media_file)
|
||||
if size is not None:
|
||||
info["size"] = size
|
||||
if info:
|
||||
canonical["file_info"] = info
|
||||
|
||||
|
||||
def _scan_roots(output_dirs):
|
||||
"""(absolute root, public label, listing index), the most specific first.
|
||||
|
||||
The index is the root's place in ``output_dirs`` (``_get_all_output_dirs``),
|
||||
the ``root_index`` the workspace listing names it by.
|
||||
"""
|
||||
public_path, _ = _public_helpers()
|
||||
roots = [
|
||||
(os.path.abspath(str(d)), public_path(d), index)
|
||||
for index, d in enumerate(output_dirs)
|
||||
]
|
||||
return sorted(roots, key=lambda root: len(root[0]), reverse=True)
|
||||
|
||||
|
||||
def _scan_location(media_file, roots):
|
||||
"""``media_file`` relative to its output root, that root's label and index."""
|
||||
full = os.path.abspath(str(media_file))
|
||||
folded = os.path.normcase(full)
|
||||
for root, label, index in roots:
|
||||
base = os.path.normcase(root).rstrip("\\/") + os.sep
|
||||
if folded.startswith(base):
|
||||
return Path(os.path.relpath(full, root)).as_posix(), label, index
|
||||
return Path(full).name, "", None
|
||||
|
||||
|
||||
def _scan_outcome(media_file, roots, status, title=None):
|
||||
"""The (group, row, log line) one scanned file records in its job."""
|
||||
if status in SCAN_GROUPS:
|
||||
group = SCAN_GROUPS[status]
|
||||
else:
|
||||
group = "errors" if status == "error" else "no_metadata"
|
||||
template = SCAN_TITLED_DETAILS.get(status) if title else None
|
||||
detail = template.format(title=title) if template else SCAN_DETAILS[status]
|
||||
path, label, index = _scan_location(media_file, roots)
|
||||
row = {"path": path, "root": label, "root_index": index, "detail": detail}
|
||||
where = f"{label}/{path}" if label else path
|
||||
return group, row, f"{SCAN_LOG_PREFIXES[group]} {where} · {detail}"
|
||||
|
||||
|
||||
def _inspect_restore_upload(path):
|
||||
"""Checks on a restore upload beyond PromptModel.verify_database_file.
|
||||
|
||||
@@ -437,7 +569,7 @@ class AdminRoutesMixin:
|
||||
guard = self._acquire_long_job(self.LONG_JOB_DUPLICATES)
|
||||
if guard is None:
|
||||
return self._long_job_busy_response("Duplicate scan or delete")
|
||||
job = ScanJob()
|
||||
job = Job("duplicates", "Duplicate scan")
|
||||
self._duplicate_job = job
|
||||
job.task = asyncio.ensure_future(self._run_duplicate_job(job, guard))
|
||||
return await self._stream_scan_job(request, job)
|
||||
@@ -1319,14 +1451,18 @@ class AdminRoutesMixin:
|
||||
async def scan_images(self, request):
|
||||
"""Scan ComfyUI output images for prompt metadata and add them to the database.
|
||||
|
||||
The scan runs as a background task that outlives this request, so a
|
||||
closed tab never leaves a half-finished scan. Progress streams as
|
||||
server-sent events; while a scan is running, a new request attaches
|
||||
to it instead of starting another one.
|
||||
The scan runs as the registry's ``scan`` job, which outlives this
|
||||
request, so a closed tab never leaves a half-finished scan. Progress
|
||||
streams as the legacy server-sent events; while a scan is running
|
||||
(started here or from the workspace), a new request attaches to it
|
||||
instead of starting another one.
|
||||
"""
|
||||
job = self._scan_job
|
||||
if job is None or job.finished:
|
||||
job = self._start_scan_job()
|
||||
job = self._jobs.active(SCAN_JOB_KIND)
|
||||
if job is None:
|
||||
try:
|
||||
job = self._start_scan_job("all")
|
||||
except JobBusy as busy:
|
||||
return self._long_job_busy_response(busy.running.title)
|
||||
return await self._stream_scan_job(request, job)
|
||||
|
||||
async def cancel_scan(self, request):
|
||||
@@ -1348,19 +1484,37 @@ class AdminRoutesMixin:
|
||||
}
|
||||
)
|
||||
|
||||
def _start_scan_job(self):
|
||||
"""Create the job record synchronously, then run the scan as a task."""
|
||||
job = ScanJob()
|
||||
def _start_scan_job(self, files=DEFAULT_SCAN_SCOPE):
|
||||
"""Register the scan job synchronously; its runner starts on the loop.
|
||||
|
||||
``files`` is the scope (``SCAN_SCOPES``). Raises JobBusy while another
|
||||
record-changing job runs.
|
||||
"""
|
||||
if files not in SCAN_SCOPES:
|
||||
raise ValueError(f"Unknown scan scope: {files!r}")
|
||||
job = self._jobs.start(
|
||||
SCAN_JOB_KIND,
|
||||
SCAN_JOB_TITLE,
|
||||
record_changing=True,
|
||||
scope={"files": files},
|
||||
runner=self._run_scan_job,
|
||||
groups=SCAN_GROUP_NAMES,
|
||||
)
|
||||
self._scan_job = job
|
||||
job.task = asyncio.ensure_future(self._run_scan_job(job))
|
||||
return job
|
||||
|
||||
async def _run_scan_job(self, job):
|
||||
try:
|
||||
async for payload in self._scan_images_events():
|
||||
job.publish(payload)
|
||||
finally:
|
||||
job.finish()
|
||||
final = None
|
||||
async for payload in self._scan_images_events(job):
|
||||
job.publish(payload)
|
||||
final = payload.get("type")
|
||||
state = SCAN_FINAL_STATES.get(final, "finished")
|
||||
if state == "failed":
|
||||
job.log_line(f"Scan failed · {job.status_line}")
|
||||
else:
|
||||
verb = "cancelled" if state == "cancelled" else "finished"
|
||||
job.log_line(f"Scan {verb} · {_plural(job.processed, 'file')} processed")
|
||||
job.finish(state)
|
||||
|
||||
async def _stream_scan_job(self, request, job):
|
||||
"""Stream ``job`` as SSE; stop quietly if the client goes away."""
|
||||
@@ -1386,21 +1540,31 @@ class AdminRoutesMixin:
|
||||
)
|
||||
return response
|
||||
|
||||
async def _scan_images_events(self):
|
||||
"""Yield progress payloads while scanning every configured output directory."""
|
||||
async def _scan_images_events(self, job):
|
||||
"""Yield progress payloads while scanning every configured output directory.
|
||||
|
||||
The payloads are the legacy SSE events, unchanged; ``job`` also gets
|
||||
its status line, totals and one outcome per processed file.
|
||||
"""
|
||||
try:
|
||||
self.logger.info("Starting image scan operation")
|
||||
output_dirs = self._get_all_output_dirs()
|
||||
if not output_dirs:
|
||||
self.logger.error("No output directories found")
|
||||
yield {
|
||||
"type": "error",
|
||||
"message": "No output directories found. "
|
||||
"Configure scan directories in Settings.",
|
||||
}
|
||||
message = (
|
||||
"No output directories found. "
|
||||
"Configure scan directories in Settings."
|
||||
)
|
||||
job.update(status_line=message)
|
||||
yield {"type": "error", "message": message}
|
||||
return
|
||||
|
||||
dir_names = [self._public_path(d) for d in output_dirs]
|
||||
folders = _plural(len(output_dirs), "output folder")
|
||||
job.scope["output_folders"] = len(output_dirs)
|
||||
scope = SCAN_SCOPE_LOG[job.scope.get("files", "all")]
|
||||
job.log_line(f"Scan started · {folders} · {scope}")
|
||||
job.update(status_line=f"Looking for media files in {folders}")
|
||||
yield self._scan_progress(
|
||||
0, f"Scanning {len(output_dirs)} directory(ies) for media files..."
|
||||
)
|
||||
@@ -1409,11 +1573,21 @@ class AdminRoutesMixin:
|
||||
self._collect_output_media_sync, output_dirs
|
||||
)
|
||||
counts = dict.fromkeys(SCAN_COUNT_KEYS, 0)
|
||||
job.update(
|
||||
total=len(media_files),
|
||||
status_line=_scan_status_line(0, len(media_files), 0, 0),
|
||||
)
|
||||
if media_files:
|
||||
yield self._scan_progress(
|
||||
5, f"Found {len(media_files)} media files to process...", counts
|
||||
)
|
||||
async for event in self._scan_batches(media_files, counts):
|
||||
roots = _scan_roots(output_dirs)
|
||||
known = set()
|
||||
if media_files and job.scope.get("files") != "all":
|
||||
known = await self._known_scan_files(job, media_files)
|
||||
async for event in self._scan_batches(
|
||||
media_files, counts, job, roots, known
|
||||
):
|
||||
yield event
|
||||
|
||||
self.logger.info(
|
||||
@@ -1421,37 +1595,78 @@ class AdminRoutesMixin:
|
||||
f"found={counts['found']}, new_prompts_added={counts['added']}, "
|
||||
f"images_linked_to_existing={counts['linked']}"
|
||||
)
|
||||
job = self._scan_job
|
||||
final_type = "cancelled" if job and job.cancelled else "complete"
|
||||
final_type = "cancelled" if job.cancelled else "complete"
|
||||
if final_type == "complete":
|
||||
job.update(progress=100)
|
||||
yield {"type": final_type, **counts, "directories": dir_names}
|
||||
|
||||
except Exception:
|
||||
self.logger.exception("Scan error")
|
||||
job.update(status_line="The scan stopped on an internal error.")
|
||||
yield {
|
||||
"type": "error",
|
||||
"message": "An internal error occurred. Check server logs for details.",
|
||||
}
|
||||
|
||||
async def _scan_batches(self, media_files, counts):
|
||||
async def _known_scan_files(self, job, media_files):
|
||||
"""The "new" scope's skip set; a failed read falls back to reading all."""
|
||||
try:
|
||||
return await self._run_in_executor(self._known_scan_files_sync, media_files)
|
||||
except Exception:
|
||||
self.logger.warning(
|
||||
"Known files could not be read; the scan reads every file",
|
||||
exc_info=True,
|
||||
)
|
||||
job.log_line("Known files could not be read · reading every file")
|
||||
return set()
|
||||
|
||||
def _known_scan_files_sync(self, media_files):
|
||||
"""Stat each scanned file and ask the database which are known (blocking)."""
|
||||
sizes = {str(media_file): _file_size(media_file) for media_file in media_files}
|
||||
return self.db.get_known_resolved_paths(sizes)
|
||||
|
||||
async def _scan_batches(self, media_files, counts, job, roots=(), known=()):
|
||||
"""Process ``media_files`` in executor batches, updating ``counts`` in place.
|
||||
|
||||
Yields one progress event per batch.
|
||||
Records each file's outcome in ``job`` and yields one progress event
|
||||
per batch. Files in ``known`` (path strings) are not read: they count
|
||||
as processed and unchanged and are recorded as skipped.
|
||||
"""
|
||||
total = len(media_files)
|
||||
started = time.monotonic()
|
||||
batches = 0
|
||||
for batch_start in range(0, total, SCAN_BATCH_SIZE):
|
||||
if self._scan_job and self._scan_job.cancelled:
|
||||
if job.cancelled:
|
||||
return
|
||||
batch = media_files[batch_start : batch_start + SCAN_BATCH_SIZE]
|
||||
batch_results = await self._run_in_executor(
|
||||
self._extract_batch_metadata_sync, batch
|
||||
)
|
||||
batch_counts = await self._run_in_executor(
|
||||
self._ingest_scan_batch_sync, batch_results
|
||||
)
|
||||
for key in counts:
|
||||
counts[key] += batch_counts[key]
|
||||
skipped = [f for f in batch if str(f) in known]
|
||||
to_read = [f for f in batch if str(f) not in known]
|
||||
outcomes = [_scan_outcome(f, roots, "skipped") for f in skipped]
|
||||
counts["processed"] += len(skipped)
|
||||
counts["unchanged"] += len(skipped)
|
||||
if to_read:
|
||||
batch_results = await self._run_in_executor(
|
||||
self._extract_batch_metadata_sync, to_read
|
||||
)
|
||||
batch_counts = await self._run_in_executor(
|
||||
self._ingest_scan_batch_sync,
|
||||
batch_results,
|
||||
roots=roots,
|
||||
outcomes=outcomes,
|
||||
)
|
||||
for key in counts:
|
||||
counts[key] += batch_counts[key]
|
||||
for group, row, log_line in outcomes:
|
||||
job.record(group, row, log_line)
|
||||
|
||||
batches += 1
|
||||
done = min(batch_start + len(batch), total)
|
||||
job.update(
|
||||
processed=counts["processed"],
|
||||
status_line=_scan_status_line(
|
||||
done, total, batches, time.monotonic() - started
|
||||
),
|
||||
)
|
||||
yield self._scan_progress(
|
||||
int(done / total * 100), f"Processing file {done}/{total}...", counts
|
||||
)
|
||||
@@ -1490,9 +1705,19 @@ class AdminRoutesMixin:
|
||||
|
||||
return map_parallel(extract, file_batch, PromptManagerConfig.WORKER_THREADS)
|
||||
|
||||
def _ingest_scan_batch_sync(self, batch_results):
|
||||
"""Store the prompts found in a metadata batch (blocking DB work)."""
|
||||
def _ingest_scan_batch_sync(self, batch_results, roots=(), outcomes=None):
|
||||
"""Store the prompts found in a metadata batch (blocking DB work).
|
||||
|
||||
Returns the legacy counts. When ``outcomes`` is a list, every processed
|
||||
file appends its ``(group, row, log_line)`` for the scan job, with the
|
||||
path relative to its root in ``roots`` (see ``_scan_roots``).
|
||||
"""
|
||||
counts = dict.fromkeys(SCAN_COUNT_KEYS, 0)
|
||||
|
||||
def note(media_file, status, title=None):
|
||||
if outcomes is not None:
|
||||
outcomes.append(_scan_outcome(media_file, roots, status, title))
|
||||
|
||||
for media_file, metadata in batch_results:
|
||||
if self._scan_job and self._scan_job.cancelled:
|
||||
break
|
||||
@@ -1502,25 +1727,21 @@ class AdminRoutesMixin:
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error processing {media_file.name}: {e}")
|
||||
counts["failed"] += 1
|
||||
note(media_file, "error")
|
||||
continue
|
||||
if outcome is None:
|
||||
note(media_file, None)
|
||||
continue
|
||||
counts["found"] += 1
|
||||
status = outcome["status"] if isinstance(outcome, dict) else outcome
|
||||
if status in (
|
||||
"added",
|
||||
"linked",
|
||||
"enriched",
|
||||
"unchanged",
|
||||
"ambiguous",
|
||||
"unresolved",
|
||||
"conflict",
|
||||
"failed",
|
||||
):
|
||||
if status in SCAN_GROUPS:
|
||||
counts[status] += 1
|
||||
if isinstance(outcome, dict):
|
||||
counts["pairs_added"] += outcome.get("pairs_added", 0)
|
||||
counts["pair_links_added"] += outcome.get("links_added", 0)
|
||||
note(media_file, status, outcome.get("title"))
|
||||
else:
|
||||
note(media_file, status)
|
||||
return counts
|
||||
|
||||
@staticmethod
|
||||
@@ -1539,7 +1760,13 @@ class AdminRoutesMixin:
|
||||
return positive.strip() or None
|
||||
|
||||
def _ingest_scanned_file(self, media_file, metadata, detailed=False):
|
||||
"""Enrich exact pairs; retain legacy positive-only import for absent pairs."""
|
||||
"""Enrich exact pairs; retain legacy positive-only import for absent pairs.
|
||||
|
||||
Returns the file's scan status, or None without prompt metadata. With
|
||||
``detailed``, a stored or linked file returns a dict instead: its
|
||||
``status``, the prompt's display ``title`` and, for a resolved pair,
|
||||
the pair counts.
|
||||
"""
|
||||
if not metadata:
|
||||
return None
|
||||
canonical = dict(metadata)
|
||||
@@ -1552,12 +1779,18 @@ class AdminRoutesMixin:
|
||||
pass
|
||||
if not canonical.get("prompt") and not canonical.get("parameters"):
|
||||
return None
|
||||
_with_file_size(canonical, media_file)
|
||||
resolution = resolve_prompt_pair(canonical)
|
||||
if resolution["state"] == "resolved":
|
||||
result = ingest_resolved_pair(
|
||||
self.db, str(media_file), canonical, resolution
|
||||
)
|
||||
return result if detailed else result["status"]
|
||||
if not detailed:
|
||||
return result["status"]
|
||||
return {
|
||||
**result,
|
||||
"title": self._scan_title(media_file, resolution["positive"]["text"]),
|
||||
}
|
||||
if canonical.get("prompt") and resolution["state"] in (
|
||||
"ambiguous",
|
||||
"unresolved",
|
||||
@@ -1585,6 +1818,25 @@ class AdminRoutesMixin:
|
||||
return "found"
|
||||
prompt_text = prompt_text.strip()
|
||||
|
||||
status = self._store_scanned_prompt(media_file, canonical, prompt_text)
|
||||
if not detailed or status not in SCAN_TITLED_DETAILS:
|
||||
return status
|
||||
return {"status": status, "title": self._scan_title(media_file, prompt_text)}
|
||||
|
||||
def _scan_title(self, media_file, text):
|
||||
"""The linked prompt's Library title (its stored title, else ``text``'s
|
||||
first line), as the Library and the inspector name it."""
|
||||
try:
|
||||
path = str(media_file)
|
||||
summary = self.db.get_image_link_summaries([path], False).get(path)
|
||||
if summary and summary.get("title"):
|
||||
return summary["title"]
|
||||
except Exception:
|
||||
self.logger.warning("Scan row title could not be read; using its text")
|
||||
return display_title((), text)
|
||||
|
||||
def _store_scanned_prompt(self, media_file, canonical, prompt_text):
|
||||
"""Link ``media_file`` to ``prompt_text``'s prompt, saving it when new."""
|
||||
prompt_hash = generate_prompt_hash(prompt_text)
|
||||
existing = self.db.get_prompt_by_hash(prompt_hash)
|
||||
if existing:
|
||||
|
||||
@@ -705,6 +705,8 @@ def _workspace_filters(query: Mapping) -> dict:
|
||||
media = query.get("media", "all")
|
||||
order = query.get("sort", "newest")
|
||||
folder = query.get("subfolder", "").strip().replace("\\", "/")
|
||||
# ``path``: one exact relative path (Activity opens a scan row by it, D57).
|
||||
exact = query.get("path", "").strip().replace("\\", "/")
|
||||
category = query.get("category", "").strip() or None
|
||||
tags = sorted(
|
||||
{tag.strip() for tag in query.get("tags", "").split(",") if tag.strip()}
|
||||
@@ -723,12 +725,15 @@ def _workspace_filters(query: Mapping) -> dict:
|
||||
raise ValueError("Invalid gallery filters")
|
||||
if folder and (_has_traversal(folder) or len(folder) > 1024):
|
||||
raise ValueError("Invalid gallery folder")
|
||||
if exact and (_has_traversal(exact) or len(exact) > 1024):
|
||||
raise ValueError("Invalid gallery path")
|
||||
return {
|
||||
"text": text,
|
||||
"query": folded,
|
||||
"media": media,
|
||||
"order": order,
|
||||
"folder": folder,
|
||||
"path": exact.casefold(),
|
||||
"category": category,
|
||||
"tags": tags,
|
||||
"favorites": favorites == "1",
|
||||
@@ -762,7 +767,10 @@ def _filtered_page(
|
||||
native_folder = folder.replace("/", os.sep)
|
||||
nested = native_folder + os.sep
|
||||
total, page, positives = 0, [], set()
|
||||
exact = filters.get("path", "")
|
||||
for position, entry in enumerate(entries):
|
||||
if exact and entry.folded != exact:
|
||||
continue
|
||||
if folder and not (
|
||||
entry.folder == native_folder or entry.folder.startswith(nested)
|
||||
):
|
||||
@@ -1155,6 +1163,7 @@ class ImageRoutesMixin:
|
||||
filters["media"],
|
||||
filters["order"],
|
||||
filters["folder"],
|
||||
filters["path"],
|
||||
root_index,
|
||||
[str(root) for root in output_dirs],
|
||||
filters["category"],
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
"""Session job routes (Activity): list, inspect, follow and cancel jobs.
|
||||
|
||||
The registry itself is ``py/api/jobs.py``; the scan job's runner is in
|
||||
``py/api/admin.py``. Every response leaves out absolute server paths: job
|
||||
rows already carry root-relative paths and public root labels.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .jobs import DEFAULT_SCAN_SCOPE, SCAN_SCOPES, JobBusy
|
||||
|
||||
|
||||
def _not_found():
|
||||
return web.json_response({"success": False, "error": "Job not found"}, status=404)
|
||||
|
||||
|
||||
def _bad_request(message):
|
||||
return web.json_response({"success": False, "error": message}, status=400)
|
||||
|
||||
|
||||
async def _scan_scope(request):
|
||||
"""The requested scan scope ("new" or "all"), or None when the body is invalid.
|
||||
|
||||
An empty body (also a chunked one with no bytes), or one without
|
||||
``scope``, asks for the default.
|
||||
"""
|
||||
if not request.can_read_body:
|
||||
return DEFAULT_SCAN_SCOPE
|
||||
text = await request.text()
|
||||
if not text.strip(): # e.g. a chunked POST that sent no bytes
|
||||
return DEFAULT_SCAN_SCOPE
|
||||
try:
|
||||
body = json.loads(text)
|
||||
except ValueError:
|
||||
return None
|
||||
if not isinstance(body, dict):
|
||||
return None
|
||||
scope = body.get("scope", DEFAULT_SCAN_SCOPE)
|
||||
return scope if scope in SCAN_SCOPES else None
|
||||
|
||||
|
||||
def _summary_event(job):
|
||||
return f"data: {json.dumps(job.summary())}\n\n".encode("utf-8")
|
||||
|
||||
|
||||
class JobRoutesMixin:
|
||||
"""Mixin providing the /prompt_manager/jobs endpoints."""
|
||||
|
||||
def _register_job_routes(self, routes):
|
||||
@routes.get("/prompt_manager/jobs")
|
||||
async def list_jobs_route(request):
|
||||
return await self.list_jobs(request)
|
||||
|
||||
@routes.post("/prompt_manager/jobs/scan")
|
||||
async def start_scan_job_route(request):
|
||||
return await self.start_scan_job(request)
|
||||
|
||||
@routes.get("/prompt_manager/jobs/{job_id}")
|
||||
async def job_detail_route(request):
|
||||
return await self.job_detail(request)
|
||||
|
||||
@routes.get("/prompt_manager/jobs/{job_id}/report")
|
||||
async def job_report_route(request):
|
||||
return await self.job_report(request)
|
||||
|
||||
@routes.get("/prompt_manager/jobs/{job_id}/events")
|
||||
async def job_events_route(request):
|
||||
return await self.job_events(request)
|
||||
|
||||
@routes.post("/prompt_manager/jobs/{job_id}/cancel")
|
||||
async def cancel_job_route(request):
|
||||
return await self.cancel_job(request)
|
||||
|
||||
async def list_jobs(self, request):
|
||||
"""This session's jobs: running first, then finished, newest first."""
|
||||
jobs = [job.summary() for job in self._jobs.list()]
|
||||
return web.json_response({"success": True, "jobs": jobs})
|
||||
|
||||
async def job_detail(self, request):
|
||||
job = self._jobs.get(request.match_info["job_id"])
|
||||
if job is None:
|
||||
return _not_found()
|
||||
return web.json_response({"success": True, "job": job.detail()})
|
||||
|
||||
async def job_report(self, request):
|
||||
"""The job's whole record as a JSON attachment (no absolute paths)."""
|
||||
job = self._jobs.get(request.match_info["job_id"])
|
||||
if job is None:
|
||||
return _not_found()
|
||||
body = json.dumps(job.report(), indent=2)
|
||||
disposition = f'attachment; filename="{job.report_filename()}"'
|
||||
return web.Response(
|
||||
text=body,
|
||||
content_type="application/json",
|
||||
headers={
|
||||
"Content-Disposition": disposition,
|
||||
"Cache-Control": "no-store",
|
||||
},
|
||||
)
|
||||
|
||||
async def cancel_job(self, request):
|
||||
"""Stop a job at its next file boundary; a finished job is a no-op."""
|
||||
job_id = request.match_info["job_id"]
|
||||
if self._jobs.get(job_id) is None:
|
||||
return _not_found()
|
||||
requested = self._jobs.cancel(job_id)
|
||||
return web.json_response({"success": True, "cancellation_requested": requested})
|
||||
|
||||
async def start_scan_job(self, request):
|
||||
"""Start the metadata scan as a job (202), or 409 naming the busy job.
|
||||
|
||||
The body's ``scope`` is "new" (the default: new and unresolved files)
|
||||
or "all"; anything else is a 400.
|
||||
"""
|
||||
files = await _scan_scope(request)
|
||||
if files is None:
|
||||
return _bad_request('The scan scope must be "new" or "all"')
|
||||
try:
|
||||
job = self._start_scan_job(files)
|
||||
except JobBusy as busy:
|
||||
running = busy.running
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": "busy",
|
||||
"error": str(busy),
|
||||
"running": {"id": running.id, "title": running.title},
|
||||
},
|
||||
status=409,
|
||||
)
|
||||
return web.json_response({"success": True, "job": job.summary()}, status=202)
|
||||
|
||||
async def job_events(self, request):
|
||||
"""Stream the job's summary as SSE: now, after each event, and at the end."""
|
||||
job = self._jobs.get(request.match_info["job_id"])
|
||||
if job is None:
|
||||
return _not_found()
|
||||
response = web.StreamResponse(
|
||||
status=200,
|
||||
reason="OK",
|
||||
headers={
|
||||
"Content-Type": "text/event-stream",
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
},
|
||||
)
|
||||
await response.prepare(request)
|
||||
try:
|
||||
await response.write(_summary_event(job))
|
||||
async with contextlib.aclosing(job.events()) as events:
|
||||
async for _payload in events:
|
||||
await response.write(_summary_event(job))
|
||||
# The last event is published before the job takes its final state.
|
||||
await response.write(_summary_event(job))
|
||||
await response.write_eof()
|
||||
except Exception as e: # the socket closed under us; the job goes on
|
||||
self.logger.info(f"Job progress client disconnected ({e})")
|
||||
return response
|
||||
+359
@@ -0,0 +1,359 @@
|
||||
"""The session job registry: every long operation as one tracked job.
|
||||
|
||||
A job runs as an asyncio task that outlives the HTTP request that started it,
|
||||
so an accidental tab close never abandons half-finished work. Each job keeps
|
||||
its counters, a capped list of file outcomes per outcome group and a capped
|
||||
log, and fans its progress out to SSE subscribers: the request that started
|
||||
it, a reconnecting page, a second tab. A subscriber receives the latest event
|
||||
as a snapshot and then every event published after it, until the job ends.
|
||||
|
||||
Jobs live only in server memory and are gone after a restart. The registry
|
||||
keeps the running jobs and this session's last ``MAX_FINISHED_JOBS`` finished
|
||||
ones. Record-changing jobs (the metadata scan, and later maintenance, cleanup
|
||||
and restore) never run together, and one job of a kind runs at a time.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import itertools
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import PurePosixPath, PureWindowsPath
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
# Start and finish order: wall-clock times can tie (coarse Windows clocks).
|
||||
_sequence = itertools.count()
|
||||
|
||||
JOB_STATES = ("running", "finished", "cancelled", "failed")
|
||||
FINAL_STATES = JOB_STATES[1:]
|
||||
MAX_OUTCOME_ROWS = 200 # rows listed per outcome group; counters stay exact
|
||||
MAX_LOG_LINES = 2000
|
||||
LOG_CAP_LINE = "[Log capped at 2,000 lines]"
|
||||
MAX_FINISHED_JOBS = 20 # finished jobs this session keeps
|
||||
|
||||
# The metadata scan's outcome groups, in counter-tile order of the spec's
|
||||
# mapping, and the per-file scan status each group collects. Files without
|
||||
# usable metadata have no status and go to "no_metadata".
|
||||
# Which files a scan reads (#249 Activity-2): "new" skips the files the Library
|
||||
# already links with a resolved link (get_known_resolved_paths) before reading
|
||||
# them, so new, unlinked, unresolved and ambiguous files are read; "all" reads
|
||||
# every file again. The workspace job defaults to "new"; legacy /scan is "all".
|
||||
SCAN_SCOPES = ("new", "all")
|
||||
DEFAULT_SCAN_SCOPE = "new"
|
||||
SCAN_GROUP_NAMES = ("added", "known", "review", "no_metadata", "errors")
|
||||
SCAN_GROUPS = {
|
||||
"added": "added",
|
||||
"linked": "added",
|
||||
"enriched": "added",
|
||||
"unchanged": "known",
|
||||
"skipped": "known", # "new" scope: known and resolved, not read again
|
||||
"ambiguous": "review",
|
||||
"conflict": "review",
|
||||
"unresolved": "review",
|
||||
"failed": "errors",
|
||||
}
|
||||
|
||||
|
||||
def _relative_path(path) -> str:
|
||||
"""``path`` as a root-relative POSIX path; an absolute one keeps its name."""
|
||||
text = str(path or "")
|
||||
if (
|
||||
PurePosixPath(text).is_absolute()
|
||||
or PureWindowsPath(text).is_absolute()
|
||||
or PureWindowsPath(text).drive
|
||||
):
|
||||
return re.split(r"[\\/]", text)[-1]
|
||||
return text.replace("\\", "/")
|
||||
|
||||
|
||||
class JobBusy(Exception):
|
||||
"""A job could not start because ``running`` holds its slot."""
|
||||
|
||||
def __init__(self, running):
|
||||
super().__init__(f"{running.title} is already running")
|
||||
self.running = running
|
||||
|
||||
|
||||
class Job:
|
||||
"""One tracked operation: state, counters, outcomes, log and subscribers."""
|
||||
|
||||
def __init__(
|
||||
self, kind, title, *, record_changing=False, scope=None, groups=(), clock=None
|
||||
):
|
||||
self._clock = clock or time.time
|
||||
self.id = uuid.uuid4().hex
|
||||
self.kind = kind
|
||||
self.title = title
|
||||
self.record_changing = record_changing
|
||||
self.scope = dict(scope or {})
|
||||
self.state = "running"
|
||||
self.started_at = self._clock()
|
||||
self._started_seq = next(_sequence)
|
||||
self._finished_seq = None
|
||||
self.finished_at = None
|
||||
self.progress = 0
|
||||
self.status_line = ""
|
||||
self.processed = 0
|
||||
self.total = 0
|
||||
self.counters = dict.fromkeys(groups, 0)
|
||||
self.outcomes = {group: [] for group in groups}
|
||||
self.log = []
|
||||
self.cancelled = False
|
||||
self.last_event = None
|
||||
self.task = None
|
||||
self._subscribers = set()
|
||||
|
||||
@property
|
||||
def running(self):
|
||||
return self.state == "running"
|
||||
|
||||
@property
|
||||
def finished(self):
|
||||
return not self.running
|
||||
|
||||
@property
|
||||
def subscriber_count(self):
|
||||
return len(self._subscribers)
|
||||
|
||||
# ── progress ──────────────────────────────────────────────────────
|
||||
|
||||
def update(self, *, progress=None, status_line=None, processed=None, total=None):
|
||||
"""Set the summary's progress fields; ``None`` leaves one unchanged."""
|
||||
if progress is not None:
|
||||
self.progress = max(0, min(100, int(progress)))
|
||||
if status_line is not None:
|
||||
self.status_line = status_line
|
||||
if processed is not None:
|
||||
self.processed = processed
|
||||
if total is not None:
|
||||
self.total = total
|
||||
|
||||
def publish(self, payload):
|
||||
"""Record ``payload`` as the latest event and hand it to every subscriber."""
|
||||
if isinstance(payload.get("progress"), (int, float)):
|
||||
self.update(progress=payload["progress"])
|
||||
self.last_event = payload
|
||||
for queue in list(self._subscribers):
|
||||
queue.put_nowait(payload)
|
||||
|
||||
def log_line(self, text):
|
||||
"""Append a timestamped log line, up to MAX_LOG_LINES and one cap line."""
|
||||
if len(self.log) > MAX_LOG_LINES:
|
||||
return
|
||||
if len(self.log) == MAX_LOG_LINES:
|
||||
self.log.append(LOG_CAP_LINE)
|
||||
return
|
||||
stamp = time.strftime("%H:%M:%S", time.localtime(self._clock()))
|
||||
self.log.append(f"{stamp} {text}")
|
||||
|
||||
def record(self, group, row, log_line):
|
||||
"""Count one file outcome in ``group``, list its row and log it.
|
||||
|
||||
``row`` is ``{path, root, root_index, detail}``: the path relative to
|
||||
its output root, the root's public label and its listing index (the
|
||||
identity Images opens it by; None when unknown). An absolute path is
|
||||
cut to its file name, so a row never reveals the server's layout.
|
||||
"""
|
||||
if group not in self.counters:
|
||||
raise ValueError(f"Unknown outcome group {group!r} for {self.kind}")
|
||||
self.counters[group] += 1
|
||||
raw = str(row.get("path") or "")
|
||||
path = _relative_path(raw)
|
||||
if raw and raw != path:
|
||||
log_line = log_line.replace(raw, path)
|
||||
rows = self.outcomes[group]
|
||||
index = row.get("root_index")
|
||||
if isinstance(index, bool) or not isinstance(index, int) or index < 0:
|
||||
index = None
|
||||
if len(rows) < MAX_OUTCOME_ROWS:
|
||||
rows.append(
|
||||
{
|
||||
"path": path,
|
||||
"root": str(row.get("root") or ""),
|
||||
"root_index": index,
|
||||
"detail": str(row.get("detail") or ""),
|
||||
}
|
||||
)
|
||||
self.log_line(log_line)
|
||||
|
||||
def finish(self, state="finished"):
|
||||
"""End the job in ``state`` and release every subscriber.
|
||||
|
||||
Each subscriber still gets the events queued before the release. A
|
||||
second finish is a no-op, so the first final state stands.
|
||||
"""
|
||||
if state not in FINAL_STATES:
|
||||
raise ValueError(f"{state!r} is not a final job state")
|
||||
if self.finished:
|
||||
return
|
||||
self.state = state
|
||||
self.finished_at = self._clock()
|
||||
self._finished_seq = next(_sequence)
|
||||
for queue in list(self._subscribers):
|
||||
queue.put_nowait(None)
|
||||
|
||||
async def events(self):
|
||||
"""Yield the latest event, then each later one, until the job finishes.
|
||||
|
||||
Consume it under ``contextlib.aclosing`` so a consumer that stops early
|
||||
(a closed socket) drops its queue at once instead of at garbage collection.
|
||||
"""
|
||||
queue = asyncio.Queue()
|
||||
self._subscribers.add(queue)
|
||||
try:
|
||||
if self.last_event is not None:
|
||||
yield self.last_event
|
||||
if self.finished:
|
||||
return
|
||||
while True:
|
||||
payload = await queue.get()
|
||||
if payload is None:
|
||||
return
|
||||
yield payload
|
||||
finally:
|
||||
self._subscribers.discard(queue)
|
||||
|
||||
# ── views ─────────────────────────────────────────────────────────
|
||||
|
||||
def summary(self):
|
||||
"""The job as the Activity list and pill show it."""
|
||||
return {
|
||||
"id": self.id,
|
||||
"kind": self.kind,
|
||||
"title": self.title,
|
||||
"state": self.state,
|
||||
"started_at": self.started_at,
|
||||
"finished_at": self.finished_at,
|
||||
"progress": self.progress,
|
||||
"status_line": self.status_line,
|
||||
"processed": self.processed,
|
||||
"total": self.total,
|
||||
"counters": dict(self.counters),
|
||||
"scope": dict(self.scope),
|
||||
# What detail() lists, so a view re-reads it only when it changed.
|
||||
"listed": {group: len(rows) for group, rows in self.outcomes.items()},
|
||||
"log_length": len(self.log),
|
||||
}
|
||||
|
||||
def detail(self):
|
||||
"""The summary plus the listed outcome rows and the log."""
|
||||
return {
|
||||
**self.summary(),
|
||||
"outcomes": {
|
||||
group: [dict(row) for row in rows]
|
||||
for group, rows in self.outcomes.items()
|
||||
},
|
||||
"log": list(self.log),
|
||||
}
|
||||
|
||||
def report(self):
|
||||
"""The downloadable report: the summary, every outcome group and the log.
|
||||
|
||||
Each group carries its exact count, its listed rows (at most
|
||||
MAX_OUTCOME_ROWS) and whether the list is cut short. Rows and log lines
|
||||
already hold root-relative paths only.
|
||||
"""
|
||||
summary = self.summary()
|
||||
summary.pop("listed", None)
|
||||
summary.pop("log_length", None)
|
||||
return {
|
||||
"report_version": 1,
|
||||
"generated_at": self._clock(),
|
||||
"job": summary,
|
||||
"outcomes": {
|
||||
group: {
|
||||
"count": self.counters.get(group, len(rows)),
|
||||
"rows": [dict(row) for row in rows],
|
||||
"truncated": self.counters.get(group, len(rows)) > len(rows),
|
||||
}
|
||||
for group, rows in self.outcomes.items()
|
||||
},
|
||||
"log": list(self.log),
|
||||
}
|
||||
|
||||
def report_filename(self):
|
||||
"""The download name: prompt-manager-KIND-YYYYMMDD-HHMM.json (local time)."""
|
||||
stamp = time.strftime("%Y%m%d-%H%M", time.localtime(self._clock()))
|
||||
return f"prompt-manager-{self.kind}-{stamp}.json"
|
||||
|
||||
|
||||
class JobRegistry:
|
||||
"""Every job of this server session, running and recently finished."""
|
||||
|
||||
def __init__(self, logger_=None):
|
||||
self._logger = logger_ or logger
|
||||
self._jobs = {}
|
||||
|
||||
def start(self, kind, title, *, record_changing, scope, runner, groups=()) -> Job:
|
||||
"""Create a job and run ``runner(job)`` as a task.
|
||||
|
||||
Raises JobBusy, naming the job in the way, when a job of ``kind`` is
|
||||
running or when ``record_changing`` and another record-changing job
|
||||
runs. Must be called from the event loop.
|
||||
"""
|
||||
same_kind = self.active(kind)
|
||||
if same_kind is not None:
|
||||
raise JobBusy(same_kind)
|
||||
if record_changing:
|
||||
for job in self._jobs.values():
|
||||
if job.running and job.record_changing:
|
||||
raise JobBusy(job)
|
||||
self._prune()
|
||||
job = Job(
|
||||
kind, title, record_changing=record_changing, scope=scope, groups=groups
|
||||
)
|
||||
self._jobs[job.id] = job
|
||||
job.task = asyncio.ensure_future(self._run(job, runner))
|
||||
return job
|
||||
|
||||
async def _run(self, job, runner):
|
||||
try:
|
||||
await runner(job)
|
||||
except asyncio.CancelledError:
|
||||
job.finish("cancelled")
|
||||
raise
|
||||
except Exception:
|
||||
self._logger.exception(f"Job {job.kind} failed")
|
||||
job.finish("failed")
|
||||
finally:
|
||||
if job.running:
|
||||
job.finish("cancelled" if job.cancelled else "finished")
|
||||
|
||||
def _prune(self):
|
||||
"""Drop the oldest finished jobs so a new one keeps the cap."""
|
||||
finished = [job for job in self._jobs.values() if job.finished]
|
||||
finished.sort(key=lambda job: job._finished_seq)
|
||||
for job in finished[: max(0, len(finished) - MAX_FINISHED_JOBS + 1)]:
|
||||
del self._jobs[job.id]
|
||||
|
||||
def get(self, job_id):
|
||||
return self._jobs.get(job_id)
|
||||
|
||||
def active(self, kind):
|
||||
"""The running job of ``kind``, or None."""
|
||||
for job in self._jobs.values():
|
||||
if job.kind == kind and job.running:
|
||||
return job
|
||||
return None
|
||||
|
||||
def list(self):
|
||||
"""Running jobs first, then the last finished ones, newest first."""
|
||||
jobs = list(self._jobs.values())
|
||||
running = [job for job in jobs if job.running]
|
||||
finished = [job for job in jobs if job.finished]
|
||||
running.sort(key=lambda job: job._started_seq, reverse=True)
|
||||
finished.sort(key=lambda job: job._finished_seq, reverse=True)
|
||||
return running + finished[:MAX_FINISHED_JOBS]
|
||||
|
||||
def cancel(self, job_id) -> bool:
|
||||
"""Ask a running job to stop at its next safe boundary.
|
||||
|
||||
Returns whether a cancellation was requested: False for a finished or
|
||||
unknown job, so a cancel after the end is a no-op.
|
||||
"""
|
||||
job = self._jobs.get(job_id)
|
||||
if job is None or job.finished:
|
||||
return False
|
||||
job.cancelled = True
|
||||
return True
|
||||
@@ -1,62 +0,0 @@
|
||||
"""Progress fan-out for the background output scan.
|
||||
|
||||
The scan runs as an asyncio task that outlives the HTTP request that started
|
||||
it, so an accidental tab close does not abandon a half-finished scan. Every
|
||||
SSE subscriber (the request that started the scan, a reconnecting page, a
|
||||
second tab) receives the latest event as a snapshot and then every event
|
||||
published after it, until the job finishes.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
|
||||
class ScanJob:
|
||||
"""One scan run: its latest event, completion flag and live subscribers."""
|
||||
|
||||
def __init__(self):
|
||||
self.last_event = None
|
||||
self.finished = False
|
||||
self.cancelled = False
|
||||
self.task = None
|
||||
self._subscribers = set()
|
||||
|
||||
@property
|
||||
def running(self):
|
||||
return not self.finished
|
||||
|
||||
@property
|
||||
def subscriber_count(self):
|
||||
return len(self._subscribers)
|
||||
|
||||
def publish(self, payload):
|
||||
"""Record ``payload`` as the latest event and hand it to every subscriber."""
|
||||
self.last_event = payload
|
||||
for queue in list(self._subscribers):
|
||||
queue.put_nowait(payload)
|
||||
|
||||
def finish(self):
|
||||
"""Mark the job done and release every subscriber after its queued events."""
|
||||
self.finished = True
|
||||
for queue in list(self._subscribers):
|
||||
queue.put_nowait(None)
|
||||
|
||||
async def events(self):
|
||||
"""Yield the latest event, then each later one, until the job finishes.
|
||||
|
||||
Consume it under ``contextlib.aclosing`` so a consumer that stops early
|
||||
(a closed socket) drops its queue at once instead of at garbage collection.
|
||||
"""
|
||||
queue = asyncio.Queue()
|
||||
self._subscribers.add(queue)
|
||||
try:
|
||||
if self.last_event is not None:
|
||||
yield self.last_event
|
||||
if self.finished:
|
||||
return
|
||||
while True:
|
||||
payload = await queue.get()
|
||||
if payload is None:
|
||||
return
|
||||
yield payload
|
||||
finally:
|
||||
self._subscribers.discard(queue)
|
||||
@@ -0,0 +1,296 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { THEMES, seedConcept, openConcept, openProduction, diffInventory, mutate } from './concept-inventory.mjs';
|
||||
|
||||
// Activity-1 Task 3. The review dialog is compared with the concept's own `scan-review` ("Review metadata scan",
|
||||
// app.js 1428–1432) and its backup box with the concept's System `.notice`; the differences are D55. The job view
|
||||
// has no concept counterpart: it is compared with the approved mockup `activity-scan.html` (a local, ignored file),
|
||||
// whose structure and CSS values are restated below (D56). Each comparison is shown to notice a mutation.
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
page.conceptErrors = [];
|
||||
page.on('pageerror', error => page.conceptErrors.push(error.message));
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, request, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
expect(page.conceptErrors).toEqual([]);
|
||||
await request.post('/__test__/jobs/restart');
|
||||
});
|
||||
|
||||
/** The open review dialog: its body's element sequence, the meta grid, the sentence, the notice and the actions. */
|
||||
function measureReview() {
|
||||
const dialog = document.querySelector('dialog#modal[open]');
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
const name = node => [node.localName, ...[...node.classList].sort()].join('.');
|
||||
const grid = dialog.querySelector('#modal-body > .meta-grid');
|
||||
const sentence = dialog.querySelector('#modal-body > p.small');
|
||||
const actions = dialog.querySelector('#modal-body > .dialog-actions');
|
||||
return {
|
||||
title: dialog.querySelector('#modal-title').textContent,
|
||||
body: [...dialog.querySelector('#modal-body').children].map(name),
|
||||
grid: {
|
||||
tag: grid.localName,
|
||||
...css(grid, 'display', 'gridTemplateColumns', 'columnGap', 'rowGap', 'marginTop', 'marginBottom', 'fontSize'),
|
||||
entries: [...grid.children].map(entry => ({
|
||||
tag: entry.localName, dt: entry.querySelector('dt').textContent, dd: entry.querySelector('dd').textContent,
|
||||
dtStyle: css(entry.querySelector('dt'), 'color', 'fontSize'),
|
||||
ddStyle: css(entry.querySelector('dd'), 'marginTop', 'marginLeft', 'fontSize', 'color'),
|
||||
})),
|
||||
},
|
||||
sentence: { text: sentence.textContent, ...css(sentence, 'fontSize', 'color', 'lineHeight', 'marginTop') },
|
||||
actions: {
|
||||
...css(actions, 'display', 'justifyContent', 'columnGap', 'marginTop'),
|
||||
buttons: [...actions.children].map(button => ({ text: button.textContent, className: button.className,
|
||||
...css(button, 'backgroundColor', 'color', 'fontSize', 'paddingTop', 'paddingLeft', 'borderTopLeftRadius') })),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** A `.notice` box: its frame and its text and actions columns, without the copy. */
|
||||
function measureNotice(selector) {
|
||||
const notice = document.querySelector(selector);
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
return {
|
||||
frame: css(notice, 'display', 'alignItems', 'justifyContent', 'columnGap', 'borderTopWidth', 'borderTopStyle',
|
||||
'borderTopColor', 'backgroundColor', 'borderTopLeftRadius', 'paddingTop', 'paddingLeft'),
|
||||
columns: [...notice.children].map(node => node.localName + (node.className ? `.${node.className}` : '')),
|
||||
text: css(notice.querySelector(':scope > div > p'), 'fontSize', 'color', 'marginTop'),
|
||||
actions: css(notice.querySelector(':scope > .row-actions'), 'display', 'flexShrink'),
|
||||
};
|
||||
}
|
||||
|
||||
/** A concept `.setting-row` switch: its frame, its label and hint, and its checkbox (not its place-dependent margin). */
|
||||
function measureToggle(selector) {
|
||||
const row = document.querySelector(selector);
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
return {
|
||||
children: [...row.children].map(node => [node.localName, ...node.classList].join('.')),
|
||||
frame: css(row, 'display', 'alignItems', 'justifyContent', 'columnGap', 'paddingTop', 'paddingBottom',
|
||||
'borderTopWidth', 'borderTopStyle', 'borderTopColor'),
|
||||
label: css(row.querySelector('label'), 'fontSize', 'fontWeight', 'color'),
|
||||
hint: css(row.querySelector('p'), 'fontSize', 'color', 'marginTop'),
|
||||
box: css(row.querySelector('input[type="checkbox"]'), 'accentColor', 'width', 'height', 'appearance',
|
||||
'backgroundColor', 'borderTopLeftRadius'),
|
||||
role: row.querySelector('input').getAttribute('role'),
|
||||
labelled: row.querySelector('label').htmlFor === row.querySelector('input').id,
|
||||
};
|
||||
}
|
||||
|
||||
/** D55 and D58: the concept's dialog with the production differences applied. */
|
||||
function mapReviewConcept(reference) {
|
||||
return {
|
||||
...reference,
|
||||
// No sample-plan note (prototype copy); the D58 "All files" toggle follows the grid; the backup box sits
|
||||
// before the actions.
|
||||
body: [...reference.body.filter(item => item !== 'p.demo-note').slice(0, -1), 'div.notice', reference.body.at(-1)]
|
||||
.flatMap(item => item === 'div.meta-grid' ? ['dl.meta-grid', 'div.setting-row'] : [item]),
|
||||
// A `dl`, as in the concept's maintenance plan; real sources. The scope keeps the concept's
|
||||
// "New and unresolved files" until "All files" is checked.
|
||||
grid: { ...reference.grid, tag: 'dl', entries: reference.grid.entries.map(entry => ({ ...entry,
|
||||
dd: { Sources: '2 output folders', Scope: 'New and unresolved files' }[entry.dt] })) },
|
||||
actions: { ...reference.actions, buttons: reference.actions.buttons.map(button => button.text === 'Simulate scan'
|
||||
? { ...button, text: 'Start scan' } : button) },
|
||||
};
|
||||
}
|
||||
|
||||
async function conceptReview(page, theme) {
|
||||
await openConcept(page, 'images', theme);
|
||||
await page.locator('#main .heading-actions').getByRole('button', { name: 'Import', exact: true }).click();
|
||||
await page.locator('dialog#modal[open] .operation').filter({ hasText: 'Scan an output folder' })
|
||||
.getByRole('button', { name: 'Review scan', exact: true }).click();
|
||||
await expect(page.locator('dialog#modal[open] #modal-title')).toHaveText('Review metadata scan');
|
||||
await page.mouse.move(0, 0);
|
||||
const review = await page.evaluate(measureReview);
|
||||
await page.keyboard.press('Escape');
|
||||
// The concept's System → Sources notice ("Bring older images into your prompt history").
|
||||
await page.goto(`/__test__/canonical/workspace#system`);
|
||||
await page.getByRole('combobox', { name: 'Theme', exact: true }).selectOption(theme);
|
||||
await page.locator('#main').getByRole('button', { name: 'Manage sources', exact: true }).click();
|
||||
await expect(page.locator('#main .notice')).toBeVisible();
|
||||
return { review, notice: await page.evaluate(measureNotice, '#main .notice'),
|
||||
toggle: await page.evaluate(measureToggle, '#main .setting-row:has(input[type="checkbox"])') };
|
||||
}
|
||||
|
||||
async function productionReview(page, theme) {
|
||||
await openProduction(page, 'images', theme);
|
||||
await page.locator('#workspace-main .heading-actions').getByRole('button', { name: 'Import', exact: true }).click();
|
||||
await page.locator('dialog#modal[open] .operation').filter({ hasText: 'Scan for metadata' })
|
||||
.getByRole('button', { name: 'Review scan', exact: true }).click();
|
||||
await expect(page.locator('dialog#modal[open] #modal-title')).toHaveText('Review metadata scan');
|
||||
// The swapped dialog can open under the pointer that pressed Review scan; measure it at rest.
|
||||
await page.mouse.move(0, 0);
|
||||
const review = await page.evaluate(measureReview);
|
||||
const notice = await page.evaluate(measureNotice, 'dialog#modal[open] .notice');
|
||||
// The concept's switch is on; compare ours on too, then turn it off again (the default).
|
||||
const allFiles = page.getByRole('switch', { name: 'All files', exact: true });
|
||||
await allFiles.check();
|
||||
await page.mouse.move(0, 0);
|
||||
const toggle = await page.evaluate(measureToggle, 'dialog#modal[open] .setting-row');
|
||||
await allFiles.uncheck();
|
||||
await page.mouse.move(0, 0);
|
||||
return { review, notice, toggle };
|
||||
}
|
||||
|
||||
for (const theme of THEMES) test(`review dialog matches the concept scan-review ${theme}`, async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
await seedConcept(request);
|
||||
const concept = await conceptReview(page, theme);
|
||||
expect(concept.review.body).toEqual(['p.demo-note', 'div.meta-grid', 'p.small', 'div.dialog-actions']);
|
||||
const production = await productionReview(page, theme);
|
||||
expect(diffInventory(mapReviewConcept(concept.review), production.review)).toEqual([]);
|
||||
expect(diffInventory(concept.notice, production.notice)).toEqual([]);
|
||||
expect(diffInventory(concept.toggle, production.toggle)).toEqual([]);
|
||||
await expect(page.locator('dialog#modal[open] .setting-row > div > label')).toHaveText('All files');
|
||||
await expect(page.locator('dialog#modal[open] .setting-row input')).not.toBeChecked();
|
||||
await expect(page.getByRole('switch', { name: 'All files', exact: true })).toBeVisible();
|
||||
await expect(page.locator('dialog#modal[open] .notice h3')).toHaveText('Back up the database first (recommended)');
|
||||
await expect(page.locator('dialog#modal[open] .notice .row-actions button')).toHaveText(['Back up now']);
|
||||
|
||||
// Mutation check: one CSS change and one removed element are both noticed, then undone to an exact match again.
|
||||
const undoCss = await mutate(page, { css: '#modal .meta-grid { gap: 3px; } #modal .notice { padding: 2px; } #modal .setting-row { gap: 3px; }' });
|
||||
const cssMutated = await page.evaluate(measureReview);
|
||||
const noticeMutated = await page.evaluate(measureNotice, 'dialog#modal[open] .notice');
|
||||
await page.getByRole('switch', { name: 'All files', exact: true }).check();
|
||||
await page.mouse.move(0, 0);
|
||||
const toggleMutated = await page.evaluate(measureToggle, 'dialog#modal[open] .setting-row');
|
||||
await page.getByRole('switch', { name: 'All files', exact: true }).uncheck();
|
||||
await page.mouse.move(0, 0);
|
||||
expect(diffInventory(concept.toggle, toggleMutated).map(item => item.path)).toEqual(['frame.columnGap']);
|
||||
expect(diffInventory(mapReviewConcept(concept.review), cssMutated).map(item => item.path)).toEqual(expect.arrayContaining(['grid.columnGap', 'grid.rowGap']));
|
||||
expect(diffInventory(concept.notice, noticeMutated).map(item => item.path)).toEqual(['frame.paddingTop', 'frame.paddingLeft']);
|
||||
await undoCss();
|
||||
const undoDom = await mutate(page, { remove: ['#modal-body > p.small'] });
|
||||
const domMutated = await page.evaluate(() => [...document.querySelectorAll('#modal-body > *')].map(node => node.localName));
|
||||
expect(domMutated).toEqual(['dl', 'div', 'div', 'div']);
|
||||
await undoDom();
|
||||
expect(diffInventory(mapReviewConcept(concept.review), await page.evaluate(measureReview))).toEqual([]);
|
||||
});
|
||||
|
||||
// The approved mockup's job view, restated: structure first, then the values of its CSS rules.
|
||||
const MOCKUP_TREE = ['div.heading', 'section.surface', 'section.surface', 'details.term'];
|
||||
const MOCKUP_PROGRESS = ['div.panel-heading(h2,span.small)', 'progress', 'dl.stat-row(div.added.stat,div.stat,div.stat.warn,div.stat,div.bad.stat)'];
|
||||
const MOCKUP_RESULTS = ['div.panel-heading(h2,label.search-field)', 'div.outcomes(div.outcome-list,div.outcome-files)'];
|
||||
const MOCKUP_STYLES = {
|
||||
statRow: { display: 'grid', columnGap: '10px', columns: 5, marginTop: '16px', marginBottom: '16px' },
|
||||
stat: { borderTopWidth: '1px', borderTopLeftRadius: '7px', paddingTop: '12px', paddingLeft: '14px' },
|
||||
dt: { fontSize: '11px' },
|
||||
dd: { fontSize: '20px', fontWeight: '600', marginTop: '6px', marginLeft: '0px' },
|
||||
outcomes: { display: 'grid', columnGap: '16px', first: '220px', marginTop: '6px' },
|
||||
outcomeButton: { display: 'flex', justifyContent: 'space-between', marginBottom: '6px', textAlign: 'left' },
|
||||
fileRow: { display: 'grid', columnGap: '12px', paddingTop: '9px', paddingLeft: '4px', borderBottomWidth: '1px', fontSize: '12px' },
|
||||
path: { monospace: true, overflowWrap: 'anywhere' },
|
||||
search: { maxWidth: '320px' },
|
||||
term: { marginTop: '16px', borderTopWidth: '1px', borderTopLeftRadius: '7px', overflow: 'hidden' },
|
||||
summary: { paddingTop: '11px', paddingLeft: '14px', fontSize: '12px', cursor: 'pointer' },
|
||||
pre: { marginTop: '0px', paddingTop: '12px', paddingLeft: '14px', maxHeight: '180px', fontSize: '11px', lineHeight: '18.7px', overflow: 'auto' },
|
||||
};
|
||||
|
||||
function measureJobView() {
|
||||
const main = document.querySelector('#workspace-main');
|
||||
const css = (node, ...keys) => Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]]));
|
||||
const name = node => [node.localName, ...[...node.classList].sort()].join('.');
|
||||
// The rows and sections the mockup nests are listed with their children.
|
||||
const shape = node => node.matches('.panel-heading, .outcomes, dl.stat-row')
|
||||
? `${name(node)}(${[...node.children].map(name).join(',')})` : name(node);
|
||||
const [progress, results] = main.querySelectorAll(':scope > section.surface');
|
||||
const statRow = progress.querySelector('dl.stat-row');
|
||||
const stat = statRow.querySelector('.stat');
|
||||
const outcomes = results.querySelector('.outcomes');
|
||||
const button = outcomes.querySelector('.outcome-list > button');
|
||||
const row = outcomes.querySelector('.file-row');
|
||||
const term = main.querySelector(':scope > details.term');
|
||||
const pre = term.querySelector('pre');
|
||||
const columns = getComputedStyle(statRow).gridTemplateColumns.split(' ');
|
||||
return {
|
||||
tree: [...main.children].map(name),
|
||||
progress: [...progress.children].map(shape),
|
||||
results: [...results.children].map(shape),
|
||||
h2: [...main.querySelectorAll(':scope > section.surface > .panel-heading > h2')].map(node => node.textContent),
|
||||
summary: term.querySelector('summary').textContent,
|
||||
styles: {
|
||||
statRow: { ...css(statRow, 'display', 'columnGap', 'marginTop', 'marginBottom'), columns: columns.length },
|
||||
stat: css(stat, 'borderTopWidth', 'borderTopLeftRadius', 'paddingTop', 'paddingLeft'),
|
||||
dt: css(stat.querySelector('dt'), 'fontSize'),
|
||||
dd: css(stat.querySelector('dd'), 'fontSize', 'fontWeight', 'marginTop', 'marginLeft'),
|
||||
outcomes: { ...css(outcomes, 'display', 'columnGap', 'marginTop'), first: getComputedStyle(outcomes).gridTemplateColumns.split(' ')[0] },
|
||||
outcomeButton: css(button, 'display', 'justifyContent', 'marginBottom', 'textAlign'),
|
||||
fileRow: css(row, 'display', 'columnGap', 'paddingTop', 'paddingLeft', 'borderBottomWidth', 'fontSize'),
|
||||
path: { monospace: /monospace/.test(getComputedStyle(row.querySelector('.path')).fontFamily),
|
||||
overflowWrap: getComputedStyle(row.querySelector('.path')).overflowWrap },
|
||||
search: css(results.querySelector('.search-field'), 'maxWidth'),
|
||||
term: css(term, 'marginTop', 'borderTopWidth', 'borderTopLeftRadius', 'overflow'),
|
||||
summary: css(term.querySelector('summary'), 'paddingTop', 'paddingLeft', 'fontSize', 'cursor'),
|
||||
// WebKit reports the 1.7 line height of 11px as 18.700001px; compare it to the hundredth of a pixel.
|
||||
pre: { ...css(pre, 'marginTop', 'paddingTop', 'paddingLeft', 'maxHeight', 'fontSize', 'overflow'),
|
||||
lineHeight: `${Math.round(parseFloat(getComputedStyle(pre).lineHeight) * 100) / 100}px` },
|
||||
},
|
||||
// Mockup colours: the pressed outcome takes the accent border on the soft blue, the tiles their tone.
|
||||
pressed: (() => {
|
||||
const probe = document.createElement('span');
|
||||
main.append(probe);
|
||||
const tone = name => { probe.style.color = `var(${name})`; return getComputedStyle(probe).color; };
|
||||
const values = { accent: tone('--ca-accent'), soft: tone('--soft-blue'), green: tone('--green'), red: tone('--red'), warning: tone('--warning') };
|
||||
probe.remove();
|
||||
const on = outcomes.querySelector('.outcome-list > button[aria-pressed="true"]');
|
||||
const dd = selector => getComputedStyle(statRow.querySelector(`${selector} dd`)).color;
|
||||
return { border: getComputedStyle(on).borderTopColor === values.accent, background: getComputedStyle(on).backgroundColor === values.soft,
|
||||
added: dd('.stat.added') === values.green, bad: dd('.stat.bad') === values.red, warn: dd('.stat.warn') === values.warning };
|
||||
})(),
|
||||
};
|
||||
}
|
||||
|
||||
const EXPECTED_JOB_VIEW = {
|
||||
tree: MOCKUP_TREE, progress: MOCKUP_PROGRESS, results: MOCKUP_RESULTS, h2: ['Progress', 'Results'],
|
||||
summary: '● Scan log — raw detail, click to show', styles: MOCKUP_STYLES,
|
||||
pressed: { border: true, background: true, added: true, bad: true, warn: true },
|
||||
};
|
||||
|
||||
async function openRunningJob(page, request, theme, viewport = { width: 1600, height: 1100 }) {
|
||||
await page.setViewportSize(viewport);
|
||||
const started = await request.post('/__test__/jobs/start', { data: { total: 19622 } });
|
||||
await expect(started).toBeOK();
|
||||
const { job } = await started.json();
|
||||
await expect(await request.post(`/__test__/jobs/${job.id}/step`, { data: { processed: 8240,
|
||||
counters: { known: 6511, review: 37, no_metadata: 254, errors: 18 },
|
||||
records: [{ group: 'added', path: 'CCC/mabes9sans-dataset-v1/IMG_GEN_00369_.png', detail: 'Added to the Library as “Mabes portrait”' }] } })).toBeOK();
|
||||
await openProduction(page, `activity/${job.id}`, theme);
|
||||
await expect(page.locator('#workspace-main .file-row')).toHaveCount(1);
|
||||
return job;
|
||||
}
|
||||
|
||||
for (const theme of THEMES) test(`job view follows the approved mockup ${theme}`, async ({ page, request }) => {
|
||||
await seedConcept(request);
|
||||
await openRunningJob(page, request, theme);
|
||||
expect(await page.evaluate(measureJobView)).toEqual(EXPECTED_JOB_VIEW);
|
||||
|
||||
// Mutation check: a CSS change (tile gap) and a removed element (the outcome list) are both noticed.
|
||||
const undoCss = await mutate(page, { css: '#workspace-main .stat-row { gap: 3px; }' });
|
||||
expect(diffInventory(EXPECTED_JOB_VIEW, await page.evaluate(measureJobView)).map(item => item.path)).toEqual(['styles.statRow.columnGap']);
|
||||
await undoCss();
|
||||
const undoDom = await mutate(page, { remove: ['#workspace-main details.term'] });
|
||||
expect(await page.evaluate(() => [...document.querySelector('#workspace-main').children].map(node => node.localName)))
|
||||
.toEqual(['div', 'section', 'section']);
|
||||
await undoDom();
|
||||
expect(await page.evaluate(measureJobView)).toEqual(EXPECTED_JOB_VIEW);
|
||||
});
|
||||
|
||||
test('job view fits a phone without page scroll', async ({ page, request }) => {
|
||||
await seedConcept(request);
|
||||
await openRunningJob(page, request, 'dark', { width: 390, height: 844 });
|
||||
const layout = await page.evaluate(() => {
|
||||
const main = document.querySelector('#workspace-main');
|
||||
const columns = node => getComputedStyle(node).gridTemplateColumns.split(' ').length;
|
||||
return {
|
||||
scroll: document.documentElement.scrollWidth <= document.documentElement.clientWidth,
|
||||
tiles: columns(main.querySelector('dl.stat-row')),
|
||||
outcomes: columns(main.querySelector('.outcomes')),
|
||||
inside: [...main.querySelectorAll('.stat, .outcome-list button, .file-row, .search-field')]
|
||||
.every(node => node.getBoundingClientRect().right <= main.getBoundingClientRect().right + 0.5),
|
||||
};
|
||||
});
|
||||
expect(layout).toEqual({ scroll: true, tiles: 2, outcomes: 1, inside: true });
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
|
||||
test.beforeEach(async ({page, browserName}) => {
|
||||
@@ -57,7 +57,8 @@ test('Classic keeps immediate selection, search query and metadata serialization
|
||||
});
|
||||
|
||||
test('Classic detaches native text synchronization on removal',async({page})=>{
|
||||
await page.waitForTimeout(60);
|
||||
// The page installs the text sync (serializeValue and its listeners) in a 50 ms timer; wait for it, not for time.
|
||||
await expect.poll(()=>page.evaluate(()=>typeof node.widgets.find(item=>item.name==='text').serializeValue)).toBe('function');
|
||||
const values=await page.evaluate(()=>{
|
||||
const widget=node.widgets.find(item=>item.name==='text');
|
||||
widget.inputEl.value='Before removal';widget.inputEl.dispatchEvent(new Event('input'));
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// same concept selectors in the immutable concept (/__test__/canonical/workspace) and in production, which now renders
|
||||
// the concept's classes. The concept is never rewritten: each page's map<Page>Concept(reference) applies only the
|
||||
// recorded spec-§8 deviations, each commented with its D-number, and diffInventory must then report nothing.
|
||||
import {expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {expect} from './fixtures.mjs';
|
||||
export {diffInventory} from './library-editor-canonical-comparison.mjs';
|
||||
|
||||
export const CONCEPT_URL='/__test__/canonical/workspace';
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
// Shared `test` for the page workflows: every Playwright worker owns one
|
||||
// tests/browser/server.py process on a port the OS chose for it, with its own temp root
|
||||
// and database. `baseURL` (and so `page`, `request` and `browser.newContext`)
|
||||
// points at that worker's server. A server that dies fails the running test
|
||||
// with its stderr tail, and the next test in the worker gets a fresh server.
|
||||
// A test that fails gets the stderr its server wrote during it attached as
|
||||
// `server-stderr`.
|
||||
import { test as base, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { attachServerStderr, createServerSupervisor, startBrowserServer } from '../../tools/vendor/start-browser-server.mjs';
|
||||
|
||||
const SERVER_FIXTURE_TIMEOUT_MS = 90_000;
|
||||
|
||||
export const test = base.extend({
|
||||
testServer: [async ({}, use) => {
|
||||
const supervisor = createServerSupervisor({ start: () => startBrowserServer() });
|
||||
try {
|
||||
await use(supervisor);
|
||||
} finally {
|
||||
await supervisor.stop();
|
||||
}
|
||||
}, { scope: 'worker', timeout: SERVER_FIXTURE_TIMEOUT_MS }],
|
||||
workerServer: [async ({ testServer }, use, testInfo) => {
|
||||
const server = await testServer.ensure();
|
||||
const mark = server.stderrMark();
|
||||
await use(server);
|
||||
await attachServerStderr(testInfo, server, mark);
|
||||
const crash = testServer.crash(server);
|
||||
if (crash) throw crash;
|
||||
}, { auto: true, timeout: SERVER_FIXTURE_TIMEOUT_MS }],
|
||||
baseURL: async ({ workerServer }, use) => use(workerServer.url),
|
||||
});
|
||||
|
||||
export { expect };
|
||||
@@ -1,5 +1,5 @@
|
||||
import {readFileSync} from 'node:fs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
|
||||
for (const variant of ['Focused', 'Split']) {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept, openConcept, openProduction, SIZES } from './concept-inventory.mjs';
|
||||
// D10/D20: the heading is the linked positive's title (its confirmed first line here), never the pair's
|
||||
// "Inspector recipe", which names the negative variant.
|
||||
@@ -804,6 +804,7 @@ test('slideshow preference failures leave native playback usable',async({page,re
|
||||
});
|
||||
|
||||
test('full inspector attaches matched theme and mobile evidence with reachable content',async({page,request},testInfo)=>{
|
||||
test.setTimeout(120_000); // 32 evidence captures across four viewports, like the concept comparisons
|
||||
await expect(await request.post('/__test__/reset',{data:{full_inspector:true,include_video:true}})).toBeOK();
|
||||
await page.goto('/prompt_manager/workspace#/images');
|
||||
const measurements=[];
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { writeFile } from 'node:fs/promises';
|
||||
import { expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { expect } from './fixtures.mjs';
|
||||
|
||||
// Content, row counts, and image bytes come from independent synthetic datasets.
|
||||
// Saved choices and actual executions are separate production sections. The concept's
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,diffInventory,mutate,stripExact} from './concept-inventory.mjs';
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,diffInventory,mutate,stripExact,countCopy,reflow,withoutItems} from './concept-inventory.mjs';
|
||||
|
||||
@@ -7,7 +7,7 @@ import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,
|
||||
// images of the concept's six prompts, its seeds, sizes and favorites). Heading, toolbar, filter row, caption, grid,
|
||||
// the first six cards, the continuation and the panel box are compared at 1600x1100 and 390x844 in both themes, and the
|
||||
// page's behaviors under normal and reduced motion; grid, cards and panel also at 1200x900 and 1800x1100 (2 and 4
|
||||
// columns, the 275px and 310px panel). The topbar and sidebar (D5, D28, D29) are the shell spec's.
|
||||
// columns, the 275px and 310px panel). The topbar and sidebar (D28, D29, D52, D53) are the shell spec's.
|
||||
// Paired captures are artifacts only; there are no PNG baselines.
|
||||
test.use({locale:'en-US',timezoneId:'UTC'});
|
||||
|
||||
@@ -149,12 +149,15 @@ function compareLoadingRow(reference,actual,count) {
|
||||
return diffInventory(unplaced(mapContinuation(reference,{loaded,total})),unplaced(actual));
|
||||
}
|
||||
|
||||
/** D15: the Import dialog's two real operations, with the ruled intro and the Rescan wording. */
|
||||
/**
|
||||
* D15: the Import dialog's real operations, with the ruled intro and the Rescan wording. D55: the concept's own "Scan an
|
||||
* output folder → Review scan" row follows Rescan as "Scan for metadata", opening the concept's "Review metadata scan".
|
||||
*/
|
||||
function mapImportConcept(reference) {
|
||||
return {...reference,
|
||||
intro:{...reference.intro,text:'Image inspection and output scanning have different effects. Choose the workflow you want.'},
|
||||
rows:reference.rows.filter(row=>row.title!=='Import portable recipes').map(row=>row.title==='Scan an output folder'
|
||||
?{...row,description:'Find files added or removed in the output folders.',buttons:['Rescan']}:row)};
|
||||
rows:reference.rows.filter(row=>row.title!=='Import portable recipes').flatMap(row=>row.title==='Scan an output folder'
|
||||
?[{...row,description:'Find files added or removed in the output folders.',buttons:['Rescan']},{...row,title:'Scan for metadata'}]:[row])};
|
||||
}
|
||||
|
||||
const SECTIONS=['heading','toolbar','filterRow','bulkbar','caption','grid','cards','continuation','panel','noPageScroll'];
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {SIZES,THEMES,seedConcept,openConcept,openProduction,inspectorOf,diffInventory,mutate,countCopy} from './concept-inventory.mjs';
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,diffInventory,mutate,stripExact,countCopy,reflow} from './concept-inventory.mjs';
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
|
||||
// Artifact-only captures of the editor states that have no concept counterpart (archive confirmation, Show archived,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {readFileSync} from 'node:fs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
|
||||
const pair={id:4,positive_text:'Library positive',negative_text:'Library negative',category:'Portrait',tags:['studio']};
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
|
||||
const moduleUrl = '/prompt_manager/js/shared/toolkit/prompt-bridge.js';
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
|
||||
const workspace='/prompt_manager/workspace#/images';
|
||||
const details=page=>page.getByRole('complementary',{name:'Selected image details',exact:true});
|
||||
|
||||
+78
-6
@@ -1,19 +1,24 @@
|
||||
"""Serve production pages and API routes with disposable state for browser tests."""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import types
|
||||
|
||||
from aiohttp import web
|
||||
from aiohttp.web_runner import GracefulExit
|
||||
from PIL import Image
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
|
||||
from tests.browser.workspace_activity_fixture import register_activity_commands
|
||||
from tests.browser.workspace_concept_fixture import (
|
||||
MAX_EXTRA_IMAGES as CONCEPT_MAX_EXTRA_IMAGES,
|
||||
MAX_SEEDLESS_IMAGES as CONCEPT_MAX_SEEDLESS_IMAGES,
|
||||
@@ -159,6 +164,7 @@ def create_app(root: Path) -> web.Application:
|
||||
library_editor_commands = register_library_editor_commands(routes, api, root)
|
||||
register_vocabulary_commands(routes, api, root)
|
||||
reveal_stub = register_reveal_stub(routes, api, root)
|
||||
register_activity_commands(app, routes, api)
|
||||
|
||||
async def canonical_asset(request: web.Request) -> web.Response:
|
||||
filename, content_type = CANONICAL_ASSETS[request.path]
|
||||
@@ -183,6 +189,9 @@ def create_app(root: Path) -> web.Application:
|
||||
for table in (
|
||||
"saved_views",
|
||||
"image_favorites",
|
||||
"media_image_tags",
|
||||
"media_annotation_bytes",
|
||||
"media_annotation_files",
|
||||
"image_prompt_pairs",
|
||||
"prompt_pair_tags",
|
||||
"prompt_pairs",
|
||||
@@ -513,14 +522,77 @@ def create_app(root: Path) -> web.Application:
|
||||
return app
|
||||
|
||||
|
||||
def main() -> None:
|
||||
HOST = "127.0.0.1"
|
||||
PORT_ANNOUNCEMENT = "PM_TEST_SERVER_PORT="
|
||||
|
||||
|
||||
def bind_listener(port: int) -> socket.socket:
|
||||
"""Bind the server's socket before the app is built.
|
||||
|
||||
``port`` 0 lets the OS choose a free port, which this process then holds,
|
||||
so parallel launchers never race for one between choosing and binding.
|
||||
"""
|
||||
listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
try:
|
||||
if port and os.name != "nt":
|
||||
# aiohttp's own default on POSIX; on Windows it would let a
|
||||
# second process share the port.
|
||||
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
listener.bind((HOST, port))
|
||||
listener.listen(128)
|
||||
except OSError:
|
||||
listener.close()
|
||||
raise
|
||||
return listener
|
||||
|
||||
|
||||
def _graceful_exit() -> None:
|
||||
raise GracefulExit()
|
||||
|
||||
|
||||
def exit_when_stdin_closes(loop: asyncio.AbstractEventLoop) -> threading.Thread:
|
||||
"""Stop the app gracefully once stdin reaches end of file.
|
||||
|
||||
The launcher keeps the write end of this process's stdin pipe, and the OS
|
||||
closes it however the launcher dies, SIGKILL included. GracefulExit runs
|
||||
aiohttp's cleanup (closing the database) and lets the temp root be removed,
|
||||
on every OS, where a signal from a dead parent would not.
|
||||
"""
|
||||
|
||||
def watch() -> None:
|
||||
while sys.stdin.buffer.read(65536):
|
||||
pass
|
||||
loop.call_soon_threadsafe(_graceful_exit)
|
||||
|
||||
thread = threading.Thread(target=watch, name="stdin-eof", daemon=True)
|
||||
thread.start()
|
||||
return thread
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--port", type=int, default=8765)
|
||||
args = parser.parse_args()
|
||||
parser.add_argument(
|
||||
"--port", type=int, default=8765, help="0 binds an OS-chosen free port"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--exit-on-stdin-eof",
|
||||
action="store_true",
|
||||
help="shut down when stdin closes (the launcher's parent-death guard)",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
listener = bind_listener(args.port)
|
||||
# One machine-readable line tells the launcher which port this process holds.
|
||||
print(f"{PORT_ANNOUNCEMENT}{listener.getsockname()[1]}", flush=True)
|
||||
loop = asyncio.new_event_loop()
|
||||
if args.exit_on_stdin_eof:
|
||||
exit_when_stdin_closes(loop)
|
||||
original_cwd = os.getcwd()
|
||||
with tempfile.TemporaryDirectory(prefix="pm-browser-") as directory:
|
||||
web.run_app(
|
||||
create_app(Path(directory).resolve()), host="127.0.0.1", port=args.port
|
||||
)
|
||||
try:
|
||||
web.run_app(create_app(Path(directory).resolve()), sock=listener, loop=loop)
|
||||
finally:
|
||||
# create_app works inside the root; Windows cannot remove a process's cwd.
|
||||
os.chdir(original_cwd)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,diffInventory,focusRingOf,hoverStateOf,mutate} from './concept-inventory.mjs';
|
||||
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept as openConceptPage,openProduction,inventoryOf,diffInventory,focusRingOf,hoverStateOf,mutate} from './concept-inventory.mjs';
|
||||
|
||||
// Task A6 of #249: the production shell (sidebar, topbar, heading, footer) against the immutable concept, with the
|
||||
// concept-shaped real fixture so the sidebar counts are the concept's own (24 images, 6 prompts, 15 tags). Both themes,
|
||||
@@ -16,7 +16,16 @@ const FULL_HEADING=new Set(['tags']);
|
||||
|
||||
const rowNamed=(nav,text)=>nav.rows.find(row=>row.text===text);
|
||||
|
||||
const removedItem=item=>String(item.className).includes('concept-badge')||(item.tag==='button'&&item.text.startsWith('Activity'));
|
||||
const removedItem=item=>String(item.className).includes('concept-badge');
|
||||
|
||||
/**
|
||||
* The concept with its Activity count hidden (D52): production counts running jobs only and hides the count at 0, as
|
||||
* the comparison fixture has none. The concept's own CSS then lays out its button without the count.
|
||||
*/
|
||||
async function openConcept(page,route,theme,motion) {
|
||||
await openConceptPage(page,route,theme,motion);
|
||||
await mutate(page,{css:'#activity-count { display: none }'});
|
||||
}
|
||||
|
||||
/** Left offsets of flex items packed along a row of `width` by its `justify-content` and `gap`. */
|
||||
function packedLefts(items,width,justify,gap) {
|
||||
@@ -37,10 +46,12 @@ function remainingTopActions(actions) {
|
||||
const gap=parseFloat(actions.columnGap);
|
||||
// A full-width row (<=700px) keeps its width; otherwise it shrinks to its remaining items.
|
||||
const width=actions.fullWidth?actions.width:shown.reduce((sum,item)=>sum+item.width,0)+gap*(shown.length-1);
|
||||
// At <=700px the concept spreads three items with `space-between`, so its Theme select sits in the middle; with
|
||||
// Activity removed (D5) the same rule puts Theme at the row's start. Offsets come from the concept's own sizes.
|
||||
// At <=700px the concept spreads its shown items with `space-between`; the hidden pill (D53) takes no place.
|
||||
// Offsets come from the concept's own sizes.
|
||||
const lefts=new Map(shown.map((item,index)=>[item,packedLefts(shown,width,actions.justifyContent,gap)[index]]));
|
||||
return {...actions,height,width,
|
||||
// D52: the full-width row (<=700px) wraps its items when a zoomed phone is narrower than they are.
|
||||
flexWrap:actions.fullWidth?'wrap':actions.flexWrap,
|
||||
items:kept.map(item=>item.shown?{...item,top:Math.round((height-item.height)/2),left:lefts.get(item)}:item)};
|
||||
}
|
||||
|
||||
@@ -53,7 +64,7 @@ function remainingFooter(footer,spans) {
|
||||
}
|
||||
|
||||
// D28 ruling (final review A, Minor 1): Overview gets a distinct outline home glyph. The concept has no home icon, and
|
||||
// its `activity` pulse stays reserved for the Activity row D5 omits, so the two entries never share one glyph.
|
||||
// its `activity` pulse belongs to the Activity row, so the two entries never share one glyph.
|
||||
const OVERVIEW_GLYPH='M3 11 12 4l9 7M5 10v10h14V10M10 20v-6h4v6';
|
||||
|
||||
/**
|
||||
@@ -76,7 +87,9 @@ function mapShellConcept(reference) {
|
||||
if(nav.id!=='system-nav')return nav;
|
||||
const system=rowNamed(nav,'System');
|
||||
return {...nav,rows:[
|
||||
// D5: no Activity row until an activity center exists. D28: no Design review (prototype page).
|
||||
// D52: the Activity row counts running jobs only, so with none it has no count.
|
||||
{...rowNamed(nav,'Activity'),count:null},
|
||||
// D28: no Design review (prototype page).
|
||||
system,
|
||||
// D28: the production Overview hub follows System, as a concept nav row without a count. Its icon is
|
||||
// the ruling's own outline home glyph (OVERVIEW_GLYPH), never the concept's `activity` pulse.
|
||||
@@ -95,9 +108,11 @@ function mapShellConcept(reference) {
|
||||
small:'Your library, on your server',
|
||||
// D28: no fake connection state, so the dot is the neutral muted token (the concept's label color).
|
||||
dot:{...chrome.connection.dot,backgroundColor:chrome.labels[0].color}},
|
||||
// D29: no "Interactive concept" badge. D5: no Activity button. The row's size and the remaining items'
|
||||
// centered offsets follow from the concept's own item sizes.
|
||||
topActions:remainingTopActions(chrome.topActions),
|
||||
// D29: no "Interactive concept" badge. D53: the progress pill before the Activity button, hidden with no running
|
||||
// job. D52: the button's count is hidden at 0. The row's size and the remaining items' centered offsets follow
|
||||
// from the concept's own item sizes.
|
||||
topActions:remainingTopActions({...chrome.topActions,items:chrome.topActions.items.flatMap(item=>
|
||||
item.tag==='button'&&item.text==='Activity'?[{tag:'a',className:'activity-pill',shown:false},{...item,count:'0'}]:[item])}),
|
||||
footer:(()=>{
|
||||
// D29: production footer copy (one line, as the concept's own copy is), and no "What changes & why"
|
||||
// (a prototype review page); the footer then holds only its span.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
|
||||
// Artifact-only captures of Tags & categories states that have no concept counterpart: the Manage categories
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
// Each Playwright worker owns its server: `request`, `page` and manual
|
||||
// `browser.newContext()` all reach it through `baseURL`, and another server
|
||||
// process never sees its database.
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { startBrowserServer } from '../../tools/vendor/start-browser-server.mjs';
|
||||
|
||||
const search = text => `/prompt_manager/search?text=${encodeURIComponent(text)}`;
|
||||
|
||||
test('every fixture reaches this worker\'s own server and another server keeps its own database', async ({ request, page, browser, baseURL, workerServer }, testInfo) => {
|
||||
expect(baseURL).toBe(workerServer.url);
|
||||
await expect(await request.post('/__test__/reset')).toBeOK();
|
||||
const marker = `worker ${testInfo.workerIndex} isolation ${testInfo.project.name} ${testInfo.repeatEachIndex}`;
|
||||
await expect(await request.post('/prompt_manager/save', { data: { text: marker } })).toBeOK();
|
||||
expect((await (await request.get(search(marker))).json()).count).toBe(1);
|
||||
|
||||
const viaPage = await page.request.get(search(marker));
|
||||
expect(viaPage.url().startsWith(`${baseURL}/`)).toBe(true);
|
||||
expect((await viaPage.json()).count).toBe(1);
|
||||
const context = await browser.newContext();
|
||||
try {
|
||||
const response = await (await context.newPage()).goto(search(marker));
|
||||
expect(response.url().startsWith(`${baseURL}/`)).toBe(true);
|
||||
expect((await response.json()).count).toBe(1);
|
||||
} finally {
|
||||
await context.close();
|
||||
}
|
||||
|
||||
const other = await startBrowserServer();
|
||||
try {
|
||||
expect(other.url).not.toBe(baseURL);
|
||||
expect((await (await fetch(`${other.url}${search(marker)}`)).json()).count).toBe(0);
|
||||
} finally {
|
||||
await other.stop();
|
||||
}
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
|
||||
test.beforeEach(async ({ request, page, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
@@ -64,11 +64,16 @@ test('actual Tags page reloads the selected filter and opens its image viewer',
|
||||
await page.goto('/prompt_manager/admin#/tags');
|
||||
await page.locator('[data-tag-name="cats"]').click();
|
||||
await expect(page.locator('#tagPromptsGrid')).toContainText('Cat portrait');
|
||||
const dogs = page.waitForResponse(response => response.url().includes('/tags/dogs/prompts?'));
|
||||
await page.locator('[data-tag-name="dogs"]').click();
|
||||
const [shown] = (await (await dogs).json()).prompts[0].images;
|
||||
await expect(page.locator('#tagPromptsGrid')).toContainText('Dog landscape');
|
||||
await expect(page.locator('#tagPromptsGrid')).not.toContainText('Cat portrait');
|
||||
await page.locator('#tagPromptsGrid img').first().click();
|
||||
await expect(page.locator('.viewer-canvas img')).toBeVisible();
|
||||
// The viewer's title names the shown image through the page's title callback. ViewerJS renders it only once the
|
||||
// image is viewed, so closing at "visible" alone left the callback unrun on slower hosts (CI coverage missed it).
|
||||
await expect(page.locator('.viewer-title')).toHaveText(shown.filename);
|
||||
await page.locator('.viewer-container').getByRole('button', { name: 'Close', exact: true }).click();
|
||||
await expect(page.locator('.viewer-container')).toHaveCount(0);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,801 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept, openConcept, openProduction } from './concept-inventory.mjs';
|
||||
|
||||
// Activity (#249 Activity-1): the concept's Activity page (app.js renderActivity 732–738), its top-bar button
|
||||
// (index.html 51–52) and sidebar row (app.js ~341), with the recorded D52 (counts of running jobs), D53 (the progress
|
||||
// pill) and D54 (real job items). Fake jobs come from tests/browser/workspace_activity_fixture.py.
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
const main = page => page.locator('#workspace-main');
|
||||
const pill = page => page.locator('.top-actions #activity-pill');
|
||||
const activityButton = page => page.locator('.top-actions button#activity-open');
|
||||
const topCount = page => page.locator('#activity-count');
|
||||
const navRow = page => page.locator('#system-nav a[data-route="activity"]');
|
||||
const toast = page => page.locator('#workspace-toast');
|
||||
const items = page => main(page).locator('section.surface > .activity-item');
|
||||
const LOST = 'This job ended when ComfyUI restarted.';
|
||||
// The job view (Task 3, D56): the page heading, then the Progress and Results surfaces and the Scan log.
|
||||
const heading = page => main(page).locator(':scope > .heading');
|
||||
const progressSection = page => main(page).getByRole('region', { name: 'Progress', exact: true });
|
||||
const resultsSection = page => main(page).getByRole('region', { name: 'Results', exact: true });
|
||||
const tiles = page => progressSection(page).locator('dl.stat-row > div.stat');
|
||||
const outcomeButtons = page => resultsSection(page).locator('.outcome-list > button');
|
||||
const fileRows = page => resultsSection(page).locator('.outcome-files > .file-row');
|
||||
const resultsNote = page => resultsSection(page).locator('.outcome-files > p.small');
|
||||
const filter = page => resultsSection(page).getByRole('searchbox', { name: 'Filter results', exact: true });
|
||||
const scanLog = page => main(page).locator('details.term');
|
||||
const TILE_LABELS = ['Added to Library', 'Already known', 'Needs review', 'No usable metadata', 'Errors'];
|
||||
const VIEWER_HINT = 'each row opens the image in the viewer';
|
||||
|
||||
async function tileValues(page) {
|
||||
return tiles(page).evaluateAll(nodes => nodes.map(node => [node.className, node.querySelector('dt').textContent,
|
||||
node.querySelector('dd').textContent]));
|
||||
}
|
||||
|
||||
async function openJob(page, id) {
|
||||
await page.goto(`${workspace}#/activity/${id}`);
|
||||
await expect(progressSection(page)).toBeVisible();
|
||||
}
|
||||
|
||||
async function startJob(request, data = {}) {
|
||||
const response = await request.post('/__test__/jobs/start', { data: { total: 19622, ...data } });
|
||||
await expect(response).toBeOK();
|
||||
return (await response.json()).job;
|
||||
}
|
||||
|
||||
async function step(request, id, data) {
|
||||
await expect(await request.post(`/__test__/jobs/${id}/step`, { data })).toBeOK();
|
||||
}
|
||||
|
||||
async function finish(request, id, state = 'finished') {
|
||||
const response = await request.post(`/__test__/jobs/${id}/finish`, { data: { state } });
|
||||
await expect(response).toBeOK();
|
||||
return (await response.json()).job;
|
||||
}
|
||||
|
||||
async function listed(request) {
|
||||
const response = await request.get('/prompt_manager/jobs');
|
||||
await expect(response).toBeOK();
|
||||
return (await response.json()).jobs;
|
||||
}
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
await expect(await request.post('/__test__/reset')).toBeOK();
|
||||
page.activityErrors = [];
|
||||
page.on('pageerror', error => page.activityErrors.push(error.message));
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, request, browserName }) => {
|
||||
// The store's own reads can meet a route handler at teardown; that must not fail the test it outlived.
|
||||
await page.unrouteAll({ behavior: 'ignoreErrors' });
|
||||
await saveCoverage(page, browserName);
|
||||
expect(page.activityErrors).toEqual([]);
|
||||
await request.post('/__test__/jobs/restart');
|
||||
});
|
||||
|
||||
/** The concept's item structure and the computed styles its CSS gives it, measured the same way in both documents. */
|
||||
function measureItems(selector) {
|
||||
return [...document.querySelectorAll(selector)].map(item => {
|
||||
const css = (node, ...keys) => node && Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]]));
|
||||
const tree = node => ({ tag: node.localName, className: [...node.classList].sort().join(' '), children: [...node.children].map(tree) });
|
||||
const progress = item.querySelector('progress');
|
||||
return {
|
||||
tree: tree(item),
|
||||
title: item.querySelector('h3').textContent,
|
||||
chip: item.querySelector('.chip').textContent,
|
||||
buttons: [...item.querySelectorAll('.row-actions button')].map(button => button.textContent),
|
||||
styles: {
|
||||
item: css(item, 'paddingTop', 'paddingBottom', 'borderBottomWidth', 'borderBottomStyle', 'borderBottomColor'),
|
||||
heading: css(item.querySelector('.panel-heading'), 'display', 'justifyContent', 'alignItems', 'columnGap'),
|
||||
h3: css(item.querySelector('h3'), 'fontSize', 'fontWeight', 'color', 'marginTop', 'marginBottom'),
|
||||
chip: css(item.querySelector('.chip'), 'fontSize', 'color', 'backgroundColor', 'paddingTop', 'paddingLeft', 'borderTopLeftRadius'),
|
||||
p: css(item.querySelector('p'), 'fontSize', 'color', 'marginTop', 'lineHeight'),
|
||||
progress: progress && { ...css(progress, 'height', 'marginTop', 'marginBottom', 'accentColor'),
|
||||
fills: Math.abs(progress.getBoundingClientRect().width - item.getBoundingClientRect().width) < 1,
|
||||
max: progress.max, label: Boolean(progress.getAttribute('aria-label')) },
|
||||
actions: css(item.querySelector('.row-actions'), 'display', 'columnGap', 'marginTop'),
|
||||
button: css(item.querySelector('.row-actions button'), 'fontSize', 'paddingTop', 'paddingLeft', 'minHeight', 'borderTopLeftRadius'),
|
||||
},
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/** D56: an item tree whose `h3` holds the link to the job's view. */
|
||||
function withTitleLink(tree) {
|
||||
const [heading, ...rest] = tree.children;
|
||||
const link = { tag: 'a', className: '', children: [] };
|
||||
return { ...tree, children: [{ ...heading, children: heading.children.map(node => node.tag === 'h3' ? { ...node, children: [link] } : node) }, ...rest] };
|
||||
}
|
||||
|
||||
function measurePage(mainSelector) {
|
||||
const root = document.querySelector(mainSelector);
|
||||
const section = root.querySelector(':scope > section.surface');
|
||||
return {
|
||||
h1: root.querySelector('.heading h1').textContent,
|
||||
description: root.querySelector('.heading p.muted').textContent,
|
||||
actions: [...root.querySelector('.heading .heading-actions').children].map(node => node.textContent),
|
||||
h2: section.querySelector(':scope > h2').textContent,
|
||||
surface: Object.fromEntries(['paddingTop', 'borderTopWidth', 'backgroundColor', 'borderTopLeftRadius']
|
||||
.map(key => [key, getComputedStyle(section)[key]])),
|
||||
};
|
||||
}
|
||||
|
||||
for (const theme of ['dark', 'light']) test(`activity page matches the concept list markup ${theme}`, async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
await seedConcept(request);
|
||||
// The concept: its finished sample, then a running job started from System (startJob, app.js 940–952).
|
||||
await openConcept(page, 'activity', theme);
|
||||
const conceptPage = await page.evaluate(measurePage, '#main');
|
||||
const [conceptDone] = await page.evaluate(measureItems, '#main .activity-item');
|
||||
await page.goto('/__test__/canonical/workspace#system');
|
||||
await page.getByRole('combobox', { name: 'Theme', exact: true }).selectOption(theme);
|
||||
await page.getByRole('button', { name: 'Review thumbnails', exact: true }).click();
|
||||
await expect(page.locator('#main .activity-item progress')).toHaveCount(1);
|
||||
const [conceptRunning] = await page.evaluate(measureItems, '#main .activity-item');
|
||||
|
||||
const done = await startJob(request);
|
||||
await step(request, done.id, { processed: 19622, counters: { added: 1420, errors: 18 } });
|
||||
await finish(request, done.id);
|
||||
const running = await startJob(request);
|
||||
await step(request, running.id, { processed: 8240 });
|
||||
await openProduction(page, 'activity', theme);
|
||||
await expect(items(page)).toHaveCount(2);
|
||||
const productionPage = await page.evaluate(measurePage, '#workspace-main');
|
||||
const [productionRunning, productionDone] = await page.evaluate(measureItems, '#workspace-main .activity-item');
|
||||
|
||||
// D52: no "Unified activity · proposed" chip (prototype copy); the rest of the heading and the section are the concept's.
|
||||
expect(conceptPage.actions).toEqual(['Unified activity · proposed']);
|
||||
expect(productionPage).toEqual({ ...conceptPage, actions: [] });
|
||||
// A completed job is the concept's completed item; D56: its title is a link to the job's view, in the title's colour.
|
||||
expect(conceptDone.title).toBe('✓ Library metadata enriched');
|
||||
expect(productionDone).toEqual({ ...conceptDone, tree: withTitleLink(conceptDone.tree), title: '✓ Scan for metadata' });
|
||||
const links = await items(page).locator('h3 > a').evaluateAll(nodes => nodes.map(node => [node.getAttribute('href'),
|
||||
getComputedStyle(node).color === getComputedStyle(node.parentElement).color, getComputedStyle(node).textDecorationLine]));
|
||||
expect(links).toEqual([[`#/activity/${running.id}`, true, 'none'], [`#/activity/${done.id}`, true, 'none']]);
|
||||
// D54: a running job's chip names its state ("Running", where the concept's sample reads "Needs attention"), and
|
||||
// its actions are Cancel job only (no "Simulate completion").
|
||||
expect(conceptRunning.buttons).toEqual(['Simulate completion', 'Cancel job']);
|
||||
const [, ...cancelOnly] = conceptRunning.tree.children.at(-1).children;
|
||||
const conceptActions = { ...conceptRunning.tree.children.at(-1), children: cancelOnly };
|
||||
expect(productionRunning).toEqual({ ...conceptRunning,
|
||||
tree: withTitleLink({ ...conceptRunning.tree, children: [...conceptRunning.tree.children.slice(0, -1), conceptActions] }),
|
||||
title: '○ Scan for metadata', chip: 'Running', buttons: ['Cancel job'] });
|
||||
await expect(items(page).nth(1).locator('p')).toHaveText(/^Finished \d\d:\d\d · 1,420 added · 18 errors$/);
|
||||
await expect(items(page).first().locator('p')).toHaveText(/^Started \d\d:\d\d · 2 output folders · closing this view does not cancel it$/);
|
||||
await expect(items(page).first().locator('progress')).toHaveAttribute('aria-label', 'Scan for metadata progress');
|
||||
expect(await items(page).first().locator('progress').evaluate(node => node.value)).toBe(41);
|
||||
});
|
||||
|
||||
test('pill shows while a job runs and opens its view', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
await page.goto(`${workspace}#/images`);
|
||||
// D52: no running job, so neither count shows; the button is the concept's.
|
||||
await expect(activityButton(page)).toHaveAccessibleName('Activity');
|
||||
await expect(topCount(page)).toBeHidden();
|
||||
await expect(navRow(page)).toHaveText('Activity');
|
||||
await expect(navRow(page).locator('.nav-count')).toHaveCount(0);
|
||||
await expect(pill(page)).toBeHidden();
|
||||
|
||||
const job = await startJob(request);
|
||||
// The list is read again on its own (15 s), or at once on a reload or when Activity opens.
|
||||
await page.reload();
|
||||
await expect(pill(page)).toBeVisible();
|
||||
await expect(pill(page)).toHaveText('Scanning · 0% · 0 / 19,622');
|
||||
await step(request, job.id, { processed: 8240 });
|
||||
const text = 'Scanning · 41% · 8,240 / 19,622';
|
||||
await expect(pill(page)).toHaveText(text);
|
||||
await expect(pill(page)).toHaveAttribute('aria-label', text);
|
||||
await expect(pill(page)).toHaveAttribute('href', `#/activity/${job.id}`);
|
||||
expect(await pill(page).locator('.activity-pill-bar i').evaluate(node => node.style.width)).toBe('41%');
|
||||
// The pill sits before the button, in the mockup's place, and both counts read 1.
|
||||
expect(await page.locator('.top-actions > *').evaluateAll(nodes => nodes.map(node => node.id || node.className)))
|
||||
.toEqual(['activity-pill', 'activity-open', 'theme-label', 'canvas-link']);
|
||||
await expect(topCount(page)).toBeVisible();
|
||||
await expect(topCount(page)).toHaveText('1');
|
||||
await expect(navRow(page).locator('.nav-count')).toHaveText('1');
|
||||
|
||||
// The pill opens the job's own view (Task 3, D56), named by the job.
|
||||
await pill(page).click();
|
||||
await expect(page).toHaveURL(new RegExp(`#/activity/${job.id}$`));
|
||||
await expect(main(page).getByRole('heading', { level: 1 })).toHaveText('Scan for metadata');
|
||||
await expect(page.locator('#workspace-breadcrumb')).toHaveText('Activity');
|
||||
await expect(page).toHaveTitle('Scan for metadata · Prompt Manager');
|
||||
await expect(navRow(page)).toHaveAttribute('aria-current', 'page');
|
||||
await expect(progressSection(page).locator('progress')).toHaveJSProperty('value', 41);
|
||||
|
||||
// The Activity button opens the list itself from any page.
|
||||
await page.locator('nav [data-route="library"]').click();
|
||||
await activityButton(page).click();
|
||||
await expect(page).toHaveURL(/#\/activity$/);
|
||||
await expect(items(page)).toHaveCount(1);
|
||||
|
||||
await finish(request, job.id);
|
||||
await expect(pill(page)).toBeHidden();
|
||||
await expect(topCount(page)).toBeHidden();
|
||||
await expect(navRow(page).locator('.nav-count')).toHaveCount(0);
|
||||
await expect(items(page).first().locator('.chip.green')).toHaveText('Completed');
|
||||
await expect(items(page).first().locator('progress, .row-actions')).toHaveCount(0);
|
||||
// A finished job's title still opens its view (D56).
|
||||
await items(page).first().locator('h3 > a').click();
|
||||
await expect(page).toHaveURL(new RegExp(`#/activity/${job.id}$`));
|
||||
await expect(heading(page).locator('p.muted')).toHaveText(/^Finished \d\d:\d\d · 0 added · 0 errors$/);
|
||||
});
|
||||
|
||||
test('a phone keeps the pill and its controls inside the topbar', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 390, height: 844 });
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 8240 });
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await expect(pill(page)).toHaveText('Scanning · 41% · 8,240 / 19,622');
|
||||
await expect(pill(page)).toHaveAttribute('aria-label', 'Scanning · 41% · 8,240 / 19,622');
|
||||
await expect(pill(page).locator('.activity-pill-files, .activity-pill-bar')).toHaveCount(2);
|
||||
await expect(pill(page).locator('.activity-pill-files')).toBeHidden();
|
||||
const layout = await page.locator('.top-actions').evaluate(row => ({
|
||||
scroll: document.documentElement.scrollWidth <= document.documentElement.clientWidth,
|
||||
inside: [...row.children].filter(node => node.getBoundingClientRect().width)
|
||||
.every(node => node.getBoundingClientRect().right <= row.getBoundingClientRect().right + 0.5
|
||||
&& node.getBoundingClientRect().left >= row.getBoundingClientRect().left - 0.5),
|
||||
}));
|
||||
expect(layout).toEqual({ scroll: true, inside: true });
|
||||
});
|
||||
|
||||
test('reload during a scan reattaches to the running job', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 1000 });
|
||||
await page.goto(`${workspace}#/activity`);
|
||||
await expect(items(page)).toHaveCount(1);
|
||||
await expect(pill(page)).toHaveText('Scanning · 5% · 1,000 / 19,622');
|
||||
await page.reload();
|
||||
await expect(pill(page)).toHaveText('Scanning · 5% · 1,000 / 19,622');
|
||||
// The reloaded page follows the same job live and started nothing.
|
||||
await step(request, job.id, { processed: 9811 });
|
||||
await expect(pill(page)).toHaveText('Scanning · 50% · 9,811 / 19,622');
|
||||
expect(await items(page).first().locator('progress').evaluate(node => node.value)).toBe(50);
|
||||
await expect(items(page)).toHaveCount(1);
|
||||
expect((await listed(request)).map(item => [item.id, item.state])).toEqual([[job.id, 'running']]);
|
||||
|
||||
// A second tab attaches to the same job and sees the same progress.
|
||||
const other = await page.context().newPage();
|
||||
await other.goto(`${workspace}#/activity`);
|
||||
await expect(pill(other)).toHaveText('Scanning · 50% · 9,811 / 19,622');
|
||||
await expect(other.locator('#workspace-main .activity-item')).toHaveCount(1);
|
||||
await other.close();
|
||||
expect((await listed(request)).length).toBe(1);
|
||||
|
||||
// Cancel job asks the server to stop at a file boundary; the job ends cancelled with its partial counts.
|
||||
await step(request, job.id, { counters: { added: 7 } });
|
||||
await items(page).first().getByRole('button', { name: 'Cancel job', exact: true }).focus();
|
||||
await page.keyboard.press('Enter');
|
||||
await expect(toast(page)).toHaveText('Scan cancelled · 7 added');
|
||||
// The removed Cancel job hands its focus to the item's title link (review M4).
|
||||
await expect(items(page).first().locator('h3 > a')).toBeFocused();
|
||||
await expect(items(page).first().locator('.chip.blue')).toHaveText('Cancelled');
|
||||
await expect(items(page).first().locator('p')).toHaveText(/^Cancelled \d\d:\d\d at a file boundary · 7 added · completed work is kept$/);
|
||||
await expect(pill(page)).toBeHidden();
|
||||
});
|
||||
|
||||
test('lost server clears the pill and marks jobs ended', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 4000 });
|
||||
await page.goto(`${workspace}#/activity`);
|
||||
await expect(pill(page)).toBeVisible();
|
||||
await expect(topCount(page)).toHaveText('1');
|
||||
const reads = [];
|
||||
page.on('request', item => { if (new URL(item.url()).pathname === '/prompt_manager/jobs') reads.push(Date.now()); });
|
||||
// ComfyUI restarts: the stream drops without a final event and the new registry is empty.
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
await expect(pill(page)).toBeHidden();
|
||||
await expect(topCount(page)).toBeHidden();
|
||||
await expect(items(page).first().locator('.chip.blue')).toHaveText('Ended');
|
||||
await expect(items(page).first().locator('p')).toHaveText(LOST);
|
||||
await expect(items(page).first().locator('progress, .row-actions')).toHaveCount(0);
|
||||
await expect(navRow(page).locator('.nav-count')).toHaveCount(0);
|
||||
// One re-read found the restart; there is no retry storm and no finish toast for a job that only vanished.
|
||||
await page.waitForTimeout(2500);
|
||||
expect(reads.length).toBeLessThanOrEqual(2);
|
||||
await expect(toast(page)).toHaveText('');
|
||||
|
||||
// A server that cannot be read at all also ends the running job, then is read again with a growing backoff.
|
||||
const second = await startJob(request);
|
||||
await page.reload();
|
||||
await expect(pill(page)).toBeVisible();
|
||||
const failed = [];
|
||||
await page.route('**/prompt_manager/jobs**', route => { failed.push(Date.now()); return route.abort(); });
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
await expect(pill(page)).toBeHidden();
|
||||
await expect(items(page).filter({ hasText: LOST })).toHaveCount(1);
|
||||
await page.waitForTimeout(3500);
|
||||
// 1 s, then 2 s: at most two re-reads in 3.5 s.
|
||||
expect(failed.length).toBeLessThanOrEqual(2);
|
||||
await page.unroute('**/prompt_manager/jobs**');
|
||||
expect(second.id).not.toBe(job.id);
|
||||
});
|
||||
|
||||
test('start and finish toasts', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await expect(pill(page)).toBeVisible();
|
||||
await step(request, job.id, { processed: 19622, counters: { added: 1420, errors: 18 } });
|
||||
await finish(request, job.id);
|
||||
await expect(toast(page)).toHaveText('Scan finished · 1,420 added · 18 errors');
|
||||
await expect(pill(page)).toBeHidden();
|
||||
|
||||
const single = await startJob(request);
|
||||
await page.reload();
|
||||
await expect(pill(page)).toBeVisible();
|
||||
await step(request, single.id, { processed: 19622, counters: { added: 1, errors: 1 } });
|
||||
await finish(request, single.id, 'failed');
|
||||
await expect(toast(page)).toHaveText('Scan failed · 1 added · 1 error');
|
||||
|
||||
// The store's own start: the real metadata scan over this server's outputs, announced, followed and finished.
|
||||
await page.goto(`${workspace}#/activity`);
|
||||
const started = await page.evaluate(async () => {
|
||||
const { createActivityStore } = await import('/prompt_manager/js/workspace/activity-store.mjs');
|
||||
const controller = new AbortController();
|
||||
const store = createActivityStore({ api: window.ApiClient.createApiClient(), pollMs: 0,
|
||||
scope: { signal: controller.signal, cleanup: () => {} } });
|
||||
const result = await store.start('scan');
|
||||
// Read before the busy attempt: its list read may already see the small scan finished and toast that.
|
||||
const toast = document.getElementById('workspace-toast').textContent;
|
||||
const busy = await store.start('scan').catch(error => ({ error: error.message }));
|
||||
return { id: result.job.id, title: result.job.title, toast, busy: busy.busy?.id ?? null, message: busy.message ?? null };
|
||||
});
|
||||
expect(started.title).toBe('Scan for metadata');
|
||||
expect(started.toast).toBe('Scan started.');
|
||||
// A second start while the scan runs is refused and names it (unless the small fixture scan already ended).
|
||||
if (started.busy !== null) expect(started.busy).toBe(started.id);
|
||||
await expect(toast(page)).toHaveText(/^Scan (started\.|finished · \d+ added · \d+ errors?)$/);
|
||||
await expect.poll(async () => (await listed(request)).find(item => item.id === started.id)?.state, { timeout: 15000 }).toBe('finished');
|
||||
});
|
||||
|
||||
// ── Task 3: Scan for metadata, its review dialog and the job view ─────────────────────────────
|
||||
|
||||
async function startFromImport(page) {
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await page.getByRole('button', { name: 'Import', exact: true }).click();
|
||||
const importDialog = page.getByRole('dialog', { name: 'Import & inspect', exact: true });
|
||||
const row = importDialog.locator('.operation').filter({ hasText: 'Scan for metadata' });
|
||||
await row.getByRole('button', { name: 'Review scan', exact: true }).click();
|
||||
const review = page.getByRole('dialog', { name: 'Review metadata scan', exact: true });
|
||||
await expect(review).toBeVisible();
|
||||
await review.getByRole('button', { name: 'Start scan', exact: true }).click();
|
||||
await expect(review).toHaveCount(0);
|
||||
}
|
||||
|
||||
test('start scan creates a job, toasts, and opens its view', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const posted = [];
|
||||
page.on('request', item => { if (item.method() === 'POST' && new URL(item.url()).pathname === '/prompt_manager/jobs/scan') posted.push(item); });
|
||||
await startFromImport(page);
|
||||
await expect(page).toHaveURL(/#\/activity\/[0-9a-f]{32}$/);
|
||||
const id = page.url().split('/').at(-1);
|
||||
expect(posted).toHaveLength(1);
|
||||
// The fixture scan is small and may already have finished; the start toast came first either way.
|
||||
await expect(toast(page)).toHaveText(/^Scan (started\.|finished · \d+ added · \d+ errors?)$/);
|
||||
await expect(heading(page).locator('h1')).toHaveText('Scan for metadata');
|
||||
await expect(heading(page).locator('h1')).toBeFocused();
|
||||
expect((await listed(request)).map(item => item.id)).toEqual([id]);
|
||||
await expect.poll(async () => (await listed(request))[0].state, { timeout: 15000 }).toBe('finished');
|
||||
await expect(heading(page).locator('p.muted')).toHaveText(/^Finished \d\d:\d\d · \d+ added · \d+ errors?$/);
|
||||
// The finished scan's tiles add up to the files it processed, and its log is there to open.
|
||||
const job = (await listed(request))[0];
|
||||
const values = await tileValues(page);
|
||||
expect(values.map(([, label]) => label)).toEqual(TILE_LABELS);
|
||||
expect(values.reduce((sum, [, , value]) => sum + Number(value.replaceAll(',', '')), 0)).toBe(job.processed);
|
||||
// Opening the log reads its last lines (review M12).
|
||||
await scanLog(page).locator('summary').click();
|
||||
await expect(scanLog(page).locator('pre')).toContainText('Scan finished');
|
||||
});
|
||||
|
||||
// ── Task 4 (Activity-2): scan scope, new and unresolved files by default ─────────────────────
|
||||
|
||||
test('all files checkbox changes the scope line and the request', async ({ page }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const bodies = [];
|
||||
page.on('request', item => {
|
||||
if (item.method() === 'POST' && new URL(item.url()).pathname === '/prompt_manager/jobs/scan') bodies.push(item.postDataJSON());
|
||||
});
|
||||
const review = page.getByRole('dialog', { name: 'Review metadata scan', exact: true });
|
||||
const scopeValue = review.locator('.meta-grid > div').filter({ has: page.locator('dt', { hasText: 'Scope' }) }).locator('dd');
|
||||
// The concept's switch: a checkbox with role=switch.
|
||||
const allFiles = review.getByRole('switch', { name: 'All files', exact: true });
|
||||
const openReview = async () => {
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await page.getByRole('button', { name: 'Import', exact: true }).click();
|
||||
await page.getByRole('dialog', { name: 'Import & inspect', exact: true }).locator('.operation')
|
||||
.filter({ hasText: 'Scan for metadata' }).getByRole('button', { name: 'Review scan', exact: true }).click();
|
||||
await expect(review).toBeVisible();
|
||||
};
|
||||
|
||||
// The default: the concept's "New and unresolved files", the box unchecked; the request asks for "new".
|
||||
await openReview();
|
||||
await expect(scopeValue).toHaveText('New and unresolved files');
|
||||
await expect(allFiles).not.toBeChecked();
|
||||
await expect(review.locator('.setting-row p')).toHaveText('Also read the files already linked to their prompt again.');
|
||||
await expect(allFiles).toHaveAccessibleDescription('Also read the files already linked to their prompt again.');
|
||||
// Checking and unchecking switches the Scope value both ways.
|
||||
await allFiles.check();
|
||||
await expect(scopeValue).toHaveText('All files');
|
||||
await allFiles.uncheck();
|
||||
await expect(scopeValue).toHaveText('New and unresolved files');
|
||||
await review.getByRole('button', { name: 'Start scan', exact: true }).click();
|
||||
await expect(page).toHaveURL(/#\/activity\/[0-9a-f]{32}$/);
|
||||
await expect.poll(() => bodies.length).toBe(1);
|
||||
expect(bodies[0]).toEqual({ scope: 'new' });
|
||||
await expect(heading(page).locator('h1')).toHaveText('Scan for metadata');
|
||||
// Wait for the first scan to end so the second is not refused as busy.
|
||||
await scanLog(page).locator('summary').click();
|
||||
await expect(scanLog(page).locator('pre')).toContainText('Scan finished', { timeout: 15000 });
|
||||
await expect(scanLog(page).locator('pre')).toContainText('new and unresolved files');
|
||||
|
||||
// Checked: "All files", and the request asks for "all".
|
||||
await openReview();
|
||||
await allFiles.check();
|
||||
await expect(scopeValue).toHaveText('All files');
|
||||
await review.getByRole('button', { name: 'Start scan', exact: true }).click();
|
||||
await expect.poll(() => bodies.length).toBe(2);
|
||||
expect(bodies[1]).toEqual({ scope: 'all' });
|
||||
await scanLog(page).locator('summary').click();
|
||||
await expect(scanLog(page).locator('pre')).toContainText('Scan started · ', { timeout: 15000 });
|
||||
await expect(scanLog(page).locator('pre')).toContainText('all files');
|
||||
});
|
||||
|
||||
test('job view counters follow the mapping and update live', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 8240, counters: { added: 1420, known: 6511, review: 37, no_metadata: 254, errors: 18 } });
|
||||
await openJob(page, job.id);
|
||||
// The header: the title, the detail line, then Download report (enabled in Task 5) and Cancel scan (danger).
|
||||
await expect(heading(page).locator('h1')).toHaveText('Scan for metadata');
|
||||
await expect(heading(page).locator('p.muted')).toHaveText(/^Started \d\d:\d\d · 2 output folders · closing this view does not cancel it$/);
|
||||
await expect(heading(page).locator('.heading-actions > button')).toHaveText(['Download report', 'Cancel scan']);
|
||||
await expect(heading(page).getByRole('button', { name: 'Download report', exact: true })).toBeEnabled();
|
||||
await expect(heading(page).getByRole('button', { name: 'Cancel scan', exact: true })).toHaveClass(/danger/);
|
||||
// Progress: the bar, the server's line and the five tiles in order and colour.
|
||||
await expect(progressSection(page).locator('.panel-heading > span.small')).toHaveText('Processing file 8,240 of 19,622');
|
||||
await expect(progressSection(page).locator('progress')).toHaveAttribute('aria-label', 'Scan progress');
|
||||
await expect(progressSection(page).locator('progress')).toHaveJSProperty('value', 41);
|
||||
expect(await tileValues(page)).toEqual([
|
||||
['stat added', 'Added to Library', '1,420'], ['stat', 'Already known', '6,511'], ['stat warn', 'Needs review', '37'],
|
||||
['stat', 'No usable metadata', '254'], ['stat bad', 'Errors', '18'],
|
||||
]);
|
||||
const colours = await tiles(page).evaluateAll(nodes => nodes.map(node => getComputedStyle(node.querySelector('dd')).color));
|
||||
const expected = await page.evaluate(() => ['--green', '--ca-text', '--warning', '--ca-text', '--red'].map(name => {
|
||||
const probe = document.createElement('span');
|
||||
probe.style.color = `var(${name})`;
|
||||
document.querySelector('#workspace-main').append(probe);
|
||||
const colour = getComputedStyle(probe).color;
|
||||
probe.remove();
|
||||
return colour;
|
||||
}));
|
||||
expect([colours[0], colours[2], colours[4]]).toEqual([expected[0], expected[2], expected[4]]);
|
||||
expect(colours[1]).toBe(colours[3]);
|
||||
// The outcome list follows the mockup's order with the same counts.
|
||||
await expect(outcomeButtons(page)).toHaveText(['Added to Library1,420', 'Needs review37', 'Errors18', 'No usable metadata254', 'Already known6,511']);
|
||||
|
||||
// Live: the next step updates the bar, the line and the tiles without a reload.
|
||||
await step(request, job.id, { processed: 9811, counters: { added: 1500, errors: 19 }, records: [
|
||||
{ group: 'added', path: 'live/new.png', detail: 'Added to the Library as “Live”' },
|
||||
] });
|
||||
await expect(progressSection(page).locator('.panel-heading > span.small')).toHaveText('Processing file 9,811 of 19,622');
|
||||
await expect(progressSection(page).locator('progress')).toHaveJSProperty('value', 50);
|
||||
await expect(tiles(page).first().locator('dd')).toHaveText('1,500');
|
||||
await expect(tiles(page).last().locator('dd')).toHaveText('19');
|
||||
await expect(fileRows(page)).toHaveCount(1);
|
||||
// A focused row keeps its focus while later reads append rows.
|
||||
await fileRows(page).first().focus();
|
||||
await step(request, job.id, { records: [{ group: 'added', path: 'live/next.png', detail: 'Added' }] });
|
||||
await expect(fileRows(page)).toHaveCount(2);
|
||||
await expect(fileRows(page).first()).toBeFocused();
|
||||
// The log is collapsed by default and holds the raw timestamped lines.
|
||||
await expect(scanLog(page)).not.toHaveAttribute('open', '');
|
||||
await expect(scanLog(page).locator('summary')).toContainText('Scan log');
|
||||
await scanLog(page).locator('summary').click();
|
||||
await expect(scanLog(page).locator('pre')).toHaveAttribute('tabindex', '0');
|
||||
await expect(scanLog(page).locator('pre')).toHaveText(/^\d\d:\d\d:\d\d \+ live\/new\.png · Added to the Library as “Live”\n\d\d:\d\d:\d\d \+ live\/next\.png · Added$/);
|
||||
|
||||
// Finished: the header says so, Cancel scan is gone and the counters stay.
|
||||
await finish(request, job.id);
|
||||
await expect(heading(page).locator('p.muted')).toHaveText(/^Finished \d\d:\d\d · 1,501 added · 19 errors$/);
|
||||
await expect(heading(page).locator('.heading-actions > button')).toHaveText(['Download report']);
|
||||
await expect(tiles(page).first().locator('dd')).toHaveText('1,501');
|
||||
});
|
||||
|
||||
test('added rows open the viewer, error rows are plain', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
// The fixture's two listed files share a name in the output and input roots; the row's root label tells them apart.
|
||||
await step(request, job.id, { processed: 3, records: [
|
||||
{ group: 'added', path: 'shared/same.png', root: 'output', detail: 'Added to the Library as “Cat portrait”' },
|
||||
{ group: 'known', path: 'shared/same.png', root: 'input', detail: 'Already linked to “Dog portrait”' },
|
||||
{ group: 'errors', path: 'broken/truncated.png', root: 'output', detail: 'Error: the file could not be processed (see the server log)' },
|
||||
] });
|
||||
await openJob(page, job.id);
|
||||
await expect(outcomeButtons(page).first()).toHaveAttribute('aria-pressed', 'true');
|
||||
await expect(fileRows(page)).toHaveCount(1);
|
||||
const added = fileRows(page).first();
|
||||
expect(await added.evaluate(node => node.localName)).toBe('button');
|
||||
await expect(added.locator('.path')).toHaveText('shared/same.png');
|
||||
await expect(added.locator('.why')).toHaveText('Added to the Library as “Cat portrait”');
|
||||
await expect(resultsNote(page)).toHaveText('Each row opens the image in the viewer.');
|
||||
await added.click();
|
||||
const viewer = page.locator('dialog#viewer-dialog[open]');
|
||||
await expect(viewer).toBeVisible();
|
||||
await expect(viewer.locator('#viewer-title')).toContainText('same.png');
|
||||
await expect(viewer.locator('#viewer-stage img').first()).toHaveAttribute('src', /[?&]root=0(&|$)/);
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(viewer).toHaveCount(0);
|
||||
await expect(added).toBeFocused();
|
||||
|
||||
// Already known opens the input root's file.
|
||||
await outcomeButtons(page).filter({ hasText: 'Already known' }).click();
|
||||
await expect(outcomeButtons(page).filter({ hasText: 'Already known' })).toHaveAttribute('aria-pressed', 'true');
|
||||
await fileRows(page).first().click();
|
||||
await expect(page.locator('dialog#viewer-dialog[open] #viewer-stage img').first()).toHaveAttribute('src', /[?&]root=1(&|$)/);
|
||||
await page.keyboard.press('Escape');
|
||||
|
||||
// Errors: plain text rows, no viewer hint.
|
||||
await outcomeButtons(page).filter({ hasText: 'Errors' }).click();
|
||||
await expect(fileRows(page)).toHaveCount(1);
|
||||
expect(await fileRows(page).first().evaluate(node => node.localName)).toBe('div');
|
||||
await expect(fileRows(page).first().locator('.path')).toHaveText('broken/truncated.png');
|
||||
await expect(resultsNote(page)).toHaveCount(0);
|
||||
// An empty group says so.
|
||||
await outcomeButtons(page).filter({ hasText: 'Needs review' }).click();
|
||||
await expect(resultsSection(page).locator('.outcome-files > p.empty-state')).toHaveText('No files in this group.');
|
||||
|
||||
// A listed row whose file has left the listing toasts instead of opening an empty viewer.
|
||||
await step(request, job.id, { records: [{ group: 'added', path: 'gone/missing.png', root: 'output', detail: 'Added' }] });
|
||||
await outcomeButtons(page).first().click();
|
||||
await expect(fileRows(page)).toHaveCount(2);
|
||||
await fileRows(page).nth(1).click();
|
||||
await expect(toast(page)).toHaveText('This file is no longer in Images. Rescan from Import and try again.');
|
||||
await expect(page.locator('dialog#viewer-dialog')).toHaveCount(0);
|
||||
// A row opens only its own root's file: a root whose label no longer matches (a replaced source) never opens the
|
||||
// other root's same path, and a row without a root index opens nothing (review M8, M9).
|
||||
const lookups = [];
|
||||
page.on('request', item => { const url = new URL(item.url()); if (url.pathname === '/prompt_manager/images/output' && url.searchParams.get('path')) lookups.push(url.search); });
|
||||
await step(request, job.id, { records: [
|
||||
{ group: 'added', path: 'shared/same.png', root: 'output', root_index: 1, detail: 'Added' },
|
||||
{ group: 'added', path: 'shared/same.png', root: 'output', root_index: null, detail: 'Added' },
|
||||
] });
|
||||
await expect(fileRows(page)).toHaveCount(4);
|
||||
for (const index of [2, 3]) {
|
||||
await fileRows(page).nth(index).click();
|
||||
await expect(toast(page)).toHaveText('This file is no longer in Images. Rescan from Import and try again.');
|
||||
await expect(page.locator('dialog#viewer-dialog')).toHaveCount(0);
|
||||
}
|
||||
expect(lookups).toEqual([]);
|
||||
// The lookup is the exact identity, not a name search.
|
||||
await fileRows(page).first().click();
|
||||
await expect(page.locator('dialog#viewer-dialog[open]')).toBeVisible();
|
||||
expect(lookups).toEqual(['?mode=workspace&root=0&path=shared%2Fsame.png&limit=5']);
|
||||
await page.keyboard.press('Escape');
|
||||
});
|
||||
|
||||
test('filter narrows the selected outcome', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 4, records: [
|
||||
{ group: 'added', path: 'CCC/portrait_0001.png', detail: 'Added' },
|
||||
{ group: 'added', path: 'CCC/portrait_0002.png', detail: 'Added' },
|
||||
{ group: 'added', path: 'forest/pines_0008.png', detail: 'Added' },
|
||||
{ group: 'errors', path: 'CCC/broken.png', detail: 'Error' },
|
||||
] });
|
||||
await openJob(page, job.id);
|
||||
await expect(fileRows(page)).toHaveCount(3);
|
||||
await expect(filter(page)).toHaveAttribute('placeholder', 'Filter by file path…');
|
||||
await filter(page).fill('ccc/');
|
||||
await expect(fileRows(page).locator('.path')).toHaveText(['CCC/portrait_0001.png', 'CCC/portrait_0002.png']);
|
||||
await expect(resultsNote(page)).toHaveText(`Showing 2 of 3 · ${VIEWER_HINT}`);
|
||||
// The filter stays while another outcome is chosen.
|
||||
await outcomeButtons(page).filter({ hasText: 'Errors' }).click();
|
||||
await expect(fileRows(page).locator('.path')).toHaveText(['CCC/broken.png']);
|
||||
await filter(page).fill('pines');
|
||||
await expect(fileRows(page)).toHaveCount(0);
|
||||
await expect(resultsSection(page).locator('.outcome-files > p.empty-state')).toHaveText('No listed file matches this filter.');
|
||||
await outcomeButtons(page).first().click();
|
||||
await expect(fileRows(page).locator('.path')).toHaveText(['forest/pines_0008.png']);
|
||||
await filter(page).fill('');
|
||||
await expect(fileRows(page)).toHaveCount(3);
|
||||
});
|
||||
|
||||
test('job view renders 200-row cap note', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 260, bulk: { group: 'added', count: 250 }, records: [
|
||||
...Array.from({ length: 10 }, (_, index) => ({ group: 'no_metadata', path: `misc/plain_${index}.png`, detail: 'No prompt metadata in the file' })),
|
||||
] });
|
||||
// Every detail read of this job: in flight, and whether one has returned the full listing.
|
||||
const detailUrl = `/prompt_manager/jobs/${job.id}`;
|
||||
const isDetail = item => new URL(item.url()).pathname === detailUrl;
|
||||
const detailReads = { inFlight: 0, full: false };
|
||||
page.on('request', item => { if (isDetail(item)) detailReads.inFlight += 1; });
|
||||
page.on('requestfailed', item => { if (isDetail(item)) detailReads.inFlight -= 1; });
|
||||
page.on('response', async response => {
|
||||
if (!isDetail(response)) return;
|
||||
const body = await response.json().catch(() => null);
|
||||
const outcomes = body?.job?.outcomes ?? {};
|
||||
if (outcomes.added?.length === 200 && outcomes.no_metadata?.length === 10) detailReads.full = true;
|
||||
detailReads.inFlight -= 1;
|
||||
});
|
||||
await openJob(page, job.id);
|
||||
await expect(tiles(page).first().locator('dd')).toHaveText('250');
|
||||
await expect(fileRows(page)).toHaveCount(200);
|
||||
await expect(resultsNote(page)).toHaveText(`Showing 200 of 250 · ${VIEWER_HINT}`);
|
||||
await outcomeButtons(page).filter({ hasText: 'No usable metadata' }).click();
|
||||
await expect(fileRows(page)).toHaveCount(10);
|
||||
await expect(resultsNote(page)).toHaveCount(0);
|
||||
// Every listed row is in the page and it stays responsive: a filter answers at once.
|
||||
await outcomeButtons(page).first().click();
|
||||
await filter(page).fill('file_00199');
|
||||
await expect(fileRows(page)).toHaveCount(1);
|
||||
await expect(resultsNote(page)).toHaveText(`Showing 1 of 250 · ${VIEWER_HINT}`);
|
||||
|
||||
// Review M12: with the list capped and the log closed, more files only update the counters; the detail is read
|
||||
// again only when the log opens with new lines.
|
||||
// The reads from opening the view have settled: one returned the full listing and none is in flight. The view
|
||||
// re-reads only while what the summary lists differs from the last read, so no further read is due.
|
||||
await expect.poll(() => detailReads.full && detailReads.inFlight === 0).toBe(true);
|
||||
const reads = [];
|
||||
page.on('request', item => { if (new URL(item.url()).pathname === `/prompt_manager/jobs/${job.id}`) reads.push(Date.now()); });
|
||||
await step(request, job.id, { processed: 300, bulk: { group: 'added', count: 40 } });
|
||||
await expect(tiles(page).first().locator('dd')).toHaveText('290');
|
||||
await page.waitForTimeout(1500);
|
||||
expect(reads).toEqual([]);
|
||||
await scanLog(page).locator('summary').click();
|
||||
await expect.poll(() => reads.length).toBe(1);
|
||||
await expect(scanLog(page).locator('pre')).toContainText('bulk/file_00039.png');
|
||||
});
|
||||
|
||||
test('cancel scan ends cancelled with partial counts', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 9, counters: { added: 7, errors: 2 } });
|
||||
await openJob(page, job.id);
|
||||
// From the keyboard: when the job ends and Cancel scan goes away, focus moves to the job's heading (review M4).
|
||||
await heading(page).getByRole('button', { name: 'Cancel scan', exact: true }).focus();
|
||||
await page.keyboard.press('Enter');
|
||||
await expect(toast(page)).toHaveText('Scan cancelled · 7 added');
|
||||
await expect(heading(page).locator('h1')).toBeFocused();
|
||||
await expect(heading(page).locator('p.muted')).toHaveText(/^Cancelled \d\d:\d\d at a file boundary · 7 added · completed work is kept$/);
|
||||
await expect(heading(page).getByRole('button', { name: 'Cancel scan', exact: true })).toHaveCount(0);
|
||||
await expect(progressSection(page).locator('.panel-heading > span.small')).toHaveText('Cancelled after 9 of 19,622 files');
|
||||
expect((await tileValues(page)).map(([, , value]) => value)).toEqual(['7', '0', '0', '0', '2']);
|
||||
expect((await listed(request))[0].state).toBe('cancelled');
|
||||
|
||||
// A failed cancel request keeps the button and says so.
|
||||
const second = await startJob(request);
|
||||
await openJob(page, second.id);
|
||||
await page.route('**/cancel', route => route.fulfill({ status: 500, json: { success: false, error: 'nope' } }));
|
||||
await heading(page).getByRole('button', { name: 'Cancel scan', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('The job could not be cancelled. Try again.');
|
||||
await expect(heading(page).getByRole('button', { name: 'Cancel scan', exact: true })).not.toHaveAttribute('aria-disabled', 'true');
|
||||
});
|
||||
|
||||
test('busy start names the running job', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const running = await startJob(request, { title: 'Database maintenance', kind: 'maintenance' });
|
||||
await startFromImport(page);
|
||||
await expect(toast(page)).toHaveText('Another job is running: Database maintenance.');
|
||||
await expect(toast(page).getByRole('link', { name: 'Database maintenance', exact: true })).toHaveAttribute('href', `#/activity/${running.id}`);
|
||||
await expect(page).toHaveURL(/#\/images$/);
|
||||
expect((await listed(request)).map(item => item.id)).toEqual([running.id]);
|
||||
// Review M14: a toast with a link outlasts the plain 6.5 s, so the link stays reachable.
|
||||
await page.waitForTimeout(7000);
|
||||
await expect(toast(page).getByRole('link', { name: 'Database maintenance', exact: true })).toBeVisible();
|
||||
await toast(page).getByRole('link', { name: 'Database maintenance', exact: true }).click();
|
||||
await expect(heading(page).locator('h1')).toHaveText('Database maintenance');
|
||||
// A job of another kind cancels as "Cancel job".
|
||||
await expect(heading(page).locator('.heading-actions > button')).toHaveText(['Download report', 'Cancel job']);
|
||||
});
|
||||
|
||||
test('an unknown or restarted job view says so', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
await page.goto(`${workspace}#/activity/${'0'.repeat(32)}`);
|
||||
await expect(main(page).locator('section.surface p.empty-state')).toHaveText('This job is not in this session’s Activity. Jobs are kept until ComfyUI restarts.');
|
||||
await expect(main(page).getByRole('link', { name: 'All activity', exact: true })).toHaveAttribute('href', '#/activity');
|
||||
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 5, records: [{ group: 'added', path: 'kept/row.png', detail: 'Added' }] });
|
||||
await openJob(page, job.id);
|
||||
await expect(fileRows(page)).toHaveCount(1);
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
await expect(heading(page).locator('p.muted')).toHaveText(LOST);
|
||||
await expect(heading(page).getByRole('button', { name: 'Cancel scan', exact: true })).toHaveCount(0);
|
||||
// A lost job has no report on the server (review M9): Download report is disabled rather than failing.
|
||||
await expect(heading(page).getByRole('button', { name: 'Download report', exact: true })).toBeDisabled();
|
||||
// What was read before the restart stays on the page.
|
||||
await expect(fileRows(page)).toHaveCount(1);
|
||||
await expect(pill(page)).toBeHidden();
|
||||
});
|
||||
|
||||
// Review M3: a finished job the server no longer lists is gone from this tab too, and its open view says so.
|
||||
test('a finished job the server pruned leaves its open view honest', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 1, records: [{ group: 'added', path: 'kept/row.png', detail: 'Added' }] });
|
||||
await finish(request, job.id);
|
||||
await openJob(page, job.id);
|
||||
await expect(fileRows(page)).toHaveCount(1);
|
||||
await filter(page).focus();
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
// The next periodic list read (15 s) no longer has the finished job.
|
||||
test.slow();
|
||||
await expect(main(page).locator('section.surface p.empty-state')).toHaveText('This job is not in this session’s Activity. Jobs are kept until ComfyUI restarts.', { timeout: 20000 });
|
||||
await expect(progressSection(page)).toHaveCount(0);
|
||||
await expect(heading(page).locator('h1')).toHaveText('Activity');
|
||||
await expect(heading(page).locator('h1')).toBeFocused();
|
||||
await expect(page).toHaveTitle('Activity · Prompt Manager');
|
||||
});
|
||||
|
||||
// Activity-2 Task 5 (D59): Download report and the log's scroll position.
|
||||
test('download report saves the job JSON', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 2, records: [
|
||||
{ group: 'added', path: 'sub/a.png', root: 'output', detail: 'Added' },
|
||||
{ group: 'errors', path: 'broken/b.png', root: 'output', detail: 'Error' },
|
||||
] });
|
||||
await finish(request, job.id);
|
||||
await openJob(page, job.id);
|
||||
const button = heading(page).getByRole('button', { name: 'Download report', exact: true });
|
||||
await expect(button).toBeEnabled();
|
||||
const [download] = await Promise.all([page.waitForEvent('download'), button.click()]);
|
||||
expect(download.suggestedFilename()).toMatch(/^prompt-manager-scan-\d{8}-\d{4}\.json$/);
|
||||
const path = await download.path();
|
||||
const report = JSON.parse(await (await import('node:fs/promises')).readFile(path, 'utf8'));
|
||||
expect(report.job.id).toBe(job.id);
|
||||
expect(report.outcomes.added.rows.map(row => row.path)).toEqual(['sub/a.png']);
|
||||
expect(report.outcomes.errors.truncated).toBe(false);
|
||||
expect(report.log.length).toBeGreaterThan(0);
|
||||
await expect(toast(page)).toHaveText('Job report downloaded.');
|
||||
});
|
||||
|
||||
test('log keeps the reader\'s scroll position', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
const more = (from, count) => Array.from({ length: count }, (_, index) => ({ group: 'known', path: `log/f_${from + index}.png`,
|
||||
root: 'output', detail: 'Known' }));
|
||||
await step(request, job.id, { processed: 60, records: more(0, 60) });
|
||||
await openJob(page, job.id);
|
||||
await scanLog(page).locator('summary').click();
|
||||
const pre = scanLog(page).locator('pre');
|
||||
await expect(pre).toContainText('log/f_59.png');
|
||||
const metrics = () => pre.evaluate(node => ({ top: node.scrollTop, max: node.scrollHeight - node.clientHeight }));
|
||||
// Opening starts at the newest line.
|
||||
await expect.poll(async () => { const m = await metrics(); return m.max > 0 && m.top >= m.max - 2; }).toBe(true);
|
||||
// Pinned to the bottom: new lines are followed.
|
||||
await step(request, job.id, { processed: 90, records: more(60, 30) });
|
||||
await expect(pre).toContainText('log/f_89.png');
|
||||
await expect.poll(async () => { const m = await metrics(); return m.top >= m.max - 2; }).toBe(true);
|
||||
// Scrolled up: the position stays while more lines arrive.
|
||||
await pre.evaluate(node => { node.scrollTop = 40; });
|
||||
await step(request, job.id, { processed: 120, records: more(90, 30) });
|
||||
await expect(pre).toContainText('log/f_119.png');
|
||||
expect((await metrics()).top).toBe(40);
|
||||
// Closed and opened again (review M6): every open starts at the newest line, also after lines arrived meanwhile.
|
||||
await scanLog(page).locator('summary').click();
|
||||
await expect(scanLog(page)).not.toHaveAttribute('open', '');
|
||||
await step(request, job.id, { processed: 150, records: more(120, 30) });
|
||||
await scanLog(page).locator('summary').click();
|
||||
await expect(pre).toContainText('log/f_149.png');
|
||||
await expect.poll(async () => { const m = await metrics(); return m.max > 0 && m.top >= m.max - 2; }).toBe(true);
|
||||
});
|
||||
|
||||
test('a failed report download says so and leaves the button usable', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await finish(request, job.id);
|
||||
await openJob(page, job.id);
|
||||
await page.route(`**/prompt_manager/jobs/${job.id}/report`, route => route.fulfill({ status: 500, body: 'no' }));
|
||||
const button = heading(page).getByRole('button', { name: 'Download report', exact: true });
|
||||
await button.click();
|
||||
await expect(toast(page)).toHaveText('The report could not be downloaded. Try again.');
|
||||
await expect(button).not.toHaveAttribute('aria-disabled', 'true');
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {readFile,writeFile} from 'node:fs/promises';
|
||||
import {createHash} from 'node:crypto';
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept, openConcept, openProduction } from './concept-inventory.mjs';
|
||||
|
||||
const workspace = '/prompt_manager/workspace#/images';
|
||||
@@ -1512,6 +1512,25 @@ test('Refine opened before discovery settles lists the discovered sources and fo
|
||||
});
|
||||
|
||||
// #249 final review m1: as in the old field row, a Source offers only its own folders.
|
||||
test('a search reload keeps the Filters label node, so a click landing during the reload is not lost', async ({ page, request }) => {
|
||||
// WebKit drops a click whose mousedown hit a text node that was replaced before mouseup; the reload reports the
|
||||
// (unchanged) active count, so the label must keep its node.
|
||||
await expect(await request.post('/__test__/reset', { data: { extra_images: 80 } })).toBeOK();
|
||||
await page.goto(workspace);
|
||||
const button = filtersButton(page);
|
||||
await button.click();await expect(refineDialog(page).getByLabel('Folder', { exact: true }).locator('option[value="bulk"]')).toHaveCount(1);await page.keyboard.press('Escape');
|
||||
await refine(page, { Source: '0', Folder: 'bulk' });
|
||||
await expect(button).toHaveText('Filters · 2');
|
||||
const label = await button.evaluateHandle(node => node.firstChild);
|
||||
const reloaded = page.waitForResponse(response => {
|
||||
const url = new URL(response.url());return imagePages(url) && url.searchParams.get('q') === 'png';
|
||||
});
|
||||
await search(page).fill('png');
|
||||
await reloaded;
|
||||
await expect(button).toHaveText('Filters · 2');
|
||||
expect(await button.evaluate((node, text) => node.firstChild === text && text.isConnected, label)).toBe(true);
|
||||
});
|
||||
|
||||
test('Refine offers only the chosen source folders', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/add-nested-image')).toBeOK();
|
||||
await page.goto(workspace);
|
||||
@@ -2037,3 +2056,50 @@ test('Set category limits the field to the server length, shows the server count
|
||||
await expect(bulkbar(page)).toBeEmpty();
|
||||
expect(answers).toHaveLength(0);
|
||||
});
|
||||
|
||||
// Activity-1 Task 3 (D55): Import → "Scan for metadata" → Review scan opens the concept's "Review metadata scan".
|
||||
const scanReview = page => page.getByRole('dialog', { name: 'Review metadata scan', exact: true });
|
||||
async function openScanReview(page) {
|
||||
await page.getByRole('button', { name: 'Import', exact: true }).click();
|
||||
const row = importDialog(page).locator('.operation').filter({ has: page.getByRole('heading', { name: 'Scan for metadata', exact: true }) });
|
||||
await row.getByRole('button', { name: 'Review scan', exact: true }).click();
|
||||
await expect(scanReview(page)).toBeVisible();
|
||||
}
|
||||
|
||||
test('scan for metadata opens the review dialog from Import', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
await page.goto(workspace);
|
||||
await expect(cards(page)).toHaveCount(2);
|
||||
// Rescan stays the quick refresh; the metadata scan is its own row under it.
|
||||
await page.getByRole('button', { name: 'Import', exact: true }).click();
|
||||
await expect(importDialog(page).locator('.operation h3')).toHaveText(['Inspect a local image', 'Scan an output folder', 'Scan for metadata']);
|
||||
await expect(importDialog(page).locator('.operation').nth(2).locator('p')).toHaveText('Discover and enrich existing files using the configured sources.');
|
||||
await page.keyboard.press('Escape');
|
||||
await openScanReview(page);
|
||||
const dialog = scanReview(page);
|
||||
await expect(dialog.locator('dl.meta-grid > div > dt')).toHaveText(['Sources', 'Scope']);
|
||||
await expect(dialog.locator('dl.meta-grid > div > dd')).toHaveText(['2 output folders', 'New and unresolved files']);
|
||||
await expect(dialog.locator('#modal-body > p.small')).toHaveText('Enrich positive / negative associations, reuse canonical text and report unchanged files. Do not inflate run counts or recreate existing thumbnails.');
|
||||
await expect(dialog.locator('.notice h3')).toHaveText('Back up the database first (recommended)');
|
||||
await expect(dialog.locator('.dialog-actions button')).toHaveText(['Cancel', 'Start scan']);
|
||||
await expect(dialog.getByRole('button', { name: 'Start scan', exact: true })).toHaveClass(/ca-primary/);
|
||||
|
||||
// Back up now downloads the existing backup, toasts, and keeps the dialog open for the scan.
|
||||
const download = page.waitForEvent('download');
|
||||
await dialog.getByRole('button', { name: 'Back up now', exact: true }).click();
|
||||
expect((await download).suggestedFilename()).toMatch(/^prompts_backup_\d{8}_\d{6}\.db$/);
|
||||
await expect(toast(page)).toHaveText('Database backup downloaded.');
|
||||
await expect(dialog).toBeVisible();
|
||||
// A failed backup says so and downloads nothing.
|
||||
await page.route('**/prompt_manager/backup', route => route.fulfill({ status: 500, json: { success: false, error: 'disk full' } }));
|
||||
await dialog.getByRole('button', { name: 'Back up now', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('The database could not be backed up. Try again.');
|
||||
|
||||
// Cancel closes it and starts nothing.
|
||||
await dialog.getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(scanReview(page)).toHaveCount(0);
|
||||
await expect(page.getByRole('button', { name: 'Import', exact: true })).toBeFocused();
|
||||
const jobs = await (await request.get('/prompt_manager/jobs')).json();
|
||||
expect(jobs.jobs).toEqual([]);
|
||||
});
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {inventoryOf,mapConcept,diffInventory} from './library-editor-canonical-comparison.mjs';
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
|
||||
const route='/prompt_manager/workspace#/library';
|
||||
const cards=page=>page.locator('article[data-prompt-id]');
|
||||
@@ -25,6 +25,8 @@ async function loadShelfPages(page,more,pages) {
|
||||
await more.evaluate(node=>node.click());await response;await expect(more).toHaveAttribute('aria-disabled','false');
|
||||
}
|
||||
}
|
||||
/** Resolves in the rendering frame after every frame callback the page had already queued (callbacks run in request order). */
|
||||
const nextFrame=page=>page.evaluate(()=>new Promise(resolve=>requestAnimationFrame(()=>resolve())));
|
||||
async function openHistory(page,text='Cat portrait') {
|
||||
await cards(page).filter({has:page.getByText(text,{exact:true})}).getByRole('button',{name:'Explore history',exact:true}).click();
|
||||
await expect(history(page)).toBeVisible();
|
||||
@@ -103,7 +105,8 @@ test('library infinite loading keeps unique parents and keyboard-accessible hist
|
||||
const appended=await (await nextPage).json();expect(appended.items.length).toBeGreaterThan(0);hasMore=appended.has_more;
|
||||
if(hasMore)await expect(more).toHaveAttribute('aria-disabled','false');
|
||||
}
|
||||
await expect(more).toBeHidden();expect(await cards(page).count()).toBeLessThan(80);
|
||||
// The last page's render queued the window update for the next frame; count after that frame, not before it.
|
||||
await expect(more).toBeHidden();await nextFrame(page);expect(await cards(page).count()).toBeLessThan(80);
|
||||
const ids=await cards(page).evaluateAll(nodes=>nodes.map(node=>node.dataset.promptId));expect(new Set(ids).size).toBe(ids.length);
|
||||
await expect(page.getByRole('button',{name:/next page|previous page/i})).toHaveCount(0);
|
||||
const firstId=await cards(page).first().getAttribute('data-prompt-id');
|
||||
@@ -376,14 +379,18 @@ async function observeLibrary(page, hold=false) {
|
||||
holdFrames(){holdingFrames=true;},
|
||||
releaseFrames(){holdingFrames=false;const callbacks=[...heldFrames.values()];heldFrames.clear();for(const callback of callbacks)callback();},
|
||||
equal(){for(const record of records){const entries=[...record.latest.values()].filter(entry=>record.targets.has(entry.target));if(entries.length)record.callback(entries,record.native);}},
|
||||
// Resolves once two consecutive rendering frames find no Library frame queued. Each check runs inside a native
|
||||
// frame callback, so a frame the page queues later in that update (a scroll event, a ResizeObserver delivery) is
|
||||
// still pending there. A timer is not a frame: under CPU load a 60 ms wait could end before the scroll's frame.
|
||||
// A frame that never drains (a rAF loop, or settling while frames are held) fails after `limit` frames with the stats.
|
||||
settled(limit=600){return new Promise((resolve,reject)=>{let quiet=0,frames=0;const check=()=>{frames++;quiet=pending.size?0:quiet+1;
|
||||
if(quiet>=2)resolve();else if(frames>=limit)reject(new Error(`Library never settled within ${limit} frames: ${JSON.stringify(window.libraryObservation.stats())}`));else raf(check);};raf(check);});},
|
||||
stats(){return {requested,pending:pending.size,heldFrames:heldFrames.size,measurements:records.reduce((sum,record)=>sum+record.entries.size,0),mutations,unmounted,targets:records.reduce((sum,record)=>sum+record.targets.size,0)};},
|
||||
};
|
||||
},{hold});
|
||||
}
|
||||
async function settleLibrary(page) {
|
||||
await expect.poll(async()=>page.evaluate(()=>window.libraryObservation.stats().pending)).toBe(0);
|
||||
await page.evaluate(()=>new Promise(resolve=>setTimeout(resolve,60)));
|
||||
await expect.poll(async()=>page.evaluate(()=>window.libraryObservation.stats().pending)).toBe(0);
|
||||
await page.evaluate(()=>window.libraryObservation.settled());
|
||||
}
|
||||
async function firstVisibleParent(page) {
|
||||
return cards(page).evaluateAll(nodes=>{
|
||||
@@ -683,6 +690,17 @@ test('title and Explore history are buttons that open history',async({page,reque
|
||||
await page.keyboard.press('ArrowLeft');await expect(cards(page).nth(1).locator('.card-title > button')).toBeFocused();
|
||||
});
|
||||
|
||||
test('settling fails with the observation stats when a Library frame stays queued',async({page,request})=>{
|
||||
await request.post('/__test__/reset',{data:{native_library:true,extra_prompts:200}});
|
||||
await observeLibrary(page);await page.addInitScript(()=>{window.IntersectionObserver=undefined;});
|
||||
await page.goto(route);await expect(cards(page).first()).toBeVisible();await settleLibrary(page);
|
||||
await page.evaluate(()=>{window.libraryObservation.holdFrames();dispatchEvent(new Event('scroll'));});
|
||||
await expect.poll(()=>page.evaluate(()=>window.libraryObservation.stats().heldFrames)).toBeGreaterThan(0);
|
||||
const failure=await page.evaluate(()=>window.libraryObservation.settled(30).then(()=>'settled',error=>error.message));
|
||||
expect(failure).toMatch(/^Library never settled within 30 frames: \{.*"heldFrames":[1-9]/);
|
||||
await page.evaluate(()=>window.libraryObservation.releaseFrames());await settleLibrary(page);
|
||||
});
|
||||
|
||||
test('virtualized rows keep measured heights with the clamp',async({page,request,isMobile})=>{
|
||||
await request.post('/__test__/reset',{data:{native_library:true,extra_prompts:200}});
|
||||
await observeLibrary(page);await page.addInitScript(()=>{window.IntersectionObserver=undefined;});
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
|
||||
// Saved views (#249 D16, maintainer ruling 2026-10-21): the toolbar's Save view (concept app.js a359, a1330–1346),
|
||||
// the sidebar SAVED VIEWS rows (a331–335) and `#/saved/<id>` (a1158–1162), on Images and the Prompt library.
|
||||
|
||||
@@ -3,12 +3,12 @@ import { readFile } from 'node:fs/promises';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { captureCanonicalState, historyGeometry, compareCanonicalHistory, nativeDialogState, exerciseCurrentPairSend, assertRecordedHistory } from './history-canonical-comparison.mjs';
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
|
||||
test.use({ locale: 'en-US', timezoneId: 'UTC', reducedMotion: 'reduce' });
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
const primary = ['images', 'library', 'tags', 'inspect'];
|
||||
const tools = ['system', 'overview'];
|
||||
const tools = ['activity', 'system', 'overview'];
|
||||
|
||||
async function noOverflow(page) {
|
||||
expect(await page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth)).toBeLessThanOrEqual(1);
|
||||
@@ -556,7 +556,7 @@ test('approved icons are decorative safe SVGs and reject unknown vocabulary', as
|
||||
await page.goto(`${workspace}#/images`);
|
||||
const icons = await page.evaluate(async () => {
|
||||
const { createWorkspaceIcon } = await import('/prompt_manager/js/workspace/icons.mjs');
|
||||
const names = ['images', 'library', 'tags', 'upload', 'overview', 'settings', 'grid', 'list', 'masonry', 'search', 'info', 'copy', 'branch', 'star', 'tagImage', 'folder', 'history', 'review'];
|
||||
const names = ['images', 'library', 'tags', 'upload', 'overview', 'activity', 'settings', 'grid', 'list', 'masonry', 'search', 'info', 'copy', 'branch', 'star', 'tagImage', 'folder', 'history', 'review'];
|
||||
const otherDocument = document.implementation.createHTMLDocument();
|
||||
const rendered = names.map(name => {
|
||||
const icon = createWorkspaceIcon(name, otherDocument);
|
||||
@@ -575,7 +575,8 @@ test('approved icons are decorative safe SVGs and reject unknown vocabulary', as
|
||||
expect(icon).toMatchObject({ svg: true, className: 'icon', owner: true, viewBox: '0 0 24 24', hidden: 'true', focusable: 'false', fill: 'none', stroke: 'currentColor', width: '1.65', children: 1 });
|
||||
expect(icon.path.length).toBeGreaterThan(10);
|
||||
}
|
||||
// D28 ruling: Overview's own outline home glyph, not the concept's `activity` pulse ('M3 12h4l3-8 4 16 3-8h4').
|
||||
// D28 ruling: Overview's own outline home glyph; the concept's `activity` pulse is the Activity row's.
|
||||
expect(icons.rendered.find(icon => icon.name === 'activity').path).toBe('M3 12h4l3-8 4 16 3-8h4');
|
||||
expect(icons.rendered.find(icon => icon.name === 'overview').path).toBe('M3 11 12 4l9 7M5 10v10h14V10M10 20v-6h4v6');
|
||||
expect(new Set(icons.rendered.map(icon => icon.path)).size).toBe(icons.rendered.length);
|
||||
});
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {pageInventoryOf,inventoryOf,diffInventory,mapVocabularyConcept,mapVocabularyDialog} from './vocabulary-canonical-comparison.mjs';
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
const statistic = (page, label) => page.locator('#workspace-main dt')
|
||||
@@ -20,6 +20,8 @@ const routes = [
|
||||
['system', 'System', '/prompt_manager/admin'],
|
||||
// #249 D51: SAVED VIEWS' Favorites, Images filtered to favorites.
|
||||
['favorites', 'Favorites', null],
|
||||
// #249 Activity-1: the concept's Activity page in the tools-and-system group.
|
||||
['activity', 'Activity', null],
|
||||
];
|
||||
|
||||
async function countThemeWrites(page) {
|
||||
@@ -46,6 +48,8 @@ test.beforeEach(async ({ page, request, browserName }) => {
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, browserName }) => {
|
||||
// A route handler still running at teardown must not fail the test it outlived.
|
||||
await page.unrouteAll({ behavior: 'ignoreErrors' });
|
||||
await saveCoverage(page, browserName);
|
||||
expect(page.workspaceErrors).toEqual([]);
|
||||
});
|
||||
@@ -176,7 +180,8 @@ test('theme System follows ComfyUI and persists an explicit light or dark prefer
|
||||
await expect(theme).toHaveAttribute('data-ca-theme', 'light');
|
||||
});
|
||||
|
||||
// Task A4 (D4, D5): the concept topbar (index.html 45-63, style.css 225-277) without the prototype badge or Activity.
|
||||
// Task A4 (D4, D29): the concept topbar (index.html 45-63, style.css 225-277) without the prototype badge; Activity
|
||||
// (D52, D53) shows its button, and its pill only while a job runs.
|
||||
test('topbar matches the concept actions', async ({ page }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
await page.goto(`${workspace}#/images`);
|
||||
@@ -204,7 +209,7 @@ test('topbar matches the concept actions', async ({ page }) => {
|
||||
expect(shape).toEqual({
|
||||
children: ['div.breadcrumbs', 'div.top-actions'],
|
||||
crumbs: ['ComfyAssets', 'span:/', 'span#workspace-breadcrumb:Images'],
|
||||
actions: ['label.theme-label', 'a.canvas-link'],
|
||||
actions: ['a.activity-pill', 'button.', 'label.theme-label', 'a.canvas-link'],
|
||||
label: 'Theme', labelFor: 'theme',
|
||||
options: [['dark', 'Dark'], ['light', 'Light'], ['system', 'System']],
|
||||
arrow: [['↗', 'true']],
|
||||
@@ -218,11 +223,12 @@ test('topbar matches the concept actions', async ({ page }) => {
|
||||
await expect(link).toHaveAttribute('href', '/');
|
||||
expect(await link.evaluate(node => node.href === `${location.origin}/`)).toBe(true);
|
||||
await expect(page.getByRole('combobox', { name: 'Appearance' })).toHaveCount(0);
|
||||
await expect(page.getByRole('button', { name: /^Activity/ })).toHaveCount(0);
|
||||
await expect(page.locator('.concept-badge, #activity-open')).toHaveCount(0);
|
||||
await expect(page.getByRole('button', { name: 'Activity', exact: true })).toHaveCount(1);
|
||||
await expect(page.locator('#activity-pill')).toBeHidden();
|
||||
await expect(page.locator('.concept-badge')).toHaveCount(0);
|
||||
await page.locator('nav [data-route="library"]').click();
|
||||
await expect(page.locator('#workspace-breadcrumb')).toHaveText('Prompt library');
|
||||
await expect(topbar.locator('.top-actions > *')).toHaveCount(2);
|
||||
await expect(topbar.locator('.top-actions > *')).toHaveCount(4);
|
||||
});
|
||||
|
||||
// Concept style.css 1375-1391: at <=700px the breadcrumbs hide, the actions spread across the bar and the Theme
|
||||
@@ -242,13 +248,18 @@ test('mobile topbar hides breadcrumbs and spreads actions', async ({ page }) =>
|
||||
expected: bar.width - 28,
|
||||
label: getComputedStyle(node.querySelector('.theme-label')).fontSize,
|
||||
select: getComputedStyle(node.querySelector('#theme')).fontSize,
|
||||
labelLeft: box(node.querySelector('.theme-label')).left - bar.left,
|
||||
activityLeft: box(node.querySelector('#activity-open')).left - bar.left,
|
||||
actionsGap: [box(node.querySelector('#activity-open')).right, box(node.querySelector('.theme-label')).left,
|
||||
box(node.querySelector('.theme-label')).right, box(node.querySelector('.canvas-link')).left],
|
||||
linkRight: bar.right - box(node.querySelector('.canvas-link')).right,
|
||||
};
|
||||
});
|
||||
expect(layout).toMatchObject({ padding: '14px', actions: { justify: 'space-between', gap: '8px' }, label: '0px', select: '11px' });
|
||||
expect(Math.abs(layout.actions.width - layout.expected)).toBeLessThanOrEqual(1);
|
||||
expect(Math.abs(layout.labelLeft - 14)).toBeLessThanOrEqual(1);
|
||||
// The concept's three items spread with space-between: Activity first, Theme between equal gaps, ComfyUI last.
|
||||
expect(Math.abs(layout.activityLeft - 14)).toBeLessThanOrEqual(1);
|
||||
const [activityRight, themeLeft, themeRight, linkLeft] = layout.actionsGap;
|
||||
expect(Math.abs((themeLeft - activityRight) - (linkLeft - themeRight))).toBeLessThanOrEqual(1);
|
||||
expect(Math.abs(layout.linkRight - 14)).toBeLessThanOrEqual(1);
|
||||
await expect(page.getByRole('link', { name: 'ComfyUI', exact: true })).toBeVisible();
|
||||
await theme.selectOption('light');
|
||||
@@ -506,18 +517,18 @@ test('a slow Images count does not delay the Library and Tags badges', async ({
|
||||
await expect(navCount(page, 'images')).toHaveText(String(totals.images));
|
||||
});
|
||||
|
||||
test('sidebar bottom holds System and Overview with a neutral connection', async ({ page }) => {
|
||||
test('sidebar bottom holds Activity, System and Overview with a neutral connection', async ({ page }) => {
|
||||
await page.goto(`${workspace}#/system`);
|
||||
const bottom = page.locator('aside.sidebar > .sidebar-bottom');
|
||||
expect(await childShape(bottom)).toEqual(['nav#system-nav', 'div.connection']);
|
||||
const system = page.locator('nav#system-nav');
|
||||
await expect(system).toHaveAttribute('aria-label', 'Tools and system');
|
||||
expect(await system.locator(':scope > a').evaluateAll(nodes => nodes.map(node => [node.getAttribute('href'), node.textContent]))).toEqual([
|
||||
['#/system', 'System'], ['#/overview', 'Overview'],
|
||||
['#/activity', 'Activity'], ['#/system', 'System'], ['#/overview', 'Overview'],
|
||||
]);
|
||||
await expect(system.locator('.nav-count')).toHaveCount(0);
|
||||
await expect(system.locator('a[href="#/system"]')).toHaveAttribute('aria-current', 'page');
|
||||
await expect(system.locator('svg.icon')).toHaveCount(2);
|
||||
await expect(system.locator('svg.icon')).toHaveCount(3);
|
||||
const connection = bottom.locator('.connection');
|
||||
expect(await childShape(connection)).toEqual(['i', 'div']);
|
||||
await expect(connection.locator(':scope > div')).toHaveText('ComfyUI workspaceYour library, on your server');
|
||||
@@ -679,8 +690,14 @@ async function serveUnderProxyMount(page) {
|
||||
await page.route(url => url.pathname.startsWith(`${PROXY_MOUNT}/`), async route => {
|
||||
const url = new URL(route.request().url());
|
||||
url.pathname = url.pathname.slice(PROXY_MOUNT.length);
|
||||
const response = await route.fetch({ url: url.href });
|
||||
await route.fulfill({ response });
|
||||
// A request still in flight when the test ends (a thumbnail the last route started) meets a page that is
|
||||
// closing: its fetched response is disposed under it. Only that teardown error is ignored.
|
||||
try {
|
||||
const response = await route.fetch({ url: url.href });
|
||||
await route.fulfill({ response });
|
||||
} catch (error) {
|
||||
if (!/has been disposed|has been closed|Target page, context or browser has been closed/.test(String(error?.message))) throw error;
|
||||
}
|
||||
});
|
||||
return escaped;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
"""Fake Activity jobs for the browser tests (#249 Activity-1).
|
||||
|
||||
A fake job is a real registry job whose runner waits for commands instead of
|
||||
reading files, so a test sets its progress and counters exactly:
|
||||
|
||||
- ``POST /__test__/jobs/start`` starts one (``kind``, ``title``, ``total``,
|
||||
``output_folders``) and returns its summary, or 409 while one runs;
|
||||
- ``POST /__test__/jobs/{id}/step`` records ``records`` (``{group, path,
|
||||
root, root_index, detail}`` file outcomes, counted, listed and logged as the scan
|
||||
records them) and ``bulk`` (``{group, count, root}``: ``count`` numbered
|
||||
rows), sets ``processed`` (and ``total``) and ``counters``, then publishes
|
||||
the progress to every subscriber;
|
||||
- ``POST /__test__/jobs/{id}/finish`` ends it as ``finished``, ``cancelled``
|
||||
or ``failed``; a cancel through the real route ends it ``cancelled``;
|
||||
- ``POST /__test__/jobs/restart`` acts out a ComfyUI restart: the open job
|
||||
streams drop without a final event and the registry starts empty.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
EVENTS_PATH = re.compile(r"^/prompt_manager/jobs/[0-9a-f]{32}/events$")
|
||||
FINAL_STATES = {"finished", "cancelled", "failed"}
|
||||
POLL_SECONDS = 0.02
|
||||
|
||||
|
||||
PREFIXES = {"added": "+", "known": "=", "review": "?", "errors": "!"}
|
||||
PREFIXES["no_metadata"] = "-"
|
||||
ROOT_INDEXES = {"output": 0, "input": 1}
|
||||
|
||||
|
||||
def outcome_records(data):
|
||||
"""The step's file outcomes: its ``records``, then its ``bulk`` rows."""
|
||||
records = data.get("records", [])
|
||||
bulk = data.get("bulk")
|
||||
if not isinstance(records, list):
|
||||
raise web.HTTPBadRequest()
|
||||
rows = []
|
||||
for record in records:
|
||||
if not isinstance(record, dict) or record.get("group") not in PREFIXES:
|
||||
raise web.HTTPBadRequest()
|
||||
root = str(record.get("root", "output"))
|
||||
rows.append(
|
||||
{
|
||||
"group": record["group"],
|
||||
"path": str(record.get("path", "")),
|
||||
"root": root,
|
||||
# The fixture's listing order: output is root 0, input root 1.
|
||||
"root_index": record.get("root_index", ROOT_INDEXES.get(root)),
|
||||
"detail": str(record.get("detail", "")),
|
||||
}
|
||||
)
|
||||
if bulk is not None:
|
||||
if not isinstance(bulk, dict) or bulk.get("group") not in PREFIXES:
|
||||
raise web.HTTPBadRequest()
|
||||
total = bulk.get("count")
|
||||
if isinstance(total, bool) or not isinstance(total, int) or total < 0:
|
||||
raise web.HTTPBadRequest()
|
||||
rows.extend(
|
||||
{
|
||||
"group": bulk["group"],
|
||||
"path": f"bulk/file_{index:05d}.png",
|
||||
"root": str(bulk.get("root", "output")),
|
||||
"root_index": 0,
|
||||
"detail": f"Bulk row {index}",
|
||||
}
|
||||
for index in range(total)
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def register_activity_commands(app, routes, api):
|
||||
from py.api.jobs import SCAN_GROUP_NAMES, JobBusy, JobRegistry
|
||||
|
||||
streams = set()
|
||||
endings = {}
|
||||
|
||||
@web.middleware
|
||||
async def track_streams(request, handler):
|
||||
if not EVENTS_PATH.match(request.path):
|
||||
return await handler(request)
|
||||
transport = request.transport
|
||||
streams.add(transport)
|
||||
try:
|
||||
return await handler(request)
|
||||
finally:
|
||||
streams.discard(transport)
|
||||
|
||||
app.middlewares.append(track_streams)
|
||||
|
||||
def fake_runner(ending):
|
||||
async def run(job):
|
||||
while not ending.done() and not job.cancelled:
|
||||
await asyncio.sleep(POLL_SECONDS)
|
||||
state = ending.result() if ending.done() else "cancelled"
|
||||
job.publish({"type": "complete", "progress": job.progress})
|
||||
job.finish(state)
|
||||
|
||||
return run
|
||||
|
||||
def job_or_404(request):
|
||||
job = api._jobs.get(request.match_info["job_id"])
|
||||
if job is None or job.id not in endings:
|
||||
raise web.HTTPNotFound()
|
||||
return job
|
||||
|
||||
async def body(request):
|
||||
data = await request.json() if request.can_read_body else {}
|
||||
if not isinstance(data, dict):
|
||||
raise web.HTTPBadRequest()
|
||||
return data
|
||||
|
||||
def count(value, default=0):
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
||||
raise web.HTTPBadRequest()
|
||||
return value
|
||||
|
||||
@routes.post("/__test__/jobs/start")
|
||||
async def start_job(request):
|
||||
data = await body(request)
|
||||
ending = asyncio.get_running_loop().create_future()
|
||||
try:
|
||||
job = api._jobs.start(
|
||||
str(data.get("kind", "scan")),
|
||||
str(data.get("title", "Scan for metadata")),
|
||||
record_changing=True,
|
||||
scope={
|
||||
"files": str(data.get("files", "new")),
|
||||
"output_folders": count(data.get("output_folders"), 2),
|
||||
},
|
||||
runner=fake_runner(ending),
|
||||
groups=SCAN_GROUP_NAMES,
|
||||
)
|
||||
except JobBusy:
|
||||
raise web.HTTPConflict()
|
||||
job.update(total=count(data.get("total")))
|
||||
endings[job.id] = ending
|
||||
return web.json_response({"success": True, "job": job.summary()})
|
||||
|
||||
@routes.post("/__test__/jobs/{job_id}/step")
|
||||
async def step_job(request):
|
||||
job = job_or_404(request)
|
||||
data = await body(request)
|
||||
total = count(data.get("total"), job.total)
|
||||
processed = count(data.get("processed"), job.processed)
|
||||
counters = data.get("counters", {})
|
||||
if not isinstance(counters, dict) or set(counters) - set(job.counters):
|
||||
raise web.HTTPBadRequest()
|
||||
for record in outcome_records(data):
|
||||
group = record["group"]
|
||||
path = record["path"]
|
||||
detail = record["detail"]
|
||||
job.record(group, record, f"{PREFIXES[group]} {path} · {detail}")
|
||||
for key, value in counters.items():
|
||||
job.counters[key] = count(value)
|
||||
progress = processed * 100 // total if total else 0
|
||||
job.update(
|
||||
progress=progress,
|
||||
processed=processed,
|
||||
total=total,
|
||||
status_line=f"Processing file {processed:,} of {total:,}",
|
||||
)
|
||||
job.publish({"type": "progress", "progress": progress})
|
||||
return web.json_response({"success": True, "job": job.summary()})
|
||||
|
||||
@routes.post("/__test__/jobs/{job_id}/finish")
|
||||
async def finish_job(request):
|
||||
job = job_or_404(request)
|
||||
state = (await body(request)).get("state", "finished")
|
||||
if state not in FINAL_STATES:
|
||||
raise web.HTTPBadRequest()
|
||||
ending = endings[job.id]
|
||||
if not ending.done():
|
||||
ending.set_result(state)
|
||||
while job.running:
|
||||
await asyncio.sleep(POLL_SECONDS)
|
||||
return web.json_response({"success": True, "job": job.summary()})
|
||||
|
||||
@routes.post("/__test__/jobs/restart")
|
||||
async def restart(request):
|
||||
old = api._jobs
|
||||
api._jobs = JobRegistry(api.logger)
|
||||
endings.clear()
|
||||
for transport in list(streams):
|
||||
transport.close()
|
||||
tasks = [job.task for job in old.list() if job.task and not job.task.done()]
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
return web.json_response({"success": True})
|
||||
@@ -170,6 +170,9 @@ _TABLES = (
|
||||
"execution_context_texts",
|
||||
"saved_views",
|
||||
"image_favorites",
|
||||
"media_image_tags",
|
||||
"media_annotation_bytes",
|
||||
"media_annotation_files",
|
||||
"image_prompt_pairs",
|
||||
"prompt_pair_tags",
|
||||
"prompt_pairs",
|
||||
|
||||
@@ -28,6 +28,9 @@ EDITOR_METADATA = {
|
||||
_TABLES = (
|
||||
"saved_views",
|
||||
"image_favorites",
|
||||
"media_image_tags",
|
||||
"media_annotation_bytes",
|
||||
"media_annotation_files",
|
||||
"image_prompt_pairs",
|
||||
"prompt_pair_tags",
|
||||
"prompt_pairs",
|
||||
|
||||
@@ -114,6 +114,9 @@ def seed_workspace_visual(api: PromptManagerAPI, root: Path) -> None:
|
||||
for table in (
|
||||
"saved_views",
|
||||
"image_favorites",
|
||||
"media_image_tags",
|
||||
"media_annotation_bytes",
|
||||
"media_annotation_files",
|
||||
"image_prompt_pairs",
|
||||
"prompt_pair_tags",
|
||||
"prompt_pairs",
|
||||
|
||||
@@ -81,6 +81,9 @@ MAX_EXTRA_CATEGORIES = 200
|
||||
_TABLES = (
|
||||
"saved_views",
|
||||
"image_favorites",
|
||||
"media_image_tags",
|
||||
"media_annotation_bytes",
|
||||
"media_annotation_files",
|
||||
"image_prompt_pairs",
|
||||
"prompt_pair_tags",
|
||||
"prompt_pairs",
|
||||
|
||||
@@ -0,0 +1,371 @@
|
||||
// Activity (#249 Activity-1): the session job store behind the top-bar button and pill and the Activity page.
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
|
||||
const source = import('../../web/js/workspace/activity-store.mjs');
|
||||
|
||||
const ID_A = 'a'.repeat(32);
|
||||
const ID_B = 'b'.repeat(32);
|
||||
const ID_C = 'c'.repeat(32);
|
||||
|
||||
function summary(overrides = {}) {
|
||||
return { id: ID_A, kind: 'scan', title: 'Scan for metadata', state: 'running', started_at: 1000, finished_at: null,
|
||||
progress: 0, status_line: '', processed: 0, total: 0,
|
||||
counters: { added: 0, known: 0, review: 0, no_metadata: 0, errors: 0 }, scope: { files: 'new', output_folders: 2 },
|
||||
...overrides };
|
||||
}
|
||||
|
||||
function apiError(status, message, code) {
|
||||
return Object.assign(new Error(message), { status, code });
|
||||
}
|
||||
|
||||
/** A fake world: the API's answers, every opened stream, the pending timers and the toasts. */
|
||||
function harness() {
|
||||
const world = { list: [], listError: null, posts: [], postResult: null, postError: null, streams: [], timers: [],
|
||||
toasts: [], gets: 0 };
|
||||
const cleanups = [];
|
||||
const controller = new AbortController();
|
||||
const scope = { signal: controller.signal, cleanup: fn => cleanups.push(fn) };
|
||||
const api = {
|
||||
async get(url, options) {
|
||||
assert.equal(url, '/prompt_manager/jobs');
|
||||
assert.equal(options.cache, 'no-store');
|
||||
world.gets += 1;
|
||||
if (world.listError) throw world.listError;
|
||||
return { success: true, jobs: world.list };
|
||||
},
|
||||
async post(url, body) {
|
||||
world.posts.push([url, body]);
|
||||
if (world.postError) throw world.postError;
|
||||
return world.postResult;
|
||||
},
|
||||
};
|
||||
const connect = url => {
|
||||
const stream = { url, closed: false, onmessage: null, onerror: null, close() { this.closed = true; } };
|
||||
world.streams.push(stream);
|
||||
return stream;
|
||||
};
|
||||
const timers = {
|
||||
setTimeout(fn, delay) { const timer = { fn, delay, cleared: false }; world.timers.push(timer); return timer; },
|
||||
clearTimeout(timer) { timer.cleared = true; },
|
||||
};
|
||||
world.dispose = () => { controller.abort(); cleanups.forEach(fn => fn()); };
|
||||
world.fire = async () => {
|
||||
const timer = world.timers.find(item => !item.cleared && !item.fired);
|
||||
timer.fired = true;
|
||||
timer.fn();
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
return timer.delay;
|
||||
};
|
||||
world.open = () => world.streams.filter(stream => !stream.closed);
|
||||
return { world, api, scope, connect, timers, pollMs: 0, notify: message => world.toasts.push(message) };
|
||||
}
|
||||
|
||||
const send = (stream, job) => stream.onmessage({ data: JSON.stringify(job) });
|
||||
|
||||
test('store merges list and SSE updates and caps backoff', async () => {
|
||||
const { createActivityStore, BACKOFF_MAX_MS, LOST_DETAIL } = await source;
|
||||
const { world, ...deps } = harness();
|
||||
world.list = [summary(), summary({ id: ID_B, state: 'finished', finished_at: 900, counters: { added: 3, errors: 0 } })];
|
||||
const store = createActivityStore(deps);
|
||||
let changes = 0;
|
||||
store.subscribe(() => { changes += 1; });
|
||||
await store.load();
|
||||
assert.equal(store.status, 'ready');
|
||||
assert.deepEqual(store.jobs().map(job => [job.id, job.state]), [[ID_A, 'running'], [ID_B, 'finished']]);
|
||||
assert.deepEqual(store.running().map(job => job.id), [ID_A]);
|
||||
// One stream per running job only; a finished job is not followed.
|
||||
assert.deepEqual(world.streams.map(stream => stream.url), [`/prompt_manager/jobs/${ID_A}/events`]);
|
||||
assert.ok(changes >= 1);
|
||||
|
||||
// An SSE summary replaces the listed one; garbage and another job's summary are ignored.
|
||||
const [stream] = world.streams;
|
||||
send(stream, summary({ progress: 42, processed: 8240, total: 19622 }));
|
||||
stream.onmessage({ data: 'not json' });
|
||||
send(stream, summary({ id: ID_B, progress: 99 }));
|
||||
assert.equal(store.find(ID_A).progress, 42);
|
||||
assert.equal(store.find(ID_B).state, 'finished');
|
||||
assert.deepEqual(world.toasts, []);
|
||||
|
||||
// The end of the job: the finish toast, and the stream is closed.
|
||||
send(stream, summary({ state: 'finished', progress: 100, finished_at: 2000, counters: { added: 1420, errors: 18 } }));
|
||||
assert.deepEqual(world.toasts, ['Scan finished · 1,420 added · 18 errors']);
|
||||
assert.equal(stream.closed, true);
|
||||
assert.deepEqual(store.running(), []);
|
||||
|
||||
// A new running job from another tab, then the server goes away: every re-read waits longer, up to 30 s.
|
||||
world.list = [summary({ id: ID_C, state: 'running' })];
|
||||
await store.load();
|
||||
const second = world.open()[0];
|
||||
assert.equal(second.url, `/prompt_manager/jobs/${ID_C}/events`);
|
||||
world.listError = new Error('connection refused');
|
||||
second.onerror();
|
||||
assert.equal(second.closed, true);
|
||||
const delays = [];
|
||||
for (let index = 0; index < 8; index += 1) delays.push(await world.fire());
|
||||
assert.deepEqual(delays, [1000, 2000, 4000, 8000, 16000, 30000, 30000, 30000]);
|
||||
assert.ok(delays.every(delay => delay <= BACKOFF_MAX_MS));
|
||||
// While the server cannot be read the running job shows as ended, so no pill stays.
|
||||
assert.equal(store.status, 'error');
|
||||
assert.equal(store.find(ID_C).state, 'lost');
|
||||
assert.deepEqual(store.running(), []);
|
||||
assert.equal(typeof LOST_DETAIL, 'string');
|
||||
|
||||
// The server is back without the job (a restart): it stays ended; finished jobs the server no longer lists are
|
||||
// dropped, so the list never outgrows the server's (review M3).
|
||||
world.listError = null;
|
||||
world.list = [];
|
||||
await world.fire();
|
||||
assert.equal(store.status, 'ready');
|
||||
assert.deepEqual(store.jobs().map(job => [job.id, job.state]), [[ID_C, 'lost']]);
|
||||
assert.equal(world.timers.filter(timer => !timer.cleared && !timer.fired).length, 0, 'no retry after a good read');
|
||||
assert.equal(world.toasts.length, 1, 'an ended job has no finish toast');
|
||||
world.dispose();
|
||||
});
|
||||
|
||||
test('a dropped stream of a still running job reattaches after the backoff, then resets it', async () => {
|
||||
const { createActivityStore } = await source;
|
||||
const { world, ...deps } = harness();
|
||||
world.list = [summary()];
|
||||
const store = createActivityStore(deps);
|
||||
await store.load();
|
||||
world.streams[0].onerror();
|
||||
assert.equal(await world.fire(), 1000);
|
||||
const again = world.open();
|
||||
assert.equal(again.length, 1);
|
||||
assert.equal(store.find(ID_A).state, 'running');
|
||||
again[0].onerror();
|
||||
assert.equal(await world.fire(), 2000);
|
||||
send(world.open()[0], summary({ progress: 5 }));
|
||||
world.open()[0].onerror();
|
||||
assert.equal(await world.fire(), 1000, 'a received message resets the backoff');
|
||||
// A second drop while a re-read is already scheduled does not schedule another.
|
||||
world.open()[0].onerror();
|
||||
store.load();
|
||||
assert.equal(world.timers.filter(timer => !timer.cleared && !timer.fired).length, 1);
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
world.dispose();
|
||||
assert.ok(world.streams.every(stream => stream.closed), 'dispose closes every stream');
|
||||
assert.ok(world.timers.every(timer => timer.cleared || timer.fired), 'dispose clears the pending re-read');
|
||||
});
|
||||
|
||||
test('start returns busy with the running job', async () => {
|
||||
const { createActivityStore } = await source;
|
||||
const { world, ...deps } = harness();
|
||||
const store = createActivityStore(deps);
|
||||
world.postResult = { success: true, job: summary() };
|
||||
const started = await store.start('scan');
|
||||
assert.equal(started.job.id, ID_A);
|
||||
assert.deepEqual(world.posts, [['/prompt_manager/jobs/scan', {}]]);
|
||||
assert.deepEqual(world.toasts, ['Scan started.']);
|
||||
assert.equal(world.open().length, 1, 'the started job is followed');
|
||||
|
||||
world.postError = apiError(409, 'Scan for metadata is already running', 'busy');
|
||||
world.list = [summary()];
|
||||
const busy = await store.start('scan');
|
||||
assert.equal(busy.busy.id, ID_A);
|
||||
assert.equal(busy.busy.title, 'Scan for metadata');
|
||||
assert.equal(busy.message, 'Scan for metadata is already running');
|
||||
assert.equal(world.toasts.length, 1, 'a refused start has no start toast');
|
||||
|
||||
world.postError = apiError(500, 'boom', null);
|
||||
await assert.rejects(store.start('scan'), /boom/);
|
||||
world.postError = null;
|
||||
world.postResult = { success: true, job: { id: 'nope' } };
|
||||
await assert.rejects(store.start('scan'), /Invalid job/);
|
||||
|
||||
world.postResult = { success: true, cancellation_requested: true };
|
||||
assert.equal(await store.cancel(ID_A), true);
|
||||
assert.deepEqual(world.posts.at(-1), [`/prompt_manager/jobs/${ID_A}/cancel`, {}]);
|
||||
world.postResult = { success: true, cancellation_requested: false };
|
||||
assert.equal(await store.cancel(ID_A), false);
|
||||
world.dispose();
|
||||
});
|
||||
|
||||
test('a list that is not a job list counts as a failed read', async () => {
|
||||
const { createActivityStore } = await source;
|
||||
const { world, api, ...deps } = harness();
|
||||
const store = createActivityStore({ ...deps, api: { ...api, get: async () => ({ success: true }) } });
|
||||
await store.load();
|
||||
assert.equal(store.status, 'error');
|
||||
assert.equal(world.timers.length, 1);
|
||||
world.dispose();
|
||||
await store.load();
|
||||
assert.equal(store.status, 'error', 'a disposed store reads nothing');
|
||||
});
|
||||
|
||||
test('a visible tab reads the list again every 15 s; a hidden one skips its turn', async () => {
|
||||
const { createActivityStore, POLL_MS } = await source;
|
||||
const { world, ...deps } = harness();
|
||||
let visible = true;
|
||||
const store = createActivityStore({ ...deps, pollMs: POLL_MS, isVisible: () => visible });
|
||||
await store.load();
|
||||
assert.equal(world.gets, 1);
|
||||
// A job started in another tab appears on the next read.
|
||||
world.list = [summary()];
|
||||
assert.equal(await world.fire(), 15000);
|
||||
assert.equal(world.gets, 2);
|
||||
assert.deepEqual(store.running().map(job => job.id), [ID_A]);
|
||||
visible = false;
|
||||
assert.equal(await world.fire(), 15000);
|
||||
assert.equal(world.gets, 2, 'a hidden tab does not read');
|
||||
visible = true;
|
||||
await world.fire();
|
||||
assert.equal(world.gets, 3);
|
||||
// A manual read replaces the pending poll rather than adding one.
|
||||
await store.load();
|
||||
assert.equal(world.timers.filter(timer => !timer.cleared && !timer.fired).length, 1);
|
||||
world.dispose();
|
||||
assert.ok(world.timers.every(timer => timer.cleared || timer.fired));
|
||||
});
|
||||
|
||||
test('a late or older answer never brings an ended job back, so it is announced once', async () => {
|
||||
const { createActivityStore } = await source;
|
||||
const { world, api, ...deps } = harness();
|
||||
const answers = [];
|
||||
const slowApi = { ...api, get: () => new Promise(resolve => answers.push(resolve)) };
|
||||
const store = createActivityStore({ ...deps, api: slowApi });
|
||||
const first = store.load();
|
||||
const second = store.load();
|
||||
// The newer read answers first; the older one, answering later with stale data, is dropped.
|
||||
answers[1]({ success: true, jobs: [summary()] });
|
||||
await second;
|
||||
answers[0]({ success: true, jobs: [] });
|
||||
await first;
|
||||
assert.deepEqual(store.jobs().map(job => job.state), ['running']);
|
||||
send(world.open()[0], summary({ state: 'finished', counters: { added: 2, errors: 0 } }));
|
||||
assert.deepEqual(world.toasts, ['Scan finished · 2 added · 0 errors']);
|
||||
// A list read sent before the end still says running: the job stays finished and no stream reopens.
|
||||
const third = store.load();
|
||||
answers[2]({ success: true, jobs: [summary()] });
|
||||
await third;
|
||||
assert.equal(store.find(ID_A).state, 'finished');
|
||||
assert.equal(world.open().length, 0);
|
||||
assert.equal(world.toasts.length, 1);
|
||||
// A stale failure is dropped too.
|
||||
const failing = store.load();
|
||||
const newest = store.load();
|
||||
answers[4]({ success: true, jobs: [] });
|
||||
await newest;
|
||||
answers[3](Promise.reject(new Error('late failure')));
|
||||
await failing;
|
||||
assert.equal(store.status, 'ready');
|
||||
world.dispose();
|
||||
});
|
||||
|
||||
test('summaries are validated and the copy follows the spec', async () => {
|
||||
const { parseJob, finishToast, startToast, pillParts, outcomeLine, backoffDelay } = await source;
|
||||
for (const raw of [null, [], { id: 'x', state: 'running' }, summary({ state: 'lost' }), summary({ id: ID_A.toUpperCase() })]) {
|
||||
assert.equal(parseJob(raw), null);
|
||||
}
|
||||
const loose = parseJob({ id: ID_A, state: 'running', progress: 140.4, processed: -1, total: 1.5, counters: { added: 'x', errors: 2 },
|
||||
title: ' ', started_at: 'x', scope: [] });
|
||||
assert.deepEqual(loose, { id: ID_A, kind: '', title: 'Background job', state: 'running', started_at: null, finished_at: null,
|
||||
progress: 100, status_line: '', processed: 0, total: 0, counters: { added: 0, errors: 2 }, scope: {}, listed: {}, log_length: 0 });
|
||||
assert.equal(parseJob(summary({ progress: -3 })).progress, 0);
|
||||
|
||||
const scan = parseJob(summary({ progress: 42, processed: 8240, total: 19622, counters: { added: 1, errors: 1 } }));
|
||||
assert.deepEqual(pillParts(scan), { label: 'Scanning · 42%', files: ' · 8,240 / 19,622', text: 'Scanning · 42% · 8,240 / 19,622' });
|
||||
assert.equal(pillParts({ ...scan, total: 0 }).text, 'Scanning · 42%');
|
||||
assert.equal(pillParts({ ...scan, kind: 'thumbnails', title: 'Thumbnails' }).text, 'Thumbnails · 42% · 8,240 / 19,622');
|
||||
assert.equal(finishToast({ ...scan, state: 'finished' }), 'Scan finished · 1 added · 1 error');
|
||||
assert.equal(finishToast({ ...scan, state: 'cancelled' }), 'Scan cancelled · 1 added');
|
||||
assert.equal(finishToast({ ...scan, state: 'failed' }), 'Scan failed · 1 added · 1 error');
|
||||
assert.equal(finishToast({ ...scan, counters: {}, state: 'finished' }), 'Scan finished · 0 added · 0 errors');
|
||||
const other = { ...scan, kind: 'backup', title: 'Backup' };
|
||||
assert.equal(finishToast({ ...other, state: 'finished' }), 'Backup finished.');
|
||||
assert.equal(finishToast({ ...other, state: 'cancelled' }), 'Backup was cancelled.');
|
||||
assert.equal(finishToast({ ...other, state: 'failed' }), 'Backup failed.');
|
||||
assert.equal(startToast(scan), 'Scan started.');
|
||||
assert.equal(startToast(other), 'Backup started.');
|
||||
assert.equal(outcomeLine(scan), '1 added · 1 error');
|
||||
assert.equal(outcomeLine(other), '');
|
||||
assert.deepEqual([-1, 0, 1, 4, 5, 6, 1e9, 0.5].map(backoffDelay), [1000, 1000, 2000, 16000, 30000, 30000, 30000, 1000]);
|
||||
});
|
||||
|
||||
test('pagehide stops polling, closes streams and aborts the in-flight read; pageshow resumes', async () => {
|
||||
const { createActivityStore } = await source;
|
||||
const { world, api, ...deps } = harness();
|
||||
const lifecycle = new EventTarget();
|
||||
const signals = [];
|
||||
let answer = null;
|
||||
const pendingApi = { ...api, get: (url, options) => {
|
||||
signals.push(options.signal);
|
||||
world.gets += 1;
|
||||
if (world.gets === 1) return Promise.resolve({ success: true, jobs: [summary()] });
|
||||
return new Promise((resolve, reject) => {
|
||||
answer = resolve;
|
||||
options.signal.addEventListener('abort', () => reject(Object.assign(new Error('aborted'), { name: 'AbortError' })));
|
||||
});
|
||||
} };
|
||||
const store = createActivityStore({ ...deps, api: pendingApi, pollMs: 15000, lifecycle });
|
||||
await store.load();
|
||||
assert.equal(world.open().length, 1);
|
||||
const poll = world.timers.find(timer => !timer.cleared && timer.delay === 15000);
|
||||
assert.ok(poll, 'a visible tab polls');
|
||||
// A read in flight when the page goes away (a closed tab, a test teardown) is aborted, nothing is scheduled again.
|
||||
const read = store.load();
|
||||
lifecycle.dispatchEvent(new Event('pagehide'));
|
||||
await read;
|
||||
assert.equal(signals.at(-1).aborted, true);
|
||||
assert.equal(world.open().length, 0);
|
||||
assert.ok(world.timers.every(timer => timer.cleared || timer.fired));
|
||||
assert.equal(store.find(ID_A).state, 'running', 'a hidden page marks nothing lost');
|
||||
await store.load();
|
||||
assert.equal(world.gets, 2, 'a suspended store reads nothing');
|
||||
// Back from the cache, it reads the list again and follows the job.
|
||||
const resumed = Object.assign(new Event('pageshow'), { persisted: true });
|
||||
lifecycle.dispatchEvent(resumed);
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
assert.equal(world.gets, 3);
|
||||
answer({ success: true, jobs: [summary()] });
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
assert.equal(world.open().length, 1);
|
||||
world.dispose();
|
||||
assert.equal(signals.at(-1).aborted, true, 'disposing aborts the store reads');
|
||||
assert.equal(world.open().length, 0);
|
||||
});
|
||||
|
||||
test('one failed read is not a restart; two in a row are, and a good read resets the backoff', async () => {
|
||||
const { createActivityStore } = await source;
|
||||
const { world, ...deps } = harness();
|
||||
world.list = [summary(), summary({ id: ID_B, state: 'finished', finished_at: 900 })];
|
||||
const store = createActivityStore(deps);
|
||||
await store.load();
|
||||
// A single failed read (a proxy's 502) keeps the job running and its stream open (review M1).
|
||||
world.listError = apiError(502, 'Bad gateway', null);
|
||||
await store.load();
|
||||
assert.equal(store.status, 'error');
|
||||
assert.equal(store.find(ID_A).state, 'running');
|
||||
assert.equal(world.open().length, 1);
|
||||
assert.equal(await world.fire(), 1000);
|
||||
// The second failure in a row: the server is gone, so the running job ended with it.
|
||||
assert.equal(store.find(ID_A).state, 'lost');
|
||||
assert.equal(world.open().length, 0);
|
||||
for (let index = 0; index < 3; index += 1) await world.fire();
|
||||
// The server is back with a new running job: the backoff starts again at 1 s (review M2).
|
||||
world.listError = null;
|
||||
world.list = [summary({ id: ID_C })];
|
||||
await world.fire();
|
||||
assert.equal(store.status, 'ready');
|
||||
assert.deepEqual(store.jobs().map(job => [job.id, job.state]), [[ID_C, 'running'], [ID_A, 'lost']]);
|
||||
world.open()[0].onerror();
|
||||
assert.equal(await world.fire(), 1000);
|
||||
// A summary carries what its detail lists, for the job view's reads (review M12).
|
||||
send(world.open()[0], summary({ id: ID_C, listed: { added: 3, errors: -1 }, log_length: 7 }));
|
||||
assert.deepEqual([store.find(ID_C).listed, store.find(ID_C).log_length], [{ added: 3, errors: 0 }, 7]);
|
||||
world.dispose();
|
||||
});
|
||||
|
||||
test('numbers group the way the server writes them, whatever the browser locale', async () => {
|
||||
const { pillParts, finishToast, parseJob } = await source;
|
||||
const original = Number.prototype.toLocaleString;
|
||||
Number.prototype.toLocaleString = function (locale, ...rest) { return original.call(this, locale ?? 'de-DE', ...rest); };
|
||||
try {
|
||||
const job = parseJob(summary({ progress: 41, processed: 8240, total: 19622, counters: { added: 1420, errors: 18 } }));
|
||||
assert.equal(pillParts(job).text, 'Scanning · 41% · 8,240 / 19,622');
|
||||
assert.equal(finishToast({ ...job, state: 'finished' }), 'Scan finished · 1,420 added · 18 errors');
|
||||
} finally {
|
||||
Number.prototype.toLocaleString = original;
|
||||
}
|
||||
});
|
||||
@@ -4,10 +4,10 @@ const source = import('../../web/js/workspace/state.mjs');
|
||||
|
||||
test('workspace routes resolve only supported deep links', async () => {
|
||||
const { resolveRoute } = await source;
|
||||
for (const route of ['overview', 'images', 'library', 'tags', 'inspect', 'system']) {
|
||||
for (const route of ['overview', 'images', 'library', 'tags', 'inspect', 'system', 'activity']) {
|
||||
assert.equal(resolveRoute(`#/${route}`).id, route);
|
||||
}
|
||||
for (const hash of ['', '#', '#/unknown', '#/images/other', '#/<script>']) {
|
||||
for (const hash of ['', '#', '#/unknown', '#/images/other', '#/<script>', '#/activity/abc', `#/activity/${'A'.repeat(32)}`]) {
|
||||
assert.equal(resolveRoute(hash).id, 'images');
|
||||
}
|
||||
});
|
||||
@@ -23,10 +23,33 @@ test('tags is a native route titled Tags; only unbuilt routes hand off to legacy
|
||||
assert.equal(routeTitle(route), route.label);
|
||||
}
|
||||
assert.equal(routeTitle(resolveRoute('#/library/7')), 'Prompt library');
|
||||
for (const id of ['images', 'library', 'tags', 'overview']) assert.equal(isHandoff(resolveRoute(`#/${id}`)), false, id);
|
||||
for (const id of ['images', 'library', 'tags', 'overview', 'activity']) assert.equal(isHandoff(resolveRoute(`#/${id}`)), false, id);
|
||||
for (const id of ['inspect', 'system']) assert.equal(isHandoff(resolveRoute(`#/${id}`)), true, id);
|
||||
});
|
||||
|
||||
test('a job of the Activity page is its own deep link', async () => {
|
||||
const { resolveRoute, routeTitle, isHandoff } = await source;
|
||||
const id = '0123456789abcdef'.repeat(2);
|
||||
const route = resolveRoute(`#/activity/${id}`);
|
||||
assert.deepEqual([route.id, route.jobId, route.hash, routeTitle(route), isHandoff(route)],
|
||||
['activity', id, `#/activity/${id}`, 'Activity', false]);
|
||||
assert.equal(resolveRoute('#/activity').jobId, undefined);
|
||||
assert.equal(route.href, undefined, 'Activity has no legacy card on Overview');
|
||||
// The router renders again when only the job changes, and not for the same job.
|
||||
const { mountRouter } = await source;
|
||||
const other = 'f'.repeat(32);
|
||||
const win = new EventTarget();
|
||||
win.location = { hash: `#/activity/${id}` };
|
||||
win.history = { state: null, replaceState(state, unused, hash) { win.location.hash = hash; } };
|
||||
const seen = [];
|
||||
const dispose = mountRouter(win, next => seen.push([next.id, next.jobId]));
|
||||
win.dispatchEvent(new Event('hashchange'));
|
||||
win.location.hash = `#/activity/${other}`; win.dispatchEvent(new Event('hashchange'));
|
||||
win.location.hash = '#/activity'; win.dispatchEvent(new Event('hashchange'));
|
||||
assert.deepEqual(seen, [['activity', id], ['activity', other], ['activity', undefined]]);
|
||||
dispose();
|
||||
});
|
||||
|
||||
test('statistics preserve zero and reject incomplete or invalid counts', async () => {
|
||||
const { summarizeStats } = await source;
|
||||
assert.deepEqual(summarizeStats({total_prompts:0, total_images:12, total_tags:3}), [
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
// The shell's shared toast: plain messages clear after 6.5 s; a message with a link stays 20 s (D57), and any toast
|
||||
// stays while the pointer or keyboard focus rests on it.
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
|
||||
const source = import('../../web/js/workspace/toast.mjs');
|
||||
|
||||
class FakeToast extends EventTarget {
|
||||
constructor() { super(); this.held = false; this.nodes = []; this.text = ''; }
|
||||
get textContent() { return this.text; }
|
||||
set textContent(value) { this.text = value; this.nodes = []; }
|
||||
replaceChildren(...nodes) { this.nodes = nodes; this.text = nodes.map(node => node.textContent).join(''); }
|
||||
matches(selector) { assert.equal(selector, ':hover, :focus-within'); return this.held; }
|
||||
}
|
||||
|
||||
function install() {
|
||||
const toast = new FakeToast();
|
||||
const previous = global.document;
|
||||
global.document = {
|
||||
getElementById: id => id === 'workspace-toast' ? toast : null,
|
||||
createTextNode: text => ({ textContent: text }),
|
||||
createElement: tag => ({ tag, href: '', textContent: '' }),
|
||||
};
|
||||
return { toast, restore: () => { global.document = previous; } };
|
||||
}
|
||||
|
||||
test('a plain toast clears after 6.5 s and a linked one after 20 s', async t => {
|
||||
const { showWorkspaceToast, TOAST_MS, LINK_TOAST_MS } = await source;
|
||||
t.mock.timers.enable({ apis: ['setTimeout'] });
|
||||
const { toast, restore } = install();
|
||||
try {
|
||||
showWorkspaceToast('Scan started.');
|
||||
assert.equal(toast.textContent, 'Scan started.');
|
||||
t.mock.timers.tick(TOAST_MS);
|
||||
assert.equal(toast.textContent, '');
|
||||
|
||||
showWorkspaceToast(['Another job is running: ', { text: 'Scan', href: '#/activity/x' }, '.']);
|
||||
assert.equal(toast.textContent, 'Another job is running: Scan.');
|
||||
assert.deepEqual(toast.nodes[1], { tag: 'a', href: '#/activity/x', textContent: 'Scan' });
|
||||
t.mock.timers.tick(TOAST_MS);
|
||||
assert.equal(toast.textContent, 'Another job is running: Scan.');
|
||||
t.mock.timers.tick(LINK_TOAST_MS - TOAST_MS);
|
||||
assert.equal(toast.textContent, '');
|
||||
} finally {
|
||||
restore();
|
||||
}
|
||||
});
|
||||
|
||||
test('a toast under the pointer or focus waits until they leave', async t => {
|
||||
const { showWorkspaceToast, TOAST_MS } = await source;
|
||||
t.mock.timers.enable({ apis: ['setTimeout'] });
|
||||
const { toast, restore } = install();
|
||||
try {
|
||||
showWorkspaceToast('Prompt copied.');
|
||||
toast.held = true;
|
||||
t.mock.timers.tick(TOAST_MS);
|
||||
assert.equal(toast.textContent, 'Prompt copied.', 'held: it stays');
|
||||
// Focus moving inside the toast keeps it; leaving it starts a fresh 6.5 s.
|
||||
toast.dispatchEvent(new Event('focusout'));
|
||||
t.mock.timers.tick(0);
|
||||
toast.dispatchEvent(new Event('pointerleave'));
|
||||
assert.equal(toast.textContent, 'Prompt copied.');
|
||||
toast.held = false;
|
||||
toast.dispatchEvent(new Event('focusout'));
|
||||
t.mock.timers.tick(0);
|
||||
t.mock.timers.tick(TOAST_MS - 1);
|
||||
assert.equal(toast.textContent, 'Prompt copied.');
|
||||
t.mock.timers.tick(1);
|
||||
assert.equal(toast.textContent, '');
|
||||
|
||||
// A newer toast replaces a held one and stops waiting for the old one's pointer.
|
||||
showWorkspaceToast('First.');
|
||||
toast.held = true;
|
||||
t.mock.timers.tick(TOAST_MS);
|
||||
showWorkspaceToast('Second.');
|
||||
toast.held = false;
|
||||
toast.dispatchEvent(new Event('pointerleave'));
|
||||
t.mock.timers.tick(TOAST_MS - 1);
|
||||
assert.equal(toast.textContent, 'Second.');
|
||||
t.mock.timers.tick(1);
|
||||
assert.equal(toast.textContent, '');
|
||||
} finally {
|
||||
restore();
|
||||
}
|
||||
});
|
||||
|
||||
test('a page without the toast shows nothing and does not fail', async () => {
|
||||
const { showWorkspaceToast } = await source;
|
||||
const previous = global.document;
|
||||
global.document = { getElementById: () => null };
|
||||
try {
|
||||
assert.doesNotThrow(() => showWorkspaceToast('Nothing.'));
|
||||
} finally {
|
||||
global.document = previous;
|
||||
}
|
||||
});
|
||||
@@ -765,6 +765,20 @@ test('the Filters button shows the active refinement count', async () => {
|
||||
assert.equal(filtersButton.textContent, 'Filters');
|
||||
for (const bad of [-1, 1.5, '1', null, Number.NaN]) assert.throws(() => toolbar.setActiveFilters(bad), TypeError);
|
||||
assert.equal(filtersButton.textContent, 'Filters');
|
||||
// An unchanged count leaves the label's text node alone: every listing reload reports the count, and WebKit
|
||||
// drops a click whose mousedown hit a text node that was replaced before mouseup.
|
||||
let writes = 0;
|
||||
const proto = Object.getPrototypeOf(filtersButton);
|
||||
const descriptor = Object.getOwnPropertyDescriptor(proto, 'textContent');
|
||||
Object.defineProperty(filtersButton, 'textContent', {
|
||||
get() { return descriptor.get.call(this); },
|
||||
set(text) { writes += 1; descriptor.set.call(this, text); },
|
||||
configurable: true,
|
||||
});
|
||||
toolbar.setActiveFilters(0);
|
||||
assert.equal(writes, 0);
|
||||
toolbar.setActiveFilters(3);
|
||||
assert.deepEqual([writes, filtersButton.textContent], [1, 'Filters · 3']);
|
||||
assert.throws(() => mountToolbar(new dom.Element('div'), { scope, categories: [], sorts: [['newest', 'Recently added']],
|
||||
activeFilters: -1, onChange() {}, onFilters() {}, onSaveView() {} }), TypeError);
|
||||
} finally {
|
||||
|
||||
@@ -507,10 +507,49 @@ test('category preview text states the real impact and whether it renames or con
|
||||
'“Landscapes” is used by 2 other saved prompts. Apply renames it to “Scenery”.');
|
||||
});
|
||||
|
||||
test('accepts image_count and names images in the delete text', async () => {
|
||||
const {createVocabularyStream, createVocabularyCommand, readTagImpact, deleteTagText} = await helpers;
|
||||
const rows = [{...tag(1, 'mist', 2), image_count: 3}, tag(2, 'dusk', 0)];
|
||||
const stream = createVocabularyStream({api: queued([tagPage(rows)]).api, kind: 'tags'});
|
||||
assert.equal(await stream.loadMore(), 2);
|
||||
assert.deepEqual(stream.state.items.map(row => row.image_count), [3, undefined]);
|
||||
for (const bad of [{...tag(1), image_count: -1}, {...tag(1), image_count: '3'}, {...tag(1), image_count: 1.5}]) {
|
||||
const rejected = createVocabularyStream({api: queued([tagPage([bad])]).api, kind: 'tags'});
|
||||
await rejected.loadMore();
|
||||
assert.ok(rejected.state.error instanceof TypeError, JSON.stringify(bad));
|
||||
assert.deepEqual(rejected.state.items, []);
|
||||
}
|
||||
const withImages = {...impact, images: 2};
|
||||
const api = {get: async () => ({success: true, library_epoch: EPOCH, impact: withImages})};
|
||||
assert.deepEqual((await readTagImpact(api, {id: 4, name: 'mist'})).impact, withImages);
|
||||
await assert.rejects(readTagImpact({get: async () => ({success: true, library_epoch: EPOCH, impact: {...impact, images: -2}})},
|
||||
{id: 4, name: 'mist'}), TypeError);
|
||||
const command = createVocabularyCommand({api: {
|
||||
patch: async () => renameReceipt({tag: {...tag(1, 'dusk', 2), image_count: 1}, images_changed: 1}),
|
||||
post: async () => ({success: true, library_epoch: EPOCH, deleted: true, components_changed: 0, pairs_changed: 0, images_changed: 2}),
|
||||
}});
|
||||
await command.renameTag({id: 1, name: 'dawn', prompt_count: 2}, 'dusk', EPOCH);
|
||||
assert.equal(command.state.phase, 'done');
|
||||
assert.equal(command.state.receipt.images_changed, 1);
|
||||
command.reset();
|
||||
await command.deleteTag({id: 1, name: 'dusk'}, EPOCH);
|
||||
assert.equal(command.state.phase, 'done');
|
||||
const malformed = createVocabularyCommand({api: {post: async () => ({success: true, library_epoch: EPOCH, deleted: true,
|
||||
components_changed: 0, pairs_changed: 0, images_changed: 'two'})}});
|
||||
await malformed.deleteTag({id: 1, name: 'dusk'}, EPOCH);
|
||||
assert.equal(malformed.state.phase, 'unknown-outcome');
|
||||
assert.equal(deleteTagText('mist', withImages),
|
||||
'Deleting “mist” removes the name and its links from 3 positive prompts, 3 negative variants (1 archived), 1 other saved prompt and 2 images. The prompts, negative variants, images and history themselves are kept.');
|
||||
assert.equal(deleteTagText('mist', {...zero, images: 1}),
|
||||
'Deleting “mist” removes the name and its links from 1 image. The prompts, negative variants, images and history themselves are kept.');
|
||||
assert.equal(deleteTagText('unused', {...zero, images: 0}),
|
||||
'“unused” is not linked to any saved prompt or image. Deleting it removes only the name.');
|
||||
});
|
||||
|
||||
test('delete text names the tag, its real links and what stays unchanged', async () => {
|
||||
const {deleteTagText} = await helpers;
|
||||
assert.equal(deleteTagText('mist', impact),
|
||||
'Deleting “mist” removes the name and its links from 3 positive prompts, 3 negative variants (1 archived) and 1 other saved prompt. The prompts, negative variants, images and history themselves are kept.');
|
||||
assert.equal(deleteTagText('unused', zero),
|
||||
'“unused” is not linked to any saved prompt. Deleting it removes only the name.');
|
||||
'“unused” is not linked to any saved prompt or image. Deleting it removes only the name.');
|
||||
});
|
||||
|
||||
@@ -0,0 +1,602 @@
|
||||
"""Image annotation schema group (#249): install, validation and restore safety."""
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
try:
|
||||
from tests.open_handles import assert_closed
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from open_handles import assert_closed
|
||||
|
||||
from database.models import PromptModel
|
||||
|
||||
try: # guarded so a missing module fails assertions, not test loading
|
||||
from database.annotation_schema import (
|
||||
annotation_schema_error,
|
||||
migrate_annotation_state,
|
||||
)
|
||||
except ImportError:
|
||||
annotation_schema_error = None
|
||||
migrate_annotation_state = None
|
||||
|
||||
METADATA = "media_annotation_metadata"
|
||||
FILES = "media_annotation_files"
|
||||
BYTES = "media_annotation_bytes"
|
||||
IMAGE_TAGS = "media_image_tags"
|
||||
TABLES = (METADATA, FILES, BYTES, IMAGE_TAGS)
|
||||
# The exact DDL the plan pins; the module must produce precisely these objects.
|
||||
DDL = {
|
||||
METADATA: (
|
||||
f"CREATE TABLE {METADATA}(singleton_id INTEGER PRIMARY KEY "
|
||||
"CHECK(singleton_id=1), schema_version INTEGER NOT NULL "
|
||||
"CHECK(schema_version=1), image_index_revision INTEGER NOT NULL DEFAULT 0 "
|
||||
"CHECK(image_index_revision>=0))"
|
||||
),
|
||||
FILES: (
|
||||
f"CREATE TABLE {FILES}(id TEXT PRIMARY KEY CHECK(length(id)=36), "
|
||||
"locator TEXT NOT NULL UNIQUE, created_at TEXT NOT NULL)"
|
||||
),
|
||||
BYTES: (
|
||||
f"CREATE TABLE {BYTES}(id TEXT PRIMARY KEY CHECK(length(id)=36), "
|
||||
f"file_id TEXT NOT NULL REFERENCES {FILES}(id) ON DELETE CASCADE, "
|
||||
"sha256 TEXT NOT NULL CHECK(length(sha256)=64 AND sha256 NOT GLOB "
|
||||
"'*[^0-9a-f]*'), size INTEGER NOT NULL CHECK(size>=0), "
|
||||
"mtime_ns INTEGER NOT NULL, verified_at TEXT NOT NULL, "
|
||||
"first_seen_at TEXT NOT NULL, tag_revision INTEGER NOT NULL DEFAULT 0 "
|
||||
"CHECK(tag_revision>=0), UNIQUE(file_id, sha256))"
|
||||
),
|
||||
IMAGE_TAGS: (
|
||||
f"CREATE TABLE {IMAGE_TAGS}(byte_id TEXT NOT NULL REFERENCES {BYTES}(id) "
|
||||
"ON DELETE CASCADE, tag_id INTEGER NOT NULL REFERENCES tags(id) "
|
||||
"ON DELETE CASCADE, position INTEGER NOT NULL CHECK(position>=0), "
|
||||
"PRIMARY KEY(byte_id, tag_id))"
|
||||
),
|
||||
}
|
||||
INDEXES = {
|
||||
"idx_media_annotation_bytes_fingerprint": (
|
||||
f"CREATE INDEX idx_media_annotation_bytes_fingerprint ON {BYTES}"
|
||||
"(file_id, size, mtime_ns)"
|
||||
),
|
||||
"idx_media_image_tags_tag": (
|
||||
f"CREATE INDEX idx_media_image_tags_tag ON {IMAGE_TAGS}(tag_id)"
|
||||
),
|
||||
"idx_media_image_tags_position": (
|
||||
f"CREATE UNIQUE INDEX idx_media_image_tags_position ON {IMAGE_TAGS}"
|
||||
"(byte_id, position)"
|
||||
),
|
||||
}
|
||||
AUTOINDEXES = {FILES: 2, BYTES: 2, IMAGE_TAGS: 1}
|
||||
LEGACY_TABLES = (
|
||||
"prompts",
|
||||
"generated_images",
|
||||
"tags",
|
||||
"prompt_tags",
|
||||
"prompt_pair_tags",
|
||||
"image_favorites",
|
||||
)
|
||||
FILE_ID = "0f8e4b2a-1c3d-4e5f-8a9b-0c1d2e3f4a5b"
|
||||
BYTE_ID = "6a1b2c3d-4e5f-4a6b-8c9d-0e1f2a3b4c5d"
|
||||
OTHER_ID = "7b2c3d4e-5f6a-4b7c-9d0e-1f2a3b4c5d6e"
|
||||
DIGEST = hashlib.sha256(b"img").hexdigest()
|
||||
NOW = "2026-10-22T00:00:00.000000+00:00"
|
||||
MAX_LOCATOR = 4096
|
||||
|
||||
|
||||
def _compact(sql):
|
||||
return re.sub(r"\s+", "", sql or "").lower()
|
||||
|
||||
|
||||
def expected_catalog():
|
||||
objects = {("table", name, _compact(sql)) for name, sql in DDL.items()}
|
||||
objects |= {("index", name, _compact(sql)) for name, sql in INDEXES.items()}
|
||||
objects |= {
|
||||
("index", f"sqlite_autoindex_{table}_{n}", "")
|
||||
for table, count in AUTOINDEXES.items()
|
||||
for n in range(1, count + 1)
|
||||
}
|
||||
return objects
|
||||
|
||||
|
||||
def catalog(conn):
|
||||
names = tuple(TABLES) + tuple(INDEXES)
|
||||
marks = ",".join("?" * len(names))
|
||||
table_marks = ",".join("?" * len(TABLES))
|
||||
return {
|
||||
(kind, name, _compact(sql))
|
||||
for kind, name, sql in conn.execute(
|
||||
"SELECT type,name,sql FROM sqlite_master "
|
||||
f"WHERE name IN ({marks}) OR tbl_name IN ({table_marks})", # nosec B608
|
||||
names + tuple(TABLES),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def insert_group_rows(conn, tag_name="img-only", locator="/out/img.png"):
|
||||
"""One file, one byte record and one image tag carrying *tag_name*."""
|
||||
tag_id = conn.execute("INSERT INTO tags(name) VALUES(?)", (tag_name,)).lastrowid
|
||||
conn.execute(f"INSERT INTO {FILES} VALUES(?,?,?)", (FILE_ID, locator, NOW))
|
||||
conn.execute(
|
||||
f"INSERT INTO {BYTES} VALUES(?,?,?,?,?,?,?,?)",
|
||||
(BYTE_ID, FILE_ID, DIGEST, 3, 1_700_000_000_000_000_000, NOW, NOW, 1),
|
||||
)
|
||||
conn.execute(f"INSERT INTO {IMAGE_TAGS} VALUES(?,?,0)", (BYTE_ID, tag_id))
|
||||
return tag_id
|
||||
|
||||
|
||||
def rows(conn, sql, *params):
|
||||
"""Plain tuples whatever row factory *conn* uses."""
|
||||
return [tuple(row) for row in conn.execute(sql, params)]
|
||||
|
||||
|
||||
def image_tags(conn):
|
||||
return rows(
|
||||
conn,
|
||||
f"SELECT t.name, it.position FROM {IMAGE_TAGS} it "
|
||||
"JOIN tags t ON t.id = it.tag_id ORDER BY it.position",
|
||||
)
|
||||
|
||||
|
||||
def metadata(conn):
|
||||
return rows(conn, f"SELECT schema_version,image_index_revision FROM {METADATA}")
|
||||
|
||||
|
||||
def table_rows(conn, tables):
|
||||
found = {}
|
||||
for table in tables:
|
||||
present = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)
|
||||
).fetchone()
|
||||
if present:
|
||||
found[table] = rows(conn, f"SELECT * FROM {table}") # nosec B608
|
||||
return found
|
||||
|
||||
|
||||
def populate_legacy(conn):
|
||||
conn.execute(
|
||||
"INSERT INTO prompts(id,text,hash,category,notes,run_count) VALUES"
|
||||
"(7,'positive','p','keep','notes',12)"
|
||||
)
|
||||
conn.execute("INSERT INTO prompts(id,text,hash) VALUES(9,'negative','n')")
|
||||
conn.execute(
|
||||
"INSERT INTO generated_images(id,prompt_id,image_path,filename,file_path) "
|
||||
"VALUES(4,7,'/tmp/image.png','image.png','/tmp/image.png')"
|
||||
)
|
||||
conn.execute("INSERT INTO tags(id,name) VALUES(3,'keep-tag')")
|
||||
conn.execute("INSERT INTO prompt_tags VALUES(7,3)")
|
||||
conn.execute(
|
||||
"INSERT INTO prompt_pairs(id,positive_prompt_id,negative_prompt_id) "
|
||||
"VALUES(6,7,9)"
|
||||
)
|
||||
conn.execute("INSERT INTO prompt_pair_tags VALUES(6,3)")
|
||||
conn.execute("INSERT INTO image_favorites VALUES('root','a.png',?)", (NOW,))
|
||||
|
||||
|
||||
class AnnotationSchemaTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.addCleanup(assert_closed, self, self.temp.name)
|
||||
self.addCleanup(PromptModel.reset_schema_cache)
|
||||
self.models = []
|
||||
self.addCleanup(self.close_models)
|
||||
self.path = str(Path(self.temp.name) / "live.db")
|
||||
|
||||
def close_models(self):
|
||||
for model in self.models:
|
||||
model.close_all()
|
||||
|
||||
def model(self, path=None):
|
||||
model = PromptModel(path or self.path)
|
||||
self.models.append(model)
|
||||
return model
|
||||
|
||||
def require_module(self):
|
||||
self.assertIsNotNone(
|
||||
annotation_schema_error, "database.annotation_schema is not implemented"
|
||||
)
|
||||
|
||||
def drop_group(self, path):
|
||||
"""Remove the group from a closed database file (an old backup)."""
|
||||
with closing(sqlite3.connect(path)) as raw, raw:
|
||||
raw.execute("PRAGMA foreign_keys=OFF")
|
||||
for table in reversed(TABLES):
|
||||
raw.execute(f"DROP TABLE {table}")
|
||||
|
||||
def test_fresh_install_and_reopen_is_idempotent(self):
|
||||
self.require_module()
|
||||
model = self.model()
|
||||
with model.get_connection() as conn:
|
||||
self.assertEqual(catalog(conn), expected_catalog())
|
||||
self.assertIsNone(annotation_schema_error(conn, allow_missing=False))
|
||||
self.assertEqual(metadata(conn), [(1, 0)])
|
||||
insert_group_rows(conn)
|
||||
self.assertEqual(
|
||||
PromptModel.verify_database_file(self.path, require_library_state=True),
|
||||
(True, "ok"),
|
||||
)
|
||||
model.close_all()
|
||||
for _ in range(2):
|
||||
PromptModel.reset_schema_cache()
|
||||
model = self.model()
|
||||
with model.get_connection() as conn:
|
||||
self.assertEqual(catalog(conn), expected_catalog())
|
||||
self.assertIsNone(annotation_schema_error(conn, allow_missing=False))
|
||||
self.assertEqual(metadata(conn), [(1, 0)])
|
||||
self.assertEqual(image_tags(conn), [("img-only", 0)])
|
||||
# Running the migration again on a current database changes nothing.
|
||||
migrate_annotation_state(conn)
|
||||
self.assertEqual(metadata(conn), [(1, 0)])
|
||||
self.assertEqual(image_tags(conn), [("img-only", 0)])
|
||||
model.close_all()
|
||||
# A tag delete and a file delete cascade into the junction and bytes.
|
||||
PromptModel.reset_schema_cache()
|
||||
model = self.model()
|
||||
with model.get_connection() as conn:
|
||||
conn.execute("DELETE FROM tags WHERE name='img-only'")
|
||||
self.assertEqual(image_tags(conn), [])
|
||||
conn.execute(f"DELETE FROM {FILES}")
|
||||
self.assertEqual(rows(conn, f"SELECT * FROM {BYTES}"), [])
|
||||
self.assertIsNone(annotation_schema_error(conn, allow_missing=False))
|
||||
|
||||
def test_old_database_without_group_migrates_without_touching_legacy_rows(self):
|
||||
self.require_module()
|
||||
model = self.model()
|
||||
with model.get_connection() as conn:
|
||||
populate_legacy(conn)
|
||||
model.close_all()
|
||||
self.drop_group(self.path)
|
||||
with closing(sqlite3.connect(self.path)) as raw:
|
||||
self.assertIsNone(annotation_schema_error(raw))
|
||||
self.assertEqual(
|
||||
annotation_schema_error(raw, allow_missing=False),
|
||||
"Missing image annotation schema",
|
||||
)
|
||||
legacy_rows_before = table_rows(raw, LEGACY_TABLES)
|
||||
self.assertTrue(PromptModel.verify_database_file(self.path)[0])
|
||||
self.assertFalse(
|
||||
PromptModel.verify_database_file(self.path, require_library_state=True)[0]
|
||||
)
|
||||
PromptModel.reset_schema_cache()
|
||||
|
||||
model = self.model()
|
||||
|
||||
with model.get_connection() as conn:
|
||||
self.assertEqual(catalog(conn), expected_catalog())
|
||||
self.assertIsNone(annotation_schema_error(conn, allow_missing=False))
|
||||
self.assertEqual(metadata(conn), [(1, 0)])
|
||||
legacy_rows_after = table_rows(conn, LEGACY_TABLES)
|
||||
# No backfill: existing prompt tags never become image tags.
|
||||
counts = [
|
||||
conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]
|
||||
for table in (FILES, BYTES, IMAGE_TAGS)
|
||||
]
|
||||
self.assertEqual(counts, [0, 0, 0])
|
||||
self.assertEqual(legacy_rows_after, legacy_rows_before)
|
||||
self.assertEqual(
|
||||
PromptModel.verify_database_file(self.path, require_library_state=True),
|
||||
(True, "ok"),
|
||||
)
|
||||
|
||||
def tampered(self, name, statements, *, checks_off=False):
|
||||
"""A closed current database whose group was altered after the fact."""
|
||||
path = str(Path(self.temp.name) / f"{name.replace(' ', '-')}.db")
|
||||
model = self.model(path)
|
||||
with model.get_connection() as conn:
|
||||
populate_legacy(conn)
|
||||
insert_group_rows(conn)
|
||||
model.close_all()
|
||||
with closing(sqlite3.connect(path)) as raw, raw:
|
||||
raw.execute("PRAGMA foreign_keys=OFF")
|
||||
if checks_off:
|
||||
raw.execute("PRAGMA ignore_check_constraints=ON")
|
||||
for statement in statements:
|
||||
if isinstance(statement, tuple):
|
||||
raw.execute(*statement)
|
||||
else:
|
||||
raw.execute(statement)
|
||||
return path
|
||||
|
||||
def assert_rejected_unrepaired(self, path):
|
||||
with closing(sqlite3.connect(path)) as raw:
|
||||
schema_before = raw.execute(
|
||||
"SELECT type,name,tbl_name,sql FROM sqlite_master ORDER BY name"
|
||||
).fetchall()
|
||||
rows_before = table_rows(raw, TABLES)
|
||||
for allow_missing in (False, True):
|
||||
error = annotation_schema_error(raw, allow_missing=allow_missing)
|
||||
self.assertIsNotNone(error)
|
||||
self.assertIn("image annotation", error.lower())
|
||||
self.assertFalse(PromptModel.verify_database_file(path)[0])
|
||||
self.assertFalse(
|
||||
PromptModel.verify_database_file(path, require_library_state=True)[0]
|
||||
)
|
||||
PromptModel.reset_schema_cache()
|
||||
with self.assertRaises(sqlite3.DatabaseError):
|
||||
self.model(path)
|
||||
PromptModel.reset_schema_cache()
|
||||
with closing(sqlite3.connect(path)) as raw:
|
||||
self.assertEqual(
|
||||
raw.execute(
|
||||
"SELECT type,name,tbl_name,sql FROM sqlite_master ORDER BY name"
|
||||
).fetchall(),
|
||||
schema_before,
|
||||
)
|
||||
self.assertEqual(table_rows(raw, TABLES), rows_before)
|
||||
|
||||
def test_partial_or_malformed_group_is_rejected_without_repair(self):
|
||||
self.require_module()
|
||||
cases = {
|
||||
"missing metadata": (f"DROP TABLE {METADATA}",),
|
||||
"missing files": (f"DROP TABLE {FILES}",),
|
||||
"missing bytes": (f"DROP TABLE {BYTES}",),
|
||||
"missing image tags": (f"DROP TABLE {IMAGE_TAGS}",),
|
||||
"reserved view": (
|
||||
f"DROP TABLE {FILES}",
|
||||
f"CREATE VIEW {FILES} AS SELECT 'x' AS id",
|
||||
),
|
||||
"extra trigger": (
|
||||
f"CREATE TRIGGER media_annotation_trigger AFTER INSERT ON {FILES} "
|
||||
"BEGIN SELECT 1; END",
|
||||
),
|
||||
"extra index": (
|
||||
f"CREATE INDEX idx_media_annotation_extra ON {FILES}(created_at)",
|
||||
),
|
||||
"extra column": (
|
||||
f"DROP TABLE {IMAGE_TAGS}",
|
||||
DDL[IMAGE_TAGS].replace(
|
||||
"position INTEGER", "note TEXT, position INTEGER"
|
||||
),
|
||||
),
|
||||
"wrong primary key": (
|
||||
f"DROP TABLE {FILES}",
|
||||
DDL[FILES].replace("id TEXT PRIMARY KEY", "id TEXT"),
|
||||
),
|
||||
"wrong foreign key": (
|
||||
f"DROP TABLE {BYTES}",
|
||||
DDL[BYTES].replace("ON DELETE CASCADE", "ON DELETE RESTRICT"),
|
||||
INDEXES["idx_media_annotation_bytes_fingerprint"],
|
||||
),
|
||||
"wrong check": (
|
||||
f"DROP TABLE {METADATA}",
|
||||
DDL[METADATA].replace("schema_version=1", "schema_version=2"),
|
||||
f"INSERT INTO {METADATA} VALUES(1,2,0)",
|
||||
),
|
||||
"missing unique": (
|
||||
f"DROP TABLE {FILES}",
|
||||
DDL[FILES].replace("NOT NULL UNIQUE", "NOT NULL"),
|
||||
),
|
||||
"missing index": ("DROP INDEX idx_media_image_tags_tag",),
|
||||
"non-unique position index": (
|
||||
"DROP INDEX idx_media_image_tags_position",
|
||||
INDEXES["idx_media_image_tags_position"].replace("UNIQUE ", ""),
|
||||
),
|
||||
"wrong index columns": (
|
||||
"DROP INDEX idx_media_annotation_bytes_fingerprint",
|
||||
INDEXES["idx_media_annotation_bytes_fingerprint"].replace(
|
||||
", mtime_ns", ""
|
||||
),
|
||||
),
|
||||
"orphan tag": (f"INSERT INTO {IMAGE_TAGS} VALUES('{BYTE_ID}',404,1)",),
|
||||
"orphan byte": (
|
||||
(
|
||||
f"INSERT INTO {BYTES} VALUES(?,?,?,1,1,?,?,0)",
|
||||
(
|
||||
OTHER_ID,
|
||||
"9" * 8 + "-0000-4000-8000-" + "0" * 12,
|
||||
DIGEST,
|
||||
NOW,
|
||||
NOW,
|
||||
),
|
||||
),
|
||||
),
|
||||
"bad file uuid": (
|
||||
(f"UPDATE {FILES} SET id=?", ("x" * 36,)),
|
||||
(f"UPDATE {BYTES} SET file_id=?", ("x" * 36,)),
|
||||
),
|
||||
"uppercase byte uuid": (
|
||||
f"UPDATE {BYTES} SET id=upper(id)",
|
||||
f"UPDATE {IMAGE_TAGS} SET byte_id=upper(byte_id)",
|
||||
),
|
||||
"null file id": ((f"INSERT INTO {FILES} VALUES(NULL,?,?)", ("/n", NOW)),),
|
||||
"blob ids": (
|
||||
f"UPDATE {FILES} SET id=CAST(id AS BLOB)",
|
||||
f"UPDATE {BYTES} SET file_id=CAST(file_id AS BLOB)",
|
||||
),
|
||||
"no metadata row": (f"DELETE FROM {METADATA}",),
|
||||
"long created_at": ((f"UPDATE {FILES} SET created_at=?", ("x" * 65,)),),
|
||||
"blob verified_at": (f"UPDATE {BYTES} SET verified_at=X'00'",),
|
||||
"text mtime": (f"UPDATE {BYTES} SET mtime_ns='soon'",),
|
||||
"duplicate position": (
|
||||
"DROP INDEX idx_media_image_tags_position",
|
||||
"INSERT INTO tags(id,name) VALUES(77,'dup')",
|
||||
f"INSERT INTO {IMAGE_TAGS} VALUES('{BYTE_ID}',77,0)",
|
||||
),
|
||||
"long locator": (
|
||||
(f"UPDATE {FILES} SET locator=?", ("x" * (MAX_LOCATOR + 1),)),
|
||||
),
|
||||
"multibyte long locator": (
|
||||
(f"UPDATE {FILES} SET locator=?", ("é" * (MAX_LOCATOR + 1),)),
|
||||
),
|
||||
"nul locator bypass": (
|
||||
(f"UPDATE {FILES} SET locator=?", ("x\0" + "y" * MAX_LOCATOR,)),
|
||||
),
|
||||
}
|
||||
for label, statements in cases.items():
|
||||
with self.subTest(label):
|
||||
self.assert_rejected_unrepaired(self.tampered(label, statements))
|
||||
unchecked = {
|
||||
"two metadata rows": (f"INSERT INTO {METADATA} VALUES(2,1,0)",),
|
||||
"wrong schema version": (f"UPDATE {METADATA} SET schema_version=2",),
|
||||
"negative index revision": (
|
||||
f"UPDATE {METADATA} SET image_index_revision=-1",
|
||||
),
|
||||
"bad digest": (f"UPDATE {BYTES} SET sha256=upper(sha256)",),
|
||||
"short digest": (f"UPDATE {BYTES} SET sha256=substr(sha256,1,63)",),
|
||||
"negative size": (f"UPDATE {BYTES} SET size=-1",),
|
||||
"negative tag revision": (f"UPDATE {BYTES} SET tag_revision=-1",),
|
||||
"negative position": (f"UPDATE {IMAGE_TAGS} SET position=-1",),
|
||||
}
|
||||
for label, statements in unchecked.items():
|
||||
with self.subTest(label):
|
||||
path = self.tampered(label, statements, checks_off=True)
|
||||
self.assert_rejected_unrepaired(path)
|
||||
accepted = {
|
||||
"locator at bound": ("x" * MAX_LOCATOR,),
|
||||
"multibyte locator at bound": ("é" * MAX_LOCATOR,),
|
||||
}
|
||||
for label, (locator,) in accepted.items():
|
||||
with self.subTest(label):
|
||||
path = self.tampered(
|
||||
label, ((f"UPDATE {FILES} SET locator=?", (locator,)),)
|
||||
)
|
||||
with closing(sqlite3.connect(path)) as raw:
|
||||
self.assertIsNone(annotation_schema_error(raw, allow_missing=False))
|
||||
self.assertEqual(
|
||||
PromptModel.verify_database_file(path, require_library_state=True),
|
||||
(True, "ok"),
|
||||
)
|
||||
|
||||
def test_install_failure_rolls_back_and_retries(self):
|
||||
self.require_module()
|
||||
with patch(
|
||||
"database.annotation_schema.annotation_schema_error",
|
||||
side_effect=[None, "injected final validation failure"],
|
||||
):
|
||||
with self.assertRaisesRegex(sqlite3.DatabaseError, "injected"):
|
||||
self.model()
|
||||
with closing(sqlite3.connect(self.path)) as raw:
|
||||
self.assertEqual(catalog(raw), set())
|
||||
self.assertIsNotNone(
|
||||
raw.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE name='saved_views'"
|
||||
).fetchone()
|
||||
)
|
||||
PromptModel.reset_schema_cache()
|
||||
model = self.model()
|
||||
self.assertEqual(
|
||||
PromptModel.verify_database_file(model.db_path, require_library_state=True),
|
||||
(True, "ok"),
|
||||
)
|
||||
with model.get_connection() as conn:
|
||||
self.assertEqual(metadata(conn), [(1, 0)])
|
||||
|
||||
def test_wal_backup_restore_round_trips_group(self):
|
||||
self.require_module()
|
||||
source_path = str(Path(self.temp.name) / "source.db")
|
||||
source = self.model(source_path)
|
||||
with source.get_connection() as conn:
|
||||
conn.execute("PRAGMA wal_autocheckpoint=0")
|
||||
conn.execute("INSERT INTO prompts(id,text,hash) VALUES(7,'p','p')")
|
||||
insert_group_rows(conn)
|
||||
conn.commit()
|
||||
source_files = rows(conn, f"SELECT * FROM {FILES}")
|
||||
source_bytes = rows(conn, f"SELECT * FROM {BYTES}")
|
||||
self.assertGreater(Path(source_path + "-wal").stat().st_size, 0)
|
||||
live = self.model()
|
||||
with live.get_connection() as conn:
|
||||
conn.execute("INSERT INTO prompts(id,text,hash) VALUES(1,'live','live')")
|
||||
insert_group_rows(conn, tag_name="live-only", locator="/out/live.png")
|
||||
source_before = Path(source_path).read_bytes()
|
||||
self.assertEqual(
|
||||
PromptModel.verify_database_file(source_path, require_library_state=True),
|
||||
(True, "ok"),
|
||||
)
|
||||
# The rows still sit in the WAL; a backup file must carry them too.
|
||||
backup_file = str(Path(self.temp.name) / "backup.db")
|
||||
self.assertTrue(source.backup_database(backup_file))
|
||||
with closing(sqlite3.connect(backup_file)) as raw:
|
||||
self.assertEqual(image_tags(raw), [("img-only", 0)])
|
||||
self.assertIsNone(annotation_schema_error(raw, allow_missing=False))
|
||||
|
||||
safety = live.restore_from_file(source_path)
|
||||
|
||||
self.assertTrue(Path(safety).is_file())
|
||||
self.assertEqual(Path(source_path).read_bytes(), source_before)
|
||||
with live.get_connection() as conn:
|
||||
restored_image_tags = image_tags(conn)
|
||||
self.assertEqual(rows(conn, f"SELECT * FROM {FILES}"), source_files)
|
||||
self.assertEqual(rows(conn, f"SELECT * FROM {BYTES}"), source_bytes)
|
||||
self.assertEqual(metadata(conn), [(1, 0)])
|
||||
self.assertIsNone(annotation_schema_error(conn, allow_missing=False))
|
||||
self.assertEqual(rows(conn, "SELECT id FROM prompts"), [(7,)])
|
||||
self.assertEqual(restored_image_tags, [("img-only", 0)])
|
||||
with closing(sqlite3.connect(safety)) as raw:
|
||||
self.assertEqual(image_tags(raw), [("live-only", 0)])
|
||||
self.assertEqual(raw.execute("SELECT id FROM prompts").fetchall(), [(1,)])
|
||||
# The same rows restore from the backup file as well.
|
||||
PromptModel.reset_schema_cache()
|
||||
live.restore_from_file(backup_file)
|
||||
with live.get_connection() as conn:
|
||||
self.assertEqual(image_tags(conn), [("img-only", 0)])
|
||||
PromptModel.close_all_instances()
|
||||
assert_closed(self, self.temp.name)
|
||||
|
||||
def test_staged_failure_preserves_source_live_safety_and_handles(self):
|
||||
self.require_module()
|
||||
live = self.model()
|
||||
with live.get_connection() as conn:
|
||||
conn.execute("INSERT INTO prompts(id,text,hash) VALUES(1,'live','live')")
|
||||
insert_group_rows(conn, tag_name="live-only")
|
||||
# A source with a malformed group is refused before any change.
|
||||
bad = self.tampered(
|
||||
"restore bad",
|
||||
(f"UPDATE {BYTES} SET sha256=upper(sha256)",),
|
||||
checks_off=True,
|
||||
)
|
||||
# A staging failure while installing the group must be refused too.
|
||||
old = str(Path(self.temp.name) / "old.db")
|
||||
self.model(old).close_all()
|
||||
self.drop_group(old)
|
||||
for source, patched in ((bad, False), (old, True)):
|
||||
with self.subTest(source=Path(source).name):
|
||||
before = Path(source).read_bytes()
|
||||
safety = sorted(Path(self.temp.name).glob("live.db.backup_*"))
|
||||
if patched:
|
||||
context = patch(
|
||||
"database.models.migrate_annotation_state",
|
||||
side_effect=sqlite3.OperationalError("staging failed"),
|
||||
)
|
||||
expected = sqlite3.OperationalError
|
||||
else:
|
||||
context = patch.object(PromptModel, "backup_database")
|
||||
expected = ValueError
|
||||
with context as hook, self.assertRaises(expected):
|
||||
live.restore_from_file(source)
|
||||
if not patched:
|
||||
hook.assert_not_called()
|
||||
self.assertEqual(Path(source).read_bytes(), before)
|
||||
self.assertEqual(
|
||||
sorted(Path(self.temp.name).glob("live.db.backup_*")), safety
|
||||
)
|
||||
assert_closed(self, source)
|
||||
with live.get_connection() as conn:
|
||||
self.assertEqual(image_tags(conn), [("live-only", 0)])
|
||||
self.assertEqual(metadata(conn), [(1, 0)])
|
||||
self.assertFalse(
|
||||
PromptModel.verify_database_file(old, require_library_state=True)[0]
|
||||
)
|
||||
self.assertTrue(PromptModel.verify_database_file(old)[0])
|
||||
|
||||
def test_verify_requires_group_only_with_require_library_state(self):
|
||||
self.require_module()
|
||||
self.model().close_all()
|
||||
self.assertEqual(
|
||||
PromptModel.verify_database_file(self.path, require_library_state=True),
|
||||
(True, "ok"),
|
||||
)
|
||||
self.drop_group(self.path)
|
||||
self.assertEqual(PromptModel.verify_database_file(self.path), (True, "ok"))
|
||||
self.assertEqual(
|
||||
PromptModel.verify_database_file(self.path, require_library_state=True),
|
||||
(False, "Missing image annotation schema"),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,598 @@
|
||||
"""Per-image tag sets (#249 A2): byte-exact storage in the shared vocabulary."""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
try:
|
||||
from tests.open_handles import assert_closed
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from open_handles import assert_closed
|
||||
|
||||
from database.annotation_schema import annotation_schema_error
|
||||
from database.library_commands import LibraryCommandError
|
||||
from database.operations import PromptDatabase
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
try: # guarded so a missing module fails assertions, not test loading
|
||||
from database import annotations as annotations_module
|
||||
from database.annotations import (
|
||||
MAX_IMAGE_TAGS,
|
||||
AnnotationCommandError,
|
||||
normalize_image_tags,
|
||||
)
|
||||
except ImportError:
|
||||
annotations_module = None
|
||||
MAX_IMAGE_TAGS = None
|
||||
AnnotationCommandError = None
|
||||
normalize_image_tags = None
|
||||
|
||||
UUID = re.compile(r"[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}")
|
||||
DIGEST_A = hashlib.sha256(b"bytes a").hexdigest()
|
||||
DIGEST_B = hashlib.sha256(b"bytes b").hexdigest()
|
||||
SIZE = 7
|
||||
MTIME = 1_700_000_000_123_456_789
|
||||
ANNOTATION_TABLES = (
|
||||
"media_annotation_files",
|
||||
"media_annotation_bytes",
|
||||
"media_image_tags",
|
||||
"media_annotation_metadata",
|
||||
)
|
||||
LEGACY_TABLES = (
|
||||
"tags",
|
||||
"prompt_tags",
|
||||
"prompt_pair_tags",
|
||||
"generated_images",
|
||||
"image_prompt_pairs",
|
||||
)
|
||||
|
||||
|
||||
class AnnotationTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.assertIsNotNone(
|
||||
annotations_module, "database.annotations is not implemented"
|
||||
)
|
||||
directory = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(directory.cleanup)
|
||||
self.addCleanup(assert_closed, self, directory.name)
|
||||
self.root = directory.name
|
||||
self.db = PromptDatabase(os.path.join(self.root, "annotations.db"))
|
||||
self.addCleanup(self.db.close_all)
|
||||
self.epoch = self.db.get_execution_epoch()
|
||||
|
||||
def locator(self, name: str) -> str:
|
||||
return os.path.normcase(os.path.join(os.path.realpath(self.root), name))
|
||||
|
||||
def replace(self, loc, digest=DIGEST_A, tags=(), *, revision=0, **over):
|
||||
options = {
|
||||
"locator": loc,
|
||||
"sha256": digest,
|
||||
"size": SIZE,
|
||||
"mtime_ns": MTIME,
|
||||
"expected_epoch": self.epoch,
|
||||
"expected_tag_revision": revision,
|
||||
"tags": list(tags) if isinstance(tags, tuple) else tags,
|
||||
}
|
||||
options.update(over)
|
||||
return self.db.replace_image_tags(**options)
|
||||
|
||||
def add(self, loc, digest=DIGEST_A, tags=(), **over):
|
||||
options = {
|
||||
"locator": loc,
|
||||
"sha256": digest,
|
||||
"size": SIZE,
|
||||
"mtime_ns": MTIME,
|
||||
"expected_epoch": self.epoch,
|
||||
"tags": list(tags) if isinstance(tags, tuple) else tags,
|
||||
}
|
||||
options.update(over)
|
||||
return self.db.add_image_tags(**options)
|
||||
|
||||
def read(self, loc, digest=DIGEST_A):
|
||||
return self.db.read_image_tags(loc, digest)
|
||||
|
||||
def rows(self, *tables):
|
||||
"""Every row of each table; ``verified_at`` is a fingerprint, not content."""
|
||||
conn = self.db.model.get_connection()
|
||||
result = []
|
||||
for table in tables:
|
||||
columns = [
|
||||
row[1]
|
||||
for row in conn.execute(f"PRAGMA table_info({table})")
|
||||
if row[1] != "verified_at"
|
||||
]
|
||||
result.append(
|
||||
[
|
||||
tuple(row)
|
||||
for row in conn.execute(
|
||||
f"SELECT {','.join(columns)} FROM {table} ORDER BY 1,2"
|
||||
)
|
||||
]
|
||||
)
|
||||
return result
|
||||
|
||||
def counts(self):
|
||||
conn = self.db.model.get_connection()
|
||||
return {
|
||||
table: conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]
|
||||
for table in ANNOTATION_TABLES
|
||||
}
|
||||
|
||||
def index_revision(self):
|
||||
return (
|
||||
self.db.model.get_connection()
|
||||
.execute("SELECT image_index_revision FROM media_annotation_metadata")
|
||||
.fetchone()[0]
|
||||
)
|
||||
|
||||
def schema_clean(self):
|
||||
self.assertIsNone(
|
||||
annotation_schema_error(self.db.model.get_connection(), allow_missing=False)
|
||||
)
|
||||
|
||||
|
||||
class NormalizeImageTagsTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.assertIsNotNone(
|
||||
normalize_image_tags, "database.annotations is not implemented"
|
||||
)
|
||||
|
||||
def test_case_insensitive_dedupe_keeps_first_spelling_and_position(self):
|
||||
self.assertEqual(
|
||||
normalize_image_tags([" Forest ", "mist", "forest", "MIST", "Rain"]),
|
||||
["Forest", "mist", "Rain"],
|
||||
)
|
||||
self.assertEqual(normalize_image_tags([]), [])
|
||||
self.assertEqual(MAX_IMAGE_TAGS, 100)
|
||||
|
||||
def test_limit_and_invalid_names_rejected(self):
|
||||
for bad in (
|
||||
None,
|
||||
"forest",
|
||||
{"a": 1},
|
||||
[1],
|
||||
[None],
|
||||
["a,b"],
|
||||
[""],
|
||||
[" "],
|
||||
["x" * 101],
|
||||
[f"t{i}" for i in range(MAX_IMAGE_TAGS + 1)],
|
||||
):
|
||||
with self.subTest(bad=bad), self.assertRaises(ValueError):
|
||||
normalize_image_tags(bad)
|
||||
# Duplicates fold before the cap is applied.
|
||||
many = [f"t{i}" for i in range(MAX_IMAGE_TAGS)]
|
||||
self.assertEqual(normalize_image_tags(many + ["T0", "T99"]), many)
|
||||
|
||||
|
||||
class ReplaceImageTagsTests(AnnotationTestCase):
|
||||
def test_first_save_creates_owner_and_byte_atomically_including_empty_set(self):
|
||||
loc = self.locator("a.png")
|
||||
receipt = self.replace(loc, tags=["Forest", "mist", "forest"])
|
||||
self.assertEqual(receipt["tags"], ["Forest", "mist"])
|
||||
self.assertEqual(receipt["tag_revision"], 1)
|
||||
self.assertEqual(
|
||||
receipt,
|
||||
{
|
||||
"library_epoch": self.epoch,
|
||||
"sha256": DIGEST_A,
|
||||
"tags": ["Forest", "mist"],
|
||||
"tag_revision": 1,
|
||||
"replayed": False,
|
||||
},
|
||||
)
|
||||
conn = self.db.model.get_connection()
|
||||
files = conn.execute("SELECT * FROM media_annotation_files").fetchall()
|
||||
self.assertEqual(len(files), 1)
|
||||
self.assertRegex(files[0]["id"], UUID)
|
||||
self.assertEqual(files[0]["locator"], loc)
|
||||
byte = conn.execute("SELECT * FROM media_annotation_bytes").fetchone()
|
||||
self.assertRegex(byte["id"], UUID)
|
||||
self.assertEqual(
|
||||
(byte["file_id"], byte["sha256"], byte["size"], byte["mtime_ns"]),
|
||||
(files[0]["id"], DIGEST_A, SIZE, MTIME),
|
||||
)
|
||||
self.assertEqual(byte["tag_revision"], 1)
|
||||
self.assertEqual(byte["verified_at"], byte["first_seen_at"])
|
||||
self.assertEqual(
|
||||
[
|
||||
tuple(row)
|
||||
for row in conn.execute(
|
||||
"SELECT t.name, mit.position FROM media_image_tags mit "
|
||||
"JOIN tags t ON t.id = mit.tag_id ORDER BY mit.position"
|
||||
)
|
||||
],
|
||||
[("Forest", 0), ("mist", 1)],
|
||||
)
|
||||
self.assertEqual(self.index_revision(), 1)
|
||||
# An explicit empty set is a real save: owner and byte record exist.
|
||||
empty = self.replace(self.locator("b.png"), DIGEST_B, [])
|
||||
self.assertEqual((empty["tags"], empty["tag_revision"]), ([], 1))
|
||||
self.assertEqual(self.counts()["media_annotation_files"], 2)
|
||||
self.assertEqual(self.counts()["media_annotation_bytes"], 2)
|
||||
self.assertEqual(
|
||||
self.db.read_image_tags(self.locator("b.png"), DIGEST_B),
|
||||
{"tags": [], "tag_revision": 1, "prior_count": 0},
|
||||
)
|
||||
self.assertEqual(self.index_revision(), 2)
|
||||
self.schema_clean()
|
||||
|
||||
def test_save_changes_only_target_byte_set(self):
|
||||
a, b, c = self.locator("a.png"), self.locator("b.png"), self.locator("c.png")
|
||||
prompt = self.db.save_prompt(
|
||||
"Shared forest",
|
||||
tags=["shared"],
|
||||
prompt_hash=generate_prompt_hash("Shared forest"),
|
||||
)
|
||||
pair = self.db.save_prompt_pair("Shared forest", "blurry", tags=["pairtag"])
|
||||
image_a = self.db.link_image_to_prompt(prompt, a)
|
||||
image_b = self.db.link_image_to_prompt(prompt, b)
|
||||
self.db.link_image_to_pair(pair, image_a, {"seed": 1})
|
||||
self.db.link_image_to_pair(pair, image_b, {"seed": 2})
|
||||
self.replace(b, DIGEST_A, ["sibling"])
|
||||
self.replace(c, DIGEST_A, ["copy"]) # equal bytes at another path
|
||||
before = self.rows(*LEGACY_TABLES)
|
||||
sibling_before = self.db.read_image_tags(b, DIGEST_A)
|
||||
copy_before = self.db.read_image_tags(c, DIGEST_A)
|
||||
|
||||
receipt = self.replace(a, DIGEST_A, ["only a"])
|
||||
|
||||
self.assertEqual((receipt["tags"], receipt["tag_revision"]), (["only a"], 1))
|
||||
after = self.rows(*LEGACY_TABLES)
|
||||
# The registry gained the new name; nothing else moved.
|
||||
self.assertEqual(after[1:], before[1:])
|
||||
self.assertEqual([row for row in after[0] if row[1] != "only a"], before[0])
|
||||
self.assertEqual(self.db.read_image_tags(b, DIGEST_A), sibling_before)
|
||||
self.assertEqual(self.db.read_image_tags(c, DIGEST_A), copy_before)
|
||||
self.assertEqual(self.db.get_prompt_by_id(prompt)["tags"], ["shared"])
|
||||
self.assertEqual(self.db.get_prompt_pair(pair)["tags"], ["pairtag"])
|
||||
|
||||
def test_changed_bytes_at_same_locator_start_empty_and_keep_prior(self):
|
||||
loc = self.locator("a.png")
|
||||
self.replace(loc, DIGEST_A, ["old"])
|
||||
self.assertEqual(
|
||||
self.db.read_image_tags(loc, DIGEST_B),
|
||||
{"tags": [], "tag_revision": 0, "prior_count": 1},
|
||||
)
|
||||
receipt = self.replace(loc, DIGEST_B, ["new"], mtime_ns=MTIME + 5)
|
||||
self.assertEqual((receipt["tags"], receipt["tag_revision"]), (["new"], 1))
|
||||
self.assertEqual(
|
||||
self.db.read_image_tags(loc, DIGEST_A),
|
||||
{"tags": ["old"], "tag_revision": 1, "prior_count": 1},
|
||||
)
|
||||
self.assertEqual(
|
||||
self.db.read_image_tags(loc, DIGEST_B),
|
||||
{"tags": ["new"], "tag_revision": 1, "prior_count": 1},
|
||||
)
|
||||
self.assertEqual(self.counts()["media_annotation_files"], 1)
|
||||
self.assertEqual(self.counts()["media_annotation_bytes"], 2)
|
||||
self.assertEqual(
|
||||
self.db.read_image_tags(self.locator("unseen.png"), DIGEST_A),
|
||||
{"tags": [], "tag_revision": 0, "prior_count": 0},
|
||||
)
|
||||
|
||||
def test_revision_conflict_returns_current_set(self):
|
||||
loc = self.locator("a.png")
|
||||
self.replace(loc, tags=["a"])
|
||||
before = self.rows(*ANNOTATION_TABLES, "tags")
|
||||
with self.assertRaises(AnnotationCommandError) as ctx:
|
||||
self.replace(loc, tags=["b"], revision=0)
|
||||
self.assertEqual(
|
||||
(ctx.exception.code, ctx.exception.current["tag_revision"]),
|
||||
("tag-conflict", 1),
|
||||
)
|
||||
self.assertIsInstance(ctx.exception, LibraryCommandError)
|
||||
self.assertEqual(ctx.exception.epoch, self.epoch)
|
||||
self.assertEqual(ctx.exception.current["tags"], ["a"])
|
||||
self.assertEqual(ctx.exception.current["sha256"], DIGEST_A)
|
||||
with self.assertRaises(AnnotationCommandError) as ahead:
|
||||
self.replace(loc, tags=["a"], revision=5)
|
||||
self.assertEqual(ahead.exception.code, "tag-conflict")
|
||||
self.assertEqual(self.rows(*ANNOTATION_TABLES, "tags"), before)
|
||||
self.assertFalse(self.db.model.get_connection().in_transaction)
|
||||
|
||||
def test_identical_replay_after_commit_is_success_replayed(self):
|
||||
loc = self.locator("a.png")
|
||||
first = self.replace(loc, tags=["a", "b"])
|
||||
self.assertEqual((first["tag_revision"], first["replayed"]), (1, False))
|
||||
before = self.rows(*ANNOTATION_TABLES)
|
||||
# The lost-response retry: same set, the revision it was sent against.
|
||||
retry = self.replace(loc, tags=["a", "b"], revision=0)
|
||||
self.assertEqual(retry, {**first, "replayed": True})
|
||||
# Saving the current set at the current revision writes nothing either.
|
||||
again = self.replace(loc, tags=["a", "b"], revision=1)
|
||||
self.assertEqual(again, {**first, "replayed": True})
|
||||
self.assertEqual(self.rows(*ANNOTATION_TABLES), before)
|
||||
# Order is part of the set.
|
||||
reordered = self.replace(loc, tags=["b", "a"], revision=1)
|
||||
self.assertEqual(
|
||||
(reordered["tags"], reordered["tag_revision"], reordered["replayed"]),
|
||||
(["b", "a"], 2, False),
|
||||
)
|
||||
self.assertEqual(self.index_revision(), 2)
|
||||
|
||||
def test_replay_with_new_fingerprint_moves_only_the_index_revision(self):
|
||||
loc = self.locator("a.png")
|
||||
first = self.replace(loc, tags=["a"])
|
||||
index = self.index_revision()
|
||||
same = self.replace(loc, tags=["a"], revision=1)
|
||||
self.assertEqual(same, {**first, "replayed": True})
|
||||
self.assertEqual(self.index_revision(), index) # identical: nothing moved
|
||||
touched = self.replace(loc, tags=["a"], revision=1, mtime_ns=MTIME + 5)
|
||||
self.assertEqual(touched, {**first, "replayed": True})
|
||||
self.assertEqual(self.index_revision(), index + 1) # currency changed
|
||||
self.assertEqual(
|
||||
self.db.current_image_tags({loc: (SIZE, MTIME + 5)}), {loc: ["a"]}
|
||||
)
|
||||
self.assertEqual(self.db.current_image_tags({loc: (SIZE, MTIME)}), {})
|
||||
self.assertEqual(self.read(loc)["tag_revision"], 1)
|
||||
|
||||
def test_epoch_checked_inside_transaction(self):
|
||||
loc = self.locator("a.png")
|
||||
seen = []
|
||||
original = PromptDatabase._library_epoch
|
||||
|
||||
def spy(db, conn, expected_epoch):
|
||||
seen.append(conn.in_transaction)
|
||||
return original(db, conn, expected_epoch)
|
||||
|
||||
with patch.object(PromptDatabase, "_library_epoch", spy):
|
||||
self.replace(loc, tags=["a"])
|
||||
self.add(loc, tags=["b"])
|
||||
self.assertEqual(seen, [True, True])
|
||||
stale = "11111111-2222-4333-8444-555555555555"
|
||||
for call in (
|
||||
lambda: self.replace(
|
||||
self.locator("x.png"), tags=["a"], expected_epoch=stale
|
||||
),
|
||||
lambda: self.add(self.locator("x.png"), tags=["a"], expected_epoch=stale),
|
||||
):
|
||||
with self.assertRaises(LibraryCommandError) as ctx:
|
||||
call()
|
||||
self.assertEqual(ctx.exception.code, "storage-changed")
|
||||
self.assertEqual(ctx.exception.epoch, self.epoch)
|
||||
self.assertEqual(self.counts()["media_annotation_files"], 1)
|
||||
for bad_epoch in (None, 7, "not-a-uuid"):
|
||||
with self.assertRaises(ValueError):
|
||||
self.replace(loc, tags=["a"], expected_epoch=bad_epoch)
|
||||
|
||||
def test_limit_and_invalid_names_rejected_without_partial_write(self):
|
||||
loc = self.locator("a.png")
|
||||
before = self.rows(*ANNOTATION_TABLES, "tags")
|
||||
for tags in (
|
||||
["ok", "a,b"],
|
||||
["ok", ""],
|
||||
["ok", 3],
|
||||
"ok",
|
||||
[f"t{i}" for i in range(101)],
|
||||
):
|
||||
with self.subTest(tags=tags), self.assertRaises(ValueError):
|
||||
self.replace(loc, tags=tags)
|
||||
with self.subTest(tags=tags, command="add"), self.assertRaises(ValueError):
|
||||
self.add(loc, tags=tags)
|
||||
for field, value in (
|
||||
("locator", ""),
|
||||
("locator", None),
|
||||
("locator", "x" * 4097),
|
||||
("sha256", DIGEST_A.upper()),
|
||||
("sha256", DIGEST_A[:-1]),
|
||||
("sha256", None),
|
||||
("size", -1),
|
||||
("size", True),
|
||||
("size", "7"),
|
||||
("mtime_ns", 1.5),
|
||||
("mtime_ns", None),
|
||||
("expected_tag_revision", -1),
|
||||
("expected_tag_revision", False),
|
||||
):
|
||||
with self.subTest(field=field, value=value), self.assertRaises(ValueError):
|
||||
self.replace(loc, tags=["ok"], **{field: value})
|
||||
self.assertEqual(self.rows(*ANNOTATION_TABLES, "tags"), before)
|
||||
self.assertEqual(self.index_revision(), 0)
|
||||
|
||||
def test_writers_supply_uuid_ids_and_refuse_to_write_without_one(self):
|
||||
loc = self.locator("a.png")
|
||||
before = self.rows(*ANNOTATION_TABLES)
|
||||
with patch.object(annotations_module, "uuid4", return_value=""):
|
||||
with self.assertRaises(ValueError):
|
||||
self.replace(loc, tags=["a"])
|
||||
self.assertEqual(self.rows(*ANNOTATION_TABLES), before)
|
||||
self.replace(loc, tags=["a"])
|
||||
self.add(self.locator("b.png"), DIGEST_B, ["b"])
|
||||
conn = self.db.model.get_connection()
|
||||
for table in ("media_annotation_files", "media_annotation_bytes"):
|
||||
for (value,) in conn.execute(f"SELECT id FROM {table}"):
|
||||
self.assertRegex(value, UUID)
|
||||
self.schema_clean()
|
||||
|
||||
def test_never_resolves_owner_through_prompt_lookup(self):
|
||||
loc = self.locator("a.png")
|
||||
prompt = self.db.save_prompt("Linked", tags=["prompt-tag"])
|
||||
self.db.link_image_to_prompt(prompt, loc)
|
||||
with patch.object(
|
||||
PromptDatabase,
|
||||
"get_prompt_id_for_image",
|
||||
side_effect=AssertionError("prompt lookup is not an owner"),
|
||||
):
|
||||
self.replace(loc, tags=["img"])
|
||||
self.add(loc, tags=["more"])
|
||||
self.assertEqual(
|
||||
self.db.read_image_tags(loc, DIGEST_A)["tags"], ["img", "more"]
|
||||
)
|
||||
self.assertEqual(
|
||||
self.db.current_image_tags({loc: (SIZE, MTIME)}),
|
||||
{loc: ["img", "more"]},
|
||||
)
|
||||
self.assertTrue(
|
||||
self.db.refresh_image_fingerprint(loc, DIGEST_A, SIZE, MTIME + 1)
|
||||
)
|
||||
list(self.db.iter_image_tag_records())
|
||||
self.assertEqual(self.db.get_prompt_by_id(prompt)["tags"], ["prompt-tag"])
|
||||
|
||||
|
||||
class AddImageTagsTests(AnnotationTestCase):
|
||||
def test_add_merges_without_removing_and_caps_at_100(self):
|
||||
loc = self.locator("a.png")
|
||||
self.replace(loc, tags=["Forest", "mist"])
|
||||
receipt = self.add(loc, tags=["forest", "Rain", "rain"])
|
||||
self.assertEqual(
|
||||
receipt,
|
||||
{
|
||||
"library_epoch": self.epoch,
|
||||
"sha256": DIGEST_A,
|
||||
"tags": ["Forest", "mist", "Rain"],
|
||||
"tag_revision": 2,
|
||||
"replayed": False,
|
||||
"added": 1,
|
||||
},
|
||||
)
|
||||
nothing = self.add(loc, tags=["MIST"])
|
||||
self.assertEqual(
|
||||
(nothing["tags"], nothing["tag_revision"], nothing["replayed"]),
|
||||
(["Forest", "mist", "Rain"], 2, True),
|
||||
)
|
||||
self.assertEqual(nothing["added"], 0)
|
||||
before = self.rows(*ANNOTATION_TABLES, "tags")
|
||||
with self.assertRaises(AnnotationCommandError) as ctx:
|
||||
self.add(loc, tags=[f"t{i}" for i in range(98)])
|
||||
self.assertEqual(ctx.exception.code, "tag-limit")
|
||||
self.assertEqual(ctx.exception.current["tag_revision"], 2)
|
||||
self.assertEqual(self.rows(*ANNOTATION_TABLES, "tags"), before)
|
||||
exact = self.add(loc, tags=[f"t{i}" for i in range(97)])
|
||||
self.assertEqual((len(exact["tags"]), exact["added"]), (100, 97))
|
||||
# An unseen file: add creates the owner and byte record like a save.
|
||||
fresh = self.add(self.locator("b.png"), DIGEST_B, ["one"])
|
||||
self.assertEqual(
|
||||
(fresh["tags"], fresh["tag_revision"], fresh["added"]), (["one"], 1, 1)
|
||||
)
|
||||
self.schema_clean()
|
||||
|
||||
|
||||
class CurrentImageTagsTests(AnnotationTestCase):
|
||||
def test_current_tags_require_matching_fingerprint(self):
|
||||
loc = self.locator("a.png")
|
||||
other = self.locator("b.png")
|
||||
self.replace(loc, tags=["a"])
|
||||
self.replace(other, DIGEST_B, [], mtime_ns=MTIME + 9)
|
||||
self.assertEqual(
|
||||
self.db.current_image_tags({loc: (SIZE, MTIME), other: (SIZE, MTIME + 9)}),
|
||||
{loc: ["a"], other: []},
|
||||
)
|
||||
self.assertEqual(self.db.current_image_tags({loc: (SIZE, MTIME + 1)}), {})
|
||||
self.assertEqual(self.db.current_image_tags({loc: (SIZE + 1, MTIME)}), {})
|
||||
self.assertEqual(self.db.current_image_tags({}), {})
|
||||
self.assertEqual(
|
||||
self.db.current_image_tags({self.locator("none.png"): (SIZE, MTIME)}), {}
|
||||
)
|
||||
# Refreshing the verified fingerprint makes the record current again.
|
||||
self.assertTrue(
|
||||
self.db.refresh_image_fingerprint(loc, DIGEST_A, SIZE, MTIME + 1)
|
||||
)
|
||||
self.assertFalse(self.db.refresh_image_fingerprint(loc, DIGEST_B, SIZE, MTIME))
|
||||
self.assertFalse(
|
||||
self.db.refresh_image_fingerprint(
|
||||
self.locator("x.png"), DIGEST_A, SIZE, MTIME
|
||||
)
|
||||
)
|
||||
self.assertEqual(
|
||||
self.db.current_image_tags({loc: (SIZE, MTIME + 1)}), {loc: ["a"]}
|
||||
)
|
||||
self.assertEqual(self.db.current_image_tags({loc: (SIZE, MTIME)}), {})
|
||||
self.assertEqual(
|
||||
self.db.read_image_tags(loc, DIGEST_A),
|
||||
{"tags": ["a"], "tag_revision": 1, "prior_count": 0},
|
||||
)
|
||||
for bad in (None, [], {loc: (SIZE,)}, {loc: (SIZE, 1.5)}, {3: (SIZE, MTIME)}):
|
||||
with self.subTest(bad=bad), self.assertRaises(ValueError):
|
||||
self.db.current_image_tags(bad)
|
||||
|
||||
def test_newest_verification_wins_on_equal_fingerprints(self):
|
||||
loc = self.locator("a.png")
|
||||
stamps = iter(f"2026-10-22T00:00:0{i}.000000+00:00" for i in range(10))
|
||||
with patch.object(annotations_module, "utc_now_iso", side_effect=stamps):
|
||||
self.replace(loc, DIGEST_A, ["old bytes"])
|
||||
self.replace(loc, DIGEST_B, ["new bytes"]) # same size and mtime
|
||||
self.assertEqual(
|
||||
self.db.current_image_tags({loc: (SIZE, MTIME)}), {loc: ["new bytes"]}
|
||||
)
|
||||
with patch.object(
|
||||
annotations_module,
|
||||
"utc_now_iso",
|
||||
return_value="2026-10-22T00:01:00.000000+00:00",
|
||||
):
|
||||
self.assertTrue(
|
||||
self.db.refresh_image_fingerprint(loc, DIGEST_A, SIZE, MTIME)
|
||||
)
|
||||
self.assertEqual(
|
||||
self.db.current_image_tags({loc: (SIZE, MTIME)}), {loc: ["old bytes"]}
|
||||
)
|
||||
|
||||
def test_lookup_is_chunked_and_records_stream_in_position_order(self):
|
||||
locators = [self.locator(f"{i:04}.png") for i in range(503)]
|
||||
self.replace(locators[0], tags=["z", "y"])
|
||||
self.replace(locators[502], DIGEST_B, ["x"], mtime_ns=MTIME + 2)
|
||||
self.replace(locators[250], DIGEST_B, [], mtime_ns=MTIME + 1)
|
||||
statements = []
|
||||
conn = self.db.model.get_connection()
|
||||
conn.set_trace_callback(statements.append)
|
||||
try:
|
||||
current = self.db.current_image_tags(
|
||||
{
|
||||
loc: (SIZE, MTIME + (2 if i == 502 else 0))
|
||||
for i, loc in enumerate(locators)
|
||||
}
|
||||
)
|
||||
finally:
|
||||
conn.set_trace_callback(None)
|
||||
self.assertEqual(current, {locators[0]: ["z", "y"], locators[502]: ["x"]})
|
||||
lookups = [s for s in statements if "media_annotation_files" in s]
|
||||
self.assertEqual(len(lookups), 2)
|
||||
self.assertEqual(
|
||||
list(self.db.iter_image_tag_records()),
|
||||
[
|
||||
(locators[0], SIZE, MTIME, ["z", "y"]),
|
||||
(locators[250], SIZE, MTIME + 1, []),
|
||||
(locators[502], SIZE, MTIME + 2, ["x"]),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class AnnotationIsolationTests(AnnotationTestCase):
|
||||
def test_cascade_from_vocabulary_delete_keeps_byte_records(self):
|
||||
loc = self.locator("a.png")
|
||||
self.replace(loc, tags=["gone", "kept"])
|
||||
self.db.delete_tag_all_prompts("gone")
|
||||
self.assertEqual(
|
||||
self.db.read_image_tags(loc, DIGEST_A),
|
||||
{
|
||||
"tags": ["kept"],
|
||||
"tag_revision": 2,
|
||||
"prior_count": 0,
|
||||
}, # moved by the delete
|
||||
)
|
||||
self.schema_clean()
|
||||
|
||||
def test_junction_failure_leaves_nothing_behind(self):
|
||||
loc = self.locator("a.png")
|
||||
self.replace(loc, tags=["a"])
|
||||
conn = self.db.model.get_connection()
|
||||
conn.execute(
|
||||
"CREATE TRIGGER reject_image_tag BEFORE INSERT ON media_image_tags "
|
||||
"BEGIN SELECT RAISE(ABORT,'image tag failure'); END"
|
||||
)
|
||||
conn.commit()
|
||||
before = self.rows(*ANNOTATION_TABLES, "tags")
|
||||
with self.assertRaises(sqlite3.IntegrityError):
|
||||
self.replace(loc, tags=["b"], revision=1)
|
||||
with self.assertRaises(sqlite3.IntegrityError):
|
||||
self.add(self.locator("new.png"), DIGEST_B, ["c"])
|
||||
self.assertFalse(conn.in_transaction)
|
||||
self.assertEqual(self.rows(*ANNOTATION_TABLES, "tags"), before)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+465
-1
@@ -1965,6 +1965,419 @@ class TestOutputScanIsABackgroundJob(AdminAPITestCase):
|
||||
self.assertEqual(data["last_event"]["type"], "error")
|
||||
|
||||
|
||||
class TestActivityJobs(AdminAPITestCase):
|
||||
"""The scan runs as a registry job; legacy routes keep their payloads."""
|
||||
|
||||
def _gate_collect(self):
|
||||
import threading
|
||||
|
||||
gate = threading.Event()
|
||||
calls = []
|
||||
original = self.api._collect_output_media_sync
|
||||
|
||||
def gated(output_dirs):
|
||||
calls.append(output_dirs)
|
||||
gate.wait(5)
|
||||
return original(output_dirs)
|
||||
|
||||
self.api._collect_output_media_sync = gated
|
||||
return gate, calls
|
||||
|
||||
async def _json(self, method, url):
|
||||
resp = await self.client.request(method, url)
|
||||
return resp.status, await resp.json()
|
||||
|
||||
async def test_legacy_scan_route_reuses_registry_job_and_payload_shape(self):
|
||||
from py.api.admin import SCAN_COUNT_KEYS
|
||||
|
||||
_write_png(self.output_dir / "gen.png", prompt_text="a red square")
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
gate, calls = self._gate_collect()
|
||||
|
||||
# The workspace starts the job; the legacy page then attaches to it.
|
||||
status, started = await self._json("POST", "/prompt_manager/jobs/scan")
|
||||
self.assertEqual(status, 202)
|
||||
self.assertTrue(started["success"])
|
||||
legacy = asyncio.ensure_future(
|
||||
self.client.request("POST", "/prompt_manager/scan", data="{}")
|
||||
)
|
||||
await asyncio.sleep(0.05)
|
||||
gate.set()
|
||||
body = await (await legacy).text()
|
||||
await asyncio.wait_for(self.api._scan_job.task, 10)
|
||||
|
||||
self.assertEqual(len(calls), 1)
|
||||
payloads = _sse_payloads(body)
|
||||
progress = [p for p in payloads if p["type"] == "progress"]
|
||||
self.assertTrue(progress)
|
||||
legacy_keys = set(SCAN_COUNT_KEYS) | {"type", "progress", "status", "processed"}
|
||||
for payload in progress:
|
||||
self.assertEqual(set(payload), legacy_keys)
|
||||
final = payloads[-1]
|
||||
self.assertEqual(final["type"], "complete")
|
||||
self.assertEqual(set(final), set(SCAN_COUNT_KEYS) | {"type", "directories"})
|
||||
self.assertEqual(final["added"], 1)
|
||||
self.assertEqual(final["directories"], ["output"])
|
||||
|
||||
status, listed = await self._json("GET", "/prompt_manager/jobs")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(len(listed["jobs"]), 1)
|
||||
job = listed["jobs"][0]
|
||||
self.assertEqual(job["id"], started["job"]["id"])
|
||||
self.assertEqual(job["id"], self.api._scan_job.id)
|
||||
self.assertEqual((job["kind"], job["state"]), ("scan", "finished"))
|
||||
self.assertEqual(job["counters"]["added"], 1)
|
||||
self.assertEqual(job["progress"], 100)
|
||||
self.assertEqual(job["scope"]["output_folders"], 1)
|
||||
|
||||
async def test_scan_records_outcomes_by_group(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
(self.output_dir / "sub").mkdir()
|
||||
names = {
|
||||
"sub/added.png": {"status": "added", "title": "A red fox"},
|
||||
"linked.png": {"status": "linked", "title": "A grey cat"},
|
||||
"unchanged.png": {"status": "unchanged", "title": "A brown dog"},
|
||||
"ambiguous.png": {"status": "ambiguous"},
|
||||
"failed.png": RuntimeError(f"{self.tmpdir}/secret boom"),
|
||||
"none.png": None,
|
||||
}
|
||||
for name in names:
|
||||
(self.output_dir / name).write_bytes(b"not an image")
|
||||
|
||||
def fake_ingest(media_file, metadata, detailed=False):
|
||||
key = Path(media_file).relative_to(self.output_dir).as_posix()
|
||||
outcome = names[key]
|
||||
if isinstance(outcome, Exception):
|
||||
raise outcome
|
||||
return outcome
|
||||
|
||||
self.api._ingest_scanned_file = fake_ingest
|
||||
with self.assertLogs("prompt_manager.api", level="ERROR"):
|
||||
status, started = await self._json("POST", "/prompt_manager/jobs/scan")
|
||||
self.assertEqual(status, 202)
|
||||
await asyncio.wait_for(self.api._scan_job.task, 10)
|
||||
|
||||
resp = await self.client.request(
|
||||
"GET", f"/prompt_manager/jobs/{started['job']['id']}"
|
||||
)
|
||||
text = await resp.text()
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertNotIn(self.tmpdir, text)
|
||||
job = json.loads(text)["job"]
|
||||
self.assertEqual(
|
||||
job["counters"],
|
||||
{"added": 2, "known": 1, "review": 1, "no_metadata": 1, "errors": 1},
|
||||
)
|
||||
outcomes = job["outcomes"]
|
||||
self.assertEqual(
|
||||
{k: len(v) for k, v in outcomes.items()},
|
||||
{"added": 2, "known": 1, "review": 1, "no_metadata": 1, "errors": 1},
|
||||
)
|
||||
added = {row["path"]: row for row in outcomes["added"]}
|
||||
self.assertEqual(set(added), {"sub/added.png", "linked.png"})
|
||||
self.assertIn("A red fox", added["sub/added.png"]["detail"])
|
||||
self.assertIn("A grey cat", added["linked.png"]["detail"])
|
||||
self.assertEqual(added["linked.png"]["root"], "output")
|
||||
self.assertIn("A brown dog", outcomes["known"][0]["detail"])
|
||||
self.assertIn("review", outcomes["review"][0]["detail"].lower())
|
||||
self.assertEqual(outcomes["errors"][0]["path"], "failed.png")
|
||||
self.assertTrue(outcomes["errors"][0]["detail"])
|
||||
self.assertEqual(outcomes["no_metadata"][0]["path"], "none.png")
|
||||
self.assertTrue(outcomes["no_metadata"][0]["detail"])
|
||||
prefixes = {line.split(" ", 2)[1] for line in job["log"][1:-1]}
|
||||
self.assertEqual(prefixes, {"+", "=", "?", "!", "-"})
|
||||
self.assertEqual(job["state"], "finished")
|
||||
self.assertTrue(job["status_line"].startswith("Processing file 6 of 6"))
|
||||
|
||||
legacy = self.api._scan_job.last_event
|
||||
self.assertEqual(
|
||||
(legacy["processed"], legacy["found"], legacy["failed"]), (6, 4, 1)
|
||||
)
|
||||
|
||||
async def test_report_is_attachment_without_absolute_paths(self):
|
||||
"""Task 5: the report downloads as JSON, whole and without server paths."""
|
||||
import re
|
||||
|
||||
from py.api.jobs import MAX_OUTCOME_ROWS
|
||||
|
||||
async def runner(job):
|
||||
absolute = f"{self.tmpdir}/output/deep/abs.png"
|
||||
job.record(
|
||||
"added",
|
||||
{"path": absolute, "root": "output", "root_index": 0, "detail": "ok"},
|
||||
f"+ {absolute} · ok",
|
||||
)
|
||||
for index in range(MAX_OUTCOME_ROWS + 5):
|
||||
job.record(
|
||||
"known",
|
||||
{"path": f"k/{index}.png", "root": "output", "detail": "x"},
|
||||
f"= k/{index}.png",
|
||||
)
|
||||
job.update(processed=MAX_OUTCOME_ROWS + 6, total=MAX_OUTCOME_ROWS + 6)
|
||||
|
||||
job = self.api._jobs.start(
|
||||
"scan",
|
||||
"Scan for metadata",
|
||||
record_changing=True,
|
||||
scope={"files": "new", "output_folders": 1},
|
||||
runner=runner,
|
||||
groups=("added", "known", "review", "no_metadata", "errors"),
|
||||
)
|
||||
await asyncio.wait_for(job.task, 10)
|
||||
|
||||
resp = await self.client.request("GET", f"/prompt_manager/jobs/{job.id}/report")
|
||||
text = await resp.text()
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertIn("application/json", resp.headers["Content-Type"])
|
||||
disposition = resp.headers["Content-Disposition"]
|
||||
self.assertRegex(
|
||||
disposition,
|
||||
r'^attachment; filename="prompt-manager-scan-\d{8}-\d{4}\.json"$',
|
||||
)
|
||||
self.assertNotIn(self.tmpdir, text)
|
||||
self.assertIsNone(
|
||||
re.search(r"[A-Za-z]:[\\/]|(?<![\w.])/(?:tmp|home|var)/", text)
|
||||
)
|
||||
report = json.loads(text)
|
||||
self.assertEqual(report["job"]["id"], job.id)
|
||||
self.assertEqual(report["job"]["scope"]["files"], "new")
|
||||
self.assertEqual(report["job"]["counters"]["known"], MAX_OUTCOME_ROWS + 5)
|
||||
added = report["outcomes"]["added"]
|
||||
self.assertEqual(
|
||||
(added["count"], len(added["rows"]), added["truncated"]), (1, 1, False)
|
||||
)
|
||||
self.assertEqual(added["rows"][0]["path"], "abs.png")
|
||||
known = report["outcomes"]["known"]
|
||||
self.assertEqual(
|
||||
(known["count"], len(known["rows"]), known["truncated"]),
|
||||
(MAX_OUTCOME_ROWS + 5, MAX_OUTCOME_ROWS, True),
|
||||
)
|
||||
self.assertEqual(report["outcomes"]["errors"]["rows"], [])
|
||||
self.assertFalse(report["outcomes"]["errors"]["truncated"])
|
||||
self.assertEqual(report["log"], job.detail()["log"])
|
||||
self.assertIn("generated_at", report)
|
||||
|
||||
resp = await self.client.request("GET", "/prompt_manager/jobs/nope/report")
|
||||
self.assertEqual(resp.status, 404)
|
||||
|
||||
async def test_cancel_after_first_batch_keeps_partial_counts(self):
|
||||
"""Review M7: the real scan stops at the batch boundary, keeping batch 1."""
|
||||
import threading
|
||||
from unittest import mock
|
||||
|
||||
from py.api import admin
|
||||
|
||||
_write_png(self.output_dir / "a.png", prompt_text="a red square")
|
||||
_write_png(self.output_dir / "b.png", prompt_text="a blue circle")
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
first_done, release = threading.Event(), threading.Event()
|
||||
original = self.api._ingest_scan_batch_sync
|
||||
|
||||
def gated(batch_results, roots=(), outcomes=None):
|
||||
counts = original(batch_results, roots=roots, outcomes=outcomes)
|
||||
first_done.set()
|
||||
release.wait(5)
|
||||
return counts
|
||||
|
||||
self.api._ingest_scan_batch_sync = gated
|
||||
with mock.patch.object(admin, "SCAN_BATCH_SIZE", 1):
|
||||
status, started = await self._json("POST", "/prompt_manager/jobs/scan")
|
||||
self.assertEqual(status, 202)
|
||||
job_id = started["job"]["id"]
|
||||
await asyncio.get_running_loop().run_in_executor(None, first_done.wait, 5)
|
||||
status, cancelled = await self._json(
|
||||
"POST", f"/prompt_manager/jobs/{job_id}/cancel"
|
||||
)
|
||||
self.assertEqual((status, cancelled["cancellation_requested"]), (200, True))
|
||||
release.set()
|
||||
await asyncio.wait_for(self.api._scan_job.task, 10)
|
||||
|
||||
status, body = await self._json("GET", f"/prompt_manager/jobs/{job_id}")
|
||||
job = body["job"]
|
||||
self.assertEqual(job["state"], "cancelled")
|
||||
self.assertEqual((job["processed"], job["total"]), (1, 2))
|
||||
self.assertEqual(sum(job["counters"].values()), job["processed"])
|
||||
self.assertEqual(job["counters"]["added"], 1)
|
||||
listed = [row["path"] for rows in job["outcomes"].values() for row in rows]
|
||||
self.assertEqual(len(listed), 1)
|
||||
self.assertIn(listed[0], {"a.png", "b.png"})
|
||||
self.assertIn("Scan cancelled · 1 file processed", job["log"][-1])
|
||||
# A cancel after the end is a no-op.
|
||||
status, again = await self._json(
|
||||
"POST", f"/prompt_manager/jobs/{job_id}/cancel"
|
||||
)
|
||||
self.assertEqual((status, again["cancellation_requested"]), (200, False))
|
||||
|
||||
async def test_scan_rows_carry_their_root_index(self):
|
||||
"""Review M9: two roots labelled "output" stay apart by listing index."""
|
||||
second = self.outside_dir / "output"
|
||||
second.mkdir(parents=True)
|
||||
os.environ[EXTRA_ROOTS_ENV] = str(second)
|
||||
_write_png(self.output_dir / "same.png", prompt_text="a red square")
|
||||
_write_png(second / "same.png", prompt_text="a blue circle")
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [
|
||||
self._root(self.output_dir),
|
||||
self._root(second),
|
||||
]
|
||||
output_dirs = [Path(d) for d in self.api._get_all_output_dirs()]
|
||||
self.assertEqual(len(output_dirs), 2)
|
||||
status, started = await self._json("POST", "/prompt_manager/jobs/scan")
|
||||
await asyncio.wait_for(self.api._scan_job.task, 10)
|
||||
status, body = await self._json(
|
||||
"GET", f"/prompt_manager/jobs/{started['job']['id']}"
|
||||
)
|
||||
rows = body["job"]["outcomes"]["added"]
|
||||
self.assertEqual(len(rows), 2)
|
||||
self.assertEqual({row["root"] for row in rows}, {"output"})
|
||||
by_index = {row["root_index"]: row for row in rows}
|
||||
self.assertEqual(set(by_index), {0, 1})
|
||||
for index, row in by_index.items():
|
||||
self.assertEqual(row["path"], "same.png")
|
||||
self.assertTrue((output_dirs[index] / row["path"]).is_file())
|
||||
texts = {
|
||||
index: (
|
||||
"red"
|
||||
if output_dirs[index] == Path(self._root(self.output_dir))
|
||||
else "blue"
|
||||
)
|
||||
for index in by_index
|
||||
}
|
||||
for index, row in by_index.items():
|
||||
self.assertIn(texts[index], row["detail"])
|
||||
|
||||
async def test_scan_row_names_the_prompt_by_its_stored_title(self):
|
||||
"""Review M10: a row names the prompt as the Library does."""
|
||||
_write_png(self.output_dir / "gen.png", prompt_text="a red square\nmore")
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
summaries = self.api.db.get_image_link_summaries
|
||||
|
||||
def titled(paths, allow_filename_fallback=True):
|
||||
found = summaries(paths, allow_filename_fallback)
|
||||
return {
|
||||
path: {**summary, "title": "Stored square title"}
|
||||
for path, summary in found.items()
|
||||
}
|
||||
|
||||
self.api.db.get_image_link_summaries = titled
|
||||
status, started = await self._json("POST", "/prompt_manager/jobs/scan")
|
||||
await asyncio.wait_for(self.api._scan_job.task, 10)
|
||||
status, body = await self._json(
|
||||
"GET", f"/prompt_manager/jobs/{started['job']['id']}"
|
||||
)
|
||||
(row,) = body["job"]["outcomes"]["added"]
|
||||
self.assertIn("“Stored square title”", row["detail"])
|
||||
|
||||
# Without a stored title (or an unreadable one): the text's first line.
|
||||
def broken(paths, allow_filename_fallback=True):
|
||||
raise RuntimeError("unreadable")
|
||||
|
||||
self.api.db.get_image_link_summaries = broken
|
||||
_write_png(self.output_dir / "next.png", prompt_text="a green cone\nmore")
|
||||
status, started = await self._json("POST", "/prompt_manager/jobs/scan")
|
||||
await asyncio.wait_for(self.api._scan_job.task, 10)
|
||||
status, body = await self._json(
|
||||
"GET", f"/prompt_manager/jobs/{started['job']['id']}"
|
||||
)
|
||||
details = {r["path"]: r["detail"] for r in body["job"]["outcomes"]["added"]}
|
||||
self.assertIn("“a green cone”", details["next.png"])
|
||||
|
||||
async def test_jobs_routes_404_and_409(self):
|
||||
_write_png(self.output_dir / "gen.png", prompt_text="a red square")
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
for method, url in (
|
||||
("GET", "/prompt_manager/jobs/nope"),
|
||||
("GET", "/prompt_manager/jobs/nope/events"),
|
||||
("POST", "/prompt_manager/jobs/nope/cancel"),
|
||||
):
|
||||
status, data = await self._json(method, url)
|
||||
self.assertEqual(status, 404, url)
|
||||
self.assertFalse(data["success"])
|
||||
|
||||
gate, calls = self._gate_collect()
|
||||
status, started = await self._json("POST", "/prompt_manager/jobs/scan")
|
||||
self.assertEqual(status, 202)
|
||||
job_id = started["job"]["id"]
|
||||
self.assertEqual(started["job"]["state"], "running")
|
||||
|
||||
status, busy = await self._json("POST", "/prompt_manager/jobs/scan")
|
||||
self.assertEqual(status, 409)
|
||||
self.assertEqual(
|
||||
busy["running"], {"id": job_id, "title": started["job"]["title"]}
|
||||
)
|
||||
self.assertEqual((busy["success"], busy["code"]), (False, "busy"))
|
||||
|
||||
watcher = asyncio.ensure_future(
|
||||
self.client.request("GET", f"/prompt_manager/jobs/{job_id}/events")
|
||||
)
|
||||
await asyncio.sleep(0.05)
|
||||
status, cancel = await self._json(
|
||||
"POST", f"/prompt_manager/jobs/{job_id}/cancel"
|
||||
)
|
||||
self.assertEqual((status, cancel["cancellation_requested"]), (200, True))
|
||||
gate.set()
|
||||
events = _sse_payloads(await (await watcher).text())
|
||||
await asyncio.wait_for(self.api._scan_job.task, 10)
|
||||
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.assertEqual(events[0]["id"], job_id)
|
||||
self.assertEqual(events[-1]["state"], "cancelled")
|
||||
status, cancel = await self._json(
|
||||
"POST", f"/prompt_manager/jobs/{job_id}/cancel"
|
||||
)
|
||||
self.assertEqual((status, cancel["cancellation_requested"]), (200, False))
|
||||
status, legacy = await self._json("GET", "/prompt_manager/scan/status")
|
||||
self.assertEqual(legacy["last_event"]["type"], "cancelled")
|
||||
|
||||
async def test_scan_without_directories_fails_the_job(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.outside_dir)]
|
||||
sys.modules.pop("folder_paths", None)
|
||||
self.api._find_comfyui_output_dir = lambda: None
|
||||
|
||||
status, started = await self._json("POST", "/prompt_manager/jobs/scan")
|
||||
await asyncio.wait_for(self.api._scan_job.task, 10)
|
||||
status, data = await self._json(
|
||||
"GET", f"/prompt_manager/jobs/{started['job']['id']}"
|
||||
)
|
||||
self.assertEqual(data["job"]["state"], "failed")
|
||||
self.assertIn("No output directories", data["job"]["status_line"])
|
||||
|
||||
async def test_legacy_scan_refused_while_another_record_job_runs(self):
|
||||
hold = asyncio.Event()
|
||||
|
||||
async def runner(job):
|
||||
await hold.wait()
|
||||
|
||||
other = self.api._jobs.start(
|
||||
"maintenance", "Maintenance", record_changing=True, scope={}, runner=runner
|
||||
)
|
||||
resp = await self.client.request("POST", "/prompt_manager/scan", data="{}")
|
||||
self.assertEqual(resp.status, 409)
|
||||
self.assertIn("Maintenance", (await resp.json())["error"])
|
||||
hold.set()
|
||||
await asyncio.wait_for(other.task, 5)
|
||||
|
||||
|
||||
class TestScanStatusLine(unittest.TestCase):
|
||||
|
||||
def test_estimate_appears_after_two_batches(self):
|
||||
from py.api.admin import _scan_status_line
|
||||
|
||||
self.assertEqual(
|
||||
_scan_status_line(50, 200, 1, 10.0), "Processing file 50 of 200"
|
||||
)
|
||||
self.assertEqual(
|
||||
_scan_status_line(100, 1000, 2, 30.0),
|
||||
"Processing file 100 of 1,000 · about 5 min left",
|
||||
)
|
||||
self.assertEqual(
|
||||
_scan_status_line(100, 120, 2, 1.0),
|
||||
"Processing file 100 of 120 · about 1 min left",
|
||||
)
|
||||
self.assertEqual(
|
||||
_scan_status_line(120, 120, 3, 9.0), "Processing file 120 of 120"
|
||||
)
|
||||
|
||||
|
||||
class TestWorkerThreadsSetting(AdminAPITestCase):
|
||||
"""The worker-thread count is exposed, validated, persisted and used by the scan."""
|
||||
|
||||
@@ -2063,7 +2476,8 @@ def _write_mp4(path, comment):
|
||||
|
||||
|
||||
class TestScanReadsVideos(AdminAPITestCase):
|
||||
"""Videos saved by ComfyUI carry the graph in their comment tag."""
|
||||
"""Videos carry the graph in per-key container tags (current savers) or in
|
||||
the old VideoHelperSuite comment envelope; the scan reads one carrier."""
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
@@ -2093,6 +2507,56 @@ class TestScanReadsVideos(AdminAPITestCase):
|
||||
images = self.api.db.get_prompt_images(prompts[0]["id"])
|
||||
self.assertTrue(images[0]["image_path"].endswith("clip.mp4"))
|
||||
|
||||
async def test_scan_links_per_key_tag_video_with_default_profile(self):
|
||||
"""A current saver's per-key tags scan exactly like the same graph as PNG."""
|
||||
from utils import video_metadata
|
||||
|
||||
_write_png(self.output_dir / "same.png", prompt_text="a cat")
|
||||
graph = {
|
||||
"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "a cat"}},
|
||||
"2": {"class_type": "CLIPTextEncode", "inputs": {"text": "bad"}},
|
||||
"3": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {"positive": ["1", 0], "negative": ["2", 0]},
|
||||
},
|
||||
}
|
||||
(self.output_dir / "clip.mkv").write_bytes(b"\x1aE\xdf\xa3 fixture")
|
||||
probed = {
|
||||
"format": {
|
||||
"tags": {
|
||||
"PROMPT": json.dumps(graph),
|
||||
"WORKFLOW": '{"nodes":[]}',
|
||||
"COMMENT": json.dumps({"prompt": "must not win"}),
|
||||
}
|
||||
}
|
||||
}
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
|
||||
with (
|
||||
patch.object(video_metadata.shutil, "which", return_value="/bin/ffprobe"),
|
||||
patch.object(
|
||||
video_metadata.subprocess,
|
||||
"run",
|
||||
return_value=MagicMock(returncode=0, stdout=json.dumps(probed)),
|
||||
) as run,
|
||||
self.assertNoLogs(level="ERROR"),
|
||||
):
|
||||
body = await (
|
||||
await self.client.request("POST", "/prompt_manager/scan")
|
||||
).text()
|
||||
|
||||
self.assertEqual(run.call_args.args[0][-1], str(self.output_dir / "clip.mkv"))
|
||||
self.assertIn('"found": 2', body)
|
||||
prompts = self.api.db.get_recent_prompts(limit=10)["prompts"]
|
||||
self.assertEqual(sorted(p["text"] for p in prompts), ["a cat", "bad"])
|
||||
positive = next(p for p in prompts if p["text"] == "a cat")
|
||||
images = self.api.db.get_prompt_images(positive["id"])
|
||||
self.assertEqual(
|
||||
sorted(os.path.basename(image["image_path"]) for image in images),
|
||||
["clip.mkv", "same.png"],
|
||||
)
|
||||
self.assertNotIn("must not win", json.dumps(prompts))
|
||||
|
||||
async def test_scan_without_ffprobe_skips_videos_quietly(self):
|
||||
from unittest.mock import patch
|
||||
from utils import video_metadata
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
"""Physical-file prompt inspection is bounded, conservative and read-only."""
|
||||
|
||||
import io
|
||||
import json
|
||||
from unittest.mock import patch
|
||||
import shutil
|
||||
import subprocess
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from PIL import Image, PngImagePlugin
|
||||
|
||||
@@ -15,6 +19,9 @@ except ImportError:
|
||||
from utils.pair_execution import EXECUTION_KEY, record_pair_execution
|
||||
from database.pair_ingestion import ingest_resolved_pair
|
||||
from utils.prompt_pair_resolver import resolve_prompt_pair
|
||||
from utils import video_metadata
|
||||
|
||||
HAVE_FFMPEG = bool(shutil.which("ffmpeg") and shutil.which("ffprobe"))
|
||||
|
||||
|
||||
def graph(positive="portrait", negative="blurry"):
|
||||
@@ -28,6 +35,54 @@ def graph(positive="portrait", negative="blurry"):
|
||||
}
|
||||
|
||||
|
||||
def write_video(path, metadata_tags=True, **tags):
|
||||
"""Tiny black clip carrying one container tag per keyword (needs ffmpeg).
|
||||
|
||||
MP4 keeps arbitrary keys only with ``use_metadata_tags`` (mdta atoms), the
|
||||
movflag the core SaveVideo node sets; without it ffmpeg writes the legacy
|
||||
ilst atoms that old VideoHelperSuite files carry. Matroska and WebM
|
||||
upper-case the key names.
|
||||
"""
|
||||
command = [
|
||||
"ffmpeg",
|
||||
"-v",
|
||||
"quiet",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"color=c=black:s=16x16:d=0.2",
|
||||
]
|
||||
if metadata_tags and path.suffix.lower() in (".mp4", ".mov", ".m4v"):
|
||||
command += ["-movflags", "use_metadata_tags"]
|
||||
for key, value in tags.items():
|
||||
command += ["-metadata", f"{key}={value}"]
|
||||
subprocess.run(command + ["-y", str(path)], check=True, timeout=60)
|
||||
return path
|
||||
|
||||
|
||||
class ProbeOutput(io.BytesIO):
|
||||
"""ffprobe stdout that records the bounded read size asked of it."""
|
||||
|
||||
requested_size = None
|
||||
|
||||
def read(self, size=-1):
|
||||
self.requested_size = size
|
||||
return super().read(size)
|
||||
|
||||
|
||||
def probe_process(payload, poll=0, returncode=0):
|
||||
"""A fake ffprobe whose stdout is ``payload`` bytes."""
|
||||
process = MagicMock()
|
||||
process.stdout = ProbeOutput(payload)
|
||||
process.poll.return_value = poll
|
||||
process.returncode = returncode
|
||||
return process
|
||||
|
||||
|
||||
def probe_payload(tags):
|
||||
return json.dumps({"format": {"tags": tags}}).encode()
|
||||
|
||||
|
||||
def png(path, **metadata):
|
||||
info = PngImagePlugin.PngInfo()
|
||||
for key, value in metadata.items():
|
||||
@@ -312,7 +367,7 @@ class TestPromptInspector(ImageAPITestCase):
|
||||
async def test_corrupt_truncated_duplicate_and_expanding_metadata_fail_closed(self):
|
||||
import struct
|
||||
import zlib
|
||||
from utils.prompt_inspector import MAX_METADATA_BYTES
|
||||
from utils.prompt_inspector import VIDEO_REPROBE_BYTES
|
||||
|
||||
path = self.output_dir / "image.png"
|
||||
base = png(path, prompt=graph()).read_bytes()
|
||||
@@ -325,8 +380,8 @@ class TestPromptInspector(ImageAPITestCase):
|
||||
cases.append(bytes(broken))
|
||||
for kind, payload in (
|
||||
(b"tEXt", b"prompt\0{}"),
|
||||
(b"zTXt", b"prompt\0\0" + zlib.compress(b"x" * (MAX_METADATA_BYTES + 1))),
|
||||
(b"tEXt", b"prompt\0" + b"x" * (MAX_METADATA_BYTES + 1)),
|
||||
(b"zTXt", b"prompt\0\0" + zlib.compress(b"x" * (VIDEO_REPROBE_BYTES + 1))),
|
||||
(b"tEXt", b"prompt\0" + b"x" * (VIDEO_REPROBE_BYTES + 1)),
|
||||
(b"tEXt", b"bad-key-without-separator"),
|
||||
(b"zTXt", b"prompt\0\1bad-compression"),
|
||||
(b"iTXt", b"prompt\0\2\0\0\0text"),
|
||||
@@ -385,9 +440,7 @@ class TestPromptInspector(ImageAPITestCase):
|
||||
self.assertEqual(result["prompts"]["positive"]["status"], "absent")
|
||||
|
||||
async def test_video_inspection_uses_bounded_container_evidence(self):
|
||||
import io
|
||||
from unittest.mock import MagicMock
|
||||
from utils.prompt_inspector import MAX_METADATA_BYTES
|
||||
from utils.prompt_inspector import VIDEO_PROBE_BYTES, VIDEO_REPROBE_BYTES
|
||||
|
||||
(self.output_dir / "clip.mp4").write_bytes(b"video fixture")
|
||||
metadata = {"prompt": graph("Video portrait", "")}
|
||||
@@ -412,14 +465,25 @@ class TestPromptInspector(ImageAPITestCase):
|
||||
process.wait.assert_called()
|
||||
self.assertNotIn("shell", spawn.call_args.kwargs)
|
||||
self.assertIn(str(self.output_dir / "clip.mp4"), spawn.call_args.args[0])
|
||||
process.stdout = io.BytesIO(b"x" * (MAX_METADATA_BYTES + 1))
|
||||
# An envelope over the probe bound cannot be split: the role-only
|
||||
# re-probe overflows too and the read fails instead of guessing.
|
||||
processes = [
|
||||
probe_process(b"x" * (VIDEO_PROBE_BYTES + 1)),
|
||||
probe_process(b"x" * (VIDEO_REPROBE_BYTES + 1)),
|
||||
]
|
||||
with (
|
||||
patch("utils.prompt_inspector.ffprobe_path", return_value="/tool/ffprobe"),
|
||||
patch("utils.prompt_inspector.subprocess.Popen", return_value=process),
|
||||
patch(
|
||||
"utils.prompt_inspector.subprocess.Popen", side_effect=processes
|
||||
) as spawn,
|
||||
):
|
||||
_, result = await self.inspect("clip.mp4")
|
||||
self.assertEqual(spawn.call_count, 2)
|
||||
self.assertEqual(result["prompts"]["positive"]["status"], "unresolved")
|
||||
self.assertTrue(process.stdout.closed)
|
||||
self.assertEqual(result["metadata"]["status"], "unreadable")
|
||||
for process in processes:
|
||||
self.assertTrue(process.stdout.closed)
|
||||
process.wait.assert_called()
|
||||
|
||||
async def test_video_missing_probe_and_timeout_preserve_uncertainty(self):
|
||||
import io
|
||||
@@ -895,11 +959,9 @@ class TestPromptInspector(ImageAPITestCase):
|
||||
self.assertNotIn("\ufffd".encode(), body)
|
||||
self.assertLessEqual(len(body), 8 * 1024 * 1024)
|
||||
|
||||
async def video_inspect(self, comment=None, description=None):
|
||||
import io
|
||||
async def video_inspect(self, comment=None, description=None, **per_key):
|
||||
import threading
|
||||
from unittest.mock import MagicMock
|
||||
from utils.prompt_inspector import MAX_METADATA_BYTES
|
||||
from utils.prompt_inspector import VIDEO_PROBE_BYTES
|
||||
|
||||
path = self.output_dir / "clip.mp4"
|
||||
path.write_bytes(b"fixture")
|
||||
@@ -908,17 +970,8 @@ class TestPromptInspector(ImageAPITestCase):
|
||||
for key, value in (("comment", comment), ("description", description))
|
||||
if value is not None
|
||||
}
|
||||
payload = json.dumps({"format": {"tags": tags}}).encode()
|
||||
|
||||
class ProbeOutput(io.BytesIO):
|
||||
def read(self, size=-1):
|
||||
self.requested_size = size
|
||||
return super().read(size)
|
||||
|
||||
process = MagicMock()
|
||||
process.stdout = ProbeOutput(payload)
|
||||
process.poll.return_value = 0
|
||||
process.returncode = 0
|
||||
tags.update(per_key)
|
||||
process = probe_process(probe_payload(tags))
|
||||
timers = []
|
||||
timer_type = threading.Timer
|
||||
|
||||
@@ -937,11 +990,156 @@ class TestPromptInspector(ImageAPITestCase):
|
||||
response, result = await self.inspect("clip.mp4")
|
||||
self.assertEqual(spawn.call_count, 1)
|
||||
self.assertTrue(process.stdout.closed)
|
||||
self.assertEqual(process.stdout.requested_size, MAX_METADATA_BYTES + 1)
|
||||
self.assertEqual(process.stdout.requested_size, VIDEO_PROBE_BYTES + 1)
|
||||
self.assertEqual(len(timers), 1)
|
||||
self.assertTrue(timers[0].finished.is_set())
|
||||
self.assertEqual(
|
||||
spawn.call_args.args[0][spawn.call_args.args[0].index("-show_entries") + 1],
|
||||
"format_tags=prompt,workflow,parameters,"
|
||||
"comfyassets_prompt_pairs,comment,description",
|
||||
)
|
||||
return response, result
|
||||
|
||||
async def test_video_separate_prompt_workflow_tags_are_read(self):
|
||||
source = graph("per-key portrait", "per-key negative")
|
||||
workflow = '{"nodes":[{"id":"per-key"}],"cfg":6.5000000000000000001}'
|
||||
for tags in (
|
||||
{"prompt": json.dumps(source), "workflow": workflow},
|
||||
{"PROMPT": json.dumps(source), "WORKFLOW": workflow},
|
||||
):
|
||||
_, result = await self.video_inspect(
|
||||
json.dumps({"prompt": graph("comment must not win")}), **tags
|
||||
)
|
||||
self.assertEqual(
|
||||
result["prompts"]["positive"], self.side("per-key portrait")
|
||||
)
|
||||
self.assertEqual(
|
||||
result["prompts"]["negative"], self.side("per-key negative")
|
||||
)
|
||||
self.assertEqual(
|
||||
result["metadata"]["record_formats"],
|
||||
{"prompt": "original-text", "workflow": "original-text"},
|
||||
)
|
||||
self.assertEqual(result["metadata"]["raw"]["workflow"], workflow)
|
||||
self.assertEqual(
|
||||
result["metadata"]["workflow_graph"]["value"]["cfg"],
|
||||
"6.5000000000000000001",
|
||||
)
|
||||
self.assertNotIn("must not win", json.dumps(result))
|
||||
_, result = await self.video_inspect(
|
||||
**{"PROMPT": json.dumps(source), "prompt": json.dumps(graph("other"))}
|
||||
)
|
||||
self.assertEqual(result["prompts"]["positive"]["status"], "unresolved")
|
||||
self.assertEqual(result["metadata"]["status"], "unreadable")
|
||||
|
||||
@unittest.skipUnless(HAVE_FFMPEG, "ffmpeg/ffprobe not installed")
|
||||
async def test_video_per_key_and_comment_tags_read_from_real_containers(self):
|
||||
video_metadata.ffprobe_path.cache_clear()
|
||||
self.addCleanup(video_metadata.ffprobe_path.cache_clear)
|
||||
source = json.dumps(graph("real portrait", ""))
|
||||
workflow = '{"nodes":[{"id":"real"}]}'
|
||||
for name in ("per_key.mp4", "per_key.mkv", "per_key.webm"):
|
||||
write_video(self.output_dir / name, prompt=source, workflow=workflow)
|
||||
response, result = await self.inspect(name)
|
||||
self.assertEqual(response.status, 200, name)
|
||||
self.assertEqual(
|
||||
result["prompts"]["positive"], self.side("real portrait"), name
|
||||
)
|
||||
self.assertEqual(result["metadata"]["raw"]["workflow"], workflow, name)
|
||||
self.assertEqual(
|
||||
result["metadata"]["record_formats"],
|
||||
{"prompt": "original-text", "workflow": "original-text"},
|
||||
name,
|
||||
)
|
||||
# Old VideoHelperSuite envelope in the legacy ilst comment atom, unchanged.
|
||||
write_video(
|
||||
self.output_dir / "vhs.mp4",
|
||||
metadata_tags=False,
|
||||
comment=json.dumps({"prompt": json.loads(source), "workflow": {"v": 1}}),
|
||||
)
|
||||
_, result = await self.inspect("vhs.mp4")
|
||||
self.assertEqual(result["prompts"]["positive"], self.side("real portrait"))
|
||||
self.assertEqual(
|
||||
result["metadata"]["record_formats"],
|
||||
{"prompt": "reserialized-json", "workflow": "reserialized-json"},
|
||||
)
|
||||
self.assertEqual(result["metadata"]["workflow_graph"]["value"], {"v": 1})
|
||||
|
||||
async def test_video_oversize_workflow_reprobes_role_only_and_marks_limited(self):
|
||||
from utils.prompt_inspector import VIDEO_PROBE_BYTES, VIDEO_REPROBE_BYTES
|
||||
|
||||
path = self.output_dir / "clip.mp4"
|
||||
path.write_bytes(b"fixture")
|
||||
source = json.dumps(graph("bounded portrait", ""))
|
||||
processes = [
|
||||
probe_process(b"x" * (VIDEO_PROBE_BYTES + 1)),
|
||||
probe_process(probe_payload({"PROMPT": source})),
|
||||
]
|
||||
with (
|
||||
patch("utils.prompt_inspector.ffprobe_path", return_value="/tool/ffprobe"),
|
||||
patch(
|
||||
"utils.prompt_inspector.subprocess.Popen", side_effect=processes
|
||||
) as spawn,
|
||||
):
|
||||
response, result = await self.inspect("clip.mp4")
|
||||
self.assertEqual(response.status, 200)
|
||||
self.assertEqual(spawn.call_count, 2)
|
||||
first, second = (call.args[0] for call in spawn.call_args_list)
|
||||
self.assertEqual(
|
||||
first[first.index("-show_entries") + 1],
|
||||
"format_tags=prompt,workflow,parameters,"
|
||||
"comfyassets_prompt_pairs,comment,description",
|
||||
)
|
||||
self.assertEqual(
|
||||
second[second.index("-show_entries") + 1],
|
||||
"format_tags=prompt,parameters,comfyassets_prompt_pairs,"
|
||||
"comment,description",
|
||||
)
|
||||
self.assertEqual(processes[0].stdout.requested_size, VIDEO_PROBE_BYTES + 1)
|
||||
self.assertEqual(processes[1].stdout.requested_size, VIDEO_REPROBE_BYTES + 1)
|
||||
for process in processes:
|
||||
self.assertTrue(process.stdout.closed)
|
||||
process.wait.assert_called()
|
||||
self.assertEqual(result["prompts"]["positive"], self.side("bounded portrait"))
|
||||
self.assertEqual(result["metadata"]["status"], "available")
|
||||
self.assertEqual(
|
||||
result["metadata"]["record_formats"],
|
||||
{"prompt": "original-text", "workflow": "limited"},
|
||||
)
|
||||
self.assertNotIn("workflow", result["metadata"]["raw"])
|
||||
self.assertEqual(result["metadata"]["workflow_graph"]["status"], "limited")
|
||||
# The dropped per-key workflow belonged to an unselected carrier when
|
||||
# an envelope wins the re-probe: that envelope is whole, not limited.
|
||||
envelope = json.dumps({"prompt": json.loads(source), "workflow": {"v": 2}})
|
||||
processes = [
|
||||
probe_process(b"x" * (VIDEO_PROBE_BYTES + 1)),
|
||||
probe_process(probe_payload({"comment": envelope})),
|
||||
]
|
||||
with (
|
||||
patch("utils.prompt_inspector.ffprobe_path", return_value="/tool/ffprobe"),
|
||||
patch("utils.prompt_inspector.subprocess.Popen", side_effect=processes),
|
||||
):
|
||||
_, result = await self.inspect("clip.mp4")
|
||||
self.assertEqual(result["prompts"]["positive"], self.side("bounded portrait"))
|
||||
self.assertEqual(result["metadata"]["workflow_graph"]["value"], {"v": 2})
|
||||
self.assertEqual(
|
||||
result["metadata"]["record_formats"]["workflow"], "reserialized-json"
|
||||
)
|
||||
# A workflow-only per-key video over budget is limited, not absent.
|
||||
processes = [
|
||||
probe_process(b"x" * (VIDEO_PROBE_BYTES + 1)),
|
||||
probe_process(probe_payload({})),
|
||||
]
|
||||
with (
|
||||
patch("utils.prompt_inspector.ffprobe_path", return_value="/tool/ffprobe"),
|
||||
patch("utils.prompt_inspector.subprocess.Popen", side_effect=processes),
|
||||
):
|
||||
_, result = await self.inspect("clip.mp4")
|
||||
self.assertEqual(result["metadata"]["status"], "available")
|
||||
self.assertEqual(result["metadata"]["record_formats"], {"workflow": "limited"})
|
||||
self.assertEqual(result["metadata"]["workflow_graph"]["status"], "limited")
|
||||
self.assertEqual(result["prompts"]["positive"]["status"], "absent")
|
||||
|
||||
async def test_video_role_description_wins_over_workflow_only_comment(self):
|
||||
evidence = {}
|
||||
source = graph("chosen", "")
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
"""Vocabulary read routes page with signed, epoch-scoped cursors."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
@@ -184,6 +186,7 @@ class TestVocabularyAPI(ImageAPITestCase):
|
||||
"id": created["tag"]["id"],
|
||||
"name": "sunset",
|
||||
"prompt_count": 0,
|
||||
"image_count": 0,
|
||||
},
|
||||
},
|
||||
)
|
||||
@@ -202,6 +205,7 @@ class TestVocabularyAPI(ImageAPITestCase):
|
||||
"other_prompts": 0,
|
||||
"negative_variants": 1,
|
||||
"archived_negative_variants": 0,
|
||||
"images": 0,
|
||||
},
|
||||
},
|
||||
)
|
||||
@@ -221,9 +225,11 @@ class TestVocabularyAPI(ImageAPITestCase):
|
||||
"id": created["tag"]["id"],
|
||||
"name": "sunset",
|
||||
"prompt_count": 2,
|
||||
"image_count": 0,
|
||||
},
|
||||
"components_changed": 1,
|
||||
"pairs_changed": 1,
|
||||
"images_changed": 0,
|
||||
},
|
||||
)
|
||||
self.assertEqual(self.api.db.get_prompt_by_id(parent)["tags"], ["sunset"])
|
||||
@@ -242,6 +248,7 @@ class TestVocabularyAPI(ImageAPITestCase):
|
||||
"deleted": True,
|
||||
"components_changed": 1,
|
||||
"pairs_changed": 1,
|
||||
"images_changed": 0,
|
||||
},
|
||||
)
|
||||
self.assertEqual(self.api.db.get_all_tags(), [])
|
||||
@@ -253,6 +260,95 @@ class TestVocabularyAPI(ImageAPITestCase):
|
||||
self.assertEqual(response.status, 400, bad)
|
||||
self.assertNotIn("code", result)
|
||||
|
||||
def image(self, name, tags, digest="a", mtime_ns=1):
|
||||
"""Tag the bytes ``digest`` of the file ``name``; return its locator."""
|
||||
locator = os.path.normcase(str(self.output_dir / name))
|
||||
self.api.db.replace_image_tags(
|
||||
locator=locator,
|
||||
sha256=hashlib.sha256(digest.encode()).hexdigest(),
|
||||
size=1,
|
||||
mtime_ns=mtime_ns,
|
||||
expected_epoch=self.api.db.get_execution_epoch(),
|
||||
expected_tag_revision=0,
|
||||
tags=tags,
|
||||
)
|
||||
return locator
|
||||
|
||||
async def test_tag_rows_report_image_count_by_locator(self):
|
||||
self._save_prompt("Prompt only", tags=["prompt-only", "shared"])
|
||||
self.image("one.png", ["twice", "shared"], digest="old")
|
||||
self.image("one.png", ["twice"], digest="new", mtime_ns=2)
|
||||
self.image("two.png", ["shared"], digest="new")
|
||||
self.image("three.png", ["prior-only"], digest="first")
|
||||
self.image("three.png", [], digest="second", mtime_ns=2)
|
||||
response, page = await self.get("tags")
|
||||
self.assertEqual(response.status, 200, page)
|
||||
rows = {row["name"]: row for row in page["tags"]}
|
||||
self.assertEqual(
|
||||
{
|
||||
name: (row["prompt_count"], row["image_count"])
|
||||
for name, row in rows.items()
|
||||
},
|
||||
{
|
||||
"prompt-only": (1, 0),
|
||||
"shared": (1, 2),
|
||||
"twice": (0, 1),
|
||||
"prior-only": (0, 1),
|
||||
},
|
||||
)
|
||||
row = rows["shared"]
|
||||
self.assertEqual(row["image_count"], 2)
|
||||
self.assertEqual(set(row), {"id", "name", "prompt_count", "image_count"})
|
||||
|
||||
async def test_impact_and_delete_counts_include_images(self):
|
||||
epoch = self.api.db.get_execution_epoch()
|
||||
self._save_prompt("Parent", tags=["moody"])
|
||||
self.image("a.png", ["moody", "keep"])
|
||||
self.image("b.png", ["moody"])
|
||||
_, page = await self.get("tags")
|
||||
moody = next(row for row in page["tags"] if row["name"] == "moody")
|
||||
self.assertEqual(moody["image_count"], 2)
|
||||
response, impact = await self.get(f"tags/{moody['id']}/impact")
|
||||
self.assertEqual(response.status, 200, impact)
|
||||
self.assertEqual(
|
||||
impact["impact"],
|
||||
{
|
||||
"positive_prompts": 1,
|
||||
"other_prompts": 0,
|
||||
"negative_variants": 0,
|
||||
"archived_negative_variants": 0,
|
||||
"images": 2,
|
||||
},
|
||||
)
|
||||
response, deleted = await self.send(
|
||||
"POST",
|
||||
f"tags/{moody['id']}/delete",
|
||||
{"expected_epoch": epoch, "name": "moody"},
|
||||
)
|
||||
self.assertEqual(response.status, 200, deleted)
|
||||
self.assertEqual(
|
||||
deleted,
|
||||
{
|
||||
"success": True,
|
||||
"library_epoch": epoch,
|
||||
"deleted": True,
|
||||
"components_changed": 1,
|
||||
"pairs_changed": 0,
|
||||
"images_changed": 2,
|
||||
},
|
||||
)
|
||||
keep = next(
|
||||
row for row in (await self.get("tags"))[1]["tags"] if row["name"] == "keep"
|
||||
)
|
||||
response, renamed = await self.send(
|
||||
"PATCH",
|
||||
f"tags/{keep['id']}",
|
||||
{"expected_epoch": epoch, "name": "keep", "new_name": "kept"},
|
||||
)
|
||||
self.assertEqual(response.status, 200, renamed)
|
||||
self.assertEqual(renamed["images_changed"], 1)
|
||||
self.assertEqual(renamed["tag"]["image_count"], 1)
|
||||
|
||||
async def test_create_route_validates_names(self):
|
||||
epoch = self.api.db.get_execution_epoch()
|
||||
before = self.tag_rows()
|
||||
|
||||
@@ -190,6 +190,26 @@ class TestWorkspaceImages(ImageAPITestCase):
|
||||
self.assertEqual(data["images"], [])
|
||||
self.assertIsNone(data["next_cursor"])
|
||||
|
||||
async def test_path_filter_lists_only_that_exact_file(self):
|
||||
# Activity (D57): a scan row opens its file by root and exact relative path,
|
||||
# never by a substring that also matches the same name in other folders.
|
||||
self.media("ComfyUI_00001_.png")
|
||||
for day in range(3):
|
||||
self.media(f"2026-10-0{day + 1}/ComfyUI_00001_.png")
|
||||
self.media("2026-10-01/ComfyUI_00001_.png.bak.png")
|
||||
_, data = await self.get(path="ComfyUI_00001_.png")
|
||||
self.assertEqual([i["path"] for i in data["images"]], ["ComfyUI_00001_.png"])
|
||||
_, data = await self.get(path="2026-10-02/ComfyUI_00001_.png")
|
||||
self.assertEqual(
|
||||
[i["path"] for i in data["images"]], ["2026-10-02/ComfyUI_00001_.png"]
|
||||
)
|
||||
_, data = await self.get(path="missing.png")
|
||||
self.assertEqual(data["images"], [])
|
||||
for bad in ("../outside.png", "a" * 1025):
|
||||
with self.subTest(bad=bad[:12]):
|
||||
response, _ = await self.get(path=bad)
|
||||
self.assertEqual(response.status, 400)
|
||||
|
||||
async def test_cursor_cannot_be_reused_with_changed_filters_or_tampered(self):
|
||||
self.media("a.png")
|
||||
self.media("b.png")
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
"""The page-workflow test server ends with its launcher (#274 review M2).
|
||||
|
||||
A Playwright worker that is SIGKILLed never runs its fixture teardown, so the
|
||||
server it started must notice on its own: with ``--exit-on-stdin-eof`` it shuts
|
||||
down when the launcher's end of its stdin pipe closes, and still removes its
|
||||
temp root. This runs on every OS in CI, where Windows is the strict case.
|
||||
"""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
SERVER_TIMEOUT_S = 60
|
||||
|
||||
|
||||
class TestBrowserServerLifetime(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
|
||||
def start(self, *extra):
|
||||
env = {**os.environ, "TMPDIR": self.tmp.name}
|
||||
env.update(TEMP=self.tmp.name, TMP=self.tmp.name)
|
||||
process = subprocess.Popen(
|
||||
[sys.executable, "-m", "tests.browser.server", "--port", "0", *extra],
|
||||
cwd=REPO_ROOT,
|
||||
env=env,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
|
||||
def stop():
|
||||
if process.poll() is None:
|
||||
process.kill()
|
||||
process.wait(timeout=SERVER_TIMEOUT_S)
|
||||
# communicate() would flush a closed stdin, which Python < 3.13 refuses.
|
||||
for stream in (process.stdin, process.stdout, process.stderr):
|
||||
stream.close()
|
||||
|
||||
self.addCleanup(stop)
|
||||
line = process.stdout.readline().decode()
|
||||
self.assertTrue(line.startswith("PM_TEST_SERVER_PORT="), line)
|
||||
url = f"http://127.0.0.1:{int(line.split('=', 1)[1])}/prompt_manager/test"
|
||||
deadline = time.monotonic() + SERVER_TIMEOUT_S
|
||||
while True:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=1) as response: # nosec B310
|
||||
self.assertEqual(response.status, 200)
|
||||
return process
|
||||
except OSError:
|
||||
self.assertIsNone(process.poll(), "the server exited early")
|
||||
self.assertLess(time.monotonic(), deadline, "never ready")
|
||||
time.sleep(0.1)
|
||||
|
||||
def roots(self):
|
||||
return sorted(path.name for path in Path(self.tmp.name).glob("pm-browser-*"))
|
||||
|
||||
def test_server_exits_and_removes_its_temp_root_when_stdin_closes(self):
|
||||
process = self.start("--exit-on-stdin-eof")
|
||||
self.assertEqual(len(self.roots()), 1, "the server works in its own temp root")
|
||||
process.stdin.close()
|
||||
code = process.wait(timeout=SERVER_TIMEOUT_S)
|
||||
self.assertEqual(
|
||||
code, 0, process.stderr.read().decode(errors="replace")[-4000:]
|
||||
)
|
||||
self.assertEqual(self.roots(), [], "the temp root and its database are removed")
|
||||
|
||||
def test_server_keeps_running_with_stdin_closed_without_the_flag(self):
|
||||
process = self.start()
|
||||
process.stdin.close()
|
||||
with self.assertRaises(subprocess.TimeoutExpired):
|
||||
process.wait(timeout=1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -452,6 +452,44 @@ class TestRestoreFromFile(BackupTestCase):
|
||||
).fetchone()
|
||||
)
|
||||
|
||||
def test_restore_old_backup_without_annotations(self):
|
||||
tables = (
|
||||
"media_image_tags",
|
||||
"media_annotation_bytes",
|
||||
"media_annotation_files",
|
||||
"media_annotation_metadata",
|
||||
)
|
||||
src = self._make_source_db(["predates image annotations"])
|
||||
with closing(sqlite3.connect(src)) as conn:
|
||||
conn.execute("PRAGMA foreign_keys=OFF")
|
||||
for table in tables:
|
||||
conn.execute(f"DROP TABLE {table}")
|
||||
conn.commit()
|
||||
self.assertTrue(PromptModel.verify_database_file(src)[0])
|
||||
ok, reason = PromptModel.verify_database_file(src, require_library_state=True)
|
||||
self.assertFalse(ok)
|
||||
self.assertEqual(reason, "Missing image annotation schema")
|
||||
|
||||
self.db.model.restore_from_file(src)
|
||||
|
||||
self.assertEqual(_texts(self.path), ["predates image annotations"])
|
||||
with closing(sqlite3.connect(self.path)) as conn:
|
||||
self.assertEqual(
|
||||
conn.execute("SELECT * FROM media_annotation_metadata").fetchall(),
|
||||
[(1, 1, 0)],
|
||||
)
|
||||
ok, reason = PromptModel.verify_database_file(
|
||||
self.path, require_library_state=True
|
||||
)
|
||||
self.assertTrue(ok, reason)
|
||||
with closing(sqlite3.connect(src)) as conn: # the source is untouched
|
||||
self.assertEqual(
|
||||
conn.execute(
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE name LIKE 'media_%'"
|
||||
).fetchone()[0],
|
||||
0,
|
||||
)
|
||||
|
||||
def test_restore_preserves_favorites_and_closes_handles(self):
|
||||
self.db.set_image_favorite("output", "live.png")
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
|
||||
@@ -200,6 +200,104 @@ class TestIdempotentMigrations(MigrationTestCase):
|
||||
|
||||
self.assertEqual(self._columns("prompt_library_state"), ["prompt_id", "title"])
|
||||
|
||||
def test_legacy_database_gains_annotation_group(self):
|
||||
self._legacy_db([])
|
||||
|
||||
self._open()
|
||||
|
||||
for table in (
|
||||
"media_annotation_metadata",
|
||||
"media_annotation_files",
|
||||
"media_annotation_bytes",
|
||||
"media_image_tags",
|
||||
):
|
||||
self.assertIn(table, self._table_names())
|
||||
conn = sqlite3.connect(self.path)
|
||||
try:
|
||||
self.assertEqual(
|
||||
conn.execute("SELECT * FROM media_annotation_metadata").fetchall(),
|
||||
[(1, 1, 0)],
|
||||
)
|
||||
# No backfill: a legacy database starts with no image tags.
|
||||
self.assertEqual(
|
||||
conn.execute("SELECT COUNT(*) FROM media_image_tags").fetchone()[0], 0
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def test_rebuild_keeps_annotation_rows(self):
|
||||
# The prompts rebuild runs with foreign keys off; image tags that
|
||||
# reference the tags table must survive it untouched.
|
||||
file_id = "0f8e4b2a-1c3d-4e5f-8a9b-0c1d2e3f4a5b"
|
||||
byte_id = "6a1b2c3d-4e5f-4a6b-8c9d-0e1f2a3b4c5d"
|
||||
self._legacy_db(
|
||||
[
|
||||
"INSERT INTO prompts (text, hash, workflow_name)"
|
||||
" VALUES ('a', 'h1', 'wf')",
|
||||
"CREATE TABLE tags (id INTEGER PRIMARY KEY AUTOINCREMENT,"
|
||||
" name TEXT NOT NULL UNIQUE)",
|
||||
"INSERT INTO tags (id, name) VALUES (3, 'img-only')",
|
||||
"CREATE TABLE media_annotation_metadata (singleton_id INTEGER"
|
||||
" PRIMARY KEY CHECK(singleton_id=1), schema_version INTEGER NOT"
|
||||
" NULL CHECK(schema_version=1), image_index_revision INTEGER NOT"
|
||||
" NULL DEFAULT 0 CHECK(image_index_revision>=0))",
|
||||
"INSERT INTO media_annotation_metadata VALUES (1, 1, 4)",
|
||||
"CREATE TABLE media_annotation_files (id TEXT PRIMARY KEY"
|
||||
" CHECK(length(id)=36), locator TEXT NOT NULL UNIQUE,"
|
||||
" created_at TEXT NOT NULL)",
|
||||
f"INSERT INTO media_annotation_files VALUES ('{file_id}',"
|
||||
" '/out/a.png', '2026-10-22T00:00:00.000000+00:00')",
|
||||
"CREATE TABLE media_annotation_bytes (id TEXT PRIMARY KEY"
|
||||
" CHECK(length(id)=36), file_id TEXT NOT NULL REFERENCES"
|
||||
" media_annotation_files(id) ON DELETE CASCADE, sha256 TEXT NOT"
|
||||
" NULL CHECK(length(sha256)=64 AND sha256 NOT GLOB"
|
||||
" '*[^0-9a-f]*'), size INTEGER NOT NULL CHECK(size>=0), mtime_ns"
|
||||
" INTEGER NOT NULL, verified_at TEXT NOT NULL, first_seen_at TEXT"
|
||||
" NOT NULL, tag_revision INTEGER NOT NULL DEFAULT 0"
|
||||
" CHECK(tag_revision>=0), UNIQUE(file_id, sha256))",
|
||||
"CREATE INDEX idx_media_annotation_bytes_fingerprint ON"
|
||||
" media_annotation_bytes(file_id, size, mtime_ns)",
|
||||
f"INSERT INTO media_annotation_bytes VALUES ('{byte_id}',"
|
||||
f" '{file_id}', '{'a' * 64}', 3, 1700000000000000000,"
|
||||
" '2026-10-22T00:00:00.000000+00:00',"
|
||||
" '2026-10-22T00:00:00.000000+00:00', 2)",
|
||||
"CREATE TABLE media_image_tags (byte_id TEXT NOT NULL REFERENCES"
|
||||
" media_annotation_bytes(id) ON DELETE CASCADE, tag_id INTEGER"
|
||||
" NOT NULL REFERENCES tags(id) ON DELETE CASCADE, position"
|
||||
" INTEGER NOT NULL CHECK(position>=0), PRIMARY KEY(byte_id,"
|
||||
" tag_id))",
|
||||
"CREATE INDEX idx_media_image_tags_tag ON media_image_tags(tag_id)",
|
||||
"CREATE UNIQUE INDEX idx_media_image_tags_position ON"
|
||||
" media_image_tags(byte_id, position)",
|
||||
f"INSERT INTO media_image_tags VALUES ('{byte_id}', 3, 0)",
|
||||
]
|
||||
)
|
||||
|
||||
self._open()
|
||||
|
||||
self.assertNotIn("workflow_name", self._columns("prompts"))
|
||||
conn = sqlite3.connect(self.path)
|
||||
try:
|
||||
self.assertEqual(
|
||||
conn.execute("SELECT * FROM media_annotation_metadata").fetchall(),
|
||||
[(1, 1, 4)],
|
||||
)
|
||||
self.assertEqual(
|
||||
conn.execute(
|
||||
"SELECT t.name, it.position FROM media_image_tags it"
|
||||
" JOIN tags t ON t.id = it.tag_id"
|
||||
).fetchall(),
|
||||
[("img-only", 0)],
|
||||
)
|
||||
self.assertEqual(
|
||||
conn.execute(
|
||||
"SELECT tag_revision FROM media_annotation_bytes"
|
||||
).fetchall(),
|
||||
[(2,)],
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def test_migration_error_is_logged_and_raised(self):
|
||||
# The legacy table has no CHECK on rating; the rebuilt one does, so the
|
||||
# copy into prompts_new fails inside the migration.
|
||||
|
||||
@@ -823,6 +823,88 @@ def _png_with_chunks(path, chunks):
|
||||
return path
|
||||
|
||||
|
||||
class TestVideoProbeBoundsMatchPngBudgets(unittest.TestCase):
|
||||
"""ffprobe escapes tag text; the stdout bounds must never gate a record
|
||||
that the identical PNG reads within its decoded budget (I3)."""
|
||||
|
||||
@staticmethod
|
||||
def _dense(size):
|
||||
# Quote-dense JSON text: every other byte escapes, so ffprobe's JSON
|
||||
# output is close to twice the decoded size.
|
||||
unit = '{"k":"v",' + "".join('"' for _ in range(6)) + "}"
|
||||
return (unit * (size // len(unit) + 1))[:size]
|
||||
|
||||
def _video_read(self, path, tags, bound):
|
||||
payload = json.dumps({"format": {"tags": tags}}).encode()
|
||||
self.assertGreater(len(payload), bound)
|
||||
process = MagicMock()
|
||||
process.stdout = io.BytesIO(payload)
|
||||
process.poll.return_value = 0
|
||||
process.returncode = 0
|
||||
with (
|
||||
patch("utils.prompt_inspector.ffprobe_path", return_value="/tool/ffprobe"),
|
||||
patch(
|
||||
"utils.prompt_inspector.subprocess.Popen", return_value=process
|
||||
) as spawn,
|
||||
):
|
||||
read = read_inspection_file(path)
|
||||
self.assertEqual(spawn.call_count, 1)
|
||||
self.assertTrue(process.stdout.closed)
|
||||
return read
|
||||
|
||||
def test_near_budget_records_read_the_same_as_png(self):
|
||||
from utils.prompt_inspector import (
|
||||
ROLE_METADATA_BYTES,
|
||||
VIDEO_PROBE_BYTES,
|
||||
VIDEO_REPROBE_BYTES,
|
||||
WORKFLOW_METADATA_BYTES,
|
||||
)
|
||||
|
||||
prompt = self._dense(ROLE_METADATA_BYTES - 1024)
|
||||
workflow = self._dense(WORKFLOW_METADATA_BYTES - 1024)
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
png = _png_with_chunks(
|
||||
Path(directory) / "same.png",
|
||||
[
|
||||
_text_chunk(b"tEXt", "prompt", prompt.encode("latin-1")),
|
||||
_text_chunk(b"tEXt", "workflow", workflow.encode("latin-1")),
|
||||
],
|
||||
)
|
||||
expected = read_inspection_file(png)
|
||||
self.assertEqual(
|
||||
expected.record_formats,
|
||||
{"prompt": "original-text", "workflow": "original-text"},
|
||||
)
|
||||
video = Path(directory) / "same.mp4"
|
||||
video.write_bytes(b"fixture")
|
||||
# Escaped output is over the old 6 MiB + 64 KiB stdout bound.
|
||||
read = self._video_read(
|
||||
video,
|
||||
{"prompt": prompt, "workflow": workflow},
|
||||
ROLE_METADATA_BYTES + WORKFLOW_METADATA_BYTES + 64 * 1024,
|
||||
)
|
||||
self.assertEqual(read.metadata, expected.metadata)
|
||||
self.assertEqual(read.record_formats, expected.record_formats)
|
||||
self.assertEqual(read.limited_records, frozenset())
|
||||
self.assertGreaterEqual(
|
||||
VIDEO_PROBE_BYTES,
|
||||
2 * (ROLE_METADATA_BYTES + WORKFLOW_METADATA_BYTES) + 64 * 1024,
|
||||
)
|
||||
self.assertGreaterEqual(
|
||||
VIDEO_REPROBE_BYTES, 2 * ROLE_METADATA_BYTES + 64 * 1024
|
||||
)
|
||||
# A role record near its budget alone escapes past 2 MiB and must
|
||||
# still read as the PNG does.
|
||||
png = _png_with_chunks(
|
||||
Path(directory) / "role.png",
|
||||
[_text_chunk(b"tEXt", "prompt", prompt.encode("latin-1"))],
|
||||
)
|
||||
expected = read_inspection_file(png)
|
||||
read = self._video_read(video, {"prompt": prompt}, ROLE_METADATA_BYTES)
|
||||
self.assertEqual(read.metadata, expected.metadata)
|
||||
self.assertEqual(read.record_formats, {"prompt": "original-text"})
|
||||
|
||||
|
||||
class TestPerRecordBudgets(unittest.TestCase):
|
||||
"""Role records keep the 2 MiB budget; a workflow has its own 4 MiB budget."""
|
||||
|
||||
|
||||
@@ -0,0 +1,436 @@
|
||||
"""Unit tests for py/api/jobs.py: the in-memory session job registry.
|
||||
|
||||
Fake runners only: no files, no database. Ordering is driven by events and
|
||||
awaited tasks, never by wall-clock bounds.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from py.api.jobs import ( # noqa: E402
|
||||
DEFAULT_SCAN_SCOPE,
|
||||
LOG_CAP_LINE,
|
||||
MAX_FINISHED_JOBS,
|
||||
MAX_LOG_LINES,
|
||||
MAX_OUTCOME_ROWS,
|
||||
SCAN_GROUP_NAMES,
|
||||
SCAN_GROUPS,
|
||||
SCAN_SCOPES,
|
||||
Job,
|
||||
JobBusy,
|
||||
JobRegistry,
|
||||
)
|
||||
|
||||
|
||||
async def _collect(job, stop_after=None):
|
||||
seen = []
|
||||
async with contextlib.aclosing(job.events()) as events:
|
||||
async for payload in events:
|
||||
seen.append(payload)
|
||||
if stop_after is not None and len(seen) >= stop_after:
|
||||
break
|
||||
return seen
|
||||
|
||||
|
||||
def _gated_runner(gate, publish=None):
|
||||
"""A runner that publishes ``publish`` (if any) and waits for ``gate``."""
|
||||
|
||||
async def runner(job):
|
||||
if publish is not None:
|
||||
job.publish(publish)
|
||||
await gate.wait()
|
||||
|
||||
return runner
|
||||
|
||||
|
||||
async def _noop(job):
|
||||
return None
|
||||
|
||||
|
||||
class TestJobFanOut(unittest.IsolatedAsyncioTestCase):
|
||||
"""The SSE fan-out the scan job always had, now on every Job."""
|
||||
|
||||
async def test_new_job_is_running_with_no_event(self):
|
||||
job = Job("scan", "Scan")
|
||||
self.assertTrue(job.running)
|
||||
self.assertIsNone(job.last_event)
|
||||
self.assertEqual(job.state, "running")
|
||||
self.assertEqual(len(job.id), 32)
|
||||
|
||||
async def test_subscriber_receives_events_until_finish(self):
|
||||
job = Job("scan", "Scan")
|
||||
consumer = asyncio.ensure_future(_collect(job))
|
||||
await asyncio.sleep(0)
|
||||
|
||||
job.publish({"type": "progress", "progress": 10})
|
||||
job.publish({"type": "complete"})
|
||||
job.finish("finished")
|
||||
|
||||
self.assertEqual(
|
||||
await asyncio.wait_for(consumer, 2),
|
||||
[{"type": "progress", "progress": 10}, {"type": "complete"}],
|
||||
)
|
||||
self.assertFalse(job.running)
|
||||
|
||||
async def test_late_subscriber_gets_latest_snapshot_then_live_events(self):
|
||||
job = Job("scan", "Scan")
|
||||
job.publish({"type": "progress", "progress": 10})
|
||||
job.publish({"type": "progress", "progress": 40})
|
||||
|
||||
consumer = asyncio.ensure_future(_collect(job))
|
||||
await asyncio.sleep(0)
|
||||
job.publish({"type": "complete"})
|
||||
job.finish()
|
||||
|
||||
self.assertEqual(
|
||||
await asyncio.wait_for(consumer, 2),
|
||||
[{"type": "progress", "progress": 40}, {"type": "complete"}],
|
||||
)
|
||||
|
||||
async def test_subscriber_after_finish_gets_only_the_final_event(self):
|
||||
job = Job("scan", "Scan")
|
||||
job.publish({"type": "progress", "progress": 99})
|
||||
job.publish({"type": "complete"})
|
||||
job.finish()
|
||||
|
||||
self.assertEqual(await _collect(job), [{"type": "complete"}])
|
||||
|
||||
async def test_departed_subscriber_is_forgotten(self):
|
||||
job = Job("scan", "Scan")
|
||||
job.publish({"type": "progress", "progress": 1})
|
||||
await _collect(job, stop_after=1)
|
||||
|
||||
self.assertEqual(job.subscriber_count, 0)
|
||||
job.publish({"type": "complete"}) # must not raise or queue anywhere
|
||||
job.finish()
|
||||
|
||||
async def test_second_subscriber_attaches_to_running_job(self):
|
||||
registry = JobRegistry()
|
||||
gate = asyncio.Event()
|
||||
job = registry.start(
|
||||
"scan",
|
||||
"Scan",
|
||||
record_changing=True,
|
||||
scope={},
|
||||
runner=_gated_runner(gate, {"type": "progress", "progress": 30}),
|
||||
)
|
||||
await asyncio.sleep(0) # the runner publishes its first event
|
||||
first = asyncio.ensure_future(_collect(job))
|
||||
second = asyncio.ensure_future(_collect(job))
|
||||
await asyncio.sleep(0)
|
||||
self.assertEqual(job.subscriber_count, 2)
|
||||
self.assertIs(registry.active("scan"), job)
|
||||
|
||||
job.publish({"type": "progress", "progress": 60})
|
||||
gate.set()
|
||||
await asyncio.wait_for(job.task, 2)
|
||||
|
||||
expected = [
|
||||
{"type": "progress", "progress": 30},
|
||||
{"type": "progress", "progress": 60},
|
||||
]
|
||||
self.assertEqual(await asyncio.wait_for(first, 2), expected)
|
||||
self.assertEqual(await asyncio.wait_for(second, 2), expected)
|
||||
self.assertEqual(job.summary()["progress"], 60)
|
||||
|
||||
async def test_finish_rejects_unknown_state_and_is_final(self):
|
||||
job = Job("scan", "Scan")
|
||||
with self.assertRaises(ValueError):
|
||||
job.finish("running")
|
||||
job.finish("cancelled")
|
||||
job.finish("finished") # a second finish keeps the first state
|
||||
self.assertEqual(job.state, "cancelled")
|
||||
self.assertIsNotNone(job.finished_at)
|
||||
|
||||
|
||||
class TestJobRegistry(unittest.IsolatedAsyncioTestCase):
|
||||
|
||||
async def test_start_lists_and_finishes_a_job(self):
|
||||
registry = JobRegistry()
|
||||
gate = asyncio.Event()
|
||||
job = registry.start(
|
||||
"scan",
|
||||
"Scan for metadata",
|
||||
record_changing=True,
|
||||
scope={"files": "new", "output_folders": 2},
|
||||
runner=_gated_runner(gate),
|
||||
)
|
||||
self.assertEqual(job.state, "running")
|
||||
self.assertIs(registry.get(job.id), job)
|
||||
self.assertIsNone(registry.get("missing"))
|
||||
summary = job.summary()
|
||||
self.assertEqual(
|
||||
set(summary),
|
||||
{
|
||||
"id",
|
||||
"kind",
|
||||
"title",
|
||||
"state",
|
||||
"started_at",
|
||||
"finished_at",
|
||||
"progress",
|
||||
"status_line",
|
||||
"processed",
|
||||
"total",
|
||||
"counters",
|
||||
"scope",
|
||||
"listed",
|
||||
"log_length",
|
||||
},
|
||||
)
|
||||
self.assertEqual(summary["scope"], {"files": "new", "output_folders": 2})
|
||||
# What the detail would list, so a view reads it only when that changed.
|
||||
self.assertEqual((summary["listed"], summary["log_length"]), ({}, 0))
|
||||
self.assertIsNone(summary["finished_at"])
|
||||
|
||||
gate.set()
|
||||
await asyncio.wait_for(job.task, 2)
|
||||
self.assertEqual(job.state, "finished")
|
||||
self.assertIsNone(registry.active("scan"))
|
||||
|
||||
# Running jobs list first, then finished ones newest first.
|
||||
older = registry.start(
|
||||
"thumbs", "Thumbnails", record_changing=False, scope={}, runner=_noop
|
||||
)
|
||||
await asyncio.wait_for(older.task, 2)
|
||||
hold = asyncio.Event()
|
||||
running = registry.start(
|
||||
"other",
|
||||
"Other",
|
||||
record_changing=False,
|
||||
scope={},
|
||||
runner=_gated_runner(hold),
|
||||
)
|
||||
listed = registry.list()
|
||||
self.assertEqual([j.id for j in listed], [running.id, older.id, job.id])
|
||||
|
||||
# Only the last MAX_FINISHED_JOBS finished jobs are kept.
|
||||
for index in range(MAX_FINISHED_JOBS + 5):
|
||||
done = registry.start(
|
||||
f"k{index}", "Fake", record_changing=False, scope={}, runner=_noop
|
||||
)
|
||||
await asyncio.wait_for(done.task, 2)
|
||||
listed = registry.list()
|
||||
finished = [j for j in listed if not j.running]
|
||||
self.assertEqual(len(finished), MAX_FINISHED_JOBS)
|
||||
self.assertEqual(finished[0].id, done.id)
|
||||
self.assertIsNone(registry.get(job.id))
|
||||
self.assertIs(listed[0], running)
|
||||
hold.set()
|
||||
await asyncio.wait_for(running.task, 2)
|
||||
|
||||
async def test_record_changing_jobs_are_exclusive(self):
|
||||
registry = JobRegistry()
|
||||
gate = asyncio.Event()
|
||||
first = registry.start(
|
||||
"scan",
|
||||
"Scan for metadata",
|
||||
record_changing=True,
|
||||
scope={},
|
||||
runner=_gated_runner(gate),
|
||||
)
|
||||
with self.assertRaises(JobBusy) as raised:
|
||||
registry.start(
|
||||
"maintenance",
|
||||
"Maintenance",
|
||||
record_changing=True,
|
||||
scope={},
|
||||
runner=_noop,
|
||||
)
|
||||
self.assertIs(raised.exception.running, first)
|
||||
self.assertIn("Scan for metadata", str(raised.exception))
|
||||
|
||||
reader = registry.start(
|
||||
"report", "Report", record_changing=False, scope={}, runner=_noop
|
||||
)
|
||||
await asyncio.wait_for(reader.task, 2)
|
||||
self.assertEqual(reader.state, "finished")
|
||||
|
||||
# One job per kind, even a read-only one.
|
||||
hold = asyncio.Event()
|
||||
registry.start(
|
||||
"report",
|
||||
"Report",
|
||||
record_changing=False,
|
||||
scope={},
|
||||
runner=_gated_runner(hold),
|
||||
)
|
||||
with self.assertRaises(JobBusy):
|
||||
registry.start(
|
||||
"report", "Report", record_changing=False, scope={}, runner=_noop
|
||||
)
|
||||
gate.set()
|
||||
hold.set()
|
||||
await asyncio.wait_for(first.task, 2)
|
||||
|
||||
async def test_cancel_is_idempotent_and_keeps_partial_counts(self):
|
||||
registry = JobRegistry()
|
||||
reached = asyncio.Event()
|
||||
proceed = asyncio.Event()
|
||||
|
||||
async def runner(job):
|
||||
for index in range(3):
|
||||
if job.cancelled:
|
||||
return
|
||||
job.record("added", {"path": f"a{index}.png", "root": "output"}, "+")
|
||||
if index == 0:
|
||||
reached.set()
|
||||
await proceed.wait()
|
||||
|
||||
job = registry.start(
|
||||
"scan",
|
||||
"Scan",
|
||||
record_changing=True,
|
||||
scope={},
|
||||
runner=runner,
|
||||
groups=SCAN_GROUP_NAMES,
|
||||
)
|
||||
await asyncio.wait_for(reached.wait(), 2)
|
||||
self.assertTrue(registry.cancel(job.id))
|
||||
self.assertTrue(registry.cancel(job.id)) # still running: still requested
|
||||
proceed.set()
|
||||
await asyncio.wait_for(job.task, 2)
|
||||
|
||||
self.assertEqual(job.state, "cancelled")
|
||||
self.assertEqual(job.counters["added"], 1) # the work done before the stop
|
||||
self.assertFalse(registry.cancel(job.id)) # finished: a no-op
|
||||
self.assertFalse(registry.cancel("missing"))
|
||||
self.assertEqual(job.state, "cancelled")
|
||||
|
||||
async def test_failing_runner_ends_failed_and_frees_the_slot(self):
|
||||
registry = JobRegistry()
|
||||
|
||||
async def runner(job):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
job = registry.start(
|
||||
"scan", "Scan", record_changing=True, scope={}, runner=runner
|
||||
)
|
||||
with self.assertLogs("py.api.jobs", level="ERROR"):
|
||||
await asyncio.wait_for(job.task, 2)
|
||||
self.assertEqual(job.state, "failed")
|
||||
self.assertIsNone(registry.active("scan"))
|
||||
|
||||
async def test_runner_that_finishes_its_own_job_keeps_its_state(self):
|
||||
registry = JobRegistry()
|
||||
|
||||
async def runner(job):
|
||||
job.finish("failed")
|
||||
|
||||
job = registry.start(
|
||||
"scan", "Scan", record_changing=True, scope={}, runner=runner
|
||||
)
|
||||
await asyncio.wait_for(job.task, 2)
|
||||
self.assertEqual(job.state, "failed")
|
||||
|
||||
|
||||
class TestJobOutcomesAndLog(unittest.TestCase):
|
||||
|
||||
def test_outcome_and_log_caps(self):
|
||||
job = Job("scan", "Scan", groups=SCAN_GROUP_NAMES)
|
||||
for index in range(MAX_OUTCOME_ROWS + 50):
|
||||
job.record("errors", {"path": f"f{index}.png", "root": "output"}, "!")
|
||||
self.assertEqual(len(job.outcomes["errors"]), MAX_OUTCOME_ROWS)
|
||||
self.assertEqual(job.counters["errors"], MAX_OUTCOME_ROWS + 50)
|
||||
self.assertEqual(job.outcomes["errors"][0]["path"], "f0.png")
|
||||
|
||||
job = Job("scan", "Scan")
|
||||
for index in range(MAX_LOG_LINES + 100):
|
||||
job.log_line(f"line {index}")
|
||||
self.assertEqual(len(job.log), MAX_LOG_LINES + 1)
|
||||
self.assertEqual(job.log[-1], LOG_CAP_LINE)
|
||||
self.assertTrue(job.log[0].endswith("line 0"))
|
||||
self.assertEqual(LOG_CAP_LINE, "[Log capped at 2,000 lines]")
|
||||
|
||||
def test_paths_are_root_relative(self):
|
||||
job = Job("scan", "Scan", groups=SCAN_GROUP_NAMES)
|
||||
absolute = os.path.abspath(os.path.join("outputs", "sub", "a.png"))
|
||||
job.record(
|
||||
"added",
|
||||
{"path": absolute, "root": "output", "detail": "x"},
|
||||
f"+ {absolute}",
|
||||
)
|
||||
job.record("known", {"path": "sub/b.png", "root": "output"}, "= sub/b.png")
|
||||
job.record("review", {"path": "C:\\out\\c.png", "root": "output"}, "?")
|
||||
rows = [row for group in job.outcomes.values() for row in group]
|
||||
for row in rows:
|
||||
self.assertFalse(os.path.isabs(row["path"]), row)
|
||||
self.assertNotIn(":", row["path"])
|
||||
self.assertEqual(job.outcomes["added"][0]["path"], "a.png")
|
||||
self.assertEqual(job.outcomes["known"][0]["path"], "sub/b.png")
|
||||
self.assertEqual(job.outcomes["review"][0]["path"], "c.png")
|
||||
self.assertEqual(job.outcomes["known"][0]["detail"], "")
|
||||
self.assertNotIn(absolute, "\n".join(job.log))
|
||||
|
||||
def test_record_rejects_unknown_group(self):
|
||||
job = Job("scan", "Scan", groups=SCAN_GROUP_NAMES)
|
||||
with self.assertRaises(ValueError):
|
||||
job.record("bogus", {"path": "a.png", "root": "output"}, "x")
|
||||
|
||||
def test_detail_and_progress(self):
|
||||
job = Job(
|
||||
"scan",
|
||||
"Scan",
|
||||
scope={"files": "new", "output_folders": 2},
|
||||
groups=SCAN_GROUP_NAMES,
|
||||
)
|
||||
job.update(progress=150, status_line="Processing file 3 of 4", processed=3)
|
||||
job.update(total=4)
|
||||
job.record("known", {"path": "a.png", "root": "output"}, "= a.png")
|
||||
detail = job.detail()
|
||||
self.assertEqual(detail["progress"], 100)
|
||||
self.assertEqual(detail["status_line"], "Processing file 3 of 4")
|
||||
self.assertEqual((detail["processed"], detail["total"]), (3, 4))
|
||||
self.assertEqual(detail["counters"]["known"], 1)
|
||||
self.assertEqual(detail["outcomes"]["known"][0]["path"], "a.png")
|
||||
self.assertEqual(len(detail["log"]), 1)
|
||||
detail["outcomes"]["known"].clear() # a copy, not the live list
|
||||
self.assertEqual(len(job.outcomes["known"]), 1)
|
||||
job.publish({"type": "progress", "progress": -5})
|
||||
self.assertEqual(job.summary()["progress"], 0)
|
||||
|
||||
def test_scan_groups_follow_the_counter_mapping(self):
|
||||
self.assertEqual(
|
||||
SCAN_GROUPS,
|
||||
{
|
||||
"added": "added",
|
||||
"linked": "added",
|
||||
"enriched": "added",
|
||||
"unchanged": "known",
|
||||
# Activity-2: a file the "new" scope skipped is Already known.
|
||||
"skipped": "known",
|
||||
"ambiguous": "review",
|
||||
"conflict": "review",
|
||||
"unresolved": "review",
|
||||
"failed": "errors",
|
||||
},
|
||||
)
|
||||
self.assertEqual((SCAN_SCOPES, DEFAULT_SCAN_SCOPE), (("new", "all"), "new"))
|
||||
self.assertEqual(
|
||||
SCAN_GROUP_NAMES, ("added", "known", "review", "no_metadata", "errors")
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
class TestListedSummary(unittest.TestCase):
|
||||
def test_summary_counts_listed_rows_and_log_lines(self):
|
||||
job = Job("scan", "Scan", groups=("added", "errors"))
|
||||
job.record(
|
||||
"added", {"path": "a.png", "root": "output", "root_index": 0}, "+ a.png"
|
||||
)
|
||||
job.record("errors", {"path": "b.png", "root_index": "x"}, "! b.png")
|
||||
summary = job.summary()
|
||||
self.assertEqual(summary["listed"], {"added": 1, "errors": 1})
|
||||
self.assertEqual(summary["log_length"], 2)
|
||||
rows = job.detail()["outcomes"]
|
||||
self.assertEqual(rows["added"][0]["root_index"], 0)
|
||||
# Only a whole non-negative number is a root index.
|
||||
self.assertIsNone(rows["errors"][0]["root_index"])
|
||||
@@ -0,0 +1,886 @@
|
||||
"""Bounded private capture store (#249 A3): exact bytes, hard limits, cleanup."""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import stat
|
||||
import struct
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
try:
|
||||
from tests.open_handles import assert_closed, open_handles_under
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from open_handles import assert_closed, open_handles_under
|
||||
|
||||
try: # guarded so a missing module fails assertions, not test loading
|
||||
from utils import media_capture as capture_module
|
||||
from utils.media_capture import (
|
||||
CAPTURE_EXTENSIONS,
|
||||
CaptureCapacity,
|
||||
CaptureChanged,
|
||||
CaptureDenied,
|
||||
CaptureStore,
|
||||
CaptureTooLarge,
|
||||
)
|
||||
except ImportError:
|
||||
capture_module = None
|
||||
CAPTURE_EXTENSIONS = None
|
||||
CaptureCapacity = CaptureChanged = CaptureDenied = CaptureTooLarge = None
|
||||
CaptureStore = None
|
||||
|
||||
EPOCH = "epoch-1"
|
||||
SCOPE = "a" * 64
|
||||
WAIT = 30.0 # generous ceiling for cross-thread handoffs, never a short clock
|
||||
|
||||
|
||||
def _chunk(kind: bytes, data: bytes) -> bytes:
|
||||
return (
|
||||
struct.pack(">I", len(data))
|
||||
+ kind
|
||||
+ data
|
||||
+ struct.pack(">I", zlib.crc32(kind + data))
|
||||
)
|
||||
|
||||
|
||||
def png_bytes(width: int, height: int, pixels: bytes = b"\x00" * 16) -> bytes:
|
||||
"""A structurally valid PNG whose header declares *width* x *height*."""
|
||||
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
return (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ _chunk(b"IHDR", ihdr)
|
||||
+ _chunk(b"IDAT", pixels)
|
||||
+ _chunk(b"IEND", b"")
|
||||
)
|
||||
|
||||
|
||||
class FakeClock:
|
||||
def __init__(self, now: float = 1_000.0):
|
||||
self.now = now
|
||||
|
||||
def __call__(self) -> float:
|
||||
return self.now
|
||||
|
||||
def advance(self, seconds: float) -> None:
|
||||
self.now += seconds
|
||||
|
||||
|
||||
class CaptureStoreTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.assertIsNotNone(capture_module, "utils.media_capture is not implemented")
|
||||
parent = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(parent.cleanup)
|
||||
self.addCleanup(assert_closed, self, parent.name)
|
||||
# The resolved path: macOS /var -> /private/var, Windows 8.3 names.
|
||||
self.parent = Path(os.path.realpath(parent.name))
|
||||
self.root = self.parent / "output"
|
||||
self.root.mkdir()
|
||||
self.clock = FakeClock()
|
||||
|
||||
def make_store(self, **options) -> "CaptureStore":
|
||||
options.setdefault("clock", self.clock)
|
||||
store = CaptureStore(self.parent / "captures", **options)
|
||||
self.addCleanup(store.close)
|
||||
return store
|
||||
|
||||
def write(self, relative: str, data: bytes) -> Path:
|
||||
path = self.root / relative
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(data)
|
||||
return path
|
||||
|
||||
def symlink(self, target, link):
|
||||
try:
|
||||
os.symlink(str(target), str(link))
|
||||
except (OSError, NotImplementedError) as error:
|
||||
self.skipTest(f"symlinks unavailable here: {error}")
|
||||
|
||||
def wait_for_change_time_after(self, ns: int) -> None:
|
||||
"""Block until a new file's ``st_ctime_ns`` passes *ns*.
|
||||
|
||||
File timestamps tick coarsely (Linux jiffies, Windows ~1-16 ms). Each
|
||||
probe is a fresh name: Windows reports the creation time (or, on
|
||||
Python 3.12+, the change time) and tunnels the creation time of a
|
||||
reused name for 15 s.
|
||||
"""
|
||||
deadline = time.monotonic() + WAIT
|
||||
for index in range(1_000_000):
|
||||
probe = self.parent / f"tick-{index}"
|
||||
probe.write_bytes(b"x")
|
||||
advanced = os.stat(probe).st_ctime_ns > ns
|
||||
probe.unlink()
|
||||
if advanced:
|
||||
return
|
||||
if time.monotonic() >= deadline:
|
||||
break
|
||||
time.sleep(0.001)
|
||||
self.fail("the filesystem clock did not advance") # pragma: no cover
|
||||
|
||||
def capture(self, store, relative="a.png", **options):
|
||||
options.setdefault("epoch", EPOCH)
|
||||
options.setdefault("scope", SCOPE)
|
||||
return store.capture(self.root, relative, **options)
|
||||
|
||||
def assert_empty(self, store):
|
||||
self.assertFalse(any(Path(store.directory).iterdir()))
|
||||
self.assertEqual(
|
||||
store.stats(), {"captures": 0, "reserved_bytes": 0, "workers": 0}
|
||||
)
|
||||
|
||||
# -- exact bytes and identity -------------------------------------------
|
||||
|
||||
def test_capture_hashes_and_copies_exact_bytes_owner_only(self):
|
||||
source_bytes = png_bytes(3, 2, os.urandom(40))
|
||||
source = self.write("a.png", source_bytes)
|
||||
store = self.make_store()
|
||||
|
||||
cap = self.capture(store, root_index=1)
|
||||
|
||||
self.assertEqual(Path(cap.path).read_bytes(), source_bytes)
|
||||
self.assertEqual(cap.sha256, hashlib.sha256(source_bytes).hexdigest())
|
||||
self.assertEqual(cap.mtime_ns, os.stat(source).st_mtime_ns)
|
||||
self.assertEqual(cap.size, len(source_bytes))
|
||||
self.assertEqual(
|
||||
cap.locator,
|
||||
os.path.normcase(os.path.join(os.path.realpath(self.root), "a.png")),
|
||||
)
|
||||
self.assertEqual((cap.root_index, cap.scope, cap.epoch), (1, SCOPE, EPOCH))
|
||||
self.assertEqual((cap.media_type, cap.width, cap.height), ("image", 3, 2))
|
||||
self.assertTrue(cap.inferable)
|
||||
self.assertIsNone(cap.inferable_reason)
|
||||
self.assertIsInstance(cap.token, str)
|
||||
self.assertGreaterEqual(len(cap.token), 32)
|
||||
self.assertNotIn(cap.token, str(cap.path))
|
||||
self.assertNotIn(str(self.root), str(cap.path))
|
||||
capture_path = Path(cap.path)
|
||||
self.assertEqual(capture_path.parent, Path(store.directory))
|
||||
self.assertEqual(capture_path.suffix, ".png")
|
||||
self.assertTrue(
|
||||
Path(store.directory).name.startswith("prompt-manager-captures-")
|
||||
)
|
||||
if os.name == "posix":
|
||||
self.assertEqual(stat.S_IMODE(os.stat(store.directory).st_mode) & 0o077, 0)
|
||||
self.assertEqual(stat.S_IMODE(os.stat(cap.path).st_mode) & 0o077, 0)
|
||||
self.assertIs(store.get(cap.token), cap)
|
||||
self.assertEqual(
|
||||
store.stats(),
|
||||
{"captures": 1, "reserved_bytes": len(source_bytes), "workers": 0},
|
||||
)
|
||||
self.assertTrue(store.release(cap.token))
|
||||
self.assertFalse(store.release(cap.token))
|
||||
self.assertIsNone(store.get(cap.token))
|
||||
self.assert_empty(store)
|
||||
assert_closed(self, self.parent)
|
||||
|
||||
def test_streams_in_bounded_chunks_never_the_whole_file(self):
|
||||
data = png_bytes(1, 1, os.urandom(5 * capture_module.CHUNK_SIZE + 7))
|
||||
self.write("big.png", data)
|
||||
store = self.make_store(max_file=len(data))
|
||||
sizes = []
|
||||
original = CaptureStore._read
|
||||
|
||||
def hook(store_, stream, size):
|
||||
sizes.append(size)
|
||||
return original(store_, stream, size)
|
||||
|
||||
with patch.object(CaptureStore, "_read", hook):
|
||||
cap = self.capture(store, "big.png")
|
||||
self.assertTrue(sizes)
|
||||
self.assertLessEqual(max(sizes), capture_module.CHUNK_SIZE)
|
||||
self.assertEqual(cap.sha256, hashlib.sha256(data).hexdigest())
|
||||
|
||||
def test_symlinked_file_or_root_escape_is_denied(self):
|
||||
outside = self.parent / "outside"
|
||||
outside.mkdir()
|
||||
(outside / "real.png").write_bytes(png_bytes(1, 1))
|
||||
self.write("a.png", png_bytes(1, 1))
|
||||
self.write("notes.txt", b"text")
|
||||
(self.root / "folder").mkdir()
|
||||
store = self.make_store()
|
||||
|
||||
denied = [
|
||||
"../outside/real.png",
|
||||
"a/../a.png",
|
||||
"./a.png",
|
||||
"a.png/",
|
||||
"",
|
||||
"/a.png",
|
||||
"C:/a.png",
|
||||
"a\\b.png",
|
||||
"a\x00.png",
|
||||
"notes.txt",
|
||||
"folder",
|
||||
"x" * 1021 + ".png",
|
||||
]
|
||||
for relative in denied:
|
||||
with self.subTest(relative=relative):
|
||||
with self.assertRaises(CaptureDenied):
|
||||
self.capture(store, relative)
|
||||
with self.assertRaises(CaptureDenied):
|
||||
self.capture(store, "a.png", root_index=True)
|
||||
with self.assertRaises(CaptureChanged):
|
||||
self.capture(store, "missing.png")
|
||||
self.assert_empty(store)
|
||||
|
||||
self.symlink(outside / "real.png", self.root / "link.png")
|
||||
self.symlink(outside, self.root / "linked-dir")
|
||||
with self.assertRaises(CaptureDenied):
|
||||
self.capture(store, "link.png")
|
||||
with self.assertRaises(CaptureDenied):
|
||||
self.capture(store, "linked-dir/real.png")
|
||||
self.assert_empty(store)
|
||||
# A symlink to a file inside the root is still an alias, never a locator.
|
||||
self.symlink(self.root / "a.png", self.root / "alias.png")
|
||||
with self.assertRaises(CaptureDenied):
|
||||
self.capture(store, "alias.png")
|
||||
self.assert_empty(store)
|
||||
|
||||
def test_locator_matches_listing_identity_for_symlinked_root(self):
|
||||
real_root = self.parent / "real-output"
|
||||
real_root.mkdir()
|
||||
(real_root / "sub").mkdir()
|
||||
source_bytes = png_bytes(2, 2)
|
||||
(real_root / "sub" / "a.png").write_bytes(source_bytes)
|
||||
link_root = self.parent / "link-output"
|
||||
self.symlink(real_root, link_root)
|
||||
store = self.make_store()
|
||||
|
||||
cap = store.capture(link_root, "sub/a.png", epoch=EPOCH, scope=SCOPE)
|
||||
|
||||
expected = os.path.normcase(
|
||||
os.path.join(os.path.realpath(link_root), "sub", "a.png")
|
||||
)
|
||||
self.assertEqual(cap.locator, expected)
|
||||
self.assertEqual(
|
||||
cap.locator, os.path.normcase(os.path.realpath(link_root / "sub/a.png"))
|
||||
)
|
||||
self.assertTrue(
|
||||
cap.locator.startswith(os.path.normcase(os.path.realpath(real_root)))
|
||||
)
|
||||
self.assertEqual(cap.sha256, hashlib.sha256(source_bytes).hexdigest())
|
||||
|
||||
# -- change detection ----------------------------------------------------
|
||||
|
||||
def test_change_during_copy_is_detected_and_temp_removed(self):
|
||||
source_bytes = png_bytes(1, 1, b"\x11" * 40)
|
||||
source = self.write("a.png", source_bytes)
|
||||
store = self.make_store(max_file=4096)
|
||||
original = CaptureStore._read
|
||||
|
||||
def appending(store_, stream, size):
|
||||
data = original(store_, stream, 16)
|
||||
if data and not getattr(appending, "done", False):
|
||||
appending.done = True
|
||||
with open(source, "ab") as handle:
|
||||
handle.write(b"\x22" * 8)
|
||||
return data
|
||||
|
||||
with patch.object(CaptureStore, "_read", appending):
|
||||
with self.assertRaises(CaptureChanged):
|
||||
store.capture(self.root, "a.png", epoch=EPOCH, scope=SCOPE)
|
||||
self.assert_empty(store)
|
||||
|
||||
# Same size, different bytes, mtime restored: only the change time
|
||||
# (POSIX ctime) can tell. Windows exposes no change time through stat,
|
||||
# so there the torn copy completes and its digest describes the copy.
|
||||
source.write_bytes(source_bytes)
|
||||
before = os.stat(source)
|
||||
self.wait_for_change_time_after(before.st_ctime_ns)
|
||||
rewritten = bytes(b ^ 0xFF for b in source_bytes)
|
||||
|
||||
def rewriting(store_, stream, size):
|
||||
data = original(store_, stream, 16)
|
||||
if data and not getattr(rewriting, "done", False):
|
||||
rewriting.done = True
|
||||
with open(source, "r+b") as handle:
|
||||
handle.write(rewritten)
|
||||
os.utime(source, ns=(before.st_atime_ns, before.st_mtime_ns))
|
||||
return data
|
||||
|
||||
# Only a change time can tell: POSIX always has one; Windows exposes
|
||||
# it from stat on Python 3.12+ (where the OS supports the fast path)
|
||||
# and never from fstat, so detection there is best effort. The test
|
||||
# asserts the outcome, not the platform: a detected change leaves no
|
||||
# temp file; an undetected one (Windows only) completes with a digest
|
||||
# that describes the copied bytes.
|
||||
with patch.object(CaptureStore, "_read", rewriting):
|
||||
try:
|
||||
cap = store.capture(self.root, "a.png", epoch=EPOCH, scope=SCOPE)
|
||||
except CaptureChanged:
|
||||
detected = True
|
||||
else:
|
||||
detected = False
|
||||
self.assertEqual(os.name, "nt", "only Windows may miss the rewrite")
|
||||
copied = Path(cap.path).read_bytes()
|
||||
self.assertEqual(cap.sha256, hashlib.sha256(copied).hexdigest())
|
||||
store.release(cap.token)
|
||||
if os.name == "posix":
|
||||
self.assertTrue(detected)
|
||||
self.assert_empty(store)
|
||||
self.assertEqual(os.stat(source).st_mtime_ns, before.st_mtime_ns)
|
||||
|
||||
if os.name == "posix":
|
||||
# The path now names another file (new inode) once the copy ends.
|
||||
source.write_bytes(source_bytes)
|
||||
|
||||
def replacing(store_, stream, size):
|
||||
data = original(store_, stream, size)
|
||||
if not data and not getattr(replacing, "done", False):
|
||||
replacing.done = True
|
||||
other = self.root / "other.png"
|
||||
other.write_bytes(source_bytes)
|
||||
os.replace(other, source)
|
||||
return data
|
||||
|
||||
with patch.object(CaptureStore, "_read", replacing):
|
||||
with self.assertRaises(CaptureChanged):
|
||||
store.capture(self.root, "a.png", epoch=EPOCH, scope=SCOPE)
|
||||
self.assert_empty(store)
|
||||
assert_closed(self, self.parent)
|
||||
|
||||
def test_size_cap_on_stat_and_stream(self):
|
||||
store = self.make_store(max_file=64)
|
||||
self.write("large.png", bytes(65))
|
||||
self.write("fits.png", bytes(64))
|
||||
|
||||
def never(store_, stream, size): # pragma: no cover - guard only
|
||||
raise AssertionError("an over-cap file must not be read")
|
||||
|
||||
with patch.object(CaptureStore, "_read", never):
|
||||
with self.assertRaises(CaptureTooLarge):
|
||||
self.capture(store, "large.png")
|
||||
self.assert_empty(store)
|
||||
|
||||
cap = self.capture(store, "fits.png")
|
||||
self.assertEqual(cap.size, 64)
|
||||
store.release(cap.token)
|
||||
|
||||
growing = self.write("growing.png", bytes(40))
|
||||
original = CaptureStore._read
|
||||
|
||||
def grow(store_, stream, size):
|
||||
data = original(store_, stream, 16)
|
||||
if data and not getattr(grow, "done", False):
|
||||
grow.done = True
|
||||
with open(growing, "ab") as handle:
|
||||
handle.write(bytes(40))
|
||||
return data
|
||||
|
||||
with patch.object(CaptureStore, "_read", grow):
|
||||
with self.assertRaises(CaptureTooLarge):
|
||||
self.capture(store, "growing.png")
|
||||
self.assert_empty(store)
|
||||
|
||||
# -- admission -----------------------------------------------------------
|
||||
|
||||
def test_count_byte_and_worker_admission_is_nonblocking(self):
|
||||
for name in ("a", "b", "c"):
|
||||
self.write(f"{name}.png", bytes(60))
|
||||
store = self.make_store(max_captures=2, max_bytes=1000)
|
||||
first = self.capture(store, "a.png")
|
||||
self.capture(store, "b.png")
|
||||
with self.assertRaises(CaptureCapacity):
|
||||
self.capture(store, "c.png")
|
||||
self.assertTrue(store.release(first.token))
|
||||
third = self.capture(store, "c.png")
|
||||
self.assertEqual(store.stats()["captures"], 2)
|
||||
store.invalidate_all()
|
||||
self.assertIsNone(store.get(third.token))
|
||||
self.assert_empty(store)
|
||||
|
||||
bytes_store = self.make_store(max_bytes=100)
|
||||
kept = self.capture(bytes_store, "a.png")
|
||||
with self.assertRaises(CaptureCapacity):
|
||||
self.capture(bytes_store, "b.png")
|
||||
self.assertEqual(bytes_store.stats()["reserved_bytes"], 60)
|
||||
bytes_store.release(kept.token)
|
||||
self.capture(bytes_store, "b.png")
|
||||
|
||||
workers = self.make_store(max_workers=1)
|
||||
entered, proceed = threading.Event(), threading.Event()
|
||||
original = CaptureStore._read
|
||||
results = {}
|
||||
|
||||
def blocking(store_, stream, size):
|
||||
if not entered.is_set():
|
||||
entered.set()
|
||||
self.assertTrue(proceed.wait(WAIT))
|
||||
return original(store_, stream, size)
|
||||
|
||||
def worker():
|
||||
try:
|
||||
results["capture"] = self.capture(workers, "a.png")
|
||||
except Exception as error: # pragma: no cover - reported below
|
||||
results["error"] = error
|
||||
|
||||
with patch.object(CaptureStore, "_read", blocking):
|
||||
thread = threading.Thread(target=worker)
|
||||
thread.start()
|
||||
try:
|
||||
self.assertTrue(entered.wait(WAIT))
|
||||
self.assertEqual(workers.stats()["workers"], 1)
|
||||
with self.assertRaises(CaptureCapacity):
|
||||
self.capture(workers, "b.png")
|
||||
finally:
|
||||
proceed.set()
|
||||
thread.join(WAIT)
|
||||
self.assertFalse(thread.is_alive())
|
||||
self.assertNotIn("error", results)
|
||||
self.assertEqual(workers.stats()["workers"], 0)
|
||||
self.assertEqual(results["capture"].size, 60)
|
||||
self.capture(workers, "b.png")
|
||||
|
||||
# -- lifetime ------------------------------------------------------------
|
||||
|
||||
def test_idle_and_hard_expiry_and_in_use_retention(self):
|
||||
self.write("a.png", png_bytes(1, 1))
|
||||
self.write("b.png", png_bytes(1, 1))
|
||||
store = self.make_store(idle=900, hard=1800)
|
||||
cap = self.capture(store, "a.png")
|
||||
|
||||
self.clock.advance(899)
|
||||
self.assertIs(store.get(cap.token), cap) # touches idle
|
||||
self.assertAlmostEqual(store.expires_in(cap.token), 900)
|
||||
self.clock.advance(899)
|
||||
self.assertIs(store.get(cap.token), cap)
|
||||
self.assertAlmostEqual(store.expires_in(cap.token), 2)
|
||||
self.clock.advance(2)
|
||||
self.assertIsNone(store.get(cap.token)) # hard limit despite touches
|
||||
self.assertIsNone(store.expires_in(cap.token))
|
||||
self.assertFalse(Path(cap.path).exists())
|
||||
self.assert_empty(store)
|
||||
|
||||
idle = self.capture(store, "a.png")
|
||||
self.clock.advance(900)
|
||||
store.sweep()
|
||||
self.assertIsNone(store.get(idle.token))
|
||||
self.assert_empty(store)
|
||||
|
||||
used = self.capture(store, "b.png")
|
||||
self.assertIs(store.acquire(used.token), used)
|
||||
self.assertIs(store.acquire(used.token), used)
|
||||
self.clock.advance(901)
|
||||
store.sweep()
|
||||
self.assertIs(store.get(used.token), used) # in use is not idle
|
||||
self.clock.advance(900)
|
||||
store.sweep()
|
||||
self.assertIsNone(store.get(used.token)) # expired for new requests
|
||||
self.assertIsNone(store.acquire(used.token))
|
||||
self.assertTrue(Path(used.path).exists()) # retained while in use
|
||||
self.assertEqual(store.stats()["captures"], 1)
|
||||
store.release_use(used.token)
|
||||
self.assertTrue(Path(used.path).exists())
|
||||
store.release_use(used.token)
|
||||
self.assertFalse(Path(used.path).exists())
|
||||
self.assert_empty(store)
|
||||
store.release_use(used.token) # unknown token: no error, nothing freed
|
||||
self.assertIsNone(store.acquire("unknown"))
|
||||
|
||||
def test_release_while_in_use_keeps_file_until_last_use_ends(self):
|
||||
self.write("a.png", png_bytes(1, 1))
|
||||
store = self.make_store()
|
||||
cap = self.capture(store, "a.png")
|
||||
self.assertIs(store.acquire(cap.token), cap)
|
||||
self.assertTrue(store.release(cap.token))
|
||||
self.assertIsNone(store.get(cap.token))
|
||||
self.assertTrue(Path(cap.path).exists())
|
||||
store.invalidate_all()
|
||||
self.assertTrue(Path(cap.path).exists())
|
||||
store.release_use(cap.token)
|
||||
self.assertFalse(Path(cap.path).exists())
|
||||
self.assert_empty(store)
|
||||
|
||||
def test_invalidate_all_and_close_remove_owned_directory_only(self):
|
||||
self.write("a.png", png_bytes(1, 1))
|
||||
self.write("b.png", png_bytes(1, 1))
|
||||
sibling = self.parent / "captures" / "sibling.txt"
|
||||
sibling.parent.mkdir()
|
||||
sibling.write_bytes(b"keep")
|
||||
store = CaptureStore(self.parent / "captures", clock=self.clock)
|
||||
try:
|
||||
a = self.capture(store, "a.png")
|
||||
b = self.capture(store, "b.png")
|
||||
self.assertEqual(Path(store.directory).parent, sibling.parent)
|
||||
store.invalidate_all()
|
||||
self.assertIsNone(store.get(a.token))
|
||||
self.assertIsNone(store.get(b.token))
|
||||
self.assert_empty(store)
|
||||
self.assertTrue(Path(store.directory).is_dir())
|
||||
c = self.capture(store, "a.png")
|
||||
self.assertTrue(Path(c.path).exists())
|
||||
finally:
|
||||
store.close()
|
||||
self.assertFalse(Path(store.directory).exists())
|
||||
self.assertEqual(sibling.read_bytes(), b"keep")
|
||||
self.assertTrue((self.root / "a.png").exists())
|
||||
store.close() # idempotent
|
||||
with self.assertRaises(RuntimeError):
|
||||
self.capture(store, "a.png")
|
||||
self.assertIsNone(store.get(c.token))
|
||||
assert_closed(self, self.parent)
|
||||
|
||||
def test_windows_handles_closed_before_unlink(self):
|
||||
source_bytes = png_bytes(1, 1, b"\x33" * 40)
|
||||
source = self.write("a.png", source_bytes)
|
||||
store = self.make_store()
|
||||
original_unlink = os.unlink
|
||||
unlinked = []
|
||||
|
||||
def checked_unlink(path, *args, **kwargs):
|
||||
self.assertEqual(open_handles_under(path), [])
|
||||
self.assertTrue(store._lock.acquire(blocking=False)) # no I/O under lock
|
||||
store._lock.release()
|
||||
unlinked.append(os.fspath(path))
|
||||
return original_unlink(path, *args, **kwargs)
|
||||
|
||||
with patch("os.unlink", checked_unlink):
|
||||
cap = self.capture(store, "a.png")
|
||||
self.assertEqual(open_handles_under(cap.path), [])
|
||||
self.assertEqual(open_handles_under(source), [])
|
||||
store.release(cap.token)
|
||||
original = CaptureStore._read
|
||||
|
||||
def appending(store_, stream, size):
|
||||
data = original(store_, stream, 16)
|
||||
if data and not getattr(appending, "done", False):
|
||||
appending.done = True
|
||||
with open(source, "ab") as handle:
|
||||
handle.write(b"\x44")
|
||||
return data
|
||||
|
||||
with patch.object(CaptureStore, "_read", appending):
|
||||
with self.assertRaises(CaptureChanged):
|
||||
self.capture(store, "a.png")
|
||||
self.assertEqual(open_handles_under(source), [])
|
||||
self.assertEqual(len(unlinked), 2)
|
||||
self.assertTrue(all(p.startswith(str(store.directory)) for p in unlinked))
|
||||
self.assert_empty(store)
|
||||
assert_closed(self, self.parent)
|
||||
|
||||
# -- inference readiness -------------------------------------------------
|
||||
|
||||
def test_dimension_limits_mark_generate_unsupported(self):
|
||||
cases = {
|
||||
"edge.png": (png_bytes(4096, 4096), True),
|
||||
"wide.png": (png_bytes(4097, 1), False),
|
||||
"tall.png": (png_bytes(1, 4097), False),
|
||||
}
|
||||
for name, (data, _) in cases.items():
|
||||
self.write(name, data)
|
||||
self.write("garbage.jpg", b"\xff\xd8not really a jpeg")
|
||||
store = self.make_store()
|
||||
|
||||
for name, (data, supported) in cases.items():
|
||||
with self.subTest(name=name):
|
||||
cap = self.capture(store, name)
|
||||
self.assertEqual(cap.inferable, supported)
|
||||
self.assertEqual((cap.width, cap.height) != (None, None), True)
|
||||
if supported:
|
||||
self.assertIsNone(cap.inferable_reason)
|
||||
else:
|
||||
self.assertIn("4096", cap.inferable_reason)
|
||||
store.release(cap.token)
|
||||
self.assertEqual(
|
||||
(capture_module.MAX_INFERENCE_SIDE, capture_module.MAX_INFERENCE_PIXELS),
|
||||
(4096, 16_777_216),
|
||||
)
|
||||
cap = self.capture(store, "garbage.jpg")
|
||||
self.assertEqual((cap.media_type, cap.width, cap.height), ("image", None, None))
|
||||
self.assertFalse(cap.inferable)
|
||||
self.assertIn("header", cap.inferable_reason)
|
||||
self.assertEqual(
|
||||
cap.sha256, hashlib.sha256(b"\xff\xd8not really a jpeg").hexdigest()
|
||||
)
|
||||
|
||||
def test_video_capture_is_not_inferable(self):
|
||||
data = b"\x00\x00\x00\x18ftypmp42" + os.urandom(32)
|
||||
self.write("clip.mp4", data)
|
||||
store = self.make_store()
|
||||
|
||||
def never(path): # pragma: no cover - guard only
|
||||
raise AssertionError("video captures are never probed")
|
||||
|
||||
with patch.object(capture_module, "read_inspection_file", never):
|
||||
cap = self.capture(store, "clip.mp4")
|
||||
self.assertEqual((cap.media_type, cap.width, cap.height), ("video", None, None))
|
||||
self.assertFalse(cap.inferable)
|
||||
self.assertEqual(
|
||||
cap.inferable_reason,
|
||||
"Auto-tagging is available for supported still images only.",
|
||||
)
|
||||
self.assertEqual(Path(cap.path).suffix, ".mp4")
|
||||
self.assertEqual(cap.sha256, hashlib.sha256(data).hexdigest())
|
||||
|
||||
# -- verification without a copy -------------------------------------
|
||||
|
||||
def test_verify_hashes_current_file_without_retaining_a_copy(self):
|
||||
source_bytes = png_bytes(1, 1, b"\x55" * 30)
|
||||
source = self.write("a.png", source_bytes)
|
||||
store = self.make_store(max_file=128, max_workers=1)
|
||||
|
||||
facts = store.verify(self.root, "a.png")
|
||||
|
||||
self.assertEqual(facts.sha256, hashlib.sha256(source_bytes).hexdigest())
|
||||
self.assertEqual(facts.size, len(source_bytes))
|
||||
self.assertEqual(facts.mtime_ns, os.stat(source).st_mtime_ns)
|
||||
self.assertEqual(
|
||||
facts.locator,
|
||||
os.path.normcase(os.path.join(os.path.realpath(self.root), "a.png")),
|
||||
)
|
||||
self.assert_empty(store)
|
||||
self.write("large.png", bytes(129))
|
||||
with self.assertRaises(CaptureTooLarge):
|
||||
store.verify(self.root, "large.png")
|
||||
with self.assertRaises(CaptureDenied):
|
||||
store.verify(self.root, "../a.png")
|
||||
self.assert_empty(store)
|
||||
|
||||
# -- arguments and failure paths ---------------------------------------
|
||||
|
||||
def test_constructor_and_arguments_are_validated(self):
|
||||
for options in (
|
||||
{"max_file": 0},
|
||||
{"max_captures": True},
|
||||
{"max_bytes": -1},
|
||||
{"max_workers": "2"},
|
||||
{"idle": 0},
|
||||
{"hard": -5},
|
||||
):
|
||||
with self.subTest(options=options):
|
||||
with self.assertRaises(ValueError):
|
||||
CaptureStore(self.parent / "captures", **options)
|
||||
self.write("a.png", png_bytes(1, 1))
|
||||
store = self.make_store()
|
||||
with self.assertRaises(ValueError):
|
||||
store.capture(self.root, "a.png", epoch="", scope=SCOPE)
|
||||
with self.assertRaises(ValueError):
|
||||
store.capture(self.root, "a.png", epoch=EPOCH, scope=None)
|
||||
with self.assertRaises(CaptureDenied):
|
||||
self.capture(store, root_index=-1)
|
||||
with self.assertRaises(CaptureDenied):
|
||||
store.capture(
|
||||
self.parent / "missing-root", "a.png", epoch=EPOCH, scope=SCOPE
|
||||
)
|
||||
self.assert_empty(store)
|
||||
|
||||
default = CaptureStore(clock=self.clock)
|
||||
try:
|
||||
self.assertEqual(
|
||||
Path(default.directory).parent, Path(tempfile.gettempdir())
|
||||
)
|
||||
self.assertTrue(Path(default.directory).is_dir())
|
||||
finally:
|
||||
default.close()
|
||||
self.assertFalse(Path(default.directory).exists())
|
||||
|
||||
def test_directory_target_and_unreadable_file(self):
|
||||
(self.root / "dir.png").mkdir()
|
||||
store = self.make_store()
|
||||
with self.assertRaises(CaptureDenied):
|
||||
self.capture(store, "dir.png")
|
||||
self.assert_empty(store)
|
||||
if os.name != "posix" or os.geteuid() == 0:
|
||||
return # read permission cannot be withdrawn here
|
||||
locked = self.write("locked.png", png_bytes(1, 1))
|
||||
os.chmod(locked, 0)
|
||||
try:
|
||||
with self.assertRaises(CaptureChanged):
|
||||
self.capture(store, "locked.png")
|
||||
finally:
|
||||
os.chmod(locked, 0o600)
|
||||
self.assert_empty(store)
|
||||
|
||||
def test_removal_failures_are_logged_and_never_touch_other_files(self):
|
||||
self.write("a.png", png_bytes(1, 1))
|
||||
store = CaptureStore(self.parent / "captures", clock=self.clock)
|
||||
stray = Path(store.directory) / "stray.bin"
|
||||
try:
|
||||
cap = self.capture(store, "a.png")
|
||||
self.assertIs(store.acquire(cap.token), cap)
|
||||
self.assertAlmostEqual(store.expires_in(cap.token), 1800) # in use
|
||||
store.release_use(cap.token)
|
||||
original_unlink = os.unlink
|
||||
|
||||
def failing(path, *args, **kwargs):
|
||||
if os.fspath(path) == cap.path:
|
||||
raise PermissionError("busy")
|
||||
return original_unlink(path, *args, **kwargs)
|
||||
|
||||
with patch("os.unlink", failing):
|
||||
with self.assertLogs("utils.media_capture", "WARNING") as logs:
|
||||
self.assertTrue(store.release(cap.token))
|
||||
self.assertIn("could not be removed", logs.output[0])
|
||||
self.assertIsNone(store.get(cap.token))
|
||||
self.assertTrue(Path(cap.path).exists())
|
||||
os.unlink(cap.path)
|
||||
stray.write_bytes(b"not ours")
|
||||
finally:
|
||||
with self.assertLogs("utils.media_capture", "WARNING") as logs:
|
||||
store.close()
|
||||
self.assertIn("could not be removed", logs.output[0])
|
||||
self.assertEqual(stray.read_bytes(), b"not ours")
|
||||
stray.unlink()
|
||||
os.rmdir(store.directory)
|
||||
assert_closed(self, self.parent)
|
||||
|
||||
# -- platform identity checks -------------------------------------------
|
||||
|
||||
def test_path_and_handle_stats_are_never_compared_across_apis(self):
|
||||
"""Windows (Python 3.12+) reports the change time from stat but the
|
||||
creation time from fstat, and may encode dev/ino differently; only
|
||||
size and mtime_ns may be compared across the two APIs."""
|
||||
source_bytes = png_bytes(2, 2, b"\x66" * 40)
|
||||
self.write("a.png", source_bytes)
|
||||
store = self.make_store()
|
||||
original_fstat = os.fstat
|
||||
|
||||
def shifted(fd):
|
||||
st = original_fstat(fd)
|
||||
return os.stat_result(
|
||||
(
|
||||
st.st_mode,
|
||||
st.st_ino + 7,
|
||||
st.st_dev + 7,
|
||||
st.st_nlink,
|
||||
st.st_uid,
|
||||
st.st_gid,
|
||||
st.st_size,
|
||||
st.st_atime,
|
||||
st.st_mtime,
|
||||
st.st_ctime + 1,
|
||||
),
|
||||
{
|
||||
"st_atime_ns": st.st_atime_ns,
|
||||
"st_mtime_ns": st.st_mtime_ns,
|
||||
"st_ctime_ns": st.st_ctime_ns + 1_000_000_000,
|
||||
},
|
||||
)
|
||||
|
||||
with patch.object(capture_module.os, "fstat", shifted):
|
||||
cap = self.capture(store, "a.png")
|
||||
facts = store.verify(self.root, "a.png")
|
||||
self.assertEqual(cap.sha256, hashlib.sha256(source_bytes).hexdigest())
|
||||
self.assertEqual(facts.sha256, cap.sha256)
|
||||
store.release(cap.token)
|
||||
self.assert_empty(store)
|
||||
|
||||
def test_change_time_moving_between_fstat_reads_is_detected(self):
|
||||
"""The Windows 3.13 / POSIX signal, forced on every platform: the
|
||||
descriptor's change time differs after the copy, size and mtime equal."""
|
||||
self.write("a.png", png_bytes(1, 1, b"\x77" * 40))
|
||||
store = self.make_store()
|
||||
original_fstat = os.fstat
|
||||
calls = []
|
||||
|
||||
def drifting(fd):
|
||||
st = original_fstat(fd)
|
||||
calls.append(fd)
|
||||
if len(calls) < 2:
|
||||
return st
|
||||
return os.stat_result(
|
||||
tuple(st)[:10],
|
||||
{
|
||||
"st_atime_ns": st.st_atime_ns,
|
||||
"st_mtime_ns": st.st_mtime_ns,
|
||||
"st_ctime_ns": st.st_ctime_ns + 1,
|
||||
},
|
||||
)
|
||||
|
||||
with patch.object(capture_module.os, "fstat", drifting):
|
||||
with self.assertRaises(CaptureChanged):
|
||||
self.capture(store, "a.png")
|
||||
self.assertEqual(len(calls), 2)
|
||||
self.assert_empty(store)
|
||||
|
||||
def test_fresh_file_captures_twice_on_every_platform(self):
|
||||
data = png_bytes(1, 1, os.urandom(3 * capture_module.CHUNK_SIZE))
|
||||
self.write("fresh.png", data)
|
||||
store = self.make_store()
|
||||
first = self.capture(store, "fresh.png")
|
||||
second = self.capture(store, "fresh.png")
|
||||
self.assertEqual(first.sha256, hashlib.sha256(data).hexdigest())
|
||||
self.assertEqual(second.sha256, first.sha256)
|
||||
self.assertNotEqual(first.token, second.token)
|
||||
self.assertEqual(store.stats()["captures"], 2)
|
||||
|
||||
def test_error_messages_never_name_server_paths(self):
|
||||
self.write("a.png", png_bytes(1, 1))
|
||||
self.write("dir.png/x", b"") # dir.png is a directory
|
||||
store = self.make_store(max_file=64)
|
||||
attempts = [
|
||||
lambda: self.capture(store, "missing.png"),
|
||||
lambda: self.capture(store, "a.png"), # over the 64-byte cap
|
||||
lambda: self.capture(store, "dir.png"),
|
||||
lambda: self.capture(store, "../a.png"),
|
||||
lambda: self.capture(store, "notes.txt"),
|
||||
lambda: store.verify(self.root, "missing.png"),
|
||||
]
|
||||
if os.name == "posix" and os.geteuid() != 0:
|
||||
locked = self.write("locked.png", bytes(10))
|
||||
os.chmod(locked, 0)
|
||||
self.addCleanup(os.chmod, locked, 0o600)
|
||||
attempts.append(lambda: self.capture(store, "locked.png"))
|
||||
for attempt in attempts:
|
||||
with self.assertRaises(capture_module.CaptureError) as caught:
|
||||
attempt()
|
||||
message = str(caught.exception)
|
||||
self.assertTrue(message)
|
||||
self.assertNotIn(str(self.root), message)
|
||||
self.assertNotIn(str(self.parent), message)
|
||||
self.assertNotIn(os.sep, message)
|
||||
self.assert_empty(store)
|
||||
|
||||
def test_close_during_a_capture_removes_the_directory_once_it_finishes(self):
|
||||
self.write("a.png", png_bytes(1, 1, bytes(64)))
|
||||
store = CaptureStore(self.parent / "captures", clock=self.clock)
|
||||
original = CaptureStore._read
|
||||
|
||||
def closing(store_, stream, size):
|
||||
data = original(store_, stream, 16)
|
||||
if data and not getattr(closing, "done", False):
|
||||
closing.done = True
|
||||
store.close()
|
||||
return data
|
||||
|
||||
with patch.object(CaptureStore, "_read", closing):
|
||||
with self.assertRaises(RuntimeError):
|
||||
self.capture(store, "a.png")
|
||||
self.assertFalse(Path(store.directory).exists())
|
||||
self.assertEqual(store.stats()["reserved_bytes"], 0)
|
||||
store.close()
|
||||
assert_closed(self, self.parent)
|
||||
|
||||
# -- constants ---------------------------------------------------------
|
||||
|
||||
def test_defaults_match_the_plan(self):
|
||||
store = self.make_store()
|
||||
self.assertEqual(
|
||||
(
|
||||
store.max_file,
|
||||
store.max_captures,
|
||||
store.max_bytes,
|
||||
store.max_workers,
|
||||
store.idle,
|
||||
store.hard,
|
||||
),
|
||||
(256 * 2**20, 8, 512 * 2**20, 2, 900, 1800),
|
||||
)
|
||||
from py.api.inspection_service import SERVABLE_EXTENSIONS
|
||||
|
||||
self.assertEqual(CAPTURE_EXTENSIONS, SERVABLE_EXTENSIONS)
|
||||
|
||||
def test_exceptions_carry_codes(self):
|
||||
self.assertEqual(CaptureCapacity.code, "review-capacity")
|
||||
self.assertEqual(CaptureChanged.code, "file-changed")
|
||||
self.assertEqual(CaptureTooLarge.code, "unsupported-media")
|
||||
self.assertEqual(CaptureDenied.code, "source-changed")
|
||||
for cls in (CaptureCapacity, CaptureChanged, CaptureTooLarge, CaptureDenied):
|
||||
self.assertTrue(issubclass(cls, capture_module.CaptureError))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -120,9 +120,9 @@ class TestPairScan(AdminAPITestCase):
|
||||
self.assertEqual(self.api._ingest_scanned_file(path, meta), "unchanged")
|
||||
|
||||
async def test_cancel_finishes_at_safe_boundary_and_restart_is_idempotent(self):
|
||||
from py.api.scan_job import ScanJob
|
||||
from py.api.jobs import Job
|
||||
|
||||
job = ScanJob()
|
||||
job = Job("scan", "Scan")
|
||||
self.api._scan_job = job
|
||||
response = await self.client.post("/prompt_manager/scan/cancel")
|
||||
self.assertEqual(response.status, 200)
|
||||
@@ -133,7 +133,7 @@ class TestPairScan(AdminAPITestCase):
|
||||
self.assertEqual(counts["processed"], 0)
|
||||
self.assertEqual(self.api.db.list_prompt_pairs()["total"], 0)
|
||||
job.finish()
|
||||
self.api._scan_job = ScanJob()
|
||||
self.api._scan_job = Job("scan", "Scan")
|
||||
first = self.api._ingest_scan_batch_sync(
|
||||
[(self.output_dir / "cancel.png", {"prompt": json.dumps(graph())})]
|
||||
)
|
||||
|
||||
@@ -251,7 +251,19 @@ class PairSchemaTests(unittest.TestCase):
|
||||
[r[1] for r in conn.execute("PRAGMA table_info(prompt_library_state)")],
|
||||
["prompt_id", "title"],
|
||||
)
|
||||
self.assertEqual(
|
||||
[
|
||||
tuple(r)
|
||||
for r in conn.execute("SELECT * FROM media_annotation_metadata")
|
||||
],
|
||||
[(1, 1, 0)],
|
||||
)
|
||||
self.assertTrue(PromptModel.verify_database_file(live.db_path)[0])
|
||||
self.assertTrue(
|
||||
PromptModel.verify_database_file(live.db_path, require_library_state=True)[
|
||||
0
|
||||
]
|
||||
)
|
||||
|
||||
def test_restore_staged_verification_failure_does_not_replace_live_pages(self):
|
||||
live = self.model()
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Global tag maintenance preserves pair and standalone tag relationships."""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import sqlite3
|
||||
import tempfile
|
||||
@@ -10,6 +11,7 @@ try:
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from open_handles import assert_closed
|
||||
|
||||
from database.library_commands import LibraryCommandError
|
||||
from database.operations import PromptDatabase
|
||||
|
||||
|
||||
@@ -18,6 +20,7 @@ class TestPairTagMaintenance(unittest.TestCase):
|
||||
directory = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(directory.cleanup)
|
||||
self.addCleanup(assert_closed, self, directory.name)
|
||||
self.root = directory.name
|
||||
self.db = PromptDatabase(os.path.join(directory.name, "pair-tags.db"))
|
||||
self.addCleanup(self.db.close_all)
|
||||
|
||||
@@ -27,6 +30,36 @@ class TestPairTagMaintenance(unittest.TestCase):
|
||||
def tags(self, pair_id: int) -> list:
|
||||
return self.db.get_prompt_pair(pair_id)["tags"]
|
||||
|
||||
def image(self, name: str, tags: list, digest: str = "a", revision: int = 0):
|
||||
"""Tag the bytes ``digest`` of the file ``name``; return its locator."""
|
||||
locator = os.path.normcase(os.path.join(os.path.realpath(self.root), name))
|
||||
replace = getattr(self.db, "replace_image_tags", None)
|
||||
self.assertTrue(callable(replace), "replace_image_tags is missing")
|
||||
replace(
|
||||
locator=locator,
|
||||
sha256=hashlib.sha256(digest.encode()).hexdigest(),
|
||||
size=1,
|
||||
mtime_ns=1,
|
||||
expected_epoch=self.db.get_execution_epoch(),
|
||||
expected_tag_revision=revision,
|
||||
tags=tags,
|
||||
)
|
||||
return locator
|
||||
|
||||
def image_tags(self, locator: str, digest: str = "a") -> list:
|
||||
return self.db.read_image_tags(
|
||||
locator, hashlib.sha256(digest.encode()).hexdigest()
|
||||
)["tags"]
|
||||
|
||||
def image_rows(self) -> list:
|
||||
conn = self.db.model.get_connection()
|
||||
return [
|
||||
tuple(row)
|
||||
for row in conn.execute(
|
||||
"SELECT byte_id, tag_id, position FROM media_image_tags ORDER BY 1,3"
|
||||
)
|
||||
]
|
||||
|
||||
def test_rename_to_existing_tag_preserves_pair_links_and_deduplicates(self):
|
||||
first = self.pair("portrait", ["old", "new", "keep"])
|
||||
second = self.pair("landscape", ["old"])
|
||||
@@ -136,6 +169,207 @@ class TestPairTagMaintenance(unittest.TestCase):
|
||||
self.assertEqual(self.tags(other), ["source"])
|
||||
self.assertEqual(self.db.get_prompt_by_id(prompt)["tags"], ["source"])
|
||||
|
||||
def test_rename_and_merge_move_image_links_with_dedup(self):
|
||||
pair = self.pair("portrait", ["old", "new"])
|
||||
both = self.image("both.png", ["new", "old", "keep"])
|
||||
only_old = self.image("old.png", ["old"])
|
||||
result = self.db.rename_tag_all_prompts("old", "new")
|
||||
self.assertEqual(result["affected_count"], 0) # prompts only, as before
|
||||
self.assertEqual(self.tags(pair), ["new"])
|
||||
self.assertEqual(self.image_tags(both), ["new", "keep"])
|
||||
self.assertEqual(self.image_tags(only_old), ["new"])
|
||||
self.assertNotIn("old", self.db.get_all_tags())
|
||||
self.db.rename_tag_all_prompts("keep", "kept")
|
||||
self.assertEqual(self.image_tags(both), ["new", "kept"])
|
||||
merged = self.db.merge_tags(["kept", "new"], "final")
|
||||
self.assertEqual(merged["tags_merged"], 2)
|
||||
self.assertEqual(merged["affected_count"], 0)
|
||||
self.assertEqual(self.image_tags(both), ["final"])
|
||||
self.assertEqual(self.image_tags(only_old), ["final"])
|
||||
self.assertEqual(self.tags(pair), ["final"])
|
||||
epoch = self.db.get_execution_epoch()
|
||||
conn = self.db.model.get_connection()
|
||||
final = conn.execute("SELECT id FROM tags WHERE name='final'").fetchone()[0]
|
||||
native = self.db.rename_vocabulary_tag(
|
||||
final, expected_epoch=epoch, name="final", new_name="native"
|
||||
)
|
||||
self.assertEqual(native["images_changed"], 2)
|
||||
self.assertEqual(self.image_tags(both), ["native"])
|
||||
self.assertEqual(self.image_tags(only_old), ["native"])
|
||||
self.image("again.png", ["native", "merge-me"])
|
||||
source = conn.execute("SELECT id FROM tags WHERE name='merge-me'").fetchone()
|
||||
receipt = self.db.rename_vocabulary_tag(
|
||||
source[0], expected_epoch=epoch, name="merge-me", new_name="native"
|
||||
)
|
||||
self.assertTrue(receipt["merged"])
|
||||
self.assertEqual(receipt["images_changed"], 1)
|
||||
self.assertEqual(receipt["tag"]["image_count"], 3)
|
||||
self.assertEqual(len(self.image_rows()), 3)
|
||||
|
||||
def test_delete_cascades_image_links_and_keeps_legacy_counts(self):
|
||||
pair = self.pair("portrait", ["remove", "keep"])
|
||||
prompt = self.db.save_prompt("standalone", tags=["remove", "keep"])
|
||||
locator = self.image("a.png", ["keep", "remove"])
|
||||
result = self.db.delete_tag_all_prompts("remove")
|
||||
self.assertEqual(result["affected_count"], 1)
|
||||
self.assertEqual(self.tags(pair), ["keep"])
|
||||
self.assertEqual(self.db.get_prompt_by_id(prompt)["tags"], ["keep"])
|
||||
self.assertEqual(self.image_tags(locator), ["keep"])
|
||||
self.assertEqual(len(self.image_rows()), 1)
|
||||
epoch = self.db.get_execution_epoch()
|
||||
conn = self.db.model.get_connection()
|
||||
keep = conn.execute("SELECT id FROM tags WHERE name='keep'").fetchone()[0]
|
||||
impact = self.db.vocabulary_tag_impact(keep)["impact"]
|
||||
self.assertEqual(impact["images"], 1)
|
||||
deleted = self.db.delete_vocabulary_tag(keep, expected_epoch=epoch, name="keep")
|
||||
self.assertEqual(
|
||||
(
|
||||
deleted["components_changed"],
|
||||
deleted["pairs_changed"],
|
||||
deleted["images_changed"],
|
||||
),
|
||||
(1, 1, 1),
|
||||
)
|
||||
self.assertEqual(self.image_tags(locator), [])
|
||||
self.assertEqual(self.image_rows(), [])
|
||||
self.assertEqual(
|
||||
conn.execute("SELECT COUNT(*) FROM media_annotation_bytes").fetchone()[0], 1
|
||||
)
|
||||
|
||||
def index_revision(self) -> int:
|
||||
return (
|
||||
self.db.model.get_connection()
|
||||
.execute("SELECT image_index_revision FROM media_annotation_metadata")
|
||||
.fetchone()[0]
|
||||
)
|
||||
|
||||
def revision(self, locator: str, digest: str = "a") -> int:
|
||||
return self.db.read_image_tags(
|
||||
locator, hashlib.sha256(digest.encode()).hexdigest()
|
||||
)["tag_revision"]
|
||||
|
||||
def tag_id(self, name: str) -> int:
|
||||
conn = self.db.model.get_connection()
|
||||
return conn.execute("SELECT id FROM tags WHERE name=?", (name,)).fetchone()[0]
|
||||
|
||||
def test_vocabulary_maintenance_bumps_image_revisions_so_stale_saves_conflict(
|
||||
self,
|
||||
):
|
||||
forest = self.image("a.png", ["forest"])
|
||||
misty = self.image("b.png", ["mist"])
|
||||
self.pair("portrait", ["unlinked"])
|
||||
index = self.index_revision()
|
||||
epoch = self.db.get_execution_epoch()
|
||||
|
||||
# A rename of a tag no image carries moves nothing image-side.
|
||||
self.db.rename_vocabulary_tag(
|
||||
self.tag_id("unlinked"), expected_epoch=epoch, name="unlinked", new_name="u"
|
||||
)
|
||||
self.assertEqual(self.index_revision(), index)
|
||||
|
||||
self.db.rename_vocabulary_tag(
|
||||
self.tag_id("forest"), expected_epoch=epoch, name="forest", new_name="woods"
|
||||
)
|
||||
self.assertEqual(self.image_tags(forest), ["woods"])
|
||||
self.assertEqual((self.revision(forest), self.revision(misty)), (2, 1))
|
||||
self.assertEqual(self.index_revision(), index + 1)
|
||||
# The Tag dialog still open at revision 1 cannot recreate "forest".
|
||||
with self.assertRaises(LibraryCommandError) as caught:
|
||||
self.db.replace_image_tags(
|
||||
locator=forest,
|
||||
sha256=hashlib.sha256(b"a").hexdigest(),
|
||||
size=1,
|
||||
mtime_ns=1,
|
||||
expected_epoch=epoch,
|
||||
expected_tag_revision=1,
|
||||
tags=["forest", "mist"],
|
||||
)
|
||||
self.assertEqual(caught.exception.code, "tag-conflict")
|
||||
self.assertEqual(caught.exception.current["tags"], ["woods"])
|
||||
self.assertEqual(caught.exception.current["tag_revision"], 2)
|
||||
self.assertNotIn("forest", self.db.get_all_tags())
|
||||
|
||||
self.db.rename_tag_all_prompts("woods", "trees") # legacy rename
|
||||
self.assertEqual((self.revision(forest), self.revision(misty)), (3, 1))
|
||||
self.db.merge_tags(["trees"], "mist") # legacy merge: only the source side
|
||||
self.assertEqual(self.image_tags(forest), ["mist"])
|
||||
self.assertEqual((self.revision(forest), self.revision(misty)), (4, 1))
|
||||
self.db.rename_tag_all_prompts("mist", "fog") # both carry it now
|
||||
self.assertEqual((self.revision(forest), self.revision(misty)), (5, 2))
|
||||
self.db.delete_vocabulary_tag(
|
||||
self.tag_id("fog"), expected_epoch=epoch, name="fog"
|
||||
)
|
||||
self.assertEqual((self.revision(forest), self.revision(misty)), (6, 3))
|
||||
self.assertEqual((self.image_tags(forest), self.image_tags(misty)), ([], []))
|
||||
self.image("c.png", ["gone"])
|
||||
self.db.delete_tag_all_prompts("gone") # legacy delete
|
||||
self.assertEqual(self.revision(forest), 6)
|
||||
self.assertEqual(
|
||||
self.index_revision(), index + 7
|
||||
) # 6 maintenance ops + the c.png save
|
||||
|
||||
def test_image_only_and_prior_only_names_participate(self):
|
||||
only = self.image("only.png", ["image-only"])
|
||||
prior = self.image("prior.png", ["prior-only"], digest="old")
|
||||
self.image("prior.png", [], digest="new") # the file's bytes changed
|
||||
self.db.save_prompt("standalone", tags=["target"])
|
||||
merged = self.db.merge_tags(["image-only"], "target")
|
||||
self.assertEqual((merged["tags_merged"], merged["affected_count"]), (1, 0))
|
||||
self.assertEqual(self.image_tags(only), ["target"])
|
||||
self.assertEqual(
|
||||
self.db.rename_tag_all_prompts("prior-only", "renamed")["affected_count"], 0
|
||||
)
|
||||
self.assertEqual(self.image_tags(prior, "old"), ["renamed"])
|
||||
self.assertEqual(self.image_tags(prior, "new"), [])
|
||||
self.assertEqual(set(self.db.get_all_tags()), {"target", "renamed"})
|
||||
epoch = self.db.get_execution_epoch()
|
||||
conn = self.db.model.get_connection()
|
||||
renamed = conn.execute("SELECT id FROM tags WHERE name='renamed'").fetchone()[0]
|
||||
receipt = self.db.rename_vocabulary_tag(
|
||||
renamed, expected_epoch=epoch, name="renamed", new_name="target"
|
||||
)
|
||||
self.assertTrue(receipt["merged"])
|
||||
self.assertEqual(receipt["images_changed"], 1)
|
||||
self.assertEqual(self.image_tags(prior, "old"), ["target"])
|
||||
self.assertEqual(receipt["tag"]["image_count"], 2)
|
||||
|
||||
def test_image_junction_failure_rolls_back_prompt_pair_and_name_changes(self):
|
||||
pair = self.pair("portrait", ["source", "target"])
|
||||
prompt = self.db.save_prompt("standalone", tags=["source"])
|
||||
locator = self.image("a.png", ["source"])
|
||||
conn = self.db.model.get_connection()
|
||||
tables = ("tags", "prompt_tags", "prompt_pair_tags", "media_image_tags")
|
||||
before = [
|
||||
conn.execute(f"SELECT * FROM {table} ORDER BY 1,2").fetchall()
|
||||
for table in tables
|
||||
]
|
||||
source = conn.execute("SELECT id FROM tags WHERE name='source'").fetchone()[0]
|
||||
conn.execute(
|
||||
"CREATE TRIGGER reject_image_tag_change BEFORE UPDATE "
|
||||
"ON media_image_tags BEGIN SELECT RAISE(ABORT,'image tag failure'); END"
|
||||
)
|
||||
conn.commit()
|
||||
with self.assertRaises(sqlite3.IntegrityError):
|
||||
self.db.rename_vocabulary_tag(
|
||||
source,
|
||||
expected_epoch=self.db.get_execution_epoch(),
|
||||
name="source",
|
||||
new_name="target",
|
||||
)
|
||||
with self.assertRaises(sqlite3.IntegrityError):
|
||||
self.db.merge_tags(["source"], "target")
|
||||
with self.assertRaises(sqlite3.IntegrityError):
|
||||
self.db.rename_tag_all_prompts("source", "target")
|
||||
self.assertFalse(conn.in_transaction)
|
||||
after = [
|
||||
conn.execute(f"SELECT * FROM {table} ORDER BY 1,2").fetchall()
|
||||
for table in tables
|
||||
]
|
||||
self.assertEqual(after, before)
|
||||
self.assertEqual(self.tags(pair), ["source", "target"])
|
||||
self.assertEqual(self.db.get_prompt_by_id(prompt)["tags"], ["source"])
|
||||
self.assertEqual(self.image_tags(locator), ["source"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,92 +0,0 @@
|
||||
"""Unit tests for py/api/scan_job.py: fan-out of scan progress to SSE subscribers."""
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from py.api.scan_job import ScanJob # noqa: E402
|
||||
|
||||
|
||||
async def _collect(job, stop_after=None):
|
||||
seen = []
|
||||
async with contextlib.aclosing(job.events()) as events:
|
||||
async for payload in events:
|
||||
seen.append(payload)
|
||||
if stop_after is not None and len(seen) >= stop_after:
|
||||
break
|
||||
return seen
|
||||
|
||||
|
||||
class TestScanJob(unittest.IsolatedAsyncioTestCase):
|
||||
|
||||
async def test_new_job_is_running_with_no_event(self):
|
||||
job = ScanJob()
|
||||
self.assertTrue(job.running)
|
||||
self.assertIsNone(job.last_event)
|
||||
|
||||
async def test_subscriber_receives_events_until_finish(self):
|
||||
job = ScanJob()
|
||||
consumer = asyncio.ensure_future(_collect(job))
|
||||
await asyncio.sleep(0)
|
||||
|
||||
job.publish({"type": "progress", "progress": 10})
|
||||
job.publish({"type": "complete"})
|
||||
job.finish()
|
||||
|
||||
self.assertEqual(
|
||||
await asyncio.wait_for(consumer, 2),
|
||||
[{"type": "progress", "progress": 10}, {"type": "complete"}],
|
||||
)
|
||||
self.assertFalse(job.running)
|
||||
|
||||
async def test_late_subscriber_gets_latest_snapshot_then_live_events(self):
|
||||
job = ScanJob()
|
||||
job.publish({"type": "progress", "progress": 10})
|
||||
job.publish({"type": "progress", "progress": 40})
|
||||
|
||||
consumer = asyncio.ensure_future(_collect(job))
|
||||
await asyncio.sleep(0)
|
||||
job.publish({"type": "complete"})
|
||||
job.finish()
|
||||
|
||||
self.assertEqual(
|
||||
await asyncio.wait_for(consumer, 2),
|
||||
[{"type": "progress", "progress": 40}, {"type": "complete"}],
|
||||
)
|
||||
|
||||
async def test_subscriber_after_finish_gets_only_the_final_event(self):
|
||||
job = ScanJob()
|
||||
job.publish({"type": "progress", "progress": 99})
|
||||
job.publish({"type": "complete"})
|
||||
job.finish()
|
||||
|
||||
self.assertEqual(await _collect(job), [{"type": "complete"}])
|
||||
|
||||
async def test_two_subscribers_both_see_every_event(self):
|
||||
job = ScanJob()
|
||||
first = asyncio.ensure_future(_collect(job))
|
||||
second = asyncio.ensure_future(_collect(job))
|
||||
await asyncio.sleep(0)
|
||||
|
||||
job.publish({"type": "progress", "progress": 1})
|
||||
job.finish()
|
||||
|
||||
self.assertEqual(await first, [{"type": "progress", "progress": 1}])
|
||||
self.assertEqual(await second, [{"type": "progress", "progress": 1}])
|
||||
|
||||
async def test_departed_subscriber_is_forgotten(self):
|
||||
job = ScanJob()
|
||||
job.publish({"type": "progress", "progress": 1})
|
||||
await _collect(job, stop_after=1)
|
||||
|
||||
self.assertEqual(job.subscriber_count, 0)
|
||||
job.publish({"type": "complete"}) # must not raise or queue anywhere
|
||||
job.finish()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,390 @@
|
||||
"""Scan scope (#249 Activity-2 Task 4): new and unresolved files by default.
|
||||
|
||||
A "Scan for metadata" job started without ``scope`` (or with ``"new"``) skips
|
||||
the files the Library already links with a resolved link, before their
|
||||
metadata is read, and counts them as Already known. ``"all"`` reads every file
|
||||
again, as the legacy ``/prompt_manager/scan`` route always does.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
try:
|
||||
from tests.test_api_admin import AdminAPITestCase, _sse_payloads, _write_png
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from test_api_admin import AdminAPITestCase, _sse_payloads, _write_png
|
||||
|
||||
from database import operations # noqa: E402
|
||||
from database.pair_ingestion import ingest_resolved_pair # noqa: E402
|
||||
from py.config import GalleryConfig # noqa: E402
|
||||
from utils.prompt_pair_resolver import resolve_prompt_pair # noqa: E402
|
||||
|
||||
SKIPPED_DETAIL = "Skipped · already linked"
|
||||
ON_DISK = object() # _link_resolved: store the file's size on disk
|
||||
|
||||
|
||||
def _graph(text):
|
||||
return {
|
||||
"1": {"class_type": "CLIPTextEncode", "inputs": {"text": text}},
|
||||
"2": {"class_type": "CLIPTextEncode", "inputs": {"text": "bad"}},
|
||||
"3": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {"positive": ["1", 0], "negative": ["2", 0]},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class ScanScopeTestCase(AdminAPITestCase):
|
||||
"""Helpers: link files the way the scan and the monitor do, spy on reads."""
|
||||
|
||||
def _link_resolved(self, path, text, size=ON_DISK):
|
||||
"""``path`` linked with a resolved pair link (what a resolved scan writes).
|
||||
|
||||
The row stores ``size`` as its file size: by default the file's size on
|
||||
disk (None when the file does not exist).
|
||||
"""
|
||||
if size is ON_DISK:
|
||||
size = os.path.getsize(path) if os.path.exists(path) else None
|
||||
metadata = {"prompt": _graph(text)}
|
||||
if size is not None:
|
||||
metadata["file_info"] = {"size": size}
|
||||
result = ingest_resolved_pair(
|
||||
self.api.db, str(path), metadata, resolve_prompt_pair(metadata)
|
||||
)
|
||||
self.assertEqual(result["status"], "added")
|
||||
|
||||
def _link_without_pair(self, path, text):
|
||||
"""``path`` linked to a prompt with no pair link (fallback / FIFO link)."""
|
||||
prompt_id = self.api.db.save_prompt(text, None, [], None, None)
|
||||
self.assertTrue(self.api.db.link_image_to_prompt(prompt_id, str(path)))
|
||||
return prompt_id
|
||||
|
||||
def _spy_reads(self):
|
||||
reads = []
|
||||
original = self.api._extract_comfyui_metadata
|
||||
|
||||
def spy(path):
|
||||
reads.append(os.path.basename(path))
|
||||
return original(path)
|
||||
|
||||
self.api._extract_comfyui_metadata = spy
|
||||
return reads
|
||||
|
||||
async def _scan(self, body=None):
|
||||
kwargs = {} if body is None else {"json": body}
|
||||
resp = await self.client.request("POST", "/prompt_manager/jobs/scan", **kwargs)
|
||||
data = await resp.json()
|
||||
self.assertEqual(resp.status, 202, data)
|
||||
await asyncio.wait_for(self.api._scan_job.task, 10)
|
||||
detail = await self.client.request(
|
||||
"GET", f"/prompt_manager/jobs/{data['job']['id']}"
|
||||
)
|
||||
return (await detail.json())["job"]
|
||||
|
||||
|
||||
class TestScanScope(ScanScopeTestCase):
|
||||
|
||||
async def test_new_scope_skips_known_resolved_files_without_reading_them(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
known = self.output_dir / "known.png"
|
||||
_write_png(known, prompt_text="a red square")
|
||||
_write_png(self.output_dir / "new.png", prompt_text="a blue circle")
|
||||
self._link_resolved(known, "a red square")
|
||||
reads = self._spy_reads()
|
||||
|
||||
job = await self._scan()
|
||||
|
||||
self.assertEqual(reads, ["new.png"])
|
||||
self.assertEqual(job["scope"]["files"], "new")
|
||||
self.assertEqual(job["counters"]["known"], 1)
|
||||
self.assertEqual(job["counters"]["added"], 1)
|
||||
self.assertEqual(job["processed"], 2)
|
||||
self.assertEqual(
|
||||
job["outcomes"]["known"],
|
||||
[
|
||||
{
|
||||
"path": "known.png",
|
||||
"root": "output",
|
||||
"root_index": 0,
|
||||
"detail": SKIPPED_DETAIL,
|
||||
}
|
||||
],
|
||||
)
|
||||
self.assertIn(f"= output/known.png · {SKIPPED_DETAIL}", "\n".join(job["log"]))
|
||||
self.assertTrue(
|
||||
job["log"][0].endswith(
|
||||
"Scan started · 1 output folder · new and unresolved files"
|
||||
)
|
||||
)
|
||||
legacy = self.api._scan_job.last_event
|
||||
self.assertEqual((legacy["processed"], legacy["unchanged"]), (2, 1))
|
||||
self.assertTrue(job["status_line"].startswith("Processing file 2 of 2"))
|
||||
|
||||
async def test_whole_known_batch_skips_the_executor_reads(self):
|
||||
from py.api import admin
|
||||
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
for name in ("a.png", "b.png"):
|
||||
_write_png(self.output_dir / name, prompt_text=f"prompt {name}")
|
||||
self._link_resolved(self.output_dir / name, f"prompt {name}")
|
||||
_write_png(self.output_dir / "c.png", prompt_text="prompt c")
|
||||
reads = self._spy_reads()
|
||||
|
||||
with patch.object(admin, "SCAN_BATCH_SIZE", 1):
|
||||
job = await self._scan({"scope": "new"})
|
||||
|
||||
self.assertEqual(reads, ["c.png"])
|
||||
self.assertEqual(job["counters"]["known"], 2)
|
||||
self.assertEqual(job["counters"]["added"], 1)
|
||||
|
||||
async def test_unresolved_files_are_rescanned_in_new_scope(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
names = ("fallback.png", "conflict.png", "unlinked.png")
|
||||
for name in names:
|
||||
_write_png(self.output_dir / name, prompt_text=f"text of {name}")
|
||||
# Linked without a resolved pair link (the monitor's fallback / FIFO link).
|
||||
self._link_without_pair(self.output_dir / "fallback.png", "text of fallback")
|
||||
# Resolved once, then also linked to a second prompt: ambiguous.
|
||||
conflict = self.output_dir / "conflict.png"
|
||||
self._link_resolved(conflict, "text of conflict.png")
|
||||
self._link_without_pair(conflict, "another prompt")
|
||||
reads = self._spy_reads()
|
||||
|
||||
job = await self._scan()
|
||||
|
||||
self.assertEqual(sorted(reads), sorted(names))
|
||||
self.assertNotIn(SKIPPED_DETAIL, json.dumps(job["outcomes"]))
|
||||
|
||||
async def test_all_scope_reads_everything(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
known = self.output_dir / "known.png"
|
||||
_write_png(known, prompt_text="a red square")
|
||||
self._link_resolved(known, "a red square")
|
||||
reads = self._spy_reads()
|
||||
|
||||
job = await self._scan({"scope": "all"})
|
||||
|
||||
self.assertEqual(reads, ["known.png"])
|
||||
self.assertEqual(job["scope"]["files"], "all")
|
||||
self.assertEqual(job["counters"]["known"], 1)
|
||||
self.assertNotEqual(job["outcomes"]["known"][0]["detail"], SKIPPED_DETAIL)
|
||||
self.assertTrue(
|
||||
job["log"][0].endswith("Scan started · 1 output folder · all files")
|
||||
)
|
||||
|
||||
async def test_legacy_scan_route_reads_everything(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
known = self.output_dir / "known.png"
|
||||
_write_png(known, prompt_text="a red square")
|
||||
self._link_resolved(known, "a red square")
|
||||
reads = self._spy_reads()
|
||||
|
||||
resp = await self.client.request("POST", "/prompt_manager/scan", data="{}")
|
||||
payloads = _sse_payloads(await resp.text())
|
||||
|
||||
self.assertEqual(reads, ["known.png"])
|
||||
self.assertEqual(self.api._scan_job.scope["files"], "all")
|
||||
self.assertEqual(payloads[-1]["type"], "complete")
|
||||
self.assertEqual(payloads[-1]["unchanged"], 1)
|
||||
|
||||
async def test_file_replaced_at_a_known_path_is_reread_and_needs_review(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
path = self.output_dir / "out_00012.png"
|
||||
_write_png(path, prompt_text="the first prompt")
|
||||
self._link_resolved(path, "the first prompt")
|
||||
before = os.path.getsize(path)
|
||||
# ComfyUI reused the name for a new image with another prompt.
|
||||
_write_png(path, prompt_text="a much longer second prompt for the same name")
|
||||
self.assertNotEqual(os.path.getsize(path), before)
|
||||
reads = self._spy_reads()
|
||||
|
||||
job = await self._scan()
|
||||
|
||||
self.assertEqual(reads, ["out_00012.png"])
|
||||
self.assertEqual(job["counters"]["review"], 1)
|
||||
self.assertEqual(job["counters"]["known"], 0)
|
||||
self.assertIn("different prompt", job["outcomes"]["review"][0]["detail"])
|
||||
|
||||
async def test_scanned_files_store_their_size_so_the_next_new_scan_skips(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
_write_png(self.output_dir / "fresh.png", prompt_text="a fresh prompt")
|
||||
first = await self._scan()
|
||||
self.assertEqual(first["counters"]["added"], 1)
|
||||
reads = self._spy_reads()
|
||||
|
||||
second = await self._scan()
|
||||
|
||||
self.assertEqual(reads, [])
|
||||
self.assertEqual(second["outcomes"]["known"][0]["detail"], SKIPPED_DETAIL)
|
||||
|
||||
async def test_stored_null_size_is_reread(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
path = self.output_dir / "old.png"
|
||||
_write_png(path, prompt_text="an old prompt")
|
||||
self._link_resolved(path, "an old prompt", size=None)
|
||||
reads = self._spy_reads()
|
||||
|
||||
job = await self._scan()
|
||||
|
||||
self.assertEqual(reads, ["old.png"])
|
||||
# The read stores the size it was missing, so the file is known next time.
|
||||
self.assertEqual(job["counters"]["added"], 1)
|
||||
self.assertEqual(
|
||||
self.api.db.get_known_resolved_paths({str(path): os.path.getsize(path)}),
|
||||
{str(path)},
|
||||
)
|
||||
|
||||
async def test_known_set_failure_falls_back_to_reading_every_file(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
known = self.output_dir / "known.png"
|
||||
_write_png(known, prompt_text="a red square")
|
||||
self._link_resolved(known, "a red square")
|
||||
reads = self._spy_reads()
|
||||
|
||||
def broken(files):
|
||||
raise sqlite3.OperationalError("database is locked")
|
||||
|
||||
self.api.db.get_known_resolved_paths = broken
|
||||
with self.assertLogs("prompt_manager.api", level="WARNING"):
|
||||
job = await self._scan()
|
||||
|
||||
self.assertEqual(job["state"], "finished")
|
||||
self.assertEqual(reads, ["known.png"])
|
||||
self.assertIn(
|
||||
"Known files could not be read · reading every file", "\n".join(job["log"])
|
||||
)
|
||||
|
||||
async def test_empty_read_body_asks_for_the_default_scope(self):
|
||||
"""A chunked POST with no bytes can read a body: still the default."""
|
||||
from py.api.job_routes import _scan_scope
|
||||
|
||||
class Request:
|
||||
can_read_body = True
|
||||
|
||||
def __init__(self, text):
|
||||
self._text = text
|
||||
|
||||
async def text(self):
|
||||
return self._text
|
||||
|
||||
self.assertEqual(await _scan_scope(Request("")), "new")
|
||||
self.assertEqual(await _scan_scope(Request(" \n")), "new")
|
||||
self.assertEqual(await _scan_scope(Request('{"scope": "all"}')), "all")
|
||||
self.assertIsNone(await _scan_scope(Request("{")))
|
||||
|
||||
async def test_invalid_scope_is_refused(self):
|
||||
for body in ({"scope": "everything"}, {"scope": 1}, ["new"]):
|
||||
with self.subTest(body=body):
|
||||
resp = await self.client.request(
|
||||
"POST", "/prompt_manager/jobs/scan", json=body
|
||||
)
|
||||
self.assertEqual(resp.status, 400)
|
||||
data = await resp.json()
|
||||
self.assertFalse(data["success"])
|
||||
resp = await self.client.request(
|
||||
"POST", "/prompt_manager/jobs/scan", data="not json"
|
||||
)
|
||||
self.assertEqual(resp.status, 400)
|
||||
self.assertIsNone(self.api._scan_job)
|
||||
|
||||
|
||||
class TestKnownResolvedPaths(ScanScopeTestCase):
|
||||
"""``PromptDatabase.get_known_resolved_paths``: one streamed query."""
|
||||
|
||||
async def test_returns_the_given_paths_that_are_known_and_resolved(self):
|
||||
known, fallback = self.output_dir / "known.png", self.output_dir / "fb.png"
|
||||
self._link_resolved(known, "a red square", size=10)
|
||||
self._link_without_pair(fallback, "a fallback")
|
||||
asked = {str(known): 10, str(fallback): 10, str(self.output_dir / "n.png"): 10}
|
||||
|
||||
self.assertEqual(self.api.db.get_known_resolved_paths(asked), {str(known)})
|
||||
self.assertEqual(self.api.db.get_known_resolved_paths({}), set())
|
||||
|
||||
async def test_size_must_match_and_a_stored_null_size_is_not_known(self):
|
||||
same, other = self.output_dir / "same.png", self.output_dir / "other.png"
|
||||
unknown = self.output_dir / "unknown.png"
|
||||
self._link_resolved(same, "same size", size=10)
|
||||
self._link_resolved(other, "other size", size=10)
|
||||
self._link_resolved(unknown, "no stored size", size=None)
|
||||
asked = {str(same): 10, str(other): 11, str(unknown): 10}
|
||||
|
||||
self.assertEqual(self.api.db.get_known_resolved_paths(asked), {str(same)})
|
||||
# A file the walk could not stat is not known either.
|
||||
self.assertEqual(self.api.db.get_known_resolved_paths({str(same): None}), set())
|
||||
|
||||
async def test_row_without_file_path_matches_through_image_path(self):
|
||||
known = self.output_dir / "legacy.png"
|
||||
self._link_resolved(known, "a legacy row", size=10)
|
||||
with sqlite3.connect(self.api.db.model.db_path) as conn:
|
||||
conn.execute("UPDATE generated_images SET file_path=NULL")
|
||||
conn.close()
|
||||
|
||||
found = self.api.db.get_known_resolved_paths({str(known): 10})
|
||||
self.assertEqual(found, {str(known)})
|
||||
|
||||
async def test_stored_spelling_is_normalized_like_the_scanned_path(self):
|
||||
known = self.output_dir / "sub" / "known.png"
|
||||
self._link_resolved(known, "a red square", size=10)
|
||||
spelled = os.path.join(str(self.output_dir), "sub", "..", "sub", "known.png")
|
||||
with sqlite3.connect(self.api.db.model.db_path) as conn:
|
||||
conn.execute("UPDATE generated_images SET file_path=?", (spelled,))
|
||||
conn.close()
|
||||
|
||||
found = self.api.db.get_known_resolved_paths({str(known): 10})
|
||||
self.assertEqual(found, {str(known)})
|
||||
|
||||
async def test_one_query_streamed_in_small_fetches(self):
|
||||
paths = [self.output_dir / f"{index}.png" for index in range(5)]
|
||||
for path in paths:
|
||||
self._link_resolved(path, f"prompt {path.name}", size=10)
|
||||
original = self.api.db.model.get_connection
|
||||
opened = []
|
||||
|
||||
def counting():
|
||||
opened.append(1)
|
||||
return original()
|
||||
|
||||
self.api.db.model.get_connection = counting
|
||||
with patch.object(operations, "KNOWN_PATHS_FETCH_SIZE", 2):
|
||||
found = self.api.db.get_known_resolved_paths({str(p): 10 for p in paths})
|
||||
self.assertEqual(found, {str(p) for p in paths})
|
||||
self.assertEqual(len(opened), 1)
|
||||
|
||||
async def test_windows_path_case_matches_known_set(self):
|
||||
stored = self.output_dir / "Sub" / "Known.PNG"
|
||||
self._link_resolved(stored, "a red square", size=10)
|
||||
# A row written before paths were case-folded keeps its own spelling.
|
||||
with sqlite3.connect(self.api.db.model.db_path) as conn:
|
||||
conn.execute(
|
||||
"UPDATE generated_images SET file_path=?", (str(stored).upper(),)
|
||||
)
|
||||
conn.close()
|
||||
asked = str(self.output_dir / "sub" / "known.png")
|
||||
|
||||
with patch("os.path.normcase", lambda path: path.lower()):
|
||||
self.assertEqual(self.api.db.get_known_resolved_paths({asked: 10}), {asked})
|
||||
if os.path.normcase("A") == "A": # a case-sensitive platform
|
||||
self.assertEqual(self.api.db.get_known_resolved_paths({asked: 10}), set())
|
||||
|
||||
|
||||
class TestSkippedOutcome(unittest.TestCase):
|
||||
|
||||
def test_skipped_file_is_already_known_with_its_own_detail(self):
|
||||
from py.api.admin import _scan_outcome
|
||||
|
||||
root = os.path.abspath("out")
|
||||
group, row, line = _scan_outcome(
|
||||
os.path.join(root, "a", "b.png"), [(root, "output", 0)], "skipped"
|
||||
)
|
||||
self.assertEqual(group, "known")
|
||||
self.assertEqual(row["detail"], SKIPPED_DETAIL)
|
||||
self.assertEqual(row["path"], "a/b.png")
|
||||
self.assertEqual(line, f"= output/a/b.png · {SKIPPED_DETAIL}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,8 +1,11 @@
|
||||
"""Tests for utils/video_metadata.py: prompt metadata from video containers.
|
||||
|
||||
ComfyUI's video savers store ``{"prompt": ..., "workflow": ...}`` as JSON in
|
||||
the container's ``comment`` tag. ffprobe reads it; when ffprobe is missing
|
||||
videos are skipped without noise.
|
||||
Current ComfyUI savers (core SaveVideo/SaveWEBM, VideoHelperSuite) write one
|
||||
container tag per key (``prompt``, ``workflow``, ``parameters``,
|
||||
``comfyassets_prompt_pairs``); older VideoHelperSuite wrote one ``comment``
|
||||
JSON envelope, with ``description`` as a fallback. One selector picks a single
|
||||
carrier for the inspector and the scan. ffprobe reads the tags; when ffprobe
|
||||
is missing videos are skipped without noise.
|
||||
"""
|
||||
|
||||
import json
|
||||
@@ -19,34 +22,44 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from utils import video_metadata # noqa: E402
|
||||
from utils.video_metadata import ( # noqa: E402
|
||||
VIDEO_EXTENSIONS,
|
||||
DuplicateVideoTagError,
|
||||
is_video_path,
|
||||
metadata_from_tags,
|
||||
read_video_metadata,
|
||||
select_video_carrier,
|
||||
)
|
||||
|
||||
HAVE_FFMPEG = bool(shutil.which("ffmpeg") and shutil.which("ffprobe"))
|
||||
GRAPH = json.dumps({"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "a cat"}}})
|
||||
|
||||
|
||||
def write_video(path, metadata_tags=True, **tags):
|
||||
"""Write a tiny black video carrying one container tag per keyword.
|
||||
|
||||
MP4 keeps arbitrary keys only with ``use_metadata_tags`` (mdta atoms), the
|
||||
movflag the core SaveVideo node sets; without it ffmpeg writes the legacy
|
||||
ilst atoms (``comment`` becomes ``©cmt``) that old VideoHelperSuite files
|
||||
carry. Matroska and WebM upper-case the key names.
|
||||
"""
|
||||
command = [
|
||||
"ffmpeg",
|
||||
"-v",
|
||||
"quiet",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"color=c=black:s=16x16:d=0.2",
|
||||
]
|
||||
if metadata_tags and str(path).lower().endswith((".mp4", ".mov", ".m4v")):
|
||||
command += ["-movflags", "use_metadata_tags"]
|
||||
for key, value in tags.items():
|
||||
command += ["-metadata", f"{key}={value}"]
|
||||
subprocess.run(command + ["-y", str(path)], check=True, timeout=60)
|
||||
|
||||
|
||||
def write_mp4(path, comment):
|
||||
"""Write a tiny black video carrying ``comment`` in its container tags."""
|
||||
subprocess.run(
|
||||
[
|
||||
"ffmpeg",
|
||||
"-v",
|
||||
"quiet",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"color=c=black:s=16x16:d=0.2",
|
||||
"-metadata",
|
||||
f"comment={comment}",
|
||||
"-y",
|
||||
str(path),
|
||||
],
|
||||
check=True,
|
||||
timeout=60,
|
||||
)
|
||||
"""Write a tiny black MP4 with ``comment`` in its legacy ilst ``©cmt`` atom."""
|
||||
write_video(path, metadata_tags=False, comment=comment)
|
||||
|
||||
|
||||
class TestMetadataFromTags(unittest.TestCase):
|
||||
@@ -69,6 +82,158 @@ class TestMetadataFromTags(unittest.TestCase):
|
||||
self.assertEqual(metadata_from_tags(tags), {"prompt": GRAPH})
|
||||
|
||||
|
||||
class TestSelectVideoCarrier(unittest.TestCase):
|
||||
"""Spec 2026-10-19 resolution parity section 8: one carrier, never merged."""
|
||||
|
||||
WORKFLOW = json.dumps({"nodes": [{"id": "per-key"}]})
|
||||
|
||||
def test_per_key_tags_win_over_comment(self):
|
||||
tags = {
|
||||
"prompt": GRAPH,
|
||||
"workflow": self.WORKFLOW,
|
||||
"comment": json.dumps({"prompt": json.dumps({"9": {}})}),
|
||||
}
|
||||
metadata, formats, carrier = select_video_carrier(tags)
|
||||
self.assertEqual(metadata, {"prompt": GRAPH, "workflow": self.WORKFLOW})
|
||||
self.assertEqual(
|
||||
formats, {"prompt": "original-text", "workflow": "original-text"}
|
||||
)
|
||||
self.assertIsNone(carrier)
|
||||
|
||||
def test_matroska_uppercase_keys_fold(self):
|
||||
tags = {"PROMPT": GRAPH, "Workflow": self.WORKFLOW, "ENCODER": "Lavf"}
|
||||
metadata, formats, carrier = select_video_carrier(tags)
|
||||
self.assertEqual(metadata, {"prompt": GRAPH, "workflow": self.WORKFLOW})
|
||||
self.assertEqual(
|
||||
formats, {"prompt": "original-text", "workflow": "original-text"}
|
||||
)
|
||||
self.assertIsNone(carrier)
|
||||
# Envelope keys are read exactly as legacy did: no fold, so a Matroska
|
||||
# upper-cased COMMENT is not an envelope.
|
||||
self.assertIsNone(
|
||||
select_video_carrier({"COMMENT": json.dumps({"prompt": GRAPH})})
|
||||
)
|
||||
|
||||
def test_folded_duplicate_keys_fail(self):
|
||||
for tags in (
|
||||
{"PROMPT": GRAPH, "prompt": json.dumps({"9": {}})},
|
||||
{"Workflow": self.WORKFLOW, "WORKFLOW": "{}", "prompt": GRAPH},
|
||||
):
|
||||
with self.assertRaises(DuplicateVideoTagError):
|
||||
select_video_carrier(tags)
|
||||
# Byte-identical duplicates are one record, as for PNG text chunks.
|
||||
metadata, _, _ = select_video_carrier({"PROMPT": GRAPH, "prompt": GRAPH})
|
||||
self.assertEqual(metadata, {"prompt": GRAPH})
|
||||
# Envelope keys never fold, so a foreign COMMENT note is not a duplicate.
|
||||
envelope = json.dumps({"prompt": GRAPH})
|
||||
metadata, _, carrier = select_video_carrier(
|
||||
{"comment": envelope, "COMMENT": "a free-text note"}
|
||||
)
|
||||
self.assertEqual(metadata, {"prompt": GRAPH})
|
||||
self.assertEqual(carrier, envelope)
|
||||
|
||||
def test_comment_envelope_still_read(self):
|
||||
envelope = json.dumps({"prompt": GRAPH, "workflow": {"nodes": []}})
|
||||
metadata, formats, carrier = select_video_carrier(
|
||||
{"comment": envelope, "encoder": "Lavf"}
|
||||
)
|
||||
self.assertEqual(metadata["prompt"], GRAPH)
|
||||
self.assertEqual(json.loads(metadata["workflow"]), {"nodes": []})
|
||||
self.assertEqual(
|
||||
formats, {"prompt": "original-text", "workflow": "reserialized-json"}
|
||||
)
|
||||
self.assertEqual(carrier, envelope)
|
||||
|
||||
def test_description_fallback(self):
|
||||
envelope = json.dumps({"prompt": GRAPH})
|
||||
tags = {"comment": "not json", "description": envelope}
|
||||
self.assertEqual(
|
||||
select_video_carrier(tags),
|
||||
({"prompt": GRAPH}, {"prompt": "original-text"}, envelope),
|
||||
)
|
||||
self.assertIsNone(select_video_carrier({"comment": "note", "encoder": "x"}))
|
||||
self.assertIsNone(select_video_carrier({"comment": "[1]"}))
|
||||
self.assertIsNone(select_video_carrier({}))
|
||||
self.assertIsNone(select_video_carrier(None))
|
||||
self.assertIsNone(select_video_carrier({"comment": 7}))
|
||||
|
||||
def test_role_carrier_beats_workflow_only_carrier(self):
|
||||
envelope = json.dumps({"prompt": GRAPH, "workflow": {"nodes": ["c"]}})
|
||||
metadata, formats, carrier = select_video_carrier(
|
||||
{"workflow": self.WORKFLOW, "comment": envelope}
|
||||
)
|
||||
self.assertEqual(metadata["prompt"], GRAPH)
|
||||
self.assertEqual(json.loads(metadata["workflow"]), {"nodes": ["c"]})
|
||||
self.assertEqual(carrier, envelope)
|
||||
for role in ("parameters", "comfyassets_prompt_pairs"):
|
||||
metadata, _, carrier = select_video_carrier(
|
||||
{"comment": json.dumps({"workflow": {}}), role: "x"}
|
||||
)
|
||||
self.assertEqual(metadata, {role: "x"})
|
||||
self.assertIsNone(carrier)
|
||||
# Without a role anywhere, the first workflow-only carrier wins.
|
||||
metadata, _, carrier = select_video_carrier(
|
||||
{"workflow": self.WORKFLOW, "comment": json.dumps({"workflow": {}})}
|
||||
)
|
||||
self.assertEqual(metadata, {"workflow": self.WORKFLOW})
|
||||
self.assertIsNone(carrier)
|
||||
|
||||
def test_carriers_never_merge(self):
|
||||
metadata, _, carrier = select_video_carrier(
|
||||
{
|
||||
"prompt": GRAPH,
|
||||
"comment": json.dumps({"workflow": {"nodes": ["comment"]}}),
|
||||
"description": json.dumps({"parameters": "Steps: 1"}),
|
||||
}
|
||||
)
|
||||
self.assertEqual(metadata, {"prompt": GRAPH})
|
||||
self.assertIsNone(carrier)
|
||||
metadata, _, carrier = select_video_carrier(
|
||||
{
|
||||
"workflow": self.WORKFLOW,
|
||||
"comment": json.dumps({"prompt": GRAPH}),
|
||||
"description": json.dumps({"parameters": "Steps: 1"}),
|
||||
}
|
||||
)
|
||||
self.assertEqual(metadata, {"prompt": GRAPH})
|
||||
self.assertEqual(carrier, json.dumps({"prompt": GRAPH}))
|
||||
|
||||
def test_per_key_values_are_original_text_and_envelope_objects_reserialized(
|
||||
self,
|
||||
):
|
||||
exact = '{"3":{"inputs":{"cfg":6.5000000000000000001}}}'
|
||||
metadata, formats, _ = select_video_carrier({"prompt": exact})
|
||||
self.assertEqual(metadata, {"prompt": exact})
|
||||
self.assertEqual(formats, {"prompt": "original-text"})
|
||||
envelope = json.dumps(
|
||||
{"prompt": {"3": {}}, "workflow": {"nodes": []}, "parameters": "Steps: 1"}
|
||||
)
|
||||
metadata, formats, carrier = select_video_carrier({"comment": envelope})
|
||||
self.assertEqual(json.loads(metadata["prompt"]), {"3": {}})
|
||||
self.assertEqual(json.loads(metadata["workflow"]), {"nodes": []})
|
||||
self.assertEqual(metadata["parameters"], "Steps: 1")
|
||||
self.assertEqual(
|
||||
formats,
|
||||
{
|
||||
"prompt": "reserialized-json",
|
||||
"workflow": "reserialized-json",
|
||||
"parameters": "original-text",
|
||||
},
|
||||
)
|
||||
self.assertEqual(carrier, envelope)
|
||||
|
||||
def test_null_role_markers_and_invalid_workflow_keep_their_shape(self):
|
||||
# The inspector refuses a null role marker rather than falling back.
|
||||
metadata, formats, _ = select_video_carrier(
|
||||
{"comment": json.dumps({"prompt": None, "workflow": [1]})}
|
||||
)
|
||||
self.assertEqual(metadata, {"prompt": None})
|
||||
self.assertEqual(formats, {"workflow": "invalid"})
|
||||
metadata, formats, _ = select_video_carrier({"prompt": 7, "workflow": 1})
|
||||
self.assertEqual(metadata, {"prompt": None})
|
||||
self.assertEqual(formats, {"workflow": "invalid"})
|
||||
|
||||
|
||||
class TestIsVideoPath(unittest.TestCase):
|
||||
|
||||
def test_extensions(self):
|
||||
@@ -125,6 +290,37 @@ class TestReadVideoMetadataWithoutFfprobe(unittest.TestCase):
|
||||
):
|
||||
self.assertEqual(read_video_metadata("clip.mp4"), {})
|
||||
|
||||
def test_read_video_metadata_returns_per_key_prompt(self):
|
||||
payload = {
|
||||
"format": {
|
||||
"tags": {
|
||||
"PROMPT": GRAPH,
|
||||
"WORKFLOW": '{"nodes":[]}',
|
||||
"COMFYASSETS_PROMPT_PAIRS": '{"version":1}',
|
||||
"COMMENT": json.dumps({"prompt": "must not win"}),
|
||||
"ENCODER": "Lavf",
|
||||
}
|
||||
}
|
||||
}
|
||||
with mock.patch.object(
|
||||
video_metadata.shutil, "which", return_value="/bin/ffprobe"
|
||||
):
|
||||
with self._run_returning(stdout=json.dumps(payload)):
|
||||
self.assertEqual(
|
||||
read_video_metadata("clip.mkv"),
|
||||
{
|
||||
"prompt": GRAPH,
|
||||
"workflow": '{"nodes":[]}',
|
||||
"comfyassets_prompt_pairs": '{"version":1}',
|
||||
},
|
||||
)
|
||||
duplicates = {"format": {"tags": {"PROMPT": GRAPH, "prompt": "{}"}}}
|
||||
with self._run_returning(stdout=json.dumps(duplicates)):
|
||||
self.assertEqual(read_video_metadata("clip.mkv"), {})
|
||||
markers = {"format": {"tags": {"comment": json.dumps({"prompt": None})}}}
|
||||
with self._run_returning(stdout=json.dumps(markers)):
|
||||
self.assertEqual(read_video_metadata("clip.mp4"), {})
|
||||
|
||||
def test_ffprobe_output_is_parsed(self):
|
||||
payload = {"format": {"tags": {"comment": json.dumps({"prompt": GRAPH})}}}
|
||||
with mock.patch.object(
|
||||
@@ -155,6 +351,39 @@ class TestReadVideoMetadataWithRealFfprobe(unittest.TestCase):
|
||||
self.assertEqual(meta["prompt"], GRAPH)
|
||||
self.assertEqual(json.loads(meta["workflow"]), {"v": 1})
|
||||
|
||||
def test_per_key_tags_round_trip_through_real_containers(self):
|
||||
for name in ("per_key.mp4", "per_key.mkv", "per_key.webm"):
|
||||
path = os.path.join(self.tmpdir, name)
|
||||
write_video(path, prompt=GRAPH, workflow='{"nodes":[]}')
|
||||
self.assertEqual(
|
||||
read_video_metadata(path),
|
||||
{"prompt": GRAPH, "workflow": '{"nodes":[]}'},
|
||||
name,
|
||||
)
|
||||
|
||||
def test_legacy_ilst_comment_atom_is_read(self):
|
||||
path = os.path.join(self.tmpdir, "vhs.mp4")
|
||||
write_mp4(path, json.dumps({"prompt": GRAPH}))
|
||||
probe = subprocess.run(
|
||||
[
|
||||
shutil.which("ffprobe"),
|
||||
"-v",
|
||||
"quiet",
|
||||
"-show_entries",
|
||||
"format_tags",
|
||||
"-of",
|
||||
"json",
|
||||
path,
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=60,
|
||||
)
|
||||
tags = json.loads(probe.stdout)["format"]["tags"]
|
||||
self.assertIn("comment", tags)
|
||||
self.assertEqual(read_video_metadata(path), {"prompt": GRAPH})
|
||||
|
||||
def test_video_without_a_comment_is_empty(self):
|
||||
path = os.path.join(self.tmpdir, "plain.mp4")
|
||||
write_mp4(path, "")
|
||||
|
||||
+124
-4
@@ -1,5 +1,7 @@
|
||||
"""Bounded vocabulary reads count exactly what Library Browse shows."""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import tempfile
|
||||
@@ -61,7 +63,48 @@ class TestVocabularyReads(unittest.TestCase):
|
||||
by_name = {row["name"]: row for row in page["tags"]}
|
||||
self.assertEqual(by_name["z"]["prompt_count"], 0)
|
||||
self.assertEqual(by_name["a"]["prompt_count"], 2)
|
||||
self.assertEqual(set(by_name["c"]), {"id", "name", "prompt_count"})
|
||||
self.assertEqual(
|
||||
set(by_name["c"]), {"id", "name", "prompt_count", "image_count"}
|
||||
)
|
||||
self.assertEqual({row["image_count"] for row in page["tags"]}, {0})
|
||||
|
||||
def image(self, name, tags, digest="a", mtime_ns=1):
|
||||
"""Tag the bytes ``digest`` of the file ``name``; return its locator."""
|
||||
locator = os.path.normcase(os.path.join(os.path.realpath(self.temp.name), name))
|
||||
self.read(
|
||||
"replace_image_tags",
|
||||
locator=locator,
|
||||
sha256=hashlib.sha256(digest.encode()).hexdigest(),
|
||||
size=1,
|
||||
mtime_ns=mtime_ns,
|
||||
expected_epoch=self.db.get_execution_epoch(),
|
||||
expected_tag_revision=0,
|
||||
tags=tags,
|
||||
)
|
||||
return locator
|
||||
|
||||
def test_tag_rows_report_image_count_by_locator(self):
|
||||
self.save("Prompt only", tags=["prompt-only"])
|
||||
self.image("one.png", ["twice", "shared"], digest="old")
|
||||
self.image(
|
||||
"one.png", ["twice"], digest="new", mtime_ns=2
|
||||
) # two bytes, one file
|
||||
self.image("two.png", ["shared"], digest="new") # equal digest, other locator
|
||||
self.image("three.png", ["prior-only"], digest="first")
|
||||
self.image("three.png", [], digest="second", mtime_ns=2)
|
||||
rows = {row["name"]: row for row in self.all_tags()}
|
||||
self.assertEqual(rows["twice"]["image_count"], 1)
|
||||
row = rows["shared"]
|
||||
self.assertEqual(row["image_count"], 2)
|
||||
self.assertEqual(rows["prior-only"]["image_count"], 1)
|
||||
self.assertEqual(rows["prompt-only"]["image_count"], 0)
|
||||
self.assertEqual(
|
||||
{name: r["prompt_count"] for name, r in rows.items()},
|
||||
{"prompt-only": 1, "twice": 0, "shared": 0, "prior-only": 0},
|
||||
)
|
||||
self.assertEqual(
|
||||
self.db.list_library(tags=["shared"])["total"], row["prompt_count"]
|
||||
)
|
||||
|
||||
def test_prompt_count_equals_library_browse_total_for_every_tag(self):
|
||||
self.save("Parent only", tags=["parent-only"])
|
||||
@@ -302,7 +345,12 @@ class TestVocabularyTagCommands(unittest.TestCase):
|
||||
self.assertTrue(created["created"])
|
||||
self.assertEqual(
|
||||
created["tag"],
|
||||
{"id": self.tag_id("sunset"), "name": "sunset", "prompt_count": 0},
|
||||
{
|
||||
"id": self.tag_id("sunset"),
|
||||
"name": "sunset",
|
||||
"prompt_count": 0,
|
||||
"image_count": 0,
|
||||
},
|
||||
)
|
||||
after = self.junction_rows()
|
||||
self.assertEqual(after[1:], before[1:])
|
||||
@@ -345,9 +393,15 @@ class TestVocabularyTagCommands(unittest.TestCase):
|
||||
{
|
||||
"library_epoch": self.epoch,
|
||||
"merged": False,
|
||||
"tag": {"id": tag_id, "name": "dramatic", "prompt_count": 2},
|
||||
"tag": {
|
||||
"id": tag_id,
|
||||
"name": "dramatic",
|
||||
"prompt_count": 2,
|
||||
"image_count": 0,
|
||||
},
|
||||
"components_changed": 2,
|
||||
"pairs_changed": 2,
|
||||
"images_changed": 0,
|
||||
},
|
||||
)
|
||||
self.assertIsNone(self.tag_id("moody"))
|
||||
@@ -397,7 +451,8 @@ class TestVocabularyTagCommands(unittest.TestCase):
|
||||
)
|
||||
self.assertFalse(result["merged"])
|
||||
self.assertEqual(
|
||||
result["tag"], {"id": tag_id, "name": "cat", "prompt_count": 1}
|
||||
result["tag"],
|
||||
{"id": tag_id, "name": "cat", "prompt_count": 1, "image_count": 0},
|
||||
)
|
||||
self.assertEqual(self.db.get_prompt_by_id(prompt)["tags"], ["cat"])
|
||||
|
||||
@@ -467,6 +522,7 @@ class TestVocabularyTagCommands(unittest.TestCase):
|
||||
"other_prompts": 1,
|
||||
"negative_variants": 1,
|
||||
"archived_negative_variants": 1,
|
||||
"images": 0,
|
||||
},
|
||||
},
|
||||
)
|
||||
@@ -482,6 +538,7 @@ class TestVocabularyTagCommands(unittest.TestCase):
|
||||
"deleted": True,
|
||||
"components_changed": 2,
|
||||
"pairs_changed": 2,
|
||||
"images_changed": 0,
|
||||
},
|
||||
)
|
||||
self.assertIsNone(self.tag_id("moody"))
|
||||
@@ -494,6 +551,69 @@ class TestVocabularyTagCommands(unittest.TestCase):
|
||||
)
|
||||
self.assertEqual(caught.exception.code, "vocabulary-conflict")
|
||||
|
||||
def image(self, name, tags, digest="a", mtime_ns=1):
|
||||
locator = os.path.normcase(os.path.join(os.path.realpath(self.temp.name), name))
|
||||
self.command(
|
||||
"replace_image_tags",
|
||||
locator=locator,
|
||||
sha256=hashlib.sha256(digest.encode()).hexdigest(),
|
||||
size=1,
|
||||
mtime_ns=mtime_ns,
|
||||
expected_epoch=self.epoch,
|
||||
expected_tag_revision=0,
|
||||
tags=tags,
|
||||
)
|
||||
return locator
|
||||
|
||||
def test_impact_and_delete_counts_include_images(self):
|
||||
self.owners()
|
||||
self.image("a.png", ["moody", "keep"])
|
||||
self.image("a.png", ["moody"], digest="prior", mtime_ns=2)
|
||||
self.image("b.png", ["moody"])
|
||||
tag_id = self.tag_id("moody")
|
||||
impact = self.command("vocabulary_tag_impact", tag_id)["impact"]
|
||||
self.assertEqual(impact["images"], 2)
|
||||
self.assertEqual(
|
||||
self.command("vocabulary_tag_impact", self.tag_id("keep"))["impact"],
|
||||
{
|
||||
"positive_prompts": 0,
|
||||
"other_prompts": 0,
|
||||
"negative_variants": 0,
|
||||
"archived_negative_variants": 0,
|
||||
"images": 1,
|
||||
},
|
||||
)
|
||||
result = self.command(
|
||||
"delete_vocabulary_tag", tag_id, expected_epoch=self.epoch, name="moody"
|
||||
)
|
||||
self.assertEqual(
|
||||
(
|
||||
result["components_changed"],
|
||||
result["pairs_changed"],
|
||||
result["images_changed"],
|
||||
),
|
||||
(2, 2, 2),
|
||||
)
|
||||
with self.db.model.get_connection() as conn:
|
||||
self.assertEqual(
|
||||
conn.execute("SELECT COUNT(*) FROM media_image_tags").fetchone()[0], 1
|
||||
)
|
||||
self.assertEqual(
|
||||
conn.execute("SELECT COUNT(*) FROM media_annotation_bytes").fetchone()[
|
||||
0
|
||||
],
|
||||
3,
|
||||
)
|
||||
renamed = self.command(
|
||||
"rename_vocabulary_tag",
|
||||
self.tag_id("keep"),
|
||||
expected_epoch=self.epoch,
|
||||
name="keep",
|
||||
new_name="kept",
|
||||
)
|
||||
self.assertEqual(renamed["images_changed"], 1)
|
||||
self.assertEqual(renamed["tag"]["image_count"], 1)
|
||||
|
||||
|
||||
_OLD_STAMP = "2000-01-01T00:00:00+00:00"
|
||||
_DML = re.compile(r"\s*(INSERT|UPDATE|DELETE|REPLACE)\b", re.IGNORECASE)
|
||||
|
||||
+234
@@ -0,0 +1,234 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createServer } from 'node:net';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { once } from 'node:events';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
import { startBrowserServer, createServerSupervisor, attachServerStderr } from '../../tools/vendor/start-browser-server.mjs';
|
||||
import { workflowWorkers } from '../../tools/vendor/test-workers.mjs';
|
||||
|
||||
const fakeServer = fileURLToPath(new URL('./fake-browser-server.mjs', import.meta.url));
|
||||
const fake = (...extra) => ({ command: process.execPath, args: port => [fakeServer, '--port', String(port), ...extra] });
|
||||
const REAL_SERVER_TIMEOUT_MS = 90_000;
|
||||
const launcher = fileURLToPath(new URL('../../tools/vendor/start-browser-server.mjs', import.meta.url));
|
||||
|
||||
/** Polls `check` until it is truthy, failing with `what` after `ms`. */
|
||||
async function waitFor(check, what, ms = 5_000) {
|
||||
const deadline = Date.now() + ms;
|
||||
while (!check()) {
|
||||
if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}`);
|
||||
await delay(20);
|
||||
}
|
||||
}
|
||||
|
||||
async function writeStderr(server, text) {
|
||||
await (await fetch(`${server.url}/__stderr?text=${encodeURIComponent(text)}`)).text();
|
||||
await waitFor(() => server.stderrTail().endsWith(text.trimEnd()), `stderr ${JSON.stringify(text)}`);
|
||||
}
|
||||
|
||||
/** A process that exited counts as gone even while a non-reaping PID 1 (CI's container) leaves it a zombie. */
|
||||
function alive(pid) {
|
||||
try { process.kill(pid, 0); } catch { return false; }
|
||||
try { return readFileSync(`/proc/${pid}/stat`, 'utf8').replace(/^.*\) /s, '')[0] !== 'Z'; } catch { return true; }
|
||||
}
|
||||
|
||||
/** Starts a server from a separate Node worker process, SIGKILLs that worker and returns the server's pid. */
|
||||
async function serverOfKilledWorker(options) {
|
||||
const script = `import { startBrowserServer } from ${JSON.stringify(pathToFileURL(launcher).href)};
|
||||
const server = await startBrowserServer(${options});
|
||||
console.log('SERVER_PID=' + server.pid);
|
||||
setInterval(() => {}, 1000);`;
|
||||
const worker = spawn(process.execPath, ['--input-type=module', '-e', script], { stdio: ['ignore', 'pipe', 'inherit'] });
|
||||
let out = '';
|
||||
worker.stdout.setEncoding('utf8');
|
||||
worker.stdout.on('data', chunk => { out += chunk; });
|
||||
await waitFor(() => /SERVER_PID=\d+\n/.test(out) || worker.exitCode !== null, 'the worker to start its server', REAL_SERVER_TIMEOUT_MS);
|
||||
const pid = Number(/SERVER_PID=(\d+)/.exec(out)?.[1]);
|
||||
assert.ok(pid > 0, `the worker started a server (output: ${out})`);
|
||||
assert.ok(alive(pid), 'the server runs while its worker lives');
|
||||
worker.kill('SIGKILL');
|
||||
await once(worker, 'exit');
|
||||
return pid;
|
||||
}
|
||||
|
||||
function listen(port = 0) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const server = createServer();
|
||||
server.once('error', reject);
|
||||
server.listen(port, '127.0.0.1', () => resolve(server));
|
||||
});
|
||||
}
|
||||
|
||||
async function releasedPort() {
|
||||
const probe = await listen();
|
||||
const { port } = probe.address();
|
||||
await new Promise(resolve => probe.close(resolve));
|
||||
return port;
|
||||
}
|
||||
|
||||
test('free port allocation never reuses a bound port', async t => {
|
||||
const held = await listen();
|
||||
t.after(() => held.close());
|
||||
const heldPort = held.address().port;
|
||||
const servers = await Promise.all(Array.from({ length: 6 }, () => startBrowserServer(fake())));
|
||||
t.after(() => Promise.all(servers.map(server => server.stop())));
|
||||
const ports = servers.map(server => server.port);
|
||||
for (const server of servers) {
|
||||
assert.ok(Number.isInteger(server.port) && server.port > 0 && server.port < 65536, `port ${server.port} is a TCP port`);
|
||||
assert.equal(server.url, `http://127.0.0.1:${server.port}`);
|
||||
assert.equal(await (await fetch(`${server.url}/prompt_manager/test`)).text(), 'ready');
|
||||
}
|
||||
assert.ok(!ports.includes(heldPort), 'a port another process holds is never handed out');
|
||||
assert.equal(new Set(ports).size, ports.length, 'concurrent servers never share a port');
|
||||
});
|
||||
|
||||
test('startBrowserServer asks the server to bind port 0 and reads the port it announces', async () => {
|
||||
const asked = [];
|
||||
const server = await startBrowserServer({ command: process.execPath, args: port => { asked.push(port); return fake().args(port); } });
|
||||
try {
|
||||
assert.deepEqual(asked, [0], 'no port is reserved and released before the server binds');
|
||||
assert.ok(server.port > 0);
|
||||
} finally {
|
||||
await server.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test('startBrowserServer serves the readiness page on a requested port and stops cleanly', async () => {
|
||||
const port = await releasedPort();
|
||||
const server = await startBrowserServer({ port, ...fake() });
|
||||
assert.equal(server.port, port);
|
||||
assert.equal(server.url, `http://127.0.0.1:${port}`);
|
||||
assert.equal(await (await fetch(`${server.url}/prompt_manager/test`)).text(), 'ready');
|
||||
await server.stop();
|
||||
await assert.rejects(fetch(`${server.url}/prompt_manager/test`));
|
||||
});
|
||||
|
||||
test('startBrowserServer reports a startup failure with the server\'s stderr tail', async () => {
|
||||
await assert.rejects(startBrowserServer(fake('--fail')), error => {
|
||||
assert.match(error.message, /exited with code 4 before it was ready/);
|
||||
assert.match(error.message, /last fake stderr line/);
|
||||
return true;
|
||||
});
|
||||
});
|
||||
|
||||
test('startBrowserServer gives up on a server that never announces a port or becomes ready', async () => {
|
||||
const silent = { command: process.execPath, args: () => ['-e', 'setInterval(() => {}, 1000)'] };
|
||||
await assert.rejects(startBrowserServer({ ...silent, timeoutMs: 500 }), /not ready .* within 500 ms/);
|
||||
});
|
||||
|
||||
test('worker server restarts after an unexpected exit', async t => {
|
||||
const supervisor = createServerSupervisor({ start: () => startBrowserServer(fake()) });
|
||||
t.after(() => supervisor.stop());
|
||||
const first = await supervisor.ensure();
|
||||
assert.equal(supervisor.crash(first), null, 'a running server reports no crash');
|
||||
await fetch(`${first.url}/__exit`);
|
||||
await first.exited;
|
||||
const crash = supervisor.crash(first);
|
||||
assert.match(crash.message, /test server on port \d+ exited unexpectedly \(code 3\)/);
|
||||
assert.match(crash.message, /crashed on purpose/);
|
||||
const second = await supervisor.ensure();
|
||||
assert.notEqual(second, first, 'the next test gets a fresh server');
|
||||
assert.equal(await (await fetch(`${second.url}/prompt_manager/test`)).text(), 'ready');
|
||||
assert.equal(await supervisor.ensure(), second, 'a healthy server is reused');
|
||||
});
|
||||
|
||||
test('supervisor reports a startup failure once and tries again on the next ensure', async t => {
|
||||
let attempts = 0;
|
||||
const supervisor = createServerSupervisor({
|
||||
start: async () => {
|
||||
attempts += 1;
|
||||
if (attempts === 1) throw new Error('ImportError: no module');
|
||||
return startBrowserServer(fake());
|
||||
},
|
||||
});
|
||||
t.after(() => supervisor.stop());
|
||||
await assert.rejects(supervisor.ensure(), /ImportError/);
|
||||
assert.equal(attempts, 1, 'a failed startup is not retried behind the caller\'s back');
|
||||
const server = await supervisor.ensure();
|
||||
assert.equal(attempts, 2);
|
||||
assert.equal(await (await fetch(`${server.url}/prompt_manager/test`)).text(), 'ready');
|
||||
});
|
||||
|
||||
test('the port announcement counts only once its line is complete', async t => {
|
||||
const server = await startBrowserServer({ ...fake('--split-announce'), timeoutMs: 5_000 });
|
||||
t.after(() => server.stop());
|
||||
assert.equal(await (await fetch(`${server.url}/prompt_manager/test`)).text(), 'ready');
|
||||
assert.equal(server.url, `http://127.0.0.1:${server.port}`);
|
||||
});
|
||||
|
||||
test('server stderr since a mark holds only what came after it, bounded to the newest output', async t => {
|
||||
const server = await startBrowserServer({ ...fake(), stderrLogChars: 64 });
|
||||
t.after(() => server.stop());
|
||||
await writeStderr(server, 'earlier test\n');
|
||||
const mark = server.stderrMark();
|
||||
assert.equal(server.stderrSince(mark), '');
|
||||
await writeStderr(server, 'Traceback: this test\n');
|
||||
assert.equal(server.stderrSince(mark), 'Traceback: this test\n');
|
||||
const flood = `${'x'.repeat(80)}newest\n`;
|
||||
await writeStderr(server, flood);
|
||||
const kept = server.stderrSince(mark);
|
||||
assert.match(kept, /^\[earlier server stderr dropped\]\n/);
|
||||
assert.ok(kept.endsWith('newest\n'), 'the newest output is kept');
|
||||
assert.ok(kept.length <= 64 + '[earlier server stderr dropped]\n'.length, `bounded, got ${kept.length} chars`);
|
||||
});
|
||||
|
||||
test('a failed test gets the server stderr written during it attached; a passing one does not', async t => {
|
||||
const server = await startBrowserServer(fake());
|
||||
t.after(() => server.stop());
|
||||
const attached = [];
|
||||
const info = status => ({ status, expectedStatus: 'passed', attach: async (name, options) => { attached.push({ name, ...options }); } });
|
||||
await writeStderr(server, 'before the test\n');
|
||||
const mark = server.stderrMark();
|
||||
await attachServerStderr(info('failed'), server, mark);
|
||||
assert.deepEqual(attached, [], 'nothing to attach when the server wrote nothing');
|
||||
await writeStderr(server, 'Traceback (most recent call last): boom\n');
|
||||
await attachServerStderr(info('passed'), server, mark);
|
||||
assert.deepEqual(attached, [], 'a passing test attaches nothing');
|
||||
await attachServerStderr(info('timedOut'), server, mark);
|
||||
assert.equal(attached.length, 1);
|
||||
assert.equal(attached[0].name, 'server-stderr');
|
||||
assert.equal(attached[0].contentType, 'text/plain');
|
||||
assert.equal(String(attached[0].body), 'Traceback (most recent call last): boom\n');
|
||||
});
|
||||
|
||||
test('a server exits with its worker even when the worker is SIGKILLed', async () => {
|
||||
const pid = await serverOfKilledWorker(`{ command: process.execPath, args: port => [${JSON.stringify(fakeServer)}, '--port', String(port), '--exit-on-stdin-eof'] }`);
|
||||
await waitFor(() => !alive(pid), `server ${pid} to exit after its worker died`);
|
||||
});
|
||||
|
||||
test('tests/browser/server.py exits with a SIGKILLed worker', { timeout: 2 * REAL_SERVER_TIMEOUT_MS }, async () => {
|
||||
const pid = await serverOfKilledWorker(`{ timeoutMs: ${REAL_SERVER_TIMEOUT_MS} }`);
|
||||
await waitFor(() => !alive(pid), `server.py ${pid} to exit after its worker died`, 15_000);
|
||||
});
|
||||
|
||||
test('tests/browser/server.py binds an OS-chosen port for --port 0 and still honours --port N', { timeout: 2 * REAL_SERVER_TIMEOUT_MS }, async t => {
|
||||
const held = await listen();
|
||||
t.after(() => held.close());
|
||||
const chosen = await startBrowserServer({ timeoutMs: REAL_SERVER_TIMEOUT_MS });
|
||||
try {
|
||||
assert.notEqual(chosen.port, held.address().port);
|
||||
assert.equal((await fetch(`${chosen.url}/prompt_manager/test`)).status, 200);
|
||||
} finally {
|
||||
await chosen.stop();
|
||||
}
|
||||
const port = await releasedPort();
|
||||
const fixed = await startBrowserServer({ port, timeoutMs: REAL_SERVER_TIMEOUT_MS });
|
||||
try {
|
||||
assert.equal(fixed.port, port);
|
||||
assert.equal((await fetch(`http://127.0.0.1:${port}/prompt_manager/test`)).status, 200);
|
||||
} finally {
|
||||
await fixed.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test('worker count follows PM_TEST_WORKERS, then one in CI, then a quarter of the CPUs up to eight', () => {
|
||||
assert.equal(workflowWorkers({ env: { PM_TEST_WORKERS: '3', CI: '1' }, cpus: 32 }), 3);
|
||||
assert.equal(workflowWorkers({ env: { CI: '1' }, cpus: 32 }), 1);
|
||||
assert.equal(workflowWorkers({ env: {}, cpus: 32 }), 8);
|
||||
assert.equal(workflowWorkers({ env: {}, cpus: 12 }), 3);
|
||||
assert.equal(workflowWorkers({ env: {}, cpus: 2 }), 1);
|
||||
assert.throws(() => workflowWorkers({ env: { PM_TEST_WORKERS: 'many' }, cpus: 8 }), /PM_TEST_WORKERS must be a positive integer/);
|
||||
assert.throws(() => workflowWorkers({ env: { PM_TEST_WORKERS: '0' }, cpus: 8 }), /PM_TEST_WORKERS must be a positive integer/);
|
||||
});
|
||||
Vendored
+223
@@ -0,0 +1,223 @@
|
||||
// Structural checks on .github/workflows/viewerjs-check.yml. The tooling has no YAML parser, so these
|
||||
// read the workflow's fixed two-space layout: jobs at two spaces, their keys at four, steps at six.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { mkdtemp, mkdir, readdir, rm, writeFile } from 'node:fs/promises';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
const workflow = readFileSync(new URL('../../.github/workflows/viewerjs-check.yml', import.meta.url), 'utf8');
|
||||
const PINNED_IMAGE = /^\s+image: mcr\.microsoft\.com\/playwright:v\d+\.\d+\.\d+-noble$/m;
|
||||
|
||||
function jobs(text) {
|
||||
const lines = text.split('\n');
|
||||
const start = lines.indexOf('jobs:');
|
||||
assert.notEqual(start, -1, 'workflow has a jobs section');
|
||||
const found = new Map();
|
||||
let current = null;
|
||||
for (const line of lines.slice(start + 1)) {
|
||||
const header = /^ {2}([\w-]+):\s*$/.exec(line);
|
||||
if (header) {
|
||||
current = header[1];
|
||||
found.set(current, []);
|
||||
} else if (current && (line.startsWith(' ') || line.trim() === '' || /^ {2}#/.test(line))) {
|
||||
found.get(current).push(line);
|
||||
} else if (current && /^\S/.test(line)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return new Map([...found].map(([id, body]) => [id, body.join('\n')]));
|
||||
}
|
||||
|
||||
function steps(jobText) {
|
||||
const parts = jobText.split(/^(?= {6}- )/m);
|
||||
return parts.slice(1);
|
||||
}
|
||||
|
||||
function runScript(stepText) {
|
||||
const lines = stepText.split('\n');
|
||||
const index = lines.findIndex(line => /^ {8}run: \|\s*$/.test(line));
|
||||
assert.notEqual(index, -1, 'step has a block run script');
|
||||
const body = [];
|
||||
for (const line of lines.slice(index + 1)) {
|
||||
if (line.trim() !== '' && !line.startsWith(' ')) break;
|
||||
body.push(line.slice(10));
|
||||
}
|
||||
return body.join('\n');
|
||||
}
|
||||
|
||||
function needsOf(jobText) {
|
||||
const match = /^ {4}needs: \[([^\]]*)\]\s*$/m.exec(jobText) ?? /^ {4}needs: ([\w-]+)\s*$/m.exec(jobText);
|
||||
assert.ok(match, 'job declares needs');
|
||||
return match[1].split(',').map(value => value.trim()).filter(Boolean);
|
||||
}
|
||||
|
||||
function matrixList(jobText, key) {
|
||||
const match = new RegExp(`^ {8}${key}: \\[([^\\]]*)\\]\\s*$`, 'm').exec(jobText);
|
||||
assert.ok(match, `job has a ${key} matrix`);
|
||||
return match[1].split(',').map(value => value.trim());
|
||||
}
|
||||
|
||||
const allJobs = jobs(workflow);
|
||||
const browserJobIds = [...allJobs].filter(([, body]) => /npm run --prefix tools\/vendor test:(workflows|coverage)/.test(body))
|
||||
.map(([id]) => id);
|
||||
const mergeJob = () => allJobs.get('frontend-coverage-merge');
|
||||
const verifyStep = () => steps(mergeJob()).find(step => /name: Verify every shard's raw coverage arrived/.test(step));
|
||||
|
||||
test('viewerjs keeps its name and needs every frontend job', () => {
|
||||
const viewerjs = allJobs.get('viewerjs');
|
||||
assert.ok(viewerjs, 'the required context is the job id viewerjs');
|
||||
assert.doesNotMatch(viewerjs, /^ {4}name:/m, 'no display name may replace the context "viewerjs"');
|
||||
const needs = needsOf(viewerjs);
|
||||
const others = [...allJobs.keys()].filter(id => id !== 'viewerjs');
|
||||
assert.deepEqual([...needs].sort(), [...others].sort(), 'viewerjs needs every other job in the workflow');
|
||||
assert.deepEqual([...needs].sort(), ['browser-workflows', 'frontend-coverage', 'frontend-coverage-merge']);
|
||||
assert.match(viewerjs, /^ {4}if: always\(\)$/m, 'viewerjs runs (and fails) when a needed job failed or was skipped');
|
||||
const script = runScript(steps(viewerjs).find(step => /run: \|/.test(step)));
|
||||
for (const id of needs) {
|
||||
const variable = new RegExp(`^ {10}([A-Z_]+): \\$\\{\\{ needs\\.${id}\\.result \\}\\}$`, 'm').exec(viewerjs);
|
||||
assert.ok(variable, `${id}'s result reaches the check`);
|
||||
assert.match(script, new RegExp(`test "\\$${variable[1]}" = success`), `${id} must be exactly success`);
|
||||
}
|
||||
});
|
||||
|
||||
test('viewerjs fails unless every needed job succeeded', () => {
|
||||
const viewerjs = allJobs.get('viewerjs');
|
||||
const script = runScript(steps(viewerjs).find(step => /run: \|/.test(step)));
|
||||
const variables = [...viewerjs.matchAll(/^ {10}([A-Z_]+): \$\{\{ needs\.[\w-]+\.result \}\}$/gm)].map(match => match[1]);
|
||||
const run = env => {
|
||||
try {
|
||||
execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', script], { env: { PATH: process.env.PATH, ...env }, stdio: 'pipe' });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
const allSuccess = Object.fromEntries(variables.map(name => [name, 'success']));
|
||||
assert.equal(run(allSuccess), true);
|
||||
for (const name of variables) {
|
||||
for (const result of ['failure', 'cancelled', 'skipped']) {
|
||||
assert.equal(run({ ...allSuccess, [name]: result }), false, `${name}=${result} must fail viewerjs`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('coverage runs as three shards that each upload their raw coverage', () => {
|
||||
const coverage = allJobs.get('frontend-coverage');
|
||||
assert.deepEqual(matrixList(coverage, 'shard'), ['1', '2', '3']);
|
||||
assert.match(coverage, /^ {4}name: frontend coverage \(shard \$\{\{ matrix\.shard \}\}\/3\)$/m);
|
||||
assert.match(coverage, /npm run --prefix tools\/vendor test:coverage:shard -- \$\{\{ matrix\.shard \}\}\/3/);
|
||||
const upload = steps(coverage).find(step => /name: coverage-raw-\$\{\{ matrix\.shard \}\}/.test(step));
|
||||
assert.ok(upload, 'each shard uploads coverage-raw-N');
|
||||
assert.match(upload, /^ {10}path: coverage\/js\/raw\/?$/m);
|
||||
assert.match(upload, /^ {10}retention-days: 3$/m);
|
||||
assert.match(upload, /^ {10}if-no-files-found: error$/m);
|
||||
});
|
||||
|
||||
test('merge job downloads every shard and runs the merge-only report and check', () => {
|
||||
const merge = mergeJob();
|
||||
assert.ok(merge, 'frontend-coverage-merge exists');
|
||||
assert.deepEqual(needsOf(merge), ['frontend-coverage']);
|
||||
assert.match(merge, /^ {4}name: frontend coverage \(merge\)$/m);
|
||||
const download = steps(merge).find(step => /uses: actions\/download-artifact@/.test(step));
|
||||
assert.ok(download, 'merge downloads the shard artifacts');
|
||||
assert.match(download, /^ {10}pattern: coverage-raw-\*$/m);
|
||||
assert.doesNotMatch(download, /merge-multiple: true/, 'each artifact keeps its own directory so the check can count them');
|
||||
assert.ok(verifyStep(), 'merge has an explicit artifact count check');
|
||||
assert.match(verifyStep(), new RegExp(`^ {10}EXPECTED_SHARDS: ${matrixList(allJobs.get('frontend-coverage'), 'shard').length}$`, 'm'),
|
||||
'the expected count matches the shard matrix');
|
||||
const order = steps(merge).map(step => (/name: (.*)/.exec(step) ?? [, step.trim().split('\n')[0]])[1]);
|
||||
const verifyAt = order.findIndex(name => /Verify every shard/.test(name));
|
||||
const mergeAt = steps(merge).findIndex(step => /npm run --prefix tools\/vendor coverage:merge/.test(step));
|
||||
assert.ok(verifyAt >= 0 && mergeAt > verifyAt, 'the count check runs before coverage:merge');
|
||||
const report = steps(merge).find(step => /name: frontend-test-reports$/m.test(step));
|
||||
assert.ok(report, 'merge uploads the HTML report as frontend-test-reports');
|
||||
assert.match(report, /!coverage\/js\/raw\/\*\*/);
|
||||
});
|
||||
|
||||
async function stagedShards(root, shards) {
|
||||
const staged = join(root, 'staged');
|
||||
for (const [shard, files] of shards) {
|
||||
const directory = join(staged, `coverage-raw-${shard}`);
|
||||
await mkdir(directory, { recursive: true });
|
||||
for (const file of files) await writeFile(join(directory, file), '{"result":[]}');
|
||||
}
|
||||
await mkdir(staged, { recursive: true });
|
||||
return staged;
|
||||
}
|
||||
|
||||
async function verify(t, shards) {
|
||||
const root = await mkdtemp(join(tmpdir(), 'pm-ci-merge-'));
|
||||
t.after(() => rm(root, { recursive: true, force: true }));
|
||||
const staged = await stagedShards(root, shards);
|
||||
try {
|
||||
execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', runScript(verifyStep())], {
|
||||
cwd: root, stdio: 'pipe',
|
||||
env: { PATH: process.env.PATH, STAGED_RAW: staged, EXPECTED_SHARDS: '3' },
|
||||
});
|
||||
} catch (error) {
|
||||
return { ok: false, output: `${error.stdout}${error.stderr}` };
|
||||
}
|
||||
return { ok: true, raw: (await readdir(join(root, 'coverage/js/raw'))).sort() };
|
||||
}
|
||||
|
||||
test('merge refuses when a shard\'s raw coverage is missing', async t => {
|
||||
const missing = await verify(t, [[1, ['a.json']], [3, ['c.json']]]);
|
||||
assert.equal(missing.ok, false);
|
||||
assert.match(missing.output, /2 of 3/);
|
||||
const empty = await verify(t, [[1, ['a.json']], [2, []], [3, ['c.json']]]);
|
||||
assert.equal(empty.ok, false, 'an artifact without raw files counts as missing');
|
||||
const collision = await verify(t, [[1, ['a.json']], [2, ['a.json']], [3, ['c.json']]]);
|
||||
assert.equal(collision.ok, false, 'two shards writing the same file name must not silently drop one');
|
||||
const complete = await verify(t, [[1, ['a.json', 'node.json']], [2, ['b.json']], [3, ['c.json']]]);
|
||||
assert.deepEqual(complete, { ok: true, raw: ['a.json', 'b.json', 'c.json', 'node.json'] });
|
||||
});
|
||||
|
||||
test('every browser job sets PM_TEST_WORKERS and runs in the pinned image', () => {
|
||||
assert.deepEqual([...browserJobIds].sort(), ['browser-workflows', 'frontend-coverage']);
|
||||
for (const id of browserJobIds) {
|
||||
const body = allJobs.get(id);
|
||||
assert.match(body, /^ {6}PM_TEST_WORKERS: 3$/m, `${id} sets PM_TEST_WORKERS for the whole job`);
|
||||
assert.match(body, PINNED_IMAGE, `${id} keeps the pinned Playwright image`);
|
||||
assert.match(body, /PIP_SHA256: [0-9a-f]{64}/, `${id} keeps the pinned pip bootstrap`);
|
||||
assert.match(body, /^ {4}timeout-minutes: \d+$/m);
|
||||
}
|
||||
assert.match(mergeJob(), PINNED_IMAGE, 'the merge sees the same workspace paths as the shards');
|
||||
});
|
||||
|
||||
test('page workflows run each browser in two shards', () => {
|
||||
const workflows = allJobs.get('browser-workflows');
|
||||
assert.deepEqual(matrixList(workflows, 'browser'), ['firefox', 'webkit']);
|
||||
assert.deepEqual(matrixList(workflows, 'shard'), ['1', '2']);
|
||||
assert.match(workflows, /test:workflows -- --project \$\{\{ matrix\.browser \}\} --shard \$\{\{ matrix\.shard \}\}\/2/);
|
||||
assert.match(workflows, /^ {6}fail-fast: false$/m);
|
||||
assert.match(workflows, /name: browser-traces-\$\{\{ matrix\.browser \}\}-shard-\$\{\{ matrix\.shard \}\}/);
|
||||
});
|
||||
|
||||
test('every npm ci restores the npm download cache keyed on the vendor lockfile first', () => {
|
||||
const installing = [...allJobs].filter(([, body]) => /npm ci --prefix tools\/vendor/.test(body));
|
||||
assert.deepEqual(installing.map(([id]) => id).sort(), ['browser-workflows', 'frontend-coverage', 'frontend-coverage-merge']);
|
||||
for (const [id, body] of installing) {
|
||||
const jobSteps = steps(body);
|
||||
const cacheAt = jobSteps.findIndex(step => /uses: actions\/cache@v\d+/.test(step));
|
||||
const installAt = jobSteps.findIndex(step => /npm ci --prefix tools\/vendor/.test(step));
|
||||
assert.ok(cacheAt >= 0 && cacheAt < installAt, `${id} restores the npm cache before npm ci`);
|
||||
assert.match(jobSteps[cacheAt], /^ {10}path: ~\/\.npm$/m, `${id} caches npm's download cache, not node_modules`);
|
||||
assert.match(jobSteps[cacheAt], /hashFiles\('tools\/vendor\/package-lock\.json'\)/, `${id} keys the cache on the lockfile`);
|
||||
assert.match(jobSteps[installAt], /npm ci --prefix tools\/vendor --ignore-scripts/, `${id} still installs without scripts`);
|
||||
// The key names the job's own pinned image, so an image bump starts a fresh cache.
|
||||
const image = /^ {6}image: mcr\.microsoft\.com\/playwright:(v\d+\.\d+\.\d+)-noble$/m.exec(body)?.[1];
|
||||
assert.ok(image, `${id} runs in the pinned Playwright image`);
|
||||
assert.match(jobSteps[cacheAt], new RegExp(`key: npm-\\$\\{\\{ runner\\.os \\}\\}-playwright-${image.replaceAll('.', '\\.')}-\\$\\{\\{ hashFiles`),
|
||||
`${id} keys the cache on its image ${image}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('browser job timeouts keep a real margin over measured runs', () => {
|
||||
for (const id of ['frontend-coverage', 'browser-workflows']) {
|
||||
const minutes = Number(/^ {4}timeout-minutes: (\d+)$/m.exec(allJobs.get(id))[1]);
|
||||
assert.ok(minutes >= 20, `${id} allows at least 20 minutes, since runner speed varies`);
|
||||
}
|
||||
});
|
||||
Vendored
+44
@@ -0,0 +1,44 @@
|
||||
// A stand-in for tests/browser/server.py: `--port N` (0 = an OS-chosen port)
|
||||
// serves the readiness page and announces the bound port on stdout as
|
||||
// `PM_TEST_SERVER_PORT=<n>`, `--fail` writes to stderr and exits before
|
||||
// listening, and GET /__exit makes the process die unexpectedly after writing a
|
||||
// last stderr line. GET /__stderr?text=T writes T to stderr before answering.
|
||||
// `--split-announce` writes the announcement in two chunks, the first ending
|
||||
// mid-number. `--exit-on-stdin-eof` exits when its stdin closes, as server.py does.
|
||||
import { createServer } from 'node:http';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const port = Number(args[args.indexOf('--port') + 1]);
|
||||
if (args.includes('--fail')) {
|
||||
process.stderr.write('fake server: refusing to start\nlast fake stderr line\n');
|
||||
process.exit(4);
|
||||
}
|
||||
const server = createServer((request, response) => {
|
||||
const url = new URL(request.url, 'http://fake');
|
||||
if (url.pathname === '/__stderr') {
|
||||
process.stderr.write(url.searchParams.get('text'));
|
||||
response.end('written');
|
||||
return;
|
||||
}
|
||||
if (request.url === '/__exit') {
|
||||
process.stderr.write('fake server: crashed on purpose\n');
|
||||
response.end('bye', () => process.exit(3));
|
||||
return;
|
||||
}
|
||||
response.end(request.url === '/prompt_manager/test' ? 'ready' : `pid ${process.pid}`);
|
||||
});
|
||||
server.listen(port, '127.0.0.1', () => {
|
||||
const line = `PM_TEST_SERVER_PORT=${server.address().port}\n`;
|
||||
if (!args.includes('--split-announce')) {
|
||||
process.stdout.write(`fake server: listening\n${line}`);
|
||||
return;
|
||||
}
|
||||
// The first chunk is a complete-looking number, one digit short of the real port.
|
||||
process.stdout.write(`fake server: listening\n${line.slice(0, -2)}`);
|
||||
setTimeout(() => process.stdout.write(line.slice(-2)), 300);
|
||||
});
|
||||
if (args.includes('--exit-on-stdin-eof')) {
|
||||
process.stdin.resume();
|
||||
process.stdin.once('end', () => process.exit(0));
|
||||
}
|
||||
for (const signal of ['SIGTERM', 'SIGINT']) process.once(signal, () => server.close(() => process.exit(0)));
|
||||
+93
@@ -0,0 +1,93 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { existsSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { plan, runCoverage } from '../../tools/vendor/frontend-coverage.mjs';
|
||||
|
||||
const steps = argv => plan(argv).map(({ step, ...options }) => (Object.keys(options).length ? { step, ...options } : step));
|
||||
|
||||
function tempPaths(t) {
|
||||
const root = mkdtempSync(join(tmpdir(), 'pm-coverage-cli-'));
|
||||
t.after(() => rmSync(root, { recursive: true, force: true }));
|
||||
const reportRoot = join(root, 'coverage/js');
|
||||
return { root, paths: { repositoryRoot: root, tools: join(root, 'tools'), reportRoot, raw: join(reportRoot, 'raw'), sourceRoot: join(root, 'web/js') } };
|
||||
}
|
||||
|
||||
function recorder() {
|
||||
const calls = [];
|
||||
const run = (args, env = {}) => { calls.push({ args, env }); };
|
||||
return { calls, run };
|
||||
}
|
||||
|
||||
test('no arguments plans the unchanged full run', () => {
|
||||
assert.deepEqual(steps([]), [
|
||||
'clean', 'node-units', 'viewer', { step: 'workflows', shard: null }, 'report', { step: 'check', updateBaseline: false },
|
||||
]);
|
||||
assert.deepEqual(steps(['--update-baseline']), [
|
||||
'clean', 'node-units', 'viewer', { step: 'workflows', shard: null }, 'report', { step: 'check', updateBaseline: true },
|
||||
]);
|
||||
});
|
||||
|
||||
test('--shard 2/3 plans only workflows with --shard 2/3 and no report', () => {
|
||||
assert.deepEqual(steps(['--shard', '2/3']), ['prepare', { step: 'workflows', shard: '2/3' }]);
|
||||
assert.deepEqual(steps(['--shard=3/3']), ['prepare', { step: 'workflows', shard: '3/3' }]);
|
||||
});
|
||||
|
||||
test('--shard 1/N also runs the node units and the viewer suite', () => {
|
||||
assert.deepEqual(steps(['--shard', '1/3']), ['prepare', 'node-units', 'viewer', { step: 'workflows', shard: '1/3' }]);
|
||||
assert.deepEqual(steps(['--shard', '1/1']), ['prepare', 'node-units', 'viewer', { step: 'workflows', shard: '1/1' }]);
|
||||
});
|
||||
|
||||
test('--merge-only plans the report and the check over existing raw coverage', () => {
|
||||
assert.deepEqual(steps(['--merge-only']), ['require-raw', 'report', { step: 'check', updateBaseline: false }]);
|
||||
assert.deepEqual(steps(['--merge-only', '--update-baseline']), ['require-raw', 'report', { step: 'check', updateBaseline: true }]);
|
||||
});
|
||||
|
||||
test('invalid shard and mode combinations are refused', () => {
|
||||
for (const argv of [['--shard'], ['--shard', '0/3'], ['--shard', '4/3'], ['--shard', '2'], ['--shard', 'a/b'], ['--shard', '1/0'], ['--shard', '1.5/3']]) {
|
||||
assert.throws(() => plan(argv), /--shard needs I\/N/, argv.join(' '));
|
||||
}
|
||||
assert.throws(() => plan(['--shard', '1/2', '--merge-only']), /either --shard or --merge-only/);
|
||||
assert.throws(() => plan(['--shard', '1/2', '--update-baseline']), /--update-baseline .* --merge-only or a full run/);
|
||||
assert.throws(() => plan(['--shard', '1/2', '--shard', '2/2']), /--shard once/);
|
||||
assert.throws(() => plan(['--report']), /Unknown argument: --report/);
|
||||
});
|
||||
|
||||
test('--merge-only refuses an empty raw dir', async t => {
|
||||
const { paths } = tempPaths(t);
|
||||
const { calls, run } = recorder();
|
||||
await assert.rejects(runCoverage({ argv: ['--merge-only'], run, paths }), /no raw coverage .*coverage\/js\/raw.* run the --shard steps first/i);
|
||||
mkdirSync(paths.raw, { recursive: true });
|
||||
await assert.rejects(runCoverage({ argv: ['--merge-only'], run, paths }), /no raw coverage/i);
|
||||
writeFileSync(join(paths.raw, 'notes.txt'), 'not coverage');
|
||||
await assert.rejects(runCoverage({ argv: ['--merge-only'], run, paths }), /no raw coverage/i);
|
||||
assert.deepEqual(calls, [], 'nothing is reported or checked over partial or empty coverage');
|
||||
});
|
||||
|
||||
test('a shard keeps raw coverage other shards wrote and passes its shard to Playwright', async t => {
|
||||
const { paths } = tempPaths(t);
|
||||
mkdirSync(paths.raw, { recursive: true });
|
||||
const earlier = join(paths.raw, 'browser-earlier-shard.json');
|
||||
writeFileSync(earlier, '{"result":[]}');
|
||||
const { calls, run } = recorder();
|
||||
await runCoverage({ argv: ['--shard', '2/3'], run, paths });
|
||||
assert.ok(existsSync(earlier), 'raw files from another shard survive');
|
||||
assert.equal(calls.length, 1, 'shard 2 runs only the workflows');
|
||||
const [{ args, env }] = calls;
|
||||
assert.deepEqual(args.slice(-6), ['--project', 'chromium', '--project', 'touch', '--shard', '2/3']);
|
||||
assert.equal(env.PM_JS_COVERAGE, paths.raw);
|
||||
});
|
||||
|
||||
test('shard 1 runs the node units into the shared raw dir, then the viewer suite, then its workflows', async t => {
|
||||
const { root, paths } = tempPaths(t);
|
||||
mkdirSync(join(root, 'tests/js'), { recursive: true });
|
||||
writeFileSync(join(root, 'tests/js/a.test.js'), '');
|
||||
const { calls, run } = recorder();
|
||||
await runCoverage({ argv: ['--shard', '1/3'], run, paths });
|
||||
assert.equal(calls.length, 3);
|
||||
assert.deepEqual(calls[0].args, ['--test', 'tests/js/a.test.js']);
|
||||
assert.equal(calls[0].env.NODE_V8_COVERAGE, paths.raw);
|
||||
assert.ok(calls[1].args.includes(join(paths.tools, 'playwright.config.mjs')));
|
||||
assert.deepEqual(calls[2].args.slice(-2), ['--shard', '1/3']);
|
||||
});
|
||||
+38
-14
@@ -155,12 +155,36 @@
|
||||
"functions": 98,
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/file-inspector.mjs": {
|
||||
"workspace/activity-job-view.mjs": {
|
||||
"lines": 97,
|
||||
"branches": 83,
|
||||
"functions": 98,
|
||||
"statements": 97
|
||||
},
|
||||
"workspace/activity-scan.mjs": {
|
||||
"lines": 95,
|
||||
"branches": 92,
|
||||
"functions": 88,
|
||||
"branches": 64,
|
||||
"functions": 98,
|
||||
"statements": 95
|
||||
},
|
||||
"workspace/activity-store.mjs": {
|
||||
"lines": 98,
|
||||
"branches": 95,
|
||||
"functions": 98,
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/activity.mjs": {
|
||||
"lines": 95,
|
||||
"branches": 87,
|
||||
"functions": 98,
|
||||
"statements": 95
|
||||
},
|
||||
"workspace/file-inspector.mjs": {
|
||||
"lines": 96,
|
||||
"branches": 92,
|
||||
"functions": 92,
|
||||
"statements": 96
|
||||
},
|
||||
"workspace/heading.mjs": {
|
||||
"lines": 98,
|
||||
"branches": 98,
|
||||
@@ -229,7 +253,7 @@
|
||||
},
|
||||
"workspace/index.mjs": {
|
||||
"lines": 98,
|
||||
"branches": 96,
|
||||
"branches": 97,
|
||||
"functions": 98,
|
||||
"statements": 98
|
||||
},
|
||||
@@ -264,10 +288,10 @@
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/library.mjs": {
|
||||
"lines": 95,
|
||||
"lines": 96,
|
||||
"branches": 85,
|
||||
"functions": 96,
|
||||
"statements": 95
|
||||
"statements": 96
|
||||
},
|
||||
"workspace/media-viewer.mjs": {
|
||||
"lines": 98,
|
||||
@@ -282,10 +306,10 @@
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/prompt-blocks.mjs": {
|
||||
"lines": 89,
|
||||
"branches": 89,
|
||||
"lines": 90,
|
||||
"branches": 92,
|
||||
"functions": 98,
|
||||
"statements": 89
|
||||
"statements": 90
|
||||
},
|
||||
"workspace/prompt-inspector.mjs": {
|
||||
"lines": 98,
|
||||
@@ -302,7 +326,7 @@
|
||||
"workspace/recorded-run.mjs": {
|
||||
"lines": 95,
|
||||
"branches": 83,
|
||||
"functions": 77,
|
||||
"functions": 80,
|
||||
"statements": 95
|
||||
},
|
||||
"workspace/saved-views.mjs": {
|
||||
@@ -325,15 +349,15 @@
|
||||
},
|
||||
"workspace/toast.mjs": {
|
||||
"lines": 98,
|
||||
"branches": 64,
|
||||
"branches": 98,
|
||||
"functions": 98,
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/toolbar.mjs": {
|
||||
"lines": 94,
|
||||
"branches": 93,
|
||||
"lines": 96,
|
||||
"branches": 96,
|
||||
"functions": 98,
|
||||
"statements": 94
|
||||
"statements": 96
|
||||
},
|
||||
"workspace/topbar.mjs": {
|
||||
"lines": 98,
|
||||
|
||||
Vendored
+99
-20
@@ -1,4 +1,4 @@
|
||||
import { readFileSync, readdirSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { existsSync, readFileSync, readdirSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { dirname, join, relative } from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
@@ -7,33 +7,69 @@ import { repositoryRoot } from './vendor-viewerjs.mjs';
|
||||
import { discoverSources, checkCoverage } from '@comfyassets/toolkit/testing/coverage';
|
||||
export { discoverSources, checkCoverage };
|
||||
const metrics = ['lines', 'branches', 'functions', 'statements'];
|
||||
const SHARD = /^([1-9]\d*)\/([1-9]\d*)$/;
|
||||
const USAGE = 'usage: frontend-coverage.mjs [--update-baseline] | --shard I/N | --merge-only [--update-baseline]';
|
||||
|
||||
function run(args, env = {}) {
|
||||
function runNode(args, env = {}) {
|
||||
const result = spawnSync(process.execPath, args, { cwd: repositoryRoot, stdio: 'inherit',
|
||||
env: { ...process.env, ...env } });
|
||||
if (result.status !== 0) throw new Error(`Command failed (${result.status}): ${args.join(' ')}`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const tools = dirname(fileURLToPath(import.meta.url));
|
||||
function defaultPaths() {
|
||||
const reportRoot = join(repositoryRoot, 'coverage/js');
|
||||
const raw = join(reportRoot, 'raw');
|
||||
const sourceRoot = join(repositoryRoot, 'web/js');
|
||||
return { repositoryRoot, tools: dirname(fileURLToPath(import.meta.url)), reportRoot,
|
||||
raw: join(reportRoot, 'raw'), sourceRoot: join(repositoryRoot, 'web/js') };
|
||||
}
|
||||
|
||||
function parseShard(value) {
|
||||
const match = SHARD.exec(value ?? '');
|
||||
if (!match || Number(match[1]) > Number(match[2])) {
|
||||
throw new Error(`--shard needs I/N with 1 <= I <= N, got ${value ?? 'nothing'}\n${USAGE}`);
|
||||
}
|
||||
return { index: Number(match[1]), total: Number(match[2]), value };
|
||||
}
|
||||
|
||||
// Turn the command line into the ordered steps of one coverage run:
|
||||
// - no mode: the full run (clean, node units, viewer suite, chromium+touch
|
||||
// workflows, report, check), exactly as before shards existed;
|
||||
// - --shard I/N: one slice of the workflows into coverage/js/raw, without wiping
|
||||
// it or reporting; shard 1 also runs the node units and the viewer suite;
|
||||
// - --merge-only: report and check over raw coverage the shards already wrote.
|
||||
export function plan(argv) {
|
||||
let shard = null;
|
||||
let mergeOnly = false;
|
||||
let updateBaseline = false;
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === '--update-baseline') updateBaseline = true;
|
||||
else if (arg === '--merge-only') mergeOnly = true;
|
||||
else if (arg === '--shard' || arg.startsWith('--shard=')) {
|
||||
if (shard) throw new Error(`Pass --shard once\n${USAGE}`);
|
||||
shard = parseShard(arg === '--shard' ? argv[(index += 1)] : arg.slice('--shard='.length));
|
||||
} else throw new Error(`Unknown argument: ${arg}\n${USAGE}`);
|
||||
}
|
||||
if (shard && mergeOnly) throw new Error(`Pass either --shard or --merge-only\n${USAGE}`);
|
||||
if (shard && updateBaseline) throw new Error(`--update-baseline needs --merge-only or a full run, not a shard\n${USAGE}`);
|
||||
const check = { step: 'check', updateBaseline };
|
||||
if (mergeOnly) return [{ step: 'require-raw' }, { step: 'report' }, check];
|
||||
if (shard) {
|
||||
const first = shard.index === 1 ? [{ step: 'node-units' }, { step: 'viewer' }] : [];
|
||||
return [{ step: 'prepare' }, ...first, { step: 'workflows', shard: shard.value }];
|
||||
}
|
||||
return [{ step: 'clean' }, { step: 'node-units' }, { step: 'viewer' }, { step: 'workflows', shard: null },
|
||||
{ step: 'report' }, check];
|
||||
}
|
||||
|
||||
function rawCoverageFiles(raw) {
|
||||
return existsSync(raw) ? readdirSync(raw).filter(file => file.endsWith('.json')) : [];
|
||||
}
|
||||
|
||||
function checkSummary({ tools, reportRoot, sourceRoot }, updateBaseline) {
|
||||
const files = discoverSources(sourceRoot);
|
||||
rmSync(reportRoot, { recursive: true, force: true });
|
||||
mkdirSync(raw, { recursive: true });
|
||||
const tests = readdirSync(join(repositoryRoot, 'tests/js')).filter(file => file.endsWith('.test.js')).map(file => `tests/js/${file}`);
|
||||
run(['--test', ...tests], { NODE_V8_COVERAGE: raw });
|
||||
const playwright = join(tools, 'node_modules/@playwright/test/cli.js');
|
||||
run([playwright, 'test', '--config', join(tools, 'playwright.config.mjs')]);
|
||||
run([playwright, 'test', '--config', join(tools, 'playwright.workflows.config.mjs'),
|
||||
'--project', 'chromium', '--project', 'touch'], { PM_JS_COVERAGE: raw });
|
||||
run([join(tools, 'node_modules/c8/bin/c8.js'), 'report', '--all', '--src', sourceRoot,
|
||||
'--include', 'web/js/**/*.js', '--include', 'web/js/**/*.mjs', '--temp-directory', raw, '--reports-dir', reportRoot,
|
||||
'--reporter', 'text', '--reporter', 'json-summary', '--reporter', 'html']);
|
||||
const summary = JSON.parse(readFileSync(join(reportRoot, 'coverage-summary.json'), 'utf8'));
|
||||
const baselinePath = join(tools, 'frontend-coverage-baseline.json');
|
||||
if (process.argv.includes('--update-baseline')) {
|
||||
if (updateBaseline) {
|
||||
const previous = JSON.parse(readFileSync(baselinePath, 'utf8'));
|
||||
const failures = checkCoverage(summary, previous, sourceRoot, files.filter(file => previous[file]));
|
||||
if (failures.length) throw new Error(`Baseline updates cannot lower coverage:\n${failures.join('\n')}`);
|
||||
@@ -53,8 +89,51 @@ async function main() {
|
||||
const failures = checkCoverage(summary, baseline, sourceRoot, files);
|
||||
if (failures.length) throw new Error(`Frontend coverage regressed:\n${failures.join('\n')}`);
|
||||
}
|
||||
console.log(`Frontend coverage passed for ${files.length} files. Report: ${relative(repositoryRoot, reportRoot)}/index.html`);
|
||||
return files.length;
|
||||
}
|
||||
|
||||
export async function runCoverage({ argv = process.argv.slice(2), run = runNode, paths = defaultPaths() } = {}) {
|
||||
const { repositoryRoot: root, tools, reportRoot, raw, sourceRoot } = paths;
|
||||
const playwright = join(tools, 'node_modules/@playwright/test/cli.js');
|
||||
const actions = {
|
||||
clean() {
|
||||
rmSync(reportRoot, { recursive: true, force: true });
|
||||
mkdirSync(raw, { recursive: true });
|
||||
},
|
||||
prepare() {
|
||||
mkdirSync(raw, { recursive: true });
|
||||
},
|
||||
'require-raw'() {
|
||||
if (!rawCoverageFiles(raw).length) {
|
||||
throw new Error(`No raw coverage in ${relative(root, raw) || raw}: run the --shard steps first, `
|
||||
+ 'or put every shard\'s raw files there');
|
||||
}
|
||||
},
|
||||
'node-units'() {
|
||||
const tests = readdirSync(join(root, 'tests/js')).filter(file => file.endsWith('.test.js'))
|
||||
.map(file => `tests/js/${file}`);
|
||||
run(['--test', ...tests], { NODE_V8_COVERAGE: raw });
|
||||
},
|
||||
viewer() {
|
||||
run([playwright, 'test', '--config', join(tools, 'playwright.config.mjs')]);
|
||||
},
|
||||
workflows({ shard }) {
|
||||
run([playwright, 'test', '--config', join(tools, 'playwright.workflows.config.mjs'),
|
||||
'--project', 'chromium', '--project', 'touch', ...(shard ? ['--shard', shard] : [])], { PM_JS_COVERAGE: raw });
|
||||
},
|
||||
report() {
|
||||
run([join(tools, 'node_modules/c8/bin/c8.js'), 'report', '--all', '--src', sourceRoot,
|
||||
'--include', 'web/js/**/*.js', '--include', 'web/js/**/*.mjs', '--temp-directory', raw, '--reports-dir', reportRoot,
|
||||
'--reporter', 'text', '--reporter', 'json-summary', '--reporter', 'html']);
|
||||
},
|
||||
check({ updateBaseline }) {
|
||||
const count = checkSummary(paths, updateBaseline);
|
||||
console.log(`Frontend coverage passed for ${count} files. Report: ${relative(root, reportRoot)}/index.html`);
|
||||
},
|
||||
};
|
||||
for (const { step, ...options } of plan(argv)) await actions[step](options);
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch(error => { console.error(error.message); process.exitCode = 1; });
|
||||
runCoverage().catch(error => { console.error(error.message); process.exitCode = 1; });
|
||||
}
|
||||
|
||||
Vendored
+2
@@ -15,6 +15,8 @@
|
||||
"test:comfyui": "playwright test --config comfyui.config.mjs",
|
||||
"test:executions": "playwright test --config playwright.execution.config.mjs",
|
||||
"test:coverage": "node frontend-coverage.mjs",
|
||||
"test:coverage:shard": "node frontend-coverage.mjs --shard",
|
||||
"coverage:merge": "node frontend-coverage.mjs --merge-only",
|
||||
"coverage:baseline-check": "node coverage-baseline-check.mjs",
|
||||
"coverage:update": "node frontend-coverage.mjs --update-baseline",
|
||||
"test:compat": "playwright test --config pack-compat.config.mjs"
|
||||
|
||||
+6
-11
@@ -1,23 +1,18 @@
|
||||
import { defineConfig, devices } from '@playwright/test';
|
||||
import { repositoryRoot } from './vendor-viewerjs.mjs';
|
||||
const port = Number(process.env.PM_TEST_PORT || 8765);
|
||||
import { workflowWorkers } from './test-workers.mjs';
|
||||
|
||||
// No global webServer: tests/browser/fixtures.mjs starts one server per worker
|
||||
// on an OS-chosen port. Files run in parallel; tests inside a file keep their order.
|
||||
export default defineConfig({
|
||||
testDir: '../../tests/browser',
|
||||
outputDir: './test-results/workflows',
|
||||
testMatch: '*.spec.mjs',
|
||||
workers: 1,
|
||||
workers: workflowWorkers(),
|
||||
fullyParallel: false,
|
||||
forbidOnly: Boolean(process.env.CI),
|
||||
retries: 0,
|
||||
reporter: 'list',
|
||||
use: { baseURL: `http://127.0.0.1:${port}`, trace: 'retain-on-failure', screenshot: 'only-on-failure' },
|
||||
webServer: {
|
||||
command: `node tools/vendor/start-browser-server.mjs --port ${port}`,
|
||||
cwd: repositoryRoot,
|
||||
url: `http://127.0.0.1:${port}/prompt_manager/test`,
|
||||
reuseExistingServer: false,
|
||||
gracefulShutdown: { signal: 'SIGTERM', timeout: 5000 },
|
||||
},
|
||||
use: { trace: 'retain-on-failure', screenshot: 'only-on-failure' },
|
||||
projects: [
|
||||
{ name: 'chromium', outputDir: './test-results/workflows/chromium', use: { browserName: 'chromium' } },
|
||||
{ name: 'firefox', outputDir: './test-results/workflows/firefox', use: { browserName: 'firefox' } },
|
||||
|
||||
+187
-6
@@ -1,8 +1,189 @@
|
||||
import { spawn } from 'node:child_process';
|
||||
const child = spawn(process.env.PM_TEST_PYTHON || 'python',
|
||||
['-m', 'tests.browser.server', ...process.argv.slice(2)], { stdio: 'inherit' });
|
||||
child.on('error', error => { console.error(error.message); process.exitCode = 1; });
|
||||
child.on('exit', code => { process.exitCode = code ?? 1; });
|
||||
for (const signal of ['SIGTERM', 'SIGINT']) {
|
||||
process.once(signal, () => child.kill(signal));
|
||||
import { resolve } from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
import { repositoryRoot } from './vendor-viewerjs.mjs';
|
||||
|
||||
const READY_PATH = '/prompt_manager/test';
|
||||
const STARTUP_TIMEOUT_MS = 60_000;
|
||||
const STOP_TIMEOUT_MS = 5_000;
|
||||
const POLL_INTERVAL_MS = 100;
|
||||
const PROBE_TIMEOUT_MS = 1_000;
|
||||
const STDERR_TAIL_BYTES = 8_192;
|
||||
// Server stderr kept for the failing test's attachment; the newest output wins.
|
||||
const STDERR_LOG_CHARS = 256 * 1024;
|
||||
const STDERR_DROPPED = '[earlier server stderr dropped]\n';
|
||||
const HOST = '127.0.0.1';
|
||||
// tests/browser/server.py prints this once it holds its listening socket. Only a
|
||||
// complete line counts: a chunk may end mid-number.
|
||||
const PORT_ANNOUNCEMENT = /^PM_TEST_SERVER_PORT=(\d+)\r?\n/m;
|
||||
|
||||
// The server exits when its stdin closes, so a worker that dies without teardown
|
||||
// (SIGKILL) leaves no server behind; the launcher holds the pipe's other end.
|
||||
const pythonServer = port => ['-m', 'tests.browser.server', '--port', String(port), '--exit-on-stdin-eof'];
|
||||
|
||||
function describeExit({ code, signal }) {
|
||||
return signal ? `signal ${signal}` : `code ${code}`;
|
||||
}
|
||||
|
||||
// Start one disposable tests/browser/server.py and wait until it serves the
|
||||
// readiness page. `port` 0 (the default) lets the server bind an OS-chosen port
|
||||
// itself and announce it on stdout, so nothing can take the port between
|
||||
// choosing and binding. Resolves to {port, url, exited, exitStatus,
|
||||
// pid, stderrTail(), stderrMark(), stderrSince(mark), stop()}; rejects with the
|
||||
// server's stderr tail when it exits or stays unready. stdin is a pipe this
|
||||
// process holds open for the server's life.
|
||||
export async function startBrowserServer({
|
||||
port = 0,
|
||||
command = process.env.PM_TEST_PYTHON || 'python',
|
||||
args = pythonServer,
|
||||
cwd = repositoryRoot,
|
||||
timeoutMs = STARTUP_TIMEOUT_MS,
|
||||
stderrLogChars = STDERR_LOG_CHARS,
|
||||
} = {}) {
|
||||
if (!Number.isInteger(port) || port < 0 || port > 65535) {
|
||||
throw new Error(`startBrowserServer needs a port from 0 to 65535, got ${port}`);
|
||||
}
|
||||
const child = spawn(command, args(port), { cwd, stdio: ['pipe', 'pipe', 'pipe'] });
|
||||
// Never written; closing it (or this process dying) tells the server to exit.
|
||||
child.stdin.on('error', () => {});
|
||||
let stderr = '';
|
||||
let stderrTotal = 0;
|
||||
child.stderr.setEncoding('utf8');
|
||||
child.stderr.on('data', chunk => {
|
||||
stderr = (stderr + chunk).slice(-Math.max(stderrLogChars, STDERR_TAIL_BYTES));
|
||||
stderrTotal += chunk.length;
|
||||
});
|
||||
let stdout = '';
|
||||
let announced = null;
|
||||
child.stdout.setEncoding('utf8');
|
||||
// Keep draining stdout for the server's whole life so it never blocks on a full pipe.
|
||||
child.stdout.on('data', chunk => {
|
||||
if (announced !== null) return;
|
||||
stdout = (stdout + chunk).slice(-STDERR_TAIL_BYTES);
|
||||
const match = PORT_ANNOUNCEMENT.exec(stdout);
|
||||
if (match) announced = Number(match[1]);
|
||||
});
|
||||
const killOnExit = () => child.kill('SIGKILL');
|
||||
process.once('exit', killOnExit);
|
||||
const server = {
|
||||
pid: child.pid,
|
||||
port: port || null,
|
||||
url: port ? `http://${HOST}:${port}` : null,
|
||||
exitStatus: null,
|
||||
stopping: false,
|
||||
stderrTail: () => stderr.slice(-STDERR_TAIL_BYTES).trimEnd(),
|
||||
// A position in the server's stderr stream, for stderrSince().
|
||||
stderrMark: () => stderrTotal,
|
||||
// What the server wrote after `mark`, at most stderrLogChars of the newest of it.
|
||||
stderrSince(mark) {
|
||||
const kept = Math.min(stderr.length, stderrLogChars);
|
||||
const keptFrom = stderrTotal - kept;
|
||||
const log = stderr.slice(stderr.length - kept);
|
||||
return mark >= keptFrom ? log.slice(mark - keptFrom) : STDERR_DROPPED + log;
|
||||
},
|
||||
exited: new Promise(done => {
|
||||
child.once('error', error => {
|
||||
stderr += `\n${error.message}`;
|
||||
server.exitStatus = { code: null, signal: null, error };
|
||||
done(server.exitStatus);
|
||||
});
|
||||
child.once('exit', (code, signal) => {
|
||||
server.exitStatus = { code, signal };
|
||||
done(server.exitStatus);
|
||||
});
|
||||
}),
|
||||
async stop() {
|
||||
server.stopping = true;
|
||||
child.stdin.destroy();
|
||||
if (server.exitStatus === null) {
|
||||
child.kill('SIGTERM');
|
||||
const stopped = await Promise.race([server.exited, delay(STOP_TIMEOUT_MS).then(() => false)]);
|
||||
if (!stopped) {
|
||||
child.kill('SIGKILL');
|
||||
await server.exited;
|
||||
}
|
||||
}
|
||||
process.removeListener('exit', killOnExit);
|
||||
},
|
||||
};
|
||||
server.exited.then(() => process.removeListener('exit', killOnExit));
|
||||
const failure = async reason => {
|
||||
await server.stop();
|
||||
const tail = server.stderrTail();
|
||||
return new Error(`test server on port ${server.port ?? '(not yet announced)'} ${reason}`
|
||||
+ (tail ? `\n--- server stderr tail ---\n${tail}` : ''));
|
||||
};
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
if (server.exitStatus !== null) {
|
||||
throw await failure(`exited with ${describeExit(server.exitStatus)} before it was ready`);
|
||||
}
|
||||
if (announced !== null && server.url === null) {
|
||||
server.port = announced;
|
||||
server.url = `http://${HOST}:${announced}`;
|
||||
}
|
||||
if (announced !== null && announced !== server.port) {
|
||||
throw await failure(`announced port ${announced} instead`);
|
||||
}
|
||||
if (server.url === null) {
|
||||
await delay(POLL_INTERVAL_MS);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const response = await fetch(`${server.url}${READY_PATH}`, { signal: AbortSignal.timeout(PROBE_TIMEOUT_MS) });
|
||||
await response.arrayBuffer();
|
||||
if (response.ok) return server;
|
||||
} catch {
|
||||
// Not listening yet.
|
||||
}
|
||||
await delay(POLL_INTERVAL_MS);
|
||||
}
|
||||
throw await failure(`was not ready at ${READY_PATH} within ${timeoutMs} ms`);
|
||||
}
|
||||
|
||||
// After a test: when it did not end as expected, attach what the server wrote to
|
||||
// stderr since `mark` (an aiohttp traceback names the cause a "got 500" hides).
|
||||
export async function attachServerStderr(testInfo, server, mark) {
|
||||
if (testInfo.status === testInfo.expectedStatus) return;
|
||||
const log = server.stderrSince(mark);
|
||||
if (log) await testInfo.attach('server-stderr', { body: log, contentType: 'text/plain' });
|
||||
}
|
||||
|
||||
// Keep one live server for a Playwright worker. ensure() returns the running
|
||||
// server, or starts a fresh one when the last one died (a failed start is
|
||||
// reported, not retried); crash(server) turns an unexpected exit into an error
|
||||
// carrying the stderr tail.
|
||||
export function createServerSupervisor({ start }) {
|
||||
let current = null;
|
||||
return {
|
||||
async ensure() {
|
||||
if (current && current.exitStatus === null) return current;
|
||||
current = null;
|
||||
current = await start();
|
||||
return current;
|
||||
},
|
||||
crash(server) {
|
||||
if (!server || server.stopping || server.exitStatus === null) return null;
|
||||
const tail = server.stderrTail();
|
||||
return new Error(`test server on port ${server.port} exited unexpectedly (${describeExit(server.exitStatus)})`
|
||||
+ (tail ? `\n--- server stderr tail ---\n${tail}` : ''));
|
||||
},
|
||||
async stop() {
|
||||
const server = current;
|
||||
current = null;
|
||||
await server?.stop();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// CLI: `node start-browser-server.mjs --port N` runs one server in the foreground.
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
|
||||
const child = spawn(process.env.PM_TEST_PYTHON || 'python',
|
||||
['-m', 'tests.browser.server', ...process.argv.slice(2)], { stdio: 'inherit' });
|
||||
child.on('error', error => { console.error(error.message); process.exitCode = 1; });
|
||||
child.on('exit', code => { process.exitCode = code ?? 1; });
|
||||
for (const signal of ['SIGTERM', 'SIGINT']) {
|
||||
process.once(signal, () => child.kill(signal));
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+18
@@ -0,0 +1,18 @@
|
||||
import { availableParallelism } from 'node:os';
|
||||
|
||||
const MAX_LOCAL_WORKERS = 8;
|
||||
const CPUS_PER_WORKER = 4;
|
||||
|
||||
// PM_TEST_WORKERS wins; CI without it keeps one worker, so CI only changes
|
||||
// through explicit workflow edits; locally a quarter of the CPUs, at most eight.
|
||||
export function workflowWorkers({ env = process.env, cpus = availableParallelism() } = {}) {
|
||||
if (env.PM_TEST_WORKERS !== undefined) {
|
||||
const workers = Number(env.PM_TEST_WORKERS);
|
||||
if (!Number.isInteger(workers) || workers < 1) {
|
||||
throw new Error(`PM_TEST_WORKERS must be a positive integer, got "${env.PM_TEST_WORKERS}"`);
|
||||
}
|
||||
return workers;
|
||||
}
|
||||
if (env.CI) return 1;
|
||||
return Math.max(1, Math.min(MAX_LOCAL_WORKERS, Math.floor(cpus / CPUS_PER_WORKER)));
|
||||
}
|
||||
@@ -0,0 +1,632 @@
|
||||
"""Bounded private capture store for image annotations (#249 A3).
|
||||
|
||||
One explicit Tag or review action captures exactly one selected media file:
|
||||
the rooted file is streamed to a private, owner-only temporary copy while its
|
||||
whole-file SHA-256 is computed, and the identity of the opened descriptor is
|
||||
checked against the path before and after the copy. Preview, inference and the
|
||||
metadata read use only that copy, so what the user reviewed is what the model
|
||||
saw and what Save re-checks. Everything is bounded: bytes per file, completed
|
||||
captures, reserved bytes, concurrent capture/verify workers, idle and hard
|
||||
lifetimes.
|
||||
|
||||
The contract is "these bytes at this location, last checked at this time".
|
||||
Stat equality is never content proof: a same-size rewrite that restores
|
||||
``mtime_ns`` is visible only through the change time between the two
|
||||
``fstat`` reads, which Windows before Python 3.13 reports as the creation time
|
||||
(no signal).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import secrets
|
||||
import stat as stat_module
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import warnings
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
from PIL import Image
|
||||
|
||||
try:
|
||||
from .prompt_inspector import MetadataReadError, read_inspection_file
|
||||
from .video_metadata import VIDEO_EXTENSIONS, is_video_path
|
||||
except ImportError: # imported with the repo root on sys.path
|
||||
from utils.prompt_inspector import MetadataReadError, read_inspection_file
|
||||
from utils.video_metadata import VIDEO_EXTENSIONS, is_video_path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CHUNK_SIZE = 1024 * 1024
|
||||
DEFAULT_MAX_FILE = 256 * 2**20
|
||||
DEFAULT_MAX_CAPTURES = 8
|
||||
DEFAULT_MAX_BYTES = 512 * 2**20
|
||||
DEFAULT_MAX_WORKERS = 2
|
||||
DEFAULT_IDLE_SECONDS = 900
|
||||
DEFAULT_HARD_SECONDS = 1800
|
||||
MAX_INFERENCE_SIDE = 4096
|
||||
MAX_INFERENCE_PIXELS = 16_777_216
|
||||
MAX_RELATIVE_LENGTH = 1024
|
||||
|
||||
VIDEO_ONLY_REASON = "Auto-tagging is available for supported still images only."
|
||||
HEADER_REASON = "The image header could not be read."
|
||||
DIMENSION_REASON = (
|
||||
"Images larger than 4096 px per side or 16,777,216 px are not supported."
|
||||
)
|
||||
|
||||
# The listing's media allowlist (``py.api.inspection_service.SERVABLE_EXTENSIONS``),
|
||||
# restated here so utilities never import the aiohttp API package; a test pins
|
||||
# the two sets equal.
|
||||
_STILL_EXTENSIONS = frozenset(
|
||||
{".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".tiff", ".tif"}
|
||||
)
|
||||
CAPTURE_EXTENSIONS = _STILL_EXTENSIONS | frozenset(VIDEO_EXTENSIONS)
|
||||
|
||||
_DIRECTORY_PREFIX = "prompt-manager-captures-"
|
||||
_OPEN_FLAGS = (
|
||||
os.O_RDONLY
|
||||
| getattr(os, "O_BINARY", 0)
|
||||
| getattr(os, "O_NOFOLLOW", 0)
|
||||
| getattr(os, "O_CLOEXEC", 0)
|
||||
)
|
||||
|
||||
|
||||
class CaptureError(Exception):
|
||||
"""Base of the capture outcomes; ``code`` is the API error code."""
|
||||
|
||||
code = "capture-error"
|
||||
|
||||
|
||||
class CaptureCapacity(CaptureError):
|
||||
"""Count, byte or worker admission failed (nonblocking)."""
|
||||
|
||||
code = "review-capacity"
|
||||
|
||||
|
||||
class CaptureTooLarge(CaptureError):
|
||||
"""The file exceeds the per-file byte cap on stat or on the stream."""
|
||||
|
||||
code = "unsupported-media"
|
||||
|
||||
|
||||
class CaptureChanged(CaptureError):
|
||||
"""The file changed, vanished or was replaced across the copy (retryable)."""
|
||||
|
||||
code = "file-changed"
|
||||
|
||||
|
||||
class CaptureDenied(CaptureError):
|
||||
"""The target is not a listed rooted media file (alias, escape, non-media)."""
|
||||
|
||||
code = "source-changed"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FileFacts:
|
||||
"""Whole-file facts of the current rooted file, without a retained copy."""
|
||||
|
||||
locator: str
|
||||
sha256: str
|
||||
size: int
|
||||
mtime_ns: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Capture:
|
||||
"""One completed private copy; ``path`` is the owner-only temporary file."""
|
||||
|
||||
token: str
|
||||
locator: str
|
||||
root_index: int
|
||||
scope: str
|
||||
epoch: str
|
||||
sha256: str
|
||||
size: int
|
||||
mtime_ns: int
|
||||
path: str
|
||||
media_type: str
|
||||
width: int | None
|
||||
height: int | None
|
||||
inferable: bool
|
||||
inferable_reason: str | None
|
||||
|
||||
|
||||
class _Entry:
|
||||
__slots__ = ("capture", "created_at", "last_used_at", "uses", "doomed")
|
||||
|
||||
def __init__(self, capture: Capture, now: float):
|
||||
self.capture = capture
|
||||
self.created_at = now
|
||||
self.last_used_at = now
|
||||
self.uses = 0
|
||||
self.doomed = False
|
||||
|
||||
|
||||
def _same_facts(before: os.stat_result, after: os.stat_result) -> bool:
|
||||
"""The facts every stat API reports alike: size and modification time.
|
||||
|
||||
This is the only comparison made across APIs (a path ``stat`` against the
|
||||
opened descriptor's ``fstat``). Windows reports the creation time from
|
||||
``fstat`` but, on Python 3.12+, the change time from ``stat``, and may
|
||||
encode device and inode differently, so those fields are compared only
|
||||
between two reads of the same kind (``_same_identity``).
|
||||
"""
|
||||
return before.st_size == after.st_size and before.st_mtime_ns == after.st_mtime_ns
|
||||
|
||||
|
||||
def _same_identity(before: os.stat_result, after: os.stat_result) -> bool:
|
||||
"""Full identity between two reads of the same kind (both ``fstat`` or both
|
||||
path stats): size, mtime, change time, and device/inode where meaningful."""
|
||||
if not _same_facts(before, after) or before.st_ctime_ns != after.st_ctime_ns:
|
||||
return False
|
||||
if before.st_ino and after.st_ino and before.st_ino != after.st_ino:
|
||||
return False
|
||||
if before.st_dev and after.st_dev and before.st_dev != after.st_dev:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _reason(error: OSError) -> str:
|
||||
"""The OS error text without the path, so messages never name server paths."""
|
||||
return error.strerror or type(error).__name__
|
||||
|
||||
|
||||
def _unlink_quietly(path: str) -> None:
|
||||
try:
|
||||
os.unlink(path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except OSError as error:
|
||||
logger.warning("Capture file %s could not be removed: %s", path, error)
|
||||
|
||||
|
||||
class CaptureStore:
|
||||
"""Bounded, lock-protected registry of private media captures."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
temp_parent: os.PathLike | str | None = None,
|
||||
*,
|
||||
max_file: int = DEFAULT_MAX_FILE,
|
||||
max_captures: int = DEFAULT_MAX_CAPTURES,
|
||||
max_bytes: int = DEFAULT_MAX_BYTES,
|
||||
max_workers: int = DEFAULT_MAX_WORKERS,
|
||||
idle: float = DEFAULT_IDLE_SECONDS,
|
||||
hard: float = DEFAULT_HARD_SECONDS,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
):
|
||||
for name, value in (
|
||||
("max_file", max_file),
|
||||
("max_captures", max_captures),
|
||||
("max_bytes", max_bytes),
|
||||
("max_workers", max_workers),
|
||||
):
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value < 1:
|
||||
raise ValueError(f"{name} must be a positive integer")
|
||||
if not (idle > 0 and hard > 0):
|
||||
raise ValueError("idle and hard must be positive")
|
||||
self.max_file = max_file
|
||||
self.max_captures = max_captures
|
||||
self.max_bytes = max_bytes
|
||||
self.max_workers = max_workers
|
||||
self.idle = idle
|
||||
self.hard = hard
|
||||
self._clock = clock
|
||||
parent = None if temp_parent is None else os.fspath(temp_parent)
|
||||
if parent is not None:
|
||||
os.makedirs(parent, exist_ok=True)
|
||||
# mkdtemp creates the directory with owner-only permissions.
|
||||
self.directory = tempfile.mkdtemp(prefix=_DIRECTORY_PREFIX, dir=parent)
|
||||
self._lock = threading.Lock()
|
||||
self._entries: dict[str, _Entry] = {}
|
||||
self._completed_bytes = 0
|
||||
self._pending = 0
|
||||
self._pending_bytes = 0
|
||||
self._workers = 0
|
||||
self._closed = False
|
||||
self._stale: list[str] = []
|
||||
|
||||
# -- public API ------------------------------------------------------------
|
||||
|
||||
def capture(
|
||||
self,
|
||||
root: os.PathLike | str,
|
||||
relative: str,
|
||||
*,
|
||||
epoch: str,
|
||||
scope: str,
|
||||
root_index: int = 0,
|
||||
) -> Capture:
|
||||
"""Copy and hash one rooted media file into a private capture."""
|
||||
if isinstance(root_index, bool) or not isinstance(root_index, int):
|
||||
raise CaptureDenied("Invalid gallery root")
|
||||
if root_index < 0:
|
||||
raise CaptureDenied("Invalid gallery root")
|
||||
for name, value in (("epoch", epoch), ("scope", scope)):
|
||||
if not isinstance(value, str) or not value:
|
||||
raise ValueError(f"{name} must be a non-empty string")
|
||||
parts, extension = _relative_parts(relative)
|
||||
candidate, locator = _resolve(root, parts)
|
||||
before = self._pre_stat(candidate)
|
||||
self._admit(before.st_size, reserve=True)
|
||||
try:
|
||||
capture = self._copy(
|
||||
candidate, locator, extension, before, root_index, epoch, scope
|
||||
)
|
||||
finally:
|
||||
self._release_admission(before.st_size, reserve=True)
|
||||
return capture
|
||||
|
||||
def verify(self, root: os.PathLike | str, relative: str) -> FileFacts:
|
||||
"""Hash the current rooted file under the same bounds, keeping no copy."""
|
||||
parts, _extension = _relative_parts(relative)
|
||||
candidate, locator = _resolve(root, parts)
|
||||
before = self._pre_stat(candidate)
|
||||
self._admit(0, reserve=False)
|
||||
try:
|
||||
sha256 = self._stream(candidate, locator, before, None)
|
||||
finally:
|
||||
self._release_admission(0, reserve=False)
|
||||
return FileFacts(locator, sha256, before.st_size, before.st_mtime_ns)
|
||||
|
||||
def get(self, token: str) -> Capture | None:
|
||||
"""The live capture for *token*, touching its idle clock; else None.
|
||||
|
||||
``get`` does not pin the file: a reader that streams ``capture.path``
|
||||
(the preview route) must bracket the read with ``acquire`` and
|
||||
``release_use``, or a concurrent ``sweep``/``release`` may unlink the
|
||||
file under it (and, on Windows, leak it until ``close``).
|
||||
"""
|
||||
try:
|
||||
with self._lock:
|
||||
entry = self._live_locked(token)
|
||||
if entry is None:
|
||||
return None
|
||||
entry.last_used_at = self._clock()
|
||||
return entry.capture
|
||||
finally:
|
||||
self._flush()
|
||||
|
||||
def acquire(self, token: str) -> Capture | None:
|
||||
"""Mark the capture in use (inference); it outlives expiry until released."""
|
||||
try:
|
||||
with self._lock:
|
||||
entry = self._live_locked(token)
|
||||
if entry is None:
|
||||
return None
|
||||
entry.uses += 1
|
||||
entry.last_used_at = self._clock()
|
||||
return entry.capture
|
||||
finally:
|
||||
self._flush()
|
||||
|
||||
def release_use(self, token: str) -> None:
|
||||
"""End one ``acquire``; a doomed capture is removed with its last use."""
|
||||
with self._lock:
|
||||
entry = self._entries.get(token)
|
||||
if entry is not None:
|
||||
entry.uses = max(0, entry.uses - 1)
|
||||
entry.last_used_at = self._clock()
|
||||
if entry.doomed and entry.uses == 0:
|
||||
self._remove_locked(entry)
|
||||
self._flush()
|
||||
|
||||
def release(self, token: str) -> bool:
|
||||
"""Explicit release; True when the token was live."""
|
||||
with self._lock:
|
||||
entry = self._entries.get(token)
|
||||
live = entry is not None and not entry.doomed
|
||||
if live:
|
||||
self._doom_locked(entry)
|
||||
self._flush()
|
||||
return live
|
||||
|
||||
def invalidate_all(self) -> None:
|
||||
"""Expire every capture now (storage epoch changed, restore, shutdown)."""
|
||||
with self._lock:
|
||||
for entry in list(self._entries.values()):
|
||||
self._doom_locked(entry)
|
||||
self._flush()
|
||||
|
||||
def sweep(self) -> None:
|
||||
"""Remove idle and hard-expired captures that are not in use."""
|
||||
with self._lock:
|
||||
self._sweep_locked(self._clock())
|
||||
self._flush()
|
||||
|
||||
def expires_in(self, token: str) -> float | None:
|
||||
"""Seconds until *token* expires without further use; None when gone."""
|
||||
try:
|
||||
with self._lock:
|
||||
entry = self._live_locked(token)
|
||||
if entry is None:
|
||||
return None
|
||||
now = self._clock()
|
||||
remaining = self.hard - (now - entry.created_at)
|
||||
if entry.uses == 0:
|
||||
remaining = min(remaining, self.idle - (now - entry.last_used_at))
|
||||
return max(0.0, remaining)
|
||||
finally:
|
||||
self._flush()
|
||||
|
||||
def stats(self) -> dict:
|
||||
with self._lock:
|
||||
return {
|
||||
"captures": len(self._entries),
|
||||
"reserved_bytes": self._completed_bytes + self._pending_bytes,
|
||||
"workers": self._workers,
|
||||
}
|
||||
|
||||
def close(self) -> None:
|
||||
"""Remove every owned capture file and the owned directory. Idempotent.
|
||||
|
||||
A capture still in flight finishes first: its worker removes its own
|
||||
temp file and then the directory (``_release_admission``).
|
||||
"""
|
||||
with self._lock:
|
||||
self._closed = True
|
||||
for entry in list(self._entries.values()):
|
||||
self._remove_locked(entry)
|
||||
busy = self._pending > 0
|
||||
self._flush()
|
||||
if not busy:
|
||||
self._remove_directory()
|
||||
|
||||
def _remove_directory(self) -> None:
|
||||
try:
|
||||
os.rmdir(self.directory)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except OSError as error:
|
||||
logger.warning(
|
||||
"Capture directory %s could not be removed: %s", self.directory, error
|
||||
)
|
||||
|
||||
# -- admission -------------------------------------------------------------
|
||||
|
||||
def _admit(self, size: int, *, reserve: bool) -> None:
|
||||
try:
|
||||
self._admit_locked(size, reserve=reserve)
|
||||
finally:
|
||||
self._flush()
|
||||
|
||||
def _admit_locked(self, size: int, *, reserve: bool) -> None:
|
||||
with self._lock:
|
||||
if self._closed:
|
||||
raise RuntimeError("The capture store is closed")
|
||||
self._sweep_locked(self._clock())
|
||||
if self._workers >= self.max_workers:
|
||||
raise CaptureCapacity("Too many captures are in progress")
|
||||
if reserve:
|
||||
if len(self._entries) + self._pending >= self.max_captures:
|
||||
raise CaptureCapacity("Too many captures are held")
|
||||
reserved = self._completed_bytes + self._pending_bytes
|
||||
if reserved + size > self.max_bytes:
|
||||
raise CaptureCapacity("Capture byte budget exhausted")
|
||||
self._pending += 1
|
||||
self._pending_bytes += size
|
||||
self._workers += 1
|
||||
|
||||
def _release_admission(self, size: int, *, reserve: bool) -> None:
|
||||
with self._lock:
|
||||
self._workers -= 1
|
||||
if reserve:
|
||||
self._pending -= 1
|
||||
self._pending_bytes -= size
|
||||
last_out = self._closed and self._pending == 0 and self._workers == 0
|
||||
if last_out:
|
||||
self._remove_directory()
|
||||
|
||||
def _register(self, capture: Capture) -> None:
|
||||
with self._lock:
|
||||
if self._closed:
|
||||
raise RuntimeError("The capture store is closed")
|
||||
self._completed_bytes += capture.size
|
||||
self._entries[capture.token] = _Entry(capture, self._clock())
|
||||
|
||||
# -- lifetime (all under the lock) -----------------------------------------
|
||||
|
||||
def _expired(self, entry: _Entry, now: float) -> bool:
|
||||
if now - entry.created_at >= self.hard:
|
||||
return True
|
||||
return entry.uses == 0 and now - entry.last_used_at >= self.idle
|
||||
|
||||
def _live_locked(self, token: str) -> _Entry | None:
|
||||
entry = self._entries.get(token)
|
||||
if entry is None or entry.doomed:
|
||||
return None
|
||||
if self._expired(entry, self._clock()):
|
||||
self._doom_locked(entry)
|
||||
return None
|
||||
return entry
|
||||
|
||||
def _sweep_locked(self, now: float) -> None:
|
||||
for entry in list(self._entries.values()):
|
||||
if entry.doomed or self._expired(entry, now):
|
||||
self._doom_locked(entry)
|
||||
|
||||
def _doom_locked(self, entry: _Entry) -> None:
|
||||
entry.doomed = True
|
||||
if entry.uses == 0:
|
||||
self._remove_locked(entry)
|
||||
|
||||
def _remove_locked(self, entry: _Entry) -> None:
|
||||
"""Forget the entry; its file is unlinked by ``_flush`` after the lock."""
|
||||
if self._entries.pop(entry.capture.token, None) is not None:
|
||||
self._completed_bytes -= entry.capture.size
|
||||
self._stale.append(entry.capture.path)
|
||||
|
||||
def _flush(self) -> None:
|
||||
"""Unlink files released under the lock, without holding it (slow I/O)."""
|
||||
with self._lock:
|
||||
stale, self._stale = self._stale, []
|
||||
for path in stale:
|
||||
_unlink_quietly(path)
|
||||
|
||||
# -- copy and hash -----------------------------------------------------------
|
||||
|
||||
def _pre_stat(self, candidate: str) -> os.stat_result:
|
||||
try:
|
||||
facts = os.lstat(candidate)
|
||||
except FileNotFoundError:
|
||||
raise CaptureChanged("The file is no longer there") from None
|
||||
except OSError as error:
|
||||
logger.debug("Capture pre-stat failed for %s: %s", candidate, error)
|
||||
raise CaptureChanged(
|
||||
f"The file could not be read ({_reason(error)})"
|
||||
) from None
|
||||
if stat_module.S_ISLNK(facts.st_mode):
|
||||
raise CaptureDenied("Symbolic links are not captured")
|
||||
if not stat_module.S_ISREG(facts.st_mode):
|
||||
raise CaptureDenied("Only regular files are captured")
|
||||
if facts.st_size > self.max_file:
|
||||
raise CaptureTooLarge("The file is larger than the capture limit")
|
||||
return facts
|
||||
|
||||
def _copy(
|
||||
self,
|
||||
candidate: str,
|
||||
locator: str,
|
||||
extension: str,
|
||||
before: os.stat_result,
|
||||
root_index: int,
|
||||
epoch: str,
|
||||
scope: str,
|
||||
) -> Capture:
|
||||
fd, temp_path = tempfile.mkstemp(
|
||||
prefix="capture-", suffix=extension, dir=self.directory
|
||||
)
|
||||
sink = os.fdopen(fd, "wb")
|
||||
completed = False
|
||||
try:
|
||||
sha256 = self._stream(candidate, locator, before, sink)
|
||||
sink.close()
|
||||
media_type = "video" if is_video_path(candidate) else "image"
|
||||
width, height, inferable, reason = self._probe(temp_path, media_type)
|
||||
capture = Capture(
|
||||
token=secrets.token_urlsafe(32),
|
||||
locator=locator,
|
||||
root_index=root_index,
|
||||
scope=scope,
|
||||
epoch=epoch,
|
||||
sha256=sha256,
|
||||
size=before.st_size,
|
||||
mtime_ns=before.st_mtime_ns,
|
||||
path=temp_path,
|
||||
media_type=media_type,
|
||||
width=width,
|
||||
height=height,
|
||||
inferable=inferable,
|
||||
inferable_reason=reason,
|
||||
)
|
||||
self._register(capture)
|
||||
completed = True
|
||||
return capture
|
||||
finally:
|
||||
if not sink.closed:
|
||||
sink.close()
|
||||
if not completed:
|
||||
_unlink_quietly(temp_path)
|
||||
|
||||
def _read(self, stream, size: int) -> bytes:
|
||||
"""One bounded read of the source; a seam for change-detection tests."""
|
||||
return stream.read(size)
|
||||
|
||||
def _stream(self, candidate: str, locator: str, before, sink) -> str:
|
||||
"""Hash (and copy into *sink*) the file; every handle closes here."""
|
||||
digest = hashlib.sha256()
|
||||
total = 0
|
||||
try:
|
||||
fd = os.open(candidate, _OPEN_FLAGS)
|
||||
except OSError as error:
|
||||
logger.debug("Capture open failed for %s: %s", candidate, error)
|
||||
raise CaptureChanged(
|
||||
f"The file could not be opened ({_reason(error)})"
|
||||
) from None
|
||||
with os.fdopen(fd, "rb") as stream:
|
||||
opened = os.fstat(stream.fileno())
|
||||
if not _same_facts(before, opened):
|
||||
raise CaptureChanged("The file changed before it was read")
|
||||
while True:
|
||||
chunk = self._read(stream, CHUNK_SIZE)
|
||||
if not chunk:
|
||||
break
|
||||
total += len(chunk)
|
||||
if total > self.max_file:
|
||||
raise CaptureTooLarge("The file grew past the capture limit")
|
||||
digest.update(chunk)
|
||||
if sink is not None:
|
||||
sink.write(chunk)
|
||||
after = os.fstat(stream.fileno())
|
||||
if total != before.st_size or not _same_identity(opened, after):
|
||||
raise CaptureChanged("The file changed while it was read")
|
||||
try:
|
||||
current = os.lstat(candidate)
|
||||
except OSError:
|
||||
raise CaptureChanged("The file is no longer there") from None
|
||||
if not _same_identity(before, current):
|
||||
raise CaptureChanged("The path names another file now")
|
||||
if os.path.normcase(os.path.realpath(candidate)) != locator:
|
||||
raise CaptureChanged("The path resolves elsewhere now")
|
||||
return digest.hexdigest()
|
||||
|
||||
def _probe(self, path: str, media_type: str):
|
||||
"""Header-only dimensions of the capture; never decodes pixels.
|
||||
|
||||
Uses the inspection reader, so a PNG whose prompt metadata is over the
|
||||
inspection budgets is reported as unreadable (not inferable) although
|
||||
its pixels decode; B2 may switch to a dimensions-only header read.
|
||||
"""
|
||||
if media_type == "video":
|
||||
return None, None, False, VIDEO_ONLY_REASON
|
||||
try:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", Image.DecompressionBombWarning)
|
||||
read = read_inspection_file(Path(path))
|
||||
except (MetadataReadError, Image.DecompressionBombError):
|
||||
return None, None, False, HEADER_REASON
|
||||
width, height = read.width, read.height
|
||||
if width is None or height is None:
|
||||
return None, None, False, HEADER_REASON
|
||||
if (
|
||||
width > MAX_INFERENCE_SIDE
|
||||
or height > MAX_INFERENCE_SIDE
|
||||
or width * height > MAX_INFERENCE_PIXELS
|
||||
):
|
||||
return width, height, False, DIMENSION_REASON
|
||||
return width, height, True, None
|
||||
|
||||
|
||||
def _relative_parts(relative) -> tuple[list[str], str]:
|
||||
"""Split a safe relative POSIX media path; anything else is denied."""
|
||||
if not isinstance(relative, str) or not relative:
|
||||
raise CaptureDenied("Invalid file path")
|
||||
if len(relative) > MAX_RELATIVE_LENGTH or "\x00" in relative:
|
||||
raise CaptureDenied("Invalid file path")
|
||||
if "\\" in relative or relative.startswith("/"):
|
||||
raise CaptureDenied("Invalid file path")
|
||||
parts = relative.split("/")
|
||||
for part in parts:
|
||||
if part in ("", ".", "..") or ":" in part:
|
||||
raise CaptureDenied("Invalid file path")
|
||||
extension = os.path.splitext(parts[-1])[1].lower()
|
||||
if extension not in CAPTURE_EXTENSIONS:
|
||||
raise CaptureDenied("Unsupported media type")
|
||||
return parts, extension
|
||||
|
||||
|
||||
def _resolve(root, parts: list[str]) -> tuple[str, str]:
|
||||
"""The candidate path and its locator; aliases and escapes are denied."""
|
||||
real_root = os.path.realpath(os.fspath(root))
|
||||
if not os.path.isdir(real_root):
|
||||
raise CaptureDenied("Unknown gallery root")
|
||||
candidate = os.path.join(real_root, *parts)
|
||||
locator = os.path.normcase(candidate)
|
||||
if os.path.normcase(os.path.realpath(candidate)) != locator:
|
||||
raise CaptureDenied("The file is not a listed rooted file")
|
||||
return candidate, locator
|
||||
+65
-48
@@ -23,7 +23,15 @@ from .prompt_pair_resolver import (
|
||||
ResolverProfile,
|
||||
resolve_prompt_pair,
|
||||
)
|
||||
from .video_metadata import ffprobe_path, is_video_path
|
||||
from .video_metadata import (
|
||||
ENVELOPE_TAG_KEYS,
|
||||
ROLE_TAG_KEYS,
|
||||
VIDEO_TAG_KEYS,
|
||||
DuplicateVideoTagError,
|
||||
ffprobe_path,
|
||||
is_video_path,
|
||||
select_video_carrier,
|
||||
)
|
||||
|
||||
# Role records (prompt, parameters, the pair execution record) share one
|
||||
# budget, encoded and decoded: roles cannot be trusted from a partial record,
|
||||
@@ -36,7 +44,18 @@ WORKFLOW_METADATA_BYTES = 4 * 1024 * 1024
|
||||
# repeated chunks cost bounded I/O instead of file size.
|
||||
DUPLICATE_WORKFLOW_BYTES = 2 * WORKFLOW_METADATA_BYTES
|
||||
DUPLICATE_ROLE_BYTES = 2 * ROLE_METADATA_BYTES
|
||||
MAX_METADATA_BYTES = ROLE_METADATA_BYTES # video stdout bound, unchanged here
|
||||
# ffprobe stdout bounds. ffprobe JSON-escapes tag text (at most doubling it),
|
||||
# and the decoded per-record budgets below are the only gate that may make a
|
||||
# record fail or ``limited``: a video must read exactly as the identical PNG.
|
||||
# So the first probe allows twice the role and workflow budgets plus 64 KiB of
|
||||
# JSON envelope; past that only the per-key ``workflow`` tag can be dropped,
|
||||
# and the re-probe asks for role and envelope tags within twice the role
|
||||
# budget (an envelope cannot be split, so an over-budget one fails the read).
|
||||
MAX_METADATA_BYTES = ROLE_METADATA_BYTES
|
||||
VIDEO_PROBE_BYTES = 2 * (ROLE_METADATA_BYTES + WORKFLOW_METADATA_BYTES) + 64 * 1024
|
||||
VIDEO_REPROBE_BYTES = 2 * ROLE_METADATA_BYTES + 64 * 1024
|
||||
VIDEO_PROBE_SECONDS = 3
|
||||
_VIDEO_ROLE_PROBE_KEYS = ROLE_TAG_KEYS + ENVELOPE_TAG_KEYS
|
||||
MAX_KEYWORD_BYTES = 80 # a 1-79 byte PNG keyword plus its NUL separator
|
||||
MAX_HEADER_BYTES = 8 * 1024 * 1024
|
||||
MAX_PNG_CHUNKS = 10_000
|
||||
@@ -86,6 +105,7 @@ class _VideoInspection:
|
||||
metadata: dict[str, str | None]
|
||||
record_formats: dict[str, str]
|
||||
carrier_text: str | None
|
||||
limited: frozenset[str] = dataclass_field(default_factory=frozenset)
|
||||
|
||||
|
||||
def _inflate(data: bytes, limit: int) -> bytes | None:
|
||||
@@ -274,16 +294,16 @@ def _png_metadata(stream: Any, size: int) -> tuple[dict, frozenset, int, int]:
|
||||
raise MetadataReadError("PNG chunk count exceeds inspection limits")
|
||||
|
||||
|
||||
def _video_metadata(path: Path) -> _VideoInspection:
|
||||
probe = ffprobe_path()
|
||||
if not probe:
|
||||
raise MetadataReadError("Video metadata reader is unavailable")
|
||||
def _probe_video_tags(
|
||||
probe: str, path: Path, keys: tuple[str, ...], limit: int
|
||||
) -> bytes | None:
|
||||
"""ffprobe's JSON for the container tags ``keys``; ``None`` past ``limit``."""
|
||||
command = [
|
||||
probe,
|
||||
"-v",
|
||||
"quiet",
|
||||
"-show_entries",
|
||||
"format_tags=comment,description",
|
||||
"format_tags=" + ",".join(keys),
|
||||
"-of",
|
||||
"json",
|
||||
str(path),
|
||||
@@ -303,13 +323,13 @@ def _video_metadata(path: Path) -> _VideoInspection:
|
||||
# Timeout and final cleanup can race with the process exiting.
|
||||
pass
|
||||
|
||||
timer = threading.Timer(3, stop)
|
||||
timer = threading.Timer(VIDEO_PROBE_SECONDS, stop)
|
||||
timer.daemon = True
|
||||
timer.start()
|
||||
try:
|
||||
raw = process.stdout.read(MAX_METADATA_BYTES + 1)
|
||||
if len(raw) > MAX_METADATA_BYTES:
|
||||
raise MetadataReadError("Video metadata exceeds inspection limits")
|
||||
raw = process.stdout.read(limit + 1)
|
||||
if len(raw) > limit:
|
||||
return None
|
||||
process.wait()
|
||||
if process.returncode:
|
||||
raise MetadataReadError("Video metadata could not be read")
|
||||
@@ -318,49 +338,45 @@ def _video_metadata(path: Path) -> _VideoInspection:
|
||||
stop()
|
||||
process.wait()
|
||||
process.stdout.close()
|
||||
return raw
|
||||
|
||||
|
||||
def _video_metadata(path: Path) -> _VideoInspection:
|
||||
probe = ffprobe_path()
|
||||
if not probe:
|
||||
raise MetadataReadError("Video metadata reader is unavailable")
|
||||
limited = frozenset()
|
||||
raw = _probe_video_tags(probe, path, VIDEO_TAG_KEYS, VIDEO_PROBE_BYTES)
|
||||
if raw is None:
|
||||
raw = _probe_video_tags(
|
||||
probe, path, _VIDEO_ROLE_PROBE_KEYS, VIDEO_REPROBE_BYTES
|
||||
)
|
||||
if raw is None:
|
||||
raise MetadataReadError("Video metadata exceeds inspection limits")
|
||||
# Only the per-key workflow tag was left out, so it is the record over
|
||||
# budget: limited when the per-key carrier (or no carrier) is selected.
|
||||
limited = frozenset({"workflow"})
|
||||
probed = json.loads(raw)
|
||||
if not isinstance(probed, dict) or not isinstance(probed.get("format"), dict):
|
||||
raise MetadataReadError("Malformed video metadata")
|
||||
tags = probed["format"].get("tags", {})
|
||||
if not isinstance(tags, dict):
|
||||
raise MetadataReadError("Malformed video metadata")
|
||||
# The scan normalizer drops null fields and unknown keys. Inspection must
|
||||
# retain explicit prompt/execution markers so malformed evidence cannot
|
||||
# silently fall back to an unrelated saved recipe or another comment.
|
||||
candidates = []
|
||||
for key in ("comment", "description"):
|
||||
raw = tags.get(key)
|
||||
if not isinstance(raw, str):
|
||||
continue
|
||||
try:
|
||||
envelope = json.loads(raw)
|
||||
except ValueError:
|
||||
continue
|
||||
if not isinstance(envelope, dict):
|
||||
continue
|
||||
candidates.append((raw, envelope))
|
||||
selected = next(
|
||||
(item for item in candidates if _ROLE_METADATA_KEYS.intersection(item[1])),
|
||||
None,
|
||||
)
|
||||
# The shared selector keeps explicit null role markers so malformed
|
||||
# evidence cannot silently fall back to an unrelated saved recipe or
|
||||
# another carrier; ``read_inspection_file`` refuses them.
|
||||
try:
|
||||
selected = select_video_carrier(tags)
|
||||
except DuplicateVideoTagError:
|
||||
raise MetadataReadError("Duplicate prompt metadata") from None
|
||||
if selected is None:
|
||||
selected = next((item for item in candidates if "workflow" in item[1]), None)
|
||||
if selected is not None:
|
||||
raw, envelope = selected
|
||||
metadata, formats = {}, {}
|
||||
for field in _METADATA_KEYS.intersection(envelope):
|
||||
value = envelope[field]
|
||||
if field == "workflow" and not isinstance(value, (str, dict)):
|
||||
formats[field] = "invalid"
|
||||
continue
|
||||
metadata[field] = (
|
||||
value if value is None or isinstance(value, str) else json.dumps(value)
|
||||
)
|
||||
formats[field] = (
|
||||
"original-text" if isinstance(value, str) else "reserialized-json"
|
||||
)
|
||||
return _VideoInspection(metadata, formats, raw)
|
||||
return _VideoInspection({}, {}, None)
|
||||
return _VideoInspection({}, {}, None, limited)
|
||||
metadata, formats, carrier = selected
|
||||
if carrier is not None:
|
||||
# An envelope carrier is whole; the dropped per-key workflow belonged
|
||||
# to the carrier that lost.
|
||||
limited = frozenset()
|
||||
return _VideoInspection(metadata, formats, carrier, limited)
|
||||
|
||||
|
||||
def read_inspection_file(path: Path) -> InspectionRead:
|
||||
@@ -370,10 +386,11 @@ def read_inspection_file(path: Path) -> InspectionRead:
|
||||
format_name = width = height = None
|
||||
if is_video_path(path):
|
||||
video = _video_metadata(path)
|
||||
metadata, formats, carrier = (
|
||||
metadata, formats, carrier, limited = (
|
||||
video.metadata,
|
||||
video.record_formats,
|
||||
video.carrier_text,
|
||||
video.limited,
|
||||
)
|
||||
else:
|
||||
with path.open("rb") as stream:
|
||||
|
||||
+126
-27
@@ -1,14 +1,18 @@
|
||||
"""Prompt metadata from video containers.
|
||||
|
||||
ComfyUI's video savers (core SaveVideo, VideoHelperSuite, ...) write the
|
||||
queued graph into the container's ``comment`` tag as JSON:
|
||||
``{"prompt": "<graph json>", "workflow": {...}}``. ffprobe reads it on every
|
||||
platform; when ffprobe is not installed, videos are skipped with a single
|
||||
notice instead of one error per file.
|
||||
Current ComfyUI video savers (core SaveVideo/SaveWEBM, VideoHelperSuite since
|
||||
``c67ff3e``) write one container tag per key, each a JSON string:
|
||||
``prompt``, ``workflow``, ``parameters``, ``comfyassets_prompt_pairs``. Older
|
||||
VideoHelperSuite wrote one ``comment`` tag holding a JSON envelope
|
||||
``{"prompt": "<graph json>", "workflow": {...}}``; ``description`` is read as a
|
||||
fallback envelope. ffprobe reads the tags on every platform; when ffprobe is
|
||||
not installed, videos are skipped with a single notice instead of one error
|
||||
per file.
|
||||
|
||||
``select_video_carrier`` picks exactly one carrier for both the inspector and
|
||||
the scan, so Images and ingestion read the same prompts from the same file.
|
||||
The result uses the same shape as PNG text chunks (string values keyed
|
||||
``prompt`` / ``workflow`` / ``parameters``) so the existing prompt parser
|
||||
handles both.
|
||||
``prompt`` / ``workflow`` / ...) so the existing prompt parser handles both.
|
||||
"""
|
||||
|
||||
import functools
|
||||
@@ -20,10 +24,15 @@ import shutil
|
||||
# Fixed argv, absolute binary, no shell.
|
||||
import subprocess # nosec B404
|
||||
|
||||
from .pair_execution import EXECUTION_KEY
|
||||
|
||||
VIDEO_EXTENSIONS = (".mp4", ".mov", ".m4v", ".webm", ".mkv", ".avi", ".wmv")
|
||||
FFPROBE_TIMEOUT = 20 # seconds; a stuck probe must not hang a worker thread
|
||||
_COMMENT_TAGS = ("comment", "description")
|
||||
_METADATA_KEYS = ("prompt", "workflow", "parameters")
|
||||
# Carrier order: per-key tags, then the ``comment`` envelope, then ``description``.
|
||||
ROLE_TAG_KEYS = ("prompt", "parameters", EXECUTION_KEY)
|
||||
METADATA_TAG_KEYS = ("prompt", "workflow", "parameters", EXECUTION_KEY)
|
||||
ENVELOPE_TAG_KEYS = ("comment", "description")
|
||||
VIDEO_TAG_KEYS = METADATA_TAG_KEYS + ENVELOPE_TAG_KEYS
|
||||
_NO_WINDOW = getattr(subprocess, "CREATE_NO_WINDOW", 0) # no console flash on Windows
|
||||
|
||||
logger = logging.getLogger("prompt_manager.video")
|
||||
@@ -46,32 +55,122 @@ def reset_missing_notice():
|
||||
_missing_notice_shown = False
|
||||
|
||||
|
||||
def metadata_from_tags(tags) -> dict:
|
||||
"""Normalise container tags into PNG-text-chunk shaped metadata.
|
||||
class DuplicateVideoTagError(ValueError):
|
||||
"""Two container tags fold to the same metadata key with different text."""
|
||||
|
||||
Returns ``{}`` when no tag carries a JSON object with a known key.
|
||||
|
||||
def _folded_tags(tags: dict) -> dict:
|
||||
"""Known tags keyed by their carrier name.
|
||||
|
||||
Per-key prompt carriers fold ASCII case: Matroska muxers upper-case tag
|
||||
names and ffprobe reports them as written. Two tags folding to one key are
|
||||
a duplicate record: byte-identical duplicates are one record (as for PNG
|
||||
text chunks), differing ones fail the read. Envelope keys (``comment``,
|
||||
``description``) are read exactly as the legacy readers did, never folded.
|
||||
"""
|
||||
if not isinstance(tags, dict):
|
||||
return {}
|
||||
for tag in _COMMENT_TAGS:
|
||||
raw = tags.get(tag)
|
||||
folded = {}
|
||||
for key, value in tags.items():
|
||||
if not isinstance(key, str):
|
||||
continue
|
||||
if key in ENVELOPE_TAG_KEYS:
|
||||
folded[key] = value
|
||||
continue
|
||||
name = key.lower()
|
||||
if name not in METADATA_TAG_KEYS:
|
||||
continue
|
||||
if name in folded and folded[name] != value:
|
||||
raise DuplicateVideoTagError(f"Duplicate video metadata tag: {name}")
|
||||
folded[name] = value
|
||||
return folded
|
||||
|
||||
|
||||
def _record(value, key):
|
||||
"""One carrier field as (metadata text or None, record format or None).
|
||||
|
||||
A null marker (or a non-text role value) has no record format: it is kept
|
||||
in the metadata as ``None`` so the inspector refuses it instead of letting
|
||||
fallback evidence win, while scans drop it.
|
||||
"""
|
||||
if key == "workflow" and not isinstance(value, (str, dict)):
|
||||
return None, "invalid"
|
||||
if isinstance(value, str):
|
||||
return value, "original-text"
|
||||
if isinstance(value, dict):
|
||||
return json.dumps(value), "reserialized-json"
|
||||
return None, None
|
||||
|
||||
|
||||
def _carriers(folded: dict):
|
||||
"""Candidate carriers in order: (fields, raw envelope text or None)."""
|
||||
per_key = {key: folded[key] for key in METADATA_TAG_KEYS if key in folded}
|
||||
if per_key:
|
||||
yield per_key, None
|
||||
for tag in ENVELOPE_TAG_KEYS:
|
||||
raw = folded.get(tag)
|
||||
if not isinstance(raw, str):
|
||||
continue
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
envelope = json.loads(raw)
|
||||
except ValueError:
|
||||
continue
|
||||
if not isinstance(data, dict):
|
||||
if isinstance(envelope, dict):
|
||||
yield envelope, raw
|
||||
|
||||
|
||||
def select_video_carrier(tags):
|
||||
"""Choose the one container carrier of prompt metadata.
|
||||
|
||||
Returns ``(metadata, record_formats, carrier_text)`` or ``None`` when no
|
||||
carrier holds a known key. ``metadata`` maps known keys to their text, or
|
||||
to ``None`` for an explicit null marker that the inspector must refuse;
|
||||
``record_formats`` is ``original-text``, ``reserialized-json`` (an envelope
|
||||
object) or ``invalid`` (a workflow that is neither text nor an object), and
|
||||
``carrier_text`` is the exact envelope for reserialized values (``None``
|
||||
for per-key tags). The first carrier holding a role key wins; otherwise the
|
||||
first holding ``workflow``. Carriers are never merged.
|
||||
Raises :class:`DuplicateVideoTagError` for differing fold-duplicate tags.
|
||||
"""
|
||||
if not isinstance(tags, dict):
|
||||
return None
|
||||
candidates = list(_carriers(_folded_tags(tags)))
|
||||
selected = next(
|
||||
(item for item in candidates if any(key in item[0] for key in ROLE_TAG_KEYS)),
|
||||
None,
|
||||
)
|
||||
if selected is None:
|
||||
selected = next((item for item in candidates if "workflow" in item[0]), None)
|
||||
if selected is None:
|
||||
return None
|
||||
fields, carrier_text = selected
|
||||
metadata, formats = {}, {}
|
||||
for key in METADATA_TAG_KEYS:
|
||||
if key not in fields:
|
||||
continue
|
||||
found = {}
|
||||
for key in _METADATA_KEYS:
|
||||
value = data.get(key)
|
||||
if value is None:
|
||||
continue
|
||||
found[key] = value if isinstance(value, str) else json.dumps(value)
|
||||
if found:
|
||||
return found
|
||||
return {}
|
||||
text, record_format = _record(fields[key], key)
|
||||
if record_format == "invalid":
|
||||
formats[key] = record_format
|
||||
continue
|
||||
metadata[key] = text
|
||||
if record_format is not None:
|
||||
formats[key] = record_format
|
||||
return metadata, formats, carrier_text
|
||||
|
||||
|
||||
def metadata_from_tags(tags) -> dict:
|
||||
"""Normalise container tags into PNG-text-chunk shaped metadata for scans.
|
||||
|
||||
Returns ``{}`` when no carrier holds a known key, and when fold-duplicate
|
||||
tags make the file's evidence untrustworthy.
|
||||
"""
|
||||
try:
|
||||
selected = select_video_carrier(tags)
|
||||
except DuplicateVideoTagError as e:
|
||||
logger.debug(f"Video tags skipped: {e}")
|
||||
return {}
|
||||
if selected is None:
|
||||
return {}
|
||||
metadata, _, _ = selected
|
||||
return {key: value for key, value in metadata.items() if isinstance(value, str)}
|
||||
|
||||
|
||||
def read_video_metadata(path, timeout=FFPROBE_TIMEOUT) -> dict:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
/**
|
||||
* State of the output-image scan as seen by the dashboard.
|
||||
*
|
||||
* The scan runs on the server as a background job (see py/api/scan_job.py) and
|
||||
* The scan runs on the server as a background job (see py/api/jobs.py) and
|
||||
* reports progress over SSE. This module turns those events into a small
|
||||
* immutable state object and answers the UI questions that depend on it:
|
||||
* what dismissing the modal should do, what the minimized pill should say, and
|
||||
|
||||
@@ -0,0 +1,445 @@
|
||||
/**
|
||||
* One job's Activity view (#249 Activity-1, D56): `#/activity/<id>`, the approved mockup `activity-scan.html` in the
|
||||
* concept's classes.
|
||||
*
|
||||
* - Header: the page heading names the job; its line is the Activity item's (D54), and its actions are Download report
|
||||
* (the job's JSON report, D59) and, while the job runs, "Cancel scan" (danger).
|
||||
* - Progress (`section.surface`): `<progress>`, the server's "Processing file N of M · about X min left", and the
|
||||
* outcome counters as `dl.stat-row` tiles: Added to Library (green), Already known, Needs review (warning), No usable
|
||||
* metadata, Errors (red).
|
||||
* - Results (`section.surface`): the outcome list (`button[aria-pressed]` with counts, in the mockup's order), a path
|
||||
* filter, and the selected outcome's listed rows (at most 200 per outcome, so "Showing N of M"). Rows under Added and
|
||||
* Already known open the file in the existing viewer through its listing identity (D57); the others are plain text.
|
||||
* - Log: `details.term` "Scan log", collapsed, holding the raw timestamped lines in `pre[tabindex=0]`.
|
||||
*
|
||||
* The summary follows the Activity store (one SSE stream per running job); the listed rows and the log are read from
|
||||
* `GET /prompt_manager/jobs/<id>` at most once a second while the job runs, and once more when it ends.
|
||||
*/
|
||||
import { createButton } from '../shared/toolkit/core.js';
|
||||
import { pageHeading } from './heading.mjs';
|
||||
import { showWorkspaceToast } from './toast.mjs';
|
||||
import { noun } from './plural.mjs';
|
||||
import { parseJob, JOBS_URL, LOST_DETAIL } from './activity-store.mjs';
|
||||
import { jobDetail, CANCEL_FAILED_TOAST } from './activity.mjs';
|
||||
import { downloadJobReport } from './activity-scan.mjs';
|
||||
import { createMediaViewer } from './media-viewer.mjs';
|
||||
import { createInspectionController } from './inspection-state.mjs';
|
||||
import { createPromptSender } from './prompt-sender.mjs';
|
||||
import { imageKey } from './images-state.mjs';
|
||||
|
||||
export const DETAIL_INTERVAL_MS = 1000;
|
||||
export const NOT_FOUND = 'This job is not in this session’s Activity. Jobs are kept until ComfyUI restarts.';
|
||||
export const EMPTY_GROUP = 'No files in this group.';
|
||||
export const NO_MATCH = 'No listed file matches this filter.';
|
||||
export const VIEWER_HINT = 'each row opens the image in the viewer';
|
||||
export const FILE_GONE = 'This file is no longer in Images. Rescan from Import and try again.';
|
||||
export const OPEN_FAILED = 'The file could not be opened. Try again.';
|
||||
const LISTING_LIMIT = 5;
|
||||
|
||||
/** The counter tiles, in the spec's order and the mockup's tones. */
|
||||
export const TILES = Object.freeze([
|
||||
['added', 'Added to Library', 'added'], ['known', 'Already known', ''], ['review', 'Needs review', 'warn'],
|
||||
['no_metadata', 'No usable metadata', ''], ['errors', 'Errors', 'bad'],
|
||||
]);
|
||||
/** The outcome list, in the mockup's order: what got added and what went wrong first. */
|
||||
export const OUTCOMES = Object.freeze([
|
||||
['added', 'Added to Library'], ['review', 'Needs review'], ['errors', 'Errors'],
|
||||
['no_metadata', 'No usable metadata'], ['known', 'Already known'],
|
||||
]);
|
||||
const OPENABLE = new Set(['added', 'known']);
|
||||
|
||||
function element(tag, text, className) {
|
||||
const node = document.createElement(tag);
|
||||
if (text !== undefined) node.textContent = text;
|
||||
if (className) node.className = className;
|
||||
return node;
|
||||
}
|
||||
|
||||
// Grouped as the server writes its lines, whatever the browser's locale (as images.mjs does).
|
||||
const number = value => value.toLocaleString('en-US');
|
||||
const isText = value => typeof value === 'string';
|
||||
|
||||
/** A job's detail read: the summary plus its listed rows per outcome and its log; null when it is not one. */
|
||||
export function parseDetail(raw) {
|
||||
const job = parseJob(raw);
|
||||
if (!job) return null;
|
||||
const outcomes = {};
|
||||
const groups = raw.outcomes !== null && typeof raw.outcomes === 'object' ? raw.outcomes : {};
|
||||
for (const [group, rows] of Object.entries(groups)) {
|
||||
if (!Array.isArray(rows)) continue;
|
||||
outcomes[group] = rows.filter(row => row && isText(row.path) && row.path)
|
||||
.map(row => ({ path: row.path, root: isText(row.root) ? row.root : '',
|
||||
root_index: Number.isSafeInteger(row.root_index) && row.root_index >= 0 ? row.root_index : null,
|
||||
detail: isText(row.detail) ? row.detail : '' }));
|
||||
}
|
||||
const log = Array.isArray(raw.log) ? raw.log.filter(isText) : [];
|
||||
return { job, outcomes, log };
|
||||
}
|
||||
|
||||
/** The progress line: the server's while running, then what the job got through. */
|
||||
export function progressLine(job) {
|
||||
const files = `${number(job.processed)} of ${number(job.total)} ${noun(job.total, 'file')}`;
|
||||
if (job.state === 'running') return job.status_line || `${job.progress}%`;
|
||||
if (job.state === 'finished') return `${files} processed`;
|
||||
if (job.state === 'cancelled') return `Cancelled after ${files}`;
|
||||
if (job.state === 'lost') return LOST_DETAIL;
|
||||
return job.status_line || 'The job stopped on an error.';
|
||||
}
|
||||
|
||||
/** The note under the rows: "Showing N of M" when not every file is listed, and the viewer hint for openable rows. */
|
||||
export function resultsNote(shown, total, openable) {
|
||||
const parts = [];
|
||||
if (shown < total) parts.push(`Showing ${number(shown)} of ${number(total)}`);
|
||||
if (openable && shown > 0) parts.push(VIEWER_HINT);
|
||||
if (!parts.length) return '';
|
||||
const text = parts.join(' · ');
|
||||
return parts.length === 1 && openable && shown >= total ? `${text[0].toUpperCase()}${text.slice(1)}.` : text;
|
||||
}
|
||||
|
||||
/** Rows whose path (or root label and path) holds the case-folded filter. */
|
||||
export function filterRows(rows, filter) {
|
||||
const wanted = filter.trim().toLocaleLowerCase();
|
||||
if (!wanted) return rows;
|
||||
return rows.filter(row => `${row.root}/${row.path}`.toLocaleLowerCase().includes(wanted));
|
||||
}
|
||||
|
||||
/** The first outcome with files, so a view opens on what happened; Added when nothing has yet. */
|
||||
export function defaultOutcome(counters) {
|
||||
return OUTCOMES.find(([group]) => (counters[group] ?? 0) > 0)?.[0] ?? 'added';
|
||||
}
|
||||
|
||||
function section(title, id) {
|
||||
const node = element('section', undefined, 'surface');
|
||||
const heading = element('div', undefined, 'panel-heading');
|
||||
const h2 = element('h2', title);
|
||||
h2.id = id;
|
||||
node.setAttribute('aria-labelledby', id);
|
||||
heading.append(h2);
|
||||
node.append(heading);
|
||||
return { node, heading };
|
||||
}
|
||||
|
||||
/** `#/activity/<id>`: one job's view. */
|
||||
export function mountActivityJobView(main, { store, scope, api, jobId }) {
|
||||
const parts = pageHeading(main);
|
||||
const download = createButton('Download report');
|
||||
const cancel = createButton('Cancel scan', 'danger');
|
||||
|
||||
const progressPart = section('Progress', 'job-progress-title');
|
||||
const status = element('span', '', 'small');
|
||||
progressPart.heading.append(status);
|
||||
const bar = element('progress');
|
||||
bar.max = 100;
|
||||
const stats = element('dl', undefined, 'stat-row');
|
||||
const tileValues = new Map();
|
||||
for (const [group, label, tone] of TILES) {
|
||||
const tile = element('div', undefined, tone ? `stat ${tone}` : 'stat');
|
||||
const value = element('dd', '0');
|
||||
tile.append(element('dt', label), value);
|
||||
tileValues.set(group, value);
|
||||
stats.append(tile);
|
||||
}
|
||||
progressPart.node.append(bar, stats);
|
||||
|
||||
const resultsPart = section('Results', 'job-results-title');
|
||||
const search = element('label', undefined, 'search-field');
|
||||
const input = element('input');
|
||||
input.type = 'search';
|
||||
input.placeholder = 'Filter by file path…';
|
||||
input.setAttribute('aria-label', 'Filter results');
|
||||
search.append(input);
|
||||
resultsPart.heading.append(search);
|
||||
const outcomes = element('div', undefined, 'outcomes');
|
||||
const list = element('div', undefined, 'outcome-list');
|
||||
list.setAttribute('role', 'group');
|
||||
list.setAttribute('aria-label', 'Outcomes');
|
||||
const files = element('div', undefined, 'outcome-files');
|
||||
const buttons = new Map();
|
||||
for (const [group, label] of OUTCOMES) {
|
||||
const button = createButton('');
|
||||
const count = element('span', '0');
|
||||
button.append(element('span', label), count);
|
||||
button.setAttribute('aria-pressed', 'false');
|
||||
button.count = count;
|
||||
buttons.set(group, button);
|
||||
list.append(button);
|
||||
}
|
||||
outcomes.append(list, files);
|
||||
resultsPart.node.append(outcomes);
|
||||
|
||||
const log = element('details', undefined, 'term');
|
||||
const summary = element('summary');
|
||||
const logLabel = document.createTextNode('');
|
||||
summary.append(logLabel, element('span', ' — raw detail, click to show'));
|
||||
const pre = element('pre', '');
|
||||
pre.tabIndex = 0;
|
||||
log.append(summary, pre);
|
||||
|
||||
const missing = element('section', undefined, 'surface');
|
||||
const back = element('a', 'All activity');
|
||||
back.href = '#/activity';
|
||||
missing.append(element('p', NOT_FOUND, 'empty-state'), back);
|
||||
|
||||
let detail = null;
|
||||
let selected = null;
|
||||
let mounted = false;
|
||||
let lastRead = 0;
|
||||
let reading = false;
|
||||
let readAgain = false;
|
||||
let readTimer = null;
|
||||
let roots = null;
|
||||
let lastState = null;
|
||||
// Set once this page's own list read answered, or the detail read said the job is unknown.
|
||||
let listedOnce = false;
|
||||
let notFound = false;
|
||||
// The store is the truth once it has read the list: a job it no longer has was pruned by the server.
|
||||
const current = () => store.find(jobId) ?? (listedOnce ? null : detail?.job ?? null);
|
||||
|
||||
const inspection = createInspectionController({ api });
|
||||
scope.cleanup(() => inspection.dispose());
|
||||
const viewer = createMediaViewer({ api, container: main, parentScope: scope, inspectionController: inspection,
|
||||
sender: createPromptSender({ scope }), onSelect: image => inspection.select(image) });
|
||||
|
||||
function showBody(job) {
|
||||
if (mounted) return;
|
||||
mounted = true;
|
||||
missing.remove();
|
||||
parts.actions.append(download, cancel);
|
||||
main.append(progressPart.node, resultsPart.node, log);
|
||||
selected ??= defaultOutcome(job.counters);
|
||||
}
|
||||
|
||||
/** The job is not (or no longer) in this session: say so instead of an empty view. */
|
||||
function showMissing() {
|
||||
if (mounted) {
|
||||
mounted = false;
|
||||
const focused = main.contains(document.activeElement) && document.activeElement !== parts.h1;
|
||||
download.remove();
|
||||
cancel.remove();
|
||||
progressPart.node.remove();
|
||||
resultsPart.node.remove();
|
||||
log.remove();
|
||||
parts.h1.textContent = 'Activity';
|
||||
parts.description.textContent = '';
|
||||
document.title = 'Activity · Prompt Manager';
|
||||
if (focused) parts.h1.focus();
|
||||
}
|
||||
if (!missing.isConnected) main.append(missing);
|
||||
}
|
||||
|
||||
// Listed rows only ever grow, so a row keeps its node across reads: a focused row (or the viewer's trigger) stays.
|
||||
let rowNodes = new Map();
|
||||
function rowNode(row, index, openable) {
|
||||
const key = `${selected}\n${index}\n${row.root}\n${row.path}`;
|
||||
let node = rowNodes.get(key);
|
||||
if (!node) {
|
||||
node = openable ? createButton('', 'file-row') : element('div', undefined, 'file-row');
|
||||
node.append(element('span', row.path, 'path'), element('span', row.detail, 'why'));
|
||||
if (openable) node.row = row;
|
||||
}
|
||||
return [key, node];
|
||||
}
|
||||
|
||||
function renderRows(job) {
|
||||
const rows = detail?.outcomes[selected] ?? [];
|
||||
const openable = OPENABLE.has(selected);
|
||||
const wanted = filterRows(rows.map((row, index) => ({ ...row, index })), input.value);
|
||||
const kept = new Map(wanted.map(row => rowNode(row, row.index, openable)));
|
||||
rowNodes = kept;
|
||||
const nodes = [...kept.values()];
|
||||
// The counter is exact; at most 200 of its files are listed, and the filter narrows those.
|
||||
const total = Math.max(job.counters[selected] ?? 0, rows.length);
|
||||
if (!nodes.length) nodes.push(element('p', rows.length ? NO_MATCH : EMPTY_GROUP, 'empty-state'));
|
||||
const note = wanted.length ? resultsNote(wanted.length, total, openable) : '';
|
||||
if (note) nodes.push(element('p', note, 'small'));
|
||||
for (const child of [...files.children]) if (!nodes.includes(child)) child.remove();
|
||||
nodes.forEach((node, index) => {
|
||||
// Only a node out of place moves, so focus survives a re-read.
|
||||
if (files.children[index] !== node) files.insertBefore(node, files.children[index] ?? null);
|
||||
});
|
||||
}
|
||||
|
||||
function render() {
|
||||
const job = current();
|
||||
if (!job) {
|
||||
if (listedOnce || notFound) showMissing();
|
||||
return;
|
||||
}
|
||||
showBody(job);
|
||||
const scan = job.kind === 'scan';
|
||||
parts.h1.textContent = job.title;
|
||||
document.title = `${job.title} · Prompt Manager`;
|
||||
parts.description.textContent = jobDetail(job);
|
||||
cancel.textContent = scan ? 'Cancel scan' : 'Cancel job';
|
||||
// A job lost with the server has no report there (review M9).
|
||||
const lost = job.state === 'lost';
|
||||
if (lost && document.activeElement === download) parts.h1.focus();
|
||||
download.disabled = lost;
|
||||
if (job.state === 'running') {
|
||||
if (!cancel.isConnected) parts.actions.append(cancel);
|
||||
} else if (cancel.isConnected) {
|
||||
// A focused Cancel that goes away hands its focus to the job's heading, never to the page body.
|
||||
if (document.activeElement === cancel) parts.h1.focus();
|
||||
cancel.remove();
|
||||
}
|
||||
status.textContent = progressLine(job);
|
||||
bar.value = job.progress;
|
||||
bar.textContent = `${job.progress}%`;
|
||||
bar.setAttribute('aria-label', scan ? 'Scan progress' : `${job.title} progress`);
|
||||
for (const [group, value] of tileValues) value.textContent = number(job.counters[group] ?? 0);
|
||||
for (const [group, button] of buttons) {
|
||||
button.count.textContent = number(job.counters[group] ?? 0);
|
||||
button.setAttribute('aria-pressed', String(group === selected));
|
||||
}
|
||||
logLabel.textContent = `● ${scan ? 'Scan' : 'Job'} log`;
|
||||
const text = detail?.log.length ? detail.log.join('\n') : 'No log lines yet.';
|
||||
if (pre.textContent !== text) {
|
||||
// Replacing the text clamps the scroll position, so keep it: pinned readers follow the end, others stay.
|
||||
const { scrollTop } = pre;
|
||||
const atEnd = pre.scrollTop + pre.clientHeight >= pre.scrollHeight - 2;
|
||||
pre.textContent = text;
|
||||
pre.scrollTop = atEnd ? pre.scrollHeight : scrollTop;
|
||||
}
|
||||
renderRows(job);
|
||||
}
|
||||
|
||||
/** Read the rows and the log, at most once a second; a request while one is in flight reads once more after it. */
|
||||
function scheduleRead() {
|
||||
if (scope.signal.aborted) return;
|
||||
if (reading) { readAgain = true; return; }
|
||||
if (readTimer !== null) return;
|
||||
const wait = Math.max(0, lastRead + DETAIL_INTERVAL_MS - Date.now());
|
||||
readTimer = setTimeout(() => { readTimer = null; read(); }, wait);
|
||||
}
|
||||
scope.cleanup(() => { if (readTimer !== null) clearTimeout(readTimer); });
|
||||
|
||||
async function read() {
|
||||
reading = true;
|
||||
lastRead = Date.now();
|
||||
try {
|
||||
const result = await api.get(`${JOBS_URL}/${jobId}`, { signal: scope.signal, cache: 'no-store' });
|
||||
if (scope.signal.aborted) return;
|
||||
const parsed = parseDetail(result?.job);
|
||||
if (parsed && parsed.job.id === jobId) detail = parsed;
|
||||
} catch (error) {
|
||||
if (scope.signal.aborted) return;
|
||||
// An unknown job: the page says so unless the store still knows it (then the rows wait for a later read).
|
||||
if (error?.status === 404) notFound = true;
|
||||
} finally {
|
||||
reading = false;
|
||||
}
|
||||
render();
|
||||
// Read once more only if the summary moved past what this read returned (review: no redundant re-read).
|
||||
if (readAgain) {
|
||||
readAgain = false;
|
||||
const job = store.find(jobId);
|
||||
if (job && job.state !== 'lost' && detailChanged(job)) scheduleRead();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the detail read would change the page: more listed rows in any outcome, or (with the log open) more log
|
||||
* lines. Counters come with the summary, so a job whose lists are capped and whose log is closed is not re-read.
|
||||
*/
|
||||
function detailChanged(job) {
|
||||
if (!detail) return true;
|
||||
const groups = new Set([...Object.keys(job.listed), ...Object.keys(detail.outcomes)]);
|
||||
for (const group of groups) if ((job.listed[group] ?? 0) !== (detail.outcomes[group]?.length ?? 0)) return true;
|
||||
return log.open && job.log_length !== detail.log.length;
|
||||
}
|
||||
|
||||
function follow() {
|
||||
const job = store.find(jobId);
|
||||
render();
|
||||
if (!job) return;
|
||||
// A job lost with the server cannot be read; any other is read while what it lists changes.
|
||||
if (job.state !== 'lost' && (job.state === 'running' || lastState === 'running') && detailChanged(job)) scheduleRead();
|
||||
lastState = job.state;
|
||||
}
|
||||
|
||||
scope.listen(log, 'toggle', () => {
|
||||
// Every open starts at the newest line: while closed the log has no layout box (Firefox, WebKit), so a kept
|
||||
// position would be lost anyway (review M6).
|
||||
if (log.open) pre.scrollTop = pre.scrollHeight;
|
||||
const job = store.find(jobId);
|
||||
if (log.open && job && job.state !== 'lost' && detailChanged(job)) scheduleRead();
|
||||
});
|
||||
|
||||
scope.listen(download, 'click', async () => {
|
||||
if (download.disabled || download.getAttribute('aria-disabled') === 'true') return;
|
||||
download.setAttribute('aria-disabled', 'true');
|
||||
try {
|
||||
// A view left while the report downloads shows no toast (review M10).
|
||||
await downloadJobReport(jobId, { signal: scope.signal });
|
||||
} finally {
|
||||
download.removeAttribute('aria-disabled');
|
||||
}
|
||||
});
|
||||
scope.listen(cancel, 'click', async () => {
|
||||
if (cancel.getAttribute('aria-disabled') === 'true') return;
|
||||
cancel.setAttribute('aria-disabled', 'true');
|
||||
try {
|
||||
await store.cancel(jobId);
|
||||
} catch {
|
||||
if (!scope.signal.aborted) showWorkspaceToast(CANCEL_FAILED_TOAST);
|
||||
} finally {
|
||||
cancel.removeAttribute('aria-disabled');
|
||||
}
|
||||
});
|
||||
scope.listen(list, 'click', event => {
|
||||
const button = event.target.closest('button');
|
||||
if (!button) return;
|
||||
selected = [...buttons].find(([, node]) => node === button)?.[0] ?? selected;
|
||||
render();
|
||||
});
|
||||
scope.listen(input, 'input', () => { const job = current(); if (job) renderRows(job); });
|
||||
scope.listen(files, 'click', event => {
|
||||
const button = event.target.closest('button.file-row');
|
||||
if (button?.row) openRow(button.row, button);
|
||||
});
|
||||
|
||||
/**
|
||||
* The listing's own item for a row, by its exact identity (D57): the root index the scan recorded, still carrying
|
||||
* the row's label (so a reordered or replaced source never opens another root's file), and its exact relative path.
|
||||
*/
|
||||
async function listed(row) {
|
||||
if (row.root_index === null) return null;
|
||||
if (!roots) {
|
||||
const data = await api.get('/prompt_manager/gallery/subfolders?mode=workspace', { signal: scope.signal });
|
||||
roots = Array.isArray(data?.roots) ? data.roots : [];
|
||||
}
|
||||
const root = roots.find(item => item?.index === row.root_index);
|
||||
if (!root || root.label !== row.root) return null;
|
||||
const params = new URLSearchParams({ mode: 'workspace', root: String(root.index), path: row.path,
|
||||
limit: String(LISTING_LIMIT) });
|
||||
const page = await api.get(`/prompt_manager/images/output?${params}`, { signal: scope.signal });
|
||||
return (Array.isArray(page?.images) ? page.images : []).find(item => item?.path === row.path
|
||||
&& item.root_index === root.index) ?? null;
|
||||
}
|
||||
|
||||
async function openRow(row, trigger) {
|
||||
if (trigger.getAttribute('aria-disabled') === 'true') return;
|
||||
trigger.setAttribute('aria-disabled', 'true');
|
||||
try {
|
||||
const item = await listed(row);
|
||||
if (scope.signal.aborted) return;
|
||||
if (!item) { showWorkspaceToast(FILE_GONE); return; }
|
||||
viewer.open({ items: [item], selectedKey: imageKey(item), trigger });
|
||||
} catch {
|
||||
if (!scope.signal.aborted) showWorkspaceToast(OPEN_FAILED);
|
||||
} finally {
|
||||
trigger.removeAttribute('aria-disabled');
|
||||
}
|
||||
}
|
||||
|
||||
lastState = store.find(jobId)?.state ?? null;
|
||||
scope.cleanup(store.subscribe(follow));
|
||||
render();
|
||||
read();
|
||||
// A job that finished before this page loaded is only in the list read; once it answered, the store decides.
|
||||
store.load().then(() => {
|
||||
if (scope.signal.aborted || store.status !== 'ready') return;
|
||||
listedOnce = true;
|
||||
render();
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
/**
|
||||
* Scan for metadata (#249 Activity-1, D55): Images → Import → "Scan for metadata" → Review scan opens the concept's
|
||||
* "Review metadata scan". Its backup box downloads the existing `/prompt_manager/backup` copy and toasts the outcome;
|
||||
* Start scan starts the session job through the Activity store over the chosen scope (new and unresolved files, or
|
||||
* all files with the D58 checkbox), which toasts "Scan started.", and opens the job's view.
|
||||
* A start refused because another record-changing job runs toasts "Another job is running: <title>." with a link to
|
||||
* that job (D57).
|
||||
*/
|
||||
import { openScanReviewDialog } from './workspace-dialogs.mjs';
|
||||
import { showWorkspaceToast } from './toast.mjs';
|
||||
import { navigateTo } from './state.mjs';
|
||||
|
||||
export const BACKUP_URL = '/prompt_manager/backup';
|
||||
export const BACKUP_DONE = 'Database backup downloaded.';
|
||||
export const BACKUP_FAILED = 'The database could not be backed up. Try again.';
|
||||
export const REPORT_DONE = 'Job report downloaded.';
|
||||
export const REPORT_FAILED = 'The report could not be downloaded. Try again.';
|
||||
const JOBS_REPORT_BASE = '/prompt_manager/jobs';
|
||||
export const START_FAILED = 'The scan could not start. Try again.';
|
||||
const FALLBACK_NAME = 'prompts_backup.db';
|
||||
const REPORT_FALLBACK_NAME = 'prompt-manager-report.json';
|
||||
|
||||
/** The download name from a Content-Disposition header, or `fallback`. */
|
||||
export function backupFilename(disposition, fallback = FALLBACK_NAME) {
|
||||
const match = /filename="?([^";]+)"?/i.exec(disposition || '');
|
||||
return match ? match[1] : fallback;
|
||||
}
|
||||
|
||||
/** Save a fetched response through a temporary link, named by `filename`. */
|
||||
function saveResponse(blob, filename) {
|
||||
const url = URL.createObjectURL(blob);
|
||||
const link = document.createElement('a');
|
||||
link.href = url;
|
||||
link.download = filename;
|
||||
document.body.append(link);
|
||||
link.click();
|
||||
link.remove();
|
||||
setTimeout(() => URL.revokeObjectURL(url), 1000);
|
||||
}
|
||||
|
||||
/** Download the prompts database copy the legacy admin page also uses; toasts the outcome. */
|
||||
export async function downloadDatabaseBackup({ fetchUrl = window.ApiClient.fetchUrl, notify = showWorkspaceToast } = {}) {
|
||||
try {
|
||||
const response = await fetchUrl(BACKUP_URL);
|
||||
if (!response.ok) throw new Error(`Backup failed with ${response.status}`);
|
||||
const blob = await response.blob();
|
||||
saveResponse(blob, backupFilename(response.headers.get('Content-Disposition')));
|
||||
notify(BACKUP_DONE);
|
||||
return true;
|
||||
} catch {
|
||||
notify(BACKUP_FAILED);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Download a job's JSON report (Activity-2, D59) through the api-client URL, so a proxy mount works; toasts the outcome
|
||||
* unless `signal` (the view's scope) was aborted meanwhile.
|
||||
*/
|
||||
export async function downloadJobReport(jobId, { fetchUrl = window.ApiClient.fetchUrl, notify = showWorkspaceToast,
|
||||
signal = null } = {}) {
|
||||
try {
|
||||
const response = await fetchUrl(`${JOBS_REPORT_BASE}/${encodeURIComponent(jobId)}/report`);
|
||||
if (!response.ok) throw new Error(`Report failed with ${response.status}`);
|
||||
const blob = await response.blob();
|
||||
saveResponse(blob, backupFilename(response.headers.get('Content-Disposition'), REPORT_FALLBACK_NAME));
|
||||
// The file is saved either way; a view left meanwhile (`signal` aborted) shows no toast.
|
||||
if (!signal?.aborted) notify(REPORT_DONE);
|
||||
return true;
|
||||
} catch {
|
||||
if (!signal?.aborted) notify(REPORT_FAILED);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** The toast for a refused start, linking the job that holds the slot. */
|
||||
export function busyToast(running) {
|
||||
if (!running) return 'Another job is running. Try again when it ends.';
|
||||
return ['Another job is running: ', { text: running.title, href: `#/activity/${running.id}` }, '.'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the scan job over `files` ('new': new and unresolved files, or 'all') and open its view; a refused or failed
|
||||
* start says why and stays on the page.
|
||||
*/
|
||||
export async function startMetadataScan({ store, scope, files = 'new', notify = showWorkspaceToast, navigate = navigateTo }) {
|
||||
try {
|
||||
const result = await store.start('scan', { scope: files });
|
||||
if (scope.signal.aborted) return;
|
||||
if (result.busy !== undefined) {
|
||||
notify(busyToast(result.busy));
|
||||
return;
|
||||
}
|
||||
navigate(`#/activity/${result.job.id}`);
|
||||
} catch {
|
||||
if (!scope.signal.aborted) notify(START_FAILED);
|
||||
}
|
||||
}
|
||||
|
||||
/** Open "Review metadata scan" for `sources` output folders. */
|
||||
export function openMetadataScan({ scope, store, sources, trigger = null }) {
|
||||
openScanReviewDialog({ scope, sources, trigger,
|
||||
onBackup: () => downloadDatabaseBackup(),
|
||||
onStart: ({ scope: files }) => startMetadataScan({ store, scope, files }) });
|
||||
}
|
||||
@@ -0,0 +1,313 @@
|
||||
/**
|
||||
* The Activity store (#249 Activity-1): this server session's jobs, for the top-bar button and pill and the Activity
|
||||
* page. One `GET /prompt_manager/jobs` read, then one SSE stream (`/jobs/<id>/events`) per running job.
|
||||
*
|
||||
* - A dropped stream re-reads the list after a backoff that doubles from 1 s and is capped at 30 s, so a stopped
|
||||
* server never sees a retry storm. A successful stream message resets the backoff.
|
||||
* - While the tab is visible the list is read again every 15 s, so a job started elsewhere (another tab, the legacy
|
||||
* admin page) appears without a reload.
|
||||
* - Jobs live only in server memory. A running job the server no longer lists (ComfyUI restarted), or one whose
|
||||
* server cannot be read at all, is shown as ended (`lost`), so no pill stays stuck.
|
||||
* - Toasts announce what happened; Activity is where progress lives. "Scan started." follows this tab's own start;
|
||||
* the finish toast follows any job this store saw running when it ends.
|
||||
*/
|
||||
import { showWorkspaceToast } from './toast.mjs';
|
||||
import { noun } from './plural.mjs';
|
||||
|
||||
export const JOBS_URL = '/prompt_manager/jobs';
|
||||
export const BACKOFF_FIRST_MS = 1000;
|
||||
export const BACKOFF_MAX_MS = 30000;
|
||||
export const POLL_MS = 15000;
|
||||
export const LOST_AFTER_FAILURES = 2;
|
||||
export const LOST_DETAIL = 'This job ended when ComfyUI restarted.';
|
||||
const SERVER_STATES = new Set(['running', 'finished', 'cancelled', 'failed']);
|
||||
const FINAL_STATES = new Set(['finished', 'cancelled', 'failed']);
|
||||
const JOB_ID = /^[0-9a-f]{32}$/;
|
||||
// The pill's verb for each job kind; a kind without one shows its title.
|
||||
const PILL_VERBS = Object.freeze({ scan: 'Scanning' });
|
||||
const START_TOASTS = Object.freeze({ scan: 'Scan started.' });
|
||||
|
||||
/** The wait before the `attempt`-th re-read (0-based): 1 s, 2 s, 4 s … never more than 30 s. */
|
||||
export function backoffDelay(attempt) {
|
||||
const step = Number.isSafeInteger(attempt) && attempt > 0 ? Math.min(attempt, 16) : 0;
|
||||
return Math.min(BACKOFF_MAX_MS, BACKOFF_FIRST_MS * 2 ** step);
|
||||
}
|
||||
|
||||
const count = value => Number.isSafeInteger(value) && value >= 0 ? value : 0;
|
||||
const time = value => typeof value === 'number' && Number.isFinite(value) ? value : null;
|
||||
const plainObject = value => value !== null && typeof value === 'object' && !Array.isArray(value);
|
||||
|
||||
/** A job summary from the API, or null when it is not one (unknown state, bad id). */
|
||||
export function parseJob(raw) {
|
||||
if (!plainObject(raw) || typeof raw.id !== 'string' || !JOB_ID.test(raw.id) || !SERVER_STATES.has(raw.state)) return null;
|
||||
const counters = {};
|
||||
if (plainObject(raw.counters)) for (const [key, value] of Object.entries(raw.counters)) counters[key] = count(value);
|
||||
// What the job's detail lists per outcome, and its log length: a view re-reads the detail only when they change.
|
||||
const listed = {};
|
||||
if (plainObject(raw.listed)) for (const [key, value] of Object.entries(raw.listed)) listed[key] = count(value);
|
||||
const progress = typeof raw.progress === 'number' && Number.isFinite(raw.progress) ? Math.round(raw.progress) : 0;
|
||||
return {
|
||||
id: raw.id,
|
||||
kind: typeof raw.kind === 'string' ? raw.kind : '',
|
||||
title: typeof raw.title === 'string' && raw.title.trim() ? raw.title : 'Background job',
|
||||
state: raw.state,
|
||||
started_at: time(raw.started_at),
|
||||
finished_at: time(raw.finished_at),
|
||||
progress: Math.max(0, Math.min(100, progress)),
|
||||
status_line: typeof raw.status_line === 'string' ? raw.status_line : '',
|
||||
processed: count(raw.processed),
|
||||
total: count(raw.total),
|
||||
counters,
|
||||
scope: plainObject(raw.scope) ? { ...raw.scope } : {},
|
||||
listed,
|
||||
log_length: count(raw.log_length),
|
||||
};
|
||||
}
|
||||
|
||||
// Grouped as the server writes its status lines ("8,240 of 19,622"), whatever the browser's locale.
|
||||
const number = value => value.toLocaleString('en-US');
|
||||
const errorsOf = job => `${number(job.counters.errors ?? 0)} ${noun(job.counters.errors ?? 0, 'error')}`;
|
||||
/** "1,420 added". */
|
||||
export const addedOf = job => `${number(job.counters.added ?? 0)} added`;
|
||||
|
||||
/** The toast when a job ends: "Scan finished · N added · M errors", "Scan cancelled · N added". */
|
||||
export function finishToast(job) {
|
||||
if (job.kind === 'scan') {
|
||||
if (job.state === 'cancelled') return `Scan cancelled · ${addedOf(job)}`;
|
||||
if (job.state === 'failed') return `Scan failed · ${addedOf(job)} · ${errorsOf(job)}`;
|
||||
return `Scan finished · ${addedOf(job)} · ${errorsOf(job)}`;
|
||||
}
|
||||
if (job.state === 'cancelled') return `${job.title} was cancelled.`;
|
||||
if (job.state === 'failed') return `${job.title} failed.`;
|
||||
return `${job.title} finished.`;
|
||||
}
|
||||
|
||||
/** The toast after this tab starts a job. */
|
||||
export function startToast(job) {
|
||||
return START_TOASTS[job.kind] ?? `${job.title} started.`;
|
||||
}
|
||||
|
||||
/** The pill's text: "Scanning · 42% · 8,240 / 19,622"; the file counts only once the total is known. */
|
||||
export function pillParts(job) {
|
||||
const label = `${PILL_VERBS[job.kind] ?? job.title} · ${job.progress}%`;
|
||||
const files = job.total > 0 ? ` · ${number(job.processed)} / ${number(job.total)}` : '';
|
||||
return { label, files, text: label + files };
|
||||
}
|
||||
|
||||
/** The scan's counters as one line: "N added · M errors"; other kinds have none yet. */
|
||||
export function outcomeLine(job) {
|
||||
return job.kind === 'scan' ? `${addedOf(job)} · ${errorsOf(job)}` : '';
|
||||
}
|
||||
|
||||
function defaultConnect(url) {
|
||||
const resolve = globalThis.ApiClient?.resolveUrl ?? (value => value);
|
||||
return new EventSource(resolve(url));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {{api: {get: Function, post: Function}, scope: {signal: AbortSignal, cleanup: Function},
|
||||
* connect?: (url: string) => {onmessage: Function|null, onerror: Function|null, close: Function},
|
||||
* notify?: (message: string) => void, timers?: {setTimeout: Function, clearTimeout: Function},
|
||||
* pollMs?: number, isVisible?: () => boolean, lifecycle?: EventTarget}} options
|
||||
*
|
||||
* `lifecycle` (the window) ends the store's work on `pagehide`: the periodic and backoff reads stop, the streams close
|
||||
* and a read in flight is aborted, so nothing answers into a page that is going away (a closed tab, a test teardown).
|
||||
* A `pageshow` from the back-forward cache reads the list again.
|
||||
*/
|
||||
export function createActivityStore({ api, scope, connect = defaultConnect, notify = showWorkspaceToast,
|
||||
timers = globalThis, pollMs = POLL_MS, isVisible = () => globalThis.document?.visibilityState !== 'hidden',
|
||||
lifecycle = globalThis.window ?? null } = {}) {
|
||||
let jobs = [];
|
||||
let status = 'idle';
|
||||
let attempt = 0;
|
||||
// Failed list reads in a row: one (a proxy's 502, a dropped poll) is not yet a restart.
|
||||
let failures = 0;
|
||||
let retryTimer = null;
|
||||
let pollTimer = null;
|
||||
let reads = 0;
|
||||
let disposed = false;
|
||||
let suspended = false;
|
||||
let reading = null;
|
||||
const streams = new Map();
|
||||
const listeners = new Set();
|
||||
|
||||
const emit = () => { for (const listener of [...listeners]) listener(store); };
|
||||
const find = id => jobs.find(job => job.id === id);
|
||||
|
||||
function announce(previous, next) {
|
||||
if (previous?.state === 'running' && FINAL_STATES.has(next.state)) notify(finishToast(next));
|
||||
}
|
||||
|
||||
function closeStream(id) {
|
||||
const source = streams.get(id);
|
||||
if (!source) return;
|
||||
streams.delete(id);
|
||||
source.onmessage = null;
|
||||
source.onerror = null;
|
||||
source.close();
|
||||
}
|
||||
|
||||
function follow() {
|
||||
for (const id of [...streams.keys()]) if (find(id)?.state !== 'running') closeStream(id);
|
||||
for (const job of jobs) if (job.state === 'running' && !streams.has(job.id)) openStream(job.id);
|
||||
}
|
||||
|
||||
function openStream(id) {
|
||||
const source = connect(`${JOBS_URL}/${id}/events`);
|
||||
streams.set(id, source);
|
||||
source.onmessage = event => {
|
||||
let job;
|
||||
try { job = parseJob(JSON.parse(event.data)); } catch { job = null; }
|
||||
if (!job || job.id !== id) return;
|
||||
attempt = 0;
|
||||
upsert(job);
|
||||
};
|
||||
source.onerror = () => {
|
||||
closeStream(id);
|
||||
if (find(id)?.state === 'running') scheduleResync({ drop: true });
|
||||
};
|
||||
}
|
||||
|
||||
/** A job never runs again once it has ended, whatever an older read in flight still says. */
|
||||
const settled = (previous, next) => FINAL_STATES.has(previous?.state) && next.state === 'running' ? previous : next;
|
||||
|
||||
function upsert(job) {
|
||||
const previous = find(job.id);
|
||||
job = settled(previous, job);
|
||||
jobs = previous ? jobs.map(item => item.id === job.id ? job : item) : [job, ...jobs];
|
||||
announce(previous, job);
|
||||
follow();
|
||||
emit();
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace the jobs with the server's list. A running job it no longer has ended with the server and stays as
|
||||
* ended; a finished job it no longer lists was pruned there and is dropped here too.
|
||||
*/
|
||||
function merge(list) {
|
||||
const next = list.map(parseJob).filter(Boolean).map(job => settled(find(job.id), job));
|
||||
const ids = new Set(next.map(job => job.id));
|
||||
const carried = jobs.filter(job => !ids.has(job.id) && (job.state === 'running' || job.state === 'lost'))
|
||||
.map(job => ({ ...job, state: 'lost' }));
|
||||
for (const job of next) announce(find(job.id), job);
|
||||
jobs = [...next, ...carried];
|
||||
follow();
|
||||
}
|
||||
|
||||
function markLost() {
|
||||
jobs = jobs.map(job => job.state === 'running' ? { ...job, state: 'lost' } : job);
|
||||
follow();
|
||||
}
|
||||
|
||||
/** Re-read after the backoff; `drop` marks a re-read for a dropped stream, which keeps the backoff growing. */
|
||||
function scheduleResync({ drop = false } = {}) {
|
||||
if (disposed || suspended || retryTimer !== null) return;
|
||||
const delay = backoffDelay(attempt);
|
||||
attempt += 1;
|
||||
retryTimer = timers.setTimeout(() => {
|
||||
retryTimer = null;
|
||||
load({ drop });
|
||||
}, delay);
|
||||
}
|
||||
|
||||
/** The next periodic read; a hidden tab skips its turn. */
|
||||
function schedulePoll() {
|
||||
if (disposed || suspended || pollMs <= 0) return;
|
||||
if (pollTimer !== null) timers.clearTimeout(pollTimer);
|
||||
pollTimer = timers.setTimeout(() => {
|
||||
pollTimer = null;
|
||||
if (isVisible()) load();
|
||||
else schedulePoll();
|
||||
}, pollMs);
|
||||
}
|
||||
|
||||
/** Read this session's jobs. A failed read marks running jobs ended and tries again after the backoff. */
|
||||
async function load({ drop = false } = {}) {
|
||||
if (disposed || suspended) return;
|
||||
const read = ++reads;
|
||||
// Only the newest read applies, so an older one still in flight is aborted.
|
||||
reading?.abort();
|
||||
const controller = new AbortController();
|
||||
reading = controller;
|
||||
try {
|
||||
const result = await api.get(JOBS_URL, { signal: controller.signal, cache: 'no-store' });
|
||||
// An older read answering late is dropped.
|
||||
if (disposed || suspended || read !== reads) return;
|
||||
if (!Array.isArray(result?.jobs)) throw new TypeError('Invalid job list');
|
||||
merge(result.jobs);
|
||||
status = 'ready';
|
||||
// A good read ends the backoff (the server is back, or nothing dropped), except for a stream that keeps
|
||||
// dropping while the list reads fine: that waits longer each time, and a received message resets it.
|
||||
if (failures > 0 || !drop) attempt = 0;
|
||||
failures = 0;
|
||||
schedulePoll();
|
||||
} catch (error) {
|
||||
if (disposed || suspended || scope.signal?.aborted || controller.signal.aborted || read !== reads) return;
|
||||
failures += 1;
|
||||
if (failures >= LOST_AFTER_FAILURES) markLost();
|
||||
status = 'error';
|
||||
scheduleResync();
|
||||
}
|
||||
emit();
|
||||
}
|
||||
|
||||
const store = {
|
||||
get status() { return status; },
|
||||
jobs: () => [...jobs],
|
||||
running: () => jobs.filter(job => job.state === 'running'),
|
||||
find,
|
||||
load,
|
||||
subscribe(listener) {
|
||||
listeners.add(listener);
|
||||
return () => listeners.delete(listener);
|
||||
},
|
||||
/** Start a job of `kind`: {job}, or {busy, message} naming the running job that holds its slot. */
|
||||
async start(kind, body = {}) {
|
||||
try {
|
||||
const result = await api.post(`${JOBS_URL}/${encodeURIComponent(kind)}`, body, { signal: scope.signal });
|
||||
const job = parseJob(result?.job);
|
||||
if (!job) throw new TypeError('Invalid job');
|
||||
upsert(job);
|
||||
notify(startToast(job));
|
||||
return { job };
|
||||
} catch (error) {
|
||||
if (error?.status !== 409 || error?.code !== 'busy') throw error;
|
||||
await load();
|
||||
return { busy: store.running()[0] ?? null, message: error.message };
|
||||
}
|
||||
},
|
||||
/** Ask a running job to stop at its next file boundary; false once it has ended. */
|
||||
async cancel(id) {
|
||||
const result = await api.post(`${JOBS_URL}/${encodeURIComponent(id)}/cancel`, {}, { signal: scope.signal });
|
||||
return result?.cancellation_requested === true;
|
||||
},
|
||||
};
|
||||
|
||||
/** Stop every timer, stream and read in flight. */
|
||||
function halt() {
|
||||
if (retryTimer !== null) timers.clearTimeout(retryTimer);
|
||||
if (pollTimer !== null) timers.clearTimeout(pollTimer);
|
||||
retryTimer = null;
|
||||
pollTimer = null;
|
||||
reading?.abort();
|
||||
reading = null;
|
||||
for (const id of [...streams.keys()]) closeStream(id);
|
||||
}
|
||||
const onPageHide = () => { suspended = true; halt(); };
|
||||
const onPageShow = event => {
|
||||
if (!suspended || !event.persisted) return;
|
||||
suspended = false;
|
||||
load();
|
||||
};
|
||||
lifecycle?.addEventListener('pagehide', onPageHide);
|
||||
lifecycle?.addEventListener('pageshow', onPageShow);
|
||||
|
||||
scope.cleanup(() => {
|
||||
disposed = true;
|
||||
halt();
|
||||
lifecycle?.removeEventListener('pagehide', onPageHide);
|
||||
lifecycle?.removeEventListener('pageshow', onPageShow);
|
||||
listeners.clear();
|
||||
});
|
||||
return store;
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
/**
|
||||
* Activity (#249 Activity-1): the concept's Activity page (app.js renderActivity 732–738), its top-bar button
|
||||
* (`button#activity-open` with `span.count#activity-count`, index.html 51–52) and its sidebar row (app.js ~341), fed
|
||||
* by the session job store (activity-store.mjs).
|
||||
*
|
||||
* - D52: both counts show the running jobs only and are hidden at 0; the heading has no "Unified activity · proposed"
|
||||
* chip (prototype copy).
|
||||
* - D53: while a job runs, the progress pill (`a.activity-pill`, the approved mockup's) sits before the button on every
|
||||
* page: "Scanning · 42% · 8,240 / 19,622" and a mini bar, linking `#/activity/<id>`.
|
||||
* - D56: each item's title links its job's own view (activity-job-view.mjs).
|
||||
* - D54: each `.activity-item` is a real job. Its chip names the state (green "Completed", or blue "Running",
|
||||
* "Cancelled", "Failed", "Ended"), its line says when and what, and a running job shows its `<progress>` and
|
||||
* "Cancel job" (no "Simulate completion").
|
||||
*/
|
||||
import { createButton } from '../shared/toolkit/core.js';
|
||||
import { navigateTo } from './state.mjs';
|
||||
import { pageHeading } from './heading.mjs';
|
||||
import { showWorkspaceToast } from './toast.mjs';
|
||||
import { countOf } from './plural.mjs';
|
||||
import { pillParts, outcomeLine, addedOf, LOST_DETAIL } from './activity-store.mjs';
|
||||
|
||||
export const ACTIVITY_DESCRIPTION = 'Scans, tagging, downloads and maintenance share one history.';
|
||||
export const EMPTY_ACTIVITY = 'No jobs have run since ComfyUI started. Scans and other long operations appear here.';
|
||||
export const LOAD_FAILED = 'Activity could not be read. It is tried again shortly.';
|
||||
export const CANCEL_FAILED_TOAST = 'The job could not be cancelled. Try again.';
|
||||
const CHIPS = Object.freeze({
|
||||
running: ['blue', 'Running'], finished: ['green', 'Completed'], cancelled: ['blue', 'Cancelled'],
|
||||
failed: ['blue', 'Failed'], lost: ['blue', 'Ended'],
|
||||
});
|
||||
|
||||
function element(tag, text, className) {
|
||||
const node = document.createElement(tag);
|
||||
if (text !== undefined) node.textContent = text;
|
||||
if (className) node.className = className;
|
||||
return node;
|
||||
}
|
||||
|
||||
/** "09:41" in the browser's local time; an unknown time is empty. */
|
||||
export function clockTime(seconds) {
|
||||
if (typeof seconds !== 'number' || !Number.isFinite(seconds)) return '';
|
||||
const date = new Date(seconds * 1000);
|
||||
return `${String(date.getHours()).padStart(2, '0')}:${String(date.getMinutes()).padStart(2, '0')}`;
|
||||
}
|
||||
|
||||
const join = parts => parts.filter(Boolean).join(' · ');
|
||||
|
||||
/** What a job covers: "2 output folders" for a scan. */
|
||||
function scopeLine(job) {
|
||||
const folders = job.scope?.output_folders;
|
||||
return Number.isSafeInteger(folders) && folders >= 0 ? countOf(folders, 'output folder') : '';
|
||||
}
|
||||
|
||||
/** The item's line under its title, by state. */
|
||||
export function jobDetail(job) {
|
||||
const at = clockTime(job.finished_at);
|
||||
if (job.state === 'running') {
|
||||
return join([`Started ${clockTime(job.started_at)}`, scopeLine(job), 'closing this view does not cancel it']);
|
||||
}
|
||||
if (job.state === 'lost') return LOST_DETAIL;
|
||||
if (job.state === 'cancelled') {
|
||||
return join([`Cancelled ${at} at a file boundary`, job.kind === 'scan' ? addedOf(job) : '',
|
||||
'completed work is kept']);
|
||||
}
|
||||
if (job.state === 'failed') return join([`Stopped on an error at ${at}`, outcomeLine(job)]);
|
||||
return join([`Finished ${at}`, outcomeLine(job)]);
|
||||
}
|
||||
|
||||
/** The shell's Activity indicators: the top-bar count and pill, and the sidebar row's count. */
|
||||
export function mountActivityTopbar({ store, scope, doc = document }) {
|
||||
const button = doc.getElementById('activity-open');
|
||||
const count = doc.getElementById('activity-count');
|
||||
const pill = doc.getElementById('activity-pill');
|
||||
scope.listen(button, 'click', () => navigateTo('#/activity'));
|
||||
const label = element('span', '', 'activity-pill-label');
|
||||
const files = element('span', '', 'activity-pill-files');
|
||||
const bar = element('span', undefined, 'activity-pill-bar');
|
||||
const fill = element('i');
|
||||
bar.setAttribute('aria-hidden', 'true');
|
||||
bar.append(fill);
|
||||
pill.replaceChildren(label, files, bar);
|
||||
const render = () => {
|
||||
const running = store.running();
|
||||
count.textContent = String(running.length);
|
||||
count.hidden = running.length === 0;
|
||||
const row = doc.querySelector('#system-nav a[data-route="activity"]');
|
||||
row?.querySelector('.nav-count')?.remove();
|
||||
if (row && running.length) row.append(element('span', running.length.toLocaleString(), 'nav-count'));
|
||||
const [job] = running;
|
||||
pill.hidden = !job;
|
||||
if (!job) {
|
||||
pill.removeAttribute('href');
|
||||
pill.removeAttribute('aria-label');
|
||||
return;
|
||||
}
|
||||
const parts = pillParts(job);
|
||||
label.textContent = parts.label;
|
||||
files.textContent = parts.files;
|
||||
fill.style.width = `${job.progress}%`;
|
||||
pill.href = `#/activity/${job.id}`;
|
||||
pill.setAttribute('aria-label', parts.text);
|
||||
};
|
||||
scope.cleanup(store.subscribe(render));
|
||||
render();
|
||||
}
|
||||
|
||||
function itemFor(job, scope) {
|
||||
const item = element('div', undefined, 'activity-item');
|
||||
item.dataset.jobId = job.id;
|
||||
const heading = element('div', undefined, 'panel-heading');
|
||||
const title = element('h3');
|
||||
// D56: the title opens the job's own view.
|
||||
const link = element('a');
|
||||
link.href = `#/activity/${job.id}`;
|
||||
title.append(link);
|
||||
const chip = element('span');
|
||||
heading.append(title, chip);
|
||||
const detail = element('p');
|
||||
const progress = element('progress');
|
||||
progress.max = 100;
|
||||
const actions = element('div', undefined, 'row-actions');
|
||||
const cancel = createButton('Cancel job');
|
||||
actions.append(cancel);
|
||||
item.append(heading, detail, progress, actions);
|
||||
scope.listen(cancel, 'click', async () => {
|
||||
if (cancel.getAttribute('aria-disabled') === 'true') return;
|
||||
cancel.setAttribute('aria-disabled', 'true');
|
||||
try {
|
||||
await item.cancelJob();
|
||||
} catch (error) {
|
||||
if (scope.signal.aborted) return;
|
||||
cancel.removeAttribute('aria-disabled');
|
||||
showWorkspaceToast(CANCEL_FAILED_TOAST);
|
||||
}
|
||||
});
|
||||
item.update = next => {
|
||||
const [tone, state] = CHIPS[next.state];
|
||||
link.textContent = `${next.state === 'finished' ? '✓' : '○'} ${next.title}`;
|
||||
chip.className = `chip ${tone}`;
|
||||
chip.textContent = state;
|
||||
detail.textContent = jobDetail(next);
|
||||
const running = next.state === 'running';
|
||||
if (running) {
|
||||
progress.value = next.progress;
|
||||
progress.textContent = `${next.progress}%`;
|
||||
progress.setAttribute('aria-label', `${next.title} progress`);
|
||||
if (!progress.isConnected) item.insertBefore(progress, actions.isConnected ? actions : null);
|
||||
if (!actions.isConnected) item.append(actions);
|
||||
} else {
|
||||
// A focused Cancel job that goes away hands its focus to the item's title link, never to the page body.
|
||||
if (actions.contains(document.activeElement)) link.focus();
|
||||
progress.remove();
|
||||
actions.remove();
|
||||
}
|
||||
};
|
||||
return item;
|
||||
}
|
||||
|
||||
/** The Activity page: the concept's "Recent work" list of this session's jobs (one job's view is D56). */
|
||||
export function mountActivityPage(main, { store, scope }) {
|
||||
pageHeading(main).description.textContent = ACTIVITY_DESCRIPTION;
|
||||
const section = element('section', undefined, 'surface');
|
||||
const status = element('p', undefined, 'empty-state');
|
||||
section.append(element('h2', 'Recent work'));
|
||||
main.append(section);
|
||||
const items = new Map();
|
||||
const render = () => {
|
||||
const jobs = store.jobs();
|
||||
for (const [id, item] of items) if (!jobs.some(job => job.id === id)) { item.remove(); items.delete(id); }
|
||||
jobs.forEach((job, index) => {
|
||||
let item = items.get(job.id);
|
||||
if (!item) {
|
||||
item = itemFor(job, scope);
|
||||
item.cancelJob = () => store.cancel(job.id);
|
||||
items.set(job.id, item);
|
||||
}
|
||||
item.update(job);
|
||||
// Only a node out of place moves, so a focused Cancel job keeps its focus across updates.
|
||||
const at = section.children[index + 1];
|
||||
if (at !== item) section.insertBefore(item, at ?? null);
|
||||
});
|
||||
const failed = store.status === 'error' && !jobs.length;
|
||||
status.textContent = failed ? LOAD_FAILED : EMPTY_ACTIVITY;
|
||||
if (failed) status.setAttribute('role', 'alert');
|
||||
else status.removeAttribute('role');
|
||||
if (jobs.length || store.status === 'idle') status.remove();
|
||||
else if (!status.isConnected) section.append(status);
|
||||
};
|
||||
scope.cleanup(store.subscribe(render));
|
||||
render();
|
||||
// Opening Activity reads the list at once instead of waiting for the next periodic read.
|
||||
store.load();
|
||||
}
|
||||
@@ -3,8 +3,9 @@ const paths = Object.freeze({
|
||||
"library": "M4 4h5v16H4z M11 4h4v16h-4z M17 5l3-1 3 15-3 1z",
|
||||
"tags": "M3 3h8l10 10-8 8L3 11V3ZM7 7h.01",
|
||||
"upload": "M12 16V3m-5 5 5-5 5 5M3 15v6h18v-6",
|
||||
// D28 ruling: an outline home glyph of its own; the concept's `activity` pulse stays with the Activity row D5 omits.
|
||||
// D28 ruling: an outline home glyph of its own; the concept's `activity` pulse belongs to the Activity row.
|
||||
"overview": "M3 11 12 4l9 7M5 10v10h14V10M10 20v-6h4v6",
|
||||
"activity": "M3 12h4l3-8 4 16 3-8h4",
|
||||
"settings": "M12 8a4 4 0 1 0 0 8 4 4 0 0 0 0-8ZM12 2v3m0 14v3M2 12h3m14 0h3M5 5l2 2m10 10 2 2M5 19l2-2M17 7l2-2",
|
||||
"grid": "M3 3h7v7H3z M14 3h7v7h-7z M3 14h7v7H3z M14 14h7v7h-7z",
|
||||
"list": "M9 5h12M9 12h12M9 19h12M3 5h.01M3 12h.01M3 19h.01",
|
||||
|
||||
@@ -12,6 +12,7 @@ import { noun } from './plural.mjs';
|
||||
import { pageHeading } from './heading.mjs';
|
||||
import { mountToolbar } from './toolbar.mjs';
|
||||
import { openImportDialog, openRefineDialog } from './workspace-dialogs.mjs';
|
||||
import { openMetadataScan } from './activity-scan.mjs';
|
||||
import { showWorkspaceToast } from './toast.mjs';
|
||||
import { imagesFilters, imagesViewState } from './saved-views.mjs';
|
||||
|
||||
@@ -112,7 +113,7 @@ export function onFavoriteChange(listener) {
|
||||
* Images last visit.
|
||||
*/
|
||||
export function mountImages(main, api, parentScope, {embedded = false, fetchPage, emptyText, focusTarget = null, savedState = null,
|
||||
favorites = false, onSaveView = () => {}} = {}) {
|
||||
favorites = false, onSaveView = () => {}, activity = null} = {}) {
|
||||
const scope = createScope();
|
||||
const dispose = () => scope.dispose();
|
||||
parentScope.signal.addEventListener('abort', dispose, {once:true});
|
||||
@@ -165,7 +166,9 @@ export function mountImages(main, api, parentScope, {embedded = false, fetchPage
|
||||
scope.cleanup(() => importButton.remove());
|
||||
scope.listen(importButton, 'click', () => {
|
||||
importDialog = openImportDialog({scope, trigger:importButton, rescanBusy:rescanning,
|
||||
onInspect:() => { window.location.hash = '#/inspect'; }, onRescan:rescan});
|
||||
onInspect:() => { window.location.hash = '#/inspect'; }, onRescan:rescan,
|
||||
// D55: Scan for metadata reviews the session scan job over every discovered source.
|
||||
onMetadataScan:activity ? trigger => openMetadataScan({scope, store:activity, sources:roots.length, trigger}) : null});
|
||||
});
|
||||
}
|
||||
// The shared concept toolbar: search (filename, linked prompt text, title and tags), category and sort (D12–D14).
|
||||
|
||||
@@ -11,6 +11,9 @@ import { bindCanvasLink, canvasHref } from './topbar.mjs';
|
||||
import { renderHeading, pageHeading } from './heading.mjs';
|
||||
import { closeWorkspaceDialog } from './workspace-dialogs.mjs';
|
||||
import { createSavedViews, openSaveViewDialog, openEditViewDialog, savedViewHash } from './saved-views.mjs';
|
||||
import { createActivityStore } from './activity-store.mjs';
|
||||
import { mountActivityPage, mountActivityTopbar } from './activity.mjs';
|
||||
import { mountActivityJobView } from './activity-job-view.mjs';
|
||||
|
||||
const { createApiClient, resolveUrl } = window.ApiClient;
|
||||
|
||||
@@ -43,7 +46,7 @@ function mountWorkspace() {
|
||||
const preference = document.getElementById('theme');
|
||||
preference.value = theme.preference;
|
||||
scope.listen(preference, 'change', () => theme.setPreference(preference.value));
|
||||
// D6: ComfyUI ↗ opens the canvas; on Images with a resolved selection it sends that pair (D5: no Activity).
|
||||
// D6: ComfyUI ↗ opens the canvas; on Images with a resolved selection it sends that pair (a send is not an Activity job).
|
||||
let images = null;
|
||||
scope.cleanup(bindCanvasLink(document.querySelector('.topbar .canvas-link'), {
|
||||
href: canvasHref(resolveUrl, window.location.pathname),
|
||||
@@ -60,8 +63,13 @@ function mountWorkspace() {
|
||||
return link;
|
||||
});
|
||||
primaryNav.replaceChildren(...navLinks([['images', 'images'], ['library', 'library'], ['tags', 'tags'], ['inspect', 'upload']]));
|
||||
// D28: System, then the production Overview hub, in the concept's tools-and-system group.
|
||||
systemNav.replaceChildren(...navLinks([['system', 'settings'], ['overview', 'overview']]));
|
||||
// The concept's tools-and-system group: Activity, System, then (D28) the production Overview hub.
|
||||
systemNav.replaceChildren(...navLinks([['activity', 'activity'], ['system', 'settings'], ['overview', 'overview']]));
|
||||
// Activity (#249 Activity-1): one store of this session's jobs feeds the top-bar button and pill, the sidebar
|
||||
// row's count and the Activity page, on every route.
|
||||
const activity = createActivityStore({ api, scope, connect: url => new EventSource(resolveUrl(url)) });
|
||||
mountActivityTopbar({ store: activity, scope });
|
||||
activity.load();
|
||||
const sidebar = document.querySelector('aside.sidebar');
|
||||
// The route key a nav link carries in `data-route`: a saved view's is `saved/<id>`.
|
||||
const routeKey = route => route.id === 'saved' ? `saved/${route.savedId}` : route.id;
|
||||
@@ -210,7 +218,7 @@ function mountWorkspace() {
|
||||
if (route.id === 'images' || route.id === 'favorites') {
|
||||
mountNewPromptAction(main, api, view);
|
||||
images = mountImages(main, api, view, {savedState:saved?.state, favorites:route.id === 'favorites',
|
||||
onSaveView:saveView('images', view)});
|
||||
onSaveView:saveView('images', view), activity});
|
||||
return;
|
||||
}
|
||||
if (route.id === 'library') {
|
||||
@@ -221,6 +229,12 @@ function mountWorkspace() {
|
||||
mountVocabulary(main, api, view);
|
||||
return;
|
||||
}
|
||||
if (route.id === 'activity') {
|
||||
// D56: `#/activity/<id>` is that job's own view; `#/activity` lists the session's jobs.
|
||||
if (route.jobId) mountActivityJobView(main, { store: activity, scope: view, api, jobId: route.jobId });
|
||||
else mountActivityPage(main, { store: activity, scope: view });
|
||||
return;
|
||||
}
|
||||
if (isHandoff(route)) {
|
||||
description.textContent = route.description;
|
||||
const panel = element('section', undefined, 'surface handoff');
|
||||
|
||||
@@ -11,11 +11,13 @@ export const routes = [
|
||||
{ id: 'overview', label: 'Overview' },
|
||||
// SAVED VIEWS' first row (#249 D51): Images filtered to favorites. No href, so Overview lists no card for it.
|
||||
{ id: 'favorites', label: 'Favorites' },
|
||||
// The concept's Activity (app.js renderActivity 732–738): this server session's jobs. No href: no Overview card.
|
||||
{ id: 'activity', label: 'Activity' },
|
||||
{ id: 'system', label: 'System', href: '/prompt_manager/admin',
|
||||
description: 'Open the current dashboard, then choose Settings or Maintenance for configuration, scans, backups and diagnostics.', action: 'Open existing administration' },
|
||||
];
|
||||
|
||||
const NATIVE_ROUTES = new Set(['images', 'library', 'tags', 'overview', 'saved', 'favorites']);
|
||||
const NATIVE_ROUTES = new Set(['images', 'library', 'tags', 'overview', 'saved', 'favorites', 'activity']);
|
||||
|
||||
/** A saved view (#249 D16): the shell resolves its id to the view's page, name and state. */
|
||||
const SAVED_ROUTE = Object.freeze({ id: 'saved', label: 'Saved view' });
|
||||
@@ -33,6 +35,9 @@ export function isHandoff(route) {
|
||||
export function resolveRoute(hash) {
|
||||
const saved = /^#\/saved\/([1-9]\d*)$/.exec(hash);
|
||||
if (saved && Number.isSafeInteger(Number(saved[1]))) return {...SAVED_ROUTE, savedId:Number(saved[1]), hash};
|
||||
// One job of the Activity page (its id is the registry's uuid4 hex).
|
||||
const job = /^#\/activity\/([0-9a-f]{32})$/.exec(hash);
|
||||
if (job) return {...routes.find(route => route.id === 'activity'), jobId:job[1], hash};
|
||||
const match = /^#\/library\/([1-9]\d*)$/.exec(hash);
|
||||
if (match && Number.isSafeInteger(Number(match[1]))) {
|
||||
return {...routes.find(route => route.id === 'library'), positiveId:Number(match[1]), hash};
|
||||
@@ -110,7 +115,8 @@ export function mountRouter(win, onRoute) {
|
||||
const route = resolveRoute(win.location.hash);
|
||||
const hash = route.hash || `#/${route.id}`;
|
||||
if (win.location.hash !== hash) win.history.replaceState(win.history.state, '', hash);
|
||||
if (current?.id === route.id && current?.positiveId === route.positiveId && current?.savedId === route.savedId) return;
|
||||
if (current?.id === route.id && current?.positiveId === route.positiveId && current?.savedId === route.savedId &&
|
||||
current?.jobId === route.jobId) return;
|
||||
current = route;
|
||||
onRoute(route);
|
||||
};
|
||||
|
||||
@@ -1,12 +1,60 @@
|
||||
/** The shell's shared toast (`#workspace-toast`, role="status"), used by every workspace page. */
|
||||
const TOAST_MS = 6500;
|
||||
export const TOAST_MS = 6500;
|
||||
/** A toast with a link stays longer, so a keyboard user can reach it (D57, WCAG 2.2.1). */
|
||||
export const LINK_TOAST_MS = 20000;
|
||||
let toastTimer;
|
||||
// Stops waiting for the pointer or focus to leave a toast that a newer one replaced.
|
||||
let releaseWait = null;
|
||||
|
||||
/** Show one outcome in the shared workspace toast; it clears itself like the concept's. */
|
||||
/** Whether the pointer or keyboard focus rests on the toast. */
|
||||
const held = toast => typeof toast.matches === 'function' && toast.matches(':hover, :focus-within');
|
||||
|
||||
/** Clear the toast when its time is up, unless the pointer or focus is on it: then wait until they leave. */
|
||||
function scheduleClear(toast, delay) {
|
||||
clearTimeout(toastTimer);
|
||||
toastTimer = setTimeout(() => {
|
||||
if (held(toast)) {
|
||||
const release = () => {
|
||||
toast.removeEventListener('pointerleave', leave);
|
||||
toast.removeEventListener('focusout', later);
|
||||
releaseWait = null;
|
||||
};
|
||||
const leave = () => {
|
||||
if (held(toast)) return;
|
||||
release();
|
||||
scheduleClear(toast, TOAST_MS);
|
||||
};
|
||||
// focusout fires before the next element takes focus, so it checks once focus has moved.
|
||||
const later = () => setTimeout(leave);
|
||||
toast.addEventListener('pointerleave', leave);
|
||||
toast.addEventListener('focusout', later);
|
||||
releaseWait = release;
|
||||
return;
|
||||
}
|
||||
toast.textContent = '';
|
||||
}, delay);
|
||||
}
|
||||
|
||||
/**
|
||||
* Show one outcome in the shared workspace toast; it clears itself like the concept's. A message given as parts
|
||||
* (`['Another job is running: ', {text, href}, '.']`) renders each `{text, href}` as a link (D57); such a toast stays
|
||||
* for LINK_TOAST_MS and while the pointer or focus is on it.
|
||||
*/
|
||||
export function showWorkspaceToast(message) {
|
||||
const toast = document.getElementById('workspace-toast');
|
||||
if (!toast) return;
|
||||
toast.textContent = message;
|
||||
clearTimeout(toastTimer);
|
||||
toastTimer = setTimeout(() => {toast.textContent = '';}, TOAST_MS);
|
||||
releaseWait?.();
|
||||
if (Array.isArray(message)) {
|
||||
toast.replaceChildren(...message.map(part => {
|
||||
if (typeof part === 'string') return document.createTextNode(part);
|
||||
const link = document.createElement('a');
|
||||
link.href = part.href;
|
||||
link.textContent = part.text;
|
||||
return link;
|
||||
}));
|
||||
} else {
|
||||
toast.textContent = message;
|
||||
}
|
||||
const linked = Array.isArray(message) && message.some(part => typeof part !== 'string');
|
||||
scheduleClear(toast, linked ? LINK_TOAST_MS : TOAST_MS);
|
||||
}
|
||||
|
||||
@@ -107,7 +107,10 @@ export function mountToolbar(container, { scope, categories = [], sorts, value =
|
||||
category.value = [...category.options].some(node => node.value === selected) ? selected : '';
|
||||
}
|
||||
function setActiveFilters(count) {
|
||||
filters.textContent = filtersLabel(count);
|
||||
const next = filtersLabel(count);
|
||||
// Every listing reload reports the count; only a changed label rewrites the text node, because WebKit drops a
|
||||
// click whose mousedown hit a text node that was replaced before mouseup.
|
||||
if (filters.textContent !== next) filters.textContent = next;
|
||||
}
|
||||
setValue(value);
|
||||
|
||||
|
||||
@@ -16,7 +16,8 @@ import { countOf } from './plural.mjs';
|
||||
const LIMIT = 100;
|
||||
const BASE = '/prompt_manager/vocabulary';
|
||||
const KINDS = {
|
||||
tags: {key: row => row.id, valid: row => id(row.id) && typeof row.name === 'string' && row.name.length > 0},
|
||||
// image_count (#249 image tags) is optional until every server reports it.
|
||||
tags: {key: row => row.id, valid: row => id(row.id) && typeof row.name === 'string' && row.name.length > 0 && optionalCount(row.image_count)},
|
||||
categories: {key: row => row.value, valid: row => typeof row.value === 'string' && row.value.length > 0},
|
||||
};
|
||||
const RECOVERY = {'storage-changed': 'storage-changed', 'vocabulary-conflict': 'conflict'};
|
||||
@@ -29,6 +30,7 @@ const IMPACT_FIELDS = ['positive_prompts', 'other_prompts', 'negative_variants',
|
||||
const object = value => value !== null && typeof value === 'object' && !Array.isArray(value);
|
||||
const id = value => Number.isSafeInteger(value) && value > 0;
|
||||
const count = value => Number.isSafeInteger(value) && value >= 0;
|
||||
const optionalCount = value => value === undefined || count(value);
|
||||
const epochOf = value => typeof value === 'string' && /^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/.test(value);
|
||||
const hexToken = value => typeof value === 'string' && /^[0-9a-f]{64}$/.test(value);
|
||||
|
||||
@@ -108,7 +110,7 @@ export function createVocabularyStream({api, kind, onChange = () => {}}) {
|
||||
}
|
||||
|
||||
function validateTag(value) {return object(value) && KINDS.tags.valid(value) && count(value.prompt_count);}
|
||||
function validateImpact(value) {return object(value) && IMPACT_FIELDS.every(name => count(value[name]));}
|
||||
function validateImpact(value) {return object(value) && IMPACT_FIELDS.every(name => count(value[name])) && optionalCount(value.images);}
|
||||
function requireRow(row) {requireValue(object(row) && id(row.id) && typeof row.name === 'string', 'Invalid tag row');}
|
||||
function requireEpoch(epoch) {requireValue(epochOf(epoch), 'Invalid library epoch');}
|
||||
function requireText(value, name) {requireValue(typeof value === 'string', `${name} must be a string`);}
|
||||
@@ -116,8 +118,9 @@ function requireText(value, name) {requireValue(typeof value === 'string', `${na
|
||||
const RECEIPTS = {
|
||||
'create-tag': value => typeof value.created === 'boolean' && validateTag(value.tag),
|
||||
'rename-tag': value => typeof value.merged === 'boolean' && validateTag(value.tag) &&
|
||||
count(value.components_changed) && count(value.pairs_changed),
|
||||
'delete-tag': value => value.deleted === true && count(value.components_changed) && count(value.pairs_changed),
|
||||
count(value.components_changed) && count(value.pairs_changed) && optionalCount(value.images_changed),
|
||||
'delete-tag': value => value.deleted === true && count(value.components_changed) && count(value.pairs_changed) &&
|
||||
optionalCount(value.images_changed),
|
||||
'apply-category': (value, body) => value.source === body.source && value.target === body.target &&
|
||||
typeof value.consolidated === 'boolean' && validateImpact(value.impact),
|
||||
};
|
||||
@@ -270,6 +273,7 @@ function impactList(impact) {
|
||||
parts.push(`${countOf(negatives, 'negative variant')}${archived}`);
|
||||
}
|
||||
if (impact.other_prompts) parts.push(countOf(impact.other_prompts, 'other saved prompt'));
|
||||
if (impact.images) parts.push(countOf(impact.images, 'image'));
|
||||
return parts.length > 1 ? `${parts.slice(0, -1).join(', ')} and ${parts.at(-1)}` : parts.join('');
|
||||
}
|
||||
|
||||
@@ -282,6 +286,6 @@ export function categoryPreviewText(shown) {
|
||||
/** The Delete tag confirmation sentence (decision D14). */
|
||||
export function deleteTagText(name, impact) {
|
||||
const list = impactList(impact);
|
||||
if (!list) return `“${name}” is not linked to any saved prompt. Deleting it removes only the name.`;
|
||||
if (!list) return `“${name}” is not linked to any saved prompt or image. Deleting it removes only the name.`;
|
||||
return `Deleting “${name}” removes the name and its links from ${list}. The prompts, negative variants, images and history themselves are kept.`;
|
||||
}
|
||||
|
||||
@@ -8,7 +8,10 @@
|
||||
* mock-status note because every field is a real filter, and no "Prompt evidence" field because evidence is only
|
||||
* known per file (D14). Clear also clears the toolbar category, as the concept's does; the page applies it.
|
||||
* - "Import & inspect" (a1483–1491): only the real operations, Inspect a local image and Scan an output folder →
|
||||
* Rescan; no "Import portable recipes" (D15). Rescan shows its running state (aria-disabled "Rescanning…").
|
||||
* Rescan; no "Import portable recipes" (D15). Rescan shows its running state (aria-disabled "Rescanning…"). D55: the
|
||||
* concept's own "Scan an output folder → Review scan" row follows as "Scan for metadata".
|
||||
* - "Review metadata scan" (a1428–1432 `scan-review`, D55): Sources and Scope in a `dl.meta-grid`, the concept's
|
||||
* sentence, a backup `.notice` ("Back up now") and Cancel / Start scan; no sample-plan note.
|
||||
* - "Copy prompt" (a1223–1241): the select-to-copy fallback when the clipboard is unavailable (D22).
|
||||
* - One-value input (a1138 `simpleInput`): a "Category"-style field (`input#action-input`) and Cancel / a primary
|
||||
* action; used by the Images selection bar's "Set category" (#249 D8) and the toolbar's Save view.
|
||||
@@ -17,7 +20,7 @@
|
||||
*
|
||||
* Every opener takes the page `scope`: disposing it closes and removes the shell, so a dialog never outlives its page.
|
||||
*/
|
||||
import { createButton } from '../shared/toolkit/core.js';
|
||||
import { createButton, createSwitch } from '../shared/toolkit/core.js';
|
||||
|
||||
const FIELD_LABELS = Object.freeze({ source: 'Source', folder: 'Folder', media: 'Media', tags: 'Tags' });
|
||||
const FIELD_ORDER = Object.keys(FIELD_LABELS);
|
||||
@@ -247,16 +250,23 @@ const RESCAN_LABEL = 'Rescan';
|
||||
const RESCAN_BUSY_LABEL = 'Rescanning…';
|
||||
|
||||
/**
|
||||
* "Import & inspect" with the two real operations (D15). While `rescanBusy`, Rescan reads "Rescanning…" and is
|
||||
* "Import & inspect" with the real operations (D15). While `rescanBusy`, Rescan reads "Rescanning…" and is
|
||||
* aria-disabled, so a second press is visibly ignored. Returns `{setRescanBusy}` for the page to end that state.
|
||||
* With `onMetadataScan`, the concept's "Review scan" row follows as "Scan for metadata" (D55).
|
||||
*/
|
||||
export function openImportDialog({ scope, onInspect, onRescan, trigger = null, rescanBusy = false }) {
|
||||
export function openImportDialog({ scope, onInspect, onRescan, onMetadataScan = null, trigger = null, rescanBusy = false }) {
|
||||
requireScope(scope);
|
||||
requireCallbacks({ onInspect, onRescan });
|
||||
requireCallbacks({ onInspect, onRescan, ...(onMetadataScan ? { onMetadataScan } : {}) });
|
||||
const list = el('div', undefined, 'operation-list');
|
||||
const inspect = operation('Inspect a local image', 'Read embedded metadata before deciding to add it.', 'Open inspector', onInspect);
|
||||
const rescan = operation('Scan an output folder', 'Find files added or removed in the output folders.', RESCAN_LABEL, onRescan);
|
||||
list.append(inspect.row, rescan.row);
|
||||
if (onMetadataScan) {
|
||||
// The opened review dialog replaces this one in the shell, so its Cancel returns focus to Import.
|
||||
const review = operation('Scan for metadata', 'Discover and enrich existing files using the configured sources.',
|
||||
'Review scan', () => onMetadataScan(trigger));
|
||||
list.append(review.row);
|
||||
}
|
||||
function setRescanBusy(busy) {
|
||||
rescan.action.textContent = busy ? RESCAN_BUSY_LABEL : RESCAN_LABEL;
|
||||
if (busy) rescan.action.setAttribute('aria-disabled', 'true');
|
||||
@@ -326,3 +336,78 @@ export function openContentDialog({ scope, title, nodes, onClose = null, trigger
|
||||
if (onClose !== null && typeof onClose !== 'function') throw new TypeError('The dialog onClose must be a function');
|
||||
show(title, nodes, { scope, trigger, onClose });
|
||||
}
|
||||
|
||||
export const SCAN_REVIEW_SENTENCE = 'Enrich positive / negative associations, reuse canonical text and report unchanged files. ' +
|
||||
'Do not inflate run counts or recreate existing thumbnails.';
|
||||
const BACKUP_LABEL = 'Back up now';
|
||||
const BACKUP_BUSY_LABEL = 'Backing up…';
|
||||
|
||||
export const SCAN_SCOPE_LABELS = { new: 'New and unresolved files', all: 'All files' };
|
||||
export const SCAN_ALL_FILES_HINT = 'Also read the files already linked to their prompt again.';
|
||||
|
||||
/**
|
||||
* The concept's "Review metadata scan" (`scan-review`, D55). `sources` is the number of output folders the scan reads.
|
||||
* The scope is the concept's "New and unresolved files"; the "All files" checkbox (D58, the concept's `.setting-row`
|
||||
* switch, unchecked) switches the Scope value to "All files". `onBackup` runs the database backup (the button is
|
||||
* aria-disabled and reads "Backing up…" until it settles, and the dialog stays open); Start scan closes the dialog, then
|
||||
* calls `onStart({scope: 'new' | 'all'})`.
|
||||
*/
|
||||
export function openScanReviewDialog({ scope, sources, onBackup, onStart, trigger = null }) {
|
||||
requireScope(scope);
|
||||
requireCallbacks({ onBackup, onStart });
|
||||
if (!Number.isSafeInteger(sources) || sources < 0) throw new TypeError('The scan review needs its source count');
|
||||
const grid = el('dl', undefined, 'meta-grid');
|
||||
const values = {};
|
||||
for (const [term, value] of [['Sources', `${sources.toLocaleString('en-US')} output ${sources === 1 ? 'folder' : 'folders'}`],
|
||||
['Scope', SCAN_SCOPE_LABELS.new]]) {
|
||||
const entry = el('div');
|
||||
values[term] = el('dd', value);
|
||||
entry.append(el('dt', term), values[term]);
|
||||
grid.append(entry);
|
||||
}
|
||||
// The concept's System toggle: a `.setting-row` whose checkbox is the Toolkit switch (concept `hydrateSwitches`).
|
||||
const toggle = createSwitch('', '', 'scan-all-files');
|
||||
const allFiles = toggle.querySelector('input');
|
||||
allFiles.checked = false;
|
||||
const option = el('div', undefined, 'setting-row');
|
||||
const optionText = el('div');
|
||||
const optionLabel = el('label', SCAN_SCOPE_LABELS.all);
|
||||
optionLabel.htmlFor = allFiles.id;
|
||||
const hint = el('p', SCAN_ALL_FILES_HINT);
|
||||
hint.id = 'scan-all-files-hint';
|
||||
allFiles.setAttribute('aria-describedby', hint.id);
|
||||
optionText.append(optionLabel, hint);
|
||||
option.append(optionText, toggle);
|
||||
const chosenScope = () => (allFiles.checked ? 'all' : 'new');
|
||||
allFiles.addEventListener('change', () => { values.Scope.textContent = SCAN_SCOPE_LABELS[chosenScope()]; });
|
||||
const notice = el('div', undefined, 'notice');
|
||||
const copy = el('div');
|
||||
copy.append(el('h3', 'Back up the database first (recommended)'),
|
||||
el('p', 'Downloads a copy of the prompts database before the scan changes any records.'));
|
||||
const backupActions = el('div', undefined, 'row-actions');
|
||||
const backup = createButton(BACKUP_LABEL);
|
||||
backupActions.append(backup);
|
||||
notice.append(copy, backupActions);
|
||||
backup.addEventListener('click', async () => {
|
||||
if (backup.getAttribute('aria-disabled') === 'true') return;
|
||||
backup.setAttribute('aria-disabled', 'true');
|
||||
backup.textContent = BACKUP_BUSY_LABEL;
|
||||
try {
|
||||
await onBackup();
|
||||
} finally {
|
||||
backup.removeAttribute('aria-disabled');
|
||||
backup.textContent = BACKUP_LABEL;
|
||||
}
|
||||
});
|
||||
const cancel = createButton('Cancel');
|
||||
const start = createButton('Start scan', 'ca-primary');
|
||||
cancel.addEventListener('click', () => finish());
|
||||
start.addEventListener('click', () => {
|
||||
const files = chosenScope();
|
||||
finish();
|
||||
onStart({ scope: files });
|
||||
});
|
||||
const actions = el('div', undefined, 'dialog-actions');
|
||||
actions.append(cancel, start);
|
||||
show('Review metadata scan', [grid, option, el('p', SCAN_REVIEW_SENTENCE, 'small'), notice, actions], { scope, trigger });
|
||||
}
|
||||
|
||||
@@ -17,6 +17,58 @@ main:focus-visible { outline: 2px solid var(--ca-focus); outline-offset: 4px; }
|
||||
nav a[aria-current="page"] { border: 2px solid Highlight; color: LinkText; }
|
||||
}
|
||||
|
||||
/* D53: the running job's progress pill before the Activity button (the approved Activity mockup's `.pill`), in the
|
||||
concept's tokens. At <=700px it keeps its verb and percentage; the file counts and bar stay in its accessible name. */
|
||||
.activity-pill:not([hidden]) { display: inline-flex; align-items: center; gap: 8px; padding: 6px 12px; border: 1px solid var(--line);
|
||||
border-radius: 999px; font-size: 12px; white-space: nowrap; color: var(--ca-text); background: var(--ca-surface); /* D53 */ }
|
||||
.activity-pill:hover { border-color: var(--ca-accent); /* D53 */ }
|
||||
.activity-pill-bar { display: block; inline-size: 70px; block-size: 5px; border-radius: 3px; background: var(--line); overflow: hidden; /* D53 */ }
|
||||
.activity-pill-bar i { display: block; block-size: 100%; inline-size: 0; background: var(--ca-accent); /* D53 */ }
|
||||
@media (pointer: coarse) { .activity-pill:not([hidden]) { min-block-size: 40px; /* D53 */ } }
|
||||
@media (max-width: 700px) {
|
||||
/* D52: with the Activity button the full-width row can be wider than a zoomed phone, so its items wrap. */
|
||||
.top-actions { flex-wrap: wrap; /* D52 */ }
|
||||
.activity-pill:not([hidden]) { gap: 0; padding-inline: 10px; /* D53 */ }
|
||||
.activity-pill-files, .activity-pill-bar { display: none; /* D53 */ }
|
||||
}
|
||||
|
||||
/* D56: each Activity item's title links its job's view in the title's own colour. */
|
||||
.activity-item h3 a { color: inherit; text-decoration: none; /* D56 */ }
|
||||
.activity-item h3 a:hover { text-decoration: underline; /* D56 */ }
|
||||
|
||||
/* D56: one job's view, the approved Activity mockup (activity-scan.html) in the concept's tokens: the counter tiles,
|
||||
the outcome list beside the selected outcome's files, and the collapsible log. */
|
||||
.stat-row { display: grid; grid-template-columns: repeat(5, minmax(0, 1fr)); gap: 10px; margin: 16px 0; /* D56 */ }
|
||||
.stat-row .stat { border: 1px solid var(--line); border-radius: 7px; padding: 12px 14px; background: var(--ca-surface); min-inline-size: 0; /* D56 */ }
|
||||
.stat-row .stat dt { font-size: 11px; color: var(--ca-muted); /* D56 */ }
|
||||
.stat-row .stat dd { margin: 6px 0 0; font-size: 20px; font-weight: 600; /* D56 */ }
|
||||
.stat-row .stat.added dd { color: var(--green); /* D56 */ }
|
||||
.stat-row .stat.warn dd { color: var(--warning); /* D56 */ }
|
||||
.stat-row .stat.bad dd { color: var(--red); /* D56 */ }
|
||||
.surface > .panel-heading > .search-field { max-width: 320px; /* D56 */ }
|
||||
.outcomes { display: grid; grid-template-columns: 220px minmax(0, 1fr); gap: 16px; margin-top: 6px; /* D56 */ }
|
||||
.ca-tools .outcome-list button { display: flex; justify-content: space-between; width: 100%; margin-bottom: 6px; text-align: left; /* D56 */ }
|
||||
.ca-tools .outcome-list button[aria-pressed="true"] { border-color: var(--ca-accent); background: var(--soft-blue); /* D56 */ }
|
||||
.outcome-files .file-row { display: grid; grid-template-columns: minmax(0, 1fr) auto; gap: 12px; padding: 9px 4px;
|
||||
border-bottom: 1px solid var(--line); font-size: 12px; /* D56 */ }
|
||||
.ca-tools button.file-row { width: 100%; min-height: 0; justify-content: normal; align-items: start; border: 0;
|
||||
border-bottom: 1px solid var(--line); border-radius: 0; background: transparent; color: inherit; text-align: left;
|
||||
white-space: normal; font: inherit; font-size: 12px; /* D56 */ }
|
||||
.ca-tools button.file-row:hover:not(:disabled) { background: var(--ca-hover); /* D56 */ }
|
||||
.file-row .path { font-family: ui-monospace, monospace; overflow-wrap: anywhere; /* D56 */ }
|
||||
.file-row .why { color: var(--ca-muted); /* D56 */ }
|
||||
.outcome-files > p.small { color: var(--ca-muted); margin-top: 10px; /* D56 */ }
|
||||
.term { margin-top: 16px; border: 1px solid var(--line); border-radius: 7px; overflow: hidden; /* D56 */ }
|
||||
.term summary { padding: 11px 14px; cursor: pointer; font-size: 12px; background: var(--ca-surface); /* D56 */ }
|
||||
.term summary span { color: var(--ca-muted); /* D56 */ }
|
||||
.term pre { margin: 0; padding: 12px 14px; max-height: 180px; overflow: auto; font: 11px/1.7 ui-monospace, monospace;
|
||||
color: var(--ca-muted); background: var(--page); white-space: pre-wrap; overflow-wrap: anywhere; /* D56 */ }
|
||||
@media (max-width: 700px) {
|
||||
.stat-row { grid-template-columns: repeat(2, minmax(0, 1fr)); /* D56 */ }
|
||||
.outcomes { grid-template-columns: minmax(0, 1fr); /* D56 */ }
|
||||
.outcome-files .file-row { grid-template-columns: minmax(0, 1fr); gap: 4px; /* D56 */ }
|
||||
}
|
||||
|
||||
/* Overview: real library statistics and the legacy tool cards inside a concept `.surface`. */
|
||||
.stat-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 16px; margin: 0 0 26px; }
|
||||
.stat { padding: 22px; background: var(--ca-surface); border: 1px solid var(--line); border-radius: 7px; }
|
||||
|
||||
+1
-1
@@ -32,7 +32,7 @@
|
||||
<div class="workspace">
|
||||
<header class="topbar">
|
||||
<div class="breadcrumbs">ComfyAssets <span>/</span> <span id="workspace-breadcrumb">Images</span></div>
|
||||
<div class="top-actions"><label class="theme-label" for="theme">Theme
|
||||
<div class="top-actions"><!-- D53: the progress pill, shown while a job runs. D52: the count shows running jobs, hidden at 0. --><a id="activity-pill" class="activity-pill" hidden></a><button id="activity-open" type="button">Activity <span class="count" id="activity-count" hidden>0</span></button><label class="theme-label" for="theme">Theme
|
||||
<select id="theme" aria-label="Theme">
|
||||
<option value="dark">Dark</option>
|
||||
<option value="light">Light</option>
|
||||
|
||||
Reference in New Issue
Block a user