Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
21a2cc0932 | ||
|
|
3ebf5c48cf | ||
|
|
0156d3fadb | ||
|
|
357c0f09e7 | ||
|
|
2cecfcf0ac |
@@ -16,9 +16,13 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
# Timeouts are the measured job time plus about 50%, so a hung run stops early.
|
||||
# Before sharding (main 2cecfcf, one worker): frontend coverage 32m10s, page workflows firefox 18m59s,
|
||||
# webkit 19m26s. Sharded with 3 workers per job: estimates pending the first CI run (coverage shard ~10m,
|
||||
# merge ~5m, workflow shard ~8m). Re-measure when a job passes its estimate.
|
||||
jobs:
|
||||
frontend-coverage:
|
||||
name: frontend coverage (chromium and touch)
|
||||
name: frontend coverage (shard ${{ matrix.shard }}/3)
|
||||
runs-on: ubuntu-latest
|
||||
# Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the
|
||||
# same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json;
|
||||
@@ -26,12 +30,15 @@ jobs:
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble
|
||||
options: --user 1001
|
||||
# This job runs every page workflow twice (chromium, then touch) on one worker with V8 coverage on, so it
|
||||
# takes about twice a single-browser job. Measured 2026-10-20: whole job 18m26s-24m57s (main 5ded3cf
|
||||
# 22m15s; PR head 6359999 was cancelled at the old 25-minute limit while still passing), coverage step
|
||||
# alone 20m55s-22m17s+, against ~12-15 minutes for the firefox/webkit jobs. 40 minutes leaves headroom
|
||||
# for suite growth while still stopping a hung run; re-measure if a run passes 30 minutes.
|
||||
timeout-minutes: 40
|
||||
timeout-minutes: 15
|
||||
# Public-repo ubuntu-latest runners have 4 vCPUs: three Playwright workers, each with its own test
|
||||
# server, leave one CPU for the servers and the runner.
|
||||
env:
|
||||
PM_TEST_WORKERS: 3
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: [1, 2, 3]
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
# The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache
|
||||
@@ -51,23 +58,92 @@ jobs:
|
||||
echo "$venv/bin" >> "$GITHUB_PATH"
|
||||
- run: python --version && node --version && node --version | grep -q '^v24\.'
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
- run: npm run --prefix tools/vendor check
|
||||
- run: npm run --prefix tools/vendor test
|
||||
- name: Frontend unit, viewer, workflow, and coverage checks
|
||||
run: npm run --prefix tools/vendor test:coverage
|
||||
- name: Upload frontend coverage and Chromium traces
|
||||
# Shard 1 also runs the node units and the viewer suite, so the vendor pins and the tooling tests
|
||||
# run on the lightest shard instead.
|
||||
- if: matrix.shard == 3
|
||||
run: npm run --prefix tools/vendor check
|
||||
- if: matrix.shard == 3
|
||||
run: npm run --prefix tools/vendor test
|
||||
- name: Page workflows with coverage, shard ${{ matrix.shard }}/3
|
||||
run: npm run --prefix tools/vendor test:coverage:shard -- ${{ matrix.shard }}/3
|
||||
- name: Upload raw coverage
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: coverage-raw-${{ matrix.shard }}
|
||||
path: coverage/js/raw/
|
||||
if-no-files-found: error
|
||||
retention-days: 3
|
||||
- name: Upload Chromium and touch traces
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: frontend-traces-shard-${{ matrix.shard }}
|
||||
path: tools/vendor/test-results/
|
||||
retention-days: 14
|
||||
|
||||
frontend-coverage-merge:
|
||||
name: frontend coverage (merge)
|
||||
needs: frontend-coverage
|
||||
runs-on: ubuntu-latest
|
||||
# Same image as the shards: the raw files name sources by the container's workspace path.
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble
|
||||
options: --user 1001
|
||||
timeout-minutes: 8
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- run: node --version && node --version | grep -q '^v24\.'
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
# Each artifact lands in its own coverage-raw-N directory, so the next step can count them.
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: coverage-raw-*
|
||||
path: ${{ runner.temp }}/coverage-raw
|
||||
# coverage:merge only checks that raw files exist; merging two of three shards would report
|
||||
# partial coverage, so a missing or empty shard artifact fails here.
|
||||
- name: Verify every shard's raw coverage arrived
|
||||
shell: bash
|
||||
env:
|
||||
STAGED_RAW: ${{ runner.temp }}/coverage-raw
|
||||
EXPECTED_SHARDS: 3
|
||||
run: |
|
||||
found=0
|
||||
for shard in $(seq 1 "$EXPECTED_SHARDS"); do
|
||||
directory="$STAGED_RAW/coverage-raw-$shard"
|
||||
count=0
|
||||
if [ -d "$directory" ]; then count=$(find "$directory" -maxdepth 1 -type f -name '*.json' | wc -l); fi
|
||||
echo "coverage-raw-$shard: $count raw files"
|
||||
if [ "$count" -gt 0 ]; then found=$((found + 1)); fi
|
||||
done
|
||||
echo "shard artifacts with raw coverage: $found of $EXPECTED_SHARDS"
|
||||
if [ "$found" -ne "$EXPECTED_SHARDS" ]; then
|
||||
echo "::error::Expected raw coverage from $EXPECTED_SHARDS shards, got $found of $EXPECTED_SHARDS"
|
||||
exit 1
|
||||
fi
|
||||
staged=$(find "$STAGED_RAW" -mindepth 2 -maxdepth 2 -type f -name '*.json' | wc -l)
|
||||
mkdir -p coverage/js/raw
|
||||
# cp -n never overwrites; a clash shows up as fewer merged files than staged ones.
|
||||
find "$STAGED_RAW" -mindepth 2 -maxdepth 2 -type f -name '*.json' -exec cp -n -t coverage/js/raw {} + || true
|
||||
merged=$(find coverage/js/raw -maxdepth 1 -type f -name '*.json' | wc -l)
|
||||
if [ "$merged" -ne "$staged" ]; then
|
||||
echo "::error::Shards wrote clashing raw file names: $staged staged, $merged merged"
|
||||
exit 1
|
||||
fi
|
||||
echo "merged $merged raw files"
|
||||
- name: Frontend coverage report and baseline check
|
||||
run: npm run --prefix tools/vendor coverage:merge
|
||||
- name: Upload frontend coverage report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: frontend-test-reports
|
||||
path: |
|
||||
tools/vendor/test-results/
|
||||
coverage/js/
|
||||
!coverage/js/raw/**
|
||||
retention-days: 14
|
||||
|
||||
browser-workflows:
|
||||
name: page workflows (${{ matrix.browser }})
|
||||
name: page workflows (${{ matrix.browser }}, shard ${{ matrix.shard }}/2)
|
||||
runs-on: ubuntu-latest
|
||||
# Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the
|
||||
# same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json;
|
||||
@@ -75,11 +151,16 @@ jobs:
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble
|
||||
options: --user 1001
|
||||
timeout-minutes: 25
|
||||
timeout-minutes: 12
|
||||
# Public-repo ubuntu-latest runners have 4 vCPUs: three Playwright workers, each with its own test
|
||||
# server, leave one CPU for the servers and the runner.
|
||||
env:
|
||||
PM_TEST_WORKERS: 3
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
browser: [firefox, webkit]
|
||||
shard: [1, 2]
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
# The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache
|
||||
@@ -99,21 +180,21 @@ jobs:
|
||||
echo "$venv/bin" >> "$GITHUB_PATH"
|
||||
- run: python --version && node --version && node --version | grep -q '^v24\.'
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
- name: Page workflows in ${{ matrix.browser }}
|
||||
run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }}
|
||||
- name: Page workflows in ${{ matrix.browser }}, shard ${{ matrix.shard }}/2
|
||||
run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }} --shard ${{ matrix.shard }}/2
|
||||
- name: Upload browser traces
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: browser-traces-${{ matrix.browser }}
|
||||
name: browser-traces-${{ matrix.browser }}-shard-${{ matrix.shard }}
|
||||
path: tools/vendor/test-results/
|
||||
retention-days: 14
|
||||
|
||||
# Branch protection requires the exact context "viewerjs". This job keeps that
|
||||
# name and fails unless every frontend job above succeeded, including when one
|
||||
# was cancelled or skipped.
|
||||
# name and fails unless every frontend job above succeeded (every shard, the
|
||||
# coverage merge and every browser shard), including when one was cancelled or skipped.
|
||||
viewerjs:
|
||||
needs: [frontend-coverage, browser-workflows]
|
||||
needs: [frontend-coverage, frontend-coverage-merge, browser-workflows]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
@@ -121,8 +202,10 @@ jobs:
|
||||
- name: Require every frontend job to succeed
|
||||
env:
|
||||
COVERAGE_RESULT: ${{ needs.frontend-coverage.result }}
|
||||
MERGE_RESULT: ${{ needs.frontend-coverage-merge.result }}
|
||||
WORKFLOWS_RESULT: ${{ needs.browser-workflows.result }}
|
||||
run: |
|
||||
echo "frontend-coverage: $COVERAGE_RESULT"
|
||||
echo "frontend-coverage-merge: $MERGE_RESULT"
|
||||
echo "browser-workflows: $WORKFLOWS_RESULT"
|
||||
test "$COVERAGE_RESULT" = success && test "$WORKFLOWS_RESULT" = success
|
||||
test "$COVERAGE_RESULT" = success && test "$MERGE_RESULT" = success && test "$WORKFLOWS_RESULT" = success
|
||||
|
||||
+85
-16
@@ -39,9 +39,13 @@ npm run --prefix tools/vendor test:coverage
|
||||
npm run --prefix tools/vendor test:workflows -- --project firefox --project webkit
|
||||
```
|
||||
|
||||
`PM_TEST_PYTHON` selects the Python executable; `PM_TEST_PORT` selects the local
|
||||
port (default 8765). Set these environment variables when Python is outside PATH
|
||||
or the port is occupied. Test servers are never reused. On unsupported Linux
|
||||
`PM_TEST_PYTHON` selects the Python executable; set it when Python is outside
|
||||
PATH. Every Playwright worker starts its own `tests/browser/server.py --port 0`,
|
||||
which binds a port the OS picks and prints it as `PM_TEST_SERVER_PORT=<n>`, with
|
||||
its own temp root and database (`tests/browser/fixtures.mjs`), so
|
||||
parallel workers and concurrent runs never share state. `PM_TEST_WORKERS` sets the
|
||||
worker count (default 1 under `CI`, otherwise a quarter of the CPUs, at most 8).
|
||||
Test servers are never reused across runs. On unsupported Linux
|
||||
distributions, browser system dependencies may need manual installation. For a
|
||||
quick browser run, use `test:workflows -- --project chromium`.
|
||||
|
||||
@@ -114,16 +118,23 @@ second without a browser and fails, naming the files, when a measured
|
||||
`web/js` file has no baseline entry or the baseline lists a file that no longer
|
||||
exists. It uses the gate's own `discoverSources`, and `npm test` runs the same
|
||||
check, so run it before pushing a new or deleted frontend file instead of
|
||||
waiting for the `frontend coverage` job.
|
||||
waiting for the `frontend coverage` jobs.
|
||||
|
||||
The `viewerjs` check runs on every PR, including backend-only changes. It is an
|
||||
aggregate job that passes only when the `frontend coverage (chromium and touch)`
|
||||
job and both `page workflows (firefox)` and `page workflows (webkit)` jobs
|
||||
succeed; the browsers run on separate runners in parallel. GitHub artifacts
|
||||
named `frontend-test-reports` contain HTML coverage, a JSON summary, and the
|
||||
Chromium/touch failure screenshots/traces; `browser-traces-firefox` and
|
||||
`browser-traces-webkit` hold the other browsers' traces. Open traces with
|
||||
Playwright's `show-trace` tool.
|
||||
aggregate job that passes only when every frontend job succeeds: the three
|
||||
`frontend coverage (shard N/3)` jobs, `frontend coverage (merge)`, and the four
|
||||
`page workflows (firefox|webkit, shard N/2)` jobs. A failed, cancelled or skipped
|
||||
job fails it. Every browser job runs with `PM_TEST_WORKERS: 3` (the public
|
||||
runners have 4 vCPUs). Each coverage shard runs `test:coverage:shard -- N/3`
|
||||
(shard 1 also runs the node units and the viewer suite, shard 3 the vendor pins
|
||||
and tooling tests) and uploads its raw V8 files as `coverage-raw-N` (kept 3
|
||||
days). The merge job downloads them, fails unless all three arrived with raw
|
||||
files and without clashing names, then runs `coverage:merge` (report plus
|
||||
baseline check). `tests/vendor/ci-workflow.test.mjs` checks this shape. GitHub
|
||||
artifacts named `frontend-test-reports` contain the merged HTML coverage and
|
||||
JSON summary; `frontend-traces-shard-N` hold the Chromium/touch failure
|
||||
screenshots/traces and `browser-traces-<browser>-shard-N` the other browsers'
|
||||
traces. Open traces with Playwright's `show-trace` tool.
|
||||
|
||||
The real-host suite covers both Classic and Nodes 2.0 startup, switching both
|
||||
ways, actual database prompt selection, execution-text serialization, workflow
|
||||
@@ -550,8 +561,9 @@ Recorded shell deviations:
|
||||
configured sources.", button "Review scan"). It opens the concept's `scan-review`,
|
||||
"Review metadata scan": no sample-plan `.demo-note` (prototype copy); Sources and
|
||||
Scope in a `dl.meta-grid` (the concept's maintenance plan uses `dl`; `scan-review`
|
||||
uses a `div`) reading "N output folders" (the discovered sources) and "All files"
|
||||
(fixed in Activity-1; Activity-2 adds the scope option); the concept's sentence; then
|
||||
uses a `div`) reading "N output folders" (the discovered sources) and the concept's
|
||||
"New and unresolved files" (the D58 checkbox switches it to "All files"); the
|
||||
concept's sentence; then
|
||||
a backup `.notice` (the concept's System notice box): `h3` "Back up the database first
|
||||
(recommended)", "Downloads a copy of the prompts database before the scan changes any
|
||||
records." and "Back up now", which downloads the existing `/prompt_manager/backup`
|
||||
@@ -624,6 +636,63 @@ Recorded shell deviations:
|
||||
is running: <title>." with the title linking that job's view (the shared toast now
|
||||
accepts link parts). A toast with a link stays 20 s instead of 6.5 s, and any toast
|
||||
stays while the pointer or focus is on it.
|
||||
- **D58** (scan scope, Activity-2 Task 4): the concept's `scan-review` states a fixed
|
||||
scope, "New and unresolved files". Production keeps that value as the default and adds
|
||||
the concept's own System toggle after the `dl.meta-grid`: a `.setting-row` with the
|
||||
label "All files", the hint "Also read the files already linked to their prompt
|
||||
again." (the switch's `aria-describedby`) and the Toolkit switch (`label.ca-switch`, a
|
||||
checkbox with `role="switch"`, as the concept's `hydrateSwitches` renders it),
|
||||
unchecked. Checking it switches the Scope value to "All files"; Start scan sends
|
||||
`POST /prompt_manager/jobs/scan {scope: "new" | "all"}` (no body, an empty body or no
|
||||
`scope` is "new"; another value is a 400).
|
||||
- In "new" the scan skips, before reading any metadata, the files the Library already
|
||||
links with a resolved link and that are unchanged on disk: exactly one
|
||||
`generated_images` row for the file, exactly one distinct pair link on it
|
||||
(`image_prompt_pairs`, written only by resolved ingestion), and a stored
|
||||
`file_size` equal to the file's size in this walk. Both paths go through
|
||||
`normalize_image_path` (absolute, normalized, `os.path.normcase`); a row without
|
||||
`file_path` matches by its `image_path`.
|
||||
- Read again: files linked without a pair (the monitor's fallback and FIFO links, the
|
||||
legacy positive-only import), linked to more than one prompt or pair, not linked, a
|
||||
file whose size changed (one replaced at the same path, e.g. a reused ComfyUI
|
||||
counter name, is read and its conflict lands under Needs review), and a row with no
|
||||
stored size (unknown is not known). The scan now stores each file's size when it
|
||||
ingests it, so a file read once is skipped by the next "new" pass; the first pass
|
||||
after the upgrade fills the missing sizes, which reports those files as enriched
|
||||
(Added) instead of unchanged.
|
||||
- The known set is one streamed query that keeps only the scanned paths. If it cannot
|
||||
be read, the scan logs "Known files could not be read · reading every file" and
|
||||
reads every file.
|
||||
- A skipped file counts as processed and unchanged in the legacy counts (not as
|
||||
found) and under Already known, with the detail "Skipped · already linked" and the
|
||||
log line "= <root>/<path> · Skipped · already linked"; the log's first line names
|
||||
the scope ("Scan started · N output folders · new and unresolved files" or "· all
|
||||
files"). A legacy-page scan that attaches to a running "new" job therefore shows
|
||||
the skipped files in its `unchanged` count.
|
||||
- "All files" reads every file, as the scan did in Activity-1, and the legacy
|
||||
`/prompt_manager/scan` route always does when it starts the scan (its SSE payload
|
||||
keys are unchanged).
|
||||
- `activity-concept.spec.mjs` compares the switch with the concept's System → Sources
|
||||
toggle ("Watch for new outputs"), without its place-dependent top margin;
|
||||
`workspace-activity.spec.mjs` covers the Scope value, the description and the
|
||||
request both ways; `tests/test_scan_scope.py` covers the skip, the size rule, the
|
||||
replaced file, the re-read of unresolved files, "all", the legacy route, the
|
||||
fallback, the empty body, the 400 and Windows case folding.
|
||||
- **D59** (job report and log, Activity-2 Task 5): the job view's "Download report" is
|
||||
enabled. It fetches `GET /prompt_manager/jobs/<id>/report` through the api-client URL
|
||||
(so a proxy mount works) and saves a JSON attachment named
|
||||
`prompt-manager-<kind>-<YYYYMMDD-HHMM>.json` (server local time), then toasts "Job
|
||||
report downloaded." (or "The report could not be downloaded. Try again."). The report
|
||||
holds the summary, every counter, each outcome group's exact `count`, its listed
|
||||
`rows` (at most 200) and a `truncated` flag, the scope and the log; rows and log lines
|
||||
carry root-relative paths only, never an absolute path (an unknown id is a 404).
|
||||
A job lost with a restart has no report on the server, so its Download report is
|
||||
disabled; a view left while the report downloads still saves it but shows no toast.
|
||||
The scan log opens at its newest line every time it is opened and follows new lines
|
||||
only while the reader is at the bottom; a reader who scrolled up keeps the position
|
||||
while it stays open (checked on Chromium, Firefox and WebKit).
|
||||
`test_report_is_attachment_without_absolute_paths` and the browser tests `download
|
||||
report saves the job JSON` and `log keeps the reader's scroll position` cover it.
|
||||
- **D29**: no prototype copy. There is no "Interactive concept" badge and no
|
||||
"What changes & why" button. The footer reads "ComfyAssets · Local tools. Your
|
||||
library.", and the title has no " concept" suffix. Save view's dialog action reads
|
||||
@@ -1498,7 +1567,7 @@ bytes at the same path without changing the immutable receipt.
|
||||
The focused acceptance command retains all four projects:
|
||||
|
||||
```sh
|
||||
PM_TEST_PORT=8766 npm run --prefix tools/vendor test:workflows -- \
|
||||
npm run --prefix tools/vendor test:workflows -- \
|
||||
workspace-history.spec.mjs workspace-visual.spec.mjs \
|
||||
--project chromium --project touch --project firefox --project webkit
|
||||
npm run --prefix tools/vendor test:executions
|
||||
@@ -1591,7 +1660,7 @@ acceptance; a baseline is added only after the maintainer accepts them.
|
||||
Focused commands:
|
||||
|
||||
```sh
|
||||
PM_TEST_PORT=8766 npm run --prefix tools/vendor test:workflows -- \
|
||||
npm run --prefix tools/vendor test:workflows -- \
|
||||
workspace-library-editor.spec.mjs library-editor-visual.spec.mjs \
|
||||
--project chromium --project touch --project firefox --project webkit
|
||||
npm run --prefix tools/vendor test:comfyui # library-sender.spec.mjs host regression
|
||||
@@ -1691,7 +1760,7 @@ mobile, the concept geometry/token/control comparison and its mutation check),
|
||||
|
||||
```sh
|
||||
node --test tests/js/workspace-vocabulary-state.test.js tests/js/workspace-state.test.js
|
||||
PM_TEST_PORT=8766 npm run --prefix tools/vendor test:workflows -- \
|
||||
npm run --prefix tools/vendor test:workflows -- \
|
||||
workspace-vocabulary.spec.mjs vocabulary-visual.spec.mjs focused-node.spec.mjs \
|
||||
--project chromium --project touch --project firefox --project webkit
|
||||
```
|
||||
|
||||
@@ -181,6 +181,21 @@ AND p.id NOT IN (SELECT prompt_id FROM live_hint_prompts)
|
||||
"""
|
||||
|
||||
|
||||
# The scan's known set (#249 Activity-2): every image row, with its stored file
|
||||
# size and whether its pair link is resolved: exactly one distinct pair, the rule
|
||||
# _LINK_SUMMARY_SQL and get_image_prompt_info use. A file is known and resolved
|
||||
# when it has exactly one image row, that row is resolved, and the row's stored
|
||||
# size equals the file's size now; several rows (one file linked to more than
|
||||
# one prompt) are ambiguous, and an unknown (NULL) stored size is not known. The
|
||||
# rows are streamed in fetches of KNOWN_PATHS_FETCH_SIZE and only the scanned
|
||||
# paths are kept, so memory follows the scan (at most MAX_SCAN_FILES paths), not
|
||||
# the table.
|
||||
_KNOWN_PATHS_SQL = """SELECT gi.file_path, gi.image_path, gi.file_size,
|
||||
(SELECT COUNT(DISTINCT ip.pair_id) FROM image_prompt_pairs ip
|
||||
WHERE ip.image_id = gi.id) = 1
|
||||
FROM generated_images gi"""
|
||||
KNOWN_PATHS_FETCH_SIZE = 1000
|
||||
|
||||
# One SELECT reads the link summaries of at most this many paths: three bound
|
||||
# parameters per path stays under SQLite's historical 999-variable limit.
|
||||
LINK_SUMMARY_BATCH = 200
|
||||
@@ -2280,6 +2295,52 @@ class PromptDatabase(
|
||||
"parameters": row[8] if len(row) > 8 else None,
|
||||
}
|
||||
|
||||
def get_known_resolved_paths(self, files: Dict[str, Optional[int]]) -> Set[str]:
|
||||
"""The paths in ``files`` the Library already links with a resolved link.
|
||||
|
||||
``files`` maps each scanned path to its size on disk (None when it could
|
||||
not be read). A path qualifies when exactly one ``generated_images`` row
|
||||
claims the file, that row has exactly one pair link
|
||||
(``image_prompt_pairs``, written only by resolved ingestion), and its
|
||||
stored ``file_size`` is not NULL and equals the given size, so a file
|
||||
replaced at the same path is read again. Files linked without a pair
|
||||
(the monitor's fallback and FIFO links, the legacy positive-only
|
||||
import), linked to several prompts or pairs, or not linked at all do
|
||||
not qualify. Both sides go through ``normalize_image_path`` (absolute,
|
||||
normalized, ``os.path.normcase``), so a Windows row stored in another
|
||||
case or spelling still matches. One query, streamed; returns the
|
||||
matching keys of ``files``.
|
||||
"""
|
||||
wanted: Dict[str, List[Tuple[str, Optional[int]]]] = {}
|
||||
for path, size in files.items():
|
||||
key = normalize_image_path(str(path))
|
||||
wanted.setdefault(key, []).append((str(path), size))
|
||||
if not wanted:
|
||||
return set()
|
||||
rows_seen: Dict[str, int] = {}
|
||||
stored_sizes: Dict[str, int] = {}
|
||||
with self.model.get_connection() as conn:
|
||||
cursor = conn.execute(_KNOWN_PATHS_SQL)
|
||||
while True:
|
||||
rows = cursor.fetchmany(KNOWN_PATHS_FETCH_SIZE)
|
||||
if not rows:
|
||||
break
|
||||
for file_path, image_path, file_size, is_resolved in rows:
|
||||
# image_path is NOT NULL; file_path is NULL only on old rows.
|
||||
key = normalize_image_path(file_path or image_path)
|
||||
if key not in wanted:
|
||||
continue
|
||||
rows_seen[key] = rows_seen.get(key, 0) + 1
|
||||
if is_resolved and file_size is not None:
|
||||
stored_sizes[key] = file_size
|
||||
return {
|
||||
path
|
||||
for key, stored_size in stored_sizes.items()
|
||||
if rows_seen[key] == 1
|
||||
for path, size in wanted[key]
|
||||
if size is not None and size == stored_size
|
||||
}
|
||||
|
||||
def get_image_link_summaries(
|
||||
self, paths: List[str], allow_filename_fallback: bool = True
|
||||
) -> Dict[str, Dict[str, Any]]:
|
||||
|
||||
+88
-21
@@ -17,7 +17,14 @@ from pathlib import Path
|
||||
from aiohttp import web
|
||||
|
||||
from .prompts import thumbnail_url_for
|
||||
from .jobs import SCAN_GROUP_NAMES, SCAN_GROUPS, Job, JobBusy
|
||||
from .jobs import (
|
||||
DEFAULT_SCAN_SCOPE,
|
||||
SCAN_GROUP_NAMES,
|
||||
SCAN_GROUPS,
|
||||
SCAN_SCOPES,
|
||||
Job,
|
||||
JobBusy,
|
||||
)
|
||||
|
||||
try:
|
||||
from ...utils.parallel import map_parallel
|
||||
@@ -150,6 +157,8 @@ SCAN_FINAL_STATES = {
|
||||
"cancelled": "cancelled",
|
||||
"error": "failed",
|
||||
}
|
||||
# The scope as the scan log's first line names it (the review dialog's words).
|
||||
SCAN_SCOPE_LOG = {"new": "new and unresolved files", "all": "all files"}
|
||||
MIN_BATCHES_FOR_ESTIMATE = 2 # "about X min left" waits for this many batches
|
||||
# Log line prefix per outcome group (spec: + added, = known, ? review, ! error,
|
||||
# - no usable metadata).
|
||||
@@ -174,6 +183,7 @@ SCAN_DETAILS = {
|
||||
"linked": "Linked to an existing prompt",
|
||||
"enriched": "Metadata added to its prompt",
|
||||
"unchanged": "Already linked to its prompt",
|
||||
"skipped": "Skipped · already linked",
|
||||
"ambiguous": "Needs review: more than one prompt could match",
|
||||
"unresolved": "Needs review: the prompt could not be traced in the workflow",
|
||||
"conflict": "Needs review: already linked to a different prompt",
|
||||
@@ -197,6 +207,30 @@ def _scan_status_line(done, total, batches, elapsed):
|
||||
return f"{line} · about {minutes} min left"
|
||||
|
||||
|
||||
def _file_size(path):
|
||||
"""``path``'s size on disk, or None when it cannot be read."""
|
||||
try:
|
||||
return os.path.getsize(path)
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def _with_file_size(canonical, media_file):
|
||||
"""Record the file's size in ``file_info`` unless the metadata carries one.
|
||||
|
||||
Stored as ``generated_images.file_size``; the "new" scan scope skips a known
|
||||
file only while its size is unchanged.
|
||||
"""
|
||||
info = canonical.get("file_info")
|
||||
info = dict(info) if isinstance(info, dict) else {}
|
||||
if info.get("size") is None:
|
||||
size = _file_size(media_file)
|
||||
if size is not None:
|
||||
info["size"] = size
|
||||
if info:
|
||||
canonical["file_info"] = info
|
||||
|
||||
|
||||
def _scan_roots(output_dirs):
|
||||
"""(absolute root, public label, listing index), the most specific first.
|
||||
|
||||
@@ -1426,7 +1460,7 @@ class AdminRoutesMixin:
|
||||
job = self._jobs.active(SCAN_JOB_KIND)
|
||||
if job is None:
|
||||
try:
|
||||
job = self._start_scan_job()
|
||||
job = self._start_scan_job("all")
|
||||
except JobBusy as busy:
|
||||
return self._long_job_busy_response(busy.running.title)
|
||||
return await self._stream_scan_job(request, job)
|
||||
@@ -1450,16 +1484,19 @@ class AdminRoutesMixin:
|
||||
}
|
||||
)
|
||||
|
||||
def _start_scan_job(self):
|
||||
def _start_scan_job(self, files=DEFAULT_SCAN_SCOPE):
|
||||
"""Register the scan job synchronously; its runner starts on the loop.
|
||||
|
||||
Raises JobBusy while another record-changing job runs.
|
||||
``files`` is the scope (``SCAN_SCOPES``). Raises JobBusy while another
|
||||
record-changing job runs.
|
||||
"""
|
||||
if files not in SCAN_SCOPES:
|
||||
raise ValueError(f"Unknown scan scope: {files!r}")
|
||||
job = self._jobs.start(
|
||||
SCAN_JOB_KIND,
|
||||
SCAN_JOB_TITLE,
|
||||
record_changing=True,
|
||||
scope={"mode": "all"},
|
||||
scope={"files": files},
|
||||
runner=self._run_scan_job,
|
||||
groups=SCAN_GROUP_NAMES,
|
||||
)
|
||||
@@ -1525,7 +1562,8 @@ class AdminRoutesMixin:
|
||||
dir_names = [self._public_path(d) for d in output_dirs]
|
||||
folders = _plural(len(output_dirs), "output folder")
|
||||
job.scope["output_folders"] = len(output_dirs)
|
||||
job.log_line(f"Scan started · {folders}")
|
||||
scope = SCAN_SCOPE_LOG[job.scope.get("files", "all")]
|
||||
job.log_line(f"Scan started · {folders} · {scope}")
|
||||
job.update(status_line=f"Looking for media files in {folders}")
|
||||
yield self._scan_progress(
|
||||
0, f"Scanning {len(output_dirs)} directory(ies) for media files..."
|
||||
@@ -1544,7 +1582,12 @@ class AdminRoutesMixin:
|
||||
5, f"Found {len(media_files)} media files to process...", counts
|
||||
)
|
||||
roots = _scan_roots(output_dirs)
|
||||
async for event in self._scan_batches(media_files, counts, job, roots):
|
||||
known = set()
|
||||
if media_files and job.scope.get("files") != "all":
|
||||
known = await self._known_scan_files(job, media_files)
|
||||
async for event in self._scan_batches(
|
||||
media_files, counts, job, roots, known
|
||||
):
|
||||
yield event
|
||||
|
||||
self.logger.info(
|
||||
@@ -1565,11 +1608,29 @@ class AdminRoutesMixin:
|
||||
"message": "An internal error occurred. Check server logs for details.",
|
||||
}
|
||||
|
||||
async def _scan_batches(self, media_files, counts, job, roots=()):
|
||||
async def _known_scan_files(self, job, media_files):
|
||||
"""The "new" scope's skip set; a failed read falls back to reading all."""
|
||||
try:
|
||||
return await self._run_in_executor(self._known_scan_files_sync, media_files)
|
||||
except Exception:
|
||||
self.logger.warning(
|
||||
"Known files could not be read; the scan reads every file",
|
||||
exc_info=True,
|
||||
)
|
||||
job.log_line("Known files could not be read · reading every file")
|
||||
return set()
|
||||
|
||||
def _known_scan_files_sync(self, media_files):
|
||||
"""Stat each scanned file and ask the database which are known (blocking)."""
|
||||
sizes = {str(media_file): _file_size(media_file) for media_file in media_files}
|
||||
return self.db.get_known_resolved_paths(sizes)
|
||||
|
||||
async def _scan_batches(self, media_files, counts, job, roots=(), known=()):
|
||||
"""Process ``media_files`` in executor batches, updating ``counts`` in place.
|
||||
|
||||
Records each file's outcome in ``job`` and yields one progress event
|
||||
per batch.
|
||||
per batch. Files in ``known`` (path strings) are not read: they count
|
||||
as processed and unchanged and are recorded as skipped.
|
||||
"""
|
||||
total = len(media_files)
|
||||
started = time.monotonic()
|
||||
@@ -1578,18 +1639,23 @@ class AdminRoutesMixin:
|
||||
if job.cancelled:
|
||||
return
|
||||
batch = media_files[batch_start : batch_start + SCAN_BATCH_SIZE]
|
||||
batch_results = await self._run_in_executor(
|
||||
self._extract_batch_metadata_sync, batch
|
||||
)
|
||||
outcomes = []
|
||||
batch_counts = await self._run_in_executor(
|
||||
self._ingest_scan_batch_sync,
|
||||
batch_results,
|
||||
roots=roots,
|
||||
outcomes=outcomes,
|
||||
)
|
||||
for key in counts:
|
||||
counts[key] += batch_counts[key]
|
||||
skipped = [f for f in batch if str(f) in known]
|
||||
to_read = [f for f in batch if str(f) not in known]
|
||||
outcomes = [_scan_outcome(f, roots, "skipped") for f in skipped]
|
||||
counts["processed"] += len(skipped)
|
||||
counts["unchanged"] += len(skipped)
|
||||
if to_read:
|
||||
batch_results = await self._run_in_executor(
|
||||
self._extract_batch_metadata_sync, to_read
|
||||
)
|
||||
batch_counts = await self._run_in_executor(
|
||||
self._ingest_scan_batch_sync,
|
||||
batch_results,
|
||||
roots=roots,
|
||||
outcomes=outcomes,
|
||||
)
|
||||
for key in counts:
|
||||
counts[key] += batch_counts[key]
|
||||
for group, row, log_line in outcomes:
|
||||
job.record(group, row, log_line)
|
||||
|
||||
@@ -1713,6 +1779,7 @@ class AdminRoutesMixin:
|
||||
pass
|
||||
if not canonical.get("prompt") and not canonical.get("parameters"):
|
||||
return None
|
||||
_with_file_size(canonical, media_file)
|
||||
resolution = resolve_prompt_pair(canonical)
|
||||
if resolution["state"] == "resolved":
|
||||
result = ingest_resolved_pair(
|
||||
|
||||
+55
-3
@@ -10,13 +10,38 @@ import json
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .jobs import JobBusy
|
||||
from .jobs import DEFAULT_SCAN_SCOPE, SCAN_SCOPES, JobBusy
|
||||
|
||||
|
||||
def _not_found():
|
||||
return web.json_response({"success": False, "error": "Job not found"}, status=404)
|
||||
|
||||
|
||||
def _bad_request(message):
|
||||
return web.json_response({"success": False, "error": message}, status=400)
|
||||
|
||||
|
||||
async def _scan_scope(request):
|
||||
"""The requested scan scope ("new" or "all"), or None when the body is invalid.
|
||||
|
||||
An empty body (also a chunked one with no bytes), or one without
|
||||
``scope``, asks for the default.
|
||||
"""
|
||||
if not request.can_read_body:
|
||||
return DEFAULT_SCAN_SCOPE
|
||||
text = await request.text()
|
||||
if not text.strip(): # e.g. a chunked POST that sent no bytes
|
||||
return DEFAULT_SCAN_SCOPE
|
||||
try:
|
||||
body = json.loads(text)
|
||||
except ValueError:
|
||||
return None
|
||||
if not isinstance(body, dict):
|
||||
return None
|
||||
scope = body.get("scope", DEFAULT_SCAN_SCOPE)
|
||||
return scope if scope in SCAN_SCOPES else None
|
||||
|
||||
|
||||
def _summary_event(job):
|
||||
return f"data: {json.dumps(job.summary())}\n\n".encode("utf-8")
|
||||
|
||||
@@ -37,6 +62,10 @@ class JobRoutesMixin:
|
||||
async def job_detail_route(request):
|
||||
return await self.job_detail(request)
|
||||
|
||||
@routes.get("/prompt_manager/jobs/{job_id}/report")
|
||||
async def job_report_route(request):
|
||||
return await self.job_report(request)
|
||||
|
||||
@routes.get("/prompt_manager/jobs/{job_id}/events")
|
||||
async def job_events_route(request):
|
||||
return await self.job_events(request)
|
||||
@@ -56,6 +85,22 @@ class JobRoutesMixin:
|
||||
return _not_found()
|
||||
return web.json_response({"success": True, "job": job.detail()})
|
||||
|
||||
async def job_report(self, request):
|
||||
"""The job's whole record as a JSON attachment (no absolute paths)."""
|
||||
job = self._jobs.get(request.match_info["job_id"])
|
||||
if job is None:
|
||||
return _not_found()
|
||||
body = json.dumps(job.report(), indent=2)
|
||||
disposition = f'attachment; filename="{job.report_filename()}"'
|
||||
return web.Response(
|
||||
text=body,
|
||||
content_type="application/json",
|
||||
headers={
|
||||
"Content-Disposition": disposition,
|
||||
"Cache-Control": "no-store",
|
||||
},
|
||||
)
|
||||
|
||||
async def cancel_job(self, request):
|
||||
"""Stop a job at its next file boundary; a finished job is a no-op."""
|
||||
job_id = request.match_info["job_id"]
|
||||
@@ -65,9 +110,16 @@ class JobRoutesMixin:
|
||||
return web.json_response({"success": True, "cancellation_requested": requested})
|
||||
|
||||
async def start_scan_job(self, request):
|
||||
"""Start the metadata scan as a job (202), or 409 naming the busy job."""
|
||||
"""Start the metadata scan as a job (202), or 409 naming the busy job.
|
||||
|
||||
The body's ``scope`` is "new" (the default: new and unresolved files)
|
||||
or "all"; anything else is a 400.
|
||||
"""
|
||||
files = await _scan_scope(request)
|
||||
if files is None:
|
||||
return _bad_request('The scan scope must be "new" or "all"')
|
||||
try:
|
||||
job = self._start_scan_job()
|
||||
job = self._start_scan_job(files)
|
||||
except JobBusy as busy:
|
||||
running = busy.running
|
||||
return web.json_response(
|
||||
|
||||
@@ -35,12 +35,19 @@ MAX_FINISHED_JOBS = 20 # finished jobs this session keeps
|
||||
# The metadata scan's outcome groups, in counter-tile order of the spec's
|
||||
# mapping, and the per-file scan status each group collects. Files without
|
||||
# usable metadata have no status and go to "no_metadata".
|
||||
# Which files a scan reads (#249 Activity-2): "new" skips the files the Library
|
||||
# already links with a resolved link (get_known_resolved_paths) before reading
|
||||
# them, so new, unlinked, unresolved and ambiguous files are read; "all" reads
|
||||
# every file again. The workspace job defaults to "new"; legacy /scan is "all".
|
||||
SCAN_SCOPES = ("new", "all")
|
||||
DEFAULT_SCAN_SCOPE = "new"
|
||||
SCAN_GROUP_NAMES = ("added", "known", "review", "no_metadata", "errors")
|
||||
SCAN_GROUPS = {
|
||||
"added": "added",
|
||||
"linked": "added",
|
||||
"enriched": "added",
|
||||
"unchanged": "known",
|
||||
"skipped": "known", # "new" scope: known and resolved, not read again
|
||||
"ambiguous": "review",
|
||||
"conflict": "review",
|
||||
"unresolved": "review",
|
||||
@@ -240,6 +247,36 @@ class Job:
|
||||
"log": list(self.log),
|
||||
}
|
||||
|
||||
def report(self):
|
||||
"""The downloadable report: the summary, every outcome group and the log.
|
||||
|
||||
Each group carries its exact count, its listed rows (at most
|
||||
MAX_OUTCOME_ROWS) and whether the list is cut short. Rows and log lines
|
||||
already hold root-relative paths only.
|
||||
"""
|
||||
summary = self.summary()
|
||||
summary.pop("listed", None)
|
||||
summary.pop("log_length", None)
|
||||
return {
|
||||
"report_version": 1,
|
||||
"generated_at": self._clock(),
|
||||
"job": summary,
|
||||
"outcomes": {
|
||||
group: {
|
||||
"count": self.counters.get(group, len(rows)),
|
||||
"rows": [dict(row) for row in rows],
|
||||
"truncated": self.counters.get(group, len(rows)) > len(rows),
|
||||
}
|
||||
for group, rows in self.outcomes.items()
|
||||
},
|
||||
"log": list(self.log),
|
||||
}
|
||||
|
||||
def report_filename(self):
|
||||
"""The download name: prompt-manager-KIND-YYYYMMDD-HHMM.json (local time)."""
|
||||
stamp = time.strftime("%Y%m%d-%H%M", time.localtime(self._clock()))
|
||||
return f"prompt-manager-{self.kind}-{stamp}.json"
|
||||
|
||||
|
||||
class JobRegistry:
|
||||
"""Every job of this server session, running and recently finished."""
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { THEMES, seedConcept, openConcept, openProduction, diffInventory, mutate } from './concept-inventory.mjs';
|
||||
|
||||
// Activity-1 Task 3. The review dialog is compared with the concept's own `scan-review` ("Review metadata scan",
|
||||
@@ -63,16 +63,35 @@ function measureNotice(selector) {
|
||||
};
|
||||
}
|
||||
|
||||
/** D55: the concept's dialog with the production differences applied. */
|
||||
/** A concept `.setting-row` switch: its frame, its label and hint, and its checkbox (not its place-dependent margin). */
|
||||
function measureToggle(selector) {
|
||||
const row = document.querySelector(selector);
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
return {
|
||||
children: [...row.children].map(node => [node.localName, ...node.classList].join('.')),
|
||||
frame: css(row, 'display', 'alignItems', 'justifyContent', 'columnGap', 'paddingTop', 'paddingBottom',
|
||||
'borderTopWidth', 'borderTopStyle', 'borderTopColor'),
|
||||
label: css(row.querySelector('label'), 'fontSize', 'fontWeight', 'color'),
|
||||
hint: css(row.querySelector('p'), 'fontSize', 'color', 'marginTop'),
|
||||
box: css(row.querySelector('input[type="checkbox"]'), 'accentColor', 'width', 'height', 'appearance',
|
||||
'backgroundColor', 'borderTopLeftRadius'),
|
||||
role: row.querySelector('input').getAttribute('role'),
|
||||
labelled: row.querySelector('label').htmlFor === row.querySelector('input').id,
|
||||
};
|
||||
}
|
||||
|
||||
/** D55 and D58: the concept's dialog with the production differences applied. */
|
||||
function mapReviewConcept(reference) {
|
||||
return {
|
||||
...reference,
|
||||
// No sample-plan note (prototype copy); the backup box sits before the actions.
|
||||
// No sample-plan note (prototype copy); the D58 "All files" toggle follows the grid; the backup box sits
|
||||
// before the actions.
|
||||
body: [...reference.body.filter(item => item !== 'p.demo-note').slice(0, -1), 'div.notice', reference.body.at(-1)]
|
||||
.map(item => item === 'div.meta-grid' ? 'dl.meta-grid' : item),
|
||||
// A `dl`, as in the concept's maintenance plan; real sources and the fixed Activity-1 scope.
|
||||
.flatMap(item => item === 'div.meta-grid' ? ['dl.meta-grid', 'div.setting-row'] : [item]),
|
||||
// A `dl`, as in the concept's maintenance plan; real sources. The scope keeps the concept's
|
||||
// "New and unresolved files" until "All files" is checked.
|
||||
grid: { ...reference.grid, tag: 'dl', entries: reference.grid.entries.map(entry => ({ ...entry,
|
||||
dd: { Sources: '2 output folders', Scope: 'All files' }[entry.dt] })) },
|
||||
dd: { Sources: '2 output folders', Scope: 'New and unresolved files' }[entry.dt] })) },
|
||||
actions: { ...reference.actions, buttons: reference.actions.buttons.map(button => button.text === 'Simulate scan'
|
||||
? { ...button, text: 'Start scan' } : button) },
|
||||
};
|
||||
@@ -92,7 +111,8 @@ async function conceptReview(page, theme) {
|
||||
await page.getByRole('combobox', { name: 'Theme', exact: true }).selectOption(theme);
|
||||
await page.locator('#main').getByRole('button', { name: 'Manage sources', exact: true }).click();
|
||||
await expect(page.locator('#main .notice')).toBeVisible();
|
||||
return { review, notice: await page.evaluate(measureNotice, '#main .notice') };
|
||||
return { review, notice: await page.evaluate(measureNotice, '#main .notice'),
|
||||
toggle: await page.evaluate(measureToggle, '#main .setting-row:has(input[type="checkbox"])') };
|
||||
}
|
||||
|
||||
async function productionReview(page, theme) {
|
||||
@@ -103,7 +123,16 @@ async function productionReview(page, theme) {
|
||||
await expect(page.locator('dialog#modal[open] #modal-title')).toHaveText('Review metadata scan');
|
||||
// The swapped dialog can open under the pointer that pressed Review scan; measure it at rest.
|
||||
await page.mouse.move(0, 0);
|
||||
return { review: await page.evaluate(measureReview), notice: await page.evaluate(measureNotice, 'dialog#modal[open] .notice') };
|
||||
const review = await page.evaluate(measureReview);
|
||||
const notice = await page.evaluate(measureNotice, 'dialog#modal[open] .notice');
|
||||
// The concept's switch is on; compare ours on too, then turn it off again (the default).
|
||||
const allFiles = page.getByRole('switch', { name: 'All files', exact: true });
|
||||
await allFiles.check();
|
||||
await page.mouse.move(0, 0);
|
||||
const toggle = await page.evaluate(measureToggle, 'dialog#modal[open] .setting-row');
|
||||
await allFiles.uncheck();
|
||||
await page.mouse.move(0, 0);
|
||||
return { review, notice, toggle };
|
||||
}
|
||||
|
||||
for (const theme of THEMES) test(`review dialog matches the concept scan-review ${theme}`, async ({ page, request }) => {
|
||||
@@ -114,19 +143,29 @@ for (const theme of THEMES) test(`review dialog matches the concept scan-review
|
||||
const production = await productionReview(page, theme);
|
||||
expect(diffInventory(mapReviewConcept(concept.review), production.review)).toEqual([]);
|
||||
expect(diffInventory(concept.notice, production.notice)).toEqual([]);
|
||||
expect(diffInventory(concept.toggle, production.toggle)).toEqual([]);
|
||||
await expect(page.locator('dialog#modal[open] .setting-row > div > label')).toHaveText('All files');
|
||||
await expect(page.locator('dialog#modal[open] .setting-row input')).not.toBeChecked();
|
||||
await expect(page.getByRole('switch', { name: 'All files', exact: true })).toBeVisible();
|
||||
await expect(page.locator('dialog#modal[open] .notice h3')).toHaveText('Back up the database first (recommended)');
|
||||
await expect(page.locator('dialog#modal[open] .notice .row-actions button')).toHaveText(['Back up now']);
|
||||
|
||||
// Mutation check: one CSS change and one removed element are both noticed, then undone to an exact match again.
|
||||
const undoCss = await mutate(page, { css: '#modal .meta-grid { gap: 3px; } #modal .notice { padding: 2px; }' });
|
||||
const undoCss = await mutate(page, { css: '#modal .meta-grid { gap: 3px; } #modal .notice { padding: 2px; } #modal .setting-row { gap: 3px; }' });
|
||||
const cssMutated = await page.evaluate(measureReview);
|
||||
const noticeMutated = await page.evaluate(measureNotice, 'dialog#modal[open] .notice');
|
||||
await page.getByRole('switch', { name: 'All files', exact: true }).check();
|
||||
await page.mouse.move(0, 0);
|
||||
const toggleMutated = await page.evaluate(measureToggle, 'dialog#modal[open] .setting-row');
|
||||
await page.getByRole('switch', { name: 'All files', exact: true }).uncheck();
|
||||
await page.mouse.move(0, 0);
|
||||
expect(diffInventory(concept.toggle, toggleMutated).map(item => item.path)).toEqual(['frame.columnGap']);
|
||||
expect(diffInventory(mapReviewConcept(concept.review), cssMutated).map(item => item.path)).toEqual(expect.arrayContaining(['grid.columnGap', 'grid.rowGap']));
|
||||
expect(diffInventory(concept.notice, noticeMutated).map(item => item.path)).toEqual(['frame.paddingTop', 'frame.paddingLeft']);
|
||||
await undoCss();
|
||||
const undoDom = await mutate(page, { remove: ['#modal-body > p.small'] });
|
||||
const domMutated = await page.evaluate(() => [...document.querySelectorAll('#modal-body > *')].map(node => node.localName));
|
||||
expect(domMutated).toEqual(['dl', 'div', 'div']);
|
||||
expect(domMutated).toEqual(['dl', 'div', 'div', 'div']);
|
||||
await undoDom();
|
||||
expect(diffInventory(mapReviewConcept(concept.review), await page.evaluate(measureReview))).toEqual([]);
|
||||
});
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
|
||||
test.beforeEach(async ({page, browserName}) => {
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// same concept selectors in the immutable concept (/__test__/canonical/workspace) and in production, which now renders
|
||||
// the concept's classes. The concept is never rewritten: each page's map<Page>Concept(reference) applies only the
|
||||
// recorded spec-§8 deviations, each commented with its D-number, and diffInventory must then report nothing.
|
||||
import {expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {expect} from './fixtures.mjs';
|
||||
export {diffInventory} from './library-editor-canonical-comparison.mjs';
|
||||
|
||||
export const CONCEPT_URL='/__test__/canonical/workspace';
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
// Shared `test` for the page workflows: every Playwright worker owns one
|
||||
// tests/browser/server.py process on a port the OS chose for it, with its own temp root
|
||||
// and database. `baseURL` (and so `page`, `request` and `browser.newContext`)
|
||||
// points at that worker's server. A server that dies fails the running test
|
||||
// with its stderr tail, and the next test in the worker gets a fresh server.
|
||||
import { test as base, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { createServerSupervisor, startBrowserServer } from '../../tools/vendor/start-browser-server.mjs';
|
||||
|
||||
const SERVER_FIXTURE_TIMEOUT_MS = 90_000;
|
||||
|
||||
export const test = base.extend({
|
||||
testServer: [async ({}, use) => {
|
||||
const supervisor = createServerSupervisor({ start: () => startBrowserServer() });
|
||||
try {
|
||||
await use(supervisor);
|
||||
} finally {
|
||||
await supervisor.stop();
|
||||
}
|
||||
}, { scope: 'worker', timeout: SERVER_FIXTURE_TIMEOUT_MS }],
|
||||
workerServer: [async ({ testServer }, use) => {
|
||||
const server = await testServer.ensure();
|
||||
await use(server);
|
||||
const crash = testServer.crash(server);
|
||||
if (crash) throw crash;
|
||||
}, { auto: true, timeout: SERVER_FIXTURE_TIMEOUT_MS }],
|
||||
baseURL: async ({ workerServer }, use) => use(workerServer.url),
|
||||
});
|
||||
|
||||
export { expect };
|
||||
@@ -1,5 +1,5 @@
|
||||
import {readFileSync} from 'node:fs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
|
||||
for (const variant of ['Focused', 'Split']) {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept, openConcept, openProduction, SIZES } from './concept-inventory.mjs';
|
||||
// D10/D20: the heading is the linked positive's title (its confirmed first line here), never the pair's
|
||||
// "Inspector recipe", which names the negative variant.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { writeFile } from 'node:fs/promises';
|
||||
import { expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { expect } from './fixtures.mjs';
|
||||
|
||||
// Content, row counts, and image bytes come from independent synthetic datasets.
|
||||
// Saved choices and actual executions are separate production sections. The concept's
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,diffInventory,mutate,stripExact} from './concept-inventory.mjs';
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,diffInventory,mutate,stripExact,countCopy,reflow,withoutItems} from './concept-inventory.mjs';
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {SIZES,THEMES,seedConcept,openConcept,openProduction,inspectorOf,diffInventory,mutate,countCopy} from './concept-inventory.mjs';
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,diffInventory,mutate,stripExact,countCopy,reflow} from './concept-inventory.mjs';
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
|
||||
// Artifact-only captures of the editor states that have no concept counterpart (archive confirmation, Show archived,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {readFileSync} from 'node:fs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
|
||||
const pair={id:4,positive_text:'Library positive',negative_text:'Library negative',category:'Portrait',tags:['studio']};
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
|
||||
const moduleUrl = '/prompt_manager/js/shared/toolkit/prompt-bridge.js';
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
|
||||
const workspace='/prompt_manager/workspace#/images';
|
||||
const details=page=>page.getByRole('complementary',{name:'Selected image details',exact:true});
|
||||
|
||||
+34
-6
@@ -4,6 +4,7 @@ import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
@@ -515,14 +516,41 @@ def create_app(root: Path) -> web.Application:
|
||||
return app
|
||||
|
||||
|
||||
def main() -> None:
|
||||
HOST = "127.0.0.1"
|
||||
PORT_ANNOUNCEMENT = "PM_TEST_SERVER_PORT="
|
||||
|
||||
|
||||
def bind_listener(port: int) -> socket.socket:
|
||||
"""Bind the server's socket before the app is built.
|
||||
|
||||
``port`` 0 lets the OS choose a free port, which this process then holds,
|
||||
so parallel launchers never race for one between choosing and binding.
|
||||
"""
|
||||
listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
try:
|
||||
if port and os.name != "nt":
|
||||
# aiohttp's own default on POSIX; on Windows it would let a
|
||||
# second process share the port.
|
||||
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
listener.bind((HOST, port))
|
||||
listener.listen(128)
|
||||
except OSError:
|
||||
listener.close()
|
||||
raise
|
||||
return listener
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--port", type=int, default=8765)
|
||||
args = parser.parse_args()
|
||||
parser.add_argument(
|
||||
"--port", type=int, default=8765, help="0 binds an OS-chosen free port"
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
listener = bind_listener(args.port)
|
||||
# One machine-readable line tells the launcher which port this process holds.
|
||||
print(f"{PORT_ANNOUNCEMENT}{listener.getsockname()[1]}", flush=True)
|
||||
with tempfile.TemporaryDirectory(prefix="pm-browser-") as directory:
|
||||
web.run_app(
|
||||
create_app(Path(directory).resolve()), host="127.0.0.1", port=args.port
|
||||
)
|
||||
web.run_app(create_app(Path(directory).resolve()), sock=listener)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept as openConceptPage,openProduction,inventoryOf,diffInventory,focusRingOf,hoverStateOf,mutate} from './concept-inventory.mjs';
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
|
||||
// Artifact-only captures of Tags & categories states that have no concept counterpart: the Manage categories
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
// Each Playwright worker owns its server: `request`, `page` and manual
|
||||
// `browser.newContext()` all reach it through `baseURL`, and another server
|
||||
// process never sees its database.
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { startBrowserServer } from '../../tools/vendor/start-browser-server.mjs';
|
||||
|
||||
const search = text => `/prompt_manager/search?text=${encodeURIComponent(text)}`;
|
||||
|
||||
test('every fixture reaches this worker\'s own server and another server keeps its own database', async ({ request, page, browser, baseURL, workerServer }, testInfo) => {
|
||||
expect(baseURL).toBe(workerServer.url);
|
||||
await expect(await request.post('/__test__/reset')).toBeOK();
|
||||
const marker = `worker ${testInfo.workerIndex} isolation ${testInfo.project.name} ${testInfo.repeatEachIndex}`;
|
||||
await expect(await request.post('/prompt_manager/save', { data: { text: marker } })).toBeOK();
|
||||
expect((await (await request.get(search(marker))).json()).count).toBe(1);
|
||||
|
||||
const viaPage = await page.request.get(search(marker));
|
||||
expect(viaPage.url().startsWith(`${baseURL}/`)).toBe(true);
|
||||
expect((await viaPage.json()).count).toBe(1);
|
||||
const context = await browser.newContext();
|
||||
try {
|
||||
const response = await (await context.newPage()).goto(search(marker));
|
||||
expect(response.url().startsWith(`${baseURL}/`)).toBe(true);
|
||||
expect((await response.json()).count).toBe(1);
|
||||
} finally {
|
||||
await context.close();
|
||||
}
|
||||
|
||||
const other = await startBrowserServer();
|
||||
try {
|
||||
expect(other.url).not.toBe(baseURL);
|
||||
expect((await (await fetch(`${other.url}${search(marker)}`)).json()).count).toBe(0);
|
||||
} finally {
|
||||
await other.stop();
|
||||
}
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
|
||||
test.beforeEach(async ({ request, page, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept, openConcept, openProduction } from './concept-inventory.mjs';
|
||||
|
||||
// Activity (#249 Activity-1): the concept's Activity page (app.js renderActivity 732–738), its top-bar button
|
||||
@@ -392,16 +392,69 @@ test('start scan creates a job, toasts, and opens its view', async ({ page, requ
|
||||
await expect(scanLog(page).locator('pre')).toContainText('Scan finished');
|
||||
});
|
||||
|
||||
// ── Task 4 (Activity-2): scan scope, new and unresolved files by default ─────────────────────
|
||||
|
||||
test('all files checkbox changes the scope line and the request', async ({ page }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const bodies = [];
|
||||
page.on('request', item => {
|
||||
if (item.method() === 'POST' && new URL(item.url()).pathname === '/prompt_manager/jobs/scan') bodies.push(item.postDataJSON());
|
||||
});
|
||||
const review = page.getByRole('dialog', { name: 'Review metadata scan', exact: true });
|
||||
const scopeValue = review.locator('.meta-grid > div').filter({ has: page.locator('dt', { hasText: 'Scope' }) }).locator('dd');
|
||||
// The concept's switch: a checkbox with role=switch.
|
||||
const allFiles = review.getByRole('switch', { name: 'All files', exact: true });
|
||||
const openReview = async () => {
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await page.getByRole('button', { name: 'Import', exact: true }).click();
|
||||
await page.getByRole('dialog', { name: 'Import & inspect', exact: true }).locator('.operation')
|
||||
.filter({ hasText: 'Scan for metadata' }).getByRole('button', { name: 'Review scan', exact: true }).click();
|
||||
await expect(review).toBeVisible();
|
||||
};
|
||||
|
||||
// The default: the concept's "New and unresolved files", the box unchecked; the request asks for "new".
|
||||
await openReview();
|
||||
await expect(scopeValue).toHaveText('New and unresolved files');
|
||||
await expect(allFiles).not.toBeChecked();
|
||||
await expect(review.locator('.setting-row p')).toHaveText('Also read the files already linked to their prompt again.');
|
||||
await expect(allFiles).toHaveAccessibleDescription('Also read the files already linked to their prompt again.');
|
||||
// Checking and unchecking switches the Scope value both ways.
|
||||
await allFiles.check();
|
||||
await expect(scopeValue).toHaveText('All files');
|
||||
await allFiles.uncheck();
|
||||
await expect(scopeValue).toHaveText('New and unresolved files');
|
||||
await review.getByRole('button', { name: 'Start scan', exact: true }).click();
|
||||
await expect(page).toHaveURL(/#\/activity\/[0-9a-f]{32}$/);
|
||||
await expect.poll(() => bodies.length).toBe(1);
|
||||
expect(bodies[0]).toEqual({ scope: 'new' });
|
||||
await expect(heading(page).locator('h1')).toHaveText('Scan for metadata');
|
||||
// Wait for the first scan to end so the second is not refused as busy.
|
||||
await scanLog(page).locator('summary').click();
|
||||
await expect(scanLog(page).locator('pre')).toContainText('Scan finished', { timeout: 15000 });
|
||||
await expect(scanLog(page).locator('pre')).toContainText('new and unresolved files');
|
||||
|
||||
// Checked: "All files", and the request asks for "all".
|
||||
await openReview();
|
||||
await allFiles.check();
|
||||
await expect(scopeValue).toHaveText('All files');
|
||||
await review.getByRole('button', { name: 'Start scan', exact: true }).click();
|
||||
await expect.poll(() => bodies.length).toBe(2);
|
||||
expect(bodies[1]).toEqual({ scope: 'all' });
|
||||
await scanLog(page).locator('summary').click();
|
||||
await expect(scanLog(page).locator('pre')).toContainText('Scan started · ', { timeout: 15000 });
|
||||
await expect(scanLog(page).locator('pre')).toContainText('all files');
|
||||
});
|
||||
|
||||
test('job view counters follow the mapping and update live', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 8240, counters: { added: 1420, known: 6511, review: 37, no_metadata: 254, errors: 18 } });
|
||||
await openJob(page, job.id);
|
||||
// The header: the title, the detail line, then Download report (Task 5) and Cancel scan (danger).
|
||||
// The header: the title, the detail line, then Download report (enabled in Task 5) and Cancel scan (danger).
|
||||
await expect(heading(page).locator('h1')).toHaveText('Scan for metadata');
|
||||
await expect(heading(page).locator('p.muted')).toHaveText(/^Started \d\d:\d\d · 2 output folders · closing this view does not cancel it$/);
|
||||
await expect(heading(page).locator('.heading-actions > button')).toHaveText(['Download report', 'Cancel scan']);
|
||||
await expect(heading(page).getByRole('button', { name: 'Download report', exact: true })).toBeDisabled();
|
||||
await expect(heading(page).getByRole('button', { name: 'Download report', exact: true })).toBeEnabled();
|
||||
await expect(heading(page).getByRole('button', { name: 'Cancel scan', exact: true })).toHaveClass(/danger/);
|
||||
// Progress: the bar, the server's line and the five tiles in order and colour.
|
||||
await expect(progressSection(page).locator('.panel-heading > span.small')).toHaveText('Processing file 8,240 of 19,622');
|
||||
@@ -558,6 +611,19 @@ test('job view renders 200-row cap note', async ({ page, request }) => {
|
||||
await step(request, job.id, { processed: 260, bulk: { group: 'added', count: 250 }, records: [
|
||||
...Array.from({ length: 10 }, (_, index) => ({ group: 'no_metadata', path: `misc/plain_${index}.png`, detail: 'No prompt metadata in the file' })),
|
||||
] });
|
||||
// Every detail read of this job: in flight, and whether one has returned the full listing.
|
||||
const detailUrl = `/prompt_manager/jobs/${job.id}`;
|
||||
const isDetail = item => new URL(item.url()).pathname === detailUrl;
|
||||
const detailReads = { inFlight: 0, full: false };
|
||||
page.on('request', item => { if (isDetail(item)) detailReads.inFlight += 1; });
|
||||
page.on('requestfailed', item => { if (isDetail(item)) detailReads.inFlight -= 1; });
|
||||
page.on('response', async response => {
|
||||
if (!isDetail(response)) return;
|
||||
const body = await response.json().catch(() => null);
|
||||
const outcomes = body?.job?.outcomes ?? {};
|
||||
if (outcomes.added?.length === 200 && outcomes.no_metadata?.length === 10) detailReads.full = true;
|
||||
detailReads.inFlight -= 1;
|
||||
});
|
||||
await openJob(page, job.id);
|
||||
await expect(tiles(page).first().locator('dd')).toHaveText('250');
|
||||
await expect(fileRows(page)).toHaveCount(200);
|
||||
@@ -573,6 +639,9 @@ test('job view renders 200-row cap note', async ({ page, request }) => {
|
||||
|
||||
// Review M12: with the list capped and the log closed, more files only update the counters; the detail is read
|
||||
// again only when the log opens with new lines.
|
||||
// The reads from opening the view have settled: one returned the full listing and none is in flight. The view
|
||||
// re-reads only while what the summary lists differs from the last read, so no further read is due.
|
||||
await expect.poll(() => detailReads.full && detailReads.inFlight === 0).toBe(true);
|
||||
const reads = [];
|
||||
page.on('request', item => { if (new URL(item.url()).pathname === `/prompt_manager/jobs/${job.id}`) reads.push(Date.now()); });
|
||||
await step(request, job.id, { processed: 300, bulk: { group: 'added', count: 40 } });
|
||||
@@ -639,6 +708,8 @@ test('an unknown or restarted job view says so', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
await expect(heading(page).locator('p.muted')).toHaveText(LOST);
|
||||
await expect(heading(page).getByRole('button', { name: 'Cancel scan', exact: true })).toHaveCount(0);
|
||||
// A lost job has no report on the server (review M9): Download report is disabled rather than failing.
|
||||
await expect(heading(page).getByRole('button', { name: 'Download report', exact: true })).toBeDisabled();
|
||||
// What was read before the restart stays on the page.
|
||||
await expect(fileRows(page)).toHaveCount(1);
|
||||
await expect(pill(page)).toBeHidden();
|
||||
@@ -662,3 +733,69 @@ test('a finished job the server pruned leaves its open view honest', async ({ pa
|
||||
await expect(heading(page).locator('h1')).toBeFocused();
|
||||
await expect(page).toHaveTitle('Activity · Prompt Manager');
|
||||
});
|
||||
|
||||
// Activity-2 Task 5 (D59): Download report and the log's scroll position.
|
||||
test('download report saves the job JSON', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 2, records: [
|
||||
{ group: 'added', path: 'sub/a.png', root: 'output', detail: 'Added' },
|
||||
{ group: 'errors', path: 'broken/b.png', root: 'output', detail: 'Error' },
|
||||
] });
|
||||
await finish(request, job.id);
|
||||
await openJob(page, job.id);
|
||||
const button = heading(page).getByRole('button', { name: 'Download report', exact: true });
|
||||
await expect(button).toBeEnabled();
|
||||
const [download] = await Promise.all([page.waitForEvent('download'), button.click()]);
|
||||
expect(download.suggestedFilename()).toMatch(/^prompt-manager-scan-\d{8}-\d{4}\.json$/);
|
||||
const path = await download.path();
|
||||
const report = JSON.parse(await (await import('node:fs/promises')).readFile(path, 'utf8'));
|
||||
expect(report.job.id).toBe(job.id);
|
||||
expect(report.outcomes.added.rows.map(row => row.path)).toEqual(['sub/a.png']);
|
||||
expect(report.outcomes.errors.truncated).toBe(false);
|
||||
expect(report.log.length).toBeGreaterThan(0);
|
||||
await expect(toast(page)).toHaveText('Job report downloaded.');
|
||||
});
|
||||
|
||||
test('log keeps the reader\'s scroll position', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
const more = (from, count) => Array.from({ length: count }, (_, index) => ({ group: 'known', path: `log/f_${from + index}.png`,
|
||||
root: 'output', detail: 'Known' }));
|
||||
await step(request, job.id, { processed: 60, records: more(0, 60) });
|
||||
await openJob(page, job.id);
|
||||
await scanLog(page).locator('summary').click();
|
||||
const pre = scanLog(page).locator('pre');
|
||||
await expect(pre).toContainText('log/f_59.png');
|
||||
const metrics = () => pre.evaluate(node => ({ top: node.scrollTop, max: node.scrollHeight - node.clientHeight }));
|
||||
// Opening starts at the newest line.
|
||||
await expect.poll(async () => { const m = await metrics(); return m.max > 0 && m.top >= m.max - 2; }).toBe(true);
|
||||
// Pinned to the bottom: new lines are followed.
|
||||
await step(request, job.id, { processed: 90, records: more(60, 30) });
|
||||
await expect(pre).toContainText('log/f_89.png');
|
||||
await expect.poll(async () => { const m = await metrics(); return m.top >= m.max - 2; }).toBe(true);
|
||||
// Scrolled up: the position stays while more lines arrive.
|
||||
await pre.evaluate(node => { node.scrollTop = 40; });
|
||||
await step(request, job.id, { processed: 120, records: more(90, 30) });
|
||||
await expect(pre).toContainText('log/f_119.png');
|
||||
expect((await metrics()).top).toBe(40);
|
||||
// Closed and opened again (review M6): every open starts at the newest line, also after lines arrived meanwhile.
|
||||
await scanLog(page).locator('summary').click();
|
||||
await expect(scanLog(page)).not.toHaveAttribute('open', '');
|
||||
await step(request, job.id, { processed: 150, records: more(120, 30) });
|
||||
await scanLog(page).locator('summary').click();
|
||||
await expect(pre).toContainText('log/f_149.png');
|
||||
await expect.poll(async () => { const m = await metrics(); return m.max > 0 && m.top >= m.max - 2; }).toBe(true);
|
||||
});
|
||||
|
||||
test('a failed report download says so and leaves the button usable', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await finish(request, job.id);
|
||||
await openJob(page, job.id);
|
||||
await page.route(`**/prompt_manager/jobs/${job.id}/report`, route => route.fulfill({ status: 500, body: 'no' }));
|
||||
const button = heading(page).getByRole('button', { name: 'Download report', exact: true });
|
||||
await button.click();
|
||||
await expect(toast(page)).toHaveText('The report could not be downloaded. Try again.');
|
||||
await expect(button).not.toHaveAttribute('aria-disabled', 'true');
|
||||
});
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {readFile,writeFile} from 'node:fs/promises';
|
||||
import {createHash} from 'node:crypto';
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept, openConcept, openProduction } from './concept-inventory.mjs';
|
||||
|
||||
const workspace = '/prompt_manager/workspace#/images';
|
||||
@@ -2060,7 +2060,7 @@ test('scan for metadata opens the review dialog from Import', async ({ page, req
|
||||
await openScanReview(page);
|
||||
const dialog = scanReview(page);
|
||||
await expect(dialog.locator('dl.meta-grid > div > dt')).toHaveText(['Sources', 'Scope']);
|
||||
await expect(dialog.locator('dl.meta-grid > div > dd')).toHaveText(['2 output folders', 'All files']);
|
||||
await expect(dialog.locator('dl.meta-grid > div > dd')).toHaveText(['2 output folders', 'New and unresolved files']);
|
||||
await expect(dialog.locator('#modal-body > p.small')).toHaveText('Enrich positive / negative associations, reuse canonical text and report unchanged files. Do not inflate run counts or recreate existing thumbnails.');
|
||||
await expect(dialog.locator('.notice h3')).toHaveText('Back up the database first (recommended)');
|
||||
await expect(dialog.locator('.dialog-actions button')).toHaveText(['Cancel', 'Start scan']);
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {inventoryOf,mapConcept,diffInventory} from './library-editor-canonical-comparison.mjs';
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
|
||||
const route='/prompt_manager/workspace#/library';
|
||||
const cards=page=>page.locator('article[data-prompt-id]');
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
|
||||
// Saved views (#249 D16, maintainer ruling 2026-10-21): the toolbar's Save view (concept app.js a359, a1330–1346),
|
||||
// the sidebar SAVED VIEWS rows (a331–335) and `#/saved/<id>` (a1158–1162), on Images and the Prompt library.
|
||||
|
||||
@@ -3,7 +3,7 @@ import { readFile } from 'node:fs/promises';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { captureCanonicalState, historyGeometry, compareCanonicalHistory, nativeDialogState, exerciseCurrentPairSend, assertRecordedHistory } from './history-canonical-comparison.mjs';
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
|
||||
test.use({ locale: 'en-US', timezoneId: 'UTC', reducedMotion: 'reduce' });
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {pageInventoryOf,inventoryOf,diffInventory,mapVocabularyConcept,mapVocabularyDialog} from './vocabulary-canonical-comparison.mjs';
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
const statistic = (page, label) => page.locator('#workspace-main dt')
|
||||
|
||||
@@ -129,7 +129,7 @@ def register_activity_commands(app, routes, api):
|
||||
str(data.get("title", "Scan for metadata")),
|
||||
record_changing=True,
|
||||
scope={
|
||||
"mode": "all",
|
||||
"files": str(data.get("files", "new")),
|
||||
"output_folders": count(data.get("output_folders"), 2),
|
||||
},
|
||||
runner=fake_runner(ending),
|
||||
|
||||
@@ -11,7 +11,7 @@ const ID_C = 'c'.repeat(32);
|
||||
function summary(overrides = {}) {
|
||||
return { id: ID_A, kind: 'scan', title: 'Scan for metadata', state: 'running', started_at: 1000, finished_at: null,
|
||||
progress: 0, status_line: '', processed: 0, total: 0,
|
||||
counters: { added: 0, known: 0, review: 0, no_metadata: 0, errors: 0 }, scope: { mode: 'all', output_folders: 2 },
|
||||
counters: { added: 0, known: 0, review: 0, no_metadata: 0, errors: 0 }, scope: { files: 'new', output_folders: 2 },
|
||||
...overrides };
|
||||
}
|
||||
|
||||
|
||||
@@ -2094,6 +2094,72 @@ class TestActivityJobs(AdminAPITestCase):
|
||||
(legacy["processed"], legacy["found"], legacy["failed"]), (6, 4, 1)
|
||||
)
|
||||
|
||||
async def test_report_is_attachment_without_absolute_paths(self):
|
||||
"""Task 5: the report downloads as JSON, whole and without server paths."""
|
||||
import re
|
||||
|
||||
from py.api.jobs import MAX_OUTCOME_ROWS
|
||||
|
||||
async def runner(job):
|
||||
absolute = f"{self.tmpdir}/output/deep/abs.png"
|
||||
job.record(
|
||||
"added",
|
||||
{"path": absolute, "root": "output", "root_index": 0, "detail": "ok"},
|
||||
f"+ {absolute} · ok",
|
||||
)
|
||||
for index in range(MAX_OUTCOME_ROWS + 5):
|
||||
job.record(
|
||||
"known",
|
||||
{"path": f"k/{index}.png", "root": "output", "detail": "x"},
|
||||
f"= k/{index}.png",
|
||||
)
|
||||
job.update(processed=MAX_OUTCOME_ROWS + 6, total=MAX_OUTCOME_ROWS + 6)
|
||||
|
||||
job = self.api._jobs.start(
|
||||
"scan",
|
||||
"Scan for metadata",
|
||||
record_changing=True,
|
||||
scope={"files": "new", "output_folders": 1},
|
||||
runner=runner,
|
||||
groups=("added", "known", "review", "no_metadata", "errors"),
|
||||
)
|
||||
await asyncio.wait_for(job.task, 10)
|
||||
|
||||
resp = await self.client.request("GET", f"/prompt_manager/jobs/{job.id}/report")
|
||||
text = await resp.text()
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertIn("application/json", resp.headers["Content-Type"])
|
||||
disposition = resp.headers["Content-Disposition"]
|
||||
self.assertRegex(
|
||||
disposition,
|
||||
r'^attachment; filename="prompt-manager-scan-\d{8}-\d{4}\.json"$',
|
||||
)
|
||||
self.assertNotIn(self.tmpdir, text)
|
||||
self.assertIsNone(
|
||||
re.search(r"[A-Za-z]:[\\/]|(?<![\w.])/(?:tmp|home|var)/", text)
|
||||
)
|
||||
report = json.loads(text)
|
||||
self.assertEqual(report["job"]["id"], job.id)
|
||||
self.assertEqual(report["job"]["scope"]["files"], "new")
|
||||
self.assertEqual(report["job"]["counters"]["known"], MAX_OUTCOME_ROWS + 5)
|
||||
added = report["outcomes"]["added"]
|
||||
self.assertEqual(
|
||||
(added["count"], len(added["rows"]), added["truncated"]), (1, 1, False)
|
||||
)
|
||||
self.assertEqual(added["rows"][0]["path"], "abs.png")
|
||||
known = report["outcomes"]["known"]
|
||||
self.assertEqual(
|
||||
(known["count"], len(known["rows"]), known["truncated"]),
|
||||
(MAX_OUTCOME_ROWS + 5, MAX_OUTCOME_ROWS, True),
|
||||
)
|
||||
self.assertEqual(report["outcomes"]["errors"]["rows"], [])
|
||||
self.assertFalse(report["outcomes"]["errors"]["truncated"])
|
||||
self.assertEqual(report["log"], job.detail()["log"])
|
||||
self.assertIn("generated_at", report)
|
||||
|
||||
resp = await self.client.request("GET", "/prompt_manager/jobs/nope/report")
|
||||
self.assertEqual(resp.status, 404)
|
||||
|
||||
async def test_cancel_after_first_batch_keeps_partial_counts(self):
|
||||
"""Review M7: the real scan stops at the batch boundary, keeping batch 1."""
|
||||
import threading
|
||||
|
||||
+13
-3
@@ -13,12 +13,14 @@ import unittest
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from py.api.jobs import ( # noqa: E402
|
||||
DEFAULT_SCAN_SCOPE,
|
||||
LOG_CAP_LINE,
|
||||
MAX_FINISHED_JOBS,
|
||||
MAX_LOG_LINES,
|
||||
MAX_OUTCOME_ROWS,
|
||||
SCAN_GROUP_NAMES,
|
||||
SCAN_GROUPS,
|
||||
SCAN_SCOPES,
|
||||
Job,
|
||||
JobBusy,
|
||||
JobRegistry,
|
||||
@@ -155,7 +157,7 @@ class TestJobRegistry(unittest.IsolatedAsyncioTestCase):
|
||||
"scan",
|
||||
"Scan for metadata",
|
||||
record_changing=True,
|
||||
scope={"mode": "all"},
|
||||
scope={"files": "new", "output_folders": 2},
|
||||
runner=_gated_runner(gate),
|
||||
)
|
||||
self.assertEqual(job.state, "running")
|
||||
@@ -181,7 +183,7 @@ class TestJobRegistry(unittest.IsolatedAsyncioTestCase):
|
||||
"log_length",
|
||||
},
|
||||
)
|
||||
self.assertEqual(summary["scope"], {"mode": "all"})
|
||||
self.assertEqual(summary["scope"], {"files": "new", "output_folders": 2})
|
||||
# What the detail would list, so a view reads it only when that changed.
|
||||
self.assertEqual((summary["listed"], summary["log_length"]), ({}, 0))
|
||||
self.assertIsNone(summary["finished_at"])
|
||||
@@ -371,7 +373,12 @@ class TestJobOutcomesAndLog(unittest.TestCase):
|
||||
job.record("bogus", {"path": "a.png", "root": "output"}, "x")
|
||||
|
||||
def test_detail_and_progress(self):
|
||||
job = Job("scan", "Scan", scope={"mode": "all"}, groups=SCAN_GROUP_NAMES)
|
||||
job = Job(
|
||||
"scan",
|
||||
"Scan",
|
||||
scope={"files": "new", "output_folders": 2},
|
||||
groups=SCAN_GROUP_NAMES,
|
||||
)
|
||||
job.update(progress=150, status_line="Processing file 3 of 4", processed=3)
|
||||
job.update(total=4)
|
||||
job.record("known", {"path": "a.png", "root": "output"}, "= a.png")
|
||||
@@ -395,12 +402,15 @@ class TestJobOutcomesAndLog(unittest.TestCase):
|
||||
"linked": "added",
|
||||
"enriched": "added",
|
||||
"unchanged": "known",
|
||||
# Activity-2: a file the "new" scope skipped is Already known.
|
||||
"skipped": "known",
|
||||
"ambiguous": "review",
|
||||
"conflict": "review",
|
||||
"unresolved": "review",
|
||||
"failed": "errors",
|
||||
},
|
||||
)
|
||||
self.assertEqual((SCAN_SCOPES, DEFAULT_SCAN_SCOPE), (("new", "all"), "new"))
|
||||
self.assertEqual(
|
||||
SCAN_GROUP_NAMES, ("added", "known", "review", "no_metadata", "errors")
|
||||
)
|
||||
|
||||
@@ -0,0 +1,390 @@
|
||||
"""Scan scope (#249 Activity-2 Task 4): new and unresolved files by default.
|
||||
|
||||
A "Scan for metadata" job started without ``scope`` (or with ``"new"``) skips
|
||||
the files the Library already links with a resolved link, before their
|
||||
metadata is read, and counts them as Already known. ``"all"`` reads every file
|
||||
again, as the legacy ``/prompt_manager/scan`` route always does.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
try:
|
||||
from tests.test_api_admin import AdminAPITestCase, _sse_payloads, _write_png
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from test_api_admin import AdminAPITestCase, _sse_payloads, _write_png
|
||||
|
||||
from database import operations # noqa: E402
|
||||
from database.pair_ingestion import ingest_resolved_pair # noqa: E402
|
||||
from py.config import GalleryConfig # noqa: E402
|
||||
from utils.prompt_pair_resolver import resolve_prompt_pair # noqa: E402
|
||||
|
||||
SKIPPED_DETAIL = "Skipped · already linked"
|
||||
ON_DISK = object() # _link_resolved: store the file's size on disk
|
||||
|
||||
|
||||
def _graph(text):
|
||||
return {
|
||||
"1": {"class_type": "CLIPTextEncode", "inputs": {"text": text}},
|
||||
"2": {"class_type": "CLIPTextEncode", "inputs": {"text": "bad"}},
|
||||
"3": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {"positive": ["1", 0], "negative": ["2", 0]},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class ScanScopeTestCase(AdminAPITestCase):
|
||||
"""Helpers: link files the way the scan and the monitor do, spy on reads."""
|
||||
|
||||
def _link_resolved(self, path, text, size=ON_DISK):
|
||||
"""``path`` linked with a resolved pair link (what a resolved scan writes).
|
||||
|
||||
The row stores ``size`` as its file size: by default the file's size on
|
||||
disk (None when the file does not exist).
|
||||
"""
|
||||
if size is ON_DISK:
|
||||
size = os.path.getsize(path) if os.path.exists(path) else None
|
||||
metadata = {"prompt": _graph(text)}
|
||||
if size is not None:
|
||||
metadata["file_info"] = {"size": size}
|
||||
result = ingest_resolved_pair(
|
||||
self.api.db, str(path), metadata, resolve_prompt_pair(metadata)
|
||||
)
|
||||
self.assertEqual(result["status"], "added")
|
||||
|
||||
def _link_without_pair(self, path, text):
|
||||
"""``path`` linked to a prompt with no pair link (fallback / FIFO link)."""
|
||||
prompt_id = self.api.db.save_prompt(text, None, [], None, None)
|
||||
self.assertTrue(self.api.db.link_image_to_prompt(prompt_id, str(path)))
|
||||
return prompt_id
|
||||
|
||||
def _spy_reads(self):
|
||||
reads = []
|
||||
original = self.api._extract_comfyui_metadata
|
||||
|
||||
def spy(path):
|
||||
reads.append(os.path.basename(path))
|
||||
return original(path)
|
||||
|
||||
self.api._extract_comfyui_metadata = spy
|
||||
return reads
|
||||
|
||||
async def _scan(self, body=None):
|
||||
kwargs = {} if body is None else {"json": body}
|
||||
resp = await self.client.request("POST", "/prompt_manager/jobs/scan", **kwargs)
|
||||
data = await resp.json()
|
||||
self.assertEqual(resp.status, 202, data)
|
||||
await asyncio.wait_for(self.api._scan_job.task, 10)
|
||||
detail = await self.client.request(
|
||||
"GET", f"/prompt_manager/jobs/{data['job']['id']}"
|
||||
)
|
||||
return (await detail.json())["job"]
|
||||
|
||||
|
||||
class TestScanScope(ScanScopeTestCase):
|
||||
|
||||
async def test_new_scope_skips_known_resolved_files_without_reading_them(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
known = self.output_dir / "known.png"
|
||||
_write_png(known, prompt_text="a red square")
|
||||
_write_png(self.output_dir / "new.png", prompt_text="a blue circle")
|
||||
self._link_resolved(known, "a red square")
|
||||
reads = self._spy_reads()
|
||||
|
||||
job = await self._scan()
|
||||
|
||||
self.assertEqual(reads, ["new.png"])
|
||||
self.assertEqual(job["scope"]["files"], "new")
|
||||
self.assertEqual(job["counters"]["known"], 1)
|
||||
self.assertEqual(job["counters"]["added"], 1)
|
||||
self.assertEqual(job["processed"], 2)
|
||||
self.assertEqual(
|
||||
job["outcomes"]["known"],
|
||||
[
|
||||
{
|
||||
"path": "known.png",
|
||||
"root": "output",
|
||||
"root_index": 0,
|
||||
"detail": SKIPPED_DETAIL,
|
||||
}
|
||||
],
|
||||
)
|
||||
self.assertIn(f"= output/known.png · {SKIPPED_DETAIL}", "\n".join(job["log"]))
|
||||
self.assertTrue(
|
||||
job["log"][0].endswith(
|
||||
"Scan started · 1 output folder · new and unresolved files"
|
||||
)
|
||||
)
|
||||
legacy = self.api._scan_job.last_event
|
||||
self.assertEqual((legacy["processed"], legacy["unchanged"]), (2, 1))
|
||||
self.assertTrue(job["status_line"].startswith("Processing file 2 of 2"))
|
||||
|
||||
async def test_whole_known_batch_skips_the_executor_reads(self):
|
||||
from py.api import admin
|
||||
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
for name in ("a.png", "b.png"):
|
||||
_write_png(self.output_dir / name, prompt_text=f"prompt {name}")
|
||||
self._link_resolved(self.output_dir / name, f"prompt {name}")
|
||||
_write_png(self.output_dir / "c.png", prompt_text="prompt c")
|
||||
reads = self._spy_reads()
|
||||
|
||||
with patch.object(admin, "SCAN_BATCH_SIZE", 1):
|
||||
job = await self._scan({"scope": "new"})
|
||||
|
||||
self.assertEqual(reads, ["c.png"])
|
||||
self.assertEqual(job["counters"]["known"], 2)
|
||||
self.assertEqual(job["counters"]["added"], 1)
|
||||
|
||||
async def test_unresolved_files_are_rescanned_in_new_scope(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
names = ("fallback.png", "conflict.png", "unlinked.png")
|
||||
for name in names:
|
||||
_write_png(self.output_dir / name, prompt_text=f"text of {name}")
|
||||
# Linked without a resolved pair link (the monitor's fallback / FIFO link).
|
||||
self._link_without_pair(self.output_dir / "fallback.png", "text of fallback")
|
||||
# Resolved once, then also linked to a second prompt: ambiguous.
|
||||
conflict = self.output_dir / "conflict.png"
|
||||
self._link_resolved(conflict, "text of conflict.png")
|
||||
self._link_without_pair(conflict, "another prompt")
|
||||
reads = self._spy_reads()
|
||||
|
||||
job = await self._scan()
|
||||
|
||||
self.assertEqual(sorted(reads), sorted(names))
|
||||
self.assertNotIn(SKIPPED_DETAIL, json.dumps(job["outcomes"]))
|
||||
|
||||
async def test_all_scope_reads_everything(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
known = self.output_dir / "known.png"
|
||||
_write_png(known, prompt_text="a red square")
|
||||
self._link_resolved(known, "a red square")
|
||||
reads = self._spy_reads()
|
||||
|
||||
job = await self._scan({"scope": "all"})
|
||||
|
||||
self.assertEqual(reads, ["known.png"])
|
||||
self.assertEqual(job["scope"]["files"], "all")
|
||||
self.assertEqual(job["counters"]["known"], 1)
|
||||
self.assertNotEqual(job["outcomes"]["known"][0]["detail"], SKIPPED_DETAIL)
|
||||
self.assertTrue(
|
||||
job["log"][0].endswith("Scan started · 1 output folder · all files")
|
||||
)
|
||||
|
||||
async def test_legacy_scan_route_reads_everything(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
known = self.output_dir / "known.png"
|
||||
_write_png(known, prompt_text="a red square")
|
||||
self._link_resolved(known, "a red square")
|
||||
reads = self._spy_reads()
|
||||
|
||||
resp = await self.client.request("POST", "/prompt_manager/scan", data="{}")
|
||||
payloads = _sse_payloads(await resp.text())
|
||||
|
||||
self.assertEqual(reads, ["known.png"])
|
||||
self.assertEqual(self.api._scan_job.scope["files"], "all")
|
||||
self.assertEqual(payloads[-1]["type"], "complete")
|
||||
self.assertEqual(payloads[-1]["unchanged"], 1)
|
||||
|
||||
async def test_file_replaced_at_a_known_path_is_reread_and_needs_review(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
path = self.output_dir / "out_00012.png"
|
||||
_write_png(path, prompt_text="the first prompt")
|
||||
self._link_resolved(path, "the first prompt")
|
||||
before = os.path.getsize(path)
|
||||
# ComfyUI reused the name for a new image with another prompt.
|
||||
_write_png(path, prompt_text="a much longer second prompt for the same name")
|
||||
self.assertNotEqual(os.path.getsize(path), before)
|
||||
reads = self._spy_reads()
|
||||
|
||||
job = await self._scan()
|
||||
|
||||
self.assertEqual(reads, ["out_00012.png"])
|
||||
self.assertEqual(job["counters"]["review"], 1)
|
||||
self.assertEqual(job["counters"]["known"], 0)
|
||||
self.assertIn("different prompt", job["outcomes"]["review"][0]["detail"])
|
||||
|
||||
async def test_scanned_files_store_their_size_so_the_next_new_scan_skips(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
_write_png(self.output_dir / "fresh.png", prompt_text="a fresh prompt")
|
||||
first = await self._scan()
|
||||
self.assertEqual(first["counters"]["added"], 1)
|
||||
reads = self._spy_reads()
|
||||
|
||||
second = await self._scan()
|
||||
|
||||
self.assertEqual(reads, [])
|
||||
self.assertEqual(second["outcomes"]["known"][0]["detail"], SKIPPED_DETAIL)
|
||||
|
||||
async def test_stored_null_size_is_reread(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
path = self.output_dir / "old.png"
|
||||
_write_png(path, prompt_text="an old prompt")
|
||||
self._link_resolved(path, "an old prompt", size=None)
|
||||
reads = self._spy_reads()
|
||||
|
||||
job = await self._scan()
|
||||
|
||||
self.assertEqual(reads, ["old.png"])
|
||||
# The read stores the size it was missing, so the file is known next time.
|
||||
self.assertEqual(job["counters"]["added"], 1)
|
||||
self.assertEqual(
|
||||
self.api.db.get_known_resolved_paths({str(path): os.path.getsize(path)}),
|
||||
{str(path)},
|
||||
)
|
||||
|
||||
async def test_known_set_failure_falls_back_to_reading_every_file(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
|
||||
known = self.output_dir / "known.png"
|
||||
_write_png(known, prompt_text="a red square")
|
||||
self._link_resolved(known, "a red square")
|
||||
reads = self._spy_reads()
|
||||
|
||||
def broken(files):
|
||||
raise sqlite3.OperationalError("database is locked")
|
||||
|
||||
self.api.db.get_known_resolved_paths = broken
|
||||
with self.assertLogs("prompt_manager.api", level="WARNING"):
|
||||
job = await self._scan()
|
||||
|
||||
self.assertEqual(job["state"], "finished")
|
||||
self.assertEqual(reads, ["known.png"])
|
||||
self.assertIn(
|
||||
"Known files could not be read · reading every file", "\n".join(job["log"])
|
||||
)
|
||||
|
||||
async def test_empty_read_body_asks_for_the_default_scope(self):
|
||||
"""A chunked POST with no bytes can read a body: still the default."""
|
||||
from py.api.job_routes import _scan_scope
|
||||
|
||||
class Request:
|
||||
can_read_body = True
|
||||
|
||||
def __init__(self, text):
|
||||
self._text = text
|
||||
|
||||
async def text(self):
|
||||
return self._text
|
||||
|
||||
self.assertEqual(await _scan_scope(Request("")), "new")
|
||||
self.assertEqual(await _scan_scope(Request(" \n")), "new")
|
||||
self.assertEqual(await _scan_scope(Request('{"scope": "all"}')), "all")
|
||||
self.assertIsNone(await _scan_scope(Request("{")))
|
||||
|
||||
async def test_invalid_scope_is_refused(self):
|
||||
for body in ({"scope": "everything"}, {"scope": 1}, ["new"]):
|
||||
with self.subTest(body=body):
|
||||
resp = await self.client.request(
|
||||
"POST", "/prompt_manager/jobs/scan", json=body
|
||||
)
|
||||
self.assertEqual(resp.status, 400)
|
||||
data = await resp.json()
|
||||
self.assertFalse(data["success"])
|
||||
resp = await self.client.request(
|
||||
"POST", "/prompt_manager/jobs/scan", data="not json"
|
||||
)
|
||||
self.assertEqual(resp.status, 400)
|
||||
self.assertIsNone(self.api._scan_job)
|
||||
|
||||
|
||||
class TestKnownResolvedPaths(ScanScopeTestCase):
|
||||
"""``PromptDatabase.get_known_resolved_paths``: one streamed query."""
|
||||
|
||||
async def test_returns_the_given_paths_that_are_known_and_resolved(self):
|
||||
known, fallback = self.output_dir / "known.png", self.output_dir / "fb.png"
|
||||
self._link_resolved(known, "a red square", size=10)
|
||||
self._link_without_pair(fallback, "a fallback")
|
||||
asked = {str(known): 10, str(fallback): 10, str(self.output_dir / "n.png"): 10}
|
||||
|
||||
self.assertEqual(self.api.db.get_known_resolved_paths(asked), {str(known)})
|
||||
self.assertEqual(self.api.db.get_known_resolved_paths({}), set())
|
||||
|
||||
async def test_size_must_match_and_a_stored_null_size_is_not_known(self):
|
||||
same, other = self.output_dir / "same.png", self.output_dir / "other.png"
|
||||
unknown = self.output_dir / "unknown.png"
|
||||
self._link_resolved(same, "same size", size=10)
|
||||
self._link_resolved(other, "other size", size=10)
|
||||
self._link_resolved(unknown, "no stored size", size=None)
|
||||
asked = {str(same): 10, str(other): 11, str(unknown): 10}
|
||||
|
||||
self.assertEqual(self.api.db.get_known_resolved_paths(asked), {str(same)})
|
||||
# A file the walk could not stat is not known either.
|
||||
self.assertEqual(self.api.db.get_known_resolved_paths({str(same): None}), set())
|
||||
|
||||
async def test_row_without_file_path_matches_through_image_path(self):
|
||||
known = self.output_dir / "legacy.png"
|
||||
self._link_resolved(known, "a legacy row", size=10)
|
||||
with sqlite3.connect(self.api.db.model.db_path) as conn:
|
||||
conn.execute("UPDATE generated_images SET file_path=NULL")
|
||||
conn.close()
|
||||
|
||||
found = self.api.db.get_known_resolved_paths({str(known): 10})
|
||||
self.assertEqual(found, {str(known)})
|
||||
|
||||
async def test_stored_spelling_is_normalized_like_the_scanned_path(self):
|
||||
known = self.output_dir / "sub" / "known.png"
|
||||
self._link_resolved(known, "a red square", size=10)
|
||||
spelled = os.path.join(str(self.output_dir), "sub", "..", "sub", "known.png")
|
||||
with sqlite3.connect(self.api.db.model.db_path) as conn:
|
||||
conn.execute("UPDATE generated_images SET file_path=?", (spelled,))
|
||||
conn.close()
|
||||
|
||||
found = self.api.db.get_known_resolved_paths({str(known): 10})
|
||||
self.assertEqual(found, {str(known)})
|
||||
|
||||
async def test_one_query_streamed_in_small_fetches(self):
|
||||
paths = [self.output_dir / f"{index}.png" for index in range(5)]
|
||||
for path in paths:
|
||||
self._link_resolved(path, f"prompt {path.name}", size=10)
|
||||
original = self.api.db.model.get_connection
|
||||
opened = []
|
||||
|
||||
def counting():
|
||||
opened.append(1)
|
||||
return original()
|
||||
|
||||
self.api.db.model.get_connection = counting
|
||||
with patch.object(operations, "KNOWN_PATHS_FETCH_SIZE", 2):
|
||||
found = self.api.db.get_known_resolved_paths({str(p): 10 for p in paths})
|
||||
self.assertEqual(found, {str(p) for p in paths})
|
||||
self.assertEqual(len(opened), 1)
|
||||
|
||||
async def test_windows_path_case_matches_known_set(self):
|
||||
stored = self.output_dir / "Sub" / "Known.PNG"
|
||||
self._link_resolved(stored, "a red square", size=10)
|
||||
# A row written before paths were case-folded keeps its own spelling.
|
||||
with sqlite3.connect(self.api.db.model.db_path) as conn:
|
||||
conn.execute(
|
||||
"UPDATE generated_images SET file_path=?", (str(stored).upper(),)
|
||||
)
|
||||
conn.close()
|
||||
asked = str(self.output_dir / "sub" / "known.png")
|
||||
|
||||
with patch("os.path.normcase", lambda path: path.lower()):
|
||||
self.assertEqual(self.api.db.get_known_resolved_paths({asked: 10}), {asked})
|
||||
if os.path.normcase("A") == "A": # a case-sensitive platform
|
||||
self.assertEqual(self.api.db.get_known_resolved_paths({asked: 10}), set())
|
||||
|
||||
|
||||
class TestSkippedOutcome(unittest.TestCase):
|
||||
|
||||
def test_skipped_file_is_already_known_with_its_own_detail(self):
|
||||
from py.api.admin import _scan_outcome
|
||||
|
||||
root = os.path.abspath("out")
|
||||
group, row, line = _scan_outcome(
|
||||
os.path.join(root, "a", "b.png"), [(root, "output", 0)], "skipped"
|
||||
)
|
||||
self.assertEqual(group, "known")
|
||||
self.assertEqual(row["detail"], SKIPPED_DETAIL)
|
||||
self.assertEqual(row["path"], "a/b.png")
|
||||
self.assertEqual(line, f"= output/a/b.png · {SKIPPED_DETAIL}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createServer } from 'node:net';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { startBrowserServer, createServerSupervisor } from '../../tools/vendor/start-browser-server.mjs';
|
||||
import { workflowWorkers } from '../../tools/vendor/test-workers.mjs';
|
||||
|
||||
const fakeServer = fileURLToPath(new URL('./fake-browser-server.mjs', import.meta.url));
|
||||
const fake = (...extra) => ({ command: process.execPath, args: port => [fakeServer, '--port', String(port), ...extra] });
|
||||
const REAL_SERVER_TIMEOUT_MS = 90_000;
|
||||
|
||||
function listen(port = 0) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const server = createServer();
|
||||
server.once('error', reject);
|
||||
server.listen(port, '127.0.0.1', () => resolve(server));
|
||||
});
|
||||
}
|
||||
|
||||
async function releasedPort() {
|
||||
const probe = await listen();
|
||||
const { port } = probe.address();
|
||||
await new Promise(resolve => probe.close(resolve));
|
||||
return port;
|
||||
}
|
||||
|
||||
test('free port allocation never reuses a bound port', async t => {
|
||||
const held = await listen();
|
||||
t.after(() => held.close());
|
||||
const heldPort = held.address().port;
|
||||
const servers = await Promise.all(Array.from({ length: 6 }, () => startBrowserServer(fake())));
|
||||
t.after(() => Promise.all(servers.map(server => server.stop())));
|
||||
const ports = servers.map(server => server.port);
|
||||
for (const server of servers) {
|
||||
assert.ok(Number.isInteger(server.port) && server.port > 0 && server.port < 65536, `port ${server.port} is a TCP port`);
|
||||
assert.equal(server.url, `http://127.0.0.1:${server.port}`);
|
||||
assert.equal(await (await fetch(`${server.url}/prompt_manager/test`)).text(), 'ready');
|
||||
}
|
||||
assert.ok(!ports.includes(heldPort), 'a port another process holds is never handed out');
|
||||
assert.equal(new Set(ports).size, ports.length, 'concurrent servers never share a port');
|
||||
});
|
||||
|
||||
test('startBrowserServer asks the server to bind port 0 and reads the port it announces', async () => {
|
||||
const asked = [];
|
||||
const server = await startBrowserServer({ command: process.execPath, args: port => { asked.push(port); return fake().args(port); } });
|
||||
try {
|
||||
assert.deepEqual(asked, [0], 'no port is reserved and released before the server binds');
|
||||
assert.ok(server.port > 0);
|
||||
} finally {
|
||||
await server.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test('startBrowserServer serves the readiness page on a requested port and stops cleanly', async () => {
|
||||
const port = await releasedPort();
|
||||
const server = await startBrowserServer({ port, ...fake() });
|
||||
assert.equal(server.port, port);
|
||||
assert.equal(server.url, `http://127.0.0.1:${port}`);
|
||||
assert.equal(await (await fetch(`${server.url}/prompt_manager/test`)).text(), 'ready');
|
||||
await server.stop();
|
||||
await assert.rejects(fetch(`${server.url}/prompt_manager/test`));
|
||||
});
|
||||
|
||||
test('startBrowserServer reports a startup failure with the server\'s stderr tail', async () => {
|
||||
await assert.rejects(startBrowserServer(fake('--fail')), error => {
|
||||
assert.match(error.message, /exited with code 4 before it was ready/);
|
||||
assert.match(error.message, /last fake stderr line/);
|
||||
return true;
|
||||
});
|
||||
});
|
||||
|
||||
test('startBrowserServer gives up on a server that never announces a port or becomes ready', async () => {
|
||||
const silent = { command: process.execPath, args: () => ['-e', 'setInterval(() => {}, 1000)'] };
|
||||
await assert.rejects(startBrowserServer({ ...silent, timeoutMs: 500 }), /not ready .* within 500 ms/);
|
||||
});
|
||||
|
||||
test('worker server restarts after an unexpected exit', async t => {
|
||||
const supervisor = createServerSupervisor({ start: () => startBrowserServer(fake()) });
|
||||
t.after(() => supervisor.stop());
|
||||
const first = await supervisor.ensure();
|
||||
assert.equal(supervisor.crash(first), null, 'a running server reports no crash');
|
||||
await fetch(`${first.url}/__exit`);
|
||||
await first.exited;
|
||||
const crash = supervisor.crash(first);
|
||||
assert.match(crash.message, /test server on port \d+ exited unexpectedly \(code 3\)/);
|
||||
assert.match(crash.message, /crashed on purpose/);
|
||||
const second = await supervisor.ensure();
|
||||
assert.notEqual(second, first, 'the next test gets a fresh server');
|
||||
assert.equal(await (await fetch(`${second.url}/prompt_manager/test`)).text(), 'ready');
|
||||
assert.equal(await supervisor.ensure(), second, 'a healthy server is reused');
|
||||
});
|
||||
|
||||
test('supervisor reports a startup failure once and tries again on the next ensure', async t => {
|
||||
let attempts = 0;
|
||||
const supervisor = createServerSupervisor({
|
||||
start: async () => {
|
||||
attempts += 1;
|
||||
if (attempts === 1) throw new Error('ImportError: no module');
|
||||
return startBrowserServer(fake());
|
||||
},
|
||||
});
|
||||
t.after(() => supervisor.stop());
|
||||
await assert.rejects(supervisor.ensure(), /ImportError/);
|
||||
assert.equal(attempts, 1, 'a failed startup is not retried behind the caller\'s back');
|
||||
const server = await supervisor.ensure();
|
||||
assert.equal(attempts, 2);
|
||||
assert.equal(await (await fetch(`${server.url}/prompt_manager/test`)).text(), 'ready');
|
||||
});
|
||||
|
||||
test('tests/browser/server.py binds an OS-chosen port for --port 0 and still honours --port N', { timeout: 2 * REAL_SERVER_TIMEOUT_MS }, async t => {
|
||||
const held = await listen();
|
||||
t.after(() => held.close());
|
||||
const chosen = await startBrowserServer({ timeoutMs: REAL_SERVER_TIMEOUT_MS });
|
||||
try {
|
||||
assert.notEqual(chosen.port, held.address().port);
|
||||
assert.equal((await fetch(`${chosen.url}/prompt_manager/test`)).status, 200);
|
||||
} finally {
|
||||
await chosen.stop();
|
||||
}
|
||||
const port = await releasedPort();
|
||||
const fixed = await startBrowserServer({ port, timeoutMs: REAL_SERVER_TIMEOUT_MS });
|
||||
try {
|
||||
assert.equal(fixed.port, port);
|
||||
assert.equal((await fetch(`http://127.0.0.1:${port}/prompt_manager/test`)).status, 200);
|
||||
} finally {
|
||||
await fixed.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test('worker count follows PM_TEST_WORKERS, then one in CI, then a quarter of the CPUs up to eight', () => {
|
||||
assert.equal(workflowWorkers({ env: { PM_TEST_WORKERS: '3', CI: '1' }, cpus: 32 }), 3);
|
||||
assert.equal(workflowWorkers({ env: { CI: '1' }, cpus: 32 }), 1);
|
||||
assert.equal(workflowWorkers({ env: {}, cpus: 32 }), 8);
|
||||
assert.equal(workflowWorkers({ env: {}, cpus: 12 }), 3);
|
||||
assert.equal(workflowWorkers({ env: {}, cpus: 2 }), 1);
|
||||
assert.throws(() => workflowWorkers({ env: { PM_TEST_WORKERS: 'many' }, cpus: 8 }), /PM_TEST_WORKERS must be a positive integer/);
|
||||
assert.throws(() => workflowWorkers({ env: { PM_TEST_WORKERS: '0' }, cpus: 8 }), /PM_TEST_WORKERS must be a positive integer/);
|
||||
});
|
||||
Vendored
+197
@@ -0,0 +1,197 @@
|
||||
// Structural checks on .github/workflows/viewerjs-check.yml. The tooling has no YAML parser, so these
|
||||
// read the workflow's fixed two-space layout: jobs at two spaces, their keys at four, steps at six.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { mkdtemp, mkdir, readdir, rm, writeFile } from 'node:fs/promises';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
const workflow = readFileSync(new URL('../../.github/workflows/viewerjs-check.yml', import.meta.url), 'utf8');
|
||||
const PINNED_IMAGE = /^\s+image: mcr\.microsoft\.com\/playwright:v\d+\.\d+\.\d+-noble$/m;
|
||||
|
||||
function jobs(text) {
|
||||
const lines = text.split('\n');
|
||||
const start = lines.indexOf('jobs:');
|
||||
assert.notEqual(start, -1, 'workflow has a jobs section');
|
||||
const found = new Map();
|
||||
let current = null;
|
||||
for (const line of lines.slice(start + 1)) {
|
||||
const header = /^ {2}([\w-]+):\s*$/.exec(line);
|
||||
if (header) {
|
||||
current = header[1];
|
||||
found.set(current, []);
|
||||
} else if (current && (line.startsWith(' ') || line.trim() === '' || /^ {2}#/.test(line))) {
|
||||
found.get(current).push(line);
|
||||
} else if (current && /^\S/.test(line)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return new Map([...found].map(([id, body]) => [id, body.join('\n')]));
|
||||
}
|
||||
|
||||
function steps(jobText) {
|
||||
const parts = jobText.split(/^(?= {6}- )/m);
|
||||
return parts.slice(1);
|
||||
}
|
||||
|
||||
function runScript(stepText) {
|
||||
const lines = stepText.split('\n');
|
||||
const index = lines.findIndex(line => /^ {8}run: \|\s*$/.test(line));
|
||||
assert.notEqual(index, -1, 'step has a block run script');
|
||||
const body = [];
|
||||
for (const line of lines.slice(index + 1)) {
|
||||
if (line.trim() !== '' && !line.startsWith(' ')) break;
|
||||
body.push(line.slice(10));
|
||||
}
|
||||
return body.join('\n');
|
||||
}
|
||||
|
||||
function needsOf(jobText) {
|
||||
const match = /^ {4}needs: \[([^\]]*)\]\s*$/m.exec(jobText) ?? /^ {4}needs: ([\w-]+)\s*$/m.exec(jobText);
|
||||
assert.ok(match, 'job declares needs');
|
||||
return match[1].split(',').map(value => value.trim()).filter(Boolean);
|
||||
}
|
||||
|
||||
function matrixList(jobText, key) {
|
||||
const match = new RegExp(`^ {8}${key}: \\[([^\\]]*)\\]\\s*$`, 'm').exec(jobText);
|
||||
assert.ok(match, `job has a ${key} matrix`);
|
||||
return match[1].split(',').map(value => value.trim());
|
||||
}
|
||||
|
||||
const allJobs = jobs(workflow);
|
||||
const browserJobIds = [...allJobs].filter(([, body]) => /npm run --prefix tools\/vendor test:(workflows|coverage)/.test(body))
|
||||
.map(([id]) => id);
|
||||
const mergeJob = () => allJobs.get('frontend-coverage-merge');
|
||||
const verifyStep = () => steps(mergeJob()).find(step => /name: Verify every shard's raw coverage arrived/.test(step));
|
||||
|
||||
test('viewerjs keeps its name and needs every frontend job', () => {
|
||||
const viewerjs = allJobs.get('viewerjs');
|
||||
assert.ok(viewerjs, 'the required context is the job id viewerjs');
|
||||
assert.doesNotMatch(viewerjs, /^ {4}name:/m, 'no display name may replace the context "viewerjs"');
|
||||
const needs = needsOf(viewerjs);
|
||||
const others = [...allJobs.keys()].filter(id => id !== 'viewerjs');
|
||||
assert.deepEqual([...needs].sort(), [...others].sort(), 'viewerjs needs every other job in the workflow');
|
||||
assert.deepEqual([...needs].sort(), ['browser-workflows', 'frontend-coverage', 'frontend-coverage-merge']);
|
||||
assert.match(viewerjs, /^ {4}if: always\(\)$/m, 'viewerjs runs (and fails) when a needed job failed or was skipped');
|
||||
const script = runScript(steps(viewerjs).find(step => /run: \|/.test(step)));
|
||||
for (const id of needs) {
|
||||
const variable = new RegExp(`^ {10}([A-Z_]+): \\$\\{\\{ needs\\.${id}\\.result \\}\\}$`, 'm').exec(viewerjs);
|
||||
assert.ok(variable, `${id}'s result reaches the check`);
|
||||
assert.match(script, new RegExp(`test "\\$${variable[1]}" = success`), `${id} must be exactly success`);
|
||||
}
|
||||
});
|
||||
|
||||
test('viewerjs fails unless every needed job succeeded', () => {
|
||||
const viewerjs = allJobs.get('viewerjs');
|
||||
const script = runScript(steps(viewerjs).find(step => /run: \|/.test(step)));
|
||||
const variables = [...viewerjs.matchAll(/^ {10}([A-Z_]+): \$\{\{ needs\.[\w-]+\.result \}\}$/gm)].map(match => match[1]);
|
||||
const run = env => {
|
||||
try {
|
||||
execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', script], { env: { PATH: process.env.PATH, ...env }, stdio: 'pipe' });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
const allSuccess = Object.fromEntries(variables.map(name => [name, 'success']));
|
||||
assert.equal(run(allSuccess), true);
|
||||
for (const name of variables) {
|
||||
for (const result of ['failure', 'cancelled', 'skipped']) {
|
||||
assert.equal(run({ ...allSuccess, [name]: result }), false, `${name}=${result} must fail viewerjs`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('coverage runs as three shards that each upload their raw coverage', () => {
|
||||
const coverage = allJobs.get('frontend-coverage');
|
||||
assert.deepEqual(matrixList(coverage, 'shard'), ['1', '2', '3']);
|
||||
assert.match(coverage, /^ {4}name: frontend coverage \(shard \$\{\{ matrix\.shard \}\}\/3\)$/m);
|
||||
assert.match(coverage, /npm run --prefix tools\/vendor test:coverage:shard -- \$\{\{ matrix\.shard \}\}\/3/);
|
||||
const upload = steps(coverage).find(step => /name: coverage-raw-\$\{\{ matrix\.shard \}\}/.test(step));
|
||||
assert.ok(upload, 'each shard uploads coverage-raw-N');
|
||||
assert.match(upload, /^ {10}path: coverage\/js\/raw\/?$/m);
|
||||
assert.match(upload, /^ {10}retention-days: 3$/m);
|
||||
assert.match(upload, /^ {10}if-no-files-found: error$/m);
|
||||
});
|
||||
|
||||
test('merge job downloads every shard and runs the merge-only report and check', () => {
|
||||
const merge = mergeJob();
|
||||
assert.ok(merge, 'frontend-coverage-merge exists');
|
||||
assert.deepEqual(needsOf(merge), ['frontend-coverage']);
|
||||
assert.match(merge, /^ {4}name: frontend coverage \(merge\)$/m);
|
||||
const download = steps(merge).find(step => /uses: actions\/download-artifact@/.test(step));
|
||||
assert.ok(download, 'merge downloads the shard artifacts');
|
||||
assert.match(download, /^ {10}pattern: coverage-raw-\*$/m);
|
||||
assert.doesNotMatch(download, /merge-multiple: true/, 'each artifact keeps its own directory so the check can count them');
|
||||
assert.ok(verifyStep(), 'merge has an explicit artifact count check');
|
||||
assert.match(verifyStep(), new RegExp(`^ {10}EXPECTED_SHARDS: ${matrixList(allJobs.get('frontend-coverage'), 'shard').length}$`, 'm'),
|
||||
'the expected count matches the shard matrix');
|
||||
const order = steps(merge).map(step => (/name: (.*)/.exec(step) ?? [, step.trim().split('\n')[0]])[1]);
|
||||
const verifyAt = order.findIndex(name => /Verify every shard/.test(name));
|
||||
const mergeAt = steps(merge).findIndex(step => /npm run --prefix tools\/vendor coverage:merge/.test(step));
|
||||
assert.ok(verifyAt >= 0 && mergeAt > verifyAt, 'the count check runs before coverage:merge');
|
||||
const report = steps(merge).find(step => /name: frontend-test-reports$/m.test(step));
|
||||
assert.ok(report, 'merge uploads the HTML report as frontend-test-reports');
|
||||
assert.match(report, /!coverage\/js\/raw\/\*\*/);
|
||||
});
|
||||
|
||||
async function stagedShards(root, shards) {
|
||||
const staged = join(root, 'staged');
|
||||
for (const [shard, files] of shards) {
|
||||
const directory = join(staged, `coverage-raw-${shard}`);
|
||||
await mkdir(directory, { recursive: true });
|
||||
for (const file of files) await writeFile(join(directory, file), '{"result":[]}');
|
||||
}
|
||||
await mkdir(staged, { recursive: true });
|
||||
return staged;
|
||||
}
|
||||
|
||||
async function verify(t, shards) {
|
||||
const root = await mkdtemp(join(tmpdir(), 'pm-ci-merge-'));
|
||||
t.after(() => rm(root, { recursive: true, force: true }));
|
||||
const staged = await stagedShards(root, shards);
|
||||
try {
|
||||
execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', runScript(verifyStep())], {
|
||||
cwd: root, stdio: 'pipe',
|
||||
env: { PATH: process.env.PATH, STAGED_RAW: staged, EXPECTED_SHARDS: '3' },
|
||||
});
|
||||
} catch (error) {
|
||||
return { ok: false, output: `${error.stdout}${error.stderr}` };
|
||||
}
|
||||
return { ok: true, raw: (await readdir(join(root, 'coverage/js/raw'))).sort() };
|
||||
}
|
||||
|
||||
test('merge refuses when a shard\'s raw coverage is missing', async t => {
|
||||
const missing = await verify(t, [[1, ['a.json']], [3, ['c.json']]]);
|
||||
assert.equal(missing.ok, false);
|
||||
assert.match(missing.output, /2 of 3/);
|
||||
const empty = await verify(t, [[1, ['a.json']], [2, []], [3, ['c.json']]]);
|
||||
assert.equal(empty.ok, false, 'an artifact without raw files counts as missing');
|
||||
const collision = await verify(t, [[1, ['a.json']], [2, ['a.json']], [3, ['c.json']]]);
|
||||
assert.equal(collision.ok, false, 'two shards writing the same file name must not silently drop one');
|
||||
const complete = await verify(t, [[1, ['a.json', 'node.json']], [2, ['b.json']], [3, ['c.json']]]);
|
||||
assert.deepEqual(complete, { ok: true, raw: ['a.json', 'b.json', 'c.json', 'node.json'] });
|
||||
});
|
||||
|
||||
test('every browser job sets PM_TEST_WORKERS and runs in the pinned image', () => {
|
||||
assert.deepEqual([...browserJobIds].sort(), ['browser-workflows', 'frontend-coverage']);
|
||||
for (const id of browserJobIds) {
|
||||
const body = allJobs.get(id);
|
||||
assert.match(body, /^ {6}PM_TEST_WORKERS: 3$/m, `${id} sets PM_TEST_WORKERS for the whole job`);
|
||||
assert.match(body, PINNED_IMAGE, `${id} keeps the pinned Playwright image`);
|
||||
assert.match(body, /PIP_SHA256: [0-9a-f]{64}/, `${id} keeps the pinned pip bootstrap`);
|
||||
assert.match(body, /^ {4}timeout-minutes: \d+$/m);
|
||||
}
|
||||
assert.match(mergeJob(), PINNED_IMAGE, 'the merge sees the same workspace paths as the shards');
|
||||
});
|
||||
|
||||
test('page workflows run each browser in two shards', () => {
|
||||
const workflows = allJobs.get('browser-workflows');
|
||||
assert.deepEqual(matrixList(workflows, 'browser'), ['firefox', 'webkit']);
|
||||
assert.deepEqual(matrixList(workflows, 'shard'), ['1', '2']);
|
||||
assert.match(workflows, /test:workflows -- --project \$\{\{ matrix\.browser \}\} --shard \$\{\{ matrix\.shard \}\}\/2/);
|
||||
assert.match(workflows, /^ {6}fail-fast: false$/m);
|
||||
assert.match(workflows, /name: browser-traces-\$\{\{ matrix\.browser \}\}-shard-\$\{\{ matrix\.shard \}\}/);
|
||||
});
|
||||
Vendored
+25
@@ -0,0 +1,25 @@
|
||||
// A stand-in for tests/browser/server.py: `--port N` (0 = an OS-chosen port)
|
||||
// serves the readiness page and announces the bound port on stdout as
|
||||
// `PM_TEST_SERVER_PORT=<n>`, `--fail` writes to stderr and exits before
|
||||
// listening, and GET /__exit makes the process die unexpectedly after writing a
|
||||
// last stderr line.
|
||||
import { createServer } from 'node:http';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const port = Number(args[args.indexOf('--port') + 1]);
|
||||
if (args.includes('--fail')) {
|
||||
process.stderr.write('fake server: refusing to start\nlast fake stderr line\n');
|
||||
process.exit(4);
|
||||
}
|
||||
const server = createServer((request, response) => {
|
||||
if (request.url === '/__exit') {
|
||||
process.stderr.write('fake server: crashed on purpose\n');
|
||||
response.end('bye', () => process.exit(3));
|
||||
return;
|
||||
}
|
||||
response.end(request.url === '/prompt_manager/test' ? 'ready' : `pid ${process.pid}`);
|
||||
});
|
||||
server.listen(port, '127.0.0.1', () => {
|
||||
process.stdout.write(`fake server: listening\nPM_TEST_SERVER_PORT=${server.address().port}\n`);
|
||||
});
|
||||
for (const signal of ['SIGTERM', 'SIGINT']) process.once(signal, () => server.close(() => process.exit(0)));
|
||||
+93
@@ -0,0 +1,93 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { existsSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { plan, runCoverage } from '../../tools/vendor/frontend-coverage.mjs';
|
||||
|
||||
const steps = argv => plan(argv).map(({ step, ...options }) => (Object.keys(options).length ? { step, ...options } : step));
|
||||
|
||||
function tempPaths(t) {
|
||||
const root = mkdtempSync(join(tmpdir(), 'pm-coverage-cli-'));
|
||||
t.after(() => rmSync(root, { recursive: true, force: true }));
|
||||
const reportRoot = join(root, 'coverage/js');
|
||||
return { root, paths: { repositoryRoot: root, tools: join(root, 'tools'), reportRoot, raw: join(reportRoot, 'raw'), sourceRoot: join(root, 'web/js') } };
|
||||
}
|
||||
|
||||
function recorder() {
|
||||
const calls = [];
|
||||
const run = (args, env = {}) => { calls.push({ args, env }); };
|
||||
return { calls, run };
|
||||
}
|
||||
|
||||
test('no arguments plans the unchanged full run', () => {
|
||||
assert.deepEqual(steps([]), [
|
||||
'clean', 'node-units', 'viewer', { step: 'workflows', shard: null }, 'report', { step: 'check', updateBaseline: false },
|
||||
]);
|
||||
assert.deepEqual(steps(['--update-baseline']), [
|
||||
'clean', 'node-units', 'viewer', { step: 'workflows', shard: null }, 'report', { step: 'check', updateBaseline: true },
|
||||
]);
|
||||
});
|
||||
|
||||
test('--shard 2/3 plans only workflows with --shard 2/3 and no report', () => {
|
||||
assert.deepEqual(steps(['--shard', '2/3']), ['prepare', { step: 'workflows', shard: '2/3' }]);
|
||||
assert.deepEqual(steps(['--shard=3/3']), ['prepare', { step: 'workflows', shard: '3/3' }]);
|
||||
});
|
||||
|
||||
test('--shard 1/N also runs the node units and the viewer suite', () => {
|
||||
assert.deepEqual(steps(['--shard', '1/3']), ['prepare', 'node-units', 'viewer', { step: 'workflows', shard: '1/3' }]);
|
||||
assert.deepEqual(steps(['--shard', '1/1']), ['prepare', 'node-units', 'viewer', { step: 'workflows', shard: '1/1' }]);
|
||||
});
|
||||
|
||||
test('--merge-only plans the report and the check over existing raw coverage', () => {
|
||||
assert.deepEqual(steps(['--merge-only']), ['require-raw', 'report', { step: 'check', updateBaseline: false }]);
|
||||
assert.deepEqual(steps(['--merge-only', '--update-baseline']), ['require-raw', 'report', { step: 'check', updateBaseline: true }]);
|
||||
});
|
||||
|
||||
test('invalid shard and mode combinations are refused', () => {
|
||||
for (const argv of [['--shard'], ['--shard', '0/3'], ['--shard', '4/3'], ['--shard', '2'], ['--shard', 'a/b'], ['--shard', '1/0'], ['--shard', '1.5/3']]) {
|
||||
assert.throws(() => plan(argv), /--shard needs I\/N/, argv.join(' '));
|
||||
}
|
||||
assert.throws(() => plan(['--shard', '1/2', '--merge-only']), /either --shard or --merge-only/);
|
||||
assert.throws(() => plan(['--shard', '1/2', '--update-baseline']), /--update-baseline .* --merge-only or a full run/);
|
||||
assert.throws(() => plan(['--shard', '1/2', '--shard', '2/2']), /--shard once/);
|
||||
assert.throws(() => plan(['--report']), /Unknown argument: --report/);
|
||||
});
|
||||
|
||||
test('--merge-only refuses an empty raw dir', async t => {
|
||||
const { paths } = tempPaths(t);
|
||||
const { calls, run } = recorder();
|
||||
await assert.rejects(runCoverage({ argv: ['--merge-only'], run, paths }), /no raw coverage .*coverage\/js\/raw.* run the --shard steps first/i);
|
||||
mkdirSync(paths.raw, { recursive: true });
|
||||
await assert.rejects(runCoverage({ argv: ['--merge-only'], run, paths }), /no raw coverage/i);
|
||||
writeFileSync(join(paths.raw, 'notes.txt'), 'not coverage');
|
||||
await assert.rejects(runCoverage({ argv: ['--merge-only'], run, paths }), /no raw coverage/i);
|
||||
assert.deepEqual(calls, [], 'nothing is reported or checked over partial or empty coverage');
|
||||
});
|
||||
|
||||
test('a shard keeps raw coverage other shards wrote and passes its shard to Playwright', async t => {
|
||||
const { paths } = tempPaths(t);
|
||||
mkdirSync(paths.raw, { recursive: true });
|
||||
const earlier = join(paths.raw, 'browser-earlier-shard.json');
|
||||
writeFileSync(earlier, '{"result":[]}');
|
||||
const { calls, run } = recorder();
|
||||
await runCoverage({ argv: ['--shard', '2/3'], run, paths });
|
||||
assert.ok(existsSync(earlier), 'raw files from another shard survive');
|
||||
assert.equal(calls.length, 1, 'shard 2 runs only the workflows');
|
||||
const [{ args, env }] = calls;
|
||||
assert.deepEqual(args.slice(-6), ['--project', 'chromium', '--project', 'touch', '--shard', '2/3']);
|
||||
assert.equal(env.PM_JS_COVERAGE, paths.raw);
|
||||
});
|
||||
|
||||
test('shard 1 runs the node units into the shared raw dir, then the viewer suite, then its workflows', async t => {
|
||||
const { root, paths } = tempPaths(t);
|
||||
mkdirSync(join(root, 'tests/js'), { recursive: true });
|
||||
writeFileSync(join(root, 'tests/js/a.test.js'), '');
|
||||
const { calls, run } = recorder();
|
||||
await runCoverage({ argv: ['--shard', '1/3'], run, paths });
|
||||
assert.equal(calls.length, 3);
|
||||
assert.deepEqual(calls[0].args, ['--test', 'tests/js/a.test.js']);
|
||||
assert.equal(calls[0].env.NODE_V8_COVERAGE, paths.raw);
|
||||
assert.ok(calls[1].args.includes(join(paths.tools, 'playwright.config.mjs')));
|
||||
assert.deepEqual(calls[2].args.slice(-2), ['--shard', '1/3']);
|
||||
});
|
||||
Vendored
+99
-20
@@ -1,4 +1,4 @@
|
||||
import { readFileSync, readdirSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { existsSync, readFileSync, readdirSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { dirname, join, relative } from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
@@ -7,33 +7,69 @@ import { repositoryRoot } from './vendor-viewerjs.mjs';
|
||||
import { discoverSources, checkCoverage } from '@comfyassets/toolkit/testing/coverage';
|
||||
export { discoverSources, checkCoverage };
|
||||
const metrics = ['lines', 'branches', 'functions', 'statements'];
|
||||
const SHARD = /^([1-9]\d*)\/([1-9]\d*)$/;
|
||||
const USAGE = 'usage: frontend-coverage.mjs [--update-baseline] | --shard I/N | --merge-only [--update-baseline]';
|
||||
|
||||
function run(args, env = {}) {
|
||||
function runNode(args, env = {}) {
|
||||
const result = spawnSync(process.execPath, args, { cwd: repositoryRoot, stdio: 'inherit',
|
||||
env: { ...process.env, ...env } });
|
||||
if (result.status !== 0) throw new Error(`Command failed (${result.status}): ${args.join(' ')}`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const tools = dirname(fileURLToPath(import.meta.url));
|
||||
function defaultPaths() {
|
||||
const reportRoot = join(repositoryRoot, 'coverage/js');
|
||||
const raw = join(reportRoot, 'raw');
|
||||
const sourceRoot = join(repositoryRoot, 'web/js');
|
||||
return { repositoryRoot, tools: dirname(fileURLToPath(import.meta.url)), reportRoot,
|
||||
raw: join(reportRoot, 'raw'), sourceRoot: join(repositoryRoot, 'web/js') };
|
||||
}
|
||||
|
||||
function parseShard(value) {
|
||||
const match = SHARD.exec(value ?? '');
|
||||
if (!match || Number(match[1]) > Number(match[2])) {
|
||||
throw new Error(`--shard needs I/N with 1 <= I <= N, got ${value ?? 'nothing'}\n${USAGE}`);
|
||||
}
|
||||
return { index: Number(match[1]), total: Number(match[2]), value };
|
||||
}
|
||||
|
||||
// Turn the command line into the ordered steps of one coverage run:
|
||||
// - no mode: the full run (clean, node units, viewer suite, chromium+touch
|
||||
// workflows, report, check), exactly as before shards existed;
|
||||
// - --shard I/N: one slice of the workflows into coverage/js/raw, without wiping
|
||||
// it or reporting; shard 1 also runs the node units and the viewer suite;
|
||||
// - --merge-only: report and check over raw coverage the shards already wrote.
|
||||
export function plan(argv) {
|
||||
let shard = null;
|
||||
let mergeOnly = false;
|
||||
let updateBaseline = false;
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === '--update-baseline') updateBaseline = true;
|
||||
else if (arg === '--merge-only') mergeOnly = true;
|
||||
else if (arg === '--shard' || arg.startsWith('--shard=')) {
|
||||
if (shard) throw new Error(`Pass --shard once\n${USAGE}`);
|
||||
shard = parseShard(arg === '--shard' ? argv[(index += 1)] : arg.slice('--shard='.length));
|
||||
} else throw new Error(`Unknown argument: ${arg}\n${USAGE}`);
|
||||
}
|
||||
if (shard && mergeOnly) throw new Error(`Pass either --shard or --merge-only\n${USAGE}`);
|
||||
if (shard && updateBaseline) throw new Error(`--update-baseline needs --merge-only or a full run, not a shard\n${USAGE}`);
|
||||
const check = { step: 'check', updateBaseline };
|
||||
if (mergeOnly) return [{ step: 'require-raw' }, { step: 'report' }, check];
|
||||
if (shard) {
|
||||
const first = shard.index === 1 ? [{ step: 'node-units' }, { step: 'viewer' }] : [];
|
||||
return [{ step: 'prepare' }, ...first, { step: 'workflows', shard: shard.value }];
|
||||
}
|
||||
return [{ step: 'clean' }, { step: 'node-units' }, { step: 'viewer' }, { step: 'workflows', shard: null },
|
||||
{ step: 'report' }, check];
|
||||
}
|
||||
|
||||
function rawCoverageFiles(raw) {
|
||||
return existsSync(raw) ? readdirSync(raw).filter(file => file.endsWith('.json')) : [];
|
||||
}
|
||||
|
||||
function checkSummary({ tools, reportRoot, sourceRoot }, updateBaseline) {
|
||||
const files = discoverSources(sourceRoot);
|
||||
rmSync(reportRoot, { recursive: true, force: true });
|
||||
mkdirSync(raw, { recursive: true });
|
||||
const tests = readdirSync(join(repositoryRoot, 'tests/js')).filter(file => file.endsWith('.test.js')).map(file => `tests/js/${file}`);
|
||||
run(['--test', ...tests], { NODE_V8_COVERAGE: raw });
|
||||
const playwright = join(tools, 'node_modules/@playwright/test/cli.js');
|
||||
run([playwright, 'test', '--config', join(tools, 'playwright.config.mjs')]);
|
||||
run([playwright, 'test', '--config', join(tools, 'playwright.workflows.config.mjs'),
|
||||
'--project', 'chromium', '--project', 'touch'], { PM_JS_COVERAGE: raw });
|
||||
run([join(tools, 'node_modules/c8/bin/c8.js'), 'report', '--all', '--src', sourceRoot,
|
||||
'--include', 'web/js/**/*.js', '--include', 'web/js/**/*.mjs', '--temp-directory', raw, '--reports-dir', reportRoot,
|
||||
'--reporter', 'text', '--reporter', 'json-summary', '--reporter', 'html']);
|
||||
const summary = JSON.parse(readFileSync(join(reportRoot, 'coverage-summary.json'), 'utf8'));
|
||||
const baselinePath = join(tools, 'frontend-coverage-baseline.json');
|
||||
if (process.argv.includes('--update-baseline')) {
|
||||
if (updateBaseline) {
|
||||
const previous = JSON.parse(readFileSync(baselinePath, 'utf8'));
|
||||
const failures = checkCoverage(summary, previous, sourceRoot, files.filter(file => previous[file]));
|
||||
if (failures.length) throw new Error(`Baseline updates cannot lower coverage:\n${failures.join('\n')}`);
|
||||
@@ -53,8 +89,51 @@ async function main() {
|
||||
const failures = checkCoverage(summary, baseline, sourceRoot, files);
|
||||
if (failures.length) throw new Error(`Frontend coverage regressed:\n${failures.join('\n')}`);
|
||||
}
|
||||
console.log(`Frontend coverage passed for ${files.length} files. Report: ${relative(repositoryRoot, reportRoot)}/index.html`);
|
||||
return files.length;
|
||||
}
|
||||
|
||||
export async function runCoverage({ argv = process.argv.slice(2), run = runNode, paths = defaultPaths() } = {}) {
|
||||
const { repositoryRoot: root, tools, reportRoot, raw, sourceRoot } = paths;
|
||||
const playwright = join(tools, 'node_modules/@playwright/test/cli.js');
|
||||
const actions = {
|
||||
clean() {
|
||||
rmSync(reportRoot, { recursive: true, force: true });
|
||||
mkdirSync(raw, { recursive: true });
|
||||
},
|
||||
prepare() {
|
||||
mkdirSync(raw, { recursive: true });
|
||||
},
|
||||
'require-raw'() {
|
||||
if (!rawCoverageFiles(raw).length) {
|
||||
throw new Error(`No raw coverage in ${relative(root, raw) || raw}: run the --shard steps first, `
|
||||
+ 'or put every shard\'s raw files there');
|
||||
}
|
||||
},
|
||||
'node-units'() {
|
||||
const tests = readdirSync(join(root, 'tests/js')).filter(file => file.endsWith('.test.js'))
|
||||
.map(file => `tests/js/${file}`);
|
||||
run(['--test', ...tests], { NODE_V8_COVERAGE: raw });
|
||||
},
|
||||
viewer() {
|
||||
run([playwright, 'test', '--config', join(tools, 'playwright.config.mjs')]);
|
||||
},
|
||||
workflows({ shard }) {
|
||||
run([playwright, 'test', '--config', join(tools, 'playwright.workflows.config.mjs'),
|
||||
'--project', 'chromium', '--project', 'touch', ...(shard ? ['--shard', shard] : [])], { PM_JS_COVERAGE: raw });
|
||||
},
|
||||
report() {
|
||||
run([join(tools, 'node_modules/c8/bin/c8.js'), 'report', '--all', '--src', sourceRoot,
|
||||
'--include', 'web/js/**/*.js', '--include', 'web/js/**/*.mjs', '--temp-directory', raw, '--reports-dir', reportRoot,
|
||||
'--reporter', 'text', '--reporter', 'json-summary', '--reporter', 'html']);
|
||||
},
|
||||
check({ updateBaseline }) {
|
||||
const count = checkSummary(paths, updateBaseline);
|
||||
console.log(`Frontend coverage passed for ${count} files. Report: ${relative(root, reportRoot)}/index.html`);
|
||||
},
|
||||
};
|
||||
for (const { step, ...options } of plan(argv)) await actions[step](options);
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch(error => { console.error(error.message); process.exitCode = 1; });
|
||||
runCoverage().catch(error => { console.error(error.message); process.exitCode = 1; });
|
||||
}
|
||||
|
||||
Vendored
+2
@@ -15,6 +15,8 @@
|
||||
"test:comfyui": "playwright test --config comfyui.config.mjs",
|
||||
"test:executions": "playwright test --config playwright.execution.config.mjs",
|
||||
"test:coverage": "node frontend-coverage.mjs",
|
||||
"test:coverage:shard": "node frontend-coverage.mjs --shard",
|
||||
"coverage:merge": "node frontend-coverage.mjs --merge-only",
|
||||
"coverage:baseline-check": "node coverage-baseline-check.mjs",
|
||||
"coverage:update": "node frontend-coverage.mjs --update-baseline",
|
||||
"test:compat": "playwright test --config pack-compat.config.mjs"
|
||||
|
||||
+6
-11
@@ -1,23 +1,18 @@
|
||||
import { defineConfig, devices } from '@playwright/test';
|
||||
import { repositoryRoot } from './vendor-viewerjs.mjs';
|
||||
const port = Number(process.env.PM_TEST_PORT || 8765);
|
||||
import { workflowWorkers } from './test-workers.mjs';
|
||||
|
||||
// No global webServer: tests/browser/fixtures.mjs starts one server per worker
|
||||
// on an OS-chosen port. Files run in parallel; tests inside a file keep their order.
|
||||
export default defineConfig({
|
||||
testDir: '../../tests/browser',
|
||||
outputDir: './test-results/workflows',
|
||||
testMatch: '*.spec.mjs',
|
||||
workers: 1,
|
||||
workers: workflowWorkers(),
|
||||
fullyParallel: false,
|
||||
forbidOnly: Boolean(process.env.CI),
|
||||
retries: 0,
|
||||
reporter: 'list',
|
||||
use: { baseURL: `http://127.0.0.1:${port}`, trace: 'retain-on-failure', screenshot: 'only-on-failure' },
|
||||
webServer: {
|
||||
command: `node tools/vendor/start-browser-server.mjs --port ${port}`,
|
||||
cwd: repositoryRoot,
|
||||
url: `http://127.0.0.1:${port}/prompt_manager/test`,
|
||||
reuseExistingServer: false,
|
||||
gracefulShutdown: { signal: 'SIGTERM', timeout: 5000 },
|
||||
},
|
||||
use: { trace: 'retain-on-failure', screenshot: 'only-on-failure' },
|
||||
projects: [
|
||||
{ name: 'chromium', outputDir: './test-results/workflows/chromium', use: { browserName: 'chromium' } },
|
||||
{ name: 'firefox', outputDir: './test-results/workflows/firefox', use: { browserName: 'firefox' } },
|
||||
|
||||
+154
-6
@@ -1,8 +1,156 @@
|
||||
import { spawn } from 'node:child_process';
|
||||
const child = spawn(process.env.PM_TEST_PYTHON || 'python',
|
||||
['-m', 'tests.browser.server', ...process.argv.slice(2)], { stdio: 'inherit' });
|
||||
child.on('error', error => { console.error(error.message); process.exitCode = 1; });
|
||||
child.on('exit', code => { process.exitCode = code ?? 1; });
|
||||
for (const signal of ['SIGTERM', 'SIGINT']) {
|
||||
process.once(signal, () => child.kill(signal));
|
||||
import { resolve } from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
import { repositoryRoot } from './vendor-viewerjs.mjs';
|
||||
|
||||
const READY_PATH = '/prompt_manager/test';
|
||||
const STARTUP_TIMEOUT_MS = 60_000;
|
||||
const STOP_TIMEOUT_MS = 5_000;
|
||||
const POLL_INTERVAL_MS = 100;
|
||||
const PROBE_TIMEOUT_MS = 1_000;
|
||||
const STDERR_TAIL_BYTES = 8_192;
|
||||
const HOST = '127.0.0.1';
|
||||
// tests/browser/server.py prints this once it holds its listening socket.
|
||||
const PORT_ANNOUNCEMENT = /^PM_TEST_SERVER_PORT=(\d+)\s*$/m;
|
||||
|
||||
const pythonServer = port => ['-m', 'tests.browser.server', '--port', String(port)];
|
||||
|
||||
function describeExit({ code, signal }) {
|
||||
return signal ? `signal ${signal}` : `code ${code}`;
|
||||
}
|
||||
|
||||
// Start one disposable tests/browser/server.py and wait until it serves the
|
||||
// readiness page. `port` 0 (the default) lets the server bind an OS-chosen port
|
||||
// itself and announce it on stdout, so nothing can take the port between
|
||||
// choosing and binding. Resolves to {port, url, exited, exitStatus,
|
||||
// stderrTail(), stop()}; rejects with the server's stderr tail when it exits
|
||||
// or stays unready.
|
||||
export async function startBrowserServer({
|
||||
port = 0,
|
||||
command = process.env.PM_TEST_PYTHON || 'python',
|
||||
args = pythonServer,
|
||||
cwd = repositoryRoot,
|
||||
timeoutMs = STARTUP_TIMEOUT_MS,
|
||||
} = {}) {
|
||||
if (!Number.isInteger(port) || port < 0 || port > 65535) {
|
||||
throw new Error(`startBrowserServer needs a port from 0 to 65535, got ${port}`);
|
||||
}
|
||||
const child = spawn(command, args(port), { cwd, stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
let stderr = '';
|
||||
child.stderr.setEncoding('utf8');
|
||||
child.stderr.on('data', chunk => { stderr = (stderr + chunk).slice(-STDERR_TAIL_BYTES); });
|
||||
let stdout = '';
|
||||
let announced = null;
|
||||
child.stdout.setEncoding('utf8');
|
||||
// Keep draining stdout for the server's whole life so it never blocks on a full pipe.
|
||||
child.stdout.on('data', chunk => {
|
||||
if (announced !== null) return;
|
||||
stdout = (stdout + chunk).slice(-STDERR_TAIL_BYTES);
|
||||
const match = PORT_ANNOUNCEMENT.exec(stdout);
|
||||
if (match) announced = Number(match[1]);
|
||||
});
|
||||
const killOnExit = () => child.kill('SIGKILL');
|
||||
process.once('exit', killOnExit);
|
||||
const server = {
|
||||
port: port || null,
|
||||
url: port ? `http://${HOST}:${port}` : null,
|
||||
exitStatus: null,
|
||||
stopping: false,
|
||||
stderrTail: () => stderr.trimEnd(),
|
||||
exited: new Promise(done => {
|
||||
child.once('error', error => {
|
||||
stderr += `\n${error.message}`;
|
||||
server.exitStatus = { code: null, signal: null, error };
|
||||
done(server.exitStatus);
|
||||
});
|
||||
child.once('exit', (code, signal) => {
|
||||
server.exitStatus = { code, signal };
|
||||
done(server.exitStatus);
|
||||
});
|
||||
}),
|
||||
async stop() {
|
||||
server.stopping = true;
|
||||
if (server.exitStatus === null) {
|
||||
child.kill('SIGTERM');
|
||||
const stopped = await Promise.race([server.exited, delay(STOP_TIMEOUT_MS).then(() => false)]);
|
||||
if (!stopped) {
|
||||
child.kill('SIGKILL');
|
||||
await server.exited;
|
||||
}
|
||||
}
|
||||
process.removeListener('exit', killOnExit);
|
||||
},
|
||||
};
|
||||
server.exited.then(() => process.removeListener('exit', killOnExit));
|
||||
const failure = async reason => {
|
||||
await server.stop();
|
||||
const tail = server.stderrTail();
|
||||
return new Error(`test server on port ${server.port ?? '(not yet announced)'} ${reason}`
|
||||
+ (tail ? `\n--- server stderr tail ---\n${tail}` : ''));
|
||||
};
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
if (server.exitStatus !== null) {
|
||||
throw await failure(`exited with ${describeExit(server.exitStatus)} before it was ready`);
|
||||
}
|
||||
if (announced !== null && server.url === null) {
|
||||
server.port = announced;
|
||||
server.url = `http://${HOST}:${announced}`;
|
||||
}
|
||||
if (announced !== null && announced !== server.port) {
|
||||
throw await failure(`announced port ${announced} instead`);
|
||||
}
|
||||
if (server.url === null) {
|
||||
await delay(POLL_INTERVAL_MS);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const response = await fetch(`${server.url}${READY_PATH}`, { signal: AbortSignal.timeout(PROBE_TIMEOUT_MS) });
|
||||
await response.arrayBuffer();
|
||||
if (response.ok) return server;
|
||||
} catch {
|
||||
// Not listening yet.
|
||||
}
|
||||
await delay(POLL_INTERVAL_MS);
|
||||
}
|
||||
throw await failure(`was not ready at ${READY_PATH} within ${timeoutMs} ms`);
|
||||
}
|
||||
|
||||
// Keep one live server for a Playwright worker. ensure() returns the running
|
||||
// server, or starts a fresh one when the last one died (a failed start is
|
||||
// reported, not retried); crash(server) turns an unexpected exit into an error
|
||||
// carrying the stderr tail.
|
||||
export function createServerSupervisor({ start }) {
|
||||
let current = null;
|
||||
return {
|
||||
async ensure() {
|
||||
if (current && current.exitStatus === null) return current;
|
||||
current = null;
|
||||
current = await start();
|
||||
return current;
|
||||
},
|
||||
crash(server) {
|
||||
if (!server || server.stopping || server.exitStatus === null) return null;
|
||||
const tail = server.stderrTail();
|
||||
return new Error(`test server on port ${server.port} exited unexpectedly (${describeExit(server.exitStatus)})`
|
||||
+ (tail ? `\n--- server stderr tail ---\n${tail}` : ''));
|
||||
},
|
||||
async stop() {
|
||||
const server = current;
|
||||
current = null;
|
||||
await server?.stop();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// CLI: `node start-browser-server.mjs --port N` runs one server in the foreground.
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
|
||||
const child = spawn(process.env.PM_TEST_PYTHON || 'python',
|
||||
['-m', 'tests.browser.server', ...process.argv.slice(2)], { stdio: 'inherit' });
|
||||
child.on('error', error => { console.error(error.message); process.exitCode = 1; });
|
||||
child.on('exit', code => { process.exitCode = code ?? 1; });
|
||||
for (const signal of ['SIGTERM', 'SIGINT']) {
|
||||
process.once(signal, () => child.kill(signal));
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+18
@@ -0,0 +1,18 @@
|
||||
import { availableParallelism } from 'node:os';
|
||||
|
||||
const MAX_LOCAL_WORKERS = 8;
|
||||
const CPUS_PER_WORKER = 4;
|
||||
|
||||
// PM_TEST_WORKERS wins; CI without it keeps one worker, so CI only changes
|
||||
// through explicit workflow edits; locally a quarter of the CPUs, at most eight.
|
||||
export function workflowWorkers({ env = process.env, cpus = availableParallelism() } = {}) {
|
||||
if (env.PM_TEST_WORKERS !== undefined) {
|
||||
const workers = Number(env.PM_TEST_WORKERS);
|
||||
if (!Number.isInteger(workers) || workers < 1) {
|
||||
throw new Error(`PM_TEST_WORKERS must be a positive integer, got "${env.PM_TEST_WORKERS}"`);
|
||||
}
|
||||
return workers;
|
||||
}
|
||||
if (env.CI) return 1;
|
||||
return Math.max(1, Math.min(MAX_LOCAL_WORKERS, Math.floor(cpus / CPUS_PER_WORKER)));
|
||||
}
|
||||
@@ -3,7 +3,7 @@
|
||||
* concept's classes.
|
||||
*
|
||||
* - Header: the page heading names the job; its line is the Activity item's (D54), and its actions are Download report
|
||||
* (disabled until Activity-2) and, while the job runs, "Cancel scan" (danger).
|
||||
* (the job's JSON report, D59) and, while the job runs, "Cancel scan" (danger).
|
||||
* - Progress (`section.surface`): `<progress>`, the server's "Processing file N of M · about X min left", and the
|
||||
* outcome counters as `dl.stat-row` tiles: Added to Library (green), Already known, Needs review (warning), No usable
|
||||
* metadata, Errors (red).
|
||||
@@ -21,6 +21,7 @@ import { showWorkspaceToast } from './toast.mjs';
|
||||
import { noun } from './plural.mjs';
|
||||
import { parseJob, JOBS_URL, LOST_DETAIL } from './activity-store.mjs';
|
||||
import { jobDetail, CANCEL_FAILED_TOAST } from './activity.mjs';
|
||||
import { downloadJobReport } from './activity-scan.mjs';
|
||||
import { createMediaViewer } from './media-viewer.mjs';
|
||||
import { createInspectionController } from './inspection-state.mjs';
|
||||
import { createPromptSender } from './prompt-sender.mjs';
|
||||
@@ -122,7 +123,6 @@ function section(title, id) {
|
||||
export function mountActivityJobView(main, { store, scope, api, jobId }) {
|
||||
const parts = pageHeading(main);
|
||||
const download = createButton('Download report');
|
||||
download.disabled = true;
|
||||
const cancel = createButton('Cancel scan', 'danger');
|
||||
|
||||
const progressPart = section('Progress', 'job-progress-title');
|
||||
@@ -271,6 +271,10 @@ export function mountActivityJobView(main, { store, scope, api, jobId }) {
|
||||
document.title = `${job.title} · Prompt Manager`;
|
||||
parts.description.textContent = jobDetail(job);
|
||||
cancel.textContent = scan ? 'Cancel scan' : 'Cancel job';
|
||||
// A job lost with the server has no report there (review M9).
|
||||
const lost = job.state === 'lost';
|
||||
if (lost && document.activeElement === download) parts.h1.focus();
|
||||
download.disabled = lost;
|
||||
if (job.state === 'running') {
|
||||
if (!cancel.isConnected) parts.actions.append(cancel);
|
||||
} else if (cancel.isConnected) {
|
||||
@@ -290,9 +294,11 @@ export function mountActivityJobView(main, { store, scope, api, jobId }) {
|
||||
logLabel.textContent = `● ${scan ? 'Scan' : 'Job'} log`;
|
||||
const text = detail?.log.length ? detail.log.join('\n') : 'No log lines yet.';
|
||||
if (pre.textContent !== text) {
|
||||
// Replacing the text clamps the scroll position, so keep it: pinned readers follow the end, others stay.
|
||||
const { scrollTop } = pre;
|
||||
const atEnd = pre.scrollTop + pre.clientHeight >= pre.scrollHeight - 2;
|
||||
pre.textContent = text;
|
||||
if (atEnd) pre.scrollTop = pre.scrollHeight;
|
||||
pre.scrollTop = atEnd ? pre.scrollHeight : scrollTop;
|
||||
}
|
||||
renderRows(job);
|
||||
}
|
||||
@@ -323,7 +329,12 @@ export function mountActivityJobView(main, { store, scope, api, jobId }) {
|
||||
reading = false;
|
||||
}
|
||||
render();
|
||||
if (readAgain) { readAgain = false; scheduleRead(); }
|
||||
// Read once more only if the summary moved past what this read returned (review: no redundant re-read).
|
||||
if (readAgain) {
|
||||
readAgain = false;
|
||||
const job = store.find(jobId);
|
||||
if (job && job.state !== 'lost' && detailChanged(job)) scheduleRead();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -347,10 +358,23 @@ export function mountActivityJobView(main, { store, scope, api, jobId }) {
|
||||
}
|
||||
|
||||
scope.listen(log, 'toggle', () => {
|
||||
// Every open starts at the newest line: while closed the log has no layout box (Firefox, WebKit), so a kept
|
||||
// position would be lost anyway (review M6).
|
||||
if (log.open) pre.scrollTop = pre.scrollHeight;
|
||||
const job = store.find(jobId);
|
||||
if (log.open && job && job.state !== 'lost' && detailChanged(job)) scheduleRead();
|
||||
});
|
||||
|
||||
scope.listen(download, 'click', async () => {
|
||||
if (download.disabled || download.getAttribute('aria-disabled') === 'true') return;
|
||||
download.setAttribute('aria-disabled', 'true');
|
||||
try {
|
||||
// A view left while the report downloads shows no toast (review M10).
|
||||
await downloadJobReport(jobId, { signal: scope.signal });
|
||||
} finally {
|
||||
download.removeAttribute('aria-disabled');
|
||||
}
|
||||
});
|
||||
scope.listen(cancel, 'click', async () => {
|
||||
if (cancel.getAttribute('aria-disabled') === 'true') return;
|
||||
cancel.setAttribute('aria-disabled', 'true');
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
/**
|
||||
* Scan for metadata (#249 Activity-1, D55): Images → Import → "Scan for metadata" → Review scan opens the concept's
|
||||
* "Review metadata scan". Its backup box downloads the existing `/prompt_manager/backup` copy and toasts the outcome;
|
||||
* Start scan starts the session job through the Activity store, which toasts "Scan started.", and opens the job's view.
|
||||
* Start scan starts the session job through the Activity store over the chosen scope (new and unresolved files, or
|
||||
* all files with the D58 checkbox), which toasts "Scan started.", and opens the job's view.
|
||||
* A start refused because another record-changing job runs toasts "Another job is running: <title>." with a link to
|
||||
* that job (D57).
|
||||
*/
|
||||
@@ -12,13 +13,29 @@ import { navigateTo } from './state.mjs';
|
||||
export const BACKUP_URL = '/prompt_manager/backup';
|
||||
export const BACKUP_DONE = 'Database backup downloaded.';
|
||||
export const BACKUP_FAILED = 'The database could not be backed up. Try again.';
|
||||
export const REPORT_DONE = 'Job report downloaded.';
|
||||
export const REPORT_FAILED = 'The report could not be downloaded. Try again.';
|
||||
const JOBS_REPORT_BASE = '/prompt_manager/jobs';
|
||||
export const START_FAILED = 'The scan could not start. Try again.';
|
||||
const FALLBACK_NAME = 'prompts_backup.db';
|
||||
const REPORT_FALLBACK_NAME = 'prompt-manager-report.json';
|
||||
|
||||
/** The download name from a Content-Disposition header, or the fallback. */
|
||||
export function backupFilename(disposition) {
|
||||
/** The download name from a Content-Disposition header, or `fallback`. */
|
||||
export function backupFilename(disposition, fallback = FALLBACK_NAME) {
|
||||
const match = /filename="?([^";]+)"?/i.exec(disposition || '');
|
||||
return match ? match[1] : FALLBACK_NAME;
|
||||
return match ? match[1] : fallback;
|
||||
}
|
||||
|
||||
/** Save a fetched response through a temporary link, named by `filename`. */
|
||||
function saveResponse(blob, filename) {
|
||||
const url = URL.createObjectURL(blob);
|
||||
const link = document.createElement('a');
|
||||
link.href = url;
|
||||
link.download = filename;
|
||||
document.body.append(link);
|
||||
link.click();
|
||||
link.remove();
|
||||
setTimeout(() => URL.revokeObjectURL(url), 1000);
|
||||
}
|
||||
|
||||
/** Download the prompts database copy the legacy admin page also uses; toasts the outcome. */
|
||||
@@ -27,14 +44,7 @@ export async function downloadDatabaseBackup({ fetchUrl = window.ApiClient.fetch
|
||||
const response = await fetchUrl(BACKUP_URL);
|
||||
if (!response.ok) throw new Error(`Backup failed with ${response.status}`);
|
||||
const blob = await response.blob();
|
||||
const url = URL.createObjectURL(blob);
|
||||
const link = document.createElement('a');
|
||||
link.href = url;
|
||||
link.download = backupFilename(response.headers.get('Content-Disposition'));
|
||||
document.body.append(link);
|
||||
link.click();
|
||||
link.remove();
|
||||
setTimeout(() => URL.revokeObjectURL(url), 1000);
|
||||
saveResponse(blob, backupFilename(response.headers.get('Content-Disposition')));
|
||||
notify(BACKUP_DONE);
|
||||
return true;
|
||||
} catch {
|
||||
@@ -43,16 +53,39 @@ export async function downloadDatabaseBackup({ fetchUrl = window.ApiClient.fetch
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Download a job's JSON report (Activity-2, D59) through the api-client URL, so a proxy mount works; toasts the outcome
|
||||
* unless `signal` (the view's scope) was aborted meanwhile.
|
||||
*/
|
||||
export async function downloadJobReport(jobId, { fetchUrl = window.ApiClient.fetchUrl, notify = showWorkspaceToast,
|
||||
signal = null } = {}) {
|
||||
try {
|
||||
const response = await fetchUrl(`${JOBS_REPORT_BASE}/${encodeURIComponent(jobId)}/report`);
|
||||
if (!response.ok) throw new Error(`Report failed with ${response.status}`);
|
||||
const blob = await response.blob();
|
||||
saveResponse(blob, backupFilename(response.headers.get('Content-Disposition'), REPORT_FALLBACK_NAME));
|
||||
// The file is saved either way; a view left meanwhile (`signal` aborted) shows no toast.
|
||||
if (!signal?.aborted) notify(REPORT_DONE);
|
||||
return true;
|
||||
} catch {
|
||||
if (!signal?.aborted) notify(REPORT_FAILED);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** The toast for a refused start, linking the job that holds the slot. */
|
||||
export function busyToast(running) {
|
||||
if (!running) return 'Another job is running. Try again when it ends.';
|
||||
return ['Another job is running: ', { text: running.title, href: `#/activity/${running.id}` }, '.'];
|
||||
}
|
||||
|
||||
/** Start the scan job and open its view; a refused or failed start says why and stays on the page. */
|
||||
export async function startMetadataScan({ store, scope, notify = showWorkspaceToast, navigate = navigateTo }) {
|
||||
/**
|
||||
* Start the scan job over `files` ('new': new and unresolved files, or 'all') and open its view; a refused or failed
|
||||
* start says why and stays on the page.
|
||||
*/
|
||||
export async function startMetadataScan({ store, scope, files = 'new', notify = showWorkspaceToast, navigate = navigateTo }) {
|
||||
try {
|
||||
const result = await store.start('scan');
|
||||
const result = await store.start('scan', { scope: files });
|
||||
if (scope.signal.aborted) return;
|
||||
if (result.busy !== undefined) {
|
||||
notify(busyToast(result.busy));
|
||||
@@ -68,5 +101,5 @@ export async function startMetadataScan({ store, scope, notify = showWorkspaceTo
|
||||
export function openMetadataScan({ scope, store, sources, trigger = null }) {
|
||||
openScanReviewDialog({ scope, sources, trigger,
|
||||
onBackup: () => downloadDatabaseBackup(),
|
||||
onStart: () => startMetadataScan({ store, scope }) });
|
||||
onStart: ({ scope: files }) => startMetadataScan({ store, scope, files }) });
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
*
|
||||
* Every opener takes the page `scope`: disposing it closes and removes the shell, so a dialog never outlives its page.
|
||||
*/
|
||||
import { createButton } from '../shared/toolkit/core.js';
|
||||
import { createButton, createSwitch } from '../shared/toolkit/core.js';
|
||||
|
||||
const FIELD_LABELS = Object.freeze({ source: 'Source', folder: 'Folder', media: 'Media', tags: 'Tags' });
|
||||
const FIELD_ORDER = Object.keys(FIELD_LABELS);
|
||||
@@ -342,22 +342,44 @@ export const SCAN_REVIEW_SENTENCE = 'Enrich positive / negative associations, re
|
||||
const BACKUP_LABEL = 'Back up now';
|
||||
const BACKUP_BUSY_LABEL = 'Backing up…';
|
||||
|
||||
export const SCAN_SCOPE_LABELS = { new: 'New and unresolved files', all: 'All files' };
|
||||
export const SCAN_ALL_FILES_HINT = 'Also read the files already linked to their prompt again.';
|
||||
|
||||
/**
|
||||
* The concept's "Review metadata scan" (`scan-review`, D55). `sources` is the number of output folders the scan reads;
|
||||
* the scope is fixed to "All files" in Activity-1. `onBackup` runs the database backup (the button is aria-disabled
|
||||
* and reads "Backing up…" until it settles, and the dialog stays open); Start scan closes the dialog, then `onStart`.
|
||||
* The concept's "Review metadata scan" (`scan-review`, D55). `sources` is the number of output folders the scan reads.
|
||||
* The scope is the concept's "New and unresolved files"; the "All files" checkbox (D58, the concept's `.setting-row`
|
||||
* switch, unchecked) switches the Scope value to "All files". `onBackup` runs the database backup (the button is
|
||||
* aria-disabled and reads "Backing up…" until it settles, and the dialog stays open); Start scan closes the dialog, then
|
||||
* calls `onStart({scope: 'new' | 'all'})`.
|
||||
*/
|
||||
export function openScanReviewDialog({ scope, sources, onBackup, onStart, trigger = null }) {
|
||||
requireScope(scope);
|
||||
requireCallbacks({ onBackup, onStart });
|
||||
if (!Number.isSafeInteger(sources) || sources < 0) throw new TypeError('The scan review needs its source count');
|
||||
const grid = el('dl', undefined, 'meta-grid');
|
||||
const values = {};
|
||||
for (const [term, value] of [['Sources', `${sources.toLocaleString('en-US')} output ${sources === 1 ? 'folder' : 'folders'}`],
|
||||
['Scope', 'All files']]) {
|
||||
['Scope', SCAN_SCOPE_LABELS.new]]) {
|
||||
const entry = el('div');
|
||||
entry.append(el('dt', term), el('dd', value));
|
||||
values[term] = el('dd', value);
|
||||
entry.append(el('dt', term), values[term]);
|
||||
grid.append(entry);
|
||||
}
|
||||
// The concept's System toggle: a `.setting-row` whose checkbox is the Toolkit switch (concept `hydrateSwitches`).
|
||||
const toggle = createSwitch('', '', 'scan-all-files');
|
||||
const allFiles = toggle.querySelector('input');
|
||||
allFiles.checked = false;
|
||||
const option = el('div', undefined, 'setting-row');
|
||||
const optionText = el('div');
|
||||
const optionLabel = el('label', SCAN_SCOPE_LABELS.all);
|
||||
optionLabel.htmlFor = allFiles.id;
|
||||
const hint = el('p', SCAN_ALL_FILES_HINT);
|
||||
hint.id = 'scan-all-files-hint';
|
||||
allFiles.setAttribute('aria-describedby', hint.id);
|
||||
optionText.append(optionLabel, hint);
|
||||
option.append(optionText, toggle);
|
||||
const chosenScope = () => (allFiles.checked ? 'all' : 'new');
|
||||
allFiles.addEventListener('change', () => { values.Scope.textContent = SCAN_SCOPE_LABELS[chosenScope()]; });
|
||||
const notice = el('div', undefined, 'notice');
|
||||
const copy = el('div');
|
||||
copy.append(el('h3', 'Back up the database first (recommended)'),
|
||||
@@ -381,10 +403,11 @@ export function openScanReviewDialog({ scope, sources, onBackup, onStart, trigge
|
||||
const start = createButton('Start scan', 'ca-primary');
|
||||
cancel.addEventListener('click', () => finish());
|
||||
start.addEventListener('click', () => {
|
||||
const files = chosenScope();
|
||||
finish();
|
||||
onStart();
|
||||
onStart({ scope: files });
|
||||
});
|
||||
const actions = el('div', undefined, 'dialog-actions');
|
||||
actions.append(cancel, start);
|
||||
show('Review metadata scan', [grid, el('p', SCAN_REVIEW_SENTENCE, 'small'), notice, actions], { scope, trigger });
|
||||
show('Review metadata scan', [grid, option, el('p', SCAN_REVIEW_SENTENCE, 'small'), notice, actions], { scope, trigger });
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user