Compare commits
10
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
82ec8782b4 | ||
|
|
14cfb8aedf | ||
|
|
ed809f7fa4 | ||
|
|
cbc8a61b1d | ||
|
|
b665aa687d | ||
|
|
f85b45d9a8 | ||
|
|
21a2cc0932 | ||
|
|
3ebf5c48cf | ||
|
|
0156d3fadb | ||
|
|
357c0f09e7 |
@@ -16,9 +16,19 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
# Timeouts are the measured job time plus at least 50%, and never under 20 minutes for a browser job,
|
||||
# because runner speed varies. Before sharding (main 2cecfcf, one worker): frontend coverage 32m10s,
|
||||
# page workflows firefox 18m59s, webkit 19m26s. First sharded run (PR #274 run 39712909047, 3 workers,
|
||||
# no npm cache, where npm ci stalled for 5m in six of eight jobs): coverage shards 11m19s/10m47s/10m49s
|
||||
# (tests 5m20s/4m42s/4m46s), merge 5m52s (report and check 4s), firefox shards 7m53s/9m27s (tests
|
||||
# 4m41s/3m29s), webkit shards 12m25s+ (cancelled at the old 12-minute limit after its 267 tests passed in
|
||||
# 6m29s) and 4m53s (tests 3m08s). With the npm cache hit (run 39725176320 at cbc8a61, npm ci 0-2s):
|
||||
# coverage shards 7m34s/6m04s/5m22s (tests 6m38s/4m53s/4m04s), merge 54s, firefox 7m57s/5m21s (tests
|
||||
# 7m08s/4m34s), webkit 8m16s/5m03s (tests 7m26s/3m58s); a cache miss adds up to 5m to npm ci, which
|
||||
# the 20-minute limits absorb.
|
||||
jobs:
|
||||
frontend-coverage:
|
||||
name: frontend coverage (chromium and touch)
|
||||
name: frontend coverage (shard ${{ matrix.shard }}/3)
|
||||
runs-on: ubuntu-latest
|
||||
# Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the
|
||||
# same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json;
|
||||
@@ -26,12 +36,15 @@ jobs:
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble
|
||||
options: --user 1001
|
||||
# This job runs every page workflow twice (chromium, then touch) on one worker with V8 coverage on, so it
|
||||
# takes about twice a single-browser job. Measured 2026-10-20: whole job 18m26s-24m57s (main 5ded3cf
|
||||
# 22m15s; PR head 6359999 was cancelled at the old 25-minute limit while still passing), coverage step
|
||||
# alone 20m55s-22m17s+, against ~12-15 minutes for the firefox/webkit jobs. 40 minutes leaves headroom
|
||||
# for suite growth while still stopping a hung run; re-measure if a run passes 30 minutes.
|
||||
timeout-minutes: 40
|
||||
timeout-minutes: 20
|
||||
# Public-repo ubuntu-latest runners have 4 vCPUs: three Playwright workers, each with its own test
|
||||
# server, leave one CPU for the servers and the runner.
|
||||
env:
|
||||
PM_TEST_WORKERS: 3
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: [1, 2, 3]
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
# The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache
|
||||
@@ -50,24 +63,107 @@ jobs:
|
||||
"$venv/bin/python" -m pip install --no-cache-dir -r requirements-dev.txt
|
||||
echo "$venv/bin" >> "$GITHUB_PATH"
|
||||
- run: python --version && node --version && node --version | grep -q '^v24\.'
|
||||
# npm's download cache, keyed on the lockfile; inside the container ~ is the job's HOME (/github/home),
|
||||
# which npm also uses. node_modules itself is still built fresh by npm ci.
|
||||
- name: Cache npm downloads
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/.npm
|
||||
key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }}
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
- run: npm run --prefix tools/vendor check
|
||||
- run: npm run --prefix tools/vendor test
|
||||
- name: Frontend unit, viewer, workflow, and coverage checks
|
||||
run: npm run --prefix tools/vendor test:coverage
|
||||
- name: Upload frontend coverage and Chromium traces
|
||||
# Shard 1 also runs the node units and the viewer suite, so the vendor pins and the tooling tests
|
||||
# run on the lightest shard instead.
|
||||
- if: matrix.shard == 3
|
||||
run: npm run --prefix tools/vendor check
|
||||
- if: matrix.shard == 3
|
||||
run: npm run --prefix tools/vendor test
|
||||
- name: Page workflows with coverage, shard ${{ matrix.shard }}/3
|
||||
run: npm run --prefix tools/vendor test:coverage:shard -- ${{ matrix.shard }}/3
|
||||
- name: Upload raw coverage
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: coverage-raw-${{ matrix.shard }}
|
||||
path: coverage/js/raw/
|
||||
if-no-files-found: error
|
||||
retention-days: 3
|
||||
- name: Upload Chromium and touch traces
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: frontend-traces-shard-${{ matrix.shard }}
|
||||
path: tools/vendor/test-results/
|
||||
retention-days: 14
|
||||
|
||||
frontend-coverage-merge:
|
||||
name: frontend coverage (merge)
|
||||
needs: frontend-coverage
|
||||
runs-on: ubuntu-latest
|
||||
# Same image as the shards: the raw files name sources by the container's workspace path.
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble
|
||||
options: --user 1001
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- run: node --version && node --version | grep -q '^v24\.'
|
||||
# npm's download cache, keyed on the lockfile; inside the container ~ is the job's HOME (/github/home),
|
||||
# which npm also uses. node_modules itself is still built fresh by npm ci.
|
||||
- name: Cache npm downloads
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/.npm
|
||||
key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }}
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
# Each artifact lands in its own coverage-raw-N directory, so the next step can count them.
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: coverage-raw-*
|
||||
path: ${{ runner.temp }}/coverage-raw
|
||||
# coverage:merge only checks that raw files exist; merging two of three shards would report
|
||||
# partial coverage, so a missing or empty shard artifact fails here.
|
||||
- name: Verify every shard's raw coverage arrived
|
||||
shell: bash
|
||||
env:
|
||||
STAGED_RAW: ${{ runner.temp }}/coverage-raw
|
||||
EXPECTED_SHARDS: 3
|
||||
run: |
|
||||
found=0
|
||||
for shard in $(seq 1 "$EXPECTED_SHARDS"); do
|
||||
directory="$STAGED_RAW/coverage-raw-$shard"
|
||||
count=0
|
||||
if [ -d "$directory" ]; then count=$(find "$directory" -maxdepth 1 -type f -name '*.json' | wc -l); fi
|
||||
echo "coverage-raw-$shard: $count raw files"
|
||||
if [ "$count" -gt 0 ]; then found=$((found + 1)); fi
|
||||
done
|
||||
echo "shard artifacts with raw coverage: $found of $EXPECTED_SHARDS"
|
||||
if [ "$found" -ne "$EXPECTED_SHARDS" ]; then
|
||||
echo "::error::Expected raw coverage from $EXPECTED_SHARDS shards, got $found of $EXPECTED_SHARDS"
|
||||
exit 1
|
||||
fi
|
||||
staged=$(find "$STAGED_RAW" -mindepth 2 -maxdepth 2 -type f -name '*.json' | wc -l)
|
||||
mkdir -p coverage/js/raw
|
||||
# cp -n never overwrites; a clash shows up as fewer merged files than staged ones.
|
||||
find "$STAGED_RAW" -mindepth 2 -maxdepth 2 -type f -name '*.json' -exec cp -n -t coverage/js/raw {} + || true
|
||||
merged=$(find coverage/js/raw -maxdepth 1 -type f -name '*.json' | wc -l)
|
||||
if [ "$merged" -ne "$staged" ]; then
|
||||
echo "::error::Shards wrote clashing raw file names: $staged staged, $merged merged"
|
||||
exit 1
|
||||
fi
|
||||
echo "merged $merged raw files"
|
||||
- name: Frontend coverage report and baseline check
|
||||
run: npm run --prefix tools/vendor coverage:merge
|
||||
- name: Upload frontend coverage report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: frontend-test-reports
|
||||
path: |
|
||||
tools/vendor/test-results/
|
||||
coverage/js/
|
||||
!coverage/js/raw/**
|
||||
retention-days: 14
|
||||
|
||||
browser-workflows:
|
||||
name: page workflows (${{ matrix.browser }})
|
||||
name: page workflows (${{ matrix.browser }}, shard ${{ matrix.shard }}/2)
|
||||
runs-on: ubuntu-latest
|
||||
# Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the
|
||||
# same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json;
|
||||
@@ -75,11 +171,16 @@ jobs:
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble
|
||||
options: --user 1001
|
||||
timeout-minutes: 25
|
||||
timeout-minutes: 20
|
||||
# Public-repo ubuntu-latest runners have 4 vCPUs: three Playwright workers, each with its own test
|
||||
# server, leave one CPU for the servers and the runner.
|
||||
env:
|
||||
PM_TEST_WORKERS: 3
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
browser: [firefox, webkit]
|
||||
shard: [1, 2]
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
# The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache
|
||||
@@ -98,22 +199,29 @@ jobs:
|
||||
"$venv/bin/python" -m pip install --no-cache-dir -r requirements-dev.txt
|
||||
echo "$venv/bin" >> "$GITHUB_PATH"
|
||||
- run: python --version && node --version && node --version | grep -q '^v24\.'
|
||||
# npm's download cache, keyed on the lockfile; inside the container ~ is the job's HOME (/github/home),
|
||||
# which npm also uses. node_modules itself is still built fresh by npm ci.
|
||||
- name: Cache npm downloads
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/.npm
|
||||
key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }}
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
- name: Page workflows in ${{ matrix.browser }}
|
||||
run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }}
|
||||
- name: Page workflows in ${{ matrix.browser }}, shard ${{ matrix.shard }}/2
|
||||
run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }} --shard ${{ matrix.shard }}/2
|
||||
- name: Upload browser traces
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: browser-traces-${{ matrix.browser }}
|
||||
name: browser-traces-${{ matrix.browser }}-shard-${{ matrix.shard }}
|
||||
path: tools/vendor/test-results/
|
||||
retention-days: 14
|
||||
|
||||
# Branch protection requires the exact context "viewerjs". This job keeps that
|
||||
# name and fails unless every frontend job above succeeded, including when one
|
||||
# was cancelled or skipped.
|
||||
# name and fails unless every frontend job above succeeded (every shard, the
|
||||
# coverage merge and every browser shard), including when one was cancelled or skipped.
|
||||
viewerjs:
|
||||
needs: [frontend-coverage, browser-workflows]
|
||||
needs: [frontend-coverage, frontend-coverage-merge, browser-workflows]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
@@ -121,8 +229,10 @@ jobs:
|
||||
- name: Require every frontend job to succeed
|
||||
env:
|
||||
COVERAGE_RESULT: ${{ needs.frontend-coverage.result }}
|
||||
MERGE_RESULT: ${{ needs.frontend-coverage-merge.result }}
|
||||
WORKFLOWS_RESULT: ${{ needs.browser-workflows.result }}
|
||||
run: |
|
||||
echo "frontend-coverage: $COVERAGE_RESULT"
|
||||
echo "frontend-coverage-merge: $MERGE_RESULT"
|
||||
echo "browser-workflows: $WORKFLOWS_RESULT"
|
||||
test "$COVERAGE_RESULT" = success && test "$WORKFLOWS_RESULT" = success
|
||||
test "$COVERAGE_RESULT" = success && test "$MERGE_RESULT" = success && test "$WORKFLOWS_RESULT" = success
|
||||
|
||||
+72
-14
@@ -39,9 +39,44 @@ npm run --prefix tools/vendor test:coverage
|
||||
npm run --prefix tools/vendor test:workflows -- --project firefox --project webkit
|
||||
```
|
||||
|
||||
`PM_TEST_PYTHON` selects the Python executable; `PM_TEST_PORT` selects the local
|
||||
port (default 8765). Set these environment variables when Python is outside PATH
|
||||
or the port is occupied. Test servers are never reused. On unsupported Linux
|
||||
`PM_TEST_PYTHON` selects the Python executable; set it when Python is outside
|
||||
PATH. Every Playwright worker starts its own `tests/browser/server.py --port 0`,
|
||||
which binds a port the OS picks and prints it as `PM_TEST_SERVER_PORT=<n>`, with
|
||||
its own temp root and database (`tests/browser/fixtures.mjs`), so
|
||||
parallel workers and concurrent runs never share state. `PM_TEST_WORKERS` sets the
|
||||
worker count (default 1 under `CI`, otherwise a quarter of the CPUs, at most 8).
|
||||
Test servers are never reused across runs. The launcher holds each server's
|
||||
stdin open and the server (`--exit-on-stdin-eof`) shuts down cleanly, removing its
|
||||
temp root, when that pipe closes, so a worker killed without teardown leaves no
|
||||
server behind. A test that fails gets the stderr its server wrote during the test
|
||||
attached to the report as `server-stderr` (an aiohttp traceback behind a 500).
|
||||
|
||||
The coverage run can be split the way CI splits it, and each shard keeps the raw
|
||||
V8 files it writes under `coverage/js/raw`:
|
||||
|
||||
```sh
|
||||
PM_TEST_WORKERS=8 npm run --prefix tools/vendor test:coverage # whole run, one report
|
||||
npm run --prefix tools/vendor test:coverage:shard -- 1/3 # node units, viewer suite, workflows 1/3
|
||||
npm run --prefix tools/vendor test:coverage:shard -- 2/3 # workflows 2/3 (likewise 3/3)
|
||||
npm run --prefix tools/vendor coverage:merge # report and baseline check of every raw file
|
||||
npm run --prefix tools/vendor test:workflows -- --project webkit --shard 2/2
|
||||
```
|
||||
|
||||
`coverage:merge` refuses an empty raw directory; it does not count shards, so run
|
||||
every shard first (CI's merge job checks all three arrived). Measured on a 32-CPU
|
||||
machine in the pinned image (2026-10-21): `test:coverage` takes about 5 minutes
|
||||
with 8 workers and 24 minutes with 1; all four projects, 8 workers, each test 3
|
||||
times (6132 tests), 24 minutes. A CI-sized run (`docker run --cpus=4`,
|
||||
`PM_TEST_WORKERS=3`, all four projects once, 2044 tests) takes 25.5 minutes.
|
||||
|
||||
Specs must pass beside each other, not only alone. Wait for the page's own
|
||||
signal, never for time: a rendering frame (`requestAnimationFrame`) for layout
|
||||
work queued by a render or a scroll, a response or a visible state for data. A
|
||||
timer that "usually" covers a frame is a race on a loaded runner. Library
|
||||
virtualization specs settle with `libraryObservation.settled()` (two consecutive
|
||||
frames with no Library frame queued). To look for such races, repeat a spec under
|
||||
load (`--workers 8 --repeat-each 10`, and in the image with `--cpus=1` or `2`).
|
||||
On unsupported Linux
|
||||
distributions, browser system dependencies may need manual installation. For a
|
||||
quick browser run, use `test:workflows -- --project chromium`.
|
||||
|
||||
@@ -114,16 +149,39 @@ second without a browser and fails, naming the files, when a measured
|
||||
`web/js` file has no baseline entry or the baseline lists a file that no longer
|
||||
exists. It uses the gate's own `discoverSources`, and `npm test` runs the same
|
||||
check, so run it before pushing a new or deleted frontend file instead of
|
||||
waiting for the `frontend coverage` job.
|
||||
waiting for the `frontend coverage` jobs.
|
||||
|
||||
The `viewerjs` check runs on every PR, including backend-only changes. It is an
|
||||
aggregate job that passes only when the `frontend coverage (chromium and touch)`
|
||||
job and both `page workflows (firefox)` and `page workflows (webkit)` jobs
|
||||
succeed; the browsers run on separate runners in parallel. GitHub artifacts
|
||||
named `frontend-test-reports` contain HTML coverage, a JSON summary, and the
|
||||
Chromium/touch failure screenshots/traces; `browser-traces-firefox` and
|
||||
`browser-traces-webkit` hold the other browsers' traces. Open traces with
|
||||
Playwright's `show-trace` tool.
|
||||
aggregate job that passes only when every frontend job succeeds: the three
|
||||
`frontend coverage (shard N/3)` jobs, `frontend coverage (merge)`, and the four
|
||||
`page workflows (firefox|webkit, shard N/2)` jobs. A failed, cancelled or skipped
|
||||
job fails it. Every browser job runs with `PM_TEST_WORKERS: 3` (the public
|
||||
runners have 4 vCPUs). Each coverage shard runs `test:coverage:shard -- N/3`
|
||||
(shard 1 also runs the node units and the viewer suite, shard 3 the vendor pins
|
||||
and tooling tests) and uploads its raw V8 files as `coverage-raw-N` (kept 3
|
||||
days). The merge job downloads them, fails unless all three arrived with raw
|
||||
files and without clashing names, then runs `coverage:merge` (report plus
|
||||
baseline check). `tests/vendor/ci-workflow.test.mjs` checks this shape. GitHub
|
||||
artifacts named `frontend-test-reports` contain the merged HTML coverage and
|
||||
JSON summary; `frontend-traces-shard-N` hold the Chromium/touch failure
|
||||
screenshots/traces and `browser-traces-<browser>-shard-N` the other browsers'
|
||||
traces. Open traces with Playwright's `show-trace` tool.
|
||||
|
||||
Every `npm ci` first restores npm's download cache (`~/.npm`), keyed on the vendor
|
||||
lockfile and the job's pinned image tag (`tests/vendor/ci-workflow.test.mjs` checks
|
||||
the key names the same tag as the job's `container.image`). Without it, `npm ci` stalled for about 300 s
|
||||
(npm's fetch timeout) in most of the eight concurrent jobs; with a hit it takes
|
||||
0 to 2 s. A lockfile change misses the cache once, so expect one slow run after it.
|
||||
Measured on PR #274 (run 39725176320 at cbc8a61, cache hit): the whole workflow
|
||||
took 8m38s, against about 33 minutes with one worker and no shards. Jobs: coverage
|
||||
shards 7m34s/6m04s/5m22s (tests 6m38s/4m53s/4m04s), merge 54s, firefox shards
|
||||
7m57s/5m21s (tests 7m08s/4m34s), webkit shards 8m16s/5m03s (tests 7m26s/3m58s).
|
||||
Browser jobs keep a 20-minute limit: the slowest job is about 8.5 minutes, a cache
|
||||
miss adds up to 5 to `npm ci`, and runner speed varies (the first sharded run was
|
||||
cancelled at a 12-minute limit).
|
||||
Playwright shards by test count, whole files at a time, so shard 1 carries the
|
||||
slow concept-comparison specs (`*-concept.spec.mjs`, alphabetically first) and
|
||||
runs longer; the remedy, if it starts to matter, is more shards, not reordering.
|
||||
|
||||
The real-host suite covers both Classic and Nodes 2.0 startup, switching both
|
||||
ways, actual database prompt selection, execution-text serialization, workflow
|
||||
@@ -1556,7 +1614,7 @@ bytes at the same path without changing the immutable receipt.
|
||||
The focused acceptance command retains all four projects:
|
||||
|
||||
```sh
|
||||
PM_TEST_PORT=8766 npm run --prefix tools/vendor test:workflows -- \
|
||||
npm run --prefix tools/vendor test:workflows -- \
|
||||
workspace-history.spec.mjs workspace-visual.spec.mjs \
|
||||
--project chromium --project touch --project firefox --project webkit
|
||||
npm run --prefix tools/vendor test:executions
|
||||
@@ -1649,7 +1707,7 @@ acceptance; a baseline is added only after the maintainer accepts them.
|
||||
Focused commands:
|
||||
|
||||
```sh
|
||||
PM_TEST_PORT=8766 npm run --prefix tools/vendor test:workflows -- \
|
||||
npm run --prefix tools/vendor test:workflows -- \
|
||||
workspace-library-editor.spec.mjs library-editor-visual.spec.mjs \
|
||||
--project chromium --project touch --project firefox --project webkit
|
||||
npm run --prefix tools/vendor test:comfyui # library-sender.spec.mjs host regression
|
||||
@@ -1749,7 +1807,7 @@ mobile, the concept geometry/token/control comparison and its mutation check),
|
||||
|
||||
```sh
|
||||
node --test tests/js/workspace-vocabulary-state.test.js tests/js/workspace-state.test.js
|
||||
PM_TEST_PORT=8766 npm run --prefix tools/vendor test:workflows -- \
|
||||
npm run --prefix tools/vendor test:workflows -- \
|
||||
workspace-vocabulary.spec.mjs vocabulary-visual.spec.mjs focused-node.spec.mjs \
|
||||
--project chromium --project touch --project firefox --project webkit
|
||||
```
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { THEMES, seedConcept, openConcept, openProduction, diffInventory, mutate } from './concept-inventory.mjs';
|
||||
|
||||
// Activity-1 Task 3. The review dialog is compared with the concept's own `scan-review` ("Review metadata scan",
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
|
||||
test.beforeEach(async ({page, browserName}) => {
|
||||
@@ -57,7 +57,8 @@ test('Classic keeps immediate selection, search query and metadata serialization
|
||||
});
|
||||
|
||||
test('Classic detaches native text synchronization on removal',async({page})=>{
|
||||
await page.waitForTimeout(60);
|
||||
// The page installs the text sync (serializeValue and its listeners) in a 50 ms timer; wait for it, not for time.
|
||||
await expect.poll(()=>page.evaluate(()=>typeof node.widgets.find(item=>item.name==='text').serializeValue)).toBe('function');
|
||||
const values=await page.evaluate(()=>{
|
||||
const widget=node.widgets.find(item=>item.name==='text');
|
||||
widget.inputEl.value='Before removal';widget.inputEl.dispatchEvent(new Event('input'));
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// same concept selectors in the immutable concept (/__test__/canonical/workspace) and in production, which now renders
|
||||
// the concept's classes. The concept is never rewritten: each page's map<Page>Concept(reference) applies only the
|
||||
// recorded spec-§8 deviations, each commented with its D-number, and diffInventory must then report nothing.
|
||||
import {expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {expect} from './fixtures.mjs';
|
||||
export {diffInventory} from './library-editor-canonical-comparison.mjs';
|
||||
|
||||
export const CONCEPT_URL='/__test__/canonical/workspace';
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
// Shared `test` for the page workflows: every Playwright worker owns one
|
||||
// tests/browser/server.py process on a port the OS chose for it, with its own temp root
|
||||
// and database. `baseURL` (and so `page`, `request` and `browser.newContext`)
|
||||
// points at that worker's server. A server that dies fails the running test
|
||||
// with its stderr tail, and the next test in the worker gets a fresh server.
|
||||
// A test that fails gets the stderr its server wrote during it attached as
|
||||
// `server-stderr`.
|
||||
import { test as base, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { attachServerStderr, createServerSupervisor, startBrowserServer } from '../../tools/vendor/start-browser-server.mjs';
|
||||
|
||||
const SERVER_FIXTURE_TIMEOUT_MS = 90_000;
|
||||
|
||||
export const test = base.extend({
|
||||
testServer: [async ({}, use) => {
|
||||
const supervisor = createServerSupervisor({ start: () => startBrowserServer() });
|
||||
try {
|
||||
await use(supervisor);
|
||||
} finally {
|
||||
await supervisor.stop();
|
||||
}
|
||||
}, { scope: 'worker', timeout: SERVER_FIXTURE_TIMEOUT_MS }],
|
||||
workerServer: [async ({ testServer }, use, testInfo) => {
|
||||
const server = await testServer.ensure();
|
||||
const mark = server.stderrMark();
|
||||
await use(server);
|
||||
await attachServerStderr(testInfo, server, mark);
|
||||
const crash = testServer.crash(server);
|
||||
if (crash) throw crash;
|
||||
}, { auto: true, timeout: SERVER_FIXTURE_TIMEOUT_MS }],
|
||||
baseURL: async ({ workerServer }, use) => use(workerServer.url),
|
||||
});
|
||||
|
||||
export { expect };
|
||||
@@ -1,5 +1,5 @@
|
||||
import {readFileSync} from 'node:fs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
|
||||
for (const variant of ['Focused', 'Split']) {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept, openConcept, openProduction, SIZES } from './concept-inventory.mjs';
|
||||
// D10/D20: the heading is the linked positive's title (its confirmed first line here), never the pair's
|
||||
// "Inspector recipe", which names the negative variant.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { writeFile } from 'node:fs/promises';
|
||||
import { expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { expect } from './fixtures.mjs';
|
||||
|
||||
// Content, row counts, and image bytes come from independent synthetic datasets.
|
||||
// Saved choices and actual executions are separate production sections. The concept's
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,diffInventory,mutate,stripExact} from './concept-inventory.mjs';
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,diffInventory,mutate,stripExact,countCopy,reflow,withoutItems} from './concept-inventory.mjs';
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {SIZES,THEMES,seedConcept,openConcept,openProduction,inspectorOf,diffInventory,mutate,countCopy} from './concept-inventory.mjs';
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,diffInventory,mutate,stripExact,countCopy,reflow} from './concept-inventory.mjs';
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
|
||||
// Artifact-only captures of the editor states that have no concept counterpart (archive confirmation, Show archived,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {readFileSync} from 'node:fs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
|
||||
const pair={id:4,positive_text:'Library positive',negative_text:'Library negative',category:'Portrait',tags:['studio']};
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
|
||||
const moduleUrl = '/prompt_manager/js/shared/toolkit/prompt-bridge.js';
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
|
||||
const workspace='/prompt_manager/workspace#/images';
|
||||
const details=page=>page.getByRole('complementary',{name:'Selected image details',exact:true});
|
||||
|
||||
+73
-6
@@ -1,16 +1,20 @@
|
||||
"""Serve production pages and API routes with disposable state for browser tests."""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import types
|
||||
|
||||
from aiohttp import web
|
||||
from aiohttp.web_runner import GracefulExit
|
||||
from PIL import Image
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
|
||||
@@ -515,14 +519,77 @@ def create_app(root: Path) -> web.Application:
|
||||
return app
|
||||
|
||||
|
||||
def main() -> None:
|
||||
HOST = "127.0.0.1"
|
||||
PORT_ANNOUNCEMENT = "PM_TEST_SERVER_PORT="
|
||||
|
||||
|
||||
def bind_listener(port: int) -> socket.socket:
|
||||
"""Bind the server's socket before the app is built.
|
||||
|
||||
``port`` 0 lets the OS choose a free port, which this process then holds,
|
||||
so parallel launchers never race for one between choosing and binding.
|
||||
"""
|
||||
listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
try:
|
||||
if port and os.name != "nt":
|
||||
# aiohttp's own default on POSIX; on Windows it would let a
|
||||
# second process share the port.
|
||||
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
listener.bind((HOST, port))
|
||||
listener.listen(128)
|
||||
except OSError:
|
||||
listener.close()
|
||||
raise
|
||||
return listener
|
||||
|
||||
|
||||
def _graceful_exit() -> None:
|
||||
raise GracefulExit()
|
||||
|
||||
|
||||
def exit_when_stdin_closes(loop: asyncio.AbstractEventLoop) -> threading.Thread:
|
||||
"""Stop the app gracefully once stdin reaches end of file.
|
||||
|
||||
The launcher keeps the write end of this process's stdin pipe, and the OS
|
||||
closes it however the launcher dies, SIGKILL included. GracefulExit runs
|
||||
aiohttp's cleanup (closing the database) and lets the temp root be removed,
|
||||
on every OS, where a signal from a dead parent would not.
|
||||
"""
|
||||
|
||||
def watch() -> None:
|
||||
while sys.stdin.buffer.read(65536):
|
||||
pass
|
||||
loop.call_soon_threadsafe(_graceful_exit)
|
||||
|
||||
thread = threading.Thread(target=watch, name="stdin-eof", daemon=True)
|
||||
thread.start()
|
||||
return thread
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--port", type=int, default=8765)
|
||||
args = parser.parse_args()
|
||||
parser.add_argument(
|
||||
"--port", type=int, default=8765, help="0 binds an OS-chosen free port"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--exit-on-stdin-eof",
|
||||
action="store_true",
|
||||
help="shut down when stdin closes (the launcher's parent-death guard)",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
listener = bind_listener(args.port)
|
||||
# One machine-readable line tells the launcher which port this process holds.
|
||||
print(f"{PORT_ANNOUNCEMENT}{listener.getsockname()[1]}", flush=True)
|
||||
loop = asyncio.new_event_loop()
|
||||
if args.exit_on_stdin_eof:
|
||||
exit_when_stdin_closes(loop)
|
||||
original_cwd = os.getcwd()
|
||||
with tempfile.TemporaryDirectory(prefix="pm-browser-") as directory:
|
||||
web.run_app(
|
||||
create_app(Path(directory).resolve()), host="127.0.0.1", port=args.port
|
||||
)
|
||||
try:
|
||||
web.run_app(create_app(Path(directory).resolve()), sock=listener, loop=loop)
|
||||
finally:
|
||||
# create_app works inside the root; Windows cannot remove a process's cwd.
|
||||
os.chdir(original_cwd)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept as openConceptPage,openProduction,inventoryOf,diffInventory,focusRingOf,hoverStateOf,mutate} from './concept-inventory.mjs';
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
|
||||
// Artifact-only captures of Tags & categories states that have no concept counterpart: the Manage categories
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
// Each Playwright worker owns its server: `request`, `page` and manual
|
||||
// `browser.newContext()` all reach it through `baseURL`, and another server
|
||||
// process never sees its database.
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { startBrowserServer } from '../../tools/vendor/start-browser-server.mjs';
|
||||
|
||||
const search = text => `/prompt_manager/search?text=${encodeURIComponent(text)}`;
|
||||
|
||||
test('every fixture reaches this worker\'s own server and another server keeps its own database', async ({ request, page, browser, baseURL, workerServer }, testInfo) => {
|
||||
expect(baseURL).toBe(workerServer.url);
|
||||
await expect(await request.post('/__test__/reset')).toBeOK();
|
||||
const marker = `worker ${testInfo.workerIndex} isolation ${testInfo.project.name} ${testInfo.repeatEachIndex}`;
|
||||
await expect(await request.post('/prompt_manager/save', { data: { text: marker } })).toBeOK();
|
||||
expect((await (await request.get(search(marker))).json()).count).toBe(1);
|
||||
|
||||
const viaPage = await page.request.get(search(marker));
|
||||
expect(viaPage.url().startsWith(`${baseURL}/`)).toBe(true);
|
||||
expect((await viaPage.json()).count).toBe(1);
|
||||
const context = await browser.newContext();
|
||||
try {
|
||||
const response = await (await context.newPage()).goto(search(marker));
|
||||
expect(response.url().startsWith(`${baseURL}/`)).toBe(true);
|
||||
expect((await response.json()).count).toBe(1);
|
||||
} finally {
|
||||
await context.close();
|
||||
}
|
||||
|
||||
const other = await startBrowserServer();
|
||||
try {
|
||||
expect(other.url).not.toBe(baseURL);
|
||||
expect((await (await fetch(`${other.url}${search(marker)}`)).json()).count).toBe(0);
|
||||
} finally {
|
||||
await other.stop();
|
||||
}
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
|
||||
test.beforeEach(async ({ request, page, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
@@ -64,11 +64,16 @@ test('actual Tags page reloads the selected filter and opens its image viewer',
|
||||
await page.goto('/prompt_manager/admin#/tags');
|
||||
await page.locator('[data-tag-name="cats"]').click();
|
||||
await expect(page.locator('#tagPromptsGrid')).toContainText('Cat portrait');
|
||||
const dogs = page.waitForResponse(response => response.url().includes('/tags/dogs/prompts?'));
|
||||
await page.locator('[data-tag-name="dogs"]').click();
|
||||
const [shown] = (await (await dogs).json()).prompts[0].images;
|
||||
await expect(page.locator('#tagPromptsGrid')).toContainText('Dog landscape');
|
||||
await expect(page.locator('#tagPromptsGrid')).not.toContainText('Cat portrait');
|
||||
await page.locator('#tagPromptsGrid img').first().click();
|
||||
await expect(page.locator('.viewer-canvas img')).toBeVisible();
|
||||
// The viewer's title names the shown image through the page's title callback. ViewerJS renders it only once the
|
||||
// image is viewed, so closing at "visible" alone left the callback unrun on slower hosts (CI coverage missed it).
|
||||
await expect(page.locator('.viewer-title')).toHaveText(shown.filename);
|
||||
await page.locator('.viewer-container').getByRole('button', { name: 'Close', exact: true }).click();
|
||||
await expect(page.locator('.viewer-container')).toHaveCount(0);
|
||||
});
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept, openConcept, openProduction } from './concept-inventory.mjs';
|
||||
|
||||
// Activity (#249 Activity-1): the concept's Activity page (app.js renderActivity 732–738), its top-bar button
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {readFile,writeFile} from 'node:fs/promises';
|
||||
import {createHash} from 'node:crypto';
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept, openConcept, openProduction } from './concept-inventory.mjs';
|
||||
|
||||
const workspace = '/prompt_manager/workspace#/images';
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {inventoryOf,mapConcept,diffInventory} from './library-editor-canonical-comparison.mjs';
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
|
||||
const route='/prompt_manager/workspace#/library';
|
||||
const cards=page=>page.locator('article[data-prompt-id]');
|
||||
@@ -25,6 +25,8 @@ async function loadShelfPages(page,more,pages) {
|
||||
await more.evaluate(node=>node.click());await response;await expect(more).toHaveAttribute('aria-disabled','false');
|
||||
}
|
||||
}
|
||||
/** Resolves in the rendering frame after every frame callback the page had already queued (callbacks run in request order). */
|
||||
const nextFrame=page=>page.evaluate(()=>new Promise(resolve=>requestAnimationFrame(()=>resolve())));
|
||||
async function openHistory(page,text='Cat portrait') {
|
||||
await cards(page).filter({has:page.getByText(text,{exact:true})}).getByRole('button',{name:'Explore history',exact:true}).click();
|
||||
await expect(history(page)).toBeVisible();
|
||||
@@ -103,7 +105,8 @@ test('library infinite loading keeps unique parents and keyboard-accessible hist
|
||||
const appended=await (await nextPage).json();expect(appended.items.length).toBeGreaterThan(0);hasMore=appended.has_more;
|
||||
if(hasMore)await expect(more).toHaveAttribute('aria-disabled','false');
|
||||
}
|
||||
await expect(more).toBeHidden();expect(await cards(page).count()).toBeLessThan(80);
|
||||
// The last page's render queued the window update for the next frame; count after that frame, not before it.
|
||||
await expect(more).toBeHidden();await nextFrame(page);expect(await cards(page).count()).toBeLessThan(80);
|
||||
const ids=await cards(page).evaluateAll(nodes=>nodes.map(node=>node.dataset.promptId));expect(new Set(ids).size).toBe(ids.length);
|
||||
await expect(page.getByRole('button',{name:/next page|previous page/i})).toHaveCount(0);
|
||||
const firstId=await cards(page).first().getAttribute('data-prompt-id');
|
||||
@@ -376,14 +379,18 @@ async function observeLibrary(page, hold=false) {
|
||||
holdFrames(){holdingFrames=true;},
|
||||
releaseFrames(){holdingFrames=false;const callbacks=[...heldFrames.values()];heldFrames.clear();for(const callback of callbacks)callback();},
|
||||
equal(){for(const record of records){const entries=[...record.latest.values()].filter(entry=>record.targets.has(entry.target));if(entries.length)record.callback(entries,record.native);}},
|
||||
// Resolves once two consecutive rendering frames find no Library frame queued. Each check runs inside a native
|
||||
// frame callback, so a frame the page queues later in that update (a scroll event, a ResizeObserver delivery) is
|
||||
// still pending there. A timer is not a frame: under CPU load a 60 ms wait could end before the scroll's frame.
|
||||
// A frame that never drains (a rAF loop, or settling while frames are held) fails after `limit` frames with the stats.
|
||||
settled(limit=600){return new Promise((resolve,reject)=>{let quiet=0,frames=0;const check=()=>{frames++;quiet=pending.size?0:quiet+1;
|
||||
if(quiet>=2)resolve();else if(frames>=limit)reject(new Error(`Library never settled within ${limit} frames: ${JSON.stringify(window.libraryObservation.stats())}`));else raf(check);};raf(check);});},
|
||||
stats(){return {requested,pending:pending.size,heldFrames:heldFrames.size,measurements:records.reduce((sum,record)=>sum+record.entries.size,0),mutations,unmounted,targets:records.reduce((sum,record)=>sum+record.targets.size,0)};},
|
||||
};
|
||||
},{hold});
|
||||
}
|
||||
async function settleLibrary(page) {
|
||||
await expect.poll(async()=>page.evaluate(()=>window.libraryObservation.stats().pending)).toBe(0);
|
||||
await page.evaluate(()=>new Promise(resolve=>setTimeout(resolve,60)));
|
||||
await expect.poll(async()=>page.evaluate(()=>window.libraryObservation.stats().pending)).toBe(0);
|
||||
await page.evaluate(()=>window.libraryObservation.settled());
|
||||
}
|
||||
async function firstVisibleParent(page) {
|
||||
return cards(page).evaluateAll(nodes=>{
|
||||
@@ -683,6 +690,17 @@ test('title and Explore history are buttons that open history',async({page,reque
|
||||
await page.keyboard.press('ArrowLeft');await expect(cards(page).nth(1).locator('.card-title > button')).toBeFocused();
|
||||
});
|
||||
|
||||
test('settling fails with the observation stats when a Library frame stays queued',async({page,request})=>{
|
||||
await request.post('/__test__/reset',{data:{native_library:true,extra_prompts:200}});
|
||||
await observeLibrary(page);await page.addInitScript(()=>{window.IntersectionObserver=undefined;});
|
||||
await page.goto(route);await expect(cards(page).first()).toBeVisible();await settleLibrary(page);
|
||||
await page.evaluate(()=>{window.libraryObservation.holdFrames();dispatchEvent(new Event('scroll'));});
|
||||
await expect.poll(()=>page.evaluate(()=>window.libraryObservation.stats().heldFrames)).toBeGreaterThan(0);
|
||||
const failure=await page.evaluate(()=>window.libraryObservation.settled(30).then(()=>'settled',error=>error.message));
|
||||
expect(failure).toMatch(/^Library never settled within 30 frames: \{.*"heldFrames":[1-9]/);
|
||||
await page.evaluate(()=>window.libraryObservation.releaseFrames());await settleLibrary(page);
|
||||
});
|
||||
|
||||
test('virtualized rows keep measured heights with the clamp',async({page,request,isMobile})=>{
|
||||
await request.post('/__test__/reset',{data:{native_library:true,extra_prompts:200}});
|
||||
await observeLibrary(page);await page.addInitScript(()=>{window.IntersectionObserver=undefined;});
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
|
||||
// Saved views (#249 D16, maintainer ruling 2026-10-21): the toolbar's Save view (concept app.js a359, a1330–1346),
|
||||
// the sidebar SAVED VIEWS rows (a331–335) and `#/saved/<id>` (a1158–1162), on Images and the Prompt library.
|
||||
|
||||
@@ -3,7 +3,7 @@ import { readFile } from 'node:fs/promises';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { captureCanonicalState, historyGeometry, compareCanonicalHistory, nativeDialogState, exerciseCurrentPairSend, assertRecordedHistory } from './history-canonical-comparison.mjs';
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
|
||||
test.use({ locale: 'en-US', timezoneId: 'UTC', reducedMotion: 'reduce' });
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {pageInventoryOf,inventoryOf,diffInventory,mapVocabularyConcept,mapVocabularyDialog} from './vocabulary-canonical-comparison.mjs';
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
const statistic = (page, label) => page.locator('#workspace-main dt')
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
"""The page-workflow test server ends with its launcher (#274 review M2).
|
||||
|
||||
A Playwright worker that is SIGKILLed never runs its fixture teardown, so the
|
||||
server it started must notice on its own: with ``--exit-on-stdin-eof`` it shuts
|
||||
down when the launcher's end of its stdin pipe closes, and still removes its
|
||||
temp root. This runs on every OS in CI, where Windows is the strict case.
|
||||
"""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
SERVER_TIMEOUT_S = 60
|
||||
|
||||
|
||||
class TestBrowserServerLifetime(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
|
||||
def start(self, *extra):
|
||||
env = {**os.environ, "TMPDIR": self.tmp.name}
|
||||
env.update(TEMP=self.tmp.name, TMP=self.tmp.name)
|
||||
process = subprocess.Popen(
|
||||
[sys.executable, "-m", "tests.browser.server", "--port", "0", *extra],
|
||||
cwd=REPO_ROOT,
|
||||
env=env,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
|
||||
def stop():
|
||||
if process.poll() is None:
|
||||
process.kill()
|
||||
process.wait(timeout=SERVER_TIMEOUT_S)
|
||||
# communicate() would flush a closed stdin, which Python < 3.13 refuses.
|
||||
for stream in (process.stdin, process.stdout, process.stderr):
|
||||
stream.close()
|
||||
|
||||
self.addCleanup(stop)
|
||||
line = process.stdout.readline().decode()
|
||||
self.assertTrue(line.startswith("PM_TEST_SERVER_PORT="), line)
|
||||
url = f"http://127.0.0.1:{int(line.split('=', 1)[1])}/prompt_manager/test"
|
||||
deadline = time.monotonic() + SERVER_TIMEOUT_S
|
||||
while True:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=1) as response: # nosec B310
|
||||
self.assertEqual(response.status, 200)
|
||||
return process
|
||||
except OSError:
|
||||
self.assertIsNone(process.poll(), "the server exited early")
|
||||
self.assertLess(time.monotonic(), deadline, "never ready")
|
||||
time.sleep(0.1)
|
||||
|
||||
def roots(self):
|
||||
return sorted(path.name for path in Path(self.tmp.name).glob("pm-browser-*"))
|
||||
|
||||
def test_server_exits_and_removes_its_temp_root_when_stdin_closes(self):
|
||||
process = self.start("--exit-on-stdin-eof")
|
||||
self.assertEqual(len(self.roots()), 1, "the server works in its own temp root")
|
||||
process.stdin.close()
|
||||
code = process.wait(timeout=SERVER_TIMEOUT_S)
|
||||
self.assertEqual(
|
||||
code, 0, process.stderr.read().decode(errors="replace")[-4000:]
|
||||
)
|
||||
self.assertEqual(self.roots(), [], "the temp root and its database are removed")
|
||||
|
||||
def test_server_keeps_running_with_stdin_closed_without_the_flag(self):
|
||||
process = self.start()
|
||||
process.stdin.close()
|
||||
with self.assertRaises(subprocess.TimeoutExpired):
|
||||
process.wait(timeout=1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+234
@@ -0,0 +1,234 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createServer } from 'node:net';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { once } from 'node:events';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
import { startBrowserServer, createServerSupervisor, attachServerStderr } from '../../tools/vendor/start-browser-server.mjs';
|
||||
import { workflowWorkers } from '../../tools/vendor/test-workers.mjs';
|
||||
|
||||
const fakeServer = fileURLToPath(new URL('./fake-browser-server.mjs', import.meta.url));
|
||||
const fake = (...extra) => ({ command: process.execPath, args: port => [fakeServer, '--port', String(port), ...extra] });
|
||||
const REAL_SERVER_TIMEOUT_MS = 90_000;
|
||||
const launcher = fileURLToPath(new URL('../../tools/vendor/start-browser-server.mjs', import.meta.url));
|
||||
|
||||
/** Polls `check` until it is truthy, failing with `what` after `ms`. */
|
||||
async function waitFor(check, what, ms = 5_000) {
|
||||
const deadline = Date.now() + ms;
|
||||
while (!check()) {
|
||||
if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}`);
|
||||
await delay(20);
|
||||
}
|
||||
}
|
||||
|
||||
async function writeStderr(server, text) {
|
||||
await (await fetch(`${server.url}/__stderr?text=${encodeURIComponent(text)}`)).text();
|
||||
await waitFor(() => server.stderrTail().endsWith(text.trimEnd()), `stderr ${JSON.stringify(text)}`);
|
||||
}
|
||||
|
||||
/** A process that exited counts as gone even while a non-reaping PID 1 (CI's container) leaves it a zombie. */
|
||||
function alive(pid) {
|
||||
try { process.kill(pid, 0); } catch { return false; }
|
||||
try { return readFileSync(`/proc/${pid}/stat`, 'utf8').replace(/^.*\) /s, '')[0] !== 'Z'; } catch { return true; }
|
||||
}
|
||||
|
||||
/** Starts a server from a separate Node worker process, SIGKILLs that worker and returns the server's pid. */
|
||||
async function serverOfKilledWorker(options) {
|
||||
const script = `import { startBrowserServer } from ${JSON.stringify(pathToFileURL(launcher).href)};
|
||||
const server = await startBrowserServer(${options});
|
||||
console.log('SERVER_PID=' + server.pid);
|
||||
setInterval(() => {}, 1000);`;
|
||||
const worker = spawn(process.execPath, ['--input-type=module', '-e', script], { stdio: ['ignore', 'pipe', 'inherit'] });
|
||||
let out = '';
|
||||
worker.stdout.setEncoding('utf8');
|
||||
worker.stdout.on('data', chunk => { out += chunk; });
|
||||
await waitFor(() => /SERVER_PID=\d+\n/.test(out) || worker.exitCode !== null, 'the worker to start its server', REAL_SERVER_TIMEOUT_MS);
|
||||
const pid = Number(/SERVER_PID=(\d+)/.exec(out)?.[1]);
|
||||
assert.ok(pid > 0, `the worker started a server (output: ${out})`);
|
||||
assert.ok(alive(pid), 'the server runs while its worker lives');
|
||||
worker.kill('SIGKILL');
|
||||
await once(worker, 'exit');
|
||||
return pid;
|
||||
}
|
||||
|
||||
function listen(port = 0) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const server = createServer();
|
||||
server.once('error', reject);
|
||||
server.listen(port, '127.0.0.1', () => resolve(server));
|
||||
});
|
||||
}
|
||||
|
||||
async function releasedPort() {
|
||||
const probe = await listen();
|
||||
const { port } = probe.address();
|
||||
await new Promise(resolve => probe.close(resolve));
|
||||
return port;
|
||||
}
|
||||
|
||||
test('free port allocation never reuses a bound port', async t => {
|
||||
const held = await listen();
|
||||
t.after(() => held.close());
|
||||
const heldPort = held.address().port;
|
||||
const servers = await Promise.all(Array.from({ length: 6 }, () => startBrowserServer(fake())));
|
||||
t.after(() => Promise.all(servers.map(server => server.stop())));
|
||||
const ports = servers.map(server => server.port);
|
||||
for (const server of servers) {
|
||||
assert.ok(Number.isInteger(server.port) && server.port > 0 && server.port < 65536, `port ${server.port} is a TCP port`);
|
||||
assert.equal(server.url, `http://127.0.0.1:${server.port}`);
|
||||
assert.equal(await (await fetch(`${server.url}/prompt_manager/test`)).text(), 'ready');
|
||||
}
|
||||
assert.ok(!ports.includes(heldPort), 'a port another process holds is never handed out');
|
||||
assert.equal(new Set(ports).size, ports.length, 'concurrent servers never share a port');
|
||||
});
|
||||
|
||||
test('startBrowserServer asks the server to bind port 0 and reads the port it announces', async () => {
|
||||
const asked = [];
|
||||
const server = await startBrowserServer({ command: process.execPath, args: port => { asked.push(port); return fake().args(port); } });
|
||||
try {
|
||||
assert.deepEqual(asked, [0], 'no port is reserved and released before the server binds');
|
||||
assert.ok(server.port > 0);
|
||||
} finally {
|
||||
await server.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test('startBrowserServer serves the readiness page on a requested port and stops cleanly', async () => {
|
||||
const port = await releasedPort();
|
||||
const server = await startBrowserServer({ port, ...fake() });
|
||||
assert.equal(server.port, port);
|
||||
assert.equal(server.url, `http://127.0.0.1:${port}`);
|
||||
assert.equal(await (await fetch(`${server.url}/prompt_manager/test`)).text(), 'ready');
|
||||
await server.stop();
|
||||
await assert.rejects(fetch(`${server.url}/prompt_manager/test`));
|
||||
});
|
||||
|
||||
test('startBrowserServer reports a startup failure with the server\'s stderr tail', async () => {
|
||||
await assert.rejects(startBrowserServer(fake('--fail')), error => {
|
||||
assert.match(error.message, /exited with code 4 before it was ready/);
|
||||
assert.match(error.message, /last fake stderr line/);
|
||||
return true;
|
||||
});
|
||||
});
|
||||
|
||||
test('startBrowserServer gives up on a server that never announces a port or becomes ready', async () => {
|
||||
const silent = { command: process.execPath, args: () => ['-e', 'setInterval(() => {}, 1000)'] };
|
||||
await assert.rejects(startBrowserServer({ ...silent, timeoutMs: 500 }), /not ready .* within 500 ms/);
|
||||
});
|
||||
|
||||
test('worker server restarts after an unexpected exit', async t => {
|
||||
const supervisor = createServerSupervisor({ start: () => startBrowserServer(fake()) });
|
||||
t.after(() => supervisor.stop());
|
||||
const first = await supervisor.ensure();
|
||||
assert.equal(supervisor.crash(first), null, 'a running server reports no crash');
|
||||
await fetch(`${first.url}/__exit`);
|
||||
await first.exited;
|
||||
const crash = supervisor.crash(first);
|
||||
assert.match(crash.message, /test server on port \d+ exited unexpectedly \(code 3\)/);
|
||||
assert.match(crash.message, /crashed on purpose/);
|
||||
const second = await supervisor.ensure();
|
||||
assert.notEqual(second, first, 'the next test gets a fresh server');
|
||||
assert.equal(await (await fetch(`${second.url}/prompt_manager/test`)).text(), 'ready');
|
||||
assert.equal(await supervisor.ensure(), second, 'a healthy server is reused');
|
||||
});
|
||||
|
||||
test('supervisor reports a startup failure once and tries again on the next ensure', async t => {
|
||||
let attempts = 0;
|
||||
const supervisor = createServerSupervisor({
|
||||
start: async () => {
|
||||
attempts += 1;
|
||||
if (attempts === 1) throw new Error('ImportError: no module');
|
||||
return startBrowserServer(fake());
|
||||
},
|
||||
});
|
||||
t.after(() => supervisor.stop());
|
||||
await assert.rejects(supervisor.ensure(), /ImportError/);
|
||||
assert.equal(attempts, 1, 'a failed startup is not retried behind the caller\'s back');
|
||||
const server = await supervisor.ensure();
|
||||
assert.equal(attempts, 2);
|
||||
assert.equal(await (await fetch(`${server.url}/prompt_manager/test`)).text(), 'ready');
|
||||
});
|
||||
|
||||
test('the port announcement counts only once its line is complete', async t => {
|
||||
const server = await startBrowserServer({ ...fake('--split-announce'), timeoutMs: 5_000 });
|
||||
t.after(() => server.stop());
|
||||
assert.equal(await (await fetch(`${server.url}/prompt_manager/test`)).text(), 'ready');
|
||||
assert.equal(server.url, `http://127.0.0.1:${server.port}`);
|
||||
});
|
||||
|
||||
test('server stderr since a mark holds only what came after it, bounded to the newest output', async t => {
|
||||
const server = await startBrowserServer({ ...fake(), stderrLogChars: 64 });
|
||||
t.after(() => server.stop());
|
||||
await writeStderr(server, 'earlier test\n');
|
||||
const mark = server.stderrMark();
|
||||
assert.equal(server.stderrSince(mark), '');
|
||||
await writeStderr(server, 'Traceback: this test\n');
|
||||
assert.equal(server.stderrSince(mark), 'Traceback: this test\n');
|
||||
const flood = `${'x'.repeat(80)}newest\n`;
|
||||
await writeStderr(server, flood);
|
||||
const kept = server.stderrSince(mark);
|
||||
assert.match(kept, /^\[earlier server stderr dropped\]\n/);
|
||||
assert.ok(kept.endsWith('newest\n'), 'the newest output is kept');
|
||||
assert.ok(kept.length <= 64 + '[earlier server stderr dropped]\n'.length, `bounded, got ${kept.length} chars`);
|
||||
});
|
||||
|
||||
test('a failed test gets the server stderr written during it attached; a passing one does not', async t => {
|
||||
const server = await startBrowserServer(fake());
|
||||
t.after(() => server.stop());
|
||||
const attached = [];
|
||||
const info = status => ({ status, expectedStatus: 'passed', attach: async (name, options) => { attached.push({ name, ...options }); } });
|
||||
await writeStderr(server, 'before the test\n');
|
||||
const mark = server.stderrMark();
|
||||
await attachServerStderr(info('failed'), server, mark);
|
||||
assert.deepEqual(attached, [], 'nothing to attach when the server wrote nothing');
|
||||
await writeStderr(server, 'Traceback (most recent call last): boom\n');
|
||||
await attachServerStderr(info('passed'), server, mark);
|
||||
assert.deepEqual(attached, [], 'a passing test attaches nothing');
|
||||
await attachServerStderr(info('timedOut'), server, mark);
|
||||
assert.equal(attached.length, 1);
|
||||
assert.equal(attached[0].name, 'server-stderr');
|
||||
assert.equal(attached[0].contentType, 'text/plain');
|
||||
assert.equal(String(attached[0].body), 'Traceback (most recent call last): boom\n');
|
||||
});
|
||||
|
||||
test('a server exits with its worker even when the worker is SIGKILLed', async () => {
|
||||
const pid = await serverOfKilledWorker(`{ command: process.execPath, args: port => [${JSON.stringify(fakeServer)}, '--port', String(port), '--exit-on-stdin-eof'] }`);
|
||||
await waitFor(() => !alive(pid), `server ${pid} to exit after its worker died`);
|
||||
});
|
||||
|
||||
test('tests/browser/server.py exits with a SIGKILLed worker', { timeout: 2 * REAL_SERVER_TIMEOUT_MS }, async () => {
|
||||
const pid = await serverOfKilledWorker(`{ timeoutMs: ${REAL_SERVER_TIMEOUT_MS} }`);
|
||||
await waitFor(() => !alive(pid), `server.py ${pid} to exit after its worker died`, 15_000);
|
||||
});
|
||||
|
||||
test('tests/browser/server.py binds an OS-chosen port for --port 0 and still honours --port N', { timeout: 2 * REAL_SERVER_TIMEOUT_MS }, async t => {
|
||||
const held = await listen();
|
||||
t.after(() => held.close());
|
||||
const chosen = await startBrowserServer({ timeoutMs: REAL_SERVER_TIMEOUT_MS });
|
||||
try {
|
||||
assert.notEqual(chosen.port, held.address().port);
|
||||
assert.equal((await fetch(`${chosen.url}/prompt_manager/test`)).status, 200);
|
||||
} finally {
|
||||
await chosen.stop();
|
||||
}
|
||||
const port = await releasedPort();
|
||||
const fixed = await startBrowserServer({ port, timeoutMs: REAL_SERVER_TIMEOUT_MS });
|
||||
try {
|
||||
assert.equal(fixed.port, port);
|
||||
assert.equal((await fetch(`http://127.0.0.1:${port}/prompt_manager/test`)).status, 200);
|
||||
} finally {
|
||||
await fixed.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test('worker count follows PM_TEST_WORKERS, then one in CI, then a quarter of the CPUs up to eight', () => {
|
||||
assert.equal(workflowWorkers({ env: { PM_TEST_WORKERS: '3', CI: '1' }, cpus: 32 }), 3);
|
||||
assert.equal(workflowWorkers({ env: { CI: '1' }, cpus: 32 }), 1);
|
||||
assert.equal(workflowWorkers({ env: {}, cpus: 32 }), 8);
|
||||
assert.equal(workflowWorkers({ env: {}, cpus: 12 }), 3);
|
||||
assert.equal(workflowWorkers({ env: {}, cpus: 2 }), 1);
|
||||
assert.throws(() => workflowWorkers({ env: { PM_TEST_WORKERS: 'many' }, cpus: 8 }), /PM_TEST_WORKERS must be a positive integer/);
|
||||
assert.throws(() => workflowWorkers({ env: { PM_TEST_WORKERS: '0' }, cpus: 8 }), /PM_TEST_WORKERS must be a positive integer/);
|
||||
});
|
||||
Vendored
+223
@@ -0,0 +1,223 @@
|
||||
// Structural checks on .github/workflows/viewerjs-check.yml. The tooling has no YAML parser, so these
|
||||
// read the workflow's fixed two-space layout: jobs at two spaces, their keys at four, steps at six.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { mkdtemp, mkdir, readdir, rm, writeFile } from 'node:fs/promises';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
const workflow = readFileSync(new URL('../../.github/workflows/viewerjs-check.yml', import.meta.url), 'utf8');
|
||||
const PINNED_IMAGE = /^\s+image: mcr\.microsoft\.com\/playwright:v\d+\.\d+\.\d+-noble$/m;
|
||||
|
||||
function jobs(text) {
|
||||
const lines = text.split('\n');
|
||||
const start = lines.indexOf('jobs:');
|
||||
assert.notEqual(start, -1, 'workflow has a jobs section');
|
||||
const found = new Map();
|
||||
let current = null;
|
||||
for (const line of lines.slice(start + 1)) {
|
||||
const header = /^ {2}([\w-]+):\s*$/.exec(line);
|
||||
if (header) {
|
||||
current = header[1];
|
||||
found.set(current, []);
|
||||
} else if (current && (line.startsWith(' ') || line.trim() === '' || /^ {2}#/.test(line))) {
|
||||
found.get(current).push(line);
|
||||
} else if (current && /^\S/.test(line)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return new Map([...found].map(([id, body]) => [id, body.join('\n')]));
|
||||
}
|
||||
|
||||
function steps(jobText) {
|
||||
const parts = jobText.split(/^(?= {6}- )/m);
|
||||
return parts.slice(1);
|
||||
}
|
||||
|
||||
function runScript(stepText) {
|
||||
const lines = stepText.split('\n');
|
||||
const index = lines.findIndex(line => /^ {8}run: \|\s*$/.test(line));
|
||||
assert.notEqual(index, -1, 'step has a block run script');
|
||||
const body = [];
|
||||
for (const line of lines.slice(index + 1)) {
|
||||
if (line.trim() !== '' && !line.startsWith(' ')) break;
|
||||
body.push(line.slice(10));
|
||||
}
|
||||
return body.join('\n');
|
||||
}
|
||||
|
||||
function needsOf(jobText) {
|
||||
const match = /^ {4}needs: \[([^\]]*)\]\s*$/m.exec(jobText) ?? /^ {4}needs: ([\w-]+)\s*$/m.exec(jobText);
|
||||
assert.ok(match, 'job declares needs');
|
||||
return match[1].split(',').map(value => value.trim()).filter(Boolean);
|
||||
}
|
||||
|
||||
function matrixList(jobText, key) {
|
||||
const match = new RegExp(`^ {8}${key}: \\[([^\\]]*)\\]\\s*$`, 'm').exec(jobText);
|
||||
assert.ok(match, `job has a ${key} matrix`);
|
||||
return match[1].split(',').map(value => value.trim());
|
||||
}
|
||||
|
||||
const allJobs = jobs(workflow);
|
||||
const browserJobIds = [...allJobs].filter(([, body]) => /npm run --prefix tools\/vendor test:(workflows|coverage)/.test(body))
|
||||
.map(([id]) => id);
|
||||
const mergeJob = () => allJobs.get('frontend-coverage-merge');
|
||||
const verifyStep = () => steps(mergeJob()).find(step => /name: Verify every shard's raw coverage arrived/.test(step));
|
||||
|
||||
test('viewerjs keeps its name and needs every frontend job', () => {
|
||||
const viewerjs = allJobs.get('viewerjs');
|
||||
assert.ok(viewerjs, 'the required context is the job id viewerjs');
|
||||
assert.doesNotMatch(viewerjs, /^ {4}name:/m, 'no display name may replace the context "viewerjs"');
|
||||
const needs = needsOf(viewerjs);
|
||||
const others = [...allJobs.keys()].filter(id => id !== 'viewerjs');
|
||||
assert.deepEqual([...needs].sort(), [...others].sort(), 'viewerjs needs every other job in the workflow');
|
||||
assert.deepEqual([...needs].sort(), ['browser-workflows', 'frontend-coverage', 'frontend-coverage-merge']);
|
||||
assert.match(viewerjs, /^ {4}if: always\(\)$/m, 'viewerjs runs (and fails) when a needed job failed or was skipped');
|
||||
const script = runScript(steps(viewerjs).find(step => /run: \|/.test(step)));
|
||||
for (const id of needs) {
|
||||
const variable = new RegExp(`^ {10}([A-Z_]+): \\$\\{\\{ needs\\.${id}\\.result \\}\\}$`, 'm').exec(viewerjs);
|
||||
assert.ok(variable, `${id}'s result reaches the check`);
|
||||
assert.match(script, new RegExp(`test "\\$${variable[1]}" = success`), `${id} must be exactly success`);
|
||||
}
|
||||
});
|
||||
|
||||
test('viewerjs fails unless every needed job succeeded', () => {
|
||||
const viewerjs = allJobs.get('viewerjs');
|
||||
const script = runScript(steps(viewerjs).find(step => /run: \|/.test(step)));
|
||||
const variables = [...viewerjs.matchAll(/^ {10}([A-Z_]+): \$\{\{ needs\.[\w-]+\.result \}\}$/gm)].map(match => match[1]);
|
||||
const run = env => {
|
||||
try {
|
||||
execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', script], { env: { PATH: process.env.PATH, ...env }, stdio: 'pipe' });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
const allSuccess = Object.fromEntries(variables.map(name => [name, 'success']));
|
||||
assert.equal(run(allSuccess), true);
|
||||
for (const name of variables) {
|
||||
for (const result of ['failure', 'cancelled', 'skipped']) {
|
||||
assert.equal(run({ ...allSuccess, [name]: result }), false, `${name}=${result} must fail viewerjs`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('coverage runs as three shards that each upload their raw coverage', () => {
|
||||
const coverage = allJobs.get('frontend-coverage');
|
||||
assert.deepEqual(matrixList(coverage, 'shard'), ['1', '2', '3']);
|
||||
assert.match(coverage, /^ {4}name: frontend coverage \(shard \$\{\{ matrix\.shard \}\}\/3\)$/m);
|
||||
assert.match(coverage, /npm run --prefix tools\/vendor test:coverage:shard -- \$\{\{ matrix\.shard \}\}\/3/);
|
||||
const upload = steps(coverage).find(step => /name: coverage-raw-\$\{\{ matrix\.shard \}\}/.test(step));
|
||||
assert.ok(upload, 'each shard uploads coverage-raw-N');
|
||||
assert.match(upload, /^ {10}path: coverage\/js\/raw\/?$/m);
|
||||
assert.match(upload, /^ {10}retention-days: 3$/m);
|
||||
assert.match(upload, /^ {10}if-no-files-found: error$/m);
|
||||
});
|
||||
|
||||
test('merge job downloads every shard and runs the merge-only report and check', () => {
|
||||
const merge = mergeJob();
|
||||
assert.ok(merge, 'frontend-coverage-merge exists');
|
||||
assert.deepEqual(needsOf(merge), ['frontend-coverage']);
|
||||
assert.match(merge, /^ {4}name: frontend coverage \(merge\)$/m);
|
||||
const download = steps(merge).find(step => /uses: actions\/download-artifact@/.test(step));
|
||||
assert.ok(download, 'merge downloads the shard artifacts');
|
||||
assert.match(download, /^ {10}pattern: coverage-raw-\*$/m);
|
||||
assert.doesNotMatch(download, /merge-multiple: true/, 'each artifact keeps its own directory so the check can count them');
|
||||
assert.ok(verifyStep(), 'merge has an explicit artifact count check');
|
||||
assert.match(verifyStep(), new RegExp(`^ {10}EXPECTED_SHARDS: ${matrixList(allJobs.get('frontend-coverage'), 'shard').length}$`, 'm'),
|
||||
'the expected count matches the shard matrix');
|
||||
const order = steps(merge).map(step => (/name: (.*)/.exec(step) ?? [, step.trim().split('\n')[0]])[1]);
|
||||
const verifyAt = order.findIndex(name => /Verify every shard/.test(name));
|
||||
const mergeAt = steps(merge).findIndex(step => /npm run --prefix tools\/vendor coverage:merge/.test(step));
|
||||
assert.ok(verifyAt >= 0 && mergeAt > verifyAt, 'the count check runs before coverage:merge');
|
||||
const report = steps(merge).find(step => /name: frontend-test-reports$/m.test(step));
|
||||
assert.ok(report, 'merge uploads the HTML report as frontend-test-reports');
|
||||
assert.match(report, /!coverage\/js\/raw\/\*\*/);
|
||||
});
|
||||
|
||||
async function stagedShards(root, shards) {
|
||||
const staged = join(root, 'staged');
|
||||
for (const [shard, files] of shards) {
|
||||
const directory = join(staged, `coverage-raw-${shard}`);
|
||||
await mkdir(directory, { recursive: true });
|
||||
for (const file of files) await writeFile(join(directory, file), '{"result":[]}');
|
||||
}
|
||||
await mkdir(staged, { recursive: true });
|
||||
return staged;
|
||||
}
|
||||
|
||||
async function verify(t, shards) {
|
||||
const root = await mkdtemp(join(tmpdir(), 'pm-ci-merge-'));
|
||||
t.after(() => rm(root, { recursive: true, force: true }));
|
||||
const staged = await stagedShards(root, shards);
|
||||
try {
|
||||
execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', runScript(verifyStep())], {
|
||||
cwd: root, stdio: 'pipe',
|
||||
env: { PATH: process.env.PATH, STAGED_RAW: staged, EXPECTED_SHARDS: '3' },
|
||||
});
|
||||
} catch (error) {
|
||||
return { ok: false, output: `${error.stdout}${error.stderr}` };
|
||||
}
|
||||
return { ok: true, raw: (await readdir(join(root, 'coverage/js/raw'))).sort() };
|
||||
}
|
||||
|
||||
test('merge refuses when a shard\'s raw coverage is missing', async t => {
|
||||
const missing = await verify(t, [[1, ['a.json']], [3, ['c.json']]]);
|
||||
assert.equal(missing.ok, false);
|
||||
assert.match(missing.output, /2 of 3/);
|
||||
const empty = await verify(t, [[1, ['a.json']], [2, []], [3, ['c.json']]]);
|
||||
assert.equal(empty.ok, false, 'an artifact without raw files counts as missing');
|
||||
const collision = await verify(t, [[1, ['a.json']], [2, ['a.json']], [3, ['c.json']]]);
|
||||
assert.equal(collision.ok, false, 'two shards writing the same file name must not silently drop one');
|
||||
const complete = await verify(t, [[1, ['a.json', 'node.json']], [2, ['b.json']], [3, ['c.json']]]);
|
||||
assert.deepEqual(complete, { ok: true, raw: ['a.json', 'b.json', 'c.json', 'node.json'] });
|
||||
});
|
||||
|
||||
test('every browser job sets PM_TEST_WORKERS and runs in the pinned image', () => {
|
||||
assert.deepEqual([...browserJobIds].sort(), ['browser-workflows', 'frontend-coverage']);
|
||||
for (const id of browserJobIds) {
|
||||
const body = allJobs.get(id);
|
||||
assert.match(body, /^ {6}PM_TEST_WORKERS: 3$/m, `${id} sets PM_TEST_WORKERS for the whole job`);
|
||||
assert.match(body, PINNED_IMAGE, `${id} keeps the pinned Playwright image`);
|
||||
assert.match(body, /PIP_SHA256: [0-9a-f]{64}/, `${id} keeps the pinned pip bootstrap`);
|
||||
assert.match(body, /^ {4}timeout-minutes: \d+$/m);
|
||||
}
|
||||
assert.match(mergeJob(), PINNED_IMAGE, 'the merge sees the same workspace paths as the shards');
|
||||
});
|
||||
|
||||
test('page workflows run each browser in two shards', () => {
|
||||
const workflows = allJobs.get('browser-workflows');
|
||||
assert.deepEqual(matrixList(workflows, 'browser'), ['firefox', 'webkit']);
|
||||
assert.deepEqual(matrixList(workflows, 'shard'), ['1', '2']);
|
||||
assert.match(workflows, /test:workflows -- --project \$\{\{ matrix\.browser \}\} --shard \$\{\{ matrix\.shard \}\}\/2/);
|
||||
assert.match(workflows, /^ {6}fail-fast: false$/m);
|
||||
assert.match(workflows, /name: browser-traces-\$\{\{ matrix\.browser \}\}-shard-\$\{\{ matrix\.shard \}\}/);
|
||||
});
|
||||
|
||||
test('every npm ci restores the npm download cache keyed on the vendor lockfile first', () => {
|
||||
const installing = [...allJobs].filter(([, body]) => /npm ci --prefix tools\/vendor/.test(body));
|
||||
assert.deepEqual(installing.map(([id]) => id).sort(), ['browser-workflows', 'frontend-coverage', 'frontend-coverage-merge']);
|
||||
for (const [id, body] of installing) {
|
||||
const jobSteps = steps(body);
|
||||
const cacheAt = jobSteps.findIndex(step => /uses: actions\/cache@v\d+/.test(step));
|
||||
const installAt = jobSteps.findIndex(step => /npm ci --prefix tools\/vendor/.test(step));
|
||||
assert.ok(cacheAt >= 0 && cacheAt < installAt, `${id} restores the npm cache before npm ci`);
|
||||
assert.match(jobSteps[cacheAt], /^ {10}path: ~\/\.npm$/m, `${id} caches npm's download cache, not node_modules`);
|
||||
assert.match(jobSteps[cacheAt], /hashFiles\('tools\/vendor\/package-lock\.json'\)/, `${id} keys the cache on the lockfile`);
|
||||
assert.match(jobSteps[installAt], /npm ci --prefix tools\/vendor --ignore-scripts/, `${id} still installs without scripts`);
|
||||
// The key names the job's own pinned image, so an image bump starts a fresh cache.
|
||||
const image = /^ {6}image: mcr\.microsoft\.com\/playwright:(v\d+\.\d+\.\d+)-noble$/m.exec(body)?.[1];
|
||||
assert.ok(image, `${id} runs in the pinned Playwright image`);
|
||||
assert.match(jobSteps[cacheAt], new RegExp(`key: npm-\\$\\{\\{ runner\\.os \\}\\}-playwright-${image.replaceAll('.', '\\.')}-\\$\\{\\{ hashFiles`),
|
||||
`${id} keys the cache on its image ${image}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('browser job timeouts keep a real margin over measured runs', () => {
|
||||
for (const id of ['frontend-coverage', 'browser-workflows']) {
|
||||
const minutes = Number(/^ {4}timeout-minutes: (\d+)$/m.exec(allJobs.get(id))[1]);
|
||||
assert.ok(minutes >= 20, `${id} allows at least 20 minutes, since runner speed varies`);
|
||||
}
|
||||
});
|
||||
Vendored
+44
@@ -0,0 +1,44 @@
|
||||
// A stand-in for tests/browser/server.py: `--port N` (0 = an OS-chosen port)
|
||||
// serves the readiness page and announces the bound port on stdout as
|
||||
// `PM_TEST_SERVER_PORT=<n>`, `--fail` writes to stderr and exits before
|
||||
// listening, and GET /__exit makes the process die unexpectedly after writing a
|
||||
// last stderr line. GET /__stderr?text=T writes T to stderr before answering.
|
||||
// `--split-announce` writes the announcement in two chunks, the first ending
|
||||
// mid-number. `--exit-on-stdin-eof` exits when its stdin closes, as server.py does.
|
||||
import { createServer } from 'node:http';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const port = Number(args[args.indexOf('--port') + 1]);
|
||||
if (args.includes('--fail')) {
|
||||
process.stderr.write('fake server: refusing to start\nlast fake stderr line\n');
|
||||
process.exit(4);
|
||||
}
|
||||
const server = createServer((request, response) => {
|
||||
const url = new URL(request.url, 'http://fake');
|
||||
if (url.pathname === '/__stderr') {
|
||||
process.stderr.write(url.searchParams.get('text'));
|
||||
response.end('written');
|
||||
return;
|
||||
}
|
||||
if (request.url === '/__exit') {
|
||||
process.stderr.write('fake server: crashed on purpose\n');
|
||||
response.end('bye', () => process.exit(3));
|
||||
return;
|
||||
}
|
||||
response.end(request.url === '/prompt_manager/test' ? 'ready' : `pid ${process.pid}`);
|
||||
});
|
||||
server.listen(port, '127.0.0.1', () => {
|
||||
const line = `PM_TEST_SERVER_PORT=${server.address().port}\n`;
|
||||
if (!args.includes('--split-announce')) {
|
||||
process.stdout.write(`fake server: listening\n${line}`);
|
||||
return;
|
||||
}
|
||||
// The first chunk is a complete-looking number, one digit short of the real port.
|
||||
process.stdout.write(`fake server: listening\n${line.slice(0, -2)}`);
|
||||
setTimeout(() => process.stdout.write(line.slice(-2)), 300);
|
||||
});
|
||||
if (args.includes('--exit-on-stdin-eof')) {
|
||||
process.stdin.resume();
|
||||
process.stdin.once('end', () => process.exit(0));
|
||||
}
|
||||
for (const signal of ['SIGTERM', 'SIGINT']) process.once(signal, () => server.close(() => process.exit(0)));
|
||||
+93
@@ -0,0 +1,93 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { existsSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { plan, runCoverage } from '../../tools/vendor/frontend-coverage.mjs';
|
||||
|
||||
const steps = argv => plan(argv).map(({ step, ...options }) => (Object.keys(options).length ? { step, ...options } : step));
|
||||
|
||||
function tempPaths(t) {
|
||||
const root = mkdtempSync(join(tmpdir(), 'pm-coverage-cli-'));
|
||||
t.after(() => rmSync(root, { recursive: true, force: true }));
|
||||
const reportRoot = join(root, 'coverage/js');
|
||||
return { root, paths: { repositoryRoot: root, tools: join(root, 'tools'), reportRoot, raw: join(reportRoot, 'raw'), sourceRoot: join(root, 'web/js') } };
|
||||
}
|
||||
|
||||
function recorder() {
|
||||
const calls = [];
|
||||
const run = (args, env = {}) => { calls.push({ args, env }); };
|
||||
return { calls, run };
|
||||
}
|
||||
|
||||
test('no arguments plans the unchanged full run', () => {
|
||||
assert.deepEqual(steps([]), [
|
||||
'clean', 'node-units', 'viewer', { step: 'workflows', shard: null }, 'report', { step: 'check', updateBaseline: false },
|
||||
]);
|
||||
assert.deepEqual(steps(['--update-baseline']), [
|
||||
'clean', 'node-units', 'viewer', { step: 'workflows', shard: null }, 'report', { step: 'check', updateBaseline: true },
|
||||
]);
|
||||
});
|
||||
|
||||
test('--shard 2/3 plans only workflows with --shard 2/3 and no report', () => {
|
||||
assert.deepEqual(steps(['--shard', '2/3']), ['prepare', { step: 'workflows', shard: '2/3' }]);
|
||||
assert.deepEqual(steps(['--shard=3/3']), ['prepare', { step: 'workflows', shard: '3/3' }]);
|
||||
});
|
||||
|
||||
test('--shard 1/N also runs the node units and the viewer suite', () => {
|
||||
assert.deepEqual(steps(['--shard', '1/3']), ['prepare', 'node-units', 'viewer', { step: 'workflows', shard: '1/3' }]);
|
||||
assert.deepEqual(steps(['--shard', '1/1']), ['prepare', 'node-units', 'viewer', { step: 'workflows', shard: '1/1' }]);
|
||||
});
|
||||
|
||||
test('--merge-only plans the report and the check over existing raw coverage', () => {
|
||||
assert.deepEqual(steps(['--merge-only']), ['require-raw', 'report', { step: 'check', updateBaseline: false }]);
|
||||
assert.deepEqual(steps(['--merge-only', '--update-baseline']), ['require-raw', 'report', { step: 'check', updateBaseline: true }]);
|
||||
});
|
||||
|
||||
test('invalid shard and mode combinations are refused', () => {
|
||||
for (const argv of [['--shard'], ['--shard', '0/3'], ['--shard', '4/3'], ['--shard', '2'], ['--shard', 'a/b'], ['--shard', '1/0'], ['--shard', '1.5/3']]) {
|
||||
assert.throws(() => plan(argv), /--shard needs I\/N/, argv.join(' '));
|
||||
}
|
||||
assert.throws(() => plan(['--shard', '1/2', '--merge-only']), /either --shard or --merge-only/);
|
||||
assert.throws(() => plan(['--shard', '1/2', '--update-baseline']), /--update-baseline .* --merge-only or a full run/);
|
||||
assert.throws(() => plan(['--shard', '1/2', '--shard', '2/2']), /--shard once/);
|
||||
assert.throws(() => plan(['--report']), /Unknown argument: --report/);
|
||||
});
|
||||
|
||||
test('--merge-only refuses an empty raw dir', async t => {
|
||||
const { paths } = tempPaths(t);
|
||||
const { calls, run } = recorder();
|
||||
await assert.rejects(runCoverage({ argv: ['--merge-only'], run, paths }), /no raw coverage .*coverage\/js\/raw.* run the --shard steps first/i);
|
||||
mkdirSync(paths.raw, { recursive: true });
|
||||
await assert.rejects(runCoverage({ argv: ['--merge-only'], run, paths }), /no raw coverage/i);
|
||||
writeFileSync(join(paths.raw, 'notes.txt'), 'not coverage');
|
||||
await assert.rejects(runCoverage({ argv: ['--merge-only'], run, paths }), /no raw coverage/i);
|
||||
assert.deepEqual(calls, [], 'nothing is reported or checked over partial or empty coverage');
|
||||
});
|
||||
|
||||
test('a shard keeps raw coverage other shards wrote and passes its shard to Playwright', async t => {
|
||||
const { paths } = tempPaths(t);
|
||||
mkdirSync(paths.raw, { recursive: true });
|
||||
const earlier = join(paths.raw, 'browser-earlier-shard.json');
|
||||
writeFileSync(earlier, '{"result":[]}');
|
||||
const { calls, run } = recorder();
|
||||
await runCoverage({ argv: ['--shard', '2/3'], run, paths });
|
||||
assert.ok(existsSync(earlier), 'raw files from another shard survive');
|
||||
assert.equal(calls.length, 1, 'shard 2 runs only the workflows');
|
||||
const [{ args, env }] = calls;
|
||||
assert.deepEqual(args.slice(-6), ['--project', 'chromium', '--project', 'touch', '--shard', '2/3']);
|
||||
assert.equal(env.PM_JS_COVERAGE, paths.raw);
|
||||
});
|
||||
|
||||
test('shard 1 runs the node units into the shared raw dir, then the viewer suite, then its workflows', async t => {
|
||||
const { root, paths } = tempPaths(t);
|
||||
mkdirSync(join(root, 'tests/js'), { recursive: true });
|
||||
writeFileSync(join(root, 'tests/js/a.test.js'), '');
|
||||
const { calls, run } = recorder();
|
||||
await runCoverage({ argv: ['--shard', '1/3'], run, paths });
|
||||
assert.equal(calls.length, 3);
|
||||
assert.deepEqual(calls[0].args, ['--test', 'tests/js/a.test.js']);
|
||||
assert.equal(calls[0].env.NODE_V8_COVERAGE, paths.raw);
|
||||
assert.ok(calls[1].args.includes(join(paths.tools, 'playwright.config.mjs')));
|
||||
assert.deepEqual(calls[2].args.slice(-2), ['--shard', '1/3']);
|
||||
});
|
||||
Vendored
+99
-20
@@ -1,4 +1,4 @@
|
||||
import { readFileSync, readdirSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { existsSync, readFileSync, readdirSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { dirname, join, relative } from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
@@ -7,33 +7,69 @@ import { repositoryRoot } from './vendor-viewerjs.mjs';
|
||||
import { discoverSources, checkCoverage } from '@comfyassets/toolkit/testing/coverage';
|
||||
export { discoverSources, checkCoverage };
|
||||
const metrics = ['lines', 'branches', 'functions', 'statements'];
|
||||
const SHARD = /^([1-9]\d*)\/([1-9]\d*)$/;
|
||||
const USAGE = 'usage: frontend-coverage.mjs [--update-baseline] | --shard I/N | --merge-only [--update-baseline]';
|
||||
|
||||
function run(args, env = {}) {
|
||||
function runNode(args, env = {}) {
|
||||
const result = spawnSync(process.execPath, args, { cwd: repositoryRoot, stdio: 'inherit',
|
||||
env: { ...process.env, ...env } });
|
||||
if (result.status !== 0) throw new Error(`Command failed (${result.status}): ${args.join(' ')}`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const tools = dirname(fileURLToPath(import.meta.url));
|
||||
function defaultPaths() {
|
||||
const reportRoot = join(repositoryRoot, 'coverage/js');
|
||||
const raw = join(reportRoot, 'raw');
|
||||
const sourceRoot = join(repositoryRoot, 'web/js');
|
||||
return { repositoryRoot, tools: dirname(fileURLToPath(import.meta.url)), reportRoot,
|
||||
raw: join(reportRoot, 'raw'), sourceRoot: join(repositoryRoot, 'web/js') };
|
||||
}
|
||||
|
||||
function parseShard(value) {
|
||||
const match = SHARD.exec(value ?? '');
|
||||
if (!match || Number(match[1]) > Number(match[2])) {
|
||||
throw new Error(`--shard needs I/N with 1 <= I <= N, got ${value ?? 'nothing'}\n${USAGE}`);
|
||||
}
|
||||
return { index: Number(match[1]), total: Number(match[2]), value };
|
||||
}
|
||||
|
||||
// Turn the command line into the ordered steps of one coverage run:
|
||||
// - no mode: the full run (clean, node units, viewer suite, chromium+touch
|
||||
// workflows, report, check), exactly as before shards existed;
|
||||
// - --shard I/N: one slice of the workflows into coverage/js/raw, without wiping
|
||||
// it or reporting; shard 1 also runs the node units and the viewer suite;
|
||||
// - --merge-only: report and check over raw coverage the shards already wrote.
|
||||
export function plan(argv) {
|
||||
let shard = null;
|
||||
let mergeOnly = false;
|
||||
let updateBaseline = false;
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === '--update-baseline') updateBaseline = true;
|
||||
else if (arg === '--merge-only') mergeOnly = true;
|
||||
else if (arg === '--shard' || arg.startsWith('--shard=')) {
|
||||
if (shard) throw new Error(`Pass --shard once\n${USAGE}`);
|
||||
shard = parseShard(arg === '--shard' ? argv[(index += 1)] : arg.slice('--shard='.length));
|
||||
} else throw new Error(`Unknown argument: ${arg}\n${USAGE}`);
|
||||
}
|
||||
if (shard && mergeOnly) throw new Error(`Pass either --shard or --merge-only\n${USAGE}`);
|
||||
if (shard && updateBaseline) throw new Error(`--update-baseline needs --merge-only or a full run, not a shard\n${USAGE}`);
|
||||
const check = { step: 'check', updateBaseline };
|
||||
if (mergeOnly) return [{ step: 'require-raw' }, { step: 'report' }, check];
|
||||
if (shard) {
|
||||
const first = shard.index === 1 ? [{ step: 'node-units' }, { step: 'viewer' }] : [];
|
||||
return [{ step: 'prepare' }, ...first, { step: 'workflows', shard: shard.value }];
|
||||
}
|
||||
return [{ step: 'clean' }, { step: 'node-units' }, { step: 'viewer' }, { step: 'workflows', shard: null },
|
||||
{ step: 'report' }, check];
|
||||
}
|
||||
|
||||
function rawCoverageFiles(raw) {
|
||||
return existsSync(raw) ? readdirSync(raw).filter(file => file.endsWith('.json')) : [];
|
||||
}
|
||||
|
||||
function checkSummary({ tools, reportRoot, sourceRoot }, updateBaseline) {
|
||||
const files = discoverSources(sourceRoot);
|
||||
rmSync(reportRoot, { recursive: true, force: true });
|
||||
mkdirSync(raw, { recursive: true });
|
||||
const tests = readdirSync(join(repositoryRoot, 'tests/js')).filter(file => file.endsWith('.test.js')).map(file => `tests/js/${file}`);
|
||||
run(['--test', ...tests], { NODE_V8_COVERAGE: raw });
|
||||
const playwright = join(tools, 'node_modules/@playwright/test/cli.js');
|
||||
run([playwright, 'test', '--config', join(tools, 'playwright.config.mjs')]);
|
||||
run([playwright, 'test', '--config', join(tools, 'playwright.workflows.config.mjs'),
|
||||
'--project', 'chromium', '--project', 'touch'], { PM_JS_COVERAGE: raw });
|
||||
run([join(tools, 'node_modules/c8/bin/c8.js'), 'report', '--all', '--src', sourceRoot,
|
||||
'--include', 'web/js/**/*.js', '--include', 'web/js/**/*.mjs', '--temp-directory', raw, '--reports-dir', reportRoot,
|
||||
'--reporter', 'text', '--reporter', 'json-summary', '--reporter', 'html']);
|
||||
const summary = JSON.parse(readFileSync(join(reportRoot, 'coverage-summary.json'), 'utf8'));
|
||||
const baselinePath = join(tools, 'frontend-coverage-baseline.json');
|
||||
if (process.argv.includes('--update-baseline')) {
|
||||
if (updateBaseline) {
|
||||
const previous = JSON.parse(readFileSync(baselinePath, 'utf8'));
|
||||
const failures = checkCoverage(summary, previous, sourceRoot, files.filter(file => previous[file]));
|
||||
if (failures.length) throw new Error(`Baseline updates cannot lower coverage:\n${failures.join('\n')}`);
|
||||
@@ -53,8 +89,51 @@ async function main() {
|
||||
const failures = checkCoverage(summary, baseline, sourceRoot, files);
|
||||
if (failures.length) throw new Error(`Frontend coverage regressed:\n${failures.join('\n')}`);
|
||||
}
|
||||
console.log(`Frontend coverage passed for ${files.length} files. Report: ${relative(repositoryRoot, reportRoot)}/index.html`);
|
||||
return files.length;
|
||||
}
|
||||
|
||||
export async function runCoverage({ argv = process.argv.slice(2), run = runNode, paths = defaultPaths() } = {}) {
|
||||
const { repositoryRoot: root, tools, reportRoot, raw, sourceRoot } = paths;
|
||||
const playwright = join(tools, 'node_modules/@playwright/test/cli.js');
|
||||
const actions = {
|
||||
clean() {
|
||||
rmSync(reportRoot, { recursive: true, force: true });
|
||||
mkdirSync(raw, { recursive: true });
|
||||
},
|
||||
prepare() {
|
||||
mkdirSync(raw, { recursive: true });
|
||||
},
|
||||
'require-raw'() {
|
||||
if (!rawCoverageFiles(raw).length) {
|
||||
throw new Error(`No raw coverage in ${relative(root, raw) || raw}: run the --shard steps first, `
|
||||
+ 'or put every shard\'s raw files there');
|
||||
}
|
||||
},
|
||||
'node-units'() {
|
||||
const tests = readdirSync(join(root, 'tests/js')).filter(file => file.endsWith('.test.js'))
|
||||
.map(file => `tests/js/${file}`);
|
||||
run(['--test', ...tests], { NODE_V8_COVERAGE: raw });
|
||||
},
|
||||
viewer() {
|
||||
run([playwright, 'test', '--config', join(tools, 'playwright.config.mjs')]);
|
||||
},
|
||||
workflows({ shard }) {
|
||||
run([playwright, 'test', '--config', join(tools, 'playwright.workflows.config.mjs'),
|
||||
'--project', 'chromium', '--project', 'touch', ...(shard ? ['--shard', shard] : [])], { PM_JS_COVERAGE: raw });
|
||||
},
|
||||
report() {
|
||||
run([join(tools, 'node_modules/c8/bin/c8.js'), 'report', '--all', '--src', sourceRoot,
|
||||
'--include', 'web/js/**/*.js', '--include', 'web/js/**/*.mjs', '--temp-directory', raw, '--reports-dir', reportRoot,
|
||||
'--reporter', 'text', '--reporter', 'json-summary', '--reporter', 'html']);
|
||||
},
|
||||
check({ updateBaseline }) {
|
||||
const count = checkSummary(paths, updateBaseline);
|
||||
console.log(`Frontend coverage passed for ${count} files. Report: ${relative(root, reportRoot)}/index.html`);
|
||||
},
|
||||
};
|
||||
for (const { step, ...options } of plan(argv)) await actions[step](options);
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch(error => { console.error(error.message); process.exitCode = 1; });
|
||||
runCoverage().catch(error => { console.error(error.message); process.exitCode = 1; });
|
||||
}
|
||||
|
||||
Vendored
+2
@@ -15,6 +15,8 @@
|
||||
"test:comfyui": "playwright test --config comfyui.config.mjs",
|
||||
"test:executions": "playwright test --config playwright.execution.config.mjs",
|
||||
"test:coverage": "node frontend-coverage.mjs",
|
||||
"test:coverage:shard": "node frontend-coverage.mjs --shard",
|
||||
"coverage:merge": "node frontend-coverage.mjs --merge-only",
|
||||
"coverage:baseline-check": "node coverage-baseline-check.mjs",
|
||||
"coverage:update": "node frontend-coverage.mjs --update-baseline",
|
||||
"test:compat": "playwright test --config pack-compat.config.mjs"
|
||||
|
||||
+6
-11
@@ -1,23 +1,18 @@
|
||||
import { defineConfig, devices } from '@playwright/test';
|
||||
import { repositoryRoot } from './vendor-viewerjs.mjs';
|
||||
const port = Number(process.env.PM_TEST_PORT || 8765);
|
||||
import { workflowWorkers } from './test-workers.mjs';
|
||||
|
||||
// No global webServer: tests/browser/fixtures.mjs starts one server per worker
|
||||
// on an OS-chosen port. Files run in parallel; tests inside a file keep their order.
|
||||
export default defineConfig({
|
||||
testDir: '../../tests/browser',
|
||||
outputDir: './test-results/workflows',
|
||||
testMatch: '*.spec.mjs',
|
||||
workers: 1,
|
||||
workers: workflowWorkers(),
|
||||
fullyParallel: false,
|
||||
forbidOnly: Boolean(process.env.CI),
|
||||
retries: 0,
|
||||
reporter: 'list',
|
||||
use: { baseURL: `http://127.0.0.1:${port}`, trace: 'retain-on-failure', screenshot: 'only-on-failure' },
|
||||
webServer: {
|
||||
command: `node tools/vendor/start-browser-server.mjs --port ${port}`,
|
||||
cwd: repositoryRoot,
|
||||
url: `http://127.0.0.1:${port}/prompt_manager/test`,
|
||||
reuseExistingServer: false,
|
||||
gracefulShutdown: { signal: 'SIGTERM', timeout: 5000 },
|
||||
},
|
||||
use: { trace: 'retain-on-failure', screenshot: 'only-on-failure' },
|
||||
projects: [
|
||||
{ name: 'chromium', outputDir: './test-results/workflows/chromium', use: { browserName: 'chromium' } },
|
||||
{ name: 'firefox', outputDir: './test-results/workflows/firefox', use: { browserName: 'firefox' } },
|
||||
|
||||
+187
-6
@@ -1,8 +1,189 @@
|
||||
import { spawn } from 'node:child_process';
|
||||
const child = spawn(process.env.PM_TEST_PYTHON || 'python',
|
||||
['-m', 'tests.browser.server', ...process.argv.slice(2)], { stdio: 'inherit' });
|
||||
child.on('error', error => { console.error(error.message); process.exitCode = 1; });
|
||||
child.on('exit', code => { process.exitCode = code ?? 1; });
|
||||
for (const signal of ['SIGTERM', 'SIGINT']) {
|
||||
process.once(signal, () => child.kill(signal));
|
||||
import { resolve } from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
import { repositoryRoot } from './vendor-viewerjs.mjs';
|
||||
|
||||
const READY_PATH = '/prompt_manager/test';
|
||||
const STARTUP_TIMEOUT_MS = 60_000;
|
||||
const STOP_TIMEOUT_MS = 5_000;
|
||||
const POLL_INTERVAL_MS = 100;
|
||||
const PROBE_TIMEOUT_MS = 1_000;
|
||||
const STDERR_TAIL_BYTES = 8_192;
|
||||
// Server stderr kept for the failing test's attachment; the newest output wins.
|
||||
const STDERR_LOG_CHARS = 256 * 1024;
|
||||
const STDERR_DROPPED = '[earlier server stderr dropped]\n';
|
||||
const HOST = '127.0.0.1';
|
||||
// tests/browser/server.py prints this once it holds its listening socket. Only a
|
||||
// complete line counts: a chunk may end mid-number.
|
||||
const PORT_ANNOUNCEMENT = /^PM_TEST_SERVER_PORT=(\d+)\r?\n/m;
|
||||
|
||||
// The server exits when its stdin closes, so a worker that dies without teardown
|
||||
// (SIGKILL) leaves no server behind; the launcher holds the pipe's other end.
|
||||
const pythonServer = port => ['-m', 'tests.browser.server', '--port', String(port), '--exit-on-stdin-eof'];
|
||||
|
||||
function describeExit({ code, signal }) {
|
||||
return signal ? `signal ${signal}` : `code ${code}`;
|
||||
}
|
||||
|
||||
// Start one disposable tests/browser/server.py and wait until it serves the
|
||||
// readiness page. `port` 0 (the default) lets the server bind an OS-chosen port
|
||||
// itself and announce it on stdout, so nothing can take the port between
|
||||
// choosing and binding. Resolves to {port, url, exited, exitStatus,
|
||||
// pid, stderrTail(), stderrMark(), stderrSince(mark), stop()}; rejects with the
|
||||
// server's stderr tail when it exits or stays unready. stdin is a pipe this
|
||||
// process holds open for the server's life.
|
||||
export async function startBrowserServer({
|
||||
port = 0,
|
||||
command = process.env.PM_TEST_PYTHON || 'python',
|
||||
args = pythonServer,
|
||||
cwd = repositoryRoot,
|
||||
timeoutMs = STARTUP_TIMEOUT_MS,
|
||||
stderrLogChars = STDERR_LOG_CHARS,
|
||||
} = {}) {
|
||||
if (!Number.isInteger(port) || port < 0 || port > 65535) {
|
||||
throw new Error(`startBrowserServer needs a port from 0 to 65535, got ${port}`);
|
||||
}
|
||||
const child = spawn(command, args(port), { cwd, stdio: ['pipe', 'pipe', 'pipe'] });
|
||||
// Never written; closing it (or this process dying) tells the server to exit.
|
||||
child.stdin.on('error', () => {});
|
||||
let stderr = '';
|
||||
let stderrTotal = 0;
|
||||
child.stderr.setEncoding('utf8');
|
||||
child.stderr.on('data', chunk => {
|
||||
stderr = (stderr + chunk).slice(-Math.max(stderrLogChars, STDERR_TAIL_BYTES));
|
||||
stderrTotal += chunk.length;
|
||||
});
|
||||
let stdout = '';
|
||||
let announced = null;
|
||||
child.stdout.setEncoding('utf8');
|
||||
// Keep draining stdout for the server's whole life so it never blocks on a full pipe.
|
||||
child.stdout.on('data', chunk => {
|
||||
if (announced !== null) return;
|
||||
stdout = (stdout + chunk).slice(-STDERR_TAIL_BYTES);
|
||||
const match = PORT_ANNOUNCEMENT.exec(stdout);
|
||||
if (match) announced = Number(match[1]);
|
||||
});
|
||||
const killOnExit = () => child.kill('SIGKILL');
|
||||
process.once('exit', killOnExit);
|
||||
const server = {
|
||||
pid: child.pid,
|
||||
port: port || null,
|
||||
url: port ? `http://${HOST}:${port}` : null,
|
||||
exitStatus: null,
|
||||
stopping: false,
|
||||
stderrTail: () => stderr.slice(-STDERR_TAIL_BYTES).trimEnd(),
|
||||
// A position in the server's stderr stream, for stderrSince().
|
||||
stderrMark: () => stderrTotal,
|
||||
// What the server wrote after `mark`, at most stderrLogChars of the newest of it.
|
||||
stderrSince(mark) {
|
||||
const kept = Math.min(stderr.length, stderrLogChars);
|
||||
const keptFrom = stderrTotal - kept;
|
||||
const log = stderr.slice(stderr.length - kept);
|
||||
return mark >= keptFrom ? log.slice(mark - keptFrom) : STDERR_DROPPED + log;
|
||||
},
|
||||
exited: new Promise(done => {
|
||||
child.once('error', error => {
|
||||
stderr += `\n${error.message}`;
|
||||
server.exitStatus = { code: null, signal: null, error };
|
||||
done(server.exitStatus);
|
||||
});
|
||||
child.once('exit', (code, signal) => {
|
||||
server.exitStatus = { code, signal };
|
||||
done(server.exitStatus);
|
||||
});
|
||||
}),
|
||||
async stop() {
|
||||
server.stopping = true;
|
||||
child.stdin.destroy();
|
||||
if (server.exitStatus === null) {
|
||||
child.kill('SIGTERM');
|
||||
const stopped = await Promise.race([server.exited, delay(STOP_TIMEOUT_MS).then(() => false)]);
|
||||
if (!stopped) {
|
||||
child.kill('SIGKILL');
|
||||
await server.exited;
|
||||
}
|
||||
}
|
||||
process.removeListener('exit', killOnExit);
|
||||
},
|
||||
};
|
||||
server.exited.then(() => process.removeListener('exit', killOnExit));
|
||||
const failure = async reason => {
|
||||
await server.stop();
|
||||
const tail = server.stderrTail();
|
||||
return new Error(`test server on port ${server.port ?? '(not yet announced)'} ${reason}`
|
||||
+ (tail ? `\n--- server stderr tail ---\n${tail}` : ''));
|
||||
};
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
if (server.exitStatus !== null) {
|
||||
throw await failure(`exited with ${describeExit(server.exitStatus)} before it was ready`);
|
||||
}
|
||||
if (announced !== null && server.url === null) {
|
||||
server.port = announced;
|
||||
server.url = `http://${HOST}:${announced}`;
|
||||
}
|
||||
if (announced !== null && announced !== server.port) {
|
||||
throw await failure(`announced port ${announced} instead`);
|
||||
}
|
||||
if (server.url === null) {
|
||||
await delay(POLL_INTERVAL_MS);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const response = await fetch(`${server.url}${READY_PATH}`, { signal: AbortSignal.timeout(PROBE_TIMEOUT_MS) });
|
||||
await response.arrayBuffer();
|
||||
if (response.ok) return server;
|
||||
} catch {
|
||||
// Not listening yet.
|
||||
}
|
||||
await delay(POLL_INTERVAL_MS);
|
||||
}
|
||||
throw await failure(`was not ready at ${READY_PATH} within ${timeoutMs} ms`);
|
||||
}
|
||||
|
||||
// After a test: when it did not end as expected, attach what the server wrote to
|
||||
// stderr since `mark` (an aiohttp traceback names the cause a "got 500" hides).
|
||||
export async function attachServerStderr(testInfo, server, mark) {
|
||||
if (testInfo.status === testInfo.expectedStatus) return;
|
||||
const log = server.stderrSince(mark);
|
||||
if (log) await testInfo.attach('server-stderr', { body: log, contentType: 'text/plain' });
|
||||
}
|
||||
|
||||
// Keep one live server for a Playwright worker. ensure() returns the running
|
||||
// server, or starts a fresh one when the last one died (a failed start is
|
||||
// reported, not retried); crash(server) turns an unexpected exit into an error
|
||||
// carrying the stderr tail.
|
||||
export function createServerSupervisor({ start }) {
|
||||
let current = null;
|
||||
return {
|
||||
async ensure() {
|
||||
if (current && current.exitStatus === null) return current;
|
||||
current = null;
|
||||
current = await start();
|
||||
return current;
|
||||
},
|
||||
crash(server) {
|
||||
if (!server || server.stopping || server.exitStatus === null) return null;
|
||||
const tail = server.stderrTail();
|
||||
return new Error(`test server on port ${server.port} exited unexpectedly (${describeExit(server.exitStatus)})`
|
||||
+ (tail ? `\n--- server stderr tail ---\n${tail}` : ''));
|
||||
},
|
||||
async stop() {
|
||||
const server = current;
|
||||
current = null;
|
||||
await server?.stop();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// CLI: `node start-browser-server.mjs --port N` runs one server in the foreground.
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
|
||||
const child = spawn(process.env.PM_TEST_PYTHON || 'python',
|
||||
['-m', 'tests.browser.server', ...process.argv.slice(2)], { stdio: 'inherit' });
|
||||
child.on('error', error => { console.error(error.message); process.exitCode = 1; });
|
||||
child.on('exit', code => { process.exitCode = code ?? 1; });
|
||||
for (const signal of ['SIGTERM', 'SIGINT']) {
|
||||
process.once(signal, () => child.kill(signal));
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+18
@@ -0,0 +1,18 @@
|
||||
import { availableParallelism } from 'node:os';
|
||||
|
||||
const MAX_LOCAL_WORKERS = 8;
|
||||
const CPUS_PER_WORKER = 4;
|
||||
|
||||
// PM_TEST_WORKERS wins; CI without it keeps one worker, so CI only changes
|
||||
// through explicit workflow edits; locally a quarter of the CPUs, at most eight.
|
||||
export function workflowWorkers({ env = process.env, cpus = availableParallelism() } = {}) {
|
||||
if (env.PM_TEST_WORKERS !== undefined) {
|
||||
const workers = Number(env.PM_TEST_WORKERS);
|
||||
if (!Number.isInteger(workers) || workers < 1) {
|
||||
throw new Error(`PM_TEST_WORKERS must be a positive integer, got "${env.PM_TEST_WORKERS}"`);
|
||||
}
|
||||
return workers;
|
||||
}
|
||||
if (env.CI) return 1;
|
||||
return Math.max(1, Math.min(MAX_LOCAL_WORKERS, Math.floor(cpus / CPUS_PER_WORKER)));
|
||||
}
|
||||
Reference in New Issue
Block a user