diff --git a/.github/workflows/viewerjs-check.yml b/.github/workflows/viewerjs-check.yml index 41ddf1b..7c57cd2 100644 --- a/.github/workflows/viewerjs-check.yml +++ b/.github/workflows/viewerjs-check.yml @@ -16,9 +16,19 @@ concurrency: group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +# Timeouts are the measured job time plus at least 50%, and never under 20 minutes for a browser job, +# because runner speed varies. Before sharding (main 2cecfcf, one worker): frontend coverage 32m10s, +# page workflows firefox 18m59s, webkit 19m26s. First sharded run (PR #274 run 39712909047, 3 workers, +# no npm cache, where npm ci stalled for 5m in six of eight jobs): coverage shards 11m19s/10m47s/10m49s +# (tests 5m20s/4m42s/4m46s), merge 5m52s (report and check 4s), firefox shards 7m53s/9m27s (tests +# 4m41s/3m29s), webkit shards 12m25s+ (cancelled at the old 12-minute limit after its 267 tests passed in +# 6m29s) and 4m53s (tests 3m08s). With the npm cache hit (run 39725176320 at cbc8a61, npm ci 0-2s): +# coverage shards 7m34s/6m04s/5m22s (tests 6m38s/4m53s/4m04s), merge 54s, firefox 7m57s/5m21s (tests +# 7m08s/4m34s), webkit 8m16s/5m03s (tests 7m26s/3m58s); a cache miss adds up to 5m to npm ci, which +# the 20-minute limits absorb. jobs: frontend-coverage: - name: frontend coverage (chromium and touch) + name: frontend coverage (shard ${{ matrix.shard }}/3) runs-on: ubuntu-latest # Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the # same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json; @@ -26,12 +36,15 @@ jobs: container: image: mcr.microsoft.com/playwright:v1.64.1-noble options: --user 1001 - # This job runs every page workflow twice (chromium, then touch) on one worker with V8 coverage on, so it - # takes about twice a single-browser job. Measured 2026-10-20: whole job 18m26s-24m57s (main 5ded3cf - # 22m15s; PR head 6359999 was cancelled at the old 25-minute limit while still passing), coverage step - # alone 20m55s-22m17s+, against ~12-15 minutes for the firefox/webkit jobs. 40 minutes leaves headroom - # for suite growth while still stopping a hung run; re-measure if a run passes 30 minutes. - timeout-minutes: 40 + timeout-minutes: 20 + # Public-repo ubuntu-latest runners have 4 vCPUs: three Playwright workers, each with its own test + # server, leave one CPU for the servers and the runner. + env: + PM_TEST_WORKERS: 3 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] steps: - uses: actions/checkout@v7 # The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache @@ -50,24 +63,107 @@ jobs: "$venv/bin/python" -m pip install --no-cache-dir -r requirements-dev.txt echo "$venv/bin" >> "$GITHUB_PATH" - run: python --version && node --version && node --version | grep -q '^v24\.' + # npm's download cache, keyed on the lockfile; inside the container ~ is the job's HOME (/github/home), + # which npm also uses. node_modules itself is still built fresh by npm ci. + - name: Cache npm downloads + uses: actions/cache@v6 + with: + path: ~/.npm + key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }} - run: npm ci --prefix tools/vendor --ignore-scripts - - run: npm run --prefix tools/vendor check - - run: npm run --prefix tools/vendor test - - name: Frontend unit, viewer, workflow, and coverage checks - run: npm run --prefix tools/vendor test:coverage - - name: Upload frontend coverage and Chromium traces + # Shard 1 also runs the node units and the viewer suite, so the vendor pins and the tooling tests + # run on the lightest shard instead. + - if: matrix.shard == 3 + run: npm run --prefix tools/vendor check + - if: matrix.shard == 3 + run: npm run --prefix tools/vendor test + - name: Page workflows with coverage, shard ${{ matrix.shard }}/3 + run: npm run --prefix tools/vendor test:coverage:shard -- ${{ matrix.shard }}/3 + - name: Upload raw coverage + uses: actions/upload-artifact@v7 + with: + name: coverage-raw-${{ matrix.shard }} + path: coverage/js/raw/ + if-no-files-found: error + retention-days: 3 + - name: Upload Chromium and touch traces + if: always() + uses: actions/upload-artifact@v7 + with: + name: frontend-traces-shard-${{ matrix.shard }} + path: tools/vendor/test-results/ + retention-days: 14 + + frontend-coverage-merge: + name: frontend coverage (merge) + needs: frontend-coverage + runs-on: ubuntu-latest + # Same image as the shards: the raw files name sources by the container's workspace path. + container: + image: mcr.microsoft.com/playwright:v1.64.1-noble + options: --user 1001 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + - run: node --version && node --version | grep -q '^v24\.' + # npm's download cache, keyed on the lockfile; inside the container ~ is the job's HOME (/github/home), + # which npm also uses. node_modules itself is still built fresh by npm ci. + - name: Cache npm downloads + uses: actions/cache@v6 + with: + path: ~/.npm + key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }} + - run: npm ci --prefix tools/vendor --ignore-scripts + # Each artifact lands in its own coverage-raw-N directory, so the next step can count them. + - uses: actions/download-artifact@v8 + with: + pattern: coverage-raw-* + path: ${{ runner.temp }}/coverage-raw + # coverage:merge only checks that raw files exist; merging two of three shards would report + # partial coverage, so a missing or empty shard artifact fails here. + - name: Verify every shard's raw coverage arrived + shell: bash + env: + STAGED_RAW: ${{ runner.temp }}/coverage-raw + EXPECTED_SHARDS: 3 + run: | + found=0 + for shard in $(seq 1 "$EXPECTED_SHARDS"); do + directory="$STAGED_RAW/coverage-raw-$shard" + count=0 + if [ -d "$directory" ]; then count=$(find "$directory" -maxdepth 1 -type f -name '*.json' | wc -l); fi + echo "coverage-raw-$shard: $count raw files" + if [ "$count" -gt 0 ]; then found=$((found + 1)); fi + done + echo "shard artifacts with raw coverage: $found of $EXPECTED_SHARDS" + if [ "$found" -ne "$EXPECTED_SHARDS" ]; then + echo "::error::Expected raw coverage from $EXPECTED_SHARDS shards, got $found of $EXPECTED_SHARDS" + exit 1 + fi + staged=$(find "$STAGED_RAW" -mindepth 2 -maxdepth 2 -type f -name '*.json' | wc -l) + mkdir -p coverage/js/raw + # cp -n never overwrites; a clash shows up as fewer merged files than staged ones. + find "$STAGED_RAW" -mindepth 2 -maxdepth 2 -type f -name '*.json' -exec cp -n -t coverage/js/raw {} + || true + merged=$(find coverage/js/raw -maxdepth 1 -type f -name '*.json' | wc -l) + if [ "$merged" -ne "$staged" ]; then + echo "::error::Shards wrote clashing raw file names: $staged staged, $merged merged" + exit 1 + fi + echo "merged $merged raw files" + - name: Frontend coverage report and baseline check + run: npm run --prefix tools/vendor coverage:merge + - name: Upload frontend coverage report if: always() uses: actions/upload-artifact@v7 with: name: frontend-test-reports path: | - tools/vendor/test-results/ coverage/js/ !coverage/js/raw/** retention-days: 14 browser-workflows: - name: page workflows (${{ matrix.browser }}) + name: page workflows (${{ matrix.browser }}, shard ${{ matrix.shard }}/2) runs-on: ubuntu-latest # Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the # same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json; @@ -75,11 +171,16 @@ jobs: container: image: mcr.microsoft.com/playwright:v1.64.1-noble options: --user 1001 - timeout-minutes: 25 + timeout-minutes: 20 + # Public-repo ubuntu-latest runners have 4 vCPUs: three Playwright workers, each with its own test + # server, leave one CPU for the servers and the runner. + env: + PM_TEST_WORKERS: 3 strategy: fail-fast: false matrix: browser: [firefox, webkit] + shard: [1, 2] steps: - uses: actions/checkout@v7 # The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache @@ -98,22 +199,29 @@ jobs: "$venv/bin/python" -m pip install --no-cache-dir -r requirements-dev.txt echo "$venv/bin" >> "$GITHUB_PATH" - run: python --version && node --version && node --version | grep -q '^v24\.' + # npm's download cache, keyed on the lockfile; inside the container ~ is the job's HOME (/github/home), + # which npm also uses. node_modules itself is still built fresh by npm ci. + - name: Cache npm downloads + uses: actions/cache@v6 + with: + path: ~/.npm + key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }} - run: npm ci --prefix tools/vendor --ignore-scripts - - name: Page workflows in ${{ matrix.browser }} - run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }} + - name: Page workflows in ${{ matrix.browser }}, shard ${{ matrix.shard }}/2 + run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }} --shard ${{ matrix.shard }}/2 - name: Upload browser traces if: always() uses: actions/upload-artifact@v7 with: - name: browser-traces-${{ matrix.browser }} + name: browser-traces-${{ matrix.browser }}-shard-${{ matrix.shard }} path: tools/vendor/test-results/ retention-days: 14 # Branch protection requires the exact context "viewerjs". This job keeps that - # name and fails unless every frontend job above succeeded, including when one - # was cancelled or skipped. + # name and fails unless every frontend job above succeeded (every shard, the + # coverage merge and every browser shard), including when one was cancelled or skipped. viewerjs: - needs: [frontend-coverage, browser-workflows] + needs: [frontend-coverage, frontend-coverage-merge, browser-workflows] if: always() runs-on: ubuntu-latest timeout-minutes: 5 @@ -121,8 +229,10 @@ jobs: - name: Require every frontend job to succeed env: COVERAGE_RESULT: ${{ needs.frontend-coverage.result }} + MERGE_RESULT: ${{ needs.frontend-coverage-merge.result }} WORKFLOWS_RESULT: ${{ needs.browser-workflows.result }} run: | echo "frontend-coverage: $COVERAGE_RESULT" + echo "frontend-coverage-merge: $MERGE_RESULT" echo "browser-workflows: $WORKFLOWS_RESULT" - test "$COVERAGE_RESULT" = success && test "$WORKFLOWS_RESULT" = success + test "$COVERAGE_RESULT" = success && test "$MERGE_RESULT" = success && test "$WORKFLOWS_RESULT" = success diff --git a/TESTING.md b/TESTING.md index 21aaa78..76031db 100644 --- a/TESTING.md +++ b/TESTING.md @@ -39,9 +39,44 @@ npm run --prefix tools/vendor test:coverage npm run --prefix tools/vendor test:workflows -- --project firefox --project webkit ``` -`PM_TEST_PYTHON` selects the Python executable; `PM_TEST_PORT` selects the local -port (default 8765). Set these environment variables when Python is outside PATH -or the port is occupied. Test servers are never reused. On unsupported Linux +`PM_TEST_PYTHON` selects the Python executable; set it when Python is outside +PATH. Every Playwright worker starts its own `tests/browser/server.py --port 0`, +which binds a port the OS picks and prints it as `PM_TEST_SERVER_PORT=`, with +its own temp root and database (`tests/browser/fixtures.mjs`), so +parallel workers and concurrent runs never share state. `PM_TEST_WORKERS` sets the +worker count (default 1 under `CI`, otherwise a quarter of the CPUs, at most 8). +Test servers are never reused across runs. The launcher holds each server's +stdin open and the server (`--exit-on-stdin-eof`) shuts down cleanly, removing its +temp root, when that pipe closes, so a worker killed without teardown leaves no +server behind. A test that fails gets the stderr its server wrote during the test +attached to the report as `server-stderr` (an aiohttp traceback behind a 500). + +The coverage run can be split the way CI splits it, and each shard keeps the raw +V8 files it writes under `coverage/js/raw`: + +```sh +PM_TEST_WORKERS=8 npm run --prefix tools/vendor test:coverage # whole run, one report +npm run --prefix tools/vendor test:coverage:shard -- 1/3 # node units, viewer suite, workflows 1/3 +npm run --prefix tools/vendor test:coverage:shard -- 2/3 # workflows 2/3 (likewise 3/3) +npm run --prefix tools/vendor coverage:merge # report and baseline check of every raw file +npm run --prefix tools/vendor test:workflows -- --project webkit --shard 2/2 +``` + +`coverage:merge` refuses an empty raw directory; it does not count shards, so run +every shard first (CI's merge job checks all three arrived). Measured on a 32-CPU +machine in the pinned image (2026-10-21): `test:coverage` takes about 5 minutes +with 8 workers and 24 minutes with 1; all four projects, 8 workers, each test 3 +times (6132 tests), 24 minutes. A CI-sized run (`docker run --cpus=4`, +`PM_TEST_WORKERS=3`, all four projects once, 2044 tests) takes 25.5 minutes. + +Specs must pass beside each other, not only alone. Wait for the page's own +signal, never for time: a rendering frame (`requestAnimationFrame`) for layout +work queued by a render or a scroll, a response or a visible state for data. A +timer that "usually" covers a frame is a race on a loaded runner. Library +virtualization specs settle with `libraryObservation.settled()` (two consecutive +frames with no Library frame queued). To look for such races, repeat a spec under +load (`--workers 8 --repeat-each 10`, and in the image with `--cpus=1` or `2`). +On unsupported Linux distributions, browser system dependencies may need manual installation. For a quick browser run, use `test:workflows -- --project chromium`. @@ -114,16 +149,39 @@ second without a browser and fails, naming the files, when a measured `web/js` file has no baseline entry or the baseline lists a file that no longer exists. It uses the gate's own `discoverSources`, and `npm test` runs the same check, so run it before pushing a new or deleted frontend file instead of -waiting for the `frontend coverage` job. +waiting for the `frontend coverage` jobs. The `viewerjs` check runs on every PR, including backend-only changes. It is an -aggregate job that passes only when the `frontend coverage (chromium and touch)` -job and both `page workflows (firefox)` and `page workflows (webkit)` jobs -succeed; the browsers run on separate runners in parallel. GitHub artifacts -named `frontend-test-reports` contain HTML coverage, a JSON summary, and the -Chromium/touch failure screenshots/traces; `browser-traces-firefox` and -`browser-traces-webkit` hold the other browsers' traces. Open traces with -Playwright's `show-trace` tool. +aggregate job that passes only when every frontend job succeeds: the three +`frontend coverage (shard N/3)` jobs, `frontend coverage (merge)`, and the four +`page workflows (firefox|webkit, shard N/2)` jobs. A failed, cancelled or skipped +job fails it. Every browser job runs with `PM_TEST_WORKERS: 3` (the public +runners have 4 vCPUs). Each coverage shard runs `test:coverage:shard -- N/3` +(shard 1 also runs the node units and the viewer suite, shard 3 the vendor pins +and tooling tests) and uploads its raw V8 files as `coverage-raw-N` (kept 3 +days). The merge job downloads them, fails unless all three arrived with raw +files and without clashing names, then runs `coverage:merge` (report plus +baseline check). `tests/vendor/ci-workflow.test.mjs` checks this shape. GitHub +artifacts named `frontend-test-reports` contain the merged HTML coverage and +JSON summary; `frontend-traces-shard-N` hold the Chromium/touch failure +screenshots/traces and `browser-traces--shard-N` the other browsers' +traces. Open traces with Playwright's `show-trace` tool. + +Every `npm ci` first restores npm's download cache (`~/.npm`), keyed on the vendor +lockfile and the job's pinned image tag (`tests/vendor/ci-workflow.test.mjs` checks +the key names the same tag as the job's `container.image`). Without it, `npm ci` stalled for about 300 s +(npm's fetch timeout) in most of the eight concurrent jobs; with a hit it takes +0 to 2 s. A lockfile change misses the cache once, so expect one slow run after it. +Measured on PR #274 (run 39725176320 at cbc8a61, cache hit): the whole workflow +took 8m38s, against about 33 minutes with one worker and no shards. Jobs: coverage +shards 7m34s/6m04s/5m22s (tests 6m38s/4m53s/4m04s), merge 54s, firefox shards +7m57s/5m21s (tests 7m08s/4m34s), webkit shards 8m16s/5m03s (tests 7m26s/3m58s). +Browser jobs keep a 20-minute limit: the slowest job is about 8.5 minutes, a cache +miss adds up to 5 to `npm ci`, and runner speed varies (the first sharded run was +cancelled at a 12-minute limit). +Playwright shards by test count, whole files at a time, so shard 1 carries the +slow concept-comparison specs (`*-concept.spec.mjs`, alphabetically first) and +runs longer; the remedy, if it starts to matter, is more shards, not reordering. The real-host suite covers both Classic and Nodes 2.0 startup, switching both ways, actual database prompt selection, execution-text serialization, workflow @@ -1556,7 +1614,7 @@ bytes at the same path without changing the immutable receipt. The focused acceptance command retains all four projects: ```sh -PM_TEST_PORT=8766 npm run --prefix tools/vendor test:workflows -- \ +npm run --prefix tools/vendor test:workflows -- \ workspace-history.spec.mjs workspace-visual.spec.mjs \ --project chromium --project touch --project firefox --project webkit npm run --prefix tools/vendor test:executions @@ -1649,7 +1707,7 @@ acceptance; a baseline is added only after the maintainer accepts them. Focused commands: ```sh -PM_TEST_PORT=8766 npm run --prefix tools/vendor test:workflows -- \ +npm run --prefix tools/vendor test:workflows -- \ workspace-library-editor.spec.mjs library-editor-visual.spec.mjs \ --project chromium --project touch --project firefox --project webkit npm run --prefix tools/vendor test:comfyui # library-sender.spec.mjs host regression @@ -1749,7 +1807,7 @@ mobile, the concept geometry/token/control comparison and its mutation check), ```sh node --test tests/js/workspace-vocabulary-state.test.js tests/js/workspace-state.test.js -PM_TEST_PORT=8766 npm run --prefix tools/vendor test:workflows -- \ +npm run --prefix tools/vendor test:workflows -- \ workspace-vocabulary.spec.mjs vocabulary-visual.spec.mjs focused-node.spec.mjs \ --project chromium --project touch --project firefox --project webkit ``` diff --git a/tests/browser/activity-concept.spec.mjs b/tests/browser/activity-concept.spec.mjs index e3c8f9f..2f4ef4d 100644 --- a/tests/browser/activity-concept.spec.mjs +++ b/tests/browser/activity-concept.spec.mjs @@ -1,5 +1,5 @@ import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs'; -import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import { test, expect } from './fixtures.mjs'; import { THEMES, seedConcept, openConcept, openProduction, diffInventory, mutate } from './concept-inventory.mjs'; // Activity-1 Task 3. The review dialog is compared with the concept's own `scan-review` ("Review metadata scan", diff --git a/tests/browser/classic-node.spec.mjs b/tests/browser/classic-node.spec.mjs index a915acd..f416062 100644 --- a/tests/browser/classic-node.spec.mjs +++ b/tests/browser/classic-node.spec.mjs @@ -1,5 +1,5 @@ import { readFileSync } from 'node:fs'; -import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import { test, expect } from './fixtures.mjs'; import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs'; test.beforeEach(async ({page, browserName}) => { @@ -57,7 +57,8 @@ test('Classic keeps immediate selection, search query and metadata serialization }); test('Classic detaches native text synchronization on removal',async({page})=>{ - await page.waitForTimeout(60); + // The page installs the text sync (serializeValue and its listeners) in a 50 ms timer; wait for it, not for time. + await expect.poll(()=>page.evaluate(()=>typeof node.widgets.find(item=>item.name==='text').serializeValue)).toBe('function'); const values=await page.evaluate(()=>{ const widget=node.widgets.find(item=>item.name==='text'); widget.inputEl.value='Before removal';widget.inputEl.dispatchEvent(new Event('input')); diff --git a/tests/browser/concept-inventory.mjs b/tests/browser/concept-inventory.mjs index 966bd64..0bc9dfb 100644 --- a/tests/browser/concept-inventory.mjs +++ b/tests/browser/concept-inventory.mjs @@ -2,7 +2,7 @@ // same concept selectors in the immutable concept (/__test__/canonical/workspace) and in production, which now renders // the concept's classes. The concept is never rewritten: each page's mapConcept(reference) applies only the // recorded spec-§8 deviations, each commented with its D-number, and diffInventory must then report nothing. -import {expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import {expect} from './fixtures.mjs'; export {diffInventory} from './library-editor-canonical-comparison.mjs'; export const CONCEPT_URL='/__test__/canonical/workspace'; diff --git a/tests/browser/fixtures.mjs b/tests/browser/fixtures.mjs new file mode 100644 index 0000000..511e512 --- /dev/null +++ b/tests/browser/fixtures.mjs @@ -0,0 +1,33 @@ +// Shared `test` for the page workflows: every Playwright worker owns one +// tests/browser/server.py process on a port the OS chose for it, with its own temp root +// and database. `baseURL` (and so `page`, `request` and `browser.newContext`) +// points at that worker's server. A server that dies fails the running test +// with its stderr tail, and the next test in the worker gets a fresh server. +// A test that fails gets the stderr its server wrote during it attached as +// `server-stderr`. +import { test as base, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import { attachServerStderr, createServerSupervisor, startBrowserServer } from '../../tools/vendor/start-browser-server.mjs'; + +const SERVER_FIXTURE_TIMEOUT_MS = 90_000; + +export const test = base.extend({ + testServer: [async ({}, use) => { + const supervisor = createServerSupervisor({ start: () => startBrowserServer() }); + try { + await use(supervisor); + } finally { + await supervisor.stop(); + } + }, { scope: 'worker', timeout: SERVER_FIXTURE_TIMEOUT_MS }], + workerServer: [async ({ testServer }, use, testInfo) => { + const server = await testServer.ensure(); + const mark = server.stderrMark(); + await use(server); + await attachServerStderr(testInfo, server, mark); + const crash = testServer.crash(server); + if (crash) throw crash; + }, { auto: true, timeout: SERVER_FIXTURE_TIMEOUT_MS }], + baseURL: async ({ workerServer }, use) => use(workerServer.url), +}); + +export { expect }; diff --git a/tests/browser/focused-node.spec.mjs b/tests/browser/focused-node.spec.mjs index c99e9b3..6f9eb31 100644 --- a/tests/browser/focused-node.spec.mjs +++ b/tests/browser/focused-node.spec.mjs @@ -1,5 +1,5 @@ import {readFileSync} from 'node:fs'; -import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import {test,expect} from './fixtures.mjs'; import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs'; for (const variant of ['Focused', 'Split']) { diff --git a/tests/browser/full-inspector.spec.mjs b/tests/browser/full-inspector.spec.mjs index 6c6e2f9..5c82d3e 100644 --- a/tests/browser/full-inspector.spec.mjs +++ b/tests/browser/full-inspector.spec.mjs @@ -1,5 +1,5 @@ import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs'; -import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import { test, expect } from './fixtures.mjs'; import { seedConcept, openConcept, openProduction, SIZES } from './concept-inventory.mjs'; // D10/D20: the heading is the linked positive's title (its confirmed first line here), never the pair's // "Inspector recipe", which names the negative variant. diff --git a/tests/browser/history-canonical-comparison.mjs b/tests/browser/history-canonical-comparison.mjs index 0433d91..e7c829c 100644 --- a/tests/browser/history-canonical-comparison.mjs +++ b/tests/browser/history-canonical-comparison.mjs @@ -1,5 +1,5 @@ import { writeFile } from 'node:fs/promises'; -import { expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import { expect } from './fixtures.mjs'; // Content, row counts, and image bytes come from independent synthetic datasets. // Saved choices and actual executions are separate production sections. The concept's diff --git a/tests/browser/history-concept.spec.mjs b/tests/browser/history-concept.spec.mjs index bac5df4..353b25d 100644 --- a/tests/browser/history-concept.spec.mjs +++ b/tests/browser/history-concept.spec.mjs @@ -1,5 +1,5 @@ import {writeFile} from 'node:fs/promises'; -import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import {test,expect} from './fixtures.mjs'; import {captureCanonicalState} from './history-canonical-comparison.mjs'; import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,diffInventory,mutate,stripExact} from './concept-inventory.mjs'; diff --git a/tests/browser/images-concept.spec.mjs b/tests/browser/images-concept.spec.mjs index 6d3ffbe..309badc 100644 --- a/tests/browser/images-concept.spec.mjs +++ b/tests/browser/images-concept.spec.mjs @@ -1,5 +1,5 @@ import {writeFile} from 'node:fs/promises'; -import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import {test,expect} from './fixtures.mjs'; import {captureCanonicalState} from './history-canonical-comparison.mjs'; import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,diffInventory,mutate,stripExact,countCopy,reflow,withoutItems} from './concept-inventory.mjs'; diff --git a/tests/browser/inspector-concept.spec.mjs b/tests/browser/inspector-concept.spec.mjs index 6a7f32f..951c833 100644 --- a/tests/browser/inspector-concept.spec.mjs +++ b/tests/browser/inspector-concept.spec.mjs @@ -1,5 +1,5 @@ import {writeFile} from 'node:fs/promises'; -import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import {test,expect} from './fixtures.mjs'; import {captureCanonicalState} from './history-canonical-comparison.mjs'; import {SIZES,THEMES,seedConcept,openConcept,openProduction,inspectorOf,diffInventory,mutate,countCopy} from './concept-inventory.mjs'; diff --git a/tests/browser/library-concept.spec.mjs b/tests/browser/library-concept.spec.mjs index afc73ae..eeab9b9 100644 --- a/tests/browser/library-concept.spec.mjs +++ b/tests/browser/library-concept.spec.mjs @@ -1,5 +1,5 @@ import {writeFile} from 'node:fs/promises'; -import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import {test,expect} from './fixtures.mjs'; import {captureCanonicalState} from './history-canonical-comparison.mjs'; import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,diffInventory,mutate,stripExact,countCopy,reflow} from './concept-inventory.mjs'; diff --git a/tests/browser/library-editor-visual.spec.mjs b/tests/browser/library-editor-visual.spec.mjs index e277f8e..8b92c22 100644 --- a/tests/browser/library-editor-visual.spec.mjs +++ b/tests/browser/library-editor-visual.spec.mjs @@ -1,4 +1,4 @@ -import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import {test,expect} from './fixtures.mjs'; import {captureCanonicalState} from './history-canonical-comparison.mjs'; // Artifact-only captures of the editor states that have no concept counterpart (archive confirmation, Show archived, diff --git a/tests/browser/pair-node.spec.mjs b/tests/browser/pair-node.spec.mjs index d124574..1d0e4bd 100644 --- a/tests/browser/pair-node.spec.mjs +++ b/tests/browser/pair-node.spec.mjs @@ -1,5 +1,5 @@ import {readFileSync} from 'node:fs'; -import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import {test,expect} from './fixtures.mjs'; import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs'; const pair={id:4,positive_text:'Library positive',negative_text:'Library negative',category:'Portrait',tags:['studio']}; diff --git a/tests/browser/prompt-bridge.spec.mjs b/tests/browser/prompt-bridge.spec.mjs index 0219dfc..f6cdb91 100644 --- a/tests/browser/prompt-bridge.spec.mjs +++ b/tests/browser/prompt-bridge.spec.mjs @@ -1,4 +1,4 @@ -import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import { test, expect } from './fixtures.mjs'; import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs'; const moduleUrl = '/prompt_manager/js/shared/toolkit/prompt-bridge.js'; diff --git a/tests/browser/prompt-inspector.spec.mjs b/tests/browser/prompt-inspector.spec.mjs index 9ebca0e..8841598 100644 --- a/tests/browser/prompt-inspector.spec.mjs +++ b/tests/browser/prompt-inspector.spec.mjs @@ -1,5 +1,5 @@ import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs'; -import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import {test,expect} from './fixtures.mjs'; const workspace='/prompt_manager/workspace#/images'; const details=page=>page.getByRole('complementary',{name:'Selected image details',exact:true}); diff --git a/tests/browser/server.py b/tests/browser/server.py index 5e2888d..77dc0f4 100644 --- a/tests/browser/server.py +++ b/tests/browser/server.py @@ -1,16 +1,20 @@ """Serve production pages and API routes with disposable state for browser tests.""" import argparse +import asyncio import json import os import shutil +import socket from pathlib import Path import sys import tempfile +import threading import time import types from aiohttp import web +from aiohttp.web_runner import GracefulExit from PIL import Image from PIL.PngImagePlugin import PngInfo @@ -515,14 +519,77 @@ def create_app(root: Path) -> web.Application: return app -def main() -> None: +HOST = "127.0.0.1" +PORT_ANNOUNCEMENT = "PM_TEST_SERVER_PORT=" + + +def bind_listener(port: int) -> socket.socket: + """Bind the server's socket before the app is built. + + ``port`` 0 lets the OS choose a free port, which this process then holds, + so parallel launchers never race for one between choosing and binding. + """ + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + if port and os.name != "nt": + # aiohttp's own default on POSIX; on Windows it would let a + # second process share the port. + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind((HOST, port)) + listener.listen(128) + except OSError: + listener.close() + raise + return listener + + +def _graceful_exit() -> None: + raise GracefulExit() + + +def exit_when_stdin_closes(loop: asyncio.AbstractEventLoop) -> threading.Thread: + """Stop the app gracefully once stdin reaches end of file. + + The launcher keeps the write end of this process's stdin pipe, and the OS + closes it however the launcher dies, SIGKILL included. GracefulExit runs + aiohttp's cleanup (closing the database) and lets the temp root be removed, + on every OS, where a signal from a dead parent would not. + """ + + def watch() -> None: + while sys.stdin.buffer.read(65536): + pass + loop.call_soon_threadsafe(_graceful_exit) + + thread = threading.Thread(target=watch, name="stdin-eof", daemon=True) + thread.start() + return thread + + +def main(argv: list[str] | None = None) -> None: parser = argparse.ArgumentParser() - parser.add_argument("--port", type=int, default=8765) - args = parser.parse_args() + parser.add_argument( + "--port", type=int, default=8765, help="0 binds an OS-chosen free port" + ) + parser.add_argument( + "--exit-on-stdin-eof", + action="store_true", + help="shut down when stdin closes (the launcher's parent-death guard)", + ) + args = parser.parse_args(argv) + listener = bind_listener(args.port) + # One machine-readable line tells the launcher which port this process holds. + print(f"{PORT_ANNOUNCEMENT}{listener.getsockname()[1]}", flush=True) + loop = asyncio.new_event_loop() + if args.exit_on_stdin_eof: + exit_when_stdin_closes(loop) + original_cwd = os.getcwd() with tempfile.TemporaryDirectory(prefix="pm-browser-") as directory: - web.run_app( - create_app(Path(directory).resolve()), host="127.0.0.1", port=args.port - ) + try: + web.run_app(create_app(Path(directory).resolve()), sock=listener, loop=loop) + finally: + # create_app works inside the root; Windows cannot remove a process's cwd. + os.chdir(original_cwd) if __name__ == "__main__": diff --git a/tests/browser/shell-concept.spec.mjs b/tests/browser/shell-concept.spec.mjs index 509bdc4..8249311 100644 --- a/tests/browser/shell-concept.spec.mjs +++ b/tests/browser/shell-concept.spec.mjs @@ -1,5 +1,5 @@ import {writeFile} from 'node:fs/promises'; -import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import {test,expect} from './fixtures.mjs'; import {captureCanonicalState} from './history-canonical-comparison.mjs'; import {SIZES,THEMES,MOTIONS,seedConcept,openConcept as openConceptPage,openProduction,inventoryOf,diffInventory,focusRingOf,hoverStateOf,mutate} from './concept-inventory.mjs'; diff --git a/tests/browser/vocabulary-visual.spec.mjs b/tests/browser/vocabulary-visual.spec.mjs index 9d354dd..2d83795 100644 --- a/tests/browser/vocabulary-visual.spec.mjs +++ b/tests/browser/vocabulary-visual.spec.mjs @@ -1,4 +1,4 @@ -import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import {test,expect} from './fixtures.mjs'; import {captureCanonicalState} from './history-canonical-comparison.mjs'; // Artifact-only captures of Tags & categories states that have no concept counterpart: the Manage categories diff --git a/tests/browser/worker-server.spec.mjs b/tests/browser/worker-server.spec.mjs new file mode 100644 index 0000000..65d2b8d --- /dev/null +++ b/tests/browser/worker-server.spec.mjs @@ -0,0 +1,35 @@ +// Each Playwright worker owns its server: `request`, `page` and manual +// `browser.newContext()` all reach it through `baseURL`, and another server +// process never sees its database. +import { test, expect } from './fixtures.mjs'; +import { startBrowserServer } from '../../tools/vendor/start-browser-server.mjs'; + +const search = text => `/prompt_manager/search?text=${encodeURIComponent(text)}`; + +test('every fixture reaches this worker\'s own server and another server keeps its own database', async ({ request, page, browser, baseURL, workerServer }, testInfo) => { + expect(baseURL).toBe(workerServer.url); + await expect(await request.post('/__test__/reset')).toBeOK(); + const marker = `worker ${testInfo.workerIndex} isolation ${testInfo.project.name} ${testInfo.repeatEachIndex}`; + await expect(await request.post('/prompt_manager/save', { data: { text: marker } })).toBeOK(); + expect((await (await request.get(search(marker))).json()).count).toBe(1); + + const viaPage = await page.request.get(search(marker)); + expect(viaPage.url().startsWith(`${baseURL}/`)).toBe(true); + expect((await viaPage.json()).count).toBe(1); + const context = await browser.newContext(); + try { + const response = await (await context.newPage()).goto(search(marker)); + expect(response.url().startsWith(`${baseURL}/`)).toBe(true); + expect((await response.json()).count).toBe(1); + } finally { + await context.close(); + } + + const other = await startBrowserServer(); + try { + expect(other.url).not.toBe(baseURL); + expect((await (await fetch(`${other.url}${search(marker)}`)).json()).count).toBe(0); + } finally { + await other.stop(); + } +}); diff --git a/tests/browser/workflows.spec.mjs b/tests/browser/workflows.spec.mjs index 091bfb6..ab5d842 100644 --- a/tests/browser/workflows.spec.mjs +++ b/tests/browser/workflows.spec.mjs @@ -1,5 +1,5 @@ import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs'; -import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import { test, expect } from './fixtures.mjs'; test.beforeEach(async ({ request, page, browserName }) => { await startCoverage(page, browserName); @@ -64,11 +64,16 @@ test('actual Tags page reloads the selected filter and opens its image viewer', await page.goto('/prompt_manager/admin#/tags'); await page.locator('[data-tag-name="cats"]').click(); await expect(page.locator('#tagPromptsGrid')).toContainText('Cat portrait'); + const dogs = page.waitForResponse(response => response.url().includes('/tags/dogs/prompts?')); await page.locator('[data-tag-name="dogs"]').click(); + const [shown] = (await (await dogs).json()).prompts[0].images; await expect(page.locator('#tagPromptsGrid')).toContainText('Dog landscape'); await expect(page.locator('#tagPromptsGrid')).not.toContainText('Cat portrait'); await page.locator('#tagPromptsGrid img').first().click(); await expect(page.locator('.viewer-canvas img')).toBeVisible(); + // The viewer's title names the shown image through the page's title callback. ViewerJS renders it only once the + // image is viewed, so closing at "visible" alone left the callback unrun on slower hosts (CI coverage missed it). + await expect(page.locator('.viewer-title')).toHaveText(shown.filename); await page.locator('.viewer-container').getByRole('button', { name: 'Close', exact: true }).click(); await expect(page.locator('.viewer-container')).toHaveCount(0); }); diff --git a/tests/browser/workspace-activity.spec.mjs b/tests/browser/workspace-activity.spec.mjs index 3970de0..05cb2d7 100644 --- a/tests/browser/workspace-activity.spec.mjs +++ b/tests/browser/workspace-activity.spec.mjs @@ -1,5 +1,5 @@ import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs'; -import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import { test, expect } from './fixtures.mjs'; import { seedConcept, openConcept, openProduction } from './concept-inventory.mjs'; // Activity (#249 Activity-1): the concept's Activity page (app.js renderActivity 732–738), its top-bar button diff --git a/tests/browser/workspace-history.spec.mjs b/tests/browser/workspace-history.spec.mjs index c28e591..405718c 100644 --- a/tests/browser/workspace-history.spec.mjs +++ b/tests/browser/workspace-history.spec.mjs @@ -1,5 +1,5 @@ import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs'; -import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import {test,expect} from './fixtures.mjs'; import {readFile,writeFile} from 'node:fs/promises'; import {createHash} from 'node:crypto'; diff --git a/tests/browser/workspace-images.spec.mjs b/tests/browser/workspace-images.spec.mjs index 99dbbba..c600759 100644 --- a/tests/browser/workspace-images.spec.mjs +++ b/tests/browser/workspace-images.spec.mjs @@ -1,5 +1,5 @@ import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs'; -import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import { test, expect } from './fixtures.mjs'; import { seedConcept, openConcept, openProduction } from './concept-inventory.mjs'; const workspace = '/prompt_manager/workspace#/images'; diff --git a/tests/browser/workspace-library-editor.spec.mjs b/tests/browser/workspace-library-editor.spec.mjs index 7560fbf..cac5777 100644 --- a/tests/browser/workspace-library-editor.spec.mjs +++ b/tests/browser/workspace-library-editor.spec.mjs @@ -1,6 +1,6 @@ import {writeFile} from 'node:fs/promises'; import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs'; -import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import {test,expect} from './fixtures.mjs'; import {captureCanonicalState} from './history-canonical-comparison.mjs'; import {inventoryOf,mapConcept,diffInventory} from './library-editor-canonical-comparison.mjs'; diff --git a/tests/browser/workspace-library.spec.mjs b/tests/browser/workspace-library.spec.mjs index bca40b4..85a9ca3 100644 --- a/tests/browser/workspace-library.spec.mjs +++ b/tests/browser/workspace-library.spec.mjs @@ -1,5 +1,5 @@ import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs'; -import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import {test,expect} from './fixtures.mjs'; const route='/prompt_manager/workspace#/library'; const cards=page=>page.locator('article[data-prompt-id]'); @@ -25,6 +25,8 @@ async function loadShelfPages(page,more,pages) { await more.evaluate(node=>node.click());await response;await expect(more).toHaveAttribute('aria-disabled','false'); } } +/** Resolves in the rendering frame after every frame callback the page had already queued (callbacks run in request order). */ +const nextFrame=page=>page.evaluate(()=>new Promise(resolve=>requestAnimationFrame(()=>resolve()))); async function openHistory(page,text='Cat portrait') { await cards(page).filter({has:page.getByText(text,{exact:true})}).getByRole('button',{name:'Explore history',exact:true}).click(); await expect(history(page)).toBeVisible(); @@ -103,7 +105,8 @@ test('library infinite loading keeps unique parents and keyboard-accessible hist const appended=await (await nextPage).json();expect(appended.items.length).toBeGreaterThan(0);hasMore=appended.has_more; if(hasMore)await expect(more).toHaveAttribute('aria-disabled','false'); } - await expect(more).toBeHidden();expect(await cards(page).count()).toBeLessThan(80); + // The last page's render queued the window update for the next frame; count after that frame, not before it. + await expect(more).toBeHidden();await nextFrame(page);expect(await cards(page).count()).toBeLessThan(80); const ids=await cards(page).evaluateAll(nodes=>nodes.map(node=>node.dataset.promptId));expect(new Set(ids).size).toBe(ids.length); await expect(page.getByRole('button',{name:/next page|previous page/i})).toHaveCount(0); const firstId=await cards(page).first().getAttribute('data-prompt-id'); @@ -376,14 +379,18 @@ async function observeLibrary(page, hold=false) { holdFrames(){holdingFrames=true;}, releaseFrames(){holdingFrames=false;const callbacks=[...heldFrames.values()];heldFrames.clear();for(const callback of callbacks)callback();}, equal(){for(const record of records){const entries=[...record.latest.values()].filter(entry=>record.targets.has(entry.target));if(entries.length)record.callback(entries,record.native);}}, + // Resolves once two consecutive rendering frames find no Library frame queued. Each check runs inside a native + // frame callback, so a frame the page queues later in that update (a scroll event, a ResizeObserver delivery) is + // still pending there. A timer is not a frame: under CPU load a 60 ms wait could end before the scroll's frame. + // A frame that never drains (a rAF loop, or settling while frames are held) fails after `limit` frames with the stats. + settled(limit=600){return new Promise((resolve,reject)=>{let quiet=0,frames=0;const check=()=>{frames++;quiet=pending.size?0:quiet+1; + if(quiet>=2)resolve();else if(frames>=limit)reject(new Error(`Library never settled within ${limit} frames: ${JSON.stringify(window.libraryObservation.stats())}`));else raf(check);};raf(check);});}, stats(){return {requested,pending:pending.size,heldFrames:heldFrames.size,measurements:records.reduce((sum,record)=>sum+record.entries.size,0),mutations,unmounted,targets:records.reduce((sum,record)=>sum+record.targets.size,0)};}, }; },{hold}); } async function settleLibrary(page) { - await expect.poll(async()=>page.evaluate(()=>window.libraryObservation.stats().pending)).toBe(0); - await page.evaluate(()=>new Promise(resolve=>setTimeout(resolve,60))); - await expect.poll(async()=>page.evaluate(()=>window.libraryObservation.stats().pending)).toBe(0); + await page.evaluate(()=>window.libraryObservation.settled()); } async function firstVisibleParent(page) { return cards(page).evaluateAll(nodes=>{ @@ -683,6 +690,17 @@ test('title and Explore history are buttons that open history',async({page,reque await page.keyboard.press('ArrowLeft');await expect(cards(page).nth(1).locator('.card-title > button')).toBeFocused(); }); +test('settling fails with the observation stats when a Library frame stays queued',async({page,request})=>{ + await request.post('/__test__/reset',{data:{native_library:true,extra_prompts:200}}); + await observeLibrary(page);await page.addInitScript(()=>{window.IntersectionObserver=undefined;}); + await page.goto(route);await expect(cards(page).first()).toBeVisible();await settleLibrary(page); + await page.evaluate(()=>{window.libraryObservation.holdFrames();dispatchEvent(new Event('scroll'));}); + await expect.poll(()=>page.evaluate(()=>window.libraryObservation.stats().heldFrames)).toBeGreaterThan(0); + const failure=await page.evaluate(()=>window.libraryObservation.settled(30).then(()=>'settled',error=>error.message)); + expect(failure).toMatch(/^Library never settled within 30 frames: \{.*"heldFrames":[1-9]/); + await page.evaluate(()=>window.libraryObservation.releaseFrames());await settleLibrary(page); +}); + test('virtualized rows keep measured heights with the clamp',async({page,request,isMobile})=>{ await request.post('/__test__/reset',{data:{native_library:true,extra_prompts:200}}); await observeLibrary(page);await page.addInitScript(()=>{window.IntersectionObserver=undefined;}); diff --git a/tests/browser/workspace-saved-views.spec.mjs b/tests/browser/workspace-saved-views.spec.mjs index e8ad366..ce8f09f 100644 --- a/tests/browser/workspace-saved-views.spec.mjs +++ b/tests/browser/workspace-saved-views.spec.mjs @@ -1,5 +1,5 @@ import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs'; -import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import { test, expect } from './fixtures.mjs'; // Saved views (#249 D16, maintainer ruling 2026-10-21): the toolbar's Save view (concept app.js a359, a1330–1346), // the sidebar SAVED VIEWS rows (a331–335) and `#/saved/` (a1158–1162), on Images and the Prompt library. diff --git a/tests/browser/workspace-visual.spec.mjs b/tests/browser/workspace-visual.spec.mjs index a9e2e28..8896505 100644 --- a/tests/browser/workspace-visual.spec.mjs +++ b/tests/browser/workspace-visual.spec.mjs @@ -3,7 +3,7 @@ import { readFile } from 'node:fs/promises'; import { createHash } from 'node:crypto'; import { captureCanonicalState, historyGeometry, compareCanonicalHistory, nativeDialogState, exerciseCurrentPairSend, assertRecordedHistory } from './history-canonical-comparison.mjs'; import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs'; -import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import { test, expect } from './fixtures.mjs'; test.use({ locale: 'en-US', timezoneId: 'UTC', reducedMotion: 'reduce' }); const workspace = '/prompt_manager/workspace'; diff --git a/tests/browser/workspace-vocabulary.spec.mjs b/tests/browser/workspace-vocabulary.spec.mjs index 913c680..9cc0742 100644 --- a/tests/browser/workspace-vocabulary.spec.mjs +++ b/tests/browser/workspace-vocabulary.spec.mjs @@ -1,6 +1,6 @@ import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs'; import {writeFile} from 'node:fs/promises'; -import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import {test,expect} from './fixtures.mjs'; import {captureCanonicalState} from './history-canonical-comparison.mjs'; import {pageInventoryOf,inventoryOf,diffInventory,mapVocabularyConcept,mapVocabularyDialog} from './vocabulary-canonical-comparison.mjs'; diff --git a/tests/browser/workspace.spec.mjs b/tests/browser/workspace.spec.mjs index e353121..c8ed505 100644 --- a/tests/browser/workspace.spec.mjs +++ b/tests/browser/workspace.spec.mjs @@ -1,5 +1,5 @@ import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs'; -import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import { test, expect } from './fixtures.mjs'; const workspace = '/prompt_manager/workspace'; const statistic = (page, label) => page.locator('#workspace-main dt') diff --git a/tests/test_browser_server_lifetime.py b/tests/test_browser_server_lifetime.py new file mode 100644 index 0000000..eb68bbb --- /dev/null +++ b/tests/test_browser_server_lifetime.py @@ -0,0 +1,83 @@ +"""The page-workflow test server ends with its launcher (#274 review M2). + +A Playwright worker that is SIGKILLed never runs its fixture teardown, so the +server it started must notice on its own: with ``--exit-on-stdin-eof`` it shuts +down when the launcher's end of its stdin pipe closes, and still removes its +temp root. This runs on every OS in CI, where Windows is the strict case. +""" + +import os +import subprocess +import sys +import tempfile +import time +import unittest +import urllib.request +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +SERVER_TIMEOUT_S = 60 + + +class TestBrowserServerLifetime(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + + def start(self, *extra): + env = {**os.environ, "TMPDIR": self.tmp.name} + env.update(TEMP=self.tmp.name, TMP=self.tmp.name) + process = subprocess.Popen( + [sys.executable, "-m", "tests.browser.server", "--port", "0", *extra], + cwd=REPO_ROOT, + env=env, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + def stop(): + if process.poll() is None: + process.kill() + process.wait(timeout=SERVER_TIMEOUT_S) + # communicate() would flush a closed stdin, which Python < 3.13 refuses. + for stream in (process.stdin, process.stdout, process.stderr): + stream.close() + + self.addCleanup(stop) + line = process.stdout.readline().decode() + self.assertTrue(line.startswith("PM_TEST_SERVER_PORT="), line) + url = f"http://127.0.0.1:{int(line.split('=', 1)[1])}/prompt_manager/test" + deadline = time.monotonic() + SERVER_TIMEOUT_S + while True: + try: + with urllib.request.urlopen(url, timeout=1) as response: # nosec B310 + self.assertEqual(response.status, 200) + return process + except OSError: + self.assertIsNone(process.poll(), "the server exited early") + self.assertLess(time.monotonic(), deadline, "never ready") + time.sleep(0.1) + + def roots(self): + return sorted(path.name for path in Path(self.tmp.name).glob("pm-browser-*")) + + def test_server_exits_and_removes_its_temp_root_when_stdin_closes(self): + process = self.start("--exit-on-stdin-eof") + self.assertEqual(len(self.roots()), 1, "the server works in its own temp root") + process.stdin.close() + code = process.wait(timeout=SERVER_TIMEOUT_S) + self.assertEqual( + code, 0, process.stderr.read().decode(errors="replace")[-4000:] + ) + self.assertEqual(self.roots(), [], "the temp root and its database are removed") + + def test_server_keeps_running_with_stdin_closed_without_the_flag(self): + process = self.start() + process.stdin.close() + with self.assertRaises(subprocess.TimeoutExpired): + process.wait(timeout=1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/vendor/browser-server-fixture.test.mjs b/tests/vendor/browser-server-fixture.test.mjs new file mode 100644 index 0000000..9b05c4b --- /dev/null +++ b/tests/vendor/browser-server-fixture.test.mjs @@ -0,0 +1,234 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createServer } from 'node:net'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { setTimeout as delay } from 'node:timers/promises'; +import { startBrowserServer, createServerSupervisor, attachServerStderr } from '../../tools/vendor/start-browser-server.mjs'; +import { workflowWorkers } from '../../tools/vendor/test-workers.mjs'; + +const fakeServer = fileURLToPath(new URL('./fake-browser-server.mjs', import.meta.url)); +const fake = (...extra) => ({ command: process.execPath, args: port => [fakeServer, '--port', String(port), ...extra] }); +const REAL_SERVER_TIMEOUT_MS = 90_000; +const launcher = fileURLToPath(new URL('../../tools/vendor/start-browser-server.mjs', import.meta.url)); + +/** Polls `check` until it is truthy, failing with `what` after `ms`. */ +async function waitFor(check, what, ms = 5_000) { + const deadline = Date.now() + ms; + while (!check()) { + if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}`); + await delay(20); + } +} + +async function writeStderr(server, text) { + await (await fetch(`${server.url}/__stderr?text=${encodeURIComponent(text)}`)).text(); + await waitFor(() => server.stderrTail().endsWith(text.trimEnd()), `stderr ${JSON.stringify(text)}`); +} + +/** A process that exited counts as gone even while a non-reaping PID 1 (CI's container) leaves it a zombie. */ +function alive(pid) { + try { process.kill(pid, 0); } catch { return false; } + try { return readFileSync(`/proc/${pid}/stat`, 'utf8').replace(/^.*\) /s, '')[0] !== 'Z'; } catch { return true; } +} + +/** Starts a server from a separate Node worker process, SIGKILLs that worker and returns the server's pid. */ +async function serverOfKilledWorker(options) { + const script = `import { startBrowserServer } from ${JSON.stringify(pathToFileURL(launcher).href)}; + const server = await startBrowserServer(${options}); + console.log('SERVER_PID=' + server.pid); + setInterval(() => {}, 1000);`; + const worker = spawn(process.execPath, ['--input-type=module', '-e', script], { stdio: ['ignore', 'pipe', 'inherit'] }); + let out = ''; + worker.stdout.setEncoding('utf8'); + worker.stdout.on('data', chunk => { out += chunk; }); + await waitFor(() => /SERVER_PID=\d+\n/.test(out) || worker.exitCode !== null, 'the worker to start its server', REAL_SERVER_TIMEOUT_MS); + const pid = Number(/SERVER_PID=(\d+)/.exec(out)?.[1]); + assert.ok(pid > 0, `the worker started a server (output: ${out})`); + assert.ok(alive(pid), 'the server runs while its worker lives'); + worker.kill('SIGKILL'); + await once(worker, 'exit'); + return pid; +} + +function listen(port = 0) { + return new Promise((resolve, reject) => { + const server = createServer(); + server.once('error', reject); + server.listen(port, '127.0.0.1', () => resolve(server)); + }); +} + +async function releasedPort() { + const probe = await listen(); + const { port } = probe.address(); + await new Promise(resolve => probe.close(resolve)); + return port; +} + +test('free port allocation never reuses a bound port', async t => { + const held = await listen(); + t.after(() => held.close()); + const heldPort = held.address().port; + const servers = await Promise.all(Array.from({ length: 6 }, () => startBrowserServer(fake()))); + t.after(() => Promise.all(servers.map(server => server.stop()))); + const ports = servers.map(server => server.port); + for (const server of servers) { + assert.ok(Number.isInteger(server.port) && server.port > 0 && server.port < 65536, `port ${server.port} is a TCP port`); + assert.equal(server.url, `http://127.0.0.1:${server.port}`); + assert.equal(await (await fetch(`${server.url}/prompt_manager/test`)).text(), 'ready'); + } + assert.ok(!ports.includes(heldPort), 'a port another process holds is never handed out'); + assert.equal(new Set(ports).size, ports.length, 'concurrent servers never share a port'); +}); + +test('startBrowserServer asks the server to bind port 0 and reads the port it announces', async () => { + const asked = []; + const server = await startBrowserServer({ command: process.execPath, args: port => { asked.push(port); return fake().args(port); } }); + try { + assert.deepEqual(asked, [0], 'no port is reserved and released before the server binds'); + assert.ok(server.port > 0); + } finally { + await server.stop(); + } +}); + +test('startBrowserServer serves the readiness page on a requested port and stops cleanly', async () => { + const port = await releasedPort(); + const server = await startBrowserServer({ port, ...fake() }); + assert.equal(server.port, port); + assert.equal(server.url, `http://127.0.0.1:${port}`); + assert.equal(await (await fetch(`${server.url}/prompt_manager/test`)).text(), 'ready'); + await server.stop(); + await assert.rejects(fetch(`${server.url}/prompt_manager/test`)); +}); + +test('startBrowserServer reports a startup failure with the server\'s stderr tail', async () => { + await assert.rejects(startBrowserServer(fake('--fail')), error => { + assert.match(error.message, /exited with code 4 before it was ready/); + assert.match(error.message, /last fake stderr line/); + return true; + }); +}); + +test('startBrowserServer gives up on a server that never announces a port or becomes ready', async () => { + const silent = { command: process.execPath, args: () => ['-e', 'setInterval(() => {}, 1000)'] }; + await assert.rejects(startBrowserServer({ ...silent, timeoutMs: 500 }), /not ready .* within 500 ms/); +}); + +test('worker server restarts after an unexpected exit', async t => { + const supervisor = createServerSupervisor({ start: () => startBrowserServer(fake()) }); + t.after(() => supervisor.stop()); + const first = await supervisor.ensure(); + assert.equal(supervisor.crash(first), null, 'a running server reports no crash'); + await fetch(`${first.url}/__exit`); + await first.exited; + const crash = supervisor.crash(first); + assert.match(crash.message, /test server on port \d+ exited unexpectedly \(code 3\)/); + assert.match(crash.message, /crashed on purpose/); + const second = await supervisor.ensure(); + assert.notEqual(second, first, 'the next test gets a fresh server'); + assert.equal(await (await fetch(`${second.url}/prompt_manager/test`)).text(), 'ready'); + assert.equal(await supervisor.ensure(), second, 'a healthy server is reused'); +}); + +test('supervisor reports a startup failure once and tries again on the next ensure', async t => { + let attempts = 0; + const supervisor = createServerSupervisor({ + start: async () => { + attempts += 1; + if (attempts === 1) throw new Error('ImportError: no module'); + return startBrowserServer(fake()); + }, + }); + t.after(() => supervisor.stop()); + await assert.rejects(supervisor.ensure(), /ImportError/); + assert.equal(attempts, 1, 'a failed startup is not retried behind the caller\'s back'); + const server = await supervisor.ensure(); + assert.equal(attempts, 2); + assert.equal(await (await fetch(`${server.url}/prompt_manager/test`)).text(), 'ready'); +}); + +test('the port announcement counts only once its line is complete', async t => { + const server = await startBrowserServer({ ...fake('--split-announce'), timeoutMs: 5_000 }); + t.after(() => server.stop()); + assert.equal(await (await fetch(`${server.url}/prompt_manager/test`)).text(), 'ready'); + assert.equal(server.url, `http://127.0.0.1:${server.port}`); +}); + +test('server stderr since a mark holds only what came after it, bounded to the newest output', async t => { + const server = await startBrowserServer({ ...fake(), stderrLogChars: 64 }); + t.after(() => server.stop()); + await writeStderr(server, 'earlier test\n'); + const mark = server.stderrMark(); + assert.equal(server.stderrSince(mark), ''); + await writeStderr(server, 'Traceback: this test\n'); + assert.equal(server.stderrSince(mark), 'Traceback: this test\n'); + const flood = `${'x'.repeat(80)}newest\n`; + await writeStderr(server, flood); + const kept = server.stderrSince(mark); + assert.match(kept, /^\[earlier server stderr dropped\]\n/); + assert.ok(kept.endsWith('newest\n'), 'the newest output is kept'); + assert.ok(kept.length <= 64 + '[earlier server stderr dropped]\n'.length, `bounded, got ${kept.length} chars`); +}); + +test('a failed test gets the server stderr written during it attached; a passing one does not', async t => { + const server = await startBrowserServer(fake()); + t.after(() => server.stop()); + const attached = []; + const info = status => ({ status, expectedStatus: 'passed', attach: async (name, options) => { attached.push({ name, ...options }); } }); + await writeStderr(server, 'before the test\n'); + const mark = server.stderrMark(); + await attachServerStderr(info('failed'), server, mark); + assert.deepEqual(attached, [], 'nothing to attach when the server wrote nothing'); + await writeStderr(server, 'Traceback (most recent call last): boom\n'); + await attachServerStderr(info('passed'), server, mark); + assert.deepEqual(attached, [], 'a passing test attaches nothing'); + await attachServerStderr(info('timedOut'), server, mark); + assert.equal(attached.length, 1); + assert.equal(attached[0].name, 'server-stderr'); + assert.equal(attached[0].contentType, 'text/plain'); + assert.equal(String(attached[0].body), 'Traceback (most recent call last): boom\n'); +}); + +test('a server exits with its worker even when the worker is SIGKILLed', async () => { + const pid = await serverOfKilledWorker(`{ command: process.execPath, args: port => [${JSON.stringify(fakeServer)}, '--port', String(port), '--exit-on-stdin-eof'] }`); + await waitFor(() => !alive(pid), `server ${pid} to exit after its worker died`); +}); + +test('tests/browser/server.py exits with a SIGKILLed worker', { timeout: 2 * REAL_SERVER_TIMEOUT_MS }, async () => { + const pid = await serverOfKilledWorker(`{ timeoutMs: ${REAL_SERVER_TIMEOUT_MS} }`); + await waitFor(() => !alive(pid), `server.py ${pid} to exit after its worker died`, 15_000); +}); + +test('tests/browser/server.py binds an OS-chosen port for --port 0 and still honours --port N', { timeout: 2 * REAL_SERVER_TIMEOUT_MS }, async t => { + const held = await listen(); + t.after(() => held.close()); + const chosen = await startBrowserServer({ timeoutMs: REAL_SERVER_TIMEOUT_MS }); + try { + assert.notEqual(chosen.port, held.address().port); + assert.equal((await fetch(`${chosen.url}/prompt_manager/test`)).status, 200); + } finally { + await chosen.stop(); + } + const port = await releasedPort(); + const fixed = await startBrowserServer({ port, timeoutMs: REAL_SERVER_TIMEOUT_MS }); + try { + assert.equal(fixed.port, port); + assert.equal((await fetch(`http://127.0.0.1:${port}/prompt_manager/test`)).status, 200); + } finally { + await fixed.stop(); + } +}); + +test('worker count follows PM_TEST_WORKERS, then one in CI, then a quarter of the CPUs up to eight', () => { + assert.equal(workflowWorkers({ env: { PM_TEST_WORKERS: '3', CI: '1' }, cpus: 32 }), 3); + assert.equal(workflowWorkers({ env: { CI: '1' }, cpus: 32 }), 1); + assert.equal(workflowWorkers({ env: {}, cpus: 32 }), 8); + assert.equal(workflowWorkers({ env: {}, cpus: 12 }), 3); + assert.equal(workflowWorkers({ env: {}, cpus: 2 }), 1); + assert.throws(() => workflowWorkers({ env: { PM_TEST_WORKERS: 'many' }, cpus: 8 }), /PM_TEST_WORKERS must be a positive integer/); + assert.throws(() => workflowWorkers({ env: { PM_TEST_WORKERS: '0' }, cpus: 8 }), /PM_TEST_WORKERS must be a positive integer/); +}); diff --git a/tests/vendor/ci-workflow.test.mjs b/tests/vendor/ci-workflow.test.mjs new file mode 100644 index 0000000..b2d8064 --- /dev/null +++ b/tests/vendor/ci-workflow.test.mjs @@ -0,0 +1,223 @@ +// Structural checks on .github/workflows/viewerjs-check.yml. The tooling has no YAML parser, so these +// read the workflow's fixed two-space layout: jobs at two spaces, their keys at four, steps at six. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdtemp, mkdir, readdir, rm, writeFile } from 'node:fs/promises'; +import { readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +const workflow = readFileSync(new URL('../../.github/workflows/viewerjs-check.yml', import.meta.url), 'utf8'); +const PINNED_IMAGE = /^\s+image: mcr\.microsoft\.com\/playwright:v\d+\.\d+\.\d+-noble$/m; + +function jobs(text) { + const lines = text.split('\n'); + const start = lines.indexOf('jobs:'); + assert.notEqual(start, -1, 'workflow has a jobs section'); + const found = new Map(); + let current = null; + for (const line of lines.slice(start + 1)) { + const header = /^ {2}([\w-]+):\s*$/.exec(line); + if (header) { + current = header[1]; + found.set(current, []); + } else if (current && (line.startsWith(' ') || line.trim() === '' || /^ {2}#/.test(line))) { + found.get(current).push(line); + } else if (current && /^\S/.test(line)) { + break; + } + } + return new Map([...found].map(([id, body]) => [id, body.join('\n')])); +} + +function steps(jobText) { + const parts = jobText.split(/^(?= {6}- )/m); + return parts.slice(1); +} + +function runScript(stepText) { + const lines = stepText.split('\n'); + const index = lines.findIndex(line => /^ {8}run: \|\s*$/.test(line)); + assert.notEqual(index, -1, 'step has a block run script'); + const body = []; + for (const line of lines.slice(index + 1)) { + if (line.trim() !== '' && !line.startsWith(' ')) break; + body.push(line.slice(10)); + } + return body.join('\n'); +} + +function needsOf(jobText) { + const match = /^ {4}needs: \[([^\]]*)\]\s*$/m.exec(jobText) ?? /^ {4}needs: ([\w-]+)\s*$/m.exec(jobText); + assert.ok(match, 'job declares needs'); + return match[1].split(',').map(value => value.trim()).filter(Boolean); +} + +function matrixList(jobText, key) { + const match = new RegExp(`^ {8}${key}: \\[([^\\]]*)\\]\\s*$`, 'm').exec(jobText); + assert.ok(match, `job has a ${key} matrix`); + return match[1].split(',').map(value => value.trim()); +} + +const allJobs = jobs(workflow); +const browserJobIds = [...allJobs].filter(([, body]) => /npm run --prefix tools\/vendor test:(workflows|coverage)/.test(body)) + .map(([id]) => id); +const mergeJob = () => allJobs.get('frontend-coverage-merge'); +const verifyStep = () => steps(mergeJob()).find(step => /name: Verify every shard's raw coverage arrived/.test(step)); + +test('viewerjs keeps its name and needs every frontend job', () => { + const viewerjs = allJobs.get('viewerjs'); + assert.ok(viewerjs, 'the required context is the job id viewerjs'); + assert.doesNotMatch(viewerjs, /^ {4}name:/m, 'no display name may replace the context "viewerjs"'); + const needs = needsOf(viewerjs); + const others = [...allJobs.keys()].filter(id => id !== 'viewerjs'); + assert.deepEqual([...needs].sort(), [...others].sort(), 'viewerjs needs every other job in the workflow'); + assert.deepEqual([...needs].sort(), ['browser-workflows', 'frontend-coverage', 'frontend-coverage-merge']); + assert.match(viewerjs, /^ {4}if: always\(\)$/m, 'viewerjs runs (and fails) when a needed job failed or was skipped'); + const script = runScript(steps(viewerjs).find(step => /run: \|/.test(step))); + for (const id of needs) { + const variable = new RegExp(`^ {10}([A-Z_]+): \\$\\{\\{ needs\\.${id}\\.result \\}\\}$`, 'm').exec(viewerjs); + assert.ok(variable, `${id}'s result reaches the check`); + assert.match(script, new RegExp(`test "\\$${variable[1]}" = success`), `${id} must be exactly success`); + } +}); + +test('viewerjs fails unless every needed job succeeded', () => { + const viewerjs = allJobs.get('viewerjs'); + const script = runScript(steps(viewerjs).find(step => /run: \|/.test(step))); + const variables = [...viewerjs.matchAll(/^ {10}([A-Z_]+): \$\{\{ needs\.[\w-]+\.result \}\}$/gm)].map(match => match[1]); + const run = env => { + try { + execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', script], { env: { PATH: process.env.PATH, ...env }, stdio: 'pipe' }); + return true; + } catch { + return false; + } + }; + const allSuccess = Object.fromEntries(variables.map(name => [name, 'success'])); + assert.equal(run(allSuccess), true); + for (const name of variables) { + for (const result of ['failure', 'cancelled', 'skipped']) { + assert.equal(run({ ...allSuccess, [name]: result }), false, `${name}=${result} must fail viewerjs`); + } + } +}); + +test('coverage runs as three shards that each upload their raw coverage', () => { + const coverage = allJobs.get('frontend-coverage'); + assert.deepEqual(matrixList(coverage, 'shard'), ['1', '2', '3']); + assert.match(coverage, /^ {4}name: frontend coverage \(shard \$\{\{ matrix\.shard \}\}\/3\)$/m); + assert.match(coverage, /npm run --prefix tools\/vendor test:coverage:shard -- \$\{\{ matrix\.shard \}\}\/3/); + const upload = steps(coverage).find(step => /name: coverage-raw-\$\{\{ matrix\.shard \}\}/.test(step)); + assert.ok(upload, 'each shard uploads coverage-raw-N'); + assert.match(upload, /^ {10}path: coverage\/js\/raw\/?$/m); + assert.match(upload, /^ {10}retention-days: 3$/m); + assert.match(upload, /^ {10}if-no-files-found: error$/m); +}); + +test('merge job downloads every shard and runs the merge-only report and check', () => { + const merge = mergeJob(); + assert.ok(merge, 'frontend-coverage-merge exists'); + assert.deepEqual(needsOf(merge), ['frontend-coverage']); + assert.match(merge, /^ {4}name: frontend coverage \(merge\)$/m); + const download = steps(merge).find(step => /uses: actions\/download-artifact@/.test(step)); + assert.ok(download, 'merge downloads the shard artifacts'); + assert.match(download, /^ {10}pattern: coverage-raw-\*$/m); + assert.doesNotMatch(download, /merge-multiple: true/, 'each artifact keeps its own directory so the check can count them'); + assert.ok(verifyStep(), 'merge has an explicit artifact count check'); + assert.match(verifyStep(), new RegExp(`^ {10}EXPECTED_SHARDS: ${matrixList(allJobs.get('frontend-coverage'), 'shard').length}$`, 'm'), + 'the expected count matches the shard matrix'); + const order = steps(merge).map(step => (/name: (.*)/.exec(step) ?? [, step.trim().split('\n')[0]])[1]); + const verifyAt = order.findIndex(name => /Verify every shard/.test(name)); + const mergeAt = steps(merge).findIndex(step => /npm run --prefix tools\/vendor coverage:merge/.test(step)); + assert.ok(verifyAt >= 0 && mergeAt > verifyAt, 'the count check runs before coverage:merge'); + const report = steps(merge).find(step => /name: frontend-test-reports$/m.test(step)); + assert.ok(report, 'merge uploads the HTML report as frontend-test-reports'); + assert.match(report, /!coverage\/js\/raw\/\*\*/); +}); + +async function stagedShards(root, shards) { + const staged = join(root, 'staged'); + for (const [shard, files] of shards) { + const directory = join(staged, `coverage-raw-${shard}`); + await mkdir(directory, { recursive: true }); + for (const file of files) await writeFile(join(directory, file), '{"result":[]}'); + } + await mkdir(staged, { recursive: true }); + return staged; +} + +async function verify(t, shards) { + const root = await mkdtemp(join(tmpdir(), 'pm-ci-merge-')); + t.after(() => rm(root, { recursive: true, force: true })); + const staged = await stagedShards(root, shards); + try { + execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', runScript(verifyStep())], { + cwd: root, stdio: 'pipe', + env: { PATH: process.env.PATH, STAGED_RAW: staged, EXPECTED_SHARDS: '3' }, + }); + } catch (error) { + return { ok: false, output: `${error.stdout}${error.stderr}` }; + } + return { ok: true, raw: (await readdir(join(root, 'coverage/js/raw'))).sort() }; +} + +test('merge refuses when a shard\'s raw coverage is missing', async t => { + const missing = await verify(t, [[1, ['a.json']], [3, ['c.json']]]); + assert.equal(missing.ok, false); + assert.match(missing.output, /2 of 3/); + const empty = await verify(t, [[1, ['a.json']], [2, []], [3, ['c.json']]]); + assert.equal(empty.ok, false, 'an artifact without raw files counts as missing'); + const collision = await verify(t, [[1, ['a.json']], [2, ['a.json']], [3, ['c.json']]]); + assert.equal(collision.ok, false, 'two shards writing the same file name must not silently drop one'); + const complete = await verify(t, [[1, ['a.json', 'node.json']], [2, ['b.json']], [3, ['c.json']]]); + assert.deepEqual(complete, { ok: true, raw: ['a.json', 'b.json', 'c.json', 'node.json'] }); +}); + +test('every browser job sets PM_TEST_WORKERS and runs in the pinned image', () => { + assert.deepEqual([...browserJobIds].sort(), ['browser-workflows', 'frontend-coverage']); + for (const id of browserJobIds) { + const body = allJobs.get(id); + assert.match(body, /^ {6}PM_TEST_WORKERS: 3$/m, `${id} sets PM_TEST_WORKERS for the whole job`); + assert.match(body, PINNED_IMAGE, `${id} keeps the pinned Playwright image`); + assert.match(body, /PIP_SHA256: [0-9a-f]{64}/, `${id} keeps the pinned pip bootstrap`); + assert.match(body, /^ {4}timeout-minutes: \d+$/m); + } + assert.match(mergeJob(), PINNED_IMAGE, 'the merge sees the same workspace paths as the shards'); +}); + +test('page workflows run each browser in two shards', () => { + const workflows = allJobs.get('browser-workflows'); + assert.deepEqual(matrixList(workflows, 'browser'), ['firefox', 'webkit']); + assert.deepEqual(matrixList(workflows, 'shard'), ['1', '2']); + assert.match(workflows, /test:workflows -- --project \$\{\{ matrix\.browser \}\} --shard \$\{\{ matrix\.shard \}\}\/2/); + assert.match(workflows, /^ {6}fail-fast: false$/m); + assert.match(workflows, /name: browser-traces-\$\{\{ matrix\.browser \}\}-shard-\$\{\{ matrix\.shard \}\}/); +}); + +test('every npm ci restores the npm download cache keyed on the vendor lockfile first', () => { + const installing = [...allJobs].filter(([, body]) => /npm ci --prefix tools\/vendor/.test(body)); + assert.deepEqual(installing.map(([id]) => id).sort(), ['browser-workflows', 'frontend-coverage', 'frontend-coverage-merge']); + for (const [id, body] of installing) { + const jobSteps = steps(body); + const cacheAt = jobSteps.findIndex(step => /uses: actions\/cache@v\d+/.test(step)); + const installAt = jobSteps.findIndex(step => /npm ci --prefix tools\/vendor/.test(step)); + assert.ok(cacheAt >= 0 && cacheAt < installAt, `${id} restores the npm cache before npm ci`); + assert.match(jobSteps[cacheAt], /^ {10}path: ~\/\.npm$/m, `${id} caches npm's download cache, not node_modules`); + assert.match(jobSteps[cacheAt], /hashFiles\('tools\/vendor\/package-lock\.json'\)/, `${id} keys the cache on the lockfile`); + assert.match(jobSteps[installAt], /npm ci --prefix tools\/vendor --ignore-scripts/, `${id} still installs without scripts`); + // The key names the job's own pinned image, so an image bump starts a fresh cache. + const image = /^ {6}image: mcr\.microsoft\.com\/playwright:(v\d+\.\d+\.\d+)-noble$/m.exec(body)?.[1]; + assert.ok(image, `${id} runs in the pinned Playwright image`); + assert.match(jobSteps[cacheAt], new RegExp(`key: npm-\\$\\{\\{ runner\\.os \\}\\}-playwright-${image.replaceAll('.', '\\.')}-\\$\\{\\{ hashFiles`), + `${id} keys the cache on its image ${image}`); + } +}); + +test('browser job timeouts keep a real margin over measured runs', () => { + for (const id of ['frontend-coverage', 'browser-workflows']) { + const minutes = Number(/^ {4}timeout-minutes: (\d+)$/m.exec(allJobs.get(id))[1]); + assert.ok(minutes >= 20, `${id} allows at least 20 minutes, since runner speed varies`); + } +}); diff --git a/tests/vendor/fake-browser-server.mjs b/tests/vendor/fake-browser-server.mjs new file mode 100644 index 0000000..827b174 --- /dev/null +++ b/tests/vendor/fake-browser-server.mjs @@ -0,0 +1,44 @@ +// A stand-in for tests/browser/server.py: `--port N` (0 = an OS-chosen port) +// serves the readiness page and announces the bound port on stdout as +// `PM_TEST_SERVER_PORT=`, `--fail` writes to stderr and exits before +// listening, and GET /__exit makes the process die unexpectedly after writing a +// last stderr line. GET /__stderr?text=T writes T to stderr before answering. +// `--split-announce` writes the announcement in two chunks, the first ending +// mid-number. `--exit-on-stdin-eof` exits when its stdin closes, as server.py does. +import { createServer } from 'node:http'; + +const args = process.argv.slice(2); +const port = Number(args[args.indexOf('--port') + 1]); +if (args.includes('--fail')) { + process.stderr.write('fake server: refusing to start\nlast fake stderr line\n'); + process.exit(4); +} +const server = createServer((request, response) => { + const url = new URL(request.url, 'http://fake'); + if (url.pathname === '/__stderr') { + process.stderr.write(url.searchParams.get('text')); + response.end('written'); + return; + } + if (request.url === '/__exit') { + process.stderr.write('fake server: crashed on purpose\n'); + response.end('bye', () => process.exit(3)); + return; + } + response.end(request.url === '/prompt_manager/test' ? 'ready' : `pid ${process.pid}`); +}); +server.listen(port, '127.0.0.1', () => { + const line = `PM_TEST_SERVER_PORT=${server.address().port}\n`; + if (!args.includes('--split-announce')) { + process.stdout.write(`fake server: listening\n${line}`); + return; + } + // The first chunk is a complete-looking number, one digit short of the real port. + process.stdout.write(`fake server: listening\n${line.slice(0, -2)}`); + setTimeout(() => process.stdout.write(line.slice(-2)), 300); +}); +if (args.includes('--exit-on-stdin-eof')) { + process.stdin.resume(); + process.stdin.once('end', () => process.exit(0)); +} +for (const signal of ['SIGTERM', 'SIGINT']) process.once(signal, () => server.close(() => process.exit(0))); diff --git a/tests/vendor/frontend-coverage-cli.test.mjs b/tests/vendor/frontend-coverage-cli.test.mjs new file mode 100644 index 0000000..8243181 --- /dev/null +++ b/tests/vendor/frontend-coverage-cli.test.mjs @@ -0,0 +1,93 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { existsSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { plan, runCoverage } from '../../tools/vendor/frontend-coverage.mjs'; + +const steps = argv => plan(argv).map(({ step, ...options }) => (Object.keys(options).length ? { step, ...options } : step)); + +function tempPaths(t) { + const root = mkdtempSync(join(tmpdir(), 'pm-coverage-cli-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const reportRoot = join(root, 'coverage/js'); + return { root, paths: { repositoryRoot: root, tools: join(root, 'tools'), reportRoot, raw: join(reportRoot, 'raw'), sourceRoot: join(root, 'web/js') } }; +} + +function recorder() { + const calls = []; + const run = (args, env = {}) => { calls.push({ args, env }); }; + return { calls, run }; +} + +test('no arguments plans the unchanged full run', () => { + assert.deepEqual(steps([]), [ + 'clean', 'node-units', 'viewer', { step: 'workflows', shard: null }, 'report', { step: 'check', updateBaseline: false }, + ]); + assert.deepEqual(steps(['--update-baseline']), [ + 'clean', 'node-units', 'viewer', { step: 'workflows', shard: null }, 'report', { step: 'check', updateBaseline: true }, + ]); +}); + +test('--shard 2/3 plans only workflows with --shard 2/3 and no report', () => { + assert.deepEqual(steps(['--shard', '2/3']), ['prepare', { step: 'workflows', shard: '2/3' }]); + assert.deepEqual(steps(['--shard=3/3']), ['prepare', { step: 'workflows', shard: '3/3' }]); +}); + +test('--shard 1/N also runs the node units and the viewer suite', () => { + assert.deepEqual(steps(['--shard', '1/3']), ['prepare', 'node-units', 'viewer', { step: 'workflows', shard: '1/3' }]); + assert.deepEqual(steps(['--shard', '1/1']), ['prepare', 'node-units', 'viewer', { step: 'workflows', shard: '1/1' }]); +}); + +test('--merge-only plans the report and the check over existing raw coverage', () => { + assert.deepEqual(steps(['--merge-only']), ['require-raw', 'report', { step: 'check', updateBaseline: false }]); + assert.deepEqual(steps(['--merge-only', '--update-baseline']), ['require-raw', 'report', { step: 'check', updateBaseline: true }]); +}); + +test('invalid shard and mode combinations are refused', () => { + for (const argv of [['--shard'], ['--shard', '0/3'], ['--shard', '4/3'], ['--shard', '2'], ['--shard', 'a/b'], ['--shard', '1/0'], ['--shard', '1.5/3']]) { + assert.throws(() => plan(argv), /--shard needs I\/N/, argv.join(' ')); + } + assert.throws(() => plan(['--shard', '1/2', '--merge-only']), /either --shard or --merge-only/); + assert.throws(() => plan(['--shard', '1/2', '--update-baseline']), /--update-baseline .* --merge-only or a full run/); + assert.throws(() => plan(['--shard', '1/2', '--shard', '2/2']), /--shard once/); + assert.throws(() => plan(['--report']), /Unknown argument: --report/); +}); + +test('--merge-only refuses an empty raw dir', async t => { + const { paths } = tempPaths(t); + const { calls, run } = recorder(); + await assert.rejects(runCoverage({ argv: ['--merge-only'], run, paths }), /no raw coverage .*coverage\/js\/raw.* run the --shard steps first/i); + mkdirSync(paths.raw, { recursive: true }); + await assert.rejects(runCoverage({ argv: ['--merge-only'], run, paths }), /no raw coverage/i); + writeFileSync(join(paths.raw, 'notes.txt'), 'not coverage'); + await assert.rejects(runCoverage({ argv: ['--merge-only'], run, paths }), /no raw coverage/i); + assert.deepEqual(calls, [], 'nothing is reported or checked over partial or empty coverage'); +}); + +test('a shard keeps raw coverage other shards wrote and passes its shard to Playwright', async t => { + const { paths } = tempPaths(t); + mkdirSync(paths.raw, { recursive: true }); + const earlier = join(paths.raw, 'browser-earlier-shard.json'); + writeFileSync(earlier, '{"result":[]}'); + const { calls, run } = recorder(); + await runCoverage({ argv: ['--shard', '2/3'], run, paths }); + assert.ok(existsSync(earlier), 'raw files from another shard survive'); + assert.equal(calls.length, 1, 'shard 2 runs only the workflows'); + const [{ args, env }] = calls; + assert.deepEqual(args.slice(-6), ['--project', 'chromium', '--project', 'touch', '--shard', '2/3']); + assert.equal(env.PM_JS_COVERAGE, paths.raw); +}); + +test('shard 1 runs the node units into the shared raw dir, then the viewer suite, then its workflows', async t => { + const { root, paths } = tempPaths(t); + mkdirSync(join(root, 'tests/js'), { recursive: true }); + writeFileSync(join(root, 'tests/js/a.test.js'), ''); + const { calls, run } = recorder(); + await runCoverage({ argv: ['--shard', '1/3'], run, paths }); + assert.equal(calls.length, 3); + assert.deepEqual(calls[0].args, ['--test', 'tests/js/a.test.js']); + assert.equal(calls[0].env.NODE_V8_COVERAGE, paths.raw); + assert.ok(calls[1].args.includes(join(paths.tools, 'playwright.config.mjs'))); + assert.deepEqual(calls[2].args.slice(-2), ['--shard', '1/3']); +}); diff --git a/tools/vendor/frontend-coverage.mjs b/tools/vendor/frontend-coverage.mjs index ec9af1d..30de415 100644 --- a/tools/vendor/frontend-coverage.mjs +++ b/tools/vendor/frontend-coverage.mjs @@ -1,4 +1,4 @@ -import { readFileSync, readdirSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import { existsSync, readFileSync, readdirSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; import { dirname, join, relative } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { spawnSync } from 'node:child_process'; @@ -7,33 +7,69 @@ import { repositoryRoot } from './vendor-viewerjs.mjs'; import { discoverSources, checkCoverage } from '@comfyassets/toolkit/testing/coverage'; export { discoverSources, checkCoverage }; const metrics = ['lines', 'branches', 'functions', 'statements']; +const SHARD = /^([1-9]\d*)\/([1-9]\d*)$/; +const USAGE = 'usage: frontend-coverage.mjs [--update-baseline] | --shard I/N | --merge-only [--update-baseline]'; -function run(args, env = {}) { +function runNode(args, env = {}) { const result = spawnSync(process.execPath, args, { cwd: repositoryRoot, stdio: 'inherit', env: { ...process.env, ...env } }); if (result.status !== 0) throw new Error(`Command failed (${result.status}): ${args.join(' ')}`); } -async function main() { - const tools = dirname(fileURLToPath(import.meta.url)); +function defaultPaths() { const reportRoot = join(repositoryRoot, 'coverage/js'); - const raw = join(reportRoot, 'raw'); - const sourceRoot = join(repositoryRoot, 'web/js'); + return { repositoryRoot, tools: dirname(fileURLToPath(import.meta.url)), reportRoot, + raw: join(reportRoot, 'raw'), sourceRoot: join(repositoryRoot, 'web/js') }; +} + +function parseShard(value) { + const match = SHARD.exec(value ?? ''); + if (!match || Number(match[1]) > Number(match[2])) { + throw new Error(`--shard needs I/N with 1 <= I <= N, got ${value ?? 'nothing'}\n${USAGE}`); + } + return { index: Number(match[1]), total: Number(match[2]), value }; +} + +// Turn the command line into the ordered steps of one coverage run: +// - no mode: the full run (clean, node units, viewer suite, chromium+touch +// workflows, report, check), exactly as before shards existed; +// - --shard I/N: one slice of the workflows into coverage/js/raw, without wiping +// it or reporting; shard 1 also runs the node units and the viewer suite; +// - --merge-only: report and check over raw coverage the shards already wrote. +export function plan(argv) { + let shard = null; + let mergeOnly = false; + let updateBaseline = false; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === '--update-baseline') updateBaseline = true; + else if (arg === '--merge-only') mergeOnly = true; + else if (arg === '--shard' || arg.startsWith('--shard=')) { + if (shard) throw new Error(`Pass --shard once\n${USAGE}`); + shard = parseShard(arg === '--shard' ? argv[(index += 1)] : arg.slice('--shard='.length)); + } else throw new Error(`Unknown argument: ${arg}\n${USAGE}`); + } + if (shard && mergeOnly) throw new Error(`Pass either --shard or --merge-only\n${USAGE}`); + if (shard && updateBaseline) throw new Error(`--update-baseline needs --merge-only or a full run, not a shard\n${USAGE}`); + const check = { step: 'check', updateBaseline }; + if (mergeOnly) return [{ step: 'require-raw' }, { step: 'report' }, check]; + if (shard) { + const first = shard.index === 1 ? [{ step: 'node-units' }, { step: 'viewer' }] : []; + return [{ step: 'prepare' }, ...first, { step: 'workflows', shard: shard.value }]; + } + return [{ step: 'clean' }, { step: 'node-units' }, { step: 'viewer' }, { step: 'workflows', shard: null }, + { step: 'report' }, check]; +} + +function rawCoverageFiles(raw) { + return existsSync(raw) ? readdirSync(raw).filter(file => file.endsWith('.json')) : []; +} + +function checkSummary({ tools, reportRoot, sourceRoot }, updateBaseline) { const files = discoverSources(sourceRoot); - rmSync(reportRoot, { recursive: true, force: true }); - mkdirSync(raw, { recursive: true }); - const tests = readdirSync(join(repositoryRoot, 'tests/js')).filter(file => file.endsWith('.test.js')).map(file => `tests/js/${file}`); - run(['--test', ...tests], { NODE_V8_COVERAGE: raw }); - const playwright = join(tools, 'node_modules/@playwright/test/cli.js'); - run([playwright, 'test', '--config', join(tools, 'playwright.config.mjs')]); - run([playwright, 'test', '--config', join(tools, 'playwright.workflows.config.mjs'), - '--project', 'chromium', '--project', 'touch'], { PM_JS_COVERAGE: raw }); - run([join(tools, 'node_modules/c8/bin/c8.js'), 'report', '--all', '--src', sourceRoot, - '--include', 'web/js/**/*.js', '--include', 'web/js/**/*.mjs', '--temp-directory', raw, '--reports-dir', reportRoot, - '--reporter', 'text', '--reporter', 'json-summary', '--reporter', 'html']); const summary = JSON.parse(readFileSync(join(reportRoot, 'coverage-summary.json'), 'utf8')); const baselinePath = join(tools, 'frontend-coverage-baseline.json'); - if (process.argv.includes('--update-baseline')) { + if (updateBaseline) { const previous = JSON.parse(readFileSync(baselinePath, 'utf8')); const failures = checkCoverage(summary, previous, sourceRoot, files.filter(file => previous[file])); if (failures.length) throw new Error(`Baseline updates cannot lower coverage:\n${failures.join('\n')}`); @@ -53,8 +89,51 @@ async function main() { const failures = checkCoverage(summary, baseline, sourceRoot, files); if (failures.length) throw new Error(`Frontend coverage regressed:\n${failures.join('\n')}`); } - console.log(`Frontend coverage passed for ${files.length} files. Report: ${relative(repositoryRoot, reportRoot)}/index.html`); + return files.length; } + +export async function runCoverage({ argv = process.argv.slice(2), run = runNode, paths = defaultPaths() } = {}) { + const { repositoryRoot: root, tools, reportRoot, raw, sourceRoot } = paths; + const playwright = join(tools, 'node_modules/@playwright/test/cli.js'); + const actions = { + clean() { + rmSync(reportRoot, { recursive: true, force: true }); + mkdirSync(raw, { recursive: true }); + }, + prepare() { + mkdirSync(raw, { recursive: true }); + }, + 'require-raw'() { + if (!rawCoverageFiles(raw).length) { + throw new Error(`No raw coverage in ${relative(root, raw) || raw}: run the --shard steps first, ` + + 'or put every shard\'s raw files there'); + } + }, + 'node-units'() { + const tests = readdirSync(join(root, 'tests/js')).filter(file => file.endsWith('.test.js')) + .map(file => `tests/js/${file}`); + run(['--test', ...tests], { NODE_V8_COVERAGE: raw }); + }, + viewer() { + run([playwright, 'test', '--config', join(tools, 'playwright.config.mjs')]); + }, + workflows({ shard }) { + run([playwright, 'test', '--config', join(tools, 'playwright.workflows.config.mjs'), + '--project', 'chromium', '--project', 'touch', ...(shard ? ['--shard', shard] : [])], { PM_JS_COVERAGE: raw }); + }, + report() { + run([join(tools, 'node_modules/c8/bin/c8.js'), 'report', '--all', '--src', sourceRoot, + '--include', 'web/js/**/*.js', '--include', 'web/js/**/*.mjs', '--temp-directory', raw, '--reports-dir', reportRoot, + '--reporter', 'text', '--reporter', 'json-summary', '--reporter', 'html']); + }, + check({ updateBaseline }) { + const count = checkSummary(paths, updateBaseline); + console.log(`Frontend coverage passed for ${count} files. Report: ${relative(root, reportRoot)}/index.html`); + }, + }; + for (const { step, ...options } of plan(argv)) await actions[step](options); +} + if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - main().catch(error => { console.error(error.message); process.exitCode = 1; }); + runCoverage().catch(error => { console.error(error.message); process.exitCode = 1; }); } diff --git a/tools/vendor/package.json b/tools/vendor/package.json index c64b903..167fb59 100644 --- a/tools/vendor/package.json +++ b/tools/vendor/package.json @@ -15,6 +15,8 @@ "test:comfyui": "playwright test --config comfyui.config.mjs", "test:executions": "playwright test --config playwright.execution.config.mjs", "test:coverage": "node frontend-coverage.mjs", + "test:coverage:shard": "node frontend-coverage.mjs --shard", + "coverage:merge": "node frontend-coverage.mjs --merge-only", "coverage:baseline-check": "node coverage-baseline-check.mjs", "coverage:update": "node frontend-coverage.mjs --update-baseline", "test:compat": "playwright test --config pack-compat.config.mjs" diff --git a/tools/vendor/playwright.workflows.config.mjs b/tools/vendor/playwright.workflows.config.mjs index 40226f1..67e7c21 100644 --- a/tools/vendor/playwright.workflows.config.mjs +++ b/tools/vendor/playwright.workflows.config.mjs @@ -1,23 +1,18 @@ import { defineConfig, devices } from '@playwright/test'; -import { repositoryRoot } from './vendor-viewerjs.mjs'; -const port = Number(process.env.PM_TEST_PORT || 8765); +import { workflowWorkers } from './test-workers.mjs'; + +// No global webServer: tests/browser/fixtures.mjs starts one server per worker +// on an OS-chosen port. Files run in parallel; tests inside a file keep their order. export default defineConfig({ testDir: '../../tests/browser', outputDir: './test-results/workflows', testMatch: '*.spec.mjs', - workers: 1, + workers: workflowWorkers(), fullyParallel: false, forbidOnly: Boolean(process.env.CI), retries: 0, reporter: 'list', - use: { baseURL: `http://127.0.0.1:${port}`, trace: 'retain-on-failure', screenshot: 'only-on-failure' }, - webServer: { - command: `node tools/vendor/start-browser-server.mjs --port ${port}`, - cwd: repositoryRoot, - url: `http://127.0.0.1:${port}/prompt_manager/test`, - reuseExistingServer: false, - gracefulShutdown: { signal: 'SIGTERM', timeout: 5000 }, - }, + use: { trace: 'retain-on-failure', screenshot: 'only-on-failure' }, projects: [ { name: 'chromium', outputDir: './test-results/workflows/chromium', use: { browserName: 'chromium' } }, { name: 'firefox', outputDir: './test-results/workflows/firefox', use: { browserName: 'firefox' } }, diff --git a/tools/vendor/start-browser-server.mjs b/tools/vendor/start-browser-server.mjs index f41dc62..f307016 100644 --- a/tools/vendor/start-browser-server.mjs +++ b/tools/vendor/start-browser-server.mjs @@ -1,8 +1,189 @@ import { spawn } from 'node:child_process'; -const child = spawn(process.env.PM_TEST_PYTHON || 'python', - ['-m', 'tests.browser.server', ...process.argv.slice(2)], { stdio: 'inherit' }); -child.on('error', error => { console.error(error.message); process.exitCode = 1; }); -child.on('exit', code => { process.exitCode = code ?? 1; }); -for (const signal of ['SIGTERM', 'SIGINT']) { - process.once(signal, () => child.kill(signal)); +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { setTimeout as delay } from 'node:timers/promises'; +import { repositoryRoot } from './vendor-viewerjs.mjs'; + +const READY_PATH = '/prompt_manager/test'; +const STARTUP_TIMEOUT_MS = 60_000; +const STOP_TIMEOUT_MS = 5_000; +const POLL_INTERVAL_MS = 100; +const PROBE_TIMEOUT_MS = 1_000; +const STDERR_TAIL_BYTES = 8_192; +// Server stderr kept for the failing test's attachment; the newest output wins. +const STDERR_LOG_CHARS = 256 * 1024; +const STDERR_DROPPED = '[earlier server stderr dropped]\n'; +const HOST = '127.0.0.1'; +// tests/browser/server.py prints this once it holds its listening socket. Only a +// complete line counts: a chunk may end mid-number. +const PORT_ANNOUNCEMENT = /^PM_TEST_SERVER_PORT=(\d+)\r?\n/m; + +// The server exits when its stdin closes, so a worker that dies without teardown +// (SIGKILL) leaves no server behind; the launcher holds the pipe's other end. +const pythonServer = port => ['-m', 'tests.browser.server', '--port', String(port), '--exit-on-stdin-eof']; + +function describeExit({ code, signal }) { + return signal ? `signal ${signal}` : `code ${code}`; +} + +// Start one disposable tests/browser/server.py and wait until it serves the +// readiness page. `port` 0 (the default) lets the server bind an OS-chosen port +// itself and announce it on stdout, so nothing can take the port between +// choosing and binding. Resolves to {port, url, exited, exitStatus, +// pid, stderrTail(), stderrMark(), stderrSince(mark), stop()}; rejects with the +// server's stderr tail when it exits or stays unready. stdin is a pipe this +// process holds open for the server's life. +export async function startBrowserServer({ + port = 0, + command = process.env.PM_TEST_PYTHON || 'python', + args = pythonServer, + cwd = repositoryRoot, + timeoutMs = STARTUP_TIMEOUT_MS, + stderrLogChars = STDERR_LOG_CHARS, +} = {}) { + if (!Number.isInteger(port) || port < 0 || port > 65535) { + throw new Error(`startBrowserServer needs a port from 0 to 65535, got ${port}`); + } + const child = spawn(command, args(port), { cwd, stdio: ['pipe', 'pipe', 'pipe'] }); + // Never written; closing it (or this process dying) tells the server to exit. + child.stdin.on('error', () => {}); + let stderr = ''; + let stderrTotal = 0; + child.stderr.setEncoding('utf8'); + child.stderr.on('data', chunk => { + stderr = (stderr + chunk).slice(-Math.max(stderrLogChars, STDERR_TAIL_BYTES)); + stderrTotal += chunk.length; + }); + let stdout = ''; + let announced = null; + child.stdout.setEncoding('utf8'); + // Keep draining stdout for the server's whole life so it never blocks on a full pipe. + child.stdout.on('data', chunk => { + if (announced !== null) return; + stdout = (stdout + chunk).slice(-STDERR_TAIL_BYTES); + const match = PORT_ANNOUNCEMENT.exec(stdout); + if (match) announced = Number(match[1]); + }); + const killOnExit = () => child.kill('SIGKILL'); + process.once('exit', killOnExit); + const server = { + pid: child.pid, + port: port || null, + url: port ? `http://${HOST}:${port}` : null, + exitStatus: null, + stopping: false, + stderrTail: () => stderr.slice(-STDERR_TAIL_BYTES).trimEnd(), + // A position in the server's stderr stream, for stderrSince(). + stderrMark: () => stderrTotal, + // What the server wrote after `mark`, at most stderrLogChars of the newest of it. + stderrSince(mark) { + const kept = Math.min(stderr.length, stderrLogChars); + const keptFrom = stderrTotal - kept; + const log = stderr.slice(stderr.length - kept); + return mark >= keptFrom ? log.slice(mark - keptFrom) : STDERR_DROPPED + log; + }, + exited: new Promise(done => { + child.once('error', error => { + stderr += `\n${error.message}`; + server.exitStatus = { code: null, signal: null, error }; + done(server.exitStatus); + }); + child.once('exit', (code, signal) => { + server.exitStatus = { code, signal }; + done(server.exitStatus); + }); + }), + async stop() { + server.stopping = true; + child.stdin.destroy(); + if (server.exitStatus === null) { + child.kill('SIGTERM'); + const stopped = await Promise.race([server.exited, delay(STOP_TIMEOUT_MS).then(() => false)]); + if (!stopped) { + child.kill('SIGKILL'); + await server.exited; + } + } + process.removeListener('exit', killOnExit); + }, + }; + server.exited.then(() => process.removeListener('exit', killOnExit)); + const failure = async reason => { + await server.stop(); + const tail = server.stderrTail(); + return new Error(`test server on port ${server.port ?? '(not yet announced)'} ${reason}` + + (tail ? `\n--- server stderr tail ---\n${tail}` : '')); + }; + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (server.exitStatus !== null) { + throw await failure(`exited with ${describeExit(server.exitStatus)} before it was ready`); + } + if (announced !== null && server.url === null) { + server.port = announced; + server.url = `http://${HOST}:${announced}`; + } + if (announced !== null && announced !== server.port) { + throw await failure(`announced port ${announced} instead`); + } + if (server.url === null) { + await delay(POLL_INTERVAL_MS); + continue; + } + try { + const response = await fetch(`${server.url}${READY_PATH}`, { signal: AbortSignal.timeout(PROBE_TIMEOUT_MS) }); + await response.arrayBuffer(); + if (response.ok) return server; + } catch { + // Not listening yet. + } + await delay(POLL_INTERVAL_MS); + } + throw await failure(`was not ready at ${READY_PATH} within ${timeoutMs} ms`); +} + +// After a test: when it did not end as expected, attach what the server wrote to +// stderr since `mark` (an aiohttp traceback names the cause a "got 500" hides). +export async function attachServerStderr(testInfo, server, mark) { + if (testInfo.status === testInfo.expectedStatus) return; + const log = server.stderrSince(mark); + if (log) await testInfo.attach('server-stderr', { body: log, contentType: 'text/plain' }); +} + +// Keep one live server for a Playwright worker. ensure() returns the running +// server, or starts a fresh one when the last one died (a failed start is +// reported, not retried); crash(server) turns an unexpected exit into an error +// carrying the stderr tail. +export function createServerSupervisor({ start }) { + let current = null; + return { + async ensure() { + if (current && current.exitStatus === null) return current; + current = null; + current = await start(); + return current; + }, + crash(server) { + if (!server || server.stopping || server.exitStatus === null) return null; + const tail = server.stderrTail(); + return new Error(`test server on port ${server.port} exited unexpectedly (${describeExit(server.exitStatus)})` + + (tail ? `\n--- server stderr tail ---\n${tail}` : '')); + }, + async stop() { + const server = current; + current = null; + await server?.stop(); + }, + }; +} + +// CLI: `node start-browser-server.mjs --port N` runs one server in the foreground. +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + const child = spawn(process.env.PM_TEST_PYTHON || 'python', + ['-m', 'tests.browser.server', ...process.argv.slice(2)], { stdio: 'inherit' }); + child.on('error', error => { console.error(error.message); process.exitCode = 1; }); + child.on('exit', code => { process.exitCode = code ?? 1; }); + for (const signal of ['SIGTERM', 'SIGINT']) { + process.once(signal, () => child.kill(signal)); + } } diff --git a/tools/vendor/test-workers.mjs b/tools/vendor/test-workers.mjs new file mode 100644 index 0000000..92dff98 --- /dev/null +++ b/tools/vendor/test-workers.mjs @@ -0,0 +1,18 @@ +import { availableParallelism } from 'node:os'; + +const MAX_LOCAL_WORKERS = 8; +const CPUS_PER_WORKER = 4; + +// PM_TEST_WORKERS wins; CI without it keeps one worker, so CI only changes +// through explicit workflow edits; locally a quarter of the CPUs, at most eight. +export function workflowWorkers({ env = process.env, cpus = availableParallelism() } = {}) { + if (env.PM_TEST_WORKERS !== undefined) { + const workers = Number(env.PM_TEST_WORKERS); + if (!Number.isInteger(workers) || workers < 1) { + throw new Error(`PM_TEST_WORKERS must be a positive integer, got "${env.PM_TEST_WORKERS}"`); + } + return workers; + } + if (env.CI) return 1; + return Math.max(1, Math.min(MAX_LOCAL_WORKERS, Math.floor(cpus / CPUS_PER_WORKER))); +}