Compare commits

...
Author SHA1 Message Date
Vito Sansevero 21a2cc0932 ci(viewerjs): run browser suites with workers and shards
Frontend coverage runs as three shards (PM_TEST_WORKERS=3 each) that upload
their raw V8 files as coverage-raw-N; a merge job downloads them, fails unless
all three arrived with raw files and no clashing names, then runs
coverage:merge. Firefox and WebKit page workflows run as two shards each with
three workers. The required viewerjs context keeps its name and now needs the
shards, the merge and the browser shards, failing unless every one succeeded.
2026-10-21 19:41:54 -07:00
Vito Sansevero 3ebf5c48cf ci(coverage): shard and merge modes for the frontend coverage run
frontend-coverage.mjs now plans its steps from the command line.
No arguments still runs everything, then reports and checks.
--shard I/N runs one slice of the chromium+touch workflows into
coverage/js/raw without wiping it or reporting; shard 1 also runs the
node units and the viewer suite. --merge-only reports and checks over raw
coverage the shards wrote, with --update-baseline as before, and refuses
an empty or missing raw dir. npm scripts: test:coverage:shard and
coverage:merge.
2026-10-21 19:34:08 -07:00
Vito Sansevero 0156d3fadb test(browser): test servers bind an OS-chosen port themselves
tests/browser/server.py accepts --port 0: it binds 127.0.0.1:0 before
building the app, holds that socket for its whole life and prints
PM_TEST_SERVER_PORT=<n> to stdout. startBrowserServer defaults to port 0
and reads the announced port, so no port is reserved and released before
the server binds and two launchers can no longer race for one. The
"address already in use" retry and free-port.mjs are gone; --port N still
works and is checked against the announcement.
2026-10-21 18:53:35 -07:00
Vito Sansevero 357c0f09e7 test(browser): one test server per Playwright worker
The page workflows started one global webServer and ran with one worker.
Each worker now owns its own tests/browser/server.py on an OS-chosen port,
with its own temp root and database, through a shared `test` in
tests/browser/fixtures.mjs. baseURL, and so page, request and
browser.newContext, point at that worker's server. A server that dies
fails the running test with its stderr tail and the next test gets a
fresh one. The worker count comes from PM_TEST_WORKERS, else 1 under CI,
else a quarter of the CPUs up to 8.

Chromium workflows in the pinned image: 510 tests in 606 s with one
worker on main, 511 (one new isolation spec) in 119 s with eight.
2026-10-21 18:30:22 -07:00
40 changed files with 986 additions and 105 deletions
+105 -22
View File
@@ -16,9 +16,13 @@ concurrency:
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# Timeouts are the measured job time plus about 50%, so a hung run stops early.
# Before sharding (main 2cecfcf, one worker): frontend coverage 32m10s, page workflows firefox 18m59s,
# webkit 19m26s. Sharded with 3 workers per job: estimates pending the first CI run (coverage shard ~10m,
# merge ~5m, workflow shard ~8m). Re-measure when a job passes its estimate.
jobs:
frontend-coverage:
name: frontend coverage (chromium and touch)
name: frontend coverage (shard ${{ matrix.shard }}/3)
runs-on: ubuntu-latest
# Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the
# same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json;
@@ -26,12 +30,15 @@ jobs:
container:
image: mcr.microsoft.com/playwright:v1.64.1-noble
options: --user 1001
# This job runs every page workflow twice (chromium, then touch) on one worker with V8 coverage on, so it
# takes about twice a single-browser job. Measured 2026-10-20: whole job 18m26s-24m57s (main 5ded3cf
# 22m15s; PR head 6359999 was cancelled at the old 25-minute limit while still passing), coverage step
# alone 20m55s-22m17s+, against ~12-15 minutes for the firefox/webkit jobs. 40 minutes leaves headroom
# for suite growth while still stopping a hung run; re-measure if a run passes 30 minutes.
timeout-minutes: 40
timeout-minutes: 15
# Public-repo ubuntu-latest runners have 4 vCPUs: three Playwright workers, each with its own test
# server, leave one CPU for the servers and the runner.
env:
PM_TEST_WORKERS: 3
strategy:
fail-fast: false
matrix:
shard: [1, 2, 3]
steps:
- uses: actions/checkout@v7
# The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache
@@ -51,23 +58,92 @@ jobs:
echo "$venv/bin" >> "$GITHUB_PATH"
- run: python --version && node --version && node --version | grep -q '^v24\.'
- run: npm ci --prefix tools/vendor --ignore-scripts
- run: npm run --prefix tools/vendor check
- run: npm run --prefix tools/vendor test
- name: Frontend unit, viewer, workflow, and coverage checks
run: npm run --prefix tools/vendor test:coverage
- name: Upload frontend coverage and Chromium traces
# Shard 1 also runs the node units and the viewer suite, so the vendor pins and the tooling tests
# run on the lightest shard instead.
- if: matrix.shard == 3
run: npm run --prefix tools/vendor check
- if: matrix.shard == 3
run: npm run --prefix tools/vendor test
- name: Page workflows with coverage, shard ${{ matrix.shard }}/3
run: npm run --prefix tools/vendor test:coverage:shard -- ${{ matrix.shard }}/3
- name: Upload raw coverage
uses: actions/upload-artifact@v7
with:
name: coverage-raw-${{ matrix.shard }}
path: coverage/js/raw/
if-no-files-found: error
retention-days: 3
- name: Upload Chromium and touch traces
if: always()
uses: actions/upload-artifact@v7
with:
name: frontend-traces-shard-${{ matrix.shard }}
path: tools/vendor/test-results/
retention-days: 14
frontend-coverage-merge:
name: frontend coverage (merge)
needs: frontend-coverage
runs-on: ubuntu-latest
# Same image as the shards: the raw files name sources by the container's workspace path.
container:
image: mcr.microsoft.com/playwright:v1.64.1-noble
options: --user 1001
timeout-minutes: 8
steps:
- uses: actions/checkout@v7
- run: node --version && node --version | grep -q '^v24\.'
- run: npm ci --prefix tools/vendor --ignore-scripts
# Each artifact lands in its own coverage-raw-N directory, so the next step can count them.
- uses: actions/download-artifact@v8
with:
pattern: coverage-raw-*
path: ${{ runner.temp }}/coverage-raw
# coverage:merge only checks that raw files exist; merging two of three shards would report
# partial coverage, so a missing or empty shard artifact fails here.
- name: Verify every shard's raw coverage arrived
shell: bash
env:
STAGED_RAW: ${{ runner.temp }}/coverage-raw
EXPECTED_SHARDS: 3
run: |
found=0
for shard in $(seq 1 "$EXPECTED_SHARDS"); do
directory="$STAGED_RAW/coverage-raw-$shard"
count=0
if [ -d "$directory" ]; then count=$(find "$directory" -maxdepth 1 -type f -name '*.json' | wc -l); fi
echo "coverage-raw-$shard: $count raw files"
if [ "$count" -gt 0 ]; then found=$((found + 1)); fi
done
echo "shard artifacts with raw coverage: $found of $EXPECTED_SHARDS"
if [ "$found" -ne "$EXPECTED_SHARDS" ]; then
echo "::error::Expected raw coverage from $EXPECTED_SHARDS shards, got $found of $EXPECTED_SHARDS"
exit 1
fi
staged=$(find "$STAGED_RAW" -mindepth 2 -maxdepth 2 -type f -name '*.json' | wc -l)
mkdir -p coverage/js/raw
# cp -n never overwrites; a clash shows up as fewer merged files than staged ones.
find "$STAGED_RAW" -mindepth 2 -maxdepth 2 -type f -name '*.json' -exec cp -n -t coverage/js/raw {} + || true
merged=$(find coverage/js/raw -maxdepth 1 -type f -name '*.json' | wc -l)
if [ "$merged" -ne "$staged" ]; then
echo "::error::Shards wrote clashing raw file names: $staged staged, $merged merged"
exit 1
fi
echo "merged $merged raw files"
- name: Frontend coverage report and baseline check
run: npm run --prefix tools/vendor coverage:merge
- name: Upload frontend coverage report
if: always()
uses: actions/upload-artifact@v7
with:
name: frontend-test-reports
path: |
tools/vendor/test-results/
coverage/js/
!coverage/js/raw/**
retention-days: 14
browser-workflows:
name: page workflows (${{ matrix.browser }})
name: page workflows (${{ matrix.browser }}, shard ${{ matrix.shard }}/2)
runs-on: ubuntu-latest
# Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the
# same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json;
@@ -75,11 +151,16 @@ jobs:
container:
image: mcr.microsoft.com/playwright:v1.64.1-noble
options: --user 1001
timeout-minutes: 25
timeout-minutes: 12
# Public-repo ubuntu-latest runners have 4 vCPUs: three Playwright workers, each with its own test
# server, leave one CPU for the servers and the runner.
env:
PM_TEST_WORKERS: 3
strategy:
fail-fast: false
matrix:
browser: [firefox, webkit]
shard: [1, 2]
steps:
- uses: actions/checkout@v7
# The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache
@@ -99,21 +180,21 @@ jobs:
echo "$venv/bin" >> "$GITHUB_PATH"
- run: python --version && node --version && node --version | grep -q '^v24\.'
- run: npm ci --prefix tools/vendor --ignore-scripts
- name: Page workflows in ${{ matrix.browser }}
run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }}
- name: Page workflows in ${{ matrix.browser }}, shard ${{ matrix.shard }}/2
run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }} --shard ${{ matrix.shard }}/2
- name: Upload browser traces
if: always()
uses: actions/upload-artifact@v7
with:
name: browser-traces-${{ matrix.browser }}
name: browser-traces-${{ matrix.browser }}-shard-${{ matrix.shard }}
path: tools/vendor/test-results/
retention-days: 14
# Branch protection requires the exact context "viewerjs". This job keeps that
# name and fails unless every frontend job above succeeded, including when one
# was cancelled or skipped.
# name and fails unless every frontend job above succeeded (every shard, the
# coverage merge and every browser shard), including when one was cancelled or skipped.
viewerjs:
needs: [frontend-coverage, browser-workflows]
needs: [frontend-coverage, frontend-coverage-merge, browser-workflows]
if: always()
runs-on: ubuntu-latest
timeout-minutes: 5
@@ -121,8 +202,10 @@ jobs:
- name: Require every frontend job to succeed
env:
COVERAGE_RESULT: ${{ needs.frontend-coverage.result }}
MERGE_RESULT: ${{ needs.frontend-coverage-merge.result }}
WORKFLOWS_RESULT: ${{ needs.browser-workflows.result }}
run: |
echo "frontend-coverage: $COVERAGE_RESULT"
echo "frontend-coverage-merge: $MERGE_RESULT"
echo "browser-workflows: $WORKFLOWS_RESULT"
test "$COVERAGE_RESULT" = success && test "$WORKFLOWS_RESULT" = success
test "$COVERAGE_RESULT" = success && test "$MERGE_RESULT" = success && test "$WORKFLOWS_RESULT" = success
+25 -14
View File
@@ -39,9 +39,13 @@ npm run --prefix tools/vendor test:coverage
npm run --prefix tools/vendor test:workflows -- --project firefox --project webkit
```
`PM_TEST_PYTHON` selects the Python executable; `PM_TEST_PORT` selects the local
port (default 8765). Set these environment variables when Python is outside PATH
or the port is occupied. Test servers are never reused. On unsupported Linux
`PM_TEST_PYTHON` selects the Python executable; set it when Python is outside
PATH. Every Playwright worker starts its own `tests/browser/server.py --port 0`,
which binds a port the OS picks and prints it as `PM_TEST_SERVER_PORT=<n>`, with
its own temp root and database (`tests/browser/fixtures.mjs`), so
parallel workers and concurrent runs never share state. `PM_TEST_WORKERS` sets the
worker count (default 1 under `CI`, otherwise a quarter of the CPUs, at most 8).
Test servers are never reused across runs. On unsupported Linux
distributions, browser system dependencies may need manual installation. For a
quick browser run, use `test:workflows -- --project chromium`.
@@ -114,16 +118,23 @@ second without a browser and fails, naming the files, when a measured
`web/js` file has no baseline entry or the baseline lists a file that no longer
exists. It uses the gate's own `discoverSources`, and `npm test` runs the same
check, so run it before pushing a new or deleted frontend file instead of
waiting for the `frontend coverage` job.
waiting for the `frontend coverage` jobs.
The `viewerjs` check runs on every PR, including backend-only changes. It is an
aggregate job that passes only when the `frontend coverage (chromium and touch)`
job and both `page workflows (firefox)` and `page workflows (webkit)` jobs
succeed; the browsers run on separate runners in parallel. GitHub artifacts
named `frontend-test-reports` contain HTML coverage, a JSON summary, and the
Chromium/touch failure screenshots/traces; `browser-traces-firefox` and
`browser-traces-webkit` hold the other browsers' traces. Open traces with
Playwright's `show-trace` tool.
aggregate job that passes only when every frontend job succeeds: the three
`frontend coverage (shard N/3)` jobs, `frontend coverage (merge)`, and the four
`page workflows (firefox|webkit, shard N/2)` jobs. A failed, cancelled or skipped
job fails it. Every browser job runs with `PM_TEST_WORKERS: 3` (the public
runners have 4 vCPUs). Each coverage shard runs `test:coverage:shard -- N/3`
(shard 1 also runs the node units and the viewer suite, shard 3 the vendor pins
and tooling tests) and uploads its raw V8 files as `coverage-raw-N` (kept 3
days). The merge job downloads them, fails unless all three arrived with raw
files and without clashing names, then runs `coverage:merge` (report plus
baseline check). `tests/vendor/ci-workflow.test.mjs` checks this shape. GitHub
artifacts named `frontend-test-reports` contain the merged HTML coverage and
JSON summary; `frontend-traces-shard-N` hold the Chromium/touch failure
screenshots/traces and `browser-traces-<browser>-shard-N` the other browsers'
traces. Open traces with Playwright's `show-trace` tool.
The real-host suite covers both Classic and Nodes 2.0 startup, switching both
ways, actual database prompt selection, execution-text serialization, workflow
@@ -1556,7 +1567,7 @@ bytes at the same path without changing the immutable receipt.
The focused acceptance command retains all four projects:
```sh
PM_TEST_PORT=8766 npm run --prefix tools/vendor test:workflows -- \
npm run --prefix tools/vendor test:workflows -- \
workspace-history.spec.mjs workspace-visual.spec.mjs \
--project chromium --project touch --project firefox --project webkit
npm run --prefix tools/vendor test:executions
@@ -1649,7 +1660,7 @@ acceptance; a baseline is added only after the maintainer accepts them.
Focused commands:
```sh
PM_TEST_PORT=8766 npm run --prefix tools/vendor test:workflows -- \
npm run --prefix tools/vendor test:workflows -- \
workspace-library-editor.spec.mjs library-editor-visual.spec.mjs \
--project chromium --project touch --project firefox --project webkit
npm run --prefix tools/vendor test:comfyui # library-sender.spec.mjs host regression
@@ -1749,7 +1760,7 @@ mobile, the concept geometry/token/control comparison and its mutation check),
```sh
node --test tests/js/workspace-vocabulary-state.test.js tests/js/workspace-state.test.js
PM_TEST_PORT=8766 npm run --prefix tools/vendor test:workflows -- \
npm run --prefix tools/vendor test:workflows -- \
workspace-vocabulary.spec.mjs vocabulary-visual.spec.mjs focused-node.spec.mjs \
--project chromium --project touch --project firefox --project webkit
```
+1 -1
View File
@@ -1,5 +1,5 @@
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import { test, expect } from './fixtures.mjs';
import { THEMES, seedConcept, openConcept, openProduction, diffInventory, mutate } from './concept-inventory.mjs';
// Activity-1 Task 3. The review dialog is compared with the concept's own `scan-review` ("Review metadata scan",
+1 -1
View File
@@ -1,5 +1,5 @@
import { readFileSync } from 'node:fs';
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import { test, expect } from './fixtures.mjs';
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
test.beforeEach(async ({page, browserName}) => {
+1 -1
View File
@@ -2,7 +2,7 @@
// same concept selectors in the immutable concept (/__test__/canonical/workspace) and in production, which now renders
// the concept's classes. The concept is never rewritten: each page's map<Page>Concept(reference) applies only the
// recorded spec-§8 deviations, each commented with its D-number, and diffInventory must then report nothing.
import {expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import {expect} from './fixtures.mjs';
export {diffInventory} from './library-editor-canonical-comparison.mjs';
export const CONCEPT_URL='/__test__/canonical/workspace';
+29
View File
@@ -0,0 +1,29 @@
// Shared `test` for the page workflows: every Playwright worker owns one
// tests/browser/server.py process on a port the OS chose for it, with its own temp root
// and database. `baseURL` (and so `page`, `request` and `browser.newContext`)
// points at that worker's server. A server that dies fails the running test
// with its stderr tail, and the next test in the worker gets a fresh server.
import { test as base, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import { createServerSupervisor, startBrowserServer } from '../../tools/vendor/start-browser-server.mjs';
const SERVER_FIXTURE_TIMEOUT_MS = 90_000;
export const test = base.extend({
testServer: [async ({}, use) => {
const supervisor = createServerSupervisor({ start: () => startBrowserServer() });
try {
await use(supervisor);
} finally {
await supervisor.stop();
}
}, { scope: 'worker', timeout: SERVER_FIXTURE_TIMEOUT_MS }],
workerServer: [async ({ testServer }, use) => {
const server = await testServer.ensure();
await use(server);
const crash = testServer.crash(server);
if (crash) throw crash;
}, { auto: true, timeout: SERVER_FIXTURE_TIMEOUT_MS }],
baseURL: async ({ workerServer }, use) => use(workerServer.url),
});
export { expect };
+1 -1
View File
@@ -1,5 +1,5 @@
import {readFileSync} from 'node:fs';
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import {test,expect} from './fixtures.mjs';
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
for (const variant of ['Focused', 'Split']) {
+1 -1
View File
@@ -1,5 +1,5 @@
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import { test, expect } from './fixtures.mjs';
import { seedConcept, openConcept, openProduction, SIZES } from './concept-inventory.mjs';
// D10/D20: the heading is the linked positive's title (its confirmed first line here), never the pair's
// "Inspector recipe", which names the negative variant.
@@ -1,5 +1,5 @@
import { writeFile } from 'node:fs/promises';
import { expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import { expect } from './fixtures.mjs';
// Content, row counts, and image bytes come from independent synthetic datasets.
// Saved choices and actual executions are separate production sections. The concept's
+1 -1
View File
@@ -1,5 +1,5 @@
import {writeFile} from 'node:fs/promises';
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import {test,expect} from './fixtures.mjs';
import {captureCanonicalState} from './history-canonical-comparison.mjs';
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,diffInventory,mutate,stripExact} from './concept-inventory.mjs';
+1 -1
View File
@@ -1,5 +1,5 @@
import {writeFile} from 'node:fs/promises';
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import {test,expect} from './fixtures.mjs';
import {captureCanonicalState} from './history-canonical-comparison.mjs';
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,diffInventory,mutate,stripExact,countCopy,reflow,withoutItems} from './concept-inventory.mjs';
+1 -1
View File
@@ -1,5 +1,5 @@
import {writeFile} from 'node:fs/promises';
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import {test,expect} from './fixtures.mjs';
import {captureCanonicalState} from './history-canonical-comparison.mjs';
import {SIZES,THEMES,seedConcept,openConcept,openProduction,inspectorOf,diffInventory,mutate,countCopy} from './concept-inventory.mjs';
+1 -1
View File
@@ -1,5 +1,5 @@
import {writeFile} from 'node:fs/promises';
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import {test,expect} from './fixtures.mjs';
import {captureCanonicalState} from './history-canonical-comparison.mjs';
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,diffInventory,mutate,stripExact,countCopy,reflow} from './concept-inventory.mjs';
+1 -1
View File
@@ -1,4 +1,4 @@
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import {test,expect} from './fixtures.mjs';
import {captureCanonicalState} from './history-canonical-comparison.mjs';
// Artifact-only captures of the editor states that have no concept counterpart (archive confirmation, Show archived,
+1 -1
View File
@@ -1,5 +1,5 @@
import {readFileSync} from 'node:fs';
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import {test,expect} from './fixtures.mjs';
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
const pair={id:4,positive_text:'Library positive',negative_text:'Library negative',category:'Portrait',tags:['studio']};
+1 -1
View File
@@ -1,4 +1,4 @@
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import { test, expect } from './fixtures.mjs';
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
const moduleUrl = '/prompt_manager/js/shared/toolkit/prompt-bridge.js';
+1 -1
View File
@@ -1,5 +1,5 @@
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import {test,expect} from './fixtures.mjs';
const workspace='/prompt_manager/workspace#/images';
const details=page=>page.getByRole('complementary',{name:'Selected image details',exact:true});
+34 -6
View File
@@ -4,6 +4,7 @@ import argparse
import json
import os
import shutil
import socket
from pathlib import Path
import sys
import tempfile
@@ -515,14 +516,41 @@ def create_app(root: Path) -> web.Application:
return app
def main() -> None:
HOST = "127.0.0.1"
PORT_ANNOUNCEMENT = "PM_TEST_SERVER_PORT="
def bind_listener(port: int) -> socket.socket:
"""Bind the server's socket before the app is built.
``port`` 0 lets the OS choose a free port, which this process then holds,
so parallel launchers never race for one between choosing and binding.
"""
listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
if port and os.name != "nt":
# aiohttp's own default on POSIX; on Windows it would let a
# second process share the port.
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
listener.bind((HOST, port))
listener.listen(128)
except OSError:
listener.close()
raise
return listener
def main(argv: list[str] | None = None) -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--port", type=int, default=8765)
args = parser.parse_args()
parser.add_argument(
"--port", type=int, default=8765, help="0 binds an OS-chosen free port"
)
args = parser.parse_args(argv)
listener = bind_listener(args.port)
# One machine-readable line tells the launcher which port this process holds.
print(f"{PORT_ANNOUNCEMENT}{listener.getsockname()[1]}", flush=True)
with tempfile.TemporaryDirectory(prefix="pm-browser-") as directory:
web.run_app(
create_app(Path(directory).resolve()), host="127.0.0.1", port=args.port
)
web.run_app(create_app(Path(directory).resolve()), sock=listener)
if __name__ == "__main__":
+1 -1
View File
@@ -1,5 +1,5 @@
import {writeFile} from 'node:fs/promises';
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import {test,expect} from './fixtures.mjs';
import {captureCanonicalState} from './history-canonical-comparison.mjs';
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept as openConceptPage,openProduction,inventoryOf,diffInventory,focusRingOf,hoverStateOf,mutate} from './concept-inventory.mjs';
+1 -1
View File
@@ -1,4 +1,4 @@
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import {test,expect} from './fixtures.mjs';
import {captureCanonicalState} from './history-canonical-comparison.mjs';
// Artifact-only captures of Tags & categories states that have no concept counterpart: the Manage categories
+35
View File
@@ -0,0 +1,35 @@
// Each Playwright worker owns its server: `request`, `page` and manual
// `browser.newContext()` all reach it through `baseURL`, and another server
// process never sees its database.
import { test, expect } from './fixtures.mjs';
import { startBrowserServer } from '../../tools/vendor/start-browser-server.mjs';
const search = text => `/prompt_manager/search?text=${encodeURIComponent(text)}`;
test('every fixture reaches this worker\'s own server and another server keeps its own database', async ({ request, page, browser, baseURL, workerServer }, testInfo) => {
expect(baseURL).toBe(workerServer.url);
await expect(await request.post('/__test__/reset')).toBeOK();
const marker = `worker ${testInfo.workerIndex} isolation ${testInfo.project.name} ${testInfo.repeatEachIndex}`;
await expect(await request.post('/prompt_manager/save', { data: { text: marker } })).toBeOK();
expect((await (await request.get(search(marker))).json()).count).toBe(1);
const viaPage = await page.request.get(search(marker));
expect(viaPage.url().startsWith(`${baseURL}/`)).toBe(true);
expect((await viaPage.json()).count).toBe(1);
const context = await browser.newContext();
try {
const response = await (await context.newPage()).goto(search(marker));
expect(response.url().startsWith(`${baseURL}/`)).toBe(true);
expect((await response.json()).count).toBe(1);
} finally {
await context.close();
}
const other = await startBrowserServer();
try {
expect(other.url).not.toBe(baseURL);
expect((await (await fetch(`${other.url}${search(marker)}`)).json()).count).toBe(0);
} finally {
await other.stop();
}
});
+1 -1
View File
@@ -1,5 +1,5 @@
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import { test, expect } from './fixtures.mjs';
test.beforeEach(async ({ request, page, browserName }) => {
await startCoverage(page, browserName);
+1 -1
View File
@@ -1,5 +1,5 @@
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import { test, expect } from './fixtures.mjs';
import { seedConcept, openConcept, openProduction } from './concept-inventory.mjs';
// Activity (#249 Activity-1): the concept's Activity page (app.js renderActivity 732–738), its top-bar button
+1 -1
View File
@@ -1,5 +1,5 @@
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import {test,expect} from './fixtures.mjs';
import {readFile,writeFile} from 'node:fs/promises';
import {createHash} from 'node:crypto';
+1 -1
View File
@@ -1,5 +1,5 @@
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import { test, expect } from './fixtures.mjs';
import { seedConcept, openConcept, openProduction } from './concept-inventory.mjs';
const workspace = '/prompt_manager/workspace#/images';
@@ -1,6 +1,6 @@
import {writeFile} from 'node:fs/promises';
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import {test,expect} from './fixtures.mjs';
import {captureCanonicalState} from './history-canonical-comparison.mjs';
import {inventoryOf,mapConcept,diffInventory} from './library-editor-canonical-comparison.mjs';
+1 -1
View File
@@ -1,5 +1,5 @@
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import {test,expect} from './fixtures.mjs';
const route='/prompt_manager/workspace#/library';
const cards=page=>page.locator('article[data-prompt-id]');
+1 -1
View File
@@ -1,5 +1,5 @@
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import { test, expect } from './fixtures.mjs';
// Saved views (#249 D16, maintainer ruling 2026-10-21): the toolbar's Save view (concept app.js a359, a1330–1346),
// the sidebar SAVED VIEWS rows (a331–335) and `#/saved/<id>` (a1158–1162), on Images and the Prompt library.
+1 -1
View File
@@ -3,7 +3,7 @@ import { readFile } from 'node:fs/promises';
import { createHash } from 'node:crypto';
import { captureCanonicalState, historyGeometry, compareCanonicalHistory, nativeDialogState, exerciseCurrentPairSend, assertRecordedHistory } from './history-canonical-comparison.mjs';
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import { test, expect } from './fixtures.mjs';
test.use({ locale: 'en-US', timezoneId: 'UTC', reducedMotion: 'reduce' });
const workspace = '/prompt_manager/workspace';
+1 -1
View File
@@ -1,6 +1,6 @@
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
import {writeFile} from 'node:fs/promises';
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import {test,expect} from './fixtures.mjs';
import {captureCanonicalState} from './history-canonical-comparison.mjs';
import {pageInventoryOf,inventoryOf,diffInventory,mapVocabularyConcept,mapVocabularyDialog} from './vocabulary-canonical-comparison.mjs';
+1 -1
View File
@@ -1,5 +1,5 @@
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
import { test, expect } from './fixtures.mjs';
const workspace = '/prompt_manager/workspace';
const statistic = (page, label) => page.locator('#workspace-main dt')
+138
View File
@@ -0,0 +1,138 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { createServer } from 'node:net';
import { fileURLToPath } from 'node:url';
import { startBrowserServer, createServerSupervisor } from '../../tools/vendor/start-browser-server.mjs';
import { workflowWorkers } from '../../tools/vendor/test-workers.mjs';
const fakeServer = fileURLToPath(new URL('./fake-browser-server.mjs', import.meta.url));
const fake = (...extra) => ({ command: process.execPath, args: port => [fakeServer, '--port', String(port), ...extra] });
const REAL_SERVER_TIMEOUT_MS = 90_000;
function listen(port = 0) {
return new Promise((resolve, reject) => {
const server = createServer();
server.once('error', reject);
server.listen(port, '127.0.0.1', () => resolve(server));
});
}
async function releasedPort() {
const probe = await listen();
const { port } = probe.address();
await new Promise(resolve => probe.close(resolve));
return port;
}
test('free port allocation never reuses a bound port', async t => {
const held = await listen();
t.after(() => held.close());
const heldPort = held.address().port;
const servers = await Promise.all(Array.from({ length: 6 }, () => startBrowserServer(fake())));
t.after(() => Promise.all(servers.map(server => server.stop())));
const ports = servers.map(server => server.port);
for (const server of servers) {
assert.ok(Number.isInteger(server.port) && server.port > 0 && server.port < 65536, `port ${server.port} is a TCP port`);
assert.equal(server.url, `http://127.0.0.1:${server.port}`);
assert.equal(await (await fetch(`${server.url}/prompt_manager/test`)).text(), 'ready');
}
assert.ok(!ports.includes(heldPort), 'a port another process holds is never handed out');
assert.equal(new Set(ports).size, ports.length, 'concurrent servers never share a port');
});
test('startBrowserServer asks the server to bind port 0 and reads the port it announces', async () => {
const asked = [];
const server = await startBrowserServer({ command: process.execPath, args: port => { asked.push(port); return fake().args(port); } });
try {
assert.deepEqual(asked, [0], 'no port is reserved and released before the server binds');
assert.ok(server.port > 0);
} finally {
await server.stop();
}
});
test('startBrowserServer serves the readiness page on a requested port and stops cleanly', async () => {
const port = await releasedPort();
const server = await startBrowserServer({ port, ...fake() });
assert.equal(server.port, port);
assert.equal(server.url, `http://127.0.0.1:${port}`);
assert.equal(await (await fetch(`${server.url}/prompt_manager/test`)).text(), 'ready');
await server.stop();
await assert.rejects(fetch(`${server.url}/prompt_manager/test`));
});
test('startBrowserServer reports a startup failure with the server\'s stderr tail', async () => {
await assert.rejects(startBrowserServer(fake('--fail')), error => {
assert.match(error.message, /exited with code 4 before it was ready/);
assert.match(error.message, /last fake stderr line/);
return true;
});
});
test('startBrowserServer gives up on a server that never announces a port or becomes ready', async () => {
const silent = { command: process.execPath, args: () => ['-e', 'setInterval(() => {}, 1000)'] };
await assert.rejects(startBrowserServer({ ...silent, timeoutMs: 500 }), /not ready .* within 500 ms/);
});
test('worker server restarts after an unexpected exit', async t => {
const supervisor = createServerSupervisor({ start: () => startBrowserServer(fake()) });
t.after(() => supervisor.stop());
const first = await supervisor.ensure();
assert.equal(supervisor.crash(first), null, 'a running server reports no crash');
await fetch(`${first.url}/__exit`);
await first.exited;
const crash = supervisor.crash(first);
assert.match(crash.message, /test server on port \d+ exited unexpectedly \(code 3\)/);
assert.match(crash.message, /crashed on purpose/);
const second = await supervisor.ensure();
assert.notEqual(second, first, 'the next test gets a fresh server');
assert.equal(await (await fetch(`${second.url}/prompt_manager/test`)).text(), 'ready');
assert.equal(await supervisor.ensure(), second, 'a healthy server is reused');
});
test('supervisor reports a startup failure once and tries again on the next ensure', async t => {
let attempts = 0;
const supervisor = createServerSupervisor({
start: async () => {
attempts += 1;
if (attempts === 1) throw new Error('ImportError: no module');
return startBrowserServer(fake());
},
});
t.after(() => supervisor.stop());
await assert.rejects(supervisor.ensure(), /ImportError/);
assert.equal(attempts, 1, 'a failed startup is not retried behind the caller\'s back');
const server = await supervisor.ensure();
assert.equal(attempts, 2);
assert.equal(await (await fetch(`${server.url}/prompt_manager/test`)).text(), 'ready');
});
test('tests/browser/server.py binds an OS-chosen port for --port 0 and still honours --port N', { timeout: 2 * REAL_SERVER_TIMEOUT_MS }, async t => {
const held = await listen();
t.after(() => held.close());
const chosen = await startBrowserServer({ timeoutMs: REAL_SERVER_TIMEOUT_MS });
try {
assert.notEqual(chosen.port, held.address().port);
assert.equal((await fetch(`${chosen.url}/prompt_manager/test`)).status, 200);
} finally {
await chosen.stop();
}
const port = await releasedPort();
const fixed = await startBrowserServer({ port, timeoutMs: REAL_SERVER_TIMEOUT_MS });
try {
assert.equal(fixed.port, port);
assert.equal((await fetch(`http://127.0.0.1:${port}/prompt_manager/test`)).status, 200);
} finally {
await fixed.stop();
}
});
test('worker count follows PM_TEST_WORKERS, then one in CI, then a quarter of the CPUs up to eight', () => {
assert.equal(workflowWorkers({ env: { PM_TEST_WORKERS: '3', CI: '1' }, cpus: 32 }), 3);
assert.equal(workflowWorkers({ env: { CI: '1' }, cpus: 32 }), 1);
assert.equal(workflowWorkers({ env: {}, cpus: 32 }), 8);
assert.equal(workflowWorkers({ env: {}, cpus: 12 }), 3);
assert.equal(workflowWorkers({ env: {}, cpus: 2 }), 1);
assert.throws(() => workflowWorkers({ env: { PM_TEST_WORKERS: 'many' }, cpus: 8 }), /PM_TEST_WORKERS must be a positive integer/);
assert.throws(() => workflowWorkers({ env: { PM_TEST_WORKERS: '0' }, cpus: 8 }), /PM_TEST_WORKERS must be a positive integer/);
});
+197
View File
@@ -0,0 +1,197 @@
// Structural checks on .github/workflows/viewerjs-check.yml. The tooling has no YAML parser, so these
// read the workflow's fixed two-space layout: jobs at two spaces, their keys at four, steps at six.
import test from 'node:test';
import assert from 'node:assert/strict';
import { execFileSync } from 'node:child_process';
import { mkdtemp, mkdir, readdir, rm, writeFile } from 'node:fs/promises';
import { readFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
const workflow = readFileSync(new URL('../../.github/workflows/viewerjs-check.yml', import.meta.url), 'utf8');
const PINNED_IMAGE = /^\s+image: mcr\.microsoft\.com\/playwright:v\d+\.\d+\.\d+-noble$/m;
function jobs(text) {
const lines = text.split('\n');
const start = lines.indexOf('jobs:');
assert.notEqual(start, -1, 'workflow has a jobs section');
const found = new Map();
let current = null;
for (const line of lines.slice(start + 1)) {
const header = /^ {2}([\w-]+):\s*$/.exec(line);
if (header) {
current = header[1];
found.set(current, []);
} else if (current && (line.startsWith(' ') || line.trim() === '' || /^ {2}#/.test(line))) {
found.get(current).push(line);
} else if (current && /^\S/.test(line)) {
break;
}
}
return new Map([...found].map(([id, body]) => [id, body.join('\n')]));
}
function steps(jobText) {
const parts = jobText.split(/^(?= {6}- )/m);
return parts.slice(1);
}
function runScript(stepText) {
const lines = stepText.split('\n');
const index = lines.findIndex(line => /^ {8}run: \|\s*$/.test(line));
assert.notEqual(index, -1, 'step has a block run script');
const body = [];
for (const line of lines.slice(index + 1)) {
if (line.trim() !== '' && !line.startsWith(' ')) break;
body.push(line.slice(10));
}
return body.join('\n');
}
function needsOf(jobText) {
const match = /^ {4}needs: \[([^\]]*)\]\s*$/m.exec(jobText) ?? /^ {4}needs: ([\w-]+)\s*$/m.exec(jobText);
assert.ok(match, 'job declares needs');
return match[1].split(',').map(value => value.trim()).filter(Boolean);
}
function matrixList(jobText, key) {
const match = new RegExp(`^ {8}${key}: \\[([^\\]]*)\\]\\s*$`, 'm').exec(jobText);
assert.ok(match, `job has a ${key} matrix`);
return match[1].split(',').map(value => value.trim());
}
const allJobs = jobs(workflow);
const browserJobIds = [...allJobs].filter(([, body]) => /npm run --prefix tools\/vendor test:(workflows|coverage)/.test(body))
.map(([id]) => id);
const mergeJob = () => allJobs.get('frontend-coverage-merge');
const verifyStep = () => steps(mergeJob()).find(step => /name: Verify every shard's raw coverage arrived/.test(step));
test('viewerjs keeps its name and needs every frontend job', () => {
const viewerjs = allJobs.get('viewerjs');
assert.ok(viewerjs, 'the required context is the job id viewerjs');
assert.doesNotMatch(viewerjs, /^ {4}name:/m, 'no display name may replace the context "viewerjs"');
const needs = needsOf(viewerjs);
const others = [...allJobs.keys()].filter(id => id !== 'viewerjs');
assert.deepEqual([...needs].sort(), [...others].sort(), 'viewerjs needs every other job in the workflow');
assert.deepEqual([...needs].sort(), ['browser-workflows', 'frontend-coverage', 'frontend-coverage-merge']);
assert.match(viewerjs, /^ {4}if: always\(\)$/m, 'viewerjs runs (and fails) when a needed job failed or was skipped');
const script = runScript(steps(viewerjs).find(step => /run: \|/.test(step)));
for (const id of needs) {
const variable = new RegExp(`^ {10}([A-Z_]+): \\$\\{\\{ needs\\.${id}\\.result \\}\\}$`, 'm').exec(viewerjs);
assert.ok(variable, `${id}'s result reaches the check`);
assert.match(script, new RegExp(`test "\\$${variable[1]}" = success`), `${id} must be exactly success`);
}
});
test('viewerjs fails unless every needed job succeeded', () => {
const viewerjs = allJobs.get('viewerjs');
const script = runScript(steps(viewerjs).find(step => /run: \|/.test(step)));
const variables = [...viewerjs.matchAll(/^ {10}([A-Z_]+): \$\{\{ needs\.[\w-]+\.result \}\}$/gm)].map(match => match[1]);
const run = env => {
try {
execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', script], { env: { PATH: process.env.PATH, ...env }, stdio: 'pipe' });
return true;
} catch {
return false;
}
};
const allSuccess = Object.fromEntries(variables.map(name => [name, 'success']));
assert.equal(run(allSuccess), true);
for (const name of variables) {
for (const result of ['failure', 'cancelled', 'skipped']) {
assert.equal(run({ ...allSuccess, [name]: result }), false, `${name}=${result} must fail viewerjs`);
}
}
});
test('coverage runs as three shards that each upload their raw coverage', () => {
const coverage = allJobs.get('frontend-coverage');
assert.deepEqual(matrixList(coverage, 'shard'), ['1', '2', '3']);
assert.match(coverage, /^ {4}name: frontend coverage \(shard \$\{\{ matrix\.shard \}\}\/3\)$/m);
assert.match(coverage, /npm run --prefix tools\/vendor test:coverage:shard -- \$\{\{ matrix\.shard \}\}\/3/);
const upload = steps(coverage).find(step => /name: coverage-raw-\$\{\{ matrix\.shard \}\}/.test(step));
assert.ok(upload, 'each shard uploads coverage-raw-N');
assert.match(upload, /^ {10}path: coverage\/js\/raw\/?$/m);
assert.match(upload, /^ {10}retention-days: 3$/m);
assert.match(upload, /^ {10}if-no-files-found: error$/m);
});
test('merge job downloads every shard and runs the merge-only report and check', () => {
const merge = mergeJob();
assert.ok(merge, 'frontend-coverage-merge exists');
assert.deepEqual(needsOf(merge), ['frontend-coverage']);
assert.match(merge, /^ {4}name: frontend coverage \(merge\)$/m);
const download = steps(merge).find(step => /uses: actions\/download-artifact@/.test(step));
assert.ok(download, 'merge downloads the shard artifacts');
assert.match(download, /^ {10}pattern: coverage-raw-\*$/m);
assert.doesNotMatch(download, /merge-multiple: true/, 'each artifact keeps its own directory so the check can count them');
assert.ok(verifyStep(), 'merge has an explicit artifact count check');
assert.match(verifyStep(), new RegExp(`^ {10}EXPECTED_SHARDS: ${matrixList(allJobs.get('frontend-coverage'), 'shard').length}$`, 'm'),
'the expected count matches the shard matrix');
const order = steps(merge).map(step => (/name: (.*)/.exec(step) ?? [, step.trim().split('\n')[0]])[1]);
const verifyAt = order.findIndex(name => /Verify every shard/.test(name));
const mergeAt = steps(merge).findIndex(step => /npm run --prefix tools\/vendor coverage:merge/.test(step));
assert.ok(verifyAt >= 0 && mergeAt > verifyAt, 'the count check runs before coverage:merge');
const report = steps(merge).find(step => /name: frontend-test-reports$/m.test(step));
assert.ok(report, 'merge uploads the HTML report as frontend-test-reports');
assert.match(report, /!coverage\/js\/raw\/\*\*/);
});
async function stagedShards(root, shards) {
const staged = join(root, 'staged');
for (const [shard, files] of shards) {
const directory = join(staged, `coverage-raw-${shard}`);
await mkdir(directory, { recursive: true });
for (const file of files) await writeFile(join(directory, file), '{"result":[]}');
}
await mkdir(staged, { recursive: true });
return staged;
}
async function verify(t, shards) {
const root = await mkdtemp(join(tmpdir(), 'pm-ci-merge-'));
t.after(() => rm(root, { recursive: true, force: true }));
const staged = await stagedShards(root, shards);
try {
execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', runScript(verifyStep())], {
cwd: root, stdio: 'pipe',
env: { PATH: process.env.PATH, STAGED_RAW: staged, EXPECTED_SHARDS: '3' },
});
} catch (error) {
return { ok: false, output: `${error.stdout}${error.stderr}` };
}
return { ok: true, raw: (await readdir(join(root, 'coverage/js/raw'))).sort() };
}
test('merge refuses when a shard\'s raw coverage is missing', async t => {
const missing = await verify(t, [[1, ['a.json']], [3, ['c.json']]]);
assert.equal(missing.ok, false);
assert.match(missing.output, /2 of 3/);
const empty = await verify(t, [[1, ['a.json']], [2, []], [3, ['c.json']]]);
assert.equal(empty.ok, false, 'an artifact without raw files counts as missing');
const collision = await verify(t, [[1, ['a.json']], [2, ['a.json']], [3, ['c.json']]]);
assert.equal(collision.ok, false, 'two shards writing the same file name must not silently drop one');
const complete = await verify(t, [[1, ['a.json', 'node.json']], [2, ['b.json']], [3, ['c.json']]]);
assert.deepEqual(complete, { ok: true, raw: ['a.json', 'b.json', 'c.json', 'node.json'] });
});
test('every browser job sets PM_TEST_WORKERS and runs in the pinned image', () => {
assert.deepEqual([...browserJobIds].sort(), ['browser-workflows', 'frontend-coverage']);
for (const id of browserJobIds) {
const body = allJobs.get(id);
assert.match(body, /^ {6}PM_TEST_WORKERS: 3$/m, `${id} sets PM_TEST_WORKERS for the whole job`);
assert.match(body, PINNED_IMAGE, `${id} keeps the pinned Playwright image`);
assert.match(body, /PIP_SHA256: [0-9a-f]{64}/, `${id} keeps the pinned pip bootstrap`);
assert.match(body, /^ {4}timeout-minutes: \d+$/m);
}
assert.match(mergeJob(), PINNED_IMAGE, 'the merge sees the same workspace paths as the shards');
});
test('page workflows run each browser in two shards', () => {
const workflows = allJobs.get('browser-workflows');
assert.deepEqual(matrixList(workflows, 'browser'), ['firefox', 'webkit']);
assert.deepEqual(matrixList(workflows, 'shard'), ['1', '2']);
assert.match(workflows, /test:workflows -- --project \$\{\{ matrix\.browser \}\} --shard \$\{\{ matrix\.shard \}\}\/2/);
assert.match(workflows, /^ {6}fail-fast: false$/m);
assert.match(workflows, /name: browser-traces-\$\{\{ matrix\.browser \}\}-shard-\$\{\{ matrix\.shard \}\}/);
});
+25
View File
@@ -0,0 +1,25 @@
// A stand-in for tests/browser/server.py: `--port N` (0 = an OS-chosen port)
// serves the readiness page and announces the bound port on stdout as
// `PM_TEST_SERVER_PORT=<n>`, `--fail` writes to stderr and exits before
// listening, and GET /__exit makes the process die unexpectedly after writing a
// last stderr line.
import { createServer } from 'node:http';
const args = process.argv.slice(2);
const port = Number(args[args.indexOf('--port') + 1]);
if (args.includes('--fail')) {
process.stderr.write('fake server: refusing to start\nlast fake stderr line\n');
process.exit(4);
}
const server = createServer((request, response) => {
if (request.url === '/__exit') {
process.stderr.write('fake server: crashed on purpose\n');
response.end('bye', () => process.exit(3));
return;
}
response.end(request.url === '/prompt_manager/test' ? 'ready' : `pid ${process.pid}`);
});
server.listen(port, '127.0.0.1', () => {
process.stdout.write(`fake server: listening\nPM_TEST_SERVER_PORT=${server.address().port}\n`);
});
for (const signal of ['SIGTERM', 'SIGINT']) process.once(signal, () => server.close(() => process.exit(0)));
+93
View File
@@ -0,0 +1,93 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { existsSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { plan, runCoverage } from '../../tools/vendor/frontend-coverage.mjs';
const steps = argv => plan(argv).map(({ step, ...options }) => (Object.keys(options).length ? { step, ...options } : step));
function tempPaths(t) {
const root = mkdtempSync(join(tmpdir(), 'pm-coverage-cli-'));
t.after(() => rmSync(root, { recursive: true, force: true }));
const reportRoot = join(root, 'coverage/js');
return { root, paths: { repositoryRoot: root, tools: join(root, 'tools'), reportRoot, raw: join(reportRoot, 'raw'), sourceRoot: join(root, 'web/js') } };
}
function recorder() {
const calls = [];
const run = (args, env = {}) => { calls.push({ args, env }); };
return { calls, run };
}
test('no arguments plans the unchanged full run', () => {
assert.deepEqual(steps([]), [
'clean', 'node-units', 'viewer', { step: 'workflows', shard: null }, 'report', { step: 'check', updateBaseline: false },
]);
assert.deepEqual(steps(['--update-baseline']), [
'clean', 'node-units', 'viewer', { step: 'workflows', shard: null }, 'report', { step: 'check', updateBaseline: true },
]);
});
test('--shard 2/3 plans only workflows with --shard 2/3 and no report', () => {
assert.deepEqual(steps(['--shard', '2/3']), ['prepare', { step: 'workflows', shard: '2/3' }]);
assert.deepEqual(steps(['--shard=3/3']), ['prepare', { step: 'workflows', shard: '3/3' }]);
});
test('--shard 1/N also runs the node units and the viewer suite', () => {
assert.deepEqual(steps(['--shard', '1/3']), ['prepare', 'node-units', 'viewer', { step: 'workflows', shard: '1/3' }]);
assert.deepEqual(steps(['--shard', '1/1']), ['prepare', 'node-units', 'viewer', { step: 'workflows', shard: '1/1' }]);
});
test('--merge-only plans the report and the check over existing raw coverage', () => {
assert.deepEqual(steps(['--merge-only']), ['require-raw', 'report', { step: 'check', updateBaseline: false }]);
assert.deepEqual(steps(['--merge-only', '--update-baseline']), ['require-raw', 'report', { step: 'check', updateBaseline: true }]);
});
test('invalid shard and mode combinations are refused', () => {
for (const argv of [['--shard'], ['--shard', '0/3'], ['--shard', '4/3'], ['--shard', '2'], ['--shard', 'a/b'], ['--shard', '1/0'], ['--shard', '1.5/3']]) {
assert.throws(() => plan(argv), /--shard needs I\/N/, argv.join(' '));
}
assert.throws(() => plan(['--shard', '1/2', '--merge-only']), /either --shard or --merge-only/);
assert.throws(() => plan(['--shard', '1/2', '--update-baseline']), /--update-baseline .* --merge-only or a full run/);
assert.throws(() => plan(['--shard', '1/2', '--shard', '2/2']), /--shard once/);
assert.throws(() => plan(['--report']), /Unknown argument: --report/);
});
test('--merge-only refuses an empty raw dir', async t => {
const { paths } = tempPaths(t);
const { calls, run } = recorder();
await assert.rejects(runCoverage({ argv: ['--merge-only'], run, paths }), /no raw coverage .*coverage\/js\/raw.* run the --shard steps first/i);
mkdirSync(paths.raw, { recursive: true });
await assert.rejects(runCoverage({ argv: ['--merge-only'], run, paths }), /no raw coverage/i);
writeFileSync(join(paths.raw, 'notes.txt'), 'not coverage');
await assert.rejects(runCoverage({ argv: ['--merge-only'], run, paths }), /no raw coverage/i);
assert.deepEqual(calls, [], 'nothing is reported or checked over partial or empty coverage');
});
test('a shard keeps raw coverage other shards wrote and passes its shard to Playwright', async t => {
const { paths } = tempPaths(t);
mkdirSync(paths.raw, { recursive: true });
const earlier = join(paths.raw, 'browser-earlier-shard.json');
writeFileSync(earlier, '{"result":[]}');
const { calls, run } = recorder();
await runCoverage({ argv: ['--shard', '2/3'], run, paths });
assert.ok(existsSync(earlier), 'raw files from another shard survive');
assert.equal(calls.length, 1, 'shard 2 runs only the workflows');
const [{ args, env }] = calls;
assert.deepEqual(args.slice(-6), ['--project', 'chromium', '--project', 'touch', '--shard', '2/3']);
assert.equal(env.PM_JS_COVERAGE, paths.raw);
});
test('shard 1 runs the node units into the shared raw dir, then the viewer suite, then its workflows', async t => {
const { root, paths } = tempPaths(t);
mkdirSync(join(root, 'tests/js'), { recursive: true });
writeFileSync(join(root, 'tests/js/a.test.js'), '');
const { calls, run } = recorder();
await runCoverage({ argv: ['--shard', '1/3'], run, paths });
assert.equal(calls.length, 3);
assert.deepEqual(calls[0].args, ['--test', 'tests/js/a.test.js']);
assert.equal(calls[0].env.NODE_V8_COVERAGE, paths.raw);
assert.ok(calls[1].args.includes(join(paths.tools, 'playwright.config.mjs')));
assert.deepEqual(calls[2].args.slice(-2), ['--shard', '1/3']);
});
+99 -20
View File
@@ -1,4 +1,4 @@
import { readFileSync, readdirSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
import { existsSync, readFileSync, readdirSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
import { dirname, join, relative } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { spawnSync } from 'node:child_process';
@@ -7,33 +7,69 @@ import { repositoryRoot } from './vendor-viewerjs.mjs';
import { discoverSources, checkCoverage } from '@comfyassets/toolkit/testing/coverage';
export { discoverSources, checkCoverage };
const metrics = ['lines', 'branches', 'functions', 'statements'];
const SHARD = /^([1-9]\d*)\/([1-9]\d*)$/;
const USAGE = 'usage: frontend-coverage.mjs [--update-baseline] | --shard I/N | --merge-only [--update-baseline]';
function run(args, env = {}) {
function runNode(args, env = {}) {
const result = spawnSync(process.execPath, args, { cwd: repositoryRoot, stdio: 'inherit',
env: { ...process.env, ...env } });
if (result.status !== 0) throw new Error(`Command failed (${result.status}): ${args.join(' ')}`);
}
async function main() {
const tools = dirname(fileURLToPath(import.meta.url));
function defaultPaths() {
const reportRoot = join(repositoryRoot, 'coverage/js');
const raw = join(reportRoot, 'raw');
const sourceRoot = join(repositoryRoot, 'web/js');
return { repositoryRoot, tools: dirname(fileURLToPath(import.meta.url)), reportRoot,
raw: join(reportRoot, 'raw'), sourceRoot: join(repositoryRoot, 'web/js') };
}
function parseShard(value) {
const match = SHARD.exec(value ?? '');
if (!match || Number(match[1]) > Number(match[2])) {
throw new Error(`--shard needs I/N with 1 <= I <= N, got ${value ?? 'nothing'}\n${USAGE}`);
}
return { index: Number(match[1]), total: Number(match[2]), value };
}
// Turn the command line into the ordered steps of one coverage run:
// - no mode: the full run (clean, node units, viewer suite, chromium+touch
// workflows, report, check), exactly as before shards existed;
// - --shard I/N: one slice of the workflows into coverage/js/raw, without wiping
// it or reporting; shard 1 also runs the node units and the viewer suite;
// - --merge-only: report and check over raw coverage the shards already wrote.
export function plan(argv) {
let shard = null;
let mergeOnly = false;
let updateBaseline = false;
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === '--update-baseline') updateBaseline = true;
else if (arg === '--merge-only') mergeOnly = true;
else if (arg === '--shard' || arg.startsWith('--shard=')) {
if (shard) throw new Error(`Pass --shard once\n${USAGE}`);
shard = parseShard(arg === '--shard' ? argv[(index += 1)] : arg.slice('--shard='.length));
} else throw new Error(`Unknown argument: ${arg}\n${USAGE}`);
}
if (shard && mergeOnly) throw new Error(`Pass either --shard or --merge-only\n${USAGE}`);
if (shard && updateBaseline) throw new Error(`--update-baseline needs --merge-only or a full run, not a shard\n${USAGE}`);
const check = { step: 'check', updateBaseline };
if (mergeOnly) return [{ step: 'require-raw' }, { step: 'report' }, check];
if (shard) {
const first = shard.index === 1 ? [{ step: 'node-units' }, { step: 'viewer' }] : [];
return [{ step: 'prepare' }, ...first, { step: 'workflows', shard: shard.value }];
}
return [{ step: 'clean' }, { step: 'node-units' }, { step: 'viewer' }, { step: 'workflows', shard: null },
{ step: 'report' }, check];
}
function rawCoverageFiles(raw) {
return existsSync(raw) ? readdirSync(raw).filter(file => file.endsWith('.json')) : [];
}
function checkSummary({ tools, reportRoot, sourceRoot }, updateBaseline) {
const files = discoverSources(sourceRoot);
rmSync(reportRoot, { recursive: true, force: true });
mkdirSync(raw, { recursive: true });
const tests = readdirSync(join(repositoryRoot, 'tests/js')).filter(file => file.endsWith('.test.js')).map(file => `tests/js/${file}`);
run(['--test', ...tests], { NODE_V8_COVERAGE: raw });
const playwright = join(tools, 'node_modules/@playwright/test/cli.js');
run([playwright, 'test', '--config', join(tools, 'playwright.config.mjs')]);
run([playwright, 'test', '--config', join(tools, 'playwright.workflows.config.mjs'),
'--project', 'chromium', '--project', 'touch'], { PM_JS_COVERAGE: raw });
run([join(tools, 'node_modules/c8/bin/c8.js'), 'report', '--all', '--src', sourceRoot,
'--include', 'web/js/**/*.js', '--include', 'web/js/**/*.mjs', '--temp-directory', raw, '--reports-dir', reportRoot,
'--reporter', 'text', '--reporter', 'json-summary', '--reporter', 'html']);
const summary = JSON.parse(readFileSync(join(reportRoot, 'coverage-summary.json'), 'utf8'));
const baselinePath = join(tools, 'frontend-coverage-baseline.json');
if (process.argv.includes('--update-baseline')) {
if (updateBaseline) {
const previous = JSON.parse(readFileSync(baselinePath, 'utf8'));
const failures = checkCoverage(summary, previous, sourceRoot, files.filter(file => previous[file]));
if (failures.length) throw new Error(`Baseline updates cannot lower coverage:\n${failures.join('\n')}`);
@@ -53,8 +89,51 @@ async function main() {
const failures = checkCoverage(summary, baseline, sourceRoot, files);
if (failures.length) throw new Error(`Frontend coverage regressed:\n${failures.join('\n')}`);
}
console.log(`Frontend coverage passed for ${files.length} files. Report: ${relative(repositoryRoot, reportRoot)}/index.html`);
return files.length;
}
export async function runCoverage({ argv = process.argv.slice(2), run = runNode, paths = defaultPaths() } = {}) {
const { repositoryRoot: root, tools, reportRoot, raw, sourceRoot } = paths;
const playwright = join(tools, 'node_modules/@playwright/test/cli.js');
const actions = {
clean() {
rmSync(reportRoot, { recursive: true, force: true });
mkdirSync(raw, { recursive: true });
},
prepare() {
mkdirSync(raw, { recursive: true });
},
'require-raw'() {
if (!rawCoverageFiles(raw).length) {
throw new Error(`No raw coverage in ${relative(root, raw) || raw}: run the --shard steps first, `
+ 'or put every shard\'s raw files there');
}
},
'node-units'() {
const tests = readdirSync(join(root, 'tests/js')).filter(file => file.endsWith('.test.js'))
.map(file => `tests/js/${file}`);
run(['--test', ...tests], { NODE_V8_COVERAGE: raw });
},
viewer() {
run([playwright, 'test', '--config', join(tools, 'playwright.config.mjs')]);
},
workflows({ shard }) {
run([playwright, 'test', '--config', join(tools, 'playwright.workflows.config.mjs'),
'--project', 'chromium', '--project', 'touch', ...(shard ? ['--shard', shard] : [])], { PM_JS_COVERAGE: raw });
},
report() {
run([join(tools, 'node_modules/c8/bin/c8.js'), 'report', '--all', '--src', sourceRoot,
'--include', 'web/js/**/*.js', '--include', 'web/js/**/*.mjs', '--temp-directory', raw, '--reports-dir', reportRoot,
'--reporter', 'text', '--reporter', 'json-summary', '--reporter', 'html']);
},
check({ updateBaseline }) {
const count = checkSummary(paths, updateBaseline);
console.log(`Frontend coverage passed for ${count} files. Report: ${relative(root, reportRoot)}/index.html`);
},
};
for (const { step, ...options } of plan(argv)) await actions[step](options);
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
main().catch(error => { console.error(error.message); process.exitCode = 1; });
runCoverage().catch(error => { console.error(error.message); process.exitCode = 1; });
}
+2
View File
@@ -15,6 +15,8 @@
"test:comfyui": "playwright test --config comfyui.config.mjs",
"test:executions": "playwright test --config playwright.execution.config.mjs",
"test:coverage": "node frontend-coverage.mjs",
"test:coverage:shard": "node frontend-coverage.mjs --shard",
"coverage:merge": "node frontend-coverage.mjs --merge-only",
"coverage:baseline-check": "node coverage-baseline-check.mjs",
"coverage:update": "node frontend-coverage.mjs --update-baseline",
"test:compat": "playwright test --config pack-compat.config.mjs"
+6 -11
View File
@@ -1,23 +1,18 @@
import { defineConfig, devices } from '@playwright/test';
import { repositoryRoot } from './vendor-viewerjs.mjs';
const port = Number(process.env.PM_TEST_PORT || 8765);
import { workflowWorkers } from './test-workers.mjs';
// No global webServer: tests/browser/fixtures.mjs starts one server per worker
// on an OS-chosen port. Files run in parallel; tests inside a file keep their order.
export default defineConfig({
testDir: '../../tests/browser',
outputDir: './test-results/workflows',
testMatch: '*.spec.mjs',
workers: 1,
workers: workflowWorkers(),
fullyParallel: false,
forbidOnly: Boolean(process.env.CI),
retries: 0,
reporter: 'list',
use: { baseURL: `http://127.0.0.1:${port}`, trace: 'retain-on-failure', screenshot: 'only-on-failure' },
webServer: {
command: `node tools/vendor/start-browser-server.mjs --port ${port}`,
cwd: repositoryRoot,
url: `http://127.0.0.1:${port}/prompt_manager/test`,
reuseExistingServer: false,
gracefulShutdown: { signal: 'SIGTERM', timeout: 5000 },
},
use: { trace: 'retain-on-failure', screenshot: 'only-on-failure' },
projects: [
{ name: 'chromium', outputDir: './test-results/workflows/chromium', use: { browserName: 'chromium' } },
{ name: 'firefox', outputDir: './test-results/workflows/firefox', use: { browserName: 'firefox' } },
+154 -6
View File
@@ -1,8 +1,156 @@
import { spawn } from 'node:child_process';
const child = spawn(process.env.PM_TEST_PYTHON || 'python',
['-m', 'tests.browser.server', ...process.argv.slice(2)], { stdio: 'inherit' });
child.on('error', error => { console.error(error.message); process.exitCode = 1; });
child.on('exit', code => { process.exitCode = code ?? 1; });
for (const signal of ['SIGTERM', 'SIGINT']) {
process.once(signal, () => child.kill(signal));
import { resolve } from 'node:path';
import { pathToFileURL } from 'node:url';
import { setTimeout as delay } from 'node:timers/promises';
import { repositoryRoot } from './vendor-viewerjs.mjs';
const READY_PATH = '/prompt_manager/test';
const STARTUP_TIMEOUT_MS = 60_000;
const STOP_TIMEOUT_MS = 5_000;
const POLL_INTERVAL_MS = 100;
const PROBE_TIMEOUT_MS = 1_000;
const STDERR_TAIL_BYTES = 8_192;
const HOST = '127.0.0.1';
// tests/browser/server.py prints this once it holds its listening socket.
const PORT_ANNOUNCEMENT = /^PM_TEST_SERVER_PORT=(\d+)\s*$/m;
const pythonServer = port => ['-m', 'tests.browser.server', '--port', String(port)];
function describeExit({ code, signal }) {
return signal ? `signal ${signal}` : `code ${code}`;
}
// Start one disposable tests/browser/server.py and wait until it serves the
// readiness page. `port` 0 (the default) lets the server bind an OS-chosen port
// itself and announce it on stdout, so nothing can take the port between
// choosing and binding. Resolves to {port, url, exited, exitStatus,
// stderrTail(), stop()}; rejects with the server's stderr tail when it exits
// or stays unready.
export async function startBrowserServer({
port = 0,
command = process.env.PM_TEST_PYTHON || 'python',
args = pythonServer,
cwd = repositoryRoot,
timeoutMs = STARTUP_TIMEOUT_MS,
} = {}) {
if (!Number.isInteger(port) || port < 0 || port > 65535) {
throw new Error(`startBrowserServer needs a port from 0 to 65535, got ${port}`);
}
const child = spawn(command, args(port), { cwd, stdio: ['ignore', 'pipe', 'pipe'] });
let stderr = '';
child.stderr.setEncoding('utf8');
child.stderr.on('data', chunk => { stderr = (stderr + chunk).slice(-STDERR_TAIL_BYTES); });
let stdout = '';
let announced = null;
child.stdout.setEncoding('utf8');
// Keep draining stdout for the server's whole life so it never blocks on a full pipe.
child.stdout.on('data', chunk => {
if (announced !== null) return;
stdout = (stdout + chunk).slice(-STDERR_TAIL_BYTES);
const match = PORT_ANNOUNCEMENT.exec(stdout);
if (match) announced = Number(match[1]);
});
const killOnExit = () => child.kill('SIGKILL');
process.once('exit', killOnExit);
const server = {
port: port || null,
url: port ? `http://${HOST}:${port}` : null,
exitStatus: null,
stopping: false,
stderrTail: () => stderr.trimEnd(),
exited: new Promise(done => {
child.once('error', error => {
stderr += `\n${error.message}`;
server.exitStatus = { code: null, signal: null, error };
done(server.exitStatus);
});
child.once('exit', (code, signal) => {
server.exitStatus = { code, signal };
done(server.exitStatus);
});
}),
async stop() {
server.stopping = true;
if (server.exitStatus === null) {
child.kill('SIGTERM');
const stopped = await Promise.race([server.exited, delay(STOP_TIMEOUT_MS).then(() => false)]);
if (!stopped) {
child.kill('SIGKILL');
await server.exited;
}
}
process.removeListener('exit', killOnExit);
},
};
server.exited.then(() => process.removeListener('exit', killOnExit));
const failure = async reason => {
await server.stop();
const tail = server.stderrTail();
return new Error(`test server on port ${server.port ?? '(not yet announced)'} ${reason}`
+ (tail ? `\n--- server stderr tail ---\n${tail}` : ''));
};
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
if (server.exitStatus !== null) {
throw await failure(`exited with ${describeExit(server.exitStatus)} before it was ready`);
}
if (announced !== null && server.url === null) {
server.port = announced;
server.url = `http://${HOST}:${announced}`;
}
if (announced !== null && announced !== server.port) {
throw await failure(`announced port ${announced} instead`);
}
if (server.url === null) {
await delay(POLL_INTERVAL_MS);
continue;
}
try {
const response = await fetch(`${server.url}${READY_PATH}`, { signal: AbortSignal.timeout(PROBE_TIMEOUT_MS) });
await response.arrayBuffer();
if (response.ok) return server;
} catch {
// Not listening yet.
}
await delay(POLL_INTERVAL_MS);
}
throw await failure(`was not ready at ${READY_PATH} within ${timeoutMs} ms`);
}
// Keep one live server for a Playwright worker. ensure() returns the running
// server, or starts a fresh one when the last one died (a failed start is
// reported, not retried); crash(server) turns an unexpected exit into an error
// carrying the stderr tail.
export function createServerSupervisor({ start }) {
let current = null;
return {
async ensure() {
if (current && current.exitStatus === null) return current;
current = null;
current = await start();
return current;
},
crash(server) {
if (!server || server.stopping || server.exitStatus === null) return null;
const tail = server.stderrTail();
return new Error(`test server on port ${server.port} exited unexpectedly (${describeExit(server.exitStatus)})`
+ (tail ? `\n--- server stderr tail ---\n${tail}` : ''));
},
async stop() {
const server = current;
current = null;
await server?.stop();
},
};
}
// CLI: `node start-browser-server.mjs --port N` runs one server in the foreground.
if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
const child = spawn(process.env.PM_TEST_PYTHON || 'python',
['-m', 'tests.browser.server', ...process.argv.slice(2)], { stdio: 'inherit' });
child.on('error', error => { console.error(error.message); process.exitCode = 1; });
child.on('exit', code => { process.exitCode = code ?? 1; });
for (const signal of ['SIGTERM', 'SIGINT']) {
process.once(signal, () => child.kill(signal));
}
}
+18
View File
@@ -0,0 +1,18 @@
import { availableParallelism } from 'node:os';
const MAX_LOCAL_WORKERS = 8;
const CPUS_PER_WORKER = 4;
// PM_TEST_WORKERS wins; CI without it keeps one worker, so CI only changes
// through explicit workflow edits; locally a quarter of the CPUs, at most eight.
export function workflowWorkers({ env = process.env, cpus = availableParallelism() } = {}) {
if (env.PM_TEST_WORKERS !== undefined) {
const workers = Number(env.PM_TEST_WORKERS);
if (!Number.isInteger(workers) || workers < 1) {
throw new Error(`PM_TEST_WORKERS must be a positive integer, got "${env.PM_TEST_WORKERS}"`);
}
return workers;
}
if (env.CI) return 1;
return Math.max(1, Math.min(MAX_LOCAL_WORKERS, Math.floor(cpus / CPUS_PER_WORKER)));
}