import test from 'node:test'; import assert from 'node:assert/strict'; import { createServer } from 'node:net'; import { spawn } from 'node:child_process'; import { EventEmitter, once } from 'node:events'; import { readFileSync } from 'node:fs'; import { isAbsolute } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { setTimeout as delay } from 'node:timers/promises'; import { startBrowserServer, createServerSupervisor, attachServerStderr, forwardPrefixedLines, SLOW_RESET_PREFIX } from '../../tools/vendor/start-browser-server.mjs'; import { reporters, workflowWorkers } from '../../tools/vendor/test-workers.mjs'; import RunningTestsReporter from '../../tools/vendor/running-tests-reporter.mjs'; const fakeServer = fileURLToPath(new URL('./fake-browser-server.mjs', import.meta.url)); const fake = (...extra) => ({ command: process.execPath, args: port => [fakeServer, '--port', String(port), ...extra] }); const REAL_SERVER_TIMEOUT_MS = 90_000; const launcher = fileURLToPath(new URL('../../tools/vendor/start-browser-server.mjs', import.meta.url)); /** Polls `check` until it is truthy, failing with `what` after `ms`. */ async function waitFor(check, what, ms = 5_000) { const deadline = Date.now() + ms; while (!check()) { if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}`); await delay(20); } } async function writeStderr(server, text) { await (await fetch(`${server.url}/__stderr?text=${encodeURIComponent(text)}`)).text(); await waitFor(() => server.stderrTail().endsWith(text.trimEnd()), `stderr ${JSON.stringify(text)}`); } /** A process that exited counts as gone even while a non-reaping PID 1 (CI's container) leaves it a zombie. */ function alive(pid) { try { process.kill(pid, 0); } catch { return false; } try { return readFileSync(`/proc/${pid}/stat`, 'utf8').replace(/^.*\) /s, '')[0] !== 'Z'; } catch { return true; } } /** Starts a server from a separate Node worker process, SIGKILLs that worker and returns the server's pid. */ async function serverOfKilledWorker(options) { const script = `import { startBrowserServer } from ${JSON.stringify(pathToFileURL(launcher).href)}; const server = await startBrowserServer(${options}); console.log('SERVER_PID=' + server.pid); setInterval(() => {}, 1000);`; const worker = spawn(process.execPath, ['--input-type=module', '-e', script], { stdio: ['ignore', 'pipe', 'inherit'] }); let out = ''; worker.stdout.setEncoding('utf8'); worker.stdout.on('data', chunk => { out += chunk; }); await waitFor(() => /SERVER_PID=\d+\n/.test(out) || worker.exitCode !== null, 'the worker to start its server', REAL_SERVER_TIMEOUT_MS); const pid = Number(/SERVER_PID=(\d+)/.exec(out)?.[1]); assert.ok(pid > 0, `the worker started a server (output: ${out})`); assert.ok(alive(pid), 'the server runs while its worker lives'); worker.kill('SIGKILL'); await once(worker, 'exit'); return pid; } function listen(port = 0) { return new Promise((resolve, reject) => { const server = createServer(); server.once('error', reject); server.listen(port, '127.0.0.1', () => resolve(server)); }); } async function releasedPort() { const probe = await listen(); const { port } = probe.address(); await new Promise(resolve => probe.close(resolve)); return port; } test('free port allocation never reuses a bound port', async t => { const held = await listen(); t.after(() => held.close()); const heldPort = held.address().port; const servers = await Promise.all(Array.from({ length: 6 }, () => startBrowserServer(fake()))); t.after(() => Promise.all(servers.map(server => server.stop()))); const ports = servers.map(server => server.port); for (const server of servers) { assert.ok(Number.isInteger(server.port) && server.port > 0 && server.port < 65536, `port ${server.port} is a TCP port`); assert.equal(server.url, `http://127.0.0.1:${server.port}`); assert.equal(await (await fetch(`${server.url}/prompt_manager/test`)).text(), 'ready'); } assert.ok(!ports.includes(heldPort), 'a port another process holds is never handed out'); assert.equal(new Set(ports).size, ports.length, 'concurrent servers never share a port'); }); test('startBrowserServer asks the server to bind port 0 and reads the port it announces', async () => { const asked = []; const server = await startBrowserServer({ command: process.execPath, args: port => { asked.push(port); return fake().args(port); } }); try { assert.deepEqual(asked, [0], 'no port is reserved and released before the server binds'); assert.ok(server.port > 0); } finally { await server.stop(); } }); test('startBrowserServer serves the readiness page on a requested port and stops cleanly', async () => { const port = await releasedPort(); const server = await startBrowserServer({ port, ...fake() }); assert.equal(server.port, port); assert.equal(server.url, `http://127.0.0.1:${port}`); assert.equal(await (await fetch(`${server.url}/prompt_manager/test`)).text(), 'ready'); await server.stop(); await assert.rejects(fetch(`${server.url}/prompt_manager/test`)); }); test('startBrowserServer reports a startup failure with the server\'s stderr tail', async () => { await assert.rejects(startBrowserServer(fake('--fail')), error => { assert.match(error.message, /exited with code 4 before it was ready/); assert.match(error.message, /last fake stderr line/); return true; }); }); test('startBrowserServer gives up on a server that never announces a port or becomes ready', async () => { const silent = { command: process.execPath, args: () => ['-e', 'setInterval(() => {}, 1000)'] }; await assert.rejects(startBrowserServer({ ...silent, timeoutMs: 500 }), /not ready .* within 500 ms/); }); test('worker server restarts after an unexpected exit', async t => { const supervisor = createServerSupervisor({ start: () => startBrowserServer(fake()) }); t.after(() => supervisor.stop()); const first = await supervisor.ensure(); assert.equal(supervisor.crash(first), null, 'a running server reports no crash'); await fetch(`${first.url}/__exit`); await first.exited; const crash = supervisor.crash(first); assert.match(crash.message, /test server on port \d+ exited unexpectedly \(code 3\)/); assert.match(crash.message, /crashed on purpose/); const second = await supervisor.ensure(); assert.notEqual(second, first, 'the next test gets a fresh server'); assert.equal(await (await fetch(`${second.url}/prompt_manager/test`)).text(), 'ready'); assert.equal(await supervisor.ensure(), second, 'a healthy server is reused'); }); test('supervisor reports a startup failure once and tries again on the next ensure', async t => { let attempts = 0; const supervisor = createServerSupervisor({ start: async () => { attempts += 1; if (attempts === 1) throw new Error('ImportError: no module'); return startBrowserServer(fake()); }, }); t.after(() => supervisor.stop()); await assert.rejects(supervisor.ensure(), /ImportError/); assert.equal(attempts, 1, 'a failed startup is not retried behind the caller\'s back'); const server = await supervisor.ensure(); assert.equal(attempts, 2); assert.equal(await (await fetch(`${server.url}/prompt_manager/test`)).text(), 'ready'); }); test('the port announcement counts only once its line is complete', async t => { const server = await startBrowserServer({ ...fake('--split-announce'), timeoutMs: 5_000 }); t.after(() => server.stop()); assert.equal(await (await fetch(`${server.url}/prompt_manager/test`)).text(), 'ready'); assert.equal(server.url, `http://127.0.0.1:${server.port}`); }); test('server stderr since a mark holds only what came after it, bounded to the newest output', async t => { const server = await startBrowserServer({ ...fake(), stderrLogChars: 64 }); t.after(() => server.stop()); await writeStderr(server, 'earlier test\n'); const mark = server.stderrMark(); assert.equal(server.stderrSince(mark), ''); await writeStderr(server, 'Traceback: this test\n'); assert.equal(server.stderrSince(mark), 'Traceback: this test\n'); const flood = `${'x'.repeat(80)}newest\n`; await writeStderr(server, flood); const kept = server.stderrSince(mark); assert.match(kept, /^\[earlier server stderr dropped\]\n/); assert.ok(kept.endsWith('newest\n'), 'the newest output is kept'); assert.ok(kept.length <= 64 + '[earlier server stderr dropped]\n'.length, `bounded, got ${kept.length} chars`); }); test('a failed test gets the server stderr written during it attached; a passing one does not', async t => { const server = await startBrowserServer(fake()); t.after(() => server.stop()); const attached = []; const info = status => ({ status, expectedStatus: 'passed', attach: async (name, options) => { attached.push({ name, ...options }); } }); await writeStderr(server, 'before the test\n'); const mark = server.stderrMark(); await attachServerStderr(info('failed'), server, mark); assert.deepEqual(attached, [], 'nothing to attach when the server wrote nothing'); await writeStderr(server, 'Traceback (most recent call last): boom\n'); await attachServerStderr(info('passed'), server, mark); assert.deepEqual(attached, [], 'a passing test attaches nothing'); await attachServerStderr(info('timedOut'), server, mark); assert.equal(attached.length, 1); assert.equal(attached[0].name, 'server-stderr'); assert.equal(attached[0].contentType, 'text/plain'); assert.equal(String(attached[0].body), 'Traceback (most recent call last): boom\n'); }); test('only complete slow-reset report lines are forwarded, however the chunks split them', () => { const written = []; const forward = forwardPrefixedLines(SLOW_RESET_PREFIX, text => written.push(text)); forward(`aiohttp access line\n${SLOW_RESET_PREFIX}POST /__test__/reset still`); assert.deepEqual(written, [], 'a report line waits for its newline'); forward(` running after 2.000s\nplain stderr\n${SLOW_RESET_PREFIX}Thread 0x1 [asyncio_0]\n${SLOW_RESET_PREFIX.slice(0, 5)}`); forward(`${SLOW_RESET_PREFIX.slice(5)} File "server.py", line 1\nnot a report ${SLOW_RESET_PREFIX}\n`); assert.deepEqual(written, [ `${SLOW_RESET_PREFIX}POST /__test__/reset still running after 2.000s\n${SLOW_RESET_PREFIX}Thread 0x1 [asyncio_0]\n`, `${SLOW_RESET_PREFIX} File "server.py", line 1\n`, ]); }); test('a slow-reset report on server stderr reaches the worker stderr; other stderr does not', async t => { const reported = []; const server = await startBrowserServer({ ...fake(), report: text => reported.push(text) }); t.after(() => server.stop()); await writeStderr(server, 'Traceback: ordinary stderr\n'); await writeStderr(server, `${SLOW_RESET_PREFIX}POST /__test__/reset answered 200 after 16.064s\n`); assert.deepEqual(reported, [`${SLOW_RESET_PREFIX}POST /__test__/reset answered 200 after 16.064s\n`]); assert.ok(server.stderrTail().includes('Traceback: ordinary stderr'), 'the stderr tail still keeps everything'); }); test('a server exits with its worker even when the worker is SIGKILLed', async () => { const pid = await serverOfKilledWorker(`{ command: process.execPath, args: port => [${JSON.stringify(fakeServer)}, '--port', String(port), '--exit-on-stdin-eof'] }`); await waitFor(() => !alive(pid), `server ${pid} to exit after its worker died`); }); test('tests/browser/server.py exits with a SIGKILLed worker', { timeout: 2 * REAL_SERVER_TIMEOUT_MS }, async () => { const pid = await serverOfKilledWorker(`{ timeoutMs: ${REAL_SERVER_TIMEOUT_MS} }`); await waitFor(() => !alive(pid), `server.py ${pid} to exit after its worker died`, 15_000); }); test('tests/browser/server.py binds an OS-chosen port for --port 0 and still honours --port N', { timeout: 2 * REAL_SERVER_TIMEOUT_MS }, async t => { const held = await listen(); t.after(() => held.close()); const chosen = await startBrowserServer({ timeoutMs: REAL_SERVER_TIMEOUT_MS }); try { assert.notEqual(chosen.port, held.address().port); assert.equal((await fetch(`${chosen.url}/prompt_manager/test`)).status, 200); } finally { await chosen.stop(); } const port = await releasedPort(); const fixed = await startBrowserServer({ port, timeoutMs: REAL_SERVER_TIMEOUT_MS }); try { assert.equal(fixed.port, port); assert.equal((await fetch(`http://127.0.0.1:${port}/prompt_manager/test`)).status, 200); } finally { await fixed.stop(); } }); test('worker count follows PM_TEST_WORKERS, then one in CI, then a quarter of the CPUs up to eight', () => { assert.equal(workflowWorkers({ env: { PM_TEST_WORKERS: '3', CI: '1' }, cpus: 32 }), 3); assert.equal(workflowWorkers({ env: { CI: '1' }, cpus: 32 }), 1); assert.equal(workflowWorkers({ env: {}, cpus: 32 }), 8); assert.equal(workflowWorkers({ env: {}, cpus: 12 }), 3); assert.equal(workflowWorkers({ env: {}, cpus: 2 }), 1); assert.throws(() => workflowWorkers({ env: { PM_TEST_WORKERS: 'many' }, cpus: 8 }), /PM_TEST_WORKERS must be a positive integer/); assert.throws(() => workflowWorkers({ env: { PM_TEST_WORKERS: '0' }, cpus: 8 }), /PM_TEST_WORKERS must be a positive integer/); }); test('CI adds the running-tests reporter and a blob report under test-results/blob/, which the trace upload covers; local runs keep list', () => { assert.equal(reporters({ env: {}, name: 'workflows' }), 'list'); const [list, running, blob] = reporters({ env: { CI: '1' }, name: 'workflows' }); assert.deepEqual(list, ['list']); // Absolute: a config imported from another directory (workflow-artifacts.test.mjs) still finds it. assert.ok(isAbsolute(running[0]) && running[0].endsWith('running-tests-reporter.mjs'), running[0]); assert.deepEqual(blob, ['blob', { outputDir: './test-results/blob/workflows' }]); }); function fakeHost() { const host = new EventEmitter(); Object.assign(host, { pid: 4242, signals: [], cwd: () => '/repo/tools/vendor' }); host.kill = (pid, signal) => host.signals.push([pid, signal]); return host; } const spec = (id, title, line) => ({ id, title, location: { file: '/repo/tests/browser/a.spec.mjs', line }, parent: { project: () => ({ name: 'firefox' }) } }); test('the running-tests reporter names only the tests that never ended, relative to rootDir', () => { let clock = 1_000; const output = []; const host = fakeHost(); const reporter = new RunningTestsReporter({ now: () => clock, write: text => output.push(text), host }); reporter.onBegin({ rootDir: '/repo/tests/browser' }); const done = spec('1', 'finishes', 10); const stuck = spec('2', 'hangs on a fetch', 20); reporter.onTestBegin(done, { retry: 0 }); reporter.onTestBegin(stuck, { retry: 0 }); clock += 500; reporter.onTestEnd(done, { retry: 0 }); clock += 41_500; reporter.onEnd({ status: 'interrupted' }); assert.equal(output.join(''), '\nStill running when the run ended (interrupted): 1\n' + ' [firefox] › a.spec.mjs:20 › hangs on a fetch (running 42.0s)\n'); assert.equal(host.listenerCount('SIGTERM'), 0, 'the run is over: SIGTERM gets its default again'); output.length = 0; reporter.onTestEnd(stuck, { retry: 0 }); reporter.onEnd({ status: 'passed' }); assert.deepEqual(output, [], 'a run where every test ended prints nothing'); }); test('on the first SIGTERM the reporter prints the running tests and hands over to Playwright\'s SIGINT', () => { const output = []; const host = fakeHost(); const reporter = new RunningTestsReporter({ now: () => 0, write: text => output.push(text), host }); reporter.onBegin({ rootDir: '/repo/tests/browser' }); reporter.onTestBegin(spec('2', 'hangs on a fetch', 20), { retry: 0 }); host.emit('SIGTERM'); assert.match(output.join(''), /Still running when the run ended \(SIGTERM\): 1\n {2}\[firefox\] › a\.spec\.mjs:20 › hangs on a fetch/); assert.deepEqual(host.signals, [[4242, 'SIGINT']], 'Playwright\'s own interrupt ends the run and writes the blob'); assert.equal(host.listenerCount('SIGTERM'), 0, 'a second SIGTERM kills the process as usual'); });