test(comfyui): refuse a real-host port another run holds (#310)

The real-host config waited on the /system_stats URL, which any server on
the port answers. When two runs started on 8189 within a few seconds of
each other, the second run's ComfyUI failed to bind ("Port 8189 is already
in use"), but Playwright had already seen the first run's host answer and
ran the second run's tests against it. When the first run finished and
stopped its host, the second run failed mid-test with ECONNREFUSED,
ECONNRESET or ERR_CONNECTION_REFUSED, and passed when rerun alone.

The launcher now refuses a port that is in use, and reports ready only
when /system_stats names its own base directory. The config waits for
that line instead of the URL, and every test checks again that the host
answering the port is the one its run started.

tests/comfyui/host-fixture.mjs

@@ -1,11 +1,20 @@
 import { expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
 import runtime from '../../tools/vendor/comfyui-runtime.json' with { type: 'json' };
+import { hostBaseDirectory } from '../../tools/vendor/comfyui-host.mjs';
 
 export function installHostHooks(test, expectedTypes = ['PromptManager', 'PromptManagerText', 'PromptSearchList']) {
     test.beforeEach(async ({ page, context, request }, testInfo) => {
         const response = await request.get('/system_stats');
         expect(response.ok()).toBe(true);
-        const info = (await response.json()).system;
+        const stats = await response.json();
+        // comfyui.config.mjs's launcher reports the host it owns. The execution suite has no
+        // web server: it starts its own host per worker behind a port refusal.
+        if (testInfo.config.webServer) {
+            const ownedBase = process.env.PM_COMFY_HOST_BASE;
+            expect(ownedBase, 'the launcher reported the host it started').toBeTruthy();
+            expect(hostBaseDirectory(stats), 'the host answering the port is the one this run started').toBe(ownedBase);
+        }
+        const info = stats.system;
         expect(info.comfyui_version).toBe(runtime.serverVersion);
         expect(info.pytorch_version).toBe(`${runtime.torchVersion}+cpu`);
         expect(info.required_frontend_version).toBe(runtime.frontendVersion);

tests/vendor/comfyui-config.test.mjs

@@ -51,3 +51,17 @@ test('CI retains stock, execution and pack suites in order with uploaded executi
     assert.deepEqual(commands, ['test:comfyui', 'test:executions', 'test:compat']);
     assert.match(workflow, /path: tools\/vendor\/test-results\//);
 });
+
+test('real-host config waits for its own launcher, never for whichever server holds the port', async t => {
+    const previous = process.env.PM_COMFY_ROOT;
+    process.env.PM_COMFY_ROOT = '/unused-pinned-checkout';
+    t.after(() => { if (previous === undefined) delete process.env.PM_COMFY_ROOT; else process.env.PM_COMFY_ROOT = previous; });
+    const { default: config } = await import('../../tools/vendor/comfyui.config.mjs');
+    const [server] = config.webServer;
+    assert.equal(server.url, undefined);
+    assert.equal(server.port, undefined);
+    assert.equal(server.reuseExistingServer, false);
+    const ready = server.wait.stdout.exec('Starting server\nPM_COMFY_HOST_READY base-directory=/tmp/comfyassets-host-a b\r\nlater');
+    assert.equal(ready.groups.pm_comfy_host_base, '/tmp/comfyassets-host-a b');
+    assert.equal(server.wait.stdout.exec('To see the GUI go to: http://127.0.0.1:8189'), null);
+});

tests/vendor/comfyui-host.test.mjs

@@ -6,7 +6,8 @@ import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { once } from 'node:events';
 import { createServer } from 'node:net';
-import { launchComfyHost } from '../../tools/vendor/comfyui-host.mjs';
+import { createServer as createHttpServer } from 'node:http';
+import { launchComfyHost, assertPortFree, waitForOwnHost, hostBaseDirectory } from '../../tools/vendor/comfyui-host.mjs';
 
 async function fixture(t, program) {
     const directory = await mkdtemp(join(tmpdir(), 'pm-host-test-'));
@@ -232,3 +233,51 @@ test('execution launcher refuses an occupied port without touching an existing s
         await new Promise(resolve => server.close(resolve));
     }
 });
+
+// A stand-in ComfyUI that answers /system_stats with its own argv, as the real host does.
+const answersOwnArgv = `const args = process.argv.slice(1);
+require('node:http').createServer((request, response) => {
+    response.setHeader('content-type', 'application/json');
+    response.end(JSON.stringify({ system: { argv: args } }));
+}).listen(Number(args[args.indexOf('--port') + 1]), '127.0.0.1');`;
+
+async function listen(t, port, handler = (request, response) => response.end()) {
+    const server = createHttpServer(handler);
+    await new Promise((resolve, reject) => { server.once('error', reject); server.listen(port, '127.0.0.1', resolve); });
+    t.after(() => new Promise(resolve => server.close(resolve)));
+    return server;
+}
+
+test('real-host launcher refuses a port another run already holds and leaves that server running', async t => {
+    const server = await listen(t, 8199);
+    await assert.rejects(assertPortFree(8199), /8199 is already in use.*never shared/s);
+    assert.equal(server.listening, true);
+});
+
+test('real-host launcher is ready only when its own ComfyUI answers the port', async t => {
+    const options = await fixture(t, answersOwnArgv);
+    await assertPortFree(options.port);
+    const host = await launchComfyHost(options);
+    t.after(() => host.stop());
+    await waitForOwnHost(host, options.port);
+    const stats = await (await fetch(`http://127.0.0.1:${options.port}/system_stats`)).json();
+    assert.equal(hostBaseDirectory(stats), host.stateDirectory);
+});
+
+test('real-host launcher refuses a host it did not start that answers before its own binds', async t => {
+    await listen(t, 8199, (request, response) => {
+        response.setHeader('content-type', 'application/json');
+        response.end(JSON.stringify({ system: { argv: ['main.py', '--base-directory', '/other-run-state'] } }));
+    });
+    const options = await fixture(t, idle);
+    const host = await launchComfyHost(options);
+    t.after(() => host.stop());
+    await assert.rejects(waitForOwnHost(host, options.port), /did not start \(base directory \/other-run-state\)/);
+});
+
+test('real-host launcher reports a host that exits before it answers', async t => {
+    const options = await fixture(t, 'process.exit(3);');
+    const host = await launchComfyHost(options);
+    t.after(() => host.stop());
+    await assert.rejects(waitForOwnHost(host, options.port), /exited before it answered on port 8199 \(exit=3/);
+});