From f98f6fa08dd807b2c76732b7eb89bd8035a46e71 Mon Sep 17 00:00:00 2001 From: Vito Date: Fri, 16 Oct 2026 20:56:50 -0700 Subject: [PATCH] test(ci): validate shared toolkit in real PromptManager hosts (#189) * test(ci): prove shared toolkit with real PromptManager host coverage * test(ci): match KikoTools Python 3.13 compatibility coverage --- .github/branch-protection.json | 6 +- .github/workflows/test.yml | 70 ++++++++++++- .gitignore | 3 + TESTING.md | 54 ++++++++-- docs/shared-ui-adoption.md | 71 +++++++++++++ tests/comfyui/prompt-manager.spec.mjs | 133 ++++++++++++++++++++++++ tests/vendor/browser-coverage.test.mjs | 29 ++++++ tests/vendor/comfyui-config.test.mjs | 15 +++ tests/vendor/comfyui-host.test.mjs | 61 +++++++++++ tests/vendor/frontend-coverage.test.mjs | 13 ++- tools/vendor/README.md | 9 ++ tools/vendor/browser-coverage.mjs | 8 +- tools/vendor/comfyui-host.mjs | 39 +++++++ tools/vendor/comfyui-runtime.json | 9 ++ tools/vendor/comfyui.config.mjs | 28 +++++ tools/vendor/frontend-coverage.mjs | 24 +---- tools/vendor/package-lock.json | 11 ++ tools/vendor/package.json | 2 + 18 files changed, 549 insertions(+), 36 deletions(-) create mode 100644 docs/shared-ui-adoption.md create mode 100644 tests/comfyui/prompt-manager.spec.mjs create mode 100644 tests/vendor/browser-coverage.test.mjs create mode 100644 tests/vendor/comfyui-config.test.mjs create mode 100644 tests/vendor/comfyui-host.test.mjs create mode 100644 tools/vendor/comfyui-host.mjs create mode 100644 tools/vendor/comfyui-runtime.json create mode 100644 tools/vendor/comfyui.config.mjs diff --git a/.github/branch-protection.json b/.github/branch-protection.json index da18229..7c12c40 100644 --- a/.github/branch-protection.json +++ b/.github/branch-protection.json @@ -5,16 +5,20 @@ "py3.10 on ubuntu-latest", "py3.11 on ubuntu-latest", "py3.12 on ubuntu-latest", + "py3.13 on ubuntu-latest", "py3.10 on windows-latest", "py3.11 on windows-latest", "py3.12 on windows-latest", + "py3.13 on windows-latest", "py3.10 on macos-latest", "py3.11 on macos-latest", "py3.12 on macos-latest", + "py3.13 on macos-latest", "coverage gate (85%)", "black and flake8", "frontend modules", - "viewerjs" + "viewerjs", + "real ComfyUI compatibility" ] }, "enforce_admins": true, diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9a2425a..fac8f32 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v7 @@ -37,6 +37,62 @@ jobs: - name: Run tests run: python -m unittest discover -s tests -v + comfyui: + name: real ComfyUI compatibility + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-node@v7 + with: + node-version: "24" + cache: npm + cache-dependency-path: tools/vendor/package-lock.json + - name: Read the reviewed runtime pins + id: runtime + run: | + node --input-type=module <<'JS' >> "$GITHUB_OUTPUT" + import runtime from './tools/vendor/comfyui-runtime.json' with { type: 'json' }; + for (const [key, value] of Object.entries(runtime)) console.log(`${key}=${value}`); + JS + - name: Check out pinned ComfyUI + uses: actions/checkout@v7 + with: + repository: ${{ steps.runtime.outputs.repository }} + ref: ${{ steps.runtime.outputs.revision }} + path: .ci-comfyui + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: ${{ steps.runtime.outputs.pythonVersion }} + cache: pip + cache-dependency-path: | + requirements-dev.txt + .ci-comfyui/requirements.txt + tools/vendor/comfyui-runtime.json + - name: Install CPU runtime and test dependencies + run: | + python -m pip install --upgrade pip + pip install "torch==${{ steps.runtime.outputs.torchVersion }}" "torchvision==${{ steps.runtime.outputs.torchvisionVersion }}" --index-url https://download.pytorch.org/whl/cpu + pip install -r .ci-comfyui/requirements.txt -r requirements-dev.txt "torch==${{ steps.runtime.outputs.torchVersion }}" "torchvision==${{ steps.runtime.outputs.torchvisionVersion }}" + - run: npm ci --prefix tools/vendor --ignore-scripts + - run: npx --prefix tools/vendor playwright install --with-deps chromium + - name: Classic and Nodes 2.0 startup and compatibility + env: + PM_COMFY_ROOT: ${{ github.workspace }}/.ci-comfyui + PM_TEST_PYTHON: python + run: npm run --prefix tools/vendor test:comfyui + - name: Upload real-host reports and startup logs + if: always() + uses: actions/upload-artifact@v7 + with: + name: real-comfyui-reports + path: tools/vendor/test-results/ + if-no-files-found: ignore + retention-days: 14 + coverage: name: coverage gate (85%) runs-on: ubuntu-latest @@ -78,6 +134,16 @@ jobs: steps: - uses: actions/checkout@v7 - # Pure-JS frontend modules, tested with the runner's preinstalled Node + - uses: actions/setup-node@v7 + with: + node-version: "24" + + - name: Check shipped extension and page modules parse + run: | + shopt -s globstar nullglob + for f in web/comfy/**/*.js web/comfy/**/*.mjs web/js/**/*.js web/js/**/*.mjs; do + node --input-type=module --check < "$f" || exit 1 + done + - name: Run JS tests run: node --test tests/js/*.test.js diff --git a/.gitignore b/.gitignore index 3a9145d..7e6132c 100644 --- a/.gitignore +++ b/.gitignore @@ -208,3 +208,6 @@ playwright-report/ # Combined frontend coverage reports coverage/ + +# Disposable upstream checkout for real-host CI +.ci-comfyui/ diff --git a/TESTING.md b/TESTING.md index 53086ea..4c89b53 100644 --- a/TESTING.md +++ b/TESTING.md @@ -10,6 +10,7 @@ reviewed. Add a regression test for each bug; prove it fails before the fix. | Python tests | Domain logic, SQLite, API handlers, filesystem checks | ComfyUI host and ML runtimes where needed | | JavaScript tests | Frontend modules and controller methods | DOM/network fixtures for focused edge cases | | Viewer checks | Production Viewer.js and frontend methods in Chromium desktop/touch | Controlled images and page fixtures | +| Real ComfyUI | Pinned CPU host, actual extension, node definitions, widgets, routes and SQLite | Inference is not queued; browser startup settings and host preference writes are isolated | | Page workflows | Production HTML, startup, click handlers, HTTP routes, SQLite, PNG files, thumbnails | ComfyUI host services and the tagging model | Page workflows cover save/edit/reload, duplicate-edit rejection, gallery image @@ -44,11 +45,49 @@ or the port is occupied. Test servers are never reused. On unsupported Linux distributions, browser system dependencies may need manual installation; CI uses Ubuntu. For a quick browser run, use `test:workflows -- --project chromium`. +## Managed real ComfyUI + +The shared engine comes from an exact commit of +[ComfyAssets Toolkit](https://github.com/ComfyAssets/ComfyAssets-Toolkit), recorded +in `tools/vendor/package.json` and its integrity-checked lockfile. The local +adapter sets PromptManager's database/configuration and monitoring policy; +source discovery, coverage mapping/gates and process lifecycle are shared code. +KikoTools remains the reference and is unchanged by this integration. + +Use a **dedicated disposable** ComfyUI checkout at the revision in +`tools/vendor/comfyui-runtime.json`. With a separate Python environment, install +that checkout's requirements, this repo's `requirements-dev.txt`, and the exact +CPU torch/torchvision pins from the JSON (using PyTorch's CPU wheel index). +Install Chromium using the Playwright command above, then run: + +```sh +PM_COMFY_ROOT=/path/to/disposable/ComfyUI \ +PM_TEST_PYTHON=/path/to/cpu-env/bin/python \ +npm run --prefix tools/vendor test:comfyui +``` + +CI performs this setup automatically. `PM_COMFY_URL` can select another +`http://127.0.0.1:PORT/` address (default port 8189). This is an owned managed +server: a running host is never reused. Do not point it at your live checkout. +The launcher verifies the upstream SHA and clean tracked files, snapshots only +tracked PromptManager files, and creates temporary user/input/output/database/ +configuration/log directories. Personal untracked files are excluded. It runs +CPU/offline with only PromptManager enabled and removes owned state on shutdown. +The upstream checkout must also be disposable: ComfyUI may initialize local +runtime files there. Browser host-settings writes are intercepted; actual +PromptManager routes use the disposable host and database. + +`real-comfyui-reports` retains startup logs, version attachments, screenshots and +traces for 14 days. Update upstream/frontend/CPU pins together after reviewing +and running both startup projects; never replace the SHA with a moving branch. +Update the toolkit via an exact reviewed commit archive and commit both npm +manifest and lockfile. No toolkit or npm installation is needed by end users. + ## Coverage and failures Python CI enforces 85% combined statement/branch coverage. Frontend coverage merges Node V8 unit coverage with Chromium page-workflow coverage through c8. Every -`web/js/*.js` file appears, including untouched files at zero coverage. The +`web/js/**/*.js` and `web/js/**/*.mjs` file appears, including untouched files at zero coverage. The controller tests provide V8 filenames so their executed code is counted too. `tools/vendor/frontend-coverage-baseline.json` holds a floor for every file and @@ -64,14 +103,17 @@ The `viewerjs` check runs on every PR, including backend-only changes. GitHub artifacts named `frontend-test-reports` contain HTML coverage, a JSON summary, and failure screenshots/traces. Open traces with Playwright's `show-trace` tool. -Live checks remain useful for ComfyUI host compatibility, real inference/GPU or -model dependency changes, and visual design changes. The automated fixtures do -not certify those environments or every existing product feature. +The real-host suite covers both Classic and Nodes 2.0 startup, switching both +ways, actual database prompt selection, execution-text serialization, workflow +reload, widget/style cleanup, node schemas and standalone-page opening. It does +not queue inference or certify every existing feature, arbitrary host versions, +GPU/model changes, cross-pack installation or visual design changes. Those still +need targeted validation. ## Required merge checks -`.github/branch-protection.json` records the required checks: all nine Python/OS -combinations, Python coverage, formatting, frontend modules, and `viewerjs`. +`.github/branch-protection.json` records the required checks: all twelve Python/OS +combinations, Python coverage, formatting, frontend modules, `viewerjs`, and `real ComfyUI compatibility`. The branch must be current with main, and administrators follow the same checks. It does not require an additional human approval. diff --git a/docs/shared-ui-adoption.md b/docs/shared-ui-adoption.md new file mode 100644 index 0000000..c14723b --- /dev/null +++ b/docs/shared-ui-adoption.md @@ -0,0 +1,71 @@ +# Shared ComfyAssets UI adoption + +Status: approved direction; toolkit/testing foundation in progress. +Product UI brainstorming comes before node changes. + +Tracking: [PromptManager #186](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/186) +and [KikoTools #81](https://github.com/ComfyAssets/ComfyUI-KikoTools/issues/81). + +KikoTools is the validated model. The +[ComfyAssets Toolkit](https://github.com/ComfyAssets/ComfyAssets-Toolkit) is the +shared source; PromptManager proves the package boundaries before a later +KikoTools adoption PR. Other custom nodes can become additional consumers. + +## Approved sequence + +1. Extract reusable controls and test infrastructure from the working KikoTools + reference into the toolkit. Keep routes, node schemas and controllers local. +2. Bring PromptManager's automated tests up to parity using the pinned toolkit. +3. Brainstorm PromptManager node layouts, controls and behavior with the maintainer + before changing its production UI. Select one bounded first consumer together. +4. Validate shared distribution, themes, cleanup and compatibility through that + consumer, then migrate incrementally. Test both packs together as appropriate. +5. Backport toolkit adoption to KikoTools after PromptManager proves it, preserving + the existing KikoTools appearance, behavior and workflow compatibility. + +KikoTools is not being migrated again during this phase. The original +[foundation mockup](https://github.com/ComfyAssets/ComfyUI-KikoTools/pull/82) +records the approved visual direction and inspiration. + +## PromptManager requirements + +- Preserve existing node names, inputs/outputs, API routes, saved workflows, + gallery behavior, and existing test/coverage gates. +- Use shared control factories, editor slots, settings/help/window behavior and + the ComfyUI adapter where applicable. Keep PromptManager controllers and HTTP + behavior in this repository. +- Both projects must look identical: one shared set of controls, typography, + spacing, colors and interaction states. Follow ComfyUI light/dark mode with + our own styling, a blue primary accent and strong red errors/destructive actions. + Use compact buttons, pill switches, segmented selections and responsive rows. + Review the pilot mockup before production styling. Existing `--pm-*` tokens + are migration context, not an approved design to retain. +- Theme standalone pages explicitly: `window.open` pages do not inherit the + host's CSS variables. Test the agreed palette policy, live changes, fallbacks + and persistence, including errors when host settings are unavailable. +- Ship a pinned, reproducible browser distribution. Installation must work + without npm or KikoTools, including when both packs ship different core versions. +- Attribute [Pixaroma](https://github.com/pixaroma/ComfyUI-Pixaroma) as architectural + inspiration. Do not copy/vendor his code, styles, or assets. + +## First migration + +With the maintainer, inventory the PromptManager nodes and gallery dialogs and +choose one bounded first UI consumer after brainstorming. Record its current behavior with tests before changing it. +Verify keyboard opening/closing, focus restoration, accessible labels, confirm +and cancel behavior, network failures, and light/dark appearances. Keep the +existing gallery and destructive-action browser workflows passing. + +Build shared-source extraction and distribution checks before importing the core. +Test both packs alone and together, scoped styling, isolated settings/storage, +and independent cleanup. ComfyUI-facing changes require the real-host Classic, +Nodes 2.0 and live-switching matrix from the foundation proposal. Standalone +dialogs additionally run through PromptManager's existing browser harness. + +## Delivery + +Use small issue-linked PRs and TDD: verify each new behavioral test fails before +implementing it, then run relevant suites and required CI checks. Include visual +evidence for appearance changes. Migrate additional nodes/dialogs/editors only after +the first consumer validates the package boundaries; do not rewrite all pages +or change backend schemas as part of this initiative. diff --git a/tests/comfyui/prompt-manager.spec.mjs b/tests/comfyui/prompt-manager.spec.mjs new file mode 100644 index 0000000..6596ed0 --- /dev/null +++ b/tests/comfyui/prompt-manager.spec.mjs @@ -0,0 +1,133 @@ +import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs'; +import runtime from '../../tools/vendor/comfyui-runtime.json' with { type: 'json' }; + +test.beforeEach(async ({ page, context, request }, testInfo) => { + const response = await request.get('/system_stats'); + expect(response.ok()).toBe(true); + const info = (await response.json()).system; + expect(info.comfyui_version).toBe(runtime.serverVersion); + expect(info.pytorch_version).toBe(`${runtime.torchVersion}+cpu`); + expect(info.required_frontend_version).toBe(runtime.frontendVersion); + expect(info.comfy_package_versions.find(item => item.name === 'comfyui-frontend-package')?.installed).toBe(runtime.frontendVersion); + await testInfo.attach('host-versions.json', { body: JSON.stringify(info, null, 2), contentType: 'application/json' }); + page.hostErrors = []; + page.on('pageerror', error => page.hostErrors.push(error.message)); + await context.route('**/*', async route => { + const request = route.request(); + const path = new URL(request.url()).pathname.replace(/^\/api\//, '/'); + if (path === '/settings' && request.method() === 'GET') { + const response = await route.fetch(); + return route.fulfill({ json: { ...await response.json(), + 'Comfy.VueNodes.Enabled': testInfo.project.metadata.initialRenderer, + 'Comfy.TutorialCompleted': true, 'Comfy.Workflow.Persist': false, + 'Comfy.Notification.ShowVersionUpdates': false } }); + } + if (!['GET', 'HEAD', 'OPTIONS'].includes(request.method()) && !path.startsWith('/prompt_manager/')) return route.fulfill({ json: {} }); + return route.continue(); + }); + await page.goto('/', { waitUntil: 'domcontentloaded' }); + await expect(page.getByRole('status', { name: 'Loading ComfyUI' })).toBeHidden({ timeout: 30000 }); + await page.waitForFunction(async () => { + const { app } = await import('/scripts/app.js'); + return !!app.graph && ['PromptManager', 'PromptManagerText', 'PromptSearchList'].every(name => globalThis.LiteGraph?.registered_node_types?.[name]); + }); + await expect.poll(() => page.evaluate(() => LiteGraph.vueNodesMode)).toBe(testInfo.project.metadata.initialRenderer); + await page.evaluate(async () => { const { app } = await import('/scripts/app.js'); app.graph.clear(); }); +}); + +test.afterEach(async ({ page }) => { + expect(page.hostErrors).toEqual([]); +}); + +for (const type of ['PromptManager', 'PromptManagerText']) { + test(`${type} selects real database prompts and preserves text across renderers and reload`, async ({ page, request }, testInfo) => { + const text = `Host test ${type} ${testInfo.project.name}`; + const saved = await request.post('/prompt_manager/save', { data: { text, category: 'Host tests', tags: ['host-test'] } }); + expect(saved.ok()).toBe(true); + expect((await saved.json()).success).toBe(true); + const id = await page.evaluate(async type => { + const { app } = await import('/scripts/app.js'); + const node = LiteGraph.createNode(type); + app.graph.add(node); + node.setSize([650, 800]); + app.canvas.centerOnNode(node); + return node.id; + }, type); + const recent = page.getByRole('button', { name: '📋 Recent', exact: true }); + await expect(recent).toBeVisible(); + await recent.click(); + await page.getByText(text, { exact: true }).click(); + const state = () => page.evaluate(async id => { + const { app } = await import('/scripts/app.js'); + const node = app.graph.getNodeById(id); + return Object.fromEntries(node.widgets.filter(widget => ['text', 'category', 'tags'].includes(widget.name)).map(widget => [widget.name, widget.value])); + }, id); + await expect.poll(state).toMatchObject({ text, category: 'Host tests', tags: 'host-test' }); + for (const renderer of [!testInfo.project.metadata.initialRenderer, testInfo.project.metadata.initialRenderer]) { + await page.evaluate(async renderer => { + const { app } = await import('/scripts/app.js'); + await app.ui.settings.setSettingValue('Comfy.VueNodes.Enabled', renderer); + app.canvas.ds.scale = .8; + app.canvas.centerOnNode(app.graph._nodes[0]); + }, renderer); + await expect.poll(() => page.evaluate(() => LiteGraph.vueNodesMode)).toBe(renderer); + await expect(recent).toHaveCount(1); + await expect(recent).toBeVisible(); + expect(await state()).toMatchObject({ text }); + } + const output = await page.evaluate(async id => { + const { app } = await import('/scripts/app.js'); + const prompt = await app.graphToPrompt(); + const workflow = app.graph.serialize(); + await app.loadGraphData(workflow); + return prompt.output[String(id)].inputs; + }, id); + expect(output.text).toBe(text); + await expect.poll(state).toMatchObject({ text, category: 'Host tests', tags: 'host-test' }); + await expect(recent).toHaveCount(1); + await page.screenshot({ path: testInfo.outputPath(`${type}-reloaded.png`) }); + const removedStyle = await page.evaluate(async id => { + const { app } = await import('/scripts/app.js'); + const node = app.graph.getNodeById(id); + const style = node._styleSheet; + app.graph.remove(node); + return !style.isConnected; + }, id); + expect(removedStyle).toBe(true); + await expect(recent).toHaveCount(0); + }); +} + +test('all node schemas and extension entrypoints load from the real host', async ({ page, request }) => { + const definitions = await (await request.get('/object_info')).json(); + for (const name of ['PromptManager', 'PromptManagerText', 'PromptSearchList']) expect(definitions[name]?.name).toBe(name); + const extensions = await (await request.get('/extensions')).json(); + expect(extensions.filter(path => path.startsWith('/extensions/promptmanager/'))).toEqual(['/extensions/promptmanager/prompt_manager.js']); + const result = await page.evaluate(async () => { + const { app } = await import('/scripts/app.js'); + const node = LiteGraph.createNode('PromptSearchList'); + app.graph.add(node); + const workflow = app.graph.serialize(); + await app.loadGraphData(workflow); + return { type: app.graph._nodes[0].type, extension: app.extensions.some(item => item.name === 'PromptManager.UI') }; + }); + expect(result).toEqual({ type: 'PromptSearchList', extension: true }); +}); + +test('node Web UI button opens actual standalone pages', async ({ page }) => { + await page.evaluate(async () => { + const { app } = await import('/scripts/app.js'); + const node = LiteGraph.createNode('PromptManagerText'); + app.graph.add(node); + app.canvas.centerOnNode(node); + }); + const popupPromise = page.waitForEvent('popup'); + await page.getByRole('button', { name: '🌐 Web UI', exact: true }).click(); + const popup = await popupPromise; + await popup.waitForLoadState('domcontentloaded'); + await expect(popup).toHaveURL(/\/prompt_manager\/(web|admin)/); + await expect(popup.locator('body')).toContainText('Prompt'); + await popup.goto(new URL('/prompt_manager/gallery', page.url()).href); + await expect(popup.locator('#addPromptBtn')).toBeVisible(); + await popup.close(); +}); diff --git a/tests/vendor/browser-coverage.test.mjs b/tests/vendor/browser-coverage.test.mjs new file mode 100644 index 0000000..39dcde5 --- /dev/null +++ b/tests/vendor/browser-coverage.test.mjs @@ -0,0 +1,29 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync, readdirSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { fileURLToPath } from 'node:url'; +import { saveCoverage } from '../../tools/vendor/browser-coverage.mjs'; + +test('browser coverage maps nested JS/MJS and excludes unrelated and traversal paths', async t => { + const directory = mkdtempSync(join(tmpdir(), 'pm-browser-coverage-')); + const previous = process.env.PM_JS_COVERAGE; + process.env.PM_JS_COVERAGE = directory; + t.after(() => { + if (previous === undefined) delete process.env.PM_JS_COVERAGE; + else process.env.PM_JS_COVERAGE = previous; + rmSync(directory, { recursive: true, force: true }); + }); + const urls = [ + '/prompt_manager/js/gallery.js', + '/prompt_manager/js/shared/controls/button.mjs?v=1', + '/prompt_manager/js/shared/panel.js', + '/scripts/app.js', '/prompt_manager/js/../private.js', + '/prompt_manager/js/shared/theme.css', + ]; + await saveCoverage({ coverage: { stopJSCoverage: async () => urls.map(url => ({ url: `http://127.0.0.1:8765${url}`, functions: [] })) } }, 'chromium'); + const report = JSON.parse(readFileSync(join(directory, readdirSync(directory)[0]), 'utf8')); + const root = fileURLToPath(new URL('../../web/js/', import.meta.url)); + assert.deepEqual(report.result.map(entry => fileURLToPath(entry.url)), [join(root, 'gallery.js'), join(root, 'shared/controls/button.mjs'), join(root, 'shared/panel.js')]); +}); diff --git a/tests/vendor/comfyui-config.test.mjs b/tests/vendor/comfyui-config.test.mjs new file mode 100644 index 0000000..3895637 --- /dev/null +++ b/tests/vendor/comfyui-config.test.mjs @@ -0,0 +1,15 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const root = fileURLToPath(new URL('../../', import.meta.url)); +test('real-ComfyUI suite registers Classic and Nodes 2.0 startup projects', () => { + const output = execFileSync(process.execPath, ['tools/vendor/node_modules/@playwright/test/cli.js', + 'test', '--config', 'tools/vendor/comfyui.config.mjs', '--list'], { + cwd: root, encoding: 'utf8', env: { ...process.env, + PM_COMFY_ROOT: '/unused-pinned-checkout', PM_COMFY_URL: 'http://127.0.0.1:8199/' }, + }); + assert.match(output, /\[classic-start\]/); + assert.match(output, /\[nodes2-start\]/); +}); diff --git a/tests/vendor/comfyui-host.test.mjs b/tests/vendor/comfyui-host.test.mjs new file mode 100644 index 0000000..fb5c9cd --- /dev/null +++ b/tests/vendor/comfyui-host.test.mjs @@ -0,0 +1,61 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdtemp, writeFile, mkdir, readFile, realpath, rm, access } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { once } from 'node:events'; +import { launchComfyHost } from '../../tools/vendor/comfyui-host.mjs'; + +async function fixture(t, program) { + const directory = await mkdtemp(join(tmpdir(), 'pm-host-test-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const checkout = join(directory, 'host'); + const repository = join(directory, 'plugin'); + await mkdir(checkout); + await mkdir(repository); + await writeFile(join(repository, '__init__.py'), ''); + execFileSync('git', ['init', '--quiet', repository]); + execFileSync('git', ['-C', repository, 'add', '__init__.py']); + await writeFile(join(repository, 'config.json'), 'private-user-config'); + await writeFile(join(repository, 'prompts.db'), 'private-user-database'); + await writeFile(join(checkout, 'main.py'), program); + const git = args => execFileSync('git', ['-C', checkout, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); + git(['init', '--quiet']); + git(['add', 'main.py']); + git(['-c', 'user.name=PromptManager Tests', '-c', 'user.email=tests@example.invalid', 'commit', '--quiet', '-m', 'fixture']); + return { checkout, repository, python: process.execPath, port: 8199, + pins: { revision: git(['rev-parse', 'HEAD']) }, logPath: join(directory, 'host.log'), forwardOutput: false }; +} + +const idle = `console.log('ready'); setInterval(() => {}, 1000);`; + +test('PromptManager adapter isolates database, configuration and monitored directories', async t => { + const options = await fixture(t, idle); + const host = await launchComfyHost(options); + t.after(() => host.stop()); + await once(host.child.stdout, 'data'); + const args = host.child.spawnargs; + for (const flag of ['--cpu', '--offline', '--disable-all-custom-nodes']) assert.ok(args.includes(flag)); + assert.equal(args[args.indexOf('--base-directory') + 1], host.stateDirectory); + assert.equal(args[args.indexOf('--listen') + 1], '127.0.0.1'); + assert.equal(args[args.indexOf('--whitelist-custom-nodes') + 1], 'promptmanager'); + const installed = join(host.stateDirectory, 'custom_nodes/promptmanager'); + assert.notEqual(await realpath(installed), await realpath(options.repository)); + assert.equal(await readFile(join(installed, '__init__.py'), 'utf8'), ''); + await assert.rejects(access(join(installed, 'prompts.db')), { code: 'ENOENT' }); + assert.equal(JSON.parse(await readFile(join(installed, 'config.json'), 'utf8')).database.default_path, join(host.stateDirectory, 'prompts.db')); + await writeFile(join(installed, 'prompts.db'), 'test data'); + assert.equal(await readFile(join(options.repository, 'prompts.db'), 'utf8'), 'private-user-database'); + assert.equal(await readFile(join(options.repository, 'config.json'), 'utf8'), 'private-user-config'); + const config = JSON.parse(await readFile(join(installed, 'config.json'), 'utf8')); + assert.equal(config.gallery.monitoring.enabled, false); + assert.deepEqual(config.gallery.monitoring.directories, [join(host.stateDirectory, 'output')]); + assert.equal(config.web_ui.result_timeout, 0); + await access(join(host.stateDirectory, 'user')); + await host.stop(); + await assert.rejects(access(host.stateDirectory), { code: 'ENOENT' }); + await access(join(options.checkout, 'main.py')); + await access(join(options.repository, '__init__.py')); +}); + diff --git a/tests/vendor/frontend-coverage.test.mjs b/tests/vendor/frontend-coverage.test.mjs index ba1845d..be3fe01 100644 --- a/tests/vendor/frontend-coverage.test.mjs +++ b/tests/vendor/frontend-coverage.test.mjs @@ -1,8 +1,10 @@ import test from 'node:test'; import assert from 'node:assert/strict'; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; import { join } from 'node:path'; const sourceRoot = join(process.cwd(), 'web/js'); -import { checkCoverage } from '../../tools/vendor/frontend-coverage.mjs'; +import { checkCoverage, discoverSources } from '../../tools/vendor/frontend-coverage.mjs'; const metrics = value => Object.fromEntries(['lines', 'branches', 'functions', 'statements'].map(key => [key, { pct: value }])); const summary = { total: metrics(90), [join(sourceRoot, 'a.js')]: metrics(85) }; const baseline = { 'a.js': { lines: 80, branches: 80, functions: 80, statements: 80 } }; @@ -20,3 +22,12 @@ test('coverage gate rejects invalid baselines and missing metrics', () => { assert.throws(() => checkCoverage(summary, { 'a.js': { lines: -1 } }, sourceRoot, ['a.js']), /baseline/i); assert.match(checkCoverage({ total: metrics(100), [join(sourceRoot, 'a.js')]: {} }, baseline, sourceRoot, ['a.js']).join('\n'), /lines/); }); + +test('coverage discovers nested JS and MJS while excluding non-source files', t => { + const root = mkdtempSync(join(tmpdir(), 'pm-coverage-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + mkdirSync(join(root, 'shared/controls'), { recursive: true }); + for (const file of ['entry.js', 'shared/core.mjs', 'shared/controls/button.js', 'shared/theme.css', 'shared/notes.txt']) writeFileSync(join(root, file), ''); + assert.deepEqual(discoverSources(root), ['entry.js', 'shared/controls/button.js', 'shared/core.mjs']); + assert.match(checkCoverage({}, {}, root, discoverSources(root)).join('\n'), /baseline.*shared\/controls\/button.js/i); +}); diff --git a/tools/vendor/README.md b/tools/vendor/README.md index 22cd817..b7a5289 100644 --- a/tools/vendor/README.md +++ b/tools/vendor/README.md @@ -78,3 +78,12 @@ windowing is not a replacement for bounding the supplied image list: native slideshow playback can still preload every supplied original. Review new preloading defaults, gesture behavior, CSS changes, and any adapter use of Viewer.js internal methods before adopting another release. + +## Shared ComfyAssets test tooling + +The exact toolkit commit archive in `package.json` / `package-lock.json` supplies +host orchestration and coverage helpers. `comfyui-host.mjs` is PromptManager's +configuration adapter; keep reusable lifecycle changes in the toolkit source. +The existing npm scripts remain the public maintainer entry points. See +[TESTING.md](../../TESTING.md) for the managed real-host matrix and update policy. +This first integration does not yet ship toolkit UI assets to end users. diff --git a/tools/vendor/browser-coverage.mjs b/tools/vendor/browser-coverage.mjs index 222b2da..4aaee07 100644 --- a/tools/vendor/browser-coverage.mjs +++ b/tools/vendor/browser-coverage.mjs @@ -1,6 +1,6 @@ import { mkdirSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; -import { pathToFileURL } from 'node:url'; +import { mapBrowserCoverage } from '@comfyassets/toolkit/testing/coverage'; import { randomUUID } from 'node:crypto'; import { repositoryRoot } from './vendor-viewerjs.mjs'; @@ -12,11 +12,7 @@ export async function startCoverage(page, browserName) { export async function saveCoverage(page, browserName) { if (!process.env.PM_JS_COVERAGE || browserName !== 'chromium') return; const entries = await page.coverage.stopJSCoverage(); - const result = []; - for (const entry of entries) { - const match = entry.url.match(/\/prompt_manager\/js\/([a-z0-9-]+\.js)(?:\?.*)?$/i); - if (match) result.push({ ...entry, url: pathToFileURL(join(repositoryRoot, 'web/js', match[1])).href }); - } + const result = mapBrowserCoverage(entries, { urlPrefix: '/prompt_manager/js/', sourceRoot: join(repositoryRoot, 'web/js') }); mkdirSync(process.env.PM_JS_COVERAGE, { recursive: true }); writeFileSync(join(process.env.PM_JS_COVERAGE, `browser-${randomUUID()}.json`), JSON.stringify({ result })); } diff --git a/tools/vendor/comfyui-host.mjs b/tools/vendor/comfyui-host.mjs new file mode 100644 index 0000000..56b95b0 --- /dev/null +++ b/tools/vendor/comfyui-host.mjs @@ -0,0 +1,39 @@ +import { writeFile } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { launchComfyHost as launchSharedHost } from '@comfyassets/toolkit/testing/host'; +import runtime from './comfyui-runtime.json' with { type: 'json' }; + +const repositoryRoot = fileURLToPath(new URL('../../', import.meta.url)); + +export function launchComfyHost({ checkout, repository = repositoryRoot, + python = process.env.PM_TEST_PYTHON || 'python', port = 8189, pins = runtime, + logPath = join(repositoryRoot, 'tools/vendor/test-results/comfyui-host.log'), forwardOutput = true }) { + return launchSharedHost({ checkout, repository, python, port, pins, logPath, forwardOutput, + pluginName: 'promptmanager', + async preparePlugin({ installed, stateDirectory }) { + const configPath = join(installed, 'config.json'); + await writeFile(configPath, JSON.stringify({ + database: { default_path: join(stateDirectory, 'prompts.db') }, + gallery: { monitoring: { enabled: false, directories: [join(stateDirectory, 'output')] } }, + web_ui: { result_timeout: 0 }, + }), { mode: 0o600 }); + return { PROMPT_MANAGER_CONFIG_PATH: configPath, PROMPT_MANAGER_EXTRA_GALLERY_ROOTS: '' }; + }, + }); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + try { + const host = await launchComfyHost({ checkout: process.env.PM_COMFY_ROOT, + port: Number(process.env.PM_COMFY_PORT || 8189) }); + const stop = () => { host.stop().catch(error => { console.error(error.message); process.exitCode = 1; }); }; + process.once('SIGINT', stop); + process.once('SIGTERM', stop); + const result = await host.finished; + process.removeListener('SIGINT', stop); + process.removeListener('SIGTERM', stop); + if (result.code) console.error(`ComfyUI host exited with status ${result.code}; see tools/vendor/test-results/comfyui-host.log`); + process.exitCode = result.code; + } catch (error) { console.error(error.message); process.exitCode = 1; } +} diff --git a/tools/vendor/comfyui-runtime.json b/tools/vendor/comfyui-runtime.json new file mode 100644 index 0000000..6bb4828 --- /dev/null +++ b/tools/vendor/comfyui-runtime.json @@ -0,0 +1,9 @@ +{ + "repository": "Comfy-Org/ComfyUI", + "revision": "e8665ca8e4089ab917567cdb491308e1397663d4", + "serverVersion": "0.39.0", + "frontendVersion": "1.55.14", + "pythonVersion": "3.12", + "torchVersion": "2.11.0", + "torchvisionVersion": "0.26.0" +} diff --git a/tools/vendor/comfyui.config.mjs b/tools/vendor/comfyui.config.mjs new file mode 100644 index 0000000..c1a0c1a --- /dev/null +++ b/tools/vendor/comfyui.config.mjs @@ -0,0 +1,28 @@ +import { defineConfig } from './node_modules/@playwright/test/index.mjs'; +import { fileURLToPath } from 'node:url'; + +if (!process.env.PM_COMFY_ROOT) throw new Error('Set PM_COMFY_ROOT to a pinned disposable ComfyUI checkout'); +const host = new URL(process.env.PM_COMFY_URL || 'http://127.0.0.1:8189/'); +if (host.protocol !== 'http:' || host.hostname !== '127.0.0.1' || !host.port) throw new Error('A managed host needs an HTTP 127.0.0.1 URL with an explicit port'); +const servers = [{ + command: 'node tools/vendor/comfyui-host.mjs', + cwd: fileURLToPath(new URL('../../', import.meta.url)), + url: new URL('system_stats', host).href, timeout: 120000, reuseExistingServer: false, + env: { PM_COMFY_PORT: host.port }, + gracefulShutdown: { signal: 'SIGTERM', timeout: 10000 }, +}]; + +export default defineConfig({ + testDir: '../../tests/comfyui', + outputDir: './test-results/comfyui', + timeout: 60000, + workers: 1, + forbidOnly: Boolean(process.env.CI), + retries: 0, + use: { baseURL: host.href, browserName: 'chromium', viewport: { width: 1600, height: 1100 }, trace: 'retain-on-failure', screenshot: 'only-on-failure' }, + webServer: servers, + projects: [ + { name: 'classic-start', metadata: { initialRenderer: false } }, + { name: 'nodes2-start', metadata: { initialRenderer: true } }, + ], +}); diff --git a/tools/vendor/frontend-coverage.mjs b/tools/vendor/frontend-coverage.mjs index e3ceae4..ec9af1d 100644 --- a/tools/vendor/frontend-coverage.mjs +++ b/tools/vendor/frontend-coverage.mjs @@ -4,25 +4,9 @@ import { fileURLToPath, pathToFileURL } from 'node:url'; import { spawnSync } from 'node:child_process'; import { repositoryRoot } from './vendor-viewerjs.mjs'; +import { discoverSources, checkCoverage } from '@comfyassets/toolkit/testing/coverage'; +export { discoverSources, checkCoverage }; const metrics = ['lines', 'branches', 'functions', 'statements']; -export function checkCoverage(summary, baseline, sourceRoot, files) { - const failures = []; - for (const file of files) { - const floor = baseline[file]; - if (!floor) { failures.push(`Missing baseline for ${file}`); continue; } - const measured = summary[join(sourceRoot, file)]; - if (!measured) { failures.push(`Missing coverage for ${file}`); continue; } - for (const metric of metrics) { - if (!Number.isFinite(floor[metric]) || floor[metric] < 0 || floor[metric] > 100) { - throw new Error(`Invalid baseline for ${file}: ${metric}`); - } - if (!Number.isFinite(measured[metric]?.pct) || measured[metric].pct < floor[metric]) { - failures.push(`${file} ${metric}: ${measured[metric]?.pct ?? 'missing'} < ${floor[metric]}`); - } - } - } - return failures; -} function run(args, env = {}) { const result = spawnSync(process.execPath, args, { cwd: repositoryRoot, stdio: 'inherit', @@ -35,7 +19,7 @@ async function main() { const reportRoot = join(repositoryRoot, 'coverage/js'); const raw = join(reportRoot, 'raw'); const sourceRoot = join(repositoryRoot, 'web/js'); - const files = readdirSync(sourceRoot).filter(file => file.endsWith('.js')).sort(); + const files = discoverSources(sourceRoot); rmSync(reportRoot, { recursive: true, force: true }); mkdirSync(raw, { recursive: true }); const tests = readdirSync(join(repositoryRoot, 'tests/js')).filter(file => file.endsWith('.test.js')).map(file => `tests/js/${file}`); @@ -45,7 +29,7 @@ async function main() { run([playwright, 'test', '--config', join(tools, 'playwright.workflows.config.mjs'), '--project', 'chromium', '--project', 'touch'], { PM_JS_COVERAGE: raw }); run([join(tools, 'node_modules/c8/bin/c8.js'), 'report', '--all', '--src', sourceRoot, - '--include', 'web/js/*.js', '--temp-directory', raw, '--reports-dir', reportRoot, + '--include', 'web/js/**/*.js', '--include', 'web/js/**/*.mjs', '--temp-directory', raw, '--reports-dir', reportRoot, '--reporter', 'text', '--reporter', 'json-summary', '--reporter', 'html']); const summary = JSON.parse(readFileSync(join(reportRoot, 'coverage-summary.json'), 'utf8')); const baselinePath = join(tools, 'frontend-coverage-baseline.json'); diff --git a/tools/vendor/package-lock.json b/tools/vendor/package-lock.json index 2567e97..b1a3324 100644 --- a/tools/vendor/package-lock.json +++ b/tools/vendor/package-lock.json @@ -8,6 +8,7 @@ "name": "promptmanager-vendor-tools", "version": "1.0.0", "devDependencies": { + "@comfyassets/toolkit": "https://codeload.github.com/ComfyAssets/ComfyAssets-Toolkit/tar.gz/66644f5620e10efeaba090dca2d5be54baa2ee14", "@playwright/test": "1.64.1", "c8": "12.0.0", "viewerjs": "1.15.2" @@ -23,6 +24,16 @@ "node": ">=18" } }, + "node_modules/@comfyassets/toolkit": { + "version": "0.1.0", + "resolved": "https://codeload.github.com/ComfyAssets/ComfyAssets-Toolkit/tar.gz/66644f5620e10efeaba090dca2d5be54baa2ee14", + "integrity": "sha512-jvDj3iYJHjRCH7nZimoF4KNLwEVdzKVQIOZrMfoLg9aLnHm98R8YvBUOvnlJ+JNrM8/LnD2m30u3ggSTnR0/sw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=24" + } + }, "node_modules/@istanbuljs/schema": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.6.tgz", diff --git a/tools/vendor/package.json b/tools/vendor/package.json index 13acd0e..aed04ae 100644 --- a/tools/vendor/package.json +++ b/tools/vendor/package.json @@ -10,10 +10,12 @@ "test": "node --test ../../tests/vendor/*.test.mjs", "test:browser": "playwright test --config playwright.config.mjs", "test:workflows": "playwright test --config playwright.workflows.config.mjs", + "test:comfyui": "playwright test --config comfyui.config.mjs", "test:coverage": "node frontend-coverage.mjs", "coverage:update": "node frontend-coverage.mjs --update-baseline" }, "devDependencies": { + "@comfyassets/toolkit": "https://codeload.github.com/ComfyAssets/ComfyAssets-Toolkit/tar.gz/66644f5620e10efeaba090dca2d5be54baa2ee14", "@playwright/test": "1.64.1", "c8": "12.0.0", "viewerjs": "1.15.2"