From 800f9dd48ce7b60212bad46b1f12f62e8d5deb8f Mon Sep 17 00:00:00 2001 From: atmaranto <35581227+atmaranto@users.noreply.github.com> Date: Tue, 21 Jul 2026 04:40:36 -0500 Subject: [PATCH] Test workflow --- .github/workflows/daily-e2e.yml | 130 +++++++++++++++++++++ test/e2e_save_as_script.py | 201 ++++++++++++++++++++++++++++++++ test/test_workflow.json | 42 +++++++ test/verify_generated_script.py | 59 ++++++++++ 4 files changed, 432 insertions(+) create mode 100644 .github/workflows/daily-e2e.yml create mode 100644 test/e2e_save_as_script.py create mode 100644 test/test_workflow.json create mode 100644 test/verify_generated_script.py diff --git a/.github/workflows/daily-e2e.yml b/.github/workflows/daily-e2e.yml new file mode 100644 index 0000000..ab3c263 --- /dev/null +++ b/.github/workflows/daily-e2e.yml @@ -0,0 +1,130 @@ +name: Daily end-to-end test + +# Exercises the whole path a user takes: run ComfyUI, open a workflow through +# File > Open, hit "Save as Script", and check the exported script actually runs. +on: + schedule: + # 00:00 UTC every day + - cron: '0 0 * * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: daily-e2e + cancel-in-progress: true + +jobs: + export-and-run: + runs-on: ubuntu-latest + timeout-minutes: 60 + + container: + image: python:3.12-slim + # Chromium needs more shared memory than a container gets by default + options: --shm-size=2g + + env: + COMFYUI_URL: http://127.0.0.1:8188 + # Where the exported script and its output land + EXPORT_DIR: ${{ github.workspace }}/exported + + steps: + - name: Install system packages + run: | + apt-get update + apt-get install -y --no-install-recommends git curl ca-certificates + + - name: Check out the extension + uses: actions/checkout@v4 + with: + path: extension + + - name: Clone ComfyUI + # The directory must be named ComfyUI: the generated script locates it by name + run: git clone --depth 1 https://github.com/comfy-org/ComfyUI.git ComfyUI + + - name: Install PyTorch (CPU build) + # Installed before ComfyUI's requirements so pip doesn't pull the CUDA wheels, + # which are several GB and useless on a CPU runner + run: | + python -m pip install --upgrade pip + pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu + + - name: Install ComfyUI requirements + working-directory: ComfyUI + run: pip install -r requirements.txt + + - name: Install the extension + run: | + cp -r extension ComfyUI/custom_nodes/ComfyUI-SaveAsScript + pip install -r ComfyUI/custom_nodes/ComfyUI-SaveAsScript/requirements.txt + + - name: Install Playwright + run: | + pip install playwright + playwright install --with-deps chromium + + - name: Start ComfyUI + working-directory: ComfyUI + run: | + python main.py --cpu --listen 127.0.0.1 --port 8188 --disable-auto-launch \ + > comfyui.log 2>&1 & + echo $! > comfyui.pid + + for _ in $(seq 1 60); do + if curl -sf "$COMFYUI_URL/system_stats" > /dev/null; then + echo "ComfyUI is up" + exit 0 + fi + if ! kill -0 "$(cat comfyui.pid)" 2>/dev/null; then + echo "::error::ComfyUI exited during startup" + cat comfyui.log + exit 1 + fi + sleep 5 + done + + echo "::error::ComfyUI did not become ready within 5 minutes" + cat comfyui.log + exit 1 + + - name: Export the workflow through the browser + run: | + python extension/test/e2e_save_as_script.py \ + --url "$COMFYUI_URL" \ + --workflow extension/test/test_workflow.json \ + --output "$EXPORT_DIR/test_workflow.py" \ + --expect-node EmptyImage \ + --artifacts "$EXPORT_DIR/artifacts" + + - name: Stop ComfyUI + working-directory: ComfyUI + run: kill "$(cat comfyui.pid)" 2>/dev/null || true + + - name: Run the exported script + # From inside ComfyUI, as a user would run it + working-directory: ComfyUI + run: | + python "$EXPORT_DIR/test_workflow.py" \ + --queue-size 1 \ + --output "$EXPORT_DIR/output.png" + + - name: Check the export and its output + run: | + python extension/test/verify_generated_script.py \ + --script "$EXPORT_DIR/test_workflow.py" \ + --expect EmptyImage RandomCropImages SaveImage \ + --image "$EXPORT_DIR/output.png" + + - name: Upload artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-artifacts + path: | + ${{ github.workspace }}/exported + ${{ github.workspace }}/ComfyUI/comfyui.log + if-no-files-found: ignore + retention-days: 14 diff --git a/test/e2e_save_as_script.py b/test/e2e_save_as_script.py new file mode 100644 index 0000000..fc837a4 --- /dev/null +++ b/test/e2e_save_as_script.py @@ -0,0 +1,201 @@ +"""Drives a running ComfyUI through the browser to export a workflow as a script. + +Opens a workflow, clicks the "Save as Script" button, and writes the downloaded Python +file to --output. Used by the daily CI run, but it works against any local ComfyUI: + + python e2e_save_as_script.py --workflow test_workflow.json --output out.py +""" + +import argparse +import os +import re +import sys + +from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutError + +# The frontend takes a while to boot on a cold CPU-only container +READY_TIMEOUT = 180_000 +ACTION_TIMEOUT = 60_000 +# Generating the script imports every node module, so give it room +DOWNLOAD_TIMEOUT = 300_000 + + +def dismiss_startup_dialogs(page): + """Closes the welcome/template dialogs some frontend versions open on first load.""" + for _ in range(3): + closed = False + for selector in ('[data-pc-name="dialog"] [aria-label="Close"]', + '[data-pc-name="dialog"] .p-dialog-close-button', + '.p-dialog-header-close'): + button = page.locator(selector) + if button.count() and button.first.is_visible(): + button.first.click() + closed = True + page.wait_for_timeout(500) + if not closed: + break + page.keyboard.press("Escape") + page.wait_for_timeout(500) + + +def first_visible(locator): + """Returns the first visible match, or None.""" + for i in range(locator.count()): + if locator.nth(i).is_visible(): + return locator.nth(i) + return None + + +def click_menu_item(page, label): + """Clicks a menu entry by its visible label.""" + exact = re.compile(rf"^\s*{re.escape(label)}\s*$", re.IGNORECASE) + for candidate in (page.get_by_role("menuitem", name=exact), + page.locator(f'xpath=//*[normalize-space(text())="{label}"]')): + item = first_visible(candidate) + if item: + item.click() + return True + return False + + +def open_workflow(page, workflow): + """Loads `workflow` the way a user opens a workflow file. + + Current ComfyUI has no File > Open menu; the open-workflow command + (Comfy.OpenWorkflow) is bound to Ctrl+O and clicks a hidden file input. Older + frontends exposed the same command under File > Open, so that is tried second. + """ + try: + with page.expect_file_chooser(timeout=20_000) as chooser_info: + page.keyboard.press("Control+o") + chooser_info.value.set_files(workflow) + return + except PlaywrightTimeoutError: + print("Ctrl+O did not open a file chooser; trying the File > Open menu", + file=sys.stderr) + + with page.expect_file_chooser(timeout=ACTION_TIMEOUT) as chooser_info: + if not click_menu_item(page, "File"): + raise RuntimeError( + "Could not open a workflow: Ctrl+O opened no file chooser and there is " + "no 'File' menu. ComfyUI's open-workflow entry point may have changed." + ) + page.wait_for_timeout(500) + if not click_menu_item(page, "Open"): + raise RuntimeError("Found a 'File' menu but no 'Open' item inside it.") + chooser_info.value.set_files(workflow) + + +def wait_for_workflow(page, expect_node): + """Waits until the loaded graph actually contains `expect_node`.""" + if not expect_node: + page.wait_for_timeout(2000) + return + try: + page.wait_for_function( + """(type) => (window.app?.graph?._nodes ?? []).some( + (n) => n?.type === type || n?.comfyClass === type)""", + arg=expect_node, + timeout=ACTION_TIMEOUT, + ) + except PlaywrightTimeoutError: + # Not every frontend exposes window.app; the exported script is checked + # separately, so this is a warning rather than a failure. + print(f"warning: could not confirm a {expect_node} node via window.app", + file=sys.stderr) + page.wait_for_timeout(1000) + + +def find_save_as_script_button(page): + """Returns the visible "Save as Script" button. + + Matching is on button text, not accessible name: the menubar button carries an + aria-label ("Save the current workflow as a Python script") that would otherwise + hide it from a name-based lookup. The extension also adds a button to the legacy + .comfy-menu sidebar, which is normally hidden. + """ + label = re.compile("Save as Script", re.IGNORECASE) + for candidate in (page.locator("button").filter(has_text=label), + page.get_by_role("button", name=label), + page.locator('button[aria-label*="Python script" i]')): + button = first_visible(candidate) + if button: + return button + + raise RuntimeError( + "The 'Save as Script' button is not present. The extension's setup() may have " + "failed to run - check the browser console output above." + ) + + +def export_script(page, name): + """Clicks "Save as Script", answers the filename prompt, returns the download.""" + button = find_save_as_script_button(page) + + # The extension asks for the filename with window.prompt(); without a handler + # Playwright auto-dismisses it and no download is ever produced. + page.once("dialog", lambda dialog: dialog.accept(name)) + + with page.expect_download(timeout=DOWNLOAD_TIMEOUT) as download_info: + button.click() + return download_info.value + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--url", default="http://127.0.0.1:8188", help="Where ComfyUI is serving") + parser.add_argument("--workflow", required=True, help="Workflow JSON to open") + parser.add_argument("--output", required=True, help="Where to write the exported script") + parser.add_argument("--expect-node", default=None, + help="Node type that must appear in the graph once loaded") + parser.add_argument("--artifacts", default=None, help="Directory for failure screenshots") + parser.add_argument("--headed", action="store_true", help="Show the browser (for debugging)") + args = parser.parse_args() + + workflow = os.path.abspath(args.workflow) + output = os.path.abspath(args.output) + os.makedirs(os.path.dirname(output) or ".", exist_ok=True) + if args.artifacts: + os.makedirs(args.artifacts, exist_ok=True) + + with sync_playwright() as playwright: + browser = playwright.chromium.launch( + headless=not args.headed, + # Chromium crashes with the small /dev/shm containers get by default + args=["--disable-dev-shm-usage", "--no-sandbox"], + ) + context = browser.new_context(accept_downloads=True) + page = context.new_page() + page.set_default_timeout(ACTION_TIMEOUT) + page.on("console", lambda msg: print(f"[browser:{msg.type}] {msg.text}")) + page.on("pageerror", lambda err: print(f"[browser:pageerror] {err}", file=sys.stderr)) + + try: + page.goto(args.url, wait_until="domcontentloaded", timeout=READY_TIMEOUT) + page.wait_for_selector("#graph-canvas", timeout=READY_TIMEOUT) + page.wait_for_timeout(3000) + dismiss_startup_dialogs(page) + + open_workflow(page, workflow) + wait_for_workflow(page, args.expect_node) + + download = export_script(page, os.path.splitext(os.path.basename(output))[0]) + download.save_as(output) + print(f"Exported script to {output}") + except Exception: + if args.artifacts: + page.screenshot(path=os.path.join(args.artifacts, "failure.png"), full_page=True) + with open(os.path.join(args.artifacts, "failure.html"), "w", encoding="utf-8") as f: + f.write(page.content()) + print(f"Wrote failure artifacts to {args.artifacts}", file=sys.stderr) + raise + finally: + context.close() + browser.close() + + if not os.path.getsize(output): + raise SystemExit("The downloaded script is empty") + + +if __name__ == "__main__": + main() diff --git a/test/test_workflow.json b/test/test_workflow.json new file mode 100644 index 0000000..8db19ed --- /dev/null +++ b/test/test_workflow.json @@ -0,0 +1,42 @@ +{ + "2": { + "inputs": { + "width": 512, + "height": 512, + "batch_size": 1, + "color": 0 + }, + "class_type": "EmptyImage", + "_meta": { + "title": "Empty Image" + } + }, + "3": { + "inputs": { + "width": 512, + "height": 512, + "seed": 0, + "images": [ + "2", + 0 + ] + }, + "class_type": "RandomCropImages", + "_meta": { + "title": "Crop Image (Random)" + } + }, + "5": { + "inputs": { + "filename_prefix": "ComfyUI", + "images": [ + "3", + 0 + ] + }, + "class_type": "SaveImage", + "_meta": { + "title": "Save Image" + } + } +} \ No newline at end of file diff --git a/test/verify_generated_script.py b/test/verify_generated_script.py new file mode 100644 index 0000000..de1e62f --- /dev/null +++ b/test/verify_generated_script.py @@ -0,0 +1,59 @@ +"""Checks that an exported script looks right and produced a real image. + +Run after e2e_save_as_script.py has downloaded a script and that script has been +executed, to confirm the export reflects the workflow rather than just being +syntactically valid Python. +""" + +import argparse +import ast +import os +import sys + +PNG_MAGIC = b"\x89PNG\r\n\x1a\n" + + +def check_script(path, expected): + with open(path, encoding="utf-8") as f: + source = f.read() + + # A truncated or error-page download would fail to parse + try: + ast.parse(source) + except SyntaxError as e: + raise SystemExit(f"Exported script is not valid Python: {e}") + + missing = [node for node in expected if node not in source] + if missing: + raise SystemExit( + f"Exported script does not reference {missing}; the workflow may not have " + "loaded before it was exported." + ) + + print(f"{path}: valid Python, references {', '.join(expected)}") + + +def check_image(path): + if not os.path.isfile(path): + raise SystemExit(f"The generated script did not produce {path}") + with open(path, "rb") as f: + header = f.read(len(PNG_MAGIC)) + if header != PNG_MAGIC: + raise SystemExit(f"{path} is not a PNG (got {header!r})") + print(f"{path}: valid PNG, {os.path.getsize(path)} bytes") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--script", required=True, help="The exported Python script") + parser.add_argument("--expect", nargs="*", default=[], help="Node types the script must reference") + parser.add_argument("--image", default=None, help="PNG the script was expected to write") + args = parser.parse_args() + + check_script(args.script, args.expect) + if args.image: + check_image(args.image) + + +if __name__ == "__main__": + main()