Test workflow
This commit is contained in:
atmaranto
@@ -0,0 +1,130 @@
|
||||
name: Daily end-to-end test
|
||||
|
||||
# Exercises the whole path a user takes: run ComfyUI, open a workflow through
|
||||
# File > Open, hit "Save as Script", and check the exported script actually runs.
|
||||
on:
|
||||
schedule:
|
||||
# 00:00 UTC every day
|
||||
- cron: '0 0 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: daily-e2e
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
export-and-run:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
|
||||
container:
|
||||
image: python:3.12-slim
|
||||
# Chromium needs more shared memory than a container gets by default
|
||||
options: --shm-size=2g
|
||||
|
||||
env:
|
||||
COMFYUI_URL: http://127.0.0.1:8188
|
||||
# Where the exported script and its output land
|
||||
EXPORT_DIR: ${{ github.workspace }}/exported
|
||||
|
||||
steps:
|
||||
- name: Install system packages
|
||||
run: |
|
||||
apt-get update
|
||||
apt-get install -y --no-install-recommends git curl ca-certificates
|
||||
|
||||
- name: Check out the extension
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
path: extension
|
||||
|
||||
- name: Clone ComfyUI
|
||||
# The directory must be named ComfyUI: the generated script locates it by name
|
||||
run: git clone --depth 1 https://github.com/comfy-org/ComfyUI.git ComfyUI
|
||||
|
||||
- name: Install PyTorch (CPU build)
|
||||
# Installed before ComfyUI's requirements so pip doesn't pull the CUDA wheels,
|
||||
# which are several GB and useless on a CPU runner
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
|
||||
|
||||
- name: Install ComfyUI requirements
|
||||
working-directory: ComfyUI
|
||||
run: pip install -r requirements.txt
|
||||
|
||||
- name: Install the extension
|
||||
run: |
|
||||
cp -r extension ComfyUI/custom_nodes/ComfyUI-SaveAsScript
|
||||
pip install -r ComfyUI/custom_nodes/ComfyUI-SaveAsScript/requirements.txt
|
||||
|
||||
- name: Install Playwright
|
||||
run: |
|
||||
pip install playwright
|
||||
playwright install --with-deps chromium
|
||||
|
||||
- name: Start ComfyUI
|
||||
working-directory: ComfyUI
|
||||
run: |
|
||||
python main.py --cpu --listen 127.0.0.1 --port 8188 --disable-auto-launch \
|
||||
> comfyui.log 2>&1 &
|
||||
echo $! > comfyui.pid
|
||||
|
||||
for _ in $(seq 1 60); do
|
||||
if curl -sf "$COMFYUI_URL/system_stats" > /dev/null; then
|
||||
echo "ComfyUI is up"
|
||||
exit 0
|
||||
fi
|
||||
if ! kill -0 "$(cat comfyui.pid)" 2>/dev/null; then
|
||||
echo "::error::ComfyUI exited during startup"
|
||||
cat comfyui.log
|
||||
exit 1
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
|
||||
echo "::error::ComfyUI did not become ready within 5 minutes"
|
||||
cat comfyui.log
|
||||
exit 1
|
||||
|
||||
- name: Export the workflow through the browser
|
||||
run: |
|
||||
python extension/test/e2e_save_as_script.py \
|
||||
--url "$COMFYUI_URL" \
|
||||
--workflow extension/test/test_workflow.json \
|
||||
--output "$EXPORT_DIR/test_workflow.py" \
|
||||
--expect-node EmptyImage \
|
||||
--artifacts "$EXPORT_DIR/artifacts"
|
||||
|
||||
- name: Stop ComfyUI
|
||||
working-directory: ComfyUI
|
||||
run: kill "$(cat comfyui.pid)" 2>/dev/null || true
|
||||
|
||||
- name: Run the exported script
|
||||
# From inside ComfyUI, as a user would run it
|
||||
working-directory: ComfyUI
|
||||
run: |
|
||||
python "$EXPORT_DIR/test_workflow.py" \
|
||||
--queue-size 1 \
|
||||
--output "$EXPORT_DIR/output.png"
|
||||
|
||||
- name: Check the export and its output
|
||||
run: |
|
||||
python extension/test/verify_generated_script.py \
|
||||
--script "$EXPORT_DIR/test_workflow.py" \
|
||||
--expect EmptyImage RandomCropImages SaveImage \
|
||||
--image "$EXPORT_DIR/output.png"
|
||||
|
||||
- name: Upload artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: e2e-artifacts
|
||||
path: |
|
||||
${{ github.workspace }}/exported
|
||||
${{ github.workspace }}/ComfyUI/comfyui.log
|
||||
if-no-files-found: ignore
|
||||
retention-days: 14
|
||||
@@ -0,0 +1,201 @@
|
||||
"""Drives a running ComfyUI through the browser to export a workflow as a script.
|
||||
|
||||
Opens a workflow, clicks the "Save as Script" button, and writes the downloaded Python
|
||||
file to --output. Used by the daily CI run, but it works against any local ComfyUI:
|
||||
|
||||
python e2e_save_as_script.py --workflow test_workflow.json --output out.py
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutError
|
||||
|
||||
# The frontend takes a while to boot on a cold CPU-only container
|
||||
READY_TIMEOUT = 180_000
|
||||
ACTION_TIMEOUT = 60_000
|
||||
# Generating the script imports every node module, so give it room
|
||||
DOWNLOAD_TIMEOUT = 300_000
|
||||
|
||||
|
||||
def dismiss_startup_dialogs(page):
|
||||
"""Closes the welcome/template dialogs some frontend versions open on first load."""
|
||||
for _ in range(3):
|
||||
closed = False
|
||||
for selector in ('[data-pc-name="dialog"] [aria-label="Close"]',
|
||||
'[data-pc-name="dialog"] .p-dialog-close-button',
|
||||
'.p-dialog-header-close'):
|
||||
button = page.locator(selector)
|
||||
if button.count() and button.first.is_visible():
|
||||
button.first.click()
|
||||
closed = True
|
||||
page.wait_for_timeout(500)
|
||||
if not closed:
|
||||
break
|
||||
page.keyboard.press("Escape")
|
||||
page.wait_for_timeout(500)
|
||||
|
||||
|
||||
def first_visible(locator):
|
||||
"""Returns the first visible match, or None."""
|
||||
for i in range(locator.count()):
|
||||
if locator.nth(i).is_visible():
|
||||
return locator.nth(i)
|
||||
return None
|
||||
|
||||
|
||||
def click_menu_item(page, label):
|
||||
"""Clicks a menu entry by its visible label."""
|
||||
exact = re.compile(rf"^\s*{re.escape(label)}\s*$", re.IGNORECASE)
|
||||
for candidate in (page.get_by_role("menuitem", name=exact),
|
||||
page.locator(f'xpath=//*[normalize-space(text())="{label}"]')):
|
||||
item = first_visible(candidate)
|
||||
if item:
|
||||
item.click()
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def open_workflow(page, workflow):
|
||||
"""Loads `workflow` the way a user opens a workflow file.
|
||||
|
||||
Current ComfyUI has no File > Open menu; the open-workflow command
|
||||
(Comfy.OpenWorkflow) is bound to Ctrl+O and clicks a hidden file input. Older
|
||||
frontends exposed the same command under File > Open, so that is tried second.
|
||||
"""
|
||||
try:
|
||||
with page.expect_file_chooser(timeout=20_000) as chooser_info:
|
||||
page.keyboard.press("Control+o")
|
||||
chooser_info.value.set_files(workflow)
|
||||
return
|
||||
except PlaywrightTimeoutError:
|
||||
print("Ctrl+O did not open a file chooser; trying the File > Open menu",
|
||||
file=sys.stderr)
|
||||
|
||||
with page.expect_file_chooser(timeout=ACTION_TIMEOUT) as chooser_info:
|
||||
if not click_menu_item(page, "File"):
|
||||
raise RuntimeError(
|
||||
"Could not open a workflow: Ctrl+O opened no file chooser and there is "
|
||||
"no 'File' menu. ComfyUI's open-workflow entry point may have changed."
|
||||
)
|
||||
page.wait_for_timeout(500)
|
||||
if not click_menu_item(page, "Open"):
|
||||
raise RuntimeError("Found a 'File' menu but no 'Open' item inside it.")
|
||||
chooser_info.value.set_files(workflow)
|
||||
|
||||
|
||||
def wait_for_workflow(page, expect_node):
|
||||
"""Waits until the loaded graph actually contains `expect_node`."""
|
||||
if not expect_node:
|
||||
page.wait_for_timeout(2000)
|
||||
return
|
||||
try:
|
||||
page.wait_for_function(
|
||||
"""(type) => (window.app?.graph?._nodes ?? []).some(
|
||||
(n) => n?.type === type || n?.comfyClass === type)""",
|
||||
arg=expect_node,
|
||||
timeout=ACTION_TIMEOUT,
|
||||
)
|
||||
except PlaywrightTimeoutError:
|
||||
# Not every frontend exposes window.app; the exported script is checked
|
||||
# separately, so this is a warning rather than a failure.
|
||||
print(f"warning: could not confirm a {expect_node} node via window.app",
|
||||
file=sys.stderr)
|
||||
page.wait_for_timeout(1000)
|
||||
|
||||
|
||||
def find_save_as_script_button(page):
|
||||
"""Returns the visible "Save as Script" button.
|
||||
|
||||
Matching is on button text, not accessible name: the menubar button carries an
|
||||
aria-label ("Save the current workflow as a Python script") that would otherwise
|
||||
hide it from a name-based lookup. The extension also adds a button to the legacy
|
||||
.comfy-menu sidebar, which is normally hidden.
|
||||
"""
|
||||
label = re.compile("Save as Script", re.IGNORECASE)
|
||||
for candidate in (page.locator("button").filter(has_text=label),
|
||||
page.get_by_role("button", name=label),
|
||||
page.locator('button[aria-label*="Python script" i]')):
|
||||
button = first_visible(candidate)
|
||||
if button:
|
||||
return button
|
||||
|
||||
raise RuntimeError(
|
||||
"The 'Save as Script' button is not present. The extension's setup() may have "
|
||||
"failed to run - check the browser console output above."
|
||||
)
|
||||
|
||||
|
||||
def export_script(page, name):
|
||||
"""Clicks "Save as Script", answers the filename prompt, returns the download."""
|
||||
button = find_save_as_script_button(page)
|
||||
|
||||
# The extension asks for the filename with window.prompt(); without a handler
|
||||
# Playwright auto-dismisses it and no download is ever produced.
|
||||
page.once("dialog", lambda dialog: dialog.accept(name))
|
||||
|
||||
with page.expect_download(timeout=DOWNLOAD_TIMEOUT) as download_info:
|
||||
button.click()
|
||||
return download_info.value
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--url", default="http://127.0.0.1:8188", help="Where ComfyUI is serving")
|
||||
parser.add_argument("--workflow", required=True, help="Workflow JSON to open")
|
||||
parser.add_argument("--output", required=True, help="Where to write the exported script")
|
||||
parser.add_argument("--expect-node", default=None,
|
||||
help="Node type that must appear in the graph once loaded")
|
||||
parser.add_argument("--artifacts", default=None, help="Directory for failure screenshots")
|
||||
parser.add_argument("--headed", action="store_true", help="Show the browser (for debugging)")
|
||||
args = parser.parse_args()
|
||||
|
||||
workflow = os.path.abspath(args.workflow)
|
||||
output = os.path.abspath(args.output)
|
||||
os.makedirs(os.path.dirname(output) or ".", exist_ok=True)
|
||||
if args.artifacts:
|
||||
os.makedirs(args.artifacts, exist_ok=True)
|
||||
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(
|
||||
headless=not args.headed,
|
||||
# Chromium crashes with the small /dev/shm containers get by default
|
||||
args=["--disable-dev-shm-usage", "--no-sandbox"],
|
||||
)
|
||||
context = browser.new_context(accept_downloads=True)
|
||||
page = context.new_page()
|
||||
page.set_default_timeout(ACTION_TIMEOUT)
|
||||
page.on("console", lambda msg: print(f"[browser:{msg.type}] {msg.text}"))
|
||||
page.on("pageerror", lambda err: print(f"[browser:pageerror] {err}", file=sys.stderr))
|
||||
|
||||
try:
|
||||
page.goto(args.url, wait_until="domcontentloaded", timeout=READY_TIMEOUT)
|
||||
page.wait_for_selector("#graph-canvas", timeout=READY_TIMEOUT)
|
||||
page.wait_for_timeout(3000)
|
||||
dismiss_startup_dialogs(page)
|
||||
|
||||
open_workflow(page, workflow)
|
||||
wait_for_workflow(page, args.expect_node)
|
||||
|
||||
download = export_script(page, os.path.splitext(os.path.basename(output))[0])
|
||||
download.save_as(output)
|
||||
print(f"Exported script to {output}")
|
||||
except Exception:
|
||||
if args.artifacts:
|
||||
page.screenshot(path=os.path.join(args.artifacts, "failure.png"), full_page=True)
|
||||
with open(os.path.join(args.artifacts, "failure.html"), "w", encoding="utf-8") as f:
|
||||
f.write(page.content())
|
||||
print(f"Wrote failure artifacts to {args.artifacts}", file=sys.stderr)
|
||||
raise
|
||||
finally:
|
||||
context.close()
|
||||
browser.close()
|
||||
|
||||
if not os.path.getsize(output):
|
||||
raise SystemExit("The downloaded script is empty")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"2": {
|
||||
"inputs": {
|
||||
"width": 512,
|
||||
"height": 512,
|
||||
"batch_size": 1,
|
||||
"color": 0
|
||||
},
|
||||
"class_type": "EmptyImage",
|
||||
"_meta": {
|
||||
"title": "Empty Image"
|
||||
}
|
||||
},
|
||||
"3": {
|
||||
"inputs": {
|
||||
"width": 512,
|
||||
"height": 512,
|
||||
"seed": 0,
|
||||
"images": [
|
||||
"2",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "RandomCropImages",
|
||||
"_meta": {
|
||||
"title": "Crop Image (Random)"
|
||||
}
|
||||
},
|
||||
"5": {
|
||||
"inputs": {
|
||||
"filename_prefix": "ComfyUI",
|
||||
"images": [
|
||||
"3",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "SaveImage",
|
||||
"_meta": {
|
||||
"title": "Save Image"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Checks that an exported script looks right and produced a real image.
|
||||
|
||||
Run after e2e_save_as_script.py has downloaded a script and that script has been
|
||||
executed, to confirm the export reflects the workflow rather than just being
|
||||
syntactically valid Python.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import os
|
||||
import sys
|
||||
|
||||
PNG_MAGIC = b"\x89PNG\r\n\x1a\n"
|
||||
|
||||
|
||||
def check_script(path, expected):
|
||||
with open(path, encoding="utf-8") as f:
|
||||
source = f.read()
|
||||
|
||||
# A truncated or error-page download would fail to parse
|
||||
try:
|
||||
ast.parse(source)
|
||||
except SyntaxError as e:
|
||||
raise SystemExit(f"Exported script is not valid Python: {e}")
|
||||
|
||||
missing = [node for node in expected if node not in source]
|
||||
if missing:
|
||||
raise SystemExit(
|
||||
f"Exported script does not reference {missing}; the workflow may not have "
|
||||
"loaded before it was exported."
|
||||
)
|
||||
|
||||
print(f"{path}: valid Python, references {', '.join(expected)}")
|
||||
|
||||
|
||||
def check_image(path):
|
||||
if not os.path.isfile(path):
|
||||
raise SystemExit(f"The generated script did not produce {path}")
|
||||
with open(path, "rb") as f:
|
||||
header = f.read(len(PNG_MAGIC))
|
||||
if header != PNG_MAGIC:
|
||||
raise SystemExit(f"{path} is not a PNG (got {header!r})")
|
||||
print(f"{path}: valid PNG, {os.path.getsize(path)} bytes")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--script", required=True, help="The exported Python script")
|
||||
parser.add_argument("--expect", nargs="*", default=[], help="Node types the script must reference")
|
||||
parser.add_argument("--image", default=None, help="PNG the script was expected to write")
|
||||
args = parser.parse_args()
|
||||
|
||||
check_script(args.script, args.expect)
|
||||
if args.image:
|
||||
check_image(args.image)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user