Vito
3f085d0724
* ci: run the full matrix once per PR, in the merge queue (#420) Pull requests now run a fast subset: Python 3.12 on ubuntu-latest, the coverage gate, black/flake8, frontend modules, bandit and the Chromium/touch frontend coverage shards plus merge. The merge queue, pushes to main and manual runs keep the full matrix: Linux/Windows/macOS x Python 3.12-3.14, real ComfyUI compatibility and the Firefox/WebKit page workflows. A dropped matrix leg never reports a check, so the new required "all tests" gate requires every test.yml job to succeed outside pull requests and real ComfyUI to be skipped only on pull requests; viewerjs does the same for the browser shards. branch-protection.json drops the eight legs a pull request no longer runs and adds "all tests". * docs(ci): ruleset edit keeps every rule and integration id; find merge-queue runs * ci: move Windows and macOS out of the blocking path (#420) The maintainer deferred Windows and macOS to the end of the redesign. The merge queue now runs Linux on Python 3.12-3.14, real ComfyUI, the frontend coverage shards and the Firefox/WebKit page workflows; pull requests keep the Linux 3.12 subset. The six Windows/macOS legs move to the new portability.yml, which runs on push to main, nightly and on demand, keeps the 27-minute limit and is never a required check. Red runs there are fixed at their cause before the cutover (#334). * docs(ci): required checks live in the ruleset and the classic protection (#420)
296 lines
13 KiB
YAML
296 lines
13 KiB
YAML
name: Tests
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
on:
|
|
merge_group:
|
|
pull_request:
|
|
branches: [main]
|
|
push:
|
|
branches: [main]
|
|
workflow_dispatch:
|
|
|
|
# Superseded PR pushes cancel their older run. Every main push gets its own
|
|
# group (run_id), so no main commit's run is ever cancelled or dropped as pending.
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
|
|
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
|
|
|
env:
|
|
PYTHONFAULTHANDLER: "1"
|
|
|
|
# Every job has a time limit: ceil(1.5 x the slowest of three recent green runs on main) + 3
|
|
# minutes, never under 5. The 3 minutes leave room for the Python runner's watchdog (120 s, see
|
|
# tests/run_suite.py) to print its stack dump before GitHub cancels the job. A limit changes only
|
|
# with new measurements written here (TESTING.md "Time limits and hangs").
|
|
# Measured 2026-10-26 on runs 40257217681 (3be0ada), 40240706568 (fb9134f), 40228167328 (f9835aa):
|
|
# py matrix, slowest py3.10 on windows-latest: 12m43s / 15m54s / 14m53s -> 27 min
|
|
# (ubuntu 5m27s-6m27s, macos 6m14s-8m30s, windows 11m42s-15m54s)
|
|
# 3.10 and 3.11 left the matrix and 3.14 joined it in #356; re-measure after three green main runs.
|
|
# Windows and macOS moved to portability.yml (#420), which keeps this 27-minute limit; this
|
|
# Linux-only matrix keeps it too until it is re-measured.
|
|
# real ComfyUI compatibility: 13m32s / 8m55s / 11m46s -> 24 min
|
|
# coverage gate (85%): 7m26s / 7m22s / 7m54s -> 15 min
|
|
# perf budget 30s / 29s / 27s, black and flake8 39s / 37s / 39s,
|
|
# frontend modules 23s / 19s / 29s -> 5 min (the floor)
|
|
# all tests: a result check without setup, 5 min (the floor)
|
|
#
|
|
# Pull requests run a fast subset (#420): Python 3.12 on ubuntu-latest, coverage, lint and frontend
|
|
# modules. The merge queue (merge_group), pushes to main and manual runs add Python 3.13 and 3.14 on
|
|
# Linux and the real ComfyUI job. "all tests" is the required gate that holds the queue to that set.
|
|
# Windows and macOS run in portability.yml (push to main, nightly, on demand) and never gate a merge.
|
|
jobs:
|
|
test:
|
|
name: py${{ matrix.python-version }} on ${{ matrix.os }}
|
|
runs-on: ${{ matrix.os }}
|
|
timeout-minutes: 27
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
os: [ubuntu-latest]
|
|
python-version: ${{ github.event_name == 'pull_request' && fromJSON('["3.12"]') || fromJSON('["3.12", "3.13", "3.14"]') }}
|
|
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
|
|
- name: Set up Python ${{ matrix.python-version }}
|
|
uses: actions/setup-python@v7
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
cache: pip
|
|
cache-dependency-path: requirements-dev.txt
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install -r requirements-dev.txt
|
|
|
|
- name: Run tests
|
|
run: python -m tests.run_suite -v
|
|
|
|
comfyui:
|
|
name: real ComfyUI compatibility
|
|
# Merge queue, main and manual runs only. A job skipped by its condition reports Success, so this
|
|
# required context never blocks a pull request; "all tests" fails if it is skipped anywhere else.
|
|
if: github.event_name != 'pull_request'
|
|
runs-on: ubuntu-latest
|
|
# Chromium and its system libraries come from the pinned Playwright image (tag = @playwright/test in
|
|
# tools/vendor/package-lock.json; tests/vendor/browser-image.test.mjs checks it). The runner install
|
|
# pulled them from Ubuntu's apt mirror on every run, and three runs (#349, #378, #382) crawled at
|
|
# ~26 kB/s for 11-12 minutes until the job hit its limit after every test had passed.
|
|
container:
|
|
image: mcr.microsoft.com/playwright:v1.64.1-noble
|
|
options: --user 1001
|
|
timeout-minutes: 24
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
with:
|
|
persist-credentials: false
|
|
- name: Read the reviewed runtime pins
|
|
id: runtime
|
|
run: |
|
|
node --input-type=module <<'JS' >> "$GITHUB_OUTPUT"
|
|
import runtime from './tools/vendor/comfyui-runtime.json' with { type: 'json' };
|
|
for (const [key, value] of Object.entries(runtime)) console.log(`${key}=${value}`);
|
|
import crossPack from './tools/vendor/cross-pack-runtime.json' with { type: 'json' };
|
|
console.log(`kikoRepository=${crossPack.repository}`);
|
|
console.log(`kikoRevision=${crossPack.revision}`);
|
|
JS
|
|
- name: Check out pinned ComfyUI
|
|
uses: actions/checkout@v7
|
|
with:
|
|
repository: ${{ steps.runtime.outputs.repository }}
|
|
ref: ${{ steps.runtime.outputs.revision }}
|
|
path: .ci-comfyui
|
|
persist-credentials: false
|
|
- name: Check out pinned KikoTools for compatibility
|
|
uses: actions/checkout@v7
|
|
with:
|
|
repository: ${{ steps.runtime.outputs.kikoRepository }}
|
|
ref: ${{ steps.runtime.outputs.kikoRevision }}
|
|
path: .ci-kikotools
|
|
persist-credentials: false
|
|
# The image's own Python and Node 24: setup-python/setup-node install the runner's toolcache builds,
|
|
# whose entry points name host paths that do not exist in the container. The image has no ensurepip
|
|
# and the job has no root, so a checksum-pinned pip wheel bootstraps a user-space venv. The image
|
|
# Python must be the reviewed host Python, so a pin or image bump that disagrees fails here.
|
|
- name: Python environment on the reviewed host Python
|
|
env:
|
|
PIP_WHEEL: https://files.pythonhosted.org/packages/f3/6e/1736e5b4ae2b778ef2f81c47d797de9f891d4d8acb047a24ca37a60294dd/pip-26.2.1-py3-none-any.whl
|
|
PIP_SHA256: 71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e
|
|
PINNED_PYTHON: ${{ steps.runtime.outputs.pythonVersion }}
|
|
run: |
|
|
image="$(python3 -c 'import sys; print("%d.%d" % sys.version_info[:2])')"
|
|
[ "$image" = "$PINNED_PYTHON" ] || { echo "::error::image Python $image is not the pinned $PINNED_PYTHON"; exit 1; }
|
|
venv="$RUNNER_TEMP/venv" wheel="$RUNNER_TEMP/pip-26.2.1-py3-none-any.whl"
|
|
python3 -m venv --without-pip "$venv"
|
|
curl -fsSL --retry 3 -o "$wheel" "$PIP_WHEEL"
|
|
echo "$PIP_SHA256 $wheel" | sha256sum -c -
|
|
"$venv/bin/python" "$wheel/pip" install --no-index --no-cache-dir "$wheel"
|
|
echo "$venv/bin" >> "$GITHUB_PATH"
|
|
- run: python --version && node --version && node --version | grep -q '^v24\.'
|
|
# pip's download cache (torch alone is ~200 MB); inside the container ~ is the job's HOME (/github/home).
|
|
- name: Cache pip downloads
|
|
uses: actions/cache@v6
|
|
with:
|
|
path: ~/.cache/pip
|
|
key: pip-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('requirements-dev.txt', '.ci-comfyui/requirements.txt', 'tools/vendor/comfyui-runtime.json') }}
|
|
- name: Install CPU runtime and test dependencies
|
|
run: |
|
|
pip install "torch==${{ steps.runtime.outputs.torchVersion }}" "torchvision==${{ steps.runtime.outputs.torchvisionVersion }}" --index-url https://download.pytorch.org/whl/cpu
|
|
pip install -r .ci-comfyui/requirements.txt -r requirements-dev.txt "torch==${{ steps.runtime.outputs.torchVersion }}" "torchvision==${{ steps.runtime.outputs.torchvisionVersion }}"
|
|
# npm's download cache, keyed on the lockfile. node_modules itself is still built fresh by npm ci.
|
|
- name: Cache npm downloads
|
|
uses: actions/cache@v6
|
|
with:
|
|
path: ~/.npm
|
|
key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }}
|
|
- run: npm ci --prefix tools/vendor --ignore-scripts
|
|
- name: Classic and Nodes 2.0 startup and compatibility
|
|
env:
|
|
PM_COMFY_ROOT: ${{ github.workspace }}/.ci-comfyui
|
|
PM_TEST_PYTHON: python
|
|
run: npm run --prefix tools/vendor test:comfyui
|
|
- name: Real cached and headless execution history
|
|
env:
|
|
PM_COMFY_ROOT: ${{ github.workspace }}/.ci-comfyui
|
|
PM_TEST_PYTHON: python
|
|
run: npm run --prefix tools/vendor test:executions
|
|
- name: Independent and combined pack compatibility
|
|
env:
|
|
PM_COMFY_ROOT: ${{ github.workspace }}/.ci-comfyui
|
|
PM_KIKO_ROOT: ${{ github.workspace }}/.ci-kikotools
|
|
PM_TEST_PYTHON: python
|
|
run: npm run --prefix tools/vendor test:compat
|
|
- name: Upload real-host reports and startup logs
|
|
if: always()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: real-comfyui-reports
|
|
path: tools/vendor/test-results/
|
|
if-no-files-found: ignore
|
|
retention-days: 14
|
|
|
|
coverage:
|
|
name: coverage gate (85%)
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
|
|
- uses: actions/setup-python@v7
|
|
with:
|
|
python-version: "3.12"
|
|
cache: pip
|
|
cache-dependency-path: requirements-dev.txt
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install -r requirements-dev.txt
|
|
|
|
- name: Measure coverage
|
|
run: |
|
|
python -m coverage run -m tests.run_suite
|
|
python -m coverage combine
|
|
python -m coverage report --fail-under=85
|
|
|
|
perf-budget:
|
|
# The reference wall-clock measurement (TESTING.md "Performance budgets").
|
|
# Shared-runner timing varies, so it never gates a pull request: it runs on
|
|
# each push to main and on demand. A red run on main means fix the code.
|
|
name: perf budget
|
|
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 5
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
|
|
- uses: actions/setup-python@v7
|
|
with:
|
|
python-version: "3.13"
|
|
cache: pip
|
|
cache-dependency-path: requirements-dev.txt
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install -r requirements-dev.txt
|
|
|
|
- name: Workspace filter budget
|
|
env:
|
|
PROMPTMANAGER_PERF_BUDGET: "1"
|
|
run: python -m unittest -v tests.test_api_workspace_images.TestWorkspaceImages.test_filter_budget
|
|
|
|
lint:
|
|
name: black and flake8
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 5
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- uses: actions/setup-python@v7
|
|
with:
|
|
python-version: "3.12"
|
|
cache: pip
|
|
cache-dependency-path: requirements-dev.txt
|
|
# The exact black and flake8 pins from requirements-dev.txt, so a new release cannot turn CI red.
|
|
- run: pip install $(grep -E '^(black|flake8)==' requirements-dev.txt)
|
|
- run: black --check .
|
|
- run: flake8 .
|
|
|
|
js-test:
|
|
name: frontend modules
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 5
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
|
|
- uses: actions/setup-node@v7
|
|
with:
|
|
node-version: "24"
|
|
cache: npm
|
|
cache-dependency-path: tools/vendor/package-lock.json
|
|
|
|
- name: Check shipped extension and page modules parse
|
|
run: |
|
|
shopt -s globstar nullglob
|
|
for f in web/comfy/**/*.js web/comfy/**/*.mjs web/js/**/*.js web/js/**/*.mjs; do
|
|
node --input-type=module --check < "$f" || exit 1
|
|
done
|
|
|
|
- name: Install pinned shared test contracts
|
|
run: npm ci --prefix tools/vendor --ignore-scripts
|
|
|
|
- name: Run JS tests
|
|
run: node --test tests/js/*.test.js
|
|
|
|
# The required context "all tests". A dropped matrix leg never reports a check at all, so the eight
|
|
# legs that pull requests skip cannot be required by name; this gate requires them instead. Every
|
|
# job must succeed; real ComfyUI must be skipped on a pull request and succeed on every other event.
|
|
all-tests:
|
|
name: all tests
|
|
needs: [test, comfyui, coverage, lint, js-test]
|
|
if: always()
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 5
|
|
steps:
|
|
- name: Require every test job for this event
|
|
env:
|
|
EVENT_NAME: ${{ github.event_name }}
|
|
TEST_RESULT: ${{ needs.test.result }}
|
|
COMFYUI_RESULT: ${{ needs.comfyui.result }}
|
|
COVERAGE_RESULT: ${{ needs.coverage.result }}
|
|
LINT_RESULT: ${{ needs.lint.result }}
|
|
JS_RESULT: ${{ needs.js-test.result }}
|
|
run: |
|
|
queue_only=success
|
|
if [ "$EVENT_NAME" = pull_request ]; then queue_only=skipped; fi
|
|
echo "event: $EVENT_NAME (real ComfyUI must be $queue_only)"
|
|
echo "test matrix: $TEST_RESULT"
|
|
echo "comfyui: $COMFYUI_RESULT"
|
|
echo "coverage: $COVERAGE_RESULT"
|
|
echo "lint: $LINT_RESULT"
|
|
echo "js-test: $JS_RESULT"
|
|
test "$TEST_RESULT" = success && test "$COVERAGE_RESULT" = success && test "$LINT_RESULT" = success && test "$JS_RESULT" = success && test "$COMFYUI_RESULT" = "$queue_only"
|