wildminder-ComfyUI-KEEP/tests/test_ci_workflow.py

"""P1.5 - the CI workflow.

A workflow file is not a test, but it *is* code: a workflow without install
steps cannot run at all, and a workflow that runs a different command than the
local gate tests something else than the developer tested.

The checks below therefore assert that

* the workflow is valid YAML and has a ``lint`` and a ``test`` job,
* the lint job installs a **pinned** ruff and runs ``ruff check .``,
* the test job carries the real install steps (CPU torch, numpy, opencv,
  einops, scipy, tomli on 3.10, **PyYAML**) and the matrix 3.10 / 3.12 / 3.13,
* the test job runs exactly the local gate command and leaves ``COMFYUI_ROOT``
  empty (so the environment dependent tests skip themselves),
* ``publish.yml`` is still the untouched registry workflow.

**The module never skips.**  The plan (P1.5) says: "workflow YAML parses via
embedded python's ``yaml`` (available: PyYAML in env? - if absent, parse with
a minimal regex check instead; **assert either way**)".  A module level
``pytest.importorskip("yaml")`` would skip *all* checks on a CI image without
PyYAML - exactly the image where the workflow is broken.  So the file parses
the workflow with PyYAML when it is importable and with a minimal indentation
parser otherwise, and the assertions below always run.  The fallback is
cross-checked against the PyYAML parse whenever PyYAML is available, so it
cannot rot.
"""

from __future__ import annotations

import ast
import re
from pathlib import Path

import pytest

try:  # PyYAML is optional: the checks below must run without it.
    import yaml
except ImportError:  # pragma: no cover - exercised by the CI-env simulation
    yaml = None  # type: ignore[assignment]

REPO_ROOT = Path(__file__).resolve().parent.parent
WORKFLOW = REPO_ROOT / ".github" / "workflows" / "tests.yml"
PUBLISH_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "publish.yml"

#: The local gate of the plan (section 4).
LOCAL_GATE = 'pytest -q -m "unit or mock_integration or headless"'


# ---------------------------------------------------------------------------
# the fallback parser (no PyYAML)
# ---------------------------------------------------------------------------
_JOB_RE = re.compile(r"^  ([A-Za-z0-9_.-]+):\s*$")
_KEY_RE = re.compile(r"^\s*([A-Za-z0-9_.-]+):\s*(.*)$")


def _indent(line: str) -> int:
    return len(line) - len(line.lstrip())


def _block_after(lines: list[str], index: int) -> list[str]:
    """The indented block that follows ``lines[index]``."""
    parent = _indent(lines[index])
    block: list[str] = []
    for line in lines[index + 1 :]:
        if line.strip() and _indent(line) <= parent:
            break
        block.append(line)
    return block


def _dedent(lines: list[str]) -> str:
    """The content of a YAML block scalar, without its indentation."""
    indents = [_indent(line) for line in lines if line.strip()]
    if not indents:
        return ""
    margin = min(indents)
    return "\n".join(line[margin:] if line.strip() else "" for line in lines)


def _scalar(value: str) -> str:
    return value.strip().strip("\"'")


def _inline_list(value: str) -> list[str]:
    """``[a, b]`` -> ``["a", "b"]`` (the quotes are optional)."""
    value = value.strip()
    if not (value.startswith("[") and value.endswith("]")):
        return [_scalar(value)]
    try:
        parsed = ast.literal_eval(value)
    except (ValueError, SyntaxError):
        parsed = None
    if isinstance(parsed, list):
        return [str(item) for item in parsed]
    return [_scalar(item) for item in value[1:-1].split(",") if item.strip()]


def _parse_mapping(block: list[str]) -> dict[str, object]:
    """A flat ``key: value`` mapping (values may be inline lists)."""
    mapping: dict[str, object] = {}
    for line in block:
        if not line.strip() or line.lstrip().startswith("#"):
            continue
        match = _KEY_RE.match(line)
        if not match:
            continue
        key, raw = match.group(1), match.group(2).strip()
        if raw.startswith("[") and raw.endswith("]"):
            mapping[key] = _inline_list(raw)
        else:
            mapping[key] = _scalar(raw)
    return mapping


def _parse_steps(body: list[str]) -> list[dict[str, str]]:
    """The ``run:`` text of every step of a job."""
    steps: list[dict[str, str]] = []
    index = 0
    while index < len(body):
        line = body[index]
        if not line.lstrip().startswith("- "):
            index += 1
            continue
        step_indent = _indent(line)
        attribute_lines: list[str] = []
        index += 1
        while index < len(body):
            candidate = body[index]
            if candidate.strip() and _indent(candidate) <= step_indent:
                break
            attribute_lines.append(candidate)
            index += 1
        for position, attribute in enumerate(attribute_lines):
            match = re.match(r"^(\s*)run:\s*(.*)$", attribute)
            if not match:
                continue
            inline = match.group(2).strip()
            if inline in {"|", ">", "|-", ">-"}:
                content = _dedent(_block_after(attribute_lines, position))
            else:
                content = inline
            steps.append({"run": content.rstrip()})
            break
    return steps


def _parse_job(body: list[str]) -> dict[str, object]:
    """One job: its ``run:`` steps, its ``env`` and its matrix."""
    env: dict[str, str] = {}
    matrix: dict[str, list[str]] = {}
    for index, line in enumerate(body):
        stripped = line.strip()
        if stripped == "env:":
            for key, value in _parse_mapping(_block_after(body, index)).items():
                env[key] = str(value)
        elif stripped == "matrix:":
            for key, value in _parse_mapping(_block_after(body, index)).items():
                matrix[key] = value if isinstance(value, list) else [str(value)]
    return {"steps": _parse_steps(body), "env": env, "matrix": matrix}


def _fallback_parse(text: str) -> dict[str, object]:
    """A minimal parser for the workflow file (no PyYAML needed).

    It understands the subset the workflow uses: the top-level ``on:``
    triggers, the ``jobs:`` mapping, the per-job ``steps`` with ``run:`` block
    scalars, the per-job ``env:`` mapping and ``strategy.matrix``.
    """
    lines = text.splitlines()
    jobs: dict[str, object] = {}
    triggers: list[str] = []

    in_jobs = False
    job_name: str | None = None
    job_body: list[str] = []

    def close_job() -> None:
        nonlocal job_name, job_body
        if job_name is not None:
            jobs[job_name] = _parse_job(job_body)
        job_name, job_body = None, []

    for index, line in enumerate(lines):
        stripped = line.strip()
        if not in_jobs:
            if stripped == "jobs:":
                in_jobs = True
            elif stripped == "on:":
                for trigger_line in _block_after(lines, index):
                    match = re.match(r"^  ([A-Za-z0-9_.-]+):", trigger_line)
                    if match:
                        triggers.append(match.group(1))
            continue
        if stripped and not line.startswith(" "):
            break  # the next top-level key ends the jobs section
        match = _JOB_RE.match(line)
        if match:
            close_job()
            job_name = match.group(1)
        elif job_name is not None:
            job_body.append(line)
    close_job()

    return {"jobs": jobs, "triggers": sorted(triggers)}


def _yaml_parse(text: str) -> dict[str, object]:
    """The same normalized view, parsed with PyYAML."""
    loaded = yaml.safe_load(text)
    assert isinstance(loaded, dict), "the workflow must parse into a mapping"

    jobs: dict[str, object] = {}
    for name, job in (loaded.get("jobs") or {}).items():
        steps = []
        for step in job.get("steps") or []:
            if isinstance(step, dict) and "run" in step:
                steps.append({"run": str(step["run"]).rstrip()})
        env = {str(key): str(value) for key, value in (job.get("env") or {}).items()}
        matrix: dict[str, list[str]] = {}
        raw_matrix = (job.get("strategy") or {}).get("matrix") or {}
        for key, value in raw_matrix.items():
            matrix[str(key)] = (
                [str(item) for item in value] if isinstance(value, list) else [str(value)]
            )
        jobs[str(name)] = {"steps": steps, "env": env, "matrix": matrix}

    # PyYAML parses the YAML 1.1 boolean "on" as the key True.
    triggers = sorted(str(key) for key in (loaded.get("on") or loaded.get(True) or {}))
    return {"jobs": jobs, "triggers": triggers}


@pytest.fixture(scope="module")
def workflow():
    """The workflow as a normalized mapping (PyYAML, or the fallback parser)."""
    assert WORKFLOW.is_file(), f"the CI workflow is missing: {WORKFLOW}"
    text = WORKFLOW.read_text(encoding="utf-8")
    if yaml is not None:
        return _yaml_parse(text)
    return _fallback_parse(text)


def _steps(job: dict) -> list[dict]:
    return list(job.get("steps", []))


def _run_text(job: dict) -> str:
    return "\n".join(str(step.get("run", "")) for step in _steps(job))


@pytest.mark.unit
def test_workflow_parses_and_has_the_two_jobs(workflow):
    """The file is valid YAML with a lint job and a test job."""
    jobs = workflow["jobs"]
    assert "lint" in jobs, "the workflow needs a lint job"
    assert "test" in jobs, "the workflow needs a test job"
    assert workflow["triggers"], "the workflow needs a trigger"


@pytest.mark.unit
def test_the_fallback_parser_agrees_with_the_yaml_parse():
    """The PyYAML-free fallback must see the same workflow as PyYAML.

    This keeps the fallback honest on every runner that *has* PyYAML (the CI
    job installs it); without the cross-check the fallback would only ever run
    on a machine that is already broken.
    """
    if yaml is None:
        pytest.skip("PyYAML is not installed - the fallback parser is the only one")
    text = WORKFLOW.read_text(encoding="utf-8")
    assert _fallback_parse(text) == _yaml_parse(text)


@pytest.mark.unit
def test_lint_job_installs_a_pinned_ruff_and_checks_the_repository(workflow):
    """The lint job installs ruff (pinned) and runs ``ruff check .``."""
    lint = workflow["jobs"]["lint"]
    text = _run_text(lint)

    assert "ruff==0.8.*" in text, "ruff must be pinned to the 0.8 series"
    assert "pip install" in text
    assert "ruff check ." in text


@pytest.mark.unit
def test_test_job_has_the_python_matrix(workflow):
    """The test job runs on python 3.10, 3.12 and 3.13."""
    matrix = workflow["jobs"]["test"].get("matrix", {})
    versions = [str(version) for version in matrix.get("python-version", [])]
    assert versions == ["3.10", "3.12", "3.13"], (
        f"unexpected python matrix: {versions}"
    )


@pytest.mark.unit
def test_test_job_installs_the_real_dependencies(workflow):
    """The test job installs what the package and the suite actually need."""
    text = _run_text(workflow["jobs"]["test"])

    assert "--index-url https://download.pytorch.org/whl/cpu" in text, (
        "torch must be installed from the CPU index"
    )
    assert "torchvision" in text, "the package imports torchvision"
    assert "opencv-python-headless" in text
    assert "einops" in text
    assert "scipy" in text
    assert "Pillow" in text
    assert "tqdm" in text
    assert "pytest" in text
    # The vendored tree imports yaml eagerly
    # (modules/deps/basicsr/utils/options.py:5), so the unit tier dies with
    # "No module named 'yaml'" when PyYAML is missing.
    assert "pyyaml" in text.lower(), (
        "PyYAML must be installed: the vendored tree imports it eagerly"
    )
    assert "numpy<2.3" in text, "numpy must be pinned for the 3.10 job"
    assert "tomli" in text, "tomli backs tomllib on python 3.10"
    assert '"3.10"' in text, "the 3.10-only installs must be conditional"


#: The distributions of ``requirements.txt`` that the CI may satisfy with a
#: different build - the headless OpenCV wheel has no GUI dependencies.
_CI_INSTALL_ALIASES = {
    "opencv-python": ("opencv-python", "opencv-python-headless"),
}


def _runtime_requirements() -> list[str]:
    """The distributions of ``requirements.txt`` (the true runtime set)."""
    text = (REPO_ROOT / "requirements.txt").read_text(encoding="utf-8")
    return [
        line.strip()
        for line in text.splitlines()
        if line.strip() and not line.strip().startswith("#")
    ]


def _installed_distributions(run_text: str) -> set[str]:
    """Every distribution the job installs (the tokens of its pip calls)."""
    installed: set[str] = set()
    for line in run_text.splitlines():
        parts = line.split()
        if "pip" not in parts or "install" not in parts:
            continue
        for token in parts[parts.index("install") + 1 :]:
            token = token.strip().strip('"').strip("'").lower()
            if not token or token.startswith("-"):
                continue
            for separator in ("<", ">", "=", "["):
                token = token.split(separator)[0]
            installed.add(token)
    return installed


@pytest.mark.unit
def test_the_ci_installs_every_runtime_requirement(workflow):
    """The CI job installs every distribution of ``requirements.txt``.

    ``requirements.txt`` is the true runtime set of the package (P2.6).  A
    distribution that is missing from the install list kills the unit tier
    on a clean runner with a ``ModuleNotFoundError`` - ``requests`` is
    imported **eagerly** by ``modules/deps/basicsr/utils/download_util.py``
    (which the package imports in ``modules/utils.py``).
    """
    installed = _installed_distributions(_run_text(workflow["jobs"]["test"]))
    missing = []
    for requirement in _runtime_requirements():
        name = requirement.lower().replace("_", "-")
        aliases = _CI_INSTALL_ALIASES.get(name, (name,))
        if not any(alias in installed for alias in aliases):
            missing.append(requirement)
    assert not missing, (
        f"the CI job does not install {missing} - requirements.txt is the "
        f"true runtime set (P2.6)"
    )


@pytest.mark.unit
def test_test_job_runs_the_local_gate_command(workflow):
    """The CI command is exactly the local gate command."""
    run_commands = [
        str(step.get("run", "")).strip()
        for step in _steps(workflow["jobs"]["test"])
        if "pytest" in str(step.get("run", "")) and "pip install" not in str(step.get("run", ""))
    ]
    assert run_commands, "the test job must run pytest"

    command = run_commands[0]
    assert command.endswith(LOCAL_GATE), (
        f"the CI command {command!r} does not end with the local gate {LOCAL_GATE!r}"
    )
    assert command.startswith("python -m pytest"), (
        "the CI must run pytest through the interpreter of the job"
    )


@pytest.mark.unit
def test_test_job_leaves_comfyui_root_empty(workflow):
    """``COMFYUI_ROOT`` is empty so the environment dependent tests skip."""
    env = workflow["jobs"]["test"].get("env", {})
    assert "COMFYUI_ROOT" in env, "the test job must define COMFYUI_ROOT"
    assert str(env["COMFYUI_ROOT"]) == "", (
        "COMFYUI_ROOT must stay empty: the gate runs without environment variables"
    )


@pytest.mark.unit
def test_publish_workflow_is_untouched():
    """``publish.yml`` is still the registry publish workflow."""
    assert PUBLISH_WORKFLOW.is_file()
    text = PUBLISH_WORKFLOW.read_text(encoding="utf-8")
    assert "Comfy-Org/publish-node-action" in text
    assert "tests.yml" not in text, "publish.yml must not depend on the test workflow"