WildAi
ec9640f34b
The audit artifacts and the plan moved to the local .dev/docs tree, so the references in the code, the tools and the tests follow them there. The license file names the tool that produces the record instead of a path.
304 lines
9.6 KiB
Python
304 lines
9.6 KiB
Python
"""P0.3 - compatibility scanner tests.
|
|
|
|
The scanner is unit-tested against fixtures (known-bad snippets are flagged,
|
|
known-good ones are not), including a fixture for the vendored ``pip.main``
|
|
shim. The tests then check the scanner against the **real** vendored tree and
|
|
against the audit document that lists the fix list.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
|
SCANNER = REPO_ROOT / "tools" / "scan_compat.py"
|
|
AUDIT_DOC = REPO_ROOT / ".dev" / "docs" / "audit" / "2026-10-22-compat-scan.md"
|
|
DEPS_DIR = REPO_ROOT / "modules" / "deps"
|
|
|
|
#: The audit artifacts are development documentation: they are not shipped with
|
|
#: the pack and may be absent from a working tree.
|
|
pytestmark = pytest.mark.skipif(
|
|
not AUDIT_DOC.exists(),
|
|
reason=f"the audit artifact {AUDIT_DOC.name} is not in the working tree",
|
|
)
|
|
|
|
KNOWN_BAD = '''\
|
|
import pdb
|
|
import imp
|
|
import pip
|
|
from re import T
|
|
from distutils.version import LooseVersion
|
|
from collections import Mapping
|
|
import numpy as np
|
|
|
|
x = np.float(1.0)
|
|
y = np.int(2)
|
|
z = np.bool(True)
|
|
o = np.object()
|
|
s = np.str("a")
|
|
n = np.NaN
|
|
amp = torch.cuda.amp
|
|
prof = torch.profiler.profile
|
|
pip.main(["install", "--user", "ffmpeg-python"])
|
|
pdb.set_trace()
|
|
'''
|
|
|
|
KNOWN_GOOD = '''\
|
|
import math
|
|
import numpy as np
|
|
import torch
|
|
|
|
from torch import nn
|
|
|
|
def add(a, b):
|
|
"""A perfectly fine module."""
|
|
return a + b + float(np.pi) + torch.tensor(1.0).sum()
|
|
'''
|
|
|
|
PIP_SHIM_FIXTURE = '''\
|
|
import cv2
|
|
import sys
|
|
import numpy as np
|
|
|
|
try:
|
|
import ffmpeg
|
|
except ImportError:
|
|
import pip
|
|
pip.main(['install', '--user', 'ffmpeg-python'])
|
|
import ffmpeg
|
|
'''
|
|
|
|
|
|
def _load_scanner():
|
|
spec = importlib.util.spec_from_file_location("keep_scan_compat", SCANNER)
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def scanner():
|
|
return _load_scanner()
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_known_bad_fixture_is_fully_flagged(scanner):
|
|
findings = scanner.scan_source(KNOWN_BAD, "fixture_bad.py")
|
|
flagged = {f["pattern"] for f in findings}
|
|
expected = {
|
|
"re_import",
|
|
"np_float",
|
|
"np_int",
|
|
"np_bool",
|
|
"np_object",
|
|
"np_str",
|
|
"np_nan",
|
|
"distutils",
|
|
"import_imp",
|
|
"torch_cuda_amp",
|
|
"pdb",
|
|
"torch_profiler",
|
|
"collections_mapping",
|
|
"import_pip",
|
|
"pip_main",
|
|
}
|
|
assert expected <= flagged, f"not flagged: {sorted(expected - flagged)}"
|
|
# the pdb pattern fires twice (import + set_trace), the pip pattern twice
|
|
assert len([f for f in findings if f["pattern"] == "pdb"]) == 2
|
|
assert len([f for f in findings if f["pattern"] == "pip_main"]) == 1
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_known_good_fixture_is_not_flagged(scanner):
|
|
assert scanner.scan_source(KNOWN_GOOD, "fixture_good.py") == []
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_pip_main_shim_fixture_is_flagged(scanner):
|
|
findings = scanner.scan_source(PIP_SHIM_FIXTURE, "video_util.py")
|
|
by_pattern = {f["pattern"]: f for f in findings}
|
|
assert "import_pip" in by_pattern, findings
|
|
assert "pip_main" in by_pattern, findings
|
|
assert by_pattern["import_pip"]["line"] == 8
|
|
assert by_pattern["pip_main"]["line"] == 9
|
|
assert "pip.main" in by_pattern["pip_main"]["snippet"]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_finding_line_numbers_are_exact(scanner):
|
|
findings = scanner.scan_source(KNOWN_BAD, "fixture_bad.py")
|
|
lines = {f["pattern"]: f["line"] for f in findings}
|
|
assert lines["re_import"] == 4
|
|
assert lines["import_pip"] == 3
|
|
assert lines["np_nan"] == 14
|
|
assert lines["pip_main"] == 17
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_real_tree_findings(scanner):
|
|
"""The live tree must not contain any finding any more.
|
|
|
|
P2.1 removed ``from re import T``, the ``pdb`` imports, the
|
|
``torch.profiler`` import and the pip shim module; P2.3 removed the
|
|
remaining ``distutils`` import. The list below is therefore the
|
|
*expected* state of the tree - a new finding fails this test.
|
|
"""
|
|
report = scanner.scan_repo(REPO_ROOT)
|
|
findings = report["findings"]
|
|
assert report["closure_file_count"] > 0
|
|
located = {(f["path"], f["line"], f["pattern"]) for f in findings}
|
|
|
|
assert located == set(), (
|
|
"the vendored closure must be free of compatibility findings, "
|
|
f"got {sorted(located)}"
|
|
)
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_closure_contains_the_runtime_files(scanner):
|
|
report = scanner.scan_repo(REPO_ROOT)
|
|
closure = set(report["closure_files"])
|
|
for expected in (
|
|
"modules/deps/basicsr/archs/keep_arch.py",
|
|
"modules/deps/basicsr/archs/gmflow_arch.py",
|
|
"modules/deps/basicsr/archs/arch_util.py",
|
|
"modules/deps/basicsr/utils/img_util.py",
|
|
"modules/deps/facelib/utils/face_restoration_helper.py",
|
|
):
|
|
assert expected in closure, f"{expected} is not in the runtime closure"
|
|
# P2.1 deleted the pip-shim module - it must not be in the closure any more
|
|
assert "modules/deps/basicsr/utils/video_util.py" not in closure
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_closure_contains_the_explicitly_listed_modules(scanner):
|
|
"""The modules of the explicit import lists belong to the closure.
|
|
|
|
P2.4 replaced the four eager folder scans of the vendored ``basicsr``
|
|
packages with explicit import lists - the same modules are loaded by
|
|
``import basicsr``, but the closure computation no longer has to model a
|
|
dynamic scan.
|
|
"""
|
|
report = scanner.scan_repo(REPO_ROOT)
|
|
closure = set(report["closure_files"])
|
|
for expected in (
|
|
# models/__init__.py - explicit list
|
|
"modules/deps/basicsr/models/keep_model.py",
|
|
"modules/deps/basicsr/models/keep_gan_model.py",
|
|
"modules/deps/basicsr/models/base_model.py",
|
|
# data/__init__.py - explicit list
|
|
"modules/deps/basicsr/data/reds_dataset.py",
|
|
"modules/deps/basicsr/data/single_image_dataset.py",
|
|
# losses/__init__.py - explicit list
|
|
"modules/deps/basicsr/losses/basic_loss.py",
|
|
# reached through 'from <package> import <submodule>'
|
|
"modules/deps/basicsr/models/lr_scheduler.py",
|
|
"modules/deps/basicsr/utils/flow_util.py",
|
|
):
|
|
assert expected in closure, (
|
|
f"{expected} is not in the runtime closure - the explicit import "
|
|
"lists of the vendored basicsr are not resolved"
|
|
)
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_exit_code_contract(scanner, monkeypatch):
|
|
"""Exit code 0 = no findings, 1 = findings found (a result, not a crash)."""
|
|
assert scanner.main(["--quiet"]) == 0, (
|
|
"the real tree must be free of findings, so the scanner must exit 0"
|
|
)
|
|
monkeypatch.setattr(
|
|
scanner,
|
|
"scan_repo",
|
|
lambda *args, **kwargs: {
|
|
"entry_points": [],
|
|
"closure_file_count": 0,
|
|
"closure_files": [],
|
|
"finding_count": 0,
|
|
"findings_by_pattern": {},
|
|
"findings": [],
|
|
},
|
|
)
|
|
assert scanner.main(["--quiet"]) == 0, (
|
|
"a clean scan must exit 0"
|
|
)
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_audit_doc_lists_every_finding(scanner):
|
|
report = scanner.scan_repo(REPO_ROOT)
|
|
text = AUDIT_DOC.read_text(encoding="utf-8")
|
|
for finding in report["findings"]:
|
|
location = f"{finding['path']}:{finding['line']}"
|
|
assert location in text, f"{location} is not in the fix list"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# P2.3 - the mechanical Python 3.13 / numpy 2 fixes
|
|
# ---------------------------------------------------------------------------
|
|
|
|
#: Imports every module of the runtime closure in a **fresh** interpreter (the
|
|
#: import must not leak into the pytest session) and reports the failures.
|
|
_CLOSURE_IMPORT_SCRIPT = """\
|
|
import importlib, json, sys
|
|
|
|
sys.path.insert(0, sys.argv[1])
|
|
import conftest # registers the package alias and the ComfyUI mocks
|
|
|
|
failures = []
|
|
for name in json.loads(sys.argv[2]):
|
|
try:
|
|
importlib.import_module(name)
|
|
except BaseException as exc:
|
|
failures.append({"module": name, "type": type(exc).__name__, "error": str(exc)})
|
|
print("KEEP_CLOSURE_FAILURES:" + json.dumps(failures))
|
|
"""
|
|
|
|
|
|
def _closure_module_names(report):
|
|
"""The importable module names of the vendored part of the closure."""
|
|
names = []
|
|
for rel in report["closure_files"]:
|
|
if not rel.startswith("modules/deps/"):
|
|
continue
|
|
dotted = rel[len("modules/deps/"):].replace("/", ".").removesuffix(".py")
|
|
if dotted.endswith(".__init__"):
|
|
dotted = dotted[: -len(".__init__")]
|
|
names.append("ComfyUI_KEEP.modules.deps." + dotted)
|
|
return names
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_every_closure_module_imports(scanner):
|
|
"""P2.3 acceptance: every module of the runtime closure still imports.
|
|
|
|
The imports run in a subprocess: the vendored packages register their
|
|
classes in global registries and must not pollute the pytest session.
|
|
"""
|
|
report = scanner.scan_repo(REPO_ROOT)
|
|
names = _closure_module_names(report)
|
|
assert names, "the runtime closure is empty"
|
|
|
|
result = subprocess.run(
|
|
[sys.executable, "-c", _CLOSURE_IMPORT_SCRIPT, str(REPO_ROOT), json.dumps(names)],
|
|
cwd=str(REPO_ROOT),
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=600,
|
|
)
|
|
sentinel = [
|
|
line for line in result.stdout.splitlines() if line.startswith("KEEP_CLOSURE_FAILURES:")
|
|
]
|
|
assert sentinel, (
|
|
f"the closure import probe produced no result\n"
|
|
f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
|
|
)
|
|
failures = json.loads(sentinel[-1].split(":", 1)[1])
|
|
assert failures == [], f"closure modules failed to import: {failures}"
|