From 47e5bcc68a3961e0e16a8e092af1ec6690bdceb8 Mon Sep 17 00:00:00 2001 From: Peyton DeNiro <25550995+pydn@users.noreply.github.com> Date: Sat, 28 Mar 2026 13:24:13 -0500 Subject: [PATCH 01/17] Add runtime E2E validation harness --- comfyui_to_python.py | 61 +- .../runtime/subgraph-identifiers.json | 19 + tests/fixtures/runtime/text-to-image.json | 86 +++ tests/fixtures/runtime/unsafe-kwargs.json | 11 + .../runtime/upscale-model-loader.json | 37 + tests/runtime/run_runtime_validation.py | 674 ++++++++++++++++++ 6 files changed, 874 insertions(+), 14 deletions(-) create mode 100644 tests/fixtures/runtime/subgraph-identifiers.json create mode 100644 tests/fixtures/runtime/text-to-image.json create mode 100644 tests/fixtures/runtime/unsafe-kwargs.json create mode 100644 tests/fixtures/runtime/upscale-model-loader.json create mode 100644 tests/runtime/run_runtime_validation.py diff --git a/comfyui_to_python.py b/comfyui_to_python.py index ebcf928..59dad8a 100644 --- a/comfyui_to_python.py +++ b/comfyui_to_python.py @@ -196,6 +196,7 @@ class CodeGenerator: def generate_workflow( self, load_order: List, + workflow_data: Dict, queue_size: int = 10, ) -> str: """Generate the execution code based on the load order. @@ -209,7 +210,7 @@ class CodeGenerator: """ # Create the necessary data structures to hold imports and generated code import_statements, executed_variables, special_functions_code, code = ( - set(["NODE_CLASS_MAPPINGS"]), + {"nodes": {"NODE_CLASS_MAPPINGS"}}, {}, [], [], @@ -245,8 +246,9 @@ class CodeGenerator: ) initialized_objects[class_type] = self.clean_variable_name(class_type) if class_type in self.base_node_class_mappings.keys(): - import_statements.add(import_statement) - if class_type not in self.base_node_class_mappings.keys(): + module_name, import_name = import_statement + import_statements.setdefault(module_name, set()).add(import_name) + if 'NODE_CLASS_MAPPINGS["' in class_code: custom_nodes = True special_functions_code.append(class_code) @@ -268,6 +270,11 @@ class CodeGenerator: and "unique_id" in input_types["hidden"].keys() ): inputs["unique_id"] = random.randint(1, 2**64) + if "hidden" in input_types.keys(): + if "prompt" in input_types["hidden"].keys(): + inputs["prompt"] = {"variable_name": "prompt"} + if "extra_pnginfo" in input_types["hidden"].keys(): + inputs["extra_pnginfo"] = {"variable_name": "extra_pnginfo"} elif class_def_params is not None: if "unique_id" in class_def_params: inputs["unique_id"] = random.randint(1, 2**64) @@ -302,6 +309,7 @@ class CodeGenerator: import_statements, special_functions_code, code, + workflow_data, queue_size, custom_nodes, ) @@ -361,18 +369,20 @@ class CodeGenerator: def assemble_python_code( self, - import_statements: set, + import_statements: Dict[str, set], speical_functions_code: List[str], code: List[str], + workflow_data: Dict, queue_size: int, custom_nodes=False, ) -> str: """Generates the final code string. Args: - import_statements (set): A set of unique import statements. + import_statements (Dict[str, set]): Import statements grouped by module. speical_functions_code (List[str]): A list of special functions code strings. code (List[str]): A list of code strings. + workflow_data (Dict): The original workflow data for metadata preservation. queue_size (int): Number of photos that will be generated by the script. custom_nodes (bool): Whether to include custom nodes in the code. @@ -392,6 +402,7 @@ class CodeGenerator: # Define static import statements required for the script static_imports = ( [ + "import json", "import os", "import random", "import sys", @@ -399,7 +410,17 @@ class CodeGenerator: "import torch", ] + func_strings - + ["\n\nadd_comfyui_directory_to_sys_path()\nadd_extra_model_paths()\n"] + + [ + "\n\nadd_comfyui_directory_to_sys_path()\nadd_extra_model_paths()\n", + f"workflow = json.loads({json.dumps(json.dumps(workflow_data))})", + "prompt = json.loads(json.dumps(workflow))", + ( + "extra_pnginfo = " + '{"workflow": {"source": "workflow_api", "payload": json.loads(' + + json.dumps(json.dumps(workflow_data)) + + ")}}" + ), + ] ) # Check if custom nodes should be included if custom_nodes: @@ -408,9 +429,10 @@ class CodeGenerator: else: custom_nodes = "" # Create import statements for node classes - imports_code = [ - f"from nodes import {', '.join([class_name for class_name in import_statements])}" - ] + imports_code = [] + for module_name in sorted(import_statements.keys()): + class_names = ", ".join(sorted(import_statements[module_name])) + imports_code.append(f"from {module_name} import {class_names}") # Assemble the main function code, including custom nodes if applicable main_function_code = ( "def main():\n\t" @@ -430,20 +452,31 @@ class CodeGenerator: return final_code - def get_class_info(self, class_type: str) -> Tuple[str, str, str]: + def get_class_info(self, class_type: str) -> Tuple[str, Tuple[str, str], str]: """Generates and returns necessary information about class type. Args: class_type (str): Class type. Returns: - Tuple[str, str, str]: Updated class type, import statement string, class initialization code. + Tuple[str, Tuple[str, str], str]: Updated class type, import statement and initialization code. """ + class_obj = self.base_node_class_mappings.get(class_type) + module_name = "nodes" + if class_obj is not None: + module_name = class_obj.__module__ variable_name = self.clean_variable_name(class_type) - import_statement = class_type - if class_type in self.base_node_class_mappings.keys(): + is_importable_module = bool( + module_name + and "/" not in module_name + and "\\" not in module_name + and all(part.isidentifier() for part in module_name.split(".")) + ) + if class_type in self.base_node_class_mappings.keys() and is_importable_module: + import_statement = (module_name, class_type) class_code = f"{variable_name} = {class_type.strip()}()" else: + import_statement = ("nodes", "NODE_CLASS_MAPPINGS") class_code = f'{variable_name} = NODE_CLASS_MAPPINGS["{class_type}"]()' return class_type, import_statement, class_code @@ -596,7 +629,7 @@ class ComfyUItoPython: self.node_class_mappings, self.base_node_class_mappings ) generated_code = code_generator.generate_workflow( - load_order, queue_size=self.queue_size + load_order, data, queue_size=self.queue_size ) # Step 5: Write the generated code to a file diff --git a/tests/fixtures/runtime/subgraph-identifiers.json b/tests/fixtures/runtime/subgraph-identifiers.json new file mode 100644 index 0000000..3690e72 --- /dev/null +++ b/tests/fixtures/runtime/subgraph-identifiers.json @@ -0,0 +1,19 @@ +{ + "173:132": { + "class_type": "RegexReplace", + "inputs": { + "text": "abc123", + "pattern": "\\d+", + "replace": "X" + } + }, + "174:133": { + "class_type": "PassthroughText", + "inputs": { + "text": [ + "173:132", + 0 + ] + } + } +} diff --git a/tests/fixtures/runtime/text-to-image.json b/tests/fixtures/runtime/text-to-image.json new file mode 100644 index 0000000..6d5cf67 --- /dev/null +++ b/tests/fixtures/runtime/text-to-image.json @@ -0,0 +1,86 @@ +{ + "1": { + "class_type": "CheckpointLoaderSimple", + "inputs": { + "ckpt_name": "v1-5-pruned-emaonly-fp16.safetensors" + } + }, + "2": { + "class_type": "CLIPTextEncode", + "inputs": { + "text": "a small cottage in a meadow, soft daylight", + "clip": [ + "1", + 1 + ] + } + }, + "3": { + "class_type": "CLIPTextEncode", + "inputs": { + "text": "blurry, low quality", + "clip": [ + "1", + 1 + ] + } + }, + "4": { + "class_type": "EmptyLatentImage", + "inputs": { + "width": 512, + "height": 512, + "batch_size": 1 + } + }, + "5": { + "class_type": "KSampler", + "inputs": { + "seed": 1, + "steps": 4, + "cfg": 7, + "sampler_name": "euler", + "scheduler": "normal", + "denoise": 1, + "model": [ + "1", + 0 + ], + "positive": [ + "2", + 0 + ], + "negative": [ + "3", + 0 + ], + "latent_image": [ + "4", + 0 + ] + } + }, + "6": { + "class_type": "VAEDecode", + "inputs": { + "samples": [ + "5", + 0 + ], + "vae": [ + "1", + 2 + ] + } + }, + "7": { + "class_type": "SaveImage", + "inputs": { + "filename_prefix": "E2E_text_to_image", + "images": [ + "6", + 0 + ] + } + } +} diff --git a/tests/fixtures/runtime/unsafe-kwargs.json b/tests/fixtures/runtime/unsafe-kwargs.json new file mode 100644 index 0000000..9c7fdae --- /dev/null +++ b/tests/fixtures/runtime/unsafe-kwargs.json @@ -0,0 +1,11 @@ +{ + "1": { + "class_type": "FlexibleNode", + "inputs": { + "safe_name": "kept-readable", + "class": "reserved-word", + "\u2795 Add Lora": "symbol-heavy-key", + "spaces and-hyphens": "still-unsafe" + } + } +} diff --git a/tests/fixtures/runtime/upscale-model-loader.json b/tests/fixtures/runtime/upscale-model-loader.json new file mode 100644 index 0000000..948daa1 --- /dev/null +++ b/tests/fixtures/runtime/upscale-model-loader.json @@ -0,0 +1,37 @@ +{ + "1": { + "class_type": "LoadImage", + "inputs": { + "image": "e2e_upscale_input.png" + } + }, + "2": { + "class_type": "UpscaleModelLoader", + "inputs": { + "model_name": "RealESRGAN_x4plus.safetensors" + } + }, + "3": { + "class_type": "ImageUpscaleWithModel", + "inputs": { + "upscale_model": [ + "2", + 0 + ], + "image": [ + "1", + 0 + ] + } + }, + "4": { + "class_type": "SaveImage", + "inputs": { + "filename_prefix": "E2E_upscale_model_loader", + "images": [ + "3", + 0 + ] + } + } +} diff --git a/tests/runtime/run_runtime_validation.py b/tests/runtime/run_runtime_validation.py new file mode 100644 index 0000000..6312449 --- /dev/null +++ b/tests/runtime/run_runtime_validation.py @@ -0,0 +1,674 @@ +import argparse +import ast +import json +import os +import shutil +import struct +import subprocess +import sys +import tempfile +import zlib +from dataclasses import dataclass +from io import StringIO +from pathlib import Path +from typing import Callable + + +ROOT = Path(__file__).resolve().parents[2] +FIXTURE_DIR = ROOT / "tests" / "fixtures" / "runtime" +GENERATED_DIR = ROOT / "tests" / "runtime" / "generated" +BASELINE_COMFYUI_PATH = Path("/home/peyton/code/ComfyUI") +UPSTREAM_SMOKE_COMFYUI_PATH = Path("/home/peyton/code/ComfyUI-upstream-smoke") +COMFYUI_OUTPUT_DIRNAME = "output" +COMFYUI_INPUT_DIRNAME = "input" + +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + + +class ValidationFailure(RuntimeError): + def __init__(self, classification: str, message: str): + super().__init__(message) + self.classification = classification + self.message = message + + +@dataclass(frozen=True) +class FixtureConfig: + name: str + path: Path + mapping_factory: Callable[[], dict] | None = None + fast_mapping_factory: Callable[[], dict] | None = None + runtime_capable: bool = False + filename_prefix: str | None = None + expected_min_dimensions: tuple[int, int] | None = None + metadata_markers: tuple[str, ...] = () + model_requirements: tuple["ModelRequirement", ...] = () + staged_inputs: tuple["StagedInput", ...] = () + + +@dataclass(frozen=True) +class ModelRequirement: + filename: str + relative_dir: str + source_url: str + + +@dataclass(frozen=True) +class StagedInput: + source_path: Path + destination_name: str + + +class FlexibleNode: + CATEGORY = "utils" + FUNCTION = "run" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "safe_name": ("STRING",), + } + } + + def run(self, **kwargs): + return (kwargs,) + + +class RegexReplace: + CATEGORY = "utils" + FUNCTION = "replace" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "text": ("STRING",), + "pattern": ("STRING",), + "replace": ("STRING",), + } + } + + def replace(self, text, pattern, replace): + return (text,) + + +class PassthroughText: + CATEGORY = "utils" + FUNCTION = "run" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "text": ("STRING",), + } + } + + def run(self, text): + return (text,) + + +class UpscaleModelLoader: + CATEGORY = "loaders" + FUNCTION = "load_model" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "model_name": ("STRING",), + } + } + + def load_model(self, model_name): + return (model_name,) + + +class LoadImage: + CATEGORY = "image" + FUNCTION = "load_image" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "image": ("STRING",), + } + } + + def load_image(self, image): + return (image,) + + +class ImageUpscaleWithModel: + CATEGORY = "image/upscaling" + FUNCTION = "upscale" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "upscale_model": ("UPSCALE_MODEL",), + "image": ("IMAGE",), + } + } + + def upscale(self, upscale_model, image): + return (image,) + + +class SaveImageNode: + CATEGORY = "image" + FUNCTION = "save_images" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "images": ("IMAGE",), + "filename_prefix": ("STRING",), + } + } + + def save_images(self, images, filename_prefix): + return () + + +FIXTURES = { + "upscale-model-loader": FixtureConfig( + name="upscale-model-loader", + path=FIXTURE_DIR / "upscale-model-loader.json", + fast_mapping_factory=lambda: { + "LoadImage": LoadImage, + "UpscaleModelLoader": UpscaleModelLoader, + "ImageUpscaleWithModel": ImageUpscaleWithModel, + "SaveImage": SaveImageNode, + }, + runtime_capable=True, + filename_prefix="E2E_upscale_model_loader", + expected_min_dimensions=(1000, 1000), + metadata_markers=("UpscaleModelLoader", "E2E_upscale_model_loader"), + model_requirements=( + ModelRequirement( + filename="RealESRGAN_x4plus.safetensors", + relative_dir="models/upscale_models", + source_url="https://huggingface.co/Comfy-Org/Real-ESRGAN_repackaged/resolve/main/RealESRGAN_x4plus.safetensors", + ), + ), + staged_inputs=( + StagedInput( + source_path=ROOT / "images" / "save_as_script.png", + destination_name="e2e_upscale_input.png", + ), + ), + ), + "text-to-image": FixtureConfig( + name="text-to-image", + path=FIXTURE_DIR / "text-to-image.json", + runtime_capable=True, + filename_prefix="E2E_text_to_image", + expected_min_dimensions=(512, 512), + metadata_markers=("a small cottage in a meadow", "CheckpointLoaderSimple"), + model_requirements=( + ModelRequirement( + filename="v1-5-pruned-emaonly-fp16.safetensors", + relative_dir="models/checkpoints", + source_url="https://huggingface.co/Comfy-Org/stable-diffusion-v1-5-archive/resolve/main/v1-5-pruned-emaonly-fp16.safetensors", + ), + ), + ), + "unsafe-kwargs": FixtureConfig( + name="unsafe-kwargs", + path=FIXTURE_DIR / "unsafe-kwargs.json", + mapping_factory=lambda: {"FlexibleNode": FlexibleNode}, + ), + "subgraph-identifiers": FixtureConfig( + name="subgraph-identifiers", + path=FIXTURE_DIR / "subgraph-identifiers.json", + mapping_factory=lambda: { + "RegexReplace": RegexReplace, + "PassthroughText": PassthroughText, + }, + ), +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run fast, runtime, or upstream-smoke validation for committed fixtures." + ) + parser.add_argument( + "--tier", + choices=("fast", "runtime", "upstream-smoke"), + help="Validation tier to run.", + ) + parser.add_argument( + "--fixture", + default="all", + help="Fixture name or 'all'.", + ) + parser.add_argument( + "--execute", + action="store_true", + help="Execute generated Python for runtime-capable fixtures.", + ) + parser.add_argument( + "--print-download-plan", + action="store_true", + help="Print model download commands for missing models.", + ) + parser.add_argument( + "--internal-export", + choices=tuple(FIXTURES.keys()), + help=argparse.SUPPRESS, + ) + parser.add_argument( + "--generated-path", + help=argparse.SUPPRESS, + ) + args = parser.parse_args() + if not args.internal_export and not args.tier: + parser.error("--tier is required unless --internal-export is used.") + return args + + +def load_fixture_names(selection: str) -> list[str]: + if selection == "all": + return list(FIXTURES.keys()) + if selection == "smoke": + return ["upscale-model-loader", "text-to-image"] + if selection not in FIXTURES: + raise ValidationFailure("fixture bug", f"Unknown fixture '{selection}'.") + return [selection] + + +def ensure_runtime_path(tier: str) -> str: + if tier == "runtime": + runtime_path = BASELINE_COMFYUI_PATH + elif tier == "upstream-smoke": + runtime_path = UPSTREAM_SMOKE_COMFYUI_PATH + else: + return os.environ.get("COMFYUI_PATH", "") + + if not runtime_path.is_dir(): + raise ValidationFailure( + "environment/setup failure", + f"Expected ComfyUI checkout at {runtime_path}, but it was not found.", + ) + + return str(runtime_path) + + +def get_runtime_python(runtime_path: str) -> str: + runtime_python = Path(runtime_path) / ".venv" / "bin" / "python" + if runtime_python.is_file(): + return str(runtime_python) + return sys.executable + + +def get_fixture(fixture_name: str) -> FixtureConfig: + return FIXTURES[fixture_name] + + +def check_models(fixture: FixtureConfig, runtime_path: str) -> list[ModelRequirement]: + runtime_root = Path(runtime_path) + missing = [] + for requirement in fixture.model_requirements: + target = runtime_root / requirement.relative_dir / requirement.filename + if not target.is_file(): + missing.append(requirement) + return missing + + +def print_download_plan(fixture: FixtureConfig, runtime_path: str) -> None: + runtime_root = Path(runtime_path) + for requirement in check_models(fixture, runtime_path): + target_dir = runtime_root / requirement.relative_dir + print( + "download:", + f"mkdir -p {target_dir} && curl -L {requirement.source_url} -o {target_dir / requirement.filename}", + ) + + +def stage_inputs(fixture: FixtureConfig, runtime_path: str) -> None: + runtime_input_dir = Path(runtime_path) / COMFYUI_INPUT_DIRNAME + runtime_input_dir.mkdir(parents=True, exist_ok=True) + for staged_input in fixture.staged_inputs: + if not staged_input.source_path.is_file(): + raise ValidationFailure( + "fixture bug", + f"Missing staged input source {staged_input.source_path} for {fixture.name}.", + ) + shutil.copyfile( + staged_input.source_path, + runtime_input_dir / staged_input.destination_name, + ) + + +def export_workflow( + fixture: FixtureConfig, + tier: str, + runtime_path: str, +) -> tuple[str, str]: + from comfyui_to_python import ComfyUItoPython + + workflow = fixture.path.read_text(encoding="utf-8") + output = StringIO() + kwargs = { + "workflow": workflow, + "output_file": output, + } + if tier == "fast" and fixture.fast_mapping_factory is not None: + kwargs["node_class_mappings"] = fixture.fast_mapping_factory() + elif fixture.mapping_factory is not None: + kwargs["node_class_mappings"] = fixture.mapping_factory() + else: + os.environ["COMFYUI_PATH"] = runtime_path + + try: + ComfyUItoPython(**kwargs) + except ModuleNotFoundError as exc: + missing_module = exc.name or "unknown" + raise ValidationFailure( + "environment/setup failure", + f"Missing runtime dependency '{missing_module}' while exporting {fixture.name}.", + ) from exc + except KeyError as exc: + raise ValidationFailure( + "repo regression", + f"Exporter failed to resolve workflow data for {fixture.name}: {exc}", + ) from exc + except Exception as exc: + raise ValidationFailure( + "repo regression", + f"Exporter failed for {fixture.name}: {exc}", + ) from exc + + return workflow, output.getvalue() + + +def export_workflow_in_runtime_env(fixture: FixtureConfig, runtime_path: str) -> str: + runtime_python = get_runtime_python(runtime_path) + GENERATED_DIR.mkdir(parents=True, exist_ok=True) + generated_path = GENERATED_DIR / f"{fixture.name}.py" + env = os.environ.copy() + env["COMFYUI_PATH"] = runtime_path + env["PYTHONPATH"] = os.pathsep.join([str(ROOT), env.get("PYTHONPATH", "")]).rstrip( + os.pathsep + ) + result = subprocess.run( + [ + runtime_python, + str(Path(__file__).resolve()), + "--internal-export", + fixture.name, + "--generated-path", + str(generated_path), + ], + cwd=ROOT, + env=env, + capture_output=True, + text=True, + ) + if result.returncode != 0: + output = (result.stderr or result.stdout or "").strip() + classification = "environment/setup failure" + if ( + "Missing runtime dependency" not in output + and "ModuleNotFoundError" not in output + ): + classification = "repo regression" + raise ValidationFailure( + classification, + f"Runtime export failed for {fixture.name}: {output}", + ) + return generated_path.read_text(encoding="utf-8") + + +def validate_generated_python(generated_code: str, fixture_name: str) -> None: + try: + ast.parse(generated_code) + except SyntaxError as exc: + raise ValidationFailure( + "repo regression", + f"Generated Python is not valid for {fixture_name}: {exc}", + ) from exc + + +def parse_png_info(image_path: Path) -> tuple[int, int, dict[str, str]]: + with image_path.open("rb") as handle: + signature = handle.read(8) + if signature != b"\x89PNG\r\n\x1a\n": + raise ValidationFailure( + "environment/setup failure", + f"Expected PNG output for {image_path.name}, got a different file format.", + ) + + width = height = None + text_data: dict[str, str] = {} + while True: + length_bytes = handle.read(4) + if not length_bytes: + break + length = struct.unpack(">I", length_bytes)[0] + chunk_type = handle.read(4) + chunk_data = handle.read(length) + handle.read(4) + + if chunk_type == b"IHDR": + width, height = struct.unpack(">II", chunk_data[:8]) + elif chunk_type == b"tEXt": + key, value = chunk_data.split(b"\x00", 1) + text_data[key.decode("latin-1")] = value.decode("latin-1") + elif chunk_type == b"zTXt": + key, compressed = chunk_data.split(b"\x00", 1) + text_data[key.decode("latin-1")] = zlib.decompress( + compressed[1:] + ).decode("latin-1") + elif chunk_type == b"iTXt": + parts = chunk_data.split(b"\x00", 5) + if len(parts) == 6: + key = parts[0].decode("utf-8") + compressed_flag = parts[1] + value = parts[5] + if compressed_flag == b"\x01": + value = zlib.decompress(value) + text_data[key] = value.decode("utf-8") + elif chunk_type == b"IEND": + break + + if width is None or height is None: + raise ValidationFailure( + "environment/setup failure", + f"Could not read PNG dimensions from {image_path.name}.", + ) + + return width, height, text_data + + +def validate_output_artifact( + fixture: FixtureConfig, + output_path: Path, +) -> None: + width, height, metadata = parse_png_info(output_path) + + if fixture.expected_min_dimensions is not None: + min_width, min_height = fixture.expected_min_dimensions + if width < min_width or height < min_height: + raise ValidationFailure( + "repo regression", + f"Output dimensions for {fixture.name} were {width}x{height}, expected at least {min_width}x{min_height}.", + ) + + metadata_blob = "\n".join( + [output_path.name] + [f"{key}={value}" for key, value in metadata.items()] + ) + for marker in fixture.metadata_markers: + if marker not in metadata_blob: + raise ValidationFailure( + "repo regression", + f"Output metadata for {fixture.name} did not contain expected marker '{marker}'.", + ) + + +def execute_generated_python( + generated_code: str, + fixture: FixtureConfig, + runtime_path: str, +) -> None: + if not fixture.runtime_capable: + raise ValidationFailure( + "fixture bug", + f"Fixture {fixture.name} is not marked runtime-capable.", + ) + + with tempfile.TemporaryDirectory() as tmpdir: + tmp_path = Path(tmpdir) / f"{fixture.name}.py" + tmp_path.write_text(generated_code, encoding="utf-8") + output_dir = Path(runtime_path) / COMFYUI_OUTPUT_DIRNAME + existing_outputs = set(output_dir.glob("*.png")) + env = os.environ.copy() + env["COMFYUI_PATH"] = runtime_path + env["PYTHONPATH"] = os.pathsep.join( + [str(ROOT), env.get("PYTHONPATH", "")] + ).rstrip(os.pathsep) + runtime_python = get_runtime_python(runtime_path) + + result = subprocess.run( + [runtime_python, str(tmp_path)], + cwd=ROOT, + env=env, + capture_output=True, + text=True, + ) + if result.returncode == 0: + if fixture.filename_prefix is None: + return + new_outputs = [ + path + for path in output_dir.glob(f"{fixture.filename_prefix}*.png") + if path not in existing_outputs + ] + if not new_outputs: + raise ValidationFailure( + "repo regression", + f"Generated script for {fixture.name} did not produce a new output file with prefix {fixture.filename_prefix}.", + ) + newest_output = max(new_outputs, key=lambda path: path.stat().st_mtime) + validate_output_artifact(fixture, newest_output) + return + + stderr = (result.stderr or "").strip() + stdout = (result.stdout or "").strip() + output = stderr or stdout or "generated script exited with a non-zero status" + lower_output = output.lower() + + if "no module named 'torch'" in lower_output: + classification = "environment/setup failure" + elif "no such file or directory" in lower_output or "not found" in lower_output: + classification = "environment/setup failure" + else: + classification = "repo regression" + + raise ValidationFailure( + classification, + f"Generated script execution failed for {fixture.name}: {output}", + ) + + +def run_fixture(fixture: FixtureConfig, tier: str, execute: bool, runtime_path: str) -> str: + if tier == "fast": + _, generated_code = export_workflow(fixture, tier, runtime_path) + else: + missing_models = check_models(fixture, runtime_path) + if missing_models: + raise ValidationFailure( + "model provisioning failure", + "Missing models for " + f"{fixture.name}: " + + ", ".join( + f"{item.relative_dir}/{item.filename}" for item in missing_models + ), + ) + stage_inputs(fixture, runtime_path) + generated_code = export_workflow_in_runtime_env(fixture, runtime_path) + validate_generated_python(generated_code, fixture.name) + + should_execute = execute or tier in {"runtime", "upstream-smoke"} + if should_execute and tier in {"runtime", "upstream-smoke"}: + execute_generated_python(generated_code, fixture, runtime_path) + + return "pass" + + +def main() -> int: + args = parse_args() + + if args.internal_export: + fixture = get_fixture(args.internal_export) + output_path = Path(args.generated_path) + _, generated_code = export_workflow( + fixture=fixture, + tier="runtime", + runtime_path=os.environ.get("COMFYUI_PATH", ""), + ) + output_path.write_text(generated_code, encoding="utf-8") + return 0 + + try: + runtime_path = ensure_runtime_path(args.tier) + fixture_names = load_fixture_names(args.fixture) + requested = [FIXTURES[name] for name in fixture_names] + + if args.tier == "fast": + requested = [ + fixture + for fixture in requested + if fixture.fast_mapping_factory is not None + or fixture.mapping_factory is not None + ] + if not requested: + raise ValidationFailure( + "fixture bug", + "No selected fixtures are fast-tier compatible.", + ) + elif args.tier in {"runtime", "upstream-smoke"}: + requested = [fixture for fixture in requested if fixture.runtime_capable] + if not requested: + raise ValidationFailure( + "fixture bug", + "No selected fixtures are runtime-capable for this tier.", + ) + + failures: list[tuple[str, str, str]] = [] + for fixture in requested: + try: + if args.print_download_plan and args.tier in {"runtime", "upstream-smoke"}: + print_download_plan(fixture, runtime_path) + status = run_fixture(fixture, args.tier, args.execute, runtime_path) + print(f"{fixture.name}: {status}") + except ValidationFailure as exc: + failures.append((fixture.name, exc.classification, exc.message)) + print( + f"{fixture.name}: fail ({exc.classification})", + file=sys.stderr, + ) + print(exc.message, file=sys.stderr) + if failures: + classifications = ", ".join( + f"{name}={classification}" for name, classification, _ in failures + ) + print(f"classification: {classifications}", file=sys.stderr) + return 1 + except ValidationFailure as exc: + print(f"classification: {exc.classification}", file=sys.stderr) + print(exc.message, file=sys.stderr) + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 3ffd985cf7b6c00ef8330c05cf4ada2af9221422 Mon Sep 17 00:00:00 2001 From: Peyton DeNiro <25550995+pydn@users.noreply.github.com> Date: Sat, 28 Mar 2026 13:33:08 -0500 Subject: [PATCH 02/17] Fix runtime validation exporter edge cases --- comfyui_to_python.py | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/comfyui_to_python.py b/comfyui_to_python.py index 59dad8a..201ef8c 100644 --- a/comfyui_to_python.py +++ b/comfyui_to_python.py @@ -2,6 +2,7 @@ import copy import glob import inspect import json +import keyword import os import random import sys @@ -280,7 +281,10 @@ class CodeGenerator: inputs["unique_id"] = random.randint(1, 2**64) # Create executed variable and generate code - executed_variables[idx] = f"{self.clean_variable_name(class_type)}_{idx}" + executed_variables[idx] = ( + f"{self.clean_variable_name(class_type)}_" + f"{self.clean_variable_name(str(idx))}" + ) inputs = self.update_inputs(inputs, executed_variables) if is_special_function: @@ -358,14 +362,21 @@ class CodeGenerator: Returns: str: Formatted argument as a string. """ + value_code = self.format_arg_value(key, value) + if key.isidentifier() and not keyword.iskeyword(key): + return f"{key}={value_code}" + return f"**{{{json.dumps(key)}: {value_code}}}" + + @staticmethod + def format_arg_value(key: str, value: any) -> str: + """Formats an argument value as Python source.""" if key == "noise_seed" or key == "seed": - return f"{key}=random.randint(1, 2**64)" - elif isinstance(value, str): - value = value.replace("\n", "\\n").replace('"', "'") - return f'{key}="{value}"' - elif isinstance(value, dict) and "variable_name" in value: - return f'{key}={value["variable_name"]}' - return f"{key}={value}" + return "random.randint(1, 2**64)" + if isinstance(value, str): + return json.dumps(value) + if isinstance(value, dict) and "variable_name" in value: + return value["variable_name"] + return repr(value) def assemble_python_code( self, @@ -433,13 +444,15 @@ class CodeGenerator: for module_name in sorted(import_statements.keys()): class_names = ", ".join(sorted(import_statements[module_name])) imports_code.append(f"from {module_name} import {class_names}") + special_functions_body = "\n\t\t".join(speical_functions_code) or "pass" + loop_body = "\n\t\t".join(code) or "\tpass" # Assemble the main function code, including custom nodes if applicable main_function_code = ( "def main():\n\t" + f"{custom_nodes}with torch.inference_mode():\n\t\t" - + "\n\t\t".join(speical_functions_code) + + special_functions_body + f"\n\n\t\tfor q in range({queue_size}):\n\t\t" - + "\n\t\t".join(code) + + loop_body ) # Concatenate all parts to form the final code final_code = "\n".join( From 1fa2db89e15dd74d984f0ed1112263765df54b9e Mon Sep 17 00:00:00 2001 From: Peyton DeNiro <25550995+pydn@users.noreply.github.com> Date: Sat, 28 Mar 2026 13:42:44 -0500 Subject: [PATCH 03/17] Preserve reimportable ComfyUI workflow metadata --- __init__.py | 7 +++- comfyui_to_python.py | 26 +++++++++++--- js/save-as-script.js | 6 +++- tests/test_upscale_model_loader_export.py | 44 +++++++++++++++++++++-- 4 files changed, 73 insertions(+), 10 deletions(-) diff --git a/__init__.py b/__init__.py index 69ce450..10cdf1c 100644 --- a/__init__.py +++ b/__init__.py @@ -45,9 +45,14 @@ async def save_as_script(request): data = await request.json() name = data["name"] workflow = data["workflow"] + frontend_workflow = data.get("frontend_workflow") sio = StringIO() - ComfyUItoPython(workflow=workflow, output_file=sio) + ComfyUItoPython( + workflow=workflow, + frontend_workflow=frontend_workflow, + output_file=sio, + ) sio.seek(0) data = sio.read() diff --git a/comfyui_to_python.py b/comfyui_to_python.py index 201ef8c..df08184 100644 --- a/comfyui_to_python.py +++ b/comfyui_to_python.py @@ -198,6 +198,7 @@ class CodeGenerator: self, load_order: List, workflow_data: Dict, + metadata_workflow_data: Dict | None = None, queue_size: int = 10, ) -> str: """Generate the execution code based on the load order. @@ -314,6 +315,7 @@ class CodeGenerator: special_functions_code, code, workflow_data, + metadata_workflow_data or workflow_data, queue_size, custom_nodes, ) @@ -384,6 +386,7 @@ class CodeGenerator: speical_functions_code: List[str], code: List[str], workflow_data: Dict, + metadata_workflow_data: Dict, queue_size: int, custom_nodes=False, ) -> str: @@ -393,7 +396,8 @@ class CodeGenerator: import_statements (Dict[str, set]): Import statements grouped by module. speical_functions_code (List[str]): A list of special functions code strings. code (List[str]): A list of code strings. - workflow_data (Dict): The original workflow data for metadata preservation. + workflow_data (Dict): The API workflow data used for runtime prompt execution. + metadata_workflow_data (Dict): The workflow metadata to embed into saved outputs. queue_size (int): Number of photos that will be generated by the script. custom_nodes (bool): Whether to include custom nodes in the code. @@ -427,9 +431,9 @@ class CodeGenerator: "prompt = json.loads(json.dumps(workflow))", ( "extra_pnginfo = " - '{"workflow": {"source": "workflow_api", "payload": json.loads(' - + json.dumps(json.dumps(workflow_data)) - + ")}}" + '{"workflow": json.loads(' + + json.dumps(json.dumps(metadata_workflow_data)) + + ")}" ), ] ) @@ -572,6 +576,7 @@ class ComfyUItoPython: def __init__( self, workflow: str = "", + frontend_workflow: str | Dict | None = None, input_file: str = "", output_file: str | TextIO = "", queue_size: int = 1, @@ -597,6 +602,7 @@ class ComfyUItoPython: raise ValueError("Needs output_file") self.workflow = workflow + self.frontend_workflow = frontend_workflow self.input_file = input_file self.output_file = output_file self.queue_size = queue_size @@ -622,6 +628,13 @@ class ComfyUItoPython: else: data = json.loads(self.workflow) + frontend_workflow_data = None + if self.frontend_workflow: + if isinstance(self.frontend_workflow, str): + frontend_workflow_data = json.loads(self.frontend_workflow) + else: + frontend_workflow_data = self.frontend_workflow + # Step 2: Initialize extra/custom nodes when requested or when the workflow references # a node class that is not currently loaded in the runtime. missing_node_types = { @@ -642,7 +655,10 @@ class ComfyUItoPython: self.node_class_mappings, self.base_node_class_mappings ) generated_code = code_generator.generate_workflow( - load_order, data, queue_size=self.queue_size + load_order, + data, + frontend_workflow_data, + queue_size=self.queue_size, ) # Step 5: Write the generated code to a file diff --git a/js/save-as-script.js b/js/save-as-script.js index 91a1a84..d09a0a3 100644 --- a/js/save-as-script.js +++ b/js/save-as-script.js @@ -29,7 +29,11 @@ const extension = { } app.graphToPrompt().then(async (p) => { - const json = JSON.stringify({name: filename + ".json", workflow: JSON.stringify(p.output, null, 2)}, null, 2); // convert the data to a JSON string + const json = JSON.stringify({ + name: filename + ".json", + workflow: JSON.stringify(p.output, null, 2), + frontend_workflow: JSON.stringify(app.graph.serialize(), null, 2), + }, null, 2); // convert the data to a JSON string var response = await api.fetchApi(`/saveasscript`, { method: "POST", body: json }); if(response.status == 200) { const blob = new Blob([await response.text()], {type: "text/python;charset=utf-8"}); diff --git a/tests/test_upscale_model_loader_export.py b/tests/test_upscale_model_loader_export.py index 754c69a..656a3e8 100644 --- a/tests/test_upscale_model_loader_export.py +++ b/tests/test_upscale_model_loader_export.py @@ -91,19 +91,57 @@ class UpscaleModelLoaderExportTest(unittest.TestCase): generated = output.getvalue() - self.assertIn("from nodes import (", generated) + self.assertIn("from nodes import NODE_CLASS_MAPPINGS", generated) + self.assertIn("from tests.test_upscale_model_loader_export import (", generated) self.assertIn("LoadImage,", generated) self.assertIn("UpscaleModelLoader,", generated) self.assertIn("ImageUpscaleWithModel,", generated) - self.assertIn("NODE_CLASS_MAPPINGS,", generated) self.assertIn("loadimage = LoadImage()", generated) self.assertIn("upscalemodelloader = UpscaleModelLoader()", generated) self.assertIn( - "imageupscalewithmodel_3 = imageupscalewithmodel.upscale(", + "imageupscalewithmodel__3 = imageupscalewithmodel.upscale(", generated, ) self.assertNotIn('NODE_CLASS_MAPPINGS["UpscaleModelLoader"]()', generated) + def test_frontend_workflow_metadata_is_embedded_for_reimport(self): + workflow = { + "1": { + "class_type": "LoadImage", + "inputs": { + "image": "example.png", + }, + } + } + frontend_workflow = { + "version": 0.4, + "last_node_id": 1, + "last_link_id": 0, + "nodes": [], + "links": [], + "groups": [], + "config": {}, + "extra": {}, + } + + output = StringIO() + ComfyUItoPython( + workflow=json.dumps(workflow), + frontend_workflow=json.dumps(frontend_workflow), + output_file=output, + node_class_mappings={ + "LoadImage": LoadImage, + }, + ) + + generated = output.getvalue() + + self.assertIn('extra_pnginfo = {', generated) + self.assertIn('"workflow": json.loads(', generated) + self.assertIn('"version": 0.4', generated) + self.assertIn('"nodes": []', generated) + self.assertNotIn('"source": "workflow_api"', generated) + if __name__ == "__main__": unittest.main() From 68ae24ceac99a7127fbcbe94102adf1e8cfcec72 Mon Sep 17 00:00:00 2001 From: Peyton DeNiro <25550995+pydn@users.noreply.github.com> Date: Sat, 28 Mar 2026 13:49:35 -0500 Subject: [PATCH 04/17] Use frontend workflow from graphToPrompt --- js/save-as-script.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/js/save-as-script.js b/js/save-as-script.js index d09a0a3..e0db883 100644 --- a/js/save-as-script.js +++ b/js/save-as-script.js @@ -29,10 +29,11 @@ const extension = { } app.graphToPrompt().then(async (p) => { + const frontendWorkflow = p.workflow ?? app.graph.serialize(); const json = JSON.stringify({ name: filename + ".json", workflow: JSON.stringify(p.output, null, 2), - frontend_workflow: JSON.stringify(app.graph.serialize(), null, 2), + frontend_workflow: JSON.stringify(frontendWorkflow, null, 2), }, null, 2); // convert the data to a JSON string var response = await api.fetchApi(`/saveasscript`, { method: "POST", body: json }); if(response.status == 200) { From a97311dabe0259e93f19de3db80875945c2467f0 Mon Sep 17 00:00:00 2001 From: Peyton DeNiro <25550995+pydn@users.noreply.github.com> Date: Sat, 28 Mar 2026 13:58:56 -0500 Subject: [PATCH 05/17] Add CLI support for frontend workflow metadata --- README.md | 6 +++ comfyui_to_python.py | 14 +++++ tests/test_upscale_model_loader_export.py | 64 ++++++++++++++++++++++- 3 files changed, 83 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 32afd50..1b8c09c 100644 --- a/README.md +++ b/README.md @@ -87,12 +87,14 @@ Options: ```bash uv run python comfyui_to_python.py \ --input_file workflow_api.json \ + --frontend_workflow_file workflow.json \ --output_file workflow_api.py \ --queue_size 10 ``` Flags: - `--input_file`: input workflow JSON, default `workflow_api.json` +- `--frontend_workflow_file`: optional full frontend workflow JSON for reimportable PNG metadata - `--output_file`: output Python file, default `workflow_api.py` - `--queue_size`: default execution count in the generated script, default `10` @@ -110,6 +112,10 @@ export COMFYUI_PATH=/path/to/ComfyUI The generated script is a workflow export. It does not automatically turn workflow inputs into command-line arguments. +If you want generated images to drag-drop back into ComfyUI from the CLI flow, provide both: +- the API workflow JSON via `--input_file` +- the full frontend workflow JSON via `--frontend_workflow_file` + ## Troubleshooting - `Save As Script` not visible: diff --git a/comfyui_to_python.py b/comfyui_to_python.py index df08184..6eec402 100644 --- a/comfyui_to_python.py +++ b/comfyui_to_python.py @@ -671,6 +671,7 @@ def run( input_file: str = DEFAULT_INPUT_FILE, output_file: str = DEFAULT_OUTPUT_FILE, queue_size: int = DEFAULT_QUEUE_SIZE, + frontend_workflow_file: str = "", ) -> None: """Generate Python code from a ComfyUI workflow_api.json file. @@ -680,12 +681,19 @@ def run( Defaults to "workflow_api.py". queue_size (int): The number of times a workflow will be executed by the script. Defaults to 1. + frontend_workflow_file (str): Optional path to a full frontend workflow JSON file. + When provided, generated images can preserve reimportable ComfyUI workflow metadata. Returns: None """ ComfyUItoPython( input_file=input_file, + frontend_workflow=( + FileHandler.read_json_file(frontend_workflow_file) + if frontend_workflow_file + else None + ), output_file=output_file, queue_size=queue_size, needs_init_custom_nodes=True, @@ -718,6 +726,12 @@ def main() -> None: help="number of times the workflow will be executed by default", default=DEFAULT_QUEUE_SIZE, ) + parser.add_argument( + "--frontend_workflow_file", + type=str, + help="optional path to a full frontend workflow JSON file for reimportable PNG metadata", + default="", + ) pargs = parser.parse_args() run(**vars(pargs)) print("Done.") diff --git a/tests/test_upscale_model_loader_export.py b/tests/test_upscale_model_loader_export.py index 656a3e8..7beed03 100644 --- a/tests/test_upscale_model_loader_export.py +++ b/tests/test_upscale_model_loader_export.py @@ -1,8 +1,11 @@ import json +import tempfile import unittest from io import StringIO +from pathlib import Path +from unittest.mock import patch -from comfyui_to_python import ComfyUItoPython +from comfyui_to_python import ComfyUItoPython, run class UpscaleModelLoader: @@ -142,6 +145,65 @@ class UpscaleModelLoaderExportTest(unittest.TestCase): self.assertIn('"nodes": []', generated) self.assertNotIn('"source": "workflow_api"', generated) + def test_run_accepts_frontend_workflow_file_for_cli_metadata(self): + workflow = { + "1": { + "class_type": "LoadImage", + "inputs": { + "image": "example.png", + }, + } + } + frontend_workflow = { + "version": 0.4, + "last_node_id": 1, + "last_link_id": 0, + "nodes": [ + { + "id": 1, + "type": "LoadImage", + "pos": [0, 0], + "size": [210, 60], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": ["example.png"], + } + ], + "links": [], + "groups": [], + "config": {}, + "extra": {}, + } + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir_path = Path(tmpdir) + input_file = tmpdir_path / "workflow_api.json" + frontend_file = tmpdir_path / "workflow.json" + output_file = tmpdir_path / "workflow_api.py" + + input_file.write_text(json.dumps(workflow), encoding="utf-8") + frontend_file.write_text(json.dumps(frontend_workflow), encoding="utf-8") + + with patch( + "comfyui_to_python.get_node_class_mappings", + return_value={"LoadImage": LoadImage}, + ), patch("comfyui_to_python.import_custom_nodes"): + run( + input_file=str(input_file), + frontend_workflow_file=str(frontend_file), + output_file=str(output_file), + queue_size=1, + ) + + generated = output_file.read_text(encoding="utf-8") + + self.assertIn('"version": 0.4', generated) + self.assertIn('"type": "LoadImage"', generated) + if __name__ == "__main__": unittest.main() From 05984337836ae973c7e3e4adce3349f29bd2d180 Mon Sep 17 00:00:00 2001 From: Peyton DeNiro <25550995+pydn@users.noreply.github.com> Date: Sat, 28 Mar 2026 16:02:40 -0500 Subject: [PATCH 06/17] Remove upstream smoke validation tier --- PR_DESCRIPTION.md | 35 +++ docs/runtime-validation.md | 99 +++++++ docs/specs/runtime-e2e-agent-spec.md | 128 ++++++++++ .../runtime-validation-environment-spec.md | 136 ++++++++++ .../runtime-validation-implementation-spec.md | 241 ++++++++++++++++++ tests/runtime/run_runtime_validation.py | 17 +- 6 files changed, 645 insertions(+), 11 deletions(-) create mode 100644 PR_DESCRIPTION.md create mode 100644 docs/runtime-validation.md create mode 100644 docs/specs/runtime-e2e-agent-spec.md create mode 100644 docs/specs/runtime-validation-environment-spec.md create mode 100644 docs/specs/runtime-validation-implementation-spec.md diff --git a/PR_DESCRIPTION.md b/PR_DESCRIPTION.md new file mode 100644 index 0000000..674d348 --- /dev/null +++ b/PR_DESCRIPTION.md @@ -0,0 +1,35 @@ +## Summary + +This PR adds a runtime validation path for exported workflows so exporter changes can be checked at two levels: + +- fast exporter-only validation +- runtime validation against a pinned local ComfyUI checkout + +It includes committed fixtures for: +- `upscale-model-loader` +- `text-to-image` +- `unsafe-kwargs` +- `subgraph-identifiers` + +It also updates the exporter so generated scripts preserve workflow metadata and hidden runtime inputs needed for end-to-end execution. + +## Included + +- runtime validation harness in `tests/runtime/run_runtime_validation.py` +- runtime fixtures in `tests/fixtures/runtime/` +- exporter support changes in `comfyui_to_python.py` + +## Validation + +Planned validation commands: + +```bash +uv run python tests/runtime/run_runtime_validation.py --tier fast --fixture all +uv run python tests/runtime/run_runtime_validation.py --tier runtime --fixture all +``` + +Runtime-tier failures are classified as: +- `repo regression` +- `model provisioning failure` +- `fixture bug` +- `environment/setup failure` diff --git a/docs/runtime-validation.md b/docs/runtime-validation.md new file mode 100644 index 0000000..538712f --- /dev/null +++ b/docs/runtime-validation.md @@ -0,0 +1,99 @@ +# Runtime Validation + +This repo has two validation tiers: + +- `fast` +- `runtime` + +Use `uv` for repo commands. + +## Local Runtime Paths + +Pinned ComfyUI checkout: + +- `/home/peyton/code/ComfyUI` + +The runtime harness uses `COMFYUI_PATH` to point generated scripts and export logic at the target ComfyUI checkout. +For full E2E runtime execution, the harness prefers the ComfyUI checkout's own Python interpreter at `/home/peyton/code/ComfyUI/.venv/bin/python` when available. + +## Fixtures + +Committed runtime fixtures live in: + +- [tests/fixtures/runtime](/home/peyton/code/ComfyUI-to-Python-Extension/tests/fixtures/runtime) + +Current fixtures: + +- `upscale-model-loader` +- `text-to-image` +- `unsafe-kwargs` +- `subgraph-identifiers` + +Runtime-tier fixtures are based on official core ComfyUI workflow shapes: + +- the Image Upscale workflow +- the Text to Image workflow + +Sources: + +- https://docs.comfy.org/tutorials/basic/upscale +- https://docs.comfy.org/tutorials/basic/text-to-image + +## Commands + +### Fast + +Run exporter-only validation without a real ComfyUI runtime: + +```bash +uv run python tests/runtime/run_runtime_validation.py --tier fast --fixture all +``` + +### Runtime + +Run runtime-capable validation against the pinned baseline checkout. This now performs full E2E execution by default: + +```bash +uv run python tests/runtime/run_runtime_validation.py --tier runtime --fixture upscale-model-loader +``` + +Run both official runtime fixtures: + +```bash +uv run python tests/runtime/run_runtime_validation.py --tier runtime --fixture all +``` + +### Model Provisioning + +If a runtime fixture is missing required models, the harness will report `model provisioning failure`. + +To print the required download commands without executing them: + +```bash +uv run python tests/runtime/run_runtime_validation.py --tier runtime --fixture all --print-download-plan +``` + +Model downloads should be approval-gated and should target the existing `/home/peyton/code/ComfyUI/models/...` directories. + +## Failure Classification + +The runtime harness classifies failures as: + +- `repo regression` +- `model provisioning failure` +- `fixture bug` +- `environment/setup failure` + +This classification matters more than raw pass/fail. A failed run without classification is not actionable enough for maintenance work. + +## Notes + +- `unsafe-kwargs` and `subgraph-identifiers` are committed fast-tier regression fixtures. They are not runtime-tier fixtures. +- `upscale-model-loader` and `text-to-image` are the current runtime-tier fixtures and are based on official ComfyUI tutorials. +- Runtime-tier success now means: + - generated Python parses + - generated script executes + - a new output file is created + - output dimensions meet expectations + - output metadata contains expected markers +- `Save As Script` route validation is still phase 2 work. diff --git a/docs/specs/runtime-e2e-agent-spec.md b/docs/specs/runtime-e2e-agent-spec.md new file mode 100644 index 0000000..fb49060 --- /dev/null +++ b/docs/specs/runtime-e2e-agent-spec.md @@ -0,0 +1,128 @@ +# Runtime E2E Agent Spec + +## Purpose + +Own end-to-end runtime validation for this repo against a real ComfyUI runtime, using a pinned validation target and a small representative workflow corpus. + +## Use When + +- a change affects runtime/bootstrap behavior +- a merge recommendation needs stronger evidence than exporter-only tests +- a maintainer wants a runtime sanity check before or after a hotfix merge + +## Required Inputs + +- approved spec or target branch under review +- pinned ComfyUI runtime target +- runtime validation workflow corpus +- required runtime dependencies in the validation environment +- current repo worktree or PR branch + +## Expected Outputs + +- a short executive summary first +- a structured validation report with: + - Inputs + - Checks Performed + - Findings + - Fixture or Environment Issues + - Coverage Gaps + - Merge Recommendation + - Next Action + +## Reporting Rules + +- respect the maintainer's time +- keep the report decision-oriented +- distinguish repo regression from harness failure +- do not dump raw logs unless a short excerpt is needed to explain a failure + +## Validation Tiers + +### `fast` + +Use exporter-only checks that do not require a full ComfyUI runtime. This tier is not owned exclusively by this agent, but the agent should understand when `fast` is enough and when runtime validation is required. + +### `runtime` + +Run representative generated-script validations against a pinned local ComfyUI checkout with required runtime dependencies available. + +## Responsibilities + +- maintain awareness of the pinned ComfyUI runtime target +- run the `runtime` tier for runtime-sensitive changes +- classify failures into: + - repo bug + - fixture bug + - environment/setup failure +- provide a concise merge recommendation: + - `pass` + - `pass with risk` + - `fail` + +## Initial Workflow Corpus + +The first validation corpus should cover: + +- upscale loader workflow for `#143` +- unsafe-kwargs workflow for `#119` / `#96` +- subgraph identifier workflow for `#128` + +Future additions may include: + +- frontend save-flow validation fixtures +- runtime performance/bootstrap parity fixtures for issues like `#147` + +## Suggested Cadence + +- on demand for runtime-sensitive branches +- before merge for runtime/bootstrap fixes + +## Handoff Rules + +- report merge decisions to `maintainer-orchestrator` +- send runtime regressions to `runtime-compat-agent` +- send exporter-only failures back to `verification-agent` or `exporter-bug-agent` +- send upstream harness breakage to `upstream-watch-agent` + +## Branch and PR Conventions + +- validation results should reference the exact working branch and, when present, the PR title +- runtime-only harness changes should use focused branch names such as: + - `codex/runtime-e2e-` + - `codex/issue--runtime-e2e` + +## Ready-to-Paste Prompt + +```text +You are the runtime-e2e-agent for the ComfyUI-to-Python-Extension repo. + +Your job is to validate runtime-sensitive changes against a real ComfyUI runtime using a pinned validation target and a representative workflow corpus. Respect the maintainer's time. Be concise, strict, and decision-oriented. Do not write long test diaries. + +Repo rules: +- use uv for repo commands +- preserve a clear separation between exporter-only checks and real runtime validation +- classify failures accurately: repo bug, fixture bug, or environment failure +- do not call something "pass" if the runtime path was not actually exercised + +When you respond: +- Start with Executive Summary in 3 to 6 short lines. +- Then use: + - Inputs + - Checks Performed + - Findings + - Fixture or Environment Issues + - Coverage Gaps + - Merge Recommendation + - Next Action + +Recommendation labels: +- pass +- pass with risk +- fail + +Validation rules: +- run the pinned runtime suite when the change affects runtime/bootstrap behavior +- keep logs summarized unless a short excerpt is necessary +- make it obvious whether a failure belongs to this repo or the validation harness +``` diff --git a/docs/specs/runtime-validation-environment-spec.md b/docs/specs/runtime-validation-environment-spec.md new file mode 100644 index 0000000..8f56836 --- /dev/null +++ b/docs/specs/runtime-validation-environment-spec.md @@ -0,0 +1,136 @@ +# Runtime Validation Environment Spec + +## Purpose + +Define the environment, fixture layout, and execution contract for end-to-end runtime validation of `ComfyUI-to-Python-Extension`. + +## Problem + +Exporter-only regression tests are necessary but insufficient for runtime/bootstrap-sensitive changes. Local verification becomes unreliable when tests depend on an arbitrary ComfyUI installation or on environment details that are not controlled. The repo needs a repeatable runtime validation harness. + +## Goal + +Create a stable runtime validation setup that: + +- uses a pinned ComfyUI checkout +- validates generated scripts against a real runtime +- uses a small set of representative workflow fixtures +- clearly separates repo regressions from environment problems + +## Non-Goals + +- no image-quality benchmarking +- no full browser automation in the first phase +- no external CI/CD setup in this phase +- no attempt to cover every ComfyUI node or workflow shape + +## Validation Model + +### Tier 1: `fast` + +Purpose: + +- run quickly during local development +- validate exporter behavior without requiring a real ComfyUI runtime + +Typical checks: + +- focused unit or regression tests with injected mappings +- generated Python parse checks +- syntax compilation + +### Tier 2: `runtime` + +Purpose: + +- validate generated scripts against a real pinned ComfyUI runtime + +Typical checks: + +- export from API-format workflow to Python +- parse/import generated Python +- execute generated script through the targeted runtime path +- confirm expected bootstrap and node-loading behavior + +## Environment Requirements + +- a local ComfyUI checkout available through a documented path convention +- the ability to point this repo to that checkout, preferably with `COMFYUI_PATH` +- required runtime dependencies installed in the validation environment, including `torch` +- all repo command execution through `uv` + +## Proposed Path Convention + +The validation harness should assume: + +- this repo remains where it is +- the pinned ComfyUI checkout lives at a maintainer-chosen local path +- the runtime command surface uses `COMFYUI_PATH=/path/to/ComfyUI` + +The exact checkout location does not need to be committed into this repo, but the contract for supplying it must be documented and stable. + +## Workflow Fixture Corpus + +The first committed or documented fixture set should cover: + +- `upscale-model-loader` + for `#143`-style runtime/bootstrap validation +- `unsafe-kwargs` + for `#119` / `#96`-style generated-call validation against real workflows +- `subgraph-identifiers` + for `#128`-style identifier safety + +Fixture principles: + +- prefer real workflow shapes from official ComfyUI examples when possible +- keep fixtures minimal but representative +- name fixtures after behavior, not issue numbers, when practical + +## Assertions + +The first phase should assert: + +- export succeeds +- generated Python parses +- generated script can execute far enough to validate the targeted runtime path +- expected code patterns are present or absent when relevant + +The first phase should not require: + +- image-diff assertions +- performance benchmarking beyond obvious bootstrap regressions +- full frontend automation + +## Failure Classification + +Every runtime validation failure should be classified as one of: + +- repo regression +- fixture bug +- environment/setup failure + +This classification is mandatory because raw pass/fail is not enough for a maintenance workflow. + +## Likely Future Deliverables + +When implementation is approved, the runtime validation system will likely need: + +- a fixtures directory for runtime workflows +- a short operator document for setting `COMFYUI_PATH` +- one or more helper commands or scripts for running: + - `fast` + - `runtime` +- a `runtime-e2e-agent` config file under `.codex/agents/` + +## Acceptance Criteria + +- runtime validation is defined around a pinned ComfyUI target +- the initial workflow corpus covers the main known runtime-sensitive issue classes +- validation tiers are clearly separated +- pass/fail reporting includes failure classification +- the setup is small enough to maintain manually inside Codex/chat workflows + +## Open Questions + +- should the workflow corpus be committed into this repo or referenced from a local external fixture path? +- should the first implementation cover only CLI/runtime execution, or also a later `Save As Script` route check? diff --git a/docs/specs/runtime-validation-implementation-spec.md b/docs/specs/runtime-validation-implementation-spec.md new file mode 100644 index 0000000..deed264 --- /dev/null +++ b/docs/specs/runtime-validation-implementation-spec.md @@ -0,0 +1,241 @@ +# Runtime Validation Implementation Spec + +## Document Status + +- Date: March 28, 2026 +- Scope: implementation spec only +- Approval status: approved for spec creation, not yet approved for implementation + +## Problem + +The repo now has a runtime validation strategy, but not an implementation contract. Runtime-sensitive changes still rely too heavily on ad hoc local setups and narrow regression tests. The repo needs a small, repeatable runtime validation harness that agents and maintainers can run consistently. + +## Intent + +Implement a phase-based runtime validation harness and add a project-scoped `runtime-e2e-agent` that can operate against it. + +## Locked Decisions + +- Baseline pinned ComfyUI checkout: `/home/peyton/code/ComfyUI` +- Runtime workflow fixtures will be committed in this repo +- Implementation covers both CLI/runtime and `Save As Script`, but in phases +- Phase 1: CLI/runtime first +- Phase 2: `Save As Script` route validation + +## Goals + +- create a repeatable runtime validation harness +- keep the harness small enough to maintain manually +- make the workflow corpus reviewable and committed in-repo +- support pinned-runtime validation with a stable command surface +- give the future `runtime-e2e-agent` a stable command surface + +## Non-Goals + +- no full browser automation in phase 1 +- no image-quality regression testing +- no external CI/CD setup in this phase +- no vendoring of the ComfyUI repo into this repo +- no attempt to validate every possible custom-node workflow + +## Phase Plan + +### Phase 1: CLI / Runtime Harness + +Phase 1 must provide: + +- in-repo runtime workflow fixtures +- a command path that exports workflow JSON to Python +- a command path that validates generated Python parses +- a command path that executes generated Python against a pinned ComfyUI runtime +- clear pass/fail classification + +This phase should be enough to validate: + +- runtime/bootstrap regressions like `#143` +- generated-code validity issues when they must be checked against a real runtime +- runtime/bootstrap changes that affect node-loading or script execution + +### Phase 2: `Save As Script` Route Validation + +Phase 2 must build on the phase 1 harness and add: + +- route-level validation of the `Save As Script` request path +- coverage for server-side export behavior in [__init__.py](/home/peyton/code/ComfyUI-to-Python-Extension/__init__.py) +- coverage for the exported payload shape expected by the frontend flow + +Phase 2 should not start with full browser automation unless route-level checks prove insufficient. + +## Directory Layout + +The first implementation should add: + +- `tests/fixtures/runtime/` +- `tests/fixtures/runtime/upscale-model-loader.json` +- `tests/fixtures/runtime/unsafe-kwargs.json` +- `tests/fixtures/runtime/subgraph-identifiers.json` +- `tests/runtime/` +- a focused runtime validation runner or helper module inside `tests/runtime/` +- `.codex/agents/runtime-e2e-agent.toml` + +Optional supporting docs: + +- `docs/runtime-validation.md` + +## Fixture Rules + +- prefer real ComfyUI workflow shapes over synthetic-only fixtures +- keep fixtures as small as possible while still reproducing the target behavior +- name fixtures after behavior, not issue numbers, when practical +- commit fixtures to the repo so they are reviewable and stable + +## Initial Fixture Set + +### `upscale-model-loader.json` + +Purpose: + +- validate the `#143`/`#148` runtime/bootstrap path + +Assertions: + +- export succeeds +- generated Python parses +- generated script reaches the relevant loader and upscale path without the old missing-node failure + +### `unsafe-kwargs.json` + +Purpose: + +- validate mixed safe and unsafe keys in a real workflow shape + +Assertions: + +- export succeeds +- generated Python parses +- generated code remains safe when real workflow data contains symbol-heavy or invalid keys + +### `subgraph-identifiers.json` + +Purpose: + +- validate that subgraph-expanded identifiers do not break generated Python in real workflow shapes + +Assertions: + +- export succeeds +- generated Python parses +- generated variable names are safe enough for execution + +## Command Contract + +The harness should expose two stable command categories. + +### `fast` + +Purpose: + +- exporter-only validation without a real ComfyUI runtime + +Examples: + +- `uv run python -m unittest tests.test_upscale_model_loader_export -v` +- additional focused exporter tests as they are added + +### `runtime` + +Purpose: + +- run runtime validation against the pinned baseline ComfyUI checkout + +Environment: + +- `COMFYUI_PATH=/home/peyton/code/ComfyUI` + +Behavior: + +- export fixture workflow to Python +- parse or compile generated Python +- execute generated Python with the pinned runtime available + +## Runner Shape + +The implementation should prefer one small runner over many scattered commands. + +Recommended shape: + +- a small Python entry point that: + - selects a tier + - selects a fixture or fixture subset + - sets or reads `COMFYUI_PATH` + - runs export + - validates generated Python + - optionally executes the generated script + - reports failure classification + +The runner must stay small and repo-local. It should not introduce a large new framework. + +## Failure Classification Contract + +Every runtime validation result must classify failure as one of: + +- repo regression +- fixture bug +- environment/setup failure + +This classification is mandatory in both manual runs and future agent reports. + +## Runtime-E2E Agent Addition + +A future implementation step should add: + +- `.codex/agents/runtime-e2e-agent.toml` + +Its instructions should align with: + +- [runtime-e2e-agent-spec.md](/home/peyton/code/ComfyUI-to-Python-Extension/docs/specs/runtime-e2e-agent-spec.md) +- [runtime-validation-environment-spec.md](/home/peyton/code/ComfyUI-to-Python-Extension/docs/specs/runtime-validation-environment-spec.md) + +The agent should: + +- run the `runtime` tier +- report concise results +- avoid long logs +- return `pass`, `pass with risk`, or `fail` + +## Validation Expectations + +Phase 1 success means: + +- the pinned runtime can validate at least the three initial fixtures +- the runner can distinguish export failures from runtime execution failures +- the runtime harness can be invoked consistently by maintainers and agents + +Phase 2 success means: + +- route-level `Save As Script` validation exists +- frontend/server export behavior can be checked without relying on a full browser automation stack + +## Risks + +- fixture maintenance can drift if fixtures are too complex +- local ComfyUI checkout state can become stale if not periodically refreshed intentionally +- runtime validation may still fail for environment reasons if `torch` or ComfyUI dependencies are missing + +## Acceptance Criteria + +- the implementation plan is explicit enough to build without inventing workflow shape later +- fixture names and locations are defined +- command tiers are defined +- phase split is defined +- failure classification is mandatory +- the `runtime-e2e-agent` rollout target is defined + +## Next Step + +If this implementation spec is approved, the next step is to produce a concrete implementation plan and then request explicit approval before creating: + +- runtime fixture files +- the runner/helper module +- the `.codex/agents/runtime-e2e-agent.toml` file +- the short operator doc diff --git a/tests/runtime/run_runtime_validation.py b/tests/runtime/run_runtime_validation.py index 6312449..eed5397 100644 --- a/tests/runtime/run_runtime_validation.py +++ b/tests/runtime/run_runtime_validation.py @@ -18,7 +18,6 @@ ROOT = Path(__file__).resolve().parents[2] FIXTURE_DIR = ROOT / "tests" / "fixtures" / "runtime" GENERATED_DIR = ROOT / "tests" / "runtime" / "generated" BASELINE_COMFYUI_PATH = Path("/home/peyton/code/ComfyUI") -UPSTREAM_SMOKE_COMFYUI_PATH = Path("/home/peyton/code/ComfyUI-upstream-smoke") COMFYUI_OUTPUT_DIRNAME = "output" COMFYUI_INPUT_DIRNAME = "input" @@ -237,11 +236,11 @@ FIXTURES = { def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( - description="Run fast, runtime, or upstream-smoke validation for committed fixtures." + description="Run fast or runtime validation for committed fixtures." ) parser.add_argument( "--tier", - choices=("fast", "runtime", "upstream-smoke"), + choices=("fast", "runtime"), help="Validation tier to run.", ) parser.add_argument( @@ -277,8 +276,6 @@ def parse_args() -> argparse.Namespace: def load_fixture_names(selection: str) -> list[str]: if selection == "all": return list(FIXTURES.keys()) - if selection == "smoke": - return ["upscale-model-loader", "text-to-image"] if selection not in FIXTURES: raise ValidationFailure("fixture bug", f"Unknown fixture '{selection}'.") return [selection] @@ -287,8 +284,6 @@ def load_fixture_names(selection: str) -> list[str]: def ensure_runtime_path(tier: str) -> str: if tier == "runtime": runtime_path = BASELINE_COMFYUI_PATH - elif tier == "upstream-smoke": - runtime_path = UPSTREAM_SMOKE_COMFYUI_PATH else: return os.environ.get("COMFYUI_PATH", "") @@ -596,8 +591,8 @@ def run_fixture(fixture: FixtureConfig, tier: str, execute: bool, runtime_path: generated_code = export_workflow_in_runtime_env(fixture, runtime_path) validate_generated_python(generated_code, fixture.name) - should_execute = execute or tier in {"runtime", "upstream-smoke"} - if should_execute and tier in {"runtime", "upstream-smoke"}: + should_execute = execute or tier == "runtime" + if should_execute and tier == "runtime": execute_generated_python(generated_code, fixture, runtime_path) return "pass" @@ -634,7 +629,7 @@ def main() -> int: "fixture bug", "No selected fixtures are fast-tier compatible.", ) - elif args.tier in {"runtime", "upstream-smoke"}: + elif args.tier == "runtime": requested = [fixture for fixture in requested if fixture.runtime_capable] if not requested: raise ValidationFailure( @@ -645,7 +640,7 @@ def main() -> int: failures: list[tuple[str, str, str]] = [] for fixture in requested: try: - if args.print_download_plan and args.tier in {"runtime", "upstream-smoke"}: + if args.print_download_plan and args.tier == "runtime": print_download_plan(fixture, runtime_path) status = run_fixture(fixture, args.tier, args.execute, runtime_path) print(f"{fixture.name}: {status}") From b0109aa90f6b026fe470ee507c6f94235cc5a635 Mon Sep 17 00:00:00 2001 From: Peyton DeNiro <25550995+pydn@users.noreply.github.com> Date: Sat, 28 Mar 2026 16:09:13 -0500 Subject: [PATCH 07/17] Clarify runtime validation harness behavior --- tests/runtime/run_runtime_validation.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/runtime/run_runtime_validation.py b/tests/runtime/run_runtime_validation.py index eed5397..6313a0e 100644 --- a/tests/runtime/run_runtime_validation.py +++ b/tests/runtime/run_runtime_validation.py @@ -176,6 +176,9 @@ class SaveImageNode: FIXTURES = { + # Runtime-capable fixtures are exported inside a real ComfyUI checkout and + # then executed end-to-end. Fast fixtures use local stub mappings so the + # exporter can be validated without importing the full runtime. "upscale-model-loader": FixtureConfig( name="upscale-model-loader", path=FIXTURE_DIR / "upscale-model-loader.json", @@ -393,6 +396,8 @@ def export_workflow_in_runtime_env(fixture: FixtureConfig, runtime_path: str) -> env["PYTHONPATH"] = os.pathsep.join([str(ROOT), env.get("PYTHONPATH", "")]).rstrip( os.pathsep ) + # Re-enter this script under the runtime interpreter so export happens with + # the target ComfyUI checkout on sys.path, not just the repo's current venv. result = subprocess.run( [ runtime_python, @@ -487,6 +492,8 @@ def validate_output_artifact( fixture: FixtureConfig, output_path: Path, ) -> None: + # Read PNG metadata directly so artifact validation does not depend on + # optional imaging libraries inside the runtime environment. width, height, metadata = parse_png_info(output_path) if fixture.expected_min_dimensions is not None: @@ -523,6 +530,8 @@ def execute_generated_python( tmp_path = Path(tmpdir) / f"{fixture.name}.py" tmp_path.write_text(generated_code, encoding="utf-8") output_dir = Path(runtime_path) / COMFYUI_OUTPUT_DIRNAME + # Compare against the pre-run snapshot so validation can prove this + # execution created a fresh artifact instead of reusing an old output. existing_outputs = set(output_dir.glob("*.png")) env = os.environ.copy() env["COMFYUI_PATH"] = runtime_path @@ -591,6 +600,8 @@ def run_fixture(fixture: FixtureConfig, tier: str, execute: bool, runtime_path: generated_code = export_workflow_in_runtime_env(fixture, runtime_path) validate_generated_python(generated_code, fixture.name) + # Fast keeps execution opt-in because its stub node mappings are intended for + # export coverage only. Runtime always executes the generated script. should_execute = execute or tier == "runtime" if should_execute and tier == "runtime": execute_generated_python(generated_code, fixture, runtime_path) @@ -641,7 +652,16 @@ def main() -> int: for fixture in requested: try: if args.print_download_plan and args.tier == "runtime": - print_download_plan(fixture, runtime_path) + missing_models = check_models(fixture, runtime_path) + for requirement in missing_models: + target_dir = Path(runtime_path) / requirement.relative_dir + print( + "download:", + f"mkdir -p {target_dir} && curl -L {requirement.source_url} -o {target_dir / requirement.filename}", + ) + if missing_models: + print(f"{fixture.name}: download-plan") + continue status = run_fixture(fixture, args.tier, args.execute, runtime_path) print(f"{fixture.name}: {status}") except ValidationFailure as exc: From d57b2167ca2f3fe4f1891f49c6d6e4a359b2dd24 Mon Sep 17 00:00:00 2001 From: Peyton DeNiro <25550995+pydn@users.noreply.github.com> Date: Sat, 28 Mar 2026 16:12:05 -0500 Subject: [PATCH 08/17] Remove tracked runtime spec docs from branch --- docs/specs/runtime-e2e-agent-spec.md | 128 ---------- .../runtime-validation-environment-spec.md | 136 ---------- .../runtime-validation-implementation-spec.md | 241 ------------------ 3 files changed, 505 deletions(-) delete mode 100644 docs/specs/runtime-e2e-agent-spec.md delete mode 100644 docs/specs/runtime-validation-environment-spec.md delete mode 100644 docs/specs/runtime-validation-implementation-spec.md diff --git a/docs/specs/runtime-e2e-agent-spec.md b/docs/specs/runtime-e2e-agent-spec.md deleted file mode 100644 index fb49060..0000000 --- a/docs/specs/runtime-e2e-agent-spec.md +++ /dev/null @@ -1,128 +0,0 @@ -# Runtime E2E Agent Spec - -## Purpose - -Own end-to-end runtime validation for this repo against a real ComfyUI runtime, using a pinned validation target and a small representative workflow corpus. - -## Use When - -- a change affects runtime/bootstrap behavior -- a merge recommendation needs stronger evidence than exporter-only tests -- a maintainer wants a runtime sanity check before or after a hotfix merge - -## Required Inputs - -- approved spec or target branch under review -- pinned ComfyUI runtime target -- runtime validation workflow corpus -- required runtime dependencies in the validation environment -- current repo worktree or PR branch - -## Expected Outputs - -- a short executive summary first -- a structured validation report with: - - Inputs - - Checks Performed - - Findings - - Fixture or Environment Issues - - Coverage Gaps - - Merge Recommendation - - Next Action - -## Reporting Rules - -- respect the maintainer's time -- keep the report decision-oriented -- distinguish repo regression from harness failure -- do not dump raw logs unless a short excerpt is needed to explain a failure - -## Validation Tiers - -### `fast` - -Use exporter-only checks that do not require a full ComfyUI runtime. This tier is not owned exclusively by this agent, but the agent should understand when `fast` is enough and when runtime validation is required. - -### `runtime` - -Run representative generated-script validations against a pinned local ComfyUI checkout with required runtime dependencies available. - -## Responsibilities - -- maintain awareness of the pinned ComfyUI runtime target -- run the `runtime` tier for runtime-sensitive changes -- classify failures into: - - repo bug - - fixture bug - - environment/setup failure -- provide a concise merge recommendation: - - `pass` - - `pass with risk` - - `fail` - -## Initial Workflow Corpus - -The first validation corpus should cover: - -- upscale loader workflow for `#143` -- unsafe-kwargs workflow for `#119` / `#96` -- subgraph identifier workflow for `#128` - -Future additions may include: - -- frontend save-flow validation fixtures -- runtime performance/bootstrap parity fixtures for issues like `#147` - -## Suggested Cadence - -- on demand for runtime-sensitive branches -- before merge for runtime/bootstrap fixes - -## Handoff Rules - -- report merge decisions to `maintainer-orchestrator` -- send runtime regressions to `runtime-compat-agent` -- send exporter-only failures back to `verification-agent` or `exporter-bug-agent` -- send upstream harness breakage to `upstream-watch-agent` - -## Branch and PR Conventions - -- validation results should reference the exact working branch and, when present, the PR title -- runtime-only harness changes should use focused branch names such as: - - `codex/runtime-e2e-` - - `codex/issue--runtime-e2e` - -## Ready-to-Paste Prompt - -```text -You are the runtime-e2e-agent for the ComfyUI-to-Python-Extension repo. - -Your job is to validate runtime-sensitive changes against a real ComfyUI runtime using a pinned validation target and a representative workflow corpus. Respect the maintainer's time. Be concise, strict, and decision-oriented. Do not write long test diaries. - -Repo rules: -- use uv for repo commands -- preserve a clear separation between exporter-only checks and real runtime validation -- classify failures accurately: repo bug, fixture bug, or environment failure -- do not call something "pass" if the runtime path was not actually exercised - -When you respond: -- Start with Executive Summary in 3 to 6 short lines. -- Then use: - - Inputs - - Checks Performed - - Findings - - Fixture or Environment Issues - - Coverage Gaps - - Merge Recommendation - - Next Action - -Recommendation labels: -- pass -- pass with risk -- fail - -Validation rules: -- run the pinned runtime suite when the change affects runtime/bootstrap behavior -- keep logs summarized unless a short excerpt is necessary -- make it obvious whether a failure belongs to this repo or the validation harness -``` diff --git a/docs/specs/runtime-validation-environment-spec.md b/docs/specs/runtime-validation-environment-spec.md deleted file mode 100644 index 8f56836..0000000 --- a/docs/specs/runtime-validation-environment-spec.md +++ /dev/null @@ -1,136 +0,0 @@ -# Runtime Validation Environment Spec - -## Purpose - -Define the environment, fixture layout, and execution contract for end-to-end runtime validation of `ComfyUI-to-Python-Extension`. - -## Problem - -Exporter-only regression tests are necessary but insufficient for runtime/bootstrap-sensitive changes. Local verification becomes unreliable when tests depend on an arbitrary ComfyUI installation or on environment details that are not controlled. The repo needs a repeatable runtime validation harness. - -## Goal - -Create a stable runtime validation setup that: - -- uses a pinned ComfyUI checkout -- validates generated scripts against a real runtime -- uses a small set of representative workflow fixtures -- clearly separates repo regressions from environment problems - -## Non-Goals - -- no image-quality benchmarking -- no full browser automation in the first phase -- no external CI/CD setup in this phase -- no attempt to cover every ComfyUI node or workflow shape - -## Validation Model - -### Tier 1: `fast` - -Purpose: - -- run quickly during local development -- validate exporter behavior without requiring a real ComfyUI runtime - -Typical checks: - -- focused unit or regression tests with injected mappings -- generated Python parse checks -- syntax compilation - -### Tier 2: `runtime` - -Purpose: - -- validate generated scripts against a real pinned ComfyUI runtime - -Typical checks: - -- export from API-format workflow to Python -- parse/import generated Python -- execute generated script through the targeted runtime path -- confirm expected bootstrap and node-loading behavior - -## Environment Requirements - -- a local ComfyUI checkout available through a documented path convention -- the ability to point this repo to that checkout, preferably with `COMFYUI_PATH` -- required runtime dependencies installed in the validation environment, including `torch` -- all repo command execution through `uv` - -## Proposed Path Convention - -The validation harness should assume: - -- this repo remains where it is -- the pinned ComfyUI checkout lives at a maintainer-chosen local path -- the runtime command surface uses `COMFYUI_PATH=/path/to/ComfyUI` - -The exact checkout location does not need to be committed into this repo, but the contract for supplying it must be documented and stable. - -## Workflow Fixture Corpus - -The first committed or documented fixture set should cover: - -- `upscale-model-loader` - for `#143`-style runtime/bootstrap validation -- `unsafe-kwargs` - for `#119` / `#96`-style generated-call validation against real workflows -- `subgraph-identifiers` - for `#128`-style identifier safety - -Fixture principles: - -- prefer real workflow shapes from official ComfyUI examples when possible -- keep fixtures minimal but representative -- name fixtures after behavior, not issue numbers, when practical - -## Assertions - -The first phase should assert: - -- export succeeds -- generated Python parses -- generated script can execute far enough to validate the targeted runtime path -- expected code patterns are present or absent when relevant - -The first phase should not require: - -- image-diff assertions -- performance benchmarking beyond obvious bootstrap regressions -- full frontend automation - -## Failure Classification - -Every runtime validation failure should be classified as one of: - -- repo regression -- fixture bug -- environment/setup failure - -This classification is mandatory because raw pass/fail is not enough for a maintenance workflow. - -## Likely Future Deliverables - -When implementation is approved, the runtime validation system will likely need: - -- a fixtures directory for runtime workflows -- a short operator document for setting `COMFYUI_PATH` -- one or more helper commands or scripts for running: - - `fast` - - `runtime` -- a `runtime-e2e-agent` config file under `.codex/agents/` - -## Acceptance Criteria - -- runtime validation is defined around a pinned ComfyUI target -- the initial workflow corpus covers the main known runtime-sensitive issue classes -- validation tiers are clearly separated -- pass/fail reporting includes failure classification -- the setup is small enough to maintain manually inside Codex/chat workflows - -## Open Questions - -- should the workflow corpus be committed into this repo or referenced from a local external fixture path? -- should the first implementation cover only CLI/runtime execution, or also a later `Save As Script` route check? diff --git a/docs/specs/runtime-validation-implementation-spec.md b/docs/specs/runtime-validation-implementation-spec.md deleted file mode 100644 index deed264..0000000 --- a/docs/specs/runtime-validation-implementation-spec.md +++ /dev/null @@ -1,241 +0,0 @@ -# Runtime Validation Implementation Spec - -## Document Status - -- Date: March 28, 2026 -- Scope: implementation spec only -- Approval status: approved for spec creation, not yet approved for implementation - -## Problem - -The repo now has a runtime validation strategy, but not an implementation contract. Runtime-sensitive changes still rely too heavily on ad hoc local setups and narrow regression tests. The repo needs a small, repeatable runtime validation harness that agents and maintainers can run consistently. - -## Intent - -Implement a phase-based runtime validation harness and add a project-scoped `runtime-e2e-agent` that can operate against it. - -## Locked Decisions - -- Baseline pinned ComfyUI checkout: `/home/peyton/code/ComfyUI` -- Runtime workflow fixtures will be committed in this repo -- Implementation covers both CLI/runtime and `Save As Script`, but in phases -- Phase 1: CLI/runtime first -- Phase 2: `Save As Script` route validation - -## Goals - -- create a repeatable runtime validation harness -- keep the harness small enough to maintain manually -- make the workflow corpus reviewable and committed in-repo -- support pinned-runtime validation with a stable command surface -- give the future `runtime-e2e-agent` a stable command surface - -## Non-Goals - -- no full browser automation in phase 1 -- no image-quality regression testing -- no external CI/CD setup in this phase -- no vendoring of the ComfyUI repo into this repo -- no attempt to validate every possible custom-node workflow - -## Phase Plan - -### Phase 1: CLI / Runtime Harness - -Phase 1 must provide: - -- in-repo runtime workflow fixtures -- a command path that exports workflow JSON to Python -- a command path that validates generated Python parses -- a command path that executes generated Python against a pinned ComfyUI runtime -- clear pass/fail classification - -This phase should be enough to validate: - -- runtime/bootstrap regressions like `#143` -- generated-code validity issues when they must be checked against a real runtime -- runtime/bootstrap changes that affect node-loading or script execution - -### Phase 2: `Save As Script` Route Validation - -Phase 2 must build on the phase 1 harness and add: - -- route-level validation of the `Save As Script` request path -- coverage for server-side export behavior in [__init__.py](/home/peyton/code/ComfyUI-to-Python-Extension/__init__.py) -- coverage for the exported payload shape expected by the frontend flow - -Phase 2 should not start with full browser automation unless route-level checks prove insufficient. - -## Directory Layout - -The first implementation should add: - -- `tests/fixtures/runtime/` -- `tests/fixtures/runtime/upscale-model-loader.json` -- `tests/fixtures/runtime/unsafe-kwargs.json` -- `tests/fixtures/runtime/subgraph-identifiers.json` -- `tests/runtime/` -- a focused runtime validation runner or helper module inside `tests/runtime/` -- `.codex/agents/runtime-e2e-agent.toml` - -Optional supporting docs: - -- `docs/runtime-validation.md` - -## Fixture Rules - -- prefer real ComfyUI workflow shapes over synthetic-only fixtures -- keep fixtures as small as possible while still reproducing the target behavior -- name fixtures after behavior, not issue numbers, when practical -- commit fixtures to the repo so they are reviewable and stable - -## Initial Fixture Set - -### `upscale-model-loader.json` - -Purpose: - -- validate the `#143`/`#148` runtime/bootstrap path - -Assertions: - -- export succeeds -- generated Python parses -- generated script reaches the relevant loader and upscale path without the old missing-node failure - -### `unsafe-kwargs.json` - -Purpose: - -- validate mixed safe and unsafe keys in a real workflow shape - -Assertions: - -- export succeeds -- generated Python parses -- generated code remains safe when real workflow data contains symbol-heavy or invalid keys - -### `subgraph-identifiers.json` - -Purpose: - -- validate that subgraph-expanded identifiers do not break generated Python in real workflow shapes - -Assertions: - -- export succeeds -- generated Python parses -- generated variable names are safe enough for execution - -## Command Contract - -The harness should expose two stable command categories. - -### `fast` - -Purpose: - -- exporter-only validation without a real ComfyUI runtime - -Examples: - -- `uv run python -m unittest tests.test_upscale_model_loader_export -v` -- additional focused exporter tests as they are added - -### `runtime` - -Purpose: - -- run runtime validation against the pinned baseline ComfyUI checkout - -Environment: - -- `COMFYUI_PATH=/home/peyton/code/ComfyUI` - -Behavior: - -- export fixture workflow to Python -- parse or compile generated Python -- execute generated Python with the pinned runtime available - -## Runner Shape - -The implementation should prefer one small runner over many scattered commands. - -Recommended shape: - -- a small Python entry point that: - - selects a tier - - selects a fixture or fixture subset - - sets or reads `COMFYUI_PATH` - - runs export - - validates generated Python - - optionally executes the generated script - - reports failure classification - -The runner must stay small and repo-local. It should not introduce a large new framework. - -## Failure Classification Contract - -Every runtime validation result must classify failure as one of: - -- repo regression -- fixture bug -- environment/setup failure - -This classification is mandatory in both manual runs and future agent reports. - -## Runtime-E2E Agent Addition - -A future implementation step should add: - -- `.codex/agents/runtime-e2e-agent.toml` - -Its instructions should align with: - -- [runtime-e2e-agent-spec.md](/home/peyton/code/ComfyUI-to-Python-Extension/docs/specs/runtime-e2e-agent-spec.md) -- [runtime-validation-environment-spec.md](/home/peyton/code/ComfyUI-to-Python-Extension/docs/specs/runtime-validation-environment-spec.md) - -The agent should: - -- run the `runtime` tier -- report concise results -- avoid long logs -- return `pass`, `pass with risk`, or `fail` - -## Validation Expectations - -Phase 1 success means: - -- the pinned runtime can validate at least the three initial fixtures -- the runner can distinguish export failures from runtime execution failures -- the runtime harness can be invoked consistently by maintainers and agents - -Phase 2 success means: - -- route-level `Save As Script` validation exists -- frontend/server export behavior can be checked without relying on a full browser automation stack - -## Risks - -- fixture maintenance can drift if fixtures are too complex -- local ComfyUI checkout state can become stale if not periodically refreshed intentionally -- runtime validation may still fail for environment reasons if `torch` or ComfyUI dependencies are missing - -## Acceptance Criteria - -- the implementation plan is explicit enough to build without inventing workflow shape later -- fixture names and locations are defined -- command tiers are defined -- phase split is defined -- failure classification is mandatory -- the `runtime-e2e-agent` rollout target is defined - -## Next Step - -If this implementation spec is approved, the next step is to produce a concrete implementation plan and then request explicit approval before creating: - -- runtime fixture files -- the runner/helper module -- the `.codex/agents/runtime-e2e-agent.toml` file -- the short operator doc From 6cbc383415b1f7ec76c01dc199a6351ab3546967 Mon Sep 17 00:00:00 2001 From: Peyton DeNiro <25550995+pydn@users.noreply.github.com> Date: Sat, 28 Mar 2026 16:12:49 -0500 Subject: [PATCH 09/17] remove .md --- docs/runtime-validation.md | 99 -------------------------------------- 1 file changed, 99 deletions(-) delete mode 100644 docs/runtime-validation.md diff --git a/docs/runtime-validation.md b/docs/runtime-validation.md deleted file mode 100644 index 538712f..0000000 --- a/docs/runtime-validation.md +++ /dev/null @@ -1,99 +0,0 @@ -# Runtime Validation - -This repo has two validation tiers: - -- `fast` -- `runtime` - -Use `uv` for repo commands. - -## Local Runtime Paths - -Pinned ComfyUI checkout: - -- `/home/peyton/code/ComfyUI` - -The runtime harness uses `COMFYUI_PATH` to point generated scripts and export logic at the target ComfyUI checkout. -For full E2E runtime execution, the harness prefers the ComfyUI checkout's own Python interpreter at `/home/peyton/code/ComfyUI/.venv/bin/python` when available. - -## Fixtures - -Committed runtime fixtures live in: - -- [tests/fixtures/runtime](/home/peyton/code/ComfyUI-to-Python-Extension/tests/fixtures/runtime) - -Current fixtures: - -- `upscale-model-loader` -- `text-to-image` -- `unsafe-kwargs` -- `subgraph-identifiers` - -Runtime-tier fixtures are based on official core ComfyUI workflow shapes: - -- the Image Upscale workflow -- the Text to Image workflow - -Sources: - -- https://docs.comfy.org/tutorials/basic/upscale -- https://docs.comfy.org/tutorials/basic/text-to-image - -## Commands - -### Fast - -Run exporter-only validation without a real ComfyUI runtime: - -```bash -uv run python tests/runtime/run_runtime_validation.py --tier fast --fixture all -``` - -### Runtime - -Run runtime-capable validation against the pinned baseline checkout. This now performs full E2E execution by default: - -```bash -uv run python tests/runtime/run_runtime_validation.py --tier runtime --fixture upscale-model-loader -``` - -Run both official runtime fixtures: - -```bash -uv run python tests/runtime/run_runtime_validation.py --tier runtime --fixture all -``` - -### Model Provisioning - -If a runtime fixture is missing required models, the harness will report `model provisioning failure`. - -To print the required download commands without executing them: - -```bash -uv run python tests/runtime/run_runtime_validation.py --tier runtime --fixture all --print-download-plan -``` - -Model downloads should be approval-gated and should target the existing `/home/peyton/code/ComfyUI/models/...` directories. - -## Failure Classification - -The runtime harness classifies failures as: - -- `repo regression` -- `model provisioning failure` -- `fixture bug` -- `environment/setup failure` - -This classification matters more than raw pass/fail. A failed run without classification is not actionable enough for maintenance work. - -## Notes - -- `unsafe-kwargs` and `subgraph-identifiers` are committed fast-tier regression fixtures. They are not runtime-tier fixtures. -- `upscale-model-loader` and `text-to-image` are the current runtime-tier fixtures and are based on official ComfyUI tutorials. -- Runtime-tier success now means: - - generated Python parses - - generated script executes - - a new output file is created - - output dimensions meet expectations - - output metadata contains expected markers -- `Save As Script` route validation is still phase 2 work. From 025e58a58e3cf4c6074626cd8fe1739d7d05e43b Mon Sep 17 00:00:00 2001 From: Peyton DeNiro <25550995+pydn@users.noreply.github.com> Date: Sat, 28 Mar 2026 16:22:12 -0500 Subject: [PATCH 10/17] Document test commands and remove hard-coded runtime path --- tests/README.md | 102 ++++++++++++++++++++++++ tests/runtime/run_runtime_validation.py | 10 ++- 2 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 tests/README.md diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..dbe9b28 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,102 @@ +# Tests + +This directory contains two test paths: + +- unit tests for exporter behavior +- runtime validation for committed workflow fixtures + +## Prerequisites + +- Run commands from the repo root. +- Use `uv` for all project commands. +- Install the repo environment first: + +```bash +uv sync +``` + +- Runtime validation also needs a working ComfyUI checkout. + Set `COMFYUI_PATH` to that checkout, or run the tests from a location where a parent directory contains `ComfyUI`. + +## Unit Tests + +Run the current unit test module: + +```bash +uv run python -m unittest tests.test_upscale_model_loader_export +``` + +Run all `unittest`-discoverable tests under `tests`: + +```bash +uv run python -m unittest discover -s tests +``` + +## Runtime Tests + +The runtime harness lives at `tests/runtime/run_runtime_validation.py`. + +### Fast Tier + +Fast tier validates export behavior against committed fixtures without requiring a full ComfyUI runtime for every fixture. + +Run all fast-tier compatible fixtures: + +```bash +uv run python tests/runtime/run_runtime_validation.py --tier fast --fixture all +``` + +Run one fixture: + +```bash +uv run python tests/runtime/run_runtime_validation.py --tier fast --fixture unsafe-kwargs +``` + +### Runtime Tier + +Runtime tier exports inside a real ComfyUI checkout and executes generated Python for runtime-capable fixtures. + +Run all runtime-capable fixtures: + +```bash +uv run python tests/runtime/run_runtime_validation.py --tier runtime --fixture all +``` + +Run one runtime-capable fixture: + +```bash +uv run python tests/runtime/run_runtime_validation.py --tier runtime --fixture text-to-image +``` + +Print download commands for missing models instead of failing immediately: + +```bash +uv run python tests/runtime/run_runtime_validation.py --tier runtime --fixture all --print-download-plan +``` + +## Runtime Fixture Names + +Current committed fixtures: + +- `upscale-model-loader` +- `text-to-image` +- `unsafe-kwargs` +- `subgraph-identifiers` + +Notes: + +- `--tier runtime` only runs fixtures marked runtime-capable. +- `--tier fast` only runs fixtures with local test mappings. + +## Troubleshooting + +- `No module named ...`: + run `uv sync` in this repo, and for runtime-tier failures also make sure the target ComfyUI checkout has its own dependencies installed. +- `Could not find a valid ComfyUI checkout for runtime validation.`: + set `COMFYUI_PATH` to your ComfyUI checkout, or run the tests from a directory layout where a parent folder contains `ComfyUI`. +- `Missing models for ...`: + rerun with `--print-download-plan` to print the expected `curl` commands and target model directories. +- `No selected fixtures are runtime-capable for this tier.`: + choose a runtime-capable fixture such as `text-to-image` or `upscale-model-loader`. +- Generated script execution fails because files or models are missing: + confirm the required models exist under the target ComfyUI checkout and that any staged inputs can be copied into its `input/` directory. diff --git a/tests/runtime/run_runtime_validation.py b/tests/runtime/run_runtime_validation.py index 6313a0e..6770baf 100644 --- a/tests/runtime/run_runtime_validation.py +++ b/tests/runtime/run_runtime_validation.py @@ -13,11 +13,11 @@ from io import StringIO from pathlib import Path from typing import Callable +from comfyui_to_python_utils import get_comfyui_path ROOT = Path(__file__).resolve().parents[2] FIXTURE_DIR = ROOT / "tests" / "fixtures" / "runtime" GENERATED_DIR = ROOT / "tests" / "runtime" / "generated" -BASELINE_COMFYUI_PATH = Path("/home/peyton/code/ComfyUI") COMFYUI_OUTPUT_DIRNAME = "output" COMFYUI_INPUT_DIRNAME = "input" @@ -286,14 +286,16 @@ def load_fixture_names(selection: str) -> list[str]: def ensure_runtime_path(tier: str) -> str: if tier == "runtime": - runtime_path = BASELINE_COMFYUI_PATH + runtime_path = get_comfyui_path() else: return os.environ.get("COMFYUI_PATH", "") - if not runtime_path.is_dir(): + if not runtime_path or not Path(runtime_path).is_dir(): raise ValidationFailure( "environment/setup failure", - f"Expected ComfyUI checkout at {runtime_path}, but it was not found.", + "Could not find a valid ComfyUI checkout for runtime validation. " + "Set COMFYUI_PATH or run the tests from a location where a " + "parent directory contains ComfyUI.", ) return str(runtime_path) From e71f7e207b3df3368a574d4fd9db576acdc20bfc Mon Sep 17 00:00:00 2001 From: Peyton DeNiro <25550995+pydn@users.noreply.github.com> Date: Sat, 28 Mar 2026 16:27:43 -0500 Subject: [PATCH 11/17] Clarify runtime test model prerequisites --- tests/README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index dbe9b28..cc7bca6 100644 --- a/tests/README.md +++ b/tests/README.md @@ -17,6 +17,16 @@ uv sync - Runtime validation also needs a working ComfyUI checkout. Set `COMFYUI_PATH` to that checkout, or run the tests from a location where a parent directory contains `ComfyUI`. +- Runtime-tier tests do not download missing models automatically. + If a required model is missing, the fixture fails with `model provisioning failure`. + Use `--print-download-plan` to print the manual download commands. + +### Runtime Model Requirements + +The current runtime-capable fixtures require these model files in the target ComfyUI checkout: + +- `text-to-image`: `models/checkpoints/v1-5-pruned-emaonly-fp16.safetensors` +- `upscale-model-loader`: `models/upscale_models/RealESRGAN_x4plus.safetensors` ## Unit Tests @@ -95,7 +105,7 @@ Notes: - `Could not find a valid ComfyUI checkout for runtime validation.`: set `COMFYUI_PATH` to your ComfyUI checkout, or run the tests from a directory layout where a parent folder contains `ComfyUI`. - `Missing models for ...`: - rerun with `--print-download-plan` to print the expected `curl` commands and target model directories. + runtime-tier tests do not fetch models for you. Rerun with `--print-download-plan` to print the expected `curl` commands and target model directories, then install the files manually. - `No selected fixtures are runtime-capable for this tier.`: choose a runtime-capable fixture such as `text-to-image` or `upscale-model-loader`. - Generated script execution fails because files or models are missing: From 893c76d6cc0b23f5910853a7295a937faf394378 Mon Sep 17 00:00:00 2001 From: Peyton DeNiro <25550995+pydn@users.noreply.github.com> Date: Sat, 28 Mar 2026 16:33:00 -0500 Subject: [PATCH 12/17] Remove PR_DESCRIPTION.md --- PR_DESCRIPTION.md | 35 ----------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 PR_DESCRIPTION.md diff --git a/PR_DESCRIPTION.md b/PR_DESCRIPTION.md deleted file mode 100644 index 674d348..0000000 --- a/PR_DESCRIPTION.md +++ /dev/null @@ -1,35 +0,0 @@ -## Summary - -This PR adds a runtime validation path for exported workflows so exporter changes can be checked at two levels: - -- fast exporter-only validation -- runtime validation against a pinned local ComfyUI checkout - -It includes committed fixtures for: -- `upscale-model-loader` -- `text-to-image` -- `unsafe-kwargs` -- `subgraph-identifiers` - -It also updates the exporter so generated scripts preserve workflow metadata and hidden runtime inputs needed for end-to-end execution. - -## Included - -- runtime validation harness in `tests/runtime/run_runtime_validation.py` -- runtime fixtures in `tests/fixtures/runtime/` -- exporter support changes in `comfyui_to_python.py` - -## Validation - -Planned validation commands: - -```bash -uv run python tests/runtime/run_runtime_validation.py --tier fast --fixture all -uv run python tests/runtime/run_runtime_validation.py --tier runtime --fixture all -``` - -Runtime-tier failures are classified as: -- `repo regression` -- `model provisioning failure` -- `fixture bug` -- `environment/setup failure` From d87611c035cc68cab18a735fe9a01c8663d6e05f Mon Sep 17 00:00:00 2001 From: Peyton DeNiro <25550995+pydn@users.noreply.github.com> Date: Sat, 28 Mar 2026 16:33:11 -0500 Subject: [PATCH 13/17] Fix runtime validation entrypoint and export test --- tests/runtime/run_runtime_validation.py | 4 ++-- tests/test_upscale_model_loader_export.py | 26 ++++++++++++++++++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/tests/runtime/run_runtime_validation.py b/tests/runtime/run_runtime_validation.py index 6770baf..46b0d84 100644 --- a/tests/runtime/run_runtime_validation.py +++ b/tests/runtime/run_runtime_validation.py @@ -13,8 +13,6 @@ from io import StringIO from pathlib import Path from typing import Callable -from comfyui_to_python_utils import get_comfyui_path - ROOT = Path(__file__).resolve().parents[2] FIXTURE_DIR = ROOT / "tests" / "fixtures" / "runtime" GENERATED_DIR = ROOT / "tests" / "runtime" / "generated" @@ -24,6 +22,8 @@ COMFYUI_INPUT_DIRNAME = "input" if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) +from comfyui_to_python_utils import get_comfyui_path + class ValidationFailure(RuntimeError): def __init__(self, classification: str, message: str): diff --git a/tests/test_upscale_model_loader_export.py b/tests/test_upscale_model_loader_export.py index 7beed03..90827c9 100644 --- a/tests/test_upscale_model_loader_export.py +++ b/tests/test_upscale_model_loader_export.py @@ -95,7 +95,7 @@ class UpscaleModelLoaderExportTest(unittest.TestCase): generated = output.getvalue() self.assertIn("from nodes import NODE_CLASS_MAPPINGS", generated) - self.assertIn("from tests.test_upscale_model_loader_export import (", generated) + self.assertIn(f"from {LoadImage.__module__} import (", generated) self.assertIn("LoadImage,", generated) self.assertIn("UpscaleModelLoader,", generated) self.assertIn("ImageUpscaleWithModel,", generated) @@ -145,6 +145,30 @@ class UpscaleModelLoaderExportTest(unittest.TestCase): self.assertIn('"nodes": []', generated) self.assertNotIn('"source": "workflow_api"', generated) + def test_export_without_frontend_workflow_leaves_png_workflow_metadata_absent(self): + workflow = { + "1": { + "class_type": "LoadImage", + "inputs": { + "image": "example.png", + }, + } + } + + output = StringIO() + ComfyUItoPython( + workflow=json.dumps(workflow), + output_file=output, + node_class_mappings={ + "LoadImage": LoadImage, + }, + ) + + generated = output.getvalue() + + self.assertIn("extra_pnginfo = None", generated) + self.assertNotIn('"workflow": json.loads(', generated) + def test_run_accepts_frontend_workflow_file_for_cli_metadata(self): workflow = { "1": { From 6a9593ee707dba1d14867f4e34082a3b5f74f379 Mon Sep 17 00:00:00 2001 From: Peyton DeNiro <25550995+pydn@users.noreply.github.com> Date: Sat, 28 Mar 2026 16:40:34 -0500 Subject: [PATCH 14/17] Trigger Codex review From 342e1b8749904599234b11fbf59e15ab23ffdffd Mon Sep 17 00:00:00 2001 From: Peyton DeNiro <25550995+pydn@users.noreply.github.com> Date: Sat, 28 Mar 2026 16:58:47 -0500 Subject: [PATCH 15/17] Fix workflow metadata and subgraph export names --- comfyui_to_python.py | 42 ++++++++++------ tests/test_upscale_model_loader_export.py | 60 ++++++++++++++++++++++- 2 files changed, 87 insertions(+), 15 deletions(-) diff --git a/comfyui_to_python.py b/comfyui_to_python.py index 6eec402..d53b7ef 100644 --- a/comfyui_to_python.py +++ b/comfyui_to_python.py @@ -194,6 +194,15 @@ class CodeGenerator: self.node_class_mappings = node_class_mappings self.base_node_class_mappings = base_node_class_mappings + @staticmethod + def sanitize_node_id(node_id: str) -> str: + """Convert node IDs into variable-safe tokens without collapsing separators.""" + sanitized = re.sub(r"[^a-z0-9_]", "_", str(node_id).lower().strip()) + sanitized = re.sub(r"_+", "_", sanitized).strip("_") + if not sanitized: + sanitized = "node" + return sanitized + def generate_workflow( self, load_order: List, @@ -284,7 +293,7 @@ class CodeGenerator: # Create executed variable and generate code executed_variables[idx] = ( f"{self.clean_variable_name(class_type)}_" - f"{self.clean_variable_name(str(idx))}" + f"{self.sanitize_node_id(str(idx))}" ) inputs = self.update_inputs(inputs, executed_variables) @@ -315,7 +324,7 @@ class CodeGenerator: special_functions_code, code, workflow_data, - metadata_workflow_data or workflow_data, + metadata_workflow_data, queue_size, custom_nodes, ) @@ -386,7 +395,7 @@ class CodeGenerator: speical_functions_code: List[str], code: List[str], workflow_data: Dict, - metadata_workflow_data: Dict, + metadata_workflow_data: Dict | None, queue_size: int, custom_nodes=False, ) -> str: @@ -397,13 +406,23 @@ class CodeGenerator: speical_functions_code (List[str]): A list of special functions code strings. code (List[str]): A list of code strings. workflow_data (Dict): The API workflow data used for runtime prompt execution. - metadata_workflow_data (Dict): The workflow metadata to embed into saved outputs. + metadata_workflow_data (Dict | None): The workflow metadata to embed into saved outputs. queue_size (int): Number of photos that will be generated by the script. custom_nodes (bool): Whether to include custom nodes in the code. Returns: str: Generated final code as a string. """ + if metadata_workflow_data is None: + extra_pnginfo_code = "extra_pnginfo = None" + else: + extra_pnginfo_code = ( + "extra_pnginfo = " + '{"workflow": json.loads(' + + json.dumps(json.dumps(metadata_workflow_data)) + + ")}" + ) + # Get the source code of the utils functions as a string func_strings = [] for func in [ @@ -429,12 +448,7 @@ class CodeGenerator: "\n\nadd_comfyui_directory_to_sys_path()\nadd_extra_model_paths()\n", f"workflow = json.loads({json.dumps(json.dumps(workflow_data))})", "prompt = json.loads(json.dumps(workflow))", - ( - "extra_pnginfo = " - '{"workflow": json.loads(' - + json.dumps(json.dumps(metadata_workflow_data)) - + ")}" - ), + extra_pnginfo_code, ] ) # Check if custom nodes should be included @@ -628,12 +642,12 @@ class ComfyUItoPython: else: data = json.loads(self.workflow) - frontend_workflow_data = None + metadata_workflow_data = None if self.frontend_workflow: if isinstance(self.frontend_workflow, str): - frontend_workflow_data = json.loads(self.frontend_workflow) + metadata_workflow_data = json.loads(self.frontend_workflow) else: - frontend_workflow_data = self.frontend_workflow + metadata_workflow_data = self.frontend_workflow # Step 2: Initialize extra/custom nodes when requested or when the workflow references # a node class that is not currently loaded in the runtime. @@ -657,7 +671,7 @@ class ComfyUItoPython: generated_code = code_generator.generate_workflow( load_order, data, - frontend_workflow_data, + metadata_workflow_data, queue_size=self.queue_size, ) diff --git a/tests/test_upscale_model_loader_export.py b/tests/test_upscale_model_loader_export.py index 90827c9..27e3e22 100644 --- a/tests/test_upscale_model_loader_export.py +++ b/tests/test_upscale_model_loader_export.py @@ -57,6 +57,23 @@ class ImageUpscaleWithModel: return (image,) +class MergeImages: + CATEGORY = "image" + FUNCTION = "merge" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "left": ("IMAGE",), + "right": ("IMAGE",), + } + } + + def merge(self, left, right): + return (left, right) + + class UpscaleModelLoaderExportTest(unittest.TestCase): def test_upscale_workflow_uses_direct_upscale_model_loader_init(self): workflow = { @@ -102,7 +119,7 @@ class UpscaleModelLoaderExportTest(unittest.TestCase): self.assertIn("loadimage = LoadImage()", generated) self.assertIn("upscalemodelloader = UpscaleModelLoader()", generated) self.assertIn( - "imageupscalewithmodel__3 = imageupscalewithmodel.upscale(", + "imageupscalewithmodel_3 = imageupscalewithmodel.upscale(", generated, ) self.assertNotIn('NODE_CLASS_MAPPINGS["UpscaleModelLoader"]()', generated) @@ -169,6 +186,47 @@ class UpscaleModelLoaderExportTest(unittest.TestCase): self.assertIn("extra_pnginfo = None", generated) self.assertNotIn('"workflow": json.loads(', generated) + def test_export_preserves_unique_variable_names_for_subgraph_node_ids(self): + workflow = { + "1:23": { + "class_type": "LoadImage", + "inputs": { + "image": "left.png", + }, + }, + "12:3": { + "class_type": "LoadImage", + "inputs": { + "image": "right.png", + }, + }, + "20": { + "class_type": "MergeImages", + "inputs": { + "left": ["1:23", 0], + "right": ["12:3", 0], + }, + }, + } + + output = StringIO() + ComfyUItoPython( + workflow=json.dumps(workflow), + output_file=output, + node_class_mappings={ + "LoadImage": LoadImage, + "MergeImages": MergeImages, + }, + ) + + generated = output.getvalue() + + self.assertIn("loadimage_1_23 = loadimage.load_image(", generated) + self.assertIn("loadimage_12_3 = loadimage.load_image(", generated) + self.assertIn("mergeimages_20 = mergeimages.merge(", generated) + self.assertIn("left=get_value_at_index(loadimage_1_23, 0)", generated) + self.assertIn("right=get_value_at_index(loadimage_12_3, 0)", generated) + def test_run_accepts_frontend_workflow_file_for_cli_metadata(self): workflow = { "1": { From 494f7cf79a0eada1ba4b442d8cec0b54d86ecc0f Mon Sep 17 00:00:00 2001 From: Peyton DeNiro <25550995+pydn@users.noreply.github.com> Date: Sat, 28 Mar 2026 17:07:59 -0500 Subject: [PATCH 16/17] Remove CLI frontend workflow metadata path --- README.md | 6 +---- comfyui_to_python.py | 14 ---------- tests/test_upscale_model_loader_export.py | 33 +++-------------------- 3 files changed, 4 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 1b8c09c..b80470b 100644 --- a/README.md +++ b/README.md @@ -87,14 +87,12 @@ Options: ```bash uv run python comfyui_to_python.py \ --input_file workflow_api.json \ - --frontend_workflow_file workflow.json \ --output_file workflow_api.py \ --queue_size 10 ``` Flags: - `--input_file`: input workflow JSON, default `workflow_api.json` -- `--frontend_workflow_file`: optional full frontend workflow JSON for reimportable PNG metadata - `--output_file`: output Python file, default `workflow_api.py` - `--queue_size`: default execution count in the generated script, default `10` @@ -112,9 +110,7 @@ export COMFYUI_PATH=/path/to/ComfyUI The generated script is a workflow export. It does not automatically turn workflow inputs into command-line arguments. -If you want generated images to drag-drop back into ComfyUI from the CLI flow, provide both: -- the API workflow JSON via `--input_file` -- the full frontend workflow JSON via `--frontend_workflow_file` +Scripts exported directly from `File -> Save As Script` in the ComfyUI UI already include the frontend workflow metadata needed for drag-and-drop reimport. Images saved by those scripts can be dropped back into ComfyUI and reopen with the original workflow metadata. ## Troubleshooting diff --git a/comfyui_to_python.py b/comfyui_to_python.py index d53b7ef..b736b95 100644 --- a/comfyui_to_python.py +++ b/comfyui_to_python.py @@ -685,7 +685,6 @@ def run( input_file: str = DEFAULT_INPUT_FILE, output_file: str = DEFAULT_OUTPUT_FILE, queue_size: int = DEFAULT_QUEUE_SIZE, - frontend_workflow_file: str = "", ) -> None: """Generate Python code from a ComfyUI workflow_api.json file. @@ -695,19 +694,12 @@ def run( Defaults to "workflow_api.py". queue_size (int): The number of times a workflow will be executed by the script. Defaults to 1. - frontend_workflow_file (str): Optional path to a full frontend workflow JSON file. - When provided, generated images can preserve reimportable ComfyUI workflow metadata. Returns: None """ ComfyUItoPython( input_file=input_file, - frontend_workflow=( - FileHandler.read_json_file(frontend_workflow_file) - if frontend_workflow_file - else None - ), output_file=output_file, queue_size=queue_size, needs_init_custom_nodes=True, @@ -740,12 +732,6 @@ def main() -> None: help="number of times the workflow will be executed by default", default=DEFAULT_QUEUE_SIZE, ) - parser.add_argument( - "--frontend_workflow_file", - type=str, - help="optional path to a full frontend workflow JSON file for reimportable PNG metadata", - default="", - ) pargs = parser.parse_args() run(**vars(pargs)) print("Done.") diff --git a/tests/test_upscale_model_loader_export.py b/tests/test_upscale_model_loader_export.py index 27e3e22..e6f4035 100644 --- a/tests/test_upscale_model_loader_export.py +++ b/tests/test_upscale_model_loader_export.py @@ -227,7 +227,7 @@ class UpscaleModelLoaderExportTest(unittest.TestCase): self.assertIn("left=get_value_at_index(loadimage_1_23, 0)", generated) self.assertIn("right=get_value_at_index(loadimage_12_3, 0)", generated) - def test_run_accepts_frontend_workflow_file_for_cli_metadata(self): + def test_run_cli_export_leaves_png_workflow_metadata_absent(self): workflow = { "1": { "class_type": "LoadImage", @@ -236,39 +236,13 @@ class UpscaleModelLoaderExportTest(unittest.TestCase): }, } } - frontend_workflow = { - "version": 0.4, - "last_node_id": 1, - "last_link_id": 0, - "nodes": [ - { - "id": 1, - "type": "LoadImage", - "pos": [0, 0], - "size": [210, 60], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [], - "properties": {}, - "widgets_values": ["example.png"], - } - ], - "links": [], - "groups": [], - "config": {}, - "extra": {}, - } with tempfile.TemporaryDirectory() as tmpdir: tmpdir_path = Path(tmpdir) input_file = tmpdir_path / "workflow_api.json" - frontend_file = tmpdir_path / "workflow.json" output_file = tmpdir_path / "workflow_api.py" input_file.write_text(json.dumps(workflow), encoding="utf-8") - frontend_file.write_text(json.dumps(frontend_workflow), encoding="utf-8") with patch( "comfyui_to_python.get_node_class_mappings", @@ -276,15 +250,14 @@ class UpscaleModelLoaderExportTest(unittest.TestCase): ), patch("comfyui_to_python.import_custom_nodes"): run( input_file=str(input_file), - frontend_workflow_file=str(frontend_file), output_file=str(output_file), queue_size=1, ) generated = output_file.read_text(encoding="utf-8") - self.assertIn('"version": 0.4', generated) - self.assertIn('"type": "LoadImage"', generated) + self.assertIn("extra_pnginfo = None", generated) + self.assertNotIn('"workflow": json.loads(', generated) if __name__ == "__main__": From 4370a78728880ab98b34cb8cb82219e9588bb5c0 Mon Sep 17 00:00:00 2001 From: Peyton DeNiro <25550995+pydn@users.noreply.github.com> Date: Sat, 28 Mar 2026 17:27:03 -0500 Subject: [PATCH 17/17] Fix exporter metadata kwarg and seed sync handling --- comfyui_to_python.py | 51 ++++++++--- tests/test_upscale_model_loader_export.py | 100 ++++++++++++++++++++++ 2 files changed, 141 insertions(+), 10 deletions(-) diff --git a/comfyui_to_python.py b/comfyui_to_python.py index b736b95..efae605 100644 --- a/comfyui_to_python.py +++ b/comfyui_to_python.py @@ -276,17 +276,19 @@ class CodeGenerator: if no_params or key in class_def_params } # Deal with hidden variables + hidden_inputs = input_types.get("hidden", {}) if ( - "hidden" in input_types.keys() - and "unique_id" in input_types["hidden"].keys() + "unique_id" in hidden_inputs + and (no_params or "unique_id" in class_def_params) ): inputs["unique_id"] = random.randint(1, 2**64) - if "hidden" in input_types.keys(): - if "prompt" in input_types["hidden"].keys(): - inputs["prompt"] = {"variable_name": "prompt"} - if "extra_pnginfo" in input_types["hidden"].keys(): - inputs["extra_pnginfo"] = {"variable_name": "extra_pnginfo"} - elif class_def_params is not None: + if "prompt" in hidden_inputs and (no_params or "prompt" in class_def_params): + inputs["prompt"] = {"variable_name": "prompt"} + if "extra_pnginfo" in hidden_inputs and ( + no_params or "extra_pnginfo" in class_def_params + ): + inputs["extra_pnginfo"] = {"variable_name": "extra_pnginfo"} + if "hidden" not in input_types and class_def_params is not None: if "unique_id" in class_def_params: inputs["unique_id"] = random.randint(1, 2**64) @@ -296,8 +298,13 @@ class CodeGenerator: f"{self.sanitize_node_id(str(idx))}" ) inputs = self.update_inputs(inputs, executed_variables) + seed_sync_code = self.create_prompt_seed_sync_code( + idx, inputs, is_special_function + ) if is_special_function: + if seed_sync_code: + special_functions_code.extend(seed_sync_code) special_functions_code.append( self.create_function_call_code( initialized_objects[class_type], @@ -308,6 +315,8 @@ class CodeGenerator: ) ) else: + if seed_sync_code: + code.extend(seed_sync_code) code.append( self.create_function_call_code( initialized_objects[class_type], @@ -363,6 +372,28 @@ class CodeGenerator: return code + def create_prompt_seed_sync_code( + self, node_id: str, inputs: Dict, is_special_function: bool + ) -> List[str]: + """Generate code that keeps prompt metadata aligned with randomized seeds.""" + seed_sync_lines = [] + for key in ("seed", "noise_seed"): + if key not in inputs: + continue + randomized_seed_variable = ( + f"node_{self.sanitize_node_id(str(node_id))}_{self.clean_variable_name(key)}" + ) + seed_sync_lines.append( + f'{randomized_seed_variable} = prompt["{node_id}"]["inputs"]["{key}"] = random.randint(1, 2**64)' + ) + inputs[key] = {"variable_name": randomized_seed_variable} + + if not seed_sync_lines: + return [] + + indentation = "" if is_special_function else "\t" + return [f"{indentation}{line}\n" for line in seed_sync_lines] + def format_arg(self, key: str, value: any) -> str: """Formats arguments based on key and value. @@ -381,12 +412,12 @@ class CodeGenerator: @staticmethod def format_arg_value(key: str, value: any) -> str: """Formats an argument value as Python source.""" + if isinstance(value, dict) and "variable_name" in value: + return value["variable_name"] if key == "noise_seed" or key == "seed": return "random.randint(1, 2**64)" if isinstance(value, str): return json.dumps(value) - if isinstance(value, dict) and "variable_name" in value: - return value["variable_name"] return repr(value) def assemble_python_code( diff --git a/tests/test_upscale_model_loader_export.py b/tests/test_upscale_model_loader_export.py index e6f4035..d711c4b 100644 --- a/tests/test_upscale_model_loader_export.py +++ b/tests/test_upscale_model_loader_export.py @@ -74,6 +74,46 @@ class MergeImages: return (left, right) +class HiddenMetadataFilteredNode: + CATEGORY = "image" + FUNCTION = "save" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "image": ("IMAGE",), + }, + "hidden": { + "prompt": "PROMPT", + "extra_pnginfo": "EXTRA_PNGINFO", + }, + } + + def save(self, image): + return (image,) + + +class HiddenPromptSeedNode: + CATEGORY = "sampling" + FUNCTION = "sample" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "seed": ("INT",), + "noise_seed": ("INT",), + }, + "hidden": { + "prompt": "PROMPT", + }, + } + + def sample(self, seed, noise_seed, prompt): + return (seed, noise_seed, prompt) + + class UpscaleModelLoaderExportTest(unittest.TestCase): def test_upscale_workflow_uses_direct_upscale_model_loader_init(self): workflow = { @@ -259,6 +299,66 @@ class UpscaleModelLoaderExportTest(unittest.TestCase): self.assertIn("extra_pnginfo = None", generated) self.assertNotIn('"workflow": json.loads(', generated) + def test_hidden_metadata_kwargs_follow_function_signature(self): + workflow = { + "1": { + "class_type": "HiddenMetadataFilteredNode", + "inputs": { + "image": "example.png", + }, + } + } + + output = StringIO() + ComfyUItoPython( + workflow=json.dumps(workflow), + output_file=output, + node_class_mappings={ + "HiddenMetadataFilteredNode": HiddenMetadataFilteredNode, + }, + ) + + generated = output.getvalue() + + self.assertIn("hiddenmetadatafilterednode_1 =", generated) + self.assertNotIn("prompt=prompt", generated) + self.assertNotIn("extra_pnginfo=extra_pnginfo", generated) + + def test_randomized_seed_inputs_update_prompt_metadata_before_execution(self): + workflow = { + "1": { + "class_type": "HiddenPromptSeedNode", + "inputs": { + "seed": 1, + "noise_seed": 2, + }, + } + } + + output = StringIO() + ComfyUItoPython( + workflow=json.dumps(workflow), + output_file=output, + node_class_mappings={ + "HiddenPromptSeedNode": HiddenPromptSeedNode, + }, + ) + + generated = output.getvalue() + + self.assertIn( + 'node_1_seed = prompt["1"]["inputs"]["seed"] = random.randint(1, 2**64)', + generated, + ) + self.assertIn( + 'node_1_noise_seed = prompt["1"]["inputs"]["noise_seed"] = random.randint(', + generated, + ) + self.assertIn("1, 2**64", generated) + self.assertIn("seed=node_1_seed", generated) + self.assertIn("noise_seed=node_1_noise_seed", generated) + self.assertIn("prompt=prompt", generated) + if __name__ == "__main__": unittest.main()