From b87d17e34d62f1ef552a007eea76ed919efde6fa Mon Sep 17 00:00:00 2001 From: Marcel Hofer <49859266+goodguy1963@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:08:23 +0100 Subject: [PATCH] feat: add native Decision API backends (v3.0.2) Add native OpenAI Decisions and OpenRouter Decisions API routing to the image and text Decision Router nodes while keeping local in-process llama.cpp inference as the default. Discover OpenRouter decision models by input modality and decision output capability, then enforce server profile image policy and optional model allowlists both in the UI model catalog and before inference. Keep credentials and endpoints in server-side profiles; apply profile input, image, and timeout limits; never fall back to chat-generated JSON. Preserve saved workflow widget order, release any retained local model when switching to API mode, and normalize typed route choices, probabilities, and confidence into decision_json. Add API setup documentation and adapter/router/security regression coverage. Validation: 240 tests passed across six test modules run individually in bundled ComfyUI Python 3.12; Python and JavaScript syntax checks, API catalog validation, version metadata checks, and git diff --check passed. Provider inference is covered with mocked responses; no authenticated live inference was run. --- README.md | 9 +- __init__.py | 2 +- custom_nodes.json | 2 +- docs/DECISION_ROUTER_API.md | 52 ++++++ docs/OPENAI_COMPATIBLE_API.md | 10 ++ nodes/decision_router.py | 166 +++++++++++++++-- nodes/openai_compatible_api.py | 18 +- pyproject.toml | 2 +- tests/test_decision_api.py | 248 ++++++++++++++++++++++++++ tests/test_decision_router.py | 8 +- tests/test_openai_api_security.py | 12 ++ thinkingllm_api_profiles.example.json | 2 + thinkingllm_core/decision_api.py | 193 ++++++++++++++++++++ update.md | 18 ++ web/js/decision_router.js | 76 +++++++- web_commercial/js/decision_router.js | 76 +++++++- 16 files changed, 869 insertions(+), 25 deletions(-) create mode 100644 docs/DECISION_ROUTER_API.md create mode 100644 tests/test_decision_api.py create mode 100644 thinkingllm_core/decision_api.py diff --git a/README.md b/README.md index 541ff30..9544dc6 100644 --- a/README.md +++ b/README.md @@ -51,8 +51,8 @@ The Decision Router needs native llama.cpp decision APIs that are not present in | `ThinkingLLM Prompt Enhancer` | Transformers / HF | Text prompt enhancement | | `ThinkingLLM (GGUF)` | llama.cpp | GGUF image and video understanding | | `ThinkingLLM (GGUF Advanced)` | llama.cpp | GGUF advanced controls and `MASK_PREVIEW` | -| `ThinkingLLM Decision Router (Image)` | llama.cpp | Image decisions routed to named branches | -| `ThinkingLLM Decision Router (Text)` | llama.cpp | Text decisions routed to named branches | +| `ThinkingLLM Decision Router (Image)` | llama.cpp / OpenAI / OpenRouter | Image decisions routed to named branches | +| `ThinkingLLM Decision Router (Text)` | llama.cpp / OpenAI / OpenRouter | Text decisions routed to named branches | | `ThinkingLLM Gemma 4 Audio (GGUF)` | llama.cpp | Gemma 4 audio understanding | | `ThinkingLLM Whisper ASR` | faster-whisper | Speech-to-text transcription | | `ThinkingLLM Prompt Enhancer (GGUF)` | llama.cpp | GGUF prompt enhancement | @@ -63,7 +63,9 @@ The Transformers/HF nodes are the recommended default when you want the simplest ### Decision Router -The separate Image and Text Decision Router nodes classify their input and send each image or text item to one named route. Configure a system prompt plus route names and descriptions for any task, such as subject, document, or product classification. They offer Clef-Flash 9B, Clef 27B, and OpenJev 27B GGUFs. Model files download on first use into `ComfyUI/models/LLM/GGUF`; image routing also downloads the matching vision projector. Both run in-process without a separate llama.cpp server. Each node exposes up to 52 configured route sockets and returns per-item decisions and normalized route probabilities on `decision_json`. Empty route outputs are blocked, so their downstream branches do not execute. `keep_model_loaded` controls whether the model stays resident between runs (off by default); `include_reason` optionally adds a short post-hoc explanation to each JSON decision without changing the route or scores. +The separate Image and Text Decision Router nodes classify their input and send each image or text item to one named route. Configure a system prompt plus route names and descriptions for any task, such as subject, document, or product classification. They offer Clef-Flash 9B, Clef 27B, and OpenJev 27B GGUFs. Model files download on first use into `ComfyUI/models/LLM/GGUF`; image routing also downloads the matching vision projector. Local inference runs in-process without a separate llama.cpp server. Each node exposes up to 52 configured route sockets and returns per-item decisions and normalized route probabilities on `decision_json`. Empty route outputs are blocked, so their downstream branches do not execute. `keep_model_loaded` controls whether the model stays resident between runs (off by default); `include_reason` optionally adds a short post-hoc explanation to each JSON decision without changing the route or scores. + +You can also select the native **API** backend on either router. It supports OpenAI Decisions (`gpt-6-luna`) and OpenRouter's Decisions API; OpenRouter models are listed according to their text/image capabilities. API credentials stay in server-side profiles/environment variables rather than workflow files. See the [Decision Router API guide](docs/DECISION_ROUTER_API.md) for setup and limits. Clef models require llama.cpp's native decision-batch APIs; OpenJev uses llama.cpp's final-position logits API. Image routing additionally requires the MTMD vision APIs; text routing does not. Each node checks the selected capabilities before downloading model weights and reports missing symbols with an installation hint. See [Decision Router llama.cpp requirements](docs/LLAMA_CPP_SYSTEMONE_INSTALL.md) for the runtime details. OpenJev is CC BY-NC 4.0 and is hidden in commercial builds; Clef models are Apache-2.0. @@ -112,6 +114,7 @@ See [Models and backends](docs/MODELS_AND_BACKENDS.md) for exact families, model | Supported models, discovery, locations, and backend selection | [Models and backends](docs/MODELS_AND_BACKENDS.md) | | GGUF / llama.cpp installation on Windows and Linux | [llama.cpp backend installation](docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md) | | Decision model runtime requirements | [Decision Router llama.cpp requirements](docs/LLAMA_CPP_SYSTEMONE_INSTALL.md) | +| Native OpenAI and OpenRouter decision APIs | [Decision Router API guide](docs/DECISION_ROUTER_API.md) | | Duration-aware LTX and MiniMax H3 prompting | [Duration-aware video prompts](docs/VIDEO_PROMPT_DURATION.md) | | Video preset recommendations and rationale | [Video preset settings](docs/VIDEO_PRESET_SETTINGS.md) | | OpenAI-compatible API node | [API setup and security](docs/OPENAI_COMPATIBLE_API.md) | diff --git a/__init__.py b/__init__.py index d6a7ead..4454d64 100644 --- a/__init__.py +++ b/__init__.py @@ -3,7 +3,7 @@ "name": "ComfyUI-ThinkingLLM", "description": "A multimodal ComfyUI AI node pack with Qwen, Gemma 4, GGUF, Whisper, image and text decision routing, and live thinking output.", "author": "goodguy1963", - "version": "3.0.1", + "version": "3.0.2", "url": "https://github.com/goodguy1963/ComfyUI-ThinkingLLM", "category": "image" } diff --git a/custom_nodes.json b/custom_nodes.json index 0b133ac..b1ac437 100644 --- a/custom_nodes.json +++ b/custom_nodes.json @@ -9,5 +9,5 @@ "description": "A multimodal ComfyUI AI node pack with Qwen, Gemma 4, GGUF, Whisper, image and text decision routing, and live thinking output.", "category": "image", "tags": ["vision", "language", "multimodal", "thinkingllm", "decision-router", "qwen", "gguf", "flash_attention", "performance", "wan2.2", "video", "i2v", "t2v", "storyboard", "caching", "fork", "gpl-3.0"], - "version": "3.0.1" + "version": "3.0.2" } diff --git a/docs/DECISION_ROUTER_API.md b/docs/DECISION_ROUTER_API.md new file mode 100644 index 0000000..df46922 --- /dev/null +++ b/docs/DECISION_ROUTER_API.md @@ -0,0 +1,52 @@ +# Decision Router API mode + +The Image and Text Decision Router nodes support both local in-process inference and native API inference. Local remains the default. API mode sends each item in a batch as a separate decision request and routes the original input item according to the provider's typed `choice` response. + +## Supported native providers + +| Profile | Protocol | Endpoint | Initial model selection | Image input | +| --- | --- | --- | --- | --- | +| OpenAI | `openai_decisions` | `https://api.openai.com/v1/decisions` | `gpt-6-luna` | Supported | +| OpenRouter | `openrouter_decisions` | `https://openrouter.ai/api/alpha/decisions` | Decision models returned by its Models API | Only models whose input modalities include `image` | + +Other providers in the OpenAI-compatible chat node are not exposed as native Decision APIs by this feature. The router does not fall back to a chat completion that generates JSON. + +## Credentials and profile setup + +The built-in `OpenAI` and `OpenRouter` profiles use `OPENAI_API_KEY` and `OPENROUTER_API_KEY`. Set the key in the environment of the process that launches ComfyUI, then restart ComfyUI. Restart after adding or changing a custom profile as well, so the node's profile list is refreshed. The key and endpoint do not enter the workflow. + +Custom profiles can opt into a native protocol using the server-side profile file: + +```json +{ + "profiles": { + "OpenRouter Decision Production": { + "provider": "OpenRouter", + "base_url": "https://openrouter.ai/api/v1", + "decision_protocol": "openrouter_decisions", + "decision_endpoint": "https://openrouter.ai/api/alpha/decisions", + "auth": "bearer_env", + "api_key_env": "OPENROUTER_PRODUCTION_API_KEY", + "allowed_models": ["cloudflare/clef-flash", "typesafe/jev-1.14"], + "allow_images": true, + "max_image_pixels": 16777216, + "max_image_bytes": 20000000, + "max_timeout_seconds": 300 + } + } +} +``` + +`decision_protocol` and `decision_endpoint` must appear together. The endpoint is trusted server configuration, validated at profile load, and is not a node input. For shared ComfyUI servers, use an exact `allowed_models` list. + +## Model and image capability handling + +The node's API model list is filtered by decision output, required input modality, the profile's `allow_images` policy, and its optional `allowed_models` list. OpenRouter metadata is cached for up to one hour. If a stale cached list exists during a metadata outage, it can still be displayed; the selected model is checked against that list before inference. A model absent from the capability catalog is rejected rather than guessed to support images. + +Image tensors are converted to bounded PNG data URLs using the selected server profile's `allow_images`, `max_image_pixels`, and `max_image_bytes` settings. OpenAI receives an `input_image` part; OpenRouter receives an `image_url` part in `state`. The profile's `max_input_chars` also bounds the system prompt, input text, and configured route names/descriptions. Provider-specific image limits can still apply and are reported as inference errors. + +## Output and limits + +The APIs return typed route selections and probabilities, which are normalized into the router's existing `decision_json` output. Provider confidence is included when returned. Route descriptions become the choice criteria, and the system prompt becomes the decision instruction. The existing 52-route cap and dynamic output sockets continue to apply. + +Native Decisions APIs do not generate a rationale. The local-only `include_reason` control is hidden in API mode; if it remains enabled in a saved workflow, the result explicitly marks the reason as unavailable. `keep_model_loaded` has no effect on remote inference, and switching to API mode releases a local model previously retained by the node. diff --git a/docs/OPENAI_COMPATIBLE_API.md b/docs/OPENAI_COMPATIBLE_API.md index 400abd4..dcbda68 100644 --- a/docs/OPENAI_COMPATIBLE_API.md +++ b/docs/OPENAI_COMPATIBLE_API.md @@ -253,6 +253,8 @@ The string `OPENROUTER_PRODUCTION_API_KEY` is only the **name** of an environmen | `thinking_mode` | Currently empty or `qwen`. | | `max_tokens_field` | `max_tokens` or `max_completion_tokens`. | | `allow_insecure_http` | Explicit opt-in for unauthenticated non-loopback HTTP. | +| `decision_protocol` | Optional native Decision API adapter: `openai_decisions` or `openrouter_decisions`. Profiles without it cannot be selected by the Decision Router API mode. | +| `decision_endpoint` | Optional fixed HTTPS endpoint paired with `decision_protocol`; this is server-side profile data and is never workflow-controlled. | Unknown profile fields are rejected instead of silently ignored. This is intentional: a typo such as `max_token_limit` must not quietly disable a security limit. @@ -260,6 +262,14 @@ A custom profile with the same name as a built-in is treated as an override. If Literal secret/header fields such as `api_key`, `token`, `Authorization`, `headers`, or `secret` are rejected. +### Native Decisions API profiles + +The Image and Text Decision Router nodes can use the built-in OpenAI and OpenRouter profiles in API mode. These profiles use their dedicated Decisions APIs, not `/chat/completions`. To enable a custom profile, set both `decision_protocol` and `decision_endpoint`, keep the credential in the configured environment variable, and configure an exact `allowed_models` list for shared deployments. OpenAI uses `openai_decisions` with `/v1/decisions`; OpenRouter uses `openrouter_decisions` with `/api/alpha/decisions`. See the [Decision Router API guide](DECISION_ROUTER_API.md) for configuration details. + +OpenAI currently exposes `gpt-6-luna` through its Decisions API. OpenRouter model choices are fetched from its model catalog and filtered by `output_modalities=decisions` plus this node's input modality. Image support is a model capability, not a provider-wide setting. If model metadata is unavailable, the UI may retain a previously fetched list, but the server validates the selected model before making an inference request. + +Each input item in a ComfyUI batch is sent as its own native decision request. Local mode remains the default and does not require API credentials. `keep_model_loaded` and generated rationale output apply only to local models; native Decisions APIs return typed choices and probabilities, not free-form explanations. + --- ## Production lock-down mode diff --git a/nodes/decision_router.py b/nodes/decision_router.py index a9bef42..5feb171 100644 --- a/nodes/decision_router.py +++ b/nodes/decision_router.py @@ -76,6 +76,16 @@ def _model_options() -> list[str]: return COMMUNITY_MODEL_OPTIONS.copy() +def _api_profile_options() -> list[str]: + try: + from openai_compatible_api import _load_api_profiles + + profiles = _load_api_profiles() + except Exception: + return ["No Decision API profiles configured"] + return [name for name, profile in profiles.items() if profile.get("decision_protocol")] or ["No Decision API profiles configured"] + + def _gguf_base_dir() -> Path: if folder_paths is None: raise RuntimeError("ComfyUI folder_paths is unavailable; the Decision Router must run inside ComfyUI.") @@ -294,6 +304,19 @@ class _DecisionRouterBase: "STRING", {"default": "", "multiline": False}, ) + # Append new controls so saved workflows keep their original positional widget values. + required["decision_backend"] = ( + ["Local (in-process)", "API"], + {"default": "Local (in-process)", "tooltip": "Use the local native model or a server-configured native Decisions API."}, + ) + required["api_profile"] = ( + _api_profile_options(), + {"tooltip": "Server-side API profile. Only profiles explicitly configured for OpenAI Decisions or OpenRouter Decisions are available."}, + ) + required["api_model"] = ( + "STRING", + {"default": "gpt-6-luna", "multiline": False, "tooltip": "Decision model ID. The UI lists models matching this node's input type."}, + ) return {"required": required} CATEGORY = "ThinkingLLM/Decision" @@ -403,26 +426,93 @@ class _DecisionRouterBase: if len(routes) > len("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"): raise ValueError("The selected native decision protocol supports at most 52 routes.") - model_key = MODEL_KEYS_BY_DISPLAY.get(str(decision_model)) - if model_key is None or (COMMERCIAL_RELEASE and model_key == "openjev"): - raise ValueError(f"Decision model is unavailable in this build: {decision_model!r}.") - + api_backend = str(kwargs.get("decision_backend", "Local (in-process)")) == "API" keep_model_loaded = bool(kwargs.get("keep_model_loaded", False)) + if api_backend and (self.llm is not None or self.chat_handler is not None): + self.clear() if item_count == 0: - if not keep_model_loaded: + if not api_backend and not keep_model_loaded: self.clear() + selected_model = str(kwargs.get("api_model", "")).strip() if api_backend else str(decision_model) outputs = [ExecutionBlocker(None) for _ in range(MAX_DECISION_ROUTES)] - return (json.dumps({"model": decision_model, "input_type": input_kind, "decisions": []}, ensure_ascii=False), *outputs) + return (json.dumps({"model": selected_model, "input_type": input_kind, "decisions": []}, ensure_ascii=False), *outputs) + model_key = None + api_profile = None + api_model = "" + api_key = "" + if api_backend: + try: + from openai_compatible_api import ( + _credential_value, + _image_to_data_url, + _load_api_profiles, + _NO_REDIRECT_OPENER, + _validate_input_text, + ) + from thinkingllm_core.decision_api import decision_models_for_profile, request_decision + except Exception as exc: + raise RuntimeError("Decision API support could not be loaded. Restart ComfyUI and check the API node installation.") from exc + profile_name = str(kwargs.get("api_profile", "")).strip() + api_profile = _load_api_profiles().get(profile_name) + if not api_profile or not api_profile.get("decision_protocol"): + raise ValueError(f"API profile {profile_name!r} is not configured for a native Decisions API.") + api_model = str(kwargs.get("api_model", "")).strip() + if not api_model: + raise ValueError("Select a Decision API model.") + api_key = _credential_value(api_profile) + api_opener = _NO_REDIRECT_OPENER + if use_image: + if not api_profile.get("allow_images", False): + raise ValueError(f"Image input is disabled by API profile {profile_name!r}.") + supported_models = decision_models_for_profile(api_profile, input_kind) + selected_api_model = next((item for item in supported_models if item.get("id") == api_model), None) + if selected_api_model is None: + modality = "image-capable " if use_image else "" + raise ValueError(f"Model {api_model!r} is not a listed {modality}Decision API model allowed by profile {profile_name!r}.") + if use_image and not selected_api_model.get("vision", False): + raise ValueError(f"Decision API model {api_model!r} is not an image-capable model.") + else: + model_key = MODEL_KEYS_BY_DISPLAY.get(str(decision_model)) + if model_key is None or (COMMERCIAL_RELEASE and model_key == "openjev"): + raise ValueError(f"Decision model is unavailable in this build: {decision_model!r}.") include_reason = bool(kwargs.get("include_reason", False)) + api_route_text = "\n".join( + f"{route['name']}: {route.get('description') or ''}" for route in routes + ) if api_backend else "" try: - self._load_model(model_key, require_vision=use_image) + if not api_backend: + self._load_model(model_key, require_vision=use_image) decisions = [] chosen_names = [] for item_index in range(item_count): - image_data_url = _encode_image(items[item_index]) if use_image else None + if api_backend: + item_text = str(items[item_index]) if not use_image else "Classify the supplied image." + _validate_input_text( + api_profile, + system_prompt, + f"{item_text}\n{api_route_text}", + ) + image_data_url = None + if use_image: + if api_backend: + image_data_url = _image_to_data_url(api_profile, items[item_index]) + else: + image_data_url = _encode_image(items[item_index]) try: - if model_key == "openjev": + if api_backend: + chosen, probability_map, confidence = request_decision( + api_profile, + api_model, + input_kind, + items[item_index], + image_data_url, + str(system_prompt or ""), + routes, + api_key=api_key, + opener=api_opener, + ) + elif model_key == "openjev": if use_image: probabilities = openjev_probabilities(self.llm, self.chat_handler, image_data_url, str(system_prompt or ""), routes) else: @@ -435,10 +525,12 @@ class _DecisionRouterBase: sorted_probabilities, sorted_routes = clef_text_probabilities(self.llm, items[item_index], str(system_prompt or ""), routes) sorted_map = map_route_probabilities(sorted_routes, sorted_probabilities) probability_map = {str(route["name"]): sorted_map[str(route["name"])] for route in routes} - chosen = selected_route(routes, probability_map) + if not api_backend: + chosen = selected_route(routes, probability_map) except Exception as exc: + model_label = api_model if api_backend else decision_model raise RuntimeError( - f"Decision inference failed for {'image' if use_image else 'text'} item {item_index + 1} with {decision_model}: {exc}" + f"Decision inference failed for {'image' if use_image else 'text'} item {item_index + 1} with {model_label}: {exc}" ) from exc chosen_names.append(chosen) decision = { @@ -446,7 +538,13 @@ class _DecisionRouterBase: "route": chosen, "probabilities": probability_map, } - if include_reason: + if api_backend: + if confidence is not None: + decision["confidence"] = confidence + if include_reason: + decision["reason"] = None + decision["reason_error"] = "The selected Decisions API returns typed decisions, not generated explanations." + elif include_reason: chosen_route = next(route for route in routes if route["name"] == chosen) try: decision["reason"] = _decision_reason( @@ -476,14 +574,17 @@ class _DecisionRouterBase: else: output_batches.append([items[index] for index in batch_indices]) + metadata = {"model": api_model if api_backend else decision_model, "input_type": input_kind, "decisions": decisions} + if api_backend: + metadata["provider"] = api_profile["provider"] decision_json = json.dumps( - {"model": decision_model, "input_type": input_kind, "decisions": decisions}, + metadata, ensure_ascii=False, separators=(",", ":"), ) return (decision_json, *output_batches) finally: - if not keep_model_loaded: + if not api_backend and not keep_model_loaded: self.clear() @@ -532,3 +633,40 @@ NODE_DISPLAY_NAME_MAPPINGS = { "ThinkingLLM_DecisionRouter": "ThinkingLLM Decision Router (Image)", "ThinkingLLM_DecisionRouterText": "ThinkingLLM Decision Router (Text)", } + + +def _register_decision_model_route(): + try: + from aiohttp import web + from server import PromptServer + from thinkingllm_core.decision_api import decision_models_for_profile + + server = PromptServer.instance + if server is None: + return + + async def list_decision_models(request): + from openai_compatible_api import _load_api_profiles + + profile_name = str(request.query.get("profile", "")) + input_kind = str(request.query.get("input_kind", "text")) + if input_kind not in {"text", "image"}: + raise web.HTTPBadRequest(text="input_kind must be text or image") + profile = _load_api_profiles().get(profile_name) + if not profile or not profile.get("decision_protocol"): + raise web.HTTPBadRequest(text="Unknown Decision API profile") + try: + models = decision_models_for_profile(profile, input_kind) + except RuntimeError as exc: + raise web.HTTPServiceUnavailable(text="Decision model catalog is unavailable") from exc + return web.json_response({"models": models}) + + server.routes.get("/thinkingllm/decision/models")(list_decision_models) + except Exception as exc: + # The module is also imported in lightweight tests without ComfyUI's server. + if "server" in str(exc).lower() or "PromptServer" in str(exc): + return + print(f"[ThinkingLLM Decision Router] Could not register Decision API model route: {exc}") + + +_register_decision_model_route() diff --git a/nodes/openai_compatible_api.py b/nodes/openai_compatible_api.py index b414f77..c8d654b 100644 --- a/nodes/openai_compatible_api.py +++ b/nodes/openai_compatible_api.py @@ -55,6 +55,7 @@ _PROFILE_ALLOWED_FIELDS = { "allowed_extra_body_fields", "send_seed", "thinking_mode", "max_tokens_field", "allow_insecure_http", "allow_images", "allow_image_url", "max_image_pixels", "max_image_bytes", + "decision_protocol", "decision_endpoint", } @@ -86,13 +87,17 @@ _BUILTIN_PROFILES = { "OpenAI": _cloud_profile( "OpenAI", "https://api.openai.com/v1", "OPENAI_API_KEY", max_tokens_field="max_completion_tokens", + decision_protocol="openai_decisions", + decision_endpoint="https://api.openai.com/v1/decisions", ), "QwenCloud (Singapore)": _cloud_profile( "QwenCloud", "https://dashscope-intl.aliyuncs.com/compatible-mode/v1", "DASHSCOPE_API_KEY", thinking_mode="qwen", ), "OpenRouter": _cloud_profile( - "OpenRouter", "https://openrouter.ai/api/v1", "OPENROUTER_API_KEY" + "OpenRouter", "https://openrouter.ai/api/v1", "OPENROUTER_API_KEY", + decision_protocol="openrouter_decisions", + decision_endpoint="https://openrouter.ai/api/alpha/decisions", ), "Together AI": _cloud_profile( "Together AI", "https://api.together.ai/v1", "TOGETHER_API_KEY" @@ -280,6 +285,15 @@ def _normalize_profile(name: str, raw: dict) -> dict: if allow_image_url and not allow_images: raise ValueError(f"API profile {profile_name!r} cannot enable allow_image_url while allow_images is false.") + decision_protocol = str(raw.get("decision_protocol") or "").strip().lower() + if decision_protocol not in {"", "openai_decisions", "openrouter_decisions"}: + raise ValueError(f"API profile {profile_name!r} has unsupported decision_protocol {decision_protocol!r}.") + decision_endpoint = str(raw.get("decision_endpoint") or "").strip() + if decision_endpoint: + decision_endpoint = _validate_base_url(decision_endpoint, auth, allow_insecure_http) + if bool(decision_protocol) != bool(decision_endpoint): + raise ValueError(f"API profile {profile_name!r} must configure both decision_protocol and decision_endpoint, or neither.") + return { "name": profile_name, "provider": provider, @@ -300,6 +314,8 @@ def _normalize_profile(name: str, raw: dict) -> dict: "allow_image_url": allow_image_url, "max_image_pixels": _profile_int(raw, "max_image_pixels", DEFAULT_MAX_IMAGE_PIXELS, maximum=100_000_000), "max_image_bytes": _profile_int(raw, "max_image_bytes", DEFAULT_MAX_IMAGE_BYTES, maximum=100_000_000), + "decision_protocol": decision_protocol, + "decision_endpoint": decision_endpoint, } diff --git a/pyproject.toml b/pyproject.toml index 4a885c1..17321d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "ComfyUI-ThinkingLLM" description = "A multimodal ComfyUI AI node pack with Qwen, Gemma 4, GGUF, Whisper, image and text decision routing, and live thinking output." -version = "3.0.1" +version = "3.0.2" license = {file = "LICENSE"} dependencies = ["transformers>=5.2.0", "torch", "huggingface-hub", "hf_xet", "psutil", "numpy", "Pillow", "opencv-python", "bitsandbytes", "accelerate", "sentencepiece", "tiktoken", "faster-whisper", "llama-cpp-python"] diff --git a/tests/test_decision_api.py b/tests/test_decision_api.py new file mode 100644 index 0000000..e4f5116 --- /dev/null +++ b/tests/test_decision_api.py @@ -0,0 +1,248 @@ +from __future__ import annotations + +import asyncio +import importlib.util +import json +import sys +import types +import unittest +from pathlib import Path +from unittest import mock + +import torch + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from thinkingllm_core import decision_api + + +ROUTES = [ + {"name": "fox", "description": "A fox."}, + {"name": "dog", "description": "A domestic dog."}, +] + + +class DecisionApiAdapterTests(unittest.TestCase): + def test_openai_request_uses_native_choice_and_input_image_schema(self): + payload = decision_api.build_decision_request( + "openai_decisions", "gpt-6-luna", "image", None, + "data:image/png;base64,AA==", "Pick the best route.", ROUTES, + ) + self.assertEqual(payload["model"], "gpt-6-luna") + self.assertEqual(payload["questions"][0]["type"], "choice") + self.assertEqual(payload["questions"][0]["choices"][0], {"value": "fox", "description": "A fox."}) + self.assertEqual(payload["input"][0]["content"][1], {"type": "input_image", "image_url": "data:image/png;base64,AA=="}) + + def test_openrouter_request_uses_native_systemone_schema(self): + payload = decision_api.build_decision_request( + "openrouter_decisions", "cloudflare/clef-flash", "image", None, + "data:image/png;base64,AA==", "Pick the best route.", ROUTES, + ) + self.assertEqual(payload["state"], [{"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}}]) + self.assertEqual(payload["questions"]["route"]["criteria"], {"fox": "A fox.", "dog": "A domestic dog."}) + + def test_openai_and_openrouter_response_schemas_normalize(self): + openai = {"answers": [{"name": "route", "type": "choice", "choice": "dog", "confidence": 0.8, + "probabilities": [{"value": "fox", "probability": 0.2}, {"value": "dog", "probability": 0.8}]}]} + openrouter = {"answers": {"route": {"type": "choice", "choice": "dog", "confidence": 0.8, + "probabilities": {"fox": 0.2, "dog": 0.8}}}} + expected = ("dog", {"fox": 0.2, "dog": 0.8}, 0.8) + self.assertEqual(decision_api.normalize_decision_response("openai_decisions", openai, ["fox", "dog"]), expected) + self.assertEqual(decision_api.normalize_decision_response("openrouter_decisions", openrouter, ["fox", "dog"]), expected) + + def test_refusal_unknown_routes_and_malformed_probabilities_fail_closed(self): + with self.assertRaisesRegex(RuntimeError, "did not return a route"): + decision_api.normalize_decision_response("openai_decisions", {"answers": [{"name": "route", "type": "refusal"}]}, ["fox", "dog"]) + with self.assertRaisesRegex(RuntimeError, "unknown route"): + decision_api.normalize_decision_response("openrouter_decisions", {"answers": {"route": {"type": "choice", "choice": "cat", "probabilities": {"fox": 0.2, "dog": 0.8}}}}, ["fox", "dog"]) + with self.assertRaisesRegex(RuntimeError, "invalid route probability"): + decision_api.normalize_decision_response("openrouter_decisions", {"answers": {"route": {"type": "choice", "choice": "fox", "probabilities": {"fox": float("nan"), "dog": 0.8}}}}, ["fox", "dog"]) + with self.assertRaisesRegex(RuntimeError, "duplicate route probabilities"): + decision_api.normalize_decision_response( + "openai_decisions", + {"answers": [{"name": "route", "type": "choice", "choice": "fox", "probabilities": [ + {"value": "fox", "probability": 0.5}, {"value": "fox", "probability": 0.4}, + {"value": "dog", "probability": 0.1}, + ]}]}, + ["fox", "dog"], + ) + + def test_openrouter_model_catalog_filters_by_required_modality(self): + response = { + "data": [ + {"id": "decision/vision", "architecture": {"input_modalities": ["text", "image"], "output_modalities": ["decisions"]}}, + {"id": "decision/text", "architecture": {"input_modalities": ["text"], "output_modalities": ["decisions"]}}, + {"id": "chat/vision", "architecture": {"input_modalities": ["text", "image"], "output_modalities": ["text"]}}, + ] + } + opener = mock.Mock() + http_response = mock.MagicMock() + http_response.read.return_value = json.dumps(response).encode() + http_response.__enter__.return_value = http_response + opener.open.return_value = http_response + decision_api._MODEL_CACHE.clear() + with mock.patch.object(decision_api, "_request_json", return_value=response): + image_models = decision_api.openrouter_decision_models("image", now=100) + text_models = decision_api.openrouter_decision_models("text", now=100) + self.assertEqual([model["id"] for model in image_models], ["decision/vision"]) + self.assertEqual([model["id"] for model in text_models], ["decision/vision", "decision/text"]) + + def test_profile_allowlist_restricts_api_model_choices(self): + with mock.patch.object(decision_api, "openrouter_decision_models", return_value=[{"id": "a"}, {"id": "b"}]): + models = decision_api.decision_models_for_profile({"decision_protocol": "openrouter_decisions", "allowed_models": ["b"]}, "text") + self.assertEqual(models, [{"id": "b"}]) + + def test_profile_image_policy_hides_vision_models(self): + with mock.patch.object(decision_api, "openrouter_decision_models") as catalog: + models = decision_api.decision_models_for_profile( + {"decision_protocol": "openrouter_decisions", "allow_images": False}, "image", + ) + self.assertEqual(models, []) + catalog.assert_not_called() + + +def load_router_module(): + module_name = "thinkingllm_decision_router_api_test" + spec = importlib.util.spec_from_file_location(module_name, ROOT / "nodes" / "decision_router.py") + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module + + +class DecisionApiRouterTests(unittest.TestCase): + def setUp(self): + self.profile = { + "name": "Test OpenAI", + "provider": "OpenAI", + "decision_protocol": "openai_decisions", + "decision_endpoint": "https://api.openai.com/v1/decisions", + "allow_images": True, + "max_image_pixels": 100, + "max_image_bytes": 100000, + } + self.api_module = types.ModuleType("openai_compatible_api") + self.api_module._load_api_profiles = lambda: {"Test OpenAI": self.profile} + self.api_module._credential_value = lambda _profile: "test-secret" + self.api_module._image_to_data_url = lambda _profile, _image: "data:image/png;base64,AA==" + self.api_module._NO_REDIRECT_OPENER = object() + def validate_input_text(profile, system_prompt, prompt): + if system_prompt is not None and not isinstance(system_prompt, str): + raise ValueError("system_prompt must be a string.") + if prompt is not None and not isinstance(prompt, str): + raise ValueError("prompt must be a string.") + total = len(system_prompt or "") + len(prompt or "") + limit = profile.get("max_input_chars", 1_000_000) + if total > limit: + raise ValueError(f"Combined input length exceeds server-side max_input_chars={limit}.") + self.api_module._validate_input_text = validate_input_text + self.previous_api_module = sys.modules.get("openai_compatible_api") + sys.modules["openai_compatible_api"] = self.api_module + + def tearDown(self): + if self.previous_api_module is None: + sys.modules.pop("openai_compatible_api", None) + else: + sys.modules["openai_compatible_api"] = self.previous_api_module + + def test_registers_model_catalog_route_and_returns_filtered_models(self): + registered = {} + + def route_decorator(path): + return lambda handler: registered.setdefault(path, handler) + + fake_server = types.ModuleType("server") + fake_server.PromptServer = types.SimpleNamespace( + instance=types.SimpleNamespace(routes=types.SimpleNamespace(get=route_decorator)), + ) + router_module = load_router_module() + with mock.patch.dict(sys.modules, {"server": fake_server}): + with mock.patch.object( + decision_api, + "decision_models_for_profile", + return_value=[{"id": "gpt-6-luna", "vision": True}], + ): + router_module._register_decision_model_route() + handler = registered["/thinkingllm/decision/models"] + response = asyncio.run(handler(types.SimpleNamespace( + query={"profile": "Test OpenAI", "input_kind": "image"}, + ))) + self.assertEqual(response.status, 200) + self.assertEqual(json.loads(response.text)["models"], [{"id": "gpt-6-luna", "vision": True}]) + + def test_api_text_batch_routes_without_loading_local_model(self): + router_module = load_router_module() + node = router_module.ThinkingLLMDecisionRouterText() + node._load_model = mock.Mock(side_effect=AssertionError("API route must not load a local model")) + node.llm = mock.Mock() + node.current_signature = ("previously-retained-local-model",) + route_fields = {"route_01_name": "fox", "route_02_name": "dog"} + api_args = {"decision_backend": "API", "api_profile": "Test OpenAI", "api_model": "gpt-6-luna"} + with ( + mock.patch.object(decision_api, "decision_models_for_profile", return_value=[{"id": "gpt-6-luna", "vision": True}]), + mock.patch.object(decision_api, "request_decision", side_effect=[("fox", {"fox": 0.9, "dog": 0.1}, 0.9), ("dog", {"fox": 0.1, "dog": 0.9}, 0.9)]), + ): + result = node.route_text(["fox text", "dog text"], "unused local", "Choose.", **api_args, **route_fields) + payload = json.loads(result[0]) + self.assertEqual(payload["provider"], "OpenAI") + self.assertEqual([item["route"] for item in payload["decisions"]], ["fox", "dog"]) + self.assertEqual(result[1], ["fox text"]) + self.assertEqual(result[2], ["dog text"]) + node._load_model.assert_not_called() + self.assertIsNone(node.llm) + self.assertIsNone(node.current_signature) + + def test_api_image_uses_profile_limited_data_url_and_blocks_empty_routes(self): + router_module = load_router_module() + node = router_module.ThinkingLLMDecisionRouter() + image = torch.zeros((1, 2, 2, 3)) + route_fields = {"route_01_name": "fox", "route_02_name": "dog"} + api_args = {"decision_backend": "API", "api_profile": "Test OpenAI", "api_model": "gpt-6-luna"} + with ( + mock.patch.object(decision_api, "decision_models_for_profile", return_value=[{"id": "gpt-6-luna", "vision": True}]), + mock.patch.object(decision_api, "request_decision", return_value=("dog", {"fox": 0.1, "dog": 0.9}, 0.9)) as request_mock, + ): + result = node.route_images(image, "unused local", "Choose.", **api_args, **route_fields) + self.assertEqual(json.loads(result[0])["decisions"][0]["route"], "dog") + self.assertEqual(request_mock.call_args.args[4], "data:image/png;base64,AA==") + self.assertIsInstance(result[1], router_module.ExecutionBlocker) + self.assertEqual(tuple(result[2].shape), (1, 2, 2, 3)) + + def test_api_mode_rejects_unknown_image_model_before_request(self): + router_module = load_router_module() + node = router_module.ThinkingLLMDecisionRouter() + with ( + mock.patch.object(decision_api, "decision_models_for_profile", return_value=[{"id": "text-only", "vision": False}]), + mock.patch.object(decision_api, "request_decision") as request_mock, + ): + with self.assertRaisesRegex(ValueError, "image-capable"): + node.route_images( + torch.zeros((1, 1, 1, 3)), "unused", "Choose.", + decision_backend="API", api_profile="Test OpenAI", api_model="text-only", + route_01_name="a", route_02_name="b", + ) + request_mock.assert_not_called() + + def test_api_mode_enforces_profile_input_limit_before_request(self): + router_module = load_router_module() + node = router_module.ThinkingLLMDecisionRouterText() + self.profile["max_input_chars"] = 20 + with ( + mock.patch.object(decision_api, "decision_models_for_profile", return_value=[{"id": "gpt-6-luna"}]), + mock.patch.object(decision_api, "request_decision") as request_mock, + ): + with self.assertRaisesRegex(ValueError, "exceeds server-side max_input_chars"): + node.route_text( + "cat", "unused", "Choose.", + decision_backend="API", api_profile="Test OpenAI", api_model="gpt-6-luna", + route_01_name="fox", route_01_description="long route description", + route_02_name="dog", + ) + request_mock.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_decision_router.py b/tests/test_decision_router.py index a11463d..4f0df61 100644 --- a/tests/test_decision_router.py +++ b/tests/test_decision_router.py @@ -478,8 +478,12 @@ class DecisionRouterNodeTests(unittest.TestCase): self.assertNotIn("text", image_node.INPUT_TYPES()["required"]) self.assertEqual(text_node.INPUT_TYPES()["required"]["text"][0], "STRING") self.assertNotIn("image", text_node.INPUT_TYPES()["required"]) - self.assertEqual(len(image_node.INPUT_TYPES()["required"]), 109) - self.assertEqual(len(text_node.INPUT_TYPES()["required"]), 109) + self.assertEqual(len(image_node.INPUT_TYPES()["required"]), 112) + self.assertEqual(len(text_node.INPUT_TYPES()["required"]), 112) + required_names = list(image_node.INPUT_TYPES()["required"]) + self.assertEqual(required_names[:5], ["image", "decision_model", "keep_model_loaded", "include_reason", "system_prompt"]) + self.assertEqual(required_names[-3:], ["decision_backend", "api_profile", "api_model"]) + self.assertEqual(image_node.INPUT_TYPES()["required"]["decision_backend"][1]["default"], "Local (in-process)") self.assertEqual(image_node.INPUT_TYPES()["required"]["keep_model_loaded"][1]["default"], False) self.assertEqual(text_node.INPUT_TYPES()["required"]["keep_model_loaded"][1]["default"], False) self.assertEqual(image_node.INPUT_TYPES()["required"]["include_reason"][1]["default"], False) diff --git a/tests/test_openai_api_security.py b/tests/test_openai_api_security.py index 3df80e9..e2675da 100644 --- a/tests/test_openai_api_security.py +++ b/tests/test_openai_api_security.py @@ -93,6 +93,18 @@ class SecurityTests(unittest.TestCase): 'api_key_env': 'TEST_KEY', }) + def test_native_decision_profile_requires_a_supported_protocol_and_endpoint_pair(self): + profile = self.profile( + decision_protocol='openrouter_decisions', + decision_endpoint='https://openrouter.ai/api/alpha/decisions', + ) + self.assertEqual(profile['decision_protocol'], 'openrouter_decisions') + self.assertEqual(profile['decision_endpoint'], 'https://openrouter.ai/api/alpha/decisions') + with self.assertRaisesRegex(ValueError, 'both decision_protocol and decision_endpoint'): + self.profile(decision_protocol='openai_decisions') + with self.assertRaisesRegex(ValueError, 'unsupported decision_protocol'): + self.profile(decision_protocol='chat_json', decision_endpoint='https://example.test/decisions') + def test_plain_http_non_loopback_requires_explicit_opt_in(self): with self.assertRaisesRegex(ValueError, 'Plain HTTP'): api_node._normalize_profile('LAN', { diff --git a/thinkingllm_api_profiles.example.json b/thinkingllm_api_profiles.example.json index 2b9d5b2..9287551 100644 --- a/thinkingllm_api_profiles.example.json +++ b/thinkingllm_api_profiles.example.json @@ -3,6 +3,8 @@ "OpenRouter Production": { "provider": "OpenRouter", "base_url": "https://openrouter.ai/api/v1", + "decision_protocol": "openrouter_decisions", + "decision_endpoint": "https://openrouter.ai/api/alpha/decisions", "auth": "bearer_env", "api_key_env": "OPENROUTER_PRODUCTION_API_KEY", "allowed_models": [ diff --git a/thinkingllm_core/decision_api.py b/thinkingllm_core/decision_api.py new file mode 100644 index 0000000..93e0784 --- /dev/null +++ b/thinkingllm_core/decision_api.py @@ -0,0 +1,193 @@ +"""Native OpenAI Decisions and OpenRouter Decisions API adapters.""" + +from __future__ import annotations + +import json +import math +import time +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Mapping + + +OPENAI_DECISION_MODEL = "gpt-6-luna" +OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models" +MODEL_CACHE_TTL_SECONDS = 3600 +_MODEL_CACHE: dict[str, tuple[float, list[dict]]] = {} + + +def _request_json(url: str, *, headers: dict[str, str], payload=None, timeout: int = 60, opener=None): + data = None if payload is None else json.dumps(payload, ensure_ascii=False).encode("utf-8") + request_headers = {"Accept": "application/json", **headers} + if data is not None: + request_headers["Content-Type"] = "application/json" + req = urllib.request.Request(url, data=data, headers=request_headers, method="GET" if data is None else "POST") + try: + response = (opener or urllib.request.build_opener()).open(req, timeout=timeout) + with response: + raw = response.read() + except urllib.error.HTTPError as exc: + # Do not include provider response text: it can echo user content or media. + raise RuntimeError(f"Decision API returned HTTP {exc.code}.") from exc + except urllib.error.URLError as exc: + raise RuntimeError(f"Could not reach the Decision API ({type(exc.reason).__name__}).") from exc + try: + result = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError("Decision API returned invalid JSON.") from exc + if not isinstance(result, dict): + raise RuntimeError("Decision API returned an unexpected response shape.") + if result.get("error"): + raise RuntimeError("Decision API reported an inference error.") + return result + + +def _route_criteria(routes): + return {str(route["name"]): str(route.get("description") or "") or None for route in routes} + + +def build_decision_request(protocol: str, model: str, input_kind: str, item, image_data_url: str | None, system_prompt: str, routes): + criteria = _route_criteria(routes) + instruction = str(system_prompt or "").strip() or "Choose the single route that best matches the input." + if protocol == "openai_decisions": + content = [{"type": "input_text", "text": str(item) if input_kind == "text" else "Classify the supplied image."}] + if image_data_url: + content.append({"type": "input_image", "image_url": image_data_url}) + return { + "model": model, + "input": [{"role": "user", "content": content}], + "questions": [{ + "type": "choice", + "name": "route", + "instructions": instruction, + "choices": [{"value": name, "description": description or ""} for name, description in criteria.items()], + }], + } + if protocol == "openrouter_decisions": + state = str(item) if input_kind == "text" else [] + if input_kind == "image": + state = [{"type": "image_url", "image_url": {"url": image_data_url}}] + return { + "model": model, + "state": state, + "questions": { + "route": {"type": "choice", "instructions": instruction, "criteria": criteria}, + }, + } + raise ValueError(f"Unsupported Decision API protocol: {protocol!r}.") + + +def _probability_map(raw, route_names): + if isinstance(raw, Mapping): + values = {str(key): value for key, value in raw.items()} + elif isinstance(raw, list): + values = {} + for item in raw: + if not isinstance(item, dict) or "value" not in item: + raise RuntimeError("Decision API returned malformed route probabilities.") + value = str(item["value"]) + if value in values: + raise RuntimeError("Decision API returned duplicate route probabilities.") + values[value] = item.get("probability") + else: + raise RuntimeError("Decision API returned no route probabilities.") + if set(values) != set(route_names): + raise RuntimeError("Decision API probabilities do not match the configured route names.") + normalized = {} + for name in route_names: + value = values[name] + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(float(value)) or not 0 <= float(value) <= 1: + raise RuntimeError("Decision API returned an invalid route probability.") + normalized[name] = float(value) + return normalized + + +def normalize_decision_response(protocol: str, response: dict, route_names: list[str]): + if protocol == "openai_decisions": + answers = response.get("answers") + answer = next((item for item in answers if isinstance(item, dict) and item.get("name") == "route"), None) if isinstance(answers, list) else None + if not isinstance(answer, dict) or answer.get("type") != "choice": + raise RuntimeError("OpenAI Decisions API did not return a route choice.") + choice = answer.get("choice") + probability_map = _probability_map(answer.get("probabilities"), route_names) + confidence = answer.get("confidence") + elif protocol == "openrouter_decisions": + answers = response.get("answers") + answer = answers.get("route") if isinstance(answers, dict) else None + if not isinstance(answer, dict) or answer.get("type") != "choice": + raise RuntimeError("OpenRouter Decisions API did not return a route choice.") + choice = answer.get("choice") + probability_map = _probability_map(answer.get("probabilities"), route_names) + confidence = answer.get("confidence") + else: + raise ValueError(f"Unsupported Decision API protocol: {protocol!r}.") + if not isinstance(choice, str) or choice not in route_names: + raise RuntimeError("Decision API selected an unknown route.") + if confidence is not None and (isinstance(confidence, bool) or not isinstance(confidence, (int, float)) or not math.isfinite(float(confidence)) or not 0 <= float(confidence) <= 1): + raise RuntimeError("Decision API returned an invalid confidence value.") + return choice, probability_map, None if confidence is None else float(confidence) + + +def request_decision(profile: dict, model: str, input_kind: str, item, image_data_url: str | None, system_prompt: str, routes, *, api_key: str, opener=None): + protocol = profile.get("decision_protocol") + payload = build_decision_request(protocol, model, input_kind, item, image_data_url, system_prompt, routes) + response = _request_json( + profile["decision_endpoint"], + headers={"Authorization": f"Bearer {api_key}"} if api_key else {}, + payload=payload, + timeout=int(profile.get("max_timeout_seconds", 300)), + opener=opener, + ) + return normalize_decision_response(protocol, response, [str(route["name"]) for route in routes]) + + +def openrouter_decision_models(input_kind: str, *, timeout: int = 20, now: float | None = None, opener=None): + """Return OpenRouter model ids with decision output and requested input capability.""" + modality = "image" if input_kind == "image" else "text" + now = time.monotonic() if now is None else now + cached = _MODEL_CACHE.get(modality) + if cached and now - cached[0] < MODEL_CACHE_TTL_SECONDS: + return [dict(model) for model in cached[1]] + query = urllib.parse.urlencode({"output_modalities": "decisions", "input_modalities": modality}) + try: + data = _request_json(f"{OPENROUTER_MODELS_URL}?{query}", headers={}, timeout=timeout, opener=opener) + items = data.get("data") + if not isinstance(items, list): + raise RuntimeError("OpenRouter model catalog returned an unexpected response shape.") + models = [] + for item in items: + if not isinstance(item, dict) or not isinstance(item.get("id"), str): + continue + architecture = item.get("architecture") + modalities = architecture.get("input_modalities") if isinstance(architecture, dict) else None + output_modalities = architecture.get("output_modalities") if isinstance(architecture, dict) else None + if not isinstance(modalities, list) or modality not in modalities: + continue + if not isinstance(output_modalities, list) or "decisions" not in output_modalities: + continue + models.append({"id": item["id"], "label": item.get("name") or item["id"], "vision": "image" in modalities}) + if not models: + raise RuntimeError("OpenRouter returned no decision models for the requested input type.") + _MODEL_CACHE[modality] = (now, models) + return [dict(model) for model in models] + except RuntimeError: + if cached: + return [dict(model) for model in cached[1]] + raise + + +def decision_models_for_profile(profile: dict, input_kind: str): + if input_kind == "image" and not profile.get("allow_images", False): + return [] + protocol = profile.get("decision_protocol") + if protocol == "openai_decisions": + models = [{"id": OPENAI_DECISION_MODEL, "label": "GPT-6 Luna Decisions", "vision": True}] + elif protocol == "openrouter_decisions": + models = openrouter_decision_models(input_kind) + else: + return [] + allowed = profile.get("allowed_models") + if allowed is not None: + models = [model for model in models if model["id"] in allowed] + return models diff --git a/update.md b/update.md index 7ba1109..37232d6 100644 --- a/update.md +++ b/update.md @@ -1,5 +1,23 @@ # ComfyUI-QwenVL Update Log +## Version 3.0.2 (2026/10/29) - Native Decision API Backends + +### 3.0.2 Changes + +- Added an optional native Decisions API backend to both image and text Decision Router nodes. Local in-process llama.cpp inference remains the default. +- Added direct OpenAI Decisions support with `gpt-6-luna` and dynamic OpenRouter Decisions model discovery, filtered by decision output, input modality, profile image policy, and optional model allowlists. +- Preserved typed route selection and provider probabilities in `decision_json`; image items are encoded within profile limits and each batch item receives its own request. The router does not fall back to chat-generated JSON. +- Reused server-side API profiles and environment-based credentials, enforced each profile's input-size and timeout limits, and kept credentials out of workflows. +- Kept existing widget order stable for saved workflows. Switching to API mode releases a locally retained decision model. +- Added API configuration documentation and updated the README. + +### 3.0.2 Validation + +- All 240 tests passed across the six relevant test modules when run individually in the bundled ComfyUI Python 3.12 runtime, including 12 Decision API adapter/router tests, 29 local Decision Router tests, and 38 API security/profile tests. +- Python compilation, both JavaScript syntax checks, JSON/TOML and mirrored-version checks, API model-catalog validation, and `git diff --check` passed. +- A live public OpenRouter model-catalog request returned decision models with text/image capabilities, confirming the dynamic modality filter against the current catalog. +- Provider inference tests use mocked responses; no authenticated OpenAI or OpenRouter inference request was made. A previous combined `unittest discover` attempt showed test-order/native-extension interference, so the six modules were verified in separate processes. + ## Version 3.0.1 (2026/10/29) - Decision Router VRAM and Explanation Controls ### 3.0.1 Changes diff --git a/web/js/decision_router.js b/web/js/decision_router.js index c749de3..613c563 100644 --- a/web/js/decision_router.js +++ b/web/js/decision_router.js @@ -5,6 +5,66 @@ const NODE_OUTPUT_TYPES = { ThinkingLLM_DecisionRouterText: "STRING", }; const MAX_ROUTES = 52; +const API_MODELS_URL = "/thinkingllm/decision/models"; +const apiModelRequestIds = new WeakMap(); + +function findWidget(node, name) { + return node.widgets?.find((widget) => widget.name === name); +} + +function widgetValue(widget) { + return String(widget?.value ?? "").trim(); +} + +async function refreshApiModels(node, inputKind) { + const profileWidget = findWidget(node, "api_profile"); + const modelWidget = findWidget(node, "api_model"); + if (!profileWidget || !modelWidget || widgetValue(findWidget(node, "decision_backend")) !== "API") return; + const profile = widgetValue(profileWidget); + const requestId = (apiModelRequestIds.get(node) ?? 0) + 1; + apiModelRequestIds.set(node, requestId); + const query = new URLSearchParams({ profile, input_kind: inputKind }); + try { + const response = await fetch(`${API_MODELS_URL}?${query.toString()}`); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const payload = await response.json(); + if (apiModelRequestIds.get(node) !== requestId + || widgetValue(findWidget(node, "decision_backend")) !== "API" + || widgetValue(findWidget(node, "api_profile")) !== profile) return; + const ids = (Array.isArray(payload.models) ? payload.models : []) + .map((model) => String(model?.id ?? "")).filter(Boolean); + if (!ids.length) throw new Error("No compatible decision models returned"); + modelWidget.type = "combo"; + modelWidget.serialize = true; + modelWidget.options ??= {}; + modelWidget.options.values = ids; + if (!ids.includes(widgetValue(modelWidget))) { + modelWidget.value = ids[0]; + modelWidget.options.value = ids[0]; + } + } catch (error) { + if (apiModelRequestIds.get(node) !== requestId + || widgetValue(findWidget(node, "decision_backend")) !== "API" + || widgetValue(findWidget(node, "api_profile")) !== profile) return; + console.warn("[ThinkingLLM Decision Router] Could not load API decision models.", error); + modelWidget.type = "text"; + } + node.size = node.computeSize?.() ?? node.size; + node.setDirtyCanvas?.(true, true); +} + +function synchronizeBackend(node, inputKind) { + const apiMode = widgetValue(findWidget(node, "decision_backend")) === "API"; + for (const name of ["decision_model", "api_profile", "api_model", "keep_model_loaded", "include_reason"]) { + const widget = findWidget(node, name); + if (widget) widget.hidden = apiMode + ? ["decision_model", "keep_model_loaded", "include_reason"].includes(name) + : ["api_profile", "api_model"].includes(name); + } + if (apiMode) refreshApiModels(node, inputKind); + node.size = node.computeSize?.() ?? node.size; + node.setDirtyCanvas?.(true, true); +} function widgetName(index, field) { return `route_${String(index + 1).padStart(2, "0")}_${field}`; @@ -97,6 +157,7 @@ app.registerExtension({ async beforeRegisterNodeDef(nodeType, nodeData) { const routeType = NODE_OUTPUT_TYPES[nodeData.name]; if (!routeType) return; + const inputKind = nodeData.name === "ThinkingLLM_DecisionRouter" ? "image" : "text"; const originalCreated = nodeType.prototype.onNodeCreated; nodeType.prototype.onNodeCreated = function () { @@ -104,6 +165,15 @@ app.registerExtension({ const node = this; return afterLifecycle(node, routeType, result, (createdNode, outputType) => { for (const widget of createdNode.widgets ?? []) { + if (["decision_backend", "api_profile"].includes(widget.name)) { + const originalCallback = widget.callback; + widget.callback = function (value, ...args) { + const callbackResult = originalCallback?.call(this, value, ...args); + synchronizeBackend(createdNode, inputKind); + return callbackResult; + }; + continue; + } if (!widget.name?.startsWith("route_")) continue; const originalCallback = widget.callback; widget.callback = function (value, ...args) { @@ -112,13 +182,17 @@ app.registerExtension({ }; } scheduleSynchronize(createdNode, outputType); + synchronizeBackend(createdNode, inputKind); }); }; const originalConfigure = nodeType.prototype.onConfigure; nodeType.prototype.onConfigure = function () { const result = originalConfigure?.apply(this, arguments); - return afterLifecycle(this, routeType, result, scheduleSynchronize); + return afterLifecycle(this, routeType, result, (configuredNode, outputType) => { + scheduleSynchronize(configuredNode, outputType); + synchronizeBackend(configuredNode, inputKind); + }); }; }, }); diff --git a/web_commercial/js/decision_router.js b/web_commercial/js/decision_router.js index 945e058..0e45b64 100644 --- a/web_commercial/js/decision_router.js +++ b/web_commercial/js/decision_router.js @@ -5,6 +5,66 @@ const NODE_OUTPUT_TYPES = { ThinkingLLM_DecisionRouterText: "STRING", }; const MAX_ROUTES = 52; +const API_MODELS_URL = "/thinkingllm/decision/models"; +const apiModelRequestIds = new WeakMap(); + +function findWidget(node, name) { + return node.widgets?.find((widget) => widget.name === name); +} + +function widgetValue(widget) { + return String(widget?.value ?? "").trim(); +} + +async function refreshApiModels(node, inputKind) { + const profileWidget = findWidget(node, "api_profile"); + const modelWidget = findWidget(node, "api_model"); + if (!profileWidget || !modelWidget || widgetValue(findWidget(node, "decision_backend")) !== "API") return; + const profile = widgetValue(profileWidget); + const requestId = (apiModelRequestIds.get(node) ?? 0) + 1; + apiModelRequestIds.set(node, requestId); + const query = new URLSearchParams({ profile, input_kind: inputKind }); + try { + const response = await fetch(`${API_MODELS_URL}?${query.toString()}`); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const payload = await response.json(); + if (apiModelRequestIds.get(node) !== requestId + || widgetValue(findWidget(node, "decision_backend")) !== "API" + || widgetValue(findWidget(node, "api_profile")) !== profile) return; + const ids = (Array.isArray(payload.models) ? payload.models : []) + .map((model) => String(model?.id ?? "")).filter(Boolean); + if (!ids.length) throw new Error("No compatible decision models returned"); + modelWidget.type = "combo"; + modelWidget.serialize = true; + modelWidget.options ??= {}; + modelWidget.options.values = ids; + if (!ids.includes(widgetValue(modelWidget))) { + modelWidget.value = ids[0]; + modelWidget.options.value = ids[0]; + } + } catch (error) { + if (apiModelRequestIds.get(node) !== requestId + || widgetValue(findWidget(node, "decision_backend")) !== "API" + || widgetValue(findWidget(node, "api_profile")) !== profile) return; + console.warn("[ThinkingLLM Decision Router] Could not load API decision models.", error); + modelWidget.type = "text"; + } + node.size = node.computeSize?.() ?? node.size; + node.setDirtyCanvas?.(true, true); +} + +function synchronizeBackend(node, inputKind) { + const apiMode = widgetValue(findWidget(node, "decision_backend")) === "API"; + for (const name of ["decision_model", "api_profile", "api_model", "keep_model_loaded", "include_reason"]) { + const widget = findWidget(node, name); + if (widget) widget.hidden = apiMode + ? ["decision_model", "keep_model_loaded", "include_reason"].includes(name) + : ["api_profile", "api_model"].includes(name); + } + if (apiMode) refreshApiModels(node, inputKind); + node.size = node.computeSize?.() ?? node.size; + node.setDirtyCanvas?.(true, true); +} function widgetName(index, field) { return `route_${String(index + 1).padStart(2, "0")}_${field}`; @@ -97,6 +157,7 @@ app.registerExtension({ async beforeRegisterNodeDef(nodeType, nodeData) { const routeType = NODE_OUTPUT_TYPES[nodeData.name]; if (!routeType) return; + const inputKind = nodeData.name === "ThinkingLLM_DecisionRouter" ? "image" : "text"; const originalCreated = nodeType.prototype.onNodeCreated; nodeType.prototype.onNodeCreated = function () { @@ -104,6 +165,15 @@ app.registerExtension({ const node = this; return afterLifecycle(node, routeType, result, (createdNode, outputType) => { for (const widget of createdNode.widgets ?? []) { + if (["decision_backend", "api_profile"].includes(widget.name)) { + const originalCallback = widget.callback; + widget.callback = function (value, ...args) { + const callbackResult = originalCallback?.call(this, value, ...args); + synchronizeBackend(createdNode, inputKind); + return callbackResult; + }; + continue; + } if (!widget.name?.startsWith("route_")) continue; const originalCallback = widget.callback; widget.callback = function (value, ...args) { @@ -112,13 +182,17 @@ app.registerExtension({ }; } scheduleSynchronize(createdNode, outputType); + synchronizeBackend(createdNode, inputKind); }); }; const originalConfigure = nodeType.prototype.onConfigure; nodeType.prototype.onConfigure = function () { const result = originalConfigure?.apply(this, arguments); - return afterLifecycle(this, routeType, result, scheduleSynchronize); + return afterLifecycle(this, routeType, result, (configuredNode, outputType) => { + scheduleSynchronize(configuredNode, outputType); + synchronizeBackend(configuredNode, inputKind); + }); }; }, });