diff --git a/README.md b/README.md index 541ff30..9544dc6 100644 --- a/README.md +++ b/README.md @@ -51,8 +51,8 @@ The Decision Router needs native llama.cpp decision APIs that are not present in | `ThinkingLLM Prompt Enhancer` | Transformers / HF | Text prompt enhancement | | `ThinkingLLM (GGUF)` | llama.cpp | GGUF image and video understanding | | `ThinkingLLM (GGUF Advanced)` | llama.cpp | GGUF advanced controls and `MASK_PREVIEW` | -| `ThinkingLLM Decision Router (Image)` | llama.cpp | Image decisions routed to named branches | -| `ThinkingLLM Decision Router (Text)` | llama.cpp | Text decisions routed to named branches | +| `ThinkingLLM Decision Router (Image)` | llama.cpp / OpenAI / OpenRouter | Image decisions routed to named branches | +| `ThinkingLLM Decision Router (Text)` | llama.cpp / OpenAI / OpenRouter | Text decisions routed to named branches | | `ThinkingLLM Gemma 4 Audio (GGUF)` | llama.cpp | Gemma 4 audio understanding | | `ThinkingLLM Whisper ASR` | faster-whisper | Speech-to-text transcription | | `ThinkingLLM Prompt Enhancer (GGUF)` | llama.cpp | GGUF prompt enhancement | @@ -63,7 +63,9 @@ The Transformers/HF nodes are the recommended default when you want the simplest ### Decision Router -The separate Image and Text Decision Router nodes classify their input and send each image or text item to one named route. Configure a system prompt plus route names and descriptions for any task, such as subject, document, or product classification. They offer Clef-Flash 9B, Clef 27B, and OpenJev 27B GGUFs. Model files download on first use into `ComfyUI/models/LLM/GGUF`; image routing also downloads the matching vision projector. Both run in-process without a separate llama.cpp server. Each node exposes up to 52 configured route sockets and returns per-item decisions and normalized route probabilities on `decision_json`. Empty route outputs are blocked, so their downstream branches do not execute. `keep_model_loaded` controls whether the model stays resident between runs (off by default); `include_reason` optionally adds a short post-hoc explanation to each JSON decision without changing the route or scores. +The separate Image and Text Decision Router nodes classify their input and send each image or text item to one named route. Configure a system prompt plus route names and descriptions for any task, such as subject, document, or product classification. They offer Clef-Flash 9B, Clef 27B, and OpenJev 27B GGUFs. Model files download on first use into `ComfyUI/models/LLM/GGUF`; image routing also downloads the matching vision projector. Local inference runs in-process without a separate llama.cpp server. Each node exposes up to 52 configured route sockets and returns per-item decisions and normalized route probabilities on `decision_json`. Empty route outputs are blocked, so their downstream branches do not execute. `keep_model_loaded` controls whether the model stays resident between runs (off by default); `include_reason` optionally adds a short post-hoc explanation to each JSON decision without changing the route or scores. + +You can also select the native **API** backend on either router. It supports OpenAI Decisions (`gpt-6-luna`) and OpenRouter's Decisions API; OpenRouter models are listed according to their text/image capabilities. API credentials stay in server-side profiles/environment variables rather than workflow files. See the [Decision Router API guide](docs/DECISION_ROUTER_API.md) for setup and limits. Clef models require llama.cpp's native decision-batch APIs; OpenJev uses llama.cpp's final-position logits API. Image routing additionally requires the MTMD vision APIs; text routing does not. Each node checks the selected capabilities before downloading model weights and reports missing symbols with an installation hint. See [Decision Router llama.cpp requirements](docs/LLAMA_CPP_SYSTEMONE_INSTALL.md) for the runtime details. OpenJev is CC BY-NC 4.0 and is hidden in commercial builds; Clef models are Apache-2.0. @@ -112,6 +114,7 @@ See [Models and backends](docs/MODELS_AND_BACKENDS.md) for exact families, model | Supported models, discovery, locations, and backend selection | [Models and backends](docs/MODELS_AND_BACKENDS.md) | | GGUF / llama.cpp installation on Windows and Linux | [llama.cpp backend installation](docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md) | | Decision model runtime requirements | [Decision Router llama.cpp requirements](docs/LLAMA_CPP_SYSTEMONE_INSTALL.md) | +| Native OpenAI and OpenRouter decision APIs | [Decision Router API guide](docs/DECISION_ROUTER_API.md) | | Duration-aware LTX and MiniMax H3 prompting | [Duration-aware video prompts](docs/VIDEO_PROMPT_DURATION.md) | | Video preset recommendations and rationale | [Video preset settings](docs/VIDEO_PRESET_SETTINGS.md) | | OpenAI-compatible API node | [API setup and security](docs/OPENAI_COMPATIBLE_API.md) | diff --git a/__init__.py b/__init__.py index d6a7ead..4454d64 100644 --- a/__init__.py +++ b/__init__.py @@ -3,7 +3,7 @@ "name": "ComfyUI-ThinkingLLM", "description": "A multimodal ComfyUI AI node pack with Qwen, Gemma 4, GGUF, Whisper, image and text decision routing, and live thinking output.", "author": "goodguy1963", - "version": "3.0.1", + "version": "3.0.2", "url": "https://github.com/goodguy1963/ComfyUI-ThinkingLLM", "category": "image" } diff --git a/custom_nodes.json b/custom_nodes.json index 0b133ac..b1ac437 100644 --- a/custom_nodes.json +++ b/custom_nodes.json @@ -9,5 +9,5 @@ "description": "A multimodal ComfyUI AI node pack with Qwen, Gemma 4, GGUF, Whisper, image and text decision routing, and live thinking output.", "category": "image", "tags": ["vision", "language", "multimodal", "thinkingllm", "decision-router", "qwen", "gguf", "flash_attention", "performance", "wan2.2", "video", "i2v", "t2v", "storyboard", "caching", "fork", "gpl-3.0"], - "version": "3.0.1" + "version": "3.0.2" } diff --git a/docs/DECISION_ROUTER_API.md b/docs/DECISION_ROUTER_API.md new file mode 100644 index 0000000..df46922 --- /dev/null +++ b/docs/DECISION_ROUTER_API.md @@ -0,0 +1,52 @@ +# Decision Router API mode + +The Image and Text Decision Router nodes support both local in-process inference and native API inference. Local remains the default. API mode sends each item in a batch as a separate decision request and routes the original input item according to the provider's typed `choice` response. + +## Supported native providers + +| Profile | Protocol | Endpoint | Initial model selection | Image input | +| --- | --- | --- | --- | --- | +| OpenAI | `openai_decisions` | `https://api.openai.com/v1/decisions` | `gpt-6-luna` | Supported | +| OpenRouter | `openrouter_decisions` | `https://openrouter.ai/api/alpha/decisions` | Decision models returned by its Models API | Only models whose input modalities include `image` | + +Other providers in the OpenAI-compatible chat node are not exposed as native Decision APIs by this feature. The router does not fall back to a chat completion that generates JSON. + +## Credentials and profile setup + +The built-in `OpenAI` and `OpenRouter` profiles use `OPENAI_API_KEY` and `OPENROUTER_API_KEY`. Set the key in the environment of the process that launches ComfyUI, then restart ComfyUI. Restart after adding or changing a custom profile as well, so the node's profile list is refreshed. The key and endpoint do not enter the workflow. + +Custom profiles can opt into a native protocol using the server-side profile file: + +```json +{ + "profiles": { + "OpenRouter Decision Production": { + "provider": "OpenRouter", + "base_url": "https://openrouter.ai/api/v1", + "decision_protocol": "openrouter_decisions", + "decision_endpoint": "https://openrouter.ai/api/alpha/decisions", + "auth": "bearer_env", + "api_key_env": "OPENROUTER_PRODUCTION_API_KEY", + "allowed_models": ["cloudflare/clef-flash", "typesafe/jev-1.14"], + "allow_images": true, + "max_image_pixels": 16777216, + "max_image_bytes": 20000000, + "max_timeout_seconds": 300 + } + } +} +``` + +`decision_protocol` and `decision_endpoint` must appear together. The endpoint is trusted server configuration, validated at profile load, and is not a node input. For shared ComfyUI servers, use an exact `allowed_models` list. + +## Model and image capability handling + +The node's API model list is filtered by decision output, required input modality, the profile's `allow_images` policy, and its optional `allowed_models` list. OpenRouter metadata is cached for up to one hour. If a stale cached list exists during a metadata outage, it can still be displayed; the selected model is checked against that list before inference. A model absent from the capability catalog is rejected rather than guessed to support images. + +Image tensors are converted to bounded PNG data URLs using the selected server profile's `allow_images`, `max_image_pixels`, and `max_image_bytes` settings. OpenAI receives an `input_image` part; OpenRouter receives an `image_url` part in `state`. The profile's `max_input_chars` also bounds the system prompt, input text, and configured route names/descriptions. Provider-specific image limits can still apply and are reported as inference errors. + +## Output and limits + +The APIs return typed route selections and probabilities, which are normalized into the router's existing `decision_json` output. Provider confidence is included when returned. Route descriptions become the choice criteria, and the system prompt becomes the decision instruction. The existing 52-route cap and dynamic output sockets continue to apply. + +Native Decisions APIs do not generate a rationale. The local-only `include_reason` control is hidden in API mode; if it remains enabled in a saved workflow, the result explicitly marks the reason as unavailable. `keep_model_loaded` has no effect on remote inference, and switching to API mode releases a local model previously retained by the node. diff --git a/docs/OPENAI_COMPATIBLE_API.md b/docs/OPENAI_COMPATIBLE_API.md index 400abd4..dcbda68 100644 --- a/docs/OPENAI_COMPATIBLE_API.md +++ b/docs/OPENAI_COMPATIBLE_API.md @@ -253,6 +253,8 @@ The string `OPENROUTER_PRODUCTION_API_KEY` is only the **name** of an environmen | `thinking_mode` | Currently empty or `qwen`. | | `max_tokens_field` | `max_tokens` or `max_completion_tokens`. | | `allow_insecure_http` | Explicit opt-in for unauthenticated non-loopback HTTP. | +| `decision_protocol` | Optional native Decision API adapter: `openai_decisions` or `openrouter_decisions`. Profiles without it cannot be selected by the Decision Router API mode. | +| `decision_endpoint` | Optional fixed HTTPS endpoint paired with `decision_protocol`; this is server-side profile data and is never workflow-controlled. | Unknown profile fields are rejected instead of silently ignored. This is intentional: a typo such as `max_token_limit` must not quietly disable a security limit. @@ -260,6 +262,14 @@ A custom profile with the same name as a built-in is treated as an override. If Literal secret/header fields such as `api_key`, `token`, `Authorization`, `headers`, or `secret` are rejected. +### Native Decisions API profiles + +The Image and Text Decision Router nodes can use the built-in OpenAI and OpenRouter profiles in API mode. These profiles use their dedicated Decisions APIs, not `/chat/completions`. To enable a custom profile, set both `decision_protocol` and `decision_endpoint`, keep the credential in the configured environment variable, and configure an exact `allowed_models` list for shared deployments. OpenAI uses `openai_decisions` with `/v1/decisions`; OpenRouter uses `openrouter_decisions` with `/api/alpha/decisions`. See the [Decision Router API guide](DECISION_ROUTER_API.md) for configuration details. + +OpenAI currently exposes `gpt-6-luna` through its Decisions API. OpenRouter model choices are fetched from its model catalog and filtered by `output_modalities=decisions` plus this node's input modality. Image support is a model capability, not a provider-wide setting. If model metadata is unavailable, the UI may retain a previously fetched list, but the server validates the selected model before making an inference request. + +Each input item in a ComfyUI batch is sent as its own native decision request. Local mode remains the default and does not require API credentials. `keep_model_loaded` and generated rationale output apply only to local models; native Decisions APIs return typed choices and probabilities, not free-form explanations. + --- ## Production lock-down mode diff --git a/nodes/decision_router.py b/nodes/decision_router.py index a9bef42..5feb171 100644 --- a/nodes/decision_router.py +++ b/nodes/decision_router.py @@ -76,6 +76,16 @@ def _model_options() -> list[str]: return COMMUNITY_MODEL_OPTIONS.copy() +def _api_profile_options() -> list[str]: + try: + from openai_compatible_api import _load_api_profiles + + profiles = _load_api_profiles() + except Exception: + return ["No Decision API profiles configured"] + return [name for name, profile in profiles.items() if profile.get("decision_protocol")] or ["No Decision API profiles configured"] + + def _gguf_base_dir() -> Path: if folder_paths is None: raise RuntimeError("ComfyUI folder_paths is unavailable; the Decision Router must run inside ComfyUI.") @@ -294,6 +304,19 @@ class _DecisionRouterBase: "STRING", {"default": "", "multiline": False}, ) + # Append new controls so saved workflows keep their original positional widget values. + required["decision_backend"] = ( + ["Local (in-process)", "API"], + {"default": "Local (in-process)", "tooltip": "Use the local native model or a server-configured native Decisions API."}, + ) + required["api_profile"] = ( + _api_profile_options(), + {"tooltip": "Server-side API profile. Only profiles explicitly configured for OpenAI Decisions or OpenRouter Decisions are available."}, + ) + required["api_model"] = ( + "STRING", + {"default": "gpt-6-luna", "multiline": False, "tooltip": "Decision model ID. The UI lists models matching this node's input type."}, + ) return {"required": required} CATEGORY = "ThinkingLLM/Decision" @@ -403,26 +426,93 @@ class _DecisionRouterBase: if len(routes) > len("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"): raise ValueError("The selected native decision protocol supports at most 52 routes.") - model_key = MODEL_KEYS_BY_DISPLAY.get(str(decision_model)) - if model_key is None or (COMMERCIAL_RELEASE and model_key == "openjev"): - raise ValueError(f"Decision model is unavailable in this build: {decision_model!r}.") - + api_backend = str(kwargs.get("decision_backend", "Local (in-process)")) == "API" keep_model_loaded = bool(kwargs.get("keep_model_loaded", False)) + if api_backend and (self.llm is not None or self.chat_handler is not None): + self.clear() if item_count == 0: - if not keep_model_loaded: + if not api_backend and not keep_model_loaded: self.clear() + selected_model = str(kwargs.get("api_model", "")).strip() if api_backend else str(decision_model) outputs = [ExecutionBlocker(None) for _ in range(MAX_DECISION_ROUTES)] - return (json.dumps({"model": decision_model, "input_type": input_kind, "decisions": []}, ensure_ascii=False), *outputs) + return (json.dumps({"model": selected_model, "input_type": input_kind, "decisions": []}, ensure_ascii=False), *outputs) + model_key = None + api_profile = None + api_model = "" + api_key = "" + if api_backend: + try: + from openai_compatible_api import ( + _credential_value, + _image_to_data_url, + _load_api_profiles, + _NO_REDIRECT_OPENER, + _validate_input_text, + ) + from thinkingllm_core.decision_api import decision_models_for_profile, request_decision + except Exception as exc: + raise RuntimeError("Decision API support could not be loaded. Restart ComfyUI and check the API node installation.") from exc + profile_name = str(kwargs.get("api_profile", "")).strip() + api_profile = _load_api_profiles().get(profile_name) + if not api_profile or not api_profile.get("decision_protocol"): + raise ValueError(f"API profile {profile_name!r} is not configured for a native Decisions API.") + api_model = str(kwargs.get("api_model", "")).strip() + if not api_model: + raise ValueError("Select a Decision API model.") + api_key = _credential_value(api_profile) + api_opener = _NO_REDIRECT_OPENER + if use_image: + if not api_profile.get("allow_images", False): + raise ValueError(f"Image input is disabled by API profile {profile_name!r}.") + supported_models = decision_models_for_profile(api_profile, input_kind) + selected_api_model = next((item for item in supported_models if item.get("id") == api_model), None) + if selected_api_model is None: + modality = "image-capable " if use_image else "" + raise ValueError(f"Model {api_model!r} is not a listed {modality}Decision API model allowed by profile {profile_name!r}.") + if use_image and not selected_api_model.get("vision", False): + raise ValueError(f"Decision API model {api_model!r} is not an image-capable model.") + else: + model_key = MODEL_KEYS_BY_DISPLAY.get(str(decision_model)) + if model_key is None or (COMMERCIAL_RELEASE and model_key == "openjev"): + raise ValueError(f"Decision model is unavailable in this build: {decision_model!r}.") include_reason = bool(kwargs.get("include_reason", False)) + api_route_text = "\n".join( + f"{route['name']}: {route.get('description') or ''}" for route in routes + ) if api_backend else "" try: - self._load_model(model_key, require_vision=use_image) + if not api_backend: + self._load_model(model_key, require_vision=use_image) decisions = [] chosen_names = [] for item_index in range(item_count): - image_data_url = _encode_image(items[item_index]) if use_image else None + if api_backend: + item_text = str(items[item_index]) if not use_image else "Classify the supplied image." + _validate_input_text( + api_profile, + system_prompt, + f"{item_text}\n{api_route_text}", + ) + image_data_url = None + if use_image: + if api_backend: + image_data_url = _image_to_data_url(api_profile, items[item_index]) + else: + image_data_url = _encode_image(items[item_index]) try: - if model_key == "openjev": + if api_backend: + chosen, probability_map, confidence = request_decision( + api_profile, + api_model, + input_kind, + items[item_index], + image_data_url, + str(system_prompt or ""), + routes, + api_key=api_key, + opener=api_opener, + ) + elif model_key == "openjev": if use_image: probabilities = openjev_probabilities(self.llm, self.chat_handler, image_data_url, str(system_prompt or ""), routes) else: @@ -435,10 +525,12 @@ class _DecisionRouterBase: sorted_probabilities, sorted_routes = clef_text_probabilities(self.llm, items[item_index], str(system_prompt or ""), routes) sorted_map = map_route_probabilities(sorted_routes, sorted_probabilities) probability_map = {str(route["name"]): sorted_map[str(route["name"])] for route in routes} - chosen = selected_route(routes, probability_map) + if not api_backend: + chosen = selected_route(routes, probability_map) except Exception as exc: + model_label = api_model if api_backend else decision_model raise RuntimeError( - f"Decision inference failed for {'image' if use_image else 'text'} item {item_index + 1} with {decision_model}: {exc}" + f"Decision inference failed for {'image' if use_image else 'text'} item {item_index + 1} with {model_label}: {exc}" ) from exc chosen_names.append(chosen) decision = { @@ -446,7 +538,13 @@ class _DecisionRouterBase: "route": chosen, "probabilities": probability_map, } - if include_reason: + if api_backend: + if confidence is not None: + decision["confidence"] = confidence + if include_reason: + decision["reason"] = None + decision["reason_error"] = "The selected Decisions API returns typed decisions, not generated explanations." + elif include_reason: chosen_route = next(route for route in routes if route["name"] == chosen) try: decision["reason"] = _decision_reason( @@ -476,14 +574,17 @@ class _DecisionRouterBase: else: output_batches.append([items[index] for index in batch_indices]) + metadata = {"model": api_model if api_backend else decision_model, "input_type": input_kind, "decisions": decisions} + if api_backend: + metadata["provider"] = api_profile["provider"] decision_json = json.dumps( - {"model": decision_model, "input_type": input_kind, "decisions": decisions}, + metadata, ensure_ascii=False, separators=(",", ":"), ) return (decision_json, *output_batches) finally: - if not keep_model_loaded: + if not api_backend and not keep_model_loaded: self.clear() @@ -532,3 +633,40 @@ NODE_DISPLAY_NAME_MAPPINGS = { "ThinkingLLM_DecisionRouter": "ThinkingLLM Decision Router (Image)", "ThinkingLLM_DecisionRouterText": "ThinkingLLM Decision Router (Text)", } + + +def _register_decision_model_route(): + try: + from aiohttp import web + from server import PromptServer + from thinkingllm_core.decision_api import decision_models_for_profile + + server = PromptServer.instance + if server is None: + return + + async def list_decision_models(request): + from openai_compatible_api import _load_api_profiles + + profile_name = str(request.query.get("profile", "")) + input_kind = str(request.query.get("input_kind", "text")) + if input_kind not in {"text", "image"}: + raise web.HTTPBadRequest(text="input_kind must be text or image") + profile = _load_api_profiles().get(profile_name) + if not profile or not profile.get("decision_protocol"): + raise web.HTTPBadRequest(text="Unknown Decision API profile") + try: + models = decision_models_for_profile(profile, input_kind) + except RuntimeError as exc: + raise web.HTTPServiceUnavailable(text="Decision model catalog is unavailable") from exc + return web.json_response({"models": models}) + + server.routes.get("/thinkingllm/decision/models")(list_decision_models) + except Exception as exc: + # The module is also imported in lightweight tests without ComfyUI's server. + if "server" in str(exc).lower() or "PromptServer" in str(exc): + return + print(f"[ThinkingLLM Decision Router] Could not register Decision API model route: {exc}") + + +_register_decision_model_route() diff --git a/nodes/openai_compatible_api.py b/nodes/openai_compatible_api.py index b414f77..c8d654b 100644 --- a/nodes/openai_compatible_api.py +++ b/nodes/openai_compatible_api.py @@ -55,6 +55,7 @@ _PROFILE_ALLOWED_FIELDS = { "allowed_extra_body_fields", "send_seed", "thinking_mode", "max_tokens_field", "allow_insecure_http", "allow_images", "allow_image_url", "max_image_pixels", "max_image_bytes", + "decision_protocol", "decision_endpoint", } @@ -86,13 +87,17 @@ _BUILTIN_PROFILES = { "OpenAI": _cloud_profile( "OpenAI", "https://api.openai.com/v1", "OPENAI_API_KEY", max_tokens_field="max_completion_tokens", + decision_protocol="openai_decisions", + decision_endpoint="https://api.openai.com/v1/decisions", ), "QwenCloud (Singapore)": _cloud_profile( "QwenCloud", "https://dashscope-intl.aliyuncs.com/compatible-mode/v1", "DASHSCOPE_API_KEY", thinking_mode="qwen", ), "OpenRouter": _cloud_profile( - "OpenRouter", "https://openrouter.ai/api/v1", "OPENROUTER_API_KEY" + "OpenRouter", "https://openrouter.ai/api/v1", "OPENROUTER_API_KEY", + decision_protocol="openrouter_decisions", + decision_endpoint="https://openrouter.ai/api/alpha/decisions", ), "Together AI": _cloud_profile( "Together AI", "https://api.together.ai/v1", "TOGETHER_API_KEY" @@ -280,6 +285,15 @@ def _normalize_profile(name: str, raw: dict) -> dict: if allow_image_url and not allow_images: raise ValueError(f"API profile {profile_name!r} cannot enable allow_image_url while allow_images is false.") + decision_protocol = str(raw.get("decision_protocol") or "").strip().lower() + if decision_protocol not in {"", "openai_decisions", "openrouter_decisions"}: + raise ValueError(f"API profile {profile_name!r} has unsupported decision_protocol {decision_protocol!r}.") + decision_endpoint = str(raw.get("decision_endpoint") or "").strip() + if decision_endpoint: + decision_endpoint = _validate_base_url(decision_endpoint, auth, allow_insecure_http) + if bool(decision_protocol) != bool(decision_endpoint): + raise ValueError(f"API profile {profile_name!r} must configure both decision_protocol and decision_endpoint, or neither.") + return { "name": profile_name, "provider": provider, @@ -300,6 +314,8 @@ def _normalize_profile(name: str, raw: dict) -> dict: "allow_image_url": allow_image_url, "max_image_pixels": _profile_int(raw, "max_image_pixels", DEFAULT_MAX_IMAGE_PIXELS, maximum=100_000_000), "max_image_bytes": _profile_int(raw, "max_image_bytes", DEFAULT_MAX_IMAGE_BYTES, maximum=100_000_000), + "decision_protocol": decision_protocol, + "decision_endpoint": decision_endpoint, } diff --git a/pyproject.toml b/pyproject.toml index 4a885c1..17321d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "ComfyUI-ThinkingLLM" description = "A multimodal ComfyUI AI node pack with Qwen, Gemma 4, GGUF, Whisper, image and text decision routing, and live thinking output." -version = "3.0.1" +version = "3.0.2" license = {file = "LICENSE"} dependencies = ["transformers>=5.2.0", "torch", "huggingface-hub", "hf_xet", "psutil", "numpy", "Pillow", "opencv-python", "bitsandbytes", "accelerate", "sentencepiece", "tiktoken", "faster-whisper", "llama-cpp-python"] diff --git a/tests/test_decision_api.py b/tests/test_decision_api.py new file mode 100644 index 0000000..e4f5116 --- /dev/null +++ b/tests/test_decision_api.py @@ -0,0 +1,248 @@ +from __future__ import annotations + +import asyncio +import importlib.util +import json +import sys +import types +import unittest +from pathlib import Path +from unittest import mock + +import torch + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from thinkingllm_core import decision_api + + +ROUTES = [ + {"name": "fox", "description": "A fox."}, + {"name": "dog", "description": "A domestic dog."}, +] + + +class DecisionApiAdapterTests(unittest.TestCase): + def test_openai_request_uses_native_choice_and_input_image_schema(self): + payload = decision_api.build_decision_request( + "openai_decisions", "gpt-6-luna", "image", None, + "data:image/png;base64,AA==", "Pick the best route.", ROUTES, + ) + self.assertEqual(payload["model"], "gpt-6-luna") + self.assertEqual(payload["questions"][0]["type"], "choice") + self.assertEqual(payload["questions"][0]["choices"][0], {"value": "fox", "description": "A fox."}) + self.assertEqual(payload["input"][0]["content"][1], {"type": "input_image", "image_url": "data:image/png;base64,AA=="}) + + def test_openrouter_request_uses_native_systemone_schema(self): + payload = decision_api.build_decision_request( + "openrouter_decisions", "cloudflare/clef-flash", "image", None, + "data:image/png;base64,AA==", "Pick the best route.", ROUTES, + ) + self.assertEqual(payload["state"], [{"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}}]) + self.assertEqual(payload["questions"]["route"]["criteria"], {"fox": "A fox.", "dog": "A domestic dog."}) + + def test_openai_and_openrouter_response_schemas_normalize(self): + openai = {"answers": [{"name": "route", "type": "choice", "choice": "dog", "confidence": 0.8, + "probabilities": [{"value": "fox", "probability": 0.2}, {"value": "dog", "probability": 0.8}]}]} + openrouter = {"answers": {"route": {"type": "choice", "choice": "dog", "confidence": 0.8, + "probabilities": {"fox": 0.2, "dog": 0.8}}}} + expected = ("dog", {"fox": 0.2, "dog": 0.8}, 0.8) + self.assertEqual(decision_api.normalize_decision_response("openai_decisions", openai, ["fox", "dog"]), expected) + self.assertEqual(decision_api.normalize_decision_response("openrouter_decisions", openrouter, ["fox", "dog"]), expected) + + def test_refusal_unknown_routes_and_malformed_probabilities_fail_closed(self): + with self.assertRaisesRegex(RuntimeError, "did not return a route"): + decision_api.normalize_decision_response("openai_decisions", {"answers": [{"name": "route", "type": "refusal"}]}, ["fox", "dog"]) + with self.assertRaisesRegex(RuntimeError, "unknown route"): + decision_api.normalize_decision_response("openrouter_decisions", {"answers": {"route": {"type": "choice", "choice": "cat", "probabilities": {"fox": 0.2, "dog": 0.8}}}}, ["fox", "dog"]) + with self.assertRaisesRegex(RuntimeError, "invalid route probability"): + decision_api.normalize_decision_response("openrouter_decisions", {"answers": {"route": {"type": "choice", "choice": "fox", "probabilities": {"fox": float("nan"), "dog": 0.8}}}}, ["fox", "dog"]) + with self.assertRaisesRegex(RuntimeError, "duplicate route probabilities"): + decision_api.normalize_decision_response( + "openai_decisions", + {"answers": [{"name": "route", "type": "choice", "choice": "fox", "probabilities": [ + {"value": "fox", "probability": 0.5}, {"value": "fox", "probability": 0.4}, + {"value": "dog", "probability": 0.1}, + ]}]}, + ["fox", "dog"], + ) + + def test_openrouter_model_catalog_filters_by_required_modality(self): + response = { + "data": [ + {"id": "decision/vision", "architecture": {"input_modalities": ["text", "image"], "output_modalities": ["decisions"]}}, + {"id": "decision/text", "architecture": {"input_modalities": ["text"], "output_modalities": ["decisions"]}}, + {"id": "chat/vision", "architecture": {"input_modalities": ["text", "image"], "output_modalities": ["text"]}}, + ] + } + opener = mock.Mock() + http_response = mock.MagicMock() + http_response.read.return_value = json.dumps(response).encode() + http_response.__enter__.return_value = http_response + opener.open.return_value = http_response + decision_api._MODEL_CACHE.clear() + with mock.patch.object(decision_api, "_request_json", return_value=response): + image_models = decision_api.openrouter_decision_models("image", now=100) + text_models = decision_api.openrouter_decision_models("text", now=100) + self.assertEqual([model["id"] for model in image_models], ["decision/vision"]) + self.assertEqual([model["id"] for model in text_models], ["decision/vision", "decision/text"]) + + def test_profile_allowlist_restricts_api_model_choices(self): + with mock.patch.object(decision_api, "openrouter_decision_models", return_value=[{"id": "a"}, {"id": "b"}]): + models = decision_api.decision_models_for_profile({"decision_protocol": "openrouter_decisions", "allowed_models": ["b"]}, "text") + self.assertEqual(models, [{"id": "b"}]) + + def test_profile_image_policy_hides_vision_models(self): + with mock.patch.object(decision_api, "openrouter_decision_models") as catalog: + models = decision_api.decision_models_for_profile( + {"decision_protocol": "openrouter_decisions", "allow_images": False}, "image", + ) + self.assertEqual(models, []) + catalog.assert_not_called() + + +def load_router_module(): + module_name = "thinkingllm_decision_router_api_test" + spec = importlib.util.spec_from_file_location(module_name, ROOT / "nodes" / "decision_router.py") + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module + + +class DecisionApiRouterTests(unittest.TestCase): + def setUp(self): + self.profile = { + "name": "Test OpenAI", + "provider": "OpenAI", + "decision_protocol": "openai_decisions", + "decision_endpoint": "https://api.openai.com/v1/decisions", + "allow_images": True, + "max_image_pixels": 100, + "max_image_bytes": 100000, + } + self.api_module = types.ModuleType("openai_compatible_api") + self.api_module._load_api_profiles = lambda: {"Test OpenAI": self.profile} + self.api_module._credential_value = lambda _profile: "test-secret" + self.api_module._image_to_data_url = lambda _profile, _image: "data:image/png;base64,AA==" + self.api_module._NO_REDIRECT_OPENER = object() + def validate_input_text(profile, system_prompt, prompt): + if system_prompt is not None and not isinstance(system_prompt, str): + raise ValueError("system_prompt must be a string.") + if prompt is not None and not isinstance(prompt, str): + raise ValueError("prompt must be a string.") + total = len(system_prompt or "") + len(prompt or "") + limit = profile.get("max_input_chars", 1_000_000) + if total > limit: + raise ValueError(f"Combined input length exceeds server-side max_input_chars={limit}.") + self.api_module._validate_input_text = validate_input_text + self.previous_api_module = sys.modules.get("openai_compatible_api") + sys.modules["openai_compatible_api"] = self.api_module + + def tearDown(self): + if self.previous_api_module is None: + sys.modules.pop("openai_compatible_api", None) + else: + sys.modules["openai_compatible_api"] = self.previous_api_module + + def test_registers_model_catalog_route_and_returns_filtered_models(self): + registered = {} + + def route_decorator(path): + return lambda handler: registered.setdefault(path, handler) + + fake_server = types.ModuleType("server") + fake_server.PromptServer = types.SimpleNamespace( + instance=types.SimpleNamespace(routes=types.SimpleNamespace(get=route_decorator)), + ) + router_module = load_router_module() + with mock.patch.dict(sys.modules, {"server": fake_server}): + with mock.patch.object( + decision_api, + "decision_models_for_profile", + return_value=[{"id": "gpt-6-luna", "vision": True}], + ): + router_module._register_decision_model_route() + handler = registered["/thinkingllm/decision/models"] + response = asyncio.run(handler(types.SimpleNamespace( + query={"profile": "Test OpenAI", "input_kind": "image"}, + ))) + self.assertEqual(response.status, 200) + self.assertEqual(json.loads(response.text)["models"], [{"id": "gpt-6-luna", "vision": True}]) + + def test_api_text_batch_routes_without_loading_local_model(self): + router_module = load_router_module() + node = router_module.ThinkingLLMDecisionRouterText() + node._load_model = mock.Mock(side_effect=AssertionError("API route must not load a local model")) + node.llm = mock.Mock() + node.current_signature = ("previously-retained-local-model",) + route_fields = {"route_01_name": "fox", "route_02_name": "dog"} + api_args = {"decision_backend": "API", "api_profile": "Test OpenAI", "api_model": "gpt-6-luna"} + with ( + mock.patch.object(decision_api, "decision_models_for_profile", return_value=[{"id": "gpt-6-luna", "vision": True}]), + mock.patch.object(decision_api, "request_decision", side_effect=[("fox", {"fox": 0.9, "dog": 0.1}, 0.9), ("dog", {"fox": 0.1, "dog": 0.9}, 0.9)]), + ): + result = node.route_text(["fox text", "dog text"], "unused local", "Choose.", **api_args, **route_fields) + payload = json.loads(result[0]) + self.assertEqual(payload["provider"], "OpenAI") + self.assertEqual([item["route"] for item in payload["decisions"]], ["fox", "dog"]) + self.assertEqual(result[1], ["fox text"]) + self.assertEqual(result[2], ["dog text"]) + node._load_model.assert_not_called() + self.assertIsNone(node.llm) + self.assertIsNone(node.current_signature) + + def test_api_image_uses_profile_limited_data_url_and_blocks_empty_routes(self): + router_module = load_router_module() + node = router_module.ThinkingLLMDecisionRouter() + image = torch.zeros((1, 2, 2, 3)) + route_fields = {"route_01_name": "fox", "route_02_name": "dog"} + api_args = {"decision_backend": "API", "api_profile": "Test OpenAI", "api_model": "gpt-6-luna"} + with ( + mock.patch.object(decision_api, "decision_models_for_profile", return_value=[{"id": "gpt-6-luna", "vision": True}]), + mock.patch.object(decision_api, "request_decision", return_value=("dog", {"fox": 0.1, "dog": 0.9}, 0.9)) as request_mock, + ): + result = node.route_images(image, "unused local", "Choose.", **api_args, **route_fields) + self.assertEqual(json.loads(result[0])["decisions"][0]["route"], "dog") + self.assertEqual(request_mock.call_args.args[4], "data:image/png;base64,AA==") + self.assertIsInstance(result[1], router_module.ExecutionBlocker) + self.assertEqual(tuple(result[2].shape), (1, 2, 2, 3)) + + def test_api_mode_rejects_unknown_image_model_before_request(self): + router_module = load_router_module() + node = router_module.ThinkingLLMDecisionRouter() + with ( + mock.patch.object(decision_api, "decision_models_for_profile", return_value=[{"id": "text-only", "vision": False}]), + mock.patch.object(decision_api, "request_decision") as request_mock, + ): + with self.assertRaisesRegex(ValueError, "image-capable"): + node.route_images( + torch.zeros((1, 1, 1, 3)), "unused", "Choose.", + decision_backend="API", api_profile="Test OpenAI", api_model="text-only", + route_01_name="a", route_02_name="b", + ) + request_mock.assert_not_called() + + def test_api_mode_enforces_profile_input_limit_before_request(self): + router_module = load_router_module() + node = router_module.ThinkingLLMDecisionRouterText() + self.profile["max_input_chars"] = 20 + with ( + mock.patch.object(decision_api, "decision_models_for_profile", return_value=[{"id": "gpt-6-luna"}]), + mock.patch.object(decision_api, "request_decision") as request_mock, + ): + with self.assertRaisesRegex(ValueError, "exceeds server-side max_input_chars"): + node.route_text( + "cat", "unused", "Choose.", + decision_backend="API", api_profile="Test OpenAI", api_model="gpt-6-luna", + route_01_name="fox", route_01_description="long route description", + route_02_name="dog", + ) + request_mock.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_decision_router.py b/tests/test_decision_router.py index a11463d..4f0df61 100644 --- a/tests/test_decision_router.py +++ b/tests/test_decision_router.py @@ -478,8 +478,12 @@ class DecisionRouterNodeTests(unittest.TestCase): self.assertNotIn("text", image_node.INPUT_TYPES()["required"]) self.assertEqual(text_node.INPUT_TYPES()["required"]["text"][0], "STRING") self.assertNotIn("image", text_node.INPUT_TYPES()["required"]) - self.assertEqual(len(image_node.INPUT_TYPES()["required"]), 109) - self.assertEqual(len(text_node.INPUT_TYPES()["required"]), 109) + self.assertEqual(len(image_node.INPUT_TYPES()["required"]), 112) + self.assertEqual(len(text_node.INPUT_TYPES()["required"]), 112) + required_names = list(image_node.INPUT_TYPES()["required"]) + self.assertEqual(required_names[:5], ["image", "decision_model", "keep_model_loaded", "include_reason", "system_prompt"]) + self.assertEqual(required_names[-3:], ["decision_backend", "api_profile", "api_model"]) + self.assertEqual(image_node.INPUT_TYPES()["required"]["decision_backend"][1]["default"], "Local (in-process)") self.assertEqual(image_node.INPUT_TYPES()["required"]["keep_model_loaded"][1]["default"], False) self.assertEqual(text_node.INPUT_TYPES()["required"]["keep_model_loaded"][1]["default"], False) self.assertEqual(image_node.INPUT_TYPES()["required"]["include_reason"][1]["default"], False) diff --git a/tests/test_openai_api_security.py b/tests/test_openai_api_security.py index 3df80e9..e2675da 100644 --- a/tests/test_openai_api_security.py +++ b/tests/test_openai_api_security.py @@ -93,6 +93,18 @@ class SecurityTests(unittest.TestCase): 'api_key_env': 'TEST_KEY', }) + def test_native_decision_profile_requires_a_supported_protocol_and_endpoint_pair(self): + profile = self.profile( + decision_protocol='openrouter_decisions', + decision_endpoint='https://openrouter.ai/api/alpha/decisions', + ) + self.assertEqual(profile['decision_protocol'], 'openrouter_decisions') + self.assertEqual(profile['decision_endpoint'], 'https://openrouter.ai/api/alpha/decisions') + with self.assertRaisesRegex(ValueError, 'both decision_protocol and decision_endpoint'): + self.profile(decision_protocol='openai_decisions') + with self.assertRaisesRegex(ValueError, 'unsupported decision_protocol'): + self.profile(decision_protocol='chat_json', decision_endpoint='https://example.test/decisions') + def test_plain_http_non_loopback_requires_explicit_opt_in(self): with self.assertRaisesRegex(ValueError, 'Plain HTTP'): api_node._normalize_profile('LAN', { diff --git a/thinkingllm_api_profiles.example.json b/thinkingllm_api_profiles.example.json index 2b9d5b2..9287551 100644 --- a/thinkingllm_api_profiles.example.json +++ b/thinkingllm_api_profiles.example.json @@ -3,6 +3,8 @@ "OpenRouter Production": { "provider": "OpenRouter", "base_url": "https://openrouter.ai/api/v1", + "decision_protocol": "openrouter_decisions", + "decision_endpoint": "https://openrouter.ai/api/alpha/decisions", "auth": "bearer_env", "api_key_env": "OPENROUTER_PRODUCTION_API_KEY", "allowed_models": [ diff --git a/thinkingllm_core/decision_api.py b/thinkingllm_core/decision_api.py new file mode 100644 index 0000000..93e0784 --- /dev/null +++ b/thinkingllm_core/decision_api.py @@ -0,0 +1,193 @@ +"""Native OpenAI Decisions and OpenRouter Decisions API adapters.""" + +from __future__ import annotations + +import json +import math +import time +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Mapping + + +OPENAI_DECISION_MODEL = "gpt-6-luna" +OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models" +MODEL_CACHE_TTL_SECONDS = 3600 +_MODEL_CACHE: dict[str, tuple[float, list[dict]]] = {} + + +def _request_json(url: str, *, headers: dict[str, str], payload=None, timeout: int = 60, opener=None): + data = None if payload is None else json.dumps(payload, ensure_ascii=False).encode("utf-8") + request_headers = {"Accept": "application/json", **headers} + if data is not None: + request_headers["Content-Type"] = "application/json" + req = urllib.request.Request(url, data=data, headers=request_headers, method="GET" if data is None else "POST") + try: + response = (opener or urllib.request.build_opener()).open(req, timeout=timeout) + with response: + raw = response.read() + except urllib.error.HTTPError as exc: + # Do not include provider response text: it can echo user content or media. + raise RuntimeError(f"Decision API returned HTTP {exc.code}.") from exc + except urllib.error.URLError as exc: + raise RuntimeError(f"Could not reach the Decision API ({type(exc.reason).__name__}).") from exc + try: + result = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError("Decision API returned invalid JSON.") from exc + if not isinstance(result, dict): + raise RuntimeError("Decision API returned an unexpected response shape.") + if result.get("error"): + raise RuntimeError("Decision API reported an inference error.") + return result + + +def _route_criteria(routes): + return {str(route["name"]): str(route.get("description") or "") or None for route in routes} + + +def build_decision_request(protocol: str, model: str, input_kind: str, item, image_data_url: str | None, system_prompt: str, routes): + criteria = _route_criteria(routes) + instruction = str(system_prompt or "").strip() or "Choose the single route that best matches the input." + if protocol == "openai_decisions": + content = [{"type": "input_text", "text": str(item) if input_kind == "text" else "Classify the supplied image."}] + if image_data_url: + content.append({"type": "input_image", "image_url": image_data_url}) + return { + "model": model, + "input": [{"role": "user", "content": content}], + "questions": [{ + "type": "choice", + "name": "route", + "instructions": instruction, + "choices": [{"value": name, "description": description or ""} for name, description in criteria.items()], + }], + } + if protocol == "openrouter_decisions": + state = str(item) if input_kind == "text" else [] + if input_kind == "image": + state = [{"type": "image_url", "image_url": {"url": image_data_url}}] + return { + "model": model, + "state": state, + "questions": { + "route": {"type": "choice", "instructions": instruction, "criteria": criteria}, + }, + } + raise ValueError(f"Unsupported Decision API protocol: {protocol!r}.") + + +def _probability_map(raw, route_names): + if isinstance(raw, Mapping): + values = {str(key): value for key, value in raw.items()} + elif isinstance(raw, list): + values = {} + for item in raw: + if not isinstance(item, dict) or "value" not in item: + raise RuntimeError("Decision API returned malformed route probabilities.") + value = str(item["value"]) + if value in values: + raise RuntimeError("Decision API returned duplicate route probabilities.") + values[value] = item.get("probability") + else: + raise RuntimeError("Decision API returned no route probabilities.") + if set(values) != set(route_names): + raise RuntimeError("Decision API probabilities do not match the configured route names.") + normalized = {} + for name in route_names: + value = values[name] + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(float(value)) or not 0 <= float(value) <= 1: + raise RuntimeError("Decision API returned an invalid route probability.") + normalized[name] = float(value) + return normalized + + +def normalize_decision_response(protocol: str, response: dict, route_names: list[str]): + if protocol == "openai_decisions": + answers = response.get("answers") + answer = next((item for item in answers if isinstance(item, dict) and item.get("name") == "route"), None) if isinstance(answers, list) else None + if not isinstance(answer, dict) or answer.get("type") != "choice": + raise RuntimeError("OpenAI Decisions API did not return a route choice.") + choice = answer.get("choice") + probability_map = _probability_map(answer.get("probabilities"), route_names) + confidence = answer.get("confidence") + elif protocol == "openrouter_decisions": + answers = response.get("answers") + answer = answers.get("route") if isinstance(answers, dict) else None + if not isinstance(answer, dict) or answer.get("type") != "choice": + raise RuntimeError("OpenRouter Decisions API did not return a route choice.") + choice = answer.get("choice") + probability_map = _probability_map(answer.get("probabilities"), route_names) + confidence = answer.get("confidence") + else: + raise ValueError(f"Unsupported Decision API protocol: {protocol!r}.") + if not isinstance(choice, str) or choice not in route_names: + raise RuntimeError("Decision API selected an unknown route.") + if confidence is not None and (isinstance(confidence, bool) or not isinstance(confidence, (int, float)) or not math.isfinite(float(confidence)) or not 0 <= float(confidence) <= 1): + raise RuntimeError("Decision API returned an invalid confidence value.") + return choice, probability_map, None if confidence is None else float(confidence) + + +def request_decision(profile: dict, model: str, input_kind: str, item, image_data_url: str | None, system_prompt: str, routes, *, api_key: str, opener=None): + protocol = profile.get("decision_protocol") + payload = build_decision_request(protocol, model, input_kind, item, image_data_url, system_prompt, routes) + response = _request_json( + profile["decision_endpoint"], + headers={"Authorization": f"Bearer {api_key}"} if api_key else {}, + payload=payload, + timeout=int(profile.get("max_timeout_seconds", 300)), + opener=opener, + ) + return normalize_decision_response(protocol, response, [str(route["name"]) for route in routes]) + + +def openrouter_decision_models(input_kind: str, *, timeout: int = 20, now: float | None = None, opener=None): + """Return OpenRouter model ids with decision output and requested input capability.""" + modality = "image" if input_kind == "image" else "text" + now = time.monotonic() if now is None else now + cached = _MODEL_CACHE.get(modality) + if cached and now - cached[0] < MODEL_CACHE_TTL_SECONDS: + return [dict(model) for model in cached[1]] + query = urllib.parse.urlencode({"output_modalities": "decisions", "input_modalities": modality}) + try: + data = _request_json(f"{OPENROUTER_MODELS_URL}?{query}", headers={}, timeout=timeout, opener=opener) + items = data.get("data") + if not isinstance(items, list): + raise RuntimeError("OpenRouter model catalog returned an unexpected response shape.") + models = [] + for item in items: + if not isinstance(item, dict) or not isinstance(item.get("id"), str): + continue + architecture = item.get("architecture") + modalities = architecture.get("input_modalities") if isinstance(architecture, dict) else None + output_modalities = architecture.get("output_modalities") if isinstance(architecture, dict) else None + if not isinstance(modalities, list) or modality not in modalities: + continue + if not isinstance(output_modalities, list) or "decisions" not in output_modalities: + continue + models.append({"id": item["id"], "label": item.get("name") or item["id"], "vision": "image" in modalities}) + if not models: + raise RuntimeError("OpenRouter returned no decision models for the requested input type.") + _MODEL_CACHE[modality] = (now, models) + return [dict(model) for model in models] + except RuntimeError: + if cached: + return [dict(model) for model in cached[1]] + raise + + +def decision_models_for_profile(profile: dict, input_kind: str): + if input_kind == "image" and not profile.get("allow_images", False): + return [] + protocol = profile.get("decision_protocol") + if protocol == "openai_decisions": + models = [{"id": OPENAI_DECISION_MODEL, "label": "GPT-6 Luna Decisions", "vision": True}] + elif protocol == "openrouter_decisions": + models = openrouter_decision_models(input_kind) + else: + return [] + allowed = profile.get("allowed_models") + if allowed is not None: + models = [model for model in models if model["id"] in allowed] + return models diff --git a/update.md b/update.md index 7ba1109..37232d6 100644 --- a/update.md +++ b/update.md @@ -1,5 +1,23 @@ # ComfyUI-QwenVL Update Log +## Version 3.0.2 (2026/10/29) - Native Decision API Backends + +### 3.0.2 Changes + +- Added an optional native Decisions API backend to both image and text Decision Router nodes. Local in-process llama.cpp inference remains the default. +- Added direct OpenAI Decisions support with `gpt-6-luna` and dynamic OpenRouter Decisions model discovery, filtered by decision output, input modality, profile image policy, and optional model allowlists. +- Preserved typed route selection and provider probabilities in `decision_json`; image items are encoded within profile limits and each batch item receives its own request. The router does not fall back to chat-generated JSON. +- Reused server-side API profiles and environment-based credentials, enforced each profile's input-size and timeout limits, and kept credentials out of workflows. +- Kept existing widget order stable for saved workflows. Switching to API mode releases a locally retained decision model. +- Added API configuration documentation and updated the README. + +### 3.0.2 Validation + +- All 240 tests passed across the six relevant test modules when run individually in the bundled ComfyUI Python 3.12 runtime, including 12 Decision API adapter/router tests, 29 local Decision Router tests, and 38 API security/profile tests. +- Python compilation, both JavaScript syntax checks, JSON/TOML and mirrored-version checks, API model-catalog validation, and `git diff --check` passed. +- A live public OpenRouter model-catalog request returned decision models with text/image capabilities, confirming the dynamic modality filter against the current catalog. +- Provider inference tests use mocked responses; no authenticated OpenAI or OpenRouter inference request was made. A previous combined `unittest discover` attempt showed test-order/native-extension interference, so the six modules were verified in separate processes. + ## Version 3.0.1 (2026/10/29) - Decision Router VRAM and Explanation Controls ### 3.0.1 Changes diff --git a/web/js/decision_router.js b/web/js/decision_router.js index c749de3..613c563 100644 --- a/web/js/decision_router.js +++ b/web/js/decision_router.js @@ -5,6 +5,66 @@ const NODE_OUTPUT_TYPES = { ThinkingLLM_DecisionRouterText: "STRING", }; const MAX_ROUTES = 52; +const API_MODELS_URL = "/thinkingllm/decision/models"; +const apiModelRequestIds = new WeakMap(); + +function findWidget(node, name) { + return node.widgets?.find((widget) => widget.name === name); +} + +function widgetValue(widget) { + return String(widget?.value ?? "").trim(); +} + +async function refreshApiModels(node, inputKind) { + const profileWidget = findWidget(node, "api_profile"); + const modelWidget = findWidget(node, "api_model"); + if (!profileWidget || !modelWidget || widgetValue(findWidget(node, "decision_backend")) !== "API") return; + const profile = widgetValue(profileWidget); + const requestId = (apiModelRequestIds.get(node) ?? 0) + 1; + apiModelRequestIds.set(node, requestId); + const query = new URLSearchParams({ profile, input_kind: inputKind }); + try { + const response = await fetch(`${API_MODELS_URL}?${query.toString()}`); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const payload = await response.json(); + if (apiModelRequestIds.get(node) !== requestId + || widgetValue(findWidget(node, "decision_backend")) !== "API" + || widgetValue(findWidget(node, "api_profile")) !== profile) return; + const ids = (Array.isArray(payload.models) ? payload.models : []) + .map((model) => String(model?.id ?? "")).filter(Boolean); + if (!ids.length) throw new Error("No compatible decision models returned"); + modelWidget.type = "combo"; + modelWidget.serialize = true; + modelWidget.options ??= {}; + modelWidget.options.values = ids; + if (!ids.includes(widgetValue(modelWidget))) { + modelWidget.value = ids[0]; + modelWidget.options.value = ids[0]; + } + } catch (error) { + if (apiModelRequestIds.get(node) !== requestId + || widgetValue(findWidget(node, "decision_backend")) !== "API" + || widgetValue(findWidget(node, "api_profile")) !== profile) return; + console.warn("[ThinkingLLM Decision Router] Could not load API decision models.", error); + modelWidget.type = "text"; + } + node.size = node.computeSize?.() ?? node.size; + node.setDirtyCanvas?.(true, true); +} + +function synchronizeBackend(node, inputKind) { + const apiMode = widgetValue(findWidget(node, "decision_backend")) === "API"; + for (const name of ["decision_model", "api_profile", "api_model", "keep_model_loaded", "include_reason"]) { + const widget = findWidget(node, name); + if (widget) widget.hidden = apiMode + ? ["decision_model", "keep_model_loaded", "include_reason"].includes(name) + : ["api_profile", "api_model"].includes(name); + } + if (apiMode) refreshApiModels(node, inputKind); + node.size = node.computeSize?.() ?? node.size; + node.setDirtyCanvas?.(true, true); +} function widgetName(index, field) { return `route_${String(index + 1).padStart(2, "0")}_${field}`; @@ -97,6 +157,7 @@ app.registerExtension({ async beforeRegisterNodeDef(nodeType, nodeData) { const routeType = NODE_OUTPUT_TYPES[nodeData.name]; if (!routeType) return; + const inputKind = nodeData.name === "ThinkingLLM_DecisionRouter" ? "image" : "text"; const originalCreated = nodeType.prototype.onNodeCreated; nodeType.prototype.onNodeCreated = function () { @@ -104,6 +165,15 @@ app.registerExtension({ const node = this; return afterLifecycle(node, routeType, result, (createdNode, outputType) => { for (const widget of createdNode.widgets ?? []) { + if (["decision_backend", "api_profile"].includes(widget.name)) { + const originalCallback = widget.callback; + widget.callback = function (value, ...args) { + const callbackResult = originalCallback?.call(this, value, ...args); + synchronizeBackend(createdNode, inputKind); + return callbackResult; + }; + continue; + } if (!widget.name?.startsWith("route_")) continue; const originalCallback = widget.callback; widget.callback = function (value, ...args) { @@ -112,13 +182,17 @@ app.registerExtension({ }; } scheduleSynchronize(createdNode, outputType); + synchronizeBackend(createdNode, inputKind); }); }; const originalConfigure = nodeType.prototype.onConfigure; nodeType.prototype.onConfigure = function () { const result = originalConfigure?.apply(this, arguments); - return afterLifecycle(this, routeType, result, scheduleSynchronize); + return afterLifecycle(this, routeType, result, (configuredNode, outputType) => { + scheduleSynchronize(configuredNode, outputType); + synchronizeBackend(configuredNode, inputKind); + }); }; }, }); diff --git a/web_commercial/js/decision_router.js b/web_commercial/js/decision_router.js index 945e058..0e45b64 100644 --- a/web_commercial/js/decision_router.js +++ b/web_commercial/js/decision_router.js @@ -5,6 +5,66 @@ const NODE_OUTPUT_TYPES = { ThinkingLLM_DecisionRouterText: "STRING", }; const MAX_ROUTES = 52; +const API_MODELS_URL = "/thinkingllm/decision/models"; +const apiModelRequestIds = new WeakMap(); + +function findWidget(node, name) { + return node.widgets?.find((widget) => widget.name === name); +} + +function widgetValue(widget) { + return String(widget?.value ?? "").trim(); +} + +async function refreshApiModels(node, inputKind) { + const profileWidget = findWidget(node, "api_profile"); + const modelWidget = findWidget(node, "api_model"); + if (!profileWidget || !modelWidget || widgetValue(findWidget(node, "decision_backend")) !== "API") return; + const profile = widgetValue(profileWidget); + const requestId = (apiModelRequestIds.get(node) ?? 0) + 1; + apiModelRequestIds.set(node, requestId); + const query = new URLSearchParams({ profile, input_kind: inputKind }); + try { + const response = await fetch(`${API_MODELS_URL}?${query.toString()}`); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const payload = await response.json(); + if (apiModelRequestIds.get(node) !== requestId + || widgetValue(findWidget(node, "decision_backend")) !== "API" + || widgetValue(findWidget(node, "api_profile")) !== profile) return; + const ids = (Array.isArray(payload.models) ? payload.models : []) + .map((model) => String(model?.id ?? "")).filter(Boolean); + if (!ids.length) throw new Error("No compatible decision models returned"); + modelWidget.type = "combo"; + modelWidget.serialize = true; + modelWidget.options ??= {}; + modelWidget.options.values = ids; + if (!ids.includes(widgetValue(modelWidget))) { + modelWidget.value = ids[0]; + modelWidget.options.value = ids[0]; + } + } catch (error) { + if (apiModelRequestIds.get(node) !== requestId + || widgetValue(findWidget(node, "decision_backend")) !== "API" + || widgetValue(findWidget(node, "api_profile")) !== profile) return; + console.warn("[ThinkingLLM Decision Router] Could not load API decision models.", error); + modelWidget.type = "text"; + } + node.size = node.computeSize?.() ?? node.size; + node.setDirtyCanvas?.(true, true); +} + +function synchronizeBackend(node, inputKind) { + const apiMode = widgetValue(findWidget(node, "decision_backend")) === "API"; + for (const name of ["decision_model", "api_profile", "api_model", "keep_model_loaded", "include_reason"]) { + const widget = findWidget(node, name); + if (widget) widget.hidden = apiMode + ? ["decision_model", "keep_model_loaded", "include_reason"].includes(name) + : ["api_profile", "api_model"].includes(name); + } + if (apiMode) refreshApiModels(node, inputKind); + node.size = node.computeSize?.() ?? node.size; + node.setDirtyCanvas?.(true, true); +} function widgetName(index, field) { return `route_${String(index + 1).padStart(2, "0")}_${field}`; @@ -97,6 +157,7 @@ app.registerExtension({ async beforeRegisterNodeDef(nodeType, nodeData) { const routeType = NODE_OUTPUT_TYPES[nodeData.name]; if (!routeType) return; + const inputKind = nodeData.name === "ThinkingLLM_DecisionRouter" ? "image" : "text"; const originalCreated = nodeType.prototype.onNodeCreated; nodeType.prototype.onNodeCreated = function () { @@ -104,6 +165,15 @@ app.registerExtension({ const node = this; return afterLifecycle(node, routeType, result, (createdNode, outputType) => { for (const widget of createdNode.widgets ?? []) { + if (["decision_backend", "api_profile"].includes(widget.name)) { + const originalCallback = widget.callback; + widget.callback = function (value, ...args) { + const callbackResult = originalCallback?.call(this, value, ...args); + synchronizeBackend(createdNode, inputKind); + return callbackResult; + }; + continue; + } if (!widget.name?.startsWith("route_")) continue; const originalCallback = widget.callback; widget.callback = function (value, ...args) { @@ -112,13 +182,17 @@ app.registerExtension({ }; } scheduleSynchronize(createdNode, outputType); + synchronizeBackend(createdNode, inputKind); }); }; const originalConfigure = nodeType.prototype.onConfigure; nodeType.prototype.onConfigure = function () { const result = originalConfigure?.apply(this, arguments); - return afterLifecycle(this, routeType, result, scheduleSynchronize); + return afterLifecycle(this, routeType, result, (configuredNode, outputType) => { + scheduleSynchronize(configuredNode, outputType); + synchronizeBackend(configuredNode, inputKind); + }); }; }, });