Marcel Hofer
21b75d3b58
* feat: full per-provider model lists with release-time auto-refresh and dedicated custom model field * feat: add setup help widget to API node (Windows/Linux API key instructions)
178 lines
6.5 KiB
Python
178 lines
6.5 KiB
Python
"""Validate the ThinkingLLM API model catalogs.
|
|
|
|
Checks that web/api_model_catalogs.json stays well-formed, that every
|
|
cloud profile shipped by the API node has a curated entry, and that model
|
|
IDs are safe to send in a request body (no control characters, no
|
|
leading/trailing whitespace, no duplicates).
|
|
|
|
Run in CI on every PR, push to main, and before every release:
|
|
|
|
python tools/check_api_model_catalogs.py
|
|
|
|
Exit code 0 on success, 1 on any validation failure.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
CATALOG_PATH = REPO_ROOT / "web" / "api_model_catalogs.json"
|
|
|
|
# Cloud profiles exposed by the API node (nodes/openai_compatible_api.py).
|
|
# Local endpoints (Ollama, vLLM, llama.cpp) are intentionally excluded
|
|
# because their model IDs are runtime/environment specific.
|
|
REQUIRED_PROFILE_KEYS = {
|
|
"OpenAI": "OpenAI",
|
|
"QwenCloud (Singapore)": "QwenCloud",
|
|
"OpenRouter": "OpenRouter",
|
|
"Together AI": "Together AI",
|
|
"Fireworks AI": "Fireworks AI",
|
|
"DeepInfra": "DeepInfra",
|
|
"Groq": "Groq",
|
|
"Featherless": "Featherless",
|
|
"OrcaRouter": "OrcaRouter",
|
|
}
|
|
|
|
# Reasonable bound to catch accidental mass-paste or generated blobs.
|
|
# Public providers ship full lists (OpenRouter 413, Featherless up to 500).
|
|
MAX_MODELS_PER_PROFILE = 600
|
|
MAX_MODEL_ID_CHARS = 512
|
|
MAX_MODEL_LABEL_CHARS = 128
|
|
MAX_NOTE_CHARS = 512
|
|
|
|
_MODEL_ID_BLOCKLIST = re.compile(
|
|
r"[\x00-\x1f\x7f-\x9f]"
|
|
)
|
|
|
|
_ERRORS: list[str] = []
|
|
|
|
|
|
def _error(message: str) -> None:
|
|
_ERRORS.append(message)
|
|
|
|
|
|
def _require(condition: bool, message: str) -> None:
|
|
if not condition:
|
|
_error(message)
|
|
|
|
|
|
def _validate_model_entry(profile_name: str, index: int, entry: object) -> None:
|
|
if not isinstance(entry, dict):
|
|
_error(f"profile {profile_name!r}: models[{index}] must be an object.")
|
|
return
|
|
model_id = entry.get("id")
|
|
if not isinstance(model_id, str) or not model_id.strip():
|
|
_error(f"profile {profile_name!r}: models[{index}].id must be a non-empty string.")
|
|
else:
|
|
model_id = model_id.strip()
|
|
if model_id != entry["id"]:
|
|
_error(f"profile {profile_name!r}: models[{index}].id has leading/trailing whitespace.")
|
|
if len(model_id) > MAX_MODEL_ID_CHARS:
|
|
_error(f"profile {profile_name!r}: models[{index}].id exceeds {MAX_MODEL_ID_CHARS} characters.")
|
|
if _MODEL_ID_BLOCKLIST.search(model_id):
|
|
_error(f"profile {profile_name!r}: models[{index}].id contains control characters.")
|
|
|
|
label = entry.get("label")
|
|
if not isinstance(label, str) or not label.strip():
|
|
_error(f"profile {profile_name!r}: models[{index}].label must be a non-empty string.")
|
|
elif len(label) > MAX_MODEL_LABEL_CHARS:
|
|
_error(f"profile {profile_name!r}: models[{index}].label exceeds {MAX_MODEL_LABEL_CHARS} characters.")
|
|
|
|
if "vision" in entry and not isinstance(entry["vision"], bool):
|
|
_error(f"profile {profile_name!r}: models[{index}].vision must be a boolean.")
|
|
|
|
if "notes" in entry:
|
|
if not isinstance(entry["notes"], str):
|
|
_error(f"profile {profile_name!r}: models[{index}].notes must be a string.")
|
|
elif len(entry["notes"]) > MAX_NOTE_CHARS:
|
|
_error(f"profile {profile_name!r}: models[{index}].notes exceeds {MAX_NOTE_CHARS} characters.")
|
|
|
|
|
|
def _validate_profile(profile_name: str, raw: object) -> None:
|
|
if not isinstance(raw, dict):
|
|
_error(f"profile {profile_name!r} must be an object.")
|
|
return
|
|
provider = raw.get("provider")
|
|
_require(
|
|
isinstance(provider, str) and provider.strip(),
|
|
f"profile {profile_name!r}: missing non-empty 'provider'.",
|
|
)
|
|
models = raw.get("models")
|
|
if not isinstance(models, list):
|
|
_error(f"profile {profile_name!r}: 'models' must be a list.")
|
|
return
|
|
if len(models) > MAX_MODELS_PER_PROFILE:
|
|
_error(f"profile {profile_name!r}: more than {MAX_MODELS_PER_PROFILE} models; is this intentional?")
|
|
seen_ids: set[str] = set()
|
|
for index, entry in enumerate(models):
|
|
_validate_model_entry(profile_name, index, entry)
|
|
if isinstance(entry, dict) and isinstance(entry.get("id"), str):
|
|
model_id = entry["id"].strip()
|
|
if model_id in seen_ids:
|
|
_error(f"profile {profile_name!r}: duplicate model id {model_id!r}.")
|
|
seen_ids.add(model_id)
|
|
|
|
|
|
def main() -> int:
|
|
try:
|
|
with CATALOG_PATH.open("r", encoding="utf-8") as handle:
|
|
payload = json.load(handle)
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
print(f"ERROR: cannot read {CATALOG_PATH}: {exc}", file=sys.stderr)
|
|
return 1
|
|
|
|
if not isinstance(payload, dict):
|
|
print(f"ERROR: {CATALOG_PATH} must contain a JSON object.", file=sys.stderr)
|
|
return 1
|
|
|
|
version = payload.get("version")
|
|
if version != 1:
|
|
_error(f"catalog 'version' must be 1, got {version!r}.")
|
|
|
|
updated_at = payload.get("updated_at")
|
|
if not isinstance(updated_at, str) or not updated_at.strip():
|
|
_error("catalog is missing 'updated_at'.")
|
|
|
|
profiles = payload.get("profiles")
|
|
if not isinstance(profiles, dict):
|
|
_error("catalog is missing 'profiles' object.")
|
|
profiles = {}
|
|
|
|
for required_key, expected_provider in REQUIRED_PROFILE_KEYS.items():
|
|
if required_key not in profiles:
|
|
_error(f"missing required profile {required_key!r} for provider {expected_provider!r}.")
|
|
continue
|
|
raw = profiles[required_key]
|
|
_validate_profile(required_key, raw)
|
|
if isinstance(raw, dict):
|
|
actual_provider = (raw.get("provider") or "").strip()
|
|
if actual_provider != expected_provider:
|
|
_error(
|
|
f"profile {required_key!r}: provider {actual_provider!r} "
|
|
f"does not match expected {expected_provider!r}.",
|
|
)
|
|
|
|
for profile_name, raw in profiles.items():
|
|
if profile_name not in REQUIRED_PROFILE_KEYS:
|
|
_validate_profile(profile_name, raw)
|
|
|
|
if _ERRORS:
|
|
print(f"ERROR: {CATALOG_PATH} failed validation:", file=sys.stderr)
|
|
for message in _ERRORS:
|
|
print(f" - {message}", file=sys.stderr)
|
|
return 1
|
|
|
|
model_count = sum(
|
|
len(p.get("models") or []) for p in profiles.values() if isinstance(p, dict)
|
|
)
|
|
print(f"OK: {CATALOG_PATH} valid ({len(profiles)} profiles, {model_count} models).")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|