From ba13db7f8a33a3d6e5fe8a1ac2a1c19b15ea9bf6 Mon Sep 17 00:00:00 2001 From: Arcum42 Date: Sat, 4 Jul 2026 12:58:32 -0700 Subject: [PATCH] More refactoring. --- routes/llm_routes.py | 1234 +++++------------------- utils/llm/providers/lmstudio/client.py | 39 + utils/llm/routes_helpers.py | 239 +++-- utils/llm/service.py | 372 +++++++ 4 files changed, 823 insertions(+), 1061 deletions(-) diff --git a/routes/llm_routes.py b/routes/llm_routes.py index 4f75fa0..f733a9d 100644 --- a/routes/llm_routes.py +++ b/routes/llm_routes.py @@ -4,7 +4,6 @@ Handles LLM chat endpoints for Ollama, LM Studio, and Native CLIP integration. """ import json -from contextvars import ContextVar from aiohttp import web from pathlib import Path from datetime import datetime @@ -17,7 +16,16 @@ from ..utils.logger import get_logger from ..utils.settings import get_setting from ..utils.llm import native as llm_native, presets, routes_helpers, set_llm_error_reporter, system_prompts from ..utils.llm import service as llm -from ..utils.llm.routes_helpers import get_compatible_models, get_available_presets_full +from ..utils.llm.routes_helpers import ( + get_compatible_models, + get_available_presets_full, + clear_last_llm_error, + set_last_llm_error, + write_sse_error_from_exception, + write_provider_and_native_error_if_unavailable, + error_response as llm_error_response, + error_response_from_exception as llm_error_response_from_exception, +) from ..utils.llm.provider_keys import ( LMSTUDIO_REST_KEY, NATIVE_KEY, @@ -39,7 +47,14 @@ def _status_key(provider_key: str) -> str: # Route list for documentation and registration tracking _route_list = [] -_last_llm_error = ContextVar('sageutils_last_llm_error', default=None) + + +def _append_route(method: str, path: str, description: str) -> None: + _route_list.append({ + 'method': method, + 'path': path, + 'description': description, + }) def _ensure_promptserver_progress_context(): @@ -64,464 +79,73 @@ def _ensure_promptserver_progress_context(): def handle_llm_error_callback(payload): """Capture structured LLM errors raised in provider helpers for route responses.""" - _last_llm_error.set(payload) + set_last_llm_error(payload) -def _clear_last_llm_error(): - """Clear any stale LLM error payload before route handling.""" - _last_llm_error.set(None) - - -def _error_response_with_metadata(message, status=500, error_code='LLM_ERROR', provider=None, operation=None, cause=None): - """Return a standardized error response including optional LLM metadata.""" - extra = {'error_code': error_code} - if provider: - extra['provider'] = provider - if operation: - extra['operation'] = operation - if cause: - extra['cause'] = cause - return error_response(message, status=status, **extra) - - -def _error_response_from_exception(prefix, exc, status=500, default_error_code='LLM_ROUTE_ERROR'): - """Build a standardized error response from the last structured LLM error (if available).""" - payload = _last_llm_error.get() - _clear_last_llm_error() - - if payload: - message = payload.get('scoped_message') or payload.get('message') or f'{prefix}: {str(exc)}' - error_type = str(payload.get('error_type', 'ERROR')).upper() - return _error_response_with_metadata( - message, - status=status, - error_code=f'LLM_{error_type}', - provider=payload.get('provider'), - operation=payload.get('operation'), - cause=payload.get('cause'), - ) - - return _error_response_with_metadata( - f'{prefix}: {str(exc)}', - status=status, - error_code=default_error_code, - ) - - -def register_routes(routes_instance): - """ - Register LLM-related routes. - - Args: - routes_instance: The PromptServer routes instance - - Returns: - int: Number of routes registered - """ - - # Wire up structured LLM error payloads so frontend can display backend messages. - set_llm_error_reporter(handle_llm_error_callback) - - global _route_list - _route_list.clear() - +def _register_status_and_model_routes(routes_instance): @routes_instance.get('/sage_llm/status') @route_error_handler async def get_llm_status(request): """ Check if REST/OpenAI/Native providers are available and enabled. - - Returns: - { - "success": true, - "lmstudio_rest": { - "available": bool, - "enabled": bool, - "url": str (if custom) - }, - "ollama_rest": { - "available": bool, - "enabled": bool, - "url": str (if custom) - }, - "openai": { - "available": bool, - "enabled": bool, - "url": str (if custom) - }, - "native": { - "available": bool, - "enabled": bool - } - } """ try: - # Ensure initialization has been attempted - try: - llm.ensure_llm_initialized() - except Exception as init_error: - logger.warning(f"LLM initialization warning: {init_error}") - - # Check settings - lmstudio_rest_enabled = get_setting("enable_lmstudio_rest", False) - ollama_rest_enabled = get_setting("enable_ollama_rest", False) - openai_enabled = get_setting("enable_openai", False) - - # Check availability - lmstudio_rest_available = llm.LMSTUDIO_REST_AVAILABLE and lmstudio_rest_enabled - ollama_rest_available = llm.OLLAMA_REST_AVAILABLE and ollama_rest_enabled - openai_available = llm.OPENAI_AVAILABLE and openai_enabled - native_models = llm_native.get_native_models() - native_available = len(native_models) > 0 - - # Get custom URLs if applicable - lmstudio_rest_info = { - "available": lmstudio_rest_available, - "enabled": lmstudio_rest_enabled, - } - ollama_rest_info = { - "available": ollama_rest_available, - "enabled": ollama_rest_enabled, - } - openai_info = { - "available": openai_available, - "enabled": openai_enabled, - } - + status = llm.get_llm_status() + if get_setting("lmstudio_use_custom_url", False): - lmstudio_rest_info["url"] = get_setting("lmstudio_custom_url", "") + status[LMSTUDIO_REST_KEY]["url"] = get_setting("lmstudio_custom_url", "") if get_setting("openai_use_custom_url", False): - openai_info["url"] = get_setting("openai_base_url", "") + status[OPENAI_KEY]["url"] = get_setting("openai_base_url", "") - native_info = { - "available": native_available, - "enabled": True, - } - - return success_response(data={ - LMSTUDIO_REST_KEY: lmstudio_rest_info, - OLLAMA_REST_KEY: ollama_rest_info, - OPENAI_KEY: openai_info, - NATIVE_KEY: native_info, - }) - + return success_response(data=status) except Exception as e: logger.error(f"LLM status error: {str(e)}") return error_response(f"Failed to get LLM status: {str(e)}", status=500) - - _route_list.append({ - "method": "GET", - "path": "/sage_llm/status", - "description": "Check REST/OpenAI/Native availability" - }) - + + _append_route("GET", "/sage_llm/status", "Check REST/OpenAI/Native availability") + @routes_instance.get('/sage_llm/models') @route_error_handler async def get_llm_models(request): - """ - Get available text models from REST providers, OpenAI, and Native CLIP. - - Query params: - force: bool - Force re-initialization of LLM providers (default: False) - - Returns: - { - "success": true, - "models": { - "lmstudio_rest": ["model1", "model2", ...], - "ollama_rest": ["model1", "model2", ...], - "openai": ["model1", "model2", ...], - "native": ["clip1.safetensors", ...] - }, - "status": { - "lmstudio_rest_available": bool, - "ollama_rest_available": bool, - "openai_available": bool, - "native_available": bool - } - } - """ + """Get available text models from REST providers, OpenAI, and Native CLIP.""" try: - # Check if force re-initialization is requested force = request.rel_url.query.get('force', '').lower() == 'true' - - if force: - logger.info("Force re-initializing LLM providers...") - llm.reset_llm_initialization_state() - - # Initialize LLM services if needed - llm.ensure_llm_initialized() - - # Get models from configured providers - lmstudio_rest_models = [] - native_models = llm_native.get_native_models() - lmstudio_rest_tool_models = [] - ollama_rest_tool_models = [] - openai_tool_models = [] - lmstudio_rest_reasoning_models = [] - ollama_rest_reasoning_models = [] - openai_reasoning_models = [] - lmstudio_rest_capabilities = {} - ollama_rest_capabilities = {} - openai_capabilities = {} - - lmstudio_rest_enabled = get_setting("enable_lmstudio_rest", False) - ollama_rest_enabled = get_setting("enable_ollama_rest", False) - openai_enabled = get_setting("enable_openai", False) - - if lmstudio_rest_enabled and llm.LMSTUDIO_REST_AVAILABLE: - try: - models = llm.get_lmstudio_rest_models() - lmstudio_rest_models = get_compatible_models(LMSTUDIO_REST_KEY, models) - lmstudio_rest_tool_models = get_compatible_models(LMSTUDIO_REST_KEY, llm.get_lmstudio_rest_tool_models()) - lmstudio_rest_reasoning_models = get_compatible_models(LMSTUDIO_REST_KEY, llm.get_lmstudio_rest_reasoning_models()) - lmstudio_rest_capabilities = llm.get_lmstudio_rest_model_capabilities_map() - except Exception as e: - logger.warning(f"Failed to get LM Studio REST models: {e}") - - ollama_rest_models = [] - openai_models = [] - - if ollama_rest_enabled and llm.OLLAMA_REST_AVAILABLE: - try: - models = llm.get_ollama_rest_models() - ollama_rest_models = get_compatible_models(OLLAMA_REST_KEY, models) - ollama_rest_tool_models = get_compatible_models(OLLAMA_REST_KEY, llm.get_ollama_rest_tool_models()) - ollama_rest_reasoning_models = get_compatible_models(OLLAMA_REST_KEY, llm.get_ollama_rest_reasoning_models()) - ollama_rest_capabilities = llm.get_ollama_rest_model_capabilities_map() - except Exception as e: - logger.warning(f"Failed to get Ollama REST models: {e}") - - if openai_enabled and llm.OPENAI_AVAILABLE: - try: - models = llm.get_openai_models() - openai_models = get_compatible_models(OPENAI_KEY, models) - openai_tool_models = get_compatible_models(OPENAI_KEY, llm.get_openai_tool_models()) - openai_reasoning_models = get_compatible_models(OPENAI_KEY, llm.get_openai_reasoning_models()) - openai_capabilities = llm.get_openai_model_capabilities_map() - except Exception as e: - logger.warning(f"Failed to get OpenAI models: {e}") - - native_capabilities = { - model_name: { - 'name': model_name, - 'provider': NATIVE_KEY, - 'vision': False, - 'tool_use': False, - 'reasoning': False, - 'thinking': False, - 'supported_modalities': ['text'], - 'confidence': 'guess', - } - for model_name in native_models - } - - return success_response(data={ - "models": { - LMSTUDIO_REST_KEY: lmstudio_rest_models, - OLLAMA_REST_KEY: ollama_rest_models, - OPENAI_KEY: openai_models, - NATIVE_KEY: native_models, - }, - "capabilities": { - LMSTUDIO_REST_KEY: lmstudio_rest_capabilities, - OLLAMA_REST_KEY: ollama_rest_capabilities, - OPENAI_KEY: openai_capabilities, - NATIVE_KEY: native_capabilities, - }, - "tool_models": { - LMSTUDIO_REST_KEY: lmstudio_rest_tool_models, - OLLAMA_REST_KEY: ollama_rest_tool_models, - OPENAI_KEY: openai_tool_models, - NATIVE_KEY: [], - }, - "reasoning_models": { - LMSTUDIO_REST_KEY: lmstudio_rest_reasoning_models, - OLLAMA_REST_KEY: ollama_rest_reasoning_models, - OPENAI_KEY: openai_reasoning_models, - NATIVE_KEY: [], - }, - "status": { - _status_key(LMSTUDIO_REST_KEY): len(lmstudio_rest_models) > 0, - _status_key(OLLAMA_REST_KEY): len(ollama_rest_models) > 0, - _status_key(OPENAI_KEY): len(openai_models) > 0, - _status_key(NATIVE_KEY): len(native_models) > 0, - } - }) - + data = llm.get_llm_models(force=force) + return success_response(data=data) except Exception as e: logger.error(f"Failed to get models: {str(e)}") return error_response(f"Failed to get models: {str(e)}", status=500) - - _route_list.append({ - "method": "GET", - "path": "/sage_llm/models", - "description": "Get available text models" - }) - + + _append_route("GET", "/sage_llm/models", "Get available text models") + @routes_instance.get('/sage_llm/vision_models') @route_error_handler async def get_llm_vision_models(request): - """ - Get available vision models from REST providers and OpenAI. - - Query params: - force: bool - Force re-initialization of LLM providers (default: False) - - Returns: - { - "success": true, - "models": { - "lmstudio_rest": ["llava-v1.6", ...], - "ollama_rest": ["llava", "bakllava", ...], - "openai": ["gpt-4o", ...], - "native": [] - }, - "status": { - "lmstudio_rest_available": bool, - "ollama_rest_available": bool, - "openai_available": bool, - "native_available": bool - } - } - """ + """Get available vision models from REST providers and OpenAI.""" try: - # Check if force re-initialization is requested force = request.rel_url.query.get('force', '').lower() == 'true' - - if force: - logger.info("Force re-initializing LLM providers for vision models...") - llm.reset_llm_initialization_state() - - # Initialize LLM services if needed - llm.ensure_llm_initialized() - - # Get vision models from configured providers - lmstudio_rest_models = [] - native_models = [] - lmstudio_rest_tool_models = [] - ollama_rest_tool_models = [] - openai_tool_models = [] - lmstudio_rest_reasoning_models = [] - ollama_rest_reasoning_models = [] - openai_reasoning_models = [] - lmstudio_rest_capabilities = {} - ollama_rest_capabilities = {} - openai_capabilities = {} - - lmstudio_rest_enabled = get_setting("enable_lmstudio_rest", False) - ollama_rest_enabled = get_setting("enable_ollama_rest", False) - openai_enabled = get_setting("enable_openai", False) - - if lmstudio_rest_enabled and llm.LMSTUDIO_REST_AVAILABLE: - try: - models = llm.get_lmstudio_rest_vision_models() - lmstudio_rest_models = get_compatible_models(LMSTUDIO_REST_KEY, models) - lmstudio_rest_tool_models = get_compatible_models(LMSTUDIO_REST_KEY, llm.get_lmstudio_rest_tool_models()) - lmstudio_rest_reasoning_models = get_compatible_models(LMSTUDIO_REST_KEY, llm.get_lmstudio_rest_reasoning_models()) - lmstudio_rest_capabilities = llm.get_lmstudio_rest_model_capabilities_map() - except Exception as e: - logger.warning(f"Failed to get LM Studio REST vision models: {e}") - - ollama_rest_models = [] - openai_vision_models = [] - - if ollama_rest_enabled and llm.OLLAMA_REST_AVAILABLE: - try: - models = llm.get_ollama_rest_vision_models() - ollama_rest_models = get_compatible_models(OLLAMA_REST_KEY, models) - ollama_rest_tool_models = get_compatible_models(OLLAMA_REST_KEY, llm.get_ollama_rest_tool_models()) - ollama_rest_reasoning_models = get_compatible_models(OLLAMA_REST_KEY, llm.get_ollama_rest_reasoning_models()) - ollama_rest_capabilities = llm.get_ollama_rest_model_capabilities_map() - except Exception as e: - logger.warning(f"Failed to get Ollama REST vision models: {e}") - - if openai_enabled and llm.OPENAI_AVAILABLE: - try: - models = llm.get_openai_vision_models() - openai_vision_models = get_compatible_models(OPENAI_KEY, models) - openai_tool_models = get_compatible_models(OPENAI_KEY, llm.get_openai_tool_models()) - openai_reasoning_models = get_compatible_models(OPENAI_KEY, llm.get_openai_reasoning_models()) - openai_capabilities = llm.get_openai_model_capabilities_map() - except Exception as e: - logger.warning(f"Failed to get OpenAI vision models: {e}") - - native_capabilities = {} - - return success_response(data={ - "models": { - LMSTUDIO_REST_KEY: lmstudio_rest_models, - OLLAMA_REST_KEY: ollama_rest_models, - OPENAI_KEY: openai_vision_models, - NATIVE_KEY: native_models, - }, - "capabilities": { - LMSTUDIO_REST_KEY: lmstudio_rest_capabilities, - OLLAMA_REST_KEY: ollama_rest_capabilities, - OPENAI_KEY: openai_capabilities, - NATIVE_KEY: native_capabilities, - }, - "tool_models": { - LMSTUDIO_REST_KEY: lmstudio_rest_tool_models, - OLLAMA_REST_KEY: ollama_rest_tool_models, - OPENAI_KEY: openai_tool_models, - NATIVE_KEY: [], - }, - "reasoning_models": { - LMSTUDIO_REST_KEY: lmstudio_rest_reasoning_models, - OLLAMA_REST_KEY: ollama_rest_reasoning_models, - OPENAI_KEY: openai_reasoning_models, - NATIVE_KEY: [], - }, - "status": { - _status_key(LMSTUDIO_REST_KEY): len(lmstudio_rest_models) > 0, - _status_key(OLLAMA_REST_KEY): len(ollama_rest_models) > 0, - _status_key(OPENAI_KEY): len(openai_vision_models) > 0, - _status_key(NATIVE_KEY): False, - } - }) - + data = llm.get_llm_vision_models(force=force) + return success_response(data=data) except Exception as e: logger.error(f"Failed to get vision models: {str(e)}") return error_response(f"Failed to get vision models: {str(e)}", status=500) - - _route_list.append({ - "method": "GET", - "path": "/sage_llm/vision_models", - "description": "Get available vision models" - }) - + + _append_route("GET", "/sage_llm/vision_models", "Get available vision models") + + +def _register_prompt_routes(routes_instance): @routes_instance.get('/sage_llm/prompts') @route_error_handler async def get_llm_prompts(request): - """ - Get available LLM prompt templates from llm_prompts.json. - - Returns: - { - "success": true, - "prompts": { - "base": {...}, - "extra": {...} - } - } - """ + """Get available LLM prompt templates from llm_prompts.json.""" try: - - return success_response(data={ - "prompts": llm_prompts - }) - + return success_response(data={"prompts": llm_prompts}) except Exception as e: logger.error(f"Failed to get prompts: {str(e)}") return error_response(f"Failed to get prompts: {str(e)}", status=500) - - _route_list.append({ - "method": "GET", - "path": "/sage_llm/prompts", - "description": "Get LLM prompt templates" - }) + + _append_route("GET", "/sage_llm/prompts", "Get LLM prompt templates") @routes_instance.get('/sage_llm/integration_profiles') @route_error_handler @@ -534,46 +158,21 @@ def register_routes(routes_instance): logger.error(f"Failed to get integration profiles: {str(e)}") return error_response(f"Failed to get integration profiles: {str(e)}", status=500) - _route_list.append({ - "method": "GET", - "path": "/sage_llm/integration_profiles", - "description": "Get tool and MCP profile metadata" - }) - + _append_route("GET", "/sage_llm/integration_profiles", "Get tool and MCP profile metadata") + + +def _register_generation_routes(routes_instance): @routes_instance.post('/sage_llm/generate') @route_error_handler async def generate_llm_response(request): - """ - Generate a response from an LLM (non-streaming for Phase 1). - - Request body: - { - "provider": "lmstudio_rest" | "ollama_rest" | "openai" | "native", - "model": str, - "prompt": str, - "system_prompt": str (optional), - "options": { - "temperature": float, - "seed": int, - ... - } - } - - Returns: - { - "success": true, - "response": str, - "provider": str, - "model": str - } - """ + """Generate a response from an LLM (non-streaming for Phase 1).""" try: - _clear_last_llm_error() + clear_last_llm_error() data = await request.json() is_valid, error_msg, payload = routes_helpers.parse_generation_request(data) if not is_valid: - return _error_response_with_metadata(error_msg, status=400, error_code='LLM_VALIDATION_ERROR') + return llm_error_response(error_msg, status=400, error_code='LLM_VALIDATION_ERROR') provider = payload['provider'] model = payload['model'] @@ -581,31 +180,19 @@ def register_routes(routes_instance): system_prompt = payload['system_prompt'] options = payload['options'] - llm.ensure_llm_initialized(force=True) - - is_available, error_msg, error_code, status_code = routes_helpers.get_provider_availability_error( - provider, - 'generate', - ) - if not is_available: - return _error_response_with_metadata( - error_msg, - status=status_code, - error_code=error_code, + if not llm.is_provider_available(provider, force=True): + return llm_error_response( + f"Provider {provider} is not available", + status=503, + error_code='LLM_PROVIDER_UNAVAILABLE', provider=provider, operation='generate', ) - - is_native_valid, native_msg, native_code, native_status = routes_helpers.validate_native_model_availability( - provider, - model, - 'generate', - ) - if not is_native_valid: - return _error_response_with_metadata( - native_msg, - status=native_status, - error_code=native_code, + if provider == routes_helpers.NATIVE_KEY and not llm.is_native_model_available(model): + return llm_error_response( + f"Native CLIP model '{model}' is not available", + status=404, + error_code='LLM_MODEL_NOT_FOUND', provider=provider, operation='generate', ) @@ -624,51 +211,26 @@ def register_routes(routes_instance): "provider": provider, "model": model, }) - except ValueError as e: logger.error(f"Validation error in generate: {str(e)}") - return _error_response_from_exception('Validation error in generate', e, status=400, default_error_code='LLM_VALIDATION_ERROR') + return llm_error_response_from_exception('Validation error in generate', e, status=400, default_error_code='LLM_VALIDATION_ERROR') except Exception as e: logger.error(f"Failed to generate response: {str(e)}") - return _error_response_from_exception('Failed to generate response', e, status=500, default_error_code='LLM_GENERATION_ERROR') - - _route_list.append({ - "method": "POST", - "path": "/sage_llm/generate", - "description": "Generate text response from LLM" - }) - + return llm_error_response_from_exception('Failed to generate response', e, status=500, default_error_code='LLM_GENERATION_ERROR') + + _append_route("POST", "/sage_llm/generate", "Generate text response from LLM") + @routes_instance.post('/sage_llm/generate_stream') @route_error_handler async def generate_llm_response_stream(request): - """ - Generate a streaming response from an LLM using Server-Sent Events. - - Request body: - { - "provider": "lmstudio_rest" | "ollama_rest" | "openai" | "native", - "model": str, - "prompt": str, - "system_prompt": str (optional), - "options": { - "temperature": float, - "seed": int, - ... - } - } - - Response: Server-Sent Events (SSE) stream - data: {"chunk": "text", "done": false} - data: {"chunk": "more", "done": false} - data: {"chunk": "", "done": true, "full_response": "complete text"} - """ + """Generate a streaming response from an LLM using Server-Sent Events.""" try: - _clear_last_llm_error() + clear_last_llm_error() data = await request.json() is_valid, error_msg, payload = routes_helpers.parse_generation_request(data) if not is_valid: - return _error_response_with_metadata(error_msg, status=400, error_code='LLM_VALIDATION_ERROR') + return llm_error_response(error_msg, status=400, error_code='LLM_VALIDATION_ERROR') provider = payload['provider'] model = payload['model'] @@ -679,24 +241,25 @@ def register_routes(routes_instance): response = await routes_helpers.prepare_sse_response(request) try: - if await routes_helpers.write_provider_availability_error_if_unavailable( - response, - provider, - 'generate_stream', - ): + if not llm.is_provider_available(provider, force=True): + await routes_helpers.write_sse_error_and_close( + response, + f"Provider {provider} is not available", + error_code='LLM_PROVIDER_UNAVAILABLE', + provider=provider, + operation='generate_stream', + ) return response - if provider == NATIVE_KEY: - is_valid_native, native_error = routes_helpers.validate_native_model_availability(provider, model) - if not is_valid_native: - await routes_helpers.write_sse_error_and_close( - response, - native_error, - error_code='LLM_MODEL_NOT_FOUND', - provider=NATIVE_KEY, - operation='generate_stream', - ) - return response + if provider == routes_helpers.NATIVE_KEY and not llm.is_native_model_available(model): + await routes_helpers.write_sse_error_and_close( + response, + f"Native CLIP model '{model}' is not available", + error_code='LLM_MODEL_NOT_FOUND', + provider=provider, + operation='generate_stream', + ) + return response await routes_helpers.stream_sse_chunks( response, @@ -711,63 +274,27 @@ def register_routes(routes_instance): ) await response.write_eof() return response - except Exception as e: logger.error(f"Error during streaming: {str(e)}") - payload = _last_llm_error.get() - _clear_last_llm_error() - error_chunk = routes_helpers.format_sse_error_chunk( - payload.get('scoped_message') if payload else str(e), - error_code=f"LLM_{str(payload.get('error_type', 'STREAM_ERROR')).upper()}" if payload else 'LLM_STREAM_ERROR', - provider=payload.get('provider') if payload else provider, - operation=payload.get('operation') if payload else 'generate_stream', - cause=payload.get('cause') if payload else None, - ) - await response.write(error_chunk.encode('utf-8')) - await response.write_eof() + await write_sse_error_from_exception(response, e, 'generate_stream', prefix='Streaming error') return response - except Exception as e: logger.error(f"Failed to initialize streaming: {str(e)}") - return _error_response_from_exception('Failed to initialize streaming', e, status=500, default_error_code='LLM_STREAM_INIT_ERROR') - - _route_list.append({ - "method": "POST", - "path": "/sage_llm/generate_stream", - "description": "Generate streaming text response from LLM (SSE)" - }) - + return llm_error_response_from_exception('Failed to initialize streaming', e, status=500, default_error_code='LLM_STREAM_INIT_ERROR') + + _append_route("POST", "/sage_llm/generate_stream", "Generate streaming text response from LLM (SSE)") + @routes_instance.post('/sage_llm/vision_generate') @route_error_handler async def generate_vision_response(request): - """ - Generate a response from a vision LLM (non-streaming). - - Request body: - { - "provider": "lmstudio_rest" | "ollama_rest" | "openai", - "model": str, - "prompt": str, - "images": [base64_encoded_image, ...], - "system_prompt": str (optional), - "options": {...} - } - - Returns: - { - "success": true, - "response": str, - "provider": str, - "model": str - } - """ + """Generate a response from a vision LLM (non-streaming).""" try: - _clear_last_llm_error() + clear_last_llm_error() data = await request.json() is_valid, error_msg, payload = routes_helpers.parse_vision_generation_request(data) if not is_valid: - return _error_response_with_metadata(error_msg, status=400, error_code='LLM_VALIDATION_ERROR') + return llm_error_response(error_msg, status=400, error_code='LLM_VALIDATION_ERROR') provider = payload['provider'] model = payload['model'] @@ -776,31 +303,33 @@ def register_routes(routes_instance): system_prompt = payload['system_prompt'] options = payload['options'] - if provider == NATIVE_KEY: - return _error_response_with_metadata( - 'Native provider does not support vision generation', - status=400, - error_code='LLM_MODEL_CAPABILITY_ERROR', - provider=NATIVE_KEY, - ) - - can_do_vision, capability_error = routes_helpers.check_model_vision_capability(provider, model) + can_do_vision, capability_error = llm.is_model_vision_capable(provider, model) if not can_do_vision: - return _error_response_with_metadata( + return llm_error_response( capability_error, status=400, error_code='LLM_MODEL_CAPABILITY_ERROR', provider=provider, ) - llm.ensure_llm_initialized(force=True) - - is_available, error_msg, error_code, status_code = routes_helpers.get_provider_availability_error( - provider, - 'vision_generate', - ) - if not is_available: - return _error_response_with_metadata( + if not llm.is_provider_available(provider, force=True): + return llm_error_response( + f"Provider {provider} is not available", + status=503, + error_code='LLM_PROVIDER_UNAVAILABLE', + provider=provider, + operation='vision_generate', + ) + if provider == routes_helpers.NATIVE_KEY and not llm.is_native_model_available(model): + return llm_error_response( + f"Native CLIP model '{model}' is not available", + status=404, + error_code='LLM_MODEL_NOT_FOUND', + provider=provider, + operation='vision_generate', + ) + if not is_valid: + return llm_error_response( error_msg, status=status_code, error_code=error_code, @@ -822,45 +351,26 @@ def register_routes(routes_instance): "provider": provider, "model": model, }) - except ValueError as e: logger.error(f"Validation error in vision generate: {str(e)}") - return _error_response_from_exception('Validation error in vision generate', e, status=400, default_error_code='LLM_VALIDATION_ERROR') + return llm_error_response_from_exception('Validation error in vision generate', e, status=400, default_error_code='LLM_VALIDATION_ERROR') except Exception as e: logger.error(f"Failed to generate vision response: {str(e)}") - return _error_response_from_exception('Failed to generate vision response', e, status=500, default_error_code='LLM_VISION_GENERATION_ERROR') - - _route_list.append({ - "method": "POST", - "path": "/sage_llm/vision_generate", - "description": "Generate vision response from LLM" - }) - + return llm_error_response_from_exception('Failed to generate vision response', e, status=500, default_error_code='LLM_VISION_GENERATION_ERROR') + + _append_route("POST", "/sage_llm/vision_generate", "Generate vision response from LLM") + @routes_instance.post('/sage_llm/vision_generate_stream') @route_error_handler async def generate_vision_response_stream(request): - """ - Generate a streaming response from a vision LLM using Server-Sent Events. - - Request body: - { - "provider": "lmstudio_rest" | "ollama_rest" | "openai", - "model": str, - "prompt": str, - "images": [base64_encoded_image, ...], - "system_prompt": str (optional), - "options": {...} - } - - Response: Server-Sent Events (SSE) stream - """ + """Generate a streaming response from a vision LLM using Server-Sent Events.""" try: - _clear_last_llm_error() + clear_last_llm_error() data = await request.json() is_valid, error_msg, payload = routes_helpers.parse_vision_generation_request(data) if not is_valid: - return _error_response_with_metadata(error_msg, status=400, error_code='LLM_VALIDATION_ERROR') + return llm_error_response(error_msg, status=400, error_code='LLM_VALIDATION_ERROR') provider = payload['provider'] model = payload['model'] @@ -869,25 +379,36 @@ def register_routes(routes_instance): system_prompt = payload['system_prompt'] options = payload['options'] - can_do_vision, capability_error = routes_helpers.check_model_vision_capability(provider, model) + can_do_vision, capability_error = llm.is_model_vision_capable(provider, model) if not can_do_vision: - return _error_response_with_metadata( + return llm_error_response( capability_error, status=400, error_code='LLM_MODEL_CAPABILITY_ERROR', provider=provider, ) - llm.ensure_llm_initialized(force=True) - response = await routes_helpers.prepare_sse_response(request) try: - if await routes_helpers.write_provider_availability_error_if_unavailable( - response, - provider, - 'vision_generate_stream', - ): + if not llm.is_provider_available(provider, force=True): + await routes_helpers.write_sse_error_and_close( + response, + f"Provider {provider} is not available", + error_code='LLM_PROVIDER_UNAVAILABLE', + provider=provider, + operation='vision_generate_stream', + ) + return response + + if provider == routes_helpers.NATIVE_KEY and not llm.is_native_model_available(model): + await routes_helpers.write_sse_error_and_close( + response, + f"Native CLIP model '{model}' is not available", + error_code='LLM_MODEL_NOT_FOUND', + provider=provider, + operation='vision_generate_stream', + ) return response await routes_helpers.stream_sse_chunks( @@ -903,44 +424,22 @@ def register_routes(routes_instance): ) await response.write_eof() return response - except Exception as e: logger.error(f"Error during vision streaming: {str(e)}") - payload = _last_llm_error.get() - _clear_last_llm_error() - error_chunk = routes_helpers.format_sse_error_chunk( - payload.get('scoped_message') if payload else str(e), - error_code=f"LLM_{str(payload.get('error_type', 'STREAM_ERROR')).upper()}" if payload else 'LLM_STREAM_ERROR', - provider=payload.get('provider') if payload else provider, - operation=payload.get('operation') if payload else 'vision_generate_stream', - cause=payload.get('cause') if payload else None, - ) - await response.write(error_chunk.encode('utf-8')) - await response.write_eof() + await write_sse_error_from_exception(response, e, 'vision_generate_stream', prefix='Vision streaming error') return response - except Exception as e: logger.error(f"Failed to initialize vision streaming: {str(e)}") - return _error_response_from_exception('Failed to initialize vision streaming', e, status=500, default_error_code='LLM_VISION_STREAM_INIT_ERROR') - - _route_list.append({ - "method": "POST", - "path": "/sage_llm/vision_generate_stream", - "description": "Generate streaming vision response from LLM (SSE)" - }) - + return llm_error_response_from_exception('Failed to initialize vision streaming', e, status=500, default_error_code='LLM_VISION_STREAM_INIT_ERROR') + + _append_route("POST", "/sage_llm/vision_generate_stream", "Generate streaming vision response from LLM (SSE)") + + +def _register_system_prompt_routes(routes_instance): @routes_instance.get('/sage_utils/system_prompts/{prompt_id}.md') @route_error_handler async def get_system_prompt(request): - """ - Serve system prompt markdown files from assets (built-in) or user directory (custom). - - Args: - prompt_id: ID of the system prompt to retrieve - - Returns: - Markdown content of the system prompt - """ + """Serve system prompt markdown files from assets or user directory.""" try: prompt_id = request.match_info['prompt_id'] content = system_prompts.get_system_prompt_text(prompt_id) @@ -949,31 +448,16 @@ def register_routes(routes_instance): return error_response(f"System prompt '{prompt_id}' not found", status=404) return web.Response(text=content, content_type='text/markdown') - except Exception as e: logger.error(f"Error loading system prompt: {str(e)}") return error_response(f"Failed to load system prompt: {str(e)}", status=500) - - _route_list.append({ - "method": "GET", - "path": "/sage_utils/system_prompts/{prompt_id}.md", - "description": "Get system prompt markdown file" - }) - + + _append_route("GET", "/sage_utils/system_prompts/{prompt_id}.md", "Get system prompt markdown file") + @routes_instance.post('/sage_llm/system_prompts/save') @route_error_handler async def save_system_prompt(request): - """ - Save a custom system prompt to user directory. - - Body: - { - "id": str, - "name": str, - "content": str, - "description": str (optional) - } - """ + """Save a custom system prompt to user directory.""" try: data = await request.json() @@ -988,29 +472,19 @@ def register_routes(routes_instance): payload['description'], ) return success_response({"id": payload['id'], "name": payload['name']}) - except ValueError as e: logger.error(f"Validation error saving system prompt: {str(e)}") return error_response(str(e), status=400) except Exception as e: logger.error(f"Error saving system prompt: {str(e)}") return error_response(f"Failed to save system prompt: {str(e)}", status=500) - - _route_list.append({ - "method": "POST", - "path": "/sage_llm/system_prompts/save", - "description": "Save custom system prompt" - }) - + + _append_route("POST", "/sage_llm/system_prompts/save", "Save custom system prompt") + @routes_instance.post('/sage_llm/system_prompts/delete') @route_error_handler async def delete_system_prompt(request): - """ - Delete a custom system prompt. - - Body: - {"id": str} - """ + """Delete a custom system prompt.""" try: data = await request.json() @@ -1023,84 +497,43 @@ def register_routes(routes_instance): return error_response(f"System prompt '{payload['id']}' not found", status=404) return success_response({"deleted": payload['id']}) - except Exception as e: logger.error(f"Error deleting system prompt: {str(e)}") return error_response(f"Failed to delete system prompt: {str(e)}", status=500) - - _route_list.append({ - "method": "POST", - "path": "/sage_llm/system_prompts/delete", - "description": "Delete custom system prompt" - }) - + + _append_route("POST", "/sage_llm/system_prompts/delete", "Delete custom system prompt") + @routes_instance.get('/sage_llm/system_prompts/list') @route_error_handler async def list_system_prompts(request): - """ - List all available system prompts (built-in + custom). - - Returns: - { - "success": true, - "prompts": { - "prompt_id": {"name": str, "description": str, "isBuiltin": bool} - } - } - """ + """List all available system prompts (built-in + custom).""" try: prompts = system_prompts.list_system_prompts() return success_response({"prompts": prompts}) - except Exception as e: logger.error(f"Error listing system prompts: {str(e)}") return error_response(f"Failed to list system prompts: {str(e)}", status=500) - - _route_list.append({ - "method": "GET", - "path": "/sage_llm/system_prompts/list", - "description": "List all system prompts" - }) - + + _append_route("GET", "/sage_llm/system_prompts/list", "List all system prompts") + + +def _register_preset_routes(routes_instance): @routes_instance.get('/sage_llm/presets/list') @route_error_handler async def list_presets(request): - """ - List all available LLM presets (built-in + custom). - - Returns: - { - "success": true, - "presets": { - "preset_id": {preset_data} - } - } - """ + """List all available LLM presets (built-in + custom).""" try: return success_response({"presets": presets.list_presets()}) - except Exception as e: logger.error(f"Error listing presets: {str(e)}") return error_response(f"Failed to list presets: {str(e)}", status=500) - - _route_list.append({ - "method": "GET", - "path": "/sage_llm/presets/list", - "description": "List all LLM presets" - }) - + + _append_route("GET", "/sage_llm/presets/list", "List all LLM presets") + @routes_instance.post('/sage_llm/presets/save') @route_error_handler async def save_preset(request): - """ - Save an LLM preset to user directory. - - Body: - { - "id": str, - "preset": {preset_data} - } - """ + """Save an LLM preset to user directory.""" try: data = await request.json() @@ -1110,29 +543,19 @@ def register_routes(routes_instance): saved_preset = presets.save_preset(payload['id'], payload['preset']) return success_response({"id": payload['id'], "preset": saved_preset}) - except ValueError as e: logger.error(f"Validation error saving preset: {str(e)}") return error_response(str(e), status=400) except Exception as e: logger.error(f"Error saving preset: {str(e)}") return error_response(f"Failed to save preset: {str(e)}", status=500) - - _route_list.append({ - "method": "POST", - "path": "/sage_llm/presets/save", - "description": "Save LLM preset" - }) - + + _append_route("POST", "/sage_llm/presets/save", "Save LLM preset") + @routes_instance.post('/sage_llm/presets/delete') @route_error_handler async def delete_preset(request): - """ - Delete a custom preset. - - Body: - {"id": str} - """ + """Delete a custom preset.""" try: data = await request.json() @@ -1145,81 +568,29 @@ def register_routes(routes_instance): return error_response(f"Preset '{payload['id']}' not found", status=404) return success_response({"deleted": payload['id']}) - except Exception as e: logger.error(f"Error deleting preset: {str(e)}") return error_response(f"Failed to delete preset: {str(e)}", status=500) - - _route_list.append({ - "method": "POST", - "path": "/sage_llm/presets/delete", - "description": "Delete LLM preset" - }) - + + _append_route("POST", "/sage_llm/presets/delete", "Delete LLM preset") + @routes_instance.get('/sage_llm/presets/all') @route_error_handler async def get_all_presets(request): - """ - Get all available LLM presets (built-in + custom) with full details. - This is useful for programmatic access from other parts of the code. - - Returns: - { - "success": true, - "presets": { - "preset_id": { - "name": str, - "description": str, - "provider": str, - "model": str, - "category": str, - "isBuiltin": bool, - "promptTemplate": str, - "systemPrompt": str, - "settings": {...} - } - } - } - """ + """Get all available LLM presets (built-in + custom) with full details.""" try: all_presets = get_available_presets_full() return success_response({"presets": all_presets}) - except Exception as e: logger.error(f"Error getting all presets: {str(e)}") return error_response(f"Failed to get presets: {str(e)}", status=500) - - _route_list.append({ - "method": "GET", - "path": "/sage_llm/presets/all", - "description": "Get all LLM presets with full details" - }) - + + _append_route("GET", "/sage_llm/presets/all", "Get all LLM presets with full details") + @routes_instance.post('/sage_llm/presets/generate_with_image') @route_error_handler async def generate_with_preset_and_image(request): - """ - Generate a response using a preset with an image. - This allows other parts of the code to use LLM presets programmatically. - - Request body: - { - "preset_id": str, # ID of the preset to use - "images": [base64, ...], # Array of base64-encoded images - "prompt_override": str, # Optional: Override the preset's prompt template - "system_prompt_override": str, # Optional: Override the preset's system prompt - "settings_override": {...} # Optional: Override specific settings - } - - Returns: - { - "success": true, - "response": str, - "preset_used": str, - "provider": str, - "model": str - } - """ + """Generate a response using a preset with an image.""" try: data = await request.json() @@ -1244,7 +615,6 @@ def register_routes(routes_instance): if not model: return error_response(f"Preset '{preset_id}' does not specify a model", status=400) - # Resolve prompt and system prompt text prompt_text = routes_helpers.resolve_preset_prompt_text(preset, prompt_override) if not prompt_text: prompt_text = "Describe this image in detail." @@ -1254,20 +624,19 @@ def register_routes(routes_instance): system_prompt_override, ) - # Merge settings and build options for LLM options = routes_helpers.build_preset_generation_options( provider, preset.get('settings', {}), settings_override, ) - # Validate provider availability before generation - is_available, error_msg, error_code, status_code = routes_helpers.get_provider_availability_error( + is_valid, error_msg, error_code, status_code = routes_helpers.get_provider_and_native_error( provider, + model, 'preset_generate_with_image', ) - if not is_available: - return _error_response_with_metadata( + if not is_valid: + return llm_error_response( error_msg, status=status_code, error_code=error_code, @@ -1275,9 +644,6 @@ def register_routes(routes_instance): operation='preset_generate_with_image', ) - # Initialize LLM services for active preset generation - llm.ensure_llm_initialized(force=True) - response_text = llm.generate_vision( provider, model=model, @@ -1294,7 +660,6 @@ def register_routes(routes_instance): "provider": provider, "model": model, }) - except ValueError as e: logger.error(f"Validation error in preset generation: {str(e)}") return error_response(str(e), status=400) @@ -1304,48 +669,21 @@ def register_routes(routes_instance): traceback.print_exc() return error_response(f"Failed to generate with preset: {str(e)}", status=500) - _route_list.append({ - "method": "POST", - "path": "/sage_llm/presets/generate_with_image", - "description": "Generate response using preset with image" - }) + _append_route("POST", "/sage_llm/presets/generate_with_image", "Generate response using preset with image") + +def _register_load_routes(routes_instance): @routes_instance.post('/sage_llm/load_model') @route_error_handler async def load_llm_model(request): - """ - Load a model into memory without generating a response. - - For LM Studio this loads the model handle and immediately unloads it - (the purpose is just to confirm the model can be loaded and to log the - event). Pass keep_alive > 0 to keep it resident. - - For Ollama this sends a blank-prompt generate to pre-warm the model so - it is resident in GPU memory for subsequent /sage_llm/generate_only - calls. - - Request body: - { - "provider": "lmstudio_rest" | "ollama_rest" | "openai" | "native", - "model": str, - "keep_alive": float | int (optional, seconds; default 60) - } - - Returns: - { - "success": true, - "loaded": true, - "provider": str, - "model": str - } - """ + """Load a model into memory without generating a response.""" try: - _clear_last_llm_error() + clear_last_llm_error() data = await request.json() is_valid, error_msg, payload = routes_helpers.parse_load_model_request(data) if not is_valid: - return _error_response_with_metadata(error_msg, status=400, error_code='LLM_VALIDATION_ERROR') + return llm_error_response(error_msg, status=400, error_code='LLM_VALIDATION_ERROR') provider = payload['provider'] model = payload['model'] @@ -1353,106 +691,55 @@ def register_routes(routes_instance): from ..utils.llm import service as llm - llm.ensure_llm_initialized(force=True) - - is_available, error_msg, error_code, status_code = routes_helpers.get_provider_availability_error( - provider, - 'load_model', - ) - if not is_available: - return _error_response_with_metadata( - error_msg, - status=status_code, - error_code=error_code, + if not llm.is_provider_available(provider, force=True): + return llm_error_response( + f"Provider {provider} is not available", + status=503, + error_code='LLM_PROVIDER_UNAVAILABLE', + provider=provider, + operation='load_model', + ) + if provider == routes_helpers.NATIVE_KEY and not llm.is_native_model_available(model): + return llm_error_response( + f"Native CLIP model '{model}' is not available", + status=404, + error_code='LLM_MODEL_NOT_FOUND', provider=provider, operation='load_model', ) - if provider == LMSTUDIO_REST_KEY: - # LM Studio REST can return load failures even when the model is already - # resident/usable. Treat preload as a readiness check here and let - # generation endpoints perform the real request path. - available_models = llm.get_lmstudio_rest_models() - if model not in available_models: - return _error_response_with_metadata( - f"Model '{model}' is not available in LM Studio REST", - status=404, - error_code='LLM_MODEL_NOT_FOUND', - provider=LMSTUDIO_REST_KEY, - operation='load_model', - ) - - if provider == NATIVE_KEY: - is_native_valid, native_msg, native_code, native_status = routes_helpers.validate_native_model_availability( - provider, - model, - 'load_model', + load_success = llm.load_model(provider, model, keep_alive=keep_alive) + if not load_success: + return llm_error_response( + f"Model '{model}' could not be loaded for provider {provider}", + status=500, + error_code='LLM_MODEL_LOAD_ERROR', + provider=provider, + operation='load_model', ) - if not is_native_valid: - return _error_response_with_metadata( - native_msg, - status=native_status, - error_code=native_code, - provider=provider, - operation='load_model', - ) return success_response(data={ "loaded": True, "provider": provider, "model": model, }) - except Exception as e: logger.error(f"Failed to load model: {str(e)}") - return _error_response_from_exception('Failed to load model', e, status=500, default_error_code='LLM_LOAD_ERROR') + return llm_error_response_from_exception('Failed to load model', e, status=500, default_error_code='LLM_LOAD_ERROR') - _route_list.append({ - "method": "POST", - "path": "/sage_llm/load_model", - "description": "Validate/preload selected model for REST/OpenAI/Native providers" - }) + _append_route("POST", "/sage_llm/load_model", "Validate/preload selected model for REST/OpenAI/Native providers") @routes_instance.post('/sage_llm/generate_only') @route_error_handler async def generate_only(request): - """ - Generate a response from an already-loaded model. - - Semantically equivalent to /sage_llm/generate but signals to the - backend (and logs) that the model is expected to already be in memory. - Callers should call /sage_llm/load_model first if they want to separate - the two phases. - - Request body: - { - "provider": "lmstudio_rest" | "ollama_rest" | "openai" | "native", - "model": str, - "prompt": str, - "system_prompt": str (optional, Ollama only), - "keep_alive": float | int (optional, seconds; default 0), - "options": { - "temperature": float, - "seed": int, - ... - } - } - - Returns: - { - "success": true, - "response": str, - "provider": str, - "model": str - } - """ + """Generate a response from an already-loaded model.""" try: - _clear_last_llm_error() + clear_last_llm_error() data = await request.json() is_valid, error_msg, payload = routes_helpers.parse_generation_request(data) if not is_valid: - return _error_response_with_metadata(error_msg, status=400, error_code='LLM_VALIDATION_ERROR') + return llm_error_response(error_msg, status=400, error_code='LLM_VALIDATION_ERROR') provider = payload['provider'] model = payload['model'] @@ -1463,62 +750,51 @@ def register_routes(routes_instance): from ..utils.llm import service as llm - llm.ensure_llm_initialized(force=True) - - response_text = "" - - is_available, error_msg, error_code, status_code = routes_helpers.get_provider_availability_error( - provider, - 'generate_only', - ) - if not is_available: - return _error_response_with_metadata( - error_msg, - status=status_code, - error_code=error_code, + try: + response_text = llm.generate_only( + provider, + model, + prompt, + options=options, + system_prompt=system_prompt, + keep_alive=keep_alive, + ) + except ValueError as e: + return llm_error_response( + str(e), + status=400, + error_code='LLM_VALIDATION_ERROR', provider=provider, operation='generate_only', ) - - is_native_valid, native_msg, native_code, native_status = routes_helpers.validate_native_model_availability( - provider, - model, - 'generate_only', - ) - if not is_native_valid: - return _error_response_with_metadata( - native_msg, - status=native_status, - error_code=native_code, - provider=provider, - operation='generate_only', - ) - response_text = llm.generate( - provider, - model=model, - prompt=prompt, - options=options, - system_prompt=system_prompt, - keep_alive=keep_alive, - ) return success_response(data={ "response": response_text, "provider": provider, "model": model, }) - except ValueError as e: logger.error(f"Validation error in generate_only: {str(e)}") - return _error_response_from_exception('Validation error in generate_only', e, status=400, default_error_code='LLM_VALIDATION_ERROR') + return llm_error_response_from_exception('Validation error in generate_only', e, status=400, default_error_code='LLM_VALIDATION_ERROR') except Exception as e: logger.error(f"Failed to generate response: {str(e)}") - return _error_response_from_exception('Failed to generate response', e, status=500, default_error_code='LLM_GENERATION_ERROR') + return llm_error_response_from_exception('Failed to generate response', e, status=500, default_error_code='LLM_GENERATION_ERROR') - _route_list.append({ - "method": "POST", - "path": "/sage_llm/generate_only", - "description": "Generate text response assuming model is already loaded" - }) + _append_route("POST", "/sage_llm/generate_only", "Generate text response assuming model is already loaded") + + +def register_routes(routes_instance): + """Register LLM-related routes.""" + set_llm_error_reporter(handle_llm_error_callback) + + global _route_list + _route_list.clear() + + _register_status_and_model_routes(routes_instance) + _register_prompt_routes(routes_instance) + _register_generation_routes(routes_instance) + _register_system_prompt_routes(routes_instance) + _register_preset_routes(routes_instance) + _register_load_routes(routes_instance) return len(_route_list) @@ -1526,7 +802,7 @@ def register_routes(routes_instance): def get_route_list(): """ Get list of registered routes for documentation. - + Returns: list: Route information """ diff --git a/utils/llm/providers/lmstudio/client.py b/utils/llm/providers/lmstudio/client.py index 1f0f1ab..c3b1588 100644 --- a/utils/llm/providers/lmstudio/client.py +++ b/utils/llm/providers/lmstudio/client.py @@ -193,6 +193,45 @@ def get_models(enabled: bool) -> list[str]: label='LM Studio REST models', ) + +def get_models_payload(enabled: bool) -> list[dict[str, Any]]: + """Retrieve raw model payloads from LM Studio REST.""" + if is_provider_unavailable(enabled): + return [] + + try: + response = lmstudio_request_json_models() + return _extract_models_payload(response) + except Exception as e: + report_fetch_error( + llm_report, + 'Error retrieving raw model payloads from LM Studio REST', + provider=_PROVIDER_NAME, + operation='get_models_payload', + cause=e, + fallback=[], + ) + return [] + + +def is_model_loaded(enabled: bool, model: str) -> bool: + """Return whether an LM Studio model already has a loaded instance.""" + if is_provider_unavailable(enabled): + return False + + try: + for model_obj in get_models_payload(enabled): + model_name = _extract_model_name(model_obj) + if model_name != model: + continue + + loaded_instances = model_obj.get('loaded_instances') + return bool(loaded_instances) + except Exception: + pass + + return False + def get_vision_models(enabled: bool) -> list[str]: """Retrieve a list of available vision models from LM Studio REST.""" if is_provider_unavailable(enabled): diff --git a/utils/llm/routes_helpers.py b/utils/llm/routes_helpers.py index 3056e6f..fcd10bb 100644 --- a/utils/llm/routes_helpers.py +++ b/utils/llm/routes_helpers.py @@ -4,6 +4,7 @@ import base64 import json import os import tempfile +from contextvars import ContextVar from pathlib import Path from typing import Any, Optional @@ -21,6 +22,7 @@ from .provider_keys import ( ) logger = get_logger('llm.routes_helpers') +_last_llm_error = ContextVar('sageutils_last_llm_error', default=None) _BUILTIN_TOOL_PROFILES: dict[str, list[dict[str, Any]]] = { @@ -725,16 +727,18 @@ def get_provider_availability_error( """Return a standardized provider availability error tuple if unavailable.""" provider = normalize_provider(provider) - if provider == LMSTUDIO_REST_KEY and not llm.LMSTUDIO_REST_AVAILABLE: - return False, 'LM Studio REST is not available', 'LLM_PROVIDER_UNAVAILABLE', 503 + if provider == NATIVE_KEY: + if not llm.is_provider_available(provider, force=True): + return False, 'Native CLIP provider is not available', 'LLM_PROVIDER_UNAVAILABLE', 503 + return True, None, None, None - if provider == OLLAMA_REST_KEY and not llm.OLLAMA_REST_AVAILABLE: - return False, 'Ollama REST is not available', 'LLM_PROVIDER_UNAVAILABLE', 503 + if provider in {LMSTUDIO_REST_KEY, OLLAMA_REST_KEY, OPENAI_KEY}: + if not llm.is_provider_available(provider, force=True): + provider_name = provider.replace('_', ' ').title() + return False, f'{provider_name} is not available', 'LLM_PROVIDER_UNAVAILABLE', 503 + return True, None, None, None - if provider == OPENAI_KEY and not llm.OPENAI_AVAILABLE: - return False, 'OpenAI provider is not available', 'LLM_PROVIDER_UNAVAILABLE', 503 - - return True, None, None, None + return False, f'Invalid provider: {provider}', 'LLM_INVALID_PROVIDER', 400 def validate_native_model_availability( @@ -747,12 +751,27 @@ def validate_native_model_availability( if provider != NATIVE_KEY: return True, None, None, None - available_native = llm.get_native_models() - if model not in available_native: + if not llm.is_native_model_available(model): return False, f"Native CLIP model '{model}' is not available", 'LLM_MODEL_NOT_FOUND', 404 return True, None, None, None +def get_provider_and_native_error( + provider: str, + model: str, + operation: str, +) -> tuple[bool, Optional[str], Optional[str], Optional[int]]: + """Validate provider availability and native model selection in one check.""" + is_available, error_msg, error_code, status_code = get_provider_availability_error( + provider, + operation, + ) + if not is_available: + return is_available, error_msg, error_code, status_code + + return validate_native_model_availability(provider, model, operation) + + def format_sse_chunk(chunk_data: dict[str, Any]) -> str: """Format a data chunk as Server-Sent Events (SSE) format.""" return f"data: {json.dumps(chunk_data)}\n\n" @@ -801,44 +820,142 @@ async def write_sse_error_and_close( await response.write_eof() -async def write_provider_availability_error_if_unavailable( +def set_last_llm_error(payload: Any) -> None: + """Store the last structured LLM error payload for route error handling.""" + _last_llm_error.set(payload) + + +def clear_last_llm_error() -> None: + """Clear any stale LLM error payload before route handling.""" + _last_llm_error.set(None) + + +def pop_last_llm_error() -> Any: + """Retrieve and clear the last structured LLM error payload.""" + payload = _last_llm_error.get() + _last_llm_error.set(None) + return payload + + +def build_error_payload( + message: str, + *, + error_code: str = 'LLM_ERROR', + provider: str | None = None, + operation: str | None = None, + cause: str | None = None, +) -> dict[str, Any]: + payload = { + 'success': False, + 'error': message, + 'error_code': error_code, + } + if provider: + payload['provider'] = provider + if operation: + payload['operation'] = operation + if cause: + payload['cause'] = cause + return payload + + +def error_response( + message: str, + status: int = 500, + *, + error_code: str = 'LLM_ERROR', + provider: str | None = None, + operation: str | None = None, + cause: str | None = None, +) -> web.Response: + return web.json_response( + build_error_payload( + message, + error_code=error_code, + provider=provider, + operation=operation, + cause=cause, + ), + status=status, + ) + + +def error_response_from_exception( + prefix: str, + exc: Exception, + status: int = 500, + default_error_code: str = 'LLM_ROUTE_ERROR', +) -> web.Response: + payload = pop_last_llm_error() + if payload: + message = payload.get('scoped_message') or payload.get('message') or f'{prefix}: {str(exc)}' + error_type = str(payload.get('error_type', 'ERROR')).upper() + return error_response( + message, + status=status, + error_code=f'LLM_{error_type}', + provider=payload.get('provider'), + operation=payload.get('operation'), + cause=payload.get('cause'), + ) + + return error_response( + f'{prefix}: {str(exc)}', + status=status, + error_code=default_error_code, + ) + + +async def write_sse_error_from_exception( + response: web.StreamResponse, + exc: Exception, + operation: str, + prefix: str = 'Stream error', + default_error_code: str = 'LLM_STREAM_ERROR', +) -> None: + payload = pop_last_llm_error() + if payload: + message = str(payload.get('scoped_message') or payload.get('message') or f'{prefix}: {str(exc)}') + error_type = str(payload.get('error_type', 'STREAM_ERROR')).upper() + await write_sse_error_and_close( + response, + message, + error_code=f'LLM_{error_type}', + provider=payload.get('provider'), + operation=payload.get('operation') or operation, + cause=payload.get('cause'), + ) + return + + await write_sse_error_and_close( + response, + f'{prefix}: {str(exc)}', + error_code=default_error_code, + operation=operation, + ) + + +async def write_provider_and_native_error_if_unavailable( response: web.StreamResponse, provider: str, + model: str, operation: str, ) -> bool: - """Write a provider availability SSE error and return True if unavailable.""" - provider = normalize_provider(provider) - - if provider == LMSTUDIO_REST_KEY and not llm.LMSTUDIO_REST_AVAILABLE: + """Write a shared SSE error chunk for provider or native model availability failures.""" + is_valid, error_msg, error_code, status_code = get_provider_and_native_error( + provider, + model, + operation, + ) + if not is_valid: await write_sse_error_and_close( response, - 'LM Studio REST is not available', - error_code='LLM_PROVIDER_UNAVAILABLE', - provider=LMSTUDIO_REST_KEY, + str(error_msg or 'Provider or model unavailable'), + error_code=error_code or 'LLM_PROVIDER_UNAVAILABLE', + provider=provider, operation=operation, ) return True - - if provider == OLLAMA_REST_KEY and not llm.OLLAMA_REST_AVAILABLE: - await write_sse_error_and_close( - response, - 'Ollama REST is not available', - error_code='LLM_PROVIDER_UNAVAILABLE', - provider=OLLAMA_REST_KEY, - operation=operation, - ) - return True - - if provider == OPENAI_KEY and not llm.OPENAI_AVAILABLE: - await write_sse_error_and_close( - response, - 'OpenAI provider is not available', - error_code='LLM_PROVIDER_UNAVAILABLE', - provider=OPENAI_KEY, - operation=operation, - ) - return True - return False @@ -915,48 +1032,6 @@ def validate_generation_data( def check_model_vision_capability(provider: str, model: str) -> tuple[bool, Optional[str]]: """ Check if a model supports vision capability before attempting vision operations. - - Args: - provider: Provider name (e.g., 'ollama', 'lmstudio', 'openai') - model: Model name - - Returns: - Tuple of (supports_vision, error_message) - - If supports_vision is True, error_message is None - - If supports_vision is False, error_message describes why """ provider = normalize_provider(provider) - - try: - from . import service as llm - # Ensure LLM services are initialized for active model capability validation - llm.ensure_llm_initialized(force=True) - - # Get capability map for the provider - if provider == 'lmstudio_rest': - cap_map = llm.get_lmstudio_rest_model_capabilities_map() - elif provider == 'ollama_rest': - cap_map = llm.get_ollama_rest_model_capabilities_map() - elif provider == 'openai': - cap_map = llm.get_openai_model_capabilities_map() - else: - return False, f"Unknown provider: {provider}" - - # Check if model exists and supports vision - if not cap_map: - return False, f"No capability data available for {provider}" - - if model not in cap_map: - available = ', '.join(sorted(cap_map.keys())[:5]) - return False, f"Model '{model}' not found in {provider}. Available: {available}..." - - capabilities = cap_map[model] - if not capabilities.get('vision', False): - return False, f"Model '{model}' does not support vision capability on provider {provider}" - - return True, None - - except Exception as e: - logger.warning(f'Error checking vision capability for {provider}:{model}: {e}') - # Degrade gracefully. If we can't check, allow the attempt. - return True, None + return llm.is_model_vision_capable(provider, model) diff --git a/utils/llm/service.py b/utils/llm/service.py index df8f715..0088fba 100644 --- a/utils/llm/service.py +++ b/utils/llm/service.py @@ -1,6 +1,8 @@ import time +from typing import Any, Optional from ..logger import get_logger +from ..settings import get_setting from .providers.settings import ( is_lmstudio_rest_enabled, is_ollama_rest_enabled, @@ -165,6 +167,16 @@ def _get_model_capabilities_dict(provider_key: str) -> dict[str, dict[str, objec return {model_name: capabilities.to_dict() for model_name, capabilities in capability_map.items()} +def _status_key(provider_key: str) -> str: + return f'{provider_key}_available' + + +def _filter_compatible_models(model_list: list[str]) -> list[str]: + if not model_list: + return [] + return [model for model in model_list if model and not model.startswith('(')] + + def _generate_non_streaming( provider_key: str, model: str, @@ -493,6 +505,155 @@ def _unload_provider_model( return provider.unload_model(enabled, model) +def load_model( + provider_key: str, + model: str, + *, + keep_alive: int = 0, +) -> bool: + provider_key = normalize_provider_key(provider_key) + + if provider_key == NATIVE_KEY: + native_models = get_native_models() + return model in native_models + + if provider_key == LMSTUDIO_REST_KEY: + ensure_lmstudio_rest_initialized(force=True) + enabled = _provider_enabled(provider_key) + if _provider_client(provider_key).is_model_loaded(enabled, model): + return True + return _load_provider_model(provider_key, model, keep_alive=keep_alive) + + if provider_key == OLLAMA_REST_KEY: + ensure_ollama_rest_initialized(force=True) + enabled = _provider_enabled(provider_key) + if _provider_client(provider_key).is_model_loaded(enabled, model): + return True + return _load_provider_model(provider_key, model, keep_alive=keep_alive) + + if provider_key == OPENAI_KEY: + ensure_openai_initialized(force=True) + compatible_models = _filter_compatible_models(get_openai_models()) + return model in compatible_models + + raise ValueError(f'Unsupported provider for load_model: {provider_key}') + + +def is_model_loaded(provider_key: str, model: str) -> bool: + provider_key = normalize_provider_key(provider_key) + + if provider_key == NATIVE_KEY: + return model in get_native_models() + + if provider_key == LMSTUDIO_REST_KEY: + ensure_lmstudio_rest_initialized(force=True) + enabled = _provider_enabled(provider_key) + return _provider_client(provider_key).is_model_loaded(enabled, model) + + if provider_key == OLLAMA_REST_KEY: + ensure_ollama_rest_initialized(force=True) + enabled = _provider_enabled(provider_key) + return _provider_client(provider_key).is_model_loaded(enabled, model) + + if provider_key == OPENAI_KEY: + ensure_openai_initialized(force=True) + compatible_models = _filter_compatible_models(get_openai_models()) + return model in compatible_models + + raise ValueError(f'Unsupported provider for is_model_loaded: {provider_key}') + + +def generate_only( + provider_key: str, + model: str, + prompt: str, + *, + options=None, + system_prompt: str = '', + keep_alive=None, +): + provider_key = normalize_provider_key(provider_key) + + if not is_provider_available(provider_key, force=True): + raise ValueError(f'Provider {provider_key} is not available') + + if provider_key == NATIVE_KEY and not is_native_model_available(model): + raise ValueError(f"Native CLIP model '{model}' is not available") + + if provider_key in {LMSTUDIO_REST_KEY, OLLAMA_REST_KEY}: + if not is_model_loaded(provider_key, model): + raise ValueError(f"Model '{model}' is not loaded for provider {provider_key}") + + return generate( + provider_key, + model, + prompt, + options=options, + system_prompt=system_prompt, + keep_alive=keep_alive, + ) + + +def is_provider_available(provider_key: str, *, force: bool = False) -> bool: + provider_key = normalize_provider_key(provider_key) + + if provider_key == NATIVE_KEY: + return len(get_native_models()) > 0 + + if provider_key == LMSTUDIO_REST_KEY: + return ensure_lmstudio_rest_initialized(force=force) and LMSTUDIO_REST_AVAILABLE and is_lmstudio_rest_enabled() + + if provider_key == OLLAMA_REST_KEY: + return ensure_ollama_rest_initialized(force=force) and OLLAMA_REST_AVAILABLE and is_ollama_rest_enabled() + + if provider_key == OPENAI_KEY: + return ensure_openai_initialized(force=force) and OPENAI_AVAILABLE and is_openai_enabled() + + return False + + +def is_native_model_available(model: str) -> bool: + return model in get_native_models() + + +def get_provider_model_capabilities_map(provider_key: str) -> dict[str, dict[str, object]]: + provider_key = normalize_provider_key(provider_key) + + if provider_key == LMSTUDIO_REST_KEY: + return get_lmstudio_rest_model_capabilities_map() + if provider_key == OLLAMA_REST_KEY: + return get_ollama_rest_model_capabilities_map() + if provider_key == OPENAI_KEY: + return get_openai_model_capabilities_map() + + return {} + + +def is_model_vision_capable(provider_key: str, model: str) -> tuple[bool, Optional[str]]: + provider_key = normalize_provider_key(provider_key) + + if provider_key == NATIVE_KEY: + return False, 'Native provider does not support vision generation' + + if provider_key not in {LMSTUDIO_REST_KEY, OLLAMA_REST_KEY, OPENAI_KEY}: + return False, f'Unknown provider: {provider_key}' + + ensure_llm_initialized(force=True) + capability_map = get_provider_model_capabilities_map(provider_key) + if not capability_map: + return False, f'No capability data available for provider {provider_key}' + + if model not in capability_map: + available = ', '.join(sorted(capability_map.keys())[:10]) + return False, f"Model '{model}' not found for {provider_key}. Available: {available}" + + capabilities = capability_map[model] + if not capabilities.get('vision', False): + return False, f"Model '{model}' does not support vision capability on provider {provider_key}" + + return True, None + + # ============================================================================ # MODEL DISCOVERY # ============================================================================ @@ -798,6 +959,217 @@ def _ensure_registered_providers(force: bool = False) -> dict[str, bool]: return results +def get_llm_status(force: bool = False) -> dict[str, dict[str, object]]: + """Return availability and enabled state for all supported providers.""" + if force: + reset_llm_initialization_state() + + ensure_llm_initialized(force=force) + + status = { + LMSTUDIO_REST_KEY: { + 'available': LMSTUDIO_REST_AVAILABLE and is_lmstudio_rest_enabled(), + 'enabled': is_lmstudio_rest_enabled(), + }, + OLLAMA_REST_KEY: { + 'available': OLLAMA_REST_AVAILABLE and is_ollama_rest_enabled(), + 'enabled': is_ollama_rest_enabled(), + }, + OPENAI_KEY: { + 'available': OPENAI_AVAILABLE and is_openai_enabled(), + 'enabled': is_openai_enabled(), + }, + } + + native_models = get_native_models() + status[NATIVE_KEY] = { + 'available': len(native_models) > 0, + 'enabled': True, + } + + return status + + +def get_llm_models(force: bool = False) -> dict[str, object]: + """Return model lists, capabilities, tool models, reasoning models, and status.""" + if force: + reset_llm_initialization_state() + + ensure_llm_initialized(force=force) + + lmstudio_rest_enabled = is_lmstudio_rest_enabled() + ollama_rest_enabled = is_ollama_rest_enabled() + openai_enabled = is_openai_enabled() + + lmstudio_rest_models = [] + lmstudio_rest_tool_models = [] + lmstudio_rest_reasoning_models = [] + lmstudio_rest_capabilities = {} + ollama_rest_models = [] + ollama_rest_tool_models = [] + ollama_rest_reasoning_models = [] + ollama_rest_capabilities = {} + openai_models = [] + openai_tool_models = [] + openai_reasoning_models = [] + openai_capabilities = {} + + if lmstudio_rest_enabled and LMSTUDIO_REST_AVAILABLE: + try: + models = get_lmstudio_rest_models() + lmstudio_rest_models = _filter_compatible_models(models) + lmstudio_rest_tool_models = _filter_compatible_models(get_lmstudio_rest_tool_models()) + lmstudio_rest_reasoning_models = _filter_compatible_models(get_lmstudio_rest_reasoning_models()) + lmstudio_rest_capabilities = get_lmstudio_rest_model_capabilities_map() + except Exception as e: + logger.warning(f'Failed to get LM Studio REST models: {e}') + + if ollama_rest_enabled and OLLAMA_REST_AVAILABLE: + try: + models = get_ollama_rest_models() + ollama_rest_models = _filter_compatible_models(models) + ollama_rest_tool_models = _filter_compatible_models(get_ollama_rest_tool_models()) + ollama_rest_reasoning_models = _filter_compatible_models(get_ollama_rest_reasoning_models()) + ollama_rest_capabilities = get_ollama_rest_model_capabilities_map() + except Exception as e: + logger.warning(f'Failed to get Ollama REST models: {e}') + + if openai_enabled and OPENAI_AVAILABLE: + try: + models = get_openai_models() + openai_models = _filter_compatible_models(models) + openai_tool_models = _filter_compatible_models(get_openai_tool_models()) + openai_reasoning_models = _filter_compatible_models(get_openai_reasoning_models()) + openai_capabilities = get_openai_model_capabilities_map() + except Exception as e: + logger.warning(f'Failed to get OpenAI models: {e}') + + native_models = get_native_models() + + return { + 'models': { + LMSTUDIO_REST_KEY: lmstudio_rest_models, + OLLAMA_REST_KEY: ollama_rest_models, + OPENAI_KEY: openai_models, + NATIVE_KEY: native_models, + }, + 'capabilities': { + LMSTUDIO_REST_KEY: lmstudio_rest_capabilities, + OLLAMA_REST_KEY: ollama_rest_capabilities, + OPENAI_KEY: openai_capabilities, + NATIVE_KEY: {}, + }, + 'tool_models': { + LMSTUDIO_REST_KEY: lmstudio_rest_tool_models, + OLLAMA_REST_KEY: ollama_rest_tool_models, + OPENAI_KEY: openai_tool_models, + NATIVE_KEY: [], + }, + 'reasoning_models': { + LMSTUDIO_REST_KEY: lmstudio_rest_reasoning_models, + OLLAMA_REST_KEY: ollama_rest_reasoning_models, + OPENAI_KEY: openai_reasoning_models, + NATIVE_KEY: [], + }, + 'status': { + _status_key(LMSTUDIO_REST_KEY): len(lmstudio_rest_models) > 0, + _status_key(OLLAMA_REST_KEY): len(ollama_rest_models) > 0, + _status_key(OPENAI_KEY): len(openai_models) > 0, + _status_key(NATIVE_KEY): len(native_models) > 0, + }, + } + + +def get_llm_vision_models(force: bool = False) -> dict[str, object]: + """Return vision model lists, capabilities, tool models, reasoning models, and status.""" + if force: + reset_llm_initialization_state() + + ensure_llm_initialized(force=force) + + lmstudio_rest_enabled = is_lmstudio_rest_enabled() + ollama_rest_enabled = is_ollama_rest_enabled() + openai_enabled = is_openai_enabled() + + lmstudio_rest_models = [] + lmstudio_rest_tool_models = [] + lmstudio_rest_reasoning_models = [] + lmstudio_rest_capabilities = {} + ollama_rest_models = [] + ollama_rest_tool_models = [] + ollama_rest_reasoning_models = [] + ollama_rest_capabilities = {} + openai_models = [] + openai_tool_models = [] + openai_reasoning_models = [] + openai_capabilities = {} + + if lmstudio_rest_enabled and LMSTUDIO_REST_AVAILABLE: + try: + models = get_lmstudio_rest_vision_models() + lmstudio_rest_models = _filter_compatible_models(models) + lmstudio_rest_tool_models = _filter_compatible_models(get_lmstudio_rest_tool_models()) + lmstudio_rest_reasoning_models = _filter_compatible_models(get_lmstudio_rest_reasoning_models()) + lmstudio_rest_capabilities = get_lmstudio_rest_model_capabilities_map() + except Exception as e: + logger.warning(f'Failed to get LM Studio REST vision models: {e}') + + if ollama_rest_enabled and OLLAMA_REST_AVAILABLE: + try: + models = get_ollama_rest_vision_models() + ollama_rest_models = _filter_compatible_models(models) + ollama_rest_tool_models = _filter_compatible_models(get_ollama_rest_tool_models()) + ollama_rest_reasoning_models = _filter_compatible_models(get_ollama_rest_reasoning_models()) + ollama_rest_capabilities = get_ollama_rest_model_capabilities_map() + except Exception as e: + logger.warning(f'Failed to get Ollama REST vision models: {e}') + + if openai_enabled and OPENAI_AVAILABLE: + try: + models = get_openai_vision_models() + openai_models = _filter_compatible_models(models) + openai_tool_models = _filter_compatible_models(get_openai_tool_models()) + openai_reasoning_models = _filter_compatible_models(get_openai_reasoning_models()) + openai_capabilities = get_openai_model_capabilities_map() + except Exception as e: + logger.warning(f'Failed to get OpenAI vision models: {e}') + + native_models = [] + + return { + 'models': { + LMSTUDIO_REST_KEY: lmstudio_rest_models, + OLLAMA_REST_KEY: ollama_rest_models, + OPENAI_KEY: openai_models, + NATIVE_KEY: native_models, + }, + 'capabilities': { + LMSTUDIO_REST_KEY: lmstudio_rest_capabilities, + OLLAMA_REST_KEY: ollama_rest_capabilities, + OPENAI_KEY: openai_capabilities, + NATIVE_KEY: {}, + }, + 'tool_models': { + LMSTUDIO_REST_KEY: lmstudio_rest_tool_models, + OLLAMA_REST_KEY: ollama_rest_tool_models, + OPENAI_KEY: openai_tool_models, + NATIVE_KEY: [], + }, + 'reasoning_models': { + LMSTUDIO_REST_KEY: lmstudio_rest_reasoning_models, + OLLAMA_REST_KEY: ollama_rest_reasoning_models, + OPENAI_KEY: openai_reasoning_models, + NATIVE_KEY: [], + }, + 'status': { + _status_key(LMSTUDIO_REST_KEY): len(lmstudio_rest_models) > 0, + _status_key(OLLAMA_REST_KEY): len(ollama_rest_models) > 0, + _status_key(OPENAI_KEY): len(openai_models) > 0, + _status_key(NATIVE_KEY): False, + }, + } + + def init_llm() -> None: """Initialize all configured REST/OpenAI providers.""" _init_registered_providers()