slahiriandClaude Opus 4.5
b01c117175
- Add attention_mode selection (auto/flash_attention_2/sdpa/eager) - Add torch.compile support for CUDA acceleration (~20-30% speedup) - Add repetition_penalty and top_p parameters for better output control - Implement response validation with retry logic for short responses - Add adaptive resolution based on Qwen model version (Qwen3 vs Qwen2.5) - Add quality metrics tracking (word count, tokens/sec) - Add provider-specific cleanup patterns (Groq, Together, Fireworks, etc.) - Update providers.toml with local model configurations and VRAM guidelines 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
159 lines
4.9 KiB
TOML
159 lines
4.9 KiB
TOML
# =============================================================================
|
|
# Provider Configuration for SID Photography Toolkit
|
|
# =============================================================================
|
|
# This file defines provider tiers, capabilities, and stop strings.
|
|
# Tiers: "advanced" (cloud APIs), "standard" (capable local), "basic" (simple local)
|
|
|
|
[tiers]
|
|
# Provider tier mappings - determines prompt complexity
|
|
anthropic = "advanced"
|
|
openai = "advanced"
|
|
google = "advanced"
|
|
gemini = "advanced"
|
|
mistral = "advanced"
|
|
deepseek = "advanced"
|
|
cohere = "advanced"
|
|
together = "standard"
|
|
fireworks = "standard"
|
|
groq = "standard"
|
|
perplexity = "standard"
|
|
openrouter = "advanced"
|
|
xai = "advanced"
|
|
grok = "advanced"
|
|
|
|
# Local providers - simpler prompts work better
|
|
ollama = "basic"
|
|
lmstudio = "basic"
|
|
kobold = "basic"
|
|
local = "basic"
|
|
custom = "standard" # Assume standard unless proven otherwise
|
|
|
|
[tier_descriptions]
|
|
advanced = "Full-featured cloud APIs (Claude, GPT-4, Gemini) - complex multi-step prompts"
|
|
standard = "Capable models with some limitations - moderate prompt complexity"
|
|
basic = "Local/lightweight models - simple, direct prompts work best"
|
|
|
|
|
|
# =============================================================================
|
|
# Stop Strings by Provider
|
|
# =============================================================================
|
|
# These sequences signal the LLM to stop generating, preventing text leakage.
|
|
|
|
[stop_strings]
|
|
# Local providers need aggressive stop strings to prevent rambling
|
|
ollama = ["\n\n\n\n\n", "\n\n\n\n", "---", "```\n\n"]
|
|
lmstudio = ["\n\n\n", "```\n\n", "---", "\n\n---"]
|
|
kobold = ["\n\n\n\n\n", "###", "---", "```"]
|
|
|
|
# Cloud providers generally handle stopping well
|
|
mistral = ["\n\n"]
|
|
anthropic = [] # Claude handles this excellently
|
|
openai = [] # GPT handles this well
|
|
google = []
|
|
default = []
|
|
|
|
|
|
# =============================================================================
|
|
# Provider Capabilities
|
|
# =============================================================================
|
|
# What each provider tier can handle
|
|
|
|
[capabilities.advanced]
|
|
max_prompt_items = 12 # Can handle long numbered lists
|
|
supports_json_output = true # Reliably outputs valid JSON
|
|
supports_system_prompt = true # Full system prompt support
|
|
supports_reasoning = true # Extended thinking capability
|
|
max_context_tokens = 128000 # Large context windows
|
|
recommended_temperature = 0.7
|
|
|
|
[capabilities.standard]
|
|
max_prompt_items = 8
|
|
supports_json_output = true
|
|
supports_system_prompt = true
|
|
supports_reasoning = false
|
|
max_context_tokens = 32000
|
|
recommended_temperature = 0.5
|
|
|
|
[capabilities.basic]
|
|
max_prompt_items = 4 # Keep prompts short and simple
|
|
supports_json_output = false # Often produces malformed JSON
|
|
supports_system_prompt = true # Most support system prompts
|
|
supports_reasoning = false
|
|
max_context_tokens = 8000
|
|
recommended_temperature = 0.3 # Lower temperature for consistency
|
|
|
|
|
|
# =============================================================================
|
|
# Image Size Limits by Provider
|
|
# =============================================================================
|
|
|
|
[image_limits]
|
|
# Maximum image dimensions for optimal processing
|
|
anthropic = 1568
|
|
openai = 2048
|
|
ollama = 1024
|
|
lmstudio = 1024
|
|
local = 1280 # QwenVL optimal range
|
|
default = 1024
|
|
|
|
|
|
# =============================================================================
|
|
# Local Model Configuration (QwenVL, Florence, etc.)
|
|
# =============================================================================
|
|
|
|
[local_models]
|
|
# Default generation parameters for local models
|
|
default_temperature = 0.3
|
|
default_repetition_penalty = 1.2
|
|
default_top_p = 0.9
|
|
default_num_beams = 1
|
|
|
|
# Attention mode options
|
|
attention_modes = ["auto", "flash_attention_2", "sdpa", "eager"]
|
|
default_attention_mode = "auto"
|
|
|
|
# Response validation
|
|
min_response_words = 20
|
|
max_retries = 2
|
|
retry_temp_increase = 0.2
|
|
|
|
[local_models.qwenvl]
|
|
# QwenVL-specific settings
|
|
# Qwen2.5-VL uses patch_size=28, Qwen3-VL uses patch_size=14
|
|
# Optimal resolution: 480x480 to 2560x2560 (per Qwen docs)
|
|
min_pixels_qwen3 = 50176 # 256 * 14 * 14
|
|
max_pixels_qwen3 = 250880 # 1280 * 14 * 14
|
|
min_pixels_qwen25 = 200704 # 256 * 28 * 28
|
|
max_pixels_qwen25 = 1003520 # 1280 * 28 * 28
|
|
|
|
# Torch compile settings
|
|
enable_torch_compile = false # Default off (first run slower)
|
|
torch_compile_mode = "reduce-overhead"
|
|
min_torch_version = "2.1"
|
|
|
|
# Model recommendations by VRAM
|
|
# <8GB: Qwen3-VL-2B (4-bit)
|
|
# 8-12GB: Qwen3-VL-4B (8-bit)
|
|
# 12-16GB: Qwen3-VL-4B (FP16) or Qwen3-VL-8B (8-bit)
|
|
# >16GB: Qwen3-VL-8B (FP16)
|
|
|
|
[local_models.florence2]
|
|
# Florence-2 is ultra-fast, good for quick captions
|
|
default_temperature = 0.2
|
|
max_output_tokens = 1024
|
|
|
|
[local_models.moondream2]
|
|
# Moondream is efficient, runs on CPU too
|
|
default_temperature = 0.3
|
|
max_output_tokens = 2048
|
|
|
|
[local_models.smolvlm]
|
|
# SmolVLM for edge/mobile use
|
|
default_temperature = 0.2
|
|
max_output_tokens = 1024
|
|
|
|
[local_models.phi35_vision]
|
|
# Phi-3.5-Vision for high quality
|
|
default_temperature = 0.4
|
|
max_output_tokens = 4096
|