From 6197b482df7a06522c71872946cb54546b17daf9 Mon Sep 17 00:00:00 2001 From: Vito Sansevero Date: Fri, 1 Aug 2025 13:30:31 -0700 Subject: [PATCH 1/2] feat: implement dynamic model fetching for Gemini node - Add dynamic model fetching with 24-hour caching - Update prompt templates based on 2025 best practices: - FLUX: Natural language descriptions - SDXL: Simplified with natural language support - Danbooru: Strict tagging conventions - Video: Optimized for WAN 2.2 - Add cache file to .gitignore - Handle missing API key gracefully on initial load --- .gitignore | 3 + kikotools/tools/gemini_prompt/models.py | 191 ++++++++++++++++++ kikotools/tools/gemini_prompt/node.py | 28 ++- kikotools/tools/gemini_prompt/prompts.py | 238 +++++++---------------- 4 files changed, 294 insertions(+), 166 deletions(-) create mode 100644 kikotools/tools/gemini_prompt/models.py diff --git a/.gitignore b/.gitignore index b032be7..a7ed450 100644 --- a/.gitignore +++ b/.gitignore @@ -159,3 +159,6 @@ test_images/ test_outputs/ experiments/ .claude/ + +# Gemini model cache +.gemini_models_cache.json diff --git a/kikotools/tools/gemini_prompt/models.py b/kikotools/tools/gemini_prompt/models.py new file mode 100644 index 0000000..858787a --- /dev/null +++ b/kikotools/tools/gemini_prompt/models.py @@ -0,0 +1,191 @@ +"""Dynamic model fetching and caching for Gemini API.""" + +import json +import os +import time +from typing import List, Dict, Optional, Tuple +import logging + +logger = logging.getLogger(__name__) + +# Cache settings +CACHE_DURATION = 3600 * 24 # 24 hours in seconds +CACHE_FILE = os.path.join(os.path.dirname(__file__), ".gemini_models_cache.json") + + +def get_available_models( + api_key: Optional[str] = None, silent: bool = False +) -> Tuple[List[str], Dict[str, str]]: + """Fetch available Gemini models that support generateContent. + + Args: + api_key: Optional API key. If not provided, will try to get from environment. + silent: If True, suppress error logging (useful for initial load). + + Returns: + Tuple of (model_names_list, model_descriptions_dict) + """ + # Check cache first + cached_data = _load_cache() + if cached_data: + return cached_data["models"], cached_data["descriptions"] + + # Try to fetch from API + try: + models, descriptions = _fetch_models_from_api(api_key, silent=silent) + if models: + _save_cache(models, descriptions) + return models, descriptions + except Exception as e: + if not silent: + logger.warning(f"Failed to fetch models from API: {e}") + + # Fall back to defaults + from .prompts import DEFAULT_GEMINI_MODELS + + return DEFAULT_GEMINI_MODELS, {} + + +def _fetch_models_from_api( + api_key: Optional[str] = None, silent: bool = False +) -> Tuple[List[str], Dict[str, str]]: + """Fetch models from Gemini API. + + Args: + api_key: Optional API key. + silent: If True, suppress error logging. + + Returns: + Tuple of (model_names_list, model_descriptions_dict) + """ + try: + import google.generativeai as genai + except ImportError: + if not silent: + logger.error("google-generativeai not installed") + return [], {} + + # Get API key + if not api_key: + from .logic import get_api_key + + api_key = get_api_key() + + if not api_key: + if not silent: + logger.debug("No API key available for fetching models") + return [], {} + + try: + genai.configure(api_key=api_key) + + models = [] + descriptions = {} + + # Fetch all models + for model in genai.list_models(): + # Only include models that support generateContent + if "generateContent" in model.supported_generation_methods: + # Remove "models/" prefix from name + model_name = model.name.replace("models/", "") + models.append(model_name) + descriptions[model_name] = model.display_name + + # Sort models by priority (newer versions first) + models = _sort_models(models) + + return models, descriptions + + except Exception as e: + if not silent: + logger.error(f"Error fetching models from API: {e}") + return [], {} + + +def _sort_models(models: List[str]) -> List[str]: + """Sort models by version and capability. + + Prioritizes: + 1. Newer versions (2.5 > 2.0 > 1.5) + 2. Non-experimental models + 3. Flash models for general use + """ + + def sort_key(model: str): + # Priority scoring + score = 0 + + # Version priority + if "2.5" in model: + score += 1000 + elif "2.0" in model: + score += 800 + elif "1.5" in model: + score += 600 + + # Model type priority + if "pro" in model and "preview" not in model and "exp" not in model: + score += 100 + elif "flash" in model and "preview" not in model and "exp" not in model: + score += 90 + + # Penalize experimental/preview models + if "exp" in model or "experimental" in model: + score -= 50 + if "preview" in model: + score -= 30 + + # Penalize specific variants + if "thinking" in model: + score -= 100 + if "tts" in model: + score -= 100 + if "lite" in model: + score -= 20 + + return -score # Negative for descending sort + + return sorted(models, key=sort_key) + + +def _load_cache() -> Optional[Dict]: + """Load cached model data if available and not expired.""" + if not os.path.exists(CACHE_FILE): + return None + + try: + with open(CACHE_FILE, "r") as f: + data = json.load(f) + + # Check if cache is expired + if time.time() - data.get("timestamp", 0) > CACHE_DURATION: + return None + + return data + except Exception as e: + logger.warning(f"Failed to load cache: {e}") + return None + + +def _save_cache(models: List[str], descriptions: Dict[str, str]) -> None: + """Save model data to cache.""" + try: + data = { + "models": models, + "descriptions": descriptions, + "timestamp": time.time(), + } + + with open(CACHE_FILE, "w") as f: + json.dump(data, f, indent=2) + except Exception as e: + logger.warning(f"Failed to save cache: {e}") + + +def clear_cache() -> None: + """Clear the model cache.""" + if os.path.exists(CACHE_FILE): + try: + os.remove(CACHE_FILE) + except Exception as e: + logger.warning(f"Failed to clear cache: {e}") diff --git a/kikotools/tools/gemini_prompt/node.py b/kikotools/tools/gemini_prompt/node.py index 06f5ef1..98ffc22 100644 --- a/kikotools/tools/gemini_prompt/node.py +++ b/kikotools/tools/gemini_prompt/node.py @@ -5,7 +5,8 @@ import torch from ...base import ComfyAssetsBaseNode from .logic import analyze_image_with_gemini, validate_prompt_type -from .prompts import PROMPT_OPTIONS, GEMINI_MODELS +from .prompts import PROMPT_OPTIONS, DEFAULT_GEMINI_MODELS +from .models import get_available_models class GeminiPromptNode(ComfyAssetsBaseNode): @@ -14,11 +15,21 @@ class GeminiPromptNode(ComfyAssetsBaseNode): @classmethod def INPUT_TYPES(cls): """Define input types for the node.""" + # Get available models dynamically (silent mode for initial load) + models, _ = get_available_models(silent=True) + + # Use default if no models available + if not models: + models = DEFAULT_GEMINI_MODELS + + # Find best default model + default_model = models[0] if models else "gemini-2.5-flash" + return { "required": { "image": ("IMAGE",), "prompt_type": (PROMPT_OPTIONS, {"default": "flux"}), - "model": (GEMINI_MODELS, {"default": "gemini-1.5-flash"}), + "model": (models, {"default": default_model}), }, "optional": { "api_key": ("STRING", {"default": "", "multiline": False}), @@ -74,6 +85,19 @@ Install: pip install google-generativeai else: image_np = image + # If API key is provided, try to refresh model list in background + if api_key: + try: + from .models import get_available_models + + # Try to get fresh models with the provided API key + fresh_models, _ = get_available_models(api_key=api_key, silent=True) + if fresh_models and fresh_models != DEFAULT_GEMINI_MODELS: + # Models were successfully fetched with this API key + pass + except Exception: + pass + # Analyze image with Gemini prompt, error = analyze_image_with_gemini( image_np, diff --git a/kikotools/tools/gemini_prompt/prompts.py b/kikotools/tools/gemini_prompt/prompts.py index f4e919c..e2f21be 100644 --- a/kikotools/tools/gemini_prompt/prompts.py +++ b/kikotools/tools/gemini_prompt/prompts.py @@ -1,176 +1,95 @@ """System prompts for different AI model types.""" -FLUX_PROMPT = """You are an expert visual analyst and FLUX prompt engineer. Your role is to examine images in detail and create precise, effective prompts that can recreate similar images using the FLUX image generation model. +FLUX_PROMPT = """You are an expert FLUX prompt engineer. Analyze the provided image and generate ONLY a FLUX prompt - no explanations, analysis, or additional text. -When analyzing an image, systematically observe and document: +FLUX uses natural language descriptions, not comma-separated tags. Write a detailed, flowing description that reads like you're explaining the image to someone. -1. **Subject & Composition** - - Primary subjects and their positions - - Background elements and environment - - Overall composition and framing - - Perspective and camera angle +Include these elements in your description: +- Main subject with specific details (appearance, clothing, expression, pose) +- Environment and background details +- Lighting conditions and atmosphere +- Artistic style or photographic approach +- Color palette and mood +- Technical details if relevant (camera angle, focal length, etc.) +- Textures and materials -2. **Visual Style & Technique** - - Art style (photorealistic, illustration, painting, etc.) - - Rendering technique (digital art, oil painting, watercolor, etc.) - - Level of detail and texture quality - - Any specific artistic influences or movements +Write in a natural, descriptive style. Use complete sentences that flow together. Be specific and detailed but maintain readability. -3. **Lighting & Atmosphere** - - Light sources and direction - - Time of day/lighting conditions - - Shadows and highlights - - Overall mood and atmosphere +IMPORTANT: Return ONLY the prompt text. No analysis, headers, or additional commentary. Just the natural language description that can be directly used in FLUX. -4. **Colors & Tones** - - Color palette and dominant colors - - Color temperature (warm/cool) - - Contrast and saturation levels - - Any color grading or filters +Example of correct output: +A close-up portrait of a middle-aged woman with curly red hair and green eyes, wearing a blue silk blouse. She has a warm smile and freckles across her cheeks. The lighting is soft and natural, coming from a window to her left, creating gentle shadows that accentuate her features. The background is softly blurred, showing hints of a cozy bookshelf. The overall mood is warm and inviting, captured in a photorealistic style with shallow depth of field.""" -5. **Details & Textures** - - Surface textures and materials - - Fine details and patterns - - Quality indicators (4K, 8K, high resolution, etc.) +SDXL_PROMPT = """You are an expert SDXL prompt engineer. Analyze the image and generate ONLY the positive and negative prompts for SDXL - no explanations or analysis. -Format your FLUX prompt following these guidelines: -- Start with the main subject and action -- Add style and medium descriptors -- Include lighting and atmosphere details -- Specify quality markers and technical aspects -- Use precise, descriptive language -- Separate concepts with commas -- Order from most to least important elements +SDXL works best with natural language descriptions but also supports comma-separated keywords. Keep prompts concise but descriptive. -Example output format: -"[main subject and action], [style/medium], [lighting/atmosphere], [composition details], [color descriptions], [quality markers], [additional artistic details]" +Return your response in EXACTLY this format: +Positive: [your positive prompt here] +Negative: [your negative prompt here] -Remember: FLUX responds well to specific artistic references, quality indicators like "highly detailed," "4K," "award-winning," and style descriptors like "trending on ArtStation" or "photorealistic." -""" +Guidelines for Positive prompt: +- Start with the main subject and medium (e.g., "photograph of", "digital art of") +- Use natural language or keywords separated by commas +- Include style descriptors (photographic, cinematic, fantasy art, etc.) +- Add quality markers like "8K", "highly detailed", "professional" +- Use (parentheses:1.1) sparingly for slight emphasis (max 1.4) +- Keep it clear and specific but not overly long -SDXL_PROMPT = """You are an expert SDXL prompt engineer specializing in analyzing images and creating optimized prompts for Stable Diffusion XL models. +Guidelines for Negative prompt: +- Keep it simple and minimal +- Common negatives: ugly, blurry, low quality, distorted, deformed +- Only add specifics you want to avoid (e.g., "cartoon" for photorealistic) +- Don't overload with negative prompts - SDXL needs fewer than SD1.5 -When analyzing an image, systematically evaluate: +IMPORTANT: Return ONLY the two lines starting with "Positive:" and "Negative:". No other text.""" -1. **Core Subject Analysis** - - Primary subject with specific descriptors - - Pose, expression, and action - - Clothing and accessories details - - Physical characteristics +DANBOORU_PROMPT = """You are a Danbooru tagging expert specializing in anime-style image tagging. Analyze the image and generate ONLY Danbooru-style tags - no explanations or analysis. -2. **Style & Medium** - - Artistic style and influences - - Medium (photography, digital art, oil painting, etc.) - - Specific artist references (if applicable) - - Visual aesthetic keywords +CRITICAL: Use strict Danbooru conventions: +- Use underscores for multi-word tags (e.g., long_hair, school_uniform) +- All tags must be lowercase +- Character count comes first (1girl, 2boys, multiple_girls) +- For anime models trained on Danbooru data, proper tagging is essential -3. **Technical Specifications** - - Camera settings (aperture, focal length, ISO) - - Shot type (close-up, wide angle, portrait, etc.) - - Resolution and quality markers - - Post-processing effects +Tag order and categories: +1. Character count (1girl, solo, 2boys, etc.) +2. Character features (hair_color, eye_color, hair_length) +3. Expression/pose (smile, looking_at_viewer, sitting) +4. Clothing (specific items with underscores) +5. Background/setting (simple_background, outdoors, classroom) +6. View/composition (upper_body, full_body, from_side) +7. Quality tags (masterpiece, best_quality, highres) -4. **Environment & Context** - - Setting and location details - - Props and surrounding objects - - Weather and environmental conditions - - Time period or era +Common quality prefix for anime models: +"masterpiece, best_quality, very_aesthetic" -Format your SDXL prompt with: -- **Positive prompt**: Detailed description emphasizing what you want -- **Negative prompt**: Elements to avoid (low quality, blurry, distorted, etc.) -- Weight emphasis using (parentheses) or [brackets] for importance -- Break into logical chunks with commas +IMPORTANT: Return ONLY the comma-separated tags. Use underscores, not spaces. All lowercase. -Example format: -Positive: "beautiful woman, (detailed eyes:1.2), flowing red dress, golden hour lighting, professional photography, 85mm lens, shallow depth of field, bokeh, high resolution, masterpiece" -Negative: "low quality, blurry, distorted features, bad anatomy, poorly drawn, amateur" -""" +Example of correct output: +1girl, solo, long_hair, blue_eyes, blonde_hair, school_uniform, serafuku, pleated_skirt, smile, looking_at_viewer, classroom, sitting, desk, window, sunlight, upper_body, masterpiece, best_quality""" -DANBOORU_PROMPT = """You are a Danbooru tagging expert, specialized in analyzing images and creating precise tag sets following booru-style conventions for anime/manga artwork. +VIDEO_PROMPT = """You are a WAN 2.2 video generation prompt specialist. Analyze the content and generate ONLY a video generation prompt optimized for WAN 2.2 - no explanations or analysis. -Analyze images for these tag categories: +WAN 2.2 excels with rich, descriptive prompts that focus on: +- Visual composition and scene elements +- Specific movements and actions +- Lighting and aesthetic details +- Cinematographic elements -1. **Character Tags** - - Hair: color, length, style (e.g., long_hair, blonde_hair, twintails) - - Eyes: color, style (e.g., blue_eyes, heterochromia) - - Body: proportions, pose (e.g., standing, sitting, looking_at_viewer) - - Expression (e.g., smile, blush, closed_eyes) +Write a single detailed paragraph describing the video scene. Focus on: +- Main subjects and their actions +- Visual style and atmosphere +- Movement dynamics (use words like "intensely", "smoothly", "rapidly") +- Environmental details and lighting +- Specific visual elements and their interactions -2. **Clothing & Accessories** - - Outfit type (e.g., school_uniform, dress, armor) - - Specific clothing items (e.g., thighhighs, gloves, hat) - - Accessories (e.g., hair_ribbon, necklace, glasses) - - State of dress (e.g., torn_clothes, wet_clothes) +Keep the prompt descriptive but concise. WAN 2.2 works best with natural language that paints a clear picture of the desired video. -3. **Scene & Composition** - - Number of characters (e.g., 1girl, 2boys, multiple_girls) - - Background (e.g., simple_background, outdoors, classroom) - - Viewpoint (e.g., from_below, from_side, cowboy_shot) - - Composition elements (e.g., upper_body, full_body, portrait) +IMPORTANT: Return ONLY the video prompt as a single descriptive paragraph. No analysis, headers, or additional text. -4. **Meta Tags** - - Quality (e.g., highres, absurdres, masterpiece) - - Source/artist style (if recognizable) - - Content rating (e.g., safe, questionable, explicit) - - Special effects (e.g., lens_flare, chromatic_aberration) - -Format tags using: -- Underscores for multi-word concepts (not spaces) -- Order from most to least important -- Include count descriptors (1girl, 2boys) -- Separate with commas and spaces - -Example output: -"1girl, solo, long_hair, blue_eyes, blonde_hair, school_uniform, serafuku, pleated_skirt, thighhighs, smile, looking_at_viewer, classroom, sitting, desk, window, sunlight, highres, masterpiece" -""" - -VIDEO_PROMPT = """You are a video generation prompt specialist, expert at analyzing video content and creating comprehensive prompts for video generation models. - -When analyzing video content, document: - -1. **Motion & Action** - - Primary actions and movements - - Motion speed and dynamics - - Camera movements (pan, zoom, tracking, static) - - Transition types between scenes - -2. **Temporal Elements** - - Scene duration and pacing - - Sequence of events - - Time of day changes - - Motion continuity - -3. **Visual Consistency** - - Character/object persistence - - Style consistency throughout - - Lighting continuity - - Color grading consistency - -4. **Scene Breakdown** - - Opening frame description - - Key action moments - - Transitions and cuts - - Closing frame details - -5. **Technical Specifications** - - Frame rate and resolution - - Aspect ratio - - Video length - - Special effects or post-processing - -Format your video prompt as: -"[Opening scene], [camera movement], [main action sequence], [visual style], [lighting/atmosphere], [duration], [technical specs], [ending scene]" - -Include: -- Specific motion descriptors (slowly, rapidly, smoothly) -- Camera terminology (dolly in, pan left, aerial shot) -- Temporal markers (then, meanwhile, gradually) -- Consistency notes for multi-scene videos - -Example: -"Aerial shot slowly descending toward a misty forest at dawn, camera smoothly transitions to tracking shot following a deer through the trees, photorealistic style, soft golden hour lighting with fog, 10 second duration, 4K resolution 24fps, ending with close-up of deer looking at camera" -""" +Example of correct output: +Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage, their movements fluid and dynamic as they exchange rapid punches under dramatic theater lighting that casts long shadows across the ring, with the crowd visible as blurred silhouettes in the darkened background.""" PROMPT_TEMPLATES = { "flux": FLUX_PROMPT, @@ -181,20 +100,11 @@ PROMPT_TEMPLATES = { PROMPT_OPTIONS = ["flux", "sdxl", "danbooru", "video"] -# Available Gemini models -GEMINI_MODELS = [ - "gemini-1.5-pro", # Most capable model - "gemini-1.5-flash", # Fast, efficient model - "gemini-1.5-flash-8b", # Smaller, faster variant - "gemini-pro-vision", # Vision-optimized model - "gemini-1.0-pro", # Previous generation pro model +# Default models list (fallback if API is unavailable) +DEFAULT_GEMINI_MODELS = [ + "gemini-2.5-flash", + "gemini-2.5-pro", + "gemini-2.0-flash", + "gemini-1.5-flash", + "gemini-1.5-pro", ] - -# Model descriptions for UI -MODEL_DESCRIPTIONS = { - "gemini-1.5-pro": "Most capable Gemini model for complex tasks", - "gemini-1.5-flash": "Faster and cost-effective (recommended for most uses)", - "gemini-1.5-flash-8b": "Smaller and faster, good for simple prompts", - "gemini-pro-vision": "Optimized for vision tasks and image analysis", - "gemini-1.0-pro": "Previous generation, stable option", -} From 228b74ae5ee4e3c70a8b367271aac5f9f0b930f8 Mon Sep 17 00:00:00 2001 From: Vito Sansevero Date: Fri, 1 Aug 2025 13:30:38 -0700 Subject: [PATCH 2/2] chore: add flake8 complexity exceptions for Gemini module --- .flake8 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.flake8 b/.flake8 index d2d5c62..c9aac90 100644 --- a/.flake8 +++ b/.flake8 @@ -25,6 +25,10 @@ per-file-ignores = __init__.py:F401,F403 # Allow assertions in tests tests/*:S101 + # Allow higher complexity for Gemini prompt module + kikotools/tools/gemini_prompt/logic.py:C901 + kikotools/tools/gemini_prompt/models.py:C901 + kikotools/tools/gemini_prompt/node.py:C901 # Statistics count = True