commit e538c2cfdc369595c1948cbfbf2ef425f708d7ef Author: Chris Rutherford Date: Sat Apr 11 13:16:01 2026 +0100 init diff --git a/README.md b/README.md new file mode 100644 index 0000000..6301916 --- /dev/null +++ b/README.md @@ -0,0 +1,120 @@ +# ComfyUI-ImageToPrompt + +A custom ComfyUI node that sends images to OpenAI-compatible vision APIs (like GPT-4 Vision, LLaVA, etc.) and returns the LLM's text response. Perfect for image analysis, captioning, and visual question answering workflows. + +## Features + +- 📷 **Upload images** via ComfyUI's image input nodes +- 🤖 **Send to any OpenAI-compatible API** (GPT-4 Vision, local LLaVA, llama.cpp, etc.) +- 💬 **Optional system and user prompts** for controlling the analysis +- ⚙️ **Configurable parameters**: temperature, max tokens, model selection +- 📝 **Text output** that can be used in downstream nodes or displayed in the UI +- 🔍 **Debug logging** to monitor API calls and responses + +## Installation + +### Prerequisites + +- [ComfyUI](https://github.com/comfyanonymous/ComfyUI) installed and working + +### Install via ComfyUI Manager (Recommended) + +1. Open ComfyUI Manager +2. Search for "ComfyUI-ImageToPrompt" +3. Click Install +4. Restart ComfyUI + +### Manual Installation + +1. Navigate to your ComfyUI custom nodes directory: + ```bash + cd ComfyUI/custom_nodes/ + ``` + +2. Clone this repository: + ```bash + git clone https://github.com/yourusername/ComfyUI-ImageToPrompt.git + ``` + +3. Install dependencies: + ```bash + cd ComfyUI-ImageToPrompt + pip install -r requirements.txt + ``` + +4. Restart ComfyUI + +## Usage + +1. **Add the node**: In ComfyUI, add the "Image to LLM Prompt" node (found under the "LLM" category) + +2. **Connect an image**: Connect an image output (from LoadImage, etc.) to the node's image input + +3. **Configure the API**: + - `base_url`: Your OpenAI-compatible API endpoint (e.g., `http://localhost:8080/v1`) + - `api_key`: Your API key (use `sk-test` for local servers that don't require authentication) + - `model`: Model name (e.g., `gpt-4o`, `llava`, etc.) + +4. **Add prompts (optional)**: + - `system_prompt`: Instructions for the AI (e.g., "You are an expert art critic") + - `user_prompt`: Specific question about the image (e.g., "Describe this image in detail") + +5. **Adjust parameters (optional)**: + - `temperature`: Controls randomness (0.0 = deterministic, 2.0 = very random) + - `max_tokens`: Maximum response length + +6. **Execute the workflow**: The LLM's response will appear in the node's output and can be passed to other nodes + +## Example Workflow + +``` +LoadImage → ImageToLLMPrompt → (use text output in other nodes) +``` + +The node outputs a STRING that can be connected to: +- Text processing nodes +- Prompt nodes for image generation +- File writing nodes +- Any node that accepts STRING input + +## API Compatibility + +This node works with any OpenAI-compatible vision API endpoint, including: + +- **OpenAI GPT-4 Vision** (`https://api.openai.com/v1`) +- **Local LLaVA servers** (via llama.cpp, ollama, etc.) +- **Azure OpenAI** with vision models +- **Other compatible backends** + +## Configuration Examples + +### OpenAI GPT-4 Vision +``` +base_url: https://api.openai.com/v1 +api_key: sk-your-actual-api-key +model: gpt-4o +``` + +### Local LLaVA (llama.cpp server) +``` +base_url: http://localhost:8080/v1 +api_key: sk-test +model: llava +``` + +## Troubleshooting + +- **"LLM API call failed" errors**: Check that your base_url is correct and the server is running +- **Empty responses**: Verify your model supports vision/image inputs +- **Timeout errors**: Increase the timeout in the code or use a faster model +- **Node not appearing**: Restart ComfyUI and check the console for import errors + +## Dependencies + +- `openai` - OpenAI Python client library +- `pillow` - Python Imaging Library for image processing +- `numpy` - Array processing (typically already installed with ComfyUI) + +## Contributing + +Contributions are welcome! Please feel free to submit a Pull Request. diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..a3367df --- /dev/null +++ b/__init__.py @@ -0,0 +1,153 @@ +import base64 +import io +from PIL import Image +import numpy as np +from openai import OpenAI + + +class ImageToLLMPrompt: + """ + A custom node that uploads an image and sends it to an OpenAI-compatible API + with optional system and user prompts, returning the LLM response as text. + """ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "image": ("IMAGE",), + "base_url": ("STRING", { + "default": "http://localhost:8080/v1", + "multiline": False + }), + "api_key": ("STRING", { + "default": "sk-test", + "multiline": False + }), + "model": ("STRING", { + "default": "gpt-4o", + "multiline": False + }), + }, + "optional": { + "system_prompt": ("STRING", { + "default": "", + "multiline": True + }), + "user_prompt": ("STRING", { + "default": "", + "multiline": True + }), + "temperature": ("FLOAT", { + "default": 0.7, + "min": 0.0, + "max": 2.0, + "step": 0.1 + }), + "max_tokens": ("INT", { + "default": 2048, + "min": 1, + "max": 32768, + "step": 1 + }), + } + } + + RETURN_TYPES = ("STRING",) + RETURN_NAMES = ("text",) + FUNCTION = "process_image" + CATEGORY = "LLM" + OUTPUT_NODE = True + + def tensor_to_base64(self, image_tensor): + """Convert ComfyUI image tensor to base64 string""" + # ComfyUI images are in format [B, H, W, C] with values 0-1 + image_np = (image_tensor[0].cpu().numpy() * 255).astype(np.uint8) + + # Convert to PIL Image + pil_image = Image.fromarray(image_np) + + # Convert to base64 + buffer = io.BytesIO() + pil_image.save(buffer, format="PNG") + buffer.seek(0) + base64_image = base64.b64encode(buffer.read()).decode('utf-8') + + return base64_image + + def process_image(self, image, base_url, api_key, model, + system_prompt="", user_prompt="", temperature=0.7, max_tokens=2048): + """Send image to LLM and return response""" + + # Convert image to base64 + base64_image = self.tensor_to_base64(image) + + # Create OpenAI client + client = OpenAI( + base_url=base_url, + api_key=api_key, + timeout=180 + ) + + # Build messages array + messages = [] + + # Add optional system prompt + if system_prompt and system_prompt.strip(): + messages.append({ + "role": "system", + "content": system_prompt + }) + + # Build user message content + user_content = [] + + # Add optional user prompt text + if user_prompt and user_prompt.strip(): + user_content.append({ + "type": "text", + "text": user_prompt + }) + + # Add the image + user_content.append({ + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{base64_image}" + } + }) + + messages.append({ + "role": "user", + "content": user_content + }) + + # Make API call + try: + print(f"[ImageToLLMPrompt] Sending request to {base_url} with model {model}") + response = client.chat.completions.create( + model=model, + temperature=temperature, + max_tokens=max_tokens, + messages=messages + ) + + result = response.choices[0].message.content + print(f"[ImageToLLMPrompt] LLM Response received (length: {len(result)} chars)") + print(f"[ImageToLLMPrompt] Response content:\n{result}") + return {"ui": {"text": (result,)}, "result": (result,)} + + except Exception as e: + error_msg = f"LLM API call failed: {str(e)}" + print(f"[ImageToLLMPrompt] ERROR: {error_msg}") + return {"ui": {"text": (error_msg,)}, "result": (error_msg,)} + + +# Node registration +NODE_CLASS_MAPPINGS = { + "ImageToLLMPrompt": ImageToLLMPrompt +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "ImageToLLMPrompt": "Image to LLM Prompt" +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a5872bb --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +openai>=1.0.0 +pillow>=10.0.0 +numpy>=1.24.0