init
This commit is contained in:
Chris Rutherford
@@ -0,0 +1,120 @@
|
||||
# ComfyUI-ImageToPrompt
|
||||
|
||||
A custom ComfyUI node that sends images to OpenAI-compatible vision APIs (like GPT-4 Vision, LLaVA, etc.) and returns the LLM's text response. Perfect for image analysis, captioning, and visual question answering workflows.
|
||||
|
||||
## Features
|
||||
|
||||
- 📷 **Upload images** via ComfyUI's image input nodes
|
||||
- 🤖 **Send to any OpenAI-compatible API** (GPT-4 Vision, local LLaVA, llama.cpp, etc.)
|
||||
- 💬 **Optional system and user prompts** for controlling the analysis
|
||||
- ⚙️ **Configurable parameters**: temperature, max tokens, model selection
|
||||
- 📝 **Text output** that can be used in downstream nodes or displayed in the UI
|
||||
- 🔍 **Debug logging** to monitor API calls and responses
|
||||
|
||||
## Installation
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- [ComfyUI](https://github.com/comfyanonymous/ComfyUI) installed and working
|
||||
|
||||
### Install via ComfyUI Manager (Recommended)
|
||||
|
||||
1. Open ComfyUI Manager
|
||||
2. Search for "ComfyUI-ImageToPrompt"
|
||||
3. Click Install
|
||||
4. Restart ComfyUI
|
||||
|
||||
### Manual Installation
|
||||
|
||||
1. Navigate to your ComfyUI custom nodes directory:
|
||||
```bash
|
||||
cd ComfyUI/custom_nodes/
|
||||
```
|
||||
|
||||
2. Clone this repository:
|
||||
```bash
|
||||
git clone https://github.com/yourusername/ComfyUI-ImageToPrompt.git
|
||||
```
|
||||
|
||||
3. Install dependencies:
|
||||
```bash
|
||||
cd ComfyUI-ImageToPrompt
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
4. Restart ComfyUI
|
||||
|
||||
## Usage
|
||||
|
||||
1. **Add the node**: In ComfyUI, add the "Image to LLM Prompt" node (found under the "LLM" category)
|
||||
|
||||
2. **Connect an image**: Connect an image output (from LoadImage, etc.) to the node's image input
|
||||
|
||||
3. **Configure the API**:
|
||||
- `base_url`: Your OpenAI-compatible API endpoint (e.g., `http://localhost:8080/v1`)
|
||||
- `api_key`: Your API key (use `sk-test` for local servers that don't require authentication)
|
||||
- `model`: Model name (e.g., `gpt-4o`, `llava`, etc.)
|
||||
|
||||
4. **Add prompts (optional)**:
|
||||
- `system_prompt`: Instructions for the AI (e.g., "You are an expert art critic")
|
||||
- `user_prompt`: Specific question about the image (e.g., "Describe this image in detail")
|
||||
|
||||
5. **Adjust parameters (optional)**:
|
||||
- `temperature`: Controls randomness (0.0 = deterministic, 2.0 = very random)
|
||||
- `max_tokens`: Maximum response length
|
||||
|
||||
6. **Execute the workflow**: The LLM's response will appear in the node's output and can be passed to other nodes
|
||||
|
||||
## Example Workflow
|
||||
|
||||
```
|
||||
LoadImage → ImageToLLMPrompt → (use text output in other nodes)
|
||||
```
|
||||
|
||||
The node outputs a STRING that can be connected to:
|
||||
- Text processing nodes
|
||||
- Prompt nodes for image generation
|
||||
- File writing nodes
|
||||
- Any node that accepts STRING input
|
||||
|
||||
## API Compatibility
|
||||
|
||||
This node works with any OpenAI-compatible vision API endpoint, including:
|
||||
|
||||
- **OpenAI GPT-4 Vision** (`https://api.openai.com/v1`)
|
||||
- **Local LLaVA servers** (via llama.cpp, ollama, etc.)
|
||||
- **Azure OpenAI** with vision models
|
||||
- **Other compatible backends**
|
||||
|
||||
## Configuration Examples
|
||||
|
||||
### OpenAI GPT-4 Vision
|
||||
```
|
||||
base_url: https://api.openai.com/v1
|
||||
api_key: sk-your-actual-api-key
|
||||
model: gpt-4o
|
||||
```
|
||||
|
||||
### Local LLaVA (llama.cpp server)
|
||||
```
|
||||
base_url: http://localhost:8080/v1
|
||||
api_key: sk-test
|
||||
model: llava
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
- **"LLM API call failed" errors**: Check that your base_url is correct and the server is running
|
||||
- **Empty responses**: Verify your model supports vision/image inputs
|
||||
- **Timeout errors**: Increase the timeout in the code or use a faster model
|
||||
- **Node not appearing**: Restart ComfyUI and check the console for import errors
|
||||
|
||||
## Dependencies
|
||||
|
||||
- `openai` - OpenAI Python client library
|
||||
- `pillow` - Python Imaging Library for image processing
|
||||
- `numpy` - Array processing (typically already installed with ComfyUI)
|
||||
|
||||
## Contributing
|
||||
|
||||
Contributions are welcome! Please feel free to submit a Pull Request.
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
import base64
|
||||
import io
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
from openai import OpenAI
|
||||
|
||||
|
||||
class ImageToLLMPrompt:
|
||||
"""
|
||||
A custom node that uploads an image and sends it to an OpenAI-compatible API
|
||||
with optional system and user prompts, returning the LLM response as text.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"image": ("IMAGE",),
|
||||
"base_url": ("STRING", {
|
||||
"default": "http://localhost:8080/v1",
|
||||
"multiline": False
|
||||
}),
|
||||
"api_key": ("STRING", {
|
||||
"default": "sk-test",
|
||||
"multiline": False
|
||||
}),
|
||||
"model": ("STRING", {
|
||||
"default": "gpt-4o",
|
||||
"multiline": False
|
||||
}),
|
||||
},
|
||||
"optional": {
|
||||
"system_prompt": ("STRING", {
|
||||
"default": "",
|
||||
"multiline": True
|
||||
}),
|
||||
"user_prompt": ("STRING", {
|
||||
"default": "",
|
||||
"multiline": True
|
||||
}),
|
||||
"temperature": ("FLOAT", {
|
||||
"default": 0.7,
|
||||
"min": 0.0,
|
||||
"max": 2.0,
|
||||
"step": 0.1
|
||||
}),
|
||||
"max_tokens": ("INT", {
|
||||
"default": 2048,
|
||||
"min": 1,
|
||||
"max": 32768,
|
||||
"step": 1
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
RETURN_NAMES = ("text",)
|
||||
FUNCTION = "process_image"
|
||||
CATEGORY = "LLM"
|
||||
OUTPUT_NODE = True
|
||||
|
||||
def tensor_to_base64(self, image_tensor):
|
||||
"""Convert ComfyUI image tensor to base64 string"""
|
||||
# ComfyUI images are in format [B, H, W, C] with values 0-1
|
||||
image_np = (image_tensor[0].cpu().numpy() * 255).astype(np.uint8)
|
||||
|
||||
# Convert to PIL Image
|
||||
pil_image = Image.fromarray(image_np)
|
||||
|
||||
# Convert to base64
|
||||
buffer = io.BytesIO()
|
||||
pil_image.save(buffer, format="PNG")
|
||||
buffer.seek(0)
|
||||
base64_image = base64.b64encode(buffer.read()).decode('utf-8')
|
||||
|
||||
return base64_image
|
||||
|
||||
def process_image(self, image, base_url, api_key, model,
|
||||
system_prompt="", user_prompt="", temperature=0.7, max_tokens=2048):
|
||||
"""Send image to LLM and return response"""
|
||||
|
||||
# Convert image to base64
|
||||
base64_image = self.tensor_to_base64(image)
|
||||
|
||||
# Create OpenAI client
|
||||
client = OpenAI(
|
||||
base_url=base_url,
|
||||
api_key=api_key,
|
||||
timeout=180
|
||||
)
|
||||
|
||||
# Build messages array
|
||||
messages = []
|
||||
|
||||
# Add optional system prompt
|
||||
if system_prompt and system_prompt.strip():
|
||||
messages.append({
|
||||
"role": "system",
|
||||
"content": system_prompt
|
||||
})
|
||||
|
||||
# Build user message content
|
||||
user_content = []
|
||||
|
||||
# Add optional user prompt text
|
||||
if user_prompt and user_prompt.strip():
|
||||
user_content.append({
|
||||
"type": "text",
|
||||
"text": user_prompt
|
||||
})
|
||||
|
||||
# Add the image
|
||||
user_content.append({
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": f"data:image/png;base64,{base64_image}"
|
||||
}
|
||||
})
|
||||
|
||||
messages.append({
|
||||
"role": "user",
|
||||
"content": user_content
|
||||
})
|
||||
|
||||
# Make API call
|
||||
try:
|
||||
print(f"[ImageToLLMPrompt] Sending request to {base_url} with model {model}")
|
||||
response = client.chat.completions.create(
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
messages=messages
|
||||
)
|
||||
|
||||
result = response.choices[0].message.content
|
||||
print(f"[ImageToLLMPrompt] LLM Response received (length: {len(result)} chars)")
|
||||
print(f"[ImageToLLMPrompt] Response content:\n{result}")
|
||||
return {"ui": {"text": (result,)}, "result": (result,)}
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"LLM API call failed: {str(e)}"
|
||||
print(f"[ImageToLLMPrompt] ERROR: {error_msg}")
|
||||
return {"ui": {"text": (error_msg,)}, "result": (error_msg,)}
|
||||
|
||||
|
||||
# Node registration
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"ImageToLLMPrompt": ImageToLLMPrompt
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"ImageToLLMPrompt": "Image to LLM Prompt"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
openai>=1.0.0
|
||||
pillow>=10.0.0
|
||||
numpy>=1.24.0
|
||||
Reference in New Issue
Block a user