diff --git a/README.md b/README.md index 956263e..3d38adb 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ The **ListHelper** collection is a comprehensive set of custom nodes for ComfyUI 1. [AudioListCombine](#audiolistcombine-node) 2. [NumberListGenerator](#numberlistgenerator-node) 3. [PromptSplitByDelimiter](#promptsplitbydelimiter-node) +4. OpenRouterLLM (For Free Bananas) +![Demo](Readme/demo6.jpg) --- diff --git a/Readme/demo6.jpg b/Readme/demo6.jpg new file mode 100644 index 0000000..2b6d69f Binary files /dev/null and b/Readme/demo6.jpg differ diff --git a/models.json b/models.json new file mode 100644 index 0000000..7b5df14 --- /dev/null +++ b/models.json @@ -0,0 +1,15 @@ +{ + "models": [ + "google/gemini-2.5-flash-image-preview:free", + "google/gemma-3-27b-it:free", + "qwen/qwen2.5-vl-32b-instruct:free", + "mistralai/mistral-small-3.2-24b-instruct:free", + "meta-llama/llama-4-maverick:free", + "openai/gpt-oss-20b:free", + "cognitivecomputations/dolphin-mistral-24b-venice-edition:free", + "moonshotai/kimi-dev-72b:free", + "qwen/qwen2.5-vl-72b-instruct:free", + "moonshotai/kimi-k2:free", + "deepseek/deepseek-chat-v3.1:free" + ] +} \ No newline at end of file diff --git a/nodes.py b/nodes.py index a9b3bf0..1b21797 100644 --- a/nodes.py +++ b/nodes.py @@ -24,6 +24,7 @@ from comfy.cli_args import args from typing import List, Dict, Any, Tuple from random import Random from datetime import datetime +from .openrouter_llm import OpenRouterLLM class AudioListGenerator: @classmethod @@ -1069,6 +1070,7 @@ NODE_CLASS_MAPPINGS = { "LoadVideoPath": LoadVideoPath, "SaveVideoPath": SaveVideoPath, "FrameMatch": FrameMatch, + "OpenRouterLLM": OpenRouterLLM, } NODE_DISPLAY_NAME_MAPPINGS = { @@ -1082,5 +1084,6 @@ NODE_DISPLAY_NAME_MAPPINGS = { "LoadVideoPath": "LoadVideoPath", "SaveVideoPath": "SaveVideoPath", "FrameMatch": "FrameMatch", + "OpenRouterLLM": "OpenRouter LLM", } diff --git a/openrouter_llm.py b/openrouter_llm.py new file mode 100644 index 0000000..5ff9cbe --- /dev/null +++ b/openrouter_llm.py @@ -0,0 +1,495 @@ +import requests +import json +import base64 +import os +import io +import re +import torch +from PIL import Image +import numpy as np + +class OpenRouterLLM: + """ + OpenRouter LLM節點,用於處理OpenRouter的LLM類型 + 支援圖片和文字輸入輸出,API金鑰管理,動態模型管理 + """ + + @classmethod + def _load_models(cls): + """載入模型列表從models.json檔案""" + models_file = os.path.join(os.path.dirname(__file__), "models.json") + try: + with open(models_file, 'r', encoding='utf-8') as f: + data = json.load(f) + return data.get('models', []) + except Exception as e: + print(f"⚠️ 載入models.json失敗: {e}") + # 返回默認模型列表 + return [ + "google/gemini-2.5-flash-image-preview:free", + "google/gemma-3-27b-it:free", + "qwen/qwen2.5-vl-32b-instruct:free", + "mistralai/mistral-small-3.2-24b-instruct:free", + "meta-llama/llama-4-maverick:free", + "openai/gpt-oss-20b:free", + "cognitivecomputations/dolphin-mistral-24b-venice-edition:free", + "moonshotai/kimi-dev-72b:free", + "qwen/qwen2.5-vl-72b-instruct:free", + "moonshotai/kimi-k2:free" + ] + + @classmethod + def _save_models(cls, models_list): + """保存模型列表到models.json檔案""" + models_file = os.path.join(os.path.dirname(__file__), "models.json") + try: + with open(models_file, 'w', encoding='utf-8') as f: + json.dump({"models": models_list}, f, ensure_ascii=False, indent=2) + except Exception as e: + print(f"❌ 保存models.json失敗: {e}") + + def _add_custom_model(self, model_name): + """添加自定義模型到models.json檔案""" + try: + current_models = self._load_models() + + # 檢查模型是否已存在 + if model_name not in current_models: + current_models.append(model_name) + self._save_models(current_models) + print(f"✅ 已添加新模型: {model_name}") + else: + print(f"ℹ️ 模型已存在: {model_name}") + except Exception as e: + print(f"❌ 添加自定義模型失敗: {e}") + + @classmethod + def INPUT_TYPES(cls): + # 從JSON檔案載入模型列表 + text_models = cls._load_models() + + # 在列表末尾添加"Add Custom Model..."選項 + text_models_with_add = text_models + ["Add Custom Model..."] + + return { + "required": { + "api_key": ("STRING", {"multiline": False, "default": "", "placeholder": "輸入您的OpenRouter API金鑰"}), + "user_prompt": ("STRING", {"multiline": True, "default": "Please analyze the provided content."}), + "text_model": (text_models_with_add, {"default": text_models_with_add[0]}), + }, + "optional": { + "custom_model": ("STRING", {"multiline": False, "default": "", "placeholder": "輸入自定義模型名稱 (選擇Add Custom Model...時使用)"}), + "system_prompt": ("STRING", {"multiline": True, "default": ""}), + "image_input_1": ("IMAGE",), + "image_input_2": ("IMAGE",), + "image_input_3": ("IMAGE",), + } + } + + RETURN_TYPES = ("IMAGE", "STRING") + RETURN_NAMES = ("image_output", "text_output") + FUNCTION = "process_llm" + CATEGORY = "ListHelper" + + def __init__(self): + self.config_file = "config.json" + + def _load_config(self): + """載入配置文件""" + try: + if os.path.exists(self.config_file): + with open(self.config_file, 'r', encoding='utf-8') as f: + return json.load(f) + except Exception as e: + print(f"載入配置文件失敗: {e}") + return {} + + def _save_config(self, config): + """保存配置文件""" + try: + with open(self.config_file, 'w', encoding='utf-8') as f: + json.dump(config, f, ensure_ascii=False, indent=2) + except Exception as e: + print(f"保存配置文件失敗: {e}") + + def _get_api_key(self, api_key_input): + """獲取API金鑰""" + config = self._load_config() + + # 如果輸入了新的API金鑰,則更新配置 + if api_key_input.strip(): + config['openrouter_api_key'] = api_key_input.strip() + self._save_config(config) + return api_key_input.strip() + + # 否則使用配置文件中的金鑰 + return config.get('openrouter_api_key', '') + + def _tensor_to_base64(self, tensor): + """將ComfyUI圖像tensor轉換為base64編碼""" + # tensor shape: [H, W, C] (0-1 range) + if tensor.dim() == 4: + tensor = tensor.squeeze(0) # 移除批次維度 + + # 轉換為numpy並調整範圍到0-255 + numpy_image = (tensor.cpu().numpy() * 255).astype(np.uint8) + + # 轉換為PIL圖像 + pil_image = Image.fromarray(numpy_image) + + # 轉換為base64 + buffer = io.BytesIO() + pil_image.save(buffer, format='PNG') + image_base64 = base64.b64encode(buffer.getvalue()).decode('utf-8') + + return f"data:image/png;base64,{image_base64}" + + def _base64_to_tensor(self, base64_str): + """將base64編碼轉換為ComfyUI圖像tensor""" + try: + # 移除各種可能的前綴 + prefixes = [ + 'data:image/png;base64,', + 'data:image/jpeg;base64,', + 'data:image/jpg;base64,', + ] + + original_str = base64_str + for prefix in prefixes: + if base64_str.startswith(prefix): + base64_str = base64_str[len(prefix):] + break + + # 清理base64字符串(移除可能的換行符和空格) + base64_str = re.sub(r'[\n\r\s]', '', base64_str) + + # 解碼base64 + image_bytes = base64.b64decode(base64_str) + + # 轉換為PIL圖像 + pil_image = Image.open(io.BytesIO(image_bytes)) + + # 確保是RGB模式 + if pil_image.mode != 'RGB': + pil_image = pil_image.convert('RGB') + + # 轉換為numpy數組 + numpy_image = np.array(pil_image).astype(np.float32) / 255.0 + + # 轉換為tensor並添加批次維度 [1, H, W, C] + tensor = torch.from_numpy(numpy_image).unsqueeze(0) + + return tensor + + except Exception as e: + print(f"❌ base64轉tensor失敗: {e}") + return torch.zeros(1, 1, 1, 3) + + def _url_to_tensor(self, image_url): + """從URL下載圖像並轉換為ComfyUI圖像tensor""" + try: + # 下載圖像 + response = requests.get(image_url, timeout=30) + response.raise_for_status() + + # 轉換為PIL圖像 + pil_image = Image.open(io.BytesIO(response.content)) + + # 確保是RGB模式 + if pil_image.mode != 'RGB': + pil_image = pil_image.convert('RGB') + + # 轉換為numpy數組 + numpy_image = np.array(pil_image).astype(np.float32) / 255.0 + + # 轉換為tensor並添加批次維度 [1, H, W, C] + tensor = torch.from_numpy(numpy_image).unsqueeze(0) + + return tensor + + except Exception as e: + print(f"❌ 從URL載入圖像失敗: {e}") + return torch.zeros(1, 1, 1, 3) + + def _call_openrouter_api(self, api_key, model, messages): + """調用OpenRouter API""" + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + "HTTP-Referer": "", + "X-Title": "ComfyUI" + } + + data = { + "model": model, + "messages": messages + } + + try: + response = requests.post( + "https://openrouter.ai/api/v1/chat/completions", + headers=headers, + data=json.dumps(data), + timeout=60 + ) + + if response.status_code == 200: + return response.json() + elif response.status_code == 429: + # Rate limit錯誤 + try: + error_data = response.json() + return {"error": {"code": 429, "message": error_data.get("error", {}).get("message", "Rate limit exceeded")}} + except: + return {"error": {"code": 429, "message": "Rate limit exceeded"}} + else: + try: + error_data = response.json() + return {"error": error_data.get("error", {"message": f"HTTP {response.status_code}"})} + except: + return {"error": {"message": f"HTTP {response.status_code}: {response.text[:200]}"}} + + except Exception as e: + print(f"❌ API調用異常: {e}") + return None + + def process_llm(self, api_key, user_prompt, text_model, custom_model="", system_prompt="", + image_input_1=None, image_input_2=None, image_input_3=None): + """處理LLM請求""" + + # 獲取API金鑰 + actual_api_key = self._get_api_key(api_key) + if not actual_api_key: + return (torch.zeros(1, 1, 1, 3), "❌ 錯誤: 請提供OpenRouter API金鑰") + + # 處理模型選擇 + if text_model == "Add Custom Model...": + if not custom_model or not custom_model.strip(): + return (torch.zeros(1, 1, 1, 3), "❌ 請在custom_model欄位輸入自定義模型名稱") + + # 使用自定義模型 + selected_model = custom_model.strip() + + # 將新模型添加到models.json中 + self._add_custom_model(selected_model) + else: + # 使用選擇的預設模型 + selected_model = text_model + + # 準備消息內容 + messages = [] + + # 添加系統提示(如果有的話) + if system_prompt and system_prompt.strip(): + messages.append({"role": "system", "content": system_prompt.strip()}) + + # 構建用戶消息 + user_content = [] + + # 添加文字內容 + if user_prompt and user_prompt.strip(): + user_content.append({ + "type": "text", + "text": user_prompt.strip() + }) + + # 檢查是否有圖像輸入 + has_images = any([image_input_1 is not None, image_input_2 is not None, image_input_3 is not None]) + + # 確定使用的模型 + if has_images: + # 有圖像輸入時,強制使用gemini-2.5-flash-image-preview:free + actual_model = "google/gemini-2.5-flash-image-preview:free" + else: + # 無圖像輸入時,使用用戶選擇的模型 + actual_model = selected_model + + # 檢查是否使用圖像生成模型(無論是否有圖像輸入) + is_image_model = actual_model == "google/gemini-2.5-flash-image-preview:free" + + if has_images: + # 有圖像輸入的情況 + # 為圖像生成添加特殊提示 + if not user_prompt or not user_prompt.strip(): + combined_text = "請分析這些圖像並生成一個相關的新圖像。請返回生成的圖像。" + else: + combined_text = user_prompt.strip() + if "生成" in combined_text or "創建" in combined_text or "製作" in combined_text: + combined_text += "\n\n請返回生成的圖像。" + + # 重新添加文字內容 + user_content = [{ + "type": "text", + "text": combined_text + }] + + # 添加圖像到用戶消息 + for i, image_input in enumerate([image_input_1, image_input_2, image_input_3], 1): + if image_input is not None: + try: + base64_image = self._tensor_to_base64(image_input) + user_content.append({ + "type": "image_url", + "image_url": { + "url": base64_image + } + }) + except Exception as e: + print(f"❌ 處理圖像{i}失敗: {e}") + elif is_image_model: + # 純文字模式但使用圖像生成模型 + combined_text = user_prompt.strip() if user_prompt else "" + if combined_text and ("生成" in combined_text or "創建" in combined_text or "製作" in combined_text or "畫" in combined_text or "繪製" in combined_text): + combined_text += "\n\n請生成並返回相應的圖像。" + elif not combined_text: + combined_text = "請生成一張圖像。" + + # 添加文字內容 + user_content.append({ + "type": "text", + "text": combined_text + }) + + # 添加用戶消息 + if len(user_content) > 1: + # 多媒體內容(包含圖像) + messages.append({ + "role": "user", + "content": user_content + }) + elif user_content: + # 純文字內容 + messages.append({ + "role": "user", + "content": user_content[0]["text"] + }) + else: + # 空內容 + messages.append({ + "role": "user", + "content": "請分析提供的內容。" + }) + + + # 調用API + response = self._call_openrouter_api(actual_api_key, actual_model, messages) + + if not response: + return (torch.zeros(1, 1, 1, 3), "❌ API調用失敗") + + # 檢查rate limit錯誤 + if 'error' in response: + error_info = response['error'] + if isinstance(error_info, dict) and 'code' in error_info: + if error_info['code'] == 429 or 'rate_limit' in str(error_info).lower(): + error_msg = f"⚠️ Rate Limit達到: {error_info.get('message', '請稍後再試')}" + print(error_msg) + return (torch.zeros(1, 1, 1, 3), error_msg) + else: + error_msg = f"❌ API錯誤: {error_info.get('message', str(error_info))}" + return (torch.zeros(1, 1, 1, 3), error_msg) + else: + error_msg = f"❌ API錯誤: {str(error_info)}" + return (torch.zeros(1, 1, 1, 3), error_msg) + + if 'choices' not in response or not response['choices']: + return (torch.zeros(1, 1, 1, 3), "❌ API回應格式異常") + + # 獲取回應訊息 + message = response['choices'][0]['message'] + response_content = message.get('content', '') + + # 檢查回應是否包含圖像數據 + output_image = torch.zeros(1, 1, 1, 3) # 預設空圖像 + output_text = response_content + found_image = False + + # 首先檢查message.images字段 + if 'images' in message and message['images']: + image_data = message['images'][0] + + try: + if isinstance(image_data, str): + if image_data.startswith('data:image/'): + output_image = self._base64_to_tensor(image_data) + found_image = True + elif image_data.startswith('http'): + output_image = self._url_to_tensor(image_data) + found_image = True + else: + # 可能是純 base64,添加 data URL 前綴 + full_data_url = f"data:image/png;base64,{image_data}" + output_image = self._base64_to_tensor(full_data_url) + found_image = True + + elif isinstance(image_data, dict): + # 處理嵌套的 image_url 字典格式 + if 'type' in image_data and image_data['type'] == 'image_url' and 'image_url' in image_data: + nested_image_url = image_data['image_url'] + if isinstance(nested_image_url, dict) and 'url' in nested_image_url: + image_url_value = nested_image_url['url'] + if str(image_url_value).startswith('data:image/'): + output_image = self._base64_to_tensor(str(image_url_value)) + else: + output_image = self._url_to_tensor(str(image_url_value)) + found_image = True + # 處理直接包含 url 的格式 + elif 'url' in image_data: + url_value = str(image_data['url']) + if url_value.startswith('data:image/'): + output_image = self._base64_to_tensor(url_value) + else: + output_image = self._url_to_tensor(url_value) + found_image = True + # 處理直接包含 data 的格式 + elif 'data' in image_data: + output_image = self._base64_to_tensor(str(image_data['data'])) + found_image = True + + except Exception as e: + print(f"❌ 處理圖像失敗: {e}") + found_image = False + + # 如果在message.images中沒有找到圖像,檢查content中的嵌入圖像 + if not found_image and is_image_model: + # 檢查URL和base64格式的圖像 + url_patterns = [ + r'https?://[^\s\)]+\.(?:png|jpg|jpeg|gif|webp)', + r'!\[.*?\]\((https?://[^\s\)]+\.(?:png|jpg|jpeg|gif|webp))\)', + r']+src=["\']([^"\']+)["\'][^>]*>', + ] + + base64_patterns = [ + r'data:image/png;base64,([A-Za-z0-9+/=\n\r]+)', + r'data:image/jpeg;base64,([A-Za-z0-9+/=\n\r]+)', + r'data:image/jpg;base64,([A-Za-z0-9+/=\n\r]+)', + ] + + # 檢查URL格式 + for pattern in url_patterns: + matches = re.findall(pattern, response_content, re.DOTALL) + if matches: + try: + output_image = self._url_to_tensor(matches[0]) + found_image = True + output_text = re.sub(pattern, "[Generated Image]", response_content, flags=re.DOTALL) + break + except Exception as e: + continue + + # 檢查base64格式 + if not found_image: + for pattern in base64_patterns: + matches = re.findall(pattern, response_content, re.DOTALL) + if matches: + try: + clean_base64 = re.sub(r'[\n\r\s]', '', matches[0]) + base64_image = f"data:image/png;base64,{clean_base64}" + output_image = self._base64_to_tensor(base64_image) + found_image = True + output_text = re.sub(pattern, "[Generated Image]", response_content, flags=re.DOTALL) + break + except Exception as e: + continue + + return (output_image, output_text) \ No newline at end of file