From 681f6ebc5fd850b57eea251cdd79a4261ea6b72f Mon Sep 17 00:00:00 2001 From: jason Date: Mon, 15 Sep 2025 15:31:26 +0800 Subject: [PATCH] support text generation * add gitignore * add video api * fix: export to video * fix * fix * fix * fix * fix * add i2v * fix * fix * add openai * fix: add base url * change default models * fix: add base url * add v1 * fix models * fix models * fix models * fix models * fix models * fix bug * fix * try to fix * use IO.STRING test * image fix * add chat content * add files * fix optional * fix file read * fix file read * change name * add get models * add unused apikey * change model sort * fix IMAGE_LIST * fix: change to image * add model cache * remove json_object * change max_tokens to 65536 * change max_tokens to 6000 --------- Co-authored-by: jason.mei --- py/flux_dev.py | 22 +- py/flux_kontext_max.py | 12 +- py/flux_kontext_max_t2i.py | 13 +- py/flux_kontext_pro.py | 12 +- py/flux_kontext_pro_t2i.py | 12 +- py/modelverse_api/requests/flux_dev.py | 1 + .../requests/flux_kontext_max.py | 3 + .../requests/flux_kontext_pro.py | 3 + py/modelverse_api/requests/step1x_edit.py | 1 + py/nodes.py | 11 +- py/openai.py | 500 ++++++++++++++++++ py/step1x_edit.py | 18 +- py/wan_ai_i2v.py | 18 +- py/wan_ai_t2v.py | 10 +- requirements.txt | 1 + 15 files changed, 573 insertions(+), 64 deletions(-) create mode 100644 py/openai.py diff --git a/py/flux_dev.py b/py/flux_dev.py index b74eca5..622bd23 100644 --- a/py/flux_dev.py +++ b/py/flux_dev.py @@ -2,7 +2,7 @@ from .modelverse_api.utils import imageurl2tensor from .modelverse_api.client import ModelverseClient from .modelverse_api.requests.flux_dev import FluxDev import torch -import asyncio +from comfy.comfy_types.node_typing import IO class FluxDevNode: @@ -17,8 +17,8 @@ class FluxDevNode: return { "required": { "client": ("MODELVERSE_API_CLIENT",), - "prompt": ("STRING", {"multiline": True, "default": "", "tooltip": "Text description of the image to generate"}), - "width": ("INT", { + "prompt": (IO.STRING, {"multiline": True, "default": "", "tooltip": "Text description of the image to generate"}), + "width": (IO.INT, { "default": 1024, "min": 512, "max": 1536, @@ -26,7 +26,7 @@ class FluxDevNode: "display": "number", "tooltip": "Image width (512 to 1536)" }), - "height": ("INT", { + "height": (IO.INT, { "default": 1024, "min": 512, "max": 1536, @@ -34,7 +34,7 @@ class FluxDevNode: "display": "number", "tooltip": "Image height (512 to 1536)" }), - "strength": ("FLOAT", { + "strength": (IO.FLOAT, { "default": 0.8, "min": 0.01, "max": 1.0, @@ -42,14 +42,14 @@ class FluxDevNode: "display": "number", "tooltip": "Strength of the image-to-image transformation (0.01 to 1.0)" }), - "seed": ("INT", { + "seed": (IO.INT, { "default": -1, "min": -1, "max": 0xffffffffffffffff, "control_after_generate": True, "tooltip": "Random seed for reproducible results. -1 for random seed" }), - "num_images": ("INT", { + "num_images": (IO.INT, { "default": 1, "min": 1, "max": 4, @@ -57,7 +57,7 @@ class FluxDevNode: "display": "number", "tooltip": "Number of images to generate in a single request (1 to 4)" }), - "num_requests": ("INT", { + "num_requests": (IO.INT, { "default": 1, "min": 1, "max": 10, @@ -65,7 +65,7 @@ class FluxDevNode: "display": "number", "tooltip": "Number of request to make (1 to 10)" }), - "num_inference_steps": ("INT", { + "num_inference_steps": (IO.INT, { "default": 28, "min": 1, "max": 50, @@ -73,7 +73,7 @@ class FluxDevNode: "display": "number", "tooltip": "Number of inference steps (1 to 50)" }), - "guidance_scale": ("FLOAT", { + "guidance_scale": (IO.FLOAT, { "default": 3.5, "min": 0.0, "max": 10.0, @@ -83,7 +83,7 @@ class FluxDevNode: }), }, "optional": { - "image": ("IMAGE", { + "image": (IO.IMAGE, { "tooltip": "The image for reference.", "forceInput": False, "default": None diff --git a/py/flux_kontext_max.py b/py/flux_kontext_max.py index 029a290..bb548b3 100644 --- a/py/flux_kontext_max.py +++ b/py/flux_kontext_max.py @@ -2,7 +2,7 @@ from .modelverse_api.utils import imageurl2tensor from .modelverse_api.client import ModelverseClient from .modelverse_api.requests.flux_kontext_max import FluxKontextMax, FluxKontextMaxMulti import torch -import asyncio +from comfy.comfy_types.node_typing import IO class FluxKontextMaxNode: @@ -21,13 +21,13 @@ class FluxKontextMaxNode: return { "required": { "client": ("MODELVERSE_API_CLIENT",), - "prompt": ("STRING", {"multiline": True, "default": "", "tooltip": "Text description of the image to generate"}), - "images": ("IMAGE,IMAGE_LIST", { + "prompt": (IO.STRING, {"multiline": True, "default": "", "tooltip": "Text description of the image to generate"}), + "images": ("IMAGE_LIST", { "tooltip": "The image(s) to edit from. If not included, use text-to-image mode.", "forceInput": False, "default": None }), - "num_requests": ("INT", { + "num_requests": (IO.INT, { "default": 1, "min": 1, "max": 10, @@ -35,14 +35,14 @@ class FluxKontextMaxNode: "display": "number", "tooltip": "Number of request to make (1 to 10)" }), - "seed": ("INT", { + "seed": (IO.INT, { "default": -1, "min": -1, "max": 0xffffffffffffffff, "control_after_generate": True, "tooltip": "Random seed for reproducible results. -1 for random seed" }), - "guidance_scale": ("FLOAT", { + "guidance_scale": (IO.FLOAT, { "default": 2.5, "min": 0.0, "max": 10.0, diff --git a/py/flux_kontext_max_t2i.py b/py/flux_kontext_max_t2i.py index b6f4086..24dcd01 100644 --- a/py/flux_kontext_max_t2i.py +++ b/py/flux_kontext_max_t2i.py @@ -2,8 +2,7 @@ from .modelverse_api.utils import imageurl2tensor from .modelverse_api.client import ModelverseClient from .modelverse_api.requests.flux_kontext_max import FluxKontextMaxT2I import torch -import asyncio - +from comfy.comfy_types.node_typing import IO class FluxKontextMaxT2INode: """ @@ -18,12 +17,12 @@ class FluxKontextMaxT2INode: return { "required": { "client": ("MODELVERSE_API_CLIENT",), - "prompt": ("STRING", {"multiline": True, "default": "", "tooltip": "Text description of the image to generate"}), + "prompt": (IO.STRING, {"multiline": True, "default": "", "tooltip": "Text description of the image to generate"}), "aspect_ratio": (["21:9", "16:9", "16:10", "4:3", "1:1", "3:4", "10:16", "9:16", "9:21"], { "default": "1:1", "tooltip": "The aspect ratio of the output image, ranging from \"21:9\" to \"9:21\", default is \"1:1\"" }), - "num_images": ("INT", { + "num_images": (IO.INT, { "default": 1, "min": 1, "max": 4, @@ -31,7 +30,7 @@ class FluxKontextMaxT2INode: "display": "number", "tooltip": "Number of images to generate in a single request (1 to 4)" }), - "num_requests": ("INT", { + "num_requests": (IO.INT, { "default": 1, "min": 1, "max": 10, @@ -39,14 +38,14 @@ class FluxKontextMaxT2INode: "display": "number", "tooltip": "Number of request to make (1 to 10)" }), - "seed": ("INT", { + "seed": (IO.INT, { "default": -1, "min": -1, "max": 0xffffffffffffffff, "control_after_generate": True, "tooltip": "Random seed for reproducible results. -1 for random seed" }), - "guidance_scale": ("FLOAT", { + "guidance_scale": (IO.FLOAT, { "default": 2.5, "min": 0.0, "max": 10.0, diff --git a/py/flux_kontext_pro.py b/py/flux_kontext_pro.py index b1f2b68..f5d9809 100644 --- a/py/flux_kontext_pro.py +++ b/py/flux_kontext_pro.py @@ -2,7 +2,7 @@ from .modelverse_api.utils import imageurl2tensor from .modelverse_api.client import ModelverseClient from .modelverse_api.requests.flux_kontext_pro import FluxKontextPro, FluxKontextProMulti import torch -import asyncio +from comfy.comfy_types.node_typing import IO class FluxKontextProNode: @@ -21,13 +21,13 @@ class FluxKontextProNode: return { "required": { "client": ("MODELVERSE_API_CLIENT",), - "prompt": ("STRING", {"multiline": True, "default": "", "tooltip": "Text description of the image to generate"}), - "images": ("IMAGE,IMAGE_LIST", { + "prompt": (IO.STRING, {"multiline": True, "default": "", "tooltip": "Text description of the image to generate"}), + "images": ("IMAGE_LIST", { "tooltip": "The image(s) to edit from. If not included, use text-to-image mode.", "forceInput": False, "default": None }), - "num_requests": ("INT", { + "num_requests": (IO.INT, { "default": 1, "min": 1, "max": 10, @@ -35,14 +35,14 @@ class FluxKontextProNode: "display": "number", "tooltip": "Number of request to make (1 to 10)" }), - "seed": ("INT", { + "seed": (IO.INT, { "default": -1, "min": -1, "max": 0xffffffffffffffff, "control_after_generate": True, "tooltip": "Random seed for reproducible results. -1 for random seed" }), - "guidance_scale": ("FLOAT", { + "guidance_scale": (IO.FLOAT, { "default": 2.5, "min": 0.0, "max": 10.0, diff --git a/py/flux_kontext_pro_t2i.py b/py/flux_kontext_pro_t2i.py index 280ea70..b286450 100644 --- a/py/flux_kontext_pro_t2i.py +++ b/py/flux_kontext_pro_t2i.py @@ -2,7 +2,7 @@ from .modelverse_api.utils import imageurl2tensor from .modelverse_api.client import ModelverseClient from .modelverse_api.requests.flux_kontext_pro import FluxKontextProT2I import torch -import asyncio +from comfy.comfy_types.node_typing import IO class FluxKontextProT2INode: @@ -18,12 +18,12 @@ class FluxKontextProT2INode: return { "required": { "client": ("MODELVERSE_API_CLIENT",), - "prompt": ("STRING", {"multiline": True, "default": "", "tooltip": "Text description of the image to generate"}), + "prompt": (IO.STRING, {"multiline": True, "default": "", "tooltip": "Text description of the image to generate"}), "aspect_ratio": (["21:9", "16:9", "16:10", "4:3", "1:1", "3:4", "10:16", "9:16", "9:21"], { "default": "1:1", "tooltip": "The aspect ratio of the output image, ranging from \"21:9\" to \"9:21\", default is \"1:1\"" }), - "num_images": ("INT", { + "num_images": (IO.INT, { "default": 1, "min": 1, "max": 4, @@ -31,7 +31,7 @@ class FluxKontextProT2INode: "display": "number", "tooltip": "Number of images to generate in a single request (1 to 4)" }), - "num_requests": ("INT", { + "num_requests": (IO.INT, { "default": 1, "min": 1, "max": 10, @@ -39,14 +39,14 @@ class FluxKontextProT2INode: "display": "number", "tooltip": "Number of request to make (1 to 10)" }), - "seed": ("INT", { + "seed": (IO.INT, { "default": -1, "min": -1, "max": 0xffffffffffffffff, "control_after_generate": True, "tooltip": "Random seed for reproducible results. -1 for random seed" }), - "guidance_scale": ("FLOAT", { + "guidance_scale": (IO.FLOAT, { "default": 2.5, "min": 0.0, "max": 10.0, diff --git a/py/modelverse_api/requests/flux_dev.py b/py/modelverse_api/requests/flux_dev.py index 70fe293..6044406 100644 --- a/py/modelverse_api/requests/flux_dev.py +++ b/py/modelverse_api/requests/flux_dev.py @@ -8,6 +8,7 @@ class FluxDev(BaseRequest): """ Flux-dev text to image model, 12 billion parameter rectified flow transformer """ + API_PATH = "/v1/images/generations" guidance_scale: Optional[float] = Field( default=2.5, description="The CFG (Classifier Free Guidance) scale is a measure of how close you want the model to stick to your prompt when looking for a related image to show you.") num_images: Optional[int] = Field( diff --git a/py/modelverse_api/requests/flux_kontext_max.py b/py/modelverse_api/requests/flux_kontext_max.py index bc28cac..4d779fd 100644 --- a/py/modelverse_api/requests/flux_kontext_max.py +++ b/py/modelverse_api/requests/flux_kontext_max.py @@ -8,6 +8,7 @@ class FluxKontextMax(BaseRequest): """ Flux Kontext Max for image editing. """ + API_PATH = "/v1/images/generations" guidance_scale: Optional[float] = Field( default=2.5, description="The CFG (Classifier Free Guidance) scale is a measure of how close you want the model to stick to your prompt when looking for a related image to show you.") image: Tensor = Field(..., @@ -60,6 +61,7 @@ class FluxKontextMaxMulti(BaseRequest): """ Flux Kontext Max for multiple image inputs. """ + API_PATH = "/v1/images/generations" guidance_scale: Optional[float] = Field( default=3.5, description="The CFG (Classifier Free Guidance) scale is a measure of how close you want the model to stick to your prompt when looking for a related image to show you.") images: list[Tensor] = Field(..., @@ -112,6 +114,7 @@ class FluxKontextMaxT2I(BaseRequest): """ Flux Kontext Pro text-to-image model """ + API_PATH = "/v1/images/generations" prompt: str = Field(..., description="The prompt to generate an image from.") aspect_ratio: Optional[str] = Field( diff --git a/py/modelverse_api/requests/flux_kontext_pro.py b/py/modelverse_api/requests/flux_kontext_pro.py index a8287db..21d8a96 100644 --- a/py/modelverse_api/requests/flux_kontext_pro.py +++ b/py/modelverse_api/requests/flux_kontext_pro.py @@ -8,6 +8,7 @@ class FluxKontextPro(BaseRequest): """ Flux Kontext Pro for image editing. """ + API_PATH = "/v1/images/generations" prompt: str = Field(..., description="The prompt to generate an image from.") image: Tensor = Field(..., @@ -61,6 +62,7 @@ class FluxKontextProMulti(BaseRequest): """ Flux Kontext Pro for multiple image inputs. """ + API_PATH = "/v1/images/generations" prompt: str = Field(..., description="The prompt to generate an image from.") images: list[Tensor] = Field(..., @@ -113,6 +115,7 @@ class FluxKontextProT2I(BaseRequest): """ Flux Kontext Pro text-to-image model """ + API_PATH = "/v1/images/generations" prompt: str = Field(..., description="The prompt to generate an image from.") aspect_ratio: Optional[str] = Field( diff --git a/py/modelverse_api/requests/step1x_edit.py b/py/modelverse_api/requests/step1x_edit.py index 425caa1..18ba4f8 100644 --- a/py/modelverse_api/requests/step1x_edit.py +++ b/py/modelverse_api/requests/step1x_edit.py @@ -11,6 +11,7 @@ class Step1xEdit(BaseRequest): Step1X-Edit transforms your photos with simple instructions into stunning, professional-quality edits—rivaling top proprietary tools. """ + API_PATH = "/v1/images/generations" prompt: str = Field(..., description="The prompt to generate an image from.") image: Tensor = Field( diff --git a/py/nodes.py b/py/nodes.py index c89a966..81d0cbf 100644 --- a/py/nodes.py +++ b/py/nodes.py @@ -1,5 +1,6 @@ import os import configparser +from comfy.comfy_types.node_typing import IO try: current_dir = os.path.dirname(os.path.abspath(__file__)) @@ -89,13 +90,13 @@ class ModelverseImagePacker: def INPUT_TYPES(s): return { "required": { - "images1": ("IMAGE,IMAGE_LIST", {"tooltip": "The first image/list to be packed together. Add more if you need."}) + "images1": (IO.IMAGE, {"tooltip": "The first image/list to be packed together. Add more if you need."}) }, "optional": { - "images2": ("IMAGE,IMAGE_LIST", {"default": None, "tooltip": "The second image/list to be packed together."}), - "images3": ("IMAGE,IMAGE_LIST", {"default": None, "tooltip": "The third image/list to be packed together."}), - "images4": ("IMAGE,IMAGE_LIST", {"default": None, "tooltip": "The fourth image/list to be packed together."}), - "images5": ("IMAGE,IMAGE_LIST", {"default": None, "tooltip": "The fifth image/list to be packed together."}) + "images2": (IO.IMAGE, {"default": None, "tooltip": "The second image/list to be packed together."}), + "images3": (IO.IMAGE, {"default": None, "tooltip": "The third image/list to be packed together."}), + "images4": (IO.IMAGE, {"default": None, "tooltip": "The fourth image/list to be packed together."}), + "images5": (IO.IMAGE, {"default": None, "tooltip": "The fifth image/list to be packed together."}) } } diff --git a/py/openai.py b/py/openai.py new file mode 100644 index 0000000..be87fc1 --- /dev/null +++ b/py/openai.py @@ -0,0 +1,500 @@ +import io +import os +import json +import openai +from openai import OpenAI +import base64 +import numpy as np +from PIL import Image +from typing import Optional, List, Dict, Any +from .modelverse_api.client import ModelverseClient +from comfy.comfy_types.node_typing import IO +from server import PromptServer +import folder_paths + +# Hardcoded models list +DEFAULT_MODELS = [ + 'ByteDance/doubao-1.5-thinking-vision-pro', + 'ByteDance/doubao-seed-1.6', + 'ByteDance/doubao-seed-1.6-thinking', + 'Qwen/QwQ-32B', + 'Qwen/Qwen-Image', + 'Qwen/Qwen-Image-Edit', + 'Qwen/Qwen3-235B-A22B', + 'Qwen/Qwen3-30B-A3B', + 'Qwen/Qwen3-32B', + 'Qwen/Qwen3-Coder', + 'Wan-AI/Wan2.2-I2V', + 'Wan-AI/Wan2.2-T2V', + 'baidu/ernie-4.5-turbo-128k', + 'baidu/ernie-4.5-turbo-vl-32k', + 'baidu/ernie-x1-turbo-32k', + 'black-forest-labs/flux-kontext-max', + 'black-forest-labs/flux-kontext-max/multi', + 'black-forest-labs/flux-kontext-max/text-to-image', + 'black-forest-labs/flux-kontext-pro', + 'black-forest-labs/flux-kontext-pro/multi', + 'black-forest-labs/flux-kontext-pro/text-to-image', + 'black-forest-labs/flux.1-dev', + 'claude-4-opus', + 'claude-4-sonnet', + 'deepseek-ai/DeepSeek-Prover-V2-671B', + 'deepseek-ai/DeepSeek-R1', + 'deepseek-ai/DeepSeek-R1-0528', + 'deepseek-ai/DeepSeek-R1-Distill-Llama-70B', + 'deepseek-ai/DeepSeek-V3-0324', + 'deepseek-ai/DeepSeek-V3.1', + 'gemini-2.5-flash', + 'gemini-2.5-flash-image', + 'gemini-2.5-pro', + 'gpt-4.1-mini', + 'grok-4', + 'moonshotai/Kimi-K2-Instruct', + 'openai/gpt-4.1', + 'openai/gpt-5', + 'openai/gpt-5-mini', + 'openai/gpt-oss-120b', + 'openai/gpt-oss-20b', + 'qwen/qwen2.5-vl-72b-instruct', + 'zai-org/glm-4.5', + 'zai-org/glm-4.5v', + 'stepfun-ai/step1x-edit' +] + +# Default selected model +DEFAULT_MODEL = "zai-org/glm-4.5" + + +class ModelverseChat: + """OpenAI Chat node with support for text, images, and files""" + + # Class-level cache for models list + _cached_models = None + _cache_initialized = False + + def __init__(self): + pass + + @classmethod + def get_models_list(cls): + """Try to fetch models from ModelVerse API, fallback to default models if failed""" + # Return cached models if already fetched + if cls._cache_initialized: + return cls._cached_models + + try: + # Try to create a client and fetch models + # Note: This will work if there's a valid API key available, otherwise fallback + client = OpenAI(base_url="https://api.modelverse.cn/v1",api_key="xxxsu") + models_response = client.models.list() + + # Extract model IDs from the response + if hasattr(models_response, 'data') and models_response.data: + model_ids = [] + for model in models_response.data: + if hasattr(model, 'id') and model.id: + model_ids.append(model.id) + + if model_ids: + print(f"ModelverseChat: Successfully fetched {len(model_ids)} models from API") + cls._cached_models = sorted(model_ids) # Sort for better user experience + cls._cache_initialized = True + return cls._cached_models + + # If no models found, fall back to default + print("ModelverseChat: No models found in API response, using default models") + cls._cached_models = DEFAULT_MODELS + cls._cache_initialized = True + return cls._cached_models + + except Exception as e: + print(f"ModelverseChat: Failed to fetch models from API ({str(e)}), using default models") + cls._cached_models = DEFAULT_MODELS + cls._cache_initialized = True + return cls._cached_models + + @classmethod + def clear_models_cache(cls): + """Clear the cached models list to force a refresh on next call""" + cls._cached_models = None + cls._cache_initialized = False + print("ModelverseChat: Models cache cleared") + + def display_message_on_node(self, message: str, node_id: str) -> None: + """Display the current response message on the node UI.""" + render_spec = { + "node_id": node_id, + "component": "MessageDisplayWidget", + "props": { + "message": message, + }, + } + PromptServer.instance.send_sync( + "display_component", + render_spec, + ) + + + @classmethod + def INPUT_TYPES(cls): + # Get models list dynamically + available_models = cls.get_models_list() + + return { + "required": { + "client": ("MODELVERSE_API_CLIENT",), + "model": (available_models, {"default": DEFAULT_MODEL if DEFAULT_MODEL in available_models else available_models[0] if available_models else DEFAULT_MODEL}), + "user_prompt": (IO.STRING, { + "multiline": True, + "default": "What can you tell me about this?" + }), + "temperature": (IO.FLOAT, { + "default": 0.7, + "min": 0.0, + "max": 2.0, + "step": 0.1 + }), + "max_tokens": (IO.INT, { + "default": 6000, + "min": 1, + "max": 128000 + }), + "top_p": (IO.FLOAT, { + "default": 1.0, + "min": 0.0, + "max": 1.0, + "step": 0.01 + }), + }, + "optional": { + "system_prompt": (IO.STRING, { + "multiline": True, + "default": "You are a helpful assistant." + }), + "image_in": (IO.IMAGE, {}), + "files": ("OPENAI_INPUT_FILES", { + "default": None, + "tooltip": "Optional file(s) to use as context for the model. Accepts inputs from the OpenAI Input Files node." + }), + # "response_format": (["text", "json_object"], {"default": "text"}), + "response_format": (["text"], {"default": "text"}), + "presence_penalty": (IO.FLOAT, { + "default": 0.0, + "min": -2.0, + "max": 2.0, + "step": 0.1 + }), + "frequency_penalty": (IO.FLOAT, { + "default": 0.0, + "min": -2.0, + "max": 2.0, + "step": 0.1 + }), + }, + "hidden": { + "unique_id": "UNIQUE_ID", + }, + } + + RETURN_TYPES = (IO.STRING,) + RETURN_NAMES = ("response",) + CATEGORY = "UCLOUD_MODELVERSE" + FUNCTION = "chat" + + @classmethod + def IS_CHANGED(cls, **kwargs): + """Tell ComfyUI that this node's inputs may change""" + return "static" + + def chat(self, + client: Dict[str, str], + model: str, + user_prompt: str, + temperature: float, + max_tokens: int, + top_p: float, + unique_id: Optional[str] = None, + system_prompt: Optional[str] = "You are a helpful assistant.", + image_in: Optional[Any] = None, + files: Optional[List[Any]] = None, + response_format: str = "text", + presence_penalty: float = 0.0, + frequency_penalty: float = 0.0) -> tuple: + + # Create ModelverseClient and get API key + api_key = client.get("api_key") + if not api_key: + raise ValueError("No API key found in the client") + + # Create ModelverseClient instance to get the actual API key + modelverse_client = ModelverseClient(api_key) + + # Initialize OpenAI client with the API key from ModelverseClient + openai_client = openai.OpenAI(api_key=modelverse_client.api_key,base_url="https://api.modelverse.cn/v1") + + # Build messages + messages = [] + + # Add system message if provided + if system_prompt: + messages.append({ + "role": "system", + "content": system_prompt + }) + + # Build user message content + user_content = [] + + # Add main prompt + user_content.append({ + "type": "text", + "text": user_prompt + }) + + # Add file content if provided + if files: + print(f"OpenAIChat: Processing {len(files)} files") + for file_info in files: + # Handle the actual file format from OpenAIInputFiles + if hasattr(file_info, 'file_data') and hasattr(file_info, 'filename'): + # Decode base64 content + try: + # Extract base64 data (remove data:text/plain;base64, prefix) + if file_info.file_data.startswith('data:'): + base64_data = file_info.file_data.split(',', 1)[1] + else: + base64_data = file_info.file_data + + # Decode base64 to text + content = base64.b64decode(base64_data).decode('utf-8') + + file_text = f"\n\nFile: {file_info.filename}\nContent:\n{content}" + user_content.append({ + "type": "text", + "text": file_text + }) + print(f"OpenAIChat: Added file {file_info.filename} with {len(content)} characters") + except Exception as e: + print(f"OpenAIChat: Error decoding file {file_info.filename}: {str(e)}") + elif isinstance(file_info, dict) and "content" in file_info and "filename" in file_info: + # Handle simple dict format (backup) + file_text = f"\n\nFile: {file_info['filename']}\nContent:\n{file_info['content']}" + user_content.append({ + "type": "text", + "text": file_text + }) + print(f"OpenAIChat: Added file {file_info['filename']} with {len(file_info['content'])} characters") + else: + print(f"OpenAIChat: Invalid file format: {file_info}") + else: + print("OpenAIChat: No files provided") + + # Add image if provided + if image_in is not None: + # Convert tensor to PIL Image + # Handle both single images and batches + if len(image_in.shape) == 4: + # Take first image from batch + image_in = image_in[0] + + # Convert from tensor format (H, W, C) to numpy array + image_array = image_in.cpu().numpy() + + # Ensure values are in 0-255 range + image_array = np.clip(255. * image_array, 0, 255).astype(np.uint8) + + # Create PIL Image + pil_image = Image.fromarray(image_array) + + # Convert PIL Image to base64 + buffered = io.BytesIO() + pil_image.save(buffered, format="PNG") + img_str = base64.b64encode(buffered.getvalue()).decode("utf-8") + + # Add image to content + user_content.append({ + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{img_str}" + } + }) + + # Add user message with all content + if len(user_content) == 1: + # If only text, use simple format + messages.append({ + "role": "user", + "content": user_content[0]["text"] + }) + else: + # If multiple content types, use array format + messages.append({ + "role": "user", + "content": user_content + }) + + # Prepare API call parameters + api_params = { + "model": model, + "messages": messages, + "temperature": temperature, + "max_tokens": max_tokens, + "top_p": top_p, + "presence_penalty": presence_penalty, + "frequency_penalty": frequency_penalty, + } + + # Add response format if JSON is requested + if response_format == "json_object": + api_params["response_format"] = {"type": "json_object"} + + try: + # Make API call to OpenAI + response = openai_client.chat.completions.create(**api_params) + + # Extract response content + if response.choices and len(response.choices) > 0: + content = response.choices[0].message.content + if content is None: + raise ValueError("No content in response") + + # Display the response message on the node UI + if unique_id: + self.display_message_on_node(content.strip(), unique_id) + + # Return the response as a tuple + return (content.strip(),) + else: + raise ValueError("No choices in response") + + except Exception as e: + error_msg = f"OpenAI API Error: {str(e)}" + print(error_msg) + return (error_msg,) + + +class ModelverseInputFiles: + """ + Loads and formats input files for OpenAI API. + """ + + @classmethod + def INPUT_TYPES(cls): + """ + For details about the supported file input types, see: + https://platform.openai.com/docs/guides/pdf-files?api-mode=responses + """ + input_dir = folder_paths.get_input_directory() + input_files = [ + f + for f in os.scandir(input_dir) + if f.is_file() + and (f.name.endswith(".txt") or f.name.endswith(".pdf") or f.name.endswith(".md")) + and f.stat().st_size < 32 * 1024 * 1024 + ] + input_files = sorted(input_files, key=lambda x: x.name) + input_files = [f.name for f in input_files] + return { + "required": {}, + "optional": { + "file": ( + IO.COMBO, + { + "options": input_files, + "default": input_files[0] if input_files else None, + "tooltip": "Input files to include as context for the model. Only accepts text (.txt), markdown (.md) and PDF (.pdf) files for now.", + }, + ), + "OPENAI_INPUT_FILES": ( + "OPENAI_INPUT_FILES", + { + "tooltip": "An optional additional file(s) to batch together with the file loaded from this node. Allows chaining of input files so that a single message can include multiple input files.", + "default": None, + }, + ), + }, + } + + DESCRIPTION = "Loads and prepares input files (text, markdown, pdf, etc.) to include as inputs for the OpenAI Chat Node. The files will be read by the OpenAI model when generating a response. 🛈 TIP: Can be chained together with other OpenAI Input File nodes." + RETURN_TYPES = ("OPENAI_INPUT_FILES",) + FUNCTION = "prepare_files" + CATEGORY = "UCLOUD_MODELVERSE" + + def read_file_content(self, file_path: str) -> str: + """Read content from a file.""" + try: + with open(file_path, 'r', encoding='utf-8') as f: + return f.read() + except UnicodeDecodeError: + # Try with different encoding if UTF-8 fails + try: + with open(file_path, 'r', encoding='gbk') as f: + return f.read() + except: + return f"Error: Unable to read file {file_path} - encoding issue" + except Exception as e: + return f"Error reading file {file_path}: {str(e)}" + + def text_to_data_uri(self, content: str) -> str: + """Convert text content to data URI.""" + encoded_content = base64.b64encode(content.encode('utf-8')).decode('ascii') + return f"data:text/plain;base64,{encoded_content}" + + def create_input_file_content(self, file_path: str): + """Create a file content object compatible with the reference implementation.""" + content = self.read_file_content(file_path) + + # Create a simple object that mimics the expected structure + class InputFileContent: + def __init__(self, content, filename): + self.file_data = self.text_to_data_uri(content) + self.filename = filename + self.type = "input_file" + self.file_id = None + + def text_to_data_uri(self, content: str) -> str: + encoded_content = base64.b64encode(content.encode('utf-8')).decode('ascii') + return f"data:text/plain;base64,{encoded_content}" + + return InputFileContent(content, os.path.basename(file_path)) + + def prepare_files( + self, file: str = None, OPENAI_INPUT_FILES: List[Any] = None + ) -> tuple: + """ + Loads and formats input files for OpenAI API. + """ + files = [] + + # Add current file if provided + if file is not None and file != "": + try: + file_path = folder_paths.get_annotated_filepath(file) + input_file_content = self.create_input_file_content(file_path) + files.append(input_file_content) + print(f"OpenAIInputFiles: Successfully loaded file {file}") + except Exception as e: + print(f"OpenAIInputFiles: Error loading file {file}: {str(e)}") + + # Add previous files if provided + if OPENAI_INPUT_FILES is not None: + files.extend(OPENAI_INPUT_FILES) + print(f"OpenAIInputFiles: Added {len(OPENAI_INPUT_FILES)} previous files") + + print(f"OpenAIInputFiles: Returning {len(files)} files total") + return (files,) + + + +# Node registration +NODE_CLASS_MAPPINGS = { + "ModelverseChat": ModelverseChat, + "ModelverseInputFiles": ModelverseInputFiles, + # "OpenAICaptionImage": OpenAICaptionImage, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "ModelverseChat": "Modelverse Chat", + "ModelverseInputFiles": "Modelverse Input Files", + # "OpenAICaptionImage": "OpenAI Caption Image", +} diff --git a/py/step1x_edit.py b/py/step1x_edit.py index a1dae80..3fb96aa 100644 --- a/py/step1x_edit.py +++ b/py/step1x_edit.py @@ -2,7 +2,7 @@ from .modelverse_api.utils import imageurl2tensor from .modelverse_api.client import ModelverseClient from .modelverse_api.requests.step1x_edit import Step1xEdit import torch -import asyncio +from comfy.comfy_types.node_typing import IO class Step1xEditNode: @@ -17,10 +17,10 @@ class Step1xEditNode: return { "required": { "client": ("MODELVERSE_API_CLIENT",), - "prompt": ("STRING", {"multiline": True, "default": "", "tooltip": "The prompt to guide the image edit."}), - "negative_prompt": ("STRING", {"multiline": True, "default": "", "tooltip": "The negative prompt to use."}), - "image": ("IMAGE", {"default": None, "tooltip": "The image to be edited."}), - "num_requests": ("INT", { + "prompt": (IO.STRING, {"multiline": True, "default": "", "tooltip": "The prompt to guide the image edit."}), + "negative_prompt": (IO.STRING, {"multiline": True, "default": "", "tooltip": "The negative prompt to use."}), + "image": (IO.IMAGE, {"default": None, "tooltip": "The image to be edited."}), + "num_requests": (IO.INT, { "default": 1, "min": 1, "max": 10, @@ -28,14 +28,14 @@ class Step1xEditNode: "display": "number", "tooltip": "Number of request to make (1 to 10)" }), - "seed": ("INT", { + "seed": (IO.INT, { "default": -1, "min": -1, "max": 0xffffffffffffffff, "control_after_generate": True, "tooltip": "Random seed for reproducible results. -1 for random seed" }), - "num_inference_steps": ("INT", { + "num_inference_steps": (IO.INT, { "default": 30, "min": 1, "max": 50, @@ -43,7 +43,7 @@ class Step1xEditNode: "display": "number", "tooltip": "Number of inference steps (1 to 50)" }), - "guidance_scale": ("FLOAT", { + "guidance_scale": (IO.FLOAT, { "default": 4.0, "min": 0.0, "max": 20.0, @@ -54,7 +54,7 @@ class Step1xEditNode: }, } - RETURN_TYPES = ("IMAGE", ) + RETURN_TYPES = (IO.IMAGE, ) RETURN_NAMES = ("image", ) CATEGORY = "UCLOUD_MODELVERSE" diff --git a/py/wan_ai_i2v.py b/py/wan_ai_i2v.py index 0809bfc..d0cd2d9 100644 --- a/py/wan_ai_i2v.py +++ b/py/wan_ai_i2v.py @@ -5,7 +5,7 @@ import tempfile import io from .modelverse_api.client import ModelverseClient from .modelverse_api.utils import image_to_base64 - +from comfy.comfy_types.node_typing import IO # Import VideoFromFile class for ComfyUI video handling try: # Try the newer import path first @@ -34,20 +34,20 @@ class Modelverse_WanAII2V: return { "required": { "client": ("MODELVERSE_API_CLIENT",), - "prompt": ("STRING", {"multiline": True, "default": "Convert to video","tooltip": "Text prompt to guide video generation"}), + "prompt": (IO.STRING, {"multiline": True, "default": "Convert to video","tooltip": "Text prompt to guide video generation"}), }, "optional": { - "first_frame_image": ("IMAGE",), - "first_frame_url": ("STRING", {"default": "", "tooltip": "First frame image URL (use either this OR first_frame_image, not both)"}), - "last_frame_image": ("IMAGE",), - "last_frame_url": ("STRING", {"default": "", "tooltip": "Optional: URL for the last frame of the video"}), - "negative_prompt": ("STRING", {"multiline": True, "default": "low quality, blurry","tooltip": "Negative prompt to avoid unwanted content"}), + "first_frame_image": (IO.IMAGE,), + "first_frame_url": (IO.STRING, {"default": "", "tooltip": "First frame image URL (use either this OR first_frame_image, not both)"}), + "last_frame_image": (IO.IMAGE,), + "last_frame_url": (IO.STRING, {"default": "", "tooltip": "Optional: URL for the last frame of the video"}), + "negative_prompt": (IO.STRING, {"multiline": True, "default": "low quality, blurry","tooltip": "Negative prompt to avoid unwanted content"}), "resolution": (["720P", "480P"], {"default": "720P", "tooltip": "Output video resolution"}), - "seed": ("INT", {"default": 0, "min": 0, "max": 2147483647, "tooltip": "Random seed for reproducible results"}), + "seed": (IO.INT, {"default": 0, "min": 0, "max": 2147483647, "tooltip": "Random seed for reproducible results"}), } } - RETURN_TYPES = ("VIDEO",) + RETURN_TYPES = (IO.VIDEO,) RETURN_NAMES = ("video",) FUNCTION = "generate_video" CATEGORY = "UCLOUD_MODELVERSE" diff --git a/py/wan_ai_t2v.py b/py/wan_ai_t2v.py index 8d464d0..f90bee2 100644 --- a/py/wan_ai_t2v.py +++ b/py/wan_ai_t2v.py @@ -4,7 +4,7 @@ import os import tempfile import io from .modelverse_api.client import ModelverseClient - +from comfy.comfy_types.node_typing import IO # Import VideoFromFile class for ComfyUI video handling try: # Try the newer import path first @@ -25,15 +25,15 @@ class Modelverse_WanAIT2V: return { "required": { "client": ("MODELVERSE_API_CLIENT",), - "prompt": ("STRING", {"multiline": True, "default": "a beautiful flower","tooltip": "Text prompt of the image to generate"}), - "negative_prompt": ("STRING", {"multiline": True, "default": "low quality","tooltip": "Negative prompt of the image to generate"}), + "prompt": (IO.STRING, {"multiline": True, "default": "a beautiful flower","tooltip": "Text prompt of the image to generate"}), + "negative_prompt": (IO.STRING, {"multiline": True, "default": "low quality","tooltip": "Negative prompt of the image to generate"}), "resolution": (["720P", "480P"], {"default": "720P"}), "size": (["1280x720", "720x1280", "832x480", "480x832"], {"default": "1280x720"}), - "seed": ("INT", {"default": 0, "min": 0, "max": 2147483647}), + "seed": (IO.INT, {"default": 0, "min": 0, "max": 2147483647}), } } - RETURN_TYPES = ("VIDEO",) + RETURN_TYPES = (IO.VIDEO,) RETURN_NAMES = ("video",) FUNCTION = "generate_video" CATEGORY = "UCLOUD_MODELVERSE" diff --git a/requirements.txt b/requirements.txt index d6d2140..2cbf9ad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ requests >= 2.32.4 pillow >= 11 +openai >= 1.107.0 \ No newline at end of file