nodes/ucloud-comfyui-plugin
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

support text generation

Browse Source 
* add gitignore

* add video api

* fix: export to video

* fix

* fix

* fix

* fix

* fix

* add i2v

* fix

* fix

* add openai

* fix: add base url

* change default models

* fix: add base url

* add v1

* fix models

* fix models

* fix models

* fix models

* fix models

* fix bug

* fix

* try to fix

* use IO.STRING test

* image fix

* add chat content

* add files

* fix optional

* fix file read

* fix file read

* change name

* add get models

* add unused apikey

* change model sort

* fix IMAGE_LIST

* fix: change to image

* add model cache

* remove json_object

* change max_tokens to 65536

* change max_tokens to 6000

---------

Co-authored-by: jason.mei <jason.mei@ucloud.cn>
This commit is contained in:
jason
2025-09-15 15:31:26 +08:00
committed by GitHub
co-authored by jason.mei
parent 3492952986
commit 681f6ebc5f
15 changed files with 573 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
py/flux_dev.py
+11 -11
View File
@@ -2,7 +2,7 @@ from .modelverse_api.utils import imageurl2tensor
from .modelverse_api.client import ModelverseClient
from .modelverse_api.requests.flux_dev import FluxDev
import torch
import asyncio
from comfy.comfy_types.node_typing import IO
class FluxDevNode:
@@ -17,8 +17,8 @@ class FluxDevNode:
return {
"required": {
"client": ("MODELVERSE_API_CLIENT",),
"prompt": ("STRING", {"multiline": True, "default": "", "tooltip": "Text description of the image to generate"}),
"width": ("INT", {
"prompt": (IO.STRING, {"multiline": True, "default": "", "tooltip": "Text description of the image to generate"}),
"width": (IO.INT, {
"default": 1024,
"min": 512,
"max": 1536,
@@ -26,7 +26,7 @@ class FluxDevNode:
"display": "number",
"tooltip": "Image width (512 to 1536)"
}),
"height": ("INT", {
"height": (IO.INT, {
"default": 1024,
"min": 512,
"max": 1536,
@@ -34,7 +34,7 @@ class FluxDevNode:
"display": "number",
"tooltip": "Image height (512 to 1536)"
}),
"strength": ("FLOAT", {
"strength": (IO.FLOAT, {
"default": 0.8,
"min": 0.01,
"max": 1.0,
@@ -42,14 +42,14 @@ class FluxDevNode:
"display": "number",
"tooltip": "Strength of the image-to-image transformation (0.01 to 1.0)"
}),
"seed": ("INT", {
"seed": (IO.INT, {
"default": -1,
"min": -1,
"max": 0xffffffffffffffff,
"control_after_generate": True,
"tooltip": "Random seed for reproducible results. -1 for random seed"
}),
"num_images": ("INT", {
"num_images": (IO.INT, {
"default": 1,
"min": 1,
"max": 4,
@@ -57,7 +57,7 @@ class FluxDevNode:
"display": "number",
"tooltip": "Number of images to generate in a single request (1 to 4)"
}),
"num_requests": ("INT", {
"num_requests": (IO.INT, {
"default": 1,
"min": 1,
"max": 10,
@@ -65,7 +65,7 @@ class FluxDevNode:
"display": "number",
"tooltip": "Number of request to make (1 to 10)"
}),
"num_inference_steps": ("INT", {
"num_inference_steps": (IO.INT, {
"default": 28,
"min": 1,
"max": 50,
@@ -73,7 +73,7 @@ class FluxDevNode:
"display": "number",
"tooltip": "Number of inference steps (1 to 50)"
}),
"guidance_scale": ("FLOAT", {
"guidance_scale": (IO.FLOAT, {
"default": 3.5,
"min": 0.0,
"max": 10.0,
@@ -83,7 +83,7 @@ class FluxDevNode:
}),
},
"optional": {
"image": ("IMAGE", {
"image": (IO.IMAGE, {
"tooltip": "The image for reference.",
"forceInput": False,
"default": None
py/flux_kontext_max.py
+6 -6
View File
@@ -2,7 +2,7 @@ from .modelverse_api.utils import imageurl2tensor
from .modelverse_api.client import ModelverseClient
from .modelverse_api.requests.flux_kontext_max import FluxKontextMax, FluxKontextMaxMulti
import torch
import asyncio
from comfy.comfy_types.node_typing import IO
class FluxKontextMaxNode:
@@ -21,13 +21,13 @@ class FluxKontextMaxNode:
return {
"required": {
"client": ("MODELVERSE_API_CLIENT",),
"prompt": ("STRING", {"multiline": True, "default": "", "tooltip": "Text description of the image to generate"}),
"images": ("IMAGE,IMAGE_LIST", {
"prompt": (IO.STRING, {"multiline": True, "default": "", "tooltip": "Text description of the image to generate"}),
"images": ("IMAGE_LIST", {
"tooltip": "The image(s) to edit from. If not included, use text-to-image mode.",
"forceInput": False,
"default": None
}),
"num_requests": ("INT", {
"num_requests": (IO.INT, {
"default": 1,
"min": 1,
"max": 10,
@@ -35,14 +35,14 @@ class FluxKontextMaxNode:
"display": "number",
"tooltip": "Number of request to make (1 to 10)"
}),
"seed": ("INT", {
"seed": (IO.INT, {
"default": -1,
"min": -1,
"max": 0xffffffffffffffff,
"control_after_generate": True,
"tooltip": "Random seed for reproducible results. -1 for random seed"
}),
"guidance_scale": ("FLOAT", {
"guidance_scale": (IO.FLOAT, {
"default": 2.5,
"min": 0.0,
"max": 10.0,
py/flux_kontext_max_t2i.py
+6 -7
View File
@@ -2,8 +2,7 @@ from .modelverse_api.utils import imageurl2tensor
from .modelverse_api.client import ModelverseClient
from .modelverse_api.requests.flux_kontext_max import FluxKontextMaxT2I
import torch
import asyncio
from comfy.comfy_types.node_typing import IO
class FluxKontextMaxT2INode:
"""
@@ -18,12 +17,12 @@ class FluxKontextMaxT2INode:
return {
"required": {
"client": ("MODELVERSE_API_CLIENT",),
"prompt": ("STRING", {"multiline": True, "default": "", "tooltip": "Text description of the image to generate"}),
"prompt": (IO.STRING, {"multiline": True, "default": "", "tooltip": "Text description of the image to generate"}),
"aspect_ratio": (["21:9", "16:9", "16:10", "4:3", "1:1", "3:4", "10:16", "9:16", "9:21"], {
"default": "1:1",
"tooltip": "The aspect ratio of the output image, ranging from \"21:9\" to \"9:21\", default is \"1:1\""
}),
"num_images": ("INT", {
"num_images": (IO.INT, {
"default": 1,
"min": 1,
"max": 4,
@@ -31,7 +30,7 @@ class FluxKontextMaxT2INode:
"display": "number",
"tooltip": "Number of images to generate in a single request (1 to 4)"
}),
"num_requests": ("INT", {
"num_requests": (IO.INT, {
"default": 1,
"min": 1,
"max": 10,
@@ -39,14 +38,14 @@ class FluxKontextMaxT2INode:
"display": "number",
"tooltip": "Number of request to make (1 to 10)"
}),
"seed": ("INT", {
"seed": (IO.INT, {
"default": -1,
"min": -1,
"max": 0xffffffffffffffff,
"control_after_generate": True,
"tooltip": "Random seed for reproducible results. -1 for random seed"
}),
"guidance_scale": ("FLOAT", {
"guidance_scale": (IO.FLOAT, {
"default": 2.5,
"min": 0.0,
"max": 10.0,
py/flux_kontext_pro.py
+6 -6
View File
@@ -2,7 +2,7 @@ from .modelverse_api.utils import imageurl2tensor
from .modelverse_api.client import ModelverseClient
from .modelverse_api.requests.flux_kontext_pro import FluxKontextPro, FluxKontextProMulti
import torch
import asyncio
from comfy.comfy_types.node_typing import IO
class FluxKontextProNode:
@@ -21,13 +21,13 @@ class FluxKontextProNode:
return {
"required": {
"client": ("MODELVERSE_API_CLIENT",),
"prompt": ("STRING", {"multiline": True, "default": "", "tooltip": "Text description of the image to generate"}),
"images": ("IMAGE,IMAGE_LIST", {
"prompt": (IO.STRING, {"multiline": True, "default": "", "tooltip": "Text description of the image to generate"}),
"images": ("IMAGE_LIST", {
"tooltip": "The image(s) to edit from. If not included, use text-to-image mode.",
"forceInput": False,
"default": None
}),
"num_requests": ("INT", {
"num_requests": (IO.INT, {
"default": 1,
"min": 1,
"max": 10,
@@ -35,14 +35,14 @@ class FluxKontextProNode:
"display": "number",
"tooltip": "Number of request to make (1 to 10)"
}),
"seed": ("INT", {
"seed": (IO.INT, {
"default": -1,
"min": -1,
"max": 0xffffffffffffffff,
"control_after_generate": True,
"tooltip": "Random seed for reproducible results. -1 for random seed"
}),
"guidance_scale": ("FLOAT", {
"guidance_scale": (IO.FLOAT, {
"default": 2.5,
"min": 0.0,
"max": 10.0,
py/flux_kontext_pro_t2i.py
+6 -6
View File
@@ -2,7 +2,7 @@ from .modelverse_api.utils import imageurl2tensor
from .modelverse_api.client import ModelverseClient
from .modelverse_api.requests.flux_kontext_pro import FluxKontextProT2I
import torch
import asyncio
from comfy.comfy_types.node_typing import IO
class FluxKontextProT2INode:
@@ -18,12 +18,12 @@ class FluxKontextProT2INode:
return {
"required": {
"client": ("MODELVERSE_API_CLIENT",),
"prompt": ("STRING", {"multiline": True, "default": "", "tooltip": "Text description of the image to generate"}),
"prompt": (IO.STRING, {"multiline": True, "default": "", "tooltip": "Text description of the image to generate"}),
"aspect_ratio": (["21:9", "16:9", "16:10", "4:3", "1:1", "3:4", "10:16", "9:16", "9:21"], {
"default": "1:1",
"tooltip": "The aspect ratio of the output image, ranging from \"21:9\" to \"9:21\", default is \"1:1\""
}),
"num_images": ("INT", {
"num_images": (IO.INT, {
"default": 1,
"min": 1,
"max": 4,
@@ -31,7 +31,7 @@ class FluxKontextProT2INode:
"display": "number",
"tooltip": "Number of images to generate in a single request (1 to 4)"
}),
"num_requests": ("INT", {
"num_requests": (IO.INT, {
"default": 1,
"min": 1,
"max": 10,
@@ -39,14 +39,14 @@ class FluxKontextProT2INode:
"display": "number",
"tooltip": "Number of request to make (1 to 10)"
}),
"seed": ("INT", {
"seed": (IO.INT, {
"default": -1,
"min": -1,
"max": 0xffffffffffffffff,
"control_after_generate": True,
"tooltip": "Random seed for reproducible results. -1 for random seed"
}),
"guidance_scale": ("FLOAT", {
"guidance_scale": (IO.FLOAT, {
"default": 2.5,
"min": 0.0,
"max": 10.0,
py/modelverse_api/requests/flux_dev.py
+1
View File
@@ -8,6 +8,7 @@ class FluxDev(BaseRequest):
"""
Flux-dev text to image model, 12 billion parameter rectified flow transformer
"""
API_PATH = "/v1/images/generations"
guidance_scale: Optional[float] = Field(
default=2.5, description="The CFG (Classifier Free Guidance) scale is a measure of how close you want the model to stick to your prompt when looking for a related image to show you.")
num_images: Optional[int] = Field(
py/modelverse_api/requests/flux_kontext_max.py
+3
View File
@@ -8,6 +8,7 @@ class FluxKontextMax(BaseRequest):
"""
Flux Kontext Max for image editing.
"""
API_PATH = "/v1/images/generations"
guidance_scale: Optional[float] = Field(
default=2.5, description="The CFG (Classifier Free Guidance) scale is a measure of how close you want the model to stick to your prompt when looking for a related image to show you.")
image: Tensor = Field(...,
@@ -60,6 +61,7 @@ class FluxKontextMaxMulti(BaseRequest):
"""
Flux Kontext Max for multiple image inputs.
"""
API_PATH = "/v1/images/generations"
guidance_scale: Optional[float] = Field(
default=3.5, description="The CFG (Classifier Free Guidance) scale is a measure of how close you want the model to stick to your prompt when looking for a related image to show you.")
images: list[Tensor] = Field(...,
@@ -112,6 +114,7 @@ class FluxKontextMaxT2I(BaseRequest):
"""
Flux Kontext Pro text-to-image model
"""
API_PATH = "/v1/images/generations"
prompt: str = Field(...,
description="The prompt to generate an image from.")
aspect_ratio: Optional[str] = Field(
py/modelverse_api/requests/flux_kontext_pro.py
+3
View File
@@ -8,6 +8,7 @@ class FluxKontextPro(BaseRequest):
"""
Flux Kontext Pro for image editing.
"""
API_PATH = "/v1/images/generations"
prompt: str = Field(...,
description="The prompt to generate an image from.")
image: Tensor = Field(...,
@@ -61,6 +62,7 @@ class FluxKontextProMulti(BaseRequest):
"""
Flux Kontext Pro for multiple image inputs.
"""
API_PATH = "/v1/images/generations"
prompt: str = Field(...,
description="The prompt to generate an image from.")
images: list[Tensor] = Field(...,
@@ -113,6 +115,7 @@ class FluxKontextProT2I(BaseRequest):
"""
Flux Kontext Pro text-to-image model
"""
API_PATH = "/v1/images/generations"
prompt: str = Field(...,
description="The prompt to generate an image from.")
aspect_ratio: Optional[str] = Field(
py/modelverse_api/requests/step1x_edit.py
+1
View File
@@ -11,6 +11,7 @@ class Step1xEdit(BaseRequest):
Step1X-Edit transforms your photos with simple instructions into stunning, professional-quality edits—rivaling top proprietary tools.
"""
API_PATH = "/v1/images/generations"
prompt: str = Field(...,
description="The prompt to generate an image from.")
image: Tensor = Field(
py/nodes.py
+6 -5
View File
@@ -1,5 +1,6 @@
import os
import configparser
from comfy.comfy_types.node_typing import IO
try:
current_dir = os.path.dirname(os.path.abspath(__file__))
@@ -89,13 +90,13 @@ class ModelverseImagePacker:
def INPUT_TYPES(s):
return {
"required": {
"images1": ("IMAGE,IMAGE_LIST", {"tooltip": "The first image/list to be packed together. Add more if you need."})
"images1": (IO.IMAGE, {"tooltip": "The first image/list to be packed together. Add more if you need."})
},
"optional": {
"images2": ("IMAGE,IMAGE_LIST", {"default": None, "tooltip": "The second image/list to be packed together."}),
"images3": ("IMAGE,IMAGE_LIST", {"default": None, "tooltip": "The third image/list to be packed together."}),
"images4": ("IMAGE,IMAGE_LIST", {"default": None, "tooltip": "The fourth image/list to be packed together."}),
"images5": ("IMAGE,IMAGE_LIST", {"default": None, "tooltip": "The fifth image/list to be packed together."})
"images2": (IO.IMAGE, {"default": None, "tooltip": "The second image/list to be packed together."}),
"images3": (IO.IMAGE, {"default": None, "tooltip": "The third image/list to be packed together."}),
"images4": (IO.IMAGE, {"default": None, "tooltip": "The fourth image/list to be packed together."}),
"images5": (IO.IMAGE, {"default": None, "tooltip": "The fifth image/list to be packed together."})
}
}
py/openai.py
+500
View File
@@ -0,0 +1,500 @@
import io
import os
import json
import openai
from openai import OpenAI
import base64
import numpy as np
from PIL import Image
from typing import Optional, List, Dict, Any
from .modelverse_api.client import ModelverseClient
from comfy.comfy_types.node_typing import IO
from server import PromptServer
import folder_paths
# Hardcoded models list
DEFAULT_MODELS = [
'ByteDance/doubao-1.5-thinking-vision-pro',
'ByteDance/doubao-seed-1.6',
'ByteDance/doubao-seed-1.6-thinking',
'Qwen/QwQ-32B',
'Qwen/Qwen-Image',
'Qwen/Qwen-Image-Edit',
'Qwen/Qwen3-235B-A22B',
'Qwen/Qwen3-30B-A3B',
'Qwen/Qwen3-32B',
'Qwen/Qwen3-Coder',
'Wan-AI/Wan2.2-I2V',
'Wan-AI/Wan2.2-T2V',
'baidu/ernie-4.5-turbo-128k',
'baidu/ernie-4.5-turbo-vl-32k',
'baidu/ernie-x1-turbo-32k',
'black-forest-labs/flux-kontext-max',
'black-forest-labs/flux-kontext-max/multi',
'black-forest-labs/flux-kontext-max/text-to-image',
'black-forest-labs/flux-kontext-pro',
'black-forest-labs/flux-kontext-pro/multi',
'black-forest-labs/flux-kontext-pro/text-to-image',
'black-forest-labs/flux.1-dev',
'claude-4-opus',
'claude-4-sonnet',
'deepseek-ai/DeepSeek-Prover-V2-671B',
'deepseek-ai/DeepSeek-R1',
'deepseek-ai/DeepSeek-R1-0528',
'deepseek-ai/DeepSeek-R1-Distill-Llama-70B',
'deepseek-ai/DeepSeek-V3-0324',
'deepseek-ai/DeepSeek-V3.1',
'gemini-2.5-flash',
'gemini-2.5-flash-image',
'gemini-2.5-pro',
'gpt-4.1-mini',
'grok-4',
'moonshotai/Kimi-K2-Instruct',
'openai/gpt-4.1',
'openai/gpt-5',
'openai/gpt-5-mini',
'openai/gpt-oss-120b',
'openai/gpt-oss-20b',
'qwen/qwen2.5-vl-72b-instruct',
'zai-org/glm-4.5',
'zai-org/glm-4.5v',
'stepfun-ai/step1x-edit'
]
# Default selected model
DEFAULT_MODEL = "zai-org/glm-4.5"
class ModelverseChat:
"""OpenAI Chat node with support for text, images, and files"""
# Class-level cache for models list
_cached_models = None
_cache_initialized = False
def __init__(self):
pass
@classmethod
def get_models_list(cls):
"""Try to fetch models from ModelVerse API, fallback to default models if failed"""
# Return cached models if already fetched
if cls._cache_initialized:
return cls._cached_models
try:
# Try to create a client and fetch models
# Note: This will work if there's a valid API key available, otherwise fallback
client = OpenAI(base_url="https://api.modelverse.cn/v1",api_key="xxxsu")
models_response = client.models.list()
# Extract model IDs from the response
if hasattr(models_response, 'data') and models_response.data:
model_ids = []
for model in models_response.data:
if hasattr(model, 'id') and model.id:
model_ids.append(model.id)
if model_ids:
print(f"ModelverseChat: Successfully fetched {len(model_ids)} models from API")
cls._cached_models = sorted(model_ids) # Sort for better user experience
cls._cache_initialized = True
return cls._cached_models
# If no models found, fall back to default
print("ModelverseChat: No models found in API response, using default models")
cls._cached_models = DEFAULT_MODELS
cls._cache_initialized = True
return cls._cached_models
except Exception as e:
print(f"ModelverseChat: Failed to fetch models from API ({str(e)}), using default models")
cls._cached_models = DEFAULT_MODELS
cls._cache_initialized = True
return cls._cached_models
@classmethod
def clear_models_cache(cls):
"""Clear the cached models list to force a refresh on next call"""
cls._cached_models = None
cls._cache_initialized = False
print("ModelverseChat: Models cache cleared")
def display_message_on_node(self, message: str, node_id: str) -> None:
"""Display the current response message on the node UI."""
render_spec = {
"node_id": node_id,
"component": "MessageDisplayWidget",
"props": {
"message": message,
},
}
PromptServer.instance.send_sync(
"display_component",
render_spec,
)
@classmethod
def INPUT_TYPES(cls):
# Get models list dynamically
available_models = cls.get_models_list()
return {
"required": {
"client": ("MODELVERSE_API_CLIENT",),
"model": (available_models, {"default": DEFAULT_MODEL if DEFAULT_MODEL in available_models else available_models[0] if available_models else DEFAULT_MODEL}),
"user_prompt": (IO.STRING, {
"multiline": True,
"default": "What can you tell me about this?"
}),
"temperature": (IO.FLOAT, {
"default": 0.7,
"min": 0.0,
"max": 2.0,
"step": 0.1
}),
"max_tokens": (IO.INT, {
"default": 6000,
"min": 1,
"max": 128000
}),
"top_p": (IO.FLOAT, {
"default": 1.0,
"min": 0.0,
"max": 1.0,
"step": 0.01
}),
},
"optional": {
"system_prompt": (IO.STRING, {
"multiline": True,
"default": "You are a helpful assistant."
}),
"image_in": (IO.IMAGE, {}),
"files": ("OPENAI_INPUT_FILES", {
"default": None,
"tooltip": "Optional file(s) to use as context for the model. Accepts inputs from the OpenAI Input Files node."
}),
# "response_format": (["text", "json_object"], {"default": "text"}),
"response_format": (["text"], {"default": "text"}),
"presence_penalty": (IO.FLOAT, {
"default": 0.0,
"min": -2.0,
"max": 2.0,
"step": 0.1
}),
"frequency_penalty": (IO.FLOAT, {
"default": 0.0,
"min": -2.0,
"max": 2.0,
"step": 0.1
}),
},
"hidden": {
"unique_id": "UNIQUE_ID",
},
}
RETURN_TYPES = (IO.STRING,)
RETURN_NAMES = ("response",)
CATEGORY = "UCLOUD_MODELVERSE"
FUNCTION = "chat"
@classmethod
def IS_CHANGED(cls, **kwargs):
"""Tell ComfyUI that this node's inputs may change"""
return "static"
def chat(self,
client: Dict[str, str],
model: str,
user_prompt: str,
temperature: float,
max_tokens: int,
top_p: float,
unique_id: Optional[str] = None,
system_prompt: Optional[str] = "You are a helpful assistant.",
image_in: Optional[Any] = None,
files: Optional[List[Any]] = None,
response_format: str = "text",
presence_penalty: float = 0.0,
frequency_penalty: float = 0.0) -> tuple:
# Create ModelverseClient and get API key
api_key = client.get("api_key")
if not api_key:
raise ValueError("No API key found in the client")
# Create ModelverseClient instance to get the actual API key
modelverse_client = ModelverseClient(api_key)
# Initialize OpenAI client with the API key from ModelverseClient
openai_client = openai.OpenAI(api_key=modelverse_client.api_key,base_url="https://api.modelverse.cn/v1")
# Build messages
messages = []
# Add system message if provided
if system_prompt:
messages.append({
"role": "system",
"content": system_prompt
})
# Build user message content
user_content = []
# Add main prompt
user_content.append({
"type": "text",
"text": user_prompt
})
# Add file content if provided
if files:
print(f"OpenAIChat: Processing {len(files)} files")
for file_info in files:
# Handle the actual file format from OpenAIInputFiles
if hasattr(file_info, 'file_data') and hasattr(file_info, 'filename'):
# Decode base64 content
try:
# Extract base64 data (remove data:text/plain;base64, prefix)
if file_info.file_data.startswith('data:'):
base64_data = file_info.file_data.split(',', 1)[1]
else:
base64_data = file_info.file_data
# Decode base64 to text
content = base64.b64decode(base64_data).decode('utf-8')
file_text = f"\n\nFile: {file_info.filename}\nContent:\n{content}"
user_content.append({
"type": "text",
"text": file_text
})
print(f"OpenAIChat: Added file {file_info.filename} with {len(content)} characters")
except Exception as e:
print(f"OpenAIChat: Error decoding file {file_info.filename}: {str(e)}")
elif isinstance(file_info, dict) and "content" in file_info and "filename" in file_info:
# Handle simple dict format (backup)
file_text = f"\n\nFile: {file_info['filename']}\nContent:\n{file_info['content']}"
user_content.append({
"type": "text",
"text": file_text
})
print(f"OpenAIChat: Added file {file_info['filename']} with {len(file_info['content'])} characters")
else:
print(f"OpenAIChat: Invalid file format: {file_info}")
else:
print("OpenAIChat: No files provided")
# Add image if provided
if image_in is not None:
# Convert tensor to PIL Image
# Handle both single images and batches
if len(image_in.shape) == 4:
# Take first image from batch
image_in = image_in[0]
# Convert from tensor format (H, W, C) to numpy array
image_array = image_in.cpu().numpy()
# Ensure values are in 0-255 range
image_array = np.clip(255. * image_array, 0, 255).astype(np.uint8)
# Create PIL Image
pil_image = Image.fromarray(image_array)
# Convert PIL Image to base64
buffered = io.BytesIO()
pil_image.save(buffered, format="PNG")
img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
# Add image to content
user_content.append({
"type": "image_url",
"image_url": {
"url": f"data:image/png;base64,{img_str}"
}
})
# Add user message with all content
if len(user_content) == 1:
# If only text, use simple format
messages.append({
"role": "user",
"content": user_content[0]["text"]
})
else:
# If multiple content types, use array format
messages.append({
"role": "user",
"content": user_content
})
# Prepare API call parameters
api_params = {
"model": model,
"messages": messages,
"temperature": temperature,
"max_tokens": max_tokens,
"top_p": top_p,
"presence_penalty": presence_penalty,
"frequency_penalty": frequency_penalty,
}
# Add response format if JSON is requested
if response_format == "json_object":
api_params["response_format"] = {"type": "json_object"}
try:
# Make API call to OpenAI
response = openai_client.chat.completions.create(**api_params)
# Extract response content
if response.choices and len(response.choices) > 0:
content = response.choices[0].message.content
if content is None:
raise ValueError("No content in response")
# Display the response message on the node UI
if unique_id:
self.display_message_on_node(content.strip(), unique_id)
# Return the response as a tuple
return (content.strip(),)
else:
raise ValueError("No choices in response")
except Exception as e:
error_msg = f"OpenAI API Error: {str(e)}"
print(error_msg)
return (error_msg,)
class ModelverseInputFiles:
"""
Loads and formats input files for OpenAI API.
"""
@classmethod
def INPUT_TYPES(cls):
"""
For details about the supported file input types, see:
https://platform.openai.com/docs/guides/pdf-files?api-mode=responses
"""
input_dir = folder_paths.get_input_directory()
input_files = [
f
for f in os.scandir(input_dir)
if f.is_file()
and (f.name.endswith(".txt") or f.name.endswith(".pdf") or f.name.endswith(".md"))
and f.stat().st_size < 32 * 1024 * 1024
]
input_files = sorted(input_files, key=lambda x: x.name)
input_files = [f.name for f in input_files]
return {
"required": {},
"optional": {
"file": (
IO.COMBO,
{
"options": input_files,
"default": input_files[0] if input_files else None,
"tooltip": "Input files to include as context for the model. Only accepts text (.txt), markdown (.md) and PDF (.pdf) files for now.",
},
),
"OPENAI_INPUT_FILES": (
"OPENAI_INPUT_FILES",
{
"tooltip": "An optional additional file(s) to batch together with the file loaded from this node. Allows chaining of input files so that a single message can include multiple input files.",
"default": None,
},
),
},
}
DESCRIPTION = "Loads and prepares input files (text, markdown, pdf, etc.) to include as inputs for the OpenAI Chat Node. The files will be read by the OpenAI model when generating a response. 🛈 TIP: Can be chained together with other OpenAI Input File nodes."
RETURN_TYPES = ("OPENAI_INPUT_FILES",)
FUNCTION = "prepare_files"
CATEGORY = "UCLOUD_MODELVERSE"
def read_file_content(self, file_path: str) -> str:
"""Read content from a file."""
try:
with open(file_path, 'r', encoding='utf-8') as f:
return f.read()
except UnicodeDecodeError:
# Try with different encoding if UTF-8 fails
try:
with open(file_path, 'r', encoding='gbk') as f:
return f.read()
except:
return f"Error: Unable to read file {file_path} - encoding issue"
except Exception as e:
return f"Error reading file {file_path}: {str(e)}"
def text_to_data_uri(self, content: str) -> str:
"""Convert text content to data URI."""
encoded_content = base64.b64encode(content.encode('utf-8')).decode('ascii')
return f"data:text/plain;base64,{encoded_content}"
def create_input_file_content(self, file_path: str):
"""Create a file content object compatible with the reference implementation."""
content = self.read_file_content(file_path)
# Create a simple object that mimics the expected structure
class InputFileContent:
def __init__(self, content, filename):
self.file_data = self.text_to_data_uri(content)
self.filename = filename
self.type = "input_file"
self.file_id = None
def text_to_data_uri(self, content: str) -> str:
encoded_content = base64.b64encode(content.encode('utf-8')).decode('ascii')
return f"data:text/plain;base64,{encoded_content}"
return InputFileContent(content, os.path.basename(file_path))
def prepare_files(
self, file: str = None, OPENAI_INPUT_FILES: List[Any] = None
) -> tuple:
"""
Loads and formats input files for OpenAI API.
"""
files = []
# Add current file if provided
if file is not None and file != "":
try:
file_path = folder_paths.get_annotated_filepath(file)
input_file_content = self.create_input_file_content(file_path)
files.append(input_file_content)
print(f"OpenAIInputFiles: Successfully loaded file {file}")
except Exception as e:
print(f"OpenAIInputFiles: Error loading file {file}: {str(e)}")
# Add previous files if provided
if OPENAI_INPUT_FILES is not None:
files.extend(OPENAI_INPUT_FILES)
print(f"OpenAIInputFiles: Added {len(OPENAI_INPUT_FILES)} previous files")
print(f"OpenAIInputFiles: Returning {len(files)} files total")
return (files,)
# Node registration
NODE_CLASS_MAPPINGS = {
"ModelverseChat": ModelverseChat,
"ModelverseInputFiles": ModelverseInputFiles,
# "OpenAICaptionImage": OpenAICaptionImage,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"ModelverseChat": "Modelverse Chat",
"ModelverseInputFiles": "Modelverse Input Files",
# "OpenAICaptionImage": "OpenAI Caption Image",
}
py/step1x_edit.py
+9 -9
View File
@@ -2,7 +2,7 @@ from .modelverse_api.utils import imageurl2tensor
from .modelverse_api.client import ModelverseClient
from .modelverse_api.requests.step1x_edit import Step1xEdit
import torch
import asyncio
from comfy.comfy_types.node_typing import IO
class Step1xEditNode:
@@ -17,10 +17,10 @@ class Step1xEditNode:
return {
"required": {
"client": ("MODELVERSE_API_CLIENT",),
"prompt": ("STRING", {"multiline": True, "default": "", "tooltip": "The prompt to guide the image edit."}),
"negative_prompt": ("STRING", {"multiline": True, "default": "", "tooltip": "The negative prompt to use."}),
"image": ("IMAGE", {"default": None, "tooltip": "The image to be edited."}),
"num_requests": ("INT", {
"prompt": (IO.STRING, {"multiline": True, "default": "", "tooltip": "The prompt to guide the image edit."}),
"negative_prompt": (IO.STRING, {"multiline": True, "default": "", "tooltip": "The negative prompt to use."}),
"image": (IO.IMAGE, {"default": None, "tooltip": "The image to be edited."}),
"num_requests": (IO.INT, {
"default": 1,
"min": 1,
"max": 10,
@@ -28,14 +28,14 @@ class Step1xEditNode:
"display": "number",
"tooltip": "Number of request to make (1 to 10)"
}),
"seed": ("INT", {
"seed": (IO.INT, {
"default": -1,
"min": -1,
"max": 0xffffffffffffffff,
"control_after_generate": True,
"tooltip": "Random seed for reproducible results. -1 for random seed"
}),
"num_inference_steps": ("INT", {
"num_inference_steps": (IO.INT, {
"default": 30,
"min": 1,
"max": 50,
@@ -43,7 +43,7 @@ class Step1xEditNode:
"display": "number",
"tooltip": "Number of inference steps (1 to 50)"
}),
"guidance_scale": ("FLOAT", {
"guidance_scale": (IO.FLOAT, {
"default": 4.0,
"min": 0.0,
"max": 20.0,
@@ -54,7 +54,7 @@ class Step1xEditNode:
},
}
RETURN_TYPES = ("IMAGE", )
RETURN_TYPES = (IO.IMAGE, )
RETURN_NAMES = ("image", )
CATEGORY = "UCLOUD_MODELVERSE"
py/wan_ai_i2v.py
+9 -9
View File
@@ -5,7 +5,7 @@ import tempfile
import io
from .modelverse_api.client import ModelverseClient
from .modelverse_api.utils import image_to_base64
from comfy.comfy_types.node_typing import IO
# Import VideoFromFile class for ComfyUI video handling
try:
# Try the newer import path first
@@ -34,20 +34,20 @@ class Modelverse_WanAII2V:
return {
"required": {
"client": ("MODELVERSE_API_CLIENT",),
"prompt": ("STRING", {"multiline": True, "default": "Convert to video","tooltip": "Text prompt to guide video generation"}),
"prompt": (IO.STRING, {"multiline": True, "default": "Convert to video","tooltip": "Text prompt to guide video generation"}),
},
"optional": {
"first_frame_image": ("IMAGE",),
"first_frame_url": ("STRING", {"default": "", "tooltip": "First frame image URL (use either this OR first_frame_image, not both)"}),
"last_frame_image": ("IMAGE",),
"last_frame_url": ("STRING", {"default": "", "tooltip": "Optional: URL for the last frame of the video"}),
"negative_prompt": ("STRING", {"multiline": True, "default": "low quality, blurry","tooltip": "Negative prompt to avoid unwanted content"}),
"first_frame_image": (IO.IMAGE,),
"first_frame_url": (IO.STRING, {"default": "", "tooltip": "First frame image URL (use either this OR first_frame_image, not both)"}),
"last_frame_image": (IO.IMAGE,),
"last_frame_url": (IO.STRING, {"default": "", "tooltip": "Optional: URL for the last frame of the video"}),
"negative_prompt": (IO.STRING, {"multiline": True, "default": "low quality, blurry","tooltip": "Negative prompt to avoid unwanted content"}),
"resolution": (["720P", "480P"], {"default": "720P", "tooltip": "Output video resolution"}),
"seed": ("INT", {"default": 0, "min": 0, "max": 2147483647, "tooltip": "Random seed for reproducible results"}),
"seed": (IO.INT, {"default": 0, "min": 0, "max": 2147483647, "tooltip": "Random seed for reproducible results"}),
}
}
RETURN_TYPES = ("VIDEO",)
RETURN_TYPES = (IO.VIDEO,)
RETURN_NAMES = ("video",)
FUNCTION = "generate_video"
CATEGORY = "UCLOUD_MODELVERSE"
py/wan_ai_t2v.py
+5 -5
View File
@@ -4,7 +4,7 @@ import os
import tempfile
import io
from .modelverse_api.client import ModelverseClient
from comfy.comfy_types.node_typing import IO
# Import VideoFromFile class for ComfyUI video handling
try:
# Try the newer import path first
@@ -25,15 +25,15 @@ class Modelverse_WanAIT2V:
return {
"required": {
"client": ("MODELVERSE_API_CLIENT",),
"prompt": ("STRING", {"multiline": True, "default": "a beautiful flower","tooltip": "Text prompt of the image to generate"}),
"negative_prompt": ("STRING", {"multiline": True, "default": "low quality","tooltip": "Negative prompt of the image to generate"}),
"prompt": (IO.STRING, {"multiline": True, "default": "a beautiful flower","tooltip": "Text prompt of the image to generate"}),
"negative_prompt": (IO.STRING, {"multiline": True, "default": "low quality","tooltip": "Negative prompt of the image to generate"}),
"resolution": (["720P", "480P"], {"default": "720P"}),
"size": (["1280x720", "720x1280", "832x480", "480x832"], {"default": "1280x720"}),
"seed": ("INT", {"default": 0, "min": 0, "max": 2147483647}),
"seed": (IO.INT, {"default": 0, "min": 0, "max": 2147483647}),
}
}
RETURN_TYPES = ("VIDEO",)
RETURN_TYPES = (IO.VIDEO,)
RETURN_NAMES = ("video",)
FUNCTION = "generate_video"
CATEGORY = "UCLOUD_MODELVERSE"
requirements.txt
+1
View File
@@ -1,2 +1,3 @@
requests >= 2.32.4
pillow >= 11
openai >= 1.107.0