add gpt-image,qwen,nano banana (#5)
* add gitignore * add video api * fix: export to video * fix * fix * fix * fix * fix * add i2v * fix * fix * add openai * fix: add base url * change default models * fix: add base url * add v1 * fix models * fix models * fix models * fix models * fix models * fix bug * fix * try to fix * use IO.STRING test * image fix * add chat content * add files * fix optional * fix file read * fix file read * change name * add get models * add unused apikey * change model sort * fix IMAGE_LIST * fix: change to image * add model cache * remove json_object * change max_tokens to 65536 * change max_tokens to 6000 * add pic model * fix gpt-image-1 * fix gpt-image-1 * fix gpt * fix gpt * fix gpt * fix gpt * add gpt edit * fix gpt edit * update project.toml --------- Co-authored-by: jason.mei <jason.mei@ucloud.cn>
This commit is contained in:
jason
@@ -0,0 +1,105 @@
|
||||
import base64
|
||||
import torch
|
||||
from typing import Optional, List, Dict, Any
|
||||
from comfy.comfy_types.node_typing import IO
|
||||
|
||||
from .modelverse_api.client import ModelverseClient
|
||||
from .modelverse_api.requests.gemini_flash_image import GeminiFlashImageRequest
|
||||
from .modelverse_api.utils import decode_image, images2tensor
|
||||
|
||||
|
||||
def _extract_images_from_gemini_response(resp: Dict[str, Any]) -> List[torch.Tensor]:
|
||||
"""Parse Gemini API response and return list of tensors for any inline images."""
|
||||
images = []
|
||||
try:
|
||||
candidates = resp.get("candidates", []) or []
|
||||
for cand in candidates:
|
||||
content = cand.get("content", {})
|
||||
parts = content.get("parts", [])
|
||||
for part in parts:
|
||||
inline = part.get("inlineData")
|
||||
if inline and isinstance(inline, dict):
|
||||
data_b64 = inline.get("data")
|
||||
if data_b64:
|
||||
try:
|
||||
img_bytes = base64.b64decode(data_b64)
|
||||
pil_img = decode_image(img_bytes)
|
||||
images.append(pil_img)
|
||||
except Exception:
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
if not images:
|
||||
return []
|
||||
return [images2tensor(images)]
|
||||
|
||||
|
||||
class GeminiFlashImageNode:
|
||||
"""
|
||||
Gemini 2.5 Flash Image (text-to-image and image-edit) via Modelverse API.
|
||||
|
||||
Endpoint: /v1beta/models/gemini-2.5-flash-image:generateContent
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"client": ("MODELVERSE_API_CLIENT",),
|
||||
"prompt": (IO.STRING, {"multiline": True, "default": "Create a picture of a nano banana dish in a fancy restaurant with a Gemini theme"}),
|
||||
"mime_type": (["image/png", "image/jpeg"], {"default": "image/png"}),
|
||||
"num_requests": (IO.INT, {"default": 1, "min": 1, "max": 10, "step": 1, "display": "number"}),
|
||||
},
|
||||
"optional": {
|
||||
"image": (IO.IMAGE, {"default": None, "tooltip": "Optional input image for edit"}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = (IO.IMAGE,)
|
||||
RETURN_NAMES = ("image",)
|
||||
CATEGORY = "UCLOUD_MODELVERSE"
|
||||
FUNCTION = "execute"
|
||||
|
||||
async def execute(self,
|
||||
client,
|
||||
prompt: str,
|
||||
mime_type: str = "image/png",
|
||||
num_requests: int = 1,
|
||||
image=None):
|
||||
if not prompt:
|
||||
raise ValueError("Prompt is required")
|
||||
|
||||
mv_client = ModelverseClient(client["api_key"])
|
||||
|
||||
outputs: List[torch.Tensor] = []
|
||||
for i in range(num_requests):
|
||||
req = GeminiFlashImageRequest(prompt=prompt, image=image, mime_type=mime_type)
|
||||
payload = req.build_payload()
|
||||
resp = mv_client.post(req.API_PATH, payload)
|
||||
|
||||
if isinstance(resp, dict) and resp.get("error"):
|
||||
err = resp.get("error")
|
||||
raise Exception(f"GeminiFlashImage error: {err.get('message', 'Unknown error')}")
|
||||
|
||||
tensors = _extract_images_from_gemini_response(resp)
|
||||
if not tensors:
|
||||
print("WARN:", "No image data found in Gemini response; check console for details.")
|
||||
continue
|
||||
# tensors is a list with one batch tensor; append that tensor
|
||||
outputs.append(tensors[0])
|
||||
|
||||
if not outputs:
|
||||
# Return an empty black image tensor (1x3x1x1) to avoid breaking graph
|
||||
return (torch.zeros((1, 3, 1, 1)),)
|
||||
|
||||
# Concatenate along batch dimension
|
||||
return (torch.cat(outputs, dim=0),)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"NanoBanana": GeminiFlashImageNode,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"NanoBanana": "Modelverse Gemini 2.5 Flash Image",
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
import base64
|
||||
import torch
|
||||
from typing import List
|
||||
from comfy.comfy_types.node_typing import IO
|
||||
|
||||
from .modelverse_api.utils import imageurl2tensor, decode_image, images2tensor
|
||||
from .modelverse_api.client import ModelverseClient
|
||||
from .modelverse_api.requests.gpt_image_1 import GPTImage1
|
||||
|
||||
|
||||
class GPTImage1Node:
|
||||
"""
|
||||
gpt-image-1 text-to-image via Modelverse /v1/images/generations API.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
return {
|
||||
"required": {
|
||||
"client": ("MODELVERSE_API_CLIENT",),
|
||||
"prompt": (IO.STRING, {"multiline": True, "default": "a beautiful flower"}),
|
||||
"size": (["1024x1024", "1024x1536", "1536x1024"], {"default": "1024x1024"}),
|
||||
"num_requests": (IO.INT, {"default": 1, "min": 1, "max": 10, "step": 1, "display": "number"}),
|
||||
"num_images": (IO.INT, {"default": 1, "min": 1, "max": 4, "step": 1, "display": "number"}),
|
||||
},
|
||||
"optional": {
|
||||
"quality": (["", "low", "medium", "high"], {"default": ""}),
|
||||
"output_format": (["png", "jpeg"], {"default": "png"}),
|
||||
"output_compression": (IO.INT, {"default": 100, "min": 0, "max": 100, "step": 1, "display": "number"}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = (IO.IMAGE,)
|
||||
RETURN_NAMES = ("image",)
|
||||
CATEGORY = "UCLOUD_MODELVERSE"
|
||||
FUNCTION = "execute"
|
||||
|
||||
async def execute(
|
||||
self,
|
||||
client,
|
||||
prompt: str,
|
||||
size: str = "1024x1024",
|
||||
num_requests: int = 1,
|
||||
num_images: int = 1,
|
||||
quality: str = "",
|
||||
output_format: str = "png",
|
||||
output_compression: int = 100,
|
||||
):
|
||||
|
||||
if not prompt:
|
||||
raise ValueError("Prompt is required")
|
||||
|
||||
mv_client = ModelverseClient(client["api_key"])
|
||||
|
||||
tasks = [
|
||||
mv_client.async_send_request(
|
||||
GPTImage1(
|
||||
prompt=prompt,
|
||||
num_images=num_images,
|
||||
size=size,
|
||||
quality=quality if quality != "" else None,
|
||||
output_format=output_format,
|
||||
output_compression=output_compression,
|
||||
)
|
||||
)
|
||||
for i in range(num_requests)
|
||||
]
|
||||
|
||||
results = await mv_client.run_tasks(tasks)
|
||||
|
||||
output_images_list: List[torch.Tensor] = []
|
||||
for data_list in results:
|
||||
if not data_list:
|
||||
print("WARN:", "No output in current request. Skipping...")
|
||||
continue
|
||||
|
||||
# Auto-detect output type: prefer URL if present, else decode b64_json
|
||||
has_url = False
|
||||
try:
|
||||
for item in data_list:
|
||||
if isinstance(item, dict) and item.get("url"):
|
||||
has_url = True
|
||||
break
|
||||
except Exception:
|
||||
has_url = False
|
||||
|
||||
if has_url:
|
||||
try:
|
||||
output_images = imageurl2tensor(data_list)
|
||||
except Exception:
|
||||
print("WARN:", "Failed to load URL images; attempting b64_json decode.")
|
||||
has_url = False # fall through to b64 decode
|
||||
|
||||
if not has_url:
|
||||
images = []
|
||||
for item in data_list:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
b64v = item.get("b64_json") or item.get("b64")
|
||||
if not b64v:
|
||||
continue
|
||||
if isinstance(b64v, str) and b64v.startswith("data:"):
|
||||
try:
|
||||
b64v = b64v.split(",", 1)[1]
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
img_bytes = base64.b64decode(b64v)
|
||||
pil_img = decode_image(img_bytes)
|
||||
images.append(pil_img)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if not images:
|
||||
print("WARN:", "No decodable base64 image found.")
|
||||
continue
|
||||
output_images = images2tensor(images)
|
||||
|
||||
output_images_list.append(output_images)
|
||||
|
||||
if not output_images_list:
|
||||
return (torch.zeros((1, 3, 1, 1)),)
|
||||
|
||||
return (torch.cat(output_images_list, dim=0),)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"Modelverse GPTImage1Node": GPTImage1Node,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"Modelverse GPTImage1Node": "Modelverse GPT Image 1",
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
import base64
|
||||
import torch
|
||||
from typing import List
|
||||
from comfy.comfy_types.node_typing import IO
|
||||
|
||||
from .modelverse_api.client import ModelverseClient
|
||||
from .modelverse_api.requests.gpt_image_1_edit import GPTImage1Edit
|
||||
from .modelverse_api.utils import imageurl2tensor, decode_image, images2tensor
|
||||
|
||||
|
||||
class GPTImage1EditNode:
|
||||
"""
|
||||
gpt-image-1 image edit via /v1/images/edits (multipart/form-data).
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
return {
|
||||
"required": {
|
||||
"client": ("MODELVERSE_API_CLIENT",),
|
||||
"prompt": (IO.STRING, {"multiline": True, "default": "Add a beach ball in the center"}),
|
||||
"image": (IO.IMAGE, {"tooltip": "Input image"}),
|
||||
"size": (["1024x1024", "1024x1536", "1536x1024"], {"default": "1024x1024"}),
|
||||
"num_requests": (IO.INT, {"default": 1, "min": 1, "max": 10, "step": 1, "display": "number"}),
|
||||
"num_images": (IO.INT, {"default": 1, "min": 1, "max": 4, "step": 1, "display": "number"}),
|
||||
},
|
||||
"optional": {
|
||||
"mask": (IO.IMAGE, {"default": None, "tooltip": "Optional mask image"}),
|
||||
"quality": (["", "low", "medium", "high"], {"default": ""}),
|
||||
"output_format": (["png", "jpeg"], {"default": "png"}),
|
||||
"output_compression": (IO.INT, {"default": 100, "min": 0, "max": 100, "step": 1, "display": "number"}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = (IO.IMAGE,)
|
||||
RETURN_NAMES = ("image",)
|
||||
CATEGORY = "UCLOUD_MODELVERSE"
|
||||
FUNCTION = "execute"
|
||||
|
||||
async def execute(
|
||||
self,
|
||||
client,
|
||||
prompt: str,
|
||||
image,
|
||||
size: str = "1024x1024",
|
||||
num_requests: int = 1,
|
||||
num_images: int = 1,
|
||||
mask=None,
|
||||
quality: str = "",
|
||||
output_format: str = "png",
|
||||
output_compression: int = 100,
|
||||
):
|
||||
|
||||
if not prompt:
|
||||
raise ValueError("Prompt is required")
|
||||
if image is None:
|
||||
raise ValueError("Input image is required")
|
||||
|
||||
mv_client = ModelverseClient(client["api_key"])
|
||||
|
||||
tasks = [
|
||||
mv_client.async_send_request(
|
||||
GPTImage1Edit(
|
||||
prompt=prompt,
|
||||
image=image,
|
||||
mask=mask,
|
||||
num_images=num_images,
|
||||
size=size,
|
||||
quality=quality if quality != "" else None,
|
||||
output_format=output_format,
|
||||
output_compression=output_compression,
|
||||
)
|
||||
)
|
||||
for _ in range(num_requests)
|
||||
]
|
||||
|
||||
results = await mv_client.run_tasks(tasks)
|
||||
|
||||
output_images_list: List[torch.Tensor] = []
|
||||
for data_list in results:
|
||||
if not data_list:
|
||||
print("WARN:", "No output in current request. Skipping...")
|
||||
continue
|
||||
|
||||
# Auto-detect URL vs b64_json
|
||||
has_url = any(isinstance(it, dict) and it.get("url") for it in data_list)
|
||||
if has_url:
|
||||
try:
|
||||
output_images = imageurl2tensor(data_list)
|
||||
except Exception:
|
||||
print("WARN:", "Failed to load URL images; attempting b64_json decode.")
|
||||
has_url = False
|
||||
|
||||
if not has_url:
|
||||
images = []
|
||||
for item in data_list:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
b64v = item.get("b64_json") or item.get("b64")
|
||||
if not b64v:
|
||||
continue
|
||||
if isinstance(b64v, str) and b64v.startswith("data:"):
|
||||
try:
|
||||
b64v = b64v.split(",", 1)[1]
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
img_bytes = base64.b64decode(b64v)
|
||||
pil_img = decode_image(img_bytes)
|
||||
images.append(pil_img)
|
||||
except Exception:
|
||||
continue
|
||||
if not images:
|
||||
print("WARN:", "No decodable base64 image found.")
|
||||
continue
|
||||
output_images = images2tensor(images)
|
||||
|
||||
output_images_list.append(output_images)
|
||||
|
||||
if not output_images_list:
|
||||
return (torch.zeros((1, 3, 1, 1)),)
|
||||
|
||||
return (torch.cat(output_images_list, dim=0),)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"Modelverse GPTImage1EditNode": GPTImage1EditNode,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"Modelverse GPTImage1EditNode": "Modelverse GPT Image 1 Edit",
|
||||
}
|
||||
|
||||
@@ -22,6 +22,14 @@ class ModelverseClient:
|
||||
response = requests.post(url, headers=self.headers, json=payload, timeout=timeout)
|
||||
return self._handle_response(response)
|
||||
|
||||
def post_multipart(self, endpoint, data=None, files=None, timeout=180):
|
||||
"""POST with multipart/form-data. Content-Type is set by requests automatically."""
|
||||
url = f"{self.BASE_URL}{endpoint}"
|
||||
# Do not set Content-Type explicitly when using files; requests will handle it.
|
||||
headers = {k: v for k, v in self.headers.items() if k.lower() != "content-type"}
|
||||
response = requests.post(url, headers=headers, data=data, files=files, timeout=timeout)
|
||||
return self._handle_response(response)
|
||||
|
||||
def get(self, endpoint, params=None, timeout=180):
|
||||
url = f"{self.BASE_URL}{endpoint}"
|
||||
headers = {"Authorization": f"Bearer {self.api_key}"}
|
||||
@@ -69,12 +77,16 @@ class ModelverseClient:
|
||||
|
||||
# --- Restored Async Methods for existing nodes ---
|
||||
async def async_send_request(self, request: BaseRequest):
|
||||
payload = request.build_payload()
|
||||
endpoint = request.API_PATH
|
||||
if "seed" in payload:
|
||||
payload["seed"] = payload["seed"] % 2147483647 if payload["seed"] != -1 else -1
|
||||
|
||||
response = self.post(endpoint, payload)
|
||||
# Support multipart form requests when available
|
||||
if hasattr(request, "build_multipart") and callable(getattr(request, "build_multipart")):
|
||||
data, files = request.build_multipart()
|
||||
response = self.post_multipart(endpoint, data=data, files=files)
|
||||
else:
|
||||
payload = request.build_payload()
|
||||
if isinstance(payload, dict) and "seed" in payload:
|
||||
payload["seed"] = payload["seed"] % 2147483647 if payload["seed"] != -1 else -1
|
||||
response = self.post(endpoint, payload)
|
||||
return response.get("data", [])
|
||||
|
||||
async def run_tasks(self, tasks):
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
from typing import Optional, List, Dict, Any
|
||||
from pydantic import Field
|
||||
from ..utils import BaseRequest
|
||||
from torch import Tensor
|
||||
import base64
|
||||
import io
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
|
||||
|
||||
def _tensor_to_base64(image: Tensor, mime_type: str = "image/png") -> Dict[str, str]:
|
||||
"""Convert a ComfyUI image tensor (HWC or 4D with batch) to base64 without data URI."""
|
||||
if image is None:
|
||||
return None
|
||||
|
||||
# If batch, take first frame
|
||||
if hasattr(image, 'shape') and len(image.shape) == 4:
|
||||
image = image[0]
|
||||
|
||||
# Tensor (H, W, C) in 0..1 -> uint8
|
||||
np_img = np.clip(255.0 * image.cpu().numpy(), 0, 255).astype(np.uint8)
|
||||
pil_img = Image.fromarray(np_img)
|
||||
|
||||
fmt = 'PNG' if mime_type.lower().endswith('png') else 'JPEG'
|
||||
with io.BytesIO() as bio:
|
||||
pil_img.save(bio, format=fmt)
|
||||
data = bio.getvalue()
|
||||
|
||||
return {
|
||||
"mimeType": mime_type,
|
||||
"data": base64.b64encode(data).decode("utf-8"),
|
||||
}
|
||||
|
||||
|
||||
class GeminiFlashImageRequest(BaseRequest):
|
||||
"""
|
||||
Request builder for Gemini 2.5 Flash Image generateContent endpoint.
|
||||
Supports text-to-image and image-edit (text+image to image).
|
||||
"""
|
||||
|
||||
API_PATH = "/v1beta/models/gemini-2.5-flash-image:generateContent"
|
||||
|
||||
prompt: str = Field(..., description="Text prompt")
|
||||
image: Optional[Tensor] = Field(default=None, description="Optional input image for edit")
|
||||
mime_type: str = Field(default="image/png", description="MIME type for inline image data")
|
||||
|
||||
def __init__(self, prompt: str, image: Optional[Tensor] = None, mime_type: str = "image/png", **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.prompt = prompt
|
||||
self.image = image
|
||||
self.mime_type = mime_type
|
||||
|
||||
def build_payload(self) -> Dict[str, Any]:
|
||||
parts: List[Dict[str, Any]] = []
|
||||
if self.prompt:
|
||||
parts.append({"text": self.prompt})
|
||||
|
||||
if isinstance(self.image, Tensor):
|
||||
parts.append({
|
||||
"inlineData": _tensor_to_base64(self.image, self.mime_type)
|
||||
})
|
||||
|
||||
payload = {
|
||||
"contents": [
|
||||
{
|
||||
"role": "user",
|
||||
"parts": parts,
|
||||
}
|
||||
],
|
||||
"generationConfig": {
|
||||
"responseModalities": ["TEXT", "IMAGE"],
|
||||
},
|
||||
}
|
||||
return self._remove_empty_fields(payload)
|
||||
|
||||
def field_required(self):
|
||||
return ["prompt"]
|
||||
|
||||
def field_order(self):
|
||||
return ["prompt", "image", "mime_type"]
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
from typing import Optional
|
||||
from pydantic import Field
|
||||
from ..utils import BaseRequest
|
||||
|
||||
|
||||
class GPTImage1(BaseRequest):
|
||||
"""
|
||||
gpt-image-1 text-to-image via /v1/images/generations.
|
||||
"""
|
||||
|
||||
API_PATH = "/v1/images/generations"
|
||||
|
||||
prompt: str = Field(..., description="Prompt text")
|
||||
num_images: Optional[int] = Field(default=1, ge=1, le=4, description="Number of images (n)")
|
||||
size: Optional[str] = Field(default="1024x1024", description="Output size: 1024x1024, 1024x1536, 1536x1024")
|
||||
quality: Optional[str] = Field(default=None, description="Image quality: low, medium, high")
|
||||
output_format: Optional[str] = Field(default="png", description="Output format: png or jpeg")
|
||||
output_compression: Optional[int] = Field(default=100, ge=0, le=100, description="Compression strength 0-100")
|
||||
# negative_prompt and response_format removed per API behavior
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
prompt: str,
|
||||
num_images: Optional[int] = 1,
|
||||
size: Optional[str] = "1024x1024",
|
||||
quality: Optional[str] = None,
|
||||
output_format: Optional[str] = "png",
|
||||
output_compression: Optional[int] = 100,
|
||||
# guidance_scale removed per API behavior
|
||||
# negative_prompt/response_format removed per API behavior
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self.prompt = prompt
|
||||
self.num_images = num_images
|
||||
self.size = size
|
||||
self.quality = quality
|
||||
self.output_format = output_format
|
||||
self.output_compression = output_compression
|
||||
|
||||
def build_payload(self) -> dict:
|
||||
payload = {
|
||||
"model": "gpt-image-1",
|
||||
"prompt": self.prompt,
|
||||
"n": self.num_images,
|
||||
"size": self.size,
|
||||
"quality": self.quality,
|
||||
"output_format": self.output_format,
|
||||
"output_compression": self.output_compression,
|
||||
}
|
||||
return self._remove_empty_fields(payload)
|
||||
|
||||
def field_required(self):
|
||||
return ["prompt"]
|
||||
|
||||
def field_order(self):
|
||||
return [
|
||||
"model",
|
||||
"prompt",
|
||||
"n",
|
||||
"size",
|
||||
"quality",
|
||||
"output_format",
|
||||
"output_compression",
|
||||
]
|
||||
@@ -0,0 +1,115 @@
|
||||
from typing import Optional, Tuple, Dict, Any
|
||||
from pydantic import Field
|
||||
from ..utils import BaseRequest, tensor2images
|
||||
from torch import Tensor
|
||||
import io
|
||||
from PIL import Image
|
||||
|
||||
|
||||
def _tensor_to_png_file(image: Tensor, filename: str) -> Tuple[str, bytes, str] | None:
|
||||
"""Convert a ComfyUI tensor image (3D or 4D) to a PNG file tuple for requests files."""
|
||||
if image is None:
|
||||
return None
|
||||
# Keep batch if present; tensor2images expects a batch dimension
|
||||
try:
|
||||
if hasattr(image, 'shape') and len(image.shape) == 3:
|
||||
# add batch dim
|
||||
img_batch = image.unsqueeze(0)
|
||||
else:
|
||||
img_batch = image
|
||||
pil_img = tensor2images(img_batch)[0]
|
||||
except Exception:
|
||||
# Fallback: attempt naive conversion
|
||||
from PIL import Image as _Image
|
||||
import numpy as _np
|
||||
arr = (image.cpu().numpy() * 255.0).clip(0, 255).astype('uint8')
|
||||
pil_img = _Image.fromarray(arr)
|
||||
with io.BytesIO() as bio:
|
||||
pil_img.save(bio, format="PNG")
|
||||
data = bio.getvalue()
|
||||
return (filename, data, "image/png")
|
||||
|
||||
|
||||
class GPTImage1Edit(BaseRequest):
|
||||
"""
|
||||
gpt-image-1 image edit via /v1/images/edits using multipart/form-data.
|
||||
"""
|
||||
|
||||
API_PATH = "/v1/images/edits"
|
||||
|
||||
prompt: str = Field(..., description="Edit prompt")
|
||||
image: Tensor = Field(..., description="Input image")
|
||||
mask: Optional[Tensor] = Field(default=None, description="Optional mask image")
|
||||
|
||||
num_images: Optional[int] = Field(default=1, ge=1, le=4, description="Number of images (n)")
|
||||
size: Optional[str] = Field(default="1024x1024", description="Output size: 1024x1024, 1024x1536, 1536x1024")
|
||||
quality: Optional[str] = Field(default=None, description="Image quality: low, medium, high")
|
||||
output_format: Optional[str] = Field(default="png", description="Output format: png or jpeg")
|
||||
output_compression: Optional[int] = Field(default=100, ge=0, le=100, description="Compression strength 0-100")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
prompt: str,
|
||||
image: Tensor,
|
||||
mask: Optional[Tensor] = None,
|
||||
num_images: Optional[int] = 1,
|
||||
size: Optional[str] = "1024x1024",
|
||||
quality: Optional[str] = None,
|
||||
output_format: Optional[str] = "png",
|
||||
output_compression: Optional[int] = 100,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self.prompt = prompt
|
||||
self.image = image
|
||||
self.mask = mask
|
||||
self.num_images = num_images
|
||||
self.size = size
|
||||
self.quality = quality
|
||||
self.output_format = output_format
|
||||
self.output_compression = output_compression
|
||||
|
||||
def build_multipart(self) -> Tuple[Dict[str, Any], Dict[str, Any]]:
|
||||
data = {
|
||||
"model": "gpt-image-1",
|
||||
"prompt": self.prompt,
|
||||
"n": self.num_images,
|
||||
"size": self.size,
|
||||
"output_format": self.output_format,
|
||||
"output_compression": str(self.output_compression) if self.output_compression is not None else None,
|
||||
}
|
||||
if self.quality:
|
||||
data["quality"] = self.quality
|
||||
# Filter None values
|
||||
data = {k: v for k, v in data.items() if v is not None and v != ""}
|
||||
|
||||
files = {}
|
||||
img_file = _tensor_to_png_file(self.image, "image.png")
|
||||
if img_file is not None:
|
||||
files["image"] = img_file
|
||||
if self.mask is not None:
|
||||
mask_file = _tensor_to_png_file(self.mask, "mask.png")
|
||||
if mask_file is not None:
|
||||
files["mask"] = mask_file
|
||||
|
||||
return data, files
|
||||
|
||||
# For interface completeness; not used by multipart path
|
||||
def build_payload(self) -> dict:
|
||||
return {}
|
||||
|
||||
def field_required(self):
|
||||
return ["prompt", "image"]
|
||||
|
||||
def field_order(self):
|
||||
return [
|
||||
"model",
|
||||
"prompt",
|
||||
"image",
|
||||
"mask",
|
||||
"n",
|
||||
"size",
|
||||
"quality",
|
||||
"output_format",
|
||||
"output_compression",
|
||||
]
|
||||
@@ -0,0 +1,73 @@
|
||||
from typing import Optional
|
||||
from pydantic import Field
|
||||
from ..utils import BaseRequest
|
||||
|
||||
|
||||
class QwenImage(BaseRequest):
|
||||
"""
|
||||
Qwen/Qwen-Image text-to-image via /v1/images/generations.
|
||||
"""
|
||||
|
||||
API_PATH = "/v1/images/generations"
|
||||
|
||||
prompt: str = Field(..., description="Prompt text")
|
||||
aspect_ratio: Optional[str] = Field(default="1:1", description="Aspect ratio e.g. 16:9, 1:1")
|
||||
num_images: Optional[int] = Field(default=1, ge=1, le=4, description="Number of images (n)")
|
||||
seed: Optional[int] = Field(default=-1, description="Random seed (-1 for random)")
|
||||
steps: Optional[int] = Field(default=20, ge=1, le=50, description="Inference steps")
|
||||
guidance_scale: Optional[float] = Field(default=2.5, ge=1.0, le=10.0, description="Guidance scale 1~10")
|
||||
negative_prompt: Optional[str] = Field(default="", description="Negative prompt")
|
||||
response_format: Optional[str] = Field(default="url", description='"url" or "b64_json"')
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
prompt: str,
|
||||
aspect_ratio: Optional[str] = "1:1",
|
||||
num_images: Optional[int] = 1,
|
||||
seed: Optional[int] = -1,
|
||||
steps: Optional[int] = 20,
|
||||
guidance_scale: Optional[float] = 2.5,
|
||||
negative_prompt: Optional[str] = "",
|
||||
response_format: Optional[str] = "url",
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self.prompt = prompt
|
||||
self.aspect_ratio = aspect_ratio
|
||||
self.num_images = num_images
|
||||
self.seed = seed
|
||||
self.steps = steps
|
||||
self.guidance_scale = guidance_scale
|
||||
self.negative_prompt = negative_prompt
|
||||
self.response_format = response_format
|
||||
|
||||
def build_payload(self) -> dict:
|
||||
payload = {
|
||||
"model": "Qwen/Qwen-Image",
|
||||
"prompt": self.prompt,
|
||||
"aspect_ratio": self.aspect_ratio,
|
||||
"n": self.num_images,
|
||||
"seed": self.seed,
|
||||
"steps": self.steps,
|
||||
"guidance_scale": self.guidance_scale,
|
||||
"negative_prompt": self.negative_prompt,
|
||||
"response_format": self.response_format,
|
||||
}
|
||||
return self._remove_empty_fields(payload)
|
||||
|
||||
def field_required(self):
|
||||
return ["prompt"]
|
||||
|
||||
def field_order(self):
|
||||
return [
|
||||
"model",
|
||||
"prompt",
|
||||
"aspect_ratio",
|
||||
"n",
|
||||
"seed",
|
||||
"steps",
|
||||
"guidance_scale",
|
||||
"negative_prompt",
|
||||
"response_format",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
from typing import Optional
|
||||
from pydantic import Field
|
||||
from ..utils import BaseRequest, image_to_base64
|
||||
from torch import Tensor
|
||||
|
||||
|
||||
class QwenImageEdit(BaseRequest):
|
||||
"""
|
||||
Qwen/Qwen-Image-Edit request model for /v1/images/generations (OpenAI-compatible).
|
||||
Supports image edit with a prompt and optional controls.
|
||||
"""
|
||||
|
||||
API_PATH = "/v1/images/generations"
|
||||
|
||||
prompt: str = Field(..., description="Prompt text for the edit")
|
||||
image: Tensor = Field(..., description="Input image tensor for edit")
|
||||
negative_prompt: Optional[str] = Field(default="", description="Negative prompt")
|
||||
|
||||
num_images: Optional[int] = Field(default=1, ge=1, le=4, description="Number of images to generate (n)")
|
||||
strength: Optional[float] = Field(default=0.8, ge=0.0, le=1.0, description="Edit strength 0~1")
|
||||
seed: Optional[int] = Field(default=-1, description="Random seed (-1 for random)")
|
||||
steps: Optional[int] = Field(default=20, ge=1, le=50, description="Inference steps 1~50")
|
||||
guidance_scale: Optional[float] = Field(default=2.5, ge=1.0, le=10.0, description="Guidance scale 1~10")
|
||||
response_format: Optional[str] = Field(default="url", description='"url" or "b64_json"')
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
prompt: str,
|
||||
image: Tensor,
|
||||
negative_prompt: Optional[str] = "",
|
||||
num_images: Optional[int] = 1,
|
||||
strength: Optional[float] = 0.8,
|
||||
seed: Optional[int] = -1,
|
||||
steps: Optional[int] = 20,
|
||||
guidance_scale: Optional[float] = 2.5,
|
||||
response_format: Optional[str] = "url",
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self.prompt = prompt
|
||||
self.negative_prompt = negative_prompt
|
||||
self.num_images = num_images
|
||||
self.strength = strength
|
||||
self.seed = seed
|
||||
self.steps = steps
|
||||
self.guidance_scale = guidance_scale
|
||||
self.response_format = response_format
|
||||
self.image = image_to_base64(image)
|
||||
|
||||
def build_payload(self) -> dict:
|
||||
payload = {
|
||||
"model": "Qwen/Qwen-Image-Edit",
|
||||
"prompt": self.prompt,
|
||||
"image": self.image,
|
||||
"negative_prompt": self.negative_prompt,
|
||||
"n": self.num_images,
|
||||
"strength": self.strength,
|
||||
"seed": self.seed,
|
||||
"steps": self.steps,
|
||||
"guidance_scale": self.guidance_scale,
|
||||
"response_format": self.response_format,
|
||||
}
|
||||
return self._remove_empty_fields(payload)
|
||||
|
||||
def field_required(self):
|
||||
return ["prompt", "image"]
|
||||
|
||||
def field_order(self):
|
||||
return [
|
||||
"model",
|
||||
"prompt",
|
||||
"image",
|
||||
"negative_prompt",
|
||||
"n",
|
||||
"strength",
|
||||
"seed",
|
||||
"steps",
|
||||
"guidance_scale",
|
||||
"response_format",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
import base64
|
||||
import torch
|
||||
from typing import Optional, List
|
||||
from comfy.comfy_types.node_typing import IO
|
||||
|
||||
from .modelverse_api.utils import imageurl2tensor, decode_image, images2tensor
|
||||
from .modelverse_api.client import ModelverseClient
|
||||
from .modelverse_api.requests.qwen_image_edit import QwenImageEdit
|
||||
|
||||
|
||||
class QwenImageEditNode:
|
||||
"""
|
||||
Qwen/Qwen-Image-Edit image editing via Modelverse /v1/images/generations API.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
return {
|
||||
"required": {
|
||||
"client": ("MODELVERSE_API_CLIENT",),
|
||||
"prompt": (IO.STRING, {"multiline": True, "default": "Convert to quick pencil sketch"}),
|
||||
"image": (IO.IMAGE, {"tooltip": "Input image to edit"}),
|
||||
"negative_prompt": (IO.STRING, {"multiline": True, "default": "", "tooltip": "Negative prompt"}),
|
||||
"num_requests": (IO.INT, {"default": 1, "min": 1, "max": 10, "step": 1, "display": "number"}),
|
||||
"num_images": (IO.INT, {"default": 1, "min": 1, "max": 4, "step": 1, "display": "number"}),
|
||||
"strength": (IO.FLOAT, {"default": 0.8, "min": 0.0, "max": 1.0, "step": 0.01, "display": "number"}),
|
||||
"steps": (IO.INT, {"default": 20, "min": 1, "max": 50, "step": 1, "display": "number"}),
|
||||
"seed": (IO.INT, {"default": -1, "min": -1, "max": 0xffffffffffffffff, "control_after_generate": True}),
|
||||
"guidance_scale": (IO.FLOAT, {"default": 2.5, "min": 1.0, "max": 10.0, "step": 0.1, "display": "number"}),
|
||||
"response_format": (["url", "b64_json"], {"default": "url"}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = (IO.IMAGE,)
|
||||
RETURN_NAMES = ("image",)
|
||||
|
||||
CATEGORY = "UCLOUD_MODELVERSE"
|
||||
FUNCTION = "execute"
|
||||
|
||||
async def execute(
|
||||
self,
|
||||
client,
|
||||
prompt: str,
|
||||
image,
|
||||
negative_prompt: str = "",
|
||||
num_requests: int = 1,
|
||||
num_images: int = 1,
|
||||
strength: float = 0.8,
|
||||
steps: int = 20,
|
||||
seed: int = -1,
|
||||
guidance_scale: float = 2.5,
|
||||
response_format: str = "url",
|
||||
):
|
||||
|
||||
if not prompt:
|
||||
raise ValueError("Prompt is required")
|
||||
if image is None:
|
||||
raise ValueError("Input image is required")
|
||||
|
||||
print("INFO:", "Running Qwen/Qwen-Image-Edit.")
|
||||
|
||||
mv_client = ModelverseClient(client["api_key"])
|
||||
|
||||
tasks = [
|
||||
mv_client.async_send_request(
|
||||
QwenImageEdit(
|
||||
prompt=prompt,
|
||||
image=image,
|
||||
negative_prompt=negative_prompt,
|
||||
num_images=num_images,
|
||||
strength=strength,
|
||||
seed=seed + i,
|
||||
steps=steps,
|
||||
guidance_scale=guidance_scale,
|
||||
response_format=response_format,
|
||||
)
|
||||
)
|
||||
for i in range(num_requests)
|
||||
]
|
||||
|
||||
results = await mv_client.run_tasks(tasks) # list of data lists
|
||||
|
||||
output_images_list: List[torch.Tensor] = []
|
||||
for data_list in results:
|
||||
if not data_list:
|
||||
print("WARN:", "No output in current request. Skipping...")
|
||||
continue
|
||||
|
||||
if response_format == "url":
|
||||
output_images = imageurl2tensor(data_list)
|
||||
else:
|
||||
# b64_json path
|
||||
images = []
|
||||
for item in data_list:
|
||||
b64v = item.get("b64_json") or item.get("b64")
|
||||
if not b64v:
|
||||
continue
|
||||
if isinstance(b64v, str) and b64v.startswith("data:"):
|
||||
try:
|
||||
b64v = b64v.split(",", 1)[1]
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
img_bytes = base64.b64decode(b64v)
|
||||
pil_img = decode_image(img_bytes)
|
||||
images.append(pil_img)
|
||||
except Exception:
|
||||
continue
|
||||
if not images:
|
||||
print("WARN:", "No decodable base64 image found.")
|
||||
continue
|
||||
output_images = images2tensor(images)
|
||||
|
||||
output_images_list.append(output_images)
|
||||
|
||||
if not output_images_list:
|
||||
return (torch.zeros((1, 3, 1, 1)),)
|
||||
|
||||
return (torch.cat(output_images_list, dim=0),)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"Modelverse QwenImageEditNode": QwenImageEditNode,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"Modelverse QwenImageEditNode": "Modelverse Qwen Image Edit",
|
||||
}
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
import base64
|
||||
import torch
|
||||
from typing import List
|
||||
from comfy.comfy_types.node_typing import IO
|
||||
|
||||
from .modelverse_api.utils import imageurl2tensor, decode_image, images2tensor
|
||||
from .modelverse_api.client import ModelverseClient
|
||||
from .modelverse_api.requests.qwen_image import QwenImage
|
||||
|
||||
|
||||
class QwenImageT2INode:
|
||||
"""
|
||||
Qwen/Qwen-Image text-to-image via Modelverse /v1/images/generations API.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
return {
|
||||
"required": {
|
||||
"client": ("MODELVERSE_API_CLIENT",),
|
||||
"prompt": (IO.STRING, {"multiline": True, "default": "a beautiful flower"}),
|
||||
"aspect_ratio": (["21:9", "16:9", "4:3", "3:2", "1:1", "2:3", "3:4", "9:16", "9:21"], {"default": "1:1"}),
|
||||
"num_requests": (IO.INT, {"default": 1, "min": 1, "max": 10, "step": 1, "display": "number"}),
|
||||
"num_images": (IO.INT, {"default": 1, "min": 1, "max": 4, "step": 1, "display": "number"}),
|
||||
"steps": (IO.INT, {"default": 20, "min": 1, "max": 50, "step": 1, "display": "number"}),
|
||||
"seed": (IO.INT, {"default": -1, "min": -1, "max": 0xffffffffffffffff, "control_after_generate": True}),
|
||||
"guidance_scale": (IO.FLOAT, {"default": 2.5, "min": 1.0, "max": 10.0, "step": 0.1, "display": "number"}),
|
||||
"response_format": (["url", "b64_json"], {"default": "url"}),
|
||||
},
|
||||
"optional": {
|
||||
"negative_prompt": (IO.STRING, {"multiline": True, "default": ""}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = (IO.IMAGE,)
|
||||
RETURN_NAMES = ("image",)
|
||||
CATEGORY = "UCLOUD_MODELVERSE"
|
||||
FUNCTION = "execute"
|
||||
|
||||
async def execute(
|
||||
self,
|
||||
client,
|
||||
prompt: str,
|
||||
aspect_ratio: str = "1:1",
|
||||
num_requests: int = 1,
|
||||
num_images: int = 1,
|
||||
steps: int = 20,
|
||||
seed: int = -1,
|
||||
guidance_scale: float = 2.5,
|
||||
response_format: str = "url",
|
||||
negative_prompt: str = "",
|
||||
):
|
||||
|
||||
if not prompt:
|
||||
raise ValueError("Prompt is required")
|
||||
|
||||
mv_client = ModelverseClient(client["api_key"])
|
||||
|
||||
tasks = [
|
||||
mv_client.async_send_request(
|
||||
QwenImage(
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect_ratio,
|
||||
num_images=num_images,
|
||||
seed=seed + i,
|
||||
steps=steps,
|
||||
guidance_scale=guidance_scale,
|
||||
negative_prompt=negative_prompt,
|
||||
response_format=response_format,
|
||||
)
|
||||
)
|
||||
for i in range(num_requests)
|
||||
]
|
||||
|
||||
results = await mv_client.run_tasks(tasks)
|
||||
|
||||
output_images_list: List[torch.Tensor] = []
|
||||
for data_list in results:
|
||||
if not data_list:
|
||||
print("WARN:", "No output in current request. Skipping...")
|
||||
continue
|
||||
|
||||
if response_format == "url":
|
||||
output_images = imageurl2tensor(data_list)
|
||||
else:
|
||||
images = []
|
||||
for item in data_list:
|
||||
b64v = item.get("b64_json") or item.get("b64")
|
||||
if not b64v:
|
||||
continue
|
||||
if isinstance(b64v, str) and b64v.startswith("data:"):
|
||||
try:
|
||||
b64v = b64v.split(",", 1)[1]
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
img_bytes = base64.b64decode(b64v)
|
||||
pil_img = decode_image(img_bytes)
|
||||
images.append(pil_img)
|
||||
except Exception:
|
||||
continue
|
||||
if not images:
|
||||
print("WARN:", "No decodable base64 image found.")
|
||||
continue
|
||||
output_images = images2tensor(images)
|
||||
|
||||
output_images_list.append(output_images)
|
||||
|
||||
if not output_images_list:
|
||||
return (torch.zeros((1, 3, 1, 1)),)
|
||||
|
||||
return (torch.cat(output_images_list, dim=0),)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"Modelverse QwenImageT2INode": QwenImageT2INode,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"Modelverse QwenImageT2INode": "Modelverse Qwen Image",
|
||||
}
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
[project]
|
||||
name = "ucloud-comfyui"
|
||||
description = "This is a custom node for ComfyUI that allows you to use the UCloud API directly in ComfyUI. UCloud is a high-performance AI computing platform providing powerful image generation models and industry-leading inference speeds. It offers a wide range of AI model options, including popular text-to-image and image editing models, ready to use without local GPU setup."
|
||||
version = "1.0.6"
|
||||
version = "1.0.7"
|
||||
license = {file = "LICENSE"}
|
||||
dependencies = ["requests", "pillow"]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user