feat(modelverse): add Gemini Pro Image and Vidu video generation support
- Add Gemini 3 Pro Image Preview (Nano Banana Pro) ComfyUI node with high-resolution output (1K, 2K, 4K) - Add GeminiProImageRequest builder supporting aspect ratio control, Google Search grounding, and up to 14 reference images - Add Vidu video generation nodes: t2v, i2v, ref2v, and startend2v for comprehensive video synthesis - Add Vidu extend node for video continuation and frame extension - Update .gitignore to exclude .ace-tool/ and docs/ directories - Enable professional asset production with advanced image and video generation capabilities
This commit is contained in:
jason.mei
+3
-1
@@ -4,4 +4,6 @@ py/__pycache__
|
||||
py/modelverse_api/__pycache__
|
||||
py/modelverse_api/requests/__pycache__
|
||||
/venv
|
||||
.idea/
|
||||
.idea/
|
||||
.ace-tool/
|
||||
docs
|
||||
@@ -0,0 +1,149 @@
|
||||
"""
|
||||
Gemini 3 Pro Image Preview (Nano Banana Pro) ComfyUI Node.
|
||||
Professional asset production with advanced features:
|
||||
- High-resolution output (1K, 2K, 4K)
|
||||
- Aspect ratio control
|
||||
- Google Search grounding
|
||||
- Up to 14 reference images
|
||||
"""
|
||||
import base64
|
||||
import torch
|
||||
from typing import Optional, List, Dict, Any
|
||||
from comfy.comfy_types.node_typing import IO
|
||||
|
||||
from .modelverse_api.client import ModelverseClient
|
||||
from .modelverse_api.requests.gemini_pro_image import GeminiProImageRequest
|
||||
from .modelverse_api.utils import decode_image, images2tensor
|
||||
|
||||
|
||||
def _extract_images_from_gemini_response(resp: Dict[str, Any]) -> List[torch.Tensor]:
|
||||
"""Extract image tensors from Gemini API response."""
|
||||
tensors: List[torch.Tensor] = []
|
||||
candidates = resp.get("candidates", [])
|
||||
for cand in candidates:
|
||||
content = cand.get("content", {})
|
||||
parts = content.get("parts", [])
|
||||
for part in parts:
|
||||
inline = part.get("inlineData")
|
||||
if inline:
|
||||
data_b64 = inline.get("data")
|
||||
if data_b64:
|
||||
raw = base64.b64decode(data_b64)
|
||||
pil_img = decode_image(raw)
|
||||
tensors.append(images2tensor(pil_img))
|
||||
# Also print any text response
|
||||
if part.get("text"):
|
||||
print("INFO: Gemini Pro Image text response:", part.get("text"))
|
||||
return tensors
|
||||
|
||||
|
||||
# Aspect ratio options
|
||||
ASPECT_RATIOS = ["auto", "1:1", "2:3", "3:2", "3:4", "4:3", "4:5", "5:4", "9:16", "16:9", "21:9"]
|
||||
# Resolution options
|
||||
IMAGE_SIZES = ["1K", "2K", "4K"]
|
||||
|
||||
|
||||
class GeminiProImageNode:
|
||||
"""
|
||||
Gemini 3 Pro Image Preview (Nano Banana Pro) via Modelverse API.
|
||||
|
||||
Professional asset production with:
|
||||
- High-resolution output (1K, 2K, 4K)
|
||||
- Aspect ratio control
|
||||
- Google Search grounding for real-time info
|
||||
- Up to 14 reference images
|
||||
|
||||
Endpoint: /v1beta/models/gemini-3-pro-image-preview:generateContent
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"client": ("MODELVERSE_API_CLIENT",),
|
||||
"prompt": (IO.STRING, {"multiline": True, "default": "Create a professional product photo"}),
|
||||
"mime_type": (["image/png", "image/jpeg"], {"default": "image/png"}),
|
||||
"aspect_ratio": (ASPECT_RATIOS, {"default": "auto"}),
|
||||
"image_size": (IMAGE_SIZES, {"default": "1K"}),
|
||||
"use_google_search": (IO.BOOLEAN, {"default": False, "tooltip": "Enable Google Search grounding for real-time info"}),
|
||||
"num_requests": (IO.INT, {"default": 1, "min": 1, "max": 10, "step": 1, "display": "number"}),
|
||||
},
|
||||
"optional": {
|
||||
"image1": (IO.IMAGE, {"default": None, "tooltip": "Reference image 1"}),
|
||||
"image2": (IO.IMAGE, {"default": None, "tooltip": "Reference image 2"}),
|
||||
"image3": (IO.IMAGE, {"default": None, "tooltip": "Reference image 3"}),
|
||||
"image4": (IO.IMAGE, {"default": None, "tooltip": "Reference image 4"}),
|
||||
"image5": (IO.IMAGE, {"default": None, "tooltip": "Reference image 5"}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = (IO.IMAGE,)
|
||||
RETURN_NAMES = ("image",)
|
||||
CATEGORY = "UCLOUD_MODELVERSE"
|
||||
FUNCTION = "execute"
|
||||
|
||||
async def execute(
|
||||
self,
|
||||
client,
|
||||
prompt: str,
|
||||
mime_type: str = "image/png",
|
||||
aspect_ratio: str = "auto",
|
||||
image_size: str = "1K",
|
||||
use_google_search: bool = False,
|
||||
num_requests: int = 1,
|
||||
image1=None,
|
||||
image2=None,
|
||||
image3=None,
|
||||
image4=None,
|
||||
image5=None,
|
||||
):
|
||||
if not prompt:
|
||||
raise ValueError("Prompt is required")
|
||||
|
||||
mv_client = ModelverseClient(client["api_key"])
|
||||
|
||||
# Collect input images
|
||||
images = []
|
||||
for img in [image1, image2, image3, image4, image5]:
|
||||
if img is not None:
|
||||
images.append(img)
|
||||
|
||||
# Process aspect_ratio
|
||||
ar = aspect_ratio if aspect_ratio != "auto" else None
|
||||
|
||||
outputs: List[torch.Tensor] = []
|
||||
for i in range(num_requests):
|
||||
req = GeminiProImageRequest(
|
||||
prompt=prompt,
|
||||
images=images if images else None,
|
||||
mime_type=mime_type,
|
||||
aspect_ratio=ar,
|
||||
image_size=image_size,
|
||||
use_google_search=use_google_search,
|
||||
)
|
||||
payload = req.build_payload()
|
||||
resp = mv_client.post(req.API_PATH, payload)
|
||||
|
||||
if isinstance(resp, dict) and resp.get("error"):
|
||||
err = resp.get("error")
|
||||
raise Exception(f"GeminiProImage error: {err.get('message', 'Unknown error')}")
|
||||
|
||||
tensors = _extract_images_from_gemini_response(resp)
|
||||
if not tensors:
|
||||
print("WARN:", "No image data found in Gemini Pro response; check console for details.")
|
||||
continue
|
||||
outputs.append(tensors[0])
|
||||
|
||||
if not outputs:
|
||||
raise Exception("No images generated from Gemini 3 Pro Image Preview")
|
||||
|
||||
return (torch.cat(outputs, dim=0),)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
'Gemini Pro Image (Nano Banana Pro)': GeminiProImageNode,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
'Gemini Pro Image (Nano Banana Pro)': 'Gemini 3 Pro Image Preview',
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
"""
|
||||
Gemini 3 Pro Image Preview (Nano Banana Pro) request builder.
|
||||
Supports text-to-image and image editing with advanced features like
|
||||
aspect ratio, resolution control, and Google Search grounding.
|
||||
"""
|
||||
import io
|
||||
import base64
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
from typing import Optional, List, Dict, Any
|
||||
from pydantic import Field
|
||||
from torch import Tensor
|
||||
|
||||
from ..utils import BaseRequest
|
||||
|
||||
|
||||
def _tensor_to_base64(image: Tensor, mime_type: str = "image/png") -> Dict[str, str]:
|
||||
"""Convert a ComfyUI image tensor (HWC or 4D with batch) to base64 without data URI."""
|
||||
if image is None:
|
||||
return None
|
||||
|
||||
# If batch, take first frame
|
||||
if hasattr(image, 'shape') and len(image.shape) == 4:
|
||||
image = image[0]
|
||||
|
||||
# Tensor (H, W, C) in 0..1 -> uint8
|
||||
np_img = np.clip(255.0 * image.cpu().numpy(), 0, 255).astype(np.uint8)
|
||||
pil_img = Image.fromarray(np_img)
|
||||
|
||||
fmt = 'PNG' if mime_type.lower().endswith('png') else 'JPEG'
|
||||
with io.BytesIO() as bio:
|
||||
pil_img.save(bio, format=fmt)
|
||||
data = bio.getvalue()
|
||||
|
||||
return {
|
||||
"mimeType": mime_type,
|
||||
"data": base64.b64encode(data).decode("utf-8"),
|
||||
}
|
||||
|
||||
|
||||
class GeminiProImageRequest(BaseRequest):
|
||||
"""
|
||||
Request builder for Gemini 3 Pro Image Preview (Nano Banana Pro) generateContent endpoint.
|
||||
Supports text-to-image, image editing, and advanced features like aspect ratio,
|
||||
resolution control (1K/2K/4K), and Google Search grounding.
|
||||
"""
|
||||
|
||||
API_PATH = "/v1beta/models/gemini-3-pro-image-preview:generateContent"
|
||||
|
||||
prompt: str = Field(..., description="Text prompt")
|
||||
images: Optional[List[Tensor]] = Field(default=None, description="Optional input images (up to 14)")
|
||||
mime_type: str = Field(default="image/png", description="MIME type for inline image data")
|
||||
aspect_ratio: Optional[str] = Field(default=None, description="Aspect ratio: 1:1, 2:3, 3:2, 3:4, 4:3, 4:5, 5:4, 9:16, 16:9, 21:9")
|
||||
image_size: Optional[str] = Field(default=None, description="Image resolution: 1K, 2K, 4K")
|
||||
use_google_search: bool = Field(default=False, description="Enable Google Search grounding")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
prompt: str,
|
||||
images: Optional[List[Tensor]] = None,
|
||||
mime_type: str = "image/png",
|
||||
aspect_ratio: Optional[str] = None,
|
||||
image_size: Optional[str] = None,
|
||||
use_google_search: bool = False,
|
||||
**kwargs
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self.prompt = prompt
|
||||
self.images = images
|
||||
self.mime_type = mime_type
|
||||
self.aspect_ratio = aspect_ratio
|
||||
self.image_size = image_size
|
||||
self.use_google_search = use_google_search
|
||||
|
||||
def build_payload(self) -> Dict[str, Any]:
|
||||
parts: List[Dict[str, Any]] = []
|
||||
|
||||
# Add text prompt
|
||||
if self.prompt:
|
||||
parts.append({"text": self.prompt})
|
||||
|
||||
# Add images (up to 14)
|
||||
if self.images:
|
||||
for img in self.images[:14]: # Limit to 14 images
|
||||
if isinstance(img, Tensor):
|
||||
parts.append({
|
||||
"inlineData": _tensor_to_base64(img, self.mime_type)
|
||||
})
|
||||
|
||||
payload = {
|
||||
"contents": [
|
||||
{
|
||||
"role": "user",
|
||||
"parts": parts,
|
||||
}
|
||||
],
|
||||
"generationConfig": {
|
||||
"responseModalities": ["TEXT", "IMAGE"],
|
||||
},
|
||||
}
|
||||
|
||||
# Add imageConfig if aspect_ratio or image_size specified
|
||||
if self.aspect_ratio or self.image_size:
|
||||
image_config = {}
|
||||
if self.aspect_ratio:
|
||||
image_config["aspectRatio"] = self.aspect_ratio
|
||||
if self.image_size:
|
||||
image_config["imageSize"] = self.image_size
|
||||
payload["generationConfig"]["imageConfig"] = image_config
|
||||
|
||||
# Add Google Search tool if enabled
|
||||
if self.use_google_search:
|
||||
payload["tools"] = [{"google_search": {}}]
|
||||
|
||||
return self._remove_empty_fields(payload)
|
||||
|
||||
def field_required(self):
|
||||
return ["prompt"]
|
||||
|
||||
def field_order(self):
|
||||
return ["prompt", "images", "mime_type", "aspect_ratio", "image_size", "use_google_search"]
|
||||
@@ -0,0 +1,141 @@
|
||||
"""
|
||||
Vidu Extend - 视频延长模型
|
||||
Models: viduq2-pro, viduq2-turbo
|
||||
"""
|
||||
import time
|
||||
import requests
|
||||
import io
|
||||
from .modelverse_api.client import ModelverseClient
|
||||
from .modelverse_api.utils import image_to_base64
|
||||
from comfy.comfy_types.node_typing import IO
|
||||
|
||||
try:
|
||||
from comfy_extras.nodes_video import VideoFromFile
|
||||
except ImportError:
|
||||
try:
|
||||
from comfy.model_management import VideoFromFile
|
||||
except ImportError:
|
||||
class VideoFromFile:
|
||||
def __init__(self, video_io):
|
||||
self.video_io = video_io
|
||||
self.video_data = video_io.getvalue() if hasattr(video_io, 'getvalue') else video_io
|
||||
|
||||
|
||||
MODELS = ["viduq2-pro", "viduq2-turbo"]
|
||||
RESOLUTIONS = ["540p", "720p", "1080p"]
|
||||
|
||||
|
||||
class ViduExtendNode:
|
||||
"""
|
||||
Vidu Extend - 视频延长
|
||||
Models: viduq2-pro, viduq2-turbo
|
||||
输入视频时长需4秒-1分钟，延长1-7秒
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"client": ("MODELVERSE_API_CLIENT",),
|
||||
"model": (MODELS, {"default": "viduq2-turbo", "tooltip": "viduq2-pro: 效果好, viduq2-turbo: 生成快"}),
|
||||
"video_url": (IO.STRING, {"default": "", "tooltip": "需要延长的视频URL (4秒-1分钟)"}),
|
||||
"duration": (IO.INT, {"default": 5, "min": 1, "max": 7, "step": 1, "tooltip": "延长时长(秒), 1-7"}),
|
||||
"resolution": (RESOLUTIONS, {"default": "720p", "tooltip": "分辨率"}),
|
||||
},
|
||||
"optional": {
|
||||
"last_frame_image": (IO.IMAGE, {"tooltip": "延长到尾帧的参考图像"}),
|
||||
"last_frame_url": (IO.STRING, {"default": "", "tooltip": "延长到尾帧的参考图像URL"}),
|
||||
"prompt": (IO.STRING, {"multiline": True, "default": "", "tooltip": "延长提示词，控制延长内容"}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = (IO.VIDEO,)
|
||||
RETURN_NAMES = ("video",)
|
||||
FUNCTION = "generate"
|
||||
CATEGORY = "UCLOUD_MODELVERSE/Vidu"
|
||||
|
||||
def generate(self, client, model, video_url, duration, resolution,
|
||||
last_frame_image=None, last_frame_url="", prompt=""):
|
||||
api_key = client.get("api_key")
|
||||
if not api_key:
|
||||
raise ValueError("API key is not set")
|
||||
|
||||
if not video_url or not video_url.strip():
|
||||
raise ValueError("必须提供 video_url")
|
||||
|
||||
mv_client = ModelverseClient(api_key)
|
||||
|
||||
task_input = {
|
||||
"video_url": video_url.strip(),
|
||||
}
|
||||
|
||||
# Optional last frame
|
||||
has_last_url = last_frame_url and last_frame_url.strip()
|
||||
has_last_image = last_frame_image is not None
|
||||
if has_last_url and has_last_image:
|
||||
raise ValueError("尾帧：请提供 image 或 url，不能同时提供")
|
||||
|
||||
if has_last_url:
|
||||
task_input["last_frame_url"] = last_frame_url.strip()
|
||||
elif has_last_image:
|
||||
task_input["last_frame_url"] = image_to_base64(last_frame_image)
|
||||
|
||||
if prompt and prompt.strip():
|
||||
task_input["prompt"] = prompt.strip()
|
||||
|
||||
parameters = {
|
||||
"vidu_type": "extend",
|
||||
"duration": duration,
|
||||
"resolution": resolution,
|
||||
}
|
||||
|
||||
# Submit task
|
||||
submit_res = mv_client.submit_task(model, task_input, parameters)
|
||||
task_id = submit_res.get("output", {}).get("task_id")
|
||||
if not task_id:
|
||||
raise Exception(f"Failed to submit task: {submit_res}")
|
||||
|
||||
print(f"Vidu Extend task submitted: {task_id}")
|
||||
|
||||
# Poll for result
|
||||
video_url_result = self._poll_task(mv_client, task_id)
|
||||
|
||||
# Download and return video
|
||||
return self._download_video(video_url_result)
|
||||
|
||||
def _poll_task(self, mv_client, task_id, max_retries=180):
|
||||
for i in range(max_retries):
|
||||
status_res = mv_client.get_task_status(task_id)
|
||||
task_status = status_res.get("output", {}).get("task_status")
|
||||
|
||||
if task_status == "Success":
|
||||
urls = status_res.get("output", {}).get("urls", [])
|
||||
if urls:
|
||||
return urls[0]
|
||||
raise Exception("Task succeeded but no video URL returned")
|
||||
elif task_status == "Failure":
|
||||
error = status_res.get("output", {}).get("error_message", "Unknown error")
|
||||
raise Exception(f"Task failed: {error}")
|
||||
elif task_status in ["Pending", "Running"]:
|
||||
print(f"Task {task_id}: {task_status} ({i+1}/{max_retries})")
|
||||
time.sleep(5)
|
||||
else:
|
||||
raise Exception(f"Unknown status: {task_status}")
|
||||
|
||||
raise Exception("Task timed out")
|
||||
|
||||
def _download_video(self, video_url):
|
||||
response = requests.get(video_url, timeout=120)
|
||||
response.raise_for_status()
|
||||
video_io = io.BytesIO(response.content)
|
||||
video_io.seek(0)
|
||||
return (VideoFromFile(video_io),)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"Vidu_Extend": ViduExtendNode,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"Vidu_Extend": "Vidu Extend",
|
||||
}
|
||||
+143
@@ -0,0 +1,143 @@
|
||||
"""
|
||||
Vidu Img2Video - 图生视频模型
|
||||
Models: viduq2-pro, viduq2-turbo
|
||||
"""
|
||||
import time
|
||||
import requests
|
||||
import io
|
||||
from .modelverse_api.client import ModelverseClient
|
||||
from .modelverse_api.utils import image_to_base64
|
||||
from comfy.comfy_types.node_typing import IO
|
||||
|
||||
try:
|
||||
from comfy_extras.nodes_video import VideoFromFile
|
||||
except ImportError:
|
||||
try:
|
||||
from comfy.model_management import VideoFromFile
|
||||
except ImportError:
|
||||
class VideoFromFile:
|
||||
def __init__(self, video_io):
|
||||
self.video_io = video_io
|
||||
self.video_data = video_io.getvalue() if hasattr(video_io, 'getvalue') else video_io
|
||||
|
||||
|
||||
MODELS = ["viduq2-pro", "viduq2-turbo"]
|
||||
RESOLUTIONS = ["540p", "720p", "1080p"]
|
||||
MOVEMENT_AMPLITUDES = ["auto", "small", "medium", "large"]
|
||||
|
||||
|
||||
class ViduImg2VideoNode:
|
||||
"""
|
||||
Vidu Img2Video - 图生视频
|
||||
Models: viduq2-pro (效果好，细节丰富), viduq2-turbo (效果好，生成快)
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"client": ("MODELVERSE_API_CLIENT",),
|
||||
"model": (MODELS, {"default": "viduq2-pro", "tooltip": "viduq2-pro: 效果好细节丰富, viduq2-turbo: 生成快"}),
|
||||
"duration": (IO.INT, {"default": 5, "min": 1, "max": 10, "step": 1, "tooltip": "视频时长(秒)"}),
|
||||
"resolution": (RESOLUTIONS, {"default": "720p", "tooltip": "分辨率"}),
|
||||
"movement_amplitude": (MOVEMENT_AMPLITUDES, {"default": "auto", "tooltip": "运动幅度"}),
|
||||
},
|
||||
"optional": {
|
||||
"first_frame_image": (IO.IMAGE, {"tooltip": "首帧图片"}),
|
||||
"first_frame_url": (IO.STRING, {"default": "", "tooltip": "首帧图片URL (与image二选一)"}),
|
||||
"prompt": (IO.STRING, {"multiline": True, "default": "", "tooltip": "文本提示词，最长2000字符"}),
|
||||
"seed": (IO.INT, {"default": 0, "min": 0, "max": 2147483647, "tooltip": "随机种子"}),
|
||||
"bgm": (IO.BOOLEAN, {"default": False, "tooltip": "是否添加背景音乐"}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = (IO.VIDEO,)
|
||||
RETURN_NAMES = ("video",)
|
||||
FUNCTION = "generate"
|
||||
CATEGORY = "UCLOUD_MODELVERSE/Vidu"
|
||||
|
||||
def generate(self, client, model, duration, resolution, movement_amplitude,
|
||||
first_frame_image=None, first_frame_url="", prompt="", seed=0, bgm=False):
|
||||
api_key = client.get("api_key")
|
||||
if not api_key:
|
||||
raise ValueError("API key is not set")
|
||||
|
||||
mv_client = ModelverseClient(api_key)
|
||||
|
||||
# Validate first frame input
|
||||
has_url = first_frame_url and first_frame_url.strip()
|
||||
has_image = first_frame_image is not None
|
||||
|
||||
if has_url and has_image:
|
||||
raise ValueError("请提供 first_frame_image 或 first_frame_url，不能同时提供")
|
||||
if not has_url and not has_image:
|
||||
raise ValueError("必须提供 first_frame_image 或 first_frame_url")
|
||||
|
||||
task_input = {}
|
||||
if has_url:
|
||||
task_input["first_frame_url"] = first_frame_url.strip()
|
||||
else:
|
||||
task_input["first_frame_url"] = image_to_base64(first_frame_image)
|
||||
|
||||
if prompt and prompt.strip():
|
||||
task_input["prompt"] = prompt.strip()
|
||||
|
||||
parameters = {
|
||||
"vidu_type": "img2video",
|
||||
"duration": duration,
|
||||
"resolution": resolution,
|
||||
"movement_amplitude": movement_amplitude,
|
||||
"seed": seed,
|
||||
"bgm": bgm,
|
||||
}
|
||||
|
||||
# Submit task
|
||||
submit_res = mv_client.submit_task(model, task_input, parameters)
|
||||
task_id = submit_res.get("output", {}).get("task_id")
|
||||
if not task_id:
|
||||
raise Exception(f"Failed to submit task: {submit_res}")
|
||||
|
||||
print(f"Vidu I2V task submitted: {task_id}")
|
||||
|
||||
# Poll for result
|
||||
video_url = self._poll_task(mv_client, task_id)
|
||||
|
||||
# Download and return video
|
||||
return self._download_video(video_url)
|
||||
|
||||
def _poll_task(self, mv_client, task_id, max_retries=180):
|
||||
for i in range(max_retries):
|
||||
status_res = mv_client.get_task_status(task_id)
|
||||
task_status = status_res.get("output", {}).get("task_status")
|
||||
|
||||
if task_status == "Success":
|
||||
urls = status_res.get("output", {}).get("urls", [])
|
||||
if urls:
|
||||
return urls[0]
|
||||
raise Exception("Task succeeded but no video URL returned")
|
||||
elif task_status == "Failure":
|
||||
error = status_res.get("output", {}).get("error_message", "Unknown error")
|
||||
raise Exception(f"Task failed: {error}")
|
||||
elif task_status in ["Pending", "Running"]:
|
||||
print(f"Task {task_id}: {task_status} ({i+1}/{max_retries})")
|
||||
time.sleep(5)
|
||||
else:
|
||||
raise Exception(f"Unknown status: {task_status}")
|
||||
|
||||
raise Exception("Task timed out")
|
||||
|
||||
def _download_video(self, video_url):
|
||||
response = requests.get(video_url, timeout=120)
|
||||
response.raise_for_status()
|
||||
video_io = io.BytesIO(response.content)
|
||||
video_io.seek(0)
|
||||
return (VideoFromFile(video_io),)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"Vidu_Img2Video": ViduImg2VideoNode,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"Vidu_Img2Video": "Vidu Img2Video",
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
"""
|
||||
Vidu Reference2Video - 参考生视频模型
|
||||
Model: viduq2
|
||||
支持1-7张参考图片，生成具备主体一致的视频
|
||||
"""
|
||||
import time
|
||||
import requests
|
||||
import io
|
||||
from .modelverse_api.client import ModelverseClient
|
||||
from .modelverse_api.utils import image_to_base64
|
||||
from comfy.comfy_types.node_typing import IO
|
||||
|
||||
try:
|
||||
from comfy_extras.nodes_video import VideoFromFile
|
||||
except ImportError:
|
||||
try:
|
||||
from comfy.model_management import VideoFromFile
|
||||
except ImportError:
|
||||
class VideoFromFile:
|
||||
def __init__(self, video_io):
|
||||
self.video_io = video_io
|
||||
self.video_data = video_io.getvalue() if hasattr(video_io, 'getvalue') else video_io
|
||||
|
||||
|
||||
ASPECT_RATIOS = ["16:9", "9:16", "3:4", "4:3", "1:1"]
|
||||
RESOLUTIONS = ["540p", "720p", "1080p"]
|
||||
|
||||
|
||||
class ViduReference2VideoNode:
|
||||
"""
|
||||
Vidu Reference2Video - 参考生视频
|
||||
Model: viduq2
|
||||
支持1-7张参考图片，生成具备主体一致的视频
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"client": ("MODELVERSE_API_CLIENT",),
|
||||
"prompt": (IO.STRING, {"multiline": True, "default": "make it dance", "tooltip": "文本提示词，最长2000字符"}),
|
||||
"duration": (IO.INT, {"default": 5, "min": 1, "max": 10, "step": 1, "tooltip": "视频时长(秒)"}),
|
||||
"aspect_ratio": (ASPECT_RATIOS, {"default": "16:9", "tooltip": "长宽比"}),
|
||||
"resolution": (RESOLUTIONS, {"default": "720p", "tooltip": "分辨率"}),
|
||||
},
|
||||
"optional": {
|
||||
"image1": (IO.IMAGE, {"tooltip": "参考图片1"}),
|
||||
"image2": (IO.IMAGE, {"tooltip": "参考图片2"}),
|
||||
"image3": (IO.IMAGE, {"tooltip": "参考图片3"}),
|
||||
"image4": (IO.IMAGE, {"tooltip": "参考图片4"}),
|
||||
"image5": (IO.IMAGE, {"tooltip": "参考图片5"}),
|
||||
"image6": (IO.IMAGE, {"tooltip": "参考图片6"}),
|
||||
"image7": (IO.IMAGE, {"tooltip": "参考图片7"}),
|
||||
"image_urls": (IO.STRING, {"default": "", "multiline": True, "tooltip": "参考图片URL列表，每行一个"}),
|
||||
"seed": (IO.INT, {"default": 0, "min": 0, "max": 2147483647, "tooltip": "随机种子"}),
|
||||
"bgm": (IO.BOOLEAN, {"default": False, "tooltip": "是否添加背景音乐"}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = (IO.VIDEO,)
|
||||
RETURN_NAMES = ("video",)
|
||||
FUNCTION = "generate"
|
||||
CATEGORY = "UCLOUD_MODELVERSE/Vidu"
|
||||
|
||||
def generate(self, client, prompt, duration, aspect_ratio, resolution,
|
||||
image1=None, image2=None, image3=None, image4=None,
|
||||
image5=None, image6=None, image7=None,
|
||||
image_urls="", seed=0, bgm=False):
|
||||
api_key = client.get("api_key")
|
||||
if not api_key:
|
||||
raise ValueError("API key is not set")
|
||||
|
||||
mv_client = ModelverseClient(api_key)
|
||||
|
||||
# Collect images
|
||||
images = []
|
||||
for img in [image1, image2, image3, image4, image5, image6, image7]:
|
||||
if img is not None:
|
||||
images.append(image_to_base64(img))
|
||||
|
||||
# Add URL images
|
||||
if image_urls and image_urls.strip():
|
||||
for url in image_urls.strip().split('\n'):
|
||||
url = url.strip()
|
||||
if url:
|
||||
images.append(url)
|
||||
|
||||
if not images:
|
||||
raise ValueError("至少需要提供1张参考图片")
|
||||
if len(images) > 7:
|
||||
print(f"Warning: 最多支持7张参考图片，当前{len(images)}张，将只使用前7张")
|
||||
images = images[:7]
|
||||
|
||||
task_input = {
|
||||
"images": images,
|
||||
"prompt": prompt,
|
||||
}
|
||||
|
||||
parameters = {
|
||||
"vidu_type": "reference2video",
|
||||
"duration": duration,
|
||||
"aspect_ratio": aspect_ratio,
|
||||
"resolution": resolution,
|
||||
"seed": seed,
|
||||
"bgm": bgm,
|
||||
}
|
||||
|
||||
# Submit task
|
||||
submit_res = mv_client.submit_task("viduq2", task_input, parameters)
|
||||
task_id = submit_res.get("output", {}).get("task_id")
|
||||
if not task_id:
|
||||
raise Exception(f"Failed to submit task: {submit_res}")
|
||||
|
||||
print(f"Vidu Ref2V task submitted: {task_id}")
|
||||
|
||||
# Poll for result
|
||||
video_url = self._poll_task(mv_client, task_id)
|
||||
|
||||
# Download and return video
|
||||
return self._download_video(video_url)
|
||||
|
||||
def _poll_task(self, mv_client, task_id, max_retries=180):
|
||||
for i in range(max_retries):
|
||||
status_res = mv_client.get_task_status(task_id)
|
||||
task_status = status_res.get("output", {}).get("task_status")
|
||||
|
||||
if task_status == "Success":
|
||||
urls = status_res.get("output", {}).get("urls", [])
|
||||
if urls:
|
||||
return urls[0]
|
||||
raise Exception("Task succeeded but no video URL returned")
|
||||
elif task_status == "Failure":
|
||||
error = status_res.get("output", {}).get("error_message", "Unknown error")
|
||||
raise Exception(f"Task failed: {error}")
|
||||
elif task_status in ["Pending", "Running"]:
|
||||
print(f"Task {task_id}: {task_status} ({i+1}/{max_retries})")
|
||||
time.sleep(5)
|
||||
else:
|
||||
raise Exception(f"Unknown status: {task_status}")
|
||||
|
||||
raise Exception("Task timed out")
|
||||
|
||||
def _download_video(self, video_url):
|
||||
response = requests.get(video_url, timeout=120)
|
||||
response.raise_for_status()
|
||||
video_io = io.BytesIO(response.content)
|
||||
video_io.seek(0)
|
||||
return (VideoFromFile(video_io),)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"Vidu_Reference2Video": ViduReference2VideoNode,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"Vidu_Reference2Video": "Vidu Reference2Video",
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
"""
|
||||
Vidu StartEnd2Video - 首尾帧生视频模型
|
||||
Models: viduq2-pro, viduq2-turbo
|
||||
"""
|
||||
import time
|
||||
import requests
|
||||
import io
|
||||
from .modelverse_api.client import ModelverseClient
|
||||
from .modelverse_api.utils import image_to_base64
|
||||
from comfy.comfy_types.node_typing import IO
|
||||
|
||||
try:
|
||||
from comfy_extras.nodes_video import VideoFromFile
|
||||
except ImportError:
|
||||
try:
|
||||
from comfy.model_management import VideoFromFile
|
||||
except ImportError:
|
||||
class VideoFromFile:
|
||||
def __init__(self, video_io):
|
||||
self.video_io = video_io
|
||||
self.video_data = video_io.getvalue() if hasattr(video_io, 'getvalue') else video_io
|
||||
|
||||
|
||||
MODELS = ["viduq2-pro", "viduq2-turbo"]
|
||||
RESOLUTIONS = ["540p", "720p", "1080p"]
|
||||
MOVEMENT_AMPLITUDES = ["auto", "small", "medium", "large"]
|
||||
|
||||
|
||||
class ViduStartEnd2VideoNode:
|
||||
"""
|
||||
Vidu StartEnd2Video - 首尾帧生视频
|
||||
Models: viduq2-pro (效果好，细节丰富), viduq2-turbo (效果好，生成快)
|
||||
注意：首尾帧分辨率需相近 (0.8~1.25)
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"client": ("MODELVERSE_API_CLIENT",),
|
||||
"model": (MODELS, {"default": "viduq2-pro", "tooltip": "viduq2-pro: 效果好细节丰富, viduq2-turbo: 生成快"}),
|
||||
"duration": (IO.INT, {"default": 5, "min": 1, "max": 8, "step": 1, "tooltip": "视频时长(秒), 1-8"}),
|
||||
"resolution": (RESOLUTIONS, {"default": "720p", "tooltip": "分辨率"}),
|
||||
"movement_amplitude": (MOVEMENT_AMPLITUDES, {"default": "auto", "tooltip": "运动幅度"}),
|
||||
},
|
||||
"optional": {
|
||||
"first_frame_image": (IO.IMAGE, {"tooltip": "首帧图片"}),
|
||||
"first_frame_url": (IO.STRING, {"default": "", "tooltip": "首帧图片URL"}),
|
||||
"last_frame_image": (IO.IMAGE, {"tooltip": "尾帧图片"}),
|
||||
"last_frame_url": (IO.STRING, {"default": "", "tooltip": "尾帧图片URL"}),
|
||||
"prompt": (IO.STRING, {"multiline": True, "default": "", "tooltip": "文本提示词，最长2000字符"}),
|
||||
"seed": (IO.INT, {"default": 0, "min": 0, "max": 2147483647, "tooltip": "随机种子"}),
|
||||
"bgm": (IO.BOOLEAN, {"default": False, "tooltip": "是否添加背景音乐"}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = (IO.VIDEO,)
|
||||
RETURN_NAMES = ("video",)
|
||||
FUNCTION = "generate"
|
||||
CATEGORY = "UCLOUD_MODELVERSE/Vidu"
|
||||
|
||||
def generate(self, client, model, duration, resolution, movement_amplitude,
|
||||
first_frame_image=None, first_frame_url="",
|
||||
last_frame_image=None, last_frame_url="",
|
||||
prompt="", seed=0, bgm=False):
|
||||
api_key = client.get("api_key")
|
||||
if not api_key:
|
||||
raise ValueError("API key is not set")
|
||||
|
||||
mv_client = ModelverseClient(api_key)
|
||||
|
||||
# Validate first frame
|
||||
has_first_url = first_frame_url and first_frame_url.strip()
|
||||
has_first_image = first_frame_image is not None
|
||||
if has_first_url and has_first_image:
|
||||
raise ValueError("首帧：请提供 image 或 url，不能同时提供")
|
||||
if not has_first_url and not has_first_image:
|
||||
raise ValueError("必须提供首帧图片")
|
||||
|
||||
# Validate last frame
|
||||
has_last_url = last_frame_url and last_frame_url.strip()
|
||||
has_last_image = last_frame_image is not None
|
||||
if has_last_url and has_last_image:
|
||||
raise ValueError("尾帧：请提供 image 或 url，不能同时提供")
|
||||
if not has_last_url and not has_last_image:
|
||||
raise ValueError("必须提供尾帧图片")
|
||||
|
||||
task_input = {}
|
||||
|
||||
# First frame
|
||||
if has_first_url:
|
||||
task_input["first_frame_url"] = first_frame_url.strip()
|
||||
else:
|
||||
task_input["first_frame_url"] = image_to_base64(first_frame_image)
|
||||
|
||||
# Last frame
|
||||
if has_last_url:
|
||||
task_input["last_frame_url"] = last_frame_url.strip()
|
||||
else:
|
||||
task_input["last_frame_url"] = image_to_base64(last_frame_image)
|
||||
|
||||
if prompt and prompt.strip():
|
||||
task_input["prompt"] = prompt.strip()
|
||||
|
||||
parameters = {
|
||||
"vidu_type": "start-end2video",
|
||||
"duration": duration,
|
||||
"resolution": resolution,
|
||||
"movement_amplitude": movement_amplitude,
|
||||
"seed": seed,
|
||||
"bgm": bgm,
|
||||
}
|
||||
|
||||
# Submit task
|
||||
submit_res = mv_client.submit_task(model, task_input, parameters)
|
||||
task_id = submit_res.get("output", {}).get("task_id")
|
||||
if not task_id:
|
||||
raise Exception(f"Failed to submit task: {submit_res}")
|
||||
|
||||
print(f"Vidu StartEnd2V task submitted: {task_id}")
|
||||
|
||||
# Poll for result
|
||||
video_url = self._poll_task(mv_client, task_id)
|
||||
|
||||
# Download and return video
|
||||
return self._download_video(video_url)
|
||||
|
||||
def _poll_task(self, mv_client, task_id, max_retries=180):
|
||||
for i in range(max_retries):
|
||||
status_res = mv_client.get_task_status(task_id)
|
||||
task_status = status_res.get("output", {}).get("task_status")
|
||||
|
||||
if task_status == "Success":
|
||||
urls = status_res.get("output", {}).get("urls", [])
|
||||
if urls:
|
||||
return urls[0]
|
||||
raise Exception("Task succeeded but no video URL returned")
|
||||
elif task_status == "Failure":
|
||||
error = status_res.get("output", {}).get("error_message", "Unknown error")
|
||||
raise Exception(f"Task failed: {error}")
|
||||
elif task_status in ["Pending", "Running"]:
|
||||
print(f"Task {task_id}: {task_status} ({i+1}/{max_retries})")
|
||||
time.sleep(5)
|
||||
else:
|
||||
raise Exception(f"Unknown status: {task_status}")
|
||||
|
||||
raise Exception("Task timed out")
|
||||
|
||||
def _download_video(self, video_url):
|
||||
response = requests.get(video_url, timeout=120)
|
||||
response.raise_for_status()
|
||||
video_io = io.BytesIO(response.content)
|
||||
video_io.seek(0)
|
||||
return (VideoFromFile(video_io),)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"Vidu_StartEnd2Video": ViduStartEnd2VideoNode,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"Vidu_StartEnd2Video": "Vidu StartEnd2Video",
|
||||
}
|
||||
+122
@@ -0,0 +1,122 @@
|
||||
"""
|
||||
Vidu Text2Video - 文生视频模型
|
||||
Model: viduq2
|
||||
"""
|
||||
import time
|
||||
import requests
|
||||
import io
|
||||
from .modelverse_api.client import ModelverseClient
|
||||
from comfy.comfy_types.node_typing import IO
|
||||
|
||||
try:
|
||||
from comfy_extras.nodes_video import VideoFromFile
|
||||
except ImportError:
|
||||
try:
|
||||
from comfy.model_management import VideoFromFile
|
||||
except ImportError:
|
||||
class VideoFromFile:
|
||||
def __init__(self, video_io):
|
||||
self.video_io = video_io
|
||||
self.video_data = video_io.getvalue() if hasattr(video_io, 'getvalue') else video_io
|
||||
|
||||
|
||||
ASPECT_RATIOS = ["16:9", "9:16", "3:4", "4:3", "1:1"]
|
||||
RESOLUTIONS = ["540p", "720p", "1080p"]
|
||||
|
||||
|
||||
class ViduText2VideoNode:
|
||||
"""
|
||||
Vidu Text2Video - 文生视频
|
||||
Model: viduq2
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"client": ("MODELVERSE_API_CLIENT",),
|
||||
"prompt": (IO.STRING, {"multiline": True, "default": "A beautiful sunset over the ocean", "tooltip": "文本提示词，最长2000字符"}),
|
||||
"duration": (IO.INT, {"default": 5, "min": 1, "max": 10, "step": 1, "tooltip": "视频时长(秒)"}),
|
||||
"aspect_ratio": (ASPECT_RATIOS, {"default": "16:9", "tooltip": "长宽比"}),
|
||||
"resolution": (RESOLUTIONS, {"default": "720p", "tooltip": "分辨率"}),
|
||||
},
|
||||
"optional": {
|
||||
"seed": (IO.INT, {"default": 0, "min": 0, "max": 2147483647, "tooltip": "随机种子，0表示随机"}),
|
||||
"guidance_scale": (IO.FLOAT, {"default": 7.5, "min": 1.0, "max": 20.0, "step": 0.5, "tooltip": "引导系数"}),
|
||||
"bgm": (IO.BOOLEAN, {"default": False, "tooltip": "是否添加背景音乐"}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = (IO.VIDEO,)
|
||||
RETURN_NAMES = ("video",)
|
||||
FUNCTION = "generate"
|
||||
CATEGORY = "UCLOUD_MODELVERSE/Vidu"
|
||||
|
||||
def generate(self, client, prompt, duration, aspect_ratio, resolution, seed=0, guidance_scale=7.5, bgm=False):
|
||||
api_key = client.get("api_key")
|
||||
if not api_key:
|
||||
raise ValueError("API key is not set")
|
||||
|
||||
mv_client = ModelverseClient(api_key)
|
||||
|
||||
task_input = {"prompt": prompt}
|
||||
parameters = {
|
||||
"vidu_type": "text2video",
|
||||
"duration": duration,
|
||||
"aspect_ratio": aspect_ratio,
|
||||
"resolution": resolution,
|
||||
"seed": seed,
|
||||
"guidance_scale": guidance_scale,
|
||||
"bgm": bgm,
|
||||
}
|
||||
|
||||
# Submit task
|
||||
submit_res = mv_client.submit_task("viduq2", task_input, parameters)
|
||||
task_id = submit_res.get("output", {}).get("task_id")
|
||||
if not task_id:
|
||||
raise Exception(f"Failed to submit task: {submit_res}")
|
||||
|
||||
print(f"Vidu T2V task submitted: {task_id}")
|
||||
|
||||
# Poll for result
|
||||
video_url = self._poll_task(mv_client, task_id)
|
||||
|
||||
# Download and return video
|
||||
return self._download_video(video_url)
|
||||
|
||||
def _poll_task(self, mv_client, task_id, max_retries=180):
|
||||
for i in range(max_retries):
|
||||
status_res = mv_client.get_task_status(task_id)
|
||||
task_status = status_res.get("output", {}).get("task_status")
|
||||
|
||||
if task_status == "Success":
|
||||
urls = status_res.get("output", {}).get("urls", [])
|
||||
if urls:
|
||||
return urls[0]
|
||||
raise Exception("Task succeeded but no video URL returned")
|
||||
elif task_status == "Failure":
|
||||
error = status_res.get("output", {}).get("error_message", "Unknown error")
|
||||
raise Exception(f"Task failed: {error}")
|
||||
elif task_status in ["Pending", "Running"]:
|
||||
print(f"Task {task_id}: {task_status} ({i+1}/{max_retries})")
|
||||
time.sleep(5)
|
||||
else:
|
||||
raise Exception(f"Unknown status: {task_status}")
|
||||
|
||||
raise Exception("Task timed out")
|
||||
|
||||
def _download_video(self, video_url):
|
||||
response = requests.get(video_url, timeout=120)
|
||||
response.raise_for_status()
|
||||
video_io = io.BytesIO(response.content)
|
||||
video_io.seek(0)
|
||||
return (VideoFromFile(video_io),)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"Vidu_Text2Video": ViduText2VideoNode,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"Vidu_Text2Video": "Vidu Text2Video",
|
||||
}
|
||||
Reference in New Issue
Block a user