From bd7d00b1ce433ed8945186466145e653ae73f46c Mon Sep 17 00:00:00 2001 From: "jason.mei" Date: Mon, 29 Dec 2025 16:03:39 +0800 Subject: [PATCH] feat(modelverse): add Gemini Pro Image and Vidu video generation support - Add Gemini 3 Pro Image Preview (Nano Banana Pro) ComfyUI node with high-resolution output (1K, 2K, 4K) - Add GeminiProImageRequest builder supporting aspect ratio control, Google Search grounding, and up to 14 reference images - Add Vidu video generation nodes: t2v, i2v, ref2v, and startend2v for comprehensive video synthesis - Add Vidu extend node for video continuation and frame extension - Update .gitignore to exclude .ace-tool/ and docs/ directories - Enable professional asset production with advanced image and video generation capabilities --- .gitignore | 4 +- py/gemini_pro_image.py | 149 ++++++++++++++++ .../requests/gemini_pro_image.py | 121 +++++++++++++ py/vidu_extend.py | 141 +++++++++++++++ py/vidu_i2v.py | 143 +++++++++++++++ py/vidu_ref2v.py | 157 +++++++++++++++++ py/vidu_startend2v.py | 163 ++++++++++++++++++ py/vidu_t2v.py | 122 +++++++++++++ 8 files changed, 999 insertions(+), 1 deletion(-) create mode 100644 py/gemini_pro_image.py create mode 100644 py/modelverse_api/requests/gemini_pro_image.py create mode 100644 py/vidu_extend.py create mode 100644 py/vidu_i2v.py create mode 100644 py/vidu_ref2v.py create mode 100644 py/vidu_startend2v.py create mode 100644 py/vidu_t2v.py diff --git a/.gitignore b/.gitignore index 8779f8d..278884e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,6 @@ py/__pycache__ py/modelverse_api/__pycache__ py/modelverse_api/requests/__pycache__ /venv -.idea/ \ No newline at end of file +.idea/ +.ace-tool/ +docs \ No newline at end of file diff --git a/py/gemini_pro_image.py b/py/gemini_pro_image.py new file mode 100644 index 0000000..f5cbffb --- /dev/null +++ b/py/gemini_pro_image.py @@ -0,0 +1,149 @@ +""" +Gemini 3 Pro Image Preview (Nano Banana Pro) ComfyUI Node. +Professional asset production with advanced features: +- High-resolution output (1K, 2K, 4K) +- Aspect ratio control +- Google Search grounding +- Up to 14 reference images +""" +import base64 +import torch +from typing import Optional, List, Dict, Any +from comfy.comfy_types.node_typing import IO + +from .modelverse_api.client import ModelverseClient +from .modelverse_api.requests.gemini_pro_image import GeminiProImageRequest +from .modelverse_api.utils import decode_image, images2tensor + + +def _extract_images_from_gemini_response(resp: Dict[str, Any]) -> List[torch.Tensor]: + """Extract image tensors from Gemini API response.""" + tensors: List[torch.Tensor] = [] + candidates = resp.get("candidates", []) + for cand in candidates: + content = cand.get("content", {}) + parts = content.get("parts", []) + for part in parts: + inline = part.get("inlineData") + if inline: + data_b64 = inline.get("data") + if data_b64: + raw = base64.b64decode(data_b64) + pil_img = decode_image(raw) + tensors.append(images2tensor(pil_img)) + # Also print any text response + if part.get("text"): + print("INFO: Gemini Pro Image text response:", part.get("text")) + return tensors + + +# Aspect ratio options +ASPECT_RATIOS = ["auto", "1:1", "2:3", "3:2", "3:4", "4:3", "4:5", "5:4", "9:16", "16:9", "21:9"] +# Resolution options +IMAGE_SIZES = ["1K", "2K", "4K"] + + +class GeminiProImageNode: + """ + Gemini 3 Pro Image Preview (Nano Banana Pro) via Modelverse API. + + Professional asset production with: + - High-resolution output (1K, 2K, 4K) + - Aspect ratio control + - Google Search grounding for real-time info + - Up to 14 reference images + + Endpoint: /v1beta/models/gemini-3-pro-image-preview:generateContent + """ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "client": ("MODELVERSE_API_CLIENT",), + "prompt": (IO.STRING, {"multiline": True, "default": "Create a professional product photo"}), + "mime_type": (["image/png", "image/jpeg"], {"default": "image/png"}), + "aspect_ratio": (ASPECT_RATIOS, {"default": "auto"}), + "image_size": (IMAGE_SIZES, {"default": "1K"}), + "use_google_search": (IO.BOOLEAN, {"default": False, "tooltip": "Enable Google Search grounding for real-time info"}), + "num_requests": (IO.INT, {"default": 1, "min": 1, "max": 10, "step": 1, "display": "number"}), + }, + "optional": { + "image1": (IO.IMAGE, {"default": None, "tooltip": "Reference image 1"}), + "image2": (IO.IMAGE, {"default": None, "tooltip": "Reference image 2"}), + "image3": (IO.IMAGE, {"default": None, "tooltip": "Reference image 3"}), + "image4": (IO.IMAGE, {"default": None, "tooltip": "Reference image 4"}), + "image5": (IO.IMAGE, {"default": None, "tooltip": "Reference image 5"}), + }, + } + + RETURN_TYPES = (IO.IMAGE,) + RETURN_NAMES = ("image",) + CATEGORY = "UCLOUD_MODELVERSE" + FUNCTION = "execute" + + async def execute( + self, + client, + prompt: str, + mime_type: str = "image/png", + aspect_ratio: str = "auto", + image_size: str = "1K", + use_google_search: bool = False, + num_requests: int = 1, + image1=None, + image2=None, + image3=None, + image4=None, + image5=None, + ): + if not prompt: + raise ValueError("Prompt is required") + + mv_client = ModelverseClient(client["api_key"]) + + # Collect input images + images = [] + for img in [image1, image2, image3, image4, image5]: + if img is not None: + images.append(img) + + # Process aspect_ratio + ar = aspect_ratio if aspect_ratio != "auto" else None + + outputs: List[torch.Tensor] = [] + for i in range(num_requests): + req = GeminiProImageRequest( + prompt=prompt, + images=images if images else None, + mime_type=mime_type, + aspect_ratio=ar, + image_size=image_size, + use_google_search=use_google_search, + ) + payload = req.build_payload() + resp = mv_client.post(req.API_PATH, payload) + + if isinstance(resp, dict) and resp.get("error"): + err = resp.get("error") + raise Exception(f"GeminiProImage error: {err.get('message', 'Unknown error')}") + + tensors = _extract_images_from_gemini_response(resp) + if not tensors: + print("WARN:", "No image data found in Gemini Pro response; check console for details.") + continue + outputs.append(tensors[0]) + + if not outputs: + raise Exception("No images generated from Gemini 3 Pro Image Preview") + + return (torch.cat(outputs, dim=0),) + + +NODE_CLASS_MAPPINGS = { + 'Gemini Pro Image (Nano Banana Pro)': GeminiProImageNode, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + 'Gemini Pro Image (Nano Banana Pro)': 'Gemini 3 Pro Image Preview', +} diff --git a/py/modelverse_api/requests/gemini_pro_image.py b/py/modelverse_api/requests/gemini_pro_image.py new file mode 100644 index 0000000..68506a2 --- /dev/null +++ b/py/modelverse_api/requests/gemini_pro_image.py @@ -0,0 +1,121 @@ +""" +Gemini 3 Pro Image Preview (Nano Banana Pro) request builder. +Supports text-to-image and image editing with advanced features like +aspect ratio, resolution control, and Google Search grounding. +""" +import io +import base64 +import numpy as np +from PIL import Image +from typing import Optional, List, Dict, Any +from pydantic import Field +from torch import Tensor + +from ..utils import BaseRequest + + +def _tensor_to_base64(image: Tensor, mime_type: str = "image/png") -> Dict[str, str]: + """Convert a ComfyUI image tensor (HWC or 4D with batch) to base64 without data URI.""" + if image is None: + return None + + # If batch, take first frame + if hasattr(image, 'shape') and len(image.shape) == 4: + image = image[0] + + # Tensor (H, W, C) in 0..1 -> uint8 + np_img = np.clip(255.0 * image.cpu().numpy(), 0, 255).astype(np.uint8) + pil_img = Image.fromarray(np_img) + + fmt = 'PNG' if mime_type.lower().endswith('png') else 'JPEG' + with io.BytesIO() as bio: + pil_img.save(bio, format=fmt) + data = bio.getvalue() + + return { + "mimeType": mime_type, + "data": base64.b64encode(data).decode("utf-8"), + } + + +class GeminiProImageRequest(BaseRequest): + """ + Request builder for Gemini 3 Pro Image Preview (Nano Banana Pro) generateContent endpoint. + Supports text-to-image, image editing, and advanced features like aspect ratio, + resolution control (1K/2K/4K), and Google Search grounding. + """ + + API_PATH = "/v1beta/models/gemini-3-pro-image-preview:generateContent" + + prompt: str = Field(..., description="Text prompt") + images: Optional[List[Tensor]] = Field(default=None, description="Optional input images (up to 14)") + mime_type: str = Field(default="image/png", description="MIME type for inline image data") + aspect_ratio: Optional[str] = Field(default=None, description="Aspect ratio: 1:1, 2:3, 3:2, 3:4, 4:3, 4:5, 5:4, 9:16, 16:9, 21:9") + image_size: Optional[str] = Field(default=None, description="Image resolution: 1K, 2K, 4K") + use_google_search: bool = Field(default=False, description="Enable Google Search grounding") + + def __init__( + self, + prompt: str, + images: Optional[List[Tensor]] = None, + mime_type: str = "image/png", + aspect_ratio: Optional[str] = None, + image_size: Optional[str] = None, + use_google_search: bool = False, + **kwargs + ): + super().__init__(**kwargs) + self.prompt = prompt + self.images = images + self.mime_type = mime_type + self.aspect_ratio = aspect_ratio + self.image_size = image_size + self.use_google_search = use_google_search + + def build_payload(self) -> Dict[str, Any]: + parts: List[Dict[str, Any]] = [] + + # Add text prompt + if self.prompt: + parts.append({"text": self.prompt}) + + # Add images (up to 14) + if self.images: + for img in self.images[:14]: # Limit to 14 images + if isinstance(img, Tensor): + parts.append({ + "inlineData": _tensor_to_base64(img, self.mime_type) + }) + + payload = { + "contents": [ + { + "role": "user", + "parts": parts, + } + ], + "generationConfig": { + "responseModalities": ["TEXT", "IMAGE"], + }, + } + + # Add imageConfig if aspect_ratio or image_size specified + if self.aspect_ratio or self.image_size: + image_config = {} + if self.aspect_ratio: + image_config["aspectRatio"] = self.aspect_ratio + if self.image_size: + image_config["imageSize"] = self.image_size + payload["generationConfig"]["imageConfig"] = image_config + + # Add Google Search tool if enabled + if self.use_google_search: + payload["tools"] = [{"google_search": {}}] + + return self._remove_empty_fields(payload) + + def field_required(self): + return ["prompt"] + + def field_order(self): + return ["prompt", "images", "mime_type", "aspect_ratio", "image_size", "use_google_search"] diff --git a/py/vidu_extend.py b/py/vidu_extend.py new file mode 100644 index 0000000..9e094c0 --- /dev/null +++ b/py/vidu_extend.py @@ -0,0 +1,141 @@ +""" +Vidu Extend - 视频延长模型 +Models: viduq2-pro, viduq2-turbo +""" +import time +import requests +import io +from .modelverse_api.client import ModelverseClient +from .modelverse_api.utils import image_to_base64 +from comfy.comfy_types.node_typing import IO + +try: + from comfy_extras.nodes_video import VideoFromFile +except ImportError: + try: + from comfy.model_management import VideoFromFile + except ImportError: + class VideoFromFile: + def __init__(self, video_io): + self.video_io = video_io + self.video_data = video_io.getvalue() if hasattr(video_io, 'getvalue') else video_io + + +MODELS = ["viduq2-pro", "viduq2-turbo"] +RESOLUTIONS = ["540p", "720p", "1080p"] + + +class ViduExtendNode: + """ + Vidu Extend - 视频延长 + Models: viduq2-pro, viduq2-turbo + 输入视频时长需4秒-1分钟，延长1-7秒 + """ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "client": ("MODELVERSE_API_CLIENT",), + "model": (MODELS, {"default": "viduq2-turbo", "tooltip": "viduq2-pro: 效果好, viduq2-turbo: 生成快"}), + "video_url": (IO.STRING, {"default": "", "tooltip": "需要延长的视频URL (4秒-1分钟)"}), + "duration": (IO.INT, {"default": 5, "min": 1, "max": 7, "step": 1, "tooltip": "延长时长(秒), 1-7"}), + "resolution": (RESOLUTIONS, {"default": "720p", "tooltip": "分辨率"}), + }, + "optional": { + "last_frame_image": (IO.IMAGE, {"tooltip": "延长到尾帧的参考图像"}), + "last_frame_url": (IO.STRING, {"default": "", "tooltip": "延长到尾帧的参考图像URL"}), + "prompt": (IO.STRING, {"multiline": True, "default": "", "tooltip": "延长提示词，控制延长内容"}), + } + } + + RETURN_TYPES = (IO.VIDEO,) + RETURN_NAMES = ("video",) + FUNCTION = "generate" + CATEGORY = "UCLOUD_MODELVERSE/Vidu" + + def generate(self, client, model, video_url, duration, resolution, + last_frame_image=None, last_frame_url="", prompt=""): + api_key = client.get("api_key") + if not api_key: + raise ValueError("API key is not set") + + if not video_url or not video_url.strip(): + raise ValueError("必须提供 video_url") + + mv_client = ModelverseClient(api_key) + + task_input = { + "video_url": video_url.strip(), + } + + # Optional last frame + has_last_url = last_frame_url and last_frame_url.strip() + has_last_image = last_frame_image is not None + if has_last_url and has_last_image: + raise ValueError("尾帧：请提供 image 或 url，不能同时提供") + + if has_last_url: + task_input["last_frame_url"] = last_frame_url.strip() + elif has_last_image: + task_input["last_frame_url"] = image_to_base64(last_frame_image) + + if prompt and prompt.strip(): + task_input["prompt"] = prompt.strip() + + parameters = { + "vidu_type": "extend", + "duration": duration, + "resolution": resolution, + } + + # Submit task + submit_res = mv_client.submit_task(model, task_input, parameters) + task_id = submit_res.get("output", {}).get("task_id") + if not task_id: + raise Exception(f"Failed to submit task: {submit_res}") + + print(f"Vidu Extend task submitted: {task_id}") + + # Poll for result + video_url_result = self._poll_task(mv_client, task_id) + + # Download and return video + return self._download_video(video_url_result) + + def _poll_task(self, mv_client, task_id, max_retries=180): + for i in range(max_retries): + status_res = mv_client.get_task_status(task_id) + task_status = status_res.get("output", {}).get("task_status") + + if task_status == "Success": + urls = status_res.get("output", {}).get("urls", []) + if urls: + return urls[0] + raise Exception("Task succeeded but no video URL returned") + elif task_status == "Failure": + error = status_res.get("output", {}).get("error_message", "Unknown error") + raise Exception(f"Task failed: {error}") + elif task_status in ["Pending", "Running"]: + print(f"Task {task_id}: {task_status} ({i+1}/{max_retries})") + time.sleep(5) + else: + raise Exception(f"Unknown status: {task_status}") + + raise Exception("Task timed out") + + def _download_video(self, video_url): + response = requests.get(video_url, timeout=120) + response.raise_for_status() + video_io = io.BytesIO(response.content) + video_io.seek(0) + return (VideoFromFile(video_io),) + + +NODE_CLASS_MAPPINGS = { + "Vidu_Extend": ViduExtendNode, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "Vidu_Extend": "Vidu Extend", +} diff --git a/py/vidu_i2v.py b/py/vidu_i2v.py new file mode 100644 index 0000000..5afe8dc --- /dev/null +++ b/py/vidu_i2v.py @@ -0,0 +1,143 @@ +""" +Vidu Img2Video - 图生视频模型 +Models: viduq2-pro, viduq2-turbo +""" +import time +import requests +import io +from .modelverse_api.client import ModelverseClient +from .modelverse_api.utils import image_to_base64 +from comfy.comfy_types.node_typing import IO + +try: + from comfy_extras.nodes_video import VideoFromFile +except ImportError: + try: + from comfy.model_management import VideoFromFile + except ImportError: + class VideoFromFile: + def __init__(self, video_io): + self.video_io = video_io + self.video_data = video_io.getvalue() if hasattr(video_io, 'getvalue') else video_io + + +MODELS = ["viduq2-pro", "viduq2-turbo"] +RESOLUTIONS = ["540p", "720p", "1080p"] +MOVEMENT_AMPLITUDES = ["auto", "small", "medium", "large"] + + +class ViduImg2VideoNode: + """ + Vidu Img2Video - 图生视频 + Models: viduq2-pro (效果好，细节丰富), viduq2-turbo (效果好，生成快) + """ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "client": ("MODELVERSE_API_CLIENT",), + "model": (MODELS, {"default": "viduq2-pro", "tooltip": "viduq2-pro: 效果好细节丰富, viduq2-turbo: 生成快"}), + "duration": (IO.INT, {"default": 5, "min": 1, "max": 10, "step": 1, "tooltip": "视频时长(秒)"}), + "resolution": (RESOLUTIONS, {"default": "720p", "tooltip": "分辨率"}), + "movement_amplitude": (MOVEMENT_AMPLITUDES, {"default": "auto", "tooltip": "运动幅度"}), + }, + "optional": { + "first_frame_image": (IO.IMAGE, {"tooltip": "首帧图片"}), + "first_frame_url": (IO.STRING, {"default": "", "tooltip": "首帧图片URL (与image二选一)"}), + "prompt": (IO.STRING, {"multiline": True, "default": "", "tooltip": "文本提示词，最长2000字符"}), + "seed": (IO.INT, {"default": 0, "min": 0, "max": 2147483647, "tooltip": "随机种子"}), + "bgm": (IO.BOOLEAN, {"default": False, "tooltip": "是否添加背景音乐"}), + } + } + + RETURN_TYPES = (IO.VIDEO,) + RETURN_NAMES = ("video",) + FUNCTION = "generate" + CATEGORY = "UCLOUD_MODELVERSE/Vidu" + + def generate(self, client, model, duration, resolution, movement_amplitude, + first_frame_image=None, first_frame_url="", prompt="", seed=0, bgm=False): + api_key = client.get("api_key") + if not api_key: + raise ValueError("API key is not set") + + mv_client = ModelverseClient(api_key) + + # Validate first frame input + has_url = first_frame_url and first_frame_url.strip() + has_image = first_frame_image is not None + + if has_url and has_image: + raise ValueError("请提供 first_frame_image 或 first_frame_url，不能同时提供") + if not has_url and not has_image: + raise ValueError("必须提供 first_frame_image 或 first_frame_url") + + task_input = {} + if has_url: + task_input["first_frame_url"] = first_frame_url.strip() + else: + task_input["first_frame_url"] = image_to_base64(first_frame_image) + + if prompt and prompt.strip(): + task_input["prompt"] = prompt.strip() + + parameters = { + "vidu_type": "img2video", + "duration": duration, + "resolution": resolution, + "movement_amplitude": movement_amplitude, + "seed": seed, + "bgm": bgm, + } + + # Submit task + submit_res = mv_client.submit_task(model, task_input, parameters) + task_id = submit_res.get("output", {}).get("task_id") + if not task_id: + raise Exception(f"Failed to submit task: {submit_res}") + + print(f"Vidu I2V task submitted: {task_id}") + + # Poll for result + video_url = self._poll_task(mv_client, task_id) + + # Download and return video + return self._download_video(video_url) + + def _poll_task(self, mv_client, task_id, max_retries=180): + for i in range(max_retries): + status_res = mv_client.get_task_status(task_id) + task_status = status_res.get("output", {}).get("task_status") + + if task_status == "Success": + urls = status_res.get("output", {}).get("urls", []) + if urls: + return urls[0] + raise Exception("Task succeeded but no video URL returned") + elif task_status == "Failure": + error = status_res.get("output", {}).get("error_message", "Unknown error") + raise Exception(f"Task failed: {error}") + elif task_status in ["Pending", "Running"]: + print(f"Task {task_id}: {task_status} ({i+1}/{max_retries})") + time.sleep(5) + else: + raise Exception(f"Unknown status: {task_status}") + + raise Exception("Task timed out") + + def _download_video(self, video_url): + response = requests.get(video_url, timeout=120) + response.raise_for_status() + video_io = io.BytesIO(response.content) + video_io.seek(0) + return (VideoFromFile(video_io),) + + +NODE_CLASS_MAPPINGS = { + "Vidu_Img2Video": ViduImg2VideoNode, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "Vidu_Img2Video": "Vidu Img2Video", +} diff --git a/py/vidu_ref2v.py b/py/vidu_ref2v.py new file mode 100644 index 0000000..e39ddea --- /dev/null +++ b/py/vidu_ref2v.py @@ -0,0 +1,157 @@ +""" +Vidu Reference2Video - 参考生视频模型 +Model: viduq2 +支持1-7张参考图片，生成具备主体一致的视频 +""" +import time +import requests +import io +from .modelverse_api.client import ModelverseClient +from .modelverse_api.utils import image_to_base64 +from comfy.comfy_types.node_typing import IO + +try: + from comfy_extras.nodes_video import VideoFromFile +except ImportError: + try: + from comfy.model_management import VideoFromFile + except ImportError: + class VideoFromFile: + def __init__(self, video_io): + self.video_io = video_io + self.video_data = video_io.getvalue() if hasattr(video_io, 'getvalue') else video_io + + +ASPECT_RATIOS = ["16:9", "9:16", "3:4", "4:3", "1:1"] +RESOLUTIONS = ["540p", "720p", "1080p"] + + +class ViduReference2VideoNode: + """ + Vidu Reference2Video - 参考生视频 + Model: viduq2 + 支持1-7张参考图片，生成具备主体一致的视频 + """ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "client": ("MODELVERSE_API_CLIENT",), + "prompt": (IO.STRING, {"multiline": True, "default": "make it dance", "tooltip": "文本提示词，最长2000字符"}), + "duration": (IO.INT, {"default": 5, "min": 1, "max": 10, "step": 1, "tooltip": "视频时长(秒)"}), + "aspect_ratio": (ASPECT_RATIOS, {"default": "16:9", "tooltip": "长宽比"}), + "resolution": (RESOLUTIONS, {"default": "720p", "tooltip": "分辨率"}), + }, + "optional": { + "image1": (IO.IMAGE, {"tooltip": "参考图片1"}), + "image2": (IO.IMAGE, {"tooltip": "参考图片2"}), + "image3": (IO.IMAGE, {"tooltip": "参考图片3"}), + "image4": (IO.IMAGE, {"tooltip": "参考图片4"}), + "image5": (IO.IMAGE, {"tooltip": "参考图片5"}), + "image6": (IO.IMAGE, {"tooltip": "参考图片6"}), + "image7": (IO.IMAGE, {"tooltip": "参考图片7"}), + "image_urls": (IO.STRING, {"default": "", "multiline": True, "tooltip": "参考图片URL列表，每行一个"}), + "seed": (IO.INT, {"default": 0, "min": 0, "max": 2147483647, "tooltip": "随机种子"}), + "bgm": (IO.BOOLEAN, {"default": False, "tooltip": "是否添加背景音乐"}), + } + } + + RETURN_TYPES = (IO.VIDEO,) + RETURN_NAMES = ("video",) + FUNCTION = "generate" + CATEGORY = "UCLOUD_MODELVERSE/Vidu" + + def generate(self, client, prompt, duration, aspect_ratio, resolution, + image1=None, image2=None, image3=None, image4=None, + image5=None, image6=None, image7=None, + image_urls="", seed=0, bgm=False): + api_key = client.get("api_key") + if not api_key: + raise ValueError("API key is not set") + + mv_client = ModelverseClient(api_key) + + # Collect images + images = [] + for img in [image1, image2, image3, image4, image5, image6, image7]: + if img is not None: + images.append(image_to_base64(img)) + + # Add URL images + if image_urls and image_urls.strip(): + for url in image_urls.strip().split('\n'): + url = url.strip() + if url: + images.append(url) + + if not images: + raise ValueError("至少需要提供1张参考图片") + if len(images) > 7: + print(f"Warning: 最多支持7张参考图片，当前{len(images)}张，将只使用前7张") + images = images[:7] + + task_input = { + "images": images, + "prompt": prompt, + } + + parameters = { + "vidu_type": "reference2video", + "duration": duration, + "aspect_ratio": aspect_ratio, + "resolution": resolution, + "seed": seed, + "bgm": bgm, + } + + # Submit task + submit_res = mv_client.submit_task("viduq2", task_input, parameters) + task_id = submit_res.get("output", {}).get("task_id") + if not task_id: + raise Exception(f"Failed to submit task: {submit_res}") + + print(f"Vidu Ref2V task submitted: {task_id}") + + # Poll for result + video_url = self._poll_task(mv_client, task_id) + + # Download and return video + return self._download_video(video_url) + + def _poll_task(self, mv_client, task_id, max_retries=180): + for i in range(max_retries): + status_res = mv_client.get_task_status(task_id) + task_status = status_res.get("output", {}).get("task_status") + + if task_status == "Success": + urls = status_res.get("output", {}).get("urls", []) + if urls: + return urls[0] + raise Exception("Task succeeded but no video URL returned") + elif task_status == "Failure": + error = status_res.get("output", {}).get("error_message", "Unknown error") + raise Exception(f"Task failed: {error}") + elif task_status in ["Pending", "Running"]: + print(f"Task {task_id}: {task_status} ({i+1}/{max_retries})") + time.sleep(5) + else: + raise Exception(f"Unknown status: {task_status}") + + raise Exception("Task timed out") + + def _download_video(self, video_url): + response = requests.get(video_url, timeout=120) + response.raise_for_status() + video_io = io.BytesIO(response.content) + video_io.seek(0) + return (VideoFromFile(video_io),) + + +NODE_CLASS_MAPPINGS = { + "Vidu_Reference2Video": ViduReference2VideoNode, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "Vidu_Reference2Video": "Vidu Reference2Video", +} diff --git a/py/vidu_startend2v.py b/py/vidu_startend2v.py new file mode 100644 index 0000000..3a1ff38 --- /dev/null +++ b/py/vidu_startend2v.py @@ -0,0 +1,163 @@ +""" +Vidu StartEnd2Video - 首尾帧生视频模型 +Models: viduq2-pro, viduq2-turbo +""" +import time +import requests +import io +from .modelverse_api.client import ModelverseClient +from .modelverse_api.utils import image_to_base64 +from comfy.comfy_types.node_typing import IO + +try: + from comfy_extras.nodes_video import VideoFromFile +except ImportError: + try: + from comfy.model_management import VideoFromFile + except ImportError: + class VideoFromFile: + def __init__(self, video_io): + self.video_io = video_io + self.video_data = video_io.getvalue() if hasattr(video_io, 'getvalue') else video_io + + +MODELS = ["viduq2-pro", "viduq2-turbo"] +RESOLUTIONS = ["540p", "720p", "1080p"] +MOVEMENT_AMPLITUDES = ["auto", "small", "medium", "large"] + + +class ViduStartEnd2VideoNode: + """ + Vidu StartEnd2Video - 首尾帧生视频 + Models: viduq2-pro (效果好，细节丰富), viduq2-turbo (效果好，生成快) + 注意：首尾帧分辨率需相近 (0.8~1.25) + """ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "client": ("MODELVERSE_API_CLIENT",), + "model": (MODELS, {"default": "viduq2-pro", "tooltip": "viduq2-pro: 效果好细节丰富, viduq2-turbo: 生成快"}), + "duration": (IO.INT, {"default": 5, "min": 1, "max": 8, "step": 1, "tooltip": "视频时长(秒), 1-8"}), + "resolution": (RESOLUTIONS, {"default": "720p", "tooltip": "分辨率"}), + "movement_amplitude": (MOVEMENT_AMPLITUDES, {"default": "auto", "tooltip": "运动幅度"}), + }, + "optional": { + "first_frame_image": (IO.IMAGE, {"tooltip": "首帧图片"}), + "first_frame_url": (IO.STRING, {"default": "", "tooltip": "首帧图片URL"}), + "last_frame_image": (IO.IMAGE, {"tooltip": "尾帧图片"}), + "last_frame_url": (IO.STRING, {"default": "", "tooltip": "尾帧图片URL"}), + "prompt": (IO.STRING, {"multiline": True, "default": "", "tooltip": "文本提示词，最长2000字符"}), + "seed": (IO.INT, {"default": 0, "min": 0, "max": 2147483647, "tooltip": "随机种子"}), + "bgm": (IO.BOOLEAN, {"default": False, "tooltip": "是否添加背景音乐"}), + } + } + + RETURN_TYPES = (IO.VIDEO,) + RETURN_NAMES = ("video",) + FUNCTION = "generate" + CATEGORY = "UCLOUD_MODELVERSE/Vidu" + + def generate(self, client, model, duration, resolution, movement_amplitude, + first_frame_image=None, first_frame_url="", + last_frame_image=None, last_frame_url="", + prompt="", seed=0, bgm=False): + api_key = client.get("api_key") + if not api_key: + raise ValueError("API key is not set") + + mv_client = ModelverseClient(api_key) + + # Validate first frame + has_first_url = first_frame_url and first_frame_url.strip() + has_first_image = first_frame_image is not None + if has_first_url and has_first_image: + raise ValueError("首帧：请提供 image 或 url，不能同时提供") + if not has_first_url and not has_first_image: + raise ValueError("必须提供首帧图片") + + # Validate last frame + has_last_url = last_frame_url and last_frame_url.strip() + has_last_image = last_frame_image is not None + if has_last_url and has_last_image: + raise ValueError("尾帧：请提供 image 或 url，不能同时提供") + if not has_last_url and not has_last_image: + raise ValueError("必须提供尾帧图片") + + task_input = {} + + # First frame + if has_first_url: + task_input["first_frame_url"] = first_frame_url.strip() + else: + task_input["first_frame_url"] = image_to_base64(first_frame_image) + + # Last frame + if has_last_url: + task_input["last_frame_url"] = last_frame_url.strip() + else: + task_input["last_frame_url"] = image_to_base64(last_frame_image) + + if prompt and prompt.strip(): + task_input["prompt"] = prompt.strip() + + parameters = { + "vidu_type": "start-end2video", + "duration": duration, + "resolution": resolution, + "movement_amplitude": movement_amplitude, + "seed": seed, + "bgm": bgm, + } + + # Submit task + submit_res = mv_client.submit_task(model, task_input, parameters) + task_id = submit_res.get("output", {}).get("task_id") + if not task_id: + raise Exception(f"Failed to submit task: {submit_res}") + + print(f"Vidu StartEnd2V task submitted: {task_id}") + + # Poll for result + video_url = self._poll_task(mv_client, task_id) + + # Download and return video + return self._download_video(video_url) + + def _poll_task(self, mv_client, task_id, max_retries=180): + for i in range(max_retries): + status_res = mv_client.get_task_status(task_id) + task_status = status_res.get("output", {}).get("task_status") + + if task_status == "Success": + urls = status_res.get("output", {}).get("urls", []) + if urls: + return urls[0] + raise Exception("Task succeeded but no video URL returned") + elif task_status == "Failure": + error = status_res.get("output", {}).get("error_message", "Unknown error") + raise Exception(f"Task failed: {error}") + elif task_status in ["Pending", "Running"]: + print(f"Task {task_id}: {task_status} ({i+1}/{max_retries})") + time.sleep(5) + else: + raise Exception(f"Unknown status: {task_status}") + + raise Exception("Task timed out") + + def _download_video(self, video_url): + response = requests.get(video_url, timeout=120) + response.raise_for_status() + video_io = io.BytesIO(response.content) + video_io.seek(0) + return (VideoFromFile(video_io),) + + +NODE_CLASS_MAPPINGS = { + "Vidu_StartEnd2Video": ViduStartEnd2VideoNode, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "Vidu_StartEnd2Video": "Vidu StartEnd2Video", +} diff --git a/py/vidu_t2v.py b/py/vidu_t2v.py new file mode 100644 index 0000000..3bc1e03 --- /dev/null +++ b/py/vidu_t2v.py @@ -0,0 +1,122 @@ +""" +Vidu Text2Video - 文生视频模型 +Model: viduq2 +""" +import time +import requests +import io +from .modelverse_api.client import ModelverseClient +from comfy.comfy_types.node_typing import IO + +try: + from comfy_extras.nodes_video import VideoFromFile +except ImportError: + try: + from comfy.model_management import VideoFromFile + except ImportError: + class VideoFromFile: + def __init__(self, video_io): + self.video_io = video_io + self.video_data = video_io.getvalue() if hasattr(video_io, 'getvalue') else video_io + + +ASPECT_RATIOS = ["16:9", "9:16", "3:4", "4:3", "1:1"] +RESOLUTIONS = ["540p", "720p", "1080p"] + + +class ViduText2VideoNode: + """ + Vidu Text2Video - 文生视频 + Model: viduq2 + """ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "client": ("MODELVERSE_API_CLIENT",), + "prompt": (IO.STRING, {"multiline": True, "default": "A beautiful sunset over the ocean", "tooltip": "文本提示词，最长2000字符"}), + "duration": (IO.INT, {"default": 5, "min": 1, "max": 10, "step": 1, "tooltip": "视频时长(秒)"}), + "aspect_ratio": (ASPECT_RATIOS, {"default": "16:9", "tooltip": "长宽比"}), + "resolution": (RESOLUTIONS, {"default": "720p", "tooltip": "分辨率"}), + }, + "optional": { + "seed": (IO.INT, {"default": 0, "min": 0, "max": 2147483647, "tooltip": "随机种子，0表示随机"}), + "guidance_scale": (IO.FLOAT, {"default": 7.5, "min": 1.0, "max": 20.0, "step": 0.5, "tooltip": "引导系数"}), + "bgm": (IO.BOOLEAN, {"default": False, "tooltip": "是否添加背景音乐"}), + } + } + + RETURN_TYPES = (IO.VIDEO,) + RETURN_NAMES = ("video",) + FUNCTION = "generate" + CATEGORY = "UCLOUD_MODELVERSE/Vidu" + + def generate(self, client, prompt, duration, aspect_ratio, resolution, seed=0, guidance_scale=7.5, bgm=False): + api_key = client.get("api_key") + if not api_key: + raise ValueError("API key is not set") + + mv_client = ModelverseClient(api_key) + + task_input = {"prompt": prompt} + parameters = { + "vidu_type": "text2video", + "duration": duration, + "aspect_ratio": aspect_ratio, + "resolution": resolution, + "seed": seed, + "guidance_scale": guidance_scale, + "bgm": bgm, + } + + # Submit task + submit_res = mv_client.submit_task("viduq2", task_input, parameters) + task_id = submit_res.get("output", {}).get("task_id") + if not task_id: + raise Exception(f"Failed to submit task: {submit_res}") + + print(f"Vidu T2V task submitted: {task_id}") + + # Poll for result + video_url = self._poll_task(mv_client, task_id) + + # Download and return video + return self._download_video(video_url) + + def _poll_task(self, mv_client, task_id, max_retries=180): + for i in range(max_retries): + status_res = mv_client.get_task_status(task_id) + task_status = status_res.get("output", {}).get("task_status") + + if task_status == "Success": + urls = status_res.get("output", {}).get("urls", []) + if urls: + return urls[0] + raise Exception("Task succeeded but no video URL returned") + elif task_status == "Failure": + error = status_res.get("output", {}).get("error_message", "Unknown error") + raise Exception(f"Task failed: {error}") + elif task_status in ["Pending", "Running"]: + print(f"Task {task_id}: {task_status} ({i+1}/{max_retries})") + time.sleep(5) + else: + raise Exception(f"Unknown status: {task_status}") + + raise Exception("Task timed out") + + def _download_video(self, video_url): + response = requests.get(video_url, timeout=120) + response.raise_for_status() + video_io = io.BytesIO(response.content) + video_io.seek(0) + return (VideoFromFile(video_io),) + + +NODE_CLASS_MAPPINGS = { + "Vidu_Text2Video": ViduText2VideoNode, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "Vidu_Text2Video": "Vidu Text2Video", +}