diff --git a/.env.template b/.env.template index cbdd0e1..dc829f7 100644 --- a/.env.template +++ b/.env.template @@ -1,5 +1,6 @@ # Wan API Key Template # Copy this file to .env and replace the placeholder with your actual API key -# Visit https://dashscope.console.aliyun.com/apiKey to get your API key +# Visit https://dashscope.console.aliyuncs.com/apiKey to get your API key +# This key is used for all Wan models (text-to-image and image-to-video) DASHSCOPE_API_KEY=your_actual_api_key_here \ No newline at end of file diff --git a/README.md b/README.md index 7fab628..650fb70 100644 --- a/README.md +++ b/README.md @@ -3,14 +3,14 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) -A custom node for ComfyUI that provides **seamless integration** with the **Wan text-to-image models** from **Alibaba Cloud Model Studio**. This solution delivers cutting-edge image generation capabilities directly within ComfyUI. +A custom node for ComfyUI that provides **seamless integration** with the **Wan models** (text-to-image and image-to-video) from **Alibaba Cloud Model Studio**. This solution delivers cutting-edge image and video generation capabilities directly within ComfyUI. ### Why Choose Alibaba Cloud Model Studio? This is a direct integration with Alibaba Cloud's Model Studio service, not a third-party wrapper or local model implementation. Benefits include: - **Enterprise-Grade Infrastructure**: Leverages Alibaba Cloud's battle-tested AI platform serving millions of requests daily -- **State-of-the-Art Models**: Access to the latest Wan models (wan2.2-t2i-flash and wan2.2-t2i-plus) with continuous updates +- **State-of-the-Art Models**: Access to the latest Wan models (wan2.2-t2i-flash, wan2.2-t2i-plus, wan2.2-i2v-flash, wan2.2-i2v-plus) with continuous updates - **Commercial Licensing**: Properly licensed for commercial use through Alibaba Cloud's terms of service - **Scalable Architecture**: Handles high-volume workloads with Alibaba Cloud's reliable infrastructure - **Security Compliance**: Follows Alibaba Cloud's security best practices with secure API key management @@ -19,11 +19,12 @@ This is a direct integration with Alibaba Cloud's Model Studio service, not a th ⚠️ **This is a paid service**: The Wan models are provided through Alibaba Cloud's commercial API and incur usage costs. You will be billed according to Alibaba Cloud's pricing model based on your usage. - **Model Authorization Required**: If you're using a non-default workspace or project in Alibaba Cloud, you may need to explicitly authorize access to the `wan2.2-t2i-flash` and `wan2.2-t2i-plus` models in your DashScope console. + **Model Authorization Required**: If you're using a non-default workspace or project in Alibaba Cloud, you may need to explicitly authorize access to the Wan models in your DashScope console. ## Features - Generate images from text T2I using Wan models with selectable model types +- Generate 5-second videos from images and text prompts (I2V) using Wan models - Configurable parameters: seed, resolution, prompt extension, watermark, negative prompts - Powered by Alibaba Cloud's advanced Wan models @@ -44,7 +45,7 @@ This is a direct integration with Alibaba Cloud's Model Studio service, not a th ### Obtain API Key -1. Visit [Alibaba Cloud Model Studio](https://dashscope.console.aliyun.com/apiKey) to get your API key +1. Visit [Alibaba Cloud Model Studio](https://dashscope.console.aliyuncs.com/apiKey) to get your API key 2. Create an account if you don't have one 3. Generate a new API key @@ -52,8 +53,8 @@ This is a direct integration with Alibaba Cloud's Model Studio service, not a th If you're using a workspace other than your default workspace, you may need to authorize the models: -1. Go to the [DashScope Model Management Console](https://dashscope.console.aliyun.com/model) -2. Find `wan2.2-t2i-flash` and `wan2.2-t2i-plus` models +1. Go to the [DashScope Model Management Console](https://dashscope.console.aliyuncs.com/model) +2. Find the Wan models you want to use 3. Click "Authorize" or "Subscribe" for each model 4. Select your workspace/project if prompted @@ -74,6 +75,20 @@ DASHSCOPE_API_KEY=your_actual_api_key_here 4. Configure parameters as needed (seed, resolution, etc.) 5. Execute the node +### Image-to-Video Generation + +1. Add the "Wan Image-to-Video Generator" node to your workflow +2. Provide a publicly accessible URL to the image you want to use as the first frame of your video +3. Select the desired model (wan2.2-i2v-flash or wan2.2-i2v-plus) +4. Connect a text input with your prompt describing the video content +5. Optionally configure the output directory where the video will be saved (can be browsed in ComfyUI) +6. Execute the node +7. The node will return a path to the downloaded video file +8. To preview the video, connect the output to a "Load Video (Path)" node from ComfyUI-VideoHelperSuite + +**Note**: The image URL must be publicly accessible (not behind authentication or on localhost). +You can use services like Imgur, cloud storage providers, or your own web server to host the image. + ## Node Parameters ### Text-to-Image Generator @@ -85,13 +100,34 @@ DASHSCOPE_API_KEY=your_actual_api_key_here - **seed**: Random seed for generation (0 for random) - **watermark**: Add Wan watermark to output +### Image-to-Video Generator +- **model**: Select the Wan model to use (wan2.2-i2v-flash or wan2.2-i2v-plus) +- **image_url**: Publicly accessible URL to the image for the first frame of the video +- **prompt** (required): The text prompt describing the video content +- **resolution**: Output video resolution (480P, 720P, 1080P) +- **negative_prompt**: Text describing content to avoid in the video +- **prompt_extend**: Enable intelligent prompt rewriting for better results +- **seed**: Random seed for generation (0 for random) +- **watermark**: Add Wan watermark to output +- **output_dir**: Directory where the generated video will be saved. Can be browsed and selected in ComfyUI. + +**Note**: To preview the generated video in ComfyUI, connect the output of this node to a "Load Video (Path)" node from ComfyUI-VideoHelperSuite. + ## Examples -### Text-only Generation +### Text-to-Image Generation Prompt: "Generate an image of a cat swimming under the water" ![Text-to-Image Example](media/ComfyUI_wan-t2i.png) +### Image-to-Video Generation +1. First frame: Provide a URL to an image (e.g., "https://example.com/your_image.png") +2. Prompt: "a cat swimming under the water, suddenly a barracuda swim cross over him, the cat looks suprised and suddenly attracting to the fish." +3. Output directory: "./videos" (default) or any custom path +4. To preview: Connect the output to a "Load Video (Path)" node from ComfyUI-VideoHelperSuite + +![Image-to-Video Example](media/ComfyUI_wan-i2v.png) + ## Security The API key is loaded from the `DASHSCOPE_API_KEY` environment variable and never stored in files or code, following Alibaba Cloud security best practices. diff --git a/__init__.py b/__init__.py index df703aa..13b464a 100644 --- a/__init__.py +++ b/__init__.py @@ -1,16 +1,18 @@ """ ComfyUI_Wan - A custom node for ComfyUI that integrates Wan models -for text-to-image generation. +for text-to-image and image-to-video generation. """ -from .wan_nodes import WanT2IGenerator +from .wan_nodes import WanT2IGenerator, WanI2VGenerator NODE_CLASS_MAPPINGS = { "WanT2IGenerator": WanT2IGenerator, + "WanI2VGenerator": WanI2VGenerator, } NODE_DISPLAY_NAME_MAPPINGS = { "WanT2IGenerator": "Wan Text-to-Image Generator", + "WanI2VGenerator": "Wan Image-to-Video Generator", } __all__ = ['NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS'] \ No newline at end of file diff --git a/media/ComfyUI_wan-i2v.png b/media/ComfyUI_wan-i2v.png new file mode 100644 index 0000000..bb8dcbb Binary files /dev/null and b/media/ComfyUI_wan-i2v.png differ diff --git a/wan_nodes.py b/wan_nodes.py index 65de477..5f7f103 100644 --- a/wan_nodes.py +++ b/wan_nodes.py @@ -10,6 +10,14 @@ from dotenv import load_dotenv import sys import pathlib +# Import ComfyUI's folder_paths for directory browsing +try: + import folder_paths + COMFYUI_AVAILABLE = True +except ImportError: + COMFYUI_AVAILABLE = False + print("folder_paths not available, using default directory handling") + # Load environment variables from .env file # Try to load .env file from the current directory first env_path = pathlib.Path(__file__).parent / '.env' @@ -300,4 +308,256 @@ class WanT2IGenerator(WanAPIBase): raise RuntimeError(f"Task did not complete within the expected time ({max_attempts} attempts)") +class WanI2VGenerator(WanAPIBase): + """Node for image-to-video generation using Wan model""" + + # Define available Wan i2v models + MODEL_OPTIONS = [ + "wan2.2-i2v-flash", # Speed Edition + "wan2.2-i2v-plus" # Professional Edition + ] + + # Define allowed resolutions for Wan i2v models (using uppercase P as required by API) + RESOLUTION_OPTIONS = [ + "480P", + "720P", + "1080P" + ] + + def __init__(self): + super().__init__() + self.api_url = "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/video-generation/video-synthesis" + + @classmethod + def INPUT_TYPES(cls): + # Define output directory options + if COMFYUI_AVAILABLE: + # Use ComfyUI's output directory with browseable option + output_dir_options = { + "default": "videos/", + "tooltip": "Directory where the generated video will be saved. Browse to select a custom directory." + } + else: + # Fallback to string input + output_dir_options = { + "default": "./videos", + "multiline": False + } + + return { + "required": { + "model": (cls.MODEL_OPTIONS, { + "default": "wan2.2-i2v-flash" + }), + "image_url": ("STRING", { + "default": "https://example.com/your_image.png" + }), + "prompt": ("STRING", { + "multiline": True, + "default": "A cat running on the grass" + }) + }, + "optional": { + "negative_prompt": ("STRING", { + "multiline": True, + "default": "" + }), + "resolution": (cls.RESOLUTION_OPTIONS, { + "default": "720P" + }), + "prompt_extend": ("BOOLEAN", { + "default": True + }), + "watermark": ("BOOLEAN", { + "default": False + }), + "seed": ("INT", { + "default": 0, + "min": 0, + "max": 2147483647 + }), + "output_dir": ("STRING", output_dir_options) + } + } + + RETURN_TYPES = ("STRING",) # Returns path to downloaded video file + FUNCTION = "generate" + CATEGORY = "Ru4ls/Wan" + + def generate(self, model, image_url, prompt, negative_prompt="", resolution="720p", + prompt_extend=True, watermark=False, seed=0, output_dir="videos/"): + # Check API key + self.check_api_key() + + # Prepare API payload for image-to-video generation + payload = { + "model": model, + "input": { + "prompt": prompt, + "img_url": image_url + }, + "parameters": { + "resolution": resolution, + "prompt_extend": prompt_extend, + "watermark": watermark + } + } + + # Add optional parameters if they have non-default values + if negative_prompt: + payload["input"]["negative_prompt"] = negative_prompt + if seed > 0: + payload["parameters"]["seed"] = seed + + # Set headers according to DashScope documentation + headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + "X-DashScope-Async": "enable" # Wan requires async processing + } + + try: + # Make API request + print(f"Making API request to {self.api_url}") + response = requests.post(self.api_url, headers=headers, json=payload) + print(f"Response status code: {response.status_code}") + if hasattr(response, 'text'): + print(f"Response text: {response.text[:500]}...") # Print first 500 chars + response.raise_for_status() + + # Parse response to get task_id + result = response.json() + print(f"API response received: {json.dumps(result, indent=2)[:200]}...") # Print first 200 chars + + # Check if this is a task creation response + if "output" in result and "task_id" in result["output"]: + task_id = result["output"]["task_id"] + task_status = result["output"]["task_status"] + print(f"Task created with ID: {task_id}, status: {task_status}") + + # Now we need to poll for the result + task_result = self.poll_task_result(task_id, output_dir) + return (task_result,) # Return path to downloaded video file + else: + raise ValueError(f"Unexpected API response format: {result}") + + except requests.exceptions.RequestException as e: + # More detailed error handling + if hasattr(e, 'response') and e.response is not None: + status_code = e.response.status_code + response_text = e.response.text + print(f"API request failed with status {status_code}: {response_text}") + if status_code == 401: + raise RuntimeError(f"API request failed: 401 Unauthorized. " + f"This usually means your API key is invalid or not properly configured. " + f"Error details: {response_text}") + elif status_code == 403: + raise RuntimeError(f"API request failed: 403 Forbidden. " + f"This usually means your API key is valid but you don't have access to this model. " + f"Error details: {response_text}") + elif status_code == 400: + raise RuntimeError(f"API request failed: 400 Bad Request. " + f"This usually means there's an issue with the request format. " + f"Error details: {response_text}") + else: + raise RuntimeError(f"API request failed: {status_code} {e.response.reason}. Response: {response_text}") + else: + raise RuntimeError(f"API request failed: {str(e)}") + except Exception as e: + raise RuntimeError(f"Failed to process API response: {str(e)}") + + def poll_task_result(self, task_id, output_dir="videos/"): + """Poll for task result until completion and download video""" + import time + import os + from datetime import datetime + + # URL for querying task results + query_url = f"https://dashscope-intl.aliyuncs.com/api/v1/tasks/{task_id}" + + headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json" + } + + max_attempts = 60 # Maximum polling attempts (may take longer for video) + attempt = 0 + + while attempt < max_attempts: + try: + print(f"Polling task {task_id}, attempt {attempt + 1}/{max_attempts}") + response = requests.get(query_url, headers=headers) + response.raise_for_status() + + result = response.json() + task_status = result["output"]["task_status"] + print(f"Task status: {task_status}") + + if task_status == "SUCCEEDED": + # Task completed successfully + if "video_url" in result["output"]: + video_url = result["output"]["video_url"] + + # Download the video + video_response = requests.get(video_url) + video_response.raise_for_status() + + # Create a unique filename for the video + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + video_filename = f"wan_i2v_{timestamp}.mp4" + + # Handle output directory based on ComfyUI availability + if COMFYUI_AVAILABLE and not output_dir.startswith(("./", "/")): + # Use ComfyUI's output directory structure + if output_dir.endswith("/"): + output_dir = output_dir[:-1] + full_output_folder = folder_paths.get_output_directory() + output_path = os.path.join(full_output_folder, output_dir) + else: + # Resolve output directory path (existing logic) + if output_dir.startswith("./"): + # Relative to the node directory + output_path = os.path.join(os.path.dirname(__file__), output_dir[2:]) + else: + output_path = output_dir + + # Create output directory if it doesn't exist + os.makedirs(output_path, exist_ok=True) + + # Save video to file + video_path = os.path.join(output_path, video_filename) + with open(video_path, "wb") as f: + f.write(video_response.content) + + print(f"Video downloaded and saved to: {video_path}") + # Return path relative to ComfyUI output directory if using ComfyUI + if COMFYUI_AVAILABLE and not output_dir.startswith(("./", "/")): + return os.path.join(output_dir, video_filename) if output_dir != "videos/" else video_filename + else: + return video_path # Return full path + else: + raise ValueError(f"Unexpected API response format: {result}") + + elif task_status == "FAILED": + # Task failed + error_code = result["output"].get("code", "Unknown") + error_message = result["output"].get("message", "Unknown error") + raise RuntimeError(f"Task failed with code: {error_code}, message: {error_message}") + + elif task_status in ["PENDING", "RUNNING"]: + # Task still in progress, wait and retry + time.sleep(10) # Wait 10 seconds before retrying (video generation may take longer) + attempt += 1 + continue + + else: + raise ValueError(f"Unexpected task status: {task_status}") + + except requests.exceptions.RequestException as e: + raise RuntimeError(f"Failed to query task status: {str(e)}") + + # If we've reached here, we've exceeded max attempts + raise RuntimeError(f"Task did not complete within the expected time ({max_attempts} attempts)") + + # Node class mappings are in __init__.py \ No newline at end of file