diff --git a/README.md b/README.md index 273a915..bcd9fa9 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ This is a direct integration with Alibaba Cloud's Model Studio service, not a th - **State-of-the-Art Models**: Access to the latest Wan models with continuous updates: - **Text-to-Image**: wan2.2-t2i-flash (Speed Edition), wan2.2-t2i-plus (Professional Edition) - **Image-to-Video**: wan2.2-i2v-flash (Speed Edition), wan2.2-i2v-plus (Professional Edition) + - **Image-to-Video Effects**: wan2.1-i2v-plus (Professional Edition) - **Text-to-Video**: wan2.2-t2v-plus (Professional Edition) - **Image-to-Video (First/Last Frames)**: wan2.1-kf2v-plus (Professional Edition) - **Universal Video Editing (VACE)**: wan2.1-vace-plus (Professional Edition) - Split into 5 specialized nodes for better usability @@ -43,6 +44,7 @@ All API endpoints are centrally managed in the `core/base.py` file, making it ea |-----------|----------|-------|-------------| | Wan Text-to-Image Generator | T2I | wan2.2-t2i-flash, wan2.2-t2i-plus | Generate images from text prompts with multiple resolution options. Returns both image tensor and image URL. | | Wan Image-to-Video Generator | I2V | wan2.2-i2v-flash, wan2.2-i2v-plus | Create 5-second videos from a single image and text prompt. Returns both video file path and video URL. | +| Wan Image-to-Video Effect Generator | I2V Effect | wan2.1-i2v-plus | Generate videos with predefined effects from a single image. Returns both video file path and video URL. | | Wan Text-to-Video Generator | T2V | wan2.2-t2v-plus | Generate 5-second videos directly from text prompts. Returns both video file path and video URL. | | Wan Image-to-Video (First/Last Frame) Generator | II2V | wan2.1-kf2v-plus | Create 5-second videos using both first and last frame images. Returns both video file path and video URL. | | Wan VACE - Multi-Image Reference | VACE | wan2.1-vace-plus | Generate videos from multiple reference images. Returns both video file path and video URL. | @@ -50,6 +52,7 @@ All API endpoints are centrally managed in the `core/base.py` file, making it ea | Wan VACE - Local Video Editing | VACE | wan2.1-vace-plus | Locally edit specific areas of videos. Returns both video file path and video URL. | | Wan VACE - Video Extension | VACE | wan2.1-vace-plus | Extend videos with additional content. Returns both video file path and video URL. | | Wan VACE - Video Outpainting | VACE | wan2.1-vace-plus | Scale videos in different directions. Returns both video file path and video URL. | +| Wan Image-to-Video Effect Generator | I2V Effect | wan2.1-i2v-plus | Generate videos with predefined effects from a single image. Returns both video file path and video URL. | ## Features @@ -58,6 +61,7 @@ All API endpoints are centrally managed in the `core/base.py` file, making it ea - **Specialized VACE Nodes**: The Universal Video Editing (VACE) model has been split into 5 specialized nodes for better usability and focused functionality - **Powered by Alibaba Cloud's Advanced Wan Models**: Access to state-of-the-art models with continuous updates - **Dual Output Support**: All video generation nodes now return both local file paths and remote URLs for flexible workflow integration +- **Video Effects Generation**: Create videos with predefined effects using the Image-to-Video Effect Generator node ## Installation @@ -289,6 +293,31 @@ This node scales videos in different directions using the Wan VACE model. **Note**: To preview the generated video in ComfyUI, connect the output of this node to a "Load Video (Path)" node from ComfyUI-VideoHelperSuite. +### Wan Image-to-Video Effect Generator +This node generates videos with predefined effects from a single image using the Wan model. + +**Important**: This feature is only supported in the Mainland China region. + +**Parameters:** +- **model**: Select the Wan model to use (wan2.1-i2v-plus) +- **image_url** (required): Publicly accessible URL to the image for the first frame of the video +- **template** (required): Predefined effect template to apply (e.g., "flying", "rose", "dance1", etc.) +- **region**: Select the region (mainland_china for effects) +- **resolution**: Output video resolution (480P, 720P, 1080P) +- **seed**: Random seed for generation (0 for random) +- **output_dir**: Directory where the generated video will be saved. Can be browsed and selected in ComfyUI. + +**Available Templates:** +- squish, rotation, poke, inflate, dissolve, carousel, singleheart +- dance1, dance2, dance3, mermaid, graduation, dragon, money +- flying, rose, crystalrose, hug, frenchkiss, coupleheart + +**Return Values:** +- **video_file_path**: Path to the downloaded video file on your local system +- **video_url**: URL of the generated video on Alibaba Cloud's servers + +**Note**: To preview the generated video in ComfyUI, connect the output of this node to a "Load Video (Path)" node from ComfyUI-VideoHelperSuite. + ## Examples Prompt: "Generate an image of a cat" @@ -306,6 +335,7 @@ The API key is loaded from the `DASHSCOPE_API_KEY` environment variable and neve - Updated `.env.template` to include separate API key variables for international and Mainland China regions - Modified `core/base.py` to support automatic endpoint and API key selection based on region - Updated all generator nodes (T2I, I2V, T2V, II2V) and VACE nodes to include region selection +- Added new "Wan Image-to-Video Effect Generator" node for creating videos with predefined effects - Improved documentation with updated setup instructions and regional support information ### v1.0.0 - Initial Release diff --git a/__init__.py b/__init__.py index 20849d1..1b42f06 100644 --- a/__init__.py +++ b/__init__.py @@ -5,6 +5,7 @@ for text-to-image, image-to-video, text-to-video, first-last-frame-tovideo, and from .generators.t2i import WanT2IGenerator from .generators.i2v import WanI2VGenerator +from .generators.i2v_effect import WanI2VEffectGenerator from .generators.t2v import WanT2VGenerator from .generators.ii2v import WanII2VGenerator from .vace.image_reference import WanVACEImageReference @@ -16,6 +17,7 @@ from .vace.video_outpainting import WanVACEVideoOutpainting NODE_CLASS_MAPPINGS = { "WanT2IGenerator": WanT2IGenerator, "WanI2VGenerator": WanI2VGenerator, + "WanI2VEffectGenerator": WanI2VEffectGenerator, "WanT2VGenerator": WanT2VGenerator, "WanII2VGenerator": WanII2VGenerator, "WanVACEImageReference": WanVACEImageReference, @@ -28,6 +30,7 @@ NODE_CLASS_MAPPINGS = { NODE_DISPLAY_NAME_MAPPINGS = { "WanT2IGenerator": "Wan Text-to-Image Generator", "WanI2VGenerator": "Wan Image-to-Video Generator", + "WanI2VEffectGenerator": "Wan Image-to-Video Effect Generator", "WanT2VGenerator": "Wan Text-to-Video Generator", "WanII2VGenerator": "Wan Image-to-Video (First/Last Frame) Generator", "WanVACEImageReference": "Wan VACE - Multi-Image Reference", diff --git a/generators/__init__.py b/generators/__init__.py index a9288e4..235bf9e 100644 --- a/generators/__init__.py +++ b/generators/__init__.py @@ -1,6 +1,7 @@ from .t2i import WanT2IGenerator from .i2v import WanI2VGenerator +from .i2v_effect import WanI2VEffectGenerator from .t2v import WanT2VGenerator from .ii2v import WanII2VGenerator -__all__ = ['WanT2IGenerator', 'WanI2VGenerator', 'WanT2VGenerator', 'WanII2VGenerator'] \ No newline at end of file +__all__ = ['WanT2IGenerator', 'WanI2VGenerator', 'WanI2VEffectGenerator', 'WanT2VGenerator', 'WanII2VGenerator'] \ No newline at end of file diff --git a/generators/i2v_effect.py b/generators/i2v_effect.py new file mode 100644 index 0000000..40afb26 --- /dev/null +++ b/generators/i2v_effect.py @@ -0,0 +1,298 @@ +import os +import json +import requests +from PIL import Image +import numpy as np +import torch +import io +import base64 +from dotenv import load_dotenv +import sys +import pathlib +from datetime import datetime + +# Import the base class and COMFYUI_AVAILABLE flag +from ..core.base import WanAPIBase, COMFYUI_AVAILABLE + +# Try to import folder_paths if available +try: + import folder_paths +except ImportError: + pass + +class WanI2VEffectGenerator(WanAPIBase): + """Node for image-to-video generation with effects using Wan model""" + + # Define available Wan i2v models + MODEL_OPTIONS = [ + "wan2.1-i2v-plus" # Professional Edition + ] + + # Define allowed resolutions for Wan i2v models (using uppercase P as required by API) + RESOLUTION_OPTIONS = [ + "480P", + "720P", + "1080P" + ] + + # Define region options + REGION_OPTIONS = [ + "international", + "mainland_china" + ] + + # Define video effect templates + TEMPLATE_OPTIONS = [ + "squish", + "rotation", + "poke", + "inflate", + "dissolve", + "carousel", + "singleheart", + "dance1", + "dance2", + "dance3", + "mermaid", + "graduation", + "dragon", + "money", + "flying", + "rose", + "crystalrose", + "hug", + "frenchkiss", + "coupleheart" + ] + + def __init__(self): + super().__init__() + + @classmethod + def INPUT_TYPES(cls): + # Define output directory options + if COMFYUI_AVAILABLE: + # Use ComfyUI's output directory with browseable option + output_dir_options = { + "default": "./videos", + "tooltip": "Directory where the generated video will be saved. Browse to select a custom directory." + } + else: + # Fallback to string input + output_dir_options = { + "default": "./videos", + "multiline": False + } + + return { + "required": { + "model": (cls.MODEL_OPTIONS, { + "default": "wan2.1-i2v-plus" + }), + "image_url": ("STRING", { + "default": "https://example.com/your_image.png" + }), + "template": (cls.TEMPLATE_OPTIONS, { + "default": "flying" + }), + "region": (cls.REGION_OPTIONS, { + "default": "mainland_china" # Note: Effects are only supported in Mainland China region + }) + }, + "optional": { + "resolution": (cls.RESOLUTION_OPTIONS, { + "default": "720P" + }), + "seed": ("INT", { + "default": 0, + "min": 0, + "max": 2147483647 + }), + "output_dir": ("STRING", output_dir_options) + } + } + + RETURN_TYPES = ("STRING", "STRING") # Returns path to downloaded video file and video URL + RETURN_NAMES = ("video_file_path", "video_url") + FUNCTION = "generate" + CATEGORY = "Ru4ls/Wan" + + def generate(self, model, image_url, template, region, resolution="720P", + seed=0, output_dir="./videos"): + # Check API key based on region + api_key = self.check_api_key(region) + + # Get the appropriate API endpoints based on region + endpoints = self.get_api_endpoints(region) + api_url = endpoints["video_post"] + + # Note: For effects, the prompt is ignored, so we don't include it + # Prepare API payload for image-to-video generation with effects + payload = { + "model": model, + "input": { + "img_url": image_url, + "template": template + }, + "parameters": { + "resolution": resolution + } + } + + # Add optional parameters if they have non-default values + if seed > 0: + payload["parameters"]["seed"] = seed + + # Set headers according to DashScope documentation + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + "X-DashScope-Async": "enable" # Wan requires async processing + } + + try: + # Make API request + print(f"Making API request to {api_url}") + response = requests.post(api_url, headers=headers, json=payload) + print(f"Response status code: {response.status_code}") + if hasattr(response, 'text'): + print(f"Response text: {response.text[:500]}...") # Print first 500 chars + response.raise_for_status() + + # Parse response to get task_id + result = response.json() + print(f"API response received: {json.dumps(result, indent=2)[:200]}...") # Print first 200 chars + + # Check if this is a task creation response + if "output" in result and "task_id" in result["output"]: + task_id = result["output"]["task_id"] + task_status = result["output"]["task_status"] + print(f"Task created with ID: {task_id}, status: {task_status}") + + # Now we need to poll for the result + task_result = self.poll_task_result(task_id, output_dir, region) + return task_result # Return both path to downloaded video file and video URL + else: + raise ValueError(f"Unexpected API response format: {result}") + + except requests.exceptions.RequestException as e: + # More detailed error handling + if hasattr(e, 'response') and e.response is not None: + status_code = e.response.status_code + response_text = e.response.text + print(f"API request failed with status {status_code}: {response_text}") + if status_code == 401: + raise RuntimeError(f"API request failed: 401 Unauthorized. " + f"This usually means your API key is invalid or not properly configured. " + f"Error details: {response_text}") + elif status_code == 403: + raise RuntimeError(f"API request failed: 403 Forbidden. " + f"This usually means your API key is valid but you don't have access to this model. " + f"Error details: {response_text}") + elif status_code == 400: + raise RuntimeError(f"API request failed: 400 Bad Request. " + f"This usually means there's an issue with the request format. " + f"Error details: {response_text}") + else: + raise RuntimeError(f"API request failed: {status_code} {e.response.reason}. Response: {response_text}") + else: + raise RuntimeError(f"API request failed: {str(e)}") + except Exception as e: + raise RuntimeError(f"Failed to process API response: {str(e)}") + + def poll_task_result(self, task_id, output_dir="./videos", region="mainland_china"): + """Poll for task result until completion and download video""" + import time + + # Get the appropriate API endpoints based on region + endpoints = self.get_api_endpoints(region) + query_url = endpoints["get"].format(task_id=task_id) + + # Check API key based on region + api_key = self.check_api_key(region) + + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json" + } + + max_attempts = 60 # Maximum polling attempts (may take longer for video) + attempt = 0 + + while attempt < max_attempts: + try: + print(f"Polling task {task_id}, attempt {attempt + 1}/{max_attempts}") + response = requests.get(query_url, headers=headers) + response.raise_for_status() + + result = response.json() + task_status = result["output"]["task_status"] + print(f"Task status: {task_status}") + + if task_status == "SUCCEEDED": + # Task completed successfully + if "video_url" in result["output"]: + video_url = result["output"]["video_url"] + + # Download the video + video_response = requests.get(video_url) + video_response.raise_for_status() + + # Create a unique filename for the video + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + video_filename = f"wan_i2v_effect_{timestamp}.mp4" + + # Handle output directory based on ComfyUI availability + if COMFYUI_AVAILABLE and not output_dir.startswith(("./", "/")): + # Use ComfyUI's output directory structure + if output_dir.endswith("/"): + output_dir = output_dir[:-1] + full_output_folder = folder_paths.get_output_directory() + output_path = os.path.join(full_output_folder, output_dir) + else: + # Resolve output directory path (existing logic) + if output_dir.startswith("./"): + # Relative to the node directory + output_path = os.path.join(os.path.dirname(__file__), output_dir[2:]) + else: + output_path = output_dir + + # Create output directory if it doesn't exist + os.makedirs(output_path, exist_ok=True) + + # Save video to file + video_path = os.path.join(output_path, video_filename) + with open(video_path, "wb") as f: + f.write(video_response.content) + + print(f"Video downloaded and saved to: {video_path}") + # Return path relative to ComfyUI output directory if using ComfyUI + if COMFYUI_AVAILABLE and not output_dir.startswith(("./", "/")): + return_path = os.path.join(output_dir, video_filename) if output_dir != "videos/" else video_filename + else: + return_path = video_path # Return full path + # Return both the file path and the video URL + return (return_path, video_url) + else: + raise ValueError(f"Unexpected API response format: {result}") + + elif task_status == "FAILED": + # Task failed + error_code = result["output"].get("code", "Unknown") + error_message = result["output"].get("message", "Unknown error") + raise RuntimeError(f"Task failed with code: {error_code}, message: {error_message}") + + elif task_status in ["PENDING", "RUNNING"]: + # Task still in progress, wait and retry + time.sleep(10) # Wait 10 seconds before retrying (video generation may take longer) + attempt += 1 + continue + + else: + raise ValueError(f"Unexpected task status: {task_status}") + + except requests.exceptions.RequestException as e: + raise RuntimeError(f"Failed to query task status: {str(e)}") + + # If we've reached here, we've exceeded max attempts + raise RuntimeError(f"Task did not complete within the expected time ({max_attempts} attempts)") \ No newline at end of file