commit 309ee84f56e9f0c35f9d0db8448cc86a890589bf Author: pixelworld AI Date: Thu Sep 4 11:30:01 2025 -0500 Initial commit: ComfyUI BytePlus API Integration - Video generation nodes (Seedance Lite & Pro) - Image generation nodes (Seedream & SeedEdit) - Video URL to frames utility node - Comprehensive README with usage guide - Base64 image handling (no external hosting required) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..94c5425 --- /dev/null +++ b/.gitignore @@ -0,0 +1,88 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Virtual environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# IDE files +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS files +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db + +# ComfyUI specific +ComfyUI-fal-API/ +ComfyUI-KLingAI-API/ + +# Temporary files +*.tmp +*.temp + +# Log files +*.log + +# Config files with sensitive data (optional - user can uncomment if needed) +# config.ini diff --git a/README.md b/README.md new file mode 100644 index 0000000..698742c --- /dev/null +++ b/README.md @@ -0,0 +1,161 @@ +# ComfyUI BytePlus API Integration + +This ComfyUI custom node provides comprehensive integration with BytePlus (ByteDance) AI APIs, including both video generation (Seedance) and image generation (Seedream/SeedEdit) models. + +## Features + +### Video Generation (Seedance) +- **Text-to-Video Generation**: Create videos from text descriptions +- **Image-to-Video Generation**: Generate videos using input images as first frames +- **First+Last Frame Generation**: Create videos between two specific frames +- **Reference Images Generation**: Generate videos based on multiple reference images +- **High-Quality Pro Models**: Access to BytePlus's premium Seedance Pro model + +### Image Generation +- **Text-to-Image Generation**: Create high-quality images from text prompts using Seedream 3.0 +- **Image-to-Image Editing**: Edit and transform existing images with text descriptions using SeedEdit 3.0 +- **Frame Extraction**: Convert video URLs to individual frames for further processing + +## Installation + +1. Clone this repository into your ComfyUI custom_nodes directory: +```bash +cd ComfyUI/custom_nodes/ +git clone https://github.com/FloyoAI/ComfyUI-BytePlus-API.git +``` + +2. Install requirements: +```bash +cd ComfyUI-BytePlus-API +pip install -r requirements.txt +``` + +3. Configure your BytePlus API key: + - Get your API key from: https://www.byteplus.com/ + - Edit `config.ini` and replace `` with your actual API key + - Or set the environment variable: `export BYTEPLUS_API_KEY="your_key_here"` + +4. Restart ComfyUI + +## Available Nodes + +### Video Generation Nodes (BytePlus/VideoGeneration) + +- **Seedance Lite Text-to-Video**: Generate videos from text prompts only + - Input: Text prompt, resolution, ratio, duration, FPS, seed, camera settings + - Output: Video URL (STRING) + - Use case: Create videos from scratch using descriptive text + +- **Seedance Lite Image-to-Video**: Generate videos from a starting image + - Input: Text prompt, input image, resolution, ratio, duration, FPS, seed, camera settings + - Output: Video URL (STRING) + - Use case: Animate static images or create videos with specific starting frames + +- **Seedance Lite First+Last Frame**: Generate videos between two keyframes + - Input: Text prompt, first frame image, last frame image, resolution, ratio, duration + - Output: Video URL (STRING) + - Use case: Create smooth transitions between two specific images + +- **Seedance Lite Reference Images**: Generate videos using multiple reference images + - Input: Text prompt, 2-4 reference images, resolution, ratio, duration + - Output: Video URL (STRING) + - Use case: Generate videos that maintain style/appearance consistency with reference materials + +- **Seedance Pro Text/Image-to-Video**: High-quality video generation + - Input: Text prompt, mode selection, optional input image, resolution, ratio, duration + - Output: Video URL (STRING) + - Use case: Premium quality video generation for professional use + +### Image Generation Nodes (BytePlus/ImageGeneration) + +- **Seedream Text-to-Image**: Generate images from text descriptions + - Input: Text prompt, size selection, optional seed + - Output: Generated image (IMAGE) + - Use case: Create high-quality images from detailed text descriptions + +- **SeedEdit Image-to-Image**: Edit and transform existing images + - Input: Text prompt, input image, size selection, guidance scale, optional seed + - Output: Edited image (IMAGE) + - Use case: Modify existing images based on text instructions (change colors, add/remove objects, style transfer) + +### Utility Nodes (BytePlus/Video) + +- **Video URL to Frames**: Extract all frames from a video URL + - Input: Video URL (STRING) + - Output: Frame batch (IMAGE), frame count (INT), FPS (FLOAT) + - Use case: Convert generated videos into individual frames for further image processing + +## Usage Guide + +### Video Generation Workflow +1. **Choose the appropriate video node** based on your input type: + - Text only → Seedance Lite Text-to-Video + - Single image → Seedance Lite Image-to-Video + - Two images → Seedance Lite First+Last Frame + - Multiple references → Seedance Lite Reference Images + - High quality → Seedance Pro + +2. **Configure parameters**: + - **Prompt**: Describe what you want in the video + - **Resolution**: 480p, 720p, or 1080p + - **Ratio**: Aspect ratio (16:9, 1:1, 9:16, etc.) + - **Duration**: 3-12 seconds + +3. **Connect the output** to other nodes: + - Use output URL directly with video preview nodes + - Connect to "Video URL to Frames" to extract individual frames + - Save video using external video saving nodes + +### Image Generation Workflow +1. **For new images**: Use Seedream Text-to-Image + - Write detailed prompts for best results + - Experiment with different sizes for various use cases + - Use seed values for reproducible results + +2. **For image editing**: Use SeedEdit Image-to-Image + - Load your source image + - Describe the changes you want in the prompt + - Adjust guidance scale (higher = more prompt adherence) + - Size can be "adaptive" to maintain original dimensions + +3. **Process results**: + - Images output as standard ComfyUI IMAGE tensors + - Compatible with all ComfyUI image processing nodes + - Can be saved, edited, or used as input for video generation + +## Requirements + +- ComfyUI +- Python 3.8+ +- BytePlus API key +- Required packages (see requirements.txt) + +## Key Features + +- **No External Dependencies**: Base64 image conversion means no external hosting required +- **Automatic Task Polling**: Video generation waits until completion automatically +- **Comprehensive Error Handling**: Graceful degradation with detailed error messages +- **ComfyUI Integration**: + - Video outputs as URL strings compatible with video preview nodes + - Image outputs as standard IMAGE tensors compatible with all image nodes +- **Flexible Sizing**: Support for various aspect ratios and resolutions +- **Seed Control**: Reproducible results with optional seed parameters + +## Tips for Best Results + +### Video Generation +- **Be specific in prompts**: Include details about movement, camera angles, and style +- **Use appropriate ratios**: Match your intended use case (16:9 for landscape, 9:16 for mobile, etc.) +- **Leverage reference images**: For consistent character/style appearance across videos + +### Image Generation +- **Detailed prompts work better**: Include style, lighting, composition details +- **Use SeedEdit for modifications**: Better than text-to-image for specific edits +- **Guidance scale tuning**: Start with default 5.5, increase for more prompt adherence +- **Size selection**: Use "adaptive" for SeedEdit to maintain original proportions + +## Troubleshooting + +- **API Key Issues**: Ensure your BytePlus API key is correctly set in config.ini or environment variables +- **Empty Results**: Check ComfyUI console for detailed error messages +- **Image Quality**: Try different prompts, seeds, or guidance scale values for better results diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..b17f93f --- /dev/null +++ b/__init__.py @@ -0,0 +1,23 @@ +import importlib +import importlib.util + +node_list = [ + "byteplus_video_node", + "video_to_frames_node", + "byteplus_image_node", +] + +NODE_CLASS_MAPPINGS = {} +NODE_DISPLAY_NAME_MAPPINGS = {} + +for module_name in node_list: + imported_module = importlib.import_module(f".nodes.{module_name}", __name__) + + NODE_CLASS_MAPPINGS = {**NODE_CLASS_MAPPINGS, **imported_module.NODE_CLASS_MAPPINGS} + NODE_DISPLAY_NAME_MAPPINGS = { + **NODE_DISPLAY_NAME_MAPPINGS, + **imported_module.NODE_DISPLAY_NAME_MAPPINGS, + } + + +__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] diff --git a/config.ini b/config.ini new file mode 100644 index 0000000..f5558cb --- /dev/null +++ b/config.ini @@ -0,0 +1,2 @@ +[API] +BYTEPLUS_API_KEY = diff --git a/nodes/byteplus_image_node.py b/nodes/byteplus_image_node.py new file mode 100644 index 0000000..1d640ab --- /dev/null +++ b/nodes/byteplus_image_node.py @@ -0,0 +1,209 @@ +import io +import requests +import torch +import numpy as np +from PIL import Image +from .byteplus_utils import BytePlusConfig, BytePlusImageUtils + + +class BytePlusResultProcessor: + """Utility functions for processing BytePlus image generation results.""" + + @staticmethod + def process_image_result(result): + """Process BytePlus image generation result and return tensor.""" + try: + images = [] + # BytePlus API returns data array with url field + for img_info in result["data"]: + img_url = img_info["url"] + img_response = requests.get(img_url) + img = Image.open(io.BytesIO(img_response.content)) + img_array = np.array(img).astype(np.float32) / 255.0 + images.append(img_array) + + # Stack the images along a new first dimension + stacked_images = np.stack(images, axis=0) + + # Convert to PyTorch tensor + img_tensor = torch.from_numpy(stacked_images) + return (img_tensor,) + except Exception as e: + print(f"Error processing BytePlus image result: {str(e)}") + return BytePlusResultProcessor.create_blank_image() + + @staticmethod + def create_blank_image(): + """Create a blank black image tensor.""" + blank_img = Image.new("RGB", (512, 512), color="black") + img_array = np.array(blank_img).astype(np.float32) / 255.0 + img_tensor = torch.from_numpy(img_array)[None,] + return (img_tensor,) + + +class BytePlusImageApiHandler: + """Utility functions for BytePlus image API interactions.""" + + @staticmethod + def generate_image(model, arguments): + """Generate image using BytePlus API.""" + config = BytePlusConfig() + + headers = { + "Authorization": f"Bearer {config.get_api_key()}", + "Content-Type": "application/json" + } + + payload = { + "model": model, + **arguments + } + + print(f"Making BytePlus image request to: {config.get_base_url()}/images/generations") + print(f"Model: {model}") + print(f"Requested batch size (n): {payload.get('n', 1)}") + print(f"Payload: {payload}") + + try: + response = requests.post( + f"{config.get_base_url()}/images/generations", + headers=headers, + json=payload, + timeout=60 + ) + + if response.status_code == 200: + result = response.json() + print(f"BytePlus image generation successful: {result}") + return result + else: + print(f"BytePlus image generation error: {response.status_code} - {response.text}") + raise Exception(f"API request failed: {response.status_code} - {response.text}") + + except Exception as e: + print(f"Error in BytePlus image generation: {str(e)}") + raise e + + @staticmethod + def handle_image_generation_error(model_name, error): + """Handle image generation errors consistently.""" + print(f"Error generating image with {model_name}: {str(error)}") + return BytePlusResultProcessor.create_blank_image() + + +class BytePlusSeedreamTextToImageNode: + """BytePlus Seedream Text-to-Image Generation Node""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "prompt": ("STRING", {"default": "", "multiline": True}), + "size": (["1024x1024", "1152x896", "896x1152", "1216x832", "832x1216", + "1344x768", "768x1344", "1536x640", "640x1536"], {"default": "1024x1024"}), + }, + "optional": { + "seed": ("INT", {"default": -1, "min": -1, "max": 2147483647}), + }, + } + + RETURN_TYPES = ("IMAGE",) + FUNCTION = "generate_image" + CATEGORY = "BytePlus/ImageGeneration" + + def generate_image(self, prompt, size, seed=-1): + try: + arguments = { + "prompt": prompt, + "n": 1, + "size": size, + "response_format": "url", + "watermark": False, # Always false, not exposed in UI + } + + # Only add seed if it's not -1 (random) + if seed != -1: + arguments["seed"] = seed + + result = BytePlusImageApiHandler.generate_image( + "seedream-3-0-t2i-250415", + arguments + ) + + return BytePlusResultProcessor.process_image_result(result) + + except Exception as e: + return BytePlusImageApiHandler.handle_image_generation_error( + "BytePlus Seedream Text-to-Image", str(e) + ) + + +class BytePlusSeedEditImageToImageNode: + """BytePlus SeedEdit Image-to-Image Generation Node""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "prompt": ("STRING", {"default": "", "multiline": True}), + "image": ("IMAGE",), + "size": (["adaptive", "1024x1024", "1152x896", "896x1152", "1216x832", "832x1216", + "1344x768", "768x1344", "1536x640", "640x1536"], {"default": "adaptive"}), + }, + "optional": { + "seed": ("INT", {"default": -1, "min": -1, "max": 2147483647}), + "guidance_scale": ("FLOAT", {"default": 5.5, "min": 1.0, "max": 20.0, "step": 0.1}), + }, + } + + RETURN_TYPES = ("IMAGE",) + FUNCTION = "generate_image" + CATEGORY = "BytePlus/ImageGeneration" + + def generate_image(self, prompt, image, size, seed=-1, guidance_scale=5.5): + try: + # Convert image to base64 + image_base64 = BytePlusImageUtils.image_to_base64(image) + if not image_base64: + return BytePlusImageApiHandler.handle_image_generation_error( + "BytePlus SeedEdit Image-to-Image", "Failed to convert image to base64" + ) + + arguments = { + "prompt": prompt, + "image": image_base64, + "n": 1, + "size": size, + "guidance_scale": guidance_scale, + "response_format": "url", + "watermark": False, # Always false, not exposed in UI + } + + # Only add seed if it's not -1 (random) + if seed != -1: + arguments["seed"] = seed + + result = BytePlusImageApiHandler.generate_image( + "seededit-3-0-i2i-250628", + arguments + ) + + return BytePlusResultProcessor.process_image_result(result) + + except Exception as e: + return BytePlusImageApiHandler.handle_image_generation_error( + "BytePlus SeedEdit Image-to-Image", str(e) + ) + + +# Node class mappings +NODE_CLASS_MAPPINGS = { + "BytePlusSeedreamTextToImage": BytePlusSeedreamTextToImageNode, + "BytePlusSeedEditImageToImage": BytePlusSeedEditImageToImageNode, +} + +# Node display name mappings +NODE_DISPLAY_NAME_MAPPINGS = { + "BytePlusSeedreamTextToImage": "Seedream Text-to-Image", + "BytePlusSeedEditImageToImage": "SeedEdit Image-to-Image", +} diff --git a/nodes/byteplus_utils.py b/nodes/byteplus_utils.py new file mode 100644 index 0000000..689d572 --- /dev/null +++ b/nodes/byteplus_utils.py @@ -0,0 +1,304 @@ +import base64 +import configparser +import io +import json +import os +import time +import tempfile +from typing import Optional, Dict, Any + +import numpy as np +import requests +import torch +from PIL import Image + + +class BytePlusConfig: + """Singleton class to handle BytePlus configuration and client setup.""" + + _instance = None + _api_key = None + _base_url = "https://ark.ap-southeast.bytepluses.com/api/v3" + + def __new__(cls): + if cls._instance is None: + cls._instance = super(BytePlusConfig, cls).__new__(cls) + cls._instance._initialize() + return cls._instance + + def _initialize(self): + """Initialize configuration and API key.""" + current_dir = os.path.dirname(os.path.abspath(__file__)) + parent_dir = os.path.dirname(current_dir) + config_path = os.path.join(parent_dir, "config.ini") + + config = configparser.ConfigParser() + config.read(config_path) + + try: + if os.environ.get("BYTEPLUS_API_KEY") is not None: + print("BYTEPLUS_API_KEY found in environment variables") + self._api_key = os.environ["BYTEPLUS_API_KEY"] + else: + print("BYTEPLUS_API_KEY not found in environment variables") + self._api_key = config["API"]["BYTEPLUS_API_KEY"] + print("BYTEPLUS_API_KEY found in config.ini") + os.environ["BYTEPLUS_API_KEY"] = self._api_key + print("BYTEPLUS_API_KEY set in environment variables") + + # Check if BytePlus key is the default placeholder + if self._api_key == "": + print("WARNING: You are using the default BytePlus API key placeholder!") + print("Please set your actual BytePlus API key in either:") + print("1. The config.ini file under [API] section") + print("2. Or as an environment variable named BYTEPLUS_API_KEY") + print("Get your API key from: https://console.volcengine.com/ark/region:ap-southeast-1/apiKey") + except KeyError: + print("Error: BYTEPLUS_API_KEY not found in config.ini or environment variables") + + def get_api_key(self): + """Get the BytePlus API key.""" + return self._api_key + + def get_base_url(self): + """Get the BytePlus API base URL.""" + return self._base_url + + +class BytePlusImageUtils: + """Utility functions for image processing specific to BytePlus API.""" + + @staticmethod + def tensor_to_pil(image): + """Convert image tensor to PIL Image.""" + try: + # Convert the image tensor to a numpy array + if isinstance(image, torch.Tensor): + image_np = image.cpu().numpy() + else: + image_np = np.array(image) + + # Ensure the image is in the correct format (H, W, C) + if image_np.ndim == 4: + image_np = image_np.squeeze(0) # Remove batch dimension if present + if image_np.ndim == 2: + image_np = np.stack([image_np] * 3, axis=-1) # Convert grayscale to RGB + elif image_np.shape[0] == 3: + image_np = np.transpose( + image_np, (1, 2, 0) + ) # Change from (C, H, W) to (H, W, C) + + # Normalize the image data to 0-255 range + if image_np.dtype == np.float32 or image_np.dtype == np.float64: + image_np = (image_np * 255).astype(np.uint8) + + # Convert to PIL Image + return Image.fromarray(image_np) + except Exception as e: + print(f"Error converting tensor to PIL: {str(e)}") + return None + + @staticmethod + def image_to_base64(image): + """Convert image tensor to base64 string format for BytePlus API.""" + try: + pil_image = BytePlusImageUtils.tensor_to_pil(image) + if not pil_image: + return None + + # Convert to base64 + buffered = io.BytesIO() + pil_image.save(buffered, format="PNG") + img_str = base64.b64encode(buffered.getvalue()).decode() + + # Format for BytePlus API + return f"data:image/png;base64,{img_str}" + except Exception as e: + print(f"Error converting image to base64: {str(e)}") + return None + + +class BytePlusApiHandler: + """Utility functions for BytePlus API interactions.""" + + @staticmethod + def create_video_generation_task(model: str, content: list, callback_url: Optional[str] = None) -> Optional[str]: + """Create a video generation task and return task ID.""" + config = BytePlusConfig() + + headers = { + "Authorization": f"Bearer {config.get_api_key()}", + "Content-Type": "application/json" + } + + payload = { + "model": model, + "content": content + } + + print(f"Making request to: {config.get_base_url()}/contents/generations/tasks") + print(f"Model: {model}") + print(f"API Key length: {len(config.get_api_key()) if config.get_api_key() else 0}") + print(f"API Key starts with: {config.get_api_key()[:10] if config.get_api_key() else 'None'}...") + print(f"Headers: {headers}") + print(f"Payload: {payload}") + + if callback_url: + payload["callback_url"] = callback_url + + try: + response = requests.post( + f"{config.get_base_url()}/contents/generations/tasks", + headers=headers, + json=payload, + timeout=30 + ) + + if response.status_code == 200: + result = response.json() + return result.get("id") + else: + print(f"Error creating task: {response.status_code} - {response.text}") + return None + + except Exception as e: + print(f"Error creating video generation task: {str(e)}") + return None + + @staticmethod + def query_task_status(task_id: str) -> Optional[Dict[str, Any]]: + """Query the status of a video generation task.""" + config = BytePlusConfig() + + headers = { + "Authorization": f"Bearer {config.get_api_key()}", + "Content-Type": "application/json" + } + + try: + response = requests.get( + f"{config.get_base_url()}/contents/generations/tasks/{task_id}", + headers=headers, + timeout=30 + ) + + if response.status_code == 200: + return response.json() + else: + print(f"Error querying task: {response.status_code} - {response.text}") + return None + + except Exception as e: + print(f"Error querying task status: {str(e)}") + return None + + @staticmethod + def wait_for_completion(task_id: str, max_wait_time: int = 300, poll_interval: int = 5) -> Optional[str]: + """Wait for task completion and return video URL.""" + start_time = time.time() + + while time.time() - start_time < max_wait_time: + result = BytePlusApiHandler.query_task_status(task_id) + + if not result: + print(f"Failed to query task status for {task_id}") + return None + + status = result.get("status") + print(f"Task {task_id} status: {status}") + + if status == "succeeded": + # Debug: Print the full response to understand the structure + print(f"Full successful response: {result}") + + # Look for video URL in the result - try multiple possible paths + if "content" in result and "video_url" in result["content"]: + print(f"Found video_url in content: {result['content']['video_url']}") + return result["content"]["video_url"] + elif "video_url" in result: + print(f"Found video_url at root level: {result['video_url']}") + return result["video_url"] + elif "result" in result and "video_url" in result["result"]: + print(f"Found video_url in result: {result['result']['video_url']}") + return result["result"]["video_url"] + elif "data" in result and "video_url" in result["data"]: + print(f"Found video_url in data: {result['data']['video_url']}") + return result["data"]["video_url"] + elif "output" in result and "video_url" in result["output"]: + print(f"Found video_url in output: {result['output']['video_url']}") + return result["output"]["video_url"] + elif "video" in result and "url" in result["video"]: + print(f"Found url in video: {result['video']['url']}") + return result["video"]["url"] + else: + print("Task succeeded but no video URL found") + print(f"Available keys: {list(result.keys())}") + return None + + elif status == "failed": + print(f"Task {task_id} failed: {result.get('error', 'Unknown error')}") + return None + + elif status in ["queued", "running"]: + time.sleep(poll_interval) + else: + print(f"Unknown task status: {status}") + return None + + print(f"Task {task_id} timed out after {max_wait_time} seconds") + return None + + @staticmethod + def submit_and_get_result(model: str, content: list, callback_url: Optional[str] = None) -> Optional[str]: + """Submit task and wait for completion, return video URL.""" + task_id = BytePlusApiHandler.create_video_generation_task(model, content, callback_url) + + if not task_id: + return None + + print(f"Created BytePlus task: {task_id}") + return BytePlusApiHandler.wait_for_completion(task_id) + + @staticmethod + def handle_video_generation_error(model_name: str, error: str) -> tuple: + """Handle video generation errors consistently.""" + print(f"Error generating video with {model_name}: {str(error)}") + # Return empty string instead of None to avoid URL errors + return ("",) + + +class BytePlusPromptBuilder: + """Utility class for building BytePlus prompts with text commands.""" + + @staticmethod + def build_prompt_with_commands( + prompt: str, + resolution: str = None, + ratio: str = None, + duration: int = None, + framepersecond: int = None, + watermark: bool = None, + seed: int = None, + camerafixed: bool = None + ) -> str: + """Build prompt with text commands appended.""" + commands = [] + + if resolution: + commands.append(f"--rs {resolution}") + if ratio: + commands.append(f"--rt {ratio}") + if duration is not None: + commands.append(f"--dur {duration}") + if framepersecond is not None: + commands.append(f"--fps {framepersecond}") + if watermark is not None: + commands.append(f"--wm {str(watermark).lower()}") + if seed is not None and seed != -1: + commands.append(f"--seed {seed}") + if camerafixed is not None: + commands.append(f"--cf {str(camerafixed).lower()}") + + if commands: + return f"{prompt} {' '.join(commands)}" + return prompt diff --git a/nodes/byteplus_video_node.py b/nodes/byteplus_video_node.py new file mode 100644 index 0000000..93875e2 --- /dev/null +++ b/nodes/byteplus_video_node.py @@ -0,0 +1,478 @@ +from .byteplus_utils import BytePlusApiHandler, BytePlusImageUtils, BytePlusPromptBuilder + + +class BytePlusSeedanceTextToVideoNode: + """BytePlus Seedance Text-to-Video Generation Node""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "prompt": ("STRING", {"default": "", "multiline": True}), + "resolution": (["480p", "720p", "1080p"], {"default": "720p"}), + "ratio": (["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"], {"default": "16:9"}), + "duration": ("INT", {"default": 5, "min": 3, "max": 12, "step": 1}), + }, + "optional": { + "framepersecond": ("INT", {"default": 24, "min": 24, "max": 24, "step": 1}), + "seed": ("INT", {"default": -1, "min": -1, "max": 2147483647}), + "camerafixed": ("BOOLEAN", {"default": False}), + }, + } + + RETURN_TYPES = ("STRING",) + FUNCTION = "generate_video" + CATEGORY = "BytePlus/VideoGeneration" + + def generate_video( + self, + prompt, + resolution, + ratio, + duration, + framepersecond=24, + seed=-1, + camerafixed=False + ): + try: + # Build prompt with text commands + full_prompt = BytePlusPromptBuilder.build_prompt_with_commands( + prompt=prompt, + resolution=resolution, + ratio=ratio, + duration=duration, + framepersecond=framepersecond, + watermark=False, + seed=seed, + camerafixed=camerafixed + ) + + # Prepare content for API + content = [ + { + "type": "text", + "text": full_prompt + } + ] + + # Submit to BytePlus API + video_url = BytePlusApiHandler.submit_and_get_result( + model="seedance-1-0-lite-t2v-250428", + content=content + ) + + if video_url and video_url.strip(): + return (video_url,) + else: + return BytePlusApiHandler.handle_video_generation_error( + "BytePlus Seedance Text-to-Video", "Failed to generate video" + ) + + except Exception as e: + return BytePlusApiHandler.handle_video_generation_error( + "BytePlus Seedance Text-to-Video", str(e) + ) + + +class BytePlusSeedanceImageToVideoNode: + """BytePlus Seedance Image-to-Video Generation Node (First Frame)""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "prompt": ("STRING", {"default": "", "multiline": True}), + "image": ("IMAGE",), + "resolution": (["480p", "720p", "1080p"], {"default": "720p"}), + "ratio": (["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"], {"default": "adaptive"}), + "duration": ("INT", {"default": 5, "min": 3, "max": 12, "step": 1}), + }, + "optional": { + "framepersecond": ("INT", {"default": 24, "min": 24, "max": 24, "step": 1}), + "seed": ("INT", {"default": -1, "min": -1, "max": 2147483647}), + "camerafixed": ("BOOLEAN", {"default": False}), + }, + } + + RETURN_TYPES = ("STRING",) + FUNCTION = "generate_video" + CATEGORY = "BytePlus/VideoGeneration" + + def generate_video( + self, + prompt, + image, + resolution, + ratio, + duration, + framepersecond=24, + seed=-1, + camerafixed=False + ): + try: + # Convert image to base64 + image_base64 = BytePlusImageUtils.image_to_base64(image) + if not image_base64: + return BytePlusApiHandler.handle_video_generation_error( + "BytePlus Seedance Image-to-Video", "Failed to convert image to base64" + ) + + # Build prompt with text commands + full_prompt = BytePlusPromptBuilder.build_prompt_with_commands( + prompt=prompt, + resolution=resolution, + ratio=ratio, + duration=duration, + framepersecond=framepersecond, + watermark=False, + seed=seed, + camerafixed=camerafixed + ) + + # Prepare content for API + content = [ + { + "type": "text", + "text": full_prompt + }, + { + "type": "image_url", + "image_url": { + "url": image_base64 + }, + "role": "first_frame" + } + ] + + # Submit to BytePlus API + video_url = BytePlusApiHandler.submit_and_get_result( + model="seedance-1-0-lite-i2v-250428", + content=content + ) + + if video_url and video_url.strip(): + return (video_url,) + else: + return BytePlusApiHandler.handle_video_generation_error( + "BytePlus Seedance Image-to-Video", "Failed to generate video" + ) + + except Exception as e: + return BytePlusApiHandler.handle_video_generation_error( + "BytePlus Seedance Image-to-Video", str(e) + ) + + +class BytePlusSeedanceFirstLastFrameNode: + """BytePlus Seedance First+Last Frame Video Generation Node""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "prompt": ("STRING", {"default": "", "multiline": True}), + "first_frame": ("IMAGE",), + "last_frame": ("IMAGE",), + "resolution": (["480p", "720p"], {"default": "720p"}), + "ratio": (["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"], {"default": "adaptive"}), + "duration": ("INT", {"default": 5, "min": 3, "max": 12, "step": 1}), + }, + "optional": { + "framepersecond": ("INT", {"default": 24, "min": 24, "max": 24, "step": 1}), + "seed": ("INT", {"default": -1, "min": -1, "max": 2147483647}), + }, + } + + RETURN_TYPES = ("STRING",) + FUNCTION = "generate_video" + CATEGORY = "BytePlus/VideoGeneration" + + def generate_video( + self, + prompt, + first_frame, + last_frame, + resolution, + ratio, + duration, + framepersecond=24, + seed=-1 + ): + try: + # Convert images to base64 + first_frame_base64 = BytePlusImageUtils.image_to_base64(first_frame) + last_frame_base64 = BytePlusImageUtils.image_to_base64(last_frame) + + if not first_frame_base64 or not last_frame_base64: + return BytePlusApiHandler.handle_video_generation_error( + "BytePlus Seedance First+Last Frame", "Failed to convert images to base64" + ) + + # Build prompt with text commands (no camerafixed for this mode) + full_prompt = BytePlusPromptBuilder.build_prompt_with_commands( + prompt=prompt, + resolution=resolution, + ratio=ratio, + duration=duration, + framepersecond=framepersecond, + watermark=False, + seed=seed + ) + + # Prepare content for API + content = [ + { + "type": "text", + "text": full_prompt + }, + { + "type": "image_url", + "image_url": { + "url": first_frame_base64 + }, + "role": "first_frame" + }, + { + "type": "image_url", + "image_url": { + "url": last_frame_base64 + }, + "role": "last_frame" + } + ] + + # Submit to BytePlus API (only supported by seedance-1-0-lite-i2v) + video_url = BytePlusApiHandler.submit_and_get_result( + model="seedance-1-0-lite-i2v-250428", + content=content + ) + + if video_url and video_url.strip(): + return (video_url,) + else: + return BytePlusApiHandler.handle_video_generation_error( + "BytePlus Seedance First+Last Frame", "Failed to generate video" + ) + + except Exception as e: + return BytePlusApiHandler.handle_video_generation_error( + "BytePlus Seedance First+Last Frame", str(e) + ) + + +class BytePlusSeedanceReferenceImagesNode: + """BytePlus Seedance Reference Images Video Generation Node""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "prompt": ("STRING", {"default": "", "multiline": True}), + "reference_image_1": ("IMAGE",), + "reference_image_2": ("IMAGE",), + "resolution": (["480p", "720p"], {"default": "720p"}), + "ratio": (["16:9", "4:3", "1:1", "3:4", "9:16", "21:9"], {"default": "16:9"}), + "duration": ("INT", {"default": 5, "min": 3, "max": 12, "step": 1}), + }, + "optional": { + "reference_image_3": ("IMAGE",), + "reference_image_4": ("IMAGE",), + "framepersecond": ("INT", {"default": 24, "min": 24, "max": 24, "step": 1}), + "seed": ("INT", {"default": -1, "min": -1, "max": 2147483647}), + }, + } + + RETURN_TYPES = ("STRING",) + FUNCTION = "generate_video" + CATEGORY = "BytePlus/VideoGeneration" + + def generate_video( + self, + prompt, + reference_image_1, + reference_image_2, + resolution, + ratio, + duration, + reference_image_3=None, + reference_image_4=None, + framepersecond=24, + seed=-1 + ): + try: + # Convert required images to base64 + ref_images = [reference_image_1, reference_image_2] + if reference_image_3 is not None: + ref_images.append(reference_image_3) + if reference_image_4 is not None: + ref_images.append(reference_image_4) + + # Build prompt with text commands (no camerafixed for reference images) + full_prompt = BytePlusPromptBuilder.build_prompt_with_commands( + prompt=prompt, + resolution=resolution, + ratio=ratio, + duration=duration, + framepersecond=framepersecond, + watermark=False, + seed=seed + ) + + # Prepare content for API + content = [ + { + "type": "text", + "text": full_prompt + } + ] + + # Add reference images + for i, ref_image in enumerate(ref_images): + image_base64 = BytePlusImageUtils.image_to_base64(ref_image) + if not image_base64: + return BytePlusApiHandler.handle_video_generation_error( + "BytePlus Seedance Reference Images", f"Failed to convert reference image {i+1} to base64" + ) + + content.append({ + "type": "image_url", + "image_url": { + "url": image_base64 + }, + "role": "reference_image" + }) + + # Submit to BytePlus API (only supported by seedance-1-0-lite-i2v) + video_url = BytePlusApiHandler.submit_and_get_result( + model="seedance-1-0-lite-i2v-250428", + content=content + ) + + if video_url and video_url.strip(): + return (video_url,) + else: + return BytePlusApiHandler.handle_video_generation_error( + "BytePlus Seedance Reference Images", "Failed to generate video" + ) + + except Exception as e: + return BytePlusApiHandler.handle_video_generation_error( + "BytePlus Seedance Reference Images", str(e) + ) + + +class BytePlusSeedanceProNode: + """BytePlus Seedance Pro Video Generation Node""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "prompt": ("STRING", {"default": "", "multiline": True}), + "mode": (["text-to-video", "image-to-video"], {"default": "text-to-video"}), + "resolution": (["480p", "720p", "1080p"], {"default": "1080p"}), + "ratio": (["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"], {"default": "adaptive"}), + "duration": ("INT", {"default": 5, "min": 3, "max": 12, "step": 1}), + }, + "optional": { + "image": ("IMAGE",), + "framepersecond": ("INT", {"default": 24, "min": 24, "max": 24, "step": 1}), + "seed": ("INT", {"default": -1, "min": -1, "max": 2147483647}), + "camerafixed": ("BOOLEAN", {"default": False}), + }, + } + + RETURN_TYPES = ("STRING",) + FUNCTION = "generate_video" + CATEGORY = "BytePlus/VideoGeneration" + + def generate_video( + self, + prompt, + mode, + resolution, + ratio, + duration, + image=None, + framepersecond=24, + seed=-1, + camerafixed=False + ): + try: + # Build prompt with text commands + full_prompt = BytePlusPromptBuilder.build_prompt_with_commands( + prompt=prompt, + resolution=resolution, + ratio=ratio, + duration=duration, + framepersecond=framepersecond, + watermark=False, + seed=seed, + camerafixed=camerafixed + ) + + # Prepare content for API + content = [ + { + "type": "text", + "text": full_prompt + } + ] + + # Add image if in image-to-video mode + if mode == "image-to-video": + if image is None: + return BytePlusApiHandler.handle_video_generation_error( + "BytePlus Seedance Pro", "Image is required for image-to-video mode" + ) + + image_base64 = BytePlusImageUtils.image_to_base64(image) + if not image_base64: + return BytePlusApiHandler.handle_video_generation_error( + "BytePlus Seedance Pro", "Failed to convert image to base64" + ) + + content.append({ + "type": "image_url", + "image_url": { + "url": image_base64 + }, + "role": "first_frame" + }) + + # Submit to BytePlus API + video_url = BytePlusApiHandler.submit_and_get_result( + model="seedance-1-0-pro-250528", + content=content + ) + + if video_url and video_url.strip(): + return (video_url,) + else: + return BytePlusApiHandler.handle_video_generation_error( + "BytePlus Seedance Pro", "Failed to generate video" + ) + + except Exception as e: + return BytePlusApiHandler.handle_video_generation_error( + "BytePlus Seedance Pro", str(e) + ) + + +# Node class mappings +NODE_CLASS_MAPPINGS = { + "SeedanceLiteTextToVideo": BytePlusSeedanceTextToVideoNode, + "SeedanceLiteImageToVideo": BytePlusSeedanceImageToVideoNode, + "SeedanceLiteFirstLastFrame": BytePlusSeedanceFirstLastFrameNode, + "SeedanceLiteReferenceImages": BytePlusSeedanceReferenceImagesNode, + "SeedanceProTextImageToVideo": BytePlusSeedanceProNode, +} + +# Node display name mappings +NODE_DISPLAY_NAME_MAPPINGS = { + "SeedanceLiteTextToVideo": "Seedance Lite Text-to-Video", + "SeedanceLiteImageToVideo": "Seedance Lite Image-to-Video", + "SeedanceLiteFirstLastFrame": "Seedance Lite First+Last Frame", + "SeedanceLiteReferenceImages": "Seedance Lite Reference Images", + "SeedanceProTextImageToVideo": "Seedance Pro Text/Image-to-Video", +} diff --git a/nodes/video_to_frames_node.py b/nodes/video_to_frames_node.py new file mode 100644 index 0000000..8258325 --- /dev/null +++ b/nodes/video_to_frames_node.py @@ -0,0 +1,158 @@ +import os +import torch +import requests +import tempfile +import numpy as np +from PIL import Image + +# Try to import opencv - if not available, provide error message +try: + import cv2 + OPENCV_AVAILABLE = True +except ImportError: + OPENCV_AVAILABLE = False + print("WARNING: opencv-python not installed. Video frame extraction will not work.") + print("To install: pip install opencv-python>=4.5.0") + + +def _fetch_video(url, stream=True): + """Download video content from URL.""" + return requests.get(url, stream=stream).content + + +class VideoToFrames: + """ + Simple node that extracts all frames from a video URL as a tensor batch. + Clean and focused approach without file saving complexity. + """ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "video_url": ("STRING", {"forceInput": True}), + } + } + + FUNCTION = "extract_frames" + CATEGORY = "BytePlus/Video" + + RETURN_TYPES = ("IMAGE", "INT", "FLOAT") + RETURN_NAMES = ("frames", "frame_count", "fps") + + def extract_frames(self, video_url): + """ + Simple function that downloads video and extracts all frames. + + Args: + video_url: URL of the video to download + + Returns: + frames: Extracted frames as IMAGE tensor batch + frame_count: Number of extracted frames + fps: FPS of the video + """ + + # Handle case where video_url might be a list + if isinstance(video_url, list): + video_url = video_url[0] + + # Initialize return values + frames_tensor = torch.zeros((1, 64, 64, 3), dtype=torch.float32) # Default empty frames + frame_count = 0 + video_fps = 0.0 + + # Download video content + try: + print(f"Downloading video from: {video_url}") + video_content = _fetch_video(video_url) + print(f"Downloaded {len(video_content)} bytes") + except Exception as e: + print(f"Error downloading video: {str(e)}") + return (frames_tensor, frame_count, video_fps) + + # Extract frames + if video_content: + try: + frames_tensor, frame_count, video_fps = self._extract_frames(video_content) + print(f"Extracted {frame_count} frames at {video_fps:.2f} fps") + except Exception as e: + print(f"Error extracting frames: {str(e)}") + frames_tensor = torch.zeros((1, 64, 64, 3), dtype=torch.float32) + frame_count = 0 + video_fps = 0.0 + + return (frames_tensor, frame_count, video_fps) + + def _extract_frames(self, video_content): + """Extract all frames from video content as tensor batch.""" + if not OPENCV_AVAILABLE: + print("Cannot extract frames: opencv-python not installed") + return torch.zeros((1, 64, 64, 3), dtype=torch.float32), 0, 0.0 + + # Create temporary file for OpenCV + with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as temp_file: + temp_file.write(video_content) + temp_path = temp_file.name + + try: + # Open video with OpenCV + cap = cv2.VideoCapture(temp_path) + + if not cap.isOpened(): + raise Exception("Could not open video file") + + # Get video properties + fps = cap.get(cv2.CAP_PROP_FPS) + total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) + + frames = [] + frame_count = 0 + + print(f"Video info: {total_frames} total frames at {fps:.2f} fps") + print(f"Extracting all frames...") + + while True: + ret, frame = cap.read() + if not ret: + break + + # Convert BGR to RGB + frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + + # Convert to PIL Image then to tensor + pil_image = Image.fromarray(frame_rgb) + frame_array = np.array(pil_image).astype(np.float32) / 255.0 + + frames.append(frame_array) + frame_count += 1 + + cap.release() + + if frames: + # Stack frames into tensor batch + frames_tensor = torch.from_numpy(np.stack(frames, axis=0)) + print(f"Created tensor with shape: {frames_tensor.shape}") + else: + # Return empty tensor if no frames extracted + frames_tensor = torch.zeros((1, 64, 64, 3), dtype=torch.float32) + frame_count = 0 + fps = 0.0 + + return frames_tensor, frame_count, fps + + finally: + # Clean up temporary file + if os.path.exists(temp_path): + os.unlink(temp_path) + + +# Node class mappings +NODE_CLASS_MAPPINGS = { + "VideoToFrames": VideoToFrames, +} + +# Node display name mappings +NODE_DISPLAY_NAME_MAPPINGS = { + "VideoToFrames": "Video URL to Frames", +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..af51792 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,25 @@ +[build-system] +requires = ["setuptools>=45", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "comfyui-byteplus-api" +version = "1.0.0" +description = "ComfyUI custom nodes for BytePlus (ByteDance) AI video generation API" +authors = [ + {name = "ComfyUI BytePlus Integration"} +] +license = {text = "MIT"} +readme = "README.md" +requires-python = ">=3.8" +dependencies = [ + "requests>=2.25.0", + "torch>=1.9.0", + "pillow>=8.0.0", + "numpy>=1.20.0", +] + +[project.urls] +Homepage = "https://github.com/your-repo/ComfyUI-BytePlus-API" +Repository = "https://github.com/your-repo/ComfyUI-BytePlus-API" +Issues = "https://github.com/your-repo/ComfyUI-BytePlus-API/issues" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..4aa2b2a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +requests>=2.25.0 +torch>=1.9.0 +pillow>=8.0.0 +numpy>=1.20.0 +opencv-python>=4.5.0