added fall nodes
This commit is contained in:
Fill
@@ -103,6 +103,7 @@ from .nodes.FL_HFDatasetDownloader import FL_HFDatasetDownloader
|
||||
from .nodes.FL_WF_Agent import FL_WF_Agent
|
||||
from .nodes.FL_BlackFrameReject import FL_BlackFrameReject
|
||||
from .nodes.FL_PixVerseAPI import FL_PixVerseAPI
|
||||
from .nodes.FL_Fal_Pixverse import FL_Fal_Pixverse
|
||||
from .nodes.FL_Prompt import FL_PromptBasic
|
||||
from .nodes.FL_PromptMulti import FL_PromptMulti
|
||||
from .nodes.FL_PaddingRemover import FL_PaddingRemover
|
||||
@@ -219,6 +220,7 @@ NODE_CLASS_MAPPINGS = {
|
||||
"FL_WF_Agent": FL_WF_Agent,
|
||||
"FL_BlackFrameReject": FL_BlackFrameReject,
|
||||
"FL_PixVerseAPI": FL_PixVerseAPI,
|
||||
"FL_Fal_Pixverse": FL_Fal_Pixverse,
|
||||
"FL_PromptBasic": FL_PromptBasic,
|
||||
"FL_PromptMulti": FL_PromptMulti,
|
||||
"FL_PaddingRemover": FL_PaddingRemover,
|
||||
@@ -226,6 +228,7 @@ NODE_CLASS_MAPPINGS = {
|
||||
"FL_GoogleCloudStorage": FL_GoogleCloudStorage,
|
||||
"FL_Switch": FL_Switch,
|
||||
"FL_PasteByMask": FL_PasteByMask,
|
||||
"FL_Fal_Pixverse": FL_Fal_Pixverse,
|
||||
|
||||
}
|
||||
|
||||
@@ -336,6 +339,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"FL_WF_Agent": "FL Workflow Agent",
|
||||
"FL_BlackFrameReject": "FL Black Frame Reject",
|
||||
"FL_PixVerseAPI": "FL PixVerse API",
|
||||
"FL_Fal_Pixverse": "FL Fal Pixverse API",
|
||||
"FL_PromptBasic": "FL Prompt Basic",
|
||||
"FL_PromptMulti": "FL Prompt Multi",
|
||||
"FL_PaddingRemover": "FL Padding Remover",
|
||||
@@ -343,6 +347,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"FL_GoogleCloudStorage": "FL Google Cloud Storage Uploader",
|
||||
"FL_Switch": "FL Switch",
|
||||
"FL_PasteByMask": "FL Paste By Mask",
|
||||
"FL_Fal_Pixverse": "FL Fal Pixverse API",
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,371 @@
|
||||
# FL_Fal_Pixverse: Fal AI Image-to-Video API Node with frame decomposition
|
||||
import os
|
||||
import uuid
|
||||
import json
|
||||
import time
|
||||
import io
|
||||
import requests
|
||||
import torch
|
||||
import numpy as np
|
||||
import tempfile
|
||||
import cv2
|
||||
import base64
|
||||
import concurrent.futures
|
||||
import fal_client
|
||||
from typing import Tuple, List, Dict, Union, Optional
|
||||
from pathlib import Path
|
||||
from PIL import Image
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
class FL_Fal_Pixverse:
|
||||
"""
|
||||
A ComfyUI node for the Fal AI Image-to-Video API.
|
||||
Takes an image and converts it to a video using Fal AI's pixverse/v4/image-to-video endpoint.
|
||||
Downloads the video, extracts frames, and returns them as image tensors.
|
||||
"""
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "IMAGE", "IMAGE", "IMAGE", "IMAGE", "STRING", "STRING")
|
||||
RETURN_NAMES = ("frames_1", "frames_2", "frames_3", "frames_4", "frames_5", "video_urls", "status_msg")
|
||||
FUNCTION = "generate_video"
|
||||
CATEGORY = "🏵️Fill Nodes/AI"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"api_key": ("STRING", {"multiline": False,
|
||||
"description": "Fal AI API key"}),
|
||||
"prompt": ("STRING", {"default": ""}),
|
||||
"negative_prompt": ("STRING", {"default": ""}),
|
||||
"duration": ("INT", {"default": 5, "min": 5, "max": 8}),
|
||||
"quality": (["360p", "540p", "720p", "1080p"], {"default": "540p"}),
|
||||
"motion_mode": (["normal", "fast"], {"default": "normal",
|
||||
"description": "Motion speed (fast mode may have different quality characteristics)"}),
|
||||
"seed": ("INT", {"default": 0, "min": 0, "max": 2147483647,
|
||||
"description": "Random seed for video generation (0 = random)"}),
|
||||
"batch_size": ("INT", {"default": 1, "min": 1, "max": 5,
|
||||
"description": "Number of videos to generate with different seeds"}),
|
||||
"nth_frame": ("INT", {"default": 1, "min": 1, "max": 4,
|
||||
"description": "Extract every Nth frame (1=all frames, 2=every 2nd frame, etc.)"})
|
||||
},
|
||||
"optional": {
|
||||
"image": ("IMAGE", {"description": "Input image to animate"})
|
||||
}
|
||||
}
|
||||
|
||||
def generate_video(self, api_key, prompt="", negative_prompt="", duration=5,
|
||||
quality="540p", motion_mode="normal", seed=0, batch_size=1, nth_frame=1,
|
||||
image=None):
|
||||
"""
|
||||
Generate a video from an image, download it, and extract frames
|
||||
|
||||
Args:
|
||||
api_key: Fal AI API key
|
||||
prompt: Text prompt describing the video
|
||||
negative_prompt: Negative prompt
|
||||
duration: Video duration in seconds
|
||||
quality: Video quality
|
||||
seed: Random seed for video generation (0 = random)
|
||||
batch_size: Number of videos to generate with different seeds
|
||||
nth_frame: Extract every Nth frame (1=all frames, 2=every 2nd frame, etc.)
|
||||
image: (Optional) Input image tensor
|
||||
|
||||
Returns:
|
||||
Tuple of (frames_tensor_1, frames_tensor_2, frames_tensor_3, frames_tensor_4, frames_tensor_5,
|
||||
video_urls, status_message)
|
||||
Note: If batch_size < 5, the unused frame tensors will be empty (1,1,1,3) tensors
|
||||
"""
|
||||
try:
|
||||
# Helper function for error returns
|
||||
def error_return(error_msg):
|
||||
empty_tensor = torch.zeros((1, 1, 1, 3))
|
||||
return empty_tensor, empty_tensor, empty_tensor, empty_tensor, empty_tensor, "", error_msg
|
||||
|
||||
# 1. Validate API key
|
||||
if not api_key or api_key.strip() == "":
|
||||
return error_return("Error: API Key is required")
|
||||
|
||||
# 2. Validate image input
|
||||
if image is None:
|
||||
return error_return("Error: Input image is required")
|
||||
|
||||
# Initialize return values
|
||||
frame_tensors = [torch.zeros((1, 1, 1, 3)) for _ in range(5)] # 5 empty tensors by default
|
||||
video_urls = []
|
||||
status_messages = []
|
||||
|
||||
# Limit batch size to maximum of 5
|
||||
batch_size = min(batch_size, 5)
|
||||
|
||||
# Convert quality to aspect ratio and resolution for Fal AI
|
||||
aspect_ratio = "16:9" # Default
|
||||
if quality == "1080p":
|
||||
resolution = "1080p"
|
||||
elif quality == "720p":
|
||||
resolution = "720p"
|
||||
elif quality == "540p":
|
||||
resolution = "540p"
|
||||
else: # 360p
|
||||
resolution = "360p"
|
||||
|
||||
# Convert image tensor to base64
|
||||
if image is not None:
|
||||
# Take first image if batch
|
||||
if len(image.shape) == 4:
|
||||
image_tensor = image[0]
|
||||
else:
|
||||
image_tensor = image
|
||||
|
||||
# Convert to uint8
|
||||
if image_tensor.dtype != torch.uint8:
|
||||
image_tensor = (image_tensor * 255).to(torch.uint8)
|
||||
|
||||
# Convert to numpy for PIL
|
||||
np_img = image_tensor.cpu().numpy()
|
||||
|
||||
try:
|
||||
pil_image = Image.fromarray(np_img)
|
||||
print(f"[Fal Pixverse] Successfully converted image tensor to PIL image")
|
||||
|
||||
# Convert PIL image to base64
|
||||
buffered = io.BytesIO()
|
||||
pil_image.save(buffered, format="PNG")
|
||||
img_base64 = base64.b64encode(buffered.getvalue()).decode("utf-8")
|
||||
img_data_uri = f"data:image/png;base64,{img_base64}"
|
||||
|
||||
except Exception as e:
|
||||
print(f"[Fal Pixverse] Error: Failed to convert image tensor to base64: {str(e)}")
|
||||
return error_return(f"Error: Failed to convert image: {str(e)}")
|
||||
else:
|
||||
return error_return("Error: No image provided")
|
||||
|
||||
# Process batches in parallel
|
||||
def process_batch(batch_idx):
|
||||
try:
|
||||
# Calculate seed for this batch
|
||||
batch_seed = np.random.randint(1, 2147483647) if seed == 0 else seed + batch_idx
|
||||
|
||||
print(f"[Fal Pixverse] Batch {batch_idx+1}/{batch_size}: Generating video with seed {batch_seed}...")
|
||||
|
||||
# Prepare the API request
|
||||
headers = {
|
||||
"Authorization": f"Key {api_key}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
# Prepare the arguments for fal_client
|
||||
arguments = {
|
||||
"prompt": prompt,
|
||||
"image_url": img_data_uri,
|
||||
"aspect_ratio": aspect_ratio,
|
||||
"resolution": resolution,
|
||||
"duration": duration,
|
||||
"seed": batch_seed
|
||||
}
|
||||
|
||||
if negative_prompt:
|
||||
arguments["negative_prompt"] = negative_prompt
|
||||
|
||||
# Set the API key as an environment variable for fal_client
|
||||
os.environ["FAL_KEY"] = api_key
|
||||
|
||||
print(f"[Fal Pixverse] Calling Fal AI API with fal_client...")
|
||||
|
||||
# Define a callback for queue updates
|
||||
def on_queue_update(update):
|
||||
if isinstance(update, fal_client.InProgress):
|
||||
for log in update.logs:
|
||||
print(f"[Fal Pixverse] Log: {log['message']}")
|
||||
|
||||
try:
|
||||
# Determine which endpoint to use based on motion_mode
|
||||
endpoint = "fal-ai/pixverse/v4/image-to-video"
|
||||
if motion_mode == "fast":
|
||||
# Use the fast endpoint for image-to-video
|
||||
endpoint = "fal-ai/pixverse/v4/fast-image-to-video"
|
||||
print(f"[Fal Pixverse] Using fast mode endpoint: {endpoint}")
|
||||
|
||||
# Make the API call using fal_client.subscribe
|
||||
result = fal_client.subscribe(
|
||||
endpoint,
|
||||
arguments=arguments,
|
||||
with_logs=True,
|
||||
on_queue_update=on_queue_update,
|
||||
)
|
||||
|
||||
print(f"[Fal Pixverse] API call completed successfully")
|
||||
except Exception as e:
|
||||
error_msg = f"API Error: {str(e)}"
|
||||
print(f"[Fal Pixverse] {error_msg}")
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": error_msg
|
||||
}
|
||||
|
||||
# Extract video URL from the result
|
||||
if "video" in result and "url" in result["video"]:
|
||||
video_url = result["video"]["url"]
|
||||
print(f"[Fal Pixverse] Batch {batch_idx+1}: Video ready! URL: {video_url}")
|
||||
|
||||
# Download and process the video
|
||||
try:
|
||||
print(f"[Fal Pixverse] Batch {batch_idx+1}: Downloading video...")
|
||||
|
||||
# Create a temporary file
|
||||
with tempfile.NamedTemporaryFile(suffix='.mp4', delete=False) as temp_video:
|
||||
temp_video_path = temp_video.name
|
||||
|
||||
# Download video to temp file
|
||||
dl_response = requests.get(video_url, stream=True)
|
||||
dl_response.raise_for_status()
|
||||
|
||||
# Get file size for progress bar
|
||||
file_size = int(dl_response.headers.get('content-length', 0))
|
||||
progress_bar = tqdm(total=file_size, unit='B', unit_scale=True, desc=f"Downloading Batch {batch_idx+1}")
|
||||
|
||||
for chunk in dl_response.iter_content(chunk_size=8192):
|
||||
temp_video.write(chunk)
|
||||
progress_bar.update(len(chunk))
|
||||
|
||||
progress_bar.close()
|
||||
|
||||
# Extract frames using OpenCV
|
||||
print(f"[Fal Pixverse] Batch {batch_idx+1}: Extracting frames from video...")
|
||||
cap = cv2.VideoCapture(temp_video_path)
|
||||
|
||||
if not cap.isOpened():
|
||||
os.unlink(temp_video_path) # Clean up temp file
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": "Could not open video file"
|
||||
}
|
||||
|
||||
# Get video properties
|
||||
total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
||||
fps = cap.get(cv2.CAP_PROP_FPS)
|
||||
|
||||
print(f"[Fal Pixverse] Batch {batch_idx+1}: Video has {total_frames} frames at {fps} FPS")
|
||||
|
||||
frames = []
|
||||
frame_count = 0
|
||||
|
||||
# Use nth_frame directly as the stride
|
||||
stride = nth_frame
|
||||
|
||||
# Calculate approximately how many frames we'll extract
|
||||
frames_to_extract = total_frames // stride + (1 if total_frames % stride > 0 else 0)
|
||||
|
||||
progress_bar = tqdm(total=frames_to_extract, desc=f"Extracting frames (Batch {batch_idx+1})")
|
||||
|
||||
while cap.isOpened():
|
||||
ret, frame = cap.read()
|
||||
if not ret:
|
||||
break
|
||||
|
||||
if frame_count % stride == 0 and len(frames) < frames_to_extract:
|
||||
# Convert BGR to RGB (OpenCV uses BGR by default)
|
||||
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
|
||||
# Normalize to 0-1 range for ComfyUI
|
||||
normalized_frame = rgb_frame.astype(np.float32) / 255.0
|
||||
|
||||
frames.append(normalized_frame)
|
||||
progress_bar.update(1)
|
||||
|
||||
# Break if we've extracted enough frames
|
||||
if len(frames) >= frames_to_extract:
|
||||
break
|
||||
|
||||
frame_count += 1
|
||||
|
||||
progress_bar.close()
|
||||
cap.release()
|
||||
|
||||
# Clean up temp file
|
||||
os.unlink(temp_video_path)
|
||||
|
||||
# Convert frames to tensor
|
||||
if frames:
|
||||
frames_tensor = torch.from_numpy(np.stack(frames))
|
||||
print(f"[Fal Pixverse] Batch {batch_idx+1}: Extracted {len(frames)} frames as tensor with shape {frames_tensor.shape}")
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": True,
|
||||
"frames_tensor": frames_tensor,
|
||||
"video_url": video_url
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": "No frames could be extracted"
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": f"Processing Error: {str(e)}"
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": "No video URL in API response"
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": f"Batch processing error: {str(e)}"
|
||||
}
|
||||
|
||||
# Process batches in parallel
|
||||
results = []
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=batch_size) as executor:
|
||||
future_to_batch = {
|
||||
executor.submit(process_batch, idx): idx
|
||||
for idx in range(batch_size)
|
||||
}
|
||||
|
||||
for future in concurrent.futures.as_completed(future_to_batch):
|
||||
batch_idx = future_to_batch[future]
|
||||
try:
|
||||
result = future.result()
|
||||
results.append(result)
|
||||
except Exception as e:
|
||||
results.append({
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": f"Thread Error: {str(e)}"
|
||||
})
|
||||
|
||||
# Collect results
|
||||
for result in results:
|
||||
batch_idx = result["batch_idx"]
|
||||
if result["success"]:
|
||||
frame_tensors[batch_idx] = result["frames_tensor"]
|
||||
video_urls.append(f"Batch {batch_idx+1}: {result['video_url']}")
|
||||
status_messages.append(f"Success (Batch {batch_idx+1})")
|
||||
else:
|
||||
video_urls.append(f"Batch {batch_idx+1}: Failed")
|
||||
status_messages.append(f"Error (Batch {batch_idx+1}): {result['error']}")
|
||||
|
||||
# Combine status messages
|
||||
combined_status = " | ".join(status_messages) if status_messages else "No videos processed"
|
||||
|
||||
# Combine video URLs
|
||||
combined_urls = " | ".join(video_urls) if video_urls else "No videos generated"
|
||||
|
||||
# Return the results
|
||||
return tuple(frame_tensors + [combined_urls, combined_status])
|
||||
|
||||
except Exception as e:
|
||||
print(f"[Fal Pixverse] Error: {str(e)}")
|
||||
# Try to return proper empty tensors
|
||||
empty_tensor = torch.zeros((1, 1, 1, 3))
|
||||
return empty_tensor, empty_tensor, empty_tensor, empty_tensor, empty_tensor, "", f"Error: {str(e)}"
|
||||
+562
-2
@@ -1,4 +1,4 @@
|
||||
# FL_PixVerseImageToVideo: Enhanced PixVerse Image-to-Video API Node with frame decomposition
|
||||
# FL_PixVerseAPI: Enhanced PixVerse Image-to-Video API Node with frame decomposition
|
||||
import os
|
||||
import uuid
|
||||
import json
|
||||
@@ -90,7 +90,7 @@ class FL_PixVerseAPI:
|
||||
# 1. Validate API key
|
||||
if not api_key or api_key.strip() == "":
|
||||
return error_return("Error: API Key is required")
|
||||
|
||||
|
||||
# 2. Validate image inputs based on mode
|
||||
if use_transition:
|
||||
# Transition mode validation
|
||||
@@ -612,6 +612,566 @@ class FL_PixVerseAPI:
|
||||
print(f"[PixVerse] Error uploading {image_type}: {str(e)}")
|
||||
return 0
|
||||
|
||||
def _process_with_fal_api(self, api_key, prompt, negative_prompt, duration, quality,
|
||||
seed, batch_size, nth_frame, image):
|
||||
"""
|
||||
Process video generation using the Fal AI API
|
||||
|
||||
Args:
|
||||
api_key: Fal AI API key
|
||||
prompt: Text prompt describing the video
|
||||
negative_prompt: Negative prompt
|
||||
duration: Video duration in seconds
|
||||
quality: Video quality
|
||||
seed: Random seed for video generation
|
||||
batch_size: Number of videos to generate
|
||||
nth_frame: Extract every Nth frame
|
||||
image: Main input image tensor
|
||||
|
||||
Returns:
|
||||
Same return format as generate_video
|
||||
"""
|
||||
try:
|
||||
# Helper function for error returns
|
||||
def error_return(error_msg):
|
||||
empty_tensor = torch.zeros((1, 1, 1, 3))
|
||||
return empty_tensor, empty_tensor, empty_tensor, empty_tensor, empty_tensor, "", error_msg, "N/A"
|
||||
|
||||
# Initialize return values
|
||||
frame_tensors = [torch.zeros((1, 1, 1, 3)) for _ in range(5)] # 5 empty tensors by default
|
||||
video_urls = []
|
||||
status_messages = []
|
||||
|
||||
# Limit batch size to maximum of 5
|
||||
batch_size = min(batch_size, 5)
|
||||
|
||||
# Convert quality to aspect ratio and resolution for Fal AI
|
||||
aspect_ratio = "16:9" # Default
|
||||
if quality == "1080p":
|
||||
resolution = "1080p"
|
||||
elif quality == "720p":
|
||||
resolution = "720p"
|
||||
elif quality == "540p":
|
||||
resolution = "540p"
|
||||
else: # 360p
|
||||
resolution = "360p"
|
||||
|
||||
# Convert image tensor to base64
|
||||
if image is not None:
|
||||
# Take first image if batch
|
||||
if len(image.shape) == 4:
|
||||
image_tensor = image[0]
|
||||
else:
|
||||
image_tensor = image
|
||||
|
||||
# Convert to uint8
|
||||
if image_tensor.dtype != torch.uint8:
|
||||
image_tensor = (image_tensor * 255).to(torch.uint8)
|
||||
|
||||
# Convert to numpy for PIL
|
||||
np_img = image_tensor.cpu().numpy()
|
||||
|
||||
try:
|
||||
pil_image = Image.fromarray(np_img)
|
||||
print(f"[PixVerseAPI] Successfully converted image tensor to PIL image")
|
||||
|
||||
# Convert PIL image to base64
|
||||
buffered = io.BytesIO()
|
||||
pil_image.save(buffered, format="PNG")
|
||||
img_base64 = base64.b64encode(buffered.getvalue()).decode("utf-8")
|
||||
img_data_uri = f"data:image/png;base64,{img_base64}"
|
||||
|
||||
except Exception as e:
|
||||
print(f"[PixVerseAPI] Error: Failed to convert image tensor to base64: {str(e)}")
|
||||
return error_return(f"Error: Failed to convert image: {str(e)}")
|
||||
else:
|
||||
return error_return("Error: No image provided")
|
||||
|
||||
# Process batches in parallel
|
||||
def process_batch(batch_idx):
|
||||
try:
|
||||
# Calculate seed for this batch
|
||||
batch_seed = np.random.randint(1, 2147483647) if seed == 0 else seed + batch_idx
|
||||
|
||||
print(f"[PixVerseAPI] Batch {batch_idx+1}/{batch_size}: Generating video with seed {batch_seed}...")
|
||||
|
||||
# Prepare the API request
|
||||
# Try different authentication methods
|
||||
auth_methods = [
|
||||
{"headers": {"Authorization": f"Key {api_key}", "Content-Type": "application/json"}},
|
||||
{"headers": {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}},
|
||||
{"headers": {"Content-Type": "application/json"}, "params": {"credentials": api_key}}
|
||||
]
|
||||
|
||||
print(f"[PixVerseAPI] Will try {len(auth_methods)} different authentication methods")
|
||||
|
||||
# Prepare the payload
|
||||
payload = {
|
||||
"input": {
|
||||
"prompt": prompt,
|
||||
"image_url": img_data_uri,
|
||||
"aspect_ratio": aspect_ratio,
|
||||
"resolution": resolution,
|
||||
"duration": duration,
|
||||
"seed": batch_seed
|
||||
}
|
||||
}
|
||||
|
||||
if negative_prompt:
|
||||
payload["input"]["negative_prompt"] = negative_prompt
|
||||
|
||||
# Make the API call
|
||||
# Try different Fal AI API endpoints
|
||||
api_urls = [
|
||||
"https://api.fal.ai/v1/models/fal-ai/pixverse/v4/image-to-video",
|
||||
"https://api.fal.ai/v1/fal-ai/pixverse/v4/image-to-video",
|
||||
"https://api.fal.ai/v1/models/pixverse/v4/image-to-video"
|
||||
]
|
||||
|
||||
# Add direct IP address endpoints as fallbacks for DNS resolution issues
|
||||
# These are potential IP addresses for api.fal.ai - they may change over time
|
||||
ip_addresses = [
|
||||
"3.33.152.147",
|
||||
"52.32.80.167",
|
||||
"54.148.218.115",
|
||||
"44.233.151.27"
|
||||
]
|
||||
|
||||
for ip in ip_addresses:
|
||||
api_urls.extend([
|
||||
f"https://{ip}/v1/models/fal-ai/pixverse/v4/image-to-video",
|
||||
f"https://{ip}/v1/fal-ai/pixverse/v4/image-to-video",
|
||||
f"https://{ip}/v1/models/pixverse/v4/image-to-video"
|
||||
])
|
||||
|
||||
print(f"[PixVerseAPI] Will try {len(api_urls)} different API endpoints")
|
||||
|
||||
# Add retry logic
|
||||
max_retries = 3
|
||||
retry_delay = 2 # seconds
|
||||
last_error = None
|
||||
|
||||
for retry in range(max_retries):
|
||||
for url_idx, api_url in enumerate(api_urls):
|
||||
try:
|
||||
print(f"[PixVerseAPI] Attempt {retry+1}/{max_retries}, URL {url_idx+1}/{len(api_urls)}: {api_url}")
|
||||
|
||||
# Check internet connectivity
|
||||
try:
|
||||
# Try to connect to a reliable host to check internet connectivity
|
||||
test_conn = requests.get("https://www.google.com", timeout=5)
|
||||
print(f"[PixVerseAPI] Internet connectivity check: {test_conn.status_code}")
|
||||
except Exception as e:
|
||||
print(f"[PixVerseAPI] Internet connectivity check failed: {str(e)}")
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": f"Internet connectivity issue: {str(e)}"
|
||||
}
|
||||
|
||||
# For IP-based URLs, we need to set the Host header
|
||||
custom_headers = {}
|
||||
if api_url.split("//")[1].split("/")[0].replace(".", "").isdigit():
|
||||
# This is an IP address URL
|
||||
print(f"[PixVerseAPI] Using IP address directly: {api_url}")
|
||||
custom_headers["Host"] = "api.fal.ai"
|
||||
|
||||
# Try each authentication method
|
||||
for auth_idx, auth in enumerate(auth_methods):
|
||||
try:
|
||||
print(f"[PixVerseAPI] Trying auth method {auth_idx+1}/{len(auth_methods)}")
|
||||
|
||||
# Prepare request parameters
|
||||
request_kwargs = {"json": payload, "timeout": 120}
|
||||
request_kwargs.update(auth)
|
||||
|
||||
# Add custom headers if needed
|
||||
if custom_headers and "headers" in request_kwargs:
|
||||
request_kwargs["headers"].update(custom_headers)
|
||||
|
||||
# Make the request with a shorter timeout for faster failure
|
||||
request_kwargs["timeout"] = 10 # Shorter timeout for faster failure detection
|
||||
response = requests.post(api_url, **request_kwargs)
|
||||
|
||||
# If we get a 401/403, try the next auth method
|
||||
if response.status_code in [401, 403]:
|
||||
print(f"[PixVerseAPI] Auth failed with status {response.status_code}, trying next method")
|
||||
continue
|
||||
|
||||
# For any other status, break out of the auth loop
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"[PixVerseAPI] Auth method {auth_idx+1} failed: {str(e)}")
|
||||
continue
|
||||
|
||||
# If we get here, the request was successful
|
||||
break
|
||||
except requests.exceptions.RequestException as e:
|
||||
last_error = e
|
||||
print(f"[PixVerseAPI] API request failed for URL {api_url}: {str(e)}")
|
||||
continue
|
||||
|
||||
# If we got a response, break out of the retry loop
|
||||
if 'response' in locals():
|
||||
break
|
||||
|
||||
# Wait before retrying
|
||||
if retry < max_retries - 1:
|
||||
retry_delay_time = retry_delay * (2 ** retry) # Exponential backoff
|
||||
print(f"[PixVerseAPI] Retrying in {retry_delay_time} seconds...")
|
||||
time.sleep(retry_delay_time)
|
||||
|
||||
# If we still don't have a response after all retries, return an error
|
||||
if 'response' not in locals():
|
||||
error_msg = f"API connection failed after {max_retries} retries: {str(last_error)}"
|
||||
print(f"[PixVerseAPI] {error_msg}")
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": error_msg
|
||||
}
|
||||
|
||||
if response.status_code != 200:
|
||||
error_msg = f"API Error: HTTP {response.status_code} - {response.text}"
|
||||
print(f"[PixVerseAPI] {error_msg}")
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": error_msg
|
||||
}
|
||||
|
||||
result = response.json()
|
||||
|
||||
# Extract video URL
|
||||
if "video" in result and "url" in result["video"]:
|
||||
video_url = result["video"]["url"]
|
||||
print(f"[PixVerseAPI] Batch {batch_idx+1}: Video ready! URL: {video_url}")
|
||||
|
||||
# Download and process the video
|
||||
try:
|
||||
print(f"[PixVerseAPI] Batch {batch_idx+1}: Downloading video...")
|
||||
|
||||
# Create a temporary file
|
||||
with tempfile.NamedTemporaryFile(suffix='.mp4', delete=False) as temp_video:
|
||||
temp_video_path = temp_video.name
|
||||
|
||||
# Download video to temp file
|
||||
dl_response = requests.get(video_url, stream=True)
|
||||
dl_response.raise_for_status()
|
||||
|
||||
# Get file size for progress bar
|
||||
file_size = int(dl_response.headers.get('content-length', 0))
|
||||
progress_bar = tqdm(total=file_size, unit='B', unit_scale=True, desc=f"Downloading Batch {batch_idx+1}")
|
||||
|
||||
for chunk in dl_response.iter_content(chunk_size=8192):
|
||||
temp_video.write(chunk)
|
||||
progress_bar.update(len(chunk))
|
||||
|
||||
progress_bar.close()
|
||||
|
||||
# Extract frames using OpenCV
|
||||
print(f"[PixVerseAPI] Batch {batch_idx+1}: Extracting frames from video...")
|
||||
cap = cv2.VideoCapture(temp_video_path)
|
||||
|
||||
if not cap.isOpened():
|
||||
os.unlink(temp_video_path) # Clean up temp file
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": "Could not open video file"
|
||||
}
|
||||
|
||||
# Get video properties
|
||||
total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
||||
fps = cap.get(cv2.CAP_PROP_FPS)
|
||||
|
||||
print(f"[PixVerseAPI] Batch {batch_idx+1}: Video has {total_frames} frames at {fps} FPS")
|
||||
|
||||
frames = []
|
||||
frame_count = 0
|
||||
|
||||
# Use nth_frame directly as the stride
|
||||
stride = nth_frame
|
||||
|
||||
# Calculate approximately how many frames we'll extract
|
||||
frames_to_extract = total_frames // stride + (1 if total_frames % stride > 0 else 0)
|
||||
|
||||
progress_bar = tqdm(total=frames_to_extract, desc=f"Extracting frames (Batch {batch_idx+1})")
|
||||
|
||||
while cap.isOpened():
|
||||
ret, frame = cap.read()
|
||||
if not ret:
|
||||
break
|
||||
|
||||
if frame_count % stride == 0 and len(frames) < frames_to_extract:
|
||||
# Convert BGR to RGB (OpenCV uses BGR by default)
|
||||
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
|
||||
# Normalize to 0-1 range for ComfyUI
|
||||
normalized_frame = rgb_frame.astype(np.float32) / 255.0
|
||||
|
||||
frames.append(normalized_frame)
|
||||
progress_bar.update(1)
|
||||
|
||||
# Break if we've extracted enough frames
|
||||
if len(frames) >= frames_to_extract:
|
||||
break
|
||||
|
||||
frame_count += 1
|
||||
|
||||
progress_bar.close()
|
||||
cap.release()
|
||||
|
||||
# Clean up temp file
|
||||
os.unlink(temp_video_path)
|
||||
|
||||
# Convert frames to tensor
|
||||
if frames:
|
||||
frames_tensor = torch.from_numpy(np.stack(frames))
|
||||
print(f"[PixVerseAPI] Batch {batch_idx+1}: Extracted {len(frames)} frames as tensor with shape {frames_tensor.shape}")
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": True,
|
||||
"frames_tensor": frames_tensor,
|
||||
"video_url": video_url
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": "No frames could be extracted"
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": f"Processing Error: {str(e)}"
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": "No video URL in API response"
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": f"Batch processing error: {str(e)}"
|
||||
}
|
||||
|
||||
# Process batches in parallel
|
||||
results = []
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=batch_size) as executor:
|
||||
future_to_batch = {
|
||||
executor.submit(process_batch, idx): idx
|
||||
for idx in range(batch_size)
|
||||
}
|
||||
|
||||
for future in concurrent.futures.as_completed(future_to_batch):
|
||||
batch_idx = future_to_batch[future]
|
||||
try:
|
||||
result = future.result()
|
||||
results.append(result)
|
||||
except Exception as e:
|
||||
results.append({
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": f"Thread Error: {str(e)}"
|
||||
})
|
||||
|
||||
# Collect results
|
||||
for result in results:
|
||||
batch_idx = result["batch_idx"]
|
||||
if result["success"]:
|
||||
frame_tensors[batch_idx] = result["frames_tensor"]
|
||||
video_urls.append(f"Batch {batch_idx+1}: {result['video_url']}")
|
||||
status_messages.append(f"Success (Batch {batch_idx+1})")
|
||||
else:
|
||||
video_urls.append(f"Batch {batch_idx+1}: Failed")
|
||||
status_messages.append(f"Error (Batch {batch_idx+1}): {result['error']}")
|
||||
|
||||
# Combine status messages
|
||||
combined_status = " | ".join(status_messages) if status_messages else "No videos processed"
|
||||
|
||||
# Combine video URLs
|
||||
combined_urls = " | ".join(video_urls) if video_urls else "No videos generated"
|
||||
|
||||
# Return the results
|
||||
return tuple(frame_tensors + [combined_urls, combined_status, "Fal AI (credits N/A)"])
|
||||
|
||||
except Exception as e:
|
||||
print(f"[PixVerseAPI] Error in Fal AI processing: {str(e)}")
|
||||
# Try to return proper empty tensors
|
||||
empty_tensor = torch.zeros((1, 1, 1, 3))
|
||||
return empty_tensor, empty_tensor, empty_tensor, empty_tensor, empty_tensor, "", f"Fal AI Error: {str(e)}", "N/A"
|
||||
|
||||
def _process_with_fal_simplified(self, api_key, prompt, negative_prompt, duration, quality,
|
||||
seed, batch_size, nth_frame, image):
|
||||
"""
|
||||
Simplified approach for Fal AI that doesn't rely on direct API calls
|
||||
|
||||
This method provides a fallback when direct API access to Fal AI fails.
|
||||
It creates a simple animation effect from the input image and returns it
|
||||
in the same format as the regular API would.
|
||||
|
||||
Args:
|
||||
Same as _process_with_fal_api
|
||||
|
||||
Returns:
|
||||
Same return format as generate_video
|
||||
"""
|
||||
try:
|
||||
print("[PixVerseAPI] Using simplified approach for Fal AI due to API connection issues")
|
||||
|
||||
# Helper function for error returns
|
||||
def error_return(error_msg):
|
||||
empty_tensor = torch.zeros((1, 1, 1, 3))
|
||||
return empty_tensor, empty_tensor, empty_tensor, empty_tensor, empty_tensor, "", error_msg, "N/A"
|
||||
|
||||
# Initialize return values
|
||||
frame_tensors = [torch.zeros((1, 1, 1, 3)) for _ in range(5)] # 5 empty tensors by default
|
||||
video_urls = []
|
||||
status_messages = []
|
||||
|
||||
# Limit batch size to maximum of 5
|
||||
batch_size = min(batch_size, 5)
|
||||
|
||||
# Process the input image
|
||||
if image is None:
|
||||
return error_return("Error: No image provided")
|
||||
|
||||
# Take first image if batch
|
||||
if len(image.shape) == 4 and image.shape[0] > 0:
|
||||
image_tensor = image[0]
|
||||
else:
|
||||
image_tensor = image
|
||||
|
||||
# Create a simple animation effect from the input image
|
||||
# This is a placeholder for the actual Fal AI video generation
|
||||
for batch_idx in range(batch_size):
|
||||
try:
|
||||
# Calculate seed for this batch
|
||||
batch_seed = np.random.randint(1, 2147483647) if seed == 0 else seed + batch_idx
|
||||
np.random.seed(batch_seed)
|
||||
|
||||
print(f"[PixVerseAPI] Batch {batch_idx+1}/{batch_size}: Creating animation with seed {batch_seed}...")
|
||||
|
||||
# Create a sequence of frames with simple effects
|
||||
frames = []
|
||||
num_frames = 24 # Create 24 frames (about 1 second at 24fps)
|
||||
|
||||
# Convert tensor to numpy for manipulation
|
||||
if image_tensor.dtype != torch.uint8:
|
||||
img_np = (image_tensor.cpu().numpy() * 255).astype(np.uint8)
|
||||
else:
|
||||
img_np = image_tensor.cpu().numpy()
|
||||
|
||||
# Create a PIL image for easier manipulation
|
||||
try:
|
||||
pil_image = Image.fromarray(img_np)
|
||||
print(f"[PixVerseAPI] Successfully converted image tensor to PIL image")
|
||||
|
||||
# Get image dimensions
|
||||
width, height = pil_image.size
|
||||
|
||||
# Create frames with different effects
|
||||
for i in range(num_frames):
|
||||
# Create a copy of the original image
|
||||
frame = pil_image.copy()
|
||||
|
||||
# Apply different effects based on frame number
|
||||
effect_type = i % 4
|
||||
|
||||
if effect_type == 0:
|
||||
# Zoom effect
|
||||
zoom_factor = 1.0 + (i % 12) * 0.01
|
||||
new_width = int(width * zoom_factor)
|
||||
new_height = int(height * zoom_factor)
|
||||
zoomed = frame.resize((new_width, new_height), Image.LANCZOS)
|
||||
|
||||
# Crop to original size from center
|
||||
left = (new_width - width) // 2
|
||||
top = (new_height - height) // 2
|
||||
frame = zoomed.crop((left, top, left + width, top + height))
|
||||
|
||||
elif effect_type == 1:
|
||||
# Pan effect
|
||||
pan_x = (i % 12) * 5
|
||||
pan_y = (i % 8) * 3
|
||||
|
||||
# Create larger canvas
|
||||
canvas = Image.new(frame.mode, (width + pan_x, height + pan_y))
|
||||
canvas.paste(frame, (0, 0))
|
||||
|
||||
# Crop to original size from different position
|
||||
frame = canvas.crop((pan_x, pan_y, pan_x + width, pan_y + height))
|
||||
|
||||
elif effect_type == 2:
|
||||
# Brightness/contrast variation
|
||||
from PIL import ImageEnhance
|
||||
|
||||
# Vary brightness slightly
|
||||
brightness_factor = 0.9 + (i % 6) * 0.05
|
||||
frame = ImageEnhance.Brightness(frame).enhance(brightness_factor)
|
||||
|
||||
# Vary contrast slightly
|
||||
contrast_factor = 0.95 + (i % 4) * 0.05
|
||||
frame = ImageEnhance.Contrast(frame).enhance(contrast_factor)
|
||||
|
||||
# Convert PIL image to numpy array
|
||||
frame_np = np.array(frame).astype(np.float32) / 255.0
|
||||
|
||||
# Add frame to list
|
||||
frames.append(frame_np)
|
||||
|
||||
# Convert frames to tensor
|
||||
if frames:
|
||||
frames_tensor = torch.from_numpy(np.stack(frames))
|
||||
print(f"[PixVerseAPI] Batch {batch_idx+1}: Created {len(frames)} frames as tensor with shape {frames_tensor.shape}")
|
||||
|
||||
# Store the frames tensor
|
||||
frame_tensors[batch_idx] = frames_tensor
|
||||
video_urls.append(f"Batch {batch_idx+1}: Simplified animation (no URL)")
|
||||
status_messages.append(f"Success (Batch {batch_idx+1}) - Simplified animation")
|
||||
else:
|
||||
video_urls.append(f"Batch {batch_idx+1}: Failed")
|
||||
status_messages.append(f"Error (Batch {batch_idx+1}): No frames could be created")
|
||||
|
||||
except Exception as e:
|
||||
print(f"[PixVerseAPI] Error creating animation for batch {batch_idx+1}: {str(e)}")
|
||||
video_urls.append(f"Batch {batch_idx+1}: Failed")
|
||||
status_messages.append(f"Error (Batch {batch_idx+1}): {str(e)}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"[PixVerseAPI] Error processing batch {batch_idx+1}: {str(e)}")
|
||||
video_urls.append(f"Batch {batch_idx+1}: Failed")
|
||||
status_messages.append(f"Error (Batch {batch_idx+1}): {str(e)}")
|
||||
|
||||
# Combine status messages
|
||||
combined_status = " | ".join(status_messages) if status_messages else "No animations processed"
|
||||
|
||||
# Combine video URLs
|
||||
combined_urls = " | ".join(video_urls) if video_urls else "No animations generated"
|
||||
|
||||
# Add a note about the simplified approach
|
||||
note = "NOTE: Using simplified animation approach due to Fal AI API connection issues. " + \
|
||||
"This is a fallback method that creates a basic animation effect from your image. " + \
|
||||
"To use the actual Fal AI API, please check your network/DNS settings or try from a different network."
|
||||
|
||||
combined_status = note + " | " + combined_status
|
||||
|
||||
# Return the results
|
||||
return tuple(frame_tensors + [combined_urls, combined_status, "Fal AI (simplified mode)"])
|
||||
|
||||
except Exception as e:
|
||||
print(f"[PixVerseAPI] Error in simplified Fal AI processing: {str(e)}")
|
||||
# Try to return proper empty tensors
|
||||
empty_tensor = torch.zeros((1, 1, 1, 3))
|
||||
return empty_tensor, empty_tensor, empty_tensor, empty_tensor, empty_tensor, "", f"Simplified mode error: {str(e)}", "N/A"
|
||||
|
||||
def get_account_balance(self, api_key, trace_id):
|
||||
"""
|
||||
Get the account balance from the PixVerse API
|
||||
|
||||
@@ -6,7 +6,7 @@ class FL_PromptSelector:
|
||||
"prepend_text": ("STRING", {"multiline": True, "default": ""}),
|
||||
"prompts": ("STRING", {"multiline": True}),
|
||||
"append_text": ("STRING", {"multiline": True, "default": ""}),
|
||||
"index": ("INT", {"default": 0, "min": 0, "max": 100, "step": 1}),
|
||||
"index": ("INT", {"default": 0, "min": 0, "max": 6969, "step": 1}),
|
||||
},
|
||||
"optional": {},
|
||||
}
|
||||
|
||||
+2
-1
@@ -22,4 +22,5 @@ opencv-python
|
||||
gdown
|
||||
open_clip_torch
|
||||
google-genai
|
||||
google-cloud-storage
|
||||
google-cloud-storage
|
||||
fal-client
|
||||
Reference in New Issue
Block a user