demo release
This commit is contained in:
SolitaryThinker
@@ -1,59 +0,0 @@
|
||||
# FastVideo Gradio Demo
|
||||
|
||||
This is a Gradio-based web interface for generating videos using the FastVideo framework. The demo allows users to create videos from text prompts with various customization options.
|
||||
|
||||
## Overview
|
||||
|
||||
The demo uses the FastVideo framework to generate videos based on text prompts. It provides a simple web interface built with Gradio that allows users to:
|
||||
|
||||
- Enter text prompts to generate videos
|
||||
- Customize video parameters (dimensions, number of frames, etc.)
|
||||
- Use negative prompts to guide the generation process
|
||||
- Set or randomize seeds for reproducibility
|
||||
|
||||
---
|
||||
|
||||
## Usage
|
||||
|
||||
Run the demo with:
|
||||
|
||||
```bash
|
||||
python examples/inference/gradio/gradio_demo.py
|
||||
```
|
||||
|
||||
This will start a web server at `http://0.0.0.0:7860` where you can access the interface.
|
||||
|
||||
---
|
||||
|
||||
## Model Initialization
|
||||
|
||||
This demo initializes a `VideoGenerator` with the minimum required arguments for inference. Users can seamlessly adjust inference options between generations, including prompts, resolution, video length, or even the number of inference steps, *without ever needing to reload the model*.
|
||||
|
||||
## Video Generation
|
||||
|
||||
The core functionality is in the `generate_video` function, which:
|
||||
1. Processes user inputs
|
||||
2. Uses the FastVideo VideoGenerator from earlier to run inference (`generator.generate_video()`)
|
||||
3. Returns an output path that Gradio uses to display the generated video
|
||||
|
||||
## Gradio Interface
|
||||
|
||||
The interface is built with several components:
|
||||
- A text input for the prompt
|
||||
- A video display for the result
|
||||
- Inference options in a collapsible accordion:
|
||||
- Height and width sliders
|
||||
- Number of frames slider
|
||||
- Guidance scale slider
|
||||
- Inference steps slider
|
||||
- Negative prompt options
|
||||
- Seed controls
|
||||
|
||||
### Inference Options
|
||||
|
||||
- **Height/Width**: Control the resolution of the generated video
|
||||
- **Number of Frames**: Set how many frames to generate
|
||||
- **Guidance Scale**: Control how closely the generation follows the prompt
|
||||
- **Inference Steps**: More steps can improve quality but take longer
|
||||
- **Negative Prompt**: Specify what you don't want to see in the video
|
||||
- **Seed**: Control randomness for reproducible results
|
||||
@@ -1,501 +0,0 @@
|
||||
import argparse
|
||||
import os
|
||||
import time
|
||||
import json
|
||||
import statistics
|
||||
import asyncio
|
||||
import aiohttp
|
||||
from copy import deepcopy
|
||||
from typing import List, Dict, Any
|
||||
import threading
|
||||
|
||||
import torch
|
||||
|
||||
# All the prompts for stress testing
|
||||
STRESS_TEST_PROMPTS = [
|
||||
"A person reading a book with words that float off the pages and form pictures.",
|
||||
"A person diving into a pool of liquid crystal, creating ripples of light.",
|
||||
"A handheld shot chasing after a group of friends laughing and playing on the beach at sunset.",
|
||||
"A mysterious ancient temple hidden in the jungle.",
|
||||
"A high-speed train navigating a steep descent.",
|
||||
"a toy robot wearing blue jeans and a white t shirt taking a pleasant stroll in Antarctica during a winter storm",
|
||||
"A cheetah accelerating to full speed while chasing its prey.",
|
||||
"A serene orchard is in full bloom, with trees heavy with blossoms and bees buzzing around, darting from flower to flower in a display of natural harmony.",
|
||||
"A little child let out a big yawn",
|
||||
"Subtle reflections of a woman on the window of a train moving at hyper-speed in a Japanese city.",
|
||||
"A truck left along the edge of a cliff, revealing the stunning coastal landscape below with waves crashing against the rocks.",
|
||||
"A red bird transforms into a flag",
|
||||
"A zoom-out from a single leaf on a tree to reveal the entire forest, showcasing the vastness and diversity of the woodland.",
|
||||
"A slow-motion video of a liquid droplet bouncing on a water-repellent surface.",
|
||||
"Static camera shot. A dinasour running near some lions and chasing them away.",
|
||||
"an adorable kangaroo wearing purple overalls and cowboy boots taking a pleasant stroll in Mumbai India during a beautiful sunset",
|
||||
"A zoom-in on an artist's brush touching the canvas, highlighting the texture of the paint and the strokes being made.",
|
||||
"an old man wearing blue jeans and a white t shirt taking a pleasant stroll in Mumbai India during a colorful festival",
|
||||
"A woman is ascending to the sky from the ground",
|
||||
"View out a window of a giant strange creature walking in rundown city at night, one single street lamp dimly lighting the area.",
|
||||
"An arc shot around a lone tree in a vast, foggy field at dawn, revealing the changing light and shadows.",
|
||||
"A person sculpting a statue out of a waterfall, the water solidifying under their touch.",
|
||||
"The person's forehead creased with concentration as she worked on a challenging puzzle.",
|
||||
"The person's cheeks flushed with pleasure as she savored a delicious meal.",
|
||||
"Hand-drawn simple line art, a young kid looking up into space with a wondrous expression on his face.",
|
||||
"A crab made of different jewlery is walking on the beach. As it walks, it drops different jewelry pieces like diamonds, pearls, etc",
|
||||
"Gold coins are falling out when elevator door opens",
|
||||
"the scene transitions from huge waves into a snowy mountain at sunset",
|
||||
"a giant cathedral is completely filled with cats. there are cats everywhere you look. a man enters the cathedral and bows before the giant cat king sitting on a throne.",
|
||||
"A mother dog gently picks up a piece of meat and carefully places it in her puppy's bowl, her eyes filled with warmth and care as she watches her little one eat.",
|
||||
"A soap bubble floating in the air, displaying iridescent colors that shift and change as it moves through different angles of light.",
|
||||
"A truck left alongside a train moving through the countryside, matching its speed and revealing the changing landscape.",
|
||||
"An astronaut walking between stone buildings.",
|
||||
"A close-up shot of the person's face reveals his fear and desperation as he navigates the ship through the storm.",
|
||||
"A frozen lake slowly cracking and thawing as spring arrives, with sheets of ice breaking apart and drifting across the surface.",
|
||||
"A FPV shot zooming through a tunnel into a vibrant underwater space.",
|
||||
"a toy robot wearing blue jeans and a white t shirt taking a pleasant stroll in Mumbai India during a colorful festival",
|
||||
"A person sips on a smoothie, the cool and fruity flavors refreshing her mouth.",
|
||||
"In a vibrant theater, a magician in dazzling attire stands center stage, pulling a comically oversized rubber chicken from an ornate, old-fashioned box. His costume shimmers under the stage lights, adding to the spectacle. The crowd erupts in laughter and applause, their faces filled with joy and amazement. The magician's expression hints at mischievous delight as he holds up the rubber chicken, his performance bringing cheer to the audience.",
|
||||
"A hamster running on a spinning wheel.",
|
||||
"A quaint village nestled in a valley is surrounded by blooming cherry blossoms, with petals drifting through the air as villagers go about their daily activities, adding life to the scene.",
|
||||
"In a tranquil forest clearing, a sparkling waterfall cascades down into a clear pool, surrounded by lush greenery and flowers, with occasional birds fluttering by.",
|
||||
"A woman beamed with pride as she watched her child perform on stage.",
|
||||
"an adorable kangaroo wearing blue jeans and a white t shirt taking a pleasant stroll in Mumbai India during a winter storm",
|
||||
"A man is eating salad",
|
||||
"An Asian girl wearing a bright yellow T-shirt and white pants is Hip-Hop dancing",
|
||||
"nighttime footage of a hermit crab using an incandescent lightbulb as its shell",
|
||||
"a toy robot wearing a green dress and a sun hat taking a pleasant stroll in Antarctica during a beautiful sunset",
|
||||
"A goat operating a food truck, serving gourmet grilled cheese sandwiches to a line of animals.",
|
||||
"Macro shot. Man in an antique scuba helmet with dark glass walking out of a flower",
|
||||
"A bustling train station in the heart of a vibrant city.",
|
||||
"Light filtering through a canopy of autumn leaves, casting warm, dappled patterns of yellow, orange, and red onto the ground.",
|
||||
"Chimneys in the setting sun",
|
||||
"A longboarder accelerating downhill, carving through turns.",
|
||||
"A couple runs through a sudden downpour, laughing and splashing in puddles as they try to find shelter.",
|
||||
"A glass of iced coffee condensing water on the outside, with droplets forming and sliding down the glass in slow motion.",
|
||||
"macro shot of a leaf showing tiny trains moving through its veins",
|
||||
"A corgi wearing sunglasses walks on the beach of a tropical island",
|
||||
"Borneo wildlife on the Kinabatangan River",
|
||||
"A beautiful silhouette animation shows a wolf howling at the moon, feeling lonely, until it finds its pack.",
|
||||
"an adorable kangaroo wearing blue jeans and a white t shirt taking a pleasant stroll in Johannesburg South Africa during a colorful festival",
|
||||
"A green monster made of plants walks through an airport.",
|
||||
"A close up view of a glass sphere that has a zen garden within it. There is a small dwarf in the sphere who is raking the zen garden and creating patterns in the sand.",
|
||||
"A person on a scooter colliding with a park bench, the scooter tipping over.",
|
||||
"A tilt-up from a city street, ascending to show the skyline with its mix of modern and historic architecture.",
|
||||
"A chef tossing a pancake into the air and catching it.",
|
||||
"A woman whispering a secret into a friend's ear.",
|
||||
"A vulture circling high in the sky.",
|
||||
"A medieval castle overlooking a bustling renaissance fair.",
|
||||
"a toy robot wearing purple overalls and cowboy boots taking a pleasant stroll in Mumbai India during a beautiful sunset",
|
||||
"A man standing in front of a burning building giving the 'thumbs up' sign.",
|
||||
"The person's cheeks flushed with embarrassment as he told a funny story.",
|
||||
"Llamas and Emus are playing chess",
|
||||
"A woman sipping a steaming cup of tea.",
|
||||
"A tree root bursting through the seat of an ancient, weathered bench, intertwining with the wood.",
|
||||
"Smoke rises from the chimney of a cozy log cabin nestled in the woods, with soft light glowing from the windows, suggesting a warm and inviting atmosphere.",
|
||||
"A close-up of sparkling water being poured into a glass, capturing the detailed flow and bubbles.",
|
||||
"a woman wearing blue jeans and a white t shirt taking a pleasant stroll in Antarctica during a beautiful sunset",
|
||||
"The Glenfinnan Viaduct is a historic railway bridge in Scotland, UK, that crosses over the west highland line between the towns of Mallaig and Fort William. It is a stunning sight as a steam train leaves the bridge, traveling over the arch-covered viaduct. The landscape is dotted with lush greenery and rocky mountains, creating a picturesque backdrop for the train journey. The sky is blue and the sun is shining, making for a beautiful day to explore this majestic spot.",
|
||||
"A piece of elastic fabric being pulled and stretched, then returning to its original size when the tension is released.",
|
||||
"a woman wearing a green dress and a sun hat taking a pleasant stroll in Antarctica during a beautiful sunset",
|
||||
"A video of a water jet cutting through metal, showing the powerful and precise movement of water.",
|
||||
"Car mirrors and sunsets",
|
||||
"Giant Pandas are eating hot noodles in a Chinese restaurant",
|
||||
"A rally car taking a fast turn on a track",
|
||||
"a toy robot wearing purple overalls and cowboy boots taking a pleasant stroll in Mumbai India during a colorful festival",
|
||||
"A crystal-clear icicle slowly dripping as it melts in the warmth of the midday sun, each drop sparkling as it falls.",
|
||||
"A tilt-down from a chandelier in a grand hall, revealing the ornate decor and people mingling below.",
|
||||
"A man is playing the drums under the water",
|
||||
"A person playing an electric guitar made of lightning, with thunderous sound waves.",
|
||||
"A person floating in a bubble, drifting over a bustling cityscape.",
|
||||
"A tilt-down from a starry night sky, revealing a quiet forest clearing bathed in moonlight.",
|
||||
"A pan right through a dense jungle, moving past lush vegetation and exotic wildlife.",
|
||||
"Close-up of a man eating an apple.",
|
||||
"A low-angle shot of a dancer leaping gracefully into the air, making their movement appear even more dynamic and powerful.",
|
||||
"A woman is search her bag trying to find something.",
|
||||
"A bulldozer clears debris from a demolished building, making way for new construction.",
|
||||
"A man sighed in relief as the doctor delivered the good news.",
|
||||
"A tsunami coming through an alley in Bulgaria, dynamic movement.",
|
||||
"Blooming Flowers",
|
||||
"A push-in through a dense crowd at a festival, moving towards a performer on stage who is captivating the audience.",
|
||||
"A truck right through a tranquil garden, moving past blooming flowers, trees, and a small fountain.",
|
||||
"The person's eyes sparkled with excitement as he greeted a friend.",
|
||||
"A person playing chess with a robot on a floating platform above the ocean.",
|
||||
"A gentle breeze rustles the leaves as someone walks down a serene forest path, sunlight filtering through the trees and shifting patterns on the ground as branches sway.",
|
||||
"A rollercoaster ride from a city to a desert and then to an ice world",
|
||||
"A pan left across an ancient library, moving from shelf to shelf, showcasing rows of leather-bound books.",
|
||||
"A mother otter floating on her back in a river, cradling her pup on her stomach to keep it safe and warm in the gentle current.",
|
||||
"an adorable kangaroo wearing purple overalls and cowboy boots taking a pleasant stroll in Johannesburg South Africa during a colorful festival",
|
||||
"a woman wearing a green dress and a sun hat taking a pleasant stroll in Mumbai India during a colorful festival",
|
||||
"A delicate layer of morning frost melting off a flower petal, the tiny droplets glistening like diamonds in the light.",
|
||||
"A panda is cooking for her child, her child is next to her.",
|
||||
"Macro shot of a man wearing an antique diving helmet with dark glass and a jetpack walking on the veins of a leaf. Realistic style",
|
||||
"an old man wearing purple overalls and cowboy boots taking a pleasant stroll in Johannesburg South Africa during a beautiful sunset",
|
||||
"A girl is unfolding a birthday gift.",
|
||||
"A pencil drawing an architectural plan.",
|
||||
"A handheld camera following a dog running through a park, bouncing and tilting as it captures the dog's joyful exploration.",
|
||||
"A pan left across a serene beach at sunrise, moving from the darkened shore to the brightening horizon.",
|
||||
"A group of people are clapping to celebrate",
|
||||
"Vendors set up stalls at a bustling farmer's market, displaying fresh fruits and vegetables, while people stroll through, selecting produce and enjoying the lively atmosphere.",
|
||||
"A police helicopter hovers above a high-speed chase, guiding officers on the ground to apprehend a suspect.",
|
||||
"A paper origami dragon riding a boat in waves. Realistic style.",
|
||||
"A close-up of a droplet of dew forming on a leaf, capturing the detailed surface tension.",
|
||||
"a toy robot wearing blue jeans and a white t shirt taking a pleasant stroll in Mumbai India during a beautiful sunset",
|
||||
"A dry rainbow rose is coming back to life.",
|
||||
"A glass falling off a table and shattering on the floor.",
|
||||
"A marathon runner crossing the finish line after a grueling race.",
|
||||
"A zoom-in on a drop of morning dew on a leaf, showing the reflection of the surrounding world within it.",
|
||||
"A child blowing on hot cocoa to cool it down.",
|
||||
"A squad of futsal players showcasing their skills on an indoor court.",
|
||||
"A princess is brushing her long golden hair in the garden.",
|
||||
"A close-up of a pair of eyes, revealing the subtle emotions and reflections within them.",
|
||||
"A tracking shot of a group of cyclists racing through a forest trail, with trees and foliage rushing by.",
|
||||
"A woman yawning widely at the end of a long day.",
|
||||
"an old man wearing a green dress and a sun hat taking a pleasant stroll in Johannesburg South Africa during a colorful festival",
|
||||
"Hidden within a garden, an ancient fountain trickles with water, surrounded by vibrant flowers and lush greenery that seem to whisper secrets of the past.",
|
||||
"A Chinese man sits at a table and eats noodles with chopsticks",
|
||||
"A pink pig running fast toward the camera in an alley in Tokyo.",
|
||||
"Strange creatures move through a mysterious, foggy marsh, their silhouettes barely visible through the dense mist as they navigate the eerie, otherworldly landscape.",
|
||||
"Tour of an art gallery with many beautiful works of art in different styles.",
|
||||
"FPV flying through a colorful coral lined streets of an underwater suburban neighborhood.",
|
||||
"Aerial view of Santorini during the blue hour, showcasing the stunning architecture of white Cycladic buildings with blue domes. The caldera views are breathtaking, and the lighting creates a beautiful, serene atmosphere.",
|
||||
"Camera zoom out. A couple walking along the beach as the sun sets over the ocean.",
|
||||
"an extreme close up shot of a woman's eye, with her iris appearing as earth",
|
||||
"a woman wearing purple overalls and cowboy boots taking a pleasant stroll in Mumbai India during a colorful festival",
|
||||
"an old man wearing a green dress and a sun hat taking a pleasant stroll in Mumbai India during a winter storm",
|
||||
"an adorable kangaroo wearing blue jeans and a white t shirt taking a pleasant stroll in Antarctica during a winter storm",
|
||||
"A martial artist breaking a board with a powerful punch.",
|
||||
"People gather on a peaceful beach at sunset, a bonfire crackling as they sit around, enjoying the warmth and the sight of the sun dipping below the horizon.",
|
||||
"A close-up of a waterfall, showing the detailed movement of water as it crashes down.",
|
||||
"A child is blowing bubbles",
|
||||
"a woman wearing a green dress and a sun hat taking a pleasant stroll in Johannesburg South Africa during a winter storm",
|
||||
"A wide-angle perspective of a serene lake surrounded by mountains, reflecting the sky and creating a sense of infinite space.",
|
||||
"The person's eyebrows arched in skepticism as she listened to a dubious claim.",
|
||||
"an old man wearing blue jeans and a white t shirt taking a pleasant stroll in Mumbai India during a beautiful sunset",
|
||||
"a woman wearing a green dress and a sun hat taking a pleasant stroll in Johannesburg South Africa during a colorful festival",
|
||||
"a woman wearing purple overalls and cowboy boots taking a pleasant stroll in Antarctica during a colorful festival",
|
||||
"a toy robot wearing blue jeans and a white t shirt taking a pleasant stroll in Antarctica during a colorful festival",
|
||||
"A chef flips a pancake and puts cream on it.",
|
||||
"An astronaut runs on the surface of the moon, the low angle shot shows the vast background of the moon, the movement is smooth and appears lightweight",
|
||||
"A man's face lit up with happiness as he received a heartfelt compliment.",
|
||||
"A futuristic spaceport hums with activity as ships of various shapes and sizes take off and land on multiple platforms, their engines glowing with vibrant colors.",
|
||||
"A person knitting a scarf using beams of light instead of yarn.",
|
||||
"A pedestal up from the edge of a canyon, gradually revealing the expansive landscape and river below.",
|
||||
"a woman wearing purple overalls and cowboy boots taking a pleasant stroll in Johannesburg South Africa during a colorful festival",
|
||||
"an old man wearing blue jeans and a white t shirt taking a pleasant stroll in Johannesburg South Africa during a colorful festival",
|
||||
"A person walking up a staircase made of clouds leading to a floating castle.",
|
||||
"Monks meditate in a serene mountaintop temple, sitting in quiet reflection as the wind gently moves through the surrounding trees, creating a sense of peace and tranquility.",
|
||||
"An aerial shot of a bustling city intersection at rush hour, capturing the organized chaos of cars and pedestrians.",
|
||||
"A pair of hands skillfully knitting a colorful scarf, the yarn winding through their fingers with each stitch.",
|
||||
"Close-up, a Chinese child is eating dumplings",
|
||||
"A kite losing wind and falling to the ground.",
|
||||
"Bioluminescent waves gently wash ashore on a deserted beach, illuminating the sand with each cresting wave as a figure walks along the water's edge, leaving glowing footprints.",
|
||||
"A red panda taking a bite of a pizza",
|
||||
"A close-up shot of a young woman driving a car, looking thoughtful, blurred green forest visible through the rainy car window.",
|
||||
"A high-speed video of a splash created by a stone thrown into a pond.",
|
||||
"A metal rod being bent slightly by a force and then springing back to its original straight shape when the force is removed.",
|
||||
"A hedgehog in a knight's armor, riding a toy horse into a medieval castle.",
|
||||
"A bird made of fresh oranges rushes out of the orange",
|
||||
"A low altitude first person perspective camera tracking shot of a soccer player's feet dribbling the ball on the groud in a soccer field, Sports Videography, Motion Tracking camera shot",
|
||||
"A tranquil island retreat features swaying palm trees and hammocks strung between them, inviting guests to relax and enjoy the serene beauty of the surroundings.",
|
||||
"a spooky haunted mansion, with friendly jack o lanterns and ghost characters welcoming trick or treaters to the entrance, tilt shift photography",
|
||||
"A coconut tree made of dollar bills at sunset, with bills falling off like leaves.",
|
||||
"A motocross bike accelerating out of a tight turn on a dirt track.",
|
||||
"A tranquil Zen garden with a gently flowing stream and koi fish.",
|
||||
"A green monster made of leaves walks through the airport, carrying a suitcase.",
|
||||
"A time-lapse of a frost-covered leaf gradually thawing in the morning sunlight, with tiny water droplets forming and trickling down.",
|
||||
"A woman practicing her archery skills at a range.",
|
||||
"A slow-motion video of ink being injected into a tank of water, creating intricate and beautiful patterns.",
|
||||
"a woman wearing blue jeans and a white t shirt taking a pleasant stroll in Johannesburg South Africa during a winter storm",
|
||||
"The person's forehead creased with worry as he listened to bad news.",
|
||||
"An arc shot around a grand piano being played in an empty concert hall, the motion revealing the intricate details of the instrument.",
|
||||
"A person conducting a symphony of animals in a forest clearing.",
|
||||
"A truck right alongside a flowing river, capturing the movement of the water and the surrounding forest.",
|
||||
"A rocket blasting off from the launch pad, accelerating rapidly into the sky.",
|
||||
"Workers move through a picturesque vineyard during the harvest season, carefully picking grapes and placing them into baskets as the sun bathes the vines in a warm glow.",
|
||||
"A person is eating an ice cream.",
|
||||
"An over-the-shoulder perspective of a chef meticulously plating a dish in a bustling kitchen.",
|
||||
"A man looked away in shame when confronted with his wrongdoing.",
|
||||
"A person is savoring a slice of pizza at a pizzeria."
|
||||
]
|
||||
|
||||
class BackendStressTest:
|
||||
def __init__(self, output_path: str,
|
||||
server_url: str = "http://localhost:8000", max_concurrent: int = 50):
|
||||
self.output_path = output_path
|
||||
self.server_url = server_url
|
||||
self.max_concurrent = max_concurrent
|
||||
|
||||
# Results storage
|
||||
self.results = []
|
||||
self.lock = threading.Lock()
|
||||
|
||||
async def check_health(self) -> bool:
|
||||
"""Check if the Ray Serve backend is healthy"""
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(f"{self.server_url}/health", timeout=aiohttp.ClientTimeout(total=5)) as response:
|
||||
return response.status == 200
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def build_request_params(self, prompt: str, **kwargs) -> Dict[str, Any]:
|
||||
"""Build request parameters for Ray Serve backend"""
|
||||
# Default parameters matching the Ray Serve backend
|
||||
default_params = {
|
||||
'prompt': prompt,
|
||||
'negative_prompt': None,
|
||||
'use_negative_prompt': False,
|
||||
'seed': 42,
|
||||
'guidance_scale': 7.5,
|
||||
'num_frames': 21,
|
||||
'height': 448,
|
||||
'width': 832,
|
||||
'num_inference_steps': 20,
|
||||
'randomize_seed': True,
|
||||
'return_frames': False # Don't return frames for stress testing to reduce overhead
|
||||
}
|
||||
|
||||
# Override with any provided kwargs
|
||||
for key, value in kwargs.items():
|
||||
if key in default_params:
|
||||
default_params[key] = value
|
||||
|
||||
# Randomize seed if requested
|
||||
if default_params.get('randomize_seed', True):
|
||||
default_params['seed'] = torch.randint(0, 1000000, (1,)).item()
|
||||
|
||||
# Handle negative prompt
|
||||
if not default_params.get('use_negative_prompt', False):
|
||||
default_params['negative_prompt'] = None
|
||||
|
||||
# NEW: Remove keys with None values to avoid sending nulls that may break validation
|
||||
clean_params = {k: v for k, v in default_params.items() if v is not None}
|
||||
return clean_params
|
||||
|
||||
async def test_single_request(self, session: aiohttp.ClientSession, prompt: str, request_id: int) -> Dict[str, Any]:
|
||||
"""Test a single request and measure latency"""
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
# Build request parameters
|
||||
request_params = self.build_request_params(prompt)
|
||||
|
||||
# Make request to Ray Serve backend
|
||||
async with session.post(
|
||||
f"{self.server_url}/generate_video",
|
||||
json=request_params,
|
||||
timeout=aiohttp.ClientTimeout(total=900) # 15 minute timeout for video generation
|
||||
) as response:
|
||||
|
||||
end_time = time.time()
|
||||
latency = end_time - start_time
|
||||
|
||||
if response.status == 200:
|
||||
response_data = await response.json()
|
||||
if response_data.get('success', False):
|
||||
result = {
|
||||
'request_id': request_id,
|
||||
'prompt': prompt,
|
||||
'latency': latency,
|
||||
'status': 'success',
|
||||
'response_time': latency, # Use our own timing
|
||||
'timestamp': start_time,
|
||||
'output_path': response_data.get('output_path', ''),
|
||||
'used_seed': response_data.get('seed', request_params['seed'])
|
||||
}
|
||||
else:
|
||||
result = {
|
||||
'request_id': request_id,
|
||||
'prompt': prompt,
|
||||
'latency': latency,
|
||||
'status': 'error',
|
||||
'error': response_data.get('error_message', 'Unknown backend error'),
|
||||
'timestamp': start_time
|
||||
}
|
||||
else:
|
||||
response_text = await response.text()
|
||||
result = {
|
||||
'request_id': request_id,
|
||||
'prompt': prompt,
|
||||
'latency': latency,
|
||||
'status': 'error',
|
||||
'error': f"HTTP {response.status}: {response_text}",
|
||||
'timestamp': start_time
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
end_time = time.time()
|
||||
latency = end_time - start_time
|
||||
result = {
|
||||
'request_id': request_id,
|
||||
'prompt': prompt,
|
||||
'latency': latency,
|
||||
'status': 'error',
|
||||
'error': str(e),
|
||||
'timestamp': start_time
|
||||
}
|
||||
|
||||
# Thread-safe result storage
|
||||
with self.lock:
|
||||
self.results.append(result)
|
||||
|
||||
return result
|
||||
|
||||
async def run_stress_test(self, num_iterations: int = 1, concurrent_requests: int = None):
|
||||
"""Run the stress test with multiple iterations and concurrent requests"""
|
||||
if concurrent_requests is None:
|
||||
concurrent_requests = self.max_concurrent
|
||||
|
||||
# Check backend health before starting
|
||||
print(f"Testing Ray Serve backend at {self.server_url}...")
|
||||
if not await self.check_health():
|
||||
print(f"❌ Backend is not healthy at {self.server_url}")
|
||||
print("Make sure the Ray Serve backend is running with:")
|
||||
print("python ray_serve_backend.py")
|
||||
return
|
||||
print("✅ Backend is healthy and ready for stress testing")
|
||||
|
||||
print(f"\nStarting stress test with {len(STRESS_TEST_PROMPTS)} prompts")
|
||||
print(f"Running {num_iterations} iteration(s) with {concurrent_requests} concurrent requests")
|
||||
print(f"Total requests: {len(STRESS_TEST_PROMPTS) * num_iterations}")
|
||||
print(f"Backend URL: {self.server_url}")
|
||||
print("-" * 80)
|
||||
|
||||
all_prompts = STRESS_TEST_PROMPTS * num_iterations
|
||||
request_id = 0
|
||||
|
||||
# Create semaphore to limit concurrent requests
|
||||
semaphore = asyncio.Semaphore(concurrent_requests)
|
||||
|
||||
async def limited_request(session: aiohttp.ClientSession, prompt: str, req_id: int):
|
||||
async with semaphore:
|
||||
return await self.test_single_request(session, prompt, req_id)
|
||||
|
||||
# Run concurrent requests using asyncio
|
||||
async with aiohttp.ClientSession() as session:
|
||||
# Create all tasks
|
||||
tasks = [
|
||||
limited_request(session, prompt, request_id + i)
|
||||
for i, prompt in enumerate(all_prompts)
|
||||
]
|
||||
|
||||
# Process completed requests as they finish
|
||||
completed = 0
|
||||
for coro in asyncio.as_completed(tasks):
|
||||
try:
|
||||
result = await coro
|
||||
completed += 1
|
||||
prompt = result['prompt']
|
||||
status_icon = "✅" if result['status'] == 'success' else "❌"
|
||||
output_info = f" -> {result.get('output_path', 'N/A')}" if result['status'] == 'success' else ""
|
||||
print(f"{status_icon} [{completed}/{len(all_prompts)}] {result['latency']:.2f}s - {prompt[:50]}...{output_info}")
|
||||
except Exception as e:
|
||||
completed += 1
|
||||
print(f"❌ [{completed}/{len(all_prompts)}] Exception: {e}")
|
||||
|
||||
self.analyze_results()
|
||||
|
||||
def analyze_results(self):
|
||||
"""Analyze and print test results"""
|
||||
print("\n" + "=" * 80)
|
||||
print("STRESS TEST RESULTS")
|
||||
print("=" * 80)
|
||||
|
||||
successful_requests = [r for r in self.results if r['status'] == 'success']
|
||||
failed_requests = [r for r in self.results if r['status'] == 'error']
|
||||
|
||||
print(f"Total Requests: {len(self.results)}")
|
||||
print(f"Successful: {len(successful_requests)}")
|
||||
print(f"Failed: {len(failed_requests)}")
|
||||
print(f"Success Rate: {len(successful_requests)/len(self.results)*100:.1f}%")
|
||||
|
||||
if successful_requests:
|
||||
latencies = [r['latency'] for r in successful_requests]
|
||||
print(f"\nLatency Statistics (seconds):")
|
||||
print(f" Min: {min(latencies):.2f}")
|
||||
print(f" Max: {max(latencies):.2f}")
|
||||
print(f" Mean: {statistics.mean(latencies):.2f}")
|
||||
print(f" Median: {statistics.median(latencies):.2f}")
|
||||
print(f" Std Dev: {statistics.stdev(latencies):.2f}")
|
||||
|
||||
# Percentiles
|
||||
sorted_latencies = sorted(latencies)
|
||||
p50 = sorted_latencies[int(len(sorted_latencies) * 0.5)]
|
||||
p90 = sorted_latencies[int(len(sorted_latencies) * 0.9)]
|
||||
p95 = sorted_latencies[int(len(sorted_latencies) * 0.95)]
|
||||
p99 = sorted_latencies[int(len(sorted_latencies) * 0.99)]
|
||||
|
||||
print(f" P50: {p50:.2f}")
|
||||
print(f" P90: {p90:.2f}")
|
||||
print(f" P95: {p95:.2f}")
|
||||
print(f" P99: {p99:.2f}")
|
||||
|
||||
if failed_requests:
|
||||
print(f"\nFailed Requests ({len(failed_requests)}):")
|
||||
for req in failed_requests[:5]: # Show first 5 failures
|
||||
print(f" - {req['error']}")
|
||||
if len(failed_requests) > 5:
|
||||
print(f" ... and {len(failed_requests) - 5} more")
|
||||
|
||||
# Save detailed results
|
||||
results_file = os.path.join(self.output_path, "stress_test_results.json")
|
||||
os.makedirs(self.output_path, exist_ok=True)
|
||||
|
||||
with open(results_file, 'w') as f:
|
||||
json.dump({
|
||||
'summary': {
|
||||
'total_requests': len(self.results),
|
||||
'successful_requests': len(successful_requests),
|
||||
'failed_requests': len(failed_requests),
|
||||
'success_rate': len(successful_requests)/len(self.results)*100 if self.results else 0
|
||||
},
|
||||
'latency_stats': {
|
||||
'min': min(latencies) if successful_requests else 0,
|
||||
'max': max(latencies) if successful_requests else 0,
|
||||
'mean': statistics.mean(latencies) if successful_requests else 0,
|
||||
'median': statistics.median(latencies) if successful_requests else 0,
|
||||
'std_dev': statistics.stdev(latencies) if len(successful_requests) > 1 else 0
|
||||
},
|
||||
'detailed_results': self.results
|
||||
}, f, indent=2)
|
||||
|
||||
print(f"\nDetailed results saved to: {results_file}")
|
||||
|
||||
async def main():
|
||||
parser = argparse.ArgumentParser(description="FastVideo Ray Serve Backend Stress Test")
|
||||
parser.add_argument("--output_path",
|
||||
type=str,
|
||||
default="outputs",
|
||||
help="Path to save test results")
|
||||
parser.add_argument("--server_url",
|
||||
type=str,
|
||||
default="http://localhost:8000",
|
||||
help="Ray Serve backend URL")
|
||||
parser.add_argument("--max_concurrent",
|
||||
type=int,
|
||||
default=50,
|
||||
help="Maximum concurrent requests")
|
||||
parser.add_argument("--iterations",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of iterations through all prompts")
|
||||
parser.add_argument("--concurrent_requests",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Number of concurrent requests (overrides max_concurrent)")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Create stress test instance
|
||||
stress_test = BackendStressTest(
|
||||
output_path=args.output_path,
|
||||
server_url=args.server_url,
|
||||
max_concurrent=args.max_concurrent
|
||||
)
|
||||
|
||||
# Run the stress test
|
||||
await stress_test.run_stress_test(
|
||||
num_iterations=args.iterations,
|
||||
concurrent_requests=args.concurrent_requests
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -1,169 +0,0 @@
|
||||
import argparse
|
||||
import os
|
||||
from copy import deepcopy
|
||||
|
||||
import gradio as gr
|
||||
import torch
|
||||
|
||||
from fastvideo import VideoGenerator
|
||||
from fastvideo.configs.sample.base import SamplingParam
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="FastVideo Gradio Demo")
|
||||
parser.add_argument("--model_path",
|
||||
type=str,
|
||||
default="FastVideo/FastHunyuan-diffusers",
|
||||
help="Path to the model")
|
||||
parser.add_argument("--num_gpus",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of GPUs to use")
|
||||
parser.add_argument("--output_path",
|
||||
type=str,
|
||||
default="outputs",
|
||||
help="Path to save generated videos")
|
||||
parsed_args = parser.parse_args()
|
||||
|
||||
# args = FastVideoArgs(model_path="FastVideo/FastHunyuan-Diffusers", num_gpus=2)
|
||||
|
||||
generator = VideoGenerator.from_pretrained(
|
||||
model_path=parsed_args.model_path, num_gpus=parsed_args.num_gpus)
|
||||
|
||||
default_params = SamplingParam.from_pretrained(parsed_args.model_path)
|
||||
|
||||
def generate_video(
|
||||
prompt,
|
||||
negative_prompt,
|
||||
use_negative_prompt,
|
||||
seed,
|
||||
guidance_scale,
|
||||
num_frames,
|
||||
height,
|
||||
width,
|
||||
num_inference_steps,
|
||||
randomize_seed=False,
|
||||
):
|
||||
params = deepcopy(default_params)
|
||||
params.prompt = prompt
|
||||
params.negative_prompt = negative_prompt
|
||||
params.seed = seed
|
||||
params.guidance_scale = guidance_scale
|
||||
params.num_frames = num_frames
|
||||
params.height = height
|
||||
params.width = width
|
||||
params.num_inference_steps = num_inference_steps
|
||||
|
||||
if randomize_seed:
|
||||
params.seed = torch.randint(0, 1000000, (1, )).item()
|
||||
|
||||
if not use_negative_prompt:
|
||||
params.negative_prompt = None
|
||||
|
||||
generator.generate_video(prompt=prompt, sampling_param=params)
|
||||
|
||||
output_path = os.path.join(parsed_args.output_path,
|
||||
f"{params.prompt[:100]}.mp4")
|
||||
|
||||
return output_path, params.seed
|
||||
|
||||
examples = [
|
||||
"A hand enters the frame, pulling a sheet of plastic wrap over three balls of dough placed on a wooden surface. The plastic wrap is stretched to cover the dough more securely. The hand adjusts the wrap, ensuring that it is tight and smooth over the dough. The scene focuses on the hand’s movements as it secures the edges of the plastic wrap. No new objects appear, and the camera remains stationary, focusing on the action of covering the dough.",
|
||||
"A vintage train snakes through the mountains, its plume of white steam rising dramatically against the jagged peaks. The cars glint in the late afternoon sun, their deep crimson and gold accents lending a touch of elegance. The tracks carve a precarious path along the cliffside, revealing glimpses of a roaring river far below. Inside, passengers peer out the large windows, their faces lit with awe as the landscape unfolds.",
|
||||
"A crowded rooftop bar buzzes with energy, the city skyline twinkling like a field of stars in the background. Strings of fairy lights hang above, casting a warm, golden glow over the scene. Groups of people gather around high tables, their laughter blending with the soft rhythm of live jazz. The aroma of freshly mixed cocktails and charred appetizers wafts through the air, mingling with the cool night breeze.",
|
||||
]
|
||||
|
||||
with gr.Blocks() as demo:
|
||||
gr.Markdown("# FastVideo Inference Demo")
|
||||
|
||||
with gr.Group():
|
||||
with gr.Row():
|
||||
prompt = gr.Text(
|
||||
label="Prompt",
|
||||
show_label=False,
|
||||
max_lines=1,
|
||||
placeholder="Enter your prompt",
|
||||
container=False,
|
||||
)
|
||||
run_button = gr.Button("Run", scale=0)
|
||||
result = gr.Video(label="Result", show_label=False)
|
||||
|
||||
with gr.Accordion("Advanced options", open=False):
|
||||
with gr.Group():
|
||||
with gr.Row():
|
||||
height = gr.Slider(
|
||||
label="Height",
|
||||
minimum=256,
|
||||
maximum=1024,
|
||||
step=32,
|
||||
value=default_params.height,
|
||||
)
|
||||
width = gr.Slider(label="Width",
|
||||
minimum=256,
|
||||
maximum=1024,
|
||||
step=32,
|
||||
value=default_params.width)
|
||||
|
||||
with gr.Row():
|
||||
num_frames = gr.Slider(
|
||||
label="Number of Frames",
|
||||
minimum=21,
|
||||
maximum=163,
|
||||
value=default_params.num_frames,
|
||||
)
|
||||
guidance_scale = gr.Slider(
|
||||
label="Guidance Scale",
|
||||
minimum=1,
|
||||
maximum=12,
|
||||
value=default_params.guidance_scale,
|
||||
)
|
||||
num_inference_steps = gr.Slider(
|
||||
label="Inference Steps",
|
||||
minimum=4,
|
||||
maximum=100,
|
||||
value=default_params.num_inference_steps,
|
||||
)
|
||||
|
||||
with gr.Row():
|
||||
use_negative_prompt = gr.Checkbox(
|
||||
label="Use negative prompt", value=False)
|
||||
negative_prompt = gr.Text(
|
||||
label="Negative prompt",
|
||||
max_lines=1,
|
||||
placeholder="Enter a negative prompt",
|
||||
visible=False,
|
||||
)
|
||||
|
||||
seed = gr.Slider(label="Seed",
|
||||
minimum=0,
|
||||
maximum=1000000,
|
||||
step=1,
|
||||
value=default_params.seed)
|
||||
randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
|
||||
seed_output = gr.Number(label="Used Seed")
|
||||
|
||||
gr.Examples(examples=examples, inputs=prompt)
|
||||
|
||||
use_negative_prompt.change(
|
||||
fn=lambda x: gr.update(visible=x),
|
||||
inputs=use_negative_prompt,
|
||||
outputs=default_params.negative_prompt,
|
||||
)
|
||||
|
||||
run_button.click(
|
||||
fn=generate_video,
|
||||
inputs=[
|
||||
prompt,
|
||||
negative_prompt,
|
||||
use_negative_prompt,
|
||||
seed,
|
||||
guidance_scale,
|
||||
num_frames,
|
||||
height,
|
||||
width,
|
||||
num_inference_steps,
|
||||
randomize_seed,
|
||||
],
|
||||
outputs=[result, seed_output],
|
||||
)
|
||||
|
||||
demo.queue(max_size=20).launch(server_name="0.0.0.0", server_port=7860)
|
||||
@@ -1,29 +1,17 @@
|
||||
import argparse
|
||||
import os
|
||||
import requests
|
||||
import json
|
||||
import base64
|
||||
import io
|
||||
from typing import Optional
|
||||
import time
|
||||
|
||||
import gradio as gr
|
||||
import torch
|
||||
import imageio
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
import time
|
||||
|
||||
from fastvideo.configs.sample.base import SamplingParam
|
||||
|
||||
|
||||
# Map clean model names to full paths
|
||||
|
||||
MODEL_PATH_MAPPING = {
|
||||
"FastWan2.1-T2V-1.3B": "FastVideo/FastWan2.1-T2V-1.3B-Diffusers",
|
||||
# "FastWan2.1-T2V-14B": "FastVideo/FastWan2.1-T2V-14B-Diffusers",
|
||||
"FastWan2.2-TI2V-5B-FullAttn": "FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers",
|
||||
# "Wan2.1-T2V-1.3B": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
|
||||
# "Wan2.1-T2V-14B": "Wan-AI/Wan2.1-T2V-14B-Diffusers",
|
||||
# "Wan2.2-TI2V-5B": "Wan-AI/Wan2.2-TI2V-5B-Diffusers"
|
||||
}
|
||||
|
||||
|
||||
@@ -33,7 +21,6 @@ class RayServeClient:
|
||||
self.session = requests.Session()
|
||||
|
||||
def check_health(self) -> bool:
|
||||
"""Check if the backend is healthy"""
|
||||
try:
|
||||
response = self.session.get(f"{self.backend_url}/health", timeout=5)
|
||||
return response.status_code == 200
|
||||
@@ -41,7 +28,6 @@ class RayServeClient:
|
||||
return False
|
||||
|
||||
def generate_video(self, request_data: dict) -> dict:
|
||||
"""Generate video using the backend API"""
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
@@ -51,15 +37,13 @@ class RayServeClient:
|
||||
f"{self.backend_url}/generate_video",
|
||||
json=request_data,
|
||||
headers=headers,
|
||||
timeout=300 # 5 minutes timeout
|
||||
timeout=300
|
||||
)
|
||||
|
||||
end_time = time.time()
|
||||
round_trip_time = end_time - start_time
|
||||
round_trip_time = time.time() - start_time
|
||||
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
# Calculate network time by subtracting backend total time
|
||||
backend_total = result.get("total_time", 0)
|
||||
network_time = round_trip_time - backend_total
|
||||
result["network_time"] = network_time
|
||||
@@ -69,79 +53,125 @@ class RayServeClient:
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
return {"success": False, "error_message": f"Request failed: {str(e)}"}
|
||||
|
||||
|
||||
|
||||
|
||||
def save_video_from_base64(video_data: str, output_dir: str, prompt: str) -> str:
|
||||
"""Save base64-encoded video data to a file"""
|
||||
if not video_data:
|
||||
return "No video data to save"
|
||||
return None
|
||||
|
||||
try:
|
||||
# Remove the data URL prefix if present
|
||||
if video_data.startswith('data:video/'):
|
||||
video_data = video_data.split(',')[1]
|
||||
|
||||
# Decode base64 to bytes
|
||||
video_bytes = base64.b64decode(video_data)
|
||||
|
||||
# Create safe filename from prompt
|
||||
safe_prompt = prompt[:50].replace(' ', '_').replace('/', '_').replace('\\', '_')
|
||||
video_filename = f"{safe_prompt}.mp4"
|
||||
video_path = os.path.join(output_dir, video_filename)
|
||||
|
||||
# Ensure output directory exists
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
# Save video bytes to file
|
||||
with open(video_path, 'wb') as f:
|
||||
f.write(video_bytes)
|
||||
|
||||
return f"Saved video to: {video_path}", video_path
|
||||
return video_path
|
||||
|
||||
except Exception as e:
|
||||
return f"Failed to save video: {str(e)}", ""
|
||||
print(f"Failed to save video: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def create_timing_display(inference_time, encoding_time, network_time, total_time, stage_execution_times, num_frames):
|
||||
dit_denoising_time = f"{stage_execution_times[5]:.2f}s" if len(stage_execution_times) > 5 else "N/A"
|
||||
|
||||
timing_html = f"""
|
||||
<div style="margin: 10px 0;">
|
||||
<h3 style="text-align: center; margin-bottom: 10px;">⏱️ Timing Breakdown</h3>
|
||||
<div style="display: grid; grid-template-columns: repeat(5, 1fr); gap: 10px; margin-bottom: 10px;">
|
||||
<div class="timing-card timing-card-highlight">
|
||||
<div style="font-size: 20px;">🚀</div>
|
||||
<div style="font-weight: bold; margin: 3px 0; font-size: 14px;">DiT Denoising</div>
|
||||
<div style="font-size: 16px; color: #ffa200; font-weight: bold;">{dit_denoising_time}</div>
|
||||
</div>
|
||||
<div class="timing-card">
|
||||
<div style="font-size: 20px;">🧠</div>
|
||||
<div style="font-weight: bold; margin: 3px 0; font-size: 14px;">E2E (w. vae/text encoder)</div>
|
||||
<div style="font-size: 16px; color: #2563eb;">{inference_time:.2f}s</div>
|
||||
</div>
|
||||
<div class="timing-card">
|
||||
<div style="font-size: 20px;">🎬</div>
|
||||
<div style="font-weight: bold; margin: 3px 0; font-size: 14px;">Video Encoding</div>
|
||||
<div style="font-size: 16px; color: #dc2626;">{encoding_time:.2f}s</div>
|
||||
</div>
|
||||
<div class="timing-card">
|
||||
<div style="font-size: 20px;">🌐</div>
|
||||
<div style="font-weight: bold; margin: 3px 0; font-size: 14px;">Network Transfer</div>
|
||||
<div style="font-size: 16px; color: #059669;">{network_time:.2f}s</div>
|
||||
</div>
|
||||
<div class="timing-card">
|
||||
<div style="font-size: 20px;">📊</div>
|
||||
<div style="font-weight: bold; margin: 3px 0; font-size: 14px;">Total Processing</div>
|
||||
<div style="font-size: 18px; color: #0277bd;">{total_time:.2f}s</div>
|
||||
</div>
|
||||
</div>"""
|
||||
|
||||
if inference_time > 0:
|
||||
fps = num_frames / inference_time
|
||||
timing_html += f"""
|
||||
<div class="performance-card" style="margin-top: 15px;">
|
||||
<span style="font-weight: bold;">Generation Speed: </span>
|
||||
<span style="font-size: 18px; color: #6366f1; font-weight: bold;">{fps:.1f} frames/second</span>
|
||||
</div>"""
|
||||
|
||||
return timing_html + "</div>"
|
||||
|
||||
|
||||
def load_example_prompts():
|
||||
def contains_chinese(text):
|
||||
return any('\u4e00' <= char <= '\u9fff' for char in text)
|
||||
|
||||
def load_from_file(filepath):
|
||||
prompts, labels = [], []
|
||||
try:
|
||||
with open(filepath, "r", encoding='utf-8') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line and not contains_chinese(line):
|
||||
label = line[:100] + "..." if len(line) > 100 else line
|
||||
labels.append(label)
|
||||
prompts.append(line)
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not read {filepath}: {e}")
|
||||
return prompts, labels
|
||||
|
||||
examples, example_labels = load_from_file("prompts/prompts_final.txt")
|
||||
|
||||
if not examples:
|
||||
examples = ["A crowded rooftop bar buzzes with energy, the city skyline twinkling like a field of stars in the background."]
|
||||
example_labels = ["Crowded rooftop bar at night"]
|
||||
|
||||
return examples, example_labels
|
||||
|
||||
|
||||
def create_gradio_interface(backend_url: str, default_params: dict[str, SamplingParam]):
|
||||
"""Create the Gradio interface"""
|
||||
|
||||
# Initialize the Ray Serve client
|
||||
client = RayServeClient(backend_url)
|
||||
|
||||
def generate_video(
|
||||
prompt,
|
||||
negative_prompt,
|
||||
use_negative_prompt,
|
||||
seed,
|
||||
guidance_scale,
|
||||
num_frames,
|
||||
height,
|
||||
width,
|
||||
randomize_seed=False,
|
||||
input_image=None,
|
||||
model_selection="FastVideo/FastWan2.1-T2V-1.3B-Diffusers (Text-to-Video)",
|
||||
progress=None,
|
||||
request: gr.Request = None,
|
||||
prompt, negative_prompt, use_negative_prompt, seed, guidance_scale,
|
||||
num_frames, height, width, randomize_seed, model_selection, progress
|
||||
):
|
||||
# Check backend health first
|
||||
if not client.check_health():
|
||||
return None, f"Backend is not available. Please check if Ray Serve is running at {backend_url}", ""
|
||||
|
||||
# Validate video dimensions
|
||||
# Validate dimensions
|
||||
max_pixels = 720 * 1280
|
||||
if height * width > max_pixels:
|
||||
return None, f"Video dimensions too large. Maximum allowed: 720x1280 pixels. Current: {height}x{width} = {height*width} pixels", ""
|
||||
return None, f"Video dimensions too large. Maximum: 720x1280 pixels", ""
|
||||
|
||||
# Update progress
|
||||
if progress:
|
||||
progress(0.1, desc="Checking backend health...")
|
||||
|
||||
# Prepare request data
|
||||
if progress:
|
||||
progress(0.3, desc="Preparing request...")
|
||||
|
||||
request_data = {
|
||||
"prompt": prompt,
|
||||
"negative_prompt": negative_prompt,
|
||||
@@ -151,16 +181,14 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
|
||||
"num_frames": num_frames,
|
||||
"height": height,
|
||||
"width": width,
|
||||
# "num_inference_steps": 20, # Use default value
|
||||
"randomize_seed": randomize_seed,
|
||||
"return_frames": False, # We'll get video data directly
|
||||
"image_path": None, # For T2V, we pass None as input_image
|
||||
"model_path": MODEL_PATH_MAPPING.get(model_selection, "FastVideo/FastWan2.1-T2V-1.3B-Diffusers") # Map to full path
|
||||
"return_frames": False,
|
||||
"image_path": None,
|
||||
"model_path": MODEL_PATH_MAPPING.get(model_selection, "FastVideo/FastWan2.1-T2V-1.3B-Diffusers")
|
||||
}
|
||||
|
||||
# Send request to backend
|
||||
if progress:
|
||||
progress(0.4, desc="Sending request to backend...")
|
||||
progress(0.4, desc="Generating video...")
|
||||
|
||||
response = client.generate_video(request_data)
|
||||
|
||||
@@ -170,148 +198,50 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
|
||||
if response.get("success", False):
|
||||
video_data = response.get("video_data", "")
|
||||
used_seed = response.get("seed", seed)
|
||||
generation_time = response.get("generation_time", 0.0)
|
||||
inference_time = response.get("inference_time", 0.0)
|
||||
encoding_time = response.get("encoding_time", 0.0)
|
||||
total_time = response.get("total_time", 0.0)
|
||||
network_time = response.get("network_time", 0.0)
|
||||
stage_names = response.get("stage_names", [])
|
||||
stage_execution_times = response.get("stage_execution_times", [])
|
||||
dit_denoising_time = f"{stage_execution_times[5]:.2f}s"
|
||||
|
||||
print(f"Used seed: {used_seed}")
|
||||
print(f"Inference time: {inference_time:.2f}s")
|
||||
print(f"Encoding time: {encoding_time:.2f}s")
|
||||
print(f"Network transfer: {network_time:.2f}s")
|
||||
print(f"Total time: {total_time:.2f}s")
|
||||
print(f"Stage names: {stage_names}")
|
||||
print(f"Stage execution times: {stage_execution_times}")
|
||||
timing_details = create_timing_display(
|
||||
inference_time, encoding_time, network_time, total_time,
|
||||
stage_execution_times, num_frames
|
||||
)
|
||||
|
||||
# Create detailed timing message with all cards in a single row
|
||||
timing_details = f"""
|
||||
<div style="margin: 10px 0;">
|
||||
<h3 style="text-align: center; margin-bottom: 10px;">⏱️ Timing Breakdown</h3>
|
||||
<div style="display: grid; grid-template-columns: repeat(5, 1fr); gap: 10px; margin-bottom: 10px;">
|
||||
<div class="timing-card timing-card-highlight">
|
||||
<div style="font-size: 20px;">🚀</div>
|
||||
<div style="font-weight: bold; margin: 3px 0; font-size: 14px;">DiT Denoising</div>
|
||||
<div style="font-size: 16px; color: #ffa200; font-weight: bold;">{dit_denoising_time}</div>
|
||||
</div>
|
||||
<div class="timing-card">
|
||||
<div style="font-size: 20px;">🧠</div>
|
||||
<div style="font-weight: bold; margin: 3px 0; font-size: 14px;">E2E (w. vae/text encoder)</div>
|
||||
<div style="font-size: 16px; color: #2563eb;">{inference_time:.2f}s</div>
|
||||
</div>
|
||||
<div class="timing-card">
|
||||
<div style="font-size: 20px;">🎬</div>
|
||||
<div style="font-weight: bold; margin: 3px 0; font-size: 14px;">Video Encoding</div>
|
||||
<div style="font-size: 16px; color: #dc2626;">{encoding_time:.2f}s</div>
|
||||
</div>
|
||||
<div class="timing-card">
|
||||
<div style="font-size: 20px;">🌐</div>
|
||||
<div style="font-weight: bold; margin: 3px 0; font-size: 14px;">Network Transfer</div>
|
||||
<div style="font-size: 16px; color: #059669;">{network_time:.2f}s</div>
|
||||
</div>
|
||||
<div class="timing-card">
|
||||
<div style="font-size: 20px;">📊</div>
|
||||
<div style="font-weight: bold; margin: 3px 0; font-size: 14px;">Total Processing</div>
|
||||
<div style="font-size: 18px; color: #0277bd;">{total_time:.2f}s</div>
|
||||
</div>
|
||||
</div>"""
|
||||
|
||||
# Add performance insights
|
||||
if inference_time > 0:
|
||||
fps = num_frames / inference_time
|
||||
timing_details += f"""
|
||||
<div class="performance-card" style="margin-top: 15px;">
|
||||
<span style="font-weight: bold;">Generation Speed: </span>
|
||||
<span style="font-size: 18px; color: #6366f1; font-weight: bold;">{fps:.1f} frames/second</span>
|
||||
</div>
|
||||
"""
|
||||
|
||||
timing_details += "</div>"
|
||||
|
||||
# Save video data to file for Gradio to display
|
||||
if video_data:
|
||||
try:
|
||||
if progress:
|
||||
progress(0.9, desc="Saving video...")
|
||||
|
||||
output_dir = "outputs"
|
||||
save_status, video_path = save_video_from_base64(video_data, output_dir, prompt)
|
||||
print(f"Video save status: {save_status}")
|
||||
|
||||
if progress:
|
||||
progress(1.0, desc="Generation complete!")
|
||||
|
||||
if video_path and os.path.exists(video_path):
|
||||
return video_path, used_seed, timing_details
|
||||
else:
|
||||
return None, f"Video generated but failed to save: {save_status}", ""
|
||||
except Exception as e:
|
||||
return None, f"Failed to save video: {str(e)}", ""
|
||||
if progress:
|
||||
progress(0.9, desc="Saving video...")
|
||||
|
||||
video_path = save_video_from_base64(video_data, "outputs", prompt)
|
||||
|
||||
if progress:
|
||||
progress(1.0, desc="Generation complete!")
|
||||
|
||||
if video_path and os.path.exists(video_path):
|
||||
return video_path, used_seed, timing_details
|
||||
else:
|
||||
return None, "Failed to save video", ""
|
||||
else:
|
||||
return None, "No video data received from backend", ""
|
||||
else:
|
||||
error_msg = response.get("error_message", "Unknown error occurred")
|
||||
return None, f"Generation failed: {error_msg}", ""
|
||||
|
||||
# Example prompts
|
||||
examples = []
|
||||
example_labels = []
|
||||
examples, example_labels = load_example_prompts()
|
||||
|
||||
def contains_chinese(text):
|
||||
"""Check if text contains Chinese characters"""
|
||||
for char in text:
|
||||
if '\u4e00' <= char <= '\u9fff':
|
||||
return True
|
||||
return False
|
||||
|
||||
# Load prompts from prompts/prompts_final.txt
|
||||
prompts_file = "prompts/prompts_final.txt"
|
||||
if os.path.exists(prompts_file):
|
||||
try:
|
||||
with open(prompts_file, "r", encoding='utf-8') as f:
|
||||
for line_num, line in enumerate(f, 1):
|
||||
line = line.strip()
|
||||
if line and not contains_chinese(line): # Skip empty lines and lines with Chinese text
|
||||
# Create a label from the first 100 characters
|
||||
label = line[:100] + "..." if len(line) > 100 else line
|
||||
example_labels.append(label)
|
||||
examples.append(line)
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not read {prompts_file}: {e}")
|
||||
|
||||
# Fallback to example_prompts.txt if prompts_final.txt doesn't exist or is empty
|
||||
if not examples:
|
||||
try:
|
||||
with open("example_prompts.txt", "r") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
example_labels.append(line[:100])
|
||||
examples.append(line)
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not read example_prompts.txt: {e}")
|
||||
# Add a default example if all else fails
|
||||
examples = ["A crowded rooftop bar buzzes with energy, the city skyline twinkling like a field of stars in the background. Strings of fairy lights hang above, casting a warm, golden glow over the scene. Groups of people gather around high tables, their laughter blending with the soft rhythm of live jazz. The aroma of freshly mixed cocktails and charred appetizers wafts through the air, mingling with the cool night breeze."]
|
||||
example_labels = ["Crowded rooftop bar at night"]
|
||||
|
||||
# Create a custom theme with blue styling to match the logo
|
||||
theme = gr.themes.Base().set(
|
||||
button_primary_background_fill="#2563eb", # Blue color
|
||||
button_primary_background_fill_hover="#1d4ed8", # Darker blue on hover
|
||||
button_primary_background_fill="#2563eb",
|
||||
button_primary_background_fill_hover="#1d4ed8",
|
||||
button_primary_text_color="white",
|
||||
slider_color="#2563eb", # Blue slider
|
||||
checkbox_background_color_selected="#2563eb", # Blue checkbox when selected
|
||||
slider_color="#2563eb",
|
||||
checkbox_background_color_selected="#2563eb",
|
||||
)
|
||||
|
||||
def get_default_values_for_model(model_selection_value):
|
||||
"""Get default parameter values for the specified model"""
|
||||
model_path = model_selection_value.split(" (")[0] if model_selection_value else None
|
||||
def get_default_values(model_name):
|
||||
model_path = MODEL_PATH_MAPPING.get(model_name)
|
||||
if model_path and model_path in default_params:
|
||||
params = default_params[model_path]
|
||||
|
||||
return {
|
||||
'height': params.height,
|
||||
'width': params.width,
|
||||
@@ -319,25 +249,19 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
|
||||
'guidance_scale': params.guidance_scale,
|
||||
'seed': params.seed,
|
||||
}
|
||||
else:
|
||||
# Fallback defaults if model not found
|
||||
return {
|
||||
'height': 448,
|
||||
'width': 832,
|
||||
'num_frames': 61,
|
||||
'guidance_scale': 3.0,
|
||||
'seed': 1024,
|
||||
}
|
||||
|
||||
# Get initial values for the default model
|
||||
default_model = "FastVideo/FastWan2.1-T2V-1.3B-Diffusers (Text-to-Video)"
|
||||
initial_values = get_default_values_for_model(default_model)
|
||||
|
||||
# Create Gradio interface
|
||||
with gr.Blocks(title="FastWan", theme=theme) as demo:
|
||||
|
||||
# Logo using Gradio's Image component
|
||||
gr.Image("fastvideo-logos/main/png/full.png", show_label=False, container=False, height=80)
|
||||
return {
|
||||
'height': 448,
|
||||
'width': 832,
|
||||
'num_frames': 61,
|
||||
'guidance_scale': 3.0,
|
||||
'seed': 1024,
|
||||
}
|
||||
|
||||
initial_values = get_default_values("FastWan2.1-T2V-1.3B")
|
||||
|
||||
with gr.Blocks(title="FastWan", theme=theme) as demo:
|
||||
gr.Image("fastvideo-logos/main/svg/full.svg", show_label=False, container=False, height=80)
|
||||
gr.HTML("""
|
||||
<div style="text-align: center; margin-bottom: 10px;">
|
||||
<p style="font-size: 18px;"> Make Video Generation Go Blurrrrrrr </p>
|
||||
@@ -345,7 +269,6 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
|
||||
</div>
|
||||
""")
|
||||
|
||||
# What is FastVideo accordion
|
||||
with gr.Accordion("🎥 What Is FastVideo?", open=False):
|
||||
gr.HTML("""
|
||||
<div style="padding: 20px; line-height: 1.6;">
|
||||
@@ -355,21 +278,14 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
|
||||
</div>
|
||||
""")
|
||||
|
||||
|
||||
# Model selection dropdown
|
||||
with gr.Row():
|
||||
model_selection = gr.Dropdown(
|
||||
choices=[
|
||||
"FastWan2.1-T2V-1.3B",
|
||||
"FastWan2.2-TI2V-5B-FullAttn",
|
||||
# "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers (Image-to-Video)" # I2V functionality commented out
|
||||
],
|
||||
choices=list(MODEL_PATH_MAPPING.keys()),
|
||||
value="FastWan2.1-T2V-1.3B",
|
||||
label="Select Model",
|
||||
interactive=True
|
||||
)
|
||||
|
||||
# Examples dropdown
|
||||
with gr.Row():
|
||||
example_dropdown = gr.Dropdown(
|
||||
choices=example_labels,
|
||||
@@ -379,7 +295,6 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
|
||||
allow_custom_value=False
|
||||
)
|
||||
|
||||
# Main interface
|
||||
with gr.Row():
|
||||
with gr.Column(scale=6):
|
||||
prompt = gr.Text(
|
||||
@@ -394,15 +309,12 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
|
||||
with gr.Column(scale=1, min_width=120, elem_classes="center-button"):
|
||||
run_button = gr.Button("Run", variant="primary", size="lg")
|
||||
|
||||
# Status and timing information
|
||||
with gr.Row():
|
||||
with gr.Column():
|
||||
error_output = gr.Text(label="Error", visible=False)
|
||||
timing_display = gr.Markdown(label="Timing Breakdown", visible=False)
|
||||
|
||||
# Two-column layout: Advanced options on left, Video on right
|
||||
with gr.Row(equal_height=True, elem_classes="main-content-row"):
|
||||
# Left column - Advanced options
|
||||
with gr.Column(scale=1, elem_classes="advanced-options-column"):
|
||||
with gr.Group():
|
||||
gr.HTML("<div style='margin: 0 0 15px 0; text-align: center; font-size: 16px;'>Advanced Options</div>")
|
||||
@@ -455,18 +367,16 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
|
||||
randomize_seed = gr.Checkbox(label="Randomize seed", value=False)
|
||||
seed_output = gr.Number(label="Used Seed")
|
||||
|
||||
# Right column - Video result
|
||||
with gr.Column(scale=1, elem_classes="video-column"):
|
||||
result = gr.Video(
|
||||
label="Generated Video",
|
||||
show_label=True,
|
||||
height=466, # Match approximate height of advanced options
|
||||
width=600, # Limit video width
|
||||
height=466,
|
||||
width=600,
|
||||
container=True,
|
||||
elem_classes="video-component"
|
||||
)
|
||||
|
||||
# Add CSS to position the button and constrain width
|
||||
gr.HTML("""
|
||||
<style>
|
||||
.center-button {
|
||||
@@ -476,7 +386,6 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
|
||||
padding-top: 1.4em !important;
|
||||
}
|
||||
|
||||
/* Constrain overall width */
|
||||
.gradio-container {
|
||||
max-width: 1200px !important;
|
||||
margin: 0 auto !important;
|
||||
@@ -487,18 +396,15 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
|
||||
margin: 0 auto !important;
|
||||
}
|
||||
|
||||
/* Constrain individual components */
|
||||
.gr-form, .gr-box, .gr-group {
|
||||
max-width: 1200px !important;
|
||||
}
|
||||
|
||||
/* Make video component smaller */
|
||||
.gr-video {
|
||||
max-width: 500px !important;
|
||||
margin: 0 auto !important;
|
||||
}
|
||||
|
||||
/* Ensure equal height columns and proper vertical alignment */
|
||||
.main-content-row {
|
||||
display: flex !important;
|
||||
align-items: flex-start !important;
|
||||
@@ -515,37 +421,31 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
|
||||
align-items: stretch !important;
|
||||
}
|
||||
|
||||
/* Ensure video component aligns to top of its container */
|
||||
.video-column > * {
|
||||
margin-top: 0 !important;
|
||||
}
|
||||
|
||||
/* Make sure the video container starts at the same level as advanced options */
|
||||
.video-column .gr-video,
|
||||
.video-component {
|
||||
margin-top: 0 !important;
|
||||
padding-top: 0 !important;
|
||||
}
|
||||
|
||||
/* Align video component label with advanced options header */
|
||||
.video-column .gr-video .gr-form {
|
||||
margin-top: 0 !important;
|
||||
}
|
||||
|
||||
/* Ensure the advanced options group and video component start at same vertical position */
|
||||
.advanced-options-column .gr-group,
|
||||
.video-column .gr-video {
|
||||
margin-top: 0 !important;
|
||||
vertical-align: top !important;
|
||||
}
|
||||
|
||||
/* Force equal heights regardless of content */
|
||||
.advanced-options-column > *:last-child,
|
||||
.video-column > *:last-child {
|
||||
flex-grow: 0 !important;
|
||||
}
|
||||
|
||||
/* Responsive alignment for split screen */
|
||||
@media (max-width: 1400px) {
|
||||
.main-content-row {
|
||||
min-height: 600px !important;
|
||||
@@ -570,7 +470,6 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
|
||||
}
|
||||
}
|
||||
|
||||
/* Theme-agnostic timing cards */
|
||||
.timing-card {
|
||||
background: var(--background-fill-secondary) !important;
|
||||
border: 1px solid var(--border-color-primary) !important;
|
||||
@@ -589,15 +488,6 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
|
||||
border: 2px solid var(--color-accent) !important;
|
||||
}
|
||||
|
||||
.stage-card {
|
||||
background: var(--background-fill-secondary) !important;
|
||||
border: 1px solid var(--border-color-primary) !important;
|
||||
color: var(--body-text-color) !important;
|
||||
padding: 10px;
|
||||
border-radius: 6px;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.performance-card {
|
||||
background: var(--background-fill-secondary) !important;
|
||||
border: 1px solid var(--border-color-primary) !important;
|
||||
@@ -607,7 +497,6 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
/* Style for display-only number inputs */
|
||||
.gr-number input[readonly] {
|
||||
background-color: var(--background-fill-secondary) !important;
|
||||
border: 1px solid var(--border-color-primary) !important;
|
||||
@@ -616,35 +505,9 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
|
||||
text-align: center !important;
|
||||
font-weight: 500 !important;
|
||||
}
|
||||
|
||||
/* Dark mode support */
|
||||
.dark .timing-card {
|
||||
background: var(--background-fill-secondary) !important;
|
||||
border-color: var(--border-color-primary) !important;
|
||||
color: var(--body-text-color) !important;
|
||||
}
|
||||
|
||||
.dark .timing-card-highlight {
|
||||
background: var(--background-fill-primary) !important;
|
||||
border-color: var(--color-accent) !important;
|
||||
}
|
||||
|
||||
.dark .stage-card,
|
||||
.dark .performance-card {
|
||||
background: var(--background-fill-secondary) !important;
|
||||
border-color: var(--border-color-primary) !important;
|
||||
color: var(--body-text-color) !important;
|
||||
}
|
||||
|
||||
.dark .gr-number input[readonly] {
|
||||
background-color: var(--background-fill-secondary) !important;
|
||||
border-color: var(--border-color-primary) !important;
|
||||
color: var(--body-text-color-subdued) !important;
|
||||
}
|
||||
</style>
|
||||
""")
|
||||
|
||||
# Function to update prompt when example is selected
|
||||
def on_example_select(example_label):
|
||||
if example_label and example_label in example_labels:
|
||||
index = example_labels.index(example_label)
|
||||
@@ -657,43 +520,41 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
|
||||
outputs=prompt,
|
||||
)
|
||||
|
||||
# Disclaimer text
|
||||
gr.HTML("""
|
||||
<div style="text-align: center; margin-top: 10px; margin-bottom: 15px;">
|
||||
<p style="font-size: 16px; margin: 0;">The compute for this demo is generously provided by <a href="https://www.gmicloud.ai/" target="_blank">GMI Cloud</a>. Note that this demo is meant to showcase FastWan's quality and that under a large number of requests, generation speed may be affected. We are also rate-limiting users to 3 requests per minute.</p>
|
||||
</div>
|
||||
""")
|
||||
|
||||
# Event handlers
|
||||
use_negative_prompt.change(
|
||||
fn=lambda x: gr.update(visible=x),
|
||||
inputs=use_negative_prompt,
|
||||
outputs=negative_prompt,
|
||||
)
|
||||
|
||||
# Model selection change handler
|
||||
def on_model_selection_change(selected_model):
|
||||
"""Update advanced options based on selected model's default parameters"""
|
||||
if not selected_model:
|
||||
selected_model = "FastWan2.1-T2V-1.3B"
|
||||
|
||||
# Extract model path from selection (remove the description part)
|
||||
model_path = MODEL_PATH_MAPPING[selected_model]
|
||||
model_path = MODEL_PATH_MAPPING.get(selected_model)
|
||||
|
||||
if model_path and model_path in default_params:
|
||||
params = default_params[model_path]
|
||||
|
||||
# Update each component with the model's default values
|
||||
return (
|
||||
gr.update(value=params.height), # height (Number, display-only)
|
||||
gr.update(value=params.width), # width (Number, display-only)
|
||||
gr.update(value=params.num_frames), # num_frames (Number, display-only)
|
||||
gr.update(value=params.guidance_scale), # guidance_scale (Slider, interactive)
|
||||
gr.update(value=params.seed), # seed (Slider, interactive)
|
||||
gr.update(value=params.height),
|
||||
gr.update(value=params.width),
|
||||
gr.update(value=params.num_frames),
|
||||
gr.update(value=params.guidance_scale),
|
||||
gr.update(value=params.seed),
|
||||
)
|
||||
else:
|
||||
# If model not found in default_params, return current values (no change)
|
||||
return {}, {}, {}, {}, {}
|
||||
|
||||
return (
|
||||
gr.update(value=448),
|
||||
gr.update(value=832),
|
||||
gr.update(value=61),
|
||||
gr.update(value=3.0),
|
||||
gr.update(value=1024),
|
||||
)
|
||||
|
||||
model_selection.change(
|
||||
fn=on_model_selection_change,
|
||||
@@ -702,31 +563,28 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
|
||||
)
|
||||
|
||||
def handle_generation(*args, progress=None, request: gr.Request = None):
|
||||
# Extract model selection and input image from args
|
||||
model_selection, prompt, negative_prompt, use_negative_prompt, seed, guidance_scale, num_frames, height, width, randomize_seed = args
|
||||
|
||||
# Call the generate_video function with progress tracking
|
||||
result_path, seed_or_error, timing_details = generate_video(
|
||||
prompt, negative_prompt, use_negative_prompt, seed, guidance_scale,
|
||||
num_frames, height, width, randomize_seed, None, model_selection, progress, request
|
||||
num_frames, height, width, randomize_seed, model_selection, progress
|
||||
)
|
||||
|
||||
if result_path and os.path.exists(result_path):
|
||||
return (
|
||||
result_path,
|
||||
seed_or_error,
|
||||
gr.update(visible=False), # error_output
|
||||
gr.update(visible=True, value=timing_details), # timing_display
|
||||
gr.update(visible=False),
|
||||
gr.update(visible=True, value=timing_details),
|
||||
)
|
||||
else:
|
||||
return (
|
||||
None,
|
||||
seed_or_error,
|
||||
gr.update(visible=True, value=seed_or_error), # error_output
|
||||
gr.update(visible=False), # timing_display
|
||||
gr.update(visible=True, value=seed_or_error),
|
||||
gr.update(visible=False),
|
||||
)
|
||||
|
||||
# Unified event handler
|
||||
run_button.click(
|
||||
fn=handle_generation,
|
||||
inputs=[
|
||||
@@ -750,55 +608,40 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="FastVideo Gradio Frontend")
|
||||
parser.add_argument("--backend_url",
|
||||
type=str,
|
||||
default="http://localhost:8000",
|
||||
parser.add_argument("--backend_url", type=str, default="http://localhost:8000",
|
||||
help="URL of the Ray Serve backend")
|
||||
parser.add_argument("--t2v_model_paths",
|
||||
type=str,
|
||||
parser.add_argument("--t2v_model_paths", type=str,
|
||||
default="FastVideo/FastWan2.1-T2V-1.3B-Diffusers,FastVideo/FastWan2.1-T2V-14B-Diffusers",
|
||||
help="Comma separated list of paths to the T2V model(s)")
|
||||
# parser.add_argument("--t2v_model_replicas", type=int,
|
||||
# default="4,4",
|
||||
# help="Comma separated list of number of replicas for the T2V model(s)")
|
||||
parser.add_argument("--host",
|
||||
type=str,
|
||||
default="0.0.0.0",
|
||||
parser.add_argument("--host", type=str, default="0.0.0.0",
|
||||
help="Host to bind to")
|
||||
parser.add_argument("--port",
|
||||
type=int,
|
||||
default=7860,
|
||||
parser.add_argument("--port", type=int, default=7860,
|
||||
help="Port to bind to")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Load default parameters from the models
|
||||
default_params = {}
|
||||
model_paths = args.t2v_model_paths.split(",")
|
||||
for model_path in model_paths:
|
||||
default_params[model_path] = SamplingParam.from_pretrained(model_path)
|
||||
|
||||
# Create and launch the interface
|
||||
demo = create_gradio_interface(args.backend_url, default_params)
|
||||
|
||||
print(f"Starting Gradio frontend at http://{args.host}:{args.port}")
|
||||
print(f"Backend URL: {args.backend_url}")
|
||||
print(f"T2V Models: {args.t2v_model_paths}")
|
||||
|
||||
# Use FastAPI to serve custom HTML with proper Open Graph metadata
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi import FastAPI, Request, HTTPException
|
||||
from fastapi.responses import HTMLResponse, FileResponse
|
||||
import uvicorn
|
||||
import os
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
@app.get("/logo.png")
|
||||
async def get_logo():
|
||||
from fastapi import Response
|
||||
def get_logo():
|
||||
return FileResponse(
|
||||
"fastvideo-logos/main/png/full.png",
|
||||
media_type="image/png",
|
||||
"fastvideo-logos/main/svg/full.svg",
|
||||
media_type="image/svg+xml",
|
||||
headers={
|
||||
"Cache-Control": "public, max-age=3600",
|
||||
"Access-Control-Allow-Origin": "*"
|
||||
@@ -806,28 +649,23 @@ def main():
|
||||
)
|
||||
|
||||
@app.get("/favicon.ico")
|
||||
async def get_favicon():
|
||||
from fastapi import Response, HTTPException
|
||||
import os
|
||||
|
||||
favicon_path = "fastvideo-logos/main/png/icon-simple.png"
|
||||
def get_favicon():
|
||||
favicon_path = "fastvideo-logos/main/svg/icon-simple.svg"
|
||||
|
||||
if os.path.exists(favicon_path):
|
||||
return FileResponse(
|
||||
favicon_path,
|
||||
media_type="image/png",
|
||||
media_type="image/svg+xml",
|
||||
headers={
|
||||
"Cache-Control": "public, max-age=3600",
|
||||
"Access-Control-Allow-Origin": "*"
|
||||
}
|
||||
)
|
||||
else:
|
||||
# Return a 404 if the favicon doesn't exist
|
||||
raise HTTPException(status_code=404, detail="Favicon not found")
|
||||
|
||||
@app.get("/", response_class=HTMLResponse)
|
||||
def index(request: Request):
|
||||
# Get the current host for dynamic URLs
|
||||
base_url = str(request.base_url).rstrip('/')
|
||||
return f"""
|
||||
<!DOCTYPE html>
|
||||
@@ -836,13 +674,11 @@ def main():
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
|
||||
<!-- Primary Meta Tags -->
|
||||
<title>FastWan</title>
|
||||
<meta name="title" content="FastWan">
|
||||
<meta name="description" content="Make video generation go blurrrrrrr">
|
||||
<meta name="keywords" content="FastVideo, video generation, AI, machine learning, FastWan">
|
||||
|
||||
<!-- Open Graph / Facebook -->
|
||||
<meta property="og:type" content="website">
|
||||
<meta property="og:url" content="{base_url}/">
|
||||
<meta property="og:title" content="FastWan">
|
||||
@@ -852,7 +688,6 @@ def main():
|
||||
<meta property="og:image:height" content="630">
|
||||
<meta property="og:site_name" content="FastWan">
|
||||
|
||||
<!-- Twitter -->
|
||||
<meta property="twitter:card" content="summary_large_image">
|
||||
<meta property="twitter:url" content="{base_url}/">
|
||||
<meta property="twitter:title" content="FastWan">
|
||||
@@ -885,10 +720,9 @@ def main():
|
||||
app,
|
||||
demo,
|
||||
path="/gradio",
|
||||
allowed_paths=[os.path.abspath("outputs"), os.path.abspath("temp_images"), os.path.abspath("fastvideo-logos")]
|
||||
allowed_paths=[os.path.abspath("outputs"), os.path.abspath("fastvideo-logos")]
|
||||
)
|
||||
|
||||
# Run the FastAPI server
|
||||
uvicorn.run(app, host=args.host, port=args.port)
|
||||
|
||||
|
||||
|
||||
@@ -1,182 +0,0 @@
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import threading
|
||||
import signal
|
||||
import requests
|
||||
from pathlib import Path
|
||||
|
||||
# Add the project root to the Python path
|
||||
project_root = Path(__file__).parent.parent.parent.parent
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
|
||||
def check_frontend_health(frontend_url: str, max_retries: int = 30) -> bool:
|
||||
"""Check if the frontend is healthy"""
|
||||
for i in range(max_retries):
|
||||
try:
|
||||
response = requests.get(frontend_url, timeout=5)
|
||||
if response.status_code == 200:
|
||||
print(f"✅ Frontend is healthy at {frontend_url}")
|
||||
return True
|
||||
except requests.exceptions.RequestException:
|
||||
pass
|
||||
|
||||
if i < max_retries - 1:
|
||||
print(f"⏳ Waiting for frontend to start... ({i+1}/{max_retries})")
|
||||
time.sleep(2)
|
||||
|
||||
print(f"❌ Frontend failed to start within {max_retries * 2} seconds")
|
||||
return False
|
||||
|
||||
|
||||
def start_frontend_instance(args, instance_id: int, backend_url: str):
|
||||
"""Start a single frontend instance"""
|
||||
frontend_script = Path(__file__).parent / "gradio_frontend.py"
|
||||
frontend_port = args.frontend_base_port + instance_id
|
||||
|
||||
cmd = [
|
||||
sys.executable, str(frontend_script),
|
||||
"--backend_url", backend_url,
|
||||
"--t2v_model_path", args.t2v_model_path,
|
||||
"--i2v_model_path", args.i2v_model_path,
|
||||
"--host", args.frontend_host,
|
||||
"--port", str(frontend_port)
|
||||
]
|
||||
|
||||
print(f"🎨 Starting Frontend {instance_id + 1} on port {frontend_port}...")
|
||||
print(f"Command: {' '.join(cmd)}")
|
||||
|
||||
# Start the frontend process
|
||||
frontend_process = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True,
|
||||
bufsize=1
|
||||
)
|
||||
|
||||
# Monitor frontend output
|
||||
def monitor_frontend():
|
||||
for line in frontend_process.stdout:
|
||||
print(f"[FRONTEND-{instance_id + 1}] {line.rstrip()}")
|
||||
|
||||
monitor_thread = threading.Thread(target=monitor_frontend, daemon=True)
|
||||
monitor_thread.start()
|
||||
|
||||
return frontend_process, frontend_port
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="FastVideo Multi-Frontend Launcher")
|
||||
|
||||
# Model and output settings
|
||||
parser.add_argument("--t2v_model_path",
|
||||
type=str,
|
||||
default="FastVideo/FastWan2.1-T2V-1.3B-Diffusers",
|
||||
help="Path to the T2V model")
|
||||
parser.add_argument("--i2v_model_path",
|
||||
type=str,
|
||||
default="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
|
||||
help="Path to the I2V model")
|
||||
|
||||
# Frontend settings
|
||||
parser.add_argument("--frontend_host",
|
||||
type=str,
|
||||
default="0.0.0.0",
|
||||
help="Frontend host to bind to")
|
||||
parser.add_argument("--frontend_base_port",
|
||||
type=int,
|
||||
default=7860,
|
||||
help="Base port for frontend instances")
|
||||
parser.add_argument("--num_frontends",
|
||||
type=int,
|
||||
default=2,
|
||||
help="Number of frontend instances to start")
|
||||
|
||||
# Backend settings
|
||||
parser.add_argument("--backend_url",
|
||||
type=str,
|
||||
default="http://localhost:8000",
|
||||
help="Backend URL for frontends to connect to")
|
||||
|
||||
# Other settings
|
||||
parser.add_argument("--skip_health_check",
|
||||
action="store_true",
|
||||
help="Skip frontend health check")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
print("🎬 FastVideo Multi-Frontend Launcher")
|
||||
print("=" * 50)
|
||||
print(f"T2V Model: {args.t2v_model_path}")
|
||||
print(f"I2V Model: {args.i2v_model_path}")
|
||||
print(f"Backend URL: {args.backend_url}")
|
||||
print(f"Number of Frontends: {args.num_frontends}")
|
||||
print(f"Frontend Base Port: {args.frontend_base_port}")
|
||||
print("=" * 50)
|
||||
|
||||
# Start multiple frontend instances
|
||||
frontend_processes = []
|
||||
frontend_urls = []
|
||||
|
||||
for i in range(args.num_frontends):
|
||||
process, port = start_frontend_instance(args, i, args.backend_url)
|
||||
frontend_processes.append(process)
|
||||
frontend_urls.append(f"http://{args.frontend_host}:{port}")
|
||||
|
||||
# Wait for frontends to be ready
|
||||
if not args.skip_health_check:
|
||||
print("\n⏳ Waiting for frontends to start...")
|
||||
for i, url in enumerate(frontend_urls):
|
||||
if not check_frontend_health(url):
|
||||
print(f"❌ Frontend {i + 1} failed to start. Terminating...")
|
||||
for process in frontend_processes:
|
||||
process.terminate()
|
||||
sys.exit(1)
|
||||
|
||||
print("\n🎉 All frontend instances are starting up!")
|
||||
for i, url in enumerate(frontend_urls):
|
||||
print(f"📺 Frontend {i + 1}: {url}")
|
||||
print("\nPress Ctrl+C to stop all frontend instances...")
|
||||
|
||||
# Signal handler for graceful shutdown
|
||||
def signal_handler(signum, frame):
|
||||
print("\n🛑 Shutting down frontend instances...")
|
||||
for process in frontend_processes:
|
||||
process.terminate()
|
||||
|
||||
# Wait for processes to terminate
|
||||
try:
|
||||
for process in frontend_processes:
|
||||
process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
print("⚠️ Force killing processes...")
|
||||
for process in frontend_processes:
|
||||
process.kill()
|
||||
|
||||
print("✅ Frontend instances stopped")
|
||||
sys.exit(0)
|
||||
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
|
||||
# Monitor processes
|
||||
try:
|
||||
while True:
|
||||
# Check if processes are still running
|
||||
for i, process in enumerate(frontend_processes):
|
||||
if process.poll() is not None:
|
||||
print(f"❌ Frontend {i + 1} process died unexpectedly")
|
||||
break
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
signal_handler(signal.SIGINT, None)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,181 +0,0 @@
|
||||
# Nginx configuration for FastVideo load balancing
|
||||
# This configuration implements the architecture:
|
||||
# ngrok -> nginx reverse proxy -> frontend1/frontend2 -> backend1×8/backend2×8
|
||||
|
||||
events {
|
||||
worker_connections 1024;
|
||||
}
|
||||
|
||||
http {
|
||||
# Basic settings
|
||||
sendfile on;
|
||||
tcp_nopush on;
|
||||
tcp_nodelay on;
|
||||
keepalive_timeout 65;
|
||||
types_hash_max_size 2048;
|
||||
client_max_body_size 100M; # Allow large video uploads
|
||||
|
||||
# Logging
|
||||
access_log /mnt/fast-disks/nfs/hao_lab/FastVideo/outputs/nginx_access.log;
|
||||
error_log /mnt/fast-disks/nfs/hao_lab/FastVideo/outputs/nginx_error.log;
|
||||
|
||||
# Gzip compression
|
||||
gzip on;
|
||||
gzip_vary on;
|
||||
gzip_min_length 1024;
|
||||
gzip_proxied any;
|
||||
gzip_comp_level 6;
|
||||
gzip_types
|
||||
text/plain
|
||||
text/css
|
||||
text/xml
|
||||
text/javascript
|
||||
application/json
|
||||
application/javascript
|
||||
application/xml+rss
|
||||
application/atom+xml
|
||||
image/svg+xml;
|
||||
|
||||
# Upstream for frontend load balancing
|
||||
upstream frontend_servers {
|
||||
# Round-robin load balancing between frontends
|
||||
server 127.0.0.1:7860 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:7861 weight=1 max_fails=3 fail_timeout=30s;
|
||||
upstream frontend_servers {
|
||||
# Round-robin load balancing between frontends
|
||||
server 127.0.0.1:7860 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:7861 weight=1 max_fails=3 fail_timeout=30s;
|
||||
|
||||
# Health check
|
||||
keepalive 32;
|
||||
}
|
||||
|
||||
# Upstream for backend1 load balancing
|
||||
upstream backend1_servers {
|
||||
server 127.0.0.1:8000 weight=1 max_fails=3 fail_timeout=30s; (8 replicas)
|
||||
upstream backend1_servers {
|
||||
# Round-robin load balancing for backend1 replicas
|
||||
server 127.0.0.1:8000 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8001 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8002 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8003 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8004 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8005 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8006 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8007 weight=1 max_fails=3 fail_timeout=30s;
|
||||
|
||||
keepalive 32;
|
||||
}
|
||||
|
||||
# Upstream for backend2 load balancing
|
||||
upstream backend2_servers {
|
||||
server 127.0.0.1:8000 weight=1 max_fails=3 fail_timeout=30s; (8 replicas)
|
||||
upstream backend2_servers {
|
||||
# Round-robin load balancing for backend2 replicas
|
||||
server 127.0.0.1:8010 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8011 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8012 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8013 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8014 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8015 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8016 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8017 weight=1 max_fails=3 fail_timeout=30s;
|
||||
|
||||
keepalive 32;
|
||||
}
|
||||
|
||||
# Main server block
|
||||
server {
|
||||
listen 80;
|
||||
server_name localhost;
|
||||
|
||||
# Security headers
|
||||
add_header X-Frame-Options "SAMEORIGIN" always;
|
||||
add_header X-Content-Type-Options "nosniff" always;
|
||||
add_header X-XSS-Protection "1; mode=block" always;
|
||||
add_header Referrer-Policy "no-referrer-when-downgrade" always;
|
||||
|
||||
# Frontend routes (Gradio interfaces)
|
||||
location / {
|
||||
proxy_pass http://frontend_servers;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# WebSocket support for Gradio
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
|
||||
# Timeouts
|
||||
proxy_connect_timeout 60s;
|
||||
proxy_send_timeout 60s;
|
||||
proxy_read_timeout 60s;
|
||||
|
||||
# Buffer settings
|
||||
proxy_buffering on;
|
||||
proxy_buffer_size 128k;
|
||||
proxy_buffers 4 256k;
|
||||
proxy_busy_buffers_size 256k;
|
||||
}
|
||||
|
||||
# Backend API routes for frontend1
|
||||
location /api/frontend1/ {
|
||||
# Strip the /api/frontend1/ prefix
|
||||
rewrite ^/api/frontend1/(.*) /$1 break;
|
||||
proxy_pass http://backend1_servers;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# Timeouts for video generation
|
||||
proxy_connect_timeout 300s;
|
||||
proxy_send_timeout 300s;
|
||||
proxy_read_timeout 300s;
|
||||
|
||||
# Buffer settings for large responses
|
||||
proxy_buffering on;
|
||||
proxy_buffer_size 128k;
|
||||
proxy_buffers 4 256k;
|
||||
proxy_busy_buffers_size 256k;
|
||||
}
|
||||
|
||||
# Backend API routes for frontend2
|
||||
location /api/frontend2/ {
|
||||
# Strip the /api/frontend2/ prefix
|
||||
rewrite ^/api/frontend2/(.*) /$1 break;
|
||||
proxy_pass http://backend2_servers;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# Timeouts for video generation
|
||||
proxy_connect_timeout 300s;
|
||||
proxy_send_timeout 300s;
|
||||
proxy_read_timeout 300s;
|
||||
|
||||
# Buffer settings for large responses
|
||||
proxy_buffering on;
|
||||
proxy_buffer_size 128k;
|
||||
proxy_buffers 4 256k;
|
||||
proxy_busy_buffers_size 256k;
|
||||
}
|
||||
|
||||
# Health check endpoint
|
||||
location /health {
|
||||
access_log off;
|
||||
return 200 "healthy\n";
|
||||
add_header Content-Type text/plain;
|
||||
}
|
||||
|
||||
# Static files (if needed)
|
||||
location /static/ {
|
||||
alias /var/www/static/;
|
||||
expires 1y;
|
||||
add_header Cache-Control "public, immutable";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,173 +0,0 @@
|
||||
# Nginx configuration for FastVideo load balancing
|
||||
# This configuration implements the architecture:
|
||||
# ngrok -> nginx reverse proxy -> frontend1/frontend2 -> backend1×8/backend2×8
|
||||
|
||||
events {
|
||||
worker_connections 1024;
|
||||
}
|
||||
|
||||
http {
|
||||
# Basic settings
|
||||
sendfile on;
|
||||
tcp_nopush on;
|
||||
tcp_nodelay on;
|
||||
keepalive_timeout 65;
|
||||
types_hash_max_size 2048;
|
||||
client_max_body_size 100M; # Allow large video uploads
|
||||
|
||||
# Logging
|
||||
access_log /var/log/nginx/access.log;
|
||||
error_log /var/log/nginx/error.log;
|
||||
|
||||
# Gzip compression
|
||||
gzip on;
|
||||
gzip_vary on;
|
||||
gzip_min_length 1024;
|
||||
gzip_proxied any;
|
||||
gzip_comp_level 6;
|
||||
gzip_types
|
||||
text/plain
|
||||
text/css
|
||||
text/xml
|
||||
text/javascript
|
||||
application/json
|
||||
application/javascript
|
||||
application/xml+rss
|
||||
application/atom+xml
|
||||
image/svg+xml;
|
||||
|
||||
# Upstream for frontend load balancing
|
||||
upstream frontend_servers {
|
||||
# Round-robin load balancing between frontends
|
||||
server 127.0.0.1:7860 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:7861 weight=1 max_fails=3 fail_timeout=30s;
|
||||
|
||||
# Health check
|
||||
keepalive 32;
|
||||
}
|
||||
|
||||
# Upstream for backend1 load balancing (8 replicas)
|
||||
upstream backend1_servers {
|
||||
# Round-robin load balancing for backend1 replicas
|
||||
server 127.0.0.1:8000 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8001 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8002 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8003 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8004 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8005 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8006 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8007 weight=1 max_fails=3 fail_timeout=30s;
|
||||
|
||||
keepalive 32;
|
||||
}
|
||||
|
||||
# Upstream for backend2 load balancing (8 replicas)
|
||||
upstream backend2_servers {
|
||||
# Round-robin load balancing for backend2 replicas
|
||||
server 127.0.0.1:8010 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8011 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8012 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8013 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8014 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8015 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8016 weight=1 max_fails=3 fail_timeout=30s;
|
||||
server 127.0.0.1:8017 weight=1 max_fails=3 fail_timeout=30s;
|
||||
|
||||
keepalive 32;
|
||||
}
|
||||
|
||||
# Main server block
|
||||
server {
|
||||
listen 80;
|
||||
server_name localhost;
|
||||
|
||||
# Security headers
|
||||
add_header X-Frame-Options "SAMEORIGIN" always;
|
||||
add_header X-Content-Type-Options "nosniff" always;
|
||||
add_header X-XSS-Protection "1; mode=block" always;
|
||||
add_header Referrer-Policy "no-referrer-when-downgrade" always;
|
||||
|
||||
# Frontend routes (Gradio interfaces)
|
||||
location / {
|
||||
proxy_pass http://frontend_servers;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# WebSocket support for Gradio
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
|
||||
# Timeouts
|
||||
proxy_connect_timeout 60s;
|
||||
proxy_send_timeout 60s;
|
||||
proxy_read_timeout 60s;
|
||||
|
||||
# Buffer settings
|
||||
proxy_buffering on;
|
||||
proxy_buffer_size 128k;
|
||||
proxy_buffers 4 256k;
|
||||
proxy_busy_buffers_size 256k;
|
||||
}
|
||||
|
||||
# Backend API routes for frontend1
|
||||
location /api/frontend1/ {
|
||||
# Strip the /api/frontend1/ prefix
|
||||
rewrite ^/api/frontend1/(.*) /$1 break;
|
||||
proxy_pass http://backend1_servers;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# Timeouts for video generation
|
||||
proxy_connect_timeout 300s;
|
||||
proxy_send_timeout 300s;
|
||||
proxy_read_timeout 300s;
|
||||
|
||||
# Buffer settings for large responses
|
||||
proxy_buffering on;
|
||||
proxy_buffer_size 128k;
|
||||
proxy_buffers 4 256k;
|
||||
proxy_busy_buffers_size 256k;
|
||||
}
|
||||
|
||||
# Backend API routes for frontend2
|
||||
location /api/frontend2/ {
|
||||
# Strip the /api/frontend2/ prefix
|
||||
rewrite ^/api/frontend2/(.*) /$1 break;
|
||||
proxy_pass http://backend2_servers;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# Timeouts for video generation
|
||||
proxy_connect_timeout 300s;
|
||||
proxy_send_timeout 300s;
|
||||
proxy_read_timeout 300s;
|
||||
|
||||
# Buffer settings for large responses
|
||||
proxy_buffering on;
|
||||
proxy_buffer_size 128k;
|
||||
proxy_buffers 4 256k;
|
||||
proxy_busy_buffers_size 256k;
|
||||
}
|
||||
|
||||
# Health check endpoint
|
||||
location /health {
|
||||
access_log off;
|
||||
return 200 "healthy\n";
|
||||
add_header Content-Type text/plain;
|
||||
}
|
||||
|
||||
# Static files (if needed)
|
||||
location /static/ {
|
||||
alias /var/www/static/;
|
||||
expires 1y;
|
||||
add_header Cache-Control "public, immutable";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,80 +5,45 @@ import base64
|
||||
import io
|
||||
from copy import deepcopy
|
||||
from typing import Dict, Any, Optional, List
|
||||
import json
|
||||
from datetime import datetime
|
||||
import threading
|
||||
import signal
|
||||
import sys
|
||||
|
||||
import ray
|
||||
from ray import serve
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi import FastAPI, Request, Response
|
||||
from pydantic import BaseModel
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
from slowapi import Limiter, _rate_limit_exceeded_handler
|
||||
from slowapi.util import get_remote_address
|
||||
from slowapi.errors import RateLimitExceeded
|
||||
import imageio
|
||||
from ray.serve.handle import DeploymentHandle
|
||||
from prometheus_client import Counter, Histogram, generate_latest
|
||||
|
||||
NUM_GPUS = 16
|
||||
DEFAULT_FPS = 16
|
||||
SEED_RANGE_MAX = 1_000_000
|
||||
SUPPORTED_MODELS = [
|
||||
"FastVideo/FastWan2.1-T2V-1.3B-Diffusers",
|
||||
"FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers",
|
||||
]
|
||||
|
||||
|
||||
class PromptLogger:
|
||||
"""Thread-safe prompt logging utility"""
|
||||
|
||||
def __init__(self, log_file: str = "user_prompts.jsonl"):
|
||||
self.log_file = log_file
|
||||
self.lock = threading.Lock()
|
||||
|
||||
# Ensure the directory exists
|
||||
log_dir = os.path.dirname(os.path.abspath(log_file))
|
||||
if log_dir: # Only create directory if there's actually a directory path
|
||||
os.makedirs(log_dir, exist_ok=True)
|
||||
|
||||
print(f"PromptLogger initialized with log file: {os.path.abspath(log_file)}")
|
||||
|
||||
def log_prompt(self, prompt: str, model_type: str, success: bool,
|
||||
negative_prompt: Optional[str] = None,
|
||||
generation_time: Optional[float] = None,
|
||||
user_ip: Optional[str] = None):
|
||||
"""Log a prompt with metadata"""
|
||||
log_entry = {
|
||||
"timestamp": datetime.utcnow().isoformat(),
|
||||
"prompt": prompt,
|
||||
"negative_prompt": negative_prompt,
|
||||
"model_type": model_type,
|
||||
"success": success,
|
||||
"generation_time": generation_time,
|
||||
"user_ip": user_ip
|
||||
}
|
||||
|
||||
print(f"Logging prompt: {prompt[:50]}... to {self.log_file}")
|
||||
|
||||
with self.lock:
|
||||
try:
|
||||
# Ensure the directory exists before writing
|
||||
log_dir = os.path.dirname(os.path.abspath(self.log_file))
|
||||
if log_dir:
|
||||
os.makedirs(log_dir, exist_ok=True)
|
||||
|
||||
with open(self.log_file, "a", encoding="utf-8") as f:
|
||||
f.write(json.dumps(log_entry, ensure_ascii=False) + "\n")
|
||||
f.flush() # Force write to disk
|
||||
print(f"Successfully logged prompt to {os.path.abspath(self.log_file)}")
|
||||
except Exception as e:
|
||||
print(f"Error: Failed to log prompt to {self.log_file}: {e}")
|
||||
print(f"Current working directory: {os.getcwd()}")
|
||||
print(f"Attempted log file path: {os.path.abspath(self.log_file)}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
# prompt_logger will be initialized inside the deployment class to avoid serialization issues
|
||||
MODEL_CONFIGS = {
|
||||
"1.3B": {
|
||||
"num_cpus": 2,
|
||||
"text_encoder_cpu_offload": False,
|
||||
"dit_cpu_offload": False,
|
||||
"vae_cpu_offload": False,
|
||||
"VSA_sparsity": 0.8,
|
||||
},
|
||||
"14B": {
|
||||
"num_cpus": 16,
|
||||
"text_encoder_cpu_offload": True,
|
||||
"dit_cpu_offload": True,
|
||||
"vae_cpu_offload": False,
|
||||
"VSA_sparsity": 0.9,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class VideoGenerationRequest(BaseModel):
|
||||
@@ -90,20 +55,17 @@ class VideoGenerationRequest(BaseModel):
|
||||
num_frames: int = 21
|
||||
height: int = 448
|
||||
width: int = 832
|
||||
# num_inference_steps: int = 20
|
||||
randomize_seed: bool = False
|
||||
return_frames: bool = False # Whether to return base64 encoded frames
|
||||
image_path: Optional[str] = None # Path to input image for I2V
|
||||
model_path: Optional[str] = None # Specific model path to use
|
||||
return_frames: bool = False
|
||||
model_path: Optional[str] = None
|
||||
|
||||
|
||||
class VideoGenerationResponse(BaseModel):
|
||||
video_data: Optional[str] = None # Base64 encoded video data
|
||||
video_data: Optional[str] = None
|
||||
seed: int
|
||||
success: bool
|
||||
error_message: Optional[str] = None
|
||||
generation_time: Optional[float] = None
|
||||
# Detailed timing information
|
||||
model_load_time: Optional[float] = None
|
||||
inference_time: Optional[float] = None
|
||||
encoding_time: Optional[float] = None
|
||||
@@ -112,18 +74,15 @@ class VideoGenerationResponse(BaseModel):
|
||||
stage_execution_times: Optional[List[float]] = None
|
||||
|
||||
|
||||
def encode_video_to_base64(frames: List[np.ndarray], fps: int = 24) -> str:
|
||||
"""Convert numpy frames to base64-encoded MP4 video"""
|
||||
def encode_video_to_base64(frames: List[np.ndarray], fps: int = DEFAULT_FPS) -> str:
|
||||
if not frames:
|
||||
return ""
|
||||
|
||||
try:
|
||||
# Save frames to bytes buffer as MP4
|
||||
buffer = io.BytesIO()
|
||||
imageio.mimsave(buffer, frames, fps=fps, format="mp4")
|
||||
buffer.seek(0)
|
||||
|
||||
# Encode to base64
|
||||
video_base64 = base64.b64encode(buffer.getvalue()).decode('utf-8')
|
||||
return f"data:video/mp4;base64,{video_base64}"
|
||||
|
||||
@@ -132,350 +91,183 @@ def encode_video_to_base64(frames: List[np.ndarray], fps: int = 24) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
def setup_model_environment(model_path: str) -> None:
|
||||
if "fullattn" in model_path.lower():
|
||||
os.environ["FASTVIDEO_ATTENTION_BACKEND"] = "FLASH_ATTN"
|
||||
else:
|
||||
os.environ["FASTVIDEO_ATTENTION_BACKEND"] = "VIDEO_SPARSE_ATTN"
|
||||
os.environ["FASTVIDEO_STAGE_LOGGING"] = "1"
|
||||
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Model-Specific Deployments (one per model type)
|
||||
# These deployments each load a single model and expose a `generate_video`
|
||||
# method that can be invoked via a DeploymentHandle. Each deployment can be
|
||||
# scaled independently by configuring `num_replicas` when binding.
|
||||
# ---------------------------------------------------------------------------
|
||||
def process_generation_result(result: Any) -> tuple[List[np.ndarray], float, List[str], List[float]]:
|
||||
frames = result if isinstance(result, list) else result.get("frames", [])
|
||||
generation_time = result.get("generation_time", 0.0) if isinstance(result, dict) else 0.0
|
||||
|
||||
logging_info = result.get("logging_info", None)
|
||||
if logging_info:
|
||||
stage_names = logging_info.get_execution_order()
|
||||
stage_execution_times = [
|
||||
logging_info.get_stage_info(stage_name).get("execution_time", 0.0)
|
||||
for stage_name in stage_names
|
||||
]
|
||||
else:
|
||||
stage_names = []
|
||||
stage_execution_times = []
|
||||
|
||||
return frames, generation_time, stage_names, stage_execution_times
|
||||
|
||||
|
||||
@serve.deployment( # T2V 1.3B model deployment
|
||||
def prepare_sampling_params(video_request: VideoGenerationRequest, default_params: Any) -> Any:
|
||||
params = deepcopy(default_params)
|
||||
params.prompt = video_request.prompt
|
||||
|
||||
if video_request.use_negative_prompt:
|
||||
params.negative_prompt = video_request.negative_prompt
|
||||
|
||||
params.seed = (video_request.seed if not video_request.randomize_seed
|
||||
else torch.randint(0, SEED_RANGE_MAX, (1,)).item())
|
||||
params.randomize_seed = video_request.randomize_seed
|
||||
params.guidance_scale = video_request.guidance_scale
|
||||
params.num_frames = video_request.num_frames
|
||||
params.height = video_request.height
|
||||
params.width = video_request.width
|
||||
params.save_video = False
|
||||
params.return_frames = False
|
||||
|
||||
return params
|
||||
|
||||
|
||||
class BaseModelDeployment:
|
||||
def __init__(self, model_path: str, output_path: str = "outputs"):
|
||||
self.model_path = model_path
|
||||
self.output_path = output_path
|
||||
self.generator = None
|
||||
self.default_params = None
|
||||
|
||||
os.makedirs(self.output_path, exist_ok=True)
|
||||
setup_model_environment(self.model_path)
|
||||
|
||||
def _initialize_generator(self, config: Dict[str, Any]) -> None:
|
||||
from fastvideo.entrypoints.video_generator import VideoGenerator
|
||||
from fastvideo.configs.sample.base import SamplingParam
|
||||
|
||||
print(f"Initializing model: {self.model_path}")
|
||||
self.generator = VideoGenerator.from_pretrained(
|
||||
model_path=self.model_path,
|
||||
num_gpus=1,
|
||||
use_fsdp_inference=True,
|
||||
text_encoder_cpu_offload=config["text_encoder_cpu_offload"],
|
||||
dit_cpu_offload=config["dit_cpu_offload"],
|
||||
vae_cpu_offload=config["vae_cpu_offload"],
|
||||
VSA_sparsity=config["VSA_sparsity"],
|
||||
enable_stage_verification=False,
|
||||
)
|
||||
self.default_params = SamplingParam.from_pretrained(self.model_path)
|
||||
|
||||
def generate_video(self, video_request: VideoGenerationRequest) -> VideoGenerationResponse:
|
||||
total_start_time = time.time()
|
||||
|
||||
params = prepare_sampling_params(video_request, self.default_params)
|
||||
|
||||
inference_start_time = time.time()
|
||||
result = self.generator.generate_video(
|
||||
prompt=video_request.prompt,
|
||||
sampling_param=params,
|
||||
save_video=False,
|
||||
return_frames=False,
|
||||
)
|
||||
inference_time = time.time() - inference_start_time
|
||||
|
||||
frames, generation_time, stage_names, stage_execution_times = process_generation_result(result)
|
||||
|
||||
encoding_start_time = time.time()
|
||||
video_data = encode_video_to_base64(frames, fps=DEFAULT_FPS)
|
||||
encoding_time = time.time() - encoding_start_time
|
||||
|
||||
total_time = time.time() - total_start_time
|
||||
|
||||
return VideoGenerationResponse(
|
||||
video_data=video_data,
|
||||
seed=params.seed,
|
||||
success=True,
|
||||
generation_time=generation_time,
|
||||
inference_time=inference_time,
|
||||
encoding_time=encoding_time,
|
||||
total_time=total_time,
|
||||
stage_names=stage_names,
|
||||
stage_execution_times=stage_execution_times,
|
||||
)
|
||||
|
||||
|
||||
@serve.deployment(
|
||||
ray_actor_options={"num_cpus": 2, "num_gpus": 1, "runtime_env": {"conda": "fv"}},
|
||||
)
|
||||
class T2VModelDeployment:
|
||||
"""Serve deployment wrapping the 1.3 B text-to-video model."""
|
||||
|
||||
class T2VModelDeployment(BaseModelDeployment):
|
||||
def __init__(self, t2v_model_path: str, output_path: str = "outputs"):
|
||||
self.model_path = t2v_model_path
|
||||
self.output_path = output_path
|
||||
|
||||
# Ensure output directory exists
|
||||
os.makedirs(self.output_path, exist_ok=True)
|
||||
|
||||
# Delay helps avoid GPU contention when many replicas start at once
|
||||
# time.sleep(5)
|
||||
|
||||
# Ensure correct attention backend for FastVideo
|
||||
if "fullattn" in self.model_path.lower():
|
||||
os.environ["FASTVIDEO_ATTENTION_BACKEND"] = "FLASH_ATTN"
|
||||
else:
|
||||
os.environ["FASTVIDEO_ATTENTION_BACKEND"] = "VIDEO_SPARSE_ATTN"
|
||||
os.environ["FASTVIDEO_STAGE_LOGGING"] = "1"
|
||||
|
||||
# Lazy import to keep the deployment import-safe on head node
|
||||
from fastvideo.entrypoints.video_generator import VideoGenerator
|
||||
from fastvideo.configs.sample.base import SamplingParam
|
||||
|
||||
print(f"Initializing T2V model: {self.model_path}")
|
||||
self.generator = VideoGenerator.from_pretrained(
|
||||
model_path=self.model_path,
|
||||
num_gpus=1,
|
||||
use_fsdp_inference=True,
|
||||
text_encoder_cpu_offload=False,
|
||||
dit_cpu_offload=False,
|
||||
vae_cpu_offload=False,
|
||||
VSA_sparsity=0.8,
|
||||
enable_stage_verification=False,
|
||||
)
|
||||
self.default_params = SamplingParam.from_pretrained(self.model_path)
|
||||
super().__init__(t2v_model_path, output_path)
|
||||
self._initialize_generator(MODEL_CONFIGS["1.3B"])
|
||||
print("✅ T2V model initialized successfully")
|
||||
|
||||
def generate_video(self, video_request: "VideoGenerationRequest") -> "VideoGenerationResponse":
|
||||
"""Generate a video for the given request and return an encoded response."""
|
||||
import time
|
||||
|
||||
total_start_time = time.time()
|
||||
|
||||
# Deep-copy default sampling params and override with request values
|
||||
params = deepcopy(self.default_params)
|
||||
params.prompt = video_request.prompt
|
||||
if video_request.use_negative_prompt:
|
||||
params.negative_prompt = video_request.negative_prompt
|
||||
|
||||
params.seed = video_request.seed if not video_request.randomize_seed else torch.randint(0, 1_000_000, (1,)).item()
|
||||
# Ensure the generator respects the chosen seed strategy
|
||||
params.randomize_seed = video_request.randomize_seed
|
||||
params.guidance_scale = video_request.guidance_scale
|
||||
params.num_frames = video_request.num_frames
|
||||
params.height = video_request.height
|
||||
params.width = video_request.width
|
||||
# params.num_inference_steps = video_request.num_inference_steps
|
||||
|
||||
# Do not write to disk when called via API
|
||||
params.save_video = False
|
||||
params.return_frames = False
|
||||
|
||||
# Track inference time
|
||||
inference_start_time = time.time()
|
||||
|
||||
# Generate video frames
|
||||
result = self.generator.generate_video(
|
||||
prompt=video_request.prompt,
|
||||
sampling_param=params,
|
||||
save_video=False,
|
||||
return_frames=False,
|
||||
)
|
||||
|
||||
inference_end_time = time.time()
|
||||
inference_time = inference_end_time - inference_start_time
|
||||
|
||||
frames = result if isinstance(result, list) else result.get("frames", [])
|
||||
generation_time = result.get("generation_time", 0.0) if isinstance(result, dict) else 0.0
|
||||
|
||||
|
||||
logging_info = result.get("logging_info", None)
|
||||
if logging_info:
|
||||
stage_names = logging_info.get_execution_order()
|
||||
stage_execution_times = [logging_info.get_stage_info(stage_name).get("execution_time", 0.0) for stage_name in stage_names]
|
||||
else:
|
||||
stage_names = []
|
||||
stage_execution_times = []
|
||||
|
||||
# Track encoding time
|
||||
encoding_start_time = time.time()
|
||||
|
||||
# Encode outputs
|
||||
video_data = encode_video_to_base64(frames, fps=16)
|
||||
|
||||
encoding_end_time = time.time()
|
||||
encoding_time = encoding_end_time - encoding_start_time
|
||||
|
||||
total_end_time = time.time()
|
||||
total_time = total_end_time - total_start_time
|
||||
|
||||
return VideoGenerationResponse(
|
||||
video_data=video_data,
|
||||
seed=params.seed,
|
||||
success=True,
|
||||
generation_time=generation_time,
|
||||
inference_time=inference_time,
|
||||
encoding_time=encoding_time,
|
||||
total_time=total_time,
|
||||
stage_names=stage_names,
|
||||
stage_execution_times=stage_execution_times,
|
||||
)
|
||||
|
||||
|
||||
@serve.deployment( # T2V 14B model deployment with optimized settings
|
||||
@serve.deployment(
|
||||
ray_actor_options={"num_cpus": 16, "num_gpus": 1, "runtime_env": {"conda": "fv"}},
|
||||
)
|
||||
class T2V14BModelDeployment:
|
||||
"""Serve deployment wrapping the 14B text-to-video model with optimized settings."""
|
||||
|
||||
class T2V14BModelDeployment(BaseModelDeployment):
|
||||
def __init__(self, t2v_14b_model_path: str, output_path: str = "outputs"):
|
||||
self.model_path = t2v_14b_model_path
|
||||
self.output_path = output_path
|
||||
|
||||
# Ensure output directory exists
|
||||
os.makedirs(self.output_path, exist_ok=True)
|
||||
|
||||
# Delay helps avoid GPU contention when many replicas start at once
|
||||
time.sleep(10)
|
||||
|
||||
# Ensure correct attention backend for FastVideo
|
||||
super().__init__(t2v_14b_model_path, output_path)
|
||||
# Override environment for 14B model
|
||||
os.environ["FASTVIDEO_ATTENTION_BACKEND"] = "VIDEO_SPARSE_ATTN"
|
||||
os.environ["FASTVIDEO_STAGE_LOGGING"] = "1"
|
||||
|
||||
# Lazy import to keep the deployment import-safe on head node
|
||||
from fastvideo.entrypoints.video_generator import VideoGenerator
|
||||
from fastvideo.configs.sample.base import SamplingParam
|
||||
|
||||
print(f"Initializing T2V 14B model: {self.model_path}")
|
||||
self.generator = VideoGenerator.from_pretrained(
|
||||
model_path=self.model_path,
|
||||
num_gpus=1,
|
||||
use_fsdp_inference=True,
|
||||
text_encoder_cpu_offload=True, # Enable CPU offload for 14B model
|
||||
dit_cpu_offload=True, # Enable CPU offload for 14B model
|
||||
vae_cpu_offload=False,
|
||||
VSA_sparsity=0.9, # Higher sparsity for 14B model
|
||||
enable_stage_verification=False,
|
||||
)
|
||||
self.default_params = SamplingParam.from_pretrained(self.model_path)
|
||||
self._initialize_generator(MODEL_CONFIGS["14B"])
|
||||
print("✅ T2V 14B model initialized successfully")
|
||||
|
||||
def generate_video(self, video_request: "VideoGenerationRequest") -> "VideoGenerationResponse":
|
||||
"""Generate a video for the given request and return an encoded response."""
|
||||
import time
|
||||
|
||||
total_start_time = time.time()
|
||||
|
||||
# Deep-copy default sampling params and override with request values
|
||||
params = deepcopy(self.default_params)
|
||||
params.prompt = video_request.prompt
|
||||
if video_request.use_negative_prompt:
|
||||
params.negative_prompt = video_request.negative_prompt
|
||||
|
||||
params.seed = video_request.seed if not video_request.randomize_seed else torch.randint(0, 1_000_000, (1,)).item()
|
||||
# Ensure the generator respects the chosen seed strategy
|
||||
params.randomize_seed = video_request.randomize_seed
|
||||
params.guidance_scale = video_request.guidance_scale
|
||||
params.num_frames = video_request.num_frames
|
||||
params.height = video_request.height
|
||||
params.width = video_request.width
|
||||
# params.num_inference_steps = video_request.num_inference_steps
|
||||
|
||||
# Do not write to disk when called via API
|
||||
params.save_video = False
|
||||
params.return_frames = False
|
||||
|
||||
# Track inference time
|
||||
inference_start_time = time.time()
|
||||
|
||||
# Generate video frames
|
||||
result = self.generator.generate_video(
|
||||
prompt=video_request.prompt,
|
||||
sampling_param=params,
|
||||
save_video=False,
|
||||
return_frames=False,
|
||||
)
|
||||
|
||||
inference_end_time = time.time()
|
||||
inference_time = inference_end_time - inference_start_time
|
||||
|
||||
frames = result if isinstance(result, list) else result.get("frames", [])
|
||||
generation_time = result.get("generation_time", 0.0) if isinstance(result, dict) else 0.0
|
||||
logging_info = result.get("logging_info", None)
|
||||
if logging_info:
|
||||
stage_names = logging_info.get_execution_order()
|
||||
stage_execution_times = [logging_info.get_stage_info(stage_name).get("execution_time", 0.0) for stage_name in stage_names]
|
||||
else:
|
||||
stage_names = []
|
||||
stage_execution_times = []
|
||||
|
||||
# Track encoding time
|
||||
encoding_start_time = time.time()
|
||||
|
||||
# Encode outputs
|
||||
video_data = encode_video_to_base64(frames, fps=16)
|
||||
|
||||
encoding_end_time = time.time()
|
||||
encoding_time = encoding_end_time - encoding_start_time
|
||||
|
||||
total_end_time = time.time()
|
||||
total_time = total_end_time - total_start_time
|
||||
|
||||
return VideoGenerationResponse(
|
||||
video_data=video_data,
|
||||
seed=params.seed,
|
||||
success=True,
|
||||
generation_time=generation_time,
|
||||
inference_time=inference_time,
|
||||
encoding_time=encoding_time,
|
||||
total_time=total_time,
|
||||
stage_names=stage_names,
|
||||
stage_execution_times=stage_execution_times,
|
||||
)
|
||||
|
||||
|
||||
# Create FastAPI app with rate limiting
|
||||
app = FastAPI()
|
||||
|
||||
# Initialize rate limiter
|
||||
limiter = Limiter(key_func=get_remote_address)
|
||||
app.state.limiter = limiter
|
||||
app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
|
||||
|
||||
# Import Prometheus metrics (but don't instantiate at module level)
|
||||
from prometheus_client import Counter, Histogram, generate_latest, CONTENT_TYPE_LATEST
|
||||
import time
|
||||
|
||||
|
||||
@serve.deployment(num_replicas=50, ray_actor_options={"num_cpus": 2})
|
||||
@serve.ingress(app)
|
||||
class FastVideoAPI:
|
||||
"""Ingress deployment that routes requests to either the T2V or I2V model deployment."""
|
||||
|
||||
def __init__(self, t2v_deployments: Dict[str, DeploymentHandle]): # Removed i2v_deployment
|
||||
def __init__(self, t2v_deployments: Dict[str, DeploymentHandle]):
|
||||
self.t2v_deployments = t2v_deployments
|
||||
# self.t2v_14b_handle = t2v_14b_deployment
|
||||
# self.i2v_handle = i2v_deployment # I2V functionality commented out
|
||||
|
||||
# Initialize Prometheus metrics inside the deployment
|
||||
# Initialize Prometheus metrics
|
||||
self.request_count = Counter('fastvideo_requests_total', 'Total FastVideo requests', ['model_type', 'status'])
|
||||
self.request_duration = Histogram('fastvideo_request_duration_seconds', 'FastVideo request duration', ['model_type'])
|
||||
self.video_generation_time = Histogram('fastvideo_video_generation_seconds', 'Video generation time', ['model_type'])
|
||||
|
||||
def _get_model_name(self, model_path: Optional[str]) -> str:
|
||||
return model_path.split('/')[-1] if model_path else "unknown"
|
||||
|
||||
def _record_metrics(self, model_name: str, status: str, duration: float, response: Optional[VideoGenerationResponse] = None) -> None:
|
||||
self.request_count.labels(model_type=model_name, status=status).inc()
|
||||
self.request_duration.labels(model_type=model_name).observe(duration)
|
||||
|
||||
# Initialize prompt logger inside the deployment to avoid serialization issues
|
||||
# Use absolute path to ensure we know where the file is created
|
||||
log_file_path = os.path.join(os.getcwd(), "logs", "user_prompts.jsonl")
|
||||
|
||||
# Hacky fix: Create the file upfront to ensure it exists
|
||||
log_dir = os.path.dirname(log_file_path)
|
||||
os.makedirs(log_dir, exist_ok=True)
|
||||
|
||||
# Touch the file to ensure it exists
|
||||
with open(log_file_path, "a") as f:
|
||||
pass # Just create/ensure file exists
|
||||
|
||||
print(f"Created log file at: {log_file_path}")
|
||||
self.prompt_logger = PromptLogger(log_file_path)
|
||||
if response and hasattr(response, 'generation_time') and response.generation_time:
|
||||
self.video_generation_time.labels(model_type=model_name).observe(response.generation_time)
|
||||
|
||||
@app.post("/generate_video", response_model=VideoGenerationResponse)
|
||||
@limiter.limit("10/minute") # Allow 50 requests per minute per IP
|
||||
@limiter.limit("10/minute")
|
||||
async def generate_video(self, request: Request, video_request: VideoGenerationRequest) -> VideoGenerationResponse:
|
||||
"""Route the request to the appropriate model deployment based on `model_path`."""
|
||||
"""Route the request to the appropriate model deployment based on model_path."""
|
||||
start_time = time.time()
|
||||
model_name = video_request.model_path.split('/')[-1] if video_request.model_path else "unknown"
|
||||
user_ip = get_remote_address(request)
|
||||
model_name = self._get_model_name(video_request.model_path)
|
||||
|
||||
try:
|
||||
# assert video_request.model_path in self.t2v_deployments, f"Model {video_request.model_path} not found"
|
||||
if video_request.model_path not in self.t2v_deployments:
|
||||
raise ValueError(f"Model {video_request.model_path} not found")
|
||||
|
||||
response_ref = self.t2v_deployments[video_request.model_path].generate_video.remote(video_request)
|
||||
# if video_request.model_type.lower() == "i2v": # I2V functionality commented out
|
||||
# response_ref = self.i2v_handle.generate_video.remote(video_request)
|
||||
# if video_request.model_type.lower() == "t2v":
|
||||
# # Route T2V requests based on model path
|
||||
# if video_request.model_path and "14b" in video_request.model_path.lower():
|
||||
# response_ref = self.t2v_14b_handle.generate_video.remote(video_request)
|
||||
# else:
|
||||
# # Default to 1.3B model
|
||||
# response_ref = self.t2v_handle.generate_video.remote(video_request)
|
||||
# else:
|
||||
# # Default to 1.3B T2V model
|
||||
# response_ref = self.t2v_handle.generate_video.remote(video_request)
|
||||
|
||||
# Await the remote response
|
||||
response = await response_ref
|
||||
|
||||
# Log the prompt (success case)
|
||||
self.prompt_logger.log_prompt(
|
||||
prompt=video_request.prompt,
|
||||
model_type=video_request.model_path,
|
||||
success=response.success,
|
||||
negative_prompt=video_request.negative_prompt if video_request.use_negative_prompt else None,
|
||||
generation_time=response.total_time,
|
||||
user_ip=user_ip
|
||||
)
|
||||
|
||||
# Record success metrics
|
||||
self.request_count.labels(model_type=model_name, status="success").inc()
|
||||
self.request_duration.labels(model_type=model_name).observe(time.time() - start_time)
|
||||
if hasattr(response, 'generation_time_seconds') and response.generation_time_seconds:
|
||||
self.video_generation_time.labels(model_type=model_name).observe(response.generation_time_seconds)
|
||||
|
||||
self._record_metrics(model_name, "success", time.time() - start_time, response)
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
# Log the prompt (error case)
|
||||
self.prompt_logger.log_prompt(
|
||||
prompt=video_request.prompt,
|
||||
model_type=video_request.model_path,
|
||||
success=False,
|
||||
negative_prompt=video_request.negative_prompt if video_request.use_negative_prompt else None,
|
||||
generation_time=None,
|
||||
user_ip=user_ip
|
||||
)
|
||||
|
||||
# Record error metrics
|
||||
self.request_count.labels(model_type=model_name, status="error").inc()
|
||||
self.request_duration.labels(model_type=model_name).observe(time.time() - start_time)
|
||||
self._record_metrics(model_name, "error", time.time() - start_time)
|
||||
|
||||
return VideoGenerationResponse(
|
||||
video_data=None,
|
||||
@@ -489,75 +281,71 @@ class FastVideoAPI:
|
||||
)
|
||||
|
||||
@app.get("/health")
|
||||
@limiter.limit("10/minute") # Allow 10 health checks per minute per IP
|
||||
async def health_check(self, request: Request):
|
||||
@limiter.limit("10/minute")
|
||||
async def health_check(self, request: Request) -> Dict[str, str]:
|
||||
return {"status": "healthy"}
|
||||
|
||||
@app.get("/metrics")
|
||||
async def metrics(self):
|
||||
"""Expose Prometheus metrics"""
|
||||
from fastapi import Response
|
||||
async def metrics(self) -> Response:
|
||||
return Response(generate_latest(), media_type="text/plain")
|
||||
|
||||
|
||||
def validate_configuration(model_paths: List[str], replicas: List[int]) -> None:
|
||||
assert len(model_paths) == len(replicas), "Number of models and replicas must match"
|
||||
assert sum(replicas) <= NUM_GPUS, f"Total replicas ({sum(replicas)}) must be <= {NUM_GPUS}"
|
||||
|
||||
for model, replica_count in zip(model_paths, replicas):
|
||||
assert model in SUPPORTED_MODELS, f"Model {model} not supported"
|
||||
assert replica_count > 0, f"Replicas must be greater than 0"
|
||||
|
||||
|
||||
def start_ray_serve(
|
||||
*,
|
||||
t2v_model_paths: str,
|
||||
t2v_model_replicas: str,
|
||||
# t2v_14b_model_path: str = "FastVideo/FastWan2.1-T2V-14B-Diffusers",
|
||||
# i2v_model_path: str = "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers", # I2V functionality commented out
|
||||
output_path: str = "outputs",
|
||||
host: str = "0.0.0.0",
|
||||
port: int = 8000,
|
||||
# t2v_replicas: int = 4,
|
||||
# t2v_14b_replicas: int = 4, # Reduced replicas for 14B model due to higher resource requirements
|
||||
# i2v_replicas: int = 4, # I2V functionality commented out
|
||||
):
|
||||
"""Start Ray Serve with independently scalable deployments for each model."""
|
||||
|
||||
) -> None:
|
||||
if not ray.is_initialized():
|
||||
ray.init()
|
||||
|
||||
# Bind model deployments with configurable replica counts
|
||||
model_paths = t2v_model_paths.split(",")
|
||||
replicas = [int(r) for r in t2v_model_replicas.split(",")]
|
||||
validate_configuration(model_paths, replicas)
|
||||
|
||||
t2v_deps = {}
|
||||
for model_path, replicas in zip(t2v_model_paths.split(","), t2v_model_replicas.split(",")):
|
||||
t2v_dep = T2VModelDeployment.options(num_replicas=int(replicas)).bind(model_path, output_path)
|
||||
for model_path, replica_count in zip(model_paths, replicas):
|
||||
t2v_dep = T2VModelDeployment.options(num_replicas=replica_count).bind(model_path, output_path)
|
||||
t2v_deps[model_path] = t2v_dep
|
||||
# i2v_dep = I2VModelDeployment.options(num_replicas=i2v_replicas).bind(i2v_model_path, output_path) # I2V functionality commented out
|
||||
|
||||
# Ingress
|
||||
api = FastVideoAPI.bind(t2v_deps) # Removed i2v_dep
|
||||
|
||||
api = FastVideoAPI.bind(t2v_deps)
|
||||
serve.run(api, route_prefix="/", name="fast_video")
|
||||
|
||||
print(f"Ray Serve backend started at http://{host}:{port}")
|
||||
for model_path, replicas in zip(t2v_model_paths.split(","), t2v_model_replicas.split(",")):
|
||||
print(f"T2V Model: {model_path} | Replicas: {replicas}")
|
||||
# print(f"T2V 14B Model: {t2v_14b_model_path} | Replicas: {t2v_14b_replicas}")
|
||||
# print(f"I2V Model: {i2v_model_path} | Replicas: {i2v_replicas}") # I2V functionality commented out
|
||||
for model_path, replica_count in zip(model_paths, replicas):
|
||||
print(f"T2V Model: {model_path} | Replicas: {replica_count}")
|
||||
print(f"Health check: http://{host}:{port}/health")
|
||||
print(f"Video generation endpoint: http://{host}:{port}/generate_video")
|
||||
|
||||
|
||||
def setup_signal_handlers() -> None:
|
||||
signal.signal(signal.SIGINT, lambda *_: sys.exit(0))
|
||||
signal.signal(signal.SIGTERM, lambda *_: sys.exit(0))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="FastVideo Ray Serve Backend")
|
||||
parser.add_argument("--t2v_model_paths",
|
||||
type=str,
|
||||
default="FastVideo/FastWan2.1-T2V-1.3B-Diffusers,FastVideo/FastWan2.1-T2V-14B-Diffusers",
|
||||
default="FastVideo/FastWan2.1-T2V-1.3B-Diffusers,FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers",
|
||||
help="Comma separated list of paths to the T2V model(s)")
|
||||
parser.add_argument("--t2v_model_replicas",
|
||||
type=str,
|
||||
default="4,4",
|
||||
help="Comma separated list of number of replicas for the T2V model(s)")
|
||||
# parser.add_argument("--t2v_14b_model_path",
|
||||
# type=str,
|
||||
# default="FastVideo/FastWan2.1-T2V-14B-Diffusers",
|
||||
# help="Path to the T2V 14B model")
|
||||
# parser.add_argument("--i2v_model_path", # I2V functionality commented out
|
||||
# type=str,
|
||||
# default="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
|
||||
# help="Path to the I2V model")
|
||||
parser.add_argument("--output_path",
|
||||
type=str,
|
||||
default="outputs",
|
||||
@@ -570,48 +358,22 @@ if __name__ == "__main__":
|
||||
type=int,
|
||||
default=8000,
|
||||
help="Port to bind to")
|
||||
# parser.add_argument("--t2v_replicas",
|
||||
# type=int,
|
||||
# default=4,
|
||||
# help="Number of replicas for the T2V model deployment")
|
||||
# parser.add_argument("--t2v_14b_replicas",
|
||||
# type=int,
|
||||
# default=4, # Reduced default for 14B model due to higher resource requirements
|
||||
# help="Number of replicas for the T2V 14B model deployment")
|
||||
# parser.add_argument("--i2v_replicas", # I2V functionality commented out
|
||||
# type=int,
|
||||
# default=4, # Reduced default for 14B model due to higher resource requirements
|
||||
# help="Number of replicas for the I2V model deployment")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
split_models = args.t2v_model_paths.split(",")
|
||||
split_replicas = [int(replica) for replica in args.t2v_model_replicas.split(",")]
|
||||
assert len(split_models) == len(split_replicas), "Number of models and replicas must match"
|
||||
assert sum(split_replicas) <= NUM_GPUS, "Total number of replicas must be less than or equal to 16"
|
||||
for model, replicas in zip(split_models, split_replicas):
|
||||
assert model in SUPPORTED_MODELS, f"Model {model} not supported"
|
||||
assert replicas > 0, f"Replicas must be greater than 0"
|
||||
|
||||
model_paths = args.t2v_model_paths.split(",")
|
||||
replicas = [int(r) for r in args.t2v_model_replicas.split(",")]
|
||||
validate_configuration(model_paths, replicas)
|
||||
|
||||
start_ray_serve(
|
||||
t2v_model_paths=args.t2v_model_paths,
|
||||
t2v_model_replicas=args.t2v_model_replicas,
|
||||
# t2v_14b_model_path=args.t2v_14b_model_path,
|
||||
# i2v_model_path=args.i2v_model_path, # I2V functionality commented out
|
||||
output_path=args.output_path,
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
# t2v_replicas=args.t2v_replicas,
|
||||
# t2v_14b_replicas=args.t2v_14b_replicas,
|
||||
# i2v_replicas=args.i2v_replicas, # I2V functionality commented out
|
||||
)
|
||||
|
||||
# ---- keep the process alive ---------------------------------
|
||||
import signal, sys, time
|
||||
signal.signal(signal.SIGINT, lambda *_: sys.exit(0)) # Ctrl-C
|
||||
signal.signal(signal.SIGTERM, lambda *_: sys.exit(0)) # docker stop etc.
|
||||
|
||||
setup_signal_handlers()
|
||||
print("✅ FastVideo backend is running. Press Ctrl-C to stop.")
|
||||
while True:
|
||||
time.sleep(3600)
|
||||
@@ -1,334 +0,0 @@
|
||||
import time
|
||||
import os
|
||||
import torch
|
||||
import base64
|
||||
import io
|
||||
from copy import deepcopy
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
import ray
|
||||
from ray import serve
|
||||
from fastapi import FastAPI, Request
|
||||
from pydantic import BaseModel
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
from slowapi import Limiter, _rate_limit_exceeded_handler
|
||||
from slowapi.util import get_remote_address
|
||||
from slowapi.errors import RateLimitExceeded
|
||||
|
||||
|
||||
class VideoGenerationRequest(BaseModel):
|
||||
prompt: str
|
||||
negative_prompt: Optional[str] = None
|
||||
use_negative_prompt: bool = False
|
||||
seed: int = 42
|
||||
guidance_scale: float = 7.5
|
||||
num_frames: int = 21
|
||||
height: int = 448
|
||||
width: int = 832
|
||||
num_inference_steps: int = 20
|
||||
randomize_seed: bool = False
|
||||
return_frames: bool = False # Whether to return base64 encoded frames
|
||||
image_path: Optional[str] = None # Path to input image for I2V
|
||||
model_type: str = "t2v" # "t2v" or "i2v" to specify which model to use
|
||||
|
||||
|
||||
class VideoGenerationResponse(BaseModel):
|
||||
output_path: str
|
||||
seed: int
|
||||
success: bool
|
||||
error_message: Optional[str] = None
|
||||
frames: Optional[List[str]] = None # Base64 encoded frames
|
||||
|
||||
|
||||
def encode_frames_to_base64(frames: List[np.ndarray]) -> List[str]:
|
||||
"""Convert numpy frames (0-255) to base64-encoded PNG images"""
|
||||
if not frames:
|
||||
return []
|
||||
|
||||
encoded_frames = []
|
||||
|
||||
for i, frame in enumerate(frames):
|
||||
try:
|
||||
# Ensure frame is numpy array
|
||||
if not isinstance(frame, np.ndarray):
|
||||
print(f"Warning: Frame {i} is not a numpy array, skipping")
|
||||
continue
|
||||
|
||||
# Ensure frame is uint8
|
||||
if frame.dtype != np.uint8:
|
||||
# Clip values to 0-255 range and convert to uint8
|
||||
frame = np.clip(frame, 0, 255).astype(np.uint8)
|
||||
|
||||
# Convert numpy array to PIL Image
|
||||
if len(frame.shape) == 3 and frame.shape[2] == 3:
|
||||
# RGB image
|
||||
pil_image = Image.fromarray(frame, mode='RGB')
|
||||
elif len(frame.shape) == 3 and frame.shape[2] == 4:
|
||||
# RGBA image
|
||||
pil_image = Image.fromarray(frame, mode='RGBA')
|
||||
elif len(frame.shape) == 2:
|
||||
# Grayscale image
|
||||
pil_image = Image.fromarray(frame, mode='L')
|
||||
else:
|
||||
print(f"Warning: Frame {i} has unsupported shape {frame.shape}, skipping")
|
||||
continue
|
||||
|
||||
# Save to bytes buffer as PNG
|
||||
buffer = io.BytesIO()
|
||||
pil_image.save(buffer, format='PNG')
|
||||
buffer.seek(0)
|
||||
|
||||
# Encode to base64
|
||||
img_base64 = base64.b64encode(buffer.getvalue()).decode('utf-8')
|
||||
encoded_frames.append(f"data:image/png;base64,{img_base64}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Warning: Failed to encode frame {i}: {e}")
|
||||
continue
|
||||
|
||||
return encoded_frames
|
||||
|
||||
|
||||
# Create FastAPI app with rate limiting
|
||||
app = FastAPI()
|
||||
|
||||
# Initialize rate limiter
|
||||
limiter = Limiter(key_func=get_remote_address)
|
||||
app.state.limiter = limiter
|
||||
app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
|
||||
|
||||
|
||||
@serve.deployment(
|
||||
num_replicas=3, # Set to 3 for cluster 0
|
||||
ray_actor_options={
|
||||
"num_cpus": 10,
|
||||
"num_gpus": 1,
|
||||
"runtime_env": {"conda": "fv"},
|
||||
},
|
||||
)
|
||||
@serve.ingress(app)
|
||||
class FastVideoMultiGPUAPI:
|
||||
def __init__(self, t2v_model_path: str, i2v_model_path: str, output_path: str, gpu_id: int = 0):
|
||||
self.t2v_model_path = t2v_model_path
|
||||
self.i2v_model_path = i2v_model_path
|
||||
self.output_path = output_path
|
||||
self.gpu_id = gpu_id
|
||||
|
||||
# Initialize the video generators
|
||||
self.t2v_generator = None
|
||||
self.i2v_generator = None
|
||||
self.t2v_default_params = None
|
||||
self.i2v_default_params = None
|
||||
|
||||
# Ensure output directory exists
|
||||
os.makedirs(output_path, exist_ok=True)
|
||||
time.sleep(10)
|
||||
self._initialize_models()
|
||||
|
||||
def _initialize_models(self):
|
||||
# Set VSA environment variable
|
||||
os.environ["FASTVIDEO_ATTENTION_BACKEND"] = "VIDEO_SPARSE_ATTN"
|
||||
os.environ["FASTVIDEO_ATTENTION_BACKEND"] = "FLASH_ATTN"
|
||||
|
||||
# Import only when needed
|
||||
from fastvideo.entrypoints.video_generator import VideoGenerator
|
||||
from fastvideo.configs.sample.base import SamplingParam
|
||||
|
||||
# Initialize T2V model
|
||||
if False: # Disabled for now
|
||||
print(f"Initializing T2V model on GPU {self.gpu_id}: {self.t2v_model_path}")
|
||||
self.t2v_generator = VideoGenerator.from_pretrained(
|
||||
model_path=self.t2v_model_path,
|
||||
num_gpus=1,
|
||||
use_fsdp_inference=True,
|
||||
text_encoder_cpu_offload=False,
|
||||
dit_cpu_offload=False,
|
||||
vae_cpu_offload=False,
|
||||
VSA_sparsity=0.8,
|
||||
)
|
||||
self.t2v_default_params = SamplingParam.from_pretrained(self.t2v_model_path)
|
||||
print(f"✅ T2V model initialized successfully on GPU {self.gpu_id}")
|
||||
|
||||
# Initialize I2V model
|
||||
if self.i2v_generator is None:
|
||||
print(f"Initializing I2V model on GPU {self.gpu_id}: {self.i2v_model_path}")
|
||||
self.i2v_generator = VideoGenerator.from_pretrained(
|
||||
model_path=self.i2v_model_path,
|
||||
num_gpus=1,
|
||||
use_fsdp_inference=True,
|
||||
text_encoder_cpu_offload=False,
|
||||
dit_cpu_offload=False,
|
||||
vae_cpu_offload=False,
|
||||
VSA_sparsity=0.8,
|
||||
)
|
||||
self.i2v_default_params = SamplingParam.from_pretrained(self.i2v_model_path)
|
||||
print(f"✅ I2V model initialized successfully on GPU {self.gpu_id}")
|
||||
|
||||
@app.post("/generate_video", response_model=VideoGenerationResponse)
|
||||
@limiter.limit("2/minute") # Allow 2 requests per minute per IP
|
||||
async def generate_video(self, request: Request, video_request: VideoGenerationRequest) -> VideoGenerationResponse:
|
||||
try:
|
||||
# Select the appropriate model and parameters based on model_type
|
||||
if video_request.model_type.lower() == "i2v":
|
||||
generator = self.i2v_generator
|
||||
params = deepcopy(self.i2v_default_params)
|
||||
print(f"Using I2V model for generation on GPU {self.gpu_id}")
|
||||
else:
|
||||
generator = self.t2v_generator
|
||||
params = deepcopy(self.t2v_default_params)
|
||||
print(f"Using T2V model for generation on GPU {self.gpu_id}")
|
||||
|
||||
# Update parameters with request values
|
||||
params.prompt = video_request.prompt
|
||||
|
||||
# Handle seed randomization
|
||||
if video_request.randomize_seed:
|
||||
params.seed = torch.randint(0, 1000000, (1,)).item()
|
||||
|
||||
# Ensure negative_prompt is a non-None string
|
||||
if params.negative_prompt is None:
|
||||
params.negative_prompt = ""
|
||||
|
||||
# Set up output path and video saving
|
||||
params.save_video = True
|
||||
params.output_path = self.output_path
|
||||
|
||||
# Create a clean filename from the prompt
|
||||
safe_prompt = video_request.prompt[:100].replace(' ', '_').replace('/', '_').replace('\\', '_')
|
||||
setattr(params, "output_video_name", safe_prompt)
|
||||
|
||||
# Handle image_path for I2V
|
||||
if video_request.image_path:
|
||||
params.image_path = video_request.image_path
|
||||
|
||||
# Generate the video
|
||||
result = generator.generate_video(
|
||||
prompt=video_request.prompt,
|
||||
sampling_param=params,
|
||||
save_video=True,
|
||||
)
|
||||
|
||||
frames = result.get("frames", [])
|
||||
|
||||
# Encode frames to base64 for web transmission only if requested
|
||||
encoded_frames = None
|
||||
if video_request.return_frames and frames:
|
||||
try:
|
||||
encoded_frames = encode_frames_to_base64(frames)
|
||||
except Exception as e:
|
||||
print(f"Warning: Failed to encode frames: {e}")
|
||||
encoded_frames = None
|
||||
|
||||
response = VideoGenerationResponse(
|
||||
output_path="",
|
||||
frames=encoded_frames,
|
||||
seed=params.seed,
|
||||
success=True
|
||||
)
|
||||
|
||||
# Memory cleanup to avoid OOM in repeated generations
|
||||
import gc
|
||||
gc.collect()
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
return response
|
||||
except Exception as e:
|
||||
return VideoGenerationResponse(
|
||||
output_path="",
|
||||
seed=video_request.seed,
|
||||
success=False,
|
||||
error_message=str(e)
|
||||
)
|
||||
|
||||
@app.get("/health")
|
||||
@limiter.limit("10/minute") # Allow 10 health checks per minute per IP
|
||||
async def health_check(self, request: Request):
|
||||
return {"status": "healthy", "gpu_id": self.gpu_id}
|
||||
|
||||
|
||||
def start_ray_serve_multi_gpu(
|
||||
t2v_model_path: str = "FastVideo/FastWan2.1-T2V-1.3B-Diffusers",
|
||||
i2v_model_path: str = "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
|
||||
output_path: str = "outputs",
|
||||
host: str = "0.0.0.0",
|
||||
port: int = 8000,
|
||||
num_gpus: int = 8,
|
||||
cluster_id: int = 0
|
||||
):
|
||||
"""Start the Ray Serve backend with multiple GPU replicas"""
|
||||
# Initialize Ray
|
||||
if not ray.is_initialized():
|
||||
ray.init()
|
||||
|
||||
# Use unique application name based on cluster_id
|
||||
app_name = f"fast_video_cluster_{cluster_id}"
|
||||
|
||||
# Deploy the API
|
||||
api = FastVideoMultiGPUAPI.bind(t2v_model_path, i2v_model_path, output_path)
|
||||
serve.run(api, route_prefix=f"/cluster_{cluster_id}", name=app_name)
|
||||
|
||||
print(f"Ray Serve multi-GPU backend started at http://{host}:{port}")
|
||||
print(f"T2V Model: {t2v_model_path}")
|
||||
print(f"I2V Model: {i2v_model_path}")
|
||||
print(f"Number of GPU replicas: {num_gpus}")
|
||||
print(f"Cluster ID: {cluster_id}")
|
||||
print(f"Application name: {app_name}")
|
||||
print(f"Health check: http://{host}:{port}/cluster_{cluster_id}/health")
|
||||
print(f"Video generation endpoint: http://{host}:{port}/cluster_{cluster_id}/generate_video")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="FastVideo Ray Serve Multi-GPU Backend")
|
||||
parser.add_argument("--t2v_model_path",
|
||||
type=str,
|
||||
default="FastVideo/FastWan2.1-T2V-1.3B-Diffusers",
|
||||
help="Path to the T2V model")
|
||||
parser.add_argument("--i2v_model_path",
|
||||
type=str,
|
||||
default="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
|
||||
help="Path to the I2V model")
|
||||
parser.add_argument("--output_path",
|
||||
type=str,
|
||||
default="outputs",
|
||||
help="Path to save generated videos")
|
||||
parser.add_argument("--host",
|
||||
type=str,
|
||||
default="0.0.0.0",
|
||||
help="Host to bind to")
|
||||
parser.add_argument("--port",
|
||||
type=int,
|
||||
default=8000,
|
||||
help="Port to bind to")
|
||||
parser.add_argument("--num_gpus",
|
||||
type=int,
|
||||
default=8,
|
||||
help="Number of GPU replicas")
|
||||
parser.add_argument("--cluster_id",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Cluster ID for unique naming")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
start_ray_serve_multi_gpu(
|
||||
t2v_model_path=args.t2v_model_path,
|
||||
i2v_model_path=args.i2v_model_path,
|
||||
output_path=args.output_path,
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
num_gpus=args.num_gpus,
|
||||
cluster_id=args.cluster_id,
|
||||
)
|
||||
|
||||
# Keep the process alive
|
||||
import signal, sys, time
|
||||
signal.signal(signal.SIGINT, lambda *_: sys.exit(0))
|
||||
signal.signal(signal.SIGTERM, lambda *_: sys.exit(0))
|
||||
|
||||
print("✅ FastVideo multi-GPU backend is running. Press Ctrl-C to stop.")
|
||||
while True:
|
||||
time.sleep(3600)
|
||||
@@ -1,3 +1,3 @@
|
||||
python examples/inference/gradio/start_ray_serve_app.py \
|
||||
--t2v_model_paths "FastVideo/FastWan2.1-T2V-1.3B-Diffusers,FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers" \
|
||||
--t2v_model_replicas "8,8"
|
||||
--t2v_model_replicas "4,4"
|
||||
@@ -12,231 +12,245 @@ import threading
|
||||
import signal
|
||||
import requests
|
||||
from pathlib import Path
|
||||
|
||||
# Add the project root to the Python path
|
||||
project_root = Path(__file__).parent.parent.parent.parent
|
||||
sys.path.insert(0, str(project_root))
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
|
||||
def check_backend_health(backend_url: str, max_retries: int = 100) -> bool:
|
||||
"""Check if the backend is healthy"""
|
||||
health_url = f"{backend_url}/health"
|
||||
DEFAULT_BACKEND_HOST = "0.0.0.0"
|
||||
DEFAULT_BACKEND_PORT = 8000
|
||||
DEFAULT_FRONTEND_HOST = "0.0.0.0"
|
||||
DEFAULT_FRONTEND_PORT = 7860
|
||||
DEFAULT_OUTPUT_PATH = "outputs"
|
||||
DEFAULT_T2V_MODELS = "FastVideo/FastWan2.1-T2V-1.3B-Diffusers,FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers"
|
||||
DEFAULT_T2V_REPLICAS = "4,4"
|
||||
|
||||
HEALTH_CHECK_TIMEOUT = 5
|
||||
HEALTH_CHECK_MAX_RETRIES = 100
|
||||
HEALTH_CHECK_INTERVAL = 2
|
||||
PROCESS_SHUTDOWN_TIMEOUT = 5
|
||||
PROCESS_MONITOR_INTERVAL = 1
|
||||
|
||||
PROJECT_ROOT = Path(__file__).parent.parent.parent.parent
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
|
||||
class ServiceManager:
|
||||
|
||||
for i in range(max_retries):
|
||||
def __init__(self, args: argparse.Namespace):
|
||||
self.args = args
|
||||
self.backend_process: Optional[subprocess.Popen] = None
|
||||
self.frontend_process: Optional[subprocess.Popen] = None
|
||||
self.backend_url = f"http://{args.backend_host}:{args.backend_port}"
|
||||
|
||||
def check_backend_health(self, max_retries: int = HEALTH_CHECK_MAX_RETRIES) -> bool:
|
||||
health_url = f"{self.backend_url}/health"
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
response = requests.get(health_url, timeout=HEALTH_CHECK_TIMEOUT)
|
||||
if response.status_code == 200:
|
||||
print(f"✅ Backend is healthy at {self.backend_url}")
|
||||
return True
|
||||
except requests.exceptions.RequestException:
|
||||
pass
|
||||
|
||||
if attempt < max_retries - 1:
|
||||
print(f"⏳ Waiting for backend to start... ({attempt + 1}/{max_retries})")
|
||||
time.sleep(HEALTH_CHECK_INTERVAL)
|
||||
|
||||
print(f"❌ Backend failed to start within {max_retries * HEALTH_CHECK_INTERVAL} seconds")
|
||||
return False
|
||||
|
||||
def _create_monitor_thread(self, process: subprocess.Popen, service_name: str) -> threading.Thread:
|
||||
def monitor():
|
||||
if process.stdout:
|
||||
for line in process.stdout:
|
||||
print(f"[{service_name}] {line.rstrip()}")
|
||||
|
||||
thread = threading.Thread(target=monitor, daemon=True)
|
||||
thread.start()
|
||||
return thread
|
||||
|
||||
def _start_service(self, script_name: str, args_dict: Dict[str, Any], service_name: str) -> subprocess.Popen:
|
||||
script_path = Path(__file__).parent / script_name
|
||||
|
||||
cmd = [sys.executable, str(script_path)]
|
||||
for key, value in args_dict.items():
|
||||
cmd.extend([f"--{key}", str(value)])
|
||||
|
||||
print(f"🚀 Starting {service_name}...")
|
||||
print(f"Command: {' '.join(cmd)}")
|
||||
|
||||
process = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True,
|
||||
bufsize=1
|
||||
)
|
||||
|
||||
self._create_monitor_thread(process, service_name.upper())
|
||||
return process
|
||||
|
||||
def start_backend(self) -> subprocess.Popen:
|
||||
backend_args = {
|
||||
"t2v_model_paths": self.args.t2v_model_paths,
|
||||
"t2v_model_replicas": self.args.t2v_model_replicas,
|
||||
"output_path": self.args.output_path,
|
||||
"host": self.args.backend_host,
|
||||
"port": self.args.backend_port
|
||||
}
|
||||
|
||||
self.backend_process = self._start_service("ray_serve_backend.py", backend_args, "backend")
|
||||
return self.backend_process
|
||||
|
||||
def start_frontend(self) -> subprocess.Popen:
|
||||
frontend_args = {
|
||||
"backend_url": self.backend_url,
|
||||
"t2v_model_paths": self.args.t2v_model_paths,
|
||||
"host": self.args.frontend_host,
|
||||
"port": self.args.frontend_port
|
||||
}
|
||||
|
||||
self.frontend_process = self._start_service("gradio_frontend.py", frontend_args, "frontend")
|
||||
return self.frontend_process
|
||||
|
||||
def shutdown_services(self) -> None:
|
||||
print("\n🛑 Shutting down services...")
|
||||
|
||||
processes = []
|
||||
if self.frontend_process:
|
||||
self.frontend_process.terminate()
|
||||
processes.append(("frontend", self.frontend_process))
|
||||
|
||||
if self.backend_process:
|
||||
self.backend_process.terminate()
|
||||
processes.append(("backend", self.backend_process))
|
||||
|
||||
for name, process in processes:
|
||||
try:
|
||||
process.wait(timeout=PROCESS_SHUTDOWN_TIMEOUT)
|
||||
print(f"✅ {name.capitalize()} stopped gracefully")
|
||||
except subprocess.TimeoutExpired:
|
||||
print(f"⚠️ Force killing {name} process...")
|
||||
process.kill()
|
||||
|
||||
print("✅ All services stopped")
|
||||
|
||||
def monitor_processes(self) -> None:
|
||||
if not self.backend_process or not self.frontend_process:
|
||||
print("❌ Processes not properly initialized")
|
||||
return
|
||||
|
||||
try:
|
||||
response = requests.get(health_url, timeout=5)
|
||||
if response.status_code == 200:
|
||||
print(f"✅ Backend is healthy at {backend_url}")
|
||||
return True
|
||||
except requests.exceptions.RequestException:
|
||||
while True:
|
||||
if self.frontend_process.poll() is not None:
|
||||
print("❌ Frontend process died unexpectedly")
|
||||
break
|
||||
|
||||
if self.backend_process.poll() is not None:
|
||||
print("❌ Backend process died unexpectedly")
|
||||
break
|
||||
|
||||
time.sleep(PROCESS_MONITOR_INTERVAL)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
|
||||
if i < max_retries - 1:
|
||||
print(f"⏳ Waiting for backend to start... ({i+1}/{max_retries})")
|
||||
time.sleep(2)
|
||||
|
||||
print(f"❌ Backend failed to start within {max_retries * 2} seconds")
|
||||
return False
|
||||
self.shutdown_services()
|
||||
|
||||
|
||||
def start_backend(args):
|
||||
"""Start the Ray Serve backend"""
|
||||
backend_script = Path(__file__).parent / "ray_serve_backend.py"
|
||||
|
||||
cmd = [
|
||||
sys.executable, str(backend_script),
|
||||
"--t2v_model_paths", args.t2v_model_paths,
|
||||
"--t2v_model_replicas", args.t2v_model_replicas,
|
||||
# "--i2v_model_path", args.i2v_model_path, # I2V functionality commented out
|
||||
"--output_path", args.output_path,
|
||||
"--host", args.backend_host,
|
||||
"--port", str(args.backend_port)
|
||||
]
|
||||
|
||||
print(f"🚀 Starting Ray Serve backend...")
|
||||
print(f"Command: {' '.join(cmd)}")
|
||||
|
||||
# Start the backend process
|
||||
backend_process = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True,
|
||||
bufsize=1
|
||||
)
|
||||
|
||||
# Monitor backend output
|
||||
def monitor_backend():
|
||||
for line in backend_process.stdout:
|
||||
print(f"[BACKEND] {line.rstrip()}")
|
||||
|
||||
monitor_thread = threading.Thread(target=monitor_backend, daemon=True)
|
||||
monitor_thread.start()
|
||||
|
||||
return backend_process
|
||||
|
||||
|
||||
def start_frontend(args):
|
||||
"""Start the Gradio frontend"""
|
||||
frontend_script = Path(__file__).parent / "gradio_frontend.py"
|
||||
backend_url = f"http://{args.backend_host}:{args.backend_port}"
|
||||
|
||||
cmd = [
|
||||
sys.executable, str(frontend_script),
|
||||
"--backend_url", backend_url,
|
||||
"--t2v_model_paths", args.t2v_model_paths,
|
||||
# "--t2v_model_replicas", args.t2v_model_replicas,
|
||||
# "--i2v_model_path", args.i2v_model_path, # I2V functionality commented out
|
||||
"--host", args.frontend_host,
|
||||
"--port", str(args.frontend_port)
|
||||
]
|
||||
|
||||
print(f"🎨 Starting Gradio frontend...")
|
||||
print(f"Command: {' '.join(cmd)}")
|
||||
|
||||
# Start the frontend process
|
||||
frontend_process = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True,
|
||||
bufsize=1
|
||||
)
|
||||
|
||||
# Monitor frontend output
|
||||
def monitor_frontend():
|
||||
for line in frontend_process.stdout:
|
||||
print(f"[FRONTEND] {line.rstrip()}")
|
||||
|
||||
monitor_thread = threading.Thread(target=monitor_frontend, daemon=True)
|
||||
monitor_thread.start()
|
||||
|
||||
return frontend_process
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="FastVideo Ray Serve App")
|
||||
|
||||
# Model and output settings
|
||||
parser.add_argument("--t2v_model_paths",
|
||||
type=str,
|
||||
default="FastVideo/FastWan2.1-T2V-1.3B-Diffusers,FastVideo/FastWan2.1-T2V-14B-Diffusers",
|
||||
help="Comma separated list of paths to the T2V model(s)")
|
||||
parser.add_argument("--t2v_model_replicas",
|
||||
type=str,
|
||||
default="4,4",
|
||||
help="Comma separated list of number of replicas for the T2V model(s)")
|
||||
# parser.add_argument("--t2v_14b_model_path",
|
||||
# type=str,
|
||||
# default="FastVideo/FastWan2.1-T2V-14B-Diffusers",
|
||||
# help="Path to the T2V 14B model")
|
||||
# parser.add_argument("--i2v_model_path", # I2V functionality commented out
|
||||
# type=str,
|
||||
# default="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
|
||||
# help="Path to the I2V model")
|
||||
parser.add_argument("--output_path",
|
||||
type=str,
|
||||
default="outputs",
|
||||
help="Path to save generated videos")
|
||||
|
||||
# Backend settings
|
||||
parser.add_argument("--backend_host",
|
||||
type=str,
|
||||
default="0.0.0.0",
|
||||
help="Backend host to bind to")
|
||||
parser.add_argument("--backend_port",
|
||||
type=int,
|
||||
default=8000,
|
||||
help="Backend port to bind to")
|
||||
|
||||
# Frontend settings
|
||||
parser.add_argument("--frontend_host",
|
||||
type=str,
|
||||
default="0.0.0.0",
|
||||
help="Frontend host to bind to")
|
||||
parser.add_argument("--frontend_port",
|
||||
type=int,
|
||||
default=7861,
|
||||
help="Frontend port to bind to")
|
||||
|
||||
# Other settings
|
||||
parser.add_argument("--skip_backend_check",
|
||||
action="store_true",
|
||||
help="Skip backend health check")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Ensure output directory exists
|
||||
os.makedirs(args.output_path, exist_ok=True)
|
||||
|
||||
print("🎬 FastVideo Ray Serve App")
|
||||
print("=" * 50)
|
||||
print(f"T2V Models: {args.t2v_model_paths}")
|
||||
print(f"T2V Model Replicas: {args.t2v_model_replicas}")
|
||||
# print(f"I2V Model: {args.i2v_model_path}") # I2V functionality commented out
|
||||
print(f"Output: {args.output_path}")
|
||||
print(f"Backend: http://{args.backend_host}:{args.backend_port}")
|
||||
print(f"Frontend: http://{args.frontend_host}:{args.frontend_port}")
|
||||
print("=" * 50)
|
||||
|
||||
# Start backend
|
||||
backend_process = start_backend(args)
|
||||
|
||||
# Wait for backend to be ready
|
||||
backend_url = f"http://{args.backend_host}:{args.backend_port}"
|
||||
|
||||
if not args.skip_backend_check:
|
||||
if not check_backend_health(backend_url):
|
||||
print("❌ Backend failed to start. Terminating...")
|
||||
backend_process.terminate()
|
||||
sys.exit(1)
|
||||
|
||||
# Start frontend
|
||||
frontend_process = start_frontend(args)
|
||||
|
||||
print("\n🎉 Both services are starting up!")
|
||||
print(f"📺 Frontend will be available at: http://{args.frontend_host}:{args.frontend_port}")
|
||||
print(f"🔧 Backend API will be available at: {backend_url}")
|
||||
print("\nPress Ctrl+C to stop both services...")
|
||||
# return
|
||||
|
||||
# Signal handler for graceful shutdown
|
||||
def signal_handler(signum, frame):
|
||||
print("\n🛑 Shutting down services...")
|
||||
frontend_process.terminate()
|
||||
backend_process.terminate()
|
||||
|
||||
# Wait for processes to terminate
|
||||
try:
|
||||
frontend_process.wait(timeout=5)
|
||||
backend_process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
print("⚠️ Force killing processes...")
|
||||
frontend_process.kill()
|
||||
backend_process.kill()
|
||||
|
||||
print("✅ Services stopped")
|
||||
def setup_signal_handlers(service_manager: ServiceManager) -> None:
|
||||
def signal_handler(signum: int, frame: Any) -> None:
|
||||
service_manager.shutdown_services()
|
||||
sys.exit(0)
|
||||
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
|
||||
|
||||
def print_startup_info(args: argparse.Namespace) -> None:
|
||||
print("🎬 FastVideo Ray Serve App")
|
||||
print("=" * 50)
|
||||
print(f"T2V Models: {args.t2v_model_paths}")
|
||||
print(f"T2V Model Replicas: {args.t2v_model_replicas}")
|
||||
print(f"Output: {args.output_path}")
|
||||
print(f"Backend: http://{args.backend_host}:{args.backend_port}")
|
||||
print(f"Frontend: http://{args.frontend_host}:{args.frontend_port}")
|
||||
print("=" * 50)
|
||||
|
||||
|
||||
def parse_arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="FastVideo Ray Serve App")
|
||||
|
||||
parser.add_argument("--t2v_model_paths",
|
||||
type=str,
|
||||
default=DEFAULT_T2V_MODELS,
|
||||
help="Comma separated list of paths to the T2V model(s)")
|
||||
parser.add_argument("--t2v_model_replicas",
|
||||
type=str,
|
||||
default=DEFAULT_T2V_REPLICAS,
|
||||
help="Comma separated list of number of replicas for the T2V model(s)")
|
||||
parser.add_argument("--output_path",
|
||||
type=str,
|
||||
default=DEFAULT_OUTPUT_PATH,
|
||||
help="Path to save generated videos")
|
||||
|
||||
parser.add_argument("--backend_host",
|
||||
type=str,
|
||||
default=DEFAULT_BACKEND_HOST,
|
||||
help="Backend host to bind to")
|
||||
parser.add_argument("--backend_port",
|
||||
type=int,
|
||||
default=DEFAULT_BACKEND_PORT,
|
||||
help="Backend port to bind to")
|
||||
|
||||
parser.add_argument("--frontend_host",
|
||||
type=str,
|
||||
default=DEFAULT_FRONTEND_HOST,
|
||||
help="Frontend host to bind to")
|
||||
parser.add_argument("--frontend_port",
|
||||
type=int,
|
||||
default=DEFAULT_FRONTEND_PORT,
|
||||
help="Frontend port to bind to")
|
||||
|
||||
parser.add_argument("--skip_backend_check",
|
||||
action="store_true",
|
||||
help="Skip backend health check")
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_arguments()
|
||||
|
||||
os.makedirs(args.output_path, exist_ok=True)
|
||||
print_startup_info(args)
|
||||
|
||||
service_manager = ServiceManager(args)
|
||||
setup_signal_handlers(service_manager)
|
||||
|
||||
# Monitor processes
|
||||
try:
|
||||
while True:
|
||||
# Check if processes are still running
|
||||
if frontend_process.poll() is not None:
|
||||
print("❌ Frontend process died unexpectedly")
|
||||
break
|
||||
|
||||
if backend_process.poll() is not None:
|
||||
print("❌ Backend process died unexpectedly")
|
||||
break
|
||||
|
||||
time.sleep(1)
|
||||
service_manager.start_backend()
|
||||
|
||||
if not args.skip_backend_check:
|
||||
if not service_manager.check_backend_health():
|
||||
print("❌ Backend failed to start. Terminating...")
|
||||
service_manager.shutdown_services()
|
||||
sys.exit(1)
|
||||
|
||||
service_manager.start_frontend()
|
||||
|
||||
print("\n🎉 Both services are starting up!")
|
||||
print(f"📺 Frontend will be available at: http://{args.frontend_host}:{args.frontend_port}")
|
||||
print(f"🔧 Backend API will be available at: http://{args.backend_host}:{args.backend_port}")
|
||||
print("\nPress Ctrl+C to stop both services...")
|
||||
|
||||
service_manager.monitor_processes()
|
||||
|
||||
except KeyboardInterrupt:
|
||||
signal_handler(signal.SIGINT, None)
|
||||
except Exception as e:
|
||||
print(f"❌ Unexpected error: {e}")
|
||||
service_manager.shutdown_services()
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,465 +0,0 @@
|
||||
"""
|
||||
Startup script for FastVideo scalable architecture:
|
||||
ngrok -> nginx reverse proxy -> frontend1/frontend2 -> backend1×8/backend2×8
|
||||
|
||||
This script starts:
|
||||
1. Multiple backend instances (8 GPU replicas each)
|
||||
2. Multiple frontend instances (2 instances)
|
||||
3. Nginx reverse proxy
|
||||
4. Optional ngrok tunnel
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import threading
|
||||
import signal
|
||||
import requests
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
# Add the project root to the Python path
|
||||
project_root = Path(__file__).parent.parent.parent.parent
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
|
||||
def check_service_health(url: str, max_retries: int = 50) -> bool:
|
||||
"""Check if a service is healthy"""
|
||||
for i in range(max_retries):
|
||||
try:
|
||||
response = requests.get(url, timeout=5)
|
||||
if response.status_code == 200:
|
||||
print(f"✅ Service is healthy at {url}")
|
||||
return True
|
||||
except requests.exceptions.RequestException:
|
||||
pass
|
||||
|
||||
if i < max_retries - 1:
|
||||
print(f"⏳ Waiting for service to start... ({i+1}/{max_retries})")
|
||||
time.sleep(2)
|
||||
|
||||
print(f"❌ Service failed to start within {max_retries * 2} seconds")
|
||||
return False
|
||||
|
||||
|
||||
def start_backend_cluster(args, cluster_id: int):
|
||||
"""Start one backend cluster (Ray-Serve application)."""
|
||||
backend_script = Path(__file__).parent / "ray_serve_backend_scalable.py"
|
||||
|
||||
# All Ray Serve apps share the same HTTP server (default 8000).
|
||||
# We still forward the port flag for completeness, but keep it
|
||||
# identical for every cluster.
|
||||
base_port = args.backend_base_port
|
||||
|
||||
cmd = [
|
||||
sys.executable, str(backend_script),
|
||||
"--t2v_model_path", args.t2v_model_path,
|
||||
"--i2v_model_path", args.i2v_model_path,
|
||||
"--output_path", args.output_path,
|
||||
"--host", args.backend_host,
|
||||
"--port", str(base_port),
|
||||
"--num_gpus", str(args.num_gpus_per_cluster),
|
||||
"--cluster_id", str(cluster_id),
|
||||
]
|
||||
|
||||
print(f"🚀 Starting Backend Cluster {cluster_id + 1} (HTTP port {base_port})...")
|
||||
print(f"Command: {' '.join(cmd)}")
|
||||
|
||||
# Start the backend process
|
||||
backend_process = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True,
|
||||
bufsize=1
|
||||
)
|
||||
|
||||
# Monitor backend output
|
||||
def monitor_backend():
|
||||
for line in backend_process.stdout:
|
||||
print(f"[BACKEND-{cluster_id + 1}] {line.rstrip()}")
|
||||
|
||||
monitor_thread = threading.Thread(target=monitor_backend, daemon=True)
|
||||
monitor_thread.start()
|
||||
|
||||
return backend_process, base_port
|
||||
|
||||
|
||||
def start_frontend_instance(args, instance_id: int, backend_url: str):
|
||||
"""Start a single frontend instance"""
|
||||
frontend_script = Path(__file__).parent / "gradio_frontend.py"
|
||||
frontend_port = args.frontend_base_port + instance_id
|
||||
|
||||
# Update backend URL to include cluster-specific path
|
||||
cluster_id = instance_id % args.num_backend_clusters
|
||||
backend_url_with_cluster = f"{backend_url}/cluster_{cluster_id}"
|
||||
|
||||
cmd = [
|
||||
sys.executable, str(frontend_script),
|
||||
"--backend_url", backend_url_with_cluster,
|
||||
"--t2v_model_path", args.t2v_model_path,
|
||||
"--i2v_model_path", args.i2v_model_path,
|
||||
"--host", args.frontend_host,
|
||||
"--port", str(frontend_port)
|
||||
]
|
||||
|
||||
print(f"🎨 Starting Frontend {instance_id + 1} on port {frontend_port}...")
|
||||
print(f"Command: {' '.join(cmd)}")
|
||||
|
||||
# Start the frontend process
|
||||
frontend_process = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True,
|
||||
bufsize=1
|
||||
)
|
||||
|
||||
# Monitor frontend output
|
||||
def monitor_frontend():
|
||||
for line in frontend_process.stdout:
|
||||
print(f"[FRONTEND-{instance_id + 1}] {line.rstrip()}")
|
||||
|
||||
monitor_thread = threading.Thread(target=monitor_frontend, daemon=True)
|
||||
monitor_thread.start()
|
||||
|
||||
return frontend_process, frontend_port
|
||||
|
||||
|
||||
def start_nginx(args):
|
||||
"""Start nginx reverse proxy"""
|
||||
nginx_conf = Path(__file__).parent / "nginx.conf"
|
||||
|
||||
# Update nginx configuration with actual ports
|
||||
update_nginx_config(args)
|
||||
|
||||
cmd = [
|
||||
"nginx",
|
||||
"-c", str(nginx_conf),
|
||||
"-g", "daemon off;"
|
||||
]
|
||||
|
||||
print(f"🌐 Starting Nginx reverse proxy...")
|
||||
print(f"Command: {' '.join(cmd)}")
|
||||
|
||||
# Start nginx process
|
||||
nginx_process = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True,
|
||||
bufsize=1
|
||||
)
|
||||
|
||||
# Monitor nginx output
|
||||
def monitor_nginx():
|
||||
for line in nginx_process.stdout:
|
||||
print(f"[NGINX] {line.rstrip()}")
|
||||
|
||||
monitor_thread = threading.Thread(target=monitor_nginx, daemon=True)
|
||||
monitor_thread.start()
|
||||
|
||||
return nginx_process
|
||||
|
||||
|
||||
def update_nginx_config(args):
|
||||
"""Rewrite nginx.conf with the correct ports – NO “/cluster_X” in upstreams."""
|
||||
nginx_conf = Path(__file__).parent / "nginx.conf"
|
||||
nginx_conf_backup = Path(__file__).parent / "nginx.conf.backup"
|
||||
|
||||
if not nginx_conf_backup.exists():
|
||||
nginx_conf_backup.write_text(nginx_conf.read_text())
|
||||
|
||||
config_content = nginx_conf_backup.read_text()
|
||||
|
||||
# ── 1. front-end pool ───────────────────────────────────────────────
|
||||
frontend_servers = "\n ".join(
|
||||
f"server 127.0.0.1:{args.frontend_base_port + i} "
|
||||
f"weight=1 max_fails=3 fail_timeout=30s;"
|
||||
for i in range(args.num_frontends)
|
||||
)
|
||||
config_content = config_content.replace(
|
||||
"# Upstream for frontend load balancing",
|
||||
f"# Upstream for frontend load balancing\n upstream frontend_servers {{\n"
|
||||
f" # Round-robin load balancing between frontends\n {frontend_servers}"
|
||||
)
|
||||
|
||||
# Shared Ray-Serve HTTP port
|
||||
backend_port = args.backend_base_port # default 8000
|
||||
backend_line = (f"server 127.0.0.1:{backend_port} "
|
||||
f"weight=1 max_fails=3 fail_timeout=30s;")
|
||||
|
||||
# ── 2. backend-1 pool ───────────────────────────────────────────────
|
||||
config_content = config_content.replace(
|
||||
"# Upstream for backend1 load balancing",
|
||||
f"# Upstream for backend1 load balancing\n upstream backend1_servers {{\n"
|
||||
f" {backend_line}"
|
||||
)
|
||||
|
||||
# ── 3. backend-2 pool ───────────────────────────────────────────────
|
||||
config_content = config_content.replace(
|
||||
"# Upstream for backend2 load balancing",
|
||||
f"# Upstream for backend2 load balancing\n upstream backend2_servers {{\n"
|
||||
f" {backend_line}"
|
||||
)
|
||||
|
||||
# ── 4. strip any stray “/cluster_X” fragments ───────────────────────
|
||||
config_content = config_content.replace("/cluster_0", "").replace("/cluster_1", "")
|
||||
|
||||
# ── 5. use user-writable log directory ---------------------------------
|
||||
log_dir = Path(args.output_path).resolve()
|
||||
config_content = config_content.replace(
|
||||
"access_log /var/log/nginx/access.log;",
|
||||
f"access_log {log_dir}/nginx_access.log;")
|
||||
config_content = config_content.replace(
|
||||
"error_log /var/log/nginx/error.log;",
|
||||
f"error_log {log_dir}/nginx_error.log;")
|
||||
|
||||
nginx_conf.write_text(config_content)
|
||||
print("✅ nginx.conf updated (no path suffixes & custom log paths)")
|
||||
|
||||
|
||||
def start_ngrok(args):
|
||||
"""Start ngrok tunnel"""
|
||||
if not args.use_ngrok:
|
||||
return None
|
||||
|
||||
cmd = [
|
||||
"ngrok",
|
||||
"http",
|
||||
str(args.nginx_port),
|
||||
"--log=stdout"
|
||||
]
|
||||
|
||||
print(f"🌍 Starting ngrok tunnel to port {args.nginx_port}...")
|
||||
print(f"Command: {' '.join(cmd)}")
|
||||
|
||||
# Start ngrok process
|
||||
ngrok_process = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True,
|
||||
bufsize=1
|
||||
)
|
||||
|
||||
# Monitor ngrok output
|
||||
def monitor_ngrok():
|
||||
for line in ngrok_process.stdout:
|
||||
print(f"[NGROK] {line.rstrip()}")
|
||||
|
||||
monitor_thread = threading.Thread(target=monitor_ngrok, daemon=True)
|
||||
monitor_thread.start()
|
||||
|
||||
return ngrok_process
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="FastVideo Scalable Architecture Launcher")
|
||||
|
||||
# Model and output settings
|
||||
parser.add_argument("--t2v_model_path",
|
||||
type=str,
|
||||
default="FastVideo/FastWan2.1-T2V-1.3B-Diffusers",
|
||||
help="Path to the T2V model")
|
||||
parser.add_argument("--i2v_model_path",
|
||||
type=str,
|
||||
default="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
|
||||
help="Path to the I2V model")
|
||||
parser.add_argument("--output_path",
|
||||
type=str,
|
||||
default="outputs",
|
||||
help="Path to save generated videos")
|
||||
|
||||
# Backend settings
|
||||
parser.add_argument("--backend_host",
|
||||
type=str,
|
||||
default="0.0.0.0",
|
||||
help="Backend host to bind to")
|
||||
parser.add_argument("--backend_base_port",
|
||||
type=int,
|
||||
default=8000,
|
||||
help="Base port for backend clusters")
|
||||
parser.add_argument("--num_backend_clusters",
|
||||
type=int,
|
||||
default=2,
|
||||
help="Number of backend clusters")
|
||||
parser.add_argument("--num_gpus_per_cluster",
|
||||
type=int,
|
||||
default=3, # Changed from 8 to 3 (3+3=6 GPUs total, leaving 1 GPU buffer)
|
||||
help="Number of GPUs per backend cluster")
|
||||
|
||||
# Frontend settings
|
||||
parser.add_argument("--frontend_host",
|
||||
type=str,
|
||||
default="0.0.0.0",
|
||||
help="Frontend host to bind to")
|
||||
parser.add_argument("--frontend_base_port",
|
||||
type=int,
|
||||
default=7860,
|
||||
help="Base port for frontend instances")
|
||||
parser.add_argument("--num_frontends",
|
||||
type=int,
|
||||
default=2,
|
||||
help="Number of frontend instances")
|
||||
|
||||
# Nginx settings
|
||||
parser.add_argument("--nginx_port",
|
||||
type=int,
|
||||
default=80,
|
||||
help="Port for nginx reverse proxy")
|
||||
|
||||
# Ngrok settings
|
||||
parser.add_argument("--use_ngrok",
|
||||
action="store_true",
|
||||
help="Start ngrok tunnel")
|
||||
|
||||
# Other settings
|
||||
parser.add_argument("--skip_health_check",
|
||||
action="store_true",
|
||||
help="Skip health checks")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Ensure output directory exists
|
||||
os.makedirs(args.output_path, exist_ok=True)
|
||||
|
||||
print(" FastVideo Scalable Architecture")
|
||||
print("=" * 60)
|
||||
print(f"Architecture: ngrok -> nginx -> frontend1/frontend2 -> backend1×{args.num_gpus_per_cluster}/backend2×{args.num_gpus_per_cluster}")
|
||||
print(f"T2V Model: {args.t2v_model_path}")
|
||||
print(f"I2V Model: {args.i2v_model_path}")
|
||||
print(f"Output: {args.output_path}")
|
||||
print(f"Backend Clusters: {args.num_backend_clusters}")
|
||||
print(f"GPUs per Cluster: {args.num_gpus_per_cluster}")
|
||||
print(f"Total GPUs needed: {args.num_backend_clusters * args.num_gpus_per_cluster}")
|
||||
print(f"Frontend Instances: {args.num_frontends}")
|
||||
print(f"Nginx Port: {args.nginx_port}")
|
||||
print(f"Use Ngrok: {args.use_ngrok}")
|
||||
print("=" * 60)
|
||||
|
||||
# Start backend clusters
|
||||
backend_processes = []
|
||||
backend_urls = []
|
||||
|
||||
for i in range(args.num_backend_clusters):
|
||||
process, _ = start_backend_cluster(args, i)
|
||||
backend_processes.append(process)
|
||||
backend_urls.append(f"http://{args.backend_host}:{args.backend_base_port}")
|
||||
|
||||
# Wait for backends to be ready
|
||||
if not args.skip_health_check:
|
||||
print("\n⏳ Waiting for backend clusters to start...")
|
||||
for i, url in enumerate(backend_urls):
|
||||
if not check_service_health(f"{url}/cluster_{i}/health"):
|
||||
print(f"❌ Backend cluster {i + 1} failed to start. Terminating...")
|
||||
for process in backend_processes:
|
||||
process.terminate()
|
||||
sys.exit(1)
|
||||
|
||||
# Start frontend instances
|
||||
frontend_processes = []
|
||||
frontend_urls = []
|
||||
|
||||
for i in range(args.num_frontends):
|
||||
# Each frontend connects to a different backend cluster
|
||||
backend_url = backend_urls[i % len(backend_urls)]
|
||||
process, port = start_frontend_instance(args, i, backend_url)
|
||||
frontend_processes.append(process)
|
||||
frontend_urls.append(f"http://{args.frontend_host}:{port}")
|
||||
|
||||
# Wait for frontends to be ready
|
||||
if not args.skip_health_check:
|
||||
print("\n⏳ Waiting for frontend instances to start...")
|
||||
for i, url in enumerate(frontend_urls):
|
||||
if not check_service_health(url):
|
||||
print(f"❌ Frontend {i + 1} failed to start. Terminating...")
|
||||
for process in backend_processes + frontend_processes:
|
||||
process.terminate()
|
||||
sys.exit(1)
|
||||
|
||||
# Start nginx reverse proxy
|
||||
nginx_process = start_nginx(args)
|
||||
|
||||
# Wait for nginx to be ready
|
||||
if not args.skip_health_check:
|
||||
print("\n⏳ Waiting for nginx to start...")
|
||||
if not check_service_health(f"http://localhost:{args.nginx_port}/health"):
|
||||
print("❌ Nginx failed to start. Terminating...")
|
||||
for process in backend_processes + frontend_processes + [nginx_process]:
|
||||
process.terminate()
|
||||
sys.exit(1)
|
||||
|
||||
# Start ngrok tunnel (optional)
|
||||
ngrok_process = start_ngrok(args)
|
||||
|
||||
print("\n🎉 All services are starting up!")
|
||||
print(f"🌐 Nginx reverse proxy: http://localhost:{args.nginx_port}")
|
||||
for i, url in enumerate(frontend_urls):
|
||||
print(f"📺 Frontend {i + 1}: {url}")
|
||||
for i, url in enumerate(backend_urls):
|
||||
print(f" Backend Cluster {i + 1}: {url}")
|
||||
if args.use_ngrok:
|
||||
print("🌍 Ngrok tunnel is starting...")
|
||||
print("\nPress Ctrl+C to stop all services...")
|
||||
|
||||
# Signal handler for graceful shutdown
|
||||
def signal_handler(signum, frame):
|
||||
print("\n🛑 Shutting down all services...")
|
||||
all_processes = backend_processes + frontend_processes + [nginx_process]
|
||||
if ngrok_process:
|
||||
all_processes.append(ngrok_process)
|
||||
|
||||
for process in all_processes:
|
||||
if process:
|
||||
process.terminate()
|
||||
|
||||
# Wait for processes to terminate
|
||||
try:
|
||||
for process in all_processes:
|
||||
if process:
|
||||
process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
print("⚠️ Force killing processes...")
|
||||
for process in all_processes:
|
||||
if process:
|
||||
process.kill()
|
||||
|
||||
print("✅ All services stopped")
|
||||
sys.exit(0)
|
||||
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
|
||||
# Monitor processes
|
||||
try:
|
||||
while True:
|
||||
# Check if processes are still running
|
||||
for i, process in enumerate(backend_processes):
|
||||
if process.poll() is not None:
|
||||
print(f"❌ Backend cluster {i + 1} process died unexpectedly")
|
||||
break
|
||||
|
||||
for i, process in enumerate(frontend_processes):
|
||||
if process.poll() is not None:
|
||||
print(f"❌ Frontend {i + 1} process died unexpectedly")
|
||||
break
|
||||
|
||||
if nginx_process and nginx_process.poll() is not None:
|
||||
print("❌ Nginx process died unexpectedly")
|
||||
break
|
||||
|
||||
if ngrok_process and ngrok_process.poll() is not None:
|
||||
print("❌ Ngrok process died unexpectedly")
|
||||
break
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
signal_handler(signal.SIGINT, None)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,147 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test script for T2V and I2V functionality in FastVideo Gradio app.
|
||||
This script tests both the backend and frontend modifications.
|
||||
"""
|
||||
|
||||
import requests
|
||||
import json
|
||||
import os
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
|
||||
def test_backend_t2v():
|
||||
"""Test the backend T2V functionality directly"""
|
||||
backend_url = "http://localhost:8000"
|
||||
|
||||
try:
|
||||
# Test T2V request data
|
||||
request_data = {
|
||||
"prompt": "A beautiful sunset over the ocean with gentle waves",
|
||||
"negative_prompt": "",
|
||||
"use_negative_prompt": False,
|
||||
"seed": 42,
|
||||
"guidance_scale": 7.5,
|
||||
"num_frames": 21,
|
||||
"height": 448,
|
||||
"width": 832,
|
||||
"num_inference_steps": 20,
|
||||
"randomize_seed": False,
|
||||
"return_frames": True,
|
||||
"image_path": None,
|
||||
"model_type": "t2v"
|
||||
}
|
||||
|
||||
# Send request to backend
|
||||
response = requests.post(
|
||||
f"{backend_url}/generate_video",
|
||||
json=request_data,
|
||||
timeout=300 # 5 minutes timeout
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
print("✅ Backend T2V test successful!")
|
||||
print(f"Success: {result.get('success')}")
|
||||
print(f"Seed used: {result.get('seed')}")
|
||||
if result.get('frames'):
|
||||
print(f"Frames returned: {len(result.get('frames'))}")
|
||||
else:
|
||||
print("No frames returned")
|
||||
else:
|
||||
print(f"❌ Backend T2V test failed with status {response.status_code}")
|
||||
print(f"Response: {response.text}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Backend T2V test failed with exception: {e}")
|
||||
|
||||
def test_backend_i2v():
|
||||
"""Test the backend I2V functionality directly"""
|
||||
backend_url = "http://localhost:8000"
|
||||
|
||||
# Create a simple test image
|
||||
test_image = Image.new('RGB', (256, 256), color='red')
|
||||
temp_image_path = "test_image.png"
|
||||
test_image.save(temp_image_path)
|
||||
|
||||
try:
|
||||
# Test I2V request data
|
||||
request_data = {
|
||||
"prompt": "The red square gently animates with subtle movement",
|
||||
"negative_prompt": "",
|
||||
"use_negative_prompt": False,
|
||||
"seed": 42,
|
||||
"guidance_scale": 7.5,
|
||||
"num_frames": 21,
|
||||
"height": 448,
|
||||
"width": 832,
|
||||
"num_inference_steps": 20,
|
||||
"randomize_seed": False,
|
||||
"return_frames": True,
|
||||
"image_path": temp_image_path,
|
||||
"model_type": "i2v"
|
||||
}
|
||||
|
||||
# Send request to backend
|
||||
response = requests.post(
|
||||
f"{backend_url}/generate_video",
|
||||
json=request_data,
|
||||
timeout=300 # 5 minutes timeout
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
print("✅ Backend I2V test successful!")
|
||||
print(f"Success: {result.get('success')}")
|
||||
print(f"Seed used: {result.get('seed')}")
|
||||
if result.get('frames'):
|
||||
print(f"Frames returned: {len(result.get('frames'))}")
|
||||
else:
|
||||
print("No frames returned")
|
||||
else:
|
||||
print(f"❌ Backend I2V test failed with status {response.status_code}")
|
||||
print(f"Response: {response.text}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Backend I2V test failed with exception: {e}")
|
||||
|
||||
finally:
|
||||
# Clean up test image
|
||||
if os.path.exists(temp_image_path):
|
||||
os.remove(temp_image_path)
|
||||
|
||||
def test_backend_health():
|
||||
"""Test if the backend is running"""
|
||||
backend_url = "http://localhost:8000"
|
||||
|
||||
try:
|
||||
response = requests.get(f"{backend_url}/health", timeout=5)
|
||||
if response.status_code == 200:
|
||||
print("✅ Backend is healthy")
|
||||
return True
|
||||
else:
|
||||
print(f"❌ Backend health check failed: {response.status_code}")
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f"❌ Backend health check failed: {e}")
|
||||
return False
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("🧪 Testing FastVideo T2V and I2V functionality...")
|
||||
print("=" * 50)
|
||||
|
||||
# Test backend health first
|
||||
if test_backend_health():
|
||||
# Test T2V functionality
|
||||
print("\n📝 Testing T2V functionality...")
|
||||
test_backend_t2v()
|
||||
|
||||
# Test I2V functionality
|
||||
print("\n🖼️ Testing I2V functionality...")
|
||||
test_backend_i2v()
|
||||
else:
|
||||
print("⚠️ Backend is not running. Please start the backend first.")
|
||||
print("You can start it with: python start_ray_serve_app.py")
|
||||
|
||||
print("=" * 50)
|
||||
print("Test completed!")
|
||||
Reference in New Issue
Block a user