diff --git a/examples/inference/gradio/gradio_frontend.py b/examples/inference/gradio/gradio_frontend.py index 42300af58..6c268ece5 100644 --- a/examples/inference/gradio/gradio_frontend.py +++ b/examples/inference/gradio/gradio_frontend.py @@ -93,7 +93,7 @@ def save_video_from_base64(video_data: str, output_dir: str, prompt: str) -> str return f"Failed to save video: {str(e)}", "" -def create_gradio_interface(backend_url: str, default_params: SamplingParam): +def create_gradio_interface(backend_url: str, default_params: dict[str, SamplingParam]): """Create the Gradio interface""" # Initialize the Ray Serve client @@ -185,11 +185,12 @@ def create_gradio_interface(backend_url: str, default_params: SamplingParam): "num_frames": num_frames, "height": height, "width": width, - "num_inference_steps": 20, # Use default value + # "num_inference_steps": 20, # Use default value "randomize_seed": randomize_seed, "return_frames": False, # We'll get video data directly "image_path": None, # For T2V, we pass None as input_image - "model_type": "i2v" if "I2V" in model_selection or "Image-to-Video" in model_selection else "t2v", # Use model type selection + # "model_type": "i2v" if "I2V" in model_selection or "Image-to-Video" in model_selection else "t2v", # Use model type selection + "model_type": model_selection.split(' ')[0], "model_path": model_selection.split(" (")[0] if model_selection else None # Extract model path from selection } @@ -367,6 +368,32 @@ def create_gradio_interface(backend_url: str, default_params: SamplingParam): checkbox_background_color_selected="#2563eb", # Blue checkbox when selected ) + def get_default_values_for_model(model_selection_value): + """Get default parameter values for the specified model""" + model_path = model_selection_value.split(" (")[0] if model_selection_value else None + if model_path and model_path in default_params: + params = default_params[model_path] + return { + 'height': params.height, + 'width': params.width, + 'num_frames': params.num_frames, + 'guidance_scale': params.guidance_scale, + 'seed': params.seed, + } + else: + # Fallback defaults if model not found + return { + 'height': 448, + 'width': 832, + 'num_frames': 61, + 'guidance_scale': 3.0, + 'seed': 1024, + } + + # Get initial values for the default model + default_model = "FastVideo/FastWan2.1-T2V-1.3B-Diffusers (Text-to-Video)" + initial_values = get_default_values_for_model(default_model) + # Create Gradio interface with gr.Blocks(title="FastWan", theme=theme) as demo: @@ -408,6 +435,8 @@ def create_gradio_interface(backend_url: str, default_params: SamplingParam): choices=[ "FastVideo/FastWan2.1-T2V-1.3B-Diffusers (Text-to-Video)", "FastVideo/FastWan2.1-T2V-14B-Diffusers (Text-to-Video)", + "Wan-AI/Wan2.1-T2V-1.3B-Diffusers (Text-to-Video)", + "Wan-AI/Wan2.1-T2V-14B-480P-Diffusers (Text-to-Video)", # "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers (Image-to-Video)" # I2V functionality commented out ], value="FastVideo/FastWan2.1-T2V-1.3B-Diffusers (Text-to-Video)", @@ -452,14 +481,14 @@ def create_gradio_interface(backend_url: str, default_params: SamplingParam): minimum=256, maximum=1280, step=32, - value=448, + value=initial_values['height'], ) width = gr.Slider( label="Width", minimum=256, maximum=1280, step=32, - value=832 + value=initial_values['width'] ) with gr.Row(): @@ -468,13 +497,13 @@ def create_gradio_interface(backend_url: str, default_params: SamplingParam): minimum=16, maximum=121, step=16, - value=61, + value=initial_values['num_frames'], ) guidance_scale = gr.Slider( label="Guidance Scale", minimum=1, maximum=12, - value=3.0, + value=initial_values['guidance_scale'], ) with gr.Row(): @@ -493,7 +522,7 @@ def create_gradio_interface(backend_url: str, default_params: SamplingParam): minimum=0, maximum=1000000, step=1, - value=1024 + value=initial_values['seed'], ) randomize_seed = gr.Checkbox(label="Randomize seed", value=False) seed_output = gr.Number(label="Used Seed") @@ -694,6 +723,35 @@ def create_gradio_interface(backend_url: str, default_params: SamplingParam): # outputs=[input_image, prompt], # ) + def on_model_selection_change(selected_model): + """Update advanced options based on selected model's default parameters""" + if not selected_model: + return {}, {}, {}, {}, {} # Return empty updates if no model selected + + # Extract model path from selection (remove the description part) + model_path = selected_model.split(" ")[0] if selected_model else None + + if model_path and model_path in default_params: + params = default_params[model_path] + + # Update each component with the model's default values + return ( + gr.update(value=params.height), # height + gr.update(value=params.width), # width + gr.update(value=params.num_frames), # num_frames + gr.update(value=params.guidance_scale), # guidance_scale + gr.update(value=params.seed), # seed + ) + else: + # If model not found in default_params, return current values (no change) + return {}, {}, {}, {}, {} + + model_selection.change( + fn=on_model_selection_change, + inputs=model_selection, + outputs=[height, width, num_frames, guidance_scale, seed], + ) + def handle_generation(*args, progress=None, request: gr.Request = None): # Extract model selection and input image from args - I2V functionality commented out model_selection, prompt, negative_prompt, use_negative_prompt, seed, guidance_scale, num_frames, height, width, randomize_seed = args @@ -760,14 +818,13 @@ def main(): type=str, default="http://localhost:8000", help="URL of the Ray Serve backend") - parser.add_argument("--t2v_model_path", + parser.add_argument("--t2v_model_paths", type=str, - default="FastVideo/FastWan2.1-T2V-1.3B-Diffusers", - help="Path to the T2V model (for default parameters)") - parser.add_argument("--t2v_14b_model_path", - type=str, - default="FastVideo/FastWan2.1-T2V-14B-Diffusers", - help="Path to the T2V 14B model (for default parameters)") + default="FastVideo/FastWan2.1-T2V-1.3B-Diffusers,FastVideo/FastWan2.1-T2V-14B-Diffusers", + help="Comma separated list of paths to the T2V model(s)") + # parser.add_argument("--t2v_model_replicas", type=int, + # default="4,4", + # help="Comma separated list of number of replicas for the T2V model(s)") # parser.add_argument("--i2v_model_path", # I2V functionality commented out # type=str, # default="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers", @@ -785,7 +842,10 @@ def main(): # Load default parameters from the models # try: - default_params = SamplingParam.from_pretrained(args.t2v_model_path) + default_params = {} + model_paths = args.t2v_model_paths.split(",") + for model_path in model_paths: + default_params[model_path] = SamplingParam.from_pretrained(model_path) # except Exception as e: # print(f"Warning: Could not load default parameters from {args.t2v_model_path}: {e}") # print("Using fallback default parameters...") @@ -803,8 +863,8 @@ def main(): print(f"Starting Gradio frontend at http://{args.host}:{args.port}") print(f"Backend URL: {args.backend_url}") - print(f"T2V 1.3B Model: {args.t2v_model_path}") - print(f"T2V 14B Model: {args.t2v_14b_model_path}") + print(f"T2V Models: {args.t2v_model_paths}") + # print(f"T2V Model Replicas: {args.t2v_model_replicas}") # print(f"I2V Model: {args.i2v_model_path}") # I2V functionality commented out demo.queue(max_size=20).launch( diff --git a/examples/inference/gradio/ray_serve_backend.py b/examples/inference/gradio/ray_serve_backend.py index 0794c0f7a..6ae483fe3 100644 --- a/examples/inference/gradio/ray_serve_backend.py +++ b/examples/inference/gradio/ray_serve_backend.py @@ -18,6 +18,14 @@ from slowapi.errors import RateLimitExceeded import imageio from ray.serve.handle import DeploymentHandle +NUM_GPUS = 8 +SUPPORTED_MODELS = [ + "FastVideo/FastWan2.1-T2V-1.3B-Diffusers", + "FastVideo/FastWan2.1-T2V-14B-Diffusers", + "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", + "Wan-AI/Wan2.1-T2V-14B-Diffusers", +] + class VideoGenerationRequest(BaseModel): prompt: str @@ -28,7 +36,7 @@ class VideoGenerationRequest(BaseModel): num_frames: int = 21 height: int = 448 width: int = 832 - num_inference_steps: int = 20 + # num_inference_steps: int = 20 randomize_seed: bool = False return_frames: bool = False # Whether to return base64 encoded frames image_path: Optional[str] = None # Path to input image for I2V @@ -96,10 +104,13 @@ class T2VModelDeployment: os.makedirs(self.output_path, exist_ok=True) # Delay helps avoid GPU contention when many replicas start at once - time.sleep(10) + time.sleep(5) # Ensure correct attention backend for FastVideo - os.environ["FASTVIDEO_ATTENTION_BACKEND"] = "VIDEO_SPARSE_ATTN" + if "FastVideo" in self.model_path: + os.environ["FASTVIDEO_ATTENTION_BACKEND"] = "VIDEO_SPARSE_ATTN" + else: + os.environ["FASTVIDEO_ATTENTION_BACKEND"] = "FLASH_ATTN" os.environ["FASTVIDEO_STAGE_LOGGING"] = "1" # Lazy import to keep the deployment import-safe on head node @@ -139,7 +150,7 @@ class T2VModelDeployment: params.num_frames = video_request.num_frames params.height = video_request.height params.width = video_request.width - params.num_inference_steps = video_request.num_inference_steps + # params.num_inference_steps = video_request.num_inference_steps # Do not write to disk when called via API params.save_video = False @@ -175,7 +186,7 @@ class T2VModelDeployment: encoding_start_time = time.time() # Encode outputs - video_data = encode_video_to_base64(frames, fps=24) + video_data = encode_video_to_base64(frames, fps=16) encoding_end_time = time.time() encoding_time = encoding_end_time - encoding_start_time @@ -348,7 +359,7 @@ class T2V14BModelDeployment: params.num_frames = video_request.num_frames params.height = video_request.height params.width = video_request.width - params.num_inference_steps = video_request.num_inference_steps + # params.num_inference_steps = video_request.num_inference_steps # Do not write to disk when called via API params.save_video = False @@ -382,7 +393,7 @@ class T2V14BModelDeployment: encoding_start_time = time.time() # Encode outputs - video_data = encode_video_to_base64(frames, fps=24) + video_data = encode_video_to_base64(frames, fps=16) encoding_end_time = time.time() encoding_time = encoding_end_time - encoding_start_time @@ -417,9 +428,9 @@ app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler) class FastVideoAPI: """Ingress deployment that routes requests to either the T2V or I2V model deployment.""" - def __init__(self, t2v_deployment: DeploymentHandle, t2v_14b_deployment: DeploymentHandle): # Removed i2v_deployment - self.t2v_handle = t2v_deployment - self.t2v_14b_handle = t2v_14b_deployment + def __init__(self, t2v_deployments: Dict[str, DeploymentHandle]): # Removed i2v_deployment + self.t2v_deployments = t2v_deployments + # self.t2v_14b_handle = t2v_14b_deployment # self.i2v_handle = i2v_deployment # I2V functionality commented out @app.post("/generate_video", response_model=VideoGenerationResponse) @@ -427,18 +438,22 @@ class FastVideoAPI: async def generate_video(self, request: Request, video_request: VideoGenerationRequest) -> VideoGenerationResponse: """Route the request to the appropriate model deployment based on `model_type` and `model_path`.""" try: + # assert video_request.model_type in self.t2v_deployments, f"Model {video_request.model_type} not found" + if video_request.model_type not in self.t2v_deployments: + raise ValueError(f"Model {video_request.model_type} not found") + response_ref = self.t2v_deployments[video_request.model_type].generate_video.remote(video_request) # if video_request.model_type.lower() == "i2v": # I2V functionality commented out # response_ref = self.i2v_handle.generate_video.remote(video_request) - if video_request.model_type.lower() == "t2v": - # Route T2V requests based on model path - if video_request.model_path and "14b" in video_request.model_path.lower(): - response_ref = self.t2v_14b_handle.generate_video.remote(video_request) - else: - # Default to 1.3B model - response_ref = self.t2v_handle.generate_video.remote(video_request) - else: - # Default to 1.3B T2V model - response_ref = self.t2v_handle.generate_video.remote(video_request) + # if video_request.model_type.lower() == "t2v": + # # Route T2V requests based on model path + # if video_request.model_path and "14b" in video_request.model_path.lower(): + # response_ref = self.t2v_14b_handle.generate_video.remote(video_request) + # else: + # # Default to 1.3B model + # response_ref = self.t2v_handle.generate_video.remote(video_request) + # else: + # # Default to 1.3B T2V model + # response_ref = self.t2v_handle.generate_video.remote(video_request) # Await the remote response response = await response_ref @@ -463,14 +478,15 @@ class FastVideoAPI: def start_ray_serve( *, - t2v_model_path: str = "FastVideo/FastWan2.1-T2V-1.3B-Diffusers", - t2v_14b_model_path: str = "FastVideo/FastWan2.1-T2V-14B-Diffusers", + t2v_model_paths: str, + t2v_model_replicas: str, + # t2v_14b_model_path: str = "FastVideo/FastWan2.1-T2V-14B-Diffusers", # i2v_model_path: str = "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers", # I2V functionality commented out output_path: str = "outputs", host: str = "0.0.0.0", port: int = 8000, - t2v_replicas: int = 4, - t2v_14b_replicas: int = 4, # Reduced replicas for 14B model due to higher resource requirements + # t2v_replicas: int = 4, + # t2v_14b_replicas: int = 4, # Reduced replicas for 14B model due to higher resource requirements # i2v_replicas: int = 4, # I2V functionality commented out ): """Start Ray Serve with independently scalable deployments for each model.""" @@ -479,18 +495,21 @@ def start_ray_serve( ray.init() # Bind model deployments with configurable replica counts - t2v_dep = T2VModelDeployment.options(num_replicas=t2v_replicas).bind(t2v_model_path, output_path) - t2v_14b_dep = T2V14BModelDeployment.options(num_replicas=t2v_14b_replicas).bind(t2v_14b_model_path, output_path) + t2v_deps = {} + for model_path, replicas in zip(t2v_model_paths.split(","), t2v_model_replicas.split(",")): + t2v_dep = T2VModelDeployment.options(num_replicas=int(replicas)).bind(model_path, output_path) + t2v_deps[model_path] = t2v_dep # i2v_dep = I2VModelDeployment.options(num_replicas=i2v_replicas).bind(i2v_model_path, output_path) # I2V functionality commented out # Ingress - api = FastVideoAPI.bind(t2v_dep, t2v_14b_dep) # Removed i2v_dep + api = FastVideoAPI.bind(t2v_deps) # Removed i2v_dep serve.run(api, route_prefix="/", name="fast_video") print(f"Ray Serve backend started at http://{host}:{port}") - print(f"T2V 1.3B Model: {t2v_model_path} | Replicas: {t2v_replicas}") - print(f"T2V 14B Model: {t2v_14b_model_path} | Replicas: {t2v_14b_replicas}") + for model_path, replicas in zip(t2v_model_paths.split(","), t2v_model_replicas.split(",")): + print(f"T2V Model: {model_path} | Replicas: {replicas}") + # print(f"T2V 14B Model: {t2v_14b_model_path} | Replicas: {t2v_14b_replicas}") # print(f"I2V Model: {i2v_model_path} | Replicas: {i2v_replicas}") # I2V functionality commented out print(f"Health check: http://{host}:{port}/health") print(f"Video generation endpoint: http://{host}:{port}/generate_video") @@ -500,14 +519,18 @@ if __name__ == "__main__": import argparse parser = argparse.ArgumentParser(description="FastVideo Ray Serve Backend") - parser.add_argument("--t2v_model_path", + parser.add_argument("--t2v_model_paths", type=str, - default="FastVideo/FastWan2.1-T2V-1.3B-Diffusers", - help="Path to the T2V model") - parser.add_argument("--t2v_14b_model_path", + default="FastVideo/FastWan2.1-T2V-1.3B-Diffusers,FastVideo/FastWan2.1-T2V-14B-Diffusers", + help="Comma separated list of paths to the T2V model(s)") + parser.add_argument("--t2v_model_replicas", type=str, - default="FastVideo/FastWan2.1-T2V-14B-Diffusers", - help="Path to the T2V 14B model") + default="4,4", + help="Comma separated list of number of replicas for the T2V model(s)") + # parser.add_argument("--t2v_14b_model_path", + # type=str, + # default="FastVideo/FastWan2.1-T2V-14B-Diffusers", + # help="Path to the T2V 14B model") # parser.add_argument("--i2v_model_path", # I2V functionality commented out # type=str, # default="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers", @@ -524,30 +547,40 @@ if __name__ == "__main__": type=int, default=8000, help="Port to bind to") - parser.add_argument("--t2v_replicas", - type=int, - default=4, - help="Number of replicas for the T2V model deployment") - parser.add_argument("--t2v_14b_replicas", - type=int, - default=4, # Reduced default for 14B model due to higher resource requirements - help="Number of replicas for the T2V 14B model deployment") + # parser.add_argument("--t2v_replicas", + # type=int, + # default=4, + # help="Number of replicas for the T2V model deployment") + # parser.add_argument("--t2v_14b_replicas", + # type=int, + # default=4, # Reduced default for 14B model due to higher resource requirements + # help="Number of replicas for the T2V 14B model deployment") # parser.add_argument("--i2v_replicas", # I2V functionality commented out # type=int, # default=4, # Reduced default for 14B model due to higher resource requirements # help="Number of replicas for the I2V model deployment") args = parser.parse_args() + + split_models = args.t2v_model_paths.split(",") + split_replicas = [int(replica) for replica in args.t2v_model_replicas.split(",")] + assert len(split_models) == len(split_replicas), "Number of models and replicas must match" + assert sum(split_replicas) <= NUM_GPUS, "Total number of replicas must be less than or equal to 16" + for model, replicas in zip(split_models, split_replicas): + assert model in SUPPORTED_MODELS, f"Model {model} not supported" + assert replicas > 0, f"Replicas must be greater than 0" + start_ray_serve( - t2v_model_path=args.t2v_model_path, - t2v_14b_model_path=args.t2v_14b_model_path, + t2v_model_paths=args.t2v_model_paths, + t2v_model_replicas=args.t2v_model_replicas, + # t2v_14b_model_path=args.t2v_14b_model_path, # i2v_model_path=args.i2v_model_path, # I2V functionality commented out output_path=args.output_path, host=args.host, port=args.port, - t2v_replicas=args.t2v_replicas, - t2v_14b_replicas=args.t2v_14b_replicas, + # t2v_replicas=args.t2v_replicas, + # t2v_14b_replicas=args.t2v_14b_replicas, # i2v_replicas=args.i2v_replicas, # I2V functionality commented out ) diff --git a/examples/inference/gradio/start_ray_serve_app.py b/examples/inference/gradio/start_ray_serve_app.py index 6d7e61d80..e3f8091c4 100644 --- a/examples/inference/gradio/start_ray_serve_app.py +++ b/examples/inference/gradio/start_ray_serve_app.py @@ -45,8 +45,8 @@ def start_backend(args): cmd = [ sys.executable, str(backend_script), - "--t2v_model_path", args.t2v_model_path, - "--t2v_14b_model_path", args.t2v_14b_model_path, + "--t2v_model_paths", args.t2v_model_paths, + "--t2v_model_replicas", args.t2v_model_replicas, # "--i2v_model_path", args.i2v_model_path, # I2V functionality commented out "--output_path", args.output_path, "--host", args.backend_host, @@ -84,8 +84,8 @@ def start_frontend(args): cmd = [ sys.executable, str(frontend_script), "--backend_url", backend_url, - "--t2v_model_path", args.t2v_model_path, - "--t2v_14b_model_path", args.t2v_14b_model_path, + "--t2v_model_paths", args.t2v_model_paths, + # "--t2v_model_replicas", args.t2v_model_replicas, # "--i2v_model_path", args.i2v_model_path, # I2V functionality commented out "--host", args.frontend_host, "--port", str(args.frontend_port) @@ -118,14 +118,18 @@ def main(): parser = argparse.ArgumentParser(description="FastVideo Ray Serve App") # Model and output settings - parser.add_argument("--t2v_model_path", + parser.add_argument("--t2v_model_paths", type=str, - default="FastVideo/FastWan2.1-T2V-1.3B-Diffusers", - help="Path to the T2V model") - parser.add_argument("--t2v_14b_model_path", + default="FastVideo/FastWan2.1-T2V-1.3B-Diffusers,FastVideo/FastWan2.1-T2V-14B-Diffusers", + help="Comma separated list of paths to the T2V model(s)") + parser.add_argument("--t2v_model_replicas", type=str, - default="FastVideo/FastWan2.1-T2V-14B-Diffusers", - help="Path to the T2V 14B model") + default="4,4", + help="Comma separated list of number of replicas for the T2V model(s)") + # parser.add_argument("--t2v_14b_model_path", + # type=str, + # default="FastVideo/FastWan2.1-T2V-14B-Diffusers", + # help="Path to the T2V 14B model") # parser.add_argument("--i2v_model_path", # I2V functionality commented out # type=str, # default="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers", @@ -167,8 +171,8 @@ def main(): print("🎬 FastVideo Ray Serve App") print("=" * 50) - print(f"T2V 1.3B Model: {args.t2v_model_path}") - print(f"T2V 14B Model: {args.t2v_14b_model_path}") + print(f"T2V Models: {args.t2v_model_paths}") + print(f"T2V Model Replicas: {args.t2v_model_replicas}") # print(f"I2V Model: {args.i2v_model_path}") # I2V functionality commented out print(f"Output: {args.output_path}") print(f"Backend: http://{args.backend_host}:{args.backend_port}")