update start

examples/inference/gradio/gradio_frontend.py

@@ -93,7 +93,7 @@ def save_video_from_base64(video_data: str, output_dir: str, prompt: str) -> str
         return f"Failed to save video: {str(e)}", ""
 
 
-def create_gradio_interface(backend_url: str, default_params: SamplingParam):
+def create_gradio_interface(backend_url: str, default_params: dict[str, SamplingParam]):
     """Create the Gradio interface"""
     
     # Initialize the Ray Serve client
@@ -185,11 +185,12 @@ def create_gradio_interface(backend_url: str, default_params: SamplingParam):
             "num_frames": num_frames,
             "height": height,
             "width": width,
-            "num_inference_steps": 20,  # Use default value
+            # "num_inference_steps": 20,  # Use default value
             "randomize_seed": randomize_seed,
             "return_frames": False,  # We'll get video data directly
             "image_path": None, # For T2V, we pass None as input_image
-            "model_type": "i2v" if "I2V" in model_selection or "Image-to-Video" in model_selection else "t2v",  # Use model type selection
+            # "model_type": "i2v" if "I2V" in model_selection or "Image-to-Video" in model_selection else "t2v",  # Use model type selection
+            "model_type": model_selection.split(' ')[0],
             "model_path": model_selection.split(" (")[0] if model_selection else None  # Extract model path from selection
         }
         
@@ -367,6 +368,32 @@ def create_gradio_interface(backend_url: str, default_params: SamplingParam):
         checkbox_background_color_selected="#2563eb",  # Blue checkbox when selected
     )
     
+    def get_default_values_for_model(model_selection_value):
+        """Get default parameter values for the specified model"""
+        model_path = model_selection_value.split(" (")[0] if model_selection_value else None
+        if model_path and model_path in default_params:
+            params = default_params[model_path]
+            return {
+                'height': params.height,
+                'width': params.width,
+                'num_frames': params.num_frames,
+                'guidance_scale': params.guidance_scale,
+                'seed': params.seed,
+            }
+        else:
+            # Fallback defaults if model not found
+            return {
+                'height': 448,
+                'width': 832,
+                'num_frames': 61,
+                'guidance_scale': 3.0,
+                'seed': 1024,
+            }
+    
+    # Get initial values for the default model
+    default_model = "FastVideo/FastWan2.1-T2V-1.3B-Diffusers (Text-to-Video)"
+    initial_values = get_default_values_for_model(default_model)
+    
     # Create Gradio interface
     with gr.Blocks(title="FastWan", theme=theme) as demo:
         
@@ -408,6 +435,8 @@ def create_gradio_interface(backend_url: str, default_params: SamplingParam):
                 choices=[
                     "FastVideo/FastWan2.1-T2V-1.3B-Diffusers (Text-to-Video)",
                     "FastVideo/FastWan2.1-T2V-14B-Diffusers (Text-to-Video)",
+                    "Wan-AI/Wan2.1-T2V-1.3B-Diffusers (Text-to-Video)",
+                    "Wan-AI/Wan2.1-T2V-14B-480P-Diffusers (Text-to-Video)",
                     # "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers (Image-to-Video)"  # I2V functionality commented out
                 ],
                 value="FastVideo/FastWan2.1-T2V-1.3B-Diffusers (Text-to-Video)",
@@ -452,14 +481,14 @@ def create_gradio_interface(backend_url: str, default_params: SamplingParam):
                             minimum=256,
                             maximum=1280,
                             step=32,
-                            value=448,
+                            value=initial_values['height'],
                         )
                         width = gr.Slider(
                             label="Width",
                             minimum=256,
                             maximum=1280,
                             step=32,
-                            value=832
+                            value=initial_values['width']
                         )
                     
                     with gr.Row():
@@ -468,13 +497,13 @@ def create_gradio_interface(backend_url: str, default_params: SamplingParam):
                             minimum=16,
                             maximum=121,
                             step=16,
-                            value=61,
+                            value=initial_values['num_frames'],
                         )
                         guidance_scale = gr.Slider(
                             label="Guidance Scale",
                             minimum=1,
                             maximum=12,
-                            value=3.0,
+                            value=initial_values['guidance_scale'],
                         )
                     
                     with gr.Row():
@@ -493,7 +522,7 @@ def create_gradio_interface(backend_url: str, default_params: SamplingParam):
                         minimum=0,
                         maximum=1000000,
                         step=1,
-                        value=1024
+                        value=initial_values['seed'],
                     )
                     randomize_seed = gr.Checkbox(label="Randomize seed", value=False)
                     seed_output = gr.Number(label="Used Seed")
@@ -694,6 +723,35 @@ def create_gradio_interface(backend_url: str, default_params: SamplingParam):
         #     outputs=[input_image, prompt],
         # )
         
+        def on_model_selection_change(selected_model):
+            """Update advanced options based on selected model's default parameters"""
+            if not selected_model:
+                return {}, {}, {}, {}, {}  # Return empty updates if no model selected
+            
+            # Extract model path from selection (remove the description part)
+            model_path = selected_model.split(" ")[0] if selected_model else None
+            
+            if model_path and model_path in default_params:
+                params = default_params[model_path]
+                
+                # Update each component with the model's default values
+                return (
+                    gr.update(value=params.height),  # height
+                    gr.update(value=params.width),   # width  
+                    gr.update(value=params.num_frames),  # num_frames
+                    gr.update(value=params.guidance_scale),  # guidance_scale
+                    gr.update(value=params.seed),  # seed
+                )
+            else:
+                # If model not found in default_params, return current values (no change)
+                return {}, {}, {}, {}, {}
+        
+        model_selection.change(
+            fn=on_model_selection_change,
+            inputs=model_selection,
+            outputs=[height, width, num_frames, guidance_scale, seed],
+        )
+        
         def handle_generation(*args, progress=None, request: gr.Request = None):
             # Extract model selection and input image from args - I2V functionality commented out
             model_selection, prompt, negative_prompt, use_negative_prompt, seed, guidance_scale, num_frames, height, width, randomize_seed = args
@@ -760,14 +818,13 @@ def main():
                         type=str,
                         default="http://localhost:8000",
                         help="URL of the Ray Serve backend")
-    parser.add_argument("--t2v_model_path",
+    parser.add_argument("--t2v_model_paths",
                         type=str,
-                        default="FastVideo/FastWan2.1-T2V-1.3B-Diffusers",
-                        help="Path to the T2V model (for default parameters)")
-    parser.add_argument("--t2v_14b_model_path",
-                        type=str,
-                        default="FastVideo/FastWan2.1-T2V-14B-Diffusers",
-                        help="Path to the T2V 14B model (for default parameters)")
+                        default="FastVideo/FastWan2.1-T2V-1.3B-Diffusers,FastVideo/FastWan2.1-T2V-14B-Diffusers",
+                        help="Comma separated list of paths to the T2V model(s)")
+    # parser.add_argument("--t2v_model_replicas", type=int,
+    #                     default="4,4",
+    #                     help="Comma separated list of number of replicas for the T2V model(s)")
     # parser.add_argument("--i2v_model_path",  # I2V functionality commented out
     #                     type=str,
     #                     default="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
@@ -785,7 +842,10 @@ def main():
     
     # Load default parameters from the models
     # try:
-    default_params = SamplingParam.from_pretrained(args.t2v_model_path)
+    default_params = {}
+    model_paths = args.t2v_model_paths.split(",")
+    for model_path in model_paths:
+        default_params[model_path] = SamplingParam.from_pretrained(model_path)
     # except Exception as e:
     #     print(f"Warning: Could not load default parameters from {args.t2v_model_path}: {e}")
     #     print("Using fallback default parameters...")
@@ -803,8 +863,8 @@ def main():
     
     print(f"Starting Gradio frontend at http://{args.host}:{args.port}")
     print(f"Backend URL: {args.backend_url}")
-    print(f"T2V 1.3B Model: {args.t2v_model_path}")
-    print(f"T2V 14B Model: {args.t2v_14b_model_path}")
+    print(f"T2V Models: {args.t2v_model_paths}")
+    # print(f"T2V Model Replicas: {args.t2v_model_replicas}")
     # print(f"I2V Model: {args.i2v_model_path}") # I2V functionality commented out
     
     demo.queue(max_size=20).launch(

examples/inference/gradio/ray_serve_backend.py

@@ -18,6 +18,14 @@ from slowapi.errors import RateLimitExceeded
 import imageio
 from ray.serve.handle import DeploymentHandle
 
+NUM_GPUS = 8
+SUPPORTED_MODELS = [
+    "FastVideo/FastWan2.1-T2V-1.3B-Diffusers", 
+    "FastVideo/FastWan2.1-T2V-14B-Diffusers",
+    "Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
+    "Wan-AI/Wan2.1-T2V-14B-Diffusers",
+]
+
 
 class VideoGenerationRequest(BaseModel):
     prompt: str
@@ -28,7 +36,7 @@ class VideoGenerationRequest(BaseModel):
     num_frames: int = 21
     height: int = 448
     width: int = 832
-    num_inference_steps: int = 20
+    # num_inference_steps: int = 20
     randomize_seed: bool = False
     return_frames: bool = False  # Whether to return base64 encoded frames
     image_path: Optional[str] = None  # Path to input image for I2V
@@ -96,10 +104,13 @@ class T2VModelDeployment:
         os.makedirs(self.output_path, exist_ok=True)
 
         # Delay helps avoid GPU contention when many replicas start at once
-        time.sleep(10)
+        time.sleep(5)
 
         # Ensure correct attention backend for FastVideo
-        os.environ["FASTVIDEO_ATTENTION_BACKEND"] = "VIDEO_SPARSE_ATTN"
+        if "FastVideo" in self.model_path:
+            os.environ["FASTVIDEO_ATTENTION_BACKEND"] = "VIDEO_SPARSE_ATTN"
+        else:
+            os.environ["FASTVIDEO_ATTENTION_BACKEND"] = "FLASH_ATTN"
         os.environ["FASTVIDEO_STAGE_LOGGING"] = "1"
 
         # Lazy import to keep the deployment import-safe on head node
@@ -139,7 +150,7 @@ class T2VModelDeployment:
         params.num_frames = video_request.num_frames
         params.height = video_request.height
         params.width = video_request.width
-        params.num_inference_steps = video_request.num_inference_steps
+        # params.num_inference_steps = video_request.num_inference_steps
 
         # Do not write to disk when called via API
         params.save_video = False
@@ -175,7 +186,7 @@ class T2VModelDeployment:
         encoding_start_time = time.time()
         
         # Encode outputs
-        video_data = encode_video_to_base64(frames, fps=24)
+        video_data = encode_video_to_base64(frames, fps=16)
         
         encoding_end_time = time.time()
         encoding_time = encoding_end_time - encoding_start_time
@@ -348,7 +359,7 @@ class T2V14BModelDeployment:
         params.num_frames = video_request.num_frames
         params.height = video_request.height
         params.width = video_request.width
-        params.num_inference_steps = video_request.num_inference_steps
+        # params.num_inference_steps = video_request.num_inference_steps
 
         # Do not write to disk when called via API
         params.save_video = False
@@ -382,7 +393,7 @@ class T2V14BModelDeployment:
         encoding_start_time = time.time()
         
         # Encode outputs
-        video_data = encode_video_to_base64(frames, fps=24)
+        video_data = encode_video_to_base64(frames, fps=16)
         
         encoding_end_time = time.time()
         encoding_time = encoding_end_time - encoding_start_time
@@ -417,9 +428,9 @@ app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
 class FastVideoAPI:
     """Ingress deployment that routes requests to either the T2V or I2V model deployment."""
 
-    def __init__(self, t2v_deployment: DeploymentHandle, t2v_14b_deployment: DeploymentHandle):  # Removed i2v_deployment
-        self.t2v_handle = t2v_deployment
-        self.t2v_14b_handle = t2v_14b_deployment
+    def __init__(self, t2v_deployments: Dict[str, DeploymentHandle]):  # Removed i2v_deployment
+        self.t2v_deployments = t2v_deployments
+        # self.t2v_14b_handle = t2v_14b_deployment
         # self.i2v_handle = i2v_deployment  # I2V functionality commented out
     
     @app.post("/generate_video", response_model=VideoGenerationResponse)
@@ -427,18 +438,22 @@ class FastVideoAPI:
     async def generate_video(self, request: Request, video_request: VideoGenerationRequest) -> VideoGenerationResponse:
         """Route the request to the appropriate model deployment based on `model_type` and `model_path`."""
         try:
+            # assert video_request.model_type in self.t2v_deployments, f"Model {video_request.model_type} not found"
+            if video_request.model_type not in self.t2v_deployments:
+                raise ValueError(f"Model {video_request.model_type} not found")
+            response_ref = self.t2v_deployments[video_request.model_type].generate_video.remote(video_request)
             # if video_request.model_type.lower() == "i2v":  # I2V functionality commented out
             #     response_ref = self.i2v_handle.generate_video.remote(video_request)
-            if video_request.model_type.lower() == "t2v":
-                # Route T2V requests based on model path
-                if video_request.model_path and "14b" in video_request.model_path.lower():
-                    response_ref = self.t2v_14b_handle.generate_video.remote(video_request)
-                else:
-                    # Default to 1.3B model
-                    response_ref = self.t2v_handle.generate_video.remote(video_request)
-            else:
-                # Default to 1.3B T2V model
-                response_ref = self.t2v_handle.generate_video.remote(video_request)
+            # if video_request.model_type.lower() == "t2v":
+            #     # Route T2V requests based on model path
+            #     if video_request.model_path and "14b" in video_request.model_path.lower():
+            #         response_ref = self.t2v_14b_handle.generate_video.remote(video_request)
+            #     else:
+            #         # Default to 1.3B model
+            #         response_ref = self.t2v_handle.generate_video.remote(video_request)
+            # else:
+            #     # Default to 1.3B T2V model
+            #     response_ref = self.t2v_handle.generate_video.remote(video_request)
 
             # Await the remote response
             response = await response_ref
@@ -463,14 +478,15 @@ class FastVideoAPI:
 
 def start_ray_serve(
     *,
-    t2v_model_path: str = "FastVideo/FastWan2.1-T2V-1.3B-Diffusers",
-    t2v_14b_model_path: str = "FastVideo/FastWan2.1-T2V-14B-Diffusers",
+    t2v_model_paths: str,
+    t2v_model_replicas: str,
+    # t2v_14b_model_path: str = "FastVideo/FastWan2.1-T2V-14B-Diffusers",
     # i2v_model_path: str = "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",  # I2V functionality commented out
     output_path: str = "outputs",
     host: str = "0.0.0.0",
     port: int = 8000,
-    t2v_replicas: int = 4,
-    t2v_14b_replicas: int = 4,  # Reduced replicas for 14B model due to higher resource requirements
+    # t2v_replicas: int = 4,
+    # t2v_14b_replicas: int = 4,  # Reduced replicas for 14B model due to higher resource requirements
     # i2v_replicas: int = 4,  # I2V functionality commented out
 ):
     """Start Ray Serve with independently scalable deployments for each model."""
@@ -479,18 +495,21 @@ def start_ray_serve(
         ray.init()
 
     # Bind model deployments with configurable replica counts
-    t2v_dep = T2VModelDeployment.options(num_replicas=t2v_replicas).bind(t2v_model_path, output_path)
-    t2v_14b_dep = T2V14BModelDeployment.options(num_replicas=t2v_14b_replicas).bind(t2v_14b_model_path, output_path)
+    t2v_deps = {}
+    for model_path, replicas in zip(t2v_model_paths.split(","), t2v_model_replicas.split(",")):
+        t2v_dep = T2VModelDeployment.options(num_replicas=int(replicas)).bind(model_path, output_path)
+        t2v_deps[model_path] = t2v_dep
     # i2v_dep = I2VModelDeployment.options(num_replicas=i2v_replicas).bind(i2v_model_path, output_path)  # I2V functionality commented out
 
     # Ingress
-    api = FastVideoAPI.bind(t2v_dep, t2v_14b_dep)  # Removed i2v_dep
+    api = FastVideoAPI.bind(t2v_deps)  # Removed i2v_dep
 
     serve.run(api, route_prefix="/", name="fast_video")
 
     print(f"Ray Serve backend started at http://{host}:{port}")
-    print(f"T2V 1.3B Model: {t2v_model_path} | Replicas: {t2v_replicas}")
-    print(f"T2V 14B Model: {t2v_14b_model_path} | Replicas: {t2v_14b_replicas}")
+    for model_path, replicas in zip(t2v_model_paths.split(","), t2v_model_replicas.split(",")):
+        print(f"T2V Model: {model_path} | Replicas: {replicas}")
+    # print(f"T2V 14B Model: {t2v_14b_model_path} | Replicas: {t2v_14b_replicas}")
     # print(f"I2V Model: {i2v_model_path} | Replicas: {i2v_replicas}")  # I2V functionality commented out
     print(f"Health check: http://{host}:{port}/health")
     print(f"Video generation endpoint: http://{host}:{port}/generate_video")
@@ -500,14 +519,18 @@ if __name__ == "__main__":
     import argparse
     
     parser = argparse.ArgumentParser(description="FastVideo Ray Serve Backend")
-    parser.add_argument("--t2v_model_path",
+    parser.add_argument("--t2v_model_paths",
                         type=str,
-                        default="FastVideo/FastWan2.1-T2V-1.3B-Diffusers",
-                        help="Path to the T2V model")
-    parser.add_argument("--t2v_14b_model_path",
+                        default="FastVideo/FastWan2.1-T2V-1.3B-Diffusers,FastVideo/FastWan2.1-T2V-14B-Diffusers",
+                        help="Comma separated list of paths to the T2V model(s)")
+    parser.add_argument("--t2v_model_replicas",
                         type=str,
-                        default="FastVideo/FastWan2.1-T2V-14B-Diffusers",
-                        help="Path to the T2V 14B model")
+                        default="4,4",
+                        help="Comma separated list of number of replicas for the T2V model(s)")
+    # parser.add_argument("--t2v_14b_model_path",
+    #                     type=str,
+    #                     default="FastVideo/FastWan2.1-T2V-14B-Diffusers",
+    #                     help="Path to the T2V 14B model")
     # parser.add_argument("--i2v_model_path",  # I2V functionality commented out
     #                     type=str,
     #                     default="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
@@ -524,30 +547,40 @@ if __name__ == "__main__":
                         type=int,
                         default=8000,
                         help="Port to bind to")
-    parser.add_argument("--t2v_replicas",
-                        type=int,
-                        default=4,
-                        help="Number of replicas for the T2V model deployment")
-    parser.add_argument("--t2v_14b_replicas",
-                        type=int,
-                        default=4,  # Reduced default for 14B model due to higher resource requirements
-                        help="Number of replicas for the T2V 14B model deployment")
+    # parser.add_argument("--t2v_replicas",
+    #                     type=int,
+    #                     default=4,
+    #                     help="Number of replicas for the T2V model deployment")
+    # parser.add_argument("--t2v_14b_replicas",
+    #                     type=int,
+    #                     default=4,  # Reduced default for 14B model due to higher resource requirements
+    #                     help="Number of replicas for the T2V 14B model deployment")
     # parser.add_argument("--i2v_replicas",  # I2V functionality commented out
     #                     type=int,
     #                     default=4,  # Reduced default for 14B model due to higher resource requirements
     #                     help="Number of replicas for the I2V model deployment")
     
     args = parser.parse_args()
+
+    split_models = args.t2v_model_paths.split(",")
+    split_replicas = [int(replica) for replica in args.t2v_model_replicas.split(",")]
+    assert len(split_models) == len(split_replicas), "Number of models and replicas must match"
+    assert sum(split_replicas) <= NUM_GPUS, "Total number of replicas must be less than or equal to 16"
+    for model, replicas in zip(split_models, split_replicas):
+        assert model in SUPPORTED_MODELS, f"Model {model} not supported"
+        assert replicas > 0, f"Replicas must be greater than 0"
+
     
     start_ray_serve(
-        t2v_model_path=args.t2v_model_path,
-        t2v_14b_model_path=args.t2v_14b_model_path,
+        t2v_model_paths=args.t2v_model_paths,
+        t2v_model_replicas=args.t2v_model_replicas,
+        # t2v_14b_model_path=args.t2v_14b_model_path,
         # i2v_model_path=args.i2v_model_path,  # I2V functionality commented out
         output_path=args.output_path,
         host=args.host,
         port=args.port,
-        t2v_replicas=args.t2v_replicas,
-        t2v_14b_replicas=args.t2v_14b_replicas,
+        # t2v_replicas=args.t2v_replicas,
+        # t2v_14b_replicas=args.t2v_14b_replicas,
         # i2v_replicas=args.i2v_replicas,  # I2V functionality commented out
     )
 

examples/inference/gradio/start_ray_serve_app.py

@@ -45,8 +45,8 @@ def start_backend(args):
     
     cmd = [
         sys.executable, str(backend_script),
-        "--t2v_model_path", args.t2v_model_path,
-        "--t2v_14b_model_path", args.t2v_14b_model_path,
+        "--t2v_model_paths", args.t2v_model_paths,
+        "--t2v_model_replicas", args.t2v_model_replicas,
         # "--i2v_model_path", args.i2v_model_path,  # I2V functionality commented out
         "--output_path", args.output_path,
         "--host", args.backend_host,
@@ -84,8 +84,8 @@ def start_frontend(args):
     cmd = [
         sys.executable, str(frontend_script),
         "--backend_url", backend_url,
-        "--t2v_model_path", args.t2v_model_path,
-        "--t2v_14b_model_path", args.t2v_14b_model_path,
+        "--t2v_model_paths", args.t2v_model_paths,
+        # "--t2v_model_replicas", args.t2v_model_replicas,
         # "--i2v_model_path", args.i2v_model_path,  # I2V functionality commented out
         "--host", args.frontend_host,
         "--port", str(args.frontend_port)
@@ -118,14 +118,18 @@ def main():
     parser = argparse.ArgumentParser(description="FastVideo Ray Serve App")
     
     # Model and output settings
-    parser.add_argument("--t2v_model_path",
+    parser.add_argument("--t2v_model_paths",
                         type=str,
-                        default="FastVideo/FastWan2.1-T2V-1.3B-Diffusers",
-                        help="Path to the T2V model")
-    parser.add_argument("--t2v_14b_model_path",
+                        default="FastVideo/FastWan2.1-T2V-1.3B-Diffusers,FastVideo/FastWan2.1-T2V-14B-Diffusers",
+                        help="Comma separated list of paths to the T2V model(s)")
+    parser.add_argument("--t2v_model_replicas",
                         type=str,
-                        default="FastVideo/FastWan2.1-T2V-14B-Diffusers",
-                        help="Path to the T2V 14B model")
+                        default="4,4",
+                        help="Comma separated list of number of replicas for the T2V model(s)")
+    # parser.add_argument("--t2v_14b_model_path",
+    #                     type=str,
+    #                     default="FastVideo/FastWan2.1-T2V-14B-Diffusers",
+    #                     help="Path to the T2V 14B model")
     # parser.add_argument("--i2v_model_path",  # I2V functionality commented out
     #                     type=str,
     #                     default="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
@@ -167,8 +171,8 @@ def main():
     
     print("🎬 FastVideo Ray Serve App")
     print("=" * 50)
-    print(f"T2V 1.3B Model: {args.t2v_model_path}")
-    print(f"T2V 14B Model: {args.t2v_14b_model_path}")
+    print(f"T2V Models: {args.t2v_model_paths}")
+    print(f"T2V Model Replicas: {args.t2v_model_replicas}")
     # print(f"I2V Model: {args.i2v_model_path}")  # I2V functionality commented out
     print(f"Output: {args.output_path}")
     print(f"Backend: http://{args.backend_host}:{args.backend_port}")