add full i2v demo

examples/inference/gradio/serving/gradio_frontend.py

@@ -5,6 +5,7 @@ import base64
 import time
 import json
 from pathlib import Path
+import tempfile
 
 import gradio as gr
 
@@ -240,9 +241,12 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
         return "I2V" in model_name
     
     def generate_video(
-        prompt, negative_prompt, use_negative_prompt, seed, guidance_scale,
-        num_frames, height, width, randomize_seed, model_selection, input_image, progress
+        prompt, negative_prompt, use_negative_prompt, guidance_scale,
+        num_frames, height, width, model_selection, input_image, progress
     ):
+        # Use default seed value (randomize_seed disabled)
+        seed = 1000
+        randomize_seed = False
         if not client.check_health():
             return None, f"Backend is not available. Please check if Ray Serve is running at {backend_url}", ""
         
@@ -342,14 +346,12 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
         #         'width': params.width,
         #         'num_frames': params.num_frames,
         #         'guidance_scale': params.guidance_scale,
-        #         'seed': params.seed,
         #     }
         
         return {
             'height': 480,
             'width': 832,
             'num_frames': 73,
-            'seed': 1000,
         }
     
     # Get available models based on what's loaded
@@ -363,12 +365,12 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
     initial_values = get_default_values(default_model)
     initial_show_image = is_i2v_model(default_model)
     
-    with gr.Blocks(title="FastWan", theme=theme) as demo:
+    with gr.Blocks(title="CausalWan", theme=theme) as demo:
         gr.Image("assets/logos/logo.svg", show_label=False, container=False, height=80)
         gr.HTML("""
         <div style="text-align: center; margin-bottom: 10px;">
             <p style="font-size: 18px;"> Make Video Generation Go Blurrrrrrr </p>
-            <p style="font-size: 18px;"> <a href="https://github.com/hao-ai-lab/FastVideo/tree/main" target="_blank">Code</a> | <a href="https://hao-ai-lab.github.io/blogs/fastvideo_post_training/" target="_blank">Blog</a> | <a href="https://hao-ai-lab.github.io/FastVideo/" target="_blank">Docs</a>  </p>
+            <p style="font-size: 18px;"> <a href="https://github.com/hao-ai-lab/FastVideo/tree/main" target="_blank">Code</a> | <a href="https://hao-ai-lab.github.io/blogs/fastvideo_causalwan_preview/" target="_blank">Blog</a> | <a href="https://hao-ai-lab.github.io/FastVideo/" target="_blank">Docs</a>  </p>
         </div>
         """)
         
@@ -420,6 +422,14 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
         with gr.Row(equal_height=False):
             with gr.Column(scale=1):
                 with gr.Tabs():
+                    with gr.Tab("Input Image", visible=initial_show_image) as image_tab:
+                        gr.Markdown("**Please make sure you upload a 480x832 image**")
+                        input_image = gr.Image(
+                            label="",
+                            type="filepath",
+                            height=400,
+                        )
+                    
                     with gr.Tab("Advanced Options"):
                         with gr.Group():
                             with gr.Row():
@@ -461,22 +471,8 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
                                     visible=False,
                                 )
 
-                            seed = gr.Slider(
-                                label="Seed",
-                                minimum=0,
-                                maximum=1000000,
-                                step=1,
-                                value=initial_values['seed'],
-                            )
-                            randomize_seed = gr.Checkbox(label="Randomize seed", value=False)
-                            seed_output = gr.Number(label="Used Seed")
-                    
-                    with gr.Tab("Input Image", visible=initial_show_image) as image_tab:
-                        input_image = gr.Image(
-                            label="",
-                            type="filepath",
-                            height=400,
-                        )
+                            # randomize_seed = gr.Checkbox(label="Randomize seed", value=False)
+                            seed_output = gr.Number(label="Used Seed", value=1000)
         
             with gr.Column(scale=1):
                 result = gr.Video(
@@ -528,7 +524,7 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
         
         gr.HTML("""
         <div style="text-align: center; margin-top: 10px; margin-bottom: 15px;">
-            <p style="font-size: 16px; margin: 0;">The compute for this demo is generously provided by NVIDIA. Note that this demo is meant as a preview of our distilled I2V model.</p>
+            <p style="font-size: 16px; margin: 0;">The compute for this demo is generously provided by <a href="https://www.gmicloud.ai/" target="_blank">GMI Cloud</a>.  Note that this demo is meant as a preview of our distilled I2V model. Outside of few-step distillation, we have not yet fully optimized it for speed. Stay tuned for updates!</p>
         </div>
         """)
         
@@ -552,7 +548,6 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
                     gr.update(value=params.width),
                     gr.update(value=params.num_frames),
                     gr.update(value=params.guidance_scale),
-                    gr.update(value=params.seed),
                     gr.update(visible=show_image_input),
                 )
             
@@ -561,22 +556,21 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
                 gr.update(value=832),
                 gr.update(value=20),
                 gr.update(value=3.0),
-                gr.update(value=1024),
                 gr.update(visible=show_image_input),
             )
         
         model_selection.change(
             fn=on_model_selection_change,
             inputs=model_selection,
-            outputs=[height, width, num_frames, guidance_scale, seed, image_tab],
+            outputs=[height, width, num_frames, guidance_scale, image_tab],
         )
         
         def handle_generation(*args, progress=None, request: gr.Request = None):
-            model_selection, prompt, negative_prompt, use_negative_prompt, seed, guidance_scale, num_frames, height, width, randomize_seed, input_image = args
+            model_selection, prompt, negative_prompt, use_negative_prompt, guidance_scale, num_frames, height, width, input_image = args
             
             result_path, seed_or_error, _ = generate_video(
-                prompt, negative_prompt, use_negative_prompt, seed, guidance_scale, 
-                num_frames, height, width, randomize_seed, model_selection, input_image, progress
+                prompt, negative_prompt, use_negative_prompt, guidance_scale, 
+                num_frames, height, width, model_selection, input_image, progress
             )
             
             if result_path and os.path.exists(result_path):
@@ -599,12 +593,11 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling
                 prompt,
                 negative_prompt,
                 use_negative_prompt,
-                seed,
                 guidance_scale,
                 num_frames,
                 height,
                 width,
-                randomize_seed,
+                # randomize_seed,
                 input_image,
             ],
             outputs=[result, seed_output, error_output],  # timing_display removed
@@ -694,23 +687,23 @@ def main():
             <meta charset="UTF-8" />
             <meta name="viewport" content="width=device-width, initial-scale=1.0" />
             
-            <title>FastWan</title>
-            <meta name="title" content="FastWan">
+            <title>CausalWan</title>
+            <meta name="title" content="CausalWan">
             <meta name="description" content="Make video generation go blurrrrrrr">
-            <meta name="keywords" content="FastVideo, video generation, AI, machine learning, FastWan">
+            <meta name="keywords" content="FastVideo, video generation, AI, machine learning, CausalWan">
             
             <meta property="og:type" content="website">
             <meta property="og:url" content="{base_url}/">
-            <meta property="og:title" content="FastWan">
+            <meta property="og:title" content="CausalWan">
             <meta property="og:description" content="Make video generation go blurrrrrrr">
             <meta property="og:image" content="{base_url}/logo.svg">
             <meta property="og:image:width" content="1200">
             <meta property="og:image:height" content="630">
-            <meta property="og:site_name" content="FastWan">
+            <meta property="og:site_name" content="CausalWan">
             
             <meta property="twitter:card" content="summary_large_image">
             <meta property="twitter:url" content="{base_url}/">
-            <meta property="twitter:title" content="FastWan">
+            <meta property="twitter:title" content="CausalWan">
             <meta property="twitter:description" content="Make video generation go blurrrrrrr">
             <meta property="twitter:image" content="{base_url}/logo.svg">
             <link rel="icon" type="image/png" sizes="32x32" href="/favicon.ico">
@@ -744,7 +737,9 @@ def main():
             os.path.abspath("outputs"), 
             os.path.abspath("fastvideo-logos"),
             os.path.abspath("prompts"),
-            os.path.abspath("images")
+            os.path.abspath("images"),
+            os.path.abspath(tempfile.gettempdir()),
+            os.path.abspath(os.path.join(tempfile.gettempdir(), "gradio")),
         ]
     )
     

examples/inference/gradio/serving/prompts.txt

@@ -2,4 +2,14 @@ Some friends dancing and having fun together in circles, at a party surrounded b
 A man wearing grey shorts jumps rope in a gym, weights and gym equipment in the background.
 Flying over a peninsula covered in bushy trees, while discovering the sea around it, painted a beautiful turquoise blue, on a sunny day.
 Skillful cyclist doing a wheelie on a bike while riding through a forest, on a dirt road, surrounded by many trees, in the morning.
-In the video, a lone rider guides a majestic horse across an expansive, open field as the sun sets in the background. The rider, dressed in a classic blue shirt and wide-brimmed hat, sits confidently in the saddle, silhouetted against the warm glow of the evening sky. The horse moves gracefully, its mane and tail flowing with each step, creating a sense of harmony between horse and rider. Surrounding the pair, towering trees form a natural border, their leaves gently rustling in the breeze. The shadows lengthen on the ground, accentuating the serene and timeless feel of the scene. The distant hills and wooden fences frame the horizon, adding depth to the tranquil landscape. A few horses graze peacefully in the background, blending into the pastoral setting. The overall ambiance evokes a sense of calmness and quietude, capturing a perfect moment in the golden light of dusk.
+In the video, a lone rider guides a majestic horse across an expansive, open field as the sun sets in the background. The rider, dressed in a classic blue shirt and wide-brimmed hat, sits confidently in the saddle, silhouetted against the warm glow of the evening sky. The horse moves gracefully, its mane and tail flowing with each step, creating a sense of harmony between horse and rider. Surrounding the pair, towering trees form a natural border, their leaves gently rustling in the breeze. The shadows lengthen on the ground, accentuating the serene and timeless feel of the scene. The distant hills and wooden fences frame the horizon, adding depth to the tranquil landscape. A few horses graze peacefully in the background, blending into the pastoral setting. The overall ambiance evokes a sense of calmness and quietude, capturing a perfect moment in the golden light of dusk.
+A silver SUV drives along a winding, snow-covered mountain road, with dense pine trees blanketed in snow lining both sides. The scene is serene, with the vehicle moving smoothly, possibly on a winter journey or vacation. As the SUV disappears around the bend, another, darker SUV follows, creating a sense of motion and perspective on the snow-dusted asphalt. The towering, snow-laden rock formation to the right contrasts with the dark green of the pines, highlighting the peacefulness of the wintry landscape.
+A man and a woman playing in a field with grass, during a bright afternoon, while cars pass by in the distance.
+A saxophonist wearing a blazer dances while playing a song in a park.
+Romantic couple embracing and looking at each other in the middle of a forest, during a break on a road trip through nature.
+Young woman cleaning her house decorated with plants and decorations, while dancing happily to music in her headphones.
+Man dressed in 80's style dances very happily in his kitchen while listening to music on his radio and drinking wine.
+Pair of jazz musicians performing a song with their saxophone and trombone on an abandoned train.
+A young woman with short hair wearing pink sunglasses chews gum and makes a bubble gum with the city in the background.
+Natural aerial landscape with a relief covered with abundant trees and vegetation and a thick layer of mist.
+Loving couple sitting on a log on the shore of a lake outside, sharing an affectionate hug.

examples/inference/gradio/serving/ray_serve_backend.py

@@ -100,8 +100,8 @@ def encode_video_to_base64(frames: List[np.ndarray], fps: int = DEFAULT_FPS) ->
         return ""
 
 
-def decode_image_from_base64(image_data: str) -> Optional[np.ndarray]:
-    """Decode base64 image data to numpy array."""
+def save_image_from_base64(image_data: str, output_dir: str) -> Optional[str]:
+    """Save base64 image data to a temporary file and return the path."""
     if not image_data:
         return None
     
@@ -111,11 +111,18 @@ def decode_image_from_base64(image_data: str) -> Optional[np.ndarray]:
             image_data = image_data.split(',')[1]
         
         image_bytes = base64.b64decode(image_data)
-        image_array = imageio.imread(io.BytesIO(image_bytes))
-        return image_array
+        
+        # Save to temporary file
+        os.makedirs(output_dir, exist_ok=True)
+        temp_image_path = os.path.join(output_dir, f"temp_input_{int(time.time() * 1000)}.png")
+        
+        with open(temp_image_path, 'wb') as f:
+            f.write(image_bytes)
+        
+        return temp_image_path
         
     except Exception as e:
-        print(f"Warning: Failed to decode image: {e}")
+        print(f"Warning: Failed to save image: {e}")
         return None
 
 
@@ -186,35 +193,41 @@ class BaseModelDeployment:
             num_gpus=1,
             use_fsdp_inference=True,
             text_encoder_cpu_offload=config["text_encoder_cpu_offload"],
+            dmd_denoising_steps=[1000, 850, 700, 550, 350, 275, 200, 125], # TODO: hardocde for I2V
+            dit_precision="fp32",  # TODO: hardocde for I2V
             dit_cpu_offload=config["dit_cpu_offload"],
             vae_cpu_offload=config["vae_cpu_offload"],
             VSA_sparsity=config["VSA_sparsity"],
             enable_stage_verification=False,
         )
         self.default_params = SamplingParam.from_pretrained(self.model_path)
+        self.default_params.seed = 1000
+        self.default_params.num_frames = 73
+        self.default_params.width = 832
+        self.default_params.height = 480
 
     def generate_video(self, video_request: VideoGenerationRequest) -> VideoGenerationResponse:
         total_start_time = time.time()
         
         params = prepare_sampling_params(video_request, self.default_params)
 
-        # Decode image if provided (for I2V)
-        image = None
+        # Save image if provided (for I2V)
+        image_path = None
         if video_request.image_data:
-            image = decode_image_from_base64(video_request.image_data)
-            if image is None:
+            image_path = save_image_from_base64(video_request.image_data, self.output_path)
+            if image_path is None:
                 return VideoGenerationResponse(
                     video_data=None,
                     seed=params.seed,
                     success=False,
-                    error_message="Failed to decode input image",
+                    error_message="Failed to save input image",
                 )
 
         inference_start_time = time.time()
         result = self.generator.generate_video(
             prompt=video_request.prompt,
             sampling_param=params,
-            image=image,  # Pass image for I2V
+            image_path=image_path,
             save_video=False,
             return_frames=False,
         )
@@ -227,6 +240,13 @@ class BaseModelDeployment:
         encoding_time = time.time() - encoding_start_time
         
         total_time = time.time() - total_start_time
+        
+        # Clean up temporary image file
+        if image_path and os.path.exists(image_path):
+            try:
+                os.remove(image_path)
+            except Exception as e:
+                print(f"Warning: Failed to remove temporary image file {image_path}: {e}")
 
         return VideoGenerationResponse(
             video_data=video_data,
@@ -242,7 +262,7 @@ class BaseModelDeployment:
 
 
 @serve.deployment(
-    ray_actor_options={"num_cpus": 2, "num_gpus": 1, "runtime_env": {"conda": "fv"}},
+    ray_actor_options={"num_cpus": 2, "num_gpus": 1, "runtime_env": {"conda": "demo-fv"}},
 )
 class T2VModelDeployment(BaseModelDeployment):
     def __init__(self, t2v_model_path: str, output_path: str = "outputs"):
@@ -252,7 +272,7 @@ class T2VModelDeployment(BaseModelDeployment):
 
 
 @serve.deployment(
-    ray_actor_options={"num_cpus": 16, "num_gpus": 1, "runtime_env": {"conda": "fv"}},
+    ray_actor_options={"num_cpus": 16, "num_gpus": 1, "runtime_env": {"conda": "demo-fv"}},
 )
 class T2V14BModelDeployment(BaseModelDeployment):
     def __init__(self, t2v_14b_model_path: str, output_path: str = "outputs"):
@@ -264,7 +284,7 @@ class T2V14BModelDeployment(BaseModelDeployment):
 
 
 @serve.deployment(
-    ray_actor_options={"num_cpus": 15, "num_gpus": 1, "runtime_env": {"conda": "fv"}},
+    ray_actor_options={"num_cpus": 15, "num_gpus": 1, "runtime_env": {"conda": "demo-fv"}},
 )
 class I2VModelDeployment(BaseModelDeployment):
     def __init__(self, i2v_model_path: str, output_path: str = "outputs"):

prompts/mixkit_i2v.jsonl

@@ -78,5 +78,33 @@
     {
         "prompt": "Some friends dancing and having fun together in circles, at a party surrounded by colored lights at a party, in a fancy old place, in a view from below them.",
         "image_path": "images/friends.jpg"
+    },
+    {
+        "prompt": "A saxophonist wearing a blazer dances while playing a song in a park.",
+        "image_path": "images/saxophonist.jpg"
+    },
+    {
+        "prompt": "Romantic couple embracing and looking at each other in the middle of a forest, during a break on a road trip through nature.",
+        "image_path": "images/romance.jpg"
+    },
+    {
+        "prompt": "Man dressed in 80's style dances very happily in his kitchen while listening to music on his radio and drinking wine.",
+        "image_path": "images/80s_dance.jpg"
+    },
+    {
+        "prompt": "Pair of jazz musicians performing a song with their saxophone and trombone on an abandoned train.",
+        "image_path": "images/jazz.jpg"
+    },
+    {
+        "prompt": "A young woman with short hair wearing pink sunglasses chews gum and makes a bubble gum with the city in the background.",
+        "image_path": "images/pink.jpg"
+    },
+    {
+        "prompt": "Natural aerial landscape with a relief covered with abundant trees and vegetation and a thick layer of mist.",
+        "image_path": "images/natural.jpg"
+    },
+    {
+        "prompt": "Loving couple sitting on a log on the shore of a lake outside, sharing an affectionate hug.",
+        "image_path": "images/couple.jpg"
     }
 ]