diff --git a/examples/inference/gradio/gradio_frontend.py b/examples/inference/gradio/gradio_frontend.py index e0414f70c..9e5e33429 100644 --- a/examples/inference/gradio/gradio_frontend.py +++ b/examples/inference/gradio/gradio_frontend.py @@ -138,51 +138,6 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling if progress: progress(0.1, desc="Checking backend health...") - # Determine if this is I2V based on model selection - I2V functionality commented out - # is_i2v = "I2V" in model_selection or "Image-to-Video" in model_selection - - # Handle input image for I2V - I2V functionality commented out - # image_path = None - # if is_i2v and input_image is not None: - # if progress: - # progress(0.2, desc="Processing input image...") - # try: - # # Save the uploaded image to a temporary file - # import tempfile - # temp_dir = "temp_images" - # os.makedirs(temp_dir, exist_ok=True) - # - # # Generate a unique filename with appropriate extension - # import uuid - # # Determine the best format to preserve quality - # if hasattr(input_image, 'format') and input_image.format: - # # Use original format if available - # ext = input_image.format.lower() - # if ext == 'jpeg': - # ext = 'jpg' - # else: - # # Default to PNG for lossless quality - # ext = 'png' - # - # image_filename = f"input_image_{uuid.uuid4().hex[:8]}.{ext}" - # image_path = os.path.abspath(os.path.join(temp_dir, image_filename)) - # - # # Save the image preserving original quality - # if ext == 'png': - # # Use PNG for lossless compression - # input_image.save(image_path, "PNG", optimize=False) - # elif ext == 'jpg': - # # Use high quality JPEG with minimal compression - # input_image.convert("RGB").save(image_path, "JPEG", quality=95, optimize=False) - # else: - # # For other formats, save as PNG to preserve quality - # input_image.save(image_path, "PNG", optimize=False) - # - # print(f"Saved input image to: {image_path}") - # except Exception as e: - # print(f"Warning: Failed to save input image: {e}") - # image_path = None - # Prepare request data if progress: progress(0.3, desc="Preparing request...") @@ -200,7 +155,6 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling "randomize_seed": randomize_seed, "return_frames": False, # We'll get video data directly "image_path": None, # For T2V, we pass None as input_image - "model_type": MODEL_PATH_MAPPING[model_selection], "model_path": MODEL_PATH_MAPPING.get(model_selection, "FastVideo/FastWan2.1-T2V-1.3B-Diffusers") # Map to full path } @@ -213,14 +167,6 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling if progress: progress(0.8, desc="Processing response...") - # Clean up temporary image file after processing - # if image_path and os.path.exists(image_path): - # try: - # os.remove(image_path) - # print(f"Cleaned up temporary image: {image_path}") - # except Exception as e: - # print(f"Warning: Failed to clean up temporary image {image_path}: {e}") - if response.get("success", False): video_data = response.get("video_data", "") used_seed = response.get("seed", seed) @@ -273,27 +219,6 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling """ - # timing_details += f""" - #
- #

🔄 Processing Stages

- #
- # """ - # - # # Add individual stage timing cards - # for stage_name, stage_time in zip(stage_names, stage_execution_times): - # if stage_name.strip() and stage_time > 0: # Only show non-empty stages with valid times - # timing_details += f""" - #
- #
{stage_name.strip()}
- #
{stage_time:.2f}s
- #
- # """ - # - # timing_details += """ - #
- #
- # """ - # Add performance insights if inference_time > 0: fps = num_frames / inference_time @@ -338,7 +263,7 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling def contains_chinese(text): """Check if text contains Chinese characters""" for char in text: - if '\u4e00' <= char <= '\u9fff': # CJK Unified Ideographs (Chinese characters) + if '\u4e00' <= char <= '\u9fff': return True return False @@ -430,18 +355,6 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling """) - # Backend status indicator - # status_text = gr.Text( - # label="Backend Status", - # value="Checking backend status...", - # interactive=False - # ) - - def update_status(): - if client.check_health(): - return "✅ Backend is healthy and ready" - else: - return "❌ Backend is not available" # Model selection dropdown with gr.Row(): @@ -449,8 +362,6 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling choices=[ "FastWan2.1-T2V-1.3B", "FastWan2.2-TI2V-5B-FullAttn", - # "Wan2.1-T2V-1.3B", - # "Wan2.2-TI2V-5B", # "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers (Image-to-Video)" # I2V functionality commented out ], value="FastWan2.1-T2V-1.3B", @@ -487,7 +398,6 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling with gr.Row(): with gr.Column(): error_output = gr.Text(label="Error", visible=False) - # frames_output = gr.Text(label="Generation Status", visible=False) timing_display = gr.Markdown(label="Timing Breakdown", visible=False) # Two-column layout: Advanced options on left, Video on right @@ -761,18 +671,7 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling outputs=negative_prompt, ) - # Model selection change handler - I2V functionality commented out - # def on_model_selection_change(model_selection): - # is_i2v = "I2V" in model_selection or "Image-to-Video" in model_selection - # prompt_placeholder = "Describe how the image should animate" if is_i2v else "Enter your prompt" - # return gr.update(visible=is_i2v), gr.update(placeholder=prompt_placeholder) - # - # model_selection.change( - # fn=on_model_selection_change, - # inputs=model_selection, - # outputs=[input_image, prompt], - # ) - + # Model selection change handler def on_model_selection_change(selected_model): """Update advanced options based on selected model's default parameters""" if not selected_model: @@ -803,16 +702,9 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling ) def handle_generation(*args, progress=None, request: gr.Request = None): - # Extract model selection and input image from args - I2V functionality commented out + # Extract model selection and input image from args model_selection, prompt, negative_prompt, use_negative_prompt, seed, guidance_scale, num_frames, height, width, randomize_seed = args - # Determine if this is I2V based on model selection - I2V functionality commented out - # is_i2v = "I2V" in model_selection or "Image-to-Video" in model_selection - - # For T2V, we pass None as input_image - I2V functionality commented out - # if not is_i2v: - # input_image = None - # Call the generate_video function with progress tracking result_path, seed_or_error, timing_details = generate_video( prompt, negative_prompt, use_negative_prompt, seed, guidance_scale, @@ -824,7 +716,6 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling result_path, seed_or_error, gr.update(visible=False), # error_output - # gr.update(visible=True, value="Generation completed successfully!"), # frames_output gr.update(visible=True, value=timing_details), # timing_display ) else: @@ -832,7 +723,6 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling None, seed_or_error, gr.update(visible=True, value=seed_or_error), # error_output - # gr.update(visible=False), # frames_output gr.update(visible=False), # timing_display ) @@ -850,7 +740,6 @@ def create_gradio_interface(backend_url: str, default_params: dict[str, Sampling height, width, randomize_seed, - # input_image, # Removed input_image from inputs ], outputs=[result, seed_output, error_output, timing_display], concurrency_limit=20, @@ -872,10 +761,6 @@ def main(): # parser.add_argument("--t2v_model_replicas", type=int, # default="4,4", # help="Comma separated list of number of replicas for the T2V model(s)") - # parser.add_argument("--i2v_model_path", # I2V functionality commented out - # type=str, - # default="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers", - # help="Path to the I2V model (for default parameters)") parser.add_argument("--host", type=str, default="0.0.0.0", @@ -899,8 +784,6 @@ def main(): print(f"Starting Gradio frontend at http://{args.host}:{args.port}") print(f"Backend URL: {args.backend_url}") print(f"T2V Models: {args.t2v_model_paths}") - # print(f"T2V Model Replicas: {args.t2v_model_replicas}") - # print(f"I2V Model: {args.i2v_model_path}") # I2V functionality commented out # Use FastAPI to serve custom HTML with proper Open Graph metadata from fastapi import FastAPI, Request @@ -998,7 +881,6 @@ def main(): """ - # Mount Gradio app under /gradio app = gr.mount_gradio_app( app, demo, diff --git a/examples/inference/gradio/ray_serve_backend.py b/examples/inference/gradio/ray_serve_backend.py index 7cdd10940..2d10d74c8 100644 --- a/examples/inference/gradio/ray_serve_backend.py +++ b/examples/inference/gradio/ray_serve_backend.py @@ -94,7 +94,6 @@ class VideoGenerationRequest(BaseModel): randomize_seed: bool = False return_frames: bool = False # Whether to return base64 encoded frames image_path: Optional[str] = None # Path to input image for I2V - model_type: str = "t2v" # "t2v" or "i2v" to specify which model to use model_path: Optional[str] = None # Specific model path to use @@ -261,101 +260,6 @@ class T2VModelDeployment: ) -# @serve.deployment( # I2V 14B model deployment - I2V functionality commented out -# ray_actor_options={"num_cpus": 10, "num_gpus": 1, "runtime_env": {"conda": "fv"}}, -# ) -# class I2VModelDeployment: -# """Serve deployment wrapping the 14 B image-to-video model.""" -# -# def __init__(self, i2v_model_path: str, output_path: str = "outputs"): -# self.model_path = i2v_model_path -# self.output_path = output_path -# -# os.makedirs(self.output_path, exist_ok=True) -# time.sleep(10) -# -# os.environ["FASTVIDEO_ATTENTION_BACKEND"] = "VIDEO_SPARSE_ATTN" -# -# from fastvideo.entrypoints.video_generator import VideoGenerator -# from fastvideo.configs.sample.base import SamplingParam -# -# print(f"Initializing I2V model: {self.model_path}") -# self.generator = VideoGenerator.from_pretrained( -# model_path=self.model_path, -# num_gpus=1, -# use_fsdp_inference=True, -# text_encoder_cpu_offload=False, -# dit_cpu_offload=False, -# vae_cpu_offload=False, -# VSA_sparsity=0.8, -# ) -# self.default_params = SamplingParam.from_pretrained(self.model_path) -# print("✅ I2V model initialized successfully") -# -# def generate_video(self, video_request: "VideoGenerationRequest") -> "VideoGenerationResponse": -# import time -# -# total_start_time = time.time() -# -# params = deepcopy(self.default_params) -# -# params.prompt = video_request.prompt -# if video_request.use_negative_prompt: -# params.negative_prompt = video_request.negative_prompt -# -# params.seed = video_request.seed if not video_request.randomize_seed else torch.randint(0, 1_000_000, (1,)).item() -# params.guidance_scale = video_request.guidance_scale -# params.num_frames = video_request.num_frames -# params.height = video_request.height -# params.width = video_request.width -# params.num_inference_steps = video_request.num_inference_steps -# -# if video_request.image_path: -# params.image_path = video_request.image_path -# -# params.save_video = False -# params.return_frames = True -# -# # Track inference time -# inference_start_time = time.time() -# -# result = self.generator.generate_video( -# prompt=video_request.prompt, -# sampling_param=params, -# save_video=False, -# return_frames=True, -# ) -# -# inference_end_time = time.time() -# inference_time = inference_end_time - inference_start_time -# -# frames = result if isinstance(result, list) else result.get("frames", []) -# generation_time = result.get("generation_time", 0.0) if isinstance(result, dict) else 0.0 -# -# # Track encoding time -# encoding_start_time = time.time() -# -# video_data = encode_video_to_base64(frames, fps=24) -# encoded_frames = encode_frames_to_base64(frames) if video_request.return_frames and frames else None -# -# encoding_end_time = time.time() -# encoding_time = encoding_end_time - encoding_start_time -# -# total_end_time = time.time() -# total_time = total_end_time - total_start_time -# -# return VideoGenerationResponse( -# video_data=video_data, -# frames=encoded_frames, -# seed=params.seed, -# success=True, -# generation_time=generation_time, -# inference_time=inference_time, -# encoding_time=encoding_time, -# total_time=total_time, -# ) - - @serve.deployment( # T2V 14B model deployment with optimized settings ray_actor_options={"num_cpus": 16, "num_gpus": 1, "runtime_env": {"conda": "fv"}}, ) @@ -514,16 +418,16 @@ class FastVideoAPI: @app.post("/generate_video", response_model=VideoGenerationResponse) @limiter.limit("10/minute") # Allow 50 requests per minute per IP async def generate_video(self, request: Request, video_request: VideoGenerationRequest) -> VideoGenerationResponse: - """Route the request to the appropriate model deployment based on `model_type` and `model_path`.""" + """Route the request to the appropriate model deployment based on `model_path`.""" start_time = time.time() - model_type = video_request.model_path.split('/')[-1] if video_request.model_path else "unknown" + model_name = video_request.model_path.split('/')[-1] if video_request.model_path else "unknown" user_ip = get_remote_address(request) try: - # assert video_request.model_type in self.t2v_deployments, f"Model {video_request.model_type} not found" - if video_request.model_type not in self.t2v_deployments: - raise ValueError(f"Model {video_request.model_type} not found") - response_ref = self.t2v_deployments[video_request.model_type].generate_video.remote(video_request) + # assert video_request.model_path in self.t2v_deployments, f"Model {video_request.model_path} not found" + if video_request.model_path not in self.t2v_deployments: + raise ValueError(f"Model {video_request.model_path} not found") + response_ref = self.t2v_deployments[video_request.model_path].generate_video.remote(video_request) # if video_request.model_type.lower() == "i2v": # I2V functionality commented out # response_ref = self.i2v_handle.generate_video.remote(video_request) # if video_request.model_type.lower() == "t2v": @@ -543,7 +447,7 @@ class FastVideoAPI: # Log the prompt (success case) self.prompt_logger.log_prompt( prompt=video_request.prompt, - model_type=video_request.model_type, + model_type=video_request.model_path, success=response.success, negative_prompt=video_request.negative_prompt if video_request.use_negative_prompt else None, generation_time=response.total_time, @@ -551,10 +455,10 @@ class FastVideoAPI: ) # Record success metrics - self.request_count.labels(model_type=model_type, status="success").inc() - self.request_duration.labels(model_type=model_type).observe(time.time() - start_time) + self.request_count.labels(model_type=model_name, status="success").inc() + self.request_duration.labels(model_type=model_name).observe(time.time() - start_time) if hasattr(response, 'generation_time_seconds') and response.generation_time_seconds: - self.video_generation_time.labels(model_type=model_type).observe(response.generation_time_seconds) + self.video_generation_time.labels(model_type=model_name).observe(response.generation_time_seconds) return response @@ -562,7 +466,7 @@ class FastVideoAPI: # Log the prompt (error case) self.prompt_logger.log_prompt( prompt=video_request.prompt, - model_type=video_request.model_type, + model_type=video_request.model_path, success=False, negative_prompt=video_request.negative_prompt if video_request.use_negative_prompt else None, generation_time=None, @@ -570,8 +474,8 @@ class FastVideoAPI: ) # Record error metrics - self.request_count.labels(model_type=model_type, status="error").inc() - self.request_duration.labels(model_type=model_type).observe(time.time() - start_time) + self.request_count.labels(model_type=model_name, status="error").inc() + self.request_duration.labels(model_type=model_name).observe(time.time() - start_time) return VideoGenerationResponse( video_data=None, diff --git a/fastvideo/configs/pipelines/registry.py b/fastvideo/configs/pipelines/registry.py index eb9b61555..0ff0ece7e 100644 --- a/fastvideo/configs/pipelines/registry.py +++ b/fastvideo/configs/pipelines/registry.py @@ -29,7 +29,8 @@ PIPE_NAME_TO_CONFIG: dict[str, type[PipelineConfig]] = { "Wan-AI/Wan2.1-T2V-14B-Diffusers": WanT2V720PConfig, "FastVideo/FastWan2.1-T2V-1.3B-Diffusers": FastWan2_1_T2V_480P_Config, "FastVideo/FastWan2.1-T2V-14B-480P-Diffusers": FastWan2_1_T2V_480P_Config, - "FastVideo/FastWan2.2-TI2V-5B-Diffusers": FastWan2_2_TI2V_5B_Config, + "FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers": + FastWan2_2_TI2V_5B_Config, "FastVideo/stepvideo-t2v-diffusers": StepVideoT2VConfig, "FastVideo/Wan2.1-VSA-T2V-14B-720P-Diffusers": WanT2V720PConfig, "Wan-AI/Wan2.2-TI2V-5B-Diffusers": Wan2_2_TI2V_5B_Config, diff --git a/fastvideo/configs/sample/registry.py b/fastvideo/configs/sample/registry.py index 193574282..97ba85879 100644 --- a/fastvideo/configs/sample/registry.py +++ b/fastvideo/configs/sample/registry.py @@ -30,10 +30,10 @@ SAMPLING_PARAM_REGISTRY: dict[str, Any] = { "FastVideo/stepvideo-t2v-diffusers": StepVideoT2VSamplingParam, "FastVideo/FastWan2.1-T2V-1.3B-Diffusers": Wan2_1_Fun_1_3B_InP_SamplingParam, - "FastVideo/FastWan2.1-T2V-14B-Diffusers": - Wan2_1_Fun_1_3B_InP_SamplingParam, + "FastVideo/FastWan2.1-T2V-14B-Diffusers": Wan2_1_Fun_1_3B_InP_SamplingParam, "Wan-AI/Wan2.2-TI2V-5B-Diffusers": Wan2_2_TI2V_5B_SamplingParam, - "FastVideo/FastWan2.2-TI2V-5B-Diffusers": Wan2_2_TI2V_5B_SamplingParam, + "FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers": + Wan2_2_TI2V_5B_SamplingParam, # "Wan-AI/Wan2.2-T2V-A14B-Diffusers": # Wan2_2_T2V_A14B_SamplingParam, # "Wan-AI/Wan2.2-I2V-A14B-Diffusers":