diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6cf2106b7..f50aba44b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ exclude: | fastvideo/sample/.*| fastvideo/train\.py| fastvideo/utils/.*| - fastvideo/v1/examples/.*| + examples/.*| .github/workflows/fastvideo-publish.yml| .github/workflows/sta-publish.yml| .github/workflows/build-image-template.yml diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 0745be99b..7cca6cbb1 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,25 +1,15 @@ -sphinx==6.2.1 -sphinx-argparse==0.4.0 -sphinx-book-theme==1.0.1 +sphinx==7.4.7 +sphinx-argparse==0.5.2 +sphinx-autodoc2==0.5.0 +sphinx-book-theme==1.1.4 sphinx-copybutton==0.5.2 sphinx-design==0.6.1 sphinx-togglebutton==0.3.2 myst-parser==3.0.1 msgspec -cloudpickle +commonmark # Required by sphinx-argparse when using :markdownhelp: # packages to install to build the documentation cachetools -pydantic >= 2.8 -f https://download.pytorch.org/whl/cpu -torch -py-cpuinfo -transformers -mistral_common >= 1.5.4 -aiohttp -starlette -openai # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args -fastapi # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args -partial-json-parser # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args -requests -zmq \ No newline at end of file +torch \ No newline at end of file diff --git a/docs/source/api/summary.md b/docs/source/api/summary.md new file mode 100644 index 000000000..a453b6bd2 --- /dev/null +++ b/docs/source/api/summary.md @@ -0,0 +1,19 @@ +# Summary + +## Video Generator + +```{autodoc2-summary} + fastvideo.VideoGenerator +``` + +## Initialization Configuration + +```{autodoc2-summary} + fastvideo.v1.configs.pipelines.PipelineConfig +``` + +## Sampling Configuration + +```{autodoc2-summary} + fastvideo.v1.configs.sample.SamplingParam +``` diff --git a/docs/source/autodoc2_docstring_parser.py b/docs/source/autodoc2_docstring_parser.py new file mode 100644 index 000000000..32f25d3c8 --- /dev/null +++ b/docs/source/autodoc2_docstring_parser.py @@ -0,0 +1,22 @@ +# type: ignore +# SPDX-License-Identifier: Apache-2.0 +from docutils import nodes +from myst_parser.parsers.sphinx_ import MystParser +from sphinx.ext.napoleon import docstring + + +class NapoleonParser(MystParser): + + def parse(self, input_string: str, document: nodes.document) -> None: + # Get the Sphinx configuration + config = document.settings.env.config + + parsed_content = str( + docstring.GoogleDocstring( + str(docstring.NumpyDocstring(input_string, config)), + config, + )) + return super().parse(parsed_content, document) + + +Parser = NapoleonParser diff --git a/docs/source/conf.py b/docs/source/conf.py index a4218fd29..91f1f297d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -13,17 +13,19 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. import datetime -import inspect import logging import os +import re import sys +from pathlib import Path from typing import Optional import requests -from sphinx.ext import autodoc logger = logging.getLogger(__name__) -sys.path.append(os.path.abspath("../..")) +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +print(os.path.abspath(REPO_ROOT)) +sys.path.append(os.path.abspath(REPO_ROOT)) # -- Project information ----------------------------------------------------- @@ -41,8 +43,7 @@ extensions = [ "sphinx.ext.linkcode", "sphinx.ext.intersphinx", "sphinx_copybutton", - "sphinx.ext.autodoc", - "sphinx.ext.autosummary", + "autodoc2", "myst_parser", "sphinxarg.ext", "sphinx_design", @@ -50,6 +51,31 @@ extensions = [ ] myst_enable_extensions = [ "colon_fence", + "fieldlist", +] +autodoc2_packages = [ + { + "path": "../../fastvideo", + "exclude_dirs": ["__pycache__", "third_party"], + }, +] +autodoc2_output_dir = "api" +autodoc2_render_plugin = "myst" +autodoc2_hidden_objects = ["dunder", "private", "inherited"] +autodoc2_docstring_parser_regexes = [ + (".*", "docs.source.autodoc2_docstring_parser"), +] +autodoc2_sort_names = True +autodoc2_index_template = None +autodoc2_skip_module_regexes = [ + "fastvideo.dataset", + "fastvideo.distill", + "fastvideo.data_preprocess", + "fastvideo.models", + "fastvideo.sample", + "fastvideo.utils", + "fastvideo.distill_adv", + "fastvideo.train", ] # Add any paths that contain templates here, relative to this directory. @@ -78,6 +104,11 @@ html_theme_options = { 'repository_url': 'https://github.com/hao-ai-lab/FastVideo/', 'use_repository_button': True, 'use_edit_page_button': True, + # Prevents the full API being added to the left sidebar of every page. + # Reduces build time by 2.5x and reduces build size from ~225MB to ~95MB. + 'collapse_navbar': True, + # Makes API visible in the right sidebar on API reference pages. + 'show_toc_level': 3, } # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, @@ -160,38 +191,38 @@ def linkcode_resolve(domain, info): return None if not info['module']: return None - module = info['module'] - # try to determine the correct file and line number to link to - obj = sys.modules[module] + # Get path from module name + file = Path(f"{info['module'].replace('.', '/')}.py") + path = REPO_ROOT / file + if not path.exists(): + path = REPO_ROOT / file.with_suffix("") / "__init__.py" + if not path.exists(): + return None - # get as specific as we can - lineno: int = 0 - filename: str = "" - try: - for part in info['fullname'].split('.'): - obj = getattr(obj, part) + # Get the line number of the object + with open(path) as f: + lines = f.readlines() + name = info['fullname'].split(".")[-1] + pattern = fr"^( {{4}})*((def|class) )?{name}\b.*" + for lineno, line in enumerate(lines, 1): + if not line or line.startswith("#"): + continue + if re.match(pattern, line): + break - if not (inspect.isclass(obj) or inspect.isfunction(obj) - or inspect.ismethod(obj)): - obj = obj.__class__ # type: ignore[assignment] + # If the line number is not found, return None + if lineno == len(lines): + return None - lineno = inspect.getsourcelines(obj)[1] - filename = (inspect.getsourcefile(obj) - or f"{filename}.py").split("FastVideo/", 1)[1] - except Exception: - # For some things, like a class member, won't work, so - # we'll use the line number of the parent (the class) - pass - - if filename.startswith("checkouts/"): + # If the line number is found, create the URL + filename = path.relative_to(REPO_ROOT) + if "checkouts" in path.parts: # a PR build on readthedocs - pr_number = filename.split("/")[1] - filename = filename.split("/", 2)[2] + pr_number = REPO_ROOT.name base, branch = get_repo_base_and_branch(pr_number) if base and branch: return f"https://github.com/{base}/blob/{branch}/{filename}#L{lineno}" - # Otherwise, link to the source file on the main branch return f"https://github.com/hao-ai-lab/FastVideo/blob/main/{filename}#L{lineno}" @@ -203,6 +234,8 @@ autodoc_mock_imports = [ "cpuinfo", "cv2", "torch", + "huggingface_hub", + "torchvision", "transformers", "psutil", "prometheus_client", @@ -231,18 +264,6 @@ for mock_target in autodoc_mock_imports: "been loaded into sys.modules when the sphinx build starts.", mock_target) - -class MockedClassDocumenter(autodoc.ClassDocumenter): - """Remove note about base class when a class is derived from object.""" - - def add_line(self, line: str, source: str, *lineno: int) -> None: - if line == " Bases: :py:class:`object`": - return - super().add_line(line, source, *lineno) - - -autodoc.ClassDocumenter = MockedClassDocumenter - intersphinx_mapping = { "python": ("https://docs.python.org/3", None), "typing_extensions": @@ -254,7 +275,4 @@ intersphinx_mapping = { "psutil": ("https://psutil.readthedocs.io/en/stable", None), } -autodoc_preserve_defaults = True -autodoc_warningiserror = True - navigation_with_keys = False diff --git a/docs/source/contributing/developer_env/docker.md b/docs/source/contributing/developer_env/docker.md index afd1c9112..467284545 100644 --- a/docs/source/contributing/developer_env/docker.md +++ b/docs/source/contributing/developer_env/docker.md @@ -1,3 +1,4 @@ +(docker)= # šŸ³ Using the FastVideo Docker Image If you prefer a containerized development environment or want to avoid managing dependencies manually, you can use our prebuilt Docker image: diff --git a/docs/source/generate_examples.py b/docs/source/generate_examples.py index 0931f6328..6c6ee3bbb 100644 --- a/docs/source/generate_examples.py +++ b/docs/source/generate_examples.py @@ -9,7 +9,7 @@ from typing import Optional ROOT_DIR = Path(__file__).parent.parent.parent.resolve() ROOT_DIR_RELATIVE = '../../../..' -EXAMPLE_DIR = ROOT_DIR / "fastvideo/v1/examples" +EXAMPLE_DIR = ROOT_DIR / "examples" EXAMPLE_DOC_DIR = ROOT_DIR / "docs/source/getting_started/examples" diff --git a/docs/source/getting_started/installation.md b/docs/source/getting_started/installation.md index b15098f9e..b4e33b43d 100644 --- a/docs/source/getting_started/installation.md +++ b/docs/source/getting_started/installation.md @@ -4,32 +4,33 @@ FastVideo currently only supports Linux and NVIDIA CUDA GPUs. -FastVideo has been tested on the following GPUs, but it should work on any GPUs that supports CUDA 12.4+, please create an issue if you discover any issues: -- RTX 4090 -- A40 -- L40S -- A100 -- H100 - ## Requirements -- OS: Linux -- Python: 3.10-3.12 -- CUDA 12.4+ +- **OS: Linux** +- **Python: 3.10-3.12** +- **CUDA 12.4** +- **At least 1 NVIDIA GPU** -## Installation Options - -### Option 1: Quick Install +## Quick start +We recommend using an environment manager such as [Conda](#miniconda-recommended) ```bash pip install fastvideo ``` -### Option 2: Installation from Source +Also optionally install flash-attn: -We recommend using a Python environment such as Conda. +```bash +pip install flash-attn==2.7.4.post1 --no-build-isolation +``` -#### 1. [Optional] Install Miniconda (if not already installed) +[Click here to install from source](#installation-from-source) + +## Environment Setup + +We highly recommend using a `conda` or `virtualenv` environment: +### Miniconda (Recommended) +#### 1. Install Miniconda (if not already installed) ```bash wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh @@ -37,20 +38,22 @@ bash Miniconda3-latest-Linux-x86_64.sh source ~/.bashrc ``` -#### 2. [Optional] Create and activate a Conda environment for FastVideo +#### 2. Create and activate a Conda environment for FastVideo ```bash conda create -n fastvideo python=3.10 -y conda activate fastvideo ``` -#### 3. Clone the FastVideo repository +## Installation from Source + +### 1. Clone the FastVideo repository ```bash git clone https://github.com/hao-ai-lab/FastVideo.git && cd FastVideo ``` -#### 4. Install FastVideo +#### 2. Install FastVideo Basic installation: @@ -66,10 +69,13 @@ pip install -e . pip install flash-attn==2.7.4.post1 --no-build-isolation ``` -### Sliding Tile Attention (STA) (Requires CUDA 12.4+ and H100) +### Sliding Tile Attention (STA) (Requires CUDA 12.4 and H100) To try Sliding Tile Attention (optional), please follow the instructions in [csrc/sliding_tile_attention/README.md](#sta-installation) to install STA. +## Docker Images +We also have prebuilt docker images with FastVideo dependencies pre-installed: +[Docker Images](#docker) ## Development Environment Setup If you're planning to contribute to FastVideo please see the following page: @@ -78,8 +84,7 @@ If you're planning to contribute to FastVideo please see the following page: ## Hardware Requirements ### For Basic Inference -- NVIDIA GPU with CUDA support -- Minimum 20GB VRAM for quantized models (e.g., single RTX 4090) +- NVIDIA GPU with CUDA 12.4support ### For Lora Finetuning - 40GB GPU memory each for 2 GPUs with lora diff --git a/docs/source/getting_started/quick_start.md b/docs/source/getting_started/quick_start.md new file mode 100644 index 000000000..a8b5c2892 --- /dev/null +++ b/docs/source/getting_started/quick_start.md @@ -0,0 +1,143 @@ +# Quick Start + +This page contains instructions and code to get you quickly started with video generation using FastVideo. + +## Requirements +- **OS: Linux** +- **Python: 3.10-3.12** +- **CUDA 12.4** +- **At least 1 NVIDIA GPU** + +## Installation + +We recommend using a environment manager such as `Conda`. + +```bash +pip install fastvideo +``` + +Also see the [Installation Guide](installation.md). + +## Generating Your First Video +Here's a minimal example to generate a video using the default settings. All of the following code snippets can be directly copied into a Python file and executed with + +```bash +python example.py +``` + +```python +from fastvideo import VideoGenerator + +def main(): + # Create a video generator with a pre-trained model + generator = VideoGenerator.from_pretrained( + "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", + num_gpus=1, # Adjust based on your hardware + ) + + # Define a prompt for your video + prompt = "A curious raccoon peers through a vibrant field of yellow sunflowers, its eyes wide with interest." + + # Generate the video + video = generator.generate_video(prompt) +if __name__ == '__main__': + main() +``` + +The generated video will be saved in the current directory under `outputs/` by default. + +## Customizing Generation + +You can customize various parameters when generating videos: + +```python +from fastvideo import VideoGenerator, SamplingParam + +def main(): + # Create the generator + generator = VideoGenerator.from_pretrained( + "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", + num_gpus=1, + ) + + # Create and customize sampling parameters + sampling_param = SamplingParam.from_pretrained("Wan-AI/Wan2.1-T2V-1.3B-Diffusers") + # How many frames to generate + sampling_param.num_frames = 45 + # How many steps we denoise the video + sampling_param.num_inference_steps = 30 + # How strongly does the video to conform to the prompt + sampling_param.guidance_scale = 7.5 + + # Optional: provide an initial image for image-to-video generation + sampling_param.image_path = "path/to/your/image.jpg" # Optional + + # Generate video with custom parameters + prompt = "A beautiful sunset over a calm ocean, with gentle waves." + video = generator.generate_video( + prompt, + sampling_param=sampling_param, + output_path="my_videos/", # controls where videos are saved + return_frames=True # also return frames from this call (defaults to False) + ) + + # `video` now contains frames +if __name__ == '__main__': + main() +``` + +## Available Models + +FastVideo supports various models for text-to-video generation: + +- `Wan-AI/Wan2.1-T2V-1.3B-Diffusers` - A popular text-to-video model +- `FastVideo/FastHunyuan-Diffusers` - A high-performance model for video generation + +## Advanced Configuration + +You can use PipelineConfig for more advanced customization: + +```python +from fastvideo import VideoGenerator, PipelineConfig + +# Load the default configuration for a model +config = PipelineConfig.from_pretrained("Wan-AI/Wan2.1-T2V-1.3B-Diffusers") + +# Modify configuration settings +config.vae_config.scale_factor = 16 +config.vae_config.precision = "fp16" + +# Create generator with custom config +generator = VideoGenerator.from_pretrained( + "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", + num_gpus=4, + pipeline_config=config, +) + +# Generate video +prompt = "A futuristic cityscape with flying cars and neon signs." +video = generator.generate_video(prompt) +``` + +## Multi-GPU Setup + +FastVideo automatically handles distributed setup when multiple GPUs are specified: + +```python +generator = VideoGenerator.from_pretrained( + "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", + num_gpus=4, # Will use 4 GPUs in parallel +) +``` + +## Hardware Requirements + +- **Basic Inference**: Minimum 20GB VRAM for quantized models (e.g., single RTX 4090) +- **LoRA Finetuning**: 40GB GPU memory each for 2 GPUs with LoRA +- **Full Finetuning/Distillation**: Multiple high-memory GPUs recommended (e.g., H100) + +## Next Steps + +- Explore the [API Reference](../api/index.md) for more details +- Learn about [Advanced Inference Options](../inference/overview_back.md) +- See [Examples](../examples/index.md) for more usage scenarios diff --git a/docs/source/getting_started/v1_api.md b/docs/source/getting_started/v1_api.md new file mode 100644 index 000000000..15d39a99f --- /dev/null +++ b/docs/source/getting_started/v1_api.md @@ -0,0 +1,18 @@ +# V1 API + +FastVideo's V1 API provides a streamlined interface for video generation tasks with powerful customization options. This page documents the primary components of the API. + +## Video Generator + +This class will be the primary Python API for generating videos and images. + +```{autodoc2-summary} + fastvideo.VideoGenerator +``` + +VideoGenerator.from_pretrained() should be the primary way of creating a new video generator. + +````{py:method} from_pretrained(model_path: str, device: typing.optional[str] = none, torch_dtype: typing.optional[torch.dtype] = none, pipeline_config: typing.optional[typing.union[str | fastvideo.v1.configs.pipelines.pipelineconfig]] = none, **kwargs) -> v1.entrypoints.video_generator.videogenerator +:canonical: v1.entrypoints.video_generator.videogenerator.from_pretrained +:classmethod: +``` diff --git a/docs/source/index.md b/docs/source/index.md index 2d9882737..f6791c88b 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -51,14 +51,17 @@ Dev in progress and highly experimental. :maxdepth: 1 getting_started/installation - +getting_started/quick_start +getting_started/v1_api ::: :::{toctree} :caption: Inference :maxdepth: 1 -inference/examples/examples_inference_index +inference/examples/basic +inference/examples/gradio +inference/examples/optimizations inference/v0_inference ::: @@ -96,6 +99,14 @@ contributing/developer_env/index contributing/add_pipeline ::: +:::{toctree} +:caption: API Reference +:maxdepth: 2 + +api/summary +api/fastvideo/fastvideo +::: + ## Indices and tables - {ref}`genindex` diff --git a/examples/inference/basic/README.md b/examples/inference/basic/README.md index 82ae0d88e..68047d232 100644 --- a/examples/inference/basic/README.md +++ b/examples/inference/basic/README.md @@ -1,3 +1,41 @@ -# Basic +# Basic Video Generation Tutorial +The `VideoGenerator` class provides the primary Python interface for doing offline video generation, which is interacting with a diffusion pipeline without using a separate inference api server. -The class provides the main python interface for using FastVideo's inference pipeline. +## Requirements +- At least a single NVIDIA GPU with CUDA 12.4. +- Python 3.10-3.12 + +## Installation +If you have not installed FastVideo, please following these [instructions](https://hao-ai-lab.github.io/FastVideo/getting_started/installation.html) first. + +## Usage +The first script in this example shows the most basic usage of FastVideo. If you are new to Python and FastVideo, you should start here. + +```bash +# if you have not cloned the directory: +git clone https://github.com/hao-ai-lab/FastVideo.git && cd FastVideo + +python fastvideo/v1/examples/inference/basic/basic.py +``` + +## Basic Walkthrough + +All you need to generate videos using multi-gpus from state-of-the-art diffusion pipelines is the following few lines! + +```python +from fastvideo import VideoGenerator + +def main(): + generator = VideoGenerator.from_pretrained( + "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", + num_gpus=1, + ) + + prompt = ("A curious raccoon peers through a vibrant field of yellow sunflowers, its eyes " + "wide with interest. The playful yet serene atmosphere is complemented by soft " + "natural light filtering through the petals. Mid-shot, warm and cheerful tones.") + video = generator.generate_video(prompt) + +if __name__ == "__main__": + main() +``` diff --git a/examples/inference/basic/basic.py b/examples/inference/basic/basic.py index 8c854ae3f..69bb12444 100644 --- a/examples/inference/basic/basic.py +++ b/examples/inference/basic/basic.py @@ -1 +1,41 @@ -print('Hello, world!') \ No newline at end of file +from fastvideo import VideoGenerator + +# from fastvideo.v1.configs.sample import SamplingParam + + +def main(): + # FastVideo will automatically use the optimal default arguments for the + # model. + # If a local path is provided, FastVideo will make a best effort + # attempt to identify the optimal arguments. + generator = VideoGenerator.from_pretrained( + "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", + # if num_gpus > 1, FastVideo will automatically handle distributed setup + num_gpus=1, + ) + + # sampling_param = SamplingParam.from_pretrained("Wan-AI/Wan2.1-T2V-1.3B-Diffusers") + # sampling_param.num_frames = 45 + # sampling_param.image_path = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/astronaut.jpg" + # Generate videos with the same simple API, regardless of GPU count + prompt = ( + "A curious raccoon peers through a vibrant field of yellow sunflowers, its eyes " + "wide with interest. The playful yet serene atmosphere is complemented by soft " + "natural light filtering through the petals. Mid-shot, warm and cheerful tones." + ) + video = generator.generate_video(prompt) + # video = generator.generate_video(prompt, sampling_param=sampling_param, output_path="wan_t2v_videos/") + + # Generate another video with a different prompt, without reloading the + # model! + prompt2 = ( + "A majestic lion strides across the golden savanna, its powerful frame " + "glistening under the warm afternoon sun. The tall grass ripples gently in " + "the breeze, enhancing the lion's commanding presence. The tone is vibrant, " + "embodying the raw energy of the wild. Low angle, steady tracking shot, " + "cinematic.") + video2 = generator.generate_video(prompt2) + + +if __name__ == "__main__": + main() diff --git a/examples/inference/basic/default_args.py b/examples/inference/basic/default_args.py new file mode 100644 index 000000000..cb1a1d0ab --- /dev/null +++ b/examples/inference/basic/default_args.py @@ -0,0 +1,62 @@ +from fastvideo import VideoGenerator + + +def main(): + + # This is the config class for the model initialization + config = PipelineConfig.from_pretrained("FastVideo/FastHunyuan-Diffusers") + # can be used to dump the config to a yaml file + config.dump_to_yaml("config.yaml") + print(config) + # { + # 'vae_config': { + # 'scale_factor': 8, + # 'sp': True, + # 'tiling': True, + # 'precision': 'fp16' + # }, + # 'text_encoder_config': { + # 'precision': 'fp16' + # }, + # 'dit_config': { + # 'precision': 'fp16' + # }, + # 'inference_args': { + # 'guidance_scale': 7.5, + # 'num_inference_steps': 5, + # 'seed': 1024, + # 'guidance_rescale': 0.0, + # 'flow_shift': 17, + # 'num_inference_steps': 5, + # } + # } + + config.vae_config.scale_factor = 16 + + # FastVideo will automatically used the optimal default arguments for the model + # If a local path is provided, FastVideo will make a best effort attempt to + # identify the optimal arguments. + generator = VideoGenerator.from_pretrained( + "FastVideo/FastHunyuan-Diffusers", + num_gpus=4, + config=config, + # or + config_path="config.yaml", + ) + + sampling_param = SamplingParam.from_pretrained( + "FastVideo/FastHunyuan-Diffusers") + sampling_param.num_inference_steps = 5 + + # Generate videos with the same simple API, regardless of GPU count + prompt = "A beautiful woman in a red dress walking down a street" + video = generator.generate_video(prompt, + sampling_param=sampling_param, + num_inference_steps=6) + + video2 = generator.generate_video(prompt2) + prompt2 = "A beautiful woman in a blue dress walking down a street" + + +if __name__ == "__main__": + main() diff --git a/fastvideo/v1/examples/inference/gradio/README.md b/examples/inference/gradio/README.md similarity index 87% rename from fastvideo/v1/examples/inference/gradio/README.md rename to examples/inference/gradio/README.md index d86d0d80f..6add451d2 100644 --- a/fastvideo/v1/examples/inference/gradio/README.md +++ b/examples/inference/gradio/README.md @@ -1,4 +1,4 @@ -# FastVideo VideoGenerator Gradio Demo +# FastVideo Gradio Demo This is a Gradio-based web interface for generating videos using the FastVideo framework. The demo allows users to create videos from text prompts with various customization options. @@ -16,7 +16,8 @@ The demo uses the FastVideo framework to generate videos based on text prompts. ## Requirements - Linux-based OS -- Python 3.10 +- Python 3.10-3.12 +- NVIDIA GPU - Cuda 12.4 - FastVideo @@ -40,15 +41,6 @@ This will start a web server at `http://0.0.0.0:7860` where you can access the i ## Model Initialization -```python -args = FastVideoArgs(model_path="FastVideo/FastHunyuan-Diffusers", num_gpus=2) - -generator = VideoGenerator.from_pretrained( - model_path=args.model_path, - num_gpus=args.num_gpus -) -``` - This demo initializes a `VideoGenerator` with the minimum required arguments for inference. Users can seamlessly adjust inference options between generations, including prompts, resolution, video length, or even the number of inference steps, *without ever needing to reload the model*. ## Video Generation @@ -78,4 +70,4 @@ The interface is built with several components: - **Guidance Scale**: Control how closely the generation follows the prompt - **Inference Steps**: More steps can improve quality but take longer - **Negative Prompt**: Specify what you don't want to see in the video -- **Seed**: Control randomness for reproducible results \ No newline at end of file +- **Seed**: Control randomness for reproducible results diff --git a/fastvideo/v1/examples/inference/gradio/gradio_demo.py b/examples/inference/gradio/gradio_demo.py similarity index 60% rename from fastvideo/v1/examples/inference/gradio/gradio_demo.py rename to examples/inference/gradio/gradio_demo.py index a1e047e70..68fc790f5 100644 --- a/fastvideo/v1/examples/inference/gradio/gradio_demo.py +++ b/examples/inference/gradio/gradio_demo.py @@ -1,26 +1,33 @@ -import os -import gradio as gr -import torch import argparse +import os from copy import deepcopy -from fastvideo.v1.fastvideo_args import FastVideoArgs +import gradio as gr +import torch + from fastvideo import VideoGenerator from fastvideo.v1.configs.sample.base import SamplingParam if __name__ == "__main__": parser = argparse.ArgumentParser(description="FastVideo Gradio Demo") - parser.add_argument("--model_path", type=str, default="FastVideo/FastHunyuan-diffusers", help="Path to the model") - parser.add_argument("--num_gpus", type=int, default=1, help="Number of GPUs to use") - parser.add_argument("--output_path", type=str, default="outputs", help="Path to save generated videos") + parser.add_argument("--model_path", + type=str, + default="FastVideo/FastHunyuan-diffusers", + help="Path to the model") + parser.add_argument("--num_gpus", + type=int, + default=1, + help="Number of GPUs to use") + parser.add_argument("--output_path", + type=str, + default="outputs", + help="Path to save generated videos") parsed_args = parser.parse_args() - + # args = FastVideoArgs(model_path="FastVideo/FastHunyuan-Diffusers", num_gpus=2) generator = VideoGenerator.from_pretrained( - model_path=parsed_args.model_path, - num_gpus=parsed_args.num_gpus - ) + model_path=parsed_args.model_path, num_gpus=parsed_args.num_gpus) default_params = SamplingParam.from_pretrained(parsed_args.model_path) @@ -52,19 +59,17 @@ if __name__ == "__main__": if not use_negative_prompt: params.negative_prompt = None - generator.generate_video( - prompt=prompt, - sampling_param=params - ) + generator.generate_video(prompt=prompt, sampling_param=params) - output_path = os.path.join(parsed_args.output_path, f"{params.prompt[:100]}.mp4") + output_path = os.path.join(parsed_args.output_path, + f"{params.prompt[:100]}.mp4") return output_path, params.seed examples = [ - "A hand enters the frame, pulling a sheet of plastic wrap over three balls of dough placed on a wooden surface. The plastic wrap is stretched to cover the dough more securely. The hand adjusts the wrap, ensuring that it is tight and smooth over the dough. The scene focuses on the handā€™s movements as it secures the edges of the plastic wrap. No new objects appear, and the camera remains stationary, focusing on the action of covering the dough.", - "A vintage train snakes through the mountains, its plume of white steam rising dramatically against the jagged peaks. The cars glint in the late afternoon sun, their deep crimson and gold accents lending a touch of elegance. The tracks carve a precarious path along the cliffside, revealing glimpses of a roaring river far below. Inside, passengers peer out the large windows, their faces lit with awe as the landscape unfolds.", - "A crowded rooftop bar buzzes with energy, the city skyline twinkling like a field of stars in the background. Strings of fairy lights hang above, casting a warm, golden glow over the scene. Groups of people gather around high tables, their laughter blending with the soft rhythm of live jazz. The aroma of freshly mixed cocktails and charred appetizers wafts through the air, mingling with the cool night breeze.", + "A hand enters the frame, pulling a sheet of plastic wrap over three balls of dough placed on a wooden surface. The plastic wrap is stretched to cover the dough more securely. The hand adjusts the wrap, ensuring that it is tight and smooth over the dough. The scene focuses on the handā€™s movements as it secures the edges of the plastic wrap. No new objects appear, and the camera remains stationary, focusing on the action of covering the dough.", + "A vintage train snakes through the mountains, its plume of white steam rising dramatically against the jagged peaks. The cars glint in the late afternoon sun, their deep crimson and gold accents lending a touch of elegance. The tracks carve a precarious path along the cliffside, revealing glimpses of a roaring river far below. Inside, passengers peer out the large windows, their faces lit with awe as the landscape unfolds.", + "A crowded rooftop bar buzzes with energy, the city skyline twinkling like a field of stars in the background. Strings of fairy lights hang above, casting a warm, golden glow over the scene. Groups of people gather around high tables, their laughter blending with the soft rhythm of live jazz. The aroma of freshly mixed cocktails and charred appetizers wafts through the air, mingling with the cool night breeze.", ] with gr.Blocks() as demo: @@ -92,11 +97,12 @@ if __name__ == "__main__": step=32, value=default_params.height, ) - width = gr.Slider( - label="Width", - minimum=256, - maximum=1024, step=32, value=default_params.width) - + width = gr.Slider(label="Width", + minimum=256, + maximum=1024, + step=32, + value=default_params.width) + with gr.Row(): num_frames = gr.Slider( label="Number of Frames", @@ -118,7 +124,8 @@ if __name__ == "__main__": ) with gr.Row(): - use_negative_prompt = gr.Checkbox(label="Use negative prompt", value=False) + use_negative_prompt = gr.Checkbox( + label="Use negative prompt", value=False) negative_prompt = gr.Text( label="Negative prompt", max_lines=1, @@ -126,7 +133,11 @@ if __name__ == "__main__": visible=False, ) - seed = gr.Slider(label="Seed", minimum=0, maximum=1000000, step=1, value=default_params.seed) + seed = gr.Slider(label="Seed", + minimum=0, + maximum=1000000, + step=1, + value=default_params.seed) randomize_seed = gr.Checkbox(label="Randomize seed", value=True) seed_output = gr.Number(label="Used Seed") diff --git a/examples/inference/optimizations/README.md b/examples/inference/optimizations/README.md new file mode 100644 index 000000000..86212cd0b --- /dev/null +++ b/examples/inference/optimizations/README.md @@ -0,0 +1,104 @@ +# Speeding Up Generation + +This page describes the various options for speeding up generation times. + +## Table of Contents +- Optimized Attention Backends + - [Flash Attention](#optimizations-flash) + - [Sliding Tile Attention](#optimizations-sta) + - [Sage Attention](#optimizations-sage) + +- Caching Techniques + - [TeaCache](#optimizations-teacache) + + +(optimizations-backends)= +## Attention Backends + +`attention_example.py` shows how to set `FASTVIDEO_ATTENTION_BACKEND` env var to change attention backends. To run this example: +```bash +python examples/inference/optimizations/attention_example.py +``` + +In python, set the `FASTVIDEO_ATTENTION_BACKEND` before instantiating `VideoGenerator` like this: + +```python + os.environ["FASTVIDEO_ATTENTION_BACKEND"] = "SLIDING_TILE_ATTN" +``` + +You can also set the env var when running any of the other example like this: +```bash +FASTVIDEO_ATTENTION_BACKEND=SAGE_ATTN python example.py +``` + +(optimizations-flash)= +### Flash Attention + +(optimizations-sta)= +### Sliding Tile Attention + +(optimizations-sage)= +### Sage Attention + + +(optimizations-teacache)= +## Teacache +TeaCache is an optimization technique supported in FastVideo that can significantly speed up video generation by skipping redundant calculations across diffusion steps. This guide explains how to enable and configure TeaCache for optimal performance in FastVideo. + +### What is TeaCache? + +See the official [TeaCache](https://github.com/ali-vilab/TeaCache) repo and their paper for more details. + + +### How to Enable TeaCache + +Enabling TeaCache is straightforward - simply add the `enable_teacache=True` parameter to your `generate_video()` call: + +```python +# ... previous code +generator.generate_video( + prompt="Your prompt here", + sampling_param=params, + enable_teacache=True +) +# more code ... +``` + +### Complete Example + +At the bottom is a complete example of using TeaCache for faster video generation. You can run it using the following command: + +```bash +python examples/inference/optimizations/teacache_example.py +``` + +### Advanced Configuration + +While TeaCache works well with default settings, you can fine-tune its behavior by adjusting the threshold value: + +1. Lower threshold values (e.g., 0.1) will result in more skipped calculations and faster generation with slightly more potential for quality degradation +2. Higher threshold values (e.g., 0.15-0.23) will skip fewer calculations but maintain quality closer to the original + +Note that the optimal threshold depends on your specific model and content. + +## Benchmarking different optimizations + +To benchmark the performance improvement, try generating the same video with and without TeaCache enabled and compare the generation times: + +```python +# Without TeaCache +start_time = time.time() +generator.generate_video(prompt="Your prompt", enable_teacache=False) +standard_time = time.time() - start_time + +# With TeaCache +start_time = time.time() +generator.generate_video(prompt="Your prompt", enable_teacache=True) +teacache_time = time.time() - start_time + +print(f"Standard generation: {standard_time:.2f} seconds") +print(f"TeaCache generation: {teacache_time:.2f} seconds") +print(f"Speedup: {standard_time/teacache_time:.2f}x") +``` + +Note: If you want to benchmark different attention backends, you'll need to reinstantiate `VideoGenerator`. diff --git a/examples/inference/optimizations/attention_example.py b/examples/inference/optimizations/attention_example.py new file mode 100644 index 000000000..b9d162dc2 --- /dev/null +++ b/examples/inference/optimizations/attention_example.py @@ -0,0 +1,33 @@ +import os +import time + +from fastvideo import VideoGenerator + +def main(): + # set the attention backend + os.environ["FASTVIDEO_ATTENTION_BACKEND"] = "FLASH_ATTN" + + start_time = time.time() + gen = VideoGenerator.from_pretrained( + model_path="Wan-AI/Wan2.1-T2V-1.3B-Diffusers", + num_gpus=1, + ) + load_time = time.time() - start_time + print(f"Model loading time: {load_time:.2f} seconds") + + gen_start_time = time.time() + + gen.generate_video( + prompt= + "Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of a bustling street food market. The scene captures a mix of humor and authenticity. Mid-shot framing, vibrant lighting.", + seed=1024, + output_path="example_outputs/") + + generation_time = time.time() - gen_start_time + print(f"Video generation time: {generation_time:.2f} seconds") + + total_time = time.time() - start_time + print(f"Total execution time: {total_time:.2f} seconds") + +if __name__ == "__main__": + main() diff --git a/examples/inference/optimizations/teacache_example.py b/examples/inference/optimizations/teacache_example.py new file mode 100644 index 000000000..d87940665 --- /dev/null +++ b/examples/inference/optimizations/teacache_example.py @@ -0,0 +1,44 @@ +import time + +from fastvideo import VideoGenerator, SamplingParam + + +def main(): + start_time = time.time() + + gen = VideoGenerator.from_pretrained( + model_path="Wan-AI/Wan2.1-T2V-1.3B-Diffusers", + num_gpus=1, + use_cpu_offload=False, + ) + load_time = time.time() - start_time + print(f"Model loading time: {load_time:.2f} seconds") + + gen_start_time = time.time() + + params = SamplingParam.from_pretrained( + model_path="Wan-AI/Wan2.1-T2V-1.3B-Diffusers", + ) + # this controls the threshold for the tea cache + params.teacache_params.teacache_thresh = 0.08 + gen.generate_video( + prompt= + "Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of a bustling street food market. The scene captures a mix of humor and authenticity. Mid-shot framing, vibrant lighting.", + sampling_param=params, + height=480, + width=832, + num_frames=61, # 85 ,77 + num_inference_steps=50, + enable_teacache=True, + seed=1024, + output_path="example_outputs/") + + generation_time = time.time() - gen_start_time + print(f"Video generation time: {generation_time:.2f} seconds") + + total_time = time.time() - start_time + print(f"Total execution time: {total_time:.2f} seconds") + + +if __name__ == "__main__": + main() diff --git a/fastvideo/v1/attention/backends/abstract.py b/fastvideo/v1/attention/backends/abstract.py index cbade2bbb..d88dad622 100644 --- a/fastvideo/v1/attention/backends/abstract.py +++ b/fastvideo/v1/attention/backends/abstract.py @@ -56,20 +56,6 @@ class AttentionMetadata: # Current step of diffusion process current_timestep: int - # @property - # @abstractmethod - # def inference_metadata(self) -> Optional["AttentionMetadata"]: - # """Return the attention metadata that's required to run prefill - # attention.""" - # pass - - # @property - # @abstractmethod - # def training_metadata(self) -> Optional["AttentionMetadata"]: - # """Return the attention metadata that's required to run decode - # attention.""" - # pass - def asdict_zerocopy(self, skip_fields: Optional[Set[str]] = None ) -> Dict[str, Any]: @@ -86,55 +72,6 @@ class AttentionMetadata: T = TypeVar("T", bound=AttentionMetadata) -# class AttentionState(ABC, Generic[T]): -# """Holds attention backend-specific objects reused during the -# lifetime of the model runner.""" - -# @abstractmethod -# def __init__(self, runner: "ModelRunnerBase"): -# ... - -# @abstractmethod -# @contextmanager -# def graph_capture(self, max_batch_size: int): -# """Context manager used when capturing CUDA graphs.""" -# yield - -# @abstractmethod -# def graph_clone(self, batch_size: int) -> "AttentionState[T]": -# """Clone attention state to save in CUDA graph metadata.""" -# ... - -# @abstractmethod -# def graph_capture_get_metadata_for_batch( -# self, -# batch_size: int, -# is_encoder_decoder_model: bool = False) -> T: -# """Get attention metadata for CUDA graph capture of batch_size.""" -# ... - -# @abstractmethod -# def get_graph_input_buffers( -# self, -# attn_metadata: T, -# is_encoder_decoder_model: bool = False) -> Dict[str, Any]: -# """Get attention-specific input buffers for CUDA graph capture.""" -# ... - -# @abstractmethod -# def prepare_graph_input_buffers( -# self, -# input_buffers: Dict[str, Any], -# attn_metadata: T, -# is_encoder_decoder_model: bool = False) -> None: -# """In-place modify input buffers dict for CUDA graph replay.""" -# ... - -# @abstractmethod -# def begin_forward(self, model_input: "ModelRunnerInputBase") -> None: -# """Prepare state for forward pass.""" -# ... - class AttentionMetadataBuilder(ABC, Generic[T]): """Abstract class for attention metadata builders.""" diff --git a/fastvideo/v1/configs/sample/base.py b/fastvideo/v1/configs/sample/base.py index fedd5b341..c6c01b9c0 100644 --- a/fastvideo/v1/configs/sample/base.py +++ b/fastvideo/v1/configs/sample/base.py @@ -8,6 +8,9 @@ logger = init_logger(__name__) @dataclass class SamplingParam: + """ + Sampling parameters for video generation. + """ # All fields below are copied from ForwardBatch data_type: str = "video" diff --git a/fastvideo/v1/entrypoints/video_generator.py b/fastvideo/v1/entrypoints/video_generator.py index 7f0cab131..f5e5775db 100644 --- a/fastvideo/v1/entrypoints/video_generator.py +++ b/fastvideo/v1/entrypoints/video_generator.py @@ -210,7 +210,9 @@ class VideoGenerator: n_tokens: {n_tokens} flow_shift: {fastvideo_args.flow_shift} embedded_guidance_scale: {fastvideo_args.embedded_cfg_scale} -""" # type: ignore[attr-defined] + save_video: {sampling_param.save_video} + output_path: {sampling_param.output_path} + """ # type: ignore[attr-defined] logger.info(debug_str) # Prepare batch diff --git a/fastvideo/v1/examples/inference/basic/README.md b/fastvideo/v1/examples/inference/basic/README.md deleted file mode 100644 index 7757c7df1..000000000 --- a/fastvideo/v1/examples/inference/basic/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# Basic Video Generation Tutorial -The `VideoGenerator` class provides the primary Python interface for doing offline video generation, which is interacting with a diffusion pipeline without using a separate inference api server. - - -## Usage -The first script in this example shows the most basic usage of FastVideo. If you are new to Python and FastVideo, you should start here. - -```bash -python fastvideo/v1/examples/inference/basic/basic.py -``` - -## Basic Walkthrough - -All you need to generate videos using multi-gpus from state-of-the-art diffusion pipelines is the following few lines! - -```python -from fastvideo import VideoGenerator - -generator = VideoGenerator.from_pretrained( - "FastVideo/FastHunyuan-Diffusers", - num_gpus=2, -) - -prompt = "A beautiful woman in a red dress walking down a street" -video = generator.generate_video(prompt) -``` - -More to come! These examples and APIs are still under construction! \ No newline at end of file diff --git a/fastvideo/v1/examples/inference/basic/basic.py b/fastvideo/v1/examples/inference/basic/basic.py deleted file mode 100644 index 7a767c409..000000000 --- a/fastvideo/v1/examples/inference/basic/basic.py +++ /dev/null @@ -1,29 +0,0 @@ -from fastvideo import VideoGenerator - -def main(): - # FastVideo will automatically use the optimal default arguments for the - # model. - # If a local path is provided, FastVideo will make a best effort - # attempt to identify the optimal arguments. - generator = VideoGenerator.from_pretrained( - "FastVideo/FastHunyuan-diffusers", - # if num_gpus > 1, FastVideo will automatically handle distributed setup - num_gpus=4, - ) - - # sampling_param = SamplingParam.from_pretrained("/workspace/data/Wan-AI/Wan2.1-I2V-14B-480P-Diffusers") - # sampling_param.num_frames = 45 - # sampling_param.image_path = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/astronaut.jpg" - # Generate videos with the same simple API, regardless of GPU count - prompt = "A beautiful woman in a red dress walking down a street" - video = generator.generate_video(prompt) - # video = generator.generate_video(prompt, sampling_param=sampling_param, output_path="wan_t2v_videos/") - - # Generate another video with a different prompt, without reloading the - # model! - prompt2 = "A beautiful woman in a blue dress walking down a street" - video2 = generator.generate_video(prompt2) - - -if __name__ == "__main__": - main() diff --git a/fastvideo/v1/pipelines/README.md b/fastvideo/v1/pipelines/README.md index 1229320f2..c0ecda73e 100644 --- a/fastvideo/v1/pipelines/README.md +++ b/fastvideo/v1/pipelines/README.md @@ -1,3 +1,3 @@ # Adding a New Custom Pipeline -Please see +Please see documentation [here](https://hao-ai-lab.github.io/FastVideo/contributing/add_pipeline.html)