diff --git a/.github/workflows/infra-docs.yml b/.github/workflows/infra-docs.yml index 4166d4799..69aba4c84 100644 --- a/.github/workflows/infra-docs.yml +++ b/.github/workflows/infra-docs.yml @@ -6,6 +6,7 @@ on: paths: - 'docs/**' - 'examples/**' + - 'scripts/inference/**' - 'mkdocs.yml' - 'requirements-mkdocs.in' - 'requirements-mkdocs.txt' @@ -16,6 +17,7 @@ on: paths: - 'docs/**' - 'examples/**' + - 'scripts/inference/**' - 'mkdocs.yml' - 'requirements-mkdocs.in' - 'requirements-mkdocs.txt' diff --git a/docs/assets/cookbook-recipes.json b/docs/assets/cookbook-recipes.json new file mode 100644 index 000000000..eadeee535 --- /dev/null +++ b/docs/assets/cookbook-recipes.json @@ -0,0 +1,52 @@ +{ + "recipes": [ + { + "id": "fastwan21-t2v", + "task": "Text to video", + "label": "FastWan2.1 1.3B (distilled + VSA)", + "model": "FastVideo/FastWan2.1-T2V-1.3B-Diffusers", + "source": "scripts/inference/inference_wan_VSA_DMD_1_3B.yaml", + "command": "FASTVIDEO_ATTENTION_BACKEND=VIDEO_SPARSE_ATTN fastvideo generate --config scripts/inference/inference_wan_VSA_DMD_1_3B.yaml" + }, + { + "id": "wan22-t2v", + "task": "Text to video", + "label": "Wan2.2 A14B", + "model": "Wan-AI/Wan2.2-T2V-A14B-Diffusers", + "source": "examples/inference/basic/basic_wan2_2.py", + "command": "python examples/inference/basic/basic_wan2_2.py" + }, + { + "id": "wan21-i2v", + "task": "Image to video", + "label": "Wan2.1 14B 480P", + "model": "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers", + "source": "scripts/inference/inference_wan_i2v.yaml", + "command": "fastvideo generate --config scripts/inference/inference_wan_i2v.yaml" + }, + { + "id": "turbowan22-i2v", + "task": "Image to video", + "label": "TurboWan2.2 A14B", + "model": "loayrashid/TurboWan2.2-I2V-A14B-Diffusers", + "source": "examples/inference/basic/basic_turbodiffusion_i2v.py", + "command": "python examples/inference/basic/basic_turbodiffusion_i2v.py" + }, + { + "id": "wan22-ti2v", + "task": "Text or image to video", + "label": "Wan2.2 TI2V 5B", + "model": "Wan-AI/Wan2.2-TI2V-5B-Diffusers", + "source": "examples/inference/basic/basic_wan2_2_ti2v.py", + "command": "python examples/inference/basic/basic_wan2_2_ti2v.py" + }, + { + "id": "matrix-game-2", + "task": "Interactive world", + "label": "Matrix Game 2.0", + "model": "FastVideo/Matrix-Game-2.0-Base-Distilled-Diffusers", + "source": "examples/inference/basic/basic_matrixgame2.py", + "command": "python examples/inference/basic/basic_matrixgame2.py" + } + ] +} diff --git a/docs/assets/cookbook.js b/docs/assets/cookbook.js new file mode 100644 index 000000000..1b985bc4e --- /dev/null +++ b/docs/assets/cookbook.js @@ -0,0 +1,60 @@ +(() => { + let recipesPromise; + + const loadRecipes = (url) => { + recipesPromise ||= fetch(url).then((response) => { + if (!response.ok) throw new Error(`HTTP ${response.status}`); + return response.json(); + }); + return recipesPromise; + }; + + const init = () => { + document.querySelectorAll("[data-cookbook]").forEach(async (root) => { + if (root.dataset.initialized) return; + root.dataset.initialized = "true"; + + const select = root.querySelector("[data-cookbook-recipe]"); + const model = root.querySelector("[data-cookbook-model]"); + const source = root.querySelector("[data-cookbook-source]"); + const command = root.querySelector("[data-cookbook-command]"); + const status = root.querySelector("[data-cookbook-status]"); + + try { + const { recipes } = await loadRecipes(root.dataset.recipes); + const byId = new Map(recipes.map((recipe) => [recipe.id, recipe])); + const groups = new Map(); + + select.replaceChildren(); + recipes.forEach((recipe) => { + if (!groups.has(recipe.task)) { + const group = document.createElement("optgroup"); + group.label = recipe.task; + groups.set(recipe.task, group); + select.append(group); + } + groups.get(recipe.task).append(new Option(recipe.label, recipe.id)); + }); + + const render = () => { + const recipe = byId.get(select.value); + model.textContent = recipe.model; + source.textContent = recipe.source; + source.href = `https://github.com/hao-ai-lab/FastVideo/blob/main/${recipe.source}`; + command.textContent = recipe.command; + status.textContent = `${recipe.label} selected.`; + }; + + select.addEventListener("change", render); + select.disabled = false; + render(); + } catch (error) { + status.textContent = "Recipes could not be loaded. Use the examples link below."; + console.error("Failed to load FastVideo cookbook recipes", error); + } + }); + }; + + if (window.document$) window.document$.subscribe(init); + else document.addEventListener("DOMContentLoaded", init); +})(); diff --git a/docs/assets/custom.css b/docs/assets/custom.css index a8278f62e..b87ef67c8 100644 --- a/docs/assets/custom.css +++ b/docs/assets/custom.css @@ -42,6 +42,46 @@ img { margin: 0 auto; } +.cookbook-picker { + padding: 1rem; + border: 0.05rem solid var(--md-default-fg-color--lightest); + border-radius: 0.2rem; +} + +.cookbook-picker select { + width: 100%; + padding: 0.6rem; + color: var(--md-default-fg-color); + background: var(--md-default-bg-color); + border: 0.05rem solid var(--md-default-fg-color--lighter); + border-radius: 0.2rem; +} + +.cookbook-picker dl { + display: grid; + grid-template-columns: max-content 1fr; + gap: 0.25rem 1rem; +} + +.cookbook-picker dt { + font-weight: 700; +} + +.cookbook-picker dd { + margin: 0; + min-width: 0; + overflow-wrap: anywhere; +} + +.cookbook-picker__status { + position: absolute; + width: 1px; + height: 1px; + overflow: hidden; + clip: rect(0, 0, 0, 0); + white-space: nowrap; +} + .md-typeset .copy-page-button.md-button { float: right; margin: 0 0 1rem 1rem; diff --git a/docs/cookbook/index.md b/docs/cookbook/index.md new file mode 100644 index 000000000..b5d408f6c --- /dev/null +++ b/docs/cookbook/index.md @@ -0,0 +1,44 @@ +# Inference Cookbook + +Choose a complete recipe maintained in the FastVideo repository. Each command +runs its checked-in source directly, so coupled model, GPU, offload, and +attention settings do not drift into unsupported combinations. + +The commands expect a local clone: + +```bash +git clone https://github.com/hao-ai-lab/FastVideo.git +cd FastVideo +``` + +
+ + +
+
Model
+
Loading…
+
Source
+
Browse maintained examples
+
+ 
Loading…
+

+ +
+ +## Customize a recipe + +Start from the checked-in source, then change only the settings your model +supports: + +- [Configuration](../inference/configuration.md) covers the Python and CLI + config surfaces. +- [Optimizations](../inference/optimizations.md) covers attention backends, + compilation, and memory tradeoffs. +- [Support matrix](../inference/support_matrix.md) lists supported models and + optimizations. diff --git a/docs/generate_examples.py b/docs/generate_examples.py index 6a57397a7..1e85d5d4a 100644 --- a/docs/generate_examples.py +++ b/docs/generate_examples.py @@ -2,6 +2,7 @@ # adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/docs/source/generate_examples.py import itertools +import json import os import re from dataclasses import dataclass, field @@ -19,6 +20,40 @@ GENERATED_DOC_PREFIXES = ( "training/examples/", "distillation/examples/", ) +COOKBOOK_DATA = ROOT_DIR / "docs/assets/cookbook-recipes.json" +COOKBOOK_SOURCE_ROOTS = ( + ROOT_DIR / "examples/inference", + ROOT_DIR / "scripts/inference", +) + + +def validate_cookbook() -> None: + """Keep cookbook entries tied to checked-in runnable sources.""" + recipes = json.loads(COOKBOOK_DATA.read_text(encoding="utf-8")).get("recipes") + if not isinstance(recipes, list) or not recipes: + raise ValueError(f"{COOKBOOK_DATA}: recipes must be a non-empty list") + + seen: set[str] = set() + for recipe in recipes: + required = ("id", "task", "label", "model", "source", "command") + missing = {key for key in required if not recipe.get(key)} + if missing: + raise ValueError(f"Cookbook recipe is missing: {', '.join(sorted(missing))}") + if recipe["id"] in seen: + raise ValueError(f"Duplicate cookbook recipe id: {recipe['id']}") + seen.add(recipe["id"]) + + source = (ROOT_DIR / recipe["source"]).resolve() + if not any(source.is_relative_to(root.resolve()) for root in COOKBOOK_SOURCE_ROOTS): + raise ValueError(f"Cookbook source is outside an approved directory: {recipe['source']}") + if not source.is_file(): + raise ValueError(f"Cookbook source does not exist: {recipe['source']}") + + source_text = source.read_text(encoding="utf-8") + if recipe["model"] not in source_text: + raise ValueError(f"Cookbook model is not present in {recipe['source']}: {recipe['model']}") + if recipe["source"] not in recipe["command"]: + raise ValueError(f"Cookbook command does not invoke its source: {recipe['id']}") def fix_case(text: str) -> str: @@ -536,6 +571,7 @@ def on_pre_build(config, **kwargs): MkDocs hook to generate examples before building the documentation. This function is called automatically by MkDocs' native hook system. """ + validate_cookbook() print("Generating example documentation...") generate_examples(generate_main_index=True) print("Example documentation generated successfully!") @@ -549,6 +585,7 @@ def on_page_context(context, page, **kwargs): if __name__ == "__main__": + validate_cookbook() print("Generating example documentation...") generate_examples(generate_main_index=True) print("Example documentation generated successfully!") diff --git a/docs/getting_started/installation.md b/docs/getting_started/installation.md index 47e06bd39..6d5c29639 100644 --- a/docs/getting_started/installation.md +++ b/docs/getting_started/installation.md @@ -65,6 +65,7 @@ uv pip install flash-attn --no-build-isolation -v ## Next Steps -- [Quick Start Guide](quick_start.md) - Get started with your first video generation +- [Quick Start](quick_start.md) - Generate your first video +- [Inference Cookbook](../cookbook/index.md) - Choose a maintained recipe - [Configuration](../inference/configuration.md) - Learn about configuration options -- [Examples](../inference/examples/examples_inference_index.md) - Explore example scripts and notebooks +- [Examples](../inference/examples/examples_inference_index.md) - Explore scripts and notebooks diff --git a/docs/getting_started/quick_start.md b/docs/getting_started/quick_start.md index 644f0e672..34701f3c8 100644 --- a/docs/getting_started/quick_start.md +++ b/docs/getting_started/quick_start.md @@ -23,61 +23,21 @@ Also optionally install flash-attn: uv pip install flash-attn --no-build-isolation -v ``` -## Basic Usage +## Choose a maintained recipe -### Text-to-Video Generation +The cookbook selects complete, checked-in recipes instead of mixing model, +parallelism, offload, and attention settings independently. -```python -from fastvideo import VideoGenerator +[Open the inference cookbook](../cookbook/index.md){ .md-button .md-button--primary } -def main(): - # Create a video generator with a pre-trained model - generator = VideoGenerator.from_pretrained( - "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", - num_gpus=1, # Adjust based on your hardware - ) - - # Define a prompt for your video - prompt = "A curious raccoon peers through a vibrant field of yellow sunflowers, its eyes wide with interest." - - # Generate the video - video = generator.generate_video( - prompt, - output_path="my_videos/", # Controls where videos are saved - save_video=True - ) - -if __name__ == '__main__': - main() -``` - -### Image-to-Video Generation - -```python -from fastvideo import VideoGenerator, SamplingParam - -def main(): - # Create the generator - model_name = "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers" - generator = VideoGenerator.from_pretrained(model_name, num_gpus=1) - - # Set up parameters with an initial image - sampling_param = SamplingParam.from_pretrained(model_name) - sampling_param.image_path = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/astronaut.jpg" - sampling_param.num_frames = 107 - - # Generate video based on the image - prompt = "A photograph coming to life with gentle movement" - generator.generate_video(prompt, sampling_param=sampling_param, - output_path="my_videos/", - save_video=True) - -if __name__ == '__main__': - main() -``` +!!! tip "Need more control?" + Start from a maintained recipe, then use the + [configuration](../inference/configuration.md) and + [optimization](../inference/optimizations.md) guides for supported changes. ## Next Steps +- [Inference Cookbook](../cookbook/index.md) - Choose a maintained recipe - [Installation Guide](installation.md) - Detailed installation instructions - [Configuration](../inference/configuration.md) - Learn about configuration options - [Examples](../inference/examples/examples_inference_index.md) - Explore more diff --git a/mkdocs.yml b/mkdocs.yml index caf72e898..e2180d61c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -78,7 +78,7 @@ plugins: minify_js: true minify_css: true cache_safe: true - js_files: [assets/copy-page.js] + js_files: [assets/copy-page.js, assets/cookbook.js] css_files: [assets/custom.css] - api-autonav: modules: ["fastvideo"] @@ -154,6 +154,7 @@ nav: - Apple Silicon FastWan: getting_started/installation/mps.md - Quick Start: getting_started/quick_start.md - V1 API: getting_started/v1_api.md + - Cookbook: cookbook/index.md - Inference: - Quick Start: inference/inference_quick_start.md - Configuration: inference/configuration.md @@ -238,3 +239,4 @@ extra_css: # Custom JavaScript extra_javascript: - assets/copy-page.js + - assets/cookbook.js