1363 lines
74 KiB
JSON
1363 lines
74 KiB
JSON
{
|
||
"version": 20,
|
||
"recipes": [
|
||
{
|
||
"id": "fastwan21-t2v",
|
||
"family": "wan",
|
||
"stage": "inference",
|
||
"task": "Text to video",
|
||
"label": "FastWan2.1 1.3B (distilled + VSA)",
|
||
"summary": "Generate a video in three denoising steps with the distilled FastWan2.1 1.3B checkpoint and video sparse attention.",
|
||
"model": "FastVideo/FastWan2.1-T2V-1.3B-Diffusers",
|
||
"source": "scripts/inference/inference_wan_VSA_DMD_1_3B.yaml",
|
||
"serving": {
|
||
"source": "examples/serving/openai_fastwan21_1_3b.yaml",
|
||
"install": "UV_TORCH_BACKEND=cu130 uv pip install -e .",
|
||
"env": "FASTVIDEO_ATTENTION_BACKEND=VIDEO_SPARSE_ATTN"
|
||
},
|
||
"command": "FASTVIDEO_ATTENTION_BACKEND=VIDEO_SPARSE_ATTN fastvideo generate --config scripts/inference/inference_wan_VSA_DMD_1_3B.yaml",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"expected_artifact": "MP4 videos under outputs_video_dmd_1.3B/"
|
||
},
|
||
{
|
||
"id": "wan22-t2v",
|
||
"family": "wan",
|
||
"stage": "inference",
|
||
"task": "Text to video",
|
||
"label": "Wan2.2 A14B",
|
||
"summary": "The maintained high-capacity Wan2.2 text-to-video example with CPU offload settings encoded in its checked-in Python source.",
|
||
"model": "Wan-AI/Wan2.2-T2V-A14B-Diffusers",
|
||
"source": "examples/inference/basic/basic_wan2_2.py",
|
||
"serving": {
|
||
"source": "examples/serving/openai_wan22_t2v_a14b.yaml",
|
||
"install": "UV_TORCH_BACKEND=cu130 uv pip install -e .",
|
||
"limitations": [
|
||
"Generation at 1280 × 720 on two GPUs with DiT CPU offload is slow. The client examples stop polling after 30 minutes while the job keeps running on the server; retrieve it by ID or raise the deadline."
|
||
]
|
||
},
|
||
"command": "python examples/inference/basic/basic_wan2_2.py",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"gpu_count": 2, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"expected_artifact": "MP4 videos under video_samples_wan2_2_14B_t2v/"
|
||
},
|
||
{
|
||
"id": "wan21-i2v",
|
||
"family": "wan",
|
||
"stage": "inference",
|
||
"task": "Image to video",
|
||
"label": "Wan2.1 14B 480P",
|
||
"summary": "Animate an input image at 480P using the maintained Wan2.1 YAML configuration and its recorded offload settings.",
|
||
"model": "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
|
||
"source": "scripts/inference/inference_wan_i2v.yaml",
|
||
"command": "fastvideo generate --config scripts/inference/inference_wan_i2v.yaml",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"gpu_count": 2, "evidence": "source-configured"},
|
||
"evidence": "Source-backed"
|
||
},
|
||
{
|
||
"id": "wan22-ti2v",
|
||
"family": "wan",
|
||
"stage": "inference",
|
||
"task": "Text or image to video",
|
||
"label": "Wan2.2 TI2V 5B",
|
||
"summary": "Use one maintained 5B checkpoint for text-to-video or add an image input to switch the same recipe to image-to-video.",
|
||
"model": "Wan-AI/Wan2.2-TI2V-5B-Diffusers",
|
||
"source": "examples/inference/basic/basic_wan2_2_ti2v.py",
|
||
"serving": {
|
||
"source": "examples/serving/openai_wan22_ti2v_5b.yaml",
|
||
"install": "UV_TORCH_BACKEND=cu130 uv pip install -e .",
|
||
"task": "Text to video",
|
||
"limitations": [
|
||
"This server walkthrough sends a text prompt only. The server API also accepts an image reference (input_reference) for this checkpoint, but the checked-in clients do not upload one. For the cookbook's image-to-video workflow, use the Python command."
|
||
]
|
||
},
|
||
"command": "python examples/inference/basic/basic_wan2_2_ti2v.py",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"expected_artifact": "MP4 videos under video_samples_wan2_2_5B_ti2v/"
|
||
},
|
||
{
|
||
"id": "fastmetal-1-3b-mlx",
|
||
"family": "wan",
|
||
"stage": "inference",
|
||
"task": "Text to video",
|
||
"label": "FastMetal 1.3B",
|
||
"summary": "Run the released FastMetal 1.3B QAD checkpoint through FastVideo's native Apple Silicon MLX path.",
|
||
"model": "FastVideo/FastMetal-1.3B-QAD",
|
||
"source": "examples/inference/basic/mlx_wan_prompt_to_video.py",
|
||
"command": "hf download FastVideo/FastMetal-1.3B-QAD --local-dir ./FastMetal-1.3B-QAD\npython examples/inference/basic/mlx_wan_prompt_to_video.py --model-root ./FastMetal-1.3B-QAD --mlx-checkpoint ./FastMetal-1.3B-QAD --height 480 --width 832 --num-frames 81 --prompt \"A bird's-eye view of a misty forest valley at dawn.\" --output-path ./outputs/fastmetal_1_3b.mp4",
|
||
"gpu_types": ["Apple Silicon"],
|
||
"hardware": {
|
||
"platform": "mlx",
|
||
"accelerator": "Apple M4 Max",
|
||
"system_memory": "36 GB unified memory",
|
||
"minimum_memory": "16 GB+ unified memory",
|
||
"peak_memory": "3.87 GiB peak MLX memory",
|
||
"evidence": "validated",
|
||
"evidence_url": "https://github.com/hao-ai-lab/FastVideo/pull/1638"
|
||
},
|
||
"evidence": "Verified",
|
||
"expected_artifact": "MP4 video at outputs/fastmetal_1_3b.mp4",
|
||
"modes": ["T2V", "temporal --fast", "spatial --fast-spatial", "two-pass --refine"],
|
||
"limitations": [
|
||
"Native MLX FastMetal T2V. Add --fast for temporal RIFE, --fast-spatial to denoise at half resolution, or --refine for a two-pass upsample. basic_mps.py is the older PyTorch MPS demo."
|
||
]
|
||
},
|
||
{
|
||
"id": "fastmetal-5b-mlx",
|
||
"family": "wan",
|
||
"stage": "inference",
|
||
"task": "Text to video",
|
||
"label": "FastMetal 5B",
|
||
"summary": "Run the released Wan2.2 5B FastMetal checkpoint as MLX T2V with MLX DiT denoising and MLX TAEHV decode. The CUDA Wan2.2 TI2V 5B recipe is the image-capable path.",
|
||
"model": "FastVideo/FastMetal-5B-QAD",
|
||
"source": "examples/inference/basic/mlx_wan22_generate.py",
|
||
"command": "hf download FastVideo/FastMetal-5B-QAD --local-dir ./FastMetal-5B-QAD\npython examples/inference/basic/mlx_wan22_generate.py --mlx-checkpoint ./FastMetal-5B-QAD --text-encoder-root ./FastMetal-5B-QAD --vae-root ./FastMetal-5B-QAD/vae --height 704 --width 1280 --num-frames 81 --prompt \"A cinematic portrait with soft neon lighting and smooth camera motion.\" --output-path ./outputs/fastmetal_5b.mp4",
|
||
"gpu_types": ["Apple Silicon"],
|
||
"hardware": {
|
||
"platform": "mlx",
|
||
"accelerator": "Apple M4 Max",
|
||
"system_memory": "36 GB unified memory",
|
||
"minimum_memory": "16 GB+ unified memory",
|
||
"peak_memory": "9.34 GiB peak MLX memory",
|
||
"evidence": "validated",
|
||
"evidence_url": "https://github.com/hao-ai-lab/FastVideo/pull/1638"
|
||
},
|
||
"evidence": "Verified",
|
||
"expected_artifact": "MP4 video at outputs/fastmetal_5b.mp4",
|
||
"modes": ["T2V", "temporal --fast", "spatial --fast-spatial", "two-pass --refine"],
|
||
"limitations": [
|
||
"The checked-in MLX example is T2V. Image-to-video is not in mlx_wan22_generate.py. Add --fast, --fast-spatial, or --refine on the same script."
|
||
]
|
||
},
|
||
{
|
||
"id": "fastmetal-14b-mlx",
|
||
"family": "wan",
|
||
"stage": "inference",
|
||
"task": "Text to video",
|
||
"label": "FastMetal 14B",
|
||
"summary": "Run the released 14B FastMetal QAD checkpoint through the same Apple Silicon MLX entrypoint as the 1.3B release.",
|
||
"model": "FastVideo/FastMetal-14B-QAD",
|
||
"source": "examples/inference/basic/mlx_wan_prompt_to_video.py",
|
||
"command": "hf download FastVideo/FastMetal-14B-QAD --local-dir ./FastMetal-14B-QAD\npython examples/inference/basic/mlx_wan_prompt_to_video.py --model-root ./FastMetal-14B-QAD --mlx-checkpoint ./FastMetal-14B-QAD --height 480 --width 832 --num-frames 81 --prompt \"A wide cinematic landscape at sunrise.\" --output-path ./outputs/fastmetal_14b.mp4",
|
||
"gpu_types": ["Apple Silicon"],
|
||
"hardware": {
|
||
"platform": "mlx",
|
||
"accelerator": "Apple M4 Max",
|
||
"system_memory": "36 GB unified memory",
|
||
"minimum_memory": "36 GB+ unified memory",
|
||
"peak_memory": "21.68 GiB peak MLX memory",
|
||
"evidence": "validated",
|
||
"evidence_url": "https://github.com/hao-ai-lab/FastVideo/pull/1638"
|
||
},
|
||
"evidence": "Verified",
|
||
"expected_artifact": "MP4 video at outputs/fastmetal_14b.mp4",
|
||
"modes": ["T2V", "temporal --fast", "spatial --fast-spatial", "two-pass --refine"],
|
||
"limitations": [
|
||
"Native MLX FastMetal T2V on 36 GB+ unified memory. Same --fast, --fast-spatial, and --refine flags as the 1.3B script."
|
||
]
|
||
},
|
||
{
|
||
"id": "ltx2-distilled-t2v",
|
||
"family": "ltx2",
|
||
"stage": "inference",
|
||
"task": "Text to video",
|
||
"label": "LTX-2 distilled",
|
||
"summary": "The distilled LTX-2 text-to-video checkpoint with audio, from its maintained example. The source is configured for four GPUs.",
|
||
"model": "FastVideo/LTX2-Distilled-Diffusers",
|
||
"source": "examples/inference/basic/basic_ltx2_distilled.py",
|
||
"command": "python examples/inference/basic/basic_ltx2_distilled.py",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"gpu_count": 4, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"expected_artifact": "MP4 (video with audio) at outputs_video/ltx2_basic/output_ltx2_distilled_t2v.mp4",
|
||
"related": ["ltx23-base-t2v"]
|
||
},
|
||
{
|
||
"id": "ltx23-base-t2v",
|
||
"family": "ltx2",
|
||
"stage": "inference",
|
||
"task": "Text to video",
|
||
"label": "LTX-2 base (1088p)",
|
||
"summary": "Base LTX-2 text-to-video at 1088x1920 using FastVideo default sampling for LTX2 base. The example loads a community Diffusers mirror of the base checkpoint; registered aliases include Lightricks/LTX-2 and FastVideo/LTX2-Diffusers.",
|
||
"model": "Davids048/LTX2-Base-Diffusers",
|
||
"source": "examples/inference/basic/basic_ltx2.py",
|
||
"command": "python examples/inference/basic/basic_ltx2.py",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"expected_artifact": "MP4 (video with audio) at outputs_video/ltx2_basic/output_ltx2_base_t2v_1088_1920_1.1.mp4",
|
||
"limitations": ["The maintained example loads the Davids048/LTX2-Base-Diffusers community mirror rather than a Lightricks upstream ID."],
|
||
"related": ["ltx2-distilled-t2v"]
|
||
},
|
||
{
|
||
"id": "hy15-t2v-480p",
|
||
"family": "hunyuan",
|
||
"stage": "inference",
|
||
"task": "Text to video",
|
||
"label": "HunyuanVideo 1.5 480P",
|
||
"summary": "HunyuanVideo 1.5 text-to-video at 480P with CPU offload enabled in the checked-in source for smaller GPUs.",
|
||
"model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v",
|
||
"source": "examples/inference/basic/basic_hy15.py",
|
||
"command": "python examples/inference/basic/basic_hy15.py",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"expected_artifact": "MP4 videos under video_samples_hy15/",
|
||
"related": ["hy15-1080p-upscale"]
|
||
},
|
||
{
|
||
"id": "hy15-1080p-upscale",
|
||
"family": "hunyuan",
|
||
"stage": "inference",
|
||
"task": "Text to video (upscaled)",
|
||
"label": "HunyuanVideo 1.5 1080P upscale",
|
||
"summary": "Run HunyuanVideo 1.5 through the 480p to 720p to 1080p upscale chain in one maintained script.",
|
||
"model": "weizhou03/HunyuanVideo-1.5-Diffusers-1080p-2SR",
|
||
"source": "examples/inference/basic/basic_hy15_1080p.py",
|
||
"command": "python examples/inference/basic/basic_hy15_1080p.py",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"expected_artifact": "MP4 videos under video_samples_hy15_1080p/",
|
||
"related": ["hy15-t2v-480p"]
|
||
},
|
||
{
|
||
"id": "cosmos25-t2w",
|
||
"family": "cosmos",
|
||
"stage": "inference",
|
||
"task": "Text to world",
|
||
"label": "Cosmos Predict 2.5 2B",
|
||
"summary": "Generate a navigable world video from a text prompt with Cosmos Predict 2.5 2B on a single GPU.",
|
||
"model": "KyleShao/Cosmos-Predict2.5-2B-Diffusers",
|
||
"source": "examples/inference/basic/basic_cosmos2_5_t2w.py",
|
||
"command": "python examples/inference/basic/basic_cosmos2_5_t2w.py",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
||
"evidence": "Source-backed"
|
||
},
|
||
{
|
||
"id": "kandinsky5-t2v-lite-sft",
|
||
"family": "kandinsky5",
|
||
"stage": "inference",
|
||
"task": "Text to video",
|
||
"label": "Kandinsky 5.0 T2V Lite SFT",
|
||
"summary": "Kandinsky 5.0 text-to-video (Lite SFT variant) from the maintained example; alternative Lite/Pro checkpoints are listed in the source.",
|
||
"model": "kandinskylab/Kandinsky-5.0-T2V-Lite-sft-5s-Diffusers",
|
||
"source": "examples/inference/basic/basic_kandinsky5_t2v.py",
|
||
"command": "python examples/inference/basic/basic_kandinsky5_t2v.py",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {
|
||
"platform": "cuda",
|
||
"gpu_count": 1,
|
||
"accelerator": "NVIDIA B200",
|
||
"evidence": "validated",
|
||
"evidence_url": "https://github.com/hao-ai-lab/FastVideo/pull/1471"
|
||
},
|
||
"evidence": "Verified",
|
||
"expected_artifact": "MP4 videos under video_samples_kandinsky5_t2v/",
|
||
"related": ["kandinsky5-i2v-pro-distilled"]
|
||
},
|
||
{
|
||
"id": "kandinsky5-i2v-pro-distilled",
|
||
"family": "kandinsky5",
|
||
"stage": "inference",
|
||
"task": "Image to video",
|
||
"label": "Kandinsky 5.0 I2V Pro distilled",
|
||
"summary": "Animate an input image with Kandinsky 5.0 I2V Pro (distilled) on a single GPU.",
|
||
"model": "kandinskylab/Kandinsky-5.0-I2V-Pro-distilled-5s-Diffusers",
|
||
"source": "examples/inference/basic/basic_kandinsky5_i2v.py",
|
||
"command": "python examples/inference/basic/basic_kandinsky5_i2v.py",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {
|
||
"platform": "cuda",
|
||
"gpu_count": 1,
|
||
"accelerator": "NVIDIA B200",
|
||
"peak_memory": "10,365.89 MB peak GPU memory",
|
||
"evidence": "validated",
|
||
"evidence_url": "https://github.com/hao-ai-lab/FastVideo/pull/1471"
|
||
},
|
||
"evidence": "Verified",
|
||
"expected_artifact": "MP4 videos under video_samples_kandinsky5_i2v/",
|
||
"related": ["kandinsky5-t2v-lite-sft"]
|
||
},
|
||
{
|
||
"id": "kandinsky6-ti2va-base",
|
||
"family": "kandinsky6",
|
||
"stage": "inference",
|
||
"task": "Text or image to video with audio",
|
||
"label": "Kandinsky 6 Pro TI2VA",
|
||
"summary": "Generate a five-second, 512x768 video with synchronized audio from text, or set IMAGE_PATH in the maintained example to condition on an image.",
|
||
"model": "kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers",
|
||
"source": "examples/inference/basic/basic_kandinsky6_ti2va.py",
|
||
"command": "python examples/inference/basic/basic_kandinsky6_ti2va.py",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"platform": "cuda", "gpu_count": 1, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"expected_artifact": "MP4 video with synchronized audio under video_samples_kandinsky6_ti2va/",
|
||
"related": ["kandinsky6-ti2va-piflow", "kandinsky6-vsr"]
|
||
},
|
||
{
|
||
"id": "kandinsky6-ti2va-piflow",
|
||
"family": "kandinsky6",
|
||
"stage": "inference",
|
||
"task": "Distilled text or image to video with audio",
|
||
"label": "Kandinsky 6 Pro pi-Flow",
|
||
"summary": "Generate a five-second video with synchronized audio using the distilled pi-Flow checkpoint in ten inference steps and guidance 1.0.",
|
||
"model": "kandinskylab/Kandinsky-6.0-Pro-distill-5s-Diffusers",
|
||
"source": "examples/inference/basic/basic_kandinsky6_ti2va.py",
|
||
"command": "KANDINSKY6_MODEL_PATH=kandinskylab/Kandinsky-6.0-Pro-distill-5s-Diffusers python examples/inference/basic/basic_kandinsky6_ti2va.py",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"platform": "cuda", "gpu_count": 1, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"expected_artifact": "MP4 video with synchronized audio under video_samples_kandinsky6_ti2va/",
|
||
"related": ["kandinsky6-ti2va-base", "kandinsky6-vsr-distilled"]
|
||
},
|
||
{
|
||
"id": "kandinsky6-ti2va-lite",
|
||
"family": "kandinsky6",
|
||
"stage": "inference",
|
||
"task": "Text or image to video with audio",
|
||
"label": "Kandinsky 6 Lite TI2VA",
|
||
"summary": "Generate a five-second, 512x768 video with synchronized audio from text or an image with the 3.2B Lite checkpoint in fifty inference steps and guidance 5.0.",
|
||
"model": "kandinskylab/Kandinsky-6.0-Lite-5s-Diffusers",
|
||
"source": "examples/inference/basic/basic_kandinsky6_ti2va.py",
|
||
"command": "KANDINSKY6_MODEL_PATH=kandinskylab/Kandinsky-6.0-Lite-5s-Diffusers python examples/inference/basic/basic_kandinsky6_ti2va.py",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"platform": "cuda", "gpu_count": 1, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"expected_artifact": "MP4 video with synchronized audio under video_samples_kandinsky6_ti2va/",
|
||
"related": ["kandinsky6-ti2va-lite-piflow", "kandinsky6-ti2va-base"]
|
||
},
|
||
{
|
||
"id": "kandinsky6-ti2va-lite-piflow",
|
||
"family": "kandinsky6",
|
||
"stage": "inference",
|
||
"task": "Distilled text or image to video with audio",
|
||
"label": "Kandinsky 6 Lite pi-Flow",
|
||
"summary": "Generate a five-second video with synchronized audio using the 3.2B Lite distilled pi-Flow checkpoint in ten inference steps and guidance 1.0.",
|
||
"model": "kandinskylab/Kandinsky-6.0-Lite-distill-5s-Diffusers",
|
||
"source": "examples/inference/basic/basic_kandinsky6_ti2va.py",
|
||
"command": "KANDINSKY6_MODEL_PATH=kandinskylab/Kandinsky-6.0-Lite-distill-5s-Diffusers python examples/inference/basic/basic_kandinsky6_ti2va.py",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"platform": "cuda", "gpu_count": 1, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"expected_artifact": "MP4 video with synchronized audio under video_samples_kandinsky6_ti2va/",
|
||
"related": ["kandinsky6-ti2va-lite", "kandinsky6-ti2va-piflow"]
|
||
},
|
||
{
|
||
"id": "kandinsky6-vsr",
|
||
"family": "kandinsky6",
|
||
"stage": "inference",
|
||
"task": "Video super-resolution",
|
||
"label": "Kandinsky 6 VSR",
|
||
"summary": "Upscale an existing clip by 2.25x with the four-step flow-matching VSR checkpoint while preserving its source audio.",
|
||
"model": "kandinskylab/Kandinsky-6.0-VSR-5s-Diffusers",
|
||
"source": "examples/inference/basic/basic_kandinsky6_sr.py",
|
||
"command": ": \"${INPUT_VIDEO:?Set INPUT_VIDEO to an existing video}\"\npython examples/inference/basic/basic_kandinsky6_sr.py --video-path \"$INPUT_VIDEO\" --scale 2.25",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"platform": "cuda", "gpu_count": 1, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"expected_artifact": "Upscaled MP4 under outputs_video/kandinsky6_sr/",
|
||
"related": ["kandinsky6-vsr-distilled", "kandinsky6-ti2va-base"]
|
||
},
|
||
{
|
||
"id": "kandinsky6-vsr-distilled",
|
||
"family": "kandinsky6",
|
||
"stage": "inference",
|
||
"task": "Distilled video super-resolution",
|
||
"label": "Kandinsky 6 VSR distilled",
|
||
"summary": "Upscale an existing clip by 2.25x with the distilled two-step VSR checkpoint while preserving its source audio.",
|
||
"model": "kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers",
|
||
"source": "examples/inference/basic/basic_kandinsky6_sr.py",
|
||
"command": ": \"${INPUT_VIDEO:?Set INPUT_VIDEO to an existing video}\"\npython examples/inference/basic/basic_kandinsky6_sr.py --model-path kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers --video-path \"$INPUT_VIDEO\" --scale 2.25",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"platform": "cuda", "gpu_count": 1, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"expected_artifact": "Upscaled MP4 under outputs_video/kandinsky6_sr/",
|
||
"related": ["kandinsky6-vsr", "kandinsky6-ti2va-piflow"]
|
||
},
|
||
{
|
||
"id": "flux2-klein-t2i",
|
||
"family": "flux",
|
||
"stage": "inference",
|
||
"task": "Text to image",
|
||
"label": "FLUX.2 Klein 4B",
|
||
"summary": "Generate an image in four denoising steps with the distilled FLUX.2 Klein checkpoint.",
|
||
"model": "black-forest-labs/FLUX.2-klein-4B",
|
||
"source": "examples/inference/basic/basic_flux2_klein.py",
|
||
"command": "python examples/inference/basic/basic_flux2_klein.py",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"expected_artifact": "PNG image at outputs/flux2/flux2_klein.png",
|
||
"related": ["flux2-dev-t2i"]
|
||
},
|
||
{
|
||
"id": "flux2-dev-t2i",
|
||
"family": "flux",
|
||
"stage": "inference",
|
||
"task": "Text to image",
|
||
"label": "FLUX.2 dev",
|
||
"summary": "Full FLUX.2 dev text-to-image with embedded guidance and the Mistral3 text encoder, from its maintained example.",
|
||
"model": "black-forest-labs/FLUX.2-dev",
|
||
"source": "examples/inference/basic/basic_flux2.py",
|
||
"command": "python examples/inference/basic/basic_flux2.py",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"expected_artifact": "PNG image at outputs/flux2/flux2.png",
|
||
"related": ["flux2-klein-t2i"]
|
||
},
|
||
{
|
||
"id": "flux1-dev-t2i",
|
||
"family": "flux",
|
||
"stage": "inference",
|
||
"task": "Text to image",
|
||
"label": "FLUX.1 dev",
|
||
"summary": "FLUX.1 dev text-to-image through the Diffusers-backed pipeline. The example defaults to a local weights directory, so this recipe passes the Hugging Face ID explicitly.",
|
||
"model": "black-forest-labs/FLUX.1-dev",
|
||
"source": "examples/inference/basic/basic_flux_dev.py",
|
||
"command": "python examples/inference/basic/basic_flux_dev.py --model-path black-forest-labs/FLUX.1-dev",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"expected_artifact": "PNG images under outputs/flux_dev/samples/",
|
||
"limitations": ["FLUX.1 is loadable by ID but registers no model_family in fastvideo/registry.py; it is grouped under FLUX for documentation only."]
|
||
},
|
||
{
|
||
"id": "qwen-image21-t2i",
|
||
"family": "qwen_image21",
|
||
"stage": "inference",
|
||
"task": "Text to image",
|
||
"label": "Qwen-Image-2.1",
|
||
"summary": "Generate a 1024 × 1024 image in 40 steps on one CUDA GPU, with the CPU offload settings recorded in the maintained YAML.",
|
||
"model": "Qwen/Qwen-Image-2.1",
|
||
"source": "examples/inference/basic/qwen_image21_t2i.yaml",
|
||
"command": "fastvideo generate --config examples/inference/basic/qwen_image21_t2i.yaml",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"platform": "cuda", "gpu_count": 1, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"expected_artifact": "RGBA PNG image at outputs/qwen_image21/teapot.png",
|
||
"limitations": ["The source uses DiT layerwise offload, text-encoder and VAE offload, and lazy loading. GPU memory requirements have not been established for this recipe."],
|
||
"related": ["qwen-image21-i2i"]
|
||
},
|
||
{
|
||
"id": "qwen-image21-i2i",
|
||
"family": "qwen_image21",
|
||
"stage": "inference",
|
||
"task": "Image to image",
|
||
"label": "Qwen-Image-2.1 references and editing",
|
||
"summary": "Generate or edit an image from a prompt and up to ten ordered reference images, using the maintained single-GPU Python example.",
|
||
"model": "Qwen/Qwen-Image-2.1",
|
||
"source": "examples/inference/basic/basic_qwen_image21.py",
|
||
"command": "python examples/inference/basic/basic_qwen_image21.py --reference source.png --prompt 'Change the shirt to blue; retain the subject and composition.' --output outputs/qwen_image21/edit.png",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"platform": "cuda", "gpu_count": 1, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"expected_artifact": "RGBA PNG image at outputs/qwen_image21/edit.png",
|
||
"limitations": [
|
||
"Replace source.png with an existing image. Repeat --reference for additional images and describe their order in the prompt.",
|
||
"Mask images use the same ordered-reference path; they do not impose hard pixel-preservation constraints.",
|
||
"The source enables CPU offload and lazy loading. This entry does not establish a minimum GPU memory requirement."
|
||
],
|
||
"related": ["qwen-image21-t2i"]
|
||
},
|
||
{
|
||
"id": "sd35-medium-t2i",
|
||
"family": "sd35",
|
||
"stage": "inference",
|
||
"task": "Text to image",
|
||
"label": "Stable Diffusion 3.5 Medium",
|
||
"summary": "Stable Diffusion 3.5 Medium text-to-image over a small built-in prompt set, from its maintained example.",
|
||
"model": "stabilityai/stable-diffusion-3.5-medium",
|
||
"source": "examples/inference/basic/basic_sd35_t2i.py",
|
||
"command": "python examples/inference/basic/basic_sd35_t2i.py",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"expected_artifact": "PNG images under outputs/sd35/samples/"
|
||
},
|
||
{
|
||
"id": "minimax-h3-t2v",
|
||
"family": "minimax_h3",
|
||
"stage": "inference",
|
||
"task": "Text to video (with audio)",
|
||
"label": "MiniMax H3 T2VA",
|
||
"summary": "Generate synchronized video and stereo audio from a structured text prompt with the full MiniMax H3 checkpoint.",
|
||
"model": "MiniMaxAI/MiniMax-H3",
|
||
"source": "examples/inference/basic/basic_minimax_h3_t2v.py",
|
||
"command": "python examples/inference/basic/basic_minimax_h3_t2v.py --prompt \"(S1) A presenter says <d>[English] FastVideo runs MiniMax H3.</d>\"",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"platform": "cuda", "gpu_count": 4, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"expected_artifact": "MP4 with synchronized audio at outputs/minimax_h3_t2v/minimax_h3_t2v.mp4",
|
||
"modes": ["T2VA"],
|
||
"knobs": [
|
||
{"key": "num_gpus", "label": "GPUs", "hint": "Sequence-parallel degree", "flag": "--num-gpus", "options": [1, 2, 4, 8], "default": 4}
|
||
],
|
||
"limitations": ["The checked-in example defaults to four-way sequence parallelism. It does not record a GPU model or memory requirement."]
|
||
},
|
||
{
|
||
"id": "fasth3-preview-cuda",
|
||
"group": "fasth3-preview",
|
||
"group_label": "FastH3 V1",
|
||
"group_task": "4-step text to video + audio",
|
||
"family": "minimax_h3",
|
||
"stage": "inference",
|
||
"task": "Few-step text to video (with audio)",
|
||
"label": "FastH3 V1 on CUDA",
|
||
"summary": "Run FastH3 V1 with four DiT forwards, trained H3 sparse attention, compiled decode, and synchronized audio.",
|
||
"model": "FastVideo/FastVideo-Minimax-FastH3-Preview-v0.2",
|
||
"source": "examples/inference/basic/basic_fasth3.py",
|
||
"serving": {
|
||
"source": "examples/serving/openai_fasth3.yaml",
|
||
"install": "UV_TORCH_BACKEND=cu130 uv pip install -e \".[fasth3]\"",
|
||
"audio": true
|
||
},
|
||
"command": "UV_TORCH_BACKEND=cu130 uv pip install -e \".[fasth3]\"\npython examples/inference/basic/basic_fasth3.py --prompt \"(S1) A presenter says <d>[English] FastVideo runs FastH3.</d>\" --profile all",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {
|
||
"platform": "cuda",
|
||
"gpu_count": 4,
|
||
"accelerator": "NVIDIA GB200",
|
||
"evidence": "validated",
|
||
"evidence_url": "https://github.com/hao-ai-lab/FastVideo/pull/1731"
|
||
},
|
||
"evidence": "Verified",
|
||
"expected_artifact": "Warmup and measured MP4 files under outputs/fasth3/",
|
||
"modes": ["T2VA", "4-step FastH3"],
|
||
"knobs": [
|
||
{"key": "num_gpus", "label": "GPUs", "hint": "Sequence-parallel degree", "flag": "--num-gpus", "options": [1, 2, 4, 8], "default": 4},
|
||
{"key": "video_decode_backend", "label": "VAE decode", "hint": "Fidelity vs. speed", "flag": "--video-decode-backend", "options": [{"value": "h3-vae", "label": "Full H3 VAE"}, {"value": "taeh3", "label": "TAEH3 preview"}], "default": "h3-vae"}
|
||
],
|
||
"limitations": ["The default all profile is the measured GB200 performance route and can change floating-point operation order. Use --profile strict --no-inference-torch-compile for the eager strict route."]
|
||
},
|
||
{
|
||
"id": "fasth3-preview-mlx",
|
||
"serving": {
|
||
"source": "examples/serving/mlx_fasth3.yaml",
|
||
"install": "uv pip install -e \".[mlx]\"",
|
||
"prepare": "hf download FastVideo/FastVideo-Minimax-FastH3-Preview-v0.2 --local-dir ./FastH3-Preview-v0.2\npython scripts/checkpoint_conversion/convert_minimax_h3_mlx.py --model-root ./FastH3-Preview-v0.2/transformer --out ./FastH3-MLX --formats \"int6\"",
|
||
"audio": true
|
||
},
|
||
"group": "fasth3-preview",
|
||
"group_label": "FastH3 V1",
|
||
"group_task": "4-step text to video + audio",
|
||
"family": "minimax_h3",
|
||
"stage": "inference",
|
||
"task": "Few-step text to video (with audio)",
|
||
"label": "FastH3 V1 on MLX",
|
||
"summary": "Run FastH3 V1 on Apple Silicon with a locally converted INT6 DiT, streamed Qwen3-VL conditioning, and native MLX video and audio VAEs.",
|
||
"model": "FastVideo/FastVideo-Minimax-FastH3-Preview-v0.2",
|
||
"source": "examples/inference/basic/mlx_fasth3.py",
|
||
"command": "hf download FastVideo/FastVideo-Minimax-FastH3-Preview-v0.2 --local-dir ./FastH3-Preview-v0.2\npython scripts/checkpoint_conversion/convert_minimax_h3_mlx.py --model-root ./FastH3-Preview-v0.2/transformer --out ./FastH3-MLX --formats \"int6\"\npython examples/inference/basic/mlx_fasth3.py --model-root ./FastH3-Preview-v0.2 --mlx-checkpoint ./FastH3-MLX/int6 --prompt \"(S1) A presenter says <d>[English] FastVideo runs FastH3.</d>\" --height 480 --width 832 --num-frames 124 --seed 2026 --output-path ./outputs/fasth3_int6.mp4",
|
||
"gpu_types": ["Apple Silicon"],
|
||
"hardware": {
|
||
"platform": "mlx",
|
||
"accelerator": "Apple M4 Max",
|
||
"system_memory": "36 GB unified memory",
|
||
"peak_memory": "19.63 GiB peak MLX memory during denoising",
|
||
"evidence": "validated",
|
||
"evidence_url": "https://github.com/hao-ai-lab/FastVideo/pull/1770"
|
||
},
|
||
"evidence": "Verified",
|
||
"expected_artifact": "MP4 with H.264 video and stereo AAC audio at outputs/fasth3_int6.mp4",
|
||
"modes": ["T2VA", "temporal --fast", "spatial --fast-spatial", "opt-in VSA"],
|
||
"knobs": [
|
||
{"key": "video_decode_backend", "label": "VAE decode", "hint": "Fidelity vs. speed", "flag": "--video-decode-backend", "options": [{"value": "h3-vae", "label": "Full H3 VAE"}, {"value": "taeh3", "label": "TAEH3 preview"}], "default": "h3-vae"}
|
||
],
|
||
"limitations": [
|
||
"The MLX path supports T2VA, optional temporal --fast, optional spatial --fast-spatial, and opt-in VSA on --include-vsa checkpoints. FL2VA, Ref2VA, and two-pass refinement are not wired."
|
||
]
|
||
},
|
||
{
|
||
"id": "fasth3-preview-spark",
|
||
"group": "fasth3-preview",
|
||
"group_label": "FastH3 V1",
|
||
"group_task": "4-step text to video + audio",
|
||
"family": "minimax_h3",
|
||
"stage": "inference",
|
||
"task": "Few-step text to video (with audio)",
|
||
"label": "FastH3 V1 on one DGX Spark",
|
||
"summary": "Run FastH3 V1 on one GB10 with Triton VSA, FA4 off, and lazy module load. Height, width, frames, and steps in the YAML are examples.",
|
||
"model": "FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree",
|
||
"source": "examples/inference/basic/basic_fasth3_spark.yaml",
|
||
"serving": {
|
||
"source": "examples/serving/openai_fasth3_spark.yaml",
|
||
"install": "UV_TORCH_BACKEND=cu130 uv pip install -e .",
|
||
"env": "FASTVIDEO_VSA_PTX_BLACKWELL=0 FASTVIDEO_FA4=0 FASTVIDEO_ATTENTION_BACKEND=VIDEO_SPARSE_ATTN_H3",
|
||
"audio": true
|
||
},
|
||
"command": "FASTVIDEO_VSA_PTX_BLACKWELL=0 FASTVIDEO_FA4=0 FASTVIDEO_ATTENTION_BACKEND=VIDEO_SPARSE_ATTN_H3 FASTVIDEO_STAGE_LOGGING=1 fastvideo generate --config examples/inference/basic/basic_fasth3_spark.yaml",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {
|
||
"platform": "cuda",
|
||
"device": "spark",
|
||
"gpu_count": 1,
|
||
"evidence": "source-configured"
|
||
},
|
||
"evidence": "Source-backed",
|
||
"expected_artifact": "MP4 under outputs/fasth3_spark/",
|
||
"modes": ["T2VA", "1-Spark"],
|
||
"limitations": [
|
||
"Install from the DGX Spark guide, not the generic CUDA extra. GB10 has no FA4 / sm_100a VSA kernel; keep FASTVIDEO_FA4=0 and FASTVIDEO_VSA_PTX_BLACKWELL=0.",
|
||
"Legal num_frames values are 17n+5, capped at 345 (15 s). Native 16:9 sizes include 832x480 and 1344x768.",
|
||
"Lazy module load reloads Qwen3-VL and the DiT between phases of each request. Do not pass --no-lazy-module-load on this box.",
|
||
"A 345-frame request on one Spark can OOM. Prefer 124 or 243 frames, TAEH3 decode, or two Sparks over QSFP."
|
||
]
|
||
},
|
||
{
|
||
"id": "fasth3-spark-pair",
|
||
"group": "fasth3-preview",
|
||
"group_label": "FastH3 V1",
|
||
"group_task": "4-step text to video + audio",
|
||
"family": "minimax_h3",
|
||
"stage": "inference",
|
||
"task": "Few-step text to video (with audio)",
|
||
"label": "FastH3 V1 on two DGX Sparks",
|
||
"summary": "Run one FastH3 clip across two GB10s with Ray sequence parallel over QSFP RoCE. Sequential load and lazy module load stay on because SP replicates the DiT on each node.",
|
||
"model": "FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree",
|
||
"source": "examples/inference/basic/basic_fasth3_spark_pair.yaml",
|
||
"command": "source examples/inference/optimizations/spark_pair_env.sh && FASTVIDEO_VSA_PTX_BLACKWELL=0 FASTVIDEO_FA4=0 FASTVIDEO_ATTENTION_BACKEND=VIDEO_SPARSE_ATTN_H3 FASTVIDEO_VAE_PARALLEL_DECODE=1 fastvideo generate --config examples/inference/basic/basic_fasth3_spark_pair.yaml",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {
|
||
"platform": "cuda",
|
||
"device": "spark",
|
||
"gpu_count": 2,
|
||
"accelerator": "NVIDIA GB10 (DGX Spark pair)",
|
||
"evidence": "validated",
|
||
"evidence_url": "https://github.com/hao-ai-lab/FastVideo/pull/1803"
|
||
},
|
||
"evidence": "Verified",
|
||
"expected_artifact": "MP4 under outputs/fasth3_spark_pair/",
|
||
"modes": ["T2VA", "2-Spark SP"],
|
||
"limitations": [
|
||
"Requires a two-node Ray cluster on the QSFP interconnect. There is no cookbook server for this path; use Python / generate.",
|
||
"Height, width, frames, and steps in the YAML are examples. Edit them or pass CLI flags. See docs/getting_started/installation/spark_pair.md."
|
||
]
|
||
},
|
||
{
|
||
"id": "fasth3-8step-v2-cuda",
|
||
"group": "fasth3-8step-v2",
|
||
"group_label": "FastH3 V2",
|
||
"group_task": "8-step text to video + audio",
|
||
"family": "minimax_h3",
|
||
"stage": "inference",
|
||
"task": "Few-step text to video (with audio)",
|
||
"label": "FastH3 V2 on CUDA",
|
||
"summary": "Run FastH3 V2, the eight-forward checkpoint (video/audio shifts 10/3, VSA 0.8, 64-token tiles) with the trained DMD ladder loaded from the checkpoint's fastvideo_inference.json.",
|
||
"model": "FastVideo/FastVideo-FastH3-8-Step-V2",
|
||
"source": "examples/inference/basic/basic_fasth3_8step.py",
|
||
"serving": {
|
||
"source": "examples/serving/openai_fasth3_8step.yaml",
|
||
"install": "UV_TORCH_BACKEND=cu130 uv pip install -e \".[fasth3]\"",
|
||
"audio": true
|
||
},
|
||
"command": "UV_TORCH_BACKEND=cu130 uv pip install -e \".[fasth3]\"\npython examples/inference/basic/basic_fasth3_8step.py --prompt \"(S1) A presenter says <d>[English] FastVideo runs FastH3.</d>\" --profile strict --no-inference-torch-compile --no-compile-vae",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {
|
||
"platform": "cuda",
|
||
"gpu_count": 4,
|
||
"accelerator": "NVIDIA GB200",
|
||
"evidence": "validated",
|
||
"evidence_url": "https://github.com/hao-ai-lab/FastVideo/pull/1852"
|
||
},
|
||
"evidence": "Verified",
|
||
"expected_artifact": "Warmup and measured MP4 files under outputs/fasth3_8step/",
|
||
"modes": ["T2VA", "8-step FastH3"],
|
||
"knobs": [
|
||
{"key": "num_gpus", "label": "GPUs", "hint": "Sequence-parallel degree", "flag": "--num-gpus", "options": [1, 2, 4, 8], "default": 4},
|
||
{"key": "video_decode_backend", "label": "VAE decode", "hint": "Fidelity vs. speed", "flag": "--video-decode-backend", "options": [{"value": "h3-vae", "label": "Full H3 VAE"}, {"value": "taeh3", "label": "TAEH3 preview"}], "default": "h3-vae"}
|
||
],
|
||
"limitations": [
|
||
"Nine sigma-grid points (eight transformer forwards) are fixed by the checkpoint's trained ladder; the example rejects any other --steps.",
|
||
"Validated with the eager strict route (--profile strict --no-inference-torch-compile --no-compile-vae). The compiled all profile has not been measured for this checkpoint.",
|
||
"T2AV only; no FL2VA/Ref2VA distillation and no matching LoRA. V2 uses eight forwards rather than V1's four."
|
||
]
|
||
},
|
||
{
|
||
"id": "fasth3-8step-v2-mlx",
|
||
"serving": {
|
||
"source": "examples/serving/mlx_fasth3_8step.yaml",
|
||
"install": "uv pip install -e \".[mlx]\"",
|
||
"prepare": "hf download FastVideo/FastVideo-FastH3-8-Step-V2 --local-dir ./FastH3-8-Step-V2\npython scripts/checkpoint_conversion/convert_minimax_h3_mlx.py --model-root ./FastH3-8-Step-V2/transformer --out ./FastH3-8-Step-V2-MLX --formats \"int8\" --include-vsa",
|
||
"audio": true
|
||
},
|
||
"group": "fasth3-8step-v2",
|
||
"group_label": "FastH3 V2",
|
||
"group_task": "8-step text to video + audio",
|
||
"family": "minimax_h3",
|
||
"stage": "inference",
|
||
"task": "Few-step text to video (with audio)",
|
||
"label": "FastH3 V2 on MLX",
|
||
"summary": "Run FastH3 V2 on Apple Silicon with a locally converted INT8 DiT, the checkpoint's DMD contract ladder, and trained VSA (sparsity 0.8, 64-token tiles).",
|
||
"model": "FastVideo/FastVideo-FastH3-8-Step-V2",
|
||
"source": "examples/inference/basic/mlx_fasth3_8step.py",
|
||
"command": "hf download FastVideo/FastVideo-FastH3-8-Step-V2 --local-dir ./FastH3-8-Step-V2\npython scripts/checkpoint_conversion/convert_minimax_h3_mlx.py --model-root ./FastH3-8-Step-V2/transformer --out ./FastH3-8-Step-V2-MLX --formats \"int8\" --include-vsa\npython examples/inference/basic/mlx_fasth3_8step.py --model-root ./FastH3-8-Step-V2 --mlx-checkpoint ./FastH3-8-Step-V2-MLX/int8 --prompt \"(S1) A presenter says <d>[English] FastVideo runs FastH3.</d>\" --height 480 --width 832 --num-frames 124 --seed 2026 --output-path ./outputs/fasth3_8step_int8.mp4",
|
||
"gpu_types": ["Apple Silicon"],
|
||
"hardware": {
|
||
"platform": "mlx",
|
||
"accelerator": "Apple M4 Max",
|
||
"system_memory": "36 GB unified memory",
|
||
"peak_memory": "27.39 GiB peak MLX memory during denoising",
|
||
"evidence": "validated",
|
||
"evidence_url": "https://github.com/hao-ai-lab/FastVideo/pull/1863"
|
||
},
|
||
"evidence": "Verified",
|
||
"expected_artifact": "MP4 with H.264 video and stereo AAC audio at outputs/fasth3_8step_int8.mp4",
|
||
"modes": ["T2VA", "8-step FastH3", "trained VSA"],
|
||
"knobs": [
|
||
{"key": "video_decode_backend", "label": "VAE decode", "hint": "Fidelity vs. speed", "flag": "--video-decode-backend", "options": [{"value": "h3-vae", "label": "Full H3 VAE"}, {"value": "taeh3", "label": "TAEH3 preview"}], "default": "h3-vae"}
|
||
],
|
||
"limitations": [
|
||
"Convert with --include-vsa. mlx_fasth3_8step.py turns VSA on (sparsity 0.8, tile 64). A dense export fails at configure_vsa.",
|
||
"--steps 8 or 9 both run the eight trained forwards. Reuse the preview VAE, audio VAE, text encoder, and tokenizer if those directories already exist.",
|
||
"The MLX path supports T2VA only. FL2VA, Ref2VA, and two-pass refinement are not wired."
|
||
]
|
||
},
|
||
{
|
||
"id": "minimax-h3-fl2va",
|
||
"family": "minimax_h3",
|
||
"stage": "inference",
|
||
"task": "First/last frame to video (with audio)",
|
||
"label": "MiniMax H3 FL2VA",
|
||
"summary": "Animate a first frame, optionally guide the final frame, and generate synchronized audio with the full MiniMax H3 checkpoint.",
|
||
"model": "MiniMaxAI/MiniMax-H3",
|
||
"source": "examples/inference/basic/basic_minimax_h3_fl2va.py",
|
||
"command": "python examples/inference/basic/basic_minimax_h3_fl2va.py --image path/to/first-frame.png --prompt \"(S1) The subject turns toward the camera and says <d>[English] Hello.</d>\"",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"platform": "cuda", "gpu_count": 4, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"expected_artifact": "MP4 with synchronized audio at outputs/minimax_h3_fl2va/minimax_h3_fl2va.mp4",
|
||
"modes": ["FL2VA"],
|
||
"knobs": [
|
||
{"key": "num_gpus", "label": "GPUs", "hint": "Sequence-parallel degree", "flag": "--num-gpus", "options": [1, 2, 4, 8], "default": 4}
|
||
],
|
||
"limitations": ["Pass --last-image to constrain the final frame. The checked-in source defaults to four GPUs."]
|
||
},
|
||
{
|
||
"id": "minimax-h3-ref2va",
|
||
"family": "minimax_h3",
|
||
"stage": "inference",
|
||
"task": "Reference media to video (with audio)",
|
||
"label": "MiniMax H3 Ref2VA",
|
||
"summary": "Condition H3 on an ordered reference video and optional audio reference, then generate a new synchronized video and audio result.",
|
||
"model": "MiniMaxAI/MiniMax-H3",
|
||
"source": "examples/inference/basic/basic_minimax_h3_ref2va.py",
|
||
"command": "python examples/inference/basic/basic_minimax_h3_ref2va.py --reference-video path/to/reference.mp4 --prompt \"Create a new scene that preserves the reference identity and motion language.\"",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"platform": "cuda", "gpu_count": 4, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"expected_artifact": "MP4 with synchronized audio at outputs/minimax_h3_ref2va/minimax_h3_ref2va.mp4",
|
||
"modes": ["Ref2VA"],
|
||
"knobs": [
|
||
{"key": "num_gpus", "label": "GPUs", "hint": "Sequence-parallel degree", "flag": "--num-gpus", "options": [1, 2, 4, 8], "default": 4}
|
||
],
|
||
"limitations": ["Pass --reference-audio for an additional audio reference. The checked-in source defaults to four GPUs."]
|
||
},
|
||
{
|
||
"id": "pika-h3-flex-t2va",
|
||
"family": "pika_h3_flex",
|
||
"stage": "inference",
|
||
"task": "Text to video (with audio)",
|
||
"label": "Pika H3 Flex",
|
||
"summary": "Pika's post-trained MiniMax-H3: synchronized video and stereo audio from text in six forwards, at 480p or 768p, 5-15 s, in BF16 or as FastVideo's NVFP4 QAD release on Blackwell, with optional FP4 attention.",
|
||
"model": "Pika-Labs/Pika-H3-Flex",
|
||
"source": "examples/inference/basic/basic_pika_h3_flex.py",
|
||
"command": "FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN FASTVIDEO_FA4=1 python examples/inference/basic/basic_pika_h3_flex.py --task t2va --offload resident --prompt 'A paper boat drifts down a rainy city street at night; rain patters on the water.'",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {
|
||
"platform": "cuda",
|
||
"gpu_count": 1,
|
||
"accelerator": "NVIDIA GB200",
|
||
"peak_memory": "About 129 GiB resident (BF16, 768p, 5 s)",
|
||
"evidence": "validated",
|
||
"evidence_url": "https://github.com/hao-ai-lab/FastVideo/pull/2038"
|
||
},
|
||
"evidence": "Verified",
|
||
"expected_artifact": "MP4 with synchronized stereo audio at outputs/pika_h3_flex/pika_h3_flex_t2va.mp4",
|
||
"modes": ["T2VA"],
|
||
"knobs": [
|
||
{
|
||
"key": "weights",
|
||
"label": "Weights",
|
||
"hint": "NVFP4 QAD needs Blackwell",
|
||
"flag": "--weights",
|
||
"options": [{"value": "bf16", "label": "BF16 (Pika release)"}, {"value": "nvfp4-qad", "label": "NVFP4 QAD"}, {"value": "nvfp4-qad-fp4-attention", "label": "NVFP4 QAD + FP4 attention"}],
|
||
"default": "bf16"
|
||
},
|
||
{
|
||
"key": "resolution",
|
||
"label": "Resolution",
|
||
"hint": "Pika resolution tier",
|
||
"flag": "--resolution",
|
||
"options": [{"value": "768p", "label": "768p"}, {"value": "480p", "label": "480p"}],
|
||
"default": "768p"
|
||
},
|
||
{
|
||
"key": "aspect",
|
||
"label": "Aspect ratio",
|
||
"hint": "Bucket within the tier",
|
||
"flag": "--aspect",
|
||
"options": ["21:9", "16:9", "3:2", "4:3", "5:4", "1:1", "4:5", "3:4", "2:3", "9:16", "9:21"],
|
||
"default": "16:9"
|
||
},
|
||
{
|
||
"key": "duration",
|
||
"label": "Duration",
|
||
"hint": "Rounded up to the 17n+5 frame grid",
|
||
"flag": "--duration",
|
||
"options": [
|
||
{"value": 5, "label": "5 s (124 frames)"},
|
||
{"value": 10, "label": "10 s (243 frames)"},
|
||
{"value": 15, "label": "15 s (362 frames)"}
|
||
],
|
||
"default": 5
|
||
},
|
||
{
|
||
"key": "nvfp4_decoder",
|
||
"label": "VAE decoder",
|
||
"hint": "NVFP4 needs Blackwell",
|
||
"flag": "--nvfp4-decoder",
|
||
"options": [
|
||
{"value": "", "label": "BF16 H3 VAE"},
|
||
{"value": "./fasth3-nvfp4-vae/quality-36L/decoder-nvfp4.safetensors", "label": "NVFP4 36L quality"},
|
||
{"value": "./fasth3-nvfp4-vae/fast-8L/decoder-nvfp4.safetensors", "label": "NVFP4 8L fast"}
|
||
],
|
||
"default": ""
|
||
},
|
||
{
|
||
"key": "nvfp4_text_encoder",
|
||
"label": "Text encoder",
|
||
"hint": "NVFP4 needs Blackwell",
|
||
"flag": "--nvfp4-text-encoder",
|
||
"options": [
|
||
{"value": "", "label": "BF16 Qwen3-VL"},
|
||
{"value": "./fasth3-nvfp4-encoder/text_encoder", "label": "NVFP4 Qwen3-VL"}
|
||
],
|
||
"default": ""
|
||
},
|
||
{
|
||
"key": "offload",
|
||
"label": "Memory",
|
||
"hint": "Resident is how the GB200 timings were measured",
|
||
"flag": "--offload",
|
||
"options": [
|
||
{"value": "resident", "label": "All resident (fastest)"},
|
||
{"value": "lazy", "label": "Lazy load + offload"}
|
||
],
|
||
"default": "resident"
|
||
},
|
||
{
|
||
"key": "num_gpus",
|
||
"label": "GPUs",
|
||
"hint": "Sequence-parallel degree",
|
||
"flag": "--num-gpus",
|
||
"options": [1, 2, 4, 8],
|
||
"default": 1
|
||
}
|
||
],
|
||
"limitations": [
|
||
"NVFP4 QAD release (--weights), one GB200, 768p 16:9, 5 s, warm: 16.8 s with BF16 attention and 15.2 s with FP4 attention; at 768p, 15 s, FP4 attention is 16% faster than BF16 attention (85.3 s vs 101.8 s).",
|
||
"Measured on one GB200 at 768p 16:9, 5 s, warm: official pika-h3-pro CLI 59.7 s; FastVideo BF16 26.2 s; with the NVFP4 quality-36L decoder 18.0 s; with the fast-8L decoder 17.4 s. The 36L decoder measured 40.5 dB PSNR against the BF16 decoder.",
|
||
"Peak resident memory was about 129 GiB for the all-BF16 run and 104.6 GiB with the NVFP4 text encoder (25.49 s) at 768p / 5 s on GB200; the official CLI's sampling peak was 71.7 GiB, a different measurement. The NVFP4 text encoder's first request in a fresh process pays a one-time cost of about 183 s.",
|
||
"--offload resident keeps every component on the GPU (104.6-108.5 GiB peak with the NVFP4 text encoder, about 129 GiB all-BF16, on GB200). --offload lazy loads components on demand for smaller devices; on GB200 it took 48.06 s for the same T2VA request.",
|
||
"FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN FASTVIDEO_FA4=1 selects FlashAttention-4 on Hopper or Blackwell; FA4 is never enabled just because it is installed. Without FA4, attention uses FA3/FA2 if installed and otherwise falls back to SDPA, which is slow and can run out of memory at 768p.",
|
||
"The NVFP4 decoder and text encoder are Blackwell-only. Download them before selecting them: hf download FastVideo/FastH3-NVFP4-VAE-Decoder --include 'quality-36L/decoder-nvfp4.safetensors' 'fast-8L/decoder-nvfp4.safetensors' --local-dir fasth3-nvfp4-vae; hf download FastVideo/FastVideo-FastH3-8-Step-V2-NVFP4-Consumer --include 'text_encoder/*' --local-dir fasth3-nvfp4-encoder. The 8L decoder is a distilled student; validate it on your prompts.",
|
||
"The six-forward re-noise ladder is part of the adapter contract; the example exposes no step-count flag."
|
||
],
|
||
"related": ["pika-h3-flex-keyframes", "pika-h3-flex-ref2va"]
|
||
},
|
||
{
|
||
"id": "pika-h3-flex-keyframes",
|
||
"family": "pika_h3_flex",
|
||
"stage": "inference",
|
||
"task": "First/last frame to video (with audio)",
|
||
"label": "Pika H3 Flex keyframes",
|
||
"summary": "Animate from a first frame (I2VA), toward a last frame (L2VA), or between both (FL2VA). Pika center-crops each keyframe to the selected bucket.",
|
||
"model": "Pika-Labs/Pika-H3-Flex",
|
||
"source": "examples/inference/basic/basic_pika_h3_flex.py",
|
||
"command": "FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN FASTVIDEO_FA4=1 python examples/inference/basic/basic_pika_h3_flex.py --task fl2va --offload resident --image assets/images/natural.jpg --last-image assets/images/peninsula.jpg --prompt 'The camera drifts from the misty mountain ridge down to the coastal cliffs above the sea.'",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {
|
||
"platform": "cuda",
|
||
"gpu_count": 1,
|
||
"accelerator": "NVIDIA GB200",
|
||
"peak_memory": "105.2 GiB FL2VA, 104.9 GiB I2VA (NVFP4 text encoder, 768p, 5 s)",
|
||
"evidence": "validated",
|
||
"evidence_url": "https://github.com/hao-ai-lab/FastVideo/pull/2038"
|
||
},
|
||
"evidence": "Verified",
|
||
"expected_artifact": "MP4 with synchronized stereo audio at outputs/pika_h3_flex/pika_h3_flex_<task>.mp4",
|
||
"modes": ["I2VA", "L2VA", "FL2VA"],
|
||
"knobs": [
|
||
{
|
||
"key": "weights",
|
||
"label": "Weights",
|
||
"hint": "NVFP4 QAD needs Blackwell",
|
||
"flag": "--weights",
|
||
"options": [{"value": "bf16", "label": "BF16 (Pika release)"}, {"value": "nvfp4-qad", "label": "NVFP4 QAD"}, {"value": "nvfp4-qad-fp4-attention", "label": "NVFP4 QAD + FP4 attention"}],
|
||
"default": "bf16"
|
||
},
|
||
{
|
||
"key": "task",
|
||
"label": "Task",
|
||
"hint": "Which keyframes condition the clip",
|
||
"flag": "--task",
|
||
"options": [
|
||
{"value": "i2va", "label": "First frame (I2VA)"},
|
||
{"value": "l2va", "label": "Last frame (L2VA)"},
|
||
{"value": "fl2va", "label": "First and last (FL2VA)"}
|
||
],
|
||
"default": "fl2va"
|
||
},
|
||
{
|
||
"key": "resolution",
|
||
"label": "Resolution",
|
||
"hint": "Pika resolution tier",
|
||
"flag": "--resolution",
|
||
"options": [{"value": "768p", "label": "768p"}, {"value": "480p", "label": "480p"}],
|
||
"default": "768p"
|
||
},
|
||
{
|
||
"key": "aspect",
|
||
"label": "Aspect ratio",
|
||
"hint": "Bucket within the tier",
|
||
"flag": "--aspect",
|
||
"options": ["21:9", "16:9", "3:2", "4:3", "5:4", "1:1", "4:5", "3:4", "2:3", "9:16", "9:21"],
|
||
"default": "16:9"
|
||
},
|
||
{
|
||
"key": "duration",
|
||
"label": "Duration",
|
||
"hint": "Rounded up to the 17n+5 frame grid",
|
||
"flag": "--duration",
|
||
"options": [
|
||
{"value": 5, "label": "5 s (124 frames)"},
|
||
{"value": 10, "label": "10 s (243 frames)"},
|
||
{"value": 15, "label": "15 s (362 frames)"}
|
||
],
|
||
"default": 5
|
||
},
|
||
{
|
||
"key": "nvfp4_decoder",
|
||
"label": "VAE decoder",
|
||
"hint": "NVFP4 needs Blackwell",
|
||
"flag": "--nvfp4-decoder",
|
||
"options": [
|
||
{"value": "", "label": "BF16 H3 VAE"},
|
||
{"value": "./fasth3-nvfp4-vae/quality-36L/decoder-nvfp4.safetensors", "label": "NVFP4 36L quality"},
|
||
{"value": "./fasth3-nvfp4-vae/fast-8L/decoder-nvfp4.safetensors", "label": "NVFP4 8L fast"}
|
||
],
|
||
"default": ""
|
||
},
|
||
{
|
||
"key": "nvfp4_text_encoder",
|
||
"label": "Text encoder",
|
||
"hint": "NVFP4 needs Blackwell",
|
||
"flag": "--nvfp4-text-encoder",
|
||
"options": [
|
||
{"value": "", "label": "BF16 Qwen3-VL"},
|
||
{"value": "./fasth3-nvfp4-encoder/text_encoder", "label": "NVFP4 Qwen3-VL"}
|
||
],
|
||
"default": ""
|
||
},
|
||
{
|
||
"key": "offload",
|
||
"label": "Memory",
|
||
"hint": "Resident is how the GB200 timings were measured",
|
||
"flag": "--offload",
|
||
"options": [
|
||
{"value": "resident", "label": "All resident (fastest)"},
|
||
{"value": "lazy", "label": "Lazy load + offload"}
|
||
],
|
||
"default": "resident"
|
||
},
|
||
{
|
||
"key": "num_gpus",
|
||
"label": "GPUs",
|
||
"hint": "Sequence-parallel degree",
|
||
"flag": "--num-gpus",
|
||
"options": [1, 2, 4, 8],
|
||
"default": 1
|
||
}
|
||
],
|
||
"limitations": [
|
||
"Measured on one GB200 at 768p, 5 s, seed 42, NVFP4 text encoder, all resident: FL2VA 28.95 s (first frame 39.6 dB / SSIM 0.971 and last frame 29.1 dB / 0.819 against the inputs); I2VA 27.37 s (first frame 26.5 dB / 0.715). L2VA was not measured separately.",
|
||
"The command passes both keyframes; the example uses only the ones the selected task needs. Replace them with your own images.",
|
||
"--offload resident keeps every component on the GPU (104.6-108.5 GiB peak with the NVFP4 text encoder, about 129 GiB all-BF16, on GB200). --offload lazy loads components on demand for smaller devices; on GB200 it took 48.06 s for the same T2VA request.",
|
||
"FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN FASTVIDEO_FA4=1 selects FlashAttention-4 on Hopper or Blackwell; FA4 is never enabled just because it is installed. Without FA4, attention uses FA3/FA2 if installed and otherwise falls back to SDPA, which is slow and can run out of memory at 768p.",
|
||
"The NVFP4 decoder and text encoder are Blackwell-only. Download them before selecting them: hf download FastVideo/FastH3-NVFP4-VAE-Decoder --include 'quality-36L/decoder-nvfp4.safetensors' 'fast-8L/decoder-nvfp4.safetensors' --local-dir fasth3-nvfp4-vae; hf download FastVideo/FastVideo-FastH3-8-Step-V2-NVFP4-Consumer --include 'text_encoder/*' --local-dir fasth3-nvfp4-encoder. The 8L decoder is a distilled student; validate it on your prompts."
|
||
],
|
||
"related": ["pika-h3-flex-t2va", "pika-h3-flex-ref2va"]
|
||
},
|
||
{
|
||
"id": "pika-h3-flex-ref2va",
|
||
"family": "pika_h3_flex",
|
||
"stage": "inference",
|
||
"task": "Reference media to video (with audio)",
|
||
"label": "Pika H3 Flex references",
|
||
"summary": "Generate from ordered image, video, and audio references through Pika's ref2va adapter: up to 9 images, 3 videos, and 3 audio clips.",
|
||
"model": "Pika-Labs/Pika-H3-Flex",
|
||
"source": "examples/inference/basic/basic_pika_h3_flex.py",
|
||
"command": "FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN FASTVIDEO_FA4=1 python examples/inference/basic/basic_pika_h3_flex.py --task ref2va --offload resident --reference image:assets/images/natural.jpg --reference image:assets/images/bus_terminal.jpg --prompt 'Start on the misty mountain ridge from the first reference, then cut to the night bus terminal from the second.'",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {
|
||
"platform": "cuda",
|
||
"gpu_count": 1,
|
||
"accelerator": "NVIDIA GB200",
|
||
"peak_memory": "108.5 GiB with two image references (NVFP4 text encoder, 768p, 5 s)",
|
||
"evidence": "validated",
|
||
"evidence_url": "https://github.com/hao-ai-lab/FastVideo/pull/2038"
|
||
},
|
||
"evidence": "Verified",
|
||
"expected_artifact": "MP4 with synchronized stereo audio at outputs/pika_h3_flex/pika_h3_flex_ref2va.mp4",
|
||
"modes": ["Ref2VA"],
|
||
"knobs": [
|
||
{
|
||
"key": "weights",
|
||
"label": "Weights",
|
||
"hint": "NVFP4 QAD needs Blackwell",
|
||
"flag": "--weights",
|
||
"options": [{"value": "bf16", "label": "BF16 (Pika release)"}, {"value": "nvfp4-qad", "label": "NVFP4 QAD"}, {"value": "nvfp4-qad-fp4-attention", "label": "NVFP4 QAD + FP4 attention"}],
|
||
"default": "bf16"
|
||
},
|
||
{
|
||
"key": "resolution",
|
||
"label": "Resolution",
|
||
"hint": "Pika resolution tier",
|
||
"flag": "--resolution",
|
||
"options": [{"value": "768p", "label": "768p"}, {"value": "480p", "label": "480p"}],
|
||
"default": "768p"
|
||
},
|
||
{
|
||
"key": "aspect",
|
||
"label": "Aspect ratio",
|
||
"hint": "Bucket within the tier",
|
||
"flag": "--aspect",
|
||
"options": ["21:9", "16:9", "3:2", "4:3", "5:4", "1:1", "4:5", "3:4", "2:3", "9:16", "9:21"],
|
||
"default": "16:9"
|
||
},
|
||
{
|
||
"key": "duration",
|
||
"label": "Duration",
|
||
"hint": "Rounded up to the 17n+5 frame grid",
|
||
"flag": "--duration",
|
||
"options": [
|
||
{"value": 5, "label": "5 s (124 frames)"},
|
||
{"value": 10, "label": "10 s (243 frames)"},
|
||
{"value": 15, "label": "15 s (362 frames)"}
|
||
],
|
||
"default": 5
|
||
},
|
||
{
|
||
"key": "nvfp4_decoder",
|
||
"label": "VAE decoder",
|
||
"hint": "NVFP4 needs Blackwell",
|
||
"flag": "--nvfp4-decoder",
|
||
"options": [
|
||
{"value": "", "label": "BF16 H3 VAE"},
|
||
{"value": "./fasth3-nvfp4-vae/quality-36L/decoder-nvfp4.safetensors", "label": "NVFP4 36L quality"},
|
||
{"value": "./fasth3-nvfp4-vae/fast-8L/decoder-nvfp4.safetensors", "label": "NVFP4 8L fast"}
|
||
],
|
||
"default": ""
|
||
},
|
||
{
|
||
"key": "nvfp4_text_encoder",
|
||
"label": "Text encoder",
|
||
"hint": "NVFP4 needs Blackwell",
|
||
"flag": "--nvfp4-text-encoder",
|
||
"options": [
|
||
{"value": "", "label": "BF16 Qwen3-VL"},
|
||
{"value": "./fasth3-nvfp4-encoder/text_encoder", "label": "NVFP4 Qwen3-VL"}
|
||
],
|
||
"default": ""
|
||
},
|
||
{
|
||
"key": "offload",
|
||
"label": "Memory",
|
||
"hint": "Resident is how the GB200 timings were measured",
|
||
"flag": "--offload",
|
||
"options": [
|
||
{"value": "resident", "label": "All resident (fastest)"},
|
||
{"value": "lazy", "label": "Lazy load + offload"}
|
||
],
|
||
"default": "resident"
|
||
},
|
||
{
|
||
"key": "num_gpus",
|
||
"label": "GPUs",
|
||
"hint": "Sequence-parallel degree",
|
||
"flag": "--num-gpus",
|
||
"options": [1, 2, 4, 8],
|
||
"default": 1
|
||
}
|
||
],
|
||
"limitations": [
|
||
"Measured on one GB200 at 768p, 5 s, seed 42, NVFP4 text encoder, all resident, two image references: 51.17 s (38.42 s denoise, 64,241 packed tokens); both references appear in the output.",
|
||
"Repeat --reference image:PATH, video:PATH, or audio:PATH in order. At least one image or video is required; a video reference needs at least 22 frames at 24 fps.",
|
||
"Ref2VA loads transformer_ref and the ref2va adapter, a separate 61.7 GiB trunk download. Reference images are encoded at up to a 1024 (480p) or 2048 (768p) short edge, so conditioning memory is higher than T2VA's.",
|
||
"--offload resident keeps every component on the GPU (104.6-108.5 GiB peak with the NVFP4 text encoder, about 129 GiB all-BF16, on GB200). --offload lazy loads components on demand for smaller devices; on GB200 it took 48.06 s for the same T2VA request.",
|
||
"FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN FASTVIDEO_FA4=1 selects FlashAttention-4 on Hopper or Blackwell; FA4 is never enabled just because it is installed. Without FA4, attention uses FA3/FA2 if installed and otherwise falls back to SDPA, which is slow and can run out of memory at 768p.",
|
||
"The NVFP4 decoder and text encoder are Blackwell-only. Download them before selecting them: hf download FastVideo/FastH3-NVFP4-VAE-Decoder --include 'quality-36L/decoder-nvfp4.safetensors' 'fast-8L/decoder-nvfp4.safetensors' --local-dir fasth3-nvfp4-vae; hf download FastVideo/FastVideo-FastH3-8-Step-V2-NVFP4-Consumer --include 'text_encoder/*' --local-dir fasth3-nvfp4-encoder. The 8L decoder is a distilled student; validate it on your prompts."
|
||
],
|
||
"related": ["pika-h3-flex-t2va", "pika-h3-flex-keyframes"]
|
||
},
|
||
{
|
||
"id": "open-h3-regenerate-2k",
|
||
"family": "minimax_h3",
|
||
"stage": "inference",
|
||
"task": "Video super-resolution (2K regeneration)",
|
||
"label": "Open-H3-Regenerate-2K",
|
||
"summary": "Regenerate a 768P MiniMax-H3 draft at 2560x1440 with NVlabs' three-step refiner. The output is silent.",
|
||
"model": "Efficient-Large-Model/Open-H3-Regenerate-2K",
|
||
"source": "examples/inference/basic/basic_open_h3_regenerate_2k.py",
|
||
"command": ": \"${INPUT_VIDEO:?Set INPUT_VIDEO to a 768P MiniMax-H3 draft}\"\npython examples/inference/basic/basic_open_h3_regenerate_2k.py --video \"$INPUT_VIDEO\" --prompt \"A red fox trots through fresh snow in a pine forest at golden hour.\"",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"platform": "cuda", "gpu_count": 1, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"expected_artifact": "2560x1440 MP4 at outputs/open_h3_regenerate_2k/regen2k_0.mp4",
|
||
"related": ["minimax-h3-t2v", "fasth3-8step-v2-cuda"]
|
||
},
|
||
{
|
||
"id": "fasth3-lora-preview",
|
||
"family": "minimax_h3",
|
||
"stage": "inference",
|
||
"task": "LoRA-adapted few-step video (with audio)",
|
||
"label": "FastH3 LoRA Preview",
|
||
"summary": "Apply a FastH3 preview adapter at load time while keeping the shared four-forward performance profile and synchronized audio output.",
|
||
"model": "MiniMaxAI/MiniMax-H3",
|
||
"source": "examples/inference/basic/basic_fasth3_lora_preview.py",
|
||
"command": "python examples/inference/basic/basic_fasth3_lora_preview.py --lora-path path/to/adapter.safetensors --prompt \"(S1) A presenter says <d>[English] This is an adapted Fast H3 run.</d>\"",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {
|
||
"platform": "cuda",
|
||
"gpu_count": 4,
|
||
"accelerator": "NVIDIA B200",
|
||
"evidence": "validated",
|
||
"evidence_url": "https://github.com/hao-ai-lab/FastVideo/pull/1771"
|
||
},
|
||
"evidence": "Verified",
|
||
"expected_artifact": "Warmup and measured MP4 files under outputs/fasth3_lora_preview/",
|
||
"modes": ["T2VA", "FastH3 LoRA"],
|
||
"knobs": [
|
||
{"key": "num_gpus", "label": "GPUs", "hint": "Sequence-parallel degree", "flag": "--num-gpus", "options": [1, 2, 4, 8], "default": 4},
|
||
{"key": "video_decode_backend", "label": "VAE decode", "hint": "Fidelity vs. speed", "flag": "--video-decode-backend", "options": [{"value": "h3-vae", "label": "Full H3 VAE"}, {"value": "taeh3", "label": "TAEH3 preview"}], "default": "h3-vae"}
|
||
],
|
||
"limitations": ["Supply a compatible FastH3 adapter. The script infers dense or VSA attention from the adapter payload unless you override it."]
|
||
},
|
||
{
|
||
"id": "stable-audio-open-t2a",
|
||
"family": "stable_audio",
|
||
"stage": "inference",
|
||
"task": "Text to audio",
|
||
"label": "Stable Audio Open 1.0",
|
||
"summary": "Six-second text-to-audio generation with Stable Audio Open 1.0 from its maintained example; duration and steps are documented knobs in the source.",
|
||
"model": "FastVideo/stable-audio-open-1.0-Diffusers",
|
||
"source": "examples/inference/basic/basic_stable_audio.py",
|
||
"command": "python examples/inference/basic/basic_stable_audio.py",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {
|
||
"platform": "cuda",
|
||
"gpu_count": 1,
|
||
"accelerator": "NVIDIA B200",
|
||
"evidence": "validated",
|
||
"evidence_url": "https://github.com/hao-ai-lab/FastVideo/pull/1260"
|
||
},
|
||
"evidence": "Verified",
|
||
"expected_artifact": "WAV audio at outputs_audio/stable_audio_basic/output_stable_audio.wav",
|
||
"limitations": ["Must load the FastVideo converted Diffusers repo; upstream stabilityai monolithic checkpoints are not loader-compatible (see scripts/checkpoint_conversion/stable_audio_to_diffusers.py)."],
|
||
"related": ["stable-audio-small-t2a"]
|
||
},
|
||
{
|
||
"id": "stable-audio-small-t2a",
|
||
"family": "stable_audio",
|
||
"stage": "inference",
|
||
"task": "Text to audio",
|
||
"label": "Stable Audio Open Small",
|
||
"summary": "The smaller Stable Audio Open variant with its own shorter training window, from its maintained example.",
|
||
"model": "FastVideo/stable-audio-open-small-Diffusers",
|
||
"source": "examples/inference/basic/basic_stable_audio_small.py",
|
||
"command": "python examples/inference/basic/basic_stable_audio_small.py",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"related": ["stable-audio-open-t2a"]
|
||
},
|
||
{
|
||
"id": "mmaudio-v2a",
|
||
"family": "mmaudio",
|
||
"stage": "inference",
|
||
"task": "Video/Text to audio",
|
||
"label": "MMAudio large 44k v2",
|
||
"summary": "Add synchronized audio to a video (or from a prompt) with MMAudio large 44k v2. The example reads the model path from MMAUDIO_MODEL_PATH; this recipe passes the converted Hugging Face repo explicitly.",
|
||
"model": "FastVideo/MMAudio-large-44k-v2-Diffusers",
|
||
"source": "examples/inference/basic/basic_mmaudio.py",
|
||
"command": "MMAUDIO_MODEL_PATH=FastVideo/MMAudio-large-44k-v2-Diffusers python examples/inference/basic/basic_mmaudio.py",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"limitations": ["The upstream checkpoint must be converted to Diffusers layout via scripts/checkpoint_conversion/convert_mmaudio_to_diffusers.py unless loaded from the FastVideo converted repo as done here."]
|
||
},
|
||
{
|
||
"id": "matrix-game-2",
|
||
"family": "matrixgame",
|
||
"stage": "inference",
|
||
"task": "Interactive world",
|
||
"label": "Matrix Game 2.0",
|
||
"summary": "Generate an interactive-world sequence from the maintained Matrix Game 2.0 example.",
|
||
"model": "FastVideo/Matrix-Game-2.0-Base-Distilled-Diffusers",
|
||
"source": "examples/inference/basic/basic_matrixgame2.py",
|
||
"command": "python examples/inference/basic/basic_matrixgame2.py",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"related": ["matrix-game-3-i2w"]
|
||
},
|
||
{
|
||
"id": "matrix-game-3-i2w",
|
||
"family": "matrixgame",
|
||
"stage": "inference",
|
||
"task": "Interactive world",
|
||
"label": "Matrix Game 3.0",
|
||
"summary": "Drive Matrix Game 3.0 from an input image plus prompt at 720p, three steps, from its maintained example.",
|
||
"model": "FastVideo/Matrix-Game-3.0-Base-Distilled-Diffusers",
|
||
"source": "examples/inference/basic/basic_matrixgame3.py",
|
||
"command": "python examples/inference/basic/basic_matrixgame3.py",
|
||
"gpu_types": ["NVIDIA"],
|
||
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
||
"evidence": "Source-backed",
|
||
"expected_artifact": "MP4 videos under video_samples_matrixgame3/",
|
||
"related": ["matrix-game-2"]
|
||
},
|
||
{
|
||
"id": "fasth3-trim-spark",
|
||
"family": "minimax_h3",
|
||
"stage": "inference",
|
||
"task": "Text to video with audio",
|
||
"label": "FastH3 Trim",
|
||
"summary": "Eight-step video and synchronized audio with the released NVFP4 transformer, text encoder, and lightweight VAE.",
|
||
"model": "FastVideo/FastVideo-FastH3-Trim-8-Step-NVFP4",
|
||
"source": "examples/inference/basic/basic_fasth3_spark_pruned_nvfp4.yaml",
|
||
"command": "FASTVIDEO_MINIMAX_H3_FUSIONS=all FASTVIDEO_NVFP4_MM_BACKEND=cutlass FASTVIDEO_H3_VAE_TILE_BATCH=1 FASTVIDEO_VSA_TRITON=1 FASTVIDEO_VSA_PTX_BLACKWELL=0 FASTVIDEO_FA4=0 FASTVIDEO_ATTENTION_BACKEND=VIDEO_SPARSE_ATTN_H3 FASTVIDEO_STAGE_LOGGING=1 fastvideo generate --config examples/inference/basic/basic_fasth3_spark_pruned_nvfp4.yaml",
|
||
"gpu_types": [
|
||
"NVIDIA"
|
||
],
|
||
"hardware": {
|
||
"platform": "cuda",
|
||
"gpu_count": 1,
|
||
"device": "spark",
|
||
"evidence": "source-configured"
|
||
},
|
||
"evidence": "Source-backed",
|
||
"serving": {
|
||
"source": "examples/serving/openai_fasth3_trim_spark.yaml",
|
||
"install": "Install FastVideo following the DGX Spark installation guide.",
|
||
"audio": true
|
||
}
|
||
},
|
||
{
|
||
"id": "fasth3-omniref-rtx5090",
|
||
"family": "minimax_h3",
|
||
"stage": "inference",
|
||
"task": "Reference media to video with audio",
|
||
"label": "FastH3 OmniRef on RTX 5090",
|
||
"summary": "Eight-forward video and audio from a prompt plus ordered image, video, and audio references, with the NVFP4 stack.",
|
||
"model": "FastVideo/FastH3-OmniRef-NVFP4-QAD",
|
||
"source": "examples/inference/basic/basic_fasth3_omniref_pdd.py",
|
||
"command": "python examples/inference/basic/basic_fasth3_omniref_pdd.py --model-path FastVideo/FastH3-OmniRef-NVFP4-QAD --profile rtx-5090 --image assets/images/couple.jpg --prompt 'A person on a sofa, lit by a warm lamp, speaking to camera.'",
|
||
"gpu_types": [
|
||
"NVIDIA"
|
||
],
|
||
"hardware": {
|
||
"platform": "cuda",
|
||
"gpu_count": 1,
|
||
"minimum_memory": "32 GB VRAM",
|
||
"evidence": "source-configured"
|
||
},
|
||
"evidence": "Source-backed"
|
||
},
|
||
{
|
||
"id": "fasth3-omniref-rtx-pro6000",
|
||
"family": "minimax_h3",
|
||
"stage": "inference",
|
||
"task": "Reference media to video with audio",
|
||
"label": "FastH3 OmniRef on RTX PRO 6000",
|
||
"summary": "Eight-forward video and audio from a prompt plus ordered image, video, and audio references, with the NVFP4 stack.",
|
||
"model": "FastVideo/FastH3-OmniRef-NVFP4-QAD",
|
||
"source": "examples/inference/basic/basic_fasth3_omniref_pdd.py",
|
||
"command": "python examples/inference/basic/basic_fasth3_omniref_pdd.py --model-path FastVideo/FastH3-OmniRef-NVFP4-QAD --profile rtx-pro-6000 --image assets/images/couple.jpg --prompt 'A person on a sofa, lit by a warm lamp, speaking to camera.'",
|
||
"gpu_types": [
|
||
"NVIDIA"
|
||
],
|
||
"hardware": {
|
||
"platform": "cuda",
|
||
"gpu_count": 1,
|
||
"evidence": "source-configured"
|
||
},
|
||
"evidence": "Source-backed"
|
||
},
|
||
{
|
||
"id": "fasth3-omniref-spark",
|
||
"family": "minimax_h3",
|
||
"stage": "inference",
|
||
"task": "Reference media to video with audio",
|
||
"label": "FastH3 OmniRef on DGX Spark",
|
||
"summary": "Eight-forward video and audio from a prompt plus ordered image, video, and audio references, with the NVFP4 stack.",
|
||
"model": "FastVideo/FastH3-OmniRef-NVFP4-QAD",
|
||
"source": "examples/inference/basic/basic_fasth3_omniref_pdd.py",
|
||
"command": "python examples/inference/basic/basic_fasth3_omniref_pdd.py --model-path FastVideo/FastH3-OmniRef-NVFP4-QAD --profile spark --image assets/images/couple.jpg --prompt 'A person on a sofa, lit by a warm lamp, speaking to camera.'",
|
||
"gpu_types": [
|
||
"NVIDIA"
|
||
],
|
||
"hardware": {
|
||
"platform": "cuda",
|
||
"gpu_count": 1,
|
||
"device": "spark",
|
||
"evidence": "source-configured"
|
||
},
|
||
"evidence": "Source-backed"
|
||
},
|
||
{
|
||
"id": "fasth3-omniref-mlx",
|
||
"family": "minimax_h3",
|
||
"stage": "inference",
|
||
"task": "Reference media to video with audio",
|
||
"label": "FastH3 Omni V1 on MLX",
|
||
"summary": "Run the FastH3 Omni BF16 release on Apple Silicon: compose it with the NVFP4 text-encoder stack, convert the DiT to INT6 with VSA, and decode with the LightVAE encoder and the FastH3 NVFP4 decoder.",
|
||
"model": "FastVideo/FastVideo-FastH3-Omni-8-Step-V1",
|
||
"source": "examples/inference/basic/mlx_fasth3_omniref.py",
|
||
"command": "hf download FastVideo/FastVideo-FastH3-Omni-8-Step-V1 --local-dir ./FastVideo-FastH3-Omni-8-Step-V1\nhf download FastVideo/FastH3-OmniRef-NVFP4-QAD --include 'text_encoder/*' 'tokenizer/*' 'processor/*' 'vae/config.json' 'audio_vae/*' --local-dir ./FastH3-OmniRef-NVFP4-QAD\nhf download lightx2v/LightVAE lightvae-lite-h3-encoder.safetensors --local-dir ./LightVAE\nhf download FastVideo/FastH3-NVFP4-VAE-Decoder --include 'quality-36L/decoder-nvfp4.safetensors' 'fast-8L/decoder-nvfp4.safetensors' --local-dir ./FastH3-NVFP4-VAE-Decoder\npython examples/inference/basic/mlx_fasth3_omniref.py --compose-only --export ./FastVideo-FastH3-Omni-8-Step-V1 --base-model-path ./FastH3-OmniRef-NVFP4-QAD --composed-dir ./FastH3-OmniRef-composed\npython scripts/checkpoint_conversion/convert_minimax_h3_mlx.py --model-root ./FastH3-OmniRef-composed/transformer_ref --out ./FastH3-OmniRef-MLX --formats int6 --include-vsa --nvfp4-conditioner-root ./FastH3-OmniRef-NVFP4-QAD/text_encoder\nln -sfn \"$PWD/FastH3-OmniRef-MLX/nvfp4-encoder\" ./FastH3-OmniRef-composed/text_encoder\npython examples/inference/basic/mlx_fasth3_omniref.py --model-root ./FastH3-OmniRef-composed --mlx-checkpoint ./FastH3-OmniRef-MLX/int6 --image assets/images/couple.jpg --prompt 'A person on a sofa, lit by a warm lamp, speaking to camera.' --vae-encoder ./LightVAE/lightvae-lite-h3-encoder.safetensors --vae-decoder ./FastH3-NVFP4-VAE-Decoder/quality-36L/decoder-nvfp4.safetensors --vae-dtype bf16 --mlx-cache-limit-gib 8 --metal-wired-limit-gib 42 --height 480 --width 832 --num-frames 124 --seed 2026 --output-path ./outputs/fasth3_omniref_mlx.mp4",
|
||
"gpu_types": [
|
||
"Apple Silicon"
|
||
],
|
||
"hardware": {
|
||
"platform": "mlx",
|
||
"accelerator": "Apple M5 Max",
|
||
"system_memory": "64 GB unified memory",
|
||
"minimum_memory": "36 GB unified memory",
|
||
"peak_memory": "27-31 GB peak memory, INT6 DiT of about 17 GB",
|
||
"evidence": "validated",
|
||
"evidence_url": "https://github.com/hao-ai-lab/FastVideo/pull/2000"
|
||
},
|
||
"evidence": "Verified",
|
||
"expected_artifact": "MP4 with H.264 video and stereo AAC audio at outputs/fasth3_omniref_mlx.mp4",
|
||
"modes": [
|
||
"Ref2VA",
|
||
"8-step FastH3 Omni",
|
||
"trained VSA"
|
||
],
|
||
"limitations": [
|
||
"Convert with --include-vsa. At least one image or video reference is required; pass references in the order the model should read them.",
|
||
"Validated on an M4 Max with 36 GB and an M5 Max with 64 GB. Set --metal-wired-limit-gib to about two thirds of the Mac's memory.",
|
||
"Use fast-8L/decoder-nvfp4.safetensors for the smaller, faster decoder tier. FL2VA keyframes, two-pass refinement and TAEH3 decoding are not wired for references."
|
||
]
|
||
}
|
||
]
|
||
}
|