653 lines
32 KiB
JSON
653 lines
32 KiB
JSON
{
|
|
"version": 4,
|
|
"recipes": [
|
|
{
|
|
"id": "fastwan21-t2v",
|
|
"family": "wan",
|
|
"stage": "inference",
|
|
"task": "Text to video",
|
|
"label": "FastWan2.1 1.3B (distilled + VSA)",
|
|
"summary": "Generate a video in three denoising steps with the distilled FastWan2.1 1.3B checkpoint and video sparse attention.",
|
|
"model": "FastVideo/FastWan2.1-T2V-1.3B-Diffusers",
|
|
"source": "scripts/inference/inference_wan_VSA_DMD_1_3B.yaml",
|
|
"command": "FASTVIDEO_ATTENTION_BACKEND=VIDEO_SPARSE_ATTN fastvideo generate --config scripts/inference/inference_wan_VSA_DMD_1_3B.yaml",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
|
"evidence": "Source-backed",
|
|
"expected_artifact": "MP4 videos under outputs_video_dmd_1.3B/"
|
|
},
|
|
{
|
|
"id": "wan22-t2v",
|
|
"family": "wan",
|
|
"stage": "inference",
|
|
"task": "Text to video",
|
|
"label": "Wan2.2 A14B",
|
|
"summary": "The maintained high-capacity Wan2.2 text-to-video example with CPU offload settings encoded in its checked-in Python source.",
|
|
"model": "Wan-AI/Wan2.2-T2V-A14B-Diffusers",
|
|
"source": "examples/inference/basic/basic_wan2_2.py",
|
|
"command": "python examples/inference/basic/basic_wan2_2.py",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"gpu_count": 2, "evidence": "source-configured"},
|
|
"evidence": "Source-backed",
|
|
"expected_artifact": "MP4 videos under video_samples_wan2_2_14B_t2v/"
|
|
},
|
|
{
|
|
"id": "wan21-i2v",
|
|
"family": "wan",
|
|
"stage": "inference",
|
|
"task": "Image to video",
|
|
"label": "Wan2.1 14B 480P",
|
|
"summary": "Animate an input image at 480P using the maintained Wan2.1 YAML configuration and its recorded offload settings.",
|
|
"model": "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
|
|
"source": "scripts/inference/inference_wan_i2v.yaml",
|
|
"command": "fastvideo generate --config scripts/inference/inference_wan_i2v.yaml",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"gpu_count": 2, "evidence": "source-configured"},
|
|
"evidence": "Source-backed"
|
|
},
|
|
{
|
|
"id": "wan22-ti2v",
|
|
"family": "wan",
|
|
"stage": "inference",
|
|
"task": "Text or image to video",
|
|
"label": "Wan2.2 TI2V 5B",
|
|
"summary": "Use one maintained 5B checkpoint for text-to-video or add an image input to switch the same recipe to image-to-video.",
|
|
"model": "Wan-AI/Wan2.2-TI2V-5B-Diffusers",
|
|
"source": "examples/inference/basic/basic_wan2_2_ti2v.py",
|
|
"command": "python examples/inference/basic/basic_wan2_2_ti2v.py",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
|
"evidence": "Source-backed",
|
|
"expected_artifact": "MP4 videos under video_samples_wan2_2_5B_ti2v/"
|
|
},
|
|
{
|
|
"id": "fastmetal-1-3b-mlx",
|
|
"family": "wan",
|
|
"stage": "inference",
|
|
"task": "Text to video",
|
|
"label": "FastMetal 1.3B",
|
|
"summary": "Run the released FastMetal 1.3B QAD checkpoint through FastVideo's native Apple Silicon MLX path.",
|
|
"model": "FastVideo/FastMetal-1.3B-QAD",
|
|
"source": "examples/inference/basic/mlx_wan_prompt_to_video.py",
|
|
"command": "hf download FastVideo/FastMetal-1.3B-QAD --local-dir ./FastMetal-1.3B-QAD\npython examples/inference/basic/mlx_wan_prompt_to_video.py --model-root ./FastMetal-1.3B-QAD --mlx-checkpoint ./FastMetal-1.3B-QAD --height 480 --width 832 --num-frames 81 --prompt \"A bird's-eye view of a misty forest valley at dawn.\" --output-path ./outputs/fastmetal_1_3b.mp4",
|
|
"gpu_types": ["Apple Silicon"],
|
|
"hardware": {
|
|
"platform": "mlx",
|
|
"accelerator": "Apple M4 Max",
|
|
"system_memory": "36 GB unified memory",
|
|
"minimum_memory": "16 GB+ unified memory",
|
|
"peak_memory": "3.87 GiB peak MLX memory",
|
|
"evidence": "validated",
|
|
"evidence_url": "https://github.com/hao-ai-lab/FastVideo/pull/1638"
|
|
},
|
|
"evidence": "Verified",
|
|
"expected_artifact": "MP4 video at outputs/fastmetal_1_3b.mp4",
|
|
"limitations": ["This is the native MLX FastMetal path. basic_mps.py is the older PyTorch MPS demo."]
|
|
},
|
|
{
|
|
"id": "fastmetal-5b-mlx",
|
|
"family": "wan",
|
|
"stage": "inference",
|
|
"task": "Text or image to video",
|
|
"label": "FastMetal 5B",
|
|
"summary": "Run the released Wan2.2 5B FastMetal checkpoint with MLX DiT denoising and MLX TAEHV decode.",
|
|
"model": "FastVideo/FastMetal-5B-QAD",
|
|
"source": "examples/inference/basic/mlx_wan22_generate.py",
|
|
"command": "hf download FastVideo/FastMetal-5B-QAD --local-dir ./FastMetal-5B-QAD\npython examples/inference/basic/mlx_wan22_generate.py --mlx-checkpoint ./FastMetal-5B-QAD --text-encoder-root ./FastMetal-5B-QAD --vae-root ./FastMetal-5B-QAD/vae --height 704 --width 1280 --num-frames 81 --prompt \"A cinematic portrait with soft neon lighting and smooth camera motion.\" --output-path ./outputs/fastmetal_5b.mp4",
|
|
"gpu_types": ["Apple Silicon"],
|
|
"hardware": {
|
|
"platform": "mlx",
|
|
"accelerator": "Apple M4 Max",
|
|
"system_memory": "36 GB unified memory",
|
|
"minimum_memory": "16 GB+ unified memory",
|
|
"peak_memory": "9.34 GiB peak MLX memory",
|
|
"evidence": "validated",
|
|
"evidence_url": "https://github.com/hao-ai-lab/FastVideo/pull/1638"
|
|
},
|
|
"evidence": "Verified",
|
|
"expected_artifact": "MP4 video at outputs/fastmetal_5b.mp4"
|
|
},
|
|
{
|
|
"id": "fastmetal-14b-mlx",
|
|
"family": "wan",
|
|
"stage": "inference",
|
|
"task": "Text to video",
|
|
"label": "FastMetal 14B",
|
|
"summary": "Run the released 14B FastMetal QAD checkpoint through the same Apple Silicon MLX entrypoint as the 1.3B release.",
|
|
"model": "FastVideo/FastMetal-14B-QAD",
|
|
"source": "examples/inference/basic/mlx_wan_prompt_to_video.py",
|
|
"command": "hf download FastVideo/FastMetal-14B-QAD --local-dir ./FastMetal-14B-QAD\npython examples/inference/basic/mlx_wan_prompt_to_video.py --model-root ./FastMetal-14B-QAD --mlx-checkpoint ./FastMetal-14B-QAD --height 480 --width 832 --num-frames 81 --prompt \"A wide cinematic landscape at sunrise.\" --output-path ./outputs/fastmetal_14b.mp4",
|
|
"gpu_types": ["Apple Silicon"],
|
|
"hardware": {
|
|
"platform": "mlx",
|
|
"accelerator": "Apple M4 Max",
|
|
"system_memory": "36 GB unified memory",
|
|
"minimum_memory": "36 GB+ unified memory",
|
|
"peak_memory": "21.68 GiB peak MLX memory",
|
|
"evidence": "validated",
|
|
"evidence_url": "https://github.com/hao-ai-lab/FastVideo/pull/1638"
|
|
},
|
|
"evidence": "Verified",
|
|
"expected_artifact": "MP4 video at outputs/fastmetal_14b.mp4"
|
|
},
|
|
{
|
|
"id": "turbodiffusion-wan21-1-3b-t2v",
|
|
"family": "turbodiffusion",
|
|
"stage": "inference",
|
|
"task": "Text to video",
|
|
"label": "TurboWan2.1 1.3B",
|
|
"summary": "A TurboDiffusion-accelerated Wan2.1 1.3B text-to-video run from its maintained single-GPU example.",
|
|
"model": "loayrashid/TurboWan2.1-T2V-1.3B-Diffusers",
|
|
"source": "examples/inference/basic/basic_turbodiffusion.py",
|
|
"command": "python examples/inference/basic/basic_turbodiffusion.py",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
|
"evidence": "Source-backed",
|
|
"expected_artifact": "MP4 videos under video_samples_turbodiffusion/",
|
|
"related": ["turbodiffusion-wan21-14b-t2v", "turbowan22-i2v"]
|
|
},
|
|
{
|
|
"id": "turbodiffusion-wan21-14b-t2v",
|
|
"family": "turbodiffusion",
|
|
"stage": "inference",
|
|
"task": "Text to video",
|
|
"label": "TurboWan2.1 14B",
|
|
"summary": "TurboDiffusion acceleration applied to the 14B Wan2.1 text-to-video checkpoint; the checked-in source is configured for two GPUs.",
|
|
"model": "loayrashid/TurboWan2.1-T2V-14B-Diffusers",
|
|
"source": "examples/inference/basic/basic_turbodiffusion_14b.py",
|
|
"command": "python examples/inference/basic/basic_turbodiffusion_14b.py",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"gpu_count": 2, "evidence": "source-configured"},
|
|
"evidence": "Source-backed",
|
|
"expected_artifact": "MP4 videos under video_samples_turbodiffusion_14B/",
|
|
"related": ["turbodiffusion-wan21-1-3b-t2v", "turbowan22-i2v"]
|
|
},
|
|
{
|
|
"id": "turbowan22-i2v",
|
|
"family": "turbodiffusion",
|
|
"stage": "inference",
|
|
"task": "Image to video",
|
|
"label": "TurboWan2.2 A14B",
|
|
"summary": "A one-to-four-step image-to-video path using TurboDiffusion and the SLA attention backend from its maintained example.",
|
|
"model": "loayrashid/TurboWan2.2-I2V-A14B-Diffusers",
|
|
"source": "examples/inference/basic/basic_turbodiffusion_i2v.py",
|
|
"command": "python examples/inference/basic/basic_turbodiffusion_i2v.py",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"gpu_count": 2, "evidence": "source-configured"},
|
|
"evidence": "Source-backed",
|
|
"related": ["turbodiffusion-wan21-14b-t2v"]
|
|
},
|
|
{
|
|
"id": "ltx2-distilled-t2v",
|
|
"family": "ltx2",
|
|
"stage": "inference",
|
|
"task": "Text to video",
|
|
"label": "LTX-2 distilled",
|
|
"summary": "The distilled LTX-2 text-to-video checkpoint with audio, from its maintained example. The source is configured for four GPUs.",
|
|
"model": "FastVideo/LTX2-Distilled-Diffusers",
|
|
"source": "examples/inference/basic/basic_ltx2_distilled.py",
|
|
"command": "python examples/inference/basic/basic_ltx2_distilled.py",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"gpu_count": 4, "evidence": "source-configured"},
|
|
"evidence": "Source-backed",
|
|
"expected_artifact": "MP4 (video with audio) at outputs_video/ltx2_basic/output_ltx2_distilled_t2v.mp4",
|
|
"related": ["ltx23-base-t2v"]
|
|
},
|
|
{
|
|
"id": "ltx23-base-t2v",
|
|
"family": "ltx2",
|
|
"stage": "inference",
|
|
"task": "Text to video",
|
|
"label": "LTX-2 base (1088p)",
|
|
"summary": "Base LTX-2 text-to-video at 1088x1920 using FastVideo default sampling for LTX2 base. The example loads a community Diffusers mirror of the base checkpoint; registered aliases include Lightricks/LTX-2 and FastVideo/LTX2-Diffusers.",
|
|
"model": "Davids048/LTX2-Base-Diffusers",
|
|
"source": "examples/inference/basic/basic_ltx2.py",
|
|
"command": "python examples/inference/basic/basic_ltx2.py",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
|
"evidence": "Source-backed",
|
|
"expected_artifact": "MP4 (video with audio) at outputs_video/ltx2_basic/output_ltx2_base_t2v_1088_1920_1.1.mp4",
|
|
"limitations": ["The maintained example loads the Davids048/LTX2-Base-Diffusers community mirror rather than a Lightricks upstream ID."],
|
|
"related": ["ltx2-distilled-t2v"]
|
|
},
|
|
{
|
|
"id": "hy15-t2v-480p",
|
|
"family": "hunyuan",
|
|
"stage": "inference",
|
|
"task": "Text to video",
|
|
"label": "HunyuanVideo 1.5 480P",
|
|
"summary": "HunyuanVideo 1.5 text-to-video at 480P with CPU offload enabled in the checked-in source for smaller GPUs.",
|
|
"model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v",
|
|
"source": "examples/inference/basic/basic_hy15.py",
|
|
"command": "python examples/inference/basic/basic_hy15.py",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
|
"evidence": "Source-backed",
|
|
"expected_artifact": "MP4 videos under video_samples_hy15/",
|
|
"related": ["hy15-1080p-upscale"]
|
|
},
|
|
{
|
|
"id": "hy15-1080p-upscale",
|
|
"family": "hunyuan",
|
|
"stage": "inference",
|
|
"task": "Text to video (upscaled)",
|
|
"label": "HunyuanVideo 1.5 1080P upscale",
|
|
"summary": "Run HunyuanVideo 1.5 through the 480p to 720p to 1080p upscale chain in one maintained script.",
|
|
"model": "weizhou03/HunyuanVideo-1.5-Diffusers-1080p-2SR",
|
|
"source": "examples/inference/basic/basic_hy15_1080p.py",
|
|
"command": "python examples/inference/basic/basic_hy15_1080p.py",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
|
"evidence": "Source-backed",
|
|
"expected_artifact": "MP4 videos under video_samples_hy15_1080p/",
|
|
"related": ["hy15-t2v-480p"]
|
|
},
|
|
{
|
|
"id": "cosmos25-t2w",
|
|
"family": "cosmos",
|
|
"stage": "inference",
|
|
"task": "Text to world",
|
|
"label": "Cosmos Predict 2.5 2B",
|
|
"summary": "Generate a navigable world video from a text prompt with Cosmos Predict 2.5 2B on a single GPU.",
|
|
"model": "KyleShao/Cosmos-Predict2.5-2B-Diffusers",
|
|
"source": "examples/inference/basic/basic_cosmos2_5_t2w.py",
|
|
"command": "python examples/inference/basic/basic_cosmos2_5_t2w.py",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
|
"evidence": "Source-backed"
|
|
},
|
|
{
|
|
"id": "kandinsky5-t2v-lite-sft",
|
|
"family": "kandinsky5",
|
|
"stage": "inference",
|
|
"task": "Text to video",
|
|
"label": "Kandinsky 5.0 T2V Lite SFT",
|
|
"summary": "Kandinsky 5.0 text-to-video (Lite SFT variant) from the maintained example; alternative Lite/Pro checkpoints are listed in the source.",
|
|
"model": "kandinskylab/Kandinsky-5.0-T2V-Lite-sft-5s-Diffusers",
|
|
"source": "examples/inference/basic/basic_kandinsky5_t2v.py",
|
|
"command": "python examples/inference/basic/basic_kandinsky5_t2v.py",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {
|
|
"platform": "cuda",
|
|
"gpu_count": 1,
|
|
"accelerator": "NVIDIA B200",
|
|
"evidence": "validated",
|
|
"evidence_url": "https://github.com/hao-ai-lab/FastVideo/pull/1471"
|
|
},
|
|
"evidence": "Verified",
|
|
"expected_artifact": "MP4 videos under video_samples_kandinsky5_t2v/",
|
|
"related": ["kandinsky5-i2v-pro-distilled"]
|
|
},
|
|
{
|
|
"id": "kandinsky5-i2v-pro-distilled",
|
|
"family": "kandinsky5",
|
|
"stage": "inference",
|
|
"task": "Image to video",
|
|
"label": "Kandinsky 5.0 I2V Pro distilled",
|
|
"summary": "Animate an input image with Kandinsky 5.0 I2V Pro (distilled) on a single GPU.",
|
|
"model": "kandinskylab/Kandinsky-5.0-I2V-Pro-distilled-5s-Diffusers",
|
|
"source": "examples/inference/basic/basic_kandinsky5_i2v.py",
|
|
"command": "python examples/inference/basic/basic_kandinsky5_i2v.py",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {
|
|
"platform": "cuda",
|
|
"gpu_count": 1,
|
|
"accelerator": "NVIDIA B200",
|
|
"peak_memory": "10,365.89 MB peak GPU memory",
|
|
"evidence": "validated",
|
|
"evidence_url": "https://github.com/hao-ai-lab/FastVideo/pull/1471"
|
|
},
|
|
"evidence": "Verified",
|
|
"expected_artifact": "MP4 videos under video_samples_kandinsky5_i2v/",
|
|
"related": ["kandinsky5-t2v-lite-sft"]
|
|
},
|
|
{
|
|
"id": "flux2-klein-t2i",
|
|
"family": "flux",
|
|
"stage": "inference",
|
|
"task": "Text to image",
|
|
"label": "FLUX.2 Klein 4B",
|
|
"summary": "Generate an image in four denoising steps with the distilled FLUX.2 Klein checkpoint.",
|
|
"model": "black-forest-labs/FLUX.2-klein-4B",
|
|
"source": "examples/inference/basic/basic_flux2_klein.py",
|
|
"command": "python examples/inference/basic/basic_flux2_klein.py",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
|
"evidence": "Source-backed",
|
|
"expected_artifact": "PNG image at outputs/flux2/flux2_klein.png",
|
|
"related": ["flux2-dev-t2i"]
|
|
},
|
|
{
|
|
"id": "flux2-dev-t2i",
|
|
"family": "flux",
|
|
"stage": "inference",
|
|
"task": "Text to image",
|
|
"label": "FLUX.2 dev",
|
|
"summary": "Full FLUX.2 dev text-to-image with embedded guidance and the Mistral3 text encoder, from its maintained example.",
|
|
"model": "black-forest-labs/FLUX.2-dev",
|
|
"source": "examples/inference/basic/basic_flux2.py",
|
|
"command": "python examples/inference/basic/basic_flux2.py",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
|
"evidence": "Source-backed",
|
|
"expected_artifact": "PNG image at outputs/flux2/flux2.png",
|
|
"related": ["flux2-klein-t2i"]
|
|
},
|
|
{
|
|
"id": "flux1-dev-t2i",
|
|
"family": "flux",
|
|
"stage": "inference",
|
|
"task": "Text to image",
|
|
"label": "FLUX.1 dev",
|
|
"summary": "FLUX.1 dev text-to-image through the Diffusers-backed pipeline. The example defaults to a local weights directory, so this recipe passes the Hugging Face ID explicitly.",
|
|
"model": "black-forest-labs/FLUX.1-dev",
|
|
"source": "examples/inference/basic/basic_flux_dev.py",
|
|
"command": "python examples/inference/basic/basic_flux_dev.py --model-path black-forest-labs/FLUX.1-dev",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
|
"evidence": "Source-backed",
|
|
"expected_artifact": "PNG images under outputs/flux_dev/samples/",
|
|
"limitations": ["FLUX.1 is loadable by ID but registers no model_family in fastvideo/registry.py; it is grouped under FLUX for documentation only."]
|
|
},
|
|
{
|
|
"id": "glm-image-t2i",
|
|
"family": "glm_image",
|
|
"stage": "inference",
|
|
"task": "Text to image",
|
|
"label": "GLM-Image",
|
|
"summary": "GLM-Image text-to-image generation from its maintained example.",
|
|
"model": "zai-org/GLM-Image",
|
|
"source": "examples/inference/basic/basic_glm_image.py",
|
|
"command": "python examples/inference/basic/basic_glm_image.py",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
|
"evidence": "Source-backed",
|
|
"expected_artifact": "PNG image at image_output/landscape.png",
|
|
"related": ["glm-image-edit"]
|
|
},
|
|
{
|
|
"id": "glm-image-edit",
|
|
"family": "glm_image",
|
|
"stage": "inference",
|
|
"task": "Image editing",
|
|
"label": "GLM-Image editing",
|
|
"summary": "Edit an input image with an instruction prompt using GLM-Image, from its maintained editing example.",
|
|
"model": "zai-org/GLM-Image",
|
|
"source": "examples/inference/basic/edit_glm_image.py",
|
|
"command": "python examples/inference/basic/edit_glm_image.py",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
|
"evidence": "Source-backed",
|
|
"expected_artifact": "PNG image at image_output/edited.png (input: assets/images/couple.jpg)",
|
|
"related": ["glm-image-t2i"]
|
|
},
|
|
{
|
|
"id": "zimage-turbo-t2i",
|
|
"family": "zimage",
|
|
"stage": "inference",
|
|
"task": "Text to image",
|
|
"label": "Z-Image Turbo",
|
|
"summary": "Z-Image Turbo text-to-image on a single GPU from its maintained example.",
|
|
"model": "Tongyi-MAI/Z-Image-Turbo",
|
|
"source": "examples/inference/basic/basic_zimage.py",
|
|
"command": "python examples/inference/basic/basic_zimage.py",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
|
"evidence": "Source-backed",
|
|
"expected_artifact": "PNG image at outputs/zimage/zimage_turbo.png"
|
|
},
|
|
{
|
|
"id": "sd35-medium-t2i",
|
|
"family": "sd35",
|
|
"stage": "inference",
|
|
"task": "Text to image",
|
|
"label": "Stable Diffusion 3.5 Medium",
|
|
"summary": "Stable Diffusion 3.5 Medium text-to-image over a small built-in prompt set, from its maintained example.",
|
|
"model": "stabilityai/stable-diffusion-3.5-medium",
|
|
"source": "examples/inference/basic/basic_sd35_t2i.py",
|
|
"command": "python examples/inference/basic/basic_sd35_t2i.py",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
|
"evidence": "Source-backed",
|
|
"expected_artifact": "PNG images under outputs/sd35/samples/"
|
|
},
|
|
{
|
|
"id": "minimax-h3-t2v",
|
|
"family": "minimax_h3",
|
|
"stage": "inference",
|
|
"task": "Text to video (with audio)",
|
|
"label": "MiniMax H3 T2VA",
|
|
"summary": "Generate synchronized video and stereo audio from a structured text prompt with the full MiniMax H3 checkpoint.",
|
|
"model": "MiniMaxAI/MiniMax-H3",
|
|
"source": "examples/inference/basic/basic_minimax_h3_t2v.py",
|
|
"command": "python examples/inference/basic/basic_minimax_h3_t2v.py --prompt \"(S1) A presenter says <d>[English] FastVideo runs MiniMax H3.</d>\"",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"platform": "cuda", "gpu_count": 4, "evidence": "source-configured"},
|
|
"evidence": "Source-backed",
|
|
"expected_artifact": "MP4 with synchronized audio at outputs/minimax_h3_t2v/minimax_h3_t2v.mp4",
|
|
"limitations": ["The checked-in example defaults to four-way sequence parallelism. It does not record a GPU model or memory requirement."]
|
|
},
|
|
{
|
|
"id": "fasth3-preview-cuda",
|
|
"group": "fasth3-preview",
|
|
"group_label": "FastH3 Preview",
|
|
"group_task": "4-step text to video + audio",
|
|
"family": "minimax_h3",
|
|
"stage": "inference",
|
|
"task": "Few-step text to video (with audio)",
|
|
"label": "FastH3 Preview on CUDA",
|
|
"summary": "Run the DMD2-distilled FastH3 Preview with four DiT forwards, trained H3 sparse attention, compiled decode, and synchronized audio.",
|
|
"model": "FastVideo/FastVideo-Minimax-FastH3-Preview-v0.2",
|
|
"source": "examples/inference/basic/basic_fasth3.py",
|
|
"command": "UV_TORCH_BACKEND=cu130 uv pip install -e \".[fasth3]\"\npython examples/inference/basic/basic_fasth3.py --prompt \"(S1) A presenter says <d>[English] FastVideo runs FastH3.</d>\" --profile all",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {
|
|
"platform": "cuda",
|
|
"gpu_count": 4,
|
|
"accelerator": "NVIDIA GB200",
|
|
"evidence": "validated",
|
|
"evidence_url": "https://github.com/hao-ai-lab/FastVideo/pull/1731"
|
|
},
|
|
"evidence": "Verified",
|
|
"expected_artifact": "Warmup and measured MP4 files under outputs/fasth3/",
|
|
"limitations": ["The default all profile is the measured GB200 performance route and can change floating-point operation order. Use --profile strict --no-inference-torch-compile for the eager strict route."]
|
|
},
|
|
{
|
|
"id": "fasth3-preview-mlx",
|
|
"group": "fasth3-preview",
|
|
"group_label": "FastH3 Preview",
|
|
"group_task": "4-step text to video + audio",
|
|
"family": "minimax_h3",
|
|
"stage": "inference",
|
|
"task": "Few-step text to video (with audio)",
|
|
"label": "FastH3 Preview on MLX",
|
|
"summary": "Run FastH3 Preview on Apple Silicon with a locally converted INT6 DiT, streamed Qwen3-VL conditioning, and native MLX video and audio VAEs.",
|
|
"model": "FastVideo/FastVideo-Minimax-FastH3-Preview-v0.2",
|
|
"source": "examples/inference/basic/mlx_fasth3.py",
|
|
"command": "hf download FastVideo/FastVideo-Minimax-FastH3-Preview-v0.2 --local-dir ./FastH3-Preview-v0.2\npython scripts/checkpoint_conversion/convert_minimax_h3_mlx.py --model-root ./FastH3-Preview-v0.2/transformer --out ./FastH3-MLX --formats \"int6\"\npython examples/inference/basic/mlx_fasth3.py --model-root ./FastH3-Preview-v0.2 --mlx-checkpoint ./FastH3-MLX/int6 --prompt \"(S1) A presenter says <d>[English] FastVideo runs FastH3.</d>\" --height 480 --width 832 --num-frames 124 --seed 2026 --output-path ./outputs/fasth3_int6.mp4",
|
|
"gpu_types": ["Apple Silicon"],
|
|
"hardware": {
|
|
"platform": "mlx",
|
|
"accelerator": "Apple M4 Max",
|
|
"system_memory": "36 GB unified memory",
|
|
"peak_memory": "19.63 GiB peak MLX memory during denoising",
|
|
"evidence": "validated",
|
|
"evidence_url": "https://github.com/hao-ai-lab/FastVideo/pull/1770"
|
|
},
|
|
"evidence": "Verified",
|
|
"expected_artifact": "MP4 with H.264 video and stereo AAC audio at outputs/fasth3_int6.mp4",
|
|
"limitations": ["The MLX path supports T2VA and optional temporal --fast mode. FL2VA, Ref2VA, spatial fast mode, two-pass refinement, and VSA are not wired."]
|
|
},
|
|
{
|
|
"id": "minimax-h3-fl2va",
|
|
"family": "minimax_h3",
|
|
"stage": "inference",
|
|
"task": "First/last frame to video (with audio)",
|
|
"label": "MiniMax H3 FL2VA",
|
|
"summary": "Animate a first frame, optionally guide the final frame, and generate synchronized audio with the full MiniMax H3 checkpoint.",
|
|
"model": "MiniMaxAI/MiniMax-H3",
|
|
"source": "examples/inference/basic/basic_minimax_h3_fl2va.py",
|
|
"command": "python examples/inference/basic/basic_minimax_h3_fl2va.py --image path/to/first-frame.png --prompt \"(S1) The subject turns toward the camera and says <d>[English] Hello.</d>\"",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"platform": "cuda", "gpu_count": 4, "evidence": "source-configured"},
|
|
"evidence": "Source-backed",
|
|
"expected_artifact": "MP4 with synchronized audio at outputs/minimax_h3_fl2va/minimax_h3_fl2va.mp4",
|
|
"limitations": ["Pass --last-image to constrain the final frame. The checked-in source defaults to four GPUs."]
|
|
},
|
|
{
|
|
"id": "minimax-h3-ref2va",
|
|
"family": "minimax_h3",
|
|
"stage": "inference",
|
|
"task": "Reference media to video (with audio)",
|
|
"label": "MiniMax H3 Ref2VA",
|
|
"summary": "Condition H3 on an ordered reference video and optional audio reference, then generate a new synchronized video and audio result.",
|
|
"model": "MiniMaxAI/MiniMax-H3",
|
|
"source": "examples/inference/basic/basic_minimax_h3_ref2va.py",
|
|
"command": "python examples/inference/basic/basic_minimax_h3_ref2va.py --reference-video path/to/reference.mp4 --prompt \"Create a new scene that preserves the reference identity and motion language.\"",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"platform": "cuda", "gpu_count": 4, "evidence": "source-configured"},
|
|
"evidence": "Source-backed",
|
|
"expected_artifact": "MP4 with synchronized audio at outputs/minimax_h3_ref2va/minimax_h3_ref2va.mp4",
|
|
"limitations": ["Pass --reference-audio for an additional audio reference. The checked-in source defaults to four GPUs."]
|
|
},
|
|
{
|
|
"id": "fasth3-lora-preview",
|
|
"family": "minimax_h3",
|
|
"stage": "inference",
|
|
"task": "LoRA-adapted few-step video (with audio)",
|
|
"label": "FastH3 LoRA Preview",
|
|
"summary": "Apply a FastH3 preview adapter at load time while keeping the shared four-forward performance profile and synchronized audio output.",
|
|
"model": "MiniMaxAI/MiniMax-H3",
|
|
"source": "examples/inference/basic/basic_fasth3_lora_preview.py",
|
|
"command": "python examples/inference/basic/basic_fasth3_lora_preview.py --lora-path path/to/adapter.safetensors --prompt \"(S1) A presenter says <d>[English] This is an adapted Fast H3 run.</d>\"",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {
|
|
"platform": "cuda",
|
|
"gpu_count": 4,
|
|
"accelerator": "NVIDIA B200",
|
|
"evidence": "validated",
|
|
"evidence_url": "https://github.com/hao-ai-lab/FastVideo/pull/1771"
|
|
},
|
|
"evidence": "Verified",
|
|
"expected_artifact": "Warmup and measured MP4 files under outputs/fasth3_lora_preview/",
|
|
"limitations": ["Supply a compatible FastH3 adapter. The script infers dense or VSA attention from the adapter payload unless you override it."]
|
|
},
|
|
{
|
|
"id": "longcat-t2v",
|
|
"family": "longcat",
|
|
"stage": "inference",
|
|
"task": "Text to video",
|
|
"label": "LongCat Video T2V",
|
|
"summary": "LongCat Video text-to-video at 480p (50 steps), with distilled and 720p refinement passes included in the same maintained script.",
|
|
"model": "FastVideo/LongCat-Video-T2V-Diffusers",
|
|
"source": "examples/inference/basic/basic_longcat_t2v.py",
|
|
"command": "python examples/inference/basic/basic_longcat_t2v.py",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
|
"evidence": "Source-backed",
|
|
"expected_artifact": "MP4 videos under outputs_video/longcat_t2v_basic/, longcat_t2v_distill/, and longcat_t2v_refine_720p/",
|
|
"related": ["longcat-i2v"]
|
|
},
|
|
{
|
|
"id": "longcat-i2v",
|
|
"family": "longcat",
|
|
"stage": "inference",
|
|
"task": "Image to video",
|
|
"label": "LongCat Video I2V",
|
|
"summary": "LongCat Video image-to-video with optional distilled and refinement passes, from its maintained example.",
|
|
"model": "FastVideo/LongCat-Video-I2V-Diffusers",
|
|
"source": "examples/inference/basic/basic_longcat_i2v.py",
|
|
"command": "python examples/inference/basic/basic_longcat_i2v.py",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
|
"evidence": "Source-backed",
|
|
"expected_artifact": "MP4 videos under outputs_video/longcat_i2v_basic/ and longcat_i2v_distill/",
|
|
"related": ["longcat-t2v"]
|
|
},
|
|
{
|
|
"id": "stable-audio-open-t2a",
|
|
"family": "stable_audio",
|
|
"stage": "inference",
|
|
"task": "Text to audio",
|
|
"label": "Stable Audio Open 1.0",
|
|
"summary": "Six-second text-to-audio generation with Stable Audio Open 1.0 from its maintained example; duration and steps are documented knobs in the source.",
|
|
"model": "FastVideo/stable-audio-open-1.0-Diffusers",
|
|
"source": "examples/inference/basic/basic_stable_audio.py",
|
|
"command": "python examples/inference/basic/basic_stable_audio.py",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {
|
|
"platform": "cuda",
|
|
"gpu_count": 1,
|
|
"accelerator": "NVIDIA B200",
|
|
"evidence": "validated",
|
|
"evidence_url": "https://github.com/hao-ai-lab/FastVideo/pull/1260"
|
|
},
|
|
"evidence": "Verified",
|
|
"expected_artifact": "WAV audio at outputs_audio/stable_audio_basic/output_stable_audio.wav",
|
|
"limitations": ["Must load the FastVideo converted Diffusers repo; upstream stabilityai monolithic checkpoints are not loader-compatible (see scripts/checkpoint_conversion/stable_audio_to_diffusers.py)."],
|
|
"related": ["stable-audio-small-t2a"]
|
|
},
|
|
{
|
|
"id": "stable-audio-small-t2a",
|
|
"family": "stable_audio",
|
|
"stage": "inference",
|
|
"task": "Text to audio",
|
|
"label": "Stable Audio Open Small",
|
|
"summary": "The smaller Stable Audio Open variant with its own shorter training window, from its maintained example.",
|
|
"model": "FastVideo/stable-audio-open-small-Diffusers",
|
|
"source": "examples/inference/basic/basic_stable_audio_small.py",
|
|
"command": "python examples/inference/basic/basic_stable_audio_small.py",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
|
"evidence": "Source-backed",
|
|
"related": ["stable-audio-open-t2a"]
|
|
},
|
|
{
|
|
"id": "mmaudio-v2a",
|
|
"family": "mmaudio",
|
|
"stage": "inference",
|
|
"task": "Video/Text to audio",
|
|
"label": "MMAudio large 44k v2",
|
|
"summary": "Add synchronized audio to a video (or from a prompt) with MMAudio large 44k v2. The example reads the model path from MMAUDIO_MODEL_PATH; this recipe passes the converted Hugging Face repo explicitly.",
|
|
"model": "FastVideo/MMAudio-large-44k-v2-Diffusers",
|
|
"source": "examples/inference/basic/basic_mmaudio.py",
|
|
"command": "MMAUDIO_MODEL_PATH=FastVideo/MMAudio-large-44k-v2-Diffusers python examples/inference/basic/basic_mmaudio.py",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
|
"evidence": "Source-backed",
|
|
"limitations": ["The upstream checkpoint must be converted to Diffusers layout via scripts/checkpoint_conversion/convert_mmaudio_to_diffusers.py unless loaded from the FastVideo converted repo as done here."]
|
|
},
|
|
{
|
|
"id": "matrix-game-2",
|
|
"family": "matrixgame",
|
|
"stage": "inference",
|
|
"task": "Interactive world",
|
|
"label": "Matrix Game 2.0",
|
|
"summary": "Generate an interactive-world sequence from the maintained Matrix Game 2.0 example.",
|
|
"model": "FastVideo/Matrix-Game-2.0-Base-Distilled-Diffusers",
|
|
"source": "examples/inference/basic/basic_matrixgame2.py",
|
|
"command": "python examples/inference/basic/basic_matrixgame2.py",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
|
"evidence": "Source-backed",
|
|
"related": ["matrix-game-3-i2w"]
|
|
},
|
|
{
|
|
"id": "matrix-game-3-i2w",
|
|
"family": "matrixgame",
|
|
"stage": "inference",
|
|
"task": "Interactive world",
|
|
"label": "Matrix Game 3.0",
|
|
"summary": "Drive Matrix Game 3.0 from an input image plus prompt at 720p, three steps, from its maintained example.",
|
|
"model": "FastVideo/Matrix-Game-3.0-Base-Distilled-Diffusers",
|
|
"source": "examples/inference/basic/basic_matrixgame3.py",
|
|
"command": "python examples/inference/basic/basic_matrixgame3.py",
|
|
"gpu_types": ["NVIDIA"],
|
|
"hardware": {"gpu_count": 1, "evidence": "source-configured"},
|
|
"evidence": "Source-backed",
|
|
"expected_artifact": "MP4 videos under video_samples_matrixgame3/",
|
|
"related": ["matrix-game-2"]
|
|
}
|
|
]
|
|
}
|