From 01be8e707ad50cdbef0f67c112fcf6a6bc9a37ea Mon Sep 17 00:00:00 2001 From: Aryan Kumar Date: Fri, 30 Oct 2026 11:35:24 -0700 Subject: [PATCH] [misc]: serve the Regenerate-2K examples on TORCH_SDPA and drop narrating comments FLASH_ATTN bypassed the cuDNN attention the refine stage selects; TORCH_SDPA keeps it. --- docs/inference/open_h3_regenerate_2k.md | 4 +++- examples/inference/basic/basic_open_h3_regenerate_2k.py | 2 +- examples/serving/openai_open_h3_regenerate_2k.yaml | 2 +- fastvideo/models/loader/component_loader.py | 1 - .../open_h3_regenerate_2k/open_h3_regenerate_2k_pipeline.py | 1 - 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/inference/open_h3_regenerate_2k.md b/docs/inference/open_h3_regenerate_2k.md index d741e1a86..53da256e1 100644 --- a/docs/inference/open_h3_regenerate_2k.md +++ b/docs/inference/open_h3_regenerate_2k.md @@ -87,10 +87,12 @@ The response headers carry `X-Request-Id`, `X-Model`, `X-Inference-Time-S` and ` - `size`, `width` and `height` may be omitted or `2560x1440`; `num_frames` may be omitted or 124 (the frame count of the VAE grid; the video has 121 frames); `fps` must be 24. `seconds` is not accepted. - `num_inference_steps` is 3 (default) or 2; `seed` sets the refine noise. +- Serve with `attention_backend: TORCH_SDPA`: the refine stage selects cuDNN attention, which at about 133k tokens is + about 3x faster than PyTorch's default flash dispatch. The `FLASH_ATTN` backend bypasses that selection. ## Parity with the reference -Measured on one GB200 (tray hpc-rack-1-5, BF16, dense attention) against the reference `standard_fwd` with the released +Measured on one GB200 (BF16, `TORCH_SDPA` with cuDNN attention) against the reference `standard_fwd` with the released VAE, same draft, seed (1766580932), prompt and VAE posterior seed (42). The reference's noise is a CUDA generator draw that FastVideo reproduces (the same `torch.Generator(device="cuda")` seed), so no injected noise was needed. diff --git a/examples/inference/basic/basic_open_h3_regenerate_2k.py b/examples/inference/basic/basic_open_h3_regenerate_2k.py index c3ab88720..cb2ff8c20 100644 --- a/examples/inference/basic/basic_open_h3_regenerate_2k.py +++ b/examples/inference/basic/basic_open_h3_regenerate_2k.py @@ -38,7 +38,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--output", type=Path, default=Path("outputs/open_h3_regenerate_2k")) parser.add_argument("--steps", type=int, choices=(2, 3), default=3, help="refine steps (3 is the released setting)") parser.add_argument("--seed", type=int, default=1766580932) - parser.add_argument("--attention-backend", default="FLASH_ATTN") + parser.add_argument("--attention-backend", default="TORCH_SDPA") parser.add_argument("--draft-size", choices=("768p", "576p"), default="768p", help="size the draft is encoded at; 576p for drafts from other video models") parser.add_argument("--runs", type=int, default=1, help="repeat the request to measure warm timings") diff --git a/examples/serving/openai_open_h3_regenerate_2k.yaml b/examples/serving/openai_open_h3_regenerate_2k.yaml index 06117818c..ffeafe536 100644 --- a/examples/serving/openai_open_h3_regenerate_2k.yaml +++ b/examples/serving/openai_open_h3_regenerate_2k.yaml @@ -17,7 +17,7 @@ generator: vae_enabled: false pipeline: experimental: - attention_backend: FLASH_ATTN + attention_backend: TORCH_SDPA # 768p for MiniMax-H3 drafts; 576p encodes drafts from other video models at 1024x576. draft_size: 768p diff --git a/fastvideo/models/loader/component_loader.py b/fastvideo/models/loader/component_loader.py index a9e3b801a..9c9ffb90f 100644 --- a/fastvideo/models/loader/component_loader.py +++ b/fastvideo/models/loader/component_loader.py @@ -1655,7 +1655,6 @@ class PipelineComponentLoader: component_model_path, ) - # A component under a subfolder (e.g. "base_model/vae") dispatches on its own name. module_name = module_name.rsplit("/", 1)[-1] # Get the appropriate loader for this module type loader = ComponentLoader.for_module_type(module_name, transformers_or_diffusers) diff --git a/fastvideo/pipelines/basic/open_h3_regenerate_2k/open_h3_regenerate_2k_pipeline.py b/fastvideo/pipelines/basic/open_h3_regenerate_2k/open_h3_regenerate_2k_pipeline.py index b5bf46360..2a5868094 100644 --- a/fastvideo/pipelines/basic/open_h3_regenerate_2k/open_h3_regenerate_2k_pipeline.py +++ b/fastvideo/pipelines/basic/open_h3_regenerate_2k/open_h3_regenerate_2k_pipeline.py @@ -54,7 +54,6 @@ class OpenH3Regenerate2KPipeline(MiniMaxH3Pipeline): revision=revision, allow_patterns=self.get_hf_download_allow_patterns()) if self.fastvideo_args.lora_path is None: - # The merged refine LoRA is part of the release; it is a constructor-time adapter like any H3 LoRA. self.fastvideo_args.lora_path = os.path.join(self.model_path, LORA_FILE) return verify_model_config_and_directory(os.path.join(self.model_path, BASE_MODEL_DIR), required_component_dirs=())