[feat]: expose the NVFP4 H3 video decoder in Open-H3-Regenerate-2K

The pipeline already loads its base_model VAE through VAELoader, so the
existing h3_vae_nvfp4_decoder swap applies; this exposes it: a
--h3-vae-nvfp4-decoder CLI flag, a --vae-decoder example flag, the serve
option in the regen2k YAML, and the measured 2K decode/quality table in
the guide.

CPU tests pin the regen2k VAE load applying the exported decoder, the
serve-config field reaching FastVideoArgs, and the new CLI flag.

docs-site/content/cookbook/pan2/pan2.mdx

@@ -1,7 +0,0 @@
----
-title: PAN 2
-description: Configure text-to-video and first-frame image-to-video generation with PAN 2.
-mode: frame
----
-
-<Cookbook modelId="pan2" />

docs-site/lib/cookbook-pages.ts

@@ -66,15 +66,6 @@ export const recipePages = [
     description: "Turn a reference image into video with Wan2.1.",
     tags: ["Image to video", "14B", "480p"],
   },
-  {
-    id: "pan2",
-    family: "PAN 2",
-    title: "PAN 2",
-    mark: "PAN",
-    path: "/cookbook/pan2/pan2",
-    description: "Text-to-video and first-frame image-to-video from the 29B PAN 2 model.",
-    tags: ["Text or image to video", "29B"],
-  },
   {
     id: "ltx2-distilled",
     family: "LTX",
@@ -119,7 +110,6 @@ const familyInfo: Record<string, { title: string; path: string; logo: string; ca
   hunyuan: { title: "Hunyuan", path: "hunyuan", logo: "tencent-hunyuan", category: "Video" },
   kandinsky5: { title: "Kandinsky 5", path: "kandinsky5", logo: "kandinsky", category: "Video" },
   kandinsky6: { title: "Kandinsky 6", path: "kandinsky6", logo: "kandinsky", category: "Video" },
-  pan2: { title: "PAN 2", path: "pan2", logo: "ifm", category: "Video" },
   flux: { title: "FLUX", path: "flux", logo: "black-forest-labs", category: "Image" },
   qwen_image21: { title: "Qwen-Image-2.1", path: "qwen-image21", logo: "qwen", category: "Image" },
   sd35: { title: "Stable Diffusion", path: "stable-diffusion", logo: "stabilityai", category: "Image" },
@@ -137,7 +127,6 @@ const backendModels: Record<string, string> = {
   "FastVideo/FastWan2.1-T2V-1.3B-Diffusers": "fastwan21",
   "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers": "wan21-i2v",
   "FastVideo/LTX2-Distilled-Diffusers": "ltx2-distilled",
-  "IFM/PAN2": "pan2",
 };
 const TASK_GROUPS: Array<[RegExp, string]> = [
   [/world/, "Text to world"],
@@ -175,7 +164,6 @@ export const modelDirectory = [...new Set(sourceRecipes.map((recipe) => recipe.m
       kandinsky: "Kandinsky Lab",
       "black-forest-labs": "Black Forest Labs",
       qwen: "Qwen",
-      ifm: "IFM",
       stabilityai: "Stability AI",
       nvidia: "NVIDIA",
     };

docs-site/navigation.json

@@ -127,10 +127,6 @@
         ],
         "display": "group"
       },
-      {
-        "label": "PAN 2 text/image to video",
-        "href": "/inference/pan2"
-      },
       {
         "label": "Examples",
         "href": "/inference/examples/examples_inference_index"
@@ -253,25 +249,6 @@
           }
         ]
       },
-      {
-        "label": "PAN 2",
-        "display": "group",
-        "collapsed": true,
-        "items": [
-          {
-            "label": "PAN 2",
-            "href": "/cookbook/pan2/pan2"
-          },
-          {
-            "label": "Family guide",
-            "href": "/cookbook/pan2"
-          },
-          {
-            "label": "Model reference",
-            "href": "/inference/pan2"
-          }
-        ]
-      },
       {
         "label": "FLUX",
         "href": "/cookbook/flux"

docs-site/tests/cookbook.spec.ts

@@ -223,7 +223,7 @@ test("inherited values, explicit false, zero and null retain their meanings", as
 test("catalog discovery opens dedicated model recipes and filters by family and task", async ({ page }) => {
   await page.goto("/FastVideo/cookbook");
   const cards = page.locator(".recipe-card");
-  await expect(cards).toHaveCount(38);
+  await expect(cards).toHaveCount(37);
   await expect(cards.first()).toContainText("FastH3 OmniRef");
   await page.getByLabel("Model family", { exact: true }).selectOption("Wan");
   await page.getByLabel("Search recipes").fill("Wan2.1 I2V");
@@ -277,7 +277,7 @@ test("output tabs support keyboard navigation and precision chips use the backen
   await page.getByLabel("Search recipes").fill("does not exist");
   await expect(page.getByRole("heading", { name: "No matching recipes" })).toBeVisible();
   await page.getByRole("button", { name: "Clear filters" }).click();
-  await expect(page.locator(".recipe-card")).toHaveCount(38);
+  await expect(page.locator(".recipe-card")).toHaveCount(37);
 });
 
 test("cookbook cards and controls remain readable in dark mode and reflow at 320px", async ({ page }) => {
@@ -443,26 +443,6 @@ test("source-backed serving pages expose their actual server settings", async ({
   await expect(page.getByRole("button", { name: "Copy config + command", exact: true })).toBeEnabled();
 });
 
-test("PAN 2 has a family card with the IFM logo and an editable serving deployment", async ({ page }) => {
-  await page.goto("/FastVideo/cookbook");
-  await page.getByLabel("Search recipes").fill("PAN 2");
-  const pan2 = page.locator(".recipe-card");
-  await expect(pan2).toHaveCount(1);
-  await expect(pan2.locator("img")).toHaveAttribute("src", /\/ifm\.webp$/);
-  await expect(pan2.locator(".recipe-card-kicker")).toHaveText("IFM");
-  await pan2.click();
-  await expect(page).toHaveURL(/cookbook\/pan2\/pan2$/);
-  await expect(page.locator(".recipe-matrix")).toBeVisible({ timeout: 30000 });
-  await expect(page.getByRole("heading", { level: 1 })).toHaveText("PAN 2");
-  await settings(page, "Serving");
-  await page.getByLabel("Port", { exact: true }).fill("9010");
-  await output(page, "config.yaml");
-  await expect(page.getByLabel("Generated YAML", { exact: true })).toContainText(/"port": 9010/);
-  await expect(page.getByLabel("Generated YAML", { exact: true })).toContainText(/"model_path": "IFM\/PAN2"/);
-  await expect(page.getByLabel("Generated YAML", { exact: true })).toContainText(/"dit_layerwise": false/);
-  await expect(page.getByRole("button", { name: "Download YAML", exact: false })).toBeEnabled();
-});
-
 test("source model hardware preferences survive hydration and reload", async ({ page }) => {
   const errors: string[] = [];
   page.on("pageerror", (error) => errors.push(error.message));

docs/assets/cookbook-recipes.json

@@ -1,5 +1,5 @@
 {
-  "version": 20,
+  "version": 19,
   "recipes": [
     {
       "id": "fastwan21-t2v",
@@ -381,33 +381,6 @@
       "expected_artifact": "Upscaled MP4 under outputs_video/kandinsky6_sr/",
       "related": ["kandinsky6-vsr", "kandinsky6-ti2va-piflow"]
     },
-    {
-      "id": "pan2-ti2v",
-      "family": "pan2",
-      "stage": "inference",
-      "task": "Text or image to video",
-      "label": "PAN 2",
-      "summary": "Generate a 13-second, 704 × 1248 video at 24 fps from text, or set IMAGE_PATH in the maintained example to condition on a first frame.",
-      "model": "IFM/PAN2",
-      "source": "examples/inference/basic/basic_pan2.py",
-      "serving": {
-        "source": "examples/serving/openai_pan2.yaml",
-        "install": "UV_TORCH_BACKEND=cu130 uv pip install -e .",
-        "task": "Text to video",
-        "limitations": [
-          "This server walkthrough sends a text prompt only. The server API also accepts an image reference (image_reference or input_reference) and then runs image-to-video from that first frame, but the checked-in clients do not upload one."
-        ]
-      },
-      "command": "python examples/inference/basic/basic_pan2.py",
-      "gpu_types": ["NVIDIA"],
-      "hardware": {"platform": "cuda", "gpu_count": 1, "evidence": "source-configured"},
-      "evidence": "Source-backed",
-      "expected_artifact": "MP4 video under video_samples_pan2/",
-      "limitations": [
-        "The full IFM/PAN2 checkpoint is not public yet; numerical parity is checked with a public random-weight checkpoint.",
-        "The example keeps the 29B DiT resident in bf16 and offloads only the text encoder. GPU memory requirements have not been established for this recipe; set PAN2_NUM_GPUS to shard video tokens with sequence parallelism, which does not shard the DiT weights."
-      ]
-    },
     {
       "id": "flux2-klein-t2i",
       "family": "flux",

docs/assets/logos/SOURCES.md

@@ -13,7 +13,6 @@ plain typographic tile instead — that tile is a UI placeholder, not a logo.
 | `nvidia.webp` | [Official NVIDIA Hugging Face organization avatar](https://huggingface.co/nvidia) | Cosmos and GEN3C cards, Cosmos page header |
 | `kandinsky.webp` | [Official Kandinsky Lab Hugging Face organization avatar](https://huggingface.co/kandinskylab) | Kandinsky 5 and Kandinsky 6 cards and page headers |
 | `black-forest-labs.webp` | [Official Black Forest Labs Hugging Face organization avatar](https://huggingface.co/black-forest-labs) | FLUX family card and page header |
-| `ifm.webp` | [Official Institute of Foundation Models (IFM) Hugging Face organization avatar](https://huggingface.co/IFM) | PAN 2 family card and page header |
 | `qwen.webp` | [Official Qwen Hugging Face organization avatar](https://huggingface.co/Qwen) | Qwen-Image-2.1 family card and page header |
 | `minimax.webp` | [Official MiniMax Hugging Face organization avatar](https://huggingface.co/MiniMaxAI) | MiniMax H3 family card and page header |
 | `stabilityai.webp` | [Official Stability AI Hugging Face organization avatar](https://huggingface.co/stabilityai) | Stable Diffusion and Stable Audio cards and page headers |

docs/cookbook/index.md

@@ -6,7 +6,6 @@ The interactive catalog groups models by family and generates YAML for supported
 - [FastH3](minimax-h3.md)
 - [Wan and FastMetal](wan.md)
 - [LTX](ltx.md)
-- [PAN 2](pan2.md)
 - [FLUX](flux.md)
 - [Qwen-Image-2.1](qwen-image21.md)
 - [Stable Diffusion](stable-diffusion.md)

docs/cookbook/pan2.md

@@ -1,76 +0,0 @@
-# PAN 2 recipes
-
-<RecipeFamily family="pan2" />
-
-Generate a video from text, or from a first frame, with `IFM/PAN2`. One pipeline
-serves both modes: leave out the first frame for text-to-video and supply one
-for image-to-video. The defaults are 704×1248, 313 frames (13 s at 24 fps),
-50 steps and guidance 3.0. See [PAN 2 Text/Image to Video](../inference/pan2.md)
-for the model, sampler and parallelism details.
-
-## Run on a GPU
-
-Follow the [CUDA installation guide](../getting_started/installation/gpu.md), then run
-from the repository root. The full `IFM/PAN2` checkpoint is not public yet; pass a
-local directory or private repo you can access with `PAN2_MODEL_PATH`.
-
-```bash
-python examples/inference/basic/basic_pan2.py
-```
-
-Set `IMAGE_PATH` in the script to a first frame for image-to-video. With more
-GPUs, `PAN2_NUM_GPUS=4` shards the video tokens with sequence parallelism; the
-count must divide PAN 2's 32 attention heads. Each rank still holds the full
-29B DiT, so extra GPUs speed up generation but do not reduce per-device weight
-memory. See [CPU offloading](../inference/offloading.md) for devices that cannot
-hold the DiT.
-
-## Serve with the OpenAI-compatible API
-
-`examples/serving/openai_pan2.yaml` serves PAN 2 through the shared
-`/v1/videos` routes. It mirrors the Python example: one GPU, the text encoder
-offloaded between requests, and the pipeline's own sampling defaults.
-
-```bash
-fastvideo serve --config examples/serving/openai_pan2.yaml --server.host 127.0.0.1
-```
-
-Text-to-video uses the same request as any other served video model, with the
-alias `pan2`:
-
-```bash
-curl --fail-with-body http://127.0.0.1:8000/v1/videos \
-  -H 'Content-Type: application/json' \
-  -d '{"model": "pan2", "prompt": "A red fox trots through fresh snow in a pine forest at dawn."}'
-```
-
-For image-to-video, add a first frame in `image_reference` (or the legacy
-`input_reference`). The server then runs image-to-video; no second deployment or
-workload is needed.
-
-```bash
-curl --fail-with-body http://127.0.0.1:8000/v1/videos \
-  -H 'Content-Type: application/json' \
-  -d '{"model": "pan2", "prompt": "The fox turns toward the camera.",
-       "image_reference": {"image_url": "https://example.com/first-frame.png"}}'
-```
-
-Local image paths must sit under `server.media_roots`, and private or loopback
-URLs need `server.allow_private_media_urls`. Poll `/v1/videos/{id}` and download
-`/v1/videos/{id}/content` as in the [serving guide](openai-api.md). Generation
-time for the default 13-second clip is not measured, so raise the client's
-polling deadline for long jobs instead of treating a slow job as failed.
-
-The cookbook's generated serving clients send a text prompt only. The image
-request above is the supported way to supply a first frame over HTTP, and no
-recorded generation backs it yet.
-
-## Evidence
-
-Both recipes are **Source-backed**: the model ID, example and serving YAML are
-checked against each other, and the serving defaults are tested against the
-registered `pan2_t2v` preset. No GPU generation, memory measurement or HTTP
-client run is recorded, so hardware fit and throughput are unknown. Numerical
-parity with the reference implementation is checked on CPU with a public
-random-weight checkpoint, which verifies the port and produces noise rather than
-useful video.

docs/cookbook/recipes/pan2.yaml

@@ -1,38 +0,0 @@
-title: PAN 2
-summary: Text-to-video and first-frame image-to-video from the 29B PAN 2 mixture-of-transformers model.
-
-deployments:
-  cuda-rest:
-    runtime: fastvideo-cuda-rest
-    workload: t2v
-    defaults: examples/serving/openai_pan2.yaml
-    requirements:
-      - >-
-        Requests without a reference run text-to-video. Add image_reference or input_reference
-        to condition on a first frame; the same server and pipeline handle both.
-      - >-
-        To use more than one GPU, set sp_size equal to num_gpus (Ulysses sequence parallelism).
-        It must divide PAN 2's 32 attention heads, and each rank still holds the full DiT weights.
-    options:
-      - server.*
-      - generator.engine.num_gpus
-      - generator.engine.parallelism.*
-      - "!generator.engine.parallelism.hsdp_*"
-      - generator.engine.offload.text_encoder
-      - generator.engine.offload.vae
-      - default_request.sampling.num_frames
-      - default_request.sampling.height
-      - default_request.sampling.width
-      - default_request.sampling.fps
-    overrides:
-      default_request.sampling.num_frames:
-        default: 313
-        description: num_frames - 1 must be a multiple of 4.
-      default_request.sampling.height:
-        default: 704
-        minimum: 32
-        multipleOf: 32
-      default_request.sampling.width:
-        default: 1248
-        minimum: 32
-        multipleOf: 32

docs/cookbook/serving-coverage.md

@@ -49,7 +49,6 @@ never deleted. See `examples/serving/README.md`.
 | `openai_hunyuan15_i2v_480p.yaml` | HunyuanVideo 1.5 I2V 480p | `i2v` | CUDA REST | `hunyuan15-i2v-480p` | Source-backed; preset renders 720x1280 |
 | `openai_kandinsky6_ti2va.yaml` | Kandinsky 6 Pro TI2VA | `t2v` | CUDA REST | `kandinsky6-ti2va` | Source-backed |
 | `openai_kandinsky6_ti2va_distilled.yaml` | Kandinsky 6 Pro-distill TI2VA | `t2v` | CUDA REST | `kandinsky6-ti2va-distilled` | Source-backed |
-| `openai_pan2.yaml` | PAN 2 | `t2v` (a reference switches to image-to-video) | CUDA REST | `pan2/cuda-rest` | Source-backed; checkpoint not public, no recorded run |
 | `openai_ltx2_distilled.yaml` | LTX2 Distilled | `t2v` | CUDA REST | `ltx2-distilled/cuda-rest` | Source-backed |
 | `streaming_demo.yaml` | LTX2 Distilled | `t2v` | CUDA streaming | `ltx2-distilled/cuda-streaming` | Source-backed |
 | `mlx_fasth3.yaml` | FastH3 V1 converted | `t2v` | MLX REST | none (legacy catalog only) | Source runtime; T2VA only |

docs/design/inference_schema_parity_inventory.yaml

@@ -283,18 +283,6 @@ surfaces:
           - fastvideo.configs.pipelines.hunyuan15.Hunyuan15T2V720PConfig
           - fastvideo.configs.pipelines.hyworld.HYWorldConfig
           - fastvideo.configs.pipelines.hyworld.Hunyuan15T2V480PConfig
-      text_system_prompt:
-        sources:
-          - fastvideo.configs.pipelines.pan2.PAN2PipelineConfig
-      text_encoder_skip_layers:
-        sources:
-          - fastvideo.configs.pipelines.pan2.PAN2PipelineConfig
-      text_max_sequence_length:
-        sources:
-          - fastvideo.configs.pipelines.pan2.PAN2PipelineConfig
-      i2v_skip_first_latent_cfg:
-        sources:
-          - fastvideo.configs.pipelines.pan2.PAN2PipelineConfig
       text_encoder_max_lengths:
         sources:
           - fastvideo.configs.pipelines.hunyuan15.Hunyuan15I2V480PStepDistilledConfig

docs/generate_examples.py

@@ -35,7 +35,6 @@ COOKBOOK_FAMILIES = {
     "cosmos",
     "kandinsky5",
     "kandinsky6",
-    "pan2",
     "flux",
     "qwen_image21",
     "sd35",

docs/inference/open_h3_regenerate_2k.md

@@ -0,0 +1,196 @@
+# Open-H3-Regenerate-2K
+
+`OpenH3Regenerate2KPipeline` regenerates a 768P [MiniMax-H3](https://huggingface.co/MiniMaxAI/MiniMax-H3) draft video
+(1344x768, 121 frames, 24 fps) at **2560x1440** with three refine steps. It is video to video: the draft is the input,
+and the output is a silent 2K MP4. FastVideo ports NVlabs'
+[Open-H3-Regenerate-2K](https://huggingface.co/Efficient-Large-Model/Open-H3-Regenerate-2K)
+([project page](https://nvlabs.github.io/Sana/Open-H3-Regenerate-2K/),
+[reference code](https://github.com/NVlabs/Sana/tree/sol-engine/models/Open-H3-Regenerate-2K)); the dense BF16
+`standard_fwd` path of that repo is the parity reference. The accelerated `sol_engine_fwd` path (Sol-Attn, MXFP8, CUDA
+graphs) and the fast 24-layer VAE decoder are not ported.
+
+```text
+draft video -> H3 VAE encode -> latent upscaler (2x, 3D conv) -> 3 Euler refine steps -> H3 VAE decode -> 2560x1440
+                                                 H3 DiT + merged LoRA + Sharp curve AdaLN branches
+```
+
+## What runs
+
+| Step | Details |
+|---|---|
+| Draft encode | `ffmpeg` resamples to 24 fps, cover-scales (bilinear) and centre-crops to 1344x768 (`draft_size: 768p`, or 1024x576 for `576p`), holds the last frame to 121 frames plus 3 copies. The FP32 H3 VAE posterior is sampled under seed 42. |
+| Latent upscale | The author's 3D-conv upscaler (`fastvideo/models/upsamplers/minimax_h3_latent_upscaler.py`), BF16, to the 160x90 latent grid. |
+| Refine | Euler on sigmas `[0.909375, 0.857143, 0.666667, 0]`, CFG 1, model time `1 - sigma`, initial state `(1 - 0.909375) * upscaled + 0.909375 * noise`. `num_inference_steps: 2` uses `[0.7, 0.494118, 0]` (softer, about 30% faster). |
+| Transformer | The base MiniMax-H3 DiT (the same module as the other H3 pipelines) with the released merged LoRA (FP32, rank 576 on 312 linears, scale 1) and 50 LMS Sharp curve AdaLN branches. |
+| Decode | The released H3 VAE, or an NVFP4 decoder on Blackwell (see below); the output is trimmed to 121 frames. |
+
+The merged LoRA loads through the standard H3 community-LoRA path as a constructor-time adapter: the pipeline sets
+`lora_path` to `<model>/lora/open_h3_regenerate_2k_lora.safetensors` when you do not pass one. The Sharp branches take an
+8-D timestep curve (`lora/adaln_t_table.bin`, 1025x8) instead of the layer input, so they cannot be folded into weights;
+`sharp_curve.py` adds them to the AdaLN projection outputs with forward hooks.
+
+**Audio.** The reference feeds an encoded-silence audio stream to the DiT as conditioning and writes the video with no
+audio track (`-an`); it neither regenerates nor passes through the draft's audio. FastVideo does the same: the draft's
+soundtrack is ignored and the output MP4 has no audio.
+
+## Weights
+
+The release repo keeps the unmodified MiniMax-H3 components (a copy of `MiniMaxAI/MiniMax-H3` at `6818f6c`) under
+`base_model/` next to `lora/` and `latent_upsampler/`. Point `model_path` at the Hub id or at a local copy of that layout.
+`adaln_t_table.bin` is base-derived and is not redistributed in the release repo: put the 32,800-byte file (SHA-256
+`c02a6c11888297688c1e6278185ea1f947023acfc69f9003bbcdcec9a229a8e7`) at `lora/adaln_t_table.bin`. The reference
+`download.py` fetches it by byte range from `Comfy-Org/MiniMax-H3` at revision
+`e5eb578a89295337b8ff433a035929ce0279e0b6` (tensor `adaln_t_table` of
+`diffusion_models/minimax_h3_ref2va_pruned_int8_convrot.safetensors`) and verifies the hash.
+
+A GB200/B200-class GPU is needed (about 150 GB with the DiT, the 2K activations and the VAE).
+
+## Usage
+
+```bash
+python examples/inference/basic/basic_open_h3_regenerate_2k.py \
+    --model-path Efficient-Large-Model/Open-H3-Regenerate-2K \
+    --video draft_768p.mp4 --prompt "A red fox trots through fresh snow" --output outputs/regen2k
+```
+
+The prompt conditions the refine steps (use the draft's own prompt). Drafts from other video models are better encoded at
+576P (`--draft-size 576p`). `--steps 2` runs the two-step variant. The output is always 2560x1440; the first 121 frames of
+the draft are used.
+
+## Serving
+
+```bash
+fastvideo serve --config examples/serving/openai_open_h3_regenerate_2k.yaml
+```
+
+`examples/serving/openai_open_h3_regenerate_2k.yaml` loads the release on one GPU, sets `media_roots`, and pins the 2K
+defaults. The draft arrives as the typed `video_reference` of `POST /v1/videos/sync` (the response body is the MP4) or of
+`POST /v1/videos` (an async job; poll `GET /v1/videos/{id}` and download `GET /v1/videos/{id}/content`):
+
+```bash
+curl -sS -X POST http://localhost:8000/v1/videos/sync \
+  -H 'Content-Type: application/json' \
+  -d '{
+        "prompt": "A red fox trots through fresh snow",
+        "video_reference": {"video_url": "https://example.com/draft_768p.mp4"},
+        "seed": 1766580932
+      }' \
+  -o regen_2k.mp4 -D -
+```
+
+The response headers carry `X-Request-Id`, `X-Model`, `X-Inference-Time-S` and `X-Stage-Durations`. Rules:
+
+- Exactly one `video_reference` (or the legacy `video_path` / `video_url`) and no image or audio references.
+- `http(s)` URLs must resolve to public addresses unless `server.allow_private_media_urls` is set. A local path is accepted
+  only inside a directory listed in `server.media_roots`. `data:` URLs are refused for video references; this policy is
+  unchanged.
+- `size`, `width` and `height` may be omitted or `2560x1440`; `num_frames` may be omitted or 124 (the frame count of the
+  VAE grid; the video has 121 frames); `fps` must be 24. `seconds` is not accepted.
+- `num_inference_steps` is 3 (default) or 2; `seed` sets the refine noise.
+
+## NVFP4 video decoder
+
+On Blackwell GPUs the released 36-block video decoder can be replaced by a pre-packed NVFP4 one; the latent space is
+unchanged, so the draft encode, the upscale and the refine steps are the same and only the decode changes. Both tiers of
+[FastVideo/FastH3-NVFP4-VAE-Decoder](https://huggingface.co/FastVideo/FastH3-NVFP4-VAE-Decoder) apply here, because the
+release carries MiniMax-H3's full 36-block VAE (`base_model/vae`):
+
+| Tier | Decoder blocks | File | Pick it when |
+|---|---|---|---|
+| `quality-36L` | 36 | `quality-36L/decoder-nvfp4.safetensors` | you want the released detail back; the default choice |
+| `fast-8L` | 8 (evenly spaced; the load drops 28 blocks) | `fast-8L/decoder-nvfp4.safetensors` | decode time or memory dominate and some fine detail can go |
+
+```bash
+hf download FastVideo/FastH3-NVFP4-VAE-Decoder --include 'quality-36L/decoder-nvfp4.safetensors' \
+    'fast-8L/decoder-nvfp4.safetensors' --local-dir fasth3-nvfp4-vae
+```
+
+Select it through the surface you drive the pipeline with; the field, the CLI flag and the environment variable all
+set `fastvideo_args.h3_vae_nvfp4_decoder`, and the serve-config field wins over the environment.
+
+```bash
+# example script
+python examples/inference/basic/basic_open_h3_regenerate_2k.py --video draft.mp4 --prompt "..." \
+    --vae-decoder fasth3-nvfp4-vae/quality-36L/decoder-nvfp4.safetensors
+# `fastvideo generate` and the serve config use the same field
+fastvideo generate --model-path Efficient-Large-Model/Open-H3-Regenerate-2K --video draft.mp4 --prompt "..." \
+    --h3-vae-nvfp4-decoder fasth3-nvfp4-vae/quality-36L/decoder-nvfp4.safetensors
+```
+
+```yaml
+generator:
+  pipeline:
+    experimental:
+      h3_vae_nvfp4_decoder: ./fasth3-nvfp4-vae/quality-36L/decoder-nvfp4.safetensors
+```
+
+`FASTVIDEO_H3_VAE_NVFP4_DECODER` stays as the fallback when neither the serve config nor the CLI sets the field. The
+swap happens in the VAE loader, so the dense decoder never reaches the GPU: it is replaced on the CPU by the exported
+checkpoint (2.0 GiB of packed weights instead of 9.7 GiB of FP32), moved to the device, and decoded under bf16 autocast.
+The decoder keeps the pipeline's tiling (at 2560x1440: 8x13 tiles of 256 px with 64 px overlap) and the fused Triton
+block kernels, which are on by default (`FASTVIDEO_H3_VAE_NVFP4_FUSED`); `FASTVIDEO_H3_VAE_TILE_BATCH` decodes several
+tiles per decoder call (4 is bit-identical and faster here), and `FASTVIDEO_H3_VAE_NVFP4_CUDA_GRAPH` replays the small
+fused calls. The MP4 is unaffected: the same 121 frames are trimmed and written.
+
+Warm decode of one refined latent at 2560x1440 (124 frames) on one GB200, median of 3, with the released decode as the
+reference:
+
+| Decoder | Decode (s) | Peak GPU memory (GiB) | PSNR (dB) | SSIM |
+|---|---:|---:|---:|---:|
+| released H3 VAE (dense, fp16 autocast) | 23.3 | 18.4 | - | - |
+| `quality-36L` (fused) | 6.6 | 6.2 | 41.3 | 0.979 |
+| `quality-36L` (fused, tile batch 4) | 5.0 | 6.3 | 41.3 | 0.979 |
+| `quality-36L` (eager NVFP4 GEMM) | 67.0 | 6.2 | 41.3 | 0.979 |
+| `fast-8L` (fused) | 1.7 | 5.3 | 32.8 | 0.892 |
+| `fast-8L` (eager NVFP4 GEMM) | 16.1 | 5.2 | 32.8 | 0.892 |
+
+The fused kernels are an exact speedup: the eager and fused rows decode bit-identical frames, and so does batching
+tiles, so the PSNR/SSIM columns measure the tier and nothing else. The eager NVFP4 GEMM path loses to the dense fp16
+decoder at these tile sizes (67 s against 23 s), so keep `FASTVIDEO_H3_VAE_NVFP4_FUSED` on. PSNR against the released
+decode is bounded by the released decode itself: the refined latent already sits 45.2 dB from the NVlabs reference (see
+below), so 41.3 dB is a decoder-only difference of the same order. `fast-8L` softens fine texture (fur, foliage) while
+keeping structure; use it only when the decode budget is tight.
+
+## Parity with the reference
+
+Measured on one GB200 (tray hpc-rack-1-5, BF16, dense attention) against the reference `standard_fwd` with the released
+VAE, same draft, seed (1766580932), prompt and VAE posterior seed (42). The reference's noise is a CUDA generator draw
+that FastVideo reproduces (the same `torch.Generator(device="cuda")` seed), so no injected noise was needed.
+
+| Stage / quantity | Fox draft (FastVideo H3 t2v, 768P) | Dance draft (576P, project page) |
+|---|---|---|
+| Draft latent, upscaled latent, DiT input | bit-identical | bit-identical |
+| One DiT forward (LoRA + Sharp), relative L2 | 1.1e-2 | 2.6e-2 |
+| Refined latent, relative L2 | 1.3e-2 | 3.7e-2 |
+| Decoded video PSNR / SSIM (21 frames) | 45.2 dB / 0.993 | 42.1 dB / 0.992 |
+
+The differences come from text-encoder rounding (relative L2 1e-3 on the prompt embeddings) and BF16 kernel choices;
+the reference reports the same order (1.4% relative L2 on the refined latent between its own runs).
+
+Warm timings, one GB200 (seconds):
+
+| | Upscale | Refine (3 steps) | VAE decode | Total |
+|---|---:|---:|---:|---:|
+| Reference `standard_fwd` | 0.87 | 71.6 | 15.0 | 87.5 |
+| FastVideo (end to end, incl. draft encode 5-10 s and decode to host) | 5.9-10.7 (encode + upscale + silence) | 69.5 | 25.0 | 107.5 |
+
+The first request builds nothing extra but loads weights (about 70 s once). The decode stage is slower than the
+reference because it also copies 121 frames to host memory and the MP4 encode is outside both columns.
+
+## Licences
+
+The release is not covered by one licence; its `README.md` says so, and the runtime code's licence does not replace the
+model licences. FastVideo only loads these weights and does not redistribute them.
+
+- The refiner LoRA, the fast H3 VAE decoder and `base_model/` (a copy of `MiniMaxAI/MiniMax-H3` at `6818f6c`: DiT, VAEs,
+  text encoder) follow the
+  [MiniMax H3 Community License](https://huggingface.co/MiniMaxAI/MiniMax-H3/blob/6818f6c32d12b210915e44ad56a4228c2608f160/LICENSE),
+  including its territory, use and redistribution terms (`base_model/LICENSE`, `NOTICE`). The Qwen3-VL text encoder's
+  Apache-2.0 text ships as `LICENSE-Qwen.txt`.
+- The latent upscaler follows its own licence:
+  [LBH-123-AI/Comfyui_Minimax_h3_latent_Upscaler](https://github.com/LBH-123-AI/Comfyui_Minimax_h3_latent_Upscaler)
+  (MIT, per the reference code) and its [weights](https://huggingface.co/LBH-123-AI/Minimax_h3_latent_Upscaler)
+  (`latent_upsampler/LICENSE`, Apache-2.0). `minimax_h3_latent_upscaler.py` is a port of that node's network.
+- The merged LoRA includes [LightX2V FL2VA Turbo](https://huggingface.co/lightx2v/Minimax-h3-Turbo) and
+  [LMS Sharp](https://huggingface.co/Alissonerdx/Minimax-H3-ComfyUI), which keep their own licences; the Sharp
+  `adaln_t_table` is read from [Comfy-Org/MiniMax-H3](https://huggingface.co/Comfy-Org/MiniMax-H3).

docs/inference/pan2.md

@@ -1,86 +0,0 @@
-# PAN 2 Text/Image to Video
-
-PAN 2 is a 29B mixture-of-transformers video DiT with separate video and text streams, a Qwen3.5 text encoder and the
-Wan2.2 VAE. One pipeline serves text-to-video and image-to-video: pass `image_path` to condition on a first frame and
-leave it out for text only.
-
-## Model
-
-| Hub repo | Pipeline | Workloads | Example |
-|---|---|---|---|
-| `IFM/PAN2` | `PAN2ModularPipeline` | T2V, I2V | [`basic_pan2.py`](https://github.com/hao-ai-lab/FastVideo/blob/main/examples/inference/basic/basic_pan2.py) |
-
-PAN2 checkpoints use the Modular Diffusers layout (`modular_model_index.json`). Any repo id or local directory whose
-name contains `pan2` (or whose manifest declares `PAN2ModularPipeline`) resolves to this pipeline. The defaults come
-from the `pan2_t2v` and `pan2_i2v` presets:
-
-| Setting | Default |
-|---|---|
-| Resolution | 704x1248 (height and width must be multiples of 32) |
-| Frames | 313, 13 s at 24 fps (`num_frames - 1` must be a multiple of 4) |
-| Steps | 50, uniform sigmas from 1 to 0 with the checkpoint's flow shift (9 for `IFM/PAN2-TEST`, the reference sampler's value) |
-| Guidance | 3.0, classifier-free guidance. Image-to-video keeps the first latent frame out of guidance (`i2v_skip_first_latent_cfg`, on by default, as in the reference sampler) |
-| Negative prompt | None unless the request carries one. The reference sampler always sends the release's structured negative caption (`negative_prompt.json` of the PAN2 repository) with its three `time_range_sec` values set to the clip length, `0 to 20` for 481 frames at 24 fps |
-
-PAN 2 is trained on the structured captions its prompt rewriter produces, serialized as one-line JSON. Short raw
-prompts work, with weaker results.
-
-## Usage
-
-```bash
-python examples/inference/basic/basic_pan2.py
-```
-
-```python
-from fastvideo import VideoGenerator
-
-generator = VideoGenerator.from_pretrained(
-    "IFM/PAN2",
-    num_gpus=1,
-    dit_cpu_offload=False,
-    dit_layerwise_offload=False,
-    text_encoder_cpu_offload=True,
-)
-generator.generate_video(
-    "A red fox trots through fresh snow in a pine forest at dawn.",
-    image_path=None,  # or the path of a first frame for image-to-video
-    output_path="video_samples_pan2",
-)
-```
-
-For image-to-video, the first frame is scaled to cover the requested resolution and center cropped, then encoded by the
-VAE into the first latent frame of the conditioning input.
-
-## Parallelism
-
-With `num_gpus > 1`, FastVideo runs Ulysses sequence parallelism: the video tokens are sharded across ranks and the
-text tokens are replicated. The number of GPUs must divide the 32 attention heads. Sequence-parallel padding of the
-video tokens is trimmed before attention, so no attention mask is needed.
-
-Padding is always excluded, without an opt-in setting. At 704x1248 with 313 frames, the 67,782 video tokens need
-two padding tokens at SP=4. They are removed before joining the text tokens in every attention block. This provides
-the same exclusion as vLLM-Omni's unconditional padding mask in
-[revision `e0c4c731`](https://github.com/vllm-project/vllm-omni/commit/e0c4c73158a5e4f44eda8eaa8879f7fd9cedbd74).
-The trimmed sequence can use either FlashAttention or Torch SDPA.
-
-```bash
-PAN2_NUM_GPUS=4 python examples/inference/basic/basic_pan2.py
-```
-
-## Memory and offloading
-
-The example keeps the DiT on the GPUs and offloads the text encoder between requests. Sequence parallelism shards
-video tokens; each rank still holds the DiT weights. Increasing `num_gpus` alone does not shard those weights.
-For devices that cannot hold the DiT, see [CPU offloading](offloading.md) for `dit_cpu_offload` and
-`dit_layerwise_offload`. Hardware recipes and timings reported by vLLM-Omni use its own offload implementation;
-they have not been validated for FastVideo.
-
-The full `IFM/PAN2` checkpoint is not public yet. Local parity tests use the public random-weight
-`wuqing157/tiny-pan2-modular-pipe` checkpoint; it verifies the integration and produces noise rather than useful video.
-
-## Differences from the vLLM-Omni integration
-
-- The optional prompt and video guardrails (`pan2-guardrail`) are not included.
-- The Cache-DiT `quality` policy and CFG-parallel execution are vLLM-Omni serving features and are not ported.
-- The text encoder is a FastVideo-native Qwen3.5 port. Its Gated DeltaNet layers use a PyTorch chunked delta rule
-  instead of the `fla` kernels.

docs/inference/support_matrix.md

@@ -68,7 +68,7 @@ column links a runnable script in `examples/inference/basic/` where one exists.
 | matrixgame | `FastVideo/Matrix-Game-2.0-Base-Distilled-Diffusers`<br />`FastVideo/Matrix-Game-2.0-GTA-Distilled-Diffusers`<br />`FastVideo/Matrix-Game-2.0-TempleRun-Distilled-Diffusers`<br />`FastVideo/Matrix-Game-2.0-Base-Diffusers`<br />`FastVideo/Matrix-Game-2.0-GTA-Diffusers`<br />`FastVideo/Matrix-Game-2.0-TempleRun-Diffusers`<br />`mignonjia/mg_longtuning_distilled_zelda`<br />`mignonjia/mg_sf_distilled_zelda_1k_steps`<br />`mignonjia/mg_sf_distilled_zelda`<br />`mignonjia/mg_causal_zelda`<br />`mignonjia/mg_bidirectional_zelda` | I2V | [basic_matrixgame2.py](https://github.com/hao-ai-lab/FastVideo/blob/main/examples/inference/basic/basic_matrixgame2.py) |
 | matrixgame | `FastVideo/Matrix-Game-3.0-Base-Distilled-Diffusers` | I2V | [basic_matrixgame3.py](https://github.com/hao-ai-lab/FastVideo/blob/main/examples/inference/basic/basic_matrixgame3.py) |
 | minimax_h3 | `MiniMaxAI/MiniMax-H3` | T2V, I2V | [T2VA](https://github.com/hao-ai-lab/FastVideo/blob/main/examples/inference/basic/basic_minimax_h3_t2v.py)<br />[FL2VA](https://github.com/hao-ai-lab/FastVideo/blob/main/examples/inference/basic/basic_minimax_h3_fl2va.py)<br />[Ref2VA](https://github.com/hao-ai-lab/FastVideo/blob/main/examples/inference/basic/basic_minimax_h3_ref2va.py) |
-| pan2 | `IFM/PAN2` | T2V, I2V | [basic_pan2.py](https://github.com/hao-ai-lab/FastVideo/blob/main/examples/inference/basic/basic_pan2.py) |
+| open_h3_regenerate_2k | `Efficient-Large-Model/Open-H3-Regenerate-2K` | — | [basic_open_h3_regenerate_2k.py](https://github.com/hao-ai-lab/FastVideo/blob/main/examples/inference/basic/basic_open_h3_regenerate_2k.py) |
 | sd35 | `stabilityai/stable-diffusion-3.5-medium` | T2I | [basic_sd35_t2i.py](https://github.com/hao-ai-lab/FastVideo/blob/main/examples/inference/basic/basic_sd35_t2i.py) |
 | qwen_image21 | `Qwen/Qwen-Image-2.1` | T2I, I2I | [basic_qwen_image21.py](https://github.com/hao-ai-lab/FastVideo/blob/main/examples/inference/basic/basic_qwen_image21.py) |
 | stable_audio | `FastVideo/stable-audio-open-1.0-Diffusers` | T2V | [basic_stable_audio.py](https://github.com/hao-ai-lab/FastVideo/blob/main/examples/inference/basic/basic_stable_audio.py) |
@@ -122,6 +122,9 @@ and distilled) generate video with audio from text, optionally plus an image (se
 `kandinsky6_sr` IDs upscale an existing video (see the
 [Video SR guide](kandinsky6_sr.md)).
 
+**Note (Open-H3-Regenerate-2K)**: `open_h3_regenerate_2k` regenerates an existing 768P MiniMax-H3 draft video at
+2560x1440 (video to video, silent output); see the [Open-H3-Regenerate-2K guide](open_h3_regenerate_2k.md).
+
 **Note (LTX-2.5)**: T2V and I2V produce synchronized video and audio. The
 official `Lightricks/LTX-2.5` repository uses gated, split component files, so
 it must first be converted to FastVideo's component layout. See the

examples/inference/basic/basic_open_h3_regenerate_2k.py

@@ -0,0 +1,93 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Regenerate a 768P MiniMax-H3 draft video at 2560x1440 with Open-H3-Regenerate-2K.
+
+The draft is VAE-encoded (the first 121 frames at 24 fps, 1344x768), its latent is upscaled, and three refine steps
+of the H3 transformer with the released LoRA run at 2560x1440. The output video is silent. See
+docs/inference/open_h3_regenerate_2k.md.
+
+    python examples/inference/basic/basic_open_h3_regenerate_2k.py --video draft.mp4 --prompt "..." --output out/
+"""
+
+from __future__ import annotations
+
+import argparse
+import time
+from pathlib import Path
+
+from fastvideo import VideoGenerator
+from fastvideo.api import (
+    ComponentConfig,
+    EngineConfig,
+    GenerationRequest,
+    GeneratorConfig,
+    InputConfig,
+    OffloadConfig,
+    OutputConfig,
+    PipelineSelection,
+    SamplingConfig,
+)
+
+DEFAULT_MODEL = "Efficient-Large-Model/Open-H3-Regenerate-2K"
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--model-path", default=DEFAULT_MODEL, help="release repo id or a local copy of it")
+    parser.add_argument("--video", required=True, help="768P draft video (URL-free local path)")
+    parser.add_argument("--prompt", required=True, help="text the refine steps are conditioned on")
+    parser.add_argument("--output", type=Path, default=Path("outputs/open_h3_regenerate_2k"))
+    parser.add_argument("--steps", type=int, choices=(2, 3), default=3, help="refine steps (3 is the released setting)")
+    parser.add_argument("--seed", type=int, default=1766580932)
+    parser.add_argument("--attention-backend", default="FLASH_ATTN")
+    parser.add_argument("--draft-size", choices=("768p", "576p"), default="768p",
+                        help="size the draft is encoded at; 576p for drafts from other video models")
+    parser.add_argument("--vae-decoder", default=None,
+                        help="NVFP4 decoder-nvfp4.safetensors that replaces the VAE decoder at load "
+                        "(Blackwell only, see docs/inference/open_h3_regenerate_2k.md)")
+    parser.add_argument("--runs", type=int, default=1, help="repeat the request to measure warm timings")
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    args.output.mkdir(parents=True, exist_ok=True)
+    experimental = {
+        "attention_backend": args.attention_backend,
+        "draft_size": args.draft_size,
+    }
+    if args.vae_decoder:
+        experimental["h3_vae_nvfp4_decoder"] = args.vae_decoder
+    generator = VideoGenerator.from_config(
+        GeneratorConfig(
+            model_path=args.model_path,
+            engine=EngineConfig(num_gpus=1, offload=OffloadConfig(dit=False, dit_layerwise=False, text_encoder=True, vae=False)),
+            pipeline=PipelineSelection(
+                components=ComponentConfig(),
+                experimental=experimental,
+            ),
+        ))
+    try:
+        for run in range(args.runs):
+            started = time.perf_counter()
+            result = generator.generate(
+                GenerationRequest(
+                    prompt=args.prompt,
+                    inputs=InputConfig(video_path=args.video),
+                    sampling=SamplingConfig(height=1440,
+                                            width=2560,
+                                            num_frames=124,
+                                            fps=24,
+                                            num_inference_steps=args.steps,
+                                            guidance_scale=1.0,
+                                            seed=args.seed),
+                    output=OutputConfig(output_path=str(args.output / f"regen2k_{run}.mp4"),
+                                        save_video=True,
+                                        return_frames=False),
+                ))
+            print(f"run {run}: {time.perf_counter() - started:.1f}s -> {result.video_path}")
+    finally:
+        generator.shutdown()
+
+
+if __name__ == "__main__":
+    main()

examples/inference/basic/basic_pan2.py

@@ -1,40 +0,0 @@
-import os
-
-from fastvideo import VideoGenerator
-
-# PAN2 is trained on the structured captions its prompt rewriter produces (serialized as one-line JSON). Plain
-# prompts like this one work, with weaker results.
-PROMPT = ("A red fox trots through fresh snow in a pine forest at dawn, its breath visible in the cold air. "
-          "The camera tracks it from the side at ground level as low golden light filters through the trees.")
-
-OUTPUT_PATH = "video_samples_pan2"
-MODEL_PATH = os.environ.get("PAN2_MODEL_PATH", "IFM/PAN2")
-
-# Set to a first frame (e.g. "assets/images/girl.png") for image-to-video; the same pipeline runs both modes.
-IMAGE_PATH = None
-
-
-def main():
-    # The 29B DiT runs in bf16; with more GPUs, num_gpus enables Ulysses sequence parallelism (the 32 attention
-    # heads must be divisible by it).
-    generator = VideoGenerator.from_pretrained(
-        MODEL_PATH,
-        num_gpus=int(os.environ.get("PAN2_NUM_GPUS", "1")),
-        use_fsdp_inference=False,
-        dit_cpu_offload=False,
-        dit_layerwise_offload=False,
-        vae_cpu_offload=False,
-        text_encoder_cpu_offload=True,
-        pin_cpu_memory=True,
-    )
-
-    generator.generate_video(
-        PROMPT,
-        image_path=IMAGE_PATH,
-        output_path=OUTPUT_PATH,
-        save_video=True,
-    )
-
-
-if __name__ == "__main__":
-    main()

examples/serving/README.md

@@ -90,7 +90,6 @@ uncovered registry families.
 | `openai_hunyuan15_i2v_480p.yaml` | HunyuanVideo 1.5 I2V 480p | image-to-video | 1 |
 | `openai_kandinsky6_ti2va.yaml` | Kandinsky 6 Pro TI2VA | text/image-to-video with audio | 1 |
 | `openai_kandinsky6_ti2va_distilled.yaml` | Kandinsky 6 Pro-distill TI2VA | text/image-to-video with audio | 1 |
-| `openai_pan2.yaml` | PAN 2 | text/image-to-video | 1 |
 | `openai_ltx2_distilled.yaml` | LTX2 Distilled | text-to-video with audio | 4 |
 | `openai_flux1_dev.yaml` | FLUX.1-dev | text-to-image | 1 |
 | `openai_flux2_dev.yaml` | FLUX.2-dev | text-to-image | 1 |

examples/serving/openai_open_h3_regenerate_2k.yaml

@@ -0,0 +1,46 @@
+# OpenAI-compatible Open-H3-Regenerate-2K server (one GPU with about 150 GB free, e.g. B200 / GB200).
+# POST /v1/videos/sync (or /v1/videos) with a 768P H3 draft as video_reference; the response is the 2560x1440 MP4.
+# Local draft paths are accepted only inside server.media_roots; http(s) URLs must resolve to public addresses unless
+# server.allow_private_media_urls is set. data: URLs are always refused. See docs/inference/open_h3_regenerate_2k.md.
+generator:
+  model_path: Efficient-Large-Model/Open-H3-Regenerate-2K
+  engine:
+    num_gpus: 1
+    offload:
+      dit: false
+      dit_layerwise: false
+      text_encoder: true
+      vae: false
+      pin_cpu_memory: true
+    compile:
+      enabled: false
+      vae_enabled: false
+  pipeline:
+    experimental:
+      attention_backend: FLASH_ATTN
+      # 768p for MiniMax-H3 drafts; 576p encodes drafts from other video models at 1024x576.
+      draft_size: 768p
+      # NVFP4 video decoder (Blackwell only). Download one tier from FastVideo/FastH3-NVFP4-VAE-Decoder and point
+      # this at it; the release VAE has the full 36 blocks, so both tiers apply. `fast-8L` is the cheap option.
+      # h3_vae_nvfp4_decoder: ./fasth3-nvfp4-vae/quality-36L/decoder-nvfp4.safetensors
+
+server:
+  host: 0.0.0.0
+  port: 8000
+  output_dir: outputs/openai_open_h3_regenerate_2k
+  served_model_name: open-h3-regenerate-2k
+  media_roots:
+    - /data/drafts
+  media_reference_max_bytes: 536870912
+
+default_request:
+  sampling:
+    height: 1440
+    width: 2560
+    num_frames: 124
+    fps: 24
+    num_inference_steps: 3
+    guidance_scale: 1.0
+    seed: 1766580932
+  output:
+    return_frames: false

examples/serving/openai_pan2.yaml

@@ -1,36 +0,0 @@
-# OpenAI-compatible server for PAN 2 text-to-video and first-frame image-to-video.
-#   fastvideo serve --config examples/serving/openai_pan2.yaml
-# Omit image_reference for text-to-video; send it (or input_reference) for image-to-video.
-generator:
-  model_path: IFM/PAN2
-  engine:
-    num_gpus: 1
-    use_fsdp_inference: false
-    parallelism:
-      tp_size: 1
-      sp_size: 1
-    offload:
-      dit: false
-      dit_layerwise: false
-      vae: false
-      text_encoder: true
-      pin_cpu_memory: true
-  pipeline:
-    workload_type: t2v
-
-server:
-  host: 0.0.0.0
-  port: 8000
-  output_dir: outputs/openai_pan2
-  served_model_name: pan2
-
-default_request:
-  sampling:
-    height: 704
-    width: 1248
-    num_frames: 313
-    fps: 24
-    num_inference_steps: 50
-    guidance_scale: 3.0
-  output:
-    return_frames: false

fastvideo/configs/models/dits/__init__.py

@@ -8,7 +8,6 @@ from fastvideo.configs.models.dits.hunyuanvideo15 import HunyuanVideo15Config
 from fastvideo.configs.models.dits.ltx2 import LTX2VideoConfig
 from fastvideo.configs.models.dits.magi_human import MagiHumanVideoConfig
 from fastvideo.configs.models.dits.minimax_h3 import MiniMaxH3Config
-from fastvideo.configs.models.dits.pan2 import PAN2VideoConfig
 from fastvideo.configs.models.dits.mmaudio import (
     MMAUDIO_44K_TRAINING_VARIANTS,
     MMAUDIO_TRAINING_VARIANTS,
@@ -33,7 +32,7 @@ __all__ = [
     "FluxDiTConfig", "Flux2Config", "LTX2VideoConfig", "HYWorldConfig", "Kandinsky5VideoConfig", "Kandinsky6ArchConfig",
     "Kandinsky6VideoAudioConfig", "Kandinsky6SRArchConfig", "Kandinsky6SRConfig", "MagiHumanVideoConfig",
     "StableAudioConfig", "HeliosConfig", "LingBotWorld2CausalFastVideoConfig", "LingBotWorldFastVideoConfig",
-    "LingBotVideoConfig", "MiniMaxH3Config", "PAN2VideoConfig", "MMAudioArchConfig", "MMAudioTransformerConfig",
+    "LingBotVideoConfig", "MiniMaxH3Config", "MMAudioArchConfig", "MMAudioTransformerConfig",
     "MMAUDIO_VARIANT_ARCHITECTURES", "MMAUDIO_TRAINING_VARIANTS", "MMAUDIO_44K_TRAINING_VARIANTS",
     "get_mmaudio_transformer_config", "QwenImage21ArchConfig", "QwenImage21Config"
 ]

fastvideo/configs/models/dits/pan2.py

@@ -1,86 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-from dataclasses import dataclass, field
-
-from fastvideo.configs.models.dits.base import DiTArchConfig, DiTConfig
-from fastvideo.platforms import AttentionBackendEnum
-
-
-def is_transformer_block(n: str, m) -> bool:
-    return "transformer_blocks" in n and str.isdigit(n.split(".")[-1])
-
-
-def is_refiner_block(n: str, m) -> bool:
-    return "context_refiner" in n and str.isdigit(n.split(".")[-1])
-
-
-@dataclass
-class PAN2ArchConfig(DiTArchConfig):
-    _fsdp_shard_conditions: list = field(default_factory=lambda: [is_transformer_block, is_refiner_block])
-    _supported_attention_backends: tuple[AttentionBackendEnum, ...] = (
-        AttentionBackendEnum.FLASH_ATTN,
-        AttentionBackendEnum.TORCH_SDPA,
-    )
-
-    # Diffusers PAN2Transformer3DModel keys -> native keys. The per-stream Q/K/V projections are fused on load.
-    param_names_mapping: dict = field(
-        default_factory=lambda: {
-            r"^x_embedder\.(.*)$": r"x_embedder.proj.\1",
-            r"^time_embedder\.linear_1\.(.*)$": r"time_embedder.mlp.fc_in.\1",
-            r"^time_embedder\.linear_2\.(.*)$": r"time_embedder.mlp.fc_out.\1",
-            r"^context_embedder\.0\.(.*)$": r"context_norm.\1",
-            r"^context_embedder\.1\.(.*)$": r"context_proj.\1",
-            r"^context_refiner\.(\d+)\.attn\.to_q\.(.*)$": (r"context_refiner.\1.attn.to_qkv.\2", 0, 3),
-            r"^context_refiner\.(\d+)\.attn\.to_k\.(.*)$": (r"context_refiner.\1.attn.to_qkv.\2", 1, 3),
-            r"^context_refiner\.(\d+)\.attn\.to_v\.(.*)$": (r"context_refiner.\1.attn.to_qkv.\2", 2, 3),
-            r"^context_refiner\.(\d+)\.attn\.to_out\.0\.(.*)$": r"context_refiner.\1.attn.to_out.\2",
-            r"^context_refiner\.(\d+)\.ff\.net\.0\.proj\.(.*)$": r"context_refiner.\1.ff.fc_in.\2",
-            r"^context_refiner\.(\d+)\.ff\.net\.2\.(.*)$": r"context_refiner.\1.ff.fc_out.\2",
-            r"^transformer_blocks\.(\d+)\.attn\.to_q\.(.*)$": (r"transformer_blocks.\1.attn.to_qkv.\2", 0, 3),
-            r"^transformer_blocks\.(\d+)\.attn\.to_k\.(.*)$": (r"transformer_blocks.\1.attn.to_qkv.\2", 1, 3),
-            r"^transformer_blocks\.(\d+)\.attn\.to_v\.(.*)$": (r"transformer_blocks.\1.attn.to_qkv.\2", 2, 3),
-            r"^transformer_blocks\.(\d+)\.attn\.add_q_proj\.(.*)$":
-            (r"transformer_blocks.\1.attn.add_qkv_proj.\2", 0, 3),
-            r"^transformer_blocks\.(\d+)\.attn\.add_k_proj\.(.*)$":
-            (r"transformer_blocks.\1.attn.add_qkv_proj.\2", 1, 3),
-            r"^transformer_blocks\.(\d+)\.attn\.add_v_proj\.(.*)$":
-            (r"transformer_blocks.\1.attn.add_qkv_proj.\2", 2, 3),
-            r"^transformer_blocks\.(\d+)\.attn\.to_out\.0\.(.*)$": r"transformer_blocks.\1.attn.to_out.\2",
-            r"^transformer_blocks\.(\d+)\.(ff|ff_context)\.net\.0\.proj\.(.*)$": r"transformer_blocks.\1.\2.fc_in.\3",
-            r"^transformer_blocks\.(\d+)\.(ff|ff_context)\.net\.2\.(.*)$": r"transformer_blocks.\1.\2.fc_out.\3",
-        })
-
-    reverse_param_names_mapping: dict = field(default_factory=lambda: {})
-
-    patch_size: tuple[int, ...] = (1, 2, 2)
-    in_channels: int = 97
-    out_channels: int = 48
-    num_attention_heads: int = 32
-    attention_head_dim: int = 128
-    num_layers: int = 44
-    num_refiner_layers: int = 4
-    mlp_ratio: float = 8.0
-    context_mlp_ratio: float = 4.0
-    refiner_mlp_ratio: float = 4.0
-    text_embed_dim: int = 4096
-    rope_axes_dim: tuple[int, ...] = (16, 56, 56)
-    rope_theta: float = 256.0
-    # Frame positions are rescaled to the frame rate the rotary embedding was trained at (16 fps of 24 fps).
-    rope_temporal_scale: float = 16 / 24
-    # PAN2 runs its text stream in the DiT dtype.
-    cast_prompt_embeds_to_dit_dtype: bool = True
-    exclude_lora_layers: list[str] = field(
-        default_factory=lambda: ["x_embedder", "time_embedder", "context_norm", "context_proj", "context_refiner"])
-
-    def __post_init__(self):
-        super().__post_init__()
-        self.patch_size = tuple(self.patch_size)
-        self.rope_axes_dim = tuple(self.rope_axes_dim)
-        self.hidden_size: int = self.attention_head_dim * self.num_attention_heads
-        self.num_channels_latents: int = self.out_channels
-
-
-@dataclass
-class PAN2VideoConfig(DiTConfig):
-    arch_config: DiTArchConfig = field(default_factory=PAN2ArchConfig)
-
-    prefix: str = "PAN2"

fastvideo/configs/models/encoders/__init__.py

@@ -16,7 +16,6 @@ from fastvideo.configs.models.encoders.mistral3 import Mistral3TextConfig
 from fastvideo.configs.models.encoders.minimax_h3_qwen3_vl import (MiniMaxH3Qwen3VLArchConfig, MiniMaxH3Qwen3VLConfig)
 from fastvideo.configs.models.encoders.qwen3 import Qwen3TextConfig
 from fastvideo.configs.models.encoders.qwen_image21 import QwenImage21Qwen3VLArchConfig, QwenImage21Qwen3VLConfig
-from fastvideo.configs.models.encoders.qwen3_5 import Qwen3_5TextArchConfig, Qwen3_5TextConfig
 from fastvideo.configs.models.encoders.lingbot_video import LingBotVideoQwen3VLTextConfig
 from fastvideo.configs.models.encoders.stable_audio_conditioner import (
     StableAudioConditionerArchConfig,
@@ -53,8 +52,6 @@ __all__ = [
     "Qwen3TextConfig",
     "QwenImage21Qwen3VLArchConfig",
     "QwenImage21Qwen3VLConfig",
-    "Qwen3_5TextArchConfig",
-    "Qwen3_5TextConfig",
     "Mistral3TextConfig",
     "LingBotWorld2UMT5ArchConfig",
     "LingBotWorld2UMT5Config",

fastvideo/configs/models/encoders/qwen3_5.py

@@ -1,119 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""Qwen3.5 text encoder configuration (hybrid Gated DeltaNet + gated attention)."""
-from dataclasses import dataclass, field
-from typing import Any
-
-from fastvideo.configs.models.encoders.base import (
-    TextEncoderArchConfig,
-    TextEncoderConfig,
-)
-
-_LEGACY_LAYER_TYPES = {
-    "mamba": "linear_attention",
-    "conv": "linear_attention",
-    "attention": "full_attention",
-}
-_DEFAULT_ROPE_THETA = 10000.0
-_DEFAULT_PARTIAL_ROTARY_FACTOR = 0.25
-
-
-def _is_transformer_layer(n: str, m: Any) -> bool:
-    return "layers" in n and str.isdigit(n.split(".")[-1])
-
-
-def _is_embeddings(n: str, m: Any) -> bool:
-    return n.endswith("embed_tokens")
-
-
-def _is_final_norm(n: str, m: Any) -> bool:
-    return n == "norm"
-
-
-@dataclass
-class Qwen3_5TextArchConfig(TextEncoderArchConfig):
-    """Architecture config for the Qwen3.5 text model (``Qwen3_5TextModel``).
-
-    Layers alternate between Gated DeltaNet linear attention and gated full
-    attention according to ``layer_types``. The defaults describe the 9B text
-    tower; checkpoint ``config.json`` values override them at load time.
-    """
-
-    vocab_size: int = 248320
-    hidden_size: int = 4096
-    intermediate_size: int = 12288
-    num_hidden_layers: int = 32
-    num_attention_heads: int = 16
-    num_key_value_heads: int = 4
-    head_dim: int = 256
-    hidden_act: str = "silu"
-    max_position_embeddings: int = 32768
-    initializer_range: float = 0.02
-    rms_norm_eps: float = 1e-6
-    use_cache: bool = True
-    tie_word_embeddings: bool = False
-    attention_bias: bool = False
-    attention_dropout: float = 0.0
-    linear_conv_kernel_dim: int = 4
-    linear_key_head_dim: int = 128
-    linear_value_head_dim: int = 128
-    linear_num_key_heads: int = 16
-    linear_num_value_heads: int = 32
-    layer_types: list[str] | None = None
-    full_attention_interval: int = 4
-    # RoPE settings as written by Transformers 5 (``rope_parameters``) or by
-    # older configs (top-level ``rope_theta`` / ``partial_rotary_factor`` /
-    # ``rope_scaling``). Values inside ``rope_parameters`` take precedence.
-    rope_parameters: dict[str, Any] | None = None
-    rope_scaling: dict[str, Any] | None = None
-    rope_theta: float | None = None
-    partial_rotary_factor: float | None = None
-    pad_token_id: int | None = None
-    bos_token_id: int | None = None
-    eos_token_id: int | list[int] | None = None
-
-    _fsdp_shard_conditions: list = field(
-        default_factory=lambda: [_is_transformer_layer, _is_embeddings, _is_final_norm])
-
-    def __post_init__(self) -> None:
-        super().__post_init__()
-        # Validate eagerly; the resolved views are recomputed on demand because
-        # ``update_model_arch`` re-runs this hook after replacing raw fields.
-        self.resolved_layer_types()
-        self.resolved_rope_parameters()
-
-    def resolved_layer_types(self) -> list[str]:
-        """Return one ``linear_attention`` / ``full_attention`` entry per layer."""
-        if self.layer_types is None:
-            interval = self.full_attention_interval
-            layer_types = [
-                "linear_attention" if (index + 1) % interval else "full_attention"
-                for index in range(self.num_hidden_layers)
-            ]
-        else:
-            layer_types = [_LEGACY_LAYER_TYPES.get(kind, kind) for kind in self.layer_types]
-        if len(layer_types) < self.num_hidden_layers:
-            raise ValueError(f"Qwen3.5 layer_types has {len(layer_types)} entries for "
-                             f"{self.num_hidden_layers} layers")
-        unknown = set(layer_types) - {"linear_attention", "full_attention"}
-        if unknown:
-            raise ValueError(f"Unsupported Qwen3.5 layer types: {sorted(unknown)}")
-        return layer_types[:self.num_hidden_layers]
-
-    def resolved_rope_parameters(self) -> dict[str, Any]:
-        """Return the effective text RoPE settings with Transformers' precedence."""
-        rope = dict(self.rope_scaling or self.rope_parameters or {})
-        rope.setdefault("rope_type", rope.get("type", "default"))
-        rope.setdefault("rope_theta", self.rope_theta if self.rope_theta is not None else _DEFAULT_ROPE_THETA)
-        rope.setdefault(
-            "partial_rotary_factor",
-            self.partial_rotary_factor if self.partial_rotary_factor is not None else _DEFAULT_PARTIAL_ROTARY_FACTOR,
-        )
-        return rope
-
-
-@dataclass
-class Qwen3_5TextConfig(TextEncoderConfig):
-    """Top-level config for the Qwen3.5 text encoder."""
-
-    arch_config: TextEncoderArchConfig = field(default_factory=Qwen3_5TextArchConfig)
-    prefix: str = "qwen3_5"

fastvideo/configs/pipelines/__init__.py

@@ -15,7 +15,6 @@ from fastvideo.configs.pipelines.matrixgame2 import MatrixGame2I2V480PConfig
 from fastvideo.configs.pipelines.matrixgame3 import MatrixGame3I2V720PConfig
 from fastvideo.configs.pipelines.mmaudio import MMAudioV2AConfig
 from fastvideo.configs.pipelines.qwen_image21 import QwenImage21PipelineConfig
-from fastvideo.configs.pipelines.pan2 import PAN2PipelineConfig
 from fastvideo.pipelines.basic.ltx2.pipeline_configs import LTX2T2VConfig
 from fastvideo.registry import get_pipeline_config_cls_from_name
 from fastvideo.models.wan.pipeline_config import (LucyEditDevConfig, SelfForcingWanT2V480PConfig, WanI2V480PConfig,
@@ -28,5 +27,5 @@ __all__ = [
     "Kandinsky5T2VConfig", "Kandinsky5I2VConfig", "Kandinsky5DMDConfig", "Kandinsky6TI2VAConfig",
     "Kandinsky6SRPipelineConfig", "LingBotWorld2CausalFastI2V480PConfig", "LingBotWorldFastI2V480PConfig",
     "LingBotVideoT2VConfig", "MatrixGame2I2V480PConfig", "MatrixGame3I2V720PConfig", "MMAudioV2AConfig",
-    "QwenImage21PipelineConfig", "PAN2PipelineConfig", "get_pipeline_config_cls_from_name"
+    "QwenImage21PipelineConfig", "get_pipeline_config_cls_from_name"
 ]

fastvideo/configs/pipelines/open_h3_regenerate_2k.py

@@ -0,0 +1,30 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Pipeline configuration for Open-H3-Regenerate-2K (768P H3 draft -> 2560x1440)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from fastvideo.configs.pipelines.minimax_h3 import MiniMaxH3PipelineConfig
+
+DRAFT_SIZE_CHOICES = ("768p", "576p")
+
+
+@dataclass
+class OpenH3Regenerate2KPipelineConfig(MiniMaxH3PipelineConfig):
+    """MiniMax-H3 policy plus the refine-stage knobs.
+
+    ``draft_size`` is the size the draft video is VAE-encoded at: 768p (1344x768) for MiniMax-H3 drafts, 576p
+    (1024x576) for drafts from other video models. ``refine_start_sigma`` re-noises the upscaled draft to another
+    level; the later steps then follow the shift-6 rule (None keeps the released grid).
+    """
+
+    draft_size: str = "768p"
+    refine_start_sigma: float | None = None
+
+    def check_pipeline_config(self) -> None:
+        super().check_pipeline_config()
+        if self.draft_size not in DRAFT_SIZE_CHOICES:
+            raise ValueError(f"draft_size must be one of {DRAFT_SIZE_CHOICES}, got {self.draft_size!r}.")
+        if self.refine_start_sigma is not None and not 0.0 < self.refine_start_sigma < 1.0:
+            raise ValueError(f"refine_start_sigma must be in (0, 1), got {self.refine_start_sigma}.")

fastvideo/configs/pipelines/pan2.py

@@ -1,46 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""PAN2 pipeline configuration: text-to-video, and image-to-video from a first frame."""
-
-from dataclasses import dataclass, field
-
-from fastvideo.configs.models import DiTConfig, EncoderConfig, VAEConfig
-from fastvideo.configs.models.dits import PAN2VideoConfig
-from fastvideo.configs.models.encoders import Qwen3_5TextConfig
-from fastvideo.configs.pipelines.base import PipelineConfig
-from fastvideo.models.wan.vae_config import WanVAEConfig
-
-PAN2_SYSTEM_PROMPT = "You are a helpful assistant. Generate the video with the following instructions."
-
-
-@dataclass
-class PAN2PipelineConfig(PipelineConfig):
-    """One configuration serves text-to-video and, given a first frame, image-to-video."""
-
-    dit_config: DiTConfig = field(default_factory=PAN2VideoConfig)
-    # The Wan2.2 48-channel VAE; its architecture and latent statistics come from vae/config.json at load time.
-    vae_config: VAEConfig = field(default_factory=WanVAEConfig)
-    text_encoder_configs: tuple[EncoderConfig, ...] = field(default_factory=lambda: (Qwen3_5TextConfig(), ))
-    text_encoder_precisions: tuple[str, ...] = field(default_factory=lambda: ("bf16", ))
-
-    dit_precision: str = "bf16"
-    vae_precision: str = "fp32"
-    # Unset: the checkpoint's scheduler config decides, which is shift 9 for IFM/PAN2-TEST, the reference sampler's
-    # fixed value. A fixed default here would override the released shift with whatever a test checkpoint declares.
-    flow_shift: float | None = None
-    # Image-to-video: the first latent frame (the conditioning image) takes the positive prediction instead of the
-    # guided one, as the reference sampler does by default (``--i2v_skip_first_latent_cfg``).
-    i2v_skip_first_latent_cfg: bool = True
-    # PAN2 samples on fp32 latents; keep the Euler update out of autocast.
-    scheduler_step_in_fp32: bool = True
-
-    # Prompt encoding: PAN2 conditions on the tokens after the chat template's user header, taken from the text
-    # encoder's hidden states with its last two layers skipped (``hidden_states[-3]``).
-    text_system_prompt: str = PAN2_SYSTEM_PROMPT
-    text_encoder_skip_layers: int = 2
-    text_max_sequence_length: int = 1536
-
-    def __post_init__(self) -> None:
-        # The encoder is needed for the image-to-video first frame.
-        self.vae_config.load_encoder = True
-        self.vae_config.load_decoder = True
-        self.text_encoder_configs[0].arch_config.output_hidden_states = True

fastvideo/entrypoints/openai/request_adapter.py

@@ -41,6 +41,10 @@ class RequestAdaptationError(ValueError):
     """The transport request cannot be represented by the loaded pipeline."""
 
 
+OPEN_H3_REGENERATE_2K_FAMILY = "open_h3_regenerate_2k"
+OPEN_H3_REGENERATE_2K_SIZE = (2560, 1440)
+OPEN_H3_REGENERATE_2K_NUM_FRAMES = 124
+
 _VIDEO_SUFFIXES = frozenset({".avi", ".m4v", ".mkv", ".mov", ".mp4", ".mpeg", ".mpg", ".webm"})
 
 
@@ -381,6 +385,13 @@ def _apply_reference_inputs(
     if request.task is not None and model_family != "minimax_h3":
         raise RequestAdaptationError("The task selector is only defined for MiniMax-H3 servers.")
 
+    if model_family == OPEN_H3_REGENERATE_2K_FAMILY:
+        if len(videos) != 1 or images or audios:
+            raise RequestAdaptationError("Open-H3-Regenerate-2K takes exactly one video_reference (the 768P draft) "
+                                         "and no image or audio references.")
+        kwargs["video_path"] = videos[0]
+        return
+
     if model_family == "minimax_h3" and (videos or audios):
         raise RequestAdaptationError("MiniMax-H3 video/audio references require a server configured with "
                                      "override_pipeline_cls_name=MiniMaxH3Ref2VAModularPipeline.")
@@ -522,6 +533,16 @@ def build_generation_request(
     if height is not None and (not isinstance(height, int) or height <= 0):
         raise RequestAdaptationError(f"height must be a positive integer, got {height!r}")
 
+    if model_family == OPEN_H3_REGENERATE_2K_FAMILY:
+        if (width, height) not in ((None, None), OPEN_H3_REGENERATE_2K_SIZE):
+            raise RequestAdaptationError("Open-H3-Regenerate-2K always renders 2560x1440; omit size/width/height "
+                                         f"or pass 2560x1440, got {width}x{height}.")
+        if kwargs.get("num_frames", OPEN_H3_REGENERATE_2K_NUM_FRAMES) != OPEN_H3_REGENERATE_2K_NUM_FRAMES:
+            raise RequestAdaptationError("Open-H3-Regenerate-2K regenerates the draft's first 121 frames (24 fps); "
+                                         "omit seconds and num_frames.")
+        if kwargs["fps"] != 24:
+            raise RequestAdaptationError(f"Open-H3-Regenerate-2K requires fps=24, got {kwargs['fps']}.")
+
     if model_family == "minimax_h3":
         from fastvideo.pipelines.basic.minimax_h3.packing import (
             MINIMAX_H3_CANVAS_MULTIPLE,

fastvideo/fastvideo_args.py

@@ -881,6 +881,13 @@ class FastVideoArgs:
             default=FastVideoArgs.taeh3_chunk_size,
             help="TAEH3 latent frames per execution chunk.",
         )
+        parser.add_argument(
+            "--h3-vae-nvfp4-decoder",
+            type=str,
+            default=FastVideoArgs.h3_vae_nvfp4_decoder,
+            help="Exported NVFP4 MiniMax-H3 decoder (decoder-nvfp4.safetensors) that replaces the VAE decoder at "
+            "load. Blackwell only. Overrides FASTVIDEO_H3_VAE_NVFP4_DECODER.",
+        )
         parser.add_argument(
             "--vae-parallel-decode",
             action=StoreBoolean,

fastvideo/layers/lora/linear.py

@@ -155,37 +155,43 @@ class BaseLayerWithLoRA(nn.Module):
         """The wrapped layer's bias, or ``None`` when it has none."""
         return getattr(self.base_layer, "bias", None)
 
-    @torch.compile()
     def forward(self, x: torch.Tensor) -> torch.Tensor:
-        lora_A = self.lora_A
-        lora_B = self.lora_B
-        if isinstance(self.lora_B, DTensor):
-            lora_B = self.lora_B.to_local()
-            lora_A = self.lora_A.to_local()
-
-        if self.side_branch and (self.lora_A is None or self._strength_is_zero):
-            # No adapter, or strength 0, is exactly the unadapted layer, down to the output dtype.
+        # The wrapper adds no compiled region around the base layer, so it runs exactly as it would unwrapped: eagerly,
+        # or inside a caller's compiled block. Compiling a packed NVFP4/FP8 linear on its own changes its numerics, and
+        # a wrapped layer with no adapter or strength 0 must equal the unwrapped layer exactly.
+        if self.lora_A is None or (self.side_branch and self._strength_is_zero):
+            # No adapter, or strength 0 on a side branch, is exactly the unadapted layer, down to the output dtype.
+            # A merge-mode wrapper reaches this branch while its first adapter is still loading or after a load
+            # failed: the pipeline wraps every linear before it reads the weights, so an unadapted request must
+            # not depend on an adapter that is not there.
             return self.base_layer(x)
 
-        if not self.merged and not self.disable_lora:
-            lora_A_sliced = self.slice_lora_a_weights(lora_A.to(x, non_blocking=True))
-            lora_B_sliced = self.slice_lora_b_weights(lora_B.to(x, non_blocking=True))
-            alpha_over_rank = 1.0
-            if self.lora_alpha != self.lora_rank:
-                alpha_over_rank = float(self.lora_alpha) / float(self.lora_rank)  # type: ignore[arg-type]
-            out, output_bias = self.base_layer(x)
-            if self.side_branch:
-                # Keep chunk temporaries opaque to Inductor so fullgraph capture cannot retain a full-size delta.
-                _add_low_rank_inplace(out, x, lora_A_sliced, lora_B_sliced, self._side_branch_strength, alpha_over_rank)
-                return out, output_bias
-            scale = float(self.lora_strength) * alpha_over_rank
-            delta = x @ lora_A_sliced.transpose(0, 1) @ lora_B_sliced.transpose(0, 1)
-            if scale != 1.0:
-                delta = delta * scale
-            return out + delta, output_bias
-        else:
-            out, output_bias = self.base_layer(x)
+        out, output_bias = self.base_layer(x)
+        if self.merged or self.disable_lora:
             return out.to(x), output_bias
+        return self._add_adapter(out, x), output_bias
+
+    @torch.compile()
+    def _add_adapter(self, out: torch.Tensor, x: torch.Tensor) -> torch.Tensor:
+        lora_A = self.lora_A
+        lora_B = self.lora_B
+        if isinstance(lora_B, DTensor):
+            lora_B = lora_B.to_local()
+            lora_A = lora_A.to_local()
+        lora_A_sliced = self.slice_lora_a_weights(lora_A.to(x, non_blocking=True))
+        lora_B_sliced = self.slice_lora_b_weights(lora_B.to(x, non_blocking=True))
+        alpha_over_rank = 1.0
+        if self.lora_alpha != self.lora_rank:
+            alpha_over_rank = float(self.lora_alpha) / float(self.lora_rank)  # type: ignore[arg-type]
+        if self.side_branch:
+            # Keep chunk temporaries opaque to Inductor so fullgraph capture cannot retain a full-size delta.
+            _add_low_rank_inplace(out, x, lora_A_sliced, lora_B_sliced, self._side_branch_strength, alpha_over_rank)
+            return out
+        scale = float(self.lora_strength) * alpha_over_rank
+        delta = x @ lora_A_sliced.transpose(0, 1) @ lora_B_sliced.transpose(0, 1)
+        if scale != 1.0:
+            delta = delta * scale
+        return out + delta
 
     def slice_lora_a_weights(self, A: torch.Tensor) -> torch.Tensor:
         return A
@@ -271,12 +277,14 @@ class BaseLayerWithLoRA(nn.Module):
             current_device = self.base_layer.weight.data.device
             data = self.base_layer.weight.data.to(get_local_torch_device())
 
-            # Apply LoRA with alpha scaling
-            lora_delta = (
-                self.slice_lora_b_weights(self.lora_B.to(data)) @ self.slice_lora_a_weights(self.lora_A.to(data)))
+            # Apply LoRA with alpha scaling. An adapter stored wider than the base (an FP32 adapter on a BF16 DiT)
+            # is multiplied and added at its own precision and rounded once, as a merge-then-cast reference does.
+            accumulate_dtype = torch.promote_types(data.dtype, self.lora_A.dtype)
+            lora_delta = (self.slice_lora_b_weights(self.lora_B.to(accumulate_dtype)) @ self.slice_lora_a_weights(
+                self.lora_A.to(accumulate_dtype)))
             scale = ((self.lora_alpha / self.lora_rank) if (self.lora_alpha and self.lora_rank) else 1.0)
             lora_delta *= scale * self.lora_strength
-            data += lora_delta
+            data = (data.to(accumulate_dtype) + lora_delta).to(data.dtype)
             self.base_layer.weight.data = data.to(current_device, non_blocking=True)
 
         self.merged = True

fastvideo/models/dits/pan2.py

@@ -1,398 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""PAN2 video transformer.
-
-A mixture-of-transformers DiT: the video and text streams keep separate weights and meet only in one joint
-attention per block, over ``[video; text]``. The text stream is modulated by a learned, timestep-free table.
-Under sequence parallelism the video tokens are sharded and the text tokens are replicated on every rank.
-"""
-
-from typing import Any
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-from fastvideo.attention import DistributedAttention, LocalAttention
-from fastvideo.configs.models.dits.pan2 import PAN2VideoConfig
-from fastvideo.distributed.communication_op import (sequence_model_parallel_all_gather_with_unpad,
-                                                    sequence_model_parallel_shard)
-from fastvideo.distributed.parallel_state import get_sp_world_size
-from fastvideo.layers.layernorm import RMSNorm
-from fastvideo.layers.linear import ReplicatedLinear
-from fastvideo.layers.mlp import MLP
-from fastvideo.layers.rotary_embedding import get_1d_rotary_pos_embed
-from fastvideo.layers.visual_embedding import PatchEmbed, TimestepEmbedder
-from fastvideo.models.dits.base import BaseDiT
-from fastvideo.platforms import AttentionBackendEnum
-
-
-class PAN2SwiGLU(nn.Module):
-    """SwiGLU MLP of the text refiner; the fused input projection stores ``[value; gate]``."""
-
-    def __init__(self, dim: int, inner_dim: int, prefix: str = ""):
-        super().__init__()
-        self.fc_in = ReplicatedLinear(dim, 2 * inner_dim, bias=False, prefix=f"{prefix}.fc_in")
-        self.fc_out = ReplicatedLinear(inner_dim, dim, bias=False, prefix=f"{prefix}.fc_out")
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        x, _ = self.fc_in(x)
-        value, gate = x.chunk(2, dim=-1)
-        x, _ = self.fc_out(value * F.silu(gate))
-        return x
-
-
-class PAN2RefinerAttention(nn.Module):
-    """Text-only self-attention of the refiner; the text is replicated on every sequence-parallel rank."""
-
-    def __init__(self,
-                 dim: int,
-                 num_heads: int,
-                 head_dim: int,
-                 eps: float,
-                 supported_attention_backends: tuple[AttentionBackendEnum, ...] | None = None,
-                 prefix: str = ""):
-        super().__init__()
-        self.num_heads = num_heads
-        self.to_qkv = ReplicatedLinear(dim, 3 * num_heads * head_dim, bias=False, prefix=f"{prefix}.to_qkv")
-        self.norm_q = RMSNorm(head_dim, eps=eps)
-        self.norm_k = RMSNorm(head_dim, eps=eps)
-        self.to_out = ReplicatedLinear(num_heads * head_dim, dim, bias=False, prefix=f"{prefix}.to_out")
-        self.attn = LocalAttention(num_heads=num_heads,
-                                   head_size=head_dim,
-                                   causal=False,
-                                   supported_attention_backends=supported_attention_backends)
-
-    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
-        qkv, _ = self.to_qkv(hidden_states)
-        query, key, value = qkv.unflatten(-1, (3, self.num_heads, -1)).unbind(2)
-        query = self.norm_q(query).to(value.dtype)
-        key = self.norm_k(key).to(value.dtype)
-        hidden_states = self.attn(query, key, value)
-        hidden_states, _ = self.to_out(hidden_states.flatten(2))
-        return hidden_states
-
-
-class PAN2TokenRefinerBlock(nn.Module):
-
-    def __init__(self,
-                 dim: int,
-                 num_heads: int,
-                 head_dim: int,
-                 mlp_ratio: float,
-                 eps: float,
-                 supported_attention_backends: tuple[AttentionBackendEnum, ...] | None = None,
-                 prefix: str = ""):
-        super().__init__()
-        self.norm1 = RMSNorm(dim, eps=eps)
-        self.attn = PAN2RefinerAttention(dim,
-                                         num_heads,
-                                         head_dim,
-                                         eps,
-                                         supported_attention_backends=supported_attention_backends,
-                                         prefix=f"{prefix}.attn")
-        self.norm2 = RMSNorm(dim, eps=eps)
-        self.ff = PAN2SwiGLU(dim, int(dim * mlp_ratio), prefix=f"{prefix}.ff")
-
-    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
-        hidden_states = hidden_states + self.attn(self.norm1(hidden_states))
-        hidden_states = hidden_states + self.ff(self.norm2(hidden_states))
-        return hidden_states
-
-
-class PAN2AdaLayerNormZero(nn.Module):
-    """Timestep modulation of the video stream: modulated attention input plus the gate/shift/scale for the rest."""
-
-    def __init__(self, dim: int, eps: float = 1e-6, prefix: str = ""):
-        super().__init__()
-        self.linear = ReplicatedLinear(dim, 6 * dim, bias=True, prefix=f"{prefix}.linear")
-        self.norm = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
-
-    def forward(self, x: torch.Tensor,
-                temb: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
-        emb, _ = self.linear(F.silu(temb))
-        shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = emb.chunk(6, dim=-1)
-        x = self.norm(x) * (1 + scale_msa[:, None]) + shift_msa[:, None]
-        return x, gate_msa, shift_mlp, scale_mlp, gate_mlp
-
-
-class PAN2AdaLayerNormOut(nn.Module):
-
-    def __init__(self, dim: int, eps: float = 1e-6, prefix: str = ""):
-        super().__init__()
-        self.linear = ReplicatedLinear(dim, 2 * dim, bias=True, prefix=f"{prefix}.linear")
-        self.norm = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
-
-    def forward(self, x: torch.Tensor, temb: torch.Tensor) -> torch.Tensor:
-        emb, _ = self.linear(F.silu(temb).to(x.dtype))
-        scale, shift = emb.chunk(2, dim=-1)
-        return self.norm(x) * (1 + scale)[:, None, :] + shift[:, None, :]
-
-
-class PAN2Attention(nn.Module):
-    """Joint attention over ``[video; text]`` with per-stream projections and Q/K RMSNorm.
-
-    RoPE applies to the video tokens only; ``DistributedAttention`` applies it after its all-to-all, where each rank
-    holds the full (unpadded) video sequence for its heads.
-    """
-
-    def __init__(self,
-                 dim: int,
-                 num_heads: int,
-                 head_dim: int,
-                 eps: float,
-                 supported_attention_backends: tuple[AttentionBackendEnum, ...] | None = None,
-                 prefix: str = ""):
-        super().__init__()
-        self.num_heads = num_heads
-        inner_dim = num_heads * head_dim
-
-        self.to_qkv = ReplicatedLinear(dim, 3 * inner_dim, bias=True, prefix=f"{prefix}.to_qkv")
-        self.norm_q = RMSNorm(head_dim, eps=eps)
-        self.norm_k = RMSNorm(head_dim, eps=eps)
-        self.to_out = ReplicatedLinear(inner_dim, dim, bias=True, prefix=f"{prefix}.to_out")
-
-        self.add_qkv_proj = ReplicatedLinear(dim, 3 * inner_dim, bias=True, prefix=f"{prefix}.add_qkv_proj")
-        self.norm_added_q = RMSNorm(head_dim, eps=eps)
-        self.norm_added_k = RMSNorm(head_dim, eps=eps)
-        self.to_add_out = ReplicatedLinear(inner_dim, dim, bias=True, prefix=f"{prefix}.to_add_out")
-
-        self.attn = DistributedAttention(num_heads=num_heads,
-                                         head_size=head_dim,
-                                         causal=False,
-                                         supported_attention_backends=supported_attention_backends,
-                                         prefix=f"{prefix}.attn")
-
-    def _project(self, projection: ReplicatedLinear,
-                 hidden_states: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
-        qkv, _ = projection(hidden_states)
-        query, key, value = qkv.unflatten(-1, (3, self.num_heads, -1)).unbind(2)
-        return query, key, value
-
-    def forward(
-        self,
-        hidden_states: torch.Tensor,
-        encoder_hidden_states: torch.Tensor,
-        freqs_cis: tuple[torch.Tensor, torch.Tensor],
-        original_seq_len: int,
-    ) -> tuple[torch.Tensor, torch.Tensor]:
-        query, key, value = self._project(self.to_qkv, hidden_states)
-        query = self.norm_q(query).to(value.dtype)
-        key = self.norm_k(key).to(value.dtype)
-
-        encoder_query, encoder_key, encoder_value = self._project(self.add_qkv_proj, encoder_hidden_states)
-        encoder_query = self.norm_added_q(encoder_query).to(encoder_value.dtype)
-        encoder_key = self.norm_added_k(encoder_key).to(encoder_value.dtype)
-
-        hidden_states, encoder_hidden_states = self.attn(query,
-                                                         key,
-                                                         value,
-                                                         original_seq_len,
-                                                         replicated_q=encoder_query,
-                                                         replicated_k=encoder_key,
-                                                         replicated_v=encoder_value,
-                                                         freqs_cis=freqs_cis)
-        hidden_states, _ = self.to_out(hidden_states.flatten(2))
-        encoder_hidden_states, _ = self.to_add_out(encoder_hidden_states.flatten(2))
-        return hidden_states, encoder_hidden_states
-
-
-class PAN2TransformerBlock(nn.Module):
-
-    def __init__(self,
-                 dim: int,
-                 num_heads: int,
-                 head_dim: int,
-                 mlp_ratio: float,
-                 context_mlp_ratio: float,
-                 eps: float = 1e-6,
-                 supported_attention_backends: tuple[AttentionBackendEnum, ...] | None = None,
-                 prefix: str = ""):
-        super().__init__()
-        self.norm1 = PAN2AdaLayerNormZero(dim, eps=eps, prefix=f"{prefix}.norm1")
-        self.norm1_context = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
-        self.scale_shift_table_context = nn.Parameter(torch.zeros(6 * dim))
-
-        self.attn = PAN2Attention(dim,
-                                  num_heads,
-                                  head_dim,
-                                  eps,
-                                  supported_attention_backends=supported_attention_backends,
-                                  prefix=f"{prefix}.attn")
-
-        self.norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
-        self.ff = MLP(dim, int(dim * mlp_ratio), act_type="gelu_pytorch_tanh", prefix=f"{prefix}.ff")
-        self.norm2_context = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
-        self.ff_context = MLP(dim,
-                              int(dim * context_mlp_ratio),
-                              act_type="gelu_pytorch_tanh",
-                              prefix=f"{prefix}.ff_context")
-
-    def forward(
-        self,
-        hidden_states: torch.Tensor,
-        encoder_hidden_states: torch.Tensor,
-        temb: torch.Tensor,
-        freqs_cis: tuple[torch.Tensor, torch.Tensor],
-        original_seq_len: int,
-    ) -> tuple[torch.Tensor, torch.Tensor]:
-        norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1(hidden_states, temb)
-        (
-            context_shift_msa,
-            context_scale_msa,
-            context_gate_msa,
-            context_shift_mlp,
-            context_scale_mlp,
-            context_gate_mlp,
-        ) = self.scale_shift_table_context.to(temb.dtype).chunk(6, dim=-1)
-        norm_encoder_hidden_states = (self.norm1_context(encoder_hidden_states) *
-                                      (1 + context_scale_msa) + context_shift_msa)
-
-        attn_output, context_attn_output = self.attn(norm_hidden_states, norm_encoder_hidden_states, freqs_cis,
-                                                     original_seq_len)
-        hidden_states = hidden_states + attn_output * gate_msa.unsqueeze(1)
-        encoder_hidden_states = encoder_hidden_states + context_attn_output * context_gate_msa
-
-        norm_hidden_states = self.norm2(hidden_states) * (1 + scale_mlp[:, None]) + shift_mlp[:, None]
-        norm_encoder_hidden_states = (self.norm2_context(encoder_hidden_states) *
-                                      (1 + context_scale_mlp) + context_shift_mlp)
-
-        hidden_states = hidden_states + self.ff(norm_hidden_states) * gate_mlp.unsqueeze(1)
-        encoder_hidden_states = encoder_hidden_states + self.ff_context(norm_encoder_hidden_states) * context_gate_mlp
-        return hidden_states, encoder_hidden_states
-
-
-class PAN2Transformer3DModel(BaseDiT):
-    """PAN2 DiT. The input stacks the noisy latents, the conditioning latents and a one-channel conditioning mask
-    along channels, so one model serves text-to-video and first-frame image-to-video."""
-
-    _fsdp_shard_conditions = PAN2VideoConfig()._fsdp_shard_conditions
-    _compile_conditions = PAN2VideoConfig()._compile_conditions
-    _supported_attention_backends = PAN2VideoConfig()._supported_attention_backends
-    param_names_mapping = PAN2VideoConfig().param_names_mapping
-    reverse_param_names_mapping = PAN2VideoConfig().reverse_param_names_mapping
-    lora_param_names_mapping = PAN2VideoConfig().lora_param_names_mapping
-
-    def __init__(self, config: PAN2VideoConfig, hf_config: dict[str, Any], **kwargs) -> None:
-        super().__init__(config=config, hf_config=hf_config)
-        inner_dim = config.num_attention_heads * config.attention_head_dim
-        self.hidden_size = inner_dim
-        self.num_attention_heads = config.num_attention_heads
-        self.num_channels_latents = config.num_channels_latents
-        self.out_channels = config.out_channels
-        self.patch_size = tuple(config.patch_size)
-        self.rope_axes_dim = tuple(config.rope_axes_dim)
-        if sum(self.rope_axes_dim) != config.attention_head_dim:
-            raise ValueError(f"rope_axes_dim {self.rope_axes_dim} must sum to the head dim {config.attention_head_dim}")
-        if config.num_attention_heads % get_sp_world_size() != 0:
-            raise ValueError(f"The number of attention heads ({config.num_attention_heads}) must be divisible by the "
-                             f"sequence parallel size ({get_sp_world_size()})")
-
-        self.x_embedder = PatchEmbed(self.patch_size,
-                                     config.in_channels,
-                                     inner_dim,
-                                     prefix=f"{config.prefix}.x_embedder")
-        self.time_embedder = TimestepEmbedder(inner_dim, act_layer="silu", frequency_embedding_size=256)
-
-        self.context_norm = RMSNorm(config.text_embed_dim, eps=1e-5)
-        self.context_proj = ReplicatedLinear(config.text_embed_dim,
-                                             inner_dim,
-                                             bias=True,
-                                             prefix=f"{config.prefix}.context_proj")
-        # Layerwise offload hooks the first nn.ModuleList child, which must be the main blocks.
-        self.transformer_blocks = nn.ModuleList([
-            PAN2TransformerBlock(inner_dim,
-                                 config.num_attention_heads,
-                                 config.attention_head_dim,
-                                 config.mlp_ratio,
-                                 config.context_mlp_ratio,
-                                 supported_attention_backends=self._supported_attention_backends,
-                                 prefix=f"{config.prefix}.transformer_blocks.{i}") for i in range(config.num_layers)
-        ])
-
-        self.context_refiner = nn.ModuleList([
-            PAN2TokenRefinerBlock(inner_dim,
-                                  config.num_attention_heads,
-                                  config.attention_head_dim,
-                                  config.refiner_mlp_ratio,
-                                  eps=1e-5,
-                                  supported_attention_backends=self._supported_attention_backends,
-                                  prefix=f"{config.prefix}.context_refiner.{i}")
-            for i in range(config.num_refiner_layers)
-        ])
-        self.context_type_embedding = nn.Parameter(torch.zeros(inner_dim))
-
-        self.norm_out = PAN2AdaLayerNormOut(inner_dim, eps=1e-6, prefix=f"{config.prefix}.norm_out")
-        self.proj_out = ReplicatedLinear(inner_dim,
-                                         self.patch_size[0] * self.patch_size[1] * self.patch_size[2] *
-                                         self.out_channels,
-                                         bias=True,
-                                         prefix=f"{config.prefix}.proj_out")
-
-        self.gradient_checkpointing = False
-        self.__post_init__()
-
-    def _rotary_pos_embed(self, num_frames: int, height: int, width: int,
-                          device: torch.device) -> tuple[torch.Tensor, torch.Tensor]:
-        """Half-dim cos/sin of the 3-axis ``(t, h, w)`` RoPE over the full (unsharded) video token grid."""
-        axes = [torch.arange(size, device=device, dtype=torch.float32) for size in (num_frames, height, width)]
-        grid = torch.stack(torch.meshgrid(*axes, indexing="ij"), dim=0)
-        grid[0] = grid[0] * self.config.rope_temporal_scale
-        cos_sin = [
-            get_1d_rotary_pos_embed(dim, grid[i].reshape(-1), theta=self.config.rope_theta, use_real=False)
-            for i, dim in enumerate(self.rope_axes_dim)
-        ]
-        freqs_cos = torch.cat([cos for cos, _ in cos_sin], dim=1).float()
-        freqs_sin = torch.cat([sin for _, sin in cos_sin], dim=1).float()
-        return freqs_cos, freqs_sin
-
-    def forward(self,
-                hidden_states: torch.Tensor,
-                encoder_hidden_states: torch.Tensor | list[torch.Tensor],
-                timestep: torch.LongTensor,
-                encoder_hidden_states_image: torch.Tensor | list[torch.Tensor] | None = None,
-                guidance=None,
-                **kwargs) -> torch.Tensor:
-        if not isinstance(encoder_hidden_states, torch.Tensor):
-            encoder_hidden_states = encoder_hidden_states[0]
-
-        batch_size, _, num_frames, height, width = hidden_states.shape
-        p_t, p_h, p_w = self.patch_size
-        post_patch_num_frames = num_frames // p_t
-        post_patch_height = height // p_h
-        post_patch_width = width // p_w
-
-        freqs_cis = self._rotary_pos_embed(post_patch_num_frames, post_patch_height, post_patch_width,
-                                           hidden_states.device)
-
-        hidden_states = self.x_embedder(hidden_states)
-        hidden_states, original_seq_len = sequence_model_parallel_shard(hidden_states, dim=1)
-
-        if timestep.dim() == 0:
-            timestep = timestep.expand(batch_size)
-        temb = self.time_embedder(timestep).to(hidden_states.dtype)
-
-        encoder_hidden_states, _ = self.context_proj(self.context_norm(encoder_hidden_states))
-        for refiner_block in self.context_refiner:
-            encoder_hidden_states = refiner_block(encoder_hidden_states)
-        encoder_hidden_states = encoder_hidden_states + self.context_type_embedding
-
-        for block in self.transformer_blocks:
-            if torch.is_grad_enabled() and self.gradient_checkpointing:
-                hidden_states, encoder_hidden_states = self._gradient_checkpointing_func(
-                    block, hidden_states, encoder_hidden_states, temb, freqs_cis, original_seq_len)
-            else:
-                hidden_states, encoder_hidden_states = block(hidden_states, encoder_hidden_states, temb, freqs_cis,
-                                                             original_seq_len)
-
-        hidden_states = self.norm_out(hidden_states, temb)
-        hidden_states = sequence_model_parallel_all_gather_with_unpad(hidden_states, original_seq_len, dim=1)
-        hidden_states, _ = self.proj_out(hidden_states)
-
-        # proj_out emits channels before the patch offsets: (C, p_t, p_h, p_w).
-        hidden_states = hidden_states.reshape(batch_size, post_patch_num_frames, post_patch_height, post_patch_width,
-                                              -1, p_t, p_h, p_w)
-        hidden_states = hidden_states.permute(0, 4, 1, 5, 2, 6, 3, 7)
-        return hidden_states.flatten(6, 7).flatten(4, 5).flatten(2, 3)
-
-
-EntryClass = PAN2Transformer3DModel

fastvideo/models/encoders/qwen3_5.py

@@ -1,574 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""Qwen3.5 text encoder: hybrid Gated DeltaNet linear attention + gated full attention.
-
-The encoder runs replicated (no tensor or sequence parallelism) and
-inference-only, so the linear-attention layers use a pure-PyTorch chunked gated
-delta rule instead of the fused ``fla`` / ``causal_conv1d`` kernels. Full
-attention calls ``scaled_dot_product_attention`` directly with head-major
-``[B, H, L, D]`` inputs: that is the layout the reference text model feeds SDPA,
-and keeping it makes bf16 hidden states reproducible against that reference.
-"""
-from collections.abc import Iterable
-from typing import Any
-
-import torch
-import torch.nn.functional as F
-from torch import nn
-
-from fastvideo.configs.models.encoders import BaseEncoderOutput
-from fastvideo.configs.models.encoders.qwen3_5 import Qwen3_5TextArchConfig, Qwen3_5TextConfig
-from fastvideo.layers.linear import ReplicatedLinear
-from fastvideo.layers.quantization import QuantizationConfig
-from fastvideo.layers.vocab_parallel_embedding import VocabParallelEmbedding
-from fastvideo.logger import init_logger
-from fastvideo.models.encoders.base import TextEncoder
-from fastvideo.models.loader.weight_utils import default_weight_loader
-
-logger = init_logger(__name__)
-
-_GATED_DELTA_CHUNK_SIZE = 64
-# Checkpoints saved from the multimodal wrapper nest the text tower.
-_CHECKPOINT_PREFIXES = ("model.language_model.", "language_model.", "model.")
-_SKIPPED_CHECKPOINT_PREFIXES = ("lm_head.", "mtp.", "visual.", "model.visual.")
-
-
-class Qwen3_5RMSNorm(nn.Module):
-    """Zero-centered RMSNorm: the stored weight is an offset from one."""
-
-    def __init__(self, hidden_size: int, eps: float = 1e-6) -> None:
-        super().__init__()
-        self.eps = eps
-        self.weight = nn.Parameter(torch.zeros(hidden_size))
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        output = x.float()
-        output = output * torch.rsqrt(output.pow(2).mean(-1, keepdim=True) + self.eps)
-        # The scale is applied in float32 before the downcast.
-        output = output * (1.0 + self.weight.float())
-        return output.type_as(x)
-
-
-class Qwen3_5RMSNormGated(nn.Module):
-    """Per-head RMSNorm followed by a SiLU output gate (plain, not zero-centered, weight)."""
-
-    def __init__(self, hidden_size: int, eps: float = 1e-6) -> None:
-        super().__init__()
-        self.eps = eps
-        self.weight = nn.Parameter(torch.ones(hidden_size))
-
-    def forward(self, hidden_states: torch.Tensor, gate: torch.Tensor) -> torch.Tensor:
-        input_dtype = hidden_states.dtype
-        hidden_states = hidden_states.to(torch.float32)
-        variance = hidden_states.pow(2).mean(-1, keepdim=True)
-        hidden_states = hidden_states * torch.rsqrt(variance + self.eps)
-        hidden_states = self.weight * hidden_states.to(input_dtype)
-        hidden_states = hidden_states * F.silu(gate.to(torch.float32))
-        return hidden_states.to(input_dtype)
-
-
-def _l2norm(x: torch.Tensor, eps: float = 1e-6) -> torch.Tensor:
-    return x * torch.rsqrt((x * x).sum(dim=-1, keepdim=True) + eps)
-
-
-def chunk_gated_delta_rule(
-    query: torch.Tensor,
-    key: torch.Tensor,
-    value: torch.Tensor,
-    g: torch.Tensor,
-    beta: torch.Tensor,
-    chunk_size: int = _GATED_DELTA_CHUNK_SIZE,
-) -> torch.Tensor:
-    """Chunked gated delta rule over a full prefix with a zero initial state.
-
-    Inputs are ``[batch, seq, heads, dim]`` (``g`` / ``beta`` are
-    ``[batch, seq, heads]``); q/k must already be L2-normalized. The recurrence
-    runs in float32 and the output is returned in the query dtype.
-    """
-    output_dtype = query.dtype
-    query, key, value, beta, g = (x.transpose(1, 2).contiguous().to(torch.float32)
-                                  for x in (query, key, value, beta, g))
-
-    batch_size, num_heads, sequence_length, k_head_dim = key.shape
-    v_head_dim = value.shape[-1]
-    pad_size = (chunk_size - sequence_length % chunk_size) % chunk_size
-    query = F.pad(query, (0, 0, 0, pad_size))
-    key = F.pad(key, (0, 0, 0, pad_size))
-    value = F.pad(value, (0, 0, 0, pad_size))
-    beta = F.pad(beta, (0, pad_size))
-    g = F.pad(g, (0, pad_size))
-    total_sequence_length = sequence_length + pad_size
-    query = query * (1 / (query.shape[-1]**0.5))
-
-    v_beta = value * beta.unsqueeze(-1)
-    k_beta = key * beta.unsqueeze(-1)
-    query, key, value, k_beta, v_beta = (x.reshape(x.shape[0], x.shape[1], -1, chunk_size, x.shape[-1])
-                                         for x in (query, key, value, k_beta, v_beta))
-    g = g.reshape(g.shape[0], g.shape[1], -1, chunk_size)
-    upper_with_diagonal = torch.triu(torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=query.device),
-                                     diagonal=0)
-
-    # Intra-chunk decay and the (I + strictly-lower)^-1 WY transform, solved by
-    # forward substitution row by row.
-    g = g.cumsum(dim=-1)
-    decay_mask = ((g.unsqueeze(-1) - g.unsqueeze(-2)).tril().exp().float()).tril()
-    attn = -((k_beta @ key.transpose(-1, -2)) * decay_mask).masked_fill(upper_with_diagonal, 0)
-    for i in range(1, chunk_size):
-        row = attn[..., i, :i].clone()
-        sub = attn[..., :i, :i].clone()
-        attn[..., i, :i] = row + (row.unsqueeze(-1) * sub).sum(-2)
-    attn = attn + torch.eye(chunk_size, dtype=attn.dtype, device=attn.device)
-    value = attn @ v_beta
-    k_cumdecay = attn @ (k_beta * g.exp().unsqueeze(-1))
-
-    recurrent_state = torch.zeros(batch_size,
-                                  num_heads,
-                                  k_head_dim,
-                                  v_head_dim,
-                                  dtype=value.dtype,
-                                  device=value.device)
-    core_attn_out = torch.zeros_like(value)
-    for i in range(total_sequence_length // chunk_size):
-        q_i, k_i, v_i = query[:, :, i], key[:, :, i], value[:, :, i]
-        # decay_mask is already zero above the diagonal, which keeps this causal.
-        attn = q_i @ k_i.transpose(-1, -2) * decay_mask[:, :, i]
-        v_prime = k_cumdecay[:, :, i] @ recurrent_state
-        v_new = v_i - v_prime
-        attn_inter = (q_i * g[:, :, i, :, None].exp()) @ recurrent_state
-        core_attn_out[:, :, i] = attn_inter + attn @ v_new
-        recurrent_state = (recurrent_state * g[:, :, i, -1, None, None].exp() +
-                           (k_i * (g[:, :, i, -1, None] - g[:, :, i]).exp()[..., None]).transpose(-1, -2) @ v_new)
-
-    core_attn_out = core_attn_out.reshape(core_attn_out.shape[0], core_attn_out.shape[1], -1, core_attn_out.shape[-1])
-    core_attn_out = core_attn_out[:, :, :sequence_length]
-    return core_attn_out.transpose(1, 2).contiguous().to(output_dtype)
-
-
-class Qwen3_5GatedDeltaNet(nn.Module):
-    """Gated DeltaNet token mixer used by the ``linear_attention`` layers."""
-
-    def __init__(
-        self,
-        config: Qwen3_5TextArchConfig,
-        quant_config: QuantizationConfig | None = None,
-        prefix: str = "",
-    ) -> None:
-        super().__init__()
-        self.hidden_size = config.hidden_size
-        self.num_v_heads = config.linear_num_value_heads
-        self.num_k_heads = config.linear_num_key_heads
-        if self.num_v_heads % self.num_k_heads:
-            raise ValueError(f"Qwen3.5 linear value heads ({self.num_v_heads}) must be a multiple of key heads "
-                             f"({self.num_k_heads})")
-        self.head_k_dim = config.linear_key_head_dim
-        self.head_v_dim = config.linear_value_head_dim
-        self.key_dim = self.head_k_dim * self.num_k_heads
-        self.value_dim = self.head_v_dim * self.num_v_heads
-        self.conv_kernel_size = config.linear_conv_kernel_dim
-        if config.hidden_act != "silu":
-            raise ValueError(f"Unsupported Qwen3.5 activation: {config.hidden_act}")
-
-        self.conv_dim = self.key_dim * 2 + self.value_dim
-        self.conv1d = nn.Conv1d(
-            in_channels=self.conv_dim,
-            out_channels=self.conv_dim,
-            bias=False,
-            kernel_size=self.conv_kernel_size,
-            groups=self.conv_dim,
-            padding=self.conv_kernel_size - 1,
-        )
-        self.dt_bias = nn.Parameter(torch.ones(self.num_v_heads))
-        self.A_log = nn.Parameter(torch.zeros(self.num_v_heads))
-        self.norm = Qwen3_5RMSNormGated(self.head_v_dim, eps=config.rms_norm_eps)
-        self.out_proj = ReplicatedLinear(self.value_dim,
-                                         self.hidden_size,
-                                         bias=False,
-                                         quant_config=quant_config,
-                                         prefix=f"{prefix}.out_proj")
-        self.in_proj_qkv = ReplicatedLinear(self.hidden_size,
-                                            self.conv_dim,
-                                            bias=False,
-                                            quant_config=quant_config,
-                                            prefix=f"{prefix}.in_proj_qkv")
-        self.in_proj_z = ReplicatedLinear(self.hidden_size,
-                                          self.value_dim,
-                                          bias=False,
-                                          quant_config=quant_config,
-                                          prefix=f"{prefix}.in_proj_z")
-        self.in_proj_b = ReplicatedLinear(self.hidden_size,
-                                          self.num_v_heads,
-                                          bias=False,
-                                          quant_config=quant_config,
-                                          prefix=f"{prefix}.in_proj_b")
-        self.in_proj_a = ReplicatedLinear(self.hidden_size,
-                                          self.num_v_heads,
-                                          bias=False,
-                                          quant_config=quant_config,
-                                          prefix=f"{prefix}.in_proj_a")
-
-    def forward(self, hidden_states: torch.Tensor, padding_mask: torch.Tensor | None = None) -> torch.Tensor:
-        if padding_mask is not None:
-            # Zero padded tokens so they cannot leak into the recurrent state.
-            hidden_states = (hidden_states * padding_mask[:, :, None]).to(hidden_states.dtype)
-        batch_size, seq_len, _ = hidden_states.shape
-
-        mixed_qkv, _ = self.in_proj_qkv(hidden_states)
-        z, _ = self.in_proj_z(hidden_states)
-        b, _ = self.in_proj_b(hidden_states)
-        a, _ = self.in_proj_a(hidden_states)
-
-        # Causal depthwise conv: left padding of kernel-1, keep the first seq_len outputs.
-        mixed_qkv = mixed_qkv.transpose(1, 2)
-        mixed_qkv = F.silu(self.conv1d(mixed_qkv.to(self.conv1d.weight.dtype))[:, :, :seq_len]).to(mixed_qkv.dtype)
-        mixed_qkv = mixed_qkv.transpose(1, 2)
-        query, key, value = torch.split(mixed_qkv, [self.key_dim, self.key_dim, self.value_dim], dim=-1)
-        query = query.reshape(batch_size, seq_len, -1, self.head_k_dim)
-        key = key.reshape(batch_size, seq_len, -1, self.head_k_dim)
-        value = value.reshape(batch_size, seq_len, -1, self.head_v_dim)
-
-        beta = b.sigmoid()
-        # float32 keeps exp(A_log) finite for half-precision weights.
-        g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias)
-        if self.num_v_heads > self.num_k_heads:
-            repeats = self.num_v_heads // self.num_k_heads
-            query = query.repeat_interleave(repeats, dim=2)
-            key = key.repeat_interleave(repeats, dim=2)
-
-        core_attn_out = chunk_gated_delta_rule(_l2norm(query), _l2norm(key), value, g=g, beta=beta)
-
-        core_attn_out = self.norm(core_attn_out.reshape(-1, self.head_v_dim), z.reshape(-1, self.head_v_dim))
-        core_attn_out = core_attn_out.reshape(batch_size, seq_len, -1)
-        output, _ = self.out_proj(core_attn_out)
-        return output
-
-
-class Qwen3_5RotaryEmbedding(nn.Module):
-    """Text-only partial RoPE.
-
-    Qwen3.5's interleaved M-RoPE collapses to standard RoPE when the temporal,
-    height and width positions are identical, which holds for text tokens.
-    """
-
-    def __init__(self, config: Qwen3_5TextArchConfig) -> None:
-        super().__init__()
-        rope = config.resolved_rope_parameters()
-        if rope["rope_type"] != "default":
-            raise NotImplementedError(f"Qwen3.5 rope_type {rope['rope_type']!r} is not supported")
-        rotary_dim = int(config.head_dim * rope["partial_rotary_factor"])
-        inv_freq = 1.0 / (rope["rope_theta"]**(
-            torch.arange(0, rotary_dim, 2, dtype=torch.float32, device="cpu") / rotary_dim))
-        self.register_buffer("inv_freq", inv_freq, persistent=False)
-
-    @torch.no_grad()
-    def forward(self, hidden_states: torch.Tensor, position_ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
-        inv_freq = self.inv_freq.to(device=hidden_states.device, dtype=torch.float32)
-        device_type = hidden_states.device.type if hidden_states.device.type != "mps" else "cpu"
-        with torch.autocast(device_type=device_type, enabled=False):
-            inv_freq = inv_freq[None, :, None].expand(position_ids.shape[0], -1, 1)
-            freqs = (inv_freq @ position_ids[:, None, :].float()).transpose(1, 2)
-            emb = torch.cat((freqs, freqs), dim=-1)
-            cos = emb.cos()
-            sin = emb.sin()
-        return cos.to(hidden_states.dtype), sin.to(hidden_states.dtype)
-
-
-def _rotate_half(x: torch.Tensor) -> torch.Tensor:
-    x1, x2 = x.chunk(2, dim=-1)
-    return torch.cat((-x2, x1), dim=-1)
-
-
-def _repeat_kv(x: torch.Tensor, groups: int) -> torch.Tensor:
-    batch_size, num_kv_heads, seq_len, head_dim = x.shape
-    x = x[:, :, None].expand(batch_size, num_kv_heads, groups, seq_len, head_dim)
-    return x.reshape(batch_size, num_kv_heads * groups, seq_len, head_dim)
-
-
-def _apply_partial_rotary(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
-    """Rotate the leading ``cos.shape[-1]`` channels of ``[B, H, L, D]`` and pass the rest through."""
-    rotary_dim = cos.shape[-1]
-    cos = cos.unsqueeze(1)
-    sin = sin.unsqueeze(1)
-    x_rot, x_pass = x[..., :rotary_dim], x[..., rotary_dim:]
-    x_rot = x_rot * cos + _rotate_half(x_rot) * sin
-    return torch.cat([x_rot, x_pass], dim=-1)
-
-
-class Qwen3_5Attention(nn.Module):
-    """Gated full attention with per-head QK-norm, partial RoPE and GQA."""
-
-    def __init__(
-        self,
-        config: Qwen3_5TextConfig,
-        quant_config: QuantizationConfig | None = None,
-        prefix: str = "",
-    ) -> None:
-        super().__init__()
-        arch = config.arch_config
-        self.num_heads = arch.num_attention_heads
-        self.num_kv_heads = arch.num_key_value_heads
-        if self.num_heads % self.num_kv_heads:
-            raise ValueError(f"Qwen3.5 attention heads ({self.num_heads}) must be a multiple of KV heads "
-                             f"({self.num_kv_heads})")
-        self.head_dim = arch.head_dim
-        self.scaling = self.head_dim**-0.5
-        # q_proj packs, per head, the query followed by its output gate.
-        self.q_proj = ReplicatedLinear(arch.hidden_size,
-                                       self.num_heads * self.head_dim * 2,
-                                       bias=arch.attention_bias,
-                                       quant_config=quant_config,
-                                       prefix=f"{prefix}.q_proj")
-        self.k_proj = ReplicatedLinear(arch.hidden_size,
-                                       self.num_kv_heads * self.head_dim,
-                                       bias=arch.attention_bias,
-                                       quant_config=quant_config,
-                                       prefix=f"{prefix}.k_proj")
-        self.v_proj = ReplicatedLinear(arch.hidden_size,
-                                       self.num_kv_heads * self.head_dim,
-                                       bias=arch.attention_bias,
-                                       quant_config=quant_config,
-                                       prefix=f"{prefix}.v_proj")
-        self.o_proj = ReplicatedLinear(self.num_heads * self.head_dim,
-                                       arch.hidden_size,
-                                       bias=arch.attention_bias,
-                                       quant_config=quant_config,
-                                       prefix=f"{prefix}.o_proj")
-        self.q_norm = Qwen3_5RMSNorm(self.head_dim, eps=arch.rms_norm_eps)
-        self.k_norm = Qwen3_5RMSNorm(self.head_dim, eps=arch.rms_norm_eps)
-
-    def forward(
-        self,
-        hidden_states: torch.Tensor,
-        position_embeddings: tuple[torch.Tensor, torch.Tensor],
-        padding_mask: torch.Tensor | None = None,
-    ) -> torch.Tensor:
-        batch_size, seq_len, _ = hidden_states.shape
-        packed, _ = self.q_proj(hidden_states)
-        query, gate = torch.chunk(packed.view(batch_size, seq_len, -1, self.head_dim * 2), 2, dim=-1)
-        gate = gate.reshape(batch_size, seq_len, -1)
-        key, _ = self.k_proj(hidden_states)
-        value, _ = self.v_proj(hidden_states)
-
-        query = self.q_norm(query).transpose(1, 2)
-        key = self.k_norm(key.view(batch_size, seq_len, -1, self.head_dim)).transpose(1, 2)
-        value = value.view(batch_size, seq_len, -1, self.head_dim).transpose(1, 2)
-        cos, sin = position_embeddings
-        query = _apply_partial_rotary(query, cos, sin)
-        key = _apply_partial_rotary(key, cos, sin)
-
-        groups = self.num_heads // self.num_kv_heads
-        # Native GQA is only used where SDPA keeps a fused kernel (no mask, head_dim <= 256).
-        native_gqa = padding_mask is None and self.head_dim <= 256
-        if groups > 1 and not native_gqa:
-            key = _repeat_kv(key, groups)
-            value = _repeat_kv(value, groups)
-        if padding_mask is None:
-            attn_output = F.scaled_dot_product_attention(
-                query,
-                key,
-                value,
-                dropout_p=0.0,
-                is_causal=seq_len > 1,
-                scale=self.scaling,
-                enable_gqa=groups > 1 and native_gqa,
-            )
-        else:
-            causal = torch.ones(seq_len, seq_len, device=query.device, dtype=torch.bool).tril()
-            mask = causal[None, None] & padding_mask.to(device=query.device, dtype=torch.bool)[:, None, None, :]
-            attn_output = F.scaled_dot_product_attention(
-                query,
-                key,
-                value,
-                attn_mask=mask,
-                dropout_p=0.0,
-                is_causal=False,
-                scale=self.scaling,
-            )
-
-        attn_output = attn_output.transpose(1, 2).reshape(batch_size, seq_len, -1)
-        attn_output = attn_output * torch.sigmoid(gate)
-        output, _ = self.o_proj(attn_output)
-        return output
-
-
-class Qwen3_5MLP(nn.Module):
-
-    def __init__(
-        self,
-        config: Qwen3_5TextArchConfig,
-        quant_config: QuantizationConfig | None = None,
-        prefix: str = "",
-    ) -> None:
-        super().__init__()
-        if config.hidden_act != "silu":
-            raise ValueError(f"Unsupported Qwen3.5 activation: {config.hidden_act}")
-        self.gate_proj = ReplicatedLinear(config.hidden_size,
-                                          config.intermediate_size,
-                                          bias=False,
-                                          quant_config=quant_config,
-                                          prefix=f"{prefix}.gate_proj")
-        self.up_proj = ReplicatedLinear(config.hidden_size,
-                                        config.intermediate_size,
-                                        bias=False,
-                                        quant_config=quant_config,
-                                        prefix=f"{prefix}.up_proj")
-        self.down_proj = ReplicatedLinear(config.intermediate_size,
-                                          config.hidden_size,
-                                          bias=False,
-                                          quant_config=quant_config,
-                                          prefix=f"{prefix}.down_proj")
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        gate, _ = self.gate_proj(x)
-        up, _ = self.up_proj(x)
-        output, _ = self.down_proj(F.silu(gate) * up)
-        return output
-
-
-class Qwen3_5DecoderLayer(nn.Module):
-
-    def __init__(
-        self,
-        config: Qwen3_5TextConfig,
-        layer_type: str,
-        quant_config: QuantizationConfig | None = None,
-        prefix: str = "",
-    ) -> None:
-        super().__init__()
-        arch = config.arch_config
-        self.layer_type = layer_type
-        if layer_type == "linear_attention":
-            self.linear_attn = Qwen3_5GatedDeltaNet(arch, quant_config=quant_config, prefix=f"{prefix}.linear_attn")
-        else:
-            self.self_attn = Qwen3_5Attention(config, quant_config=quant_config, prefix=f"{prefix}.self_attn")
-        self.mlp = Qwen3_5MLP(arch, quant_config=quant_config, prefix=f"{prefix}.mlp")
-        self.input_layernorm = Qwen3_5RMSNorm(arch.hidden_size, eps=arch.rms_norm_eps)
-        self.post_attention_layernorm = Qwen3_5RMSNorm(arch.hidden_size, eps=arch.rms_norm_eps)
-
-    def forward(
-        self,
-        hidden_states: torch.Tensor,
-        position_embeddings: tuple[torch.Tensor, torch.Tensor],
-        padding_mask: torch.Tensor | None = None,
-    ) -> torch.Tensor:
-        residual = hidden_states
-        hidden_states = self.input_layernorm(hidden_states)
-        if self.layer_type == "linear_attention":
-            hidden_states = self.linear_attn(hidden_states, padding_mask)
-        else:
-            hidden_states = self.self_attn(hidden_states, position_embeddings, padding_mask)
-        hidden_states = residual + hidden_states
-
-        residual = hidden_states
-        hidden_states = self.post_attention_layernorm(hidden_states)
-        hidden_states = self.mlp(hidden_states)
-        return residual + hidden_states
-
-
-class Qwen3_5TextModel(TextEncoder):
-    """Qwen3.5 text tower used as a diffusion text encoder (e.g. PAN2).
-
-    ``hidden_states`` follows the Transformers contract: entry 0 is the token
-    embeddings, entry ``i`` is the output of layer ``i - 1``, and the last entry
-    is the final-normed output (it replaces the raw last-layer output).
-    """
-
-    def __init__(self, config: Qwen3_5TextConfig) -> None:
-        super().__init__(config)
-        self.config = config
-        arch = config.arch_config
-        self.quant_config = getattr(config, "quant_config", None)
-        self.layer_types = arch.resolved_layer_types()
-
-        self.embed_tokens = VocabParallelEmbedding(
-            arch.vocab_size,
-            arch.hidden_size,
-            org_num_embeddings=arch.vocab_size,
-            quant_config=self.quant_config,
-        )
-        self.layers = nn.ModuleList([
-            Qwen3_5DecoderLayer(
-                config,
-                layer_type,
-                quant_config=self.quant_config,
-                prefix=f"{config.prefix}.layers.{index}",
-            ) for index, layer_type in enumerate(self.layer_types)
-        ])
-        self.norm = Qwen3_5RMSNorm(arch.hidden_size, eps=arch.rms_norm_eps)
-        self.rotary_emb = Qwen3_5RotaryEmbedding(arch)
-
-    def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
-        return self.embed_tokens(input_ids)
-
-    def forward(
-        self,
-        input_ids: torch.Tensor | None = None,
-        position_ids: torch.Tensor | None = None,
-        attention_mask: torch.Tensor | None = None,
-        inputs_embeds: torch.Tensor | None = None,
-        output_hidden_states: bool | None = None,
-        **kwargs: Any,
-    ) -> BaseEncoderOutput:
-        output_hidden_states = (output_hidden_states
-                                if output_hidden_states is not None else self.config.arch_config.output_hidden_states)
-        if (input_ids is None) == (inputs_embeds is None):
-            raise ValueError("Specify exactly one of input_ids or inputs_embeds")
-        if inputs_embeds is None:
-            assert input_ids is not None
-            if input_ids.ndim == 1:
-                input_ids = input_ids.unsqueeze(0)
-            inputs_embeds = self.get_input_embeddings(input_ids)
-        hidden_states = inputs_embeds
-        batch_size, seq_len = hidden_states.shape[:2]
-
-        if position_ids is None:
-            position_ids = torch.arange(seq_len, device=hidden_states.device).unsqueeze(0).expand(batch_size, -1)
-        elif position_ids.ndim == 1:
-            position_ids = position_ids.unsqueeze(0).expand(batch_size, -1)
-        elif position_ids.ndim != 2:
-            raise ValueError("Qwen3.5 text encoder expects text position_ids shaped [batch, seq]")
-
-        # An all-ones mask is equivalent to no mask; only real padding takes the masked path.
-        padding_mask = None
-        if attention_mask is not None:
-            if attention_mask.ndim != 2:
-                raise ValueError("Qwen3.5 text encoder expects a 2D [batch, seq] padding mask")
-            if not bool(attention_mask.to(torch.bool).all()):
-                padding_mask = attention_mask.to(hidden_states.device)
-
-        position_embeddings = self.rotary_emb(hidden_states, position_ids)
-        all_hidden_states: tuple[torch.Tensor, ...] | None = () if output_hidden_states else None
-        for layer in self.layers:
-            if all_hidden_states is not None:
-                all_hidden_states += (hidden_states, )
-            hidden_states = layer(hidden_states, position_embeddings, padding_mask)
-
-        hidden_states = self.norm(hidden_states)
-        if all_hidden_states is not None:
-            all_hidden_states += (hidden_states, )
-
-        return BaseEncoderOutput(
-            last_hidden_state=hidden_states,
-            hidden_states=all_hidden_states,
-            attention_mask=attention_mask,
-        )
-
-    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
-        params_dict = dict(self.named_parameters())
-        loaded_params: set[str] = set()
-        for name, loaded_weight in weights:
-            if name.startswith(_SKIPPED_CHECKPOINT_PREFIXES) or "rotary_emb.inv_freq" in name:
-                continue
-            for checkpoint_prefix in _CHECKPOINT_PREFIXES:
-                if name.startswith(checkpoint_prefix):
-                    name = name[len(checkpoint_prefix):]
-                    break
-            if name not in params_dict:
-                logger.debug("Skipping unexpected Qwen3.5 checkpoint tensor %s", name)
-                continue
-            param = params_dict[name]
-            weight_loader = getattr(param, "weight_loader", default_weight_loader)
-            weight_loader(param, loaded_weight)
-            loaded_params.add(name)
-        return loaded_params
-
-
-EntryClass = Qwen3_5TextModel

fastvideo/models/loader/component_loader.py

@@ -1655,6 +1655,8 @@ class PipelineComponentLoader:
             component_model_path,
         )
 
+        # A component under a subfolder (e.g. "base_model/vae") dispatches on its own name.
+        module_name = module_name.rsplit("/", 1)[-1]
         # Get the appropriate loader for this module type
         loader = ComponentLoader.for_module_type(module_name, transformers_or_diffusers)
 

fastvideo/models/registry.py

@@ -44,7 +44,6 @@ _TEXT_TO_VIDEO_DIT_MODELS = {
     "Kandinsky5Transformer3DModel": ("dits", "kandinsky5", "Kandinsky5Transformer3DModel"),
     "Kandinsky6Transformer3DModel": ("dits", "kandinsky6", "Kandinsky6Transformer3DModel"),
     "Flux2Transformer2DModel": ("dits", "flux_2", "Flux2Transformer2DModel"),
-    "PAN2Transformer3DModel": ("dits", "pan2", "PAN2Transformer3DModel"),
 }
 
 _IMAGE_TO_VIDEO_DIT_MODELS = {
@@ -92,7 +91,6 @@ _TEXT_ENCODER_MODELS = {
     "LTX2GemmaTextEncoderModel": ("encoders", "gemma", "LTX2GemmaTextEncoderModel"),
     "Qwen3ForCausalLM": ("encoders", "qwen3", "Qwen3ForCausalLM"),
     "Mistral3ForConditionalGeneration": ("encoders", "mistral3", "Mistral3ForConditionalGeneration"),
-    "Qwen3_5TextModel": ("encoders", "qwen3_5", "Qwen3_5TextModel"),
 }
 
 _IMAGE_ENCODER_MODELS: dict[str, tuple] = {

fastvideo/models/upsamplers/minimax_h3_latent_upscaler.py

@@ -0,0 +1,161 @@
+# SPDX-License-Identifier: Apache-2.0
+"""MiniMax-H3 3D-convolution latent upscaler (LBH-123-AI/Comfyui_Minimax_h3_latent_Upscaler, MIT).
+
+Port of the upstream node's ``LatentResizer3D`` and its ``target dimensions`` size logic, without the ComfyUI shims,
+attention blocks (the node forces them off) and temporal chunking (Open-H3-Regenerate-2K runs without it). Module names
+match the upstream ``latent_upsampler/*.safetensors`` checkpoint, so it loads without remapping.
+
+The network works on a latent that was normalized with the statistics below, not the VAE-normalized latent the DiT
+sees: ``upscale_latent`` therefore re-normalizes its input and de-normalizes the result with the same statistics.
+"""
+from __future__ import annotations
+
+import re
+
+import torch
+import torch.nn.functional as F
+from safetensors.torch import load_file
+from torch import nn
+
+VAE_DOWNSAMPLE = 16
+PIXEL_ALIGNMENT = 32
+
+LATENTS_MEAN = (
+    0.858090341091156, -0.9606591463088989, 1.0661640167236328, -0.5090325474739075, -0.2727581858634949,
+    -1.3675414323806763, -0.2553254961967468, -0.26907554268836975, -0.5376840829849243, -0.0464097298681736,
+    0.6657370328903198, 0.19690127670764923, -0.5460608005523682, -0.4035342037677765, -0.23683024942874908,
+    0.25928452610969543, -0.30133944749832153, 0.211341992020607, -1.1206848621368408, 0.3581933379173279,
+    -0.04225143790245056, 0.2604829967021942, 0.22864092886447906, 0.7056031823158264)
+LATENTS_STD = (
+    1.2223774194717407, 1.2767263650894165, 1.6831774711608887, 1.7549455165863037, 1.5636216402053833,
+    2.194143533706665, 0.9653137922286987, 1.0569885969161987, 0.841948926448822, 0.7729952931404114,
+    1.8955937623977661, 0.946841835975647, 0.7996809482574463, 0.44988900423049927, 0.7197399735450745,
+    0.6936293244361877, 2.961095094680786, 2.7694199085235596, 3.0496184825897217, 2.1088054180145264,
+    3.276226282119751, 3.1627357006073, 2.2816812992095947, 2.6127843856811523)
+
+
+def _norm(channels: int) -> nn.GroupNorm:
+    return nn.GroupNorm(32, channels)
+
+
+class ResBlockEmb3D(nn.Module):
+
+    def __init__(self, channels: int, emb_channels: int) -> None:
+        super().__init__()
+        self.in_layers = nn.Sequential(_norm(channels), nn.SiLU(), nn.Conv3d(channels, channels, 3, padding=1))
+        self.emb_layers = nn.Sequential(nn.SiLU(), nn.Linear(emb_channels, 2 * channels))
+        self.out_norm = _norm(channels)
+        self.out_layers = nn.Sequential(nn.SiLU(), nn.Dropout(p=0.0), nn.Conv3d(channels, channels, 3, padding=1))
+
+    def forward(self, x: torch.Tensor, emb: torch.Tensor) -> torch.Tensor:
+        h = self.in_layers(x)
+        scale, shift = self.emb_layers(emb).type(h.dtype)[..., None, None, None].chunk(2, dim=1)
+        h = self.out_norm(h) * (1 + scale) + shift
+        return x + self.out_layers(h)
+
+
+class TemporalConv(nn.Module):
+
+    def __init__(self, channels: int, kernel_size: int = 5) -> None:
+        super().__init__()
+        self.norm = _norm(channels)
+        self.dwconv = nn.Conv3d(channels, channels, (kernel_size, 1, 1), padding=(kernel_size // 2, 0, 0),
+                                groups=channels)
+        self.pwconv = nn.Conv3d(channels, channels, 1)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return x + self.pwconv(self.dwconv(F.silu(self.norm(x))))
+
+
+def _blocks(count: int, channels: int, emb_dim: int, temporal_every: int, kernel: int) -> nn.ModuleList:
+    blocks = nn.ModuleList()
+    for index in range(count):
+        blocks.append(ResBlockEmb3D(channels, emb_dim))
+        if temporal_every > 0 and index % temporal_every == 0:
+            blocks.append(TemporalConv(channels, kernel))
+    return blocks
+
+
+class MiniMaxH3LatentUpscaler(nn.Module):
+
+    def __init__(self,
+                 in_channels: int = 24,
+                 in_blocks: int = 12,
+                 out_blocks: int = 12,
+                 channels: int = 512,
+                 temporal_every: int = 2,
+                 temporal_kernel: int = 5) -> None:
+        super().__init__()
+        emb_dim = 64
+        self.conv_in = nn.Conv3d(in_channels, channels, 3, padding=1)
+        self.embed = nn.Sequential(nn.Linear(1, emb_dim), nn.SiLU(), nn.Linear(emb_dim, emb_dim))
+        self.in_blocks = _blocks(in_blocks, channels, emb_dim, temporal_every, temporal_kernel)
+        self.out_blocks = _blocks(out_blocks, channels, emb_dim, temporal_every, temporal_kernel)
+        self.norm_out = _norm(channels)
+        self.conv_out = nn.Conv3d(channels, in_channels, 3, padding=1)
+
+    def _run(self, blocks: nn.ModuleList, x: torch.Tensor, emb: torch.Tensor) -> torch.Tensor:
+        for block in blocks:
+            x = block(x, emb.expand(x.shape[0], -1)) if isinstance(block, ResBlockEmb3D) else block(x)
+        return x
+
+    def forward(self, x: torch.Tensor, scale: float, target_size: tuple[int, int, int]) -> torch.Tensor:
+        """``x``: [B, C, T, h, w] normalized latent; returns [B, C, T, *target_size[1:]]."""
+        emb = self.embed(torch.tensor([[scale - 1.0]], dtype=x.dtype, device=x.device))
+        x = self._run(self.in_blocks, self.conv_in(x), emb)
+        x = F.interpolate(x, size=target_size, mode="trilinear", align_corners=False)
+        x = self._run(self.out_blocks, x, emb)
+        return self.conv_out(F.silu(self.norm_out(x)))
+
+    @classmethod
+    def from_checkpoint(cls, path: str) -> MiniMaxH3LatentUpscaler:
+        state = load_file(path, device="cpu")
+        model = cls(in_channels=state["conv_in.weight"].shape[1],
+                    channels=state["conv_in.weight"].shape[0],
+                    in_blocks=_count_blocks(state, "in_blocks"),
+                    out_blocks=_count_blocks(state, "out_blocks"),
+                    temporal_kernel=_temporal_kernel(state))
+        model.load_state_dict(state, strict=True)
+        return model.eval().requires_grad_(False)
+
+
+def _count_blocks(state: dict[str, torch.Tensor], prefix: str) -> int:
+    pattern = re.compile(rf"{prefix}\.(\d+)\.in_layers\.0\.weight")
+    return len({int(m.group(1)) for key in state if (m := pattern.match(key))})
+
+
+def _temporal_kernel(state: dict[str, torch.Tensor]) -> int:
+    return next(tensor.shape[2] for key, tensor in state.items() if key.endswith("dwconv.weight"))
+
+
+def resolve_upscale_geometry(input_size: tuple[int, int], target_pixels: tuple[int, int]) -> tuple[tuple[int, int], float]:
+    """Latent (h, w) after aligning the target pixel size to 32, and the scale the network is conditioned on.
+
+    ``input_size`` is the latent (h, w); ``target_pixels`` the requested (height, width).
+    """
+    h_in, w_in = input_size
+    h_target, w_target = target_pixels
+    scale = (w_target / (w_in * VAE_DOWNSAMPLE) + h_target / (h_in * VAE_DOWNSAMPLE)) / 2.0
+    aligned = (round(round(h_target / PIXEL_ALIGNMENT) * PIXEL_ALIGNMENT / VAE_DOWNSAMPLE),
+               round(round(w_target / PIXEL_ALIGNMENT) * PIXEL_ALIGNMENT / VAE_DOWNSAMPLE))
+    return (max(1, aligned[0]), max(1, aligned[1])), scale
+
+
+@torch.inference_mode()
+def upscale_latent(model: MiniMaxH3LatentUpscaler, latent: torch.Tensor, target_pixels: tuple[int, int],
+                   dtype: torch.dtype = torch.bfloat16) -> torch.Tensor:
+    """Upscale a [C, T, h, w] latent to ``target_pixels`` (height, width); FP32 [C, T, H, W] out.
+
+    The input is returned unchanged when it already has the target latent size.
+    """
+    channels, frames, h_in, w_in = latent.shape
+    (h_out, w_out), scale = resolve_upscale_geometry((h_in, w_in), target_pixels)
+    if (h_out, w_out) == (h_in, w_in):
+        return latent.float()
+    if scale < 1.0:
+        raise ValueError("The MiniMax-H3 latent upscaler only upscales (effective scale >= 1.0).")
+    mean = torch.tensor(LATENTS_MEAN, dtype=dtype, device=latent.device).view(1, -1, 1, 1, 1)
+    std = torch.tensor(LATENTS_STD, dtype=dtype, device=latent.device).view(1, -1, 1, 1, 1)
+    source = (latent[None].to(dtype) - mean) / std
+    result = model(source, scale, (frames, h_out, w_out))
+    return (result * std + mean).float()[0]

fastvideo/pipelines/basic/open_h3_regenerate_2k/__init__.py

@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from fastvideo.pipelines.basic.open_h3_regenerate_2k.open_h3_regenerate_2k_pipeline import (
+    EntryClass,
+    OpenH3Regenerate2KPipeline,
+)
+
+__all__ = ["EntryClass", "OpenH3Regenerate2KPipeline"]

fastvideo/pipelines/basic/open_h3_regenerate_2k/draft_video.py

@@ -0,0 +1,54 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Decode a draft video into the 124-frame pixel block the H3 VAE turns into 37 latent frames."""
+
+from __future__ import annotations
+
+import subprocess
+
+import numpy as np
+import torch
+
+import fastvideo.envs as envs
+
+DRAFT_SIZES: dict[str, tuple[int, int]] = {"768p": (1344, 768), "576p": (1024, 576)}
+DRAFT_FRAMES = 121
+# The causal VAE maps 124 frames to 37 latent frames; the last frame is held for the three extra ones.
+VAE_PADDING_FRAMES = 3
+# Seconds of held last frame ffmpeg appends so a short draft still yields DRAFT_FRAMES.
+_TAIL_PAD_SECONDS = 6
+_DECODE_TIMEOUT_S = 600
+
+
+def ffmpeg_decode_command(path: str, width: int, height: int) -> list[str]:
+    """fps=24, cover-scale (bilinear) and centre-crop, last frame held, exactly DRAFT_FRAMES frames, no audio."""
+    video_filter = (f"fps=24,scale={width}:{height}:force_original_aspect_ratio=increase:flags=bilinear,"
+                    f"crop={width}:{height},tpad=stop_mode=clone:stop_duration={_TAIL_PAD_SECONDS}")
+    return [
+        envs.FASTVIDEO_FFMPEG_BIN.get(), "-v", "error", "-i", path, "-vf", video_filter, "-frames:v",
+        str(DRAFT_FRAMES), "-an", "-f", "rawvideo", "-pix_fmt", "rgb24", "pipe:1"
+    ]
+
+
+def load_draft_pixels(path: str, size: str = "768p") -> torch.Tensor:
+    """uint8 pixels [1, 3, 124, H, W] on the CPU for the draft video at ``path``."""
+    if size not in DRAFT_SIZES:
+        raise ValueError(f"draft size must be one of {sorted(DRAFT_SIZES)}, got {size!r}.")
+    width, height = DRAFT_SIZES[size]
+    try:
+        raw = subprocess.run(ffmpeg_decode_command(path, width, height),
+                             capture_output=True,
+                             check=True,
+                             timeout=_DECODE_TIMEOUT_S).stdout
+    except FileNotFoundError as error:
+        raise RuntimeError(f"ffmpeg ({envs.FASTVIDEO_FFMPEG_BIN.get()!r}) is required to read the draft video; "
+                           "set FASTVIDEO_FFMPEG_BIN.") from error
+    except subprocess.CalledProcessError as error:
+        raise ValueError(
+            f"Could not decode the draft video {path!r}: {error.stderr.decode(errors='replace')}") from error
+    expected = DRAFT_FRAMES * height * width * 3
+    if len(raw) != expected:
+        raise ValueError(f"The draft video {path!r} decoded to {len(raw)} bytes, expected {expected} "
+                         f"({DRAFT_FRAMES} frames at {width}x{height}).")
+    frames = torch.from_numpy(np.frombuffer(raw, np.uint8).reshape(DRAFT_FRAMES, height, width, 3).copy())
+    pixels = frames.permute(3, 0, 1, 2)[None]
+    return torch.cat([pixels, pixels[:, :, -1:].expand(-1, -1, VAE_PADDING_FRAMES, -1, -1)], dim=2).contiguous()

fastvideo/pipelines/basic/open_h3_regenerate_2k/open_h3_regenerate_2k_pipeline.py

@@ -0,0 +1,102 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Open-H3-Regenerate-2K: regenerate a 768P MiniMax-H3 draft video at 2560x1440.
+
+``draft video -> H3 VAE encode -> latent upscaler -> 3 fixed-sigma Euler steps (H3 DiT + merged LoRA + Sharp curve
+AdaLN branches) -> H3 VAE decode``. The release repo holds the unmodified MiniMax-H3 components under ``base_model/``
+next to ``lora/`` and ``latent_upsampler/``; see ``docs/inference/open_h3_regenerate_2k.md``.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any
+
+import torch
+
+from fastvideo.fastvideo_args import FastVideoArgs
+from fastvideo.logger import init_logger
+from fastvideo.models.upsamplers.minimax_h3_latent_upscaler import MiniMaxH3LatentUpscaler
+from fastvideo.pipelines.basic.minimax_h3.minimax_h3_pipeline import MiniMaxH3Pipeline
+from fastvideo.pipelines.basic.minimax_h3.stages import MiniMaxH3VideoDecodingStage
+from fastvideo.pipelines.basic.open_h3_regenerate_2k.sharp_curve import SharpCurveBranches
+from fastvideo.pipelines.basic.open_h3_regenerate_2k.stages import (
+    Regenerate2KDenoisingStage,
+    Regenerate2KFinalizeStage,
+    Regenerate2KLatentPreparationStage,
+)
+from fastvideo.utils import maybe_download_model, verify_model_config_and_directory
+
+logger = init_logger(__name__)
+
+BASE_MODEL_DIR = "base_model"
+LORA_FILE = "lora/open_h3_regenerate_2k_lora.safetensors"
+UPSCALER_FILE = "latent_upsampler/minimax_h3_latent_upscaler_3d_conv_v1_bf16.safetensors"
+_BASE_COMPONENTS = ("text_encoder", "tokenizer", "processor", "vae", "audio_vae", "transformer", "scheduler",
+                    "audio_scheduler")
+
+
+class OpenH3Regenerate2KPipeline(MiniMaxH3Pipeline):
+    """MiniMax-H3 T2VA layout with a draft-latent start, fixed sigmas and a silent output."""
+
+    _extra_config_module_map = {name: f"{BASE_MODEL_DIR}/{name}" for name in _BASE_COMPONENTS}
+    # The merged LoRA reaches only these linears; the AdaLN projections carry the Sharp curve branches instead.
+    lora_target_modules = ["attn.to_q", "attn.to_k", "attn.to_v", "attn.to_out", "ff.fc_in", "ff.fc_out"]
+
+    @classmethod
+    def get_hf_download_allow_patterns(cls) -> list[str]:
+        components = [f"{BASE_MODEL_DIR}/{name}/**" for name in _BASE_COMPONENTS]
+        return [f"{BASE_MODEL_DIR}/model_index.json", *components, "lora/**", "latent_upsampler/*.safetensors"]
+
+    def _load_config(self, model_path: str) -> dict[str, Any]:
+        revision = getattr(self.fastvideo_args, "revision", None)
+        self.model_path = maybe_download_model(model_path,
+                                               revision=revision,
+                                               allow_patterns=self.get_hf_download_allow_patterns())
+        if self.fastvideo_args.lora_path is None:
+            # The merged refine LoRA is part of the release; it is a constructor-time adapter like any H3 LoRA.
+            self.fastvideo_args.lora_path = os.path.join(self.model_path, LORA_FILE)
+        return verify_model_config_and_directory(os.path.join(self.model_path, BASE_MODEL_DIR),
+                                                 required_component_dirs=())
+
+    def initialize_pipeline(self, fastvideo_args: FastVideoArgs) -> None:
+        super().initialize_pipeline(fastvideo_args)
+        self._upscaler: MiniMaxH3LatentUpscaler | None = None
+        self._curve: SharpCurveBranches | None = None
+
+    def _load_upscaler(self) -> MiniMaxH3LatentUpscaler:
+        if self._upscaler is None:
+            model = MiniMaxH3LatentUpscaler.from_checkpoint(os.path.join(self.model_path, UPSCALER_FILE))
+            self._upscaler = model.to(device=self.device, dtype=torch.bfloat16)
+        return self._upscaler
+
+    def _load_curve(self) -> SharpCurveBranches:
+        if self._curve is None:
+            self._curve = SharpCurveBranches(Path(self.model_path))
+        return self._curve
+
+    def _add_denoise_stages(self, *, ref2va: bool) -> None:
+        config = self.fastvideo_args.pipeline_config
+        vae = self.get_module("vae")
+        audio_vae = self.get_module("audio_vae")
+        transformer = self.get_module("transformer")
+        if vae is None or audio_vae is None or transformer is None:
+            raise RuntimeError("Open-H3-Regenerate-2K needs the video VAE, audio VAE and transformer loaded.")
+        self.add_stage(
+            "latent_preparation_stage",
+            Regenerate2KLatentPreparationStage(vae,
+                                               audio_vae,
+                                               self.get_module("scheduler"),
+                                               load_upscaler=self._load_upscaler,
+                                               draft_size=config.draft_size,
+                                               start_sigma=config.refine_start_sigma))
+        self.add_stage("denoising_stage",
+                       Regenerate2KDenoisingStage(transformer, self._load_curve(), config.refine_start_sigma))
+        self.add_stage("video_decoding_stage", MiniMaxH3VideoDecodingStage(vae=vae))
+        self.add_stage("audio_decoding_stage", Regenerate2KFinalizeStage())
+        self._denoise_stages_ready = True
+
+
+EntryClass = OpenH3Regenerate2KPipeline
+
+__all__ = ["EntryClass", "OpenH3Regenerate2KPipeline"]

fastvideo/pipelines/basic/open_h3_regenerate_2k/presets.py

@@ -0,0 +1,35 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Inference preset for Open-H3-Regenerate-2K."""
+
+from fastvideo.api.presets import InferencePreset, PresetStageSpec
+
+_REFINE_STAGE = PresetStageSpec(
+    name="refine",
+    kind="denoising",
+    description="Latent upscale, then three fixed-sigma Euler refine steps",
+    allowed_overrides=frozenset({"num_inference_steps"}),
+)
+
+OPEN_H3_REGENERATE_2K = InferencePreset(
+    name="open_h3_regenerate_2k",
+    version=1,
+    model_family="open_h3_regenerate_2k",
+    description="Open-H3-Regenerate-2K: regenerate a 768P MiniMax-H3 draft video at 2560x1440 (silent output)",
+    workload_type=None,
+    stage_schemas=(_REFINE_STAGE, ),
+    defaults={
+        "height": 1440,
+        "width": 2560,
+        "num_frames": 124,
+        "fps": 24,
+        "guidance_scale": 1.0,
+        "batch_cfg": False,
+        "negative_prompt": "",
+        "num_inference_steps": 3,
+        "seed": 1766580932,
+    },
+)
+
+ALL_PRESETS = (OPEN_H3_REGENERATE_2K, )
+
+__all__ = ["ALL_PRESETS", "OPEN_H3_REGENERATE_2K"]

fastvideo/pipelines/basic/open_h3_regenerate_2k/sampling.py

@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Fixed Open-H3-Regenerate-2K refine schedule: Euler, CFG 1, model time ``1 - sigma``."""
+
+from __future__ import annotations
+
+import torch
+
+REFINE_SHIFT = 6.0
+DEFAULT_REFINE_STEPS = 3
+# Start noise of the two-step variant; the three-step grid starts at 0.909375.
+TWO_STEP_START_SIGMA = 0.7
+
+
+def shifted_grid(start: float, steps: int, shift: float = REFINE_SHIFT) -> list[float]:
+    """Euler grid from ``start`` to 0: even spacing in unshifted sigma, then ``s * x / (1 + (s - 1) * x)``."""
+    if not 0.0 < start < 1.0:
+        raise ValueError(f"start sigma must be in (0, 1), got {start}.")
+    if steps < 1:
+        raise ValueError(f"steps must be positive, got {steps}.")
+    raw = start / (shift - (shift - 1) * start)
+    mapped = [shift * x / (1 + (shift - 1) * x) for x in (raw * (1 - i / steps) for i in range(1, steps))]
+    return [start, *mapped, 0.0]
+
+
+REFINE_SCHEDULES: dict[int, tuple[float, ...]] = {
+    3: (0.909375, 6 / 7, 2 / 3, 0.0),
+    2: tuple(shifted_grid(TWO_STEP_START_SIGMA, 2)),
+}
+
+
+def refine_sigmas(steps: int = DEFAULT_REFINE_STEPS, start_sigma: float | None = None) -> list[float]:
+    """Sigma grid of ``steps`` Euler steps, ending at 0."""
+    if start_sigma is not None:
+        return shifted_grid(start_sigma, steps)
+    if steps not in REFINE_SCHEDULES:
+        raise ValueError(f"Open-H3-Regenerate-2K supports {sorted(REFINE_SCHEDULES)} refine steps, got {steps}.")
+    return list(REFINE_SCHEDULES[steps])
+
+
+def mix_initial_state(upscaled: torch.Tensor, noise: torch.Tensor, sigma: float) -> torch.Tensor:
+    """Re-noise the upscaled draft latent to the first sigma: ``(1 - sigma) * upscaled + sigma * noise``."""
+    return (1.0 - sigma) * upscaled + sigma * noise

fastvideo/pipelines/basic/open_h3_regenerate_2k/sharp_curve.py

@@ -0,0 +1,86 @@
+# SPDX-License-Identifier: Apache-2.0
+"""LMS Sharp curve AdaLN branches.
+
+Fifty AdaLN projections get an extra low-rank term whose input is an 8-D curve interpolated from the model timestep
+(``adaln_t_table``, 1025 x 8) instead of the layer input, so it cannot be folded into the LoRA. The term is added to the
+projection output through a forward hook; ``set_timesteps`` must be called with the per-forward unique timesteps first.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import torch
+import torch.nn.functional as F
+from safetensors.torch import load_file
+
+CURVE_BRANCHES_FILE = "lora/sharp_curve_branches.safetensors"
+CURVE_TABLE_FILE = "lora/adaln_t_table.bin"
+CURVE_TABLE_SHAPE = (1025, 8)
+NUM_CURVE_BRANCHES = 50
+_KEY_PREFIX = "transformer."
+_A_SUFFIX = ".curve_A.weight"
+_B_SUFFIX = ".curve_B.weight"
+
+
+def interpolate_curve(table: torch.Tensor, timesteps: torch.Tensor) -> torch.Tensor:
+    """Linear interpolation of ``table`` rows at model times in [0, 1]; ``timesteps`` [U] -> [U, 8]."""
+    t = timesteps.float()
+    if t.numel() and (t.min() < 0 or t.max() > 1):
+        raise ValueError("Sharp curve timesteps must lie in [0, 1].")
+    table = table.to(t.device)
+    position = t * (len(table) - 1)
+    low = position.floor().long().clamp(max=len(table) - 2)
+    return torch.lerp(table[low], table[low + 1], (position - low).unsqueeze(-1))
+
+
+class SharpCurveBranches:
+
+    def __init__(self, weights_dir: str | Path) -> None:
+        root = Path(weights_dir)
+        raw = torch.frombuffer(bytearray((root / CURVE_TABLE_FILE).read_bytes()), dtype=torch.float32)
+        if raw.numel() != CURVE_TABLE_SHAPE[0] * CURVE_TABLE_SHAPE[1]:
+            raise ValueError(f"{root / CURVE_TABLE_FILE} must hold a {CURVE_TABLE_SHAPE} float32 table.")
+        self.table = raw.reshape(CURVE_TABLE_SHAPE).clone()
+        tensors = load_file(str(root / CURVE_BRANCHES_FILE), device="cpu")
+        self.factors: dict[str, tuple[torch.Tensor, torch.Tensor]] = {
+            key[len(_KEY_PREFIX):-len(_A_SUFFIX)]: (value.float(), tensors[key[:-len(_A_SUFFIX)] + _B_SUFFIX].float())
+            for key, value in tensors.items() if key.endswith(_A_SUFFIX)
+        }
+        if len(self.factors) != NUM_CURVE_BRANCHES:
+            raise ValueError(f"Expected {NUM_CURVE_BRANCHES} Sharp curve branches, found {len(self.factors)}.")
+        self.coords: torch.Tensor | None = None
+        self._attached: set[int] = set()
+        self._device_factors: dict[str, tuple[torch.Tensor, torch.Tensor]] = {}
+
+    def set_timesteps(self, unique_timesteps: torch.Tensor) -> None:
+        self.coords = interpolate_curve(self.table, unique_timesteps)
+
+    def attach(self, transformer: torch.nn.Module) -> None:
+        """Register the branch hooks on ``transformer`` once."""
+        if id(transformer) in self._attached:
+            return
+        modules = dict(transformer.named_modules())
+        missing = sorted(set(self.factors) - set(modules))
+        if missing:
+            raise ValueError(f"Sharp curve branches match no transformer module: {missing[:5]}")
+        for name in self.factors:
+            modules[name].register_forward_hook(self._hook(name))
+        self._attached.add(id(transformer))
+
+    def _hook(self, name: str):
+
+        def add_branch(module, inputs, output):
+            projected = output[0] if isinstance(output, tuple) else output
+            coords = self.coords
+            if coords is None or coords.shape[0] != projected.shape[0]:
+                raise RuntimeError("Sharp curve timesteps were not set for this forward.")
+            factors = self._device_factors.get(name)
+            if factors is None or factors[0].device != projected.device:
+                factors = tuple(f.to(projected.device) for f in self.factors[name])
+                self._device_factors[name] = factors
+            delta = F.linear(F.linear(coords, factors[0]), factors[1]).to(projected.dtype)
+            projected = projected + delta
+            return (projected, *output[1:]) if isinstance(output, tuple) else projected
+
+        return add_branch

fastvideo/pipelines/basic/open_h3_regenerate_2k/stages/__init__.py

@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from fastvideo.pipelines.basic.open_h3_regenerate_2k.stages.denoising import Regenerate2KDenoisingStage
+from fastvideo.pipelines.basic.open_h3_regenerate_2k.stages.finalize import Regenerate2KFinalizeStage
+from fastvideo.pipelines.basic.open_h3_regenerate_2k.stages.latent_preparation import (
+    Regenerate2KLatentPreparationStage, )
+
+__all__ = [
+    "Regenerate2KDenoisingStage",
+    "Regenerate2KFinalizeStage",
+    "Regenerate2KLatentPreparationStage",
+]

fastvideo/pipelines/basic/open_h3_regenerate_2k/stages/denoising.py

@@ -0,0 +1,109 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Fixed-sigma Euler refine loop of Open-H3-Regenerate-2K."""
+
+from __future__ import annotations
+
+import contextlib
+from typing import Any
+
+import torch
+from torch.nn.attention import SDPBackend, sdpa_kernel
+
+from fastvideo.distributed import get_local_torch_device
+from fastvideo.fastvideo_args import FastVideoArgs
+from fastvideo.forward_context import set_forward_context
+from fastvideo.logger import init_logger
+from fastvideo.pipelines.basic.minimax_h3.packing import MiniMaxH3PackedLayout, build_row_timesteps
+from fastvideo.pipelines.basic.minimax_h3.stages.minimax_h3_latent_preparation import MINIMAX_H3_LAYOUT_KEY
+from fastvideo.pipelines.basic.open_h3_regenerate_2k.sampling import refine_sigmas
+from fastvideo.pipelines.basic.open_h3_regenerate_2k.sharp_curve import SharpCurveBranches
+from fastvideo.pipelines.lazy_module import is_lazy_module
+from fastvideo.pipelines.pipeline_batch_info import ForwardBatch
+from fastvideo.pipelines.stages.base import PipelineStage
+from fastvideo.pipelines.stages.validators import StageValidators as V
+from fastvideo.pipelines.stages.validators import VerificationResult
+
+logger = init_logger(__name__)
+
+
+class Regenerate2KDenoisingStage(PipelineStage):
+    """Euler steps over the refine sigmas with the video clock at ``1 - sigma`` and the silent audio held clean."""
+
+    performance_component_metric = "dit_time_s"
+
+    def __init__(self, transformer: Any, curve: SharpCurveBranches, start_sigma: float | None) -> None:
+        super().__init__()
+        self.transformer = transformer
+        self.curve = curve
+        self.start_sigma = start_sigma
+
+    def verify_input(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> VerificationResult:
+        result = VerificationResult()
+        result.add_check("layout", batch.extra.get(MINIMAX_H3_LAYOUT_KEY), V.not_none)
+        result.add_check("prompt_embeds", batch.prompt_embeds, V.list_of_tensors_dims(3))
+        result.add_check("latents", batch.latents, V.with_dims(2))
+        result.add_check("audio_latents", batch.audio_latents, V.with_dims(2))
+        return result
+
+    def verify_output(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> VerificationResult:
+        result = VerificationResult()
+        result.add_check("latents", batch.latents, V.with_dims(2))
+        return result
+
+    @torch.no_grad()
+    def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch:
+        transformer = self.transformer
+        if is_lazy_module(transformer):
+            transformer.materialize()
+        self.curve.attach(transformer)
+        layout = batch.extra.get(MINIMAX_H3_LAYOUT_KEY)
+        if not isinstance(layout, MiniMaxH3PackedLayout):
+            raise ValueError("The MiniMax-H3 packed layout is missing before denoising.")
+        device = get_local_torch_device()
+        sigmas = refine_sigmas(batch.num_inference_steps, self.start_sigma)
+        state = batch.latents.to(device)
+        with self._attention_kernel(device):
+            state = self._euler_loop(batch, layout, state, sigmas, device, transformer)
+        batch.latents = state
+        return batch
+
+    @staticmethod
+    def _attention_kernel(device: torch.device) -> contextlib.AbstractContextManager[None]:
+        """cuDNN attention: at 133k tokens PyTorch's default SDPA dispatch picks its flash kernel, 3x slower on B200."""
+        return sdpa_kernel(SDPBackend.CUDNN_ATTENTION) if device.type == "cuda" else contextlib.nullcontext()
+
+    def _euler_loop(self, batch: ForwardBatch, layout: MiniMaxH3PackedLayout, state: torch.Tensor, sigmas: list[float],
+                    device: torch.device, transformer: Any) -> torch.Tensor:
+        position_ids = layout.position_ids.to(device)
+        token_tags = layout.token_tags.to(device)
+        video_indices = layout.video_indices.to(device)
+        audio_indices = layout.audio_indices.to(device)
+        text_indices = layout.text_indices.to(device)
+        prompt_embeds = batch.prompt_embeds[0].to(device)
+        audio = batch.audio_latents.to(device=device, dtype=torch.bfloat16)[None]
+        for index, (sigma, following) in enumerate(zip(sigmas[:-1], sigmas[1:], strict=True)):
+            unique, inverse = build_row_timesteps(layout,
+                                                  video_timestep=1.0 - sigma,
+                                                  audio_timestep=1.0,
+                                                  condition_video_timestep=1.0,
+                                                  condition_audio_timestep=1.0)
+            unique, inverse = unique.to(device), inverse.to(device)
+            self.curve.set_timesteps(unique)
+            torch.compiler.cudagraph_mark_step_begin()
+            with set_forward_context(current_timestep=index, attn_metadata=None, forward_batch=batch):
+                velocity, _ = transformer(hidden_states=state.to(torch.bfloat16)[None],
+                                          audio_hidden_states=audio,
+                                          encoder_hidden_states=prompt_embeds,
+                                          timestep=unique,
+                                          timestep_indices=inverse,
+                                          token_tags=token_tags,
+                                          position_ids=position_ids,
+                                          video_indices=video_indices,
+                                          audio_indices=audio_indices,
+                                          text_indices=text_indices)
+            state = state + (sigma - following) * velocity[0].float()
+            batch.step_index = index
+        return state
+
+
+__all__ = ["Regenerate2KDenoisingStage"]

fastvideo/pipelines/basic/open_h3_regenerate_2k/stages/finalize.py

@@ -0,0 +1,39 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Trim the decoded video to the draft length and drop the per-request H3 state."""
+
+from __future__ import annotations
+
+from fastvideo.fastvideo_args import FastVideoArgs
+from fastvideo.pipelines.basic.open_h3_regenerate_2k.draft_video import DRAFT_FRAMES
+from fastvideo.pipelines.pipeline_batch_info import ForwardBatch
+from fastvideo.pipelines.stages.base import PipelineStage
+from fastvideo.pipelines.stages.validators import StageValidators as V
+from fastvideo.pipelines.stages.validators import VerificationResult
+
+
+class Regenerate2KFinalizeStage(PipelineStage):
+    """The released pipeline writes a silent video: the audio stream only conditions the DiT and is never decoded."""
+
+    def verify_input(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> VerificationResult:
+        result = VerificationResult()
+        result.add_check("output", batch.output, V.with_dims(5))
+        return result
+
+    def verify_output(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> VerificationResult:
+        result = VerificationResult()
+        result.add_check("output", batch.output, V.with_dims(5))
+        return result
+
+    def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch:
+        # The VAE decodes 124 frames (37 latent frames); the draft has 121.
+        batch.output = batch.output[:, :, :DRAFT_FRAMES]
+        batch.prompt_embeds = []
+        batch.latents = None
+        batch.audio_latents = None
+        for key in tuple(batch.extra):
+            if key.startswith("minimax_h3_"):
+                batch.extra.pop(key)
+        return batch
+
+
+__all__ = ["Regenerate2KFinalizeStage"]

fastvideo/pipelines/basic/open_h3_regenerate_2k/stages/latent_preparation.py

@@ -0,0 +1,129 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Draft video -> upscaled, re-noised H3 latent plus the fixed silent-audio rows."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any
+
+import torch
+
+from fastvideo.distributed import get_local_torch_device
+from fastvideo.fastvideo_args import FastVideoArgs
+from fastvideo.logger import init_logger
+from fastvideo.models import pinned_offload
+from fastvideo.models.upsamplers.minimax_h3_latent_upscaler import MiniMaxH3LatentUpscaler, upscale_latent
+from fastvideo.pipelines.basic.minimax_h3.packing import (
+    MINIMAX_H3_AUDIO_CHANNELS,
+    MINIMAX_H3_FPS,
+    MINIMAX_H3_KEYFRAME_ENCODE_SEED,
+    h3_dit_patch_size,
+    patchify_video_latents,
+)
+from fastvideo.pipelines.basic.minimax_h3.stages.minimax_h3_latent_preparation import (
+    MINIMAX_H3_LAYOUT_KEY,
+    MiniMaxH3LatentPreparationStage,
+)
+from fastvideo.pipelines.basic.open_h3_regenerate_2k.draft_video import load_draft_pixels
+from fastvideo.pipelines.basic.open_h3_regenerate_2k.sampling import mix_initial_state, refine_sigmas
+from fastvideo.pipelines.pipeline_batch_info import ForwardBatch
+from fastvideo.pipelines.stages.validators import VerificationResult
+
+logger = init_logger(__name__)
+
+DRAFT_FRAME_COUNT = 124
+
+
+class Regenerate2KLatentPreparationStage(MiniMaxH3LatentPreparationStage):
+    """Encode the draft with the H3 VAE, upscale its latent to the target grid and re-noise it to the first sigma."""
+
+    def __init__(self, vae: Any, audio_vae: Any, scheduler: Any, *, load_upscaler: Callable[[],
+                                                                                            MiniMaxH3LatentUpscaler],
+                 draft_size: str, start_sigma: float | None) -> None:
+        super().__init__(vae, audio_vae, scheduler, ref2va=False)
+        self.load_upscaler = load_upscaler
+        self.draft_size = draft_size
+        self.start_sigma = start_sigma
+        self._silence: tuple[tuple[str, int], torch.Tensor] | None = None
+
+    def verify_input(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> VerificationResult:
+        result = super().verify_input(batch, fastvideo_args)
+        result.add_check("video_path", batch.video_path, lambda value: isinstance(value, str) and bool(value))
+        return result
+
+    def _encode_draft(self, video_path: str, device: torch.device, fastvideo_args: FastVideoArgs) -> torch.Tensor:
+        """Normalized native latent [C, T, h, w] of the draft, with the VAE posterior sampled under a fixed seed."""
+        pixels = load_draft_pixels(video_path, self.draft_size)
+        pinned_offload.load(self.vae, device, pin=fastvideo_args.pin_cpu_memory)
+        try:
+            posterior = self.vae.encode_pixels(pixels).latent_dist
+            sample = posterior.sample(generator=torch.Generator("cpu").manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED))
+            return self.vae.normalize_latents(sample.float())[0].to(device)
+        finally:
+            if fastvideo_args.vae_cpu_offload:
+                pinned_offload.unload(self.vae)
+
+    def _silent_audio_rows(self, num_frames: int, device: torch.device, fastvideo_args: FastVideoArgs) -> torch.Tensor:
+        """The encoded-silence audio stream, cached: it is the same for every request of this length."""
+        key = (str(device), num_frames)
+        if self._silence is not None and self._silence[0] == key:
+            return self._silence[1]
+        pinned_offload.load(self.audio_vae, device, pin=fastvideo_args.pin_cpu_memory)
+        try:
+            samples = round(num_frames / MINIMAX_H3_FPS * int(self.audio_vae.sampling_rate))
+            silence = torch.zeros(MINIMAX_H3_AUDIO_CHANNELS, 1, samples, device=device)
+            posterior = self.audio_vae.encode(silence).latent_dist
+            latents = self.audio_vae.normalize_latents(posterior.mode().float()).transpose(1, 2)
+            rows = latents.reshape(-1, self.audio_vae.latent_channels).to(torch.bfloat16)
+        finally:
+            if fastvideo_args.vae_cpu_offload:
+                pinned_offload.unload(self.audio_vae)
+        self._silence = (key, rows)
+        return rows
+
+    @staticmethod
+    def _noise_rows(shape: tuple[int, int], device: torch.device, seed: int) -> torch.Tensor:
+        generator = torch.Generator(device=device).manual_seed(seed)
+        return torch.randn((1, *shape), device=device, dtype=torch.float32, generator=generator)[0]
+
+    @torch.no_grad()
+    def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch:
+        if batch.num_frames != DRAFT_FRAME_COUNT:
+            raise ValueError(f"Open-H3-Regenerate-2K refines {DRAFT_FRAME_COUNT} frames, got {batch.num_frames}.")
+        device = get_local_torch_device()
+        patch_size = h3_dit_patch_size(fastvideo_args)
+        _, channels, num_latent_frames, latent_height, latent_width = batch.raw_latent_shape
+
+        draft = self._encode_draft(batch.video_path, device, fastvideo_args)
+        upscaler = self.load_upscaler()
+        upscaled = upscale_latent(upscaler, draft, (batch.height, batch.width))
+        if tuple(upscaled.shape) != (channels, num_latent_frames, latent_height, latent_width):
+            raise ValueError(f"The upscaled latent has shape {tuple(upscaled.shape)}, expected "
+                             f"{(channels, num_latent_frames, latent_height, latent_width)}.")
+        upscaled_rows = patchify_video_latents(upscaled[None], patch_size)
+
+        if batch.latents is None:
+            noise_rows = self._noise_rows(tuple(upscaled_rows.shape), device, 0 if batch.seed is None else batch.seed)
+        else:
+            expected = (1, channels, num_latent_frames, latent_height, latent_width)
+            if tuple(batch.latents.shape) != expected:
+                raise ValueError(f"Injected noise must have shape {expected}, got {tuple(batch.latents.shape)}.")
+            noise_rows = patchify_video_latents(batch.latents.to(device=device, dtype=torch.float32), patch_size)
+
+        layout = self._build_layout(batch, fastvideo_args)
+        audio_rows = self._silent_audio_rows(batch.num_frames, device, fastvideo_args)
+        if audio_rows.shape[0] != layout.audio_indices.numel():
+            raise ValueError(f"The silent audio has {audio_rows.shape[0]} rows, the packed layout expects "
+                             f"{layout.audio_indices.numel()} ({layout.num_audio_latents} latents).")
+
+        batch.latents = mix_initial_state(upscaled_rows, noise_rows,
+                                          refine_sigmas(batch.num_inference_steps, self.start_sigma)[0])
+        batch.audio_latents = audio_rows
+        batch.extra[MINIMAX_H3_LAYOUT_KEY] = layout
+        for key in tuple(batch.extra):
+            if key.startswith("minimax_h3_") and key != MINIMAX_H3_LAYOUT_KEY:
+                batch.extra.pop(key)
+        return batch
+
+
+__all__ = ["Regenerate2KLatentPreparationStage"]

fastvideo/pipelines/basic/pan2/__init__.py

@@ -1 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0

fastvideo/pipelines/basic/pan2/pan2_pipeline.py

@@ -1,49 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""PAN2 video generation: text-to-video, and image-to-video from a first frame."""
-
-import torch
-
-from fastvideo.fastvideo_args import FastVideoArgs
-from fastvideo.pipelines.basic.pan2.stages import (
-    PAN2ConditioningStage,
-    PAN2DenoisingStage,
-    PAN2InputValidationStage,
-    PAN2TextEncodingStage,
-    PAN2TimestepPreparationStage,
-)
-from fastvideo.pipelines.composed_pipeline_base import ComposedPipelineBase
-from fastvideo.pipelines.stages import DecodingStage, LatentPreparationStage
-
-
-class PAN2Pipeline(ComposedPipelineBase):
-    """One stage chain serves both modes: a request with ``image_path`` (or a PIL ``pil_image``) runs image-to-video
-    from that first frame, and one without runs text-to-video."""
-
-    _required_config_modules = ["text_encoder", "tokenizer", "vae", "transformer", "scheduler"]
-
-    def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None:
-        self.add_stage(stage_name="input_validation_stage", stage=PAN2InputValidationStage())
-        self.add_stage(stage_name="prompt_encoding_stage",
-                       stage=PAN2TextEncodingStage(
-                           text_encoders=[self.get_module("text_encoder")],
-                           tokenizers=[self.get_module("tokenizer")],
-                       ))
-        self.add_stage(stage_name="timestep_preparation_stage",
-                       stage=PAN2TimestepPreparationStage(scheduler=self.get_module("scheduler")))
-        self.add_stage(stage_name="latent_preparation_stage",
-                       stage=LatentPreparationStage(scheduler=self.get_module("scheduler"),
-                                                    transformer=self.get_module("transformer"),
-                                                    latent_dtype=torch.float32))
-        self.add_stage(stage_name="conditioning_stage", stage=PAN2ConditioningStage(vae=self.get_module("vae")))
-        self.add_stage(stage_name="denoising_stage",
-                       stage=PAN2DenoisingStage(transformer=self.get_module("transformer"),
-                                                scheduler=self.get_module("scheduler"),
-                                                pipeline=self))
-        self.add_stage(stage_name="decoding_stage", stage=DecodingStage(vae=self.get_module("vae"), pipeline=self))
-
-
-class PAN2ModularPipeline(PAN2Pipeline):
-    """The pipeline class PAN2 checkpoints declare in their ``modular_model_index.json``."""
-
-
-EntryClass = [PAN2Pipeline, PAN2ModularPipeline]

fastvideo/pipelines/basic/pan2/presets.py

@@ -1,48 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""Inference presets for PAN2 text-to-video and image-to-video."""
-
-from fastvideo.api.presets import InferencePreset, PresetStageSpec
-
-_DENOISE_STAGE = PresetStageSpec(
-    name="denoise",
-    kind="denoising",
-    description="PAN2 flow-matching denoising",
-    allowed_overrides=frozenset({"num_inference_steps", "guidance_scale"}),
-)
-
-# 704x1248, 313 frames (13 s at 24 fps), 50 steps, CFG 3. The preset sends no negative prompt (an empty one is encoded
-# as a single space); the reference sampler sends the release's structured negative caption, so pass it as the
-# request's ``negative_prompt`` to follow that recipe.
-_SHARED_DEFAULTS = {
-    "height": 704,
-    "width": 1248,
-    "num_frames": 313,
-    "fps": 24,
-    "num_inference_steps": 50,
-    "guidance_scale": 3.0,
-    "negative_prompt": "",
-}
-
-PAN2_T2V = InferencePreset(
-    name="pan2_t2v",
-    version=1,
-    model_family="pan2",
-    description="PAN2 text-to-video",
-    workload_type="t2v",
-    stage_schemas=(_DENOISE_STAGE, ),
-    defaults=dict(_SHARED_DEFAULTS),
-)
-
-PAN2_I2V = InferencePreset(
-    name="pan2_i2v",
-    version=1,
-    model_family="pan2",
-    description="PAN2 image-to-video from a first frame",
-    workload_type="i2v",
-    stage_schemas=(_DENOISE_STAGE, ),
-    defaults=dict(_SHARED_DEFAULTS),
-)
-
-ALL_PRESETS = (PAN2_T2V, PAN2_I2V)
-
-__all__ = ["ALL_PRESETS", "PAN2_I2V", "PAN2_T2V"]

fastvideo/pipelines/basic/pan2/stages.py

@@ -1,210 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""PAN2-specific pipeline stages: prompt encoding, first-frame conditioning, sigma schedule and fp32 sampling."""
-
-from __future__ import annotations
-
-from collections.abc import Sequence
-from typing import Any
-
-import numpy as np
-import PIL.Image
-import torch
-import torchvision.transforms.functional as TF
-from torch.distributed.tensor import DTensor
-
-from fastvideo.distributed import get_local_torch_device
-from fastvideo.fastvideo_args import FastVideoArgs
-from fastvideo.forward_context import set_forward_context
-from fastvideo.pipelines.pipeline_batch_info import ForwardBatch
-from fastvideo.pipelines.stages.base import PipelineStage
-from fastvideo.pipelines.stages.denoising import DenoisingStage
-from fastvideo.pipelines.stages.input_validation import InputValidationStage
-from fastvideo.pipelines.stages.text_encoding import TextEncodingStage
-from fastvideo.pipelines.stages.timestep_preparation import TimestepPreparationStage
-from fastvideo.pipelines.stages.validators import StageValidators as V
-from fastvideo.pipelines.stages.validators import VerificationResult
-from fastvideo.utils import PRECISION_TO_TYPE
-
-_USER_HEADER = "<|im_start|>user\n"
-
-
-class PAN2InputValidationStage(InputValidationStage):
-    """Rejects geometry the PAN2 VAE and patch embedding cannot represent."""
-
-    def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch:
-        batch = super().forward(batch, fastvideo_args)
-        pipeline_config = fastvideo_args.pipeline_config
-        vae_arch = pipeline_config.vae_config.arch_config
-        spatial_multiple = vae_arch.spatial_compression_ratio * pipeline_config.dit_config.arch_config.patch_size[1]
-        if batch.height % spatial_multiple != 0 or batch.width % spatial_multiple != 0:
-            raise ValueError(f"PAN2 height and width must be divisible by {spatial_multiple}, "
-                             f"got {batch.height} and {batch.width}.")
-        if (batch.num_frames - 1) % vae_arch.temporal_compression_ratio != 0:
-            raise ValueError(f"PAN2 num_frames - 1 must be divisible by {vae_arch.temporal_compression_ratio}, "
-                             f"got {batch.num_frames}.")
-        if batch.pil_image is not None and not isinstance(batch.pil_image, PIL.Image.Image):
-            raise ValueError("PAN2 image-to-video expects the first frame as a PIL image or an image_path.")
-        return batch
-
-
-class PAN2TextEncodingStage(TextEncodingStage):
-    """Encodes a prompt with PAN2's chat template.
-
-    PAN2 conditions on the tokens after the ``<|im_start|>user\\n`` header (the prompt and the assistant header),
-    read from the text encoder's hidden states with its last ``text_encoder_skip_layers`` layers skipped.
-    """
-
-    @staticmethod
-    def tokenize(tokenizer: Any, prompt: str, system_prompt: str, max_length: int) -> tuple[torch.Tensor, int]:
-        """Token ids truncated to ``max_length`` tokens after the user header, and where that header ends."""
-        # An empty prompt is encoded as a single space, as in training.
-        messages = [{"role": "system", "content": system_prompt}, {"role": "user", "content": prompt or " "}]
-        input_ids = tokenizer.apply_chat_template(messages, add_generation_prompt=True, tokenize=True,
-                                                  return_dict=True)["input_ids"]
-        if isinstance(input_ids, torch.Tensor):
-            input_ids = input_ids.tolist()
-        if input_ids and isinstance(input_ids[0], list):
-            input_ids = input_ids[0]
-        header_ids = tokenizer(_USER_HEADER, add_special_tokens=False)["input_ids"]
-        crop_start = next((i + len(header_ids)
-                           for i in range(len(input_ids)) if input_ids[i:i + len(header_ids)] == header_ids), None)
-        if crop_start is None:
-            raise ValueError(f"The PAN2 tokenizer's chat template has no {_USER_HEADER!r} header.")
-        return torch.tensor([input_ids[:crop_start + max_length]], dtype=torch.long), crop_start
-
-    @torch.no_grad()
-    def encode_text(
-        self,
-        text: str | list[str],
-        fastvideo_args: FastVideoArgs,
-        encoder_index: int | list[int] | None = None,
-        return_attention_mask: bool = False,
-        return_type: str = "list",
-        device: torch.device | str | None = None,
-        dtype: torch.dtype | None = None,
-        max_length: int | Sequence[int | None] | None = None,
-        truncation: bool | None = None,
-        padding: bool | str | None = None,
-    ):
-        pipeline_config = fastvideo_args.pipeline_config
-        texts = [text] if isinstance(text, str) else list(text)
-        if len(texts) != 1:
-            raise ValueError(f"PAN2 encodes one prompt per request, got {len(texts)}.")
-        if isinstance(max_length, Sequence):
-            max_length = max_length[0]
-        max_length = max_length or pipeline_config.text_max_sequence_length
-
-        tokenizer = self.tokenizers[0]
-        text_encoder = self.text_encoders[0]
-        target_device = torch.device(device) if device is not None else get_local_torch_device()
-        # A module parked on CPU by text_encoder_cpu_offload computes on the target device and goes back afterwards;
-        # FSDP-wrapped (DTensor) encoders move their own shards.
-        first_param = next(text_encoder.parameters(), None)
-        input_device = first_param.device if first_param is not None else target_device
-        moved_for_forward = False
-        if (first_param is not None and not isinstance(first_param, DTensor)
-                and input_device.type != target_device.type):
-            text_encoder = text_encoder.to(target_device)
-            input_device = target_device
-            moved_for_forward = True
-
-        input_ids, crop_start = self.tokenize(tokenizer, texts[0], pipeline_config.text_system_prompt, max_length)
-        with set_forward_context(current_timestep=0, attn_metadata=None):
-            outputs = text_encoder(input_ids=input_ids.to(input_device), output_hidden_states=True)
-        prompt_embeds = outputs.hidden_states[-(pipeline_config.text_encoder_skip_layers + 1)][:, crop_start:]
-        # fp32 is exact for the bf16 encoder output; the denoiser casts the embeddings to the DiT dtype.
-        prompt_embeds = prompt_embeds.to(device=target_device, dtype=dtype or torch.float32, copy=True)
-        attention_mask = torch.ones(prompt_embeds.shape[:2], dtype=torch.long, device=target_device)
-        if moved_for_forward and fastvideo_args.text_encoder_cpu_offload:
-            text_encoder.to("cpu")
-        return self.return_embeds([prompt_embeds], [attention_mask], return_type, return_attention_mask, [0])
-
-
-class PAN2TimestepPreparationStage(TimestepPreparationStage):
-    """PAN2 samples on uniformly spaced sigmas from 1 down to (excluding) 0; the scheduler applies the shift."""
-
-    def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch:
-        if batch.timesteps is None and batch.sigmas is None:
-            batch.sigmas = np.linspace(1.0, 0.0, batch.num_inference_steps + 1)[:-1].tolist()
-        return super().forward(batch, fastvideo_args)
-
-
-class PAN2ConditioningStage(PipelineStage):
-    """Builds the conditioning latents and mask that PAN2 stacks after the noisy latents along channels.
-
-    Text-to-video conditions on zeros. Image-to-video fills the first latent frame with the normalized VAE latents
-    of the first frame, scaled to cover the output and center cropped, and sets that frame's mask channel to one.
-    """
-
-    def __init__(self, vae) -> None:
-        super().__init__()
-        self.vae = vae
-
-    @staticmethod
-    def preprocess_image(image: PIL.Image.Image, height: int, width: int) -> torch.Tensor:
-        """``[1, 3, 1, H, W]`` pixels in ``[-1, 1]``: scale to cover the canvas, round the size, center crop."""
-        image = image.convert("RGB")
-        scale = max(width / image.width, height / image.height)
-        image = image.resize((round(image.width * scale), round(image.height * scale)), resample=PIL.Image.LANCZOS)
-        left, top = round((image.width - width) / 2), round((image.height - height) / 2)
-        image = image.crop((left, top, left + width, top + height))
-        return (TF.to_tensor(image) * 2.0 - 1.0)[None, :, None]
-
-    def _encode_first_frame(self, image: PIL.Image.Image, batch: ForwardBatch,
-                            fastvideo_args: FastVideoArgs) -> torch.Tensor:
-        device = get_local_torch_device()
-        vae_dtype = PRECISION_TO_TYPE[fastvideo_args.pipeline_config.vae_precision]
-        self.vae = self.vae.to(device)
-        pixels = self.preprocess_image(image, batch.height, batch.width).to(device=device, dtype=vae_dtype)
-        latents = self.vae.encode(pixels).mode().float()
-        shift = self.vae.shift_factor.to(latents.device, latents.dtype)
-        scale = self.vae.scaling_factor.to(latents.device, latents.dtype)
-        if fastvideo_args.vae_cpu_offload:
-            self.vae.to("cpu")
-        return (latents - shift) * scale
-
-    @torch.no_grad()
-    def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch:
-        latents = batch.latents
-        batch_size, num_channels, num_frames, height, width = latents.shape
-        condition = torch.zeros(batch_size,
-                                num_channels + 1,
-                                num_frames,
-                                height,
-                                width,
-                                device=latents.device,
-                                dtype=torch.float32)
-        if batch.pil_image is not None:
-            image_latents = self._encode_first_frame(batch.pil_image, batch, fastvideo_args)
-            condition[:, :num_channels, :1] = image_latents.to(condition.device)
-            condition[:, num_channels:, :1] = 1.0
-        batch.image_latent = condition
-        return batch
-
-    def verify_input(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> VerificationResult:
-        result = VerificationResult()
-        result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)])
-        result.add_check("height", batch.height, V.positive_int)
-        result.add_check("width", batch.width, V.positive_int)
-        return result
-
-    def verify_output(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> VerificationResult:
-        result = VerificationResult()
-        result.add_check("image_latent", batch.image_latent, [V.is_tensor, V.with_dims(5)])
-        return result
-
-
-class PAN2DenoisingStage(DenoisingStage):
-    """Keeps the latents in fp32 across the loop; they are cast to the DiT dtype for each forward.
-
-    Image-to-video leaves the conditioning image's latent frame out of classifier-free guidance: that frame takes the
-    positive prediction, as in the reference sampler.
-    """
-
-    def finish_guidance(self, noise_pred, noise_pred_text, batch, fastvideo_args) -> torch.Tensor:
-        if batch.pil_image is not None and fastvideo_args.pipeline_config.i2v_skip_first_latent_cfg:
-            noise_pred[:, :, :1] = noise_pred_text[:, :, :1]
-        return noise_pred
-
-    def prepare_scheduler_model_output(self, noise_pred, state) -> torch.Tensor:
-        return noise_pred.float()

fastvideo/pipelines/stages/denoising.py

@@ -472,9 +472,6 @@ class DenoisingStage(PipelineStage):
                                 noise_pred_text,
                                 guidance_rescale=batch.guidance_rescale,
                             )
-
-                        noise_pred = self.finish_guidance(noise_pred, noise_pred_text, batch, fastvideo_args)
-                noise_pred = self.prepare_scheduler_model_output(noise_pred, state)
                 if scheduler_fp32:
                     # Diffusers-style: fp32 Euler update outside autocast avoids BF16 drift.
                     latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
@@ -608,14 +605,6 @@ class DenoisingStage(PipelineStage):
     def prepare_timestep(self, timestep, latent_model_input, batch, state) -> torch.Tensor:
         return timestep.repeat(latent_model_input.shape[0])
 
-    def finish_guidance(self, noise_pred, noise_pred_text, batch, fastvideo_args) -> torch.Tensor:
-        """The guided prediction handed on to the scheduler; a model overrides it to keep part of the output out of CFG."""
-        return noise_pred
-
-    def prepare_scheduler_model_output(self, noise_pred, state) -> torch.Tensor:
-        """The model output handed to the scheduler step, which returns the next latents in its dtype."""
-        return noise_pred
-
     def finish_step(self, latents, state) -> torch.Tensor:
         return latents
 

fastvideo/pipelines/stages/input_validation.py

@@ -64,8 +64,7 @@ class InputValidationStage(PipelineStage):
             raise ValueError("Either `prompt` or `prompt_embeds` must be provided")
 
         # Ensure negative prompt is properly formatted if using classifier-free guidance
-        if (batch.do_classifier_free_guidance and batch.negative_prompt is None
-                and batch.negative_prompt_embeds is None):
+        if (batch.do_classifier_free_guidance and batch.negative_prompt is None and not batch.negative_prompt_embeds):
             raise ValueError("For classifier-free guidance, either `negative_prompt` or "
                              "`negative_prompt_embeds` must be provided")
 

fastvideo/pipelines/stages/latent_preparation.py

@@ -30,16 +30,11 @@ class LatentPreparationStage(PipelineStage):
     denoised during the diffusion process.
     """
 
-    def __init__(self,
-                 scheduler,
-                 transformer,
-                 use_btchw_layout: bool = False,
-                 latent_dtype: torch.dtype | None = None) -> None:
+    def __init__(self, scheduler, transformer, use_btchw_layout: bool = False) -> None:
         super().__init__()
         self.scheduler = scheduler
         self.transformer = transformer
         self.use_btchw_layout = use_btchw_layout
-        self.latent_dtype = latent_dtype
 
     def forward(
         self,
@@ -93,7 +88,7 @@ class LatentPreparationStage(PipelineStage):
             batch.prompt_embeds = [dummy_prompt]
             batch.negative_prompt_embeds = []
             batch.do_classifier_free_guidance = False
-        dtype = self.latent_dtype or batch.prompt_embeds[0].dtype
+        dtype = batch.prompt_embeds[0].dtype
         device = get_local_torch_device()
         generator = batch.generator
         latents = batch.latents

fastvideo/registry.py

@@ -10,7 +10,6 @@ from __future__ import annotations
 
 import dataclasses
 import os
-import re
 from collections.abc import Callable
 from functools import lru_cache
 from typing import TYPE_CHECKING, Any
@@ -43,7 +42,7 @@ from fastvideo.configs.pipelines.flux_2 import (
 from fastvideo.configs.pipelines.matrixgame2 import MatrixGame2I2V480PConfig
 from fastvideo.configs.pipelines.matrixgame3 import MatrixGame3I2V720PConfig
 from fastvideo.configs.pipelines.minimax_h3 import MiniMaxH3PipelineConfig
-from fastvideo.configs.pipelines.pan2 import PAN2PipelineConfig
+from fastvideo.configs.pipelines.open_h3_regenerate_2k import OpenH3Regenerate2KPipelineConfig
 from fastvideo.configs.pipelines.mmaudio import (
     MMAudioLarge44kV2PipelineConfig,
     MMAudioLarge44kV2AConfig,
@@ -218,6 +217,13 @@ def _resolve_class_name(config: dict[str, Any]) -> str | None:
     return class_name
 
 
+OPEN_H3_REGENERATE_2K_HF_PATH = "Efficient-Large-Model/Open-H3-Regenerate-2K"
+
+
+def _is_open_h3_regenerate_2k_release(model_path: str) -> bool:
+    return os.path.isfile(os.path.join(model_path, "lora", "open_h3_regenerate_2k_lora.safetensors"))
+
+
 def _get_config_info(
     model_path: str,
     *,
@@ -225,14 +231,21 @@ def _get_config_info(
     revision: str | None = None,
     allow_remote: bool = True,
 ) -> ConfigInfo | None:
+    # Normalize only the lookup key; keep the real path for manifest IO and detectors.
+    lookup_path = model_path.rstrip("/") or model_path
+    parts = lookup_path.split("/")
+    # Standard Hugging Face cache layout: models--{org}--{name}/snapshots/{revision}.
+    if len(parts) >= 3 and parts[-2] == "snapshots" and parts[-3].startswith("models--"):
+        lookup_path = parts[-3][len("models--"):].replace("--", "/", 1)
+
     # 1. Exact match
-    if model_path in _MODEL_HF_PATH_TO_NAME:
-        model_id = _MODEL_HF_PATH_TO_NAME[model_path]
+    if lookup_path in _MODEL_HF_PATH_TO_NAME:
+        model_id = _MODEL_HF_PATH_TO_NAME[lookup_path]
         logger.debug("Resolved model path '%s' from exact path match.", model_path)
         return _CONFIG_REGISTRY.get(model_id)
 
     # 2. Partial match: use short model name.
-    model_name = get_model_short_name(model_path.lower())
+    model_name = get_model_short_name(lookup_path.lower())
     all_model_hf_paths = sorted(_MODEL_HF_PATH_TO_NAME.keys(), key=len, reverse=True)
     for registered_model_hf_id in all_model_hf_paths:
         registered_model_name = get_model_short_name(registered_model_hf_id.lower())
@@ -242,6 +255,9 @@ def _get_config_info(
             return _CONFIG_REGISTRY.get(model_id)
 
     # 3. Use detectors (path or model_index pipeline name).
+    if os.path.exists(model_path) and _is_open_h3_regenerate_2k_release(model_path):
+        # The release keeps its Diffusers manifest under base_model/, so a renamed local copy has no root manifest.
+        return _CONFIG_REGISTRY.get(_MODEL_HF_PATH_TO_NAME[OPEN_H3_REGENERATE_2K_HF_PATH])
     if os.path.exists(model_path):
         config = verify_model_config_and_directory(model_path, required_component_dirs=[])
     elif allow_remote:
@@ -1147,6 +1163,22 @@ def _register_configs() -> None:
 
     _register_wan_configs(WAN_MODEL_DEFINITIONS)
 
+    # Open-H3-Regenerate-2K (video -> video). Registered before MiniMax H3 so the first matching detector wins. There is
+    # no V2V WorkloadType, so, like the Kandinsky6 SR entries, it declares no workload.
+    register_configs(
+        sampling_param_cls=None,
+        pipeline_config_cls=OpenH3Regenerate2KPipelineConfig,
+        workload_types=(),
+        hf_model_paths=[OPEN_H3_REGENERATE_2K_HF_PATH],
+        model_detectors=[
+            lambda path: any(token in path.lower()
+                             for token in ("open-h3-regenerate", "open_h3_regenerate", "openh3regenerate2kpipeline")),
+        ],
+        model_family="open_h3_regenerate_2k",
+        default_preset="open_h3_regenerate_2k",
+        pipeline_cls_name="OpenH3Regenerate2KPipeline",
+    )
+
     # MiniMax H3
     register_configs(
         sampling_param_cls=None,
@@ -1173,22 +1205,6 @@ def _register_configs() -> None:
         pipeline_cls_name="MiniMaxH3ModularPipeline",
     )
 
-    # PAN2: one pipeline runs text-to-video, and image-to-video when the request carries a first frame.
-    def _is_pan2(name: str) -> bool:
-        name = name.lower()
-        return "pan2modularpipeline" in name or re.search(r"(^|[/_-])pan-?2([/_.-]|$)", name) is not None
-
-    register_configs(
-        sampling_param_cls=None,
-        pipeline_config_cls=PAN2PipelineConfig,
-        workload_types=(WorkloadType.T2V, WorkloadType.I2V),
-        hf_model_paths=["IFM/PAN2"],
-        model_detectors=[_is_pan2],
-        model_family="pan2",
-        default_preset="pan2_t2v",
-        pipeline_cls_name="PAN2ModularPipeline",
-    )
-
     # SD3.5
     register_configs(
         sampling_param_cls=None,
@@ -1274,19 +1290,18 @@ def get_model_info(
         # is triggered just to log the replaced name).
         pipeline_name: str | None = override_pipeline_cls_name
         logger.info("Using override pipeline class name %s", pipeline_name)
+    elif config_info.pipeline_cls_name is not None:
+        # The resolved (path/detector-based) config pins the pipeline class, e.g. an I2V checkpoint whose
+        # `_class_name` would otherwise resolve to the T2V pipeline. The manifest is not read: a release such as
+        # Open-H3-Regenerate-2K keeps it in a subfolder.
+        pipeline_name = config_info.pipeline_cls_name
+        logger.info("Pinned pipeline class name %s", pipeline_name)
     else:
         if os.path.exists(model_path):
             config = verify_model_config_and_directory(model_path, required_component_dirs=[])
         else:
             config = maybe_download_model_index(model_path, revision=revision)
-
         pipeline_name = _resolve_class_name(config)
-        if config_info.pipeline_cls_name is not None:
-            # The resolved (path/detector-based) config pins the pipeline class,
-            # e.g. an I2V checkpoint whose `_class_name` would otherwise resolve
-            # to the T2V pipeline.
-            logger.info("Pinning pipeline class name from %s to %s", pipeline_name, config_info.pipeline_cls_name)
-            pipeline_name = config_info.pipeline_cls_name
 
     if pipeline_name is None:
         raise ValueError("Model config does not contain a _class_name attribute. "
@@ -1361,10 +1376,10 @@ def _register_presets() -> None:
         ALL_PRESETS as MATRIXGAME3_PRESETS, )
     from fastvideo.pipelines.basic.minimax_h3.presets import (
         ALL_PRESETS as MINIMAX_H3_PRESETS, )
+    from fastvideo.pipelines.basic.open_h3_regenerate_2k.presets import (
+        ALL_PRESETS as OPEN_H3_REGENERATE_2K_PRESETS, )
     from fastvideo.pipelines.basic.mmaudio.presets import (
         ALL_PRESETS as MMAUDIO_PRESETS, )
-    from fastvideo.pipelines.basic.pan2.presets import (
-        ALL_PRESETS as PAN2_PRESETS, )
     from fastvideo.pipelines.basic.sd35.presets import (
         ALL_PRESETS as SD35_PRESETS, )
     from fastvideo.pipelines.basic.stable_audio.presets import (
@@ -1396,8 +1411,8 @@ def _register_presets() -> None:
         MATRIXGAME2_PRESETS,
         MATRIXGAME3_PRESETS,
         MINIMAX_H3_PRESETS,
+        OPEN_H3_REGENERATE_2K_PRESETS,
         MMAUDIO_PRESETS,
-        PAN2_PRESETS,
         SD35_PRESETS,
         STABLE_AUDIO_PRESETS,
         WAN_PRESETS,

fastvideo/tests/api/test_wan_definitions.py

@@ -80,6 +80,40 @@ def test_legacy_aliases_and_sampling_defaults(model_path, config_name, preset_na
             assert getattr(sampling, name) == value
 
 
+@pytest.mark.parametrize("hf_id", [
+    "Wan-AI/Wan2.2-TI2V-5B-Diffusers",
+    "Wan-AI/Wan2.1-T2V-14B-Diffusers",
+])
+@pytest.mark.parametrize("path_form", ["renamed_parent", "trailing_slash", "snapshot", "snapshot_slash"])
+def test_local_path_forms_resolve_like_hub_id(tmp_path, hf_id, path_form):
+    expected = registry._get_config_info(hf_id)
+    org, name = hf_id.split("/", 1)
+    if path_form.startswith("snapshot"):
+        checkpoint = tmp_path / "hub" / f"models--{org}--{name}" / "snapshots" / ("0" * 40)
+    else:
+        checkpoint = tmp_path / "renamed-parent" / name
+    checkpoint.mkdir(parents=True)
+    (checkpoint / "model_index.json").write_text(json.dumps({
+        "_class_name": "WanPipeline", "_diffusers_version": "0.33.0",
+    }))
+    path = str(checkpoint) + ("/" if path_form.endswith("slash") else "")
+    info = registry._get_config_info(path)
+    assert info.pipeline_config_cls is expected.pipeline_config_cls
+    assert info.default_preset == expected.default_preset
+
+
+@pytest.mark.parametrize("suffix", ["", "/"])
+def test_unknown_snapshot_falls_back_to_real_manifest(tmp_path, suffix):
+    checkpoint = tmp_path / "models--x--y" / "snapshots" / ("0" * 40)
+    checkpoint.mkdir(parents=True)
+    (checkpoint / "model_index.json").write_text(json.dumps({
+        "_class_name": "WanImageToVideoPipeline", "_diffusers_version": "0.33.0",
+    }))
+    info = registry._get_config_info(str(checkpoint) + suffix)
+    assert info.pipeline_config_cls is pipeline_config.WanI2V480PConfig
+    assert info.default_preset == "wan_i2v_14b_480p"
+
+
 @pytest.mark.parametrize("directory,manifest_class,expected_family,expected_preset", [
     ("custom", "WanPipeline", "wan", "wan_t2v_1_3b"),
     ("custom", "WanImageToVideoPipeline", "wan", "wan_i2v_14b_480p"),

fastvideo/tests/entrypoints/test_openai_open_h3_regenerate_2k.py

@@ -0,0 +1,145 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Open-H3-Regenerate-2K through the OpenAI-compatible video routes, with a fake generator (no GPU)."""
+
+import asyncio
+import time
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+from fastapi.testclient import TestClient
+from PIL import Image
+
+from fastvideo.entrypoints.cli.inference_config import build_serve_config
+from fastvideo.entrypoints.openai.api_server import create_app
+from fastvideo.entrypoints.openai.media_policy import MediaPolicy
+from fastvideo.entrypoints.openai.stores import VIDEO_STORE
+
+ROOT = Path(__file__).resolve().parents[3]
+MODEL = "Efficient-Large-Model/Open-H3-Regenerate-2K"
+ARTIFACT = b"fake-2k-video-for-transport-tests"
+PROMPT = "a quiet street at dusk"
+
+
+class FileGenerator:
+
+    def __init__(self):
+        self.requests = []
+
+    def generate(self, request):
+        self.requests.append(request)
+        output = Path(request.output.output_path)
+        output.parent.mkdir(parents=True, exist_ok=True)
+        output.write_bytes(ARTIFACT)
+        return SimpleNamespace(video_path=str(output), generation_time=0.01)
+
+    def shutdown(self):
+        pass
+
+
+@pytest.fixture
+def served(monkeypatch, tmp_path):
+    generator = FileGenerator()
+    monkeypatch.setattr("fastvideo.entrypoints.openai.api_server.VideoGenerator.from_fastvideo_args",
+                        lambda args: generator)
+    drafts = tmp_path / "drafts"
+    drafts.mkdir()
+    (drafts / "draft.mp4").write_bytes(b"draft")
+    Image.new("RGB", (8, 8)).save(drafts / "still.png")
+    args = SimpleNamespace(model_path=MODEL, lora_path=None, lora_nickname="default", lora_strength=1.0,
+                           override_pipeline_cls_name=None)
+    config = build_serve_config(SimpleNamespace(config=str(ROOT / "examples/serving/openai_open_h3_regenerate_2k.yaml")),
+                                overrides=[])
+    app = create_app(args,
+                     str(tmp_path / "server"),
+                     config.default_request,
+                     config.server.served_model_name,
+                     media_policy=MediaPolicy(roots=(str(drafts), )))
+    with TestClient(app) as client:
+        yield client, generator, drafts
+    asyncio.run(VIDEO_STORE.clear())
+
+
+def _body(drafts, **extra):
+    return {"prompt": PROMPT, "video_reference": {"video_url": str(drafts / "draft.mp4")}, **extra}
+
+
+def test_serving_yaml_pins_the_2k_defaults():
+    config = build_serve_config(SimpleNamespace(config=str(ROOT / "examples/serving/openai_open_h3_regenerate_2k.yaml")),
+                                overrides=[])
+    sampling = config.default_request.sampling
+    assert (sampling.height, sampling.width, sampling.num_frames, sampling.num_inference_steps) == (1440, 2560, 124, 3)
+    assert config.generator.model_path == MODEL
+
+
+def test_sync_route_returns_the_mp4_and_maps_the_draft_to_video_path(served):
+    client, generator, drafts = served
+    response = client.post("/v1/videos/sync", json=_body(drafts, seed=7))
+    assert response.status_code == 200, response.text
+    assert response.content == ARTIFACT
+    assert response.headers["content-type"] == "video/mp4"
+    assert response.headers["x-model"] == "open-h3-regenerate-2k"
+    request = generator.requests[0]
+    assert request.prompt == PROMPT
+    assert request.inputs.video_path == str((drafts / "draft.mp4").resolve())
+    assert (request.sampling.height, request.sampling.width, request.sampling.num_frames) == (1440, 2560, 124)
+    assert request.sampling.num_inference_steps == 3 and request.sampling.seed == 7
+
+
+def test_async_route_completes_and_serves_the_content(served):
+    client, generator, drafts = served
+    created = client.post("/v1/videos", json=_body(drafts))
+    assert created.status_code == 200, created.text
+    video_id = created.json()["id"]
+    deadline = time.monotonic() + 10
+    while time.monotonic() < deadline:
+        status = client.get(f"/v1/videos/{video_id}").json()["status"]
+        if status in ("completed", "failed"):
+            break
+        time.sleep(0.02)
+    assert status == "completed"
+    assert client.get(f"/v1/videos/{video_id}/content").content == ARTIFACT
+    assert generator.requests[0].inputs.video_path.endswith("draft.mp4")
+
+
+def test_two_refine_steps_can_be_requested(served):
+    client, generator, drafts = served
+    assert client.post("/v1/videos/sync", json=_body(drafts, num_inference_steps=2)).status_code == 200
+    assert generator.requests[0].sampling.num_inference_steps == 2
+
+
+@pytest.mark.parametrize("reference, message", [
+    ({"video_url": "data:video/mp4;base64,AAAA"}, "Data video references are not supported"),
+    ({"video_url": "/etc/passwd"}, "Local media paths are not accepted"),
+    ({"file_id": "file_1"}, "file_id video references are not supported"),
+])
+def test_unusable_video_references_are_rejected(served, reference, message):
+    client, generator, _ = served
+    response = client.post("/v1/videos/sync", json={"prompt": PROMPT, "video_reference": reference})
+    assert response.status_code == 400 and message in response.text
+    assert not generator.requests
+
+
+@pytest.mark.parametrize("extra, message", [
+    ({"video_reference": None}, "exactly one video_reference"),
+    ({"image_reference": {"image_url": "STILL"}}, "exactly one video_reference"),
+    ({"size": "1920x1080"}, "always renders 2560x1440"),
+    ({"seconds": 5}, "first 121 frames"),
+    ({"fps": 30}, "requires fps=24"),
+])
+def test_requests_the_pipeline_cannot_honor_are_rejected(served, extra, message):
+    client, generator, drafts = served
+    body = {**_body(drafts), **extra}
+    if "image_reference" in body:
+        body["image_reference"] = {"image_url": str(drafts / "still.png")}
+    body = {key: value for key, value in body.items() if value is not None}
+    response = client.post("/v1/videos/sync", json=body)
+    assert response.status_code == 400 and message in response.text
+    assert not generator.requests
+
+
+def test_two_drafts_are_rejected(served):
+    client, generator, drafts = served
+    reference = {"video_url": str(drafts / "draft.mp4")}
+    response = client.post("/v1/videos/sync", json={"prompt": PROMPT, "video_reference": [reference, reference]})
+    assert response.status_code == 400 and "exactly one video_reference" in response.text

fastvideo/tests/entrypoints/test_serving_family_baselines.py

@@ -25,7 +25,6 @@ from fastvideo.entrypoints.openai.request_adapter import (
     prepare_reference_media,
 )
 from fastvideo.pipelines.basic.hunyuan15.presets import HUNYUAN15_I2V_480P_DISTILLED
-from fastvideo.pipelines.basic.pan2.presets import PAN2_T2V
 
 ROOT = Path(__file__).resolve().parents[3]
 
@@ -38,7 +37,6 @@ BASELINES = {
     "openai_kandinsky6_ti2va.yaml": "t2v",
     "openai_kandinsky6_ti2va_distilled.yaml": "t2v",
     "openai_ltx2_distilled.yaml": "t2v",
-    "openai_pan2.yaml": "t2v",
     "openai_flux1_dev.yaml": "t2i",
     "openai_flux2_dev.yaml": "t2i",
     "openai_flux2_klein.yaml": "t2i",
@@ -114,32 +112,6 @@ def test_hunyuan_i2v_480p_keeps_the_registered_checkpoint_semantics(tmp_path: Pa
     assert adapted.output.output_path.endswith(".mp4")
 
 
-def test_pan2_serving_matches_its_preset_and_runs_image_to_video_from_a_reference(tmp_path: Path) -> None:
-    config = _load("openai_pan2.yaml")
-    sampling = config.default_request.sampling
-    assert config.generator.model_path == "IFM/PAN2"
-    assert config.generator.engine.offload.dit is False
-    assert config.generator.engine.offload.dit_layerwise is False
-    for field in ("height", "width", "num_frames", "fps", "num_inference_steps", "guidance_scale"):
-        assert getattr(sampling, field) == PAN2_T2V.defaults[field], field
-
-    image = tmp_path / "first-frame.png"
-    image.write_bytes(b"not-decoded-at-admission")
-    for reference in (None, {"image_url": str(image)}):
-        request = VideoGenerationRequest(prompt="a red fox", image_reference=reference)
-        adapted = build_generation_request(
-            "pan2_test",
-            request,
-            _args(config, "t2v"),
-            served_model_name=config.server.served_model_name or "",
-            output_dir=str(tmp_path),
-            default_request=config.default_request,
-        )
-        assert adapted.sampling.num_frames == 313
-        assert adapted.inputs.image_path == (str(image) if reference else None)
-        assert adapted.output.output_path.endswith(".mp4")
-
-
 def test_stable_audio_seconds_override_produces_wav(tmp_path: Path) -> None:
     config = _load("openai_stable_audio.yaml")
     request = VideoGenerationRequest(prompt="rain on a metal roof", seconds=6)

fastvideo/tests/layers/test_lora_merge_precision.py

@@ -0,0 +1,44 @@
+# SPDX-License-Identifier: Apache-2.0
+"""CPU contract: merging an adapter stored wider than the base rounds once, after a wide add."""
+
+from __future__ import annotations
+
+import pytest
+import torch
+
+from fastvideo.layers.linear import ReplicatedLinear
+from fastvideo.layers.lora import linear as lora_linear
+from fastvideo.layers.lora.linear import get_lora_layer
+
+IN, OUT, RANK = 32, 24, 16
+
+
+@pytest.fixture(autouse=True)
+def _cpu_merge(monkeypatch: pytest.MonkeyPatch):
+    monkeypatch.setattr(torch._dynamo.config, "disable", True)
+    monkeypatch.setattr(lora_linear, "get_local_torch_device", lambda: torch.device("cpu"))
+
+
+def _merged(adapter_dtype: torch.dtype) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
+    torch.manual_seed(0)
+    layer = ReplicatedLinear(IN, OUT, bias=False, params_dtype=torch.bfloat16)
+    with torch.no_grad():
+        layer.weight.copy_(torch.randn(OUT, IN))
+    base = layer.weight.detach().clone()
+    wrapped = get_lora_layer(layer)
+    a, b = torch.randn(RANK, IN, dtype=adapter_dtype), torch.randn(OUT, RANK, dtype=adapter_dtype) * 0.1
+    wrapped.set_lora_weights(a, b, lora_alpha=RANK, strength=1.0)
+    wrapped.merge_lora_weights()
+    return wrapped.base_layer.weight.detach(), base, a, b
+
+
+def test_fp32_adapter_is_added_in_fp32_and_rounded_once() -> None:
+    merged, base, a, b = _merged(torch.float32)
+    expected = (base.float() + b @ a).to(torch.bfloat16)
+    assert merged.dtype == torch.bfloat16
+    assert torch.equal(merged, expected)
+
+
+def test_bf16_adapter_keeps_the_bf16_merge() -> None:
+    merged, base, a, b = _merged(torch.bfloat16)
+    assert torch.equal(merged, base + b @ a)

fastvideo/tests/layers/test_lora_side_branch.py

@@ -54,6 +54,89 @@ def _packed_linear(seed: int = 0) -> ReplicatedLinear:
     return layer
 
 
+def test_a_wrapped_layer_without_an_adapter_runs_the_base_layer() -> None:
+    """A dense wrapper with no adapter is the base layer.
+
+    The pipeline wraps every linear when the first runtime adapter is set. A render that then runs without one, or a
+    load that failed part-way, must still produce the base model's output instead of failing on the missing factors.
+    """
+    base = _dense_linear()
+    wrapped = get_lora_layer(base)
+    assert not wrapped.side_branch and wrapped.lora_A is None
+    x = torch.randn(2, IN, dtype=torch.bfloat16)
+    with torch.no_grad():
+        assert torch.equal(wrapped(x)[0], base(x)[0])
+
+
+@pytest.mark.parametrize("packed", [False, True])
+def test_wrapped_layer_at_strength_zero_is_bitwise_the_base_layer(packed: bool) -> None:
+    base = _packed_linear() if packed else _dense_linear()
+    x = _x()
+    with torch.no_grad():
+        # Taken before wrapping, so a merge that wrongly rewrites the shared base weight cannot also change it.
+        expected = base(x)[0].clone()
+    layer = get_lora_layer(base)
+    layer.set_lora_weights(*_factors(), lora_alpha=8, strength=0.0)
+    assert layer.lora_A is not None and layer.lora_strength == 0.0
+    with torch.no_grad():
+        assert torch.equal(layer(x)[0], expected)
+
+
+@pytest.mark.parametrize("backend", ["eager", "inductor"])
+@pytest.mark.parametrize("adapter", [False, True])
+def test_wrapping_does_not_change_a_compiled_block(monkeypatch: pytest.MonkeyPatch, backend: str,
+                                                   adapter: bool) -> None:
+    """Under a caller's compile the base layer is traced with the block, as it is unwrapped; the wrapper adds no
+    region of its own, so the compiled block is unchanged with no adapter or at strength 0."""
+    if backend == "inductor" and not torch.cuda.is_available():
+        pytest.skip("Inductor execution requires a CUDA GPU for this regression")
+    device = torch.device("cuda" if backend == "inductor" else "cpu")
+    monkeypatch.setattr(torch._dynamo.config, "disable", False)
+    torch._dynamo.reset()
+
+    class Block(torch.nn.Module):
+
+        def __init__(self, proj: torch.nn.Module) -> None:
+            super().__init__()
+            self.proj = proj
+
+        def forward(self, x: torch.Tensor) -> torch.Tensor:
+            return self.proj(x)[0] + x[..., :OUT]
+
+    x = _x().to(device)
+    try:
+        with torch.no_grad():
+            expected = torch.compile(Block(_packed_linear().to(device)), backend=backend, fullgraph=True)(x)
+            torch._dynamo.reset()
+            layer = get_lora_layer(_packed_linear().to(device))
+            if adapter:
+                layer.set_lora_weights(*(factor.to(device) for factor in _factors()), lora_alpha=8, strength=0.0)
+            actual = torch.compile(Block(layer), backend=backend, fullgraph=True)(x)
+        assert torch.equal(actual, expected)
+    finally:
+        torch._dynamo.reset()
+
+
+def test_wrapper_adds_no_compiled_region_around_the_base_layer(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Compiling a packed base layer on its own changes its numerics, so an uncompiled caller runs it eagerly."""
+    monkeypatch.setattr(torch._dynamo.config, "disable", False)
+    torch._dynamo.reset()
+    base = _packed_linear()
+    layer = get_lora_layer(base)
+    layer.set_lora_weights(*_factors(), lora_alpha=8, strength=0.5)
+    seen: list[bool] = []
+    original = base.forward
+
+    def spy(x: torch.Tensor):
+        seen.append(torch.compiler.is_compiling())
+        return original(x)
+
+    monkeypatch.setattr(base, "forward", spy)
+    with torch.no_grad():
+        layer(_x())
+    assert seen == [False]
+
+
 def _factors(seed: int = 1) -> tuple[torch.Tensor, torch.Tensor]:
     generator = torch.Generator().manual_seed(seed)
     return torch.randn(RANK, IN, generator=generator), torch.randn(OUT, RANK, generator=generator)

fastvideo/tests/pipelines/test_open_h3_regenerate_2k.py

@@ -0,0 +1,300 @@
+# SPDX-License-Identifier: Apache-2.0
+"""CPU unit tests for the Open-H3-Regenerate-2K building blocks."""
+
+import numpy as np
+import pytest
+import torch
+from safetensors.torch import save_file
+
+from fastvideo.models.upsamplers.minimax_h3_latent_upscaler import (
+    MiniMaxH3LatentUpscaler,
+    resolve_upscale_geometry,
+    upscale_latent,
+)
+from fastvideo.pipelines.basic.open_h3_regenerate_2k import draft_video
+from fastvideo.pipelines.basic.open_h3_regenerate_2k.sampling import (
+    mix_initial_state,
+    refine_sigmas,
+    shifted_grid,
+)
+from fastvideo.pipelines.basic.open_h3_regenerate_2k.sharp_curve import (
+    NUM_CURVE_BRANCHES,
+    SharpCurveBranches,
+    interpolate_curve,
+)
+
+
+def test_default_sigmas_match_the_released_grid():
+    assert refine_sigmas(3) == pytest.approx([0.909375, 0.857143, 0.666667, 0.0], abs=1e-6)
+
+
+def test_two_step_sigmas_start_at_0_7():
+    assert refine_sigmas(2) == pytest.approx([0.7, 0.494118, 0.0], abs=1e-6)
+
+
+def test_shifted_grid_from_the_released_start_is_the_original_four_step_grid():
+    assert shifted_grid(0.909375, 4) == pytest.approx([0.909375, 0.841446, 0.732075, 0.526697, 0.0], abs=1e-6)
+
+
+def test_custom_start_sigma_is_decreasing_and_ends_at_zero():
+    grid = refine_sigmas(3, start_sigma=0.8)
+    assert grid[0] == 0.8 and grid[-1] == 0.0
+    assert all(a > b for a, b in zip(grid, grid[1:]))
+
+
+@pytest.mark.parametrize("steps", [0, 4])
+def test_unsupported_step_count_is_rejected(steps):
+    with pytest.raises(ValueError):
+        refine_sigmas(steps)
+
+
+def test_initial_state_mix_weights_noise_by_sigma():
+    upscaled, noise = torch.ones(2, 3), torch.zeros(2, 3)
+    assert torch.allclose(mix_initial_state(upscaled, noise, 0.909375), torch.full((2, 3), 0.090625))
+    assert torch.equal(mix_initial_state(upscaled, noise, 0.0), upscaled)
+
+
+def test_upscale_geometry_targets_2k_from_768p_and_576p():
+    assert resolve_upscale_geometry((48, 84), (1440, 2560))[0] == (90, 160)
+    shape, scale = resolve_upscale_geometry((36, 64), (1440, 2560))
+    assert shape == (90, 160) and scale == pytest.approx(2.5)
+
+
+def test_tiny_upscaler_forward_shape_and_identity_passthrough():
+    model = MiniMaxH3LatentUpscaler(channels=32, in_blocks=2, out_blocks=2).eval()
+    latent = torch.randn(24, 5, 4, 6)
+    out = upscale_latent(model, latent, (96, 160), dtype=torch.float32)
+    assert out.shape == (24, 5, 6, 10) and out.dtype == torch.float32
+    assert torch.equal(upscale_latent(model, latent, (64, 96)), latent)
+
+
+def test_upscaler_checkpoint_roundtrip(tmp_path):
+    model = MiniMaxH3LatentUpscaler(channels=32, in_blocks=3, out_blocks=2)
+    path = str(tmp_path / "upscaler.safetensors")
+    save_file(model.state_dict(), path)
+    loaded = MiniMaxH3LatentUpscaler.from_checkpoint(path)
+    assert len(loaded.in_blocks) == len(model.in_blocks) and len(loaded.out_blocks) == len(model.out_blocks)
+    for key, value in model.state_dict().items():
+        assert torch.equal(loaded.state_dict()[key], value)
+
+
+def test_curve_interpolation_hits_table_rows_and_midpoints():
+    table = torch.arange(1025 * 8, dtype=torch.float32).reshape(1025, 8)
+    coords = interpolate_curve(table, torch.tensor([0.0, 1.0, 0.5, 1 / 2048]))
+    assert torch.equal(coords[0], table[0]) and torch.equal(coords[1], table[1024])
+    assert torch.equal(coords[2], table[512])
+    assert torch.allclose(coords[3], (table[0] + table[1]) / 2)
+    with pytest.raises(ValueError):
+        interpolate_curve(table, torch.tensor([1.5]))
+
+
+class _Projection(torch.nn.Module):
+
+    def __init__(self, out_features):
+        super().__init__()
+        self.linear = torch.nn.Linear(4, out_features)
+
+    def forward(self, x):
+        return self.linear(x), None
+
+
+def _write_curve_release(root, out_features, count):
+    (root / "lora").mkdir()
+    (root / "lora" / "adaln_t_table.bin").write_bytes(np.random.rand(1025 * 8).astype(np.float32).tobytes())
+    tensors = {}
+    for index in range(count):
+        tensors[f"transformer.blocks.{index}.proj.curve_A.weight"] = torch.randn(3, 8)
+        tensors[f"transformer.blocks.{index}.proj.curve_B.weight"] = torch.randn(out_features, 3)
+    save_file(tensors, str(root / "lora" / "sharp_curve_branches.safetensors"))
+
+
+def test_sharp_branches_require_all_fifty(tmp_path):
+    _write_curve_release(tmp_path, 6, 3)
+    with pytest.raises(ValueError, match="50"):
+        SharpCurveBranches(tmp_path)
+
+
+def test_sharp_branch_adds_curve_term_to_projection_output(tmp_path):
+    _write_curve_release(tmp_path, 6, NUM_CURVE_BRANCHES)
+    curve = SharpCurveBranches(tmp_path)
+    model = torch.nn.Module()
+    model.blocks = torch.nn.ModuleList()
+    for _ in range(NUM_CURVE_BRANCHES):
+        block = torch.nn.Module()
+        block.proj = _Projection(6)
+        model.blocks.append(block)
+    curve.attach(model)
+    curve.attach(model)  # idempotent: a second attach must not double the term
+    timesteps = torch.tensor([0.1, 0.9])
+    x = torch.randn(2, 4)
+    base, _ = model.blocks[0].proj.linear(x), None
+    curve.set_timesteps(timesteps)
+    out, extra = model.blocks[0].proj(x)
+    a, b = curve.factors["blocks.0.proj"]
+    expected = base + torch.nn.functional.linear(torch.nn.functional.linear(curve.coords, a), b)
+    assert extra is None and torch.allclose(out, expected, atol=1e-6)
+
+
+def test_sharp_branch_without_timesteps_fails_loudly(tmp_path):
+    _write_curve_release(tmp_path, 6, NUM_CURVE_BRANCHES)
+    curve = SharpCurveBranches(tmp_path)
+    model = torch.nn.Module()
+    model.blocks = torch.nn.ModuleList()
+    for _ in range(NUM_CURVE_BRANCHES):
+        block = torch.nn.Module()
+        block.proj = _Projection(6)
+        model.blocks.append(block)
+    curve.attach(model)
+    with pytest.raises(RuntimeError, match="timesteps"):
+        model.blocks[0].proj(torch.randn(2, 4))
+
+
+def test_ffmpeg_command_cover_scales_and_caps_at_121_frames():
+    command = draft_video.ffmpeg_decode_command("clip.mp4", 1344, 768)
+    video_filter = command[command.index("-vf") + 1]
+    assert "fps=24" in video_filter and "scale=1344:768:force_original_aspect_ratio=increase" in video_filter
+    assert "crop=1344:768" in video_filter
+    assert command[command.index("-frames:v") + 1] == "121"
+
+
+def test_draft_pixels_pad_the_last_frame_to_124(monkeypatch):
+    width, height = draft_video.DRAFT_SIZES["576p"]
+    frames = np.arange(121, dtype=np.uint8).reshape(121, 1, 1, 1) * np.ones((121, height, width, 3), np.uint8)
+
+    class _Done:
+        stdout = frames.tobytes()
+
+    monkeypatch.setattr(draft_video.subprocess, "run", lambda *a, **k: _Done())
+    pixels = draft_video.load_draft_pixels("clip.mp4", "576p")
+    assert pixels.shape == (1, 3, 124, height, width) and pixels.dtype == torch.uint8
+    assert torch.equal(pixels[:, :, 120], pixels[:, :, 123])
+
+
+def test_draft_pixels_reject_truncated_decode(monkeypatch):
+
+    class _Short:
+        stdout = b"\x00" * 10
+
+    monkeypatch.setattr(draft_video.subprocess, "run", lambda *a, **k: _Short())
+    with pytest.raises(ValueError, match="decoded to 10 bytes"):
+        draft_video.load_draft_pixels("clip.mp4")
+
+
+class _ConstantVelocityTransformer:
+    """Predicts a constant video velocity and records what the stage feeds it."""
+
+    def __init__(self, velocity):
+        self.velocity = velocity
+        self.calls = []
+
+    def __call__(self, **kwargs):
+        self.calls.append(kwargs)
+        rows = kwargs["hidden_states"].shape[1]
+        audio_rows = kwargs["audio_hidden_states"].shape[1]
+        return torch.full((1, rows, 8), self.velocity), torch.zeros(1, audio_rows, 4)
+
+
+class _RecordingCurve:
+
+    def __init__(self):
+        self.timesteps = []
+        self.attached = 0
+
+    def attach(self, transformer):
+        self.attached += 1
+
+    def set_timesteps(self, unique):
+        self.timesteps.append(unique.clone())
+
+
+def _denoise(monkeypatch, steps, velocity=0.5):
+    from fastvideo.pipelines.basic.minimax_h3.packing import build_packed_sequence
+    from fastvideo.pipelines.basic.minimax_h3.stages.minimax_h3_latent_preparation import MINIMAX_H3_LAYOUT_KEY
+    from fastvideo.pipelines.basic.open_h3_regenerate_2k.stages import denoising
+    from fastvideo.pipelines.pipeline_batch_info import ForwardBatch
+
+    monkeypatch.setattr(denoising, "get_local_torch_device", lambda: torch.device("cpu"))
+    layout = build_packed_sequence(torch.ones(3, dtype=torch.long), 2, 2, 2, 2, (1, 2, 2))
+    batch = ForwardBatch(data_type="video")
+    batch.num_inference_steps = steps
+    batch.prompt_embeds = [torch.zeros(1, 3, 4)]
+    batch.latents = torch.zeros(layout.video_indices.numel(), 8)
+    batch.audio_latents = torch.ones(layout.audio_indices.numel(), 4)
+    batch.extra[MINIMAX_H3_LAYOUT_KEY] = layout
+    transformer, curve = _ConstantVelocityTransformer(velocity), _RecordingCurve()
+    stage = denoising.Regenerate2KDenoisingStage(transformer, curve, None)
+    return stage.forward(batch, None), transformer, curve
+
+
+def test_euler_steps_integrate_the_constant_velocity_over_sigma_zero_to_start(monkeypatch):
+    batch, transformer, curve = _denoise(monkeypatch, steps=3, velocity=0.5)
+    assert len(transformer.calls) == 3 and curve.attached == 1
+    assert torch.allclose(batch.latents, torch.full_like(batch.latents, 0.909375 * 0.5))
+
+
+def test_video_clock_is_one_minus_sigma_and_audio_is_held_clean(monkeypatch):
+    _, transformer, curve = _denoise(monkeypatch, steps=3)
+    for call, unique, sigma in zip(transformer.calls, curve.timesteps, [0.909375, 6 / 7, 2 / 3]):
+        assert torch.allclose(unique.sort().values, torch.tensor([1.0 - sigma, 1.0]).sort().values.float(), atol=1e-6)
+        assert call["hidden_states"].dtype == torch.bfloat16 and call["audio_hidden_states"].dtype == torch.bfloat16
+        assert torch.equal(call["audio_hidden_states"].float(), torch.ones_like(call["audio_hidden_states"]).float())
+
+
+def test_two_step_request_runs_two_forwards(monkeypatch):
+    batch, transformer, _ = _denoise(monkeypatch, steps=2, velocity=1.0)
+    assert len(transformer.calls) == 2
+    assert torch.allclose(batch.latents, torch.full_like(batch.latents, 0.7), atol=1e-6)
+
+
+def test_registry_resolves_the_release_to_the_regenerate_pipeline(tmp_path):
+    from fastvideo.configs.pipelines.open_h3_regenerate_2k import OpenH3Regenerate2KPipelineConfig
+    from fastvideo.registry import get_model_info, get_pipeline_config_cls_from_name, get_preset_selection
+
+    hub_id = "Efficient-Large-Model/Open-H3-Regenerate-2K"
+    (tmp_path / "lora").mkdir()
+    (tmp_path / "lora" / "open_h3_regenerate_2k_lora.safetensors").write_bytes(b"")
+    for path in (hub_id, str(tmp_path)):
+        assert get_pipeline_config_cls_from_name(path) is OpenH3Regenerate2KPipelineConfig
+        assert get_model_info(path).pipeline_cls.__name__ == "OpenH3Regenerate2KPipeline"
+    preset, family = get_preset_selection(hub_id)
+    assert (preset, family) == ("open_h3_regenerate_2k", "open_h3_regenerate_2k")
+
+
+def test_serve_config_forwards_the_nvfp4_decoder_option(monkeypatch):
+    """`generator.pipeline.experimental.h3_vae_nvfp4_decoder` reaches FastVideoArgs through the serve path."""
+    from fastvideo import fastvideo_args as fva
+    from fastvideo.api import ComponentConfig, EngineConfig, GeneratorConfig, PipelineSelection
+    from fastvideo.api.compat import generator_config_to_fastvideo_args
+
+    captured = {}
+
+    class _Captured:
+
+        def __init__(self, **kwargs):
+            self.kwargs = kwargs
+
+    def _capture(**kwargs):
+        captured.update(kwargs)
+        return _Captured(**kwargs)
+
+    monkeypatch.setattr(fva.FastVideoArgs, "from_kwargs", _capture)
+    generator_config_to_fastvideo_args(
+        GeneratorConfig(model_path="Efficient-Large-Model/Open-H3-Regenerate-2K",
+                        engine=EngineConfig(num_gpus=1),
+                        pipeline=PipelineSelection(components=ComponentConfig(),
+                                                   experimental={
+                                                       "draft_size": "768p",
+                                                       "h3_vae_nvfp4_decoder": "/tmp/decoder-nvfp4.safetensors",
+                                                   })))
+    assert captured["h3_vae_nvfp4_decoder"] == "/tmp/decoder-nvfp4.safetensors"
+    assert captured["draft_size"] == "768p"
+
+
+def test_cli_flag_sets_the_nvfp4_decoder_option():
+    from fastvideo.fastvideo_args import FastVideoArgs
+    from fastvideo.utils import FlexibleArgumentParser
+
+    args = FastVideoArgs.add_cli_args(FlexibleArgumentParser()).parse_args(
+        ["--h3-vae-nvfp4-decoder", "/tmp/decoder-nvfp4.safetensors"])
+    assert args.h3_vae_nvfp4_decoder == "/tmp/decoder-nvfp4.safetensors"
+    assert FastVideoArgs.add_cli_args(FlexibleArgumentParser()).parse_args([]).h3_vae_nvfp4_decoder is None

fastvideo/tests/stages/test_input_validation.py

@@ -0,0 +1,94 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Weight-free regression tests for negative conditioning validation."""
+
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+from fastvideo.pipelines.pipeline_batch_info import ForwardBatch
+from fastvideo.pipelines.stages.input_validation import InputValidationStage
+from fastvideo.pipelines.stages.text_encoding import TextEncodingStage
+
+
+@pytest.fixture(autouse=True)
+def disable_gpu_stage_timing(monkeypatch):
+    monkeypatch.setattr(
+        "fastvideo.pipelines.stages.base.envs.FASTVIDEO_STAGE_LOGGING",
+        SimpleNamespace(get=lambda: False),
+    )
+
+
+def _batch(**overrides):
+    kwargs = dict(
+        data_type="video",
+        prompt="a red fox",
+        negative_prompt=None,
+        height=16,
+        width=16,
+        num_frames=1,
+        seed=0,
+        num_inference_steps=2,
+        guidance_scale=5.0,
+    )
+    kwargs.update(overrides)
+    return ForwardBatch(**kwargs)
+
+
+def _args(verify):
+    return SimpleNamespace(
+        enable_stage_verification=verify,
+        pipeline_config=SimpleNamespace(ti2v_task=False, is_causal=False),
+    )
+
+
+@pytest.mark.parametrize("verify", [False, True])
+@pytest.mark.parametrize("negative_prompt_embeds", [None, []])
+def test_cfg_rejects_missing_negative_conditioning(verify, negative_prompt_embeds):
+    batch = _batch(negative_prompt_embeds=negative_prompt_embeds)
+    assert batch.do_classifier_free_guidance
+    assert batch.negative_prompt_embeds == []
+    with pytest.raises(ValueError, match="For classifier-free guidance, either `negative_prompt`"):
+        InputValidationStage()(batch, _args(verify))
+
+
+@pytest.mark.parametrize("verify", [False, True])
+def test_cfg_accepts_empty_negative_prompt(verify):
+    batch = _batch(negative_prompt="")
+    assert InputValidationStage()(batch, _args(verify)) is batch
+    assert batch.negative_prompt == ""
+
+
+@pytest.mark.parametrize("verify", [False, True])
+def test_cfg_accepts_precomputed_negative_embeddings(verify):
+    embeddings = [torch.ones(1, 2, 3)]
+    batch = _batch(negative_prompt_embeds=embeddings)
+    assert InputValidationStage()(batch, _args(verify)) is batch
+    assert batch.negative_prompt_embeds is embeddings
+    assert torch.equal(batch.negative_prompt_embeds[0], torch.ones(1, 2, 3))
+
+
+@pytest.mark.parametrize("verify", [False, True])
+@pytest.mark.parametrize("negative_prompt_embeds", [None, []])
+def test_without_cfg_does_not_require_negative_conditioning(verify, negative_prompt_embeds):
+    batch = _batch(guidance_scale=1.0, negative_prompt_embeds=negative_prompt_embeds)
+    assert not batch.do_classifier_free_guidance
+    assert InputValidationStage()(batch, _args(verify)) is batch
+
+
+def test_empty_negative_prompt_reaches_text_encoding(monkeypatch):
+    batch = _batch(negative_prompt="")
+    args = _args(False)
+    args.pipeline_config.text_encoder_configs = [SimpleNamespace()]
+    encoder = TextEncodingStage([object()], [object()])
+    prompts = []
+
+    def encode_text(prompt, *args, **kwargs):
+        prompts.append(prompt)
+        return [torch.ones(1, 2, 3)], [torch.ones(1, 2, dtype=torch.long)]
+
+    monkeypatch.setattr(encoder, "encode_text", encode_text)
+    InputValidationStage()(batch, args)
+    assert encoder(batch, args) is batch
+    assert prompts == ["a red fox", ""]
+    assert len(batch.prompt_embeds) == len(batch.negative_prompt_embeds) == 1

fastvideo/tests/stages/test_pan2_first_latent_cfg.py

@@ -1,57 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""PAN2 image-to-video keeps the first latent frame out of classifier-free guidance; no weights or CUDA needed."""
-
-import pytest
-import torch
-
-from fastvideo.tests.stages._denoising_fixtures import (
-    NullProgressBar, TinyDenoiser, TinyScheduler, _patch_denoising_module, _tiny_args, _tiny_batch,
-)
-
-
-def _one_step(monkeypatch, env_overrides, *, image_to_video: bool, skip_first_latent: bool, guidance_rescale=0.0,
-              frame_values=(0.0, 0.0, 0.0)):
-    _patch_denoising_module(monkeypatch, env_overrides, None)
-    from fastvideo.pipelines.basic.pan2.stages import PAN2DenoisingStage
-
-    stage = PAN2DenoisingStage(TinyDenoiser(), TinyScheduler())
-    stage.progress_bar = lambda iterable=None, total=None: NullProgressBar()
-    batch = _tiny_batch()
-    batch.latents = torch.tensor(frame_values).reshape(1, 1, 3, 1, 1)
-    batch.raw_latent_shape = (1, 1, 3, 1, 1)
-    batch.timesteps = torch.tensor([0.0])
-    batch.num_inference_steps = 1
-    batch.pil_image = object() if image_to_video else None
-    batch.guidance_rescale = guidance_rescale
-    args = _tiny_args()
-    args.pipeline_config.i2v_skip_first_latent_cfg = skip_first_latent
-    args.pipeline_config.scheduler_step_in_fp32 = True
-    return stage.forward(batch, args).latents.flatten().tolist()
-
-
-def test_image_to_video_takes_the_positive_prediction_for_the_first_latent_frame(monkeypatch, env_overrides):
-    # With a zero latent and timestep the cond prediction is 0.5 and the uncond -0.5, so CFG 2 guides to 1.5.
-    latents = _one_step(monkeypatch, env_overrides, image_to_video=True, skip_first_latent=True)
-
-    assert latents == pytest.approx([-0.005, -0.015, -0.015])
-
-
-@pytest.mark.parametrize("image_to_video,skip_first_latent", [(True, False), (False, True)])
-def test_every_frame_is_guided_without_a_conditioning_image_or_with_the_skip_off(monkeypatch, env_overrides,
-                                                                                image_to_video, skip_first_latent):
-    latents = _one_step(monkeypatch, env_overrides, image_to_video=image_to_video, skip_first_latent=skip_first_latent)
-
-    assert latents == pytest.approx([-0.015, -0.015, -0.015])
-
-
-def test_first_latent_frame_keeps_the_positive_prediction_after_guidance_rescale(monkeypatch, env_overrides):
-    # Frames 0, 1, 2 predict 0.5, 0.7, 0.9 (cond) and 1.5, 1.7, 1.9 (CFG 2), so the rescale factor is 1 and only the
-    # first frame's replacement by 0.5 remains. Replacing before the rescale would scale it by std(cond)/std(mixed).
-    latents = _one_step(monkeypatch,
-                        env_overrides,
-                        image_to_video=True,
-                        skip_first_latent=True,
-                        guidance_rescale=1.0,
-                        frame_values=(0.0, 1.0, 2.0))
-
-    assert latents == pytest.approx([-0.005, 0.983, 1.981])

fastvideo/tests/stages/test_pan2_latent_preparation.py

@@ -1,67 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""PAN2 draws its initial noise in fp32 whatever dtype the prompt embeddings arrive in; no weights or CUDA needed."""
-
-import types
-
-import pytest
-import torch
-
-from fastvideo.pipelines.basic.pan2 import pan2_pipeline
-from fastvideo.pipelines.pipeline_batch_info import ForwardBatch
-from fastvideo.pipelines.stages import LatentPreparationStage, latent_preparation
-
-
-@pytest.fixture()
-def cpu_device(monkeypatch):
-    monkeypatch.setattr(latent_preparation, "get_local_torch_device", lambda: torch.device("cpu"))
-
-
-def _args() -> types.SimpleNamespace:
-    arch = types.SimpleNamespace(temporal_compression_ratio=4, spatial_compression_ratio=16)
-    return types.SimpleNamespace(pipeline_config=types.SimpleNamespace(vae_config=types.SimpleNamespace(arch_config=arch)))
-
-
-def _noise(stage, embeds_dtype: torch.dtype) -> torch.Tensor:
-    batch = ForwardBatch(data_type="video",
-                         prompt="a cat",
-                         prompt_embeds=[torch.zeros(1, 4, 8, dtype=embeds_dtype)],
-                         generator=torch.Generator("cpu").manual_seed(7),
-                         height=32,
-                         width=32,
-                         num_frames=5)
-    return stage.forward(batch, _args()).latents
-
-
-def test_noise_is_fp32_and_independent_of_the_prompt_embedding_dtype(cpu_device):
-    transformer = types.SimpleNamespace(num_channels_latents=3)
-    stage = LatentPreparationStage(scheduler=types.SimpleNamespace(),
-                                   transformer=transformer,
-                                   latent_dtype=torch.float32)
-
-    from_bf16, from_fp32 = _noise(stage, torch.bfloat16), _noise(stage, torch.float32)
-
-    assert from_bf16.dtype == torch.float32
-    assert torch.equal(from_bf16, from_fp32)
-    assert _noise(LatentPreparationStage(types.SimpleNamespace(), transformer), torch.bfloat16).dtype == torch.bfloat16
-
-
-def test_pan2_pipeline_requests_fp32_latents(monkeypatch):
-    seen = {}
-
-    class _Stage:
-
-        def __init__(self, *args, **kwargs):
-            seen.setdefault("kwargs", []).append(kwargs)
-
-    for name in ("PAN2InputValidationStage", "PAN2TextEncodingStage", "PAN2TimestepPreparationStage",
-                 "PAN2ConditioningStage", "PAN2DenoisingStage", "DecodingStage"):
-        monkeypatch.setattr(pan2_pipeline, name, _Stage)
-    monkeypatch.setattr(pan2_pipeline, "LatentPreparationStage", _Stage)
-    pipeline = pan2_pipeline.PAN2Pipeline.__new__(pan2_pipeline.PAN2Pipeline)
-    stages = {}
-    pipeline.add_stage = lambda stage_name, stage: stages.__setitem__(stage_name, stage)
-    pipeline.get_module = lambda name: name
-
-    pipeline.create_pipeline_stages(None)
-
-    assert any(kwargs.get("latent_dtype") is torch.float32 for kwargs in seen["kwargs"])

fastvideo/tests/vaes/test_minimax_h3_nvfp4_decoder.py

@@ -487,19 +487,34 @@ def _write_deploy_checkpoint(path, student_layers: int, base_layers: int, heads:
     return decoder
 
 
-def _load_h3_vae(vae_dir, cpu: bool = True, decoder: str | None = None):
+def _load_h3_vae(vae_dir, cpu: bool = True, decoder: str | None = None, pipeline_config=None):
     from fastvideo.configs.models.vaes.minimax_h3_video import MiniMaxH3VideoVAEConfig
     from fastvideo.configs.pipelines import PipelineConfig
     from fastvideo.fastvideo_args import FastVideoArgs
     from fastvideo.models.loader.component_loader import VAELoader
 
-    args = FastVideoArgs(model_path=str(vae_dir),
-                         pipeline_config=PipelineConfig(vae_config=MiniMaxH3VideoVAEConfig(), vae_precision="fp32"),
-                         h3_vae_nvfp4_decoder=decoder)
+    args = FastVideoArgs(
+        model_path=str(vae_dir),
+        pipeline_config=pipeline_config or PipelineConfig(vae_config=MiniMaxH3VideoVAEConfig(), vae_precision="fp32"),
+        h3_vae_nvfp4_decoder=decoder)
     args.vae_cpu_offload = cpu  # load on CPU
     return VAELoader().load(str(vae_dir), args)
 
 
+def test_open_h3_regenerate_2k_vae_load_applies_the_exported_decoder(tmp_path):
+    """Regenerate-2K loads its ``base_model/vae`` through the same swap, so the option applies there too."""
+    from fastvideo.configs.pipelines.open_h3_regenerate_2k import OpenH3Regenerate2KPipelineConfig
+
+    _write_vae_dir(tmp_path / "vae", decoder_layers=5)
+    _write_deploy_checkpoint(tmp_path / "d3.pt", student_layers=3, base_layers=5)
+    config = OpenH3Regenerate2KPipelineConfig()
+    assert len(_load_h3_vae(tmp_path / "vae", pipeline_config=config).decoder.transformer_blocks) == 5
+    vae = _load_h3_vae(tmp_path / "vae", decoder=str(tmp_path / "d3.pt"), pipeline_config=config)
+    assert len(vae.decoder.transformer_blocks) == 3
+    assert vae.decode_autocast_dtype == torch.bfloat16
+    assert isinstance(vae.decoder.transformer_blocks[1].ff.net[2], NVFP4DecoderLinear)
+
+
 def test_loader_applies_exported_nvfp4_decoder(tmp_path):
     import fastvideo.envs as envs
 

tests/local_tests/encoders/test_pan2_qwen3_5_parity.py

@@ -1,339 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""Qwen3.5 text-encoder parity for PAN2 against Transformers' ``Qwen3_5TextModel``.
-
-Coverage scope: both. The checkpoint test loads the public tiny PAN2 text
-encoder through FastVideo's production ``TextEncoderLoader`` and feeds both
-models the exact PAN2 chat-templated token ids. The random-config tests use a
-larger hybrid stack (GQA, partial RoPE, more value than key heads in the Gated
-DeltaNet, sequences that cross several 64-token delta-rule chunks) and also
-cover bf16 and padded batches.
-
-Everything runs on CPU. Set ``PAN2_TINY_MODEL_DIR`` to a local copy of
-``wuqing157/tiny-pan2-modular-pipe``; otherwise the HF cache is used and the
-checkpoint test skips when it is absent.
-"""
-
-from __future__ import annotations
-
-import copy
-import os
-import socket
-from pathlib import Path
-from types import SimpleNamespace
-from typing import Any
-
-import pytest
-import torch
-from torch.testing import assert_close
-
-from fastvideo.configs.models.encoders.qwen3_5 import Qwen3_5TextConfig
-from fastvideo.forward_context import set_forward_context
-from fastvideo.models.loader.utils import set_default_torch_dtype
-
-PARITY_SCOPE = "both"
-TINY_REPO_ID = "wuqing157/tiny-pan2-modular-pipe"
-PAN2_SYSTEM_MESSAGE = "You are a helpful assistant. Generate the video with the following instructions."
-PAN2_TEXT_ENCODER_LAYERS_TO_SKIP = 2
-PAN2_PROMPTS = (
-    "A red fox crosses fresh snow at sunrise.",
-    "A slow dolly shot glides through a rain-soaked neon market at night. Steam rises from food stalls while "
-    "vendors in bright raincoats shout over the hum of generators. Reflections of pink, teal and amber signs ripple "
-    "across puddles as a cyclist weaves between crowds, her umbrella catching droplets that sparkle in the light. "
-    "In the background, a tram rattles past an old clock tower, its windows glowing warm yellow. The camera rises to "
-    "reveal rooftops crowded with antennas and laundry lines, then tilts toward a thunderstorm rolling in over the "
-    "harbor, lightning briefly silhouetting cranes and container ships. Cinematic lighting, shallow depth of field, "
-    "35mm film grain, realistic motion blur.",
-)
-FP32_TOLERANCE = {"atol": 1e-5, "rtol": 1e-5}
-BF16_TOLERANCE = {"atol": 2e-2, "rtol": 2e-2}
-
-
-def _hf_qwen3_5() -> tuple[Any, Any]:
-    module = pytest.importorskip("transformers.models.qwen3_5", reason="transformers without Qwen3.5 support")
-    return module.Qwen3_5TextConfig, module.Qwen3_5TextModel
-
-
-def _tiny_model_dir() -> Path | None:
-    local = os.getenv("PAN2_TINY_MODEL_DIR")
-    if local:
-        return Path(local)
-    try:
-        from huggingface_hub import snapshot_download
-
-        return Path(snapshot_download(TINY_REPO_ID, local_files_only=True))
-    except Exception:
-        return None
-
-
-def _free_port() -> int:
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
-        sock.bind(("127.0.0.1", 0))
-        return sock.getsockname()[1]
-
-
-@pytest.fixture(scope="module")
-def cpu_single_rank():
-    """Single-rank CPU runtime; the loader would otherwise target ``mps`` on CPU-only Linux hosts."""
-    import fastvideo.distributed.parallel_state as parallel_state
-    import fastvideo.models.loader.component_loader as component_loader
-    from fastvideo.distributed import cleanup_dist_env_and_memory, maybe_init_distributed_environment_and_model_parallel
-
-    with pytest.MonkeyPatch.context() as patch:
-        patch.setattr(parallel_state, "get_local_torch_device", lambda: torch.device("cpu"))
-        patch.setattr(component_loader, "get_local_torch_device", lambda: torch.device("cpu"))
-        for name, value in (("MASTER_ADDR", "127.0.0.1"), ("MASTER_PORT", str(_free_port())), ("RANK", "0"),
-                            ("WORLD_SIZE", "1"), ("LOCAL_RANK", "0")):
-            patch.setenv(name, value)
-        maybe_init_distributed_environment_and_model_parallel(1, 1)
-        yield
-        cleanup_dist_env_and_memory()
-
-
-def _loader_args(config: Qwen3_5TextConfig, precision: str = "fp32") -> SimpleNamespace:
-    return SimpleNamespace(
-        pipeline_config=SimpleNamespace(text_encoder_configs=(config, ), text_encoder_precisions=(precision, )),
-        text_encoder_cpu_offload=False,
-        override_text_encoder_quant=None,
-        override_text_encoder_safetensors=None,
-        pin_cpu_memory=False,
-        disable_offload_on_unified_memory=lambda device_id, offload_flag=None: False,
-    )
-
-
-def _pan2_input_ids(tokenizer: Any, prompt: str) -> tuple[torch.Tensor, int]:
-    """Tokenize exactly like PAN2's ``encode_prompt`` and return ids plus the user-header crop."""
-    messages = [{"role": "system", "content": PAN2_SYSTEM_MESSAGE}, {"role": "user", "content": prompt or " "}]
-    input_ids = tokenizer.apply_chat_template(messages, add_generation_prompt=True, tokenize=True,
-                                              return_dict=True).input_ids
-    header_ids = tokenizer("<|im_start|>user\n", add_special_tokens=False).input_ids
-    crop_start = next(
-        (i + len(header_ids) for i in range(len(input_ids)) if input_ids[i:i + len(header_ids)] == header_ids),
-        None,
-    )
-    assert crop_start is not None, "PAN2 chat template has no user header"
-    return torch.tensor([input_ids]), crop_start
-
-
-def _run_both(reference: torch.nn.Module, native: torch.nn.Module, **inputs: Any) -> tuple[Any, Any]:
-    with torch.inference_mode():
-        expected = reference(**inputs, output_hidden_states=True)
-        with set_forward_context(current_timestep=0, attn_metadata=None):
-            actual = native(**inputs, output_hidden_states=True)
-    return expected, actual
-
-
-def _assert_hidden_states_close(
-    expected: tuple[torch.Tensor, ...],
-    actual: tuple[torch.Tensor, ...] | None,
-    label: str,
-    tolerance: dict[str, float],
-    valid: torch.Tensor | None = None,
-) -> float:
-    assert actual is not None, f"{label}: native encoder returned no hidden_states"
-    assert len(actual) == len(expected), f"{label}: {len(actual)} hidden states, reference has {len(expected)}"
-    worst = 0.0
-    for index, (reference_state, native_state) in enumerate(zip(expected, actual, strict=True)):
-        assert native_state.shape == reference_state.shape, f"{label} hidden_states[{index}] shape"
-        assert torch.isfinite(native_state).all(), f"{label} hidden_states[{index}] has non-finite values"
-        if valid is not None:
-            reference_state = reference_state[valid]
-            native_state = native_state[valid]
-        drift = (native_state.float() - reference_state.float()).abs().max().item()
-        worst = max(worst, drift)
-        print(f"{label} hidden_states[{index}]: max_abs={drift:.3e}", flush=True)
-        assert_close(native_state, reference_state, **tolerance, msg=lambda m: f"{label} hidden_states[{index}]: {m}")
-    return worst
-
-
-def test_pan2_qwen3_5_arch_config_matches_transformers_rope_and_layer_types() -> None:
-    """Config.json fields resolve to the same RoPE settings and layer schedule as Transformers."""
-    hf_config_cls, _ = _hf_qwen3_5()
-    cases = (
-        {},
-        {
-            "rope_theta": 1_000_000.0
-        },
-        {
-            "partial_rotary_factor": 0.5,
-            "rope_parameters": {
-                "rope_type": "default",
-                "rope_theta": 5_000_000.0
-            }
-        },
-        {
-            "partial_rotary_factor": 0.5,
-            "rope_parameters": {
-                "rope_type": "default",
-                "rope_theta": 5_000_000.0,
-                "partial_rotary_factor": 0.25
-            }
-        },
-        {
-            "num_hidden_layers": 6,
-            "full_attention_interval": 3
-        },
-        {
-            "num_hidden_layers": 3,
-            "layer_types": ["mamba", "attention", "linear_attention"]
-        },
-    )
-    for case in cases:
-        reference = hf_config_cls.from_dict(dict(case))
-        config = Qwen3_5TextConfig()
-        config.update_model_arch(dict(case))
-        rope = config.arch_config.resolved_rope_parameters()
-        for key in ("rope_type", "rope_theta", "partial_rotary_factor"):
-            assert rope[key] == reference.rope_parameters[key], f"{case}: {key}"
-        assert config.arch_config.resolved_layer_types() == list(reference.layer_types), case
-
-
-def test_pan2_qwen3_5_tiny_checkpoint_loader_parity(cpu_single_rank) -> None:
-    """Production-loader encoder matches every HF hidden state on real PAN2 prompts (fp32 CPU)."""
-    _, hf_model_cls = _hf_qwen3_5()
-    model_dir = _tiny_model_dir()
-    if model_dir is None or not (model_dir / "text_encoder" / "config.json").is_file():
-        pytest.skip(f"{TINY_REPO_ID} is not available locally; set PAN2_TINY_MODEL_DIR")
-    if not (model_dir / "tokenizer").is_dir():
-        pytest.skip(f"{TINY_REPO_ID} tokenizer is not available locally")
-    from transformers import AutoTokenizer
-
-    from fastvideo.models.encoders.qwen3_5 import Qwen3_5TextModel
-    from fastvideo.models.loader.component_loader import TextEncoderLoader
-
-    text_encoder_dir = model_dir / "text_encoder"
-    tokenizer = AutoTokenizer.from_pretrained(model_dir / "tokenizer", local_files_only=True)
-    reference, loading_info = hf_model_cls.from_pretrained(
-        text_encoder_dir,
-        local_files_only=True,
-        dtype=torch.float32,
-        output_loading_info=True,
-    )
-    load_errors = {
-        name: loading_info.get(name)
-        for name in ("missing_keys", "unexpected_keys", "mismatched_keys", "error_msgs") if loading_info.get(name)
-    }
-    assert not load_errors, f"Reference Qwen3.5 checkpoint did not load strictly: {load_errors}"
-    reference = reference.eval()
-
-    native = TextEncoderLoader().load(str(text_encoder_dir), _loader_args(Qwen3_5TextConfig()))
-    assert isinstance(native, Qwen3_5TextModel)
-    assert native.layer_types == list(reference.config.layer_types)
-    assert next(native.parameters()).dtype == torch.float32
-    assert_close(native.rotary_emb.inv_freq, reference.rotary_emb.inv_freq.float(), atol=0.0, rtol=0.0)
-
-    for prompt_index, prompt in enumerate(PAN2_PROMPTS):
-        input_ids, crop_start = _pan2_input_ids(tokenizer, prompt)
-        label = f"tiny prompt{prompt_index} (L={input_ids.shape[1]})"
-        expected, actual = _run_both(reference, native, input_ids=input_ids)
-        worst = _assert_hidden_states_close(expected.hidden_states, actual.hidden_states, label, FP32_TOLERANCE)
-        assert_close(actual.last_hidden_state, expected.last_hidden_state, **FP32_TOLERANCE)
-
-        skip = PAN2_TEXT_ENCODER_LAYERS_TO_SKIP + 1
-        reference_embeds = expected.hidden_states[-skip][:, crop_start:]
-        native_embeds = actual.hidden_states[-skip][:, crop_start:]
-        assert_close(native_embeds, reference_embeds, **FP32_TOLERANCE)
-        print(
-            f"{label} PAN2 prompt_embeds hidden_states[-{skip}][:, {crop_start}:] "
-            f"max_abs={(native_embeds - reference_embeds).abs().max().item():.3e} all_layers_max_abs={worst:.3e}",
-            flush=True,
-        )
-
-
-@pytest.fixture(scope="module")
-def random_pair(cpu_single_rank):
-    """A randomly initialized 8-layer hybrid reference and the native model loaded from its state dict."""
-    hf_config_cls, hf_model_cls = _hf_qwen3_5()
-    from fastvideo.models.encoders.qwen3_5 import Qwen3_5TextModel
-
-    torch.manual_seed(0)
-    hf_config = hf_config_cls(
-        vocab_size=1024,
-        hidden_size=128,
-        intermediate_size=256,
-        num_hidden_layers=8,
-        num_attention_heads=4,
-        num_key_value_heads=2,
-        head_dim=32,
-        linear_conv_kernel_dim=4,
-        linear_key_head_dim=16,
-        linear_value_head_dim=24,
-        linear_num_key_heads=2,
-        linear_num_value_heads=4,
-        layer_types=[
-            "linear_attention",
-            "linear_attention",
-            "full_attention",
-            "linear_attention",
-            "full_attention",
-            "linear_attention",
-            "linear_attention",
-            "full_attention",
-        ],
-        rope_parameters={
-            "rope_type": "default",
-            "rope_theta": 1_000_000.0,
-            "partial_rotary_factor": 0.25
-        },
-        initializer_range=0.08,
-    )
-    reference = hf_model_cls(hf_config).eval()
-    with torch.no_grad():
-        # Move the zero-centered norms, gates and decay parameters off their
-        # init values so every term contributes.
-        for name, parameter in reference.named_parameters():
-            if name.endswith("norm.weight") or name.endswith(("A_log", "dt_bias")):
-                parameter.add_(0.2 * torch.randn_like(parameter))
-
-    config = Qwen3_5TextConfig()
-    config.update_model_arch(hf_config.to_dict())
-    with set_default_torch_dtype(torch.float32):
-        native = Qwen3_5TextModel(config).eval()
-    loaded = native.load_weights(reference.state_dict().items())
-    expected_names = {name for name, _ in native.named_parameters()}
-    assert loaded == expected_names, f"unloaded native parameters: {sorted(expected_names - loaded)}"
-    assert set(reference.state_dict()) == expected_names
-    return reference, native
-
-
-def test_pan2_qwen3_5_random_config_fp32_parity(random_pair) -> None:
-    """Multi-chunk delta rule (L=301), GQA and partial RoPE match HF in fp32."""
-    reference, native = random_pair
-    generator = torch.Generator().manual_seed(1)
-    input_ids = torch.randint(0, reference.config.vocab_size, (2, 301), generator=generator)
-    expected, actual = _run_both(reference, native, input_ids=input_ids)
-    worst = _assert_hidden_states_close(expected.hidden_states, actual.hidden_states, "random fp32", FP32_TOLERANCE)
-    print(f"random fp32 all_layers_max_abs={worst:.3e}", flush=True)
-    # Each layer must move the residual stream, or the comparison is vacuous.
-    for index in range(1, len(expected.hidden_states) - 1):
-        change = (expected.hidden_states[index] - expected.hidden_states[index - 1]).abs().max().item()
-        assert change > 0.1, f"layer {index - 1} barely changes the hidden state ({change:.3e})"
-
-
-def test_pan2_qwen3_5_random_config_padding_parity(random_pair) -> None:
-    """Right- and left-padded rows match HF on every non-padded position."""
-    reference, native = random_pair
-    generator = torch.Generator().manual_seed(2)
-    input_ids = torch.randint(0, reference.config.vocab_size, (2, 200), generator=generator)
-    attention_mask = torch.ones_like(input_ids)
-    attention_mask[0, 150:] = 0
-    attention_mask[1, :37] = 0
-    expected, actual = _run_both(reference, native, input_ids=input_ids, attention_mask=attention_mask)
-    worst = _assert_hidden_states_close(
-        expected.hidden_states,
-        actual.hidden_states,
-        "random padded",
-        FP32_TOLERANCE,
-        valid=attention_mask.bool(),
-    )
-    print(f"random padded all_layers_max_abs={worst:.3e}", flush=True)
-
-
-def test_pan2_qwen3_5_random_config_bf16_parity(random_pair) -> None:
-    """Half-precision dtype handling (fp32 norms, gates and delta rule) follows HF."""
-    reference, native = (copy.deepcopy(model).to(torch.bfloat16) for model in random_pair)
-    generator = torch.Generator().manual_seed(3)
-    input_ids = torch.randint(0, reference.config.vocab_size, (1, 173), generator=generator)
-    expected, actual = _run_both(reference, native, input_ids=input_ids)
-    assert actual.hidden_states is not None and actual.hidden_states[-1].dtype == torch.bfloat16
-    worst = _assert_hidden_states_close(expected.hidden_states, actual.hidden_states, "random bf16", BF16_TOLERANCE)
-    print(f"random bf16 all_layers_max_abs={worst:.3e}", flush=True)

tests/local_tests/open_h3_regenerate_2k/test_upscaler_parity.py

@@ -0,0 +1,62 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Opt-in parity of the ported latent upscaler against the author's ComfyUI node.
+
+Needs the release weights (OPEN_H3_REGEN_WEIGHTS) and a checkout of NVlabs/Sana on the ``sol-engine`` branch
+(OPEN_H3_REGEN_REF, the models/Open-H3-Regenerate-2K directory). GPU only: the node runs the network in bf16.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import os
+import sys
+import types
+from pathlib import Path
+
+import pytest
+import torch
+
+from fastvideo.models.upsamplers.minimax_h3_latent_upscaler import MiniMaxH3LatentUpscaler, upscale_latent
+
+WEIGHTS = os.environ.get("OPEN_H3_REGEN_WEIGHTS")
+REFERENCE = os.environ.get("OPEN_H3_REGEN_REF")
+CHECKPOINT = "latent_upsampler/minimax_h3_latent_upscaler_3d_conv_v1_bf16.safetensors"
+
+pytestmark = pytest.mark.skipif(not (WEIGHTS and REFERENCE and torch.cuda.is_available()),
+                                reason="needs OPEN_H3_REGEN_WEIGHTS, OPEN_H3_REGEN_REF and a GPU")
+
+
+def _load_author_node(checkpoint: str):
+    folder = types.ModuleType("folder_paths")
+    folder.folder_names_and_paths = {"latent_upscale_models": ([], set())}
+    folder.get_full_path_or_raise = lambda category, name: checkpoint
+    sys.modules["folder_paths"] = folder
+    sys.modules["comfy.model_management"] = None
+    sys.modules["comfy_api.latest"] = None
+    spec = importlib.util.spec_from_file_location("author_upscaler",
+                                                  Path(REFERENCE) / "open_h3" / "upscaler_node.py")
+    node = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(node)
+    node.io.NodeOutput = lambda value: (value, )
+    return node
+
+
+@pytest.mark.parametrize("shape", [(24, 37, 48, 84), (24, 37, 36, 64)])
+def test_upscaler_matches_the_author_node(shape):
+    checkpoint = str(Path(WEIGHTS) / CHECKPOINT)
+    node = _load_author_node(checkpoint)
+    torch.manual_seed(0)
+    latent = torch.randn(shape)
+    mode = {"mode": node.UpscaleMode.TARGET_DIMENSIONS, "width": 2560, "height": 1440}
+    with torch.inference_mode():
+        (expected, ) = node.MinimaxH3LatentUpscaler3D.execute({"samples": latent[None].float()},
+                                                              "upscaler",
+                                                              mode,
+                                                              align=32,
+                                                              enable_temporal_chunking=False,
+                                                              force_unload=False,
+                                                              device="cuda",
+                                                              precision="bf16")
+    model = MiniMaxH3LatentUpscaler.from_checkpoint(checkpoint).to("cuda", torch.bfloat16)
+    actual = upscale_latent(model, latent.cuda(), (1440, 2560)).cpu()
+    assert torch.equal(actual, expected["samples"].float()[0])

tests/local_tests/pan2/PORT_STATUS.md

@@ -1,71 +0,0 @@
-# PAN2 Port Status
-
-## Summary
-- model_family: pan2
-- workload_types: T2V, I2V (one pipeline; I2V when the request carries a first frame)
-- official_ref: vllm-project/vllm-omni#8602 at `e0c4c73158a5e4f44eda8eaa8879f7fd9cedbd74` (the diffusers integration is private)
-- official_ref_dir: none; rebuilt in `tests/local_tests/pan2/reference.py`
-- hf_weights_path: `IFM/PAN2` (not public); tests use `wuqing157/tiny-pan2-modular-pipe@3ff3130b`
-- local_weights_dir: HF cache, or `PAN2_TINY_MODEL_DIR`
-- source_layout: Modular Diffusers (`modular_model_index.json`), diffusers component subfolders
-- local_tests_readme: `tests/local_tests/pan2/README.md`
-
-## Current Phase
-- phase: 11 (post-parity review and handoff)
-- status: in_progress (GPU and real-weight validation pending, see I001 and I002)
-- owner: orchestrator
-- last_updated: 2026-10-20
-
-## Component Matrix
-| Component | Type | Reuse/Port | Official Definition | Official Instantiation | FastVideo Target | Prototype | Conversion | Parity | Open Issues |
-|---|---|---|---|---|---|---|---|---|---|
-| transformer | DiT | port | vLLM-Omni `pan2_transformer.py` (diffusers `PAN2Transformer3DModel`) | `transformer/config.json` | `fastvideo/models/dits/pan2.py`, `fastvideo/configs/models/dits/pan2.py` | done | load-time key mapping (Q/K/V fused) | non_skip_pass | I002 |
-| text_encoder | encoder | port | transformers `Qwen3_5TextModel` | `text_encoder/config.json` | `fastvideo/models/encoders/qwen3_5.py`, `fastvideo/configs/models/encoders/qwen3_5.py` | done | passthrough | non_skip_pass | I002 |
-| tokenizer | tokenizer | reuse | `Qwen2Tokenizer` | `tokenizer/` | AutoTokenizer | n/a | passthrough | covered by pipeline parity | — |
-| vae | VAE | reuse | diffusers `AutoencoderKLWan` (Wan2.2, z_dim 48) | `vae/config.json` | `fastvideo/models/wan/vae.py` | done | passthrough | non_skip_pass | — |
-| scheduler | scheduler | reuse | diffusers `FlowMatchEulerDiscreteScheduler` | `scheduler/scheduler_config.json` (shift 7) | `fastvideo/models/schedulers/scheduling_flow_match_euler_discrete.py` | done | passthrough | non_skip_pass | — |
-
-## Conversion State
-- conversion_script: none; the transformer maps diffusers keys at load time (`PAN2ArchConfig.param_names_mapping`)
-- converted_weights_dir: n/a
-- source_layout: diffusers
-- strict_load_status: tiny checkpoint loads strictly (no missing or unexpected keys)
-- passthrough_components: text_encoder, tokenizer, vae, scheduler
-- retry_history: none
-
-## Parity Commands
-| Scope | Command | Last Result | Notes |
-|---|---|---|---|
-| DiT | `pytest tests/local_tests/transformers/test_pan2_transformer_parity.py` | 13 passed (CPU) | SP=2/4 vs one rank over gloo, T2V/I2V, one/two pad tokens; max abs 7.7e-6 |
-| Text encoder | `pytest tests/local_tests/encoders/test_pan2_qwen3_5_parity.py` | 5 passed (CPU) | exact |
-| VAE | `pytest tests/local_tests/vaes/test_pan2_vae_parity.py` | 9 passed (CPU) | exact |
-| Scheduler | `pytest tests/local_tests/schedulers/test_pan2_scheduler_parity.py` | 9 passed (CPU) | exact |
-| Pipeline | `pytest tests/local_tests/pipelines/test_pan2_pipeline_parity.py` | 3 passed (CPU) | T2V and I2V latents exact |
-
-## Open Questions
-| ID | Question | Owner | Needed By Phase | Status | Resolution |
-|---|---|---|---|---|---|
-| Q001 | Which reference, with the diffusers integration private? | orchestrator | 2 | resolved | vLLM-Omni #8602, rebuilt from diffusers/transformers blocks in `reference.py` |
-| Q002 | Port the `pan2-guardrail` prompt/video guardrails? | orchestrator | 7 | resolved | Not ported: FastVideo has no guardrail stage. |
-| Q003 | Port Cache-DiT `quality` and CFG-parallel? | orchestrator | 7 | resolved | Not ported: vLLM-Omni serving features. Ulysses SP is supported through `DistributedAttention`. |
-
-## Issues And Blockers
-| ID | Phase | Component | Severity | Issue | Evidence | Owner | Status | Resolution |
-|---|---|---|---|---|---|---|---|---|
-| I001 | 9-10 | all | medium | No real-weight parity or SSIM regression | `IFM/PAN2` is not public | orchestrator | open | Add `fastvideo/tests/ssim/test_pan2_similarity.py` once weights are public. |
-| I002 | 9 | transformer, text_encoder | medium | No GPU parity run yet (bf16 + FlashAttention, NCCL SP, 9B encoder speed) | Local parity tests ran on a CPU login node | orchestrator | open | Run the same tests on a GPU node. |
-| I003 | 7 | pipeline | low | The shared denoiser casts latents to the model-output dtype in the scheduler step; PAN2 samples on fp32 latents | pipeline parity fails (8e-2) without the fix | orchestrator | resolved | `DenoisingStage.prepare_scheduler_model_output` hook; `PAN2DenoisingStage` returns `noise_pred.float()`. |
-
-## Escape Hatches
-| ID | Phase | Decision Type | Question | Recommended Option | Status | Resolution |
-|---|---|---|---|---|---|---|
-
-## Decisions
-| Date | Decision | Rationale | Impact |
-|---|---|---|---|
-| 2026-10-15 | Native Qwen3.5 encoder with a PyTorch chunked gated delta rule | No `fla`/`causal_conv1d` dependency; encoder runs once per prompt | Slower than fused kernels for long prompts |
-| 2026-10-15 | Fuse the per-stream Q/K/V projections at load time | One GEMM per stream instead of three | Mapping in `PAN2ArchConfig.param_names_mapping` |
-| 2026-10-15 | Text embeddings kept in fp32 after encoding | The shared latent stage draws noise in the embedding dtype; PAN2 uses fp32 noise and latents | Denoiser casts embeddings to the DiT dtype (`cast_prompt_embeds_to_dit_dtype`) |
-
-## Handoff Notes
-- The remaining work is I001 and I002. Both need hardware or weights, not code changes.

tests/local_tests/pan2/README.md

@@ -1,67 +0,0 @@
-# PAN2 Local Tests
-
-Runbook for validating the PAN2 port (text-to-video and first-frame image-to-video). These tests run manually and are
-not part of the default CI suite.
-
-## Reference
-
-PAN2's diffusers integration (`PAN2Transformer3DModel`, `PAN2ModularPipeline`) is not public yet, and the full
-`IFM/PAN2` weights are not released. The reference is the vLLM-Omni port,
-[vllm-project/vllm-omni#8602](https://github.com/vllm-project/vllm-omni/pull/8602) (head `e0c4c731`), which reports
-parity with the diffusers implementation. `reference.py` rebuilds it from diffusers and transformers building blocks:
-the DiT from `AdaLayerNormZero`, `AdaLayerNormContinuous`, `FeedForward` (gelu-approximate and swiglu), `Timesteps`,
-`TimestepEmbedding` and the diffusers RoPE helpers, using the checkpoint's module names so the checkpoint loads
-strictly. The pipeline is `reference_generate`, a step-by-step copy of the vLLM-Omni `PAN2Pipeline.forward`
-using `Qwen3_5TextModel`, `AutoencoderKLWan` and `FlowMatchEulerDiscreteScheduler`.
-
-The checkpoint is the public random-weight [`wuqing157/tiny-pan2-modular-pipe`](https://huggingface.co/wuqing157/tiny-pan2-modular-pipe)
-(revision `3ff3130b`), the same one vLLM-Omni tests against. It has two DiT blocks, a four-layer Qwen3.5 text model
-(three Gated DeltaNet layers and one full-attention layer) and a small Wan2.2-layout VAE.
-
-## Setup
-
-```bash
-huggingface-cli download wuqing157/tiny-pan2-modular-pipe --revision 3ff3130b4527f61aaab7bde01465c9884dc84cdd
-# or point the tests at a local copy:
-export PAN2_TINY_MODEL_DIR=/path/to/tiny-pan2-modular-pipe
-```
-
-The tests need `diffusers` with `AutoencoderKLWan` and `transformers` with `Qwen3_5TextModel` (5.x). They run on CPU
-when no GPU is present. On a GPU-less host, `import fastvideo` also needs the `fastvideo_kernel` stub described in
-`fastvideo/tests/AGENTS.md`.
-
-## Tests
-
-| Area | Command | Result (CPU, fp32 unless noted) |
-|---|---|---|
-| DiT: tiny checkpoint, strict load + forward | `pytest tests/local_tests/transformers/test_pan2_transformer_parity.py -k tiny_checkpoint` | max abs 6.0e-7 |
-| DiT: random weights, batch 1 and 2 | `pytest tests/local_tests/transformers/test_pan2_transformer_parity.py -k random_weights` | max abs 0 and 1.9e-5 (outputs ~16) |
-| DiT: Ulysses SP=2/4 vs one rank, T2V/I2V with 15/18 tokens | `pytest tests/local_tests/transformers/test_pan2_transformer_parity.py -k sequence_parallel` | max abs 4.8e-6 / 7.7e-6 (gloo on CPU; NCCL on 2/4 GPUs) |
-| Qwen3.5 text encoder | `pytest tests/local_tests/encoders/test_pan2_qwen3_5_parity.py` | all hidden states exact (fp32, bf16, padded batch) |
-| Wan2.2 VAE encode/decode + normalization | `pytest tests/local_tests/vaes/test_pan2_vae_parity.py` | exact; normalization 2.4e-7 |
-| Flow-match scheduler, PAN2 sigmas + shift | `pytest tests/local_tests/schedulers/test_pan2_scheduler_parity.py` | exact |
-| Pipeline, T2V and I2V (bf16 DiT) | `pytest tests/local_tests/pipelines/test_pan2_pipeline_parity.py` | final latents and video exact |
-
-The pipeline test builds the FastVideo pipeline through the registry and component loaders and compares the final
-latents (relative L2 < 2e-2, cosine > 0.9995) and the decoded video (PSNR > 40 dB) with `reference_generate`. To
-confirm the threshold catches real deviations, three injected changes were each run against it: conditioning on
-`hidden_states[-2]`, letting the latents fall to bf16 between steps, and shifting the first-frame crop by one pixel.
-Each moved the latents by about 8e-2 relative L2.
-
-Upstream revision `e0c4c731` always masks video padding, fixing Ulysses=4 drift when padding was left unmasked.
-FastVideo already excludes this padding by passing the original video length to `DistributedAttention`, which trims
-it before RoPE and joint attention, then pads the output for the reverse all-to-all. The distributed test covers one
-pad token (15 tokens at SP=2/4), two pad tokens (18 at SP=4, matching 704x1248x313 modulo 4), and an unpadded
-control (18 at SP=2), with both zero T2V and first-frame I2V conditioning. No masking toggle is required.
-The pipeline math and checkpoint layout are unchanged between upstream revisions `59dd8c9a` and `e0c4c731`.
-
-On 2026-10-20, all 39 PAN2 local tests passed on CPU (Torch 2.12.0, Transformers 5.15.0, Diffusers 0.39.0).
-The broader API/contract suite had 499 passes and two failures also reproduced on clean `main` at `cdda1b753`:
-the schema inventory lacks MiniMax-H3's `audio_scheduler_shift`/`video_scheduler_shift`, and
-`test_load_kandinsky5.py` reads `FASTVIDEO_ATTENTION_BACKEND` outside the environment registry.
-Importing the PAN2 tests and running their fixtures preserves both an unset and a pre-existing attention backend.
-
-## Not covered
-
-- Real-weight parity and an SSIM regression: `IFM/PAN2` is not public.
-- GPU runs: everything above ran on CPU. The bf16 GPU path, FlashAttention, and SP over NCCL still need a GPU node.

tests/local_tests/pan2/reference.py

@@ -1,302 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""PAN2 reference implementation for local parity tests.
-
-The diffusers ``PAN2Transformer3DModel`` and ``PAN2ModularPipeline`` are not public yet. The reference here is
-assembled from diffusers' and transformers' own building blocks (``AdaLayerNormZero``, ``AdaLayerNormContinuous``,
-``FeedForward`` with ``gelu-approximate``/``swiglu``, ``Timesteps``/``TimestepEmbedding``,
-``get_1d_rotary_pos_embed``/``apply_rotary_emb``, ``Qwen3_5TextModel``, ``AutoencoderKLWan``,
-``FlowMatchEulerDiscreteScheduler``) and follows the vLLM-Omni PAN2 port (vllm-project/vllm-omni#8602), which was
-validated against the diffusers implementation (upstream revision ``e0c4c731``). Module names are the checkpoint's,
-so checkpoints load strictly. Its unconditional SP padding mask is equivalent to unpadded reference attention.
-"""
-
-from __future__ import annotations
-
-import os
-from pathlib import Path
-
-import numpy as np
-import PIL.Image
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from diffusers.models.attention import FeedForward
-from diffusers.models.embeddings import TimestepEmbedding, Timesteps, apply_rotary_emb, get_1d_rotary_pos_embed
-from diffusers.models.normalization import AdaLayerNormContinuous, AdaLayerNormZero, RMSNorm
-
-TINY_REPO_ID = "wuqing157/tiny-pan2-modular-pipe"
-TINY_REVISION = "3ff3130b4527f61aaab7bde01465c9884dc84cdd"
-SYSTEM_PROMPT = "You are a helpful assistant. Generate the video with the following instructions."
-TEXT_ENCODER_LAYERS_TO_SKIP = 2
-MAX_SEQUENCE_LENGTH = 1536
-
-
-def tiny_model_dir() -> Path | None:
-    """``PAN2_TINY_MODEL_DIR``, else the pinned tiny checkpoint in the local HF cache."""
-    local = os.getenv("PAN2_TINY_MODEL_DIR")
-    if local:
-        return Path(local)
-    try:
-        from huggingface_hub import snapshot_download
-
-        return Path(snapshot_download(TINY_REPO_ID, revision=TINY_REVISION, local_files_only=True))
-    except Exception:
-        return None
-
-
-def _sdpa(query: torch.Tensor, key: torch.Tensor, value: torch.Tensor) -> torch.Tensor:
-    out = F.scaled_dot_product_attention(query.transpose(1, 2), key.transpose(1, 2), value.transpose(1, 2))
-    return out.transpose(1, 2).flatten(2)
-
-
-class _RefRefinerAttention(nn.Module):
-
-    def __init__(self, dim: int, heads: int, dim_head: int, eps: float):
-        super().__init__()
-        self.heads = heads
-        self.to_q = nn.Linear(dim, heads * dim_head, bias=False)
-        self.to_k = nn.Linear(dim, heads * dim_head, bias=False)
-        self.to_v = nn.Linear(dim, heads * dim_head, bias=False)
-        self.norm_q = RMSNorm(dim_head, eps=eps)
-        self.norm_k = RMSNorm(dim_head, eps=eps)
-        self.to_out = nn.ModuleList([nn.Linear(heads * dim_head, dim, bias=False)])
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        query = self.norm_q(self.to_q(x).unflatten(-1, (self.heads, -1)))
-        key = self.norm_k(self.to_k(x).unflatten(-1, (self.heads, -1)))
-        value = self.to_v(x).unflatten(-1, (self.heads, -1))
-        return self.to_out[0](_sdpa(query, key, value))
-
-
-class _RefRefinerBlock(nn.Module):
-
-    def __init__(self, dim: int, heads: int, dim_head: int, mlp_ratio: float, eps: float):
-        super().__init__()
-        self.norm1 = RMSNorm(dim, eps=eps)
-        self.attn = _RefRefinerAttention(dim, heads, dim_head, eps)
-        self.norm2 = RMSNorm(dim, eps=eps)
-        self.ff = FeedForward(dim, inner_dim=int(dim * mlp_ratio), activation_fn="swiglu", bias=False)
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        x = x + self.attn(self.norm1(x))
-        return x + self.ff(self.norm2(x))
-
-
-class _RefJointAttention(nn.Module):
-
-    def __init__(self, dim: int, heads: int, dim_head: int, eps: float):
-        super().__init__()
-        self.heads = heads
-        self.to_q = nn.Linear(dim, heads * dim_head)
-        self.to_k = nn.Linear(dim, heads * dim_head)
-        self.to_v = nn.Linear(dim, heads * dim_head)
-        self.norm_q = RMSNorm(dim_head, eps=eps)
-        self.norm_k = RMSNorm(dim_head, eps=eps)
-        self.to_out = nn.ModuleList([nn.Linear(heads * dim_head, dim)])
-        self.add_q_proj = nn.Linear(dim, heads * dim_head)
-        self.add_k_proj = nn.Linear(dim, heads * dim_head)
-        self.add_v_proj = nn.Linear(dim, heads * dim_head)
-        self.norm_added_q = RMSNorm(dim_head, eps=eps)
-        self.norm_added_k = RMSNorm(dim_head, eps=eps)
-        self.to_add_out = nn.Linear(heads * dim_head, dim)
-
-    def forward(self, x: torch.Tensor, context: torch.Tensor,
-                rope: tuple[torch.Tensor, torch.Tensor]) -> tuple[torch.Tensor, torch.Tensor]:
-        query = self.norm_q(self.to_q(x).unflatten(-1, (self.heads, -1)))
-        key = self.norm_k(self.to_k(x).unflatten(-1, (self.heads, -1)))
-        value = self.to_v(x).unflatten(-1, (self.heads, -1))
-        # Interleaved (GPT-J) rotary embedding on the video tokens only.
-        query = apply_rotary_emb(query, rope, use_real=True, use_real_unbind_dim=-1, sequence_dim=1)
-        key = apply_rotary_emb(key, rope, use_real=True, use_real_unbind_dim=-1, sequence_dim=1)
-
-        context_query = self.norm_added_q(self.add_q_proj(context).unflatten(-1, (self.heads, -1)))
-        context_key = self.norm_added_k(self.add_k_proj(context).unflatten(-1, (self.heads, -1)))
-        context_value = self.add_v_proj(context).unflatten(-1, (self.heads, -1))
-
-        out = _sdpa(torch.cat([query, context_query], dim=1), torch.cat([key, context_key], dim=1),
-                    torch.cat([value, context_value], dim=1))
-        out, context_out = out.split([x.shape[1], context.shape[1]], dim=1)
-        return self.to_out[0](out), self.to_add_out(context_out)
-
-
-class _RefBlock(nn.Module):
-
-    def __init__(self, dim: int, heads: int, dim_head: int, mlp_ratio: float, context_mlp_ratio: float):
-        super().__init__()
-        eps = 1e-6
-        self.norm1 = AdaLayerNormZero(dim)
-        self.norm1_context = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
-        self.scale_shift_table_context = nn.Parameter(torch.zeros(6 * dim))
-        self.attn = _RefJointAttention(dim, heads, dim_head, eps)
-        self.norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
-        self.ff = FeedForward(dim, inner_dim=int(dim * mlp_ratio), activation_fn="gelu-approximate")
-        self.norm2_context = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
-        self.ff_context = FeedForward(dim, inner_dim=int(dim * context_mlp_ratio), activation_fn="gelu-approximate")
-
-    def forward(self, x, context, temb, rope):
-        norm_x, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1(x, emb=temb)
-        c_shift_msa, c_scale_msa, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp = (
-            self.scale_shift_table_context.chunk(6, dim=-1))
-        norm_context = self.norm1_context(context) * (1 + c_scale_msa) + c_shift_msa
-        attn_out, context_attn_out = self.attn(norm_x, norm_context, rope)
-        x = x + attn_out * gate_msa.unsqueeze(1)
-        context = context + context_attn_out * c_gate_msa
-        norm_x = self.norm2(x) * (1 + scale_mlp[:, None]) + shift_mlp[:, None]
-        norm_context = self.norm2_context(context) * (1 + c_scale_mlp) + c_shift_mlp
-        x = x + self.ff(norm_x) * gate_mlp.unsqueeze(1)
-        context = context + self.ff_context(norm_context) * c_gate_mlp
-        return x, context
-
-
-class ReferencePAN2Transformer(nn.Module):
-
-    def __init__(self, patch_size=(1, 2, 2), in_channels=97, out_channels=48, num_attention_heads=32,
-                 attention_head_dim=128, num_layers=44, num_refiner_layers=4, mlp_ratio=8.0, context_mlp_ratio=4.0,
-                 refiner_mlp_ratio=4.0, text_embed_dim=4096, rope_axes_dim=(16, 56, 56), rope_theta=256.0,
-                 rope_temporal_scale=16 / 24, **_):
-        super().__init__()
-        self.patch_size = tuple(patch_size)
-        self.rope_axes_dim = tuple(rope_axes_dim)
-        self.rope_theta = rope_theta
-        self.rope_temporal_scale = rope_temporal_scale
-        inner_dim = num_attention_heads * attention_head_dim
-        self.x_embedder = nn.Conv3d(in_channels, inner_dim, kernel_size=self.patch_size, stride=self.patch_size)
-        self.time_proj = Timesteps(num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=0)
-        self.time_embedder = TimestepEmbedding(in_channels=256, time_embed_dim=inner_dim)
-        self.context_embedder = nn.Sequential(RMSNorm(text_embed_dim, eps=1e-5), nn.Linear(text_embed_dim, inner_dim))
-        self.context_refiner = nn.ModuleList([
-            _RefRefinerBlock(inner_dim, num_attention_heads, attention_head_dim, refiner_mlp_ratio, eps=1e-5)
-            for _ in range(num_refiner_layers)
-        ])
-        self.context_type_embedding = nn.Parameter(torch.zeros(inner_dim))
-        self.transformer_blocks = nn.ModuleList([
-            _RefBlock(inner_dim, num_attention_heads, attention_head_dim, mlp_ratio, context_mlp_ratio)
-            for _ in range(num_layers)
-        ])
-        self.norm_out = AdaLayerNormContinuous(inner_dim, inner_dim, elementwise_affine=False, eps=1e-6)
-        self.proj_out = nn.Linear(inner_dim, self.patch_size[0] * self.patch_size[1] * self.patch_size[2] * out_channels)
-
-    def _rope(self, num_frames: int, height: int, width: int, device) -> tuple[torch.Tensor, torch.Tensor]:
-        axes = [torch.arange(size, device=device, dtype=torch.float32) for size in (num_frames, height, width)]
-        grid = torch.stack(torch.meshgrid(*axes, indexing="ij"), dim=0)
-        grid[0] = grid[0] * self.rope_temporal_scale
-        cos, sin = zip(*[
-            get_1d_rotary_pos_embed(dim, grid[i].reshape(-1), self.rope_theta, use_real=True,
-                                    repeat_interleave_real=True) for i, dim in enumerate(self.rope_axes_dim)
-        ])
-        return torch.cat(cos, dim=1).float(), torch.cat(sin, dim=1).float()
-
-    def forward(self, hidden_states, timestep, encoder_hidden_states):
-        batch_size, _, num_frames, height, width = hidden_states.shape
-        p_t, p_h, p_w = self.patch_size
-        t, h, w = num_frames // p_t, height // p_h, width // p_w
-        rope = self._rope(t, h, w, hidden_states.device)
-        x = self.x_embedder(hidden_states).flatten(2).transpose(1, 2)
-        temb = self.time_embedder(self.time_proj(timestep).to(x.dtype))
-        context = self.context_embedder(encoder_hidden_states)
-        for block in self.context_refiner:
-            context = block(context)
-        context = context + self.context_type_embedding
-        for block in self.transformer_blocks:
-            x, context = block(x, context, temb, rope)
-        x = self.proj_out(self.norm_out(x, temb))
-        x = x.reshape(batch_size, t, h, w, -1, p_t, p_h, p_w).permute(0, 4, 1, 5, 2, 6, 3, 7)
-        return x.flatten(6, 7).flatten(4, 5).flatten(2, 3)
-
-
-def reference_encode_prompt(tokenizer, text_encoder, prompt: str, device) -> torch.Tensor:
-    messages = [{"role": "system", "content": SYSTEM_PROMPT}, {"role": "user", "content": prompt or " "}]
-    input_ids = tokenizer.apply_chat_template(messages, add_generation_prompt=True, tokenize=True,
-                                              return_dict=True).input_ids
-    header_ids = tokenizer("<|im_start|>user\n", add_special_tokens=False).input_ids
-    crop_start = next(i + len(header_ids) for i in range(len(input_ids))
-                      if input_ids[i:i + len(header_ids)] == header_ids)
-    input_ids = torch.tensor([input_ids[:crop_start + MAX_SEQUENCE_LENGTH]], device=device)
-    hidden_states = text_encoder(input_ids=input_ids, output_hidden_states=True).hidden_states
-    return hidden_states[-(TEXT_ENCODER_LAYERS_TO_SKIP + 1)][:, crop_start:]
-
-
-def reference_condition_latents(vae, latents: torch.Tensor, image: PIL.Image.Image | None, height: int,
-                                width: int) -> torch.Tensor:
-    batch_size, num_channels, num_frames, latent_height, latent_width = latents.shape
-    condition = latents.new_zeros(batch_size, num_channels + 1, num_frames, latent_height, latent_width)
-    if image is None:
-        return condition
-    from diffusers.video_processor import VideoProcessor
-
-    image = image.convert("RGB")
-    scale = max(width / image.width, height / image.height)
-    image = image.resize((round(image.width * scale), round(image.height * scale)), resample=PIL.Image.LANCZOS)
-    left, top = round((image.width - width) / 2), round((image.height - height) / 2)
-    image = image.crop((left, top, left + width, top + height))
-    pixels = VideoProcessor(vae_scale_factor=16).preprocess(image, height=height, width=width)
-    pixels = pixels.unsqueeze(2).to(device=latents.device, dtype=vae.dtype)
-    image_latents = vae.encode(pixels).latent_dist.mode()
-    mean = torch.tensor(vae.config.latents_mean).view(1, -1, 1, 1, 1).to(image_latents)
-    std = torch.tensor(vae.config.latents_std).view(1, -1, 1, 1, 1).to(image_latents)
-    condition[:, :num_channels, :1] = (image_latents - mean) / std
-    condition[:, num_channels:, :1] = 1.0
-    return condition
-
-
-@torch.no_grad()
-def reference_generate(components: dict, *, prompt: str, negative_prompt: str, image: PIL.Image.Image | None,
-                       height: int, width: int, num_frames: int, num_inference_steps: int, guidance_scale: float,
-                       seed: int, dit_dtype: torch.dtype, device="cpu",
-                       skip_first_latent_cfg: bool = True) -> tuple[torch.Tensor, torch.Tensor]:
-    """Final latents and the decoded video in ``[0, 1]``, following the vLLM-Omni pipeline step by step.
-
-    ``skip_first_latent_cfg`` is the reference sampler's ``--i2v_skip_first_latent_cfg``: with an image, the first
-    latent frame takes the positive prediction instead of the guided one.
-    """
-    from diffusers.utils.torch_utils import randn_tensor
-
-    tokenizer, text_encoder = components["tokenizer"], components["text_encoder"]
-    transformer, vae, scheduler = components["transformer"], components["vae"], components["scheduler"]
-    do_cfg = guidance_scale > 1.0
-    prompt_embeds = reference_encode_prompt(tokenizer, text_encoder, prompt, device).to(dit_dtype)
-    negative_embeds = (reference_encode_prompt(tokenizer, text_encoder, negative_prompt, device).to(dit_dtype)
-                       if do_cfg else None)
-
-    shape = (1, vae.config.z_dim, (num_frames - 1) // vae.config.scale_factor_temporal + 1,
-             height // vae.config.scale_factor_spatial, width // vae.config.scale_factor_spatial)
-    generator = [torch.Generator("cpu").manual_seed(seed)]
-    latents = randn_tensor(shape, generator=generator, device=torch.device(device), dtype=torch.float32)
-    condition = reference_condition_latents(vae, latents, image, height, width).to(dit_dtype)
-
-    scheduler.set_timesteps(sigmas=np.linspace(1.0, 0.0, num_inference_steps + 1)[:-1], device=device)
-    for t in scheduler.timesteps:
-        model_input = torch.cat([latents.to(dit_dtype), condition], dim=1)
-        timestep = t.expand(model_input.shape[0])
-        noise_pred = transformer(model_input, timestep, prompt_embeds)
-        if do_cfg:
-            noise_uncond = transformer(model_input, timestep, negative_embeds)
-            guided = noise_uncond + guidance_scale * (noise_pred - noise_uncond)
-            if image is not None and skip_first_latent_cfg:
-                guided[:, :, :1] = noise_pred[:, :, :1]
-            noise_pred = guided
-        latents = scheduler.step(noise_pred.float(), t, latents, return_dict=False)[0]
-
-    mean = torch.tensor(vae.config.latents_mean).view(1, -1, 1, 1, 1).to(latents)
-    std = torch.tensor(vae.config.latents_std).view(1, -1, 1, 1, 1).to(latents)
-    video = vae.decode((latents * std + mean).to(vae.dtype), return_dict=False)[0]
-    return latents, (video.float() / 2 + 0.5).clamp(0, 1)
-
-
-def load_reference_components(model_dir: Path, dit_dtype: torch.dtype) -> dict:
-    import json
-
-    from diffusers import AutoencoderKLWan, FlowMatchEulerDiscreteScheduler
-    from safetensors.torch import load_file
-    from transformers import AutoTokenizer, Qwen3_5TextModel
-
-    transformer_dir = model_dir / "transformer"
-    transformer = ReferencePAN2Transformer(**json.loads((transformer_dir / "config.json").read_text()))
-    transformer.load_state_dict(load_file(str(transformer_dir / "diffusion_pytorch_model.safetensors")), strict=True)
-    return {
-        "tokenizer": AutoTokenizer.from_pretrained(model_dir / "tokenizer"),
-        "text_encoder": Qwen3_5TextModel.from_pretrained(model_dir / "text_encoder", torch_dtype=torch.float32).eval(),
-        "transformer": transformer.to(dit_dtype).eval(),
-        "vae": AutoencoderKLWan.from_pretrained(model_dir / "vae", torch_dtype=torch.float32).eval(),
-        "scheduler": FlowMatchEulerDiscreteScheduler.from_pretrained(model_dir / "scheduler"),
-    }

tests/local_tests/pipelines/test_pan2_pipeline_parity.py

@@ -1,159 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""PAN2 end-to-end pipeline parity on the public tiny checkpoint.
-
-The FastVideo pipeline (built through the real registry and component loaders) is compared against
-``tests/local_tests/pan2/reference.py::reference_generate``, which follows the vLLM-Omni ``PAN2Pipeline.forward``
-step by step with transformers/diffusers components: prompt template and crop, fp32 noise, first-frame
-conditioning, the uniform sigma schedule with shift 7, sequential CFG, fp32 Euler steps and the VAE decode.
-
-Both sides run the DiT in bf16 (FastVideo's denoiser always feeds it bf16), so the comparison allows bf16 rounding.
-Runs on CPU when no GPU is present. Set ``PAN2_TINY_MODEL_DIR`` to a local copy of the tiny checkpoint.
-"""
-
-from __future__ import annotations
-
-import socket
-from contextlib import ExitStack
-
-import PIL.Image
-import pytest
-import torch
-
-from fastvideo import envs
-from tests.local_tests.pan2.reference import load_reference_components, reference_generate, tiny_model_dir
-
-HEIGHT, WIDTH, NUM_FRAMES = 64, 96, 9
-NUM_INFERENCE_STEPS = 3
-# A large scale makes the guided steps visible through the tiny random-weight VAE.
-GUIDANCE_SCALE = 30.0
-SEED = 42
-PROMPT = "a cat walks on the grass"
-NEGATIVE_PROMPT = "blurry"
-# Not the output aspect ratio, so the cover-resize and center crop are exercised.
-CONDITION_IMAGE = PIL.Image.radial_gradient("L").resize((120, 64)).convert("RGB")
-
-
-def _free_port() -> int:
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
-        sock.bind(("127.0.0.1", 0))
-        return sock.getsockname()[1]
-
-
-@pytest.fixture(scope="module")
-def pan2_runtime():
-    model_dir = tiny_model_dir()
-    if model_dir is None or not (model_dir / "modular_model_index.json").is_file():
-        pytest.skip("wuqing157/tiny-pan2-modular-pipe is not available locally; set PAN2_TINY_MODEL_DIR")
-    from fastvideo.configs.pipelines.pan2 import PAN2PipelineConfig
-    from fastvideo.distributed import cleanup_dist_env_and_memory, maybe_init_distributed_environment_and_model_parallel
-    import fastvideo.distributed.parallel_state as parallel_state
-    from fastvideo.fastvideo_args import FastVideoArgs
-    from fastvideo.pipelines import build_pipeline
-
-    with ExitStack() as stack:
-        stack.enter_context(envs.FASTVIDEO_ATTENTION_BACKEND.override("TORCH_SDPA"))
-        monkeypatch = stack.enter_context(pytest.MonkeyPatch.context())
-        if not torch.cuda.is_available():
-            # Reach modules that imported the device helper by name on a GPU-less Linux host.
-            def _cpu_device() -> torch.device:
-                return torch.device("cpu")
-
-            monkeypatch.setattr(parallel_state.get_local_torch_device, "__code__", _cpu_device.__code__)
-        for name, value in (("MASTER_ADDR", "127.0.0.1"), ("MASTER_PORT", str(_free_port())), ("RANK", "0"),
-                            ("WORLD_SIZE", "1"), ("LOCAL_RANK", "0")):
-            stack.enter_context(envs.override_external(name, value))
-        stack.callback(cleanup_dist_env_and_memory)
-        maybe_init_distributed_environment_and_model_parallel(1, 1)
-
-        pipeline_config = PAN2PipelineConfig(dit_precision="bf16",
-                                             vae_precision="fp32",
-                                             text_encoder_precisions=("fp32", ))
-        args = FastVideoArgs(model_path=str(model_dir),
-                             num_gpus=1,
-                             pipeline_config=pipeline_config,
-                             dit_cpu_offload=False,
-                             dit_layerwise_offload=False,
-                             text_encoder_cpu_offload=False,
-                             vae_cpu_offload=False,
-                             use_fsdp_inference=False)
-        pipeline = build_pipeline(args)
-        device = parallel_state.get_local_torch_device()
-        yield pipeline, args, model_dir, device
-
-
-def _fastvideo_generate(pipeline, args, image, guidance_scale: float = GUIDANCE_SCALE):
-    from fastvideo.pipelines import ForwardBatch
-
-    batch = ForwardBatch(data_type="video",
-                         prompt=PROMPT,
-                         negative_prompt=NEGATIVE_PROMPT,
-                         seed=SEED,
-                         height=HEIGHT,
-                         width=WIDTH,
-                         num_frames=NUM_FRAMES,
-                         num_inference_steps=NUM_INFERENCE_STEPS,
-                         guidance_scale=guidance_scale,
-                         pil_image=image)
-    with torch.inference_mode():
-        batch = pipeline.forward(batch, args)
-    return batch.latents.float().cpu(), batch.output.float().cpu()
-
-
-def _psnr(a: torch.Tensor, b: torch.Tensor) -> float:
-    mse = (a - b).pow(2).mean().clamp_min(1e-12)
-    return float(10 * torch.log10(1.0 / mse))
-
-
-@pytest.mark.parametrize(
-    ("image", "skip_first_latent_cfg"),
-    [(None, True), (CONDITION_IMAGE, True), (CONDITION_IMAGE, False)],
-    ids=["t2v", "i2v", "i2v-guided-first-frame"],
-)
-def test_pan2_pipeline_matches_reference(pan2_runtime, monkeypatch, image, skip_first_latent_cfg):
-    pipeline, args, model_dir, device = pan2_runtime
-    monkeypatch.setattr(args.pipeline_config, "i2v_skip_first_latent_cfg", skip_first_latent_cfg)
-    latents, video = _fastvideo_generate(pipeline, args, image)
-
-    components = load_reference_components(model_dir, torch.bfloat16)
-    components = {
-        name: module.to(device) if isinstance(module, torch.nn.Module) else module
-        for name, module in components.items()
-    }
-    ref_latents, ref_video = reference_generate(components,
-                                                prompt=PROMPT,
-                                                negative_prompt=NEGATIVE_PROMPT,
-                                                image=image,
-                                                height=HEIGHT,
-                                                width=WIDTH,
-                                                num_frames=NUM_FRAMES,
-                                                num_inference_steps=NUM_INFERENCE_STEPS,
-                                                guidance_scale=GUIDANCE_SCALE,
-                                                seed=SEED,
-                                                dit_dtype=torch.bfloat16,
-                                                device=device,
-                                                skip_first_latent_cfg=skip_first_latent_cfg)
-    ref_latents, ref_video = ref_latents.float().cpu(), ref_video.float().cpu()
-
-    assert latents.dtype == torch.float32
-    assert latents.shape == ref_latents.shape == (1, 48, 3, HEIGHT // 16, WIDTH // 16)
-    assert video.shape == ref_video.shape == (1, 3, NUM_FRAMES, HEIGHT, WIDTH)
-    latent_rel = float((latents - ref_latents).norm() / ref_latents.norm())
-    cosine = float(torch.nn.functional.cosine_similarity(latents.flatten(), ref_latents.flatten(), dim=0))
-    psnr = _psnr(video, ref_video)
-    print(f"latents rel_l2={latent_rel:.3e} cosine={cosine:.6f}; video psnr={psnr:.1f} dB")
-    assert latent_rel < 2e-2
-    assert cosine > 0.9995
-    assert psnr > 40.0
-
-
-def test_pan2_image_conditioning_and_guidance_change_every_frame(pan2_runtime):
-    pipeline, args, _, _ = pan2_runtime
-    _, t2v = _fastvideo_generate(pipeline, args, None)
-    _, i2v = _fastvideo_generate(pipeline, args, CONDITION_IMAGE)
-    _, unguided = _fastvideo_generate(pipeline, args, None, guidance_scale=1.0)
-
-    def per_frame_max_diff(a, b):
-        return (a - b).abs().amax(dim=(0, 1, 3, 4))
-
-    assert per_frame_max_diff(t2v, i2v).max() > 1e-3
-    assert per_frame_max_diff(t2v, unguided).min() > 1e-3

tests/local_tests/schedulers/test_pan2_scheduler_parity.py

@@ -1,200 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""PAN2 flow-matching Euler schedule on FastVideo's FlowMatchEulerDiscreteScheduler.
-
-Coverage scope: both. PAN2 loads Diffusers' ``FlowMatchEulerDiscreteScheduler``
-from ``<checkpoint>/scheduler`` (shift 7.0, no dynamic shifting), optionally
-overrides the shift with ``flow_shift``, calls
-``set_timesteps(sigmas=np.linspace(1.0, 0.0, N + 1)[:-1])`` and steps float32
-latents. The FastVideo side is loaded by the production ``SchedulerLoader`` and
-its schedule is built by the stock ``TimestepPreparationStage`` from
-``batch.sigmas``. Timesteps, sigmas, and every Euler update must be bit-exact.
-
-The default asset is the tiny PAN2 checkpoint (config only), so this runs on a
-CPU-only host::
-
-    pytest tests/local_tests/schedulers/test_pan2_scheduler_parity.py -v -s
-
-Set ``PAN2_MODEL_PATH`` to a local PAN2 root containing ``scheduler/`` to use
-another checkpoint.
-"""
-
-from __future__ import annotations
-
-import json
-import os
-from pathlib import Path
-from types import SimpleNamespace
-
-import numpy as np
-import pytest
-import torch
-from torch.testing import assert_close
-
-try:
-    from diffusers import FlowMatchEulerDiscreteScheduler as DiffusersFlowMatchEulerDiscreteScheduler
-except ImportError:  # pragma: no cover - environment dependent
-    pytest.skip("diffusers with FlowMatchEulerDiscreteScheduler is required", allow_module_level=True)
-
-from fastvideo.models.loader.component_loader import SchedulerLoader
-from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import (
-    FlowMatchEulerDiscreteScheduler as FastVideoFlowMatchEulerDiscreteScheduler, )
-from fastvideo.pipelines.pipeline_batch_info import ForwardBatch
-from fastvideo.pipelines.stages.timestep_preparation import TimestepPreparationStage
-
-PARITY_SCOPE = "both"
-TINY_PAN2_REPO = "wuqing157/tiny-pan2-modular-pipe"
-TINY_PAN2_REVISION = "3ff3130b4527f61aaab7bde01465c9884dc84cdd"
-MODEL_PATH_ENV = "PAN2_MODEL_PATH"
-LATENT_SHAPE = (1, 48, 3, 4, 6)
-
-
-def _pan2_root(subfolder: str) -> Path:
-    """Resolve a PAN2 root that contains ``subfolder``; skip when unavailable."""
-    override = os.getenv(MODEL_PATH_ENV)
-    if override:
-        root = Path(override)
-    else:
-        try:
-            from huggingface_hub import snapshot_download
-        except ImportError:
-            pytest.skip("huggingface_hub is required to resolve the tiny PAN2 checkpoint")
-        root = None
-        last_error: Exception | None = None
-        for local_files_only in (True, False):
-            try:
-                root = Path(
-                    snapshot_download(
-                        TINY_PAN2_REPO,
-                        revision=TINY_PAN2_REVISION,
-                        allow_patterns=[f"{subfolder}/*"],
-                        local_files_only=local_files_only,
-                    ))
-                break
-            except Exception as exc:  # noqa: BLE001 - offline/missing cache must skip.
-                last_error = exc
-        if root is None:
-            pytest.skip(f"Tiny PAN2 checkpoint unavailable: {last_error}")
-    if not (root / subfolder).is_dir():
-        pytest.skip(f"PAN2 {subfolder}/ missing under {root}")
-    return root
-
-
-@pytest.fixture(scope="module")
-def pan2_root() -> Path:
-    root = _pan2_root("scheduler")
-    if not (root / "scheduler" / "scheduler_config.json").is_file():
-        pytest.skip(f"PAN2 scheduler config missing under {root}")
-    return root
-
-
-@pytest.fixture
-def cpu_runtime(monkeypatch: pytest.MonkeyPatch) -> None:
-    """Pin FastVideo's device helper to CPU; it reports ``mps`` on CPU-only Linux."""
-    monkeypatch.setattr("fastvideo.pipelines.stages.timestep_preparation.get_local_torch_device",
-                        lambda: torch.device("cpu"))
-
-
-def _pan2_sigmas(num_steps: int) -> np.ndarray:
-    """PAN2's pre-shift grid: uniform from 1 down to, but excluding, 0."""
-    return np.linspace(1.0, 0.0, num_steps + 1)[:-1]
-
-
-def _reference(root: Path, flow_shift: float | None = None) -> DiffusersFlowMatchEulerDiscreteScheduler:
-    """Mirror the upstream PAN2 pipeline's scheduler construction."""
-    scheduler = DiffusersFlowMatchEulerDiscreteScheduler.from_pretrained(str(root),
-                                                                         subfolder="scheduler",
-                                                                         local_files_only=True)
-    if flow_shift is not None:
-        scheduler._shift = flow_shift
-    return scheduler
-
-
-def _fastvideo(root: Path, flow_shift: float | None = None) -> FastVideoFlowMatchEulerDiscreteScheduler:
-    """Load through the production SchedulerLoader, which applies ``flow_shift``."""
-    fastvideo_args = SimpleNamespace(pipeline_config=SimpleNamespace(flow_shift=flow_shift))
-    scheduler = SchedulerLoader().load(str(root / "scheduler"), fastvideo_args)
-    assert type(scheduler) is FastVideoFlowMatchEulerDiscreteScheduler
-    return scheduler
-
-
-def _prepare_fastvideo_timesteps(scheduler: FastVideoFlowMatchEulerDiscreteScheduler,
-                                 num_steps: int) -> ForwardBatch:
-    """Drive the stock TimestepPreparationStage with PAN2's custom sigmas."""
-    batch = ForwardBatch(data_type="video", num_inference_steps=num_steps, sigmas=_pan2_sigmas(num_steps).tolist())
-    stage = TimestepPreparationStage(scheduler)
-    assert stage.verify_input(batch, SimpleNamespace()).is_valid()
-    batch = stage.forward(batch, SimpleNamespace())
-    assert batch.timesteps is scheduler.timesteps
-    return batch
-
-
-def test_pan2_scheduler_loader_keeps_checkpoint_config(pan2_root: Path) -> None:
-    """Without ``flow_shift`` the loader keeps every released config value, including shift 7.0."""
-    raw = json.loads((pan2_root / "scheduler" / "scheduler_config.json").read_text())
-    assert raw["_class_name"] == "FlowMatchEulerDiscreteScheduler"
-    scheduler = _fastvideo(pan2_root)
-    for key, value in raw.items():
-        if not key.startswith("_"):
-            assert scheduler.config[key] == value, key
-    assert scheduler.shift == raw["shift"] == 7.0
-    assert not scheduler.config.use_dynamic_shifting
-
-
-@pytest.mark.parametrize("num_steps", [1, 4, 8, 30, 50])
-def test_pan2_scheduler_schedule_matches_diffusers(num_steps: int, pan2_root: Path, cpu_runtime: None) -> None:
-    """Timesteps and sigmas from ``batch.sigmas`` are bit-exact for short and long schedules."""
-    reference = _reference(pan2_root)
-    reference.set_timesteps(sigmas=_pan2_sigmas(num_steps), device="cpu")
-    fastvideo = _fastvideo(pan2_root)
-    batch = _prepare_fastvideo_timesteps(fastvideo, num_steps)
-
-    print(f"[pan2-scheduler] N={num_steps} timesteps[:4]={reference.timesteps[:4].tolist()} "
-          f"sigmas[-3:]={reference.sigmas[-3:].tolist()}")
-    assert len(batch.timesteps) == num_steps
-    assert_close(batch.timesteps, reference.timesteps, atol=0, rtol=0)
-    assert_close(fastvideo.sigmas, reference.sigmas, atol=0, rtol=0)
-    assert fastvideo.sigmas[0].item() == 1.0 and fastvideo.sigmas[-1].item() == 0.0
-
-
-@pytest.mark.parametrize("flow_shift", [None, 5.0], ids=["checkpoint_shift", "flow_shift_override"])
-def test_pan2_scheduler_steps_match_diffusers(flow_shift: float | None, pan2_root: Path, cpu_runtime: None) -> None:
-    """Five float32 Euler updates are bit-exact, with and without a shift override."""
-    num_steps = 5
-    reference = _reference(pan2_root, flow_shift)
-    reference.set_timesteps(sigmas=_pan2_sigmas(num_steps), device="cpu")
-    fastvideo = _fastvideo(pan2_root, flow_shift)
-    _prepare_fastvideo_timesteps(fastvideo, num_steps)
-    assert fastvideo.shift == reference.shift == (flow_shift or 7.0)
-    assert_close(fastvideo.timesteps, reference.timesteps, atol=0, rtol=0)
-
-    generator = torch.Generator().manual_seed(2026)
-    reference_latents = torch.randn(LATENT_SHAPE, generator=generator)
-    fastvideo_latents = reference_latents.clone()
-    max_diff = 0.0
-    for index in range(num_steps):
-        noise_pred = torch.randn(LATENT_SHAPE, generator=generator)
-        reference_latents = reference.step(noise_pred, reference.timesteps[index], reference_latents).prev_sample
-        fastvideo_latents = fastvideo.step(noise_pred, fastvideo.timesteps[index], fastvideo_latents).prev_sample
-        assert fastvideo_latents.dtype == torch.float32
-        max_diff = max(max_diff, (fastvideo_latents - reference_latents).abs().max().item())
-        assert_close(fastvideo_latents, reference_latents, atol=0, rtol=0)
-    print(f"[pan2-scheduler] flow_shift={flow_shift} steps={num_steps} max_abs={max_diff:.3e}")
-    assert fastvideo.step_index == reference.step_index == num_steps
-
-
-def test_pan2_scheduler_step_returns_model_output_dtype(pan2_root: Path, cpu_runtime: None) -> None:
-    """Both implementations cast ``prev_sample`` to the model output dtype.
-
-    PAN2 keeps float32 latents by stepping with ``noise_pred.float()``; a bf16
-    noise prediction would silently drop the latents to bf16 on either side.
-    """
-    reference = _reference(pan2_root)
-    reference.set_timesteps(sigmas=_pan2_sigmas(2), device="cpu")
-    fastvideo = _fastvideo(pan2_root)
-    _prepare_fastvideo_timesteps(fastvideo, 2)
-    latents = torch.randn(LATENT_SHAPE, generator=torch.Generator().manual_seed(5))
-    noise_pred = torch.randn(LATENT_SHAPE, generator=torch.Generator().manual_seed(6)).to(torch.bfloat16)
-    expected = reference.step(noise_pred, reference.timesteps[0], latents).prev_sample
-    actual = fastvideo.step(noise_pred, fastvideo.timesteps[0], latents).prev_sample
-    assert expected.dtype == actual.dtype == torch.bfloat16
-    assert_close(actual, expected, atol=0, rtol=0)

tests/local_tests/test_cookbook_serving.py

@@ -264,23 +264,3 @@ def test_kandinsky6_cookbook_recipes_match_maintained_examples():
     assert all("INPUT_VIDEO" in by_id[recipe_id]["command"] for recipe_id in ("kandinsky6-vsr", "kandinsky6-vsr-distilled"))
     assert "VSR-distilled2steps" in by_id["kandinsky6-vsr-distilled"]["command"]
     validate_cookbook()
-
-
-def test_pan2_recipe_serves_text_and_image_from_one_deployment():
-    recipes = json.loads(COOKBOOK_DATA.read_text())["recipes"]
-    recipe = next(item for item in recipes if item["family"] == "pan2")
-    assert recipe["id"] == "pan2-ti2v"
-    assert recipe["model"] == "IFM/PAN2"
-    assert recipe["evidence"] == "Source-backed"
-    assert recipe["hardware"] == {"platform": "cuda", "gpu_count": 1, "evidence": "source-configured"}
-    profile = cookbook_serving_profile(recipe)
-    assert profile["command"] == "fastvideo serve --config examples/serving/openai_pan2.yaml --server.host 127.0.0.1"
-    assert profile["model"] == "pan2"
-    assert profile["playground_url"] is None
-    assert profile["sampling"]["num_frames"] == 313
-    assert any("image_reference" in item for item in recipe["serving"]["limitations"])
-    manifest = yaml.safe_load((ROOT / "docs/cookbook/recipes/pan2.yaml").read_text())
-    deployment = manifest["deployments"]["cuda-rest"]
-    assert deployment["workload"] == "t2v"
-    assert deployment["defaults"] == recipe["serving"]["source"]
-    validate_cookbook()

tests/local_tests/transformers/test_pan2_transformer_parity.py

@@ -1,328 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""PAN2 transformer weight-schema and forward parity.
-
-The reference is ``tests/local_tests/pan2/reference.py``: diffusers building blocks arranged as the (not yet public)
-diffusers ``PAN2Transformer3DModel``, following the vLLM-Omni port. Its module names are the checkpoint's, so the
-public tiny checkpoint loads into it strictly.
-
-Set ``PAN2_TINY_MODEL_DIR`` to a local copy of ``wuqing157/tiny-pan2-modular-pipe``; otherwise the HF cache is
-used and the checkpoint test skips when it is absent.
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import os
-import socket
-import subprocess
-import sys
-from contextlib import ExitStack
-from dataclasses import fields
-from pathlib import Path
-
-import pytest
-import torch
-import torch.nn as nn
-from torch.testing import assert_close
-
-from fastvideo import envs
-from fastvideo.forward_context import set_forward_context
-from fastvideo.models.loader.fsdp_load import load_model_from_full_model_state_dict
-from fastvideo.models.loader.utils import get_param_names_mapping
-from tests.local_tests.pan2.reference import TINY_REPO_ID, ReferencePAN2Transformer, tiny_model_dir
-
-REPO_ROOT = Path(__file__).resolve().parents[3]
-SP_WORLD_SIZES = (2, 4)
-
-
-def _free_port() -> int:
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
-        sock.bind(("127.0.0.1", 0))
-        return sock.getsockname()[1]
-
-
-def _small_kwargs() -> dict:
-    """Larger than the tiny checkpoint: several heads, asymmetric RoPE axes, more blocks, distinct MLP ratios."""
-    return {
-        "patch_size": (1, 2, 2),
-        "in_channels": 9,
-        "out_channels": 4,
-        "num_attention_heads": 4,
-        "attention_head_dim": 32,
-        "num_layers": 3,
-        "num_refiner_layers": 2,
-        "mlp_ratio": 3.0,
-        "context_mlp_ratio": 2.0,
-        "refiner_mlp_ratio": 1.5,
-        "text_embed_dim": 48,
-        "rope_axes_dim": (8, 12, 12),
-        "rope_theta": 256.0,
-        "rope_temporal_scale": 16 / 24,
-    }
-
-
-@pytest.fixture
-def single_rank_dist(monkeypatch):
-    from fastvideo.distributed import cleanup_dist_env_and_memory, maybe_init_distributed_environment_and_model_parallel
-    import fastvideo.distributed.parallel_state as parallel_state
-
-    if not torch.cuda.is_available():
-        def _cpu_device() -> torch.device:
-            return torch.device("cpu")
-
-        monkeypatch.setattr(parallel_state.get_local_torch_device, "__code__", _cpu_device.__code__)
-    with ExitStack() as stack:
-        stack.enter_context(envs.FASTVIDEO_ATTENTION_BACKEND.override("TORCH_SDPA"))
-        for name, value in (("MASTER_ADDR", "127.0.0.1"), ("MASTER_PORT", str(_free_port())), ("RANK", "0"),
-                            ("WORLD_SIZE", "1"), ("LOCAL_RANK", "0")):
-            stack.enter_context(envs.override_external(name, value))
-        try:
-            maybe_init_distributed_environment_and_model_parallel(1, 1)
-            yield
-        finally:
-            cleanup_dist_env_and_memory()
-
-
-def _build_native(arch_kwargs: dict):
-    from fastvideo.configs.models.dits.pan2 import PAN2VideoConfig
-    from fastvideo.models.dits.pan2 import PAN2Transformer3DModel
-
-    config = PAN2VideoConfig()
-    config.update_model_arch(arch_kwargs)
-    return PAN2Transformer3DModel(config=config, hf_config=arch_kwargs).float().eval()
-
-
-def _load_into_native(native, weights) -> None:
-    incompatible = load_model_from_full_model_state_dict(
-        native,
-        iter(weights),
-        device=torch.device("cpu"),
-        param_dtype=torch.float32,
-        strict=True,
-        param_names_mapping=get_param_names_mapping(native.param_names_mapping),
-        training_mode=False,
-    )
-    assert incompatible.missing_keys == []
-    assert incompatible.unexpected_keys == []
-
-
-def _randomize(module: nn.Module, seed: int) -> None:
-    generator = torch.Generator().manual_seed(seed)
-    with torch.no_grad():
-        for param in module.parameters():
-            param.copy_(torch.randn(param.shape, generator=generator) * 0.2)
-
-
-def _make_inputs(in_channels: int, text_dim: int, batch_size: int = 2, seed: int = 0) -> dict[str, torch.Tensor]:
-    generator = torch.Generator().manual_seed(seed)
-    return {
-        # 9 frames at 64x96 in latent space: 3 latent frames of 4x6.
-        "hidden_states": torch.randn(batch_size, in_channels, 3, 4, 6, generator=generator),
-        "timestep": torch.tensor([700.0, 125.5][:batch_size]),
-        "encoder_hidden_states": torch.randn(batch_size, 13, text_dim, generator=generator),
-    }
-
-
-def _compare(native, reference, inputs: dict[str, torch.Tensor], scope: str) -> None:
-    with torch.inference_mode():
-        expected = reference(**inputs)
-        with set_forward_context(current_timestep=0, attn_metadata=None):
-            actual = native(**inputs)
-    assert actual.shape == expected.shape
-    diff = (actual - expected).abs()
-    print(f"{scope}: shape={tuple(actual.shape)} ref_abs_mean={expected.abs().mean().item():.3e} "
-          f"diff_max={diff.max().item():.3e} diff_mean={diff.mean().item():.3e}")
-    assert torch.isfinite(expected).all() and expected.abs().mean() > 1e-3
-    assert_close(actual, expected, atol=2e-5, rtol=2e-5)
-
-
-def test_pan2_transformer_registry_resolves_native_class():
-    from fastvideo.models.dits.pan2 import PAN2Transformer3DModel
-    from fastvideo.models.registry import ModelRegistry
-
-    model_cls, architecture = ModelRegistry.resolve_model_cls("PAN2Transformer3DModel")
-    assert model_cls is PAN2Transformer3DModel
-    assert architecture == "PAN2Transformer3DModel"
-
-
-@pytest.mark.parametrize(("module_name", "expected"), [
-    ("transformer_blocks.0", True),
-    ("transformer_blocks.43", True),
-    ("context_refiner.3", True),
-    ("transformer_blocks.0.attn.to_qkv", False),
-    ("transformer_blocks.0.ff.fc_in", False),
-])
-def test_pan2_fsdp_shard_conditions_match_only_blocks(module_name: str, expected: bool):
-    from fastvideo.configs.models.dits.pan2 import PAN2ArchConfig
-
-    conditions = PAN2ArchConfig()._fsdp_shard_conditions
-    assert any(condition(module_name, None) for condition in conditions) is expected
-
-
-def test_pan2_default_arch_config_is_the_29b_model():
-    from fastvideo.configs.models.dits.pan2 import PAN2ArchConfig
-
-    config = PAN2ArchConfig()
-    assert config.hidden_size == 4096
-    assert config.num_channels_latents == 48
-    assert config.in_channels == 2 * 48 + 1
-    assert sum(config.rope_axes_dim) == config.attention_head_dim
-
-
-def test_pan2_tiny_checkpoint_config_fields_are_arch_fields():
-    model_dir = tiny_model_dir()
-    if model_dir is None or not (model_dir / "transformer" / "config.json").is_file():
-        pytest.skip(f"{TINY_REPO_ID} is not available locally; set PAN2_TINY_MODEL_DIR")
-    from fastvideo.configs.models.dits.pan2 import PAN2ArchConfig
-
-    checkpoint_config = json.loads((model_dir / "transformer" / "config.json").read_text())
-    assert checkpoint_config.pop("_class_name") == "PAN2Transformer3DModel"
-    checkpoint_config.pop("_diffusers_version", None)
-    assert set(checkpoint_config) <= {field.name for field in fields(PAN2ArchConfig)}
-
-
-def test_pan2_tiny_checkpoint_strict_load_and_forward_parity(single_rank_dist):
-    model_dir = tiny_model_dir()
-    if model_dir is None or not (model_dir / "transformer").is_dir():
-        pytest.skip(f"{TINY_REPO_ID} is not available locally; set PAN2_TINY_MODEL_DIR")
-    from safetensors.torch import load_file
-
-    transformer_dir = model_dir / "transformer"
-    arch_kwargs = json.loads((transformer_dir / "config.json").read_text())
-    state_dict = load_file(str(transformer_dir / "diffusion_pytorch_model.safetensors"))
-
-    reference = ReferencePAN2Transformer(**arch_kwargs).float().eval()
-    reference.load_state_dict(state_dict, strict=True)
-    native = _build_native(arch_kwargs)
-    _load_into_native(native, state_dict.items())
-
-    inputs = _make_inputs(arch_kwargs["in_channels"], arch_kwargs["text_embed_dim"])
-    _compare(native, reference, inputs, "tiny checkpoint")
-
-
-def test_pan2_layerwise_offload_hooks_the_main_blocks(single_rank_dist):
-    native = _build_native(_small_kwargs())
-
-    first_module_list = next(child for child in native.children() if isinstance(child, torch.nn.ModuleList))
-
-    assert first_module_list is native.transformer_blocks
-
-
-@pytest.mark.parametrize("batch_size", [1, 2])
-def test_pan2_random_weights_forward_parity(single_rank_dist, batch_size: int):
-    kwargs = _small_kwargs()
-    reference = ReferencePAN2Transformer(**kwargs).float().eval()
-    # Zero-initialized tables (text modulation, type embedding) must be exercised too.
-    _randomize(reference, seed=1234)
-    native = _build_native(kwargs)
-    _load_into_native(native, reference.state_dict().items())
-
-    inputs = _make_inputs(kwargs["in_channels"], kwargs["text_embed_dim"], batch_size=batch_size, seed=batch_size)
-    _compare(native, reference, inputs, f"random weights, batch {batch_size}")
-
-
-def _run_sp_worker(sp_size: int, output_path: Path) -> None:
-    from fastvideo.configs.models.dits.pan2 import PAN2VideoConfig
-    from fastvideo.distributed import cleanup_dist_env_and_memory, maybe_init_distributed_environment_and_model_parallel
-    from fastvideo.models.dits.pan2 import PAN2Transformer3DModel
-    import torch.distributed as dist
-
-    if torch.cuda.is_available():
-        device = torch.device(f"cuda:{int(os.environ.get('LOCAL_RANK', '0'))}")
-        torch.cuda.set_device(device)
-        torch.backends.cuda.matmul.allow_tf32 = False
-        torch.backends.cudnn.allow_tf32 = False
-    else:
-        # GPU-less host: gloo ranks on CPU. fastvideo resolves the local device to mps there; swapping the code
-        # object reaches every module that imported get_local_torch_device by name.
-        import fastvideo.distributed.parallel_state as parallel_state
-
-        def _cpu_device() -> torch.device:
-            return torch.device("cpu")
-
-        parallel_state.get_local_torch_device.__code__ = _cpu_device.__code__
-        device = torch.device("cpu")
-    try:
-        maybe_init_distributed_environment_and_model_parallel(1, sp_size)
-        kwargs = _small_kwargs()
-        reference = ReferencePAN2Transformer(**kwargs).float()
-        _randomize(reference, seed=1234)
-        config = PAN2VideoConfig()
-        config.update_model_arch(kwargs)
-        native = PAN2Transformer3DModel(config=config, hf_config=kwargs).float().eval()
-        _load_into_native(native, reference.state_dict().items())
-        native = native.to(device)
-        outputs = {}
-        # 15 tokens need one pad at SP=2/4; 18 need two at SP=4, as at 704x1248x313.
-        for latent_frames in (5, 6):
-            for image_conditioned in (False, True):
-                inputs = _make_inputs(kwargs["in_channels"], kwargs["text_embed_dim"])
-                video = torch.randn(2, kwargs["in_channels"], latent_frames, 2, 6,
-                                    generator=torch.Generator().manual_seed(5))
-                video[:, kwargs["out_channels"]:] = 0
-                if image_conditioned:
-                    video[:, kwargs["out_channels"]:-1, :1] = torch.randn(
-                        2, kwargs["out_channels"], 1, 2, 6, generator=torch.Generator().manual_seed(6))
-                    video[:, -1:, :1] = 1
-                inputs["hidden_states"] = video
-                inputs = {name: value.to(device) for name, value in inputs.items()}
-                with torch.inference_mode(), set_forward_context(current_timestep=0, attn_metadata=None):
-                    output = native(**inputs)
-                outputs[f"{latent_frames}_{image_conditioned}"] = output.detach().cpu()
-        if int(os.environ.get("RANK", "0")) == 0:
-            torch.save(outputs, output_path)
-        dist.barrier()
-    finally:
-        cleanup_dist_env_and_memory()
-
-
-def _run_torchrun(nproc_per_node: int, output_path: Path) -> None:
-    command = [
-        sys.executable, "-m", "torch.distributed.run", "--nnodes", "1", "--nproc_per_node",
-        str(nproc_per_node), "--master_port",
-        str(_free_port()),
-        str(Path(__file__).resolve()), "--pan2-sp-worker", "--sp-size", str(nproc_per_node), "--output",
-        str(output_path)
-    ]
-    environment = os.environ.copy()
-    environment["PYTHONPATH"] = os.pathsep.join(filter(None, [str(REPO_ROOT), environment.get("PYTHONPATH")]))
-    environment["FASTVIDEO_ATTENTION_BACKEND"] = "TORCH_SDPA"
-    process = subprocess.run(command, capture_output=True, text=True, env=environment, timeout=180)
-    if process.returncode != 0:
-        raise RuntimeError(f"SP={nproc_per_node} worker failed with code {process.returncode}\n"
-                           f"STDOUT:\n{process.stdout}\nSTDERR:\n{process.stderr}")
-
-
-@pytest.fixture(scope="module")
-def single_rank_sp_outputs(tmp_path_factory):
-    path = tmp_path_factory.mktemp("pan2-single") / "single.pt"
-    _run_torchrun(1, path)
-    return torch.load(path, map_location="cpu", weights_only=True)
-
-
-@pytest.mark.parametrize("sp_size", SP_WORLD_SIZES)
-def test_pan2_sequence_parallel_matches_single_rank(tmp_path: Path, single_rank_sp_outputs, sp_size: int):
-    """T2V and I2V joint attention excludes padding at Ulysses SP=2/4. Uses gloo on CPU or NCCL on 2/4 GPUs."""
-    if torch.cuda.is_available() and torch.cuda.device_count() < sp_size:
-        pytest.skip(f"PAN2 SP parity requires {sp_size} CUDA devices.")
-    sp_path = tmp_path / f"sp{sp_size}.pt"
-    _run_torchrun(sp_size, sp_path)
-    sp_outputs = torch.load(sp_path, map_location="cpu", weights_only=True)
-    assert sp_outputs.keys() == single_rank_sp_outputs.keys()
-    for case, single in single_rank_sp_outputs.items():
-        sp = sp_outputs[case]
-        diff = (sp - single).abs()
-        print(f"sp{sp_size}-vs-single {case} diff_max={diff.max().item():.3e}")
-        assert_close(sp, single, atol=1e-5, rtol=1e-5)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--pan2-sp-worker", action="store_true")
-    parser.add_argument("--sp-size", type=int, choices=[1, *SP_WORLD_SIZES], required=True)
-    parser.add_argument("--output", type=str, required=True)
-    args = parser.parse_args()
-    if not args.pan2_sp_worker:
-        raise SystemExit("This module is intended to be run by pytest.")
-    _run_sp_worker(args.sp_size, Path(args.output))

tests/local_tests/vaes/test_pan2_vae_parity.py

@@ -1,387 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""PAN2 reuse of FastVideo's Wan2.2 (48-channel) VAE.
-
-Coverage scope: both. PAN2 ships a Diffusers ``AutoencoderKLWan`` in the
-Wan2.2 layout (z_dim 48, residual blocks, 2x2 patchify, 16x spatial and 4x
-temporal compression). The FastVideo side loads it through ``VAELoader``, the
-path a pipeline uses, and is compared on CPU fp32 against
-``diffusers.AutoencoderKLWan.from_pretrained``:
-
-* posterior mode/logvar for a single first frame (PAN2's I2V conditioning),
-* posterior mode/logvar for a 9-frame clip (4x temporal downsampling + cache),
-* decode of 3 latent frames (first-chunk handling + temporal upsampling cache).
-
-The normalization tests use a copy of the checkpoint with non-trivial
-``latents_mean``/``latents_std`` (the tiny checkpoint ships 0/1, which would
-hide a missing or doubled normalization) and pin where it happens: the raw
-``encode``/``decode`` never normalize, ``PAN2ConditioningStage`` applies
-PAN2's ``(z - mean) / std`` once, and ``DecodingStage`` applies
-``z * std + mean`` once.
-
-The default asset is the random-weight tiny PAN2 checkpoint, so this runs on a
-CPU-only host::
-
-    pytest tests/local_tests/vaes/test_pan2_vae_parity.py -v -s
-
-Set ``PAN2_MODEL_PATH`` to a local PAN2 root containing ``vae/`` to use other
-weights.
-"""
-
-from __future__ import annotations
-
-import json
-import os
-from pathlib import Path
-from types import SimpleNamespace
-
-import pytest
-import torch
-from safetensors.torch import load_file
-from torch.testing import assert_close
-
-try:
-    from diffusers import AutoencoderKLWan as DiffusersAutoencoderKLWan
-except ImportError:  # pragma: no cover - environment dependent
-    pytest.skip("diffusers with AutoencoderKLWan is required", allow_module_level=True)
-
-from fastvideo.models.loader.component_loader import VAELoader
-from fastvideo.models.wan.vae import AutoencoderKLWan as FastVideoAutoencoderKLWan
-from fastvideo.models.wan.vae_config import WanVAEConfig
-
-PARITY_SCOPE = "both"
-TINY_PAN2_REPO = "wuqing157/tiny-pan2-modular-pipe"
-TINY_PAN2_REVISION = "3ff3130b4527f61aaab7bde01465c9884dc84cdd"
-MODEL_PATH_ENV = "PAN2_MODEL_PATH"
-
-# H and W must be multiples of 32 (16x spatial compression times the 2x2
-# patchify inside the VAE).
-HEIGHT, WIDTH = 64, 96
-VIDEO_FRAMES = 9
-LATENT_FRAMES = (VIDEO_FRAMES - 1) // 4 + 1
-ATOL = RTOL = 1e-5
-
-# Structural fields of Wan-AI/Wan2.2-TI2V-5B-Diffusers vae/config.json. The
-# statistics do not affect parameter shapes, so placeholders keep this inline.
-WAN22_TI2V_VAE_CONFIG = {
-    "attn_scales": [],
-    "base_dim": 160,
-    "clip_output": False,
-    "decoder_base_dim": 256,
-    "dim_mult": [1, 2, 4, 4],
-    "dropout": 0.0,
-    "in_channels": 12,
-    "is_residual": True,
-    "latents_mean": [0.0] * 48,
-    "latents_std": [1.0] * 48,
-    "num_res_blocks": 2,
-    "out_channels": 12,
-    "patch_size": 2,
-    "scale_factor_spatial": 16,
-    "scale_factor_temporal": 4,
-    "temperal_downsample": [False, True, True],
-    "z_dim": 48,
-}
-
-
-def _pan2_root(subfolder: str) -> Path:
-    """Resolve a PAN2 root that contains ``subfolder``; skip when unavailable."""
-    override = os.getenv(MODEL_PATH_ENV)
-    if override:
-        root = Path(override)
-    else:
-        try:
-            from huggingface_hub import snapshot_download
-        except ImportError:
-            pytest.skip("huggingface_hub is required to resolve the tiny PAN2 checkpoint")
-        root = None
-        last_error: Exception | None = None
-        for local_files_only in (True, False):
-            try:
-                root = Path(
-                    snapshot_download(
-                        TINY_PAN2_REPO,
-                        revision=TINY_PAN2_REVISION,
-                        allow_patterns=[f"{subfolder}/*"],
-                        local_files_only=local_files_only,
-                    ))
-                break
-            except Exception as exc:  # noqa: BLE001 - offline/missing cache must skip.
-                last_error = exc
-        if root is None:
-            pytest.skip(f"Tiny PAN2 checkpoint unavailable: {last_error}")
-    if not (root / subfolder).is_dir():
-        pytest.skip(f"PAN2 {subfolder}/ missing under {root}")
-    return root
-
-
-@pytest.fixture(scope="module")
-def vae_dir() -> Path:
-    path = _pan2_root("vae") / "vae"
-    for name in ("config.json", "diffusion_pytorch_model.safetensors"):
-        if not (path / name).is_file():
-            pytest.skip(f"PAN2 VAE asset missing: {path / name}")
-    return path
-
-
-@pytest.fixture
-def cpu_runtime(monkeypatch: pytest.MonkeyPatch) -> None:
-    """Pin FastVideo's device helper to CPU; it reports ``mps`` on CPU-only Linux."""
-    for module in (
-            "fastvideo.models.loader.component_loader",
-            "fastvideo.pipelines.stages.decoding",
-            "fastvideo.pipelines.basic.pan2.stages",
-    ):
-        monkeypatch.setattr(f"{module}.get_local_torch_device", lambda: torch.device("cpu"))
-
-
-def _pan2_vae_config() -> WanVAEConfig:
-    """The VAE config a PAN2 pipeline config would carry (arch filled by the loader)."""
-    config = WanVAEConfig()
-    config.load_encoder = True
-    config.load_decoder = True
-    return config
-
-
-def _load_fastvideo(vae_dir: Path) -> FastVideoAutoencoderKLWan:
-    """Load through the production VAELoader, exactly as a pipeline does."""
-    fastvideo_args = SimpleNamespace(
-        model_paths={},
-        pipeline_config=SimpleNamespace(vae_config=_pan2_vae_config(), vae_precision="fp32"),
-        vae_cpu_offload=False,
-    )
-    vae = VAELoader().load(str(vae_dir), fastvideo_args)
-    assert isinstance(vae, FastVideoAutoencoderKLWan)
-    assert fastvideo_args.model_paths["vae"] == str(vae_dir)
-    assert all(param.dtype == torch.float32 and param.device.type == "cpu" for param in vae.parameters())
-    return vae
-
-
-def _load_diffusers(vae_dir: Path) -> DiffusersAutoencoderKLWan:
-    model, info = DiffusersAutoencoderKLWan.from_pretrained(
-        str(vae_dir),
-        torch_dtype=torch.float32,
-        local_files_only=True,
-        output_loading_info=True,
-    )
-    problems = {key: value for key, value in info.items() if value}
-    assert not problems, f"Diffusers reference did not load strictly: {problems}"
-    return model.eval()
-
-
-def _pixels(num_frames: int, seed: int) -> torch.Tensor:
-    generator = torch.Generator().manual_seed(seed)
-    return torch.rand((1, 3, num_frames, HEIGHT, WIDTH), generator=generator) * 2.0 - 1.0
-
-
-def _latents(seed: int) -> torch.Tensor:
-    generator = torch.Generator().manual_seed(seed)
-    return torch.randn((1, 48, LATENT_FRAMES, HEIGHT // 16, WIDTH // 16), generator=generator)
-
-
-def _report(name: str, actual: torch.Tensor, expected: torch.Tensor, clamp_bound: float | None = None) -> None:
-    assert actual.shape == expected.shape, f"{name}: {tuple(actual.shape)} != {tuple(expected.shape)}"
-    diff = (actual.float() - expected.float()).abs()
-    message = (f"[pan2-vae] {name}: shape={tuple(actual.shape)} max_abs={diff.max().item():.3e} "
-               f"mean_abs={diff.mean().item():.3e} ref_absmax={expected.abs().max().item():.3e}")
-    if clamp_bound is not None:
-        # Clamped pixels compare trivially; report how many there are.
-        message += f" clamped={(expected.abs() >= clamp_bound).float().mean().item():.2%}"
-    print(message)
-
-
-def _nontrivial_stats() -> tuple[list[float], list[float]]:
-    mean = [round(0.05 * (index - 24), 4) for index in range(48)]
-    std = [round(0.4 + 0.03 * index, 4) for index in range(48)]
-    return mean, std
-
-
-@pytest.fixture
-def stats_vae_dir(vae_dir: Path, tmp_path: Path) -> Path:
-    """Checkpoint copy with non-trivial latent statistics (weights symlinked)."""
-    config = json.loads((vae_dir / "config.json").read_text())
-    config["latents_mean"], config["latents_std"] = _nontrivial_stats()
-    target = tmp_path / "vae"
-    target.mkdir()
-    (target / "config.json").write_text(json.dumps(config))
-    (target / "diffusion_pytorch_model.safetensors").symlink_to(
-        (vae_dir / "diffusion_pytorch_model.safetensors").resolve())
-    return target
-
-
-def _diffusers_stats(model: DiffusersAutoencoderKLWan, like: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
-    mean = torch.tensor(model.config.latents_mean).view(1, -1, 1, 1, 1).to(like)
-    std = torch.tensor(model.config.latents_std).view(1, -1, 1, 1, 1).to(like)
-    return mean, std
-
-
-def test_pan2_vae_production_loader_matches_checkpoint(vae_dir: Path, cpu_runtime: None) -> None:
-    """The loader fills the tiny Wan2.2 arch from config.json and covers every weight.
-
-    ``VAELoader`` loads ``AutoencoderKLWan`` with ``strict=False``, so key
-    coverage is checked here instead of trusting the loader.
-    """
-    raw = json.loads((vae_dir / "config.json").read_text())
-    vae = _load_fastvideo(vae_dir)
-    arch = vae.config.arch_config
-    for key in ("base_dim", "decoder_base_dim", "z_dim", "num_res_blocks", "in_channels", "out_channels",
-                "is_residual", "patch_size", "scale_factor_spatial", "scale_factor_temporal", "dropout"):
-        assert getattr(arch, key) == raw[key], key
-    for key in ("dim_mult", "attn_scales", "temperal_downsample", "latents_mean", "latents_std"):
-        assert list(getattr(arch, key)) == list(raw[key]), key
-    assert vae.config.use_feature_cache and not vae.config.use_tiling
-    assert arch.spatial_compression_ratio == 16 and arch.temporal_compression_ratio == 4
-    assert tuple(vae.scaling_factor.shape) == tuple(vae.shift_factor.shape) == (1, 48, 1, 1, 1)
-    assert_close(vae.scaling_factor.flatten(), 1.0 / torch.tensor(raw["latents_std"]))
-    assert_close(vae.shift_factor.flatten(), torch.tensor(raw["latents_mean"]))
-
-    checkpoint = load_file(str(vae_dir / "diffusion_pytorch_model.safetensors"))
-    state = vae.state_dict()
-    assert set(state) == set(checkpoint), (sorted(set(checkpoint) - set(state)), sorted(set(state) - set(checkpoint)))
-    for key, tensor in checkpoint.items():
-        assert torch.equal(state[key], tensor.float()), key
-
-
-@pytest.mark.parametrize("variant", ["wan22_ti2v_5b", "tiny_pan2"])
-def test_pan2_vae_config_shapes_match_diffusers(variant: str, request: pytest.FixtureRequest) -> None:
-    """``WanVAEConfig`` expresses both the real Wan2.2 and the tiny PAN2 VAE configs.
-
-    Both models are built on the meta device, so the full-size config costs no
-    memory; parameter names and shapes must match one-to-one.
-    """
-    if variant == "tiny_pan2":
-        raw = json.loads((request.getfixturevalue("vae_dir") / "config.json").read_text())
-        config = {key: value for key, value in raw.items() if not key.startswith("_")}
-    else:
-        config = WAN22_TI2V_VAE_CONFIG
-    fastvideo_config = _pan2_vae_config()
-    fastvideo_config.update_model_arch(config)
-    with torch.device("meta"):
-        fastvideo = FastVideoAutoencoderKLWan(fastvideo_config)
-        reference = DiffusersAutoencoderKLWan.from_config(config)
-    fastvideo_shapes = {key: tuple(value.shape) for key, value in fastvideo.state_dict().items()}
-    reference_shapes = {key: tuple(value.shape) for key, value in reference.state_dict().items()}
-    assert fastvideo_shapes == reference_shapes
-
-
-def test_pan2_vae_wan22_topology_random_weight_parity() -> None:
-    """Numerically cover the Wan2.2 paths the tiny config skips.
-
-    The tiny checkpoint uses ``dim_mult`` [1, 1, 1, 1] and no separate decoder
-    width, so channel-changing shortcuts and resamplers never run. This narrow
-    random-weight model keeps the real topology (``dim_mult`` [1, 2, 4, 4], two
-    res blocks, ``decoder_base_dim`` > ``base_dim``).
-    """
-    config = dict(WAN22_TI2V_VAE_CONFIG, base_dim=8, decoder_base_dim=16)
-    torch.manual_seed(0)
-    reference = DiffusersAutoencoderKLWan.from_config(config).eval()
-    fastvideo_config = _pan2_vae_config()
-    fastvideo_config.update_model_arch(config)
-    fastvideo = FastVideoAutoencoderKLWan(fastvideo_config).eval()
-    fastvideo.load_state_dict(reference.state_dict(), strict=True)
-
-    video = _pixels(VIDEO_FRAMES, seed=11)
-    latents = _latents(seed=12)
-    with torch.inference_mode():
-        expected_mode = reference.encode(video).latent_dist.mode()
-        actual_mode = fastvideo.encode(video).mode()
-        expected_video = reference.decode(latents).sample
-        actual_video = fastvideo.decode(latents)
-    _report("wan22-topology encode mode", actual_mode, expected_mode)
-    _report("wan22-topology decode", actual_video, expected_video, clamp_bound=1.0)
-    assert_close(actual_mode, expected_mode, atol=ATOL, rtol=RTOL)
-    assert_close(actual_video, expected_video, atol=ATOL, rtol=RTOL)
-
-
-@pytest.mark.parametrize("num_frames", [1, VIDEO_FRAMES], ids=["first_frame", "video_9f"])
-def test_pan2_vae_encode_matches_diffusers(num_frames: int, vae_dir: Path, cpu_runtime: None) -> None:
-    """Posterior mode (PAN2 uses ``latent_dist.mode()``) and logvar match Diffusers.
-
-    FastVideo's ``encode`` returns the ``DiagonalGaussianDistribution`` itself,
-    so the port calls ``vae.encode(x).mode()`` instead of ``.latent_dist.mode()``.
-    """
-    reference = _load_diffusers(vae_dir)
-    fastvideo = _load_fastvideo(vae_dir)
-    pixels = _pixels(num_frames, seed=100 + num_frames)
-    with torch.inference_mode():
-        expected = reference.encode(pixels).latent_dist
-        actual = fastvideo.encode(pixels)
-    assert tuple(actual.mode().shape) == (1, 48, (num_frames - 1) // 4 + 1, HEIGHT // 16, WIDTH // 16)
-    _report(f"encode mode ({num_frames}f)", actual.mode(), expected.mode())
-    _report(f"encode logvar ({num_frames}f)", actual.logvar, expected.logvar)
-    assert_close(actual.mode(), expected.mode(), atol=ATOL, rtol=RTOL)
-    assert_close(actual.logvar, expected.logvar, atol=ATOL, rtol=RTOL)
-
-
-def test_pan2_vae_decode_matches_diffusers(vae_dir: Path, cpu_runtime: None) -> None:
-    """Decode three latent frames so first-chunk and temporal cache paths run."""
-    reference = _load_diffusers(vae_dir)
-    fastvideo = _load_fastvideo(vae_dir)
-    latents = _latents(seed=7)
-    with torch.inference_mode():
-        expected = reference.decode(latents).sample
-        actual = fastvideo.decode(latents)
-    assert tuple(actual.shape) == (1, 3, VIDEO_FRAMES, HEIGHT, WIDTH)
-    _report("decode", actual, expected, clamp_bound=1.0)
-    assert_close(actual, expected, atol=ATOL, rtol=RTOL)
-
-
-def test_pan2_conditioning_stage_matches_reference(stats_vae_dir: Path, cpu_runtime: None) -> None:
-    """``PAN2ConditioningStage`` builds the reference I2V condition: cover-resize and center crop the first frame,
-    encode it, apply ``(z - mean) / std`` once, and set the mask channel on the first latent frame only.
-
-    The raw ``encode`` is unnormalized, so the stage is the only place the statistics apply.
-    """
-    import PIL.Image
-
-    from fastvideo.pipelines.basic.pan2.stages import PAN2ConditioningStage
-    from tests.local_tests.pan2.reference import reference_condition_latents
-
-    reference = _load_diffusers(stats_vae_dir)
-    fastvideo = _load_fastvideo(stats_vae_dir)
-    # Not the output aspect ratio, so the resize and crop are exercised.
-    image = PIL.Image.radial_gradient("L").resize((120, 64)).convert("RGB")
-    latents = torch.zeros(1, 48, LATENT_FRAMES, HEIGHT // 16, WIDTH // 16)
-    with torch.inference_mode():
-        pixels = PAN2ConditioningStage.preprocess_image(image, HEIGHT, WIDTH)
-        raw = reference.encode(pixels).latent_dist.mode()
-        assert_close(fastvideo.encode(pixels).mode(), raw, atol=ATOL, rtol=RTOL)
-        expected = reference_condition_latents(reference, latents, image, HEIGHT, WIDTH)
-
-        batch = SimpleNamespace(latents=latents, pil_image=image, height=HEIGHT, width=WIDTH, image_latent=None)
-        stage_args = SimpleNamespace(pipeline_config=SimpleNamespace(vae_precision="fp32"), vae_cpu_offload=False)
-        actual = PAN2ConditioningStage(fastvideo).forward(batch, stage_args).image_latent
-    _report("PAN2 I2V condition", actual, expected)
-    assert actual.shape == (1, 49, LATENT_FRAMES, HEIGHT // 16, WIDTH // 16)
-    assert_close(actual, expected, atol=ATOL, rtol=RTOL)
-    assert torch.equal(actual[:, 48, 0], torch.ones_like(actual[:, 48, 0]))
-    assert not actual[:, :, 1:].any()
-    assert not torch.allclose(raw, expected[:, :48, :1], atol=1e-2), "statistics too close to identity"
-
-
-def test_pan2_decoding_stage_denormalizes_once(stats_vae_dir: Path, cpu_runtime: None) -> None:
-    """``DecodingStage`` performs PAN2's ``z * std + mean`` itself.
-
-    The PAN2 denoising loop must hand normalized latents to ``DecodingStage``.
-    Denormalizing beforehand (as the upstream pipeline does inline) would apply
-    the statistics twice, which the negative check below detects.
-    """
-    from fastvideo.pipelines.stages.decoding import DecodingStage
-
-    reference = _load_diffusers(stats_vae_dir)
-    fastvideo = _load_fastvideo(stats_vae_dir)
-    latents = _latents(seed=31)
-    # vae_tiling=True is the PipelineConfig default; the feature-cache decode ignores tiling, so it must not matter.
-    decode_args = SimpleNamespace(
-        pipeline_config=SimpleNamespace(vae_precision="fp32", vae_decode_precision="fp32", vae_tiling=True),
-        disable_autocast=False,
-    )
-    stage = DecodingStage(fastvideo)
-    with torch.inference_mode():
-        mean, std = _diffusers_stats(reference, latents)
-        expected = reference.decode(latents * std + mean).sample
-        expected = (expected * 0.5 + 0.5).clamp(0, 1)
-        actual = stage.decode(latents, decode_args)
-        doubled = stage.decode(latents * std + mean, decode_args)
-    _report("DecodingStage output", actual, expected)
-    assert_close(actual, expected, atol=ATOL, rtol=RTOL)
-    assert (doubled - expected).abs().max().item() > 1e-3, "double denormalization went undetected"