From 62fc3548244b7f10260b1ac18775596ddd8d5c76 Mon Sep 17 00:00:00 2001 From: aryan5v Date: Thu, 29 Oct 2026 15:17:41 -0700 Subject: [PATCH] [feat]: wire NVFP4 decoder and text encoder into Pika H3 Flex Expose the exported H3 NVFP4 VAE decoder through ComponentConfig and map it to FastVideoArgs, alongside the existing text-encoder weight override. Add example flags and docs for the 36L quality and 8L fast decoder tiers, including the 8L depth-cut and no-NVFP4-VAE-encoder caveats. Add compat translation tests. --- .../inference_schema_parity_inventory.yaml | 2 +- docs/inference/pika_h3_flex.md | 41 +++++++++++++++++++ .../inference/basic/basic_pika_h3_flex.py | 12 +++++- fastvideo/api/compat.py | 4 ++ fastvideo/api/schema.py | 1 + .../tests/api/test_compat_translation.py | 33 ++++++++++++++- 6 files changed, 90 insertions(+), 3 deletions(-) diff --git a/docs/design/inference_schema_parity_inventory.yaml b/docs/design/inference_schema_parity_inventory.yaml index 9027bb660..ca98232dc 100644 --- a/docs/design/inference_schema_parity_inventory.yaml +++ b/docs/design/inference_schema_parity_inventory.yaml @@ -88,7 +88,7 @@ surfaces: video_decode_backend: "MiniMax-H3 video decoder selection (full VAE vs TAEH3 preview); model-specific optimization not yet represented in the typed public schema." taeh3_checkpoint: "Optional local TAEH3 safetensors path; model-specific optimization not yet represented in the typed public schema." taeh3_chunk_size: "TAEH3 temporal chunk length; model-specific optimization not yet represented in the typed public schema." - h3_vae_nvfp4_decoder: "Exported NVFP4 MiniMax-H3 video decoder file that replaces the VAE decoder at load; model-specific optimization not yet represented in the typed public schema." + h3_vae_nvfp4_decoder: "Exported NVFP4 MiniMax-H3 video decoder file that replaces the VAE decoder at load; represented as ComponentConfig.h3_vae_nvfp4_decoder." vae_parallel_encode: "MiniMax-H3 sequence-parallel reference VAE encode opt-in; model-specific optimization not yet represented in the typed public schema." vae_parallel_decode_strategy: "Chunk-transport collective for vae_parallel_decode; model-specific optimization not yet represented in the typed public schema." attention_backend: "Process-wide default attention-backend request applied per component at load time; kernel-selection knob not yet represented in the typed public schema." diff --git a/docs/inference/pika_h3_flex.md b/docs/inference/pika_h3_flex.md index 31c223384..c43dbd8e7 100644 --- a/docs/inference/pika_h3_flex.md +++ b/docs/inference/pika_h3_flex.md @@ -97,6 +97,47 @@ lossless switches: starts. Kernel work beyond the existing exact kernels is intentionally not part of this integration. +## NVFP4 on Blackwell + +Pika can reuse FastVideo's existing H3 NVFP4 VAE decoder and NVFP4 Qwen3-VL +text encoder. Both are weight formats, not new sampler paths; the six-forward +Pika schedule is unchanged. + +```bash +# 36-block quality decoder: needs the full 36-block H3 VAE +python examples/inference/basic/basic_pika_h3_flex.py --task t2va \ + --prompt "a paper boat drifting down a rainy street" \ + --nvfp4-decoder ./FastH3-NVFP4-VAE-Decoder/quality-36L/decoder-nvfp4.safetensors + +# 8-block fast decoder: depth-cuts any H3 VAE with at least 8 blocks +python examples/inference/basic/basic_pika_h3_flex.py --task ref2va \ + --reference image:person.png --reference video:dance.mp4 \ + --prompt "the person dances to the music" \ + --nvfp4-decoder ./FastH3-NVFP4-VAE-Decoder/fast-8L/decoder-nvfp4.safetensors \ + --nvfp4-text-encoder ./fasth3-nvfp4-encoder/text_encoder +``` + +* `quality-36L` keeps the full decoder depth and is the closest match to Pika's + dense decoder. `fast-8L` is much smaller and faster but is a distilled + student; expect a visible quality drop. Pika's six-step re-noise latents are + not the student's original training distribution, so validate 8L on your + prompts before using it for delivery. +* The NVFP4 decoder replaces only `vae.decoder`; keyframe and reference + encoding still use the FP32 VAE encoder. +* The NVFP4 text encoder quantizes the Qwen3-VL language stack while keeping + the vision tower in bf16, so Ref2VA reference images/videos still work. +* Do not select `taeh3` with an NVFP4 decoder; `h3-vae` is required. +* Both options require Blackwell FP4 hardware. The fused decoder kernels and + CUDA graphs are already enabled by default; see + `FASTVIDEO_H3_VAE_NVFP4_FUSED` and `FASTVIDEO_H3_VAE_NVFP4_CUDA_GRAPH`. +* There is no NVFP4 **VAE encoder** path. The H3 video VAE encoder is a causal + 3D CNN and stays FP32; quantizing it would need a separate convolution + kernel/quantization effort. + +If a process first loads the 8L decoder it depth-cuts the VAE to eight blocks; +loading 36L afterwards in the same process is not supported. Choose one decoder +tier per process. + ## Hardware The selected H3 trunk plus Qwen3-VL does not fit comfortably on a 24 GB card diff --git a/examples/inference/basic/basic_pika_h3_flex.py b/examples/inference/basic/basic_pika_h3_flex.py index 19da44082..f0383d62c 100644 --- a/examples/inference/basic/basic_pika_h3_flex.py +++ b/examples/inference/basic/basic_pika_h3_flex.py @@ -12,7 +12,9 @@ Five task shapes share one model release: The six-forward ladder is part of the adapter contract; this script deliberately does not expose a step-count flag. ``--lossless-accel`` turns on the existing training-free FastVideo switches (bit-identical kernels, reference memo, -parallel VAE tiles) before the generator starts. Kernel/VSA work is separate. +parallel VAE tiles) before the generator starts. ``--nvfp4-decoder`` and +``--nvfp4-text-encoder`` opt into the existing Blackwell FP4 weight paths; +they change weights, not the six-forward sampler. Kernel/VSA work is separate. Examples: python examples/inference/basic/basic_pika_h3_flex.py --task t2va \ @@ -90,6 +92,10 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--compile", action="store_true", help="torch.compile the DiT") parser.add_argument("--lossless-accel", action="store_true", help="enable bit-identical speed-ups") parser.add_argument("--low-vram", action="store_true", help="stream the DiT layerwise") + parser.add_argument("--nvfp4-decoder", + help="Blackwell-only NVFP4 H3 decoder (quality-36L or fast-8L decoder-nvfp4.safetensors)") + parser.add_argument("--nvfp4-text-encoder", + help="Blackwell-only serialized NVFP4 Qwen3-VL text_encoder directory") parser.add_argument("--output", default="outputs/pika_h3_flex") args = parser.parse_args() @@ -117,6 +123,10 @@ def _pipeline(args: argparse.Namespace) -> PipelineSelection: components = ComponentConfig() if args.task == "ref2va": components.override_pipeline_cls_name = "PikaH3FlexRef2VAModularPipeline" + if args.nvfp4_decoder: + components.h3_vae_nvfp4_decoder = args.nvfp4_decoder + if args.nvfp4_text_encoder: + components.text_encoder_weights = args.nvfp4_text_encoder return PipelineSelection( workload_type="t2v" if args.task == "t2va" else "i2v", components=components, diff --git a/fastvideo/api/compat.py b/fastvideo/api/compat.py index e199bc8f0..9e7c23202 100644 --- a/fastvideo/api/compat.py +++ b/fastvideo/api/compat.py @@ -199,6 +199,8 @@ def legacy_from_pretrained_to_config( components["transformer_weights"] = value elif key == "init_weights_from_safetensors_2": components["transformer_2_weights"] = value + elif key == "h3_vae_nvfp4_decoder": + components["h3_vae_nvfp4_decoder"] = value else: experimental[key] = deepcopy(value) @@ -331,6 +333,8 @@ def generator_config_to_fastvideo_args(config: GeneratorConfig | Mapping[str, An kwargs["init_weights_from_safetensors"] = components.transformer_weights if components.transformer_2_weights is not None: kwargs["init_weights_from_safetensors_2"] = components.transformer_2_weights + if components.h3_vae_nvfp4_decoder is not None: + kwargs["h3_vae_nvfp4_decoder"] = components.h3_vae_nvfp4_decoder if components.config_root is not None: kwargs["config_model_path"] = components.config_root if components.upsampler_weights is not None: diff --git a/fastvideo/api/schema.py b/fastvideo/api/schema.py index 0ece1ab24..44222d157 100644 --- a/fastvideo/api/schema.py +++ b/fastvideo/api/schema.py @@ -120,6 +120,7 @@ class ComponentConfig: transformer_weights: str | None = None transformer_2_weights: str | None = None vae_weights: str | None = None + h3_vae_nvfp4_decoder: str | None = None upsampler_weights: str | None = None lora_path: str | None = None lora_nickname: str = "default" diff --git a/fastvideo/tests/api/test_compat_translation.py b/fastvideo/tests/api/test_compat_translation.py index 2c8002eda..f84545532 100644 --- a/fastvideo/tests/api/test_compat_translation.py +++ b/fastvideo/tests/api/test_compat_translation.py @@ -11,7 +11,7 @@ from fastvideo.api.compat import ( request_to_sampling_param, ) from fastvideo.api.parser import parse_config -from fastvideo.api.schema import CompileConfig, GenerationRequest, GeneratorConfig +from fastvideo.api.schema import CompileConfig, ComponentConfig, GenerationRequest, GeneratorConfig, PipelineSelection from fastvideo.api.sampling_param import SamplingParam @@ -78,6 +78,37 @@ class TestCompileConfigRoundTrip: assert args.kwargs["torch_compile_kwargs"] == {} +class TestH3Nvfp4ComponentTranslation: + """Pika/H3 NVFP4 decoder and text-encoder overrides stay on ComponentConfig.""" + + def test_decoder_and_text_encoder_translate(self, monkeypatch) -> None: + _stub_fastvideo_args_from_kwargs(monkeypatch) + config = GeneratorConfig( + model_path="/models/pika", + pipeline=PipelineSelection(components=ComponentConfig( + h3_vae_nvfp4_decoder="/models/quality-36L/decoder-nvfp4.safetensors", + text_encoder_weights="/models/nvfp4-encoder/text_encoder", + )), + ) + args = generator_config_to_fastvideo_args(config) + assert args.kwargs["h3_vae_nvfp4_decoder"] == "/models/quality-36L/decoder-nvfp4.safetensors" + assert args.kwargs["override_text_encoder_safetensors"] == "/models/nvfp4-encoder/text_encoder" + + def test_legacy_flat_decoder_promotes_to_component(self) -> None: + config = legacy_from_pretrained_to_config( + "/models/pika", + {"h3_vae_nvfp4_decoder": "/models/fast-8L/decoder-nvfp4.safetensors"}, + ) + assert config.pipeline.components.h3_vae_nvfp4_decoder == ( + "/models/fast-8L/decoder-nvfp4.safetensors") + + def test_unset_fields_are_not_emitted(self, monkeypatch) -> None: + _stub_fastvideo_args_from_kwargs(monkeypatch) + args = generator_config_to_fastvideo_args(GeneratorConfig(model_path="/models/pika")) + assert "h3_vae_nvfp4_decoder" not in args.kwargs + assert "override_text_encoder_safetensors" not in args.kwargs + + class TestLegacyLtx2VaeTilingTranslation: """``ltx2_vae_tiling`` flat kwarg promotes to ``generator.pipeline.vae_tiling``; reverse direction emits the