From 6b5eef47112f4b480704d0c3f0635d20ad5a972a Mon Sep 17 00:00:00 2001 From: William Lin Date: Sat, 7 Nov 2026 10:27:53 +0000 Subject: [PATCH] [bugfix]: Preserve PAN2 negative prompt (#2029) Preserve PAN2's trained single-space unconditional prompt through shared normalization and PAN2 input validation. Add scoped regression coverage and remove redundant comments. --- examples/inference/basic/basic_pan2.py | 2 - fastvideo/configs/pipelines/base.py | 4 +- fastvideo/configs/pipelines/pan2.py | 4 +- fastvideo/entrypoints/video_generator.py | 5 +- fastvideo/pipelines/basic/pan2/stages.py | 14 +++-- .../tests/entrypoints/test_video_generator.py | 32 ++++++++++++ .../tests/stages/test_pan2_negative_prompt.py | 52 ++++++++++++++++++- .../tests/transformers/test_pan2_fusions.py | 10 ---- 8 files changed, 103 insertions(+), 20 deletions(-) diff --git a/examples/inference/basic/basic_pan2.py b/examples/inference/basic/basic_pan2.py index 06cd63453..2a2f1ee8a 100644 --- a/examples/inference/basic/basic_pan2.py +++ b/examples/inference/basic/basic_pan2.py @@ -120,8 +120,6 @@ NUM_FRAMES = 241 # 10 s at 24 fps, the caption's length. OUTPUT_PATH = "video_samples_pan2" MODEL_PATH = os.environ.get("PAN2_MODEL_PATH", "IFM/PAN2") -# Set to a first frame (e.g. "assets/images/girl.png") for image-to-video; the same pipeline runs both modes. The -# caption's `init_frame_visual_description` should then describe that frame. IMAGE_PATH = None diff --git a/fastvideo/configs/pipelines/base.py b/fastvideo/configs/pipelines/base.py index 7faf283b4..c76ff8dac 100644 --- a/fastvideo/configs/pipelines/base.py +++ b/fastvideo/configs/pipelines/base.py @@ -2,7 +2,7 @@ import json from collections.abc import Callable from dataclasses import asdict, dataclass, field, fields -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, ClassVar, cast import torch @@ -76,6 +76,8 @@ class PipelineConfig: image_encoder_precisions: tuple[str, ...] | None = None # Text encoder configuration + # This is a model-fixed normalization policy, not a serialized request knob. + preserve_single_space_negative_prompt: ClassVar[bool] = False DEFAULT_TEXT_ENCODER_PRECISIONS = ("fp32", ) text_encoder_configs: tuple[EncoderConfig, ...] = field(default_factory=lambda: (EncoderConfig(), )) text_encoder_precisions: tuple[str, ...] = field(default_factory=lambda: ("fp32", )) diff --git a/fastvideo/configs/pipelines/pan2.py b/fastvideo/configs/pipelines/pan2.py index 6ddde8cd4..70a8e0e2c 100644 --- a/fastvideo/configs/pipelines/pan2.py +++ b/fastvideo/configs/pipelines/pan2.py @@ -2,6 +2,7 @@ """PAN2 pipeline configuration: text-to-video, and image-to-video from a first frame.""" from dataclasses import dataclass, field +from typing import ClassVar from fastvideo.configs.models import DiTConfig, EncoderConfig, VAEConfig from fastvideo.configs.models.dits import PAN2VideoConfig @@ -51,6 +52,8 @@ class PAN2PipelineConfig(PipelineConfig): # Keep the guard models on CPU and move each to the GPU only for its check. offload_guardrail_models: bool = False + # A single space is PAN2's trained unconditional prompt; normalization must not turn it into an empty string. + preserve_single_space_negative_prompt: ClassVar[bool] = True # Prompt encoding: PAN2 conditions on the tokens after the chat template's user header, taken from the text # encoder's hidden states with its last two layers skipped (``hidden_states[-3]``). text_system_prompt: str = PAN2_SYSTEM_PROMPT @@ -58,7 +61,6 @@ class PAN2PipelineConfig(PipelineConfig): text_max_sequence_length: int = 1536 def __post_init__(self) -> None: - # The encoder is needed for the image-to-video first frame. self.vae_config.load_encoder = True self.vae_config.load_decoder = True self.text_encoder_configs[0].arch_config.output_hidden_states = True diff --git a/fastvideo/entrypoints/video_generator.py b/fastvideo/entrypoints/video_generator.py index 464145d07..f0bba8f7c 100644 --- a/fastvideo/entrypoints/video_generator.py +++ b/fastvideo/entrypoints/video_generator.py @@ -868,7 +868,10 @@ class VideoGenerator: sampling_param.prompt = prompt # Process negative prompt if sampling_param.negative_prompt is not None: - sampling_param.negative_prompt = sampling_param.negative_prompt.strip() + keep_single_space = (sampling_param.negative_prompt == " " and getattr( + fastvideo_args.pipeline_config, "preserve_single_space_negative_prompt", False)) + if not keep_single_space: + sampling_param.negative_prompt = sampling_param.negative_prompt.strip() # Validate dimensions if (sampling_param.height <= 0 or sampling_param.width <= 0 or sampling_param.num_frames <= 0): diff --git a/fastvideo/pipelines/basic/pan2/stages.py b/fastvideo/pipelines/basic/pan2/stages.py index a3c1b345b..2dec0fc90 100644 --- a/fastvideo/pipelines/basic/pan2/stages.py +++ b/fastvideo/pipelines/basic/pan2/stages.py @@ -30,6 +30,14 @@ from fastvideo.utils import PRECISION_TO_TYPE _USER_HEADER = "<|im_start|>user\n" +def _set_default_negative_prompt(batch: ForwardBatch) -> None: + if not getattr(batch, "do_classifier_free_guidance", False) or getattr(batch, "negative_prompt", None): + return + num_frames = batch.num_frames[0] if isinstance(batch.num_frames, list) else batch.num_frames + fps = batch.fps[0] if isinstance(batch.fps, list) else batch.fps + batch.negative_prompt = render_pan2_default_negative_prompt(num_frames, fps or 24) + + class PAN2InputValidationStage(InputValidationStage): """Rejects geometry the PAN2 VAE and patch embedding cannot represent. @@ -38,6 +46,7 @@ class PAN2InputValidationStage(InputValidationStage): """ def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch: + _set_default_negative_prompt(batch) image_path = batch.image_path if image_path is not None and not image_path.endswith(".mp4"): batch.pil_image = load_image(image_path, exif_transpose=False) @@ -71,10 +80,7 @@ class PAN2TextEncodingStage(TextEncodingStage): """ def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch: - if batch.do_classifier_free_guidance and not batch.negative_prompt: - num_frames = batch.num_frames[0] if isinstance(batch.num_frames, list) else batch.num_frames - fps = batch.fps[0] if isinstance(batch.fps, list) else batch.fps - batch.negative_prompt = render_pan2_default_negative_prompt(num_frames, fps or 24) + _set_default_negative_prompt(batch) return super().forward(batch, fastvideo_args) @staticmethod diff --git a/fastvideo/tests/entrypoints/test_video_generator.py b/fastvideo/tests/entrypoints/test_video_generator.py index 8143305fe..1573e20fc 100644 --- a/fastvideo/tests/entrypoints/test_video_generator.py +++ b/fastvideo/tests/entrypoints/test_video_generator.py @@ -634,6 +634,38 @@ def test_generate_single_video_latent_metadata_skips_cpu_materialization(tmp_pat assert result["video_path"] is None +@pytest.mark.parametrize(("preserve_space", "negative_prompt", "expected"), [ + (False, " ", ""), + (True, " ", " "), + (True, " custom negative ", "custom negative"), +]) +def test_generate_single_video_respects_single_space_negative_prompt_policy( + tmp_path, preserve_space, negative_prompt, expected): + output_batch = _single_video_output_batch(torch.zeros(1, 3, 2, 16, 16)) + fastvideo_args = _single_video_args() + fastvideo_args.pipeline_config.preserve_single_space_negative_prompt = preserve_space + generator = _single_video_generator(output_batch, fastvideo_args) + captured = {} + + def execute_forward(batch, args): + captured["batch"] = batch + return output_batch + + generator.executor.execute_forward = execute_forward + sampling = _small_sampling_param() + sampling.negative_prompt = negative_prompt + generator._prepare_output_path = lambda *args: str(tmp_path / "unused.mp4") + + generator._generate_single_video( + prompt="negative prompt", + sampling_param=sampling, + fastvideo_args=fastvideo_args, + output_path=str(tmp_path / "unused.mp4"), + ) + + assert captured["batch"].negative_prompt == expected + + def test_generate_single_video_non_output_rank_suppresses_all_user_payloads(tmp_path): audio = torch.ones(8) audio_latents = torch.ones(2) diff --git a/fastvideo/tests/stages/test_pan2_negative_prompt.py b/fastvideo/tests/stages/test_pan2_negative_prompt.py index 2c8c6342e..96c3b59df 100644 --- a/fastvideo/tests/stages/test_pan2_negative_prompt.py +++ b/fastvideo/tests/stages/test_pan2_negative_prompt.py @@ -2,13 +2,18 @@ """PAN2 falls back to the release's default negative prompt, rendered for the clip; no weights or CUDA needed.""" import json +from dataclasses import fields from types import SimpleNamespace import pytest +import torch +from fastvideo.configs.pipelines.base import PipelineConfig +from fastvideo.configs.pipelines.pan2 import PAN2PipelineConfig from fastvideo.pipelines.basic.pan2.negative_prompt import (PAN2_DEFAULT_NEGATIVE_PROMPT, render_pan2_default_negative_prompt) -from fastvideo.pipelines.basic.pan2.stages import PAN2TextEncodingStage +from fastvideo.pipelines.basic.pan2.stages import PAN2InputValidationStage, PAN2TextEncodingStage +from fastvideo.pipelines.pipeline_batch_info import ForwardBatch from fastvideo.pipelines.stages.text_encoding import TextEncodingStage @@ -29,6 +34,12 @@ def test_the_default_spans_the_clip_and_is_otherwise_the_release_template(): assert _time_ranges(json.loads(render_pan2_default_negative_prompt(313, 24))) == ["0 to 13"] * 3 +def test_single_space_preservation_is_a_pan2_policy_not_a_global_change(): + assert PAN2PipelineConfig.preserve_single_space_negative_prompt is True + assert PipelineConfig.preserve_single_space_negative_prompt is False + assert "preserve_single_space_negative_prompt" not in {field.name for field in fields(PAN2PipelineConfig)} + + @pytest.fixture def encode(monkeypatch): seen = [] @@ -58,3 +69,42 @@ def test_a_given_negative_prompt_or_a_space_is_kept(encode): def test_without_guidance_the_negative_prompt_is_left_alone(encode): assert encode("", cfg=False) == "" + + +@pytest.mark.parametrize("negative_prompt", [None, "", " ", "blurry"]) +def test_validation_then_text_encoding_resolves_the_negative_prompt(monkeypatch, negative_prompt): + batch = ForwardBatch( + data_type="video", + prompt="a red fox", + negative_prompt=negative_prompt, + height=32, + width=32, + num_frames=5, + fps=24, + seed=0, + num_inference_steps=2, + guidance_scale=2.5, + ) + args = SimpleNamespace(pipeline_config=SimpleNamespace( + text_encoder_configs=[SimpleNamespace()], + ti2v_task=False, + is_causal=False, + vae_config=SimpleNamespace(arch_config=SimpleNamespace(spatial_compression_ratio=8, + temporal_compression_ratio=4)), + dit_config=SimpleNamespace(arch_config=SimpleNamespace(patch_size=(1, 2, 2))), + )) + stage = PAN2TextEncodingStage([object()], [object()]) + prompts = [] + + def encode_text(prompt, *args, **kwargs): + prompts.append(prompt) + return [torch.ones(1, 2, 3)], [torch.ones(1, 2, dtype=torch.long)] + + monkeypatch.setattr(stage, "encode_text", encode_text) + assert PAN2InputValidationStage().forward(batch, args) is batch + assert stage.forward(batch, args) is batch + + expected = negative_prompt or render_pan2_default_negative_prompt(5, 24) + assert batch.negative_prompt == expected + assert prompts == ["a red fox", expected] + assert len(batch.prompt_embeds) == len(batch.negative_prompt_embeds) == 1 diff --git a/fastvideo/tests/transformers/test_pan2_fusions.py b/fastvideo/tests/transformers/test_pan2_fusions.py index 49f48e393..eaea99797 100644 --- a/fastvideo/tests/transformers/test_pan2_fusions.py +++ b/fastvideo/tests/transformers/test_pan2_fusions.py @@ -89,7 +89,6 @@ def _randomize(model: torch.nn.Module, seed: int) -> None: parameter.copy_(torch.randn(parameter.shape, generator=generator) * 0.1) - def _qkv(batch: int, seq: int, heads: int, dim: int, dtype: torch.dtype, seed: int = 0): generator = torch.Generator().manual_seed(seed) qkv = torch.randn(batch, seq, 3 * heads * dim, generator=generator).to(dtype) @@ -113,9 +112,6 @@ def _eager_qk(x: torch.Tensor, weight: torch.Tensor, eps: float, rope: tuple[tor return _apply_rotary_emb(rows, cos, sin, is_neox_style=False).unflatten(0, x.shape[:2]) -# ---- references against the eager arithmetic ----------------------------------------------------------------------- - - @pytest.mark.parametrize("with_rope", [False, True]) def test_qk_reference_is_the_eager_chain_bit_for_bit(with_rope: bool) -> None: q, k, _ = _qkv(2, 13, 3, 16, torch.bfloat16) @@ -186,9 +182,6 @@ def test_fused_ops_compile_with_their_backwards_on_gpu() -> None: torch.testing.assert_close(actual, expected, atol=1e-5, rtol=1e-5) -# ---- the Triton kernels against the references ---------------------------------------------------------------------- - - def check_triton_kernels(device: str, dtypes: tuple[torch.dtype, ...]) -> None: """Every Triton kernel against its reference, forward and backward; raises on a mismatch.""" for dtype in dtypes: @@ -239,9 +232,6 @@ def test_triton_kernels_match_their_references(env_overrides) -> None: assert process.returncode == 0, f"Triton interpreter check failed:\n{process.stdout}\n{process.stderr}" -# ---- the fused model against the eager model ----------------------------------------------------------------------- - - def _forward_backward(model: torch.nn.Module, inputs: dict[str, torch.Tensor]): model.zero_grad(set_to_none=True) with set_forward_context(current_timestep=0, attn_metadata=None, forward_batch=ForwardBatch(data_type="dummy")):