diff --git a/docs/cookbook/pan2.md b/docs/cookbook/pan2.md index e5581f144..c0a69b947 100644 --- a/docs/cookbook/pan2.md +++ b/docs/cookbook/pan2.md @@ -5,7 +5,7 @@ Generate a video from text, or from a first frame, with `IFM/PAN2`. One pipeline serves both modes: leave out the first frame for text-to-video and supply one for image-to-video. The defaults are 704×1248, 313 frames (13 s at 24 fps), -50 steps and guidance 3.0. See [PAN 2 Text/Image to Video](../inference/pan2.md) +50 steps, guidance 2.5 and PAN 2's default negative prompt. See [PAN 2 Text/Image to Video](../inference/pan2.md) for the model, sampler and parallelism details. ## Run on a GPU diff --git a/docs/inference/pan2.md b/docs/inference/pan2.md index 1687832c6..2580da165 100644 --- a/docs/inference/pan2.md +++ b/docs/inference/pan2.md @@ -20,7 +20,7 @@ from the `pan2_t2v` and `pan2_i2v` presets: | Frames | 313, 13 s at 24 fps (`num_frames - 1` must be a multiple of 4) | | Steps | 50, uniform sigmas from 1 to 0 with the checkpoint's flow shift (9 for `IFM/PAN2-TEST`, the reference sampler's value) | | Guidance | 2.5, classifier-free guidance, the reference launcher's default (3 before 2026-10-23). Image-to-video keeps the first latent frame out of guidance (`i2v_skip_first_latent_cfg`, on by default, as in the reference sampler) | -| Negative prompt | None unless the request carries one. The reference sampler always sends the release's structured negative caption (`negative_prompt.json` of the PAN2 repository) with its three `time_range_sec` values set to the clip length, `0 to 20` for 481 frames at 24 fps | +| Negative prompt | The release's structured negative caption (`negative_prompt.json` of the PAN2 repository) with its three `time_range_sec` values set to the clip length, `0 to 20` for 481 frames at 24 fps, as in the reference sampler. It is used whenever the request's negative prompt is empty; a single space gives the empty unconditional prompt PAN 2 trained with | PAN 2 is trained on the structured captions its prompt rewriter produces, serialized as one-line JSON. Short raw prompts work, with weaker results. diff --git a/examples/serving/openai_pan2.yaml b/examples/serving/openai_pan2.yaml index 17bc68793..9f44b4da5 100644 --- a/examples/serving/openai_pan2.yaml +++ b/examples/serving/openai_pan2.yaml @@ -25,12 +25,14 @@ server: served_model_name: pan2 default_request: + # Empty: PAN 2's default negative prompt, its time ranges set to the clip's duration. + negative_prompt: "" sampling: height: 704 width: 1248 num_frames: 313 fps: 24 num_inference_steps: 50 - guidance_scale: 3.0 + guidance_scale: 2.5 output: return_frames: false diff --git a/fastvideo/pipelines/basic/pan2/negative_prompt.py b/fastvideo/pipelines/basic/pan2/negative_prompt.py new file mode 100644 index 000000000..3fa307225 --- /dev/null +++ b/fastvideo/pipelines/basic/pan2/negative_prompt.py @@ -0,0 +1,165 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The negative prompt PAN2 was released with (``inference/prompts/negative_prompt.json`` in ifm-ai/PAN2). + +Its ``time_range_sec`` fields are a ``0 to 10`` template; ``render_pan2_default_negative_prompt`` sets them to span +the clip, as the reference sampler does for records without a negative prompt of their own. +""" + +import json + +PAN2_DEFAULT_NEGATIVE_PROMPT = { + "video_style": "low quality, blur, softness, reduced detail, noise, compression, clipped highlights, " + "blurred sidebar, black border, letterboxing, watermark, subtitle, AIGC text, oily waxy " + "plastic texture, oversaturation or color cast, color clipping or chroma bleeding, flicker, " + "jitter, ghosting, motion smear, choppy or laggy motion, slow motion, Ken Burns effect, " + "unmotivated slow zoom over a frozen scene, pan across a still image, global frame scaling " + "or drifting, simulated movement from a static photograph, repeated micro-motion loop, " + "discontinuous shot, scene cut, hard cut, crossfade, fade-in, fade-out, fading transition, " + "scene replacement, spontaneous object or person pop-in or disappearance, duplicated or " + "missing people or props, identity or count drift, rigid-object deformation or morphing, " + "bending or melting rigid structures, structural shape or component drift", + "lighting": "overexposure, clipped highlights, underexposure, harsh lighting, overly deep skin shadows, " + "brightness fading, excessive lighting saturation, chroma-clipped highlights, neon-like skin " + "tones, colored spill overwhelming local material colors, oversaturated bloom, unnatural neon " + "color casts, hue drift, chroma noise, bad white balance, color pulsing, flickering light, " + "unnatural lighting", + "perspective_view": "warped geometry, distorted perspective, inconsistent spatial relationships, " + "unmotivated layout changes, camera teleport or jump, viewpoint reset, wrong depth or " + "occlusion", + "subjects_lifecycle": [ + { + "subject_id": "<|Subject1|>", + "display_name": "malformed inconsistent subject", + "subject_attribute": "broken anatomy, distorted body proportions, reversed pelvis, " + "twisted torso, deformed arms or legs, reversed joints, fused " + "feet or toes, warped or collapsing faces, transparent or " + "floating head and body, missing body parts, detached or " + "disembodied body parts, floating hands or limbs, detached " + "articulated parts, disconnected joints or supports, hands " + "disconnected from wrists or forearms, discontinuous " + "limb-to-torso connections, blurred or fused fingers, missing " + "thumbs, extra or missing fingers, malformed wrists, extra " + "limbs, duplicated hands, mirrored face, identity drift, wrong " + "relative sizes, human-object fusion, unintended body-material " + "fusion, loss of separate material boundaries, opaque white " + "nail polish, gray nail polish, chalky opaque fingernails, " + "artificial acrylic nails", + } + ], + "init_frame_visual_description": "blur or softness, reduced detail, noise or compression artifacts, clipped " + "highlights, blurred sidebar, black border or letterboxing, oily waxy " + "plastic surfaces, oversaturation or color cast, exaggerated local " + "contrast, exposure fading, bad white balance, smeared dense textures, " + "blurred or distorted face, transparent or missing body parts, detached or " + "disembodied body parts, floating hands or limbs, detached articulated " + "parts, disconnected joints or supports, disconnected wrists or forearms, " + "discontinuous limb-to-torso connections, twisted torso, fused or missing " + "fingers, missing thumbs, malformed wrists, duplicated limbs, deformed " + "arms or legs, reversed knees, fused toes, warped geometry, deformed or " + "asymmetrically warped rigid objects, bent rigid structures, incorrect " + "rigid part geometry, topology, proportions, or component count, wrong " + "occlusion, floating or hovering objects, unsupported objects, object or " + "human-object fusion, unintended fusion of distinct surfaces or layers, " + "loss of separate material boundaries, merged objects, hand-object " + "penetration, object-object penetration, penetration, collision, " + "impossible contact, wrong sizes.", + "events": [ + { + "event_index": 0, + "time_range_sec": "0 to 10", + "event_description": "blur, softness, reduced detail, compression artifacts, clipped " + "highlights, flicker, jitter, ghosting, motion smear, choppy or laggy " + "motion, slow motion, Ken Burns effect, unmotivated slow zoom over a " + "frozen scene, pan across a still image, global frame scaling or drifting, " + "simulated movement from a static photograph, repeated micro-motion loop, " + "oily oversaturated or color-pulsing surfaces, shimmering surface detail, " + "fingers fusing or changing count, hands smearing, facial features " + "merging, reversed joints, upper body rotating independently of attached " + "hips and feet, corkscrewed waist, limbs changing length or attachment " + "after occlusion, transparent body or head, detached or disembodied body " + "parts, floating hands or limbs, detached articulated parts, disconnected " + "joints or supports, hands disconnected from wrists or forearms, body " + "parts moving independently, vanishing limb-to-torso connections, merged " + "people or limbs, limbs fused into torso, merged or fused feet and toes, " + "floating or hovering objects, unsupported objects, object or human-object " + "fusion, unintended fusion of distinct surfaces or layers, loss of " + "separate material boundaries, merged objects, hand-object penetration, " + "object-object penetration, penetration or clipping, objects colliding or " + "morphing, rigid objects tearing, fragmenting, fusing, bending, melting or " + "changing dimensions, topology, silhouette, component count or layout, " + "subjects changing identity, splitting or duplicating, objects appearing " + "or disappearing without entry, exit or occlusion, changed identity or " + "count after occlusion, state changes inconsistent with the causing " + "action, broken physics, uncaused deformation or motion, state changes " + "preceding the causing interaction, loss of coupling during contact, pose " + "jumps, discontinuous shot, camera jump, hard scene cut, crossfade, " + "fade-in, fade-out, fading transition, scene replacement or viewpoint " + "reset.", + "camera_motion": [ + {"time_range_sec": "0 to 10", "camera_motion": ["camera_jump", "hard_cut", "viewpoint_reset"]} + ], + "contained_actions": [ + { + "time_range_sec": "0 to 10", + "action": "<|Subject1|> fingers fusing during grip changes, palms " + "merging with props, wrists collapsing, crossed limbs fusing, " + "intersecting or swapping attachment, duplicated hands or " + "feet, detached or disembodied hands or limbs, hands " + "disconnected from wrists or forearms, discontinuous " + "limb-to-torso connections, merged or fused feet and toes, " + "unstable contact, unintended grip slip, hand-object " + "penetration, body-object penetration, passage through solid " + "surfaces instead of openings, object-object penetration, " + "merged or fused objects, rigid objects fragmenting, fusing, " + "smearing or changing dimensions during motion, unintended " + "fusion of distinct surfaces or layers, loss of separate " + "material boundaries, floating or hovering objects, " + "unsupported objects, objects duplicating, splitting or " + "disappearing during interaction or occlusion, unchanged " + "action result, pose jump", + } + ], + } + ], + "quality": { + "video_quality": "low resolution, blur, softness, reduced detail, noise, compression, clipped " + "highlights, blurred sidebar, black border, letterboxing, oily waxy plastic " + "surfaces, greasy highlights, oversaturated colors, faded or washed-out " + "colors, exposure fading, smeared faces, hands or held objects, surface detail " + "smearing or swimming, material or existing markings changing without a " + "physical cause, color or shape morphing, object pop-in or pop-out, sudden " + "disappearance, missing persistent objects, object duplication or count drift, " + "rigid-object warping or deformation, changing rigid proportions, topology, " + "silhouette, or component layout", + "text_logos_overlays": "watermark, subtitle, captions, AIGC text or logo", + "artifacts": [ + "watermark", + "template_or_text_dominated", + "talking_head", + "visual_quality_issue", + "edited_or_discontinuous", + "video_game", + "animation", + "ai_generated_content", + "black_border", + "ken_burn", + "time_lapse", + "blurred_sidebar", + "hard_scene_cuts", + ], + }, +} + + +def render_pan2_default_negative_prompt(num_frames: int, fps: float) -> str: + """The default negative prompt as compact one-line JSON, with every time range spanning the clip.""" + span = f"0 to {(num_frames - 1) / fps:g}" + + def fill(node): + if isinstance(node, dict): + return {key: span if key == "time_range_sec" else fill(value) for key, value in node.items()} + if isinstance(node, list): + return [fill(item) for item in node] + return node + + return json.dumps(fill(PAN2_DEFAULT_NEGATIVE_PROMPT), ensure_ascii=False, separators=(",", ":")) diff --git a/fastvideo/pipelines/basic/pan2/presets.py b/fastvideo/pipelines/basic/pan2/presets.py index af93fe8f0..87a2904c2 100644 --- a/fastvideo/pipelines/basic/pan2/presets.py +++ b/fastvideo/pipelines/basic/pan2/presets.py @@ -11,9 +11,8 @@ _DENOISE_STAGE = PresetStageSpec( ) # 704x1248, 313 frames (13 s at 24 fps), 50 steps, CFG 2.5 (the release launcher's default since ifm-ai/PAN2 -# 86d5023, 2026-10-23; 3 before). The preset sends no negative prompt (an empty one is encoded -# as a single space); the reference sampler sends the release's structured negative caption, so pass it as the -# request's ``negative_prompt`` to follow that recipe. +# 86d5023, 2026-10-23; 3 before). The empty negative prompt selects the release's structured negative caption, +# rendered for the clip's duration by ``PAN2TextEncodingStage`` as in the reference sampler. _SHARED_DEFAULTS = { "height": 704, "width": 1248, diff --git a/fastvideo/pipelines/basic/pan2/stages.py b/fastvideo/pipelines/basic/pan2/stages.py index 6ef9001b2..8ea67dcf9 100644 --- a/fastvideo/pipelines/basic/pan2/stages.py +++ b/fastvideo/pipelines/basic/pan2/stages.py @@ -15,6 +15,7 @@ from torch.distributed.tensor import DTensor from fastvideo.distributed import get_local_torch_device from fastvideo.fastvideo_args import FastVideoArgs from fastvideo.forward_context import set_forward_context +from fastvideo.pipelines.basic.pan2.negative_prompt import render_pan2_default_negative_prompt from fastvideo.pipelines.pipeline_batch_info import ForwardBatch from fastvideo.pipelines.stages.base import PipelineStage from fastvideo.pipelines.stages.denoising import DenoisingStage @@ -52,8 +53,18 @@ class PAN2TextEncodingStage(TextEncodingStage): PAN2 conditions on the tokens after the ``<|im_start|>user\\n`` header (the prompt and the assistant header), read from the text encoder's hidden states with its last ``text_encoder_skip_layers`` layers skipped. + + An empty or missing negative prompt is replaced by the release's default one, its time ranges set to the clip's + duration, as in the reference sampler. A single space encodes the empty unconditional prompt PAN2 trained with. """ + def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch: + if batch.do_classifier_free_guidance and not batch.negative_prompt: + num_frames = batch.num_frames[0] if isinstance(batch.num_frames, list) else batch.num_frames + fps = batch.fps[0] if isinstance(batch.fps, list) else batch.fps + batch.negative_prompt = render_pan2_default_negative_prompt(num_frames, fps or 24) + return super().forward(batch, fastvideo_args) + @staticmethod def tokenize(tokenizer: Any, prompt: str, system_prompt: str, max_length: int) -> tuple[torch.Tensor, int]: """Token ids truncated to ``max_length`` tokens after the user header, and where that header ends.""" diff --git a/fastvideo/tests/entrypoints/test_serving_family_baselines.py b/fastvideo/tests/entrypoints/test_serving_family_baselines.py index be67249f3..db0ffce74 100644 --- a/fastvideo/tests/entrypoints/test_serving_family_baselines.py +++ b/fastvideo/tests/entrypoints/test_serving_family_baselines.py @@ -122,6 +122,7 @@ def test_pan2_serving_matches_its_preset_and_runs_image_to_video_from_a_referenc assert config.generator.engine.offload.dit_layerwise is False for field in ("height", "width", "num_frames", "fps", "num_inference_steps", "guidance_scale"): assert getattr(sampling, field) == PAN2_T2V.defaults[field], field + assert config.default_request.negative_prompt == PAN2_T2V.defaults["negative_prompt"] image = tmp_path / "first-frame.png" image.write_bytes(b"not-decoded-at-admission") diff --git a/fastvideo/tests/stages/test_pan2_negative_prompt.py b/fastvideo/tests/stages/test_pan2_negative_prompt.py new file mode 100644 index 000000000..2c8c6342e --- /dev/null +++ b/fastvideo/tests/stages/test_pan2_negative_prompt.py @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: Apache-2.0 +"""PAN2 falls back to the release's default negative prompt, rendered for the clip; no weights or CUDA needed.""" + +import json +from types import SimpleNamespace + +import pytest + +from fastvideo.pipelines.basic.pan2.negative_prompt import (PAN2_DEFAULT_NEGATIVE_PROMPT, + render_pan2_default_negative_prompt) +from fastvideo.pipelines.basic.pan2.stages import PAN2TextEncodingStage +from fastvideo.pipelines.stages.text_encoding import TextEncodingStage + + +def _time_ranges(node): + if isinstance(node, dict): + return [v for k, v in node.items() if k == "time_range_sec"] + sum(map(_time_ranges, node.values()), []) + if isinstance(node, list): + return sum(map(_time_ranges, node), []) + return [] + + +def test_the_default_spans_the_clip_and_is_otherwise_the_release_template(): + rendered = json.loads(render_pan2_default_negative_prompt(481, 24)) + assert _time_ranges(rendered) == ["0 to 20"] * 3 + assert _time_ranges(PAN2_DEFAULT_NEGATIVE_PROMPT) == ["0 to 10"] * 3 + assert render_pan2_default_negative_prompt(313, 24) == json.dumps( + json.loads(render_pan2_default_negative_prompt(313, 24)), ensure_ascii=False, separators=(",", ":")) + assert _time_ranges(json.loads(render_pan2_default_negative_prompt(313, 24))) == ["0 to 13"] * 3 + + +@pytest.fixture +def encode(monkeypatch): + seen = [] + monkeypatch.setattr(TextEncodingStage, "forward", lambda self, batch, args: seen.append(batch.negative_prompt)) + stage = PAN2TextEncodingStage.__new__(PAN2TextEncodingStage) + + def run(negative_prompt, cfg=True, num_frames=121, fps=24): + batch = SimpleNamespace(negative_prompt=negative_prompt, do_classifier_free_guidance=cfg, + num_frames=num_frames, fps=fps) + stage.forward(batch, None) + return seen[-1] + + return run + + +@pytest.mark.parametrize("negative_prompt", ["", None]) +def test_an_empty_negative_prompt_becomes_the_default(encode, negative_prompt): + assert encode(negative_prompt) == render_pan2_default_negative_prompt(121, 24) + assert encode(negative_prompt, num_frames=[241], fps=[24]) == render_pan2_default_negative_prompt(241, 24) + assert encode(negative_prompt, fps=None) == render_pan2_default_negative_prompt(121, 24) + + +def test_a_given_negative_prompt_or_a_space_is_kept(encode): + assert encode("blurry") == "blurry" + assert encode(" ") == " " + + +def test_without_guidance_the_negative_prompt_is_left_alone(encode): + assert encode("", cfg=False) == ""