[feat] PAN2: default to the released negative prompt; align serving and cookbook with CFG 2.5
An empty negative prompt now selects the release's structured negative caption with its time ranges set to the clip's duration, as the reference sampler does; a single space still gives the empty unconditional prompt. The serving config and cookbook now say guidance 2.5, matching the presets. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
William Lin
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
7490986ec8
commit
910ee7d6bb
@@ -5,7 +5,7 @@
|
||||
Generate a video from text, or from a first frame, with `IFM/PAN2`. One pipeline
|
||||
serves both modes: leave out the first frame for text-to-video and supply one
|
||||
for image-to-video. The defaults are 704×1248, 313 frames (13 s at 24 fps),
|
||||
50 steps and guidance 3.0. See [PAN 2 Text/Image to Video](../inference/pan2.md)
|
||||
50 steps, guidance 2.5 and PAN 2's default negative prompt. See [PAN 2 Text/Image to Video](../inference/pan2.md)
|
||||
for the model, sampler and parallelism details.
|
||||
|
||||
## Run on a GPU
|
||||
|
||||
@@ -20,7 +20,7 @@ from the `pan2_t2v` and `pan2_i2v` presets:
|
||||
| Frames | 313, 13 s at 24 fps (`num_frames - 1` must be a multiple of 4) |
|
||||
| Steps | 50, uniform sigmas from 1 to 0 with the checkpoint's flow shift (9 for `IFM/PAN2-TEST`, the reference sampler's value) |
|
||||
| Guidance | 2.5, classifier-free guidance, the reference launcher's default (3 before 2026-10-23). Image-to-video keeps the first latent frame out of guidance (`i2v_skip_first_latent_cfg`, on by default, as in the reference sampler) |
|
||||
| Negative prompt | None unless the request carries one. The reference sampler always sends the release's structured negative caption (`negative_prompt.json` of the PAN2 repository) with its three `time_range_sec` values set to the clip length, `0 to 20` for 481 frames at 24 fps |
|
||||
| Negative prompt | The release's structured negative caption (`negative_prompt.json` of the PAN2 repository) with its three `time_range_sec` values set to the clip length, `0 to 20` for 481 frames at 24 fps, as in the reference sampler. It is used whenever the request's negative prompt is empty; a single space gives the empty unconditional prompt PAN 2 trained with |
|
||||
|
||||
PAN 2 is trained on the structured captions its prompt rewriter produces, serialized as one-line JSON. Short raw
|
||||
prompts work, with weaker results.
|
||||
|
||||
@@ -25,12 +25,14 @@ server:
|
||||
served_model_name: pan2
|
||||
|
||||
default_request:
|
||||
# Empty: PAN 2's default negative prompt, its time ranges set to the clip's duration.
|
||||
negative_prompt: ""
|
||||
sampling:
|
||||
height: 704
|
||||
width: 1248
|
||||
num_frames: 313
|
||||
fps: 24
|
||||
num_inference_steps: 50
|
||||
guidance_scale: 3.0
|
||||
guidance_scale: 2.5
|
||||
output:
|
||||
return_frames: false
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""The negative prompt PAN2 was released with (``inference/prompts/negative_prompt.json`` in ifm-ai/PAN2).
|
||||
|
||||
Its ``time_range_sec`` fields are a ``0 to 10`` template; ``render_pan2_default_negative_prompt`` sets them to span
|
||||
the clip, as the reference sampler does for records without a negative prompt of their own.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
PAN2_DEFAULT_NEGATIVE_PROMPT = {
|
||||
"video_style": "low quality, blur, softness, reduced detail, noise, compression, clipped highlights, "
|
||||
"blurred sidebar, black border, letterboxing, watermark, subtitle, AIGC text, oily waxy "
|
||||
"plastic texture, oversaturation or color cast, color clipping or chroma bleeding, flicker, "
|
||||
"jitter, ghosting, motion smear, choppy or laggy motion, slow motion, Ken Burns effect, "
|
||||
"unmotivated slow zoom over a frozen scene, pan across a still image, global frame scaling "
|
||||
"or drifting, simulated movement from a static photograph, repeated micro-motion loop, "
|
||||
"discontinuous shot, scene cut, hard cut, crossfade, fade-in, fade-out, fading transition, "
|
||||
"scene replacement, spontaneous object or person pop-in or disappearance, duplicated or "
|
||||
"missing people or props, identity or count drift, rigid-object deformation or morphing, "
|
||||
"bending or melting rigid structures, structural shape or component drift",
|
||||
"lighting": "overexposure, clipped highlights, underexposure, harsh lighting, overly deep skin shadows, "
|
||||
"brightness fading, excessive lighting saturation, chroma-clipped highlights, neon-like skin "
|
||||
"tones, colored spill overwhelming local material colors, oversaturated bloom, unnatural neon "
|
||||
"color casts, hue drift, chroma noise, bad white balance, color pulsing, flickering light, "
|
||||
"unnatural lighting",
|
||||
"perspective_view": "warped geometry, distorted perspective, inconsistent spatial relationships, "
|
||||
"unmotivated layout changes, camera teleport or jump, viewpoint reset, wrong depth or "
|
||||
"occlusion",
|
||||
"subjects_lifecycle": [
|
||||
{
|
||||
"subject_id": "<|Subject1|>",
|
||||
"display_name": "malformed inconsistent subject",
|
||||
"subject_attribute": "broken anatomy, distorted body proportions, reversed pelvis, "
|
||||
"twisted torso, deformed arms or legs, reversed joints, fused "
|
||||
"feet or toes, warped or collapsing faces, transparent or "
|
||||
"floating head and body, missing body parts, detached or "
|
||||
"disembodied body parts, floating hands or limbs, detached "
|
||||
"articulated parts, disconnected joints or supports, hands "
|
||||
"disconnected from wrists or forearms, discontinuous "
|
||||
"limb-to-torso connections, blurred or fused fingers, missing "
|
||||
"thumbs, extra or missing fingers, malformed wrists, extra "
|
||||
"limbs, duplicated hands, mirrored face, identity drift, wrong "
|
||||
"relative sizes, human-object fusion, unintended body-material "
|
||||
"fusion, loss of separate material boundaries, opaque white "
|
||||
"nail polish, gray nail polish, chalky opaque fingernails, "
|
||||
"artificial acrylic nails",
|
||||
}
|
||||
],
|
||||
"init_frame_visual_description": "blur or softness, reduced detail, noise or compression artifacts, clipped "
|
||||
"highlights, blurred sidebar, black border or letterboxing, oily waxy "
|
||||
"plastic surfaces, oversaturation or color cast, exaggerated local "
|
||||
"contrast, exposure fading, bad white balance, smeared dense textures, "
|
||||
"blurred or distorted face, transparent or missing body parts, detached or "
|
||||
"disembodied body parts, floating hands or limbs, detached articulated "
|
||||
"parts, disconnected joints or supports, disconnected wrists or forearms, "
|
||||
"discontinuous limb-to-torso connections, twisted torso, fused or missing "
|
||||
"fingers, missing thumbs, malformed wrists, duplicated limbs, deformed "
|
||||
"arms or legs, reversed knees, fused toes, warped geometry, deformed or "
|
||||
"asymmetrically warped rigid objects, bent rigid structures, incorrect "
|
||||
"rigid part geometry, topology, proportions, or component count, wrong "
|
||||
"occlusion, floating or hovering objects, unsupported objects, object or "
|
||||
"human-object fusion, unintended fusion of distinct surfaces or layers, "
|
||||
"loss of separate material boundaries, merged objects, hand-object "
|
||||
"penetration, object-object penetration, penetration, collision, "
|
||||
"impossible contact, wrong sizes.",
|
||||
"events": [
|
||||
{
|
||||
"event_index": 0,
|
||||
"time_range_sec": "0 to 10",
|
||||
"event_description": "blur, softness, reduced detail, compression artifacts, clipped "
|
||||
"highlights, flicker, jitter, ghosting, motion smear, choppy or laggy "
|
||||
"motion, slow motion, Ken Burns effect, unmotivated slow zoom over a "
|
||||
"frozen scene, pan across a still image, global frame scaling or drifting, "
|
||||
"simulated movement from a static photograph, repeated micro-motion loop, "
|
||||
"oily oversaturated or color-pulsing surfaces, shimmering surface detail, "
|
||||
"fingers fusing or changing count, hands smearing, facial features "
|
||||
"merging, reversed joints, upper body rotating independently of attached "
|
||||
"hips and feet, corkscrewed waist, limbs changing length or attachment "
|
||||
"after occlusion, transparent body or head, detached or disembodied body "
|
||||
"parts, floating hands or limbs, detached articulated parts, disconnected "
|
||||
"joints or supports, hands disconnected from wrists or forearms, body "
|
||||
"parts moving independently, vanishing limb-to-torso connections, merged "
|
||||
"people or limbs, limbs fused into torso, merged or fused feet and toes, "
|
||||
"floating or hovering objects, unsupported objects, object or human-object "
|
||||
"fusion, unintended fusion of distinct surfaces or layers, loss of "
|
||||
"separate material boundaries, merged objects, hand-object penetration, "
|
||||
"object-object penetration, penetration or clipping, objects colliding or "
|
||||
"morphing, rigid objects tearing, fragmenting, fusing, bending, melting or "
|
||||
"changing dimensions, topology, silhouette, component count or layout, "
|
||||
"subjects changing identity, splitting or duplicating, objects appearing "
|
||||
"or disappearing without entry, exit or occlusion, changed identity or "
|
||||
"count after occlusion, state changes inconsistent with the causing "
|
||||
"action, broken physics, uncaused deformation or motion, state changes "
|
||||
"preceding the causing interaction, loss of coupling during contact, pose "
|
||||
"jumps, discontinuous shot, camera jump, hard scene cut, crossfade, "
|
||||
"fade-in, fade-out, fading transition, scene replacement or viewpoint "
|
||||
"reset.",
|
||||
"camera_motion": [
|
||||
{"time_range_sec": "0 to 10", "camera_motion": ["camera_jump", "hard_cut", "viewpoint_reset"]}
|
||||
],
|
||||
"contained_actions": [
|
||||
{
|
||||
"time_range_sec": "0 to 10",
|
||||
"action": "<|Subject1|> fingers fusing during grip changes, palms "
|
||||
"merging with props, wrists collapsing, crossed limbs fusing, "
|
||||
"intersecting or swapping attachment, duplicated hands or "
|
||||
"feet, detached or disembodied hands or limbs, hands "
|
||||
"disconnected from wrists or forearms, discontinuous "
|
||||
"limb-to-torso connections, merged or fused feet and toes, "
|
||||
"unstable contact, unintended grip slip, hand-object "
|
||||
"penetration, body-object penetration, passage through solid "
|
||||
"surfaces instead of openings, object-object penetration, "
|
||||
"merged or fused objects, rigid objects fragmenting, fusing, "
|
||||
"smearing or changing dimensions during motion, unintended "
|
||||
"fusion of distinct surfaces or layers, loss of separate "
|
||||
"material boundaries, floating or hovering objects, "
|
||||
"unsupported objects, objects duplicating, splitting or "
|
||||
"disappearing during interaction or occlusion, unchanged "
|
||||
"action result, pose jump",
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
"quality": {
|
||||
"video_quality": "low resolution, blur, softness, reduced detail, noise, compression, clipped "
|
||||
"highlights, blurred sidebar, black border, letterboxing, oily waxy plastic "
|
||||
"surfaces, greasy highlights, oversaturated colors, faded or washed-out "
|
||||
"colors, exposure fading, smeared faces, hands or held objects, surface detail "
|
||||
"smearing or swimming, material or existing markings changing without a "
|
||||
"physical cause, color or shape morphing, object pop-in or pop-out, sudden "
|
||||
"disappearance, missing persistent objects, object duplication or count drift, "
|
||||
"rigid-object warping or deformation, changing rigid proportions, topology, "
|
||||
"silhouette, or component layout",
|
||||
"text_logos_overlays": "watermark, subtitle, captions, AIGC text or logo",
|
||||
"artifacts": [
|
||||
"watermark",
|
||||
"template_or_text_dominated",
|
||||
"talking_head",
|
||||
"visual_quality_issue",
|
||||
"edited_or_discontinuous",
|
||||
"video_game",
|
||||
"animation",
|
||||
"ai_generated_content",
|
||||
"black_border",
|
||||
"ken_burn",
|
||||
"time_lapse",
|
||||
"blurred_sidebar",
|
||||
"hard_scene_cuts",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def render_pan2_default_negative_prompt(num_frames: int, fps: float) -> str:
|
||||
"""The default negative prompt as compact one-line JSON, with every time range spanning the clip."""
|
||||
span = f"0 to {(num_frames - 1) / fps:g}"
|
||||
|
||||
def fill(node):
|
||||
if isinstance(node, dict):
|
||||
return {key: span if key == "time_range_sec" else fill(value) for key, value in node.items()}
|
||||
if isinstance(node, list):
|
||||
return [fill(item) for item in node]
|
||||
return node
|
||||
|
||||
return json.dumps(fill(PAN2_DEFAULT_NEGATIVE_PROMPT), ensure_ascii=False, separators=(",", ":"))
|
||||
@@ -11,9 +11,8 @@ _DENOISE_STAGE = PresetStageSpec(
|
||||
)
|
||||
|
||||
# 704x1248, 313 frames (13 s at 24 fps), 50 steps, CFG 2.5 (the release launcher's default since ifm-ai/PAN2
|
||||
# 86d5023, 2026-10-23; 3 before). The preset sends no negative prompt (an empty one is encoded
|
||||
# as a single space); the reference sampler sends the release's structured negative caption, so pass it as the
|
||||
# request's ``negative_prompt`` to follow that recipe.
|
||||
# 86d5023, 2026-10-23; 3 before). The empty negative prompt selects the release's structured negative caption,
|
||||
# rendered for the clip's duration by ``PAN2TextEncodingStage`` as in the reference sampler.
|
||||
_SHARED_DEFAULTS = {
|
||||
"height": 704,
|
||||
"width": 1248,
|
||||
|
||||
@@ -15,6 +15,7 @@ from torch.distributed.tensor import DTensor
|
||||
from fastvideo.distributed import get_local_torch_device
|
||||
from fastvideo.fastvideo_args import FastVideoArgs
|
||||
from fastvideo.forward_context import set_forward_context
|
||||
from fastvideo.pipelines.basic.pan2.negative_prompt import render_pan2_default_negative_prompt
|
||||
from fastvideo.pipelines.pipeline_batch_info import ForwardBatch
|
||||
from fastvideo.pipelines.stages.base import PipelineStage
|
||||
from fastvideo.pipelines.stages.denoising import DenoisingStage
|
||||
@@ -52,8 +53,18 @@ class PAN2TextEncodingStage(TextEncodingStage):
|
||||
|
||||
PAN2 conditions on the tokens after the ``<|im_start|>user\\n`` header (the prompt and the assistant header),
|
||||
read from the text encoder's hidden states with its last ``text_encoder_skip_layers`` layers skipped.
|
||||
|
||||
An empty or missing negative prompt is replaced by the release's default one, its time ranges set to the clip's
|
||||
duration, as in the reference sampler. A single space encodes the empty unconditional prompt PAN2 trained with.
|
||||
"""
|
||||
|
||||
def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch:
|
||||
if batch.do_classifier_free_guidance and not batch.negative_prompt:
|
||||
num_frames = batch.num_frames[0] if isinstance(batch.num_frames, list) else batch.num_frames
|
||||
fps = batch.fps[0] if isinstance(batch.fps, list) else batch.fps
|
||||
batch.negative_prompt = render_pan2_default_negative_prompt(num_frames, fps or 24)
|
||||
return super().forward(batch, fastvideo_args)
|
||||
|
||||
@staticmethod
|
||||
def tokenize(tokenizer: Any, prompt: str, system_prompt: str, max_length: int) -> tuple[torch.Tensor, int]:
|
||||
"""Token ids truncated to ``max_length`` tokens after the user header, and where that header ends."""
|
||||
|
||||
@@ -122,6 +122,7 @@ def test_pan2_serving_matches_its_preset_and_runs_image_to_video_from_a_referenc
|
||||
assert config.generator.engine.offload.dit_layerwise is False
|
||||
for field in ("height", "width", "num_frames", "fps", "num_inference_steps", "guidance_scale"):
|
||||
assert getattr(sampling, field) == PAN2_T2V.defaults[field], field
|
||||
assert config.default_request.negative_prompt == PAN2_T2V.defaults["negative_prompt"]
|
||||
|
||||
image = tmp_path / "first-frame.png"
|
||||
image.write_bytes(b"not-decoded-at-admission")
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""PAN2 falls back to the release's default negative prompt, rendered for the clip; no weights or CUDA needed."""
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from fastvideo.pipelines.basic.pan2.negative_prompt import (PAN2_DEFAULT_NEGATIVE_PROMPT,
|
||||
render_pan2_default_negative_prompt)
|
||||
from fastvideo.pipelines.basic.pan2.stages import PAN2TextEncodingStage
|
||||
from fastvideo.pipelines.stages.text_encoding import TextEncodingStage
|
||||
|
||||
|
||||
def _time_ranges(node):
|
||||
if isinstance(node, dict):
|
||||
return [v for k, v in node.items() if k == "time_range_sec"] + sum(map(_time_ranges, node.values()), [])
|
||||
if isinstance(node, list):
|
||||
return sum(map(_time_ranges, node), [])
|
||||
return []
|
||||
|
||||
|
||||
def test_the_default_spans_the_clip_and_is_otherwise_the_release_template():
|
||||
rendered = json.loads(render_pan2_default_negative_prompt(481, 24))
|
||||
assert _time_ranges(rendered) == ["0 to 20"] * 3
|
||||
assert _time_ranges(PAN2_DEFAULT_NEGATIVE_PROMPT) == ["0 to 10"] * 3
|
||||
assert render_pan2_default_negative_prompt(313, 24) == json.dumps(
|
||||
json.loads(render_pan2_default_negative_prompt(313, 24)), ensure_ascii=False, separators=(",", ":"))
|
||||
assert _time_ranges(json.loads(render_pan2_default_negative_prompt(313, 24))) == ["0 to 13"] * 3
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def encode(monkeypatch):
|
||||
seen = []
|
||||
monkeypatch.setattr(TextEncodingStage, "forward", lambda self, batch, args: seen.append(batch.negative_prompt))
|
||||
stage = PAN2TextEncodingStage.__new__(PAN2TextEncodingStage)
|
||||
|
||||
def run(negative_prompt, cfg=True, num_frames=121, fps=24):
|
||||
batch = SimpleNamespace(negative_prompt=negative_prompt, do_classifier_free_guidance=cfg,
|
||||
num_frames=num_frames, fps=fps)
|
||||
stage.forward(batch, None)
|
||||
return seen[-1]
|
||||
|
||||
return run
|
||||
|
||||
|
||||
@pytest.mark.parametrize("negative_prompt", ["", None])
|
||||
def test_an_empty_negative_prompt_becomes_the_default(encode, negative_prompt):
|
||||
assert encode(negative_prompt) == render_pan2_default_negative_prompt(121, 24)
|
||||
assert encode(negative_prompt, num_frames=[241], fps=[24]) == render_pan2_default_negative_prompt(241, 24)
|
||||
assert encode(negative_prompt, fps=None) == render_pan2_default_negative_prompt(121, 24)
|
||||
|
||||
|
||||
def test_a_given_negative_prompt_or_a_space_is_kept(encode):
|
||||
assert encode("blurry") == "blurry"
|
||||
assert encode(" ") == " "
|
||||
|
||||
|
||||
def test_without_guidance_the_negative_prompt_is_left_alone(encode):
|
||||
assert encode("", cfg=False) == ""
|
||||
Reference in New Issue
Block a user