[feat] v2: per-model sampling defaults on ModelCard (Phase 1a)
v2 had no per-model defaults — generate_video hardcoded 30 steps/25 frames/480x832/cfg5/16fps for every model, badly wrong for e.g. LTX-2 distilled (wants 8 steps @1024x1536) or Wan2.2-TI2V (704x1280@24fps). - New SamplingDefaults dataclass + ModelCard.sampling_defaults (v2/card/specs.py), exported from v2.card. - Populated all 7 supported cards from fastvideo's InferencePreset defaults (steps/guidance/HxW/frames/fps + per-modality guidance for LTX-2.3 A/V). Negative prompts copied verbatim into v2/recipes/_prompts.py (recipe DATA, not model code -> v2-owned, no fastvideo import). - VideoGenerator stores the resolved card; generate_video applies card defaults with precedence kwargs > SamplingParam > card > generic fallback (pure _resolve_default helper, unit-tested). - test_sampling_defaults.py: per-card values + precedence (incl. empty-neg-prompt edge). CPU mini 210 pass, 2 skip.
This commit is contained in:
+2
-1
@@ -18,11 +18,12 @@ from v2.card.specs import (
|
||||
ParityTestSpec,
|
||||
PrecisionContract,
|
||||
RecipeSpec,
|
||||
SamplingDefaults,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ModelCard", "ComponentSpec", "LoopSpec", "RecipeSpec", "ParitySpec", "ParityTestSpec",
|
||||
"CheckpointManifest", "CapabilityMatrix", "CostModel", "CacheContract",
|
||||
"CheckpointManifest", "CapabilityMatrix", "CostModel", "CacheContract", "SamplingDefaults",
|
||||
"ParallelismContract", "PrecisionContract", "DataRef", "CardValidationError",
|
||||
"ModelInstance", "load_card",
|
||||
"Capability", "ConsistencyLevel", "ExecutionProfile", "LoopKind", "WorkUnitKind",
|
||||
|
||||
@@ -201,6 +201,24 @@ class CapabilityMatrix:
|
||||
return cls(frozenset(caps))
|
||||
|
||||
|
||||
@dataclass
|
||||
class SamplingDefaults:
|
||||
"""Per-model default generation params (the v2 mirror of fastvideo's per-model ``InferencePreset``
|
||||
defaults). Applied by the entrypoint when the caller didn't specify a value; ``None`` => fall back to
|
||||
the generic default. ``shift``/``sigmas``/MoE boundary already live in the card's loop policies — these
|
||||
are the user-facing knobs surfaced at request time."""
|
||||
num_steps: int | None = None
|
||||
guidance_scale: float | None = None
|
||||
guidance_per_modality: dict[str, float] = field(default_factory=dict) # joint A/V, e.g. {"video":3,"audio":7}
|
||||
height: int | None = None
|
||||
width: int | None = None
|
||||
num_frames: int | None = None
|
||||
fps: int | None = None
|
||||
negative_prompt: str | None = None
|
||||
shift: float | None = None
|
||||
sigmas: tuple[float, ...] | None = None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# ModelCard — the (recipe, runtime) pair as one versioned, validatable object #
|
||||
# --------------------------------------------------------------------------- #
|
||||
@@ -221,6 +239,7 @@ class ModelCard:
|
||||
parallelism: ParallelismContract = field(default_factory=ParallelismContract)
|
||||
precision: PrecisionContract = field(default_factory=PrecisionContract)
|
||||
checkpoint: CheckpointManifest = field(default_factory=CheckpointManifest)
|
||||
sampling_defaults: SamplingDefaults = field(default_factory=SamplingDefaults)
|
||||
|
||||
def validate(self) -> "ModelCard":
|
||||
"""Strict enough to validate before any GPU touches it (design_v3 §4.1).
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Default negative prompts per model family — copied verbatim from fastvideo's pipeline presets
|
||||
(`fastvideo/pipelines/basic/{wan,ltx2}/presets.py`). These are stable recipe DATA (not model code), so
|
||||
v2 owns its own copy rather than importing private symbols from fastvideo's pipelines package."""
|
||||
from __future__ import annotations
|
||||
|
||||
# Wan family — English (Wan2.1 T2V) and Chinese (Wan2.2 / self-forcing) negative prompts.
|
||||
WAN_NEG_EN = ("Bright tones, overexposed, static, blurred details, subtitles,"
|
||||
" style, works, paintings, images, static, overall gray, worst"
|
||||
" quality, low quality, JPEG compression residue, ugly,"
|
||||
" incomplete, extra fingers, poorly drawn hands, poorly drawn"
|
||||
" faces, deformed, disfigured, misshapen limbs, fused fingers,"
|
||||
" still picture, messy background, three legs, many people in"
|
||||
" the background, walking backwards")
|
||||
|
||||
WAN_NEG_CN = ("色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,"
|
||||
"静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,"
|
||||
"多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,"
|
||||
"形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,"
|
||||
"背景人很多,倒着走")
|
||||
|
||||
# LTX-2 family negative prompt (base / 2.3); the distilled few-step presets use "" instead.
|
||||
LTX2_NEG = ("blurry, out of focus, overexposed, underexposed, low contrast, "
|
||||
"washed out colors, excessive noise, grainy texture, poor lighting, "
|
||||
"flickering, motion blur, distorted proportions, unnatural skin "
|
||||
"tones, deformed facial features, asymmetrical face, missing facial "
|
||||
"features, extra limbs, disfigured hands, wrong hand count, "
|
||||
"artifacts around text, inconsistent perspective, camera shake, "
|
||||
"incorrect depth of field, background too sharp, background clutter, "
|
||||
"distracting reflections, harsh shadows, inconsistent lighting "
|
||||
"direction, color banding, cartoonish rendering, 3D CGI look, "
|
||||
"unrealistic materials, uncanny valley effect, incorrect ethnicity, "
|
||||
"wrong gender, exaggerated expressions, wrong gaze direction, "
|
||||
"mismatched lip sync, silent or muted audio, distorted voice, "
|
||||
"robotic voice, echo, background noise, off-sync audio, incorrect "
|
||||
"dialogue, added dialogue, repetitive speech, jittery movement, "
|
||||
"awkward pauses, incorrect timing, unnatural transitions, "
|
||||
"inconsistent framing, tilted camera, flat lighting, inconsistent "
|
||||
"tone, cinematic oversaturation, stylized filters, or AI artifacts.")
|
||||
@@ -26,9 +26,11 @@ from v2.card import (
|
||||
ParitySpec,
|
||||
PrecisionContract,
|
||||
RecipeSpec,
|
||||
SamplingDefaults,
|
||||
)
|
||||
from v2.parallel import ParallelPlan
|
||||
from v2.platform.backends.toy import ToyAudioVAE, ToyDiT, ToyTextEncoder, ToyUpsampler, ToyVAE, ToyVocoder, _seed_from
|
||||
from v2.recipes._prompts import LTX2_NEG
|
||||
from v2.recipes.ltx2.loop import BASE_SIGMAS, REFINE_SIGMAS, LTX2DenoiseLoop, LTX23DenoiseLoop
|
||||
|
||||
|
||||
@@ -86,6 +88,9 @@ def build_ltx2_card(model_id: str = "ltx2-2stage-distilled") -> ModelCard:
|
||||
Capability.VAE_DECODE),
|
||||
recipe=RecipeSpec(method="distilled", parents=["ltx2.3-base"], assumes_loop="ltx2_base",
|
||||
assumes_precision="float32", consistency_required=ConsistencyLevel.C1),
|
||||
sampling_defaults=SamplingDefaults( # two-stage distilled: 8-step base (+ 2-step refine in loop)
|
||||
num_steps=8, guidance_scale=1.0, height=1024, width=1536, num_frames=121, fps=24,
|
||||
negative_prompt=""),
|
||||
parity=ParitySpec(consistency_levels=[ConsistencyLevel.C1], interleave_required=True),
|
||||
caches={"feature": CacheContract(cache_class="feature", max_bytes=1 << 24,
|
||||
reuse_across_requests=True)},
|
||||
@@ -134,6 +139,9 @@ def build_ltx2_base_card(model_id: str = "ltx2-single-stage") -> ModelCard:
|
||||
capabilities=CapabilityMatrix.of(Capability.TEXT_TO_VIDEO, Capability.VAE_DECODE),
|
||||
recipe=RecipeSpec(method="base", assumes_loop="ltx2_single",
|
||||
assumes_precision="float32", consistency_required=ConsistencyLevel.C1),
|
||||
sampling_defaults=SamplingDefaults(
|
||||
num_steps=40, guidance_scale=3.0, height=512, width=768, num_frames=121, fps=24,
|
||||
negative_prompt=LTX2_NEG),
|
||||
parity=ParitySpec(consistency_levels=[ConsistencyLevel.C1], interleave_required=True),
|
||||
caches={"feature": CacheContract(cache_class="feature", max_bytes=1 << 24,
|
||||
reuse_across_requests=True)},
|
||||
@@ -191,6 +199,9 @@ def build_ltx2_3_card(model_id: str = "ltx2.3-distilled") -> ModelCard:
|
||||
Capability.VAE_DECODE),
|
||||
recipe=RecipeSpec(method="distilled", parents=["ltx2.3-base"], assumes_loop="ltx2_3",
|
||||
assumes_precision="float32", consistency_required=ConsistencyLevel.C1),
|
||||
sampling_defaults=SamplingDefaults( # joint A/V: per-modality cfg video 3 / audio 7
|
||||
num_steps=30, guidance_scale=3.0, height=512, width=768, num_frames=121, fps=24,
|
||||
negative_prompt=LTX2_NEG, guidance_per_modality={"video": 3.0, "audio": 7.0}),
|
||||
parity=ParitySpec(consistency_levels=[ConsistencyLevel.C1], interleave_required=True),
|
||||
caches={"feature": CacheContract(cache_class="feature", max_bytes=1 << 24,
|
||||
reuse_across_requests=True)},
|
||||
|
||||
@@ -22,6 +22,7 @@ from v2.card import (
|
||||
ParityTestSpec,
|
||||
PrecisionContract,
|
||||
RecipeSpec,
|
||||
SamplingDefaults,
|
||||
)
|
||||
from v2.loop.policies import (
|
||||
BoundaryTimestepRouting,
|
||||
@@ -32,12 +33,14 @@ from v2.loop.policies import (
|
||||
)
|
||||
from v2.parallel import ParallelPlan
|
||||
from v2.platform.backends.toy import ToyDiT, ToyTextEncoder, ToyVAE, _seed_from
|
||||
from v2.recipes._prompts import WAN_NEG_CN, WAN_NEG_EN
|
||||
from v2.recipes.wan21.loop import WanDenoiseLoop
|
||||
|
||||
|
||||
def build_wan21_card(model_id: str = "wan2.1-1.3b", *, cfg_policy=None, flow_shift: float = 3.0,
|
||||
checkpoint_root: str | None = None, latent_channels: int = 16,
|
||||
spatial_ratio: int = 8, temporal_ratio: int = 4) -> ModelCard:
|
||||
spatial_ratio: int = 8, temporal_ratio: int = 4,
|
||||
sampling_defaults: SamplingDefaults | None = None) -> ModelCard:
|
||||
seed = _seed_from(model_id)
|
||||
cost = CostModel(kind=WorkUnitKind.DIFFUSION_STEP, base_seconds=1e-4, per_unit_seconds=1e-7)
|
||||
cfg = cfg_policy or ClassicCFG()
|
||||
@@ -91,6 +94,9 @@ def build_wan21_card(model_id: str = "wan2.1-1.3b", *, cfg_policy=None, flow_shi
|
||||
valid_plans=[ParallelPlan.single(),
|
||||
ParallelPlan(axes={"sp": 2, "cfgp": 2}, mesh_order=["cfgp", "sp"])],
|
||||
default_plan=ParallelPlan.single()),
|
||||
sampling_defaults=sampling_defaults or SamplingDefaults(
|
||||
num_steps=50, guidance_scale=3.0, height=480, width=832, num_frames=81, fps=16,
|
||||
negative_prompt=WAN_NEG_EN),
|
||||
)
|
||||
card.validate()
|
||||
if checkpoint_root:
|
||||
@@ -106,7 +112,10 @@ def build_wan22_ti2v_card(model_id: str = "wan2.2-ti2v-5b", *,
|
||||
timestep (its 1D path), so no per-frame ``expand_timesteps`` is needed for pure t2v. Uses the same
|
||||
``transformer/vae/text_encoder`` subfolder layout, so ``stamp_wan21_checkpoints`` applies."""
|
||||
return build_wan21_card(model_id=model_id, flow_shift=5.0, checkpoint_root=checkpoint_root,
|
||||
latent_channels=48, spatial_ratio=16, temporal_ratio=4)
|
||||
latent_channels=48, spatial_ratio=16, temporal_ratio=4,
|
||||
sampling_defaults=SamplingDefaults(
|
||||
num_steps=50, guidance_scale=5.0, height=704, width=1280,
|
||||
num_frames=121, fps=24, negative_prompt=WAN_NEG_CN))
|
||||
|
||||
|
||||
def build_wan22_a14b_card(model_id: str = "wan2.2-t2v-a14b", *, checkpoint_root: str | None = None,
|
||||
@@ -170,6 +179,9 @@ def build_wan22_a14b_card(model_id: str = "wan2.2-t2v-a14b", *, checkpoint_root:
|
||||
precision=PrecisionContract(default_dtype="float32", training_precision="float32"),
|
||||
parallelism=ParallelismContract(valid_plans=[ParallelPlan.single()],
|
||||
default_plan=ParallelPlan.single()),
|
||||
sampling_defaults=SamplingDefaults(
|
||||
num_steps=40, guidance_scale=4.0, height=480, width=832, num_frames=81, fps=16,
|
||||
negative_prompt=WAN_NEG_CN),
|
||||
)
|
||||
card.validate()
|
||||
if checkpoint_root:
|
||||
|
||||
@@ -19,10 +19,12 @@ from v2.card import (
|
||||
ParitySpec,
|
||||
PrecisionContract,
|
||||
RecipeSpec,
|
||||
SamplingDefaults,
|
||||
)
|
||||
from v2.loop.policies import ClassicCFG, FlowShiftPolicy, PrecisionPolicy
|
||||
from v2.parallel import ParallelPlan
|
||||
from v2.platform.backends.toy import ToyDiT, ToyTextEncoder, ToyVAE, _seed_from
|
||||
from v2.recipes._prompts import WAN_NEG_CN
|
||||
from v2.recipes.wan_causal.loop import ChunkRolloutLoop
|
||||
|
||||
|
||||
@@ -76,5 +78,8 @@ def build_wan_causal_card(model_id: str = "wan-causal-sf-1.3b", *,
|
||||
precision=PrecisionContract(default_dtype="float32", training_precision="float32"),
|
||||
parallelism=ParallelismContract(valid_plans=[ParallelPlan.single()],
|
||||
default_plan=ParallelPlan.single()),
|
||||
sampling_defaults=SamplingDefaults(
|
||||
num_steps=50, guidance_scale=6.0, height=480, width=832, num_frames=81, fps=16,
|
||||
negative_prompt=WAN_NEG_CN),
|
||||
)
|
||||
return card.validate()
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Phase 1a: per-model sampling defaults on the card + the kwargs>SamplingParam>card>generic precedence."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from v2.card import SamplingDefaults
|
||||
from v2.recipes.ltx2 import build_ltx2_3_card, build_ltx2_base_card, build_ltx2_card
|
||||
from v2.recipes.wan21 import build_wan21_card, build_wan22_a14b_card, build_wan22_ti2v_card
|
||||
from v2.recipes.wan_causal import build_wan_causal_card
|
||||
from v2.video_generator import _resolve_default
|
||||
|
||||
|
||||
def test_wan21_defaults():
|
||||
sd = build_wan21_card().sampling_defaults
|
||||
assert (sd.num_steps, sd.guidance_scale, sd.height, sd.width, sd.num_frames, sd.fps) == (50, 3.0, 480, 832, 81, 16)
|
||||
assert "overexposed" in sd.negative_prompt
|
||||
|
||||
|
||||
def test_wan22_ti2v_defaults():
|
||||
sd = build_wan22_ti2v_card().sampling_defaults
|
||||
assert (sd.num_steps, sd.guidance_scale, sd.height, sd.width, sd.num_frames, sd.fps) == (50, 5.0, 704, 1280, 121, 24)
|
||||
|
||||
|
||||
def test_wan22_a14b_defaults():
|
||||
sd = build_wan22_a14b_card().sampling_defaults
|
||||
assert (sd.num_steps, sd.guidance_scale, sd.fps) == (40, 4.0, 16)
|
||||
|
||||
|
||||
def test_wan_causal_defaults():
|
||||
sd = build_wan_causal_card().sampling_defaults
|
||||
assert (sd.num_steps, sd.guidance_scale, sd.height, sd.width, sd.num_frames, sd.fps) == (50, 6.0, 480, 832, 81, 16)
|
||||
|
||||
|
||||
def test_ltx2_two_stage_defaults():
|
||||
sd = build_ltx2_card().sampling_defaults
|
||||
assert (sd.num_steps, sd.guidance_scale, sd.height, sd.width, sd.num_frames, sd.fps) == (8, 1.0, 1024, 1536, 121, 24)
|
||||
assert sd.negative_prompt == "" # distilled: empty negative prompt
|
||||
|
||||
|
||||
def test_ltx2_base_defaults():
|
||||
sd = build_ltx2_base_card().sampling_defaults
|
||||
assert (sd.num_steps, sd.guidance_scale, sd.height, sd.width) == (40, 3.0, 512, 768)
|
||||
assert "blurry" in sd.negative_prompt
|
||||
|
||||
|
||||
def test_ltx2_3_defaults():
|
||||
sd = build_ltx2_3_card().sampling_defaults
|
||||
assert (sd.num_steps, sd.guidance_scale, sd.height, sd.width) == (30, 3.0, 512, 768)
|
||||
assert sd.guidance_per_modality == {"video": 3.0, "audio": 7.0}
|
||||
|
||||
|
||||
def test_resolve_precedence():
|
||||
sd = SamplingDefaults(num_steps=8, guidance_scale=1.0, height=1024)
|
||||
# card default used when neither kwargs nor SamplingParam set it (note the num_steps alias)
|
||||
assert _resolve_default("num_inference_steps", 30, {}, None, sd, "num_steps") == 8
|
||||
assert _resolve_default("height", 480, {}, None, sd) == 1024
|
||||
# kwargs override card
|
||||
assert _resolve_default("num_inference_steps", 30, {"num_inference_steps": 4}, None, sd, "num_steps") == 4
|
||||
# SamplingParam override (no kwargs) beats card
|
||||
assert _resolve_default("height", 480, {}, SimpleNamespace(height=720), sd) == 720
|
||||
# generic fallback when card has no value and nothing else set
|
||||
assert _resolve_default("width", 832, {}, None, sd) == 832
|
||||
# empty-string negative prompt is a real value, not "unset"
|
||||
assert _resolve_default("negative_prompt", None, {}, None, SamplingDefaults(negative_prompt="")) == ""
|
||||
+27
-9
@@ -24,14 +24,33 @@ from typing import Any
|
||||
# ``v2/registry.py`` so every entrypoint (this VideoGenerator, a CLI, the server) resolves identically.
|
||||
|
||||
|
||||
def _resolve_default(key: str, generic: Any, kwargs: dict, sp: Any, sd: Any,
|
||||
card_attr: str | None = None) -> Any:
|
||||
"""Resolve one sampling param with precedence kwargs > SamplingParam override > card
|
||||
``SamplingDefaults`` > generic fallback. ``card_attr`` aliases the card field name when it differs
|
||||
from the request key (e.g. ``num_inference_steps`` -> ``SamplingDefaults.num_steps``). Pure +
|
||||
torch-free so it's unit-testable without a GPU."""
|
||||
if key in kwargs and kwargs[key] is not None:
|
||||
return kwargs[key]
|
||||
if sp is not None and getattr(sp, key, None) is not None:
|
||||
return getattr(sp, key)
|
||||
if sd is not None:
|
||||
cv = getattr(sd, card_attr or key, None)
|
||||
if cv is not None:
|
||||
return cv
|
||||
return generic
|
||||
|
||||
|
||||
class VideoGenerator:
|
||||
"""Typed t2v entrypoint over a single resident v2 ``ModelInstance`` (mirrors fastvideo's)."""
|
||||
|
||||
def __init__(self, engine: Any, model_id: str, *, instance: Any = None, supports_av: bool = False) -> None:
|
||||
def __init__(self, engine: Any, model_id: str, *, instance: Any = None, supports_av: bool = False,
|
||||
card: Any = None) -> None:
|
||||
self._engine = engine
|
||||
self._model_id = model_id
|
||||
self._inst = instance # the resident ModelInstance (to read e.g. the audio sample rate)
|
||||
self._supports_av = supports_av # model emits joint video+sound (LTX-2.3 T2VS) -> auto-request audio
|
||||
self._card = card # the v2 ModelCard (for per-model sampling_defaults)
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
@classmethod
|
||||
@@ -96,7 +115,7 @@ class VideoGenerator:
|
||||
# A model that advertises TEXT_TO_VIDEO_SOUND (LTX-2.3) generates joint video+audio; the
|
||||
# entrypoint then issues a T2VS request by default so a plain generate() yields both modalities.
|
||||
supports_av = card.capabilities.has(Capability.TEXT_TO_VIDEO_SOUND)
|
||||
return cls(eng, card.model_id, instance=inst, supports_av=supports_av)
|
||||
return cls(eng, card.model_id, instance=inst, supports_av=supports_av, card=card)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""API parity with ``fastvideo.VideoGenerator.shutdown()``. The v2 bring-up holds a single
|
||||
@@ -141,16 +160,15 @@ class VideoGenerator:
|
||||
from fastvideo.api import GenerationRequest, OutputConfig, SamplingConfig
|
||||
audio = kwargs.pop("audio", None) # None -> auto (by model capability), True/False -> force
|
||||
sp = sampling_param
|
||||
# Per-model defaults (LTX-2 wants 8/30 steps + its own res, Wan2.2-TI2V 704x1280@24fps, etc.) come
|
||||
# from the resolved card; precedence is kwargs > SamplingParam > card defaults > generic fallback.
|
||||
sd = getattr(self._card, "sampling_defaults", None) if self._card is not None else None
|
||||
|
||||
def pick(key: str, default: Any) -> Any:
|
||||
if key in kwargs and kwargs[key] is not None:
|
||||
return kwargs[key]
|
||||
if sp is not None and getattr(sp, key, None) is not None:
|
||||
return getattr(sp, key)
|
||||
return default
|
||||
def pick(key: str, default: Any, card_attr: str | None = None) -> Any:
|
||||
return _resolve_default(key, default, kwargs, sp, sd, card_attr)
|
||||
|
||||
sampling = SamplingConfig(
|
||||
num_inference_steps=int(pick("num_inference_steps", 30)),
|
||||
num_inference_steps=int(pick("num_inference_steps", 30, "num_steps")),
|
||||
seed=int(pick("seed", 1024)),
|
||||
num_frames=int(pick("num_frames", 25)),
|
||||
height=int(pick("height", 480)),
|
||||
|
||||
Reference in New Issue
Block a user