diff --git a/v2/card/__init__.py b/v2/card/__init__.py index ea1cf652d..78d388ec2 100644 --- a/v2/card/__init__.py +++ b/v2/card/__init__.py @@ -18,11 +18,12 @@ from v2.card.specs import ( ParityTestSpec, PrecisionContract, RecipeSpec, + SamplingDefaults, ) __all__ = [ "ModelCard", "ComponentSpec", "LoopSpec", "RecipeSpec", "ParitySpec", "ParityTestSpec", - "CheckpointManifest", "CapabilityMatrix", "CostModel", "CacheContract", + "CheckpointManifest", "CapabilityMatrix", "CostModel", "CacheContract", "SamplingDefaults", "ParallelismContract", "PrecisionContract", "DataRef", "CardValidationError", "ModelInstance", "load_card", "Capability", "ConsistencyLevel", "ExecutionProfile", "LoopKind", "WorkUnitKind", diff --git a/v2/card/specs.py b/v2/card/specs.py index 475d57fad..3ca5ef23b 100644 --- a/v2/card/specs.py +++ b/v2/card/specs.py @@ -201,6 +201,24 @@ class CapabilityMatrix: return cls(frozenset(caps)) +@dataclass +class SamplingDefaults: + """Per-model default generation params (the v2 mirror of fastvideo's per-model ``InferencePreset`` + defaults). Applied by the entrypoint when the caller didn't specify a value; ``None`` => fall back to + the generic default. ``shift``/``sigmas``/MoE boundary already live in the card's loop policies — these + are the user-facing knobs surfaced at request time.""" + num_steps: int | None = None + guidance_scale: float | None = None + guidance_per_modality: dict[str, float] = field(default_factory=dict) # joint A/V, e.g. {"video":3,"audio":7} + height: int | None = None + width: int | None = None + num_frames: int | None = None + fps: int | None = None + negative_prompt: str | None = None + shift: float | None = None + sigmas: tuple[float, ...] | None = None + + # --------------------------------------------------------------------------- # # ModelCard — the (recipe, runtime) pair as one versioned, validatable object # # --------------------------------------------------------------------------- # @@ -221,6 +239,7 @@ class ModelCard: parallelism: ParallelismContract = field(default_factory=ParallelismContract) precision: PrecisionContract = field(default_factory=PrecisionContract) checkpoint: CheckpointManifest = field(default_factory=CheckpointManifest) + sampling_defaults: SamplingDefaults = field(default_factory=SamplingDefaults) def validate(self) -> "ModelCard": """Strict enough to validate before any GPU touches it (design_v3 §4.1). diff --git a/v2/recipes/_prompts.py b/v2/recipes/_prompts.py new file mode 100644 index 000000000..373cca1f6 --- /dev/null +++ b/v2/recipes/_prompts.py @@ -0,0 +1,38 @@ +"""Default negative prompts per model family — copied verbatim from fastvideo's pipeline presets +(`fastvideo/pipelines/basic/{wan,ltx2}/presets.py`). These are stable recipe DATA (not model code), so +v2 owns its own copy rather than importing private symbols from fastvideo's pipelines package.""" +from __future__ import annotations + +# Wan family — English (Wan2.1 T2V) and Chinese (Wan2.2 / self-forcing) negative prompts. +WAN_NEG_EN = ("Bright tones, overexposed, static, blurred details, subtitles," + " style, works, paintings, images, static, overall gray, worst" + " quality, low quality, JPEG compression residue, ugly," + " incomplete, extra fingers, poorly drawn hands, poorly drawn" + " faces, deformed, disfigured, misshapen limbs, fused fingers," + " still picture, messy background, three legs, many people in" + " the background, walking backwards") + +WAN_NEG_CN = ("色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，" + "静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，" + "多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，" + "形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，" + "背景人很多，倒着走") + +# LTX-2 family negative prompt (base / 2.3); the distilled few-step presets use "" instead. +LTX2_NEG = ("blurry, out of focus, overexposed, underexposed, low contrast, " + "washed out colors, excessive noise, grainy texture, poor lighting, " + "flickering, motion blur, distorted proportions, unnatural skin " + "tones, deformed facial features, asymmetrical face, missing facial " + "features, extra limbs, disfigured hands, wrong hand count, " + "artifacts around text, inconsistent perspective, camera shake, " + "incorrect depth of field, background too sharp, background clutter, " + "distracting reflections, harsh shadows, inconsistent lighting " + "direction, color banding, cartoonish rendering, 3D CGI look, " + "unrealistic materials, uncanny valley effect, incorrect ethnicity, " + "wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, " + "robotic voice, echo, background noise, off-sync audio, incorrect " + "dialogue, added dialogue, repetitive speech, jittery movement, " + "awkward pauses, incorrect timing, unnatural transitions, " + "inconsistent framing, tilted camera, flat lighting, inconsistent " + "tone, cinematic oversaturation, stylized filters, or AI artifacts.") diff --git a/v2/recipes/ltx2/card.py b/v2/recipes/ltx2/card.py index ed81e89bd..bdcb6341a 100644 --- a/v2/recipes/ltx2/card.py +++ b/v2/recipes/ltx2/card.py @@ -26,9 +26,11 @@ from v2.card import ( ParitySpec, PrecisionContract, RecipeSpec, + SamplingDefaults, ) from v2.parallel import ParallelPlan from v2.platform.backends.toy import ToyAudioVAE, ToyDiT, ToyTextEncoder, ToyUpsampler, ToyVAE, ToyVocoder, _seed_from +from v2.recipes._prompts import LTX2_NEG from v2.recipes.ltx2.loop import BASE_SIGMAS, REFINE_SIGMAS, LTX2DenoiseLoop, LTX23DenoiseLoop @@ -86,6 +88,9 @@ def build_ltx2_card(model_id: str = "ltx2-2stage-distilled") -> ModelCard: Capability.VAE_DECODE), recipe=RecipeSpec(method="distilled", parents=["ltx2.3-base"], assumes_loop="ltx2_base", assumes_precision="float32", consistency_required=ConsistencyLevel.C1), + sampling_defaults=SamplingDefaults( # two-stage distilled: 8-step base (+ 2-step refine in loop) + num_steps=8, guidance_scale=1.0, height=1024, width=1536, num_frames=121, fps=24, + negative_prompt=""), parity=ParitySpec(consistency_levels=[ConsistencyLevel.C1], interleave_required=True), caches={"feature": CacheContract(cache_class="feature", max_bytes=1 << 24, reuse_across_requests=True)}, @@ -134,6 +139,9 @@ def build_ltx2_base_card(model_id: str = "ltx2-single-stage") -> ModelCard: capabilities=CapabilityMatrix.of(Capability.TEXT_TO_VIDEO, Capability.VAE_DECODE), recipe=RecipeSpec(method="base", assumes_loop="ltx2_single", assumes_precision="float32", consistency_required=ConsistencyLevel.C1), + sampling_defaults=SamplingDefaults( + num_steps=40, guidance_scale=3.0, height=512, width=768, num_frames=121, fps=24, + negative_prompt=LTX2_NEG), parity=ParitySpec(consistency_levels=[ConsistencyLevel.C1], interleave_required=True), caches={"feature": CacheContract(cache_class="feature", max_bytes=1 << 24, reuse_across_requests=True)}, @@ -191,6 +199,9 @@ def build_ltx2_3_card(model_id: str = "ltx2.3-distilled") -> ModelCard: Capability.VAE_DECODE), recipe=RecipeSpec(method="distilled", parents=["ltx2.3-base"], assumes_loop="ltx2_3", assumes_precision="float32", consistency_required=ConsistencyLevel.C1), + sampling_defaults=SamplingDefaults( # joint A/V: per-modality cfg video 3 / audio 7 + num_steps=30, guidance_scale=3.0, height=512, width=768, num_frames=121, fps=24, + negative_prompt=LTX2_NEG, guidance_per_modality={"video": 3.0, "audio": 7.0}), parity=ParitySpec(consistency_levels=[ConsistencyLevel.C1], interleave_required=True), caches={"feature": CacheContract(cache_class="feature", max_bytes=1 << 24, reuse_across_requests=True)}, diff --git a/v2/recipes/wan21/card.py b/v2/recipes/wan21/card.py index a462206e6..81665221d 100644 --- a/v2/recipes/wan21/card.py +++ b/v2/recipes/wan21/card.py @@ -22,6 +22,7 @@ from v2.card import ( ParityTestSpec, PrecisionContract, RecipeSpec, + SamplingDefaults, ) from v2.loop.policies import ( BoundaryTimestepRouting, @@ -32,12 +33,14 @@ from v2.loop.policies import ( ) from v2.parallel import ParallelPlan from v2.platform.backends.toy import ToyDiT, ToyTextEncoder, ToyVAE, _seed_from +from v2.recipes._prompts import WAN_NEG_CN, WAN_NEG_EN from v2.recipes.wan21.loop import WanDenoiseLoop def build_wan21_card(model_id: str = "wan2.1-1.3b", *, cfg_policy=None, flow_shift: float = 3.0, checkpoint_root: str | None = None, latent_channels: int = 16, - spatial_ratio: int = 8, temporal_ratio: int = 4) -> ModelCard: + spatial_ratio: int = 8, temporal_ratio: int = 4, + sampling_defaults: SamplingDefaults | None = None) -> ModelCard: seed = _seed_from(model_id) cost = CostModel(kind=WorkUnitKind.DIFFUSION_STEP, base_seconds=1e-4, per_unit_seconds=1e-7) cfg = cfg_policy or ClassicCFG() @@ -91,6 +94,9 @@ def build_wan21_card(model_id: str = "wan2.1-1.3b", *, cfg_policy=None, flow_shi valid_plans=[ParallelPlan.single(), ParallelPlan(axes={"sp": 2, "cfgp": 2}, mesh_order=["cfgp", "sp"])], default_plan=ParallelPlan.single()), + sampling_defaults=sampling_defaults or SamplingDefaults( + num_steps=50, guidance_scale=3.0, height=480, width=832, num_frames=81, fps=16, + negative_prompt=WAN_NEG_EN), ) card.validate() if checkpoint_root: @@ -106,7 +112,10 @@ def build_wan22_ti2v_card(model_id: str = "wan2.2-ti2v-5b", *, timestep (its 1D path), so no per-frame ``expand_timesteps`` is needed for pure t2v. Uses the same ``transformer/vae/text_encoder`` subfolder layout, so ``stamp_wan21_checkpoints`` applies.""" return build_wan21_card(model_id=model_id, flow_shift=5.0, checkpoint_root=checkpoint_root, - latent_channels=48, spatial_ratio=16, temporal_ratio=4) + latent_channels=48, spatial_ratio=16, temporal_ratio=4, + sampling_defaults=SamplingDefaults( + num_steps=50, guidance_scale=5.0, height=704, width=1280, + num_frames=121, fps=24, negative_prompt=WAN_NEG_CN)) def build_wan22_a14b_card(model_id: str = "wan2.2-t2v-a14b", *, checkpoint_root: str | None = None, @@ -170,6 +179,9 @@ def build_wan22_a14b_card(model_id: str = "wan2.2-t2v-a14b", *, checkpoint_root: precision=PrecisionContract(default_dtype="float32", training_precision="float32"), parallelism=ParallelismContract(valid_plans=[ParallelPlan.single()], default_plan=ParallelPlan.single()), + sampling_defaults=SamplingDefaults( + num_steps=40, guidance_scale=4.0, height=480, width=832, num_frames=81, fps=16, + negative_prompt=WAN_NEG_CN), ) card.validate() if checkpoint_root: diff --git a/v2/recipes/wan_causal/card.py b/v2/recipes/wan_causal/card.py index 8a3d24c26..fff0b94c0 100644 --- a/v2/recipes/wan_causal/card.py +++ b/v2/recipes/wan_causal/card.py @@ -19,10 +19,12 @@ from v2.card import ( ParitySpec, PrecisionContract, RecipeSpec, + SamplingDefaults, ) from v2.loop.policies import ClassicCFG, FlowShiftPolicy, PrecisionPolicy from v2.parallel import ParallelPlan from v2.platform.backends.toy import ToyDiT, ToyTextEncoder, ToyVAE, _seed_from +from v2.recipes._prompts import WAN_NEG_CN from v2.recipes.wan_causal.loop import ChunkRolloutLoop @@ -76,5 +78,8 @@ def build_wan_causal_card(model_id: str = "wan-causal-sf-1.3b", *, precision=PrecisionContract(default_dtype="float32", training_precision="float32"), parallelism=ParallelismContract(valid_plans=[ParallelPlan.single()], default_plan=ParallelPlan.single()), + sampling_defaults=SamplingDefaults( + num_steps=50, guidance_scale=6.0, height=480, width=832, num_frames=81, fps=16, + negative_prompt=WAN_NEG_CN), ) return card.validate() diff --git a/v2/tests/test_sampling_defaults.py b/v2/tests/test_sampling_defaults.py new file mode 100644 index 000000000..c5ee666cd --- /dev/null +++ b/v2/tests/test_sampling_defaults.py @@ -0,0 +1,62 @@ +"""Phase 1a: per-model sampling defaults on the card + the kwargs>SamplingParam>card>generic precedence.""" +from types import SimpleNamespace + +from v2.card import SamplingDefaults +from v2.recipes.ltx2 import build_ltx2_3_card, build_ltx2_base_card, build_ltx2_card +from v2.recipes.wan21 import build_wan21_card, build_wan22_a14b_card, build_wan22_ti2v_card +from v2.recipes.wan_causal import build_wan_causal_card +from v2.video_generator import _resolve_default + + +def test_wan21_defaults(): + sd = build_wan21_card().sampling_defaults + assert (sd.num_steps, sd.guidance_scale, sd.height, sd.width, sd.num_frames, sd.fps) == (50, 3.0, 480, 832, 81, 16) + assert "overexposed" in sd.negative_prompt + + +def test_wan22_ti2v_defaults(): + sd = build_wan22_ti2v_card().sampling_defaults + assert (sd.num_steps, sd.guidance_scale, sd.height, sd.width, sd.num_frames, sd.fps) == (50, 5.0, 704, 1280, 121, 24) + + +def test_wan22_a14b_defaults(): + sd = build_wan22_a14b_card().sampling_defaults + assert (sd.num_steps, sd.guidance_scale, sd.fps) == (40, 4.0, 16) + + +def test_wan_causal_defaults(): + sd = build_wan_causal_card().sampling_defaults + assert (sd.num_steps, sd.guidance_scale, sd.height, sd.width, sd.num_frames, sd.fps) == (50, 6.0, 480, 832, 81, 16) + + +def test_ltx2_two_stage_defaults(): + sd = build_ltx2_card().sampling_defaults + assert (sd.num_steps, sd.guidance_scale, sd.height, sd.width, sd.num_frames, sd.fps) == (8, 1.0, 1024, 1536, 121, 24) + assert sd.negative_prompt == "" # distilled: empty negative prompt + + +def test_ltx2_base_defaults(): + sd = build_ltx2_base_card().sampling_defaults + assert (sd.num_steps, sd.guidance_scale, sd.height, sd.width) == (40, 3.0, 512, 768) + assert "blurry" in sd.negative_prompt + + +def test_ltx2_3_defaults(): + sd = build_ltx2_3_card().sampling_defaults + assert (sd.num_steps, sd.guidance_scale, sd.height, sd.width) == (30, 3.0, 512, 768) + assert sd.guidance_per_modality == {"video": 3.0, "audio": 7.0} + + +def test_resolve_precedence(): + sd = SamplingDefaults(num_steps=8, guidance_scale=1.0, height=1024) + # card default used when neither kwargs nor SamplingParam set it (note the num_steps alias) + assert _resolve_default("num_inference_steps", 30, {}, None, sd, "num_steps") == 8 + assert _resolve_default("height", 480, {}, None, sd) == 1024 + # kwargs override card + assert _resolve_default("num_inference_steps", 30, {"num_inference_steps": 4}, None, sd, "num_steps") == 4 + # SamplingParam override (no kwargs) beats card + assert _resolve_default("height", 480, {}, SimpleNamespace(height=720), sd) == 720 + # generic fallback when card has no value and nothing else set + assert _resolve_default("width", 832, {}, None, sd) == 832 + # empty-string negative prompt is a real value, not "unset" + assert _resolve_default("negative_prompt", None, {}, None, SamplingDefaults(negative_prompt="")) == "" diff --git a/v2/video_generator.py b/v2/video_generator.py index 96a6afd0f..c9adfab48 100644 --- a/v2/video_generator.py +++ b/v2/video_generator.py @@ -24,14 +24,33 @@ from typing import Any # ``v2/registry.py`` so every entrypoint (this VideoGenerator, a CLI, the server) resolves identically. +def _resolve_default(key: str, generic: Any, kwargs: dict, sp: Any, sd: Any, + card_attr: str | None = None) -> Any: + """Resolve one sampling param with precedence kwargs > SamplingParam override > card + ``SamplingDefaults`` > generic fallback. ``card_attr`` aliases the card field name when it differs + from the request key (e.g. ``num_inference_steps`` -> ``SamplingDefaults.num_steps``). Pure + + torch-free so it's unit-testable without a GPU.""" + if key in kwargs and kwargs[key] is not None: + return kwargs[key] + if sp is not None and getattr(sp, key, None) is not None: + return getattr(sp, key) + if sd is not None: + cv = getattr(sd, card_attr or key, None) + if cv is not None: + return cv + return generic + + class VideoGenerator: """Typed t2v entrypoint over a single resident v2 ``ModelInstance`` (mirrors fastvideo's).""" - def __init__(self, engine: Any, model_id: str, *, instance: Any = None, supports_av: bool = False) -> None: + def __init__(self, engine: Any, model_id: str, *, instance: Any = None, supports_av: bool = False, + card: Any = None) -> None: self._engine = engine self._model_id = model_id self._inst = instance # the resident ModelInstance (to read e.g. the audio sample rate) self._supports_av = supports_av # model emits joint video+sound (LTX-2.3 T2VS) -> auto-request audio + self._card = card # the v2 ModelCard (for per-model sampling_defaults) # --------------------------------------------------------------------- # @classmethod @@ -96,7 +115,7 @@ class VideoGenerator: # A model that advertises TEXT_TO_VIDEO_SOUND (LTX-2.3) generates joint video+audio; the # entrypoint then issues a T2VS request by default so a plain generate() yields both modalities. supports_av = card.capabilities.has(Capability.TEXT_TO_VIDEO_SOUND) - return cls(eng, card.model_id, instance=inst, supports_av=supports_av) + return cls(eng, card.model_id, instance=inst, supports_av=supports_av, card=card) def shutdown(self) -> None: """API parity with ``fastvideo.VideoGenerator.shutdown()``. The v2 bring-up holds a single @@ -141,16 +160,15 @@ class VideoGenerator: from fastvideo.api import GenerationRequest, OutputConfig, SamplingConfig audio = kwargs.pop("audio", None) # None -> auto (by model capability), True/False -> force sp = sampling_param + # Per-model defaults (LTX-2 wants 8/30 steps + its own res, Wan2.2-TI2V 704x1280@24fps, etc.) come + # from the resolved card; precedence is kwargs > SamplingParam > card defaults > generic fallback. + sd = getattr(self._card, "sampling_defaults", None) if self._card is not None else None - def pick(key: str, default: Any) -> Any: - if key in kwargs and kwargs[key] is not None: - return kwargs[key] - if sp is not None and getattr(sp, key, None) is not None: - return getattr(sp, key) - return default + def pick(key: str, default: Any, card_attr: str | None = None) -> Any: + return _resolve_default(key, default, kwargs, sp, sd, card_attr) sampling = SamplingConfig( - num_inference_steps=int(pick("num_inference_steps", 30)), + num_inference_steps=int(pick("num_inference_steps", 30, "num_steps")), seed=int(pick("seed", 1024)), num_frames=int(pick("num_frames", 25)), height=int(pick("height", 480)),