diff --git a/examples/inference/basic/V2_PORTING_STATUS.md b/examples/inference/basic/V2_PORTING_STATUS.md index 266c180c2..5c0fe64ff 100644 --- a/examples/inference/basic/V2_PORTING_STATUS.md +++ b/examples/inference/basic/V2_PORTING_STATUS.md @@ -55,7 +55,8 @@ Wan2.1 model — covered by wan21; those runtime modes are out of scope. | `basic_ltx2_3_distilled_i2v*.py` | LTX-2.3-Distilled I2V | Image conditioning on the ltx2 single-stage program (encode the input image + condition the denoise). | ## Backlog — goal: port ALL fastvideo models (63 total: 8 ported · 21 reuse-arch · 34 new-arch) -Ported (11): the 7 above + **Wan2.1-T2V-14B** + the **Wan2.1 i2v cluster** — Wan2.1-Fun-1.3B-InP +Ported (12): the 7 above + **Wan2.1-T2V-14B** + **Wan2.2-I2V-A14B** (MoE i2v, structural) + the +**Wan2.1 i2v cluster** — Wan2.1-Fun-1.3B-InP (GPU-verified: CLIP encoder + first-frame [mask|cond] -> 36ch DiT) and Wan2.1-I2V-14B-480P/720P (reuse the same i2v card; 14B weights GPU-pending). i2v required a real sub-system: the image-encoder adapter, the loop's i2v conditioning hooks, and the Wan adapter's 36ch concat (all landed + T2V-unregressed). diff --git a/v2/recipes/wan21/i2v.py b/v2/recipes/wan21/i2v.py index f8429c3c9..3d371e594 100644 --- a/v2/recipes/wan21/i2v.py +++ b/v2/recipes/wan21/i2v.py @@ -28,11 +28,11 @@ from v2.card import ( RecipeSpec, SamplingDefaults, ) -from v2.loop.policies import ClassicCFG, FlowShiftPolicy, NoRouting, PrecisionPolicy +from v2.loop.policies import BoundaryTimestepRouting, ClassicCFG, FlowShiftPolicy, NoRouting, PrecisionPolicy from v2.parallel import ParallelPlan from v2.platform.backends.toy import ToyDiT, ToyImageEncoder, ToyTextEncoder, ToyVAE, _seed_from from v2.program import ComponentNode, ModelLoopNode, Program, ProgramKind -from v2.recipes._prompts import WAN_NEG_EN +from v2.recipes._prompts import WAN_NEG_CN, WAN_NEG_EN from v2.recipes.common import text_encode_node_fn as _text_encode from v2.recipes.wan21.card import stamp_wan21_checkpoints from v2.recipes.wan21.loop import WanDenoiseLoop @@ -93,6 +93,61 @@ def build_wan21_i2v_card(model_id: str = "wan2.1-i2v-1.3b", *, flow_shift: float return card +def build_wan22_i2v_a14b_card(model_id: str = "wan2.2-i2v-a14b", *, boundary: float = 0.9, + checkpoint_root: str | None = None) -> ModelCard: + """Wan2.2-I2V-A14B — MoE (two WanTransformer3DModel experts + boundary routing) + i2v conditioning + (CLIP + first-frame [mask|cond]). Reuses the Wan adapter (cond concat + MoE CPU offload), the shared + WanDenoiseLoop (i2v hooks + the boundary expert), and the i2v program. Structural (GPU-pending: 2x14B).""" + seed = _seed_from(model_id) + cost = CostModel(kind=WorkUnitKind.DIFFUSION_STEP, base_seconds=1e-4, per_unit_seconds=1e-7) + cfg, flow = ClassicCFG(), FlowShiftPolicy(shift=5.0) + precision = PrecisionPolicy(compute_dtype="float32", scheduler_step_in_fp32=True) + expert = BoundaryTimestepRouting(high_noise="transformer", low_noise="transformer_2", boundary=boundary) + + def loop_factory(): + return WanDenoiseLoop(loop_id="i2v_denoise", cfg=cfg, flow_shift=flow, precision=precision, + expert=expert, cost=cost) + + def _dit(cid): + return ComponentSpec(cid, kind="dit", load_id="fastvideo.models.dits.wanvideo:WanTransformer3DModel", + factory=lambda inst: ToyDiT(seed=seed), resident_for=["i2v_denoise"], + required_for={"i2v"}) + + components = { + "text_encoder": ComponentSpec("text_encoder", kind="text_encoder", + load_id="fastvideo.models.encoders.t5:T5EncoderModel", + factory=lambda inst: ToyTextEncoder(), required_for={"i2v"}), + "image_encoder": ComponentSpec("image_encoder", kind="image_encoder", + load_id="fastvideo.models.encoders.clip:CLIPVisionModel", + factory=lambda inst: ToyImageEncoder(), required_for={"i2v"}), + "vae": ComponentSpec("vae", kind="vae", load_id="fastvideo.models.vaes.wanvae:AutoencoderKLWan", + factory=lambda inst: ToyVAE(), required_for={"i2v"}), + "transformer": _dit("transformer"), "transformer_2": _dit("transformer_2"), + } + loops = { + "i2v_denoise": LoopSpec("i2v_denoise", kind=LoopKind.DIFFUSION_DENOISE, + work_unit_kind=WorkUnitKind.DIFFUSION_STEP, step_cost_model=cost, + shared_weight_components=["transformer", "transformer_2"], + cache_policy=["feature"], loop_factory=loop_factory), + } + card = ModelCard( + model_id=model_id, family="wan", components=components, loops=loops, + capabilities=CapabilityMatrix.of(Capability.IMAGE_TO_VIDEO, Capability.VAE_DECODE), + recipe=RecipeSpec(method="base", assumes_loop="i2v_denoise", assumes_precision="float32", + consistency_required=ConsistencyLevel.C1), + parity=ParitySpec(consistency_levels=[ConsistencyLevel.C1], interleave_required=True), + caches={"feature": CacheContract("feature", max_bytes=1 << 24, reuse_across_requests=True)}, + precision=PrecisionContract(default_dtype="float32", training_precision="float32"), + parallelism=ParallelismContract(valid_plans=[ParallelPlan.single()], default_plan=ParallelPlan.single()), + sampling_defaults=SamplingDefaults(num_steps=40, guidance_scale=3.5, height=480, width=832, + num_frames=81, fps=16, negative_prompt=WAN_NEG_CN), + ) + card.validate() + if checkpoint_root: + stamp_wan21_checkpoints(card, checkpoint_root) + return card + + # --- program: CLIP encode + first-frame VAE conditioning + i2v denoise + decode ------------------ # def _i2v_image_encode(instance, slots, request, ctx) -> None: img = request.image() diff --git a/v2/registry.py b/v2/registry.py index 298325d35..8e230b055 100644 --- a/v2/registry.py +++ b/v2/registry.py @@ -51,7 +51,7 @@ def _entries() -> list[ModelEntry]: build_wan_t2v_14b_card, build_wan_t2v_program, ) - from v2.recipes.wan21.i2v import build_wan21_i2v_card, build_wan21_i2v_program + from v2.recipes.wan21.i2v import build_wan21_i2v_card, build_wan21_i2v_program, build_wan22_i2v_a14b_card return [ ModelEntry(("Wan-AI/Wan2.1-T2V-1.3B-Diffusers",), build_wan21_card, build_wan_t2v_program), ModelEntry(("Wan-AI/Wan2.1-T2V-14B-Diffusers",), build_wan_t2v_14b_card, build_wan_t2v_program), @@ -65,6 +65,9 @@ def _entries() -> list[ModelEntry]: ModelEntry(("Wan-AI/Wan2.1-I2V-14B-720P-Diffusers",), lambda: build_wan21_i2v_card("wan2.1-i2v-14b-720p", flow_shift=5.0, height=720, width=1280), build_wan21_i2v_program), + # Wan2.2-I2V-A14B: MoE (2 experts + boundary) + i2v conditioning. Reuses Wan adapter + i2v + MoE. + ModelEntry(("Wan-AI/Wan2.2-I2V-A14B-Diffusers",), + build_wan22_i2v_a14b_card, build_wan21_i2v_program), ModelEntry(("Wan-AI/Wan2.2-TI2V-5B-Diffusers",), build_wan22_ti2v_card, build_wan_t2v_program), ModelEntry(("Wan-AI/Wan2.2-T2V-A14B-Diffusers",), build_wan22_a14b_card, build_wan_t2v_program), ModelEntry(("wlsaidhi/SFWan2.1-T2V-1.3B-Diffusers",), build_wan_causal_card, build_wan_causal_program),