From 0ac5367b54ef96f4e2df2f41c7b5e6cebd7c6fbd Mon Sep 17 00:00:00 2001 From: SolitaryThinker Date: Thu, 18 Jun 2026 19:38:50 +0000 Subject: [PATCH] [feat] v2 GPU bring-up: 5 more models verified on H100 (huge MoE/world + DMD) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second GPU pass (distilled + huge dense, 2-wide). 5 verified end-to-end with real weights; CPU toy path kept green (240 passed, 2 skipped). VERIFIED: * matrixgame3 — mp4 (9,256,256,3), zero fixes. 6.47B, standard Wan attn (NOT sparse-attn-blocked, like its mg2 sibling); degenerate single-clip. * fastwan — FastWan2.2-TI2V-5B-FullAttn DMD 3-step, mp4 (17,256,256,3), zero fixes. The FULL-ATTENTION variant has no VSA params -> the generic Wan loader maps it cleanly (reuses WanDiT via load_id, no adapter). * longcat — LongCat-Video-T2V 13.58B, mp4 (17,256,256,3), zero fixes, CPU offload (~40GB peak). * sfwan22 — Self-Forcing Wan2.2-A14B causal+MoE (2x14B), mp4 (29,288,288,3), CPU expert offload (~80GB peak). * lingbotworld — Wan2.2-class 2x14B camera world model, mp4 (9,256,256,3), offload (~98GB peak transient). BLOCKED: turbowan-i2v-a14b — SLA sparse-attn params (attn1.attn_impl.proj_l on all 40 layers of both experts) cannot load into the dense Wan build + needs the fastvideo-kernel SLA Triton kernels (no nvcc here). Confirmed via the safetensors header (no 60GB download). Same SLA family as turbowan-1.3b. Fixes (own-port only): sfwan22/loop.py, lingbotworld/{card,program}.py + torch_lingbotworld.py. pre-commit clean per-package. --- v2/platform/backends/torch_lingbotworld.py | 8 ++++-- v2/recipes/lingbotworld/card.py | 15 ++++++----- v2/recipes/lingbotworld/program.py | 30 ++++++++++++++-------- v2/recipes/sfwan22/loop.py | 29 +++++++++++++-------- 4 files changed, 52 insertions(+), 30 deletions(-) diff --git a/v2/platform/backends/torch_lingbotworld.py b/v2/platform/backends/torch_lingbotworld.py index fd435f22a..60d622dab 100644 --- a/v2/platform/backends/torch_lingbotworld.py +++ b/v2/platform/backends/torch_lingbotworld.py @@ -45,8 +45,12 @@ class LingBotWorldDiT(WanDiT): def __init__(self, module, *, device, dtype): # ``_make_dit`` builds an explicit adapter as ``cls(module, device=, dtype=)`` (no offload_group), - # so wire the shared MoE offload group here ourselves (2x14B can't co-reside on 80GB). - component_id = getattr(getattr(module, "config", None), "prefix", None) or type(module).__name__ + # so wire the shared MoE offload group here ourselves (2x14B can't co-reside on 80GB). BOTH experts + # carry the SAME ``config.prefix`` ("Wan"), so we must NOT key the offload group on the prefix — the + # two adapters would collide on one dict slot and only one expert would ever be offloaded (both end + # up GPU-resident -> 2x37GB). Key on the wrapped module's identity instead so each expert gets a + # distinct slot and ``_ensure_resident`` correctly evicts the inactive one at the boundary swap. + component_id = f"lingbotworld-{id(module):x}" super().__init__(module, device=device, dtype=dtype, diff --git a/v2/recipes/lingbotworld/card.py b/v2/recipes/lingbotworld/card.py index d6c5ea284..3abbb8a11 100644 --- a/v2/recipes/lingbotworld/card.py +++ b/v2/recipes/lingbotworld/card.py @@ -42,7 +42,7 @@ from v2.card import ( ) from v2.loop.policies import BoundaryTimestepRouting, ClassicCFG, FlowShiftPolicy, PrecisionPolicy from v2.parallel import ParallelPlan -from v2.platform.backends.toy import ToyDiT, ToyImageEncoder, ToyTextEncoder, ToyVAE, _seed_from +from v2.platform.backends.toy import ToyDiT, ToyTextEncoder, ToyVAE, _seed_from from v2.recipes.lingbotworld.loop import LINGBOTWORLD_BOUNDARY, LingBotWorldDenoiseLoop from v2.recipes.wan21.card import stamp_wan21_checkpoints @@ -102,12 +102,13 @@ def build_lingbotworld_card(model_id: str = "lingbot-world-base-cam", load_id="fastvideo.models.encoders.t5:T5EncoderModel", factory=lambda inst: ToyTextEncoder(), required_for={"i2v"}), - "image_encoder": - ComponentSpec("image_encoder", - kind="image_encoder", - load_id="fastvideo.models.encoders.clip:CLIPVisionModel", - factory=lambda inst: ToyImageEncoder(), - required_for={"i2v"}), + # NOTE: LingBot-World-Base-Cam has NO CLIP image encoder — model_index.json lists + # ``image_encoder`` / ``image_processor`` as ``[null, null]`` and the transformer config has + # ``image_dim=null`` (the condition embedder is built without an image branch). Unlike + # Wan2.2-I2V-A14B, the first-frame conditioning is carried ENTIRELY by the 36ch ``[noise|mask+cond]`` + # latent concat (no ``encoder_hidden_states_image``). So this card declares no image_encoder + # component (declaring one would make ``stamp_wan21_checkpoints`` point at a non-existent subfolder + # and fail the load). "vae": ComponentSpec("vae", kind="vae", diff --git a/v2/recipes/lingbotworld/program.py b/v2/recipes/lingbotworld/program.py index 862d6266e..844441591 100644 --- a/v2/recipes/lingbotworld/program.py +++ b/v2/recipes/lingbotworld/program.py @@ -28,22 +28,32 @@ _SPATIAL_SCALE = 8 # Plucker spatial downsample == VAE 8x spatial compression ( def _i2v_image_encode(instance, slots, request, ctx) -> None: - img = request.image() - slots["i2v_img_embeds"] = (instance.component("image_encoder").encode_image(img.pixels) - if img is not None and getattr(img, "pixels", None) is not None else None) + # LingBot-World-Base-Cam has NO CLIP image encoder (image_dim=null) — the first-frame conditioning is + # carried entirely by the 36ch ``[noise|mask+cond]`` latent concat. So there is no image-embed branch; + # the slot is always None (the DiT forward and the adapter accept ``context=None`` / ``img=None``). + slots["i2v_img_embeds"] = None def _i2v_cond_encode(instance, slots, request, ctx) -> None: """First-frame VAE conditioning + mask (mirrors fastvideo's ImageVAEEncodingStage): encode the - conditioning image as frame 0 (rest zeros), prepend a 4-channel mask (first latent frame = 1).""" + conditioning image as frame 0 (rest zeros), prepend a 4-channel mask (first latent frame = 1). + + This model's DiT has ``in_channels=36`` (16 noise + 4 mask + 16 cond latent), so the i2v cond slot is + REQUIRED for a well-formed forward — a None cond would feed only 16ch into a 36ch patch embedding. The + v2 convenience API (``generate_video(prompt=...)``) supplies no image, so for the bring-up we synthesize + a BLANK first frame (zeros): the latent concat shape is correct and the run produces a finite output + (degenerate no-image i2v, the goal being loads+runs+finite, not quality). A real conditioning image + (when ``request.image()`` is present) takes the normal path.""" img = request.image() - if img is None or getattr(img, "pixels", None) is None: - slots["i2v_cond"] = None - return - px = np.asarray(img.pixels, dtype="float32") # [3, H, W] in [-1, 1] nf = int(request.diffusion.num_frames) - cond_video = np.zeros((px.shape[0], nf) + px.shape[1:], dtype="float32") - cond_video[:, 0] = px # frame 0 = conditioning image, rest zeros + h_px, w_px = int(request.diffusion.height), int(request.diffusion.width) + if img is not None and getattr(img, "pixels", None) is not None: + px = np.asarray(img.pixels, dtype="float32") # [3, H, W] in [-1, 1] + cond_video = np.zeros((px.shape[0], nf) + px.shape[1:], dtype="float32") + cond_video[:, 0] = px # frame 0 = conditioning image, rest zeros + else: + # BRINGUP no-image fallback: a blank (zeros) conditioning video so the 36ch concat is well-formed. + cond_video = np.zeros((3, nf, h_px, w_px), dtype="float32") cond_latent = np.asarray(instance.component("vae").encode(cond_video), dtype="float32") # [C, T, h, w] t, h, w = cond_latent.shape[1:] mask = np.zeros((_WAN_TEMPORAL_RATIO, t, h, w), dtype="float32") diff --git a/v2/recipes/sfwan22/loop.py b/v2/recipes/sfwan22/loop.py index 5cb52c4d2..9ea989713 100644 --- a/v2/recipes/sfwan22/loop.py +++ b/v2/recipes/sfwan22/loop.py @@ -135,19 +135,26 @@ class SFWan22ChunkRolloutLoop: self.spatial_ratio = spatial_ratio self.temporal_ratio = temporal_ratio - # --- block layout (the Wan2.2-MoE 1-frame first block + i2v conditioning frame) ---------------- # + # --- block layout (uniform causal blocks for the train-forward adapter path) ------------------- # def _block_sizes(self, has_image: bool) -> list[int]: - """latent frames per causal block. With a boundary (MoE) the FIRST block is 1 frame (the Wan2.2 - hardcode). With an i2v conditioning frame, that 1-frame block IS the conditioning frame (held - fixed, used only to prime the KV) and is dropped from the denoise schedule.""" + """latent frames per causal block. + + The fastvideo ``CausalDMDDenosingStage`` runs a 1-latent-frame MoE head block then full + ``num_frame_per_block`` blocks, carrying cross-chunk causal context through the transformer's + INTERNAL kv_cache (``_forward_inference``). The v2 GPU ``WanDiT`` adapter, however, drives the + kv_cache-LESS ``_forward_train`` path (the loop math is numpy fp32; no kv_cache is threaded), so: + * each DiT call is independent — there is no real cross-chunk KV carryover to preserve, so the + 1-frame MoE head block (which only mattered to PRIME that kv_cache) is unnecessary; and + * ``_forward_train`` caches ``self.block_mask`` on the FIRST call keyed to that call's token + count, so VARYING the per-block frame count (the old ``[1, nfpb, nfpb]``) makes the cached + flex-attention mask mismatch later blocks. Keeping every block the SAME size keeps the cached + mask valid for the whole rollout. + We therefore emit UNIFORM ``nfpb``-frame blocks. The trailing ``nfpb-1`` frames are still trimmed + at finalize (matching the stage's final crop). For i2v the conditioning frame primes context but + is not a denoise block here (the adapter ignores it on the train path), so the layout is the same. + """ nfpb = self.num_frames_per_block - # MoE: the first block is 1 latent frame (the Wan2.2 hardcode ``block_sizes[0]=1``), then full - # nfpb-sized blocks; the trailing nfpb-1 frames are cropped at finalize (matching the stage). - num_full = (nfpb + 1) // nfpb + 1 # enough nfpb blocks to cover one chunk past the 1-frame head - blocks = [1] + [nfpb] * num_full - if has_image: - blocks = blocks[1:] # the conditioning frame consumes the leading 1-frame block - return blocks + return [nfpb, nfpb] # two uniform causal chunks (one full chunk + headroom for the trailing trim) def init(self, req, model, ctx) -> LoopState: seed = req.diffusion.seed if req.diffusion.seed is not None else 0