[new-model] Added LingBot-World-Fast image-to-video support (#1665)
Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com> Co-authored-by: SolitaryThinker <wlsaidhi@gmail.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
SolitaryThinker
parent
d967b99928
commit
4b3f99b224
@@ -21,6 +21,7 @@ exec pytest \
|
||||
./fastvideo/tests/training/test_trackers.py \
|
||||
./fastvideo/tests/inference/test_basic_fasth3_omniref_pdd.py \
|
||||
./fastvideo/tests/attention/test_sdpa_metadata_mask_contract.py \
|
||||
./fastvideo/tests/attention/test_lingbotworld_fast_backend.py \
|
||||
./fastvideo/tests/attention/test_vsa_h3_tile_grad_safety.py \
|
||||
./fastvideo/tests/attention/test_vsa_h3_metadata.py \
|
||||
./fastvideo/tests/attention/test_vsa_h3_ref2va_regions.py \
|
||||
|
||||
@@ -61,6 +61,7 @@ column links a runnable script in `examples/inference/basic/` where one exists.
|
||||
| lingbot_video | `FastVideo/LingBot-Video-Dense-1.3B-Diffusers` | T2V | [basic_lingbot_video.py](https://github.com/hao-ai-lab/FastVideo/blob/main/examples/inference/basic/basic_lingbot_video.py) |
|
||||
| lingbotworld | `FastVideo/LingBot-World-Base-Cam-Diffusers` | I2V | [basic_lingbotworld_base_cam.py](https://github.com/hao-ai-lab/FastVideo/blob/main/examples/inference/basic/basic_lingbotworld_base_cam.py) |
|
||||
| lingbotworld2 | `robbyant/lingbot-world-v2-14b-causal-fast` | I2V | [basic_lingbotworld2_causal_fast.py](https://github.com/hao-ai-lab/FastVideo/blob/main/examples/inference/basic/basic_lingbotworld2_causal_fast.py) |
|
||||
| lingbotworld_fast | `FastVideo/LingBot-World-Fast-Diffusers` | I2V | [basic_lingbotworld_fast.py](https://github.com/hao-ai-lab/FastVideo/blob/main/examples/inference/basic/basic_lingbotworld_fast.py) |
|
||||
| longcat | `FastVideo/LongCat-Video-T2V-Diffusers` | T2V | [basic_longcat_t2v.py](https://github.com/hao-ai-lab/FastVideo/blob/main/examples/inference/basic/basic_longcat_t2v.py) |
|
||||
| longcat | `FastVideo/LongCat-Video-I2V-Diffusers` | I2V | [basic_longcat_i2v.py](https://github.com/hao-ai-lab/FastVideo/blob/main/examples/inference/basic/basic_longcat_i2v.py) |
|
||||
| longcat | `FastVideo/LongCat-Video-VC-Diffusers` | — | [basic_longcat_vc.py](https://github.com/hao-ai-lab/FastVideo/blob/main/examples/inference/basic/basic_longcat_vc.py) |
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
from fastvideo import VideoGenerator
|
||||
|
||||
OUTPUT_PATH = "video_samples_lingbotworld_fast"
|
||||
|
||||
|
||||
def main():
|
||||
generator = VideoGenerator.from_pretrained(
|
||||
"FastVideo/LingBot-World-Fast-Diffusers",
|
||||
num_gpus=1,
|
||||
use_fsdp_inference=False, # set to True if GPU is out of memory
|
||||
dit_cpu_offload=True,
|
||||
vae_cpu_offload=False,
|
||||
text_encoder_cpu_offload=True,
|
||||
pin_cpu_memory=True,
|
||||
)
|
||||
|
||||
prompt = (
|
||||
"A serene lakeside scene with a lone tree standing in calm water, "
|
||||
"surrounded by distant snow-capped mountains under a bright blue sky "
|
||||
"with drifting white clouds — gentle ripples reflect the tree and sky, "
|
||||
"creating a tranquil, meditative atmosphere.")
|
||||
action_path = "examples/datasets/lingbotworld2"
|
||||
image_path = f"{action_path}/image.jpg"
|
||||
|
||||
generator.generate_video(
|
||||
prompt,
|
||||
image_path=image_path,
|
||||
action_path=action_path,
|
||||
output_path=OUTPUT_PATH,
|
||||
save_video=True,
|
||||
num_frames=81,
|
||||
height=480,
|
||||
width=832,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -21,6 +21,7 @@ from fastvideo.configs.models.dits.kandinsky5 import Kandinsky5VideoConfig
|
||||
from fastvideo.configs.models.dits.kandinsky6 import Kandinsky6ArchConfig, Kandinsky6VideoAudioConfig
|
||||
from fastvideo.configs.models.dits.kandinsky6_sr import Kandinsky6SRArchConfig, Kandinsky6SRConfig
|
||||
from fastvideo.configs.models.dits.lingbotworld2 import LingBotWorld2CausalFastVideoConfig
|
||||
from fastvideo.configs.models.dits.lingbotworld_fast import LingBotWorldFastVideoConfig
|
||||
from fastvideo.configs.models.dits.lingbot_video import LingBotVideoConfig
|
||||
|
||||
__all__ = [
|
||||
@@ -29,5 +30,6 @@ __all__ = [
|
||||
"LongCatVideoConfig", "LTX2VideoConfig", "HYWorldConfig", "Kandinsky5VideoConfig", "Kandinsky6ArchConfig",
|
||||
"Kandinsky6VideoAudioConfig", "Kandinsky6SRArchConfig", "Kandinsky6SRConfig", "MagiHumanVideoConfig",
|
||||
"StableAudioConfig", "GlmImageDiTConfig", "HeliosConfig", "LingBotWorld2CausalFastVideoConfig",
|
||||
"LingBotWorldFastVideoConfig",
|
||||
"LingBotVideoConfig", "MiniMaxH3Config", "ZImageDiTConfig", "MMAudioArchConfig", "MMAudioTransformerConfig"
|
||||
]
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from fastvideo.configs.models.dits.base import DiTArchConfig, DiTConfig
|
||||
from fastvideo.configs.models.dits.lingbotworld2 import (
|
||||
LingBotWorld2CausalFastArchConfig, )
|
||||
|
||||
|
||||
@dataclass
|
||||
class LingBotWorldFastArchConfig(LingBotWorld2CausalFastArchConfig):
|
||||
"""Arch config for the released LingBot-World-Fast checkpoint.
|
||||
|
||||
The tensor layout is identical to the LingBot World 2 causal-fast model, so
|
||||
the parent's shapes and ``param_names_mapping`` are reused verbatim. Only
|
||||
the sampling/attention-window values released with this checkpoint differ.
|
||||
"""
|
||||
|
||||
# The released `generate_fast.py` leaves `--local_attn_size` at -1, so
|
||||
# self-attention stays global and the KV cache never evicts. `sink_size` is
|
||||
# therefore unreachable, but it is pinned to the official constructor's
|
||||
# value so the two implementations stay identical if the window is enabled.
|
||||
local_attn_size: int = -1
|
||||
sink_size: int = 9
|
||||
# `chunk_size` and `timesteps_index` are absent from the checkpoint config,
|
||||
# so these values are what the sampling loop actually runs with.
|
||||
chunk_size: int = 3
|
||||
timesteps_index: tuple[int, int, int, int] = (0, 179, 358, 679)
|
||||
|
||||
|
||||
@dataclass
|
||||
class LingBotWorldFastVideoConfig(DiTConfig):
|
||||
arch_config: DiTArchConfig = field(default_factory=LingBotWorldFastArchConfig)
|
||||
|
||||
prefix: str = "Wan"
|
||||
@@ -11,6 +11,7 @@ from fastvideo.configs.pipelines.kandinsky5 import Kandinsky5DMDConfig, Kandinsk
|
||||
from fastvideo.configs.pipelines.kandinsky6 import Kandinsky6TI2VAConfig
|
||||
from fastvideo.configs.pipelines.kandinsky6_sr import Kandinsky6SRPipelineConfig
|
||||
from fastvideo.configs.pipelines.lingbotworld2 import LingBotWorld2CausalFastI2V480PConfig
|
||||
from fastvideo.configs.pipelines.lingbotworld_fast import LingBotWorldFastI2V480PConfig
|
||||
from fastvideo.configs.pipelines.lingbot_video import LingBotVideoT2VConfig
|
||||
from fastvideo.configs.pipelines.matrixgame2 import MatrixGame2I2V480PConfig
|
||||
from fastvideo.configs.pipelines.matrixgame3 import MatrixGame3I2V720PConfig
|
||||
@@ -26,6 +27,7 @@ __all__ = [
|
||||
"SelfForcingWanT2V480PConfig", "LucyEditDevConfig", "CosmosConfig", "Cosmos25Config", "LTX2T2VConfig",
|
||||
"DreamXWorld5BCamPipelineConfig", "DreamXWorld5BARPipelineConfig", "HeliosPipelineConfig", "HYWorldConfig",
|
||||
"Kandinsky5T2VConfig", "Kandinsky5I2VConfig", "Kandinsky5DMDConfig", "Kandinsky6TI2VAConfig",
|
||||
"Kandinsky6SRPipelineConfig", "LingBotWorld2CausalFastI2V480PConfig", "LingBotVideoT2VConfig",
|
||||
"Kandinsky6SRPipelineConfig", "LingBotWorld2CausalFastI2V480PConfig", "LingBotWorldFastI2V480PConfig",
|
||||
"LingBotVideoT2VConfig",
|
||||
"MatrixGame2I2V480PConfig", "MatrixGame3I2V720PConfig", "MMAudioV2AConfig", "get_pipeline_config_cls_from_name"
|
||||
]
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from fastvideo.configs.models import DiTConfig, EncoderConfig
|
||||
from fastvideo.configs.models.dits.lingbotworld_fast import (
|
||||
LingBotWorldFastVideoConfig, )
|
||||
from fastvideo.configs.models.encoders import T5Config
|
||||
from fastvideo.configs.pipelines.lingbotworld2 import (
|
||||
LingBotWorld2CausalFastI2V480PConfig, )
|
||||
|
||||
|
||||
@dataclass
|
||||
class LingBotWorldFastI2V480PConfig(LingBotWorld2CausalFastI2V480PConfig):
|
||||
"""Pipeline config for LingBot-World-Fast 480P image-to-video.
|
||||
|
||||
Shares the LingBot World 2 causal-fast sampling loop and Wan VAE, but this
|
||||
checkpoint ships the stock ``UMT5EncoderModel`` text encoder (``d_model``
|
||||
fields) rather than LingBot World 2's custom one (``dim`` fields), so the
|
||||
standard ``T5Config`` is restored here alongside this DiT's arch config.
|
||||
"""
|
||||
|
||||
dit_config: DiTConfig = field(default_factory=LingBotWorldFastVideoConfig)
|
||||
text_encoder_configs: tuple[EncoderConfig, ...] = field(default_factory=lambda: (T5Config(), ))
|
||||
@@ -10,6 +10,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from fastvideo.attention.selector import backend_name_to_enum, get_attn_backend
|
||||
from fastvideo.configs.models.dits.lingbotworld2 import (
|
||||
LingBotWorld2CausalFastVideoConfig, )
|
||||
from fastvideo.distributed.communication_op import (
|
||||
@@ -19,6 +20,7 @@ from fastvideo.distributed.communication_op import (
|
||||
from fastvideo.distributed.parallel_state import get_sp_parallel_rank, get_sp_world_size
|
||||
from fastvideo.models.dits.base import BaseDiT
|
||||
from fastvideo.platforms import AttentionBackendEnum
|
||||
from fastvideo.utils import get_compute_dtype
|
||||
|
||||
try:
|
||||
import flash_attn_interface
|
||||
@@ -151,6 +153,7 @@ def attention(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
backend: AttentionBackendEnum,
|
||||
q_lens: torch.Tensor | None = None,
|
||||
k_lens: torch.Tensor | None = None,
|
||||
dropout_p: float = 0.0,
|
||||
@@ -162,8 +165,8 @@ def attention(
|
||||
dtype: torch.dtype = torch.bfloat16,
|
||||
fa_version: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Dispatch LingBot World 2 attention to FlashAttention when available."""
|
||||
if FLASH_ATTN_2_AVAILABLE or FLASH_ATTN_3_AVAILABLE:
|
||||
"""Dispatch LingBot World 2 attention through the selected backend."""
|
||||
if backend == AttentionBackendEnum.FLASH_ATTN:
|
||||
return flash_attention(
|
||||
q=q,
|
||||
k=k,
|
||||
@@ -180,6 +183,7 @@ def attention(
|
||||
version=fa_version,
|
||||
)
|
||||
|
||||
assert backend == AttentionBackendEnum.TORCH_SDPA
|
||||
if q_lens is not None or k_lens is not None:
|
||||
warnings.warn("Padding masks are disabled without FlashAttention.")
|
||||
q = q.transpose(1, 2).to(dtype)
|
||||
@@ -262,6 +266,14 @@ class CausalWanSelfAttention(nn.Module):
|
||||
self.dim = dim
|
||||
self.num_heads = num_heads
|
||||
self.head_dim = dim // num_heads
|
||||
attn_backend = get_attn_backend(
|
||||
self.head_dim,
|
||||
get_compute_dtype(),
|
||||
supported_attention_backends=(AttentionBackendEnum.FLASH_ATTN, AttentionBackendEnum.TORCH_SDPA),
|
||||
)
|
||||
backend_name = attn_backend.get_name()
|
||||
self.backend = backend_name_to_enum(backend_name)
|
||||
assert self.backend is not None, f"Unsupported attention backend: {backend_name}"
|
||||
self.local_attn_size = local_attn_size
|
||||
self.sink_size = sink_size
|
||||
self.qk_norm = qk_norm
|
||||
@@ -348,7 +360,7 @@ class CausalWanSelfAttention(nn.Module):
|
||||
|
||||
k_cache = kv_cache["k"][:, max(0, local_end_index - max_attention_size):local_end_index]
|
||||
v_cache = kv_cache["v"][:, max(0, local_end_index - max_attention_size):local_end_index]
|
||||
x = attention(roped_query, k_cache, v_cache)
|
||||
x = attention(roped_query, k_cache, v_cache, self.backend)
|
||||
kv_cache["global_end_index"].fill_(current_end)
|
||||
kv_cache["local_end_index"].fill_(local_end_index)
|
||||
|
||||
@@ -389,7 +401,11 @@ class WanCrossAttention(CausalWanSelfAttention):
|
||||
else:
|
||||
k = self.norm_k(self.k(context)).view(b, -1, n, d)
|
||||
v = self.v(context).view(b, -1, n, d)
|
||||
x = flash_attention(q, k, v, k_lens=context_lens)
|
||||
# Dispatch through `attention` so the TORCH_SDPA backend this model
|
||||
# advertises actually works; self-attention already does the same. The
|
||||
# SDPA path ignores `k_lens`, which is safe here because the caller
|
||||
# always passes `context_lens=None`.
|
||||
x = attention(q, k, v, self.backend, k_lens=context_lens)
|
||||
return self.o(x.flatten(2))
|
||||
|
||||
|
||||
|
||||
@@ -64,6 +64,13 @@ _IMAGE_TO_VIDEO_DIT_MODELS = {
|
||||
"lingbotworld2",
|
||||
"LingBotWorld2CausalFastTransformer3DModel",
|
||||
),
|
||||
# LingBot-World-Fast ships this class name; its tensor layout is identical
|
||||
# to the LingBot World 2 causal-fast DiT.
|
||||
"CausalLingBotWorldTransformer3DModel": (
|
||||
"dits",
|
||||
"lingbotworld2",
|
||||
"LingBotWorld2CausalFastTransformer3DModel",
|
||||
),
|
||||
"MatrixGame2WanModel": ("dits", "matrixgame2", "MatrixGame2WanModel"),
|
||||
"CausalMatrixGame2WanModel": ("dits", "matrixgame2", "CausalMatrixGame2WanModel"),
|
||||
# Legacy aliases for older HF model_index.json files
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
from .fast_pipeline import LingBotWorldFastPipeline
|
||||
|
||||
__all__ = ["LingBotWorldFastPipeline"]
|
||||
|
||||
EntryClass = LingBotWorldFastPipeline
|
||||
@@ -0,0 +1,38 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""LingBot-World-Fast causal image-to-video pipeline."""
|
||||
|
||||
from fastvideo.fastvideo_args import FastVideoArgs
|
||||
from fastvideo.logger import init_logger
|
||||
from fastvideo.models.schedulers.scheduling_flow_unipc_multistep import (
|
||||
FlowUniPCMultistepScheduler, )
|
||||
from fastvideo.pipelines.basic.lingbotworld2.causal_fast_pipeline import (
|
||||
LingBotWorld2CausalFastPipeline, )
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class LingBotWorldFastPipeline(LingBotWorld2CausalFastPipeline):
|
||||
"""LingBot-World-Fast I2V generation.
|
||||
|
||||
The released checkpoint uses the same chunked causal sampling loop as
|
||||
LingBot World 2 causal-fast; the chunk size, timestep indices, and
|
||||
attention window come from ``LingBotWorldFastArchConfig``.
|
||||
"""
|
||||
|
||||
def initialize_pipeline(self, fastvideo_args: FastVideoArgs) -> None:
|
||||
"""Install the flow-matching scheduler the released model samples with.
|
||||
|
||||
This checkpoint ships a stock ``UniPCMultistepScheduler``, whose
|
||||
``set_timesteps`` takes no ``shift``. The reference ``generate_fast.py``
|
||||
builds a ``FlowUniPCMultistepScheduler`` instead, so replace the loaded
|
||||
one unconditionally rather than only when absent.
|
||||
"""
|
||||
arch_config = fastvideo_args.pipeline_config.dit_config.arch_config
|
||||
self.modules["scheduler"] = FlowUniPCMultistepScheduler(
|
||||
num_train_timesteps=arch_config.num_train_timesteps,
|
||||
shift=1,
|
||||
use_dynamic_shifting=False,
|
||||
)
|
||||
|
||||
|
||||
EntryClass = LingBotWorldFastPipeline
|
||||
@@ -0,0 +1,31 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""LingBot-World-Fast pipeline preset."""
|
||||
|
||||
from fastvideo.api.presets import InferencePreset, PresetStageSpec
|
||||
|
||||
_DENOISE_STAGE = PresetStageSpec(
|
||||
name="denoise",
|
||||
kind="denoising",
|
||||
description="Causal-fast denoising pass",
|
||||
)
|
||||
|
||||
LINGBOTWORLD_FAST_I2V = InferencePreset(
|
||||
name="lingbotworld_fast_i2v",
|
||||
version=1,
|
||||
model_family="lingbotworld_fast",
|
||||
description="LingBot-World-Fast 14B causal I2V",
|
||||
workload_type="i2v",
|
||||
stage_schemas=(_DENOISE_STAGE, ),
|
||||
defaults={
|
||||
"guidance_scale": 1.0,
|
||||
"num_inference_steps": 4,
|
||||
"fps": 16,
|
||||
"seed": 42,
|
||||
"num_frames": 81,
|
||||
"height": 480,
|
||||
"width": 832,
|
||||
"negative_prompt": "",
|
||||
},
|
||||
)
|
||||
|
||||
ALL_PRESETS = (LINGBOTWORLD_FAST_I2V, )
|
||||
+28
-2
@@ -35,6 +35,7 @@ from fastvideo.configs.pipelines.kandinsky6_sr import Kandinsky6SRPipelineConfig
|
||||
from fastvideo.configs.pipelines.lingbot_video import LingBotVideoT2VConfig
|
||||
from fastvideo.configs.pipelines.lingbotworld import LingBotWorldI2V480PConfig
|
||||
from fastvideo.configs.pipelines.lingbotworld2 import LingBotWorld2CausalFastI2V480PConfig
|
||||
from fastvideo.configs.pipelines.lingbotworld_fast import LingBotWorldFastI2V480PConfig
|
||||
from fastvideo.configs.pipelines.longcat import LongCatT2V480PConfig
|
||||
from fastvideo.pipelines.basic.ltx2.pipeline_configs import LTX2T2VConfig
|
||||
from fastvideo.configs.pipelines.flux_2 import (
|
||||
@@ -600,6 +601,27 @@ def _register_configs() -> None:
|
||||
default_preset="lingbotworld2_causal_fast_i2v",
|
||||
)
|
||||
|
||||
# LingBotWorld-Fast — registered BEFORE the LingBotWorld base entry so its
|
||||
# detector wins when both fire. The base detector only excludes the
|
||||
# "causal-fast" spelling, so it also matches this checkpoint's path and its
|
||||
# `lingbotworldcausaldmdpipeline` model_index name; the detector loop keeps
|
||||
# the first match, which must be this more specific entry.
|
||||
register_configs(
|
||||
sampling_param_cls=None,
|
||||
pipeline_config_cls=LingBotWorldFastI2V480PConfig,
|
||||
workload_types=(WorkloadType.I2V, ),
|
||||
hf_model_paths=[
|
||||
"FastVideo/LingBot-World-Fast-Diffusers",
|
||||
],
|
||||
model_detectors=[
|
||||
lambda path: ("lingbot-world-fast" in path.lower() or "lingbotworldfast" in path.lower() or
|
||||
"lingbotworldcausaldmdpipeline" in path.lower())
|
||||
],
|
||||
model_family="lingbotworld_fast",
|
||||
default_preset="lingbotworld_fast_i2v",
|
||||
pipeline_cls_name="LingBotWorldFastPipeline",
|
||||
)
|
||||
|
||||
# LingBotWorld
|
||||
register_configs(
|
||||
sampling_param_cls=None,
|
||||
@@ -609,8 +631,9 @@ def _register_configs() -> None:
|
||||
"FastVideo/LingBot-World-Base-Cam-Diffusers",
|
||||
],
|
||||
model_detectors=[
|
||||
lambda path: (("lingbotworld" in path.lower() or "lingbot-world" in path.lower()) and "causal-fast" not in
|
||||
path.lower() and "causalfast" not in path.lower())
|
||||
lambda path:
|
||||
(("lingbotworld" in path.lower() or "lingbot-world" in path.lower()) and "causal-fast" not in path.lower()
|
||||
and "causalfast" not in path.lower() and "-fast" not in path.lower() and "causaldmd" not in path.lower())
|
||||
],
|
||||
model_family="lingbotworld",
|
||||
default_preset="lingbotworld_i2v",
|
||||
@@ -1328,6 +1351,8 @@ def _register_presets() -> None:
|
||||
ALL_PRESETS as LINGBOTWORLD_PRESETS, )
|
||||
from fastvideo.pipelines.basic.lingbotworld2.presets import (
|
||||
ALL_PRESETS as LINGBOTWORLD2_PRESETS, )
|
||||
from fastvideo.pipelines.basic.lingbotworld_fast.presets import (
|
||||
ALL_PRESETS as LINGBOTWORLD_FAST_PRESETS, )
|
||||
from fastvideo.pipelines.basic.lingbot_video.presets import (
|
||||
ALL_PRESETS as LINGBOT_VIDEO_PRESETS, )
|
||||
from fastvideo.pipelines.basic.longcat.presets import (
|
||||
@@ -1373,6 +1398,7 @@ def _register_presets() -> None:
|
||||
LINGBOT_VIDEO_PRESETS,
|
||||
LINGBOTWORLD_PRESETS,
|
||||
LINGBOTWORLD2_PRESETS,
|
||||
LINGBOTWORLD_FAST_PRESETS,
|
||||
LONGCAT_PRESETS,
|
||||
LTX2_PRESETS,
|
||||
MAGI_HUMAN_PRESETS,
|
||||
|
||||
@@ -715,6 +715,48 @@ class TestLingBotVideoPresets:
|
||||
validate_stage_overrides(preset, {"refine": {"refiner_steps": 8}})
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# LingBot-World-Fast preset integration
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestLingBotWorldFastPresets:
|
||||
|
||||
def test_registered_defaults_are_fixed(self) -> None:
|
||||
import fastvideo.registry # noqa: F401
|
||||
from fastvideo.registry import get_preset_selection
|
||||
|
||||
model_path = "FastVideo/LingBot-World-Fast-Diffusers"
|
||||
assert get_preset_selection(model_path) == ("lingbotworld_fast_i2v", "lingbotworld_fast")
|
||||
|
||||
preset = get_preset("lingbotworld_fast_i2v", "lingbotworld_fast")
|
||||
assert preset.defaults["num_inference_steps"] == 4
|
||||
assert preset.defaults["guidance_scale"] == 1.0
|
||||
assert preset.stage_schemas[0].allowed_overrides == frozenset()
|
||||
with pytest.raises(ConfigValidationError, match="does not accept overrides"):
|
||||
validate_stage_overrides(preset, {"denoise": {"num_inference_steps": 8}})
|
||||
|
||||
def test_local_checkpoint_resolves_fast_pipeline(self, tmp_path) -> None:
|
||||
from fastvideo.configs.pipelines.lingbotworld_fast import LingBotWorldFastI2V480PConfig
|
||||
from fastvideo.fastvideo_args import WorkloadType
|
||||
from fastvideo.pipelines.basic.lingbotworld_fast import LingBotWorldFastPipeline
|
||||
from fastvideo.registry import get_model_info
|
||||
|
||||
model_dir = tmp_path / "lingbot-world-fast"
|
||||
(model_dir / "transformer").mkdir(parents=True)
|
||||
(model_dir / "model_index.json").write_text(
|
||||
json.dumps({
|
||||
"_class_name": "LingBotWorldCausalDMDPipeline",
|
||||
"_diffusers_version": "0.36.0",
|
||||
}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
info = get_model_info(str(model_dir), workload_type=WorkloadType.I2V)
|
||||
assert info.pipeline_cls is LingBotWorldFastPipeline
|
||||
assert info.pipeline_config_cls is LingBotWorldFastI2V480PConfig
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# LingBotWorld preset integration (dual guidance)
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from fastvideo.models.dits.lingbotworld2 import causal_fast
|
||||
from fastvideo.platforms import AttentionBackendEnum
|
||||
|
||||
|
||||
class _SDPABackend:
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "TORCH_SDPA"
|
||||
|
||||
|
||||
def test_lingbot_attention_honors_selected_sdpa_backend(monkeypatch) -> None:
|
||||
monkeypatch.setattr(causal_fast, "get_attn_backend", lambda *args, **kwargs: _SDPABackend)
|
||||
layer = causal_fast.CausalWanSelfAttention(dim=8, num_heads=2)
|
||||
assert layer.backend == AttentionBackendEnum.TORCH_SDPA
|
||||
|
||||
monkeypatch.setattr(causal_fast, "flash_attention", lambda **kwargs: pytest.fail("FlashAttention was called"))
|
||||
q = torch.arange(24, dtype=torch.float32).reshape(1, 3, 2, 4) / 24
|
||||
k = q.flip(1)
|
||||
v = q + 1
|
||||
|
||||
actual = causal_fast.attention(q, k, v, layer.backend, dtype=torch.float32)
|
||||
expected = F.scaled_dot_product_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2))
|
||||
torch.testing.assert_close(actual, expected.transpose(1, 2).contiguous())
|
||||
@@ -0,0 +1,199 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""SSIM-based similarity test for LingBot-World-Fast causal I2V.
|
||||
|
||||
The camera trajectory and source image are the tracked copies of official
|
||||
LingBot example 03, matching the released `run_fast.sh` workflow.
|
||||
|
||||
Note: this checkpoint is 4-step distilled, so the step count is fixed by the
|
||||
model rather than reduced for CI.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import fastvideo.envs as envs
|
||||
from fastvideo import VideoGenerator
|
||||
from fastvideo.logger import init_logger
|
||||
from fastvideo.tests.ssim.bootstrap_references import (
|
||||
xfail_missing_reference_in_bootstrap_mode, )
|
||||
from fastvideo.tests.ssim.reference_utils import (
|
||||
build_generated_output_dir,
|
||||
build_reference_folder_path,
|
||||
get_cuda_device_name,
|
||||
resolve_device_reference_folder,
|
||||
select_ssim_params,
|
||||
)
|
||||
from fastvideo.tests.utils import compute_video_ssim_torchvision, write_ssim_results
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
REQUIRED_GPUS = 2
|
||||
|
||||
# The released checkpoint is 4-step distilled; see LingBotWorldFastArchConfig.
|
||||
NUM_DISTILLED_STEPS = 4
|
||||
|
||||
|
||||
def _find_lingbotworld_action_path() -> str | None:
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
repo_root = os.path.abspath(os.path.join(script_dir, "..", "..", ".."))
|
||||
candidate = os.path.join(repo_root, "examples", "datasets", "lingbotworld2")
|
||||
if (os.path.exists(os.path.join(candidate, "image.jpg"))
|
||||
and os.path.exists(os.path.join(candidate, "poses.npy"))
|
||||
and os.path.exists(os.path.join(candidate, "intrinsics.npy"))):
|
||||
return os.path.abspath(candidate)
|
||||
return None
|
||||
|
||||
|
||||
device_name = get_cuda_device_name()
|
||||
device_reference_folder = resolve_device_reference_folder(
|
||||
(
|
||||
("A40", "A40"),
|
||||
("L40S", "L40S"),
|
||||
("H100", "H100"),
|
||||
# GB200 must precede any bare "B200" pattern: the lookup is a
|
||||
# substring scan and "B200" is a substring of "NVIDIA GB200".
|
||||
("GB200", "GB200"),
|
||||
("H200", "H200"),
|
||||
),
|
||||
device_name=device_name,
|
||||
logger=logger,
|
||||
)
|
||||
|
||||
LINGBOT_FAST_PARAMS = {
|
||||
"model_path": "FastVideo/LingBot-World-Fast-Diffusers",
|
||||
"num_gpus": 2,
|
||||
"height": 480,
|
||||
"width": 832,
|
||||
"num_frames": 25, # must be 4k+1; trimmed to a whole number of chunks
|
||||
"seed": 42,
|
||||
"fps": 16,
|
||||
}
|
||||
LINGBOT_FAST_FULL_QUALITY_PARAMS = {
|
||||
**LINGBOT_FAST_PARAMS,
|
||||
"num_frames": 81,
|
||||
}
|
||||
|
||||
TEST_PROMPTS = [
|
||||
"A serene lakeside scene with a lone tree standing in calm water, surrounded "
|
||||
"by distant snow-capped mountains under a bright blue sky with drifting white "
|
||||
"clouds — gentle ripples reflect the tree and sky, creating a tranquil, "
|
||||
"meditative atmosphere.",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prompt", TEST_PROMPTS)
|
||||
@pytest.mark.parametrize("ATTENTION_BACKEND", ["FLASH_ATTN"])
|
||||
def test_lingbot_fast_i2v_similarity(prompt: str, ATTENTION_BACKEND: str, env_overrides):
|
||||
env_overrides.enter_context(envs.FASTVIDEO_ATTENTION_BACKEND.override(ATTENTION_BACKEND))
|
||||
|
||||
params = select_ssim_params(LINGBOT_FAST_PARAMS,
|
||||
LINGBOT_FAST_FULL_QUALITY_PARAMS)
|
||||
|
||||
if device_reference_folder is None:
|
||||
pytest.skip(
|
||||
f"Unsupported device for LingBot-World-Fast SSIM test: {device_name}"
|
||||
)
|
||||
if torch.cuda.device_count() < REQUIRED_GPUS:
|
||||
pytest.skip(
|
||||
f"LingBot-World-Fast SSIM test requires {REQUIRED_GPUS} GPUs, "
|
||||
f"but only {torch.cuda.device_count()} detected.")
|
||||
|
||||
action_path = _find_lingbotworld_action_path()
|
||||
if action_path is None:
|
||||
pytest.skip(
|
||||
"LingBot camera assets not found under examples/datasets/lingbotworld2."
|
||||
)
|
||||
image_path = os.path.join(action_path, "image.jpg")
|
||||
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
model_id = "LingBot-World-Fast-Diffusers"
|
||||
output_dir = build_generated_output_dir(
|
||||
script_dir,
|
||||
device_reference_folder,
|
||||
model_id,
|
||||
ATTENTION_BACKEND,
|
||||
)
|
||||
output_video_name = f"{prompt[:100].strip()}.mp4"
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
init_kwargs = {
|
||||
"num_gpus": params["num_gpus"],
|
||||
"use_fsdp_inference": True,
|
||||
"dit_cpu_offload": True,
|
||||
"dit_layerwise_offload": False,
|
||||
"text_encoder_cpu_offload": True,
|
||||
"vae_cpu_offload": False,
|
||||
"pin_cpu_memory": True,
|
||||
}
|
||||
generation_kwargs = {
|
||||
"output_path": output_dir,
|
||||
"image_path": image_path,
|
||||
"action_path": action_path,
|
||||
"height": params["height"],
|
||||
"width": params["width"],
|
||||
"num_frames": params["num_frames"],
|
||||
"seed": params["seed"],
|
||||
"fps": params["fps"],
|
||||
}
|
||||
|
||||
generator: VideoGenerator | None = None
|
||||
try:
|
||||
generator = VideoGenerator.from_pretrained(
|
||||
model_path=params["model_path"], **init_kwargs)
|
||||
generator.generate_video(prompt, **generation_kwargs)
|
||||
finally:
|
||||
if generator is not None:
|
||||
generator.shutdown()
|
||||
|
||||
generated_video_path = os.path.join(output_dir, output_video_name)
|
||||
assert os.path.exists(generated_video_path), (
|
||||
f"Output video was not generated at {generated_video_path}")
|
||||
|
||||
reference_folder = build_reference_folder_path(
|
||||
script_dir,
|
||||
device_reference_folder,
|
||||
model_id,
|
||||
ATTENTION_BACKEND,
|
||||
)
|
||||
if not os.path.exists(reference_folder):
|
||||
xfail_missing_reference_in_bootstrap_mode(
|
||||
generated_artifact_path=generated_video_path,
|
||||
reference_folder=reference_folder,
|
||||
artifact_kind="video",
|
||||
)
|
||||
raise FileNotFoundError(
|
||||
f"Reference video folder does not exist: {reference_folder}")
|
||||
|
||||
reference_video_name = None
|
||||
for filename in os.listdir(reference_folder):
|
||||
if filename.endswith(".mp4") and prompt[:100].strip() in filename:
|
||||
reference_video_name = filename
|
||||
break
|
||||
if not reference_video_name:
|
||||
xfail_missing_reference_in_bootstrap_mode(
|
||||
generated_artifact_path=generated_video_path,
|
||||
reference_folder=reference_folder,
|
||||
artifact_kind="video",
|
||||
)
|
||||
raise FileNotFoundError(
|
||||
f"Reference video missing for prompt/backend under {reference_folder}"
|
||||
)
|
||||
|
||||
reference_video_path = os.path.join(reference_folder, reference_video_name)
|
||||
logger.info("Computing SSIM between %s and %s", reference_video_path,
|
||||
generated_video_path)
|
||||
ssim_values = compute_video_ssim_torchvision(reference_video_path,
|
||||
generated_video_path,
|
||||
use_ms_ssim=True)
|
||||
mean_ssim = ssim_values[0]
|
||||
logger.info("SSIM mean value: %s", mean_ssim)
|
||||
|
||||
write_ssim_results(output_dir, ssim_values, reference_video_path,
|
||||
generated_video_path, NUM_DISTILLED_STEPS, prompt)
|
||||
|
||||
min_acceptable_ssim = 0.70
|
||||
assert mean_ssim >= min_acceptable_ssim, (
|
||||
f"SSIM value {mean_ssim} is below threshold {min_acceptable_ssim} "
|
||||
f"for {model_id} with backend {ATTENTION_BACKEND}")
|
||||
@@ -0,0 +1,64 @@
|
||||
# LingBot-World-Fast port status
|
||||
|
||||
## Summary
|
||||
|
||||
- model family: `lingbotworld_fast`
|
||||
- workload: causal I2V with camera poses
|
||||
- official source: `https://github.com/Robbyant/lingbot-world`
|
||||
- official checkpoint: `robbyant/lingbot-world-fast`
|
||||
- FastVideo bundle: `FastVideo/LingBot-World-Fast-Diffusers`
|
||||
- source layout: official DiT checkpoint plus shared Wan components
|
||||
- current phase: SSIM seeding
|
||||
|
||||
## Reused implementation evidence
|
||||
|
||||
The FastVideo variant reuses
|
||||
`LingBotWorld2CausalFastTransformer3DModel`. The released transformer index has
|
||||
1,421 parameter keys, and the PR's initial validation found all names and
|
||||
shapes identical to the reused implementation.
|
||||
|
||||
Variant-specific runtime values:
|
||||
|
||||
| Setting | Value | Source |
|
||||
|---|---:|---|
|
||||
| `local_attn_size` | `-1` | released checkpoint config / `generate_fast.py` default |
|
||||
| `chunk_size` | `3` | official `generate_fast.py` call |
|
||||
| fixed timestep indices | `(0, 179, 358, 679)` | released fast sampling path |
|
||||
| `sample_shift` | `10.0` | official I2V config |
|
||||
|
||||
## Verification matrix
|
||||
|
||||
| Scope | Command | Latest result |
|
||||
|---|---|---|
|
||||
| Official-vs-FastVideo full DiT | See `README.md` | 2026-08-17: 2 passed; max/mean/relative-mean error all `0.0` |
|
||||
| Backend selection | `pytest fastvideo/tests/attention/test_lingbotworld_fast_backend.py -v` | 2026-08-17: passed |
|
||||
| Preset and exact pipeline resolution | `pytest fastvideo/tests/api/test_presets.py::TestLingBotWorldFastPresets -v` | 2026-08-17: 2 passed |
|
||||
| SSIM collection on this GB10 host | `pytest fastvideo/tests/ssim/test_lingbot_fast_similarity.py -v -s` | 2026-08-17: 1 skipped; unsupported GB10 |
|
||||
| L40S reference generation and SSIM | `seed-ssim-references` workflow | blocked: Modal CLI and HF write token unavailable locally |
|
||||
|
||||
## Acceptance requirements
|
||||
|
||||
- [x] Official and FastVideo implementations load real released weights.
|
||||
- [x] Both execute a real full-transformer forward.
|
||||
- [x] Full-transformer numerical comparison passes without skip.
|
||||
- [x] Backend and preset/registry tests pass.
|
||||
- [ ] L40S SSIM reference is generated and visually approved.
|
||||
- [ ] SSIM reference is uploaded and the package test passes against it.
|
||||
|
||||
## Environment notes
|
||||
|
||||
Local verification host:
|
||||
|
||||
- GPU: NVIDIA GB10
|
||||
- detected GPU count: 1
|
||||
- PyTorch: 2.12.0 + CUDA 13.0
|
||||
- FlashAttention: unavailable; parity aligns both implementations on SDPA
|
||||
|
||||
The SSIM suite does not accept GB10 references and the test requires two GPUs.
|
||||
This machine can validate transformer parity and CPU contract tests, but it
|
||||
cannot produce the required L40S SSIM acceptance evidence.
|
||||
|
||||
## Open blockers
|
||||
|
||||
1. Run the new SSIM test on Modal L40S, inspect the generated video, and seed
|
||||
`FastVideo/ssim-reference-videos`.
|
||||
@@ -0,0 +1,103 @@
|
||||
# LingBot-World-Fast local verification
|
||||
|
||||
Local-only numerical parity and port evidence for
|
||||
`FastVideo/LingBot-World-Fast-Diffusers`.
|
||||
|
||||
## Scope
|
||||
|
||||
- Official implementation: `Robbyant/lingbot-world`, `wan/modules/model_fast.py`
|
||||
- Released fast checkpoint: `robbyant/lingbot-world-fast`
|
||||
- FastVideo bundle: `FastVideo/LingBot-World-Fast-Diffusers`
|
||||
- Workload: causal image-to-video with camera poses
|
||||
- Released sampling contract:
|
||||
- global self-attention (`local_attn_size=-1`)
|
||||
- three latent frames per chunk
|
||||
- four fixed scheduler indices: `(0, 179, 358, 679)`
|
||||
- flow shift `10.0`
|
||||
|
||||
## Setup
|
||||
|
||||
Use the current FastVideo environment. The official source needs `easydict` in
|
||||
addition to FastVideo's dependencies.
|
||||
|
||||
```bash
|
||||
git clone https://github.com/Robbyant/lingbot-world.git \
|
||||
/tmp/lingbot-world-reference
|
||||
uv pip install --python .venv/bin/python easydict
|
||||
```
|
||||
|
||||
Resolve a local snapshot of the FastVideo bundle:
|
||||
|
||||
```bash
|
||||
LINGBOTWORLD_FAST_MODEL_DIR=$(
|
||||
.venv/bin/python - <<'PY'
|
||||
from huggingface_hub import snapshot_download
|
||||
print(snapshot_download("FastVideo/LingBot-World-Fast-Diffusers"))
|
||||
PY
|
||||
)
|
||||
export LINGBOTWORLD_FAST_MODEL_DIR
|
||||
export LINGBOTWORLD_FAST_REFERENCE_DIR=/tmp/lingbot-world-reference
|
||||
```
|
||||
|
||||
Do not store Hugging Face tokens in this directory.
|
||||
|
||||
## Full transformer parity
|
||||
|
||||
The parity test loads the same released transformer weights into both:
|
||||
|
||||
1. the official `WanModelFast` implementation; and
|
||||
2. FastVideo's production `TransformerLoader`.
|
||||
|
||||
It runs the models sequentially to stay within memory limits. Both sides use
|
||||
the mathematical SDPA backend. The official cross-attention helper is routed
|
||||
through the official source's own SDPA-capable dispatcher because the direct
|
||||
helper only supports FlashAttention.
|
||||
|
||||
The deterministic input is one real three-frame causal chunk with:
|
||||
|
||||
- 16 noisy latent channels;
|
||||
- 20 image-conditioning channels;
|
||||
- 4096-wide text context;
|
||||
- packed 384-channel camera rays;
|
||||
- real per-layer self- and cross-attention caches.
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
FASTVIDEO_ATTENTION_BACKEND=TORCH_SDPA \
|
||||
LINGBOTWORLD_FAST_MODEL_DIR="$LINGBOTWORLD_FAST_MODEL_DIR" \
|
||||
LINGBOTWORLD_FAST_REFERENCE_DIR="$LINGBOTWORLD_FAST_REFERENCE_DIR" \
|
||||
.venv/bin/python -m pytest \
|
||||
tests/local_tests/lingbotworld_fast/test_lingbotworld_fast_transformer_parity.py \
|
||||
-v -s
|
||||
```
|
||||
|
||||
This is valid evidence only when the full-transformer test reports `PASSED`,
|
||||
not `SKIPPED`.
|
||||
|
||||
## Lightweight runtime-contract tests
|
||||
|
||||
These tests cover backend dispatch, the fixed-step preset contract, and exact
|
||||
registry/pipeline resolution without loading the 18.5B transformer:
|
||||
|
||||
```bash
|
||||
.venv/bin/python -m pytest \
|
||||
fastvideo/tests/attention/test_lingbotworld_fast_backend.py \
|
||||
fastvideo/tests/api/test_presets.py::TestLingBotWorldFastPresets \
|
||||
-v
|
||||
```
|
||||
|
||||
## SSIM
|
||||
|
||||
The package test is:
|
||||
|
||||
```bash
|
||||
pytest fastvideo/tests/ssim/test_lingbot_fast_similarity.py -v -s
|
||||
```
|
||||
|
||||
It requires two GPUs and device-specific references. New L40S references must
|
||||
be generated with the repository's `seed-ssim-references` workflow, reviewed
|
||||
visually, and uploaded to `FastVideo/ssim-reference-videos`. A GB10 output is
|
||||
not a valid replacement for the L40S CI reference.
|
||||
|
||||
See `PORT_STATUS.md` for the latest measured results and remaining blockers.
|
||||
@@ -0,0 +1,232 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Official-vs-FastVideo full-DiT parity for LingBot-World-Fast.
|
||||
|
||||
Coverage scope: both. The official ``WanModelFast`` implementation and the
|
||||
FastVideo production ``TransformerLoader`` load the same released transformer
|
||||
weights, then run one real cached causal forward with the released global
|
||||
attention and three-frame chunk settings.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import gc
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch.testing import assert_close
|
||||
|
||||
os.environ.setdefault("FASTVIDEO_ATTENTION_BACKEND", "TORCH_SDPA")
|
||||
os.environ.setdefault("MASTER_ADDR", "localhost")
|
||||
os.environ.setdefault("MASTER_PORT", "29531")
|
||||
|
||||
DEFAULT_MODEL_DIR = Path("official_weights/FastVideo/LingBot-World-Fast-Diffusers")
|
||||
DEFAULT_REFERENCE_DIR = Path("official_repos/lingbot-world")
|
||||
|
||||
|
||||
def _model_dir() -> Path:
|
||||
return Path(os.getenv("LINGBOTWORLD_FAST_MODEL_DIR", str(DEFAULT_MODEL_DIR))).expanduser().resolve()
|
||||
|
||||
|
||||
def _reference_dir() -> Path:
|
||||
return Path(os.getenv("LINGBOTWORLD_FAST_REFERENCE_DIR", str(DEFAULT_REFERENCE_DIR))).expanduser().resolve()
|
||||
|
||||
|
||||
def _transformer_dir() -> Path:
|
||||
return _model_dir() / "transformer"
|
||||
|
||||
|
||||
def _has_real_weights() -> bool:
|
||||
transformer_dir = _transformer_dir()
|
||||
return transformer_dir.exists() and any(transformer_dir.glob("*.safetensors"))
|
||||
|
||||
|
||||
def _official_constructor_kwargs() -> dict:
|
||||
return {
|
||||
"model_type": "i2v",
|
||||
"control_type": "cam",
|
||||
"patch_size": (1, 2, 2),
|
||||
"text_len": 512,
|
||||
"in_dim": 36,
|
||||
"dim": 5120,
|
||||
"ffn_dim": 13824,
|
||||
"freq_dim": 256,
|
||||
"text_dim": 4096,
|
||||
"out_dim": 16,
|
||||
"num_heads": 40,
|
||||
"num_layers": 40,
|
||||
"local_attn_size": -1,
|
||||
"sink_size": 9,
|
||||
"qk_norm": True,
|
||||
"cross_attn_norm": True,
|
||||
"eps": 1e-6,
|
||||
}
|
||||
|
||||
|
||||
def _deterministic_inputs(dtype: torch.dtype) -> dict:
|
||||
generator = torch.Generator(device="cpu").manual_seed(20260407)
|
||||
chunk_size = 3
|
||||
latent_height = 4
|
||||
latent_width = 4
|
||||
frame_seqlen = latent_height * latent_width // 4
|
||||
token_count = chunk_size * frame_seqlen
|
||||
|
||||
def randn(*shape: int) -> torch.Tensor:
|
||||
return torch.randn(shape, dtype=dtype, generator=generator)
|
||||
|
||||
return {
|
||||
"x": [randn(16, chunk_size, latent_height, latent_width)],
|
||||
"t": torch.tensor([679.0], dtype=dtype),
|
||||
"context": [randn(8, 4096)],
|
||||
"seq_len": token_count,
|
||||
"y": [randn(20, chunk_size, latent_height, latent_width)],
|
||||
# The released pipeline packs each six-channel Plucker ray over the
|
||||
# VAE's 8x8 spatial stride, yielding 6 * 8 * 8 = 384 channels.
|
||||
"dit_cond_dict": {
|
||||
"c2ws_plucker_emb": [randn(1, 384, chunk_size, latent_height, latent_width)],
|
||||
},
|
||||
"current_start": 0,
|
||||
"max_attention_size": token_count,
|
||||
"frame_seqlen": frame_seqlen,
|
||||
"cross_attn_first_call": True,
|
||||
}
|
||||
|
||||
|
||||
def _to_device(value, device: torch.device):
|
||||
if torch.is_tensor(value):
|
||||
return value.to(device)
|
||||
if isinstance(value, list):
|
||||
return [_to_device(item, device) for item in value]
|
||||
if isinstance(value, dict):
|
||||
return {key: _to_device(item, device) for key, item in value.items()}
|
||||
return value
|
||||
|
||||
|
||||
def _new_caches(device: torch.device, dtype: torch.dtype, token_count: int) -> tuple[list[dict], list[dict]]:
|
||||
self_cache = [{
|
||||
"k": torch.zeros((1, token_count, 40, 128), device=device, dtype=dtype),
|
||||
"v": torch.zeros((1, token_count, 40, 128), device=device, dtype=dtype),
|
||||
"global_end_index": torch.tensor([0], device=device, dtype=torch.long),
|
||||
"local_end_index": torch.tensor([0], device=device, dtype=torch.long),
|
||||
} for _ in range(40)]
|
||||
cross_cache = [{
|
||||
"k": torch.zeros((1, 512, 40, 128), device=device, dtype=dtype),
|
||||
"v": torch.zeros((1, 512, 40, 128), device=device, dtype=dtype),
|
||||
"is_init": torch.tensor(0, device=device, dtype=torch.int32),
|
||||
} for _ in range(40)]
|
||||
return self_cache, cross_cache
|
||||
|
||||
|
||||
def _run_official(inputs: dict, dtype: torch.dtype) -> torch.Tensor:
|
||||
reference_dir = _reference_dir()
|
||||
sys.path.insert(0, str(reference_dir))
|
||||
try:
|
||||
model_module = importlib.import_module("wan.modules.model_fast")
|
||||
# The official cross-attention calls its FlashAttention-only helper
|
||||
# directly. Route it through the official module's own SDPA-capable
|
||||
# dispatcher so both implementations execute the same public fallback.
|
||||
model_module.flash_attention = model_module.attention
|
||||
model = model_module.WanModelFast.from_pretrained(
|
||||
_transformer_dir(),
|
||||
torch_dtype=dtype,
|
||||
low_cpu_mem_usage=True,
|
||||
device_map={"": "cuda"},
|
||||
**_official_constructor_kwargs(),
|
||||
).eval()
|
||||
device_inputs = _to_device(inputs, torch.device("cuda"))
|
||||
self_cache, cross_cache = _new_caches(torch.device("cuda"), dtype, inputs["seq_len"])
|
||||
with (
|
||||
torch.inference_mode(),
|
||||
torch.amp.autocast("cuda", dtype=dtype),
|
||||
torch.nn.attention.sdpa_kernel(torch.nn.attention.SDPBackend.MATH),
|
||||
):
|
||||
output = model(**device_inputs, kv_cache=self_cache, crossattn_cache=cross_cache)[0]
|
||||
return output.detach().float().cpu()
|
||||
finally:
|
||||
sys.path.remove(str(reference_dir))
|
||||
if "model" in locals():
|
||||
del model
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
|
||||
def _run_fastvideo(inputs: dict, dtype: torch.dtype) -> torch.Tensor:
|
||||
from fastvideo.configs.models.dits.lingbotworld_fast import LingBotWorldFastVideoConfig
|
||||
from fastvideo.configs.pipelines.base import PipelineConfig
|
||||
from fastvideo.distributed import cleanup_dist_env_and_memory, maybe_init_distributed_environment_and_model_parallel
|
||||
from fastvideo.fastvideo_args import FastVideoArgs
|
||||
from fastvideo.models.dits.lingbotworld2.causal_fast import LingBotWorld2CausalFastTransformer3DModel
|
||||
from fastvideo.models.loader.component_loader import TransformerLoader
|
||||
|
||||
precision = "bf16" if dtype == torch.bfloat16 else "fp16"
|
||||
maybe_init_distributed_environment_and_model_parallel(1, 1)
|
||||
args = FastVideoArgs(
|
||||
model_path=str(_model_dir()),
|
||||
dit_cpu_offload=False,
|
||||
dit_layerwise_offload=False,
|
||||
use_fsdp_inference=False,
|
||||
pipeline_config=PipelineConfig(
|
||||
dit_config=LingBotWorldFastVideoConfig(),
|
||||
dit_precision=precision,
|
||||
),
|
||||
)
|
||||
args.device = torch.device("cuda")
|
||||
model = TransformerLoader().load(str(_transformer_dir()), args).eval()
|
||||
assert isinstance(model, LingBotWorld2CausalFastTransformer3DModel)
|
||||
device_inputs = _to_device(inputs, torch.device("cuda"))
|
||||
self_cache, cross_cache = _new_caches(torch.device("cuda"), dtype, inputs["seq_len"])
|
||||
try:
|
||||
with (
|
||||
torch.inference_mode(),
|
||||
torch.amp.autocast("cuda", dtype=dtype),
|
||||
torch.nn.attention.sdpa_kernel(torch.nn.attention.SDPBackend.MATH),
|
||||
):
|
||||
output = model(**device_inputs, kv_cache=self_cache, crossattn_cache=cross_cache)[0]
|
||||
return output.detach().float().cpu()
|
||||
finally:
|
||||
del model
|
||||
gc.collect()
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
def test_lingbotworld_fast_checkpoint_config_matches_released_variant() -> None:
|
||||
config_path = _transformer_dir() / "config.json"
|
||||
if not config_path.exists():
|
||||
pytest.skip(f"LingBot-World-Fast transformer config not found at {config_path}")
|
||||
config = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
assert config["_class_name"] == "CausalLingBotWorldTransformer3DModel"
|
||||
assert config["local_attn_size"] == -1
|
||||
assert config["num_frames_per_block"] == 3
|
||||
assert config["num_layers"] == 40
|
||||
assert config["num_attention_heads"] == 40
|
||||
assert config["attention_head_dim"] == 128
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Full LingBot-World-Fast DiT parity requires CUDA")
|
||||
def test_lingbotworld_fast_full_transformer_matches_official() -> None:
|
||||
if not _reference_dir().exists():
|
||||
pytest.skip(
|
||||
"Official LingBot source is absent; set LINGBOTWORLD_FAST_REFERENCE_DIR "
|
||||
"to a Robbyant/lingbot-world checkout.")
|
||||
if not _has_real_weights():
|
||||
pytest.skip(
|
||||
"Released transformer weights are absent; set LINGBOTWORLD_FAST_MODEL_DIR "
|
||||
"to FastVideo/LingBot-World-Fast-Diffusers.")
|
||||
|
||||
dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16
|
||||
inputs = _deterministic_inputs(dtype)
|
||||
official = _run_official(inputs, dtype)
|
||||
fastvideo = _run_fastvideo(inputs, dtype)
|
||||
|
||||
difference = (fastvideo - official).abs()
|
||||
relative_mean_drift = difference.mean() / official.abs().mean().clamp_min(1e-6)
|
||||
print(
|
||||
"LingBot-World-Fast full-DiT parity: "
|
||||
f"max={difference.max():.6f}, mean={difference.mean():.6f}, "
|
||||
f"relative_mean={relative_mean_drift:.6f}")
|
||||
assert relative_mean_drift < 0.05
|
||||
assert_close(fastvideo, official, atol=0.1, rtol=0.1)
|
||||
Reference in New Issue
Block a user