diff --git a/fastvideo/configs/models/dits/__init__.py b/fastvideo/configs/models/dits/__init__.py index 64a5c118d..88f7c79c4 100644 --- a/fastvideo/configs/models/dits/__init__.py +++ b/fastvideo/configs/models/dits/__init__.py @@ -14,10 +14,11 @@ from fastvideo.configs.models.dits.stable_audio import StableAudioConfig from fastvideo.configs.models.dits.wanvideo import WanVideoConfig from fastvideo.configs.models.dits.hyworld import HYWorldConfig from fastvideo.configs.models.dits.kandinsky5 import Kandinsky5VideoConfig +from fastvideo.configs.models.dits.lingbotworld2 import LingBotWorld2CausalFastVideoConfig __all__ = [ "HunyuanVideoConfig", "HunyuanVideo15Config", "HunyuanGameCraftConfig", "WanVideoConfig", "DreamXWorldConfig", "DreamXWorldARConfig", "CosmosVideoConfig", "Cosmos25VideoConfig", "FluxDiTConfig", "Flux2Config", "LongCatVideoConfig", "LTX2VideoConfig", "HYWorldConfig", "Kandinsky5VideoConfig", "MagiHumanVideoConfig", - "StableAudioConfig", "GlmImageDiTConfig" + "StableAudioConfig", "GlmImageDiTConfig", "LingBotWorld2CausalFastVideoConfig" ] diff --git a/fastvideo/configs/models/encoders/__init__.py b/fastvideo/configs/models/encoders/__init__.py index 7636fce23..644726e96 100644 --- a/fastvideo/configs/models/encoders/__init__.py +++ b/fastvideo/configs/models/encoders/__init__.py @@ -2,6 +2,7 @@ from fastvideo.configs.models.encoders.base import (BaseEncoderOutput, EncoderCo TextEncoderConfig) from fastvideo.configs.models.encoders.clip import (CLIPTextConfig, CLIPVisionConfig, WAN2_1ControlCLIPVisionConfig) from fastvideo.configs.models.encoders.llama import LlamaConfig +from fastvideo.configs.models.encoders.lingbotworld2_t5 import LingBotWorld2UMT5ArchConfig, LingBotWorld2UMT5Config from fastvideo.configs.models.encoders.t5 import T5Config, T5LargeConfig from fastvideo.configs.models.encoders.qwen2_5 import Qwen2_5_VLConfig from fastvideo.configs.models.encoders.siglip import SiglipVisionConfig @@ -17,5 +18,6 @@ __all__ = [ "EncoderConfig", "TextEncoderConfig", "ImageEncoderConfig", "BaseEncoderOutput", "CLIPTextConfig", "CLIPVisionConfig", "WAN2_1ControlCLIPVisionConfig", "LlamaConfig", "T5Config", "T5LargeConfig", "Qwen2_5_VLConfig", "Reason1ArchConfig", "Reason1Config", "LTX2GemmaConfig", "SiglipVisionConfig", "StableAudioConditionerArchConfig", - "StableAudioConditionerConfig", "T5GemmaEncoderConfig", "Qwen3TextConfig", "Mistral3TextConfig" + "StableAudioConditionerConfig", "T5GemmaEncoderConfig", "Qwen3TextConfig", "Mistral3TextConfig", + "LingBotWorld2UMT5ArchConfig", "LingBotWorld2UMT5Config" ] diff --git a/fastvideo/configs/pipelines/__init__.py b/fastvideo/configs/pipelines/__init__.py index 3983479de..d05fb5541 100644 --- a/fastvideo/configs/pipelines/__init__.py +++ b/fastvideo/configs/pipelines/__init__.py @@ -7,6 +7,7 @@ from fastvideo.configs.pipelines.hunyuan15 import Hunyuan15T2V480PConfig, Hunyua from fastvideo.configs.pipelines.hunyuangamecraft import HunyuanGameCraftPipelineConfig from fastvideo.configs.pipelines.hyworld import HYWorldConfig from fastvideo.configs.pipelines.kandinsky5 import Kandinsky5I2VConfig, Kandinsky5T2VConfig +from fastvideo.configs.pipelines.lingbotworld2 import LingBotWorld2CausalFastI2V480PConfig from fastvideo.configs.pipelines.matrixgame2 import MatrixGame2I2V480PConfig from fastvideo.configs.pipelines.matrixgame3 import MatrixGame3I2V720PConfig from fastvideo.pipelines.basic.ltx2.pipeline_configs import LTX2T2VConfig @@ -19,5 +20,6 @@ __all__ = [ "Hunyuan15T2V720PConfig", "WanT2V480PConfig", "WanI2V480PConfig", "WanT2V720PConfig", "WanI2V720PConfig", "SelfForcingWanT2V480PConfig", "LucyEditDevConfig", "CosmosConfig", "Cosmos25Config", "LTX2T2VConfig", "DreamXWorld5BCamPipelineConfig", "DreamXWorld5BARPipelineConfig", "HYWorldConfig", "Kandinsky5T2VConfig", - "Kandinsky5I2VConfig", "MatrixGame2I2V480PConfig", "MatrixGame3I2V720PConfig", "get_pipeline_config_cls_from_name" + "Kandinsky5I2VConfig", "LingBotWorld2CausalFastI2V480PConfig", "MatrixGame2I2V480PConfig", + "MatrixGame3I2V720PConfig", "get_pipeline_config_cls_from_name" ] diff --git a/fastvideo/models/loader/component_loader.py b/fastvideo/models/loader/component_loader.py index 6b097c317..dbdd2b764 100644 --- a/fastvideo/models/loader/component_loader.py +++ b/fastvideo/models/loader/component_loader.py @@ -820,6 +820,24 @@ class VAELoader(ComponentLoader): vae.load_state_dict(sd, strict=False) return vae.eval() + if class_name == "LingBotWorld2WanVAE": + dtype = PRECISION_TO_TYPE[fastvideo_args.pipeline_config.vae_precision] + config.pop("_class_name", None) + vae_config = fastvideo_args.pipeline_config.vae_config + vae_config.update_model_arch(config) + vae_cls, _ = ModelRegistry.resolve_model_cls(class_name) + weight_path = os.path.join(model_path, "Wan2.1_VAE.pth") + if not os.path.exists(weight_path): + raise FileNotFoundError( + f"Missing LingBot World 2 VAE weights: {weight_path}" + ) + vae = vae_cls( + vae_config, + checkpoint_path=weight_path, + dtype=dtype, + ).to(target_device) + return vae.eval() + # LTX-2 uses CausalVideoAutoencoder with nested "vae" config if class_name == "CausalVideoAutoencoder" and "vae" in config: vae_cls, _ = ModelRegistry.resolve_model_cls(class_name) diff --git a/fastvideo/models/registry.py b/fastvideo/models/registry.py index fdbc4fc2e..592d2e136 100644 --- a/fastvideo/models/registry.py +++ b/fastvideo/models/registry.py @@ -42,6 +42,11 @@ _TEXT_TO_VIDEO_DIT_MODELS = { "LTX2Transformer3DModel": ("dits", "ltx2", "LTX2Transformer3DModel"), "SD3Transformer2DModel": ("dits", "sd3", "SD3Transformer2DModel"), "LingBotWorldTransformer3DModel": ("dits", "lingbotworld", "LingBotWorldTransformer3DModel"), + "LingBotWorld2CausalFastTransformer3DModel": ( + "dits", + "lingbotworld2", + "LingBotWorld2CausalFastTransformer3DModel", + ), "Gen3CTransformer3DModel": ("dits", "gen3c", "Gen3CTransformer3DModel"), "Kandinsky5Transformer3DModel": ("dits", "kandinsky5", "Kandinsky5Transformer3DModel"), "Flux2Transformer2DModel": ("dits", "flux_2", "Flux2Transformer2DModel"), @@ -53,6 +58,11 @@ _IMAGE_TO_VIDEO_DIT_MODELS = { "DreamXWorldTransformer3DModel": ("dits", "dreamx_world", "DreamXWorldTransformer3DModel"), "DreamXWorldARTransformer3DModel": ("dits", "dreamx_world_ar", "DreamXWorldARTransformer3DModel"), "CausalWanTransformer3DModel": ("dits", "causal_wanvideo", "CausalWanTransformer3DModel"), + "LingBotWorld2CausalFastTransformer3DModel": ( + "dits", + "lingbotworld2", + "LingBotWorld2CausalFastTransformer3DModel", + ), "MatrixGame2WanModel": ("dits", "matrixgame2", "MatrixGame2WanModel"), "CausalMatrixGame2WanModel": ("dits", "matrixgame2", "CausalMatrixGame2WanModel"), # Legacy aliases for older HF model_index.json files @@ -72,6 +82,7 @@ _TEXT_ENCODER_MODELS = { ("encoders", "clip", "CLIPTextModelWithProjection"), "LlamaModel": ("encoders", "llama", "LlamaModel"), "UMT5EncoderModel": ("encoders", "t5", "UMT5EncoderModel"), + "LingBotWorld2T5EncoderModel": ("encoders", "lingbotworld2_t5", "LingBotWorld2T5EncoderModel"), "T5EncoderModel": ("encoders", "t5_hf", "T5EncoderModel"), "BertModel": ("encoders", "clip", "CLIPTextModel"), "Qwen2_5_VLTextModel": ("encoders", "qwen2_5", "Qwen2_5_VLTextModel"), @@ -98,6 +109,7 @@ _VAE_MODELS = { "AutoencoderKLHYWorld": ("vaes", "hyworldvae", "AutoencoderKLHYWorld"), "AutoencoderKLHunyuanVideo15": ("vaes", "hunyuan15vae", "AutoencoderKLHunyuanVideo15"), "AutoencoderKLWan": ("vaes", "wanvae", "AutoencoderKLWan"), + "LingBotWorld2WanVAE": ("vaes", "lingbotworld2_wanvae", "LingBotWorld2WanVAE"), "AutoencoderKL": ("vaes", "autoencoder_kl", "AutoencoderKL"), "AutoencoderKLGen3CTokenizer": ("vaes", "gen3c_tokenizer_vae", "AutoencoderKLGen3CTokenizer"), diff --git a/fastvideo/pipelines/pipeline_batch_info.py b/fastvideo/pipelines/pipeline_batch_info.py index 243bd2ae0..9c7dd3528 100644 --- a/fastvideo/pipelines/pipeline_batch_info.py +++ b/fastvideo/pipelines/pipeline_batch_info.py @@ -147,8 +147,9 @@ class ForwardBatch: camera_trajectory: str | None = None # Camera trajectory file/identifier action_list: list[str] | None = None # List of actions (e.g., ['forward', 'left']) action_speed_list: list[float] | None = None # Speed for each action - # Camera control inputs (LingBotWorld) + # Camera control inputs (LingBotWorld and LingBotWorld2) c2ws_plucker_emb: torch.Tensor | None = None # Plucker embedding: [B, C, F_lat, H_lat, W_lat] + action_path: str | None = None # Directory containing poses.npy and intrinsics.npy # Camera control inputs (GEN3C) trajectory_type: str | None = None diff --git a/fastvideo/registry.py b/fastvideo/registry.py index 30d885937..2df63cfb0 100644 --- a/fastvideo/registry.py +++ b/fastvideo/registry.py @@ -30,6 +30,7 @@ from fastvideo.configs.pipelines.hunyuan15 import (Hunyuan15T2V480PConfig, Hunyu from fastvideo.configs.pipelines.hyworld import HYWorldConfig from fastvideo.configs.pipelines.kandinsky5 import Kandinsky5I2VConfig, Kandinsky5T2VConfig from fastvideo.configs.pipelines.lingbotworld import LingBotWorldI2V480PConfig +from fastvideo.configs.pipelines.lingbotworld2 import LingBotWorld2CausalFastI2V480PConfig from fastvideo.configs.pipelines.longcat import LongCatT2V480PConfig from fastvideo.pipelines.basic.ltx2.pipeline_configs import LTX2T2VConfig from fastvideo.configs.pipelines.flux_2 import ( @@ -488,6 +489,20 @@ def _register_configs() -> None: model_family="gamecraft", default_preset="gamecraft_i2v", ) + # LingBotWorld2 causal-fast + register_configs( + sampling_param_cls=None, + pipeline_config_cls=LingBotWorld2CausalFastI2V480PConfig, + workload_types=(WorkloadType.I2V, ), + hf_model_paths=[ + "robbyant/lingbot-world-v2-14b-causal-fast", + ], + model_detectors=[lambda path: ("lingbot-world-v2-14b-causal-fast" in path.lower() + or "lingbotworld2causalfastpipeline" in path.lower())], + model_family="lingbotworld2", + default_preset="lingbotworld2_causal_fast_i2v", + ) + # LingBotWorld register_configs( sampling_param_cls=None, @@ -496,7 +511,9 @@ def _register_configs() -> None: hf_model_paths=[ "FastVideo/LingBot-World-Base-Cam-Diffusers", ], - model_detectors=[lambda path: ("lingbotworld" in path.lower() or "lingbot-world" in path.lower())], + model_detectors=[lambda path: (("lingbotworld" in path.lower() or "lingbot-world" in path.lower()) + and "causal-fast" not in path.lower() + and "causalfast" not in path.lower())], model_family="lingbotworld", default_preset="lingbotworld_i2v", ) @@ -1210,6 +1227,8 @@ def _register_presets() -> None: ALL_PRESETS as KANDINSKY5_PRESETS, ) from fastvideo.pipelines.basic.lingbotworld.presets import ( ALL_PRESETS as LINGBOTWORLD_PRESETS, ) + from fastvideo.pipelines.basic.lingbotworld2.presets import ( + ALL_PRESETS as LINGBOTWORLD2_PRESETS, ) from fastvideo.pipelines.basic.longcat.presets import ( ALL_PRESETS as LONGCAT_PRESETS, ) from fastvideo.pipelines.basic.ltx2.presets import ( @@ -1240,6 +1259,7 @@ def _register_presets() -> None: HYWORLD_PRESETS, KANDINSKY5_PRESETS, LINGBOTWORLD_PRESETS, + LINGBOTWORLD2_PRESETS, LONGCAT_PRESETS, LTX2_PRESETS, MATRIXGAME2_PRESETS,