nodes/hao-ai-lab-FastVideo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Connect Lingbot world 2 registration and porting.

Browse Source
This commit is contained in:
Davids048
2026-07-13 17:10:07 -07:00
committed by SolitaryThinker
parent 6f089ba83a
commit 8031dcbdc5
7 changed files with 61 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fastvideo/configs/models/dits/__init__.py
+2 -1
View File
@@ -14,10 +14,11 @@ from fastvideo.configs.models.dits.stable_audio import StableAudioConfig
from fastvideo.configs.models.dits.wanvideo import WanVideoConfig
from fastvideo.configs.models.dits.hyworld import HYWorldConfig
from fastvideo.configs.models.dits.kandinsky5 import Kandinsky5VideoConfig
from fastvideo.configs.models.dits.lingbotworld2 import LingBotWorld2CausalFastVideoConfig
__all__ = [
"HunyuanVideoConfig", "HunyuanVideo15Config", "HunyuanGameCraftConfig", "WanVideoConfig", "DreamXWorldConfig",
"DreamXWorldARConfig", "CosmosVideoConfig", "Cosmos25VideoConfig", "FluxDiTConfig", "Flux2Config",
"LongCatVideoConfig", "LTX2VideoConfig", "HYWorldConfig", "Kandinsky5VideoConfig", "MagiHumanVideoConfig",
"StableAudioConfig", "GlmImageDiTConfig"
"StableAudioConfig", "GlmImageDiTConfig", "LingBotWorld2CausalFastVideoConfig"
]
fastvideo/configs/models/encoders/__init__.py
+3 -1
View File
@@ -2,6 +2,7 @@ from fastvideo.configs.models.encoders.base import (BaseEncoderOutput, EncoderCo
TextEncoderConfig)
from fastvideo.configs.models.encoders.clip import (CLIPTextConfig, CLIPVisionConfig, WAN2_1ControlCLIPVisionConfig)
from fastvideo.configs.models.encoders.llama import LlamaConfig
from fastvideo.configs.models.encoders.lingbotworld2_t5 import LingBotWorld2UMT5ArchConfig, LingBotWorld2UMT5Config
from fastvideo.configs.models.encoders.t5 import T5Config, T5LargeConfig
from fastvideo.configs.models.encoders.qwen2_5 import Qwen2_5_VLConfig
from fastvideo.configs.models.encoders.siglip import SiglipVisionConfig
@@ -17,5 +18,6 @@ __all__ = [
"EncoderConfig", "TextEncoderConfig", "ImageEncoderConfig", "BaseEncoderOutput", "CLIPTextConfig",
"CLIPVisionConfig", "WAN2_1ControlCLIPVisionConfig", "LlamaConfig", "T5Config", "T5LargeConfig", "Qwen2_5_VLConfig",
"Reason1ArchConfig", "Reason1Config", "LTX2GemmaConfig", "SiglipVisionConfig", "StableAudioConditionerArchConfig",
"StableAudioConditionerConfig", "T5GemmaEncoderConfig", "Qwen3TextConfig", "Mistral3TextConfig"
"StableAudioConditionerConfig", "T5GemmaEncoderConfig", "Qwen3TextConfig", "Mistral3TextConfig",
"LingBotWorld2UMT5ArchConfig", "LingBotWorld2UMT5Config"
]
fastvideo/configs/pipelines/__init__.py
+3 -1
View File
@@ -7,6 +7,7 @@ from fastvideo.configs.pipelines.hunyuan15 import Hunyuan15T2V480PConfig, Hunyua
from fastvideo.configs.pipelines.hunyuangamecraft import HunyuanGameCraftPipelineConfig
from fastvideo.configs.pipelines.hyworld import HYWorldConfig
from fastvideo.configs.pipelines.kandinsky5 import Kandinsky5I2VConfig, Kandinsky5T2VConfig
from fastvideo.configs.pipelines.lingbotworld2 import LingBotWorld2CausalFastI2V480PConfig
from fastvideo.configs.pipelines.matrixgame2 import MatrixGame2I2V480PConfig
from fastvideo.configs.pipelines.matrixgame3 import MatrixGame3I2V720PConfig
from fastvideo.pipelines.basic.ltx2.pipeline_configs import LTX2T2VConfig
@@ -19,5 +20,6 @@ __all__ = [
"Hunyuan15T2V720PConfig", "WanT2V480PConfig", "WanI2V480PConfig", "WanT2V720PConfig", "WanI2V720PConfig",
"SelfForcingWanT2V480PConfig", "LucyEditDevConfig", "CosmosConfig", "Cosmos25Config", "LTX2T2VConfig",
"DreamXWorld5BCamPipelineConfig", "DreamXWorld5BARPipelineConfig", "HYWorldConfig", "Kandinsky5T2VConfig",
"Kandinsky5I2VConfig", "MatrixGame2I2V480PConfig", "MatrixGame3I2V720PConfig", "get_pipeline_config_cls_from_name"
"Kandinsky5I2VConfig", "LingBotWorld2CausalFastI2V480PConfig", "MatrixGame2I2V480PConfig",
"MatrixGame3I2V720PConfig", "get_pipeline_config_cls_from_name"
]
fastvideo/models/loader/component_loader.py
+18
View File
@@ -820,6 +820,24 @@ class VAELoader(ComponentLoader):
vae.load_state_dict(sd, strict=False)
return vae.eval()
if class_name == "LingBotWorld2WanVAE":
dtype = PRECISION_TO_TYPE[fastvideo_args.pipeline_config.vae_precision]
config.pop("_class_name", None)
vae_config = fastvideo_args.pipeline_config.vae_config
vae_config.update_model_arch(config)
vae_cls, _ = ModelRegistry.resolve_model_cls(class_name)
weight_path = os.path.join(model_path, "Wan2.1_VAE.pth")
if not os.path.exists(weight_path):
raise FileNotFoundError(
f"Missing LingBot World 2 VAE weights: {weight_path}"
)
vae = vae_cls(
vae_config,
checkpoint_path=weight_path,
dtype=dtype,
).to(target_device)
return vae.eval()
# LTX-2 uses CausalVideoAutoencoder with nested "vae" config
if class_name == "CausalVideoAutoencoder" and "vae" in config:
vae_cls, _ = ModelRegistry.resolve_model_cls(class_name)
fastvideo/models/registry.py
+12
View File
@@ -42,6 +42,11 @@ _TEXT_TO_VIDEO_DIT_MODELS = {
"LTX2Transformer3DModel": ("dits", "ltx2", "LTX2Transformer3DModel"),
"SD3Transformer2DModel": ("dits", "sd3", "SD3Transformer2DModel"),
"LingBotWorldTransformer3DModel": ("dits", "lingbotworld", "LingBotWorldTransformer3DModel"),
"LingBotWorld2CausalFastTransformer3DModel": (
"dits",
"lingbotworld2",
"LingBotWorld2CausalFastTransformer3DModel",
),
"Gen3CTransformer3DModel": ("dits", "gen3c", "Gen3CTransformer3DModel"),
"Kandinsky5Transformer3DModel": ("dits", "kandinsky5", "Kandinsky5Transformer3DModel"),
"Flux2Transformer2DModel": ("dits", "flux_2", "Flux2Transformer2DModel"),
@@ -53,6 +58,11 @@ _IMAGE_TO_VIDEO_DIT_MODELS = {
"DreamXWorldTransformer3DModel": ("dits", "dreamx_world", "DreamXWorldTransformer3DModel"),
"DreamXWorldARTransformer3DModel": ("dits", "dreamx_world_ar", "DreamXWorldARTransformer3DModel"),
"CausalWanTransformer3DModel": ("dits", "causal_wanvideo", "CausalWanTransformer3DModel"),
"LingBotWorld2CausalFastTransformer3DModel": (
"dits",
"lingbotworld2",
"LingBotWorld2CausalFastTransformer3DModel",
),
"MatrixGame2WanModel": ("dits", "matrixgame2", "MatrixGame2WanModel"),
"CausalMatrixGame2WanModel": ("dits", "matrixgame2", "CausalMatrixGame2WanModel"),
# Legacy aliases for older HF model_index.json files
@@ -72,6 +82,7 @@ _TEXT_ENCODER_MODELS = {
("encoders", "clip", "CLIPTextModelWithProjection"),
"LlamaModel": ("encoders", "llama", "LlamaModel"),
"UMT5EncoderModel": ("encoders", "t5", "UMT5EncoderModel"),
"LingBotWorld2T5EncoderModel": ("encoders", "lingbotworld2_t5", "LingBotWorld2T5EncoderModel"),
"T5EncoderModel": ("encoders", "t5_hf", "T5EncoderModel"),
"BertModel": ("encoders", "clip", "CLIPTextModel"),
"Qwen2_5_VLTextModel": ("encoders", "qwen2_5", "Qwen2_5_VLTextModel"),
@@ -98,6 +109,7 @@ _VAE_MODELS = {
"AutoencoderKLHYWorld": ("vaes", "hyworldvae", "AutoencoderKLHYWorld"),
"AutoencoderKLHunyuanVideo15": ("vaes", "hunyuan15vae", "AutoencoderKLHunyuanVideo15"),
"AutoencoderKLWan": ("vaes", "wanvae", "AutoencoderKLWan"),
"LingBotWorld2WanVAE": ("vaes", "lingbotworld2_wanvae", "LingBotWorld2WanVAE"),
"AutoencoderKL": ("vaes", "autoencoder_kl", "AutoencoderKL"),
"AutoencoderKLGen3CTokenizer":
("vaes", "gen3c_tokenizer_vae", "AutoencoderKLGen3CTokenizer"),
fastvideo/pipelines/pipeline_batch_info.py
+2 -1
View File
@@ -147,8 +147,9 @@ class ForwardBatch:
camera_trajectory: str | None = None # Camera trajectory file/identifier
action_list: list[str] | None = None # List of actions (e.g., ['forward', 'left'])
action_speed_list: list[float] | None = None # Speed for each action
# Camera control inputs (LingBotWorld)
# Camera control inputs (LingBotWorld and LingBotWorld2)
c2ws_plucker_emb: torch.Tensor | None = None # Plucker embedding: [B, C, F_lat, H_lat, W_lat]
action_path: str | None = None # Directory containing poses.npy and intrinsics.npy
# Camera control inputs (GEN3C)
trajectory_type: str | None = None
fastvideo/registry.py
+21 -1
View File
@@ -30,6 +30,7 @@ from fastvideo.configs.pipelines.hunyuan15 import (Hunyuan15T2V480PConfig, Hunyu
from fastvideo.configs.pipelines.hyworld import HYWorldConfig
from fastvideo.configs.pipelines.kandinsky5 import Kandinsky5I2VConfig, Kandinsky5T2VConfig
from fastvideo.configs.pipelines.lingbotworld import LingBotWorldI2V480PConfig
from fastvideo.configs.pipelines.lingbotworld2 import LingBotWorld2CausalFastI2V480PConfig
from fastvideo.configs.pipelines.longcat import LongCatT2V480PConfig
from fastvideo.pipelines.basic.ltx2.pipeline_configs import LTX2T2VConfig
from fastvideo.configs.pipelines.flux_2 import (
@@ -488,6 +489,20 @@ def _register_configs() -> None:
model_family="gamecraft",
default_preset="gamecraft_i2v",
)
# LingBotWorld2 causal-fast
register_configs(
sampling_param_cls=None,
pipeline_config_cls=LingBotWorld2CausalFastI2V480PConfig,
workload_types=(WorkloadType.I2V, ),
hf_model_paths=[
"robbyant/lingbot-world-v2-14b-causal-fast",
],
model_detectors=[lambda path: ("lingbot-world-v2-14b-causal-fast" in path.lower()
or "lingbotworld2causalfastpipeline" in path.lower())],
model_family="lingbotworld2",
default_preset="lingbotworld2_causal_fast_i2v",
)
# LingBotWorld
register_configs(
sampling_param_cls=None,
@@ -496,7 +511,9 @@ def _register_configs() -> None:
hf_model_paths=[
"FastVideo/LingBot-World-Base-Cam-Diffusers",
],
model_detectors=[lambda path: ("lingbotworld" in path.lower() or "lingbot-world" in path.lower())],
model_detectors=[lambda path: (("lingbotworld" in path.lower() or "lingbot-world" in path.lower())
and "causal-fast" not in path.lower()
and "causalfast" not in path.lower())],
model_family="lingbotworld",
default_preset="lingbotworld_i2v",
)
@@ -1210,6 +1227,8 @@ def _register_presets() -> None:
ALL_PRESETS as KANDINSKY5_PRESETS, )
from fastvideo.pipelines.basic.lingbotworld.presets import (
ALL_PRESETS as LINGBOTWORLD_PRESETS, )
from fastvideo.pipelines.basic.lingbotworld2.presets import (
ALL_PRESETS as LINGBOTWORLD2_PRESETS, )
from fastvideo.pipelines.basic.longcat.presets import (
ALL_PRESETS as LONGCAT_PRESETS, )
from fastvideo.pipelines.basic.ltx2.presets import (
@@ -1240,6 +1259,7 @@ def _register_presets() -> None:
HYWORLD_PRESETS,
KANDINSKY5_PRESETS,
LINGBOTWORLD_PRESETS,
LINGBOTWORLD2_PRESETS,
LONGCAT_PRESETS,
LTX2_PRESETS,
MATRIXGAME2_PRESETS,