diff --git a/docs/design/inference_schema_parity_inventory.yaml b/docs/design/inference_schema_parity_inventory.yaml index 606e68554..6fa90b7dc 100644 --- a/docs/design/inference_schema_parity_inventory.yaml +++ b/docs/design/inference_schema_parity_inventory.yaml @@ -71,33 +71,32 @@ surfaces: VSA_tile_size: generator.engine.attention.vsa_tile_size moba_config_path: generator.engine.attention.moba_config_path inference_torch_compile: generator.engine.compile.regional - preset_owned: - ltx2_vae_spatial_tile_size_in_pixels: generator.pipeline.preset_overrides.ltx2.vae.spatial_tile_size_in_pixels - ltx2_vae_spatial_tile_overlap_in_pixels: generator.pipeline.preset_overrides.ltx2.vae.spatial_tile_overlap_in_pixels - ltx2_vae_temporal_tile_size_in_frames: generator.pipeline.preset_overrides.ltx2.vae.temporal_tile_size_in_frames - ltx2_vae_temporal_tile_overlap_in_frames: generator.pipeline.preset_overrides.ltx2.vae.temporal_tile_overlap_in_frames - ltx2_initial_latent_path: request.extensions.ltx2.initial_latent_path - ltx2_audio_latent_path: request.extensions.ltx2.audio_latent_path + vae_parallel_decode: generator.pipeline.minimax_h3.vae_parallel_decode + vae_parallel_encode: generator.pipeline.minimax_h3.vae_parallel_encode + vae_parallel_decode_strategy: generator.pipeline.minimax_h3.vae_parallel_decode_strategy + h3_sequential_load: generator.pipeline.minimax_h3.sequential_load + video_decode_backend: generator.pipeline.minimax_h3.video_decode_backend + taeh3_checkpoint: generator.pipeline.minimax_h3.taeh3_checkpoint + taeh3_chunk_size: generator.pipeline.minimax_h3.taeh3_chunk_size + refine_transformer_path: generator.pipeline.ltx2.refine.transformer_path + refine_noise_path: generator.pipeline.ltx2.refine.noise_path + refine_audio_noise_path: generator.pipeline.ltx2.refine.audio_noise_path + ltx2_refine_transformer_path: generator.pipeline.ltx2.refine.transformer_path + ltx2_refine_noise_path: generator.pipeline.ltx2.refine.noise_path + ltx2_refine_audio_noise_path: generator.pipeline.ltx2.refine.audio_noise_path + ltx2_legacy_native_noise_order: generator.pipeline.ltx2.legacy_native_noise_order + ltx2_use_distilled_sigmas: generator.pipeline.ltx2.use_distilled_sigmas + ltx2_vae_spatial_tile_size_in_pixels: generator.pipeline.ltx2.vae_spatial_tile_size_in_pixels + ltx2_vae_spatial_tile_overlap_in_pixels: generator.pipeline.ltx2.vae_spatial_tile_overlap_in_pixels + ltx2_vae_temporal_tile_size_in_frames: generator.pipeline.ltx2.vae_temporal_tile_size_in_frames + ltx2_vae_temporal_tile_overlap_in_frames: generator.pipeline.ltx2.vae_temporal_tile_overlap_in_frames + ltx2_initial_latent_path: generator.pipeline.ltx2.initial_latent_path + ltx2_audio_latent_path: generator.pipeline.ltx2.audio_latent_path compatibility_only: mode: "Legacy multi-mode FastVideoArgs switch; typed inference config should not expose execution mode." inference_mode: "Legacy boolean mirror of mode; kept only through adapters while FastVideoArgs remains." output_type: "Legacy output formatting surface pending GenerationResult cleanup." - vae_parallel_decode: "MiniMax-H3 sequence-parallel VAE decode opt-in; model-specific optimization not yet represented in the typed public schema." - h3_sequential_load: "MiniMax-H3 sequential text-encoder then DiT/VAE load; model-specific optimization not yet represented in the typed public schema." - video_decode_backend: "MiniMax-H3 video decoder selection (full VAE vs TAEH3 preview); model-specific optimization not yet represented in the typed public schema." - taeh3_checkpoint: "Optional local TAEH3 safetensors path; model-specific optimization not yet represented in the typed public schema." - taeh3_chunk_size: "TAEH3 temporal chunk length; model-specific optimization not yet represented in the typed public schema." - vae_parallel_encode: "MiniMax-H3 sequence-parallel reference VAE encode opt-in; model-specific optimization not yet represented in the typed public schema." - vae_parallel_decode_strategy: "Chunk-transport collective for vae_parallel_decode; model-specific optimization not yet represented in the typed public schema." master_port: "Executor/bootstrap compatibility field; not part of the canonical inference schema." - refine_transformer_path: "Generic stage-2 refine transformer override; no typed equivalent yet." - refine_noise_path: "Generic stage-2 refine noise override; no typed equivalent yet." - refine_audio_noise_path: "Generic stage-2 refine audio noise override; no typed equivalent yet." - ltx2_refine_transformer_path: "LTX-2 refine transformer carrier; no typed equivalent yet." - ltx2_refine_noise_path: "LTX-2 refine noise carrier; no typed equivalent yet." - ltx2_refine_audio_noise_path: "LTX-2 refine audio noise carrier; no typed equivalent yet." - ltx2_legacy_native_noise_order: "LTX-2 SSIM compatibility knob preserving legacy native latent noise ordering." - ltx2_use_distilled_sigmas: "LTX-2 compatibility knob gating use of distilled sigma schedule." private_only: ray_placement_group: "Ray deployment-only field." ray_runtime_env: "Ray deployment-only field." @@ -326,16 +325,21 @@ surfaces: - fastvideo.configs.pipelines.wan.WanT2V720PConfig bsa_cdf_threshold: sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig] + target: generator.pipeline.longcat.bsa_cdf_threshold bsa_chunk_k: sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig] + target: generator.pipeline.longcat.bsa_chunk_k bsa_chunk_q: sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig] + target: generator.pipeline.longcat.bsa_chunk_q bsa_params: sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig] bsa_sparsity: sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig] + target: generator.pipeline.longcat.bsa_sparsity enable_bsa: sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig] + target: generator.pipeline.longcat.enable_bsa enable_kv_cache: sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig] enhance_hf: diff --git a/fastvideo/api/__init__.py b/fastvideo/api/__init__.py index 2fbc0b555..c2de8bcc8 100644 --- a/fastvideo/api/__init__.py +++ b/fastvideo/api/__init__.py @@ -10,6 +10,10 @@ from fastvideo.api.schema import ( GeneratorConfig, GpuPoolConfig, InputConfig, + LTX2Config, + LTX2RefineConfig, + LongCatConfig, + MiniMaxH3Config, OffloadConfig, OutputConfig, ParallelismConfig, @@ -71,6 +75,10 @@ __all__ = [ "GeneratorConfig", "GpuPoolConfig", "InputConfig", + "LTX2Config", + "LTX2RefineConfig", + "LongCatConfig", + "MiniMaxH3Config", "OffloadConfig", "OutputConfig", "ParallelismConfig", diff --git a/fastvideo/api/compat.py b/fastvideo/api/compat.py index ef3f6e191..6a3494165 100644 --- a/fastvideo/api/compat.py +++ b/fastvideo/api/compat.py @@ -110,6 +110,9 @@ def legacy_from_pretrained_to_config( _set_dotted_path(raw, ["engine", "compile", "extras"], remaining) elif key == "pipeline_config" and not isinstance(value, str): experimental[key] = deepcopy(value) + elif key.startswith(tuple(_COMPONENT_OVERRIDE_PREFIXES)): + component, field_name = key.split(".", 1) + _set_dotted_path(raw, ["pipeline", _COMPONENT_OVERRIDE_PREFIXES[component + "."], field_name], value) elif key in _LTX2_REFINE_PRESET_KEYWORDS: _set_dotted_path(raw, ["pipeline", "preset_overrides", "refine", key[len("ltx2_refine_"):]], value) elif key in _EMPTY_MEANS_UNSET_KEYWORDS: @@ -166,6 +169,10 @@ def generator_config_to_fastvideo_args(config: GeneratorConfig | Mapping[str, An _resolved_quant_cls = get_quantization_config(quantization.transformer_quant) kwargs["transformer_quant"] = _resolved_quant_cls() + for prefix, section in _COMPONENT_OVERRIDE_PREFIXES.items(): + for field_name, value in getattr(normalized.pipeline, section).items(): + kwargs[f"{prefix}{field_name}"] = deepcopy(value) + preset_overrides = deepcopy(normalized.pipeline.preset_overrides) refine = preset_overrides.pop("refine", None) if isinstance(refine, Mapping): @@ -303,9 +310,16 @@ _SPECIALLY_MAPPED_PATHS: dict[str, str] = { "pipeline.preset": "not supported", "pipeline.preset_version": "not supported", "pipeline.components.vae_weights": "not supported", + "pipeline.dit": "keys passed as dit_config.", + "pipeline.vae": "keys passed as vae_config.", "pipeline.preset_overrides": "keys passed as flat keywords; refine keys renamed to ltx2_refine_*", "pipeline.experimental": "keys passed as flat keywords", } +# Flat keyword prefix for a component config override -> the PipelineSelection dict that holds the overrides. +_COMPONENT_OVERRIDE_PREFIXES = { + "dit_config.": "dit", + "vae_config.": "vae", +} # from_pretrained keywords whose empty-string value means "unset", and the field each one sets. # ltx2_refine_lora_path sets the main LoRA path. _EMPTY_MEANS_UNSET_KEYWORDS = { diff --git a/fastvideo/api/schema.py b/fastvideo/api/schema.py index e93338d7a..164204b51 100644 --- a/fastvideo/api/schema.py +++ b/fastvideo/api/schema.py @@ -153,6 +153,72 @@ class ComponentConfig: override_transformer_cls_name: str | None = flat_field("override_transformer_cls_name", None) +@dataclass +class LTX2RefineConfig: + """Stage-2 refine assets that ``preset_overrides.refine`` and ``components`` do not cover.""" + + transformer_path: str | None = flat_field("ltx2_refine_transformer_path", None) + noise_path: str | None = flat_field("ltx2_refine_noise_path", None) + audio_noise_path: str | None = flat_field("ltx2_refine_audio_noise_path", None) + + +@dataclass +class LTX2Config: + """LTX-2 settings. ``None`` keeps the model's default.""" + + vae_spatial_tile_size_in_pixels: int | None = flat_field("ltx2_vae_spatial_tile_size_in_pixels", None) + vae_spatial_tile_overlap_in_pixels: int | None = flat_field("ltx2_vae_spatial_tile_overlap_in_pixels", None) + vae_temporal_tile_size_in_frames: int | None = flat_field("ltx2_vae_temporal_tile_size_in_frames", None) + vae_temporal_tile_overlap_in_frames: int | None = flat_field("ltx2_vae_temporal_tile_overlap_in_frames", None) + initial_latent_path: str | None = flat_field("ltx2_initial_latent_path", None) + """Path to load or save a precomputed initial video latent.""" + audio_latent_path: str | None = flat_field("ltx2_audio_latent_path", None) + """Path to load or save a precomputed initial audio latent.""" + legacy_native_noise_order: bool | None = flat_field("ltx2_legacy_native_noise_order", None) + """Draw latent noise in the legacy native order, which earlier SSIM references use.""" + use_distilled_sigmas: bool | None = flat_field("ltx2_use_distilled_sigmas", None) + """Use the distilled sigma schedule when the checkpoint provides one.""" + refine: LTX2RefineConfig = field(default_factory=LTX2RefineConfig) + + +@dataclass +class MiniMaxH3Config: + """MiniMax-H3 settings. ``None`` keeps the model's default.""" + + sequential_load: bool | None = flat_field("h3_sequential_load", None) + """Encode with Qwen3-VL, release that encoder, then load the DiT and VAEs. ``None`` enables it on + unified-memory devices only.""" + video_decode_backend: Literal["h3-vae", "taeh3"] | None = flat_field("video_decode_backend", None) + """``h3-vae`` is the full VAE; ``taeh3`` is a fast approximate preview decoder.""" + taeh3_checkpoint: str | None = flat_field("taeh3_checkpoint", None) + """Local ``taeh3.safetensors`` path. ``None`` downloads the pinned upstream weights.""" + taeh3_chunk_size: int | None = flat_field("taeh3_chunk_size", None) + """TAEH3 latent frames per execution chunk.""" + vae_parallel_decode: bool | None = flat_field("vae_parallel_decode", None) + """Spread VAE decode chunks across the sequence-parallel ranks. ``None`` falls back to + ``FASTVIDEO_VAE_PARALLEL_DECODE``.""" + vae_parallel_encode: bool | None = flat_field("vae_parallel_encode", None) + """Spread reference-video VAE encode clips across the sequence-parallel ranks. ``None`` falls back to + ``FASTVIDEO_VAE_PARALLEL_ENCODE``.""" + vae_parallel_decode_strategy: Literal["gather", "all_gather"] | None = flat_field( + "vae_parallel_decode_strategy", None) + """Collective that moves decoded chunks. ``None`` falls back to ``FASTVIDEO_VAE_PARALLEL_DECODE_STRATEGY``, + then ``gather``.""" + + +@dataclass +class LongCatConfig: + """LongCat block sparse attention (BSA) settings. ``None`` keeps the model's default.""" + + enable_bsa: bool | None = flat_field("enable_bsa", None) + bsa_sparsity: float | None = flat_field("bsa_sparsity", None) + bsa_cdf_threshold: float | None = flat_field("bsa_cdf_threshold", None) + bsa_chunk_q: list[int] | None = flat_field("bsa_chunk_q", None) + """Query chunk shape as ``[T, H, W]``.""" + bsa_chunk_k: list[int] | None = flat_field("bsa_chunk_k", None) + """Key chunk shape as ``[T, H, W]``.""" + + @dataclass class PipelineSelection: workload_type: Literal["t2v", "i2v", "t2i", "i2i", "v2a", "t2a"] | None = flat_field("workload_type", None) @@ -169,6 +235,13 @@ class PipelineSelection: """Guidance scale that guidance-distilled models take as a DiT input. ``None`` keeps the model's default.""" dmd_denoising_steps: list[int] | None = flat_field("dmd_denoising_steps", None) """Timesteps of a few-step distilled (DMD) sampler. ``None`` keeps the model's default.""" + dit: dict[str, Any] = field(default_factory=dict) + """Overrides for fields of the model's DiT config, such as ``prefix``.""" + vae: dict[str, Any] = field(default_factory=dict) + """Overrides for fields of the model's VAE config, such as ``load_encoder`` or ``use_tiling``.""" + ltx2: LTX2Config = field(default_factory=LTX2Config) + minimax_h3: MiniMaxH3Config = field(default_factory=MiniMaxH3Config) + longcat: LongCatConfig = field(default_factory=LongCatConfig) preset_overrides: dict[str, Any] = field(default_factory=dict) experimental: dict[str, Any] = field(default_factory=dict) @@ -363,6 +436,10 @@ __all__ = [ "GeneratorConfig", "GpuPoolConfig", "InputConfig", + "LTX2Config", + "LTX2RefineConfig", + "LongCatConfig", + "MiniMaxH3Config", "OffloadConfig", "OutputConfig", "ParallelismConfig", diff --git a/fastvideo/tests/api/test_parser.py b/fastvideo/tests/api/test_parser.py index 6e0214579..ff93d9671 100644 --- a/fastvideo/tests/api/test_parser.py +++ b/fastvideo/tests/api/test_parser.py @@ -174,6 +174,39 @@ def test_load_run_config_supports_yaml_roundtrip(tmp_path) -> None: "flow_shift": None, "embedded_cfg_scale": None, "dmd_denoising_steps": None, + "dit": {}, + "vae": {}, + "ltx2": { + "vae_spatial_tile_size_in_pixels": None, + "vae_spatial_tile_overlap_in_pixels": None, + "vae_temporal_tile_size_in_frames": None, + "vae_temporal_tile_overlap_in_frames": None, + "initial_latent_path": None, + "audio_latent_path": None, + "legacy_native_noise_order": None, + "use_distilled_sigmas": None, + "refine": { + "transformer_path": None, + "noise_path": None, + "audio_noise_path": None, + }, + }, + "minimax_h3": { + "sequential_load": None, + "video_decode_backend": None, + "taeh3_checkpoint": None, + "taeh3_chunk_size": None, + "vae_parallel_decode": None, + "vae_parallel_encode": None, + "vae_parallel_decode_strategy": None, + }, + "longcat": { + "enable_bsa": None, + "bsa_sparsity": None, + "bsa_cdf_threshold": None, + "bsa_chunk_q": None, + "bsa_chunk_k": None, + }, "preset_overrides": {}, "experimental": {}, },