From b24396561acb2cef4e418df04fecd5334e40a60a Mon Sep 17 00:00:00 2001 From: Davids048 Date: Fri, 2 Oct 2026 17:28:11 +0000 Subject: [PATCH] [refactor]: add component overrides and model-family config partitions Parameters that belong to one model family, or to one component config, could be set only through argparse flags or pipeline.experimental. They gain typed GeneratorConfig fields, each with the flat name that FastVideoArgs or PipelineConfig uses: - pipeline.dit and pipeline.vae: dicts of DiT and VAE config field overrides, passed as dit_config. and vae_config. - pipeline.ltx2: VAE tile sizes, initial video and audio latent paths, noise order, distilled sigmas, and refine.{transformer,noise,audio_noise}_path - pipeline.minimax_h3: sequential load, video decode backend, TAEH3 checkpoint and chunk size, and the sequence-parallel VAE switches - pipeline.longcat: the block sparse attention (BSA) settings legacy_from_pretrained_to_config routes dit_config.* and vae_config.* keywords into the new dicts. Every new scalar field defaults to None and is not passed to FastVideoArgs while unset. Behavior is unchanged: the golden config snapshots match. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012M91wnVFmPEvJ9h39r5BH7 --- .../inference_schema_parity_inventory.yaml | 48 ++++++------ fastvideo/api/__init__.py | 8 ++ fastvideo/api/compat.py | 14 ++++ fastvideo/api/schema.py | 77 +++++++++++++++++++ fastvideo/tests/api/test_parser.py | 33 ++++++++ 5 files changed, 158 insertions(+), 22 deletions(-) diff --git a/docs/design/inference_schema_parity_inventory.yaml b/docs/design/inference_schema_parity_inventory.yaml index 606e68554..6fa90b7dc 100644 --- a/docs/design/inference_schema_parity_inventory.yaml +++ b/docs/design/inference_schema_parity_inventory.yaml @@ -71,33 +71,32 @@ surfaces: VSA_tile_size: generator.engine.attention.vsa_tile_size moba_config_path: generator.engine.attention.moba_config_path inference_torch_compile: generator.engine.compile.regional - preset_owned: - ltx2_vae_spatial_tile_size_in_pixels: generator.pipeline.preset_overrides.ltx2.vae.spatial_tile_size_in_pixels - ltx2_vae_spatial_tile_overlap_in_pixels: generator.pipeline.preset_overrides.ltx2.vae.spatial_tile_overlap_in_pixels - ltx2_vae_temporal_tile_size_in_frames: generator.pipeline.preset_overrides.ltx2.vae.temporal_tile_size_in_frames - ltx2_vae_temporal_tile_overlap_in_frames: generator.pipeline.preset_overrides.ltx2.vae.temporal_tile_overlap_in_frames - ltx2_initial_latent_path: request.extensions.ltx2.initial_latent_path - ltx2_audio_latent_path: request.extensions.ltx2.audio_latent_path + vae_parallel_decode: generator.pipeline.minimax_h3.vae_parallel_decode + vae_parallel_encode: generator.pipeline.minimax_h3.vae_parallel_encode + vae_parallel_decode_strategy: generator.pipeline.minimax_h3.vae_parallel_decode_strategy + h3_sequential_load: generator.pipeline.minimax_h3.sequential_load + video_decode_backend: generator.pipeline.minimax_h3.video_decode_backend + taeh3_checkpoint: generator.pipeline.minimax_h3.taeh3_checkpoint + taeh3_chunk_size: generator.pipeline.minimax_h3.taeh3_chunk_size + refine_transformer_path: generator.pipeline.ltx2.refine.transformer_path + refine_noise_path: generator.pipeline.ltx2.refine.noise_path + refine_audio_noise_path: generator.pipeline.ltx2.refine.audio_noise_path + ltx2_refine_transformer_path: generator.pipeline.ltx2.refine.transformer_path + ltx2_refine_noise_path: generator.pipeline.ltx2.refine.noise_path + ltx2_refine_audio_noise_path: generator.pipeline.ltx2.refine.audio_noise_path + ltx2_legacy_native_noise_order: generator.pipeline.ltx2.legacy_native_noise_order + ltx2_use_distilled_sigmas: generator.pipeline.ltx2.use_distilled_sigmas + ltx2_vae_spatial_tile_size_in_pixels: generator.pipeline.ltx2.vae_spatial_tile_size_in_pixels + ltx2_vae_spatial_tile_overlap_in_pixels: generator.pipeline.ltx2.vae_spatial_tile_overlap_in_pixels + ltx2_vae_temporal_tile_size_in_frames: generator.pipeline.ltx2.vae_temporal_tile_size_in_frames + ltx2_vae_temporal_tile_overlap_in_frames: generator.pipeline.ltx2.vae_temporal_tile_overlap_in_frames + ltx2_initial_latent_path: generator.pipeline.ltx2.initial_latent_path + ltx2_audio_latent_path: generator.pipeline.ltx2.audio_latent_path compatibility_only: mode: "Legacy multi-mode FastVideoArgs switch; typed inference config should not expose execution mode." inference_mode: "Legacy boolean mirror of mode; kept only through adapters while FastVideoArgs remains." output_type: "Legacy output formatting surface pending GenerationResult cleanup." - vae_parallel_decode: "MiniMax-H3 sequence-parallel VAE decode opt-in; model-specific optimization not yet represented in the typed public schema." - h3_sequential_load: "MiniMax-H3 sequential text-encoder then DiT/VAE load; model-specific optimization not yet represented in the typed public schema." - video_decode_backend: "MiniMax-H3 video decoder selection (full VAE vs TAEH3 preview); model-specific optimization not yet represented in the typed public schema." - taeh3_checkpoint: "Optional local TAEH3 safetensors path; model-specific optimization not yet represented in the typed public schema." - taeh3_chunk_size: "TAEH3 temporal chunk length; model-specific optimization not yet represented in the typed public schema." - vae_parallel_encode: "MiniMax-H3 sequence-parallel reference VAE encode opt-in; model-specific optimization not yet represented in the typed public schema." - vae_parallel_decode_strategy: "Chunk-transport collective for vae_parallel_decode; model-specific optimization not yet represented in the typed public schema." master_port: "Executor/bootstrap compatibility field; not part of the canonical inference schema." - refine_transformer_path: "Generic stage-2 refine transformer override; no typed equivalent yet." - refine_noise_path: "Generic stage-2 refine noise override; no typed equivalent yet." - refine_audio_noise_path: "Generic stage-2 refine audio noise override; no typed equivalent yet." - ltx2_refine_transformer_path: "LTX-2 refine transformer carrier; no typed equivalent yet." - ltx2_refine_noise_path: "LTX-2 refine noise carrier; no typed equivalent yet." - ltx2_refine_audio_noise_path: "LTX-2 refine audio noise carrier; no typed equivalent yet." - ltx2_legacy_native_noise_order: "LTX-2 SSIM compatibility knob preserving legacy native latent noise ordering." - ltx2_use_distilled_sigmas: "LTX-2 compatibility knob gating use of distilled sigma schedule." private_only: ray_placement_group: "Ray deployment-only field." ray_runtime_env: "Ray deployment-only field." @@ -326,16 +325,21 @@ surfaces: - fastvideo.configs.pipelines.wan.WanT2V720PConfig bsa_cdf_threshold: sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig] + target: generator.pipeline.longcat.bsa_cdf_threshold bsa_chunk_k: sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig] + target: generator.pipeline.longcat.bsa_chunk_k bsa_chunk_q: sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig] + target: generator.pipeline.longcat.bsa_chunk_q bsa_params: sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig] bsa_sparsity: sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig] + target: generator.pipeline.longcat.bsa_sparsity enable_bsa: sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig] + target: generator.pipeline.longcat.enable_bsa enable_kv_cache: sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig] enhance_hf: diff --git a/fastvideo/api/__init__.py b/fastvideo/api/__init__.py index 2fbc0b555..c2de8bcc8 100644 --- a/fastvideo/api/__init__.py +++ b/fastvideo/api/__init__.py @@ -10,6 +10,10 @@ from fastvideo.api.schema import ( GeneratorConfig, GpuPoolConfig, InputConfig, + LTX2Config, + LTX2RefineConfig, + LongCatConfig, + MiniMaxH3Config, OffloadConfig, OutputConfig, ParallelismConfig, @@ -71,6 +75,10 @@ __all__ = [ "GeneratorConfig", "GpuPoolConfig", "InputConfig", + "LTX2Config", + "LTX2RefineConfig", + "LongCatConfig", + "MiniMaxH3Config", "OffloadConfig", "OutputConfig", "ParallelismConfig", diff --git a/fastvideo/api/compat.py b/fastvideo/api/compat.py index ef3f6e191..6a3494165 100644 --- a/fastvideo/api/compat.py +++ b/fastvideo/api/compat.py @@ -110,6 +110,9 @@ def legacy_from_pretrained_to_config( _set_dotted_path(raw, ["engine", "compile", "extras"], remaining) elif key == "pipeline_config" and not isinstance(value, str): experimental[key] = deepcopy(value) + elif key.startswith(tuple(_COMPONENT_OVERRIDE_PREFIXES)): + component, field_name = key.split(".", 1) + _set_dotted_path(raw, ["pipeline", _COMPONENT_OVERRIDE_PREFIXES[component + "."], field_name], value) elif key in _LTX2_REFINE_PRESET_KEYWORDS: _set_dotted_path(raw, ["pipeline", "preset_overrides", "refine", key[len("ltx2_refine_"):]], value) elif key in _EMPTY_MEANS_UNSET_KEYWORDS: @@ -166,6 +169,10 @@ def generator_config_to_fastvideo_args(config: GeneratorConfig | Mapping[str, An _resolved_quant_cls = get_quantization_config(quantization.transformer_quant) kwargs["transformer_quant"] = _resolved_quant_cls() + for prefix, section in _COMPONENT_OVERRIDE_PREFIXES.items(): + for field_name, value in getattr(normalized.pipeline, section).items(): + kwargs[f"{prefix}{field_name}"] = deepcopy(value) + preset_overrides = deepcopy(normalized.pipeline.preset_overrides) refine = preset_overrides.pop("refine", None) if isinstance(refine, Mapping): @@ -303,9 +310,16 @@ _SPECIALLY_MAPPED_PATHS: dict[str, str] = { "pipeline.preset": "not supported", "pipeline.preset_version": "not supported", "pipeline.components.vae_weights": "not supported", + "pipeline.dit": "keys passed as dit_config.", + "pipeline.vae": "keys passed as vae_config.", "pipeline.preset_overrides": "keys passed as flat keywords; refine keys renamed to ltx2_refine_*", "pipeline.experimental": "keys passed as flat keywords", } +# Flat keyword prefix for a component config override -> the PipelineSelection dict that holds the overrides. +_COMPONENT_OVERRIDE_PREFIXES = { + "dit_config.": "dit", + "vae_config.": "vae", +} # from_pretrained keywords whose empty-string value means "unset", and the field each one sets. # ltx2_refine_lora_path sets the main LoRA path. _EMPTY_MEANS_UNSET_KEYWORDS = { diff --git a/fastvideo/api/schema.py b/fastvideo/api/schema.py index e93338d7a..164204b51 100644 --- a/fastvideo/api/schema.py +++ b/fastvideo/api/schema.py @@ -153,6 +153,72 @@ class ComponentConfig: override_transformer_cls_name: str | None = flat_field("override_transformer_cls_name", None) +@dataclass +class LTX2RefineConfig: + """Stage-2 refine assets that ``preset_overrides.refine`` and ``components`` do not cover.""" + + transformer_path: str | None = flat_field("ltx2_refine_transformer_path", None) + noise_path: str | None = flat_field("ltx2_refine_noise_path", None) + audio_noise_path: str | None = flat_field("ltx2_refine_audio_noise_path", None) + + +@dataclass +class LTX2Config: + """LTX-2 settings. ``None`` keeps the model's default.""" + + vae_spatial_tile_size_in_pixels: int | None = flat_field("ltx2_vae_spatial_tile_size_in_pixels", None) + vae_spatial_tile_overlap_in_pixels: int | None = flat_field("ltx2_vae_spatial_tile_overlap_in_pixels", None) + vae_temporal_tile_size_in_frames: int | None = flat_field("ltx2_vae_temporal_tile_size_in_frames", None) + vae_temporal_tile_overlap_in_frames: int | None = flat_field("ltx2_vae_temporal_tile_overlap_in_frames", None) + initial_latent_path: str | None = flat_field("ltx2_initial_latent_path", None) + """Path to load or save a precomputed initial video latent.""" + audio_latent_path: str | None = flat_field("ltx2_audio_latent_path", None) + """Path to load or save a precomputed initial audio latent.""" + legacy_native_noise_order: bool | None = flat_field("ltx2_legacy_native_noise_order", None) + """Draw latent noise in the legacy native order, which earlier SSIM references use.""" + use_distilled_sigmas: bool | None = flat_field("ltx2_use_distilled_sigmas", None) + """Use the distilled sigma schedule when the checkpoint provides one.""" + refine: LTX2RefineConfig = field(default_factory=LTX2RefineConfig) + + +@dataclass +class MiniMaxH3Config: + """MiniMax-H3 settings. ``None`` keeps the model's default.""" + + sequential_load: bool | None = flat_field("h3_sequential_load", None) + """Encode with Qwen3-VL, release that encoder, then load the DiT and VAEs. ``None`` enables it on + unified-memory devices only.""" + video_decode_backend: Literal["h3-vae", "taeh3"] | None = flat_field("video_decode_backend", None) + """``h3-vae`` is the full VAE; ``taeh3`` is a fast approximate preview decoder.""" + taeh3_checkpoint: str | None = flat_field("taeh3_checkpoint", None) + """Local ``taeh3.safetensors`` path. ``None`` downloads the pinned upstream weights.""" + taeh3_chunk_size: int | None = flat_field("taeh3_chunk_size", None) + """TAEH3 latent frames per execution chunk.""" + vae_parallel_decode: bool | None = flat_field("vae_parallel_decode", None) + """Spread VAE decode chunks across the sequence-parallel ranks. ``None`` falls back to + ``FASTVIDEO_VAE_PARALLEL_DECODE``.""" + vae_parallel_encode: bool | None = flat_field("vae_parallel_encode", None) + """Spread reference-video VAE encode clips across the sequence-parallel ranks. ``None`` falls back to + ``FASTVIDEO_VAE_PARALLEL_ENCODE``.""" + vae_parallel_decode_strategy: Literal["gather", "all_gather"] | None = flat_field( + "vae_parallel_decode_strategy", None) + """Collective that moves decoded chunks. ``None`` falls back to ``FASTVIDEO_VAE_PARALLEL_DECODE_STRATEGY``, + then ``gather``.""" + + +@dataclass +class LongCatConfig: + """LongCat block sparse attention (BSA) settings. ``None`` keeps the model's default.""" + + enable_bsa: bool | None = flat_field("enable_bsa", None) + bsa_sparsity: float | None = flat_field("bsa_sparsity", None) + bsa_cdf_threshold: float | None = flat_field("bsa_cdf_threshold", None) + bsa_chunk_q: list[int] | None = flat_field("bsa_chunk_q", None) + """Query chunk shape as ``[T, H, W]``.""" + bsa_chunk_k: list[int] | None = flat_field("bsa_chunk_k", None) + """Key chunk shape as ``[T, H, W]``.""" + + @dataclass class PipelineSelection: workload_type: Literal["t2v", "i2v", "t2i", "i2i", "v2a", "t2a"] | None = flat_field("workload_type", None) @@ -169,6 +235,13 @@ class PipelineSelection: """Guidance scale that guidance-distilled models take as a DiT input. ``None`` keeps the model's default.""" dmd_denoising_steps: list[int] | None = flat_field("dmd_denoising_steps", None) """Timesteps of a few-step distilled (DMD) sampler. ``None`` keeps the model's default.""" + dit: dict[str, Any] = field(default_factory=dict) + """Overrides for fields of the model's DiT config, such as ``prefix``.""" + vae: dict[str, Any] = field(default_factory=dict) + """Overrides for fields of the model's VAE config, such as ``load_encoder`` or ``use_tiling``.""" + ltx2: LTX2Config = field(default_factory=LTX2Config) + minimax_h3: MiniMaxH3Config = field(default_factory=MiniMaxH3Config) + longcat: LongCatConfig = field(default_factory=LongCatConfig) preset_overrides: dict[str, Any] = field(default_factory=dict) experimental: dict[str, Any] = field(default_factory=dict) @@ -363,6 +436,10 @@ __all__ = [ "GeneratorConfig", "GpuPoolConfig", "InputConfig", + "LTX2Config", + "LTX2RefineConfig", + "LongCatConfig", + "MiniMaxH3Config", "OffloadConfig", "OutputConfig", "ParallelismConfig", diff --git a/fastvideo/tests/api/test_parser.py b/fastvideo/tests/api/test_parser.py index 6e0214579..ff93d9671 100644 --- a/fastvideo/tests/api/test_parser.py +++ b/fastvideo/tests/api/test_parser.py @@ -174,6 +174,39 @@ def test_load_run_config_supports_yaml_roundtrip(tmp_path) -> None: "flow_shift": None, "embedded_cfg_scale": None, "dmd_denoising_steps": None, + "dit": {}, + "vae": {}, + "ltx2": { + "vae_spatial_tile_size_in_pixels": None, + "vae_spatial_tile_overlap_in_pixels": None, + "vae_temporal_tile_size_in_frames": None, + "vae_temporal_tile_overlap_in_frames": None, + "initial_latent_path": None, + "audio_latent_path": None, + "legacy_native_noise_order": None, + "use_distilled_sigmas": None, + "refine": { + "transformer_path": None, + "noise_path": None, + "audio_noise_path": None, + }, + }, + "minimax_h3": { + "sequential_load": None, + "video_decode_backend": None, + "taeh3_checkpoint": None, + "taeh3_chunk_size": None, + "vae_parallel_decode": None, + "vae_parallel_encode": None, + "vae_parallel_decode_strategy": None, + }, + "longcat": { + "enable_bsa": None, + "bsa_sparsity": None, + "bsa_cdf_threshold": None, + "bsa_chunk_q": None, + "bsa_chunk_k": None, + }, "preset_overrides": {}, "experimental": {}, },