[refactor]: add component overrides and model-family config partitions
Parameters that belong to one model family, or to one component config,
could be set only through argparse flags or pipeline.experimental. They gain
typed GeneratorConfig fields, each with the flat name that FastVideoArgs or
PipelineConfig uses:
- pipeline.dit and pipeline.vae: dicts of DiT and VAE config field
overrides, passed as dit_config.<key> and vae_config.<key>
- pipeline.ltx2: VAE tile sizes, initial video and audio latent paths, noise
order, distilled sigmas, and refine.{transformer,noise,audio_noise}_path
- pipeline.minimax_h3: sequential load, video decode backend, TAEH3
checkpoint and chunk size, and the sequence-parallel VAE switches
- pipeline.longcat: the block sparse attention (BSA) settings
legacy_from_pretrained_to_config routes dit_config.* and vae_config.*
keywords into the new dicts. Every new scalar field defaults to None and is
not passed to FastVideoArgs while unset. Behavior is unchanged: the golden
config snapshots match.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012M91wnVFmPEvJ9h39r5BH7
This commit is contained in:
Davids048
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
0b80dd52db
commit
b24396561a
@@ -71,33 +71,32 @@ surfaces:
|
||||
VSA_tile_size: generator.engine.attention.vsa_tile_size
|
||||
moba_config_path: generator.engine.attention.moba_config_path
|
||||
inference_torch_compile: generator.engine.compile.regional
|
||||
preset_owned:
|
||||
ltx2_vae_spatial_tile_size_in_pixels: generator.pipeline.preset_overrides.ltx2.vae.spatial_tile_size_in_pixels
|
||||
ltx2_vae_spatial_tile_overlap_in_pixels: generator.pipeline.preset_overrides.ltx2.vae.spatial_tile_overlap_in_pixels
|
||||
ltx2_vae_temporal_tile_size_in_frames: generator.pipeline.preset_overrides.ltx2.vae.temporal_tile_size_in_frames
|
||||
ltx2_vae_temporal_tile_overlap_in_frames: generator.pipeline.preset_overrides.ltx2.vae.temporal_tile_overlap_in_frames
|
||||
ltx2_initial_latent_path: request.extensions.ltx2.initial_latent_path
|
||||
ltx2_audio_latent_path: request.extensions.ltx2.audio_latent_path
|
||||
vae_parallel_decode: generator.pipeline.minimax_h3.vae_parallel_decode
|
||||
vae_parallel_encode: generator.pipeline.minimax_h3.vae_parallel_encode
|
||||
vae_parallel_decode_strategy: generator.pipeline.minimax_h3.vae_parallel_decode_strategy
|
||||
h3_sequential_load: generator.pipeline.minimax_h3.sequential_load
|
||||
video_decode_backend: generator.pipeline.minimax_h3.video_decode_backend
|
||||
taeh3_checkpoint: generator.pipeline.minimax_h3.taeh3_checkpoint
|
||||
taeh3_chunk_size: generator.pipeline.minimax_h3.taeh3_chunk_size
|
||||
refine_transformer_path: generator.pipeline.ltx2.refine.transformer_path
|
||||
refine_noise_path: generator.pipeline.ltx2.refine.noise_path
|
||||
refine_audio_noise_path: generator.pipeline.ltx2.refine.audio_noise_path
|
||||
ltx2_refine_transformer_path: generator.pipeline.ltx2.refine.transformer_path
|
||||
ltx2_refine_noise_path: generator.pipeline.ltx2.refine.noise_path
|
||||
ltx2_refine_audio_noise_path: generator.pipeline.ltx2.refine.audio_noise_path
|
||||
ltx2_legacy_native_noise_order: generator.pipeline.ltx2.legacy_native_noise_order
|
||||
ltx2_use_distilled_sigmas: generator.pipeline.ltx2.use_distilled_sigmas
|
||||
ltx2_vae_spatial_tile_size_in_pixels: generator.pipeline.ltx2.vae_spatial_tile_size_in_pixels
|
||||
ltx2_vae_spatial_tile_overlap_in_pixels: generator.pipeline.ltx2.vae_spatial_tile_overlap_in_pixels
|
||||
ltx2_vae_temporal_tile_size_in_frames: generator.pipeline.ltx2.vae_temporal_tile_size_in_frames
|
||||
ltx2_vae_temporal_tile_overlap_in_frames: generator.pipeline.ltx2.vae_temporal_tile_overlap_in_frames
|
||||
ltx2_initial_latent_path: generator.pipeline.ltx2.initial_latent_path
|
||||
ltx2_audio_latent_path: generator.pipeline.ltx2.audio_latent_path
|
||||
compatibility_only:
|
||||
mode: "Legacy multi-mode FastVideoArgs switch; typed inference config should not expose execution mode."
|
||||
inference_mode: "Legacy boolean mirror of mode; kept only through adapters while FastVideoArgs remains."
|
||||
output_type: "Legacy output formatting surface pending GenerationResult cleanup."
|
||||
vae_parallel_decode: "MiniMax-H3 sequence-parallel VAE decode opt-in; model-specific optimization not yet represented in the typed public schema."
|
||||
h3_sequential_load: "MiniMax-H3 sequential text-encoder then DiT/VAE load; model-specific optimization not yet represented in the typed public schema."
|
||||
video_decode_backend: "MiniMax-H3 video decoder selection (full VAE vs TAEH3 preview); model-specific optimization not yet represented in the typed public schema."
|
||||
taeh3_checkpoint: "Optional local TAEH3 safetensors path; model-specific optimization not yet represented in the typed public schema."
|
||||
taeh3_chunk_size: "TAEH3 temporal chunk length; model-specific optimization not yet represented in the typed public schema."
|
||||
vae_parallel_encode: "MiniMax-H3 sequence-parallel reference VAE encode opt-in; model-specific optimization not yet represented in the typed public schema."
|
||||
vae_parallel_decode_strategy: "Chunk-transport collective for vae_parallel_decode; model-specific optimization not yet represented in the typed public schema."
|
||||
master_port: "Executor/bootstrap compatibility field; not part of the canonical inference schema."
|
||||
refine_transformer_path: "Generic stage-2 refine transformer override; no typed equivalent yet."
|
||||
refine_noise_path: "Generic stage-2 refine noise override; no typed equivalent yet."
|
||||
refine_audio_noise_path: "Generic stage-2 refine audio noise override; no typed equivalent yet."
|
||||
ltx2_refine_transformer_path: "LTX-2 refine transformer carrier; no typed equivalent yet."
|
||||
ltx2_refine_noise_path: "LTX-2 refine noise carrier; no typed equivalent yet."
|
||||
ltx2_refine_audio_noise_path: "LTX-2 refine audio noise carrier; no typed equivalent yet."
|
||||
ltx2_legacy_native_noise_order: "LTX-2 SSIM compatibility knob preserving legacy native latent noise ordering."
|
||||
ltx2_use_distilled_sigmas: "LTX-2 compatibility knob gating use of distilled sigma schedule."
|
||||
private_only:
|
||||
ray_placement_group: "Ray deployment-only field."
|
||||
ray_runtime_env: "Ray deployment-only field."
|
||||
@@ -326,16 +325,21 @@ surfaces:
|
||||
- fastvideo.configs.pipelines.wan.WanT2V720PConfig
|
||||
bsa_cdf_threshold:
|
||||
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
|
||||
target: generator.pipeline.longcat.bsa_cdf_threshold
|
||||
bsa_chunk_k:
|
||||
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
|
||||
target: generator.pipeline.longcat.bsa_chunk_k
|
||||
bsa_chunk_q:
|
||||
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
|
||||
target: generator.pipeline.longcat.bsa_chunk_q
|
||||
bsa_params:
|
||||
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
|
||||
bsa_sparsity:
|
||||
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
|
||||
target: generator.pipeline.longcat.bsa_sparsity
|
||||
enable_bsa:
|
||||
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
|
||||
target: generator.pipeline.longcat.enable_bsa
|
||||
enable_kv_cache:
|
||||
sources: [fastvideo.configs.pipelines.longcat.LongCatT2V480PConfig, fastvideo.configs.pipelines.longcat.LongCatT2V704PConfig]
|
||||
enhance_hf:
|
||||
|
||||
@@ -10,6 +10,10 @@ from fastvideo.api.schema import (
|
||||
GeneratorConfig,
|
||||
GpuPoolConfig,
|
||||
InputConfig,
|
||||
LTX2Config,
|
||||
LTX2RefineConfig,
|
||||
LongCatConfig,
|
||||
MiniMaxH3Config,
|
||||
OffloadConfig,
|
||||
OutputConfig,
|
||||
ParallelismConfig,
|
||||
@@ -71,6 +75,10 @@ __all__ = [
|
||||
"GeneratorConfig",
|
||||
"GpuPoolConfig",
|
||||
"InputConfig",
|
||||
"LTX2Config",
|
||||
"LTX2RefineConfig",
|
||||
"LongCatConfig",
|
||||
"MiniMaxH3Config",
|
||||
"OffloadConfig",
|
||||
"OutputConfig",
|
||||
"ParallelismConfig",
|
||||
|
||||
@@ -110,6 +110,9 @@ def legacy_from_pretrained_to_config(
|
||||
_set_dotted_path(raw, ["engine", "compile", "extras"], remaining)
|
||||
elif key == "pipeline_config" and not isinstance(value, str):
|
||||
experimental[key] = deepcopy(value)
|
||||
elif key.startswith(tuple(_COMPONENT_OVERRIDE_PREFIXES)):
|
||||
component, field_name = key.split(".", 1)
|
||||
_set_dotted_path(raw, ["pipeline", _COMPONENT_OVERRIDE_PREFIXES[component + "."], field_name], value)
|
||||
elif key in _LTX2_REFINE_PRESET_KEYWORDS:
|
||||
_set_dotted_path(raw, ["pipeline", "preset_overrides", "refine", key[len("ltx2_refine_"):]], value)
|
||||
elif key in _EMPTY_MEANS_UNSET_KEYWORDS:
|
||||
@@ -166,6 +169,10 @@ def generator_config_to_fastvideo_args(config: GeneratorConfig | Mapping[str, An
|
||||
_resolved_quant_cls = get_quantization_config(quantization.transformer_quant)
|
||||
kwargs["transformer_quant"] = _resolved_quant_cls()
|
||||
|
||||
for prefix, section in _COMPONENT_OVERRIDE_PREFIXES.items():
|
||||
for field_name, value in getattr(normalized.pipeline, section).items():
|
||||
kwargs[f"{prefix}{field_name}"] = deepcopy(value)
|
||||
|
||||
preset_overrides = deepcopy(normalized.pipeline.preset_overrides)
|
||||
refine = preset_overrides.pop("refine", None)
|
||||
if isinstance(refine, Mapping):
|
||||
@@ -303,9 +310,16 @@ _SPECIALLY_MAPPED_PATHS: dict[str, str] = {
|
||||
"pipeline.preset": "not supported",
|
||||
"pipeline.preset_version": "not supported",
|
||||
"pipeline.components.vae_weights": "not supported",
|
||||
"pipeline.dit": "keys passed as dit_config.<key>",
|
||||
"pipeline.vae": "keys passed as vae_config.<key>",
|
||||
"pipeline.preset_overrides": "keys passed as flat keywords; refine keys renamed to ltx2_refine_*",
|
||||
"pipeline.experimental": "keys passed as flat keywords",
|
||||
}
|
||||
# Flat keyword prefix for a component config override -> the PipelineSelection dict that holds the overrides.
|
||||
_COMPONENT_OVERRIDE_PREFIXES = {
|
||||
"dit_config.": "dit",
|
||||
"vae_config.": "vae",
|
||||
}
|
||||
# from_pretrained keywords whose empty-string value means "unset", and the field each one sets.
|
||||
# ltx2_refine_lora_path sets the main LoRA path.
|
||||
_EMPTY_MEANS_UNSET_KEYWORDS = {
|
||||
|
||||
@@ -153,6 +153,72 @@ class ComponentConfig:
|
||||
override_transformer_cls_name: str | None = flat_field("override_transformer_cls_name", None)
|
||||
|
||||
|
||||
@dataclass
|
||||
class LTX2RefineConfig:
|
||||
"""Stage-2 refine assets that ``preset_overrides.refine`` and ``components`` do not cover."""
|
||||
|
||||
transformer_path: str | None = flat_field("ltx2_refine_transformer_path", None)
|
||||
noise_path: str | None = flat_field("ltx2_refine_noise_path", None)
|
||||
audio_noise_path: str | None = flat_field("ltx2_refine_audio_noise_path", None)
|
||||
|
||||
|
||||
@dataclass
|
||||
class LTX2Config:
|
||||
"""LTX-2 settings. ``None`` keeps the model's default."""
|
||||
|
||||
vae_spatial_tile_size_in_pixels: int | None = flat_field("ltx2_vae_spatial_tile_size_in_pixels", None)
|
||||
vae_spatial_tile_overlap_in_pixels: int | None = flat_field("ltx2_vae_spatial_tile_overlap_in_pixels", None)
|
||||
vae_temporal_tile_size_in_frames: int | None = flat_field("ltx2_vae_temporal_tile_size_in_frames", None)
|
||||
vae_temporal_tile_overlap_in_frames: int | None = flat_field("ltx2_vae_temporal_tile_overlap_in_frames", None)
|
||||
initial_latent_path: str | None = flat_field("ltx2_initial_latent_path", None)
|
||||
"""Path to load or save a precomputed initial video latent."""
|
||||
audio_latent_path: str | None = flat_field("ltx2_audio_latent_path", None)
|
||||
"""Path to load or save a precomputed initial audio latent."""
|
||||
legacy_native_noise_order: bool | None = flat_field("ltx2_legacy_native_noise_order", None)
|
||||
"""Draw latent noise in the legacy native order, which earlier SSIM references use."""
|
||||
use_distilled_sigmas: bool | None = flat_field("ltx2_use_distilled_sigmas", None)
|
||||
"""Use the distilled sigma schedule when the checkpoint provides one."""
|
||||
refine: LTX2RefineConfig = field(default_factory=LTX2RefineConfig)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MiniMaxH3Config:
|
||||
"""MiniMax-H3 settings. ``None`` keeps the model's default."""
|
||||
|
||||
sequential_load: bool | None = flat_field("h3_sequential_load", None)
|
||||
"""Encode with Qwen3-VL, release that encoder, then load the DiT and VAEs. ``None`` enables it on
|
||||
unified-memory devices only."""
|
||||
video_decode_backend: Literal["h3-vae", "taeh3"] | None = flat_field("video_decode_backend", None)
|
||||
"""``h3-vae`` is the full VAE; ``taeh3`` is a fast approximate preview decoder."""
|
||||
taeh3_checkpoint: str | None = flat_field("taeh3_checkpoint", None)
|
||||
"""Local ``taeh3.safetensors`` path. ``None`` downloads the pinned upstream weights."""
|
||||
taeh3_chunk_size: int | None = flat_field("taeh3_chunk_size", None)
|
||||
"""TAEH3 latent frames per execution chunk."""
|
||||
vae_parallel_decode: bool | None = flat_field("vae_parallel_decode", None)
|
||||
"""Spread VAE decode chunks across the sequence-parallel ranks. ``None`` falls back to
|
||||
``FASTVIDEO_VAE_PARALLEL_DECODE``."""
|
||||
vae_parallel_encode: bool | None = flat_field("vae_parallel_encode", None)
|
||||
"""Spread reference-video VAE encode clips across the sequence-parallel ranks. ``None`` falls back to
|
||||
``FASTVIDEO_VAE_PARALLEL_ENCODE``."""
|
||||
vae_parallel_decode_strategy: Literal["gather", "all_gather"] | None = flat_field(
|
||||
"vae_parallel_decode_strategy", None)
|
||||
"""Collective that moves decoded chunks. ``None`` falls back to ``FASTVIDEO_VAE_PARALLEL_DECODE_STRATEGY``,
|
||||
then ``gather``."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class LongCatConfig:
|
||||
"""LongCat block sparse attention (BSA) settings. ``None`` keeps the model's default."""
|
||||
|
||||
enable_bsa: bool | None = flat_field("enable_bsa", None)
|
||||
bsa_sparsity: float | None = flat_field("bsa_sparsity", None)
|
||||
bsa_cdf_threshold: float | None = flat_field("bsa_cdf_threshold", None)
|
||||
bsa_chunk_q: list[int] | None = flat_field("bsa_chunk_q", None)
|
||||
"""Query chunk shape as ``[T, H, W]``."""
|
||||
bsa_chunk_k: list[int] | None = flat_field("bsa_chunk_k", None)
|
||||
"""Key chunk shape as ``[T, H, W]``."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class PipelineSelection:
|
||||
workload_type: Literal["t2v", "i2v", "t2i", "i2i", "v2a", "t2a"] | None = flat_field("workload_type", None)
|
||||
@@ -169,6 +235,13 @@ class PipelineSelection:
|
||||
"""Guidance scale that guidance-distilled models take as a DiT input. ``None`` keeps the model's default."""
|
||||
dmd_denoising_steps: list[int] | None = flat_field("dmd_denoising_steps", None)
|
||||
"""Timesteps of a few-step distilled (DMD) sampler. ``None`` keeps the model's default."""
|
||||
dit: dict[str, Any] = field(default_factory=dict)
|
||||
"""Overrides for fields of the model's DiT config, such as ``prefix``."""
|
||||
vae: dict[str, Any] = field(default_factory=dict)
|
||||
"""Overrides for fields of the model's VAE config, such as ``load_encoder`` or ``use_tiling``."""
|
||||
ltx2: LTX2Config = field(default_factory=LTX2Config)
|
||||
minimax_h3: MiniMaxH3Config = field(default_factory=MiniMaxH3Config)
|
||||
longcat: LongCatConfig = field(default_factory=LongCatConfig)
|
||||
preset_overrides: dict[str, Any] = field(default_factory=dict)
|
||||
experimental: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
@@ -363,6 +436,10 @@ __all__ = [
|
||||
"GeneratorConfig",
|
||||
"GpuPoolConfig",
|
||||
"InputConfig",
|
||||
"LTX2Config",
|
||||
"LTX2RefineConfig",
|
||||
"LongCatConfig",
|
||||
"MiniMaxH3Config",
|
||||
"OffloadConfig",
|
||||
"OutputConfig",
|
||||
"ParallelismConfig",
|
||||
|
||||
@@ -174,6 +174,39 @@ def test_load_run_config_supports_yaml_roundtrip(tmp_path) -> None:
|
||||
"flow_shift": None,
|
||||
"embedded_cfg_scale": None,
|
||||
"dmd_denoising_steps": None,
|
||||
"dit": {},
|
||||
"vae": {},
|
||||
"ltx2": {
|
||||
"vae_spatial_tile_size_in_pixels": None,
|
||||
"vae_spatial_tile_overlap_in_pixels": None,
|
||||
"vae_temporal_tile_size_in_frames": None,
|
||||
"vae_temporal_tile_overlap_in_frames": None,
|
||||
"initial_latent_path": None,
|
||||
"audio_latent_path": None,
|
||||
"legacy_native_noise_order": None,
|
||||
"use_distilled_sigmas": None,
|
||||
"refine": {
|
||||
"transformer_path": None,
|
||||
"noise_path": None,
|
||||
"audio_noise_path": None,
|
||||
},
|
||||
},
|
||||
"minimax_h3": {
|
||||
"sequential_load": None,
|
||||
"video_decode_backend": None,
|
||||
"taeh3_checkpoint": None,
|
||||
"taeh3_chunk_size": None,
|
||||
"vae_parallel_decode": None,
|
||||
"vae_parallel_encode": None,
|
||||
"vae_parallel_decode_strategy": None,
|
||||
},
|
||||
"longcat": {
|
||||
"enable_bsa": None,
|
||||
"bsa_sparsity": None,
|
||||
"bsa_cdf_threshold": None,
|
||||
"bsa_chunk_q": None,
|
||||
"bsa_chunk_k": None,
|
||||
},
|
||||
"preset_overrides": {},
|
||||
"experimental": {},
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user