diff --git a/docs/design/inference_schema_parity_inventory.yaml b/docs/design/inference_schema_parity_inventory.yaml index 6fa90b7dc..966ca7049 100644 --- a/docs/design/inference_schema_parity_inventory.yaml +++ b/docs/design/inference_schema_parity_inventory.yaml @@ -55,13 +55,13 @@ surfaces: ltx2_vae_tiling: generator.pipeline.vae_tiling refine_enabled: generator.pipeline.preset_overrides.refine.enabled refine_upsampler_path: generator.pipeline.components.upsampler_weights - refine_lora_path: generator.pipeline.components.lora_path + refine_lora_path: generator.pipeline.ltx2.refine.lora_path refine_num_inference_steps: request.stage_overrides.refine.num_inference_steps refine_guidance_scale: request.stage_overrides.refine.guidance_scale refine_add_noise: generator.pipeline.preset_overrides.refine.add_noise ltx2_refine_enabled: generator.pipeline.preset_overrides.refine.enabled ltx2_refine_upsampler_path: generator.pipeline.components.upsampler_weights - ltx2_refine_lora_path: generator.pipeline.components.lora_path + ltx2_refine_lora_path: generator.pipeline.ltx2.refine.lora_path ltx2_refine_num_inference_steps: request.stage_overrides.refine.num_inference_steps ltx2_refine_guidance_scale: request.stage_overrides.refine.guidance_scale ltx2_refine_add_noise: generator.pipeline.preset_overrides.refine.add_noise diff --git a/docs/design/server_contracts/dynamo.md b/docs/design/server_contracts/dynamo.md index 70f70e612..f78b3e21b 100644 --- a/docs/design/server_contracts/dynamo.md +++ b/docs/design/server_contracts/dynamo.md @@ -262,7 +262,8 @@ this adapter can build the config purely from the public typed schema: def build_generator_config(args) -> "GeneratorConfig": from fastvideo.api import ( CompileConfig, ComponentConfig, EngineConfig, GeneratorConfig, - OffloadConfig, ParallelismConfig, PipelineSelection, + LTX2Options, LTX2RefineOptions, OffloadConfig, ParallelismConfig, + PipelineSelection, ) return GeneratorConfig( model_path=args.model_path, @@ -275,10 +276,8 @@ def build_generator_config(args) -> "GeneratorConfig": pipeline=PipelineSelection( workload_type=args.workload or "t2v", preset=args.preset, # e.g. "ltx2_two_stage" - components=ComponentConfig( - upsampler_weights=args.refine_upsampler, - lora_path=args.refine_lora, - ), + components=ComponentConfig(upsampler_weights=args.refine_upsampler), + ltx2=LTX2Options(refine=LTX2RefineOptions(lora_path=args.refine_lora)), ), ) ``` diff --git a/examples/inference/basic/basic_ltx2_3_distilled_i2v_typed.py b/examples/inference/basic/basic_ltx2_3_distilled_i2v_typed.py index f5ac88e0f..b249ee88f 100644 --- a/examples/inference/basic/basic_ltx2_3_distilled_i2v_typed.py +++ b/examples/inference/basic/basic_ltx2_3_distilled_i2v_typed.py @@ -56,7 +56,7 @@ Typed-API mapping (legacy kwarg ↔ typed field) - ``ltx2_vae_tiling`` ↔ ``pipeline.vae_tiling`` - ``ltx2_refine_enabled`` ↔ ``pipeline.preset_overrides["refine"]["enabled"]`` - ``ltx2_refine_upsampler_path`` ↔ ``pipeline.components.upsampler_weights`` -- ``ltx2_refine_lora_path`` ↔ ``pipeline.components.lora_path`` +- ``ltx2_refine_lora_path`` ↔ ``pipeline.ltx2.refine.lora_path`` - ``ltx2_refine_num_inference_steps`` ↔ ``pipeline.preset_overrides["refine"]["num_inference_steps"]`` - ``ltx2_refine_guidance_scale`` ↔ ``pipeline.preset_overrides["refine"]["guidance_scale"]`` - ``ltx2_refine_add_noise`` ↔ ``pipeline.preset_overrides["refine"]["add_noise"]`` diff --git a/fastvideo/api/compat.py b/fastvideo/api/compat.py index 6a3494165..93917b249 100644 --- a/fastvideo/api/compat.py +++ b/fastvideo/api/compat.py @@ -321,10 +321,8 @@ _COMPONENT_OVERRIDE_PREFIXES = { "vae_config.": "vae", } # from_pretrained keywords whose empty-string value means "unset", and the field each one sets. -# ltx2_refine_lora_path sets the main LoRA path. _EMPTY_MEANS_UNSET_KEYWORDS = { "ltx2_refine_upsampler_path": "pipeline.components.upsampler_weights", - "ltx2_refine_lora_path": "pipeline.components.lora_path", } # from_pretrained keywords that set pipeline.preset_overrides.refine.. _LTX2_REFINE_PRESET_KEYWORDS = frozenset({ diff --git a/fastvideo/api/schema.py b/fastvideo/api/schema.py index 4735f96ad..2852451a0 100644 --- a/fastvideo/api/schema.py +++ b/fastvideo/api/schema.py @@ -158,6 +158,9 @@ class LTX2RefineOptions: """Stage-2 refine assets that ``preset_overrides.refine`` and ``components`` do not cover.""" transformer_path: str | None = flat_field("ltx2_refine_transformer_path", None) + lora_path: str | None = flat_field("ltx2_refine_lora_path", None) + """LoRA applied to the refine transformer only. ``None`` uses the checkpoint's default + (``fastvideo_refine_lora_path`` in ``model_index.json``); an empty string disables the refine LoRA.""" noise_path: str | None = flat_field("ltx2_refine_noise_path", None) audio_noise_path: str | None = flat_field("ltx2_refine_audio_noise_path", None) diff --git a/fastvideo/pipelines/basic/ltx2/stage_overrides.py b/fastvideo/pipelines/basic/ltx2/stage_overrides.py index 7e9c0786c..afe25fd12 100644 --- a/fastvideo/pipelines/basic/ltx2/stage_overrides.py +++ b/fastvideo/pipelines/basic/ltx2/stage_overrides.py @@ -6,8 +6,10 @@ * ``stage_overrides.refine`` — per-request knobs (see :class:`LTX2RefineStageOverride`). -Asset paths live on :class:`~fastvideo.api.schema.ComponentConfig` -(``upsampler_weights`` and ``lora_path``). +The refine upsampler path lives on +:class:`~fastvideo.api.schema.ComponentConfig` (``upsampler_weights``), and the +refine LoRA path on :class:`~fastvideo.api.schema.LTX2RefineOptions` +(``pipeline.ltx2.refine.lora_path``). """ from __future__ import annotations diff --git a/fastvideo/tests/api/config_snapshot.py b/fastvideo/tests/api/config_snapshot.py index ed3f48013..44e83830a 100644 --- a/fastvideo/tests/api/config_snapshot.py +++ b/fastvideo/tests/api/config_snapshot.py @@ -354,6 +354,9 @@ def collect_cases() -> list[SnapshotCase]: "ltx2_refine_lora_path": (LTX2, { "ltx2_refine_lora_path": "/checkpoints/refine_lora.safetensors" }), + "ltx2_refine_lora_disabled": (LTX2, { + "ltx2_refine_lora_path": "" + }), "pipeline_config_json_path": ("FastVideo/FastHunyuan-diffusers", { "pipeline_config": pipeline_json }), diff --git a/fastvideo/tests/api/config_snapshot_goldens/kwargs/ltx2_refine_lora_disabled.json b/fastvideo/tests/api/config_snapshot_goldens/kwargs/ltx2_refine_lora_disabled.json new file mode 100644 index 000000000..4d4fe2343 --- /dev/null +++ b/fastvideo/tests/api/config_snapshot_goldens/kwargs/ltx2_refine_lora_disabled.json @@ -0,0 +1,356 @@ +{ + "fastvideo_args": { + "VSA_sparsity": 0.0, + "VSA_tile_size": 256, + "attention_backend": null, + "boundary_ratio": 0.875, + "disable_autocast": false, + "dist_timeout": null, + "distributed_executor_backend": "mp", + "dit_cpu_offload": true, + "dit_layerwise_offload": true, + "enable_stage_verification": true, + "enable_torch_compile": false, + "enable_torch_compile_audio_vae": false, + "enable_torch_compile_text_encoder": false, + "enable_torch_compile_vae": false, + "h3_sequential_load": null, + "hsdp_replicate_dim": 1, + "hsdp_shard_dim": 1, + "image_encoder_cpu_offload": true, + "inference_mode": true, + "inference_torch_compile": false, + "init_weights_from_safetensors": "", + "init_weights_from_safetensors_2": "", + "lazy_module_load": null, + "lora_nickname": "default", + "lora_path": null, + "lora_strength": 1.0, + "lora_target_modules": null, + "ltx2_audio_latent_path": null, + "ltx2_initial_latent_path": null, + "ltx2_legacy_native_noise_order": false, + "ltx2_refine_add_noise": true, + "ltx2_refine_audio_noise_path": null, + "ltx2_refine_enabled": false, + "ltx2_refine_guidance_scale": 1.0, + "ltx2_refine_lora_path": "", + "ltx2_refine_noise_path": null, + "ltx2_refine_num_inference_steps": 3, + "ltx2_refine_transformer_path": null, + "ltx2_refine_upsampler_path": null, + "ltx2_use_distilled_sigmas": true, + "ltx2_vae_spatial_tile_overlap_in_pixels": null, + "ltx2_vae_spatial_tile_size_in_pixels": null, + "ltx2_vae_temporal_tile_overlap_in_frames": null, + "ltx2_vae_temporal_tile_size_in_frames": null, + "ltx2_vae_tiling": null, + "master_port": null, + "moba_config": {}, + "moba_config_path": null, + "mode": "inference", + "model_loaded": { + "transformer": true, + "upsampler": true, + "vae": true + }, + "model_path": "FastVideo/LTX2-Distilled-Diffusers", + "model_paths": {}, + "num_gpus": 1, + "output_type": "pil", + "override_pipeline_cls_name": null, + "override_text_encoder_quant": null, + "override_text_encoder_safetensors": null, + "override_transformer_cls_name": null, + "pin_cpu_memory": true, + "preprocess_config": null, + "prompt_txt": null, + "ray_placement_group": null, + "ray_runtime_env": null, + "refine_add_noise": null, + "refine_audio_noise_path": null, + "refine_enabled": null, + "refine_guidance_scale": null, + "refine_lora_path": null, + "refine_noise_path": null, + "refine_num_inference_steps": null, + "refine_transformer_path": null, + "refine_upsampler_path": null, + "revision": null, + "sp_size": 1, + "taeh3_checkpoint": null, + "taeh3_chunk_size": 5, + "text_encoder_cpu_offload": true, + "torch_compile_kwargs": {}, + "torch_compile_kwargs_audio_vae": {}, + "torch_compile_kwargs_dit": {}, + "torch_compile_kwargs_text_encoder": {}, + "torch_compile_kwargs_vae": {}, + "tp_size": 1, + "transformer_quant": null, + "trust_remote_code": false, + "use_fsdp_inference": false, + "vae_cpu_offload": true, + "vae_parallel_decode": false, + "vae_parallel_decode_strategy": "gather", + "vae_parallel_encode": false, + "video_decode_backend": "h3-vae", + "workload_type": "t2v" + }, + "pipeline_config": { + "audio_decoder_config": { + "_resolved_attention_backend": null, + "arch_config": { + "architectures": [ + "LTX2AudioDecoder" + ], + "stacked_params_mapping": [] + } + }, + "audio_decoder_precision": "bf16", + "boundary_ratio": null, + "disable_autocast": false, + "dit_config": { + "_resolved_attention_backend": null, + "arch_config": { + "_supported_attention_backends": [ + 3, + 1, + 2, + 7, + 10, + 4, + 5, + 6, + 11, + 12 + ], + "apply_gated_attention": false, + "attention_head_dim": 128, + "attention_type": "default", + "audio_attention_head_dim": 64, + "audio_connector_attention_head_dim": 128, + "audio_connector_num_attention_heads": 30, + "audio_connector_num_layers": 2, + "audio_cross_attention_dim": 2048, + "audio_in_channels": 128, + "audio_num_attention_heads": 32, + "audio_out_channels": 128, + "audio_positional_embedding_max_pos": [ + 20 + ], + "av_ca_timestep_scale_multiplier": 1, + "boundary_ratio": null, + "caption_channels": 3840, + "caption_proj_before_connector": false, + "caption_proj_input_norm": true, + "caption_projection_first_linear": true, + "caption_projection_second_linear": true, + "cast_prompt_embeds_to_dit_dtype": false, + "connector_attention_head_dim": 128, + "connector_num_attention_heads": 30, + "connector_num_layers": 2, + "cross_attention_adaln": false, + "cross_attention_dim": 4096, + "double_precision_rope": true, + "exclude_lora_layers": [], + "hidden_size": 0, + "in_channels": 128, + "norm_eps": 1e-06, + "num_attention_heads": 32, + "num_channels_latents": 128, + "num_layers": 48, + "out_channels": 128, + "patch_size": [ + 1, + 1, + 1 + ], + "positional_embedding_max_pos": [ + 20, + 2048, + 2048 + ], + "positional_embedding_theta": 10000.0, + "rope_type": "split", + "stacked_params_mapping": [], + "stg_block_idx": 29, + "timestep_scale_multiplier": 1000, + "use_middle_indices_grid": true + }, + "prefix": "ltx2", + "quant_config": null + }, + "dit_precision": "bf16", + "dmd_denoising_steps": null, + "embedded_cfg_scale": 6.0, + "flow_shift": null, + "flow_shift_sr": null, + "image_encoder_config": { + "_resolved_attention_backend": null, + "arch_config": { + "_supported_attention_backends": [ + 1, + 2 + ], + "architectures": [], + "output_hidden_states": false, + "stacked_params_mapping": [], + "use_return_dict": true + }, + "lora_config": null, + "prefix": "", + "quant_config": null + }, + "image_encoder_configs": null, + "image_encoder_precision": "fp32", + "image_encoder_precisions": null, + "is_causal": false, + "lucy_edit_task": false, + "model_path": "FastVideo/LTX2-Distilled-Diffusers", + "pipeline_config_path": null, + "postprocess_text_funcs": [ + "fastvideo.pipelines.basic.ltx2.pipeline_configs.ltx2_postprocess_text" + ], + "preprocess_text_funcs": [ + "fastvideo.configs.pipelines.base.preprocess_text" + ], + "scheduler_step_in_fp32": false, + "text_encoder_configs": [ + { + "_resolved_attention_backend": null, + "arch_config": { + "_supported_attention_backends": [ + 1, + 2 + ], + "architectures": [ + "LTX2GemmaTextEncoderModel" + ], + "audio_connector_attention_head_dim": null, + "audio_connector_num_attention_heads": null, + "audio_connector_num_layers": null, + "audio_feature_extractor_out_features": null, + "caption_proj_before_connector": false, + "caption_proj_input_norm": true, + "caption_projection_first_linear": true, + "caption_projection_second_linear": true, + "connector_apply_gated_attention": false, + "connector_attention_head_dim": 128, + "connector_double_precision_rope": false, + "connector_num_attention_heads": 30, + "connector_num_layers": 2, + "connector_num_learnable_registers": 128, + "connector_positional_embedding_max_pos": [ + 4096 + ], + "connector_positional_embedding_theta": 10000.0, + "connector_rope_type": "split", + "decoder_start_token_id": 0, + "eos_token_id": 2, + "feature_extractor_in_features": 188160, + "feature_extractor_out_features": 3840, + "gemma_dtype": "bfloat16", + "gemma_model_path": "", + "hidden_size": 3840, + "hidden_state_skip_layer": 0, + "num_attention_heads": 30, + "num_hidden_layers": 48, + "output_hidden_states": true, + "output_past": true, + "pad_token_id": 0, + "padding_side": "left", + "require_processor": false, + "scalable_attention": true, + "stacked_params_mapping": [], + "text_len": 1024, + "tie_word_embeddings": false, + "tokenizer_kwargs": { + "max_length": 1024, + "padding": "max_length", + "return_tensors": "pt", + "truncation": true + }, + "use_return_dict": true, + "video_feature_extractor_out_features": null, + "vocab_size": 0 + }, + "chat_template_enable_thinking": false, + "is_chat_model": false, + "lora_config": null, + "prefix": "ltx2_gemma", + "quant_config": null, + "treat_empty_as_dot": false + } + ], + "text_encoder_precisions": [ + "bf16" + ], + "ti2v_task": false, + "upsampler_config": { + "_resolved_attention_backend": null, + "arch_config": { + "stacked_params_mapping": [] + } + }, + "upsampler_precision": "fp32", + "vae_config": { + "_resolved_attention_backend": null, + "arch_config": { + "_class_name": "CausalVideoAutoencoder", + "causal_decoder": false, + "decoder_blocks": [], + "decoder_spatial_padding_mode": "reflect", + "dims": 3, + "encoder_blocks": [], + "encoder_spatial_padding_mode": "zeros", + "in_channels": 3, + "latent_channels": 128, + "latent_log_var": "uniform", + "norm_layer": "pixel_norm", + "normalize_latent_channels": false, + "out_channels": 3, + "patch_size": 4, + "scaling_factor": 1.0, + "spatial_compression_ratio": 32, + "stacked_params_mapping": [], + "temporal_compression_ratio": 8, + "timestep_conditioning": true, + "use_quant_conv": false, + "z_dim": 128 + }, + "blend_num_frames": 4, + "load_decoder": true, + "load_encoder": false, + "ltx2_spatial_tile_overlap_in_pixels": 64, + "ltx2_spatial_tile_size_in_pixels": 512, + "ltx2_temporal_tile_overlap_in_frames": 24, + "ltx2_temporal_tile_size_in_frames": 64, + "tile_sample_min_height": 256, + "tile_sample_min_num_frames": 16, + "tile_sample_min_width": 256, + "tile_sample_stride_height": 192, + "tile_sample_stride_num_frames": 12, + "tile_sample_stride_width": 192, + "use_parallel_tiling": true, + "use_temporal_scaling_frames": true, + "use_temporal_tiling": true, + "use_tiling": true + }, + "vae_decode_precision": null, + "vae_precision": "bf16", + "vae_sp": false, + "vae_tiling": true, + "vocoder_config": { + "_resolved_attention_backend": null, + "arch_config": { + "architectures": [ + "LTX2Vocoder" + ], + "stacked_params_mapping": [] + } + }, + "vocoder_precision": "bf16" + }, + "pipeline_config_class": "fastvideo.pipelines.basic.ltx2.pipeline_configs.LTX2T2VConfig" +} diff --git a/fastvideo/tests/api/config_snapshot_goldens/kwargs/ltx2_refine_lora_path.json b/fastvideo/tests/api/config_snapshot_goldens/kwargs/ltx2_refine_lora_path.json index 4da00b49f..485b70e1a 100644 --- a/fastvideo/tests/api/config_snapshot_goldens/kwargs/ltx2_refine_lora_path.json +++ b/fastvideo/tests/api/config_snapshot_goldens/kwargs/ltx2_refine_lora_path.json @@ -24,7 +24,7 @@ "init_weights_from_safetensors_2": "", "lazy_module_load": null, "lora_nickname": "default", - "lora_path": "/checkpoints/refine_lora.safetensors", + "lora_path": null, "lora_strength": 1.0, "lora_target_modules": null, "ltx2_audio_latent_path": null, @@ -34,7 +34,7 @@ "ltx2_refine_audio_noise_path": null, "ltx2_refine_enabled": false, "ltx2_refine_guidance_scale": 1.0, - "ltx2_refine_lora_path": null, + "ltx2_refine_lora_path": "/checkpoints/refine_lora.safetensors", "ltx2_refine_noise_path": null, "ltx2_refine_num_inference_steps": 3, "ltx2_refine_transformer_path": null, diff --git a/fastvideo/tests/api/test_ltx2_gpu_pool_translation.py b/fastvideo/tests/api/test_ltx2_gpu_pool_translation.py index 77332e9d0..2f51a7772 100644 --- a/fastvideo/tests/api/test_ltx2_gpu_pool_translation.py +++ b/fastvideo/tests/api/test_ltx2_gpu_pool_translation.py @@ -109,9 +109,10 @@ class TestGpuPoolForwardTranslation: def test_refine_upsampler_routed_to_components(self, config) -> None: assert config.pipeline.components.upsampler_weights == ("/models/ltx2-distilled/spatial_upsampler") - def test_empty_refine_lora_becomes_none(self, config) -> None: - # gpu_pool passes "" to keep refine LoRA disabled; typed schema - # treats that as "no LoRA" rather than an empty-string path. + def test_empty_refine_lora_stays_disabled(self, config) -> None: + # gpu_pool passes "" to keep the refine LoRA disabled. None would + # load the checkpoint's default refine LoRA instead. + assert config.pipeline.ltx2.refine.lora_path == "" assert config.pipeline.components.lora_path is None def test_refine_preset_overrides(self, config) -> None: diff --git a/fastvideo/tests/api/test_parser.py b/fastvideo/tests/api/test_parser.py index ff93d9671..60e73b01c 100644 --- a/fastvideo/tests/api/test_parser.py +++ b/fastvideo/tests/api/test_parser.py @@ -187,6 +187,7 @@ def test_load_run_config_supports_yaml_roundtrip(tmp_path) -> None: "use_distilled_sigmas": None, "refine": { "transformer_path": None, + "lora_path": None, "noise_path": None, "audio_noise_path": None, }, diff --git a/fastvideo/tests/contract/test_dreamverse_shape.py b/fastvideo/tests/contract/test_dreamverse_shape.py index 0def46114..9e909d6c4 100644 --- a/fastvideo/tests/contract/test_dreamverse_shape.py +++ b/fastvideo/tests/contract/test_dreamverse_shape.py @@ -90,7 +90,8 @@ class TestDreamverseLoadKwargsShape: config = legacy_from_pretrained_to_config("/models/ltx2", _dreamverse_load_kwargs()) assert isinstance(config.pipeline.components, ComponentConfig) assert (config.pipeline.components.upsampler_weights == "/models/ltx2-refine") - assert config.pipeline.components.lora_path == "/models/ltx2-refine-lora" + assert config.pipeline.ltx2.refine.lora_path == "/models/ltx2-refine-lora" + assert config.pipeline.components.lora_path is None assert config.pipeline.components.config_root == "/models/ltx2-config" def test_torch_compile_kwargs_reach_typed_fields(self):