From c7c2d77fae389d58defe8cd30a7964622b2739aa Mon Sep 17 00:00:00 2001 From: Davids048 Date: Fri, 2 Oct 2026 18:29:37 +0000 Subject: [PATCH] [bugfix]: apply the LTX-2 refine LoRA to the refine transformer The from_pretrained keyword ltx2_refine_lora_path set pipeline.components.lora_path, which reaches FastVideoArgs.lora_path and loads as the main LoRA on the stage-1 transformer. The refine stage reads FastVideoArgs.ltx2_refine_lora_path, which stayed unset. The conversion also turned an empty string into None. An empty string is how callers such as the Dreamverse GPU pool disable the refine LoRA, and None makes LTX2Pipeline load the checkpoint default instead: fastvideo_refine_lora_path in model_index.json is FastVideo/LTX2-Distilled-LoRA for the LTX-2 Diffusers checkpoints. The refine LoRA gets its own field, pipeline.ltx2.refine.lora_path, whose flat name is ltx2_refine_lora_path. None keeps the checkpoint default, and an empty string disables the refine LoRA. The Dreamverse contract test, the Dynamo contract example, and the parity inventory point at the field. Golden changes: kwargs/ltx2_refine_lora_path now records the path on ltx2_refine_lora_path instead of lora_path, and the added case kwargs/ltx2_refine_lora_disabled pins the empty string. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012M91wnVFmPEvJ9h39r5BH7 --- .../inference_schema_parity_inventory.yaml | 4 +- docs/design/server_contracts/dynamo.md | 9 +- .../basic/basic_ltx2_3_distilled_i2v_typed.py | 2 +- fastvideo/api/compat.py | 2 - fastvideo/api/schema.py | 3 + .../pipelines/basic/ltx2/stage_overrides.py | 6 +- fastvideo/tests/api/config_snapshot.py | 3 + .../kwargs/ltx2_refine_lora_disabled.json | 356 ++++++++++++++++++ .../kwargs/ltx2_refine_lora_path.json | 4 +- .../api/test_ltx2_gpu_pool_translation.py | 7 +- fastvideo/tests/api/test_parser.py | 1 + .../tests/contract/test_dreamverse_shape.py | 3 +- 12 files changed, 382 insertions(+), 18 deletions(-) create mode 100644 fastvideo/tests/api/config_snapshot_goldens/kwargs/ltx2_refine_lora_disabled.json diff --git a/docs/design/inference_schema_parity_inventory.yaml b/docs/design/inference_schema_parity_inventory.yaml index 6fa90b7dc..966ca7049 100644 --- a/docs/design/inference_schema_parity_inventory.yaml +++ b/docs/design/inference_schema_parity_inventory.yaml @@ -55,13 +55,13 @@ surfaces: ltx2_vae_tiling: generator.pipeline.vae_tiling refine_enabled: generator.pipeline.preset_overrides.refine.enabled refine_upsampler_path: generator.pipeline.components.upsampler_weights - refine_lora_path: generator.pipeline.components.lora_path + refine_lora_path: generator.pipeline.ltx2.refine.lora_path refine_num_inference_steps: request.stage_overrides.refine.num_inference_steps refine_guidance_scale: request.stage_overrides.refine.guidance_scale refine_add_noise: generator.pipeline.preset_overrides.refine.add_noise ltx2_refine_enabled: generator.pipeline.preset_overrides.refine.enabled ltx2_refine_upsampler_path: generator.pipeline.components.upsampler_weights - ltx2_refine_lora_path: generator.pipeline.components.lora_path + ltx2_refine_lora_path: generator.pipeline.ltx2.refine.lora_path ltx2_refine_num_inference_steps: request.stage_overrides.refine.num_inference_steps ltx2_refine_guidance_scale: request.stage_overrides.refine.guidance_scale ltx2_refine_add_noise: generator.pipeline.preset_overrides.refine.add_noise diff --git a/docs/design/server_contracts/dynamo.md b/docs/design/server_contracts/dynamo.md index 70f70e612..f78b3e21b 100644 --- a/docs/design/server_contracts/dynamo.md +++ b/docs/design/server_contracts/dynamo.md @@ -262,7 +262,8 @@ this adapter can build the config purely from the public typed schema: def build_generator_config(args) -> "GeneratorConfig": from fastvideo.api import ( CompileConfig, ComponentConfig, EngineConfig, GeneratorConfig, - OffloadConfig, ParallelismConfig, PipelineSelection, + LTX2Options, LTX2RefineOptions, OffloadConfig, ParallelismConfig, + PipelineSelection, ) return GeneratorConfig( model_path=args.model_path, @@ -275,10 +276,8 @@ def build_generator_config(args) -> "GeneratorConfig": pipeline=PipelineSelection( workload_type=args.workload or "t2v", preset=args.preset, # e.g. "ltx2_two_stage" - components=ComponentConfig( - upsampler_weights=args.refine_upsampler, - lora_path=args.refine_lora, - ), + components=ComponentConfig(upsampler_weights=args.refine_upsampler), + ltx2=LTX2Options(refine=LTX2RefineOptions(lora_path=args.refine_lora)), ), ) ``` diff --git a/examples/inference/basic/basic_ltx2_3_distilled_i2v_typed.py b/examples/inference/basic/basic_ltx2_3_distilled_i2v_typed.py index f5ac88e0f..b249ee88f 100644 --- a/examples/inference/basic/basic_ltx2_3_distilled_i2v_typed.py +++ b/examples/inference/basic/basic_ltx2_3_distilled_i2v_typed.py @@ -56,7 +56,7 @@ Typed-API mapping (legacy kwarg ↔ typed field) - ``ltx2_vae_tiling`` ↔ ``pipeline.vae_tiling`` - ``ltx2_refine_enabled`` ↔ ``pipeline.preset_overrides["refine"]["enabled"]`` - ``ltx2_refine_upsampler_path`` ↔ ``pipeline.components.upsampler_weights`` -- ``ltx2_refine_lora_path`` ↔ ``pipeline.components.lora_path`` +- ``ltx2_refine_lora_path`` ↔ ``pipeline.ltx2.refine.lora_path`` - ``ltx2_refine_num_inference_steps`` ↔ ``pipeline.preset_overrides["refine"]["num_inference_steps"]`` - ``ltx2_refine_guidance_scale`` ↔ ``pipeline.preset_overrides["refine"]["guidance_scale"]`` - ``ltx2_refine_add_noise`` ↔ ``pipeline.preset_overrides["refine"]["add_noise"]`` diff --git a/fastvideo/api/compat.py b/fastvideo/api/compat.py index 6a3494165..93917b249 100644 --- a/fastvideo/api/compat.py +++ b/fastvideo/api/compat.py @@ -321,10 +321,8 @@ _COMPONENT_OVERRIDE_PREFIXES = { "vae_config.": "vae", } # from_pretrained keywords whose empty-string value means "unset", and the field each one sets. -# ltx2_refine_lora_path sets the main LoRA path. _EMPTY_MEANS_UNSET_KEYWORDS = { "ltx2_refine_upsampler_path": "pipeline.components.upsampler_weights", - "ltx2_refine_lora_path": "pipeline.components.lora_path", } # from_pretrained keywords that set pipeline.preset_overrides.refine.. _LTX2_REFINE_PRESET_KEYWORDS = frozenset({ diff --git a/fastvideo/api/schema.py b/fastvideo/api/schema.py index 4735f96ad..2852451a0 100644 --- a/fastvideo/api/schema.py +++ b/fastvideo/api/schema.py @@ -158,6 +158,9 @@ class LTX2RefineOptions: """Stage-2 refine assets that ``preset_overrides.refine`` and ``components`` do not cover.""" transformer_path: str | None = flat_field("ltx2_refine_transformer_path", None) + lora_path: str | None = flat_field("ltx2_refine_lora_path", None) + """LoRA applied to the refine transformer only. ``None`` uses the checkpoint's default + (``fastvideo_refine_lora_path`` in ``model_index.json``); an empty string disables the refine LoRA.""" noise_path: str | None = flat_field("ltx2_refine_noise_path", None) audio_noise_path: str | None = flat_field("ltx2_refine_audio_noise_path", None) diff --git a/fastvideo/pipelines/basic/ltx2/stage_overrides.py b/fastvideo/pipelines/basic/ltx2/stage_overrides.py index 7e9c0786c..afe25fd12 100644 --- a/fastvideo/pipelines/basic/ltx2/stage_overrides.py +++ b/fastvideo/pipelines/basic/ltx2/stage_overrides.py @@ -6,8 +6,10 @@ * ``stage_overrides.refine`` — per-request knobs (see :class:`LTX2RefineStageOverride`). -Asset paths live on :class:`~fastvideo.api.schema.ComponentConfig` -(``upsampler_weights`` and ``lora_path``). +The refine upsampler path lives on +:class:`~fastvideo.api.schema.ComponentConfig` (``upsampler_weights``), and the +refine LoRA path on :class:`~fastvideo.api.schema.LTX2RefineOptions` +(``pipeline.ltx2.refine.lora_path``). """ from __future__ import annotations diff --git a/fastvideo/tests/api/config_snapshot.py b/fastvideo/tests/api/config_snapshot.py index ed3f48013..44e83830a 100644 --- a/fastvideo/tests/api/config_snapshot.py +++ b/fastvideo/tests/api/config_snapshot.py @@ -354,6 +354,9 @@ def collect_cases() -> list[SnapshotCase]: "ltx2_refine_lora_path": (LTX2, { "ltx2_refine_lora_path": "/checkpoints/refine_lora.safetensors" }), + "ltx2_refine_lora_disabled": (LTX2, { + "ltx2_refine_lora_path": "" + }), "pipeline_config_json_path": ("FastVideo/FastHunyuan-diffusers", { "pipeline_config": pipeline_json }), diff --git a/fastvideo/tests/api/config_snapshot_goldens/kwargs/ltx2_refine_lora_disabled.json b/fastvideo/tests/api/config_snapshot_goldens/kwargs/ltx2_refine_lora_disabled.json new file mode 100644 index 000000000..4d4fe2343 --- /dev/null +++ b/fastvideo/tests/api/config_snapshot_goldens/kwargs/ltx2_refine_lora_disabled.json @@ -0,0 +1,356 @@ +{ + "fastvideo_args": { + "VSA_sparsity": 0.0, + "VSA_tile_size": 256, + "attention_backend": null, + "boundary_ratio": 0.875, + "disable_autocast": false, + "dist_timeout": null, + "distributed_executor_backend": "mp", + "dit_cpu_offload": true, + "dit_layerwise_offload": true, + "enable_stage_verification": true, + "enable_torch_compile": false, + "enable_torch_compile_audio_vae": false, + "enable_torch_compile_text_encoder": false, + "enable_torch_compile_vae": false, + "h3_sequential_load": null, + "hsdp_replicate_dim": 1, + "hsdp_shard_dim": 1, + "image_encoder_cpu_offload": true, + "inference_mode": true, + "inference_torch_compile": false, + "init_weights_from_safetensors": "", + "init_weights_from_safetensors_2": "", + "lazy_module_load": null, + "lora_nickname": "default", + "lora_path": null, + "lora_strength": 1.0, + "lora_target_modules": null, + "ltx2_audio_latent_path": null, + "ltx2_initial_latent_path": null, + "ltx2_legacy_native_noise_order": false, + "ltx2_refine_add_noise": true, + "ltx2_refine_audio_noise_path": null, + "ltx2_refine_enabled": false, + "ltx2_refine_guidance_scale": 1.0, + "ltx2_refine_lora_path": "", + "ltx2_refine_noise_path": null, + "ltx2_refine_num_inference_steps": 3, + "ltx2_refine_transformer_path": null, + "ltx2_refine_upsampler_path": null, + "ltx2_use_distilled_sigmas": true, + "ltx2_vae_spatial_tile_overlap_in_pixels": null, + "ltx2_vae_spatial_tile_size_in_pixels": null, + "ltx2_vae_temporal_tile_overlap_in_frames": null, + "ltx2_vae_temporal_tile_size_in_frames": null, + "ltx2_vae_tiling": null, + "master_port": null, + "moba_config": {}, + "moba_config_path": null, + "mode": "inference", + "model_loaded": { + "transformer": true, + "upsampler": true, + "vae": true + }, + "model_path": "FastVideo/LTX2-Distilled-Diffusers", + "model_paths": {}, + "num_gpus": 1, + "output_type": "pil", + "override_pipeline_cls_name": null, + "override_text_encoder_quant": null, + "override_text_encoder_safetensors": null, + "override_transformer_cls_name": null, + "pin_cpu_memory": true, + "preprocess_config": null, + "prompt_txt": null, + "ray_placement_group": null, + "ray_runtime_env": null, + "refine_add_noise": null, + "refine_audio_noise_path": null, + "refine_enabled": null, + "refine_guidance_scale": null, + "refine_lora_path": null, + "refine_noise_path": null, + "refine_num_inference_steps": null, + "refine_transformer_path": null, + "refine_upsampler_path": null, + "revision": null, + "sp_size": 1, + "taeh3_checkpoint": null, + "taeh3_chunk_size": 5, + "text_encoder_cpu_offload": true, + "torch_compile_kwargs": {}, + "torch_compile_kwargs_audio_vae": {}, + "torch_compile_kwargs_dit": {}, + "torch_compile_kwargs_text_encoder": {}, + "torch_compile_kwargs_vae": {}, + "tp_size": 1, + "transformer_quant": null, + "trust_remote_code": false, + "use_fsdp_inference": false, + "vae_cpu_offload": true, + "vae_parallel_decode": false, + "vae_parallel_decode_strategy": "gather", + "vae_parallel_encode": false, + "video_decode_backend": "h3-vae", + "workload_type": "t2v" + }, + "pipeline_config": { + "audio_decoder_config": { + "_resolved_attention_backend": null, + "arch_config": { + "architectures": [ + "LTX2AudioDecoder" + ], + "stacked_params_mapping": [] + } + }, + "audio_decoder_precision": "bf16", + "boundary_ratio": null, + "disable_autocast": false, + "dit_config": { + "_resolved_attention_backend": null, + "arch_config": { + "_supported_attention_backends": [ + 3, + 1, + 2, + 7, + 10, + 4, + 5, + 6, + 11, + 12 + ], + "apply_gated_attention": false, + "attention_head_dim": 128, + "attention_type": "default", + "audio_attention_head_dim": 64, + "audio_connector_attention_head_dim": 128, + "audio_connector_num_attention_heads": 30, + "audio_connector_num_layers": 2, + "audio_cross_attention_dim": 2048, + "audio_in_channels": 128, + "audio_num_attention_heads": 32, + "audio_out_channels": 128, + "audio_positional_embedding_max_pos": [ + 20 + ], + "av_ca_timestep_scale_multiplier": 1, + "boundary_ratio": null, + "caption_channels": 3840, + "caption_proj_before_connector": false, + "caption_proj_input_norm": true, + "caption_projection_first_linear": true, + "caption_projection_second_linear": true, + "cast_prompt_embeds_to_dit_dtype": false, + "connector_attention_head_dim": 128, + "connector_num_attention_heads": 30, + "connector_num_layers": 2, + "cross_attention_adaln": false, + "cross_attention_dim": 4096, + "double_precision_rope": true, + "exclude_lora_layers": [], + "hidden_size": 0, + "in_channels": 128, + "norm_eps": 1e-06, + "num_attention_heads": 32, + "num_channels_latents": 128, + "num_layers": 48, + "out_channels": 128, + "patch_size": [ + 1, + 1, + 1 + ], + "positional_embedding_max_pos": [ + 20, + 2048, + 2048 + ], + "positional_embedding_theta": 10000.0, + "rope_type": "split", + "stacked_params_mapping": [], + "stg_block_idx": 29, + "timestep_scale_multiplier": 1000, + "use_middle_indices_grid": true + }, + "prefix": "ltx2", + "quant_config": null + }, + "dit_precision": "bf16", + "dmd_denoising_steps": null, + "embedded_cfg_scale": 6.0, + "flow_shift": null, + "flow_shift_sr": null, + "image_encoder_config": { + "_resolved_attention_backend": null, + "arch_config": { + "_supported_attention_backends": [ + 1, + 2 + ], + "architectures": [], + "output_hidden_states": false, + "stacked_params_mapping": [], + "use_return_dict": true + }, + "lora_config": null, + "prefix": "", + "quant_config": null + }, + "image_encoder_configs": null, + "image_encoder_precision": "fp32", + "image_encoder_precisions": null, + "is_causal": false, + "lucy_edit_task": false, + "model_path": "FastVideo/LTX2-Distilled-Diffusers", + "pipeline_config_path": null, + "postprocess_text_funcs": [ + "fastvideo.pipelines.basic.ltx2.pipeline_configs.ltx2_postprocess_text" + ], + "preprocess_text_funcs": [ + "fastvideo.configs.pipelines.base.preprocess_text" + ], + "scheduler_step_in_fp32": false, + "text_encoder_configs": [ + { + "_resolved_attention_backend": null, + "arch_config": { + "_supported_attention_backends": [ + 1, + 2 + ], + "architectures": [ + "LTX2GemmaTextEncoderModel" + ], + "audio_connector_attention_head_dim": null, + "audio_connector_num_attention_heads": null, + "audio_connector_num_layers": null, + "audio_feature_extractor_out_features": null, + "caption_proj_before_connector": false, + "caption_proj_input_norm": true, + "caption_projection_first_linear": true, + "caption_projection_second_linear": true, + "connector_apply_gated_attention": false, + "connector_attention_head_dim": 128, + "connector_double_precision_rope": false, + "connector_num_attention_heads": 30, + "connector_num_layers": 2, + "connector_num_learnable_registers": 128, + "connector_positional_embedding_max_pos": [ + 4096 + ], + "connector_positional_embedding_theta": 10000.0, + "connector_rope_type": "split", + "decoder_start_token_id": 0, + "eos_token_id": 2, + "feature_extractor_in_features": 188160, + "feature_extractor_out_features": 3840, + "gemma_dtype": "bfloat16", + "gemma_model_path": "", + "hidden_size": 3840, + "hidden_state_skip_layer": 0, + "num_attention_heads": 30, + "num_hidden_layers": 48, + "output_hidden_states": true, + "output_past": true, + "pad_token_id": 0, + "padding_side": "left", + "require_processor": false, + "scalable_attention": true, + "stacked_params_mapping": [], + "text_len": 1024, + "tie_word_embeddings": false, + "tokenizer_kwargs": { + "max_length": 1024, + "padding": "max_length", + "return_tensors": "pt", + "truncation": true + }, + "use_return_dict": true, + "video_feature_extractor_out_features": null, + "vocab_size": 0 + }, + "chat_template_enable_thinking": false, + "is_chat_model": false, + "lora_config": null, + "prefix": "ltx2_gemma", + "quant_config": null, + "treat_empty_as_dot": false + } + ], + "text_encoder_precisions": [ + "bf16" + ], + "ti2v_task": false, + "upsampler_config": { + "_resolved_attention_backend": null, + "arch_config": { + "stacked_params_mapping": [] + } + }, + "upsampler_precision": "fp32", + "vae_config": { + "_resolved_attention_backend": null, + "arch_config": { + "_class_name": "CausalVideoAutoencoder", + "causal_decoder": false, + "decoder_blocks": [], + "decoder_spatial_padding_mode": "reflect", + "dims": 3, + "encoder_blocks": [], + "encoder_spatial_padding_mode": "zeros", + "in_channels": 3, + "latent_channels": 128, + "latent_log_var": "uniform", + "norm_layer": "pixel_norm", + "normalize_latent_channels": false, + "out_channels": 3, + "patch_size": 4, + "scaling_factor": 1.0, + "spatial_compression_ratio": 32, + "stacked_params_mapping": [], + "temporal_compression_ratio": 8, + "timestep_conditioning": true, + "use_quant_conv": false, + "z_dim": 128 + }, + "blend_num_frames": 4, + "load_decoder": true, + "load_encoder": false, + "ltx2_spatial_tile_overlap_in_pixels": 64, + "ltx2_spatial_tile_size_in_pixels": 512, + "ltx2_temporal_tile_overlap_in_frames": 24, + "ltx2_temporal_tile_size_in_frames": 64, + "tile_sample_min_height": 256, + "tile_sample_min_num_frames": 16, + "tile_sample_min_width": 256, + "tile_sample_stride_height": 192, + "tile_sample_stride_num_frames": 12, + "tile_sample_stride_width": 192, + "use_parallel_tiling": true, + "use_temporal_scaling_frames": true, + "use_temporal_tiling": true, + "use_tiling": true + }, + "vae_decode_precision": null, + "vae_precision": "bf16", + "vae_sp": false, + "vae_tiling": true, + "vocoder_config": { + "_resolved_attention_backend": null, + "arch_config": { + "architectures": [ + "LTX2Vocoder" + ], + "stacked_params_mapping": [] + } + }, + "vocoder_precision": "bf16" + }, + "pipeline_config_class": "fastvideo.pipelines.basic.ltx2.pipeline_configs.LTX2T2VConfig" +} diff --git a/fastvideo/tests/api/config_snapshot_goldens/kwargs/ltx2_refine_lora_path.json b/fastvideo/tests/api/config_snapshot_goldens/kwargs/ltx2_refine_lora_path.json index 4da00b49f..485b70e1a 100644 --- a/fastvideo/tests/api/config_snapshot_goldens/kwargs/ltx2_refine_lora_path.json +++ b/fastvideo/tests/api/config_snapshot_goldens/kwargs/ltx2_refine_lora_path.json @@ -24,7 +24,7 @@ "init_weights_from_safetensors_2": "", "lazy_module_load": null, "lora_nickname": "default", - "lora_path": "/checkpoints/refine_lora.safetensors", + "lora_path": null, "lora_strength": 1.0, "lora_target_modules": null, "ltx2_audio_latent_path": null, @@ -34,7 +34,7 @@ "ltx2_refine_audio_noise_path": null, "ltx2_refine_enabled": false, "ltx2_refine_guidance_scale": 1.0, - "ltx2_refine_lora_path": null, + "ltx2_refine_lora_path": "/checkpoints/refine_lora.safetensors", "ltx2_refine_noise_path": null, "ltx2_refine_num_inference_steps": 3, "ltx2_refine_transformer_path": null, diff --git a/fastvideo/tests/api/test_ltx2_gpu_pool_translation.py b/fastvideo/tests/api/test_ltx2_gpu_pool_translation.py index 77332e9d0..2f51a7772 100644 --- a/fastvideo/tests/api/test_ltx2_gpu_pool_translation.py +++ b/fastvideo/tests/api/test_ltx2_gpu_pool_translation.py @@ -109,9 +109,10 @@ class TestGpuPoolForwardTranslation: def test_refine_upsampler_routed_to_components(self, config) -> None: assert config.pipeline.components.upsampler_weights == ("/models/ltx2-distilled/spatial_upsampler") - def test_empty_refine_lora_becomes_none(self, config) -> None: - # gpu_pool passes "" to keep refine LoRA disabled; typed schema - # treats that as "no LoRA" rather than an empty-string path. + def test_empty_refine_lora_stays_disabled(self, config) -> None: + # gpu_pool passes "" to keep the refine LoRA disabled. None would + # load the checkpoint's default refine LoRA instead. + assert config.pipeline.ltx2.refine.lora_path == "" assert config.pipeline.components.lora_path is None def test_refine_preset_overrides(self, config) -> None: diff --git a/fastvideo/tests/api/test_parser.py b/fastvideo/tests/api/test_parser.py index ff93d9671..60e73b01c 100644 --- a/fastvideo/tests/api/test_parser.py +++ b/fastvideo/tests/api/test_parser.py @@ -187,6 +187,7 @@ def test_load_run_config_supports_yaml_roundtrip(tmp_path) -> None: "use_distilled_sigmas": None, "refine": { "transformer_path": None, + "lora_path": None, "noise_path": None, "audio_noise_path": None, }, diff --git a/fastvideo/tests/contract/test_dreamverse_shape.py b/fastvideo/tests/contract/test_dreamverse_shape.py index 0def46114..9e909d6c4 100644 --- a/fastvideo/tests/contract/test_dreamverse_shape.py +++ b/fastvideo/tests/contract/test_dreamverse_shape.py @@ -90,7 +90,8 @@ class TestDreamverseLoadKwargsShape: config = legacy_from_pretrained_to_config("/models/ltx2", _dreamverse_load_kwargs()) assert isinstance(config.pipeline.components, ComponentConfig) assert (config.pipeline.components.upsampler_weights == "/models/ltx2-refine") - assert config.pipeline.components.lora_path == "/models/ltx2-refine-lora" + assert config.pipeline.ltx2.refine.lora_path == "/models/ltx2-refine-lora" + assert config.pipeline.components.lora_path is None assert config.pipeline.components.config_root == "/models/ltx2-config" def test_torch_compile_kwargs_reach_typed_fields(self):