Compare commits

...
11 Commits
Author SHA1 Message Date
SolitaryThinker a8dddfaa16 missing file 2026-02-10 08:33:00 +00:00
Will Lin 7210c68f1b lint 2026-02-10 00:30:03 -08:00
SolitaryThinker 5602dc1bad revert 2026-02-10 08:22:05 +00:00
SolitaryThinker ac4bc4ab84 uipdate 2026-02-10 07:58:43 +00:00
Matthew Noto bee27f9f74 Merge branch 'main' into ltx-base 2026-02-09 17:51:37 -08:00
Will Lin dff0ea401a update 2026-02-08 00:22:21 -08:00
Davids048 becd379f58 Split LTX2 mappings and add registry coverage.
Detail:

- Merged LTX2 sampling behavior into the global registry and removed the ambiguous local converted-path default.
- Replaced single LTX2 sampling mapping with explicit model-ID mappings:
    - Lightricks/LTX-2 -> LTX2BaseSamplingParam
    - FastVideo/LTX2-base -> LTX2BaseSamplingParam
    - FastVideo/LTX2-Distilled-Diffusers -> LTX2DistilledSamplingParam
- Kept LTX2T2VConfig as the pipeline config for all explicitly mapped LTX2 IDs.
- Removed implicit mapping for converted/ltx2_diffusers to avoid guessing base vs distilled for user-local
  conversions.
- Added focused local tests at tests/local_tests/test_ltx2_registry.py for:
    - exact base/distilled sampling resolution,
    - pipeline config resolution,
    - no fallback behavior for ambiguous local converted paths.

Assumptions:

- Canonical rename/ID intent:
    - “base” names map to LTX2BaseSamplingParam.
    - “Distilled” names map to LTX2DistilledSamplingParam.
- converted/ltx2_diffusers is intentionally ambiguous across users and must not be auto-assigned.
- Unknown/non-canonical names containing “LTX”/“LTX2” (but not matching explicit registered IDs) should not auto-
  resolve to base or distilled.
    - Sampling resolver returns None (caller falls back to generic defaults/user overrides).
    - Pipeline config lookup raises a “No match found” error.

Notes:

- This change prioritizes explicitness over convenience: only predetermined, canonical model IDs get LTX2-specific
  defaults; everything else requires user intent.
2026-02-05 20:28:46 -08:00
Davids048 1ed7d7e1b0 Add gemma tokenizer to LTX2 conversion script.
- Also clean up for PR.
2026-02-05 18:53:22 -08:00
Davids048 d9fabcc5ef Add some annotations. 2026-02-05 18:53:22 -08:00
Davids048 9db48498de Update quality test script. 2026-02-05 18:53:22 -08:00
Davids048 1cd7038315 Add LTX2 base model. 2026-02-05 18:53:12 -08:00
7 changed files with 115 additions and 126 deletions
+1 -1
View File
@@ -1 +1 @@
@AGENTS.md
@AGENTS.md
+7 -2
View File
@@ -1,5 +1,6 @@
from fastvideo import VideoGenerator
PROMPT = (
"A warm sunny backyard. The camera starts in a tight cinematic close-up "
"of a woman and a man in their 30s, facing each other with serious "
@@ -16,16 +17,20 @@ PROMPT = (
def main() -> None:
# Uses FastVideo default sampling settings for LTX2 base.
generator = VideoGenerator.from_pretrained(
"FastVideo/LTX2-Distilled-Diffusers",
"Davids048/LTX2-Base-Diffusers",
num_gpus=1,
)
output_path = "outputs_video/ltx2_basic/output_ltx2_distilled_t2v.mp4"
output_path = "outputs_video/ltx2_basic/output_ltx2_base_t2v_1088_1920_1.1.mp4"
generator.generate_video(
prompt=PROMPT,
output_path=output_path,
save_video=True,
num_frames=121,
height=1088,
width=1920,
)
generator.shutdown()
@@ -1,54 +0,0 @@
from fastvideo import VideoGenerator
PROMPT = (
"A warm sunny backyard. The camera starts in a tight cinematic close-up "
"of a woman and a man in their 30s, facing each other with serious "
"expressions. The woman, emotional and dramatic, says softly, \"That's "
"it... Dad's lost it. And we've lost Dad.\" The man exhales, slightly "
"annoyed: \"Stop being so dramatic, Jess.\" A beat. He glances aside, "
"then mutters defensively, \"He's just having fun.\" The camera slowly "
"pans right, revealing the grandfather in the garden wearing enormous "
"butterfly wings, waving his arms in the air like he's trying to take "
"off. He shouts, \"Wheeeew!\" as he flaps his wings with full commitment. "
"The woman covers her face, on the verge of tears. The tone is deadpan, "
"absurd, and quietly tragic."
)
official_negative_prompt = (
"blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, "
"grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, "
"deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, "
"wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of "
"field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent "
"lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny "
"valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, "
"mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, "
"off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward "
"pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, "
"inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts."
)
def main() -> None:
# Uses FastVideo default sampling settings for LTX2 base.
generator = VideoGenerator.from_pretrained(
"Davids048/LTX2-Base-Diffusers",
num_gpus=1,
)
output_path = "outputs_video/ltx2_basic/output_ltx2_base_t2v_1088_1920_1.0.mp4"
generator.generate_video(
prompt=PROMPT,
output_path=output_path,
save_video=True,
num_frames=121,
height=1088,
width=1920,
fps=24,
negative_prompt=official_negative_prompt,
guidance_scale=3.0,
)
generator.shutdown()
if __name__ == "__main__":
main()
@@ -0,0 +1,34 @@
from fastvideo import VideoGenerator
PROMPT = (
"A warm sunny backyard. The camera starts in a tight cinematic close-up "
"of a woman and a man in their 30s, facing each other with serious "
"expressions. The woman, emotional and dramatic, says softly, \"That's "
"it... Dad's lost it. And we've lost Dad.\" The man exhales, slightly "
"annoyed: \"Stop being so dramatic, Jess.\" A beat. He glances aside, "
"then mutters defensively, \"He's just having fun.\" The camera slowly "
"pans right, revealing the grandfather in the garden wearing enormous "
"butterfly wings, waving his arms in the air like he's trying to take "
"off. He shouts, \"Wheeeew!\" as he flaps his wings with full commitment. "
"The woman covers her face, on the verge of tears. The tone is deadpan, "
"absurd, and quietly tragic."
)
def main() -> None:
generator = VideoGenerator.from_pretrained(
"FastVideo/LTX2-Distilled-Diffusers",
num_gpus=1,
)
output_path = "outputs_video/ltx2_basic/output_ltx2_distilled_t2v.mp4"
generator.generate_video(
prompt=PROMPT,
output_path=output_path,
save_video=True,
)
generator.shutdown()
if __name__ == "__main__":
main()
+4 -3
View File
@@ -249,8 +249,8 @@ def _register_configs() -> None:
"FastVideo/LTX2-base",
],
model_detectors=[
lambda path: ("ltx2" in path.lower() or "ltx-2" in path.lower())
and "distilled" not in path.lower(),
lambda path: ("ltx2" in path.lower() or "ltx-2" in path.lower()) and
"distilled" not in path.lower(),
],
)
# LTX-2 (distilled)
@@ -261,7 +261,8 @@ def _register_configs() -> None:
"FastVideo/LTX2-Distilled-Diffusers",
],
model_detectors=[
lambda path: ("ltx2" in path.lower() or "ltx-2" in path.lower()) and "distilled" in path.lower(),
lambda path: ("ltx2" in path.lower() or "ltx-2" in path.lower()) and
"distilled" in path.lower(),
],
)
+17 -16
View File
@@ -280,18 +280,19 @@ class LTX2TrainingPipeline(TrainingPipeline):
conditioning_mask = None
first_frame_p = self.training_args.ltx2_first_frame_conditioning_p
if first_frame_p > 0:
if torch.rand(1, generator=self.noise_random_generator).item(
) < first_frame_p:
conditioning_mask = torch.zeros(
(batch_size, 1, latents.shape[2], latents.shape[3],
latents.shape[4]),
dtype=torch.bool,
device=latents.device,
)
conditioning_mask[:, :, 0:1] = True
noisy_model_input = torch.where(conditioning_mask, latents,
noisy_model_input)
if (first_frame_p > 0 and torch.rand(
1,
generator=self.noise_random_generator,
).item() < first_frame_p):
conditioning_mask = torch.zeros(
(batch_size, 1, latents.shape[2], latents.shape[3],
latents.shape[4]),
dtype=torch.bool,
device=latents.device,
)
conditioning_mask[:, :, 0:1] = True
noisy_model_input = torch.where(conditioning_mask, latents,
noisy_model_input)
if conditioning_mask is None:
mask_patch = torch.zeros(
@@ -398,9 +399,9 @@ class LTX2TrainingPipeline(TrainingPipeline):
with self.tracker.timed("timing/forward_backward"), set_forward_context(
current_timestep=training_batch.current_timestep,
attn_metadata=training_batch.attn_metadata):
with torch.autocast("cuda", dtype=training_batch.latents.dtype):
with torch.autograd.set_detect_anomaly(True):
outputs = self.transformer(**input_kwargs)
with torch.autocast("cuda", dtype=training_batch.latents.dtype
), torch.autograd.set_detect_anomaly(True):
outputs = self.transformer(**input_kwargs)
if isinstance(outputs, tuple):
video_denoised, audio_denoised = outputs
else:
@@ -422,7 +423,7 @@ class LTX2TrainingPipeline(TrainingPipeline):
audio_loss = (audio_pred_velocity.float() -
audio_target.float())**2
loss = loss + audio_loss.mean()
logger.info("Audio loss: {}".format(audio_loss.mean()))
logger.info("Audio loss: %s", audio_loss.mean())
else:
logger.warning("Audio denoised is None")
+52 -50
View File
@@ -888,59 +888,61 @@ class TrainingPipeline(LoRAPipeline, ABC):
# Only sp_group leaders (rank_in_sp_group == 0) need to send their
# results to global rank 0
if self.rank_in_sp_group == 0:
if self.global_rank == 0:
# Global rank 0 collects results from all sp_group leaders
all_videos = step_videos # Start with own results
all_captions = step_captions
if self.rank_in_sp_group == 0 and self.global_rank == 0:
# Global rank 0 collects results from all sp_group leaders
all_videos = step_videos # Start with own results
all_captions = step_captions
# Receive from other sp_group leaders
for sp_group_idx in range(1, num_sp_groups):
src_rank = sp_group_idx * self.sp_world_size # Global rank of other sp_group leaders
recv_videos = world_group.recv_object(src=src_rank)
recv_captions = world_group.recv_object(src=src_rank)
all_videos.extend(recv_videos)
all_captions.extend(recv_captions)
# Receive from other sp_group leaders
for sp_group_idx in range(1, num_sp_groups):
src_rank = sp_group_idx * self.sp_world_size # Global rank of other sp_group leaders
recv_videos = world_group.recv_object(src=src_rank)
recv_captions = world_group.recv_object(src=src_rank)
all_videos.extend(recv_videos)
all_captions.extend(recv_captions)
video_filenames = []
for i, (video, caption) in enumerate(
zip(all_videos, all_captions, strict=True)):
os.makedirs(training_args.output_dir, exist_ok=True)
filename = os.path.join(
training_args.output_dir,
f"validation_step_{global_step}_inference_steps_{num_inference_steps}_video_{i}.mp4"
)
imageio.mimsave(filename, video, fps=sampling_param.fps)
# Mux audio if available
audio = output_batch.extra.get("audio")
audio_sample_rate = output_batch.extra.get(
"audio_sample_rate")
if audio is not None and audio_sample_rate is not None:
if not self._mux_audio(filename, audio,
audio_sample_rate):
logger.warning(
"Audio mux failed for validation video %s; saved video without audio.",
filename)
video_filenames.append(filename)
video_filenames = []
for i, (video, caption) in enumerate(
zip(all_videos, all_captions, strict=True)):
os.makedirs(training_args.output_dir, exist_ok=True)
filename = os.path.join(
training_args.output_dir,
f"validation_step_{global_step}_inference_steps_{num_inference_steps}_video_{i}.mp4"
)
imageio.mimsave(filename, video, fps=sampling_param.fps)
# Mux audio if available
audio = output_batch.extra.get("audio")
audio_sample_rate = output_batch.extra.get(
"audio_sample_rate")
if (audio is not None and audio_sample_rate is not None
and not self._mux_audio(
filename,
audio,
audio_sample_rate,
)):
logger.warning(
"Audio mux failed for validation video %s; saved video without audio.",
filename)
video_filenames.append(filename)
artifacts = []
for filename, caption in zip(video_filenames,
all_captions,
strict=True):
video_artifact = self.tracker.video(filename,
caption=caption)
if video_artifact is not None:
artifacts.append(video_artifact)
if artifacts:
logs = {
f"validation_videos_{num_inference_steps}_steps":
artifacts
}
self.tracker.log_artifacts(logs, global_step)
else:
# Other sp_group leaders send their results to global rank 0
world_group.send_object(step_videos, dst=0)
world_group.send_object(step_captions, dst=0)
artifacts = []
for filename, caption in zip(video_filenames,
all_captions,
strict=True):
video_artifact = self.tracker.video(filename,
caption=caption)
if video_artifact is not None:
artifacts.append(video_artifact)
if artifacts:
logs = {
f"validation_videos_{num_inference_steps}_steps":
artifacts
}
self.tracker.log_artifacts(logs, global_step)
elif self.rank_in_sp_group == 0:
# Other sp_group leaders send their results to global rank 0
world_group.send_object(step_videos, dst=0)
world_group.send_object(step_captions, dst=0)
# Re-enable gradients for training
training_args.inference_mode = False