diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..43c994c2d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/examples/inference/basic/basic_ltx2.py b/examples/inference/basic/basic_ltx2.py index d8acd56bc..b053e9364 100644 --- a/examples/inference/basic/basic_ltx2.py +++ b/examples/inference/basic/basic_ltx2.py @@ -1,5 +1,6 @@ from fastvideo import VideoGenerator + PROMPT = ( "A warm sunny backyard. The camera starts in a tight cinematic close-up " "of a woman and a man in their 30s, facing each other with serious " @@ -16,16 +17,20 @@ PROMPT = ( def main() -> None: + # Uses FastVideo default sampling settings for LTX2 base. generator = VideoGenerator.from_pretrained( - "FastVideo/LTX2-Distilled-Diffusers", + "Davids048/LTX2-Base-Diffusers", num_gpus=1, ) - output_path = "outputs_video/ltx2_basic/output_ltx2_distilled_t2v.mp4" + output_path = "outputs_video/ltx2_basic/output_ltx2_base_t2v_1088_1920_1.1.mp4" generator.generate_video( prompt=PROMPT, output_path=output_path, save_video=True, + num_frames=121, + height=1088, + width=1920, ) generator.shutdown() diff --git a/examples/inference/basic/basic_ltx2_base.py b/examples/inference/basic/basic_ltx2_base.py deleted file mode 100644 index 69de7de54..000000000 --- a/examples/inference/basic/basic_ltx2_base.py +++ /dev/null @@ -1,54 +0,0 @@ -from fastvideo import VideoGenerator - - -PROMPT = ( - "A warm sunny backyard. The camera starts in a tight cinematic close-up " - "of a woman and a man in their 30s, facing each other with serious " - "expressions. The woman, emotional and dramatic, says softly, \"That's " - "it... Dad's lost it. And we've lost Dad.\" The man exhales, slightly " - "annoyed: \"Stop being so dramatic, Jess.\" A beat. He glances aside, " - "then mutters defensively, \"He's just having fun.\" The camera slowly " - "pans right, revealing the grandfather in the garden wearing enormous " - "butterfly wings, waving his arms in the air like he's trying to take " - "off. He shouts, \"Wheeeew!\" as he flaps his wings with full commitment. " - "The woman covers her face, on the verge of tears. The tone is deadpan, " - "absurd, and quietly tragic." -) - -official_negative_prompt = ( - "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " - "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " - "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " - "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " - "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " - "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " - "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " - "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " - "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " - "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " - "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." -) -def main() -> None: - # Uses FastVideo default sampling settings for LTX2 base. - generator = VideoGenerator.from_pretrained( - "Davids048/LTX2-Base-Diffusers", - num_gpus=1, - ) - - output_path = "outputs_video/ltx2_basic/output_ltx2_base_t2v_1088_1920_1.0.mp4" - generator.generate_video( - prompt=PROMPT, - output_path=output_path, - save_video=True, - num_frames=121, - height=1088, - width=1920, - fps=24, - negative_prompt=official_negative_prompt, - guidance_scale=3.0, - ) - generator.shutdown() - - -if __name__ == "__main__": - main() diff --git a/examples/inference/basic/basic_ltx2_distilled.py b/examples/inference/basic/basic_ltx2_distilled.py new file mode 100644 index 000000000..d8acd56bc --- /dev/null +++ b/examples/inference/basic/basic_ltx2_distilled.py @@ -0,0 +1,34 @@ +from fastvideo import VideoGenerator + +PROMPT = ( + "A warm sunny backyard. The camera starts in a tight cinematic close-up " + "of a woman and a man in their 30s, facing each other with serious " + "expressions. The woman, emotional and dramatic, says softly, \"That's " + "it... Dad's lost it. And we've lost Dad.\" The man exhales, slightly " + "annoyed: \"Stop being so dramatic, Jess.\" A beat. He glances aside, " + "then mutters defensively, \"He's just having fun.\" The camera slowly " + "pans right, revealing the grandfather in the garden wearing enormous " + "butterfly wings, waving his arms in the air like he's trying to take " + "off. He shouts, \"Wheeeew!\" as he flaps his wings with full commitment. " + "The woman covers her face, on the verge of tears. The tone is deadpan, " + "absurd, and quietly tragic." +) + + +def main() -> None: + generator = VideoGenerator.from_pretrained( + "FastVideo/LTX2-Distilled-Diffusers", + num_gpus=1, + ) + + output_path = "outputs_video/ltx2_basic/output_ltx2_distilled_t2v.mp4" + generator.generate_video( + prompt=PROMPT, + output_path=output_path, + save_video=True, + ) + generator.shutdown() + + +if __name__ == "__main__": + main() diff --git a/fastvideo/registry.py b/fastvideo/registry.py index 9b08661a6..aa1449233 100644 --- a/fastvideo/registry.py +++ b/fastvideo/registry.py @@ -250,6 +250,7 @@ def _register_configs() -> None: hf_model_paths=[ "Lightricks/LTX-2", "FastVideo/LTX2-base", + "FastVideo/LTX2-Diffusers", ], model_detectors=[ lambda path: ("ltx2" in path.lower() or "ltx-2" in path.lower()) and diff --git a/fastvideo/training/ltx2_training_pipeline.py b/fastvideo/training/ltx2_training_pipeline.py index 31e3c2f1d..3d77aee8a 100644 --- a/fastvideo/training/ltx2_training_pipeline.py +++ b/fastvideo/training/ltx2_training_pipeline.py @@ -280,9 +280,10 @@ class LTX2TrainingPipeline(TrainingPipeline): conditioning_mask = None first_frame_p = self.training_args.ltx2_first_frame_conditioning_p - if first_frame_p > 0 and torch.rand( + if (first_frame_p > 0 and torch.rand( 1, - generator=self.noise_random_generator).item() < first_frame_p: + generator=self.noise_random_generator, + ).item() < first_frame_p): conditioning_mask = torch.zeros( (batch_size, 1, latents.shape[2], latents.shape[3], latents.shape[4]), diff --git a/fastvideo/training/training_pipeline.py b/fastvideo/training/training_pipeline.py index dc814fa59..755943b00 100644 --- a/fastvideo/training/training_pipeline.py +++ b/fastvideo/training/training_pipeline.py @@ -888,58 +888,61 @@ class TrainingPipeline(LoRAPipeline, ABC): # Only sp_group leaders (rank_in_sp_group == 0) need to send their # results to global rank 0 - if self.rank_in_sp_group == 0: - if self.global_rank == 0: - # Global rank 0 collects results from all sp_group leaders - all_videos = step_videos # Start with own results - all_captions = step_captions + if self.rank_in_sp_group == 0 and self.global_rank == 0: + # Global rank 0 collects results from all sp_group leaders + all_videos = step_videos # Start with own results + all_captions = step_captions - # Receive from other sp_group leaders - for sp_group_idx in range(1, num_sp_groups): - src_rank = sp_group_idx * self.sp_world_size # Global rank of other sp_group leaders - recv_videos = world_group.recv_object(src=src_rank) - recv_captions = world_group.recv_object(src=src_rank) - all_videos.extend(recv_videos) - all_captions.extend(recv_captions) + # Receive from other sp_group leaders + for sp_group_idx in range(1, num_sp_groups): + src_rank = sp_group_idx * self.sp_world_size # Global rank of other sp_group leaders + recv_videos = world_group.recv_object(src=src_rank) + recv_captions = world_group.recv_object(src=src_rank) + all_videos.extend(recv_videos) + all_captions.extend(recv_captions) - video_filenames = [] - for i, (video, caption) in enumerate( - zip(all_videos, all_captions, strict=True)): - os.makedirs(training_args.output_dir, exist_ok=True) - filename = os.path.join( - training_args.output_dir, - f"validation_step_{global_step}_inference_steps_{num_inference_steps}_video_{i}.mp4" - ) - imageio.mimsave(filename, video, fps=sampling_param.fps) - # Mux audio if available - audio = output_batch.extra.get("audio") - audio_sample_rate = output_batch.extra.get( - "audio_sample_rate") - if audio is not None and audio_sample_rate is not None and not self._mux_audio( - filename, audio, audio_sample_rate): - logger.warning( - "Audio mux failed for validation video %s; saved video without audio.", - filename) - video_filenames.append(filename) + video_filenames = [] + for i, (video, caption) in enumerate( + zip(all_videos, all_captions, strict=True)): + os.makedirs(training_args.output_dir, exist_ok=True) + filename = os.path.join( + training_args.output_dir, + f"validation_step_{global_step}_inference_steps_{num_inference_steps}_video_{i}.mp4" + ) + imageio.mimsave(filename, video, fps=sampling_param.fps) + # Mux audio if available + audio = output_batch.extra.get("audio") + audio_sample_rate = output_batch.extra.get( + "audio_sample_rate") + if (audio is not None and audio_sample_rate is not None + and not self._mux_audio( + filename, + audio, + audio_sample_rate, + )): + logger.warning( + "Audio mux failed for validation video %s; saved video without audio.", + filename) + video_filenames.append(filename) - artifacts = [] - for filename, caption in zip(video_filenames, - all_captions, - strict=True): - video_artifact = self.tracker.video(filename, - caption=caption) - if video_artifact is not None: - artifacts.append(video_artifact) - if artifacts: - logs = { - f"validation_videos_{num_inference_steps}_steps": - artifacts - } - self.tracker.log_artifacts(logs, global_step) - else: - # Other sp_group leaders send their results to global rank 0 - world_group.send_object(step_videos, dst=0) - world_group.send_object(step_captions, dst=0) + artifacts = [] + for filename, caption in zip(video_filenames, + all_captions, + strict=True): + video_artifact = self.tracker.video(filename, + caption=caption) + if video_artifact is not None: + artifacts.append(video_artifact) + if artifacts: + logs = { + f"validation_videos_{num_inference_steps}_steps": + artifacts + } + self.tracker.log_artifacts(logs, global_step) + elif self.rank_in_sp_group == 0: + # Other sp_group leaders send their results to global rank 0 + world_group.send_object(step_videos, dst=0) + world_group.send_object(step_captions, dst=0) # Re-enable gradients for training training_args.inference_mode = False