Compare commits
11
Commits
main
...
will/ltx-base
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a8dddfaa16 | ||
|
|
7210c68f1b | ||
|
|
5602dc1bad | ||
|
|
ac4bc4ab84 | ||
|
|
bee27f9f74 | ||
|
|
dff0ea401a | ||
|
|
becd379f58 | ||
|
|
1ed7d7e1b0 | ||
|
|
d9fabcc5ef | ||
|
|
9db48498de | ||
|
|
1cd7038315 |
@@ -1,5 +1,6 @@
|
|||||||
from fastvideo import VideoGenerator
|
from fastvideo import VideoGenerator
|
||||||
|
|
||||||
|
|
||||||
PROMPT = (
|
PROMPT = (
|
||||||
"A warm sunny backyard. The camera starts in a tight cinematic close-up "
|
"A warm sunny backyard. The camera starts in a tight cinematic close-up "
|
||||||
"of a woman and a man in their 30s, facing each other with serious "
|
"of a woman and a man in their 30s, facing each other with serious "
|
||||||
@@ -16,16 +17,20 @@ PROMPT = (
|
|||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
|
# Uses FastVideo default sampling settings for LTX2 base.
|
||||||
generator = VideoGenerator.from_pretrained(
|
generator = VideoGenerator.from_pretrained(
|
||||||
"FastVideo/LTX2-Distilled-Diffusers",
|
"Davids048/LTX2-Base-Diffusers",
|
||||||
num_gpus=1,
|
num_gpus=1,
|
||||||
)
|
)
|
||||||
|
|
||||||
output_path = "outputs_video/ltx2_basic/output_ltx2_distilled_t2v.mp4"
|
output_path = "outputs_video/ltx2_basic/output_ltx2_base_t2v_1088_1920_1.1.mp4"
|
||||||
generator.generate_video(
|
generator.generate_video(
|
||||||
prompt=PROMPT,
|
prompt=PROMPT,
|
||||||
output_path=output_path,
|
output_path=output_path,
|
||||||
save_video=True,
|
save_video=True,
|
||||||
|
num_frames=121,
|
||||||
|
height=1088,
|
||||||
|
width=1920,
|
||||||
)
|
)
|
||||||
generator.shutdown()
|
generator.shutdown()
|
||||||
|
|
||||||
|
|||||||
@@ -1,54 +0,0 @@
|
|||||||
from fastvideo import VideoGenerator
|
|
||||||
|
|
||||||
|
|
||||||
PROMPT = (
|
|
||||||
"A warm sunny backyard. The camera starts in a tight cinematic close-up "
|
|
||||||
"of a woman and a man in their 30s, facing each other with serious "
|
|
||||||
"expressions. The woman, emotional and dramatic, says softly, \"That's "
|
|
||||||
"it... Dad's lost it. And we've lost Dad.\" The man exhales, slightly "
|
|
||||||
"annoyed: \"Stop being so dramatic, Jess.\" A beat. He glances aside, "
|
|
||||||
"then mutters defensively, \"He's just having fun.\" The camera slowly "
|
|
||||||
"pans right, revealing the grandfather in the garden wearing enormous "
|
|
||||||
"butterfly wings, waving his arms in the air like he's trying to take "
|
|
||||||
"off. He shouts, \"Wheeeew!\" as he flaps his wings with full commitment. "
|
|
||||||
"The woman covers her face, on the verge of tears. The tone is deadpan, "
|
|
||||||
"absurd, and quietly tragic."
|
|
||||||
)
|
|
||||||
|
|
||||||
official_negative_prompt = (
|
|
||||||
"blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, "
|
|
||||||
"grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, "
|
|
||||||
"deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, "
|
|
||||||
"wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of "
|
|
||||||
"field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent "
|
|
||||||
"lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny "
|
|
||||||
"valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, "
|
|
||||||
"mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, "
|
|
||||||
"off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward "
|
|
||||||
"pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, "
|
|
||||||
"inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts."
|
|
||||||
)
|
|
||||||
def main() -> None:
|
|
||||||
# Uses FastVideo default sampling settings for LTX2 base.
|
|
||||||
generator = VideoGenerator.from_pretrained(
|
|
||||||
"Davids048/LTX2-Base-Diffusers",
|
|
||||||
num_gpus=1,
|
|
||||||
)
|
|
||||||
|
|
||||||
output_path = "outputs_video/ltx2_basic/output_ltx2_base_t2v_1088_1920_1.0.mp4"
|
|
||||||
generator.generate_video(
|
|
||||||
prompt=PROMPT,
|
|
||||||
output_path=output_path,
|
|
||||||
save_video=True,
|
|
||||||
num_frames=121,
|
|
||||||
height=1088,
|
|
||||||
width=1920,
|
|
||||||
fps=24,
|
|
||||||
negative_prompt=official_negative_prompt,
|
|
||||||
guidance_scale=3.0,
|
|
||||||
)
|
|
||||||
generator.shutdown()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
from fastvideo import VideoGenerator
|
||||||
|
|
||||||
|
PROMPT = (
|
||||||
|
"A warm sunny backyard. The camera starts in a tight cinematic close-up "
|
||||||
|
"of a woman and a man in their 30s, facing each other with serious "
|
||||||
|
"expressions. The woman, emotional and dramatic, says softly, \"That's "
|
||||||
|
"it... Dad's lost it. And we've lost Dad.\" The man exhales, slightly "
|
||||||
|
"annoyed: \"Stop being so dramatic, Jess.\" A beat. He glances aside, "
|
||||||
|
"then mutters defensively, \"He's just having fun.\" The camera slowly "
|
||||||
|
"pans right, revealing the grandfather in the garden wearing enormous "
|
||||||
|
"butterfly wings, waving his arms in the air like he's trying to take "
|
||||||
|
"off. He shouts, \"Wheeeew!\" as he flaps his wings with full commitment. "
|
||||||
|
"The woman covers her face, on the verge of tears. The tone is deadpan, "
|
||||||
|
"absurd, and quietly tragic."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
generator = VideoGenerator.from_pretrained(
|
||||||
|
"FastVideo/LTX2-Distilled-Diffusers",
|
||||||
|
num_gpus=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
output_path = "outputs_video/ltx2_basic/output_ltx2_distilled_t2v.mp4"
|
||||||
|
generator.generate_video(
|
||||||
|
prompt=PROMPT,
|
||||||
|
output_path=output_path,
|
||||||
|
save_video=True,
|
||||||
|
)
|
||||||
|
generator.shutdown()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -249,8 +249,8 @@ def _register_configs() -> None:
|
|||||||
"FastVideo/LTX2-base",
|
"FastVideo/LTX2-base",
|
||||||
],
|
],
|
||||||
model_detectors=[
|
model_detectors=[
|
||||||
lambda path: ("ltx2" in path.lower() or "ltx-2" in path.lower())
|
lambda path: ("ltx2" in path.lower() or "ltx-2" in path.lower()) and
|
||||||
and "distilled" not in path.lower(),
|
"distilled" not in path.lower(),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
# LTX-2 (distilled)
|
# LTX-2 (distilled)
|
||||||
@@ -261,7 +261,8 @@ def _register_configs() -> None:
|
|||||||
"FastVideo/LTX2-Distilled-Diffusers",
|
"FastVideo/LTX2-Distilled-Diffusers",
|
||||||
],
|
],
|
||||||
model_detectors=[
|
model_detectors=[
|
||||||
lambda path: ("ltx2" in path.lower() or "ltx-2" in path.lower()) and "distilled" in path.lower(),
|
lambda path: ("ltx2" in path.lower() or "ltx-2" in path.lower()) and
|
||||||
|
"distilled" in path.lower(),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -280,18 +280,19 @@ class LTX2TrainingPipeline(TrainingPipeline):
|
|||||||
|
|
||||||
conditioning_mask = None
|
conditioning_mask = None
|
||||||
first_frame_p = self.training_args.ltx2_first_frame_conditioning_p
|
first_frame_p = self.training_args.ltx2_first_frame_conditioning_p
|
||||||
if first_frame_p > 0:
|
if (first_frame_p > 0 and torch.rand(
|
||||||
if torch.rand(1, generator=self.noise_random_generator).item(
|
1,
|
||||||
) < first_frame_p:
|
generator=self.noise_random_generator,
|
||||||
conditioning_mask = torch.zeros(
|
).item() < first_frame_p):
|
||||||
(batch_size, 1, latents.shape[2], latents.shape[3],
|
conditioning_mask = torch.zeros(
|
||||||
latents.shape[4]),
|
(batch_size, 1, latents.shape[2], latents.shape[3],
|
||||||
dtype=torch.bool,
|
latents.shape[4]),
|
||||||
device=latents.device,
|
dtype=torch.bool,
|
||||||
)
|
device=latents.device,
|
||||||
conditioning_mask[:, :, 0:1] = True
|
)
|
||||||
noisy_model_input = torch.where(conditioning_mask, latents,
|
conditioning_mask[:, :, 0:1] = True
|
||||||
noisy_model_input)
|
noisy_model_input = torch.where(conditioning_mask, latents,
|
||||||
|
noisy_model_input)
|
||||||
|
|
||||||
if conditioning_mask is None:
|
if conditioning_mask is None:
|
||||||
mask_patch = torch.zeros(
|
mask_patch = torch.zeros(
|
||||||
@@ -398,9 +399,9 @@ class LTX2TrainingPipeline(TrainingPipeline):
|
|||||||
with self.tracker.timed("timing/forward_backward"), set_forward_context(
|
with self.tracker.timed("timing/forward_backward"), set_forward_context(
|
||||||
current_timestep=training_batch.current_timestep,
|
current_timestep=training_batch.current_timestep,
|
||||||
attn_metadata=training_batch.attn_metadata):
|
attn_metadata=training_batch.attn_metadata):
|
||||||
with torch.autocast("cuda", dtype=training_batch.latents.dtype):
|
with torch.autocast("cuda", dtype=training_batch.latents.dtype
|
||||||
with torch.autograd.set_detect_anomaly(True):
|
), torch.autograd.set_detect_anomaly(True):
|
||||||
outputs = self.transformer(**input_kwargs)
|
outputs = self.transformer(**input_kwargs)
|
||||||
if isinstance(outputs, tuple):
|
if isinstance(outputs, tuple):
|
||||||
video_denoised, audio_denoised = outputs
|
video_denoised, audio_denoised = outputs
|
||||||
else:
|
else:
|
||||||
@@ -422,7 +423,7 @@ class LTX2TrainingPipeline(TrainingPipeline):
|
|||||||
audio_loss = (audio_pred_velocity.float() -
|
audio_loss = (audio_pred_velocity.float() -
|
||||||
audio_target.float())**2
|
audio_target.float())**2
|
||||||
loss = loss + audio_loss.mean()
|
loss = loss + audio_loss.mean()
|
||||||
logger.info("Audio loss: {}".format(audio_loss.mean()))
|
logger.info("Audio loss: %s", audio_loss.mean())
|
||||||
else:
|
else:
|
||||||
logger.warning("Audio denoised is None")
|
logger.warning("Audio denoised is None")
|
||||||
|
|
||||||
|
|||||||
@@ -888,59 +888,61 @@ class TrainingPipeline(LoRAPipeline, ABC):
|
|||||||
|
|
||||||
# Only sp_group leaders (rank_in_sp_group == 0) need to send their
|
# Only sp_group leaders (rank_in_sp_group == 0) need to send their
|
||||||
# results to global rank 0
|
# results to global rank 0
|
||||||
if self.rank_in_sp_group == 0:
|
if self.rank_in_sp_group == 0 and self.global_rank == 0:
|
||||||
if self.global_rank == 0:
|
# Global rank 0 collects results from all sp_group leaders
|
||||||
# Global rank 0 collects results from all sp_group leaders
|
all_videos = step_videos # Start with own results
|
||||||
all_videos = step_videos # Start with own results
|
all_captions = step_captions
|
||||||
all_captions = step_captions
|
|
||||||
|
|
||||||
# Receive from other sp_group leaders
|
# Receive from other sp_group leaders
|
||||||
for sp_group_idx in range(1, num_sp_groups):
|
for sp_group_idx in range(1, num_sp_groups):
|
||||||
src_rank = sp_group_idx * self.sp_world_size # Global rank of other sp_group leaders
|
src_rank = sp_group_idx * self.sp_world_size # Global rank of other sp_group leaders
|
||||||
recv_videos = world_group.recv_object(src=src_rank)
|
recv_videos = world_group.recv_object(src=src_rank)
|
||||||
recv_captions = world_group.recv_object(src=src_rank)
|
recv_captions = world_group.recv_object(src=src_rank)
|
||||||
all_videos.extend(recv_videos)
|
all_videos.extend(recv_videos)
|
||||||
all_captions.extend(recv_captions)
|
all_captions.extend(recv_captions)
|
||||||
|
|
||||||
video_filenames = []
|
video_filenames = []
|
||||||
for i, (video, caption) in enumerate(
|
for i, (video, caption) in enumerate(
|
||||||
zip(all_videos, all_captions, strict=True)):
|
zip(all_videos, all_captions, strict=True)):
|
||||||
os.makedirs(training_args.output_dir, exist_ok=True)
|
os.makedirs(training_args.output_dir, exist_ok=True)
|
||||||
filename = os.path.join(
|
filename = os.path.join(
|
||||||
training_args.output_dir,
|
training_args.output_dir,
|
||||||
f"validation_step_{global_step}_inference_steps_{num_inference_steps}_video_{i}.mp4"
|
f"validation_step_{global_step}_inference_steps_{num_inference_steps}_video_{i}.mp4"
|
||||||
)
|
)
|
||||||
imageio.mimsave(filename, video, fps=sampling_param.fps)
|
imageio.mimsave(filename, video, fps=sampling_param.fps)
|
||||||
# Mux audio if available
|
# Mux audio if available
|
||||||
audio = output_batch.extra.get("audio")
|
audio = output_batch.extra.get("audio")
|
||||||
audio_sample_rate = output_batch.extra.get(
|
audio_sample_rate = output_batch.extra.get(
|
||||||
"audio_sample_rate")
|
"audio_sample_rate")
|
||||||
if audio is not None and audio_sample_rate is not None:
|
if (audio is not None and audio_sample_rate is not None
|
||||||
if not self._mux_audio(filename, audio,
|
and not self._mux_audio(
|
||||||
audio_sample_rate):
|
filename,
|
||||||
logger.warning(
|
audio,
|
||||||
"Audio mux failed for validation video %s; saved video without audio.",
|
audio_sample_rate,
|
||||||
filename)
|
)):
|
||||||
video_filenames.append(filename)
|
logger.warning(
|
||||||
|
"Audio mux failed for validation video %s; saved video without audio.",
|
||||||
|
filename)
|
||||||
|
video_filenames.append(filename)
|
||||||
|
|
||||||
artifacts = []
|
artifacts = []
|
||||||
for filename, caption in zip(video_filenames,
|
for filename, caption in zip(video_filenames,
|
||||||
all_captions,
|
all_captions,
|
||||||
strict=True):
|
strict=True):
|
||||||
video_artifact = self.tracker.video(filename,
|
video_artifact = self.tracker.video(filename,
|
||||||
caption=caption)
|
caption=caption)
|
||||||
if video_artifact is not None:
|
if video_artifact is not None:
|
||||||
artifacts.append(video_artifact)
|
artifacts.append(video_artifact)
|
||||||
if artifacts:
|
if artifacts:
|
||||||
logs = {
|
logs = {
|
||||||
f"validation_videos_{num_inference_steps}_steps":
|
f"validation_videos_{num_inference_steps}_steps":
|
||||||
artifacts
|
artifacts
|
||||||
}
|
}
|
||||||
self.tracker.log_artifacts(logs, global_step)
|
self.tracker.log_artifacts(logs, global_step)
|
||||||
else:
|
elif self.rank_in_sp_group == 0:
|
||||||
# Other sp_group leaders send their results to global rank 0
|
# Other sp_group leaders send their results to global rank 0
|
||||||
world_group.send_object(step_videos, dst=0)
|
world_group.send_object(step_videos, dst=0)
|
||||||
world_group.send_object(step_captions, dst=0)
|
world_group.send_object(step_captions, dst=0)
|
||||||
|
|
||||||
# Re-enable gradients for training
|
# Re-enable gradients for training
|
||||||
training_args.inference_mode = False
|
training_args.inference_mode = False
|
||||||
|
|||||||
Reference in New Issue
Block a user