Compare commits
11
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a8dddfaa16 | ||
|
|
7210c68f1b | ||
|
|
5602dc1bad | ||
|
|
ac4bc4ab84 | ||
|
|
bee27f9f74 | ||
|
|
dff0ea401a | ||
|
|
becd379f58 | ||
|
|
1ed7d7e1b0 | ||
|
|
d9fabcc5ef | ||
|
|
9db48498de | ||
|
|
1cd7038315 |
@@ -1,5 +1,6 @@
|
||||
from fastvideo import VideoGenerator
|
||||
|
||||
|
||||
PROMPT = (
|
||||
"A warm sunny backyard. The camera starts in a tight cinematic close-up "
|
||||
"of a woman and a man in their 30s, facing each other with serious "
|
||||
@@ -16,16 +17,20 @@ PROMPT = (
|
||||
|
||||
|
||||
def main() -> None:
|
||||
# Uses FastVideo default sampling settings for LTX2 base.
|
||||
generator = VideoGenerator.from_pretrained(
|
||||
"FastVideo/LTX2-Distilled-Diffusers",
|
||||
"Davids048/LTX2-Base-Diffusers",
|
||||
num_gpus=1,
|
||||
)
|
||||
|
||||
output_path = "outputs_video/ltx2_basic/output_ltx2_distilled_t2v.mp4"
|
||||
output_path = "outputs_video/ltx2_basic/output_ltx2_base_t2v_1088_1920_1.1.mp4"
|
||||
generator.generate_video(
|
||||
prompt=PROMPT,
|
||||
output_path=output_path,
|
||||
save_video=True,
|
||||
num_frames=121,
|
||||
height=1088,
|
||||
width=1920,
|
||||
)
|
||||
generator.shutdown()
|
||||
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
from fastvideo import VideoGenerator
|
||||
|
||||
|
||||
PROMPT = (
|
||||
"A warm sunny backyard. The camera starts in a tight cinematic close-up "
|
||||
"of a woman and a man in their 30s, facing each other with serious "
|
||||
"expressions. The woman, emotional and dramatic, says softly, \"That's "
|
||||
"it... Dad's lost it. And we've lost Dad.\" The man exhales, slightly "
|
||||
"annoyed: \"Stop being so dramatic, Jess.\" A beat. He glances aside, "
|
||||
"then mutters defensively, \"He's just having fun.\" The camera slowly "
|
||||
"pans right, revealing the grandfather in the garden wearing enormous "
|
||||
"butterfly wings, waving his arms in the air like he's trying to take "
|
||||
"off. He shouts, \"Wheeeew!\" as he flaps his wings with full commitment. "
|
||||
"The woman covers her face, on the verge of tears. The tone is deadpan, "
|
||||
"absurd, and quietly tragic."
|
||||
)
|
||||
|
||||
official_negative_prompt = (
|
||||
"blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, "
|
||||
"grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, "
|
||||
"deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, "
|
||||
"wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of "
|
||||
"field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent "
|
||||
"lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny "
|
||||
"valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, "
|
||||
"mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, "
|
||||
"off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward "
|
||||
"pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, "
|
||||
"inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts."
|
||||
)
|
||||
def main() -> None:
|
||||
# Uses FastVideo default sampling settings for LTX2 base.
|
||||
generator = VideoGenerator.from_pretrained(
|
||||
"Davids048/LTX2-Base-Diffusers",
|
||||
num_gpus=1,
|
||||
)
|
||||
|
||||
output_path = "outputs_video/ltx2_basic/output_ltx2_base_t2v_1088_1920_1.0.mp4"
|
||||
generator.generate_video(
|
||||
prompt=PROMPT,
|
||||
output_path=output_path,
|
||||
save_video=True,
|
||||
num_frames=121,
|
||||
height=1088,
|
||||
width=1920,
|
||||
fps=24,
|
||||
negative_prompt=official_negative_prompt,
|
||||
guidance_scale=3.0,
|
||||
)
|
||||
generator.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,34 @@
|
||||
from fastvideo import VideoGenerator
|
||||
|
||||
PROMPT = (
|
||||
"A warm sunny backyard. The camera starts in a tight cinematic close-up "
|
||||
"of a woman and a man in their 30s, facing each other with serious "
|
||||
"expressions. The woman, emotional and dramatic, says softly, \"That's "
|
||||
"it... Dad's lost it. And we've lost Dad.\" The man exhales, slightly "
|
||||
"annoyed: \"Stop being so dramatic, Jess.\" A beat. He glances aside, "
|
||||
"then mutters defensively, \"He's just having fun.\" The camera slowly "
|
||||
"pans right, revealing the grandfather in the garden wearing enormous "
|
||||
"butterfly wings, waving his arms in the air like he's trying to take "
|
||||
"off. He shouts, \"Wheeeew!\" as he flaps his wings with full commitment. "
|
||||
"The woman covers her face, on the verge of tears. The tone is deadpan, "
|
||||
"absurd, and quietly tragic."
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
generator = VideoGenerator.from_pretrained(
|
||||
"FastVideo/LTX2-Distilled-Diffusers",
|
||||
num_gpus=1,
|
||||
)
|
||||
|
||||
output_path = "outputs_video/ltx2_basic/output_ltx2_distilled_t2v.mp4"
|
||||
generator.generate_video(
|
||||
prompt=PROMPT,
|
||||
output_path=output_path,
|
||||
save_video=True,
|
||||
)
|
||||
generator.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -249,8 +249,8 @@ def _register_configs() -> None:
|
||||
"FastVideo/LTX2-base",
|
||||
],
|
||||
model_detectors=[
|
||||
lambda path: ("ltx2" in path.lower() or "ltx-2" in path.lower())
|
||||
and "distilled" not in path.lower(),
|
||||
lambda path: ("ltx2" in path.lower() or "ltx-2" in path.lower()) and
|
||||
"distilled" not in path.lower(),
|
||||
],
|
||||
)
|
||||
# LTX-2 (distilled)
|
||||
@@ -261,7 +261,8 @@ def _register_configs() -> None:
|
||||
"FastVideo/LTX2-Distilled-Diffusers",
|
||||
],
|
||||
model_detectors=[
|
||||
lambda path: ("ltx2" in path.lower() or "ltx-2" in path.lower()) and "distilled" in path.lower(),
|
||||
lambda path: ("ltx2" in path.lower() or "ltx-2" in path.lower()) and
|
||||
"distilled" in path.lower(),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@@ -280,18 +280,19 @@ class LTX2TrainingPipeline(TrainingPipeline):
|
||||
|
||||
conditioning_mask = None
|
||||
first_frame_p = self.training_args.ltx2_first_frame_conditioning_p
|
||||
if first_frame_p > 0:
|
||||
if torch.rand(1, generator=self.noise_random_generator).item(
|
||||
) < first_frame_p:
|
||||
conditioning_mask = torch.zeros(
|
||||
(batch_size, 1, latents.shape[2], latents.shape[3],
|
||||
latents.shape[4]),
|
||||
dtype=torch.bool,
|
||||
device=latents.device,
|
||||
)
|
||||
conditioning_mask[:, :, 0:1] = True
|
||||
noisy_model_input = torch.where(conditioning_mask, latents,
|
||||
noisy_model_input)
|
||||
if (first_frame_p > 0 and torch.rand(
|
||||
1,
|
||||
generator=self.noise_random_generator,
|
||||
).item() < first_frame_p):
|
||||
conditioning_mask = torch.zeros(
|
||||
(batch_size, 1, latents.shape[2], latents.shape[3],
|
||||
latents.shape[4]),
|
||||
dtype=torch.bool,
|
||||
device=latents.device,
|
||||
)
|
||||
conditioning_mask[:, :, 0:1] = True
|
||||
noisy_model_input = torch.where(conditioning_mask, latents,
|
||||
noisy_model_input)
|
||||
|
||||
if conditioning_mask is None:
|
||||
mask_patch = torch.zeros(
|
||||
@@ -398,9 +399,9 @@ class LTX2TrainingPipeline(TrainingPipeline):
|
||||
with self.tracker.timed("timing/forward_backward"), set_forward_context(
|
||||
current_timestep=training_batch.current_timestep,
|
||||
attn_metadata=training_batch.attn_metadata):
|
||||
with torch.autocast("cuda", dtype=training_batch.latents.dtype):
|
||||
with torch.autograd.set_detect_anomaly(True):
|
||||
outputs = self.transformer(**input_kwargs)
|
||||
with torch.autocast("cuda", dtype=training_batch.latents.dtype
|
||||
), torch.autograd.set_detect_anomaly(True):
|
||||
outputs = self.transformer(**input_kwargs)
|
||||
if isinstance(outputs, tuple):
|
||||
video_denoised, audio_denoised = outputs
|
||||
else:
|
||||
@@ -422,7 +423,7 @@ class LTX2TrainingPipeline(TrainingPipeline):
|
||||
audio_loss = (audio_pred_velocity.float() -
|
||||
audio_target.float())**2
|
||||
loss = loss + audio_loss.mean()
|
||||
logger.info("Audio loss: {}".format(audio_loss.mean()))
|
||||
logger.info("Audio loss: %s", audio_loss.mean())
|
||||
else:
|
||||
logger.warning("Audio denoised is None")
|
||||
|
||||
|
||||
@@ -888,59 +888,61 @@ class TrainingPipeline(LoRAPipeline, ABC):
|
||||
|
||||
# Only sp_group leaders (rank_in_sp_group == 0) need to send their
|
||||
# results to global rank 0
|
||||
if self.rank_in_sp_group == 0:
|
||||
if self.global_rank == 0:
|
||||
# Global rank 0 collects results from all sp_group leaders
|
||||
all_videos = step_videos # Start with own results
|
||||
all_captions = step_captions
|
||||
if self.rank_in_sp_group == 0 and self.global_rank == 0:
|
||||
# Global rank 0 collects results from all sp_group leaders
|
||||
all_videos = step_videos # Start with own results
|
||||
all_captions = step_captions
|
||||
|
||||
# Receive from other sp_group leaders
|
||||
for sp_group_idx in range(1, num_sp_groups):
|
||||
src_rank = sp_group_idx * self.sp_world_size # Global rank of other sp_group leaders
|
||||
recv_videos = world_group.recv_object(src=src_rank)
|
||||
recv_captions = world_group.recv_object(src=src_rank)
|
||||
all_videos.extend(recv_videos)
|
||||
all_captions.extend(recv_captions)
|
||||
# Receive from other sp_group leaders
|
||||
for sp_group_idx in range(1, num_sp_groups):
|
||||
src_rank = sp_group_idx * self.sp_world_size # Global rank of other sp_group leaders
|
||||
recv_videos = world_group.recv_object(src=src_rank)
|
||||
recv_captions = world_group.recv_object(src=src_rank)
|
||||
all_videos.extend(recv_videos)
|
||||
all_captions.extend(recv_captions)
|
||||
|
||||
video_filenames = []
|
||||
for i, (video, caption) in enumerate(
|
||||
zip(all_videos, all_captions, strict=True)):
|
||||
os.makedirs(training_args.output_dir, exist_ok=True)
|
||||
filename = os.path.join(
|
||||
training_args.output_dir,
|
||||
f"validation_step_{global_step}_inference_steps_{num_inference_steps}_video_{i}.mp4"
|
||||
)
|
||||
imageio.mimsave(filename, video, fps=sampling_param.fps)
|
||||
# Mux audio if available
|
||||
audio = output_batch.extra.get("audio")
|
||||
audio_sample_rate = output_batch.extra.get(
|
||||
"audio_sample_rate")
|
||||
if audio is not None and audio_sample_rate is not None:
|
||||
if not self._mux_audio(filename, audio,
|
||||
audio_sample_rate):
|
||||
logger.warning(
|
||||
"Audio mux failed for validation video %s; saved video without audio.",
|
||||
filename)
|
||||
video_filenames.append(filename)
|
||||
video_filenames = []
|
||||
for i, (video, caption) in enumerate(
|
||||
zip(all_videos, all_captions, strict=True)):
|
||||
os.makedirs(training_args.output_dir, exist_ok=True)
|
||||
filename = os.path.join(
|
||||
training_args.output_dir,
|
||||
f"validation_step_{global_step}_inference_steps_{num_inference_steps}_video_{i}.mp4"
|
||||
)
|
||||
imageio.mimsave(filename, video, fps=sampling_param.fps)
|
||||
# Mux audio if available
|
||||
audio = output_batch.extra.get("audio")
|
||||
audio_sample_rate = output_batch.extra.get(
|
||||
"audio_sample_rate")
|
||||
if (audio is not None and audio_sample_rate is not None
|
||||
and not self._mux_audio(
|
||||
filename,
|
||||
audio,
|
||||
audio_sample_rate,
|
||||
)):
|
||||
logger.warning(
|
||||
"Audio mux failed for validation video %s; saved video without audio.",
|
||||
filename)
|
||||
video_filenames.append(filename)
|
||||
|
||||
artifacts = []
|
||||
for filename, caption in zip(video_filenames,
|
||||
all_captions,
|
||||
strict=True):
|
||||
video_artifact = self.tracker.video(filename,
|
||||
caption=caption)
|
||||
if video_artifact is not None:
|
||||
artifacts.append(video_artifact)
|
||||
if artifacts:
|
||||
logs = {
|
||||
f"validation_videos_{num_inference_steps}_steps":
|
||||
artifacts
|
||||
}
|
||||
self.tracker.log_artifacts(logs, global_step)
|
||||
else:
|
||||
# Other sp_group leaders send their results to global rank 0
|
||||
world_group.send_object(step_videos, dst=0)
|
||||
world_group.send_object(step_captions, dst=0)
|
||||
artifacts = []
|
||||
for filename, caption in zip(video_filenames,
|
||||
all_captions,
|
||||
strict=True):
|
||||
video_artifact = self.tracker.video(filename,
|
||||
caption=caption)
|
||||
if video_artifact is not None:
|
||||
artifacts.append(video_artifact)
|
||||
if artifacts:
|
||||
logs = {
|
||||
f"validation_videos_{num_inference_steps}_steps":
|
||||
artifacts
|
||||
}
|
||||
self.tracker.log_artifacts(logs, global_step)
|
||||
elif self.rank_in_sp_group == 0:
|
||||
# Other sp_group leaders send their results to global rank 0
|
||||
world_group.send_object(step_videos, dst=0)
|
||||
world_group.send_object(step_captions, dst=0)
|
||||
|
||||
# Re-enable gradients for training
|
||||
training_args.inference_mode = False
|
||||
|
||||
Reference in New Issue
Block a user