debugging ..

README.md

@@ -23,7 +23,7 @@ We've prepared some debug data to facilitate development. To make sure the train
 ```
 python scripts/download_hf.py --repo_id=Stealths-Video/dummyVid --local_dir=data/dummyVid --repo_type=dataset
 python scripts/download_hf.py --repo_id=Stealths-Video/mochi --local_dir=data/mochi --repo_type=model
-python scripts/download_hf.py --repo_id=Stealths-Video/Mochi-Synthetic-Data --local_dir=data/mochi --repo_type=model
+python scripts/download_hf.py --repo_id=Stealths-Video/Mochi-Synthetic-Data --local_dir=data/Mochi-Synthetic-Data --repo_type=dataset
 ```
 
 ## Debug Training

fastvideo/dataset/latent_datasets.py

@@ -25,7 +25,7 @@ class LatentDataset(Dataset):
         prompt_embed_file = self.data_anno[idx]["prompt_embed_path"]
         prompt_attention_mask_file = self.data_anno[idx]["prompt_attention_mask"]
         # load 
-        latent = torch.load(os.path.join(self.latent_dir, latent_file), map_location="cpu", weights_only=True)[:, :self.num_latent_t]
+        latent = torch.load(os.path.join(self.latent_dir, latent_file), map_location="cpu", weights_only=True)[:, -self.num_latent_t:]
         prompt_embed = torch.load(os.path.join(self.prompt_embed_dir, prompt_embed_file), map_location="cpu", weights_only=True)
         prompt_attention_mask = torch.load(os.path.join(self.prompt_attention_mask_dir, prompt_attention_mask_file), map_location="cpu", weights_only=True)
         return latent, prompt_embed, prompt_attention_mask

infenrece_mochi.sh

@@ -1,11 +1,11 @@
 python fastvideo/sample/sample_t2v_mochi.py \
     --model_path data/mochi \
     --prompt "A person is drizzling olive oil over a pizza with a spoon from a glass jar. The pizza has a golden-brown crust topped with dollops of white cheese and red sauce. The background shows a wooden table surface, indicating the setting is likely a kitchen. There is no significant movement in the scene, focusing solely on the action of oil being poured. The lighting is bright, enhancing the colors of the food. The scene continues with the person pouring olive oil on the pizza, focusing on the central area. The amount of oil on the pizza visibly increases, giving it a shiny appearance. The spoon used for drizzling is partially visible, held by a hand entering from the upper part of the frame. The consistency of the oil is clear and fluid. The background remains unchanged, maintaining the focus on the pizza. The oil pouring action concludes, and the spoon is being withdrawn from the frame, dripping oil back into the jar. The pizza now looks glossier due to the freshly added oil, emphasizing the texture of the cheese and sauce. The hand and spoon move away, indicating the end of this specific food preparation step. The wooden table and the kitchen setting remain constant throughout. The scene ends with the pizza fully dressed and more visually appealing due to the added oil." \
-    --num_frames 163 \
+    --num_frames 79 \
     --height 480 \
     --width 848 \
-    --num_inference_steps 32 \
+    --num_inference_steps 64 \
     --guidance_scale 4.5 \
     --seed 12345 \
-    --transformer_path data/outputs/debug/checkpoint-60/transformer \
-    --output_path outputs3.mp4
+    --transformer_path data/outputs/debug_1e-5/checkpoint-300/transformer \
+    --output_path debug_1e-5_79.mp4

scripts/accelerate_configs/deepspeed_zero2_config.yaml

@@ -8,6 +8,6 @@ main_process_ip: null
 main_process_port: 29503
 main_training_function: main
 num_machines: 1
-num_processes: 8
+num_processes: 4
 # gpu_ids: 0
 use_cpu: false

t2v_debug_multi.sh

@@ -9,15 +9,15 @@ accelerate launch \
     --data_merge_path "data/Mochi-Synthetic-Data/merge.txt" \
     --gradient_checkpointing \
     --train_batch_size=1 \
-    --num_latent_t 28 \
+    --num_latent_t 14 \
     --dataloader_num_workers 1 \
-    --gradient_accumulation_steps=4 \
-    --max_train_steps=1000000 \
+    --gradient_accumulation_steps=8 \
+    --max_train_steps=500 \
     --learning_rate=5e-5 \
     --lr_scheduler="constant" \
     --lr_warmup_steps=0 \
     --mixed_precision="bf16" \
-    --checkpointing_steps=60 \
+    --checkpointing_steps=100 \
     --allow_tf32 \
     --tile_sample_stride 192 \
     --ema_start_step 0 \
@@ -27,4 +27,102 @@ accelerate launch \
     --enable_tiling \
     --sp_size 1 \
     --train_sp_batch_size 1 \
-    --output_dir="data/outputs/debug"
+    --output_dir="data/outputs/debug"
+
+
+
+
+accelerate launch \
+    --config_file scripts/accelerate_configs/deepspeed_zero2_config.yaml \
+    fastvideo/train.py \
+    --seed 42 \
+    --pretrained_model_name_or_path data/mochi \
+    --cache_dir "data/.cache" \
+    --data_merge_path "data/Mochi-Synthetic-Data/merge.txt" \
+    --gradient_checkpointing \
+    --train_batch_size=1 \
+    --num_latent_t 14 \
+    --dataloader_num_workers 1 \
+    --gradient_accumulation_steps=8 \
+    --max_train_steps=500 \
+    --learning_rate=1e-5 \
+    --lr_scheduler="constant" \
+    --lr_warmup_steps=0 \
+    --mixed_precision="bf16" \
+    --checkpointing_steps=100 \
+    --allow_tf32 \
+    --tile_sample_stride 192 \
+    --ema_start_step 0 \
+    --cfg 0.1 \
+    --resume_from_checkpoint="latest" \
+    --ema_decay 0.999 \
+    --enable_tiling \
+    --sp_size 1 \
+    --train_sp_batch_size 1 \
+    --output_dir="data/outputs/debug_1e-5"
+
+
+
+# export WANDB_MODE="offline"
+
+accelerate launch \
+    --config_file scripts/accelerate_configs/deepspeed_zero2_config.yaml \
+    fastvideo/train.py \
+    --seed 42 \
+    --pretrained_model_name_or_path data/mochi \
+    --cache_dir "data/.cache" \
+    --data_merge_path "data/Mochi-Synthetic-Data/merge.txt" \
+    --gradient_checkpointing \
+    --train_batch_size=1 \
+    --num_latent_t 14 \
+    --dataloader_num_workers 1 \
+    --gradient_accumulation_steps=16 \
+    --max_train_steps=500 \
+    --learning_rate=5e-5 \
+    --lr_scheduler="constant" \
+    --lr_warmup_steps=0 \
+    --mixed_precision="bf16" \
+    --checkpointing_steps=200 \
+    --allow_tf32 \
+    --tile_sample_stride 192 \
+    --ema_start_step 0 \
+    --cfg 0.1 \
+    --resume_from_checkpoint="latest" \
+    --ema_decay 0.999 \
+    --enable_tiling \
+    --sp_size 1 \
+    --train_sp_batch_size 1 \
+    --output_dir="data/outputs/debug_ga_16"
+
+
+
+
+accelerate launch \
+    --config_file scripts/accelerate_configs/deepspeed_zero2_config.yaml \
+    fastvideo/train.py \
+    --seed 42 \
+    --pretrained_model_name_or_path data/mochi \
+    --cache_dir "data/.cache" \
+    --data_merge_path "data/Mochi-Synthetic-Data/merge.txt" \
+    --gradient_checkpointing \
+    --train_batch_size=1 \
+    --num_latent_t 14 \
+    --dataloader_num_workers 1 \
+    --gradient_accumulation_steps=16 \
+    --max_train_steps=500 \
+    --learning_rate=1e-6 \
+    --lr_scheduler="constant" \
+    --lr_warmup_steps=0 \
+    --mixed_precision="bf16" \
+    --checkpointing_steps=200 \
+    --allow_tf32 \
+    --tile_sample_stride 192 \
+    --ema_start_step 0 \
+    --cfg 0.1 \
+    --resume_from_checkpoint="latest" \
+    --ema_decay 0.999 \
+    --enable_tiling \
+    --sp_size 1 \
+    --train_sp_batch_size 1 \
+    --output_dir="data/outputs/debug_1e-6_ga_16"
+