train vis

scripts/causalvae/reconstruction.sh

@@ -1,6 +1,6 @@
 CUDA_VISIBLE_DEVICES=0 python examples/rec_imvi_vae.py \
-    --ae_path "../../Open-Sora-Plan/CausalVAEModel_4x8x8/" \
-    --video_path origin_tiger.mp4 \
+    --ae_path "/path/to/CausalVAEModel_4x8x8/" \
+    --video_path input.mp4 \
     --rec_path rec.mp4 \
     --device cuda \
     --sample_rate 1 \

scripts/text_condition/sample_video.sh

@@ -1,12 +1,12 @@
 CUDA_VISIBLE_DEVICES=6 python opensora/sample/sample_t2v.py \
-    --model_path /remote-home1/yeyang/dev/Open-Sora-Plan/debug_mask_loss/checkpoint-500/model \
+    --model_path 512/checkpoint-20/model \
     --version 65x512x512 \
     --image_size 512 \
-    --cache_dir "/remote-home1/yeyang/sora4.8/Open-Sora-Plan/cache_dir" \
+    --cache_dir "./cache_dir" \
     --text_encoder_name DeepFloyd/t5-v1_1-xxl \
     --text_prompt examples/prompt_list_0.txt \
     --ae CausalVAEModel_4x8x8 \
-    --ae_path "/remote-home1/yeyang/sora4.8/Open-Sora-Plan/CausalVAEModel_4x8x8/" \
+    --ae_path "/dxyl_data02/CausalVAEModel_4x8x8/" \
     --save_img_path "./sample_videos" \
     --fps 24 \
     --guidance_scale 10.0 \

scripts/text_condition/train_videoae_65x512x512.sh

@@ -9,14 +9,14 @@ accelerate launch \
     --cache_dir "./cache_dir" \
     --dataset t2v \
     --ae CausalVAEModel_4x8x8 \
-    --ae_path "/dxyl_code02/dev/CausalVAEModel_4x8x8/" \
+    --ae_path "/dxyl_data02/CausalVAEModel_4x8x8/" \
     --video_data "/dxyl_code02/dev/Open-Sora-Plan/scripts/train_data/video_data.txt" \
     --sample_rate 1 \
     --num_frames 65 \
     --max_image_size 512 \
     --gradient_checkpointing \
     --attention_mode xformers \
-    --train_batch_size=4 \
+    --train_batch_size=2 \
     --dataloader_num_workers 10 \
     --gradient_accumulation_steps=1 \
     --max_train_steps=1000000 \
@@ -24,7 +24,7 @@ accelerate launch \
     --lr_scheduler="constant" \
     --lr_warmup_steps=0 \
     --mixed_precision="bf16" \
-    --report_to="tensorboard" \
+    --report_to="wandb" \
     --checkpointing_steps=100 \
     --output_dir="512" \
     --allow_tf32 \
@@ -33,4 +33,5 @@ accelerate launch \
     --use_image_num 4 \
     --enable_tiling \
     --use_img_from_vid \
-    --pretrained t2v.pt
+    --pretrained t2v.pt \
+    --enable_tracker

scripts/train_data/video_data.txt

@@ -1,2 +1,2 @@
-/dxyl_data02/mixkit,/dxyl_data02/anno_jsons/video_mixkit_65f_54735.json
-/dxyl_data02/mixkit,/dxyl_data02/anno_jsons/video_mixkit_65f_54735.json
+/dxyl_data02/mixkit,/dxyl_data02/anno_jsons/tmp_video_mixkit_65f_54735.json
+/dxyl_data02/mixkit,/dxyl_data02/anno_jsons/tmp_video_mixkit_65f_54735.json