Generate synthetic dataset

Delete Open Sora Plan Modeling

amend log validation

Change name to fast video

Remove OSP modeling

clean up name changing

remove files

 Deleted unnecessary files

commit first

commit first

training

ok

Update generate_synthetic.sh and deepspeed_zero2_config.yaml

14

debug

overfitting ....

debugging ..

Debug successful!

Add zero3

OK

update optimizer

load

small bug

random seed args

sp enable & still has bug

rename

update inference sp code / can run / still has bug / don't output normal mp4

switch to deepspeed dummyoptim

fix some bugs; still output green

SP inference done!

fix typos in readme and latent dataset debug file

scripts/accelerate_configs/deepspeed_zero2_config.yaml

@@ -8,6 +8,6 @@ main_process_ip: null
 main_process_port: 29503
 main_training_function: main
 num_machines: 1
-num_processes: 2
+num_processes: 4
 # gpu_ids: 0
 use_cpu: false

scripts/accelerate_configs/deepspeed_zero3_config.yaml

@@ -9,6 +9,5 @@ main_process_ip: null
 main_process_port: 29501
 main_training_function: main
 num_machines: 1
-num_processes: 8
-gpu_ids: 0,1,2,3,4,5,6,7
+num_processes: 4
 use_cpu: false

scripts/accelerate_configs/zero2.json

@@ -10,6 +10,25 @@
     "bf16": {
         "enabled": "auto"
     },
+    "optimizer": {
+        "type": "AdamW",
+        "params": {
+          "lr": "auto",
+          "betas": [0.9, 0.999],
+          "eps": 1e-8,
+          "weight_decay": "auto"
+        }
+      },
+      "scheduler": {
+        "type": "WarmupCosineLR",
+        "params": {
+          "warmup_min_ratio": 0.0,
+          "cos_min_ratio": 0.1,
+          "warmup_type": "linear",
+          "warmup_num_steps": "auto",
+          "total_num_steps": "auto"
+        }
+      },
     "communication_data_type": "fp32",
     "gradient_clipping": 1.0,
     "train_micro_batch_size_per_gpu": "auto",

scripts/accelerate_configs/zero3.json

@@ -20,7 +20,7 @@
         "sub_group_size": 1e9,
         "reduce_bucket_size": 5e8,
         "stage3_prefetch_bucket_size": 5e8,
-        "stage3_param_persistence_threshold": "auto",
+        "stage3_param_persistence_threshold": 1e6,
         "stage3_max_live_parameters": 1e9,
         "stage3_max_reuse_distance": 1e9,
         "stage3_gather_16bit_weights_on_model_save": true

scripts/causalvae/eval.sh

@@ -1,21 +0,0 @@
-# REAL_DATASET_DIR=/remote-home1/dataset/OpenMMLab___Kinetics-400/raw/Kinetics-400/videos_val/
-REAL_DATASET_DIR=data/dummyVid/video
-EXP_NAME=decoder
-SAMPLE_RATE=3
-NUM_FRAMES=33
-RESOLUTION=256
-SUBSET_SIZE=50
-METRIC=ssim
-
-python opensora/models/causalvideovae/eval/eval_common_metric.py \
-    --batch_size 1 \
-    --real_video_dir ${REAL_DATASET_DIR} \
-    --generated_video_dir /remote-home1/lzj/dataset/gen/${EXP_NAME}_sr${SAMPLE_RATE}_nf${NUM_FRAMES}_res${RESOLUTION}_subset${SUBSET_SIZE} \
-    --device cuda:0 \
-    --sample_fps 3 \
-    --sample_rate ${SAMPLE_RATE} \
-    --num_frames ${NUM_FRAMES} \
-    --resolution ${RESOLUTION} \
-    --subset_size ${SUBSET_SIZE} \
-    --crop_size ${RESOLUTION} \
-    --metric ${METRIC}

scripts/causalvae/prepare_eval.sh

@@ -1,24 +0,0 @@
-export CUDA_VISIBLE_DEVICES=0
-REAL_DATASET_DIR=valid/
-EXP_NAME=decoder
-SAMPLE_RATE=1
-NUM_FRAMES=33
-RESOLUTION=512
-SUBSET_SIZE=1
-CKPT=/storage/lcm/Causal-Video-VAE/results/488dim8
-
-python opensora/models/causalvideovae/sample/rec_video_vae.py \
-    --batch_size 1 \
-    --real_video_dir ${REAL_DATASET_DIR} \
-    --generated_video_dir valid_gen/${EXP_NAME}_sr${SAMPLE_RATE}_nf${NUM_FRAMES}_res${RESOLUTION}_subset${SUBSET_SIZE} \
-    --device cuda \
-    --sample_fps 24 \
-    --sample_rate ${SAMPLE_RATE} \
-    --num_frames ${NUM_FRAMES} \
-    --resolution ${RESOLUTION} \
-    --subset_size ${SUBSET_SIZE} \
-    --crop_size ${RESOLUTION} \
-    --num_workers 8 \
-    --ckpt ${CKPT} \
-    --output_origin \
-    --enable_tiling

scripts/causalvae/rec_image.sh

@@ -1,8 +0,0 @@
-CUDA_VISIBLE_DEVICES=0 python examples/rec_image.py \
-    --ae_path "/storage/dataset/test140k" \
-    --image_path /storage/dataset/image/anytext3m/ocr_data/Art/images/gt_5544.jpg \
-    --rec_path rec.jpg \
-    --device cuda \
-    --short_size 512 \
-    --ae CausalVAEModel_4x8x8 \
-    --enable_tiling

scripts/causalvae/rec_video.sh

@@ -1,11 +0,0 @@
-CUDA_VISIBLE_DEVICES=0 python opensora/sample/vae_reconstruct_video.py \
-    --ae_path "data/Open-Sora-Plan-v1.2.0/vae" \
-    --video_path data/dummyVid/for_vae_reconstruct.mp4 \
-    --rec_path rec.mp4 \
-    --device cuda \
-    --sample_rate 1 \
-    --num_frames 65 \
-    --height 480 \
-    --width 640 \
-    --ae CausalVAEModel_4x8x8 \
-    --enable_tiling --tile_overlap_factor 0.125 --save_memory

scripts/overfit.sh

@@ -0,0 +1,39 @@
+accelerate launch \
+    --config_file scripts/accelerate_configs/deepspeed_zero2_config.yaml \
+    fastvideo/train.py \
+    --seed 42 \
+    --pretrained_model_name_or_path data/mochi \
+    --cache_dir "data/.cache" \
+    --data_merge_path "data/Mochi-Synthetic-Data/merge.txt" \
+    --gradient_checkpointing \
+    --train_batch_size=1 \
+    --num_latent_t 14 \
+    --dataloader_num_workers 1 \
+    --gradient_accumulation_steps=4 \
+    --max_train_steps=200 \
+    --learning_rate=1e-5 \
+    --lr_scheduler="constant" \
+    --lr_warmup_steps=0 \
+    --mixed_precision="bf16" \
+    --checkpointing_steps=50 \
+    --allow_tf32 \
+    --tile_sample_stride 192 \
+    --ema_start_step 0 \
+    --cfg 0.1 \
+    --ema_decay 0.999 \
+    --enable_tiling \
+    --sp_size 1 \
+    --train_sp_batch_size 1 \
+    --output_dir="data/outputs/debug_1e-5"
+
+python fastvideo/sample/sample_t2v_mochi.py \
+    --model_path data/mochi \
+    --prompt "A hand enters the frame, pulling a sheet of plastic wrap over three balls of dough placed on a wooden surface. The plastic wrap is stretched to cover the dough more securely. The hand adjusts the wrap, ensuring that it is tight and smooth over the dough. The scene focuses on the hand's movements as it secures the edges of the plastic wrap. No new objects appear, and the camera remains stationary, focusing on the action of covering the dough." \
+    --num_frames 79 \
+    --height 480 \
+    --width 848 \
+    --num_inference_steps 64 \
+    --guidance_scale 4.5 \
+    --seed 12346 \
+    --transformer_path data/outputs/debug_1e-5/checkpoint-50/transformer \
+    --output_path 123.mp4

scripts/text_condition/overfit.sh

@@ -1,67 +0,0 @@
-# export WANDB_MODE="offline"
-
-export ENTITY="linbin"
-export HF_DATASETS_OFFLINE=1 
-export TRANSFORMERS_OFFLINE=1
-export PDSH_RCMD_TYPE=ssh
-# NCCL setting
-export GLOO_SOCKET_IFNAME=eth0
-export NCCL_SOCKET_IFNAME=eth0
-export NCCL_IB_HCA=mlx5_10:1,mlx5_11:1,mlx5_12:1,mlx5_13:1
-export NCCL_IB_GID_INDEX=3
-export NCCL_IB_TC=162
-export NCCL_IB_TIMEOUT=22
-export NCCL_PXN_DISABLE=0
-export NCCL_IB_QPS_PER_CONNECTION=4
-export NCCL_ALGO=Ring
-export OMP_NUM_THREADS=1
-export MKL_NUM_THREADS=1
-
-accelerate launch \
-    --config_file scripts/accelerate_configs/deepspeed_zero2_config.yaml \
-    opensora/train/train_t2v_diffusers.py \
-    --model OpenSoraT2V-ROPE-L/122 \
-    --text_encoder_name google/mt5-xxl \
-    --cache_dir "data/.cache" \
-    --dataset t2v \
-    --data "data/dummyVid/merge.txt" \
-    --ae CausalVAEModel_D4_4x8x8 \
-    --ae_path "data/Open-Sora-Plan-v1.2.0/vae/" \
-    --sample_rate 1 \
-    --num_frames 29 \
-    --max_height 480 \
-    --max_width 640 \
-    --pretrained /ephemeral/hao.zhang/codefolder/FastVideo-OSP/data/Open-Sora-Plan-v1.2.0/29x480p/diffusion_pytorch_model.safetensors \
-    --interpolation_scale_t 1.0 \
-    --interpolation_scale_h 1.0 \
-    --interpolation_scale_w 1.0 \
-    --attention_mode xformers \
-    --gradient_checkpointing \
-    --train_batch_size=1 \
-    --dataloader_num_workers 10 \
-    --gradient_accumulation_steps=8 \
-    --max_train_steps=1000000 \
-    --learning_rate=1e-4 \
-    --lr_scheduler="constant" \
-    --lr_warmup_steps=0 \
-    --mixed_precision="bf16" \
-    --report_to="wandb" \
-    --checkpointing_steps=25 \
-    --allow_tf32 \
-    --model_max_length 512 \
-    --use_image_num 0 \
-    --tile_overlap_factor 0.125 \
-    --snr_gamma 5.0 \
-    --use_ema \
-    --ema_start_step 0 \
-    --cfg 0.1 \
-    --noise_offset 0.02 \
-    --use_rope \
-    --resume_from_checkpoint="latest" \
-    --ema_decay 0.999 \
-    --enable_tiling \
-    --speed_factor 1.0 \
-    --group_frame \
-    --sp_size 4 \
-    --train_sp_batch_size 1 \
-    --output_dir="outputs/debug_1node_vid_newpipeline_acc" 

scripts/text_condition/sample_t2v.sh

@@ -1,18 +0,0 @@
-CUDA_VISIBLE_DEVICES=0 python opensora/sample/sample_t2v.py \
-    --model_path outputs/debug/checkpoint-250/model \
-    --num_frames 29 \
-    --height 480 \
-    --width 640 \
-    --cache_dir "data/.cache" \
-    --text_encoder_name google/mt5-xxl \
-    --text_prompt data/inference_prompt/overfit.txt \
-    --ae CausalVAEModel_D4_4x8x8 \
-    --ae_path "data/Open-Sora-Plan-v1.2.0/vae" \
-    --save_img_path "data/inference_prompt" \
-    --fps 24 \
-    --guidance_scale 7.5 \
-    --num_sampling_steps 100 \
-    --enable_tiling \
-    --max_sequence_length 512 \
-    --sample_method EulerAncestralDiscrete \
-    --model_type "dit"

scripts/text_condition/sample_t2v_ddp.sh

@@ -1,21 +0,0 @@
-
-torchrun --nnodes=1 --nproc_per_node 8  --master_port 29503 \
-    -m opensora.sample.sample_t2v_ddp \
-    --model_path /path/to/checkpoint-xxx/model_ema \
-    --num_frames 93 \
-    --height 720 \
-    --width 1280 \
-    --cache_dir "./cache_dir" \
-    --text_encoder_name google/mt5-xxl \
-    --text_prompt examples/prompt_list_0.txt \
-    --ae CausalVAEModel_D4_4x8x8 \
-    --ae_path "/path/to/causalvideovae" \
-    --save_img_path "./sample_93x480p_cfg7.5_step100_eulera" \
-    --fps 24 \
-    --guidance_scale 7.5 \
-    --num_sampling_steps 100 \
-    --enable_tiling \
-    --tile_overlap_factor 0.125 \
-    --max_sequence_length 512 \
-    --sample_method EulerAncestralDiscrete \
-    --model_type "dit"

scripts/text_condition/sample_t2v_sp.sh

@@ -1,21 +0,0 @@
-
-torchrun --nnodes=1 --nproc_per_node 8  --master_port 29503 \
-    -m opensora.sample.sample_t2v_sp \
-    --model_path /path/to/checkpoint-xxx/model_ema \
-    --num_frames 93 \
-    --height 720 \
-    --width 1280 \
-    --cache_dir "./cache_dir" \
-    --text_encoder_name google/mt5-xxl \
-    --text_prompt examples/prompt_list_0.txt \
-    --ae CausalVAEModel_D4_4x8x8 \
-    --ae_path "/path/to/causalvideovae" \
-    --save_img_path "./sample_video_test" \
-    --fps 24 \
-    --guidance_scale 7.5 \
-    --num_sampling_steps 100 \
-    --enable_tiling \
-    --tile_overlap_factor 0.125 \
-    --max_sequence_length 512 \
-    --sample_method EulerAncestralDiscrete \
-    --model_type "dit" 

scripts/train_t2v.sh

@@ -47,9 +47,9 @@ accelerate launch \
     --report_to="wandb" \
     --checkpointing_steps=250 \
     --allow_tf32 \
-    --model_max_length 512 \
+    --text_max_length 512 \
     --use_image_num 0 \
-    --tile_overlap_factor 0.125 \
+    --tile_sample_stride_height 0.125 \
     --snr_gamma 5.0 \
     --use_ema \
     --ema_start_step 0 \

scripts/upload_hf.py

@@ -0,0 +1,8 @@
+from huggingface_hub import HfApi
+api = HfApi()
+
+api.upload_folder(
+    folder_path="data/Mochi-Synthetic-Data",
+    repo_id="Stealths-Video/Mochi-Synthetic-Data",
+    repo_type="dataset",
+)