Preprocessing Stage: 1. Save to parquets periodically 2. Allow resuming from the middle 3. Doesn't support multi-gpu for now. Data Loader Stage: 1. Allow multi-gpu dataloader 2. Doesn't drop files even if number of parquet files is not divisible by num_gpus 3. Able to resume training

2025-05-23 00:57:39 +00:00
parent 982bfcfdc8
commit 42a2f272d5
6 changed files with 183 additions and 181 deletions
@@ -12,7 +12,6 @@ torchrun --nproc_per_node=$GPU_NUM \
    --model_path $MODEL_PATH \
    --data_merge_path $DATA_MERGE_PATH \
    --preprocess_video_batch_size=4 \
-    --preprocess_text_batch_size=4 \
    --max_height=480 \
    --max_width=832 \
    --num_frames=81 \
@@ -21,4 +20,6 @@ torchrun --nproc_per_node=$GPU_NUM \
    --model_type $MODEL_TYPE \
    --text_encoder_name $TEXT_ENCODER_PATH \
    --train_fps 16 \
-    --validation_prompt_txt $VALIDATION_PATH
+    --validation_prompt_txt $VALIDATION_PATH \
+    --samples_per_file 108 \
+    --flush_frequency 108