Small change

fastvideo/v1/pipelines/preprocess/preprocess_pipeline_i2v.py

@@ -86,9 +86,7 @@ class PreprocessPipeline_I2V(BasePreprocessPipeline):
                             enabled=True):
             encoder_outputs = self.get_module("vae").encode(video_conditions)
 
-        # For now just hardcode this. Seed is the same as the one used in training/validation
-        latent_condition = encoder_outputs.sample(generator=torch.Generator(
-                device="cpu").manual_seed(42)) 
+        latent_condition = encoder_outputs.mean
         if (hasattr(self.get_module("vae"), "shift_factor")
                 and self.get_module("vae").shift_factor is not None):
             if isinstance(self.get_module("vae").shift_factor, torch.Tensor):

scripts/finetune/finetune_v1_i2v.sh

@@ -5,11 +5,10 @@ export HOME="/mnt/weka/home/hao.zhang/wei"
 
 DATA_DIR=$HOME/FastVideo/data/wei-i2v-dataset/crush-smol_preprocessed/combined_parquet_dataset
 VALIDATION_DIR=$HOME/FastVideo/data/wei-i2v-dataset/crush-smol_preprocessed/validation_parquet_dataset
-NUM_GPUS=2
-export CUDA_VISIBLE_DEVICES=0,1
+NUM_GPUS=4
 # IP=[MASTER NODE IP]
 
-CHECKPOINT_PATH="$DATA_DIR/outputs/wan_i2v_finetune/checkpoint-5"
+CHECKPOINT_PATH="$DATA_DIR/outputs/wan_i2v_finetune/checkpoint-50"
 
 # If you do not have 32 GPUs and to fit in memory, you can: 1. increase sp_size. 2. reduce num_latent_t
 torchrun --nnodes 1 --nproc_per_node $NUM_GPUS\
@@ -27,11 +26,11 @@ torchrun --nnodes 1 --nproc_per_node $NUM_GPUS\
     --dp_shards $NUM_GPUS \
     --train_sp_batch_size 1\
     --dataloader_num_workers 1\
-    --gradient_accumulation_steps=1\
-    --max_train_steps=120 \
+    --gradient_accumulation_steps=8 \
+    --max_train_steps=5000 \
     --learning_rate=1e-6\
     --mixed_precision="bf16"\
-    --checkpointing_steps=50 \
+    --checkpointing_steps=500000 \
     --validation_steps 20\
     --validation_sampling_steps "2,4,8" \
     --log_validation \
@@ -39,8 +38,8 @@ torchrun --nnodes 1 --nproc_per_node $NUM_GPUS\
     --allow_tf32\
     --ema_start_step 0\
     --cfg 0.0\
-    --output_dir="$DATA_DIR/outputs/wan_finetune"\
-    --tracker_project_name wan_finetune \
+    --output_dir="$DATA_DIR/outputs/wan_i2v_finetune"\
+    --tracker_project_name wan_i2v_finetune \
     --num_height 480 \
     --num_width 832 \
     --num_frames  77 \
@@ -52,5 +51,4 @@ torchrun --nnodes 1 --nproc_per_node $NUM_GPUS\
     --not_apply_cfg_solver \
     --master_weight_type "fp32" \
     --max_grad_norm 1.0 \
-
-# --resume_from_checkpoint "$CHECKPOINT_PATH"
+    # --resume_from_checkpoint "$CHECKPOINT_PATH"

scripts/preprocess/v1_preprocess_wan_data_i2v.sh

@@ -1,9 +1,10 @@
-# export WANDB_MODE="offline"
+export WANDB_MODE="offline"
+export HOME="/mnt/weka/home/hao.zhang/wei"
 GPU_NUM=1 # 2,4,8
 MODEL_PATH="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers"
 MODEL_TYPE="wan"
-DATA_MERGE_PATH="finetrainers/crush-smol/merge.txt"
-OUTPUT_DIR="crush-smol_preprocess"
+DATA_MERGE_PATH="$HOME/FastVideo/data/wei-i2v-dataset/crush-smol_raw/merge.txt"
+OUTPUT_DIR="$HOME/FastVideo/data/wei-i2v-dataset/crush-smol_preprocessed"
 VALIDATION_PATH="assets/prompt.txt"
 
 torchrun --nproc_per_node=$GPU_NUM \