diff --git a/.gitignore b/.gitignore index 657c99fb8..1944a9eb9 100644 --- a/.gitignore +++ b/.gitignore @@ -83,3 +83,7 @@ docs/distillation/examples/ dmd_t2v_output/ preprocess_output_text/ +preprocess_output/ +finetune_i2v_output/ +finetune_df_output/ +sf_train_output/ diff --git a/examples/training/finetune/MatrixGame2.0/finetune_df.slurm b/examples/training/finetune/MatrixGame2.0/finetune_df.slurm index aefa4a2d0..401c2d44e 100644 --- a/examples/training/finetune/MatrixGame2.0/finetune_df.slurm +++ b/examples/training/finetune/MatrixGame2.0/finetune_df.slurm @@ -84,7 +84,7 @@ dataset_args=( validation_args=( --log_validation --validation_dataset_file "$VALIDATION_DATASET_FILE" - --validation_steps 200 + --validation_steps 500 --validation_sampling_steps "50" --validation_guidance_scale "6.0" ) diff --git a/examples/training/finetune/MatrixGame2.0/finetune_i2v.slurm b/examples/training/finetune/MatrixGame2.0/finetune_i2v.slurm index 256a6cc94..f82ace261 100644 --- a/examples/training/finetune/MatrixGame2.0/finetune_i2v.slurm +++ b/examples/training/finetune/MatrixGame2.0/finetune_i2v.slurm @@ -35,26 +35,28 @@ RUN_NAME=$(date +"%m%d_%H%M") echo "RUN_NAME: $RUN_NAME" # Configs -MODEL_PATH="../mg_models/Matrix-Game-2.0-Foundation-Diffusers" -DATA_DIR="../solaris/datasets/train/preprocessed_0227" -VALIDATION_DATASET_FILE="examples/training/finetune/MatrixGame2.0/validation_mc.json" +MODEL_PATH="../mg_models/Solaris-6K" +DATA_DIR="/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/train_81f/preprocessed_0306" +# VALIDATION_DATASET_FILE="examples/training/finetune/MatrixGame2.0/validation_mc.json" +VALIDATION_DATASET_FILE="examples/training/finetune/MatrixGame2.0/validation_solaris.json" # Training arguments training_args=( --tracker_project_name "mg_finetune_solaris" --output_dir "checkpoints/matrixgame_finetune_${RUN_NAME}" - --wandb_run_name "${RUN_NAME}_bs64_more_steps" - --max_train_steps 150000 + --wandb_run_name "${RUN_NAME}_bs64" + --override_transformer_cls_name "MatrixGameWanModel" + --max_train_steps 100000 --train_batch_size 1 --train_sp_batch_size 1 - --gradient_accumulation_steps 1 - --num_latent_t 20 + --gradient_accumulation_steps 2 + --num_latent_t 21 --num_height 352 --num_width 640 - --num_frames 77 + --num_frames 81 # --enable_gradient_checkpointing_type "full" - --reinit-action-module True - --override-keyboard-dim 23 + # --reinit-action-module True + # --override-keyboard-dim 23 ) # Parallel arguments diff --git a/examples/training/finetune/MatrixGame2.0/launch_preprocess_slurm.sh b/examples/training/finetune/MatrixGame2.0/launch_preprocess_slurm.sh index 18da89da4..c2a6d1fd3 100644 --- a/examples/training/finetune/MatrixGame2.0/launch_preprocess_slurm.sh +++ b/examples/training/finetune/MatrixGame2.0/launch_preprocess_slurm.sh @@ -5,16 +5,16 @@ mkdir -p preprocess_output # Launch 8 jobs, one for each node (Total 64 GPUs) # Each node processes 8 consecutive files (64 total files / 8 nodes = 8 files per node) -for node_id in {0..3}; do +for node_id in {0..7}; do # Calculate the starting file number for this node start_file=$((node_id * 8)) echo "Launching node $node_id with files merge_${start_file}.txt to merge_$((start_file + 7)).txt" - sbatch --job-name=mg-pre-${node_id} \ + sbatch --job-name=mg-vpt-pre-${node_id} \ --output=preprocess_output/mg-node-${node_id}.out \ --error=preprocess_output/mg-node-${node_id}.err \ $(pwd)/examples/training/finetune/MatrixGame2.0/preprocess_worker.slurm $start_file $node_id done -echo "All 4 nodes (32 GPUs) launched successfully!" +echo "All 8 nodes (64 GPUs) launched successfully!" diff --git a/examples/training/finetune/MatrixGame2.0/preprocess_worker.slurm b/examples/training/finetune/MatrixGame2.0/preprocess_worker.slurm index 22da257c1..cca9221c7 100644 --- a/examples/training/finetune/MatrixGame2.0/preprocess_worker.slurm +++ b/examples/training/finetune/MatrixGame2.0/preprocess_worker.slurm @@ -16,8 +16,8 @@ export PYTHONPATH="/mnt/weka/home/hao.zhang/kaiqin/FastVideo_clean:$PYTHONPATH" START_FILE=${1:-1} # Starting file number for this node NODE_ID=${2:-0} # Node identifier (0-7) -MODEL_PATH="../mg_models/Matrix-Game-2.0-Foundation-Diffusers" -OUTPUT_BASE="../solaris/datasets/train/preprocessed_0228_81" +MODEL_PATH="../mg_models/Matrix-Game-2.0-Base-Diffusers" +OUTPUT_BASE="../solaris/datasets/vpt/vpt/train_81f/preprocessed_0306" # Port range calculation base_port=$((29500 + NODE_ID * 100)) @@ -28,7 +28,7 @@ for i in {1..8}; do gpu=${gpu_ids[((i-1))]} file_num=$((START_FILE + i - 1)) - DATA_MERGE_PATH="../solaris/datasets/train/merge_${file_num}.txt" + DATA_MERGE_PATH="../solaris/datasets/vpt/vpt/train_81f/merge_${file_num}.txt" OUTPUT_DIR="${OUTPUT_BASE}/gpu_${gpu}_file_${file_num}" echo "DATA_MERGE_PATH: $DATA_MERGE_PATH" echo "OUTPUT_DIR: $OUTPUT_DIR" diff --git a/examples/training/finetune/MatrixGame2.0/validation_df.json b/examples/training/finetune/MatrixGame2.0/validation_df.json index 7d227d0c5..86b28f63a 100644 --- a/examples/training/finetune/MatrixGame2.0/validation_df.json +++ b/examples/training/finetune/MatrixGame2.0/validation_df.json @@ -159,6 +159,166 @@ "height": 352, "width": 640, "num_frames": 81 + }, + { + "caption": "16", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Alpha_instance_000.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Alpha_instance_000.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "17", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Alpha_instance_001.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Alpha_instance_001.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "18", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Alpha_instance_002.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Alpha_instance_002.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "19", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Alpha_instance_003.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Alpha_instance_003.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "20", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Bravo_instance_000.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Bravo_instance_000.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "21", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Bravo_instance_001.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Bravo_instance_001.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "22", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Bravo_instance_002.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Bravo_instance_002.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "23", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Bravo_instance_003.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Bravo_instance_003.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "24", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Alpha_instance_000.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Alpha_instance_000.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "25", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Alpha_instance_001.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Alpha_instance_001.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "26", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Alpha_instance_002.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Alpha_instance_002.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "27", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Alpha_instance_003.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Alpha_instance_003.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "28", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Bravo_instance_000.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Bravo_instance_000.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "29", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Bravo_instance_001.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Bravo_instance_001.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "30", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Bravo_instance_002.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Bravo_instance_002.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "31", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Bravo_instance_003.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Bravo_instance_003.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 352, + "width": 640, + "num_frames": 81 } ] } diff --git a/examples/training/finetune/MatrixGame2.0/validation_solaris.json b/examples/training/finetune/MatrixGame2.0/validation_solaris.json new file mode 100644 index 000000000..9513f27af --- /dev/null +++ b/examples/training/finetune/MatrixGame2.0/validation_solaris.json @@ -0,0 +1,324 @@ +{ + "data": [ + { + "caption": "00", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000057.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000057.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "01", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000012.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000012.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "02", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000066.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000066.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "03", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000051.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000051.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "04", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000040.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000040.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "05", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000071.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000071.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "06", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000052.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000052.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "07", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000057.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000057.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "08", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000044.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000044.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "09", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000068.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000068.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "10", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000016.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000016.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "11", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000015.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000015.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "12", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000047.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000047.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "13", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000037.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000037.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "14", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000045.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000045.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "15", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000036.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000036.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "16", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000013.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000013.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "17", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000065.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000065.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "18", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000027.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000027.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "19", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000066.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000066.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "20", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000038.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000038.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "21", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000007.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000007.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "22", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000068.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000068.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "23", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000003.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000003.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "24", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000007.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000007.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "25", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000026.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000026.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "26", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000005.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000005.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "27", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000036.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000036.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "28", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000024.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000024.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "29", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000046.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000046.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "30", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000049.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000049.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "31", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000035.png", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000035.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + } + ] +} \ No newline at end of file diff --git a/fastvideo/models/dits/matrixgame/utils.py b/fastvideo/models/dits/matrixgame/utils.py index 96ad900f0..9f97469d9 100644 --- a/fastvideo/models/dits/matrixgame/utils.py +++ b/fastvideo/models/dits/matrixgame/utils.py @@ -42,10 +42,14 @@ KEYBOARD_MAP_7 = { # templerun_distilled_model: still/w/s/left/right/a/d KEYBOARD_MAP = KEYBOARD_MAP_4 # Default for backward compatibility SOLARIS_MOVEMENT_KEY_INDICES = { - "W": 0, # forward - "S": 1, # back - "A": 2, # left - "D": 3, # right + # "W": 0, # forward + # "S": 1, # back + # "A": 2, # left + # "D": 3, # right + "W": 11, # forward + "S": 12, # back + "A": 13, # left + "D": 14, # right } def _require_cv2() -> bool: diff --git a/fastvideo/training/training_pipeline.py b/fastvideo/training/training_pipeline.py index a287ecd7f..ad1791603 100644 --- a/fastvideo/training/training_pipeline.py +++ b/fastvideo/training/training_pipeline.py @@ -819,11 +819,37 @@ class TrainingPipeline(LoRAPipeline, ABC): self.global_rank, training_args.validation_dataset_file, local_main_process_only=False) - validation_dataset = ValidationDataset( - training_args.validation_dataset_file) - validation_dataloader = DataLoader(validation_dataset, - batch_size=None, - num_workers=0) + local_validation_ok = 1 + local_validation_error = None + try: + validation_dataset = ValidationDataset( + training_args.validation_dataset_file) + validation_dataloader = DataLoader(validation_dataset, + batch_size=None, + num_workers=0) + except Exception as e: + local_validation_ok = 0 + local_validation_error = repr(e) + logger.warning( + "Rank %s failed to build validation dataset, will skip validation this round. err=%s", + self.global_rank, + local_validation_error, + local_main_process_only=False) + + validation_ok = torch.tensor(local_validation_ok, + device=get_local_torch_device(), + dtype=torch.int32) + dist.all_reduce(validation_ok, op=dist.ReduceOp.MIN) + if validation_ok.item() == 0: + if self.global_rank == 0: + logger.warning( + "Skip validation at step %s because at least one rank failed to prepare validation dataset.", + global_step) + training_args.inference_mode = False + self.transformer.train() + if getattr(self, "transformer_2", None) is not None: + self.transformer_2.train() + return self.transformer.eval() if getattr(self, "transformer_2", None) is not None: