diff --git a/examples/training/finetune/MatrixGame2.0/finetune_df.sh b/examples/training/finetune/MatrixGame2.0/finetune_df.sh new file mode 100644 index 000000000..e09bcad01 --- /dev/null +++ b/examples/training/finetune/MatrixGame2.0/finetune_df.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +export WANDB_BASE_URL="https://api.wandb.ai" +export WANDB_MODE=online +export TOKENIZERS_PARALLELISM=false +# export FASTVIDEO_ATTENTION_BACKEND=TORCH_SDPA + +MODEL_PATH="FastVideo/Matrix-Game-2.0-Foundation-Diffusers" +DATA_DIR="footsies-dataset/preprocessed/combined_parquet_dataset" +VALIDATION_DATASET_FILE="$(dirname "$0")/validation.json" +NUM_GPUS=8 +# export CUDA_VISIBLE_DEVICES=4,5 +# IP=[MASTER NODE IP] + +# Training arguments +training_args=( + --tracker_project_name "matrixgame_finetune" + --output_dir "checkpoints/matrixgame_finetune" + --override_transformer_cls_name "CausalMatrixGameWanModel" + --max_train_steps 1500 + --train_batch_size 1 + --train_sp_batch_size 1 + --gradient_accumulation_steps 4 + --num_latent_t 21 + --num_height 352 + --num_width 640 + --num_frames 81 + --enable_gradient_checkpointing_type "full" +) + +# Parallel arguments +parallel_args=( + --num_gpus $NUM_GPUS + --sp_size 2 + --tp_size 1 + --hsdp_replicate_dim 4 + --hsdp_shard_dim 2 +) + +# Model arguments +model_args=( + --model_path $MODEL_PATH + --pretrained_model_name_or_path $MODEL_PATH +) + +# Dataset arguments +dataset_args=( + --data_path "$DATA_DIR" + --dataloader_num_workers 1 +) + +# Validation arguments +validation_args=( + --log_validation + --validation_dataset_file "$VALIDATION_DATASET_FILE" + --validation_steps 100 + --validation_sampling_steps "40" + --validation_guidance_scale "6.0" +) + +# Optimizer arguments +optimizer_args=( + --learning_rate 2e-5 + --mixed_precision "bf16" + --weight_only_checkpointing_steps 100 + --training_state_checkpointing_steps 100 + --weight_decay 1e-4 + --max_grad_norm 1.0 +) + +# Miscellaneous arguments +miscellaneous_args=( + --inference_mode False + --checkpoints_total_limit 3 + --training_cfg_rate 0.1 + --multi_phased_distill_schedule "4000-1" + --not_apply_cfg_solver + --dit_precision "fp32" + --num_euler_timesteps 50 + --ema_start_step 0 +) + +# If you do not have 32 GPUs and to fit in memory, you can: 1. increase sp_size. 2. reduce num_latent_t +torchrun \ + --nnodes 1 \ + --nproc_per_node $NUM_GPUS \ + fastvideo/training/matrixgame_ar_diffusion_pipeline.py \ + "${parallel_args[@]}" \ + "${model_args[@]}" \ + "${dataset_args[@]}" \ + "${training_args[@]}" \ + "${optimizer_args[@]}" \ + "${validation_args[@]}" \ + "${miscellaneous_args[@]}" diff --git a/examples/training/finetune/MatrixGame2.0/validation.json b/examples/training/finetune/MatrixGame2.0/validation.json index 884e85782..b275ec6ed 100644 --- a/examples/training/finetune/MatrixGame2.0/validation.json +++ b/examples/training/finetune/MatrixGame2.0/validation.json @@ -1,356 +1,324 @@ { "data": [ { - "caption": "00", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000003/batch2_split_8_000026_Bravo_instance_001_clip001701.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000003/batch2_split_8_000026_Bravo_instance_001_clip001701.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000003/batch2_split_8_000026_Bravo_instance_001_clip001701.npy", + "caption": "00 Val-00: W", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "01", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_2_000068_Bravo_instance_003_clip001053.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_2_000068_Bravo_instance_003_clip001053.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_2_000068_Bravo_instance_003_clip001053.npy", + "caption": "01 Val-01: S", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "02", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_1_000039_Alpha_instance_000_clip002511.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_1_000039_Alpha_instance_000_clip002511.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_1_000039_Alpha_instance_000_clip002511.npy", + "caption": "02 Val-02: A", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "03", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_14_000059_Bravo_instance_002_clip000405.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_14_000059_Bravo_instance_002_clip000405.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_14_000059_Bravo_instance_002_clip000405.npy", + "caption": "03 Val-03: D", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "04", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_10_000057_Bravo_instance_002_clip000081.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_10_000057_Bravo_instance_002_clip000081.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_10_000057_Bravo_instance_002_clip000081.npy", + "caption": "04 Val-04: u", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/u.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "05", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_0_000020_Bravo_instance_002_clip000162.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_0_000020_Bravo_instance_002_clip000162.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_0_000020_Bravo_instance_002_clip000162.npy", + "caption": "05 Val-05: d", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000001.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/d.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "06", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_3_000026_Bravo_instance_001_clip001863.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_3_000026_Bravo_instance_001_clip001863.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_3_000026_Bravo_instance_001_clip001863.npy", + "caption": "06 Val-06: l", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000006.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/l.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "07", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_2_000068_Alpha_instance_003_clip001053.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_2_000068_Alpha_instance_003_clip001053.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_2_000068_Alpha_instance_003_clip001053.npy", + "caption": "07 Val-07: r", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000007.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/r.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "08", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000003/batch2_split_9_000041_Alpha_instance_000_clip000810.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000003/batch2_split_9_000041_Alpha_instance_000_clip000810.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000003/batch2_split_9_000041_Alpha_instance_000_clip000810.npy", + "caption": "08 Val-00: key rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_1.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "09", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_4_000076_Bravo_instance_001_clip000000.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_4_000076_Bravo_instance_001_clip000000.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_4_000076_Bravo_instance_001_clip000000.npy", + "caption": "09 Val-01: key rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_2.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "10", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_2_000041_Bravo_instance_003_clip002673.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_2_000041_Bravo_instance_003_clip002673.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_2_000041_Bravo_instance_003_clip002673.npy", + "caption": "10 Val-02: camera rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_1.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "11", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_5_000053_Alpha_instance_003_clip000243.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_5_000053_Alpha_instance_003_clip000243.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_5_000053_Alpha_instance_003_clip000243.npy", + "caption": "11 Val-03: camera rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_2.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "12", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_19_000080_Alpha_instance_002_clip001296.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_19_000080_Alpha_instance_002_clip001296.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_19_000080_Alpha_instance_002_clip001296.npy", + "caption": "12 Val-00: key+camera excl rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_1.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "13", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_1_000039_Bravo_instance_000_clip000486.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_1_000039_Bravo_instance_000_clip000486.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_1_000039_Bravo_instance_000_clip000486.npy", + "caption": "13 Val-01: key+camera excl rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_2.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "14", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_1_000039_Bravo_instance_000_clip000162.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_1_000039_Bravo_instance_000_clip000162.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_1_000039_Bravo_instance_000_clip000162.npy", + "caption": "14 Val-02: key+camera rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_1.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "15", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_2_000052_Bravo_instance_000_clip000162.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_2_000052_Bravo_instance_000_clip000162.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_2_000052_Bravo_instance_000_clip000162.npy", + "caption": "15 Val-03: key+camera rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_2.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "16", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_0_000020_Alpha_instance_002_clip000324.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_0_000020_Alpha_instance_002_clip000324.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_0_000020_Alpha_instance_002_clip000324.npy", + "caption": "16 Val-04: (simultaneous) key rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_1.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "17", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_10_000002_Alpha_instance_000_clip000243.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_10_000002_Alpha_instance_000_clip000243.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_10_000002_Alpha_instance_000_clip000243.npy", + "caption": "17 Val-05: (simultaneous) camera rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000001.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_1.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "18", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_2_000098_Alpha_instance_000_clip000162.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_2_000098_Alpha_instance_000_clip000162.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_2_000098_Alpha_instance_000_clip000162.npy", + "caption": "18 Val-06: (simultaneous) key+camera excl rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000006.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_2_action_rand_1.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "19", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_6_000011_Alpha_instance_003_clip000000.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_6_000011_Alpha_instance_003_clip000000.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_6_000011_Alpha_instance_003_clip000000.npy", + "caption": "19 Val-07: (simultaneous) key+camera rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000007.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_1.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "20", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_1_000039_Alpha_instance_000_clip002754.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_1_000039_Alpha_instance_000_clip002754.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_1_000039_Alpha_instance_000_clip002754.npy", + "caption": "20 Val-08: W+A", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/humanplay/000005.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "21", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_4_000076_Bravo_instance_001_clip002673.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_4_000076_Bravo_instance_001_clip002673.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_4_000076_Bravo_instance_001_clip002673.npy", + "caption": "21 Val-09: S+u", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/humanplay/000013.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S_u.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "22", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/75_batch2_split_8_000039_Bravo_instance_000_clip001296.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/75_batch2_split_8_000039_Bravo_instance_000_clip001296.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/75_batch2_split_8_000039_Bravo_instance_000_clip001296.npy", + "caption": "22 Val-08: Still", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/humanplay/000005.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/still.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "23", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000003/batch2_split_8_000051_Alpha_instance_001_clip000891.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000003/batch2_split_8_000051_Alpha_instance_001_clip000891.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000003/batch2_split_8_000051_Alpha_instance_001_clip000891.npy", + "caption": "23 Val-09: Still", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/humanplay/000013.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/still.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "24", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_19_000080_Alpha_instance_002_clip000891.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_19_000080_Alpha_instance_002_clip000891.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_19_000080_Alpha_instance_002_clip000891.npy", + "caption": "24 Val-06: key+camera excl rand Frame 4", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000006.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_1_f4.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "25", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_0_000020_Alpha_instance_002_clip000648.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_0_000020_Alpha_instance_002_clip000648.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_0_000020_Alpha_instance_002_clip000648.npy", + "caption": "25 Val-07: key+camera excl rand Frame 4", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000007.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_2_f4.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "26", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_1_000098_Alpha_instance_000_clip000081.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_1_000098_Alpha_instance_000_clip000081.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_1_000098_Alpha_instance_000_clip000081.npy", + "caption": "26 Train-00", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/first_frame/000500.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/videos/000500_action.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "27", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_5_000053_Alpha_instance_003_clip000000.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_5_000053_Alpha_instance_003_clip000000.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_5_000053_Alpha_instance_003_clip000000.npy", + "caption": "27 Train-01", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/first_frame/001000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/videos/001000_action.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "28", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_16_000029_Alpha_instance_003_clip000162.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_16_000029_Alpha_instance_003_clip000162.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_16_000029_Alpha_instance_003_clip000162.npy", + "caption": "28 Doom-00: W", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "29", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_0_000015_Bravo_instance_000_clip000486.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_0_000015_Bravo_instance_000_clip000486.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_0_000015_Bravo_instance_000_clip000486.npy", + "caption": "29 Doom-01: key rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000001.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_1.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "30", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_4_000042_Bravo_instance_003_clip000081.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_4_000042_Bravo_instance_003_clip000081.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_4_000042_Bravo_instance_003_clip000081.npy", + "caption": "30 Doom-02: camera rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_1.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 }, { - "caption": "31", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_19_000080_Bravo_instance_002_clip000000.png", - "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_19_000080_Bravo_instance_002_clip000000.mp4", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_19_000080_Bravo_instance_002_clip000000.npy", + "caption": "31 Doom-03: key+camera excl rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_1.npy", "video_path": null, - "num_inference_steps": 50, + "num_inference_steps": 40, "height": 352, "width": 640, - "num_frames": 81 + "num_frames": 77 } ] } \ No newline at end of file