nodes/hao-ai-lab-FastVideo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update scripts

Browse Source
This commit is contained in:
H1yori233
2026-03-24 19:22:24 -07:00
parent aa96629fb0
commit 7cbe94dbb4
2 changed files with 254 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/training/finetune/MatrixGame2.0/finetune_df.sh
+94
View File
@@ -0,0 +1,94 @@
#!/bin/bash
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_MODE=online
export TOKENIZERS_PARALLELISM=false
# export FASTVIDEO_ATTENTION_BACKEND=TORCH_SDPA
MODEL_PATH="FastVideo/Matrix-Game-2.0-Foundation-Diffusers"
DATA_DIR="footsies-dataset/preprocessed/combined_parquet_dataset"
VALIDATION_DATASET_FILE="$(dirname "$0")/validation.json"
NUM_GPUS=8
# export CUDA_VISIBLE_DEVICES=4,5
# IP=[MASTER NODE IP]
# Training arguments
training_args=(
--tracker_project_name "matrixgame_finetune"
--output_dir "checkpoints/matrixgame_finetune"
--override_transformer_cls_name "CausalMatrixGameWanModel"
--max_train_steps 1500
--train_batch_size 1
--train_sp_batch_size 1
--gradient_accumulation_steps 4
--num_latent_t 21
--num_height 352
--num_width 640
--num_frames 81
--enable_gradient_checkpointing_type "full"
)
# Parallel arguments
parallel_args=(
--num_gpus $NUM_GPUS
--sp_size 2
--tp_size 1
--hsdp_replicate_dim 4
--hsdp_shard_dim 2
)
# Model arguments
model_args=(
--model_path $MODEL_PATH
--pretrained_model_name_or_path $MODEL_PATH
)
# Dataset arguments
dataset_args=(
--data_path "$DATA_DIR"
--dataloader_num_workers 1
)
# Validation arguments
validation_args=(
--log_validation
--validation_dataset_file "$VALIDATION_DATASET_FILE"
--validation_steps 100
--validation_sampling_steps "40"
--validation_guidance_scale "6.0"
)
# Optimizer arguments
optimizer_args=(
--learning_rate 2e-5
--mixed_precision "bf16"
--weight_only_checkpointing_steps 100
--training_state_checkpointing_steps 100
--weight_decay 1e-4
--max_grad_norm 1.0
)
# Miscellaneous arguments
miscellaneous_args=(
--inference_mode False
--checkpoints_total_limit 3
--training_cfg_rate 0.1
--multi_phased_distill_schedule "4000-1"
--not_apply_cfg_solver
--dit_precision "fp32"
--num_euler_timesteps 50
--ema_start_step 0
)
# If you do not have 32 GPUs and to fit in memory, you can: 1. increase sp_size. 2. reduce num_latent_t
torchrun \
--nnodes 1 \
--nproc_per_node $NUM_GPUS \
fastvideo/training/matrixgame_ar_diffusion_pipeline.py \
"${parallel_args[@]}" \
"${model_args[@]}" \
"${dataset_args[@]}" \
"${training_args[@]}" \
"${optimizer_args[@]}" \
"${validation_args[@]}" \
"${miscellaneous_args[@]}"
examples/training/finetune/MatrixGame2.0/validation.json
+160 -192
View File
@@ -1,356 +1,324 @@
{
"data": [
{
"caption": "00",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000003/batch2_split_8_000026_Bravo_instance_001_clip001701.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000003/batch2_split_8_000026_Bravo_instance_001_clip001701.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000003/batch2_split_8_000026_Bravo_instance_001_clip001701.npy",
"caption": "00 Val-00: W",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "01",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_2_000068_Bravo_instance_003_clip001053.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_2_000068_Bravo_instance_003_clip001053.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_2_000068_Bravo_instance_003_clip001053.npy",
"caption": "01 Val-01: S",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "02",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_1_000039_Alpha_instance_000_clip002511.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_1_000039_Alpha_instance_000_clip002511.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_1_000039_Alpha_instance_000_clip002511.npy",
"caption": "02 Val-02: A",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "03",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_14_000059_Bravo_instance_002_clip000405.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_14_000059_Bravo_instance_002_clip000405.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_14_000059_Bravo_instance_002_clip000405.npy",
"caption": "03 Val-03: D",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "04",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_10_000057_Bravo_instance_002_clip000081.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_10_000057_Bravo_instance_002_clip000081.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_10_000057_Bravo_instance_002_clip000081.npy",
"caption": "04 Val-04: u",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000000.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/u.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "05",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_0_000020_Bravo_instance_002_clip000162.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_0_000020_Bravo_instance_002_clip000162.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_0_000020_Bravo_instance_002_clip000162.npy",
"caption": "05 Val-05: d",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000001.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/d.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "06",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_3_000026_Bravo_instance_001_clip001863.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_3_000026_Bravo_instance_001_clip001863.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_3_000026_Bravo_instance_001_clip001863.npy",
"caption": "06 Val-06: l",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000006.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/l.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "07",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_2_000068_Alpha_instance_003_clip001053.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_2_000068_Alpha_instance_003_clip001053.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_2_000068_Alpha_instance_003_clip001053.npy",
"caption": "07 Val-07: r",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000007.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/r.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "08",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000003/batch2_split_9_000041_Alpha_instance_000_clip000810.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000003/batch2_split_9_000041_Alpha_instance_000_clip000810.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000003/batch2_split_9_000041_Alpha_instance_000_clip000810.npy",
"caption": "08 Val-00: key rand",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_1.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "09",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_4_000076_Bravo_instance_001_clip000000.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_4_000076_Bravo_instance_001_clip000000.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_4_000076_Bravo_instance_001_clip000000.npy",
"caption": "09 Val-01: key rand",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_2.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "10",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_2_000041_Bravo_instance_003_clip002673.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_2_000041_Bravo_instance_003_clip002673.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_2_000041_Bravo_instance_003_clip002673.npy",
"caption": "10 Val-02: camera rand",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_1.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "11",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_5_000053_Alpha_instance_003_clip000243.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_5_000053_Alpha_instance_003_clip000243.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_5_000053_Alpha_instance_003_clip000243.npy",
"caption": "11 Val-03: camera rand",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_2.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "12",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_19_000080_Alpha_instance_002_clip001296.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_19_000080_Alpha_instance_002_clip001296.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_19_000080_Alpha_instance_002_clip001296.npy",
"caption": "12 Val-00: key+camera excl rand",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_1.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "13",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_1_000039_Bravo_instance_000_clip000486.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_1_000039_Bravo_instance_000_clip000486.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_1_000039_Bravo_instance_000_clip000486.npy",
"caption": "13 Val-01: key+camera excl rand",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_2.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "14",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_1_000039_Bravo_instance_000_clip000162.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_1_000039_Bravo_instance_000_clip000162.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_1_000039_Bravo_instance_000_clip000162.npy",
"caption": "14 Val-02: key+camera rand",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_1.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "15",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_2_000052_Bravo_instance_000_clip000162.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_2_000052_Bravo_instance_000_clip000162.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_2_000052_Bravo_instance_000_clip000162.npy",
"caption": "15 Val-03: key+camera rand",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_2.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "16",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_0_000020_Alpha_instance_002_clip000324.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_0_000020_Alpha_instance_002_clip000324.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_0_000020_Alpha_instance_002_clip000324.npy",
"caption": "16 Val-04: (simultaneous) key rand",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000000.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_1.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "17",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_10_000002_Alpha_instance_000_clip000243.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_10_000002_Alpha_instance_000_clip000243.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_10_000002_Alpha_instance_000_clip000243.npy",
"caption": "17 Val-05: (simultaneous) camera rand",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000001.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_1.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "18",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_2_000098_Alpha_instance_000_clip000162.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_2_000098_Alpha_instance_000_clip000162.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_2_000098_Alpha_instance_000_clip000162.npy",
"caption": "18 Val-06: (simultaneous) key+camera excl rand",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000006.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_2_action_rand_1.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "19",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_6_000011_Alpha_instance_003_clip000000.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_6_000011_Alpha_instance_003_clip000000.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_6_000011_Alpha_instance_003_clip000000.npy",
"caption": "19 Val-07: (simultaneous) key+camera rand",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000007.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_1.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "20",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_1_000039_Alpha_instance_000_clip002754.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_1_000039_Alpha_instance_000_clip002754.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_1_000039_Alpha_instance_000_clip002754.npy",
"caption": "20 Val-08: W+A",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/humanplay/000005.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "21",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_4_000076_Bravo_instance_001_clip002673.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_4_000076_Bravo_instance_001_clip002673.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_4_000076_Bravo_instance_001_clip002673.npy",
"caption": "21 Val-09: S+u",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/humanplay/000013.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S_u.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "22",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/75_batch2_split_8_000039_Bravo_instance_000_clip001296.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/75_batch2_split_8_000039_Bravo_instance_000_clip001296.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/75_batch2_split_8_000039_Bravo_instance_000_clip001296.npy",
"caption": "22 Val-08: Still",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/humanplay/000005.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/still.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "23",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000003/batch2_split_8_000051_Alpha_instance_001_clip000891.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000003/batch2_split_8_000051_Alpha_instance_001_clip000891.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000003/batch2_split_8_000051_Alpha_instance_001_clip000891.npy",
"caption": "23 Val-09: Still",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/humanplay/000013.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/still.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "24",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_19_000080_Alpha_instance_002_clip000891.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_19_000080_Alpha_instance_002_clip000891.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_19_000080_Alpha_instance_002_clip000891.npy",
"caption": "24 Val-06: key+camera excl rand Frame 4",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000006.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_1_f4.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "25",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_0_000020_Alpha_instance_002_clip000648.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_0_000020_Alpha_instance_002_clip000648.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_0_000020_Alpha_instance_002_clip000648.npy",
"caption": "25 Val-07: key+camera excl rand Frame 4",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000007.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_2_f4.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "26",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_1_000098_Alpha_instance_000_clip000081.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_1_000098_Alpha_instance_000_clip000081.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_1_000098_Alpha_instance_000_clip000081.npy",
"caption": "26 Train-00",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/first_frame/000500.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/videos/000500_action.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "27",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_5_000053_Alpha_instance_003_clip000000.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_5_000053_Alpha_instance_003_clip000000.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_5_000053_Alpha_instance_003_clip000000.npy",
"caption": "27 Train-01",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/first_frame/001000.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/videos/001000_action.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "28",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_16_000029_Alpha_instance_003_clip000162.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_16_000029_Alpha_instance_003_clip000162.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000001/batch2_split_16_000029_Alpha_instance_003_clip000162.npy",
"caption": "28 Doom-00: W",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000000.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "29",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_0_000015_Bravo_instance_000_clip000486.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_0_000015_Bravo_instance_000_clip000486.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_0_000015_Bravo_instance_000_clip000486.npy",
"caption": "29 Doom-01: key rand",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000001.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_1.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "30",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_4_000042_Bravo_instance_003_clip000081.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_4_000042_Bravo_instance_003_clip000081.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000000/75_batch2_split_4_000042_Bravo_instance_003_clip000081.npy",
"caption": "30 Doom-02: camera rand",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000002.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_1.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
},
{
"caption": "31",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_19_000080_Bravo_instance_002_clip000000.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_19_000080_Bravo_instance_002_clip000000.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train_81f_test/000002/batch2_split_19_000080_Bravo_instance_002_clip000000.npy",
"caption": "31 Doom-03: key+camera excl rand",
"image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000003.jpg",
"action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_1.npy",
"video_path": null,
"num_inference_steps": 50,
"num_inference_steps": 40,
"height": 352,
"width": 640,
"num_frames": 81
"num_frames": 77
}
]
}