nodes/hao-ai-lab-FastVideo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
H1yori233
2026-03-07 23:22:50 +00:00
parent c0f64377e4
commit fa7b8a605b
9 changed files with 546 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+4
View File
@@ -83,3 +83,7 @@ docs/distillation/examples/
dmd_t2v_output/
preprocess_output_text/
preprocess_output/
finetune_i2v_output/
finetune_df_output/
sf_train_output/
examples/training/finetune/MatrixGame2.0/finetune_df.slurm
+1 -1
View File
@@ -84,7 +84,7 @@ dataset_args=(
validation_args=(
--log_validation
--validation_dataset_file "$VALIDATION_DATASET_FILE"
--validation_steps 200
--validation_steps 500
--validation_sampling_steps "50"
--validation_guidance_scale "6.0"
)
examples/training/finetune/MatrixGame2.0/finetune_i2v.slurm
+12 -10
View File
@@ -35,26 +35,28 @@ RUN_NAME=$(date +"%m%d_%H%M")
echo "RUN_NAME: $RUN_NAME"
# Configs
MODEL_PATH="../mg_models/Matrix-Game-2.0-Foundation-Diffusers"
DATA_DIR="../solaris/datasets/train/preprocessed_0227"
VALIDATION_DATASET_FILE="examples/training/finetune/MatrixGame2.0/validation_mc.json"
MODEL_PATH="../mg_models/Solaris-6K"
DATA_DIR="/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/train_81f/preprocessed_0306"
# VALIDATION_DATASET_FILE="examples/training/finetune/MatrixGame2.0/validation_mc.json"
VALIDATION_DATASET_FILE="examples/training/finetune/MatrixGame2.0/validation_solaris.json"
# Training arguments
training_args=(
--tracker_project_name "mg_finetune_solaris"
--output_dir "checkpoints/matrixgame_finetune_${RUN_NAME}"
--wandb_run_name "${RUN_NAME}_bs64_more_steps"
--max_train_steps 150000
--wandb_run_name "${RUN_NAME}_bs64"
--override_transformer_cls_name "MatrixGameWanModel"
--max_train_steps 100000
--train_batch_size 1
--train_sp_batch_size 1
--gradient_accumulation_steps 1
--num_latent_t 20
--gradient_accumulation_steps 2
--num_latent_t 21
--num_height 352
--num_width 640
--num_frames 77
--num_frames 81
# --enable_gradient_checkpointing_type "full"
--reinit-action-module True
--override-keyboard-dim 23
# --reinit-action-module True
# --override-keyboard-dim 23
)
# Parallel arguments
examples/training/finetune/MatrixGame2.0/launch_preprocess_slurm.sh
+3 -3
View File
@@ -5,16 +5,16 @@ mkdir -p preprocess_output
# Launch 8 jobs, one for each node (Total 64 GPUs)
# Each node processes 8 consecutive files (64 total files / 8 nodes = 8 files per node)
for node_id in {0..3}; do
for node_id in {0..7}; do
# Calculate the starting file number for this node
start_file=$((node_id * 8))
echo "Launching node $node_id with files merge_${start_file}.txt to merge_$((start_file + 7)).txt"
sbatch --job-name=mg-pre-${node_id} \
sbatch --job-name=mg-vpt-pre-${node_id} \
--output=preprocess_output/mg-node-${node_id}.out \
--error=preprocess_output/mg-node-${node_id}.err \
$(pwd)/examples/training/finetune/MatrixGame2.0/preprocess_worker.slurm $start_file $node_id
done
echo "All 4 nodes (32 GPUs) launched successfully!"
echo "All 8 nodes (64 GPUs) launched successfully!"
examples/training/finetune/MatrixGame2.0/preprocess_worker.slurm
+3 -3
View File
@@ -16,8 +16,8 @@ export PYTHONPATH="/mnt/weka/home/hao.zhang/kaiqin/FastVideo_clean:$PYTHONPATH"
START_FILE=${1:-1} # Starting file number for this node
NODE_ID=${2:-0} # Node identifier (0-7)
MODEL_PATH="../mg_models/Matrix-Game-2.0-Foundation-Diffusers"
OUTPUT_BASE="../solaris/datasets/train/preprocessed_0228_81"
MODEL_PATH="../mg_models/Matrix-Game-2.0-Base-Diffusers"
OUTPUT_BASE="../solaris/datasets/vpt/vpt/train_81f/preprocessed_0306"
# Port range calculation
base_port=$((29500 + NODE_ID * 100))
@@ -28,7 +28,7 @@ for i in {1..8}; do
gpu=${gpu_ids[((i-1))]}
file_num=$((START_FILE + i - 1))
DATA_MERGE_PATH="../solaris/datasets/train/merge_${file_num}.txt"
DATA_MERGE_PATH="../solaris/datasets/vpt/vpt/train_81f/merge_${file_num}.txt"
OUTPUT_DIR="${OUTPUT_BASE}/gpu_${gpu}_file_${file_num}"
echo "DATA_MERGE_PATH: $DATA_MERGE_PATH"
echo "OUTPUT_DIR: $OUTPUT_DIR"
examples/training/finetune/MatrixGame2.0/validation_df.json
+160
View File
@@ -159,6 +159,166 @@
"height": 352,
"width": 640,
"num_frames": 81
},
{
"caption": "16",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Alpha_instance_000.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Alpha_instance_000.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 352,
"width": 640,
"num_frames": 81
},
{
"caption": "17",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Alpha_instance_001.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Alpha_instance_001.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 352,
"width": 640,
"num_frames": 81
},
{
"caption": "18",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Alpha_instance_002.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Alpha_instance_002.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 352,
"width": 640,
"num_frames": 81
},
{
"caption": "19",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Alpha_instance_003.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Alpha_instance_003.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 352,
"width": 640,
"num_frames": 81
},
{
"caption": "20",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Bravo_instance_000.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Bravo_instance_000.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 352,
"width": 640,
"num_frames": 81
},
{
"caption": "21",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Bravo_instance_001.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Bravo_instance_001.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 352,
"width": 640,
"num_frames": 81
},
{
"caption": "22",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Bravo_instance_002.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Bravo_instance_002.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 352,
"width": 640,
"num_frames": 81
},
{
"caption": "23",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Bravo_instance_003.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000054_Bravo_instance_003.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 352,
"width": 640,
"num_frames": 81
},
{
"caption": "24",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Alpha_instance_000.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Alpha_instance_000.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 352,
"width": 640,
"num_frames": 81
},
{
"caption": "25",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Alpha_instance_001.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Alpha_instance_001.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 352,
"width": 640,
"num_frames": 81
},
{
"caption": "26",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Alpha_instance_002.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Alpha_instance_002.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 352,
"width": 640,
"num_frames": 81
},
{
"caption": "27",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Alpha_instance_003.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Alpha_instance_003.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 352,
"width": 640,
"num_frames": 81
},
{
"caption": "28",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Bravo_instance_000.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Bravo_instance_000.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 352,
"width": 640,
"num_frames": 81
},
{
"caption": "29",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Bravo_instance_001.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Bravo_instance_001.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 352,
"width": 640,
"num_frames": 81
},
{
"caption": "30",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Bravo_instance_002.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Bravo_instance_002.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 352,
"width": 640,
"num_frames": 81
},
{
"caption": "31",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Bravo_instance_003.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000055_Bravo_instance_003.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 352,
"width": 640,
"num_frames": 81
}
]
}
examples/training/finetune/MatrixGame2.0/validation_solaris.json
+324
View File
@@ -0,0 +1,324 @@
{
"data": [
{
"caption": "00",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000057.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000057.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "01",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000012.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000012.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "02",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000066.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000066.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "03",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000051.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000051.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "04",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000040.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000040.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "05",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000071.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000071.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "06",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000052.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000052.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "07",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000057.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000057.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "08",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000044.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000044.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "09",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000068.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000068.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "10",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000016.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000016.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "11",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000015.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000015.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "12",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000047.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000047.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "13",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000037.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000037.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "14",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000045.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000045.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "15",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000036.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000036.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "16",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000013.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000013.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "17",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000065.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000065.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "18",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000027.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000027.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "19",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000066.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000066.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "20",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000038.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000038.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "21",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000007.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000007.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "22",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000068.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000068.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "23",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000003.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000003.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "24",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000007.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000007.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "25",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000026.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000026.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "26",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000005.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000005.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "27",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000036.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000036.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "28",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000024.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000024.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "29",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000046.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000046.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "30",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000049.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000049.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "31",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000035.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000035.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
}
]
}
fastvideo/models/dits/matrixgame/utils.py
+8 -4
View File
@@ -42,10 +42,14 @@ KEYBOARD_MAP_7 = { # templerun_distilled_model: still/w/s/left/right/a/d
KEYBOARD_MAP = KEYBOARD_MAP_4 # Default for backward compatibility
SOLARIS_MOVEMENT_KEY_INDICES = {
"W": 0, # forward
"S": 1, # back
"A": 2, # left
"D": 3, # right
# "W": 0, # forward
# "S": 1, # back
# "A": 2, # left
# "D": 3, # right
"W": 11, # forward
"S": 12, # back
"A": 13, # left
"D": 14, # right
}
def _require_cv2() -> bool:
fastvideo/training/training_pipeline.py
+31 -5
View File
@@ -819,11 +819,37 @@ class TrainingPipeline(LoRAPipeline, ABC):
self.global_rank,
training_args.validation_dataset_file,
local_main_process_only=False)
validation_dataset = ValidationDataset(
training_args.validation_dataset_file)
validation_dataloader = DataLoader(validation_dataset,
batch_size=None,
num_workers=0)
local_validation_ok = 1
local_validation_error = None
try:
validation_dataset = ValidationDataset(
training_args.validation_dataset_file)
validation_dataloader = DataLoader(validation_dataset,
batch_size=None,
num_workers=0)
except Exception as e:
local_validation_ok = 0
local_validation_error = repr(e)
logger.warning(
"Rank %s failed to build validation dataset, will skip validation this round. err=%s",
self.global_rank,
local_validation_error,
local_main_process_only=False)
validation_ok = torch.tensor(local_validation_ok,
device=get_local_torch_device(),
dtype=torch.int32)
dist.all_reduce(validation_ok, op=dist.ReduceOp.MIN)
if validation_ok.item() == 0:
if self.global_rank == 0:
logger.warning(
"Skip validation at step %s because at least one rank failed to prepare validation dataset.",
global_step)
training_args.inference_mode = False
self.transformer.train()
if getattr(self, "transformer_2", None) is not None:
self.transformer_2.train()
return
self.transformer.eval()
if getattr(self, "transformer_2", None) is not None: