diff --git a/examples/training/finetune/MatrixGame2.0/finetune_i2v.sh b/examples/training/finetune/MatrixGame2.0/finetune_i2v.sh index ce00f73f3..f161d2bc8 100644 --- a/examples/training/finetune/MatrixGame2.0/finetune_i2v.sh +++ b/examples/training/finetune/MatrixGame2.0/finetune_i2v.sh @@ -1,39 +1,41 @@ #!/bin/bash +export WANDB_API_KEY="7ff8b6e8356924f7a6dd51a0342dd1a422ea9352" export WANDB_BASE_URL="https://api.wandb.ai" export WANDB_MODE=online export TOKENIZERS_PARALLELISM=false # export FASTVIDEO_ATTENTION_BACKEND=TORCH_SDPA -MODEL_PATH="FastVideo/Matrix-Game-2.0-Foundation-Diffusers" -DATA_DIR="footsies-dataset/preprocessed/combined_parquet_dataset" -VALIDATION_DATASET_FILE="$(dirname "$0")/validation.json" -NUM_GPUS=8 +MODEL_PATH="../mg_models/Solaris-6K" +DATA_DIR="/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/train_81f/preprocessed_0306" +VALIDATION_DATASET_FILE="$(dirname "$0")/validation_test.json" +NUM_GPUS=1 # export CUDA_VISIBLE_DEVICES=4,5 # IP=[MASTER NODE IP] # Training arguments training_args=( - --tracker_project_name "matrixgame_finetune" - --output_dir "checkpoints/matrixgame_finetune" - --max_train_steps 1500 + --tracker_project_name "mg_finetune_solaris" + --wandb_run_name "reference_videos" + --output_dir "checkpoints/reference_test" + --max_train_steps 1500000 --train_batch_size 1 --train_sp_batch_size 1 - --gradient_accumulation_steps 4 - --num_latent_t 20 + --gradient_accumulation_steps 1 + --num_latent_t 21 --num_height 352 --num_width 640 - --num_frames 77 + --num_frames 81 --enable_gradient_checkpointing_type "full" ) # Parallel arguments parallel_args=( --num_gpus $NUM_GPUS - --sp_size 2 + --sp_size 1 --tp_size 1 - --hsdp_replicate_dim 4 - --hsdp_shard_dim 2 + --hsdp_replicate_dim 1 + --hsdp_shard_dim 1 ) # Model arguments diff --git a/examples/training/finetune/MatrixGame2.0/validation_test.json b/examples/training/finetune/MatrixGame2.0/validation_test.json index ec80319fc..f157e01dd 100644 --- a/examples/training/finetune/MatrixGame2.0/validation_test.json +++ b/examples/training/finetune/MatrixGame2.0/validation_test.json @@ -2,13 +2,355 @@ "data": [ { "caption": "00", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000051_Bravo_instance_002.png", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000051_Bravo_instance_002.npy", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000057.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000057.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000057.npy", "video_path": null, "num_inference_steps": 50, - "height": 352, + "height": 360, "width": 640, - "num_frames": 77 + "num_frames": 81 + }, + { + "caption": "01", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000012.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000012.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000012.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "02", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000066.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000066.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000066.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "03", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000051.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000051.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000051.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "04", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000040.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000040.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000040.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "05", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000071.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000071.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000071.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "06", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000052.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000052.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000052.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "07", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000057.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000057.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000057.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "08", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000044.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000044.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000044.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "09", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000068.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000068.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000068.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "10", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000016.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000016.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000016.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "11", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000015.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000015.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000015.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "12", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000047.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000047.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000047.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "13", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000037.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000037.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000037.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "14", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000045.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000045.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000045.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "15", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000036.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000036.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000036.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "16", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000013.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000013.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000013.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "17", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000065.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000065.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000065.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "18", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000027.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000027.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000027.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "19", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000066.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000066.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000066.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "20", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000038.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000038.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000038.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "21", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000007.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000007.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000007.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "22", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000068.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000068.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000068.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "23", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000003.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000003.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000003.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "24", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000007.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000007.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000007.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "25", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000026.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000026.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000026.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "26", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000005.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000005.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000005.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "27", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000036.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000036.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000036.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "28", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000024.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000024.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000024.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "29", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000046.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000046.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000046.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "30", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000049.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000049.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000049.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 + }, + { + "caption": "31", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000035.png", + "ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000035.mp4", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000035.npy", + "video_path": null, + "num_inference_steps": 50, + "height": 360, + "width": 640, + "num_frames": 81 } ] -} +} \ No newline at end of file diff --git a/fastvideo/training/trackers.py b/fastvideo/training/trackers.py index 281d79325..42086d80d 100644 --- a/fastvideo/training/trackers.py +++ b/fastvideo/training/trackers.py @@ -151,6 +151,17 @@ class WandbTracker(BaseTracker): self._run.log(metrics, step=step) self._timed_metrics = {} + def log_artifacts(self, artifacts: dict[str, Any], step: int) -> None: + if not artifacts: + return + + normalized_artifacts = dict(artifacts) + if "validation_ref_videos" in normalized_artifacts: + normalized_artifacts["reference_video/videos"] = ( + normalized_artifacts.pop("validation_ref_videos")) + + self.log(normalized_artifacts, step) + def finish(self) -> None: self._run.finish() diff --git a/fastvideo/training/training_pipeline.py b/fastvideo/training/training_pipeline.py index 171e8ea56..d9a7d9de2 100644 --- a/fastvideo/training/training_pipeline.py +++ b/fastvideo/training/training_pipeline.py @@ -38,6 +38,7 @@ from fastvideo.distributed import (cleanup_dist_env_and_memory, from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs from fastvideo.forward_context import set_forward_context from fastvideo.logger import init_logger +from fastvideo.models.vision_utils import load_video from fastvideo.pipelines import (ComposedPipelineBase, ForwardBatch, LoRAPipeline, TrainingBatch) from fastvideo.platforms import current_platform @@ -991,9 +992,13 @@ class TrainingPipeline(LoRAPipeline, ABC): strict=True): if filename is None: continue - video_artifact = self.tracker.video(filename, - caption=caption, - fps=sampling_param.fps) + ref_frames = np.stack( + [np.asarray(frame) for frame in load_video(filename)], + axis=0) + ref_frames = np.ascontiguousarray( + ref_frames.transpose(0, 3, 1, 2)) + video_artifact = self.tracker.video( + ref_frames, caption=caption, fps=sampling_param.fps) if video_artifact is not None: ref_artifacts.append(video_artifact) if ref_artifacts: