nodes/hao-ai-lab-FastVideo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

upload reference videos to wandb

Browse Source
This commit is contained in:
H1yori233
2026-03-08 01:04:54 +00:00
parent 82846c0711
commit bbea8d88d5
4 changed files with 381 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/training/finetune/MatrixGame2.0/finetune_i2v.sh
+15 -13
View File
@@ -1,39 +1,41 @@
#!/bin/bash
export WANDB_API_KEY="7ff8b6e8356924f7a6dd51a0342dd1a422ea9352"
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_MODE=online
export TOKENIZERS_PARALLELISM=false
# export FASTVIDEO_ATTENTION_BACKEND=TORCH_SDPA
MODEL_PATH="FastVideo/Matrix-Game-2.0-Foundation-Diffusers"
DATA_DIR="footsies-dataset/preprocessed/combined_parquet_dataset"
VALIDATION_DATASET_FILE="$(dirname "$0")/validation.json"
NUM_GPUS=8
MODEL_PATH="../mg_models/Solaris-6K"
DATA_DIR="/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/train_81f/preprocessed_0306"
VALIDATION_DATASET_FILE="$(dirname "$0")/validation_test.json"
NUM_GPUS=1
# export CUDA_VISIBLE_DEVICES=4,5
# IP=[MASTER NODE IP]
# Training arguments
training_args=(
--tracker_project_name "matrixgame_finetune"
--output_dir "checkpoints/matrixgame_finetune"
--max_train_steps 1500
--tracker_project_name "mg_finetune_solaris"
--wandb_run_name "reference_videos"
--output_dir "checkpoints/reference_test"
--max_train_steps 1500000
--train_batch_size 1
--train_sp_batch_size 1
--gradient_accumulation_steps 4
--num_latent_t 20
--gradient_accumulation_steps 1
--num_latent_t 21
--num_height 352
--num_width 640
--num_frames 77
--num_frames 81
--enable_gradient_checkpointing_type "full"
)
# Parallel arguments
parallel_args=(
--num_gpus $NUM_GPUS
--sp_size 2
--sp_size 1
--tp_size 1
--hsdp_replicate_dim 4
--hsdp_shard_dim 2
--hsdp_replicate_dim 1
--hsdp_shard_dim 1
)
# Model arguments
examples/training/finetune/MatrixGame2.0/validation_test.json
+347 -5
View File
@@ -2,13 +2,355 @@
"data": [
{
"caption": "00",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000051_Bravo_instance_002.png",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/train/000010/train/75_batch2_split_0_000051_Bravo_instance_002.npy",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000057.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000057.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000057.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 352,
"height": 360,
"width": 640,
"num_frames": 77
"num_frames": 81
},
{
"caption": "01",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000012.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000012.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000012.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "02",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000066.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000066.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000066.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "03",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000051.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000051.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000051.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "04",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000040.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000040.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000040.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "05",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000071.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000071.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000071.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "06",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000052.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000052.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000052.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "07",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000057.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000057.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000057.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "08",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000044.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000044.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000044.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "09",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000068.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000068.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000068.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "10",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000016.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000016.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000016.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "11",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000015.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000015.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000015.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "12",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000047.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000047.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000047.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "13",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000037.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000037.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000037.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "14",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000045.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000045.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000045.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "15",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000036.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000036.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000036.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "16",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000013.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000013.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000013.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "17",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000065.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000065.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000065.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "18",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000027.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000027.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000027.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "19",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000066.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000066.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000066.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "20",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000038.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000038.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000038.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "21",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000007.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000007.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-4a5cd44ff498-20220419-153826_clip000007.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "22",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000068.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000068.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000068.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "23",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000003.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000003.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000003.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "24",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000007.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000007.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000007.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "25",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000026.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000026.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000026.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "26",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000005.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000005.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000005.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "27",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000036.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000036.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-22bbc5451fc6-20220423-234543_clip000036.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "28",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000024.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000024.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000024.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "29",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000046.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000046.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000046.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "30",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000049.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000049.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-176978c155fd-20220423-170106_clip000049.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
},
{
"caption": "31",
"image_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000035.png",
"ref_video": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000035.mp4",
"action_path": "/mnt/weka/home/hao.zhang/kaiqin/solaris/datasets/vpt/vpt/test_81f/v10/cheeky-cornflower-setter-2656cd13f4b9-20220421-200328_clip000035.npy",
"video_path": null,
"num_inference_steps": 50,
"height": 360,
"width": 640,
"num_frames": 81
}
]
}
}
fastvideo/training/trackers.py
+11
View File
@@ -151,6 +151,17 @@ class WandbTracker(BaseTracker):
self._run.log(metrics, step=step)
self._timed_metrics = {}
def log_artifacts(self, artifacts: dict[str, Any], step: int) -> None:
if not artifacts:
return
normalized_artifacts = dict(artifacts)
if "validation_ref_videos" in normalized_artifacts:
normalized_artifacts["reference_video/videos"] = (
normalized_artifacts.pop("validation_ref_videos"))
self.log(normalized_artifacts, step)
def finish(self) -> None:
self._run.finish()
fastvideo/training/training_pipeline.py
+8 -3
View File
@@ -38,6 +38,7 @@ from fastvideo.distributed import (cleanup_dist_env_and_memory,
from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs
from fastvideo.forward_context import set_forward_context
from fastvideo.logger import init_logger
from fastvideo.models.vision_utils import load_video
from fastvideo.pipelines import (ComposedPipelineBase, ForwardBatch,
LoRAPipeline, TrainingBatch)
from fastvideo.platforms import current_platform
@@ -991,9 +992,13 @@ class TrainingPipeline(LoRAPipeline, ABC):
strict=True):
if filename is None:
continue
video_artifact = self.tracker.video(filename,
caption=caption,
fps=sampling_param.fps)
ref_frames = np.stack(
[np.asarray(frame) for frame in load_video(filename)],
axis=0)
ref_frames = np.ascontiguousarray(
ref_frames.transpose(0, 3, 1, 2))
video_artifact = self.tracker.video(
ref_frames, caption=caption, fps=sampling_param.fps)
if video_artifact is not None:
ref_artifacts.append(video_artifact)
if ref_artifacts: