From bdaf2e2e458088fcab546f295182229a87fb5d1e Mon Sep 17 00:00:00 2001 From: JerryZhou54 Date: Tue, 3 Feb 2026 20:23:13 +0000 Subject: [PATCH] Support custom action trajectories for validation --- .../action/000000_action.npy | Bin 0 -> 2994 bytes .../action/000001_action.npy | Bin 0 -> 2994 bytes .../action/000002_action.npy | Bin 0 -> 2994 bytes .../action/000003_action.npy | Bin 0 -> 2994 bytes .../action/000004_action.npy | Bin 0 -> 2994 bytes .../action/000005_action.npy | Bin 0 -> 2994 bytes .../action/000006_action.npy | Bin 0 -> 2994 bytes .../action/000007_action.npy | Bin 0 -> 2994 bytes .../action/000008_action.npy | Bin 0 -> 2994 bytes .../action/000009_action.npy | Bin 0 -> 2994 bytes .../action/000010_action.npy | Bin 0 -> 2994 bytes .../action/000011_action.npy | Bin 0 -> 2994 bytes .../action/000012_action.npy | Bin 0 -> 2994 bytes .../action/000013_action.npy | Bin 0 -> 2994 bytes .../action/000014_action.npy | Bin 0 -> 2994 bytes .../action/000015_action.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/action/README.md | 18 + .../WanGame2.1_1.3b_i2v/finetune_i2v.slurm | 8 +- .../WanGame2.1_1.3b_i2v/generate_actions.py | 193 +++++++++ .../WanGame2.1_1.3b_i2v/validation.json | 404 ++++++++++++++++++ fastvideo/dataset/validation_dataset.py | 21 + 21 files changed, 640 insertions(+), 4 deletions(-) create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000000_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000001_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000002_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000003_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000004_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000005_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000006_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000007_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000008_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000009_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000010_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000011_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000012_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000013_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000014_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000015_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/README.md create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/generate_actions.py create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/validation.json diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000000_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000000_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..8a6ab2ab14b855f0691ea93076134c495f72926b GIT binary patch literal 2994 zcmeHIF-yZh6i!lO#X37UWlIVrlY@v*sNi5QRl&hQ!X>#9vB`zYMF_UgrP{!4d4m5| zbBRMJZVE22TBwCFv;Vzu)Tm#Y{&VWJt9g`0sO$DI3P*JXdW=A zfa^^q@O~1uB^5Zz8qQT8-ryE2M;%5zu7|8hv8dOkCs1zMqaIfm;JWVRXB`bVV+MZt z086m=-V-i-f@lbl7U_aC>VH!@H_q-WL<6qPj=OAUTWpG8n|$;|Q>$^?+cwI4tBBM= z>d<`~(ht%P(hpq)q#wF!{LTLW^@QenO!5r>IOer_%paALjn=(duB8J1r+~0*j%u*K E06JO3F8}}l literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000001_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000001_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..e649b6c7273d76092c69f063d53cc78ae717dfbf GIT binary patch literal 2994 zcmeHIze~eF6uzX!igk8y%62J~Ob#MKp@M_KR0RhI376zb#3mOmi4bg|i`u|#d4m6~ z=7&NQw+=4%hIjAYJ-)|%e0Q(3Z!Rux74(Rnuul_@<6|8!rg)^|8RID@k;Ax8>+~6! zC$!2Z0f}j~H-;m9sc&YxtF1RYK#ht0C=27QuJD_nzNDZ`*AG(XK_TGy-7BPiDGQBNofuxIzKR`91g%+|bp?{8fwHosW#bm8^+sarn@P7={lFdlv F%uh&;#4i8< literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000002_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000002_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..3ee9a9c38c0fc4b02f535415193cac4201c448a3 GIT binary patch literal 2994 zcmeH|ze~eF6vr>AF=Cw^oU&aCC6j}QP^jQwFjc|9LBb`ua$=K|OCkhY=%O}oTb|&5 ztN9@`h?|1Tz2R~1-Qx@Q<-@(wxw*K!RnQ}P!U0Wuj*ktznBbv-r!h}CiQJe6v~Hh~ zSwgFR!pNLfwK*6XOXGL3yV`ogebj{5kFs!{Tdp5-YK7S>y(gTL9K@c>sUJs4$}>NO z5`@-Fg3zO&9Vx*^&v2@E@Cw&p+RA|SgchXvoYwQ|$KVv{SEL#8no^cAl=zfIxNe`J zS&BtJC1{St+8zwerTM$qU2VNteWeM$pXBkpaQu)m?8NyjyGM+n0{EWKa7dFhV|kcC z1zc~Cz>OqoAr-jFGn}dcyuvlut~ww+u18rh$2DG?96`Bmw|ZP%faiJVA0nFMjxq?! zJ6M9vcOGyNP{bl28qsS~Vti4$SMK%;kUrO@hh4UlZ){4 DWcb7{ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000004_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000004_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..9ce5230372d7f3a39ac9eb871b638708e37e2d55 GIT binary patch literal 2994 zcmeH|ze~eF6vva)7_rU{PT7(|$>bm+6e>6vOjU4jkZ?(^L~MR=xd_1)x>Q@ZEl=>j z)%@_lh?|3h_lA4C?>)ZoK0J7(b8~TdD@%{klM&#|C&saE%qPZBH>L@p1jTMb0$j^y zXqMqh&q9>qsx}8heWCv@c9&bPMqg@z?Z$#5$#x!a;U$O!5GtWsn}*t_rF~^@KSS7O%JlH7+sQXKsdgK`Y4uRcQ9B#YZ6$QU zffMNNhuD)Y4^D}5;aTXMK$n-`)8)Y_aV|UyofGKt`X79ty3kCGl03sdo_VD@^9P&p TTIoU(l_2m}1nQN|QT13Kr_=zg literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000005_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000005_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..02cd7ab2e0691d85f98d3d6140435496260cf2dd GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGacRJ{HmZH3|;+@f9~Tc6;6 ztNmf2h}#V=d62xkm&X_K@g=Xc?yj!yCG?D5utyV@;ZqGSXLzjPdBjpiLOWs}tVY?BdMvyJiN5UA%fgjk6x>1;*7ni3sz+mwlxM_0=Cp^TynL1AoZn`EeiZ>=zoU(Wakjdn<(2lXi4*=S99{ zLhvAX=-x!_g&u?+gdX|~2tD-K*#3WjszY-nh_ZzK+2-Y{%^wxhjod$0tOW!A$3QLF Ij8v2H4YmmYtpET3 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000007_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000007_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..b0f8a4d3e79be73f379dc65f8aca9f50c5a87b48 GIT binary patch literal 2994 zcmeH|u}i~16vi*9F=Cw^oU$c_lF30tC{%DTn5y95Amy4|iP$FLk_f>Tx>Q@ZEl=>j z)ui!(5jO`B?hW_defRj_ejL2Qy}7u&RmdZGGJTeK+&nhS*}&`?<~ZUhr{OT-KCA3A zI!RdRCjpIFdA2%TV{ZJeb{AW(W}9qE?1$Mj&h4QWab{1mNqSE?r#XndA!l9`CMnOn z6bca9rW1r7`gKbI4q3vfGJ+Sl2Fp=8K}%?Un#Zi_R|iM1Ue#AEq0GQ_-ShXdHjC3s zVYI%3Iap%nf#su!avuUU(5lct|BQ65ob6`_+Cm*4eoZ_1mPV?y{+nLEijYOG-l0k# z2{gG#K207-3Avz39|<&hNj^;;NC~;1N*@U{c}YG^9!Lqfp#Fa!$_XvBX_O`G$1$&# YWBy>>U#a+1sgl6o5~#jxM$5(i0K`53tpET3 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000008_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000008_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..8a754f020b29a79843f069690896426397c4eb96 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGarJu literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000010_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000010_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..e97638cfc59b49de2c798709bf1dc5e988c22fd0 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGarc literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000015_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000015_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..a8c7fba01479245739ac912e26c76249940eec91 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa[{k2}]" + + # Format Mouse Description + if m1 == "stop" and m2 == "stop": + m_desc = "Static" + elif m1 == m2: + m_desc = f"Hold [{m1}]" + else: + m_desc = f"Switch [{m1}]->[{m2}]" + + return f"{k_desc} + {m_desc}" + +# ========================================== +# Main Generation Logic +# ========================================== + +configs = [] +readme_content = [] + +# Group 1: Constant Keyboard, No Mouse (0-7) +keys_basic = ['W', 'S', 'A', 'D', 'WA', 'WD', 'SA', 'SD'] +for k in keys_basic: + configs.append(((k, k), ("stop", "stop"))) + +# Group 2: No Keyboard, Constant Mouse (8-15) +mouse_basic = ['up', 'down', 'left', 'right', 'up_right', 'up_left', 'down_right', 'down_left'] +for m in mouse_basic: + configs.append((("", ""), (m, m))) + +# Group 3: Split Keyboard, No Mouse (16-23) +split_keys = [ + ('W', 'S'), ('S', 'W'), + ('A', 'D'), ('D', 'A'), + ('W', 'A'), ('W', 'D'), + ('S', 'A'), ('S', 'D') +] +for k1, k2 in split_keys: + configs.append(((k1, k2), ("stop", "stop"))) + +# Group 4: No Keyboard, Split Mouse (24-31) +split_mouse = [ + ('left', 'right'), ('right', 'left'), + ('up', 'down'), ('down', 'up'), + ('up_left', 'up_right'), ('up_right', 'up_left'), + ('left', 'up'), ('right', 'down') +] +for m1, m2 in split_mouse: + configs.append((("", ""), (m1, m2))) + +# Group 5: Constant Keyboard + Constant Mouse (32-47) +combo_keys = ['W', 'S', 'W', 'S', 'A', 'D', 'WA', 'WD', 'W', 'S', 'W', 'S', 'A', 'D', 'WA', 'WD'] +combo_mice = ['left', 'left', 'right', 'right', 'up', 'up', 'down', 'down', 'up_left', 'up_left', 'up_right', 'up_right', 'down_left', 'down_right', 'right', 'left'] +for i in range(16): + configs.append(((combo_keys[i], combo_keys[i]), (combo_mice[i], combo_mice[i]))) + +# Group 6: Constant Keyboard, Split Mouse (48-55) +complex_1_keys = ['W'] * 8 +complex_1_mice = [ + ('left', 'right'), ('right', 'left'), + ('up', 'down'), ('down', 'up'), + ('left', 'up'), ('right', 'up'), + ('left', 'down'), ('right', 'down') +] +for i in range(8): + configs.append(((complex_1_keys[i], complex_1_keys[i]), complex_1_mice[i])) + +# Group 7: Split Keyboard, Constant Mouse (56-63) +complex_2_keys = [ + ('W', 'S'), ('S', 'W'), + ('A', 'D'), ('D', 'A'), + ('W', 'A'), ('W', 'D'), + ('S', 'A'), ('S', 'D') +] +complex_2_mouse = 'up' +for k1, k2 in complex_2_keys: + configs.append(((k1, k2), (complex_2_mouse, complex_2_mouse))) + + +# Execution +print(f"Preparing to generate {len(configs)} action files...") + +for i, (key_seq, mouse_seq) in enumerate(configs): + if i >= 16: break + + # Generate Data + kb_arr, ms_arr = generate_sequence(key_seq, mouse_seq) + filename = save_action(i, kb_arr, ms_arr) + + # Generate Description for README + description = generate_description(key_seq, mouse_seq) + readme_entry = f"{i:02d}. {description}" + readme_content.append(readme_entry) + + print(f"Generated {filename} -> {description}") + +# Write README +readme_path = os.path.join(BASE_OUTPUT_DIR, 'README.md') +with open(readme_path, 'w', encoding='utf-8') as f: + f.write(f"Total Files: {len(readme_content)}\n\n") + for line in readme_content: + f.write(line + '\n') + +print(f"\nProcessing complete.") +print(f"64 .npy files generated in {VIDEO_OUTPUT_DIR}") +print(f"Manifest saved to {readme_path}") \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/validation.json b/examples/training/finetune/WanGame2.1_1.3b_i2v/validation.json new file mode 100644 index 000000000..5af4cca74 --- /dev/null +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/validation.json @@ -0,0 +1,404 @@ +{ + "data": [ + { + "caption": "0", + "image_path": "../../../../mc_wasd_10/validate/000000.jpg", + "action_path": "../../../../mc_wasd_10/videos/000000_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "1", + "image_path": "../../../../mc_wasd_10/validate/000001.jpg", + "action_path": "../../../../mc_wasd_10/videos/000001_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "2", + "image_path": "../../../../mc_wasd_10/validate/000002.jpg", + "action_path": "../../../../mc_wasd_10/videos/000002_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "3", + "image_path": "../../../../mc_wasd_10/validate/000003.jpg", + "action_path": "../../../../mc_wasd_10/videos/000003_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "4", + "image_path": "../../../../mc_wasd_10/validate/000004.jpg", + "action_path": "../../../../mc_wasd_10/videos/000004_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "5", + "image_path": "../../../../mc_wasd_10/validate/000005.jpg", + "action_path": "../../../../mc_wasd_10/videos/000005_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "6", + "image_path": "../../../../mc_wasd_10/validate/000006.jpg", + "action_path": "../../../../mc_wasd_10/videos/000006_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "7", + "image_path": "../../../../mc_wasd_10/validate/000007.jpg", + "action_path": "../../../../mc_wasd_10/videos/000007_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "00. Hold [W] + Static", + "image_path": "../../../../mc_wasd_10/validate/000000.jpg", + "action_path": "action/000000_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "01. Hold [S] + Static", + "image_path": "../../../../mc_wasd_10/validate/000001.jpg", + "action_path": "action/000001_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "02. Hold [A] + Static", + "image_path": "../../../../mc_wasd_10/validate/000002.jpg", + "action_path": "action/000002_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "03. Hold [D] + Static", + "image_path": "../../../../mc_wasd_10/validate/000003.jpg", + "action_path": "action/000003_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "04. Hold [WA] + Static", + "image_path": "../../../../mc_wasd_10/validate/000004.jpg", + "action_path": "action/000004_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "05. Hold [WD] + Static", + "image_path": "../../../../mc_wasd_10/validate/000005.jpg", + "action_path": "action/000005_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "06. Hold [SA] + Static", + "image_path": "../../../../mc_wasd_10/validate/000006.jpg", + "action_path": "action/000006_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "07. Hold [SD] + Static", + "image_path": "../../../../mc_wasd_10/validate/000007.jpg", + "action_path": "action/000007_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "08. No Key + Hold [up]", + "image_path": "../../../../mc_wasd_10/validate/000000.jpg", + "action_path": "action/000008_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "09. No Key + Hold [down]", + "image_path": "../../../../mc_wasd_10/validate/000001.jpg", + "action_path": "action/000009_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "10. No Key + Hold [left]", + "image_path": "../../../../mc_wasd_10/validate/000002.jpg", + "action_path": "action/000010_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "11. No Key + Hold [right]", + "image_path": "../../../../mc_wasd_10/validate/000003.jpg", + "action_path": "action/000011_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "12. No Key + Hold [up_right]", + "image_path": "../../../../mc_wasd_10/validate/000004.jpg", + "action_path": "action/000012_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "13. No Key + Hold [up_left]", + "image_path": "../../../../mc_wasd_10/validate/000005.jpg", + "action_path": "action/000013_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "14. No Key + Hold [down_right]", + "image_path": "../../../../mc_wasd_10/validate/000006.jpg", + "action_path": "action/000014_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "15. No Key + Hold [down_left]", + "image_path": "../../../../mc_wasd_10/validate/000007.jpg", + "action_path": "action/000015_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "00. Hold [W] + Static", + "image_path": "../../../../mc_wasd_10/validate/gen_000000.jpg", + "action_path": "action/000000_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "01. Hold [S] + Static", + "image_path": "../../../../mc_wasd_10/validate/gen_000001.jpg", + "action_path": "action/000001_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "02. Hold [A] + Static", + "image_path": "../../../../mc_wasd_10/validate/gen_000002.jpg", + "action_path": "action/000002_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "03. Hold [D] + Static", + "image_path": "../../../../mc_wasd_10/validate/gen_000003.jpg", + "action_path": "action/000003_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "04. Hold [WA] + Static", + "image_path": "../../../../mc_wasd_10/validate/gen_000004.jpg", + "action_path": "action/000004_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "05. Hold [WD] + Static", + "image_path": "../../../../mc_wasd_10/validate/gen_000005.jpg", + "action_path": "action/000005_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "06. Hold [SA] + Static", + "image_path": "../../../../mc_wasd_10/validate/gen_000006.jpg", + "action_path": "action/000006_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "07. Hold [SD] + Static", + "image_path": "../../../../mc_wasd_10/validate/gen_000007.jpg", + "action_path": "action/000007_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "08. No Key + Hold [up]", + "image_path": "../../../../mc_wasd_10/validate/gen_000000.jpg", + "action_path": "action/000008_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "09. No Key + Hold [down]", + "image_path": "../../../../mc_wasd_10/validate/gen_000001.jpg", + "action_path": "action/000009_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "10. No Key + Hold [left]", + "image_path": "../../../../mc_wasd_10/validate/gen_000002.jpg", + "action_path": "action/000010_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "11. No Key + Hold [right]", + "image_path": "../../../../mc_wasd_10/validate/gen_000003.jpg", + "action_path": "action/000011_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "12. No Key + Hold [up_right]", + "image_path": "../../../../mc_wasd_10/validate/gen_000004.jpg", + "action_path": "action/000012_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "13. No Key + Hold [up_left]", + "image_path": "../../../../mc_wasd_10/validate/gen_000005.jpg", + "action_path": "action/000013_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "14. No Key + Hold [down_right]", + "image_path": "../../../../mc_wasd_10/validate/gen_000006.jpg", + "action_path": "action/000014_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "15. No Key + Hold [down_left]", + "image_path": "../../../../mc_wasd_10/validate/gen_000007.jpg", + "action_path": "action/000015_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + } + ] +} \ No newline at end of file diff --git a/fastvideo/dataset/validation_dataset.py b/fastvideo/dataset/validation_dataset.py index 5ab467d75..8d831d5df 100644 --- a/fastvideo/dataset/validation_dataset.py +++ b/fastvideo/dataset/validation_dataset.py @@ -4,6 +4,7 @@ import os import pathlib import datasets +import numpy as np from torch.utils.data import IterableDataset from fastvideo.distributed import (get_sp_world_size, get_world_rank, @@ -160,5 +161,25 @@ class ValidationDataset(IterableDataset): else: sample["control_video"] = load_video(control_video_path) + if sample.get("action_path", None) is not None: + action_path = sample["action_path"] + action_path = os.path.join(self.dir, action_path) + sample["action_path"] = action_path + if not pathlib.Path(action_path).is_file(): + logger.warning("Action file %s does not exist.", action_path) + else: + try: + action_data = np.load(action_path, allow_pickle=True) + num_frames = sample["num_frames"] + if action_data.dtype == object: action_data = action_data.item() + if isinstance(action_data, dict): + sample["keyboard_cond"] = action_data["keyboard"][:num_frames] + sample["mouse_cond"] = action_data["mouse"][:num_frames] + else: + sample["keyboard_cond"] = action_data[:num_frames] + except Exception as e: + logger.error("Error loading action file %s: %s", + action_path, e) + sample = {k: v for k, v in sample.items() if v is not None} yield sample