From 2133f715e763713ad3af07e57d12ecb1839c79df Mon Sep 17 00:00:00 2001 From: Vito Sansevero Date: Thu, 22 Oct 2026 06:35:24 -0700 Subject: [PATCH] feat(inspect): bounded browser video tag readers Add web/js/workspace/local-video.mjs: ISO-BMFF (MP4/MOV/M4V) and Matroska (MKV/WebM) walkers that read only the container tags the server asks ffprobe for, slicing box and element headers, carrier names and retained values within the server's budgets and never a byte inside mdat or a Cluster; a port of utils/video_metadata.select_video_carrier with the record budgets of read_inspection_file; and a byte-exact json.loads/json.dumps round trip so envelope objects reserialize as the server spells them. Every naming rule mirrors ffprobe 9.0.2 on hand-built files: legacy ilst atoms, mdta keys, joined repeats, data types, Matroska language and default handling, and Tags behind a Cluster found only through a SeekHead. Tests: node units over byte-exact container builders and the committed fixtures; a Python parity test comparing the local carrier with the server's selection from the same tag maps (mocked probe) and from the same files (real ffprobe, including 1 MiB + 2.5 MiB per-key records); an ffprobe characterization of the recorded fixture maps. --- tests/fixtures/README.md | 39 + .../fixtures/local-inspect/core-faststart.mp4 | Bin 0 -> 3268 bytes .../fixtures/local-inspect/core-moov-end.mp4 | Bin 0 -> 2837 bytes tests/fixtures/local-inspect/core-tags.webm | Bin 0 -> 1718 bytes tests/fixtures/local-inspect/core.m4v | Bin 0 -> 2845 bytes tests/fixtures/local-inspect/core.mkv | Bin 0 -> 2489 bytes tests/fixtures/local-inspect/core.mov | Bin 0 -> 2849 bytes .../fixtures/local-inspect/legacy-comment.mp4 | Bin 0 -> 2940 bytes .../local-inspect/legacy-comment.webm | Bin 0 -> 1888 bytes .../fixtures/local-inspect/video-tags-v1.json | 60 ++ tests/js/helpers/video-bytes.mjs | 277 ++++++ tests/js/support/local-inspect-oracle.mjs | 22 +- tests/js/workspace-local-video.test.js | 484 ++++++++++ tests/test_local_inspect_parity.py | 454 ++++++++- tests/test_utils_video_metadata.py | 39 + tools/vendor/frontend-coverage-baseline.json | 8 +- web/js/workspace/local-video.mjs | 896 ++++++++++++++++++ 17 files changed, 2256 insertions(+), 23 deletions(-) create mode 100644 tests/fixtures/local-inspect/core-faststart.mp4 create mode 100644 tests/fixtures/local-inspect/core-moov-end.mp4 create mode 100644 tests/fixtures/local-inspect/core-tags.webm create mode 100644 tests/fixtures/local-inspect/core.m4v create mode 100644 tests/fixtures/local-inspect/core.mkv create mode 100644 tests/fixtures/local-inspect/core.mov create mode 100644 tests/fixtures/local-inspect/legacy-comment.mp4 create mode 100644 tests/fixtures/local-inspect/legacy-comment.webm create mode 100644 tests/fixtures/local-inspect/video-tags-v1.json create mode 100644 tests/js/helpers/video-bytes.mjs create mode 100644 tests/js/workspace-local-video.test.js create mode 100644 web/js/workspace/local-video.mjs diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index f44f20c1..34f1409c 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -28,6 +28,45 @@ ffmpeg -f lavfi -i testsrc=size=48x32:rate=1 -frames:v 1 -q:v 10 tiny.jpg ffmpeg -f lavfi -i testsrc=size=40x24:rate=1 -frames:v 1 -c:v libwebp -quality 40 tiny.webp ``` +# Local Inspect video fixtures + +`local-inspect/core-faststart.mp4`, `core-moov-end.mp4`, `core.mov`, `core.m4v`, +`core.mkv`, `core-tags.webm`, `legacy-comment.mp4` and `legacy-comment.webm` are +0.2-second 16x16 black clips (each under 4 KB) carrying an original synthetic +ComfyUI graph in their container tags, dedicated to the public domain under +CC0-1.0. The `core-*` files carry one tag per key as core SaveVideo/SaveWEBM write +them (MP4/MOV/M4V through `-movflags use_metadata_tags`; Matroska upper-cases the +names); `core-faststart.mp4` also carries `parameters` and +`comfyassets_prompt_pairs`. The `legacy-comment.*` files carry the old +VideoHelperSuite `comment` JSON envelope (an ilst `©cmt` atom in MP4, a `COMMENT` +tag in WebM). `video-tags-v1.json` records `ffprobe -show_entries format_tags -of +json` for each file (ffmpeg 9.0.2); `tests/test_utils_video_metadata.py` +`VideoFixtureFfprobeTests` re-checks it when ffprobe is installed, and +`tests/js/workspace-local-video.test.js` plus `tests/test_local_inspect_parity.py` +hold the browser video reader to the same maps. + +Reproduce with FFmpeg, from `tests/fixtures/local-inspect`: + +```sh +PROMPT='{"3":{"class_type":"KSampler","inputs":{"seed":7,"steps":20,"cfg":8.0,"sampler_name":"euler","scheduler":"normal","denoise":1.0,"model":["4",0],"positive":["6",0],"negative":["7",0],"latent_image":["5",0]}},"4":{"class_type":"CheckpointLoaderSimple","inputs":{"ckpt_name":"synthetic-v1.safetensors"}},"5":{"class_type":"EmptyLatentImage","inputs":{"width":512,"height":512,"batch_size":1}},"6":{"class_type":"CLIPTextEncode","inputs":{"text":"a red fox in snow, detailed fur","clip":["4",1]}},"7":{"class_type":"CLIPTextEncode","inputs":{"text":"blurry, lowres","clip":["4",1]}},"8":{"class_type":"VAEDecode","inputs":{"samples":["3",0],"vae":["4",2]}},"9":{"class_type":"SaveVideo","inputs":{"filename_prefix":"video/synthetic","video":["8",0]}}}' +WORKFLOW='{"last_node_id":9,"nodes":[{"id":3,"type":"KSampler","widgets_values":[7,"fixed",20,8.0,"euler","normal",1.0]},{"id":6,"type":"CLIPTextEncode","widgets_values":["a red fox in snow, detailed fur"]},{"id":7,"type":"CLIPTextEncode","widgets_values":["blurry, lowres"]}],"links":[],"version":0.4}' +PARAMETERS=$'a red fox in snow, detailed fur\nNegative prompt: blurry, lowres\nSteps: 20, Sampler: Euler, CFG scale: 8.0, Seed: 7, Size: 512x512' +PAIRS='{"version":1,"nodes":{"6":{"records":[{"positive_text":"a red fox in snow, detailed fur","negative_text":"blurry, lowres","negative_effective_text":"blurry, lowres","prepend_text":"","append_text":""}]}}}' +COMMENT=$(PROMPT="$PROMPT" WORKFLOW="$WORKFLOW" python3 -c 'import json,os; print(json.dumps({"prompt": os.environ["PROMPT"], "workflow": json.loads(os.environ["WORKFLOW"])}))') +SRC=(-v quiet -y -f lavfi -i "color=c=black:s=16x16:d=0.2") +ffmpeg "${SRC[@]}" -movflags use_metadata_tags+faststart -metadata prompt="$PROMPT" -metadata workflow="$WORKFLOW" -metadata parameters="$PARAMETERS" -metadata comfyassets_prompt_pairs="$PAIRS" core-faststart.mp4 +ffmpeg "${SRC[@]}" -movflags use_metadata_tags -metadata prompt="$PROMPT" -metadata workflow="$WORKFLOW" core-moov-end.mp4 +ffmpeg "${SRC[@]}" -movflags use_metadata_tags -metadata prompt="$PROMPT" -metadata workflow="$WORKFLOW" core.mov +ffmpeg "${SRC[@]}" -movflags use_metadata_tags -metadata prompt="$PROMPT" -metadata workflow="$WORKFLOW" core.m4v +ffmpeg "${SRC[@]}" -metadata prompt="$PROMPT" -metadata workflow="$WORKFLOW" core.mkv +ffmpeg "${SRC[@]}" -c:v libvpx -metadata prompt="$PROMPT" -metadata workflow="$WORKFLOW" core-tags.webm +ffmpeg "${SRC[@]}" -metadata comment="$COMMENT" legacy-comment.mp4 +ffmpeg "${SRC[@]}" -c:v libvpx -metadata comment="$COMMENT" legacy-comment.webm +for f in core-faststart.mp4 core-moov-end.mp4 core.mov core.m4v core.mkv core-tags.webm legacy-comment.mp4 legacy-comment.webm; do + ffprobe -v quiet -show_entries format_tags -of json "$f" # -> video-tags-v1.json "fixtures"[f] = .format.tags +done +``` + # Inspection parity fixtures `inspection_parity/` holds small synthetic PNGs (`f01b.png` ... `f24.png`, each diff --git a/tests/fixtures/local-inspect/core-faststart.mp4 b/tests/fixtures/local-inspect/core-faststart.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5ba7aa9cd1c42e1a49161e54bbfb71c0833e53cb GIT binary patch literal 3268 zcmbVOO^6)F74FqWB3g_sjKDhRQ0eI4!|ZJL%k(W zPhB;$Gg=nxV1p07_#&_m(me!SbBZk)1q43$8i6myAvh!?w_phN=6l`KyED5AGAX8} zUcJA1_5P|2!(gGF17Qeyfv9Vy# z?{Lg*fIhN+^^o&v&369>q1@w~fi$_=tV>t@qqJ;1_l7XmUTCZcDZSgdS z1@SC=mdG>&aCVfGN@vZy5LC4DK_Q5T2GO4z7BQ>fh2JXOivfS4w5kI_JohPRSZU1a ztT1PA^(}xjX)K(*2L5ilX*X@F#bVhTx4NB*R{fvb-v`hE*TzC%EbV_~ywu0~`zODC z@S0&Pl9v{Z$FFSN`Q?IfD*F4^pPLW9`}SLsJ(F{lVf%v-{yC*f_k&#W zoTT(4UfhbpLZg;n_YZl&M|d1wu1nLk-qV)orLObP<(HU4T~m4EWzcR^%YtGPa zCo2YFob6!$VoBu%#}T3+Ipm<`ewKt2u9VQqt7N^L%aZ;A(M9TK0j1f=Z)BO)i&Q$~ zkN*U`v=y*n?3!)U*){zbsyvjxFdg%aZJy*vQL|ynG#_e397>45bk-WC(jo_@Wi?Dc z?3>Pd6OgJTycAYKgBVug%8x`)!j73{MZ#m~3`Ckq1vZ-`6IdCW&JD9;HmsWsGtZRN zas=`Xvs84QM%fn~g~LgFKU|zLs%#WLwhZ=I#c% zu1!~yR7XNfe|cm#6%XO`RAq%SN&D(C?VAYNRP+>&f5QROfMc5#iYTIjfGH8)4G04>-9^UuL;V(IlHSQLGIi@(v`s;@iKY{ zeXF$Z;xSFzd?d=YIj1SaB`MXA`685KID=YkwsH_WaF-Z5YrV>iJ!JnY$awUFh0c5l zAo&r0nCnU8_{9ctHNaJ!8@nbU+70t~WgxbFs=X19hb1pc1E!!z8YqpjDC#PxGavbW zbFWe9>>lVm7U*M|pBd=`t*~Kg=?~C)B(OL4P<^pX2Xt|OBQzVCrKV#wJMf+H5hk^e z4`+S%j2vA0`LuLd)uJ3Wf9TRSH4}%yV{F=V9JWafqro;_zrvKyW8pCJCfh~>a@ZOG zl#RnsRAWpS^cPHO`d`i(wtHy5eGt#xs?G|eW)W2Jt=od6AUXqdvo*3 za)({Iz5&ia_#kX#`6L#hMzE~*a?5J10vTzYJ1Z+YJ3Gy?-eR6Mv!cI3r!*s-#Nf$t zjXXk>ect1~Yol$2ZX3O-7ia#!wH?c`7*BaTQNp#xZO0nhcAF)_jl`I#VGqz6%hd#; z`1gvyMI}K9e=MPr^GKr&39sO3U$}Oc`4Qez?m^UMx)5*?}iW6fp-A@RT-u;^7(3xyIgyi}0;0Q>XxA!HFlNS6{(dJujaCmVt{{c|71b z-J<99Bv*8ZL{1ZCr+j64fHLHf*UxfLD`XB)e0V^&+gi1zpocF|Y*A<;75-4W9gEB? zIJp$A9D<8Yvi$KtKgWF*; zN05*Zh#_?GTZlhJ(|8*h{DOG@U8BAK{lEY5$;s~7#}6r~FMWI=zWnL;4@&s%QSBk83BZPQ?i5en5J#5$ZibLP%?=ao6< zzFGoR6#NUia8dUn-MX?5sSt22SkQ$CqA2Q03ev{kH*;^E1;s<=e&6|C=X;&+L{Svg zm!m8cX{;zm6oU9H)_2V~tDB0Vyx`}YgZCu%m;}7@+_CNLPrqCF<%^#lt$uz>|5knU z^6!5Pt!7=FSKTz{s@ZI)dCjscwOwm8jRpf_wF-fS*DtTUI$u}MUtNZz$6YX%(`*!R zU#ga2)aEV2YJeC>nY9)c27^JhA9_5ESW->%&H|lM4P+caCe37+CZeUftj%28RCDgz zHPz$oD0REG*)m#&$`Tfh1h13iCW+YJ>*vOpJ_XU^syrs$@hXxsn(zMG0Y@A)1h(Z^>ObCXW>02~0pRXC&k2Ean2qYCG8oc<9-vmBT!iQ50>b9Wp_O zggGzb8t||a$ToOs#uKNLX5f~@460o|f^FMs7-Q6l!vtGZxIE!*Pug{Z+{_uJH0L6K zdhR+q;SxW0Z8!JTxQ)D0SlHNBt!k(n6pU?Dn?N%%TtGIon`b}}l4o{Z4Kox84S^cM z7TZAmshY}7bnr?3(FaOx`{`f5-96Tv*?K@teg5OQ=)%3H_Y3^?h2e+W58flZa+}Eu z_a5FXara%|pg;d-oTh!yqPQP;fP2=|&)94!6@~tG*S-zN2lnT7C7*>I!A1K7`RaoC zw(J&XPVEt#Slik|E71vO!Z@%$NA&6OrzmsWlLEV09tm9TVvz>y9v%C9!Djn z8H@NQdU>Rlc;$T|+Y#`)LW)U2NahYrERkLaumKud^ znlL4={5`$B4WtgK<%*{q-~K^4)dBqM(L47qD#{dPX-e5TyL#)bit=3W>yKY+_n$m| zOuEw<6B!_RLEj7-dz|9_7OkU2&(m}C(Zm4((sf63jIg&fgwcma=(;bc!+ z;up4NVaqKjySB#q$}8YqAM*&%`CkE^IPk$AHzffq^2fm0-}gMp0KbVj!D79%w*}&Y zKjw^qjZrZPjQaUS63Q@BGfK@g?FMPy^`mrv{Yl#Mgq{jC&CzaHH0cy6QgnagL86m^ zE6NS6rnPQpZp4Ie@S<_8rM-5A#Tgn-)3q?kdQy;t;M~(%i@HXW0~E{9HP`QGt)(h3 zQEE5|CL}EIUa2nJfO`er(vmcfSp=INPts6843kFJVt5&8t#z%g>Bco(%Tf`_un+dS z)+|X0?_friq{WgHVTLEt!7T3-k_{qnZtAddFq-9nyWK3s99>HpGIAxP;P%A^zMQx& zMu`l#4Bh#@SryFZI7pGo7V9pa5pMUoD5JfV_>5m9?oJvR!o zaca#XJB$D07k&Z`pfG6l#R+kQ0y?TD-BB%b_=Xcp8=$&;mT~X{lE2@XmTZMmT z4q3Xw`n(u|`z-kgl3LBlFzUh~l2QD>v#?VM_iLMa zX|uUw^H8A=S^n2b?^uNoV^9Bs)dPjSwu##pg-MqJ2ioVkz<-Jw{cL MbBA}ppS%PA0Qq6w=>Px# literal 0 HcmV?d00001 diff --git a/tests/fixtures/local-inspect/core-tags.webm b/tests/fixtures/local-inspect/core-tags.webm new file mode 100644 index 0000000000000000000000000000000000000000..c6a709d581883c2512b97cce919ad3e5d23705c5 GIT binary patch literal 1718 zcmcIl-)kd99G^t*=!0Sulp=y*^g)g_m!#Jpw;(j>Mk$vxA?Xjbl(^ZQY=&-jmz_mFHN=S9bA~a09EbEL@ zMTUt9GmG$@Wm82u6qn%{D1=u8Xqf~%xXPB7;e-bnNR#`}qzCD;gA6B+%d+NLj&rzP z7Z}UE-prJcvTB*waFV91B0GnrlGB?MWJf9G34WDFffG**5zi!NGxMnCBn*W` zXh5#RU>0OKXd^XPG;2X@1c_m;2|*P(GS)d-;n`DkZ1qALaicT&AC4*L7294Hg1Wh8 zBXS(;)DY`pEFqyG-CmKf+F52r=O()MM1@m8((7ApVGhnf;ui#t(talm1RL%$j0Vp({ob$QaCKSzfd&Xd%%-3 zA6KTE-%`-+QUDqOKX~7FDgXiiyfD&gbz04i-wz&m@D=dKpVu;t?%NbR1FrkPiNFpR s1*0#X{uPV>z~RjCaP4^b$}r5o9B=Jk{D^{2-e6y-pEgf?^CP|QFP$kN8UO$Q literal 0 HcmV?d00001 diff --git a/tests/fixtures/local-inspect/core.m4v b/tests/fixtures/local-inspect/core.m4v new file mode 100644 index 0000000000000000000000000000000000000000..6c59992280136064ac43e9fe12bb629062eb04b2 GIT binary patch literal 2845 zcmbVO-D@0G6rVJ0l~QaT3Kk4FPV?ZyW-~MUk!0yW+ccPBiiK(rVjb?z+}Rm-zB2dj zS4*IRg7^pY!H2$yAmXEfplzX*pbtI?7W6>`Q4|FQDM%ZCXLdK)Yzm4uxpU9g{oQlV zIrrXG6h-x9FTGG+Qx)ZaLSrD32;F3D$50gIaWCT>x(AWVB*qU9AKKo2|J&uCKmGCE z%E#9W->COa{`Py!nInuP`W9iZGbFaB=y$*|K_OZ3&hx zcc55G(q70tshYZ8nl*K^0--NuTAQ2ebUMX$;PND7aWTmnbF@m)mr)3tB$YuDi<;`N zI&&;T&A4ZkRF~Jo#A#YaO|R)Hi&@wc+|s+Hn%*^xk{WT#=Ur8_>Y(P;RP-js_OVz5-}T74OM153Zzz_pMoo4G8d zUDWM*zyvK4WIW$jhX)N`)}c#M9@~v1g|;tDF}2Biz_!eaJ|OKVh{3AD;W2kw(kkoZ zX2vL_85cgLXO6wIUD9Wc0LLU}^3*D; zL5d`yAdo{~u?^&(vZ-7{1s~&Yy`hx0AO7|0XNRiCw{BBWpLu5{JbUxut(?Abw)^Jx z?bnH(xWVMvn|H4DY3DW2kUw@mN|H8YVbu0rj3aIGCoo&eghKzjZ4U$S7=L=#@=??Q z9JKckuO^hQ$Y#FgztxYQj{i@ty3%4U!G77`@j6~rFQGV{riM6-D(AH z(?9)(J@dldPZr;O>Z{33}~*Jdf$gK=B-G26?eg z(nQ+b-*}j4WzdRpRV!(=tC|xsA#A*CT&ro%Ut&>;!qW;`5T`9E$U<=LYPI=-MmGo~ zre4q-uc6fzil9V)h8^Rk1OjjMrwhmDZcf*##wlnKybQJ4x>nW- z`sIR_CL)kQ8|rnf+84#VfxD_Nn(vE3+~u*fai=$O%L<7%Hw!=+-_4TGoo1TgK3z>1 zB62C9&F$G6^m6FB=*80KGH_CbKJ;azptqWPb3S)wG-jQ^mA+Q1 z7-m89^EY9@)R}aATLf1q3Itr;2fVa;{^EUAz^lh?7nM zPpM=b)G<5SH=dIrZ literal 0 HcmV?d00001 diff --git a/tests/fixtures/local-inspect/core.mkv b/tests/fixtures/local-inspect/core.mkv new file mode 100644 index 0000000000000000000000000000000000000000..3cb0e60eee7f2d537395ae4c0fc1b5145703b274 GIT binary patch literal 2489 zcmb_eZEO=|9KVj2Fu#Dr7h*7;2PVdJ-Cett(Rc~#T3}&g%eoEY#LLsWr|q%p-R+*I zUCUgu+X|aP0tp5pA`#~obs-o7s8crLOGfmA7jX zlW;g(tzRp%aS4y~|9vofE=-lGkL-HxiHmQKtc-4I9opZ5@~LnrAO$2bxTA5=mb`xO zz>2mnmJXj;vgWZ@c3s+1NX4EX+oYGRV*YG8JXL!1nd_@g&RIQr(e;r}#tI*N`rDEd z=iWW$a>D6jBRNwWbGf|-x|P=U_?Eib=0sasxnaq*RrBn)~M9DR$=O$$D)ek)Ym3_8h6j^x7>qi_@21@>~xIdmT*hiW9G@NhoACs%QIK6 zd?rl)dGjX6q8mFsRa$$arG}7LhdMo!o{8R!kMC{oXlrfnjEzRG4hSJ3G9ai1B1EA@ z8w(L(TLNWm13Q9W&`mo>36mfgYeHm$Umz6QfCNRqpr+D7q#*z+Qr1vRlw}M!S5_x# z25a6rBAAwwMFwzc*tB#4ViMELS{gQlNRLn__{E)m!M2D_^*kK+2;nkh;xw9vHk2U) zQEXC0&!V&^S#ob8eAua8BnK;-L5mt04?~Q&aQw_X~^b80L2>FG|rhv4jF(pbK897fDx&o|#{_sOrc>OTn0P{r; zrM`2}U*-%ia8~5Fg83hlyub=J%vt&`SntZ|&OvBALpOVw!fy}5ej_bxXcz2I`(5XV)WT)u2}_RX*2KYsY#<*jEA z`!DgA*ImECX!#mmwH(Y#;dPRYFLGXCd)1w zI7NB5I#d%BgY^JrD77QCwf+750k4P*WCkoJUCV9>WN6j^8Ox>+Arj$Lltij5K`*A{ z5Hw)Yu+&~zihxf%GLcavSQZPRh**%M5TC_z1{ZjeOTuacZ<8XZ!c%cH8EOQS;K65E zF)55u0u9AMW*W;4uN<4M2Vdw-_4TT0xrLa)@va$#!AZ@p0VmT=JC5cb5 z0=G%8ZZnB8L7%NysT9Fft_kuq6=Lhp4*%aoL}xBU7a+`}4ZV1Mj%p*%#nX+RwwwH~-{T F{{>4ZIzRvb literal 0 HcmV?d00001 diff --git a/tests/fixtures/local-inspect/core.mov b/tests/fixtures/local-inspect/core.mov new file mode 100644 index 0000000000000000000000000000000000000000..a44a6123f27b004f1b72e783df9c2ff8b19d2aea GIT binary patch literal 2849 zcmbVO&1)T16rVJ$gj#GC0lVm89J6pSd6}8_k@V4lwrOgLDHJM2h;_Jc=Ds)Mo3G5h z@2e$PLBYSEi!S;H6mi>?wgrWNYr%>xL=Z*MofM>vzcceDc}WV2!<)J1>;CS!=bU@) zQxs*cEl265R8>Veq7(uCBhhP*cxeUxB5^T_{$PbQJQoR4v1( zE?9G*y?im}{GA#@lvP^>`~x z+^%glji#Zpn1v(3ZDUw%8bi~psu8z+K2$}o1!_r6#RyaJbu!O3D+VUvqej7ydk!I* znBinB?r_^|sIH$S5pyurRAt7)Fc6@Yhs&NTLAaZdZNLTQ-ArO`Tc&QBYMTk^q@o+7 zWKjTYrcTms3oh*iOO<|x83Yt2NtgLxoIRNc0~fxO0z-|nd<{29&@)R?V3SuW{g6bap7Zn<~qCE zC4J`FZq`<#7UD|V0%Kd%ilJ`OW^AL<0GSeSKC+?RI0J!@Jhf|TkRnMa2;>l0Y!msX zY$~@EMLEeo`ADhmJpb$WFOD_lwjV<@bN=ppc;VsmM>&1(!tmpr#~%_sb&tsl55K%! z(C+)7A)lFvlB5q=81;P*zB%!rEoFr#CeOUIuR*XbOuvDgbajHbRrBHPf-^`jhg(oMYt@&MG!LUj`8 z<%>+RT^wcJD{xMGsI0LaVVxMy13iTk&cm^-!&)ZD3>l>xk{Zce`gV< zHIVeuGc~sk`$n~5R!qZE!=OdosL&PPQ#%xh8mv~PJmvV#kIJhZjGsPv@6km?nWl(N zE8Az+?z}UlyyXA()7RRgXHTEf+}V_g6eD>>?wg96#DN$7xXIX#Fy^uL4?RyZ zjNe3@#{7A$%qVeDt}hhj*G3B|H%4{-3K^y7wJBBQEOCP*>$byW0RANLJfB3Mf&`aAI^$An;x>UAR8?a=NL-NfxmXm>!RlK)?(gNY^5G8EVZ9 zt)}V54P8qU5y+qq^@i3cL^1E6cNL2n#l3w^U9Ses)Zkz_(6^!g$6Rit3Fo_ljScV;qX_-i12 ztywoMUGwuhV$8IdbbUtzHz^7P+&BchvVQsM+k7Zj^E~czS3-;+GhFGO+D?YJJ5&)T z107eZWI;&M9%auBgLE8PGmp;F|5%2u%xt9N@*Uu6dJ*dKA=K;VS1)p^zkPleMS|Md zK+)yN?z235NWGQ&cjge%HP+{OZ`_B}#wMxMoD^*>7$O+?|DeU)nsfEqueqgFj^wYhOiBSBTy4xbFZ z%+6$4h@#+Mkb{SKj!3Sa%tIn9_zxKLA^}kpy-6;~`un=4vy;t$g2ncH??-*#S5<2m z25akKCRG|6#vX$pK8N>xJI>~8!!QoEb0Hvm76)7d-aW8?d;80umVf)^m&Yq#-}a?^DM7Bj>B5@`Gz&mAy}zFVfO9I%WuxivGZ4#pcx1sf~7PY zMxw2mV_EeX$8zSu47JV{XJ^;f*K2(lh&19!EzLW#6r~pGID$@^X_+Q!k@>vEeb;8W zXuEY5h*p&PUDsZ;7A?jT9u1XntwDX!8rXK7#lj86fT>;!*lCujAx!b><$-J0EEwU3 z#d075k38D2@N%AXgljjLALePyJ($`|=OT)v0=6(%2z(91zaG05QsBXjG!d?2Pun(Y zbEUmZb!A2xrGx94m$utVXm`e8I?Q1~j-oW}@(_f#!&8y;kxRv}Sdtepd>M01dB{YI zJmOFSzZLazKJ@%F&bS8cqeMFAGJy*y=A3leIgf=xv07d>1TF&?z4Ca#GulPVYe}vs zh|EQ?uXQ0ip>9EzW+L%AX$EP@%wXCTL-=-`d257vu}nZR<%>l4J?+j}lxEJUq`6Qb z%yZv+v0akqzU$|07PnAW+7<}csn;xaoi^iIwFc0P92bxe+{PI&lopvg$7F^kVIa^$ z_~O^mf9j@j6BB$yeD;Y^-+uo0ANTe*j%_`pp+5ilsp!J}=MM_}_JzTx+Ydh?eCjsW z7w$j0S>pbOz`;NGUYw?V@S?aM27qI0@2?=Xj46ZucGgP-*t_U+JDQWQJveAzpk7@F zo4Q-X+&iXNdD|M}m1rfY1iR>S#6LRv4C9mtv;uWfL<)zySY!de`$jqMIUWa+gU8}v z_ud__4;k-1D(q&$Aj%62KXhy>oZwYD#w$DtB0)M+-=dLiAjkS~;dDIBQ$Z;^8DLt- z7>neGdwIl4eClJRTM_VkN~@|Nq;r>UEItwO(1uh})*a#$o= zbi7x7)7drdt4klgfirPY}vS09bbfHIl1TWyB5v*t; zNnZFm315D}+3_{9H{O8k#z;nh;(rHtc-Mz~{FD^1C?5-|7kh!`fM;Vtuvo8+j{wFd z=UGNws@DD^vYfM1cXUb)PvT7>V{&Ss7wxJS*xZvlevE?c`*B=n!_064XWCq3=0?-3 zH_b%=KjKPxc*g|jruo(t9%mS|ra9d-Ws>!@D)p2Q0kG3x)4f4l#{%KEJ3tp|gsall zOK?Ym*)&D3a;5xG1SJK2lIAgwiWq@NQmIHCmkxZyc-$aayw)`5pkS?mm!(Q-*(W8i z8x=bd9o$PiM zqjxr^hKUY^mi|oNt|{IY2&>W@l_?@2=!8hiXuxo_$na8`;I0_!GSDGR=WPd)uy{RQ zkS(tL&{Oh8QLiElsxcwV((0wl?}&k3PUxS7S2m&{f?d<(c=`j@P6v2nm`c+1X*|K2 z%ZRwWati!NW>6kA#9k}HqT2r>EIqaPa2k*HI-ay5y@^p4CPuk>e)*!9xUX^mX*Q8< zz34iM*yp1;pn-DA{Lf7Ee1-Q#aoxUBu{KIVW9?g3Ntd!kw6yzD5MjY*UBm|kAAAw<4}1~z+?k}iaR(H{AwT#0 zec!p~o^$7z_;vsN#04ji_zk{4I}av)amErG?HVqW6I~~m=s3agY)cIw_8f^mk$Ts@ zo7%aUz0rPh{)1w6%en4JPc7YC1L=#@&LyxNUHU}LZfykOp7?Zj2aFp>j(zXppmQ$V z|MA1J+CIK_YxD8MY^rr>L8=xfCnB*(EE<1lZbTGb>&+4`1(y8Y%|UMIJtsgv?QK7D zw3%*vmAUF5{-dj#6XGRmgS&v1Mr$4}(=8TgYm ziPF5iZeoyg{EJzsY--rb@!=d#bhBbp&yQlPz-kKA)W#;$#-kv}MHQCQ5w@mYY+jej z9vZF;p{QKKiYEb!ZdheWb2${O8-z0Vn2S+16bzP`^Ev(`I7C+g*EA@xNtHQ)dUB}N zu`2DWr-o`xvaxRGNm)_{5vN>Jvk6du>9gx(3CnAyL3De;kQ8iX2`d6bjn)8s?ETHr zb=@vuo5;^sV-YG9G0;-Og37o|K$a%NP|rgfvB3m!IbPx{tZ+r6&JmrXx={;r3brMp zF>S@`0$C#_I1dgaw(8PCwf_TcL91BSdYIFUnuVcYuv1@_X5umX{1m!>+0@xvELJx&&(~&FQ$d=m1uOv{Y2u?!GoSjeq5Ptf9Ifn z-$AGq3ZnM{$3rND&c8cn=g_SHdMLDo#?koM6ThM{ lgqS+xude&6uN_vicifvhZ+z&W+pojB^rwx7FaPK~@E1%!QfvSK literal 0 HcmV?d00001 diff --git a/tests/fixtures/local-inspect/video-tags-v1.json b/tests/fixtures/local-inspect/video-tags-v1.json new file mode 100644 index 00000000..964a1ed7 --- /dev/null +++ b/tests/fixtures/local-inspect/video-tags-v1.json @@ -0,0 +1,60 @@ +{ + "generator": "ffmpeg -show_entries format_tags -of json over tests/fixtures/local-inspect, see tests/fixtures/README.md", + "fixtures": { + "core-faststart.mp4": { + "major_brand": "isom", + "minor_version": "512", + "compatible_brands": "isomiso2avc1mp41", + "prompt": "{\"3\":{\"class_type\":\"KSampler\",\"inputs\":{\"seed\":7,\"steps\":20,\"cfg\":8.0,\"sampler_name\":\"euler\",\"scheduler\":\"normal\",\"denoise\":1.0,\"model\":[\"4\",0],\"positive\":[\"6\",0],\"negative\":[\"7\",0],\"latent_image\":[\"5\",0]}},\"4\":{\"class_type\":\"CheckpointLoaderSimple\",\"inputs\":{\"ckpt_name\":\"synthetic-v1.safetensors\"}},\"5\":{\"class_type\":\"EmptyLatentImage\",\"inputs\":{\"width\":512,\"height\":512,\"batch_size\":1}},\"6\":{\"class_type\":\"CLIPTextEncode\",\"inputs\":{\"text\":\"a red fox in snow, detailed fur\",\"clip\":[\"4\",1]}},\"7\":{\"class_type\":\"CLIPTextEncode\",\"inputs\":{\"text\":\"blurry, lowres\",\"clip\":[\"4\",1]}},\"8\":{\"class_type\":\"VAEDecode\",\"inputs\":{\"samples\":[\"3\",0],\"vae\":[\"4\",2]}},\"9\":{\"class_type\":\"SaveVideo\",\"inputs\":{\"filename_prefix\":\"video/synthetic\",\"video\":[\"8\",0]}}}", + "workflow": "{\"last_node_id\":9,\"nodes\":[{\"id\":3,\"type\":\"KSampler\",\"widgets_values\":[7,\"fixed\",20,8.0,\"euler\",\"normal\",1.0]},{\"id\":6,\"type\":\"CLIPTextEncode\",\"widgets_values\":[\"a red fox in snow, detailed fur\"]},{\"id\":7,\"type\":\"CLIPTextEncode\",\"widgets_values\":[\"blurry, lowres\"]}],\"links\":[],\"version\":0.4}", + "parameters": "a red fox in snow, detailed fur\nNegative prompt: blurry, lowres\nSteps: 20, Sampler: Euler, CFG scale: 8.0, Seed: 7, Size: 512x512", + "comfyassets_prompt_pairs": "{\"version\":1,\"nodes\":{\"6\":{\"records\":[{\"positive_text\":\"a red fox in snow, detailed fur\",\"negative_text\":\"blurry, lowres\",\"negative_effective_text\":\"blurry, lowres\",\"prepend_text\":\"\",\"append_text\":\"\"}]}}}", + "encoder": "Lavf63.1.102" + }, + "core-moov-end.mp4": { + "major_brand": "isom", + "minor_version": "512", + "compatible_brands": "isomiso2avc1mp41", + "prompt": "{\"3\":{\"class_type\":\"KSampler\",\"inputs\":{\"seed\":7,\"steps\":20,\"cfg\":8.0,\"sampler_name\":\"euler\",\"scheduler\":\"normal\",\"denoise\":1.0,\"model\":[\"4\",0],\"positive\":[\"6\",0],\"negative\":[\"7\",0],\"latent_image\":[\"5\",0]}},\"4\":{\"class_type\":\"CheckpointLoaderSimple\",\"inputs\":{\"ckpt_name\":\"synthetic-v1.safetensors\"}},\"5\":{\"class_type\":\"EmptyLatentImage\",\"inputs\":{\"width\":512,\"height\":512,\"batch_size\":1}},\"6\":{\"class_type\":\"CLIPTextEncode\",\"inputs\":{\"text\":\"a red fox in snow, detailed fur\",\"clip\":[\"4\",1]}},\"7\":{\"class_type\":\"CLIPTextEncode\",\"inputs\":{\"text\":\"blurry, lowres\",\"clip\":[\"4\",1]}},\"8\":{\"class_type\":\"VAEDecode\",\"inputs\":{\"samples\":[\"3\",0],\"vae\":[\"4\",2]}},\"9\":{\"class_type\":\"SaveVideo\",\"inputs\":{\"filename_prefix\":\"video/synthetic\",\"video\":[\"8\",0]}}}", + "workflow": "{\"last_node_id\":9,\"nodes\":[{\"id\":3,\"type\":\"KSampler\",\"widgets_values\":[7,\"fixed\",20,8.0,\"euler\",\"normal\",1.0]},{\"id\":6,\"type\":\"CLIPTextEncode\",\"widgets_values\":[\"a red fox in snow, detailed fur\"]},{\"id\":7,\"type\":\"CLIPTextEncode\",\"widgets_values\":[\"blurry, lowres\"]}],\"links\":[],\"version\":0.4}", + "encoder": "Lavf63.1.102" + }, + "core.mov": { + "major_brand": "qt ", + "minor_version": "512", + "compatible_brands": "qt ", + "prompt": "{\"3\":{\"class_type\":\"KSampler\",\"inputs\":{\"seed\":7,\"steps\":20,\"cfg\":8.0,\"sampler_name\":\"euler\",\"scheduler\":\"normal\",\"denoise\":1.0,\"model\":[\"4\",0],\"positive\":[\"6\",0],\"negative\":[\"7\",0],\"latent_image\":[\"5\",0]}},\"4\":{\"class_type\":\"CheckpointLoaderSimple\",\"inputs\":{\"ckpt_name\":\"synthetic-v1.safetensors\"}},\"5\":{\"class_type\":\"EmptyLatentImage\",\"inputs\":{\"width\":512,\"height\":512,\"batch_size\":1}},\"6\":{\"class_type\":\"CLIPTextEncode\",\"inputs\":{\"text\":\"a red fox in snow, detailed fur\",\"clip\":[\"4\",1]}},\"7\":{\"class_type\":\"CLIPTextEncode\",\"inputs\":{\"text\":\"blurry, lowres\",\"clip\":[\"4\",1]}},\"8\":{\"class_type\":\"VAEDecode\",\"inputs\":{\"samples\":[\"3\",0],\"vae\":[\"4\",2]}},\"9\":{\"class_type\":\"SaveVideo\",\"inputs\":{\"filename_prefix\":\"video/synthetic\",\"video\":[\"8\",0]}}}", + "workflow": "{\"last_node_id\":9,\"nodes\":[{\"id\":3,\"type\":\"KSampler\",\"widgets_values\":[7,\"fixed\",20,8.0,\"euler\",\"normal\",1.0]},{\"id\":6,\"type\":\"CLIPTextEncode\",\"widgets_values\":[\"a red fox in snow, detailed fur\"]},{\"id\":7,\"type\":\"CLIPTextEncode\",\"widgets_values\":[\"blurry, lowres\"]}],\"links\":[],\"version\":0.4}", + "encoder": "Lavf63.1.102" + }, + "core.m4v": { + "major_brand": "M4V ", + "minor_version": "512", + "compatible_brands": "M4V isomiso2avc1", + "prompt": "{\"3\":{\"class_type\":\"KSampler\",\"inputs\":{\"seed\":7,\"steps\":20,\"cfg\":8.0,\"sampler_name\":\"euler\",\"scheduler\":\"normal\",\"denoise\":1.0,\"model\":[\"4\",0],\"positive\":[\"6\",0],\"negative\":[\"7\",0],\"latent_image\":[\"5\",0]}},\"4\":{\"class_type\":\"CheckpointLoaderSimple\",\"inputs\":{\"ckpt_name\":\"synthetic-v1.safetensors\"}},\"5\":{\"class_type\":\"EmptyLatentImage\",\"inputs\":{\"width\":512,\"height\":512,\"batch_size\":1}},\"6\":{\"class_type\":\"CLIPTextEncode\",\"inputs\":{\"text\":\"a red fox in snow, detailed fur\",\"clip\":[\"4\",1]}},\"7\":{\"class_type\":\"CLIPTextEncode\",\"inputs\":{\"text\":\"blurry, lowres\",\"clip\":[\"4\",1]}},\"8\":{\"class_type\":\"VAEDecode\",\"inputs\":{\"samples\":[\"3\",0],\"vae\":[\"4\",2]}},\"9\":{\"class_type\":\"SaveVideo\",\"inputs\":{\"filename_prefix\":\"video/synthetic\",\"video\":[\"8\",0]}}}", + "workflow": "{\"last_node_id\":9,\"nodes\":[{\"id\":3,\"type\":\"KSampler\",\"widgets_values\":[7,\"fixed\",20,8.0,\"euler\",\"normal\",1.0]},{\"id\":6,\"type\":\"CLIPTextEncode\",\"widgets_values\":[\"a red fox in snow, detailed fur\"]},{\"id\":7,\"type\":\"CLIPTextEncode\",\"widgets_values\":[\"blurry, lowres\"]}],\"links\":[],\"version\":0.4}", + "encoder": "Lavf63.1.102" + }, + "core.mkv": { + "WORKFLOW": "{\"last_node_id\":9,\"nodes\":[{\"id\":3,\"type\":\"KSampler\",\"widgets_values\":[7,\"fixed\",20,8.0,\"euler\",\"normal\",1.0]},{\"id\":6,\"type\":\"CLIPTextEncode\",\"widgets_values\":[\"a red fox in snow, detailed fur\"]},{\"id\":7,\"type\":\"CLIPTextEncode\",\"widgets_values\":[\"blurry, lowres\"]}],\"links\":[],\"version\":0.4}", + "PROMPT": "{\"3\":{\"class_type\":\"KSampler\",\"inputs\":{\"seed\":7,\"steps\":20,\"cfg\":8.0,\"sampler_name\":\"euler\",\"scheduler\":\"normal\",\"denoise\":1.0,\"model\":[\"4\",0],\"positive\":[\"6\",0],\"negative\":[\"7\",0],\"latent_image\":[\"5\",0]}},\"4\":{\"class_type\":\"CheckpointLoaderSimple\",\"inputs\":{\"ckpt_name\":\"synthetic-v1.safetensors\"}},\"5\":{\"class_type\":\"EmptyLatentImage\",\"inputs\":{\"width\":512,\"height\":512,\"batch_size\":1}},\"6\":{\"class_type\":\"CLIPTextEncode\",\"inputs\":{\"text\":\"a red fox in snow, detailed fur\",\"clip\":[\"4\",1]}},\"7\":{\"class_type\":\"CLIPTextEncode\",\"inputs\":{\"text\":\"blurry, lowres\",\"clip\":[\"4\",1]}},\"8\":{\"class_type\":\"VAEDecode\",\"inputs\":{\"samples\":[\"3\",0],\"vae\":[\"4\",2]}},\"9\":{\"class_type\":\"SaveVideo\",\"inputs\":{\"filename_prefix\":\"video/synthetic\",\"video\":[\"8\",0]}}}", + "ENCODER": "Lavf63.1.102" + }, + "core-tags.webm": { + "WORKFLOW": "{\"last_node_id\":9,\"nodes\":[{\"id\":3,\"type\":\"KSampler\",\"widgets_values\":[7,\"fixed\",20,8.0,\"euler\",\"normal\",1.0]},{\"id\":6,\"type\":\"CLIPTextEncode\",\"widgets_values\":[\"a red fox in snow, detailed fur\"]},{\"id\":7,\"type\":\"CLIPTextEncode\",\"widgets_values\":[\"blurry, lowres\"]}],\"links\":[],\"version\":0.4}", + "PROMPT": "{\"3\":{\"class_type\":\"KSampler\",\"inputs\":{\"seed\":7,\"steps\":20,\"cfg\":8.0,\"sampler_name\":\"euler\",\"scheduler\":\"normal\",\"denoise\":1.0,\"model\":[\"4\",0],\"positive\":[\"6\",0],\"negative\":[\"7\",0],\"latent_image\":[\"5\",0]}},\"4\":{\"class_type\":\"CheckpointLoaderSimple\",\"inputs\":{\"ckpt_name\":\"synthetic-v1.safetensors\"}},\"5\":{\"class_type\":\"EmptyLatentImage\",\"inputs\":{\"width\":512,\"height\":512,\"batch_size\":1}},\"6\":{\"class_type\":\"CLIPTextEncode\",\"inputs\":{\"text\":\"a red fox in snow, detailed fur\",\"clip\":[\"4\",1]}},\"7\":{\"class_type\":\"CLIPTextEncode\",\"inputs\":{\"text\":\"blurry, lowres\",\"clip\":[\"4\",1]}},\"8\":{\"class_type\":\"VAEDecode\",\"inputs\":{\"samples\":[\"3\",0],\"vae\":[\"4\",2]}},\"9\":{\"class_type\":\"SaveVideo\",\"inputs\":{\"filename_prefix\":\"video/synthetic\",\"video\":[\"8\",0]}}}", + "ENCODER": "Lavf63.1.102" + }, + "legacy-comment.mp4": { + "major_brand": "isom", + "minor_version": "512", + "compatible_brands": "isomiso2avc1mp41", + "encoder": "Lavf63.1.102", + "comment": "{\"prompt\": \"{\\\"3\\\":{\\\"class_type\\\":\\\"KSampler\\\",\\\"inputs\\\":{\\\"seed\\\":7,\\\"steps\\\":20,\\\"cfg\\\":8.0,\\\"sampler_name\\\":\\\"euler\\\",\\\"scheduler\\\":\\\"normal\\\",\\\"denoise\\\":1.0,\\\"model\\\":[\\\"4\\\",0],\\\"positive\\\":[\\\"6\\\",0],\\\"negative\\\":[\\\"7\\\",0],\\\"latent_image\\\":[\\\"5\\\",0]}},\\\"4\\\":{\\\"class_type\\\":\\\"CheckpointLoaderSimple\\\",\\\"inputs\\\":{\\\"ckpt_name\\\":\\\"synthetic-v1.safetensors\\\"}},\\\"5\\\":{\\\"class_type\\\":\\\"EmptyLatentImage\\\",\\\"inputs\\\":{\\\"width\\\":512,\\\"height\\\":512,\\\"batch_size\\\":1}},\\\"6\\\":{\\\"class_type\\\":\\\"CLIPTextEncode\\\",\\\"inputs\\\":{\\\"text\\\":\\\"a red fox in snow, detailed fur\\\",\\\"clip\\\":[\\\"4\\\",1]}},\\\"7\\\":{\\\"class_type\\\":\\\"CLIPTextEncode\\\",\\\"inputs\\\":{\\\"text\\\":\\\"blurry, lowres\\\",\\\"clip\\\":[\\\"4\\\",1]}},\\\"8\\\":{\\\"class_type\\\":\\\"VAEDecode\\\",\\\"inputs\\\":{\\\"samples\\\":[\\\"3\\\",0],\\\"vae\\\":[\\\"4\\\",2]}},\\\"9\\\":{\\\"class_type\\\":\\\"SaveVideo\\\",\\\"inputs\\\":{\\\"filename_prefix\\\":\\\"video/synthetic\\\",\\\"video\\\":[\\\"8\\\",0]}}}\", \"workflow\": {\"last_node_id\": 9, \"nodes\": [{\"id\": 3, \"type\": \"KSampler\", \"widgets_values\": [7, \"fixed\", 20, 8.0, \"euler\", \"normal\", 1.0]}, {\"id\": 6, \"type\": \"CLIPTextEncode\", \"widgets_values\": [\"a red fox in snow, detailed fur\"]}, {\"id\": 7, \"type\": \"CLIPTextEncode\", \"widgets_values\": [\"blurry, lowres\"]}], \"links\": [], \"version\": 0.4}}" + }, + "legacy-comment.webm": { + "COMMENT": "{\"prompt\": \"{\\\"3\\\":{\\\"class_type\\\":\\\"KSampler\\\",\\\"inputs\\\":{\\\"seed\\\":7,\\\"steps\\\":20,\\\"cfg\\\":8.0,\\\"sampler_name\\\":\\\"euler\\\",\\\"scheduler\\\":\\\"normal\\\",\\\"denoise\\\":1.0,\\\"model\\\":[\\\"4\\\",0],\\\"positive\\\":[\\\"6\\\",0],\\\"negative\\\":[\\\"7\\\",0],\\\"latent_image\\\":[\\\"5\\\",0]}},\\\"4\\\":{\\\"class_type\\\":\\\"CheckpointLoaderSimple\\\",\\\"inputs\\\":{\\\"ckpt_name\\\":\\\"synthetic-v1.safetensors\\\"}},\\\"5\\\":{\\\"class_type\\\":\\\"EmptyLatentImage\\\",\\\"inputs\\\":{\\\"width\\\":512,\\\"height\\\":512,\\\"batch_size\\\":1}},\\\"6\\\":{\\\"class_type\\\":\\\"CLIPTextEncode\\\",\\\"inputs\\\":{\\\"text\\\":\\\"a red fox in snow, detailed fur\\\",\\\"clip\\\":[\\\"4\\\",1]}},\\\"7\\\":{\\\"class_type\\\":\\\"CLIPTextEncode\\\",\\\"inputs\\\":{\\\"text\\\":\\\"blurry, lowres\\\",\\\"clip\\\":[\\\"4\\\",1]}},\\\"8\\\":{\\\"class_type\\\":\\\"VAEDecode\\\",\\\"inputs\\\":{\\\"samples\\\":[\\\"3\\\",0],\\\"vae\\\":[\\\"4\\\",2]}},\\\"9\\\":{\\\"class_type\\\":\\\"SaveVideo\\\",\\\"inputs\\\":{\\\"filename_prefix\\\":\\\"video/synthetic\\\",\\\"video\\\":[\\\"8\\\",0]}}}\", \"workflow\": {\"last_node_id\": 9, \"nodes\": [{\"id\": 3, \"type\": \"KSampler\", \"widgets_values\": [7, \"fixed\", 20, 8.0, \"euler\", \"normal\", 1.0]}, {\"id\": 6, \"type\": \"CLIPTextEncode\", \"widgets_values\": [\"a red fox in snow, detailed fur\"]}, {\"id\": 7, \"type\": \"CLIPTextEncode\", \"widgets_values\": [\"blurry, lowres\"]}], \"links\": [], \"version\": 0.4}}", + "ENCODER": "Lavf63.1.102" + } + } +} diff --git a/tests/js/helpers/video-bytes.mjs b/tests/js/helpers/video-bytes.mjs new file mode 100644 index 00000000..7b60b42d --- /dev/null +++ b/tests/js/helpers/video-bytes.mjs @@ -0,0 +1,277 @@ +// Byte-exact ISO-BMFF (MP4/MOV/M4V) and Matroska (MKV/WebM) builders for the local +// video tag reader tests, laid out as ffmpeg's muxers write them (see the dumps in +// tests/fixtures/local-inspect). The File-like blobs come from png-bytes.mjs. +import { concat, u32, utf8, latin1 } from "./png-bytes.mjs"; + +export { concat, u32, utf8, latin1 }; +export { sparseBlob, recordingBlob, slowBlob, failingBlob, blobOf } from "./png-bytes.mjs"; + +const toBytes = (part) => (typeof part === "string" ? utf8(part) : Buffer.from(part)); + +function u64(n) { + const out = new Uint8Array(8); + new DataView(out.buffer).setBigUint64(0, BigInt(n)); + return out; +} + +// ---------------------------------------------------------------- ISO-BMFF + +/** A box: 32-bit size, 4-byte type (string or raw bytes), payload. `size` overrides the field. */ +export function box(type, ...parts) { + const options = parts.length && parts[parts.length - 1] && parts[parts.length - 1].__options ? parts.pop() : {}; + const payload = concat(...parts.map(toBytes)); + const typeBytes = typeof type === "string" ? latin1(type) : Buffer.from(type); + const total = 8 + payload.length; + if (options.largesize) return concat(u32(1), typeBytes, u64(options.largesize === true ? total + 8 : options.largesize), payload); + return concat(u32(options.size ?? total), typeBytes, payload); +} + +/** Marker for `box` options: box('ftyp', ..., opts({size: 3})). */ +export function opts(options) { + return { __options: true, ...options }; +} + +export function fullBox(type, ...parts) { + return box(type, u32(0), ...parts); +} + +export function ftyp(brand = "isom", compatible = "isomiso2avc1mp41") { + return box("ftyp", latin1(brand), u32(512), latin1(compatible)); +} + +export function mdat(payload = [0, 0, 0, 0]) { + return box("mdat", payload); +} + +export const moov = (...children) => box("moov", ...children); +export const udta = (...children) => box("udta", ...children); +export const free = (count = 0) => box("free", new Uint8Array(count)); + +/** `meta` as ffmpeg writes it: a FullBox (version/flags) whose first child is hdlr. */ +export function meta(...children) { + return fullBox("meta", ...children); +} + +export function hdlr(handler = "mdta") { + return fullBox("hdlr", u32(0), latin1(handler), new Uint8Array(12), [0]); +} + +/** `keys`: one mdta entry per name (an entry may be [name, namespace]); `count` overrides the count field. */ +export function keys(names, { count } = {}) { + const entries = names.map((entry) => { + const [name, namespace] = Array.isArray(entry) ? entry : [entry, "mdta"]; + const bytes = toBytes(name); + return concat(u32(8 + bytes.length), latin1(namespace), bytes); + }); + return fullBox("keys", u32(count ?? names.length), ...entries); +} + +export const ilst = (...items) => box("ilst", ...items); + +/** A `data` atom: type indicator (1 = UTF-8), locale, payload. */ +export function data(payload, { type = 1, locale = 0, size } = {}) { + return box("data", u32(type), u32(locale), toBytes(payload), ...(size === undefined ? [] : [opts({ size })])); +} + +/** An ilst item addressed by its 1-based `keys` index. */ +export function item(index, ...children) { + return box(u32(index), ...children); +} + +/** An ilst item addressed by a legacy fourcc such as "©cmt" (Latin-1 bytes). */ +export function legacyItem(fourcc, ...children) { + return box(latin1(fourcc), ...children); +} + +/** `moov/udta/meta` carrying per-key mdta tags, as core SaveVideo writes through ffmpeg. */ +export function mdtaUdta(tags) { + const names = Object.keys(tags); + return udta(meta(hdlr("mdta"), keys(names), ilst(...names.map((name, index) => item(index + 1, data(tags[name])))))); +} + +/** `moov/udta/meta` with legacy ilst atoms: {comment, description} → ©cmt / desc. */ +export function legacyUdta({ comment, description, tool = "Lavf" } = {}) { + const items = [legacyItem("©too", data(tool))]; + if (comment !== undefined) items.push(legacyItem("©cmt", data(comment))); + if (description !== undefined) items.push(legacyItem("desc", data(description))); + return udta(meta(hdlr("mdir"), ilst(...items))); +} + +/** A whole MP4: ftyp, (moov first when faststart), mdat, moov with the given udta. */ +export function mp4(udtaBox, { brand = "isom", faststart = false, extraMoov = [], payload } = {}) { + const movie = moov(...extraMoov, udtaBox); + const media = mdat(payload); + return faststart ? concat(ftyp(brand), movie, media) : concat(ftyp(brand), free(), media, movie); +} + +export function perKeyMp4(tags, options) { + return mp4(mdtaUdta(tags), options); +} + +/** Top-level boxes of `bytes` as [{type, start, end}]. */ +export function topLevelBoxes(bytes) { + const out = []; + let offset = 0; + while (offset + 8 <= bytes.length) { + const view = new DataView(bytes.buffer, bytes.byteOffset + offset); + let size = view.getUint32(0); + if (size === 1) size = Number(view.getBigUint64(8)); + if (size === 0) size = bytes.length - offset; + out.push({ type: latin1Of(bytes.subarray(offset + 4, offset + 8)), start: offset, end: offset + size }); + offset += size; + } + return out; +} + +function latin1Of(bytes) { + return Buffer.from(bytes).toString("latin1"); +} + +// ---------------------------------------------------------------- Matroska + +export const ID = { + EBML: 0x1a45dfa3, + EBMLVersion: 0x4286, + DocType: 0x4282, + DocTypeVersion: 0x4287, + Segment: 0x18538067, + SeekHead: 0x114d9b74, + Seek: 0x4dbb, + SeekID: 0x53ab, + SeekPosition: 0x53ac, + Void: 0xec, + CRC32: 0xbf, + Info: 0x1549a966, + Tracks: 0x1654ae6b, + Tags: 0x1254c367, + Tag: 0x7373, + Targets: 0x63c0, + TargetTypeValue: 0x68ca, + TagTrackUID: 0x63c5, + SimpleTag: 0x67c8, + TagName: 0x45a3, + TagString: 0x4487, + TagLanguage: 0x447a, + TagDefault: 0x4484, + TagBinary: 0x4485, + Cluster: 0x1f43b675, + Timestamp: 0xe7, + Cues: 0x1c53bb6b, +}; + +/** The ID's bytes (EBML IDs carry their own length marker). */ +export function idBytes(id) { + const out = []; + let value = id; + while (value > 0) { + out.unshift(value & 0xff); + value = Math.floor(value / 256); + } + return Uint8Array.from(out); +} + +/** A size VINT: minimal width unless `width` is given; `unknown` writes all ones. */ +export function vint(value, { width, unknown = false } = {}) { + let length = width; + if (!length) { + length = 1; + while (length < 8 && value >= 2 ** (7 * length) - 1) length += 1; + } + const out = new Uint8Array(length); + if (unknown) { + out.fill(0xff); + out[0] = 0xff >> (length - 1); + return out; + } + let rest = BigInt(value); + for (let index = length - 1; index >= 0; index -= 1) { + out[index] = Number(rest & 0xffn); + rest >>= 8n; + } + out[0] |= 0x80 >> (length - 1); + return out; +} + +/** An element: ID, size VINT, payload. `size`/`width`/`unknown` shape the size field. */ +export function el(id, ...parts) { + const options = parts.length && parts[parts.length - 1] && parts[parts.length - 1].__options ? parts.pop() : {}; + const payload = concat(...parts.map(toBytes)); + const size = vint(options.size ?? payload.length, { width: options.width, unknown: options.unknown }); + return concat(idBytes(id), size, payload); +} + +export function uint(value, width) { + const out = []; + let rest = BigInt(value); + do { + out.unshift(Number(rest & 0xffn)); + rest >>= 8n; + } while (rest > 0n); + while (width && out.length < width) out.unshift(0); + return Uint8Array.from(out); +} + +export function ebmlHeader(docType = "matroska") { + return el(ID.EBML, el(ID.EBMLVersion, [1]), el(ID.DocType, docType), el(ID.DocTypeVersion, [4])); +} + +/** Segment with an 8-byte size field as ffmpeg writes it; `unknown` makes it unknown-sized. */ +export function segment(children, { unknown = false } = {}) { + return el(ID.Segment, ...children, opts({ width: 8, unknown })); +} + +/** SeekHead entries: [[id, position]] with positions relative to the Segment payload. */ +export function seekHead(entries) { + return el(ID.SeekHead, ...entries.map(([id, position]) => el(ID.Seek, el(ID.SeekID, idBytes(id)), el(ID.SeekPosition, uint(position))))); +} + +export function voidEl(count = 4) { + return el(ID.Void, new Uint8Array(count)); +} + +/** A SimpleTag with TagName and TagString (`value` undefined omits the string). */ +export function simpleTag(name, value, { lang, def, binary, nested = [], extra = [] } = {}) { + const children = [el(ID.TagName, name)]; + if (lang !== undefined) children.push(el(ID.TagLanguage, lang)); + if (def !== undefined) children.push(el(ID.TagDefault, [def])); + if (value !== undefined) children.push(el(ID.TagString, value)); + if (binary !== undefined) children.push(el(ID.TagBinary, binary)); + return el(ID.SimpleTag, ...children, ...nested, ...extra); +} + +/** A Tag whose Targets carry the given track UID (none → global). */ +export function tag(simpleTags, { trackUid, targetType } = {}) { + const targets = []; + if (targetType !== undefined) targets.push(el(ID.TargetTypeValue, uint(targetType))); + if (trackUid !== undefined) targets.push(el(ID.TagTrackUID, uint(trackUid, 8))); + return el(ID.Tag, el(ID.Targets, ...targets), ...simpleTags); +} + +export function tagsEl(...tagElements) { + return el(ID.Tags, ...tagElements); +} + +/** A Cluster of `size` payload bytes (zeros); `unknown` makes its size unknown. */ +export function cluster(size = 8, { unknown = false, payload } = {}) { + return el(ID.Cluster, payload ?? new Uint8Array(size), opts({ unknown })); +} + +/** A Cluster header only, declaring `size` payload bytes (for sparse layouts). */ +export function clusterHeader(size) { + return concat(idBytes(ID.Cluster), vint(size, { width: 8 })); +} + +/** A whole Matroska file: EBML header + Segment(children). */ +export function mkv(children, { docType = "matroska", unknown = false } = {}) { + return concat(ebmlHeader(docType), segment(children, { unknown })); +} + +/** A global Tags element from {name: value} (ffmpeg upper-cases names at mux time). */ +export function globalTags(tags, options) { + return tagsEl(tag(Object.entries(tags).map(([name, value]) => simpleTag(name, value)), options)); +} + +/** A Matroska file whose global Tags hold {name: value}, ffmpeg layout. */ +export function perKeyMkv(tags, { docType = "matroska" } = {}) { + return mkv([el(ID.Info, new Uint8Array(4)), globalTags(tags), cluster()], { docType }); +} diff --git a/tests/js/support/local-inspect-oracle.mjs b/tests/js/support/local-inspect-oracle.mjs index bb245227..fc3ffe83 100644 --- a/tests/js/support/local-inspect-oracle.mjs +++ b/tests/js/support/local-inspect-oracle.mjs @@ -2,9 +2,13 @@ // tests/test_local_inspect_parity.py, which compares every answer with the server's // utils.prompt_inspector.read_inspection_file on the same file. Each file is opened // as a Blob (fs.openAsBlob) so the reader slices it exactly as it slices a File. -// stdin: [{name, path}] +// stdin: [{name, path} | {name, tags, limited?}] // stdout: {name: {classified, result}} where result is readPngMetadataFile's -// answer for a PNG, readImageHeader's for another image, else null. +// answer for a PNG, readImageHeader's for another image, for a video +// {read: readVideoTags(...), selection: selectVideoCarrier(read.tags, {limited})} +// (selection null unless the read completed), else null. +// A {tags} entry answers {classified: null, result: {selection}} for the +// carrier rule alone, as the server selects from ffprobe's tag map. import fs from 'node:fs'; import path from 'node:path'; import { createRequire } from 'node:module'; @@ -13,17 +17,27 @@ import { fileURLToPath } from 'node:url'; const here = path.dirname(fileURLToPath(import.meta.url)); const require = createRequire(import.meta.url); const { classifyPrefix, readPngMetadataFile, readImageHeader } = require(path.join(here, '..', '..', '..', 'web', 'js', 'png-metadata.js')); +const { readVideoTags, selectVideoCarrier } = await import(path.join(here, '..', '..', '..', 'web', 'js', 'workspace', 'local-video.mjs')); const chunks = []; for await (const chunk of process.stdin) chunks.push(chunk); const output = {}; -for (const { name, path: filePath } of JSON.parse(Buffer.concat(chunks).toString('utf8'))) { - const file = await fs.openAsBlob(filePath); +for (const entry of JSON.parse(Buffer.concat(chunks).toString('utf8'))) { + const { name } = entry; + if (entry.path === undefined) { + output[name] = { classified: null, result: { selection: selectVideoCarrier(entry.tags, { limited: entry.limited || [] }) } }; + continue; + } + const file = await fs.openAsBlob(entry.path); const prefix = new Uint8Array(await file.slice(0, Math.min(32, file.size)).arrayBuffer()); const classified = classifyPrefix(prefix); let result = null; if (classified?.kind === 'png') result = await readPngMetadataFile(file, { prefix }); else if (classified?.media === 'image') result = await readImageHeader(file, classified.kind); + else if (classified?.media === 'video') { + const read = await readVideoTags(file, { prefix, kind: classified.kind }); + result = { read, selection: read.status === 'complete' ? selectVideoCarrier(read.tags, { limited: read.limited }) : null }; + } output[name] = { classified, result }; } process.stdout.write(JSON.stringify(output)); diff --git a/tests/js/workspace-local-video.test.js b/tests/js/workspace-local-video.test.js new file mode 100644 index 00000000..d4f40d39 --- /dev/null +++ b/tests/js/workspace-local-video.test.js @@ -0,0 +1,484 @@ +// Run with: node --test tests/js/ +// The browser video tag readers mirror what the server reads through ffprobe +// (utils/prompt_inspector.py _video_metadata) and the carrier rule of +// utils/video_metadata.py select_video_carrier. Each rule names its server twin; +// tests/test_local_inspect_parity.py proves the two agree on the committed fixtures +// and on recorded tag maps. +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import * as v from "./helpers/video-bytes.mjs"; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const FIXTURES = path.join(here, "..", "fixtures", "local-inspect"); +const RECORDED = JSON.parse(fs.readFileSync(path.join(FIXTURES, "video-tags-v1.json"), "utf8")).fixtures; +const m = await import("../../web/js/workspace/local-video.mjs").catch((error) => { + if (error.code !== "ERR_MODULE_NOT_FOUND") throw error; + return {}; +}); +const { readVideoTags, selectVideoCarrier, pythonJsonDumps, pythonJsonLoads } = m; + +const MiB = 1024 * 1024; +const GiB = 1024 * MiB; +const ROLE_BUDGET = 2 * MiB; +const WORKFLOW_BUDGET = 4 * MiB; +const TAG_READ_BYTES = 12 * MiB + 64 * 1024; +const PER_KEY = new Set(["prompt", "workflow", "parameters", "comfyassets_prompt_pairs"]); +const GRAPH = '{"3":{"class_type":"KSampler","inputs":{"seed":7}}}'; +const WORKFLOW = '{"nodes":[],"version":0.4}'; + +function knownTags(map) { + return Object.fromEntries(Object.entries(map).filter(([name]) => PER_KEY.has(name.toLowerCase()) || name === "comment" || name === "description")); +} + +function fixture(name) { + return new Uint8Array(fs.readFileSync(path.join(FIXTURES, name))); +} + +async function read(bytes, options = {}) { + return readVideoTags(v.blobOf(bytes), options); +} + +function fill(byte, count) { + return Buffer.alloc(count, byte); +} + +function complete(tags, extra = {}) { + return { status: "complete", tags, limited: [], ...extra }; +} + +test("contract: the readers, the carrier rule and the json port are exported", () => { + assert.equal(typeof readVideoTags, "function"); + assert.equal(typeof selectVideoCarrier, "function"); + assert.equal(typeof pythonJsonDumps, "function"); + assert.equal(typeof pythonJsonLoads, "function"); +}); + +test("iso-bmff reader finds per-key tags with moov after mdat", async () => { + for (const [name, recorded] of Object.entries(RECORDED)) { + const blob = v.recordingBlob(fixture(name)); + const result = await readVideoTags(blob, { prefix: fixture(name).subarray(0, 32) }); + assert.equal(result.status, "complete", name); + assert.deepEqual(result.tags, knownTags(recorded), name); + assert.equal(result.container, name.endsWith(".mkv") || name.endsWith(".webm") ? "matroska" : "iso-bmff", name); + assert.ok(["isom", "qt", "M4V", "matroska", "webm"].includes(result.format), `${name}: ${result.format}`); + assert.deepEqual(result.limited, [], name); + assert.ok(blob.ranges.length > 0, name); + } + // core-moov-end.mp4 behind a 3 GiB mdat: only the box headers and the tag bytes are sliced. + const bytes = fixture("core-moov-end.mp4"); + const boxes = v.topLevelBoxes(bytes); + const mdat = boxes.find((box) => box.type === "mdat"); + const moov = boxes.find((box) => box.type === "moov"); + const mdatSize = 3 * GiB + 16; + const header = v.concat(v.u32(1), v.latin1("mdat"), new Uint8Array(8)); + new DataView(header.buffer).setBigUint64(8, BigInt(mdatSize)); + const moovOffset = mdat.start + mdatSize; + const blob = v.sparseBlob( + [ + { offset: 0, bytes: bytes.subarray(0, mdat.start) }, + { offset: mdat.start, bytes: header }, + { offset: moovOffset, bytes: bytes.subarray(moov.start, moov.end) }, + ], + moovOffset + (moov.end - moov.start), + ); + const result = await readVideoTags(blob, { prefix: bytes.subarray(0, 32) }); + assert.deepEqual(result.tags, knownTags(RECORDED["core-moov-end.mp4"])); + const inside = blob.ranges.filter(([start]) => start >= mdat.start && start < moovOffset); + assert.ok(inside.every(([start, end]) => start === mdat.start && end - start <= 16), JSON.stringify(inside)); + const total = blob.ranges.reduce((sum, [start, end]) => sum + (end - start), 0); + assert.ok(total <= m.VIDEO_READ_CEILING && total < 64 * 1024, `${total}`); +}); + +test("iso-bmff: legacy ilst atoms, moov/meta, sticky mdta handler and data types", async () => { + // ©cmt and desc map to comment and description, as ffprobe reports them. + const legacy = await read(v.mp4(v.legacyUdta({ comment: '{"prompt":"{}"}', description: "note" }))); + assert.deepEqual(legacy, complete({ comment: '{"prompt":"{}"}', description: "note" }, { container: "iso-bmff", format: "isom" })); + // moov/meta (ISO style) is read like moov/udta/meta. + const isoMeta = await read(v.mp4(v.box("meta", v.u32(0), v.hdlr("mdta"), v.keys(["prompt"]), v.ilst(v.item(1, v.data(GRAPH)))))); + assert.deepEqual(isoMeta.tags, { prompt: GRAPH }); + // meta without the version/flags word (QuickTime) still finds hdlr. + const qtMeta = await read(v.mp4(v.udta(v.box("meta", v.hdlr("mdta"), v.keys(["prompt"]), v.ilst(v.item(1, v.data(GRAPH))))))); + assert.deepEqual(qtMeta.tags, { prompt: GRAPH }); + // Index items need an mdta handler and a keys box; with a legacy handler they are skipped. + const noKeys = await read(v.mp4(v.udta(v.meta(v.hdlr("mdir"), v.ilst(v.item(1, v.data(GRAPH))))))); + assert.deepEqual(noKeys.tags, {}); + // The mdta handler is sticky across the file, as ffmpeg's found_hdlr_mdta. + const sticky = await read(v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys(["prompt"])), v.meta(v.hdlr("mdir"), v.ilst(v.item(1, v.data(GRAPH))))))); + assert.deepEqual(sticky.tags, { prompt: GRAPH }); + // As ffprobe reports them: consecutive data children join with ";", integer data + // types print as decimals, other types are not tags, text types 0/1/4 are read, NUL + // truncates, and the run of data children stops at the first other child. + const mixed = await read(v.mp4(v.udta(v.meta( + v.hdlr("mdta"), + v.keys(["prompt", "workflow", "parameters", "comfyassets_prompt_pairs", "comment", "description"]), + v.ilst( + v.item(1, v.data("first"), v.data("second"), v.box("name", "x"), v.data("not read")), + v.item(2, v.data([0, 0, 0, 7], { type: 21 }), v.data([0xff, 0xfe], { type: 21 }), v.data([0xff, 0xfe], { type: 22 })), + v.item(3, v.data("cut\0off")), + v.item(4, v.data("sort", { type: 4 }), v.data("mac", { type: 0 })), + v.item(5, v.data([0, 0, 0, 0, 0, 0, 0], { type: 21 })), + v.item(6, v.data("float", { type: 23 })), + ), + )))); + assert.deepEqual(mixed.tags, { prompt: "first;second", workflow: "7;-2;65534", parameters: "cut", comfyassets_prompt_pairs: "sort;mac" }); + // Keys outside the mdta namespace address nothing; unknown names are skipped unread. + const blob = v.recordingBlob(v.mp4(v.udta(v.meta( + v.hdlr("mdta"), + v.keys([["prompt", "udta"], "encoder", "workflow"]), + v.ilst(v.item(1, v.data(GRAPH)), v.item(2, v.data(fill(0x41, 5000))), v.item(3, v.data(WORKFLOW))), + )))); + const result = await readVideoTags(blob); + assert.deepEqual(result.tags, { workflow: WORKFLOW }); + assert.ok(blob.ranges.every(([start, end]) => end - start < 5000), "the encoder value was never sliced"); + // Legacy fourccs are mapped even in mdta mode, and a repeated name joins with ";". + const repeated = await read(v.mp4(v.udta(v.meta( + v.hdlr("mdta"), + v.keys(["prompt", "prompt", "comment"]), + v.ilst(v.item(1, v.data("one")), v.item(2, v.data("two")), v.item(3, v.data("mdta")), v.legacyItem("©cmt", v.data("c"))), + )))); + assert.deepEqual(repeated.tags, { prompt: "one;two", comment: "mdta;c" }); + // Apple's namespaced keys are reported verbatim, so they are never an envelope. + const apple = await read(v.perKeyMp4({ "com.apple.quicktime.description": '{"prompt":"{}"}', prompt: GRAPH })); + assert.deepEqual(apple.tags, { prompt: GRAPH }); + // An item whose first child is not a data atom, or a data atom shorter than its + // 16-byte header, is skipped as ffmpeg skips it. + const junk = await read(v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys(["prompt"]), v.ilst(v.item(1, v.box("name", "x"), v.data(GRAPH))))))); + assert.deepEqual(junk.tags, {}); + const short = await read(v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys(["prompt"]), v.ilst(v.item(1, v.data(GRAPH, { size: 12 }))))))); + assert.deepEqual(short.tags, {}); + // A box declared with size 0 runs to the end of its parent; faststart puts moov first. + const fast = await read(v.perKeyMp4({ prompt: GRAPH }, { faststart: true, brand: "mp42" })); + assert.deepEqual(fast, complete({ prompt: GRAPH }, { container: "iso-bmff", format: "mp42" })); + const toEnd = await read(v.concat(v.ftyp(), v.mdat(), v.moov(v.mdtaUdta({ prompt: GRAPH }), v.opts({ size: 0 })))); + assert.deepEqual(toEnd.tags, { prompt: GRAPH }); +}); + +test("matroska reads global Tags and skips clusters", async () => { + const { ID } = v; + // ffmpeg layout: SeekHead, Info, Tracks, Tags, Cluster, Cues; names upper-cased. + const plain = await read(v.perKeyMkv({ PROMPT: GRAPH, WORKFLOW, ENCODER: "Lavf" }, { docType: "webm" })); + assert.deepEqual(plain, complete({ PROMPT: GRAPH, WORKFLOW }, { container: "matroska", format: "webm" })); + // Tags after a 3 GiB Cluster, located through the SeekHead: no slice inside the cluster. + const tags = v.globalTags({ PROMPT: GRAPH }); + const clusterSize = 3 * GiB; + // The Tags position (relative to the Segment payload) is a 4-byte uint either way, + // so the SeekHead's length can be measured with a placeholder first. + const seekLength = v.seekHead([[ID.Tags, clusterSize]]).length; + const seekHead = v.seekHead([[ID.Tags, seekLength + 12 + clusterSize]]); + assert.equal(seekHead.length, seekLength); + const head = v.concat(v.ebmlHeader(), v.idBytes(ID.Segment), v.vint(seekHead.length + 12 + clusterSize + tags.length, { width: 8 }), seekHead, v.clusterHeader(clusterSize)); + const clusterStart = v.ebmlHeader().length + 12 + seekHead.length; + const tagsOffset = head.length + clusterSize; + const blob = v.sparseBlob([{ offset: 0, bytes: head }, { offset: tagsOffset, bytes: tags }], tagsOffset + tags.length); + const seeked = await readVideoTags(blob, { prefix: head.subarray(0, 32) }); + assert.deepEqual(seeked.tags, { PROMPT: GRAPH }); + const inside = blob.ranges.filter(([start]) => start >= clusterStart && start < tagsOffset); + assert.ok(inside.every(([start, end]) => start === clusterStart && end - start <= 16), JSON.stringify(inside)); + assert.ok(blob.ranges.reduce((sum, [start, end]) => sum + (end - start), 0) <= m.VIDEO_READ_CEILING); + // ffprobe reads level-1 elements up to the first Cluster: Tags behind it are found + // only through a SeekHead, so without one they are not read (nor sliced). + const behind = v.recordingBlob(v.mkv([v.cluster(64), v.globalTags({ PROMPT: GRAPH }), el(ID.Cues, new Uint8Array(2))])); + const sequential = await readVideoTags(behind); + assert.deepEqual(sequential.tags, {}); + assert.ok(behind.ranges.every(([, end]) => end <= v.ebmlHeader().length + 12 + 12 + 32), JSON.stringify(behind.ranges)); + // Track-targeted Tags are ignored; chapter and attachment targets too. + const targeted = await read(v.mkv([ + v.tagsEl( + v.tag([v.simpleTag("PROMPT", "track")], { trackUid: 1 }), + v.tag([v.simpleTag("WORKFLOW", WORKFLOW)], { targetType: 50 }), + v.tag([v.simpleTag("PROMPT", "zero uid is global")], { trackUid: 0 }), + ), + v.cluster(), + ])); + assert.deepEqual(targeted.tags, { WORKFLOW, PROMPT: "zero uid is global" }); + // An unknown-size Cluster (live recordings) is the same stop; a SeekHead still works. + const hidden = await read(v.mkv([v.cluster(8, { unknown: true }), v.globalTags({ PROMPT: GRAPH })])); + assert.deepEqual(hidden, complete({}, { container: "matroska", format: "matroska" })); + const before = await read(v.mkv([v.globalTags({ PROMPT: GRAPH }), v.cluster(8, { unknown: true })])); + assert.deepEqual(before.tags, { PROMPT: GRAPH }); + const info = el(ID.Info, new Uint8Array(4)); + const clusterUnknown = v.cluster(8, { unknown: true }); + const seekToTags = v.seekHead([[ID.Tags, 0x7f]]); // a one-byte position, like the real one + const viaSeek = await read(v.mkv([v.seekHead([[ID.Tags, seekToTags.length + info.length + clusterUnknown.length]]), info, clusterUnknown, v.globalTags({ PROMPT: GRAPH })])); + assert.deepEqual(viaSeek.tags, { PROMPT: GRAPH }); + // An unknown-size Segment is fine (live recordings); other unknown sizes are invalid. + const liveSegment = await read(v.mkv([v.globalTags({ PROMPT: GRAPH }), v.cluster()], { unknown: true })); + assert.deepEqual(liveSegment.tags, { PROMPT: GRAPH }); + const unknownTags = await read(v.mkv([el(ID.Tags, v.tag([v.simpleTag("PROMPT", GRAPH)]), v.opts({ unknown: true }))])); + assert.deepEqual(unknownTags, { status: "unreadable", reason: "invalid-container" }); + // ffmpeg's tag naming (checked against ffprobe 9.0.2): a language other than und + // files the tag under NAME-lang only, unless TagDefault is set; nested SimpleTags + // become PARENT/CHILD (never ours); a string-less tag deletes the key; a repeated + // name keeps the last value; NUL ends the string; Void and CRC-32 are skipped. + const naming = await read(v.mkv([ + v.voidEl(8), + v.tagsEl( + el(ID.CRC32, new Uint8Array(4)), + v.tag([ + v.simpleTag("PROMPT", "eng only", { lang: "eng" }), + v.simpleTag("WORKFLOW", "eng only", { lang: "eng", def: 0 }), + v.simpleTag("workflow", "default and eng", { lang: "eng", def: 1 }), + v.simpleTag("PARAMETERS", "und", { lang: "und", nested: [v.simpleTag("prompt", "nested")] }), + v.simpleTag("comfyassets_prompt_pairs", "cut\0here"), + v.simpleTag("description", undefined, { binary: [1, 2, 3] }), + v.simpleTag("comment", "note"), + v.simpleTag("comment", undefined), + v.simpleTag("prompt", "first"), + v.simpleTag("prompt", "last", { binary: [1] }), + ]), + ), + v.cluster(), + ])); + assert.deepEqual(naming.tags, { workflow: "default and eng", PARAMETERS: "und", comfyassets_prompt_pairs: "cut", prompt: "last" }); + // Long names are never ours and are skipped without reading their strings. + const longName = v.recordingBlob(v.mkv([v.tagsEl(v.tag([v.simpleTag("N".repeat(300), fill(0x41, 4000)), v.simpleTag("PROMPT", GRAPH)])), v.cluster()])); + const skipped = await readVideoTags(longName); + assert.deepEqual(skipped.tags, { PROMPT: GRAPH }); + assert.ok(longName.ranges.every(([start, end]) => end - start < 300)); + function el(id, ...parts) { + return v.el(id, ...parts); + } +}); + +test("containers fail closed", async () => { + const { ID } = v; + const invalid = { status: "unreadable", reason: "invalid-container" }; + const rows = [ + ["box size below its header", v.concat(v.ftyp(), v.box("free", v.opts({ size: 4 })), v.mdat(), v.moov(v.mdtaUdta({ prompt: GRAPH })))], + ["child box past its parent", v.concat(v.ftyp(), v.mdat(), v.moov(v.udta(v.meta(v.hdlr("mdta"), v.keys(["prompt"]), v.ilst(v.item(1, v.data(GRAPH), v.opts({ size: 5000 })))))))], + ["top-level box past the end of the file", v.concat(v.ftyp(), v.mdat(), v.box("moov", v.opts({ size: 4000 })))], + ["64-bit size above MAX_SAFE_INTEGER", v.concat(v.ftyp(), v.box("mdat", v.opts({ largesize: 2 ** 60 })), v.moov(v.mdtaUdta({ prompt: GRAPH })))], + ["64-bit size below its header", v.concat(v.ftyp(), v.box("mdat", v.opts({ largesize: 12 })), v.moov(v.mdtaUdta({ prompt: GRAPH })))], + ["keys count larger than the entries", v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys(["prompt"], { count: 3 }), v.ilst(v.item(1, v.data(GRAPH))))))], + ["keys entry overrunning the box", v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.box("keys", v.u32(0), v.u32(1), v.u32(90), v.latin1("mdta"), v.utf8("prompt")), v.ilst(v.item(1, v.data(GRAPH))))))], + ["ilst index 0", v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys(["prompt"]), v.ilst(v.item(0, v.data(GRAPH))))))], + ["ilst index out of range", v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys(["prompt"]), v.ilst(v.item(2, v.data(GRAPH))))))], + ["invalid UTF-8 in a tag value", v.perKeyMp4({ prompt: [0x7b, 0xff, 0xfe, 0x7d] })], + ["invalid UTF-8 in a key name", v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys([Uint8Array.from([0xc3, 0x28])]), v.ilst(v.item(1, v.data(GRAPH))))))], + ["no moov box", v.concat(v.ftyp(), v.mdat(), v.free(32))], + ["not a container at all", v.utf8("#EXTM3U\n#EXT-X-VERSION:3\n".padEnd(64, "x"))], + ["bad VINT: zero ID byte", v.concat(v.ebmlHeader(), v.idBytes(ID.Segment), v.vint(20, { width: 8 }), [0x00, 0x81, 0x00], new Uint8Array(17))], + ["bad VINT: zero size byte", v.concat(v.ebmlHeader(), v.idBytes(ID.Segment), v.vint(20, { width: 8 }), v.idBytes(ID.Info), [0x00], new Uint8Array(17))], + ["element past its parent", v.mkv([v.el(ID.Tags, v.tag([v.simpleTag("PROMPT", GRAPH)]), v.opts({ size: 5000 }))])], + ["element past the end of the file", v.concat(v.ebmlHeader(), v.idBytes(ID.Segment), v.vint(5000, { width: 8 }), v.globalTags({ PROMPT: GRAPH }))], + ["unknown DocType", v.perKeyMkv({ PROMPT: GRAPH }, { docType: "avi" })], + ["EBML header without a DocType", v.concat(v.el(ID.EBML, v.el(ID.EBMLVersion, [1])), v.segment([v.globalTags({ PROMPT: GRAPH })]))], + ["invalid UTF-8 in a TagString", v.mkv([v.tagsEl(v.tag([v.simpleTag("PROMPT", [0xff, 0xfe])]))])], + ["invalid UTF-8 in a TagName", v.mkv([v.tagsEl(v.tag([v.simpleTag([0xff], GRAPH)]))])], + ["Matroska bytes after a stray size", v.concat(v.ebmlHeader(), v.idBytes(ID.Segment), v.vint(3, { width: 8 }), [0xec, 0x82, 0x00])], + ]; + for (const [name, bytes] of rows) { + assert.deepEqual(await read(bytes), invalid, name); + } + // Trailing bytes shorter than a box header are ignored, as ffmpeg does. + assert.deepEqual((await read(v.concat(v.perKeyMp4({ prompt: GRAPH }), [1, 2, 3]))).tags, { prompt: GRAPH }); + // 10,001 boxes stop the read. + const many = v.concat(v.ftyp(), ...Array.from({ length: 10001 }, () => v.free()), v.mdat(), v.moov(v.mdtaUdta({ prompt: GRAPH }))); + assert.deepEqual(await read(many), { status: "limited", reason: "chunk-limit" }); + const manyElements = v.mkv([...Array.from({ length: 10001 }, () => v.voidEl(1)), v.globalTags({ PROMPT: GRAPH })]); + assert.deepEqual(await read(manyElements), { status: "limited", reason: "chunk-limit" }); + // Exactly 10,000 elements still read (ftyp, mdat, moov, udta, meta, hdlr, keys, its entry, ilst, the item and its data atom). + const exact = v.concat(v.ftyp(), ...Array.from({ length: 10000 - 11 }, () => v.free()), v.mdat(), v.moov(v.mdtaUdta({ prompt: GRAPH }))); + assert.deepEqual((await read(exact)).tags, { prompt: GRAPH }); + // A vanished file is a read failure, not a container error. + assert.deepEqual(await readVideoTags(v.failingBlob(4096), { prefix: v.ftyp() }), { status: "unreadable", reason: "read-failure" }); + assert.deepEqual(await readVideoTags({}), { status: "unreadable", reason: "read-failure" }); +}); + +test("budgets: roles share 2 MiB, an oversized workflow is left unread, duplicates and the total are bounded", async () => { + const limited = { status: "limited", reason: "metadata-limit" }; + // Role tags over 2 MiB together stop the read (the server fails the whole read). + assert.deepEqual(await read(v.perKeyMp4({ prompt: fill(0x41, ROLE_BUDGET + 1) })), limited); + assert.deepEqual(await read(v.perKeyMkv({ PROMPT: fill(0x41, ROLE_BUDGET - 10), PARAMETERS: fill(0x42, 11) })), limited); + const atRole = await read(v.perKeyMp4({ prompt: fill(0x41, ROLE_BUDGET - 10), comfyassets_prompt_pairs: fill(0x42, 10) })); + assert.equal(atRole.status, "complete"); + assert.equal(atRole.tags.prompt.length, ROLE_BUDGET - 10); + // A per-key workflow over 4 MiB is not sliced; the result says the workflow is limited. + const big = v.recordingBlob(v.perKeyMp4({ prompt: GRAPH, workflow: fill(0x7b, WORKFLOW_BUDGET + 1) })); + const over = await readVideoTags(big); + assert.deepEqual(over, { status: "complete", container: "iso-bmff", format: "isom", tags: { prompt: GRAPH }, limited: ["workflow"] }); + assert.ok(big.ranges.every(([start, end]) => end - start < WORKFLOW_BUDGET), "the oversized workflow was never sliced"); + const bigMkv = await read(v.perKeyMkv({ WORKFLOW: fill(0x7b, WORKFLOW_BUDGET + 1), PROMPT: GRAPH })); + assert.deepEqual([bigMkv.limited, Object.keys(bigMkv.tags)], [["workflow"], ["PROMPT"]]); + const atWorkflow = await read(v.perKeyMp4({ workflow: fill(0x7b, WORKFLOW_BUDGET) })); + assert.deepEqual([atWorkflow.limited, atWorkflow.tags.workflow.length], [[], WORKFLOW_BUDGET]); + // Fold-duplicates draw from the duplicate budgets (2x each class), then stop. + const twice = await read(v.perKeyMkv({ PROMPT: fill(0x41, ROLE_BUDGET), prompt: fill(0x41, ROLE_BUDGET) })); + assert.deepEqual(Object.keys(twice.tags).sort(), ["PROMPT", "prompt"]); + const thrice = await read(v.perKeyMkv({ PROMPT: fill(0x41, ROLE_BUDGET), prompt: fill(0x41, ROLE_BUDGET), Prompt: fill(0x41, ROLE_BUDGET + 1) })); + assert.deepEqual(thrice, limited); + const workflowTwice = await read(v.perKeyMkv({ WORKFLOW: fill(0x7b, WORKFLOW_BUDGET), workflow: fill(0x7b, WORKFLOW_BUDGET) })); + assert.equal(workflowTwice.status, "complete"); + // Envelopes count against the total only; the total is the server's probe bound. + const envelope = await read(v.mp4(v.legacyUdta({ comment: fill(0x41, 5 * MiB) }))); + assert.equal(envelope.tags.comment.length, 5 * MiB); + const tooMuch = await read(v.mp4(v.legacyUdta({ comment: fill(0x41, TAG_READ_BYTES - 10), description: fill(0x42, 11) }))); + assert.deepEqual(tooMuch, limited); + assert.equal(m.VIDEO_TAG_READ_BYTES, TAG_READ_BYTES); +}); + +test("avi and asf are recognised but not read", async () => { + const avi = v.concat(v.latin1("RIFF"), v.u32(1000), v.latin1("AVI LIST"), new Uint8Array(16)); + const asf = v.concat([0x30, 0x26, 0xb2, 0x75, 0x8e, 0x66, 0xcf, 0x11, 0xa6, 0xd9, 0x00, 0xaa, 0x00, 0x62, 0xce, 0x6c], new Uint8Array(16)); + for (const bytes of [avi, asf]) { + const blob = v.recordingBlob(v.concat(bytes, new Uint8Array(4096))); + assert.deepEqual(await readVideoTags(blob, { prefix: bytes }), { status: "unsupported", reason: "video-container-unsupported" }); + assert.deepEqual(blob.ranges, []); + } + // The classifier's kind can be handed in instead of a prefix. + const blob = v.recordingBlob(new Uint8Array(64)); + assert.deepEqual(await readVideoTags(blob, { kind: "avi" }), { status: "unsupported", reason: "video-container-unsupported" }); + assert.deepEqual(blob.ranges, []); + // Without a prefix the reader slices the first 32 bytes itself. + const mp4 = v.recordingBlob(v.perKeyMp4({ prompt: GRAPH })); + assert.deepEqual((await readVideoTags(mp4)).tags, { prompt: GRAPH }); + assert.deepEqual(mp4.ranges[0], [0, 32]); +}); + +test("abort during a moov walk settles once", async () => { + const unhandled = []; + const onUnhandled = (reason) => unhandled.push(reason); + process.on("unhandledRejection", onUnhandled); + try { + const bytes = v.concat(v.ftyp(), ...Array.from({ length: 50 }, () => v.free()), v.mdat(), v.moov(v.mdtaUdta({ prompt: GRAPH, workflow: WORKFLOW }))); + const controller = new AbortController(); + let slices = 0; + const blob = { + size: bytes.length, + slice(start, end) { + slices += 1; + if (slices === 3) controller.abort(); + return { arrayBuffer: () => new Promise((resolve) => setTimeout(() => resolve(bytes.slice(start, end).buffer), 2)) }; + }, + }; + await assert.rejects(readVideoTags(blob, { signal: controller.signal }), { name: "AbortError" }); + const already = new AbortController(); + already.abort(); + await assert.rejects(readVideoTags(v.blobOf(bytes), { signal: already.signal }), { name: "AbortError" }); + const slow = v.slowBlob(bytes, 1); + const during = new AbortController(); + const pending = readVideoTags(slow, { signal: during.signal, yieldControl: async () => during.abort() }); + await assert.rejects(pending, { name: "AbortError" }); + await new Promise((resolve) => setTimeout(resolve, 10)); + assert.deepEqual(unhandled, []); + } finally { + process.off("unhandledRejection", onUnhandled); + } +}); + +test("yieldControl is awaited while walking many boxes", async () => { + let yields = 0; + const many = v.concat(v.ftyp(), ...Array.from({ length: 3000 }, () => v.free()), v.mdat(), v.moov(v.mdtaUdta({ prompt: GRAPH }))); + const result = await readVideoTags(v.slowBlob(many, 0), { yieldControl: async () => { yields += 1; } }); + assert.equal(result.status, "complete"); + assert.ok(yields > 0, "the reader handed control back"); +}); + +test("carrier selection: per-key tags, then the comment envelope, then description, never merged", () => { + const ENVELOPE_WORKFLOW = '{"last_node_id": 9, "nodes": [{"id": 3, "type": "KSampler", "widgets_values": [7, "fixed", 20, 8.0, "euler", "normal", 1.0]}, {"id": 6, "type": "CLIPTextEncode", "widgets_values": ["a red fox in snow, detailed fur"]}, {"id": 7, "type": "CLIPTextEncode", "widgets_values": ["blurry, lowres"]}], "links": [], "version": 0.4}'; + const original = (keys) => Object.fromEntries(keys.map((key) => [key, "original-text"])); + const text = (tags, keys) => Object.fromEntries(keys.map((key) => [key, tags[key] ?? tags[key.toUpperCase()]])); + // The recorded fixtures. + for (const name of ["core-faststart.mp4", "core-moov-end.mp4", "core.mov", "core.m4v", "core.mkv", "core-tags.webm"]) { + const tags = RECORDED[name]; + const keys = Object.keys(tags).map((key) => key.toLowerCase()).filter((key) => PER_KEY.has(key)); + assert.deepEqual(selectVideoCarrier(tags), { status: "complete", records: text(tags, keys), record_formats: original(keys), carrierText: null, limited: [] }, name); + } + const legacy = RECORDED["legacy-comment.mp4"]; + const comment = pythonJsonLoads(legacy.comment); + assert.deepEqual(selectVideoCarrier(legacy), { + status: "complete", + records: { prompt: comment.get("prompt"), workflow: ENVELOPE_WORKFLOW }, + record_formats: { prompt: "original-text", workflow: "reserialized-json" }, + carrierText: legacy.comment, + limited: [], + }); + assert.ok(comment.get("prompt").startsWith('{"3":{"class_type":"KSampler"')); + // Matroska upper-cases COMMENT, and envelope names are exact-case (ruling M1): nothing is read. + assert.deepEqual(selectVideoCarrier(RECORDED["legacy-comment.webm"]), { status: "complete", records: {}, record_formats: {}, carrierText: null, limited: [] }); + // Synthetic maps. + const envelope = JSON.stringify({ prompt: '{"9":{}}', workflow: { v: 1 } }); + const cases = [ + ["per-key wins over comment", { prompt: GRAPH, workflow: WORKFLOW, comment: envelope }, { records: { prompt: GRAPH, workflow: WORKFLOW }, record_formats: original(["prompt", "workflow"]), carrierText: null }], + ["Matroska names fold", { PROMPT: GRAPH, Workflow: WORKFLOW, ENCODER: "Lavf" }, { records: { prompt: GRAPH, workflow: WORKFLOW }, record_formats: original(["prompt", "workflow"]), carrierText: null }], + ["identical fold-duplicates are one record", { PROMPT: GRAPH, prompt: GRAPH }, { records: { prompt: GRAPH }, record_formats: original(["prompt"]), carrierText: null }], + ["differing fold-duplicates fail", { PROMPT: GRAPH, prompt: '{"9":{}}' }, { status: "unreadable", reason: "duplicate-metadata" }], + ["differing workflow duplicates fail even with a prompt", { Workflow: WORKFLOW, WORKFLOW: "{}", prompt: GRAPH }, { status: "unreadable", reason: "duplicate-metadata" }], + ["envelope names never fold", { comment: envelope, COMMENT: "a free-text note" }, { records: { prompt: '{"9":{}}', workflow: '{"v": 1}' }, record_formats: { prompt: "original-text", workflow: "reserialized-json" }, carrierText: envelope }], + ["non-JSON comment falls through to description", { comment: "not json", description: envelope }, { records: { prompt: '{"9":{}}', workflow: '{"v": 1}' }, record_formats: { prompt: "original-text", workflow: "reserialized-json" }, carrierText: envelope }], + ["a JSON array comment is not an envelope", { comment: '["prompt"]', description: '{"workflow": {}}' }, { records: { workflow: "{}" }, record_formats: { workflow: "reserialized-json" }, carrierText: '{"workflow": {}}' }], + ["a role envelope beats a workflow-only per-key carrier", { workflow: WORKFLOW, comment: envelope }, { records: { prompt: '{"9":{}}', workflow: '{"v": 1}' }, record_formats: { prompt: "original-text", workflow: "reserialized-json" }, carrierText: envelope }], + ["role description beats workflow-only comment", { comment: '{"workflow": {"a": 1}}', description: '{"parameters": "cat"}' }, { records: { parameters: "cat" }, record_formats: { parameters: "original-text" }, carrierText: '{"parameters": "cat"}' }], + ["workflow-only carriers: the first wins, never merged", { comment: '{"workflow": {"a": 1}}', description: '{"workflow": {"b": 2}}' }, { records: { workflow: '{"a": 1}' }, record_formats: { workflow: "reserialized-json" }, carrierText: '{"workflow": {"a": 1}}' }], + ["description only", { description: envelope }, { records: { prompt: '{"9":{}}', workflow: '{"v": 1}' }, record_formats: { prompt: "original-text", workflow: "reserialized-json" }, carrierText: envelope }], + ["no known key", { encoder: "Lavf", major_brand: "isom" }, { records: {}, record_formats: {}, carrierText: null }], + ["envelope with only unknown keys is no carrier", { comment: '{"title": "x"}' }, { records: {}, record_formats: {}, carrierText: null }], + ["null role marker is refused", { comment: '{"prompt": null, "workflow": {}}' }, { status: "unreadable", reason: "invalid-metadata" }], + ["numeric role is refused", { description: '{"parameters": 5}' }, { status: "unreadable", reason: "invalid-metadata" }], + ["a non-text non-object workflow is invalid but the roles stay", { comment: '{"prompt": "{}", "workflow": null}' }, { records: { prompt: "{}" }, record_formats: { prompt: "original-text", workflow: "invalid" }, carrierText: '{"prompt": "{}", "workflow": null}' }], + ["an object prompt is reserialized", { comment: '{"prompt": {"1": {"class_type": "A"}}}' }, { records: { prompt: '{"1": {"class_type": "A"}}' }, record_formats: { prompt: "reserialized-json" }, carrierText: '{"prompt": {"1": {"class_type": "A"}}}' }], + ["a non-text per-key role marker is refused", { prompt: 5, workflow: WORKFLOW }, { status: "unreadable", reason: "invalid-metadata" }], + ["a non-text per-key workflow is invalid", { prompt: GRAPH, workflow: 5 }, { records: { prompt: GRAPH }, record_formats: { prompt: "original-text", workflow: "invalid" }, carrierText: null }], + ["a workflow-only envelope with an invalid workflow is still the carrier", { comment: '{"workflow": 5}' }, { records: {}, record_formats: { workflow: "invalid" }, carrierText: '{"workflow": 5}' }], + ["not a map", null, { records: {}, record_formats: {}, carrierText: null }], + ]; + for (const [name, tags, expected] of cases) { + const want = expected.status ? expected : { status: "complete", limited: [], ...expected }; + assert.deepEqual(selectVideoCarrier(tags), want, name); + } + // The server's budget loop: a 4 MiB workflow is limited, roles over 2 MiB fail, exact sizes pass. + const bigWorkflow = "{" + "0".repeat(WORKFLOW_BUDGET); + assert.deepEqual(selectVideoCarrier({ prompt: GRAPH, workflow: bigWorkflow }), { status: "complete", records: { prompt: GRAPH }, record_formats: { prompt: "original-text", workflow: "limited" }, carrierText: null, limited: ["workflow"] }); + assert.deepEqual(selectVideoCarrier({ workflow: bigWorkflow.slice(1) }).records, { workflow: bigWorkflow.slice(1) }); + assert.deepEqual(selectVideoCarrier({ comment: JSON.stringify({ prompt: "x", workflow: { k: "0".repeat(WORKFLOW_BUDGET) } }) }).record_formats, { prompt: "original-text", workflow: "limited" }); + assert.deepEqual(selectVideoCarrier({ prompt: "a".repeat(ROLE_BUDGET), parameters: "b" }), { status: "limited", reason: "metadata-limit" }); + assert.deepEqual(selectVideoCarrier({ prompt: "é".repeat(ROLE_BUDGET / 2) }).records.prompt.length, ROLE_BUDGET / 2); + assert.deepEqual(selectVideoCarrier({ prompt: "é".repeat(ROLE_BUDGET / 2) + "a" }), { status: "limited", reason: "metadata-limit" }); + assert.deepEqual(selectVideoCarrier({ prompt: "\ud800".repeat(ROLE_BUDGET / 3) + "aaa" }), { status: "limited", reason: "metadata-limit" }); + assert.deepEqual(selectVideoCarrier({ prompt: "\u{1f600}".repeat(ROLE_BUDGET / 4) }).records.prompt.length, ROLE_BUDGET / 2); + assert.deepEqual(selectVideoCarrier({ prompt: "\u{1f600}".repeat(ROLE_BUDGET / 4) + "a" }), { status: "limited", reason: "metadata-limit" }); + // The reader's limited workflow (the server's role-only re-probe): kept with the per-key + // carrier or no carrier, cleared when an envelope wins. + assert.deepEqual(selectVideoCarrier({ prompt: GRAPH }, { limited: ["workflow"] }).record_formats, { prompt: "original-text", workflow: "limited" }); + assert.deepEqual(selectVideoCarrier({ encoder: "x" }, { limited: ["workflow"] }), { status: "complete", records: {}, record_formats: { workflow: "limited" }, carrierText: null, limited: ["workflow"] }); + assert.deepEqual(selectVideoCarrier({ comment: envelope }, { limited: ["workflow"] }).limited, []); + assert.deepEqual(selectVideoCarrier({ comment: envelope }, { limited: ["workflow"] }).record_formats, { prompt: "original-text", workflow: "reserialized-json" }); +}); + +test("pythonJsonLoads and pythonJsonDumps spell values as Python's json module does", () => { + const rows = [ + ["1.0", "1.0"], ["8.0", "8.0"], ["0.1", "0.1"], ["1e16", "1e+16"], ["1e15", "1000000000000000.0"], + ["123456789012345678", "123456789012345678"], ["0.0001", "0.0001"], ["0.00001", "1e-05"], ["1e-7", "1e-07"], + ["1.5e300", "1.5e+300"], ["1e400", "Infinity"], ["-0", "0"], ["-0.0", "-0.0"], ["1E2", "100.0"], ["2.5", "2.5"], + ["100.0", "100.0"], ["1e21", "1e+21"], ["123456.789", "123456.789"], ["0.000123", "0.000123"], ["-1e-05", "-1e-05"], + ["9007199254740993", "9007199254740993"], ["1.7976931348623157e308", "1.7976931348623157e+308"], ["5e-324", "5e-324"], + ["NaN", "NaN"], ["-Infinity", "-Infinity"], ["Infinity", "Infinity"], ["[1.0e+2]", "[100.0]"], ["-1e400", "-Infinity"], + ["[]", "[]"], ["{}", "{}"], ["[true, false, null]", "[true, false, null]"], ['{"k":[1,{"z":1}]}', '{"k": [1, {"z": 1}]}'], + [' \t\n\r{"a" : 1 } ', '{"a": 1}'], + ['{"a":"é\\u0001\u007f\\"\\\\\\n\\t\\b\\f\\r\\/ 😀 \\ud800"}', '{"a": "\\u00e9\\u0001\\u007f\\"\\\\\\n\\t\\b\\f\\r/ \\ud83d\\ude00 \\ud800"}'], + ['{"a":"\\ud83d\\ude00"}', '{"a": "\\ud83d\\ude00"}'], + ['{"10":1,"3":2,"b":3,"a":4}', '{"10": 1, "3": 2, "b": 3, "a": 4}'], + ['{"a":1,"b":2,"a":3}', '{"a": 3, "b": 2}'], + ]; + for (const [input, output] of rows) { + assert.equal(pythonJsonDumps(pythonJsonLoads(input)), output, input); + } + const bad = ['{"a":1} x', '{"a":"\u0001"}', '{"a":"\\x"}', "01", "1.", ".5", "+1", "[1,]", '{"a":1,}', "{a:1}", "'a'", "", " ", "[", "nul", "-", "1e", '"\\u12"', "[1 2]", "tru", "Infinit"]; + for (const input of bad) { + assert.throws(() => pythonJsonLoads(input), { name: "PythonJsonError" }, JSON.stringify(input)); + } + assert.throws(() => pythonJsonLoads("[".repeat(2000) + "]".repeat(2000)), { name: "PythonJsonError", recursion: true }); + assert.throws(() => pythonJsonDumps(undefined), { name: "PythonJsonError" }); + assert.throws(() => pythonJsonLoads(5), { name: "PythonJsonError" }); + assert.equal(pythonJsonDumps(pythonJsonLoads("[".repeat(500) + "]".repeat(500))), "[".repeat(500) + "]".repeat(500)); + const parsed = pythonJsonLoads('{"s":"x","n":1,"f":1.5,"b":true,"z":null,"l":[1]}'); + assert.ok(parsed instanceof Map); + assert.deepEqual([...parsed.keys()], ["s", "n", "f", "b", "z", "l"]); + assert.equal(parsed.get("s"), "x"); + assert.equal(parsed.get("b"), true); + assert.equal(parsed.get("z"), null); + assert.ok(Array.isArray(parsed.get("l"))); +}); diff --git a/tests/test_local_inspect_parity.py b/tests/test_local_inspect_parity.py index 780feb94..f4b4db9a 100644 --- a/tests/test_local_inspect_parity.py +++ b/tests/test_local_inspect_parity.py @@ -1,4 +1,4 @@ -"""The browser's local PNG reader answers exactly as the server's on the same file. +"""The browser's local readers answer exactly as the server's on the same file. ``web/js/png-metadata.js`` ``readPngMetadataFile`` ports the bounded PNG reader in ``utils/prompt_inspector.py``. This test hands the inspection parity corpus (the @@ -7,6 +7,15 @@ Latin-1 and batch-duplicate PNGs, and the committed JPEG/WebP header fixtures to Node oracle, and asserts per file that records, record formats, dimensions and readability equal ``read_inspection_file``. The oracle reads each file through ``fs.openAsBlob`` slices, as the page reads a chosen ``File``. + +``web/js/workspace/local-video.mjs`` reads the container tags the server asks +ffprobe for and ports ``utils/video_metadata.select_video_carrier`` with the +record budgets of ``read_inspection_file``. The video tests hand the oracle the +committed container fixtures (``tests/fixtures/local-inspect/*.{mp4,mov,m4v,mkv, +webm}`` with their recorded ffprobe tag maps in ``video-tags-v1.json``), recorded +and synthetic tag maps, and multi-megabyte per-key containers built at test time, +and assert that the local carrier equals the server's, which selects from the +same tag map through a mocked probe, or from the same file through real ffprobe. """ import io @@ -19,6 +28,7 @@ import tempfile import unittest import zlib from pathlib import Path +from unittest.mock import patch from PIL import Image from PIL.PngImagePlugin import PngInfo @@ -33,16 +43,155 @@ from utils.prompt_inspector import ( try: from tests import inspection_parity as parity from tests.open_handles import assert_closed + from tests.test_utils_video_metadata import HAVE_FFMPEG, write_video except ImportError: # pragma: no cover - discover -s tests puts tests/ first import inspection_parity as parity from open_handles import assert_closed + from test_utils_video_metadata import HAVE_FFMPEG, write_video HERE = Path(__file__).resolve().parent ORACLE = HERE / "js" / "support" / "local-inspect-oracle.mjs" HEADER_FIXTURES = HERE / "fixtures" / "local-inspect" +VIDEO_FIXTURES = HEADER_FIXTURES +RECORDED_TAGS = json.loads( + (VIDEO_FIXTURES / "video-tags-v1.json").read_text(encoding="utf-8") +)["fixtures"] NODE_TIMEOUT_SECONDS = 180 PNG_CLASS = {"media": "image", "kind": "png"} _FOOTER = "Steps: 20, Sampler: Euler, Batch index: {}, Seed: 7" +MiB = 1024 * 1024 +GRAPH = '{"3":{"class_type":"KSampler","inputs":{"seed":7}}}' +WORKFLOW = '{"nodes":[],"version":0.4}' +ENVELOPE = json.dumps({"prompt": '{"9":{}}', "workflow": {"v": 1}}) +VIDEO_CONTAINERS = { + ".mp4": "iso-bmff", + ".mov": "iso-bmff", + ".m4v": "iso-bmff", + ".mkv": "matroska", + ".webm": "matroska", +} +PER_KEY_NAMES = {"prompt", "workflow", "parameters", "comfyassets_prompt_pairs"} + +# Tag maps as ffprobe would report them, covering every branch of the carrier rule: +# (name, tags, records the reader left out). The server answer is computed, not +# written down, so a change in either side fails here. +TAG_MAP_VECTORS = [ + ( + "per_key_wins_over_comment", + {"prompt": GRAPH, "workflow": WORKFLOW, "comment": ENVELOPE}, + [], + ), + ( + "matroska_names_fold", + {"PROMPT": GRAPH, "Workflow": WORKFLOW, "ENCODER": "Lavf"}, + [], + ), + ("identical_fold_duplicates", {"PROMPT": GRAPH, "prompt": GRAPH}, []), + ("differing_fold_duplicates_fail", {"PROMPT": GRAPH, "prompt": '{"9":{}}'}, []), + ( + "differing_workflow_duplicates_fail", + {"Workflow": WORKFLOW, "WORKFLOW": "{}", "prompt": GRAPH}, + [], + ), + ( + "envelope_names_never_fold", + {"comment": ENVELOPE, "COMMENT": "a free-text note"}, + [], + ), + ( + "non_json_comment_falls_to_description", + {"comment": "not json", "description": ENVELOPE}, + [], + ), + ( + "array_comment_is_not_an_envelope", + {"comment": '["prompt"]', "description": '{"workflow": {}}'}, + [], + ), + ( + "role_envelope_beats_workflow_only_per_key", + {"workflow": WORKFLOW, "comment": ENVELOPE}, + [], + ), + ( + "role_description_beats_workflow_comment", + {"comment": '{"workflow": {"a": 1}}', "description": '{"parameters": "cat"}'}, + [], + ), + ( + "workflow_only_carriers_never_merge", + {"comment": '{"workflow": {"a": 1}}', "description": '{"workflow": {"b": 2}}'}, + [], + ), + ("description_only", {"description": ENVELOPE}, []), + ("no_known_key", {"encoder": "Lavf", "major_brand": "isom"}, []), + ("envelope_without_known_keys", {"comment": '{"title": "x"}'}, []), + ("null_role_marker_refused", {"comment": '{"prompt": null, "workflow": {}}'}, []), + ("numeric_role_refused", {"description": '{"parameters": 5}'}, []), + ( + "invalid_envelope_workflow_keeps_roles", + {"comment": '{"prompt": "{}", "workflow": null}'}, + [], + ), + ( + "object_prompt_reserialized", + { + "comment": '{"prompt": {"1": {"class_type": "A", "f": 8.0, "n": 1e16, ' + '"u": "é😀"}}}' + }, + [], + ), + ("workflow_only_invalid_envelope", {"comment": '{"workflow": 5}'}, []), + ("non_text_per_key_role_refused", {"prompt": 5, "workflow": WORKFLOW}, []), + ("non_text_per_key_workflow_invalid", {"prompt": GRAPH, "workflow": 5}, []), + ( + "python_number_spellings", + { + "description": '{"workflow": {"a": [1.0, 0.1, 1E2, 1e-7, -0.0, -0, ' + "123456789012345678, 1e400, NaN, 0.00001, 1e15], " + '"b": "\\u0001\\u007f/"}}' + }, + [], + ), + ( + "integer_like_keys_keep_order", + {"description": '{"workflow": {"10": 1, "3": 2, "b": 3, "a": 4, "3": 5}}'}, + [], + ), + ( + "workflow_over_budget_limited", + {"prompt": GRAPH, "workflow": "{" + "0" * WORKFLOW_METADATA_BYTES}, + [], + ), + ("workflow_at_budget_kept", {"workflow": "0" * WORKFLOW_METADATA_BYTES}, []), + ( + "roles_over_budget_fail", + {"prompt": "a" * ROLE_METADATA_BYTES, "parameters": "b"}, + [], + ), + ("roles_at_budget_kept", {"prompt": "é" * (ROLE_METADATA_BYTES // 2)}, []), + ( + "lone_surrogates_count_three_bytes", + {"prompt": "\ud800" * (ROLE_METADATA_BYTES // 3) + "aaa"}, + [], + ), + ( + "envelope_workflow_over_budget_limited", + { + "comment": json.dumps( + {"prompt": "x", "workflow": {"k": "0" * WORKFLOW_METADATA_BYTES}} + ) + }, + [], + ), + ("reader_limited_workflow_with_per_key", {"prompt": GRAPH}, ["workflow"]), + ("reader_limited_workflow_without_carrier", {"encoder": "x"}, ["workflow"]), + ( + "reader_limited_workflow_cleared_by_envelope", + {"comment": ENVELOPE}, + ["workflow"], + ), +] def _raw_chunk(kind, data): @@ -175,6 +324,156 @@ def _classes(results): return {name: entry["classified"] for name, entry in results.items()} +def _run_oracle(testcase, entries): + node = shutil.which("node") + if node is None: # pragma: no cover - CI runners and dev hosts ship Node + if os.environ.get("CI"): + testcase.fail("Node.js is required in CI for the local reader parity test") + testcase.skipTest("Node.js is not installed") + # Fixed local Node harness, no shell. + completed = subprocess.run( # nosec B603 + [node, str(ORACLE)], + input=json.dumps(entries).encode("utf-8"), + capture_output=True, + timeout=NODE_TIMEOUT_SECONDS, + check=False, + ) + testcase.assertEqual(completed.returncode, 0, completed.stderr.decode("utf-8")) + return json.loads(completed.stdout) + + +def _known_tags(tags): + """The tags the local reader keeps: per-key names folded, envelope names exact.""" + return { + key: value + for key, value in tags.items() + if key.lower() in PER_KEY_NAMES or key in ("comment", "description") + } + + +def _fake_probe(tags, drop_workflow=False): + """``_probe_video_tags`` answering from a tag map as ffprobe would. + + ffprobe's ``format_tags=`` filter is case-insensitive and the stdout bound is + the only reason a probe answers ``None``; ``drop_workflow`` makes the first + probe (the one asking for ``workflow``) overflow, as a huge per-key workflow + tag does, so the server takes its role-only re-probe. + """ + + def probe(binary, path, keys, limit): + if drop_workflow and "workflow" in keys: + return None + selected = {key: value for key, value in tags.items() if key.lower() in keys} + raw = json.dumps({"format": {"tags": selected}}, ensure_ascii=False).encode( + "utf-8", "surrogatepass" + ) + return None if len(raw) > limit else raw + + return probe + + +def _server_selection(tags, limited=()): + """The server's carrier for a tag map, in the oracle's shape; None when it fails.""" + with ( + patch("utils.prompt_inspector.ffprobe_path", return_value="/tool/ffprobe"), + patch( + "utils.prompt_inspector._probe_video_tags", + side_effect=_fake_probe(tags, drop_workflow="workflow" in limited), + ), + ): + try: + read = read_inspection_file(Path("vector.mp4")) + except MetadataReadError: + return None + return _read_shape(read) + + +def _read_shape(read): + return { + "records": read.metadata, + "record_formats": read.record_formats, + "carrierText": read.video_carrier_text, + "limited": sorted(read.limited_records), + } + + +def _local_shape(selection): + if selection["status"] != "complete": + return None + return { + "records": selection["records"], + "record_formats": selection["record_formats"], + "carrierText": selection["carrierText"], + "limited": sorted(selection["limited"]), + } + + +def _box(kind, *parts): + payload = b"".join(parts) + return struct.pack(">I", 8 + len(payload)) + kind + payload + + +def _full_box(kind, *parts): + return _box(kind, b"\0\0\0\0", *parts) + + +def _mdta_udta(tags): + """``moov/udta/meta`` with one mdta key per tag, laid out as ffmpeg writes it.""" + names = list(tags) + keys = _full_box( + b"keys", + struct.pack(">I", len(names)), + *( + struct.pack(">I", 8 + len(name.encode("utf-8"))) + + b"mdta" + + name.encode("utf-8") + for name in names + ), + ) + items = b"".join( + _box( + struct.pack(">I", index + 1), + _box(b"data", struct.pack(">II", 1, 0), tags[name].encode("utf-8")), + ) + for index, name in enumerate(names) + ) + hdlr = _full_box(b"hdlr", b"\0\0\0\0", b"mdta", bytes(12), b"\0") + return _box(b"udta", _full_box(b"meta", hdlr, keys, _box(b"ilst", items))) + + +def _with_udta(data, udta): + """``data`` (an MP4 with moov after mdat) with its moov's udta replaced.""" + out = bytearray() + offset = 0 + while offset < len(data): + size, kind = struct.unpack(">I4s", data[offset : offset + 8]) + if kind != b"moov": + out += data[offset : offset + size] + offset += size + continue + body = data[offset + 8 : offset + size] + children = bytearray() + inner = 0 + while inner < len(body): + child_size, child_kind = struct.unpack(">I4s", body[inner : inner + 8]) + if child_kind != b"udta": + children += body[inner : inner + child_size] + inner += child_size + children += udta + out += struct.pack(">I", 8 + len(children)) + b"moov" + bytes(children) + offset += size + return bytes(out) + + +def _text_of(size, prefix): + """A valid JSON object of exactly ``size`` UTF-8 bytes with quotes throughout.""" + unit = '"k":"v",' + body = (unit * (size // len(unit) + 1))[: size - len(prefix) - 2] + body = body[: body.rfind(",")] if body.endswith(",") else body + text = prefix + body + "}" + return text + " " * (size - len(text)) + + class LocalPngReaderParityTests(unittest.TestCase): def setUp(self): temporary = tempfile.TemporaryDirectory() @@ -199,24 +498,9 @@ class LocalPngReaderParityTests(unittest.TestCase): return files def _oracle(self, files): - node = shutil.which("node") - if node is None: # pragma: no cover - CI runners and dev hosts ship Node - if os.environ.get("CI"): - self.fail("Node.js is required in CI for the local reader parity test") - self.skipTest("Node.js is not installed") - payload = json.dumps( - [{"name": name, "path": str(path)} for name, path in files.items()] + return _run_oracle( + self, [{"name": name, "path": str(path)} for name, path in files.items()] ) - # Fixed local Node harness, no shell. - completed = subprocess.run( # nosec B603 - [node, str(ORACLE)], - input=payload.encode("utf-8"), - capture_output=True, - timeout=NODE_TIMEOUT_SECONDS, - check=False, - ) - self.assertEqual(completed.returncode, 0, completed.stderr.decode("utf-8")) - return json.loads(completed.stdout) def test_local_png_reader_matches_read_inspection_file(self): files = self._files() @@ -272,5 +556,139 @@ class LocalPngReaderParityTests(unittest.TestCase): self.assertLess(path.stat().st_size, 4096) +class LocalVideoCarrierParityTests(unittest.TestCase): + """The local video readers and carrier rule answer as the server does.""" + + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.addCleanup(assert_closed, self, temporary.name) + self.directory = Path(temporary.name) + + def test_local_carrier_matches_server_selection(self): + vectors = [(name, tags, []) for name, tags in RECORDED_TAGS.items()] + vectors += TAG_MAP_VECTORS + observed = _run_oracle( + self, + [ + {"name": name, "tags": tags, "limited": limited} + for name, tags, limited in vectors + ], + ) + outcomes = set() + for name, tags, limited in vectors: + with self.subTest(vector=name): + selection = observed[name]["result"]["selection"] + server = _server_selection(tags, limited) + self.assertEqual( + _local_shape(selection), server, selection.get("reason") + ) + if server is None: + outcomes.add(selection["status"] + ":" + selection["reason"]) + elif not server["records"] and not server["record_formats"]: + outcomes.add("no-carrier") + else: + outcomes.update(server["record_formats"].values()) + # Every outcome of the rule must occur, or agreement proves little. + self.assertEqual( + outcomes, + { + "original-text", + "reserialized-json", + "invalid", + "limited", + "no-carrier", + "unreadable:duplicate-metadata", + "unreadable:invalid-metadata", + "limited:metadata-limit", + }, + ) + + def test_local_video_readers_match_recorded_tags_and_server(self): + files = {name: VIDEO_FIXTURES / name for name in RECORDED_TAGS} + observed = _run_oracle( + self, [{"name": name, "path": str(path)} for name, path in files.items()] + ) + for name, path in files.items(): + with self.subTest(fixture=name): + self.assertLess(path.stat().st_size, 16 * 1024) + container = VIDEO_CONTAINERS[path.suffix] + self.assertEqual( + observed[name]["classified"], {"media": "video", "kind": container} + ) + read = observed[name]["result"]["read"] + self.assertEqual(read["status"], "complete", read) + self.assertEqual(read["container"], container) + self.assertEqual(read["tags"], _known_tags(RECORDED_TAGS[name])) + self.assertEqual(read["limited"], []) + local = _local_shape(observed[name]["result"]["selection"]) + self.assertEqual(local, _server_selection(RECORDED_TAGS[name])) + if HAVE_FFMPEG: + self.assertEqual(local, _read_shape(read_inspection_file(path))) + # The corpus covers per-key carriers in every container, an envelope and a + # Matroska COMMENT that the exact-case envelope rule leaves unread. + self.assertTrue( + _server_selection(RECORDED_TAGS["legacy-comment.mp4"])["carrierText"] + ) + self.assertEqual( + _server_selection(RECORDED_TAGS["legacy-comment.webm"])["records"], {} + ) + + @unittest.skipUnless(HAVE_FFMPEG, "ffmpeg/ffprobe not installed") + def test_large_per_key_videos_read_the_same_as_the_server(self): + base = self.directory / "base.mp4" + write_video(base) + source = base.read_bytes() + shapes = { + "available": { + "prompt": _text_of( + MiB, '{"1":{"class_type":"Note","inputs":{"text":"a"}},' + ), + "workflow": _text_of(5 * MiB // 2, '{"nodes":[],"version":0.4,'), + }, + "workflow_limited": { + "prompt": GRAPH, + "workflow": "[" + "0," * (WORKFLOW_METADATA_BYTES // 2) + "]", + }, + "roles_over_budget": {"prompt": "A" * (ROLE_METADATA_BYTES + 1)}, + } + files = {} + for name, tags in shapes.items(): + path = self.directory / f"{name}.mp4" + path.write_bytes(_with_udta(source, _mdta_udta(tags))) + files[name] = path + observed = _run_oracle( + self, [{"name": name, "path": str(path)} for name, path in files.items()] + ) + available = observed["available"]["result"] + self.assertEqual(available["read"]["status"], "complete") + self.assertEqual(available["read"]["limited"], []) + self.assertEqual( + available["selection"]["records"], shapes["available"], "1 MiB + 2.5 MiB" + ) + self.assertEqual( + _local_shape(available["selection"]), + _read_shape(read_inspection_file(files["available"])), + ) + limited = observed["workflow_limited"]["result"] + self.assertEqual(limited["read"]["limited"], ["workflow"]) + self.assertNotIn("workflow", limited["read"]["tags"]) + self.assertEqual( + _local_shape(limited["selection"]), + _read_shape(read_inspection_file(files["workflow_limited"])), + ) + self.assertEqual( + limited["selection"]["record_formats"], + {"prompt": "original-text", "workflow": "limited"}, + ) + over = observed["roles_over_budget"]["result"] + self.assertEqual( + over["read"], {"status": "limited", "reason": "metadata-limit"} + ) + self.assertIsNone(over["selection"]) + with self.assertRaises(MetadataReadError): + read_inspection_file(files["roles_over_budget"]) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_utils_video_metadata.py b/tests/test_utils_video_metadata.py index 1cacefa8..f8918780 100644 --- a/tests/test_utils_video_metadata.py +++ b/tests/test_utils_video_metadata.py @@ -390,5 +390,44 @@ class TestReadVideoMetadataWithRealFfprobe(unittest.TestCase): self.assertEqual(read_video_metadata(path), {}) +@unittest.skipUnless(HAVE_FFMPEG, "ffmpeg/ffprobe not installed") +class VideoFixtureFfprobeTests(unittest.TestCase): + """Characterization: the committed local-inspect video fixtures still carry the + tag maps recorded in ``video-tags-v1.json``, which the browser reader parity + tests use in place of ffprobe. A different ffprobe that reports other names or + values for the same bytes shows up here, not as a silent parity gap.""" + + FIXTURES = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "fixtures", "local-inspect" + ) + + def test_committed_fixtures_produce_their_recorded_tags(self): + with open( + os.path.join(self.FIXTURES, "video-tags-v1.json"), encoding="utf-8" + ) as stream: + recorded = json.load(stream)["fixtures"] + self.assertEqual(len(recorded), 8) + for name, tags in recorded.items(): + with self.subTest(fixture=name): + probe = subprocess.run( + [ + shutil.which("ffprobe"), + "-v", + "quiet", + "-show_entries", + "format_tags", + "-of", + "json", + os.path.join(self.FIXTURES, name), + ], + capture_output=True, + text=True, + encoding="utf-8", + check=True, + timeout=60, + ) + self.assertEqual(json.loads(probe.stdout)["format"]["tags"], tags) + + if __name__ == "__main__": unittest.main() diff --git a/tools/vendor/frontend-coverage-baseline.json b/tools/vendor/frontend-coverage-baseline.json index b4bfa2f0..25b3a65a 100644 --- a/tools/vendor/frontend-coverage-baseline.json +++ b/tools/vendor/frontend-coverage-baseline.json @@ -73,7 +73,7 @@ }, "png-metadata.js": { "lines": 98, - "branches": 90, + "branches": 96, "functions": 98, "statements": 98 }, @@ -293,6 +293,12 @@ "functions": 96, "statements": 96 }, + "workspace/local-video.mjs": { + "lines": 98, + "branches": 93, + "functions": 98, + "statements": 98 + }, "workspace/media-viewer.mjs": { "lines": 98, "branches": 93, diff --git a/web/js/workspace/local-video.mjs b/web/js/workspace/local-video.mjs new file mode 100644 index 00000000..eebf4f12 --- /dev/null +++ b/web/js/workspace/local-video.mjs @@ -0,0 +1,896 @@ +// Local video tag readers for the Inspect page (#249): bounded ISO-BMFF (MP4/MOV/M4V) +// and Matroska (MKV/WebM) walkers that read only the container tags the server asks +// ffprobe for (utils/prompt_inspector.py _video_metadata), and a port of the carrier +// rule utils/video_metadata.py select_video_carrier plus the record budgets of +// read_inspection_file. Nothing here uploads, stores or logs the file. The walkers +// slice box and element headers, the tag names and the retained tag values only, so +// a multi-gigabyte video costs at most VIDEO_READ_CEILING bytes of reads and never a +// byte inside mdat or a Cluster. tests/test_local_inspect_parity.py proves the +// answers equal the server's on the committed fixtures and recorded tag maps. +// +// What ffprobe reports is the oracle: ISO-BMFF mdta keys are reported verbatim and +// the legacy ilst atoms ©cmt/desc as comment/description; Matroska TagNames are +// reported as written (ffmpeg upper-cases them at mux time). Envelope names are +// exact-case (comment, description) and the per-key names fold ASCII case in the +// selector, exactly as the server folds them. + +// Role records (prompt, parameters, the pair execution record) share one budget: +// past it the read fails. A workflow has its own budget; past it only the workflow +// is "limited". Duplicate tags draw from separate budgets per class. The total of +// tag bytes read is the server's ffprobe stdout bound. +export const ROLE_METADATA_BYTES = 2 * 1024 * 1024; +export const WORKFLOW_METADATA_BYTES = 4 * 1024 * 1024; +export const DUPLICATE_WORKFLOW_BYTES = 2 * WORKFLOW_METADATA_BYTES; +export const DUPLICATE_ROLE_BYTES = 2 * ROLE_METADATA_BYTES; +export const VIDEO_TAG_READ_BYTES = 2 * (ROLE_METADATA_BYTES + WORKFLOW_METADATA_BYTES) + 64 * 1024; +export const MAX_VIDEO_ELEMENTS = 10000; +const MAX_HEADER_BYTES = 16; // a 64-bit ISO-BMFF box header; an EBML header is at most 12 +const MAX_NAME_BYTES = 64; // the longest carrier name is 24 bytes; longer names are never ours +// Every byte the readers can slice from a video of any size: each element's header +// and name plus the retained tag values. +export const VIDEO_READ_CEILING = MAX_VIDEO_ELEMENTS * (MAX_HEADER_BYTES + MAX_NAME_BYTES) + VIDEO_TAG_READ_BYTES; +const PREFIX_BYTES = 32; +const YIELD_INTERVAL_MS = 8; +const JSON_MAX_DEPTH = 1000; // Python's default recursion limit + +const ROLE_KEYS = ["prompt", "parameters", "comfyassets_prompt_pairs"]; +const METADATA_KEYS = ["prompt", "workflow", "parameters", "comfyassets_prompt_pairs"]; +const ENVELOPE_KEYS = ["comment", "description"]; +const PER_KEY_NAMES = new Set(METADATA_KEYS); +const ENVELOPE_NAMES = new Set(ENVELOPE_KEYS); +// ffprobe's names for the legacy ilst atoms that can carry an envelope. +const LEGACY_ATOMS = new Map([["©cmt", "comment"], ["desc", "description"], ["©des", "description"]]); +const ASF_GUID = [0x30, 0x26, 0xb2, 0x75, 0x8e, 0x66, 0xcf, 0x11, 0xa6, 0xd9, 0x00, 0xaa, 0x00, 0x62, 0xce, 0x6c]; + +const EBML = { + header: 0x1a45dfa3, + docType: 0x4282, + segment: 0x18538067, + seekHead: 0x114d9b74, + seek: 0x4dbb, + seekId: 0x53ab, + seekPosition: 0x53ac, + tags: 0x1254c367, + tag: 0x7373, + targets: 0x63c0, + trackUid: 0x63c5, + chapterUid: 0x63c4, + attachmentUid: 0x63c6, + simpleTag: 0x67c8, + tagName: 0x45a3, + tagString: 0x4487, + tagLanguage: 0x447a, + tagDefault: 0x4484, + cluster: 0x1f43b675, +}; +const TAGS_ID_BYTES = [0x12, 0x54, 0xc3, 0x67]; + +class VideoReadError extends Error { + constructor(status, reason, message) { + super(message); + this.name = "VideoReadError"; + this.status = status; + this.reason = reason; + } +} + +const invalid = (message) => new VideoReadError("unreadable", "invalid-container", message); +const limited = (reason, message) => new VideoReadError("limited", reason, message); +const readFailure = (message) => new VideoReadError("unreadable", "read-failure", message); + +function abortError() { + return new DOMException("The file read was aborted", "AbortError"); +} + +function throwIfAborted(signal) { + if (signal && signal.aborted) throw abortError(); +} + +function toBytes(input) { + if (input instanceof Uint8Array) return input; + if (input instanceof ArrayBuffer) return new Uint8Array(input); + if (Array.isArray(input)) return Uint8Array.from(input); + return null; +} + +function latin1(bytes, start, end) { + let out = ""; + for (let index = start; index < end; index += 1) out += String.fromCharCode(bytes[index]); + return out; +} + +function startsWith(bytes, prefix, at = 0) { + if (bytes.length < at + prefix.length) return false; + for (let index = 0; index < prefix.length; index += 1) if (bytes[at + index] !== prefix[index]) return false; + return true; +} + +function readUint32(bytes, offset) { + return ((bytes[offset] << 24) >>> 0) + (bytes[offset + 1] << 16) + (bytes[offset + 2] << 8) + bytes[offset + 3]; +} + +/** A big-endian unsigned integer of up to 8 bytes as a Number, or null past MAX_SAFE_INTEGER. */ +function readUintBytes(bytes, start, end) { + let value = 0n; + for (let index = start; index < end; index += 1) value = (value << 8n) | BigInt(bytes[index]); + return value <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(value) : null; +} + +function concatBytes(parts, total) { + const out = new Uint8Array(total); + let offset = 0; + for (const part of parts) { + out.set(part, offset); + offset += part.length; + } + return out; +} + +/** ffmpeg stores tag text as a C string: the first NUL ends it. */ +function untilNul(bytes) { + const nul = bytes.indexOf(0); + return nul < 0 ? bytes : bytes.subarray(0, nul); +} + +/** Strict UTF-8, as Python decodes ffprobe's output: invalid bytes fail the read. */ +function utf8Strict(bytes) { + try { + return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes); + } catch (error) { + throw invalid("Video metadata is not valid UTF-8"); + } +} + +/** UTF-8 length as Python's ``encode("utf-8", "surrogatepass")`` counts it. */ +function utf8Length(text) { + let total = 0; + for (let index = 0; index < text.length; index += 1) { + const code = text.charCodeAt(index); + if (code < 0x80) total += 1; + else if (code < 0x800) total += 2; + else if (code >= 0xd800 && code <= 0xdbff && index + 1 < text.length) { + const next = text.charCodeAt(index + 1); + if (next >= 0xdc00 && next <= 0xdfff) { + total += 4; + index += 1; + } else total += 3; + } else total += 3; + } + return total; +} + +function defaultYield() { + return new Promise((resolve) => setTimeout(resolve, 0)); +} + +/** Hands the event loop a turn about every YIELD_INTERVAL_MS of work. */ +function createPacer(yieldControl, signal) { + let last = performance.now(); + return { + async maybeYield() { + if (performance.now() - last < YIELD_INTERVAL_MS) return; + await yieldControl(); + throwIfAborted(signal); + last = performance.now(); + }, + }; +} + +/** + * Logical byte ranges of `file`, served from the retained `prefix` where it covers + * them (the twin of png-metadata.js's reader). Every sliced byte is charged against + * VIDEO_READ_CEILING, and every range is checked against `file.size` before slicing. + */ +function createSliceReader(file, prefix, signal) { + const retained = prefix || new Uint8Array(0); + let charged = 0; + return { + async read(start, end) { + throwIfAborted(signal); + if (!Number.isSafeInteger(end) || end > file.size || start > end) throw invalid("Incomplete video container"); + if (end <= retained.length) return retained.subarray(start, end); + const head = start < retained.length ? retained.subarray(start) : null; + const sliceStart = head ? retained.length : start; + charged += end - sliceStart; + if (charged > VIDEO_READ_CEILING) throw readFailure("Video read exceeds the inspection ceiling"); + const buffer = await file.slice(sliceStart, end).arrayBuffer(); + throwIfAborted(signal); + const bytes = new Uint8Array(buffer); + if (bytes.length !== end - sliceStart) throw readFailure("The file changed while it was being read"); + return head ? concatBytes([head, bytes], end - start) : bytes; + }, + }; +} + +/** Container kind from the first bytes, mirroring png-metadata.js classifyPrefix. */ +function detectKind(bytes) { + if (!bytes) return null; + if (bytes.length >= 12 && latin1(bytes, 0, 4) === "RIFF") return latin1(bytes, 8, 12) === "AVI " ? "avi" : null; + if (bytes.length >= 8 && latin1(bytes, 4, 8) === "ftyp") return "iso-bmff"; + if (startsWith(bytes, [0x1a, 0x45, 0xdf, 0xa3])) return "matroska"; + if (startsWith(bytes, ASF_GUID)) return "asf"; + return null; +} + +/** + * The shared walk state: the element count, the tag budgets and the collected tags. + * `value` is the one gate every tag value passes before it is sliced. + */ +function createWalk(reader, signal, pacer) { + const tags = {}; + const limitedRecords = []; + const seen = new Set(); + let count = 0; + let role = 0; + let workflow = 0; + let duplicateRole = 0; + let duplicateWorkflow = 0; + let total = 0; + const overLimit = () => limited("metadata-limit", "Video metadata exceeds inspection limits"); + return { + tags, + limited: limitedRecords, + async header(offset, length) { + count += 1; + if (count > MAX_VIDEO_ELEMENTS) throw limited("chunk-limit", "Video container exceeds inspection limits"); + await pacer.maybeYield(); + return reader.read(offset, offset + length); + }, + read: (start, end) => reader.read(start, end), + /** Whether `name` is a carrier the server reads (per-key folded, envelope exact). */ + known(name) { + return PER_KEY_NAMES.has(name.toLowerCase()) || ENVELOPE_NAMES.has(name); + }, + remove(name) { + delete tags[name]; + }, + /** + * Reads and keeps the tag `name` (a known carrier, see `known`) whose value + * spans [start, end), within the budgets. `join` appends to an existing value + * with that separator (ISO-BMFF repeats, as ffprobe reports them); `text` + * stores an already decoded value. + */ + async value(name, start, end, { join = null, text = null } = {}) { + const size = end - start; + const folded = name.toLowerCase(); + if (PER_KEY_NAMES.has(folded)) { + const first = !seen.has(folded); + seen.add(folded); + if (folded === "workflow") { + if (first && size > WORKFLOW_METADATA_BYTES) { + // The server's role-only re-probe: the per-key workflow is left out. + if (!limitedRecords.includes("workflow")) limitedRecords.push("workflow"); + return; + } + if (first) workflow += size; + else duplicateWorkflow += size; + if (duplicateWorkflow > DUPLICATE_WORKFLOW_BYTES) throw overLimit(); + } else { + if (first) role += size; + else duplicateRole += size; + if (role > ROLE_METADATA_BYTES || duplicateRole > DUPLICATE_ROLE_BYTES) throw overLimit(); + } + } + total += size; + if (total > VIDEO_TAG_READ_BYTES) throw overLimit(); + const value = text ?? utf8Strict(untilNul(await reader.read(start, end))); + tags[name] = join !== null && name in tags ? `${tags[name]}${join}${value}` : value; + }, + }; +} + +// ------------------------------------------------------------------ ISO-BMFF + +/** One box header at `offset` inside a parent ending at `parentEnd`, or null when fewer than 8 bytes remain. */ +async function readBox(walk, offset, parentEnd) { + if (parentEnd - offset < 8) return null; + const bytes = await walk.header(offset, Math.min(MAX_HEADER_BYTES, parentEnd - offset)); + let size = readUint32(bytes, 0); + let bodyStart = offset + 8; + if (size === 1) { + if (bytes.length < 16) throw invalid("Truncated 64-bit box size"); + const large = readUintBytes(bytes, 8, 16); + if (large === null || large < 16) throw invalid("Invalid 64-bit box size"); + size = large; + bodyStart = offset + 16; + } else if (size === 0) { + size = parentEnd - offset; + } else if (size < 8) { + throw invalid("Box size below its header"); + } + const end = offset + size; + if (end > parentEnd) throw invalid("Box extends past its parent"); + return { type: latin1(bytes, 4, 8), raw: bytes.subarray(4, 8), start: offset, bodyStart, end }; +} + +async function readIsoBmff(walk, fileSize) { + const state = { hdlrMdta: false, keys: null, format: null }; + let offset = 0; + let box; + while ((box = await readBox(walk, offset, fileSize))) { + if (box.type === "ftyp" && box.end - box.bodyStart >= 4) { + state.format = latin1(await walk.read(box.bodyStart, box.bodyStart + 4), 0, 4).replace(/[\s\0]+$/, ""); + } else if (box.type === "moov") { + await readMoov(walk, box, state); + return state.format; + } + offset = box.end; + } + throw invalid("No moov box"); +} + +async function readMoov(walk, moov, state) { + let offset = moov.bodyStart; + let box; + while ((box = await readBox(walk, offset, moov.end))) { + if (box.type === "meta") await readMeta(walk, box, state); + else if (box.type === "udta") { + let inner = box.bodyStart; + let child; + while ((child = await readBox(walk, inner, box.end))) { + if (child.type === "meta") await readMeta(walk, child, state); + inner = child.end; + } + } + offset = box.end; + } +} + +/** `meta` as ffmpeg reads it: scan 4-byte words for the first child's `hdlr` type. */ +async function readMeta(walk, meta, state) { + const scan = await walk.read(meta.bodyStart, Math.min(meta.end, meta.bodyStart + MAX_NAME_BYTES)); + let childrenStart = null; + for (let word = 4; word + 4 <= scan.length; word += 4) { + if (latin1(scan, word, word + 4) === "hdlr") { + childrenStart = meta.bodyStart + word - 4; + break; + } + } + if (childrenStart === null) return; + let offset = childrenStart; + let box; + while ((box = await readBox(walk, offset, meta.end))) { + if (box.type === "hdlr") { + if (box.end - box.bodyStart >= 12) { + const handler = await walk.read(box.bodyStart + 8, box.bodyStart + 12); + if (latin1(handler, 0, 4) === "mdta") state.hdlrMdta = true; + } + } else if (box.type === "keys") await readKeys(walk, box, state); + else if (box.type === "ilst") await readIlst(walk, box, state); + offset = box.end; + } +} + +/** The `keys` box: 1-based mdta names; other namespaces and overlong names address nothing. */ +async function readKeys(walk, box, state) { + if (box.end - box.bodyStart < 8) return; + const head = await walk.read(box.bodyStart, box.bodyStart + 8); + const count = readUint32(head, 4); + const keys = [null]; + let offset = box.bodyStart + 8; + for (let index = 1; index <= count; index += 1) { + if (box.end - offset < 8) throw invalid("Keys box count exceeds its entries"); + const entry = await walk.header(offset, 8); + const keySize = readUint32(entry, 0); + if (keySize < 8 || offset + keySize > box.end) throw invalid("Keys entry exceeds its box"); + const nameLength = keySize - 8; + let name = null; + if (latin1(entry, 4, 8) === "mdta" && nameLength <= MAX_NAME_BYTES) { + name = utf8Strict(await walk.read(offset + 8, offset + keySize)); + } + keys.push(name); + offset += keySize; + } + state.keys = keys; +} + +/** ffprobe's text for an integer `data` atom (types 21 and 22) of 1, 2, 4 or 8 bytes, else null. */ +function integerText(bytes, signed) { + if (![1, 2, 4, 8].includes(bytes.length)) return null; + let value = 0n; + for (const byte of bytes) value = (value << 8n) | BigInt(byte); + if (signed) value = BigInt.asIntN(bytes.length * 8, value); + return value.toString(); +} + +/** + * ilst items: a legacy fourcc or a `keys` index, then the leading run of `data` + * children. As ffprobe reports them, repeated values of one name join with ";", + * types 0, 1 and 4 are text, 21 and 22 are integers, other types are not tags. + */ +async function readIlst(walk, ilst, state) { + let offset = ilst.bodyStart; + let item; + while ((item = await readBox(walk, offset, ilst.end))) { + offset = item.end; + let name = LEGACY_ATOMS.get(item.type); + if (name === undefined && state.hdlrMdta && state.keys) { + const index = readUint32(item.raw, 0); + if (index === 0 || index >= state.keys.length) throw invalid("ilst index out of range"); + name = state.keys[index]; + } + if (!name || !walk.known(name)) continue; + let inner = item.bodyStart; + while (item.end - inner >= 16) { + const data = await walk.header(inner, 16); + const dataSize = readUint32(data, 0); + if (latin1(data, 4, 8) !== "data" || dataSize < 16 || dataSize > item.end - inner) break; + const type = readUint32(data, 8); + const start = inner + 16; + const end = inner + dataSize; + inner = end; + if (type === 0 || type === 1 || type === 4) await walk.value(name, start, end, { join: ";" }); + else if ((type === 21 || type === 22) && end - start <= 8) { + const text = integerText(await walk.read(start, end), type === 21); + if (text !== null) await walk.value(name, start, end, { join: ";", text }); + } + } + } +} + +// ------------------------------------------------------------------ Matroska + +/** One EBML element header at `offset` inside a parent ending at `parentEnd`. */ +async function readElement(walk, offset, parentEnd) { + if (parentEnd - offset < 2) return null; + const bytes = await walk.header(offset, Math.min(12, parentEnd - offset)); + const idLength = Math.clz32(bytes[0]) - 23; + if (bytes[0] === 0 || idLength > 4 || bytes.length < idLength + 1) throw invalid("Invalid EBML element id"); + const id = readUintBytes(bytes, 0, idLength); + const sizeByte = bytes[idLength]; + const sizeLength = Math.clz32(sizeByte) - 23; + if (sizeByte === 0 || bytes.length < idLength + sizeLength) throw invalid("Invalid EBML element size"); + let size = BigInt(sizeByte & (0xff >> sizeLength)); + for (let index = idLength + 1; index < idLength + sizeLength; index += 1) size = (size << 8n) | BigInt(bytes[index]); + const unknown = size === (1n << BigInt(7 * sizeLength)) - 1n; + if (!unknown && size > BigInt(Number.MAX_SAFE_INTEGER)) throw invalid("EBML element size exceeds inspection limits"); + const dataStart = offset + idLength + sizeLength; + const end = unknown ? null : dataStart + Number(size); + if (end !== null && end > parentEnd) throw invalid("EBML element extends past its parent"); + return { id, dataStart, end, unknown }; +} + +/** Children of a sized master element, in order. */ +async function* children(walk, element) { + let offset = element.dataStart; + let child; + while ((child = await readElement(walk, offset, element.end))) { + if (child.unknown) throw invalid("Unknown-size element inside a master element"); + yield child; + offset = child.end; + } +} + +async function readMatroska(walk, fileSize) { + const header = await readElement(walk, 0, fileSize); + if (!header || header.id !== EBML.header || header.unknown) throw invalid("Not an EBML file"); + let docType = null; + for await (const child of children(walk, header)) { + if (child.id === EBML.docType && child.end - child.dataStart <= MAX_NAME_BYTES) { + docType = latin1(untilNul(await walk.read(child.dataStart, child.end)), 0, Math.min(child.end - child.dataStart, MAX_NAME_BYTES)); + } + } + if (docType !== "matroska" && docType !== "webm") throw invalid("Unsupported EBML DocType"); + const segment = await readElement(walk, header.end, fileSize); + if (!segment || segment.id !== EBML.segment) throw invalid("No Segment element"); + const segmentEnd = segment.unknown ? fileSize : segment.end; + // As ffmpeg's demuxer: level-1 elements are read up to the first Cluster (of known + // or unknown size), then SeekHead entries locate Tags stored behind the clusters. + const seekTargets = []; + let offset = segment.dataStart; + let element; + while ((element = await readElement(walk, offset, segmentEnd))) { + if (element.id === EBML.cluster) break; + if (element.unknown) throw invalid("Unknown-size element that is not a Segment or Cluster"); + if (element.id === EBML.seekHead) await readSeekHead(walk, element, segment.dataStart, seekTargets); + else if (element.id === EBML.tags) await readTags(walk, element); + offset = element.end; + } + for (const target of seekTargets) { + if (target < offset || target >= segmentEnd) continue; + const pointed = await readElement(walk, target, segmentEnd); + if (!pointed || pointed.id !== EBML.tags || pointed.unknown) continue; + await readTags(walk, pointed); + } + return docType; +} + +async function readSeekHead(walk, seekHead, segmentStart, targets) { + for await (const seek of children(walk, seekHead)) { + if (seek.id !== EBML.seek) continue; + let isTags = false; + let position = null; + for await (const field of children(walk, seek)) { + const length = field.end - field.dataStart; + if (field.id === EBML.seekId && length <= 4) { + isTags = startsWith(await walk.read(field.dataStart, field.end), TAGS_ID_BYTES) && length === 4; + } else if (field.id === EBML.seekPosition && length <= 8) { + position = readUintBytes(await walk.read(field.dataStart, field.end), 0, length); + } + } + if (isTags && position !== null) targets.push(segmentStart + position); + } +} + +async function readTags(walk, tags) { + for await (const tag of children(walk, tags)) { + if (tag.id !== EBML.tag) continue; + let targeted = false; + const simpleTags = []; + for await (const child of children(walk, tag)) { + if (child.id === EBML.simpleTag) simpleTags.push(child); + else if (child.id === EBML.targets) targeted = await readTargets(walk, child); + } + if (targeted) continue; + for (const simpleTag of simpleTags) await readSimpleTag(walk, simpleTag); + } +} + +/** Whether the Targets name a track, chapter or attachment (ffmpeg files those tags elsewhere). */ +async function readTargets(walk, targets) { + let targeted = false; + for await (const field of children(walk, targets)) { + const length = field.end - field.dataStart; + if ((field.id === EBML.trackUid || field.id === EBML.chapterUid || field.id === EBML.attachmentUid) && length <= 8) { + if ((await walk.read(field.dataStart, field.end)).some((byte) => byte !== 0)) targeted = true; + } + } + return targeted; +} + +/** + * One SimpleTag as ffmpeg's matroska_convert_tag files it: under its plain name when + * it has no language other than und or TagDefault is set (the "-lang" name is never + * ours), a string-less tag deletes the name, nested SimpleTags become PARENT/CHILD + * (never ours). + */ +async function readSimpleTag(walk, simpleTag) { + let name = null; + let language = null; + let isDefault = 0; + let string = null; + for await (const field of children(walk, simpleTag)) { + const length = field.end - field.dataStart; + if (field.id === EBML.tagName) { + name = length <= MAX_NAME_BYTES ? utf8Strict(untilNul(await walk.read(field.dataStart, field.end))) : ""; + } else if (field.id === EBML.tagLanguage && length <= MAX_NAME_BYTES) { + language = latin1(untilNul(await walk.read(field.dataStart, field.end)), 0, Math.min(length, MAX_NAME_BYTES)); + } else if (field.id === EBML.tagDefault && length <= 8) { + isDefault = readUintBytes(await walk.read(field.dataStart, field.end), 0, length) ?? 0; + } else if (field.id === EBML.tagString) { + string = field; + } + } + if (name === null || !walk.known(name)) return; + const lang = language && language !== "und" ? language : null; + if (isDefault === 0 && lang) return; + if (string === null) walk.remove(name); + else await walk.value(name, string.dataStart, string.end); +} + +/** + * Reads the prompt carrier tags of a video File without touching its media data. + * + * @param {Blob} file - the chosen File (only `size` and `slice` are used) + * @param {Object} [options] + * @param {AbortSignal} [options.signal] - rejects the read with an AbortError + * @param {Uint8Array} [options.prefix] - the file's first bytes, already read for classification + * @param {Function} [options.yieldControl] - awaited about every 8 ms of work + * @param {string} [options.kind] - the classifier's kind, when the prefix was not kept + * @returns {Promise<{status:'complete', container:'iso-bmff'|'matroska', format:string|null, tags:Object, limited:string[]} + * | {status:'unreadable', reason:'invalid-container'|'read-failure'} + * | {status:'limited', reason:'metadata-limit'|'chunk-limit'} + * | {status:'unsupported', reason:'video-container-unsupported'}>} + */ +export async function readVideoTags(file, options = {}) { + const { signal, yieldControl = defaultYield } = options; + try { + throwIfAborted(signal); + if (!file || !Number.isSafeInteger(file.size) || typeof file.slice !== "function") throw readFailure("Not a readable file"); + let prefix = toBytes(options.prefix); + if (!prefix && options.kind === undefined) { + prefix = await createSliceReader(file, null, signal).read(0, Math.min(PREFIX_BYTES, file.size)); + } + const container = options.kind ?? detectKind(prefix); + if (container === "avi" || container === "asf") return { status: "unsupported", reason: "video-container-unsupported" }; + if (container !== "iso-bmff" && container !== "matroska") throw invalid("Not a supported video container"); + const walk = createWalk(createSliceReader(file, prefix, signal), signal, createPacer(yieldControl, signal)); + const format = container === "iso-bmff" ? await readIsoBmff(walk, file.size) : await readMatroska(walk, file.size); + return { status: "complete", container, format, tags: walk.tags, limited: walk.limited }; + } catch (error) { + if (error && error.name === "AbortError") throw error; + if (error instanceof VideoReadError) return { status: error.status, reason: error.reason }; + return { status: "unreadable", reason: "read-failure" }; + } +} + +// ------------------------------------------------- Python json port (loads/dumps) + +export class PythonJsonError extends Error { + constructor(message, recursion = false) { + super(message); + this.name = "PythonJsonError"; + this.recursion = recursion; + } +} + +/** A JSON number as Python keeps it: the lexeme, and whether it is a float. */ +class PyNumber { + constructor(text, isFloat) { + this.text = text; + this.isFloat = isFloat; + } +} + +const NUMBER = /-?(?:0|[1-9]\d*)(\.\d+)?([eE][-+]?\d+)?/y; +const STRING_STOP = /["\\\u0000-\u001f]/g; +const WHITESPACE = /[ \t\n\r]*/y; +const ESCAPES = { '"': '"', "\\": "\\", "/": "/", b: "\b", f: "\f", n: "\n", r: "\r", t: "\t" }; +const LITERALS = [["true", true], ["false", false], ["null", null], ["NaN", new PyNumber("NaN", true)], ["Infinity", new PyNumber("Infinity", true)], ["-Infinity", new PyNumber("-Infinity", true)]]; + +/** + * Python's ``json.loads``: objects become Maps (insertion order, last duplicate wins in + * the first position), numbers keep their lexeme, NaN/Infinity are accepted, raw + * control characters and trailing data are rejected. + */ +export function pythonJsonLoads(text) { + if (typeof text !== "string") throw new PythonJsonError("Not a string"); + let pos = 0; + const fail = (message) => new PythonJsonError(`${message} at ${pos}`); + const skip = () => { + WHITESPACE.lastIndex = pos; + pos = WHITESPACE.exec(text)[0].length + pos; + }; + function string() { + pos += 1; + let out = ""; + for (;;) { + STRING_STOP.lastIndex = pos; + const stop = STRING_STOP.exec(text); + if (!stop) throw fail("Unterminated string"); + out += text.slice(pos, stop.index); + pos = stop.index + 1; + const char = stop[0]; + if (char === '"') return out; + if (char !== "\\") throw fail("Invalid control character"); + const escape = text[pos]; + if (escape === "u") { + const hex = text.slice(pos + 1, pos + 5); + if (!/^[0-9a-fA-F]{4}$/.test(hex)) throw fail("Invalid \\uXXXX escape"); + pos += 5; + let code = parseInt(hex, 16); + if (code >= 0xd800 && code <= 0xdbff && text[pos] === "\\" && text[pos + 1] === "u") { + const low = text.slice(pos + 2, pos + 6); + if (/^[0-9a-fA-F]{4}$/.test(low)) { + const lowCode = parseInt(low, 16); + if (lowCode >= 0xdc00 && lowCode <= 0xdfff) { + out += String.fromCharCode(code, lowCode); + pos += 6; + continue; + } + } + } + out += String.fromCharCode(code); + } else if (escape in ESCAPES) { + out += ESCAPES[escape]; + pos += 1; + } else throw fail("Invalid \\escape"); + } + } + function value(depth) { + if (depth > JSON_MAX_DEPTH) throw new PythonJsonError("Maximum recursion depth exceeded", true); + const char = text[pos]; + if (char === '"') return string(); + if (char === "{") { + pos += 1; + const map = new Map(); + skip(); + if (text[pos] === "}") { + pos += 1; + return map; + } + for (;;) { + if (text[pos] !== '"') throw fail("Expecting property name enclosed in double quotes"); + const key = string(); + skip(); + if (text[pos] !== ":") throw fail("Expecting ':' delimiter"); + pos += 1; + skip(); + map.set(key, value(depth + 1)); + skip(); + if (text[pos] === "}") { + pos += 1; + return map; + } + if (text[pos] !== ",") throw fail("Expecting ',' delimiter"); + pos += 1; + skip(); + } + } + if (char === "[") { + pos += 1; + const list = []; + skip(); + if (text[pos] === "]") { + pos += 1; + return list; + } + for (;;) { + list.push(value(depth + 1)); + skip(); + if (text[pos] === "]") { + pos += 1; + return list; + } + if (text[pos] !== ",") throw fail("Expecting ',' delimiter"); + pos += 1; + skip(); + } + } + for (const [literal, result] of LITERALS) { + if (text.startsWith(literal, pos)) { + pos += literal.length; + return result; + } + } + NUMBER.lastIndex = pos; + const number = NUMBER.exec(text); + if (!number || number.index !== pos || !number[0]) throw fail("Expecting value"); + pos += number[0].length; + return new PyNumber(number[0], Boolean(number[1] || number[2])); + } + skip(); + const result = value(0); + skip(); + if (pos !== text.length) throw fail("Extra data"); + return result; +} + +/** Python's ``repr(float)``: shortest round-trip digits, exponent when decpt <= -4 or > 16. */ +function pythonFloatRepr(number) { + if (Number.isNaN(number)) return "NaN"; + if (number === Infinity) return "Infinity"; + if (number === -Infinity) return "-Infinity"; + if (number === 0) return Object.is(number, -0) ? "-0.0" : "0.0"; + const sign = number < 0 ? "-" : ""; + const [mantissa, exponent] = Math.abs(number).toExponential().split("e"); + const digits = mantissa.replace(".", ""); + const decpt = Number(exponent) + 1; + if (decpt <= -4 || decpt > 16) { + const shown = digits.length > 1 ? `${digits[0]}.${digits.slice(1)}` : digits; + const power = decpt - 1; + return `${sign}${shown}e${power < 0 ? "-" : "+"}${String(Math.abs(power)).padStart(2, "0")}`; + } + if (decpt <= 0) return `${sign}0.${"0".repeat(-decpt)}${digits}`; + if (decpt >= digits.length) return `${sign}${digits}${"0".repeat(decpt - digits.length)}.0`; + return `${sign}${digits.slice(0, decpt)}.${digits.slice(decpt)}`; +} + +/** Python's ``json.dumps`` default ``ensure_ascii`` string escaping. */ +function pythonEscape(text) { + let out = '"'; + for (let index = 0; index < text.length; index += 1) { + const char = text[index]; + const code = text.charCodeAt(index); + if (char === '"') out += '\\"'; + else if (char === "\\") out += "\\\\"; + else if (char === "\n") out += "\\n"; + else if (char === "\r") out += "\\r"; + else if (char === "\t") out += "\\t"; + else if (char === "\b") out += "\\b"; + else if (char === "\f") out += "\\f"; + else if (code < 0x20 || code > 0x7e) out += `\\u${code.toString(16).padStart(4, "0")}`; + else out += char; + } + return `${out}"`; +} + +/** Python's ``json.dumps(value)`` with default separators and ``ensure_ascii``, on a loads() value. */ +export function pythonJsonDumps(value) { + if (value === null) return "null"; + if (value === true) return "true"; + if (value === false) return "false"; + if (typeof value === "string") return pythonEscape(value); + if (value instanceof PyNumber) { + if (!value.isFloat) return value.text === "-0" ? "0" : value.text; + return pythonFloatRepr(Number(value.text)); + } + if (Array.isArray(value)) return `[${value.map(pythonJsonDumps).join(", ")}]`; + if (value instanceof Map) return `{${[...value].map(([key, item]) => `${pythonEscape(key)}: ${pythonJsonDumps(item)}`).join(", ")}}`; + throw new PythonJsonError("Not a JSON value"); +} + +// ------------------------------------------------------------ carrier selection + +/** utils/video_metadata._record: (text or null, record format or null). */ +function record(value, key) { + if (key === "workflow" && !(typeof value === "string" || value instanceof Map)) return [null, "invalid"]; + if (typeof value === "string") return [value, "original-text"]; + if (value instanceof Map) return [pythonJsonDumps(value), "reserialized-json"]; + return [null, null]; +} + +/** + * The one container carrier of prompt metadata, with the server's record budgets + * applied (utils/video_metadata.select_video_carrier, then the metadata loop of + * utils/prompt_inspector.read_inspection_file). + * + * @param {Object} tags - container tags as ffprobe would report them + * @param {Object} [options] + * @param {string[]} [options.limited] - records the reader left out (`['workflow']`) + * @returns {{status:'complete', records:Object, record_formats:Object, carrierText:string|null, limited:string[]} + * | {status:'unreadable', reason:'duplicate-metadata'|'invalid-metadata'} + * | {status:'limited', reason:'metadata-limit'}} + */ +export function selectVideoCarrier(tags, options = {}) { + let limitedRecords = new Set(options.limited || []); + const folded = new Map(); + const entries = tags && typeof tags === "object" && !Array.isArray(tags) ? Object.entries(tags) : []; + for (const [key, value] of entries) { + if (ENVELOPE_NAMES.has(key)) { + folded.set(key, value); + continue; + } + const name = key.toLowerCase(); + if (!PER_KEY_NAMES.has(name)) continue; + if (folded.has(name) && folded.get(name) !== value) return { status: "unreadable", reason: "duplicate-metadata" }; + folded.set(name, value); + } + const candidates = []; + const perKey = new Map(METADATA_KEYS.filter((key) => folded.has(key)).map((key) => [key, folded.get(key)])); + if (perKey.size) candidates.push([perKey, null]); + for (const tag of ENVELOPE_KEYS) { + const raw = folded.get(tag); + if (typeof raw !== "string") continue; + let envelope; + try { + envelope = pythonJsonLoads(raw); + } catch (error) { + if (error instanceof PythonJsonError && error.recursion) return { status: "unreadable", reason: "invalid-metadata" }; + continue; + } + if (envelope instanceof Map) candidates.push([envelope, raw]); + } + const selected = + candidates.find(([fields]) => ROLE_KEYS.some((key) => fields.has(key))) ?? candidates.find(([fields]) => fields.has("workflow")) ?? null; + const formats = {}; + if (selected === null) { + for (const key of limitedRecords) formats[key] = "limited"; + return { status: "complete", records: {}, record_formats: formats, carrierText: null, limited: [...limitedRecords] }; + } + const [fields, carrierText] = selected; + // An envelope carrier is whole; a left-out per-key workflow belonged to the carrier that lost. + if (carrierText !== null) limitedRecords = new Set(); + const metadata = {}; + for (const key of METADATA_KEYS) { + if (!fields.has(key)) continue; + const [text, format] = record(fields.get(key), key); + if (format === "invalid") { + formats[key] = format; + continue; + } + metadata[key] = text; + if (format !== null) formats[key] = format; + } + const records = {}; + let roleSize = 0; + for (const key of METADATA_KEYS) { + if (!(key in metadata)) continue; + const value = metadata[key]; + if (typeof value !== "string") return { status: "unreadable", reason: "invalid-metadata" }; + const size = utf8Length(value); + if (key === "workflow") { + if (size > WORKFLOW_METADATA_BYTES) { + limitedRecords.add(key); + continue; + } + } else { + roleSize += size; + if (roleSize > ROLE_METADATA_BYTES) return { status: "limited", reason: "metadata-limit" }; + } + records[key] = value; + if (!(key in formats)) formats[key] = "original-text"; + } + for (const key of limitedRecords) formats[key] = "limited"; + return { status: "complete", records, record_formats: formats, carrierText, limited: [...limitedRecords] }; +}