From c2876f5a6b31573fdb773da52c1e5cba72dc4204 Mon Sep 17 00:00:00 2001 From: nofunstudio Date: Sun, 2 Feb 2025 17:37:00 -0700 Subject: [PATCH] working with animation sequences --- __pycache__/iframe_view.cpython-312.pyc | Bin 8689 -> 11910 bytes iframe_view.py | 226 +++++++++++++----- js/iframe_view.js | 305 ++++++++++++++++-------- 3 files changed, 364 insertions(+), 167 deletions(-) diff --git a/__pycache__/iframe_view.cpython-312.pyc b/__pycache__/iframe_view.cpython-312.pyc index b1c277373ee7489751a219da4d375955bbba963c..0b6c23e68589574d97159ee73227280133eb807f 100644 GIT binary patch literal 11910 zcmd5iZEPD?a=Rp#-=F#>MawJ6woFT=9e>2J{E^5KB}D5Jdmlg_|B61VdHsZj#;(+RqE9wIdI2~}nDVHmvYf>B;(hzU0Yuz^JHUH$;Mem#z z?YvzsNjs7q6b0Iq_&)Z%nKy6Vn>X`j_7jsykHPaF*!<|tUrw1Toz+Ba~2r2^}~ zDBT+vrT;)7QSPsz^mJ7i_uDBwWq{m48U59i$?v4h{!Ns{Uqe~_n<<;0q)NtHBUa328aK={=0Dhf)Q-8PZCcd=GoyAVc8T zC`MKN1@7N^1$Q~VKuEkUg+O#hgQK*d6|<8QtP2-3$F6WR+j$a-%FFcm{!nga2n18X zPYJ(=?T}o@c#MNwdM;s%y-?#ZJG>r8eV=-GSVfszKI{ti0~`@z>i3-KA2{U+`1((t z@(9XNnuE8x$yoT}BpnD-g7N}=g+-*0&fw2O|NFp5i7MXbVOr zX~qmgRl$$l3dszX)mY!{Noi_VEzTQ5DNDnOr7>k`Txh*hmbSc@)V`QCSRO)0ERkjs3LN070Av}$<_c3lI&RTlcoB}ywpOe>4V({X5HiVvUXR{>|N zlhTV=JsV)vuZz`CG-D;_kVla5GKTd=jp3A0?tx=5V-b}mSnZtL;joR0u&=|V0T^DpF-Mef1uE`qbl zZ$-Gxh5mAhvdKNBi6XcXo}fz6Ldw(vJVBMAH$Znme#(pZu97hoV7OX&obfujRLJ=} z#I(8yXGIZCwTzRh{2(_f$R0W}#Ake)Tq@+8OD^+yXc^0Rquf@=sVcd3T2n-Yo!5*v z6?(~~qPgvo-;M82WRLF}?J44aTHuHw{cp=k`WdTS5#yP)M6GaPS`F&A=qZq$zV zR8%9tKV$ySg>QO4FS)QIzL7=2;|bu?w7F=0J9#tZ25NS_pVxuFZIchH$G8_|%C}%l z*>tg1s^g7M>iX}1Jw9#xmhm!v8JkiLVVCiz;C<7UqEUDAmM7u4R(Y)PHn~*DWxoAy z?tESmQVb!gE#I>Ea(RJz*9(bH^&~vKOgsdpB^i!|;?Zzyl*}22Bo*X>HPM)X9AW4PcO(-7Yg5elK6eQ0FS^C9&J83{o(G(d*7C6ar z3~Jpz8KY=M#9b%#t0Teib%`ScBXQ_*In0egLUChFF&6gOOrvoRMbF zjs<}*GU3_)x?i&7rG-HEhGs!6aWipb1CaJOaGq?CdNm-P=Z2EjB|^tWxv^$Jg=UsG z^lT)k&xR-x`dB=+Q(ErCz6}r!#H6hRycAhE5@(`8PFj`b#48&>oqutk%yv?ge$9!t4G{TY(i0O1OvE>EsyhcO?R<$zxh?B{j5j6fH$Ze; z9Fd%W&DP8yd-jX}f+w+OgYNC&STK^44_P8|oKNi90H%$W#7H8liIg{CLVX_HW8H%C zRL8Nzwhf?rJG*lN7zJT$PHcZ}KY8!%-=JsFBE;exh`YQL{YaPuLX=!l6a=*-zKMey zpgI-2OiqNS=tv;QFu^ONIJC3`4QOE+nwb*dX=2Al@U#daA2D1UM8>^AUk}5;4q-$H zwD97H(7$_w{%tTm+ElPDNi43aoY?MASoKU*=jyzU0 zk&p4S8}2S}8%#v#r%uXnjC`zg?;ClnBUzYsjAGr17dIe6;_C*-Y%}Sq7gSt0N(;mU zoc|nDV3!C47iU6a0+FDZI4fvIBJm)%caK1blMwV_HXLKQU@Sz#c?9Z3P({Klr0Aq$ zkVPaYL23jg>^wma9T|>AN(100B4LgxLp^k8pNgWxpc7e8#V6<(gPaY5Mp_6!h-5-g z#3lqn+ztXU2{gVWXrygI3p&qKh@RkJ#RMg+4J5%SK^X$Seg=WnnnH!WCIjG4022g zW$P(Z<^(j=z>g)tJ2rzoGGZka*X`Hr?^Mrpt*UfMy*Ja_pBxxU_xdxc;rklH48N*1 zCCw)@+TK-L#r5iI)i;KJO#H<3BU8q9bXJWTt=-F~fBovOUS0Mk&BGb(t659=tm%;& z)0WI_xuwczHi;Mx{%qfp>SNu9x{nO$L){r|Pu5U6$E6I_Stoh>_|4<M4ztn zUuX@(x)o;>|At|vA;srt5^YOVMtfpSwVSB;+)zG8-D&;!MUmlr43vlx|Ix zZk->^l)6{T>`BLKnX+KkS%16dX3s)BOlQs`zJ7VBeCgtcRhhD`Y)$?Av76z!4iQV& zr>)7}!B4sY*=o&}l^S=d#=USkU2|a0AhvgWTDm-(?i@-EhcZ@b)meMH>t@&YFE3n8 zHylbk53e}eQqH!efwc3)oK6Hdx}^Kq{GmBxJ-J%txUIdZowui}8s`YqSoO+M`N!2C zRxg>772dnnlb@q`^nA)C`-eZ7&REZ8tDGW4%v6#rc`aiMu9iEl4__Oeul}*; zCp|yv$&??PZAW{etT)*=xYBnv)ps`8doEcL$QWPCmfB}~(8^l7miPbq(60_H?@SJ! zO`Z#_oEuG@8%>`JC(n*2$0w2(FRffmq%J1X7y0DGbkh8-jP~`cwfwsAn(@}D+y0yW zv~~N!{*-n1Eb+OmV#QXQvenLSP20BTHu#~%_MdnEw0rUBvOVb=TJfDt`Oc<&0co;n z-$-(FJbCIu@r*8=W;LQuzvrj7OSfL5>wb5XmdHc zmeI5wXEJ3&*)scAYXmfW^{5UrmVb$lN%6~?4nyBw?2{dheWlo^3Z4LG@H1V}1t;uxA)Ip+l+m9nt}FkTAb@(HTbY29w`d0VTjz_Y(aCh8 z@P-dNfW77XE9GU}rT0qa0dPUE1Y@8NkdkUgu76gvLlZT_mxEz002WLj6r2$K{&3Sn zx*j^7CBZn@Gm@~eWKJ(MlhOqtQ3>`J!Xg^f`RD}M@EC6ZbNZP$`$5mgCy}WX42?Z$ zC(+Rf?h1*txZ6dDp$BFM(9V!(0-7LuB^C$VC^of3w_qsh&!UydsX0l*RX<|}%pmlRz>l4Wqpet_4?ee|y^2F{{pQZA@-G@NVeFws$Wq_RShZnQ<>%x)WN6B%4nx zpIAPUJQGh&TuH_g$;8#9na^mae`B!CT>!ApDh`O}(5kuo?dY4)8^_b;tt;BCN$u9G z);za8rFG1z*DM(HK9JTs9$7I$pVaqeRDD^4X{K{cgXy+CW>N6wJ1vfuL->bIyyc+k z!%b#XZbE4Y+VW4%&yGFpg;&=x8bfhLzk*g$_#2q)gMb*L0^*BGznW6{H4qon>cIae z{ZTrI3gUj9T-QNNP>c`iuc%#mLEkB|{4|7|){hQ>q=}~$&xiQR=e%a3yXbAOsb(Px%tF{ zN~7Jp=EQ1TS$kdX-%+2f-~Yg*sWd;>RAJSxVHG+(g92|ZEyQ(uS~@(;1k|3LxU>v9 zDw#2qgi$gMNztJhkeFCFMd7?e>@0dAG1i)2a)`Ym77J$CT$bEqa4X@)yrJR%A_`4P z`U=i{NnaH-6e8s5M;6B?+>$t76hv#bT#wkpniixrq9FFa>Bw2flej(n4j42V11sg}70zCEbC6xY$ z26=aZm4r=K@TJd%D*YN%C13VjsIsp?Rq^G|g)09VlpUxvd^7ptjbg~aQM4b8yn`}< z9olCs6}5C zvVX>ycq~527JICNT*6pn%`_)k@=h2T#{jrWMiI=VF!HtMkEr&X5r-W4(z!3XI^Bt$ zuc2s>%}3etL+-9TMsS*mF#^#`bcmc3#{-2IL2eGWpv}b%*!50X(Go@4h6+cE97raP~jc>;Vnxv!*pQW~@#cw*7GL!t3eAmzNHG*60mi}4(*L7!rL>GCoQt{2~{N%Pr^_S}7A)s3bFMSbZa~5Kq2*a*a?T zBTcE(uXW(LU#fxifA~`WL~7Unfkm77JI5RyhwxuG@s2I3Uu@E%^49u}2GvJe zHIzRx*LO6jK6*hcAE@tmLG?=qD*uwK?|4!5H_BsmQ2Dr_4ALhIE@U2psf9f}2gws( zq`}61`ho8<@^eY+(~#^tcRyHV}xR4+;Z<2T!vZ6nd4bY+05~^GR#t%HJ6}&C{#Hg+S_pecun!t`$u*sA>A&-;SvUk+ax+K z6Ds$vaW+9z4)&9UvGEIPylhN&}JI+91FB{p7ON9 zx1&QYFn`5o-z&Mo{g++a1O-G=lo0L{!yOE|c?z1bAUhTbpO+#cif~+vf8_L}7`R~2 z*FvIN6;$JFJjS4VDmY}qaddlT&^MHVaU>pruxel;$c?emZYSD2#|AnCbN}&E9?$7c z&md6O+Rvc-faKf}HM6Lc#RdE`)bI{U-bEOR4gm8C#_rCJKGSinTC2sxFcml^O9=@tD;O2jKbSZHC0}Q2WeicU8=60+p6_tu4R=TD~ z)tykSfvN+csGKp~*OmZIi+#qJ)tAj^vKrfrDyy~45cf^Cw};*wN>;wGQu$J<@}+cT zbJ}!hM*ka(m{rUkn03wVnWN_hZ*fx^sB(;!@d?dTDHVa(OhVKAph^|4HBb JZH*{C{|)cnu#f-% literal 8689 zcmc&ZYfM{Ln&NYca&0h)w7X!;=75QyUt9!W7x9LM*9?fOCPy@cS{ z9cH#a7->gC)LoINsltxt2UWWgl`1tWt+aI1PN}Bq*s?3UH;lHsJ3qGnXcK9xbfw+z zT>IK44b!f)(jM^nzVrCb`ObG9-{bzlXw)G{R?WSstpdi(ynz_)yNW!=o?1rB z)&>br9iwG*0BwxkQ_mPY4UEy#$W(ZCF(ywFWA-#NmQatp+!ADMq25wS#@;O;?}9XN z^0X=tyPK(GcfEy}Dz*V~E8F-MVp{>LW}6_dVVmDVZ|h|W0!<;N_BVuQ&m3W|`x+18 zwkt$xAmW>1MU4c`P4jj_RG*p?SbpFfpyVujb<|&2874tW_>u5?*az7XN+1DXK3zwg ztyD@N8&rpI-zQQ6c2Abp2%1Ch5SS6;sPmQ4v01y*g%JWh8JQHZ#2fRe1{uyZ{JMR zE4KzGIUw?053;lE({o}S%;TL3$FBOqUV*(K%y7`=3(v4z1x!^3KYlM{^C+h_|IOL7 zx@FVUczZl;+P7gkkTxAy>0PyEOh;3iqdC3lA#_Ck!VX~^Y2hbHYrh&xODML5q~`6J zyvIpGC`yeXO8b{U{9sv&qSPW#@<<_-jFlim3V7F~6w=683BIS4cO$7N8=p!jLel*# z^~4yQsj`_9iV;|38I`JxEBOxNQ-Y-_C3&tzf^#wI1iFp@>le!UGHQHs1apC)cviU8 zvUxP6c@ij3kt7pjs7j_PLAaX*xpU2BBk1H>0s&ohslSXd#k<28Fh@yJsFmA7_+&hh z5~Mt0QYpZDmQSHZ87o1dS;iAenMQ`87P+ki8Kc~qq=emaeQ1x2m0*G@_EeRvwIZSV z@x6?GQAS~s#{ub}(7w_PGRBx0i@cXf*vF^#sVwWG`4P?wyC}5EGlcfbSP7P$Rjn2! zw4no~HW^E35^ARE-NGG8mdnBVljKl-%W6t;`2W3SWqVV*(<+!cXz7sUP=fNy0$RZa zJ)SGI$XKbz+hsQx`W}{hlwjEoJ>@l#<;B#;^>AlYWl}p*8bihs#o3q!>5l2k*4dcQ zKmF7)yP)MStTS{>9=`nEvuT{lb2q8sEmnkAl~A4wX^ z==UUy+v$Z9@?4>A87o0qZm`b(GcA))_!Z-rYvb$2kpe7nZ{O>VADfuwVt&x=MWxL6 z1m7ikMDTF}%g}|^V04NuXl)vIXpeX9Kqv8~pABATnf7;-Ut{g?^~;cPSbKbp zAC7kjkz$0oX*PaxhXIFKuoa@10fwFQ&4dLQKg!2AdNwEoXg=VZX1nQl<*;v7;;cAf zx2VLN{)KV#jL}zpJbUyA%|`uT!Z6#oZ;w-WjPAH&2j)fuE;v0cF$ewVv0)b-V0{eB z(UV*(A`i_&ujo|!W1{Wr|Ar)z!~Lh?`*#>}Xuw4Wv8`BK+ld`|4i`jAbFlX4OoZiv z{_X6G)NGIu0+P5zRe%jn1q99tVvj$&{#3~aN((&sqEqY(6LGTr@Z@7f8~vEL6z|x9 z8m!*wF#E)f2}bG1Wc!iH$6A_)wMH4fJ$`hD0rJA!;ktFx_9js&1S70SO~Y9dL`sNp z{(wluSuVzl>dA1-CmcQ`QWE>3F31O?yx@!aS-1s3zaWxu21OlIaRQGo4)mN32L(}; zKmDTi>d_+%@CZj2UrSLLn`WaNeuzXhaDJWT1W^&47Aa|yIc%?r)C_QPTvQkKLex5M z_}OV87>kM|Yyf1w8=}g`MS0m9etQLn zZNCDmUTMiHm;mm*sKz|P84~r<{qSDJ?i0`k!(OREq$I#$laRyKps1e?-eALCiDyw? z;M2=;oT$n_BBGiN`=)u85eb&ZrZc@QaoAcGtxtO*xqfJBf*((Tz~<4WrFzMB%l2l& z{NScio6_}Wm5zVan>IDZRK?3#P0yyKW~t#;!|lr-bba{3`!8fIJqs$_Xm+h%_~Wae zy}IsBRZL_xujWkE3&t&_ovM*YTeF(ooBHZS=C0#QearIXs`59{O_S~Rp4->&w7o}W zO>LXz>Qv2W);yN0Z^D&}>ReU*M%CVQ)!yZ)Y*qWF)t0KilC}DBO?&S}?nG94GfkbV zmFcDr@5n9wq@!LEK+#hx|i3gKW+G^Va=GTaerw(_h-z@`K)<3SJxh9Dn?_W8Uv2`q}Hk;^sfjfa^VWscy690JUpN{{-@obZ0QNN|2EcLm@eH)D( z>Bf##)qU#zwM=8rq7$#8v2ShW)0-dNT#KaYN58a=J<=g#WvX&KYw&DV*Dp=nnpkf7 zVCKW*`^jvzbD?k3U`bhrQs*yjoWGPleLtT1!`~k_N-UC@YO=SBm5e z7cPG?c|RLvy`U)sR@9EU2hNWV7g;FWAE4dL|fqCM_aCQMQu|if5`IwhThJ}FJUmnxJ(W1YC-@{5!7H1Ig(_>WS z9tn}q$&mnJdo-vlQdH)FkO`vHzpP%FP%sokQ3}y2@Hl|0d}34(DJfjFZS`u1cogds zs(C}nOG!|SMs7>066z<~iV<6gUTB_ZE3PN0Nob#F(-+5t2!w#Irj(u*St~_%lKQd; zSJCGMzpNDZDteU3in7{boGWQ6t1U*flIEWlQ%vY2@6%EyQDefwRDkzs+U9*46BP+l z!aS*zywGAa@o9cIc&7E+{8>Z7u-&r^qNHB(^9-*W{`%+O?*T0NdEliMqk=G2@3yhb zPmU#6ijuCBu@Yn~rFdZodlFBi_bQ9KR+%XLO)3jsC~zftq1(9odvcbrSGnUyiiUCi z78QI@$+6()qW%CEi^gX7a*xy}#P{lcK-TLEhry4N+{0jWdImg$xQeFXnd^>gXs8E| zs5`FCdqdsv3Yso^^19;&=qks;yPtO5;|G2)=msI2=&5>p2|F_1H&99GDZ!7i{~ zY@c&|xWXVQ2nZ`+XB|pdPZUd%x{yo8yw)fbIVYy3*vPan$KyD`Hj6x8$ZL;k@Ah}s z>%)?b(`9<(sXd}FDE~81L;r!k1H--Awlscg{LP64on&!4) z=C8^6qZQQ)uW#C(TeaVRC1dMbRBb7hMtXBs3$`;?^qE~p7Ik3nHg>L3nTDeauFZyy zA7ap2lW9B-)@@DwQt(#r&DWL((>3i2{g_Yd;QF!D$OSO|E?!Ah`m%1!xp2F{{B%NGTXjI zWL`+tww3cMy{XF1tl>GZGMCQWI3T(tLTKd$)Z+p5N$IKd$XllAjRAoiCA}oFFjXO##&>FHwO1UP%JJO$}oufK}Z?4`kau6BSe&R2LJ3 z+KYj>VkJrt2}OcTC=*lyR2_uCweRAOPmtt5C0Khz{AHbdY)8u8u364EJ&l7y5I1@1 z(QGf0*-23?o#$XoB)OR=7lJA5I?fN^e|0^1Y=_9q(0Rj)y99M{TiHd~ev_7G@!MtI3CT(~oXRcZtyWO{Z29CFLt!sTRxL6NKg|Xrxsl-NuP zA=&k!#_NsZPk~-h=k-Qn%uE>L2Cw(pj4xbh(fh+bo{z9XAjWVLFtcbFcEV>$ciuQG zmfSsOog<>AZ_M30I&jV{YI=J{o&D#A$0hkl>*B_NTr%UNXM!8RMVw*NjmJjx8z@Ja zzfn|Dx`!lES++2?wI5lJ&%1I}I5RcRpN2a-fBK;N`&=X_27gnv*Vg zrdj4i*z+p)Ifv8R>*^U9@t*A&8Ulx5M68!89RnjnuAXr(7D1`ac+q*v>lhez_QL1f zar<6T0r4Obf{0Qahr=eGs1Eq}Ksb1n!@+J*5sXQJ9qH{CDJdkwbwW#Chg`uJj!(n+ z3&yaE!eQ$~G)%_A5FPSP`-A|`g>e<-aGo0L7b`|j4?CR~2Amgxk={`b-v>zx<@Jmt z{~Ne&4rg(RvL7hiqQN!Le|pqA?sU1%T@=;W;&zYrh{{v0F{eoNx|}`3pvhtoOHxVS zig{5deQkE*&(V^$6V+ZX=zFjCiCurJIhj9cFLJ+v39-+?e+AhVNf5*Xqhmby8$hWoZ2$4 z%xNt1)K^B!ug8Bmo~nIrqxN{Z_IReYJ7au)UiZ(4T2L$;Td*%4T4Wb5-X2&EFJE3A hTs?XJ+M0DuwH8>PS)WR&E@aWgf71>9TrHjZ{{cA^MzsI{ diff --git a/iframe_view.py b/iframe_view.py index 0d0a857..e85085a 100644 --- a/iframe_view.py +++ b/iframe_view.py @@ -28,33 +28,56 @@ def get_global_texture_value(unique_id, key): def process_texture_data(data): + """ + Process incoming texture data from the iframe. + We expect data to be a dict containing a base64-encoded image string (under the "base64" key) + along with width, height, and other metadata. + """ start_time = time.time() - print("[process_texture_data] Started processing texture data.") + print("[process_texture_data] Started processing texture data. Data type:", type(data)) - if not data: - print("[process_texture_data] No texture data received.") - result = torch.zeros((1, 64, 64, 3), dtype=torch.float32) + if data is None: + print("[process_texture_data] No texture data received (data is None).") + result = torch.zeros((64, 64, 3), dtype=torch.float32) print(f"[process_texture_data] Returning default tensor with shape: {result.shape}") return result + # Log the structure of the data if it's a dict. + if isinstance(data, dict): + print("[process_texture_data] Texture data keys:", list(data.keys())) + if "base64" in data: + base64_string = data.get("base64") + print("[process_texture_data] Found 'base64' key with string length:", len(base64_string)) + elif "data" in data: + base64_string = data.get("data") + print("[process_texture_data] Found 'data' key with string length:", len(base64_string) if isinstance(base64_string, str) else "non-string") + else: + print("[process_texture_data] Unexpected data format. Data:", data) + try: - print("Raw data received:") - if isinstance(data, dict) and "data" in data: - if isinstance(data["data"], str): - print("[process_texture_data] Received base64 encoded texture data.") - b64data = data["data"] + # Process if the data dict has either "base64" or "data" + if isinstance(data, dict) and ("data" in data or "base64" in data): + b64data = data.get("base64", data.get("data")) + if isinstance(b64data, str): + print("[process_texture_data] b64data is string. Checking for header...") if b64data.startswith("data:"): print("[process_texture_data] Stripping data URL header from base64 string.") - b64data = b64data.split(",")[-1] + parts = b64data.split(",") + print("[process_texture_data] Header part:", parts[0]) + b64data = parts[-1] + print("[process_texture_data] Final base64 string length:", len(b64data)) image_bytes = base64.b64decode(b64data) + print("[process_texture_data] Decoded base64, received", len(image_bytes), "bytes.") pil_image = Image.open(BytesIO(image_bytes)).convert("RGB") + print("[process_texture_data] PIL image mode:", pil_image.mode) image = np.array(pil_image) print(f"[process_texture_data] PIL image shape: {image.shape}") else: - print("[process_texture_data] Received raw numeric texture data.") + print("[process_texture_data] b64data is not a string. Type:", type(b64data)) width = data.get("width", 64) height = data.get("height", 64) - pixel_array = np.array(data["data"], dtype=np.uint8) + pixel_array = np.array(data.get("data"), dtype=np.uint8) + print("[process_texture_data] Raw pixel_array shape from 'data':", pixel_array.shape) try: image = pixel_array.reshape(height, width, 3) print(f"[process_texture_data] Reshaped image to: {image.shape}") @@ -62,24 +85,39 @@ def process_texture_data(data): print(f"[process_texture_data] Error reshaping array: {reshape_err}") raise else: - print("[process_texture_data] Data format not recognized, returning default.") - result = torch.zeros((1, 64, 64, 3), dtype=torch.float32) + print("[process_texture_data] Data format not recognized, returning default tensor.") + result = torch.zeros((64, 64, 3), dtype=torch.float32) print(f"[process_texture_data] Returning default tensor with shape: {result.shape}") return result - print(f"[process_texture_data] Post reshape image stats: shape={image.shape}, min={image.min()}, max={image.max()}") + print(f"[process_texture_data] Post-conversion image stats: shape={image.shape}, min={image.min()}, max={image.max()}") image = image.astype(np.float32) / 255.0 - tensor = torch.from_numpy(image)[None,] - print(f"[process_texture_data] Tensor shape: {tensor.shape}, range=[{tensor.min().item():.3f}, {tensor.max().item():.3f}]") + + tensor = torch.from_numpy(image) + print(f"[process_texture_data] Final tensor shape: {tensor.shape}, range=[{tensor.min().item():.3f}, {tensor.max().item():.3f}]") elapsed = time.time() - start_time print(f"[process_texture_data] Completed processing in {elapsed:.4f} seconds.") return tensor except Exception as e: print(f"[process_texture_data] Error processing texture data: {e}") traceback.print_exc() - result = torch.zeros((1, 64, 64, 3), dtype=torch.float32) + result = torch.zeros((64, 64, 3), dtype=torch.float32) return result + +async def wait_for_capture_completion(unique_id, target_frame_count, timeout=20.0, interval=0.5): + waited = 0.0 + while waited < timeout: + frames = get_global_texture_value(unique_id, "animationFrames") + frame_count = len(frames) if frames is not None else 0 + print(f"[wait_for_capture_completion] {waited:.2f}s elapsed: Found {frame_count} frames for unique_id '{unique_id}'") + if frames and frame_count >= target_frame_count: + return frames + await asyncio.sleep(interval) + waited += interval + print("[wait_for_capture_completion] Timeout reached, returning empty list.") + return [] + class IframeView: @classmethod def INPUT_TYPES(cls): @@ -90,71 +128,133 @@ class IframeView: "height": ("INT", {"default": 512, "min": 64, "max": 2048}), "scene_state": ( "STRING", - {"multiline": True, "default": "{\"camera\": {\"position\": [0,0,5]}, \"animation\": false}"} + { + "multiline": True, + "default": "{\"camera\": {\"position\": [0,0,5]}, \"animation\": false}", + }, ), + "frame_count": ("INT", {"default": 4, "min": 1, "max": 60}), }, "hidden": { "unique_id": "UNIQUE_ID", "color": "TEXTURE", "canny": "TEXTURE", "depth": "TEXTURE", - "normal": "TEXTURE" - } + "normal": "TEXTURE", + "animationFrames": "TEXTURE_SEQUENCE", + }, } - RETURN_TYPES = ("IMAGE", "IMAGE", "IMAGE", "IMAGE") - RETURN_NAMES = ("color", "canny", "depth", "normal") + RETURN_TYPES = ( + "IMAGE", "IMAGE", "IMAGE", "IMAGE", # base outputs + "IMAGE", "IMAGE", "IMAGE", "IMAGE", # animation outputs (one per channel) + ) + RETURN_NAMES = ( + "color", "canny", "depth", "normal", + "animation_color", "animation_canny", "animation_depth", "animation_normal", + ) FUNCTION = "process_iframe" CATEGORY = "lth" - async def _process_iframe_async(self, url, width, height, scene_state, unique_id, color=None, canny=None, depth=None, normal=None): + async def _process_iframe_async( + self, + url, + width, + height, + scene_state, + frame_count, + unique_id, + color=None, + canny=None, + depth=None, + normal=None, + animationFrames=None, + ): start_time = time.time() print("[process_iframe] Starting asynchronous processing at", start_time) - try: - print("\n[process_iframe] Called with inputs:") - print(" url:", url) - print(" width:", width, "height:", height) - print(" scene_state:", scene_state) - print(" unique_id:", unique_id) + print("frame_count:", frame_count) + + unique_id = str(unique_id) - # Since the iframe is always sending fully processed texture data, - # there is no need to trigger an intermediate save check. - # Directly retrieve the texture data. - if color is None: - color = get_global_texture_value(unique_id, "color") - if canny is None: - canny = get_global_texture_value(unique_id, "canny") - if depth is None: - depth = get_global_texture_value(unique_id, "depth") - if normal is None: - normal = get_global_texture_value(unique_id, "normal") + # Process base textures first. + color_tensor = process_texture_data(color) + canny_tensor = process_texture_data(canny) + depth_tensor = process_texture_data(depth) + normal_tensor = process_texture_data(normal) - try: - state_data = json.loads(scene_state) - except json.JSONDecodeError as e: - print(f"[process_iframe] Invalid scene state JSON: {e}") - state_data = {} + # Use provided hidden animation frames if available; otherwise, use fallback. + if animationFrames is not None and len(animationFrames) >= frame_count: + print("[process_iframe] Using provided animationFrames from hidden data.") + else: + print("[process_iframe] Animation frames not provided or incomplete; using fallback.") + animationFrames = await wait_for_capture_completion(unique_id, frame_count) - color_tensor = process_texture_data(color) - canny_tensor = process_texture_data(canny) - depth_tensor = process_texture_data(depth) - normal_tensor = process_texture_data(normal) - - elapsed = time.time() - start_time - print(f"[process_iframe] Asynchronous processing completed in {elapsed:.4f} seconds.") - return (color_tensor, canny_tensor, depth_tensor, normal_tensor) - except Exception as e: - print(f"[process_iframe] Error: {e}") - empty = torch.zeros((1, 64, 64, 3), dtype=torch.float32) - return (empty, empty, empty, empty) + # Process each frame's texture into a tensor. + color_frames = [ + process_texture_data(frame["textures"]["color"]) + for frame in animationFrames + if "textures" in frame and "color" in frame["textures"] + ] + canny_frames = [ + process_texture_data(frame["textures"]["canny"]) + for frame in animationFrames + if "textures" in frame and "canny" in frame["textures"] + ] + depth_frames = [ + process_texture_data(frame["textures"]["depth"]) + for frame in animationFrames + if "textures" in frame and "depth" in frame["textures"] + ] + normal_frames = [ + process_texture_data(frame["textures"]["normal"]) + for frame in animationFrames + if "textures" in frame and "normal" in frame["textures"] + ] - def process_iframe(self, url, width, height, scene_state, unique_id, color=None, canny=None, depth=None, normal=None): - """ - Synchronous wrapper for _process_iframe_async. - """ + # Convert the processed frames into batched tensors. + animation_color = torch.stack(color_frames, dim=0) if color_frames else None + animation_canny = torch.stack(canny_frames, dim=0) if canny_frames else None + animation_depth = torch.stack(depth_frames, dim=0) if depth_frames else None + animation_normal = torch.stack(normal_frames, dim=0) if normal_frames else None + + # Clear the local reference to the heavy animationFrames data. + # This lets garbage-collection release the base64 strings from memory. + animationFrames = None + + elapsed = time.time() - start_time + print(f"[process_iframe] Asynchronous processing completed in {elapsed:.4f} seconds.") + return ( + color_tensor, canny_tensor, depth_tensor, normal_tensor, + animation_color, animation_canny, animation_depth, animation_normal, + ) + + def process_iframe( + self, + url, + width, + height, + scene_state, + frame_count, + unique_id, + color=None, + canny=None, + depth=None, + normal=None, + animationFrames=None, + ): return asyncio.run( self._process_iframe_async( - url, width, height, scene_state, unique_id, color, canny, depth, normal + url, + width, + height, + scene_state, + frame_count, + unique_id, + color, + canny, + depth, + normal, + animationFrames, ) ) diff --git a/js/iframe_view.js b/js/iframe_view.js index 4df4f28..7f6023e 100644 --- a/js/iframe_view.js +++ b/js/iframe_view.js @@ -8,7 +8,7 @@ window.TEXTURE_STORE = window.TEXTURE_STORE || {}; // Function to create the iframe widget async function widgetIframe(node, nodeData, inputData, app) { console.log("[widgetIframe] Initializing iframe widget for node:", node); - + node.isCaptureComplete = false; // Create iframe wrapper with some styling const iframeWrapper = $el("div.iframeWrapper", { style: { @@ -44,6 +44,22 @@ async function widgetIframe(node, nodeData, inputData, app) { iframe.src = urlWidget.value; } + // Ensure we have a "frame_count" widget to specify animation frame count + let frameCountWidget = node.widgets.find((w) => w.name === "frame_count"); + if (!frameCountWidget) { + frameCountWidget = { + name: "frame_count", + value: "4", // default number of frames as a string + isHidden: false, // visible so user can change it + callback: function () { + // Reset frame buffer when the frame_count changes. + node.frameBuffer = []; + node.isCaptureComplete = false; + }, + }; + node.widgets.push(frameCountWidget); + } + // Message handler for communication from the iframe function messageHandler(event) { // Ensure the message comes from our iframe @@ -55,7 +71,9 @@ async function widgetIframe(node, nodeData, inputData, app) { case "IFRAME_READY": console.log("[messageHandler] IFRAME_READY received"); sendParamsToIframe(); - break; + if (iframe.contentWindow) + // iframe.contentWindow.postMessage({ type: "CAPTURE_TEXTURES" }, "*"); + break; case "TEXTURE_OUTPUT": console.log( "[messageHandler] TEXTURE_OUTPUT received with data:", @@ -96,6 +114,7 @@ async function widgetIframe(node, nodeData, inputData, app) { width: parseInt(widthWidget.value, 10) || 512, height: parseInt(heightWidget.value, 10) || 512, sceneState, + unique_id: node.unique_id, }, }; @@ -111,66 +130,73 @@ async function widgetIframe(node, nodeData, inputData, app) { } } - // Function to handle texture output coming from the iframe - async function handleTextureOutput(data) { - const startTime = performance.now(); - console.log( - "[handleTextureOutput] Processing texture output for node:", - node.unique_id, - "at", - startTime - ); + window.addEventListener("message", (event) => { + if (event.data && event.data.type === "TEXTURE_OUTPUT") { + console.log( + "[iframe_view] Received TEXTURE_OUTPUT message:", + event.data.data + ); + handleTextureOutput(event.data.data); + } + }); - // Make sure the node has a valid unique_id - if (!node.unique_id) { + // Updated handleTextureOutput: Capture frames until target is reached, then aggregate them. + function handleTextureOutput(data) { + if (node.isCaptureComplete) return; + // Validate that we received all expected textures. + if (!data || !data.color || !data.depth || !data.normal || !data.canny) { console.warn( - "[handleTextureOutput] Node unique_id is not set, cannot update texture store." + "[handleTextureOutput] Incomplete texture data received; skipping frame." ); return; } - // For each expected texture key, update the hidden widget and the global texture store. - const textureKeys = ["color", "canny", "depth", "normal"]; - textureKeys.forEach((key) => { - if (data[key]) { - node.setHiddenWidgetValue(key, { - data: data[key].data, - width: data[key].width, - height: data[key].height, - format: data[key].format, - type: data[key].type, - shape: data[key].shape, - }); - } + + // Ensure the frame buffer exists. + if (!node.frameBuffer) node.frameBuffer = []; + + // Append the complete frame data (with base64 textures) into the buffer. + node.frameBuffer.push({ + timestamp: Date.now(), + textures: { + color: data.color, + depth: data.depth, + normal: data.normal, + canny: data.canny, + }, }); console.log( - "[handleTextureOutput] Global texture store for unique_id", - node.unique_id, - window.TEXTURE_STORE[node.unique_id] + "[handleTextureOutput] Captured frame", + node.frameBuffer.length ); - // Optionally clear heavy data from the global store if it is no longer needed - // Delay the prompt so that the global store has updated - setTimeout(() => { - if (node.unique_id && window.TEXTURE_STORE[node.unique_id]) { - // You can conditionally delete or nullify large arrays after processing. - delete window.TEXTURE_STORE[node.unique_id]; - } + // Use the cached target frame count value. + let targetFrameCount = node.targetFrameCount || 4; + console.log("[handleTextureOutput] Target frame count:", targetFrameCount); - if (app) { - app.queuePrompt(); + // If we haven't reached the target, request the next capture. + if (node.frameBuffer.length < targetFrameCount) { + if (iframe && iframe.contentWindow) { + console.log("[handleTextureOutput] Requesting next frame capture..."); + if (node.frameBuffer.length === 0) { + // iframe.contentWindow.postMessage({ type: "CAPTURE_TEXTURES" }, "*"); + // console.log("[handleTextureOutput] starting capture..."); + } } else { - console.error( - "queuePrompt is not available in the current app context." + console.warn( + "[handleTextureOutput] No iframe contentWindow available." ); } - }, 50); - - const elapsedTime = performance.now() - startTime; - console.log( - "[handleTextureOutput] Processing completed in", - elapsedTime.toFixed(2), - "ms" - ); + } else { + // When all frames are captured, update the hidden widget and mark the capture complete. + node.setHiddenWidgetValue("animationFrames", node.frameBuffer); + iframe.contentWindow.postMessage({ type: "STOP_CAPTURE" }, "*"); + node.isCaptureComplete = true; + console.log( + "[handleTextureOutput] Completed capturing", + targetFrameCount, + "frames. Capture complete flag set." + ); + } } // Create the DOM widget for this node and register cleanup @@ -270,20 +296,43 @@ async function widgetIframe(node, nodeData, inputData, app) { // Updated onSerialize: Strip out the heavy "data" field to avoid localStorage quota problems. node.onSerialize = (nodeData) => { - const textureKeys = ["color", "canny", "depth", "normal"]; + // Always ensure a hidden data object exists nodeData.hidden = nodeData.hidden || {}; + + // Serialize the basic texture keys (keep any properties like base64 data) + const textureKeys = ["color", "canny", "depth", "normal"]; textureKeys.forEach((key) => { const widgetEntry = node.widgets.find((w) => w.name === key); if (widgetEntry && widgetEntry.value) { - // If the widget's value includes a "data" field, remove it from the serialization. - if ("data" in widgetEntry.value) { - const { data, ...metadata } = widgetEntry.value; - nodeData.hidden[key] = metadata; - } else { - nodeData.hidden[key] = widgetEntry.value; - } + nodeData.hidden[key] = widgetEntry.value; } }); + + // Serialize the animation frames in full. + const animWidget = node.widgets.find((w) => w.name === "animationFrames"); + nodeData.hidden["animationFrames"] = + animWidget && animWidget.value ? animWidget.value : []; + console.log("[onSerialize] Serialized animation frames:", nodeData.hidden); + }; + + // Update triggerCapture to reset the frame buffer state and cache the dynamic frame count. + node.triggerCapture = function () { + // Reset previous capture state. + node.frameBuffer = []; + node.isCaptureComplete = false; + // Clear out previous animation frames from the hidden widget/global store. + node.setHiddenWidgetValue("animationFrames", []); + // Cache the target frame count from the widget (defaulting to 4 if not provided). + node.targetFrameCount = + parseInt(node.widgets.find((w) => w.name === "frame_count")?.value, 10) || + 4; + console.log( + "[triggerCapture] Reset capture state. Target frame count:", + node.targetFrameCount + ); + if (iframe.contentWindow) + iframe.contentWindow.postMessage({ type: "CAPTURE_TEXTURES" }, "*"); + console.log(" starting capture..."); }; return widget; @@ -326,60 +375,108 @@ app.registerExtension({ const originalOnNodeCreated = nodeType.prototype.onNodeCreated; nodeType.prototype.onNodeCreated = async function () { - // If nodeData.hidden definitions exist, add any missing widgets. - if (nodeData && nodeData.hidden) { - Object.keys(nodeData.hidden).forEach((key) => { - if (!this.widgets.find((widget) => widget.name === key)) { - this.widgets.push({ - name: key, - value: nodeData.hidden[key] || null, - isHidden: true, - }); - console.log(`[onNodeCreated] Hidden widget '${key}' added.`); - } - }); - } - const r = originalOnNodeCreated - ? originalOnNodeCreated.apply(this, arguments) - : undefined; + // Create and initialize the Iframe widget. const widget = await widgetIframe(this, nodeData, nodeData, app); this.onResize(); - // Synchronize the node's unique_id with the one passed from Python. - if (nodeData && nodeData.hidden && nodeData.hidden.unique_id) { - this.unique_id = nodeData.hidden.unique_id; - console.log( - `[onNodeCreated] Using provided unique_id from nodeData: ${this.unique_id}` - ); - let uniqueWidget = this.widgets.find((w) => w.name === "unique_id"); - if (uniqueWidget) { - uniqueWidget.value = this.unique_id; - } else { - this.widgets.push({ - name: "unique_id", - value: this.unique_id, - isHidden: true, - }); - } - } else { - // Fallback: generate a unique id. - this.unique_id = Math.random().toString(36).substr(2, 9); - console.log( - `[onNodeCreated] Generated fallback unique_id: ${this.unique_id}` - ); - let uniqueWidget = this.widgets.find((w) => w.name === "unique_id"); - if (uniqueWidget) uniqueWidget.value = this.unique_id; - else - this.widgets.push({ - name: "unique_id", - value: this.unique_id, - isHidden: true, - }); + // Use node.id as the unique id to ensure consistency between JS and Python. + this.unique_id = this.id; + console.log( + "[widgetIframe] Using node id as unique id:", + this.unique_id + ); + + // Update or create the hidden unique id widget to be consistent in the global texture store. + let uniqueWidget = this.widgets.find((w) => w.name === "unique_id"); + if (uniqueWidget) uniqueWidget.value = this.unique_id; + else { + this.widgets.push({ + name: "unique_id", + value: this.unique_id, + isHidden: true, + }); } - return r; + + return originalOnNodeCreated + ? originalOnNodeCreated.apply(this, arguments) + : undefined; + }; + + // Here is the key: hook into the node processing + nodeType.prototype.onProcess = function () { + console.log("[onProcess] Node is now processing. Triggering capture."); + // Call the capture method only when the node is actually queued. + if (this.triggerCapture) this.triggerCapture(); }; } }, }); +function generateFallbackUniqueId() { + // Only used if no unique id is provided + return Math.random().toString(36).substr(2, 9); +} + +export function onNodeCreated(node) { + // Try to read the unique id from node.settings or node.hiddenInputs + let providedUniqueId = undefined; + if (node.settings && node.settings.unique_id) { + providedUniqueId = node.settings.unique_id; + } else if (node.hiddenInputs && node.hiddenInputs.unique_id) { + providedUniqueId = node.hiddenInputs.unique_id; + } + + // Force it to a string so that Python (which does str(unique_id)) can match correctly. + if (providedUniqueId) { + node.unique_id = String(providedUniqueId); + } else { + node.unique_id = generateFallbackUniqueId(); + } + + console.log("[widgetIframe] onNodeCreated: Using unique id:", node.unique_id); + + // Update or create a hidden widget for unique id to ensure consistency. + let uniqueIdWidget = node.widgets.find((w) => w.name === "unique_id"); + if (uniqueIdWidget) { + uniqueIdWidget.value = node.unique_id; + console.log( + "[widgetIframe] Updated hidden widget 'unique_id' to:", + node.unique_id + ); + } else { + node.widgets.push({ + name: "unique_id", + value: node.unique_id, + isHidden: true, + }); + console.log( + "[widgetIframe] Created hidden widget 'unique_id' with:", + node.unique_id + ); + } +} + export { widgetIframe }; + +// At the end of your file, or in a separate initialization block, +// override the global queuePrompt to trigger capture on IframeView nodes. + +const originalQueuePrompt = app.queuePrompt; +app.queuePrompt = function (...args) { + console.log("[QueuePrompt] Triggering capture for IframeView nodes."); + // Call the original queuePrompt functionality. + const result = originalQueuePrompt.apply(this, args); + // Find all nodes in the graph and trigger capture on those with comfyClass "IframeView". + if (app.graph && app.graph._nodes_by_id) { + Object.values(app.graph._nodes_by_id).forEach((node) => { + if ( + node.comfyClass === "IframeView" && + typeof node.triggerCapture === "function" + ) { + console.log(`[QueuePrompt] Triggering capture on node ${node.id}`); + node.triggerCapture(); + } + }); + } + return result; +};