From 2bf060a47fdb09004518da4459561a5c8c0aa4f4 Mon Sep 17 00:00:00 2001 From: wailovet Date: Mon, 24 Jun 2024 00:57:36 +0800 Subject: [PATCH 01/64] 'update' --- examples/captioner.png | Bin 0 -> 342124 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 examples/captioner.png diff --git a/examples/captioner.png b/examples/captioner.png new file mode 100644 index 0000000000000000000000000000000000000000..53d5c3b567f75695b397f2343e1bfd0c0bee3e1d GIT binary patch literal 342124 zcmd432|Sc<_djkcDG`05vTrGCb|NYvgoGqpGGiUa&e*q7WZw!=B!!T&k5NgsBnnwV z_FZG&;&-YM7@9*^FHUi&$;H#B~^v}dynoVARySU zc;WnI0)oA|1Oz12dv=36*IWeU2?!3#JIGyia7Nl$njnzQ1a7PdBYOu!Ya|SAXbxjN zB_zs&zk@Khw#3~LKFPxh{j)vmDP1>K+%v*FtPUs}IO{3aOLlN4bGWmjm60i&m50>^ z2|Xn)D#9Z!D5B59YH#ieo^oS7$$Cmq0E{C5{#RJUgNM}wVPpy(bps!fb}+ad>nSj% z6?hBlDe#!NwT+_#{(aU)R?vIQp%>VLH>|%`QRS?H9QYHOC7850H0c{~1bDJP9@Fm^ z%IX2$ZexeEf!jHlLsNHSRRyCnL&GtfAnlknj$QtJ`W_xU`1J`&Y`#b3nxQ-bX$QA9 zh6~CX**KUZt-)kB7AhbjbdpC*;J1|uiHqP@DzWoQ#r}1r#v5-jG)7vRn44}}GqeYr zlaSr`SNs-0bKcm39fq(+ARP?NtzmE%u))yYARQfk-(wj3hU2!^X=q+nR8?5tV@Mc6 z0^oc0W=4?Ug|^8F602#?tBLKDCXO+Z8#y#G)2 z1X{VNo;&OvWKX!4{2?_k8BI+?4GqK1dHfL#^2V*lZ__mjxgKnu9Sp$=T>=|!F z){Y3oUyNe|aCX2SNQA;Cca+~c7y+~S#r^=!tmB%>Sq zEPcibDgOE>imIAhgW&)84`U2OzZ!_n`7crOHzDnyHg2pib33@P1JVu!P<2yia~newb35R6)?ppOe0w+o3}6U%g4>}CtsTr*PXWOin<0_*5MU#K zk6?B{GCSHpLv1(&Fus#H434zHdm#wr&CFpixHao36C;E@9H^6#qdgovzxk(~krmil zH&zqy0si02%fuW3U*G2K>sr?XoWg-=0GDEAW*soKHYPLPA_XR9r$>1R4tRtAc_+Fn^i{u(eI#`qMmi z0GO?5{Tprp7XXZV!{huh%r@n;H*$g-8o`{5fDZ*qu;I3Tf6ZB#)8^v0ta0lVI}Wq$ zHCt1_E0rP8NFavw*Zktlz}$he!21|T7uLnQ;|O3+|6Js@H~a-I9PEtDt^dAQP3WJ0 zSuO-3LP9%&$Zuo)3MJqh|COIzcK}-|1{U_GmjxRJq8%ecdzgTs2>9cyHOj%v+}iYt zh$0jXT3hnj8=1fz;MVp?JAl>zluttc0SXA`71;*xAUs&#ctt2^(J(ebY`I@yVG$lt z5gcG{g@-~L{#WQEnAx9d43zC()EFm_EopCnHC~UP0Cq!$e*t8$i#s^=jbS#HvMuJs ziA#(}NF03g&yeWvZ~4!%+O}akipUynWCyo*;Dx&w8RJ7B9x+i6{~E)s9e^v_bh3ir z-j9mj(T2a|DX9llXSjj}5gD6cObS-~n3+Gq+z?l$}0psS5~qe^-}3 z^St$l07^MU@Zsh*^pwCGt-xQ<6R6X_D9l#uwmFU8{UsRw4}ZD!HLx2S^4@k6NM;96 z-61*Mz}LU$w~(|2C3Y0`ALISk29J~L@B6%?V1c#rt-ptrXFZg(Mmq55vGTy}?2u5L zfkOgjAW|^N_1Ie%O6km&1EJx5d& zex6zB0QH`evIpdR2*TsZnLj-3K0Zwpvg*kia-4Dr?e&!Mw$SmAyWZtLsw$idG0X+$ ziOxrNr9U}G+&Xw-1FOTDgKHeyM zls%8B4Dh%Pp2m`56|A24PC6DG|M^|!2h#%O3h65G8da=!hF7`Q<9Q}xX5F}(Reee_ zMeY&y4Wvs1JVZ2vLEdt&Z!DM5`^i)$_10qt7wlIgR_VPGUlFyiGmlpp<`IVwO!_*v z8zm0pCE_M^R_em$HB?Y|XkoAWE)0>OPxfo=?hN-IhhWM<PKjF(0CTz;lcmp*Mu zZk{n#rEKBcNACpnd`GtIL3rtnQdS+6+&Gv>wM9e|dxqkb$joE78;(gNQ-QTp#r#-i z?DZpqr5UA0ka*8x{N2eGsi!#ll&fOH_*o1uW5ioU41RnIS@2s(F~cgH&(GY?S-Uu% zZ`GbmIyp3SJdJ?J&0>Zs=G-UJr9H|-M!~z6&oCJ&GG{hiXBuMTA@dhGI!5f_&*76J z@8|I_G>82_vsn%$_yWn>qxOGdiU;o1(GbxX9Q{`@*%&7Cl z)?IfVr}TJ88SXvmenM9gx65PRdw^;J4V6`9c5r{~TCsjv3Uj;~Bmt)DttLwO6~CfiJ`6>kQSY&C8DwT;XXNQcS4zK) zn+LPcp-Aoery?l;t^oa!6^`WWQpG%Qmt1t38t-ebE{Nr?ABpPW@*k-EgN- zv+c(g=CgT4OMC5VZOp9^C~8#WO9~Deb@9NY5u~S-B?p$`3)-Dj0EQC^8y=#fa0rG*m0qoDYC;#6ZsdaubSpiOmn+ zEAvnl`!JKr+u(0I6cPfJJpzd`teQFU>@jZ9`ybLgt2raoc&f}+w-j& zjGY}EqOe#$HxqydO8?gZM4zfWPiDqP`aDtAbSh#TXH_}GtT-Uaq2%@Xge4q3w0>!g zvuh8Hx)}79dv$MkGB!R!Z7=_|*IRe|l$sQ|Z7Qf<31u^NL(T+pL|l{@Uxays$j1Yw zw{aj++e?1J&*1rJGD+>aZ-dnECF=D6VLU0lz4@xeax|6 zQMfw27hovQ82uVmGx{9B*L{+eUj8E4VC$;he#A0CkA@rIoPs>{o^R$i#)g9#fGl!9 zjiZnE8UuG=L0T3Ui^qDxU*Y5C%27{7zHFB_^B#9(Ztr*@I>Q#&GF=D(==O+7Isxl{ zi32MZ-R3H>1%p?CjZD=eIW+cQx&%oG#vJ^Le);f;(-ucJ!bc>M-!QRBq>N>p88|ycP~9G1-JLH;=PX!~Cy4NZ58sB`wec_yDLV(%gl9Z96o-3r)NQA&HbO^NJL2x5p(RG6BEB z$C;RC!GW~}su93_ZnJB=KRt(e>~4F8O^O$SU7YK_@Y)gkH~6UDy#XKI2@Qrs8%}wL zQMbA61)jv7nqFHgg;g$()xEi?@MvSZ)FeG=T1p_73!4$Z&hWvW;V}@e7x^g*1X}+M ze6V_HS?$VN%yTj$ZEeh{nWdo5a@|dS2Qxf2mRUqGH#76phnTckm;2E82UfdiWoKux zrIWWmX?bGWAB8$h9|3^@-U2a$gg@^O9$IDNag${l@|&$^e2)uAd_2s^{6j{sW^Y^; zX#=u`mD5L=Kp^R}ZhN+5g-dL>&~oF>cs*r#Qri3=@iCRSFS>3L$GZxM1IE*LleekQ zTI(BP0tWAd6HuQaxGQ2OG3$_P|kNfz~kTs9*yb;dOTg51=z ztzo>+X}TrN9LFg^h)$S503!B&LQZKzqBq-?%tzQ0nzKR#eBpxm^3h;io-xnQCnMtiy~zcjAA|>5VV_Jsj)1b_b}WXGHg9SpC#Q3(vI2Sm&5gA zhC2cBMQ$VYA40z;C-cO+a9N1Yr?b)^0pt*F(ndiyK| z6oBbo_pbZsM7DdwUaw|D(T{fmp#-trZhSbj#C|DC>d~nE0ZMAh7~4)`;I`ZVG+hm7 zjI;nbWh)gBZl)y^H%5X$GEoh~kegaPCobPn%v!WN1=CK5&kmYL;{(>%;rr=vTgI8%6breZSt`h3V;M{j9X7bN@chWDte#vgTwY|0N(tjSxdI z>N#3!(vK%I<@N!`>vUCY>RJmfPPlF#4CRSm^^7+@?31Yv``^NEoPxbl_LObT zxQ)<}WKhQ-C2y%-E-fuhH~lQw)X<Xyyx7hZ(t^Dw=3(X`m;Bg)Y^-S)&US3@OmE2rR0U1{8vIQoQ-@9h&eOl znY3Ymv*}Ka0yB7;BaaUi_Uvi9-$J>gsp#}V2^Y>(Yyuco zcG+%VVWu{_f?~+hbjCX-qeU&CJLic&&NV;?6lZ(t6D9MUi>5o; zc@r1)dQWy&I(w05kZ6pbRP(5JYgxs~_&2ZYAA{(I$5;dx_AGyDr}8?m!{uL=P2;Nf z{giYMq={0B88-vz1st)Quu3I!6=O+9*wDSvjvIjXpmi6c5k1(IUh3Wn0 zk2g^EY`p_;0vBG3>^WA*@yE5Fw-8!(qe$W-{bH9}E+-$XxYt8o_;&rB-!3Sv9FO## zbm5*^y-j+Dv{rmlf1DbZVNY@B3@8KoW}u~;EWM;ecNTH&3B)9sSnAQ*x25WSu)Q_h zH|Vws+!`SOa~PQ_bM2WJ?0CUYJ=$a|1KYp^2qcboJ#ja{Qte23kYWMtu@s`Q5t!!Q zm4RZxyX@WVsKp6N5LqJvP$O=gC3AS{>AHY&&K?lpW@N}D>!0%Smr>)C>6(V<0=%Xj zArK^h5mXU`5i%3|mm#|)DXKDLcGyvz9OK1i7}5@SJr2da(;uNjT(*phc7u>h6|_^- zeW}Trq>p4sp*C2<09?CmFg>|s(k?)yq zu4;9ZPNAIngW1^@MCL2M7D70Rm(8sUHq0!PwES~BD z*{izS!G`%_-Kf>gU<6MKm2KiyiqtA`ZmMDnE;iYF%)NjZ7ncAgQ4#g}PJQgbZQo>; zsCjW!k^2WjcFc2`bwFEZA%Q=t42iq)Ho5!8pM)YUS{ch5FP2HqHWB&d((UVeA>UJ` z^zgv6gx_BOU-*RW1j8ln)^4HtGk;0K&#p9Z`s9x5QsJF!Nt*-?sW(!H2sMmF zlBz7L%&W(*ZPKDasgtW$-F4gnal4c_pY; z{AFE)5-Dm#k1COvN(FohIccGQ^F=teGSQt3QhzHdawvMl@x70wy8UZBnu?wF56B_| zwnb`N+zbwQZ1?_Gz;j!s=AhRZeC%_pba}Uq+%P2wLr8GUbw6Pc#KqC$Gox6grs4s1i$WgP*GlSRvl|()l+T|V^>JsLXb9cJ4DFp8~e@}Z)7$%Gz zLj#~O(;Q+W?|&G2a^|6e z&wtAd|Bv{x$x#o_o`ow%l9Q@k`u6P`K){TQj3-eZLdC=NJl6LnOp5wM3_$>VfUHks z?OYi@&W4ym+U zP~-0@tJr4HA5{rdN_hBaZfX;e-W&f(r_D;m2}_uIX97! zXoz|~kC_ZwPp%gze_?>TVMOs_?5lgY4ZyPf7V1L2t!IC29A$4d`SyzHXLWtXSbW7N z<-rP2vCr-|5$1U}x>P^r$y5m%NoWHXLLeCy+qP>g2Qb_Ct96h7A5ba?RU;t?-Av=q zA0j{jNJ{1xZXVQkAXo~Fy{WR14e1J__q8?(r?Kc>qp~sxk_h4tLN?GCa-FzRKa6gs z?V5**BA_fHq4X>Aq(&sy(NTN(AL4%qoaKs1<-}(G*9DmX{z{IxA+&gPQ*PZdakJ}e z+mHRn+~-MACwOxFfXYf{(qURy*5k+=Qq#?`&|72W))TI$R3@Tq&`LIk{b6ST>Jan# zW~K%*>r2E}wECf1L>wZtCZ_q&3I3D&)?o(}n^><4Fy&m}=(K3B-A4(c-Ocg^{`>I{ zGdCB{tv`cDiyX~KdWQ9+@95GSP8GeVM1E-|7^eX*<1Gxu7y`}qlxfSBX z-f?t~gl-E}UVB}7cFKHLmbkCez$~#lzFM05PCJZOWqKb#p^+)OW{`M6uy8L`sPA6w z$YOMh>*O+Kb4=!qx?+fy@V_XQexU4E^rK1fn}1LYz3WT`Vb}F`P%m|U?e=TE)S7Jb zY6V#~?@D3wp{{^Vs1Ui+MQmr7MDjOp^jK<<8X{cVSq~Fzi{Hqi+qMu8mt!zRStc@mamcgne{p zNeOZ&+jFBAprq;6e!Ap>(R}M~aSi&zG!PTnZAe5+N};(_rk*1I?BmCeH7a^7rkhlE z`xHq**esax2>-TFX1Z;4h2Lv22*m17>7s!Dtg|e6VO*JBxOT`Z*S7cl@wM|5=~<`D z)>%#p>adQ73*y!LDCk5l?mr^ktaU%f7lGt1L~e@&ug6zD0rEe7Z}@c!)lA{2j|?FyA-W7y!O%Z zWrp##Wa-m!6cyq4)p0&T!g~=H8BV)8%pA{J&1Lu*ps1ZyPvhi#{Sn5;lk^m)y}y!hsgAF0CP8$#mU zn)0J`w0;o0ef#kv3n@t|LyGAZl7jf%Gu0dq=PObVhj$xvc^*K=GftiZ9LIs&RL9{d zt9MyH^HN?6s^sgvxUox%L}|hw<<5TTJQ?{>;S0fZVjqua=BKQABZgI z!kJ*U+oYerQ-hM%%BKJwrtZNjnav%0yJ1t#%n@r%GrGfO^;-;Gw6p-Vc}P zOCEJggw-BY6+f$}rHmW+3>C*o_XQh{wW6?uwg#_z4g>AkfWKV2K-sB3P@3Th>M94w z3O~l?=0S5DD-U7Cx#~Uj(o-EgooLj{Yh|7uRI+M`WcZg8#ds%p3ptH_HS@>xYJogO zEC|&PQKE+GW5g@;KZxo-;d%0hNhC*GX`L%b<*fL)`9C)hBJFncaI3YFJYdV8ti0x%*+5fZvN5v z$HP0RJWl)WZv6Z$lwUg@Y}%A1lF?NA>LC=a#KoLOzV3xpY_O3d(LG?pIhS zCV?#`;UH9e62c^dnYagO17l5rVO@sIj8S4P=)))iQ4I_A_0qkTjJQu#mL3#8F(52q zp{~jhR5LYAYs-Zze=EGZ8*lpFB)MVUnv3_vZIhi7_JDAb%Ihm{{_#0N&yP2>o~YsV zx=M*C3v4p#L-K-s%V1qH1Bmhv0YDFkfgaGa8Sq*gMU-e_h4bRYkcn|Zrm9y)#ecFw z^_BzV1X%*F7;R>VA>%t&}yCPbeq&tU_`;L zNc(!?^zI{dm`cHYA=C7OYEmyIknfNyqh+g>uha*X3KKwG<(p^wwtWw0M88B`;w?uH zyqv`KPQ^?7iE0M%6(H?!G!{I{j4MN(DdGro^t2g#;C$us_XyC66M^nY$^BMv9Lu~= z)PcE^rT~?!U@ShFw%N6qLT)}#Gtp-6y$U3L1q|7|y$eTSISl8o9#o5J%pRECP)hQVOQCY2@Y$Hb(rrv62^>XW$ zBo7gk>0IEz+n$P`rf<$&&lz31eQrrT(EPm8)`EFr;5y9VxVku~{Zn-DCvDk+ zoMWRR=d*U}5|y7L+GQ3)2EoFs^=m6?B90@Py=wknXOU}j$Y6(9ly3NEiNS;v&*}KC znaH53(N~{UHRwbUmuL;$U$_q3H_bGzzLxLGV@s!2V^{WbPNQq_qw|-00>)gudbzMW zOKDz9X+y}LKO0k(gQqzbZLp@-FzzTh`FiPPghLWcrfX$Dtf`~Oo^8rNuQFE8?oON_ zTuJ{(oIkg{BHvH)`)ap{I?njjB(1Fsj^wo%jLx5zy+yA6`H4z~dp^9DCeOB4ezq@C zExk8H^5d|T1MQ{Okk#_XuG(HlfOj+!9YfFDg3^k2gkM1uz&}ve69Lpugd@Nn>WcNFv+{_G^7vwkEahoU#m*kg@`R;>i`KZxVoq9;XWNPSepmZytGYN5 zZVSGql`C0#t(u9ov!1VHiOeD813pn-7FB`7OhR36CgEj7GV@8=W8aHWCAnkYo7;3_ zMONR_ArUS}nYz`?cqg-)BvY|xnhRUth8>3je6kyRxq&-4a=Ep!n7Zg_+N9IC+1U7* ze5vKJmrWd)^#M-yFpKEk2F^s+qBC+9ZvG*9*mix+RZ zPok$)%j!QLIQDs^xxXmz>yvOXmzim0EoGJ|Cnu-o)2bc2AXk2}ZdVs5uC9JkaQ$I0 z7ps#K`ol1y#?k}pVc5*woWUb%?_U2Iy@dCgjHUf>rtut|ixlZ|K`&8#BsDR9HE{^@ znyz8@;dHGGG#Z^TZTZBs+L*#tjlG-ICv(~_jCcH66}Q0;k1q6&=*hw3N?vjeqzkBJ zoemZhjTfh@ha4JJUQe}Ne6 z<)cX6)VSBXX!S0AeyQisAouYQpX1bzV6o0ie!KTyHe9UH=JHU^#ePm&bvXafb+$>H z?d|o1SDgFR#J~FwsUY-+t%2wdb;r23%XN)D3SbPp@SwV&!;`Qajw<4?rpIspUq7oGP~Eryn6N1Ob~4kjC!qb5owFSUTUCp-JQEc@>H{R_LvF&*`WMIdn z|5Iz`VLyMDOkkm}(#$K>uYNmBny$I|u=OU3jp^+kn& z&{W&*DV9Ffz07g6c}PapoQy^JN~Zwq0TU?WBB#-l9VSxt zI&(I?oR{OOVps!&>ZLTE>z9bOwY0c>QlPgz?=bQZ)3c1O_ULo9RTEnJE_Q(#Nd^mc z1HEXL;%fA}TN`1vfUO?AWDuFh@WyQjrp$8UV}iuXC-pk7%+$}bI}`~IkIt8^d7O`l zpzc{(U?9A7E}1&_zVw8IV*WqLamW5SR@>CVs#|lIqv7eDSjBVt?qg3 zhi1AIbB*l~ipe+H(j#mi@=b-ADr0^ontUi*oz2eRQrdgu64#l%WTHB=LMMdUu^qM& zCFs1Qi|0nW5S{2E!~WVr#+et>SmSrQe#HCI5ze&eJL{9oIST3@G|evT2{bFj8c(X& z^C{LeMf2X&JE0{p`xU{idYV%Nr4?x-cS}(E1yvDjqRVUTCzY+Z4xuJzqsp|T#5~>U zxNj9%pYzR0G?nsnbR^%^pUEfR$5V{{=X)S!TSHR(*y~voNSdUfD%1x+)gom#&cirHEBGMBskX-BDSCw@s*wg#ibb z&0M}&|HP)DSD=!*|Z-bBvK(eri(K^O_!?bgqUyAHHhI$?{(5lg!*#<8ZWpANyl;pYD0AQb_P!ojs|xC?&y> z1pANbWs7ylrEjvKE$0n_%PRQCd$cahPL60nBzt4`<*9lYFYmdjaqa1mY=)mjiX13FEqOEq zw067Kw*J|*$7T)$dVtIy1Hm*hF~YA#jI`-^t01QMUdY|MgDLEkaUYuQUr7_TXI_mc z^o)T~C=R~v>r}j9N`qorI#0y65y;$3eisaaMr#@na6z`ndfhmg>>T##1Vdf z*!21QQqRw1cd+?<3%;Tkn9pz6-f1;7T{y37!fjN@vtNmA6va)qH7z2jj=$QZx>UHj zG}@%Dz|dsJ@G(h7CVfVGvPQ6>)OFTE#BFYD=(sJ5S49T-(lK2nM}P6f`k8#E+-soF zBc(wVI(#A7g-GjYTUE?C-@8m*2V+q$k-?RrH)Evx0uL_tO$r;wbhD^dMT$+z$;2AC zK2xfJH)rZZE95c+Gs@U?IJvt`?MVsZ$B2(;r^}8kUT1T(cxz1W5?>W#+wW3^S`od#5HTNQW@6JD} zJecH$&0U#o_Q@A4^;rR9%@KQ(a3W*KrPxQ)n9sy0qx~&Jb}`$>&L!} zD(8P$ek<2m9z01arj&oN$E4KInmZ>YX#d%Ka?9tY!gY%4Uij41MqU%mt8zINa0OZY z739{HVl8pc5GUA$S$5ZOJg!b!>Ejd`H=udLPB;rP21iz16G#6@&0|q);i++{D_)bO z^C5{WDEB6VJ;uwx!h_KNR!7Jnod7D3tY> zdzqxAZRp|n*z!f13ta2LRWTnY+T7FwhH{|g$>UY=cz%0&x=-A|b>3XK`^)q7t%aVW z;kqz`{2=;0u9o4LH9dfn3Tc~q+(>FHWMiv1QW@%4tC%y=h$LQ~Ah{Mh?v!kF7 z;5<>C=0JGtQ4ZRZCLa#xp{7k77yDx~cI~4%PbAVGe=%ZWV!~J110dj`LYF zf}-Xo;@M=xhsl-2dao?hMHi`hozV+5!uFsG?6aM03IP;`hSkJ4r3Nq#7KKm(GP#bG z5wTmQa^41>eR-~MW|wEDX9T@lVXWqZgu!cloKdIs?sWAOes#L*;_fAdA7ti@SaISp=>o2k~77tj3wsv*)`w-@Ig}><_+-dZ~Ha z5y}E&?DRob_~}nBJ(tsuetaOQ-O45OL;|D0#FmvIbc!rpmF@G3^QYcqB(c=p6A@5@ zJqA3~&kL+qwF!bblPsy&dqb&P(heXZ6>OB-7qG30Q&w#?F>#tF6twpFX!;ub9EkQ_ z1Wdv+edljl`nDB^sK`D?bvT`t*MlARIL00FzF$4)rZQwNf*wwS*Db|sh_Qmam~9y+2WaenXRzXjqN6jLzS{^DIkAUMFnX&ZoZJ9BY+P6hju=^ugIVTnxYKUsQ7HbXDq1?~J zu~rzSTr}8fn)F;%qW_;146>-7#<4$I`_%F?b{*BSH@QaTw>;~cKM3>?_4x~*Kw|fu zxWqKf)>keUL^GR#KCEWF_n9geF=nPwIl3menzV6T%(~;S)}D*2R<1&)DJUY1Gotkz zsh-?HJc}#?S&$k#v5KCR(}zaG%QELKUJTJ+aV0-$ z6;3WW{wmTHOlE0VH&Uxtp=4BGq$qr)JqjUqI%UP6Ug&|1T8+9VXJy{#B3+l>eF=M$ zGWHV23~712h@04&Zq&;xDHq%&g;GD|h z$s`G;rE^4C8=7$EQ}5pG!LWkRf=(na{F}sR=F2ZY>!XjfQm>GE(&(klf1Q3^-&P_F z`hRF|H@Gc=#)-gtX4)i>E7qT@(6?1oUnv@$Hff*$je{1T64;W?66U z9kP2YHPTdrt>@qF<3HcD&vDoLg$C)V(%sh8^^ta8Dn%<5Z(_S@%K)OL2JgOGytY6! z5KKf&*LpeBwy!Gu=bqSe{s@x~qLj(rhQ5S5tlQZ|4hSXznkbW-Ng`%*CoJC1R0P=UDdjA6~&@$s1wGUinO~lsJ$n) zZonQudVNWv;_sjYDf`7w4#lHcD~mair>HfOxu^ zFvYAsMxNVy$zG-4BkK%PxkPbtpcP7xR%D@%SxWlO%ZF0c2O(NRLIMTSV!frEbeUKZ zF$%6ZiVr^!`tFW~0;wf{MVihLRp_YeQ8h6}K60_Kta{k9l1uiV-#L18egfUw)45Tx z>t1qOtH5!nLz&e_UMz@%lq23C*3h#kB9gQ3Ug1J`kB${h%l^IQmoAWn`PHaB&+q*_ zO-JR-2d4Uw@?8F_`t(n|QY~1hT&wfWFSuv0PoQUQrG72b8nbAUcm0j;g|pp*g06*Q ziAPov9w3H~kikv>^y7!s=FHWz=*DG|frus8cE6{4fo0|ObfCq13_9JZF0BR`uADo) z*cTa1J##E|;VI&Ns_dgl8xCeM*2Ip0XJuA;mXo1PiW;87ddc zN|Bik{2JoYGQ%#{z`t}|h%=~;er)qxTVm{*nZwJg=~GpASR(u)xINJV@fBGO+~ay2 zXDDj4E)TaUHho?=!TW^k{_w=>?C>-8)Su!%>hwe6pCtlp6AjtqQer(=D!ajM8L`2G z3(@f;uj{Stk`GbuQ6bXlH2_l?ZvmAX9cF;&#eH-iisxdrbK4x9te+>-&$!_w)59a{* z#|47I^BDVeAks+r#%{l`C0Qn;Ryxo%ri1W>U~$_an+)_!!;pz>u3&x9U}6RhpWao} z_t+Y#@%vtTY`^q9yp|T-*AVP0S@kndb+x0KkFaz&K?}#L&4GSZaIl9eM*3(){ZwwF z4=Ll*zW&`sPsu|EYn6z)2Q%4~%Pugxt>+Y@C)L~m`2*7c`8>CZL8M4F%rS1zDhhGp zT&&n5c7bE_(Nohc25we$T1>g{In{7Uz#@M_lbmz$OM{AQJ?1Ow7k&a(BO(wKDsGbm z=3*8+De#zn_y7ps0P!`J(olB;Mbvsy8W>tiK=?s^hiK(Rx?K3Q?a8w95Zlb`v$_t^ z(!K7L)u>shjVD}grGeZNGgZUg^yQhxn|kq6_>#a$i0s|hntN>w^_q+~jz!nvP06u57Tum|z^L!hAtK^(ZiUl|cq};#8r2lfeUhbw%AQZDO`g}{ zsk+nl2CFR~^CPWp;E71V44NC@8^?_1m)4sqt`VUY&0CfS@(e^?>gaJd+tn#MIy*~N zx(csP-%bgn1VCYy(RLO5j|yt{8Q$cJMZedEN-nj-kuDa{)X#v3%_dF})U0(B@LF|u>duB1+IyB$Y@bX~v&|>IPRd?{j#*h*pcJ_d}BGa zduIT#5zLSaS1hFu4F)2pW$(sEs3b#Ml}v-nwoUwy8M0^S4;l{PkLm3Ry-2fE9Nw}3 z9WyvjgrF(lA@7&R&*i{95@y}DLlLynAjrj4F&F4p=)d0hXtQ3u%p%c)f`r2hT&JOv zc~IdegQaAhst)5FT_VO;V=gPc#_J$i1-_I9N;V7#U7X86 z%?3y}`Z-s{GzosNlwQU_t!l6rsP2s;E!UZxz6RlZ5n42;g5QQ^{n%5L22sgaZqQM2 zn}@8RZ;w6OB@Z#rtF!_NefIqWo$KPgi{G+sag{lZZ2qe@Cz{Al9DA%4jMrwfz4SRM zLHEJ4kdTmKOVcYYOD_Jk(PhjKv7q-a#|^%n+DuS}Jj)QHO@M|j1?Sazty`Sl z4L%R5*PjZ(A)VfM0XWbZIxud$@Z)=WS{f-hU*wM?Tch_I7S(WMI}NevyHC)`^vcA| zgLB%{pcyUqp0?bZimwW;%Q!KRE=)VD>h+PY9~I)kK!ON(Q9`#oP#2$ZblK^xc=^o7 zPaI_9>0;p9HG4Y)C>Zv(3bGJ(t+qIOE!aDIb?$BC1)3Wc%gle)AbmfF`|_LQnSk}a zx@$y$8DNAdmpj z_Y$6z$$iZM)(I2H4x|>l!8ANx4mYGnx1XX1t#9m>;GEGv<4_%ws-u{HZy0aw{4L>b z_S53Fimt`JIx`KYJPN&)Rsu@aygRfpfk6O9aed@ER*VNIZ{=Y?efW(l(s1%csKeZM zx0hcXoHHTrq zTq9(j6G!uT9stp56=T>k^ZZtupdL{ciV(B_a;dFBb9DyCInj>TB4zqe+)TrG^GMvY z=5goFmFfHOjm6=7t3QI1-r#!&x%Rn7SDpdu%^U*}3ex#c;bCICvmKsf+|t=~Mh@qc1jP zfjfv0a9U^mjM^iP?ms5Dc)p522OL2F(XW;P_SNJvH8@oA&xxIkG4GwZLV(X zR~;&sQ+jTxA?oppl{8x3?cJdHky)*0QH`BX5sGl-c*5nZ182QV6ia*$_L;^Qvf0_9 z{7BcnPsm*4^;q@*MtGhGGpMD$Ubssv#QhQk@$x5d6nZk&s&q>Kep z3xX2#XF*XF9B9J#)HR?zF10p>9F(n80aa`|3^=Z80DuYT{qG7L=o|DRbY|{5%IisO$#g}XNP6>-92QceV;W^E3`>ymv>NqC-+21y$xgC=2O z{I3nnNq7TD1_lTYM_DEkfR>loJU^IyU%^YcVU5sIe28)W(UCQ zd6O5DD}-3oXpeoY6g5CLx%M%K`}0|&N9VxiK~3?ke6~@OQkM;WQc*q$BK7DsT~U}` zW!t9*a08m)A+=kb6w4QoOnjhiWPt!+L6R)4mA;WNK)$PxPo5=j{Xhcvc@m;%?}~vp zJAl-gO)(&>EqW9?9to+uv8sws~pLo+#d*p7fMf)MJ5^-|M!<4!rI7ccY#H5EIK zrXQ2#^{8@=cNKm1G5oIH^gCD&F5?Bv`Hp3{P=IjEaAXK823 zHcdIz8OKbxwa`eiRZE<&GQ$kW=Xc_cVk_kgDK_S zPf$V5Qv8BNb%lxAo75*h$IPVi%kc}Y0($t0ab-yGjk!F_mZQC6kHg$+Q!g$?gbnyu zO{IjMoVwTbbHcf)CYrnHkhUJx{->a+Yr<8;b;itCRd7|dYfd*qru~|64dsYH+fw_& zs(jP);-;B;Rc*gYiYv+!y2;I@xvKle4_gFB0timK3;iaKKKGBNp%*DlT^`>jzkKX< z&Z2&bW=D-?TWcZt-P(_jed-@{bASFxAl(UM`CnNW8WS8 zD6ZQ)0Vq=~eWzBqMRf#6bXQ)Z*E`bD;+CkE1-;3An7zb?!|K4#Hc;5!2R%o{{1YPu zknG1n|E(Z!g1d~NmcK2dB`>5h9iKj&klYCv{mZ5g9Ips}%K|xW+?{UM>rTe=@_8-8 zZqLeB$8$V?@~jM;%UdX}uy7gtk-n;bmSeU2o7XGWJK(2h0Mv=8@_@6O3fS@95BBw% z2al=cEPW6&@RZf}s+bMq-B0~iIuJPa<#flfkuDJFv#Ghm+j(e``#ahP(|!C%=QJ+W z$1sXazC&Gkn_9mD&YGpdKD$EwGBOKqD3^xcq$P}2EMw_jwpN>%vQ2YjquNH6MP>8J zd69I6mW@@&UN^d?uVoAws+dujh|MRbHS8FFk`|@s=vfhqs^e_x^tq|0c}D{|Wsxm= z(aX+hYY&#HI+$lccaNwq5FAx*U&R%blih~^}r^FohyDPsJhO_r4-*f^-f zohS4ht2ttyrYgsBJ?QwS=i!Q5=;^uJ=NA`xLlt0e8Js)R^=;nlQkUC<^3&r2^|NlD z=$e?uE^H-J_A2NI<4&istvQi)zN`zhB`je>y)v7#4Vg+EG46|YdGnoa3O|Q1sSiBu z)kY_qb&r;$S!xKqtHsf4f1@Vq#K_8Yy>|Jn z{TX*vI`!_B-Q+WFwhMQo_gwg{PCM0cxne5LGD}^^^yA(494#5yn4?G%1~WeQPdjm zYM)8GR(iv+e;Sl;hPoa+c;Mup9W^pMd?(~sx^u%v9v1DQhoKHr_R%jbC2?*S_X8^K z?Ugy-%fEQOoX;3E`68qyb&x^p$|N{6*b3!usH`+hOGa*;X;;0jsE5>Cx3Puy(Mx%I zKCFheTJ%tP9O8{^iJ$J$+*kH0XT)A5#C-B!?+mUTUc7o6@;Yl$)x02~LP3EVA<2T0^Im(A?yo8yI@d)3$^b&$2YLSO#cYjWn z3*ssZ$JS!aa=%ApUZsjvwVPI&V7sGt#1AoWAt?^Q)hEw_tg){?7DX~(w-U#0pGV#| z4`}0;OW!HI4qN>&JK{cWX&7Y9Z5%F|`-X$=+LLa*rM6KoE$zH7y#@>X`qWHM-!&&pT(1G^qeyL9ANLJQ@i!!;$1>~<8IjNL$NaqZt>^=DM1BChx?&#qeo)I z#ytJ7&U^wTE0-#Vv4>{|W!6w-E+Pi4m+eHbvBXg8jP{MOR{|wy#No+$nwOQv=0ATZ zx`(+dYF3Ux1m8>}Ov)s-!?%yTN5D-GNfr7z^+Aje{gD%kVSSS~`;L~m_tH*j4~Uc( zB#;&O=1pmoZvYdJ9jo&5F5ftUS~MeJoy?tGmdsLSXZgzb;2jVf4Poi8(tL_BMu zHRtqK^N7mqW^-4W$KD-v@8Ru7{6kV zVm!H{Ej3oom&M9Wh1S5usG^{$-7=z-3yZ3o5P{z@exq&FeN2}g9-jtI^n<3y#!=#2 zcpmw5Pj>q!NvXqDu0K6AJ77T*8i>O=af`AU{HF zb(P-!gTP3mGQZZjf1<>m%{KfW!oEAG$!^{Evx1@`pdwPDH0cORFA-Ei5s)S#O{Gbb zUIK(Dh=2m2BOss@Y0^UPDo979h8`)Q*FY!s*B@LISD(t+g8PETxTWE9 zK*^2b1V^y{Gh_BtSx)c_h1EFH&u-X6ie&6?^Odb+YjlZ6#}4EDKZ|J%Boz4I_+uDe zMIr!g!e{+6V}Baf=K|WI`skZq&11RIpB{MhXBlHKzaVkzIDT*#hVGVyOc{_%M3EK{ zwblNXStoRc$yAhQ4|iTkh2Bk{RAHC>Id0{(uR80z|pJ_(!=lK=kH%SDpt!r^KS|W=@ zzmTSA2%@6%_>gYI-4xy84}g5`%GtF~T7~G9je1}aG5N38!R!|1Ky2k=zNpC4hi!fH znv>lv^1*Nm-C5*L10=QU9t8j9l@q&#ktklgN?uxSoO`#jMMb~&0y@Lae4bZo_bGL7i#lRkeK$6ZY!y?piSv9i;na>cVBG`PoLKWRa-{>oJ+W zbl#}wJyTOdaR)8ogI$YKcXe#{bE92V;L>Tjdwn$wG+2iYign9o(_tJ*C^~GR@0~~A zsh`}cBhY>2C?%W+|6l=V^Efn5$?mR2HH|>_AXJ7ZI#X$f>~-+7vP?Sir+#^ceBnnM zBMALK3Tx6Kv>ky}cI!xOUXs>`^W0}h*|~e67-X^Od+E|9vgLGLdgfx8j|$L{b}r;p z)Xek@Nncum=d}}@*Kqb!lx<91fo^Pcj+t~bPs#}1y{xG1F@L`^LHgrJygX)#0iCfd zFF$Z)=pe$Xz%67ahf1Fb{MuFn@-~BO3u(>0bEiql9C8QsosS{TAv;)~J%@#JKI_Hr z`g6+l`|b5NPZ}(y&EYpfQ!?+zYfD+%1NxGl_XD(2yKiWnIaG=7Qak9GQ}#DAyHy-Q ziIL~G6BOUe48o`%56YheRSWPhAa+-L5LohJ+|kdEM3<+5;!yn0;Nkr@V1beg6!Jg- zysPwz=1Bkcj#1L5RZ^KP(%Go&JQF3=>X}A&zR%AWB9O2L^3p3;4Jz3UhO{B?&LcUz z2@Cx(<^Kd^@q*f5v3%o}flj?%7Rn&JKFT2wlN))B+>9kg)f;biFO9q*W{G4sO;n>Z zLJ)sS`F>ci@PAbCd}eX8Vzu9HKAWa?+f;$&%6(q#UX>7*-SDeA$S3uCGh=TGvA}oB z(PNCrm6a}aK|~khe+@(0zsuL{5rlJtb+{j!<#81VXko~c(zwW5yl+1kr*P09uFfuG z2g!fO(}3AkiGPYO7)yo-Plm-yDRtH&@~=pGK7rnAqnq5&_>egu@YqbZuo15K&)eoD z=LNrTcDD|pb?3ac*!B0ue1>gU+kr3HjZ3X@3|vhyi~_>-dw5^Kow_(CR5?vw>N?-! zdtI#&H9L@bzs04I|Nig?q-dBXb8&%jb!=Pg$*0xRYDOD;_iSMDA|W--Xh2>%H+6l1 zQ*$svOweGcC%U{n6`dgnol8SoL_>TW(yoxo&H7tClD?cgH@pH(CgX}1zB$NcDV=4q z*p%|8qJ0QP&?Zwnz%a`35&x#s_T%YduTw4_@^@Tl2f;nVISyVlS&FLO1xxYS!?Aw=H}D6G%d3i3|fG8V;1 z5bU&*IAr<*p&Juft?eE)U#cWrbDtgyWMaq`M`x@ci5>F`K~rH58elK3+r02~DE3+# zr|M6I3h*Jc&w!Jt|L()+F;fbm1h!5PagQrncz1wh><|BRCx^|ww>q&8N#3mQBf2ft zzM!8e9j9OxT0SJlR+^n;ec-BaNH2qTT8&s1g{V-+-K-bUQQtGizB}2DEH@G)wzBhg zYY#tI~M=Oe)uLK0!!6`MU$(QH~eMo$k%Rl)d)!~8? zc7qQjEaPS?fcY`$#}d62EkwS?v?IBLH2BP2N=h+DtI5oLOkdX00|?AIgu3cUE7dTw z2&`HuM|SLWxZe&gA*W;f>7h#xo!fv@!Gvnr1;wv?nc#5+n zcBb;fb@{bWwa2eKs!w`kBlivtv_glib=}527pWHXp$9RKi+lFaL94T8YwI7*rM2un z0v#~uJ!;ORZ`<5$0_19DE<@FO>c(7^1E8|U=8T)@MA{8?m_#MZ6pz!okarqC=UI zZePHgq7~CMBv*=HrWSXlh>xj*s^~L9?RXQ}O%I5eejr7-%v!xxghB`q$!{<66InTg z5^q;I)W(0}w+vyH5R#5GS`o1w5O5o^h|883_q_81<@Jl)nuwI{=G%_oXbiY&Bco`f zrs8m4#UgyiAu=VA&%HvoEc%~E=3LCr6bF4K0vOqQa>CW^0pCs>?!K!j#FVmURlN`) z2)Ln~q(-fj2P8-h68e26v7kPCr4~2t3wt++*k&%}4af{=NAKi#XGFryT^E(pbvEi!2Nf9)(X)}^HJ zkUvq(tQ+dQqC4en>@PKE^tH?=eAqO52O3pJ(U~&9lI{>H0>RK3!dh$!bQ)S6Ji^N2 zSBllnB(tD4uM2Aryc<$?7WuIh$ruDdu%U2M7jb~l>TC*+*L6Pc05~2OylpGRawXzA z0WuVdD~QZvF5=kK612tQH~Leixqj3^yq83{I2nwGw3Dyay{REC@Q1`;0Q89D@Z6|8SM{=O|{u1sjRaXJeS zy(#)hRFe})o@yygzG@Ygro;QNu2#Di|DAsUg(7PaT(=*cFuovFEe{)7%g;438Ud3` zG$V)YvWCp^g=KY$fRgN(v~^STRf;ds`cSn&+Of93Y1xsrLk}3iXr+<0YPhMH8c3=f&2Py*4vE z`+0g~b*gKOEM?xlw$8gnH{hcLY)n`uA0J}d6h}S^4F;Z%tdn?$a3J1_6t+GwB+4Y* zY*i4Jq<&p-C-xo7tfc*LN!zkIEXb+|J{%;E?l2sDu2P3%@j}j}$>D=LBL$<}*;cQ= z*f991Bb8&6vcj?5wNOWaBy3JiUxdT}uU5$XB*?x}{K#>E7xn`V@p}>W1=-`LYz%%s z(nrR}$S?6Duo4u3q0FY1U5#Pcu0YoxS>YiBW3HBH1jloiYIk09}q= zYQJ*7@?~54!DTti65=Lr%EKs+M5B1gqYqMiC#?y;_5%AzhT{cH<-{!qi$&W1tfBCn zRy)aEb)-L1uyetO@iy~|{=-|h~-zgDYC`i981?hhaY?zq(+Uq3$#RZ_ipu75~{*8w8 zSbhzaw2ey_Ajfl^{fpU%7fMcjWm+gSJEwE!&AIF|o>SG#{bY~v0u!UnXO6x%IH6H* zj#GPT(vDLM|Gh8OD(q1pn25;!I*iI80qAO!^{E{bQ=e^$aR(nNoI+_}KZqoG=lcIvdYwN)||1y6rrGw)6DP1i~{7yElm{RtUviQ~ZFYbno; z2(IXo3~&TL#Hzz6W|W;)GV)*L6UqUHEw3S|4)E#r+ujry0l}9HxPla%^}jy?*2$T| zFFFL&M=wHtclGYNgwd7%xJo&w`+7(7-)Siynh;Z%#7Dk9^ZGS|Hi(S-fCD$J=Y)X9 z5JJ_pWG5?0N$+(Tft)?gH5{SXlRix&H|T53Fk9|6@oz5}yM`CNg2k@q+lyF4xbx!aKCY;b9q z4!Ua&T8_p6Od;Y&4L($61N?{o!r5N#z3^fYc>?6sBTUEL@0;W}oIE!o<~5$I-mXd* zm9-P${53X~)avi*=Ei&Nf~%{mc@H!R$h02_oT7ey>2~I~$CorQLqqd&(+ z(-X+qLpHy^zKZe1?zfLDgqdPLJrppVST=Ee{lBbqdHjx}+Y~M;h|3=Pqth-?Ix4^6 zpT{KP0r*NEJLvy`#u<=gWe#Q8Nyop|0^2|m=9oK}_nBlH>E(LG6efx_R3?oCuf5*W zT7?6P%(ufJ8fsK1pElL6%F*A={kOBOoSa<5PIvQ`!Smi`fE(84F^HDKLN0#-v*7_N z24w%8nzqF9#BMt9vjQStet{e%P9G$S+iS1ZkzUlFj*w@2Nd24Oj!!^-!ZIE*>W+wAd#_ga zdZ$Lrz0s>5VBkc-&8L{ankS5STtfBPU8c9U#6hYN2ks1DUqUBJ$w3;^j+yBCrcdz( z96oI{^-^G%f)8%YkYShV??MceOLye~z>aR8jGN=A)ddXa*42mYQZz`m_3PIuSmDNw z3dLV=UWy_oseU$fO@GChuMQmAh6oLBoDsm@-x zpx0y!H~HCE{BU$s%cs@rZ-V7etOX0LcZ=rj~!=^62_ah{Wjw2 z$OWtWJ#!75c36;efIyVhwDcLp&+-~Q&xrIivF4C}IH({2LjSZi1SCi1V8CR3*o6QE z+887;$|fGiCn##vWHeku%M~3c@YGDO2Vx}9VK3z1`8d-2l^SK{{>%gy zpost{>F?e31?&_yihD4yDzR{!rlSDCy^(?eXSgncKkkXC!7ed%f@z{-P>u)k)CPR)rT1cBs_HPqAC+*MgVuZKhZb z@#ar3qU7+MrXf%?8BVM9{J2S>CZxd+B2EYassjfw5xL?J@E^GU2@3B2?gPdB9Qb$I zxWY>t2FF6>&JJm>i2Ck%@;#RBZhHRuIFWNLSkp(F)psW%rgXDk-&CtNbJgNPSAv*r zUxBY%>99k+K@4_qny1VM#ELBIUU{SPyp^d}R4Y2P(7oy$A#n4veAt~@Huhc@GL~$D9M)?mzop3uD16r$z$I%bAPhF_o9&gmq$-IdBy3^ z0P>OP{+%#}gXvMt9D`!3>>bNq)7brcV~D9So8^R3J$yu3@1ik$x4&j(Q4PYWW!oj7 zPfn%v@UgI&=d;4NQ)Vi6jAo=$5aoYrh!qlrIH*a=qTu5lLN+rU{(X(XI_zA6rggf4 zk2ljF7h7hyDIUJ@e~L#t4%H77fLy3%km)~89+q3vT^)}3tNMYL%4F64f1Oqrfk$=&yw!pYwO4{ewM+KYqSiw@~=%!Vm z;ieU{`l+=LYxqLf7E{-|wz_+S{<;1s>LEM_WJqVsrOTK?g3u`+_eukgk0Y~LHHE}4 zgx@)*Co$@n=e0d{mQKM1U{6{=BT&3v6tMJsfq#FuDU`WCJ98i5ik&7$CN;Ij=IR%S zM_$KCfFDc|*2TO9Vw9RXVHLj@V^w0(+Olx5qvOW6GaL>?+(#b~;=7Pap<`fI?Hou# z3&B(=4^Ysxiby0Q@2)@8W4r!yY#na48j1x(6CY4yB3@fixMIJ%mZ=Yk*p!VmU&{;x z?g9%XXzs~#OH0?kBqzTRa~SV@_}zkI{Uc4VV0))VOYDeVHQ-6+H~=Gyg=Fs)Eg<@R z-G3Y12g|DoIE=l&0^tMG8K#@vHI$-F|7+^B-rrjE(c!0ES$66F`t2BLYF1FrW;}S~ z%jFTDgo*G@$nc()a3Mjs!!$s?Ne%$P* z_;4%(DARzSx5OZ+&1v!|b;jg9yv^~N>X&T-doX?dIO(}4s%#YNzDR#%!&Ya5$HV$G z(D<8_E`?fr(+}BN&NL>qD?)V7=at8Q z&%Ap8v`QC&f@sxOA2j2lIXRw7Ma?-n8d6+Dqe)Gji>kXqY=H?YX`}&cYrB6>N=@M& zxKi22Xk`m~Jbdox6ZZ$o>oP#eaw${+ih9AZanISwxpiZX)Us_#?1@X=$H_AqyGBsV zo1jdbDX?TiKDB(~5s!uMK-v4K9BqM@e=P6B)VRvA9WJ6`aZOHq=r|MrK}5M`U(&LE zw?a*Uz6qR5T5bvkC2Hx3{n~l`oKyI>VpRroBjJZ-rhyQ=lkxBIaXPgQdsM-Sh*pD; zSPD^B#JqI-??C@z_k`W0gZFvdY~A|ZqVQU&1NG81HJVVYH5xbKP@g9c=nG2sgeW#> zZ8SIVe!sp^L0HJ*`^GG3t#xZCnPMR&osMTe*MZ@JwO0v9s0`7nqV4wUnpgoDr8KK+B(*bS4CUOMw!~E(?|u4t0Vo? zc9~&ebu0BE@VQNe)`#sQhOJZH(W5zFD_NHW-WS9Q8K>?~KXuzD{0-a?(sijLeELQlx{PtHorkPqK*KUJ~7h^Uz2*HP1# zsxAW<(H+j;(xz<#=O_g&txV51Pqbp$x^3w@{#YMSQe;e*L}|0Vj-);=`6;vdMTmkYG{c)o zu~-#TaDX5jis|8(xs+mOWBDS8OzBW{He=v%VPU739s)(Rn9X#=FGQ*dX)#>aSi+R6 z{Gu=Sox1};kHes|z-)-c-qNonJ|T=e+CrGFWk zveClQ-`)RoFW*WHRrPe6IRtxZNV_oWl>X`bWEqbaug9h(S5^_L-dYg9>?u#3!gM1) zHQNZmsT~5480tDdPW`RVpfRx1J&^L-J(K|#kmiYWlHCJ@Ap&!5NmQKu@m{qh$G!Dt z*tE7zwO;N+-fzEHIO-4!6+3#?#?kHoJ$GdNSD>yx@gITuO46Y%4`o6kp;i_x;ms-1 z-9a}gS|vFId>6Xu476H8gljjS-7SnRe*?IMVQuvR52QTT!smd>eq47OdoYrP(nJy; zxvkcEz#3IXGJ-!S4%{L*ENBuVMaa8dUH7)7anbeq;I+S1PH>(lQ8zKtVj!r!GM*r_^`dbW_}D3y$WX z#4tWolwhVh$^v={*lc z$bTw*%Af(5quI()ANr)dLJVHx`cvrxFCt8yp;gk3leVP~1H37O=_2~GUCv6`$cyfi zh54tmgFm#^**iVQ1HrqqasW?OeNjte+of;eT{s7^8_wI0V^K8y3d2#0}@I{ne&tH@c zEtgp-nd%!a-V)~%qCw-BtUhSLd3nQdlkfGXRNyMSfV(i1qEZ(P$XIP2PLY|X>KjLt z?{%8Hd8(fbjpD&!o%}3WP;ECGi}JQ0EF5enS6tU;MYREa2rRCWrmUc%MBzWp%RTe< z1c8_IzC$9e+9mI|!J2^SES>y4uHRKVGtizr*j1Y6H{)tvab^r6!UX}6o3l0IaBx~d zt0h8P0S{iO77Rk3KV2y*rID=)2ei8E6Ewbtady1q`L|KP4+jXDMGfd~j;D>0|j|%CvXpUQJ)&#k-ioM&)0mQ6^W(ZQ2{d^nO@$c5XpY@2*X-C|+T7Ym~f?b-; zsgX{VZ?MH-k~{}#ZTHnCYl*ws$1TfKl-L0^yv!Iy#S8e15QVxZn^>^=2dOJHn+YXb zyV6=SK#4{WB+s@+*}LrwmDKgK&LbPtLmM=vJAPhd0*2J8+Wu|pLr==G{Cd|tMz1f5 zWqss@&Sl^>p$kHt@@Kq5_8zZjw7c^U654*>$~tP(KJMibvIVF3`)=5f+CPX}!c#wg zYOHQl*zMQA=gVi}=70;Jdo2wyOE}x*V7t<0r)5H`p_x%WUw*I(f!%5iOu6OmxQ1l0 zlT-4UugMrNYm74gE~ym5d`c;*ZJJ0C%BM0~9iLLkP?ZXPFtzYlSmuYwr>08ry#za3 zuiu;+vEo^c>P2D@K2b9MrpAX7!gWMvyi0TyE|xis z;~R+mg!NhCzL1$}JxlMMYSpVIZraLkY2_;(AQp52`@h)1dsI?F*)Vs3F~D@tx@W(# zalAHPG3X8V$t|&(^1F8fUS^|Mux=P$tpn%Rl}31vNb_tQsI%F;mriMksa*?Gb^Gao z9eW_vWaP_X-rk)CgTY?f(p%oFiYouf<*5-V$P9Ps#GOrkNJ8;eYX*m1f{0n_-V=zt z4>$kDv5*Z**1l5HgkfZKT%zdnOmhiZ52dRAXYF7%A-a2lA_}HRa2CVm(BI@c&_#M> zEH^C~#XMH5a>MB3I2va=s}1rDihBm1%Cg;mnA6?wWbPuNGR)wN1(k2#V3ApxS@Tpg z9C<*(7LY`1>m1PE!_+L)5U#1`=>fDfKgZ)znaR@2T2K@9#V-3)h+3S`^0usYMDs&H z!H7$~keqKm1V-!H3OZr$)6uw>RoUT6cxIv3Ff6(9MLuxICdX%@5&Fn81;JpZw^$*_>5b-lcbpiWZRa!p^67 zOv+OF;;B&FWv6d3l%6;>4l%i-Cy$Bgwg5E$GLZ=tDJ3JM$>Qbp9j8%wVB*QA)pn=J z$px54?d@z-+>Ts75+j(nor_mvJQz6#*q2W`kNqordY(ll|7-S>Atvo2rh#3iOwk=; z?dCTwuY1J^iVbP=sWzdOUZkTlg6(*l{YQulr>cMs8iDEj(DQMV{zil#I2XDA-@``q zD0<#!g$tVc?s5UWt4rGHLd7+^8NMEo$<|rOpwt6apTTbn zzcpFENS}OB5MH~mthbQQysFKQ9j+^yqoi8M{^(dr;$uo9c^krh1iO*L^Ht=Q7JHI}@Y9Cz^{OBL72O`Cb^SiT85pLhJREkQ_nP`*}zPOe%O)+Yzd_o##&{B*^Zz zaG*9x8pFyXz$!T#^l@T2EAKfC+lpBozaCittm&vj2Z472eDA3XkxUi_xh0ye znvW}ct1=`xPxU=nmym?r(0H54>({p?X0d|$?|jiMswL6lzzspFUo%Hh^`hE0Z>P!h z-@28@WdQa5!d}jLT2t#4=rEB%(PHlhCsRX5{LOw!1NL0tGZ+yEZ&m`==sd#_R3R^7sd9F+t!T2duwX7)V`=Kl+BiX>K(pZUV0DE4hq>glD|mu*7r6CUq{guz&!0<1EZldlww?38-Wf7 zco6|N5xr$xfLzy62*g;I{#>m#g~UO1xWH@XoO`8fCR^=-mFP|JKrs&A`dF1 z0nDd%0rQg&e(Bn~(;~1LXE}W4JD=w|MyKZ5Vn5qN6SJopRjh?!uMQUU^b30@NqKz@LF>~0&Y=IT`+obk``)&6qKpCO zLx3Uyu@hTti`0IqoDbPw3lzI^6SMz3PNRMhCcV3x3;=gHAmx_`XWt8Erg)*tuD?ig z-|VNU&v}Eo%*MN%12N4Q$Tt|#eD!xwf33bTCGc<&C%RA#yTR9WC?X_Z`VWD;t?LX$ zQT%jir78K1ofL57fCHp56J+Wb?P%uaU}Dno$m1~*7qOtu&JntDBpZx5E*yyXOgj<|2E7??{Haf5t zfs5-c9p#~IJ4Ri>rXQfqM(`&pkg3HlW<&Z=JcI^m-g{}?t(B6p@ zJmz6l);2E1ly;t0R+?z^FW=e=pg#ZV8SHi-ZQypuRobej=iv4sZIbzr%yLQNfQv9s zrzfes1D%E$#7di?Vp~oBPm^B8p*b~@Z4Jj|_>59>;_!v#1$)j_vCQE2_~!^_D(694 zn(qT>7UY}EsRb$mhbq|-VZdSueSYWYw1vi2oRqlaNLwiWqwII1rs1$$#a0-(#R#fg zcBYJ>5)u-Zqzu1MC!2pjB=pSo$mgVOn7qH98=GhI%NnzF6NBo&po|8&zzd6b=X4J| z{uH0;|L@at8BU%r1>SUi;gee46=&qRe+%DzNqtNFcnM7H^*O3|8J++UZOTIzKyU!^ z#qFrhkCwU|4HMSzy(ad~CVhr*Malpvy6%Blw9&==FksU%(bi$xat|~8nYHF(EKJ}q znR*GH=dF`*VG53PwjRROqD9R$h+aI&z) zHp`h74e3Y2YPJlYPOsNOd^Ly5WG;3uOdOC3mquy}6s(L!x{RQrrcmdnaSzy{VM0wY ziwI|3oE8*~p-uF58gYBKrMI$FR0T_ z0LJ*Yoez@9qO+A!Xp5;!1c^WIz?kj-bKF}Yr#=l1xPa6CbjzR9P@DymzwTnpCX7MX z8x=EcmpP}&@}Ly29SgUMhsrb)>b{rNJS#4+a|-5z;3f)O@IWEVN%Xo4&w`6D=C&La7C+lG}bRxZLXY$1+vUHdhUA_QCMRg^xeEZ#if ze=r>(s4^iJtm8^ODD!n(%(rZ}(LLVTnm0seh&l+ezKgv3vc9{Fli~1GLCI9u917^dgC~r1CEhR;m7q$1> z&X5K-f$IAaXDomjb50_z*Lq;17(dtkPV6ovf5D(G2|nhrI#+>u%6|0lf%Z6F60~GQ z+Vz&jwcn{a|m6#pLy18q#?!V z#+Lm#jkiT6E&QzFKZ4Yx@4qw@a{MHF7v3Ni+Gh0IZo@CLC=bGD>>P0hB4hNp+&by~ z8x)&V4t1QZeu28}Zlwv*I1V#7O?SM))+m(^_5ThKmRCWdq08{C*y@<_!anzg|E@BL zoh^uq4ulG6g)+AJ<6HVGpVxJn%@2tVYE1jyFmY$cX$|5pyY=$Sd%&Unt@@@udgF*i zW9+%=k`#q6jjX3wX0CQW)fir6u)Ion5z}Xm41Oijg}W=7>lnNzf9$NS$gBOf(YUls z4lDr#5_wq80@>}iEThO9?u#YMRoKh8392%Cg^dy!c-2jiQfV*8aJ_87ro{h<+kPH+sA(H2X}EV}OA7T1SFC6Z6u zpW|&1cq7z4HNis*qro}evV+=%v^CZE>V7c1h8g$Ci*^@KKjVD4RgF;~2B#KFV*5CF zrCSSU+kwAfm2emdaZepgcX8>mlK~Kr3~s$?XT(h1%E4FoHmYI48{uf%?drEEicMW~ zxtJWcR>Qd@<=?&`jhPy9gHwX$r#Y(D*l+26Sz9#uiy5_Q-V&k*}u$^$m8 ztg19Od@k}p@8h$o!W3{%@2W*-)EX#h3O}Ry^Wb0a{$nDf5CZjF`y|p&u%SN znrgIhnCxhQZ#ha!qRB=WK_@dDK)TXW-k`y3EHxFsYkyXJBEL&enY$e;rCfEBx8ya* z9nNk-HaGW;x`gxF(|pbIA)JOMO=d6NpFX|pc?SiU&FGtGOABu1|(JuOk@M%0G=ohnESCbLT=H z3wl2!1?t{SzCrrtI?ZRKc+d+aN7d}kB#K)71isx`aRkd=6xgiP0=)jGfxt1tZn%WE zd#0iUIPkxWrE)x&MebVxD?v?jAb3(CUyJX6e9M|vgFj#!>d+V8P{;|!;3~lBzdZjx zjvR^!szmfDMbY=wFwVB-ls`PdXDM#3HfWe6DZxrSoDd->JZa;fhk8OK)XvmiG<8DPak3lO8rCpak`o1Ur(jBTb^Z5A8@=lo@ zCO@gl_Q;Uv{e|(`z%wk8k3Oq=3(Oa9{9F{`P%`4UX?Z^mLE3u5+hZ_ZhY*hNB@^2> z=QUf4%-nvqStx+W-N^9DeOYySYa~+h~j5)4zv#$^}tP)nv;y zu!cTa;Au&kKwa4B(ublqLDW1#{Tp{q(dcXV>&*6#)5BQ;jf6CN6+py@(ZVi++v{!$7u@uT4IQ;;{ zA(8S`cHbX(8U4D`)+ONZ&sNGMvWiAj@0C5oS0rut3;h)Q^Lh1@pCnzAlszy1dRxtC z(8tFBH%^15FjoFF7%G;snWe3LioA_2kX3m3H4qEFyJ7o;0^KSSRqF78L@!~+4(b|7@AtUC_I(5l^YF1%MO>=b8L?yQAD$`Zt30r`UAd4 zy!SYPTO=G_i9SV0h(NQ`b#toiL3>cXuE?J2oQ9w8gx511VgG&QRg*@U&+k_der2$k z;ZD_SGxqabMZ(*kZ;)sK?hw|JwREjd4V{5KqX14)4O9PKD`}%VD9?GWKb_RZi%-$8!%Ek0+cqEzf)HNiPjk4kJTi|Z zFL~43bI9e0g}AfSWg)Y=N1dHDzUtlJ9ra$Ved|iCn3~UU5Q-iy zv(F6*G_H1!0f?e{vghO}z0Cdc{gd_^{F+gUbc%*Z|E)#$RD0=0t;{?!nv?>dtt z3e;Rj8e461K$1q%3ujCwJ}pt!0lZ?K-T(xPTUQPJc!*$skV34WVJ!X^M@Qv9t+8i6 zmHa?W%O5;_{w`(FCmJ?`zd9u9sfyd@Fmac`X}!l+6k-Fk-PB?hMu)|0Ee4{BwTGqR zBL&x=D`CezfB70{p1na8A?UF(dQHx2Q`hQhu~j_}v-lVN6e9#dtZk7qa+kSAE9|m} zD!Dd)B?~W0)%9M`uq-l|pIK&t0f8ntK+&-11&tq$7 zmUH-tp}zF!jtmGem9VdBWh|oRS!W>fqFU?U12!zis!+JH5pI4e`QNp&>y6n|QF?k3 zr`h}F>c3582o(T#p_~vgTbqo}me&5OH`l}F1K)5PE?~ch3lYeYY|06|9d&-@?QgV$ zdnmFPCH6UkiBfl(#Y*U@f!oL5UOciqL0{mp*YYwy zv`x_z>ewP`-gkZ;T3G+GBd6EOP(DhkceW+j*xM)YvG`($>8mo_#VXmF!NEy-J^8i< zB==`f-O(H#(4YG2SEvq07pVy;Qzj0d=3%6B%NA`2o%G%LP6Sn*Et;zaW~0p+^wu*g zkmk~P#^w+v4qEx0k3`^6L0_vJb?Z1^8TLb0m0M)Y=Jx_ z{E>U&!nFF60+mnUBVZ9#0$#bX%w%RGu=^5YBnjQnb;)YvzfMmh+WgVCQ(2^4 zJ?Y`Sw3nmoq}-PXZ3cn)rbyK4ub)5OG4kO*bDKWY?@}pIC{O&<(%X4sFNyl@yO%0u zqxHxX-G>iV+z56C#g?U`uDzW-d4>V9`#ZW)+oLWFPaC_Rg$qxp(0^*qykBYfxYQ;M zAIc)7x{S-zKs>5)U*?`_Qjd zxa&|I4HKW5DeZPl^u6#s*UHD|o`dK}W)8VySiUk}_^ZC*V`Dq(YwLFIi4!_ix&n(Z z%-j5h6ZW(DNetmWPI#JZ16>Y>8seJ|U!d{la%%QW1n@01@6Yjll4W^0S4iE~aP9rM zx2gZnzwq6HkNS2kI}gCEc+JaRjPor*C2=^VObfL=xeMp^R$sk(+9=UAw#2N&V&Qjw zCu!9O1}_HQG_OxT9jiwoaYJYaY+c(3yK=}?O&fZS?@#@INr0_$_sZ^GcJ{38n<&&0 z+Dd7U6V^w^JGn|2+?|6A*?WGvIV9_{A02^?xU0dF>m?e%L_G?eVZjLl2Far%6c$-q zS|*c{F*2W#{v6~P6{$r>`?egqvSuuHTn@AaN;z>nQQamr##i0=bj3 zm#80r;mHZ}Zw)UnQx(4kHafaPo8zKkveO?872urhQ>**9)$}_CRD|Y(-Q9zh{N-xx z(`P)y;yFjLlk#_?j5|epA_Uw_plB{xC7YgSf#<9^>3BP(Z|5|wnxHd&`ePj>s$A#k z?*;Xl2;?*(38mx;X73?c!VKCME7( zMN+3%If$=jsiQ4WY^EwzRuZe-ROeWt0o%BCt=iV3yZtIULrCW0JUzMbvw6B)Xy|N%WFm4(D2$?it+59K#!u>aMEjF$vGB|>H)4wa5cPh}UFw9P+5DFg@XQB;yW3%@X zMqPKxzz`FROHseao?M%6zIdk~JOYz+diDhS_$qSWTgqq7QG3>3cEh!%A}z$%spOXZ z&@Eo5Vo+D@)BGia<<(aZlEUlqlM!@Qx>rwrZ{QN*8uumHU&LQwW7mO~-F@yfeX}8G zsylV7aE?ow{kS_Ikxxt1@2W>ysj0KS&Q$Yketo?qg}h{i28&^AyJDqTJbneKFe({& zcZ`-hq@1Q*nFU<}Pbro9^s(07rs50?bEYXXostp5EjoEQo*}QYzGI^x5)%S-h-7cE#af8X;fVLzkiIzQDKHrz<>}&|_m+i@x*W*md~<^> zPU|1+=ayiU;=L#>lQE#s|Lu)r?+K zLdjGCsN(w5fAE~t2?{Ctp@nyNH$UikgFMfEVzSFt2?#m zF}k1&M^RXUo}{|4za_^l*; z(n=DQ%Qh1=DAk2Is4gfY10hi`R{wqFMXbW}c24V_DX>x|9fnG_DS9tQx`+GD+9W#K zDPcQcbdF`P;EcnbN4+muPfxl)d|Y9tPYEqP9-&zu;#Xv%@1a{+AS}52F~qLk>*jpm zvV@6t0(a}gk4vMAiOGWYwQ>p53ZCV+YB#8F7v9Hl`+R?Gc=!MC z_0|DZrfu7=3JQW!1}M@g2n$iV8>Abgq!AV+xrl{?gmkBLtANC!K>_LRMq<(3&At}S zJm2$v@80{b8UGk&=H|N3>pYL+cOG}7P!}Too3-0~VzK=SaUH3staX&Ge3Y)XC&^{@ zRFG?1@S^qfj=oFM>$R6_ejME)Bs?^&h2zTuq%z>sDPc-*-O9KosnV8Eqty+x@p+oX z)tBFpsdvk_p_+Og!d3Rq<(+p1MMpr-jHN4O@W9eVvTTU>6&YqRQFTS>t4NECVxp1I zkT2Zr%ZcJ0Fuoq?2FETQi-iNO!mr+BC=n_Dud!hLa#bP;x9s^p6T!0ko1~@tBWqrk zD6tGkiq04+^rwAQmKG=70`3wCY}VP_fh}5JV>yDR!&i%md^!TRsx_+aZ)CTRBAgSy z!Oa}V`v_q6MC;SFu8h_(m>8E}Pe72{ah)E(Db}h)Ha<1Uys+Y31f~H^4Yxth>6+fv z+lz@%C>Ky$L?vxDEEWA8VT=*nZIW{fMYoPHL-S9@Z28`3WNuV zSlkjN=f{jQk}hozx)ayg~(8HLV?K}(0Wo5k+4!+m!J8Ecvs zt0XzS^%`HfI&9?5InUM$bx7{Ct96&&)W{6wGIm|h5O%i8H2Ot@zboH8FT9ndlm0|` zVw`acs#Y6yZCm4p5-Jk)%V@GMwH*QYsC_zvDnsP-nLSUR;5EiJr+$f}L@UwnfE|mpk zDe87Em+$T$G=u%ka_Tk*VBUi=y zz+2_)$v#>#hbQs)NB;CHYOTxP45hYlc6caRqUl;-Z7aMi3A*166p^0ZPsP@mDCL~+ zumJml$YCD%Bw_+IsW3=qq-uv+7LQe|%*GN6j-6M-=d(}v99!yLt~#8ZeNHvw7Br*_ zf8>U#`R9l;m1lp;PaPyoLkJkd81bxbtBk!5d=|9zZ3aPJyN%XHa^SbVfPXlIZ;=oS zxexi8=}d(;Ib%jt;^^xsm%MHhJot)j>9LYUXtcllU1B$NH^w%*OJwwAmIJDvyf;Bl$-wtTH;?;RVk|6S>`pN9=f#yqjw+&b4DD&KPue zv#uq^w>{Wdyx=AP9vakG8MIOb2lHRO>lxHTYv0}^e#o!=&#}fSAuFZ$@`(aMG5$$v zc;6yp?GRbQJ8(u#89DlAsmfh{0blG)`-o!`sYrVHN)e?ev&N#V7)Qz@@15cY&qEs` zTXe2$Vv`gdeOE^tua5M`%3bxw*TSLeO8}}kC=x4tvcj+z!3^L`lLbK%MFqcyBl^E~ zj)uNks1CleUJ!oe=F}1MJwiA%2b&_~Fwv>%*!q?IiP!DDhl9oU~E_G7c&Y@JjrY{D$Z~r-HpNaAi;o|as7zP z>dR>eff+45@Al8oAya+HnAVw0sRu|w%uv4C=be|0&3D89FVD}$>c_t>dGhly0#Wv` z6aYsgKmu!wqV; z+(Pg{bb&n{>rr!PXU&NcXgf8hZID%Nk@m~?H8;Up4cQl&&`1_|uU?SY6h9=d=XJJM zgMUxb$u|%+^lK!Nu0|qZwxl+hWA;m*OkCujUj)fQ7d8j&gFS}y?b66u<&jt>{g;n4 zA2g@xN!nOX^$1)+3Ap;#?%VaI7PdvoUnpi-em#-zi>GTy*kEa z+%+MBjkaJlqF0We$``FR3W=ZN*?l%s_edHZ8B1kVR}*tRF_lg#C3`IcXF{ zBdeA!T?gyeLWbKQmm7p?OXp*wJvLCZbj49uZP}_M_KhF*i+CP%f4%UBm@*0bOplld zj5=8E9x@y*fNx=#Zv#6R9P;~$q5W1U@eCW)-V&!j{q}nLLTrvM6cirj27h%mUB;my zJM?5G+0e+*8=wIgc*Lh&Qj3HB3yIJbLfXlw5Hip-+;*XPUsc7fr#gx^ z=2c^qO{L}1_R$1+FWs;R++79c4+RoeoI9x%MwB4Mrb%d90PXE?LFD zau0#dRMmPOYnO+vRbIGk#zq1b2@Xy`_!ewZbn@Uh&=vNtefjGI;yf}idN7~paU{Fq zqNYCdOJ%l^?bev~0mpMUWJF_%~DEu*@ z{#F=q5$p=OvnWx>va6@I)0)HT4B5pA73{Fj)J_iJa}g-PQ-M)XSXf^BIp8(frVW1n zGipU}j10ubCDr86!5z~|fdaK2{B_-2DXg(dwkUw>mYRM&s3<=J>{m`??j6QpCy($7 zYAs41PTd{X+Rm{8rtx+)B{ffE((HB60&*GQbElil*>b3G)GeSD8{ z3O2D!BjO@rWN|B#W1(`)7k}W zF7HmQ6|Q9NI91MJ>RlPo!AC8J#&E6v($J_ryrACAQHSdqWN;;WNv4m#%Cqw3+AKAF zQ#!+Fw=dsu`=rM3_RL2kY)!Ku!NpE${q=V*=4nFXwTYv(1)jdiG0X{JbvZZ7PJ(?fQ>^g#=hom2K31b$ZI`pw}4o!gn!iWD+!>ie>4SH!RL(E`D-mn;2zOgCjwD z(+AhDu>=4Q7%nt8hM|$rF$LPOf~9GKMqmwPrG$h*pK#jwJkf?1s<4t=&#=O*2hf10 z+mu)s$kQUGPJ(nkna#;c5y7ahUwIf%79((_7X5p-=o5DPvSefjjZgIf%_PwUN$V6U`gHCYK&$5^ zx@MUYfHtlGCd=*bIC|ejJY4J1(iB_`?wON~W6fpL5}1FWSfsFq!7JumOGqb*J6%#; z$^BOqX2_M&;XzXo5lY~*n^mW>BcbOL<<`pRrUL`nDd#esvM2GZ4fPJ3PUk#dpk2HP zEcq=QMgv*WmCIFOM?fH=FHGxI-S23k@6`6J8LybEt4*U58xf}&!Sz`AgIvo}*JWsA z@RRCLV|oyeL&BMf#PLIjgF(!3H>VkJakQ>W*q?$$u^eD_s>B%%|y8!;dM zkLL#&nZ#}Gu*%`Zk|uS!y!dh}=Z<9P<*wpb70d#Be=ire>5JpWI2!;T-g7+eygnNA zEa-pw2;A-h)=!9^er%I4EEISq^y2SP?IuUQ6yl!BVeTCZL%wf^x8iZnK_k$>>F3u0 z%$G-_ADTA-Fo#EnZi9&x8pFa;+g7I7c0hGNp|*zjZY(;4`RK4oW^JS(b~S#e*kcR+ zRfKZlAQ&9pG|KDgL+gSG;r4f2A9cyR#v*TLljQ`s?N#gf;$8tee<9B=rFa}847wul zxMAHw3IX+}*y-r0psa)la887nwo}=`z|n=qK5XYtKEvGluJvNo>8@oK< zW$UNBF;C!^%Hz1MU}4GL*6htXg3II6vj$66k^L4C(I3%-fk{$fwn?eEv-tU&aTe%N z*B8PIwWB2J!Vq8mc8eOH>s8W+x=~oF<0AF_lT%Y@a`|6@&X@4?U5IMroCnCibFIg zmBcoVH`4fVkZ5F&<-!6A0|!wmXcw;%xPTu9EIFS4aywqaMOI3rlst}*K6`>-P49Xy zZyIN3$2RiTDeMWrfXQ!(`QKR7?QARZ(2t$dXLyEFnH-6dZ)cZWtO8YR_+7w^@0-ko zCLyB&nLYl{rAH0lZEY^0p>?bAWnf}b-3&S*U^tVCp#eu{o8}g`+wtYa8JZs8pnf$T zG{P@_ep>t!?v3mhilx;tha-@$V8}{!qoa^1%?S)ZWAg%<7SZ^>Xl#T5ps@iZHM@)< zOeR<7v>Q9ym;#gH*EsMjbN|y{Lc>gLqZw4>KT0@POw@LOL_b;v6b1NC2qQy(eUo6x zlU&rG>*p75TC_1dYRm4sfT`0+o=j=EewidlsS3%SUtk=nlekzOS`-_se3o1{9W30k zeMy+@^3CW~iuuIE(7E?ZbHr+;=RRbew}IhiP!?myhXOo zmeX*>f<*yed~43+jM74scGv;lvdJ%EY#Ok5!}_~zG0*@pbc_ki=OM-WvySkPcN*B> zv48q}14mV8v00k**|$4cEaIiX;9UdHtCx$%Y2$^V6)BZYm4lR9kZi+*bq$lWo)*Ia zot%zKLfbGupou%E9wecvjtr-t^aNqnCyYbNOYetECMPX= zQ7u{9MIkI>DWg-HMy=-9YRr9(Yd*`R%N;X0W zQA^IukABi!KXYO`4ES*jX95HlK>!#xjhBqx7v|8C|c68Qn>@sU^voulWWoeOF8#s9C$$i-jK082#R&OKss5eZz?FL^v~ z=|a!v@VQ(TRE}*X9Af{za zvLmud51$|Mc6`ZN2=nJh{5FLI?K?f89U=m5k_(WNvXm}-A=dbM_~@isKbNO(|%6H|EmZ%svAiUViw=$)i#(j!OXFj5I~KJ zH-m31Vv|r}?qTw3He_VUVi_(>5P_CveFV2Y3565?c+lLA5?fDC0642mp=9%%c(EvO zw^fd`wD2#l=o5dX9U5!E_&=^H=+i*dRYfvWEt2rsGM#&`e*n^kQAS41ZBpHBB#juv zbyw_ei=?we1a*p_iY@uA+aGWEAI6={&a$c&bI}39lt}3Bc?oonmSej<365H}JYMu%?VKnq&5w=uOxprM@iM`*1pcQ(dqJdfMHqi6ecIk(F~8onFm~50 zmr60DUN{D$(LW^&eCjXfmESkAqw#SX#YJ#+%1?sh$snZST{aqgMDBUq)<`qf|BWzt zg+(YqT$vk;#)@=a(K!Q^4;R~dLJb8=wUA9Fs(f+DL`O@4 z+g@fTr;LRi36Ctk1sk?E2J`lBs6mk^dJeo^H4nA;|4I)v<0#mFFjO4u3~@Y~00MXiFHX)z3XRlf7o0JN?m}Ydiu7~U4^DUtmj+2Ye;!rre z;eXCf)BCHJyQks(9`8J8qCVL@Zr`Qp7^=5CpeH^X;~`*)^{SZ$N~%xev(ChSeq@*j zItv|DJJ^i%hmKJFH{|+f-4L|_ajnFHqf>I*es3Luo7hO^26jWhCB0yvT}he)Zlq)& zsJV}<6>91R#xNOaAX!#iVKdZo7dgcI(VFsI5mbI5nNzhkz}V`!0Q(LQ@Xz{i-E-u- zoYVbhVMcfOZ;75C4o%DOZ4a~X&6WFebEr|kLnCB}M0z!9gZoRD@k`3QKl%{iHeac! z>Z34EG>neY_n|cI!)c9AK){%&X3-zju_FCihw!B@BRb+i72;qCmig>(qwJDd{mI#3 zt*CeqQ5N(1jH`99ib@jAK#l-SOURec40yE266^Vvne=V2Q>u=Z7}s;?^D6)r@KO9- zYgk%;xwQxEd?)Y2vcf6eNTqmzPBlU0#o_Z}%ZbM@uyjW-bg*Lg1AMsTZ8F>?!R>Gi z5!JOkUY0oNIAQdtN~>@Na53Z{K;3cpyn6FAq~^jgcyPD?OO^H+4>@c1bnP#@!Y&vk zXyS_|3YKhpS}rh-$ZW?wCT(X(`tzjeXrfxKX6B3C*AAP`&FA|V!7uXJ-MBNLd=`>2 zd6pJz-4&5->XiBW26bOw>L1?uUTw_O3v9&u1=j|OQ>LKUZ5^P0xt@JCk`}lJu9PGd z=)dfCBy=UYlgK1Iupt8wf0w8D(SW5oZ) z(+3AiCAs3$E*IaUFnFT;zz$J+7X)=AnpTRq&Cupt^z^m<`OWfM{B7QX)dk*gN41t; z&4R$kSxi^|ZNF)wYF!gMO>JG>0^!ziU)+)Z0A@-huV}Qm>owboe}a!5iARThE;Z8< z<8KS5)*65F>0~ZQ&=4R^W}2!F8I&Uc|H*jY7NhSUAK!k*sKaWiJ8EO1yaTNKKKX2~ zsiZ_%?YM!pZv(ZflP)$cT7Zu1RV>Y4dY^?>tP8yTx5&2sr^ptDP+z|;nxW{fNGOm8^UGN5`@h{|3LUwnZy zU8XFc%6GU$`^9Ra6+9(J^S!TNwc{{eYgeu!pw2YGe{l5e=fNbL@2F$tElPC4k8NC4 zvCCSswf^^s56fJamp;ye`BK`PwvD4nW^nr}k{qWV0wZT7){VyyJa-q(pjX+zt!$lb z^c%2@Z;~z^Bm=bs0Ag)T0eZ2;#Q?`%sDTZ!zWXVNHmc)v*%FE@{ z8bX-Je&1!ztCa=srQN~SC+ww;DhDz;Sm-1GT4cVQ+O(^g_fy-i4{Gmal}qUq9u`#E z&fihx80Ro*Wy$;QuI_TMYKa_UB|}$|MGTVQh*ULA`kn=)Vw)xIDFcPgFJZ%l^vG?} z!{!P<_Clhke&sobT$SQoql@ta*7+Y#?#_;`DrO&+6kyuvzJH&Wz_HTBDsks#APq`e z;}pJ8lK@IFn7821NL36Cw^rTQ|91%niBSKP;3=9h0-qNzZaC9?3Lu4;92jky1?#^A zKj|U;tvCU}hQ=$y`OsJ%_Ukzd^nTkN@m73Wy+6EFLJFfwx_KCPxQr^y#~3tuoqo8% z*Y`3Fo`+7hi^L|&SzTB2|7rLj#jp9vr#bL@PhU#j2_^dW)MH^AAGSak`WlmrbU3+l zznExArh;DXkua~nQQkxx(199Oz1#w7=`o`df+~$Waz4MtBrq~YmMQ7*Nj%DF^apMy z-ZVJ|0Hwz>S-mm~I_^;5qtiQQqxaz-e0&!0VcR+kzt0+vPACE^U&15Q(c%MD-4mWe zmm&0X5*Tw;uKZidTq)|`9;|cQ@E@Ms1XuWve=m z#;Xf9{V7#kMBKeaH8nM9hK5SeL>$(v%`Md1BWMC21r(e1=;qfXT2)mF)q;e=6k8(O zsF57`8PO58_peDF>%)&Cm=4O>Go|rbY(spo&FYQpQ7M)&S`{#Ck(C}lZTxP>m#iu$3umFt2Z)Zk);|t?sQ4@$D`fljQLz_x4Qu~O!DtW|y z$h-^ah??~@AJdL=+&t%z(aeK*^P*%hfBs}QLCv2+?c*w~@LOJLJjU$N`O6%875`0NNytlD=i8Z1t??V?V1 z{eo{gwd^|_?NqPa><%6w!x@cUF~;t;-YeI37lVN;l+N}nUe-n?5a3B#Da>B-0kp|B z9YgB2(GmdVLUx~R%J|@VoF2(DTJw%&%k-q;u|GcJl1*lr2oC}1=y(_tTV?-*{Hp~{ z_^Y;<1P#odDB7Kum|s&Brl#u?(dG9Aa!Cb>k&~5$tqvnoVcSOUiCL%C$_Bo@8nVug zeKWz@YxQpNzK0coxGYGEog35@EPJ*MmeRw~ns-LaS+VQf0( zvnP*!`8MfD=VM62@An@bfHvimRpAj@)c%w^zAO%HO`(c+Kq0@s7 zsmMASHa6>a@abz|p?lN!y9J;m4{krQ0y}yWMFzS(mM`DLf==rKFWt1`r>)@`Yl{Pa zo%SAK09iLPuh!gw3%KT@cns@8-f#r(I)Re|#p{JQy_3B2>5ed0f!?;wMUkGs`3GOk zo(U&22!vN_MV9`V6oryX+ZJ2HHGe>#De zA-!Qq@a4c4F-+00lPHB4)UTe^>GOpKe2^u`xHEj_6iFTsxSQ&7-WtQk<`IEUCC(t9 zwEZ|97)!d(yE0#HBj_HD^+U^Hc~F2IUjRBjF&9g}XZFj0G~%8)?rEzidb*&X-f;I{ zSBN4&;Vy1C3Aeo@_$Hb9o6-j}APJVW&&XQCroK*)%&OY5c$sBYm+2?6wZV#buF5WLkrtE7?mJ8Dr&JjD{$YVq>DxIPfNW z+9Ry%+G22dw+(3?hJi2KGVfGiQv95q>Q98O>$ri%!pqZl)t8HuWdC*%i`~D;OUHa3 ze%(p7y4wY|E--krva-_d9ola!Y@wQwo8$nOH?s1%gXQABG7fsRaF6-D$-*QQ>COZe zuF-MZtUFyr%ykPDiEn|Fex&dj!`1xjp>Nk?{zsrHHs3@kZJxP{OBEy1pxkE;gi?&$ z3upAx%lpXvVcDN|_npnLO>p)`MbCaQr=ETj+Ze%u=qn#bhMpMxwip0j-4j9BBf*?B zF!&6ZqLSGNVD<>-(Q0{VypNCvC>i)A$t(B^s$lx;KtY^K!s~zzHRr(69q7u2UX@D; z`0MwM(3*Ir>3dYuyi7ibHHN(t5h>T$Lo06A5m9<|m%%-2m-gUDD;3r6XYamry1AJ| zyo0WF4Z?jTq}`not@`ioKHQQ(ye4Y{3rvb8ozCKqw%l_|-~Hl6-QO1^?9WaE*$M%D z{z9ip)E9@OfK>U{rEiX1y&FoU<~oVz+*xBFYxic-QFZV>eQZo2&kb1`M_3Q#m;F3M zXGd|wq%^mq{SLL^CpU)5BO(s-FF-q=TPMV7+w0z&yuvnZMqE>pi0k{Y1I(0zx@a4H z?)0E{snT0nXa~B3p|GH$Ey|S;shCZxWv=N%p#vHS zHCR!LdtSWu44C{S{>S8BIsTt*J@#ZKMeJ*u-H=s%RE@H?0-~{<>>GuD^MluuK0Po?I9^CNCF`N zUB=&@Ku!B&;)Xo(c0aY(oK~dGXVzXE_xDf&GUae4+sbc{iE#OHDGx(%9~*P;I`e5| zo6A>WAKDxG)<5u+!DoS3NQ6A2L9HziHX0inD}h0R{QNLqbt_+bk{obX?+!QJ(>jyg zSNlgHu)FcO<>2ODWqoo*%4uVxbba?fES?+&%3)C#Vy?QqoguFvnPe0pQ?wVsr2Ogr zQA{D9>m9LQKe*^JO*`J@0XN0Ahmhl}JhDLWvyj;FFT$GTIm9G~aJwjWP-)QCI{&hpc61SoIQ&%wJeWH>;RbQs4 ztoESed``fZ`6ohhds*}#!;EkEL zv07C{b$4rL0%57>Sopp?J+$bz)B3T@SV!E~Du=IbOt+)o2lMr+x3!#p?3sOL2$Aa1YN>uVl zpDxckF;6MqHTW8PieK3ExB>F+mP-Rg;!0S5?3LBIZ074dJbZJQj642&xOnZGi2g!!R4Qb=)8uKSC@7^^S;Bp%w2~jcEK+qi8icV(ajR1Br)#ob z;4E;@R8=j82lE5dM4X{ zC08qlWywt!SEzD(8)Vl4CZq}iMBZYH5$3$YBLT9%!kh;2^%$Lj0;XE04@79y=TAbm zrYM|SefEzF$znIO5Ok{6TGrWzuQsjqTP)mJ7;#;a7Jjz?JkSE|P8Y2SKkx9%+PDc9 z(UH)PAV!=fLtK@C=ULke##Y0cG3GtbN2Db2SHWyFxx8TlT{A@-Kg$!&QA5S5=fjB2 z0lJiL@|0g(cBGKM_@MP4^hb(hDDfWMTAZig^?7YMLdsVc;jG~^^G;%(f)+Evgq4hL zc5Bg_Vvl3pRtWK}0IUZFC#G(Go3)(oF>&&gz#CJF{qos9z~4iIvBqg0oYQu&+zNai zBdLE6sRcAxY~-XOzBq*!<#T_t>(4BA-NbI#~&cmL@**-W2U$EIq1>!n|40pIW6S1#yDr1 z(O$f3e`G|h^|)^8eRXmq^vUbTpu`MDuy}VMXDweo-HM-!1?{CDae)XY?EVL$9GkX~ zhldXJ?9P4N;C7dy2<+xs(pJsJG0d!eVm7aFH8mvM{^2w3&4HiZrcwh_TtN+BlbK%r zz9h>euuJoTexIH`0PI8M?yzT(h;zW^eKzz$oV}$U)tLs3rtC+jX~C@kJMXY8msQh! zA!GQ^YS^K_uwNE8u8IlsEVcfqE~-Z@LN|l;-U`qY5nhcd&K0jC0ot}ZQTJ!|bQXf7 zAGUk;Cb&jF(+$~BNX@o-p}N)K75i9nst+RKE!xNbI)8LE3*ICl&`SXNboypktK)Rm zwO{%xh6fOCvpHnQN66i-E%+;Z{_`;bUPfa6@jsK(U0{H>Y|ePGF~!=Np^0!+Bb;&V ziR)yQ-3nt~#bRijxT6EvrRq$Zd6K^3(62fniLn1f-SPYm`Z;sFL;b6Ow-0sD=< zUVXc{-H!kh(j8N8{hqr(TZ{?WfHQ8Mm~Hh>2fGM3E7k`tG*4KmcxhbCy2S4+2i+0= zF_ZydN^Ku}4xY3PS%J_<-JTfQ_kF`9S@0M8=X>9zZ7?v zsWa(XO%nReGWW^aX}n+w=={m=98~qOkNZdc!0kd*MST0wk<^X+&{sjSM~bY%LfSh34Vo~*);F)qaeaHR=c2nJSM zNycEK0(YVSO7QW#W2 z;(ZG@IS2Tzrqq-x6X@6RN%}}9@f%%r7`8U5UQFJcPegTzn;#FRO7vjc>?=#4YAb?0 zz*VxOR4A5FSkrP(e0Fg<6|9q+?zv>HpTrYX0NX z>d2Sw9$o7WP7<7`zr^^D+yAjgl#6L}BN{Z0uxA$T6Oz6bgzsmZMZ&RMb~fGb8&$c> zFSWaO#K=Dq8gMfgbj3Z~5Ya-dl@QV9K(v#p7}h>O(<-XLh}t#TuLQM%Kh;$F%;6U#XU$<8 z!CRXgTM7x+k8aG|Lb^vw5P#vH+Rm%OoUU}w8QAtkY%n6+fN_^4aSfY&wj?dyfPgXq zQp={DP+LX17aza4|L$dc$O1Hh2nqO(KbBJ*HedUG@o3ajOb*V-2AEnBpZ4R+6%1F& zYk-LF7TB^;6@A=!f`Lmaqm}me{$Ls3ET2mbJ!IaP(jNVx-+ZCt;Sp=UGAwp?E?|b2 zVuF8aSdbopG&U5lDBK3wNisYhl-@^HBd&gH2gwDE{)rwE^{A|U%4$#hWg4bzN-bT~ z3??VaIH)_aGed4|E+1j?P*kgM=ajTU_3Lo4)s#ekr5z#41$v~$n_ENP6Ej{J`h-+l z;{E=#aZbbSiv|)>g4XnHL=Io?#9_yq+)h#Sr+059tYezm=h*+bjkINLEy;2e9?X%~ z8)_`G5D1g|!YvIV6vQ&wsYDVKpLOkSlRTq!Kn-HE05fkN1XN@=>5fh$t!&i%4`4cQ zgrx#UAQZX<@?!p{mG;{5UmX`LHPG-bd7dvsKm*j~Rq=JPDc*}rt(i~RpCyMt^X}U~ z@|38Io{rI)S8a+Lco)UL6#nT(VYl4-i&}&Q$RS(Ca@@rrkB}`fi27}bFw_!&vVh9*cKw;joY25Py~kVj!kMHS z09p#`GXC+L3v5PjG+y;@pc&X3{mj1BE&V^eUyJ0dpb3ro$#4$ z<->Ss%L@}Iho)E3PQ~+!4yBr-u7VCz*tHz;z!NBFS;Akup-*UUPX%P$BX(6^vY4uJ z7&rg-m+=`IuSO{rlZM)|O%*4yEeeWHm-qt~8CC0_1EPYqV*{&O9Bh?n`5@+w~N;Xm0@>q?AI-3q7w^GhGg# zsV9jmV+4>nnZo$#!xcoLk?eUF^@pkm8-#w75RL%Ik8AGw^-H*_>41tv_~c?ro>XH3 zLx)fDPB0NQU^As(n{5@;vaaQ67CY<>pR*VHn`eL&piz|713{k~db;{wf0G96zHfQw z8@QKh+)-bEgS=VnWZYe1_W?^G+f)G_d~4+nDf0o+^YBB!tGCzK&kcMtH_SgZ7>4y}c@bWe5o71zy_RkZOS@n0(Z}-uW!71hW&Q*vi zo-TAX!<)6$b#h{}Q1iVrvt7FB$2QZ6Wu65PRoy?(9Rh^;-YCi-JI1tqHw)!?vi$9(516kj`y?}tyU^UQ3SNdq0scYf%HtyeM-;{r34_vg8 zAS#b*t)_%P_Lp<6d$rnQRY*insQYwkK*5AM6fAc>4TZ-sF(GW9s(PS|adNyNz!ca1 zv%OaTEDIh~r5~luzAy;GZ1B6fT&h>!(_r54hw|L1oEqd;wJRgVN^n1KIdzzA^E*+z|B`R#HDVx>*i9w za9aRIjsWXzcJ^^VG%0BHX2jZ2f#hcB6H)F${v3q*D0Z zjk^-wj+I&D9K#UkzX(Vfh4Pz5X8<+e;>qE0seK!#Mt*a;Ryy1-tpj|5$;>-!4y-b% zpm%^>#{9!~o!*)ENQ7^_e5h+4Zck(@OZ0ojKAt%Cmhuh`+PjS-Yh-0Kg zDDd9@KzWj%gYBn1y`nP7YXxF1hs+ph{Gp}R2_5+4Ob8te%ROUP&2=%5MAYkRZ?cy6 z%c1r~kDdfQu7`H|)4xR=;8M(MfA{-zdO*S&ih)d@^l;ltwtfRv?%!Yzzb1!kczk3T#Q6*#v4UQS7p|eVdpR$>?_9oxEhl@4N@1 zu-3=Xd<#{&s}l2rxPR$sG?dZ3CR60_m%*CLBEux(N4hH4-6OjkuHIb@EIeiJKd9Q8 zD;|ZK#a)ntodvP=rvf8L5Yr^!RM9+!;Jvms5c4$M;D6^HLR!1#Hy@o^`! zl;vxFArX5z>XCJU0)`a3qo~TE^yFF={I%7EB!_Z~`}yx$RG^-ON-S4!nl5;7883qd za);yT_vcrCprl9-)%%@jXE?2c7&+{iWCar6Ewr_D9=jfS-=j+A?eQ6-n+Q|Lc!A3& zJ<@#5E1G#wl!fk^Rb)#%pAn7(RFwx;opxdUZ#CVAhmSU<{wU`dL`WP>PZ^(=H#4%Z zyVVFKe>sJcvmjhn`r*D){U9?9$#q}KwX*4Lv*_QacOrJfj&<|r&egUN@M!dBTy3ns zHmnPvu8ollgSS0)#v|d-l^jEAEfdQZ;|xD6LH6KJv-XM-A`RZ*(28|OQ}u{cyTC0O}Y}>{@{Bf0ri)Mx6i#XiV)FtIi(q^ zwy=IX^uto`2eLzyH18)f`?>ZJK*!Ga9vx6Q#Ri}rU;)l=tl7Rh2N@Ly^PP6xQP$@h z90UtxDrxua(j+P?UWjXmu4BVAlN=Kp@1YtKPYtTnwhnTgIS-9jHcKB*_&QJYxCvf^ z5CdGak)_9q(GsrZ6&pHJXY|c8sw28IvAC7mY_q{IYss!PHqDt75YO00yd^uZxT8rNSb)KHJ?lc-Vla22+6)#6wK8&=^dT z5bvSXhbGE~hJT06^AFAM`5+=lCJri5w4J7H^`82V2;S-+bH-c$I<%uMw5t@>hkJaHn*+ne%M1>TTCZ!;TFf z3>|~Su~6Z}LevS{@J5|5hY+Cn%{&9f-FhpQisMcV)h8LVk!yXR1iCu7}@iJseBV0s5)n8au=tOdsk@Nrr zPvEna8p3Xu5TXF~ z$ji+tGE}s72FroSQNBX+dzI@u@uQ&1JY@4g>6)SZ^F_99A8odg&fQcgog(z2zQw3z znB;xAcEJ+LYdR3oTYqxW%JKa;&J!hLD6n9DGz(=zCuLTqNwfpEmp8tB?WTmPZrwT~ zBtr3kWAor~Ag|3P@1@rKP!VdSRK~-|rTwH`%?d@Diig!KKprO^py`2KThy)wk$OSP z`~Qj-KFHM=-qW6r1?7q(8%ScWiT;U71<}?ux>tDabN&h7}Rb#(25(>nn|^mzpZI zl4=mWFB*l~M;WN_=!XdS!*&o6iNf76@chj|(MOefu;%L4lYo2ibF4FJOJ2$cSMP`Y zC)AFta%cl$Pk}x5H>dI{*uMVr412cmZ#bi9sxw}%9I4~h^mpiHs*znX;8?aRexUjY z2-%@3f4A6Q99N7K?6S0h1QG?2`C#>#u!K)+&*|O+r|(C~#Z$rEhF$ zK_Cc~&u(i9FSLRG1E3Umq2}$Mc;A*nDb#~2-!Rwi_Ge1m&|plmGVG=b z{+a#hV%U$~u>W^DW9RDsozA$pPW;s?J^<5SGKHT|T6HM67zG)DB*x}A`;Bo~t6GPM z&J+Rl;cuEu%Gr}lpx{L=rFbys$qaIQSTKW~uDstJGo26~@!Mi#{F+(ig2l8{7 z9>*vqE%uJVJlr#Q4AUXIWq?TXf0L*~&p4d~$E!!ItZ*2#C5G>+!V)@9Grq; zJ&E+|!Y2KD^yMZT#@Y3lk&1AWN^PE@zpD1*$KeVJeg{gen;p^(_!y5U^O{uY} z_ZCK{;lZZKU?cnP1AgsDP@^OTPWvv@OlPUJ-u`Nmfr;Lr7W(2P>B8RigGOsSDw-jb z_OTTRyb5TA4ajmxms>shOg>gPLsMrYrB$q7AQ@iCKsXvo%%Nz~os=B$(|qCzYW(ov zh+NBY%`qD~Q>Ozy;M=8;*#RQGz$~Jk>Svg^rvsi3>2CtMe^~(1o1Y8_2d0^rT{+A| zm+W{sKxaI8)OfZ4s7|Si02kP{Ck7ItmwBjTJW&S)8I@3HvgrD=T>C!KK9m*E>*|oW zV_>-SPzm-=zWwAs3Z__b7MU;5!8g1A0|A2o=QftPoGiZb&;9hm^Zie(6cG>rytiqT zy2mRo@4h42c@XZiV^@1w#S9r)WIHKtj_bh=n!8{0^b zoO70_gh9RjLHT@I-6;Z)wGWC{+p|=viKXgt&Nn~RpfINf zNLrbMkOn*BQ)876@6myDM{ z6KF;=c}cWIyFB)1Jpy{yOB+B3LlmzcPoJK;99&&*EqL1h6FR?3>yI`7e&tzbq8Fsi zt=B!+!zF#~vsdHT4UEYH7fI+hh_t?{e~S?-d=X*jaG8cxqUmuF7UBxtiMc^s>q1~& zW|)(KEAl2fn3ut#$Uj}G<%c{n6iEfLL%%Z2l$)l=xSN?%x;d-tiK7mt=`Ai=Rm>go zh%E6LX#{kvI9ev@*g><*JZo^cjuCuTI+&8TZm)Y6eT={ZO>?JgL{}E?35bu)mkc}p zo<8>?p8w*xSW)p^KAd8tKYoVMFE7~cte=JI#pqSt`Rp#U z&)wRi;oKR)OCJeNJic&ye%Vv)D*J%dP}+cW_m+?$&5gqN=WxuADkCMP5V1GtLikdC z7k6MqhAnad+Y{V_oV&|-6URp!bvv?}Rr~kHYF(YYfsZj`_+o`+4zTq07`LVY{z4o6 zv^JT+fj{hKZCX<;DRG~p?5PN`V%??O$oBx>V;$E<5W^Wq`Pu7XFn@zGHLv*aaDfc5 z!IU(jvsAm#Pl1V0c5aZAPxk*|?5)G1Uc2^hMHDPT1Vp-X0O>|RL8MDWN|X?!rBkE@ zX&7mx8yva?6c`W&Nr$chVd!pt*NpCc-|zGMj`w~4+1q35u@Cvywbr@L^*L{xlqsA> zahet#2R0|tk5}qy*-7n8sc|Euj>BW3+uOu_?%y7n8Qpc;>=MlLowVOZz6t}1PdAL- z@SRuWb`n@dJ{D^EQ{XQ>G^4F45r*#R6nt67RDwqLI$we%lig3WmOsZI!mNM7Dh9HO zz4$BB{V^qnG@lR}ugNJj5`1Vt(t$zb0Fat}OiO7oClXrYUeO}UET-fD{*$SXK#lHu zb%Vdg`cWK!sv!ZtAUGXVuLSIv{R%7`Uadw-tv>Vl6^tD6*6%`R!YDsCfvT2aPT)kq zuYVV%s?ZfSOU&H6_l(*H zXVfDbDa_dA{@2|GeKsKdy_b`2m}xsdNS5x}gN%`+Mp{XLS=@Mup?Ia*&KPYDqu#Qr zCty#{w5q7ZY}^INsat0yp&!6d{9u#BtmU2V(0UcX&1D7PsYHngwrGuf9=q0`t5>C0 z3gX2GKRfrp_^$@{wuK(hUw}=3OvjIO3ZEehp0cp(7qU+OoqiHwjcsSqDNC;a+#u;g*>q$R~ojezk)bnUGxDtI6MAyZV ztK9m;qCY&6u0CmQ6Y|ywW|o8UfQQIrVi=>4O*^;dqI`EP_|UhmZ#1YYs5Uzx$9rQr z$96H!oAn9Vv@~~EYaanzh@mb=zgf9?&*00|M$i2tp`M@{07hM=oe3kvOdG33RwkUb zW6=)r1r1|oYeGvdPlShZs$=q4ZoI(U@i?ezSw7{&z@XNNDz)Z$w$x432Onp?tGFKm2X&sqgn7WO&~apQGCjP< z3s{tQ;;&0QPnv6|j4&YKP&sr9EkA(d`Ic0$0WoYCM`Dl#xfLgn@eDC%#Dm#|oY#_0 zN}n`XJD%}&d=@*9b@rg~Was=iU*Pp9*J_+)p~An1Od{{Ta!7vyH!=a!k^Qr1r?7Wq z0vy!!G(Z14$H3;AJ_>vwx@=2VkO1J%tNrX*L8rRMPbCXetaUo%?M)mJiOY%N@xk4^ z$=4|}W6;5;M>Ej!sNwCk#=643*=%Yh5<^LclYoZWX%&o>ta71t&2AiI5U5LUxj$Wf zm<*g`o0z|e$-*T7c>t0uNNbuq(l3Qd8B6ZkvsOCf^3! zt5U!KduRPzhx8d$8Y?h9^qgO*0m3RV4HXTZGyaD5gI;Y@rPxl(@jxyK%T;HfV49If!~GO}OUT#83^v^*a`TDaP% z-joOgnS4vm4dg3uu2WymZqz^6E9rg|ii|kk_DEwTS^N0u{#1oeti8E$RuK+Vt# z#j1n|gSo{z4EX7P#C7dsl&WQgvmHiuuLE-5Xz4Ro#{>I`n7rXB;O!%prJ6+FZ+Z-U zCt>IT)>!&p-J+6tQ3ODAe+y2MC`j}V`$<5cW}Cg!87(H=p#4r81?IajnYinKPC66Z zc)A3jDGa9WfN1U6K(rAwI$S(}D<7~+!*6c=&h+=qvEoKb*+l{PPq5{aB^7&P6f_e! zYO&YJdv!m5iB={I+ZfGj)AbxP*}H0j0N&N&_F@QfYy1sD#mirPQHGrh$(LskOO!8H znAr<_qlzZYeroHJ9l{`E-8S+a-#Owbc;3fA{i@YA5e( zOU(N&DSN{|@9CTK%}Nd9SU=sH+Mjp~%e0M3eI}ON9dh>GMCHUPFGPm-9QpBp><}Qh z8H96U(@pYdk?3H|Y9h7#f3GHvry@CW(O#H*6$IE7;8hjbkOHgHt9tLdatWI-GlfA{ zr0E__hJ_YcGc%5Z>{m~Gt11;*Yjm0F!A-nMQc??6*wQ6CyFAA2x8pj#tSUzdpCIUw z((5`*1TvAd(j~wjs&$8P>fZ%NuEE^~kfRn}>mKd|gw5YVLLPsG&4SHDj@owS0??}b z&nX=9pS{MzpOFLlDcr8-Z4>=DhLk!cK(*a!ZxHt4jI=WBS%B4mK!W8j>!4jN-iyV~ zwL)c`I_=$P3kt;x@Qv`PZEvv>Pg2q_3SK3t^3~JJec%&tXT(C4jl^u)&PNf7BA2Ja zRT}i)IXm_R`$6@cURDH(Wm+obhAcgF>Rn0eUl3ou0yus)NTaWW& zr55$7*BBq(C%DEKagwvBbW+V?nh+PqhBJ+V5^kJMGM!uwB*t@BgN}T8set_QvR>fz zu!PL7fz^kUVB@Y}ng7@%C|^qOiQ1`cngjSw_G~N|AtG{~w}D?ePWc=cGs&`s!~uSi z&i7=KZ#~V7-$Yw${J+)MPIM?5aj2}Q2T3IlY<-`)|9%;}qKwj^tcAk%zw`@uxzYv|y}$gxnzpVCiJ9#i^3F@vkE+Vh&#`fA1M z@_P2ZbdLk}KZzbY6z_z#`K5rTg(>zBp{s#4HL+E=cP4-;f!{5t=aZWqk019ZE2 zXWDEC>I|tz-MYcvrfw%}oQ=3IctGU&H~OOe%gs^`hQ28_U(&9i4X*tQ@-F04F2%`j zzzJAAzC?KP)Efuvcl0w$Tx+EZa8&{@r6t(EBJg#0Nd>nkr7H#uZpGsnW?>KZG4d~1 zwaF#ue;0#|cLJS8=b}j^`Dz{z%l{5{fARhESP@DPEWcUGFKelHT}x2#J$&WAF;mA} zg-%9$f#X1?>>@i~0ba9=Nk`jSSYkT<{H2Sf`<(_O6b}eM{1I2P1H%OvYRql~M1T~i zQ@{E-JE-?#vhggy<-{Djl*YS)^NHjAOv%vdUzX1P;rktKI%0 zm7)bz&vAN~S~4+N^fLkhB%}8LsNE5GEDKlwS_`|eaXN+=d%Ji00^$Ore7NQzkQ|vu z_?Wiea;5a}oQZ;3>~lBQsq^DExiJNd@7ZBhLI zdP_KVD)FN3cLOL;U;y5+*k=| z99UJtPR?4Ux_c2W?SESShU*fn>_>hE-8Sdcay2BgiY&r*T*GJSF!_Kk2+8IW9SLUcdn zOI&jU6}>*{=8S2OmaI_#SAM=8a3{Y(7suXuph(20A^j+xE>PzW;w?@=DvEQ)hJ;Cl zpSJTAohkQIUcd>m*2VFm<-|Q{4SB7J+Cp`kPJe9yJTHoztEs$l)QPEG@d`LRNX-#T zxw^san>%C5LOjFKQ+xk9CGHh3F7AsN%STc|Rv$Nc37-5XK=R3lT0jAQJW^iwxKOA# zYwc00*WQu^%wi<%7g7ZzsL+Z)iJj;c6MZ$jFiaHiL@nMLOMki3lDBec zWY_V$ClLLcnU*?EQXlh+5HOqEzmK-B{K4TD$-NTVH#|(a*d<=~lTzQ!q-Ek>M2s zSsl6DoKl4@MSD3IBZwKyF+egm2RFSwOHf9^EGJlMPhLZ>OR)ULC6vI#O#1pPBARd< zFfhbfZo(0OsdEQ;J_~|Ch3p(LQ%xDR9CMHPE38-;{#F=*=s21*U)N!-cU?)4KG`Dd zsy8bzEo?5?MjDajjxPGyUqVQ-ab6>YN~B;PVIsd1`o0QFt%nsI)IlG}Y5&Y@P>UWD zkp&Xud-0)~+NW%BAPI(M>Y!IRYc}PYph4Q1sM#}UboEcqpgud?OxLucX*JvY-l}`K8_y-J^=PHOTS*Yw*@2|Qhwzc z`K!Nvq@8A@`1h7F7nGHae}80H2`2eOx3C#dm`z9IVUuH?e@2b_G%+ zVBYi?Gl}7up?E?9X~b35UBy&3{_0deqO9Fz4N9PgPc2T`BW)T#pmyfa{VA~by7^4g8Teu1=FcD@1-A~I~O>|x@VpC1onc#E@1(!TU>6NUiPX8{~yQR{mlE* zU=u-W(U|TTSag~`w^&#CTEtOOIdfS+PWYu9zZwHahK#xcT{vI$Cy_l)@{y|3mNn+Hd+1m)#TzrA$>r>c{PsEqVB1viIk2*WnpL_foa^KG)~hn_0Y zib-odmBWpK*5$xkx?asHV&s9XMKb2*UMVK}Gy068#E-JJtoEK3?s|^#3Ir0s6Poz7 z43hn{2@5Bv-xtz?U9Kh|Cj%z6^>5(|wlZQh#lPth|Ne*$*5Jk!!3Z`$0R z^E43K$3kX?V8+d7&U`%jRZoE(RRirA1v%cAR-^G;nZdoc4ba0M#xt`^;!M81Ip^hH zcIHeTe&bdZz}jsWag=jgfII5Stnbwg)E6GJBfDHBRP^XfX+}HG@)k^y20k2~=MY*b z(bZO>B8c5Ns#&-+hHZ@UXwMQ9IgWl7l_idQ>=$E*8HP0tUbRC!7Kuh=$x&wa>sqb{ ziKOEoVIYaBap&7#2L&?pLaz>QI-@MU#IqMYc?AWUM&T;Mmv=^)z+!2wuTM>+&3|hGL+x?kZ?ZWWu!6@X}PZ#&u7S_SVJd}x*v}ZH~xZS*JQ3?9<)y=2H)?-x= zo3J+5rfq8QvC-)l>r1F#Z_)>QJCN;#o^9*vm`+5}YVCYU%{OUSDEHCGzE<@_;t_D&Waf=G>rZxe=<%jKdzVp zbknSR0WwlfzdspxuI^stiSHcAMI>?*xg`sI*L>!Y7zgEvE7q5eQj9Ipz+0s=XV|GB zxEFB(BEMomqmy4Y3l_n3i8ih=mK+b)zJWeVgM8NN8w;jQOWSx6YCjW3&Z9~HeSlbO zJhVL7I+4q|MqjEA4U}Z|Qs!P$$B+U-{jm4%alZHNwsA8GQlcErrnk+UkiWlv5!&CZ z-I~;2AL@d3d8CE5OiqOsu@RHYM zGW0s!2RgzMl`k;q8DDT@?mbVzJz@jSu72OjP!D4=UH1ytHyx9ehX!`PewpySnx`B3 zHl(E6UGTjS+|Leg4pXHlI=q{AnZOUd$1HLMnGW|^N0#TF%#t^&>t!$~Wc*n7JwM)> z_9p}}9q;WAxIoYE1Zk$Hd~vk4)8yClazYVR~ zGRu_Er0{i^v3|ybG8xH9!cL&_&ldiIm}cxqhjpb@rO{4_ZgUTGcw5P&j*KI_CJg~z zG(*BTtWIK}dzSaOfejvh7V;VsKMH2;uTLK8@c?{|i{c>K z#{e}n(zGSVQsOiJxUrTa65ZsQ>a;#&cr!Vd*5g@Kqi_Ek(?LfdP&P^cbbkH-iJAoc zK#qzQ54-d7pknl>?{1+n90fJzK0Vrv1?NlGP2urRYtN^zk}!2)m@R3No-w!BC*~im zrrz2YeMIZE;rHX?z08P?#}$`XpVqq+hu^s;YvjGFQ@cOu`8~jT$;SmG#sGJ`1x`&U{s@ao>jEs}q0s#M>r*H&~4&MSC?snXO{)*sXFw|GT# zKkPeM?`eX?H@9FDBSS-KtDir!y7kIwZtha?V~oBN_#pbFM^od=V~_$DYmfiF=N$Mg zmnFy+%HmDC!DeKoYVvUM2efFz^_FgR^=A@xUy2H!>T)Ktb335A<=x%j^xP*IB_I>~ zwU!FzX|K@^^_UDP{6p!Uv7;DPkS3Bk*`uA}4ty9`FdZMJ+r_D5tdTy~|7O5Lg(s|S zW&2T8k$BT=E6=ir;@Wnact$(2;*Y;CXYTug7x?y!G3TIbR}Yt88Z(P`ls7h4Rr8Re ze&&6OR&I1H5U?IpZ#;0CLafCm%?Vn5=}=j<5C&hN@2^X0kf&Tn!@yK8xWn*8wQ8hfGQe{abBuiC!b5X_)V=5%9;0SLFDH&hNBT&p6%g zQdO_)vnp?&dyQNK<%pMeHtXg%xqM`f4@A1<#@^vqI#luG*}U@Q2Cl&}r71@h5D+d3>!5XwiFcEwvIn>JRi!_WhU!^RO7 zg%acMc&UA>fD*Pcs;!0jc9em%<3fx%6K}W8H#^PVXkD8Ba@KS@4oef6!y>NY%J5V6a6Ei03g zF&X7&Kl`$jiW8z=t$e|8zMOGuc}w?lvojhA*_1pwJX#(x`yR$QAGXYsafx>}2Bt?t zENAl2AugEoMP7Etf&a0L_`K|CnH)hKDOj&CA;)p4WC{1VUC957G@99x0=8GJD^Q)1r#kZB@v}JMgH- z0ZY7s&(T177l1-&OGyO7#6j`2W?!+dtg?kWLt?5Bz&C=coVuQ%ZyODQYzoC_d=5-d zR|mQ?7cilL_;zbWuA>!}is^yd5vRv$O;JI=!`IG|rr{Epw&|ii@mm|~B;~d5D&1Y* zcgHK%HBgxrN<2(U$bMI^K$EQ`wt7qV+JhWzK4zis-#Lv7(jdDn0aCx^seySp6WX0; zZkQm4yLIDYly;JUWya)lX|IE_QU#@(VQ@-xXBbc<^jvwfVLom*d-A%w_jal1%fwvQ zxmLxip^f7EUA<~91!2#NQa5}=y-w^cM~f_R9!K3XJ{b$!nn@sxwelGhNlt*SZuNai z+8f#PiVLnz`o@e&NxlI3Qt5xI>Kwy6r!{MP==njZ+=5mjDX(@~i;xpvQo&l&a5&9 zwL1hKvk%wq!UsKbAGgWhjL$th_Hjucq&h$Po}Pz#Z6#d~l$IXokt-1=7^&V0=rBdJ z$Sn-0NvG23d!R12ICjOc<4ic-VpkcPi$`QlYIiM`V_Q{g*|%wsRWXa3DwHxbelmozu>>>UHdJK#vG3dSv_ATGyphwZ?O=QK z&c&CGA@ARTYD;yFe`I8vtoT{yZ`jZNT;ud-I3BLs^)R=|7x^bY#@@(rpqoY>@j`e; zP&4iAnBqx;#--;qb}4K&iO~A5KjOg7MtOHG+%AQlq$K^*wN|sWaOod;q*4YW&yT&H zd+4oHBL;YmULf)sIquj{0V5o?!-UZn&ULH!4rCEzVcI>Uc-rJF}M=A z$SZn7hWf{1v7?E-%+)vr8>+Tx@&LnM4v*bK>e$*+@DMd7oBZ*3lKSoCy>2~EbI|%F z)a{29${XqewzDFZLq}+P9(>lO7eBA*oi87K6Dvp;{?4=Pb;uA|;~*4I`F|}med}y9 zXdm{3&4W57Z6hRNW2Us&t*2alY>|AR`$g%g=$<#e2L8G^K5g$mmU#cSE?aNQ%8pk0 z3inwSfhutnNjbKPKd_hhOSeLeUzh+qd;SW93G|u4Y!^7sH0XJFJZi~4=FANWV76{; zVPxumP;OXux$x-Ng%Bt_25y3%W+7#FTgBnC zgk~G!w#5J@#B&}jFo2R98lW;#Z6F@5RCQ&RIk4sBmF}-CmxjWwiLH(|X_56BJ?(xI z&F-VI*v;y0+V)xepKAd#nGEqTYVHU1+<=*V9IW5B(C=Ko*9EqhAddsk5jx zsScMKvr-B*Se~RPwjWh=WEp@3@dtFC#2-aAt(Hbgo0q@B%)-o2U8K=23sqxlQG%~I zOxuEXKX(m9P!mvzOdqGf-MK&wU)Etp`@pa?)cfJFz?(U`k$H#J)se@MTApkd(%laz zkVlil&z{Rmrhy7E1|kPx|3lyGd4w>N~+D@zsAZ7owyYfqS0$8$s81PT}n;&c2 z1HZ|6XEM{i`|CKk2~}D+K<`D10!+CcIa+&|kO%(I&`7FeaYAj08*1d8U$H(Yb#XH^ ziDYZEyA-~hU8wV`GtWT?0$7_SFI-8OZj5S6tm(p6KyH**Tt#<2N zEW1Qow@xqWG#%xr?_T1OAuv2j_c!jJ8D}o`r^1CC zGi_ECh0nOsf%Z9%e}h`gfOrrDnb89bh~~3rEGX2r*+9;RXc;`18I;=)69T)nTv#X@ z&8=T)#SAhSc6aTq9c_WlxtUF6Nwd~T7|&3@b0SHHmz~&}3h>=y=6=4f84`VJ z+JbrXYm_lX@}R)Iq3)V3AjH%o8137&4Gg%zu`iROM;(*x^Hrq^FU`Kb-5#{t&;*_hBOPNInU|to2T?$T%sSFvlu2GdoY$|6*D`95 zb3f^`4fqLR7d?DOHcB$T*^dm)RFH<|rX?{irV5um@tn39NF-d9OOBe>BYB$Y<3U?V zE^}Pzs!Al}xhL@u^=6igjSEmTAAA+VogTWiz1l{o2E_nDRfJWdsl}_h$7>Pt!%sk` ze>sTWkc^D%+8RHIN|zXW>6iu(e9dfgFhbD6VWxHcmFFUrhz!gE1PRcN*tptldyz8Gqt){&@wY#~K4-@@MUI8!B}(PtH} z=zEfdb#Yq&^tdo+njc$teI>m3gP0CVdiyrpEnWiUNGj16xF6y41!=**kudHohvI>p z-GyV1$pRHl3TcOi9kEPZ$c&AbcP5vwNiEgs$tz9Cd*6`)#!cOAgw|Ro8NkT@ajf1* zl%2316GR{BWEk|&y_aZLy)qUAy&`*drWKI^g)69|~x-w&Vb6!5HzrQ=H!ONr=L8tMK7c?!d4`p$e7T+SW`)PWfhQ^&DnwRUciGqhxv!k%AOlRVA=H(U-4G+OZFyd zYeGN$z7(@oP4Jokr(ms`#qEV}30gt&l|E3#LcKQl@Er44%--ecXnYxk+FMopw7@L= zzL0y~6*q<%8eer}K9k2eaht04tgN(h;C4vzwW)Z|u*>9vaUP zq~=%$>8PvX5;LFM&hw#__rBVl-C;&0ms|!t#iN4!o2%_sU$F)>wenv25y@R8+YhP!NtLSaM~j64B;-moc6Q4U!DZ;rb- zbC+s_d;Yk7oEql;RdrBW(l9ycqnj(NP}MqZ!My`$2dZUCicKfo-E-4%uXFEk^3Od) z-P;Q_GVKOt&i0Nl>-^Bbqz6|u85;Chwl%NHX0edJ8_&BM^Q*aaJDGMyqq#DH{no{DlBI({ekx&qB`y1Whj)2GOsz$AjN#~<_ZZWA4DW|&qa%um% zhk4{9-a=Wsl|dA=QfxH$W`;|q{D=9kewWgy`TrC}%GjbPa?jS~SN+-{J-x(x%|0%$Y#cQI55n%jy!UuV_qtGVAKwi!QuzI}}9erY_s^muR;=yZxz6?Q6SVc7F&36z>j^(!g7 zXdi#T0_USCQ}X$cQvt`Pkf^B#hxL`6@Pl; zHXt_gY%v4{U&h5%WnqX~#orOB430C@V!>?nI9;BMc+yZI%aNdC59xKeSbmCqyTFSQ za(|NXLZoP;>Uy^Qfg8GG>o+ZJ4~6ow=anHK9qsZN4VrvY=}f?nUEPsdViA6q6}m@X zA=Db`Uz5ni%&1={sdVICRYZvFN3yVf8B|FcoRmyG6uheKS><$IydA`xw(#StxkAW& z0(xGD*`ehkT9dX$2}AjxLu{=fb46uMp^N^z4X51RImvIZ^&JMm#nhxe*Z?3>um=0!Yo=J%bdTtf?br7IqiG)+-mV#JkJlW zz=lS|3dCltT4&<5e*cpz1T=Y4bAd_H=L-nOgK4<`&ZkCYgVUQ@Jd+2dYO?s7pznSR zV1m74I4STs8~X{oIi%cf>d0lzlp6aS1T zDHU{9ek*d%0eT2q9*e#Sii?YTWN8WJBM2z?;K~nE2qXCtS(RL~KT^)1AbPh`J_Ri+ zEq?NB=0%GlNYEKH?NZZ+I!Bv8BcD^~3Jw$kx%6v_R6lFXg>L~oaRc2y zGp5?e3joi=66@(eB1N^F*+}WvI1CKBrH=I0DtBDd{7yk?sj+v&?L{IJ;yWx~^_h&e z9x;cDjQ7ud6;HIdSeo$Le_gMqep64quABP?G4U;Lb~T->HnDy*lJ0y{EU2J^Gp$0Y1Uu72oDhz^JZo>=F>kq z(R81S7am+RNbFE)qw`qZ-hlC>34bV(iYP)Gw`JhS=*Q)~P@8Xo&>H+`0K(O-v9A$$gFp2-Li}SQ{?pJ4 zj|PBBr)#5MW$nOZ5!)MK1^h~&Mx`;$z&NVKe0W7!zlkzZDd0{X2R`++G)BHO{>%(| z{b+(?_pLa%W7iVv>L&{e2~Jwu!#o}PivK+V9`SQIJ@hQt*YUGl#~>{Hc^k(HUa8Ia1@~#E;CI#(kx4pre*6b zYtS2R5e50Uh$!>Go?C=eqFxWkPQ9bOHeAXSD6~BNx9}4bV505V#uX2B5m&iGsR_jD zVSFHGoVRw{2u|5v8PI91B{A5HR&81nJ)-k%*pIxbCREE8nQTPnxw?J7tOfbGnCIlm zs$luyK3i`WtoGa*qssrpc&m~GNns{!N9vb>VyB(D&U~HT7F$9|uhhP<>zk}>n;a_c zyzDOU>ev08NpYR$ZNAR4oVDkR``yrv2FWUgVg+m|{5j#aISa;TVhZ_RrEsjq$CigM zGs8Rle#YLwZ~aQoz<|mUpOtRfCydsvk5l$-t-_(HBm;{H|w6=l<@eRXNwQJ%%FYU#yvZx_ClUrB}O7I-29aRx`UDiJSTFK4Omb92H7!5&l%Wo+Mom61!PBIWBLw;TD>O?pSnc zs4amf-2KR#)lCL@mWKiBIkABHavn^gJ&7z{q;3`qC^f8nk^m8*NaJ00=PTL5>KI_Wg+B@TvVNIjXn20-#**z^l@0CXAc|7E z=6=l7a5w&1I%e;l1GntT+eZ}`Z$7=VN-u!v?LZ$J?8yx7_W%1pIf-W{do3PMX&8Si25Po<+V|5O=frp-yvjdd;k`tWCbn}ZhJ`YVU=n`)xH@;iStGKQIf z$^B(K61c>*8qmAoy%!INJ>DQQm5<_?p}%DkS7q(4;CPYJwrgA@*+a4L*=Za7zHVAJJWwQny~e9@$mY#{<~v& zh?DeQ{{*4MF_@FEHS1TeIUkiOM*5)qpDqW+mP@#42vBSsCW1Pq9 zB%Ry#lo_2%>o;gy&g_mBAa+D`p*V^nSd0^(2wkSf0h3Gp*~tt71b6I97);)E*7%$z zzaMwoc@-~lvf-4-hjL-(vNmF!I$m$im3`lQepdfpWC^MU#f!;d@gle@z_f|v3>WFb z@za2YrDZ_qEZ&ug`kpB=ywZk){uY%K#l7q0UuK4}0=z!~youaR_^w35I8kqolV*N} z)da`p#uuIlcsEZY7urZK*Qq7-m2qGP@-P4+du96k!b;DbgJ(V0m+}^&dDy*xJX+_D zuf?uY9GFtCZfLaeWwpdnhk{O8-tM0V<#f;e7kL#}WDF!J&Cd_DD(rH^R-6yMI^x=s zzpZVC!T2nd{yZ1Kyxhlh%O17o(`SrU$L@WdGJMu6yq8yAa~j2>P~G07?8}?h*aqQH zwtWG+z``K=PvHu(N{nEPNWPcy*-Ot+=3QPf`GCS~v_MqX7G_QH05(|sCKhrkt5M%P zLPIWdBzK3Mb?K*~lT$1t_3(Z|z@ie7kj#a_c{n)WI%^!V<(`Hp2LKHq{|J=TR*pEJPEmc`Lbg5Gh=OEdcu>Fi}9?w3l5;WF+G+bvb84*fLZU^ zQw&KZp+M&c#t9S`Q!HRcz+<>Zv zVbb<*Dj6pF=G&DtV1wd+nVg`hB>%c<%zxwwP#4ucZ?IVAonQfM8T~Apr)%potB(Cy zaIku=wET8LF-cjz%4u<6l3u22@LLDSKcE0T*XIUjac7Y{Q&T1CDg$Z%dpGTp*L)7x z;|a=38g;={%|X?}6PTG-U+aW6HEgzz;`(XEGNsP6Z@wjHUSrWes+0?cAa1@{Fy%?f zl8>T~zHhoRT&$3hET%0ip)aNwnPh#~di{Fb{$pyPaYdI0^s2JVWkJD%-@}ZxO_owo zi7}AWNufxpLoQGh%w+Q~@y=_&IZ43+o-BxJ1FW>-g^yGjvln9^_K6$Ig&j8fOQU|MxqH&z~+YN&D+>&{UrI6=ezi zwW)eznxj*>&ntpsidEF7S`3%h{iH zFCoAfCNg8yCBX8y2J_~1ohtYT-85Zp)XM(k;OF(96TWFjhiSmUDhV;@10>&9fzw!K zzu)+u-jQlxmkM;+Siy^epF;(Y6RUQCGkyu5(a0E)uuHF>vyAMgkfk(L(LyBjR*7ZL z+eW5qUIu7W$dpQzL9j3b(f=rRMqX%Y1Hw&n^2a6dA^5 z{Db46-I3BKAhVfvNPs%m0n|-dCA1$Xg$V>yl(9i(8b%AR1XMcGR}cTJ&P?3n-P($F zY>B!aQzG|Tw>aMHdO|)(39Pj|1}C6OCm-De-SV~G+$dTj?oH{xgHuyh9AL(BZ_vn; zU#1-n2U_B{6ID>})FzcQgI5j9tExz^-&^4rnQ~~vamMWO{Ho}PQH5!sld>%VL8+vr$}YBKU~;VG&S53d<}oyj85bTE5;tJuyE zqF>dJDQnnP&pVaC!P zktWG!(K}EX0#$8gNK=qnjlX5~=@g&vL*G~q?n8?SHyR@*D3=}+t#3n5jhC>fw&&rX z_eBrjVFKY%qW*P`3>^v4h=?|G+GVa$MI(t@Fj@xYj6-NhSK-VupH zQTWOKcmW*e@90%oe`z>B^Xx*SzyZhkW>KoJ!JLdtXrGIxvm7P9Zk@2*TYn$02R_Tj zh{y65iQs1eFO@0w2zAGtaLQyt zNrzG?zsC@V({^&Q+0Qp&F-_6Mw4Wq0VijQSLxVm!fBGb&t$$1*#+G@sF2C`5zpkAi zQ=2}2+h3dHj8yZKs9<&cqxkS1ORS#W`1mewF1|G`c+nqU(xmG;htj%kVrj{|fV&Vr zC2&+3DdBMs60SSk6_6juUaVyS#F#G;eiD(}(yV?`U7O0l!r=1f1$?zWYO=o2_n34v&0x zelNZc!GD)FwQn04WqXSrKb%gLVZ1W=M~H=U&1-4`N}CIzydgkH{2Ungupku-=ZSU* zzqPga2Ve8S+tgf?tNnewRmy)p^z5Q%0FlBg8}N#=bx^cY1&GFFkDm915sORuY z)DQ%(?O87sk(-t3@7L|DdM5lxW3E%*iY{8fYYN*Hw-b6u0%=cA z^0<6_zE~U7-?I#iQ`p&hl>MNKk#r}+H(@J;W`p{7DwGR2s?W`f1a{aOWZg#w5lv3mPdslkq8;0dqfN-GLa4+ z@3KYmz8dP)S63%GY>+)JINN&E5gVrr7xu5X)rYG`OYuQZVUF+y0Hk^_VYsV+vzEeyFcoQ<0P zlxM{l{ca|1PuYEIizwKcm@pM+2{2 zoPyOs;@>&f+qX+@Uqopo^^a&H7v49@cnbRf&HtTs!aqk^{KVN%XzBb}`V{RQYBnoL zzRGj}hTK>VrodLEqJ9@7|L5#R>pu-EaGh4%nS^LuY4x1fbo2;u+E?;yDJqdlO5hfk zw5{WOJEEACCaI=)5yntLU76{UYV4iM77K~iH7w$F@O+wcEsXXBS)wafG zsGErJybRQ@=0>!&j8J#xYeb3TDK#r&9cBhjlfLH`g}i+z`T=`PeR}l=I~7HOv8RK| zt)1pyKeV#R6>@ljv&j!!pZq9@!&CfQ+F;))tDvAuB*jg&$aDo4Ji3+&_k#(goqk7^ z;!W^E$(XRFgqybqb|J}xb1A1nt9Dd}_K6NN8`n|EjLqzH!f1@unBBL^z{Ly3xJeBr z@GKD++v|O*ZF!+VOss`zo5%+xKo{Ly?)XZ7$!rjnsm<7wcP4vwta}cLPuBs_y0^mM zQiM#^z^7a2+i9$&HFx;VXjHhQ2xA8VpHv2!`>{|5RX<<4J2Sl(RcVxmFAKCXVIHV3 zPVQrYWe>~s_SfrmuMy_%1_HAT6!&}rmizzqCzlrM^E$vJ&@YPr688KqCHh|zq_6*( zUT8AwFSpq=2*!ueaBo{Vn_GmwefSTfPN(^gUmMJ$z~mw(71;24f*pE&lJ~&mgj&ow zs$Vg?zfpJp!N6x@w6Q)9r;)+u&I@tDhRGs->_NSh8f#M-YZy)z@HPssfNQc8EbeuH zc^xTB8$RECfJmrB-Ks!b(#UZ}55Ir=X64ttrjyq{jnY)G&GrMow|mdRcZtcnz#Q1m z$aeS8II2Srl=jTGcbm9MwqC8bTdaEwrSxna?722W8hd|d*X{lYw=hK~6#Hoz49=wi z^hswTmP{6<`}Yy~x41u2eU7orL!FV<4dN}}G0}*E{4}T1{MyDm`on*%;n@IEkAa9C zXTuzsrAUG@SqI3NCK`xywhYw17ua5hNp)C;DM~NCc*Zlb{lo6=k?}9%ph#Mbk2bVx z#>NtU!VKHo{jgl`MStmI!~k&oNb{A_!pZ)4XB~R*s<7!$>e8wa7cCAp2&=u=5TL4B zH!=rKPt__!TNA+prMSy5=6~#P+&Chv-x?xCX?aZl?Qtr+jZH?S)}(9BLtdQ!{A6+| z(PLek{goHrtvB;L% zcqKV`H&7<>5)qa4Z#g?J5W@k!@(1*9MR5SDfjHnCRKxQZNKuJY8Oz{OV5~gw2`BHl znO73YLI0)rJh?RKiI&vc_<uI^ z{|=K0^MISm<+Fj~60;TW*Ov!7Q>suzp%i$jI|_Ej7MIAq9hSn&X0Wgt;Eq1VpYD|* z;ReCRdVJU+A8kv`-gS+UZcHek#)@>E(>jCj$;Oq-3v1N%sM6;1!Uyh=HezdvRX zQ~sf6heK!SW8-;>a>cXC~r$AH>J}MUFq>k;9tdDnOd=Pj8)>W1Os+N z59()Y$S?}tA^TxOmpnCIB(<3A^AV${BEGDne|^cErV68hv>ANo<-xxB$HV|Mx3@l3 znqhUw9J_4@-kT1&l9a{yWN)+jahMJb`OW4>K<8DzR_-$NOmAnLA#=M9yQTsv*1b~a zm5Z4F%~xE|B>u6=0R}hLC_QxipGU^J+pR&)SE>Y+XV_Y&kf_=S^u78Vw& ztkK?b_t=+GxpMO^1GcKDbuvp6Sh@?snci|49>qA2_9T&Z&BI6FU?Ku+O2Qw~!x&$B zACW7F>#-7dvf>`?iYv5-Q}C!s3Mv8Q_2;^X^{y@!w%NYDRa)R_P*c0l2S$1QrN+Ly z`~Uc!TAjyiTs!}fu2EJU1b75g8{WP15@0dW_gCg5-Z#KNMCxL8CG4uYUiL#`v%rq* zW0SPx!xU?y=0wwO^>QJM#6BVv`$b*-Gw>9_qub%jYlYGa2$Vz;&qVU`2!ogQFD z@&9Fhsy#33w#}7z5vOn|y57eJidn7}oF&{plZo(h2{7&Me1b#v*XFnT5ZD**tX_&N zX#4g0y|tck%i{8zn_+d}=TRBjyN~d$eeY*;%E!!u99|O{^}k9t0T&2&s~cWRt<1qTVi@C%o3B*}$(6u#;@m4;7yHeR%NYo^Va6I@g05K98NJ$a0J76qiwo_q@$7zVlH?xa9$n3l21Hc#5c6VF%ck z)0K~;f#!L_W57J4TfDMCg^av9UNM#hGYEqZ7t^{`y&25=0>XyfmxbN(#{qn)p!CzH zFTfJ>$QvNo9NS+%P(`!a~WQ=+gv#roLvsQGDPYu6x7aCt-A-kH8H)AD{+ z{^<=qv40LrDI5_JUKN)8*&H)7pva6?D`RcdSr}XO$v?L0!v%`3x5-O^N1CEBS2kAo z)n@?*6u;q(IV)B(>M1|RBSfO-rtX*ZTS zHhD;Y-kNNQ9{=++(CqG7BOBb`__4xF>A#&mO1S%)2WF;+1c5NyETsLE5AM4?yY!PN z|Bssg6wQwxLU+dnJ!}|FAahhZ`~K?I$u==$6j4{o@&yr_5BBC6p$FjEz{JIIP+O9SR&cPF%6`H zp54l7Os4-ojJxkPn^Sop-Qa(KabT;q7 z$nU>{cKq+|_enJQ$_keq*DEg^(P0A$@ztR92 zW@5-Fra71+=H@ntCWqrIs@!i#ab^v!fquDCClSakfmJ^t}^n@H^jhB=7S0K`v> z90mIhn48Jf1@`evYXX8a?t281@)frORdKY^;BqTY!vTUlFchAf-=P=91cPSBWSK^< zC+#FGjKbHQDunMpgJym05KJj=ao!A!d7ZeA9LPoT+I65mMZ7`eFn-&YHho-E+U zH`Ev0#G8hcFHl4V?N1N4b%#<&(h1)m`~na0_qFKO_*Z~`RCI`a6w@5v%H<4Nlyb#B zjtTg^^_NkAc~I=;`HpL@arT=cSd+-M>nO}3)U5StPXG*9pXBadsrcGD#BRcI@%!T( zF(qpN@A9`8NksM7jVX&8M`+7Uas4kNlP#ci`Sr zC=t-Lb=krvMkMINg94uD_gwq)<;(1MWGfu7s(UyWzH?%bdqm9UsCk+#Zm4&jJP7_+ zR-{`aWkYk!1t|ghzk|wiYdFrWH=)KZx>Vv}Y=TH3g-dvoVpq0U%XNnxkx3}XabwYS zp(d&K$VLM3vXP^K*=18=fIxa8&>C|yLU}XR#(^3mkVUsb^?o_7GO1*X$paweBWfm@ zmlnB%=~vjNw`P|WDOLG}^!P7I%ghu+V7a9{%wk;T6ur|SNVu{QuYkb#`@q7V_vH0$ zTi_Ns;WY40H=Q;6OS17d67=RoixVjx2JMw@kP+%qW11q!@4QaKb2gwP&$7QZLCty2 zF5w+<(+sdLKyXH3+Jj6hU{PMTH-8+aCtY-RkLdx`XNC=L`lvpI&F}4!14>C zjbBzz0QG^t_M+5zHj$zq;du73OEy;!ra+Fd;Jh6iY6|9vD7N_gWItU+0gsR|?v0QHaLWorS5FSMS|w|;-+FxkbqhSD6Qa-q)Cx>3px>NQ9OTBm7m0KIUZOuQz%*SRd%tA z&)a|fanoeplzM1O|Dcm!=k)a19v^G!rCNYmFYgHs^dfw*yi^OQ=&w2@di=|MHWl}O z$ig-r<|=}ubV4E}46>|89>hvuR*Zs_WA7u9UpBKMkaaJzYk z<3#QMG{kONCenxX5*U9zRlhXf$wZHl{<3=hl4 z%@Xv2?h$|^=G;ysOy(=lQoM>ty~iu`#1clgBHDixFyC=`*2utXghxAD2ZLd*w9W(m z?c;Q2XB47ZVBxURYuW|KGI31mwt?AbmvZvK$d)+}_TqUh`cZuhyoMS`KV~Z?Be-4~ zft}2&1|-1o`G?Dka{VXYr{vYrQ}$BpPeeA0dhLH*Qg4Zt)mxh{fr?=&Fj}ZGpViH8 z83ekAE19nKU--)6RFwjYQ4x#whwlIDE$K3G-zv!;xjS(5XdKJ;^zPGv^RaZGk+mAY zc}#wGeLq%#wL$Vm=nZ_bLP7Nh3QKHpuH23Iy;6GbUWBA|R zcO4o!PZw`M8o5$+152M5KTNv#9ABWn5$%=n`e_Dk@hll6ldiv-nHxa~bD(wY0MD^k z2Iu8-OtG&xsPMaP6AgX zC5q0a+dx`*a_JJ33w*VJVSg6z9=Z>2)R#qcbV9Ky6h$nyA51YP^;+NVG!gdb+JG;Xh008**je1s3gKi2oy%k8hNcFxjhZ{(#bjxUKB(QYWknDK${ z+Y@V_0OzD^3PYg~E6@;J`jo9RU8J$>QG1+;9&YJv)Jx4w;w>l1q({5*G^K38GS*V> zPtOB+@|sLMvKY9WxZgL2w7XtwPQKi0h~|{cW6SzPf{}?S_O2Yvbk{|hdqaDtyAkG1 zk)edRoJJ#H(Uw2Uil5MjF0m*Vi0WVc+R~qAKazq^G5FOafJs3)T}q`lCREQcUHeAL zVCb5?(vZu7Qs@I8G)19<*6IFJbL0i%gj7KB1-iDhQt<>X!7ACuD7l8mWBn;pVYvNx@hgC(g;?ObT zr%w_#<>cC9f3@&5e3Dxf;_I!B@rp%fE!0qgYK5t8>jS$aOOe0De=xrQ19;0M74(^9 zq9^+7vKTn0hj$p*70igh2&?b5kqb9`Zbh#&UU>R&MRhDWqIrK={P~Xc+SqmmnQ3Ai4LulD6&Tsc<(uh*J!igM1Jk zVB0>B2`{&BztnI3%YclR(>zAzKmkcE8^e5RMBJ8TWj+Rv*$fTnX`VE8OXQt}%aiM3 zx)$ThScHKa#I@X)|8UF+nn`E z5`70}(!L#LDz5Tliet0m%+$S0@=>s;2af2^Ejp!SjK!0dAZ(_{_(N8PYoPN_H~w~} z_UN;bb+3#kkFMv4?YG2;fRgBcKojR92s9~*d`w?!J2qXg1L!cI6Rr^7a)p*e)P_>D z_kUTHe82O;E{VNZsjG{4PUxvJ&wAPV5db>qye|>|%3c!t&wrRaB=l-4=&W+r^&g1> zj8T^KS=M4NmM`6cW9LHlf$-ZicFt1wj*2OUg|GG*$ENrTU-tq$-X-k?@H#g3VFNTJ5gdgRay-W>il+LsnefyGM}S0YOTK`tq)282P2WON-L&0x`@~*v;EScQSjPOhh`WtXK4G_C_WCh zN^R3(+oL|%1LPMqeQ9Fc`#!2k)GS;~P|=b#Vo{tom$T8_;7Pkc zD3Jr(bYnTLEuS~e_>yF0ck%=UDI^CmH4IgC^5!$WfPQC?tj|6|1niLP(jn?syjA*+ zxGmi^nqHYJW0kDJvuB=-Ugc#Bk>+bBegWv61u(UukIn?ECfoI`pKtuFk26UO&q3S_ zSnb=O#0q;ciA{Wa`3a;i6Hfz{CZ7@`#LOL*JG$Q)#NGmT;T9}5A&uCtdidIa2cMBW zNY*Ck^>S|G>51wy&CWGljSsm_so4vh01$0VwhX;@#o(H~j@%0Qnwy1V}Uv+R@p-S-d3&4qL)XAX(u`1)W3e1mUF%o<7 zQ_Ivd%0S`C%5y#=ZSmuyALCkKlQ1oX@0NW7>+_w<&%i#)mQpkZUVnKV6M_w&@mjET zP5uqCsm>3R#$qd%x_D0BB4j5UZ6|U*P3-N|p_#?X4*2jX6)_gCm`=;EU|)7ZnIu&I z?)1}1+X-GfcFBhHY|x!IJO$GjSN6Q;ILT2QHAyh5nDO&Pf#n{@A&&Y zh#!@is%w{4C_VO;oIQEKRdPlCVb6Ag6_l}2)~-Fcuh1BmV7ni;j_AJb?zG;gWpgA_ z#XvjJ3`bqjqG~>bpV%&Iv_%>8;sR0!(0QJ|M@);cpb)&MDrrbs3|I-_wU@x8{?^AJ1JJb zshF7GQ^<}i1}He`JNCpHICMln04Rmj_`zh}I@s;bOP{6yK?u$dVCbY4De{X z9ewXw`k3$~Z^wJ^h!-4h%gM^mVHv!J%6#kZ(C$?WjvgLM>zp?`6bkF69SW}iq|6;U z2P2N9OF%9{5i=$n**+v(O0j1o!tcCF!*jmX7MLJZw)M++kK?k)A(f0rR)dNh6d`_> zwI@`kt~UB<9|@t_44DdPmbU)x^G3+(Li6$C>mD`NEX7vQ5mDC5B8{^!mF!ZU>9yj5 z?a!|iB+oQ_V?QqU_4uThhJBa-^RjUm0gVl-gm)!>Zb1d`|qbNY(qjTZdP`#<~ zpZ!iK{B+N`(VBe))tJby8pSGmvz^OCG4nItgDgi=z*W!vSip`f zuVos?oYwTn9x2&&RTzLIU-w!rkisvG1XV>Ip{R|bL#`I@>hfrsgHGJ*G4*U4yS2eJ z4)5UDG70IL2WD%Gwq}1fyAFeLMM_OA8sXqR-x2*KBF5TcM3|CMAZ%X?@t%UPCvWdX z0Fa$rc)HGhw;yB6$Lt}U{O9u0r}~E@aaGwZeq#vxh!4gf>9;OjVve4_rSQY2S1ynS z-lttCETATf3b45bF7cnQUm1LO@1siYa{sBe)$DL&)}j>0V&}vD%~PMxua4}xf?!*t zuYqdIxDLtjGw`hw`czQ{+}=K~Tzh~#kLKprjLiNAk;g$M!4d6HP5m`<0qF*|-gDyVr^02?g|-0>t+& zvRln9iRxRHWY)Ld5t(*Y?Fq`S^{d?l@*2%mqr7#*?;kY*6X$GI#aIv{_55^IWTdIw z!ma_!1c8-pjt%03#MlPU_dhUjw5M8 z@x;m#h%)u0W@AXM>io8h^{a?9?28X?SdRPV2FcZrKd4&_lqzL?q7*23QXxK7;>O7y zW}?6IBR|8(a+eB+`vV=@tt1;81$lWi7kWc1U?dZDqv>x-BjiaXS`A?{##v__I7#}w z5JPkN=-f4|Imk+1Y1}N-dx3{s@x&OgUpXTa%s9ec1JOo*c5ZiPSLJ0f*dK^AjO6xm zyveu8!~O>wU|<<6dE^0|!x3nI-V!$zbd?*Bqh1*)X>`VZ_w@dA48X)U`7Y(%%e2jp zw~#$5zZ-fcT)6}+3$!K@hpH`=Pr60cHdQ9bfU(%}|Lw1|WFG57`EDtyPh!5=m~fbi zZe#MZHZ-aBbUF5n4zuzJNwKa0K06+;inXVK-Ix3h$Ohq&MZaY9YQ1{iu;WkCs!8vZ z3RasB$=i^}(DG@nUl!-f3?+c{+HnJzgsN)>h3I(};Edt;n!TB7P>F?1TuXE!-*`kM zt&2bH&#{`H`HBj2T2^r33j1&$MJM75m^yR;LkrW52{U}5q(6og$DX6GR+z)WTUk%{ z|1r3or4_GFbes8tJiP?jgX#AUlE3AL>c&ePys~TV?P_UM7VecKn|Ws)alk{zAXfPDE7DIg`VT*5 z`uSJ^1udDve19=pT+bvwJuy~fr`vq| zb@d$sT`Z1zeRXxUGFjfLZPAf*TzCwq4DFm6Z&zAzIjIOzY%bR4K$!zoT`vJvt zC^PCTOZv!ELqJZZT>K%$@|5R=?GNhH$!DOtt9Zg|tvgJGlx13FBV;m=cN-H1VR!^$ z-izOf{24HFb4}F^g^hrN*F7B(vC6gn+~S$nU$l)32;Mq{u-5>yia78hc~%BN#&>&) z>j4Ej3n|_NTs~?EW_5xar^{*TPc#eOnStv{4&0XcR8rxg7ViHK4}%wF4gUdt>=eUW z3wB0r>SZR4#=Ag?oxBT5Y*dV&!nWUrgfC^LznUT|v_x=}*Cs-HM%0%g7WF`F@Ay>+HZ`T?KBMNWWf$C}Rc;c+Wz#T_hEXP^otU>>jZrg2;p{2+YgN#V_rO4I+xl)?ltziM)J>44i$Z_-|5j)2YFk4*9SSx ze>|1SLs;P>Aajg{l0Y-(CTF8q=Wij@TYd*}22<%mbqf^(J*s9n(4@E0<}4Gd&88kS z=S-;AQ}Yqa2-WrkBl1|Fxt23FR%3BLf=6>QfxC7Jqr!Sr-lo>#Evb-)gR-*+6d7JW zzqQpeAhiTcDm__x6AEXGK$KM~5Q8;0i*upoH zOG(9!h-|dVm@9f60vDNWGhVH1U;`oQlfvOF4-Cm_wTI!eON25uGCysMq{7RAn@bAw z02swufGEL!oLL_lg%6xIFxrl2%3Z%ZA!z<;QmvMIHGgDP!G4tIfWT(3KmtwZ39Gbn z6i-Dg`b~6T^CnNM&b~uX?l!I@rP9=Hzn-A)`sa}@NWJBbA1Jn4F1*JK2C|RSdMcNN zGjGGi?W|^(05eci*vvB~S~~5}X=6evT1LC%30*Z(^AeeWuomy(!T`*z+tLA=((mSOa!SAHSbgcHs#%aJ!rz zC54zhU|<&ke5t0_m^AcfJI%@kdq|wp!)hGD)KhO;@pOdxdiJ>%h29jo>Q-1r|Nc%mWYhsU!5Md7U-;0zPS#aClDG55qOrOo9uohAV7gF{eRRIM;zZFT-B%?r zEX+6zd{tTq!7sePCXmcQC+l;ljL)GKzeCgniC~tiYzdIEp-&q04g--1WAP(4K9o_M=~`z=BAErD(3R;WTdm%e(>#bO$sI^ z9Dp?BF)TIBm((Ey>i=t3m^G;RwIRQ;tEMz^Cu-U2%F*bQCucP0Pg5!u^F=C5Im73) z@UluC{o0G-I`wxys;!F(3o+N&{y18L>Dx8r5*~WxRCZRBRmXtqx5E(B@7w@!$8v!S zY*1`3kud3*2QWUKA9$HjbiArz2vdF$>m64@kyi~UYi9n|A?I$di`x!*J`Nu01=YBC z>JH|Jn(-0SliekCua6PXtRXkT_zaY3UvK`$W*5WboYWJw4pboZ z{>Sh>bAMxC60wBwbAD7k)Ze7*_xkMtXRt`%kpkQ}>-hFGkp>Mg8yMQ;@^a5q`2f!+-BBPkyYW76yCL^whu2VMn`rhg14h0>!y! zzxCiH2A!*I#JCR9$DGzAyD@=@$9;l}0#)*&)0pLY<775g*aFZQ>?~&bk?1?Tet!4a(`$WL^pl-S_0a+06DysKc|a_W|GoGVi7z z>IROem%V7J?jN!)#OGdSX{#PyX5RyZ=}Xr+`aIy!Cy$p2NO0E~S@ma&AP?34d`K>X zGvf8s^D611#kVQx!!dHN4k1>Z>ZLtJa}dRtuk2mnI@LN_T53?k+g_%k9Xb8Vg`wz)&j{Qb-=oZV9fF zi!_%3r|a4jN9=1HnUfWX%5RS5Raoz_dNY3GiziTrx&2`YN$ol8%8xe9&~F7P#FyrFgZCnlJW|Wtx}`) zSj9sQ*$HrKrZZ2*-6bHv9LD=jWEf)n4ZIkfpRu!g0FDP3+;pS2RfV3i`5vDtWHLnPwkMa9I52{8hJ9XPjp{gk`v=11ce z%OS3(UyzQfd{20^7JrCXVgb>vez*(AM`>fQP-DzaXs*1&4RM??tZ@j$ZaDcw?w4MPogRO%`m;@#pX!q(K z+Rx^;R0LG7u&Q*mvV~RCS=BC>b4Y^_793<;Fe*-bp|BFSg zSz_P6g!|Jf(wZy^vvkM2um*>cJKSuFPWwAWy{60lxrpHuTiV7@kH*gHyZVeJX-z3vV%Si7S4}l`YV5NWy=~)B9zD@ObruGP|Etj~sZd?C zdW^T>Il-F2uc9AOz7~IF1;S6PaLF%rL@qQMx5I5a)<(+*MVaC+9eb6Dh?xX+qzHyDGLqDM@bfYdEffMZAl-yUhEvx;`kzW ztH-6$zRagGWF^^^*bc2=!r6L-47jcO1t~;1x6mlnre0b*E=U^Ei#;45bBN{PG!R(@ zrU+jQ(L^it%k#v!t$B|Su@;tN6@{H1Q0prL5JVE&3%Bu6O=;^N~O{kzdAyfQoDmHUY~4DDM9I-iH^2UhxC zoTCxp1vY>SnC9FCA5U#~9v+LI-XDX@-_YT!%)u}QdpXWsvp1}#b)x11tGpjMp7NZL zoO9A!!V?NQ8p{%L$~px=OJ~hedU2y`<;W!A>BhH~5CW_%*R`MTn=34PBWryER~-}5 z)_B-@V?2;v1%~82bWWJF&M@-*fOt4k@B+tiq`5fJovgSH>49!F5~9&BU8-zV=JKz zTZwW_t*@)+E$4mYN{L$dhf!}y_^`}tQI6htL+4lQeZ|^rpQV3_R+0)Bm!x|3!QFRf z&OvguH#QuYUOFd}e-=u&-7f`Q4x!F<6ZaC=t=Yu4GUWwrUaEXHy-7W%Hf-@?rmxQo zM#7hz4tI_#+n4?NC$ZLB`-{D4^xnh~??B%4w~$MaDe>0Ft5nv9&hX&zQyr2on|>NX zJok4phxXH@-U2my8?j4*AJ6hHTs&rINC8+0*%bOwnOw%ciYf!?sOh2ev^P0E?IvBA zH(+~QQ4|s(lRI=S9t{^Ij>}KwUys~PTZkp)v?^1jQ42TK27_=iZ;M@kjoDQp{R7d6hbDN23HQWis7`h)HbjqcpEkyVG-)D=Vp~9~RVk*4> zwhUtNXxrFV_O=ekMyeXfT+a&HFbB?GIZCO4Y6`1;CT+MqlwO2=*h zNkAwo+*ayobhCV`;~$L!ZlruTL8;+`=l%}h9IV?Q!Lk+w53W3YJn_wrH8_2Srm58t zpD$`ipHjgt38&4d2b}n9XkKcvEmtAot=@3$ZL5(|t#nN;HH}#QzPzewA3d%G|C_9` zIP%*<7>VXQ>B|H8D$OMMJGO%8g8LhjdZAtd_Op^wmCS%r1%7AR;BZ44WhK0g>82(s zG^-M|N{z=hw#6Htl1tWBOY2JS67dmj4@o8@1z{vo;ecca+vX=EAK#3LR<8Bc1?hW^ ze=aJfv=BU>j_fL3uM@1Zsoh9f?Bel+92p~-1EO|;)mVH{5o6!|62*3goj>E}eX1Xz zPyR~m-=GAO1O}r^vUy;X^5zVwJ4U|ieUM;~;StN?Bt3A{=>l80$yOh=g|^Id1H+Q~Kc_~4#^@97ghwC;$v6g3i>xJc)J z{27F-S-fS>hx^pp@GSRYe7WeJm#TE~*mH{In+%k2HRw5|MCX?aA~qvaC(99X>l-wY zIxQ9A@aGkH_AOX9o!Z&JX4VvTUTeA>k9@qjR_4jf+y2^SJOEp`Ah@BY>AqQ*K-42K zj-mT)T=xk$9AkOZ63W>a2IPc?+@1<%lDbt(9^G4#HS`{M;udWe8%>oq_oX zKxt>8$a?`jbJ^6M19l68r&_V+E_QU?75XjB-;^m{M zG|HrX>i;&W$Y>6xJ|S0xd0^h(lC*m4ha;Y=oghC#Zh%ku<#~VYTW1&|H_l-#^m+PB zVvod;Vx?q3RVZ8Ilh=6Y@LN+CK&4+O&T#_V`9K<)=cC;lg&(dJU^;KkD4^ zuID_PjbF2-yn)|@OYnUJg^1zV{H-1GH`dDw$OCdQk{6SKDrO`P;~yE{$-t$XAWGMd zW$cqa^_liQfJ8?G?u3S{54pK~3~HSUsr3Z&g^Z3trf-LbD}$|>n^QOt=hub2FK7>a zwR{UbLU7CcEq7geW_%I6UrTvPgvcT_iZKobh(KLPcj$c3U!qg~Q%KK{e_4H* z=VQi-C2Z>#@y!9}=|@JTfyZ5*L z!x|yqZwGBBsu8!OVd>ITaLO#6HpC5>g@%71u4hSQ^Idb6IMS?1Amu2N zqPm{Qt`CU_4E-Oy9Dal>yyG-|@QK&e5-UpC=;dG0=AFzE11y|F8Z>M@cE!MTU4$KJ;RT}K=h`ak$U#SoBcL*I^!tb`yQ_M4!?iA ze=E97ikhs6xmhHOK~UO$JvgbO44$lP{pbOnJ1~sAH^lTAtZCGg$4;f;Y^KN%Xd z5W;~Q>T#wd^B+n-A0=79NBg-pKgWodZFO~D?SJ8EH~e>`@o&`Di^sytGNF%IAkHrS z{nw8e_0G*gncaOtUU?D4JK@G#^Lgx=OQD<$mHE(u#GGt>2RJR_3o3SjU8}5GEDMO^ zeQ+@0sG=@F@)9_6=wz})+%lvQbE5_L@#A>~vPM2Kpe?%Xi@*dAl${ zlB`Q)5DgfywbKg=3yhL=f$0V9(7nLvc(E^3K~-yMYhb3Yg7A z{I5S19##N69s>Ia)c^8$m^*d=IkH$PGZJP9+8?8rS(T2ni~z|kh7x8*xZP)HR{?7A zK5|E6Ytdj~F;CCYf~} zG#)g38vz@t#@?Cn$RcKmGLh3P;&g${_o1++9sWqgfduLGFVE@wo6g3YhJbyTtOW*U zxavN>%@-ziCq3qZcZzM`O0^es4teM6x??{r-JE&NNs)Adp@6(pcQm(=UZ_ zv_IrE>rIn8Pph7{W4(@RYhj^w3wJt?*tf`<2i*fbKAoa$2X<3UgXXO5yI(Z={HE7n z#Wt}v0VlSuK5VC-(KXn5Gdp}2;x4jooK+Ajd80R;@`d>I9{cNArQ@r`1Dttum~}!QYcY#@gx#WL!wHgzLPRmp9*sI2 z?=CxR&3w&K@xM0$y!Af6XJ5Q$xDA$2!95RAm7O45g_2e&?=YPRzk>Y^T^D~oz((e`tk75s*lMRcl zoIus)yVvD}ouy=TVOXZC5|zLvSB%OLkHs%^*)-qcp1si6%r6A2i&Y^|QaW+r^-V%z z&x*l?YD`0${`V)F?8e{Jg0LdPdu__-M{YJvfWTDyBey>i_-s8mBdtn?Yc^9M4xgG)0`JqUSAUWc(9kEBZmyMgUuB zMhO%nbN_ObvXM`6lkD#J^1AxLr-A}mp+&5tsXABNJ)z)eS(~yPct@ZOtOVb;$qv!O zEik^S73nF+yNU|G6Zi}$Tpg3>Fjkj3;7IV?`!}k z-9bH45OEL($mxuD6tXtPG4XndW{Et$xL@vBfo9p?nurI_20OxV9T^h}UvU->ZlByY z+4GEcd1=EoE)Dhzu-{KM>QsQOG2tOb2b@wZ#iMc!RNhaM=!lXDxMM*=P|zb=cMh+^ z&Q>NUW=N4-Ng*9nAa(_55qIuL3z9j*6d^PbXpQe82jaGO`5&S?+})VykA?!e!~(8I z-%Yb!bvYpaHH{OUrp)A|$Di*0 z`H44?qga$;;R>w)U8(@+(rD!+x`Yd%OZoN>173P+blz6gul}aRw(#o%6?pc5)E#)( z&3KuP@Isnj07aj!Y1x8d4kT@dv^y3P^H!VMLHiUjh^Dx%b_$<0E5ecX@?6#ufJ`6) zC{9H|RjH!`GYL5!b8;jZF3){QqFPDsjlnCS(lU?}{woiZ6Tf8uhWX)saI$Qei30w+ zMEY@PA%SV%JszP5_SpoLM*SY)gVn8Hk%C*nz%H%$=UTW|xvP@UPNLf_G_^cXB=v(X zO{$}hTcx&xuMIVY=ucs9h+&RGBMXWMuI!>B`R+}+RTM_9%S5Z$Ps^vR#c}_okpb#F zWDP6dq*e50WYIar^egAhXXEIJouF*UGwbqT5%LPgIK5aiz;x4;suY%^#FXoxDlYLa zpoC%cVBE+G#&|X%$}J_O=`;_YAt53#8zdsDqR-5pdmEgIl9fqGdBXS`1Eb&zC6zz3 zKXc4II=aI~4aL2QjmD}v8}-Sm<9*+z-rlIv(ghJ`s3D0&9z5PmQ6S z)Nwy+3wnVl7P9cDl@4G)oN}V((Wxlx)%66HRyTV|N7LvQOH$af$p!-$gzC zMXDz*avrK=L4d4mzivu^&4{$6Iur(lntDjaoWSkVt~`p$tLuX zN3r0umEncfo5PN@4Uaw*%_Y-B3dl?nb^L1BomX%IgtaJ^Y@)|rSXckK^nE0l>FjYG z_>?X`pxa?Zn&w(iDhSg3qGAaZF+r_Fx|)pS|+oZ4du}?P=X#l%saQ zREKXQW#a8(J$}FRu86}{@Zo1Ym}vzDxk;Bf#xKXK4GX46QLHR6sN?3cozi!IA`E5^ zdeT}&njyL$iU8~&c2;zJ0Go_MEz4Q_pz?h^NJu2ODBjDv+G}E;@MUz-8EuNMmJV=) zv8m3YHDY4&es2!@vIK1&SinOJ_P+F9oKicYW#rNRlKS)_k(@dM@cY+p4@_n}Krj@j zQJx>Z&B)4@+C9zg4b5;a({}YOI_G56qdV6LA^<-F2~86RX_UkM%pi(o-2@$$n?1Ga z!l-q^+3N0!BJE7(WAXPCQRBv77>@ouL_K06RI4%p8pNu-!a~bA$@JHV5-8_&M76xd+W)oV1w#|E2r`PDt4{xbjOt+1Dtc||^lNf+mMDv^`- zq1AJvY>I=6dyi;lBM-kJ7qNw|cW|uj3>IlAXz~R3;wwGh;)8y|7bA?1WnUz-D11+& zMiqEL;|dr z%N((bk)i()UhoCv>i9&ET!2F$FzRQ$Y`#8W&8kmLj+L!_xJ*9s?rEn*_KIQI@|hU< z3$o@Y^!aEEIkK`fVsMQ_n;%EV$4f6c3q|%cZ0XNtEI7^tq3(V=Qwc`1sX+ht3y_sY zW0bF)zcJ`KTJe1JtVnXEOmzti=+7WM)L8qjgYcp9p z05ZBk(-~mXv{B@p!fmE8Ub$}G`L62s&rf*T(?dhmnKLvi1Njs{+aFnJJDpdow~LDQ zN;HP*N0F9Q?R*UR?*4eJd2jk;sVeW$yvHLz8L}s+4%-CrL&`ucXg7q-vv?o_TtI)JA9)+|qlMT9FCY zes5z~%4#u@4=mj54V^~V)H*)Mp&YMi z%>p;}Cs6`#6u)`NS0VAQUu%cHZN{gMa8;@-xurIMwLLMRKL*H?&tE@}VwrOUE}lGv z5usjLIwxQ5xKHq{D0dd7Q-5(kx-iU>Kv%}8(mIIKUe`P`i(-l0No_0g18X4sf33lX z#Q)!GK(&glJu$=dmUYejzUkwxoO}u&U5twQ5`7vc1)s_?Cp4HTf>!q9vp<(fgs4>>M)5*ibG;m}j;2ipWu*}R?KvPrvMGe$b6Sv~G{EwHT{_f13sw#@d-n3tb7IMkj04s^qd7#%nVA zI71B{Ju-M_{AQC#T4_DPl981)uPs~!b0zY^I1M#2HZ6_p6Y+}|FSu4lY%>csl2TIW z+d!>HX1tTewd&m$vDA9v0qBESwWb-uXQ(@er#qi3?0(hXFb-iu2KSdB2B12<=67vW z{aMM`V@|F)8Y8kgwugs&-Dq9NxS5yVV>h4OuG%1ld!1CoHIz=f1K{4cl|viA8Blns zM%yQLrMiPY`7^d+l+2WzyVFU>_%(JZnb6-yG8x#fa{hbf_a((Vp%oN;M)iN4`4DjC zqgYyucW;PH&L!h0O7XKgmyPV6Q(~868F!!}o}wvMkr;ME*Z)?4kI90GT2-^nBhR)q zN`cCI6Xp8v> zf^;iNxl;X_RM1H?FH(;483=)YMNpgWRj2QGcfv`PA+^tT>trEd13V-5$}Mwd39%tz z_XSJ!EK=0mS^nI$H~sY^vmV9c{c+pnHj*G!Ik|`7BnJ*ZO-gGPbpb6=Y36{0r!*gg zBj-2AzG&vlzzqUq)pb-*Rv=_+&oGA0qf{P&JDD=_@w#50>4+iSt0RZWghy7LB}lSG zC!NJE;5q7uClJ=%lDga>@&DY)qTofhtZFre73u&}A8o_>P<3FO$4>Po_zhB1<P@ z_>HEQaFF0^K=}$oR1l;3Hh8PPp_>h@WJjy496azD*R*A8 zj%xg_UkLBpK0~ydF$ssdeXcf$EJDoWFsJ{drYw82^fu3}UIq!VWq5sM&q)4kPM5qn-VkX>boO?A?^n@TYf;*YJ9+W;+@!741xBC|xtWQH`F*`g zD_7-LZXMsGbwXl7f*)sP?=jl6$>BY!khF?_fw;zMhxW%*^SJbx1VInQ z$o1D7_xL`vhKYDBN$6I2q|L51*ygQ>l>3IDLv;lb+G!+GEQ=OJ+%wlA?hdXY4E;%fR$~Kp+>QHS#OLB`WC5W=p;iJds~9+&*6@QV?EMOP=gM zB0IqqC%EDyM{nutbCM}?W31c?yW7W%8*)Y%6)x_S*khZ~VtmDpVs6%itr>g!OE0;| zWVI;y_E#M(8}Jj~&z;U#NqkVrsh<5_X{K&`A>LPCUy@2?K*4&l%c!g-)Z37pv0=}& zlE6M1Q=CL5;8G&a=V)cBK&Qvi*LVEx-6?Zv`^&)s`xy^=B0@V_c^?pst_-Y5acQLL ztl+fn9Pg~*+AfYe`}T`aDSmy{bKic4Hf}+BKflAk!TF;vv74D zzg>15;3F!MNzsE$zbld9GbmhBS>0Hm+`#XroD{aZ3{#QH?Yi^UkW5XByhu(bEHt=<<4M^7lzkHR?OSU$`vP|>(#A^E9Yvn#qFyFWCTW`Ri z($`1jao)ESa0l*`ss+8n*=eGOq|j$OgAE?yFqL!0yXVLq@h&h_(9LkPKEp4H3aEFP z%1*K@v)i`R?`UDp#&#Da4|}D(|7F+yJq4&lg31X|&wdnBc)c98<-d5-({$)1-z$sl zTxCZiC?jRDbfvWY$bV8Dr?X?X@M~mV5seh>U>MNhb&t-UE2~E1J z(2W$7dd(R{r^_9&!#!V+YJJEmbBNn;lim{?5h~R_(;8$mcDRl23BAy{6V@VCkF&n_ zvSHNXrIro8;8>cdDKB*0w6XvW(^+^6t%R=mpkYvBX4K;#QpcmMvL01DJHCBq2dklT zqBikIXZ-tl)m5(#CJK#apANV_+e#FkWib)<}7A3O6%~mQ6LHvaENtfzDu6F5`JgAnm?I^PgZ2!4=LDg~oSSNT|)RKsz)TXg40P!s>L)>xBm)3BBJ zug%9jA|Jwx)T?X^#7d*NX?086k837BPfykwFeY~}eLM27qoeI)YHK=PxmjdThh;A% z>yz}p9JFGT3j{Ou`IR=4RBa9tmGddXmM{E&>T(zn>G9KQUxB#MfNg9-%^Gbc@A1$m&B0o^-)+B z{mn*nG^j9V24|r}->|dGks|I-V|NWyAvgo132ru9%pf_9FMdKWGp32%T)S}8JHb4i=1DbR@0 z@pTt)HIY)Nqn%h%44kLmwcH8R!=Sqb{htTlNAbG8bsHBQo7{<1iuN37v|mTIXsKp- zJ*K^sC1d&*$uJ@3Tp|)b`|K?=XisTQ+pi+)L{1lkwDq+twQe^ z`Pvs$d}(W^ld0`Y0-bR1go8*@Bv}Fh;gqH#_L243RgyVGwRKe-qr~8fD;5qU8KN zk>(~{=T*D;ZQaB-KOP-#@ud4ooLYMbW_zTkG-3@MX|=H#PZubs@oTE9&pzq;wNg0BJQ(Bic&2MAwY zy^BzhwNq2pOud7cKp*W_c7YBiQ5iRr|AX?dTL9*vX;xtU$yC1o0Q@yO3a`lbG4v>z z)A_z?i{~ptWrS&(J2_u6O`G2JfytKn-t5rqvEt?R2%SfhA3Z)4uM}TIhH9T&1LyE* zS!(RpkRpoc|3le(05!dJ{lcOsRuB-eQIsYipePE01f+NA(gGq#M|vj|L3#_lD~NO= zy%P&9p!6cW6M6|90^FT|KIc5|z4yD{WQI9s#(^aNz1Lpr*LGu+3mdIANh3*e^hAAN z>JK6k;5DF7%?QqRO3L0eaH?$DWsnXvpoakb$*4?>ZmBd+TN85o?Xt7|M+@+0n~;sAr7l?u7XJZ2 zc%MVYvuTWsjJn2Bh`Ux(f@>ko=&vM1hAq6-hI_pObqG#Q&O2wY$EWJ${=OfH8Ix&k z-Pr#p`?%YcdvLb zr3zLsv$8&1sv}|~#O;$4CwDK4%gChVt}>?qnZk_oa=wrK2q~N9u6~KeTAmf+bOX%{CdCKLf51 zJi~U2 zk@|N&yuRpljH!Xq=NJT-h}avoFE)vCXl#;t=9K*L-T(Anw*fcfOzS zhhTTac)P0=dcXvz-`v*eG5tU@3>G4>fnqOID^p#i_^Yr#j3uSK6WuX=DV~a{;agIt zVyf*W<7X*c-N-LBhM0Z$veA#2OY0NwS0=6RskHXG4?8bc7g(Pcw1ePY|tG7f{QA#KMicG6X{)2izcDYJ69Nk3#4}3MS*X;kL>nLS5z%B5DVB z5Cts9QN6BEgd?%f*~$6vy(gxUl!<%vkrL|Q0fdyqs&)wthdbM?N(`mv6YZvqp0jI= zj7aK3BIQ+#qf)7g_kO7G-*nb;9#1Yf&t(k($B??!>*kgt<2nRpnj z&0<~mFrTQYNf>tZ80MpAJc%_AYOP~Y&hj0gnk*ix$i8NM2(Ifg^kp1yq`pLgfcWrL zMG35ILs4{&Z!vFI=nZ%h+wz+@U)Db=x!qj2c<|EF$Addzd*s z)kN((MMC;gyv!d9(k6y&e@rUN+jSpQVWpT8=U-?cW*2EmoGDf%ar48!ClVj}Hj?EH zI##BD3JSZ_t7htiGH(W^&DPtUY1-S9zax~bcP}_Fs&B@$zw6*UbQa;USBK>!OhdVP z!z?ltn7#JojT%0sLJn3#JU=}&>B;e3{UYg>HBq0EXk2jHhALVc##3xI5Jl}c8TMf# zz6RQfR+V?;HhbRLYqE>gWhPs7=vE!iKfLRy7`vi$`!SrOW z1oKKWa(vY<8VgcBbCnPMdG&BvxBrnPxqiC-z$T@7A7`rinf;}jk<%=?EpBf7xmv}K zO=)NkR)*d3TLg+ZLp1hBA?l+p37s_GDrZB`H-o$;J|hiX>%McWc4BEmET}dWaHAS& z^L_)N6n|-~CaN2eOX!q1ky4OmJ!4h@U_i(u;wR>?3NE(R^J})rL-qXq$8QIvmytU) zZ1?|+?JOFi(yZe`xNYeE_OgIJj!Ag(pWd4rFbPj3{GmL{P;kYwDjuH`KW95#IKp$Y&5^ zKn>LVw2vhgs(HDtkEqL&hfTsfQo%i1R|fP z+tjl8^AmYyKZSd-=O0*W!%dkt5?H6Gxh-f{Tr;h+RS{S1{97MV$J;spmUg`td7ykb zJE!6o2Hi?e+Aq7ThRiJyX;~Y1i?pRT1_&RkO3r2Lg@ICz#lAwH7QQv zrCL(8zd7Jdh28+yw2Iq&vyL)&w-D~uz<;ocov<0`E$vH!Ad(JPx}|d^${oj))s_xB z_XS7B=1&AP`_JyVj&!yx^3LW)4;L(UW|uv;9n@PxC~04f2aQCv?qNbSbS&p7dyDm+ zR)&A_k{wvdW6=%-dQxa|*>1N7)i?qKoUlpLRcZs)1O6oFUom3yDhR1Y2SI$72;T~B z#UG#tf&ln;R*PFAS;Y#J6opeYMg{kvvSosFn(on^`tKRhI`q`hlHEwfI`C+9t=)NA z+FTy2oUlE>!M~GY#x$g2liC%iG|*aBF%>T}QnVi^@eFH0!K2n^3b@1~$F(;V02?&| zciE3y*W8Gg5R$&D>!~L%Px=fo$bIM2>x`00Fu@mfD!>77Dsa8AY5{$zS@aIM?z6{;j;LV9tN-i(wntGqwz6`cF= ztM7#JY2kZ%rVbaAn8NEFw4~v>7&L&2PW!=EM=f(EIn9|K=$wp7KyK`Pq!6%03U5id z(VqG}K#Z6)i6jaq=cruZyz$Jbh*k*$)h;zpYdZ`pLdV6XBXphP)Qvsu3%q0_i;?>_$@2!%gg=u)K=og17Rew_^kw7|n z#5tXVZg6?!zPh**RuXTPzLCE~$r`4tTW2POit5-LP-dMzhzacDhCIQ08wiRp*JOsaIqHeZWTen)z zG`NwrRWB>=y@M%fLcrwEqtm(1*u8T**)Zx59m!|D_rpx-MWKNE1M{EUProd?|EDB2 zMuk<8yPUx_Xe%T*-^vuaY?b6ci|g0)bNyzEYt`T7w#uNV6)OOC_79*LtE+8;ICX@a zF%f)%J>pBdt!1~UuarLQ*QEb31VqtB-fASIVdP!y&Xe|s6S6R_Cb%8!5O7B1xbKVy zaCB_+YRkcY(zEJwj`{gW>s^_dZ`@kKLXiXDAc{Al5$mbHiAN+e4HwTm36Q0 zSEnRY_q)Q~Z=Ec!yQA|_J;WTzi)KEmdtX^dvSEehU!ZHtF=dUBuh2hZ=@{R3Kg{L_ zW@(w<@5E?am5Rb6Sz=w-&h>}c)JqCXEsee8Bk zb3JQj5Gi`Mk28YaJa?CQkydr3sQ-5MkWRv+{U@ynHy<*=+zBgQaHqHm=F$*X&pw$k z%@lHxMFaEuW37x)!)w8k$r>iPBx!Ine8JFm#lGSYAXJgry9K`qou?MO0q5rxWL`eT z_hgID2Zp627}|q^4qvQp1J^-b>RCiPbZg`IWKeMOC^3yKsL!uRE|SMrPol}3HaS|K zn!9w{I&_1w1Ex>7z30E`ij%Eu<#o`J<63E>gz!16n6@38{TOK`)NP&ZbM6J2YkbfVyzv=(^~YW9vN&=&kaVuX3)U2f@9M7#3@JD z@?uH7yMT8eF6Qr<_utw2$iXXwQpQvQ*~_FYY}oGElc&X+_caOP#;;}Dtz7$Q0X);w zXPZ6}nlfrMnfyq*l%Y|YN6TX^B9$kJ3h68!`ABHS3UJkMwq8~QO|;i6Q?d(Zps7)H ze>~Elx?qs_Nu^0GLp$HPzbz|xulps1Z#8GbX&i^jia6k=C}n|Mk^$5Dn1Mw5VaIX0 zgT&fj{XHz&YN(j(4RzEkV7+)bD;C1{CJL2%xyFl4x<9YAJj*`=8+KWf*xr(Qk?WmR4x?g9$%XL5|sscx#&fu@C?T zR(feApmprTK8~o^yADQW3mCT#4?rY|)>ARNV-XR#?G@U3X4=b16qS~Qd^8aB9+ zx;TpEl!huc6Efw|H$?VzbatN7lVH3jl4W{bvP84oih+vT;%kX6z1(y7$0Sf8J~Tza zi)tg;cXl=NsF_hOvLe~^MYy}8wBmtLmApQW8;N zmV5cSQL>5`z{i7|KPhwrh|-}^>s2?2GH0nV0JHU^qQvCYNQ3FBM;NU}KMH{A6A4O$if^u8$1{IqW1(x~47UK^Pq1%6Y>h+6X2hm8=oA!y3@BHu z=(Cg7oS%ms$B@j;dQ%H}^`__&P10Q8>BF7&@|S}A*tm?lXc^Bnu@(V5OZuTf%N6B& z{9IAyR6Fv=A81_%=9EOS_#jo`ksD&vP^sRpR6U`qa~wzSUay<8%6Il*)qmG%#TD{gSXWQ zOp!nG4odn@WiN)?CHA6AK=EQL9~sB!X2CVQWh*g{2s(SVN)`I#^gRtxAshWReyQNS zO+2ZmBOB{j=YG_O)dKnnXrdTEsQ4iXe6JCkQUyU5K%>^^(6*F7EYhI24-iD2Rvp|> zgT(cbmaAEbl$f#&lF|+dE;+LJ{o;H?`$6>{)sq_4G5|&4MIxpHE9Y>?(^okFVH9)j`W943g~ylLKsfk2)tX09s} zq)O{^w)Gi+ELU&|+m3~sw0gk-|N42$^k_e@p}YB}?1A`$ecBl}b+9D&)Z|c%DdmwG z6fiVTfMb*n+6t0LM3M{cWGG7wHL|s9`&T%YU*L4_(>r0Jtj;I_n8V-suTS;b49s2+ zFBd`(lpYL7>gf?qQjDm$PHyJNm-g^4p3;)mo(|8#RvR0yD^3A-;rStc`sgHGJTdUh@ zcCTp`Lple*4BPHlGlspthqdrm8YQU{Gj9vo9?hvpz8&J<3{GuyRmi_^!PO`+kcoTA zStsWA3uxp1dN2@Rsz->itoh?;*#e!a zMEi5@s`Ff~|6q@1?S5qM;LeCP5_AmtZdlCj$G8Y(*G#m2rKhJSZNcY6;i@?8d)jvD zh!=0HvNu+Zt3QX_lyVeq`WYJ>pt=U+c^~y;qS95%W~iejBBVPGrsLlVduydP&^wmp zPXMfRu$8TJ@JI_4hb=jbIbe19p5Y&aqFb527 zp{K}iXiLA1Ha(!2B&|wsHFajB*yOl^9K_UJjeB4uD(o3FyPdh)N~^K^Ud-G31OXV{ z@8C5N%d>;qq#&eYpDIwmSj9CgfrDdMBr`M9y#YwX!Y$88b1{ED;TXg41{B8o7YGX9 zfN6;D=ku>QeF6|NMe17%J5yoG{bimvwB@M-#DZ_L)~GdnQOUhP6+a;eQ_3>Q{271l z)vq|N3y-GGC*`XbHo1`H2AMn3_wUNYBP1|7LpT*q#+&QeU_ihBLiwRhAOd-yu9Vod z;T4?LflTBpOJUM}$et~wjxM!dhK~DJ0--?<*W&q&R18st=9D5dx#gY9$~U5v@wD|< zERQS#krAc`%4%w{jUVH7WAqTxb$652Jo(1`?AG&XLuRhk6`r1yw=7b2IbT}MG~l>& zHQAvQhzWumCRfMVLYj9qMHiozkcaOY){Cvq|8}TX>JkPFcv>ALHm<5J@>6RKJ~_8CHeO9%~6J zzyCSvKkiblK$Z1Yo6E9LXOKC)>$U>ank3n?mk zwA#Q>(Ymk4+iE+JUI}TWD_`>~Jy_1QQOFG^Jc|f&4cc5Dq6zc)=zKW4B*%pSAI49( z^qVIB!%~Qyh-CW(wnxI=aUm2l8GU2z0SFbmT#oWv?h&CR0L|ialqgLUcH89isP+V} ztg31SnI@+VaQ;k;Pd$Y4?~P3rMx>ZmdjSZ@X2yr558iN|L9-?PNeH-?L=<>y8nyR{ z2oU2Wd1O=)#Q6n6S$3B;&u>clRG$}-w>^vN5uXEz-YfT)oqHfXN-+TSUoNNYecOU2= zbPzVbvtL|M-b$Wb1mz?|xR*q&oAavUdp*9^YKPWOa|l6dRJ@#N`{&bI4@S=ue2wBL zX$zEim_lZvgjb`jInio}QgGeC?`9VdKz+U0;W5Xa`lIN4WmQ;ibH?E8=bxOMIM^?M zK1y{{WUCCn_`&3@z0pPwQkuG|*ywpK6JDo8@NU%QaC zJ!=lM!*BWX_mqcb0aAY9#B?&=^Xh*PM|edJ2y$asYj5$2~~H zuX2(RsO%!>v1}-DZPq zLfiS9;F$g>1YvnzGHJQDJ}U%ZbO(3MeBHY9H#qgguZ@wdin;9HDtV|m)RFl;&7AjB zKg|NgLL{*>g@4N2!NY1QoB}js_gnA#^z?Kh7mt^duAv-$2pZBLWJPT>S zrxf$bF=e;oXJzUIqyG$ZH@%)5o#L=_izY!?;w*TiVHtWScu|>4&fRu^XQKeEmu7qE zTWzPg%yiGQ(WS#qtROAn46PV;EGYTMUVc?o!uDa6&klrR<`;c;XhJP!JW3@iSqDcp=MCOp6#<7pE-^O%T(+=6wn8cn* zC}%4LHHEaBk=Sgd$Mp*7|Nnn@??~Hn(SPeA;FCgXyKu{KAX>GZlN{AU0|t}oKyM&N z34NH5YXg*{`i)TuZ!31^vfrJJ259GJkfT6!HX2&@a0NluiNqKq=?Cx&eB8PgO-9hU zh{a~#;gGU6b%@^oKnJ-IGWwKv+L~XCP+>Ng{7_X&P7|7Y~$mcU7T_(Rw#=+9`n= z!X2{8HNFlLE5ri*pZA`1yk`RZ?j;n5s<1&%7ot%_DQP>NYJJOfS>f*m1FjN1;Z z#1_^|>0Joz!`{i?pXairo6Tu&L{ly+9FnP@#5F2Mdj~Il2Li=F0dTfYoOKLpP3c6G z1sErGs}=Pf)>-ZL618@OQE79-(t*gJ3v7cMymt2%f9KQH7m97+dp+>GWNBZdNm7`- zSf6Qs2X&fFJ28eSY~@b{HYYMT4O3=|zF*9oIE-N@;Ya0Aq zzQ6H}=q;3F=HTGS_j=!V^d1GgFLPT>`eL?hraK~tM1;f$6>dB-TrrW9lpJlMs*KX4 z5KuWYS|S@VyXY{Cr95o!iH}p(E_z*{udc3MyMgfuyFb<{(@>YaliBocATm1o0*?Uo z;ZShJ)EUR$*%jsq0G0&XAc3ak;LM+4gYCsL)oNqP3(j?JY6e^P+c8zn=?dyjZ`vBB z(i&za8umo#HSa))#oV_7C3A=t#^0ToBn72I%!5xU%wMijI8w~X$>{_w6?z@?`I3j* z4Um)cEgSQ_ib&Xg#-~s8x==ZWk?R3#np1)89s#ziV54FL_EwL8B9y~MY;tse zl=&pMa%5vH-aFN@Q)%0Rk9u461{Mw4fP<{H3WF#5TtQTny}rD2g{qi3t{ajXrl2i~-830MAgwcs902&CFjNWNY#UMAo8V;DMHuT~5>-gp6uyjs1! z8flhe-jx35qcI%&PfUMbE>jp}ZciJ~e&?oN;zogR{ot&1pA?9 zV_-`SKcHGxbqrlgRdy%gA~ftNZtREYZ!wi+qJwI^tN78D_B!cTuJJc@1m@T;S3=p$ zym00ZDr?Xxz>Oxxss`Blvp3@S*(YB`wfletM?Vo7D8$Kbb@t6}`CK;^U@*?G1ec}A zuJK(lGp4eIvF8IL$q?fkY09WC`#IR?*N?V>qe_y!;cOrpRt5xSvFEOmwt62gZ$3%*84tS!UgPp_Ywivt(#Oam-(288I4PX1CWeB);a zZoo@#pmEa-eFj2P*lIer)kvN-2M3PW}5>XaaZ-E zhMA?xUX}9f`X!t1#S*%Z)Ev_@+V)xEq(#FO+y?Wd(==;U{qBI zm0G0|Xi)aBLF`_9rpl1I?ihJeynwU);^{ulf}OAFM++Iqa{*Eim7F@4Yy}KyM_w9c z(0K`zT=Z#Y#oC1YRUC<{(5VIKalyOARaAp+NX~Ei*lZa*Yn5BJrs0~gN?eT7l_`nv zl9Winj``3Cb4jr>heHbarL*)3CP7e{(e}!ihh&s=Ld0Pn_+s31P$Q+A4XQ}V0{xkk z>N`RVERLuvSexccm@iK(D=bwON$|S`RMJaHilim1(Za!i6wQwA=?F}31md@vE zrQ>0$Xb=G(ole`+zTL_PuDQ3sU~^?Z-s z&!`>_YI-$A$t#u_p1{>iaTa_fe>+x>1dcR(V{_yFKO38T;?D_>KP=g9r$PSc{Z=@M zIIQtMSoS#4IU4l}i;stdpX~Ltq$O{;lFx8b_xf5Fqrgok0sfJ8ddy`Xp8Jm=CCUl^^DYZ0uT6&vz-xQP5cH%jCgOa{4H~kCpPp5Fl3Dcu4^n z0@8rSA5zXAoFVf|xV51Y$(B}<;Jj3xQeY)o@0ZnjxZ8`&0LJ21UQ$Dj1hwyciAYKA zPz!^C_oZM}hn~VH0~6k$H$IO*;lMS`NIW0{`qSfI9zUtB33$|{hc*d6blG&?z)SBef+}F^3nC) z#zX={`txADf#0{@!0NDIQw{PMUqyw-h*h<&)eyzIY|kDVc7QT(*iF7a(-_QQN=fb% zZ8(17pT5n1S!D)a5-G~w ze6jTu+d(t=s^>QDTYyO(%#?)V+h3u@sc(*MTEekA#Uc|SG;@LtutAD#-A|wHk+aLO zVrjDa3c?NDMw=;$Y^O)4{GtXw7w9!tRq<7zlbj8ghpX2c=|b8vY+=CQJE3_b((e*0?*D;p(%o@^eM@y~8ovi_k^qP&PV8 zkL`as z&RZcQH3x~b3Kw&>59wJ0F4!})vU_;I}49Hz*Qa zFsMhib4r%E+#y^JD>{;La!m6SSs{s!O0Tu@|E2U=!}NeNYK(-K@&fz5 zJ#&zw;l*|YQxBp7AYEYI$c9!s6?I+aCpzlaX#PmJASeIG*rNVv@wrpR)jR`?Xgs=Z>42#hrYYb=usrL(dNqc**zv%Z6$!ra2RYc&0cp zsAd-NGyddIU}uy^Nqscoq%!E<5v1O%R6Z^)??^KiYW$h{;OJ(r=3Z?#E$o965b3f7 z5YmjrO`fL)Qb#tR&-wRaUzXEVkRwo85u81~P1 zo+Mula7)9FCkj2p$-p@U92)QNV?G#9E(w;bx2kj%!n<;I?-*$w*2I^*e~42iyn=jz4Ws~CxP0hM3#tG^yN9JT6S56|8Q zjbwYQRKOH~;*x~l;-EM(VF1OsF8+n$&Mp3*ptuHniJ_~2J>uT7GUn0?G;|mZU?>_( zdnKr?uzz8@N#)jsM5BU@^f*)T$C_2@9Fwo!@&b>Tg$Lgtif%ZPZE8Wi*q9;jN_3=d zj0KO**Ax9<^r*dsFs?f9MtQ)F9hx~X2dMZ}SH*6rqC%gW05HM5B=6`{Bzjd#Qaf`w zZ0lsXTAAgK0=e)za9Jt&%c|KuC0kgQ<1?j_z!~=cl3M+9IT|vpy zi_tYGSF{Qu*8#uzb1k(mt+dUk`tUAesY#vEZf^vVP)3Q1d7fgnfn&mR&;UVmB%TMd z@Xh4e3C1j#gGk9_byteW1*g5m(olBWvHMFUUn-_4t`^1lOnPN9tH zv%_*k3vQt1vgC#Ah7LlP4kjIk3r6l5+pNmyJ{x$mpJ?Z@uF%#Grt#0=@1&EpQ{d6h zDZT;M2pYI<;<=TvHl?^pO7ug)UXxS-L+ePeMIX}5t-sp2G}m3Z)wLGw+e(^D$1mm* zynSo6+-=-CbmQHIS~m^IMLbZp+Ez_Jw(;xckC^VfAHIGl+xFddnH90C_-h`VdtBWC z|H2vnX*7=c-&G_c==7*Q@$rvsw7zWR$5w^J&1c@?$hh?ri~D+_ZD&+vnpC0)$+MJk zn26pc=!#uWfKw8QoP^OjG?Djd2n#Z zpO3YaynRcrS;tjn!v}m7ZjrYRy6d8~rO`G|#qQb74yF}3q2@k#&kw6Y!K%lBBLxr- zIXo5&!zri!t@e8ifQG-h0U{PAM^i)9&u`7Qj(@DSHYWtTB|P6rFCo99wKbS)W7wSa z;Zj~)gsl`Y?l{>TQB+hkqi8a7Yll<$wb3AMGcYiWE0DdSYVY}aLD~yo;$s@^QUDV_ zh$KP;o=Zp&yUWIATjUWr6{sJqF5d;`Y4Ts;DvTA)8(qrF1A&{%^m-81YKvg*TOBmFiq(yVtM{aYKh+`_|WQ};k#@`{nRj> zm`QkiJnPvg&GLH@!r^?g7jv#0*$xvf>i`1yVvGa0_J|Z*0fjVBD6$saj6JI-)uPR1 z)ZqmraW{m^CeKF$ttF+D{I6_UusfIDATyR&kwr&Fx4+fr0Hr-kPgiO<Tf;PD zQKPi8%Tn=2L>O zlY)|yv*WZ$08qPP!soC;yRkT+dYN5$jpfe<#nm0KT-|EX_rwzQI1`tfmG;TN2X&ir zJDlstW=|WjIS6_=(@`#E1j5B$1#C9Mz?tY_>UWI_-z?tx(=3ksZ5F}p98v}=zJC;s z4vy~8MF08mQGGNrc_!Mj51BQb!C>p8^WMG0aQX4>dOPcnz#wCV9zVkH*jNd0SA&w0 zl9zE5Dh%Zp=?~5P7(ttsy%2-1cS{#f8N6@ASK{hOJitDnlHfOYt=|lOa%q2|sU7 zkRGuDK{U4;qw1HxaLe5XLVa-@UU)~AablI4z%lpsIj2H9%TqDW(UPow-WE#_E6P z*i!{1)bnlz_ip)Dsp(TwjO*&}B7WL~``qczM%&IOIgB+Ec7kHt33{GXy9EaO@6xbwXZ18_S_8#CC};QBpctObES560 z0~6f4>49=P<2=LKUXrdN?%=+2D66~hV z8u!fG>oyIbs{{rNAT*2ufX)Eg6;>MxN=jC38g_{&tQ(9Pw(XJe%|-u)yQ$js^%X7f zr^yHi!W*MV__uxC15r`t;?XV>XPNBFE7GX^IF*QHdxPR-OD1jak6B?RQA$XdYkm&6 zJg&o?;M5&5c2YOm0|9`?(}j(bB$xCpWkT*&bB#=S48<( z{k5p}ERk%}C-1adPUE>zzsqB4n*Su8MHg@uJ8-7-tOS4fg#G!Yc3hE{k_ zP>jb3a(dv)I|xZ+#aTB4vLZD7Fi(&?s?r5c8-s_xb7qSyC(gXO0Iu+shQBB?>9v1k zKG;l;iXtLjpnSaSWyZHk14h;Lor-k8c`sv^as9)keGr!*Q*(1o3#VAm02|LVH4k>zX2hq+850#e`g%@BKkFZ-(dHc#O=%W zf^BO+&J6MR&T-{1UEP@uc-xWp*1nJZw+(OJ(<9=${f^APo0$LMBmbL&@n=XTf?JB$ zOAy)&I1lX~u|*;!s>tfSOIhtzsfI-db-Z9~_~FL|4!##w7s3g4=6c0deR;#~P2~Vd zHr%(&YRXDbx(H6(%ALw%J8zds``o;*K<)Ci*cl#^p1pB2Kn2cQ(S-*&Z7zsdPrKZG z|0-pCX=vmE25)@^-|2ZWx-BLIDH4le711#NLEpr#RdL2QLH4XQZxTuPT}REN?1dkG zrp#zIzi2)h=vo}t;l)s31A}K=KS#m{<gGPtnMpQh%b0OD#$FU8{rJmlop zS|Ld54{>Q5H&#Hr36U$KpTBfy5x?|{g`*@QIjlMvhyH2l9W`_FJX#H)J&SkRs)?fi z_OOU7E&PpeTz2Grqs0(%Dmy|6lhZ9(%AKcY!;{5dd#U_b7h^e_ed6SSn7bdax!Ijp zDlCGY8qVo65=?$z2@-7tF4O0dkUuW3Sj6sFAHGJ*UdI2S>jUQ)HCkdH1~_dkvF2^Q z@nll{rH7f`-7qOuDYdzy(Nr#WM+@VqprXQ<*B!jM?O}>C`Ptl@3WD@r0BvRgYQvxI|$n2y}*`LYnkWp^zH{5GJ+vAV(Gx9KU*)sS z5>r!+{e-sd;ocd^<_yNDl7(SeB^4KL={B&n9KzVj)jw_USTZbKging+NjicfHbVhb zE2$RLvHNU?YN)I+Qd`|03@@WciHX)?__ZAt-Ax$d@ z5p}m*`yw9Z)nQ=xjmv zM=}5A`;^R+Az?t;TpD%X9)w9{>o=s*vHE%d;KwiY(7=F&AC;E%HY%QvT{Vvm zK<+qt2I_I&G06Rr)G3nXUVkW2GRaz1?jN5ceLb@vZNuM^ zwGZRZnnxCSzkD}sj$GZg1KeSh{WVp&B;m2y`C<_Da@kJ_YTM!^JO7%&9#cGlgQUbS z62?3-Vo*{ypxTY4*Jw$4767W*t&D&C&L&pGRow5?&!N_&gPl|>J$OiN43PU9w3=3R zGobdL25S?P2o*hTkxuK4kiIOR`Fz?Y*I|ZhTsoqNj48Ir z0XWx1>K@mK4odQYEr?mxC^2QodWBJZ=|JJSGbO64!06WVK~QLXE@lw1u#9O&Trn$` z{G7vCZZmraBu*u!LOILf*|O{}SHcW?)8!5YXt36Hc6K_q`j#q}?YweB(1pg`4P9x? z-nDvCut{Y!kOErOp6c}#vxJCVw z-%(P^!QYt1zk!$kTIw|PZuBS36Np1m-?xCib2f?QsNG9w4VnAZk~VUYhUvzw{#`ls ztpy$2Vprp}k^=5SU!)s$+cHgB`0gw&0w+@2cRl|f@jAAVP(*qCm*#&L}iI5HpSY9pg!B7XJy7GBcbQ&3R2;~Q`4oU^~W zWnN`@wmU;!X#L~fdCMh5l@c4Fg30By&!0ijzY5%A(cmAaG&O{PiTjbNW)i2qq`L(Q zwgimSMtuePC&@$E-j%*zDtlfGsipCSTz2D*H~LGZXC!4qi{Dja(yNV~L={im-g5k- z1*ouFDb}^*Em{14LP!T+(*UVQHX+$~n9MlC@87^*ytQ^Hn`{0BE_UE1=eOi6t8$1I zzk?f6cb;DZQBzm>PBs3x$kV8X3!G9r8^d*ThG_ED)st_2o!NaA#W!cWAd_p3jgRSC zk~8S2?>*4r6^vxjm1TXcdsY>oA)4mY5)}!LW^JEn3u)knmlK!oU-OtpASE8R&BBFF5kle3VwDdUwPK3>(-2pK>P`@VuVg$> z00d)Y0EE8i?Fj#>SPTlNPM)`_W+8zB-3ACj{IxALoqt?2BDa9#h&abX|1EW#27N00F2Fae@c+tn%@{Rrd zsgq!ea_~CJYbx_o{6qVGuKfpCn3t!+A`jQXrFH)0JI&$V6^kAqauUz4qJ68?Mj-O3 zE&Ian*p{(DHQJj|X0O2{*;$;c8Fmi>N%jv2h(wTY-EDP$b)-rDN=vx?ISi0_Pr+5` z8KDa{;9TlM05wUkkyshdmurNAGt^J@N{3L|w36V2TeUPTMCXpRMD$I2*MGAf_GSO0 z`3#PxNyj0Cm;#J%dx$xt{rDulapmG`7W}Yy?Z0F7j5Il~eTz^NJpY~rRIP#^fB;Q!!G1Cq+; zp~&Y4P5$n4JMZzMdC2c18W~xW=dM2M6z>*(m%L*{a`G7Gq-7zG;ic=gQXdHLwhu}? zayBwsAMyt|y8-%XDJDlkoXQ#hsZ4#%3O?SE5I`&x&Q5$k*3vOFH5XeSa0gl4d_>u~ zk5=5Dow3*ceO``>`DPFy9jj&m6$cxeNTihZr#DCW74A^NvN(Mwr>774Fo0`KNl9}_ zNqpGeziK-P3vezdC?FgImsOSM8Ug*?W7BKDk|7Z3`=1-+VX-vhf=AK>MoGT2K=6%w zMM`gGNrkX?jeRH9MKw>er78!gh+NXcjwfeLZ-8*D3Wm0Sv`#>9H}UpwZs)%kBzm`_ z4%I{N7TgjQ5x6!W<;bx?iS|)Li={WHyl|4OiRtnuVo&Ly+cU#81Io9*fx84RtB&hh zorOrBAM-&59B0YFDGB&#IyyRh4_nc)^78V(`#xB;G(Pm+JLifO2gX!Ra}WuCHPO0- zM;8T@-!|w(`}d^mwrHca`wxV)e7W7(KD-W@fC3=Vc*i8+A8u-5apSX$u!DD;-|DLO${VPYJ-|=1hk+7Dko?G9w+RAkH8UxW* zTNV+AW|mo9xhB5V`f_95}c@X5Z4}81Np^g~i89 zA*wfT*v?ptI|&cd0PPOR<)df1fAH?VKi~Ns)=5N`=rE!1cu0~>jdu5twtnjxxa-k- z_tGqr#qV-gCK(5F&-VH5pd&UWBg^xj2drhmUj_fHA&yx~{3f8ExYc&rv6vVcc;Pq& z{GXIzt*i<{rdG<<b5mfzvBW6y)Kr~46&i%n7u zZ8+zs*|vXj?sswOw&Rp%WRXLQF3{VRurx;1jJ|k9im#v6bF!4$a-qoB9(C!_X+r?F54<3A^>G_q)0j(U657QMXspT5~ z7W%N#BwqGA2A}y+IGP4Nk6Sb7l6{&>Qw}JS5r`_xc>py@*fjm-9J2zdMTrds zHJpZFbUAKf86%Awqjtz9rGT^@TcfqbGlPRCw&$^BOPli<-wOu29B$F8NKzhH^}#eMFol#PFVz! zoD3Cnu~4Bpjd>x$!oNt1b8|gCKl#yL$)#Tq7XoQ}rBR3?^LMF9$LqU^u7Q6 z>1>?<0a&Ca-n4lpP5%>x`na&LK)x#HSiTLRKI(zQ%?Y<%#HFOjy{uu^aox`%`dyRR zUBA1hc@MKBBUv7SfP^^^xA&4Pxfqa!9M#}#^ee0NihGFFDy%&XSk>c1pFcgBkK1IJ z3PaGrb}2-`Se=E610pLoct{u-9T{Kz^;)Ya7a}@7{^msN2Yl8nXu=5oK=dmJC3$;u zoFL)%2as12YV9a{V|Mg-QrBNp0?)`?+Y2n(aHdczX}+5x2vAfZa#<4%Bg27O2kSKxCY# z+uF}v)lS@2^H{5tt@lMWNX|@p|Lr12RE71PtkZm2DSYW!I4C@kZ2M%&uc`2>x$!Lc zm;gLB#?~+X>3nW1|I;jlqB){TX|MYL2!wuO;Zx3@i&P0%U@_LPqKXZ*tnP~Jm^R9Z zWw5qeEFYPI@*_?k1JbSpizaJgO#dX6x(m<~#@uZLpooQ9N4O~z%K-pIAN}`$OL3XugBUC2wDuil!$L- zPrFvlyjJIb%hMP|{01b8)Xv~P*5vUV_5^nCc%PQhbNoq6qvi2G(p;&NzOd!D#LM5o z(`(uJq1R;Q618pd>-Wu>#**S-Q6Ni`IzqyIqg>3mkC*F-lPqT~BAdZ77Vwq=0#f;a z>1EC_47h?ab!n)Q=&7%8nbJmKd!8;vJff_J0~7R_25g+Qi=p5t)uJ1h8U2U6t1%(f z6@PZ8CunbCK{!3|0=#2B*{8rDDRL9HGX%!*&^|0U9(&;r;mHEndt5Z$znxs601LQ+ zYTvGj=y1(*>IPG#bxVkB2>ZwN!8qOLtTYRxR^Ejmz%hB+dH&I_&?X{{4f?1sYs6oCbFz8XhM~8N9D~StTa4U~{y?|>Zl$m`T2crB=*kWORhK!)Hu(G&<{rCb7c(&hq>U{)!t{&2X= z@)^)C@V-h`b3>Tu{)+~SKBmFw-z|!R_zXu8ETtCuZ4TC;dpjXp3M49mTZ8#d{CFMm zOcYG)Ssa!i{3;8xVhjW1&!h)#sW)oZ&0)LECnGRxoAiF;O@#llmO-iceT|3I^ zo$>f_gP~n;)dRkiYk6lH;sY7QUlI;Z_JU3{B~NkVmKRdP(Tx&BO5O*?C3y+uNJV?z zyUtUQRFl0H7i~cTB0g^x{($*o7S0Ptv@Yak)s-oxTrFIwL-_E#?B}&ZPHik^1VZ&8 zGL=2>+I=7iSY+|*28bb;WJI_)BgNqNb`G(8*p*W(+MOMr@90TgzE>;|A`4bio38-6 zMQdigv-N}J1oUb(K#6D$>^J7hZ6<<*4EJ}7i(Ga(KFA*f^mtgu%F4>{tNXms0k#}R z)0h=HuY3~|)|Lh!LB*uWm)N`Iz)Y%CQWW-nV0$9gAjwa>%xgrc+@U^~FNOIUs1w`F zHC`|!bI5!Ne*R!)c z6REpDnS#&`R`sh_uDk$Ubk{xMyC?4L zWS{8edUDjHiqQU%7bsiEA*Dvvk8w22_Rd!iT*22EH&E+?lM@#>F>KW>0!6hL5X^iY zu12JRgxK1b01rqdrh4kn5?zp}h?IV6Hn@G|K}#^z|6=Ver3vOjDc=eo{w ztz#X>vDO;v%+nA4F?oiW3?ds|HXCHK-|EZLldYPA7Prf0(eL^Pb z1*}w%HB;51m-in&W*&~P+EaMtz->DJ_(Oj}+}qg@vp4fY>0=Cum1pG3f01UBXgtm& z7g3G_igF1z&!Svme04}yU8i6rP%Ss_FA;hIynRRf3)PN;`SrPjOIVf%l=-a{Axb01 zdGi)ybNCBQW;Jv=mw=rtZaNH<*ORI0bgHzgA~_)>IdsY#+2&v<`((Wc4Z(9UV~UQg31Z088wx7VVUG6&AGc1!p^@x(Vb4B-ZOtdYw4_(}rh}IT z)ackGV`NS~O(Vgk2S3h;H(&+4KA51N=XCTCp{18~83b@Y(JH0v&5$=TabOfYHeXZ> z*o5)mWkZh5n)7zo3ojqZmE+n;iVZ;1V?Fnhp*)3mHRgxBwF`cA4T}psP_|u5*6XYwc%uo-Dtb zbEKFMZ$x8LWBzO(@r=q#l+}&3vvv;3kmwil6Tg>T|Mx@xKaF8fJBf6N7BQ-`O>UoB z&$Q#JFkd%;5Fzj261a*4JW5q-;f4V*(6zkA-tz6%7AyH5i`&m+B7%;*E<(0IDJ?d~I z-yrE%T0?Cc-085x4SVZx!M3>bWuQFOp*CkMp2>apnBfVb{l^!`l1JwZ`!ELBvS?h&N;mD3Of>((6^jNPukLYTVk;~h^ zdyguTB-x)`W@%&x2GComfNUjNF^?*#H6ET}Ye>D70et?b+D42jXxX3CGPT=alY0K# z{}k)K=1mMxrjS*-3pDpr{ofrhSL`B3=hQxQ8fATLuKXGJa|Kp@-At27pYiT zex5|72!_{a=bId2Clmr~I6@5QV83GXr7-`HRZ+K#0pw4V2 zp=q#k*Ei|ZzjmuP#C`_&0+=531o(ze7XR@L|KD5q%g=nnjs$sB$6P)7Hp%-MHTz#I zHCC3rvwr|`vVIA574st!x}K)>+~KpMO2$Z3)s|*{(zQUbp%=6;e#8$;3lxJZCd)!Xt1MnO(t=){HPOXn%ECGN42C?=+yWS&%G)sl_oa*c}8Qs zV9vbpn)~$8z2(<#ZDwI)IyeJ&==3YTqj(3@Z01Re>C8JAB`dzGpxvB!OwCMAI>Wf3 zt-J4M=PtsCfJJH#Xho2O$)tS9%JovwQJfWZ|YR+XI`}2ru z8#>C(yh4K~&bMb`>aO7QezrH|W7V7+@KqLqsw3@u7BV4$=Zjezd->+T4BoOja|KwR{+JZG}TqvuY zM3}iB7^uf@yB@obS*AyjL6rHma%0?`pz|uvFyTJ%)dcHz|9Vw7v-E7w7szjhr+=7G zHhqYW`4sBZ;Y5Arg;>Z{xR-oy+p7}k3b8SLQyt3IrU16wUgAxztr5}n|V2@BO@=LA)5&~rpzr5SH9nLo=C=kQ(f?x5O z#X?NUn}dlFqCgzTPE-tS3Ud21LIkQA=vxTZUZ`6^(Jef@ju2C9Mca!w2zB;gsYZTkkm>a zycJT;)2o=;naW_$(r1BeOku}6pdM{(ZT$(B3cdt?wtM19L0XZ}s;B8f`2O=fenJ9y z$J0Nz1Kb2?0^cq(|ErZhR~fYO8=mO(C(vG{;(y7axz{?vwQ=C{1b?$})t);8#d`gU zWB7j$KnQXs{6`x`31(_h_$mJox8IC2*{F{A5`W4Obsg)yWlo9MG^o7;GEi^8@NWg{ z+(75${s#&^cxOV3!k941B?fAIGDv9g0n^T&08F{1rK9s&U;kdQB+k$mj&Cd`PO>Ks zT|)~2LSg|_lF+2R$2-7Q(g7wuk}z;|f!XQKB{LH}{Kp&sqo0{QyQVDogwD!sr`^S4 zQt`D*>kJ@`%vs%5kR=F|XFl}SNuK{zW3-1Z*Z8$Vy% zo+sXD>YSv@Gp*$d`SOLR<9af{=zqhWStq|qx@IPq4{8BD27U==8sO3CS&!LS-oev$^X#!pOXCH>j=49Y&J(-QZr};^|@{}yywn*U_Scr2i zSPS&GsH!qjucZAxLozhu2(0Oy37{;K-Uy1yz$FmH=<{CMs9!5tVxxdN+Q#OB=1c68 zA@+^$t5!7B-MOdnio9~Dc`Z;q&QL!mzi`%JzmTo4K)ztPS7Y-gG(~RyKT)C*%PM10 zd_NJ)9qka1KC@;?u%S#1YnK0E=2`#8HmN(5=gjO5bl;dh>miY_n#ZXTk5w2U6x?5n*5#He{8@>0rnnIRO5gQ&m8) zpn0BkMoq8Cx<=|;!}Vv zFeEC9l_{iwDg2o8g4LA4+P|U^L{GwU5IHUtCMqgA8JPtKFK;c-wiX6JYCK@P?%qdd zPyvOT>|fTw_x8s>9h^5ld)=BX>DxG6cDiIM*VD5rg+WU~`;J;vM}(;M5*F5%xL>cH zh+(6@LZmw?4YdtAf9T=Qi|K8P>*tSq`||P21dUfXHy((lb6j@2j3=f;^7&%|tjLU(;Nc9qjWa{&KrY;r0gIB`Wbj($PY3G{^q;*PDpP>B97e3bEnK z4?byMS|^guTN%QD?cV&8gSGEXKR9c&6M2Z2zv|#|XxbE6nyj&cF?7NPn~dKuo#j9h zqwNlLJSMY3jdPg9MJIi0UhrPcq!Fm)y=Dz$ZSlRf*O>!)!cu?VyYK=aKY-1*+hC13 zmKgAaewa)P|0}B+Jp6Wp=+C{-mvZg|?ZvChn2z%vE^6gp2E|{S6OSCM zN6BV73m#@KirY_Th|g=;S!ycT(_XLO=`2zMo)>|u_)aL=2+fuMc9o{|!juln^O9)k zquF}%zlQpC{2ag=zL`?k8-Pj|X^ET^})AG8=YtcXQ?W3iNHlyd6S5RUg#*fVn>*xIRK z9<9RNH+_F1@!~}$`khwy);*u8`oJUzajs8-IrZW^BbIMy-sCI#`?%#=Qt={qV-%`Q zB+SCXFk$1Eb5CleuV5uO*QA=SD?pzwN2y^(%AFzPZ)8P}@OQh$bdW81uQQj2Q?0)Y zblMwFkU;U>1xEB(N4k7BN@f?AP&vX3H#;#AX2DTRQ;>EQ5R}CuY;ITW`JC3#;e^LN z0{c@1F(OX0kyYx|;L{U{IXAXNs{PO;)46yT*#x$z+#*!={CcTYYu34j8LV6)n}nCo zqC*epaRL%evniPAN8ZSLn+%QW8DRp!gbc^IIUvR*EOaF)UuIib4MG zsFj}llL>tb^r4Ta;vQ_#Ic zMWfmVw3kw%Vc+oQv3OM8tgM1x@hb0Xl?i7(VQ0jISK+z|;UR0AN(fMQ1RIf_gn5?jQ9qjNf|cmuEg$=sH;Jlyt8_ZG{9(7}1nAP_Ac01zDlnXxWKsb7+i5 zm|6u~Q`a=5rG;j-%!}_$ivIie+)-MuPu*!TSlS&5eT2T*ruXA?r)P}A4u#(S#c5nORjd}w-Hj|W2E=;mPa=YmWA-S{I_XC#doKU~!5GfND!*$Dh zb*pU!DLG|-mGa^Coa^-;sNFLASJ1s#cLwKp758Y*e>fTV;ZF<|DM2s4MX4&4@s9Z+ z_zIevy5(D*sOy(SfQeqXC;X*1e4vCm>U4-F#e>Cbw>_fND<$SVeSy{&Y0&;vq7&kA zvo601?yId)be6~nanMOsDZT}Q{T@z|6JJ2x}i3Wxjep)L8!c#zh;>;=F;Ek_sE+ye`tHsG&MXJ9bD{c z&%bV9PaB00yd>{lK|Ez{AQ9nh7B<-Eg-Gg=^)KhC1f(ZF2K zt!rywz|m|PiJ09^$l?(pZ&stg&v!wt08;Y}a8m2m;8*^6n+An; zf#JYo{M|fr5fgTnDI&3i%Qh+1^6`&cz>Qq5r&8VX8OcNn=CAT7S((%u5Oik9v8TrT zxL05Z+SY)1kSRMcIO$DNYj@@tTErupg5W;l97C^bQF`<36M1?a7)(q|&;CBpd7S5lvxp`b57%?(dYjD0|s10NM z!Dde-V0&K>%vVRRo|$d=u0LF4##EX(GSzd?vSMvUU28`;_aqY6N|3;l;|w)nBSP;Y zQ+Dp9>u9oZ1Veq}nVy$N-GV+??+k1m+Qdq?DbOt>sH$p=~uy)O@@owIhMhXe-jUft0 zm@l33)rMVq@c;)8EANSmN0^=64_SsuJ_|K-RYSYAa)OlXoc>Oq`JgEds$oIU94PWu zGKX2L+eMsxsh|>!;KM6l|BVlx%g^WE0ma}WtD>?@^fcQ@o(Hb^$o@nLyi_xcfBu2E z`|bGX<@WD&VPsc|T|8CzMXU~|2;e_MrBkL*8qWq{>n-p7;9J zOT~FFoY}mZ!*rJEauXAuON6!?hr8ztMiLjAh4tcsFEG0s!m`+D7kYO_1C~Y5@3qrQ zv5O8)&@mTo>(Pp4y=viWXWp6JQOHEC-_dQ0?p(0MyclL;{!X+C-`f*|m zFr*|Nj6GvU45lmZ{Q|kXHcTu$%9|}WC+AmI*2`_Lrz&)-C&)lf75G*&r|Yu_pc3WY zP?>aG6*Q?9M1zgfQOw_Fy(iZxqXM~5LXG0~&^>9CZRK8-d* zftKqOXhzW6S-ioe(~LD~{Z^x4GS*a6I-~1ueY_Z{%IsXG++@9*n~9v0Jb^E3@d&rU zsNB;#q#ttdg2v_$Ub{8F#^&kzn&KtcbdQ@S8^NO7_Rd6;l(Fou76+S!dV8x;Qg;Un zj2`uPb9aY^ti0b%J-&taHuy{)VmB@Jry=S>6<20pzR1$a4XEK6+WG&)u zo&!z6wsqyZyiqWA09dHyuU5}+_o@8IaOZIt9<{NTgYA~uHw*nB0}u8-7{P=&GURcF z=1Zme#i<6wi90f#PnkiCj9%Oh-V>74PSk!Ec%|e>x9|vH=8;PmqYOPz3Be*V?dSqd z;{rH6+-&Pr6LOyS5QN7M2-iQTelEbjfS*4qr7~i~+)vlu^yH^BUba6aE0^Qup-a#(23HS-}KmaA7IxB z(p(wV2Jx*adY!Z?^*@1?A9^xws5&&|r_|u?-AWjE(NZADey0&G&VNl`C2-329Fx#& z5TWYL0I1%)cv5}l|1LbP6`mI!RHCNWt2@?l*$VrrMUn}Yylj@l?YG_}j|n!(;!5G~ zb(A=G`JVRQyV@Ax!)k{xUZ)l~lmIs?Oq^7LpGcV^zOLjGtMJg(ip`<7WPH-vpw#of z@?Ffqvp*;ki@>=^T@2^hl)WkP`0tmRd=BN!uY61Ny6e;Rwmcick8jd~SeE_tu|hVz z?z`C&!rHKWUANVvR4~EtJLpw=BW!B7)1&YP$i)8D8kI{U@QyTX5ZL31_Buw?=^M+! z13(d~J%_rlYf>k!t24g`>JQvold9=(Q!i~d;KsB&oZ7bS&LVnc~!WDLs+PalcIuB%Az3v!K8C7(eKYAZ(5Q0l)>*fN=6VSvn zfaIfYg<;}ci6v8a{Ibt)>ukL;C<+?yQfZ6)v|`?S#wzUm!Gsyty(s^!ILNl|-CTK5 zuWk_6N?ZDcf6c}~+z>*3XzqT{o$SEaGvSd-)|s(VYS{Lw(~9|3BlyXW=wF^0X_=wo z&LCz=s}emsk=%w*S&9EXV{lW82JN*s_I%_}jp`?S|5|m?x`R_@uk^^!A5aBhE{X3@ z>Di6m=zhN`yg28+H1;!{{lupGPnhG3fw3_gkT!;JnA12^BS5VuW5}o1PB^(7BpyiG zQR@>t-ySD?{c5&0$Qg17XQYx9Hm~mYX@lmumhaMz{=B;7Dj?D9e4{;7$8r{@yr+S@|xa&Mxjn44kP(8^F5XA1`g6?2V;fTXN3$yh|`g zP(BL-rM1trdV;fr(pluD8KvIa*u4`!v{oARnld3J?s#EU=#{o;;)ogTyWObQ2m)}r zLLQ%eYbJ{Fv2gkK6~z3XwZG8$6$HlsMhD~$g4GH)P*sP5vXFL>IE35cWHZb6i(dTM$>Sq`NbdfhPlzlF^r_oDFmB=^z zD|yrq9+V64h&nFYsJl(^K=Y1A1v>_r2P(s1U}@48Sl-T=W@BVzd?O}?12%KG1O){d zih~1Vb96N~%zf_$?WnDwa{P6u^#vg@N`nzsv@ zTU+H_S95YW#L@AEE%|I#4aL_MUVi-(Ca>VP)GoA*jv_;EU<5Nf2geD1hg6y0j+aE! z&W?}o63%xYj;^jQ11{!~u_ZOO(N-7`f)W(e>=Hf4gE&=4udj#SSsaqNx*peOhY!^O zb7Qt!3i8(^_w3kLKxA)nt_D=p=({s%%Ts0eZ9jb3iY1;$w;>P6SrJZlPnQ`- zW&x5IqQkk22K{X!`V{xz|RF^~SvE5d+!F_Qfd39w$axbdF(v<}`X-S=o`Fu9?^}GH6PZmYP~iS5TD3 z$NyecSsj>*BWHAfL>!s_y7P%QEB)QLD-hOv?D-h1)bSP#0*zy`ia}onavmCQ6jf?g z)2rVb&3Y%y9Sf#n24s|NN-?ppyUVAZ;!A^Z}A9KV=r7VTO(bfie-+&xEirP%V z%BROn$MS$mdA(~Rf*eS0i!V??-?xt~SWP($8eZefv}kRak1WqW63LKR94sHR8IY$dxZOghfj z^4ZYSWg0(j!&{|nHPe23{s}%A+Z8-EzFcVKMCdw4bd2o`UGxKjKAZJ8vRsgiP`s$C zlM6vPVS6gi&)qK~+e==~&y%mdmeW18+>7~@);*BUWbe;mU;d?TXWyFQ3|!HC-p9-8 ze|=a)E?wiS@TtU#4}jPjoNuhGx_o9vvEAZC;1aRdO~uq1AYtv!7{-i5{?KDh4evCeqaey$%U}&0+Z+{&Od*`bkkg zGp?I-|N27F4^VZ;xcOLx)8L;=i5u>=vvfBt7i_KAap9&=dAC7~iN4QVehhJuGRL_C z_MHPlCErOJm*%jIQUdpLP6j{z_RS;| zCsWffp}eQzKXh_N`0h`kBW($hu}cMG{)LA@L2vZdNhm`fdw4LdoyPN=ypw9&zrTS2>hnb2YlaD{{F+HVNq_MF=_i#Ec@@}2aSY$ z4vDt{+cq}E(HD+u4kua;zw{bK-^MeEMemGFF~`fnWWumz;( zo_Xhv%A8r&_-2oSdLo#^!2F4~X6EB1l6kN;F4(1z(o0M&cJvN0#Gf{v^ z;H=e&dGO3e$vrR~tj%P_jR4}~>JL;3w~BAjou!d$$8y9SD|;g{;0E_dzZgBe6B-%@ zaua$S3Q>+oUOLl|!99Q&RU3YvuE#*qxK$e#7TwS=ZO*CZ_a9!43X78Ell{z#lU_Cy z-*@9U)ELpMKdO5t>cPsZo^;{w@?CT6!#|zF4w>~|_DU1U-}XN8yJ9Rq`R2VEx~>Im z>LJ@W-9uh!@LR{-i9WZIX#wCSv^p)&pKqWLA%uW~DqZ%f8+Lf)ggICR>iUE)xvhma zE>CpfRUW`oA%llQSFRWxLG$ua6g@AIFZ6+>eks2T$J#^$IPQDan51AbA{%eMbx!nK6es-zeORO-U~vt0_(hr&$Isb z*}dH#dwzEDm!32I*V&B{ACpW{;Zn3Rm>Y|U6=YU_d2x2BJ{r^-3|9`#R#n%;1PFs_g) zIKqQoTz8p?sJmH&^hM^GPW%QmGsw1F&d0WZrJyV(KTS0_O=FQSsWdND?bdC@1GowX z5iMJM5Zhdmsl2X0ag_QWur33)I}MhVSDMv7R@^&a@n3pkxG=`aCw2|Ubm#JHK(G_m z1+=V&prw2IKhP4KW?rnR-zK}|O6*o9WJ|aZl`aCd(%<;d9Ty&PWho|7OuP~qeyI2X z))|`?g~!0k>JL2I7~h)w4D?QJG*;=Ib+yt^VXzcUH`nhh_Rx{`LE9`o4W)C60l2j@ zcq(eHmPM>$?kQc$7-ZeD7}g620m&ud;=i+;``%ZC3g@7jgg1t+!grr-va}jcqVL=I z&JVlPS+11={gj9`0>le;0eL3}{A1sp;ULs2e)R5)OFe(Za~uoK-c9$iwKbm!?}>eh z>=1L>s>n%r?DQQ5K`F#BHf+=aV#Dm$HMTJTWJ5uiECPURFzk15b|u$Ho#v^T z6UXDx6SDTUfrpBXxUk9FPQ1aO`Y#}EF?}d~4rtJ@e$!TIHu?Eop)X>&ch3x)rGd;+ zRsU{Xs6^k^O`E3tSwJNI;7}Ewss2F4YCRX>EY_s1o}r-2z3k7GQE&F>|3@(O#Ioa!a(8~(ahZ)W`(bv< z>l2oIf(4%ObPgYKa#lqacf+tRuwSoGok~^gc3I06;%3T3s{E z5jt17hyby}wOryk9(Yb-OcDC|)@!H~dDJ!N^tQ0u?5C@y!LrqbUQ==>p%YZ3Qn0b} zah_&0UU&hdU`U#!E(6xlNikuD9K?`91fK--= zqaBTk5zsH3D~Wua9vV=O;?2&k+zYvJyDdzr0#P*?h&EPDnQy%}#ud)sR6 z3uce?>Mz*3_XR2=Y?u>u6PRgq;k?08pAe01?CEVLGYMvb-mFX3I)9W=Uv4XmrYT+C zsx7lif5C4N0Df!ux)9C&k8^1c+*gRQV7rOFfJB2XY-J|w>I**i*7Eli5qN)&DiU5G z@AKckm#|9$}hFn&njwaGBR$}I`Pj)je50(?u`JdRHh`_p+|^k$fz^Hr_pqv z!pT9D9-j>iQ;}Ng%gp3X8aC`}aPEBJ$Xku=IB&&Inod!_OBqCT!>01&X+BI4z zQCCS~ke--$YLCpGBQZy;J#0l~yMTb?yQpp$Z)L2=+CB~r4#WC;s)bo&{*ILpV)F!f zo+7!McZ3c1ec*6Et#i76+|X8c>LH{XKtx#Be~7Ri{#(PwxoX16b2~47vE3RtL^pDt zokMp^KuS@BaFZyTlQKE0+dCD}=1iY%S)88O^nHD4b>l&ulA%7O>thI>ySV(syF$h5 z?RkY<3>(`)l}s6!3f(_+vjC!@ge!w&tP&x_p&ZdnxnQ4BQ*mcFFMK!i)ekX1Q`}zc z&;F4TP}yFs@X6~kTjEZ!YO9H35u^dSULd%nExRHsLgZ!eX&hBoHSNIW+TM&y8LtcE z>3rlnmb?)(bmT3P3~rKp|LZPc{t|2m-I$V}gSY{c3lPWlyJ6a`DD$D8+Qgi7Dc-~R znQf@jUayDmcYzHuSwrfDCwCS(y(pn?T0^F+#2Wls`Z|{4v#yGKI{4reS(We^%7m0m z1=D_EzzxN;6f#1>l*84i?88cMi(ew@kq!N>lXQ56Byf<+W~w-Dn1)7mjziS*^4KT12-k}sq0Iw`1ft2wDC zDG_sLCw5ejbZCRABR5D%nP5ied|3x?5|_znq?*G@K80J;QbuV29JxPLHU6U`66Wx88=Sl+1ZCb0gu&{BF7ve(Y5dQCxFFK zi*`ET)(}`3ni~2tWUSEMEt7hVw|r|?$}K18k6 zcft8;zkyx>FZ=b&8}JTtWFug>&|gDxFsEZas4uv)^K~fez^W@Si=Ei2?y^ttK3tr# zgTUVFz6@14v(dh1U=Arwk)M32_124$%FPsnvjLgo{0zYbmXDgGpXjyRXBIr^xkWJY z1o$;mEI472lJtMLD~Fc~odPV&P;tQek<+FThhswz&KngOt--A=SRg?W?CudiOpsbaO!v|K6&gBC>>~_8hiJ)@BXtST>CHpIlcAJG&77U+ z`kc3e9+8~}wt#__JEORd{T#bT{-y~c7O?M8QP1&7S$@$Ll}&h3Li3}wpIC9q+)W1$ zEvVPa{7Z}OIuW_Eahnj7Y+r~75N(Lml7MO|gR{(;%cLjL_;Ba>;?8e6kL}}_=uGUN zdGN~6U=*Md{1A^~rf6u-7~XHePM1K8LYceXLpkV5{O$kFK^Ymt=o!>x0_&`lf&QslIJ~{6Bdhh)&@wG+vK9;D;kOHG&`PAUo z&L{FRP}1q#op-@N;hB$*Aw09o2k?xf(O}-e16rLpUw4TH!_*I{EcZ;ar()_j@SR9!ramVDo*@2YgsAa59r zlz4z8PjOhjU<%6{49NH!kk@+oYf+4MqEFl>$?619LLk3} zgoqr+^wcQ6@@u}kp%PH9ASahtM`29WKyD<fYA>rd651}gxWQgdyfC0I752lil zlk<854elHfFkUmvHB$Enf;#pa_`t^4Yq8{X8&&3i9uWT^|Gs4Y#ZvdjWES?sp28@( z_nQ)XrN=ELC2&@yJet?NSI2vNR)HDH$b0scd>=S9BToBP}n0hWF!$7LMI#wwZKauqFC(h{J;xMZb+0jVJiVY_RN62`N zLZ(^@2C`-onku|jz89_UG5*7tm(oNMvehmNvqOg1ssj~kc;V6a=xxVV+o7JBg0bfz zpr)VfQ|C7B{c2pXrNiwKzves7j@Xw@uU;4MaO$4@Y6duK;6tI~H<{3eP9nHF1X#Se ziQ8cJ*~)G-u^?5U*Ga|!|B3|J09cj+8!1~IkW_kZGz2=?!$5;?Wf4PFBrzx`*+3@0 zkFl;`-+~FEFxb;@oV&X68?I>pQ#2~+va3`~pkU=I`VM!#pDBi$;3$zvdAVFHl5&dK z`|U6^2UD;z)!cL5e06vrU{v_s**~S7%vCb@C$KEQh<#b31oZXwWg(EA!N-A*Wzjn| zF23g!oQD8(sFHeF`1|S^Fco!S8L0V|x#Q+_e&l)PGg4jpbGx?H;)Z*r))iQw0*-7; zPsjS7)(!PG3;Ry`_iV35MvgOE@{V_A9t`w>^OU&=LY}FA(MapLV*xONj2qkm+5Li@ zJ6God|F*#-K~Qx8^l_rUlz-QX`TgC${v*%LI3a2UNwZJFUJbiIGpfgO!CmYLYCLj? zp-R(w@jSEH9qzwhP^{oH!v;+k*y31G zQ6iIYeCg9$Lpw0deCMidsYw?nug((DDK)SQPwPb1t?)*E-{ID4cR}tJR(9~sMgCa! zm}>>=nl+h$g72R6jO#MIEC1V_a2wb52F&daCa4}Zvkgv8l$_BA3&T2wRdobw{#^F?kYGhz| zJDJ5=$^%+dxu)c!{Pqqs%Me}g-^ph_&oeL;P4Qm@@Id87?En$LAlv?*wS&Fz z)qksWk-dEro#fpUx;)9Nj$5r3c?JV-&foUKNIV8k&Nr&})PU%nvGt!>{8Si`9YLJ0 z%+bE~(K>^aD*mh_`q7AZe@naVufRPRd-?Y3$EFPy^94i3z%yX!Km|G3BXP~ByJrOS z=ISTTjnb3VU?;)o$Ba|$o#(9L_<^0dfdU4SkwZr%=fzQ{@yuCUv}-LYPqm`U>{6cB z-PYgVLung@%J<#JUkG3!iUKdx*OL9z)jom8=|=&tM+UOT(GOQ^urH3V4Ay(CqUoCV zKvT_8VGuLK`GJN3a$(UYG+nx9&!2*I(P2$#s)Ze&MC07RFXw%>R3ieJm zHa5QQ`@Q*It9aUG0}UN@Glhu8xm`rVB3pv`e88{~u(3)$zGz%Lr`6O>g)XDdd04Xj z?QNYjX(xd~ot6k~rXuj46D)EUgy-c8!qfKBBz;M(^s@EHy29apY7?WHOwtbjbF2CE zzR~!$K7_F(Y*%=;T>_%;2kmhFjrw(CrnVii&UBg&g?x4Iq)`5SudY7~N{EC@h5nn1 z_X$vklrT*!i~pFmkNI@js&sB7|6`7l_3LrR&A+@%NNeS*u?b6{yah7Ndo8y2u4fv} zOL0h}(TICh#CRsssG~%hXdksr95`_}0kxu>TxjF{zuyAWqb25G9Lk5Bxrq;Anz6!d zMy$xd@=68m*ZB?!NBrlqh4m;d?90U5?lGV|FgKXbrR1CF4g3~MYsQJ&ewI6f>Zkh( zhZFMzVKh>UTR&ju3@FimF`$eWB=_smmIrRYg#o!+j&1S6*fP!#T=fU6RHC$p`0@ny z%jzLPO1D0%9SOP5Pj9RlpwX)e0l+CCdOwckFda@xvGJZB&b__?#)tWnIetBoL*=y^ zau}iVp|W8^t_OBFSdn`LHJqry0(5J20tv(LRMTy%SZ&T2Dn4+tt(DI?6(V-Cr$DGL zQhZ-Bp}_NN;`-^!XD@_~E&Bz(t2+6(`F5F|y+!IvXQrLcgrHa^kD0?Box7W@{j~Wh z87{==YVZDHxtDb^72(?wpK>2h;&g8gn6T_>#lg$_pKssz zZjMrul+j_Y6K=DI2Zf0W$Lv3ola*6^!fe7KTsaPeC9=hW^ zH!gq=Kt7h-eabWFV|A-@N1*qlGrt@xoUjMGKIbQW*{@k{fUTEt;p)x>TW9>#MLodNh+AbDs zp5(sw{4R{{f-J8t88&CIJ`REIkB`ZAuchQo%ef(bJ4`@H)K_!f%A zK`+zNmM|ZwN^2xvuqOeG8Vy--wE6=K6cC|`!GdKEt$_uAPS%A0 zu~!pb;HF#0J@+u{JM{YmVoS`&yAbUK;RsdVo2YQSB61%Fto4j5Q%69OVO#vhP-e$YVJp+3=!m%VYr(P25eetxrwoxnwhz9?k( zeB$sN=gEOw18R`c+2eYe=9m`)s5!)Fs)lH-? zzZ$>ra~Hu>wfHD$>QruW6jJgesWxShzBU*Z7WP_H)I%tq^d|jT!PQNU-fG_P?JijV zn~|fkiJ6_q7umTpYyw)LrcRpFkx#=_trgM6dBz-r`E6F8L1g~n2-pj=@I{-g6uG{1AWR(Yg|hQ{uiXa9l<|EGoumAK5&q7qcbLhNTMV*7`S6ZFkyX3v?rEQmBMvyz>g>KSYJHy5aAl9kg%?u&^@=Q6(|xy|qT1sRDCkW9X9Hfu38)hw_l^PV#u;TE(y$ke>*}7ejJO63cslRB}S$D{y9X7^C z6;W@M@jy*Ip>R|*zDnPnWn{yR4tHAqa5(Y3C7;0^qH`d*3HJk7p$7WqnqQ5_{-=4x zI(@Is@7rU*aZ8B@Q2oTlS4#xtySSBjs;QC^Kh$*qdJUq%tP#R=otNz3n8TlFj8{Er zBxwD%MQpV?>;YER3xn}uz5%e)Y9bV?SmSx(2MTlPqrJ7%4YtBpefGDkCWSm0|0UJT zTm4(AsR$F;HzRHi@xD`*@&XcTDs4aPfpz2Y`0Qo(myeIZ_zkFyEle9kdYH2yLgM%i zibHsO-1zwM-U<4#-112e=qd2YBqZ2d9?bS7t;lQ70WEr9F97Pi@bmwSzYkqe8e^jK z_Lo~pmQFQDrp=QZK%c(()G&Eu_GTS*akjLZp=pKv#1*LLGP>7640S@$kG#W$E@xE& zx&rp23`WmV`m^)t;9v*|I_edA*9Em@boqnup>wH}s@GajJ6m*x8e({UN1jN%6Ltsj- zRlj~2S+1G{jPG$y4>*)PQ}>`|n6*(SSrW&uY9-sxL(Kx}e-~-%^1PlB_I&n;6wDxz zV8MEO6A+G42S420%DWRJbCte<72|MUAcpLS<-rM%5$S{t z8wOkk*yWBQ|3LF&F1H#s>^%R*5f#CuAHV3lHxim>y+WaY{t!6y6JAv(T4)}eyPJcZf{B$G>bNXTt^GE> z@nMyglJZ{q0|;T@3AtYzq>ebDFW;zp8Ua74Q_^e#yN)_;DEP&PGLOk?O15q{wKayW z-Hn^BaKc_xC4Cz}^-UZMnmY*;q7ua=SXG#mb_VX1f<-BFQe)6k6G*AB;#+jZ_df15 zDk?(ll0PE9`a6Fq+%B0;tyNTMgy&=gib%ZdD=Bw`!pcCst-b?*l3aT2u~{RkxjI*$ zM+*$(T-k5;H3A{}zk1ummk++$)`%>2m%r)vv@sr&+=`ANJ8J!7=X=(D9~pRk1Q!a^ zXy}(PSPDYm#&w-pl|k{ws@J};wq!lgk!~WN5@}$5V6E@8uam?_|N8jX2l(-Y7hsqT zG?9Vz54&YjAt)snbdc4NfN;M4!$~V@@O$f#oE1a;`!U0~HtO$s{6qkGoZvX>F+4zu z3FztrN_5bN#s8WYi86&EtN*hg&)A&Uab;k|OKLPM1_Ie)iJ35AG4P&;wshJ8O+D3f zy(2~CzKeXH2XLo1;o>vZ^cdjrT+OE6ixG{r1&rf58Ae>66YgoL#T=W|b#kVHyr8yRW)sHy{Nlf+63%DA9au&r+1h%$64ZBeeY75cCKA4(1lo)%H{>W>*4F^D{0b8`rDQ^?A0Ze?#K}wf z^A!VVO_9qqQ!O&v;j66FZR2rm4)e6_uAHFuqr%%KL2qRk$Q*O2${z=kU8|VgVsa8r z1_cm-*5o`Jr|J5vu#k?!sWX%LX^25F>W$ny;7de^(ZZ|`${E!PjhnWygis@rI+Ffq%ly3Kyk z2ZJ5%czJ2@q|l(7CW?MAkM)?Uo5KW zH~Cx_gE{@WWBI{1t8r#D{B+1D1Z?eV3D$9vnSYG^%nMW225p0KB2oJ7nE4Wt}@|Rm`UjikSw|`Oz9*&%G{&^vv z0E^h{V{^PqhUb^rJtFd%ldGn;8!LV{ywa+>^!01G1%D+fKNcJtTE8jH$E@fEBbCZg zW5z)?p};!fp4F%i(jB@U@P)D(6`M_#YkceZ`6C)bjYjwfviB{75X?_Nas44mblHvC z3D{sR7RkDe_m)gNe?-o+som3hN(z2h{nsh$m5;2I*1=o_;tuF7(M98yua7-;mGePz zIC&slwO!lrAAV;^+3*Eu*Sv>7G8A=#(@6Z>y9EDtl9_73A+flyW5+aOz zh*PnHv%rK9u1`@?{g!WvV*v~_l!;|h8S2#S{Bvs?uMKbxI{|((X`Jq1)8eG%m3Mx% zDbw;w0d;<;N8h2`yFJRKDbHM>qMMIUD2;%J7D`y0ty$}4D zxdVxfKJ(hGVz8;vh~^>#0MTBwv9j8AwjLo$(&qGlC`&xTkYO;*I{1V(@jT)lDn53H zT&FI?eI&2DfTDb97Fc7vf#MywdX3MH%(FEwbHRPhKxO!Q#whDu3v9)DG5ca7SEUaV zg${~vT<<=c znr(DD4m3&6vY9J)0od*xjRlw=Lk+~^G60dIdxXhQT?)UcX@VK6l=57}{AqgFRQ7%Y zo9<%)_Q>e3_wO@!dsu>Z`RV-<2O)NIRd6>84i2&_2tK!*5$4;bhTviuMFVkKzrsrj z@=u?Dp<3)n9tc}fH#do0E)EfI7f}aN`aUFj34Akhi|F;D@gk9Jzbh=SynuI9>dW)I zj1;lJA&v!5;=ipg0)vCC6uGtdE&Q0P>gnav<$)(hVcJ5AqSu}zq>uc%e4K&DxJ^xPg8S%hmFFM4`{v!LXs z^-=37T-Rw*A!|WZz1I5$`*mx3;M$t+*SCUt7k*|JzmC)L5<&0nJl+GBoG;mEs6c~TlnqaOR>#$*KfRsVicvb&%WJJ@ zY$Y`>ubMwmvj(-r2=}!cX|u&oTO0fOxh8)BKChxuLdn=bXGC&I)3s#w z-F{qJ;y6JwDk6VH^NbX{`+lpMh9IX~P_sDJT9NehBL%lk1=Qk3( zd8Mo;vJjlvkN;}d`5B@R2E;i112D*NpRypOn&7q4E+Jk{_B#Cmqv1Z6f5C1{;udK< z5|vs;+6h3UhTi}i#eitS8y5Y4l5EiAi);rT@gl&b{ouB^mLIBH4Iz^Nk)etpi_@Hj zM7K#kp_=2>YX<(kJd@@dVIkNG8bGO(#FnL~pfHxuT?VUFDkuTj1eoa;<3Pn^5B(b} z?k9k7t(g@k<+1Yv0!xsybJu3gW!oXX?e}5q0~VtnpI_EmE+CDnkHqI{_TY_)ne~#% z!M#>5Zqyx^(x`aSwfdXXL;IUwvpawSEGd}GQsG}K9CQi8WxjPA!1Hzhi|OtdQCx#@ z62jp{cy0*ug~T%&iWdE{`6FZ_lNz`*_*w=A{Ik~|_#qx~*(;m9t}^mhk4I{j-0&!Kw>RwH`k z;(Pw#IQ9m#I6BP5FFYC~?32!&D4-{Tc&d+X+9@u~H^8rM{qy+@n|d2Vqo>2or}{<% zr>6=SK*YE*xjcASew7G=!WKsha^L_WU8T_l&#m6a59cTi9;^)l1vquC3M~&ylJ~J` zZQ&mY-&FU0RRj!9HZJSeJC)^&mP~KPn^nU&IdlAFT#_Tdo?gg%ii3vthrso=wPv_3 za=~~wKV~X5rIac%WX?MvtmsTHu1hQyE_N1bC{BHU^5yLRV`hZd0nB(P3Gc9OpJVtZ z`mFoQ4btj?Vp)1JH$U6Or)=sr&~q{)==W;{$t}>a1tqw1wSpZlI3ZGS%NrSSXFTevDVG%qx1o#4X#w;RGcaBP?_ zA2amQ{m$9+1b!$XM*u1GpdV=W;Yn0P7L}gB*&z9#A3FP$?jmB_5zC;qIu!WjmJ{Ky zfQv=lu@AImMnE)#KpcqVjat7j8aIm(Atc-)3)=<2Ft~aI`xf!xW0Wq1ZpFfvxQ@?J z=#SM#^G8l+Ku^v8ngg($)?mu%#AaW$E;ano3tcOP74Li|9OD`nP6*KEvL*0f5C6z< zuC)rc7!_VyjGi=8I+@+1QaN3xQk5$@cM_evRu^#6Ki_h|og;b|5i1>PcbS%vF`5b7 z2%~lJcKoAIV*j5FGEK_>Q4}6bmt@CLr}^%*I6rmgtP#dX`YTK6^42vLM()0BaL!&WpHhABU61wWv2O!9#$6Ab*oGShE0F1bJUmJ(6^$GV;dUvS4s5?J;C?f9 zXAbnjzwZzH^ztBjd^gYdOYznoC?yn}TkJ(mr;=@r6V>fEdM>(qK~HTsIa|Wi6Qf zYWl$cdJToDXNSRu9uVB|gx95hF-`=m)@)aE1-csrY$a?aZPV-uDQUD4Ygw~`iYa&I zsMu}}?3R?ozjU7j?u5ymksS-7eIp;*ky%+w6(in%9CJs=49w!MbUuGIT#? z5P7w4X7PK}$-^GD95a}Bb#5d9qM9V#kH(FjIqxkIdO@za<|6{I@yz z6594S(k+7Qi{l>>1BgFTAUo|A5y;ol3UK44HLULkHpoh42xOxW8P2>A;T=)Axp8yl ztv^`@JK5M2n~)`p7_B5OCJB&wPvcD8>{PBrcz)Wyc50<)?em5Kr_Xl$F>vi;f)T0! zOU))*)I1wdW!bMQEI@I3QATQ>1AoaTRWS@z;NUwtxFB-Llm%;pI7MtPgKJG6Cpd4% znmPgV)JtHZlJPdnT?vB;UL$lh9h5g?IUnBx2%Ttoel?WER*vkil_nNgY5e`404 ziH(hwP?lFw$vSQSy#N$Za6rR@vhm$5lmTV8KDYr=#@gCCuk2-UFaEoL*j1ufW>6>m z_&z1jj`X|0=OK|OyHRf{K!LN;IW1@2jH=rPKT@xT>pqSX2QFB40(+`3-CFJ$t-0L; zJ-+4cBihA}(3TMai`qreF7xml!i2*UCLWue*THD-Lpx5c>%xN-XJUAwTB8S?jUBIM z;I|Y>0SRO`krJ)RPajYxzcHJ+Q7&^m@Nrt^1mS%*SU8yvoRVx9Z=a)mE@vZgNOmxK zehgE9{eA?w69$)zzoYdMhd@9o7on_(2x3%B3}X@XKXX=^fL(Z?!_Do+af0GzR?&-( zm;xC~xbr6R z60pKydGO3Gx2N=5NLiruRJ!`a-THfI-?23cP?n(I5LP|^5LRGg4;yd#rtVVwts`vz zeQHOwz4fjCWW6AfgR|Q#eI(nUgGG3Lm}{Q8PvZ1QWri$4P5l28&jH*oD|ab2JO)#2fr9CM=0-I zGO*on8vg7&aP+;A>}EOF^%JT3ow>3AQ3nrMD?LyL1mu?poc}txLOVdXaTDAKfam3c z;)%m=`|cNUru8R{Qm0EvjNgHR4Q`7ld(;H{)*!Uoo|&mw<&;X>TlSiKoO|;UfiVTK`@;PN|~IJK)fee-r$3R!J(XK6x!w{um}^=^1oeXp#Wy8O?SG(Ls%p1qytelb zVK6O4DzS3t>qLGIelu^N87Nd5^*tWw`NFA0NuXbv%NXE<4*$J?Rpx4@4m?69<;z?! z4C4!Dl@gI_Q2Gs8+G`BDdG{$$_qeR3#em5&_mFXM)ulc^d1+wPibZl)o2(nBiv%k#Hn{Aa5rb|;PVs;!M;`fdQe z9v>h7FZrl;QQCnud$iA?`v2mzm-|1R_FyCT8#X$vKffduIlT-Nn84bm$dEee5Oxr0cJBwl%rBgmPfaJzrO~)c6+v%!C1?V^M5P z!OBubRm@c2;>ddE#K=g6oZy3l1Hy)8XPE+MvepR~&{=`-NieP+LMZ578#ikxojkWN z%=Is>z#R`f5`wJ|)bccUydzAav)7W4G!TxW!SVJ_ZJz$n`{tGeEmZ$goR8PnhIJqa zvrrWk_1WSoY6vh(h+b%iKa0(TAbFF@{=Y(s{?$`*ZPZa;^Jwp(Tn`s&$}HEN#`Emf z{fHFzYi8Yj92z*-H(x5lW1Upkiwpq#LZU%t0lF7WBnp>6Jf_J*|K|?URJNSBx#KpQHv}2^S`IGZvl*|gWsJH-eRAvoea>d)bJPmFEcr8m ztfcwH!i1Nu0O?a~<0eH}j82$-9n7lk^Cst%`_8iS%AumYEx%c!x?rGu@Q>)Ks^E7( z3wXJM#j)dxeR>An=sMgc0FVZ*2t1MvsCq2km;{bbM6l0=j87U zcZ`~T-woxiB;m~moSGlr|G}yGw5WJdV#I6ayhYA~lRu7oe9cU&0CEN=#)3A3HTuK%A z`{u$8QWm7I{x$&f?b_Z$S2q~o#vL^HyH0zy%D;$cI+drYU$GXewtAK~FT!^H>H?<@ z)7QlyNf65#Eow?#{7mx+K)c-(IRlX%hG~Kq2gGMkNGNxpqtK1!Ix9WE6)KS`uKa}P z%fzC8+Ysb7DY8`7Ta88#{bp5trsW^wgGM3+iD5~?q!P`3*8&0v^i)`nOFXVTZx_*4 zboLb+KqAM&TeH+HD}B#;FAR(5Ln*M4aG%}_lnv$q3{PNw5MxsniFo->nIePzUp>S4 zV4vcQoEY*FzQ^w;ANQ_0f zGPUS^)Rdh&2tLYt00K`%dzkiXC(uHNry);h5U5g_V(`k86W>+Aj0D|S7_4wufFNuemQv2IV~;vpQ1AsUUW9G|1CPnPxR_MZ{X%a^o*B! zz=O0$WU*^~ISrvN5kcapl)mKh322xkJvv~$mN&`(0fq@TBqeriKuX`ljxT;}0Ar0{ zfmcXIMP`S2Yod&ud~7i&#E0AlC{P&gr0f=Ivnrn$E3)9f&e{0x?a{ZKc1ZYLQs_;` z1$vB5_y*cmMp$PAVgPu9IVR9>eF$~C4>ttUx>?CYJg&}ksiQYX3mP67f5iIi6TnAG9H3xh^jvEp$m(?ul#FKN}3Puvw+J+-Im@@y{qKN zA-my@iCNUt2x<`>^sB#^sz+z1F>NI~fXszT#B45nK7VSBjyE5`Q1S_22(G`Yr9@KO zi$7}dL)096e z)^R@^4r#>MbU^5n0Vnk>EWrvy+`VfE-nD9zB&~`47S65)@=z;vr=8j>XA-W=9dO)U z30FP2KTFj({`l;W2%{2r3RoA$>A=Xep+?2U_=^w^$MX|(2BI|Uy2>KF`Hk-|g6Eq? z3|M*A)0Gr0U>OOOJ`w4gjTehQD4KSm#8KP3Tn0hQ<t%EtI{<8t9u{VXME!e*TdorBW9v#J{`H0rw~2)3;m%29iU2+$G%y~s3I zS|h=Kl3+!^O&y<%it+RIZE-`0kg~}^Ou^m0o}M(1b3Fne5@7xm%cgD!q6gc+tKKvk zA%=nutZc!#lh!P7xV$E_Z%S^x3kii~yfBWZP>2HtTum|W8uY6IZYT|#ou&L zBpi6h$a)kup3R89YO*Jr80rdpNLT=>w-9T^>WC^|6D`qc|H0+0!yZW>s>SQ4LZXVUDX*Af^t8;6w z+3Uq3@^1i0%g)wc3~`Jmsc?F4Yz?l&zaI-P>9F>x0uj;NU1lahzK9v3(Oufak3m5Y z0wVAlqo(I+BvW-ERDpCx<*04#7(F(VKd1s*L_=yLPbBPc-xCtED*++2!CbXXegXV} zN`9jz^a6BY4-ndhr_YXXGR?xl`}7)A9P+U&!S7lGVM~d?=J4%Bv;?ji_?N9|7g1l; zyB?DQsltzLB9X}oOZme+qdc_=MS`b5H3R`4T@bD?88z&22~qyk`8!wR%w3@X6W!6= ziH`Y?4(LOG13Jiw0F?aMISAVxxQH6(lH%QZ_Q%$!Mc;dNw?@oSy>?yevD;^z?P0D* zY5zP1mhPN0`vF%1fnD`3OTv1tC^`*dvk&%evXyHyAC5MXFeATe)T#;BJv? zXed$Q(_d~0AQ+n&A-Qkv*sH4ew9eQj2?N9L&Rx_ns&*!2v-pGrAwW2ilb7qu zE*0ufWg7Kqnm%h9^(*lkIJr35(vksvJZSF@?GB8?szu?Ms9lzkAOj|!VI`*>R6nDf z23>Zh1F5IzU;ih=Nu3K%LjW|%INO6FgM%ZYCJxB4hdU!ME~(6a_v!`#gdYhxOjG9s ztmfW#Syfri))(5&T5>_!iRj6azCk8MxcAqPfjhtNkL_+pN%Mzy>A|z=t(i&eljSRx z4EVr+Rk4jckawjIz=~^!YNkKJ^;&-TvzDyKeG+D|7$rYjUp_V}r9iu<6pfup&UOtJ zkpI7d1ukx-RobXv_>EAuCp3{M4I~j(nCr<2JXUGRCG76)rRlW-d6tHgiE{}5G4^pI zFr9+T4L5T6)N{#W$UP%u`v!-h`#__`a{o=-Ger&iUR?;Sw5lrME}1eAGjA`IsTxpg z)dpwY4%Lj#Um>;cbL^>Z5(5(`SAiKa1As3d$xv9ho0ox@8S0x6al3-h`c4|B`Nk5? zjnWXn6u0g&VbJl*y*Qh77GO9zSe^D3l_t$wpV@_>sh$jcFWjmGy=&Ktg;Y5ZysP)G z;NAZXR#58Fc9ZY=zkH9$Mr0(Sm+?I6d$z-k`Ho*RETj2NTx}xufR?IOh30ARi?(rQ z?g&+&Iy6tTqsNGU`Lt9zJ}Io^x#<3bo_K<(=x1k!m6rr6PeIY1oKLYr%O!8PS`&{M zT5!U`$Y^gcmSkR~(ZHA>Obu4l*b^S=2(*j|XaI?+|KJaT|4e`*y!tm5R+@*!ZWGv1 zTF@xa#pJ#gv{~|dr0x#va_QxB+%4rgOH%3#q4$7cPg*r==hS96;zr zhpu!+XkplRaD%;EeA+TFifG?W-EXx7Xs>Wpo$o+J0l%`X2ETLo<7AYYL${m8M<9E9 z7@{i6)wF(bn|@ln}4qxO21MTIVp#s|MWaZEJyOlg1+`Hemh!jBg)qM1B`8;te2W9 z-@hr!fSM@uy)_VDrHCo|=;zqeVBd#3vOqZfe(5BjtqLA*IjsKl{F8(60E^})m}~tm zj46w?|F{(>o>=3&oI0ljU8!|Jx9^R)V|uwZ!?H#1XR#LQ4JuJJjbuw-*zH2h<;OMAD^pzQaT-4Z1AO1nhAfshkF zT|>xY?XjE}ALG8dCsDdJv$JO!Z=97U-MRjl^TJO6Zr(DAX|1svIX{3ajC~8?`b=k% zYwMRTvdB=WNf~)kVc{nb8>j@adNAV$ ztZ`b;5O>3?Ja#4Q?p^Bw;j{`UN)KA1P1a28ChIEb?=W3Ca>Ium2n+fM%2n1i>`m5- zUzY?7VrmM=ci|-g8oVS3JaeiF{gYY5oDih+A7C};z0&22V~1q?K-WjyRp2>vQ<{e>5UJC`nx?8Zbx>ghwe5EaG?2@J>~-3CtQ5fg zevk4V+j|UK3+>WOSOjLwjEhCo7%(0+Bbr%_!Kg`hX?L+5>28eFM#(Vwl8qOt@BL6>Mx{eZRionGw-5qtQ=>JI(kFAXx8mry7wsmOVt=&VauDF@jk&Dg+H&pi%aooXfl(qinXm{*ymT*QXqNEH>Rf9>lwSg<3GpO0So=dMA`J$M5&j*u8HP0 z1Wc5@f;(YjZaY(KY-DP%Mqr?90s6zZ9_EU^Gq|R50^~h#FsoM<`poTt43H$&`V)lR zN7)cV$T4UInM;a@NV886phq1(+9z;~r#=O?u}PQ3axYK)DZ z{|ll0wvage;ik`{6F!A+gy)nkkY7(+{>SA8_UeX3Lr7eiaXPBi$YRNA$IE@Qo+ezv zo(siF#C(=B4w8-QI1p@0N15vj4r3JXD<)D}eE$ja7uqwG)B95wf_t-zY!%#_@QBbt zfl>JIFZaUG1H-MX+21pA`Y|{p-#8jbut8ROjh}F8a@@~T`+~lXIYVH_Y)d%$p>IaMbv>{Iw}FlYCZ+ApDC8|MvyTeII^-dPA4&xI<5;PLnCm#>#G?-_FUl9Ow%`DjGcYNR4hoOg&>Xff_Df%eK_-p|G@v zkPubM@l9#@Q<`w%o`?)I0O-!fl#n`&Q*K(yeC+c;TzS%-p4jSAk*RI%Tfsw13``6Xp@aUG6-es>DsLqY*E`wl zB-ot6S+_{I`MRH+2e!D^#bv+;=zwp$c!NO0WGGd(>%`$kG{0hc&JYg$uzeQ)`o?}~ zP^s}u;12w~e2fMdvp8D<^t1baaaUevjx0S);$VH-O&k{RBL!|^e*e1+@s$E_&1On# zm9yPxnTJdi*+ik&bG|dJ{ChcRFI6%Y$U%2*EAYhqd`u&63)`sX3C23R*nKzUMO{uY zSP$@=j)|E|C3}`cYE{61>Z1%6_0_eoA)xn?q4e$ai8efqhHp7iUO@>|wD5_)fcwyi zabM$FC{hOo0}R5r+j1(JqOCR0~ zxQ^}hO0=v?2Ybf!OyfXKW8o~DtZX>EqHbw+VL~*xy&DaxrGd& zR$wScN!mXC3cus~(~42w)5EO-x*)-&lkypcVe=NOLv7Y*w`b-Pk+`gyZ?+~Yl)PE# z>65@)mc!V3abj2d%bjO$oYymnHX~iWE?52V_o+%%F{MI(}QV(l-y882q_t_Peso9C{kEQKUHx6Y7 zXK3Z5WT7-Y3I)~K&{E6cE5BDw9eJ0@h6+E^XUg4Rdx4!=GEAo=JY%CTBykiinK}5% zd5;Sn6bAChNXAMM7$h2k)}jl(J%FJ=&_*uKgkyC7I>>kWG!S_p?G| zEt`H!9;mDjBq;+E*KG=sHk`7MV0cEu3TYRLmIb0y{ zBJDR{)-(3^o1Ak!Y`M!TEAl|VMUQdck0aq@^q=2)>%K5mqu0w`w;$iTp=U?3l06i= z5#u~5!+Hv6?S6~+*8JsJGkC_EE1|f|s(z?3(>0EG%^++B0GB4Pz2)j&kwd2h1uZv2 z$SyHP=GB0VPL9+9*VelEMy6r+go{~eN&`wjX1*J`bEf8f{M>QYVx?ad+i#+X>eJ>f zkr}r86KlC)uZ&en>?L!5$_*{2osS%I`&}gXvQiBYD&d7opgLm2$DTPWYmHsv;Vx;fN(>K&yRn5n9mUcokKdg?T#;4=|#3S z-~NEJCndaB%)hI_>4a?p)+{IH4?=3237EMD!yOdvmUG@5np^ijQ#7iu1O^700BYkE z-@T$V0bRd>0W7W7QHHZx(-{8Gj37Ja5g6{JW|zQi3)jaDq(St2D0xHPFpUDTfRi(^ zYt&sFoA2b~H!HtARr_?W=NgC(^D=(f-EZZ4LL{+~NxE*eAHxFC2NT{C?-!%9*93JT z=5*65YYfH7Rh8c|6@N)w8E763{l!{SnTN!vq_Y{hsg(G>kuaBoNw3PIyS) z0y&kLpyu)B*9LmkeAvo?LlgR;Zg)wqV4xJ&*4>x2B9_d_X=Kb%?hV>XZc+ZbpS~Ia z+<95%9_RI!svZn%$-NU|64WzbB+~U(khk5mYFip!KM#-Z~C<+ zO#tl0TY^D}LQSCuJBy_?0;#p(Z(&L42)JAzl&DM6EGC?Hu!bd?i)oPc}d*0Wz zK`P>H0jYSAGcZyN%L}Th^n-<%>g!i0s`|GTd<|Koz6rZ5Wc0lREF4YL#|B{5%u_GO z@S?pObU?S;oTy_qRpnxLPU9?$mNxY0vM{nX>6EG+)@3>BjXYtOV)f*O;+Oi2>bl96 z*@vw!NfSFzvmRi{qPA?${zFexaGJK*TkByx!C6tlR!9X0SGo zOQ6o-H(Iu81Erkf0oNY*dFLY1L{?A!8m581%O z@j9@KTyx;K&%wV}8-0u_ODoX3J>S_>>3TOWe;}ff*>K2f{o!)CyJFIv&OGoNZ!@Ss zq295a(ZXwNVDs<5u3|Xq&+a%2MjG-@0%Ul#X%krhvzE8Jv+??X#0Xk}19G#4?|YKX z`A)*-cTbIfHfufDeGmx2ogPaxMIYi4HR}}fUHSD3xyXP)2^?>e#`l9JN1I0SRm_Xs zA^6r$x5i7p%y?XB@cjC$m*sJ_XgpI}r64b#u&c*T^rFIoANUDMOq#r$5$>qJAfLp< z=h}HQk7U&2;^T6(Dbh}COAO}N4=TvYvSx9jp-G$E-`~$FFOQZRM;*a3C^lw4wK=u@YIcS3n{p;N#1R-tAKi^n z(y+&Yo~~se(2YAs`7ek2GY6_VCMEZZ$ziM=D$Ru*bRX}~(5`fhO-4h! zehOian6J0irny3)!YHSbx!xmP7vh1&v1VPn8pB#3qhE?TqVQnXps@Ot{`tXObblEU zDhxG=K*-i3|ITuhr?Ho(6o1~xq?PtyFT>$|>4N%S7rq1OFh>Ue#C?>#HJ?sSziO}? z148nok05DgwVqC+s* zk{s+I)!pC0E7N7(->nZ#>RZ@K#u|fp9QUY!0EC&^^HovqOLA2+ z0@m<1h<9oInQ5i-Jyg1NyA_0E+^|_Kw}A}}Czjbhh_&V`??u%J^M$P!KhVJ%^%C>! zbx^ei>6jH%E+B)vERm;C^`3=yQRcN5{YOj3L&%c(y-FDQxhHDG8A+arQi zV{8%~9m-%j8n+sa(WfAKN16DJW2IaTDQq5WymypcQ@^(CTBDhM9N-12iWq&ym}zr?%wRkk;pQx& zP=EQZi=HmIws~Ukwt#3|IghLvL1s$)FLOUHdxZ*1u>opiT{fr_q0qCmQMS$7u8;TM z?XHe-NN<_qoy&X~FCGo6Tm9b6lIwwcV$IC#Wcc#2XwzAfuvRLMX2kU9BI}z)1OzxN zc>9@BUDs$%Hq+F=uEuj&UK^f4h< zPnwc??&*fiZ8Gp8RVI*0)*NOm()Qr`u*_D5M^6h%oKjaa3F>niECK+czZi^RRZvtk zs20LxEgrJ?2~rcuiy$%i$sIlqH^5QFYwe4FV6*zumeRcLc!n4FnPS3F!#1^mX;^xA z_-t)Z8gDSx2o00qMZ>9SRmWojt@hpo-qd1==a%mH8aAEy40LqZ_O?e{y+1xnc$|D) z4B>sLU%V7}*0S|3mKkQ5BB~+5tTsZ1R}d_TMQ77iancv=`881sBdeg+Es^dNwsM8z zWSy{*c*W=`dF~{|vxY5Oo|>BV+2{(^*I+zdUAAD^@6i8bx!V1xBvGvT`{L$Q6~rsy zxy96{_d0;(u|Lz{i3KrD`cDv;g7~h^QSM#=d)#Y~6)DkSHr{Ym^i03w$x{j365C%8 zT)kosB3=hCE}Ji`^;c^dM&V%KW8(Mt+?DN|gC0&Tc{@W$z0p)9qhBb+uMKZ>+<}Ew zGI|ktEH%@pbjoTb7M65SVg_D*(cltQC!-j)e!f1~uJXKIg46v+ES{zlg{YPDR@vyl z_m3!PY~M0!JVg1*2P|sGiVOE;BOR4_`xUsbDMLpg9Fxyw4x@(;d&CTkpYe$D@w|IH zt4mX^5IfGM#XEDReGl@mhyQ=RY0P_C^Vx*Y0XdTU_G)`q&$AJ9!R1LrT<7^ z=FotK;>Q$}9atmHfGaY|DEuL=VvNwvHWTd~aLr`#x#b)oNfrF5`mf7PD_%iW5?8dt zy7QMht0c2b$(Dwi8Ez%24)V`6r{okAgxwC_6;rhY3rA1<2*y(rCClqEV5!|1E!7vl zDEiE_?z)l3`QhYuZb@%?2np|lqNRD?W6T8sTsjbljs8l%13LJ@BO~Ft3>dX3kcWoL zV$F>kV5%!M-CpUPEP{(yEE6_qu)B<>b>3Gh7)Kr^65wXdE86adPId|DH!|mA!Zt{v z(qalyui7J`z+XNqv*`>W<@_#&#b1mc)sCf+NE$*Y(0^Do7kG962BG^=d6Xq`P-KK9 zoCs{;Ta)6UC?B4NJrv8X(VIj|d5v>hlBvqDhrwkzDrPl+F<%Npt^ROiBg?(L_-u|4?YxRS?iL%ZiEs(I7707vwY}ez5p4C*WLhUpTAUU&SAs z18xZ%f7u52i;cM%j-1Wqz;Wb0 z{iOW~^EIgLh%?jf@N&x2VN>>YF|vfl>`mHX3Kow=d{4T6sm@a1thR@PO$%6Z^G+37 zblTn)l{)Ah7)qRsdyW(~>#gADh77UHD#Ed|#Z+6f=XZvqem`pe_D&a)*IOtU6L-WT zdtZ@p#Es%K+$zZFTS!1PnGkntPp0YM@zeJFcIjt+F$Y;l)`=xnp74_N)}=(=n9cd& zVwU9Db6uMr7dPA}a|SHI?))>UfPw>jN?x>@F3_`r+J}%ba6wzuJmWa z9O-6rqo*T-@hWy`n2joiu+-xGnaX0nyfKS?V~5Awr62ZW2P>f4;X!;FK7yl}Xh2FG_ZNc*8tPZIRl2P< zY2(bnEzuKP;r=Z9dwYH*CvV@rjqAm{-ZtZh@n%XjQm&VDOib8f0b*bR4oD9Un`aw0 zz;k~)0gh+D%KI14=TqO`CJgy8cM5S=!La#cBuTe6Ek@*jEw%gML=jVMcT>RB=v4@$<46@Rlg>N4;D$}q}A+uM;v3pUO0#{iy#;AIoBF$mP^T5t z#QuxH(|n^0r=m1j!BV~MG+G{H1W(kL?@Pqd@U2{yl&$zdnncPbc=eYucQtAp{86KD zaF!{F(j00}D7NbvLTy--(Eb zc-J(ym(=WrYxGTXInai5`X6cEg#UiPlEU~@^ujw7+XJLC#^2ZRJ{y7_C``aaHb1{Y zDtY~@V)X?z7{__ZoL1Vf`Sa<5q-)}9fUqs-wWvJH)cCmNDyvhotx&)HE_=F;de70r z+SYVppNpr^5=k-tjAzdgSMB+REgQ^cpJ?bx+g$P$*#aZmx_t3HTl9CdZBXq#hitkF zw)9L+POTt*nKEfVk*X`t?7i3Am$^$j5gi-*@~~`HIvEX3kA_jP2bIoHg@OIMUGywT z0S_6o9F(AK#V3#w-%;Bq8%Ewgze{FgyysV-$Vw+26M7huZR*MIr%LL97S$VTMldiV zcAOrDB^ux&;lY$v#lDiRo`JQP6N#j=_z-Nw+TY$NH$cQx=&e9z1J)>46XiYwSx)$1 zahEXw=*$NdZF~_pNsp`-*G4y*M}U{`z)f26>o%x1pCCBEj4~=v+%s3&WwA)VJ#SB7 zWVo~#$tykF_ki>XsH8Zf*C6MMDsB|vrAClV|vDNeG1)Tz2-$`<4EhLnjd08 z%8%W+kAC?ZMKhD=@qyR51c@B%;4v}i3TmSxP1LV!31sdOP8q47k-39mg^R!z%gmvZ zHexIAs1`s!2fw})42~6HQO43o7?cMI=<07xk+apduW-esbslhR zS@R8VPfDF}<_Br7barBinfRI>$=i<;dKUDpU?ti*bS3AD1t#WMbtX4@ft+(28MpdyK7F1z#>Xfyb=)UaH!LBVE1I;J72NL0@o*(pz3LVe8IpeV+$ zZufyJpjLOwCOOmE)ccho1lrJ#Pq2QV>I2>!bAG=gBvI>c8v8@}01UAQRRau$!DBt= zuWQQe;)=`>97tcogAYht{=EY7LPeN<^o`mC+;%00c={azgS6RBwIq-jx6NC zeIYTizvz)R3Q=+Iu;^+FWQN_h#=YH13^TRMbvl2#5Z4%r7{)}XpVS{tofwcC7&it7 z^N8Q|i~?(l_;WD*I0Rlm!(*6V#`A}jh~eVk`11sTZPNFs^I|cjLn;5<2VQ-LiAxm- z#+U7aZXgMVu^xMCr&TYYx0-<2f>_`@CaiwPsh7{zuj6^AQZ@+q7u(422^~ocCgSu< z@h!`ZQ4g71wBG5T=||`=Rudj=kKL^dA@ewYNo-3xD@esF0w$)1ieAj2eyx1}o(S`2 z$TkZogo?#Aty_nOl0_K9*5r6a&-eQTQ$W}ydw5znlw4~QDWx^~{8#`Ql6rNXAyBdm zThV$Uc)pX686)^^wmzl94_@Uj;9FmUNrNt02M2@f{)n+|eW1`8a0zmP?|SnemrekI z(+y^?u?ngzSrE(+l+M{sG;>R<)`&e-wt^iDN)NujeW&HIPX3ft^TTQ{NbG4mu8|s! z+UY)WW@Wc5HZl^uf{K)L9uG*OfJ&+bz6l#vgJc=Lqj%mz&iZDrv`38n)^sy)SBMlH z78n;8ilFFVRHfz_uYn7RDm7mjSi*LL~O-xogb{`AZix_AomYi;&2 z#rv>2lw@85lp?LMe(OQ+eSmLXp+1ehDSwLU5!dykh^avE%&v6iW!+`dGd|kf~)U zCpFb%snQnL&QvUQzw!j%PI9WlK;=btuo{->x_RHKApi@p&4t#-RWrhdO^$sA!0{i6f@*(W)^*`RMhM!FZS+ zsFFs@k#NelaA7qC-##GM5GRnAKh_0duMG=Vu5R3iPB*BO&~j|04yAL@4a%^k#>FIEh{fW2 zN&o<9eO~=mOY@6`!dN~MsLzwT>>2&gNEbc#Qqd4n~=4tcr3p{LbPz(6!My zVwr7iY>gKp{g-xK=w$G|nM7pBa@iSEM+S4olWKLsOD%Yv_3F>$mK=C`-RzV@-e6WK z7<%<}3;1~Kw6or@#0EtbA5#2t`Mrp47=Md)!&0n}gDViUjraJ^ah##`Lh`Hxj#=kV@tBCHSXUQC^=!^Ny(VLczFgO zmP$NNFj6QmrB}r)wks+zQM#zL`4$lsy7q^Rl4P*V1`_+t_3k>XGf?MgR1iR?`jP zmj-}^EKS80tA&)kBF<70xihp18<`m6yR1h9uEJCq>qmq#g`!gwWF$1+CniROfS|CpZ#8Q zei+csPEjuhb62$|rl~0awMiCVS~3${mPBrg`P#rEudwi^-M8Wm+!0$Sox$!RSow7w zFmQoFTs~Hv!CU3iG|#wBs4(hQTwnGiB6!e>))bUz*>qwx{;suN!7ca9WihFU=x00I zBFO_U>=B`EwijWairtk)NQ=J|yTAu$ms(5%QtFQ9`|OGot;QmESAcs#T8`Eqh`op~HoqQOes%+@rcVG`R!s ze;xz-60JU)>XHK-ujo9zKBa@16R&=k+YPy9DA<`#vyJ?z&IQHD(faXq} zfY$gyr4Cxf-mR1wOAAPD;-d|ELV4*6EqyTzia$q$Vvg3lO2Tli)09^-QV3y}zlXk= z0hoY@f=v1qvSsX4g%W!bk}*YG*WBa(hqbqis(S6Zhec435D*lQ7Non85-F7wP(VuQ zM!Gkp0wQb>0cq)w+>|s(vkB>v2I=m8uf4%@y#LSrJfGgd7ta_D55Ftcwbq<-%~kip z9Y79WPp4)(GQ0Xna=#_NlD|eiv`cX9+g{Vo{o4hp?DUZz%J*(x?K}D)p<3#M{@j}R zKIVXD)YBJQK%H^+Iiyk1K@i-g$M0&g<`a6C!S498vRABenGF`iQH{ou*qLk}b#OL> z1@Ut+RzKy1bThLD23iOrpJ{ms{tJcZcZ8nsZ~Eqofv;5m9Ns?!&ev!pMT}(A=SdL( zl3VQM9_gdgTvt_MmqDk5pC3H6m?06`69xzfKSmC%UM~s^JD~pF-hdsMJ0w*{g`^}& z-d+i;J;=B>ekC_6>y`*0jVHWd%ViesUbX|%^KymgSl9{9(*bD6l8t+fHqfBie}2Ir z<6}utuaub8A|G_32nmQ&Tp!6`cqXwh(?yCTIc#U>YO^hzwxN z`n}{-$EX}`qd81WOugC&ZT2`F1xFF5BY|H%HF8Cub0=LfLBTk|!|^Rm=>$Xw%t{#S z6Blf?!qa?!ROX7NuH1_I4+E(Bh%0BVR;s3!SO7T^nD0SqOhFvjEN3;;3ci=Pjt6af8;7o@3mE$&8r z(8aSYQZRy}ZQ6AdLb@T~gD-?|H^x|ZGXWq2`G%P4jwbZ;=^v(k@fVXn` zz=18?&uoptPY>w6tv0DyrGm!TID`uONrJ)!g9(07^D{y4i`JJ9*$%P4?`}1YXYnsN z7;D@aMo+jOqTo2z)pzjJFgAIv!~gKkk*UyH^2<7yHhu&=nO@OVRnLR-+0pUfT)P*l3a1T*vVtC1-NI17<4Bq2t+dOb9vt~LJ@ zaVVfI5_vdnSw5m*Tm@Tpd2sAij!PZVx;Ob{1z^9ax+l{_p(tZ6^Km?2w#`QloBeJT z(|Mxxh8BT+#J+mR;CT%q&_*@sm)tJ?kKksPK7(hr{G5;xS zJupw{8P6ACW$~DII$ka~@2;@PJN`C==;kwn#qH-85)3aD{8t63E2*~ZILl=$OHIHb zO&J#qVBm@6giW+R)4nmD8;LgO9#WzLALh8{s1?H9ce$`msU}3RVLZd|h zG#hvD$#*#1va2_%nVD2K(@M?I11h|QfkP=O%=LV&Y@Wh(DKp(ZJ-E#=r)jmtV(|pq z0Z3fdy;s{d&@Z^h?JflOIKZX3h0nr2;lS-uNrBmS&17WOl*Lf(fq>G)d;fN z5sqMi0DuJaA?&5R?LrpMYY##39Jf2J>L@agNHCBWW6t+8P_Zh*{MGQ0!G4~>GQ8vxjI>I< zNRp9Y;=-4%*iac`qezRL+X2*Hz5Mk317_n6WAN*Gd2&K@ ztc51;J=p5WOvBxo=!lckjI6YH3F24e*k^}8|CU@EgDhu8NYXGueeoLCgDB~)@$eBc zn1sZ!O9Ly2w?oR%C(dUakTd6lDZj-ZSnMAOowr))7lE~=rmrMXmd(I7z)SkG22ps$ z#*265o{9fT7t_DMq=t-fnf2h_#<(deez8msYMiI^j#f^)?9uO$Qnsw=whlyE&Kc|j zJ4c?er&1`0d*;QAXtl7Ls0Y64$WTunrY+9?N2}y@&?Eps1!*EUw?FannXNtn*Vwk` z8X$Af(4+a5lyK1fQ0_O&#nXd)57|`?fx{(m1&x>I3l3+UFLB-nF1m_L-#<5|a<+j8 zf1EVTX_T9jbGrs;8t6P2?_i*{tnbHE3rGlh_#4=Gi)2sH2Nqf&-tSjvz(t}Y997K! zCGOqc$@U2SGiz*zg=+ynyWqs)-@S$+DSmgYV9MLmBx3~JkNHWJB9x&fPa)1|&6i{-4% zLWBWSCy~5(C7KOogGbY`8X|4uf$*qWM1{pj5Twh|LI)+h%dg1C2nYR0lV@G3Y!{Q5&_uv6lFl6 z__LCilB65o3L&4yPPe?v-z1BoVSE#D>(CoK?p}pKK>ddpNtL^$=6xk@tNF06$XrZ< zzR0a#`Dj97hC${YHgAz2Uk-hyC?ZzM*9Q)m$TvfD;#g6E^BfF0A* z&_f4Q-u{UeC!c~5A*KV^QWTL{e2dS<7^m)h6E2Xfuqt~S8C4NirpNgv_+FU@hutZb zY7aKk4nM2(@t`b>Y{eDr#0Q$3$LFMQg;gxcYm{2VXHuZ^d1hOmJ^TfjEHfQ|hw2z1 zzEOF#+H1)}ptNuP_vxqBF#2A@NbNf|*r0dd-kAG#hq8^a{t(dHBh1sXWgfn;zYcUG zA4C5m)ztC8>9BBV5;XqlRfqakT+Cx#=umt=w=2OE(~V}?bGzVjyA(?kJ}#207C3{S(s z*Z0TJ0a0U&*Z{vLa@AOY0%fK})${|f_p8iq5g5nvL#_xM zJkkG%swe9bs;%#Dze9Je!2mBzP+l(07=NQ)W1Daq=Mx4B_4@1FbhVmG8ZWfwyZ85t8({{9Ak2=(ul8 z7@pFe+&&4{Jbj)Zo34C}LUaT5T;N>cDBxWLI(t8OJ$gpc-md&6`pJldLEN&m0^z=QM-F0Ck#O^L??PVawcnLX=!P$Jy^Z3OywMagR zm}M8EM(z7~JK+TW-`g(2BgpT_uUTfHsG}HIK`BTr9M7|L2UekU>s^Iw`qr<}L&y2m zi=!%atOtT70#G9ysufl$NRQFn1mIafGcd)17TXW%jy*sHbS~i>Bz96XpRX~Rx%54J zHnrt>Lb-c&VRrnTXGN7Rm^b{`~=^9GkmVRuPG12z-p}J+%J$ufY?|1PZ>h<00`^+9{KtvTSa$p6_>Fe>LSaFOa zmaI2g2K=m_F2+dS5$m5ruD+5;-Y;c@<_6HkC`^bHBb<`{^zb>8)r2hZf#*In=|d1R zO%t*P1D5Yc!pf)eX2TuVkvs@^wt~3V4T0|;g291pH)fD=Y2GVLr3f>ap&Leko#XUUNLoR5&75m|poLIojxnD&Q}*32YJ z1tS*73ay682 zcCHB6tV9Z!Q=%cGegcgU*h@B*{k&;oPyJC}MZHcqTqPb2Yq@N9zxWczdR-Z3mT$7B z#GpypVimW-BxQlP2rmk-`ABb4Q!OXN>D;^+c|TO>O=)&8)zO+R-Ww6n_NeX(Yk7gE zTq6vkJV1e`$=CUt0vQa3I~c+0)uxqfy2f9onN=FBr2MTBpw0rB;al69!w&0(F3&SN zv~l1r9SJ!eooK>fBh@EEER$)G?eP5pTh|J_u*ELoia*LepdTDl<<>kp5sl_?xaH!Z zgLzE?NrGSZd7?Yq)`UJxtefdxlQor1PH$zOBCsbLTa1gK6o(?YbRN}0cQ$gAQFmT1LMw}2-vKu_DzFQ1;=n)RPgyIo3|Qd?_SM1ty%)(YcGC! zR`W1lI;EnN*04oq4p+SsjS+6g*Mrl^FL7H=Un&oE>xvEH3zmQbArL$2f7f)>GK0no>JOqXss|s zX6nBGC^Pye!IH`MD-%&yZ9TW)lf9#{RWs86)1k@a^oOgJM-51JkSp+!O&A`-S2 z(t&V_9Oe_aXl1;D(hq2vC$tsu@FsKyV^$rwky`^68(<P!8K=dXd+=K!D+ zGED2Gs$G#5-*A`x9I64%;zte8zs)3Vh3kl&^=BpEEHW4UW`$@Iz;nNAZHLI`z=bx! z$k6dDJ{pPNk$qshqMa0F37<#{?>eErZa?g3h!~NSwvK)=An^*C1)@a`f{t%Am=8!eKTmp8>WLCL!EpHFLs{m)NhullXL)Rz-RHE;=@r72>- zHf2ki)%=~7@j>N+K#yyW88%SYlY34@+bqN*cY*Kz0OI#?1tXhGTb^myX0f5eH0dka zs+` z#WJc;i!|0ip?XCGdeI(P1Vnaqx*fLrz$@0uWyz$Ph=4Zc`}BvF_12Sj7G!|h5dDD1 zZb%1`cpN=0W8jH?*?Zx-T0+6|p0A@u#jK`ve!n{<6QMLvW*|;-vn~T(Ruau2FM|d_ zVVK>3@9H7xaTMVzl0UK5bm~|rf}`Pi1sPsBV`_1iL1LkdD*^B^4f+b_OyuVY0c|~T zR(!f+KOzic+`(rt!x)*zpcG{aZi}pR-s2sxnaVeq&!ZOkg&Ew>GK4b`8zS5CdA)YR=F6Yi&2D8xhhmO#CM>c$+iu zc}3h!AqlxOvE$WSYJcNjGG=M&;TzE7NN?-$L$o%FC-;WQc$(BoQidN}~@VJ)I01G&1%T5FR; z7UG+-MjiA#Hhth=*xcLP@+GDoG#1yALav*C0IUi9;s5M@fc7;+q~i+3DF`j|h-clW zWJ%nga0bH>$%*vucRnNM*T}@(WZN4Z`#xb+l1>fRMHUEO*`K6WH}rx*@A*;nZ6#eE zQsj*xzRzO{Efm<2Rn}9kcNHq^W)a?xzX6ZH zv`n>Np+B7@>JMK5z_Q2b_!P1TV?da4+(m(Z_)n2*IP(pJ} zaPQ{G$(k6ctI!&Pkn<^F@j)*uHKbdQqEk}``;SNiEjM-b?6^4tdg4z7^8*uoWS7D?* z!pkveAZh-iyA?F2xNOMqbVPJ4IY4HA&-H%wtGN|3a6(nG&~!$4v)}W{;|ha$?`(j+ zeV_h_(bBF^VaM1!U+j{V0sRYEqm8FfVyMY)c zo;AqYm%d(@AFrv~6;rhOG+n_(M%DhSJGzJK^5om6hVu#8A^L$ygbxX^-+zkF zN}2jW@*pg(`ZMF4x>jT4#JXZRtjw@dff8hiVmM$IRzh3%Hk1XlB*aH#5;yoT-LAkU~z^?2hvbG*i3Cf0@IShlnP}?ec zWezQA-b5)?6*aTy{z20{KtcXhW0PU_`g*#A_;`O!-PM9d`R8S?(@&pLjpG{iY(Ijn zUQ36fBDQ!PeG{T2K}CLs368_zH9&(eBPbl7r!4Q)u?vXxIOXQ`7-swoPALI;aPjlp z_MKn0SjSu2v97WYuG$;uoRb}tH4ne(lrz8HfX#$xgH)vGiTGQi%ZoyIc~NNpx+q{W zwpQmCPT!)c(Dmx$V79W^TO)RY6qd!U@x!K5?L_2Jw zj?cd6%`nQ=wx}pRBtc13@f&MpO2KW`O)>TAdYS5Nb;;r#Z&M5N!4vZZ&gr;vrzF|D zYSoY5Ec8i0Y0a`(xbz)u^Y*xBioQnMb zuH}~bU@{+&+UR&7vzyt)jy7*_U+95Kood{#HwqL z#VaY>JGK1iMgMPl)*Pa1mUUW}R)~V$nSd&dKnu z^Qe;3b+H@1kst>g1YmDi7rQ&svggXxJ;d>k?+P;dI|AfDuSJ3ch8D~wx}lQ$viO7g z8(=IQ-&Y{;%wmyR?Dlb!N{FMSEipgiz%-CsG{{Ly8=b+n}rltA~ zUXfq|{Hpe;%a2LJ8Q$StfC|iRVg)Jly{D+?deNKMY;OPi$i=Wh2#+9A6ii83lVoEz7HGBF zxinoXDG(*{q1_4L&3B85jJVIVE#rtI|Ifors3&_|@;Yu9R+^9cuKqRV3ro7Z!R2-Z zBaOUAi6O+CD^u9@^%R}?z5`fbn=jEZ6gL4Z6yx3NC;3)T?)5|et(el=w>{jChgGZe zC48B~i_xv`v}pl{Ij%;f>ladBFE1{%cG!dzKVoEdwl@89P#g+nc3G`qt#orN$J@3s z?f_Y&3ItM}@e0_&+Jn*WbhovdN)ZxjCb#wBFJjhbHjQ-?A~?%mX0=5X?Y5k^81P1z z))t(dYrY7XoA0U2b>^rR*7%s2X91x$=S33QLW|}uDt2gV+ma9_+cDyh`0M1+*cyLF zmCw>6Qd7@YiOGJ0kOgO^IPT$mXk58NlFeWcK|-V@OR}fvR##CF`sVYIua-)U`7*w* zWJl%mW&W4{3M{(;G_V{%%~2oq&e~@|LZ67uu{hBBx4=IM(O(^^90-WoW={-41x-IQ z-1OrolK?W`L}etm55V~|ypMM;@?t6x&;WlQ*bsEE3_Cw|5F9>u>QG^ovHpD~I2m-l zNTEfhrluk|lipnIij-;Xs`iLwH3;?EuJb2){z+PlmmlNWH5bFTl_(BHRvGZdC0}Pj zJA`)DNClRo^R!x{Uj2_vhk`7;BSteIO4Y6JTOhSkn==KRQr_)h|{Q7^!Q zDwe$sUOyz_U=XJr$bkTyFvkC`%USNJ_u(laF*n4NCOXeKRx<_Pvl4aU;#@OI*0WyI z^6L?Ep@{O;eztm>D1%F^gRLq^QA+?YKl1Xgm%T;A-d#hpjwO(HR% zYHQK!r;Pwp<|_(g0l?%&HN<(-5gA#pi1DBq%#c;I)#++%t6f_?{^=#gju);P0mgH# zmnoG#(l|yW4R96@=C`j=5hV`SmCGkG?cXPIOqJ63dn~uH!bb#}Xf^7{Ae5rg5E0w| zsWHNPqv=a#iypi!PSeDA_&xSU%EG;inj?k4JB@WzX?t(%9tpI#Bzo?b^lfir2I4cu z9``ix%IM(AM?VNS5rkm+{n+{W*)JovgpKQFzs-$lX7l5Ke80jC1S4PB#jWFj)O0`Y z1mS&m1I$x?Gmzn!bs>J__5**1CATS%8RUit>iWHxMXbJ>-~qGwS=Qk*--RLb^uVW% zxhG$$<2OCaZ?}yASsb2b+IBJ4()Fvz9bpDx6F^JdC!s-ps*y;&#&XFKXsu=rCbaHJ z3EE{M8Z46UEu}sCy+VL&Y>f)s96A`w<`$91E)5NX~WJ zUN2h9sad|O9x)FCAPPx;NIWeyKw%nv}cc)AS!{Y;?7(b@`~1|8bB(|%)vEcgIS z+hI799gmY9mmVeR2#&gi2yu{o=gKGA$^v9`K%Kl)k_2~Q@NI$(;R&mYso6;VziMywE z5%ded2QDNYEAFi&w#aIcxY-d=)jqrg8M6z`@vjf(gn@R9zcwB3e+09B;}wyN8)a&}LSr-F-;l3@GAKP5qK%<^*oHzHt3!hY}juW|=w z?KgGTiagvbez!E;(R9u97mn8hr{KO^z0w#8=5^Glg%{n%$bW3$9^TD&41_mjp%cry zPjBp&Jxb_O919k_pSVU>eZ!vWM6mz;iw|8~fE$j9lnqN$xaS$10Qhqeo;m8QZ$TX! z*@$I8;99HeK9rr&T_CGOIoBgQ@}lgVK?hhQAGquxjag4vG$|1p2V8Jg9_H(lt^9+o zwloPkhWpmQe&-$Oo@0{_EK5FXRfFnERCpe!{t0ffi%zTvvfTw9I4d<*O zQK_M@9)H>;@5yFEp8ZX>0S4(jn=P0bqWH}`ON3G2UX1(Map)zXPHfH|m85z}_snmR59uwuE-D+y@H z@FwCmtVU6lkO^2&A;xqi)g-W=#frMm)HgT)&P4XuK>@mqx2{(+CV#w|b$A^a)NLfu zK-KF9bn(~XhWE<;NhBjM+X`_gg_=k)-$@p7;XZNPVILg<@-ai3|CUJ_L0?1jX!E+* znOr=30tkn6tuh~JxNW3`wLb@@sw&uq7i)3_gF)hJr51$s^WmcmSTL`i6i~9!I5K{V z@e%1UKLh@$y|V08k~uGnZ{e&C75G9^ljpYonCTgF9zUY(FpZUctm+ z+1%>4u4Lm9Cmv^F^SxQ&-+IP*8`Mnw1X8cRf_|QS>h^kwW)&~J@5)N(EiF?p{RM>1 z&v!wKF_>(}0bIcAe82#3ZrNRaOoL@#*vko8*Z~0Wk&M=o%Ur}o8Od=|2_y>jtS2pu zW(uh1Ag|+5*OaF`tv(a5sgiZ{2%7^u@Jmk&X`IVUi~=oW_*N8aPLl-8CJYFQ51y8Y zewHnIldcd4z!5M7I4zV$MtQQKyU3$^)Vg`~`L!zVzTQGPF!r zrOC%I-G(zRL}dI})RS$>1+XLM^wT@{;vplZ_UEX$4EUX(MeX_P(Gb_7GWfo9AoK1U zlgFncB3cs>IhvO9@54DKb<8nRJ`F3I2XH^zB62+u6ii$8hZw7KHALd<<`9J zgK{)8=1Kc^;FUk|$z3|Qwi$l*>&1W8@)umRjqr7F+Ys>=$pt;Axbqr8-WkHATSkZg z`HKhe%5J=nT~b8o^+EnT^{Zi8`{}1})O+=rLw|`w(N5 zlS^%93i|=OC^NSr^TA+{S$RGKPJoRMHQop=mO&S`4}0~mui1lk+Qs8-)>2xBPy4?- zO6}jv%F`Zzz@iL+h{n0MC9;^sHroC;*!&z5BdhDV9fbE;3QTiX(6Za@pH|<~hl;$| zVSHdm6}Wyx262{Lxx@Z=w%qEdo>KCZ#7mFrj~2lFK7swx4bT`+YD5+p*T8 zrnwnzk(aGsNW}Q$om*B-hGojm*t9F8xmcz5`5`Qu&XW3#%R^uUbOuNwZVd66_DH@W z(dLJSHI}(*u*?)H(!n97JanC0cjLsBV^G2P*!+b+qp$_8ZVg_NisSVPC>WI`k9r*& zZ@Alv3Yr@qVQLF$leMO`TU;7Ru7`rRxvEhZDDFuM-S2B*WXQ&RKV$?3Nkx47vRzcVb^Sk$Qdn7 zyD>Ou+&}iBBq-j4hcCJ4eb|zNuH!kV^)eI_yg{}-ZGkMR5QH?>Yl>Q3jeU?WBD+zh zxWlvqQ4#eNm>REVGU)zd49P$|`kCfB4)Padh~9i5aM1k*QY5e>_u%(Pv(&W;&l7eJ z-qWY>%|ZlRZP7j`WO6B+Ru;d2W`75+Y{y)}@P!4UqRO#S%3w6!?cT3RM_0668?8-N z%W*F2BlDxvbx-!WyS(_Lp!q}!*xIEYuhodzX7dEjU~X+^ zf{{Kc0lm|}$R;O+Zp46gDjsMyit&%I4Q-Ur@^gynM`WjI+{2sy10>cRKO zA2hroVQ2U((iy{v|D)MAimjN_a=cRGRd$d#L8IB}q<#x;U1~SywEKDheJmcnyYzYI z%-rVYQ8u|9MUaee$>vP&(PnFU0wzjo{MR0T>`ndP^SFX#0868V9oR@3mwa4H=c_p{ zwa}1EY1ituNLwPdWBin?^b-HEl!bBoJLz zR53Zbo1sI%sZCk(O<7*051rP&+tKUYmz=z@Y$ZUi$KSYF*veM23Z&umbBxK?^kN-s z>W_9G2{ii@Bbr>6tt-Wew4DfD_JwZz<0tYTG9IFaDS6@O>$)d&V8n;TtYH zzn<__$5>Ts7{q-nDP9R(lTK=1;B3TtKC@~j^O~EVk8Uuda#;)-*?9odLhMK|R1L2; z5tj9V?YKh{G{YDb7!F6U+K+cZ=z?zz1{<9%lRN#?b{fyrvLNx?vEL~eyeq@YNp{{z zf}4EJZ1a%8?L7z8vKS05$@T7fvAE6DWPD_mfJ|d z%t+{fdy!-17cSQmcdb%KMSESBs^jL5B#eQqGf<~Nj&6cvIsz%PM~!Ma9=vsTeksal zfN?)^e4?Uvs}g~S$1|fH0aB(CgDG3C7;y{6`mA<_u%||{6wd~8wHd|4#LWN3Fn@f7 z%6Yss%QCkQ`u5LdspR(p>xsap!FpaM7Qx5=As_H-{P|!}^#cVEl>6e>>MdjgNVt{) zV;sNI9Al_h0S<p`Y%w3SX{!`$r;)h5LvXk%|=fDBkLt zm)_Rl69sMWNQS0la4=1t)O9Y%DbsRWkMq9z8<)d%tE54q_S-dX!s97yTx2)HKtGE9 z(Qb0g8Arg&5&^l-6J_T7W9PLzFE@7wr=4=h&R4d1h&-|06BH)&_p3fBo3$sW>1=i> zyb@lh(djXDP0yQBcmHhF`W{va2v0x<+o~tK+7ou~;nCMFq087Mk~Irve> zAG;HKBs>S32V-GoCiv0A-6_m{Pw6whrmgNlhIN(f9`WXajahR7JzOhu_%VEvB*9xb zb%<;D;3Mq4ncYB3HAvh$+}+uHy+G?_XNwL9S#ZwjRiWbaf*Nw>ufxbkX-7lii)!Z!5U4y2rAI(sfI&?HlCtJirazO8!f$%^pOIK=E7t{>cVDK2 z8|OjRX@6g_8*bJ}joAHyxpQqd-VWUjuB!7ozZ+CpcYq0$@Ru`pIO`Mbc@d`xR>?rF z$oR+kRARbhg`G)o@OUt+0>hq!?yGHq?A7Z2H+j2tPt;XGBRXAESe+uXz7+CZvOo9O z$2Xphec+Il!PT$dkPKystb2-US2a@}XA3&;HyZ<4T`XMnv@F|4z7);O&dj3(<0~5e zPPWL3Vg)ud8JeY58Z;%shwWjeKfA661i)6p40q0!*EPo@KLz0;`z9m=&28?jyv7mh z-^K-L9B}2_Ze!g=S{WVyjA=af{Sd6%$9A|8<&uG&9&8X~UB-?5qtbJhAVx^;e?Y^m zPWf+_u2N$k9eU7Y+V(m{%oro&BLR^L7pe86>SbMNHL^>_PO3ZHFUDUwYC3C4sI~c^ z!(*$!e{$<1cPCzRK;Z`0aB2WU*IlkJ(!t;T&qVc&Xk!d_JhB=iwDy6jRRUC>KUXP? z&D{ZInWFus^05-!>bu>%0=47pw?yEMgvS`X2bn~nzGr|!7ARFFdWTr_sD&oQFtRVW z-)JhvF?;7cf;HK$OGM3y<>evB6#_m6Z`Lf|*hrGH-}DHyn#kxd`11Xf;a1v zdA2)ODs49qDEJpq#ua})z9n*Ya26B9{s1e3?gj4ypZMdxWEp5n?8gN3%^Ln4VjD{c z=`eP)zib6VGZ7HUTs#oB3h&6^dVz6o<^yv6ny~wYbN)AIGMcA$kCpQ}UBh4GXc@Zb zKIKCoN4Ui-kSsXzmH6taQ})LSR6=y4{sDlm5khRZG*+7-kvUR!wJh*6|jX+P)&ji`)-m^bPS4f`Pyx+JV6C#%-EsXu-keLr4x zBI>D!3g0PXC)bnBjg_(%1HzMFfoY8HJ6O}>&ZiEGA0u9(&q(=fCey}Z&s55S1S(}|nS&<@`^u-2-F%f*r&--ADj|VV@PY7IhTW>bmdP{S z6X&vclg(6$c+C>4`|`AcKu`&Qojk8JFq(CGVxc|xCFM}MO%?xyeHcSBxi!TY#D3JZ z*`+y9DlAB`YaseGX`b?29yfB~uxZGA`jzFGG?coUH8+7%&pS8t&2WoRW+&%oCpwT6 z9qD8Dy8a{{;W_U-s&VnBQ;2$8=2A92D3d1oh_|2j{d-k%YpUuVznA|t2e6?)~E~Zo^Zj8uPd%=U)d_k zne+WTey@tUGKE*|XD>Y5PLp4Jm!95xd#1U+`DY^D>G!nHZOlxmLe9fqx_M6K=eyhK z6VmoY52&U@6Vl5LG&y3woU**BRUcv;Lm_Oxi#A>On6IfejJ#uvc-7dI)NzQVmlOW> z4D>Gp{Z{wZ+^!o(Ywp?UF~PcTi&4qS$ol8EOPHs|fi%fXMl%3H5XT5#2=xA42rAq1G9Vj%2)t<`Uw7{HmJ+LRhk>kp62g z`g@t{pkk3ho8S}5a-`x2j1S%+A^I0z0}w-(@81#(wk4V#E=@DLB#qu%n5$z?uU$Kw zRy@%&zKzAk{OwknynWNt)q2F^4#1&cQS}}Cj%aO9Rl9O7&5_wh-Pt%8Z+=U4+GZLi z_QBDHiN`=I425C(kX&^$!>|$>4=9>l!mD6&#G7HA0s+?(_|_hUw2dq|eT z^nbDxE3P-gFC%PFmqn|c{%IR~%lvkCXch48dhPv%jpYl-$-8}IYk5p{iS;bl!V-% z5e{(KIssxBY&Uzs`MXP;Cr(M+q!XSI11HNUh)-V@Y>2?|)bJI$C>E)zE*}?c74qh` z0iaH49q_Z=+WZbG0`p0h*D^axrVQWv;1-#KTwI!IgXzhRi1_3c2h8+g4&Pq%r7Kb{ zTD^TleFWUGS`)%(xL(s$aPbxzjs{zr{&JXU_*r5-tk5N_2Wc-yf?n>zDQdTX%m9MC z&>8XQ8H%1duTS6HnRtb}4EE(hBAg(;@{;V~pMtmoG<_jUD_qnjP2)>VAAs8jCdQM` z9+4|jGG=L?ZdDbTc4w!M9>7-Hj+p&fQzr_)QeEI)JY1I*G2q$$kZnn#r}_B>0a) z#(3}^HprooD(Dg3gy*0E=8pmL5>YeLrcLlX}5XIACo))#lGlI1^A@1CDnSvT82 z8~XOUZHJg`skH9@NWL*~RTf9GpRJxx=H30Br%-dO-)3q1<@;f84R$K3TM6CRP(*PI z0%+s$fO=iMxb&|GE|1;r-^b3S;C>fL-XKjdA@q(`u10CRK)oRhOHVc?RwNopVt>El zEk~$1;BiD4bUY^DeHLsfv=DF{PZho%WiUg83=?vL5Ka&yCOrtpWm4faExA~*IRtX} z>vtZ=K4W?T#g_XN)~W0uD4;8#TPmmmtuvx8GJ5FKJPsC{+x~*WefbYG1&)hH zK}F_WIucMi=926&-6yl5Z$7?mtG2}sB;gzO4-^de?%xvFdfwbz^QsOyg59{RXELxu4SQ}6Mgb%$Yj|BiE`#}*iVsmK3d)h{@G z#;zu{9xAG3KQ}k8o3^gzZY^8dK47Q$AiOc?2Sq2BC^!|D$}Oa|#O^pEv3ie4tO{3@ zJ0%u#t#5&z8iXGyIt?g7E}#?ex}FUq91b$KlTD#BiwC3$1pI5-`@F0Pj+u)DQ7ghB z#elnseiZ!g$!-CfM)AKjjYPYP#wFRHRb7RapMUVDY{8`MZ;AZ|47PLk=Bk|$&0SsR z57kNB0gc8qfo!Et+jqB|+f|08jN9aXOOL4rH=Di>ifJ+z?AoZW?3G z`n5Or=;dKoK4@Q3qs~w{nk@e6d&lwG2Xeoc!Lcf zImTK{_gHY&4dBvDyTkW?*veMq%S_7^*`3u51+cm$I5EZz(ldO7nIE&cnvJOR z9LrzCEo()?8kJozAmjo&p7i+Kw*Rc#Bx}#}@1~)JYyb9n{YSy+EVB_jDN>eHy9oJ} zd-nSDvB&9x@Q&@$A^gd76Aop2QgPk@7CY!Q+yk;cV3zGaV|P)4JLe0!Q@Lkk5Di*jy1uJ6m}>b(DRG7$wJ%NYDJ2w)y@EEql!|c4Tex)35vimby?*_9 zP{%o_T6eR`QjelRZD^un=XA^jD=T>P!XI-y^eQ-W|0?gnBT8=HGJrmhHNVU^4+bSn z$42a;E3jXHY47|PaZnIy9n{-<0!M9rnY)}4jkPaMwsDJ=;iW?;^UZ!gs& zBK=*rVY|pSuf*7sZMpQ&(r$o;=v!s{amv_L?!Nbwt;GlKw-zlY(IqTm3&!{+B92ct!(n}_vPo5`g@2fF2_~D;GQ(#sL~6rXPpb_!^szmTW2e|q{n%a$ zZ2^EHHUcFM{WN zwn|k?>j0voPpCmQfVizwEIkt86oN>Vc+n>FV&{N2;r{SUpOEO+6Qi8Q+B*}XAD8y5 zI2|9Ni-pFaBh&V@|FRPQ>S!_)wT)yg6@Y7)>%p3O-P{DlSp}Zf*0q8;>b=x^3Nsb$ zZ{KD18NVPZ^*h=Txj%E&jaE;8-rZZ|mn9TPANRych;Z(@$8xl&$<`z}>3GPTSg2Kc zAGB#4$AMJ+b-GyoUj{YTssDoyF(6{^pZh0W|KIqlsI|Q7Vn_0`hO-stPPB&}I{N7L z8w5zh0YckLAs0%_MK9mw%i=eWdmJyOYE;rE4V>VQ(|$dg#~~#n8S;R3d^Mi0Mk_G< zoT-AJzG-Q6wN-VA`&;0sgvU`O(CH9^mVny_p(X@Ehu=mhqG(IVAZMn1g=#XdL;l>% z0}Y2?w3@(U`FNVSDgxn7qu@!k%)i2P$*BH6-$06J#hfwQ; ze4GG9H()yKIF+Zot>)4(^3&9*lHDpR=a&TK*%YBN$zUrYo6@KBRlGpFYjprP+pt~M zjr}=D@%e?~G4QAlkd5)0xF@?M;9l;~CnOukAA9bNqse_(GNTl0AL$hl;#lEWp@KUZ zg&AMr6DNG`IV=~+NZ}8Gff*^^znI}abJx3fpY8z$b&-&638pdbs#yzrO z8B{e7Pb>#qQ0Kq>TqE!>CuQYtLEYl$r68_~53<5NLOJzyVHKPkh@XBrFrk?`$+2>Io?#oyKTPgaXX*;`cy^wmC*l))L#Sn@7$u5 zY!~hwdw!$D53c)mSUAH265hh?1lEBSIw=IqX)ZXmk6kLaJkMoIZ<5cNdj8m6Hfe3+ zRaGx(x9&MiziEE;S%p5GgKH8hZ)LT2hTZhHu%gorZF@-bwXsPYQdb%r3IQ)2w;Aic z%{J!6a^1bJ0ePq4wsv+7rHL;JGz!yp4)mct?Zb8^GyE|>%&>5Q^cT+qL-_m`+5@{i z7I}y@CgXWNqJhby)}o)#L$uay-(fDL&dy%7`hEf zfmPbtHl{1*yj)s$4qG-p#%U+vS8`7)iTv`;ZLEBe$MKg^_vvBb{NieWbqp_|VvY3L ze41j@byP3m?E8{nlk)Fe9k`3LgZ#r3^m-_3KZm4&Y)81^4m;*f3A#~n`Qx2wV|x+s zS6`uh)KPOZz1&Vpr9{e~P?G^@x%PFX=1ot6N5Z9cI38k)D3T@E6O>X|m&Fhb8s6}& zP}bszLX9)g6Wr|+=;IXp{kRTzm)H!oR%e2)91xL^dlYOzAz_^&)w;Z{Gz6^Z9cA8n z|tKbCLOt;YhUk+@A!w&yp~UEJXY|dd5v?a zG~bWLvvWMw%*mw;t<|u8t3dHAzuYL8eQ3z}#PM#z@YccUTjOiPTN5g0Go|XQUfZWL z>$kU+V{{qzm}&$fM}n+mJBDY)7AG-tH7M6Oo_@0Xg+~{mbPmTVBG%D1|G9{m1VhZ}TXrG_zh~z*2uduhEc3 zX``ryEeG69RI3g_pY^T}NQ-*Qor)5qbF61CGQ@oG6Y%i+|3}zcfJNO!U!xKNihzKC zN~eSa3Mk#(4Z=_=4Wb|+Ehy66NOyw>gS3QF5`J-z0cZft-a4-u0uL}*rFX9KDFb_V?7os_TH|rvEdKFUrJ+G99taL-t(l?oOVm# zSB+8QW(zHEKWzzNrMbTTy~lIAho1vTVj#7{iqDkf!qfjGA2(kfxJK!Y);M`(^OzrW zXE#k>UpvOa0^z4IXw{BYKnJ+i9MDhMW{+ckT?-U#3aA zPCr&0UMhG!B$}w|ak!OzLrrYy=)8KmJQMCZLqD+yb98h>j_;%kW3xacNju}$PM@;` zsg%qa*c2bEd`T8)GJP3iJHUBR#6ZNNz~QVPT@f(DFD>m5E3+m>${wN=9F+bNW$CfI z+pZUQ5WjgVfq=3(v!u>hq`hZ<-$@7B3j2LXTyx_y{pPQfe$jmY5b_Y0#Jk>;J{@mP z_pRNR4*1$v>=I&O!JE;#))Q4XL{N*D@9g4IXpbPT7$zTu?|bYcAHywYMH^+_+TrJl zEU+!W$yh?bWj_cRQ=D7 zVnsa7aS?;GQeQ{E!cmgoyZ9f@SFjR4nrON^O|wC^p~E{_9M2~v(|Z~LqV(Gv5)}Vs zIvo_pYw7*Xzfr0lbNd>C#7wF5z`n{Pvqt2YcKL*ML22rjW}`NyeDms?PY6lk0x`Km6An=6rZC zg0ztA3bE_WGjyE^%9Sr(jD$vr9tPWW^*?w}k9{vlE3RjyR)s|X^6Z0Kg_xhMTl<>3mb{EE{0#03TCMUnE+6Ya-b^Ioi7yb%egsqCQpxf=$L!B`@0XGtYdh&~*o!mh2Z^Bt?{4YJe>patKZAB| z9k(;y=B}SzVWG3dt1|V&KXKG%NF^1FMl?ha4cuU$zNfW(#)33-?Uo zTqf1Iu4_WfdbY+4RhsV$_+30+g+x6yl#N;NIg^D6ze85giF8@HMoZ^1K~hCr`-(-_ zo|xZvPzaA^;ye+qX@0q)%EdqtBSGu^+ndUb6wZ(@C)lh1%L$V|+|AC1=#5 zn1`K=H)XAO-+H$E{JBswbSxPcFi2kUN}ZgUQyHeqNbcyxjG+-`Ct%xWC=tFWUrY6c zI})OP*hb97+!FK3_gqzWE)K>%FsEbd-IU&ARj=GyqE-Kt;6A^&^o6-{?4A}Q)cbfR zpVxll?z|g!t>%P8X=i*C9TY?{{zToj5r#j)tmoH6q0`ZLH!Z0ldy&j*)T}fY@u~>X zJtHD?YS+6;?Q*=cHD`)MhusG2_dqb3npCNEl%O;F92UszMUN*6)?N_>$k^U{3r@)I z%SDZ~=l~obLIy~SZwqvkxL4?9B8u(RJjGWb^+5qJ<+}8~jq{skJ>sul{Gt9Qsq=%R zjtvZ@@P!7-iCUgPF6v?_nh_P4axk@cX)<5-`dRF$?<=SK4&hV+&m?GWO2c9o<;`uL zFI^K5`4bfLpm8E*mZK}aiFo^4IMa*Ai0B~g=<}|evX%7Dbc4H3tp>z-J3gw-VpitX z87C`{FA`s8B)(AS(TA8EVf;f#SMoM z%4t#?Y+t`*p>}5g{1o+}K%rEHMMuFOyk3k@`(qa#)Io2f6S7mICx>`@D)-63rk)#7 z^`#*sc~7u2UKS^=FnljES@qsD*OqIkbKfTb0cF2iiPR{0pjUT#AInUs`Uyd0Zl0QX z{NsQf|DEd3J5L7LKK{*51`q9z>0FZvvMXeQUL1ar6NXR35}d&vG~+W@=5iq%j|V7M*NV$X zR0?x%uk=g6w8w~ey8MK*Oh3x(ON?0jcN)$pC7 z)F=r;IdegK5J(T%IrC1GbO)*;EZ~{h9f3h--Hq)Vo$-8x>k_Xfc*$N)yy1^;a$o5; zW*GFK;jW|U(_2n0A9nxW7aMn9>dXljg=Zor4%2SCZKkAB%47T1zcQeFgYM|$rPZkW z(>fFMVh6}jkuzh0UY_j94~>3}!4@7PE4`(6NGTtD{Ir36=kGYiM{=qNjz2MZ`l*8yP zDu9qFZ@9oQI_ja}wL^ySZhlyOG1X+&vtiLIy9u{7BBI2(i0X*k!N%G~ykUrb-HPPUmitt)EH zUW(_Cfx*uSTzcvf3N%0Rerm3NHn?HulTwr3%@wEh^%SA~fG1{to_*WVj19t({n^Kv z!b{kd#;7iK`((<-tS5k`$)8Fg4jW!Oxq$45Fg{F^vo6d$(&RTua<$jIN?~^~8rQEK z43FWTOW3XwFJicj_m=tE6~gk+yVHr3&-t#&U6cBICXptnu_~h!jK9#d_a(LjNNn&T z+8ws)!m;!*JdF5+Z`Qq9dX(OXTs^Uy`1l{1-Y+=h<>Xj#Fg9Ue{sk7dG5$$y`bkUM z>mnvePeg@|C?RG_OUVTO?LSxFB|S`vj1;dw+Q^(pN@~x^6*p75CsAhH=$-{trSEHJ z+R{VNu`N=-X@3>!3AvMUd zyHeh`!zbvzpNQnqt~58WDWXfQGvs`3LOEQn#$(wJXM!rKEXU3>{{R!ey52ppp6XOf zpW0#&OplSG*yGeN$d31x{jO-%Q&OT_ca}!~B9ww_d8ifZo|E0OlJU!@>r}pbWLvJ; z2i|%rj8oU=@#{ zqY%7!n2FyvR>q6q$Ro8EUNOHChw-vS=j*RLYedaA*v`1=m0DiapZjaCRf=%|<{4KJ!$wqon!&JD z75a&gr-Jq1hifS--}?RK8YUST*^4`o0oOlDNJj=dS)`N0a1*B)mHcjy?d$#D7sv#; zX;rVnNyhC!+`dg?wb{!BV-(SPNueH%=Zy~}ZrJ-xp5f>616YjL+@9G|i_=5~5aS_F zFtUhI+OCxDM|*2DdMcYlyN}d%AzSWIswIibPu|OSw}I+=U*})G;4|*W8zOH%3WN(E zp_U=MCyi_@q1TMS*nkWQU-Z$;plcwZC-5SV+r7+@kSa+Z*$Dy3({j!c?e0Gsf3g|oA3|i4i{eO@sw=b-2BQy!5q+!E``WBBjTAcJK zZY(~B7}61Sen9V+Q9(5ABK!T}ncabRCM~eWz@4)THVec8aaHZkLl3==aofh?jNOnb zjNyTI*LFujLS}ks8#gm)0E`_srxq7$h4#yqf3S*^)6|Tnh}0G1Tcu8)K^eaGD9FET z$kQd=#UpBdGfK+Y zv2k){;tL4GdOVI`=Pd5HAN;8^J_2RIBoIyFf19Ow?>e&y4&>ph$3GdTvsDgnv0Dd`V@dgj!K;JbTTw=BsRB z#GMqcM;+vB`qPd3{xSxtBF&qr-cVbJCsi{#28EMgqAtF|xu`B7@%*<^A*(!NTKrm; z!w~`L-nVy;)ALmI<@LQ}KiJ)<%w3!KiT>1OSG5Fj>9*J~lVw#i(cg#xhc|Qxyl+89 zSxTM%lEwWK&FlXVbeZT3{||Jn)hacvuQ$&6G&aMKJaN_);`9RbaHuohzD5Wcf>m~# zfUCUZT$9L*`@q~{w9XG~l<_C8hG@f1i76?S0GnVObJH@tEIa|zy#x0W>jL*YqWj6U zN^_@_4t?F0zD0}FOPd$XkpY)9FP6bJE1@TXX;>GsF+}Gwe&HU>e&|WKj2FN1*0CTb zhd0L~a{p)PIZNUlA#u-TP7}@x6J1?d2L}gsVciwmPklc zt^8Z+*30*o7?)C$%2NO9Lp2(+9-vpj1YJ5Cv4-HAar@luZS6;TJMg#6VF!>>sWQ0n zw_Nz5+K^uOn}w4PH&#xM_BbON<&s{NlM&f@tPl$}uXBKPc*v(va>Hri^|Df-DVLqw zlrF6aZ*;=&=!X%9Q6bSzDWl1Qe`qtjvG6@5+gYAwqdr@HlT|U3!Vz9~kWrSq0D@gl zcRlg^&ODBoM!vnSL7|kix*8i5rRwnIi}43G2qYU{pf)z< zv(ge7S=zRwK9?>Fl#s>|Qf)h{{L6NA@U>NcS=&`<=XO1Cugq=5s5*_Fs1s|uf9HZ~ zyC18unxSpB_?}n99Yqh2#=9oq?^UU0BocZ;*jQLte%t&`xYe@u2FrbYq)@A4y*0@n zcsA9*w2yhQJ6AZFm+?EX^0AnAZG3X_n&oyV6T-jp-hBjD3LC%Ho4nu3ElHK>JR_)+ zlf|MjLuI?6_Nds)A>+1G$;LC>Gi)&XMIr(#v%w7S&@WD?*m~1fIj^bxmjN!Qdo@Tm zBaOX6WKnPUsk|3yzMk`CpN-U~Q%-YDXw}3-2Dn|gsn9`L+B<^IO2-rL7Ue)r@38QU z7Ln~wsalLKz^_wgCoSMVg>_>|{t=K?5&`+n9osFDA>!LOzhiB5=aSPYia&o~GjET= zaaGg1>09<6zWRg@B%Qe?&&?}ApVO?Qfh=`G)-}us>pf-i$08#*? z9)I+HW@{ut^F)oN@$wY%gM}$5s62HbL=oTd)@+R=LhKz?N*=p9Txn23#Q9EkR71{F z()vz{U3HUui!KUKcP=We6-6GLl$6JZ6R#8}yEbWl{sLd8i`|Qh@-Bey1^ztTP=qaJ z;P){F9B(_y_y-pJyp;O~guT3ND%4}CEXuVz8x$0zZK$cZ!nLUG5c!@r^H@%o8JcIw zG$gX{>E#kDF6@-=795t|HkE;7mG@qAQisnxo)Y*pgz)*{w?QD z-Ngt&R#NaT_vvmB=lpgNET1__eN(5xX&!8lf}8rXwlprQ6lUDcH$F3)Tpj+u419FA z`I7ggnjYyv;j-Js7S$AM-ZZ~v`L4co_fbD>6(fSXD@vZNi*qMCKN?&d(slFdEnUNA z2)Kr8``Z;)x#3rDJ(GNbuOjP;OK~(r$i+s1Z@*bl`Rza1V~4LZ{^xAl9Dp4H#20j< zU*jQs23&JQ21@-Pk{C(0M?ztgnN_%zxf97xlk8Sq>Q}*aeNlVr1Kj9h{`(6&rUws5 zFD;q6IAePUI=vm9wLgc0t5cBym&s3i03pY<5hv=gWHS(4ce%ZiMYjy!O2$K?vK^;KkvWYe0ag?91uz>^>=#HIex9TLg`odVW zHYDW93TgMraFJP0i&R;z%jV}G0Zx;;AeT)+rVAXz;LHk3wM^+z0}jaD0n@0P$N;Ym zT0kYnO>dz0zU_o`C3F6ei0hW&u#i~dl@p8Z4)$>_R95zrJO@n)(wV!rD4>mjFvLaZ z;T0b3t>glHl}B#Bu{msA!HC97PrDCsN&V{NOl1CbapwmGHIN?|DhZpP{l5J6aW}f9 zn@I@V8_JzjhemC!*RPn+;t7r!=&_h_K2=m;wgC?N)h-Cwi)-488#Q=jU=I?-ZUreK zn$aLHmkG=q63x=bsjq4~LKh>xc=X@5i2zBx^yXvFJjsZ$Y0u26~4|UtD2QSqt8-}=R z$tdV$$7jSjcHys`!_pz0?dR9s#&Z+B9Wq$xzb7!Wv9W20nKyC?WqW67aASbn&HT4L<#W_iUjjL~0j9O*TqQAiN;SIDa z{FE6K1rea~*3i&6j{d6!`LasA=Tcf5E2niDmc5Mri9Gz%L*-0-KkVzw;T7Z@s5iDT z5J3irqB?2LA64HyEhz9b`{9H9PVqXhIn07y!QWXn0~s{&AM z2B$dS4=Q}(Y`=3xL9`52gCinH&6K|ObVfDjrKI3jIWBRpP1SOMyVk-)QTvSS?3F)3 z8*xt_m_IRCW+YVLy6xR+(%sdSN%|l>>a(Q|{kMM|Q)Ks)Pm5h_q#r%qnw4DUPV zZ*5CCKo$%9nD)8W2cS<_o(dDRlIOeiHH|BESXcV}tJw3UptK!_B7TuVg<442WPwI@ zMsISh^_oI{=s%IPrhnvqKmy=epe)MjY@MP&qkq!>k9Q)9La=FBYNuw#xx9u3>`&q# z=Ui8mek1y8P%pZ~T;V#v<1m=!qNX^_dfc74Wyp)vw_5#-?R38QMce8BU}4xzY>Sev znRXF+HC~ZMNG&MNH>G~h{0??c;P)_D-}kF{pe7!XsnAwU9twIOkm|HNbzSUXR;rzV zmAtFXY!mnV@~-)3tFb<*Gyy;K5~RfSsn`UUx_bV>og-?c{U~N7xS2FnN6wc#N9+NJmmx-UG(r|aeTn6<{O;39Apr)pcK6Q%N@ zq8b{)5dMb~n-x|$O;@Z$0c|Rcd&rv8H)Fil9Uf`}h*uo#v+hUN8xOz-kB~3nXk71K zLw^l8L1UWVOA@Afh&7j~lAY(5k6vF^U?I5aMbiegV$i*cNTWDi|Ikq7x`l9fZ0sUa zO})Cb-vTn0Dyh=vQX;JbT(_w>34668U1FAMJ&=*Pl4L(!MiTX~)^55^Upg{23UcyQ zBYP`H(0MI*+Z@{axhCfVg{!Hc5H#!R%aTL&ndcrh zRQa4AYuHSEE3~@kjOyOs6gF9UCqVLml)E+z@l~vl8}<(L1^Eu#zAJWt5#bGn=%tL7 z88L4BN_@N!m@VV@oH~u$oLfPXd7X2T1g5aJ;v>7#m%2A>SzqzIO|b5CZL1abc&AdY zd}g?O_8NTjWN&2^b(A023@#c}vv>8)_d%#x@f_E>I687KEiY*)SQqM%!d3rGkg-3i zE+6JhO{E{vMnb}x19;XwZJe6H_^|-Wr!#eGBWB!zyfVLT`R_n zvP^ShF%w%U%umZIE^kU^QB9EtopZSuSU3YxKonu)-~QJNAkCpymIG#l8XGE4gGusP zMi*OOI%pX4hYE|+TU#ZQd96txCQVa&!ZjkVXN27^QAexbzF$U{3bWa|`!dV`rb^1b z4~hZ*lusp9@CouO5FM((h~YXg+aoY&k*b#a83xeG=LQp73~%<@9CKPls_xXlT?h_u9yXLlgDTAkC6c9JvYv)4?J|>Q z-8j3hgGAHZr)sm0djb(WM*|NpHBXYX}3zN$0DI;TBh`hk-8B5-G ze;x@s%=3fW-;{1Y47hbqHH4!7vK;j-4ontdOPgDvd}{ZwGX{Y`6j`?m$UhJh7g6Z8bW(lYZQ7ot^=v+UsVu9x6KNu1@ zA|%^m>vSDze$c;{OnC*M^jD{H^6)_M=X3jl?p)&dgYG`=NDd|_s?GU4uA|XYI76j* zvp32LIgmNr7d;7 z(UbNQRfTlY;B6-SjG$H1%2OuBR#a#@Un?$0miwTJ$Hq^D%#AKI6*d~cFy2Dju`=WS zKlI?HWHB-n8mXI7VMg0^Ev|gPC31iAD}f5kl-c#v1w;PjM`u0v2nZ;jDvzihhu;D+ zoBpbEk%pRA>cXG!w)$zpZ91rKb83VAyrANn+>r1wk=Y1E&NN~l%2%$>%T3D}zLbR; z=UY^xDJpTXDlfq+1p)&ViI-TdHW_D^H1DWsXm?9Nh;nT+IrO1`$6-E0UAFF=d^EQr za}31eaAPUxeWBfl8^KeAqz{&OOgwk)fopY2(*k3nf4iL@KNfuV=RHNC79zs9GMDNr zmxH!K_plNycJ=4b&`?mkpcVOL)3`(LJaFZ}<=^sB7&P|@SxA9x)X<#kF^L_D50 zg6aD79yT^W#$k=MF_WF`j5n%t$^DFDEYnV@N`KmKO6|MM-m}5T7&(Hhi zkpg+#N8;*H8r^ev*D*J3AK)4ZDIQZD4Gzj}@|k53Tv9q4Y2HNMdk=S^03i=v3EAJM z2sRAwy>i_gq&XzopdA;h@OkENSs`WC^9GQ^_Sz&5;yB{6wV&F?Z=M8d9dvibN5AX( z*a^fgW+ifXAaRXH6oomqG7HoZ=%a_jQm{n~hjc397%w#e&I?0G*gMuJNG(C*%Yx@) zrHP!k16eV-b@=no51FCtL*6}m*m5b03<|Nz1*1~14S06`gHy0a%-J39NoYljgn^tx zMacf;9>Q_p_tx-5aJSd<)#$J)%Om1)XN*eb&Uk}Qn13Vh4{j-vs7T>D_)y*Uhd0r( z8TEBDrEsg0ski;tomww;6Pl6XZ@QS2W|5Io$6)x5^hbGfGX>@#a99`@F?ES!y~JDs zmXE^MvrT^DUF}NYpK5$Ay@@0=@2xtX9c~(+!f{JuOS9;>FOJ%s!xUA{Pz%0V@^z)>YCFQ!wV7kGWjwB`N(y#!=qc@ zk>2Q#GA(~25ZnX*P8bh&8ROi1e)%1+GgxNFEWy~ z(Qk3XqWp?n(-4O(YEPyLI^RSFJywVc3d+1=t}V}EYGGl4!vyW-omLL_L}mt9%`~_f z7uMH`sJ<$TyMG=)>zx56;SG8nO+Oe6ed}ZB^XGRvZxwbPfVJdwzQL+Wg8Nm5JvUGPU%^e*p=?r$2lp`T1y=WujEdwta{)UA= z_BCxno$({KYC}*Xm1&xEcEQGXR+L_sKaJ8b?!LN2zx{vLcqZZhw8jey3#>elI8LKF z?c8+<779m2<0S6SK_Ow3e3?e|E(4%joI~?Z-L$>Li$w-T=;5|lAb?2hgLZFY6*<3J z`Y>D^sxbET-=cKksWs&6-)rh75o$s-1F*5B6s>FR!oC3Wyq3^of5oOb4 zOpH)MT5pcc>H5#heW}b&u;*<*8D!qMxWFE)<#LDdU-tHw7@aY{ny!<{;dHIXw_5(C z)-aezNsNe|6#_%;(ZGk|HauBY_(`wKQ8+EjvZng)yqe1=de~*^d zwkk`kfQ1Y#k<}vr%L_2iok$1;#DGcD4V(RZ{`XJH*SEUXcbTEPZ6E=Uh1zuy-XUci zxheN!kwME{iJO^}0|=OJMa-R46si{c0Rar+7k=jM?m#TR&x%-DDjQkxZ4)Bgb`fg86}t52HbKh0K^3@; zJX(7M+m#csgmLDlYF%{fMZyn>cJVkPOq+=I+i2;t#|rhYKwIXhNzqcNnp!vCXScV!UEp=bAzil(Mr++* zU`ZGr=2g4@T9vI2TTbX8POUOupK}sk*QP_1w2H}m&di;eN?LoRJmLhlUUT;y z)R|94Nr`(7Ul@PQk3}sO8%KEXsXlY$&TUXad*dCG+&6dXnW(l6sVmlnP_X?-l16-a zX}N;{1Mdp#rytJa#ZvjkXY0ks#*&D&wf|Wi%D?GY9U9ox|43tRt+*b4^5-nuQ1shR zLM)b!&*yVBq989{yufZb->=2@-V4{QG!G!`wf>%A%f3{xf(J=#CNxtbYkb->Tj~D4 z!`tFXg0DaSQ=dqh0{Ji>_sKe)j@#E68XIm7ARE7ZxrvBY2<0w40$0r<`-HalgYCPp z6*=o&HnYG!4g>pa65efr2gPqJK_(9 z@BJVOKEVU@a*wzGCMY?9!Lvnf$a4T|czLv2ekVY$N$t!6C2%aIQ-XmiV0c69erq@I zc7=`|`R@PE?{I`i=gTNo+k>r*Au^gYec!tALjBDz3JdqtsLOUkHDiqK&{D`LKP?H* zO1Q00{ZfJ!&@RJ<*!-ut%4B^^iX3FB^v2qTk|+Lih^56-g!j_}fto@Sf45}eIS;fi_EN4E`3yD%NSo&F#$Lxc(F1#&DPSU&Ez(tX?p{LL*(& ztGyhH-#s_9vTIRURizI-9xJ1vK{yiPT*|JEiP=INY1UJy#i3QS#wO2L79OqNOjTr! znorOiHSoDIcS*-k+q^y7F5EeA#cM$wBi5blM*6uz!_v@B72;Q?#7HVURK@6(|6-it zRmk-#AjyzT>k?$A4|acdQ7U~_-D+da*jv3koqQ1bQ@Hu;dK_}$REN^(w=8Mik%Rn* zl>J_0rleY$ihMIri(lKW@z6`DO7@>ZPLX6N>8Ue)s?g_<&Ukiqv;Ep>&doVdxWUCO zu(aQshefgeX9)+`KGfM!z8tMrEI2o1LS8qWSV(upgP~yAprYc;=r~MhKKIuxHOQST zeo3;R!Z$6w-LMx<_h5}4pqRHjY`d8OG`qoeYzncS^gpDv!=<}qAYuGIoHx=73k&_k z3q$?7q9AUD(ROU$fW}r*1y|HSjy4dCS^C`YYB1)_*b7VyCIhK&K$>p2GyWemF&za- zDxoQPHhtsgbK(G*dTR2h_%R<9=rrLfHSLDZ8xQ9tFQ5F9zBnIi04n{u!5^iJV8&w=0vYOIjjGUZg zHY5|&4c7o(IAFX@Ny#x7uwvLC{ezjg2rE#w(p<49=I1P*;jjEoBjxP|R4v^R&lhd| z%70k&(_0q-)#ul)NbKL~&?_|=cT4Gw2}#!(3Hr>?1B+alRKMNy8H_)sbf{!LI{N*A zXs|PiQRj)-;Ra%VtA6{_;?AR5pau`ce5z+RQ!*2kXMg<`#Oa{kA@ z(Lo7@0U_K3->5t*I^%bfj%Ga6awu1@HUURn2*6xyO3JNPbNS+HalYcF*h~n=-Ni02 zx1F@YQBo2!H$NXgb+P%x?>!?XHai{kH2Y;v~GfmiMjP1l-kS3}76+eA;FKd59@`VqI7v1mq7 zi;rK-+{(Q5;EIM6|80qV+=!V^)ZfEJ=c@6Ag%>Ulvzw*=j6<--xxC-jlwD;thXV`_ zOm_PM@W1fR-yCsfG_{Q*`eVZUa>)%aYAoxZ!q^=UXk8k}lqv-A`htZD;V{?g zPcKuoxWaF+_^M`F7Vv#!aHiTvZc&qyOn)gi0b+9Z~P|cM`v}L+sen=k^zbb-|%cgOvcv7@{UNa8%*Y z0fEy~{m#GZb+a$lE2S`vt30oLxbB6d$>l5^yWnMvrK7BnKf0DMy~46>qGxg$Nv4+- z@r?oK{n$J~&OwE$phQG#L=$r=^>m}#FWA}H{bqx z<}%^Q%H5#9-xZIAYb2%TWW||R;rPR~>g5>*Sh#p){uPpH7Er=$k(Ms?10w_Oxzg13 zym!s=)Hd*df@NcX>Q1YioG2vKolU-OK48#ww24)SVF?OWtYjSX0k5UvPtx zNhqc>Y4NtGq`sSg5zZ?2xCZttXtTJ6?Zo&dDKOEj%ew8Q%}AD?B;OK6oqsPu{1YWL zY8t#TrPov4uaybs9e%J5S#9+Px&Dc62z*5v z@Xlw_wyEi8dG$hc%hjQo@_5`b0Yw2?9TpASI^*jasjxUs^KgiUomia-s2GyLg@WSO zF%)3vVN>>=k+M?;eg3J0k9T$gdDrUeN;pjnpFZoM5vTE8n(MR@Z$9^lquBV7&^L5? zxw(1p9*(BC#opWt=;1f~Ig--DF9O^#XSe=mUCIG+R=io-$K!XPX(tz`q%RyJO=>-p z5J@fD{l7{&vhkn(H66xb-VhD2D#UerrWBE-O=I^tI+`jrG4gkSrGml@vO3W4Q#E`h zguhA*?cEK&$pUaOqIc&uU6-*^`|DE~y$5imtkNv*M^pNI}&d7~^dKo?+4PWfH6Y=opx>$@%4BLw^0$yo|w ze$*esL}~xhaj#1Mn&bzFI2z3UFh`AHSDIJssk7#!n_@F;agb~})VG|5OEpbJ20Iv$OJXMf90;GP6L7$aow^ixXsf+HOWc zCXL{sI7Cglfr|lN9aiwx@*lHwP=j^P*ZZhXaL|fTHR|pQg7|plPk$*gy%^No?4D@0 z3vJfNspn`#3sJ-BR~<2Tyb_w=30Sf<0c|gi17xI)cRL3Dl8h1GPc>@R>DJzDo57?%j-lOi|87+)K~tgkEws&?igN)|NMCl+2rYhKo#CQ4f$kJAXNKMq(BxuMcr9Wq z)Uqavb%SG^>RlTA5m9H`Y}#kG8mmZLxv^elwfxXfr{IvVgW9jG*?YiE^Vvf~I{23M zuXg?Ni~A5FP{-ijYWO5*h%gMa80p@&I)h=&zm9~IIycqS&4RkuU>%*_f2X&J)nxgU zf%4~HMf?+}jksyg*S{ntCVqcSc;OL9XmEqgW@U1+HAdT}rpH~t5cDxx#NP9}G@lMK~Y8UTb zWuWLaH|aYwk(2?#lAj)C5sA|f`|~aSq>qTC4?$u@%1O@=jKBL=BOD3J5l(Tyk}>f& z2iJAR`K(nu7ymP@TgelY(#Ir0{!;Q*!kew0Ta;AyLB;~zw-~+#)vS5CkYKVXL0gUI z8pLFG>bj7w`Dnp#03i6g1f@eu8FDn#^vTZkA*&{rDby7qpHSPGA84CQ_a7Av8#5j4 zu@e-062k^oIt;Xx?*Azl(lbCmHCGN?9dH>p@j$K+vvT)o33Ok>Cv0ZY?)Xsxc7r&W z2c*BwHS8@E>XEwUO!@F5%;R)>bvTc!0sF3vM+-~_@U4u=`;6o6pMc-usxxaN1UB-Q zu8#s|ge=SnfI1t+O1fw9@v*lpeI)*RBz!yRzqi~XAtjahSu)k&$19NP44wmRPP>I`T)gx@$NZt@0DjuvZBV*q1hI40t8&#bFa;>clQ}R zQPvs48+7nJ9x z0=F(|t4Hxc#`xWYeZ>5Yt5D#G`O4P>+zGT3f1~UPnJ*wFa8dLVcqU4m86nW@LcKP}`4Ndpm2$<=EQFqp5^v|M$cJj*R3IYJ&D91-BLlMPO|G zc0W|q%&6gAl+8#H*)ZdcAJ!F89>D2}NqTW$kfAyB=CN&4WWXu=N8#2EW(WjQfa00h zx<5_xznodY-j>}-i1+TO6Q##d+(*o(Ye_WTF|8NRAs$2^vc`R?UvXWj#qYhPdrs~g zc8eZ%1EJU~A%uOo6)~hHYMDw2-r;%ArVGMoyoi7t9ZpC!POCXa81O-{heKM?VV0!( zED7xjOj8S{rW1Xez*XKr#LmIU@(zb1bJTXSiH+c%rd$91LPUqv+(AASiYQlDF4|LQ%`A#Xn;D)Eg_`}vZlCl9=HRD5bj)ZYE| zY!gFHL4cI%Mf{bZIpvr!-8Vn!AN*xs*~i`+b;j#g^K@-;=Irk7W*q4lN<=?AximEP z+;J;kp$cb0Un_3i_US-FO%$+#W6|Qwbn370c&!E^Wkc`Eyj}KG1f<}=#u&R<4~6kN zh0v(3_{>O70lJW()hwPBuH)Xvu}5SU(U3EuZ^>mFd;(|6i-6xmieCaurG)@!&rk(tsH{STl44I`7E19W_}S&@4KFAwOG2)>JU zZ)`Au@=~$8h2?gE5g=2)PrT-di zb;vx-e+u0CE4hGQi~ zNg34(4+Dn(#Yd25fBY@&yIsu8)XF+uKC5-qSNGHaG4J~w?-AVbSE7R?1;@cAkD^+C zhBu0x@rk2;wT(cjQT?Y_vmQX7yRW^$o<*~hS$W)wdAs_IOLSfFmA&fn30+-Xv7Zyv zm~(TDr3TML{nwKeU{pf(c;;p~!C(51P1#a~64sxf_o!a-%vp7^F44b zbmRh05=h>!RKSB;(HQep#?t&YnGxGX5yx(0gC5xf=k@PAu?f6wgMe|H%;Y z*r78%ptJw#u@yNIjQiTHDmz9pKK2d9%(;COX0}?4VFM_dH*7)S%EC^yhJF>}uvbEB z)Ai|ee2&jCmy(|pW77{N&~<4rCoy$N%diNb1@2)j82kGcmVdnf`EhB&cM;KNA`bHp zl10u)HD$}qxD%*in1Xisn~&#t>Q0uStH(PpUkFFD*1g-2?C$N|QDiI|p^2bI%b_pXCbcF^yJD5Du<&L`mlwjW8?sIfQuO=^#0?2?j`m9KKT>s3y# zA6(|ZzY3=Nn_Qsr_M8ZP#$2_{kG`$1`iD6F*Mxnl1Y$`xdVE8gg@=TqQ_8C_**`=d zHK71t{JzPMi-7d!Cqz^;4)1Cv=cvtk*1p7S$_lIW;!Hgd6#9mQ2sRu;n#03jwttJ_ ziSo_AnFa0e*}t&;+X-5s2Mke!W<7TjV&-4GlwYfMbjD@W@2>L&LS+SOP(n0zSI!S0 zf(LfSD?bxwOqw9LJrMFf8c1uw`(YM^vancGCu>s5 z#An9m-K+^{W%}}IkyRAL{LEKA;Rdj!>KaR&T=zF_yRn1{)sKRdBM1EKsV1aCJuV|U ztXs|xyd#09sB|v}e&Gi|#U&s~uPUt%-y?K>rgAH0k%gn%L^evYL}8!l&p}WZAJqR=M9`uD2I?nt(yBLk?3ttEit@4;VH4 zP9}FzLUETq1`?N}V4|6u5~;jinv(H)?e%OIE+drvOo!wFw8}~WNM{<%P)BW5RW8mD zst@-Ont2)t&H_vPa&Tz?Na=*JJ-gzmQY#%6^3r7c)5W&Q5YgF&7M7MnAdit;R_J;6 zv(=d^IxYpeY>cyH8Q<4wrH4A2^~^_ecXREtKtFLmwH)mq?_9y$WX`OjF4)Rw8Pi5x(O90%-Xu4m=UEf%oe*WI3rcoEq zb+4Hw=dDmSaWrkEu%;&FEJmh zT7;kXU1rB{%pTphV>WZmsaOfbHk<>b1GP%J zXdHi0iGi6C3)HwSc4akCE6mc&Y#>VpUwE}MOqp^^DErM|-~(pnsQPd19AV#PT$w73 z9Z`18UJ|tl$b)RjQrB%%TcL|Z^7{qAD7N*g0ZWu4qSNW4#ivzXI~t%ntaka6 zd=k5RLx(*r&D}*Yro^(zVy>jdq*Kacfmf;tJetA4vvLS@qgdK(R5%hQF>Ney<*=6n=k&dVZk}rMZNJZy;+|zY}r*KC}&X{Q$6OO540f5Jg0E zsW{$>X6@cA3X_2414VUpU}Rze7AY%?B$2vKcl!_2E;>*j9kd-7;M6QzFJn#JH zr(P;!Mq|)%d=@iqL^Qr!`C>t$VV-i5GPRHk37l)J48@&g-Hw$@aFp7>A&|rgbl@Ph zKldBAYrdu31D}p$0U0GH7q}W58#@S%SuW}(vu-bSPOGY;B|?w>xFSS|=9{D$-UB=8 zAsxwX0l@IVk$@1X^1I6Peg7|+K5G1&;LJhtPUP+1QwHmg(5MvdhH^6@xbAZ5{m`6fhvT*m4)IWWpV*8J~}ojKAy8PMfsbV zpY>M@L>8xJ;JUs}0T%W?eFhVhM%44!?T0{}fOw1lbde$X0(?dgaLP1LJuZ7<#S$>hP$ zMxRRugGa9$i{41Us^Twg_fhHa&AF!b|5rW=@Y~kcr5toanaC!Blr5MzL5ZaK1fSyT zDM1eHn6U>0B;s(;tbjSw{f8NVN-b-U`55k;g3GlEpSOMDb=x&_|Lu!gAQM8)>o70f z-HQ?^^-nu-tUsrvppnXOcmujM{Ufg@C03A5+o4Z+0OBPZAa2t#LAV;m5$6} z=g;0`0~w8&;Pm-^-Pq*+q^;`q1KW!+hz^PN4v9crd1YV@$ru#?_tF}+h?Vz_lxNR;Yy3(-+deMt0-Wv?4R=a;^vVaxNTYB2<^)4n#eK( zdi&P`t^BRwisxF7*MKW}@Njc>;AGE|SI>^M&!7B8WS2qn=}0+p+viefhihJv89xt9 zBma1V)M~oUxyrN!v-}Ir{q)LNQ1k_@9P8P3XO+BN>?EFJ@)v9pUkg7RhGU%QP*Z#z zReDruUT)lotR6GYuBzg7->O|XcZzb}?|P7)1J>QGGrnJV?-oGO@!$c0YYDXTV3AAT zeX;fjmAI}C^ZjIK{+9A0Mzg^B==nT6d-h?5;}Nr>YN*bL7GNxwJ|;Q~m^wATu`jJI zLzBt(IVnljp60J#@SETj|NcdM3Jw>Y>$o1~r3bgrk!NahW#9`yP5tXBpWp+w&%OON zPBW#Zkh0H7s`fR|NRzBwRz~&HZ~=8*g-A%YOzYhT=&H8;#}DEdNWXC>**1%(66?0* zuRuEiiw)BvBO#SrXKnYCRyUH;v7oFgU=oAd3Pp8V?{59UVaUIa;k#IX4=3n9?C1sk zhh$BSm}o8rKN^x`CGhUg=AE-yr%#UKOH1a!&-zfM)tt5OdT0DaNDJP1Z0SU<2&fLi z%Jp|glwjxJo+dT4sqwzBTxyEITb6^Z=}9Rt%w-RVF!HveC7GF00XMUr#3TWGoh-N| z>ro@H*3HlQE_)OfgeD)=eyE$}gBLfyp7|^u&~%=tNflS9?{$C`1&X|Q{eXw{`yqch zYgkpev834W0b{iaQQaEfCArD+c|{QHe9w3!SawnYT{R>)pc(6w!~1{FiUA@}`Sw3Y zx1%?qMOI=oABPrM@7zJEKA?H_atHEPScEhv zol9CuknZmM#zOar=lS0E_x^JZC-y!b=A2{P_m%3yi*RkXLuK$HMyh7k@z^dQ2fYEb zb@B^qPhPtf@FV8Swst+>HH}>}eG6Nxww_-(-)_QNXkSk8NCwl7h}7!^ zcEeu$1dv0|h8Zz*k5vsO`k1bUG_o8aGzX>Q`t`LjWzR0!Og@StS{*gyUZ zXP&KF7?2U{5>bN^9<9ZVWehW!ijGUW0<|H|; zE5_;cZUQFQ$^L$7GeqvfNiS6mNi@n0ADEwBp-8_GLfpIs$G^T8I#;un{U7NrZYqtl z2&(wZ6L3oVM>U5EUR9GT0aqzhv`(L$ItL{Nla0r%G>EjzgG|6RE^46#T(<-EV_G?Z zW=?<*6h2w_uHO&(S$+UjM8SA6@TWAny9u9MbO>Hiw9`EZRpb#Y+C2&2!O3|5RN6GR z#}%ee5-N*50sBg+0c*5U8c$3l#th!uhYR9`zgIgx+L~rEyAzpMnXX|fRuzN{PM#8a zrmPl8x}hsEOLx#wuBlXHt|sl_wD z8a~b7FtXvi4U7xm!znlq28!NFU=aq)GtY%B)7i4A!lwqb8ExQMZKSg3%fkg*+Mq@( z9xDp6QV@0M!NWIBWO#TcBzq1fW@C#_!yNJ0jmW>Rr128~C$VidK*sCf;(+?k5 z^BFC-{JK=xk+NZrXeFvll}{f$WdYTkX!zm^kgA2;sKoDy558I-ZK#3S*0cl&Et1QC zMk?IrApzxJ93q8ipGd&<1){Lj7~mFBc_in0m$O*CHI?&IMK5xM_TG}{ig04*WwEe3 z#Y=)Q|3_XR9%}&S9QemRT+G2NfsWgLWH|- zWT>|GT&kR!2DDS!=&}^biRu*tb-Q6U z(_nlg;8kOLMY^_k(q{Q9gO|kFpY?guC2G=i6-*r&#YyQwZ|%!Ze0HytnQcE%LQ5pBKiC>}_Hk`^V31twc!#9~w%!S!f>W_+*L}SY zSp4r`=G0|IF61Iyx%uHc*&|AjJ!t%`Gp6#r5oieddY*-~j58-0)0++ohI5PW8F)$1 zGISqI0+4QmQpqNxn>+3x7qDy502Am&>L3N%) z!%zL>XM$3=nFc)=7#P4A;X+-1AzSoGg(@&i39(Sht=^`z)>lNuGe?u!R&!d!vX+ z-%FRq`3JG@3bF^kd0(gZEUr=8m%Cg0>3HUrC;}U0HG*!Xy5*A9h+drhELMTA5>v|#hkF&89H@L*v#@D}7&OVv2xtzLL*3d4qp%)WQ-#y4DG!A<5e=38=5I@Q`C*nW;kb==vt{9$tPq50{$DplfYwU4KcUvxdlqjso~P(&`m;0- z|Ii;BYI1&L$>|@X31tOp(on?s%d7_Tdh+xQd8bOEOkBoqh9N)jp%IUSC_c1}e_@@j z>h&@!lA$XaMjAv8eCFf@*j_HJ3+{*sS^9Q+8ZvGM2@03%9C^Z6n-;boXkIjfP`v}Td z`wL>TKY$abX$>W(Np@UhXhQUch9gyo`lBk*Wx=b9(Epg{_HWu_k12WwK7 z54-N<`(KKXUP%u4*+2g8Acdr;n3w_YDNW$_%mVrrxFbTsaS5efkWbR@0jeI2d zxjIGT5?`8T@P2Fo+>zO%XKk75WO-B_Av*eWjx`lK+nBjD6^js_y$_6Fj(#<=uD=E_ zJ?FPF&wB^f<6hG!`q@Uk3;`jV%q&vXdg=Or-4Z3C)wGU!NuZ_>03XXL1wd3>9xq@FT;W7=iAVa`d$V?z>IOkdKxCCjm4r3l#0D_MxZEtjQ@ zGBFXw3)g?o5=kd}lt~Kj?0kII&(NuMoa}MsWW9R2*Taf3#a5#gZ{uwrdp*Q>yDR3} zjh?N6RLlC9DQe&>KZUi4k#Qxbl5FV8$s;2mf)c)L$n)(_jKhuG&p9bC?gPr9)BH;S z>;Ez8Cq%M2ry%k%{;Ey@ycd_(gb=gE)m#M$U=U1`9<=aX=H3VLQ>uRIVjxp13KZF$ z|DEcs81_Xu6a}x)@?XhJ7<)UavCkHI5b6tqd@CJK*6ru)$|nX~G>h&O7_S!7OdoJ9 z^lZPB=;`+BDvdVPW(a*NYK}l;wCy99JEr*KOvmyrrshX={1@NWvBXTf$H9eexsQ$)oTCfgPP~kp2X+fgMO@BbQx{i0x*xBa~DX0r_ z5N%x%;7$BG+!)N-fBc1aSZxbW*gJ~B_p8DiiPaSyB??$%MJJ~k@^f)Yx!Ey^6Cm=v z4S;D+ismCuTJBM7?~2mgI5adgpl3U>V2?UZ1JRvQcGN#wLZ=wBwZQ#hDV;#{5GfOL zOsLaXu$u)amAnK|fio`^_fnWE5AlhO_i6tP0UMr*u1JhV%wv9-R7xo?t!8gEfMq*> zX4T#u(4u($+805^2m^M|$ANW59FPOBFY~(S)h6^h-t_BrR;-Y_x?tdu#}O~?;&McC z#SG--2qKTv_$v@TI3oi=^BXNe_6fV3;-+{?UT(a`}EGY`BQFw4l|XH6Zx1-X z1nH`^9~e>sBqi=tVRxSgtWj!InU2KyOb;stoi~3Bbn|&>wu>VNRk1?8{==2tb-_K!|7KiA6udbej3C@l(+zZW)BIle04Eay zyC3nr70G5%K0b-yShbA2wzEOvCd?`N!{d_9NDbdwnqD&dWZwFKT?HP zCy(OJf8ewNn8*HH%L17HNo+$R?r#1sGA>`DDvBx1!FN6HQp+RVD{3OSO+Lag1UT-E zc*_mq*F4!K)O>(8`tNoM@=&TB>dDf{`4*n7cZsk z!eGQ-hhL=IXUx^F~=_ue?O)T?)(3nKWO=g{XvWh>Vq+{I=Iz(OP&%k9r zigy!t6;Q56o%cgVpPkD36W?%bzFKwMNJ-KGl_rd)#1F)>nDXl(ajm$Qk0J4ZH1Ey$ zT^B~?&cid}{-R3s$(yShMgQY#Ucuen-P_w0FMnOj=1=C8BOF;#aQ`$ThuE(GrK(ayI8^$@Il!8f*UodP zR~xY1HJau#aMO(_rs_A9!pLsR=F0?6qn zRLy&{>AtzB7nGs57BHQHUabO=GAhLL{@vJK?&-YwYSpfA`I-GU;=(MONvB4q?%;erpo6ItPlI$5n&1Ak1`)y-he>~_ za@X%&BhC4Bb<*gYp^?mkN6N-u<6fl``{@X)VP0``eQXM`#pxokeu^%d2MHG4@GD(-o@LJ4_d`mL0-d4i5PZN%MPju#R z<{{UEY>7HcAQ5`d(z?v`@bPs^-dL*e9oZ(}ql*H?q*wRvFyDN2Qjm|VvmnDs6d8rZ zdE?UFL;KZzhDx!^Ll0u1Hl8a7wHo-!Y52!v_nJPdBMh!w0N((+V5&YAldlKXNT`i03WA4z(K@myz-ni zQX^>&0DXTLjxXyMAfAfYPDW|KLxfTTp=@W4E4t%`*S&VmWqLOKw8Tfo; z0IP%PalAJ2^vs00PTseUE`q8& z4a(LNuxD~@R5BFKMHU!1{WgmiJf9_jKsRp(&TF1 z`B!{IDy8GdpcRL1xPLSN>9H0>bzpNK!G<=rhsMuGN+rsqm4-r5U|BM8f!>~oyD$fE zE5?LWr@i$~DFeWGB>{e;Pu_(g{{WE_Xr7VOy6o0~a*?mlYhxcMrnV#H`ydp?0$zzl zg85|rI+N*Iu4GF(}EsEVn!vJUj$-e6$j1v3wt|3d=88jgxE(D3_t@gc?+ zHgMIcJi{?+dyVxasZ$>^hvbyYs$6cxR-p586`I)@Q?HuiuM6$t{zI)d8^4Biz@ z#Tb!Qz#@t_rqAHB;gk$H%atPON0y`k*;RTV;v(f;1T8jg{EMzb>eI5^*I$qsyJE~f zgI5qF|CNIakklq4qLT+ihX94+IkJfY7BA0S61A?4mn5z7NXd*)#dH16XToO-lpQR~ zG#zlbmA)j8k%0%F`;3Qg0NdJ~PYGThBQB~Da;Pqt?XpEOYRuj?h=6Eye`XH=ZLB5_E!_^uk&*On?SN>dzpAJo3+@XUc6gM$bY6-4~H@po1zTW>M zo%fBuDjC_0odm1A1q4p5C8bffD*;{<;Ef z9#0&WwedVl9A(W4UI5e&O9T>5Wna*rp1dT{*{2v-?8$#M?DG22a;fo?ahSz)ZRV(> zGHd+~F@mXTHbu3Puy(I$9>AzTg9=}@VUzil?1yY*ZfjrLH=C?X;~k7XFuL|@?|Iu$ z9jA=3*Sr`y&EoOcWY|Oci-8W9tZy`Q66c-wE}zK?ZU|LLIq)!0u*3woQs%54+DGY2 z#4BiIZ^xMLBsa^G)qn^li9Je>jseQ}Or;zdE&1niXx3NS*=T^M&mn)$P7%-*Ok4(& zjJ42K3|sB}!fg&I()gOS-E=(~_=DlRzu!UzeC=sWB^y#R9!>tmO^qewANOn}L}-q4 zsEutr*llQzOT>;Iijs~5@~qvy{gMhW^&C#|R72nm1Zh}%ln};s9}r0Kv#dO{!D?u# zhQnlMR(C-%JL-Drb;G;%po~XEqfQEmwwaPw;ewNFGTto9a>`CTAgqFcIpadaa?lA7 zqmP8n&`&l@Z&Pbwq>d6?@5Hs z{uvO#4x#zX72=CCf*-Z0fv3Q#ET50`!|ZKvT%_YNsr&#u5$FBTM3FNP6_O1e`cM!7 z#|rR49{hY4I&iQ$)No}uOa@>r=k=nmoLw&r5s3!>8u_0<=e4nCQl=Y0nECnF+rNL*v91*1^PqHRI;bkcreKHfD%(_fkBeAeMNg#yQG$!nFIM5>cSgFB< z?{J5eAMzI@Pra)PS;a$$%OWOqJ!vCJ;va%|89H|fDk@`ezBE!rc*$(l>9#-?Y-j@E z;IW6E=yF|uncSs`W~6eq(FZ3F2+rNyuzqI|4k3RGP(gbrn5xu8r3msA0ML8L4{hWG zHr_!$yjZ!k$y$@vZDRf~7>vCYcCs*!Yqz%G1gvpU$jd)>F5ISKy?j#!))DO`g-fw& zs|~jiQm+5dr{bwcOVOg3jh>jQH<35x7x<~IlgEMPlOAS`D!KL1B9rbY22Mcf38(Yf zETHlOc(TG_Q+=Y^nrao`W~ksG@&#~cDjG!C`nDENF4d#6&BrC%u~+ZdHZ^NMYx=V^ zE$2s?F}!r-GL}{iN_x!@JmK7~D6b;o`=mh(sw$J!gS(gvKjhW=P5h*fQiGi!J)aeW z(kp=nnn7ce{Ag# zJglpqn|mk=s@GiR_AfIXUH?ilq1=B16#fdCRZlze8mYmC`OCV)SiQ-e8OWB!9+QC+ zl#a6gG#m3c82CyTr8w9n^vR9H#7!6IZPgjNc}!`p>7}c>t_}5Qppy$YOPULO)s`V) zUGEx~65fNV_Fz3y8?!|B#ICq*?waBVv%J=i>!F19IV-Q=i9vf02GAe-sURbTGnbcN z3m!Eh%LJ^cUxRM2^X7*=rIwzQ8bL^NsFc;E)us8VdY!gy)O*PR{!Rke+O7DJMx1O| zr&N@Tbr8rLf-78(S)6yittx42Gb;Gn^6vP6`X?!k{sD$}@8-2kS}Qwaf=U!( z6~j0%IXP_c4GfBKetf|Mfov4vdw~JDwYBZiiC6V0_YL)XIy+IFUE5z^TBT>Vi+i}Z z@$CKtHp*HTar*Nh0 z4d!Itxms^+dK?|8>Ltz88N=61XvAz!cKO1-@emU&)Xy%shI$!g;`;h3vfCy!5FVuM zfK-$C2DIpGB=wW|XGxE&`dI@n>W>4=*~`~FV?n*Ne0-XO%xXtZ$IO8(&aj-nonMl~ zqE^S%L;vu4AkW_NoY-@~_M35oP=BbL10hixe8V{mI@^CQ?7W_h(BgVv#v=rm;q4&}fhWnCnU9SWGDJiccr~ib zvV}k28-f`Giit{OJ@c3Mm&-F^a)N{@K`WFLf^3N-(5&r|rMY+_)>~U!(G(WIDsneo z!BTsT+hBliL1c_b?9eSQ1%LSEQB>#Hy6i!JlDVp`siF)P*Zd*`hxR&uRHh9%AN1!8)^U z?eUB3Zlu|G<@|_iAD_^HtD1zzG2b|xlC2`gP+$rLuXhIDntjOCS>^U;kgFE8Y;_tn7v zfuPBs2O)Nc@GB|*&Gn&V+dSLRAO7%baXJB)ztuC z+d0RXMz_`Cx~)MH^z9jsv!N7ZwW-dANTQ7&X`c=2T<=v7S_Ph5HP^xAx1OsBblh6b zZAiR>m?=&B38{`{%EuZ8j*x1$yH)ZB+~sA9m&1G;jsy`fo|6)I`rc;o_+(J|aLgE^ zJG$&kUiTwSF3;dvm~{4!vhOMF`2HCGxU`E3h8J$3k6lAwo?B+xF5A>YTJ$@?wl~y4 zR#Nrt_2Mf(kY1EeNEaOyJwAdr`^vrwHm~VlIEz?~%`d45VCN7%|e*?<8!rUUAoj6?c9bpzLyXmNM9o2ql)&G(G)lTap>w4I z#0ChnK%u&a!!}}n1(k_9eTgzKkZHU2P~!tWL~~h!zcr(>2A(LRFd|m0ZdR(nK^= zJ%rUD0`8SB@ekXp>=r^GKnNz~PbjY6lNi_rNsO{&9v5uDM)s}5`k9)|*}*C&2#*v_ zRq6yOik1#kA~qSx_NN?Nr@M{kTNxmgvicBiv<7CyyO@d$7Y{?f+>gFE2Kf<7w!Mfb zACP9*uIzh1DV9}d^s<+gT#f+nF?dKI`^SF-glXEo@);7f-~KgyNYJ#S?|A^eDnE}m zZxIe(lERIPB@ROcb|{`4*D*H0>kA#J{adVygxSFAc`BnmYgAsxLZNmZw>6? z)F2L1FLU51$f6N4`HH{BFBojy$UZ6ZiNT{ok-k{th-FMBiB}B7e`Vi6L4O>n0wDa= zi_?A2%%h3-C73z57@Qz$N>N=+LUm@MgDLt&&f}Tuae;u)kYl8QHO@KhJ7^Cw9G**M zZ#ni;Lk~j3@p@R3vhO@d&PvC2Vd%vaXD%idu{Jq0G3jAR%C@b0E|*lEke-p@mOKd4 zk>m6RO%Z&1sM2xIgn7S$UfV1 zi@s6!ulOF}-oE+%hf6-mfo7?~m6MI&0#NAtOiO%tfMic@ zMWH4(R>rnswRS0`hQn%tu;Z_2hV~D83r^}au95HSK_GF73+0AwCl&s&!_UuuZcNOF z{2iydCH4kc05pjzx*NtU&8Bjo?O;m>2`1xbk7s|Iv6+^v*$M$n0a6Q`z8|u_+VvU8 z4!T$w83FGSD7KaX9;Sk!VYc7v&HH!1`sC72~xpLlu2FGlnKPz#T4MZ+Ufmi z%qV%ag$Z-%bD8@1li|$Q{N*cAqjn>gtnzEPV>L_N@A$6|O#|K!B}KeEbZ8;J{z*D< zdfY)@2_IU!#wcQ0f7C zqydn3Yge{}8Vi)j8NihT@;efkYO}N%ZE2lhThz3)&WGPj`cB0pdt#Uy8sE*qRCsNu z&b)brQfnw@nP6EDv1zLuR|mybOvv?nau@7sb&q1x1J?#cCfMKm6{GfY&&^(x#fGDU z<4{j9x0oA;>ub$F^^9Z4H{J-u5C#?%+d0(OF8}*VT8ghml$Ben9^6=}Z=oV%+zv79 zawh{qnY(Q?O3I5bIZ_9X8NN|#82ppEZ=B*Vv}08&1j)odLR(Z{^)x?BL&p=MJHb#cpQU&;it54F_jT2!1y|b_%g!d zZXdElah=Pt#oS7?CXOEUF2$pQ}-S2y!P)Hw#oAM9& zxy-eMo5UM;otvcU_q2wn4pqu}(AUxnxcD{3b7-p)8xr+AvseuK5?ns;CvyB5=)=?a zYiDtj{v%O7d{2%80ti`;9O+uy6wIXDxFZR3BG*jc+nb~OBi)-v-O*n6^Gn?+wPIvu z%^4hI&o3$%>NC_^aJw5y8K>XlvPA?6)`qX2d92PovOC%k8*2rgg)x3MrHv2&9$6n> zVcor5m%{P01Zf5&NJibX#j>n~mLb8io(HM?HgaeJGn8_xSz>V2vLvaj;3cQ|ILsTJ zl#0Ipk(O0-aa(C7yyCQ^?JYer^fTn`S0sFZUMs@s;4Y;Lt6LF)=|CK+B85Z|W( zT!MPUXiky-=~4#>{IuBv8R!JCFZTEM-ME{Pf$alFu`_1X%`#iydN*NZd+t+T#Wq>! zI{L4TebjdI*>>if$T23!A|llts07I!L5Lk5o9V(;dp2%=~Ih| zzUagx?QiNjWU;sJ?H{g7w)s3dGC!~yJkY(n8eF7Jer9XTcxXn8p*>$##)iXuy32Q6 z-B?)fnV-*3`ACG+CJBQ^)3Y$P{ylqHLB7z{%QjZS;fZS zzl7Mjid-$U+KkW#pR>yTD7*t(4AqysCobMrMEfS+KQ>W2Q9O8msV@ToW&ye+55Sr% z5$~?AF2zJ8Eucx?!I<*=*M|&pX`2Hnkh7+r2;#2pkApsnp{2erZ8H*fQnb=G!`&~p zsy`#X(CW?@4#VEZ;cU3?{O>1%@Zq$04x>Rl9ajoUO&gp(z-~NXr$d}rS5&jI`jQq6 z;BSY6jg4)k-{bK77nGmwO8w! zkSS_9P8qG2zD`xmm{nC30+@l7!9`b9S}Iw7)T~226er_Njx?`wc{dIXQijbsmv2Pt z_zVI}j>Xv5hq1)tHMT^x@2BK;H}Vu~D)h!7Uj;7AWiPD?bQM<*BE~H?lGQL;z^;bF=b^P7qsB<&NM%Utwhjw`+~9UDSh@3&o<5;TXWi#@SoXm(j{RB`F(;E`>i z((Cttgm)sJc0iM9ZGQ;h4NYV@4C98U}P=zwY5GhK=tmfi&@sURq&=Jxldb_Pqu;kl}V$st#Sz- z(-BwsBIZNb;-mP>m7K97tR2_G5wKXEJ6gzrSm_cM7xH7`aj8O6jHA@Q+ogc7n+5b- zlNNVF_0u*mD|I7~U|YP3?hoLfS+M?jKpA-;!$=@SY`3WH}3<@rX&+g>b0xcVs8$ z1B0HXH`c8GXaQm{p%a|y!^jd>F3lfr&&k^pT4gaefW&7G%cje!*U$S;ih*4Wge3yb z`I}hv0&27b<1VoJDS!me*ZyX!fxxiNRKvI5%b`szpC!E&ITURhC4+!x2z@?^R3*N2 zWLaEZJ~yKVb1tB)y&Csth_ha?VgbIDB-0%Iq|H@g)4J$nYqIGoF#z3iEa{_sPR@X{ zReKl3^MI6b011Nvy=N|3-V%Qs1I`l=3KAi~wK1XG2qT5QpQr6eYtuQjQG_p;bY_!D$^vLD=sK^Q;GF zv~{#)6;uGm1@JI0H*_cD`CsTRKeX4;z^dk%w){eHqq4X#3NUG~%3Gi*_uG7E8MuqA z=pd61XxQW9tMP9TMPE&^-u7{0mo)0oY|t(n8j$q~UX=N>;%Km~*&^!Mu7~JImbcx9 zMJCX83$?KCqt&=tQ*FWu$BW15#3ZSdNClTeER|-+KO6TWXvD?IeCQ*9BDQA(>-uu) zW7g<{S%BbJ3{XdN&Pjm!PXZaiM#pV!=Ys*L$>G{at4U-0$F9|eg8Vf_YKCx@U=THE zw*dS)8Gy?1I~oS5ztvqPQ#CUPhk&(9w$RuiWjZ1)iUqllQUm>LTb7=sAp^goY9=_V zJFvq(ORowOWjVAV$PR!iA30=Pi2n2f|G>C=yL|hVA<&(5*IqJ$L62=fJ}N4XeDIm7 zs`6^K;wTsWpBSH8$gY&lnLW8H#sNWoso&?Yz?XEqX^M+jvywq~^cFww4=u+TdLM@~ zI~Hc0+JX0RKu7Gz8h&VY)G(BP^e;gAL~@q(Q&a2l-Ld?AcKk~;7|quH7OGcAB{sE? z(JI02gZ%cVH;74;&PM_YMx>@$&^d`r_Rhl@mBtBQ{CX9G`&%JVbd*d3ZGU>zScvud z($v@YdD*a_2_2nlNtLVxk_nmfCBawGcWV+_5J~u-eJ;RfgbjWF?t9u(tt#ydG+ot_ z`S*$b05nAKqvE8jcpDKcVUOPxVR6MMim%K zCzd*!NK&n!UME|D@nZ_oYT*_$hN{ob=I@uEi@De!lVi9PJp_-Y6c6c$@X^7)erVx! zvP;^dTrBkHFu(OJ3aZ)4oL&M;fH%-;*<6V>3$?bj%h~85mFakPvEO7m066^vsugYI z&!2A?85wZHS^DVjoAbJG&96TlSm)Spzv8r#vy=x^OG8wHdCD&;cefNaJwg(V15X_| za)He=`2O2qo~TK0z6~0Xa`In)o=>4b!Eb>9PO!?wWe#TQpUsCP- zYMAT2t1$}5uxN)0QD^|5bjE7=Q#NL(pKKb@vKj6rPeHfICF_~XCLH!H&W-eUEZipi z|Fq{I>M&Euz@8I@2;u@&VKFf=lm^whOpg#bySI>*C}*u+&4y$u8Cg2S1Wgp_cB>gu4?A0VH+`P5?@Tj2q7?8CFi0=BD z2WRgPi8O!M7fbr^^Ipvy!+Eoc2Du2RBDIFIY3-Q}7AdBf7DP~-#KQUGcM^2fk?->^ z#_;-hCB+re#X%j|qM|In7?9DYLrI-PUtD}493Rw-B~rfw5;JfRFQR4r?mghSloCZn zw9dH6Fw22FR){R&$&}3ible-eVzJ5c?znor5!%&QVpXQ`TpDyCs#xA?ZeBK`M z3c1)+0E<0FJzsOroqE0!c}t9~}p>*9qG=yrghr+AVowdFpWnXm%S*Vg+|_~wlV`OXy<;z%$N zxB}t71~4#SD&DzZQ|YBH^TvwrmC_`~I8~@b!z(y@M01JC16d>2{Y~s`i_<@_G*5h&XG&uf z0+*~CYjLueb|PgAX9npG^t|jMVa#7l#ih-Kv1z5U57z~A<~u^q65GA*AIkxU7y~P7 z==bk_4qF#7AhZ~FXBYtKPe3o!9cJwd0cEAmvW>wkqF{XM2Y}>_+uH~B1f$f=yJxP= z*g|)NQo|p|30&$11PABbeXt~NZ7olAII&|9XQsAUm<8sT63B&e~)lv1NY%Lk1MU4>-+_ z2s!=0bg>$(s%?F{D+SnuKf_=!V{o=sfYY}YVA~9IbkJd4y_t>iLr6pTsfw~P=5?r~ zl2X)3ipuXJ5m4k~i*73_7duIsB}3)Ga-||MkauY}n`8v=V1}ly7}KYiKwV=%ID-Vv zH_pdzVFm>%MWVyn8Q#ZDeP5m{ii&#qQO*{Cbc}E7>p>LQd1L&nesK8a8!?wYNcldKTg_Y5rPRU>RB8W-g?_5GMdr^wuN(&+$hPJMnmkIagLPE^t?N_?-!*@=*TBk<}sOJuqZDl>X9~3UIaw(tE(NO+*C6+p8BJ zhx}6ALGBAN<2gAiFquvLg;cjWes1D&bekdMpL?;2krc%vy(=(f*pB3KXaQO;UJs#deyF!)3JnCoesXM|0D zpkKC^%wag5q|8=!X@Z1=xL*Vq{^7LaR5c)hK% zZQI)Xc6w)7HkZdk$o)`z$YI7eCb^@#n=7}6AfUDov$TiBvJ<>BJRc0 zOX#;gz_ZD7Ic7qxdC9a+C9J|$*^NsMy7;DOdJ@Pv@K#Nx-YKcQiyfrl@*>@S^Kkx+qq@WXCg)Y0&X zRg~1>&%gk*w&BJr=a(RXu!hnAgb?``*I9&}urM%;KY9?d)Fp^+6!ako?Z|x65Z_t5 zz=MO*liT?BZwm5R6eOuq6$w*QM(`S|UPOw=Q9*w}l$A$@VFr@B<35kpjgPmq4rV_O znZT2iKnV)%E)TT*;7KPT`jo;!U9E#q!v?7RX9=k)LaXyUzD z^@oV}JKnJEeV;T2t7n&(-XC~g8cGT}UDAQ+Tt_q)5#Q`*2J{%PF3^-?+!aCNDFxO) zsk4Ph*<4VNqhPZHysO5u&7W|{$hH#z!Lhu$7_0N7RSF)VqY2razNb*IPFLe*|GR)j z$=MDii?-c~e@GL^33f}ZP%7(Ih1UcO?|DNn@HcG_dx`Q6F4*#m&e*cBI%C{FpvvPS zkpI-rAA`)C7NceqCxsmY=`dKX_>(U7Kmr8K3qtz@FA4q^9Kqdx! zBN7*VV$z-Zundt*KHH^=b<3kkwo`T*nrMqAH%_4cWW&W-AOg{Av!{_bmi+#R(eRO7H!}~sK=Vplx*o%6$+_TR-@`UxAbUh_K~q=I+o7IMNJBzBm=HQkazp!Qhc>bBGS zdr32uAVsK4uXNCuE0WsKj?}7syvGC)&|no}nlg9;;oUh)BY%64JcNG;bS6O74h;{n z3O~%E@V*~~CLETbmOtdgVG<6k!{0)wgl&e~uP#020xohz8#T-H2&SFaH(P=C_yywt zdn~iY)8u#QW+QX^&mou)uK9tq@ zV81yLJn?mV)(7M3SJM%RT2Rj{hAcyxlkjX!JVt504_Q3MSEQ;iWz5%Iup~^^;PSKC z_~W8)9Krs#uo-!LQduaM$ceu7IP&DF0CM2xoYM_0~qIEMTjSAHh6%i`0vTQBwARj*o* z?8^gQQ2<-LoATA?;0EH9zf{}pbK;{Kl1>ViLvcxa? z4&QJWu8Jz~8-GD$E-OnaReiC@>LCuWA{;zCC^VGR9+Rmfh~U-NX@@Vb$=vrA#3jVV z)4;eF^P=ws_OnwChk%`OP>q40wLwiDGXV7i5E2GV+@W%iSS6=4_Dx%!Iog?c8n*U^ z+qxFR-Xug{QkeAqgADXnISONJO{uavxuJ#aSyi>35^%DtwLLJL_8qjRF(Y0SF7To( z+Bd0*-8Zkjui1DtPu(q8g#um_Mz%=;v*V*#Gt{Z^x3uDjyy|~pRWIsN3RHqImVND) zq4Z(Q^YeNPyzHKgj8jxO1YEg%{^7~MeRV2FLK03T$OtzyC;~Ak_7fJg{J@hQ?aT+^ z=!#i8!eP(hE&A#lx}>zKzVMpLq&2!GZ+GqppVj_%#UhzUq8PaL?8U}_mgt~WL%J#| zOZ=GneRs0a)8PSr{-Hu5Ev8%))v=p&lh{o3_= zH41*VnNl)r#hE0_&Jr$$a8_V2%g=wGo4L`kAvfgOSn98(l-xl7nmdj1a>&I)fE2fi@gwSh%AjcPi*>d>Se+rRCsE!#pp8q37 zdS;Cv#yp2lLOm`%Z`JS1ghT3Gj<<&E94!f-z5Ph8Z4+P};@(B`{COa6*0k&F`D8VaS4zT7c=}(7VKgl zOfyeDkN3Z=`--wmpO|44@|_25*?8YN#FKc7`F&l=5TwI*f2ww$52rK6FcVW1TBQWF zSm=ytq!e_9RB=jFDN(*<%oM!(N&uQ(_yA{&`GE1I3z2oE4BR+zZG<*(c-0cT3))gu zN}b_r?tERr!O~Tt`aMgtVyJyQHfP)C+a7+f@~?w2%wJlmPdR8YN;Ho1PL4anzxuD# zf!8R+vrAOkeK{;DYEFW`KgB@50&jpdn)h3PnMn@kTqNy`6v5^ovSClG8!)tM{m|t6 zXDm1nFgbSqXCg(p@ZX{Pp7PW6!r#-3AYn8sj!!SQ9_QlELVO*^~vp$0cM1mD)<)k zgG%g9CqCh+^W`t~^};lJ*@G%GzfZ|zF^~8}U-|%?HM|H55F-2&EO31>JH zYYG5(`TWr@C@A#_Lzj|*=2Yr({L>c>8(Ma=%>v;_nj{S{Be=GOzGT)`wlooF^ZX2Q z)7+HQ)g_Yk1sXW#_~b%JyD}NPSJdRf$m9rjK~?-9iQ|+ssI18Z`EP`LA5cNgPE_1R zX2@FS{=5?#Phj9)5<8iJ8F7uGitF#JWDIP69#bP%*ZMf7928X`OuVLoOaav10|KE` zWHf0dA{>uMIgTnA6dd$>{@Q#;j*6l=oDl`1@zWEaHSCPijOvWy;W1j-`Ta@&=od~A zLjuF|<3uL!>IAbcQ7Ky6rA4i%YV~+Pk1Hu-#NVaz8Whl z{bW1*G{|L>HLz>-faF3kFCkp*>>GfjUv?mcN%1-wB$pq4tgIPna{)M@ds*peV6(Oh!16Y4=} zl=Ym4Jxu3KOsvbx+9Gwh;qiKj`-)akFR700d!WRd00;DnFWoqMvArtc!G$Rs7j2c% zHM|I?)19%U)7a%?vE0?0;07v2S@?z;*HA3-PX?qL)1PpC1ZhsGvBw915}Kp<7fCYw zww>!P-uPXD5;BsKbTTe|qN21%UFjnELpD0KcJhz!*F4)DGzK1bV}~^*S#rJa6xdf$ z#-UwZ46O`}@rCqEYq?=)!h6l1E{E%mM~bgbd1&z%AXQ7j>h1N^X@{|9V1DF^$ zO9+c*?Z!xyp!;al(R4#{sC^JlO7zRI|BtY@0Lnsb+eJl5Q9@c$IwYh)Qo5z3Q97hU zTDrTtySqV<20^+(y1RQnFRt}{|2}7*eTG>x%&cLtc%QoO>%PMOVOuBzXd+mG6B#7O zfN?G{twvMZ#_8#dJOKz_WHF^q)?}P5b}8JIn6qE|0*;|K#J8T|=~U@%HPT~aUS{lu zBn;;L;@pT|JINA&fl?x@(hm7og>QHo1EQzGZb3dD4Ut!0T=i8L*`R` z6)C&8&o{E?H{4G$w~`6DzBHE7xOHua)kyAf#K8_+=AuZp4JTKfJP?!(OnFv94f=ev zX2tgZWHu)KHw|Xy;41`W1C5CQ|Buc38Bj<9%4!sf3~Go)L0;aWSX#Kt7LNDG0qpI! zA56(LgrPnPiniO9v~ora6sEQhZArK@uuqMl*(c7nKJ;T)g zERX-tY9QP;2yz)GrV7O&@OQpAr9uFnoMJ#*Hw0wAdAU#~)JXpXfQ03O8LYgpFf6E1 zqct73$oP9s6)z`mZh7fsWo1XkmioX$+86#JP%MiFc`&`I(6>R6y;cVUUz5kzxNm1v z1?EeZrzwuAE+f;BR^Gnqw1Yl zSLzow@aa9x+~DHMl)Gk<$+Yzx=gmb|&Tu&X6y=v9+>K$#U3i0RbNh_|{xa}ME)lTl zA_WA>L5y*^(%|9DgEVlB{|2$mX9d@{JYWngxYKOIk!rlN-L!qZ(#;G$cgtf7<4HP= zAf2mL9_qv_ z!uUP@GM<*LN1%il0v2=AI)LB+zn&?Tv&B`P#mw;Vu4ilhrx~T1~5NAkWeaws{M_D<|ZKPPpg4Z7;v(ABh;GR z^epiiF`T<8LPd7WtDDSLVPIlUD7A8Xh?hOR^P}$^EJL!Hi&sOigPi2m7}mk>@X&ko z->#q#(q4VwJ7k*8f4PC2#GAf;3xzZ7B$e1~{%M>50`yFk9x{sX|0r+~oiGP@8_sOp zCZ)KTIO%yJ3u$uQn_N6<;4l0HkTqqX;|zOwl5g9;6x`Zd`10KY>|Ieh8qV~(&%*RIRU4TnEBS*UV_ zvh^$?4&PFklqK)Yjop5dzKQ$Uuaz^3iv>Xb@;9GsOH{|r!CUAyw3YLXkWmeIs{9!Q zf$`1%l`FW%a(s)|>5k<{7p&2BUip$iMfJ`dh_-G+Eb=#U+aM81pgsMA3*~L7qM$CW zhk=n09en+>P7`ptknmN9Fa6p4%Rzv2cnsY_8jUby_Gw<*=L)#}|MzxF9%uoenm(U) zPcfkZIl$MD)fYX$6Q2pV2WdX7-DHV_Je>j3B{K5u*#(!3_?kzIYpyrtGTlf$F;?FO z7AS^O@C=fG+`#RZEJ4bKw&(a)sPp7~m)1d{K*-GTir(iz=f_9FfJ#Mvt<-ZCTw_7;jGcJWk z%GCd0;S0M17JmF{)w(Dw1NZ;{xvJ+cvkXfdfEOc|5)L9ZT(+$@t$Fid{$|K)E)N(k z4-2LF1)C8z>7cKrzhf!bNSC9JI;VLLO8!X+>nPt14(q*2399nD=r1~#=R@*&o0kMY z2KLtdqvbYsGF%d`#l2U;3D8EZT$m0e2w%J`-2U9!?&^Ifo;dpi!UOsMsGWvZ4O4*O z3S_ask3*Vf@Eq$Onq-{klY{&vB;P4>{G}xYRZV`fY`rg~<0nzcdA$E}0XlK=e`Ye1 zRIGXVv&r5d6?bIC(oXXk462X8Jkgcd*(a+=DXv%qK+J!-GZ&;Ab^af_cMH4c|F3Rb zXy{Nj=rze%0twC!c;FW7oUiH~sCefS0pRJa2Z=wcX1wVO$OGwdm^dZACVhchJ$<~g z(&tvCu>3V0ElWdJCg&}f;y2OBLq?faCLza%Tn1*BKlKB|P2aHke0|{dpgtJ_`3#>C z;=T^dHgmJ%KDB54KDY7Ej5Nr;ctEk&V*ZJR=kJvPM{m9R&DDzUj$f*%go@;{3Db1vveh`cZw|fjJIf#@p<*TX0hH$^-Cw95`}=6}3L0jn zIKOi>Sysq_co?!{Z8x>z%9sKZWA%p2-Ezkk_RRY-zTOxl5q9pQ)oJqzD*+h*K;BrO`V|1N(V9P) zMefQ@>Yq4F?Wh|{Ixq_^Iymcehm4Qglbrl<@p|a?zQEJEL8{H=F=%hcw;45Y>ZrV( z2bt$2_pGy9MpfL{weGmJL>DZ(1fSO$d|jtZ<^{lfLVgN4s6@VSdx@(ymH@a_^ge;ylBn8j9)iBoECr!J^UZ=2m5S#c(|61N zsxA%-4O4wU#Uzrql9v_t zRlbXt{;ICto&{Tbww+x&dV_8T~r zeE`!jW|N7xGc{jv_4M@WQBOdQtYxtV7mFhqG8CgQ1$I3u9n^OM?TqdZn=Jc5KgoD=Ca2?J-f%S#TL04g48 z^%L*(>m+}v9X_@}P5P8#%ARbgO0yTQCxlwv!~p?=uc5mk&MF=#A)T^{2r8I zx3^z6B&Ev4B0l#<#sFy|P);OazjG=lpRACWXErpvrY_9Cz%Yv5@jcDn@f~~6w(mZF zR;72cpC-eq)4|MHv+137p_ApQow6Z(ErM_VNO*w%Klg<`qM!{DAtNRI2wGfty;AI} zkt%ZcQnD&g6!5{5{J4RQtiHlZXsQLib#j)L<>@##Cy2U&0Ae0n)Z+9J5tNOUmHFtq zG)g&ll(|xScuBxn_D6-9$4x!#U_862X|7NwTsqsD@b&HP7c0TeI+uf@g0$UO3dHto z$rijVI?#BA(WU~|xb2DJ#Mu+cDUAwUgqQf7Q9M_xXdqWP6^F$b9pdaLSmz8BP@Aj0 zk-#dQ;t`ex>}~W1rGQ~uz0+f}D(oA8akai89q-u+iNv4#9E&q@bSr4U-(+w;ky z$4ib?yCX(-lHDim_fYM6DOHDHq%R7^dL}AQP!PHT4|zV(P8Ps}oF+ zULd?BLOx*XAiRA|tu@`NciNq=6f(4XHPg&pc_x54%{XXDF+QP@5is-hi;AI8MiRpo zIT&?E=T%yDl5;w)z93c;NAER#kG|J?FkQy0?e^pTyGsd?_JzwUF#aq2I18bB7jL0} zh{BEzKf^EL=$vbM4d<4g=8r~RT`1Zk*{^70N8c+eC|o-KCd@_aiCOr_dTNz4RnDft zp~p(5dum~Fx{8IES5q*yS4pz7812B6|1hMPep6Rcg~hq}You^#9ynSaqgcj%dmv~4h9 z{S$iYxmrO<@daBi@CTuXJcp?BF&C+=t>qql{WM^;cD?KxVtZJ( zKjIMX@=tk|!GR&rFhLY}3jfD~l1taDib6uFJHq>>u=yUnL!h5Qg;?Q$)M!Af&XOoz z4jn}Upg;OEmg~m3dv;r#v$`R&va8PWkptgbeH;$rjO;~@LYmFdZpR&9xa%7qbOw5R zgou4W`xfG~APZ&(7W2vPDdhYP<{M)%=`|%~c&E)(>)%o#?YY`4tXSM$6$f$WL*qK&VV0+&Dp$lZ!8!PY@i}L2vsX=@5No< zCEw`3tTx<9ezo0&Rgv{N%I#8Z=4{evxsmTeV+bjz2F~DF(o3Zhkl->_;P|(Xzj+G( zc<|B)pFdSf9tDmzRto(cJtFyK*KW<2f=Av=3Tk_aDBZ}@^R9?pm){l~Timrz3nz1< zkKvIKGXY=NI6h%H8pnsp7x;$AX}Y?X;r>biHFI;STKzpy=G+NJyhXf}uf{Aw`Jv#jpsWEKv7C z!nSQ7M%FdXAkct0bN)?5w)y;-$l`!2m39_4Q;&-=^zJ{40^xG!9!S=J@adS67N?SH z?J(M8MU1MwtHivVCT!4FeC-N45uL9BjRg}Z6xmn)1R>@y(J?a0O%gFR$b5OMsGr|N z<6jhZg_!nYlZxD8MYAuG7|c7=3dt&2_!meZ+Foch&YHJj6MAMC^us}_76`49{6?Fv?NeYM+A2OODX@T7Jb1gNx&-tqhioWtU!y$(rILM1}^5-d}T5`{PPBQ z7y%|T<;91v)J72cl~+~eRno9+TzU{~r#rOm*bFF}Q%~Fn2GW_GFGXXaY)>~)x5szH zw|YRDo?)AZ!&vLXlSw93`9FqLh**QU`}0nl6N)oaWuNhZo~Bn$AqqYgi%vZ(qUp{u z064tUiZcZV0$L3faXJhzE~yRi2p8sJVv5vsSi?YVlatC>CVY0dktC-sMzWA8f-Ej2 zg|RWIX5K8+dbeku56Yb{LI3FBj_d(3s>51xpQWK>fG9G*jd6y4ySJ9CM<66nai{a~ zM3PfbG=^tapC(_x*x8hs@jl$1BwP7b)>C`vn) zFtBwoP@2|LI`2>0L>(%^OI?`ycIzHO&%+}f_7)9oxp_j=QjAJta40uogt7GC>E2`| zzuA9w0wC^yoiJ5phmL{2!rJz!Tl9IRA+>dhr3)RrJgxcgy*EZ#eK?ZmO7~?jA#Z#Py_O_|ZV+59liI?p+Y?-6 zINYe9t}b{h>F7wqFBgMB{;q#U&TTNfi5x(`r#sWBDo;RSSTLABy+RYcqry{+?WIqW zo2$HBK?i9nxM$jU_bK5vaho)eYT_F_%$L)htA1AYJeJUPpvID4>b>(B@w1-oYK}fFkge$mS>K`qZfemG}B;PSklt3;Bo)O;3)PoD+O~p z0$;Q@xg>``CpD{Tfz7W?gR?9e^igzvG3vUDdvgWIpzINxNF70{+O+n0MEW}Am_%=#<1@}DVAG5v3B2iDBpJVYu0FK`g}M0 zO!UHC^tsZaMy*~)A8`LbYPMn?te_N;snRkn?-^H+&}Ayc#$Q1e`H>xlCbIlAfTC9} zpC0J(M*XlbFrG6~pp^lFQ;;Zqi8R7sIQ`Y4WF8*Vx-@^ms$r}~gH2|+&cUmBN>Zuj zcgs#h+6EAjqZWbP-gT^BkL|-9vOj#8Iw+KdwA`BKfvz~hQQgE${Iz9>YCZ~8C?3Zy zv1UPU%$$#s!Y{rupobL41zQQw+9DqBG}k+OvsBu%R;L zj;B`(4^`tqR1H-)Z9QEs>c&ZseCUZ=yml`y`Jb_sB(u2w-RVleJ9sid=)wdbj>m#y zJAc-t#f~r)=NQmQ1lmvz!4h1x;BD+Hf^sL>G97*=bP|IjM!8j!9UWm6$pk8xjxXLgHj5EWrwJNd2bOn z83#f7hmC#(HRS^eJvNg2_~ z0Z4eb@K;5J6>4WiU6xmzj*)(=aCh#Q3ElA_R&19hz);WYXQIwX$n#-@C~_Q=5tpL) zdQpeu%B|_f$CB+5n5!)nZ?SH@H5DInq34Yh1tE|e&|e#1v? z|Ciy9RG4Y6>`(T)f>QKlRZ(!lEuZ*@ZvR5_%|~0?Xwj`-dSiL-Kz5Taq#6nTqApx>N)cT5 zR#cljmzl(5{A7{Ev_a5PUAU$hw%hD=OMF#K{< zDLX%ub^C#{L_;>x(BL3BxgLwIMm#^ciD>J6kyBw(m(nSsLr<;g1YlfQ6&sk`WHY4QMXYrzV=ZwgT11gB(4 zE69uzGE0Jpn())OuNvAN%#vGx{mm*)H>YrfCu3emJ>Qc*VrKcp`Q+ppoF)amDvIe_ zwZ9qRJwE4t`kZ>gRkwLIudva7A}%h868?7O{_dGWb!Y)_EkCB)kQdU^^SqU6`#~19 zwv;c^_Cv;;_5}NO3+woB0p_l4$k@S)gP3v-l|THQB|1(b-x6wg2}1smx0fGWuS%@I z+Mul_+Bp~RJ^^nOr>!x`wo8>2Q!=}&wBw**GVq`QnjE7S@23lH*#9G1bJE&DCxr~I zfmsmH4LbF*4VkyE%phd;t2p|wvzX~%wgjgJD~kpf<#b-`g( zmCa23i5h>5F7fzs=YhPC5C_#(8^%+z$-Tt~3Y(=5C29?pXWz#1yKxznxt>863G*cj zCn1DN)&VArdhI?_0^=GKfXkSG8;i0vHK54)^+e;cx$^(?nV0O3rN<060Ym5R@ziOm z4hIGhGO3r=A{q+@0>lM+mgRQhPToGM^Vwr4R-`|43pF>%h$ThjSjV zv3;9S$|eod2>VJ3{}`cz*!e`5B2{hG4hAYSDWMAw@!2APOkCc^l@ZQ_;1z zKyY&iEhj6>Q(k_<6?J60{W~OQ{(bm<(;en=)zY@r=AmrDLz|pxX1rSZqS&Nj+f=zIQE*>zERclU2X}yq1$gPgTsI>n(+-f4}5I3!6dY8?h*AWB^RwR zEn;3yk)^`;unv--H**rra*~oNiJOlDFeHRlCFnp|-yvAJyZaRq4$u7_ckvn2EC~^4 zF$j76zdR&I0O(&{)2Jg?#x0Kh)!P1i<7OLNt~~Jb&^TGWuCV!2#D;)?0EmxZl~yH4 zMe23AnHq_L>sT|=ogKhjqm}uCeguCp@wCdgxi#s{Khs z?1AgayN26&%wL{uoAugN1dv(Q4+xo`A(>a0xJRe8Vv+|M5M$Y~5^`PW603*ren_7c zQK-H0mVMFro?Kp|II?eBNe(~y+9s0)Rz;Zg2~hOFI`lr5+c}%m6d2u46cvEIAEPC6 z{Q4E&rT&aYweA~?OS5&YGN(gv=M#sX@bX6y5XN2EqkEY=!RIJJ%cT|ncIv!=`nc^p zr4!SSLo5G2at#mHll(IkMxi?#l^ci^dZSg(iW+>|fKk(9zFOJJ(xSK9B%4#`qX-d5 zeGm^UVnh2p#}J-z%l* zu{Biu6fw&XwfrhaZpC5Y#V+6O7FM&{kK0rE!Qb@9MbtJg%L$o6SYYY=O#KJYR1q=n z1)b(oA$b7z8khD+ou6X$ho-f&?fh|7prC-<_>+?Syu+YO2@rL~3AuU(E~QJ&1b$3ZFB8 z#)345nqbDez=0M$7f|AM#Pr-=@Od?RnKtH{Yol-hG10TBeg}v}htyJv;W;oMY|i<% z7;WSS@mHi@{aktn@&B|S7%zo-52E0q(a@&7zqI?x3Vw1bD4G>gtZwX8+1USS{GuXA zQlMzNmMWrGHM<%ey5HYEBqyswkd0j?^;f#Lgwn`ykxX7wQ#+z@p#|%VxqOsw@jw{4KD2jW zUX%tVJ6NLnLs$^yVk+}2K&iMPjx3O1A7CPw`0^40g?j_+1Du`kMETB|ScsJ}{ zB8oC7yE91pD3gP5{E&X9IBT?V7}L8z@*8&DW)=`3dLuXMk1#j+3HX(G)1O=LXR5RLdG8Q39E9wbbmU@n zD5Qr&-1agAL_z>h7Jq%o)SIl{f{iJ6$! zdi`@zw-O0JNY5h0(8$z)e?g`iqml4Co>RKLO9BWD z1%>kIZd7n^a5jjBecyS443j-z0tt z+*3ZHDwIDi9$E=U@$EOFMn+W8QBl}4yaPN2+r%q|n-6AbepAp;$H@H~zvF`s9H2r@ ze!6zijvPDockJ34D>_LXxsIc2jbKOr=Nug}LI?qWbAir`5ptQWylUUTWy3FLkE#F~ zx?u7ExfIUV(7mRX=2CRhe!H(~jln@KQa3H{MYUY+=sDe%zKHhxB4kvtCWgV)px2)S zhsU50A~zV~b?}nc_{uTvgYi*n-l3ql|0E#@e7LDV6k=AF&hxEn&>VSq{a0i2IVtxt zC+{Eg)@|u9VLaZFv4*2|0xxcOCzSIv{5^h%y5YaZDP|eqGJ7r(DWc7)2jKo-Sa*#R8TY zT#xD}rNftAE-!+POHU zdw#`7E_d)NG-aXk*K{2b>$xY!FVy*JKwq)bg$OkQ zB555s@uY&}14!Dl&zAxqgFkpYD6$N)TqFaJS|tQngsy?u&TrFV%T0%G?|XtXe=qGK zsOSEMxE|~S|EZzy$iRSr*LO+&A;GyjTDM6V5!+h(_*ysQ_z$b*X3sjVKGSbJ0qBia zLM7~l#sSOMniOAgCkv%nlLyF=P7*mY0Th{=$S1|*^bY%&8k}}%zB!T^<+Z_Sn|YMA zr5A2)ewcJB{3es;r+u5_*rXKGgZg%@c9FgtNxxD`!88JL|0-Hl$`P2xLI!34t@GQZ zadXpwQBI@kut-_`cv zJ4uL;kdVKhj^y`zzy+O-=FQ^HLaklTwgN5_!LB~G^O8P>`y8*bP;j% zasu@RO|Nd7wA&o^B|m``#yiNitCZvAtTLC1JivYPq{ZS*>Vn&WHGLWGGpsOL@K|%g ze@^`g>}!yK7Kmfln&qkR9UMFFQ(n-3Ryee_pQX5bQ-6OnG66c!OBiz0HBvjlVfhhgeVx@^RX|4{M?^WMpJH2Nc4ZPjBAuca@Z84Vvwqkc2tF zTAd#{f+X&0qWKD)-XI{7V*_4ogdh{TeO%qx7_s4EA;0SDEGFdTQf5OVpA!K35v!a1 z!S~&eB4hLND3iyxb7*}-D~G=2&@Sd|hwZ!1i{30`o^LW^vf`4GZ-JdkIDo{rYRqBD z4WGd?8$e%nB0q#}wtFKkOXG?j?Ia-}<_;{DX4X}mXY|V_@}wQ`=22+DW-@o4kYKY< zWlQz^q>IQS1dp!66e5&UFfjJ|_nXhjTn*9=;Z_r^E38j?fHa}Uu||#nICn8Gg$&y~ ztXhuM3p6)pM`bL(QazT%Y(8mYY_K?rXy!v)HY|z~LCGE0O@s&?IBC`_=YJG#cBMaq zCVIF&CvvG@(u(;W`HukvnTwO1aWA@3{J}iQ(4NLbN9qJvOzf$!aoII0=(5x zgKg>vpyoa^Drg%z-x`e-4r!7?2_8!BH?nx2f()FgAj1qu4#KT*WN%E{n4+v`6Q1mVKJg>Zg>`hx z;l3W1UkC=7z_2DPbwl`wsWBz`7@PNqe0zjU;7LSvz@5q&&l}r!PAwJZpigk-O`6{` z%UtoigL)k2T0;!GeExUCE+qbn?DKc>55c_j`Nc$puk{*cT?o<{c>sr4R;iaQYITVo zp=)x%r$6IJLX8M$hm2K}l=RtWl0HyZRD zu{OXawxbuIKl}x|WBHY8xFCrgIQ!vjFWfz^9irE0pcGA?D+veV1O(NZT>)i9vW6VY zD{ElAoF&T^d(`m;+xw)}mmL4Wk=e)uWgC~7-^-C5DX$aZ9(99?T13GXO9*9!b{hg z)>2Un9xiKmZ{J2ZtErk5HAnj+tC@y7+}anM!Tg#Ld!LlfOvvJ2fSnl)+RUMo^*qAX z%W+bW#fAR8>FNUf<+5uq(mH>D3E`D1J0R6I)I#`h47-$58~?Emtv$Z|hpruE(%_vn z@R2xDgeMCxqqf2-u|2uuyy_EPP;j>`X2pF?eSU5Q2)+ zrM}B5xge5??39kg!3%$XF^FnBzo{t-=tC)CUDpSw64=a3E)z+(fTX?F4`r~S!eYr; zIZH&EY%y1cS2pw_si-L01~jb01nkeWJcz=K-P-s_VW1Jn^5tp*9G@+^}bO5GUEa85ST* zCIxmDX9?D65sCoafBzTV@3@VMB@o92_8KLVVbrN&42bVwR6&&{&dSYG& zwiTRA?z$!}JJ6x|PuBq+`6`rwdLGBhrDnQv&^GR4aM14wa!5T61)*pID_>^E)nCMl zm(j{&4}X-XFfgL4wnuGG2X7=cSB(xRE!9ik^++(jEHBUsyIH-#00zVA35i@Iqmf2^ zceMy_)SEu>1!B6AV~9eQ9sCzn*)-?uiyII6{!UK*PoDa3L~8_JTiWyAoX9P}UzKsx zbo*_89>ud9{^@0j2um0I200706jPU`c?6xF;w70{S9dpB?)h2t2lYVR{6CvGz>{7X z4e#mzs2KhI!ne^^uC8J=GCpRgQ)h{h%O2M&tE)pYqKJnMqT%WZ&{*rgYI*OxLsh_E z15Y)#lp*ILO_m_Rkq694p4$UdR6K_y3I#ZYf)|SqVSYb)yoAV^y1Kd=w_g%Cdj>l9 zE?Ua=L`W_uz$f6r_kmBae<++CRh#xBkPp+r-_t^E_uR)#v z7*qV;DeU&-6ELZuK!gpPi%p1-ya;ZM+{dwp0|e9U8mgpL3m|0H5R*`Hk?ibjajTof zUL~6j(Nk<~i||dc?1o;5QBotmh@GW?j*oC>C) zXjy;lcUc9FYT{qO{X=)WXfa)| zj=(fLJ02=JDvnS{52bPZuz$u1nv31%C_IO54YrSyC3}KuDk^GQo1b@xk31k%_(VpR z%Vfh%Ns+-n4KN_VQ}d<|gWl*LNjd{Gn2}#mkdX;Bg{^D3P#O9BV*T=)0Ic^fWGK6Y zV+rYdEf6;<`fjK2a)s2~u;ZP|gD^ds{-jAjEE<4##=V4NUZP=h-~)nMy~U~c5Pa;m z%^&*(jt~&AARrmF+A5R8PA8fGgrB6g%#y`DQFv{~A@BF?3WszD3Ah~_gleLM8ckgr z`3(oAnC5FJrsa|y)!s#oZ?#7&EZpMf5-NkWxi<60>!qq6*IR7i_L$3;7AkShVS$}c zls#qG5s=$UmXj%~v z`B^7JP{U$YZ~10j*?@CtFRS)>{bT>lRXgg^V_MHo;QroifRqY&wfb67Fj{NGKk9lO zZ{R$wXLh7&6>i%j`6LQ)vAD0hilp?H;O##OdI{W$&Q>ziSAd+7YTIg799kbJz~#*& z`O(qu6@?i!QSPg+&zh;*51jlm^YX4GmCf&6$P|?n#Fm(5G>L%E1?V0n>_00qZ`5MAK@Omjyu{%p)b)N=in{btveG z^%+iIeuYJb;;zVkyG;@*oE(rS7LWoKSyw`fsx3M`zuR)P=QDtMTQW4Q)bOe+><@S{ zRHVfQjecG+*?LL87p|#tdTrUoL|u=e@vv-wzMp4bb>QGczP4w%sHRIZp7-d_Hz)zw zjR3%{)8MxUD5+WwF)z=Be2Cz3IQAm?U}Xc*21=sqMX1@Gvxd_-48*;M zDTf5Qk&0BGf}d3A2Yx-7AQ{CeoT1TXOV|?*85I?d1HaAyT}nkzl7K!Iu$Q*lz8`$` z`%Yi62IxU^a&{-GJD|)|t+QzPM1g~|CuGwDB1mNE=?^l%lBO_z7Bso%6U8P|1Y_eC9+0F`FflsMX)7RkQ#~ zmCZW>0w>^qv2{M%Ju*|R;t`i$b$2#7%OalxM=T8fOkvR&akn6ASJxH@cpvOAc#6#% z$oV3Xx`DpY#;4}N3_V;r3KK}O+>qL@bRJ?cfq#fZD3G_3j3s3fwK|p>WnN$bTQOI? zc~?EQ3O(61B5u)s~Af&BreByQ_mJ_i}e-qBC@F~G;VcSS7R@2opewO{n+9lm!& z0BG(HH{grw=?Yx}romuj2!JU&yDGE!nJT{+Np_s~=XCR@e$MU#eb=H{rfcn4Xyly? z)877cd$aawpbo{#!mwRB2cc;qz=O-;MjcZzh$c_IpBnkw`Ck9`QDbEX;WrnQ4-4fa zYC-g@tb@fWDqhU*u|u$*p_Dy`3bE+jGPSn0hAAJZWF|)to@WbETZ#4Z0fQ-m%Q-3< z8YaoZ3sMb01bL1S;l@yAI_>-AsUtXN;bh}CT)f?a+R}uBMr{~hvPa-*Y2PPazSZ`~ zIZobC=``vy+>Jo7+-&Xe^Z98YIfx2Brh}|i^MzUg^^s5Io~zwWD}7UjCxig zSvELn(t`nW`q2tS_^as%k<8QdEIj2A?;V#E;BLDu!%Z`7y))rLE*iD%%ANTYxb?u1 z!-4`=qDY0D3h*1qG#Vl_)!&3!wMx>hIqIW(n{FZZM%?IG@ruZ=S9bMTlM~bV``nsw zb7brs`1-~Sl|En01UIcabo9E(D4-C4; zGPG&NW&x}|j;6&I);ZHPZRc?xDo&x>wsa%(UUHa(Q#Vc1p-AGvPoaZ2#+2bqHlc2^ zhTz}uTStS9jQJ9i4?_#F-^FB;P22{a{QNQS<4FZlqF>z`KTCc-bQD1qbTL0z;w!NX z@}u>xRsGg?3vt}<5?|}I3(*U9649lBX)5Y4DRCgK?6e@O`!tp|?EMCJ(-evVA?+TGyw`0T-?2zTw5D^7<44TJWU*yE4V)K{W4t9Nro2}k#BH*g1p zO{pf(8T+ZLj-6-JlM^&TH9?yk>{a?}w!4M!0#enDH zNoRRQ#l59h>h}PpPLQW2Gg0a$#^-SqnN|>Z^+`ixZ=>9+NfTX2u7KznvWeo7O^$g}FNP6N9mQ>7SBz5afDw2H9 z$&baWE;`D5$d|j)e38*%D;rDh9ZV|OQ;twRi3^;t>+&H5Ma66{SrlM(q3kT{Bl&o?aok=@7oL-Oq(yBkMp9|{U?>JOi#RhRsxcYdhM|UUP>LrkTz?Y`e;`sVUb=-J zBA{^FW}U9&T$pTf+R1Ta$&(Vs^=TBe?Mx4vh^L9p|G0E@g>WF6AmB|H+m$1{f>mmr z{jdv%O7wI(l+V)0erw_YeG&gE^x@X43YemD+i#eoLQ(Pb1spt;cTOAXaH!ZDs?GT% z&JcRyjPW9bKl+0cw6`w9+HZokKiCRXRU#inVdatbQ2+j`tPs*@S}AP({0hBN&E^i7 z%R}AmFW)=GYi#7Ao!8@Ybys^%(1&55Me$v2fBLlcPnE~^0F+LFQkf!yM3gR8laqg< z$bnASP%>sxx@Ih0?}%PBjr=&5wOqM^485PPVY+OIklKqPnpR%R@Y4qdc?-=Ll7@!8-bUT-(N_E2n#ns#+zVv++~#w(?JGm%lQ5)yvnsefDCqXr zZnCAZC!XNz=hbQN73#vRUqpr8>@8P*snlM78~>tZhPtM8-k~$p@GD+5w0+IsU`_>g z7Oo6ak(yw56nA~-=bScCt_DFtoF2p>4%bTx$FkEA z7yP;WJtt=0HZj4|?UTs)|a!=zI=(7tg5uE`{DEMN47As7;pigo}9w^F+KNTeIeJ zh2;ao^iLDF%#_e`zZfR3^EB@7d}EWrWi#DZZQyIk`EudPwS_^dFzn9%)#6WAqRH$gf>L&vlz+ zb%$>tMF(~Ve;hKMxhyGia1|KgX^hS#lqMqtNFg%k%MU5`Ki?lD80T;sA}d<=S1hk3Ohy^cJLh0pI_*#-!_o(?Vh47K) z3P@FCWlN%nQ1@{=M7@?LvO2d$OJgWfzA5CZd;IXjS_dIZF~njXqW;cAjKEo1iNM*S zye~HrY03CG3DQ%_cCl?c)p7Q{9N^2ArI0U{CE%Mv4o?!&a8wIWy9iU9%j$TT8g%#N zsbLTXLoRAE&Bw7(^L96X;hSSwv7zzZ#_7 z6##Qqg}N8Uj1_p`BZCfz0j{zN2HZCbudaDU$z|wn7vx9Eo#KuA;M_Fc9`GEaWsZNV zm%1C-CcAA^PHJB7@4B1ws+znP<|f=jEp`xe`c3e8p|(y_$mDjIpvrgD5rwQ`t=#7u zFGi{A5&Y`L8chUQkHq=d*jQFZ+~pg7R4vF#eb<66n@Hj|jZ?QBS1bu#-aQJ`y8Un{ ziTkrbkNQtkxX$wZuCQYgt4`1hf;naGlhuIOwRtz!uqq9+wJl%yUXcp2-WX}munUj{ z*lmP7<9YvTgEkVJS(3?hxZaqYcgf)7pr{|dlB1WKFsl!LYpcoI7egd0oy4wEP(U3W zFds;NuPDvqP zLprj8Of)WGioN#i3R*h7aqW%Kx*U--T5L(C zUca#Cr51U;OyV%zFXJs}v|X0Z6+tUjW0Qc#b>L&az%nigw#8}eZ>4SsB|Vu)#DwCGNsLlck!@zj2o}#S z?txT|K-R+DIpJ-#j=cGn#A#Yj%qD%(=@ep`FkCKIl7Ohq?^=uFlXt|w#Zi8|hJxdO zkNG5$rmG?^zhblpA86`t>wJ00ILBM(CTRK4qhGOhDM24ilPet!^u!wcH|MkF{gE_EJtrm;mkr4p ztd<$hTkDCr2q`fCasl+fWq*8D>c+jKaZTy%dTyO6QN4-D`%syBC)p4a)0b>NPquj5#g?UZv5%lPK0e-?+G`%3sgYD61bxP zu$CmNKC1H8C49DPL;%oBO35#k3oS&!y?+=V7kHu=3 z1y;uiyPWfvx^J|NFJZXv4UuP_!9#PGSY>k!A`gbDClZ1;v&#lEf|dzYcc!X%hKi|KqJNW=%d zKJ?HO6&qVvwq%#xpH}c0+tPrv8FKBGVUtde1Q4A+YZ8TZ??cAvD=E;f2uM9XY*LGN z?X#vqJS*oqQ6I@tV|(GB%H(P`b%pOHn?U&S=XhV8>&+4d)=nTtF2>@HW2iItV8f@u zE4dR1_AF&Lo9eNB7q$|^(lx!6`_Jva|307$Lt$mR!r<)j5fvR_ojG#wvV*UrgQFzE zrC*_@WNHf*KVLEeNjr%J2v&^dyCcr!rKx|lItFC#{UC=D72egZ zLQnX;Vcv_oLQ_l+ndFQeNj4?cT521B2oo7PG?tU^Z#P(tD~x1yKfay zz@kJ!QaS{b5b4fMNy8>Y*wWoC9U>jljcigk-Ab2qxRH<+Hb{5fXX82F@4NT@aqrk; zoN>+==MR0~=Y7^(bIvuF&1a&nK|=z=Rtm~mjb=(1WZDIfY|h2{XK7l|>;x`yCpG$0 zPYtC1)tIH{dB)BJcUgUl0s`i`@)MrTANQ5yA11DD^cI zQFj9U-SQ|)95INQ8T>>l5o)he#}$YljGz+>@b4<<{*cTV>Mm~{dsI)aB<8jg%0Lkh zYIH;+@pT;4;NvQ0=$i9_Hox6)VKH2tyW~l$la>P>dm@`x7q;&s1YJuAY9)I1v0{VO zPiKVZu@Qy}6CpnKjq6b#PWPB%Pd5#yXrxkUa-=>)OTK^D14L&Gi-C=PeWp|d z_QG{1h{cY0^6g|y!rV9E6RnCbC=`c5sQK#4ExF{K7^Pqimpe=gCaLc5#HFM}y2;jX zh}J?L_aqJZX z&Sa)Z9xVz60z`Gh5zOw8eZ;{)+MS?WaW-DvoHp8dut0gAwi}+-<1p|-Q z+?_87(cBdKe zoslXCjmM?z$X7^N4(xY+w-Kdx;8QiLII!jd&O7%3)ejikeW1R-K8 z?;UgVs%U-;!^ms2(YTS74q%yTPlPgW#e{{r#3*yLt3C_;nzMGs$iTa<&|Dm^aeawssKjV#Za-||xZaBff-k%YMAnu-kLgsaN#-Vss1&gnaY~$DLngYgg z{b`M}^Hhgq*|+0D*~Q?1zH;_C%1X8CAX#%@sXEe&HP|QV)X8XM9p5(2u%-7von&Y= z>3(>{?n;~1b@t32n^v`oi~Avd-mQC&X@bY?oC1m4;|Y>O zXT`4Ua8kD$Tul`nl%N#N;ex5&bRVt^zFWTs=u4olQn;vflfl@t&X(_a??AEldM+f{#S z9~cR`Hp<%DbHWllv9lD}16+DHN+3X}je0tC4G$?&1x%F&VjgEvA;Y46b~hM*i@$oI zpD$CDOOQ7ru+8by7^MuQsU5h(Cc3a**)VPz>A23B=8hYEr|I0(@^PbcCmDRI>G?tZ zgcm(%t41d0Dw|&oNP9^1nu&qW(@}$e)*O8kp?_Q$ulNjLG{~LA=Ciz1uLGp)v<#4- z6Uw~zi>yoO_iO1toNtV25;0*f3XmTV=e~#vbU*n0q+aOgs|?_dLAOoGjK~mNtkGK? zkW>EI0DFJd`JJa1O#joOM^s^FN89!z3OpeH;~ZrGF*1s!)=7^dd~Tl|N{syqYqTWP z>cmB%(7`+}XV@2&)8!S>`ZL3VQEUUpBje01A>9klJ2)E_={>WR{#Puwf5i=;t-*M_Zb;JcD$jeU)qWLxY3A+>ff?4(9f@zu$cCVLh1n z4Fd+jqfJxcfb|M_opc1=56;dLY~u+@q%s9^v_T-TOOL9)_scnL07CmJinQl9sWhnR z?kHGq5cY_CbSOg5od}zcno43=nHFdwD_h=LS(W7Y<+E=kC1Fe4^#w`K-6#9(I3U_v z1?(1z%Z52QIkgz+%L;cT-X{o$3i~yAnk>xG>KQ|V@NE`eoaw-7?B2~wiR_hI*Te^& zXr;R`O0gDuya>XdFp%SOm>I}hsa1V2s9rx4=X2Jeee-5FN+|ZjX|S2G+lI~P?hwVY z6V20lbPtlT&sA*qSfeqO68-C+VC#?ET9$qh!2H@bIU zc%8ms4y^WEV%h0oDeG!~WUww%1=ddFKZRC3&3|D=_d5yfIlX-U`gl3TrlTt!9r~dL zwVP0(?Ojm#QuVVtSz`JW?e7wA3mZty3BK5Vhw{Xbc9`~p=(0OZr^c} z65$+O7X9gZr`X+oes(sAdUZJd@rLW<&o`PK6*Jr=m`dbQXLYDj2sV>-B?=Zx-|l6t z88Nz)V-VciyM#1J52cZxcPP@6-(TgBud0rfou-c?H&{HBYjI*p$&kKrU7@PsaK;0~ zq6nDf{IdB2NEe;11ctOxNg-|*MB(Mu9P{z4Fl%C!)z3OV(183~M-BeBI%+LgR3ADS z$g6*KiYfsh-^7`SeZ$flH4B&DuYo(eba+IFyFm!MZh^Slhf!};m_Ft0@A85sLWcpv z_T@XvTAxerfu1@=887q;lYaGfS+Uur90B&W^gO{$=ZT#AfBE#ux980mlRSqj%VN z=%yt^MDdAj&lF|XACn87QYDX>X#zP)(nv0!&>7aeSWCC9!6&s_es|Ou?Z{ua4`oYo z_*|UDbtgby*hF{7GIT#`+PayikK?z5>vJTY7ldul=NxeDa@}O1uB_YRh$ikfkM&tK zM|0@ZZ>jsE#xLbPJ7I`YO6?|9bcy>|A(!s5b64ZoZ;OS8_tNkvBAx@cAf@Iv2x zxc}3db%G*j%Ixm1<5>5-kif8sV-YSyMj)p zi=9C|ZK2=Yc$zMh+}@5ArKk5w9hm18vD&-jp0Vp!^A>Tpj5{-nwDSKJoCmJq?t)nQ5w;O*7U3mt25Y z`of7?CtHAlTTv-CEr~k#9X*syw`S1c=UD(gC6}jK@s#mOHwkA)57VauRX0t6?U;u1 zgZmSQ-AQ~I2kEwrmOj5u(@zX<4q2S09yr8`BDT+i8!7eea9Aie4E1jVnt2Xgnjc@E z86fP$SdnH)qC7v@=f{T~tCVpdeV2QOpUt~>i7DKD(hj#>J=X6C2VY&qwd)1`L=PVp z4$e`z^3w^=d5}m6;J1q$@&wyfi3Z>YZ^+5X)f`s5X4}RG-)A58>x^%cD%#pi%8em_ zUMj6g(RD=0b+i9204WgS)!pe%KA=^?YK6V=;Z)zK)DIeZM%vHfExM8eC^#y-y_%l* zoxi9dRcYj^hskUWQ6lz%d%m67M{G5nF4e9~6hye~^N?HE9%yUhKlWM8+bOZ};a}Zf z?-V|X7HH0|^{zH%DDeHEsjN(u2b^rfO8}^P#*qpD#XYK9U3< z?!pJ2o;&PK1s89XoK>D+e@u=t9Uzq;w)5z+;wg^tvZg*+J=4ghmMM1TfT^{F7zkMF z`f=pIals_Nyt1M?5s6JC2t{S8{@4V65;?vH4-3Hky<4NEKK3lU#?W-jlW69+M~xGl_G`Yn+cucf zxS-DkzZFi373sCzoO|5~xA4kSEkh~HoSs|CsXe-LnB6rQDpLC78aIR}JI4wt`OZ$Y zVsGtCN+^bvP#Q9rct_O@9>XeH7nn__9`9gPRW2++<%yYT9m-BXr>Yz=EX4UcjJ+q5 ziHGS|ga&zBqD>mD^qNI`Z{klh3uuqJX;OB|4E{iaB+@3SQeRPZ_!>f`n!05*`gVeS6u1ew?&y=lyUov;^%Yp7d{AIUC7{qi1XhL@#_w0LV^lkRET(`zVJJ{&|i{S!{J! zE_I}h8;{EAm-gPIqljVS!C0>m(sv9G$A=8dCdd2Nd}~% zUti#TA$KaTe=0_DE?ktS_o@H(;90)|{&p;r$Jt>PS3tbKU9^z+y4K_ofi8k7kXQ zCM0({9>-f|ZFHLykH{e_;`;>u4j>l{DA021I|$iWmT3J36Nh*hLMj8DbP!%8w0{wabpgFUwhq34BLe;{pjbO*~ntL$DdcgU(AYMI;HH?Z6E8(M7{I!yoIm<$s4|4wS;sy zF?C`%CHxu^aqQ)0y)P_NWiRk%PFCG(%yAldOaXFTKxd_&$rX^5`uj z^(2HYFcT)ms{y)ZpT_ZhN(;Qv#2+hz8qI2;TZmKBqb&#H`L<;n{Q5kwPq{l^q3llA z#~ZiKev+*3P4%bo8?O!rPlYY5NKi50eFXF_2k2|H48WnMyYq00bfWrrSGWv}BD0>p zKRNyJWx@FDsP~Oqy3d8ScKt!1$o5B0lk{SP%b14C3y-Frn@EDw$k5-LCYRsagjAio z$$0mZZ3-{A$0xe(%#yHMMeqiTPKa#TTTl=r{lHS-9-*kf!o#*WlWf%g=gu^`d&P@O z?-GNvXQomqL5@Ce-3d)>%w}E1P3x%{?8?X0rzw@0VyYu{yji9R^7{C;s;YZBiupUf zN7x8c$e0M{SpT!N>&JTuWuB7h9v5zT={<4hXlfIr{yzb0b!|8BIz+&#$V_lVQ=Oyl zk_;=2HSWOo5&LO+v`{`RzZJc77q^)uFJ&5$OZAHJ8z5Rbdwue*9X!u>xpI}JvA7tVfxpDD!+(P-8z^W!A9 ziQbmeKZz^p2829c8)j}G02$j{O|Ly^qB63*0Du_dQ6e0MZA^+JxE)E@5t+#Ez0cxn zPdn0#|E`BI&ZKgU@k_TiCLnF*S_0Gg2_1>I|1Gp9n~f`W@FEK;ZhNkCA!i}$Dz%}_PomgUP@Wo%MxD-V)taq;sjsED+cKN zdl;4Wq~xtkC=P~-kx`Dywz1qjUrQQjcJzd9sinJ zW`BlyFfJIlQnb~tnV*an^$ta6uv%vxHT;oA1B|6fcS%H^nfwYBGRClN;oim=NvG$s zqyhiYR7g(S0EsZq`c^LGS=4Nayzo*|&l}=xqj||JCQ{t@taMQ4wNY8TNgi!%JBFCN zbaK#6^YxqI)B}#EL#_ZPCQ1_W_c?ivJSYn_DmdFHG{#caxtvc4&wxBOkS# z5|+QwTI4{7wixjF*ndA>SRz9?>DznTg5OKZ4}Y<3a;J#0F(d#_`efXNU|<;i;Ax>% z(JzLptYZG@INFrwe5WO}@jxwpB}HWk>z$(s>dIS4X+FE`NH~P;mtQIioA^nXA>PCF zfe&$FV5 zpY5c7&i3!KZtfM_?u1YyzT$nl-|n>lkIuU=jes+Cu@LuCqv4VS#r0wjK}9}UMg9%0 z0ju`pGq~4 ztS|{p#8EDS`yH>Mnd$cq8W;Ce?#o593TEg-Wjh)oyoh`Y0Er@Y6yYfTPLd@Ab14Iu zCklG6iLtIdYZZY_a8vRka;I zd}Pm$99E*?Y#d0vti;OQi*ryX#I^L+*KaqS7?IhsNxqiFeabo^#aU3lb}>h3LBQag)NQ!t4f2+LPjeG&aB@ zL(l6y@A`Yg_{J4$V@Q>WNxo%pM2wMnc!^*w;y<{*?}SzI<3c|9ujwaO)G9UcBKYO)&+Y`ivm=kq zb52`Hh@P3nZJG$7M~D34{ii3b&-)DOjUv>nXHCiY&fc21o@T<|g3^8sBk5&uQdETd zzg*-ekt;M3x%^xN2EkXB+ui-i97C7gt`$q*ehr+(Zq?C`mbD&;G;kX6d>N^u5~!ex`1G10686y9Os z8G~feiyv5j8t<3;L~_T05BIz4%#vH;irx$sH^!cUq=jH6D3QV_&cof&&O28!Vw@$8 zv?5q+bwjmc6f?1a}NjzIgprB>rAXH_%=HItzeL1Lzs}_b9eT^Q#Ff>qIDmX*tNC=!`7f8MM^$LMyZC1?ALTqv2**$ ziXy^MLy(Srg}A?d1tu?HT8LVNtp^qK&(CW1EY?@ECX?|8@y};{}SxBrF&CQL|ztkx`!aUgSShYh7F5E?vWZ({$$8>K1Y=&SSN_mI)j zVk}l~mDMH$EUY0z68H$aZqXnsR-L{CxmWVEgj$Ar9^DN{rh`yK1qHbq@pf@qPq|8F zkG3U=`rqoUFw^Y5!!A&^HX%zWR$>ep*=>6F<}Eeebxe_PCKl8sI{`8W4G=`2+$Q}3 z+RYbq*Ks6!@&xTWD8&28Nhcn$ zgzj{tl*Zf-yt%gPvFWC}wsvvu4c^hNV(UL(;{w0^G`O4jFCR4T9^a2P9*rFJsQl{{ zwdHYo&W|LJ=tm&D3-1>wiy`ehBR1qp2m<7zX!kk6+T*A%M(lQ}jZ_U(W^{5q29LT@ zsowbQg}(S@Li=g!vjzf|A>@?@!Y&J^oz)etKA{i-3}q(N$R~MtELg!E5&oE4>LIHR zqX#gm6epFuvU?hzh*fLvsuXBRQc-CC+CS+guslU^VlVpn*>{#UE@uBsk3Mt$e*dic z;brx@$Ntbu(xGGV72Gso$BW#NWAPn}`J>w>6nGkLXw#1EnmK-NeXe)$$-Kx8O{8*Pvn_WY0`7@J9o$`fP?TQZJDjX{p-kdG z29v68Pmq9$zT zOM~9tLd!Q_n_GfW=k2nMg3yDc*iT;Cw-irO5vb^}bHqgzlYPVXtjF6Ie<<@s*PNQe z-GwIo|C*^FpEx5HC)&1&_PxGYoc?_yLjt$5C8~;loTsy7;aP#LCBFgPJ6)F`w-PfS zz=2vrEe+1yhL^%$!Q7YzEE-pPg6*fVKLGZlS}6BUb0#WJkl--tStsR2H2=ZxDHmH^ z6RxD~^*MjoSwJLwV8+v-UZ8%>2K^5BVpwyWE_U6m2jV>&d?c=Hk64@jdkn)FAI}!! zSWekb&`skwc=lkHI+iNj27x#4Ob79pPyh@Ov4w?&@jyDRQGi^@V;scN^7oWCPa2&< zGy}YDUUygIyiTPe5^udH8$7B z%~bAj7#m)mlw~|!8hj>Tj{V?KV%Pq}K%zp*3r73A3$KCnX@n)x0K=kGqpIT3??6ZR z1XV$oPAugoVgQx%``WT-)Dd*4y+4BSRLEf8zP0^@Pej*COhK|O9Z+l|+JQVAJDU|7 zevz?~Yh@U{*6R^@uHr0M{cM~aj5X2sw+WeEq29f7E*TYFtp(oS|Jsg}c>gM}-;Jme zL0+=kgGWTN)unp5fHUwLxWk-)g&PlPjizBe2)?N>xvN9*cs*faz^k|9>1gLpA1e1g z_h!(OZHJD5d;HA{k-IebsAru9%m=pAP<^{4MqVdVpRplzrDkTE<}tEprBCuvj0!Z# zJqbg6O1Hc+?$-&-2xoKa1CJPJBTpdUCM-J?O$1)Zee{(?7zADq=C_;dzI(Z!oqInR z!*?gU6HYcY2xzcl7dk#@(snuw2r30$$EqdBm812%??>Z}m1KVc77zfhka_^HZ496A zrcX%p1KLj^SHe4haDrC;eEnc!v$L-3XU=8j&((&$T|#O+1Xxmrv#+nw9D zF-g6^V`YtsDJM%WWK_tm)B^d`R(Toe z|8QR;5B}x8x^4pdl7FJ}DzzS@UZ&pJYT=fPk!oyQqGs1z*ljaQOOE@V!iRC0vTyWz z#apL$-iixPo%}kSAx0B7^J=#pfXlbm2$un|^npi@j{No=HR(rZX0>XX?LI?^pc2$Z z{6843D6p6zj9%K05iDpY^qmhGSKNxV7$Q`*L8Tb z(2Jp3g?X$R`@Gu9Ls)8Wv?saXk(JlhxAtKWHY#}`(ex@uishz=LcgQX?QQVL1uGt=1BuKu~!eKJ3&F(a zK33XvU-%H*%8&P{)GACUcLW&S_&IP%VJ79TRu^Ejz=U>=8%jCr-= z#{1V00vT4YCBruezZ#Iz=^nqniXx0Uoh2Irbe4eif#8u8*+x|?V4ZNpkOYh0nN&;T z;#o4(BvBb9<_XG8qfh#{kpy80R7U11rjTdLoDx))|GFPphCl1AauihMJ(eQN74#ss z6SmqFi`i1uCw8k$25SN+rJxPt$l=D$63?hzL)C)OXxr7GhiIkZ8tRiqhr*-BuYQ=E zgeNn#mR^2Ydo%KdW@o;dJw9hF_>K8jWQ9*|ui`649@4{I#$O4gF9s=A#j0RS!xXHo zta?gA|JQ$wH%`S0JEg2^dl~3R|lpSk^3E9Cg_MEQ7keEUm81 zu&^Di8e5jyS3GTnFKgZRne*O-2H2N2I!8Qs=p0^2!u|t4dkWNRW0l3(y8X0|$#j#d z8sqG2sWsf5wUvTY=D^b~3pBOtmSn)OSc<=1?u&$E2*KTJ?CU!1z*BJWV@6&=?%@Y4kml7NL#AYC1 zfc_**kSxtleDHr_5xzfIe zsHo_0<21^<%@eMd0DPi6`!6-YtB9U-z3)yocqNs%>n_Q;{ij~98G~Nwr(E?go62Nf zd02;`f^;N#Z0iz{ivBCJy>gF68<@Dz-p`HxUTFeHzuv44=c(l@)XVK-xp7Bx zXm_q}3}v-??ETK_84XYbvLAcq$0Y0u3`~&0%7B_^rUfF0EIFT}a##W|yyHY3+l5I6 zVj8X0PXK%5m1TF-uuZgMpYy(-bIb^DGJpRo52Y3|P0`4a&pq#l&prp^D+PPyR1Acl z^-!&yQU0jW(XT(H-v=w;w>hU#7o?6&mQw?&yv>*&3m`!OKj3_+SPrXQ>)ZVD#-W z9z%ZPLcUV~JGPx&R^R26<4p@ItH!f!TqR;$06mfioc@tL%3T7l#@+mP#cA-Ie2D*N z+zE8miq{IDeq8>LYgcnARD{?f_pTZDf||`JrpMf$!5IB|{d~)vv8SU|7F70iNvc`# z)dy|x{CxVGd1jWzZ#)tA0n$fcT`#8mp&FGrZcV+F;=Zr~Gl@G$n)f#8ax?IGIiBZK zZ9oT%eZVUJ*0}7{X7CsR&gl2Zzjp|Y(-pBx9K@aOUwSo}$L;{E%wfRM3=sd>me;DT z%2PDdHw?aScjJ~zXiuRzuj|ogE-tdrh8@E}T9J2i8u=Ng2~h1x;Wp1u)WdGCy#VqP@GRrl|$3IrbET=7ucz&y1T7Is@&T@Um~=x5jE>cy1& zA@~r?oBZ(SbNN$@eW}d1BosG!Ha{YfSxNn_QIcgrLqnqw*fSu=hJm}MQ=5yqc$NkeS|je1Wt;{- zQWI=;udH6*>$dGnG*wd^Bx4*O#|;q!KK41*Nzn|e0+U0#p$ZJYW|_jH{)J$6f~ zb3Ll9J1&sdm85+df6(-~`}#sjTP-u7KL;;#%_%pA1XzT5E-$;g{cjbzLG|^m$#Oof zP&U8KKRf8Jhpm+AnUP?9FX4SsDP2U{OluG%FD!Y!G|ey*n_o!3I@*NE+4Vz+A8Eg{ z^r&mq}$@(2C-8WXw8ctKop6a1Qf57v19*8T__gQy>|H)yj+k}^HzMjx4z6TGwa%E%R+#peCQu;f7zS_x3R;tDB6`h(3xHlu}duX!I8(sKfN>I|*~*1>zV|#8+~>_Sns2 zTLV1Za)9zzgoO3@h=>Y1AF*jYiH~1)Aq7{53*D87?Ur~!Z~}=*o0QZ^p`5nxsRi6f zSLL}sV0=FI+au`ea*5c!)V$JrdK}cL1k8h5!S6K4Zs6R){1_Azlza2Y4kN}g2|I5> z+?@GXg%0ZH2vLXm?1t2$|JNW95X5jojXz45dXQo{Yod5Liko{FsL3K?erS`Z>GP7@ zhKl^U9ITOtsAuPBhl(_n%&}vLB=Oz?SMh$_-}{W6Kj#R+c^9!%uq)NGBIH$~d-NsR z&R;B+)dIDaTl#0V@OWkFxvsnJaO=x=Laxi=$0(DD$HJ}WM!()&->HONs7|YI-CQ)7 z+P27BzV&9Vuc$0cuzG!{)3{xg#V9OZI+7yR`@HAPfP9^0c6i-q-nfkuP5pQvop;2s zZdcoQZaiE?mVY`qiigjGt@Fq-mx9kOw&JEy0tB6cCZ5QvBr1R`0s|s_%W88|R%6p_sQ_(c@lL11% znu05q%h(wkqRysvW&S|O3QO>tGdq1p&OT?S6}?1N-gR)hytn}}^Z2=KhGuOpCNv4h zDH_=}rmL0uU!L#%c)Zn9s=#DEbfLu<%lpZoHb#2mpnU)J| z8w9s*pgE#<*RQ`+5kX+C-n?wd(ot2-A$w}w9XS$e+$+H~^5ytvyrT+mGqu}>`(wtl zN%m@&3o$J%1p{ATx*e(Y+Af;Cjbqa6>4{hB0nNFGdr%S(h(g#fbYw{)Tt=E=H%{eG z`vn09s3EKG%?}Hxe&_`~IRB1BW`ufu<9e(EZFIaCJdU;Kr(|Xda_H}GbMM&?LlI4| ztT;Sfk#^ms88Pk+UFhht@v%?&(y9T2SOnqsO3UbrmbWwricatNs$RZ1D|T4v53{WJ z12R>J3qr*QO0;PlO6xTq3U6gn-_{=c)uGEJ7kyu}K%&OBK%3gva`A!G*?XQDDoM$- z2j6f8lp1$#OTLy6)A}>x3l_e6`QwMoDiD9fJ7(^_ts{Bg1gu{ zu90r${o_OI`-8#UH6kS;qLhK(Jmb{G5ciFisj4yvrRQg8iN(Rb(!eQ5)BBMOpAgog zNauA~WgGn9$PgY{5x}R{_QyWXbTp$-a;}U#3BF;41)xthSRUhOTlNvwY5~FDnl%w|=Q! zwLz}iUSA@C1`>xXFN^&%tz=KCv9{}vB@9NCD55~(br;Ba(91y9Is`w}JBhLodJ7U1 zBEI$|Q6>WUq|@n!McY5aGq4|@%V9u28AIaj`V3_Uf`}jrrhg0^({)a&sK$OuS_+5d zzSw<~K@6D79MJrtY!#d>h8}`&09F|wYx|pcs^5}73t#&Q2+y6JlGvrTZ7CWlc=2IA zhBlaf7)D0bmHRF>+4egneOy69bCGwaScj38#8|pzEzmL!VE1?IW2{B3RFm?P0po?= z?m#TsQRnTkGA;Lgm`T(SKx2i3RDo5*M~r+8UvUyDQ;MVCG?K4J+uA(-oX1Wb9S?M1 zMRNPoJcR)=IGx`SGqlWHD&qd!qV)KvBrD#+!q0 z;=%Ldbc`C_#}ETG=6Eeic~GQkE*BsMAxzq0uO?3Fo%U|5fSKn+*7KtE$;w$fat34! znodfau5Q5nt0w{5cPUmwMzD)84^V`1t#Ku*9QAMOtT4EdpE5Iuscu+H+rO!c`u3*H z*qPe3c8Z|BkLuTAc(43MaeIUa8?v<*a$FF3ElMRiwQJ**r4O@*(&kKK{4o4aHT06J zo8Xdq+WDsoA;tAo0^=|Hs0lho)r17+cpO6d?l7WAh`6}J5j&#+S`I~vM@i3(@-1f? zm=4Hx{Nh3fK8A=`NWQ)}8z?Ge*Rn0!`!g-Q+X&SRy&E)8eB2R1{t0_1 zIfUeabM?&R8O8&ky5&NO;mQ`$T8(NdYKKW@_tO?*D4Q9r?z~q!oH_1|gsnJS(&u~{ zHWB*CfeE0ojkZ$x0uzwro&V3gkcRQpGx$%iH}GiJvTmhww42a)@ZAMYn*P@5EQv31 z5?A<?Qug7JD2)ymd>^v5QigE8JQS%ec8k@Keo+;Iqp@Q#E_Bk0z zVQZfk>+rXbQ@fqQ-KiGBaJ4l@H^rP*xX(c|F7cp2G@L^JB*YQO>ZLvzm+PKH^hXDj zm$ao`kFh*1MJD}p3K9yrSopmu^U~Wv2o-x@xFwy! zQN}|Vl*)1m<0k7p2?b@m;YZJp&+5Mmv3Z0A;%|N=mO9JsCM4VeQ+1j`Q|GV7i7WrZ z;54qz4A2a9KQ+e$CI=%hHzZww7ouVDQW%}U66J1DaAn7fXPpj`8*TPoaoYaiE^MFY zRT1na6$qQvMM$d=b-v!#y?Cn#iD%6+#R+~%c4*xV5;~64x)Qn{uvKQ~#e-}BEm_%> z-0weX?EF`h7p>3dfE^Nw_bzM+mZ-;uD^#Ne*YuNcTZ4-e)ea|D6fUI;*29p9X2$(q zFm2r36W<&E_^q-ma2>gM%Ou+VUTCL%z*f>da6!7{mA{#{Yy7*f3|>lJR1)Vjo5I)` z`69Nbq#NDmFTX+&J@PK+g3Op1lRos2hJxH;$KbldWNb_7uATeP0}PG zmP|@BNL8~dvfrxL-F!uWq78{tE~{XHdatC+Do38ESG93gO0z6g@QM?Pg_ivn7`1D1 z`osUSzw4DsF_#rr{=t5YafIiVZG)eK)*(rHSqr~G(%<9`EH}zg?0`yk*OEj~r;Bu< zWoW3sN=6G^dp2M5GKFmbY0DGU1DV4J8~XKc&*nSTy7a2!J_NS| zV}z)unCYzXleRROgvZi=^1o+vx}sLK_nvsSnjSM#Lt9_J6gy5v8S#{vIYz(nM8?q2 zP?y-wbNrK1(v;672FdgJ)Cy+kq}4C9@Ld{n(1;vdcmjCcgUs1q}3PupJ#xn0iUx9$1yXT#Q|!7TAGM{VGdV|)sd3OwezWfE`RyeS9!(R8`k zBjn#}h5Ui!u(U2$5k(W_06r{g?PtCG*iz?N4Vp|#nQ5fRjtX^9Ek`7};XT@U$Bqm7 zq+@(u&GA#UlW10BND+sPd1NUPXugQ^Zo(S)HG3OGx z|5(FIx-r)-D&}V|j4j28o8@}=XPPCsBBX(eH%lN_n!-yy-P1ge`Xz5P>nn-}+-0>V z%=_VuRwz8FGkE-3xp9wIvIJ~nwqLLQ%_8&Z?1r4{?^ul=TujjH&G8GMrK|%WTe`itbVD)Lfs;ei3d*g;){r8q6&84tW?nNaol>DCl^r2d$#LBJ#7_MtnY9Gw9u z4oyDvYbcdXuDc13F_5hS*0IEM!@t=sU=HP< zke8Pi1i*9Vz~0-PaP*KE`x(`MDR=Lc85&7QFWR_ zmE{I1^fpF&JDvUc{w@rxL`67n-1+j>SC`lGsOzIS$TTPgqXX1>rMk7cO!{jp+{YXR z-n>jucO04s!B3^7isy;%Ps5!f64HF~?w+ofsXtO8vx(WS)_Za{SEh5g692^7d^589n zN40V+)2k;=Ax`HWb!$%bKZCT{Czqwj?v;C4TdBHe_0}i_IlWi}4jWeD-05|TXtVP# zrC9JEfI5j`bwCbCg4uxn_N!SR>hsV&hAIjlYT!kKnp}DM{Ej5JAv_Z}XyuGb9W&D& zr$rA2%cFKpiJ@e;3xs`acdyVmxuz>;-|evmFcvkDwfH~-s7O>^H7XkALjb>$G?lA&h?Cab1=O<3g4X(lW135enoDd#|SZL2*O;t2WhzQ%0LE!s9 z`vP~?%avGmK$2JUCdiJ!=Rlz>mga&fRfM7OK%1%8881DzAk*el4+rjdc3OxY!vQHH zbL({*g<8abR+=5?vy*tk5Tr_sK##Zrvn0ot7O~@1mmv%Sk;f- z|HnMZpoSjg1``rUQNC7eJDS1cTASLxqvHDcg;OT%@h|xo)}2!Ay=mVNFW1PmgA6sm zF!f$M`@e#{1~Z6D_NmCxpS1D9Q#$B;%#U)D0TCkBlD3Bd4wcLibFfDG#$TL^Y5xpE zN!owgG|3Ru1uMOoR6q4AOC*A!seYEH!cV|EV8I`}od}j3KLazEFAZg?U>d=g&E# z6BFGC7zEb$@%>We#09--zP!4R^zf6`lH={eG=7&hnI!KHo?XEA;*nuYnKbZ|_BYM3 zt$L5Y+!txDnt-*IEf;2}YHuo3mJ5_C{QjRaS?V)$i2Etk@cZnz20PkQuUnSfc@2Fi znFC6uLM*+{+^u0*LJ21N3?LD!gRa=F{W#h*_SuRJGdONB#=W`+!T)4v?j@Q6&E9A3 zAI%;-HX}Z8C4$L30^zyVSonMG_-|YiRArzU=6qN>R%&S$Xpf${0w&Y2Q9ne1!! z7{$j_n%^PD=Q@#;FqLMn%ip6OA*q+&rE?YZ)7So>T)gyqX5EsEA;H&|oiYxR*+~x_ zngpmwkY?xme1952xJ;%47XYQsOP6Zns{ipcBVf)^w+&f%s8g=wmV(HoqzTy=)-*dD z3tc;J+DY2J$pA=W=XPlXULMB zz0AnKY1O81xVn>#O;iEZyxx(4Wg{YYPMM^*RaP9#_&! zJA1KX9X-pPMJBmpd*HQXHm*^mh1Mkpkc7Nrzyv)wgHfkGb;^sz(&Zc`uHsLAF~-Bj zkhFHCj}zJ1)S!xdR;jTx+u4aJ%7dA#@sca@kDN*~pIfZX>~1iWKQB z^j_~1oSEFkI+$ZsU#PS^%kF}0m;G@ zzu$77LhsYrMVW+#IxOoP)HTBUxbG3d(SL@`CdG$#=JZ**my35c6`UmAgaREsf*QR1 z!Pf}8EiccVDg4#!E}uDirlAmE2{5s-S9IVNS7u#HQYBM7p_(J($Fj9py=v>ss{dw7 z)5u5XlR4QBHktASnc$_W8|@Fa+b~pm{DceSEVdl_mNB&veA~B(0}ZV^zeHYYCqjL_7K?d$Isxt7X}e{AOCl|cqhBV275 zyR3uLS#NmUXpuOdkwUuBF7K=$qyThFl^?ay9D*4v`&&qF0Ck8+10I}wHsb%UHqu1E zyuPAMj7b(W2BpJf^V4#mDzuXHTbjW$XK#SsdBRUQb8|C*@vSSRj68VDqNc8$dZnLL zb4Hs>Uu`JrglcR&>01A%)wsW4@Y5#cTTnLt-^>MjH2*Uf1U)F=s3KrL$(pHgff+o- zCyV2zxv^kcYg6jx=EV2#$sgfD_xsAWeF;up4v%BVdLyFYfYF;ih^e{Lf5KPdag)U1LNVou8lYq*?kCM0$(U8v>iR zdscfYFE$^@RwHn}sz~&{+vP}c_tEl?T^~#yuFQG4e~TmWa{u)~%1R$wVbrZVdaI=t z>)v@*{y#w>jDra8+*!<^9t@)wVb+Up!WndS0n`TMFyH71W-f7Yn(v&~>~E!nZSh3$ zKJhRRke6`V+Yx50O$mC_*#gRE}9K1Cv+BfhOc!aeI)kVz3Y+z>dXH?<(FIM2WQ2qF3(9L)sGTzOV8J#19? zyojI7u%!m08L9H8m_j^4{&DPoC@;E;pUa)%zH|n9C0_m%7I9m9{ejj!g;`&Z=8oCV zk9U9%QWS1vJY4oyuJyZAF#CH4_&PBHxt2Yc54%;)W%^}KuUYM#lr8g>3-}?pTKFy0 z`Wijx0gEoJv7G}RbN20!n->GP%crx%V`f0td*gR|kBZbRtn@`L5GTAVO?3aV_nl2~ zlc@c!n|%)UndI@B==emjE6h0ZTr`e>h<%*VKWi*M!6zRrGvK`{AmVesQqN207!+A} z-AJenU>#MSfcT*h6di{b^w0omDZ?biZ0aNu1_~hacl-g*I(l#qoBrOR9G2JolatA2 zU?=8%S`kH5CRjYa8M|3Ci4Xetyb&tZw;KAtwEVziH2~Suqn4ACV_(}Qe8+IOAvm2e zfQsRPWof={iC$ry2bdX3fInr?5LSpymk$r^hieU}9$#kdprE^m*jZj8W`*%=Vd5l_ zQhiP~!h%-_m_J=<1rO;`n_6NfXfw5AFtJsc5B8ZbkK>7_wj)3XzR4+)S6`H(ChAMg zbT7Y*)SeO&)S&#^C~?8z(x97h#mz9QQu&zOc+*39F#KLQKM>nfmr;G|@rd>q??`Hu z*h1B;^Y{x1!4GHjO%ylK&Ch?pJTRk;<-OW+ADd_1;fQ}~&Z8Z`G1rT1wAv-|?STiE zUJpOWAfQHKBz#&C5R8!z;?Bs)f%Y%YMmuj5;Tf;OnZ3{a?XIwA?JajF$C~asfDytn zhOM~AOTSgX$WP8#`uzBe$EuUWdKJH0$Nv(zJ{I!WS-t~W4k`7s?%Ps`k=xD+^6MGdTL29bXLuek1>;;5w2d37GkJ;mHFV0$p%P3h z>GIe`+&i3Ba)8m9L^7hmt|6$yKp2<_<7MzfJWN;jG~f^is1?OHzDxnHJ%xWJ4`XxK%PQ!t;1j-vrRHhNqtzT~ zHD66y0tfi}9{C(06j>2d&DiPE-p&XB!>z5&V1ED3qfaFJrxK%Px=ljWG}ZIy9XG#5 zTF1vb-sq~1{~izgX#e1+@@wWqRwwX)rob7mIlFZTj88#B>Kdm3P&DmT7AXkKg=?X` zR%?C1G*w)-!F71C-deb)3@S>^}NzOuVE$k!|#LS+Wx_?c}Jx5p`cS-5t zmJm-;?Z9m|&(}xq%Cxvyvl{B2jhPuF_uFioH9w`}#(J93H5w~}%!#YDx?!3OAC#{a z9bVgv@6s;Z*~6MN@(Xsd4zmafd(w4*%CuWAIh!?zBh0&KNls_K!YHuZXy)P&O1`{M z4GzDquIae>)16Z26?enx9IO7plMnmpVdqHSD}d4e#9w`Pq3?X`W%X5og;74kGaMFl z*t8A(Clsq5)7#c=yJ*V8i0MULmRS!W^lDwz`&A0)pFiumHZ@d6+0L((*!Gx~XrZQ+oc z8zjp5T`eiojF|IcMl;b4OK(MMpjY9r?FIBCX=OeR$N_@3yg3N{k-|GB9^H-CxdU8pp3^A5Qb#ZUN_+Y! zqJJL25Y5XE+Q1$${5(|NJqyi` z#7rACn|x2*LqcGE!f)Tq0$aiU@%;9fDz9n5{-pu!z6XHLO0UX{eSEQg5qKd#V=9dV z!<3n%iC=t@9onyCD&}m`TyuG!<73&R=;bp1iz)pUXE3ti?PcvQ2Cx(DeTZ0|M|94s zs_ovbN_PFluA>M(KY4dV8SNvN52l!)L};F9;LP<2cdYoO|GUlwcI<(votYMEa5 zUNP%9$ku843LWx@k)|R` zRZbR_JtqIb9vsRjzKE)g;&s*qmf_cSYivh=<{?B=`RcyZ5%IoSIf>1Aky5Zx?7hPK zRLT?GxHJ}%KXez;;oTh1<-c(cZCp*=%P@EN6HGam-D=IK$n2+6>4JmP>{B^jn0^xQ zwRx3u{D8LgJ)@fK@4YJKXTZFf%Yh|07_0}o_jw9*vFvq1dwX^pFO3xN@n8Wl=3PzhB~xC#7IelWX{ zMAjG}{j)ie5$11&U2Vl@MbXidPq)91piY9c&%e>M;&1GrFQ7|R+D+YQy+7Hu1njK-*II}F+O&t0 zx2_BvWe=67Karw#UgSKF8@^RMZ-3+&!$ZH;7z26K31xGzM|3G(8?sihoAC7n=qPu6zy9vir{&V|(rX9|%%@CxV;HS^(b2@atY&Zf8CUojI*E8W zhbue>!8yb>e{qEKD0Kg1j{M3_OpINds0gRn1#E8}iw_}aJNux`nB%^^3QCGt9 zpYC4!EYtX*px4(Cm{@LY&hrWdTJMY|D>pVAz~~BB%u}<{x4i@A^RNCqH7KLH_F#Q2 zCrhg_$9NGiYidDz1HR*c7*zYt!@X2=k|FCmWp;(PKrwOW?zG!ZdN@VnFhQ&e-7SOF zJ9=Mxhxx13zQjHSK#u}aI6|7*e%sDEYO}NY=`l3uA|7l%`!0C39a&`qW^+O3$zd68 zig{==%u4r5I z=E;_E72x0DH3RCU(H~7j{8?!aFMM%RSC3xg<)i(##C@*Z*43qL1v4~ER=c^0N}0yT zC-um)2#*VOGzhlslI7aGz5xdqPjG<*VN9n z9rXNIbu;q;A^x-fYb~|0df$5caXdY0b6PP`jmcE+l7?PZulB93@i@F6dzQdoDvT~i zcEUV<#hn;%0#8-Pn5A3E=|iX$hqFR3*PVQta0q1*M4>jww0C#Xs1^CGn5`Yzn+4B| zxrbAX0Sc@<{w7fqoDU86P>;$9=xckh>07?AaI#IbhEh|m$)NlGpp#>Gai%ctw4PzJ z>dEx$VC}&Dg=-IKx7{ua7LFKmL=MODMsI^wVjt75QES>SsLb8FlkyqpIzQt^eE+5A zZrVRj_y9J~5A_-Ta|o$BRlte@me#_0=FIkqKXs(su81N{#n|S!Cd1`$=VO^sc}P;K z$1|kxNYDE*=6f)KD{`)#bR;J-QW884PB9Iqq-r(|DGN^Ytmj)cOBl0Q5Tvwi^ zRi4RVzAa|#Zm2QR33q@+Ja<&TrLAyCm(d-n;)PHqK0E37n&I$&GAa7MO#0Xx%zPgH zUXJ8d$mT<0db&F2(OM%C_WC+%=6IJsLe*{Pd+IkrVK9VP-2ip1%mqfzq%=1C7c~b5vV2VHoWmI7=1;d>; zGym0u*GEzTfidxmq6S7%f2|8jn)rb#k$qg zJfu|?D8*y8?Xaxb8pfhy&c==L3QWm4Nd<0qTx-Y6tg^Nt8hX)*RSkcha9+9 z1E-7@7*UOBao*jZ|5`}dzHXpr@-O|7sh_w9ybO3rANyImFBOz?R~!ouY~Q@%N!-Va z`zu!guU)GZob$TZ(z83uMlWemmuC*AV;PiUr&)%%m;eNq<9u5kR3hiVQ{{Rwcc6rQsgT!V-lZwYGjMV=g! z0QW%(cv#seIn&C+1ijTckVK_Oi5tNBk2}Tx_>!G}@}ZPVYEqD{{q`m_=t@i6Q}8lN z7ncSd4$LI}c)hf5R(4C_!4~Hg&UztRsp&fL=l!~Hik#fhF=c@dZZA8H#aCZmFe~z* z!I#FzVATl_U8g>+Das^<6PFI*^DrFqpFAsx3kwJ$o)NW zXbE=1;qQ<9$P_X4Cj-yLv9qvV#=Cy-b3sH+V}0&SXw822XK0GKy}XD7n=Ieg8VsUw z-Zw!i@K~2BBNo!wcc-Q-uA>%qnvdwVFFY-TKLl+SMO|Ir{^ZeW*ERqNs6ENj;5^HN zUmMn&8e{Hsq-FQcJjWsqnGCre_-|}{{DIU;9 zC4bjk7ddgm=xrSA)%#iCLcl%(20Nu}86!Sq+;V~F)6QfRKMkX^ipfpT>px%aY6bdS zv$V~dwxFb3)W-K3$0Xgr#veCBA^gS^K=~CG;75^~@#`c)LPA1Idb==Yt(7-q3=4-_fzu%eWm_>l? zF%somTIk1s^n3_KlaFM!AE--zE0iOf6Cn+HYImLq^u}gBL!%jWEY;W+OM`ku-PS%` z6xs#N)=Sj1`C)FH^g6P(B7xr(Gx^=L@@@{|q4DDi@qF#FvJMrOWCuj*dWkP}vIcvm ztUOsCU(nBrCmWY8($MP19ff=C^)xI3_fAmJ%a<{D5<_3$QK|LE`XmOYu$K+Om#rSn z{1;GV&;hBX^5tsI5HJ#~v@J?~u&ITBdYZVb{7(B3jxVeu=tyuTM&(d!#Vd z4(7!!9!W9K6$a;As;;eO9ug3o1C3Wf{}SvWIe_VFCx2>E@xE9117w^lc|IF>FDf{o zMz%Jh_^*8Z@}M9R}6tqS)8mci|4dnR3uX?5 zcxiSHRAwQ|%>}9@OTJY>Ugs0?q>b|;`|tTVNyGm!N^Q9v4&K7MuWDD~c*}x%z1H>k zH{&TY9?jEcI>vN%JBiBs*Vy50shYO&NCKo*?*-sekQ^eGlTFXK@gJc~+1k~0-K(ws zbFSX4Fxt=pAd(-WC-KrbA^KOxY+mx40{2xk2_7p2P_&GvEoQaFXr2Gr(^H6V$!;`W z{mYZt3~RoL%hE8P$c-%^9fZbKG3*@zP100VU}cTZ0O0o+tL;s4JeUVZ6kNxJ4OPOeg;yE@p#{)93Gulb_PNP7h`QrVO9$ER!pQ% zVl7aaad#&f^<|)O#*N{3s45@Ch91Twi7x3csw&GbnvU+WD|*kQ1$ccCp>)o&*TI1` zTjFszP;%Uiv+As1gj4;MKd&Leq=g#n9()*Vw4ZfG{ zU{ecJy$VSgEPcVitgK%sGr`goW@Ka}r~kIW5Eex-qX>pnKedEi}~Rxl6!3|@8 zq8jrnEvpUK^Tht3kKuCOd(}sxHhXHes{9R?o9i`hpOC~C

m)N=8PC;0dF{A8E4p zSEvIAKgSRfPL{YosrArPh1o5H=D-Yjc?u2}J&FP~uho`I-CB4?Xkc+Mx1wj{^%W{o zrpwAz(E-tp4r4+C_H?d-SR+S7N9~+})#=V|zxGGL-6Y4*Q}el_)vrkY(@L*3e+Fc{ z;I3uGdGE`Wodjfpm+zd=ClJT%>Fg`6Zg!nomp1vH?glNM)YkiMG+H(Gis~;eeV?j~ zDb85}gq(5ZYTYT#(DS@j}>jp*;+rQLC5*v~p8)u|CvN-2BY-9x-B$NgQV&u8l* z%~Rk$@5CK(<*wRFUaF5bCj8Eg!p@2R#@z4vFcB&5u~L7Deg%kc zG_ZH_TqwM~7v5f8O4pA0;=M)lh2ePqro*VSwvZQ{1#Xy&2xsZquNj8{3}Y|y&uR*r zlo`hU-1|LD(kRY4r>h$5J-|;a=R>|*LgQjhF*0A{-y;rlxk^ za7RKmNvkoi?GMPw1P(@G?x;_LRS*$-zAYke>tqZZ-)K3Zf6Acx<_2KO-l2U}&P$8a zUnHXxJ!4~hpe+lu77Xxu>#p09FOP%NXMQ=JuzS=Pri;yt)r&uKxm=sFQ&D3pE-mA} zo}D}pXB3MtT)WGi%kTA_NL+)kZi|1*`730Z=zmOLWTrZ^7D&!IyT-xDuM>6GhgWYLQGAFAP&9GDr5FFSutG1o@H@i7tQO~DnToN-d@8@o|QIEU!Mg&-*fJhT*Zp7Pc_%yNF%b z$)8LTyNE2`qfm`|K}K7L!*I%}N-}HshGQ+CTY$6hdHRVos#i91hUn7@vzydYS0na;IGHO(0$ItsY!=OBm-^(O18T9>y>!z4KW`BZeem}*ijCYT7 z1i2TAqdvIrj=Hv&be-6~$&w|xmA)RR?PrX2(kRmK(`$RI~?d^>+Zn$7_eUvATAF*PcZ%TH?8L;GAH*9mq-1V8&N;y z>z4S(9~6+&Px#d<*1gAHU(^tZDJ)t=7R}Q*k%|>!sdiO1R5Z8LP7j%-T>^;Ta}nUo zb1MDoL=D=0Lm66;{ZT_EWBSwCMFxWF;q^yjx~PTy7OEL-WCH@*^{mm&mT;)dz^j9V zp0+s_cb8(rs#fK2gTD~l1piaCradM46u8J~n`M!cZ^3-+RJW4S)6<`|lzk-C=o=34 zW+7mGO7vHp#xPV7JvQFFR3CMEoGeI%m_PWH z!LvzoFC|gxLU=L7*Kb7P3HI5+^%r49_&+tCd-8z+!K%Nrz0=zF=mJO9y(yHR$rF*lPJFItfiylq8MxDN;Z2SgyMz90)Q5}E`* z31U+lWFFDWLp{pQjXauE*=+iNa50;SH%kcV{8%LIJ9 zE1(&!QRei^kvoG2x_zWpu2TD+W31p7iEY1L~B8bZ7VZ`iX4g#ghXd8ypb>4q2;EvpX7tkx&f3PqF{5N7d;L;$W z+y@#B4XBNT72A;7>ET+#IDKuHJ`@UYQSD9w8y0>V-KhO$#3`n4UsZg4I=lLJhtT-b zC#4^ZgW)14N1NQAzd}=4H>OD>_bBmhHNRgql;yO3iR5)AR43_de;2!!!{FT{ntGS* z+>irp95Z_eUmG%ZI45~wi&@tB9^+eXFt(W$9Gt}8J<2jiuTDAcltwB zo0jeAGjd!uB8+t_O*}G?@wWy z_t(|j_rEVK#@yYKy#YVuejzU8e&a8Ad38e@-5?y7;RC+YH;E|jN6QVV-Z6o|LH4R= z$47SX;(tm1y|5svbkdf^{j@KMI(tM0of^m-ER+EVw z)8UGt6a^}5W);X#(ely#WdPJZ(>1=}C=+vBOP+;FwS~7Z_{unO?oRmcYS-?FS@g3C zn5K6k*O%(1u;EL;*XtkcPfux@HjRSBY~Z<48D{{hh>Z=5)#rki#!gnXtgtpkbUs_! zz)G#V!aYZNGR`3fH@CrqvZ60#)_ioE%U@~6%h>qKUht!>mTi~6CXdevkEpzt*d0`{ z@|V{!FK3}=YKec@7)~U)cG^m?dMcCQW@qj5WaT(~6=}%QV0`Drr)rA}^nMbSOz89U z#@~tQCn%A`KZY;w5YQV#zm41OxUZT`Z?~-E4!`C<9ISGgvAy&nyp5=*R}kuAJ)U9_ zMnoa}H0;riaOFljOwgOZYoe~sM+6^vOu`>AhfAKAPih>l`o88dI-&<-6?sp56&Ox; zmx}6wGKRIYLe1@2!pzEDaF>_^D2Y5Gh zr3ow=8B$W;+TO*iby5zYoH=`UElD7qMiEPgOkVB>lpJjs$-}HGXaQ*-Wh>h&jW0)@ zU#;87dw2zB`-s_X89A7+!?tFgVD4)5Vr`K3>jNBYA`)KegudSv(TUuD5OOn@buC9V znd2-sRXWT(zK&s~v>?b&_1+y-Rbp+>Lm?1mKji{n8JM>eorf1c_v6uX${2J)e;`W? zoh@SZXg(}|=3g;kw{^?jh{=`(dTKHcu$45WZ#<|ey-QYf7W0~E;LY*gp#=uk(XJ!= zH%RDpaj0ZbeXhtlyKsvQ`SGJlr3CQ@YDhD?BQba5{qS7s7A(K+U+_6iDHB(MJNeeI zylOKu=RxcCI!N(b87QYD&fbG&=TAJzyt^qPvcP;KAI?AzWH+ZvW<#i=^5t)~uC2%` zJo*V5uX3tkNsrRMi z=}>H}?X0dd2|r1PdU!0b4(Q|}x^;6QR3^R5>fwrNTcYGftirw=+Afylq!%hsH3czb zLQ#j~oU#o;GTFF`_Ohfle`mm}GcHk|fIk7Y#X_rDT$KQv3QMnnDR-mu0 zx@QSRu3xuoLk??hc&~~%zo@7*V0s2G^*(&q?W;~@s+0ou9u}F`xa8L? z6$16D_uiin)Ah++xzpoz?%UIgdA_vAuD^#IHfK9D>A0PbDCKJm!EUsN`}6XW?4UC1 z)&n(55v}9Tug~IOh9%uKilWJqzBOXMb2YhVhOJK7H2KJ-0AzNi3QlwYMIMW@omR*x z+WT-%R|-J(b~>(oFj>@9;v<%b>uoZ{xOrXc$*k*N?GH@^Q|gqh$Qn8E_puhKs3iYE z36gF6kw<9*eET@g|PQAyu4ZYQ8y|Tx>{PBZU!$q zN}Z{^T`$B$0`IzgP}T8+MVUZ0E~Mz#k0dCaHq1eI_-8{mbED&=d>Q!JDc2j|N1Q}M z4xj2(4?H~^w}|?6P*7IP-YPpPlZxASJ)}Ba+gmkNLRMn^UCFc`$x6t{?n2h-SEUqj z5oXo8DY_PUUv0JYlT(P-L<4@IsZxZXb68gDiOu6;4d!Xy^4}j7(S7c*N(hfoH>Crm z1npF4z3(q+5=`}&^XK1Qo2oPWbEz;#)gw9(=C45+*%fzocjdWrR|TB2t^1DE#wB*@ zc6lR5?CNz>yf(J$ge`iO_9P;90!e?-*k0mfCC z+n>bkl~2N-YzM}XaY%xg2WvK6VgqLe7K(o8laP?m3cj%tn1 zFM}@@te=>%YWCeqXAm9tq(>o~=FSbNy-a?EtO3FVROWY45&l0l0h?5vFk$c~8ES-R zXYq9w-=o*o7E#82ZOG0%Ms`qDpCdDkczt0F5H}j$ESKJyKYz~X{9uIjJGSC8I|r!O z1jNO~nNE2s)`u58%6LNnBqW*EE(vkuH5=IUQ1##Yh9<| z3S=!VxTqjL1LhH-{5RI|5$^YIhi3$|oMnl%R^Pp;o>M>Y%fvSzDB?s6W-|eW%l?C-PlpqET#CKVxr8P5(JyH;KGARi)3DBpa;omL4pYPT|uf1 z$&u=%*e9|J3N)qVJf!u_dY_{qvsC-BZULkFyH<^^{VLyPQSiWiYSsr4Ic~mIPa23tQl7>e7S8pT)KcVZ3_ac7%TKsd=APTns27Yq@^)c z&LD56pN6#|%S^VIZq>^Deu;n5x!=yQPAC($Fr4y<9IchB);u(T#vbJ0G<2b8>)E^p z?(@%JXzQ)vH9Z0f3X1lnM0D#~XL63~OX!>xo}+t!Yi$j7a&^smLDM5Fs=x6P$r_kR zpK*4`ztZ1Nd-Fd9l@mc55^CItovB@N9tkABGBvN~Q#eNiDQT)imf*Y)Km0q9C;Wcf zYT7_myv@I5k3d1Dj|4V5c&0kk>zOj~LTVYo9u+mhiFsOQM16nA3I7YLzNeG{p=|8&6jkM3&%0F1x)3`P-r5$Ny9Qey4?P$E>$2OLg9r zC8RfQJ1@R5MKFXidaL`UDi5aLkKmQrI>(KZ`5remQV2V0wTrVLiLWAb3t*dbX^|Dv zT|B+hjZ*XV-s|OQ^2(#|r8xe1%27fgc}HzHl$$=*w|YJtq&B4rO}9Qix~i0eVLWY_ z(3E=6qYhts0dDoq_ZewHeu&F%QW}pb<`FxjHDkqg=&Ok*>hsMLs%XBVnvEW%ag)1% zGgbKXfl7EO-*M(U$?A1B{g?Y&prBf7%?5#=+M5^mfeMAN+IX=TKF~UZt|Nx5eFAsv z|GZ&Oz@uuj5&M9n{dha_f?SZ6{PnVeBlC^MEF;4ALCW_6m-Sc(07OZ$(Y|w){$bD_ z+F|`cC-V24l`jqr*y=!GH;@~sNUeklP~m>Mm!>Q1#`x&!iyZ0qS5pB8aZ*$qK>5v9 zFHrtmxLzV z?%!txd1T+t^dkZOo4qE!1xjHZe5CloD~keaVA)#%wy!Bxw5^+zID?wtvIcJlkqIXh@gaFzq8Xb8)g8$E`1$Nu zzPyL%jT{ zK_*4kliXCq?S8hu!ntAu7|!73)giS^;a^y8^_?3ZL=84xa_PLl|4e?MxbD(DI6MS> zFNn=JC9mn>OMWAkv4jd$pZ;4SBePSWc2A1KGwNh>Y}JnSQb(-}l8pKZrt^{3IV2As zGU!uj={Co{Q@T)DWYmE9oK=^S&a;z-?#5!{771-al;BeeDjOUePuTczDgsL|Rquij zz`oF)1UT-MtP#Ou&U2(P*vyB9E=|MZ>a?3(*y?X}$BnO*@tr-mi!E@PnO`@2X^`XR zAZx)6NkU`6#N8ueCq&U1;ouT%Rv}pQb63oRFgc#&az$6ifhc?hJN$+I^}WBBP>yPT zaWM+|=uyvTKU-qCr99@(yR0DVI|54qCDTn-^VSO+W1o23Z|vLZI72{?nR}PSacze7 zs&DK#o*rxa#WJK8E6T3&AUEABvdXJK|C&HN|9+(rAY29(^g7X-vFE`Lx9=|+ba`8V zRhRsX^AxrwohweuvqRNP6|F?AoP0M|?=tX>%&(hsP#~?3JNG|dEusgDpga4^AMn6= z4vYd?L&Vj^rG5Iu=;r)tc;}kDD=zA*VfAn@MN>L6{_6dPK40GT(R_!b>tPykU5+Ri zL1QC_nDk}PtmACsm+y@~NR#na@eTtC=}*zEx%YLQs)=9u@Zg)i4cXApARsE5pjR*$ zRopI6$YUz7b~n#OcY1P3r2PehM|MwdZ}j460wuX&6!o1iod2BRtdCqCM}v0jSu@`t zK6ePM=YFy8W9{nXUSG(TL#ZncA3N}I}N!Cq3c?l{HvyiS+ z4|e4!=;&zAP`|~BgUCa6(1`TtWRqxFSn1iBcl5RJyiLM-eDl2kh4><_iK7nRhF%|-A&r+Dj}__y?uYWXn39mj=a z0YR$f_SEC2TE91T<-YH2Ow|oE;kzPqClx?V!mOS)}oku7y!Ix-ihx!W5>uyGc#@)|u3b9|U7j{_gc9!$P{{9yK{*hnH+qOq%(*4AZ0tD`cb6|)0V@u}LzCS%?k?}6) z(wUuW>pw@gy?EfrdmX;%7c{j0M&6~8(?S8xQ0UWy{quUs8H$e-5p9czc|BF{4m?Xs z+kD>bhL8Yr3xBojg_pd+56cfutfu4hPIGP3>yF+QIj`eehMKQWEay7aoPN0CuxIdx z=7g&eqtc38>1s)|U&*eDb{QD|;M_Z4-Fz%VnL;GKcwDLa`&+uWn-tnfU;b~`@TqTr zj;|DZ^u+4WJBG$^R1*UL2W?%|9lF+gsCvB>tgP0F(#d^TV_SK+f@u) z2eOCJM_U@FX{TAEJh7%Zr}doBN?ayq8*|%>p{OUVMcbk_OAr}`An-~yomi%_jB^Ja zr%n|?7)Bm&fl9W1*$tP;MG#onRk(})&W(WM(-JF}wj_Ba~o^BK+vCR(wEqNyWgrhf0j;!#_XArfs^Hv9v@bo)~ z1?{s%%qT?qWIEmHfXH1T<_BRE1%=FnW$8RJ?l1Kn;GBoNhY}zK2LLev$iF((-|>I- zPeSHmd-W4r7@e~T2|cmE@vla-oUV8Mpn4kO>hE2uGjRv=zd-_k7lT2P;@46tl?Hg~ zOD&1&)&S+1DJfknfX6`R$lT(-O)Reqxl`{8nypu&CUY~T#ozpYYDtI|t>cNm7QtKh z^saxY*1ZZoU*&5Ow`}4yZQ})kGZrt-m%`f+uz1;};W>v4O!7Vp?PO`*iS&4A0Br|0WDICoP0Q+&%X^7+F;Hx2sz>JpBC zX}wdKRJ@!Zcm+2OUcAF0QBiOenu!Q{=%wn)Q|!@Dq6^_7wbt1cmjfV^@STdXZeHo@ax={guV ziO5wT2DOeiWs5l15%sKco+C6n_$X>7j$Q2d<+Xi3`P%%FR(@7-plw6ncc&OR5Zbcu zoBN+*v0Vw3X$(JFCqiXH-$w9WeH2xcL1ofqw;aK{gqVk=8W!QP1GW0UC;Aq*G(qc| zw8l-@&I7BVxzoMptX&0GDUrUX1M;6wU6ShqB;cHGtan`IfmTCujd9#YUeeS`{)}^d z3IVT%4Em?sqJy#`Dp}Q;!zN$VL34P_=@bDFe&3;E>fRf9_qEs7WJ5hMyu$l()!lIwaE1q zr0h~kbh5rtv|+UibZ%A5=%I{oSA{p2gtA#0z4USw&X3C6f_Pr>XA9>S?K(*zbK(8c#g@C(9wtrIc|O)e6kQ}LX8e3miq^+ z)tNZuUHBIuc&IJfddKe;LvnVpF-h0eQ-fl`W^uPOIJ#RF+J(}$r|-_KfPjPp8tJ^% z8DAf;M_v8rMDIXV7=y4THn;g@8RW(B|JxACE^MJbzxs}(fv9NJ=gW5E#fs?qIyAx* z5*uAhrx&f*yO(_eZ=Tv8qUC9e}vjn8J9oU{VjB`F<3}BR55vPA>wOu zv<1gc+M?T7Y_OFO3}@L!0+9SN)eTzO4@TTZhxUdo(eTKB4x+1QBPpb6XR6>M1ckV# z@H#inB0w(Hw7j(TNm=`#&f=N&tIt#Me{MVKr~8d`oK)Kc_FFS&@;sZn6|t#nj};rmrX;q-6N2!pSKV z<8qkC-*&sG2pa21kD4*^#Ibb`j^vSW>a3hK9PU~;-CUptJH_p$}(phnb|1g)yYBt)ryURVYI-=T2 zcEV3Qt*td4O6=*^$i&^G!dU(AeHe_f-xapT4n+$;Q#fCuOiuT{xt-FAGD z{gS6?co^8-lHlQ}I{ zuhy9yT>36oPt^wxZ>neH-VmmVanXZ%j&=G~b_;`9F|*)|@b<@&7WKtEcvdrrmtNG? z$?1O71GgjT+us|&>e`t}D~V+mo`3X#N7D0u>5A^|V)~LCNg0eQSjsB7t~5M~a#XY1 zsvjS$P(j=1#Og6*_AA)aX^N?ceFdg%EUMlZ*GQ=mm=w+&tUI~tP~5>1JcrK!`;+bk zS|bPkzgqqjz5i3Rb2=?FGgHl%H=JLzIa*+)EqAC;jSUvkpSJ(Xuj*+uVz0X*+0Iy> zA=)&pYLkN1>O@a{V;MRHZ@RSsNX+}z+hZ;UEQdG1PDe*?YCVb4F6dnArsT=X-W30% z+#CC`7IB~tDX=DIjiUv*5ubBe(S-W-f2?Rvasjp)t8q8sgY#P->Lq__S-)jpa?Qi* znAxVhBVrH50&YX>YyRU)XZYZo1XVBTcYM8{eq-m^KG!PLzZbEwOypa?es8E|gKX%5 zK++zgyAexa(3N#2dsN2WB`VH3VhwD+mbGMZa&nI3#)r8c2F}iS!7NKl?7rYx0mLsS zE`;kTlO**%0~1YFjUnqER{d8J<)|!7CNG&c_m3DLvCsYG|4O$&NCtX8Y;c?iKl}K0 z+-iarC>tj5FhYkgDK20JlHex~SY4u4kPB;{3 hondhZhlQ|ZdDoHY|5HhX?^H{<% zMTTWz8JBUf@V}mCQN4Tm?cZ^H|L;5QWAFCfNBi}7p8LLr^E%J-W>*~8Pygbh@+9YN z@g#Ash3I>~RF`jkT6yHW@<_gCy7CL;BFqysmh4L9oN60t)7-6?^v2L}9 z)_Co)gvwed@@gU`llHO4)T{$)qw!y8f){fz)&9vX`-rq@tua|ObMnkF&?fN@B>sJ6 zYZjO0te}Xjpl6HgF$jDJ8dK;$Fd7?b-Bd5P$78^wwD4F2*7c2Xk3dkT_`nS=%w$3X3EK>1nurUw3MK~H>lU~F&BRW zpYUIc?aZ5S=mO$2iMueuMmzDW7yb}JcJs2bi|5>zK@NwMjP!)k_W3w^xsNe{TaN*} zs5f;YHEzM$JHoau-=Ag^Y;7|3_-i9ZGgE1_a1*)ijBbb;+jfy#^F4CJK#Xnv*&(nh zH&=f`!#h;VtE>3COr1<4Z_-r?)Oh7N{#9L|wi||qlBzRb z{a*myj)!b%d#o;r(c8YSq`w{fR2Gc=6#8G^H>Q{A;?n%~yvcC`qO|USch=pHToJ-I z!vN@S-Xi)&v1nh})Q6Sqk>@;0{C1`}QT6Fr-m`M@gYUi-3Ih|=(IdRKxji>p^5yge z=^Z2(@Ol0?qO75OC}kzHXh^DE?WxOE#_-dY3J>TYgQ&c4%S(#DZYS_F$(U;-3?=(Czq; zLlxZ3{4~9!o#tDFdCya|h#`T4n10^ZXU!{WyoMa?XZqr~|8NK9emUHW*V=fR#5kL) zRvn{?)jn2q(%#I1QYx4h7o_Up?*3xIqvI}Dw+~Q?f9kDV>qAu>^9q*e)j~2{K;l3Co4*7RH}{Mt z2fEdQuy;d<&!dbO{csIl8zrPzsITmMujA{k6yf(Wl54wA!h4JPt zmGXC5r1@vt#{dE2XN5+l|LOBPP=gl@B2Z;eWB|GfIp0oj7Zf8rNt8_RM#{FbJLZ(R zme6UoA%|z<1F4q|bo7qMHqky2`h{)(Xwp+O!VOwst)|bn&opF7ySz{G+L_fi(V43I zwK~wMAt~H73SkVur3j!fa?I|0C~96EDSsNT?X@MlV3op<5ln0978)e()u_x4upCKC z`xfV1mNa0!eUoa8Crwkn3uro6 zD)Tr74WkEAY}J}SokV=U^ac-vX3Bl{4xeP@otSS01nM*|vr(~lRM0O+qB6T_J)~*) z*-rxw>bN7#;^JTSQ7p~XK4V4n&?*9;5k~%}Z+>`t4^r!*l)lZrY zX6#{e22Xv$FN= z1+}_*J=K}Dqro}~S2~?KXKZJ(AQT?j?IfH((|UsU@s$FWG#HDsvI@M2D>f<(y(YQF zW(@zO#LkpcpLZ366;q}E=8!2oTs&m0za6p%|9r^!qXHxEkRL-ktg%%S7WXD8&!UPi zBq2277v1O6G_Ro!8?bRuLo zq{V3n-agP`TXIMZcN|ym=Wa`7f6MMY2{Wc`f`{t^4jhfuO~d?Z9L1JO>ApsV$~9lOKzr5<8PQY_p>T7%$_SQ z^WT5{zRU8zk;!%?eLv6lBT(=r+&Lk#Ha<2H^GaOlDALbJsZkBexw%R59F|GJSAq{l z0G&|S>_yy=kgs)kL!UI9Ag#>0Y_lLsp^s@nt_cJ#5-Df_2suG#bd}GVdX654>F8&H zG%`@mghR}0P9BYSp75k7w(o6Un4cn7D-LUK+Usqb%97@oS(gm5g2OW>Jp?;<)!p(l9R80C3_MB^vv|&1<_6^;$Bh_i}=6@ned_vJnrbnawadzOck`pfF!h_f3W#_wIKawms)L+L% z98ySlyWi;p&-U%9Z?2mVPbirl7=kgpY#z;9uHX7fzvFhIv@u0zH)h{Wy$1o!skB?` zBm!>UT%O(Kr}%<&qa>zT0`6UL(5d@M>x+*G-K2AfpJofiNk(O)yiC7+LbtZp!EBra zkzo&gy$m=yBwrnT7G~{MCWJpHS!q@HfTCG`6k62aVc}dogoOD8_SQ}=2;e#DGXr8_ znOU!Sw{vTG-FxwPrN4U&nOQD4X1+|1+bghkuC>A0yE}KDX2RrJ0;aOWwM=B7g#&`7 zUsA5No&22O{O9g2{|~4=0mty|j=*qPYCr7zvUd z=WZyaYAP1%G1u=_g`#DlQg(U{IH4n^&P^5xz}(`t!8q_2qQn$;{1x(Fy71=Lz({jU~k=$gl6pkn8mc-OF%K?Qw`f=JTzuocN>_3D*R4~p)-w695eevcIGb*x_yTGq}D zxu_FX8Ay{~9DnqJKZLN{pBUT#f5$H+5s-ikATl}2&rf$*T))2YlZC#}<*(CW*7VPO z^ZWZ*T6({I`zCAxsZqX=57Zt;H%IIpqa+H(ILh7kcuU_a`W2J?gRL*78KA~M}V)%!Hm!3g!t%vK0Njs zVCFI43XTz$LzlN}y$-cbAPlHOU=;(@d11YG zpk{cC6-n7CS);nZxNn_cA|p!je}EsQek8$tSO;ZszOJoJdX&21u!ef&zIbPX@4;Kd z%%i}#PK+B+Sg^_hjJFJ3YLzZOUy=FPzb@Yae@`n)tMf(Z&dB0#xQ*8EiD`7mVZu@O zti!j3cnZZF!baLQMM`!TxJONrvBm_1kl#Vrb8YW6?lU3Av)|Jyx%3!sg)llhBKd*X4mX_JJ~vbtdXZ*9<`rG@gv2cM0-J2*Cmui&H%*@W1TRX--X0!F*T1UcA6AOY>a)Gl|U0wY~)&KG118vrt z^G35*M^5;MSXgToe%tEL5iBm5-OWj*kPmiV94NHnd`9(@9{-mg_-_E}Z~YlRCN%pR zllb#@V#cDD0$^Zj&~j$PxZ7#~KtJOV3iJcrwn)6l#%w6SiiioNAAQJ|8701@Z|LS; z=|hxf?YC8gDPFzb)21qwF|-|aR-rpq1|B|-e1~Rvr=o{jILbQUQ`x-NLNNZ|&4Mqd zPxokPw|K5pdJU(y#Az9d?L9pE4{)T}{omn8g}b1pfi=5lDE7Von2TlDsUP)s7YY?WMhb$s0DLM_~op8qG#Y&H*=q z#1W{G6|SnA=z3FyG+T%S^W?)YB%$P`z_0{R*u~N9>AlO(MBYU4-p+r3*4^9tmpXSi z`qFYFr!)fnU$XPmk7d47`EDD06VW^D_Hw+O;pps~S$4)}F3dgkCEv*El_KyjA6Krw z*E>uFfhOlUBF@XQWBp=ZHJ%?Rch7w6O}ums2i3eslJz_d8a=Vrf za{~JIVV0M6?fharHm1|nJzLP39bARK&Ivm2zy6)h`?VoMx`fGXkhGr#bXa9Q)&Cy2bp(=q{(pqFW5blxzB+!WaD)VN0uZ?Bbvs* zIF+^e(a!m*D6juL-(#U4&BR{NzacDpW?L&uI3v7`bN!loET1h`sd1~CQx_Kd*|pyF zVaqJ2x-B-w8lz@F`+w-ZU^*7((x0TU+!wDXX;IEn0g&=t zlicK0za^8ZhYDtyqwh@Ug*QqYx|Nz|=ynFbJhwGSZ1Obk><#!1DlbsE!n66+sq_0$ zvLn*W@N`2&U(|7bCoomkT1O1eao-z8EPv zBS#3})<*RAZ3jp;pN?hL<&S_Sa2JsUY%_>&G>s`2x($|A6AU%wjAtnUJ-1;XCtY@O z*Yc-AvG!kecbw*+3sp#-az5>=ARD@`%azBqu0r7v*enCejzBS3W#mV$_F=9Q>45~CCJb*I`{cH4*F5M#Ja ze}-0Y;qmeTv!8P6R{C4Jj!#X+H)EnJglQ%;lTJ5}LI*r$YaQhR-B%w#HEHe#r53yq zLf-mS+8D(O&npgn?v?mR5(4pi_ODwS=8e@6lS?CL1O6}{Q{qO%0x%zY&u1Modj!g) zB!HNWZV+Aub>CTBFTH{`;qmT^)#DMyWiLi=-ymMqXi}yxD%0iN^cyn1)IkoHbyWb; zqoiT>%M|4`t%3K!fxW871Cn=eyHH(aD>WOJ7-}ZWfyxW{) ze?)nYtfNV)+UQF~SQs*MTYz4TgBeZo>;)X*yPEkcJKh9m?8 zE*ncF+sNbh8~YOoc0i$k(}!$!?|@-piqj1Kz>D3qicl_+U-fQaB;cL4(&PHls-rCm zm`oPFKrFq2pp8fd{$#gb_&DbiGs|AB^+xCiwY7wj13O-S8H7E#X+8a*<3Cr?$u*6| z0zc=Dt`GTk?zA5_AOs`c2eKnHHMvsTl$qi}OE9n;3i>i3g!iZK!dQL-e>DgrHe)*2 zuD_MYmI55~Qo@t&moe`$(gU8?_S8;v6EYvjjA#9Oi6lw^~%PYj{l0wx;1J? z<%>}vE-aO%74Yv;5-8=D)Nr;*JICoxbIcfelWlG zI>iSuJX!5Gjuz5X@moj5QSE+ftDFbrJgX4ORR`ga^^dnVKd)lr7CyW8<8#V_LAX<4 zXK(uw*#?45*n^si!T?D_sNuES{>WSPR3eY3soRs}p`6A^K-=`zigU0m2*R($zqu67N>oi0?_2#Q{R zn3-STShb?AySw|2wHI7n&q+XEcUw`g7+tMED z%PptyJ?6?8x9F@&1lmvBZ(i}1#zQj!sv?vCajZW&*iouQFz0pJ_wGEMg`mlmd%EF+ zrtFM5j)QGP6{Bf=vxriz@Lc4@bjy3z+onL|TAE&7u6Og`nA-sf^hp(jQxqTXw0KDs z{Nk(S6VxSFudN3QCifn zr#jG2gt2q|2ITqKVLawqw#7vg@Zt38)3R@|$vPDvPGpcJEApq>L}e~jKl!H1*Y3ao z`w2?-r*)E%)9r>5KE_ztjBijXRsn!!aSj*h$_8vv=wpE`%IIJwIs?=A9YLA0^lO<) zNzRTx8BVA0!Tl#Teaewo%66xm!NCs9V(2MEPtrSSrrb0NkIGG%bTN?OYo*6|tY=%? z!R8h2QNo7nE2;sjd)ndhfO4U43mkBn0j0e-?PfDwdaQ*iK1DM>@JCDzL$!Ry$M($% z{s-3bHXmEM8;S3Ya@=R$6#KIc2e2;mJidP&Obi$^B%%Z>&-KIgQyH=3gCt{tVRuq4 z5Z6zB@m!c~mz-*K(5>sB4KK_OThzZfEA2h+N&m!EuuFCNBm0Ymo!*|N^VtR3>)ljJ zLoGq`E-j`yByc z5Nq0C*o}Wxq2Lipr$MOz60b6cJ5S`uO?*swrd~stKYy@bR|OhkelKM@JU`ptLg~6e z!D!HowJzJ;Jp$m#7H=Sh!70yZ#_k-sDOt`4uJLCfc1<|ZXlkUhH%A6&6{oV~?F-|H z0v*c`*_Tv$HW;f*1+YymK(xwHVHy3I+O4SysV>coQ-em`soH?Ky*pl(puj z3JMB(4|YONrIYjsfR>!sYL=lD{3QIT!2^8o(e@q9Twmdz>-Q~ljHBjr?MjHM`V@l@ z&a4UU*N_nW*QBXr%!H$w9a*%VBICs`%f!DsW zwk7To?Lh3vut{d&^ziU>g;iXlN1lM?;3Y>2TXxqv+f(3))&4Cd1gtQk$H&Jt7z!ih zr29k$t>Oh?qcXLJy?l$EVd*XHL7#O^}XbMdrW7%>Ts(EC~7O zAGgU_#5tQ|nT&n|;?u_OLDiwYgRK*Bd*BP_@ni0VxXe7n405WL5=!CsI&WnQwVCP~ z!0647l$w+IgX>iuP(AZ@Qa-Dcn?Yy6kIB3>emNp0hxeDi&d0f3Q@10r88vjUKCouI zADki~L>@^Ibsv4LOP4OquG#fLBs1T5-F0eQE2rz7KSY<5I*q>sI6jhV2IeNum|w5H z=~SWWJ>|+()~p*#x=-*LuZCUvss@}BL`!zML?Tlgj=}SWn$bKvb_6|=$l2o__rBl+ z13D|bTz=oFuZNVk&G~pwv(k&@3iw%h4;9n^81kn?!E|RYkx~=G=VjYf>|1cG(V2SH zD}^sYLc>Oc=kv-IM!+8b$oqV5FM+fAqr>!-nmKeY9C6zK$ zOs?eGEY(bF(3+_u=IZ};%459HS?sKWTXXSVp*}991ZMQFS&XS?4SUd8nt=Qf6qQFj zInFuNU7N9UEq1>KIZWY=X4k`yvISd~ z6x2YS!RrMt&fo(g-|5<4{N`IG%gce#W)O5e_Ry#R7@OcPoT2dTvo)f{OOGX&9A_Ud zCk)}+zcbo*zp$t!w@a1q2!N?Z9$W};kVcmDHWbzcGdtf0K{M^mWoZDeFBAZTeB4Bw zl#CHUl0f%5?DWy`oYCm$==*ovD#Wt{Q>=pC*J$l+fzqi2%83>!0tdTjO=^`cAOYScuj3GdB}@~hM` z*NYCnOXf26kDXtQ{%WNz2dOz}_5F8ew3Ag`G z`7YF=?t+*uz zRz|40X{0z-rVQAu7QJT(OEL+*-((VDsGFL8b683soEMWeyXi>GL6k1J6~}T|8hBWW599C!aqov%csIX?YH?IybZ%w zHY#m?UAv}2d|C}=1qWZ(NKlW2{MQw?F?MZZwfe6HCIG)nvZ+|0nEXh+aLkkDkn9ar z#dESjiX!Ie#?1WgTdjECR?@^%DcUwii$&MNnh0#_hdskJ` zVrR??!|gGkG=Zc9+)MJsL5(!6;0NW&+rc5g-M@al zd<)8#TI>x0)S;OmQ-6r7)bLb>mo`Kz z*@?_3E|2oSkB}gH8Al_3fBRG7+P*YDh}0!2uNS8g0)*Sz`n=NHp$9U>Y~Z~BH9b&= zH6|jcv)KM4==!*kdSuf+q?|(iimbJ{E;Dfn)0I*R?~D`_6dd)sb@X~eQ?d1`%}e;z1djBC#^kgc%@PrzqMLrjjY+kZ@GbAhRAopUi$Xe*h#V`!Z}uxXpGMc6H}CXS-4iRt*KOH$hC&jw2DogFJ%S)g)Q z`}fL$k^4Ly1QlMhrKP1B$1(>72W|FPkcfvbsHDHYkP|`f)WV{t_LqUEhECL5_GQmz za+bIND1Lr%O<1RVl=!2s%O8LrnR6uB=}vM0*4x{=Uf6_E1Fszsp0@3Yj(FMC%-&S( zL!?m%2w;Qeh#ZOQgJLE% z2>om85nU5%{vaZgxCtttJJ1ZJ>8&XAf7`jjwGq*z!%0<<=zzX>a8Dnc)VP+4V;I#T zE1rpGUXfD1j=tQ8&Yg}pV2YhE#rx#3WZQ~mh`ui0vu-6Ad^z4)~KO-`eT58W% z^dYi{+us)@51;uEfkq(dh_EPoc+#Y%XV7M~6w$QOKW6v% zqrc{r^85dQ1FRm*Qx;r6u?N=OGp&JR$A(l9 zbR2jMe?!JkT|$&P3Txg)u`kj(nm9I~+)yO2O2mi?9mZ|ZH(ti>p@eK*EjMoam(&UX zsG1}mx%s~T{d-+05`q%_*XLS#BD*O6tS-++bvH04{YgQ^GiR<4ROL7tjBXMfPRCHu z$fWrpp0hciTBA*~?a{ONf&nIn?jdGzwMcKyLbo&+;U;ORZ$kJNrouCC#;i%d8Bl=M zuGpo zW59^b98-RcYKR4MZ_f3?7J+bTBl+3bFE%}{$uC&Kj2}VC-c>1j`3eAVjVw?f-$kB8 zg552lfwt>`MFmNl2DtC`nL5$`3HL?m&91(#FY4{wHtPQg6!$;Z1c=>3;RV5A!Qm?H zHhuvT6;FUMbWR~V7(5ldf8wI~;$6PdT}XR0Ui2`L=G$OM4JLiC{5Mh&u}CR(7owE2 z+-*Bk+F~;{yZiBCllFHE8Z1R#p_|KIKA@76KE~*U?i{y`HBdS-U>qUF#i2xwU## zz&e<;f|z?kdRit;JC; zwP*t&Jk+2qjx}D~H~&0wtXa$j$a00N{xkLa8u-$nEw4Qi}9fCx(+UqCe*~ zQj+8EG#a*nE}Q`#wiOq<(R#M|k3KO>fn^YA`YOJ}o5H1lAl`zR=I_+ZZlu^T--+1Y zS~ep_I=HXMeAt>@26`DpY60Cw#P;>*4zz-Rt@3kRvYGOBIeIn#C2;Dy3`0jGsAD0g zI8qe3g&J-`k;XGmOLO}77xW5f8VU8GFfriRpP!!xSnt>gnK4ay{Yoy|&X>ha%7 z=shk4xgO_ErD?!AB(a={#aMjh!sPF(I0R(_Y%lo7BTq-(fw+-Ar;BddT|iq2A)ROT zN16<((}qJzgVG%1_kfzW4W(Ucr+CmB;X{;$@&mvOdahp;+4TC!sm7G9TZ6yqW$=1~ zdFPIJs@*mZoEdZql4>OPe$YOxoJH6nFTzg3uoJPke2AbkP#_G?=rvgalv#%N%R7;m z=M-(@r3s<}4N$RiUo2P$yUfjmm&S?weBoh?aN4fP)BbODJ;%^C73%8|P1Vc^+up7v zss}=FA_9&#lF)4XT#G2g#U{CPh%SpnghSSs9UGUSTUoYSgM`11 zRMwo(zw_veP?tgJaOX1en1LiRzhu2pG?hWvNr4_YVD_#5+fon%A!Z2cV_1L8@O~nB zwt8P*I0B9jycfOLn$qO$HX;eLG;~9CqL+{vNEpyBgM6J*i%WFpAeCLT zYC!A(bQ5b%aqeCkx`X*NC&~s|?dCPc*_QkC zLU>Orn;@SpWp=RZ(UC57rfS0TH=5nj`y%tF-i>rYEK+f7kaM!xXP*H_RuMer0)hEh z2P}g3h9Z$dlBNz9>Rn7oTQbLf>uV1STj@|#riDhdtjBIU^Kb=d58H?TPA} zRDVty$0}Fz{zTIqPeb6CoL&&!k-~kGt|QzTD!&_Bp7FL5W^vW z2rh@1FV*D8zQ&|FbM`Zx*+w28>B^ihnn2>l2Ni@38SJ#{0Ava%#OUwVxz&r3@Gk&w zc8(OhgbtrgDDf#2r25z(9zDBzDjb|TkjSq!MGv4d$}=5YWf~>|b-{Xn2|qagh4^Jq z8Yx;GG0bUssY{29U=%G!<}w4${1ZjcFsV0w7r;tn~cmybl!ObRvUld}pk-v+kEG;p&@ zgl-VmR^(V=Gg*0cWG?-C}?At}{Tq z=7K0<8^SSOhu?`Pswa=6HCCE`y=QLyla}@-5FV;28ecApAB3||uo9+|cJ;D+_R4;Y zeaxbueJ2SxUhrKvC%!YT{9T^@J0CQhf@^k~>v2~>t7Z1;mTW$4G)-Q91s@cynur3h zwc3gEvLYWm({9*sjg{wtG-FD%o^ya+dx^_^I`Z?X(ok}an6sNaSie}81c-kwckm<^ zb>}iszzMM0{#(xmj2lP8U-NyaR`#k*oF!(P;ura;K!SVdu)v|G^fPIoOf6%Po0(&X zXT?r{ZKN?CX2vQWpRwfN7^^whxr;;ohYpLWRYgJ%84l;K{pAR@ z)uBK*p4@c!hOV_o_SPQ_TG>Ltk<^Ej|Jz+S7Y0|&BLBmF9qrZR*okLVWTQp974A>6 zs_!cPZ^A%mTYwRMN449ZSQYRd@kK%>s~E2| zw3f&={4A0jGKgOq28V|U_OdBsc4%)7c4M1sPSx5C`dBk%!bmznrsJXMcS9+4SDjE@ zok|5$SJ!K;d8s0RcBVzC!I|ghP@I*C^aUlYL?MGk7MP7a7&L=SShFdTa4Ks%MiI3D zybZ8lzE_Y-r@C@O=BV_zU7-WBBh<|eFc5X}+Pk)?mWLA5jn)dgM+K7H*C8a*IV~Jn z7om65_!v_RGNV36q^9_Lroo*o*#xET0_osezp7xgJvcJ7j<{w$RAlA}25PG9*U`mmzA~+(Hh$q)dD-H!|YNGoDoKKtGnVJ-%t zVmU;W>Y`^Rw0gjk3&BEjl)8mV&CL=_j+{MVs1g0+^l(pFX^m6V3RK7n1>G4=3lCGS znL|UYgEc!b&yk_~qP5FxBw|!{0f3ET>qbKUT2g1ClcA%O0{0}ZA}j#!h?X0G(O7V2 zl5Vj`qQcMyA^J@2bcD%9N=|izHAMggdsZqjCma6}p~W0YOHXb`h9QWJH8RR@7^)G4 zQO2czH_8CuJv&pYN2Wyx0xRN(!d$r?QBe0Y`-8nmid^bfx(#yiSo&_RUx7m!O!jsh zq|b_V>+w06hR%^kQ8D^VIWjax@Gm>2OC16icT)0LkdL6T;(72ZL~i&yw!*~F_o^0c zR<%sX%2%RqT>L>>kZcTF7`61^8S`?@_XV2{ldz64%7Akk}U&7#NN zD$RD`0%ngag$p1vG9vMd0N~Aqi4Y#BvwqU)5n6q2O!rRv7_eQ6{;W-+@1TdQHrJA% z2E7nP-T`w2g8X~Y8&x#1RH&KWVs>6q1FfNz+h*P?F>Rp&suvFFuBYO*ARNk9-RY~) z(rHHv+BzYff{w-q*~>>)trmR`NJYTFb$TUP247>W!bPPuy-Y0xx8gTkRlUnNXX3)E z<`GZ`v5{jgas9t{LQrB~pvX?>bM${<{&aZM4q`{`Bu-)6E^^o&TY=b{p(Acczf4Ey zqnXwNV|Tlile)Jd2Qc-h9d~E}qEk+7{{r(Kuwp_UQdoq{Y=I6X^}NmTVoU7nHucY- z=B00ek{@WvqU>&Km~BnbTXZh^ZLp8)qE2PZ`-#bK7-#nH>E1n_yS@ z+bsdR7OwRo*2e8B($dnLf#Mz>9^5CJCM0>oYQ(O=035NsT#n#k)3wAe=s0LVK;G$> zCF(;8bNo#pg*WlePd8(8lh^TDwgXvtqDgmE{AQ2LI+W=5 zf+HC#avtn(^(qvEDt`{}_^&HGg+c^Hmeyc!cmabNk%4HlDNYPfT;-j1k<;M?c%9W0!HuR-K+*EY2RjUa2^; zMc%b8*C56@n(3f)stl~*{@qyZsWsRGy-?q6xL))M6jQ(m)V_;?ubZ=$R<1!S9OoJ> zT!a>wf?E=wIyRM|)8v^qxi%ywX$h4GJVSs&G^Ih+uH8lw*hDD}w>93c@o5U$-pP6U>D6uZ0fa4RD zYxb$B7$jgq+RGdkAZb3ccgDDew<0&fJ$kCP$^ft zUrnq54>z<;kUB0^M0P;%o%Mk4a2SC}Ix@W=;lpGRs^_ezsQ_;%UzRU1rDTbwhII1X zD}FD9gGf0M46ksFEH?%kzzb;4^^=$sSd_ig%5JZEC?S>ezyo%e6A3zlIq$w$s@C!r zI#bb)sQQsLC-}%wFDXm8g&{Ot(u>sSHfb;_ht*;GSiI({WjN4IXhtx z_b-yIG3mAfmX`?{z;MXM?LdjPuj7rKABvlhw;;KtQUxeT?AiiOnNVaX95=AN4Z zu;%5he=>GDyh1$S1582J>0ekQ_$O`+czWX(_}z{owC-h`tBqE$M$g4qOw=sM-&ekd zSvb0J_WUO1$n}T%ik0b;W9e@{pPllmi6=LS7Ia)c@#-j3HO2h@mkQkUgDXy4w7r|^ zdP%2d5}!}wHIDPf->UW7cBQ5TP^2C4<`pR^DaYuR^n+)&cTbfD>UJYcL#A%g8cidYnBeL_iZ`R` zl`<>+tQ1`DBjH6h7%cQa%#)e*&&yP{JE3y*?#ZtPZJCZt>{nVE6nf!u!&aVy5#tgb z+;g$XZ?|(|oza7Y`ZXiYevRrEckuseJx0MAX>3Q1;JS{3on@>o!bdfbSu>aMzne9~ z^oTj1Fjz0w<@Di-RWVodpe#7bdWGuFAsnNRR)ug+-ha$ssNYiKKcz5ez!z-;1M-6N zvUF)WjMpX2vZnV`{9GkgEDj~|fj-`4TuTOhKhRlV>f0}lAioom^bl1q&<~;`x+Qs4 zZ>;I%;;}{B#zj*8mhOU)h!4nn2z5U=KDL203qz1!_wn7arhBY;&$inl+#iJ1VwnAR z@>tvw#OCG4DNNU-fY024pBHBVWVD(_2w|!coh8Q+&b-HKRYpA%SVkwg>(j{RuHyAW zPH)u{W-7gq9LCYq6Z{+qT)^l)io45r`d6IKklgu{uP#em0M9}6{<;?W(6wNsMzJUh z5|TD6e!MT)6}bvE6mu=y^^`fI#BHpntV|?E41gM1t(Ih&uJ_ED1fi@*_x=QFmmy;W zkyRYdJA>Fxgv(s=i?lD_yQw{Qq5C<{-{d0rvd-FtOVJiO(np4V_(sxC+Qk0cGWC!bpi<+db-4 zZ(DUv&L5x@WS${3wIz9X(P{7LMAcoyd$ttVgQqiRKYka+=)d`dX`^ioSMkbG8u_sD zqJpvh#h%%=DHS?!=C*U}3Hgf-Wc%`Ox~j@jzSXJVO7JLXZ%CLMNT$)uET zgwrQOTIWG7eXn{FU639^WUK}FBu#Lk1C?e%{47H3xl&o4b`T9rlp9_OAi3W@NAox^%c9i~1^bG6PoP< zG??Zi3};GXPkc&A;4pLKUdPv^fJ~^|b}njiUamGj!S{ET*7i5loJx`>^RbXI)al4s z|A0;o#Ao(?X{c|#>CalJ2(PB)#ID1r%6&P7o5d?KBbx?>%Si#lOx7 zo%hIYs5`Zveuo9d9~LkRg*&I0aJHY9Q15o%zq4GOdStHAc-g%)iZnvJdTuqQ_Tqmd z{&J2R%-rt7tR;k=fN}93cfH@pKdK4YVj#YuQ{J;^Y!|P}hf}}?cm^Nzx4RNbiQJSZ zaY|*lfI6SoyH?!nAZPjorwF(b>(Tw--@@Xz-Yp&Kyl&lvjff9VPj!^y*JK(viEWM= z21h3d9m!d3`)0~VL70#E5-zI6NfEo=uq4eE7`!d5Z-y(?-(T^PH3l&nk6P&xNYUin140?*-O0 zZRvVo?QspkM|9sIxKsX`x43e({YL{y;=oXGq2_t%R7ITuHqNw7JHH$|dNg(QYQ;~2 zAEu;YQ!VHH0{ss%xMgUUyXhZT?>spAVThxni0{Vq$`#!Fc9D06rtf?x)Vv<7qC-WO$xh(0I^fzr%;7%+?Vm9xgsA%2-SML>Nh&m6g18-rKRpT7HA6Tyb8Z zwlk#;bHrzxjal0568G>HVO+A)TBV-btm#7O7J?31gDMvW56wQPjvc@^N~U5j5({yf zF>{=j6~;qSn609E;@MWpY=+0S1|B!(K>jFFF;6A)t?8xAWaCm|Z2Kvdp)EU$GgY)7 zEAgdU6gav_>cF~8K&MNH(0%fl&<3EIW4s;D=b)yV#E{VFd8eRvNYn8JcGrW3d?%@-v6#)E4eHz&|eDZclsH$djg^CTXMmhmPms;T8 zj0!L0+o31JcB@OQ%dEF2n;ut4`c^aHK<~IN-r2DuoqWLc`?fJo2R>Ns2WWXdxR&B$ zhO;=c{w?SinX~kk3&q`LRqE|aiw@Ug+9`mROG0 z_3LKTPcFW~uqx|tLSkiTs_P}2d|PFmh5H$7KbEt*U(Z+t-+A#{D$+IGpKkW(y=&rW zN|%aZoX<4Rtdy#2$Vno3jePUm_#quFjbFmqRr5%rK`~kJCnE9`dV8>G_GxS%uYHN94H9D_P zJo3(FeCCfJo94&6@m=~A8V8kbQWlj#eNVVJwq z$@pQv`9cwuzr#+w~v*OKS79FYSy+*e0?UWuAAWk*(wD%DaP> zN-9SAJ2vG{64&`pO6`;pP%1t~m>d2S>!odVDUUI*u5VCOl9#(}oxcll&bJvZhGjW+ znB~~{dmOGmV*PdI&nPqM9Qu;+(d?u0?h|7h`BZ*reGu5XXXJ)@(&|EYZu)Zna~#_i zWEJz9Xiz0lj?JV!VrGCiV?Iqb)BqPGEGwY2M zmxS>leQTq!l@ljbNi`E=pI5(2kxZtXAU&u7*kvxenIvn$bS={pgmkqLwguZ; z?tHK|tXyU;wN>*564Q4LqcL4dF72FyqP~r45OUNTtsCDp{je&87>4yY1UDq7lbwAS zm7zB5o2=Y);I7;)xVzX60}ghFf876={u|LTYRrJz(l0P`8#%!K_h)X?56%nihaD=F zHJ^GWoqU^JB@8Y|>l+6d-z#ppYcIi)OgMjjhT7L(4=3%eN~@KF%8bJnY>~^gBD~ub zuT$y-wT$6-B*KJLt%}${KDbM@n>C%>!{~Jj+eZHr3AB0(UsOj#8U2@8ddJS?DEgqe zCE;pEu7znys~p6+GTn>cZR0Q5k8%g|Atxq#%ee}XSbqoJEswZ9ujHL1AZMUCW-igv zvO*U}3+@Bmnig?kmKfvr^nSE!=Y#R?U|k7;nPP5RG2&4r&pMmnZjw0tgXm?)U=oMBbg zfJ4hMfgV{TB+E2Kogv=x!r<8bC*0f4@O?B>aovx}tXD^)$QM4233awD6DRZaASswM zB%Zdc;5m>i=bKwu0v{ICz0*(4T^Z^u?#aa7n&ZrTi6p2Yv!O?AtP~n6030 z`@-65PdOxq2g9AY)7RWKl(N$~q|ijD72fnzr22|VPXRq_RtFa(pZv{K!BHExAL2%J zL(LCHD?NK`%=c&$E_cjfR%lYoIUD=QIvj(o3@u$_e=Cyb!C}WutZDbfc0VfZlqlPc zWR!80L-?t0@cW00SIk8=N%KY2K2MGiZ{Za_F)Z`jTM&Z3Oy0MmC2XKLtSoG{^}y4L zY?}tGZR6#iFTKI5Xx-_0IKEjI&H+!8wj-i6#HayJ*;&-|m z`VvzsAowpnONQ{xj4}R4r7tXBZy!qAE{#bc5SnLrydvkHd-w7o=OLzqX}~^bunR9nvY2ZM==DIPdz-yz^xq?QSD)FV@Q@-#0;DS zB$+P|$RoYADGUKU)mqi!DRwBJ)`yEf+LCZ^O;VI@n!m0yS}u6XTr)06FzD(exzBxE z$t%lkE!$`vR&BWw5(gXEgC`nE5&cMUr4of38rc=skmLMtOzLO(RBYX6Rp>D1vi4Pj zk(WF(r<~}FPj-tMW2f*Vj`a#9u=t;Q`)MU_>h6tHn{HO6i^q30w3M>L*ZEK$wvl}4%k6_c@~7Ke zhK85>4{l7MtL36;U}4+;|7jo=;wK;{wFE{v;iW1)SXI-2U}{oOB{|hm+rC707n;qIJPXv^lEA?r>OJ)BkoPWnmp66(Q0k&Ow|;v;)0-3wN?QEWfMqSRJ4e=gMdUuQ1)VEU!tWI zl_e_5j;JW85F%ng1cHhXkxf9tP6UJi5m^&RNWyvEFA1RS%$fh3>pJJ6*Q5+H$(QeY z-)Fh+`?+5aC}zxv4!DHua`ti*CtSF4^biR3yr#Zvvy%n+QHo^-=8!SFnE2Rem5C9v ziIH@<(_^puO$o~4ztsF!27nE*RjacPf2gz5{5($E95N zf3+{odaol`lhd9$RzVrl7*!wLM$IMq;Gq$EgLf9!3u7v}aMEY;BBwzkk#VML3`?Az-d@u)a{qm;V&+2mts%2qe!zKhuJjfc#)`&a_+HA|iU+_~Y9Qd{nrD&47| zC2;%**W0R8cY}#fgA80*j1`-Ic1_b$t5fSh4mW@j+<$_Ht#_5 z!XE>_;%^jpmq!-5L?Xl38Z*&bGZnT%@x1J#f$zRlI+`WiYjxT>*P+=dUiyt767n9T zjPtDj=^Le^YI-tIZd`jepr$3KvQ*agI{U1pbrlVuW?%#72KFU*`j9q7o`QJ8l**oA z)M^w+>Y}PLXWd!$V~zc00loWs!n9j|j8zF%FRW}Uk6i0^Pv0?1K|Gp7HuN2}FKt^? zw%$Db4%z>`GOA-591mEPg)K8L+g#{A=#(#hH-nsWWr6*B?&g-Kt|PH#s8(1X=N_lk zA?Zs6dvfdz?MsJlB~j%P)!ZI*I!#t|Td?l5I67*#y<$YYr@x9k-aC!)kp1)zC^fO3 z=3uvF{=xtStZW%My44`sXdAqL*CA?btP0NUwvd-ykN~c`)E1HBV5&K_^@y&qTbA&n zS3w+oB{6ehXSxP*?F@n0MZ(cw6XN#`tDy_-tN};d|y#|l?dwtfFSq={d z#G6f4r2g`(BCo!FjnV6MXHl(YOB%Ah)^J)rF59>~^6v4M&N(E4aZsLeKUT_376p)Ts=Uw}HuV%E0d`OY_s}=9orW;s)JJy@& zzuNod9}5)+E-XA};iim#C%1lPM(g&#XUd0W9a{G5-oUR7EQr&4JN$=6*W!*OE?K?G zX(D&vlv0hl<_p_hBaIWP`i+dop2ek=^vkXEq4tcn`~mOLCf23=d|oAO6Y0^-4=YbFO5*TUpjZD zYu1MJG*$0A*K}HjFTb5KJNy8(G;R~u@Fts@X{l7={o-t?O_vAo1)wmT@*XQ~-?07CLihkRTE&a_V=ZT6S|<^88z=0t zoc*HfL$qrRWSx3#%+VxM!#7O!LTQv*<}frcD7kyGl$G*ucc*ch$G_3NdV|M$UL*WkS19wHj*iI(JMqUhT6J~ovsAYU8Ve>pQ5b%uOumB$3j9aCoV26j!ikx7 za=FRwtuNl+U*dQ^_LlvVQv0z$9PYB0y=b33uQ#eHF?BKoAlXx2&)!>+6mLvf zwI^5YCbYIqX;Fe-(tIPlc7ud&+Mr}{A;OWGn1!OV@$?>bcUsOPqQ)`lB1wZk&)7pqp##(tuAgs2f& zzH^-F2qW&$=db85I}ulQBBy)gffGcIg%rSsAhOXJZR;}a=r)k74=t~&Y*XL*MZqP2 zysB`}PGG@}A6)`%;gM*ZU-wkMAETn&$H-!ajk)6Kaef;5@8nzx4EBj2dfU44)n7aF z-v!}R-R}4U9UIm4y6rq1XWEcx3Nem!BcFSE@dQ}6--Yv9`aeQ)CizLBy~t%(gU%5b z6TNfooT~S325eVn!Of|}e2AN*va6@7v~DMsoTCXbgi@y!me8qU-(%%^)OtSlwaa`e zUUM;^W*r2vTQ(Hb-A&^v)DdUp~**)Up#B&s7Bh>8fpq9M+!;5ao@YG_!wrvIz9PC0vxOMX|x zhmMGZ95;jW`Ai~*{Tj#rWtZIkm*1;G6fM zy`KI>0zXPH2SYYHsXP)62k8NF@i!K~!mC;|%1WdQY7g(1&yi^XsL3_+I?1ihx6-6QDZ!&6+i&yFv$6(`rekUl_<0kJTPl7}B(ItxGWrw_g+IDzqFvjzK1?I(@*{ z)QOJ5sO*6x80(TBVZ6pg(fsk%0X!3|C+;61iS7~vcRTwcNh8LjYjl3OTkaDH+CQB6H{Wy$W-|$Xd>kk)_ zlA=pTPPubbV!+D$fX*V@f@%ry>SEJkA@|)3x$n-!YmSosrj??lCrQsjyWNWO5n7}FPJQdo`6wcx zM4jJD96=jOP6glW++$yQ^*c&SD91>>l z*m;6m&tOBI1|niMA|hO~0I2&Re^2UqM+R2^?A6*;f`e}RJp-D~@m^F;6FDIbkD@)A zHl)WkZE{4Ej1+h}KxNjMy99(A?uIsK1P9^iN9ws#upkg(Uo-E0ut%)`P)pPf>1bMI z8SHxLWjnYN@Zx3SUvWGu$d-NlJx+lK{fJ|#H=2UXcDE5Xj+Qf;y@n!o9TRPaXBk?8AD`ZE6$K)s8OILF)mJ(JZHM*tn;yfC28jSbK_``EGfErAzQg6B zBUiUUi@-f-m6$}X!HJ32K z+;Ce^z|{Lnark+Vl)R!W>-oj^vf`cD%DvAOeuBiZIUBZnj8zRhO3%B@A0B!dO8R&^ zo8vYQD5T=BRFZ%pW!!=TU58KBppMZqID>*)pj^oS_GP(iR%orc)o^kne;^eLhR?jr zoz9{%D?G;8(Ed?G;+>S8>W|Xv*QgHJ8e;nnXs?OtyOyG5(q^Rs9}UUjs4MozoX{^$ z&rjt8>f7{w^^l@dK1$s0-9~&_((+4dnU;%|G>u^C&~u|zac?TcXMdmnn4~s*mn&^r z8N0NUG#g6H9);b1IJ+S6#drys4^Snfp72}DUP^aIH;Tp&S#$@K5TR1T*u2u_SWkp( za93TwL~Y-a-OjNug^n$i{x~ol!!m$mwX*pV9;fqbNp2X`3AKX?ex7})MP4Q?{aylD zETN+Q*7YPGwf}Hyeb&VgVYC?r+iurbR@<8=1#jxzynu=VoP30w$@vz@G%|S1`oeKx zHDEpIHyTDgk{c=JxlMTX6_7YqtltRYp{SbY&wmkwvBtti+HfRq{ub0w!%3j01yGKV zKI|ufJE18$d@gdLud6(+r`DWcio)|p@y8O@7VAwptnuf#z{`Q0?7;O2c#`;|HDG8e zCzk_&u0<~&3v7-I=S^D zR3~>dtct7cDC58O`2#74O7(iR!;~4QKzTYNwdP5SR@dX2uBA6l(lw~SI6M17y5^HNtM=iSeTk{kvjLR>Rp# zHw?vtWHm-09N%Xebe{u9dF&t?@an5C_>8cT519bmEN_Ur+{E^}u479=uN77z9+Kx5 ze1bhN+Uh@Co?EkA4Hy?7U-wreX zHgzR^l<$ahXf##MW+e=A;WZ^BZozv6wz7r7vhqlTZ02k-NAQ*S$b!z5W5LO+9>w zB8_L(gv??Z%76OBt~8uB7C_ElpC z)rd2u>b14_on{foO_skICccvGGBv|9iz@E|Q!FS1VuHwKS5%J2C#sc4#=)=-93^_u z><(d^M#vrC++NsL-!E$IR|&fePj+MBKy5pTcaoG9{{dVV&j^~?hS@?^aKUhXe?b4o zAVa9)R7n$LaW}bofE7&rLTOGkIuox36oQ%YC%t(a8A`(EN*fq+@ON3`xdr37xf9~J z0)AW%DxPYz3;5;jv#qT29t%TWNpkXFU}4AT((N~Szf;F`Z|`$jwsz*K6*E_DEc`})pvrjd#C^rU-kcGp zNoDYuLcYM~0#0dQ^*gz5zxPIs`SJWPA>u(pNpYIc0gT25lLjYH<5cP2r3poYV)0z< zz4P!v4yQNT+`y9zTb{Tpi{8z~KNHYNq+n-$mzBKK#x8|^Sd(gj?S5;X5JB1$$l*v|( zQXFr&S(Dy%r3|Jfz_keVe$F|+&}(US+)qk+F{b;+6b3A)=M3B@sZr&gul&6X{k2DT zeu#ScLSAv;A*s`UVB<{6c>hLL3|Rja{3LXPO&bY?eDfc=(JRP-l+nFkLJ-rABFqI! zlA1y$2|FOs_)ozh%}0EK4D!(YY&L-};eA zFeLje_k3Zy1-325kxj>HAhE2|^cyT$r|)%P$uMW|dd&iD@3fPH2L_L3Ij54bY;zO8 zs+WvqUnI-OO*01;SYJoSB6cla*oITX*Bwnpomfq6s(H;VxnZb_cNJ2uVG-OG(G+)| zpWbM%*y_x0=@0aBxSjo%x;Snt(NM_vH)Yk04bQ#yR*XGMAXnv7gC}sHmu^XEt=4;aYT5o|S?1@p8=V}q z#+dkcoSxDd=m`ht_U1r?|Zv=9N48{E1bG`eyuX&RLs&W70?s0C#1PcOTS-K;@oei*z{X zr}JqkbcDLhT~$MQw>PU5dyNmaAm|S_ZjRIkeU)M)zrr~17PuY$60zI16Yiu}$4kq9 zz(9@>tcZa{YcOJ;RYqN=jp|aG?^CXh@F(n64YbrQ#TbW-R!eafM)ivu(^)I*JuEx{NQX8^=&6F5Ff4GD>gZ2!RI zIyq!md?0)=GqAm}kam`8SZe-<@dgV3FaMsxj&iSW{|AisTK)f57;gt+zbi4^JZF=Y zqJn~R=xK^({S*JiP8}}}8>j}-+|!qXT5gWqy(4PlTDuW1x;J?@)P5W7GI8drCALxS>HxlI) zIR>*7n|jVhB>5HhKW@~><7oA3uaPvW)@!)cZj?k5q-F>f@l412uJk5CGAKtdTo-c_ z>J2dcz&iS+{^m<=2 zTUx+VPlW)*)P87jyrT$GsUKC9JxGt@R$)S51qNaTIqbDYhUMYYxx+8};;tl~_jsg)^254eH;KH7V zyP=rwoQ;H)0P9FEq8RaX%yqKPammgCTB};}KimhjGwpI_2W#KTTGL6xw4lwgD>avB z9(ECdq(W4KN4SarXsp{%uTR7CK9PYIwpqR`+&qP)VwHKz@Bh}m!0dkBMhv}=#ltHj zaiSIr!CXjaDW$%a7-_GI>2r{Xrhog*EC;=^i?!AeJx(ytJJKf8y^IBS!WCME+oG0{ zpsbimW%l+kJ3jB9T|<3(PGnV;a9tqpoi=*erM+&cwyaxHFmj=k`j#6@mZ>K@ zXjz%zg0b@ik@BJCtgoRlRq$jh{if~Kx+^=KU;@zGvP@#!(d;~37uJoC4aGlcfanuz z=U$Chee-r`z9fc`O{Hi91*h)Rgl$u-OvIuQXQ&?O>XJ1B}$BTvu;N$hhixH zg{0?1!HP{Dl89ZsV(EFR9AU$varv`n z6QA&YS9zCOIdmOr{X?JB`ZCR3 zvSgHfGsVy~E>Xje|LQybI)rh4^hHOD8!3rL3tIrN-we6vW3v>;x|1R%4&d-yYhTV? zwvm6hjY$7xFm7!Az=f~=4J5%7C0JY#21-Wtk17QkUmOX ztu;uV;AUR?*8@b}8U-qBUr{&O0|$jh%3IT@|n;?1mv3$5OYGt4VTQ$-5IC9*iZ zL-;t7!3-~)%GIEF`w(@l?pZTbvl_vhkqFSS z<4*mhQWZyFEt#M`T2wG=JH&sN5yufx7!$-`?-}TFt~Q)ra*kQ_i0hA{^j{v0zAbeX z@W)_GkbELbe<>7HhX`kl4M0Ew0`Zesp&sAF7Xl^KckH;!^8U@?ifR3tIaer%itipq zg#;1_+1lvE^zAxs7h$17I&Zw;##{`emkBaBAZxxh{3ClWLTrTQMPrX!gE_ju)y%Wk zwmNSml;Szkm$jKU*Isn>DLg@~JSVFwq26}Fg;)Ux2N`Ts!W43Q8rh(%@Od0~!|0Bu zLAK&ryx@~9N%h)iB5HGbnO&oFGDBSa7V6_H!8ZlH4C-mK^KkXQARqui9Q{O4N|!k2 zg#OGaFJmG-Ui)+fe^B(QmY#l%>zLM^k2;0M>ZWJ zI4P?63x+bOciqk0Xin3r+?I$_+tEz~6J`Dc24IG)`;DF)2B z^v0J!t@GzCNHDGn`b3}sXth&p!=w7^my0a6eWy6Ec-7hXmxz>MG{0H}pB?+LmSlxD z2Q|m-+V#*4BWZc^E_4$@S}+-v$ApVPB2@au9QKuYnVrT3DAP+x&d`SHx(n=|#PPeb zQFsIu)t>}zhk2O^9Zn#AcPhVfV-z1a| z&jvh8c~5F~H#ty%Jp)Lc7h%W2o?Gfu;T$s`E8OjbCQ*+3fD!i*W4!2jvpgQLNzqN> zkJZjOKDn-#2C{l+`xPlkLz?l;*hME?-AuY*1L0qa8Atz`u6>*oEYzOgb1m@~98cuLV^*)#xv1s zronY)-Ln_B^i9R{q97;hC_!DLae;vJ)4sz)Q69<35`m_4xK z(0*)|YDI`dY78)U?hpNCub{z{S8^0tiOi?`P=43oOsiI+7gA?W7m0&XTIhBMY6@}=Fw0m)V(G@qzF{_a&>til zR@CuEKDOm1S%z2S9;B$pppPk`UTIeFMH&pT=s@gy={4AL!9NOxAFOKZ0_L{&`P!5L zl%lXor@&UCZn%xW*dXzRdr+_P1AdEXa4Jw05V^yHApOQROd+2*vZ|gV`kpmdR9Ak2 zqyB+*17Z&pxp8xm8hIC^Im)kw8F0(fPz3E(q3@X|kyNawmP*x(mCq82s#|mcAA^P; z@@2G2{~czsCbSmSL3S;gYp_aLxo@I&sJ_vix4?Grxc9R{@paMqar5y^67W%ljdd5~ zHE%{XL&Ygs6o15n;Op3S8^%))Lf>W-=_F$|AQi%x`2*>|%uD7Bx}imC7y=w+MZLB> zri%_!S3_FBu1nSNWq%CWE^oUb?XRKq7l(P}SFE?h*YQDB@JXxRsWFpMvX#R}NW;H5 zk0<51=U`!s8@0whMuut772GD7Oo8e8h9EyN^v8b?6RW(WVOAM+I|z>#|5 z{+hS_N5&al+TU&OGHn{5L41U#4?hCRfwS57jcD96(`v{IbT_NTuxA6mV&&?7^-*Z;d2o*mDGFHesow zAmYNK=ea#+BD@3#$!hNY=Cfjl?AKwPt9MZ^T;j;I04KSyCerGXZg6liijv`3JP^NQ zxQd-IG6%*P33|YR(D7_GTlL7(Gbe``AZW{HuQkjPzJZRWsQ}l%Amr>f7pDL7bMQBF z2w(LW!%UI?EJyl258?hY0NAmAI=>N?O^g(zI8lX(D}Np7XYo_M>u*~vQK<>GzH2u& zY-K;6n<&VroKWjgn&_XPw1pMCU0PC^=2#;!K}ltW3X((s9&TOb8~#1cpsko%kd048=nRySLQo z_&2O0Z%OM+321F}r^S!`-t17>lI@NGE#%Xp)Dl*sPF;8q+!7e!agomMP$(>qM_AGzb3h6BsYK1mWAVwNVhG8(o{8l&Cjc>_A#hw-4wSzD<&{R<*H(l6u5gHU0X`HjJ<*<9M zDSo_|nx%dz!L#9I+lSHvUR&!t<#hw}80eEub=}zVAdcT+T^k-`X&nO38)u(~%sI z9j&7I3y6vB6YO;id%BLg9`d(EV`dU1;G0ylIzv>zp}eynT1D%Y+aLCvF)%~j#s}0l zmQkWD>GsV!^z;4U*sLgkF}%@41B@X+;{a zARnvr_=@cF&Yi^%>1j7JIP;-!tH$20pNv0>qE(KF_crc*IoTeeVZ!*j$WPiMOE+I; z1&Xr7^>h`}Y^)!CzubqVG?Y$Cvps^Quxcvzqt)mejZ35dW5qL{XZ0L zvLuK1I-KcRggfZ+(#n9^J#2Ga;LK^{6m0|9KGh<~4e)i0KxO#$9MD#U4 ziqPAGsSp*Qh}~G*ENCmZy9__}u{iAdjpTG2{m>Zauk9bbc+koKSrqv_V`R8P7=*t* zPjT#BAREXaWjXM~eHS9`u5JRvzTGn%s#Y${9}nvTcJ;$okIT6gzxP z3kWTBP;Xf17R>PV1@-Dcw3gy%E6GswlH|3{YB)xPF}T^^%Uam|++q+lwAn)q*qpB| z5z3{mb^wUmA2=L2;KVj+wOvT`er7~e(Kbh5Vz5vr^kj%f9W>OgZ(jfgGC>_>Yl#n#- zrf2?#CLOehjAL$sFioNtJ1u-x9y#ulw6DBegBFK3aHHSnE1cv8%4UDvF_i~**i@u1 zpTP9E=V{1&K*+!34^(eZig(T7kC|@+!(kJ+@i}l5;6(9$kV1u)24`rgB3&PrRMBYI z+w`c+#iLgU;=VInFVe{rJsjX6am52C{k0dlPIX4FOAqsbsUundst+ZwGtB2tD}nFl zgx$}o#O4zbCtA~k6?b=Q0VE}dQsu}+L`S{@$9>Ed11}`AJ4;IMM3nBbf8?L9_ifGh z`tOZ*j*eK8Kr(G}_oZ{s5P4q<8s&!=5j;%pi6^~pear>R2mk*Ngkh@eAy*G-a7fmm zl11SHTJI(@2$Qjp8QX_$j>zwNL>HUT{S?ILp!0x$Fdg;eIKG z=rWYe6_l1EON7I{saWBvgE69S`0uc>e0s)p8Eg#Gv3s^CNrbMF7#r|vRi@bYW9?8> zcL$QuFD<|lR>pGsQt5@*1Q76KOqX~{_*3w!!w8b{jc7R>edaXhY)oj(hS-=5!54%V zXuBa*pso!1vV*fK{$PZ+Rf5^WBoGp9g~Ah}fU2sJ#)^OX17kwp|A5hZUE3GMsMOV+ zp^BRXIRGYOvTcY?e*(`fiB8@e!y{V~CvQk?bYXj=-nC`e*KYU7FSD#lEEP6ZzKldF zZS}1y_O%?%3Uzz5tPmE;W{fGml(mS>{_7148uoszpB#d;Y|yFXUwy*Mk$NWVy5;(o z8O7|7^pJhWW?9xnF%cDX@G&Ms2Dt6g&m~)ZGz?L^{QDJ+H|o7~UQ&APfYZF=YeN27 zhV2|J(x^+k5G+MB#Jar2t((qMF;cP(@Wlh4%+|tWWahlXq1zfU6z%1KX((E5-1fRU zb2Y=af*j1kp~KXZW}kOI4CMXvY!HpLCrlTOy$MR){La2~EO{M%@__hq9JR--$VM<=EgdulEu&*gc3Bkl@Z_YiE@F$;8A0m3 zat!Gg{RZ)$Q?!Fm19GoWdBYl`o#wk-gqCE;Qs=*UY-4T92%{sk_}^45|ON& zWfh}hJ6IuUtR>8P_j82uhT2t^*sak#Ot%SBH|n$p{u=>=dva}f@I(eSw~t*Hd5-sC z3?C9l8!I2h4}Rd?oi^B8J#?A#f5s1cnEZl3t$BEMMMB8XZ+;@=@E~4lEB;>rWD_rC_;NgI^Mq z0Unt#VW1}5{(?iw7tzRwU2GVs<>2`e!ylTm9aVnnIUK`3kNn( zI^>Ha=VqO8lVW#sh`QWiQx%UiCg`qJtn2ecSGr+Z;9=v(>e}`&sT7Qah{sN5 zZBQ-7kj;O{RJ1PL{H{W9V>%^7#zIiyFFcn)m;g75fG}^>DBr;_&@;e1!?q6d{N5Rv ztC1JbPcvy&?y*-^s&qaoruCJ8rQB&2TPlahfU|P z8e!Hby5x3DNpTvx5j>aRDZ?b5Y>_`(9O%0!FB$|z&|&y>UV$A+ECphqITl8r^nAZ( z{hjE@?EIAIK#S?>cTgx}s@mmlEh*rv1Um+rYS=M)Ax53n80WfS@Tp~H6FtlpIceB` zEL~Z>Uab>(1PwxhK@H3F#3|kEXf7TJdLVbwwmq!VL9+K3n0RrW{=4U)){A1WP8exT zpamPXmQ@7Btvhj?SXYbQME2TVeQQJ_iynx&R{8urV@zlft5UkMwUTrg1gZMXPdr4HmOuYx^u=GPW`i#hvu$`JQxZqbBOHQdDS+Efe0$-I@J5Wx-iTmH+CnK z>v+eT#ycg>BscDu_RBSy9<}bgb4tg%jG6&g16}>Bc@4WsGZeSX?Po?5+Hx-UjmS^? zQ==N9B#a2kHr9v58f%qeZQOgf*1Jcs;ef@f-lQsSg@_6yangpx5UDRjqEtvqYJ(NX zJOA>>;aU`|hr>k*7=w^rxQ*QhL+g^Z*|+vfZ7U}0l=mX-+nvmFapObWe=BCR!wBgd zxjj_>Fr7d<(&)ySRUXO)J8%z>`i0i;a&dmR+i)OyI;Gz@XP$C&EjeYN1a!l$Po^#~ zLfFqlF&ScI6(|j%K)4?ZIz#3C8W_XH(=hi*r^=~8Q46$g$3ROU;O(ds!TH9pZ{wq4 zHbmaXQpR`pUBO&yj9pLhafXS6=`(lLhGEiufEAc?#i5oOxo#bsz+1qz{4z4rsl1zx ziFS%Y%R=>+Yiht{V6H0=p^6B%hmZSfGkw8nqMJOeG60RsCe6e1uGrXXL*ls&Oe|cx zL&2YzMHj++l8NFgel<4Z;j)_k7JQFGKa?06_E^R~#R|39V_2c~9rBMeX;uvTS%rWN zC@Z)<+7c6BYwEYR3zcc48OmZfS?eWpBFgT z{aptSWqkd-$ugcwr%q1IT{n8a0gEMe7wfo^TcAWFY$7TRJEiF%b{pmVXNL#s)rB#i zheKEf?@t7g?we;`6kAUbo*_0~$PWk!U$0$EK}B9+Z**y0V76F6nl2$Ob>~^&8D6!i zQ!4@UjDtZ-P&mNMRkTV++y~#D?5&&%0(?UM-4iu?(E$NC^QMUE?I^_6|)mSI}b9q+QXQ`emKW1dTekVK?KsZAx2y{6=q_!zAd|+>Kee|j%p`cZRa`BrW(U@p)qOn>tz>I*B z4MhLCoP|7*unvB&oonV3OGVcoF%>6Ocr>!-pxG$+QDOTIhm*Y*^r>sjxfV4x*Z9zc zP|a8Sqm`ZF{f&lv!XH=(b&%@&3|8RbeYc4@4pc7t=wu}#5&1MbUnDfJ3_^cC(}Fz3 zFrC?T2@H>9K{MX}aA+9Rqgf>;?+vi{o>ju?1{1Pz?4~$tIX60E$O=@et26IHRH#pD zO)PgL#Xk-Aa600-2qtTUi}`dJb_ z8O=Ndb~z!!*X+cV2&^dlc~R-t^Cwp*=F#`kB~9;NB{M@w39px+rY4cjAJfs_qst}{V4@2?M zs03ef&L(&!t+E!TBtX5|$ZhtGNB;xNM%2F`*lE&gK?`&IguW8luJtMjW=~hXSgc8~ zj@&rf>13?7e+`D6V9gr;%6#lGFxi2$;Rd&C)^5yP4o1Mo0!)w*7u*JQm}<>fUwbu| z+3~~p*ol*ohsnp#xH1yaf(+_X^yx8w=wU=Vq!S3!o`tk=aO9tJ?--jAxLGxjQm3RZ z0GYxP)~2Gom(7f3^Hvj3ekh|6{97-=?1dmm*)@9A&gMaKsAiF$XI%dY%X7MfmItyym^uVDWl2sB?1? za3E&sR4xz+ql$`5O5=V&?kbByiUPl0S)YPDOn8l%0n5R|WYK@{FbR@OO2=>UQhmL? zkBbD8V@~2XuKzzn)3DCXBMZt(f|JdyeGTdeiV~UEbo$rsIT$EH=MP!Gqrgz`{sGLQ z8dkFB%fmPU%-?xuOu~l;=UvIX2bvsMHajo!o-R=n5L=vM2_HX+kYMmJJ^f@>d<`Mi!>YT-?V-6-J}D{(&-M! zqD%ITM`aPU_udG_p&G0)R=c+1Q}@`6U3xiX&@9)b+I0OiO?ZT~Gf)vvbz>j0-P#W1 zT##u28r>#!pqOJ3783x*Pd{L_!HT@!e<3SpAuE>3OD)H+mDzwh%WQ)5u$J?%B`6;F zuck;U-;G$~DlL@VVvpM#hd8j_$!@AdzJFYTVG0ZRj58JZ({7Evi7){zvh@dSpseUJ z8cA}D$mHrkq1+HN&a|!fP2j~Mk2ctfAWXb38&AU2>?i<3*Db|)T|khxdlGSePz zf!7@Rtbs2bT$3A->MMhq>cBG74KS9 zzAMt`PE>>GY#jE}$VFV1Y_r;=iqmp`2T0uB>aUe4FJp2H{A+}Q66Ks1~(d7p4o@kEh zbF`PMVLn|VfvPGf)I{^KOdAiT6{i)EN6C1EtgHKsF8}iJ<}X8a=R{xwneToHjr{9^ zB;Un%+RsI+aUG(b0soHoN=)xO#xubjg1#i#pjqun*!@v!lr(9;0{>|dhJtnms!A}{ zTz!`16n1g#yitf$JDK>)H@y3xo%|!8=S-Nbm~rzzXeWz({sCr=#XoNPUk8x}?uE5B zga$DhUgq%D!7p#8e%aB}mU6?yEq}z{<&~%2y#EAOe1c?9M1LTmI|NjByV};YH@@)& z`mcuxpj-=+p~UzbM$PA>N&@9hq6e~F6W%JnDK&}%MCKtRzc@JiC|=e*@V#x zsxTXdX*^iQo2U1f6)9ss(UF7%HvvKBvXWg=f@HBNY`+=l#ao(Ub9d9rcLQG+8Z{Tk zR_wO_xzFEqUs+vv;}}hGHrZDnjG^>&bYMD)!in`o7RDeZJO~yGwhNcZ5J%V5%nHV? z$jT1v|AP>pQ~Bv#8i|ON0vzV0VGmIdFTPm`FTzu^97oS_!5j$20OVod6+{`P71FBe zqt$0s6$<6Z75{tE{zG^(2c4gDK&GX>!77gD$hQ4hnsNYRe?XUyRD6w;aEiwz-)#s^ zK)!47$ylSE3^K>@=<(MyYiZ0{LLyMHFpA6s;lkj>tcIQQ%_L9_&+*a$laH2U%zy~y z?$1-|9H>b7xdLh4QHF_DF|+?g%|@duC+L@cupws{{^nF;)8}6n!0nX=y`qtl#m#f{vW#%aGjLnrz27B-7@EVoOTo+87;?J0Xl*V zN>=z@IRv>Es$m-+pd*5X4;zkV-ooPVWu=IGfJ{462xf= z<3DtVxzl<}BEtAxv#iO-`UD|{h-R3UnXORjqmkw~4ZpEAcPe%=2+yHbG@C!559o}K zsVoWs!}-erzq=6bEk`1az2l7WHX|C0tYJmJEdn-dW6qq8DcKcCHP0_Qi1AD|9H_C7 zqwC;$OiV?}Q_43>tYO1PWt1Y#5^`EDN=t3A2G(QBHFSwjKmaYhBfksWT#(_+8+>xk z5-;yUJIUjW$sLnNHaHBWX1)xYMh;n*LII8erx2;2j6o`_ z-HC~JD@wQJ@`Olc2Ck2&hKg+%Dx2qkay5AUMO#HUIoyU)AI5aHrAoHvhvWNCZu~Ge zZX+z9T)!H-b=747{8x{|O{g#+SU0%5?7ETl`KArQ3JoLsn^K$@4GLkkqcGQ|U)I*B z>Rn&&fyqr@mvw7*bA9n$bm66Q#%WKGk_BtQ=)XaiW9#Fm`Uh>?2%Dr{iT@Dw;dH?X zhpgr&>kEP>o;>khH#c|(nouh=cjxygorCFBvD;H6w=c*RIuMMe?1yBw!;{kc&z%be z`Q@D_2|L^fK7;k&z5@BrdnDGIW-d4hX9s($@a1Nn&sCf8$=}f%7I(Cv;nWuA!`6rM z#lJo$D865beJ3PW1P}!IeIXRv&{&3I-^040A*ijPsPX**Nk>Mld@9WwJfb!3B8 zC5o;4Em9WZHSa*)U>K2h=s9Kygv#H3Fij|5+h-N+ ziiR4|K_4+u(NT@uT1&&Xpzy%-*yy|uB+D+@aH?FJ=Yo`OFQ^zKFv)7BeFZ@T7B03p zE@445Saf7Omj}+DyarHAOEFn);}6D@cw!uUaQa!!&1%|UWv+A??pl|tifhDTZ zEY>Wy*YLh(;r4^(;g8Gx@?oqiu;^J5!5V`+Js=mt$4c&PBdag70LJo54NxxZrR@y= z%q-41XUZ&2a)to*u?b~=F#T;=h^kyKbi}}3$a06|?rly{pQeidxQfY9(^f)NrL@;P zMyiv1-vFK$(4jftfuFV`U#mn0vdfGNRKUmpo3FrD3-;yEDK!y?-^99oX!6Id0@(uE z<(GWiAn7h`*M!)wDP6e=EG_p##f0?)kNvPAjQ@2p{XcoIR9SiS-ySTW7I9H5v53>& ze)kN-RCKb53|AiPYs!vt`-;5y-81E{=LNg{J@!nWxjZg)dy!3mc8_XJ`najBzzf>0 zU`~#*AAz5rg?0)v z+DCQm)x2@bQSdE6+`x3-5-Cj!jgbBv5Y$1|>xfI{HRL_Mf19U~@LZYvZ~fy%Un!mp zVeC!n95{hN$m6$Z{u%=~mEwxK*bE2YAFx67%3L zC_Umw)NCJAeqPkzJixHwSd78}2Qmv-l!H=Y2ngwoQ89-|l>rZRQo#k0>JY~*qUyA_ z5_($J_SKTwsfInmcMw=l)Q2gBgQ)jWv&0L=!I6z%g~=fpOYN8y{kibvP0CDY!iFdA z!Wc0!Z-fyRFt?GK>!sWU0SL#`ld?-MSQn1DLM8+_&4w(_W67-Sd&Xfl!(n@ATL5|j zec4TJYWH6-$A2frR0iG_Z5_YJTi}(G_zF>4p+Ao#-v`fI$o~05J0SD(&u#d4bp!Hp z=k~fG=W}fF5k>L9dpcX%YXm3o7A*HnUv;^o56iK_Q!ba{N}*#Y3pnZWam&)dZl6t~ z!h7nLptoZuRFC_eKuQA8ZXrNAalHA5@g0%Z+D`n^?A@F{~cb1PN&}gLjBW^6?TRa2L{w3 z=qx?gC`M8Q48Qtdh7d{Vu%reM-_wRAU)KHb_o(1er#Zc(V$Gg7M{~!jjtiJQ?W1fTqXtMhz2^E`5g8hCT=@@{b8ipcBNo+PL1Tf zWiPHmVtn`0+dL7b)5HYE*oi}E_V6bDx0c`EEX3kqvxxlc?TV}oh->os8XADrPyzX z&TD#k!IHCQxHHKV+C8k@lD_0NXb}D-v9iG|;NiFttwSa!#+`Sp9twoBw2oB5cnhYJ zma^;fwk-G5`*5bd=V8q;HW|j2Q>@5IGK@e96-<9&m0cR8?rxNxy;q_-Y62{zq|bCh zN@!#t3a;2~s&U84!_s>Xrs>VXd3S%?Ne3DMK{5N=OfkX{5Zr|k_2cRndt{&}tS$9` zh>Xvx_j{eQi^DqPgfRM`orXuv5MY}L`Q}wukyUbx^ORNcvHiCQ9q9+8T}G%Sm>56Qu>0-S-h zR-!D_MsCLRWIry8S$PnRP`y4DUZ@bPJCCT)rBmh}-LdiH1bsJEV!ZkJVuMGR^meY3 z$F@F+5UiVgq?b&Lf|0_^nAUz)LF@1WC9A-0KrZ`O!J`+Y(LoX;{!jiheA~Nb1{@x< z1&w!$^^9h8IN`q17(9!{N+;MH;xOID-fGC>| zoIl%J=&k!gIWd;fQ4PeO9Gc%Y!mAB$l{9ubWq;V@-RB`DPI9PU*XBJ|W}wZMS-IUJ zDNz2T1?-W?91+woLKY21!3hVZECvgr-qHI*}u%C-tw>y2ieu3rw0w>Cm z;rQS?=8DsiXDG40ci1{PHQvq6=mp_VhG_s(B3}iiMVL*$RRMPBBHBBzoqm=CKVX<6 zgP|z6F1tE+5J+yEq4ZavAw4k>yk%xxfhI`7vBXm{;+4JW;FJmld2nldYdKsH5Ql?C zQq~%v_D8T6&cn0$mi;0G8?*(zDw?;&UbM~sHP~XI4-IYoS8ge@Z7J)$Lm3hj5c2)R zg=f~UwsIgfR(%KV7tpV-;eT_R(t1srSOV z7|9193O#P|UbXzEZk2Z>@qu;M5WW40tmbE)ys$HG18tx@0yDITBUcyLpRj-57mKihusMpTUlA9y42LFljqIlu09+TQv-PNq)SPk7ll9u9*ocj{Pat zHz&9H4cYp;ZWKF8+>!L@R_A}Z)d9rBriRG_j0LiDt%w~(AOqSQ@ag%FSH8+p)#xe> zwt`W3_96e4=4^U0d*1|X_}}t>+P_wcZM(fmMivbm8dEuXY%iMi0WX+S65 z!dBjSVTNSKfhhp#)kPR+$w*%~j7$Keqa^}e;@8rjV)63Vqt)nOgK3*u`rBRCddn0k zgYZUaJPAYlVpL$X-|6b+s7bfSBGJl_GHMx>*rry6pDpvZ|BImh<;H10X{)|F0;kO3 zA^tQoJP2_TQ!C`NA>od&BYPi9zksvNcj$BjG`A&tIj~8+C^Rp8N8hoX;EnS4%l(kp zlM%K1)<0E7(kHT*D*FNhqs(nis>AEB11-KL7F+lJGE#X9Hk0fbyJR-n)5;y=Y9;@L z&9)U!1D!_j65+)Cm4mKST^l?%U~Kkc`5~!q zIV=Td(|wa}G1$f(c-D4qY9}KhRrAXW-94FAVIzFt4LuDY=ltzcrv8if3^)m@rcei- zKdgYcwYG65935`J{6(C*R)_Qd)!UVTHFd39Ew!|YMB7?LA*u2d6`5705VT-X)2i12 zE)YQ!L_wJ%5HNuv3OJNhR04_wC(tqr0ipuZfB^#pK?owqWC#$ZFehQS`T&&9ny6>ZVk6_?tXZ$HRcQ=%Y(A zKMNgZ+6v2rCg=^0p+TWvX;I&pSqHG&1>4RR6coHzZ;>D5q1nsRLiDrJmP=g=BoiWl zW;%7kGepqBn+o=^YG{MiTcRXT=A{J*M#5FOz;Mqox+%(I_%5SrOJo_pe?^| zI~_PRKQat?zDTtgjp!5bWG8Uq_|otV%B#h787JMX!D?#bCn)9Lnn)_+mTYP-!nXC} zW@(A)V=p29qt$tAR95otIXhP8_@UC zP@E@^*iM*G!}MZBbW;$~2zP6t@Sp^3_oqh9pWQ5%MzO$pOdgOI3P`w-DWgfjI>m9c zB}CLZZ$b;epxxs}-nc_$&9P8<_kA0kI$+)oZ}ll#C3!d18>r|t=md^LSe?>qfVjL| zE1ES&Tklg3#9dE3KkBwA77UbK=d!DU(`&#qLi%Mi2Z*nUvB1&=UQCW(S`%deWV;=U zpc2c5_Cp*v^O(-B$|MPwU_=S_c!l-wxmnhFZ!?i2*S7WT8t6NU0tJ(!bHsLN-i!jdu0DY0h9chq4u^Di~v;-HtdPl$&O& z49q9s1;5jXC@q2<-mAvxyifye1aQnv1dWw|9h96BqL!X$FbA{Cl6sq(`ZU{H`S9>4 zL#{0ADJX-~6~NSz^ESK1FnjF2RTRv&^kuzdmi5$2HAf{E%~&~VU}W&e1yZER$_ zc(@n1#?Dk(vdO6afp>YNvqHPRFfhG$w8>-ihmY1i%T$%WWgVZw71?wilYoZT^MQ_< zl4Pgx*`c4ns_nfXNA)Rq+W|_80fyA> zh=qz!r^7=XAmDPpcNqZ4rh;T1v~Myf@3NA(5YCSDsQE4Yy3Yre&O^^+>v|mmGreEk zWDylPM_{`GC?i! zxVQPI;6!zTZsV#_;VLO4fK?nRP=`SU8<57S-*?TfMMs3cD0LE|)cZ!}{JswCz80l?8sxDIkIOT7+G@;1C| z2Bf&yama5venjeC3iMg9z8LIe1e@f+yhNQ)IPd?9m%t7Wz=(9A3vYlT@3+&X6&T3D z)7wK&K=+474&CiiBWA|e=DlKBzFOV(E3@7E8Am(^8&s#I*9z16F@N1fz&hPiD1v-2 zLHyIOz*kEuy78-%;H`vSEBL-qAe|lp&X9+CfkT>1aY>U64E6^ifxzYGoGR>_k|(CR zy0@YEqg7cobJGJR8S*NGy~lij_ZV1|HeCz~&%@x<7Vd1G(Dt}5x?YrHQ6F53>PGS)X7vxH+rtV(caQwZuxeEe&7m{>d9%1NDYYUoA<`~v$L}Rh7=IBV>u0+bnt~~y= z)Mf>SQttcU!F;p4h}6xATRzfI#($K%(>S5>f~KWx(xdS9U`)GNzWJ>7o)Ck3xrjz( zphoZMPiUVyS?jgD%(BF)zgt7NGHdfG>$ms)8_&cZG<}U2VsL|4auj(@1P+;l_V8=! zf6fhZz%nAS;a~5aRBX=3-XoSWlv#BjF25X0n+DpOnWGoK7NSrMP1AKBl9+heDEN9C-s11%L~-Zz7I7kuX*@>CGVZ>tNI zkTuSv1USNv63U@Fs_grt$p&pT>s?rM;W32Yy+d~Qz=6Cq`#kA7^QPl*l~V$% zd<AC=AUF<8 zNRGG93YH-JPK+-J^^bj^3O)Wc|1jKgf6Z@5<6a*IXM5`BjTc^li*~tmO-Au1UFcBz zVw1>v#Hzb`*f9AW(_UaTwk)tnau~8+kWAdAMPwxd4i zDDRX)Gjvu3U3Aqu#pVSg?STW!Q{D#oSHx#8n&~bQrWTW$mqD3^*CH+U1>Ey$2MQG2UHr zDADhV?4&^7=R=%YJfhTN>t+6c*g^LuUSxB+?YjrmkS%wlp)3jLnLqno{c0h^2DWTdORCnXv#z4jFx2H!5 zfvff;RK1!QE6yMB&kg5)OTN+MvnZ$vh6Gh3pK!-RoPw^qoTDT1Y5uvpS+eiy^ zM*COp{&}gn80as+Ky=Kn{f^JOZY~MN4j4iNV#Vt<`a{P^SZE`Ru)|QMkpsq&e!aB6iG%5gfFJp> zL=Lr;Y1|w+4TM9d;6evd!>nNSi!yg^A-0BE)O6uknKd^^+M zkIJGYIcg$X=6{9r=~?iVG$`JyGn7K~6r5L-YsWYkFg!wQZXi&Ij5yV#DKCtsQ@e-T@!p zMioW4$7v>ur{EVxVd|lMovLX4+orSOcT{~c+G*o=!!13Sgb4ojDa^+RaXZyr>fsA}iVXtt+g=QF6EuNw!(%D+=uJP=pR@H?Gn9^moclAFEg3aDOphLC92 zRdanEo}G+syVU$-Jdrd3dRx?YZH`#{(SN^4`=$E{msDklHL+q>OXRs12a21Nl{IZr@>UyR`oe+bbT;r=pb2Txu-+{jjQVGcRNUphm~7z>8Kvw- z>z-3ZDtQu8enS5j_8^P;&mygP|S!9M<;_&49^(5+~pSDUw<#^(F3`{p% zz|_+_m_-bHbprzB_Ex%FhqpZ#1mJFO<7O1k6Ik*O$Y{lDa-(ltCi0PlR>08Ee9PW6 zG48bm!ObYJPcjUi?f@aR=e8CjGUHFzLrKfbiuUOoRLI^G5|AYtBGO)t&`hmlRHY)R z25!T8P6wjXvU*PqVdivdsEd;f3vpYL$~Mf#=+?ksh;z0ozTWzl*2sLO*L&6=fyO3v zh@n%Fo9dk2^foUIsw?(WD4)}tF4r+;d6oqFD^gHT=?sIe#UOSN8(yWQ1M^_*=YTB1 zoJpA(GaCPj^KZ&uL=k>EVBKI$nC>74{!0gWj$$^;!l^wstE7`llCg^x&(gciR*cl# zQOaO`@HBcV$jt&gvdT_gb{(G)Oy~>auF0Xch~s0aEs6ouWe1^QorqeH8DOo&hvQM z5JDcwZU%_Z%_fhXP37=&GzG<9lL?N6&GdzUO@CH?@lO$g-`WgzN*wNkk?;y5k~iSz zNvX;YN}~)3sq$6Kv_nL2sd|hnI>#OaYC=P7sjb>S=MBPd(?8A|po|10CanfTj5O}S zq{X)78{=O80-P)j9rgg!J#T0jZYapOl$xG$*vd9{uMBpr*jp0EOS#U=GW*x&5;bK6 zTgtP!&V>oOtMhMhzXRVntFN1&&Bp=CeCiH2q!fYA<>C5Q^L=wSG>2kS)-(V*dC2`Q9%CEm1N+>qt zhZ?*d*rWo^qS#f`EorAk-60wYnLOu0z_~VDsR;<5CJxa>TaUkDKgoAV&0?P%!puz4 zMibc^PjIpRLfouZ)hAvCYTkT#e#|X$@An8n?O)lg1xfg|$d>dA<<)?}RO=!k)*fzL z(Z3{8!E=4c2kF~X0@4O&BtR4T-1IF3upr|^KwvG4XqRyvVjR+OnH9)Yjx(Lg?MlNz z-Ug>{-01}5Fr1N$)dp2!a9ZrGS5Qm1p+9*q&I8@X6^ax0dY=Qim=H3eT! z8p@J@p7}Gv-gw`wr`X0&HfMk?8IZ1DN~y*d=Gk?Hl-P}=Rr{rk)4_+GEpD9}ofy*z zh$JIX%kMqV*<1AyG;TsL979SJt|_gv^(kVqu0eR2JA?)N!t|T*6O#d=34Bh28-po+ z>tS>|e3)l3k;~zG<0u5j^A>2`mOk7C!avk0{wa<~We)O?psy_Qz}a@tH}_g5x-Zf0 zz^Hr+q?6?Uz{HBhR@OWy(a4ia?#GLG5F2Bn_f>&|JPGZ)&4m3@9!fq+*qwegeoGO8 z(MthU1ElVuSI}R8G?Hb8pbfZz)NwZZ$a{q1^Fti~vjI%^*45ob)?&k>_F6ussuNE_ zY3!4UBj62wg9bT}i1h0@Q4dwTqzv}cr!#aIKyFZjZofQF`=#W@ym?==?=s(h@KZ4{ z9UYQKwl(m;%~Uzz_cbW$9OI6AE)$90DW6NS=!tGbWAg!HGMc401w)p@|AZM{Sx%ZB&N#J=q@A#m3=j%g=8KN zb{OY`Wwuuv?v2u!tP&3al2ib5QG9xM3y4Vep6|lUA zb~Ql!oRgG*4rI016^{U&Z59POIR+)2FtkthY}@c<@XRa+y!)-MvLTd=?7zaR5hx&W z*?}}Rl@KOBTaVNqa~T~Y@Na&d9zoK!L(?f)s>u-oVl9_|4I0 z-45$oDaf>s0>C`yLRj`Px Date: Tue, 25 Jun 2024 02:30:26 +0800 Subject: [PATCH 02/64] 'update' --- __init__.py | 13 ++++++- configs/kohya_ss_lora/controlnet_sd1_5.json | 28 ++++++++++++++ hook_kohya_ss_run.py | 42 +++++++++++++++++++++ mz_train_tools_core.py | 32 +++++++++++++--- 4 files changed, 107 insertions(+), 8 deletions(-) create mode 100644 configs/kohya_ss_lora/controlnet_sd1_5.json diff --git a/__init__.py b/__init__.py index c47b053..471ff52 100644 --- a/__init__.py +++ b/__init__.py @@ -51,7 +51,6 @@ NODE_DISPLAY_NAME_MAPPINGS["MZ_KohyaSSInitWorkspace"] = f"{AUTHOR_NAME} - KohyaS class MZ_ImagesCopyWorkspace: - @classmethod def INPUT_TYPES(s): return { @@ -62,12 +61,17 @@ class MZ_ImagesCopyWorkspace: "resolution": ("INT", {"default": 512}), "num_repeats": ("INT", {"default": 1}), "batch_size": ("INT", {"default": 1}), - # "class_name": ("STRING", {"default": "girl", "dynamicPrompts": True}), "force_clear": (["enable", "disable"], {"default": "disable"}), "force_clear_only_images": (["enable", "disable"], {"default": "disable"}), "same_caption_generate": (["enable", "disable"], {"default": "disable"}), "same_caption": ("STRING", {"default": "", "dynamicPrompts": True, "multiline": True}), + "description": ("STRING", {"default": """ +如果训练类型是controlnet,必须传入预处理后的图片(conditioning_images) +""", "multiline": True}), }, + "optional": { + "conditioning_images": ("IMAGE",), + } } RETURN_TYPES = (f"STRING",) @@ -323,3 +327,8 @@ class MZ_LoadImagesFromDirectoryPath: NODE_CLASS_MAPPINGS["MZ_LoadImagesFromDirectoryPath"] = MZ_LoadImagesFromDirectoryPath NODE_DISPLAY_NAME_MAPPINGS[ "MZ_LoadImagesFromDirectoryPath"] = f"{AUTHOR_NAME} - LoadImagesFromDirectoryPath" + + +# 别名 +NODE_CLASS_MAPPINGS["MZ_DatasetConfig"] = MZ_ImagesCopyWorkspace +NODE_DISPLAY_NAME_MAPPINGS["MZ_DatasetConfig"] = f"{AUTHOR_NAME} - DatasetConfig" diff --git a/configs/kohya_ss_lora/controlnet_sd1_5.json b/configs/kohya_ss_lora/controlnet_sd1_5.json new file mode 100644 index 0000000..04ad320 --- /dev/null +++ b/configs/kohya_ss_lora/controlnet_sd1_5.json @@ -0,0 +1,28 @@ +{ + "metadata": { + "train_type": "controlnet_sd1_5" + }, + "train_config": { + "pretrained_model_name_or_path": "", + "max_train_steps": "4500", + "xformers": true, + "sdpa": false, + "fp8_base": false, + "mixed_precision": "fp16", + "cache_latents": true, + "cache_latents_to_disk": true, + "learning_rate": "1e-5", + "lr_scheduler": "cosine_with_restarts", + "optimizer_type": "AdamW", + "save_every_n_epochs": "20", + "shuffle_caption": true, + "lr_warmup_steps": "0", + "save_precision": "fp16", + "lr_scheduler_num_cycles": "1", + "persistent_data_loader_workers": true, + "noise_offset": "0.1", + "output_dir": "", + "output_name": "", + "lowram": false + } +} diff --git a/hook_kohya_ss_run.py b/hook_kohya_ss_run.py index d0488e6..0454550 100644 --- a/hook_kohya_ss_run.py +++ b/hook_kohya_ss_run.py @@ -161,9 +161,51 @@ def run_lora_sdxl(): trainer.train(train_args) +from types import SimpleNamespace + + +class SimpleNamespaceCNWarrper(SimpleNamespace): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.__dict__.update(kwargs) # or self.__dict__ = kwargs + self.__dict__["mid_block_type"] = "UNetMidBlock2DCrossAttn" + self.__iter__ = lambda: iter(kwargs.keys()) + # is not iterable + + def __iter__(self): + return iter(self.__dict__.keys()) + # object has no attribute 'num_attention_heads' + + def __getattr__(self, name): + return self.__dict__.get(name, None) + + +def run_controlnet_sd1_5(): + import types + types.SimpleNamespace = SimpleNamespaceCNWarrper + hook_kohya_ss_utils.hook_kohya_ss() + # 覆盖sample_images生成函数,包括进度条和生成图片功能 + import train_controlnet + + # 配置对应的pipeline + import library.train_util + global sample_images_pipe_class + sample_images_pipe_class = library.train_util.StableDiffusionLongPromptWeightingPipeline + + train_config = json.loads(train_config_json) + train_args = config2args(train_controlnet.setup_parser(), train_config) + + LOG({ + "type": "start_train", + }) + + train_controlnet.train(train_args) + + func_map = { "run_lora_sd1_5": run_lora_sd1_5, "run_lora_sdxl": run_lora_sdxl, + "run_controlnet_sd1_5": run_controlnet_sd1_5, } diff --git a/mz_train_tools_core.py b/mz_train_tools_core.py index 89bd55d..dd6d7c5 100644 --- a/mz_train_tools_core.py +++ b/mz_train_tools_core.py @@ -93,6 +93,12 @@ def MZ_ImageSelecter_call(args={}): images = args.get("images") pil_images = Utils.tensors2pil_list(images) + conditioning_images = args.get("conditioning_images", None) + conditioning_pil_images = None + if conditioning_images is not None: + conditioning_pil_images = Utils.tensors2pil_list( + conditioning_images) + resolution = args.get("resolution", 512) workspace_config = args.get("workspace_config", {}) @@ -106,6 +112,9 @@ def MZ_ImageSelecter_call(args={}): train_images_dir = os.path.join(workspace_dir, "train_images") os.makedirs(train_images_dir, exist_ok=True) + conditioning_images_dir = os.path.join( + workspace_dir, "conditioning_images") + force_clear = args.get("force_clear") == "enable" force_clear_only_images = args.get("force_clear_only_images") == "enable" if force_clear: @@ -120,10 +129,16 @@ def MZ_ImageSelecter_call(args={}): saved_images_path = [] for i, pil_image in enumerate(pil_images): pil_image = Utils.resize_max(pil_image, resolution, resolution) + width, height = pil_image.size filename = hashlib.md5(pil_image.tobytes()).hexdigest() + ".png" pil_image.save(os.path.join(train_images_dir, filename)) saved_images_path.append(filename) + if conditioning_pil_images is not None: + os.makedirs(conditioning_images_dir, exist_ok=True) + conditioning_pil_images[i].resize((width, height)).save( + os.path.join(conditioning_images_dir, filename)) + same_caption_generate = args.get("same_caption_generate") == "enable" if same_caption_generate: same_caption = args.get("same_caption").strip() @@ -136,12 +151,17 @@ def MZ_ImageSelecter_call(args={}): f.write(same_caption) dataset_config_path = os.path.join(workspace_dir, "dataset.toml") + + if conditioning_images is None: + conditioning_images_dir = None + generate_toml_config( dataset_config_path, enable_bucket=args.get("enable_bucket") == "enable", resolution=args.get("resolution"), batch_size=args.get("batch_size"), image_dir=train_images_dir, + conditioning_data_dir=conditioning_images_dir, caption_extension=".caption", num_repeats=args.get("num_repeats"), ) @@ -290,7 +310,7 @@ def check_install(): import logging -def generate_toml_config(output_path, enable_bucket=True, resolution=512, batch_size=1, image_dir=None, caption_extension=".caption", num_repeats=10, ): +def generate_toml_config(output_path, enable_bucket=True, resolution=512, batch_size=1, image_dir=None, conditioning_data_dir=None, caption_extension=".caption", num_repeats=10, ): check_install() import toml config = { @@ -304,6 +324,7 @@ def generate_toml_config(output_path, enable_bucket=True, resolution=512, batch_ 'subsets': [ { 'image_dir': image_dir, + 'conditioning_data_dir': conditioning_data_dir, 'caption_extension': caption_extension, 'num_repeats': num_repeats, }, @@ -374,11 +395,8 @@ def run_hook_kohya_ss_run_file(kohya_ss_tool_dir, train_config, trainer_func, ot if trainer_func.find("sdxl") != -1: taesd_type = "sdxl" - - - pb = Utils.progress_bar(train_config.get("max_train_steps"), taesd_type) - + import traceback import comfy.model_management @@ -401,7 +419,6 @@ def run_hook_kohya_ss_run_file(kohya_ss_tool_dir, train_config, trainer_func, ot max_side = max(xy_img.width, xy_img.height) # print(f"global_step: {global_step}, max_train_steps: {max_train_steps}") - total_steps = resp.get("total_steps") pb.update( int(global_step), int(total_steps), ("JPEG", xy_img, max_side)) @@ -517,6 +534,9 @@ def MZ_KohyaSSTrain_call(args={}): elif train_type == "lora_sdxl": run_hook_kohya_ss_run_file( kohya_ss_tool_dir, train_config, "run_lora_sdxl", other_config) + elif train_type == "controlnet_sd1_5": + run_hook_kohya_ss_run_file( + kohya_ss_tool_dir, train_config, "run_controlnet_sd1_5", other_config) else: raise Exception( f"暂时不支持的训练类型: {train_type}") From 536bc3a64bd6fe72c37a10becaf5e0f8ca984391 Mon Sep 17 00:00:00 2001 From: wailovet Date: Tue, 25 Jun 2024 13:20:49 +0800 Subject: [PATCH 03/64] 'update' --- __init__.py | 59 +++++++++++++++++++++++++++++++++++++++--- hook_kohya_ss_run.py | 33 ++++++++++++----------- hook_kohya_ss_utils.py | 53 +++++++++++++++++++++++++++++++++++-- mz_train_tools_core.py | 47 ++++++++++++++++++++++++++++----- 4 files changed, 163 insertions(+), 29 deletions(-) diff --git a/__init__.py b/__init__.py index 471ff52..289db33 100644 --- a/__init__.py +++ b/__init__.py @@ -221,7 +221,7 @@ class AlwaysEqualProxy(str): return False -class MZ_KohyaSSTrain: +class MZ_KohyaSSLoraTrain: @classmethod def INPUT_TYPES(s): @@ -282,8 +282,61 @@ class MZ_KohyaSSTrain: return mz_train_tools_core.MZ_KohyaSSTrain_call(kwargs) -NODE_CLASS_MAPPINGS["MZ_KohyaSSTrain"] = MZ_KohyaSSTrain -NODE_DISPLAY_NAME_MAPPINGS["MZ_KohyaSSTrain"] = f"{AUTHOR_NAME} - KohyaSSTrain" +NODE_CLASS_MAPPINGS["MZ_KohyaSSTrain"] = MZ_KohyaSSLoraTrain +NODE_DISPLAY_NAME_MAPPINGS[ + "MZ_KohyaSSTrain"] = f"{AUTHOR_NAME} - KohyaSSTrain(old version)" + +NODE_CLASS_MAPPINGS["MZ_KohyaSSLoraTrain"] = MZ_KohyaSSLoraTrain +NODE_DISPLAY_NAME_MAPPINGS[ + "MZ_KohyaSSLoraTrain"] = f"{AUTHOR_NAME} - KohyaSSTrain(lora)" + +class MZ_KohyaSSControlnetTrain: + + @classmethod + def INPUT_TYPES(s): + models = [ + "latest", + "empty", + ] + + comfyui_full_m_path = [] + comfyui_basemodels = folder_paths.get_filename_list("controlnet") + for b_model in comfyui_basemodels: + m_path = folder_paths.get_full_path("controlnet", b_model) + comfyui_full_m_path.append(m_path) + + # 按创建时间排序 + comfyui_full_m_path = sorted( + comfyui_full_m_path, key=lambda x: os.path.getctime(x), reverse=True) + + models = models + comfyui_full_m_path + + return { + "required": { + "train_config": ("MZ_TT_SS_TrainConfig",), + "base_controlnet": (models, {"default": "latest"}), + "sample_generate": (["enable", "disable"], {"default": "enable"}), + "sample_prompt": ("STRING", {"default:": "", "dynamicPrompts": True, "multiline": True}), + }, + "optional": { + "has_no_effect": (AlwaysEqualProxy("*"),), + }, + } + + RETURN_TYPES = () + RETURN_NAMES = () + OUTPUT_NODE = True + FUNCTION = "start" + CATEGORY = CATEGORY_NAME + + def start(self, **kwargs): + importlib.reload(mz_train_tools_core) + return mz_train_tools_core.MZ_KohyaSSTrain_call(kwargs) + + +NODE_CLASS_MAPPINGS["MZ_KohyaSSControlnetTrain"] = MZ_KohyaSSControlnetTrain +NODE_DISPLAY_NAME_MAPPINGS[ + "MZ_KohyaSSControlnetTrain"] = f"{AUTHOR_NAME} - KohyaSSTrain(controlnet)" class MZ_LoadImagesFromDirectoryPath: diff --git a/hook_kohya_ss_run.py b/hook_kohya_ss_run.py index 0454550..73b821d 100644 --- a/hook_kohya_ss_run.py +++ b/hook_kohya_ss_run.py @@ -43,6 +43,10 @@ train_config_json = "{}" sample_images_pipe_class = None +def utils_sample_images(*args, **kwargs): + return sample_images(None, *args, **kwargs) + + def sample_images(self, *args, **kwargs): # accelerator, args, epoch, global_step, device, vae, tokenizer, text_encoder, unet accelerator = args[0] @@ -58,27 +62,15 @@ def sample_images(self, *args, **kwargs): # print(f"sample_images: args = {args}") # print(f"sample_images: kwargs = {kwargs}") - # last_noise_pred = hook_kohya_ss_utils.running_info.get( - # "last_noise_pred", None) - - # noise_pred_latent_path = None - # if last_noise_pred is not None: - # noise_pred_latent_path = os.path.join( - # tempfile.gettempdir(), f"sample_images_{global_step % 5}.latent") - - # output = {} - # output["latent_tensor"] = last_noise_pred - # output["latent_format_version_0"] = torch.tensor([]) - # safetensors.torch.save_file(output, noise_pred_latent_path) + controlnet = kwargs.get("controlnet", None) if epoch is not None and cmd_args.save_every_n_epochs is not None and epoch % cmd_args.save_every_n_epochs == 0: prompt_dict_list = other_config.get("prompt_dict_list", []) if len(prompt_dict_list) == 0: seed = other_config.get("seed", 0) - - prompt_dict_list.append({ - "controlnet_image": None, + prompt_dict = { + "controlnet_image": other_config.get("controlnet_image", None), "prompt": other_config.get("sample_prompt", ""), "seed": seed, "negative_prompt": "", @@ -86,7 +78,9 @@ def sample_images(self, *args, **kwargs): "sample_sampler": "euler_a", "sample_steps": 20, "scale": 5.0, - }) + } + + prompt_dict_list.append(prompt_dict) else: for i, prompt_dict in enumerate(prompt_dict_list): if prompt_dict.get("controlnet_image", None) is None: @@ -98,6 +92,7 @@ def sample_images(self, *args, **kwargs): if prompt_dict.get("enum", None) is None: prompt_dict["enum"] = i hook_kohya_ss_utils.generate_image( + pipe_class=sample_images_pipe_class, cmd_args=cmd_args, accelerator=accelerator, epoch=epoch, @@ -105,8 +100,8 @@ def sample_images(self, *args, **kwargs): tokenizer=tokenizer, unet=unet, vae=vae, - pipe_class=sample_images_pipe_class, prompt_dict_list=prompt_dict_list, + controlnet=controlnet, ) LOG({ @@ -169,6 +164,7 @@ class SimpleNamespaceCNWarrper(SimpleNamespace): super().__init__(*args, **kwargs) self.__dict__.update(kwargs) # or self.__dict__ = kwargs self.__dict__["mid_block_type"] = "UNetMidBlock2DCrossAttn" + self.__dict__["_diffusers_version"] = "0.6.0" self.__iter__ = lambda: iter(kwargs.keys()) # is not iterable @@ -185,10 +181,13 @@ def run_controlnet_sd1_5(): types.SimpleNamespace = SimpleNamespaceCNWarrper hook_kohya_ss_utils.hook_kohya_ss() # 覆盖sample_images生成函数,包括进度条和生成图片功能 + import train_controlnet # 配置对应的pipeline import library.train_util + library.train_util.sample_images = utils_sample_images + global sample_images_pipe_class sample_images_pipe_class = library.train_util.StableDiffusionLongPromptWeightingPipeline diff --git a/hook_kohya_ss_utils.py b/hook_kohya_ss_utils.py index e1f6031..f347a1f 100644 --- a/hook_kohya_ss_utils.py +++ b/hook_kohya_ss_utils.py @@ -216,6 +216,10 @@ def hook_kohya_ss(): import library.utils import library.train_util import library.sdxl_train_util + import library.model_util + + library.model_util.convert_controlnet_state_dict_to_diffusers = convert_controlnet_state_dict_to_diffusers + library.utils.setup_logging = setup_logging library.train_util.load_tokenizer = load_tokenizers @@ -350,7 +354,7 @@ def _load_target_model(args: argparse.Namespace, weight_dtype, device="cpu", une return text_encoder, vae, unet, load_stable_diffusion_format -def generate_image(pipe_class, cmd_args, accelerator, vae, tokenizer, text_encoder, unet, epoch, prompt_dict_list): +def generate_image(pipe_class, cmd_args, accelerator, vae, tokenizer, text_encoder, unet, epoch, prompt_dict_list, controlnet=None): import library.train_util # for multi gpu distributed inference. this is a singleton, so it's safe to use it here @@ -369,6 +373,10 @@ def generate_image(pipe_class, cmd_args, accelerator, vae, tokenizer, text_encod v_parameterization=cmd_args.v_parameterization, ) + # print("unet.config._diffusers_version:", unet.config._diffusers_version) + # print("unet.config:", unet.config) + # input("press enter to continue...") + pipeline = pipe_class( text_encoder=text_encoder, vae=vae, @@ -394,7 +402,6 @@ def generate_image(pipe_class, cmd_args, accelerator, vae, tokenizer, text_encod save_dir = sample_images_path prompt_replacement = None steps = 0 - controlnet = None # save random state to restore later rng_state = torch.get_rng_state() @@ -406,6 +413,9 @@ def generate_image(pipe_class, cmd_args, accelerator, vae, tokenizer, text_encod with torch.no_grad(): for prompt_dict in prompt_dict_list: + print("prompt_dict:", json.dumps( + prompt_dict, indent=4, ensure_ascii=False)) + library.train_util.sample_image_inference( accelerator, cmd_args, pipeline, save_dir, prompt_dict, epoch, steps, prompt_replacement, controlnet=controlnet ) @@ -417,3 +427,42 @@ def generate_image(pipe_class, cmd_args, accelerator, vae, tokenizer, text_encod if cuda_rng_state is not None: torch.cuda.set_rng_state(cuda_rng_state) vae.to(org_vae_device) + + +def convert_controlnet_state_dict_to_diffusers(controlnet_state_dict): + import library.model_util as model_util + unet_conversion_map, unet_conversion_map_resnet, unet_conversion_map_layer = model_util.controlnet_conversion_map() + + mapping = {k: k for k in controlnet_state_dict.keys()} + + # print("controlnet_state_dict:", controlnet_state_dict.keys()) + keys_first = list(controlnet_state_dict.keys())[0] + + if keys_first.startswith("control_model."): + new_mapping = {} + for k, v in mapping.items(): + new_mapping[k.replace("control_model.", "")] = v.replace( + "control_model.", "") + mapping = new_mapping + + for sd_name, diffusers_name in unet_conversion_map: + mapping[sd_name] = diffusers_name + for k, v in mapping.items(): + for sd_part, diffusers_part in unet_conversion_map_layer: + v = v.replace(sd_part, diffusers_part) + mapping[k] = v + for k, v in mapping.items(): + if "resnets" in v: + for sd_part, diffusers_part in unet_conversion_map_resnet: + v = v.replace(sd_part, diffusers_part) + mapping[k] = v + + if keys_first.startswith("control_model."): + new_state_dict = { + v: controlnet_state_dict[ + f"control_model.{k}"] for k, v in mapping.items()} + else: + new_state_dict = { + v: controlnet_state_dict[k] for k, v in mapping.items()} + + return new_state_dict diff --git a/mz_train_tools_core.py b/mz_train_tools_core.py index dd6d7c5..96b3cb5 100644 --- a/mz_train_tools_core.py +++ b/mz_train_tools_core.py @@ -511,14 +511,38 @@ def MZ_KohyaSSTrain_call(args={}): if "network_dropout" in train_config: del train_config["network_dropout"] + + + + base_controlnet = args.get("base_controlnet", "empty") + if base_controlnet == "empty": + pass + elif base_controlnet == "latest": + workspace_controlnet_dir = os.path.join(workspace_dir, "output") + if os.path.exists(workspace_controlnet_dir): + workspace_controlnet_files = os.listdir(workspace_controlnet_dir) + workspace_controlnet_files = list( + filter(lambda x: x.endswith(".safetensors"), workspace_controlnet_files)) + workspace_controlnet_files = list( + map(lambda x: os.path.join(workspace_controlnet_dir, x), workspace_controlnet_files)) + # 排序 + workspace_controlnet_files = sorted( + workspace_controlnet_files, key=lambda x: os.path.getctime(x), reverse=True) + if len(workspace_controlnet_files) > 0: + base_controlnet = os.path.join( + workspace_controlnet_dir, workspace_controlnet_files[0]) + else: + base_controlnet = "empty" + else: + pass + + if base_controlnet != "empty" and os.path.exists(base_controlnet): + train_config["controlnet_model_name_or_path"] = base_controlnet + + train_type = config.get("metadata").get("train_type") - # 在ComfyUI中无法运行, 梯度有问题, 原因不清楚 - # import train_network - # train_network.logger = logging.getLogger() - # trainer = train_network.NetworkTrainer() - # train_args = config2args(train_network.setup_parser(), train_config) - # trainer.train(train_args) - # return + + sample_generate = args.get("sample_generate", "enable") sample_prompt = args.get("sample_prompt", "") if sample_generate == "enable": @@ -535,6 +559,15 @@ def MZ_KohyaSSTrain_call(args={}): run_hook_kohya_ss_run_file( kohya_ss_tool_dir, train_config, "run_lora_sdxl", other_config) elif train_type == "controlnet_sd1_5": + + conditioning_images_dir = os.path.join( + workspace_dir, "conditioning_images") + conditioning_images_onec = "" + if os.path.exists(conditioning_images_dir): + conditioning_images_onec = os.listdir(conditioning_images_dir)[0] + other_config["controlnet_image"] = os.path.join( + conditioning_images_dir, conditioning_images_onec) + run_hook_kohya_ss_run_file( kohya_ss_tool_dir, train_config, "run_controlnet_sd1_5", other_config) else: From 740e059dd91d2392dcb5fce237478e6539786cd8 Mon Sep 17 00:00:00 2001 From: wailovet Date: Thu, 27 Jun 2024 13:35:39 +0800 Subject: [PATCH 04/64] 'update' --- .gitignore | 1 + __init__.py | 244 +++++++++++- hook_HYDiT_idk_run.py | 155 ++++++++ hook_HYDiT_main.py | 710 +++++++++++++++++++++++++++++++++++ hook_HYDiT_run.py | 146 +++++++ mz_train_tools_core_HYDiT.py | 394 +++++++++++++++++++ mz_train_tools_utils.py | 15 + 7 files changed, 1657 insertions(+), 8 deletions(-) create mode 100644 hook_HYDiT_idk_run.py create mode 100644 hook_HYDiT_main.py create mode 100644 hook_HYDiT_run.py create mode 100644 mz_train_tools_core_HYDiT.py diff --git a/.gitignore b/.gitignore index 5130811..03b2b0f 100644 --- a/.gitignore +++ b/.gitignore @@ -171,3 +171,4 @@ exclude.txt *.msgpack *.onnx_data configs/models_config/**/*.png +HunyuanDiT \ No newline at end of file diff --git a/__init__.py b/__init__.py index 289db33..189e9df 100644 --- a/__init__.py +++ b/__init__.py @@ -50,7 +50,7 @@ NODE_CLASS_MAPPINGS["MZ_KohyaSSInitWorkspace"] = MZ_KohyaSSInitWorkspace NODE_DISPLAY_NAME_MAPPINGS["MZ_KohyaSSInitWorkspace"] = f"{AUTHOR_NAME} - KohyaSSInitWorkspace" -class MZ_ImagesCopyWorkspace: +class MZ_KohyaSSDatasetConfig: @classmethod def INPUT_TYPES(s): return { @@ -65,9 +65,6 @@ class MZ_ImagesCopyWorkspace: "force_clear_only_images": (["enable", "disable"], {"default": "disable"}), "same_caption_generate": (["enable", "disable"], {"default": "disable"}), "same_caption": ("STRING", {"default": "", "dynamicPrompts": True, "multiline": True}), - "description": ("STRING", {"default": """ -如果训练类型是controlnet,必须传入预处理后的图片(conditioning_images) -""", "multiline": True}), }, "optional": { "conditioning_images": ("IMAGE",), @@ -78,6 +75,9 @@ class MZ_ImagesCopyWorkspace: RETURN_NAMES = ("workspace_images_dir",) # OUTPUT_NODE = True + MZ_DESC = """ +如果训练类型是controlnet,必须传入预处理后的图片(conditioning_images) +""" FUNCTION = "start" CATEGORY = CATEGORY_NAME @@ -87,9 +87,13 @@ class MZ_ImagesCopyWorkspace: return mz_train_tools_core.MZ_ImageSelecter_call(kwargs) -NODE_CLASS_MAPPINGS["MZ_ImagesCopyWorkspace"] = MZ_ImagesCopyWorkspace +NODE_CLASS_MAPPINGS["MZ_ImagesCopyWorkspace"] = MZ_KohyaSSDatasetConfig NODE_DISPLAY_NAME_MAPPINGS["MZ_ImagesCopyWorkspace"] = f"{AUTHOR_NAME} - ImagesCopyWorkspace" +# 别名 +NODE_CLASS_MAPPINGS["MZ_KohyaSSDatasetConfig"] = MZ_KohyaSSDatasetConfig +NODE_DISPLAY_NAME_MAPPINGS["MZ_KohyaSSDatasetConfig"] = f"{AUTHOR_NAME} - KohyaSSDatasetConfig" + class MZ_KohyaSSUseConfig: train_config_template_dir = os.path.join( @@ -290,6 +294,7 @@ NODE_CLASS_MAPPINGS["MZ_KohyaSSLoraTrain"] = MZ_KohyaSSLoraTrain NODE_DISPLAY_NAME_MAPPINGS[ "MZ_KohyaSSLoraTrain"] = f"{AUTHOR_NAME} - KohyaSSTrain(lora)" + class MZ_KohyaSSControlnetTrain: @classmethod @@ -382,6 +387,229 @@ NODE_DISPLAY_NAME_MAPPINGS[ "MZ_LoadImagesFromDirectoryPath"] = f"{AUTHOR_NAME} - LoadImagesFromDirectoryPath" -# 别名 -NODE_CLASS_MAPPINGS["MZ_DatasetConfig"] = MZ_ImagesCopyWorkspace -NODE_DISPLAY_NAME_MAPPINGS["MZ_DatasetConfig"] = f"{AUTHOR_NAME} - DatasetConfig" +from . import mz_train_tools_core_HYDiT + + +class MZ_HYDiTInitWorkspace: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "train_name": ("STRING", {"default": ""}), + "branch": ("STRING", {"default": "5657364143e44ac90f72aeb47b81bd505a95665d"}), + "source": ([ + "github", + "githubfast", + "521github", + "kkgithub", + ], {"default": "github"}), + "seed": ("INT", {"default": 0}), + }, + } + + RETURN_TYPES = ("MZ_TT_HYDiT_WorkspaceConfig",) + RETURN_NAMES = ("workspace_config",) + + FUNCTION = "start" + + CATEGORY = CATEGORY_NAME + + def start(self, **kwargs): + importlib.reload(mz_train_tools_core_HYDiT) + return mz_train_tools_core_HYDiT.MZ_HYDiTInitWorkspace_call(kwargs) + + +NODE_CLASS_MAPPINGS["MZ_HYDiTInitWorkspace"] = MZ_HYDiTInitWorkspace +NODE_DISPLAY_NAME_MAPPINGS["MZ_HYDiTInitWorkspace"] = f"{AUTHOR_NAME} - HYDiTInitWorkspace" + + +class MZ_HYDiTDatasetConfig: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "workspace_config": ("MZ_TT_HYDiT_WorkspaceConfig",), + "resolution": ("INT", {"default": 1024}), + }, + "optional": { + "images": ("IMAGE",), + } + } + + RETURN_TYPES = ("STRING",) + RETURN_NAMES = ("workspace_images_dir",) + + FUNCTION = "start" + + CATEGORY = CATEGORY_NAME + + def start(self, **kwargs): + importlib.reload(mz_train_tools_core_HYDiT) + return mz_train_tools_core_HYDiT.MZ_HYDiTDatasetConfig_call(kwargs) + + +NODE_CLASS_MAPPINGS["MZ_HYDiTDatasetConfig"] = MZ_HYDiTDatasetConfig +NODE_DISPLAY_NAME_MAPPINGS["MZ_HYDiTDatasetConfig"] = f"{AUTHOR_NAME} - HYDiTDatasetConfig" + + +# class MZ_HYDiTUseConfig: +# @classmethod +# def INPUT_TYPES(s): + +# return { +# "required": { +# "workspace_config": ("MZ_TT_HYDiT_WorkspaceConfig",), +# "workspace_images_dir": ("STRING", {"forceInput": True}), +# }, +# "optional": { +# } +# } + +# RETURN_TYPES = ("MZ_TT_HYDiT_TrainConfig",) +# RETURN_NAMES = ("train_config",) + +# FUNCTION = "start" + +# CATEGORY = CATEGORY_NAME + +# def start(self, **kwargs): +# importlib.reload(mz_train_tools_core_HYDiT) +# # kwargs["train_config_template_dir"] = self.train_config_template_dir +# return mz_train_tools_core_HYDiT.MZ_HYDiTUseConfig_call(kwargs) + + +# NODE_CLASS_MAPPINGS["MZ_HYDiTUseConfig"] = MZ_HYDiTUseConfig +# NODE_DISPLAY_NAME_MAPPINGS["MZ_HYDiTUseConfig"] = f"{AUTHOR_NAME} - HYDiTUseConfig" + + +class MZ_HYDiTAdvConfig: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "learn_sigma": (["enable", "disable"], {"default": "enable"}), + "text_states_dim": ("INT", {"default": 1024}), + "text_states_dim_t5": ("INT", {"default": 2048}), + "text_len": ("INT", {"default": 77}), + "text_len_t5": ("INT", {"default": 256}), + "norm": (["layer", "instance"], {"default": "layer"}), + "infer_mode": (["torch", "deepspeed"], {"default": "torch"}), + "use_fp16": (["enable", "disable"], {"default": "enable"}), + } + } + + RETURN_TYPES = ("MZ_TT_HYDiT_AdvConfig",) + RETURN_NAMES = ("advanced_config",) + + FUNCTION = "start" + + CATEGORY = CATEGORY_NAME + + def start(self, **kwargs): + return (kwargs,) + + +NODE_CLASS_MAPPINGS["MZ_HYDiTAdvConfig"] = MZ_HYDiTAdvConfig +NODE_DISPLAY_NAME_MAPPINGS["MZ_HYDiTAdvConfig"] = f"{AUTHOR_NAME} - HYDiTAdvConfig" + + +class MZ_HYDiTTrain: + @classmethod + def INPUT_TYPES(s): + + hunyuan_models_path = os.path.join( + Utils.get_comfyui_models_path(), "hunyuan") + os.makedirs(hunyuan_models_path, exist_ok=True) + + models = Utils.get_models_by_folder(hunyuan_models_path) + folders = Utils.get_folders_by_folder(hunyuan_models_path) + + vae_models = Utils.get_models_by_folder( + os.path.join(Utils.get_comfyui_models_path(), "vae")) + unet_models = Utils.get_models_by_folder( + os.path.join(Utils.get_comfyui_models_path(), "unet")) + return { + "required": { + "workspace_config": ("MZ_TT_HYDiT_WorkspaceConfig",), + "unet_path": (["auto"] + models + unet_models, {"default": "auto"}), + "vae_ema_path": (["auto"] + folders + vae_models, {"default": "auto"}), + "text_encoder_path": (["auto"] + folders, {"default": "auto"}), + "tokenizer_path": (["auto"] + folders, {"default": "auto"}), + "t5_encoder_path": (["auto"] + folders, {"default": "auto"}), + "resolution": ("INT", {"default": 1024}), + "batch_size": ("INT", {"default": 1}), + "epochs": ("INT", {"default": 50}), + "save_every_n_epochs": ("INT", {"default": 10}), + "base_lora": (["latest", "empty"], {"default": "latest"}), + "sample_generate": (["enable", "disable"], {"default": "enable"}), + "sample_prompt": ("STRING", {"default:": "", "dynamicPrompts": True, "multiline": True}), + }, + "optional": { + "workspace_images_dir": ("STRING", {"forceInput": True}), + "has_no_effect": (AlwaysEqualProxy("*"),), + } + } + + RETURN_TYPES = () + RETURN_NAMES = () + + FUNCTION = "start" + + MZ_DESC = """ +base_size The base resolution (n, n) from which to create multiple resolutions | Recommended values: 256/512/1024 +""" + + OUTPUT_NODE = True + + CATEGORY = CATEGORY_NAME + + def start(self, **kwargs): + importlib.reload(mz_train_tools_core_HYDiT) + return mz_train_tools_core_HYDiT.MZ_HYDiTTrain_call(kwargs) + + +NODE_CLASS_MAPPINGS["MZ_HYDiTTrain"] = MZ_HYDiTTrain +NODE_DISPLAY_NAME_MAPPINGS["MZ_HYDiTTrain"] = f"{AUTHOR_NAME} - HYDiTTrain" + + +class MZ_TrainToolsDebug: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "object": (AlwaysEqualProxy("*"),), + "indent": ("INT", {"default": 2}), + "depth": ("INT", {"default": 5}), + "width": ("INT", {"default": 80}), + "compact": (["enable", "disable"], {"default": "enable"}), + "sort_keys": (["enable", "disable"], {"default": "enable"}), + "underscore_numbers": (["enable", "disable"], {"default": "enable"}), + }, + } + + RETURN_TYPES = ("STRING",) + RETURN_NAMES = ("debug",) + + FUNCTION = "start" + + CATEGORY = CATEGORY_NAME + + def start(self, **kwargs): + + from pprint import pprint, pp + object = kwargs["object"] + indent = kwargs["indent"] + depth = kwargs["depth"] + width = kwargs["width"] + compact = kwargs["compact"] == "enable" + sort_keys = kwargs["sort_keys"] == "enable" + underscore_numbers = kwargs["underscore_numbers"] == "enable" + + debug = pp(object, stream=None, indent=indent, depth=depth, width=width, + compact=compact, sort_dicts=sort_keys, underscore_numbers=underscore_numbers) + + return (debug,) + + +NODE_CLASS_MAPPINGS["MZ_TrainToolsDebug"] = MZ_TrainToolsDebug +NODE_DISPLAY_NAME_MAPPINGS["MZ_TrainToolsDebug"] = f"{AUTHOR_NAME} - TrainToolsDebug" diff --git a/hook_HYDiT_idk_run.py b/hook_HYDiT_idk_run.py new file mode 100644 index 0000000..50b8f27 --- /dev/null +++ b/hook_HYDiT_idk_run.py @@ -0,0 +1,155 @@ +import argparse + +from index_kits.dataset.make_dataset_core import startup, make_multireso +from index_kits.common import show_index_info +from index_kits import __version__ + + +def common_args(parser): + parser.add_argument('-t', '--target', type=str, + required=True, help='Save path') + + +def get_args(): + parser = argparse.ArgumentParser(description=""" + IndexKits is a tool to build and manage index files for large-scale datasets. + It supports both base index and multi-resolution index. + + Introduction + ------------ + This command line tool provides the following functionalities: + 1. Show index v2 information + 2. Build base index v2 + 3. Build multi-resolution index v2 + + Examples + -------- + 1. Show index v2 information + index_kits show /path/to/index.json + + 2. Build base index v2 + Default usage: + index_kits base -c /path/to/config.yaml -t /path/to/index.json + + Use multiple processes: + index_kits base -c /path/to/config.yaml -t /path/to/index.json -w 40 + + 3. Build multi-resolution index v2 + + Build with a configuration file: + index_kits multireso -c /path/to/config.yaml -t /path/to/index_mb_gt512.json + + Build by specifying arguments without a configuration file: + index_kits multireso --src /path/to/index.json --base-size 512 --reso-step 32 --min-size 512 -t /path/to/index_mb_gt512.json + + Build by specifying target-ratios: + index_kits multireso --src /path/to/index.json --base-size 512 --target-ratios 1:1 4:3 3:4 16:9 9:16 --min-size 512 -t /path/to/index_mb_gt512.json + + Build with multiple source index files. + index_kits multireso --src /path/to/index1.json /path/to/index2.json --base-size 512 --reso-step 32 --min-size 512 -t /path/to/index_mb_gt512.json + """, formatter_class=argparse.RawTextHelpFormatter) + sub_parsers = parser.add_subparsers(dest='task', required=True) + + # Show index message + show_parser = sub_parsers.add_parser('show', description=""" + Show base/multireso index v2 information. + + Example + ------- + index_kits show /path/to/index.json + """, formatter_class=argparse.RawTextHelpFormatter) + show_parser.add_argument( + 'src', type=str, help='Path to a base/multireso index file.') + show_parser.add_argument( + '--arrow-files', action='store_true', help='Show arrow files only.') + show_parser.add_argument('--depth', type=int, default=1, + help='Arrow file depth. Default is 1, the level of last folder in the arrow file path. ' + 'Set it to 0 to show the full path including `xxx/last_folder/*.arrow`.') + + # Single resolution bucket + base_parser = sub_parsers.add_parser('base', description=""" + Build base index v2. + + Example + ------- + index_kits base -c /path/to/config.yaml -t /path/to/index.json + """, formatter_class=argparse.RawTextHelpFormatter) + base_parser.add_argument('-c', '--config', type=str, + required=True, help='Configuration file path') + common_args(base_parser) + base_parser.add_argument('-w', '--world-size', type=int, default=1) + base_parser.add_argument('--work-dir', type=str, + default='.', help='Work directory') + base_parser.add_argument('--use-cache', action='store_true', help='Use cache to avoid reprocessing. ' + 'Perform merge pkl results directly.') + + # Multi-resolution bucket + mo_parser = sub_parsers.add_parser('multireso', description=""" + Build multi-resolution index v2 + + Example + ------- + Build with a configuration file: + index_kits multireso -c /path/to/config.yaml -t /path/to/index_mb_gt512.json + + Build by specifying arguments without a configuration file: + index_kits multireso --src /path/to/index.json --base-size 512 --reso-step 32 --min-size 512 -t /path/to/index_mb_gt512.json + + Build by specifying target-ratios: + index_kits multireso --src /path/to/index.json --base-size 512 --target-ratios 1:1 4:3 3:4 16:9 9:16 --min-size 512 -t /path/to/index_mb_gt512.json + + Build with multiple source index files. + index_kits multireso --src /path/to/index1.json /path/to/index2.json --base-size 512 --reso-step 32 --min-size 512 -t /path/to/index_mb_gt512.json + """, formatter_class=argparse.RawTextHelpFormatter) + mo_parser.add_argument('-c', '--config', type=str, default=None, + help='Configuration file path in a yaml format. Either --config or --src must be provided.') + mo_parser.add_argument('-s', '--src', type=str, nargs='+', default=None, + help='Source index files. Either --config or --src must be provided.') + common_args(mo_parser) + mo_parser.add_argument('--base-size', type=int, default=None, + help="Base size. Typically set as 256/512/1024 according to image size you train model.") + mo_parser.add_argument('--reso-step', type=int, default=None, + help="Resolution step. Either reso_step or target_ratios must be provided.") + mo_parser.add_argument('--target-ratios', type=str, nargs='+', default=None, + help="Target ratios. Either reso_step or target_ratios must be provided.") + mo_parser.add_argument('--md5-file', type=str, default=None, + help='You can provide an md5 to height and width file to accelerate the process. ' + 'It is a pickle file that contains a dict, which maps md5 to (height, width) tuple.') + mo_parser.add_argument('--align', type=int, default=16, + help="Used when --target-ratios is provided. Align size of source image height and width.") + mo_parser.add_argument('--min-size', type=int, default=0, + help="Minimum size. Images smaller than this size will be ignored.") + + # Common + parser.add_argument('-v', '--version', action='version', + version=f'%(prog)s {__version__}') + + args = parser.parse_args() + return args + + +if __name__ == '__main__': + args = get_args() + if args.task == 'show': + show_index_info(args.src, + args.arrow_files, + args.depth, + ) + elif args.task == 'base': + startup(args.config, + args.target, + args.world_size, + args.work_dir, + use_cache=args.use_cache, + ) + elif args.task == 'multireso': + make_multireso(args.target, + args.config, + args.src, + args.base_size, + args.reso_step, + args.target_ratios, + args.align, + args.min_size, + args.md5_file, + ) diff --git a/hook_HYDiT_main.py b/hook_HYDiT_main.py new file mode 100644 index 0000000..d33d547 --- /dev/null +++ b/hook_HYDiT_main.py @@ -0,0 +1,710 @@ + +import argparse +import json +import logging +import os +import sys +from types import SimpleNamespace +import gc +import json +import os +import random +import sys +import time +from functools import partial +from glob import glob +from pathlib import Path +import numpy as np + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.utils.data import DataLoader +from torchvision.transforms import functional as TF +from diffusers.models import AutoencoderKL +from transformers import BertModel, BertTokenizer, logging as tf_logging + +from hydit.config import get_args +# from hydit.constants import VAE_EMA_PATH, TEXT_ENCODER, TOKENIZER, T5_ENCODER +from hydit.lr_scheduler import WarmupLR +from hydit.data_loader.arrow_load_stream import TextImageArrowStream +from hydit.diffusion import create_diffusion +from hydit.ds_config import deepspeed_config_from_args +from hydit.modules.ema import EMA +from hydit.modules.fp16_layers import Float16Module +from hydit.modules.models import HUNYUAN_DIT_MODELS +from hydit.modules.posemb_layers import init_image_posemb +from hydit.utils.tools import set_seeds, get_trainable_params +from IndexKits.index_kits import ResolutionGroup +from IndexKits.index_kits.sampler import DistributedSamplerWithStartIndex, BlockDistributedSampler +from peft import LoraConfig, get_peft_model +from tqdm import tqdm + + +UNET_PATH = "ckpts/t2i/model/pytorch_model_ema.pt" + + +def set_unet_path(path): + global UNET_PATH + UNET_PATH = path + + +VAE_EMA_PATH = "ckpts/t2i/sdxl-vae-fp16-fix" + + +def set_vae_ema_path(path): + global VAE_EMA_PATH + VAE_EMA_PATH = path + + +TOKENIZER = "ckpts/t2i/tokenizer" + + +def set_tokenizer_path(path): + global TOKENIZER + TOKENIZER = path + + +TEXT_ENCODER = 'ckpts/t2i/clip_text_encoder' + + +def set_text_encoder_path(path): + global TEXT_ENCODER + TEXT_ENCODER = path + + +T5_ENCODER = { + 'MT5': 'ckpts/t2i/mt5', + 'attention_mask': True, + 'layer_index': -1, + 'attention_pool': True, + 'torch_dtype': torch.float16, + 'learnable_replace': True +} + + +def set_t5_encoder_path(path): + global T5_ENCODER + T5_ENCODER['MT5'] = path + + +def create_logger(log_dir=None): + logger = logging.getLogger(__name__) + logger.addHandler(logging.NullHandler()) + return logger + + +def create_exp_folder(args, rank): + if rank == 0: + os.makedirs(args.results_dir, exist_ok=True) + + # print('args.results_dir', args.results_dir) + _existed_experiments = os.listdir(args.results_dir) + # print('_existed_experiments', _existed_experiments) + existed_experiments = [] + for x in _existed_experiments: + if os.path.isdir(os.path.join(args.results_dir, x)): + existed_experiments.append(x) + + if len(existed_experiments) == 0: + experiment_index = 1 + else: + existed_experiments.sort() + # print('existed_experiments', existed_experiments) + experiment_index = max([int(x.split('-')[0]) + for x in existed_experiments]) + 1 + model_string_name = args.task_flag if args.task_flag else args.model.replace( + "/", "-") + # Create an experiment folder + experiment_dir = f"{args.results_dir}/{experiment_index:03d}-{model_string_name}" + # Stores saved model checkpoints + checkpoint_dir = f"{experiment_dir}/checkpoints" + if rank == 0: + os.makedirs(checkpoint_dir, exist_ok=True) + logger = create_logger(experiment_dir) + logger.info(f"Experiment directory created at {experiment_dir}") + else: + logger = create_logger() + experiment_dir = "" + logger.info = print + return experiment_dir, checkpoint_dir, logger + + +def save_checkpoint(args, rank, logger, model, ema, epoch, train_steps, checkpoint_dir): + def save_lora_weight(checkpoint_dir, client_state, tag=f"{train_steps:07d}.pt"): + cur_ckpt_save_dir = f"{checkpoint_dir}/{tag}" + if rank == 0: + if args.use_fp16: + try: + model.module.save_pretrained(cur_ckpt_save_dir) + except Exception as e: + model.save_pretrained(cur_ckpt_save_dir) + else: + model.module.save_pretrained(cur_ckpt_save_dir) + + checkpoint_path = "[Not rank 0. Disabled output.]" + + client_state = { + "steps": train_steps, + "epoch": epoch, + "args": args + } + if ema is not None: + client_state['ema'] = ema.state_dict() + + dst_paths = [] + + if train_steps % args.ckpt_every == 0: + checkpoint_path = f"{checkpoint_dir}/{train_steps:07d}.pt" + try: + if args.training_parts == "lora": + save_lora_weight(checkpoint_dir, client_state, + tag=f"{train_steps:07d}.pt") + else: + model.save_checkpoint( + checkpoint_dir, client_state=client_state, tag=f"{train_steps:07d}.pt") + dst_paths.append(checkpoint_path) + logger.info(f"Saved checkpoint to {checkpoint_path}") + except: + logger.error(f"Saved failed to {checkpoint_path}") + + if train_steps % args.ckpt_latest_every == 0 or train_steps == args.max_training_steps: + save_name = "latest.pt" + checkpoint_path = f"{checkpoint_dir}/{save_name}" + try: + if args.training_parts == "lora": + save_lora_weight(checkpoint_dir, client_state, + tag=f"{save_name}") + else: + model.save_checkpoint( + checkpoint_dir, client_state=client_state, tag=f"{save_name}") + dst_paths.append(checkpoint_path) + logger.info(f"Saved checkpoint to {checkpoint_path}") + except: + logger.error(f"Saved failed to {checkpoint_path}") + + if rank == 0 and len(dst_paths) > 0: + # Delete optimizer states to avoid occupying too much disk space. + for dst_path in dst_paths: + for opt_state_path in glob(f"{dst_path}/zero_dp_rank_*_tp_rank_00_pp_rank_00_optim_states.pt"): + os.remove(opt_state_path) + + return checkpoint_path + + +@torch.no_grad() +def prepare_model_inputs(args, batch, device, vae, text_encoder, text_encoder_t5, freqs_cis_img): + image, text_embedding, text_embedding_mask, text_embedding_t5, text_embedding_mask_t5, kwargs = batch + + # clip & mT5 text embedding + text_embedding = text_embedding.to(device) + text_embedding_mask = text_embedding_mask.to(device) + encoder_hidden_states = text_encoder( + text_embedding.to(device), + attention_mask=text_embedding_mask.to(device), + )[0] + text_embedding_t5 = text_embedding_t5.to(device).squeeze(1) + text_embedding_mask_t5 = text_embedding_mask_t5.to(device).squeeze(1) + with torch.no_grad(): + output_t5 = text_encoder_t5( + input_ids=text_embedding_t5, + attention_mask=text_embedding_mask_t5 if T5_ENCODER['attention_mask'] else None, + output_hidden_states=True + ) + encoder_hidden_states_t5 = output_t5['hidden_states'][T5_ENCODER['layer_index']].detach( + ) + + # additional condition + image_meta_size = kwargs['image_meta_size'].to(device) + style = kwargs['style'].to(device) + + if args.extra_fp16: + image = image.half() + image_meta_size = image_meta_size.half() if image_meta_size is not None else None + + # Map input images to latent space + normalize latents: + image = image.to(device) + vae_scaling_factor = vae.config.scaling_factor + latents = vae.encode(image).latent_dist.sample().mul_( + vae_scaling_factor).to(device) + + # positional embedding + _, _, height, width = image.shape + reso = f"{height}x{width}" + + # print("freqs_cis_img:", freqs_cis_img.keys()) + # print("freqs_cis_img[reso]:", freqs_cis_img[reso]) + + cos_cis_img, sin_cis_img = freqs_cis_img[reso] + + # Model conditions + model_kwargs = dict( + encoder_hidden_states=encoder_hidden_states, + text_embedding_mask=text_embedding_mask, + encoder_hidden_states_t5=encoder_hidden_states_t5, + text_embedding_mask_t5=text_embedding_mask_t5, + image_meta_size=image_meta_size, + style=style, + cos_cis_img=cos_cis_img, + sin_cis_img=sin_cis_img, + ) + + return latents, model_kwargs + + +def Core(args, LOG): + if args.training_parts == "lora": + args.use_ema = False + + assert torch.cuda.is_available(), "Training currently requires at least one GPU." + + batch_size = args.batch_size + grad_accu_steps = args.grad_accu_steps + global_batch_size = batch_size * grad_accu_steps + + seed = args.global_seed + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + print(f"Starting ,seed={seed}.") + rank = 0 + # Setup an experiment folder + experiment_dir, checkpoint_dir, logger = create_exp_folder(args, rank) + + # Log all the arguments + logger.info(sys.argv) + logger.info(str(args)) + # Save to a json file + args_dict = vars(args) + with open(f"{experiment_dir}/args.json", 'w') as f: + json.dump(args_dict, f, indent=4) + + # Disable the message "Some weights of the model checkpoint at ... were not used when initializing BertModel." + # If needed, just comment the following line. + tf_logging.set_verbosity_error() + + # =========================================================================== + # Building HYDIT + # =========================================================================== + + logger.info("Building HYDIT Model.") + + # --------------------------------------------------------------------------- + # Training sample base size, such as 256/512/1024. Notice that this size is + # just a base size, not necessary the actual size of training samples. Actual + # size of the training samples are correlated with `resolutions` when enabling + # multi-resolution training. + # --------------------------------------------------------------------------- + image_size = args.image_size + if type(image_size) == int: + image_size = [image_size, image_size] + if len(image_size) == 1: + image_size = [image_size[0], image_size[0]] + if len(image_size) != 2: + raise ValueError(f"Invalid image size: {args.image_size}") + assert image_size[0] % 8 == 0 and image_size[1] % 8 == 0, "Image size must be divisible by 8 (for the VAE encoder). " \ + f"got {image_size}" + latent_size = [image_size[0] // 8, image_size[1] // 8] + + # initialize model by deepspeed + model = HUNYUAN_DIT_MODELS[args.model](args, + input_size=latent_size, + log_fn=logger.info, + ) + # Multi-resolution / Single-resolution training. + if args.multireso: + resolutions = ResolutionGroup(image_size[0], + align=16, + step=args.reso_step, + target_ratios=args.target_ratios).data + else: + resolutions = ResolutionGroup(image_size[0], + align=16, + target_ratios=['1:1']).data + + freqs_cis_img = init_image_posemb(args.rope_img, + resolutions=resolutions, + patch_size=model.patch_size, + hidden_size=model.hidden_size, + num_heads=model.num_heads, + log_fn=logger.info, + rope_real=args.rope_real, + ) + + # Create EMA model and convert to fp16 if needed. + ema = None + device = "cuda" + if args.use_ema: + ema = EMA(args, model, device, logger) + + # Setup FP16 main model: + if args.use_fp16: + model = Float16Module(model, args) + logger.info( + f" Using main model with data type {'fp16' if args.use_fp16 else 'fp32'}") + + diffusion = create_diffusion( + noise_schedule=args.noise_schedule, + predict_type=args.predict_type, + learn_sigma=args.learn_sigma, + mse_loss_weight_type=args.mse_loss_weight_type, + beta_start=args.beta_start, + beta_end=args.beta_end, + noise_offset=args.noise_offset, + ) + + # Setup VAE + logger.info(f" Loading vae from {VAE_EMA_PATH}") + + if os.path.isdir(VAE_EMA_PATH): + vae = AutoencoderKL.from_pretrained(VAE_EMA_PATH) + else: + vae = AutoencoderKL.from_single_file(VAE_EMA_PATH) + # Setup BERT text encoder + logger.info(f" Loading Bert text encoder from {TEXT_ENCODER}") + text_encoder = BertModel.from_pretrained( + TEXT_ENCODER, False, revision=None) + # Setup BERT tokenizer: + logger.info(f" Loading Bert tokenizer from {TOKENIZER}") + tokenizer = BertTokenizer.from_pretrained(TOKENIZER) + # Setup T5 text encoder + from hydit.modules.text_encoder import MT5Embedder + mt5_path = T5_ENCODER['MT5'] + embedder_t5 = MT5Embedder( + mt5_path, torch_dtype=T5_ENCODER['torch_dtype'], max_length=args.text_len_t5) + tokenizer_t5 = embedder_t5.tokenizer + text_encoder_t5 = embedder_t5.model + + if args.extra_fp16: + logger.info(f" Using fp16 for extra modules: vae, text_encoder") + vae = vae.half().to(device) + text_encoder = text_encoder.half().to(device) + text_encoder_t5 = text_encoder_t5.half().to(device) + else: + vae = vae.to(device) + text_encoder = text_encoder.to(device) + text_encoder_t5 = text_encoder_t5.to(device) + + logger.info( + f" Optimizer parameters: lr={args.lr}, weight_decay={args.weight_decay}") + + # =========================================================================== + # Building Dataset + # =========================================================================== + + logger.info(f"Building Streaming Dataset.") + logger.info(f" Loading index file {args.index_file} (v2)") + world_size = 1 + + dataset = TextImageArrowStream(args=args, + resolution=image_size[0], + random_flip=args.random_flip, + log_fn=logger.info, + index_file=args.index_file, + multireso=args.multireso, + batch_size=batch_size, + world_size=world_size, + random_shrink_size_cond=args.random_shrink_size_cond, + merge_src_cond=args.merge_src_cond, + uncond_p=args.uncond_p, + text_ctx_len=args.text_len, + tokenizer=tokenizer, + uncond_p_t5=args.uncond_p_t5, + text_ctx_len_t5=args.text_len_t5, + tokenizer_t5=tokenizer_t5, + ) + if args.multireso: + sampler = BlockDistributedSampler(dataset, num_replicas=world_size, rank=rank, seed=args.global_seed, + shuffle=False, drop_last=True, batch_size=batch_size) + else: + sampler = DistributedSamplerWithStartIndex(dataset, num_replicas=world_size, rank=rank, seed=args.global_seed, + shuffle=False, drop_last=True) + loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, sampler=sampler, + num_workers=args.num_workers, pin_memory=True, drop_last=True) + logger.info(f" Dataset contains {len(dataset):,} images.") + logger.info(f" Index file: {args.index_file}.") + if args.multireso: + logger.info(f' Using MultiResolutionBucketIndexV2 with step {dataset.index_manager.step} ' + f'and base size {dataset.index_manager.base_size}') + logger.info(f'\n {dataset.index_manager.resolutions}') + + # =========================================================================== + # Loading parameter + # =========================================================================== + + logger.info(f"Loading parameter") + start_epoch = 0 + start_epoch_step = 0 + train_steps = 0 + # Resume checkpoint if needed + model, ema, start_epoch, start_epoch_step, train_steps = model_resume( + args, model, ema, logger) + + if args.training_parts == "lora": + loraconfig = LoraConfig( + r=args.rank, + lora_alpha=args.rank, + target_modules=args.target_modules + ) + if args.use_fp16: + model.module = get_peft_model(model.module, loraconfig) + else: + model = get_peft_model(model, loraconfig) + + logger.info(f" Training parts: {args.training_parts}") + + # =========================================================================== + # Training + # =========================================================================== + + model.to(device) + model.train() + if args.use_ema: + ema.eval() + + print(f" Worker {rank} ready.") + + iters_per_epoch = len(loader) + logger.info( + " ****************************** Running training ******************************") + logger.info(f" Number GPUs: {world_size}") + logger.info(f" Number training samples: {len(dataset):,}") + logger.info( + f" Number parameters: {sum(p.numel() for p in model.parameters()):,}") + logger.info( + f" Number trainable params: {sum(p.numel() for p in get_trainable_params(model)):,}") + logger.info( + " ------------------------------------------------------------------------------") + logger.info(f" Iters per epoch: {iters_per_epoch:,}") + logger.info(f" Batch size per device: {batch_size}") + logger.info( + f" Batch size all device: {batch_size * world_size * grad_accu_steps:,} (world_size * batch_size * grad_accu_steps)") + logger.info(f" Gradient Accu steps: {args.grad_accu_steps}") + logger.info( + f" Total optimization steps: {args.epochs * iters_per_epoch // grad_accu_steps:,}") + + logger.info( + f" Training epochs: {start_epoch}/{args.epochs}") + logger.info( + f" Training epoch steps: {start_epoch_step:,}/{iters_per_epoch:,}") + logger.info( + f" Training total steps: {train_steps:,}/{min(args.max_training_steps, args.epochs * iters_per_epoch):,}") + logger.info( + " ------------------------------------------------------------------------------") + logger.info(f" Noise schedule: {args.noise_schedule}") + logger.info( + f" Beta limits: ({args.beta_start}, {args.beta_end})") + logger.info(f" Learn sigma: {args.learn_sigma}") + logger.info(f" Prediction type: {args.predict_type}") + logger.info(f" Noise offset: {args.noise_offset}") + + logger.info( + " ------------------------------------------------------------------------------") + logger.info( + f" Using EMA model: {args.use_ema} ({args.ema_dtype})") + if args.use_ema: + logger.info( + f" Using EMA decay: {ema.max_value if args.use_ema else None}") + logger.info( + f" Using EMA warmup power: {ema.power if args.use_ema else None}") + logger.info(f" Using main model fp16: {args.use_fp16}") + logger.info(f" Using extra modules fp16: {args.extra_fp16}") + logger.info( + " ------------------------------------------------------------------------------") + logger.info(f" Experiment directory: {experiment_dir}") + logger.info( + " *******************************************************************************") + + if args.gc_interval > 0: + gc.disable() + gc.collect() + + # Variables for monitoring/logging purposes: + log_steps = 0 + running_loss = 0 + start_time = time.time() + + if args.async_ema: + ema_stream = torch.cuda.Stream() + + total_steps = args.epochs * len(loader) + + if args.use_fp16: + optimizer = torch.optim.AdamW( + model.parameters(), lr=args.lr, weight_decay=args.weight_decay, eps=4e-6) + else: + optimizer = torch.optim.AdamW( + model.parameters(), lr=args.lr, weight_decay=args.weight_decay) + + # torch.autograd.set_detect_anomaly(True) + # with torch.autograd.detect_anomaly(): + with tqdm(total=total_steps, initial=train_steps) as pbar: + + # Training loop + for epoch in range(start_epoch, args.epochs): + logger.info(f" Start random shuffle with seed={seed}") + # Makesure all processors use the same seed to shuffle dataset. + dataset.shuffle(seed=args.global_seed + epoch, fast=True) + logger.info(f" End of random shuffle") + + # Move sampler to start_index + if not args.multireso: + start_index = start_epoch_step * world_size * batch_size + if start_index != sampler.start_index: + sampler.start_index = start_index + # Reset start_epoch_step to zero, to ensure next epoch will start from the beginning. + start_epoch_step = 0 + logger.info( + f" Iters left this epoch: {len(loader):,}") + + logger.info(f" Beginning epoch {epoch}...") + step = 0 + mean_loss = 0 + for batch in loader: + step += 1 + + # Zero gradients, backward pass, and optimizer step: + optimizer.zero_grad() + latents, model_kwargs = prepare_model_inputs( + args, batch, device, vae, text_encoder, text_encoder_t5, freqs_cis_img) + # with torch.cuda.amp.autocast(enabled=True, dtype=torch.float16, cache_enabled=True): + + if True: + # training model by deepspeed while use fp16 + if args.use_fp16: + if args.use_ema and args.async_ema: + with torch.cuda.stream(ema_stream): + ema.update(model.module.module, step=step) + torch.cuda.current_stream().wait_stream(ema_stream) + + loss_dict = diffusion.training_losses( + model=model, x_start=latents, model_kwargs=model_kwargs) + + loss = loss_dict["loss"].mean() + + # print(f"step={step}, loss={loss.item()}") + + # model.backward(loss) + + loss.backward() + + # last_batch_iteration = ( + # train_steps + 1) // (global_batch_size // (batch_size * world_size)) + # model.step( + # lr_kwargs={'last_batch_iteration': last_batch_iteration}) + + optimizer.step() + + if args.use_ema and not args.async_ema or (args.async_ema and step == len(loader) - 1): + if args.use_fp16: + ema.update(model.module.module, step=step) + else: + ema.update(model.module, step=step) + + train_steps += 1 + mean_loss += loss.item() + pbar.update(1) + pbar.set_description(f"Epoch {epoch}, step {step}, loss {loss.item():.4f}, mean_loss {mean_loss / step:.4f}") + + # =========================================================================== + # Log loss values: + # =========================================================================== + # running_loss += loss.item() + # log_steps += 1 + # if train_steps % args.log_every == 0: + # # Measure training speed: + # torch.cuda.synchronize() + # end_time = time.time() + # steps_per_sec = log_steps / (end_time - start_time) + # # Reduce loss history over all processes: + # avg_loss = torch.tensor( + # running_loss / log_steps, device=device) + # # dist.all_reduce(avg_loss, op=dist.ReduceOp.SUM) + + # # avg_loss = avg_loss.item() / world_size + # # get lr from deepspeed fused optimizer + # logger.info(f"(step={train_steps:07d}) " + + # (f"(update_step={train_steps // args.grad_accu_steps:07d}) " if args.grad_accu_steps > 1 else "") + + # f"Train Loss: {avg_loss:.4f}, " + # # f"Lr: {opt.param_groups[0]['lr']:.6g}, " + # f"Steps/Sec: {steps_per_sec:.2f}, " + # f"Samples/Sec: {int(steps_per_sec * batch_size * world_size):d}") + # # Reset monitoring variables: + # running_loss = 0 + # log_steps = 0 + # start_time = time.time() + + # collect gc: + if args.gc_interval > 0 and (step % args.gc_interval == 0): + gc.collect() + + if (train_steps % args.ckpt_every == 0 or train_steps % args.ckpt_latest_every == 0 # or train_steps == args.max_training_steps + ) and train_steps > 0: + logger.info( + f" Saving checkpoint at step {train_steps}.") + save_checkpoint(args, rank, logger, model, ema, + epoch, train_steps, checkpoint_dir) + + if train_steps >= args.max_training_steps: + logger.info(f"Breaking step loop at {train_steps}.") + break + + LOG({ + "type": "sample_images", + "global_step": train_steps, + "total_steps": total_steps, + # "latent": noise_pred_latent_path, + }) + + if train_steps >= args.max_training_steps: + logger.info(f"Breaking epoch loop at {epoch}.") + break + + +def model_resume(args, model, ema, logger): + """ + Load pretrained weights. + """ + start_epoch = 0 + start_epoch_step = 0 + train_steps = 0 + resume_path = UNET_PATH + if not Path(resume_path).exists(): + raise FileNotFoundError( + f" Cannot find checkpoint from {resume_path}") + + logger.info(f"Resume from checkpoint {resume_path}") + + if args.resume_split: + # Resume main model + + resume_ckpt_module = torch.load( + resume_path, map_location=lambda storage, loc: storage) + model.load_state_dict(resume_ckpt_module, strict=False) + + # Resume ema model + if args.use_ema: + if args.module_to_ema: + if "resume_ckpt_module" in locals(): + logger.info(f" Resume ema model from main states.") + ema.load_state_dict(resume_ckpt_module, strict=args.strict) + else: + logger.info(f" Resume ema model from module states.") + resume_ckpt_module = torch.load( + resume_path, map_location=lambda storage, loc: storage) + ema.load_state_dict(resume_ckpt_module, strict=args.strict) + else: + if "resume_ckpt_ema" in locals(): + logger.info(f" Resume ema model from EMA states.") + ema.load_state_dict(resume_ckpt_ema, strict=args.strict) + else: + logger.info(f" Resume ema model from EMA states.") + resume_ckpt_ema = torch.load(resume_path, + map_location=lambda storage, loc: storage) + ema.load_state_dict(resume_ckpt_ema, strict=args.strict) + else: + raise ValueError( + " “If `resume` is True, then either `resume_split` must be true.”") + + return model, ema, start_epoch, start_epoch_step, train_steps diff --git a/hook_HYDiT_run.py b/hook_HYDiT_run.py new file mode 100644 index 0000000..3ced37c --- /dev/null +++ b/hook_HYDiT_run.py @@ -0,0 +1,146 @@ + +import argparse +import json +import os +import sys +from types import SimpleNamespace + + +# python -m torch.distributed.launch ./hook_HYDi_run.py --sys_path /data/ComfyUI/models/minus_zone_models/train_tools/HunyuanDiT/ + + +class SimpleNamespaceCNWarrper(SimpleNamespace): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.__dict__.update(kwargs) + self.__iter__ = lambda: iter(kwargs.keys()) + # is not iterable + + def __iter__(self): + return iter(self.__dict__.keys()) + # object has no attribute 'num_attention_heads' + + def __getattr__(self, name): + return self.__dict__.get(name, None) + + +import requests + +master_port = 0 + + +def LOG(log): + if master_port == 0: + raise Exception("master_port is 0") + # 发送http + resp = requests.request("post", f"http://127.0.0.1:{master_port}/log", data=json.dumps(log), headers={ + "Content-Type": "application/json"}) + if resp.status_code != 200: + raise Exception(f"LOG failed: {resp.text}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + conflict_handler='resolve', + ) + parser.add_argument("--sys_path", type=str, default="") + parser.add_argument("--train_config_file", type=str, default="") + parser.add_argument("--master_port", type=int, default=0) + args = parser.parse_args() + + master_port = args.master_port + + print(f"master_port = {master_port}") + + sys_path = args.sys_path + if sys_path != "": + sys.path.append(sys_path) + + print("HYDi run hook") + + try: + from . import hook_HYDiT_main + except Exception as e: + import hook_HYDiT_main + + import hydit.config + + def _handle_conflict_error(self, *args, **kwargs): + pass + + def parse_args(self, args=None, namespace=None): + args, argv = self.parse_known_args(args, namespace) + return args + + train_config_file = args.train_config_file + + if train_config_file == "": + raise ValueError("train_config_file is empty") + + train_config = {} + with open(train_config_file, "r") as f: + train_config = json.load(f) + + argparse.ArgumentParser._handle_conflict_error = _handle_conflict_error + argparse.ArgumentParser._handle_conflict_resolve = _handle_conflict_error + argparse.ArgumentParser.parse_args = parse_args + margs = hydit.config.get_args() + margs.model = train_config.get("model", "DiT-g/2") + + + margs.task_flag = train_config.get( + "task_flag", "lora_porcelain_ema_rank64") + + + margs.resume_split = train_config.get("resume_split", True) + margs.ema_to_module = train_config.get("ema_to_module", True) + margs.deepspeed = False + margs.predict_type = train_config.get("predict_type", "v_prediction") + margs.training_parts = train_config.get("training_parts", "lora") + margs.batch_size = train_config.get("batch_size", 1) + margs.grad_accu_steps = train_config.get("grad_accu_steps", 1) + margs.global_seed = train_config.get("global_seed", 0) + margs.use_flash_attn = train_config.get("use_flash_attn", False) + margs.use_fp16 = train_config.get("use_fp16", True) + margs.qk_norm = train_config.get("qk_norm", True) + margs.ema_dtype = train_config.get("ema_dtype", "fp32") + margs.async_ema = False + margs.multireso = train_config.get("multireso", True) + margs.epochs = train_config.get("epochs", 50) + margs.target_ratios = train_config.get( + "target_ratios", ['1:1', '3:4', '4:3', '16:9', '9:16']) + margs.rope_img = train_config.get("rope_img", "base512") + margs.image_size = train_config.get("image_size", 512) + margs.rope_real = train_config.get("rope_real", True) + + margs.index_file = train_config.get("index_file", None) + + margs.lr = train_config.get("lr", 1e-5) + + margs.noise_offset = train_config.get("noise_offset", 0.1) + + margs.log_every = train_config.get("log_every", 10) + + margs.results_dir = train_config.get("results_dir") + margs.mse_loss_weight_type = train_config.get("mse_loss_weight_type", "constant") + + for k, v in train_config.items(): + if hasattr(margs, k): + setattr(margs, k, v) + + hook_HYDiT_main.set_unet_path( + train_config.get("unet_path")) + + hook_HYDiT_main.set_vae_ema_path( + train_config.get("vae_ema_path")) + + hook_HYDiT_main.set_text_encoder_path( + train_config.get("text_encoder_path")) + + hook_HYDiT_main.set_tokenizer_path( + train_config.get("tokenizer_path")) + + hook_HYDiT_main.set_t5_encoder_path( + train_config.get("t5_encoder_path")) + + hook_HYDiT_main.Core(margs, LOG) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py new file mode 100644 index 0000000..7c04555 --- /dev/null +++ b/mz_train_tools_core_HYDiT.py @@ -0,0 +1,394 @@ + + +import argparse +import csv +import hashlib +import os +import shutil +import sys +import json +import subprocess +import time + +import torch + +from .mz_train_tools_utils import Utils +import folder_paths +import nodes + + +git_accelerate_urls = { + "githubfast": "githubfast.com", + "521github": "521github.com", + "kkgithub": "kkgithub.com", +} + +# 初始化工具仓库和工作区 + + +def MZ_HYDiTInitWorkspace_call(args={}): + mz_dir = Utils.get_minus_zone_models_path() + git_url = "https://github.com/Tencent/HunyuanDiT" + source = args.get("source", "github") + kohya_ss_lora_dir = os.path.join(mz_dir, "train_tools", "HunyuanDiT") + if git_accelerate_urls.get(source, None) is not None: + git_url = f"https://{git_accelerate_urls[source]}/Tencent/HunyuanDiT" + try: + if not os.path.exists(kohya_ss_lora_dir) or not os.path.exists(os.path.join(kohya_ss_lora_dir, ".git")): + subprocess.run( + ["git", "clone", "--depth", "1", git_url, kohya_ss_lora_dir], check=True) + + # 切换远程分支 git remote set-branches origin 'main' + branch = args.get("branch", "main") + + # 查看本地分支是否一致 + short_result = subprocess.run( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=kohya_ss_lora_dir, stdout=subprocess.PIPE, check=True) + result = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=kohya_ss_lora_dir, stdout=subprocess.PIPE, check=True) + + short_current_branch = short_result.stdout.decode().strip() + long_current_branch = result.stdout.decode().strip() + print( + f"当前分支(current branch): {long_current_branch}({short_current_branch})") + print(f"目标分支(target branch): {branch}") + + if branch != result.stdout.decode() and branch != short_result.stdout.decode(): + subprocess.run( + ["git", "remote", "set-branches", "origin", branch], cwd=kohya_ss_lora_dir, check=True) + subprocess.run( + ["git", "fetch", "--depth", "1", "origin", branch], cwd=kohya_ss_lora_dir, check=True) + + # 恢复所有文件 + subprocess.run( + ["git", "checkout", "."], cwd=kohya_ss_lora_dir, check=True) + + subprocess.run( + ["git", "checkout", branch], cwd=kohya_ss_lora_dir, check=True) + + except Exception as e: + raise Exception(f"克隆kohya-ss/sd-scripts或者切换分支时出现异常,详细信息请查看控制台...") + + workspace_name = args.get("train_name", None) + workspace_name = workspace_name.strip() + + if workspace_name is None or workspace_name == "": + raise Exception("训练名称不能为空(train_name is required)") + + args["workspace_name"] = workspace_name + workspaces_dir = os.path.join( + folder_paths.output_directory, "mz_train_workspaces") + + os.makedirs(workspaces_dir, exist_ok=True) + + workspace_dir = os.path.join(workspaces_dir, workspace_name) + if not os.path.exists(workspace_dir): + os.makedirs(workspace_dir) + + args["workspace_dir"] = workspace_dir + + return ( + args, + ) + + +def MZ_HYDiTDatasetConfig_call(args={}): + workspace_config = args.get("workspace_config", {}) + workspace_name = workspace_config.get("workspace_name", None) + workspace_dir = os.path.join( + folder_paths.output_directory, "mz_train_workspaces", workspace_name) + train_images_dir = os.path.join(workspace_dir, "train_images") + os.makedirs(train_images_dir, exist_ok=True) + + images = args.get("images", None) + + if images is None: + print(f"训练数据位于:{train_images_dir},未检测到传入图片默认已有训练数据,将直接跳过数据准备步骤...") + return ( + train_images_dir, + ) + + pil_images = Utils.tensors2pil_list(images) + + resolution = args.get("resolution", 512) + + if workspace_name is None or workspace_name == "": + raise Exception("训练名称不能为空(workspace_name is required)") + + saved_images_path = [] + + for i, pil_image in enumerate(pil_images): + pil_image = Utils.resize_max(pil_image, resolution, resolution) + width, height = pil_image.size + filename = hashlib.md5(pil_image.tobytes()).hexdigest() + ".png" + pil_image.save(os.path.join(train_images_dir, filename)) + saved_images_path.append(filename) + + return ( + train_images_dir, + ) + + +def MZ_HYDiTTrain_call(args={}): + # raise Exception(args) + resolution = args.get("resolution") + + workspace_config = args.get("workspace_config", {}) + workspace_name = workspace_config.get("workspace_name", None) + workspace_dir = workspace_config.get("workspace_dir", None) + + workspace_images_dir = args.get("workspace_images_dir", None) + if workspace_images_dir is None: + workspace_images_dir = os.path.join(workspace_dir, "train_images") + + full_filenames = os.listdir(workspace_images_dir) + + # 创建image_text.csv文件 + csv_filename = os.path.join(workspace_dir, "image_text.csv") + + if os.path.exists(csv_filename): + os.remove(csv_filename) + + max_width, max_height, min_width, min_height = 0, 0, 99999, 99999 + + from PIL import Image + + with open(csv_filename, "w", encoding="utf-8", newline="") as file: + writer = csv.writer(file) + + writer.writerow(["image_path", "text_zh"]) + for filename in full_filenames: + if filename.endswith(".png"): + + image_path = os.path.join(workspace_images_dir, filename) + pil_image = Image.open(image_path) + width, height = pil_image.size + if width > max_width: + max_width = width + if height > max_height: + max_height = height + if width < min_width: + min_width = width + if height < min_height: + min_height = height + + text_zh = "" + + # 去除后缀 + _caption_filename = filename.split(".")[0] + caption_filename = "" + if os.path.exists(os.path.join(workspace_images_dir, _caption_filename + ".txt")): + caption_filename = _caption_filename + ".txt" + elif os.path.exists(os.path.join(workspace_images_dir, _caption_filename + ".caption")): + caption_filename = _caption_filename + ".caption" + else: + caption_filename = "" + if caption_filename != "": + with open(os.path.join(workspace_images_dir, caption_filename), "r") as f: + caption = f.read() + text_zh = caption + + writer.writerow([image_path, text_zh]) + + if resolution > max_width and resolution > max_height: + print("resolution 大于所有图片的宽高,强制使用最大宽高作为resolution") + resolution = max(max_width, max_height) // 8 * 8 + + print(f"csv文件已生成: {csv_filename}") + + HYDiT_tool_dir = os.path.join( + Utils.get_minus_zone_models_path(), "train_tools", "HunyuanDiT") + + # python ./hydit/data_loader/csv2arrow.py ./dataset/porcelain/csvfile/image_text.csv ./dataset/porcelain/arrows + arrows_dir = os.path.join(workspace_dir, "arrows") + os.makedirs(arrows_dir, exist_ok=True) + csv2arrow_exec = os.path.join( + HYDiT_tool_dir, "hydit", "data_loader", "csv2arrow.py") + try: + subprocess.run([ + sys.executable, csv2arrow_exec, csv_filename, arrows_dir + ], check=True) + except Exception as e: + raise Exception(f"生成arrow文件时出现异常,详细信息请查看控制台...") + + arrows_files = os.listdir(arrows_dir) + + dataset_yaml_path = os.path.join(workspace_dir, "dataset.yaml") + + import yaml + with open(dataset_yaml_path, "w", encoding="utf-8") as f: + yaml.dump({ + "source": [os.path.join(arrows_dir, arrow_file) for arrow_file in arrows_files], + }, f) + + print(f"dataset.yaml文件已生成: {dataset_yaml_path}") + + idk_exec = os.path.join( + os.path.dirname(__file__), "hook_HYDiT_idk_run.py") + + try: + from index_kits import __version__ + except Exception as e: + subprocess.run([ + sys.executable, "-m", "pip", "install", "-e", os.path.join( + HYDiT_tool_dir, "IndexKits") + ], check=True) + # idk base -c dataset/yamls/porcelain.yaml -t dataset/porcelain/jsons/porcelain.json + + dataset_json = os.path.join(workspace_dir, "dataset.json") + try: + subprocess.run([ + sys.executable, idk_exec, "base", "-c", dataset_yaml_path, "-t", dataset_json, + ], check=True) + except Exception as e: + raise Exception(f"生成index文件时出现异常,详细信息请查看控制台...") + + print(f"index文件已生成: {dataset_json}") + + dataset_mt_yaml_path = os.path.join(workspace_dir, "dataset_mt.yaml") + + with open(dataset_mt_yaml_path, "w", encoding="utf-8") as f: + yaml.dump({ + "src": [dataset_json], + "base_size": resolution, + "target_ratios": ["1:1", "3:4", "4:3", "16:9", "9:16"], + }, f) + + print(f"dataset_mt.yaml文件已生成: {dataset_mt_yaml_path}") + + # idk multireso -c dataset/yamls/porcelain_mt.yaml -t dataset/porcelain/jsons/porcelain_mt.json + + dataset_mt_json = os.path.join(workspace_dir, "dataset_mt.json") + try: + subprocess.run([ + sys.executable, idk_exec, "multireso", "-c", dataset_mt_yaml_path, "-t", dataset_mt_json, + ], check=True) + except Exception as e: + raise Exception(f"生成多分辨率index文件时出现异常,详细信息请查看控制台...") + + print(f"多分辨率index文件已生成: {dataset_mt_json}") + + train_config = {} + + train_config_path = os.path.join(workspace_dir, "mz_train_config.json") + + if os.path.exists(train_config_path): + with open(train_config_path, "r", encoding="utf-8") as f: + train_config = json.load(f) + +# "unet": (["auto"] + models + unet_models, {"default": "auto"}), +# "vae": (["auto"] + folders + vae_models, {"default": "auto"}), +# "text_encoder": (["auto"] + folders, {"default": "auto"}), +# "tokenizer": (["auto"] + folders, {"default": "auto"}), +# "t5_encoder": (["auto"] + folders, {"default": "auto"}), + + output_dir = os.path.join(workspace_dir, "output") + train_config.update({ + "batch_size": args.get("batch_size", 1), + "target_ratios": train_config.get("target_ratios", ["1:1", "3:4", "4:3", "16:9", "9:16"]), + "index_file": dataset_mt_json, + "epochs": args.get("epochs", 50), + "ckpt_every": args.get("save_every_n_epochs", 1), + "rope_img": f"base{resolution}", + "rope_real": args.get("rope_real", "enable") == "enable", + "image_size": resolution, + "unet_path": args.get("unet_path"), + "vae_ema_path": args.get("vae_ema_path"), + "text_encoder_path": args.get("text_encoder_path"), + "tokenizer_path": args.get("tokenizer_path"), + "t5_encoder_path": args.get("t5_encoder_path"), + "results_dir": output_dir, + "task_flag": args.get("task_flag", workspace_name), + }) + + with open(train_config_path, "w", encoding="utf-8") as f: + json.dump(train_config, f, indent=4) + + run_hook_HYDiT_pyexec(train_config_path, train_config, HYDiT_tool_dir) + return ( + args, + ) + + +def run_hook_HYDiT_pyexec(train_config_path, train_config, HYDiT_tool_dir): + pb = Utils.progress_bar(train_config.get("epochs"), "sdxl") + import traceback + import comfy.model_management + stop_server = None + + is_running = True + + def log_callback(log): + try: + comfy.model_management.throw_exception_if_processing_interrupted() + except Exception as e: + stop_server() + return is_running + + try: + resp = log + if resp.get("type") == "sample_images": + total_steps = resp.get("total_steps") + global_step = resp.get("global_step") + xy_img = get_sample_images(train_config) + if xy_img is None: + pb.update( + int(global_step), int(total_steps), None) + return is_running + max_side = max(xy_img.width, xy_img.height) + + pb.update( + int(global_step), int(total_steps), ("JPEG", xy_img, max_side)) + else: + print(f"LOG: {log}") + except Exception as e: + print(f"LOG: {log} e: {e} ") + print(f"stack: {traceback.format_exc()}") + return is_running + stop_server, port = Utils.Simple_Server(log_callback) + + exec_pyfile = os.path.join(os.path.dirname( + __file__), "hook_HYDiT_run.py",) + + try: + subprocess.run( + [sys.executable, exec_pyfile, "--sys_path", HYDiT_tool_dir, + "--master_port", str(port), "--train_config_file", train_config_path], + check=True, + ) + stop_server() + is_running = False + except Exception as e: + stop_server() + is_running = False + raise Exception(f"训练失败!!! 具体报错信息请查看控制台...") + + +def get_sample_images(train_config): + # output_name = train_config.get("output_name") + # sample_images_dir = os.path.join( + # os.path.dirname(train_config.get("dataset_config")), "sample_images" + # ) + # pil_images = [] + # pre_render_texts_x = [] + # if os.path.exists(sample_images_dir): + # image_files = os.listdir(sample_images_dir) + # image_files = list( + # filter(lambda x: x.endswith(".png"), image_files)) + # # 筛选 output_name 前缀 + # image_files = list( + # filter(lambda x: x.startswith(output_name), image_files)) + + # image_files = sorted(image_files, key=lambda x: x) + + # for image_file in image_files: + # pil_image = Image.open(os.path.join(sample_images_dir, image_file)) + # pil_images.append([pil_image]) + # pre_render_texts_x.append(image_file) + # result = Utils.xy_image( + # pre_render_images=pil_images, + # pre_render_texts_x=pre_render_texts_x, + # pre_render_texts_y=[""], + # ) + # return result + return None diff --git a/mz_train_tools_utils.py b/mz_train_tools_utils.py index 49ac5eb..c5345eb 100644 --- a/mz_train_tools_utils.py +++ b/mz_train_tools_utils.py @@ -913,3 +913,18 @@ class Utils: label_text, font=font, fill="black") return full_canvas + + def get_models_by_folder(dir_path): + models = [] + for root, dirs, files in os.walk(dir_path): + for file in files: + if file.endswith(".pth") or file.endswith(".pt") or file.endswith(".pkl") or file.endswith(".onnx") or file.endswith(".safetensors"): + models.append(os.path.join(root, file)) + return models + + def get_folders_by_folder(dir_path): + folders = [] + for root, dirs, files in os.walk(dir_path): + for dir in dirs: + folders.append(os.path.join(root, dir)) + return folders \ No newline at end of file From 464b3b5c675ab4734f5ae769ad0cf85301043e7b Mon Sep 17 00:00:00 2001 From: wailovet Date: Thu, 27 Jun 2024 23:01:02 +0800 Subject: [PATCH 05/64] 'update' --- __init__.py | 4 +- hook_HYDiT_main.py | 161 ++++++++++++++++++++++++++++++++--- hook_HYDiT_run.py | 15 ++-- mz_train_tools_core_HYDiT.py | 93 +++++++++++++------- mz_train_tools_utils.py | 2 + 5 files changed, 223 insertions(+), 52 deletions(-) diff --git a/__init__.py b/__init__.py index 189e9df..c8a6dd6 100644 --- a/__init__.py +++ b/__init__.py @@ -58,7 +58,7 @@ class MZ_KohyaSSDatasetConfig: "workspace_config": ("MZ_TT_SS_WorkspaceConfig",), "images": ("IMAGE",), "enable_bucket": (["enable", "disable"], {"default": "enable"}), - "resolution": ("INT", {"default": 512}), + "resolution": ("INT", {"default": 1024}), "num_repeats": ("INT", {"default": 1}), "batch_size": ("INT", {"default": 1}), "force_clear": (["enable", "disable"], {"default": "disable"}), @@ -539,7 +539,7 @@ class MZ_HYDiTTrain: "resolution": ("INT", {"default": 1024}), "batch_size": ("INT", {"default": 1}), "epochs": ("INT", {"default": 50}), - "save_every_n_epochs": ("INT", {"default": 10}), + "ckpt_every": ("INT", {"default": 10}), "base_lora": (["latest", "empty"], {"default": "latest"}), "sample_generate": (["enable", "disable"], {"default": "enable"}), "sample_prompt": ("STRING", {"default:": "", "dynamicPrompts": True, "multiline": True}), diff --git a/hook_HYDiT_main.py b/hook_HYDiT_main.py index d33d547..c9eaac4 100644 --- a/hook_HYDiT_main.py +++ b/hook_HYDiT_main.py @@ -83,6 +83,14 @@ T5_ENCODER = { } +global TRAIN_CONFIG + + +def set_train_config(train_config): + global TRAIN_CONFIG + TRAIN_CONFIG = train_config + + def set_t5_encoder_path(path): global T5_ENCODER T5_ENCODER['MT5'] = path @@ -135,12 +143,9 @@ def save_checkpoint(args, rank, logger, model, ema, epoch, train_steps, checkpoi cur_ckpt_save_dir = f"{checkpoint_dir}/{tag}" if rank == 0: if args.use_fp16: - try: - model.module.save_pretrained(cur_ckpt_save_dir) - except Exception as e: - model.save_pretrained(cur_ckpt_save_dir) - else: model.module.save_pretrained(cur_ckpt_save_dir) + else: + model.save_pretrained(cur_ckpt_save_dir) checkpoint_path = "[Not rank 0. Disabled output.]" @@ -183,12 +188,6 @@ def save_checkpoint(args, rank, logger, model, ema, epoch, train_steps, checkpoi except: logger.error(f"Saved failed to {checkpoint_path}") - if rank == 0 and len(dst_paths) > 0: - # Delete optimizer states to avoid occupying too much disk space. - for dst_path in dst_paths: - for opt_state_path in glob(f"{dst_path}/zero_dp_rank_*_tp_rank_00_pp_rank_00_optim_states.pt"): - os.remove(opt_state_path) - return checkpoint_path @@ -441,6 +440,7 @@ def Core(args, LOG): model, ema, start_epoch, start_epoch_step, train_steps = model_resume( args, model, ema, logger) + if args.training_parts == "lora": loraconfig = LoraConfig( r=args.rank, @@ -537,10 +537,15 @@ def Core(args, LOG): optimizer = torch.optim.AdamW( model.parameters(), lr=args.lr, weight_decay=args.weight_decay) + + + easy_sample_images(args, vae, text_encoder, tokenizer, model, embedder_t5, + target_height=768, target_width=1280, train_steps=0) # torch.autograd.set_detect_anomaly(True) # with torch.autograd.detect_anomaly(): with tqdm(total=total_steps, initial=train_steps) as pbar: + # Training loop for epoch in range(start_epoch, args.epochs): logger.info(f" Start random shuffle with seed={seed}") @@ -605,8 +610,9 @@ def Core(args, LOG): train_steps += 1 mean_loss += loss.item() pbar.update(1) - pbar.set_description(f"Epoch {epoch}, step {step}, loss {loss.item():.4f}, mean_loss {mean_loss / step:.4f}") - + pbar.set_description( + f"Epoch {epoch}, step {step}, loss {loss.item():.4f}, mean_loss {mean_loss / step:.4f}") + # =========================================================================== # Log loss values: # =========================================================================== @@ -640,9 +646,11 @@ def Core(args, LOG): gc.collect() if (train_steps % args.ckpt_every == 0 or train_steps % args.ckpt_latest_every == 0 # or train_steps == args.max_training_steps - ) and train_steps > 0: + ) and train_steps > 0: logger.info( f" Saving checkpoint at step {train_steps}.") + easy_sample_images(args, vae, text_encoder, tokenizer, model, embedder_t5, + target_height=768, target_width=1280, train_steps=train_steps) save_checkpoint(args, rank, logger, model, ema, epoch, train_steps, checkpoint_dir) @@ -708,3 +716,128 @@ def model_resume(args, model, ema, logger): " “If `resume` is True, then either `resume_split` must be true.”") return model, ema, start_epoch, start_epoch_step, train_steps + + +def easy_sample_images( + args, + vae=None, + text_encoder=None, + tokenizer=None, + model=None, + embedder_t5=None, + target_height=768, + target_width=1280, + prompt="A photo of a girl with a hat on a sunny day", + negative_prompt="", + batch_size=1, + guidance_scale=5.0, + infer_steps=20, + sampler='ddpm', + train_steps=0, +): + + with torch.no_grad(): + + from hydit.diffusion.pipeline import StableDiffusionPipeline + from diffusers import schedulers + from hydit.constants import SAMPLER_FACTORY + from hydit.modules.posemb_layers import get_fill_resize_and_crop, get_2d_rotary_pos_embed + from hydit.modules.models import HUNYUAN_DIT_CONFIG + + workspace_dir = TRAIN_CONFIG.get("workspace_dir") + sample_config_file = TRAIN_CONFIG.get("sample_config_file", None) + if sample_config_file is None: + print("sample_config_file is not set.") + return + try: + sample_config = json.load(open(sample_config_file, "r")) + except Exception as e: + print(f"Failed to load sample_config_file: {sample_config_file}") + return + sample_images_dir = os.path.join(workspace_dir, "sample_images") + os.makedirs(sample_images_dir, exist_ok=True) + + # Load sampler from factory + kwargs = SAMPLER_FACTORY[sampler]['kwargs'] + scheduler = SAMPLER_FACTORY[sampler]['scheduler'] + + # Build scheduler according to the sampler. + scheduler_class = getattr(schedulers, scheduler) + scheduler = scheduler_class(**kwargs) + + # Set timesteps for inference steps. + scheduler.set_timesteps(infer_steps, "cuda") + + def calc_rope(height, width): + model_config = HUNYUAN_DIT_CONFIG["DiT-g/2"] + patch_size = model_config['patch_size'] + head_size = model_config['hidden_size'] // model_config['num_heads'] + th = height // 8 // patch_size + tw = width // 8 // patch_size + base_size = 512 // 8 // patch_size + start, stop = get_fill_resize_and_crop((th, tw), base_size) + sub_args = [start, stop, (th, tw)] + rope = get_2d_rotary_pos_embed(head_size, *sub_args) + return rope + + + pipeline = StableDiffusionPipeline(vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=model, + scheduler=scheduler, + feature_extractor=None, + safety_checker=None, + requires_safety_checker=False, + embedder_t5=embedder_t5, + ) + pipeline.to("cuda") + + style = torch.as_tensor([0, 0] * batch_size, device="cuda") + + src_size_cond = (target_width, target_height) + size_cond = list(src_size_cond) + [target_width, target_height, 0, 0] + image_meta_size = torch.as_tensor( + [size_cond] * 2 * batch_size, device="cuda",) + + if type(sample_config) != list: + sample_config = [sample_config] + for i, sample in enumerate(sample_config): + prompt = sample.get("prompt", "") + negative_prompt = sample.get("negative_prompt", "") + guidance_scale = sample.get("cfg", guidance_scale) + infer_steps = sample.get("steps", infer_steps) + width = sample.get("width", target_width) + height = sample.get("height", target_height) + + freqs_cis_img = calc_rope(height, width) + samples = pipeline( + height=height, + width=width, + prompt=prompt, + negative_prompt=negative_prompt, + num_images_per_prompt=batch_size, + guidance_scale=guidance_scale, + num_inference_steps=infer_steps, + style=style, + return_dict=False, + use_fp16=True, + learn_sigma=args.learn_sigma, + freqs_cis_img=freqs_cis_img, + image_meta_size=image_meta_size, + )[0] + + # print("samples:",type(samples),) + # input("Press Enter to continue...") + # print("samples:",samples,) + + if type(samples) == list: + pil_image = samples[0] + else: + pil_image = samples + + sample_filename = f"{args.task_flag}_train_steps_{train_steps:07d}.png" + sample_filename_path = os.path.join(sample_images_dir, sample_filename) + pil_image.save(sample_filename_path) + + return None diff --git a/hook_HYDiT_run.py b/hook_HYDiT_run.py index 3ced37c..180e725 100644 --- a/hook_HYDiT_run.py +++ b/hook_HYDiT_run.py @@ -5,6 +5,8 @@ import os import sys from types import SimpleNamespace +import torch + # python -m torch.distributed.launch ./hook_HYDi_run.py --sys_path /data/ComfyUI/models/minus_zone_models/train_tools/HunyuanDiT/ @@ -87,11 +89,9 @@ if __name__ == "__main__": margs = hydit.config.get_args() margs.model = train_config.get("model", "DiT-g/2") - margs.task_flag = train_config.get( "task_flag", "lora_porcelain_ema_rank64") - - + margs.resume_split = train_config.get("resume_split", True) margs.ema_to_module = train_config.get("ema_to_module", True) margs.deepspeed = False @@ -109,8 +109,8 @@ if __name__ == "__main__": margs.epochs = train_config.get("epochs", 50) margs.target_ratios = train_config.get( "target_ratios", ['1:1', '3:4', '4:3', '16:9', '9:16']) - margs.rope_img = train_config.get("rope_img", "base512") - margs.image_size = train_config.get("image_size", 512) + margs.rope_img = train_config.get("rope_img", "base1024") + margs.image_size = train_config.get("image_size", 1024) margs.rope_real = train_config.get("rope_real", True) margs.index_file = train_config.get("index_file", None) @@ -122,7 +122,8 @@ if __name__ == "__main__": margs.log_every = train_config.get("log_every", 10) margs.results_dir = train_config.get("results_dir") - margs.mse_loss_weight_type = train_config.get("mse_loss_weight_type", "constant") + margs.mse_loss_weight_type = train_config.get( + "mse_loss_weight_type", "constant") for k, v in train_config.items(): if hasattr(margs, k): @@ -143,4 +144,6 @@ if __name__ == "__main__": hook_HYDiT_main.set_t5_encoder_path( train_config.get("t5_encoder_path")) + hook_HYDiT_main.set_train_config(train_config) + hook_HYDiT_main.Core(margs, LOG) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index 7c04555..ebf84e1 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -110,7 +110,7 @@ def MZ_HYDiTDatasetConfig_call(args={}): pil_images = Utils.tensors2pil_list(images) - resolution = args.get("resolution", 512) + resolution = args.get("resolution", 1024) if workspace_name is None or workspace_name == "": raise Exception("训练名称不能为空(workspace_name is required)") @@ -282,13 +282,20 @@ def MZ_HYDiTTrain_call(args={}): # "tokenizer": (["auto"] + folders, {"default": "auto"}), # "t5_encoder": (["auto"] + folders, {"default": "auto"}), + epochs = args.get("epochs", 50) output_dir = os.path.join(workspace_dir, "output") + + datetime = time.strftime("%Y%m%d_%H%M%S", time.localtime()) + output_name = f"{workspace_name}_{datetime}" + train_config.update({ + "workspace_name": workspace_name, + "workspace_dir": workspace_dir, "batch_size": args.get("batch_size", 1), "target_ratios": train_config.get("target_ratios", ["1:1", "3:4", "4:3", "16:9", "9:16"]), "index_file": dataset_mt_json, - "epochs": args.get("epochs", 50), - "ckpt_every": args.get("save_every_n_epochs", 1), + "epochs": epochs, + "ckpt_every": int(args.get("ckpt_every")), "rope_img": f"base{resolution}", "rope_real": args.get("rope_real", "enable") == "enable", "image_size": resolution, @@ -298,11 +305,31 @@ def MZ_HYDiTTrain_call(args={}): "tokenizer_path": args.get("tokenizer_path"), "t5_encoder_path": args.get("t5_encoder_path"), "results_dir": output_dir, - "task_flag": args.get("task_flag", workspace_name), + "task_flag": output_name, }) + sample_generate = args.get("sample_generate", "enable") + sample_prompt = args.get("sample_prompt", "") + sample_config_file = os.path.join(workspace_dir, "sample_config.json") + if sample_generate == "enable": + resolution = args.get("resolution") + sample_config = { + "prompt": sample_prompt, + "negative_prompt": "", + "cfg": 5.0, + "steps": 20, + "width": resolution, + "height": resolution, + } + with open(sample_config_file, "w", encoding="utf-8") as f: + json.dump(sample_config, f, indent=4, ensure_ascii=False) + + train_config["sample_config_file"] = sample_config_file + else: + train_config["sample_config_file"] = None + with open(train_config_path, "w", encoding="utf-8") as f: - json.dump(train_config, f, indent=4) + json.dump(train_config, f, indent=4, ensure_ascii=False) run_hook_HYDiT_pyexec(train_config_path, train_config, HYDiT_tool_dir) return ( @@ -311,7 +338,7 @@ def MZ_HYDiTTrain_call(args={}): def run_hook_HYDiT_pyexec(train_config_path, train_config, HYDiT_tool_dir): - pb = Utils.progress_bar(train_config.get("epochs"), "sdxl") + pb = Utils.progress_bar(0, "sdxl") import traceback import comfy.model_management stop_server = None @@ -365,30 +392,36 @@ def run_hook_HYDiT_pyexec(train_config_path, train_config, HYDiT_tool_dir): def get_sample_images(train_config): - # output_name = train_config.get("output_name") - # sample_images_dir = os.path.join( - # os.path.dirname(train_config.get("dataset_config")), "sample_images" - # ) - # pil_images = [] - # pre_render_texts_x = [] - # if os.path.exists(sample_images_dir): - # image_files = os.listdir(sample_images_dir) - # image_files = list( - # filter(lambda x: x.endswith(".png"), image_files)) - # # 筛选 output_name 前缀 - # image_files = list( - # filter(lambda x: x.startswith(output_name), image_files)) + if train_config.get("sample_config_file") is None: + return None - # image_files = sorted(image_files, key=lambda x: x) + from PIL import Image + output_name = train_config.get("task_flag") - # for image_file in image_files: - # pil_image = Image.open(os.path.join(sample_images_dir, image_file)) - # pil_images.append([pil_image]) - # pre_render_texts_x.append(image_file) - # result = Utils.xy_image( - # pre_render_images=pil_images, - # pre_render_texts_x=pre_render_texts_x, - # pre_render_texts_y=[""], - # ) - # return result + workspace_dir = train_config.get("workspace_dir") + + sample_images_dir = os.path.join(workspace_dir, "sample_images") + + pil_images = [] + pre_render_texts_x = [] + if os.path.exists(sample_images_dir): + image_files = os.listdir(sample_images_dir) + image_files = list( + filter(lambda x: x.endswith(".png"), image_files)) + # 筛选 output_name 前缀 + image_files = list( + filter(lambda x: x.startswith(output_name), image_files)) + + image_files = sorted(image_files, key=lambda x: x) + + for image_file in image_files: + pil_image = Image.open(os.path.join(sample_images_dir, image_file)) + pil_images.append([pil_image]) + pre_render_texts_x.append(image_file) + result = Utils.xy_image( + pre_render_images=pil_images, + pre_render_texts_x=pre_render_texts_x, + pre_render_texts_y=[""], + ) + return result return None diff --git a/mz_train_tools_utils.py b/mz_train_tools_utils.py index c5345eb..b67a9d2 100644 --- a/mz_train_tools_utils.py +++ b/mz_train_tools_utils.py @@ -787,6 +787,8 @@ class Utils: # 去掉pre_render_texts_x中所有相同字符串前缀 def get_common_prefix(pre_render_texts_x): + if len(pre_render_texts_x) == 0: + return "" common_prefix = "" for i in range(len(pre_render_texts_x[0])): c = pre_render_texts_x[0][i] From eee407c8103636b4538bf34587f3bfb1121f3f48 Mon Sep 17 00:00:00 2001 From: wailovet Date: Thu, 27 Jun 2024 23:31:46 +0800 Subject: [PATCH 06/64] 'update' --- __init__.py | 1 + mz_train_tools_core_HYDiT.py | 157 +++++++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+) diff --git a/__init__.py b/__init__.py index c8a6dd6..8a53abd 100644 --- a/__init__.py +++ b/__init__.py @@ -540,6 +540,7 @@ class MZ_HYDiTTrain: "batch_size": ("INT", {"default": 1}), "epochs": ("INT", {"default": 50}), "ckpt_every": ("INT", {"default": 10}), + "ema_to_module": (["enable", "disable"], {"default": "enable"}), "base_lora": (["latest", "empty"], {"default": "latest"}), "sample_generate": (["enable", "disable"], {"default": "enable"}), "sample_prompt": ("STRING", {"default:": "", "dynamicPrompts": True, "multiline": True}), diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index ebf84e1..51f2033 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -129,7 +129,163 @@ def MZ_HYDiTDatasetConfig_call(args={}): ) +HYDiT_MODEL = { + "HunyuanDiT/t2i/model/pytorch_model_ema.pt": { + "url": "https://www.modelscope.cn/api/v1/models/modelscope/HunyuanDiT/repo?Revision=master&FilePath=t2i%2Fmodel%2Fpytorch_model_ema.pt", + }, + "HunyuanDiT/t2i/model/pytorch_model_module.pt": { + "url": "https://www.modelscope.cn/api/v1/models/modelscope/HunyuanDiT/repo?Revision=master&FilePath=t2i%2Fmodel%2Fpytorch_model_module.pt", + }, + "HunyuanDiT/t2i/sdxl-vae-fp16-fix/diffusion_pytorch_model.safetensors": { + "url": "https://www.modelscope.cn/api/v1/models/modelscope/HunyuanDiT/repo?Revision=master&FilePath=t2i%2Fsdxl-vae-fp16-fix%2Fdiffusion_pytorch_model.safetensors", + }, + "HunyuanDiT/t2i/sdxl-vae-fp16-fix/config.json": { + "url": "https://www.modelscope.cn/api/v1/models/modelscope/HunyuanDiT/repo?Revision=master&FilePath=t2i%2Fsdxl-vae-fp16-fix%2Fconfig.json", + }, + "HunyuanDiT/t2i/clip_text_encoder/pytorch_model.bin": { + "url": "https://www.modelscope.cn/api/v1/models/modelscope/HunyuanDiT/repo?Revision=master&FilePath=t2i%2Fclip_text_encoder%2Fpytorch_model.bin", + }, + "HunyuanDiT/t2i/clip_text_encoder/config.json": { + "url": "https://www.modelscope.cn/api/v1/models/modelscope/HunyuanDiT/repo?Revision=master&FilePath=t2i%2Fclip_text_encoder%2Fconfig.json", + }, + "HunyuanDiT/t2i/tokenizer/special_tokens_map.json": { + "url": "https://www.modelscope.cn/api/v1/models/modelscope/HunyuanDiT/repo?Revision=master&FilePath=t2i%2Ftokenizer%2Fspecial_tokens_map.json", + }, + "HunyuanDiT/t2i/tokenizer/tokenizer_config.json": { + "url": "https://www.modelscope.cn/api/v1/models/modelscope/HunyuanDiT/repo?Revision=master&FilePath=t2i%2Ftokenizer%2Ftokenizer_config.json", + }, + "HunyuanDiT/t2i/tokenizer/vocab.txt": { + "url": "https://www.modelscope.cn/api/v1/models/modelscope/HunyuanDiT/repo?Revision=master&FilePath=t2i%2Ftokenizer%2Fvocab.txt", + }, + "HunyuanDiT/t2i/tokenizer/vocab_org.txt": { + "url": "https://www.modelscope.cn/api/v1/models/modelscope/HunyuanDiT/repo?Revision=master&FilePath=t2i%2Ftokenizer%2Fvocab_org.txt", + }, + "HunyuanDiT/t2i/mt5/config.json": { + "url": "https://www.modelscope.cn/api/v1/models/modelscope/HunyuanDiT/repo?Revision=master&FilePath=t2i%2Fmt5%2Fconfig.json", + }, + "HunyuanDiT/t2i/mt5/generation_config.json": { + "url": "https://www.modelscope.cn/api/v1/models/modelscope/HunyuanDiT/repo?Revision=master&FilePath=t2i%2Fmt5%2Fgeneration_config.json", + }, + "HunyuanDiT/t2i/mt5/special_tokens_map.json": { + "url": "https://www.modelscope.cn/api/v1/models/modelscope/HunyuanDiT/repo?Revision=master&FilePath=t2i%2Fmt5%2Fspecial_tokens_map.json", + }, + "HunyuanDiT/t2i/mt5/spiece.model": { + "url": "https://www.modelscope.cn/api/v1/models/modelscope/HunyuanDiT/repo?Revision=master&FilePath=t2i%2Fmt5%2Fspiece.model", + }, + "HunyuanDiT/t2i/mt5/tokenizer_config.json": { + "url": "https://www.modelscope.cn/api/v1/models/modelscope/HunyuanDiT/repo?Revision=master&FilePath=t2i%2Fmt5%2Ftokenizer_config.json", + }, + "HunyuanDiT/t2i/mt5/pytorch_model.bin": { + "url": "https://www.modelscope.cn/api/v1/models/modelscope/HunyuanDiT/repo?Revision=master&FilePath=t2i%2Fmt5%2Fpytorch_model.bin", + }, +} + + +def check_model_auto_download(args): + unet_path = args.get("unet_path", "auto") + hunyuan_base_path = os.path.join( + Utils.get_comfyui_models_path(), "hunyuan") + if unet_path == "auto": + ema_to_module = args.get("ema_to_module", "enable") == "enable" + if ema_to_module: + download_file = "HunyuanDiT/t2i/model/pytorch_model_ema.pt" + else: + download_file = "HunyuanDiT/t2i/model/pytorch_model_module.pt" + download_fullpath = os.path.join( + hunyuan_base_path, download_file) + os.makedirs(os.path.dirname(download_fullpath), exist_ok=True) + success_path = Utils.download_file( + download_fullpath, + HYDiT_MODEL[download_file]["url"], + ) + if os.path.exists(success_path): + args["unet_path"] = download_fullpath + + vae_ema_path = args.get("vae_ema_path", "auto") + if vae_ema_path == "auto": + download_files = [] + prefix = "HunyuanDiT/t2i/sdxl-vae-fp16-fix/" + for key in HYDiT_MODEL: + if key.startswith(prefix): + download_files.append(key) + + for download_file in download_files: + download_fullpath = os.path.join( + hunyuan_base_path, download_file) + os.makedirs(os.path.dirname(download_fullpath), exist_ok=True) + Utils.download_file( + download_fullpath, + HYDiT_MODEL[download_file]["url"], + ) + + args["vae_ema_path"] = os.path.join( + hunyuan_base_path, prefix) + + text_encoder_path = args.get("text_encoder_path", "auto") + if text_encoder_path == "auto": + download_files = [] + prefix = "HunyuanDiT/t2i/clip_text_encoder/" + for key in HYDiT_MODEL: + if key.startswith(prefix): + download_files.append(key) + + for download_file in download_files: + download_fullpath = os.path.join( + hunyuan_base_path, download_file) + os.makedirs(os.path.dirname(download_fullpath), exist_ok=True) + Utils.download_file( + download_fullpath, + HYDiT_MODEL[download_file]["url"], + ) + + args["text_encoder_path"] = os.path.join( + hunyuan_base_path, prefix) + + tokenizer_path = args.get("tokenizer_path", "auto") + if tokenizer_path == "auto": + download_files = [] + prefix = "HunyuanDiT/t2i/tokenizer/" + for key in HYDiT_MODEL: + if key.startswith(prefix): + download_files.append(key) + + for download_file in download_files: + download_fullpath = os.path.join( + hunyuan_base_path, download_file) + os.makedirs(os.path.dirname(download_fullpath), exist_ok=True) + Utils.download_file( + download_fullpath, + HYDiT_MODEL[download_file]["url"], + ) + + args["tokenizer_path"] = os.path.join( + hunyuan_base_path, prefix) + + t5_encoder_path = args.get("t5_encoder_path", "auto") + if t5_encoder_path == "auto": + download_files = [] + prefix = "HunyuanDiT/t2i/mt5/" + for key in HYDiT_MODEL: + if key.startswith(prefix): + download_files.append(key) + + for download_file in download_files: + download_fullpath = os.path.join( + hunyuan_base_path, download_file) + os.makedirs(os.path.dirname(download_fullpath), exist_ok=True) + Utils.download_file( + download_fullpath, + HYDiT_MODEL[download_file]["url"], + ) + + args["t5_encoder_path"] = os.path.join( + hunyuan_base_path, prefix) + + return args + + def MZ_HYDiTTrain_call(args={}): + args = check_model_auto_download(args) # raise Exception(args) resolution = args.get("resolution") @@ -292,6 +448,7 @@ def MZ_HYDiTTrain_call(args={}): "workspace_name": workspace_name, "workspace_dir": workspace_dir, "batch_size": args.get("batch_size", 1), + "ema_to_module": args.get("ema_to_module", "enable") == "enable", "target_ratios": train_config.get("target_ratios", ["1:1", "3:4", "4:3", "16:9", "9:16"]), "index_file": dataset_mt_json, "epochs": epochs, From cfe811fd88f17f8f6ec524681a884b4c0bb8419f Mon Sep 17 00:00:00 2001 From: wailovet Date: Thu, 27 Jun 2024 23:56:01 +0800 Subject: [PATCH 07/64] 'update' --- mz_train_tools_core_HYDiT.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index 51f2033..8769fb2 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -195,8 +195,8 @@ def check_model_auto_download(args): hunyuan_base_path, download_file) os.makedirs(os.path.dirname(download_fullpath), exist_ok=True) success_path = Utils.download_file( - download_fullpath, HYDiT_MODEL[download_file]["url"], + download_fullpath, ) if os.path.exists(success_path): args["unet_path"] = download_fullpath @@ -214,8 +214,8 @@ def check_model_auto_download(args): hunyuan_base_path, download_file) os.makedirs(os.path.dirname(download_fullpath), exist_ok=True) Utils.download_file( - download_fullpath, HYDiT_MODEL[download_file]["url"], + download_fullpath, ) args["vae_ema_path"] = os.path.join( @@ -234,8 +234,8 @@ def check_model_auto_download(args): hunyuan_base_path, download_file) os.makedirs(os.path.dirname(download_fullpath), exist_ok=True) Utils.download_file( - download_fullpath, HYDiT_MODEL[download_file]["url"], + download_fullpath, ) args["text_encoder_path"] = os.path.join( @@ -254,8 +254,8 @@ def check_model_auto_download(args): hunyuan_base_path, download_file) os.makedirs(os.path.dirname(download_fullpath), exist_ok=True) Utils.download_file( - download_fullpath, HYDiT_MODEL[download_file]["url"], + download_fullpath, ) args["tokenizer_path"] = os.path.join( @@ -274,8 +274,8 @@ def check_model_auto_download(args): hunyuan_base_path, download_file) os.makedirs(os.path.dirname(download_fullpath), exist_ok=True) Utils.download_file( - download_fullpath, HYDiT_MODEL[download_file]["url"], + download_fullpath, ) args["t5_encoder_path"] = os.path.join( From 3c3a92a836da7f45ce7577dfb5561df3072bd94a Mon Sep 17 00:00:00 2001 From: wailovet Date: Fri, 28 Jun 2024 00:21:43 +0800 Subject: [PATCH 08/64] 'update' --- mz_train_tools_core_HYDiT.py | 65 ++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index 8769fb2..bd9b730 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -194,12 +194,16 @@ def check_model_auto_download(args): download_fullpath = os.path.join( hunyuan_base_path, download_file) os.makedirs(os.path.dirname(download_fullpath), exist_ok=True) - success_path = Utils.download_file( - HYDiT_MODEL[download_file]["url"], - download_fullpath, - ) - if os.path.exists(success_path): + + if os.path.exists(download_fullpath): args["unet_path"] = download_fullpath + else: + success_path = Utils.download_file( + HYDiT_MODEL[download_file]["url"], + download_fullpath, + ) + if os.path.exists(success_path): + args["unet_path"] = download_fullpath vae_ema_path = args.get("vae_ema_path", "auto") if vae_ema_path == "auto": @@ -213,10 +217,12 @@ def check_model_auto_download(args): download_fullpath = os.path.join( hunyuan_base_path, download_file) os.makedirs(os.path.dirname(download_fullpath), exist_ok=True) - Utils.download_file( - HYDiT_MODEL[download_file]["url"], - download_fullpath, - ) + + if not os.path.exists(download_fullpath): + Utils.download_file( + HYDiT_MODEL[download_file]["url"], + download_fullpath, + ) args["vae_ema_path"] = os.path.join( hunyuan_base_path, prefix) @@ -233,10 +239,12 @@ def check_model_auto_download(args): download_fullpath = os.path.join( hunyuan_base_path, download_file) os.makedirs(os.path.dirname(download_fullpath), exist_ok=True) - Utils.download_file( - HYDiT_MODEL[download_file]["url"], - download_fullpath, - ) + + if not os.path.exists(download_fullpath): + Utils.download_file( + HYDiT_MODEL[download_file]["url"], + download_fullpath, + ) args["text_encoder_path"] = os.path.join( hunyuan_base_path, prefix) @@ -253,10 +261,12 @@ def check_model_auto_download(args): download_fullpath = os.path.join( hunyuan_base_path, download_file) os.makedirs(os.path.dirname(download_fullpath), exist_ok=True) - Utils.download_file( - HYDiT_MODEL[download_file]["url"], - download_fullpath, - ) + + if not os.path.exists(download_fullpath): + Utils.download_file( + HYDiT_MODEL[download_file]["url"], + download_fullpath, + ) args["tokenizer_path"] = os.path.join( hunyuan_base_path, prefix) @@ -273,10 +283,12 @@ def check_model_auto_download(args): download_fullpath = os.path.join( hunyuan_base_path, download_file) os.makedirs(os.path.dirname(download_fullpath), exist_ok=True) - Utils.download_file( - HYDiT_MODEL[download_file]["url"], - download_fullpath, - ) + + if not os.path.exists(download_fullpath): + Utils.download_file( + HYDiT_MODEL[download_file]["url"], + download_fullpath, + ) args["t5_encoder_path"] = os.path.join( hunyuan_base_path, prefix) @@ -284,7 +296,18 @@ def check_model_auto_download(args): return args +def check_required(): + try: + import pandas + + except Exception as e: + subprocess.run([ + sys.executable, "-m", "pip", "install", "pandas" + ], check=True) + + def MZ_HYDiTTrain_call(args={}): + check_required() args = check_model_auto_download(args) # raise Exception(args) resolution = args.get("resolution") From 118f9abca5b4803ffc26f096c7e881a59c3e6a7a Mon Sep 17 00:00:00 2001 From: wailovet Date: Fri, 28 Jun 2024 00:23:09 +0800 Subject: [PATCH 09/64] 'update' --- mz_train_tools_core_HYDiT.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index bd9b730..f3855ca 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -299,11 +299,18 @@ def check_model_auto_download(args): def check_required(): try: import pandas - except Exception as e: subprocess.run([ sys.executable, "-m", "pip", "install", "pandas" ], check=True) + + try: + import pyarrow + except Exception as e: + subprocess.run([ + sys.executable, "-m", "pip", "install", "pyarrow" + ], check=True) + def MZ_HYDiTTrain_call(args={}): From 9602e6f5f8f91d3326186c00cd7a77a3b137f510 Mon Sep 17 00:00:00 2001 From: wailovet Date: Fri, 28 Jun 2024 00:24:03 +0800 Subject: [PATCH 10/64] 'update' --- mz_train_tools_core_HYDiT.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index f3855ca..936a876 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -311,6 +311,13 @@ def check_required(): sys.executable, "-m", "pip", "install", "pyarrow" ], check=True) + try: + import diffusers + except Exception as e: + subprocess.run([ + sys.executable, "-m", "pip", "install", "diffusers" + ], check=True) + def MZ_HYDiTTrain_call(args={}): From 108106df5113e4cbc4d24ad04ea635351275c09d Mon Sep 17 00:00:00 2001 From: wailovet Date: Fri, 28 Jun 2024 00:24:25 +0800 Subject: [PATCH 11/64] 'update' --- mz_train_tools_core_HYDiT.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index 936a876..3cce23e 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -318,6 +318,13 @@ def check_required(): sys.executable, "-m", "pip", "install", "diffusers" ], check=True) + try: + import transformers + except Exception as e: + subprocess.run([ + sys.executable, "-m", "pip", "install", "transformers" + ], check=True) + def MZ_HYDiTTrain_call(args={}): From 638b42a7e963a4854734a85a045b19817f5655cf Mon Sep 17 00:00:00 2001 From: wailovet Date: Fri, 28 Jun 2024 00:28:47 +0800 Subject: [PATCH 12/64] 'update' --- mz_train_tools_core_HYDiT.py | 60 +++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index 3cce23e..bf5a457 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -297,34 +297,38 @@ def check_model_auto_download(args): def check_required(): - try: - import pandas - except Exception as e: - subprocess.run([ - sys.executable, "-m", "pip", "install", "pandas" - ], check=True) - - try: - import pyarrow - except Exception as e: - subprocess.run([ - sys.executable, "-m", "pip", "install", "pyarrow" - ], check=True) - - try: - import diffusers - except Exception as e: - subprocess.run([ - sys.executable, "-m", "pip", "install", "diffusers" - ], check=True) - - try: - import transformers - except Exception as e: - subprocess.run([ - sys.executable, "-m", "pip", "install", "transformers" - ], check=True) - + # timm==0.9.5 + # diffusers==0.21.2 + # peft==0.10.0 + # protobuf==3.19.0 + # torchvision==0.14.1 + # transformers==4.39.1 + # peft==0.10.0 + # accelerate==0.29.3 + # loguru==0.7.2 + # einops==0.7.0 + # sentencepiece==0.1.99 + # cuda-python==11.7.1 + # onnxruntime==1.12.1 + # onnx==1.12.0 + # nvidia-pyindex==1.0.9 + # onnx-graphsurgeon==0.3.27 + # polygraphy==0.47.1 + # pandas==2.0.3 + # gradio==3.50.2 + # deepspeed==0.6.3 + # pyarrow==16.1.0 + packages = [ + "pandas", "pyarrow", "diffusers", "transformers", + "timm", "peft", "accelerate", "loguru", "einops", "sentencepiece", + "polygraphy" + ] + for package in packages: + try: + __import__(package) + except ImportError: + subprocess.run([sys.executable, "-m", "pip", + "install", package], check=True) def MZ_HYDiTTrain_call(args={}): From 50e5a6829b8326530d4db50d7bed06cb3ed0dad1 Mon Sep 17 00:00:00 2001 From: wailovet Date: Fri, 28 Jun 2024 00:30:59 +0800 Subject: [PATCH 13/64] 'update' --- mz_train_tools_core_HYDiT.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index bf5a457..62e3bb5 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -321,7 +321,7 @@ def check_required(): packages = [ "pandas", "pyarrow", "diffusers", "transformers", "timm", "peft", "accelerate", "loguru", "einops", "sentencepiece", - "polygraphy" + "polygraphy", "deepspeed" ] for package in packages: try: From 072603a28686fc5e5052506d77d441ab2d0a89c3 Mon Sep 17 00:00:00 2001 From: wailovet Date: Fri, 28 Jun 2024 00:36:16 +0800 Subject: [PATCH 14/64] 'update' --- hook_HYDiT_config.py | 235 +++++++++++++++++++++++++++++++++++ hook_HYDiT_run.py | 11 +- mz_train_tools_core_HYDiT.py | 2 +- 3 files changed, 242 insertions(+), 6 deletions(-) create mode 100644 hook_HYDiT_config.py diff --git a/hook_HYDiT_config.py b/hook_HYDiT_config.py new file mode 100644 index 0000000..a98fcfa --- /dev/null +++ b/hook_HYDiT_config.py @@ -0,0 +1,235 @@ + + +import argparse + +from hydit.constants import * +from hydit.modules.models import HUNYUAN_DIT_CONFIG, HUNYUAN_DIT_MODELS +from hydit.diffusion.gaussian_diffusion import ModelVarType + + +def model_var_type(value): + try: + return ModelVarType[value] + except KeyError: + raise ValueError( + f"Invalid choice '{value}', valid choices are {[v.name for v in ModelVarType]}") + + +def get_args(default_args=None): + parser = argparse.ArgumentParser() + parser.add_argument("--task-flag", type=str) + + # General Setting + parser.add_argument("--batch-size", type=int, + default=1, help="Per-GPU batch size") + parser.add_argument('--seed', type=int, default=42, + help="A seed for all the prompts.") + parser.add_argument("--use-fp16", action="store_true", + help="Use FP16 precision.") + parser.add_argument("--no-fp16", dest="use_fp16", action="store_false") + parser.set_defaults(use_fp16=True) + parser.add_argument("--extra-fp16", action="store_true", + help="Use extra fp16 for vae and text_encoder.") + + # HunYuan-DiT + parser.add_argument( + "--model", type=str, choices=list(HUNYUAN_DIT_CONFIG.keys()), default='DiT-g/2') + parser.add_argument("--image-size", type=int, nargs='+', default=[1024, 1024], + help='Image size (h, w). If a single value is provided, the image will be treated to ' + '(value, value).') + parser.add_argument("--qk-norm", action="store_true", + help="Query Key normalization. See http://arxiv.org/abs/2302.05442 for details.") + parser.set_defaults(qk_norm=True) + parser.add_argument( + "--norm", type=str, choices=["rms", "laryer"], default="layer", help="Normalization layer type") + parser.add_argument("--text-states-dim", type=int, + default=1024, help="Hidden size of CLIP text encoder.") + parser.add_argument("--text-len", type=int, default=77, + help="Token length of CLIP text encoder output.") + parser.add_argument("--text-states-dim-t5", type=int, + default=2048, help="Hidden size of CLIP text encoder.") + parser.add_argument("--text-len-t5", type=int, default=256, + help="Token length of T5 text encoder output.") + + # LoRA config + parser.add_argument("--training-parts", type=str, default='all', + choices=['all', 'lora'], help="Training parts") + parser.add_argument("--rank", type=int, default=64, help="Rank of LoRA") + parser.add_argument("--lora-ckpt", type=str, + default=None, help="LoRA checkpoint") + parser.add_argument('--target-modules', type=str, nargs='+', default=['Wqkv', 'q_proj', 'kv_proj', 'out_proj'], + help="Target modules for LoRA fine tune") + parser.add_argument("--output-merge-path", type=str, + default=None, help="Output path for merged model") + + # controlnet config + parser.add_argument("--control-type", type=str, default='canny', + choices=['canny', 'depth', 'pose'], help="Controlnet condition type") + parser.add_argument("--control-weight", type=float, + default=1.0, help="Controlnet weight") + parser.add_argument("--condition-image-path", type=str, + default=None, help="Inference condition image path") + + # Diffusion + parser.add_argument("--learn-sigma", action="store_true", + help="Learn extra channels for sigma.") + parser.add_argument("--no-learn-sigma", + dest="learn_sigma", action="store_false") + parser.set_defaults(learn_sigma=True) + parser.add_argument("--predict-type", type=str, default="v_prediction", + help="Diffusion predict type") + parser.add_argument("--noise-schedule", type=str, default="scaled_linear", + help="Noise schedule") + parser.add_argument("--beta-start", type=float, + default=0.00085, help="Beta start value") + parser.add_argument("--beta-end", type=float, + default=0.03, help="Beta end value") + parser.add_argument("--sigma-small", action="store_true") + parser.add_argument("--mse-loss-weight-type", type=str, default="constant", + help="Min-SNR-gamma. Can be constant or min_snr_ where gamma is a integer. 5 is recommended in the paper.") + parser.add_argument("--model-var-type", type=model_var_type, + default=None, help="Specify the model variable type.") + parser.add_argument("--noise-offset", type=float, default=0.0, + help="Add extra noise to the input image.") + + # ======================================================================================================== + # Inference + # ======================================================================================================== + + # Basic Setting + parser.add_argument("--prompt", type=str, default="一只小猫", + help="The prompt for generating images.") + parser.add_argument("--model-root", type=str, + default="ckpts", help="Model root path.") + + # Model setting + parser.add_argument("--load-key", type=str, choices=[ + "ema", "module", "distill", 'merge'], default="ema", help="Load model key for HunYuanDiT checkpoint.") + parser.add_argument('--size-cond', type=int, nargs='+', default=[1024, 1024], + help="Size condition used in sampling. 2 values are required for height and width. " + "If a single value is provided, the image will be treated to (value, value).") + parser.add_argument('--target-ratios', type=str, nargs='+', default=None, + help="Target ratios for multi-resolution training.") + parser.add_argument("--cfg-scale", type=float, default=6.0, + help="Guidance scale for classifier-free.") + parser.add_argument("--negative", type=str, + default=None, help="Negative prompt.") + + # Acceleration + parser.add_argument("--infer-mode", type=str, + choices=["fa", "torch", "trt"], default="torch", help="Inference mode") + parser.add_argument("--onnx-workdir", type=str, + default="onnx_model", help="Path to save ONNX model") + + # Sampling + parser.add_argument("--sampler", type=str, + default="ddpm", help="Diffusion sampler") + parser.add_argument("--infer-steps", type=int, + default=100, help="Inference steps") + + # Prompt enhancement + parser.add_argument("--enhance", action="store_true", + help="Enhance prompt with dialoggen.") + parser.add_argument("--no-enhance", dest="enhance", action="store_false") + parser.add_argument( + "--load-4bit", help="load DialogGen model with 4bit quantization.", action="store_true") + parser.set_defaults(enhance=True) + + # App + parser.add_argument("--lang", type=str, default="zh", + choices=["zh", "en"], help="Language") + + # ======================================================================================================== + # Training + # ======================================================================================================== + + # Basic Setting + parser.add_argument("--lr", type=float, default=1e-4) + parser.add_argument("--epochs", type=int, default=1400) + parser.add_argument("--max-training-steps", type=int, default=10_000_000) + parser.add_argument("--gc-interval", type=int, default=40, + help='To address the memory bottleneck encountered during the preprocessing of the dataset,' + ' memory fragments are reclaimed here by invoking the gc.collect() function.') + parser.add_argument("--log-every", type=int, default=100) + parser.add_argument("--ckpt-every", type=int, default=100_000, + help="Create a ckpt every a few steps.") + parser.add_argument("--ckpt-latest-every", type=int, default=10_000, + help="Create a ckpt named `latest.pt` every a few steps.") + parser.add_argument("--num-workers", type=int, default=4) + parser.add_argument("--global-seed", type=int, default=1234) + parser.add_argument("--warmup-min-lr", type=float, default=1e-6) + parser.add_argument("--warmup-num-steps", type=float, default=0) + parser.add_argument("--weight-decay", type=float, + default=0, help="weight-decay in optimizer") + parser.add_argument("--rope-img", type=str, default=None, choices=['extend', 'base512', 'base1024'], + help="Extend or interpolate the positional embedding of the image.") + parser.add_argument("--rope-real", action="store_true", + help="Use real part and imaginary part separately for RoPE.") + + # Classifier-free + parser.add_argument("--uncond-p", type=float, default=0.2, + help="The probability of dropping training text used for CLIP feature extraction") + parser.add_argument("--uncond-p-t5", type=float, default=0.2, + help="The probability of dropping training text used for mT5 feature extraction") + + # Directory + parser.add_argument("--results-dir", type=str, default="results") + parser.add_argument("--resume", type=str, default=None, + help="Resume experiment from a checkpoint") + parser.add_argument("--no-strict", dest="strict", + action="store_false", help="Strict loading of checkpoint") + parser.set_defaults(strict=True) + parser.add_argument("--resume-deepspeed", action="store_true", + help="Resume model and ema states from a checkpoint saved by Deepspeed version of DIT.") + parser.add_argument("--resume-split", action="store_true", + help="Resume model and ema states from two checkpoint separated from DeepSpeed ckpt.") + parser.add_argument("--ema-to-module", action="store_true", + help="If true, initialize the module with EMA weights.") + parser.add_argument("--module-to-ema", action="store_true", + help="if true, initialize the ema with Module weights.") + + # Dataset + parser.add_argument("--index-file", type=str, nargs='+', + help="During training, provide a JSON file with data indices.") + parser.add_argument("--random-flip", action="store_true", + help="Random flip image") + parser.add_argument("--reset-loader", action="store_true", + help="Reset the data loader. It is useful when resuming from a checkpoint but switch to a new dataset.") + parser.add_argument("--multireso", action="store_true", + help="Use multi-resolution training.") + parser.add_argument("--reso-step", type=int, default=None, + help="Step size for multi-resolution training.") + + # Additional condition + parser.add_argument("--random-shrink-size-cond", action="store_true", + help="Randomly shrink the original size condition.") + parser.add_argument("--merge-src-cond", action="store_true", + help="Merge the source condition into a single value.") + + # EMA Model + parser.add_argument("--use-ema", action="store_true", help="Use EMA model") + parser.add_argument("--ema-dtype", type=str, choices=['fp16', 'fp32', 'none'], default="none", + help="EMA data type. If none, use the same data type as the model.") + parser.add_argument("--ema-decay", type=float, default=None, + help="EMA decay rate. If None, use the default value of the model.") + parser.add_argument("--ema-warmup", action="store_true", + help="EMA warmup. If True, perform ema_decay warmup from 0 to ema_decay.") + parser.add_argument("--ema-warmup-power", type=float, default=None, + help="EMA power. If None, use the default value of the model.") + parser.add_argument("--ema-reset-decay", action="store_true", + help="Reset EMA decay to 0 and restart increasing the EMA decay." + "Only works when --ema-warmup is enabled.") + # Acceleration + parser.add_argument("--use-flash-attn", action="store_true", help="During training, " + "flash attention is used to accelerate training.") + parser.add_argument("--no-flash-attn", dest="use_flash_attn", + action="store_false", help="During training, flash attention is not used to accelerate training.") + parser.add_argument("--use-zero-stage", type=int, default=1, + help="Use AngelPTM zero stage. Support 2 and 3") + parser.add_argument("--grad-accu-steps", type=int, + default=1, help="Gradient accumulation steps.") + + + args = parser.parse_args(default_args) + + return args diff --git a/hook_HYDiT_run.py b/hook_HYDiT_run.py index 180e725..3aa8eed 100644 --- a/hook_HYDiT_run.py +++ b/hook_HYDiT_run.py @@ -8,9 +8,6 @@ from types import SimpleNamespace import torch -# python -m torch.distributed.launch ./hook_HYDi_run.py --sys_path /data/ComfyUI/models/minus_zone_models/train_tools/HunyuanDiT/ - - class SimpleNamespaceCNWarrper(SimpleNamespace): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -64,8 +61,12 @@ if __name__ == "__main__": from . import hook_HYDiT_main except Exception as e: import hook_HYDiT_main + try: + from . import hook_HYDiT_config + except Exception as e: + import hook_HYDiT_config - import hydit.config + def _handle_conflict_error(self, *args, **kwargs): pass @@ -86,7 +87,7 @@ if __name__ == "__main__": argparse.ArgumentParser._handle_conflict_error = _handle_conflict_error argparse.ArgumentParser._handle_conflict_resolve = _handle_conflict_error argparse.ArgumentParser.parse_args = parse_args - margs = hydit.config.get_args() + margs = hook_HYDiT_config.get_args() margs.model = train_config.get("model", "DiT-g/2") margs.task_flag = train_config.get( diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index 62e3bb5..bf5a457 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -321,7 +321,7 @@ def check_required(): packages = [ "pandas", "pyarrow", "diffusers", "transformers", "timm", "peft", "accelerate", "loguru", "einops", "sentencepiece", - "polygraphy", "deepspeed" + "polygraphy" ] for package in packages: try: From 390f397bc9b2b5eee93530eae87e6d100a3dda4e Mon Sep 17 00:00:00 2001 From: wailovet Date: Fri, 28 Jun 2024 00:37:26 +0800 Subject: [PATCH 15/64] 'update' --- hook_HYDiT_main.py | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/hook_HYDiT_main.py b/hook_HYDiT_main.py index c9eaac4..8b64165 100644 --- a/hook_HYDiT_main.py +++ b/hook_HYDiT_main.py @@ -24,7 +24,6 @@ from torchvision.transforms import functional as TF from diffusers.models import AutoencoderKL from transformers import BertModel, BertTokenizer, logging as tf_logging -from hydit.config import get_args # from hydit.constants import VAE_EMA_PATH, TEXT_ENCODER, TOKENIZER, T5_ENCODER from hydit.lr_scheduler import WarmupLR from hydit.data_loader.arrow_load_stream import TextImageArrowStream @@ -440,7 +439,6 @@ def Core(args, LOG): model, ema, start_epoch, start_epoch_step, train_steps = model_resume( args, model, ema, logger) - if args.training_parts == "lora": loraconfig = LoraConfig( r=args.rank, @@ -537,14 +535,11 @@ def Core(args, LOG): optimizer = torch.optim.AdamW( model.parameters(), lr=args.lr, weight_decay=args.weight_decay) - - easy_sample_images(args, vae, text_encoder, tokenizer, model, embedder_t5, target_height=768, target_width=1280, train_steps=0) # torch.autograd.set_detect_anomaly(True) # with torch.autograd.detect_anomaly(): with tqdm(total=total_steps, initial=train_steps) as pbar: - # Training loop for epoch in range(start_epoch, args.epochs): @@ -735,7 +730,7 @@ def easy_sample_images( sampler='ddpm', train_steps=0, ): - + with torch.no_grad(): from hydit.diffusion.pipeline import StableDiffusionPipeline @@ -780,17 +775,16 @@ def easy_sample_images( rope = get_2d_rotary_pos_embed(head_size, *sub_args) return rope - pipeline = StableDiffusionPipeline(vae=vae, - text_encoder=text_encoder, - tokenizer=tokenizer, - unet=model, - scheduler=scheduler, - feature_extractor=None, - safety_checker=None, - requires_safety_checker=False, - embedder_t5=embedder_t5, - ) + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=model, + scheduler=scheduler, + feature_extractor=None, + safety_checker=None, + requires_safety_checker=False, + embedder_t5=embedder_t5, + ) pipeline.to("cuda") style = torch.as_tensor([0, 0] * batch_size, device="cuda") @@ -837,7 +831,8 @@ def easy_sample_images( pil_image = samples sample_filename = f"{args.task_flag}_train_steps_{train_steps:07d}.png" - sample_filename_path = os.path.join(sample_images_dir, sample_filename) + sample_filename_path = os.path.join( + sample_images_dir, sample_filename) pil_image.save(sample_filename_path) return None From c1900f93cd31e20b750570848bb490bf04014ca6 Mon Sep 17 00:00:00 2001 From: wailovet Date: Fri, 28 Jun 2024 00:39:49 +0800 Subject: [PATCH 16/64] 'update' --- hook_HYDiT_main.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/hook_HYDiT_main.py b/hook_HYDiT_main.py index 8b64165..ae1eb2e 100644 --- a/hook_HYDiT_main.py +++ b/hook_HYDiT_main.py @@ -27,9 +27,7 @@ from transformers import BertModel, BertTokenizer, logging as tf_logging # from hydit.constants import VAE_EMA_PATH, TEXT_ENCODER, TOKENIZER, T5_ENCODER from hydit.lr_scheduler import WarmupLR from hydit.data_loader.arrow_load_stream import TextImageArrowStream -from hydit.diffusion import create_diffusion -from hydit.ds_config import deepspeed_config_from_args -from hydit.modules.ema import EMA +from hydit.diffusion import create_diffusion from hydit.modules.fp16_layers import Float16Module from hydit.modules.models import HUNYUAN_DIT_MODELS from hydit.modules.posemb_layers import init_image_posemb @@ -334,7 +332,7 @@ def Core(args, LOG): ema = None device = "cuda" if args.use_ema: - ema = EMA(args, model, device, logger) + raise ValueError("Not support EMA model.") # Setup FP16 main model: if args.use_fp16: From 1a423fc953c8ab52d2a34c5672f3ec2a98b1334d Mon Sep 17 00:00:00 2001 From: wailovet Date: Fri, 28 Jun 2024 00:42:20 +0800 Subject: [PATCH 17/64] 'update' --- mz_train_tools_core_HYDiT.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index bf5a457..bc0b9bd 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -321,7 +321,7 @@ def check_required(): packages = [ "pandas", "pyarrow", "diffusers", "transformers", "timm", "peft", "accelerate", "loguru", "einops", "sentencepiece", - "polygraphy" + "polygraphy", "protobuf" ] for package in packages: try: From 8d58f376468c5a317a1a1dbd4c1f6938f1d749a2 Mon Sep 17 00:00:00 2001 From: wailovet Date: Fri, 28 Jun 2024 00:48:55 +0800 Subject: [PATCH 18/64] 'update' --- hook_HYDiT_main.py | 70 ++++++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/hook_HYDiT_main.py b/hook_HYDiT_main.py index ae1eb2e..2f7dcc5 100644 --- a/hook_HYDiT_main.py +++ b/hook_HYDiT_main.py @@ -794,43 +794,45 @@ def easy_sample_images( if type(sample_config) != list: sample_config = [sample_config] - for i, sample in enumerate(sample_config): - prompt = sample.get("prompt", "") - negative_prompt = sample.get("negative_prompt", "") - guidance_scale = sample.get("cfg", guidance_scale) - infer_steps = sample.get("steps", infer_steps) - width = sample.get("width", target_width) - height = sample.get("height", target_height) + + with torch.cuda.amp.autocast(enabled=True, dtype=torch.float16, cache_enabled=True): + for i, sample in enumerate(sample_config): + prompt = sample.get("prompt", "") + negative_prompt = sample.get("negative_prompt", "") + guidance_scale = sample.get("cfg", guidance_scale) + infer_steps = sample.get("steps", infer_steps) + width = sample.get("width", target_width) + height = sample.get("height", target_height) - freqs_cis_img = calc_rope(height, width) - samples = pipeline( - height=height, - width=width, - prompt=prompt, - negative_prompt=negative_prompt, - num_images_per_prompt=batch_size, - guidance_scale=guidance_scale, - num_inference_steps=infer_steps, - style=style, - return_dict=False, - use_fp16=True, - learn_sigma=args.learn_sigma, - freqs_cis_img=freqs_cis_img, - image_meta_size=image_meta_size, - )[0] + freqs_cis_img = calc_rope(height, width) + samples = pipeline( + height=height, + width=width, + prompt=prompt, + negative_prompt=negative_prompt, + num_images_per_prompt=batch_size, + guidance_scale=guidance_scale, + num_inference_steps=infer_steps, + style=style, + return_dict=False, + use_fp16=True, + learn_sigma=args.learn_sigma, + freqs_cis_img=freqs_cis_img, + image_meta_size=image_meta_size, + )[0] - # print("samples:",type(samples),) - # input("Press Enter to continue...") - # print("samples:",samples,) + # print("samples:",type(samples),) + # input("Press Enter to continue...") + # print("samples:",samples,) - if type(samples) == list: - pil_image = samples[0] - else: - pil_image = samples + if type(samples) == list: + pil_image = samples[0] + else: + pil_image = samples - sample_filename = f"{args.task_flag}_train_steps_{train_steps:07d}.png" - sample_filename_path = os.path.join( - sample_images_dir, sample_filename) - pil_image.save(sample_filename_path) + sample_filename = f"{args.task_flag}_train_steps_{train_steps:07d}.png" + sample_filename_path = os.path.join( + sample_images_dir, sample_filename) + pil_image.save(sample_filename_path) return None From 7a1de072321623706b21be826bc50bb157854373 Mon Sep 17 00:00:00 2001 From: wailovet Date: Fri, 28 Jun 2024 00:53:05 +0800 Subject: [PATCH 19/64] 'update' --- hook_HYDiT_main.py | 75 +++++++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 38 deletions(-) diff --git a/hook_HYDiT_main.py b/hook_HYDiT_main.py index 2f7dcc5..d35460b 100644 --- a/hook_HYDiT_main.py +++ b/hook_HYDiT_main.py @@ -27,7 +27,7 @@ from transformers import BertModel, BertTokenizer, logging as tf_logging # from hydit.constants import VAE_EMA_PATH, TEXT_ENCODER, TOKENIZER, T5_ENCODER from hydit.lr_scheduler import WarmupLR from hydit.data_loader.arrow_load_stream import TextImageArrowStream -from hydit.diffusion import create_diffusion +from hydit.diffusion import create_diffusion from hydit.modules.fp16_layers import Float16Module from hydit.modules.models import HUNYUAN_DIT_MODELS from hydit.modules.posemb_layers import init_image_posemb @@ -639,7 +639,7 @@ def Core(args, LOG): gc.collect() if (train_steps % args.ckpt_every == 0 or train_steps % args.ckpt_latest_every == 0 # or train_steps == args.max_training_steps - ) and train_steps > 0: + ) and train_steps > 0: logger.info( f" Saving checkpoint at step {train_steps}.") easy_sample_images(args, vae, text_encoder, tokenizer, model, embedder_t5, @@ -794,45 +794,44 @@ def easy_sample_images( if type(sample_config) != list: sample_config = [sample_config] - - with torch.cuda.amp.autocast(enabled=True, dtype=torch.float16, cache_enabled=True): - for i, sample in enumerate(sample_config): - prompt = sample.get("prompt", "") - negative_prompt = sample.get("negative_prompt", "") - guidance_scale = sample.get("cfg", guidance_scale) - infer_steps = sample.get("steps", infer_steps) - width = sample.get("width", target_width) - height = sample.get("height", target_height) - freqs_cis_img = calc_rope(height, width) - samples = pipeline( - height=height, - width=width, - prompt=prompt, - negative_prompt=negative_prompt, - num_images_per_prompt=batch_size, - guidance_scale=guidance_scale, - num_inference_steps=infer_steps, - style=style, - return_dict=False, - use_fp16=True, - learn_sigma=args.learn_sigma, - freqs_cis_img=freqs_cis_img, - image_meta_size=image_meta_size, - )[0] + for i, sample in enumerate(sample_config): + prompt = sample.get("prompt", "") + negative_prompt = sample.get("negative_prompt", "") + guidance_scale = sample.get("cfg", guidance_scale) + infer_steps = sample.get("steps", infer_steps) + width = sample.get("width", target_width) + height = sample.get("height", target_height) - # print("samples:",type(samples),) - # input("Press Enter to continue...") - # print("samples:",samples,) + freqs_cis_img = calc_rope(height, width) + samples = pipeline( + height=height, + width=width, + prompt=prompt, + negative_prompt=negative_prompt, + num_images_per_prompt=batch_size, + guidance_scale=guidance_scale, + num_inference_steps=infer_steps, + style=style, + return_dict=False, + use_fp16=True, + learn_sigma=args.learn_sigma, + freqs_cis_img=freqs_cis_img, + image_meta_size=image_meta_size, + )[0] - if type(samples) == list: - pil_image = samples[0] - else: - pil_image = samples + # print("samples:",type(samples),) + # input("Press Enter to continue...") + # print("samples:",samples,) - sample_filename = f"{args.task_flag}_train_steps_{train_steps:07d}.png" - sample_filename_path = os.path.join( - sample_images_dir, sample_filename) - pil_image.save(sample_filename_path) + if type(samples) == list: + pil_image = samples[0] + else: + pil_image = samples + + sample_filename = f"{args.task_flag}_train_steps_{train_steps:07d}.png" + sample_filename_path = os.path.join( + sample_images_dir, sample_filename) + pil_image.save(sample_filename_path) return None From 218747909aa5ebebce082267f5ab725f70ada81a Mon Sep 17 00:00:00 2001 From: wailovet Date: Fri, 28 Jun 2024 00:57:54 +0800 Subject: [PATCH 20/64] 'update' --- hook_HYDiT_main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hook_HYDiT_main.py b/hook_HYDiT_main.py index d35460b..74131f7 100644 --- a/hook_HYDiT_main.py +++ b/hook_HYDiT_main.py @@ -783,6 +783,7 @@ def easy_sample_images( requires_safety_checker=False, embedder_t5=embedder_t5, ) + pipeline._execution_device = "cuda" pipeline.to("cuda") style = torch.as_tensor([0, 0] * batch_size, device="cuda") From cc4d298ebb268f975d78de434f8b9571bd1e55ba Mon Sep 17 00:00:00 2001 From: wailovet Date: Fri, 28 Jun 2024 00:59:47 +0800 Subject: [PATCH 21/64] 'update' --- hook_HYDiT_main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hook_HYDiT_main.py b/hook_HYDiT_main.py index 74131f7..35fe902 100644 --- a/hook_HYDiT_main.py +++ b/hook_HYDiT_main.py @@ -109,7 +109,8 @@ def create_exp_folder(args, rank): existed_experiments = [] for x in _existed_experiments: if os.path.isdir(os.path.join(args.results_dir, x)): - existed_experiments.append(x) + if x.split('-')[0].isdigit(): + existed_experiments.append(x) if len(existed_experiments) == 0: experiment_index = 1 From 02f414b637bf54943e676213d3f9e2eeb9ede232 Mon Sep 17 00:00:00 2001 From: wailovet Date: Fri, 28 Jun 2024 01:01:24 +0800 Subject: [PATCH 22/64] 'update' --- hook_HYDiT_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hook_HYDiT_main.py b/hook_HYDiT_main.py index 35fe902..af85446 100644 --- a/hook_HYDiT_main.py +++ b/hook_HYDiT_main.py @@ -784,7 +784,7 @@ def easy_sample_images( requires_safety_checker=False, embedder_t5=embedder_t5, ) - pipeline._execution_device = "cuda" + pipeline.to("cuda") style = torch.as_tensor([0, 0] * batch_size, device="cuda") From 2f636466cae51f995cb996682ca0dec6c563cfa2 Mon Sep 17 00:00:00 2001 From: wailovet Date: Fri, 28 Jun 2024 01:09:54 +0800 Subject: [PATCH 23/64] 'update' --- hook_HYDiT_main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hook_HYDiT_main.py b/hook_HYDiT_main.py index af85446..809bd4e 100644 --- a/hook_HYDiT_main.py +++ b/hook_HYDiT_main.py @@ -529,7 +529,7 @@ def Core(args, LOG): if args.use_fp16: optimizer = torch.optim.AdamW( - model.parameters(), lr=args.lr, weight_decay=args.weight_decay, eps=4e-6) + model.parameters(), lr=args.lr, weight_decay=args.weight_decay, eps=TRAIN_CONFIG.get("adam_epsilon", 1e-8)) else: optimizer = torch.optim.AdamW( model.parameters(), lr=args.lr, weight_decay=args.weight_decay) @@ -784,7 +784,7 @@ def easy_sample_images( requires_safety_checker=False, embedder_t5=embedder_t5, ) - + pipeline.to("cuda") style = torch.as_tensor([0, 0] * batch_size, device="cuda") From d5de0742c4da26797246c7f0c0c7579e8b1c793b Mon Sep 17 00:00:00 2001 From: wailovet Date: Fri, 28 Jun 2024 01:14:29 +0800 Subject: [PATCH 24/64] 'update' --- hook_HYDiT_main.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hook_HYDiT_main.py b/hook_HYDiT_main.py index 809bd4e..102b453 100644 --- a/hook_HYDiT_main.py +++ b/hook_HYDiT_main.py @@ -712,6 +712,11 @@ def model_resume(args, model, ema, logger): return model, ema, start_epoch, start_epoch_step, train_steps +from hydit.diffusion.pipeline import StableDiffusionPipeline +from diffusers import schedulers +from hydit.constants import SAMPLER_FACTORY +from hydit.modules.posemb_layers import get_fill_resize_and_crop, get_2d_rotary_pos_embed +from hydit.modules.models import HUNYUAN_DIT_CONFIG def easy_sample_images( args, vae=None, @@ -732,11 +737,6 @@ def easy_sample_images( with torch.no_grad(): - from hydit.diffusion.pipeline import StableDiffusionPipeline - from diffusers import schedulers - from hydit.constants import SAMPLER_FACTORY - from hydit.modules.posemb_layers import get_fill_resize_and_crop, get_2d_rotary_pos_embed - from hydit.modules.models import HUNYUAN_DIT_CONFIG workspace_dir = TRAIN_CONFIG.get("workspace_dir") sample_config_file = TRAIN_CONFIG.get("sample_config_file", None) From a076b6b4e20e1e75279dd6ff7f38f5996c0d5666 Mon Sep 17 00:00:00 2001 From: wailovet Date: Fri, 28 Jun 2024 01:24:20 +0800 Subject: [PATCH 25/64] 'update' --- hook_HYDiT_main.py | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/hook_HYDiT_main.py b/hook_HYDiT_main.py index 102b453..b0d5b6e 100644 --- a/hook_HYDiT_main.py +++ b/hook_HYDiT_main.py @@ -529,7 +529,7 @@ def Core(args, LOG): if args.use_fp16: optimizer = torch.optim.AdamW( - model.parameters(), lr=args.lr, weight_decay=args.weight_decay, eps=TRAIN_CONFIG.get("adam_epsilon", 1e-8)) + model.parameters(), lr=args.lr, weight_decay=args.weight_decay, eps=TRAIN_CONFIG.get("adam_epsilon", 1e-7)) else: optimizer = torch.optim.AdamW( model.parameters(), lr=args.lr, weight_decay=args.weight_decay) @@ -734,7 +734,6 @@ def easy_sample_images( sampler='ddpm', train_steps=0, ): - with torch.no_grad(): @@ -783,9 +782,7 @@ def easy_sample_images( safety_checker=None, requires_safety_checker=False, embedder_t5=embedder_t5, - ) - - pipeline.to("cuda") + ).to("cuda") style = torch.as_tensor([0, 0] * batch_size, device="cuda") @@ -806,21 +803,26 @@ def easy_sample_images( height = sample.get("height", target_height) freqs_cis_img = calc_rope(height, width) - samples = pipeline( - height=height, - width=width, - prompt=prompt, - negative_prompt=negative_prompt, - num_images_per_prompt=batch_size, - guidance_scale=guidance_scale, - num_inference_steps=infer_steps, - style=style, - return_dict=False, - use_fp16=True, - learn_sigma=args.learn_sigma, - freqs_cis_img=freqs_cis_img, - image_meta_size=image_meta_size, - )[0] + + try: + samples = pipeline( + height=height, + width=width, + prompt=prompt, + negative_prompt=negative_prompt, + num_images_per_prompt=batch_size, + guidance_scale=guidance_scale, + num_inference_steps=infer_steps, + style=style, + return_dict=False, + use_fp16=True, + learn_sigma=args.learn_sigma, + freqs_cis_img=freqs_cis_img, + image_meta_size=image_meta_size, + )[0] + except Exception as e: + print(f"Failed to sample images: {e}") + continue # print("samples:",type(samples),) # input("Press Enter to continue...") From 16cc345731954ae575228697b6ce267b67183a35 Mon Sep 17 00:00:00 2001 From: wailovet Date: Fri, 28 Jun 2024 01:44:08 +0800 Subject: [PATCH 26/64] 'update' --- hook_HYDiT_main.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/hook_HYDiT_main.py b/hook_HYDiT_main.py index b0d5b6e..d6e3292 100644 --- a/hook_HYDiT_main.py +++ b/hook_HYDiT_main.py @@ -576,11 +576,12 @@ def Core(args, LOG): with torch.cuda.stream(ema_stream): ema.update(model.module.module, step=step) torch.cuda.current_stream().wait_stream(ema_stream) + + with torch.cuda.amp.autocast(enabled=args.use_fp16): + loss_dict = diffusion.training_losses( + model=model, x_start=latents, model_kwargs=model_kwargs) - loss_dict = diffusion.training_losses( - model=model, x_start=latents, model_kwargs=model_kwargs) - - loss = loss_dict["loss"].mean() + loss = loss_dict["loss"].mean() # print(f"step={step}, loss={loss.item()}") From 5f4e14eac9c5b766e4931078c69aa065fe8a7616 Mon Sep 17 00:00:00 2001 From: wailovet Date: Fri, 28 Jun 2024 01:44:31 +0800 Subject: [PATCH 27/64] 'update' --- hook_HYDiT_main.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/hook_HYDiT_main.py b/hook_HYDiT_main.py index d6e3292..b65064b 100644 --- a/hook_HYDiT_main.py +++ b/hook_HYDiT_main.py @@ -576,12 +576,12 @@ def Core(args, LOG): with torch.cuda.stream(ema_stream): ema.update(model.module.module, step=step) torch.cuda.current_stream().wait_stream(ema_stream) - - with torch.cuda.amp.autocast(enabled=args.use_fp16): + + with torch.cuda.amp.autocast(enabled=True, dtype=torch.float16, cache_enabled=True): loss_dict = diffusion.training_losses( model=model, x_start=latents, model_kwargs=model_kwargs) - loss = loss_dict["loss"].mean() + loss = loss_dict["loss"].mean() # print(f"step={step}, loss={loss.item()}") @@ -641,7 +641,7 @@ def Core(args, LOG): gc.collect() if (train_steps % args.ckpt_every == 0 or train_steps % args.ckpt_latest_every == 0 # or train_steps == args.max_training_steps - ) and train_steps > 0: + ) and train_steps > 0: logger.info( f" Saving checkpoint at step {train_steps}.") easy_sample_images(args, vae, text_encoder, tokenizer, model, embedder_t5, @@ -718,6 +718,8 @@ from diffusers import schedulers from hydit.constants import SAMPLER_FACTORY from hydit.modules.posemb_layers import get_fill_resize_and_crop, get_2d_rotary_pos_embed from hydit.modules.models import HUNYUAN_DIT_CONFIG + + def easy_sample_images( args, vae=None, @@ -737,7 +739,6 @@ def easy_sample_images( ): with torch.no_grad(): - workspace_dir = TRAIN_CONFIG.get("workspace_dir") sample_config_file = TRAIN_CONFIG.get("sample_config_file", None) if sample_config_file is None: @@ -804,7 +805,7 @@ def easy_sample_images( height = sample.get("height", target_height) freqs_cis_img = calc_rope(height, width) - + try: samples = pipeline( height=height, From 989dc28dcf71237339497e2a5f30b6b1263b6a44 Mon Sep 17 00:00:00 2001 From: wailovet Date: Fri, 28 Jun 2024 03:12:26 +0800 Subject: [PATCH 28/64] 'update' --- hook_HYDiT_main.py | 31 +- hook_HYDiT_main_train_deepspeed.py | 570 +++++++++++++++++++++++++++++ hook_HYDiT_run.py | 63 ++-- hook_HYDiT_utils.py | 278 ++++++++++++++ mz_train_tools_core_HYDiT.py | 30 +- 5 files changed, 893 insertions(+), 79 deletions(-) create mode 100644 hook_HYDiT_main_train_deepspeed.py create mode 100644 hook_HYDiT_utils.py diff --git a/hook_HYDiT_main.py b/hook_HYDiT_main.py index b65064b..1d7ff01 100644 --- a/hook_HYDiT_main.py +++ b/hook_HYDiT_main.py @@ -39,37 +39,21 @@ from tqdm import tqdm UNET_PATH = "ckpts/t2i/model/pytorch_model_ema.pt" - - def set_unet_path(path): global UNET_PATH UNET_PATH = path - - VAE_EMA_PATH = "ckpts/t2i/sdxl-vae-fp16-fix" - - def set_vae_ema_path(path): global VAE_EMA_PATH VAE_EMA_PATH = path - - TOKENIZER = "ckpts/t2i/tokenizer" - - def set_tokenizer_path(path): global TOKENIZER TOKENIZER = path - - TEXT_ENCODER = 'ckpts/t2i/clip_text_encoder' - - def set_text_encoder_path(path): global TEXT_ENCODER TEXT_ENCODER = path - - T5_ENCODER = { 'MT5': 'ckpts/t2i/mt5', 'attention_mask': True, @@ -78,20 +62,13 @@ T5_ENCODER = { 'torch_dtype': torch.float16, 'learnable_replace': True } - - -global TRAIN_CONFIG - - -def set_train_config(train_config): - global TRAIN_CONFIG - TRAIN_CONFIG = train_config - - def set_t5_encoder_path(path): global T5_ENCODER T5_ENCODER['MT5'] = path - +global TRAIN_CONFIG +def set_train_config(train_config): + global TRAIN_CONFIG + TRAIN_CONFIG = train_config def create_logger(log_dir=None): logger = logging.getLogger(__name__) diff --git a/hook_HYDiT_main_train_deepspeed.py b/hook_HYDiT_main_train_deepspeed.py new file mode 100644 index 0000000..e494616 --- /dev/null +++ b/hook_HYDiT_main_train_deepspeed.py @@ -0,0 +1,570 @@ +import gc +import json +import os +import random +import sys +import time +from functools import partial +from glob import glob +from pathlib import Path +import numpy as np + +import deepspeed +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.distributed as dist +from torch.utils.data import DataLoader +from torch.distributed.optim import ZeroRedundancyOptimizer +from torchvision.transforms import functional as TF +from diffusers.models import AutoencoderKL +from transformers import BertModel, BertTokenizer, logging as tf_logging + +from hydit.config import get_args +from hydit.lr_scheduler import WarmupLR +from hydit.data_loader.arrow_load_stream import TextImageArrowStream +from hydit.diffusion import create_diffusion +from hydit.ds_config import deepspeed_config_from_args +from hydit.modules.ema import EMA +from hydit.modules.fp16_layers import Float16Module +from hydit.modules.models import HUNYUAN_DIT_MODELS +from hydit.modules.posemb_layers import init_image_posemb +from hydit.utils.tools import create_logger, set_seeds, create_exp_folder, get_trainable_params +from IndexKits.index_kits import ResolutionGroup +from IndexKits.index_kits.sampler import DistributedSamplerWithStartIndex, BlockDistributedSampler +from peft import LoraConfig, get_peft_model + + +def deepspeed_initialize(args, logger, model, opt, deepspeed_config): + logger.info(f"Initialize deepspeed...") + logger.info(f" Using deepspeed optimizer") + + def get_learning_rate_scheduler(warmup_min_lr, lr, warmup_num_steps, opt): + return WarmupLR(opt, warmup_min_lr, lr, warmup_num_steps) + + logger.info( + f" Building scheduler with warmup_min_lr={args.warmup_min_lr}, warmup_num_steps={args.warmup_num_steps}") + model, opt, _, scheduler = deepspeed.initialize( + model=model, + model_parameters=get_trainable_params(model), + config_params=deepspeed_config, + args=args, + lr_scheduler=partial(get_learning_rate_scheduler, args.warmup_min_lr, + args.lr, args.warmup_num_steps) if args.warmup_num_steps > 0 else None, + ) + return model, opt, scheduler + + +def save_checkpoint(args, rank, logger, model, ema, epoch, train_steps, checkpoint_dir): + def save_lora_weight(checkpoint_dir, client_state, tag=f"{train_steps:07d}.pt"): + cur_ckpt_save_dir = f"{checkpoint_dir}/{tag}" + if rank == 0: + if args.use_fp16: + model.module.module.save_pretrained(cur_ckpt_save_dir) + else: + model.module.save_pretrained(cur_ckpt_save_dir) + + checkpoint_path = "[Not rank 0. Disabled output.]" + + client_state = { + "steps": train_steps, + "epoch": epoch, + "args": args + } + if ema is not None: + client_state['ema'] = ema.state_dict() + + dst_paths = [] + if train_steps % args.ckpt_every == 0: + checkpoint_path = f"{checkpoint_dir}/{train_steps:07d}.pt" + try: + if args.training_parts == "lora": + save_lora_weight(checkpoint_dir, client_state, + tag=f"{train_steps:07d}.pt") + else: + model.save_checkpoint( + checkpoint_dir, client_state=client_state, tag=f"{train_steps:07d}.pt") + dst_paths.append(checkpoint_path) + logger.info(f"Saved checkpoint to {checkpoint_path}") + except: + logger.error(f"Saved failed to {checkpoint_path}") + + if train_steps % args.ckpt_latest_every == 0 or train_steps == args.max_training_steps: + save_name = "latest.pt" + checkpoint_path = f"{checkpoint_dir}/{save_name}" + try: + if args.training_parts == "lora": + save_lora_weight(checkpoint_dir, client_state, + tag=f"{save_name}") + else: + model.save_checkpoint( + checkpoint_dir, client_state=client_state, tag=f"{save_name}") + dst_paths.append(checkpoint_path) + logger.info(f"Saved checkpoint to {checkpoint_path}") + except: + logger.error(f"Saved failed to {checkpoint_path}") + + dist.barrier() + if rank == 0 and len(dst_paths) > 0: + # Delete optimizer states to avoid occupying too much disk space. + for dst_path in dst_paths: + for opt_state_path in glob(f"{dst_path}/zero_dp_rank_*_tp_rank_00_pp_rank_00_optim_states.pt"): + os.remove(opt_state_path) + + return checkpoint_path + + +@torch.no_grad() +def prepare_model_inputs(args, batch, device, vae, text_encoder, text_encoder_t5, freqs_cis_img): + try: + from .hook_HYDiT_utils import VAE_EMA_PATH, TEXT_ENCODER, TOKENIZER, T5_ENCODER, easy_sample_images, model_resume, PBar + except: + from hook_HYDiT_utils import VAE_EMA_PATH, TEXT_ENCODER, TOKENIZER, T5_ENCODER, easy_sample_images, model_resume, PBar + + image, text_embedding, text_embedding_mask, text_embedding_t5, text_embedding_mask_t5, kwargs = batch + + # clip & mT5 text embedding + text_embedding = text_embedding.to(device) + text_embedding_mask = text_embedding_mask.to(device) + encoder_hidden_states = text_encoder( + text_embedding.to(device), + attention_mask=text_embedding_mask.to(device), + )[0] + text_embedding_t5 = text_embedding_t5.to(device).squeeze(1) + text_embedding_mask_t5 = text_embedding_mask_t5.to(device).squeeze(1) + with torch.no_grad(): + output_t5 = text_encoder_t5( + input_ids=text_embedding_t5, + attention_mask=text_embedding_mask_t5 if T5_ENCODER['attention_mask'] else None, + output_hidden_states=True + ) + encoder_hidden_states_t5 = output_t5['hidden_states'][T5_ENCODER['layer_index']].detach( + ) + + # additional condition + image_meta_size = kwargs['image_meta_size'].to(device) + style = kwargs['style'].to(device) + + if args.extra_fp16: + image = image.half() + image_meta_size = image_meta_size.half() if image_meta_size is not None else None + + # Map input images to latent space + normalize latents: + image = image.to(device) + vae_scaling_factor = vae.config.scaling_factor + latents = vae.encode(image).latent_dist.sample().mul_(vae_scaling_factor) + + # positional embedding + _, _, height, width = image.shape + reso = f"{height}x{width}" + cos_cis_img, sin_cis_img = freqs_cis_img[reso] + + # Model conditions + model_kwargs = dict( + encoder_hidden_states=encoder_hidden_states, + text_embedding_mask=text_embedding_mask, + encoder_hidden_states_t5=encoder_hidden_states_t5, + text_embedding_mask_t5=text_embedding_mask_t5, + image_meta_size=image_meta_size, + style=style, + cos_cis_img=cos_cis_img, + sin_cis_img=sin_cis_img, + ) + + return latents, model_kwargs + + +def Core(args): + if args.training_parts == "lora": + args.use_ema = False + + assert torch.cuda.is_available(), "Training currently requires at least one GPU." + + dist.init_process_group("nccl") + world_size = dist.get_world_size() + batch_size = args.batch_size + grad_accu_steps = args.grad_accu_steps + global_batch_size = world_size * batch_size * grad_accu_steps + + rank = dist.get_rank() + device = rank % torch.cuda.device_count() + seed = args.global_seed * world_size + rank + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + torch.cuda.set_device(device) + print(f"Starting rank={rank}, seed={seed}, world_size={world_size}.") + deepspeed_config = deepspeed_config_from_args(args, global_batch_size) + + # Setup an experiment folder + experiment_dir, checkpoint_dir, logger = create_exp_folder(args, rank) + + # Log all the arguments + logger.info(sys.argv) + logger.info(str(args)) + # Save to a json file + args_dict = vars(args) + args_dict['world_size'] = world_size + with open(f"{experiment_dir}/args.json", 'w') as f: + json.dump(args_dict, f, indent=4) + + # Disable the message "Some weights of the model checkpoint at ... were not used when initializing BertModel." + # If needed, just comment the following line. + tf_logging.set_verbosity_error() + + # =========================================================================== + # Building HYDIT + # =========================================================================== + + logger.info("Building HYDIT Model.") + + # --------------------------------------------------------------------------- + # Training sample base size, such as 256/512/1024. Notice that this size is + # just a base size, not necessary the actual size of training samples. Actual + # size of the training samples are correlated with `resolutions` when enabling + # multi-resolution training. + # --------------------------------------------------------------------------- + image_size = args.image_size + if len(image_size) == 1: + image_size = [image_size[0], image_size[0]] + if len(image_size) != 2: + raise ValueError(f"Invalid image size: {args.image_size}") + assert image_size[0] % 8 == 0 and image_size[1] % 8 == 0, "Image size must be divisible by 8 (for the VAE encoder). " \ + f"got {image_size}" + latent_size = [image_size[0] // 8, image_size[1] // 8] + + # initialize model by deepspeed + assert args.deepspeed, f"Must enable deepspeed in this script: train_deepspeed.py" + with deepspeed.zero.Init(data_parallel_group=torch.distributed.group.WORLD, + remote_device=None if args.remote_device == 'none' else args.remote_device, + config_dict_or_path=deepspeed_config, + mpu=None, + enabled=args.zero_stage == 3): + model = HUNYUAN_DIT_MODELS[args.model](args, + input_size=latent_size, + log_fn=logger.info, + ) + # Multi-resolution / Single-resolution training. + if args.multireso: + resolutions = ResolutionGroup(image_size[0], + align=16, + step=args.reso_step, + target_ratios=args.target_ratios).data + else: + resolutions = ResolutionGroup(image_size[0], + align=16, + target_ratios=['1:1']).data + + freqs_cis_img = init_image_posemb(args.rope_img, + resolutions=resolutions, + patch_size=model.patch_size, + hidden_size=model.hidden_size, + num_heads=model.num_heads, + log_fn=logger.info, + rope_real=args.rope_real, + ) + + # Create EMA model and convert to fp16 if needed. + ema = None + if args.use_ema: + ema = EMA(args, model, device, logger) + + # Setup FP16 main model: + if args.use_fp16: + model = Float16Module(model, args) + logger.info( + f" Using main model with data type {'fp16' if args.use_fp16 else 'fp32'}") + + diffusion = create_diffusion( + noise_schedule=args.noise_schedule, + predict_type=args.predict_type, + learn_sigma=args.learn_sigma, + mse_loss_weight_type=args.mse_loss_weight_type, + beta_start=args.beta_start, + beta_end=args.beta_end, + noise_offset=args.noise_offset, + ) + + try: + from .hook_HYDiT_utils import VAE_EMA_PATH, TEXT_ENCODER, TOKENIZER, T5_ENCODER, easy_sample_images, model_resume, PBar + except: + from hook_HYDiT_utils import VAE_EMA_PATH, TEXT_ENCODER, TOKENIZER, T5_ENCODER, easy_sample_images, model_resume, PBar + + # Setup VAE + logger.info(f" Loading vae from {VAE_EMA_PATH}") + vae = AutoencoderKL.from_pretrained(VAE_EMA_PATH) + # Setup BERT text encoder + logger.info(f" Loading Bert text encoder from {TEXT_ENCODER}") + text_encoder = BertModel.from_pretrained( + TEXT_ENCODER, False, revision=None) + # Setup BERT tokenizer: + logger.info(f" Loading Bert tokenizer from {TOKENIZER}") + tokenizer = BertTokenizer.from_pretrained(TOKENIZER) + # Setup T5 text encoder + from hydit.modules.text_encoder import MT5Embedder + mt5_path = T5_ENCODER['MT5'] + embedder_t5 = MT5Embedder( + mt5_path, torch_dtype=T5_ENCODER['torch_dtype'], max_length=args.text_len_t5) + tokenizer_t5 = embedder_t5.tokenizer + text_encoder_t5 = embedder_t5.model + + if args.extra_fp16: + logger.info(f" Using fp16 for extra modules: vae, text_encoder") + vae = vae.half().to(device) + text_encoder = text_encoder.half().to(device) + text_encoder_t5 = text_encoder_t5.half().to(device) + else: + vae = vae.to(device) + text_encoder = text_encoder.to(device) + text_encoder_t5 = text_encoder_t5.to(device) + + logger.info( + f" Optimizer parameters: lr={args.lr}, weight_decay={args.weight_decay}") + logger.info(" Using deepspeed optimizer") + opt = None + + # =========================================================================== + # Building Dataset + # =========================================================================== + + logger.info(f"Building Streaming Dataset.") + logger.info(f" Loading index file {args.index_file} (v2)") + + dataset = TextImageArrowStream(args=args, + resolution=image_size[0], + random_flip=args.random_flip, + log_fn=logger.info, + index_file=args.index_file, + multireso=args.multireso, + batch_size=batch_size, + world_size=world_size, + random_shrink_size_cond=args.random_shrink_size_cond, + merge_src_cond=args.merge_src_cond, + uncond_p=args.uncond_p, + text_ctx_len=args.text_len, + tokenizer=tokenizer, + uncond_p_t5=args.uncond_p_t5, + text_ctx_len_t5=args.text_len_t5, + tokenizer_t5=tokenizer_t5, + ) + if args.multireso: + sampler = BlockDistributedSampler(dataset, num_replicas=world_size, rank=rank, seed=args.global_seed, + shuffle=False, drop_last=True, batch_size=batch_size) + else: + sampler = DistributedSamplerWithStartIndex(dataset, num_replicas=world_size, rank=rank, seed=args.global_seed, + shuffle=False, drop_last=True) + loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, sampler=sampler, + num_workers=args.num_workers, pin_memory=True, drop_last=True) + logger.info(f" Dataset contains {len(dataset):,} images.") + logger.info(f" Index file: {args.index_file}.") + if args.multireso: + logger.info(f' Using MultiResolutionBucketIndexV2 with step {dataset.index_manager.step} ' + f'and base size {dataset.index_manager.base_size}') + logger.info(f'\n {dataset.index_manager.resolutions}') + + # =========================================================================== + # Loading parameter + # =========================================================================== + + logger.info(f"Loading parameter") + start_epoch = 0 + start_epoch_step = 0 + train_steps = 0 + # Resume checkpoint if needed + # if args.resume is not None or len(args.resume) > 0: + if True: + model, ema, start_epoch, start_epoch_step, train_steps = model_resume( + args, model, ema, logger) + + if args.training_parts == "lora": + loraconfig = LoraConfig( + r=args.rank, + lora_alpha=args.rank, + target_modules=args.target_modules + ) + if args.use_fp16: + model.module = get_peft_model(model.module, loraconfig) + else: + model = get_peft_model(model, loraconfig) + + logger.info(f" Training parts: {args.training_parts}") + + model, opt, scheduler = deepspeed_initialize( + args, logger, model, opt, deepspeed_config) + + # =========================================================================== + # Training + # =========================================================================== + + model.train() + if args.use_ema: + ema.eval() + + print(f" Worker {rank} ready.") + dist.barrier() + + iters_per_epoch = len(loader) + logger.info( + " ****************************** Running training ******************************") + logger.info(f" Number GPUs: {world_size}") + logger.info(f" Number training samples: {len(dataset):,}") + logger.info( + f" Number parameters: {sum(p.numel() for p in model.parameters()):,}") + logger.info( + f" Number trainable params: {sum(p.numel() for p in get_trainable_params(model)):,}") + logger.info( + " ------------------------------------------------------------------------------") + logger.info(f" Iters per epoch: {iters_per_epoch:,}") + logger.info(f" Batch size per device: {batch_size}") + logger.info( + f" Batch size all device: {batch_size * world_size * grad_accu_steps:,} (world_size * batch_size * grad_accu_steps)") + logger.info(f" Gradient Accu steps: {args.grad_accu_steps}") + logger.info( + f" Total optimization steps: {args.epochs * iters_per_epoch // grad_accu_steps:,}") + + logger.info( + f" Training epochs: {start_epoch}/{args.epochs}") + logger.info( + f" Training epoch steps: {start_epoch_step:,}/{iters_per_epoch:,}") + logger.info( + f" Training total steps: {train_steps:,}/{min(args.max_training_steps, args.epochs * iters_per_epoch):,}") + logger.info( + " ------------------------------------------------------------------------------") + logger.info(f" Noise schedule: {args.noise_schedule}") + logger.info( + f" Beta limits: ({args.beta_start}, {args.beta_end})") + logger.info(f" Learn sigma: {args.learn_sigma}") + logger.info(f" Prediction type: {args.predict_type}") + logger.info(f" Noise offset: {args.noise_offset}") + + logger.info( + " ------------------------------------------------------------------------------") + logger.info( + f" Using EMA model: {args.use_ema} ({args.ema_dtype})") + if args.use_ema: + logger.info( + f" Using EMA decay: {ema.max_value if args.use_ema else None}") + logger.info( + f" Using EMA warmup power: {ema.power if args.use_ema else None}") + logger.info(f" Using main model fp16: {args.use_fp16}") + logger.info(f" Using extra modules fp16: {args.extra_fp16}") + logger.info( + " ------------------------------------------------------------------------------") + logger.info(f" Experiment directory: {experiment_dir}") + logger.info( + " *******************************************************************************") + + if args.gc_interval > 0: + gc.disable() + gc.collect() + + # Variables for monitoring/logging purposes: + log_steps = 0 + running_loss = 0 + start_time = time.time() + + if args.async_ema: + ema_stream = torch.cuda.Stream() + + easy_sample_images(args, vae, text_encoder, tokenizer, model, embedder_t5, + target_height=768, target_width=1280, train_steps=0) + pbar = PBar(args.epochs * len(loader)) + + # Training loop + for epoch in range(start_epoch, args.epochs): + logger.info(f" Start random shuffle with seed={seed}") + # Makesure all processors use the same seed to shuffle dataset. + dataset.shuffle(seed=args.global_seed + epoch, fast=True) + logger.info(f" End of random shuffle") + + # Move sampler to start_index + if not args.multireso: + start_index = start_epoch_step * world_size * batch_size + if start_index != sampler.start_index: + sampler.start_index = start_index + # Reset start_epoch_step to zero, to ensure next epoch will start from the beginning. + start_epoch_step = 0 + logger.info(f" Iters left this epoch: {len(loader):,}") + + logger.info(f" Beginning epoch {epoch}...") + step = 0 + for batch in loader: + step += 1 + + latents, model_kwargs = prepare_model_inputs( + args, batch, device, vae, text_encoder, text_encoder_t5, freqs_cis_img) + + # training model by deepspeed while use fp16 + if args.use_fp16: + if args.use_ema and args.async_ema: + with torch.cuda.stream(ema_stream): + ema.update(model.module.module, step=step) + torch.cuda.current_stream().wait_stream(ema_stream) + + loss_dict = diffusion.training_losses( + model=model, x_start=latents, model_kwargs=model_kwargs) + loss = loss_dict["loss"].mean() + model.backward(loss) + last_batch_iteration = ( + train_steps + 1) // (global_batch_size // (batch_size * world_size)) + model.step( + lr_kwargs={'last_batch_iteration': last_batch_iteration}) + + if args.use_ema and not args.async_ema or (args.async_ema and step == len(loader) - 1): + if args.use_fp16: + ema.update(model.module.module, step=step) + else: + ema.update(model.module, step=step) + + # =========================================================================== + # Log loss values: + # =========================================================================== + running_loss += loss.item() + log_steps += 1 + train_steps += 1 + if train_steps % args.log_every == 0: + # Measure training speed: + torch.cuda.synchronize() + end_time = time.time() + steps_per_sec = log_steps / (end_time - start_time) + # Reduce loss history over all processes: + avg_loss = torch.tensor( + running_loss / log_steps, device=device) + dist.all_reduce(avg_loss, op=dist.ReduceOp.SUM) + avg_loss = avg_loss.item() / world_size + # get lr from deepspeed fused optimizer + logger.info(f"(step={train_steps:07d}) " + + (f"(update_step={train_steps // args.grad_accu_steps:07d}) " if args.grad_accu_steps > 1 else "") + + f"Train Loss: {avg_loss:.4f}, " + f"Lr: {opt.param_groups[0]['lr']:.6g}, " + f"Steps/Sec: {steps_per_sec:.2f}, " + f"Samples/Sec: {int(steps_per_sec * batch_size * world_size):d}") + # Reset monitoring variables: + running_loss = 0 + log_steps = 0 + start_time = time.time() + + # collect gc: + if args.gc_interval > 0 and (step % args.gc_interval == 0): + gc.collect() + + pbar.step( + f"Epoch {epoch}, step {step}, loss {loss.item():.4f}, mean_loss {running_loss / step:.4f}", epoch * len(loader), train_steps) + + if (train_steps % args.ckpt_every == 0 or train_steps % args.ckpt_latest_every == 0 # or train_steps == args.max_training_steps + ) and train_steps > 0: + easy_sample_images(args, vae, text_encoder, tokenizer, model, embedder_t5, + target_height=768, target_width=1280, train_steps=train_steps) + save_checkpoint(args, rank, logger, model, ema, + epoch, train_steps, checkpoint_dir) + + if train_steps >= args.max_training_steps: + logger.info(f"Breaking step loop at {train_steps}.") + break + + if train_steps >= args.max_training_steps: + logger.info(f"Breaking epoch loop at {epoch}.") + break + + dist.destroy_process_group() diff --git a/hook_HYDiT_run.py b/hook_HYDiT_run.py index 3aa8eed..11ed62c 100644 --- a/hook_HYDiT_run.py +++ b/hook_HYDiT_run.py @@ -23,34 +23,26 @@ class SimpleNamespaceCNWarrper(SimpleNamespace): return self.__dict__.get(name, None) -import requests - -master_port = 0 - - -def LOG(log): - if master_port == 0: - raise Exception("master_port is 0") - # 发送http - resp = requests.request("post", f"http://127.0.0.1:{master_port}/log", data=json.dumps(log), headers={ - "Content-Type": "application/json"}) - if resp.status_code != 200: - raise Exception(f"LOG failed: {resp.text}") - - if __name__ == "__main__": parser = argparse.ArgumentParser( conflict_handler='resolve', ) parser.add_argument("--sys_path", type=str, default="") parser.add_argument("--train_config_file", type=str, default="") - parser.add_argument("--master_port", type=int, default=0) + parser.add_argument("--mz_master_port", type=int, default=0) args = parser.parse_args() - master_port = args.master_port + master_port = args.mz_master_port print(f"master_port = {master_port}") + try: + from . import hook_HYDiT_utils + except Exception as e: + import hook_HYDiT_utils + + hook_HYDiT_utils.set_master_port(master_port) + sys_path = args.sys_path if sys_path != "": sys.path.append(sys_path) @@ -62,11 +54,11 @@ if __name__ == "__main__": except Exception as e: import hook_HYDiT_main try: - from . import hook_HYDiT_config + from . import hook_HYDiT_main_train_deepspeed except Exception as e: - import hook_HYDiT_config + import hook_HYDiT_main_train_deepspeed - + import hydit.config def _handle_conflict_error(self, *args, **kwargs): pass @@ -87,7 +79,7 @@ if __name__ == "__main__": argparse.ArgumentParser._handle_conflict_error = _handle_conflict_error argparse.ArgumentParser._handle_conflict_resolve = _handle_conflict_error argparse.ArgumentParser.parse_args = parse_args - margs = hook_HYDiT_config.get_args() + margs = hydit.config.get_args() margs.model = train_config.get("model", "DiT-g/2") margs.task_flag = train_config.get( @@ -111,7 +103,9 @@ if __name__ == "__main__": margs.target_ratios = train_config.get( "target_ratios", ['1:1', '3:4', '4:3', '16:9', '9:16']) margs.rope_img = train_config.get("rope_img", "base1024") + margs.image_size = train_config.get("image_size", 1024) + margs.rope_real = train_config.get("rope_real", True) margs.index_file = train_config.get("index_file", None) @@ -120,7 +114,9 @@ if __name__ == "__main__": margs.noise_offset = train_config.get("noise_offset", 0.1) - margs.log_every = train_config.get("log_every", 10) + margs.log_every = train_config.get("log_every", 99999999999999) + margs.use_zero_stage = 2 + margs.deepspeed = True margs.results_dir = train_config.get("results_dir") margs.mse_loss_weight_type = train_config.get( @@ -130,21 +126,30 @@ if __name__ == "__main__": if hasattr(margs, k): setattr(margs, k, v) - hook_HYDiT_main.set_unet_path( + hook_HYDiT_utils.set_unet_path( train_config.get("unet_path")) - hook_HYDiT_main.set_vae_ema_path( + hook_HYDiT_utils.set_vae_ema_path( train_config.get("vae_ema_path")) - hook_HYDiT_main.set_text_encoder_path( + hook_HYDiT_utils.set_text_encoder_path( train_config.get("text_encoder_path")) - hook_HYDiT_main.set_tokenizer_path( + hook_HYDiT_utils.set_tokenizer_path( train_config.get("tokenizer_path")) - hook_HYDiT_main.set_t5_encoder_path( + hook_HYDiT_utils.set_t5_encoder_path( train_config.get("t5_encoder_path")) - hook_HYDiT_main.set_train_config(train_config) + hook_HYDiT_utils.set_train_config(train_config) - hook_HYDiT_main.Core(margs, LOG) + try: + # deepspeed/runtime/engine + import deepspeed.runtime.engine + deepspeed.runtime.engine.DeepSpeedEngine._do_sanity_check = lambda x: None + except Exception as e: + pass + + if type(margs.image_size) == int: + margs.image_size = [margs.image_size, margs.image_size] + hook_HYDiT_main_train_deepspeed.Core(margs) diff --git a/hook_HYDiT_utils.py b/hook_HYDiT_utils.py new file mode 100644 index 0000000..2c33682 --- /dev/null +++ b/hook_HYDiT_utils.py @@ -0,0 +1,278 @@ + + +import json +import os +import torch + + +UNET_PATH = "ckpts/t2i/model/pytorch_model_ema.pt" + + +def set_unet_path(path): + global UNET_PATH + UNET_PATH = path + + +VAE_EMA_PATH = "ckpts/t2i/sdxl-vae-fp16-fix" + + +def set_vae_ema_path(path): + global VAE_EMA_PATH + VAE_EMA_PATH = path + + +TOKENIZER = "ckpts/t2i/tokenizer" + + +def set_tokenizer_path(path): + global TOKENIZER + TOKENIZER = path + + +TEXT_ENCODER = 'ckpts/t2i/clip_text_encoder' + + +def set_text_encoder_path(path): + global TEXT_ENCODER + TEXT_ENCODER = path + + +T5_ENCODER = { + 'MT5': 'ckpts/t2i/mt5', + 'attention_mask': True, + 'layer_index': -1, + 'attention_pool': True, + 'torch_dtype': torch.float16, + 'learnable_replace': True +} + + +def set_t5_encoder_path(path): + global T5_ENCODER + T5_ENCODER['MT5'] = path + + +global TRAIN_CONFIG + + +def set_train_config(train_config): + global TRAIN_CONFIG + TRAIN_CONFIG = train_config + + +def easy_sample_images( + args, + vae=None, + text_encoder=None, + tokenizer=None, + model=None, + embedder_t5=None, + target_height=768, + target_width=1280, + prompt="A photo of a girl with a hat on a sunny day", + negative_prompt="", + batch_size=1, + guidance_scale=5.0, + infer_steps=20, + sampler='ddpm', + train_steps=0, +): + from hydit.diffusion.pipeline import StableDiffusionPipeline + from diffusers import schedulers + from hydit.constants import SAMPLER_FACTORY + from hydit.modules.posemb_layers import get_fill_resize_and_crop, get_2d_rotary_pos_embed + from hydit.modules.models import HUNYUAN_DIT_CONFIG + with torch.no_grad(): + + workspace_dir = TRAIN_CONFIG.get("workspace_dir") + sample_config_file = TRAIN_CONFIG.get("sample_config_file", None) + if sample_config_file is None: + print("sample_config_file is not set.") + return + try: + sample_config = json.load(open(sample_config_file, "r")) + except Exception as e: + print(f"Failed to load sample_config_file: {sample_config_file}") + return + sample_images_dir = os.path.join(workspace_dir, "sample_images") + os.makedirs(sample_images_dir, exist_ok=True) + + # Load sampler from factory + kwargs = SAMPLER_FACTORY[sampler]['kwargs'] + scheduler = SAMPLER_FACTORY[sampler]['scheduler'] + + # Build scheduler according to the sampler. + scheduler_class = getattr(schedulers, scheduler) + scheduler = scheduler_class(**kwargs) + + # Set timesteps for inference steps. + scheduler.set_timesteps(infer_steps, "cuda") + + def calc_rope(height, width): + model_config = HUNYUAN_DIT_CONFIG["DiT-g/2"] + patch_size = model_config['patch_size'] + head_size = model_config['hidden_size'] // model_config['num_heads'] + th = height // 8 // patch_size + tw = width // 8 // patch_size + base_size = 512 // 8 // patch_size + start, stop = get_fill_resize_and_crop((th, tw), base_size) + sub_args = [start, stop, (th, tw)] + rope = get_2d_rotary_pos_embed(head_size, *sub_args) + return rope + + pipeline = StableDiffusionPipeline(vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=model.module, + scheduler=scheduler, + feature_extractor=None, + safety_checker=None, + requires_safety_checker=False, + embedder_t5=embedder_t5, + ).to("cuda") + + style = torch.as_tensor([0, 0] * batch_size, device="cuda") + + src_size_cond = (target_width, target_height) + size_cond = list(src_size_cond) + [target_width, target_height, 0, 0] + image_meta_size = torch.as_tensor( + [size_cond] * 2 * batch_size, device="cuda",) + + if type(sample_config) != list: + sample_config = [sample_config] + + for i, sample in enumerate(sample_config): + prompt = sample.get("prompt", "") + negative_prompt = sample.get("negative_prompt", "") + guidance_scale = sample.get("cfg", guidance_scale) + infer_steps = sample.get("steps", infer_steps) + width = sample.get("width", target_width) + height = sample.get("height", target_height) + + freqs_cis_img = calc_rope(height, width) + + samples = pipeline( + height=height, + width=width, + prompt=prompt, + negative_prompt=negative_prompt, + num_images_per_prompt=batch_size, + guidance_scale=guidance_scale, + num_inference_steps=infer_steps, + style=style, + return_dict=False, + use_fp16=True, + learn_sigma=args.learn_sigma, + freqs_cis_img=freqs_cis_img, + image_meta_size=image_meta_size, + )[0] + + try: + pass + except Exception as e: + print(f"Failed to sample images: {e}") + continue + + # print("samples:",type(samples),) + # input("Press Enter to continue...") + # print("samples:",samples,) + + if type(samples) == list: + pil_image = samples[0] + else: + pil_image = samples + + sample_filename = f"{args.task_flag}_train_steps_{train_steps:07d}.png" + sample_filename_path = os.path.join( + sample_images_dir, sample_filename) + pil_image.save(sample_filename_path) + + return None + + +def model_resume(args, model, ema, logger): + """ + Load pretrained weights. + """ + start_epoch = 0 + start_epoch_step = 0 + train_steps = 0 + resume_path = UNET_PATH + + logger.info(f"Resume from checkpoint {resume_path}") + + if args.resume_split: + # Resume main model + + resume_ckpt_module = torch.load( + resume_path, map_location=lambda storage, loc: storage) + model.load_state_dict(resume_ckpt_module, strict=False) + + # Resume ema model + if args.use_ema: + if args.module_to_ema: + if "resume_ckpt_module" in locals(): + logger.info(f" Resume ema model from main states.") + ema.load_state_dict(resume_ckpt_module, strict=args.strict) + else: + logger.info(f" Resume ema model from module states.") + resume_ckpt_module = torch.load( + resume_path, map_location=lambda storage, loc: storage) + ema.load_state_dict(resume_ckpt_module, strict=args.strict) + else: + if "resume_ckpt_ema" in locals(): + logger.info(f" Resume ema model from EMA states.") + ema.load_state_dict(resume_ckpt_ema, strict=args.strict) + else: + logger.info(f" Resume ema model from EMA states.") + resume_ckpt_ema = torch.load(resume_path, + map_location=lambda storage, loc: storage) + ema.load_state_dict(resume_ckpt_ema, strict=args.strict) + else: + raise ValueError( + " “If `resume` is True, then either `resume_split` must be true.”") + + return model, ema, start_epoch, start_epoch_step, train_steps + + +import tqdm + +import requests + + +def set_master_port(port): + global master_port + master_port = port + + +master_port = 0 + + +def LOG(log): + if master_port == 0: + raise Exception("master_port is 0") + # 发送http + resp = requests.request("post", f"http://127.0.0.1:{master_port}/log", data=json.dumps(log), headers={ + "Content-Type": "application/json"}) + if resp.status_code != 200: + raise Exception(f"LOG failed: {resp.text}") + + +# with tqdm(total=total_steps, initial=train_steps) as pbar: +# pbar.update(1) +# pbar.set_description( +# f"Epoch {epoch}, step {step}, loss {loss.item():.4f}, mean_loss {mean_loss / step:.4f}") +class PBar: + def __init__(self, total): + self.pbar = tqdm.tqdm(total=total) + + def step(self, desc, total_steps, train_steps): + self.pbar.update(1) + self.pbar.set_description(desc) + + LOG({ + "type": "sample_images", + "global_step": train_steps, + "total_steps": total_steps, + # "latent": noise_pred_latent_path, + }) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index bc0b9bd..07b64d1 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -297,27 +297,6 @@ def check_model_auto_download(args): def check_required(): - # timm==0.9.5 - # diffusers==0.21.2 - # peft==0.10.0 - # protobuf==3.19.0 - # torchvision==0.14.1 - # transformers==4.39.1 - # peft==0.10.0 - # accelerate==0.29.3 - # loguru==0.7.2 - # einops==0.7.0 - # sentencepiece==0.1.99 - # cuda-python==11.7.1 - # onnxruntime==1.12.1 - # onnx==1.12.0 - # nvidia-pyindex==1.0.9 - # onnx-graphsurgeon==0.3.27 - # polygraphy==0.47.1 - # pandas==2.0.3 - # gradio==3.50.2 - # deepspeed==0.6.3 - # pyarrow==16.1.0 packages = [ "pandas", "pyarrow", "diffusers", "transformers", "timm", "peft", "accelerate", "loguru", "einops", "sentencepiece", @@ -330,6 +309,11 @@ def check_required(): subprocess.run([sys.executable, "-m", "pip", "install", package], check=True) + try: + import deepspeed + except ImportError: + raise Exception("请手动安装合适的deepspeed版本") + def MZ_HYDiTTrain_call(args={}): check_required() @@ -584,8 +568,8 @@ def run_hook_HYDiT_pyexec(train_config_path, train_config, HYDiT_tool_dir): try: subprocess.run( - [sys.executable, exec_pyfile, "--sys_path", HYDiT_tool_dir, - "--master_port", str(port), "--train_config_file", train_config_path], + [sys.executable, "-m", "torch.distributed.launch", "--use_env", exec_pyfile, "--sys_path", HYDiT_tool_dir, + "--mz_master_port", str(port), "--train_config_file", train_config_path], check=True, ) stop_server() From 8ad8efa42f03ccff942079854892c68770ae8fd0 Mon Sep 17 00:00:00 2001 From: wailovet Date: Fri, 28 Jun 2024 03:13:28 +0800 Subject: [PATCH 29/64] 'update' --- hook_HYDiT_config.py | 235 ------------- hook_HYDiT_main.py | 820 ------------------------------------------- hook_HYDiT_run.py | 5 +- 3 files changed, 1 insertion(+), 1059 deletions(-) delete mode 100644 hook_HYDiT_config.py delete mode 100644 hook_HYDiT_main.py diff --git a/hook_HYDiT_config.py b/hook_HYDiT_config.py deleted file mode 100644 index a98fcfa..0000000 --- a/hook_HYDiT_config.py +++ /dev/null @@ -1,235 +0,0 @@ - - -import argparse - -from hydit.constants import * -from hydit.modules.models import HUNYUAN_DIT_CONFIG, HUNYUAN_DIT_MODELS -from hydit.diffusion.gaussian_diffusion import ModelVarType - - -def model_var_type(value): - try: - return ModelVarType[value] - except KeyError: - raise ValueError( - f"Invalid choice '{value}', valid choices are {[v.name for v in ModelVarType]}") - - -def get_args(default_args=None): - parser = argparse.ArgumentParser() - parser.add_argument("--task-flag", type=str) - - # General Setting - parser.add_argument("--batch-size", type=int, - default=1, help="Per-GPU batch size") - parser.add_argument('--seed', type=int, default=42, - help="A seed for all the prompts.") - parser.add_argument("--use-fp16", action="store_true", - help="Use FP16 precision.") - parser.add_argument("--no-fp16", dest="use_fp16", action="store_false") - parser.set_defaults(use_fp16=True) - parser.add_argument("--extra-fp16", action="store_true", - help="Use extra fp16 for vae and text_encoder.") - - # HunYuan-DiT - parser.add_argument( - "--model", type=str, choices=list(HUNYUAN_DIT_CONFIG.keys()), default='DiT-g/2') - parser.add_argument("--image-size", type=int, nargs='+', default=[1024, 1024], - help='Image size (h, w). If a single value is provided, the image will be treated to ' - '(value, value).') - parser.add_argument("--qk-norm", action="store_true", - help="Query Key normalization. See http://arxiv.org/abs/2302.05442 for details.") - parser.set_defaults(qk_norm=True) - parser.add_argument( - "--norm", type=str, choices=["rms", "laryer"], default="layer", help="Normalization layer type") - parser.add_argument("--text-states-dim", type=int, - default=1024, help="Hidden size of CLIP text encoder.") - parser.add_argument("--text-len", type=int, default=77, - help="Token length of CLIP text encoder output.") - parser.add_argument("--text-states-dim-t5", type=int, - default=2048, help="Hidden size of CLIP text encoder.") - parser.add_argument("--text-len-t5", type=int, default=256, - help="Token length of T5 text encoder output.") - - # LoRA config - parser.add_argument("--training-parts", type=str, default='all', - choices=['all', 'lora'], help="Training parts") - parser.add_argument("--rank", type=int, default=64, help="Rank of LoRA") - parser.add_argument("--lora-ckpt", type=str, - default=None, help="LoRA checkpoint") - parser.add_argument('--target-modules', type=str, nargs='+', default=['Wqkv', 'q_proj', 'kv_proj', 'out_proj'], - help="Target modules for LoRA fine tune") - parser.add_argument("--output-merge-path", type=str, - default=None, help="Output path for merged model") - - # controlnet config - parser.add_argument("--control-type", type=str, default='canny', - choices=['canny', 'depth', 'pose'], help="Controlnet condition type") - parser.add_argument("--control-weight", type=float, - default=1.0, help="Controlnet weight") - parser.add_argument("--condition-image-path", type=str, - default=None, help="Inference condition image path") - - # Diffusion - parser.add_argument("--learn-sigma", action="store_true", - help="Learn extra channels for sigma.") - parser.add_argument("--no-learn-sigma", - dest="learn_sigma", action="store_false") - parser.set_defaults(learn_sigma=True) - parser.add_argument("--predict-type", type=str, default="v_prediction", - help="Diffusion predict type") - parser.add_argument("--noise-schedule", type=str, default="scaled_linear", - help="Noise schedule") - parser.add_argument("--beta-start", type=float, - default=0.00085, help="Beta start value") - parser.add_argument("--beta-end", type=float, - default=0.03, help="Beta end value") - parser.add_argument("--sigma-small", action="store_true") - parser.add_argument("--mse-loss-weight-type", type=str, default="constant", - help="Min-SNR-gamma. Can be constant or min_snr_ where gamma is a integer. 5 is recommended in the paper.") - parser.add_argument("--model-var-type", type=model_var_type, - default=None, help="Specify the model variable type.") - parser.add_argument("--noise-offset", type=float, default=0.0, - help="Add extra noise to the input image.") - - # ======================================================================================================== - # Inference - # ======================================================================================================== - - # Basic Setting - parser.add_argument("--prompt", type=str, default="一只小猫", - help="The prompt for generating images.") - parser.add_argument("--model-root", type=str, - default="ckpts", help="Model root path.") - - # Model setting - parser.add_argument("--load-key", type=str, choices=[ - "ema", "module", "distill", 'merge'], default="ema", help="Load model key for HunYuanDiT checkpoint.") - parser.add_argument('--size-cond', type=int, nargs='+', default=[1024, 1024], - help="Size condition used in sampling. 2 values are required for height and width. " - "If a single value is provided, the image will be treated to (value, value).") - parser.add_argument('--target-ratios', type=str, nargs='+', default=None, - help="Target ratios for multi-resolution training.") - parser.add_argument("--cfg-scale", type=float, default=6.0, - help="Guidance scale for classifier-free.") - parser.add_argument("--negative", type=str, - default=None, help="Negative prompt.") - - # Acceleration - parser.add_argument("--infer-mode", type=str, - choices=["fa", "torch", "trt"], default="torch", help="Inference mode") - parser.add_argument("--onnx-workdir", type=str, - default="onnx_model", help="Path to save ONNX model") - - # Sampling - parser.add_argument("--sampler", type=str, - default="ddpm", help="Diffusion sampler") - parser.add_argument("--infer-steps", type=int, - default=100, help="Inference steps") - - # Prompt enhancement - parser.add_argument("--enhance", action="store_true", - help="Enhance prompt with dialoggen.") - parser.add_argument("--no-enhance", dest="enhance", action="store_false") - parser.add_argument( - "--load-4bit", help="load DialogGen model with 4bit quantization.", action="store_true") - parser.set_defaults(enhance=True) - - # App - parser.add_argument("--lang", type=str, default="zh", - choices=["zh", "en"], help="Language") - - # ======================================================================================================== - # Training - # ======================================================================================================== - - # Basic Setting - parser.add_argument("--lr", type=float, default=1e-4) - parser.add_argument("--epochs", type=int, default=1400) - parser.add_argument("--max-training-steps", type=int, default=10_000_000) - parser.add_argument("--gc-interval", type=int, default=40, - help='To address the memory bottleneck encountered during the preprocessing of the dataset,' - ' memory fragments are reclaimed here by invoking the gc.collect() function.') - parser.add_argument("--log-every", type=int, default=100) - parser.add_argument("--ckpt-every", type=int, default=100_000, - help="Create a ckpt every a few steps.") - parser.add_argument("--ckpt-latest-every", type=int, default=10_000, - help="Create a ckpt named `latest.pt` every a few steps.") - parser.add_argument("--num-workers", type=int, default=4) - parser.add_argument("--global-seed", type=int, default=1234) - parser.add_argument("--warmup-min-lr", type=float, default=1e-6) - parser.add_argument("--warmup-num-steps", type=float, default=0) - parser.add_argument("--weight-decay", type=float, - default=0, help="weight-decay in optimizer") - parser.add_argument("--rope-img", type=str, default=None, choices=['extend', 'base512', 'base1024'], - help="Extend or interpolate the positional embedding of the image.") - parser.add_argument("--rope-real", action="store_true", - help="Use real part and imaginary part separately for RoPE.") - - # Classifier-free - parser.add_argument("--uncond-p", type=float, default=0.2, - help="The probability of dropping training text used for CLIP feature extraction") - parser.add_argument("--uncond-p-t5", type=float, default=0.2, - help="The probability of dropping training text used for mT5 feature extraction") - - # Directory - parser.add_argument("--results-dir", type=str, default="results") - parser.add_argument("--resume", type=str, default=None, - help="Resume experiment from a checkpoint") - parser.add_argument("--no-strict", dest="strict", - action="store_false", help="Strict loading of checkpoint") - parser.set_defaults(strict=True) - parser.add_argument("--resume-deepspeed", action="store_true", - help="Resume model and ema states from a checkpoint saved by Deepspeed version of DIT.") - parser.add_argument("--resume-split", action="store_true", - help="Resume model and ema states from two checkpoint separated from DeepSpeed ckpt.") - parser.add_argument("--ema-to-module", action="store_true", - help="If true, initialize the module with EMA weights.") - parser.add_argument("--module-to-ema", action="store_true", - help="if true, initialize the ema with Module weights.") - - # Dataset - parser.add_argument("--index-file", type=str, nargs='+', - help="During training, provide a JSON file with data indices.") - parser.add_argument("--random-flip", action="store_true", - help="Random flip image") - parser.add_argument("--reset-loader", action="store_true", - help="Reset the data loader. It is useful when resuming from a checkpoint but switch to a new dataset.") - parser.add_argument("--multireso", action="store_true", - help="Use multi-resolution training.") - parser.add_argument("--reso-step", type=int, default=None, - help="Step size for multi-resolution training.") - - # Additional condition - parser.add_argument("--random-shrink-size-cond", action="store_true", - help="Randomly shrink the original size condition.") - parser.add_argument("--merge-src-cond", action="store_true", - help="Merge the source condition into a single value.") - - # EMA Model - parser.add_argument("--use-ema", action="store_true", help="Use EMA model") - parser.add_argument("--ema-dtype", type=str, choices=['fp16', 'fp32', 'none'], default="none", - help="EMA data type. If none, use the same data type as the model.") - parser.add_argument("--ema-decay", type=float, default=None, - help="EMA decay rate. If None, use the default value of the model.") - parser.add_argument("--ema-warmup", action="store_true", - help="EMA warmup. If True, perform ema_decay warmup from 0 to ema_decay.") - parser.add_argument("--ema-warmup-power", type=float, default=None, - help="EMA power. If None, use the default value of the model.") - parser.add_argument("--ema-reset-decay", action="store_true", - help="Reset EMA decay to 0 and restart increasing the EMA decay." - "Only works when --ema-warmup is enabled.") - # Acceleration - parser.add_argument("--use-flash-attn", action="store_true", help="During training, " - "flash attention is used to accelerate training.") - parser.add_argument("--no-flash-attn", dest="use_flash_attn", - action="store_false", help="During training, flash attention is not used to accelerate training.") - parser.add_argument("--use-zero-stage", type=int, default=1, - help="Use AngelPTM zero stage. Support 2 and 3") - parser.add_argument("--grad-accu-steps", type=int, - default=1, help="Gradient accumulation steps.") - - - args = parser.parse_args(default_args) - - return args diff --git a/hook_HYDiT_main.py b/hook_HYDiT_main.py deleted file mode 100644 index 1d7ff01..0000000 --- a/hook_HYDiT_main.py +++ /dev/null @@ -1,820 +0,0 @@ - -import argparse -import json -import logging -import os -import sys -from types import SimpleNamespace -import gc -import json -import os -import random -import sys -import time -from functools import partial -from glob import glob -from pathlib import Path -import numpy as np - -import torch -import torch.nn as nn -import torch.nn.functional as F -from torch.utils.data import DataLoader -from torchvision.transforms import functional as TF -from diffusers.models import AutoencoderKL -from transformers import BertModel, BertTokenizer, logging as tf_logging - -# from hydit.constants import VAE_EMA_PATH, TEXT_ENCODER, TOKENIZER, T5_ENCODER -from hydit.lr_scheduler import WarmupLR -from hydit.data_loader.arrow_load_stream import TextImageArrowStream -from hydit.diffusion import create_diffusion -from hydit.modules.fp16_layers import Float16Module -from hydit.modules.models import HUNYUAN_DIT_MODELS -from hydit.modules.posemb_layers import init_image_posemb -from hydit.utils.tools import set_seeds, get_trainable_params -from IndexKits.index_kits import ResolutionGroup -from IndexKits.index_kits.sampler import DistributedSamplerWithStartIndex, BlockDistributedSampler -from peft import LoraConfig, get_peft_model -from tqdm import tqdm - - -UNET_PATH = "ckpts/t2i/model/pytorch_model_ema.pt" -def set_unet_path(path): - global UNET_PATH - UNET_PATH = path -VAE_EMA_PATH = "ckpts/t2i/sdxl-vae-fp16-fix" -def set_vae_ema_path(path): - global VAE_EMA_PATH - VAE_EMA_PATH = path -TOKENIZER = "ckpts/t2i/tokenizer" -def set_tokenizer_path(path): - global TOKENIZER - TOKENIZER = path -TEXT_ENCODER = 'ckpts/t2i/clip_text_encoder' -def set_text_encoder_path(path): - global TEXT_ENCODER - TEXT_ENCODER = path -T5_ENCODER = { - 'MT5': 'ckpts/t2i/mt5', - 'attention_mask': True, - 'layer_index': -1, - 'attention_pool': True, - 'torch_dtype': torch.float16, - 'learnable_replace': True -} -def set_t5_encoder_path(path): - global T5_ENCODER - T5_ENCODER['MT5'] = path -global TRAIN_CONFIG -def set_train_config(train_config): - global TRAIN_CONFIG - TRAIN_CONFIG = train_config - -def create_logger(log_dir=None): - logger = logging.getLogger(__name__) - logger.addHandler(logging.NullHandler()) - return logger - - -def create_exp_folder(args, rank): - if rank == 0: - os.makedirs(args.results_dir, exist_ok=True) - - # print('args.results_dir', args.results_dir) - _existed_experiments = os.listdir(args.results_dir) - # print('_existed_experiments', _existed_experiments) - existed_experiments = [] - for x in _existed_experiments: - if os.path.isdir(os.path.join(args.results_dir, x)): - if x.split('-')[0].isdigit(): - existed_experiments.append(x) - - if len(existed_experiments) == 0: - experiment_index = 1 - else: - existed_experiments.sort() - # print('existed_experiments', existed_experiments) - experiment_index = max([int(x.split('-')[0]) - for x in existed_experiments]) + 1 - model_string_name = args.task_flag if args.task_flag else args.model.replace( - "/", "-") - # Create an experiment folder - experiment_dir = f"{args.results_dir}/{experiment_index:03d}-{model_string_name}" - # Stores saved model checkpoints - checkpoint_dir = f"{experiment_dir}/checkpoints" - if rank == 0: - os.makedirs(checkpoint_dir, exist_ok=True) - logger = create_logger(experiment_dir) - logger.info(f"Experiment directory created at {experiment_dir}") - else: - logger = create_logger() - experiment_dir = "" - logger.info = print - return experiment_dir, checkpoint_dir, logger - - -def save_checkpoint(args, rank, logger, model, ema, epoch, train_steps, checkpoint_dir): - def save_lora_weight(checkpoint_dir, client_state, tag=f"{train_steps:07d}.pt"): - cur_ckpt_save_dir = f"{checkpoint_dir}/{tag}" - if rank == 0: - if args.use_fp16: - model.module.save_pretrained(cur_ckpt_save_dir) - else: - model.save_pretrained(cur_ckpt_save_dir) - - checkpoint_path = "[Not rank 0. Disabled output.]" - - client_state = { - "steps": train_steps, - "epoch": epoch, - "args": args - } - if ema is not None: - client_state['ema'] = ema.state_dict() - - dst_paths = [] - - if train_steps % args.ckpt_every == 0: - checkpoint_path = f"{checkpoint_dir}/{train_steps:07d}.pt" - try: - if args.training_parts == "lora": - save_lora_weight(checkpoint_dir, client_state, - tag=f"{train_steps:07d}.pt") - else: - model.save_checkpoint( - checkpoint_dir, client_state=client_state, tag=f"{train_steps:07d}.pt") - dst_paths.append(checkpoint_path) - logger.info(f"Saved checkpoint to {checkpoint_path}") - except: - logger.error(f"Saved failed to {checkpoint_path}") - - if train_steps % args.ckpt_latest_every == 0 or train_steps == args.max_training_steps: - save_name = "latest.pt" - checkpoint_path = f"{checkpoint_dir}/{save_name}" - try: - if args.training_parts == "lora": - save_lora_weight(checkpoint_dir, client_state, - tag=f"{save_name}") - else: - model.save_checkpoint( - checkpoint_dir, client_state=client_state, tag=f"{save_name}") - dst_paths.append(checkpoint_path) - logger.info(f"Saved checkpoint to {checkpoint_path}") - except: - logger.error(f"Saved failed to {checkpoint_path}") - - return checkpoint_path - - -@torch.no_grad() -def prepare_model_inputs(args, batch, device, vae, text_encoder, text_encoder_t5, freqs_cis_img): - image, text_embedding, text_embedding_mask, text_embedding_t5, text_embedding_mask_t5, kwargs = batch - - # clip & mT5 text embedding - text_embedding = text_embedding.to(device) - text_embedding_mask = text_embedding_mask.to(device) - encoder_hidden_states = text_encoder( - text_embedding.to(device), - attention_mask=text_embedding_mask.to(device), - )[0] - text_embedding_t5 = text_embedding_t5.to(device).squeeze(1) - text_embedding_mask_t5 = text_embedding_mask_t5.to(device).squeeze(1) - with torch.no_grad(): - output_t5 = text_encoder_t5( - input_ids=text_embedding_t5, - attention_mask=text_embedding_mask_t5 if T5_ENCODER['attention_mask'] else None, - output_hidden_states=True - ) - encoder_hidden_states_t5 = output_t5['hidden_states'][T5_ENCODER['layer_index']].detach( - ) - - # additional condition - image_meta_size = kwargs['image_meta_size'].to(device) - style = kwargs['style'].to(device) - - if args.extra_fp16: - image = image.half() - image_meta_size = image_meta_size.half() if image_meta_size is not None else None - - # Map input images to latent space + normalize latents: - image = image.to(device) - vae_scaling_factor = vae.config.scaling_factor - latents = vae.encode(image).latent_dist.sample().mul_( - vae_scaling_factor).to(device) - - # positional embedding - _, _, height, width = image.shape - reso = f"{height}x{width}" - - # print("freqs_cis_img:", freqs_cis_img.keys()) - # print("freqs_cis_img[reso]:", freqs_cis_img[reso]) - - cos_cis_img, sin_cis_img = freqs_cis_img[reso] - - # Model conditions - model_kwargs = dict( - encoder_hidden_states=encoder_hidden_states, - text_embedding_mask=text_embedding_mask, - encoder_hidden_states_t5=encoder_hidden_states_t5, - text_embedding_mask_t5=text_embedding_mask_t5, - image_meta_size=image_meta_size, - style=style, - cos_cis_img=cos_cis_img, - sin_cis_img=sin_cis_img, - ) - - return latents, model_kwargs - - -def Core(args, LOG): - if args.training_parts == "lora": - args.use_ema = False - - assert torch.cuda.is_available(), "Training currently requires at least one GPU." - - batch_size = args.batch_size - grad_accu_steps = args.grad_accu_steps - global_batch_size = batch_size * grad_accu_steps - - seed = args.global_seed - random.seed(seed) - np.random.seed(seed) - torch.manual_seed(seed) - torch.cuda.manual_seed_all(seed) - print(f"Starting ,seed={seed}.") - rank = 0 - # Setup an experiment folder - experiment_dir, checkpoint_dir, logger = create_exp_folder(args, rank) - - # Log all the arguments - logger.info(sys.argv) - logger.info(str(args)) - # Save to a json file - args_dict = vars(args) - with open(f"{experiment_dir}/args.json", 'w') as f: - json.dump(args_dict, f, indent=4) - - # Disable the message "Some weights of the model checkpoint at ... were not used when initializing BertModel." - # If needed, just comment the following line. - tf_logging.set_verbosity_error() - - # =========================================================================== - # Building HYDIT - # =========================================================================== - - logger.info("Building HYDIT Model.") - - # --------------------------------------------------------------------------- - # Training sample base size, such as 256/512/1024. Notice that this size is - # just a base size, not necessary the actual size of training samples. Actual - # size of the training samples are correlated with `resolutions` when enabling - # multi-resolution training. - # --------------------------------------------------------------------------- - image_size = args.image_size - if type(image_size) == int: - image_size = [image_size, image_size] - if len(image_size) == 1: - image_size = [image_size[0], image_size[0]] - if len(image_size) != 2: - raise ValueError(f"Invalid image size: {args.image_size}") - assert image_size[0] % 8 == 0 and image_size[1] % 8 == 0, "Image size must be divisible by 8 (for the VAE encoder). " \ - f"got {image_size}" - latent_size = [image_size[0] // 8, image_size[1] // 8] - - # initialize model by deepspeed - model = HUNYUAN_DIT_MODELS[args.model](args, - input_size=latent_size, - log_fn=logger.info, - ) - # Multi-resolution / Single-resolution training. - if args.multireso: - resolutions = ResolutionGroup(image_size[0], - align=16, - step=args.reso_step, - target_ratios=args.target_ratios).data - else: - resolutions = ResolutionGroup(image_size[0], - align=16, - target_ratios=['1:1']).data - - freqs_cis_img = init_image_posemb(args.rope_img, - resolutions=resolutions, - patch_size=model.patch_size, - hidden_size=model.hidden_size, - num_heads=model.num_heads, - log_fn=logger.info, - rope_real=args.rope_real, - ) - - # Create EMA model and convert to fp16 if needed. - ema = None - device = "cuda" - if args.use_ema: - raise ValueError("Not support EMA model.") - - # Setup FP16 main model: - if args.use_fp16: - model = Float16Module(model, args) - logger.info( - f" Using main model with data type {'fp16' if args.use_fp16 else 'fp32'}") - - diffusion = create_diffusion( - noise_schedule=args.noise_schedule, - predict_type=args.predict_type, - learn_sigma=args.learn_sigma, - mse_loss_weight_type=args.mse_loss_weight_type, - beta_start=args.beta_start, - beta_end=args.beta_end, - noise_offset=args.noise_offset, - ) - - # Setup VAE - logger.info(f" Loading vae from {VAE_EMA_PATH}") - - if os.path.isdir(VAE_EMA_PATH): - vae = AutoencoderKL.from_pretrained(VAE_EMA_PATH) - else: - vae = AutoencoderKL.from_single_file(VAE_EMA_PATH) - # Setup BERT text encoder - logger.info(f" Loading Bert text encoder from {TEXT_ENCODER}") - text_encoder = BertModel.from_pretrained( - TEXT_ENCODER, False, revision=None) - # Setup BERT tokenizer: - logger.info(f" Loading Bert tokenizer from {TOKENIZER}") - tokenizer = BertTokenizer.from_pretrained(TOKENIZER) - # Setup T5 text encoder - from hydit.modules.text_encoder import MT5Embedder - mt5_path = T5_ENCODER['MT5'] - embedder_t5 = MT5Embedder( - mt5_path, torch_dtype=T5_ENCODER['torch_dtype'], max_length=args.text_len_t5) - tokenizer_t5 = embedder_t5.tokenizer - text_encoder_t5 = embedder_t5.model - - if args.extra_fp16: - logger.info(f" Using fp16 for extra modules: vae, text_encoder") - vae = vae.half().to(device) - text_encoder = text_encoder.half().to(device) - text_encoder_t5 = text_encoder_t5.half().to(device) - else: - vae = vae.to(device) - text_encoder = text_encoder.to(device) - text_encoder_t5 = text_encoder_t5.to(device) - - logger.info( - f" Optimizer parameters: lr={args.lr}, weight_decay={args.weight_decay}") - - # =========================================================================== - # Building Dataset - # =========================================================================== - - logger.info(f"Building Streaming Dataset.") - logger.info(f" Loading index file {args.index_file} (v2)") - world_size = 1 - - dataset = TextImageArrowStream(args=args, - resolution=image_size[0], - random_flip=args.random_flip, - log_fn=logger.info, - index_file=args.index_file, - multireso=args.multireso, - batch_size=batch_size, - world_size=world_size, - random_shrink_size_cond=args.random_shrink_size_cond, - merge_src_cond=args.merge_src_cond, - uncond_p=args.uncond_p, - text_ctx_len=args.text_len, - tokenizer=tokenizer, - uncond_p_t5=args.uncond_p_t5, - text_ctx_len_t5=args.text_len_t5, - tokenizer_t5=tokenizer_t5, - ) - if args.multireso: - sampler = BlockDistributedSampler(dataset, num_replicas=world_size, rank=rank, seed=args.global_seed, - shuffle=False, drop_last=True, batch_size=batch_size) - else: - sampler = DistributedSamplerWithStartIndex(dataset, num_replicas=world_size, rank=rank, seed=args.global_seed, - shuffle=False, drop_last=True) - loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, sampler=sampler, - num_workers=args.num_workers, pin_memory=True, drop_last=True) - logger.info(f" Dataset contains {len(dataset):,} images.") - logger.info(f" Index file: {args.index_file}.") - if args.multireso: - logger.info(f' Using MultiResolutionBucketIndexV2 with step {dataset.index_manager.step} ' - f'and base size {dataset.index_manager.base_size}') - logger.info(f'\n {dataset.index_manager.resolutions}') - - # =========================================================================== - # Loading parameter - # =========================================================================== - - logger.info(f"Loading parameter") - start_epoch = 0 - start_epoch_step = 0 - train_steps = 0 - # Resume checkpoint if needed - model, ema, start_epoch, start_epoch_step, train_steps = model_resume( - args, model, ema, logger) - - if args.training_parts == "lora": - loraconfig = LoraConfig( - r=args.rank, - lora_alpha=args.rank, - target_modules=args.target_modules - ) - if args.use_fp16: - model.module = get_peft_model(model.module, loraconfig) - else: - model = get_peft_model(model, loraconfig) - - logger.info(f" Training parts: {args.training_parts}") - - # =========================================================================== - # Training - # =========================================================================== - - model.to(device) - model.train() - if args.use_ema: - ema.eval() - - print(f" Worker {rank} ready.") - - iters_per_epoch = len(loader) - logger.info( - " ****************************** Running training ******************************") - logger.info(f" Number GPUs: {world_size}") - logger.info(f" Number training samples: {len(dataset):,}") - logger.info( - f" Number parameters: {sum(p.numel() for p in model.parameters()):,}") - logger.info( - f" Number trainable params: {sum(p.numel() for p in get_trainable_params(model)):,}") - logger.info( - " ------------------------------------------------------------------------------") - logger.info(f" Iters per epoch: {iters_per_epoch:,}") - logger.info(f" Batch size per device: {batch_size}") - logger.info( - f" Batch size all device: {batch_size * world_size * grad_accu_steps:,} (world_size * batch_size * grad_accu_steps)") - logger.info(f" Gradient Accu steps: {args.grad_accu_steps}") - logger.info( - f" Total optimization steps: {args.epochs * iters_per_epoch // grad_accu_steps:,}") - - logger.info( - f" Training epochs: {start_epoch}/{args.epochs}") - logger.info( - f" Training epoch steps: {start_epoch_step:,}/{iters_per_epoch:,}") - logger.info( - f" Training total steps: {train_steps:,}/{min(args.max_training_steps, args.epochs * iters_per_epoch):,}") - logger.info( - " ------------------------------------------------------------------------------") - logger.info(f" Noise schedule: {args.noise_schedule}") - logger.info( - f" Beta limits: ({args.beta_start}, {args.beta_end})") - logger.info(f" Learn sigma: {args.learn_sigma}") - logger.info(f" Prediction type: {args.predict_type}") - logger.info(f" Noise offset: {args.noise_offset}") - - logger.info( - " ------------------------------------------------------------------------------") - logger.info( - f" Using EMA model: {args.use_ema} ({args.ema_dtype})") - if args.use_ema: - logger.info( - f" Using EMA decay: {ema.max_value if args.use_ema else None}") - logger.info( - f" Using EMA warmup power: {ema.power if args.use_ema else None}") - logger.info(f" Using main model fp16: {args.use_fp16}") - logger.info(f" Using extra modules fp16: {args.extra_fp16}") - logger.info( - " ------------------------------------------------------------------------------") - logger.info(f" Experiment directory: {experiment_dir}") - logger.info( - " *******************************************************************************") - - if args.gc_interval > 0: - gc.disable() - gc.collect() - - # Variables for monitoring/logging purposes: - log_steps = 0 - running_loss = 0 - start_time = time.time() - - if args.async_ema: - ema_stream = torch.cuda.Stream() - - total_steps = args.epochs * len(loader) - - if args.use_fp16: - optimizer = torch.optim.AdamW( - model.parameters(), lr=args.lr, weight_decay=args.weight_decay, eps=TRAIN_CONFIG.get("adam_epsilon", 1e-7)) - else: - optimizer = torch.optim.AdamW( - model.parameters(), lr=args.lr, weight_decay=args.weight_decay) - - easy_sample_images(args, vae, text_encoder, tokenizer, model, embedder_t5, - target_height=768, target_width=1280, train_steps=0) - # torch.autograd.set_detect_anomaly(True) - # with torch.autograd.detect_anomaly(): - with tqdm(total=total_steps, initial=train_steps) as pbar: - - # Training loop - for epoch in range(start_epoch, args.epochs): - logger.info(f" Start random shuffle with seed={seed}") - # Makesure all processors use the same seed to shuffle dataset. - dataset.shuffle(seed=args.global_seed + epoch, fast=True) - logger.info(f" End of random shuffle") - - # Move sampler to start_index - if not args.multireso: - start_index = start_epoch_step * world_size * batch_size - if start_index != sampler.start_index: - sampler.start_index = start_index - # Reset start_epoch_step to zero, to ensure next epoch will start from the beginning. - start_epoch_step = 0 - logger.info( - f" Iters left this epoch: {len(loader):,}") - - logger.info(f" Beginning epoch {epoch}...") - step = 0 - mean_loss = 0 - for batch in loader: - step += 1 - - # Zero gradients, backward pass, and optimizer step: - optimizer.zero_grad() - latents, model_kwargs = prepare_model_inputs( - args, batch, device, vae, text_encoder, text_encoder_t5, freqs_cis_img) - # with torch.cuda.amp.autocast(enabled=True, dtype=torch.float16, cache_enabled=True): - - if True: - # training model by deepspeed while use fp16 - if args.use_fp16: - if args.use_ema and args.async_ema: - with torch.cuda.stream(ema_stream): - ema.update(model.module.module, step=step) - torch.cuda.current_stream().wait_stream(ema_stream) - - with torch.cuda.amp.autocast(enabled=True, dtype=torch.float16, cache_enabled=True): - loss_dict = diffusion.training_losses( - model=model, x_start=latents, model_kwargs=model_kwargs) - - loss = loss_dict["loss"].mean() - - # print(f"step={step}, loss={loss.item()}") - - # model.backward(loss) - - loss.backward() - - # last_batch_iteration = ( - # train_steps + 1) // (global_batch_size // (batch_size * world_size)) - # model.step( - # lr_kwargs={'last_batch_iteration': last_batch_iteration}) - - optimizer.step() - - if args.use_ema and not args.async_ema or (args.async_ema and step == len(loader) - 1): - if args.use_fp16: - ema.update(model.module.module, step=step) - else: - ema.update(model.module, step=step) - - train_steps += 1 - mean_loss += loss.item() - pbar.update(1) - pbar.set_description( - f"Epoch {epoch}, step {step}, loss {loss.item():.4f}, mean_loss {mean_loss / step:.4f}") - - # =========================================================================== - # Log loss values: - # =========================================================================== - # running_loss += loss.item() - # log_steps += 1 - # if train_steps % args.log_every == 0: - # # Measure training speed: - # torch.cuda.synchronize() - # end_time = time.time() - # steps_per_sec = log_steps / (end_time - start_time) - # # Reduce loss history over all processes: - # avg_loss = torch.tensor( - # running_loss / log_steps, device=device) - # # dist.all_reduce(avg_loss, op=dist.ReduceOp.SUM) - - # # avg_loss = avg_loss.item() / world_size - # # get lr from deepspeed fused optimizer - # logger.info(f"(step={train_steps:07d}) " + - # (f"(update_step={train_steps // args.grad_accu_steps:07d}) " if args.grad_accu_steps > 1 else "") + - # f"Train Loss: {avg_loss:.4f}, " - # # f"Lr: {opt.param_groups[0]['lr']:.6g}, " - # f"Steps/Sec: {steps_per_sec:.2f}, " - # f"Samples/Sec: {int(steps_per_sec * batch_size * world_size):d}") - # # Reset monitoring variables: - # running_loss = 0 - # log_steps = 0 - # start_time = time.time() - - # collect gc: - if args.gc_interval > 0 and (step % args.gc_interval == 0): - gc.collect() - - if (train_steps % args.ckpt_every == 0 or train_steps % args.ckpt_latest_every == 0 # or train_steps == args.max_training_steps - ) and train_steps > 0: - logger.info( - f" Saving checkpoint at step {train_steps}.") - easy_sample_images(args, vae, text_encoder, tokenizer, model, embedder_t5, - target_height=768, target_width=1280, train_steps=train_steps) - save_checkpoint(args, rank, logger, model, ema, - epoch, train_steps, checkpoint_dir) - - if train_steps >= args.max_training_steps: - logger.info(f"Breaking step loop at {train_steps}.") - break - - LOG({ - "type": "sample_images", - "global_step": train_steps, - "total_steps": total_steps, - # "latent": noise_pred_latent_path, - }) - - if train_steps >= args.max_training_steps: - logger.info(f"Breaking epoch loop at {epoch}.") - break - - -def model_resume(args, model, ema, logger): - """ - Load pretrained weights. - """ - start_epoch = 0 - start_epoch_step = 0 - train_steps = 0 - resume_path = UNET_PATH - if not Path(resume_path).exists(): - raise FileNotFoundError( - f" Cannot find checkpoint from {resume_path}") - - logger.info(f"Resume from checkpoint {resume_path}") - - if args.resume_split: - # Resume main model - - resume_ckpt_module = torch.load( - resume_path, map_location=lambda storage, loc: storage) - model.load_state_dict(resume_ckpt_module, strict=False) - - # Resume ema model - if args.use_ema: - if args.module_to_ema: - if "resume_ckpt_module" in locals(): - logger.info(f" Resume ema model from main states.") - ema.load_state_dict(resume_ckpt_module, strict=args.strict) - else: - logger.info(f" Resume ema model from module states.") - resume_ckpt_module = torch.load( - resume_path, map_location=lambda storage, loc: storage) - ema.load_state_dict(resume_ckpt_module, strict=args.strict) - else: - if "resume_ckpt_ema" in locals(): - logger.info(f" Resume ema model from EMA states.") - ema.load_state_dict(resume_ckpt_ema, strict=args.strict) - else: - logger.info(f" Resume ema model from EMA states.") - resume_ckpt_ema = torch.load(resume_path, - map_location=lambda storage, loc: storage) - ema.load_state_dict(resume_ckpt_ema, strict=args.strict) - else: - raise ValueError( - " “If `resume` is True, then either `resume_split` must be true.”") - - return model, ema, start_epoch, start_epoch_step, train_steps - - -from hydit.diffusion.pipeline import StableDiffusionPipeline -from diffusers import schedulers -from hydit.constants import SAMPLER_FACTORY -from hydit.modules.posemb_layers import get_fill_resize_and_crop, get_2d_rotary_pos_embed -from hydit.modules.models import HUNYUAN_DIT_CONFIG - - -def easy_sample_images( - args, - vae=None, - text_encoder=None, - tokenizer=None, - model=None, - embedder_t5=None, - target_height=768, - target_width=1280, - prompt="A photo of a girl with a hat on a sunny day", - negative_prompt="", - batch_size=1, - guidance_scale=5.0, - infer_steps=20, - sampler='ddpm', - train_steps=0, -): - with torch.no_grad(): - - workspace_dir = TRAIN_CONFIG.get("workspace_dir") - sample_config_file = TRAIN_CONFIG.get("sample_config_file", None) - if sample_config_file is None: - print("sample_config_file is not set.") - return - try: - sample_config = json.load(open(sample_config_file, "r")) - except Exception as e: - print(f"Failed to load sample_config_file: {sample_config_file}") - return - sample_images_dir = os.path.join(workspace_dir, "sample_images") - os.makedirs(sample_images_dir, exist_ok=True) - - # Load sampler from factory - kwargs = SAMPLER_FACTORY[sampler]['kwargs'] - scheduler = SAMPLER_FACTORY[sampler]['scheduler'] - - # Build scheduler according to the sampler. - scheduler_class = getattr(schedulers, scheduler) - scheduler = scheduler_class(**kwargs) - - # Set timesteps for inference steps. - scheduler.set_timesteps(infer_steps, "cuda") - - def calc_rope(height, width): - model_config = HUNYUAN_DIT_CONFIG["DiT-g/2"] - patch_size = model_config['patch_size'] - head_size = model_config['hidden_size'] // model_config['num_heads'] - th = height // 8 // patch_size - tw = width // 8 // patch_size - base_size = 512 // 8 // patch_size - start, stop = get_fill_resize_and_crop((th, tw), base_size) - sub_args = [start, stop, (th, tw)] - rope = get_2d_rotary_pos_embed(head_size, *sub_args) - return rope - - pipeline = StableDiffusionPipeline(vae=vae, - text_encoder=text_encoder, - tokenizer=tokenizer, - unet=model, - scheduler=scheduler, - feature_extractor=None, - safety_checker=None, - requires_safety_checker=False, - embedder_t5=embedder_t5, - ).to("cuda") - - style = torch.as_tensor([0, 0] * batch_size, device="cuda") - - src_size_cond = (target_width, target_height) - size_cond = list(src_size_cond) + [target_width, target_height, 0, 0] - image_meta_size = torch.as_tensor( - [size_cond] * 2 * batch_size, device="cuda",) - - if type(sample_config) != list: - sample_config = [sample_config] - - for i, sample in enumerate(sample_config): - prompt = sample.get("prompt", "") - negative_prompt = sample.get("negative_prompt", "") - guidance_scale = sample.get("cfg", guidance_scale) - infer_steps = sample.get("steps", infer_steps) - width = sample.get("width", target_width) - height = sample.get("height", target_height) - - freqs_cis_img = calc_rope(height, width) - - try: - samples = pipeline( - height=height, - width=width, - prompt=prompt, - negative_prompt=negative_prompt, - num_images_per_prompt=batch_size, - guidance_scale=guidance_scale, - num_inference_steps=infer_steps, - style=style, - return_dict=False, - use_fp16=True, - learn_sigma=args.learn_sigma, - freqs_cis_img=freqs_cis_img, - image_meta_size=image_meta_size, - )[0] - except Exception as e: - print(f"Failed to sample images: {e}") - continue - - # print("samples:",type(samples),) - # input("Press Enter to continue...") - # print("samples:",samples,) - - if type(samples) == list: - pil_image = samples[0] - else: - pil_image = samples - - sample_filename = f"{args.task_flag}_train_steps_{train_steps:07d}.png" - sample_filename_path = os.path.join( - sample_images_dir, sample_filename) - pil_image.save(sample_filename_path) - - return None diff --git a/hook_HYDiT_run.py b/hook_HYDiT_run.py index 11ed62c..20f9422 100644 --- a/hook_HYDiT_run.py +++ b/hook_HYDiT_run.py @@ -49,10 +49,7 @@ if __name__ == "__main__": print("HYDi run hook") - try: - from . import hook_HYDiT_main - except Exception as e: - import hook_HYDiT_main + try: from . import hook_HYDiT_main_train_deepspeed except Exception as e: From 66bce4c08dc63479cea869122176661b2d896e24 Mon Sep 17 00:00:00 2001 From: wailovet Date: Sat, 29 Jun 2024 02:51:32 +0800 Subject: [PATCH 30/64] 'update' --- __init__.py | 8 +- mz_train_tools_core.py | 4 +- mz_train_tools_core_HYDiT.py | 26 ++++- mz_train_tools_utils.py | 2 +- tools/face_resize_copy_dir.py | 191 ++++++++++++++++++++++++++++++++++ 5 files changed, 224 insertions(+), 7 deletions(-) create mode 100644 tools/face_resize_copy_dir.py diff --git a/__init__.py b/__init__.py index 8a53abd..930b314 100644 --- a/__init__.py +++ b/__init__.py @@ -367,8 +367,8 @@ class MZ_LoadImagesFromDirectoryPath: if not os.path.exists(image_dir): return (images,) images = os.listdir(image_dir) - images = [x for x in images if x.endswith( - ".png") or x.endswith(".jpg")] + images = [x for x in images if x.lower().endswith( + ".png") or x.lower().endswith(".jpg")] images = [os.path.join(image_dir, x) for x in images] pil_images = [] @@ -430,6 +430,10 @@ class MZ_HYDiTDatasetConfig: "required": { "workspace_config": ("MZ_TT_HYDiT_WorkspaceConfig",), "resolution": ("INT", {"default": 1024}), + "force_clear": (["enable", "disable"], {"default": "disable"}), + "force_clear_only_images": (["enable", "disable"], {"default": "disable"}), + "same_caption_generate": (["enable", "disable"], {"default": "disable"}), + "same_caption": ("STRING", {"default": "", "dynamicPrompts": True, "multiline": True}), }, "optional": { "images": ("IMAGE",), diff --git a/mz_train_tools_core.py b/mz_train_tools_core.py index 96b3cb5..65460a2 100644 --- a/mz_train_tools_core.py +++ b/mz_train_tools_core.py @@ -120,7 +120,7 @@ def MZ_ImageSelecter_call(args={}): if force_clear: if force_clear_only_images: for file in os.listdir(train_images_dir): - if file.endswith(".png") or file.endswith(".jpg"): + if file.lower().endswith(".png") or file.lower().endswith(".jpg"): os.remove(os.path.join(train_images_dir, file)) else: shutil.rmtree(train_images_dir) @@ -359,7 +359,7 @@ def get_sample_images(train_config): if os.path.exists(sample_images_dir): image_files = os.listdir(sample_images_dir) image_files = list( - filter(lambda x: x.endswith(".png"), image_files)) + filter(lambda x: x.lower().endswith(".png"), image_files)) # 筛选 output_name 前缀 image_files = list( filter(lambda x: x.startswith(output_name), image_files)) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index 07b64d1..63a0ebb 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -115,6 +115,17 @@ def MZ_HYDiTDatasetConfig_call(args={}): if workspace_name is None or workspace_name == "": raise Exception("训练名称不能为空(workspace_name is required)") + force_clear = args.get("force_clear") == "enable" + force_clear_only_images = args.get("force_clear_only_images") == "enable" + if force_clear: + if force_clear_only_images: + for file in os.listdir(train_images_dir): + if file.lower().endswith(".png") or file.lower().endswith(".jpg"): + os.remove(os.path.join(train_images_dir, file)) + else: + shutil.rmtree(train_images_dir) + os.makedirs(train_images_dir, exist_ok=True) + saved_images_path = [] for i, pil_image in enumerate(pil_images): @@ -124,6 +135,17 @@ def MZ_HYDiTDatasetConfig_call(args={}): pil_image.save(os.path.join(train_images_dir, filename)) saved_images_path.append(filename) + same_caption_generate = args.get("same_caption_generate") == "enable" + if same_caption_generate: + same_caption = args.get("same_caption").strip() + if same_caption != "": + # 循环已经保存的图片 + for i, filename in enumerate(saved_images_path): + base_filename = os.path.splitext(filename)[0] + caption_filename = base_filename + ".caption" + with open(os.path.join(train_images_dir, caption_filename), "w", encoding="utf-8") as f: + f.write(same_caption) + return ( train_images_dir, ) @@ -346,7 +368,7 @@ def MZ_HYDiTTrain_call(args={}): writer.writerow(["image_path", "text_zh"]) for filename in full_filenames: - if filename.endswith(".png"): + if filename.lower().endswith(".png"): image_path = os.path.join(workspace_images_dir, filename) pil_image = Image.open(image_path) @@ -596,7 +618,7 @@ def get_sample_images(train_config): if os.path.exists(sample_images_dir): image_files = os.listdir(sample_images_dir) image_files = list( - filter(lambda x: x.endswith(".png"), image_files)) + filter(lambda x: x.lower().endswith(".png"), image_files)) # 筛选 output_name 前缀 image_files = list( filter(lambda x: x.startswith(output_name), image_files)) diff --git a/mz_train_tools_utils.py b/mz_train_tools_utils.py index b67a9d2..d6f2136 100644 --- a/mz_train_tools_utils.py +++ b/mz_train_tools_utils.py @@ -70,7 +70,7 @@ class Utils: files = os.listdir(path) new_files = [] for file in files: - if file.endswith(".png"): + if file.lower().endswith(".png"): new_files.append(file) files = new_files files.sort(key=lambda x: int(os.path.basename(x).split(".")[0])) diff --git a/tools/face_resize_copy_dir.py b/tools/face_resize_copy_dir.py new file mode 100644 index 0000000..ac043ff --- /dev/null +++ b/tools/face_resize_copy_dir.py @@ -0,0 +1,191 @@ +import argparse +import hashlib +import io +import os +from PIL import Image +import numpy as np +import onnxruntime as ort +import cv2 + + +class Utils: + def Md5(bytes): + return hashlib.md5(bytes).hexdigest() + + def resize_by(image, percent): + # 判断类型是否为PIL + if not isinstance(image, Image.Image): + image = Image.fromarray(image) + + width, height = image.size + new_width = int(width * percent) + new_height = int(height * percent) + return image.resize((new_width, new_height), Image.LANCZOS) + + def resize_max(im, dst_w, dst_h): + src_w, src_h = im.size + + if src_h > src_w: + newWidth = dst_w + newHeight = dst_w * src_h // src_w + else: + newWidth = dst_h * src_w // src_h + newHeight = dst_h + + newHeight = newHeight // 8 * 8 + newWidth = newWidth // 8 * 8 + + return im.resize((newWidth, newHeight), Image.Resampling.LANCZOS) + + def resize_min(im, dst_w, dst_h): + src_w, src_h = im.size + + if src_h < src_w: + newWidth = dst_w + newHeight = dst_w * src_h // src_w + else: + newWidth = dst_h * src_w // src_h + newHeight = dst_h + + newHeight = newHeight // 8 * 8 + newWidth = newWidth // 8 * 8 + + return im.resize((newWidth, newHeight), Image.Resampling.LANCZOS) + + # 根据人脸进行剪裁 + def resize_crop(im, dst_w, dst_h, focus_point): + + src_w, src_h = im.size + + if src_h > src_w: + newWidth = dst_w + newHeight = dst_w * src_h // src_w + else: + newWidth = dst_h * src_w // src_h + newHeight = dst_h + + newHeight = newHeight // 8 * 8 + newWidth = newWidth // 8 * 8 + + im = im.resize((newWidth, newHeight), Image.Resampling.LANCZOS) + + + left = (newWidth - dst_w) // 2 + top = (newHeight - dst_h) // 2 + + right = left + dst_w + bottom = top + dst_h + + + focus_point_x, focus_point_y = focus_point + offset_w = dst_w // 8 + offset_h = dst_h // 8 + if focus_point_x < left + dst_w // 8: + left = 0 + right = dst_w + elif focus_point_x > right - dst_w // 8: + right = newWidth + left = newWidth - dst_w + if focus_point_y < top + dst_h // 8: + top = 0 + bottom = dst_h + elif focus_point_y > bottom - dst_h // 8: + bottom = newHeight + top = newHeight - dst_h + + return im.crop((left, top, right, bottom)) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("-i", "--input_dir", type=str, default="") + parser.add_argument("-o", "--output_dir", type=str, default="") + parser.add_argument("-m", "--face_model_path", type=str, default="") + parser.add_argument("-s", "--score", type=float, default=0.8) + args = parser.parse_args() + limitscore = args.score + sess = ort.InferenceSession(args.face_model_path) + sess.set_providers(['CUDAExecutionProvider']) + input_dir = args.input_dir + output_dir = args.output_dir + + total = 0 + for root, dirs, files in os.walk(input_dir): + for file in files: + if not file.endswith(".png") and not file.endswith(".jpg"): + continue + total += 1 + + import tqdm + + with tqdm.tqdm(total=total) as pbar: + for root, dirs, files in os.walk(input_dir): + for file in files: + if not file.endswith(".png") and not file.endswith(".jpg"): + continue + + file_path = os.path.join(root, file) + pbar.update(1) + pbar.set_description(f"Processing {file_path}") + + with open(file_path, "rb") as f: + md5_str = Utils.Md5(f.read()) + if os.path.exists(os.path.join(output_dir, md5_str + ".png")): + continue + + image = Image.open(file_path) + image = Utils.resize_min(image, 1280, 1280) + + cv2_img = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR) + + scores, bboxes, keypoints, aligned_imgs, landmarks, affine_matrices = sess.run( + None, {"input": cv2_img}) + if len(bboxes) != 1: + continue + + bboxes = bboxes[0] + scores = scores[0] + if limitscore > scores: + continue + + affine_matrices = affine_matrices[0] + angle = np.degrees(np.arctan2( + affine_matrices[1][0], affine_matrices[0][0])) + + real_angle = 0 + if angle < -45 and angle > -135: + # 向左旋转90度 + real_angle = 90 + elif angle > 45 and angle < 135: + # 向右旋转90度 + real_angle = -90 + elif angle < -135 or angle > 135: + # 向左旋转180度 + real_angle = 180 + + if real_angle != 0: + continue + + # print(f"{file_path} bboxes: \n{bboxes} {scores} {real_angle}") + + + center_point = (bboxes[0] + bboxes[2]) / \ + 2, (bboxes[1] + bboxes[3]) / 2 + image = Utils.resize_crop(image, 960, 1280, center_point) + cv2_img = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR) + scores, bboxes, keypoints, aligned_imgs, landmarks, affine_matrices = sess.run( + None, {"input": cv2_img}) + if len(bboxes) != 1: + continue + # cv2.imshow("image", cv2.cvtColor( + # np.array(image), cv2.COLOR_RGB2BGR)) + # cv2.waitKey(0) + + output_path = os.path.join(output_dir, md5_str + ".png") + # print(f"Save {file_path} to {output_path}") + + image.save(output_path) + + +if __name__ == "__main__": + main() From 32b3c6514a53227ef927f832040a7ffb98658861 Mon Sep 17 00:00:00 2001 From: wailovet Date: Sat, 29 Jun 2024 10:17:01 +0800 Subject: [PATCH 31/64] 'update' --- mz_train_tools_core_HYDiT.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index 63a0ebb..b30d9b8 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -237,8 +237,9 @@ def check_model_auto_download(args): for download_file in download_files: download_fullpath = os.path.join( - hunyuan_base_path, download_file) - os.makedirs(os.path.dirname(download_fullpath), exist_ok=True) + hunyuan_base_path, download_file) + if not os.path.exists(download_fullpath): + os.makedirs(os.path.dirname(download_fullpath), exist_ok=True) if not os.path.exists(download_fullpath): Utils.download_file( @@ -260,7 +261,8 @@ def check_model_auto_download(args): for download_file in download_files: download_fullpath = os.path.join( hunyuan_base_path, download_file) - os.makedirs(os.path.dirname(download_fullpath), exist_ok=True) + if not os.path.exists(download_fullpath): + os.makedirs(os.path.dirname(download_fullpath), exist_ok=True) if not os.path.exists(download_fullpath): Utils.download_file( @@ -282,7 +284,8 @@ def check_model_auto_download(args): for download_file in download_files: download_fullpath = os.path.join( hunyuan_base_path, download_file) - os.makedirs(os.path.dirname(download_fullpath), exist_ok=True) + if not os.path.exists(download_fullpath): + os.makedirs(os.path.dirname(download_fullpath), exist_ok=True) if not os.path.exists(download_fullpath): Utils.download_file( @@ -304,7 +307,8 @@ def check_model_auto_download(args): for download_file in download_files: download_fullpath = os.path.join( hunyuan_base_path, download_file) - os.makedirs(os.path.dirname(download_fullpath), exist_ok=True) + if not os.path.exists(download_fullpath): + os.makedirs(os.path.dirname(download_fullpath), exist_ok=True) if not os.path.exists(download_fullpath): Utils.download_file( From c4d76a1dbb838bfe8f486afa7b4b739e493c5e9a Mon Sep 17 00:00:00 2001 From: wailovet Date: Sat, 29 Jun 2024 10:17:23 +0800 Subject: [PATCH 32/64] 'update' --- mz_train_tools_core_HYDiT.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index b30d9b8..b304d98 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -215,7 +215,8 @@ def check_model_auto_download(args): download_file = "HunyuanDiT/t2i/model/pytorch_model_module.pt" download_fullpath = os.path.join( hunyuan_base_path, download_file) - os.makedirs(os.path.dirname(download_fullpath), exist_ok=True) + if not os.path.exists(download_fullpath): + os.makedirs(os.path.dirname(download_fullpath), exist_ok=True) if os.path.exists(download_fullpath): args["unet_path"] = download_fullpath @@ -237,7 +238,7 @@ def check_model_auto_download(args): for download_file in download_files: download_fullpath = os.path.join( - hunyuan_base_path, download_file) + hunyuan_base_path, download_file) if not os.path.exists(download_fullpath): os.makedirs(os.path.dirname(download_fullpath), exist_ok=True) From d67c2c204884c3ad6b05b55271dbdf3996a875c9 Mon Sep 17 00:00:00 2001 From: wailovet Date: Sat, 29 Jun 2024 10:48:44 +0800 Subject: [PATCH 33/64] 'update' --- hook_HYDiT_utils.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/hook_HYDiT_utils.py b/hook_HYDiT_utils.py index 2c33682..a5f1a70 100644 --- a/hook_HYDiT_utils.py +++ b/hook_HYDiT_utils.py @@ -151,23 +151,23 @@ def easy_sample_images( freqs_cis_img = calc_rope(height, width) - samples = pipeline( - height=height, - width=width, - prompt=prompt, - negative_prompt=negative_prompt, - num_images_per_prompt=batch_size, - guidance_scale=guidance_scale, - num_inference_steps=infer_steps, - style=style, - return_dict=False, - use_fp16=True, - learn_sigma=args.learn_sigma, - freqs_cis_img=freqs_cis_img, - image_meta_size=image_meta_size, - )[0] - try: + samples = pipeline( + height=height, + width=width, + prompt=prompt, + negative_prompt=negative_prompt, + num_images_per_prompt=batch_size, + guidance_scale=guidance_scale, + num_inference_steps=infer_steps, + style=style, + return_dict=False, + use_fp16=True, + learn_sigma=args.learn_sigma, + freqs_cis_img=freqs_cis_img, + image_meta_size=image_meta_size, + )[0] + pass except Exception as e: print(f"Failed to sample images: {e}") From 7c5c414f0604cf049e46e231104bbb1f5fcc9452 Mon Sep 17 00:00:00 2001 From: wailovet Date: Sat, 29 Jun 2024 22:51:05 +0800 Subject: [PATCH 34/64] 'update' --- __init__.py | 64 +++++- hook_HYDiT_main_train_deepspeed.py | 15 +- hook_HYDiT_run.py | 11 +- hook_HYDiT_utils.py | 72 +++++- hook_kohya_ss_utils.py | 53 +---- mz_train_tools_core_HYDiT.py | 354 ++++++++++++++++++++++++++++- mz_train_tools_utils.py | 10 +- 7 files changed, 509 insertions(+), 70 deletions(-) diff --git a/__init__.py b/__init__.py index 930b314..9acde5a 100644 --- a/__init__.py +++ b/__init__.py @@ -539,7 +539,7 @@ class MZ_HYDiTTrain: "vae_ema_path": (["auto"] + folders + vae_models, {"default": "auto"}), "text_encoder_path": (["auto"] + folders, {"default": "auto"}), "tokenizer_path": (["auto"] + folders, {"default": "auto"}), - "t5_encoder_path": (["auto"] + folders, {"default": "auto"}), + "t5_encoder_path": (["none", "auto"] + folders, {"default": "auto"}), "resolution": ("INT", {"default": 1024}), "batch_size": ("INT", {"default": 1}), "epochs": ("INT", {"default": 50}), @@ -561,7 +561,7 @@ class MZ_HYDiTTrain: FUNCTION = "start" MZ_DESC = """ -base_size The base resolution (n, n) from which to create multiple resolutions | Recommended values: 256/512/1024 +base_size The base resolution (n, n) from which to create multiple resolutions | Recommended values: 256/512/1024 """ OUTPUT_NODE = True @@ -577,6 +577,66 @@ NODE_CLASS_MAPPINGS["MZ_HYDiTTrain"] = MZ_HYDiTTrain NODE_DISPLAY_NAME_MAPPINGS["MZ_HYDiTTrain"] = f"{AUTHOR_NAME} - HYDiTTrain" +class MZ_HYDiTSimpleT2I: + @classmethod + def INPUT_TYPES(s): + hunyuan_models_path = os.path.join( + Utils.get_comfyui_models_path(), "hunyuan") + os.makedirs(hunyuan_models_path, exist_ok=True) + + models = Utils.get_models_by_folder(hunyuan_models_path) + folders = Utils.get_folders_by_folder(hunyuan_models_path) + + vae_models = Utils.get_models_by_folder( + os.path.join(Utils.get_comfyui_models_path(), "vae")) + unet_models = Utils.get_models_by_folder( + os.path.join(Utils.get_comfyui_models_path(), "unet")) + + comfyui_full_loras = [] + comfyui_loras = folder_paths.get_filename_list("loras") + for lora in comfyui_loras: + lora_path = folder_paths.get_full_path("loras", lora) + comfyui_full_loras.append(lora_path) + + return { + "required": { + "unet_path": (["auto"] + models + unet_models, {"default": "auto"}), + "vae_ema_path": (["auto"] + folders + vae_models, {"default": "auto"}), + "text_encoder_path": (["auto"] + folders, {"default": "auto"}), + "tokenizer_path": (["auto"] + folders, {"default": "auto"}), + "t5_encoder_path": (["none", "auto"] + folders, {"default": "auto"}), + "lora_path": (["none"] + comfyui_full_loras, {"default": "none"}), + "seed": ("INT", {"default": 0}), + "steps": ("INT", {"default": 20}), + "cfg": ("FLOAT", {"default": 8.0, "min": 0.0, "max": 100.0, "step": 0.1, "round": 0.01}), + "scheduler": ([ + "ddpm", "ddim", "dpmms", + ], {"default": "ddpm"}), + "prompt": ("STRING", {"default:": "", "dynamicPrompts": True, "multiline": True}), + "negative_prompt": ("STRING", {"default:": "", "dynamicPrompts": True, "multiline": True}), + "width": ("INT", {"default": 512, "max": 8192, "step": 16}), + "height": ("INT", {"default": 512, "max": 8192, "step": 16}), + "keep_device": (["enable", "disable"], {"default": "enable"}), + } + } + + RETURN_TYPES = ("IMAGE",) + + RETURN_NAMES = ("images",) + + FUNCTION = "start" + + CATEGORY = CATEGORY_NAME + + def start(self, **kwargs): + importlib.reload(mz_train_tools_core_HYDiT) + return mz_train_tools_core_HYDiT.MZ_HYDiTSimpleT2I_call(kwargs) + + +NODE_CLASS_MAPPINGS["MZ_HYDiTSimpleT2I"] = MZ_HYDiTSimpleT2I +NODE_DISPLAY_NAME_MAPPINGS["MZ_HYDiTSimpleT2I"] = f"{AUTHOR_NAME} - HYDiTSimpleT2I" + + class MZ_TrainToolsDebug: @classmethod def INPUT_TYPES(s): diff --git a/hook_HYDiT_main_train_deepspeed.py b/hook_HYDiT_main_train_deepspeed.py index e494616..521d3fc 100644 --- a/hook_HYDiT_main_train_deepspeed.py +++ b/hook_HYDiT_main_train_deepspeed.py @@ -287,9 +287,9 @@ def Core(args): ) try: - from .hook_HYDiT_utils import VAE_EMA_PATH, TEXT_ENCODER, TOKENIZER, T5_ENCODER, easy_sample_images, model_resume, PBar + from .hook_HYDiT_utils import VAE_EMA_PATH, TEXT_ENCODER, TOKENIZER, T5_ENCODER, easy_sample_images, model_resume, PBar, CustomizeEmbeds except: - from hook_HYDiT_utils import VAE_EMA_PATH, TEXT_ENCODER, TOKENIZER, T5_ENCODER, easy_sample_images, model_resume, PBar + from hook_HYDiT_utils import VAE_EMA_PATH, TEXT_ENCODER, TOKENIZER, T5_ENCODER, easy_sample_images, model_resume, PBar, CustomizeEmbeds # Setup VAE logger.info(f" Loading vae from {VAE_EMA_PATH}") @@ -304,8 +304,11 @@ def Core(args): # Setup T5 text encoder from hydit.modules.text_encoder import MT5Embedder mt5_path = T5_ENCODER['MT5'] - embedder_t5 = MT5Embedder( - mt5_path, torch_dtype=T5_ENCODER['torch_dtype'], max_length=args.text_len_t5) + if mt5_path is None: + embedder_t5 = CustomizeEmbeds() + else: + embedder_t5 = MT5Embedder( + mt5_path, torch_dtype=T5_ENCODER['torch_dtype'], max_length=args.text_len_t5) tokenizer_t5 = embedder_t5.tokenizer text_encoder_t5 = embedder_t5.model @@ -550,10 +553,10 @@ def Core(args): gc.collect() pbar.step( - f"Epoch {epoch}, step {step}, loss {loss.item():.4f}, mean_loss {running_loss / step:.4f}", epoch * len(loader), train_steps) + f"Epoch {epoch}, step {step}, loss {loss.item():.4f}, mean_loss {running_loss / step:.4f}", args.epochs * len(loader), train_steps) if (train_steps % args.ckpt_every == 0 or train_steps % args.ckpt_latest_every == 0 # or train_steps == args.max_training_steps - ) and train_steps > 0: + ) and train_steps > 0: easy_sample_images(args, vae, text_encoder, tokenizer, model, embedder_t5, target_height=768, target_width=1280, train_steps=train_steps) save_checkpoint(args, rank, logger, model, ema, diff --git a/hook_HYDiT_run.py b/hook_HYDiT_run.py index 20f9422..e5300e3 100644 --- a/hook_HYDiT_run.py +++ b/hook_HYDiT_run.py @@ -49,7 +49,6 @@ if __name__ == "__main__": print("HYDi run hook") - try: from . import hook_HYDiT_main_train_deepspeed except Exception as e: @@ -108,14 +107,20 @@ if __name__ == "__main__": margs.index_file = train_config.get("index_file", None) margs.lr = train_config.get("lr", 1e-5) + + margs.rank = train_config.get("rank", 8) - margs.noise_offset = train_config.get("noise_offset", 0.1) + margs.noise_offset = train_config.get("noise_offset", 0.0) margs.log_every = train_config.get("log_every", 99999999999999) - margs.use_zero_stage = 2 + + margs.use_zero_stage = train_config.get("use_zero_stage", 2) + + margs.global_batch_size = train_config.get("global_batch_size", 1) margs.deepspeed = True margs.results_dir = train_config.get("results_dir") + margs.mse_loss_weight_type = train_config.get( "mse_loss_weight_type", "constant") diff --git a/hook_HYDiT_utils.py b/hook_HYDiT_utils.py index a5f1a70..eadbe30 100644 --- a/hook_HYDiT_utils.py +++ b/hook_HYDiT_utils.py @@ -2,6 +2,7 @@ import json import os +import time import torch @@ -38,7 +39,7 @@ def set_text_encoder_path(path): T5_ENCODER = { - 'MT5': 'ckpts/t2i/mt5', + 'MT5': None, 'attention_mask': True, 'layer_index': -1, 'attention_pool': True, @@ -82,6 +83,8 @@ def easy_sample_images( from hydit.constants import SAMPLER_FACTORY from hydit.modules.posemb_layers import get_fill_resize_and_crop, get_2d_rotary_pos_embed from hydit.modules.models import HUNYUAN_DIT_CONFIG + + import traceback with torch.no_grad(): workspace_dir = TRAIN_CONFIG.get("workspace_dir") @@ -130,6 +133,19 @@ def easy_sample_images( requires_safety_checker=False, embedder_t5=embedder_t5, ).to("cuda") + # attr _execution_device is not defined + if not hasattr(pipeline, "_execution_device"): + time.sleep(1) + pipeline = StableDiffusionPipeline(vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=model.module, + scheduler=scheduler, + feature_extractor=None, + safety_checker=None, + requires_safety_checker=False, + embedder_t5=embedder_t5, + ).to("cuda") style = torch.as_tensor([0, 0] * batch_size, device="cuda") @@ -152,6 +168,7 @@ def easy_sample_images( freqs_cis_img = calc_rope(height, width) try: + samples = pipeline( height=height, width=width, @@ -170,7 +187,9 @@ def easy_sample_images( pass except Exception as e: - print(f"Failed to sample images: {e}") + print(f"Failed to sample images: {e} ") + # 打印堆栈信息 + traceback.print_exc() continue # print("samples:",type(samples),) @@ -276,3 +295,52 @@ class PBar: "total_steps": total_steps, # "latent": noise_pred_latent_path, }) + + +from torch import nn + + +class CustomizeEmbedsModel(nn.Module): + dtype = torch.float16 + x = torch.zeros(1, 1, 256, 2048) + + def __init__(self, *args, **kwargs): + super().__init__() + + def to(self, *args, **kwargs): + self.dtype = torch.float16 + return self + + def forward(self, *args, **kwargs): + if self.output_hidden_states or kwargs.get("output_hidden_states", False): + return { + "hidden_states": self.x.to("cuda"), + "input_ids": torch.zeros(1, 1), + } + return self.x + + +class CustomizeTokenizer(dict): + + added_tokens_encoder = [] + input_ids = torch.zeros(1, 256) + attention_mask = torch.zeros(1, 256) + + def __init__(self): + self['added_tokens_encoder'] = self.added_tokens_encoder + self['input_ids'] = self.input_ids + self['attention_mask'] = self.attention_mask + + def tokenize(self, text): + return text + + def __call__(self, *args, **kwargs): + return self + + +class CustomizeEmbeds(): + def __init__(self): + super().__init__() + self.tokenizer = CustomizeTokenizer() + self.model = CustomizeEmbedsModel().to("cuda") + self.max_length = 256 diff --git a/hook_kohya_ss_utils.py b/hook_kohya_ss_utils.py index f347a1f..e1f6031 100644 --- a/hook_kohya_ss_utils.py +++ b/hook_kohya_ss_utils.py @@ -216,10 +216,6 @@ def hook_kohya_ss(): import library.utils import library.train_util import library.sdxl_train_util - import library.model_util - - library.model_util.convert_controlnet_state_dict_to_diffusers = convert_controlnet_state_dict_to_diffusers - library.utils.setup_logging = setup_logging library.train_util.load_tokenizer = load_tokenizers @@ -354,7 +350,7 @@ def _load_target_model(args: argparse.Namespace, weight_dtype, device="cpu", une return text_encoder, vae, unet, load_stable_diffusion_format -def generate_image(pipe_class, cmd_args, accelerator, vae, tokenizer, text_encoder, unet, epoch, prompt_dict_list, controlnet=None): +def generate_image(pipe_class, cmd_args, accelerator, vae, tokenizer, text_encoder, unet, epoch, prompt_dict_list): import library.train_util # for multi gpu distributed inference. this is a singleton, so it's safe to use it here @@ -373,10 +369,6 @@ def generate_image(pipe_class, cmd_args, accelerator, vae, tokenizer, text_encod v_parameterization=cmd_args.v_parameterization, ) - # print("unet.config._diffusers_version:", unet.config._diffusers_version) - # print("unet.config:", unet.config) - # input("press enter to continue...") - pipeline = pipe_class( text_encoder=text_encoder, vae=vae, @@ -402,6 +394,7 @@ def generate_image(pipe_class, cmd_args, accelerator, vae, tokenizer, text_encod save_dir = sample_images_path prompt_replacement = None steps = 0 + controlnet = None # save random state to restore later rng_state = torch.get_rng_state() @@ -413,9 +406,6 @@ def generate_image(pipe_class, cmd_args, accelerator, vae, tokenizer, text_encod with torch.no_grad(): for prompt_dict in prompt_dict_list: - print("prompt_dict:", json.dumps( - prompt_dict, indent=4, ensure_ascii=False)) - library.train_util.sample_image_inference( accelerator, cmd_args, pipeline, save_dir, prompt_dict, epoch, steps, prompt_replacement, controlnet=controlnet ) @@ -427,42 +417,3 @@ def generate_image(pipe_class, cmd_args, accelerator, vae, tokenizer, text_encod if cuda_rng_state is not None: torch.cuda.set_rng_state(cuda_rng_state) vae.to(org_vae_device) - - -def convert_controlnet_state_dict_to_diffusers(controlnet_state_dict): - import library.model_util as model_util - unet_conversion_map, unet_conversion_map_resnet, unet_conversion_map_layer = model_util.controlnet_conversion_map() - - mapping = {k: k for k in controlnet_state_dict.keys()} - - # print("controlnet_state_dict:", controlnet_state_dict.keys()) - keys_first = list(controlnet_state_dict.keys())[0] - - if keys_first.startswith("control_model."): - new_mapping = {} - for k, v in mapping.items(): - new_mapping[k.replace("control_model.", "")] = v.replace( - "control_model.", "") - mapping = new_mapping - - for sd_name, diffusers_name in unet_conversion_map: - mapping[sd_name] = diffusers_name - for k, v in mapping.items(): - for sd_part, diffusers_part in unet_conversion_map_layer: - v = v.replace(sd_part, diffusers_part) - mapping[k] = v - for k, v in mapping.items(): - if "resnets" in v: - for sd_part, diffusers_part in unet_conversion_map_resnet: - v = v.replace(sd_part, diffusers_part) - mapping[k] = v - - if keys_first.startswith("control_model."): - new_state_dict = { - v: controlnet_state_dict[ - f"control_model.{k}"] for k, v in mapping.items()} - else: - new_state_dict = { - v: controlnet_state_dict[k] for k, v in mapping.items()} - - return new_state_dict diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index b304d98..dd34ae2 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -10,8 +10,10 @@ import json import subprocess import time +import safetensors.torch import torch + from .mz_train_tools_utils import Utils import folder_paths import nodes @@ -519,7 +521,7 @@ def MZ_HYDiTTrain_call(args={}): "vae_ema_path": args.get("vae_ema_path"), "text_encoder_path": args.get("text_encoder_path"), "tokenizer_path": args.get("tokenizer_path"), - "t5_encoder_path": args.get("t5_encoder_path"), + "t5_encoder_path": args.get("t5_encoder_path") if args.get("t5_encoder_path") != "none" else None, "results_dir": output_dir, "task_flag": output_name, }) @@ -641,3 +643,353 @@ def get_sample_images(train_config): ) return result return None + + +def get_model_from_path(model_path, lora_path, width, height): + + from hydit.modules.models import HunYuanDiT, HUNYUAN_DIT_CONFIG + + from types import SimpleNamespace + + # self.text_states_dim = args.text_states_dim + # self.text_states_dim_t5 = args.text_states_dim_t5 + # self.text_len = args.text_len + # self.text_len_t5 = args.text_len_t5 + # self.norm = args.norm + # use_flash_attn = args.infer_mode == 'fa' or args.use_flash_attn + + args = SimpleNamespace() + args.learn_sigma = True + args.text_states_dim = 1024 + args.text_states_dim_t5 = 2048 + args.text_len = 77 + args.text_len_t5 = 256 + args.norm = "layer" + args.infer_mode = "torch" + args.use_fp16 = True + args.qk_norm = True + args.use_flash_attn = False + + model_config = HUNYUAN_DIT_CONFIG["DiT-g/2"] + + latent_size = (width // 8, height // 8) + + model = HunYuanDiT(args, + input_size=latent_size, + **model_config, + ).half().to("cuda") + state_dict = torch.load( + model_path, map_location=lambda storage, loc: storage) + model.load_state_dict(state_dict) + + if lora_path is not None: + from peft import PeftConfig + + # 查询后缀 + if lora_path.lower().endswith(".safetensors"): + from safetensors.torch import load_file + adapter_state_dict = load_file(lora_path) + else: + adapter_state_dict = torch.load(lora_path) + + peft_config = PeftConfig() + + # 相同目录下同名的json文件 + lora_json_path = os.path.splitext(lora_path)[0] + ".json" + if not os.path.exists(lora_json_path): + raise Exception(f"未找到对应的json文件: {lora_json_path}") + with open(lora_json_path, "r", encoding="utf-8") as f: + loaded_attributes = json.load(f) + + for key, value in loaded_attributes.items(): + setattr(peft_config, key, value) + + model.load_adapter( + adapter_state_dict=adapter_state_dict, + peft_config=peft_config, + ) + model.merge_and_unload() + + return model + + +def clean_unet(): + unet_data = Utils.cache_get("HYDiT_UNET") + if unet_data is not None: + unet_data["model"].cpu() + Utils.cache_set("HYDiT_UNET", None) + torch.cuda.empty_cache() + + +def clean_vae(): + vae_data = Utils.cache_get("HYDiT_VAE_EMA") + if vae_data is not None: + vae_data["model"].cpu() + Utils.cache_set("HYDiT_VAE_EM", None) + torch.cuda.empty_cache() + + +def clean_clip_text_encoder(): + clip_text_encoder = Utils.cache_get("HYDiT_CLIP_TEXT_ENCODER") + if clip_text_encoder is not None: + clip_text_encoder["model"].cpu() + Utils.cache_set("HYDiT_CLIP_TEXT_ENCODER", None) + torch.cuda.empty_cache() + + +def clean_t5_encoder(): + t5_encoder = Utils.cache_get("HYDiT_T5_ENCODER") + if t5_encoder is not None: + t5_encoder["model"].cpu() + Utils.cache_set("HYDiT_T5_ENCODER", None) + torch.cuda.empty_cache() + + +from torch import nn + + +class CustomizeEmbedsModel(nn.Module): + dtype = torch.float16 + x = torch.zeros(1, 1, 256, 2048) + + def __init__(self, *args, **kwargs): + super().__init__() + + def to(self, *args, **kwargs): + self.dtype = torch.float16 + return self + + def forward(self, *args, **kwargs): + if self.output_hidden_states or kwargs.get("output_hidden_states", False): + return { + "hidden_states": self.x.to("cuda"), + "input_ids": torch.zeros(1, 1), + } + return self.x + + +class CustomizeTokenizer(dict): + + added_tokens_encoder = [] + input_ids = torch.zeros(1, 256) + attention_mask = torch.zeros(1, 256) + + def __init__(self): + self['added_tokens_encoder'] = self.added_tokens_encoder + self['input_ids'] = self.input_ids + self['attention_mask'] = self.attention_mask + + def tokenize(self, text): + return text + + def __call__(self, *args, **kwargs): + return self + + +class CustomizeEmbeds(): + def __init__(self): + super().__init__() + self.tokenizer = CustomizeTokenizer() + self.model = CustomizeEmbedsModel() + self.max_length = 256 + + +def MZ_HYDiTSimpleT2I_call(args={}): + check_required() + args = check_model_auto_download(args) + + HYDiT_tool_dir = os.path.join( + Utils.get_minus_zone_models_path(), "train_tools", "HunyuanDiT") + if HYDiT_tool_dir not in sys.path: + sys.path.append(HYDiT_tool_dir) + + import hydit.inference + from hydit.inference import SAMPLER_FACTORY, HUNYUAN_DIT_CONFIG, get_fill_resize_and_crop, get_2d_rotary_pos_embed + from hydit.modules.text_encoder import MT5Embedder + from hydit.utils.tools import set_seeds + from PIL import Image + from diffusers import schedulers, AutoencoderKL + from transformers import BertModel, BertTokenizer + + width = args.get("width") + height = args.get("height") + + unet_path = args.get("unet_path") + lora_path = args.get("lora_path") + unet_cache_key = f"HYDiT_UNET" + unet_data = Utils.cache_get(unet_cache_key) + if unet_data is None or unet_data.get("unet_path") != unet_path or unet_data.get("lora_path") != lora_path: + clean_unet() + _lora_path = lora_path if lora_path != "none" else None + unet = get_model_from_path( + args.get("unet_path"), _lora_path, width, height) + Utils.cache_set(unet_cache_key, { + "model": unet, + "unet_path": unet_path, + "lora_path": lora_path, + }) + else: + unet = unet_data["model"] + + text_encoder_path = args.get("text_encoder_path") + clip_text_encoder_data = Utils.cache_get("HYDiT_CLIP_TEXT_ENCODER") + + if clip_text_encoder_data is None or clip_text_encoder_data.get("text_encoder_path") != text_encoder_path: + clean_clip_text_encoder() + clip_text_encoder = BertModel.from_pretrained( + text_encoder_path, False, revision=None, local_files_only=True).to("cuda") + Utils.cache_set("HYDiT_CLIP_TEXT_ENCODER", { + "model": clip_text_encoder, + "text_encoder_path": text_encoder_path, + }) + else: + clip_text_encoder = clip_text_encoder_data["model"] + + tokenizer_path = args.get("tokenizer_path") + text_tokenizer = BertTokenizer.from_pretrained( + tokenizer_path, revision=None, local_files_only=True) + + vae_ema_path = args.get("vae_ema_path") + vae_ema_data = Utils.cache_get("HYDiT_VAE_EMA") + if vae_ema_data is None or vae_ema_data.get("vae_ema_path") != vae_ema_path: + clean_vae() + if os.path.isdir(vae_ema_path): + vae = AutoencoderKL.from_pretrained( + vae_ema_path, local_files_only=True).to("cuda") + else: + vae = AutoencoderKL.from_single_file( + vae_ema_path, local_files_only=True).to("cuda") + + Utils.cache_set("HYDiT_VAE_EMA", { + "model": vae, + "vae_ema_path": vae_ema_path, + }) + else: + vae = vae_ema_data["model"] + + t5_encoder_path = args.get("t5_encoder_path") + if t5_encoder_path == "none": + t5_encoder = CustomizeEmbeds() + else: + t5_encoder_data = Utils.cache_get("HYDiT_T5_ENCODER") + if t5_encoder_data is None or t5_encoder_data.get("t5_encoder_path") != t5_encoder_path: + clean_t5_encoder() + t5_encoder = MT5Embedder( + t5_encoder_path, torch_dtype=torch.float16, max_length=256) + Utils.cache_set("HYDiT_T5_ENCODER", { + "model": t5_encoder, + "t5_encoder_path": t5_encoder_path, + }) + else: + t5_encoder = t5_encoder_data["model"] + + sampler = args.get("scheduler") + scheduler = SAMPLER_FACTORY[sampler]['scheduler'] + kwargs = SAMPLER_FACTORY[sampler]['kwargs'] + # Build scheduler according to the sampler. + scheduler_class = getattr(schedulers, scheduler) + scheduler = scheduler_class(**kwargs) + + scheduler.set_timesteps(args.get("steps"), "cuda") + + from hydit.diffusion.pipeline import StableDiffusionPipeline + pipeline = StableDiffusionPipeline(vae=vae, + text_encoder=clip_text_encoder, + tokenizer=text_tokenizer, + unet=unet, + scheduler=scheduler, + feature_extractor=None, + safety_checker=None, + requires_safety_checker=False, + progress_bar_config=None, + embedder_t5=t5_encoder, + infer_mode="torch", + ) + + pipeline = pipeline.to("cuda") + + seed = args.get("seed", 0) + target_height = int((height // 16) * 16) + target_width = int((width // 16) * 16) + prompt = args.get("prompt", "").strip() + negative_prompt = args.get("negative_prompt", "").strip() + + batch_size = 1 + style = torch.as_tensor([0, 0] * batch_size, device="cuda") + + src_size_cond = (target_width, target_height) + size_cond = list(src_size_cond) + [target_width, target_height, 0, 0] + image_meta_size = torch.as_tensor( + [size_cond] * 2 * batch_size, device="cuda",) + + def calc_rope(height, width): + model_config = HUNYUAN_DIT_CONFIG["DiT-g/2"] + patch_size = model_config['patch_size'] + head_size = model_config['hidden_size'] // model_config['num_heads'] + th = height // 8 // patch_size + tw = width // 8 // patch_size + base_size = 512 // 8 // patch_size + start, stop = get_fill_resize_and_crop((th, tw), base_size) + sub_args = [start, stop, (th, tw)] + rope = get_2d_rotary_pos_embed(head_size, *sub_args) + return rope + + freqs_cis_img = calc_rope(target_height, target_width) + guidance_scale = args.get("cfg", 7.0) + infer_steps = args.get("steps", 20) + + with torch.inference_mode(): + with torch.cuda.amp.autocast(): + + pbar = Utils.progress_bar(0, "sdxl") + preview = pbar.get_previewer() + + def callback(step: int, timestep: int, latents: torch.FloatTensor, pred_x0): + try: + pil_img = preview.decode_latent_to_preview_image( + None, + latents, + )[1] + pbar.update(step, infer_steps, pil_img) + except Exception as e: + print("decode_tensors error:", e) + generator = set_seeds(seed, device="cuda") + samples = pipeline( + height=height, + width=width, + prompt=prompt, + negative_prompt=negative_prompt, + num_images_per_prompt=batch_size, + guidance_scale=guidance_scale, + num_inference_steps=infer_steps, + style=style, + return_dict=False, + generator=generator, + use_fp16=True, + learn_sigma=True, + freqs_cis_img=freqs_cis_img, + image_meta_size=image_meta_size, + callback=callback, + callback_steps=2, + )[0] + + if type(samples) == list: + pil_image = samples[0] + else: + pil_image = samples + + tensor_image = Utils.pil2tensor(pil_image) + + keep_device = args.get("keep_device", "disable") == "enable" + if not keep_device: + del pipeline + del unet + del clip_text_encoder + del vae + del t5_encoder + clean_unet() + clean_clip_text_encoder() + clean_vae() + torch.cuda.empty_cache() + + return (Utils.list_tensor2tensor([tensor_image]),) diff --git a/mz_train_tools_utils.py b/mz_train_tools_utils.py index d6f2136..b21b147 100644 --- a/mz_train_tools_utils.py +++ b/mz_train_tools_utils.py @@ -710,12 +710,12 @@ class Utils: return previewer def update(self, step, total_steps, pil_img=None): - pil_img_info = ("JPEG", pil_img, 512) - if pil_img is None: - pil_img_info = None - if type(pil_img) == Tuple or type(pil_img) == list or type(pil_img) == tuple: - pil_img_info = pil_img try: + pil_img_info = ("JPEG", pil_img, 512) + if pil_img is None: + pil_img_info = None + if type(pil_img) == Tuple or type(pil_img) == list or type(pil_img) == tuple: + pil_img_info = pil_img # print("pil_img_info:", type(pil_img), pil_img_info) # print("step:", step, "total_steps:", total_steps) self.pbar.update_absolute( From d87d098bc83f1ee28651c69944c06e25c4525957 Mon Sep 17 00:00:00 2001 From: wailovet Date: Sun, 30 Jun 2024 01:31:22 +0800 Subject: [PATCH 35/64] 'update' --- __init__.py | 69 +++++++++++++++++++++++------- hook_HYDiT_main_train_deepspeed.py | 21 ++++++--- hook_HYDiT_run.py | 25 +++++++++-- hook_HYDiT_utils.py | 21 +++------ mz_train_tools_core_HYDiT.py | 56 ++++++++++++++++++++++-- 5 files changed, 147 insertions(+), 45 deletions(-) diff --git a/__init__.py b/__init__.py index 9acde5a..19f0b59 100644 --- a/__init__.py +++ b/__init__.py @@ -416,7 +416,7 @@ class MZ_HYDiTInitWorkspace: def start(self, **kwargs): importlib.reload(mz_train_tools_core_HYDiT) - return mz_train_tools_core_HYDiT.MZ_HYDiTInitWorkspace_call(kwargs) + return mz_train_tools_core_HYDiT.MZ_HYDiTInitWorkspace_call(kwargs.copy()) NODE_CLASS_MAPPINGS["MZ_HYDiTInitWorkspace"] = MZ_HYDiTInitWorkspace @@ -449,7 +449,7 @@ class MZ_HYDiTDatasetConfig: def start(self, **kwargs): importlib.reload(mz_train_tools_core_HYDiT) - return mz_train_tools_core_HYDiT.MZ_HYDiTDatasetConfig_call(kwargs) + return mz_train_tools_core_HYDiT.MZ_HYDiTDatasetConfig_call(kwargs.copy()) NODE_CLASS_MAPPINGS["MZ_HYDiTDatasetConfig"] = MZ_HYDiTDatasetConfig @@ -491,14 +491,40 @@ class MZ_HYDiTAdvConfig: def INPUT_TYPES(s): return { "required": { - "learn_sigma": (["enable", "disable"], {"default": "enable"}), - "text_states_dim": ("INT", {"default": 1024}), - "text_states_dim_t5": ("INT", {"default": 2048}), - "text_len": ("INT", {"default": 77}), - "text_len_t5": ("INT", {"default": 256}), - "norm": (["layer", "instance"], {"default": "layer"}), - "infer_mode": (["torch", "deepspeed"], {"default": "torch"}), - "use_fp16": (["enable", "disable"], {"default": "enable"}), + "lr": ("FLOAT", {"default": 1e-5}), + "rope_real": (["enable", "disable"], {"default": "enable"}), + # ['Wqkv', 'q_proj', 'kv_proj', 'out_proj'] + "target_modules_Wqkv": (["enable", "disable"], {"default": "enable"}), + "target_modules_q_proj": (["enable", "disable"], {"default": "enable"}), + "target_modules_kv_proj": (["enable", "disable"], {"default": "enable"}), + "target_modules_out_proj": (["enable", "disable"], {"default": "enable"}), + "warmup_min_lr": ("FLOAT", {"default": 1e-6}), + # parser.add_argument("--warmup-num-steps", type=float, default=0) + # parser.add_argument("--weight-decay", type=float, default=0, help="weight-decay in optimizer") + "weight_decay": ("FLOAT", {"default": 0}), + "warmup_num_steps": ("FLOAT", {"default": 0}), + # parser.add_argument("--uncond-p", type=float, default=0.2, + # help="The probability of dropping training text used for CLIP feature extraction") + # parser.add_argument("--uncond-p-t5", type=float, default=0.2, + # help="The probability of dropping training text used for mT5 feature extraction") + "uncond_p": ("FLOAT", {"default": 0.2}), + "uncond_p_t5": ("FLOAT", {"default": 0.2}), + + # parser.add_argument("--use-flash-attn", action="store_true", help="During training, " + # "flash attention is used to accelerate training.") + # parser.add_argument("--no-flash-attn", dest="use_flash_attn", + # action="store_false", help="During training, flash attention is not used to accelerate training.") + # parser.add_argument("--use-zero-stage", type=int, default=1, help="Use AngelPTM zero stage. Support 2 and 3") + # parser.add_argument("--grad-accu-steps", type=int, default=1, help="Gradient accumulation steps.") + "use_flash_attn": (["enable", "disable"], {"default": "disable"}), + "use_zero_stage": ("INT", {"default": 2}), + "grad_accu_steps": ("INT", {"default": 1}), + # parser.add_argument("--extra-fp16", action="store_true", help="Use extra fp16 for vae and text_encoder.") + "extra_fp16": (["enable", "disable"], {"default": "enable"}), + # parser.add_argument("--qk-norm", action="store_true", help="Query Key normalization. See http://arxiv.org/abs/2302.05442 for details.") + "qk_norm": (["enable", "disable"], {"default": "enable"}), + # parser.add_argument("--norm", type=str, choices=["rms", "laryer"], default="layer", help="Normalization layer type") + "norm": (["rms", "layer"], {"default": "layer"}), } } @@ -510,7 +536,7 @@ class MZ_HYDiTAdvConfig: CATEGORY = CATEGORY_NAME def start(self, **kwargs): - return (kwargs,) + return (kwargs.copy(),) NODE_CLASS_MAPPINGS["MZ_HYDiTAdvConfig"] = MZ_HYDiTAdvConfig @@ -532,24 +558,35 @@ class MZ_HYDiTTrain: os.path.join(Utils.get_comfyui_models_path(), "vae")) unet_models = Utils.get_models_by_folder( os.path.join(Utils.get_comfyui_models_path(), "unet")) + + workspaces_root = os.path.join( + folder_paths.output_directory, "mz_train_workspaces") + + loras = mz_train_tools_core_HYDiT.search_loras([ + workspaces_root, + os.path.join(Utils.get_comfyui_models_path(), "loras"), + ]) + return { "required": { "workspace_config": ("MZ_TT_HYDiT_WorkspaceConfig",), "unet_path": (["auto"] + models + unet_models, {"default": "auto"}), + "ema_to_module": (["enable", "disable"], {"default": "enable"}), "vae_ema_path": (["auto"] + folders + vae_models, {"default": "auto"}), "text_encoder_path": (["auto"] + folders, {"default": "auto"}), "tokenizer_path": (["auto"] + folders, {"default": "auto"}), - "t5_encoder_path": (["none", "auto"] + folders, {"default": "auto"}), + "t5_encoder_path": (["none", "auto"] + folders, {"default": "none"}), "resolution": ("INT", {"default": 1024}), "batch_size": ("INT", {"default": 1}), "epochs": ("INT", {"default": 50}), - "ckpt_every": ("INT", {"default": 10}), - "ema_to_module": (["enable", "disable"], {"default": "enable"}), - "base_lora": (["latest", "empty"], {"default": "latest"}), + "ckpt_every": ("INT", {"default": 500}), + "rank": ("INT", {"default": 8}), + "base_lora": (["latest", "empty"] + loras, {"default": "latest"}), "sample_generate": (["enable", "disable"], {"default": "enable"}), "sample_prompt": ("STRING", {"default:": "", "dynamicPrompts": True, "multiline": True}), }, "optional": { + "advanced_config": ("MZ_TT_HYDiT_AdvConfig",), "workspace_images_dir": ("STRING", {"forceInput": True}), "has_no_effect": (AlwaysEqualProxy("*"),), } @@ -570,7 +607,7 @@ base_size The base resolution (n, n) from which to create multiple resolutions | def start(self, **kwargs): importlib.reload(mz_train_tools_core_HYDiT) - return mz_train_tools_core_HYDiT.MZ_HYDiTTrain_call(kwargs) + return mz_train_tools_core_HYDiT.MZ_HYDiTTrain_call(kwargs.copy()) NODE_CLASS_MAPPINGS["MZ_HYDiTTrain"] = MZ_HYDiTTrain diff --git a/hook_HYDiT_main_train_deepspeed.py b/hook_HYDiT_main_train_deepspeed.py index 521d3fc..73d50eb 100644 --- a/hook_HYDiT_main_train_deepspeed.py +++ b/hook_HYDiT_main_train_deepspeed.py @@ -44,6 +44,9 @@ def deepspeed_initialize(args, logger, model, opt, deepspeed_config): logger.info( f" Building scheduler with warmup_min_lr={args.warmup_min_lr}, warmup_num_steps={args.warmup_num_steps}") + logger.info( + f" deepspeed_config={deepspeed_config}") + model, opt, _, scheduler = deepspeed.initialize( model=model, model_parameters=get_trainable_params(model), @@ -381,11 +384,17 @@ def Core(args): args, model, ema, logger) if args.training_parts == "lora": - loraconfig = LoraConfig( - r=args.rank, - lora_alpha=args.rank, - target_modules=args.target_modules - ) + lora_ckpt = args.lora_ckpt + if lora_ckpt is not None: + from peft import PeftConfig + loraconfig = PeftConfig.from_pretrained(lora_ckpt) + else: + loraconfig = LoraConfig( + r=args.rank, + lora_alpha=args.rank, + target_modules=args.target_modules + ) + if args.use_fp16: model.module = get_peft_model(model.module, loraconfig) else: @@ -553,7 +562,7 @@ def Core(args): gc.collect() pbar.step( - f"Epoch {epoch}, step {step}, loss {loss.item():.4f}, mean_loss {running_loss / step:.4f}", args.epochs * len(loader), train_steps) + f"Epoch {epoch}, step {step}, loss {loss.item():.4f}", args.epochs * len(loader), train_steps) if (train_steps % args.ckpt_every == 0 or train_steps % args.ckpt_latest_every == 0 # or train_steps == args.max_training_steps ) and train_steps > 0: diff --git a/hook_HYDiT_run.py b/hook_HYDiT_run.py index e5300e3..8667e6f 100644 --- a/hook_HYDiT_run.py +++ b/hook_HYDiT_run.py @@ -78,24 +78,40 @@ if __name__ == "__main__": margs = hydit.config.get_args() margs.model = train_config.get("model", "DiT-g/2") - margs.task_flag = train_config.get( - "task_flag", "lora_porcelain_ema_rank64") + margs.task_flag = train_config.get("task_flag") margs.resume_split = train_config.get("resume_split", True) + margs.ema_to_module = train_config.get("ema_to_module", True) + margs.deepspeed = False + margs.predict_type = train_config.get("predict_type", "v_prediction") + margs.training_parts = train_config.get("training_parts", "lora") + margs.batch_size = train_config.get("batch_size", 1) + margs.grad_accu_steps = train_config.get("grad_accu_steps", 1) + margs.global_seed = train_config.get("global_seed", 0) + margs.use_flash_attn = train_config.get("use_flash_attn", False) + margs.use_fp16 = train_config.get("use_fp16", True) + margs.qk_norm = train_config.get("qk_norm", True) + margs.ema_dtype = train_config.get("ema_dtype", "fp32") + margs.async_ema = False + + margs.ckpt_latest_every = 0x7fffffff + margs.multireso = train_config.get("multireso", True) + margs.epochs = train_config.get("epochs", 50) + margs.target_ratios = train_config.get( "target_ratios", ['1:1', '3:4', '4:3', '16:9', '9:16']) margs.rope_img = train_config.get("rope_img", "base1024") @@ -107,16 +123,17 @@ if __name__ == "__main__": margs.index_file = train_config.get("index_file", None) margs.lr = train_config.get("lr", 1e-5) - + margs.rank = train_config.get("rank", 8) margs.noise_offset = train_config.get("noise_offset", 0.0) margs.log_every = train_config.get("log_every", 99999999999999) - + margs.use_zero_stage = train_config.get("use_zero_stage", 2) margs.global_batch_size = train_config.get("global_batch_size", 1) + margs.deepspeed = True margs.results_dir = train_config.get("results_dir") diff --git a/hook_HYDiT_utils.py b/hook_HYDiT_utils.py index eadbe30..2afcbc2 100644 --- a/hook_HYDiT_utils.py +++ b/hook_HYDiT_utils.py @@ -85,7 +85,7 @@ def easy_sample_images( from hydit.modules.models import HUNYUAN_DIT_CONFIG import traceback - with torch.no_grad(): + with torch.cuda.amp.autocast(): workspace_dir = TRAIN_CONFIG.get("workspace_dir") sample_config_file = TRAIN_CONFIG.get("sample_config_file", None) @@ -132,20 +132,9 @@ def easy_sample_images( safety_checker=None, requires_safety_checker=False, embedder_t5=embedder_t5, - ).to("cuda") + ) + pipeline = pipeline.to("cuda") # attr _execution_device is not defined - if not hasattr(pipeline, "_execution_device"): - time.sleep(1) - pipeline = StableDiffusionPipeline(vae=vae, - text_encoder=text_encoder, - tokenizer=tokenizer, - unet=model.module, - scheduler=scheduler, - feature_extractor=None, - safety_checker=None, - requires_safety_checker=False, - embedder_t5=embedder_t5, - ).to("cuda") style = torch.as_tensor([0, 0] * batch_size, device="cuda") @@ -190,7 +179,7 @@ def easy_sample_images( print(f"Failed to sample images: {e} ") # 打印堆栈信息 traceback.print_exc() - continue + print(f"Failed to sample pipeline: {pipeline} ") # print("samples:",type(samples),) # input("Press Enter to continue...") @@ -312,7 +301,7 @@ class CustomizeEmbedsModel(nn.Module): return self def forward(self, *args, **kwargs): - if self.output_hidden_states or kwargs.get("output_hidden_states", False): + if kwargs.get("output_hidden_states", False): return { "hidden_states": self.x.to("cuda"), "input_ids": torch.zeros(1, 1), diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index dd34ae2..a8b8973 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -345,7 +345,11 @@ def check_required(): def MZ_HYDiTTrain_call(args={}): + check_required() + + advanced_config = args.get("advanced_config", {}) + args = check_model_auto_download(args) # raise Exception(args) resolution = args.get("resolution") @@ -505,6 +509,21 @@ def MZ_HYDiTTrain_call(args={}): datetime = time.strftime("%Y%m%d_%H%M%S", time.localtime()) output_name = f"{workspace_name}_{datetime}" + lora_ckpt = args.get("base_lora") + if lora_ckpt == "empty": + lora_ckpt = None + elif lora_ckpt == "latest": + lora_ckpt = None + loras = search_loras(workspace_dir) + if len(loras) > 0: + lora_ckpt = loras[0] + elif not os.path.exists(lora_ckpt): + raise Exception(f"未找到指定的lora文件: {lora_ckpt}") + + if lora_ckpt is not None and lora_ckpt.endswith(".safetensors"): + # 文件所在目录 + lora_ckpt = os.path.dirname(lora_ckpt) + print(f"使用指定的lora文件: {lora_ckpt}") train_config.update({ "workspace_name": workspace_name, "workspace_dir": workspace_dir, @@ -524,8 +543,23 @@ def MZ_HYDiTTrain_call(args={}): "t5_encoder_path": args.get("t5_encoder_path") if args.get("t5_encoder_path") != "none" else None, "results_dir": output_dir, "task_flag": output_name, + "lora_ckpt": lora_ckpt, }) + if "target_modules" in train_config: + del train_config["target_modules"] + for key in advanced_config: + if key.startswith("target_modules_"): + if advanced_config[key] == "enable": + if "target_modules" not in train_config: + train_config["target_modules"] = [] + train_config["target_modules"].append( + key.replace("target_modules_", "")) + elif advanced_config[key] == "enable" or advanced_config[key] == "disable": + train_config[key] = advanced_config[key] == "enable" + else: + train_config[key] = advanced_config[key] + sample_generate = args.get("sample_generate", "enable") sample_prompt = args.get("sample_prompt", "") sample_config_file = os.path.join(workspace_dir, "sample_config.json") @@ -645,7 +679,7 @@ def get_sample_images(train_config): return None -def get_model_from_path(model_path, lora_path, width, height): +def get_HunYuanDiT_model_from_path(model_path, lora_path, width, height): from hydit.modules.models import HunYuanDiT, HUNYUAN_DIT_CONFIG @@ -760,7 +794,7 @@ class CustomizeEmbedsModel(nn.Module): return self def forward(self, *args, **kwargs): - if self.output_hidden_states or kwargs.get("output_hidden_states", False): + if kwargs.get("output_hidden_states", False): return { "hidden_states": self.x.to("cuda"), "input_ids": torch.zeros(1, 1), @@ -821,7 +855,7 @@ def MZ_HYDiTSimpleT2I_call(args={}): if unet_data is None or unet_data.get("unet_path") != unet_path or unet_data.get("lora_path") != lora_path: clean_unet() _lora_path = lora_path if lora_path != "none" else None - unet = get_model_from_path( + unet = get_HunYuanDiT_model_from_path( args.get("unet_path"), _lora_path, width, height) Utils.cache_set(unet_cache_key, { "model": unet, @@ -993,3 +1027,19 @@ def MZ_HYDiTSimpleT2I_call(args={}): torch.cuda.empty_cache() return (Utils.list_tensor2tensor([tensor_image]),) + + +def search_loras(wdirs): + loras = [] + for wdir in wdirs: + for root, dirs, files in os.walk(wdir): + for file in files: + if file.lower().endswith(".safetensors"): + # 并且在同一目录下有相同名字的json文件 + json_file = os.path.splitext(file)[0] + ".json" + if os.path.exists(os.path.join(root, json_file)): + loras.append(os.path.join(root, file)) + + # 按时间倒序 + loras = sorted(loras, key=lambda x: os.path.getmtime(x), reverse=True) + return loras From cc62eb99957c9600156f0d1b4a641605828a671d Mon Sep 17 00:00:00 2001 From: wailovet Date: Sun, 30 Jun 2024 01:45:59 +0800 Subject: [PATCH 36/64] 'update' --- __init__.py | 6 +++--- mz_train_tools_core_HYDiT.py | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/__init__.py b/__init__.py index 19f0b59..73342b8 100644 --- a/__init__.py +++ b/__init__.py @@ -491,14 +491,14 @@ class MZ_HYDiTAdvConfig: def INPUT_TYPES(s): return { "required": { - "lr": ("FLOAT", {"default": 1e-5}), + "lr": ("STRING", {"default": "1e-5"}), "rope_real": (["enable", "disable"], {"default": "enable"}), # ['Wqkv', 'q_proj', 'kv_proj', 'out_proj'] "target_modules_Wqkv": (["enable", "disable"], {"default": "enable"}), "target_modules_q_proj": (["enable", "disable"], {"default": "enable"}), "target_modules_kv_proj": (["enable", "disable"], {"default": "enable"}), - "target_modules_out_proj": (["enable", "disable"], {"default": "enable"}), - "warmup_min_lr": ("FLOAT", {"default": 1e-6}), + "target_modules_out_proj": (["enable", "disable"], {"default": "enable"}), + "warmup_min_lr": ("STRING", {"default": "1e-6"}), # parser.add_argument("--warmup-num-steps", type=float, default=0) # parser.add_argument("--weight-decay", type=float, default=0, help="weight-decay in optimizer") "weight_decay": ("FLOAT", {"default": 0}), diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index a8b8973..ae8e91e 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -349,6 +349,12 @@ def MZ_HYDiTTrain_call(args={}): check_required() advanced_config = args.get("advanced_config", {}) + fix_float_keys = ["lr", "warmup_min_lr", + "warmup_max_lr", "weight_decay", "warmup_num_steps",] + for key in fix_float_keys: + if key in advanced_config: + if isinstance(advanced_config[key], str): + advanced_config[key] = float(advanced_config[key]) args = check_model_auto_download(args) # raise Exception(args) From 3de73b4e5cf6f2e293edb2b0fa071b5738a1407d Mon Sep 17 00:00:00 2001 From: wailovet Date: Sun, 30 Jun 2024 01:57:53 +0800 Subject: [PATCH 37/64] 'update' --- mz_train_tools_core_HYDiT.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index ae8e91e..abeb492 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -32,22 +32,22 @@ def MZ_HYDiTInitWorkspace_call(args={}): mz_dir = Utils.get_minus_zone_models_path() git_url = "https://github.com/Tencent/HunyuanDiT" source = args.get("source", "github") - kohya_ss_lora_dir = os.path.join(mz_dir, "train_tools", "HunyuanDiT") + hunyuan_lora_dir = os.path.join(mz_dir, "train_tools", "HunyuanDiT") if git_accelerate_urls.get(source, None) is not None: git_url = f"https://{git_accelerate_urls[source]}/Tencent/HunyuanDiT" try: - if not os.path.exists(kohya_ss_lora_dir) or not os.path.exists(os.path.join(kohya_ss_lora_dir, ".git")): + if not os.path.exists(hunyuan_lora_dir) or not os.path.exists(os.path.join(hunyuan_lora_dir, ".git")): subprocess.run( - ["git", "clone", "--depth", "1", git_url, kohya_ss_lora_dir], check=True) + ["git", "clone", "--depth", "1", git_url, hunyuan_lora_dir], check=True) # 切换远程分支 git remote set-branches origin 'main' branch = args.get("branch", "main") # 查看本地分支是否一致 short_result = subprocess.run( - ["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=kohya_ss_lora_dir, stdout=subprocess.PIPE, check=True) + ["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=hunyuan_lora_dir, stdout=subprocess.PIPE, check=True) result = subprocess.run( - ["git", "rev-parse", "HEAD"], cwd=kohya_ss_lora_dir, stdout=subprocess.PIPE, check=True) + ["git", "rev-parse", "HEAD"], cwd=hunyuan_lora_dir, stdout=subprocess.PIPE, check=True) short_current_branch = short_result.stdout.decode().strip() long_current_branch = result.stdout.decode().strip() @@ -57,16 +57,24 @@ def MZ_HYDiTInitWorkspace_call(args={}): if branch != result.stdout.decode() and branch != short_result.stdout.decode(): subprocess.run( - ["git", "remote", "set-branches", "origin", branch], cwd=kohya_ss_lora_dir, check=True) + ["git", "remote", "set-branches", "origin", branch], cwd=hunyuan_lora_dir, check=True) subprocess.run( - ["git", "fetch", "--depth", "1", "origin", branch], cwd=kohya_ss_lora_dir, check=True) + ["git", "fetch", "--depth", "1", "origin", branch], cwd=hunyuan_lora_dir, check=True) # 恢复所有文件 subprocess.run( - ["git", "checkout", "."], cwd=kohya_ss_lora_dir, check=True) + ["git", "checkout", "."], cwd=hunyuan_lora_dir, check=True) subprocess.run( - ["git", "checkout", branch], cwd=kohya_ss_lora_dir, check=True) + ["git", "checkout", branch], cwd=hunyuan_lora_dir, check=True) + + content = None + with open(os.path.join(hunyuan_lora_dir, "hydit/diffusion/pipeline.py"), "r", encoding="utf-8") as f: + pre_replace = "device = self._execution_device" + content = f.read() + content = content.replace(pre_replace, "device = 'cuda'") + with open(os.path.join(hunyuan_lora_dir, "hydit/diffusion/pipeline.py"), "w", encoding="utf-8") as f: + f.write(content) except Exception as e: raise Exception(f"克隆kohya-ss/sd-scripts或者切换分支时出现异常,详细信息请查看控制台...") @@ -354,7 +362,7 @@ def MZ_HYDiTTrain_call(args={}): for key in fix_float_keys: if key in advanced_config: if isinstance(advanced_config[key], str): - advanced_config[key] = float(advanced_config[key]) + advanced_config[key] = float(advanced_config[key]) args = check_model_auto_download(args) # raise Exception(args) From fa517f0222c220ccd28ec7b00ae9a9a97328a527 Mon Sep 17 00:00:00 2001 From: wailovet Date: Sun, 30 Jun 2024 02:05:44 +0800 Subject: [PATCH 38/64] 'update' --- __init__.py | 4 ++-- hook_HYDiT_run.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/__init__.py b/__init__.py index 73342b8..94898ff 100644 --- a/__init__.py +++ b/__init__.py @@ -497,7 +497,7 @@ class MZ_HYDiTAdvConfig: "target_modules_Wqkv": (["enable", "disable"], {"default": "enable"}), "target_modules_q_proj": (["enable", "disable"], {"default": "enable"}), "target_modules_kv_proj": (["enable", "disable"], {"default": "enable"}), - "target_modules_out_proj": (["enable", "disable"], {"default": "enable"}), + "target_modules_out_proj": (["enable", "disable"], {"default": "enable"}), "warmup_min_lr": ("STRING", {"default": "1e-6"}), # parser.add_argument("--warmup-num-steps", type=float, default=0) # parser.add_argument("--weight-decay", type=float, default=0, help="weight-decay in optimizer") @@ -576,7 +576,7 @@ class MZ_HYDiTTrain: "text_encoder_path": (["auto"] + folders, {"default": "auto"}), "tokenizer_path": (["auto"] + folders, {"default": "auto"}), "t5_encoder_path": (["none", "auto"] + folders, {"default": "none"}), - "resolution": ("INT", {"default": 1024}), + "resolution": ("INT", {"default": 1024, "step": 16}), "batch_size": ("INT", {"default": 1}), "epochs": ("INT", {"default": 50}), "ckpt_every": ("INT", {"default": 500}), diff --git a/hook_HYDiT_run.py b/hook_HYDiT_run.py index 8667e6f..1fe4222 100644 --- a/hook_HYDiT_run.py +++ b/hook_HYDiT_run.py @@ -114,6 +114,7 @@ if __name__ == "__main__": margs.target_ratios = train_config.get( "target_ratios", ['1:1', '3:4', '4:3', '16:9', '9:16']) + margs.rope_img = train_config.get("rope_img", "base1024") margs.image_size = train_config.get("image_size", 1024) @@ -133,7 +134,7 @@ if __name__ == "__main__": margs.use_zero_stage = train_config.get("use_zero_stage", 2) margs.global_batch_size = train_config.get("global_batch_size", 1) - + margs.deepspeed = True margs.results_dir = train_config.get("results_dir") From e6b5d0d49a163e7411d66fa6e45538d6d3da3c53 Mon Sep 17 00:00:00 2001 From: wailovet Date: Sun, 30 Jun 2024 02:13:56 +0800 Subject: [PATCH 39/64] 'update' --- __init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/__init__.py b/__init__.py index 94898ff..6ac7e2c 100644 --- a/__init__.py +++ b/__init__.py @@ -379,7 +379,7 @@ class MZ_LoadImagesFromDirectoryPath: for pil_image in pil_images: tensor_images.append(Utils.pil2tensor(pil_image)) - return (tensor_images,) + return (Utils.list_tensor2tensor(tensor_images),) NODE_CLASS_MAPPINGS["MZ_LoadImagesFromDirectoryPath"] = MZ_LoadImagesFromDirectoryPath From 81d44d641454a27f15eadf91b1f909bf563ce103 Mon Sep 17 00:00:00 2001 From: wailovet Date: Sun, 30 Jun 2024 20:22:01 +0800 Subject: [PATCH 40/64] 'update' --- mz_train_tools_core_HYDiT.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index abeb492..218fe66 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -851,7 +851,16 @@ def MZ_HYDiTSimpleT2I_call(args={}): if HYDiT_tool_dir not in sys.path: sys.path.append(HYDiT_tool_dir) - import hydit.inference + + + + + try: + import hydit + except ImportError: + sys.path.append(HYDiT_tool_dir) + + from hydit.inference import SAMPLER_FACTORY, HUNYUAN_DIT_CONFIG, get_fill_resize_and_crop, get_2d_rotary_pos_embed from hydit.modules.text_encoder import MT5Embedder from hydit.utils.tools import set_seeds From 6b67f47bc9f6663f56d1098553aab6604eadf434 Mon Sep 17 00:00:00 2001 From: wailovet Date: Sun, 30 Jun 2024 20:23:57 +0800 Subject: [PATCH 41/64] 'update' --- mz_train_tools_core_HYDiT.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index 218fe66..306e5b5 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -855,11 +855,6 @@ def MZ_HYDiTSimpleT2I_call(args={}): - try: - import hydit - except ImportError: - sys.path.append(HYDiT_tool_dir) - from hydit.inference import SAMPLER_FACTORY, HUNYUAN_DIT_CONFIG, get_fill_resize_and_crop, get_2d_rotary_pos_embed from hydit.modules.text_encoder import MT5Embedder From d3aa37a554491c1c78adf56397b24d50534fb92d Mon Sep 17 00:00:00 2001 From: wailovet Date: Sun, 30 Jun 2024 20:26:53 +0800 Subject: [PATCH 42/64] 'update' --- mz_train_tools_core_HYDiT.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index 306e5b5..9ca0e6b 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -849,6 +849,7 @@ def MZ_HYDiTSimpleT2I_call(args={}): HYDiT_tool_dir = os.path.join( Utils.get_minus_zone_models_path(), "train_tools", "HunyuanDiT") if HYDiT_tool_dir not in sys.path: + print(f"add {HYDiT_tool_dir} to sys.path") sys.path.append(HYDiT_tool_dir) From fa4e57af5832b1ec0484d0de3d37fe76ec5b697f Mon Sep 17 00:00:00 2001 From: wailovet Date: Sun, 30 Jun 2024 22:25:27 +0800 Subject: [PATCH 43/64] 'update' --- __init__.py | 2 +- mz_train_tools_core_HYDiT.py | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/__init__.py b/__init__.py index 6ac7e2c..77e78ef 100644 --- a/__init__.py +++ b/__init__.py @@ -647,7 +647,7 @@ class MZ_HYDiTSimpleT2I: "steps": ("INT", {"default": 20}), "cfg": ("FLOAT", {"default": 8.0, "min": 0.0, "max": 100.0, "step": 0.1, "round": 0.01}), "scheduler": ([ - "ddpm", "ddim", "dpmms", + "ddpm", "ddim", "dpmms", "uni_pc" ], {"default": "ddpm"}), "prompt": ("STRING", {"default:": "", "dynamicPrompts": True, "multiline": True}), "negative_prompt": ("STRING", {"default:": "", "dynamicPrompts": True, "multiline": True}), diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index 9ca0e6b..9a7ad80 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -852,11 +852,6 @@ def MZ_HYDiTSimpleT2I_call(args={}): print(f"add {HYDiT_tool_dir} to sys.path") sys.path.append(HYDiT_tool_dir) - - - - - from hydit.inference import SAMPLER_FACTORY, HUNYUAN_DIT_CONFIG, get_fill_resize_and_crop, get_2d_rotary_pos_embed from hydit.modules.text_encoder import MT5Embedder from hydit.utils.tools import set_seeds @@ -864,6 +859,20 @@ def MZ_HYDiTSimpleT2I_call(args={}): from diffusers import schedulers, AutoencoderKL from transformers import BertModel, BertTokenizer + SAMPLER_FACTORY["uni_pc"] = { + 'scheduler': 'UniPCMultistepScheduler', + 'name': 'UniPCMultistepScheduler', + 'kwargs': { + # 'beta_schedule': 'scaled_linear', + # 'beta_start': 0.00085, + # 'beta_end': 0.03, + # 'prediction_type': 'v_prediction', + # 'trained_betas': None, + # 'solver_order': 2, + # 'algorithm_type': 'dpmsolver++', + } + } + width = args.get("width") height = args.get("height") From 9fc49c303594ec62ad6c89d2092e3687a7acbf20 Mon Sep 17 00:00:00 2001 From: wailovet Date: Sun, 30 Jun 2024 22:28:07 +0800 Subject: [PATCH 44/64] 'update' --- mz_train_tools_core_HYDiT.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index 9a7ad80..a7625f0 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -863,13 +863,13 @@ def MZ_HYDiTSimpleT2I_call(args={}): 'scheduler': 'UniPCMultistepScheduler', 'name': 'UniPCMultistepScheduler', 'kwargs': { - # 'beta_schedule': 'scaled_linear', - # 'beta_start': 0.00085, - # 'beta_end': 0.03, - # 'prediction_type': 'v_prediction', - # 'trained_betas': None, - # 'solver_order': 2, - # 'algorithm_type': 'dpmsolver++', + 'beta_schedule': 'scaled_linear', + 'beta_start': 0.00085, + 'beta_end': 0.03, + 'prediction_type': 'v_prediction', + 'trained_betas': None, + 'solver_order': 2, + 'algorithm_type': 'dpmsolver++', } } From d6c91bf74afda96f46238c942fd80f9eeafd8fb1 Mon Sep 17 00:00:00 2001 From: wailovet Date: Sun, 30 Jun 2024 22:28:37 +0800 Subject: [PATCH 45/64] 'update' --- mz_train_tools_core_HYDiT.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index a7625f0..10c5df3 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -869,7 +869,7 @@ def MZ_HYDiTSimpleT2I_call(args={}): 'prediction_type': 'v_prediction', 'trained_betas': None, 'solver_order': 2, - 'algorithm_type': 'dpmsolver++', + # 'algorithm_type': 'dpmsolver++', } } From e3b2ef9b35f8ad1216ed4496e16ccdc492a2ecff Mon Sep 17 00:00:00 2001 From: wailovet Date: Sun, 30 Jun 2024 22:34:27 +0800 Subject: [PATCH 46/64] 'update' --- __init__.py | 2 +- mz_train_tools_core_HYDiT.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/__init__.py b/__init__.py index 77e78ef..7397c2a 100644 --- a/__init__.py +++ b/__init__.py @@ -647,7 +647,7 @@ class MZ_HYDiTSimpleT2I: "steps": ("INT", {"default": 20}), "cfg": ("FLOAT", {"default": 8.0, "min": 0.0, "max": 100.0, "step": 0.1, "round": 0.01}), "scheduler": ([ - "ddpm", "ddim", "dpmms", "uni_pc" + "ddpm", "ddim", "dpmms", "uni_pc", "DPMPP_2M_Karras" ], {"default": "ddpm"}), "prompt": ("STRING", {"default:": "", "dynamicPrompts": True, "multiline": True}), "negative_prompt": ("STRING", {"default:": "", "dynamicPrompts": True, "multiline": True}), diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index 10c5df3..2a3f4d4 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -872,6 +872,20 @@ def MZ_HYDiTSimpleT2I_call(args={}): # 'algorithm_type': 'dpmsolver++', } } + SAMPLER_FACTORY["DPMPP_2M_Karras"] = { + 'scheduler': 'DPMSolverMultistepScheduler', + 'name': 'DPMSolverMultistepScheduler', + 'kwargs': { + 'beta_schedule': 'scaled_linear', + 'beta_start': 0.00085, + 'beta_end': 0.03, + 'prediction_type': 'v_prediction', + 'trained_betas': None, + 'solver_order': 2, + 'algorithm_type': 'dpmsolver++', + "use_karras_sigmas": True, + } + } width = args.get("width") height = args.get("height") From 27827985e3fb0d3ba2690c0f9fa8958a7c848fa0 Mon Sep 17 00:00:00 2001 From: wailovet Date: Sun, 30 Jun 2024 23:13:08 +0800 Subject: [PATCH 47/64] 'update' --- mz_train_tools_core_HYDiT.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index 2a3f4d4..04dacf5 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -876,12 +876,8 @@ def MZ_HYDiTSimpleT2I_call(args={}): 'scheduler': 'DPMSolverMultistepScheduler', 'name': 'DPMSolverMultistepScheduler', 'kwargs': { - 'beta_schedule': 'scaled_linear', - 'beta_start': 0.00085, - 'beta_end': 0.03, - 'prediction_type': 'v_prediction', - 'trained_betas': None, - 'solver_order': 2, + 'beta_schedule': 'scaled_linear', + 'prediction_type': 'v_prediction', 'algorithm_type': 'dpmsolver++', "use_karras_sigmas": True, } From 9e15bb4cf120fcaada5cf99e97b3dcee59693bf6 Mon Sep 17 00:00:00 2001 From: wailovet Date: Sun, 30 Jun 2024 23:15:55 +0800 Subject: [PATCH 48/64] 'update' --- mz_train_tools_core_HYDiT.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index 04dacf5..d191667 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -876,9 +876,9 @@ def MZ_HYDiTSimpleT2I_call(args={}): 'scheduler': 'DPMSolverMultistepScheduler', 'name': 'DPMSolverMultistepScheduler', 'kwargs': { - 'beta_schedule': 'scaled_linear', - 'prediction_type': 'v_prediction', - 'algorithm_type': 'dpmsolver++', + 'beta_schedule': 'scaled_linear', + 'beta_start': 0.00085, + 'beta_end': 0.03, "use_karras_sigmas": True, } } From 2b1adb20a84bb98d518e2e26420b1a44f01c7006 Mon Sep 17 00:00:00 2001 From: wailovet Date: Sun, 30 Jun 2024 23:19:17 +0800 Subject: [PATCH 49/64] 'update' --- mz_train_tools_core_HYDiT.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index d191667..2a3f4d4 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -878,7 +878,11 @@ def MZ_HYDiTSimpleT2I_call(args={}): 'kwargs': { 'beta_schedule': 'scaled_linear', 'beta_start': 0.00085, - 'beta_end': 0.03, + 'beta_end': 0.03, + 'prediction_type': 'v_prediction', + 'trained_betas': None, + 'solver_order': 2, + 'algorithm_type': 'dpmsolver++', "use_karras_sigmas": True, } } From 19fe6d1435663ae4e195effcd7f21f002d92e843 Mon Sep 17 00:00:00 2001 From: wailovet Date: Mon, 1 Jul 2024 23:49:58 +0800 Subject: [PATCH 50/64] 'update' --- __init__.py | 9 +- mz_train_tools_core.py | 11 +- mz_train_tools_core_HYDiT.py | 14 +- mz_train_tools_utils.py | 22 ++- tools/aesthetic.py | 210 +++++++++++++++++++++++++++ tools/aesthetic2_5.py | 260 ++++++++++++++++++++++++++++++++++ tools/face_resize_copy_dir.py | 47 +++--- 7 files changed, 540 insertions(+), 33 deletions(-) create mode 100644 tools/aesthetic.py create mode 100644 tools/aesthetic2_5.py diff --git a/__init__.py b/__init__.py index 7397c2a..2d9847c 100644 --- a/__init__.py +++ b/__init__.py @@ -102,7 +102,8 @@ class MZ_KohyaSSUseConfig: @classmethod def INPUT_TYPES(s): - train_config_templates = os.listdir(s.train_config_template_dir) + train_config_templates = Utils.listdir(s.train_config_template_dir) + # 去掉json后缀 train_config_templates = [os.path.splitext(x)[0] for x in train_config_templates] @@ -240,6 +241,9 @@ class MZ_KohyaSSLoraTrain: # 使用walk查询所有的workspace中的所有lora模型,lora存放在每个workspace的output目录下 workspaces_loras = [] for root, dirs, files in os.walk(workspaces_dir): + + # 排除隐藏文件夹 + dirs[:] = [d for d in dirs if not d.startswith(".")] if root.endswith("output"): for file in files: if file.endswith(".safetensors"): @@ -366,7 +370,8 @@ class MZ_LoadImagesFromDirectoryPath: image_dir = kwargs["directory"] if not os.path.exists(image_dir): return (images,) - images = os.listdir(image_dir) + images = Utils.listdir(image_dir) + images = [x for x in images if x.lower().endswith( ".png") or x.lower().endswith(".jpg")] images = [os.path.join(image_dir, x) for x in images] diff --git a/mz_train_tools_core.py b/mz_train_tools_core.py index 65460a2..db1a3e0 100644 --- a/mz_train_tools_core.py +++ b/mz_train_tools_core.py @@ -119,7 +119,8 @@ def MZ_ImageSelecter_call(args={}): force_clear_only_images = args.get("force_clear_only_images") == "enable" if force_clear: if force_clear_only_images: - for file in os.listdir(train_images_dir): + images_files = Utils.listdir(train_images_dir) + for file in images_files: if file.lower().endswith(".png") or file.lower().endswith(".jpg"): os.remove(os.path.join(train_images_dir, file)) else: @@ -357,7 +358,7 @@ def get_sample_images(train_config): pil_images = [] pre_render_texts_x = [] if os.path.exists(sample_images_dir): - image_files = os.listdir(sample_images_dir) + image_files = Utils.listdir(sample_images_dir) image_files = list( filter(lambda x: x.lower().endswith(".png"), image_files)) # 筛选 output_name 前缀 @@ -483,7 +484,7 @@ def MZ_KohyaSSTrain_call(args={}): elif base_lora == "latest": workspace_lora_dir = os.path.join(workspace_dir, "output") if os.path.exists(workspace_lora_dir): - workspace_lora_files = os.listdir(workspace_lora_dir) + workspace_lora_files = Utils.listdir(workspace_lora_dir) workspace_lora_files = list( filter(lambda x: x.endswith(".safetensors"), workspace_lora_files)) workspace_lora_files = list( @@ -520,7 +521,7 @@ def MZ_KohyaSSTrain_call(args={}): elif base_controlnet == "latest": workspace_controlnet_dir = os.path.join(workspace_dir, "output") if os.path.exists(workspace_controlnet_dir): - workspace_controlnet_files = os.listdir(workspace_controlnet_dir) + workspace_controlnet_files = Utils.listdir(workspace_controlnet_dir) workspace_controlnet_files = list( filter(lambda x: x.endswith(".safetensors"), workspace_controlnet_files)) workspace_controlnet_files = list( @@ -564,7 +565,7 @@ def MZ_KohyaSSTrain_call(args={}): workspace_dir, "conditioning_images") conditioning_images_onec = "" if os.path.exists(conditioning_images_dir): - conditioning_images_onec = os.listdir(conditioning_images_dir)[0] + conditioning_images_onec = Utils.listdir(conditioning_images_dir)[0] other_config["controlnet_image"] = os.path.join( conditioning_images_dir, conditioning_images_onec) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index 2a3f4d4..e0e0d50 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -129,7 +129,8 @@ def MZ_HYDiTDatasetConfig_call(args={}): force_clear_only_images = args.get("force_clear_only_images") == "enable" if force_clear: if force_clear_only_images: - for file in os.listdir(train_images_dir): + images_files = Utils.listdir(train_images_dir) + for file in images_files: if file.lower().endswith(".png") or file.lower().endswith(".jpg"): os.remove(os.path.join(train_images_dir, file)) else: @@ -376,7 +377,7 @@ def MZ_HYDiTTrain_call(args={}): if workspace_images_dir is None: workspace_images_dir = os.path.join(workspace_dir, "train_images") - full_filenames = os.listdir(workspace_images_dir) + full_filenames = Utils.listdir(workspace_images_dir) # 创建image_text.csv文件 csv_filename = os.path.join(workspace_dir, "image_text.csv") @@ -446,7 +447,7 @@ def MZ_HYDiTTrain_call(args={}): except Exception as e: raise Exception(f"生成arrow文件时出现异常,详细信息请查看控制台...") - arrows_files = os.listdir(arrows_dir) + arrows_files = Utils.listdir(arrows_dir) dataset_yaml_path = os.path.join(workspace_dir, "dataset.yaml") @@ -671,9 +672,11 @@ def get_sample_images(train_config): pil_images = [] pre_render_texts_x = [] if os.path.exists(sample_images_dir): - image_files = os.listdir(sample_images_dir) + image_files = Utils.listdir(sample_images_dir) + image_files = list( filter(lambda x: x.lower().endswith(".png"), image_files)) + # 筛选 output_name 前缀 image_files = list( filter(lambda x: x.startswith(output_name), image_files)) @@ -1075,6 +1078,9 @@ def search_loras(wdirs): loras = [] for wdir in wdirs: for root, dirs, files in os.walk(wdir): + # 排除隐藏文件夹 + dirs[:] = [d for d in dirs if not d.startswith('.')] + for file in files: if file.lower().endswith(".safetensors"): # 并且在同一目录下有相同名字的json文件 diff --git a/mz_train_tools_utils.py b/mz_train_tools_utils.py index b21b147..04c099d 100644 --- a/mz_train_tools_utils.py +++ b/mz_train_tools_utils.py @@ -67,7 +67,7 @@ class Utils: def listdir_png(path): try: - files = os.listdir(path) + files = Utils.listdir(path) new_files = [] for file in files: if file.lower().endswith(".png"): @@ -78,6 +78,15 @@ class Utils: except Exception as e: return [] + def listdir(path): + try: + files = os.listdir(path) + # 排除.开头的文件 + files = [file for file in files if not file.startswith(".")] + return files + except Exception as e: + return [] + def tensor2pil(image): return Image.fromarray(np.clip(255.0 * image.cpu().numpy().squeeze(), 0, 255).astype(np.uint8)) @@ -676,6 +685,8 @@ class Utils: if os.path.exists(find_fullpath): for root, dirs, files in os.walk(find_fullpath): + # 排除隐藏文件夹 + dirs[:] = [d for d in dirs if not d.startswith('.')] for file in files: if target_sha256 == Utils.file_sha256(os.path.join(root, file)): return os.path.join(root, file) @@ -919,14 +930,19 @@ class Utils: def get_models_by_folder(dir_path): models = [] for root, dirs, files in os.walk(dir_path): + # 排除隐藏文件夹 + dirs[:] = [d for d in dirs if not d.startswith('.')] for file in files: if file.endswith(".pth") or file.endswith(".pt") or file.endswith(".pkl") or file.endswith(".onnx") or file.endswith(".safetensors"): models.append(os.path.join(root, file)) return models - + def get_folders_by_folder(dir_path): folders = [] for root, dirs, files in os.walk(dir_path): + + # 排除隐藏文件夹 + dirs[:] = [d for d in dirs if not d.startswith('.')] for dir in dirs: folders.append(os.path.join(root, dir)) - return folders \ No newline at end of file + return folders diff --git a/tools/aesthetic.py b/tools/aesthetic.py new file mode 100644 index 0000000..ee3891d --- /dev/null +++ b/tools/aesthetic.py @@ -0,0 +1,210 @@ +import json +import os +import shutil +from PIL import Image +import matplotlib.pyplot as plt + +from warnings import filterwarnings + +import numpy as np +import torch +import pytorch_lightning as pl +import torch.nn as nn +from PIL import Image + + +from transformers import CLIPProcessor, CLIPModel +import torch.nn.functional as F + + +class MLP(pl.LightningModule): + def __init__(self, input_size, xcol='emb', ycol='avg_rating'): + super().__init__() + self.input_size = input_size + self.xcol = xcol + self.ycol = ycol + self.layers = nn.Sequential( + nn.Linear(self.input_size, 1024), + # nn.ReLU(), + nn.Dropout(0.2), + nn.Linear(1024, 128), + # nn.ReLU(), + nn.Dropout(0.2), + nn.Linear(128, 64), + # nn.ReLU(), + nn.Dropout(0.1), + + nn.Linear(64, 16), + # nn.ReLU(), + + nn.Linear(16, 1) + ) + + def forward(self, x): + return self.layers(x) + + def training_step(self, batch, batch_idx): + x = batch[self.xcol] + y = batch[self.ycol].reshape(-1, 1) + x_hat = self.layers(x) + loss = F.mse_loss(x_hat, y) + return loss + + def validation_step(self, batch, batch_idx): + x = batch[self.xcol] + y = batch[self.ycol].reshape(-1, 1) + x_hat = self.layers(x) + loss = F.mse_loss(x_hat, y) + return loss + + def configure_optimizers(self): + optimizer = torch.optim.Adam(self.parameters(), lr=1e-3) + return optimizer + + +def normalized(a, axis=-1, order=2): + import numpy as np + + l2 = np.atleast_1d(np.linalg.norm(a, order, axis)) + l2[l2 == 0] = 1 + return a / np.expand_dims(l2, axis) + + +class Aestheic: + def __init__(self, clip_path, aes_model): + + model = MLP(768) # CLIP embedding dim is 768 for CLIP ViT L 14 + + # load the model you trained previously or the model available in this repo + s = torch.load(aes_model) + + model.load_state_dict(s) + + model.to("cuda") + model.eval() + + self.model = model + + self.device = "cuda" + + model2 = CLIPModel.from_pretrained( + clip_path, local_files_only=True).to("cuda") + preprocess = CLIPProcessor.from_pretrained( + clip_path, local_files_only=True) + + self.model2 = model2 + self.preprocess = preprocess + + def prediction(self, img_paths): + if not isinstance(img_paths, list): + img_paths = [img_paths] + + pil_images = [] + for img_path in img_paths: + pil_image = Image.open(img_path) + pil_images.append(pil_image) + + inputs = self.preprocess( + images=pil_images, return_tensors="pt", + ).to("cuda") + with torch.no_grad(): + image_features = self.model2.get_image_features(**inputs) + print(image_features.shape) + + im_emb_arr = normalized(image_features.cpu().detach().numpy()) + + prediction = self.model(torch.from_numpy( + im_emb_arr).to(self.device).type(torch.cuda.FloatTensor)) + + prediction_score = prediction.cpu().detach().numpy() + prediction_score = prediction_score.flatten() + + images_score = { + img_path: float(score) for img_path, score in zip(img_paths, prediction_score) + } + + # 按value排序 + images_score = dict( + sorted(images_score.items(), key=lambda x: x[1], reverse=True)) + + return images_score + + +import argparse +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("-i", "--input_dir", type=str, default="") + parser.add_argument("-o", "--output_dir", type=str, default="") + parser.add_argument("-c", "--clip_path", type=str, + default="D:\\数据集\\clip-vit-large-patch14") + parser.add_argument("-m", "--aes_model", type=str, + default="D:\\数据集\\sac+logos+ava1-l14-linearMSE.pth") + parser.add_argument("-s", "--score", type=float, default=5.9) + + parser.add_argument("-b", "--batch_size", type=int, default=1) + + args = parser.parse_args() + limitscore = args.score + + aes = Aestheic(args.clip_path, args.aes_model) + + input_dir = args.input_dir + output_dir = args.output_dir + batch_size = args.batch_size + total = 0 + for root, dirs, files in os.walk(input_dir): + # 排除隐藏文件夹 + dirs[:] = [d for d in dirs if not d.startswith('.')] + for file in files: + if not file.endswith(".png") and not file.endswith(".jpg"): + continue + total += 1 + + import tqdm + print(f"total: {total}") + + images = [] + + with tqdm.tqdm(total=total) as pbar: + for root, dirs, files in os.walk(input_dir): + # 排除隐藏文件夹 + dirs[:] = [d for d in dirs if not d.startswith('.')] + for file in files: + if not file.endswith(".png") and not file.endswith(".jpg"): + continue + file_path = os.path.join(root, file) + pbar.update(1) + + pbar.set_description( + f"Processing {os.path.basename(file_path)}") + + if len(images) < batch_size: + images.append(file_path) + else: + image_scroe_map = aes.prediction(images) + images = [] + # print(scroe) + # print(json.dumps(scroe, indent=4, ensure_ascii=False)) + # exit(0) + + for img_path, score in image_scroe_map.items(): + print(f"{img_path}: {score}") + + if score > limitscore: + if not os.path.exists(output_dir): + os.makedirs(output_dir) + shutil.copy(img_path, output_dir) + + if len(images) > 0: + aes.prediction(images) + images = [] + + for img_path, score in image_scroe_map.items(): + print(f"{img_path}: {score}") + + if score > limitscore: + if not os.path.exists(output_dir): + os.makedirs(output_dir) + shutil.copy(img_path, output_dir) + + diff --git a/tools/aesthetic2_5.py b/tools/aesthetic2_5.py new file mode 100644 index 0000000..d51f377 --- /dev/null +++ b/tools/aesthetic2_5.py @@ -0,0 +1,260 @@ +import json +import os +from collections import OrderedDict +from os import PathLike +import shutil +from typing import Final + +import torch +import torch.nn as nn +from transformers import ( + SiglipImageProcessor, + SiglipVisionConfig, + SiglipVisionModel, + logging, +) +from transformers.image_processing_utils import BatchFeature +from transformers.modeling_outputs import ImageClassifierOutputWithNoAttention + +logging.set_verbosity_error() + +URL: Final[str] = ( + "https://github.com/discus0434/aesthetic-predictor-v2-5/raw/main/models/aesthetic_predictor_v2_5.pth" +) + + +class AestheticPredictorV2_5Head(nn.Module): + def __init__(self, config: SiglipVisionConfig) -> None: + super().__init__() + self.scoring_head = nn.Sequential( + nn.Linear(config.hidden_size, 1024), + nn.Dropout(0.5), + nn.Linear(1024, 128), + nn.Dropout(0.5), + nn.Linear(128, 64), + nn.Dropout(0.5), + nn.Linear(64, 16), + nn.Dropout(0.2), + nn.Linear(16, 1), + ) + + def forward(self, image_embeds: torch.Tensor) -> torch.Tensor: + return self.scoring_head(image_embeds) + + +class AestheticPredictorV2_5Model(SiglipVisionModel): + PATCH_SIZE = 14 + + def __init__(self, config: SiglipVisionConfig, *args, **kwargs) -> None: + super().__init__(config, *args, **kwargs) + self.layers = AestheticPredictorV2_5Head(config) + self.post_init() + + def forward( + self, + pixel_values: torch.FloatTensor | None = None, + labels: torch.Tensor | None = None, + return_dict: bool | None = None, + ) -> tuple | ImageClassifierOutputWithNoAttention: + return_dict = ( + return_dict if return_dict is not None else self.config.use_return_dict + ) + + outputs = super().forward( + pixel_values=pixel_values, + return_dict=return_dict, + ) + image_embeds = outputs.pooler_output + image_embeds_norm = image_embeds / \ + image_embeds.norm(dim=-1, keepdim=True) + prediction = self.layers(image_embeds_norm) + + loss = None + if labels is not None: + loss_fct = nn.MSELoss() + loss = loss_fct() + + if not return_dict: + return (loss, prediction, image_embeds) + + return ImageClassifierOutputWithNoAttention( + loss=loss, + logits=prediction, + hidden_states=image_embeds, + ) + + +class AestheticPredictorV2_5Processor(SiglipImageProcessor): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + def __call__(self, *args, **kwargs) -> BatchFeature: + return super().__call__(*args, **kwargs) + + @classmethod + def from_pretrained( + self, + pretrained_model_name_or_path: str + | PathLike = "google/siglip-so400m-patch14-384", + *args, + **kwargs, + ) -> "AestheticPredictorV2_5Processor": + return super().from_pretrained(pretrained_model_name_or_path, *args, **kwargs) + + +def convert_v2_5_from_siglip( + predictor_name_or_path: str | PathLike | None = None, + encoder_model_name: str = "google/siglip-so400m-patch14-384", + *args, + **kwargs, +) -> tuple[AestheticPredictorV2_5Model, AestheticPredictorV2_5Processor]: + model = AestheticPredictorV2_5Model.from_pretrained( + encoder_model_name, *args, **kwargs + ) + + processor = AestheticPredictorV2_5Processor.from_pretrained( + encoder_model_name, *args, **kwargs + ) + + if predictor_name_or_path is None or not os.path.exists(predictor_name_or_path): + state_dict = torch.hub.load_state_dict_from_url( + URL, map_location="cpu") + else: + state_dict = torch.load(predictor_name_or_path, map_location="cpu") + + assert isinstance(state_dict, OrderedDict) + + model.layers.load_state_dict(state_dict) + model.eval() + + return model, processor + + +from PIL import Image + + +class AestheticPredictor: + def __init__(self, clip_path, aesthetic_predictor_path): + # load model and preprocessor + self.model, self.preprocessor = convert_v2_5_from_siglip( + low_cpu_mem_usage=True, + trust_remote_code=True, + encoder_model_name=clip_path, + predictor_name_or_path=aesthetic_predictor_path, + ) + if torch.cuda.is_available(): + self.model = self.model.to(torch.bfloat16).cuda() + + def inference(self, img_paths) -> float: + if not isinstance(img_paths, list): + img_paths = [img_paths] + + pil_images = [] + for img_path in img_paths: + pil_image = Image.open(img_path) + pil_images.append(pil_image) + + # preprocess image + pixel_values = self.preprocessor( + images=pil_images, return_tensors="pt" + ).pixel_values + + if torch.cuda.is_available(): + pixel_values = pixel_values.to(torch.bfloat16).cuda() + + # predict aesthetic score + with torch.inference_mode(): + score = self.model( + pixel_values).logits.squeeze().float().cpu().numpy() + prediction_score = score.flatten() + + images_score = { + img_path: float(score) for img_path, score in zip(img_paths, prediction_score) + } + + # 按value排序 + images_score = dict( + sorted(images_score.items(), key=lambda x: x[1], reverse=True)) + + return images_score + + + +import argparse +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("-i", "--input_dir", type=str, default="") + parser.add_argument("-o", "--output_dir", type=str, default="") + parser.add_argument("-c", "--clip_path", type=str, + default="D:\\数据集\\siglip-so400m-patch14-384") + parser.add_argument("-m", "--aes_model", type=str, + default="D:\\数据集\\aesthetic_predictor_v2_5.pth") + parser.add_argument("-s", "--score", type=float, default=5.9) + + parser.add_argument("-b", "--batch_size", type=int, default=1) + + args = parser.parse_args() + limitscore = args.score + + aes = AestheticPredictor(args.clip_path, args.aes_model) + + input_dir = args.input_dir + output_dir = args.output_dir + batch_size = args.batch_size + total = 0 + for root, dirs, files in os.walk(input_dir): + + # 排除隐藏文件夹 + dirs[:] = [d for d in dirs if not d.startswith('.')] + for file in files: + if not file.endswith(".png") and not file.endswith(".jpg"): + continue + total += 1 + + import tqdm + print(f"total: {total}") + + images = [] + + with tqdm.tqdm(total=total) as pbar: + for root, dirs, files in os.walk(input_dir): + # 排除隐藏文件夹 + dirs[:] = [d for d in dirs if not d.startswith('.')] + for file in files: + if not file.endswith(".png") and not file.endswith(".jpg"): + continue + file_path = os.path.join(root, file) + pbar.update(1) + + pbar.set_description( + f"Processing {os.path.basename(file_path)}") + + if os.path.exists(os.path.join(output_dir, os.path.basename(file_path))): + continue + + if len(images) < batch_size: + images.append(file_path) + else: + image_scroe_map = aes.inference(images) + images = [] + for img_path, score in image_scroe_map.items(): + print(f"{img_path}: {score}") + + if score > limitscore: + if not os.path.exists(output_dir): + os.makedirs(output_dir) + shutil.copy(img_path, output_dir) + + if len(images) > 0: + image_scroe_map = aes.inference(images) + images = [] + + for img_path, score in image_scroe_map.items(): + print(f"{img_path}: {score}") + + if score > limitscore: + if not os.path.exists(output_dir): + os.makedirs(output_dir) + shutil.copy(img_path, output_dir) + + diff --git a/tools/face_resize_copy_dir.py b/tools/face_resize_copy_dir.py index ac043ff..03eacf5 100644 --- a/tools/face_resize_copy_dir.py +++ b/tools/face_resize_copy_dir.py @@ -64,20 +64,18 @@ class Utils: newWidth = dst_h * src_w // src_h newHeight = dst_h - newHeight = newHeight // 8 * 8 - newWidth = newWidth // 8 * 8 + newHeight = int(newHeight // 8 * 8) + newWidth = int(newWidth // 8 * 8) im = im.resize((newWidth, newHeight), Image.Resampling.LANCZOS) - left = (newWidth - dst_w) // 2 top = (newHeight - dst_h) // 2 right = left + dst_w bottom = top + dst_h - - focus_point_x, focus_point_y = focus_point + focus_point_x, focus_point_y = focus_point offset_w = dst_w // 8 offset_h = dst_h // 8 if focus_point_x < left + dst_w // 8: @@ -101,32 +99,41 @@ def main(): parser.add_argument("-i", "--input_dir", type=str, default="") parser.add_argument("-o", "--output_dir", type=str, default="") parser.add_argument("-m", "--face_model_path", type=str, default="") + parser.add_argument("-r", "--resize", type=int, default=1024) parser.add_argument("-s", "--score", type=float, default=0.8) + parser.add_argument("-f", "--format", type=str, default="jpg") + args = parser.parse_args() limitscore = args.score sess = ort.InferenceSession(args.face_model_path) sess.set_providers(['CUDAExecutionProvider']) input_dir = args.input_dir output_dir = args.output_dir - + resize = args.resize + img_format = args.format total = 0 for root, dirs, files in os.walk(input_dir): + # 排除隐藏文件夹 + dirs[:] = [d for d in dirs if not d.startswith('.')] for file in files: if not file.endswith(".png") and not file.endswith(".jpg"): continue total += 1 - - import tqdm + import tqdm + print(f"total: {total}") + + os.makedirs(output_dir, exist_ok=True) with tqdm.tqdm(total=total) as pbar: + + # 排除隐藏文件夹 + dirs[:] = [d for d in dirs if not d.startswith('.')] for root, dirs, files in os.walk(input_dir): for file in files: if not file.endswith(".png") and not file.endswith(".jpg"): continue - file_path = os.path.join(root, file) pbar.update(1) - pbar.set_description(f"Processing {file_path}") with open(file_path, "rb") as f: md5_str = Utils.Md5(f.read()) @@ -134,10 +141,11 @@ def main(): continue image = Image.open(file_path) - image = Utils.resize_min(image, 1280, 1280) + image = Utils.resize_min(image, resize, resize) cv2_img = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR) + print(f"start check {file_path}") scores, bboxes, keypoints, aligned_imgs, landmarks, affine_matrices = sess.run( None, {"input": cv2_img}) if len(bboxes) != 1: @@ -167,11 +175,11 @@ def main(): continue # print(f"{file_path} bboxes: \n{bboxes} {scores} {real_angle}") - center_point = (bboxes[0] + bboxes[2]) / \ 2, (bboxes[1] + bboxes[3]) / 2 - image = Utils.resize_crop(image, 960, 1280, center_point) + image = Utils.resize_crop( + image, resize * 3 / 4, resize, center_point) cv2_img = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR) scores, bboxes, keypoints, aligned_imgs, landmarks, affine_matrices = sess.run( None, {"input": cv2_img}) @@ -179,12 +187,13 @@ def main(): continue # cv2.imshow("image", cv2.cvtColor( # np.array(image), cv2.COLOR_RGB2BGR)) - # cv2.waitKey(0) - - output_path = os.path.join(output_dir, md5_str + ".png") - # print(f"Save {file_path} to {output_path}") - - image.save(output_path) + # cv2.waitKey(0) + if img_format == "png": + output_path = os.path.join(output_dir, md5_str + ".png") + image.save(output_path) + else: + output_path = os.path.join(output_dir, md5_str + ".jpg") + image.save(output_path, "jpeg", quality=95) if __name__ == "__main__": From ec6144fa819653b9e0added32cc8d103c06ef0c0 Mon Sep 17 00:00:00 2001 From: wailovet Date: Tue, 2 Jul 2024 00:45:41 +0800 Subject: [PATCH 51/64] 'update' --- mz_train_tools_core_HYDiT.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index e0e0d50..b244919 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -394,7 +394,7 @@ def MZ_HYDiTTrain_call(args={}): writer.writerow(["image_path", "text_zh"]) for filename in full_filenames: - if filename.lower().endswith(".png"): + if filename.lower().endswith(".png") or filename.lower().endswith(".jpg"): image_path = os.path.join(workspace_images_dir, filename) pil_image = Image.open(image_path) From 6babea03a6134e60e20b68c11a6dd15d07d5faa8 Mon Sep 17 00:00:00 2001 From: wailovet Date: Tue, 2 Jul 2024 00:54:11 +0800 Subject: [PATCH 52/64] 'update' --- mz_train_tools_core_HYDiT.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index b244919..b6aa015 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -394,6 +394,7 @@ def MZ_HYDiTTrain_call(args={}): writer.writerow(["image_path", "text_zh"]) for filename in full_filenames: + print(f"处理文件: {filename}") if filename.lower().endswith(".png") or filename.lower().endswith(".jpg"): image_path = os.path.join(workspace_images_dir, filename) From 553a0ba678dc2881162a15c1e7efdab594c07999 Mon Sep 17 00:00:00 2001 From: wailovet Date: Tue, 2 Jul 2024 18:13:41 +0800 Subject: [PATCH 53/64] 'update' --- __init__.py | 8 ++++++++ mz_train_tools_core_HYDiT.py | 34 ++++++++++++++++++++++------------ 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/__init__.py b/__init__.py index 2d9847c..e7863f8 100644 --- a/__init__.py +++ b/__init__.py @@ -557,6 +557,7 @@ class MZ_HYDiTTrain: os.makedirs(hunyuan_models_path, exist_ok=True) models = Utils.get_models_by_folder(hunyuan_models_path) + folders = Utils.get_folders_by_folder(hunyuan_models_path) vae_models = Utils.get_models_by_folder( @@ -642,6 +643,13 @@ class MZ_HYDiTSimpleT2I: return { "required": { + "branch": ("STRING", {"default": "5657364143e44ac90f72aeb47b81bd505a95665d"}), + "source": ([ + "github", + "githubfast", + "521github", + "kkgithub", + ], {"default": "github"}), "unet_path": (["auto"] + models + unet_models, {"default": "auto"}), "vae_ema_path": (["auto"] + folders + vae_models, {"default": "auto"}), "text_encoder_path": (["auto"] + folders, {"default": "auto"}), diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index b6aa015..5c7fd23 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -28,7 +28,7 @@ git_accelerate_urls = { # 初始化工具仓库和工作区 -def MZ_HYDiTInitWorkspace_call(args={}): +def MZ_HYDiTInitClone_call(args={}): mz_dir = Utils.get_minus_zone_models_path() git_url = "https://github.com/Tencent/HunyuanDiT" source = args.get("source", "github") @@ -54,8 +54,8 @@ def MZ_HYDiTInitWorkspace_call(args={}): print( f"当前分支(current branch): {long_current_branch}({short_current_branch})") print(f"目标分支(target branch): {branch}") - - if branch != result.stdout.decode() and branch != short_result.stdout.decode(): + time.sleep(1) + if branch != long_current_branch and branch != short_current_branch: subprocess.run( ["git", "remote", "set-branches", "origin", branch], cwd=hunyuan_lora_dir, check=True) subprocess.run( @@ -72,13 +72,17 @@ def MZ_HYDiTInitWorkspace_call(args={}): with open(os.path.join(hunyuan_lora_dir, "hydit/diffusion/pipeline.py"), "r", encoding="utf-8") as f: pre_replace = "device = self._execution_device" content = f.read() - content = content.replace(pre_replace, "device = 'cuda'") + content = content.replace(pre_replace, "device = torch.device('cuda')") with open(os.path.join(hunyuan_lora_dir, "hydit/diffusion/pipeline.py"), "w", encoding="utf-8") as f: f.write(content) except Exception as e: raise Exception(f"克隆kohya-ss/sd-scripts或者切换分支时出现异常,详细信息请查看控制台...") + +def MZ_HYDiTInitWorkspace_call(args={}): + MZ_HYDiTInitClone_call(args) + workspace_name = args.get("train_name", None) workspace_name = workspace_name.strip() @@ -377,7 +381,7 @@ def MZ_HYDiTTrain_call(args={}): if workspace_images_dir is None: workspace_images_dir = os.path.join(workspace_dir, "train_images") - full_filenames = Utils.listdir(workspace_images_dir) + full_filenames = Utils.listdir(workspace_images_dir) # 创建image_text.csv文件 csv_filename = os.path.join(workspace_dir, "image_text.csv") @@ -448,7 +452,7 @@ def MZ_HYDiTTrain_call(args={}): except Exception as e: raise Exception(f"生成arrow文件时出现异常,详细信息请查看控制台...") - arrows_files = Utils.listdir(arrows_dir) + arrows_files = Utils.listdir(arrows_dir) dataset_yaml_path = os.path.join(workspace_dir, "dataset.yaml") @@ -673,11 +677,11 @@ def get_sample_images(train_config): pil_images = [] pre_render_texts_x = [] if os.path.exists(sample_images_dir): - image_files = Utils.listdir(sample_images_dir) - + image_files = Utils.listdir(sample_images_dir) + image_files = list( filter(lambda x: x.lower().endswith(".png"), image_files)) - + # 筛选 output_name 前缀 image_files = list( filter(lambda x: x.startswith(output_name), image_files)) @@ -699,7 +703,12 @@ def get_sample_images(train_config): def get_HunYuanDiT_model_from_path(model_path, lora_path, width, height): - from hydit.modules.models import HunYuanDiT, HUNYUAN_DIT_CONFIG + from hydit.modules.models import HUNYUAN_DIT_CONFIG + import importlib + import hydit.modules.models + importlib.reload(hydit.modules.models) + HunYuanDiT = hydit.modules.models.HunYuanDiT + from types import SimpleNamespace @@ -732,7 +741,7 @@ def get_HunYuanDiT_model_from_path(model_path, lora_path, width, height): ).half().to("cuda") state_dict = torch.load( model_path, map_location=lambda storage, loc: storage) - model.load_state_dict(state_dict) + model.load_state_dict(state_dict, strict=False) if lora_path is not None: from peft import PeftConfig @@ -848,6 +857,7 @@ class CustomizeEmbeds(): def MZ_HYDiTSimpleT2I_call(args={}): check_required() + MZ_HYDiTInitClone_call(args) args = check_model_auto_download(args) HYDiT_tool_dir = os.path.join( @@ -1079,7 +1089,7 @@ def search_loras(wdirs): loras = [] for wdir in wdirs: for root, dirs, files in os.walk(wdir): - # 排除隐藏文件夹 + # 排除隐藏文件夹 dirs[:] = [d for d in dirs if not d.startswith('.')] for file in files: From 661b8f61063c8ded4dbf68a098d6cd6ba8d283c1 Mon Sep 17 00:00:00 2001 From: wailovet Date: Wed, 3 Jul 2024 00:12:57 +0800 Subject: [PATCH 54/64] 'update' --- hook_HYDiT_main_train_deepspeed.py | 27 ++++++++++++++++++++------- hook_HYDiT_utils.py | 18 ++++++++++++------ mz_train_tools_core_HYDiT.py | 26 +++++++++++++------------- 3 files changed, 45 insertions(+), 26 deletions(-) diff --git a/hook_HYDiT_main_train_deepspeed.py b/hook_HYDiT_main_train_deepspeed.py index 73d50eb..c25fa9d 100644 --- a/hook_HYDiT_main_train_deepspeed.py +++ b/hook_HYDiT_main_train_deepspeed.py @@ -8,6 +8,7 @@ from functools import partial from glob import glob from pathlib import Path import numpy as np +import safetensors.torch import deepspeed import torch @@ -382,12 +383,19 @@ def Core(args): if True: model, ema, start_epoch, start_epoch_step, train_steps = model_resume( args, model, ema, logger) - + if args.training_parts == "lora": lora_ckpt = args.lora_ckpt if lora_ckpt is not None: - from peft import PeftConfig - loraconfig = PeftConfig.from_pretrained(lora_ckpt) + lastest_checkpoint = lora_ckpt + from peft.peft_model import PeftModel + print(f"Loading lora model from {lastest_checkpoint}") + if args.use_fp16: + model.module = PeftModel.from_pretrained( + model.module, lastest_checkpoint, is_trainable=True) + else: + model = PeftModel.from_pretrained( + model, lastest_checkpoint, is_trainable=True) else: loraconfig = LoraConfig( r=args.rank, @@ -395,10 +403,10 @@ def Core(args): target_modules=args.target_modules ) - if args.use_fp16: - model.module = get_peft_model(model.module, loraconfig) - else: - model = get_peft_model(model, loraconfig) + if args.use_fp16: + model.module = get_peft_model(model.module, loraconfig) + else: + model = get_peft_model(model, loraconfig) logger.info(f" Training parts: {args.training_parts}") @@ -409,6 +417,11 @@ def Core(args): # Training # =========================================================================== + # print model structure + # for name, param in model.named_parameters(): + # print(name, param.size()) + # raise Exception("stop") + model.train() if args.use_ema: ema.eval() diff --git a/hook_HYDiT_utils.py b/hook_HYDiT_utils.py index 2afcbc2..8d40308 100644 --- a/hook_HYDiT_utils.py +++ b/hook_HYDiT_utils.py @@ -291,8 +291,8 @@ from torch import nn class CustomizeEmbedsModel(nn.Module): dtype = torch.float16 - x = torch.zeros(1, 1, 256, 2048) - + # x = torch.zeros(1, 1, 256, 2048) + x = None def __init__(self, *args, **kwargs): super().__init__() @@ -300,7 +300,13 @@ class CustomizeEmbedsModel(nn.Module): self.dtype = torch.float16 return self - def forward(self, *args, **kwargs): + def forward(self, *args, **kwargs): + input_ids = kwargs.get("input_ids", None) + if self.x is None: + batch_size = input_ids.shape[0] + self.x = torch.zeros(1, batch_size, 256, 2048, dtype=self.dtype) + + if kwargs.get("output_hidden_states", False): return { "hidden_states": self.x.to("cuda"), @@ -315,15 +321,15 @@ class CustomizeTokenizer(dict): input_ids = torch.zeros(1, 256) attention_mask = torch.zeros(1, 256) - def __init__(self): + def __init__(self, *args, **kwargs): self['added_tokens_encoder'] = self.added_tokens_encoder self['input_ids'] = self.input_ids self['attention_mask'] = self.attention_mask - def tokenize(self, text): + def tokenize(self, text): return text - def __call__(self, *args, **kwargs): + def __call__(self, *args, **kwargs): return self diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index 5c7fd23..18e4390 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -72,7 +72,8 @@ def MZ_HYDiTInitClone_call(args={}): with open(os.path.join(hunyuan_lora_dir, "hydit/diffusion/pipeline.py"), "r", encoding="utf-8") as f: pre_replace = "device = self._execution_device" content = f.read() - content = content.replace(pre_replace, "device = torch.device('cuda')") + content = content.replace( + pre_replace, "device = torch.device('cuda')") with open(os.path.join(hunyuan_lora_dir, "hydit/diffusion/pipeline.py"), "w", encoding="utf-8") as f: f.write(content) @@ -442,6 +443,9 @@ def MZ_HYDiTTrain_call(args={}): # python ./hydit/data_loader/csv2arrow.py ./dataset/porcelain/csvfile/image_text.csv ./dataset/porcelain/arrows arrows_dir = os.path.join(workspace_dir, "arrows") + if os.path.exists(arrows_dir): + shutil.rmtree(arrows_dir) + os.makedirs(arrows_dir, exist_ok=True) csv2arrow_exec = os.path.join( HYDiT_tool_dir, "hydit", "data_loader", "csv2arrow.py") @@ -534,15 +538,12 @@ def MZ_HYDiTTrain_call(args={}): lora_ckpt = None elif lora_ckpt == "latest": lora_ckpt = None - loras = search_loras(workspace_dir) + loras = search_loras([workspace_dir]) if len(loras) > 0: lora_ckpt = loras[0] elif not os.path.exists(lora_ckpt): raise Exception(f"未找到指定的lora文件: {lora_ckpt}") - if lora_ckpt is not None and lora_ckpt.endswith(".safetensors"): - # 文件所在目录 - lora_ckpt = os.path.dirname(lora_ckpt) print(f"使用指定的lora文件: {lora_ckpt}") train_config.update({ "workspace_name": workspace_name, @@ -564,6 +565,7 @@ def MZ_HYDiTTrain_call(args={}): "results_dir": output_dir, "task_flag": output_name, "lora_ckpt": lora_ckpt, + "rank": int(args.get("rank")), }) if "target_modules" in train_config: @@ -705,10 +707,9 @@ def get_HunYuanDiT_model_from_path(model_path, lora_path, width, height): from hydit.modules.models import HUNYUAN_DIT_CONFIG import importlib - import hydit.modules.models + import hydit.modules.models importlib.reload(hydit.modules.models) HunYuanDiT = hydit.modules.models.HunYuanDiT - from types import SimpleNamespace @@ -1091,13 +1092,12 @@ def search_loras(wdirs): for root, dirs, files in os.walk(wdir): # 排除隐藏文件夹 dirs[:] = [d for d in dirs if not d.startswith('.')] - + # 存在adapter_config.json文件并且存在adapter_model.safetensors文件 for file in files: - if file.lower().endswith(".safetensors"): - # 并且在同一目录下有相同名字的json文件 - json_file = os.path.splitext(file)[0] + ".json" - if os.path.exists(os.path.join(root, json_file)): - loras.append(os.path.join(root, file)) + if file == "adapter_config.json": + lora_path = os.path.join(root, "adapter_model.safetensors") + if os.path.exists(lora_path): + loras.append(root) # 按时间倒序 loras = sorted(loras, key=lambda x: os.path.getmtime(x), reverse=True) From ac255212638a665f82cef609ccd4fe21c2ce3ec2 Mon Sep 17 00:00:00 2001 From: wailovet Date: Wed, 3 Jul 2024 00:17:34 +0800 Subject: [PATCH 55/64] 'update' --- __init__.py | 2 +- hook_HYDiT_utils.py | 41 ++++++++++++++++++++++++++++++------ mz_train_tools_core_HYDiT.py | 2 +- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/__init__.py b/__init__.py index e7863f8..5986001 100644 --- a/__init__.py +++ b/__init__.py @@ -660,7 +660,7 @@ class MZ_HYDiTSimpleT2I: "steps": ("INT", {"default": 20}), "cfg": ("FLOAT", {"default": 8.0, "min": 0.0, "max": 100.0, "step": 0.1, "round": 0.01}), "scheduler": ([ - "ddpm", "ddim", "dpmms", "uni_pc", "DPMPP_2M_Karras" + "ddpm", "ddim", "dpmms", "uni_pc", "dpmpp_2m_karras" ], {"default": "ddpm"}), "prompt": ("STRING", {"default:": "", "dynamicPrompts": True, "multiline": True}), "negative_prompt": ("STRING", {"default:": "", "dynamicPrompts": True, "multiline": True}), diff --git a/hook_HYDiT_utils.py b/hook_HYDiT_utils.py index 8d40308..84a70c5 100644 --- a/hook_HYDiT_utils.py +++ b/hook_HYDiT_utils.py @@ -100,9 +100,38 @@ def easy_sample_images( sample_images_dir = os.path.join(workspace_dir, "sample_images") os.makedirs(sample_images_dir, exist_ok=True) + sampler_factory = SAMPLER_FACTORY.copy() + + sampler_factory["uni_pc"] = { + 'scheduler': 'UniPCMultistepScheduler', + 'name': 'UniPCMultistepScheduler', + 'kwargs': { + 'beta_schedule': 'scaled_linear', + 'beta_start': 0.00085, + 'beta_end': 0.03, + 'prediction_type': 'v_prediction', + 'trained_betas': None, + 'solver_order': 2, + } + } + sampler_factory["dpmpp_2m_karras"] = { + 'scheduler': 'DPMSolverMultistepScheduler', + 'name': 'DPMSolverMultistepScheduler', + 'kwargs': { + 'beta_schedule': 'scaled_linear', + 'beta_start': 0.00085, + 'beta_end': 0.03, + 'prediction_type': 'v_prediction', + 'trained_betas': None, + 'solver_order': 2, + 'algorithm_type': 'dpmsolver++', + "use_karras_sigmas": True, + } + } + # Load sampler from factory - kwargs = SAMPLER_FACTORY[sampler]['kwargs'] - scheduler = SAMPLER_FACTORY[sampler]['scheduler'] + kwargs = sampler_factory[sampler]['kwargs'] + scheduler = sampler_factory[sampler]['scheduler'] # Build scheduler according to the sampler. scheduler_class = getattr(schedulers, scheduler) @@ -293,6 +322,7 @@ class CustomizeEmbedsModel(nn.Module): dtype = torch.float16 # x = torch.zeros(1, 1, 256, 2048) x = None + def __init__(self, *args, **kwargs): super().__init__() @@ -300,13 +330,12 @@ class CustomizeEmbedsModel(nn.Module): self.dtype = torch.float16 return self - def forward(self, *args, **kwargs): + def forward(self, *args, **kwargs): input_ids = kwargs.get("input_ids", None) if self.x is None: batch_size = input_ids.shape[0] self.x = torch.zeros(1, batch_size, 256, 2048, dtype=self.dtype) - if kwargs.get("output_hidden_states", False): return { "hidden_states": self.x.to("cuda"), @@ -326,10 +355,10 @@ class CustomizeTokenizer(dict): self['input_ids'] = self.input_ids self['attention_mask'] = self.attention_mask - def tokenize(self, text): + def tokenize(self, text): return text - def __call__(self, *args, **kwargs): + def __call__(self, *args, **kwargs): return self diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index 18e4390..d1c4623 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -887,7 +887,7 @@ def MZ_HYDiTSimpleT2I_call(args={}): # 'algorithm_type': 'dpmsolver++', } } - SAMPLER_FACTORY["DPMPP_2M_Karras"] = { + SAMPLER_FACTORY["dpmpp_2m_karras"] = { 'scheduler': 'DPMSolverMultistepScheduler', 'name': 'DPMSolverMultistepScheduler', 'kwargs': { From 9e13becee82eff68a7e41396ac7713118a5714e5 Mon Sep 17 00:00:00 2001 From: wailovet Date: Wed, 3 Jul 2024 01:31:46 +0800 Subject: [PATCH 56/64] 'update' --- hook_HYDiT_utils.py | 4 ++-- mz_train_tools_core_HYDiT.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hook_HYDiT_utils.py b/hook_HYDiT_utils.py index 84a70c5..a37f9d4 100644 --- a/hook_HYDiT_utils.py +++ b/hook_HYDiT_utils.py @@ -73,9 +73,9 @@ def easy_sample_images( prompt="A photo of a girl with a hat on a sunny day", negative_prompt="", batch_size=1, - guidance_scale=5.0, + guidance_scale=2.0, infer_steps=20, - sampler='ddpm', + sampler='dpmpp_2m_karras', train_steps=0, ): from hydit.diffusion.pipeline import StableDiffusionPipeline diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index d1c4623..3680387 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -169,10 +169,10 @@ def MZ_HYDiTDatasetConfig_call(args={}): HYDiT_MODEL = { "HunyuanDiT/t2i/model/pytorch_model_ema.pt": { - "url": "https://www.modelscope.cn/api/v1/models/modelscope/HunyuanDiT/repo?Revision=master&FilePath=t2i%2Fmodel%2Fpytorch_model_ema.pt", + "url": "https://www.modelscope.cn/models/wailovet/hy1.1/resolve/master/t2i/model/pytorch_model_module.pt", }, "HunyuanDiT/t2i/model/pytorch_model_module.pt": { - "url": "https://www.modelscope.cn/api/v1/models/modelscope/HunyuanDiT/repo?Revision=master&FilePath=t2i%2Fmodel%2Fpytorch_model_module.pt", + "url": "https://www.modelscope.cn/models/wailovet/hy1.1/resolve/master/t2i/model/pytorch_model_module.pt", }, "HunyuanDiT/t2i/sdxl-vae-fp16-fix/diffusion_pytorch_model.safetensors": { "url": "https://www.modelscope.cn/api/v1/models/modelscope/HunyuanDiT/repo?Revision=master&FilePath=t2i%2Fsdxl-vae-fp16-fix%2Fdiffusion_pytorch_model.safetensors", From 2f9518f3cca0b5910f29255eaadce2828f65f236 Mon Sep 17 00:00:00 2001 From: wailovet Date: Wed, 3 Jul 2024 02:49:02 +0800 Subject: [PATCH 57/64] 'update' --- mz_train_tools_core_HYDiT.py | 2 +- tools/face_resize_copy_dir2.py | 245 +++++++++++++++++++++++++++++++++ 2 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 tools/face_resize_copy_dir2.py diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index 3680387..d8cbf7f 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -169,7 +169,7 @@ def MZ_HYDiTDatasetConfig_call(args={}): HYDiT_MODEL = { "HunyuanDiT/t2i/model/pytorch_model_ema.pt": { - "url": "https://www.modelscope.cn/models/wailovet/hy1.1/resolve/master/t2i/model/pytorch_model_module.pt", + "url": "https://www.modelscope.cn/models/wailovet/hy1.1/resolve/master/t2i/model/pytorch_model_ema.pt", }, "HunyuanDiT/t2i/model/pytorch_model_module.pt": { "url": "https://www.modelscope.cn/models/wailovet/hy1.1/resolve/master/t2i/model/pytorch_model_module.pt", diff --git a/tools/face_resize_copy_dir2.py b/tools/face_resize_copy_dir2.py new file mode 100644 index 0000000..7a78823 --- /dev/null +++ b/tools/face_resize_copy_dir2.py @@ -0,0 +1,245 @@ +import argparse +import hashlib +import io +import os +from PIL import Image +import numpy as np +import onnxruntime as ort +import cv2 + +import piexif + + +class Utils: + def Md5(bytes): + return hashlib.md5(bytes).hexdigest() + + def resize_by(image, percent): + # 判断类型是否为PIL + if not isinstance(image, Image.Image): + image = Image.fromarray(image) + + width, height = image.size + new_width = int(width * percent) + new_height = int(height * percent) + return image.resize((new_width, new_height), Image.LANCZOS) + + def resize_max(im, dst_w, dst_h): + src_w, src_h = im.size + + if src_h > src_w: + newWidth = dst_w + newHeight = dst_w * src_h // src_w + else: + newWidth = dst_h * src_w // src_h + newHeight = dst_h + + newHeight = newHeight // 8 * 8 + newWidth = newWidth // 8 * 8 + + return im.resize((newWidth, newHeight), Image.Resampling.LANCZOS) + + def resize_min(im, dst_w, dst_h): + src_w, src_h = im.size + + if src_h < src_w: + newWidth = dst_w + newHeight = dst_w * src_h // src_w + else: + newWidth = dst_h * src_w // src_h + newHeight = dst_h + + newHeight = newHeight // 8 * 8 + newWidth = newWidth // 8 * 8 + + return im.resize((newWidth, newHeight), Image.Resampling.LANCZOS) + + # 根据人脸进行剪裁 + def resize_crop(im, dst_w, dst_h, focus_point): + + src_w, src_h = im.size + + if src_h > src_w: + newWidth = dst_w + newHeight = dst_w * src_h // src_w + else: + newWidth = dst_h * src_w // src_h + newHeight = dst_h + + newHeight = int(newHeight // 8 * 8) + newWidth = int(newWidth // 8 * 8) + + im = im.resize((newWidth, newHeight), Image.Resampling.LANCZOS) + + left = (newWidth - dst_w) // 2 + top = (newHeight - dst_h) // 2 + + right = left + dst_w + bottom = top + dst_h + + focus_point_x, focus_point_y = focus_point + offset_w = dst_w // 8 + offset_h = dst_h // 8 + if focus_point_x < left + dst_w // 8: + left = 0 + right = dst_w + elif focus_point_x > right - dst_w // 8: + right = newWidth + left = newWidth - dst_w + if focus_point_y < top + dst_h // 8: + top = 0 + bottom = dst_h + elif focus_point_y > bottom - dst_h // 8: + bottom = newHeight + top = newHeight - dst_h + + return im.crop((left, top, right, bottom)) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("-i", "--input_dir", type=str, default="") + parser.add_argument("-o", "--output_dir", type=str, default="") + parser.add_argument("-m", "--face_model_path", type=str, default="") + parser.add_argument("-r", "--resize", type=int, default=1024) + parser.add_argument("-p", "--face_proportion", type=float, default=0.5) + parser.add_argument("-f", "--format", type=str, default="jpg") + + args = parser.parse_args() + sess = ort.InferenceSession(args.face_model_path) + sess.set_providers(['CUDAExecutionProvider']) + input_dir = args.input_dir + output_dir = args.output_dir + resize = args.resize + img_format = args.format + total = 0 + for root, dirs, files in os.walk(input_dir): + # 排除隐藏文件夹 + dirs[:] = [d for d in dirs if not d.startswith('.')] + for file in files: + if not file.endswith(".png") and not file.endswith(".jpg"): + continue + total += 1 + + import tqdm + print(f"total: {total}") + + os.makedirs(output_dir, exist_ok=True) + with tqdm.tqdm(total=total) as pbar: + + # 排除隐藏文件夹 + dirs[:] = [d for d in dirs if not d.startswith('.')] + for root, dirs, files in os.walk(input_dir): + for file in files: + if not file.endswith(".png") and not file.endswith(".jpg"): + continue + file_path = os.path.join(root, file) + pbar.update(1) + + with open(file_path, "rb") as f: + md5_str = Utils.Md5(f.read()) + if os.path.exists(os.path.join(output_dir, md5_str + ".png")): + continue + + image = Image.open(file_path) + max_image = image.copy() + image = Utils.resize_min(image, 1920, 1920) + zoom = image.size[0] / max_image.size[0] + + cv2_img = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR) + + print(f"start check {file_path}") + scores, bboxes, keypoints, aligned_imgs, landmarks, affine_matrices = sess.run( + None, {"input": cv2_img}) + if len(bboxes) != 1: + continue + + bboxes = bboxes[0] + + affine_matrices = affine_matrices[0] + angle = np.degrees(np.arctan2( + affine_matrices[1][0], affine_matrices[0][0])) + + real_angle = 0 + if angle < -45 and angle > -135: + # 向左旋转90度 + real_angle = 90 + elif angle > 45 and angle < 135: + # 向右旋转90度 + real_angle = -90 + elif angle < -135 or angle > 135: + # 向左旋转180度 + real_angle = 180 + + if real_angle != 0: + continue + + # print(f"{file_path} bboxes: \n{bboxes} {scores} {real_angle}") + + bboxes = bboxes / zoom + + image = max_image + + # 预期人脸高度占图片高度的一定比例 + pr = args.face_proportion + face_width = bboxes[2] - bboxes[0] + pre_image_width = face_width / pr + + face_center_x = (bboxes[2] + bboxes[0]) / 2 + crop_x1 = face_center_x - pre_image_width / 2 + crop_x2 = face_center_x + pre_image_width / 2 + + # 判断是否超出图片边界 + if crop_x1 < 0 or crop_x2 > image.width: + print( + f"crop_x1: {crop_x1}, crop_x2: {crop_x2}, image.width: {image.width}") + continue + + face_center_y = (bboxes[3] + bboxes[1]) / 2 + # 总高度和宽度的比例是4:3 + pre_image_height = pre_image_width * 4 / 3 + + # 中点位于高度的0.382比例 + crop_y1 = face_center_y - pre_image_height * 0.382 + crop_y2 = crop_y1 + pre_image_height + + # 判断是否超出图片边界 + if crop_y1 < 0 or crop_y2 > image.height: + print( + f"crop_y1: {crop_y1}, crop_y2: {crop_y2}, image.height: {image.height}") + continue + + # 判断是否比resize小 + if pre_image_width < resize or pre_image_height < resize: + print( + f"pre_image_width: {pre_image_width}, pre_image_height: {pre_image_height}") + continue + + # 进行裁剪 + image = image.crop((crop_x1, crop_y1, crop_x2, crop_y2)) + + # 进行缩放并居中剪裁 + image = image.resize(( + resize * 3 // 4, resize + ), Image.LANCZOS) + + # cv2_img = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR) + # cv2.imshow("image", cv2_img) + # cv2.waitKey(0) + + if img_format == "png": + output_path = os.path.join(output_dir, md5_str + ".png") + image.save(output_path) + else: + output_path = os.path.join(output_dir, md5_str + ".jpg") + image.save(output_path, "jpeg", quality=95) + # 插入IPTC + # piexif.insert(output_path, piexif.ImageIFD, { + # "CameraOwnerName": "CameraOwnerName", + # }) + + print(f"save to {output_path}") + + +if __name__ == "__main__": + main() From 73f6a2b3f0586b1f8ed191a50ee06558c1cc9280 Mon Sep 17 00:00:00 2001 From: wailovet Date: Wed, 3 Jul 2024 02:54:57 +0800 Subject: [PATCH 58/64] 'update' --- mz_train_tools_core_HYDiT.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index d8cbf7f..b5ae8f3 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -119,6 +119,20 @@ def MZ_HYDiTDatasetConfig_call(args={}): if images is None: print(f"训练数据位于:{train_images_dir},未检测到传入图片默认已有训练数据,将直接跳过数据准备步骤...") + same_caption_generate = args.get("same_caption_generate") == "enable" + if same_caption_generate: + saved_images_path = Utils.listdir(train_images_dir) + same_caption = args.get("same_caption").strip() + if same_caption != "": + # 循环已经保存的图片 + for i, filename in enumerate(saved_images_path): + if filename.lower().endswith(".png") or filename.lower().endswith(".jpg"): + base_filename = os.path.splitext(filename)[0] + caption_filename = base_filename + ".caption" + with open(os.path.join(train_images_dir, caption_filename), "w", encoding="utf-8") as f: + f.write(same_caption) + + return ( train_images_dir, ) From e71258f19f228b8825d7e42ad6e96a321a0e255c Mon Sep 17 00:00:00 2001 From: wailovet Date: Wed, 3 Jul 2024 02:59:44 +0800 Subject: [PATCH 59/64] 'update' --- hook_HYDiT_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hook_HYDiT_utils.py b/hook_HYDiT_utils.py index a37f9d4..a2d9d25 100644 --- a/hook_HYDiT_utils.py +++ b/hook_HYDiT_utils.py @@ -333,7 +333,10 @@ class CustomizeEmbedsModel(nn.Module): def forward(self, *args, **kwargs): input_ids = kwargs.get("input_ids", None) if self.x is None: - batch_size = input_ids.shape[0] + if input_ids is None: + batch_size = 1 + else: + batch_size = input_ids.shape[0] self.x = torch.zeros(1, batch_size, 256, 2048, dtype=self.dtype) if kwargs.get("output_hidden_states", False): From 28227347ad52b5606f0d90e987e67dd9b44535e7 Mon Sep 17 00:00:00 2001 From: wailovet Date: Wed, 3 Jul 2024 08:48:17 +0800 Subject: [PATCH 60/64] 'update' --- hook_HYDiT_utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/hook_HYDiT_utils.py b/hook_HYDiT_utils.py index a2d9d25..6b9d865 100644 --- a/hook_HYDiT_utils.py +++ b/hook_HYDiT_utils.py @@ -289,8 +289,11 @@ def LOG(log): if master_port == 0: raise Exception("master_port is 0") # 发送http - resp = requests.request("post", f"http://127.0.0.1:{master_port}/log", data=json.dumps(log), headers={ - "Content-Type": "application/json"}) + try: + resp = requests.request("post", f"http://127.0.0.1:{master_port}/log", data=json.dumps(log), headers={ + "Content-Type": "application/json"}) + except Exception as e: + return if resp.status_code != 200: raise Exception(f"LOG failed: {resp.text}") From a73102b1af094087472912da6fad1a7b01220a69 Mon Sep 17 00:00:00 2001 From: wailovet Date: Thu, 4 Jul 2024 00:43:26 +0800 Subject: [PATCH 61/64] 'update' --- __init__.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/__init__.py b/__init__.py index 5986001..502d71d 100644 --- a/__init__.py +++ b/__init__.py @@ -102,7 +102,7 @@ class MZ_KohyaSSUseConfig: @classmethod def INPUT_TYPES(s): - train_config_templates = Utils.listdir(s.train_config_template_dir) + train_config_templates = Utils.listdir(s.train_config_template_dir) # 去掉json后缀 train_config_templates = [os.path.splitext(x)[0] @@ -241,8 +241,8 @@ class MZ_KohyaSSLoraTrain: # 使用walk查询所有的workspace中的所有lora模型,lora存放在每个workspace的output目录下 workspaces_loras = [] for root, dirs, files in os.walk(workspaces_dir): - - # 排除隐藏文件夹 + + # 排除隐藏文件夹 dirs[:] = [d for d in dirs if not d.startswith(".")] if root.endswith("output"): for file in files: @@ -370,7 +370,7 @@ class MZ_LoadImagesFromDirectoryPath: image_dir = kwargs["directory"] if not os.path.exists(image_dir): return (images,) - images = Utils.listdir(image_dir) + images = Utils.listdir(image_dir) images = [x for x in images if x.lower().endswith( ".png") or x.lower().endswith(".jpg")] @@ -635,11 +635,9 @@ class MZ_HYDiTSimpleT2I: unet_models = Utils.get_models_by_folder( os.path.join(Utils.get_comfyui_models_path(), "unet")) - comfyui_full_loras = [] - comfyui_loras = folder_paths.get_filename_list("loras") - for lora in comfyui_loras: - lora_path = folder_paths.get_full_path("loras", lora) - comfyui_full_loras.append(lora_path) + comfyui_full_loras = mz_train_tools_core_HYDiT.search_loras([ + os.path.join(Utils.get_comfyui_models_path(), "loras"), + ]) return { "required": { From e77f7f135ca9a7120ea68bff66d7dc1a6a174b88 Mon Sep 17 00:00:00 2001 From: wailovet Date: Thu, 4 Jul 2024 00:48:44 +0800 Subject: [PATCH 62/64] 'update' --- mz_train_tools_core_HYDiT.py | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index b5ae8f3..a710f0f 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -759,31 +759,7 @@ def get_HunYuanDiT_model_from_path(model_path, lora_path, width, height): model.load_state_dict(state_dict, strict=False) if lora_path is not None: - from peft import PeftConfig - - # 查询后缀 - if lora_path.lower().endswith(".safetensors"): - from safetensors.torch import load_file - adapter_state_dict = load_file(lora_path) - else: - adapter_state_dict = torch.load(lora_path) - - peft_config = PeftConfig() - - # 相同目录下同名的json文件 - lora_json_path = os.path.splitext(lora_path)[0] + ".json" - if not os.path.exists(lora_json_path): - raise Exception(f"未找到对应的json文件: {lora_json_path}") - with open(lora_json_path, "r", encoding="utf-8") as f: - loaded_attributes = json.load(f) - - for key, value in loaded_attributes.items(): - setattr(peft_config, key, value) - - model.load_adapter( - adapter_state_dict=adapter_state_dict, - peft_config=peft_config, - ) + model.load_adapter(lora_path) model.merge_and_unload() return model From b051af41851c57764956386aae423e1df9e9f945 Mon Sep 17 00:00:00 2001 From: wailovet Date: Thu, 4 Jul 2024 00:55:04 +0800 Subject: [PATCH 63/64] 'update' --- mz_train_tools_core_HYDiT.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/mz_train_tools_core_HYDiT.py b/mz_train_tools_core_HYDiT.py index a710f0f..90359a8 100644 --- a/mz_train_tools_core_HYDiT.py +++ b/mz_train_tools_core_HYDiT.py @@ -759,7 +759,37 @@ def get_HunYuanDiT_model_from_path(model_path, lora_path, width, height): model.load_state_dict(state_dict, strict=False) if lora_path is not None: - model.load_adapter(lora_path) + + lora_path_files = Utils.listdir(lora_path) + safetensors_file = None + config_json_file = None + for file in lora_path_files: + if file.lower().endswith(".safetensors"): + safetensors_file = os.path.join(lora_path, file) + if file.lower().endswith(".json"): + config_json_file = os.path.join(lora_path, file) + + if safetensors_file is None or config_json_file is None: + raise Exception("未找到对应的lora文件") + + from peft import PeftConfig + + # 查询后缀 + from safetensors.torch import load_file + adapter_state_dict = load_file(safetensors_file) + + peft_config = PeftConfig() + + with open(config_json_file, "r", encoding="utf-8") as f: + loaded_attributes = json.load(f) + + for key, value in loaded_attributes.items(): + setattr(peft_config, key, value) + + model.load_adapter( + adapter_state_dict=adapter_state_dict, + peft_config=peft_config, + ) model.merge_and_unload() return model From 29dde1643233b3c07f9384da6be27911e2bf9c58 Mon Sep 17 00:00:00 2001 From: wailovet Date: Thu, 4 Jul 2024 22:00:23 +0800 Subject: [PATCH 64/64] 'update' --- hook_HYDiT_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hook_HYDiT_utils.py b/hook_HYDiT_utils.py index 6b9d865..d5ac04b 100644 --- a/hook_HYDiT_utils.py +++ b/hook_HYDiT_utils.py @@ -77,6 +77,7 @@ def easy_sample_images( infer_steps=20, sampler='dpmpp_2m_karras', train_steps=0, + seed=0, ): from hydit.diffusion.pipeline import StableDiffusionPipeline from diffusers import schedulers @@ -186,7 +187,8 @@ def easy_sample_images( freqs_cis_img = calc_rope(height, width) try: - + generator = torch.Generator(device="cuda") + generator.manual_seed(seed) samples = pipeline( height=height, width=width, @@ -201,6 +203,7 @@ def easy_sample_images( learn_sigma=args.learn_sigma, freqs_cis_img=freqs_cis_img, image_meta_size=image_meta_size, + generator=generator, )[0] pass