From e90605d0b56602abfc9d701022f6fe1928360462 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Sepp=C3=A4nen?= <40791699+kijai@users.noreply.github.com> Date: Wed, 9 Oct 2024 19:01:32 +0300 Subject: [PATCH] initial commit --- .gitignore | 145 ++++++++++++++++++++++++++++++++ README.md | 3 + __init__.py | 3 + configs/lotus_unet_config.json | 73 +++++++++++++++++ empty_text_embed.pt | Bin 0 -> 8966 bytes nodes.py | 146 +++++++++++++++++++++++++++++++++ requirements.txt | 1 + 7 files changed, 371 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 __init__.py create mode 100644 configs/lotus_unet_config.json create mode 100755 empty_text_embed.pt create mode 100644 nodes.py create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bb46937 --- /dev/null +++ b/.gitignore @@ -0,0 +1,145 @@ +.idea/ + +training/ +lightning_logs/ +image_log/ + +*.pth +*.ckpt +*.safetensors + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + + +*.safetensors +*.ckpt + +checkpoints \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..589c298 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# ComfyUI nodes to use Lotus depth/normal prediction + +Original repo: https://github.com/EnVision-Research/Lotus \ No newline at end of file diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..2e96bd6 --- /dev/null +++ b/__init__.py @@ -0,0 +1,3 @@ +from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS + +__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] \ No newline at end of file diff --git a/configs/lotus_unet_config.json b/configs/lotus_unet_config.json new file mode 100644 index 0000000..ef720ef --- /dev/null +++ b/configs/lotus_unet_config.json @@ -0,0 +1,73 @@ +{ + "_class_name": "UNet2DConditionModel", + "_diffusers_version": "0.28.0.dev0", + "_name_or_path": "../Lotus-weights/lotus-depth-d-v1-1/unet", + "act_fn": "silu", + "addition_embed_type": null, + "addition_embed_type_num_heads": 64, + "addition_time_embed_dim": null, + "attention_head_dim": [ + 5, + 10, + 20, + 20 + ], + "attention_type": "default", + "block_out_channels": [ + 320, + 640, + 1280, + 1280 + ], + "center_input_sample": false, + "class_embed_type": "projection", + "class_embeddings_concat": false, + "conv_in_kernel": 3, + "conv_out_kernel": 3, + "cross_attention_dim": 1024, + "cross_attention_norm": null, + "down_block_types": [ + "CrossAttnDownBlock2D", + "CrossAttnDownBlock2D", + "CrossAttnDownBlock2D", + "DownBlock2D" + ], + "downsample_padding": 1, + "dropout": 0.0, + "dual_cross_attention": false, + "encoder_hid_dim": null, + "encoder_hid_dim_type": null, + "flip_sin_to_cos": true, + "freq_shift": 0, + "in_channels": 4, + "layers_per_block": 2, + "mid_block_only_cross_attention": null, + "mid_block_scale_factor": 1, + "mid_block_type": "UNetMidBlock2DCrossAttn", + "norm_eps": 1e-05, + "norm_num_groups": 32, + "num_attention_heads": null, + "num_class_embeds": null, + "only_cross_attention": false, + "out_channels": 4, + "projection_class_embeddings_input_dim": 4, + "resnet_out_scale_factor": 1.0, + "resnet_skip_time_act": false, + "resnet_time_scale_shift": "default", + "reverse_transformer_layers_per_block": null, + "sample_size": 64, + "time_cond_proj_dim": null, + "time_embedding_act_fn": null, + "time_embedding_dim": null, + "time_embedding_type": "positional", + "timestep_post_act": null, + "transformer_layers_per_block": 1, + "up_block_types": [ + "UpBlock2D", + "CrossAttnUpBlock2D", + "CrossAttnUpBlock2D", + "CrossAttnUpBlock2D" + ], + "upcast_attention": false, + "use_linear_projection": true + } \ No newline at end of file diff --git a/empty_text_embed.pt b/empty_text_embed.pt new file mode 100755 index 0000000000000000000000000000000000000000..dcedcdecbf8ec649a9f088f3a9e50996dce8f89b GIT binary patch literal 8966 zcmZ{K2{=~K*1xIDb21j12$d2k?>U=F5h9gR6d?)8SPCJE3?Va>D3VMiiFdE{Hb_dT zWL7H8X^;k`;nV+q|L?i?`|e%m*=O&w&)UCduQRQ+*ZJ952@3J?iHPw1A5rEL<@4~} z85FuX$YXcVW)I&j9`1A8U4vX_@AURrws;)hhQAUi;1=W;;I?h{=HMVNpFpwA0Uld| zy?orq2JHy+3)mboPt2cBTYJ6m7=GZGrmLriKmR|5h%NK+a}8Skk4`avf%W`jI{N;C z>jlSZx1GWMLN+aKB7Wj++X}}}{}#LdXt5%n^1ruuj{dU6 z3jeV8{~7&HyTzdea9`^=RE-zr7KV61Rj(2I?YSQ;)pmi|(sEoEUIBQsmaIa%JAE_$ z8VKHOr2Ro3P1H`Ng6rpO=4MhAy|{2a$nFhhjtKNJ%O2YE=OKiltElKUVAtpe~9C?HXme+RG{R0FUNdH74x}y+{tMx z853(UuIcq-&{oHRbs?%SIvCG<5L979%lql@NDaO2#*af~O~9Kp#C03!pq`XKw82Bz zFL@eNZXLv@hZb|9s?8W-!K-wBX*PW%l?m?Q{WL)<5@h(U!E`}==vgFAXY85`FUnZh zml_NzEuBnth&z1gHHYdbQ^>UMgZH?Rwnz>`VqYv+Tg*hBS0I#KK{(YtU}rLI<;pAWtxf%I)t5b4w0*uWg2(3l$ooa^ur?$c9zKr-5;w@ei)fxzuMRkSCqtIL9E#tS zggE0pkidBWSzkio&y7R0x>^Pp?{_rTRURUrWpQ74sPmFDC&Mf)ABg^b6~dN%q%YJ; zX_A^goX9GMtz8)){H>UdQ)>ge#N#l*S%mAiu)u`F?*S?`Dv+S_1{5sBpv^=J!mCQC z*Ki!8=Tyci?Hs0b#{;@CJ_ky)zrdB8LfUj>H~0IYT==9_1<(9qK#+Y7wG*0QkLNwE z@2(us`S6fY)kuJO#+QI!Y(Gpdvxh`EH~PM97jxVA4Y>FFg2KzW&|1|BvvszE^eRVs z-R~8*cFkfS!v=Lu7qxNeyE&jx`307{NQ3mPyKq%xG8<%QWs+apWzy%j1~#Am31;KO zO(RTR!H7#L*yiNIDvSj4`Dz%DSDQ zx&uNW;q(e}S`(QWMZKJ-p)+A~LMNP+Qirdi-c|ZxgfBS)uSgThPBG^6A@lju#Md14lK2$Ui2 zfdICdzXj%{IxuCs7(eZKjPG^{co&gG3$~LDDoH?Jr1z`QPS1|DHLZ8a@D}@|)qE;vtxQr4er2J^{bab;01O!*HY{ z8YGfu!xrC2X29DNe1{)49N6s-_0wzUjzV|1I#nNryT5_U=F{-Tk}#J?bZN|~0s3n< z<%G_5Y)HMLjP8Dyn8jjd5MOZsCa>nf#RKEf_VNd6lXw{vuIz@lBAsAL;+XlVl~Cdk z$2?wN4K}6XaP6Koj1Rg7{tYiUTIB|e(&R+$V{3?yJoyhhHTnB+K zt=z^jTRrRBfq%ypNIt#-Mgl}I@Io25ElXxXRY9${mcmVqkD(HFIHlXhWY|1z3K`qpzBxAg$ZHUhMcT zI^bCYksfkPsKaJhzs(oMJ(&$2H?ldxhmo0c(Gq$UMVZ{yGBobMe1@F0hf7&nGq zd05UEfumsw*a}~O56T)4Hd-`xZ6r9k7uu=Cj0UiB$b+{RD&fMK`yghnO3&VXO}8dz z!la07Mta|7@VbAUbEDN0K;P{_h=vWa<_+y^Wxz5aRJmlxC#AlEFov|9ISgej&`k_Ouw0@(s|?WP`7$@P>j0< z4~vW7tj_@Bc&{05Unydo+xW0hSQA`54w|T*9i<(scEW9g40!L@Mn#XO!7A}wj^IUA z5Vt=G{nAG{37^OAn~VgG?wi6se!d@;eU5+?JCDQWZ%tGsd+s+nk+}mqpKHVSnWdaM-2(8`YNHD~_%OB50G7$vz`*UfaJ_LAB+fF0D{H>f zsL$c_LzNV@O}2rRqE4{GGz?_hIvak-UZ=AUya)BYEkHJIfuq~5(Q1|-s@u8r_aX(r zs!b58nhsc#0yo4LK+lR{&bt@o(qDqV`DGqHf8~-0B9ExjVsS zt2{N-Gyp+f032)o0tG*VL1^_=+NvA@C!Cw0cVr&?c}1Zjb~bm8wIal>+Sbr>vlzD7 zBE?#1pu@jbSF3ap2z6we)y+ z7(8ryP6xv7(B8{4z-~o0%#ykV(Ro8mX#azTkh)^XS3L(-MjYzk*+@&us$iWG;VL;- z!?)lxsODq9AU_H2jW}>$ti1`UUILgn{~5hkz(RtgGt}oTgj7*CroSi?UQCT;PT!Y@ zD_f%J*&GCk+;0#!=SPM?7X>hUA00cD4D67#v7DHU(RGn;xRQda? ze(OH)mWbq}ofW00XH2KhRtup;z#6KfaDi6LwuN)S2$Ftj&|y0tIL|U5P=rE;Lm6!@ zt_3SzBV3z#fC(GYVOxhBAiZ$7VXB}IJ9|%T!-tcbpt3O?T4Lt;dF_{cUo-w366C3(FRX~#Rr6el`Mv9x2$Q=sWg!7 zYh&^!&w;&4P0$!Az%JlF3mf0&!sn$y5W%kircqO%B7HY#hRD)o>7S{eWdQ^iykTm5 zj=-NK;#e1c6db-*(L>oGFgP<6raSh6S!gBYchiF<$D&!0HTg_-O(PiD+d`9*2t?&Q zq~Y`r*pAjSz5bTa@HLN{iz*=KQw1KA3gJSvI$fzD&GJ%2*n93u^m;-z*JP_0WKBw; zb3R;w+r|}e$~}p?E_Y@3EOz8E%~ zP$-)ZqY9SwdP?0;k{VAr3g_|3*n+?4Xe<2R`9qItN+e&c#%Rvx8 zD~NE>hk9iVc3-dq=jvK5dJattQ*6v697HT0L-1<>|Zs_)I44jn24 zj4W2dTDlOL$JQf(!`I-l`FJ!?w&YBU*Z>poC?t)W4~nsQa5hI6ufJD^fFoH@^<+C! zFH6BGu#ei?mxJ|3d+1TA>`q$zaWX4@ z?FFN%EeDJ4h`^ma19b6P9=s1oh68seQq}cOsgi6sn;9Mjk1A!5bR*>Il)&4*4Q%a^ z%k<>&vk)nC7G{611l?;bbTNMuY_giknRr;4oh!NuybW$b?YSm6+Rg`eA5361{30Ra zc|6_Q)=hV33Q@_ri_nsElNPk-L+-Ct%#fP}G%V!6otqT4xvI0OBbBhbCJp8fCDVmt z{$8i=FbICv0(wmt1+^uhxtAaM&ey=v_Y=XN^BM*X>)`IpL;l7vOVod%7uzb94SRw#xQW4;P;GOMX0|#(ynH-Zm93+! zjU8JyNfui7FzmpCpUjWBO|-{X44>Hb!J^BTXb%;`j^OPuAQDO0ff4F@_ZRq{TMg!| zZQP=6b*5qMD{vMof^TX)5E4t_-IPsGWn9kKk2j%OngdM5!lFg@OINdVA21Z1{Gek^ z76{7vGI95{V0Q3E`Zy>StX-5~#;2)ZuO^NL+CHH5LINv{7eM7MeI_A4uZRyIacFHa6r(EkJkikRPb>R?Kaf>U64QGLM?;Gy0;RZPMaSijp!+|!x zi-L=9)0y1%eK5CH8K$}4qI^#)AwVmd9p5QLbyKq-v80B)vK%q7P3;B6n2U_z$$Cy( zs5GT#5|Mjt8SC#ZYSgvp9)^*VU7-w#SCnS}UOJv_6i(UKO-7Qo)##3_N5V z$r{CXK)aD7b}q;Ri&k@@v2O<%%#nrJC~NBWLIxk*iXxn<5d12xi^(5t@nm-;nb&j> z2k3rm8|lCcpEL;aezN)F^@!cpePn@sGzMjwQms;i)M@^#ObQaV=mx4idCsPOSi^>& zKZxT*^x??G$+$CZ8Vs#8BT}y?;&FLpFPv{lz6@P;+{J@#&;UKwJ&Kmkwa?;}G|&8%c{KDnIB!=OppxcqJ!DGYpw`EDPu zqxvfGzC8;o{?uXQk|v^Zq6D7H6Vr9|3NZJIA#aj<9=o|>0sczAPSeL?IXRPA#3WOK zH@C|Pk9`ou?8je|7|W1fcP_II2d&A}tI=qicn{i3Wle>|^1$fS0Cu*=5^-BfPQ@6KOim+a zI!>`uP%BK9_aXf-@}^az(mVlFdlrKwr~UBKdIgLZ zi^tZaBz(r5!7FHwV9vE%Wp5PRhpurHMjQ9y>xM;`(zp=h?hJw7QCa*l<0;jdqRUg6 zm(0pUZ$yVLzQiE?2TAIXWX}hF!PKU!C^{%g)`bg@ZLXd;!heM}IgI0-U+6<7FloT| zbtTrsAPxVmjN-Qo@V<=}aqB2yTl4eik)7M1iZ2(0&kNzI&9_l|;1>C|;sO~K_r`fg zlHlRw#aNJ&fodaf_49f9quI&Et?+et?+0KaZ1N zA4bC*L3D{bh-<|j;^nU?$m2d_pFf_)3lkOP{Z_QX-;R95IBt~8TNOrDygJKXPFap& zFE)^s4`$%ipWncB>nu~#E-4~#bs|hUFNyN9X4ql-2%To7kc|rykuM+yc2=;a)3VCg zM>8+7Rar>mHf6D^lLJBSWHYfFe!%=Z)=gelEW(C&L2YhD6}_%gf0|XE}HwHb^zsh>%XL5cJr$pMGpSN7xH(BvI%#=1y3K z0y~n}0D~?lGIS=}4v3KJcDL9?L3!vS-pmdL@!>}+UwD0yOANi*iNu@tWX9HF46*WH zy*CJAz0zawiqAl~DsxnE>m*NCsT0K+5~j_`$~ftz7OuUYhzXZpf^=yHC@-3B>S$ku zul#qish4^%Gpz=NKk=je(hYcc^)~bjbRz!CrFhL%Ram=l5!t7_k$fm!juYAiOuuFI z!J$4&(sLjWIiKoj)+IsmcGp*o`y>SqCZw}r4^>FJ+fMRpJr0sn_J7O0}!qa%Hb&xK% zUwI9hLxgGOTV%PSO3O%jY+1*nN@= zNsK4);v1OEtE-`Ly%oxOWuR2=3dWst7^d`H;cR-spfkqd9XluTy(g0lDtVD~$5$}k z@-=o_`rsltKJ0&Wk#!2RA$yOM;}qpn;J-Q<6Zm4$p=Ugf%s++aj*hJ@zvSZY@Ui&I zPSkY9Eq&S;Mq&430p7SdeXu@T8aIs0gu%%7FwiW=yYe|6kKaweTxg?*ils5o%$r&n z{KncaL7b&MpXf`e^G-c4#%7szlnDOHPH=F)ehxZ@ z93!8}buc`fM|TC^=WLVD!^@BB7@egeyc1vUp@60uj1C8|&jKn*<2`B9T8#!WHLQT$ zo6SdPxg<=>Da7c-9OCAoP5d7BL3>LXcIDn-96qgOm*1Ys^%Yr5LYAE-Mt)r|Gwv21 zzp8?%{sC<0?%Vj*Qw_ItH=#h}UGl_24_oqPqj>8$ykol^mE=Fd1lM@BOYI1&%iQB8 zO&MToVp@sY{sfF&EQQM=VuVq`iAzyF@l2n} z&UY%omacsyJw$_dVoDy0C>yZr`m*ri@KF@JJw!(>9%HSqFte`KkG=k38E^lav2Sm& z8tGDV!RW{ntm%>@?0XZ-p1z%eJw^m)GLPWqrz*03X&XptKEYqVmykW*kD16fh_RXi z=BQjc9`b**uoinRk|>=t#?B&!;kTZMOIs(PbwoOdzbnS9enHl*E^#bJtw?OvDVVyQ z636R%rSR#{Gxq%9m1yPJz(jS=$G+ZL@M=lK>sO_4c)?Sc`>BOp9;AaOR2y*8!9`?w zN(+`(tFhz~KWx5Z$jklt7B}^W!L_(qg#T(kJ8jNt(}N!Aq}6Z%8XK<0B=MK5k(v^( zZSq76SJ;GRjVGAkRbp&LQ85ULjj%EX8APIO8YqvbLc#}ed=|D6-6yB9+B0<6f~|Zw zPw@bT6gHDRp?plLOE7D+TpP7I`JlexAU1iF5EbWKc+_Z5P<2?A`2= z!5a`)FqteVnTXuvUfkX103%;lqo+eD8^sQ4-h_wkHAvD*F6w2fgV>+9EZ_7p;?n8MUXj&eN4mAibEiXN=i3?6eoYd~ zR}3`=W3gH20Dd`Li8j`bctt^o8$;VA9N3`>B!!SuT6yi>ayo>QtPjc>~kG zZzZ>KjiBPJFH)2`H!8LCCj5#H;hp#p^9*NZh_vxW3c)@;2Y;% zWS-4q_RB_fqTXG>(OB%yM)sb*awEohTxT0Fge?hMC|`;C*mg;6E91K ztU294r7Rir=j|qI3U7kcM^l_gSS=GO+QQZvi18l$ODyO zc3mT7b|tML@=NC8+-3`sv%wvAzM0I6l#*tQRnMa4wqf{_ng)tdCb-XV341Di6r|hD zc?Bnm@WD@Q{OP3v7C-bDA>;ix8X~|d3j9Wi>}E*+dK`cMNX4uJ17zCqy==Cj4W5T( zWPinVc(OdyhMI zpF!7DVN)mPa+JHKjc^K4RP-(@FEWm|U{?uxKeAzVJXXdou_$u-yeD(_)h;|cuA8%< zYAUO!u#PC|9Kk;^bLoJ+9jrN&NLHAqVfXhG_%7QEAA1r=X{R1$q)kE-B~8=sTh|eZ z5AO*_$bjv8dl#&i4RJ*mCqZRsF#cJ4n_~5G(y^kDDld2qs?|%${pn34OZ_9-)Q^Xz z_Hx|x!;%y~UP*o^#o&o2!k9Fq2>cGua8~JYe6nv2d%|Wp?a#Bpmib6UM)!{SJOi@n zwkVmWWy?OhpM|;l;$%a63VBp!O-@>su>8>{khkLvJ7||qwi;A{lAAWks&!^t{IiKm zY$Dt-E8&LgIHRc64$`B0kR)`?BZ<3JAl#}HcHD6!D^!vgJ^ng2;miSO?f%0|sY~GS zyN6@d=L&K+Qk>immL?moRk3atXP|=GeQ5r;61bH!VM|yhBXxH@p>8kmac&ew zWe9_)eK4uqsf)yO1$H<~@G_dZ!A<)i_5{S@+oT0V?ch_0pxG#0If*EAT)~;+G>PD$ z`52l!i`d(27|SJfV32JB-ncsh9VevXNv#Q{m$Pet-*)Vu%}Rb=i(es5QaQ#x%{>gE z=rx|JdMn~)BFujwq3d{4!#$hG0K>NE% zrvx%Gmjf|)aSQiwZUD~Tm;>ULGaBwhT_7*)l4xGE(5tS78pC`Jm$avP@x=b%mc3QTc4gl8T^`UslADqAM6Gd-S041|BCI=UczwKzo)b1y_rt4ajcVDD>WIA#@(WnsQLEe#rU)2 z`S5E9i`q<1e(<2OHg4z@5QcNyZV&_WKWy?3DdM8+3Tb*8ru#IYn~*D|aX;^D=~?{LqG)95R+35{Ns!i0xYSi3U;JwLGv+e3(XVl}ZE@WS-67!u>Ihw1V>@|RsueW7j^pT-!q zV66OSc7YP#qW{Y-2=NFA9HTERTWs|2=>`A9H2C*T1F^p|xOdERt-#m>{uVnc{=e1# za60~(`=I>y#J|XYWO)2@n0Z5jUD}4m^u3Y8|0tosq`-(cAEbNjQh(I`78gwzW