From 28ac439a26f69d053f6319922dd0255a65e4ed7d Mon Sep 17 00:00:00 2001 From: doggeddalle <153059103+cicalooo@users.noreply.github.com> Date: Sat, 8 Aug 2026 19:27:06 +1000 Subject: [PATCH] Add MiniMax H3 Power LoRA Stack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a ComfyUI plugin implementing a stacked multi-LoRA loader for MiniMax H3. Introduces H3PowerLoraStack and H3LoraInspector node classes, plus modules: adaln (adaLN dense↔curve basis conversion), branch (runtime low-rank branches for quantized weights), apply (stack orchestration), detect (safetensors header introspection), keymap (key normalization), nodes, and a web UI (h3_power_lora_stack.js). Includes pyproject metadata and README. Preserves quantized kernels via branching, rebases adaLN pairs, handles many naming conventions, and provides a browser picker and per-LoRA reporting. --- README.md | 139 ++++++ __init__.py | 7 + __pycache__/__init__.cpython-312.pyc | Bin 0 -> 418 bytes h3lora/__init__.py | 1 + h3lora/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 215 bytes h3lora/__pycache__/adaln.cpython-312.pyc | Bin 0 -> 17844 bytes h3lora/__pycache__/apply.cpython-312.pyc | Bin 0 -> 10290 bytes h3lora/__pycache__/branch.cpython-312.pyc | Bin 0 -> 7494 bytes h3lora/__pycache__/detect.cpython-312.pyc | Bin 0 -> 7340 bytes h3lora/__pycache__/keymap.cpython-312.pyc | Bin 0 -> 5233 bytes h3lora/__pycache__/nodes.cpython-312.pyc | Bin 0 -> 9855 bytes h3lora/adaln.py | 337 +++++++++++++ h3lora/apply.py | 197 ++++++++ h3lora/branch.py | 125 +++++ h3lora/detect.py | 113 +++++ h3lora/keymap.py | 126 +++++ h3lora/nodes.py | 202 ++++++++ pyproject.toml | 14 + web/h3_power_lora_stack.js | 502 ++++++++++++++++++++ 19 files changed, 1763 insertions(+) create mode 100644 README.md create mode 100644 __init__.py create mode 100644 __pycache__/__init__.cpython-312.pyc create mode 100644 h3lora/__init__.py create mode 100644 h3lora/__pycache__/__init__.cpython-312.pyc create mode 100644 h3lora/__pycache__/adaln.cpython-312.pyc create mode 100644 h3lora/__pycache__/apply.cpython-312.pyc create mode 100644 h3lora/__pycache__/branch.cpython-312.pyc create mode 100644 h3lora/__pycache__/detect.cpython-312.pyc create mode 100644 h3lora/__pycache__/keymap.cpython-312.pyc create mode 100644 h3lora/__pycache__/nodes.cpython-312.pyc create mode 100644 h3lora/adaln.py create mode 100644 h3lora/apply.py create mode 100644 h3lora/branch.py create mode 100644 h3lora/detect.py create mode 100644 h3lora/keymap.py create mode 100644 h3lora/nodes.py create mode 100644 pyproject.toml create mode 100644 web/h3_power_lora_stack.js diff --git a/README.md b/README.md new file mode 100644 index 0000000..928a9d9 --- /dev/null +++ b/README.md @@ -0,0 +1,139 @@ +# ComfyUI-H3-PowerLoraStack + +A stacked multi-LoRA loader for **MiniMax H3**, in the spirit of rgthree's Power +Lora Loader but built around the three things that actually break H3 LoRAs. + +## Nodes + +| Node | Purpose | +|---|---| +| **MiniMax H3 Power LoRA Stack** | Any number of LoRAs on one node, each with a toggle and strength | +| **MiniMax H3 LoRA Inspector** | Reports a LoRA's format, rank and adaLN basis without loading it | + +## Why not just use a normal LoRA loader + +### 1. Quantized weights: merging destroys the LoRA + +ComfyUI's stock path is dequantize → add delta → `requantize_from_float(scale="recalculate")`. +That round trip is **not idempotent**: re-fitting the codebook and re-rounding to +int4 injects ~1.5% relative weight noise, while a typical H3 LoRA delta is +0.01–0.08% of the weight. The merge therefore replaces the adapter with noise. +Measured on a w4a8 checkpoint, the stock merge recovers under a third of the +LoRA magnitude at cos 0.12–0.14 against the correct result, and takes 128 s +versus 6.5 s. + +This node routes quantized layers through an exact runtime low-rank branch +(`y = W_q(x) + B @ A @ x`) instead, keeping the quantized kernel and costing +~1.5% extra FLOPs at rank 64. + +`quantized_layers` controls this: + +- `auto` (default) — branch quantized layers, merge unquantized ones +- `branch` — never modify a weight, even in bf16 (fast strength A/B testing) +- `merge` — stock behaviour; only useful for comparison + +One layer is special-cased: `mlp.fc2` under `TensorWiseINT8Layout` is reached +through `comfy.ops.linear_input_act`, which fuses the activation into the INT8 +kernel and never calls `fc2.forward`. A branch there would be silently dropped, +so those layers merge even in `branch` mode. + +### 2. adaLN basis mismatch + +H3 ships in two forms. A *dense* checkpoint feeds `silu(time_embedder(t))`, a +2688-dim vector, into every `adaln_proj.linear`. A *curve* (pruned) checkpoint +drops the time embedder and stores an `adaln_t_table` of shape `[grid, 8]`. + +A LoRA trained against one form has the wrong `lora_A` width for the other, so +ComfyUI logs `shape '[96768, 8]' is invalid for input of size 260112384` and +skips the layer. **Dropping those pairs is not an acceptable fix** — on the +turbo distillation LoRAs the constant term alone is ~100% of the magnitude of +`dW @ S(t)`, so it discards essentially the whole adapter. + +This node changes basis instead, which preserves rank: + +``` +dense -> curve A' = A @ V, bias delta B @ (A @ c) +curve -> dense A' = A @ pinv(V), bias delta -B @ (A' @ c) +``` + +where `S(t) = c + V @ table(t)`, recovered by least squares of `[1 | table]` +against the silu grid. The bias delta is emitted as `.diff_b`, which comfy +applies as a `("diff",)` patch on the sibling `.bias`; without it the port is +nearly worthless. Measured end to end, the ported adapter reproduces the dense +contribution at **cos 0.999998**. + +The fit uses the *target checkpoint's own* table, because bakes differ — locally, +`PinkCherry` ships an uncentered basis (column norms 22.98, 2.67, 1.66, …) while +`10Eros_Max` ships comfy's mean-centered one (7.08, 2.09, 0.75, …). + +**This also catches a silent failure the stock loader cannot see.** Two curve +bakes have the same adaLN width, so a LoRA trained against one loads without +complaint on the other and is simply wrong — measured at **cos −0.375**, worse +than not applying it at all. When a LoRA ships its own `adaln_t_table`, this +node rebases table-to-table (exactly, cos 1.000000, no grid needed). When it +does not, the width matches and the mismatch is undetectable — so prefer LoRAs +trained against the checkpoint you are running. + +#### The silu grid + +Dense↔curve porting needs `h3_silu_temb_grid.safetensors`. It is searched for in +`models/h3_adaln/`, `models/loras/`, `models/diffusion_models/` and one level +under `custom_nodes/`. Table-to-table rebasing does not need it. Set +`adaln_port` to `off` to disable porting entirely. + +With a grid from a different build the fit bottoms out at ~1.7e-3 relative, +because the 7th and 8th curve directions are near-degenerate (σ₇ ≈ σ₈) and so +differ between bakes. That is still ~6× below the int8 quantization floor. + +### 3. Key conventions + +Every H3 LoRA naming convention resolves against the model's own key set rather +than by guessing where underscores split, so `qkv_proj` is never mistaken for +two tokens: + +| Convention | Example | +|---|---| +| ai-toolkit / diffusers | `diffusion_model.blocks.0.attn.qkv_proj.lora_A.weight` | +| bare (no prefix) | `blocks.0.attn.qkv_proj.lora_A.weight` | +| kohya / musubi | `lora_unet_blocks_0_attn_qkv_proj.lora_down.weight` + `.alpha` | +| lycoris | `lycoris_blocks_0_...` | +| peft / diffusers trainer | `base_model.model.blocks.0...`, `transformer.blocks.0...` | + +Verified against all 37 H3 LoRAs in `models/loras/h3`: **zero unmatched keys.** + +## Stacking + +Multiple LoRAs on the same layer are fused into a single pair by concatenating +along the rank axis: + +``` +sum_i s_i * B_i @ A_i @ x == [s_1 B_1 | ... | s_N B_N] @ [A_1; ...; A_N] @ x +``` + +so a ten-LoRA stack costs one extra matmul pair per layer, not ten. The factors +live in a `_LoraBank` registered via `set_additional_models`, so its VRAM is +accounted for and comfy's weakref bookkeeping stays quiet. + +## Output + +The `report` string output accounts for every LoRA (wire it to a preview/show-text +node to read it; it is also written to the console log): + +``` +base: ConvRotW4A4 x300, INT8 x50 +adaLN: curve (input dim 8) +turbo dense-adaLN @ 1: 26 merged, 275 branched, adaLN ported x51 (basis fit 1.7e-03) +GalaxyAce curve-adaLN @ 0.8: 0 merged, 258 branched +branch bank: 300 layers, 412 MB +``` + +## Limitations + +- A curve-trained LoRA on a **dense** checkpoint can only be ported if the LoRA + carries its own `adaln_t_table`; otherwise there is no record of which bake it + was trained against and the adaLN pairs are dropped with a warning. +- Runtime branches apply to `MODEL` from ComfyUI's native H3 loader. The + streaming loader in `minimaxh3chinkloader` uses its own `MINIMAX_H3_MODEL` + handle and its own LoRA path. +- DoRA, LoHa, LoKr and locon adapters always merge — only plain rank + decompositions can be branched. diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..ccdf11c --- /dev/null +++ b/__init__.py @@ -0,0 +1,7 @@ +"""ComfyUI-H3-PowerLoraStack: stacked multi-LoRA loading for MiniMax H3.""" + +from .h3lora.nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS + +WEB_DIRECTORY = "./web" + +__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"] diff --git a/__pycache__/__init__.cpython-312.pyc b/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ee877b2314ebad350cda4c1cf729565d1c89ac66 GIT binary patch literal 418 zcmZ`$J4?e*6uwECR!Rech~SjLB}3!Tt%wklV1r2-8bulcmzbN>Xp$>=P}@m=MRzxU zi@(53qJxNw(9Ox4;$!j*=X~e!9nQyj;J7v*wsYVpta-NuC6y$%D zjIDWO%8O$N@4=F!;8T(NE{eji%p;uowMy47^8zPuMR)%d?m|eE6%dM7vubTG!KjRb j{j5Wl)-v-=;yxIPq8`D~E6|?Sw{E!SwsqUe>eqY$qkeUn literal 0 HcmV?d00001 diff --git a/h3lora/__init__.py b/h3lora/__init__.py new file mode 100644 index 0000000..e421f41 --- /dev/null +++ b/h3lora/__init__.py @@ -0,0 +1 @@ +"""MiniMax H3 Power LoRA Stack.""" diff --git a/h3lora/__pycache__/__init__.cpython-312.pyc b/h3lora/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d5be8bc5bf04ef9655251d3caaecdb996f39eeb8 GIT binary patch literal 215 zcmX@j%ge<81QTDEW+?;d#~=<2FhUuhIe?7m3@Hpz43&(UOjRXTGe0jmKR2y1 zCb_h@BtJJkFFz%UUaz3?7Kcr4 geoARhs$CHq&}@(!i$U)Dz|6?Vc#A=)hy}<20Go9@2LJ#7 literal 0 HcmV?d00001 diff --git a/h3lora/__pycache__/adaln.cpython-312.pyc b/h3lora/__pycache__/adaln.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..66993d174655c92c0fe372089b0b491ab772d4c4 GIT binary patch literal 17844 zcmch9d2keImS1Mwcj+qKf)q%I5=bKoWepIU9W0@rx_siaEzM^BE6U!a1**&UiT==Yn?uSIW8JU1*>V zQk>@+#d$w7WGEMLWn4K|0Hun#!fVu-k`bf7f^D%0&OjVn}3y(a2?95JS-j zbCH*Zc|O8$d_?3KHo`H%m~feAj)o$kqwE!?qlF3f@xg(iXec6y%?<}dh<%|UkqJc@ zX*kOCM1?`InPFO)Mna=8CqL7}^PI?BxFCkYvGr1DkPq;K7kQ53h4qr(f8oML2g_{P zwqr*V7aC+P^Fb*pYy`qml;MHF5h%%WY&a4a5~BUh;ZTHUg$ozd$_Sf{%=#fA7U4O+ zBg-Yj2~prr>f;%#lF_OptqVm&iDx-RgsOOvVVNO;4@L)vViM0ewCbdQ#9j>Z7cMZ- z9%wB)!~-X1dxa3Ukr_A-(d!48z03|K91XJJ@Q8mmsZ8aMR196i4)D;p0n)Ni1n2|X zZPCG=k@q?qAP>@q>II$|4V~31 zs;sL$+HiK~w(Z-1y&dNpFmEVwnGJ_HLW^`5>HN?r4>RPA&6~Hh?0D1RzzGVI33@9$ z!iWPn50RHJJHiXl!QGM&8X5{kdLcFn^LdC32|!t7B2fw3#Rh}?5b1lSCv;_(1DZ~z zNldsHWw?+ig~HIwxGu1VRNAmuM2U?^3=Gg9!-lc1fZ|7+-*}@QSp))(LAEy%l42Z> z5f?6S@56}T0Kp7rM4${4l5hpH85SW_6l4Wh7`!M#OF}q@2p_|J(J;&!jva!!pgEnH z$vVu#qGBVxyt;n14u+{K@~~J%MqncYyVTBA7jo0S%sPgJ*#Og|6&N}Nvw-}D!ZDG# z7=oeZ_^`w>%sY^}9L>(X@)Y~U9V(cPWEzHH3#DpZ=uR0)<$WrOIb&u(U9 zbq~SqjqQboAf;0ltd?2$s!pM=)%50cDLUa%&R8J4I zB_ic)`uaGd_}~R~x!@*?k_0DJG}hb4@T|HoiF~-H2^Tgjd7#4#L+3hh(g5?!Fpm=o zdh*vORQKg>JtML`JkSxNf~a`aG!%3sakPvJ1*Nm1 z1p6Y$oM2z`Ns{xt;w0&TAn?SG!W@EJQfw&9Lq2?+&CR{6?0Ll%Afj9?9n1g=0s?IC zG*T4(d>ufTX-d-5WJ&`daoV1NGI4`U_nS$%B@fA>efA^0WxsysvIzK;4f#M1XVjpH zRyR%MuMx_7U#7~Lv~nC>0m<6eN>u)z@ooP5^37SKLXB_Pz*$4oXVCljyG#isE40d@ zwL@)H!uhv4@cb0ePgCo!f=GWAl zmw(xu(}K*rh+AGmDPIfp83R4pAzQEwYCBBS^~WY^#I}}_N;K~4D8Wcmr>X0|7^bcm zPgBD*q^^1?^?zxpe*5U&BYzyJiO%>tf3x)d9fC4gNq zqu3Lb0^F)&c+dzC@gOR$AjppBU?2k0LR3n7qG31;1S0GpF9twl1wan=i3;T}P)uRi zoE29lhZ=8ThlcnFrx>E5V!|lJ)DH&}#i3FU0Mt+lKo|v(Tv+1%LoZCdG`xWejvZPw*v;;r+p5yGq6J$;(pK^5nz@GAhPjQi8|USPhJDF~eGeR|%EM1= zNAxT+8y70KCo8w#x27uJeo&aKYMfEthu;oq^cS+bUmrcgOBLPm4|+{x^`}Oc6d&nmG8du`N2B}KRFlZCQfmw)%**y!QM+hcTj~@ei@3b%|~o;uw(woh!ov2SAE%(^Ge+H`Tn_m3``EOy`1imJKF*~mxwVxdQjg_iwTr31W z7ON?j=XWn^DVOiJ6m52YV{wQ(Z2DW zEAwUgDoFcoRq`7Kjd9pR3?R32@)#vu+RMhvwBVEUT}`I`l;U#I;mhPChDD?BhGxb9Q;C_r;|0?ToAj}|jr6mTXU4gm~G0D74Qg>LZM1%yP35%vwG zFbCW<6G$dBV4^EPc8rtzySk6Pt2j=v;TTVL2*n^p6*I}HSbLx{sb!1bEEHiOi^`Kq z!60&q30JI$wUK>6sKZP4d2*{Cg1#c_1HU4t;I82lGd81p7NRYnQe*MH~;e7y{-qfiHi1=XMe)BKkam1 z4=0^f3(i$Z=c-R*bMe`D%Gos5o-VJxc6id1H2Km_@7UqzP$!5Qd}e;Qi;)R9PXMM& z^=m0PZxz@8)XRy}f@<2pc8U@2CBBG7_GYV3xEn-304`m*WA z`Lyh(Q!znLvfunl;-5#kj@8SOY1Xp@wN0c?H^(cX?GQ1d52 zs`60flJLHMw9rqK+9)cB&|}t*zU2ocE-3JIB?;($L{Nnx*cFMQdPLRV0Oka34(d}7 z&WiwxiQ1*>*QhYKLn~%Z8X4jtU%61J3R%f&@U#6IeMq0q>sTe|>VkTl+F^aK40pXT_3q@$cM>FP8 z#rn246(gLP6ccEKMa2RdJ2u=4Cm&G~FDW*y&*6vw&?3|ui^fF7t}8Yb6RIK=b1=*f z4k^YX#||piVOEGh*D9t^1QhC7-MFen!k2`<$h9r;1QiDJCrzn;)(m+ zu>()NB{$>=dBIzo^w!R=yIq^|Zd~wgO?tPEb);35mZB@#~V092PSv2la z<7O_+*QRWB3${&3+oqIl^H}Szyd{%8Ggp$O4YzBPr5hG}o0GoHDPPMy_RF3xdVbRP zW#o&tY)1oO>sA6PS3)qtC^ z!)MGEFM$zaJ0muF)trJBGe)%+5m=#%{UxR|G==1X2hH$7fQD+O1{pC> zmqkXQ{Q@|VPy(E5DPbo*cH;w)XqUer3m=4cFnS+85OD}?_yA~53H$MJ03RsPg}3o> z2p@;>(SeUnd=O}0S6piUsLl$COPx5GFC#QrF+zidZY)9Qp$wV+bwy$h|WJ^4pLfb?qxv~zF5v{x0$(6{=}sD|k0D6FR`o;pK=&yj(n!NgJ22H zKy=~H8va#X*whrq5Ouf1E6;|f@N26WM(;>NrYp-g<{9`NX!NTnco%_*ecdQ z4+}az&TmpoXzmqyAU7!@^g>9)x^V@lliwByfWwm%2#or2d$3uH{#RH8k@pz&w6bQV z|4%oKo34A)zDjsG(iLmqHe`|NrB4G2i<>?&tyHs`I!tFRj_epT6;&Ugs9z$8K`5gS7V+T%H3soZ zR6TnLJbD=y;VM1}yA~>TRA4~-&(Qk_@PX=+g}WM@!_nIay}L5%oJK;dk>mn}%&`c; zGDr{o0@1HfLu25Xgzh#k>nG4*gb2;u2y@sBzv?EXQ&VduZDLi(*Q7b$ksTNqN5?y8 z-$m7@Mpg14(I5(HK^FaQz;Z!0cRmas=;MQ4WvcfbVm|PLVuq3MQgo)m zENo`i=%3_Kmw-(ZO(Fu1G9V)Alj;w|zqr@RxW^E~~0KS{Z{Zj3X>MW;Y-WB;f zIcfxGH@XX40b>zvHv`U-*pg60^)V7RX=dh4*kyc#xcR7%4c($(>VRTqDE4cdxucu& zG;m2?CKh2YgQy@>*4K-Ezeg1lAt){!!N*a2^x*?Vhl<*8Aqi-iAu1G9#ei|PPepks zhy+wqgWU(WZN*6@S_V|1*vja0_$$HX#vd~|42%{n5kn;I-$FUj0jmapSOK+q?OgwC ze`3{}lcs6M4;}Y6jGNQWg7LlS;ty8jQzV24@&E{0`=JEEYm22ns-#T*h$iL`XsN0&X+j@^r)$O^@r7GLT z+pix^S2J^mXAj?|XS?9@C~#9ya^pJ_-$~difA_*p6;%8dbVaM5y1X|IPaIB^AAZ#R z#C0NFwHh)XNms5KZ@+PP{BRoNm|Eudi+0HVt%y6{!ws$5+M1}pZnC#;GXC|py7orn zziTu@7;uAtySv|{3V$9u9!C%~dy?QcC>DXJV8Lh~jfRCm%xY5r3ZRcbAfPw`=(P|F zW7ri4fcs%MlVS}7xM(mC5C)K%4b+lDJov2%5v(Bwk2@3*vIJZRL5#)6E%+1H!s8nC zjp^_Ulf_&EBWdyc)|{SdgrH~svdgPfbjv#jeqgB6#)ZUa+-E>$&B0K#=n+dG5BGf7ydkkP*HpK0+;pzb}LFv*VTjSMTxqdeQ5cB5J zY9L%W;?6i2C1ht0Qg+DzhOyqPr8=KC8fTSV8Woz#mEM|!jeCGoFSJ#c*I6z#aNSN# z8qdE+QRkq=@q&1vT-cvSqI0@t;|eVnf@fO(T)JTA199@nX3MjD^Kx z^!1a_`$f1G;w5rPe`&TQT8MKKZ5e$%1nGGUXq<DEnl4Mz@o%M&i#mupX8Mp$g6$ujC5iRg-i?0Gf0^ zqw$b^6FMH48--U4R}9z!&NB*YKWGdRQ_%JQDtpI@pzPXg87(9m?iT5N>VUE@Ym*Gpk_7`fv7Tusu~H_Ki!%d6uxFw6DX4ehMX zJ5JeN^XKo070QwECB5W^o~_pVxF*}jk%ueRnk|j%n$g;teXme!mfJ?TENAY(s4xHZ zZ&)4!n6_FQmkhtPBEs1UT$9#*mN|}+_5A;5HW;q& z;7a3FE9?$sT3^Ih!HQTdue!R|4ya#VMb;}LHEDezp?Zk?WQHr(c0r*)X8O_lXxUGj z)o_Yqv>N2{mFFI6Z=v=|dG#ptkX-u5?-Ftajw(`#*RHUZigMS|3VDi`{R7x-U>8I=3}-$4JP2pK+PM2QtHma}wYh@6qaYpKAGs%u*XPtw z_UyKdvIS0tFdt7-u$#H`^IZ-53jiGNsz(Ssimh;&OFiJA3JT6ofadYkpq`$<_)B1~ zZi~;=c4kfTc=S+j9e}}o$$10w^x(lEb9{mdc=hOWGUqf0dM~&yzzAS1g*#0|yRMxE zK!Fa5FzvvJHxl4vq^g6FO*EN1@D!!qfx>$yS!1;DTD>_`hF6r-vj`zUh_PV-?x%r` z9u?w5F%B1<@Lm$04RiGAa2U-V!$7GG#`9<-bB~A|9|<+QgTsd53KU*H>qHefTt|9Y zd!FgzqcQN4RUP8+Sh*XHfVzu9gdYxwa6b+1LP_FMeR-GPtXS~m#dC@kFJ-}564UuOE2>^j~2Zgspj6f;<=5qu1ShEsi$N-_G@~4;$Sek?bSuKgYEKj z->4h?Y6Gp!>IIO|0`keh315p*GrXgHf}bAbj}2wMg-ftO5EB5NG2plOmhg|^C43hi zpuiI`d=(yA^4Ft>R^*hdhS446d^V9=aM0nx7*JNs zYVcTW z29N`B7lshjk)yFH4xXs>g zee`D1b$o1pdUfsC!Jm5z$Bk(xbK977ZUArTbY;z4^K5g%zb{q!)`WN5G|o%$I zElyZh&$r#`yxEzk+qr14*x#q0uHAYs{Gjb;oew)xYrB#zX1sNLc*d5lsGK=C;~MWw zGwa9qPc|l9HF)~(nCM9Ojwjwd^Tc%)R18kdbgdxM+cKG{0GcWQ6SQ#)>a<}67#nfWvK zcHTen#h!(h{mGX757=Z&$K(1AunSBa!q%os%4TY(4vz1~K*Q8g2o#r&x2KD%zu%sA zm(8@y?VH{A*zHfd3McD6K0LEI>8k$tFmUnATQVL8UyeEZto?R)#-1wQ4ech8Rm|L; z**$-@Z?X-DP9K~)IMX&?^0y`Hz_-4-@mYayGL$M{=3A2m_3-|tpgdi(W}*w2xiq;x z+v90(#mtGh_h;XqKQVjevA5xN+vi<(y6&@I_kPv;fc>if@x~6QeTJR2LbZCJta8ef zF07ilg#P;Tmu5#F7y54pKabprJfMGOeQ13|KXg29J_H3m<>$h);e}P3lB+h|qf@K4 zB+IuTrJ8wq)((NPswva2R83X7pl98<6i5R2fsLYzx9hlFU+8D zTC`EswSQoni>t=lp`5RBx^t>?rZ-ixZrq+OD8?Cp!?tKJxvSIVl`}hM-QfIRUURqi z9)JJTljimr9<^(qh9A|Z%8w_yKUnDgFxmZKB5)zu%_hs)Ni)p&(#o0rQ|IPS+&X#l zUZ4Vnyi02>1!Q#JOg(ga7~&tPfAa0)#;^Q{^rli@08!G{j%YUhLnHT zxN-bI!dIVk)hBK1(*DM!-|7@QeQ(|U9my?ikLzJIcyAn>I5^okBRz4|rRy7R+i#v9 zZ=1BtlqU;nldd)CH5+d=-E6u?e`)=~np(31=}n$ay1pwf-(gIYl^$;~CR)rG-t9c$FemLa zJcGGvDL4wK%B}wmfhG1ua%n+xjETel!Vov$;U}LFh%@4WD4+5vpDR(neuRWVnO{Cb z5oaRDEHHD)bRKxk;DyAi<&syU0mtM#<|dVjcJPQBPUH0pLQeK%C%8zA~qaVimZ&5C;o67C`%qy(DyZ33!?6w5{U&kF|N z0#g=~5f?_dPOMl|0HlgDjX2 z*_rh0T=2Y|^t`>`Igs=mcyKA{=^Qu0n$DT7R&w*k05qHh31{`wBA@DNIA1#zNfp(P zxBV{d^kgRYq80LeD;5GTQ@dKd)I;yu)*Z%&+i7?LM*9fj{RUweUczMzo5;lt5+j*jO-<;IT>!p#nv{xZ(}KT{`{BG7v*!I=L5?wOA0} z4plXc$bk$KX?|95t=RUcaA=?K6Uc%qRm7fwg-+AIHPq9lzXQJzdhIVM&(A6QFDchE zOW9cOf~72JDVwpSEVX0CX9b(a+-WZ}=6crPA9E($bt%f9eyerNmGJCIQLQg5RkY{% r9w%M!ym~$DeSV7e(fj_-8*k86zoFpqqK6)!>C)%tjCQ(|F!p}|%-3r? literal 0 HcmV?d00001 diff --git a/h3lora/__pycache__/apply.cpython-312.pyc b/h3lora/__pycache__/apply.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..81d449329f34f0d46332985a3f41ce5d59e871d4 GIT binary patch literal 10290 zcma(%ZERClmhbud`#Xta^6`?81d|X5p-?CV0trwGp%iGRp^PzpFNqz;PTzZm#C@KI zQ8nvy8DX}wirKbmsu^jFs5GgxyOW>HXryJde{7sK=1nRwYG+s7U$h|YYQBpp&x$@l{t!~hAEb2 zRZ)6EHLRksu8yjQ)$rCtH51xlZKX{&tS5a2^43O;6Q*Gk>C;8c6P95M)b*?(YMrnR z+ezIBb;q!i)J;%#4ZCPcMRDp#hl0%Nn>F8|%U{DD*1}rD^l*)eI!3X!cPZ9>M^)*o zWouZ+yPDxTww86m+soFmE_m0oUe*n7pOG?BY(4yZtmj?r@TO_Ce^YYLSuq$IXX1Ql zj1xpYD2C&)eM~IQPXwc3f$L@_VsViS6Nd2n zG|zH8#}0%;B7EARoDB!|GTykVgwF(!g6ct@RsZGa<{X9PU1O;CNyp z7~95kfaT>V#{@&6cp@e;@e$_gSWslffFxXuW4jq4!H)z(9LvOGj5x*t9a*j?RN)@M z2hmYe6#e)FL1&bbr~v7}2TQ|O6`Fii(698t$Z{cl6?vtoY09q};t|xZm9^1uj1y$t z1jmnZtZXP#6l!{5JPe4kvgvX#HXgV!wm|9lXpXZ#}%Mr8+OXEG6HjwC1Pw#{fu$ zsyl)Y+z+9~KB&wAEh6w>eRlKbhGyOi9ZxZ_iuUu+1B!x(btyVXjog7#TtnM|AfDp?bWhhqeL9#JT>qKvFY!C9!iafws z%P^Bc5ws&On=6EZ?=d)EglYmJz$n6HQ)pRUitmL^Km3IK@R*?V9U18~ZXkk?e3ZM=@6H2`XYWp*x zG;5$5o(ZKzsEBSIB8Y`IR=cx_%@ac#V01?0EtgLXSN~xZ*e}op)vDdVCc96I{J*|Sen$(^Dx7Y{(yXN#!tY-D} zrPN}Tff1v!@akJqvsy*`2B?dmxl$U5j+j>=l$xYb6r@03uL0ez(ooYnzkW#85_@)3 z)&vt`Ts8-y#Oen^QGth@Q0M$Qeh<8P!r5(5lbwO1r(b=&|J8xO;Wu9CKYMm?Kvu)q zmJM8t6;NFKT7D}I*^bZ}JnOQqJguw~5+ftwDFJ7L)2*CubD(TZ(GB2_LLZ)M0o501 zV!d_dM_JE&-QP8r?4FX|oWAnoAAK0f)&70+Qd2)Ptyn#o*RmIL14V1^%+VEB+p?>( z;Oflj7F1uj`qH|RxpvvyTrfA!>pnMkK5%;0#hV=~aBS(HG2=E*g zrcyLZr&Q0m9#VTb+=gMv!oS? zPmS;jm-&u08s;)zx!gfX=OBPNJ6hfOFE?Q^_M_K4M`Z2Qe8i-fp(BZ zqlo676qwJc)K}liepu7H)Td@;<^k%4{)Qo>9tBHu5hi~VsF%{G43a+LULC8{(A!o< zZrE@8fBNmK{U1=QLo$3oeWbprx_&@IrHo`<74MOZ5wzv&3hR_q0A1x9+!b6Aw8ZO* zWK=l&3>&CcY+wlPFd=oykkVhJ{O;tDvUf}T)PeB1PJvr7eop~9U2_ZIc8 z;>-dRC`mCPFz8tI5EUxh+3?6nLIC4gHmb5d5gU)iug1vn8sZNE%CZqWi}Fo?N6C`S ziCDGeDPx8T8F)cIwm}8;RoN(+(G-A}i-XCD2bHWEzp_>v8}>(o6PMZGfn+<&iD1^z zo53~7^`vJ%n1*0fH~^J0*qbnwfTBA?-D|yf=J)NtuO$Z!W>&-PLpGJ~8AswgCsWE7 z-a2Gm`OcA4F;huB(f`cU_o{BjZ{fQTzixQQ`ih6Zqc&Qjocgg-2bUocd~t19O@=q>jK z^x#o#;cUWgP%Jal!zO0tSjl3Y8ALawzU1p%_B~(lJ)hruqUd`$Z3ji&aRr*2eE4MahTe4>!sVS#x&Un+9?arx+j;>`#U%}D0aPn?z(J{E} zI8|_*x;K{p!En)WA#Z=PWOrs-GVbrxw5_((`U(uzn!crQ!awRQC4=>bW!93pnoVR! z^EDkyhR%}Neq-CaEvDh^rw1G^U;vikpe?8D&#y1hSf4rr*~*7wr` zZr$%R#sP==cXk?{p{h*;Ak>ybUo#+ZAl2F_T7VeAs?AmO73f!LqZ?1PUnl7{a@`_T zeGMyBN_C8WRa-}@x>|t(O{?EXBP*j7{eoH&^rtbT3ZwSi1)9?cx~r;(SLXvwUa?ht z3qWk3omcG;T!q z`^dUd?RQG_0F_Z){BPi`OL9fJ)&asNShNanL4qLyCb?J}y6`EthlP|XE0}aT5x;)1k>SBmxU5UK7t^qz_ zAExS_H-TT-z&58ErA-mE;_HecHy|@~)thQcHA_wZ&zZMNP0v2_Akpd+BQe*^BAQB4 zq-I581fA3@HAqdwGu$s8sY2)^riyoqI9P3Okr+v(aOJ7d7o_G0I`QiY+l*9wMjwNH zskT(B)EYtKxvo^kwmiFU9OO)BmfALiAhSywk0yOMn(r1*D9c7pDYex%+afh7lz{UI zmL@W^+N;!91|=k<3RJbk^{<|>=T@@k&5}pjd~MU3Q@&Z@$uo9C=$JH96mZm$ggZy7 zU2;UuC>X51C1$jGbBLT>ZC7eitJEg7k3is~+yc95EaRVQ%+S{l!kwUjcY$>{uZ&oI zgRFSgQRB7MUZu8bf20Owl?r{TI*?HE2g-=mx71jHM6ZvOjb;Pw+$J@`bMr@RZa845oSuW8kI%;jM0lL|Vv|4V$ z-4%P1`kpex5V_hUw*&64V7`ZJh#{eGhUeZ&gaH|t?iA!BP6el`Ta?xDk&&eCa!}y* zF-Z*4ogCW7$l6eXzrx8{7Sc{!(!y*B$0idZ1ED-dru`7q0}{#_63I0vWE?H?=F!1( za04_7!4VFkdO?;)jU`)2iYg!mL=64_fJK14hq&^$tiwzZ$0i++AVU;L-V72!Q+TyQ z8dm^Q@jgiUTtUjIObW1bo%ZA)wW0 zQU!F?FiZ=RWSB}H3A7SvDQCI>KO%3kd#o=&vUMaL8X$?Lq_rYx_yuzrtAZA$CnxS z$JFGJcnlLSK_Nbr$IH&oUV zj?3zkr;kC^Vuu~8yeKn_l1xshNZDw4PYBx_kVP|juuyaVULs~O}2NzxT5%Husu zZo>Qsz>>&%Fj|i42mz#P00SdB$d3deOY7$+5nRv3L_W+3JUU~tsY+PbCz23XgnS7^ zIAD{qzPvs!8wh&?0>(g)hdi1aSr79EAn!|*fvo1HLb9pM8IXVpSxZiz!n}|;C7Xcq z6sjR(_E&2E33ix<5I3Ywak z6B%tu@64RPulJQa-dW=x?cNn{%d&Sz!MmgA-I;!A#nZU#*;?>y&Fx!AEQX7o<7wkV zjp?sWTsx7eUGjG2jupMT@^#PU?YoyWdrFm_y7|HU=I3tJE$+#CkKEm#e`WZdl)vz1 z-hOdO^TV&ZcHen%=0wR;zwBu*c-o7e&Y4qRJH2Tw#Og^-wYF$y?%R9k2Xh^toJwm} z+}>=>$Mv`B=lA7=VtrrHy(?{4an;Rr-RxR+Z7aC8<+z2TwttI0HPf-o z>?klh7U~w;i_D>dp)qaByi}^GOPfk&Pu|RYGF;l!{_%lZ2XfmNcNI4sN}pU|{L9R~ z0<*8=@MT|GI0=ETmcw+x(wIJx)jqU)GCz3lol^7m1?|GvXZHN|L;1%3j0rH#ym8;& z3}f<4??U^cv9R-SzV}GJX`tX3_{;i-3w9=N?}jPnExGdx`wH6*EDHJVBX@iAO@jr; zV8-;dvv$sR)3@wwFF4!hBMY5H=f1S=VJ(sj^8MCQoBx;lKiU7w0|iSnV5={^Fu44} zP~nB4d(%rVypi3SZ{Pi?I``5Y+x$BP-=2IRlzzEn_vP)apY|-)Ep6>DZSKlwGm(O$ zrR4Cg)U?eX%Uvke>`fnCaW`eJ%)h-bu;kwR6i}(IY3`ky??C#i&Y#=!tAn=>F7_6> zUd(9Hhau*7V`_FPOV7TY9r-wVE1K)Q6~Axaw&3}-?=#KbG~5+q-4o)q2#EYv;B>2zNO%3`&%2}Q}S$@JAL!?e1E~yksd608t!WI zhtB*$JJ0>xk`sTGyq#QZxt+St?9abGT<}}~5KHZ{rKMnLneWQ+Ma#}*%ie-z@8Zry z@$UKjzL%FQCrg(4>>0=ezIW?vEYCq87KX$B8!{~9qWZE^IXWB7wdP*S9bXt;IGK0%FBuO1!S4Cs zB8ju8C3_cpCDOdfsF`6>CRvJ2%8+r>3y$f3xCyEUx)4C-? z{X>g2O_yvn%eGww+phec6Zd-WwdJ21TC$yn*mA}>do5Ep9^iA^>4%QC`TqI7 zysa~>UNN{Tf5GU45;<@MP6`MC-f1+r6Ie+F{-f-?e zF}i!6{$9Yy-*1?`XL~fiZE&9Lql$gTb5?b6zvY}ky=2fr9sXc}iNo?gDpY(M)Md@( zcsvT3$Z7{Mi3EfI@Soy{R(SX|`~j?!2RfMehm8!pldiL<)GNsui5UV{$lt)@Egk=> zF_M74aln5ycr`ZYqw&#Eu=lb#6puzB;{iDsq3oKNhzHU`e1{am0tmPyJr$3n{6T#G zKzvdlX;ax#Zmn1}u;ZBTCh2hE1M`@4!R1L0mE;I1DEN6@dN;HXgzUgcmB# z3UOO7(n4|$#9omUPWgW!cuA5C`@tQ6f1w=U{{cSnZX_Vqzz&*zK)v`Us`V?%_9fN! z73KT`<@%E9`jYDY6UBT-Y^i0p5 z?m-$cN~sd6GAiRBDwUF4w#)LR%21FjNx5tuHPMbTX_ho#66wR7g3qIxDxJ|Y`Wa48_*A#QXZ% zBqgJU?UZ6m#LQ(>$tHRhGe{ZJmULB1XH8qn*asw{wGZgFP4{4@G%0DSTEA&tr`_o{ z)|q`Y!)!ZxW=Kogmh{IRiGAD90u$;tG;fy~Q@6A>X*i`D8XeEl+znWgLNsZ2Vs}S7 zvNyL$<^UGwb;E)5;3bW-WW9)nlh-hK)=-kT2ij6mm8`81X;{Zev1z3Cc63p!9MROU zMz?56^r2(RTGG6s5lh0((MQ1uI#Dqx8;adjS{cmfc232}Hg;`caUFZGxQ@MHWl+&G zmd)BGO@hJbh*r+9apCW2TGl%(S5PEN%PIutPLEd`QId9>bRDhSb$j=zqK;%VgIx@r z(6++v8EPXoQKS>6-#?2}!c0^I-`I<(F4#))I-LvS4&*FNJs@>6F|#Byga4XzTF+<- zk@~S+2(ThqXpMuFkw9+A-FWZ5hK5(V1`XKtSnvrnWqB?+i+u2E5 zbQ_R!-7wsJv!%PcPAhggXSfT_;-(BGj~s`Y!UNKq31b-d$jpq9XFV;=rZ+i&gEn!+ zDFp{@;tZrT?wm;hm6vzUUQC34Lm?o}I}s%VxyLb>8H>KYPaqbWdQ4B+-ANZF+}!Uz z_!n&`{24Gz<9d1Em`~+<{i;yp27qU`qP>DDs(zq|SmZ|d-oOPe9&kdZO`;sZ?Twy1 zX&NeI0V0@}NCjS8VS&bb3&*hRR}rO$Gr~_Zkm6f~6S)j!4zh za(LYlGH6NYy%K6vof=tA8j5AfvPCb9s-`53|M$B`4_sCC0YCuc7n(khznaX!ZlvW5 z(Av6MF(&ONJKN8i!y2J-$gtyTs?$LEsw;YlY@S3h?C5GwP!rz1wxQaZK=%7_?n$V2 zB3KHopAWsZux)!$xE+47EpadR(arleAB=sr?Qki&x!6^z+dL`U9hxN5dl#bh<>=N@ zbn8t2z18>pGbbPRe`MS@9`E_&;(X%5e6(k=fqJjw-qx8@Kk@x6`Zv+XJ+tR7&F}tT zK6-g!lTW)1E^r)e-(u(Dm1jovQ*%kP&4MUD~ima zohO{0z`gR#=L(tkZNLXr_IEGdN!&-MOgq6r4ftV`xSz!EPS(<_5t&72!ZnfgcshPG z8;w@a7jNyf%|2w~91gYC3>`sWzxDOZ`ZDEzwcv$sW!Z%pl%ui?LuFLH1Z7z@ld{ajg*0Psr?w)RNzE{1InI-HsI@3uyv=q_ z==HE{3;nPf+3#<0|0JCMtq>45|E}IA9^u($Seqn1S5pP0_!knYmLV>*8z?KN0T*q- z*@8}(9WUc(%6Y8d{eH?wD1}O<7-a(&V|Ec(+t&vsF{}`^ukQd}^3q}H1Np<&k+`%| z%4K1{sG{RB($@!rOV#Zx9R{2O71gdw4H9Do?zw?!P=%&E#HA`8FK|Dz;fBqep-NUt zA8`3ZT1%TS3TY*y3~Gd}kRG~PxeBYV4?H1nff2bj1g4CQvx99AA`Nv!NDbsBD!vnr`rwrrcSs@>3&niHrzMb*w{ zHOFV0@fs$|lu=}Jwj(HH&~gITfmL|=&78exgB(8Ph|CYP=(STxXUR1=0CaeM0OiqF zW!|mqLI>-wkwNBHy2^PX~ZSE;${EF`l8e2hRtTbYQ_gZ*QkZS*m zD-yqK|Jxr5oL!~*1)n#v%3lZil5oMN`Y1Um_}wREPIeW~hN}3pxNr2}7wP{HkaT%^ zP^tI><6y=wHx6TS!sw%;RX-TMu|{0Cw6Cuy6ogG!n8bkjXMr5*o>8)1 zp$0v0D?y^P%%Gjxr5M?i;`U?yIXpfXQP!=mFPFtoFvipot*{8nPg3ZvAOxg@ss1v< z&0KYOv#ukIh!Xws8!AhL8VtB5epOGy6$vQWETS7^FKQi;>4Fnv5unaJj>rm*-_SCS zA1=QWhU;(ZgE=#2ficz6H=_6Fm;+2-)4&`}cEQM>%UZw}5Fu*`=7{V_miQqRhAYrW z1UZdWyo;apx5&o1XHjlV!{qs5fWj-R6BRflx zopX`)hq;gP_w$b%Kkk~}b)vlMTxr+2xm_1#BNt|c3$AUi>LK<(ft6|cJk#_7Z}EAa z1uvoDiyng~p>Txxm8`e-9rysN$*gq%wsFz@4nmVEOrRhZ0*5f|?X}Rfsy#1=o~Dh# zf~{oThpJ9CI#*JGwq3#HUdhsXYy6xZ)V*1jydKcH3Z9MCfTvLRXITQjs~P0I9Rh_! zoCM$-2pRY`rwZTojfK&3C952+?mL1O6k3+0LfC%Yt18rhuSC4nEY5fPt2j$)HCS3a zOL%qS=*LHLI>dx>nFCb)z4y_O=eWe{oERf0374N#2sWW|PbmB{HD(T(Q~OF2Cwhi@17}@E zPIjZMQx9hrj0T>jH*qYNLDj3XJB$p~EQ6l)ZN9+75PZf|!Ellf~-JsQbw*`@xj zoj!)SK)lY;DZ`6rOit1YzpV_CT_`b!*ok03gvn_46nTeMGpa_t11D0sFB$F>qj45B zNfceZ>A~dRS89(bkkd1TCqg-_DLS6k+VdJllcK`k1@#70EyUQ(alv>{S ztmVy8XiG6NdF{!D%`@$fHq34KT2U-s|1|X4Ge5U|KkzErFnRnNUmHKR5L;i4wUlBl znzFFHBy7Luzh~V0qj}-b({;_o$U@UsrfX*|&#q0N9Fxkij#8}S(ayQp zTgA{qsIeSsDTP{QZvN!du@(Md716#35yyP0e+=H=P!$U?d?k7K0<3^}gSRB0gun^oS729T zLEwt`MS;sg;ETRePG9g*_B0CY9P{7aC2+Uj5Wro43&QBWu56Z=BRXub=3;;c5%_BZ zg4Sj-nak$e7(}>UGr%E1D0U+d(us#_t3}dH7D1$qVAgHLa3z^Kn{HH)mo578s@R5^%iN#0HH&EYdILXe2-!II z>$-+JHz#gR@0qzaU)Nc#d%INk_FUbO@skVDT8MGe)^gL1Qqzvln%auB)LOq+JTfUv zDhtuoll@bvyQ#kzn%Vo~1K&UJ!$YODyB@t+YCKqq9-KJ-+z(p}0e`Li83cUw_-Q6O zVCP@QtYjlH08@Z4G;{nQLh%?QYh3=?=>-y%dp0Z;e;2>?QY@KacvSUZK97q0&BAZ=tmm(>O&MhjR*T2%L+4=B(CEl?N9p;YL=%vR_Xxcj4N`-d*_q50?^ zeY49IC7Hg%0Uby)vomkrym{}<%Zg=a->;8g{RURY8Cj|WSh0EnEEifTY| zj2@1}5`z;W2Mj{=ne%@0C4pn&}@Vmio?iAkmk8$f){Bf&MHJK z7mTnlO&dMPfuJ08Oo(ALZ$vQ15+DxSMvsL#V8)|qL2f$EaUe9qGG}{f#0;qY<}Z=0 ze7I}@u@fSLRxb3WTx=vLz9Eh#V_X|Z5frb<`aqBghB=uUPJ}0zl;wIX!83u;I41^V zLu?|L;3GoHbiGsy$hssK5>qYLcMmcG7Z`yB#RmMPLK1l1ACa$;D;8Ao8O@QPz*flT2_} z*74wgvMwql;xdJ-FZf6>03(KFJS6L5Tzp6j3rHz6O@Di)>+ryuq7}^#1V;riF%m#- zCk%8UV_Z7Ze)>TBtBEm=KMRKL6Jef#@Bw7>0Ty;lQ1mA!WDA-&pt?U_2mFz~gdWH; z*s{r%Wv9Czo7}kz%dYy_#+k;PIDg^(<$ISGI{)(Zf@|+0zU11U)h|1$r&5!t+}680 z=XTE5-SHP3zW<>xv*(G){h7)AtzZRyKk2k~ZYMw9PIa~!HaGT=xCNAI z>c*}$V%PwvE{(rKl$|0=gm4yr+bCh(*oP%r%~%3{Ym0;j!61E@4l9pgiMO#z^gKR6 zPUyf~8-zk$cK~_naj5(S-UF7Mh2hs~ktmZybj@>1vqK@_ng~C|XK@_6Rj&hY_#1Kz ze}MPG2Bk(AF&GBB68;xtd{{Psc}0$om&}^sfj1_bI)=D7H=g8=rkVo5MEnL9M_bz8 z5laM_m~hm;*1r|Vg)NYM{U6xZ87y!2Kd71S%#KZ^C)2qLPaMq;YQ7QBI{s`+6JB|7 zN1=D)(qJo8SQ@x|sde4baI`cZ;UiOqK@e_O@TcrzVwj5ucrFx)bG$!gg_4N{V$1}5 zAsv_nk>N#QEFy*<5Iho|_d+J1%|X+O2KXkZ|Dm;SM}xpMFm>tXOG~d_e6%mSZ>nRm zBj0fHiRBbDsH=Z#OCv7p1{t1{4Wselcw#K>BV=o!>wNF!lf4(uobTsz-uqv%-Ml%keR4e(i( zP?4{KBrN#>nWkW+Xa^{%JSMG|^igyuZAvU48toh&#|7B#5iD&GkxMB_(rYf8Hj2oY zYO7Iel_=JLkdTP1s3d89pHb;UTTJcKN0G~IO4`xWjZK+?L>LPa*J|K=FiECL+GI_v zS(ErZ;#Qpw=+kRB3goe%IVD5Iu~ycmjRi_OqGWAQbD$b%ZigmXbf;D;m(StpWL+9h zO)C}WuG%oSG9#KBxe7k& zl3a)vT$V=3T8W$Fio*T3Ik8P;U#QsM%+i`qRqWr&Zu`Cc+u0Vazr5~S{W|Q{f5T;7 z-j8Oj7rbhRvTx9C*okWEsMg{S=-aia&$^Do?u~t{51@kaS!lr$Y^w+P@g8MHs(gA% zq}qb;pty!uTQL40V9zzAT1J#*i1WvvqAdOa5hPy6SRzgQi0CVoWB6DJd4dE8$SDaV zXiO8VP9pALKO;bLCHj*pNdEgcA~1!{sxh>>^}KZm^-dGFjN073w$w{~qaz~>#1b5Y zS6n&j^3!w|ia%IrrgtM~+eVLwkr9Ybgh-0>`QcWkIG#`N{4S8a*GDQPk786=he%S@ zqj3aRnL&t!RLFXm=FwRI*Lh%s6QN6F0uarV;z8M522e7^iW5mr#*fK*r6$O=f)FUj zHh~))j{+ANlg;H;;Keft%*aL|5faA$Mx=;8O)H|wE?@xR(^!HZM&ZtAj60Rm`ACa;taaN~)jEf9~#gtR|WC4bhTE0ZLx)L796s#Z?{}N#>pG7t+Ux6f_ zh^B0S732ON4&P8dW#u!VeU^F?qvHBr?K5QNsW!nbiCR^44 zHvp*z|y{6@J9kbqh8<`J{acF+4C$;^+e*zdqV|$K8OY&S5`3U3Pv{Tzo@Ry zGRqFv)WqaO-u*`Y+V#hdz*7=4nP*-sq1A0IIB$VLvMI?30>phB@AJq+0&po{XHhns z?>mV$iA+KWscsG09YiAsPcn@)}gxg-iIWHOmZ z%DPxY0P{hNsi0a|Fgz`AtgOGm#6~%RM-Y*BDLe!YOdJt8u&0rWvkHy}@XWF@2~RJw z0p2)bOpxB~Ro4Z(WZOiIcteCS- z36T8@+X{~M&m0F9t1_pTEskmZyB(SCAi1Z~sO6-|a1UU-`w}#S6I$cQ4Idnm@GA^?Bn95BDy4Un$h~0CmI7-lBW!UGJPX zzw-wV`GWgIiAAxlL1n!{kd$rN({#6Wu66$E{egP}i>(jY!j7(jr~8TL^pfZFqq_Ws zzJli>1TwErI#iXI-Z_>Vzbnm2`L<(^n~xVg+w$ALx8ymb()^~*eVfaiT;9@j=V0cQ z<%TVHsLabnXKm(8(OEZZow5GZKB>#%*?mQeb?Wfs;py;WOLMWon>|ys))bwtS!#xw z9=dJ+%-J%(ElU-vs%KkgTJw!33RPWM!xxsS_l4ZS+o|~r@4Z>Hx~9%do_Rl*+cA&N z?Y=$o*xFKb)aABLN=2t<);?qZK$t(c;JWwHo%Ca8`zk@yyqN8RML4UcnRmvE&iYx) zj3w7~w`Z} zwfZXzM-J=u=KAh5{Pq67J@nwvlRZb4_8fWGUD(sPkqS2m6OQ_c_w|k2NoyGh&~(O=s)LPqww5ts+0Q;!yvzibA=^ zbPgjw-ClL}2>Izj9Mz9dkpI$XI#)~nvId8I!xeynNG06uhg%Olu~dA2SbJLGZSYh` zAaHvJp8L?Vp<4eL2|!2}1x}k2OQCn4id^+WYa!&bU{8hotO1496-Tdyi)liFw+smH zD$j3*`H5%wBS03%SxXwNV%>@=&b2Q5gLT`VBas6h?@N~=P&p<14knQ>kFgK)xq20( zqyU!`ahR6zy&LR+E(zr--Vc?vs~CQUSbr5i^Tg%Sp162qh6gx*8492JvLoaCv-f4Q zprFHvKny@tg-de;#tZe3sSi6fN0xRRp2y$s%eiJRL-m_gB)E>hOE z(!)5Q4f_9mW&918Xl-5WAHS{yj2F=JhPtjD9Y5z5U4Io^*m^(obM6}+J$! zx(eP84GJbqnduUmZKHVT|9_XBuKgl_;_G)UDC&G2}fh{Sz5ei)5Zu2J+fB^y)_ zDJU<2=#o@#M(cVz@M)lo!oTnlWErezu>5%Z=J>SXv7w<@RX2TOrfs?@$IrLSkKJpZ zZ_YdR<&FD_R>#!o$@&C*B>O8(%Oj z?khAO%r|{+G5S%>KQ(;Z@Te_+^^HR3K>ql(6%4>}yl~ll}RIV+GssysooIS+$XQx4&TDo7cUt fVxpWx#=7bx@zzzV97&-72f5N6t$v7Qj#VAdmJm4=t%2W29h*z?Z&blyLMz(vWwPrv0QP7;#$jH zW@c$xELF2V2MRS{^->G45ALZ+9iVA1&8f((KnoO#l{CyYK!K*0_M}!V3gp!HW=V>& z-J*vsz}eaPc{B5VzV8ixyKS2)!Sjp1mTvsIPm(^vkJb|kD)Po6&E9X2_Nc zp=%)->x!*hQ_S$Suo<})F{9U_X58%he)w9Kxy_8hJ7)Hp-SF-<6Q&Ap)l8Z_@a|bq z(kbINiX>g23&e3*!LTiEc$VvsoXf}?mSeqP+#=_Wk&Ev1Yg|*+GdC$)Xf+VaAzq%6 zvSpjZ;>2|*adX5VdBZVtB^y5+qhL96BB10 zXWKN3<6c$O!^E)0J=eA8Esq=_rj^T;U?L|ZlL96r@s=wLnl-!SJ$qYJw-uxTH+mu1{5p z{_W*agj-^q+PIGEc&^w;)^J?MLO*@_T*r%smp=gq0OgcmKcJW~l~L~6H&I&2F+nF_ z+TbLp9p;wi@-RCulNRqs8YrHAH z6M6)r@i{Gh^N=Jh%2nw`SJ2W~R^>-cNnF0Oy(*cC7islx_Vq}+q^g_?Wr5gcq_DIQ zPDd(V>44^fUDR^fqoDdNxK74TKuC^`v?07XZe)y%Y#lq22DH7B4*3cMsNcmf;5nU2 zho+{|VLu9R;Zf7?)h}L}eoddbdiL!3ubr9k6`MMKj5;PSTV6gLV%@NJ7+uN%g>Y01 zA)lY0eD!+U7y>P?XG`333t(=O^6QfbpjXe2pF1{w*)3Cc5za8<8QJ;k`D1vV>#(DO zQPheHew-I=%hLn-ECup4`07otgKBKkhqOk2ZRG?;c+|z7qYYXMdxA zaQWL04zA?xzp%1rwg1pt(bazKaa>!ENc~@u*P~K=a05h0TMUlI5(fW0RPITZFd-Pz z0Aq%#p&M-nz?s2gwvEYBFrw)ItzLYap$>B&s)jnvz1fH6UR0_buCOWDRG=?hRjP7T zISvv+OOUg8Ru)G)l$wy+ zp|)TJlV&Xwa>P7^H5e$Rl?wq?!eGo9J$Ifu1QJfK>;_6=h%kVZf&Qj^S@UHb;!_1h zGfxB|>xCg~8x$DDMcc^Ibj05lq$b?dad&>i0@}Uh%kzGB+nRnD^9hdVgIgpli8A_T zOOVB4egx#Li)m~xbi4#V{yiva(t4k?cYp27AGYnP%Z)_xZgr`;x<~skapc`Me_8o? z<^A-BFJElL2O0z0R|iKP^sjvV{=SvO>cD8dyOG%OpnrAmk+q#KuI@a#nwY4E8@q?= zk-GIkY)4~oKh(^np2p}=tlYJi?3H5=m9^gC$Fb+v6H@Q)^`sO})h;r)8f=YSJcNkd z-$4cL1~R~}%#N9%McIRv;DK0vUc|EWtVpePMk)Yv2Q0UHOpH?@U6vdfV~WXlj(JEG z;(-;0{~s&JQdPh{mT6_`%2_aZLZ7)Zeg3k3dHRgNVm1g%26i9NuTGslGd(kTY5GjN zA9yc}Vd8gXA#a)kFjpYmQ+gE%26E)T7 z8>)BxvH!XHWFygk_x95575-7;U}I?agUN@={c|hlmv1kJ8@u)`cYTuFzTPbj9Q=Dk z>goGeuaq3FD~+DS-IGfvSB?o}9(tgxOx};JbT6N*pKioc&|A9zbCSdBF)5y`T?nk( zCRx}>8=3zEADvWOl^!)w84z0j?^Z>8{Z6Q=%t(OLZOR3K&Bp|6Zeb?i?X#tYTkGAD z`5~~!C+P@KNN#A?}HY?R|D_LmC+uYrP3{-z&x(Q7P3wY;5kbxW*E)@VJMh7GZ zfhquT454ERW@uAv02XB1p{PXkZ%vFXrX#|(zA{HWHU#Z{h*4jGdmDg7420__obAM( zFjjpfZ+U)_iZGXb$OBdKQI0f-(3&=uS$~b#S}I@M6Tn zFcA-!s_9~)FS^)Ojn<;oNHtUyBz$L22T(*IgKIkOVyqgg^;MN;(etKES-Nvv$ck0F zDzKg)>oWi>;HbF}1ix&236`H0_c;oN7rxMOH4;~{wk=xhjhNr$>1pbfn1dud+Wcdq zLQS2UV{j8|UjGC|U^+<9d&b2QMT2t+r~zCQ+#OYOds46OK=G-;jI-Z7gGYbRoIjXd7ud3S)gRhPO%fR72G$#XTWWjAnrKkzqo>U0tW4F>zN^G z-hdxpgaU{&DW&?seRuDzoo@{8y;t~Q;YaS0T36~8Gg!4~O51tfe$ebW`;cjrjJD<9j}c?^&7oFn$2pc0V9({p|8rmo7by z?OE@ah7PR{Nb$sScm2d_c&CVxR=%zP>O1K7^S9LvC0>`7e&M=hN2o`ZH_G3Y?z;s^Z76O`tRfNXg zmc(uYZN`oVnm)n42*WWH`6(#Y6oK)2RO?;eq4bApiFH-l j@w&V!4Qxas^590YPfl$d-X0Ww011*H3GrcRC5jTENR(~LdRekfi4sMd0vVBVD%LK%!Y)ZrAOPHA*K+C+(=e>dfRvrtM4tLy6$6I^#}~nWVp{P;DN6 z+V9+5fD}We?X;`W#oc?)z4zR6&pG!yXZa6RRXz@n@bilkKW^r@pU_MBaat9A<=;@b z$tm0jr|^o9lkq;4#k;tPP#_i=yND8)IB2}o)b7#oTw{Ly85fQ zZ}ZmMh)?m1_yw+)Q@mF=#is^ti*`>HMpr5R3S~8;3V^Cg4OXaXl(15L#W7N=1g~%- zb&RG4G_|({d!7($t3z9;(pImA)Ow}BBx78aPNU_td`UXhB@L#hRZZ$opXru{3^{(G^B1&Mgg5d8@Q##jlVkZ*nR(!sA2) zg~yJ?86hH??(Wpg@JvRH+ncV8hA{aR2OBoSM}+9fTUgrx z-bD+)mcst^&diMIip5fDES9ad=kB!LYeBA)e#~;EVEqazTr}b^J?NDuL2vqEvB|VD zm85zg7JG9_PFg)$HE1=6&0RxQG<%^MsbB(5qM{VMMK>tzw0q^*b2h0JdMsT zKPC{Uq;rlNg{ofhLM=rI$f-C`y5_h`{D^yqi@40DlS%bb;(Ssa$k0K_$-Y!(${_K{ zry%O?cwEgGIy-Y&JD)H#S(}mGc!Q)9i@x!Olpu1ee^g7S3^k=lvZCmcY)INv%1BJA zoj#w0=XiQ@bmnZIbWu(w6xmQgB4xDnWX6!>F*%Xa4XH0Wcy>58d~Wc>(DUeJ+8|Aj zr*&1*($i8rjR~cM&Zbk;ctTMXFu<5Q4+0%}I474kE@gNZb?srCZS78 zA`Y(9Oj=_sgT;iwCX0h}VnI?bs@UHJbw-l0k#Q8dbRIfMysFw{Le~=)Rmp&ODyEQ5 zVJitkQqro$8I)eSkOIH*`Sg_0X)R!%6Ap>vsE+^qN)DAj<8N{XwX$bk%yD0ZLfy9C zc`nE2xL8G#bHXKFhto-LXsyhd<0icJfHgm8b**`|tGCMr#LaNG_-MpwI`t{+jg*$m z9E$3o{h1CNl5D!BGIUrGhvvrw8hj(CwV*J)u~v}AvlS$!#XDuae!drIR6V(cq zV=c24bUJD6uV=Xjo~n6w!L#X}r@2(WY0h!gi=VH(yl{+FJ%MZfdH?mEn|(L>e&pHA zjNvikCh;(VzKY6CCb9|C);||vYY@u%X1wQk;(7=|ov4UafW8C8xW|Hx!;1+7p6 zPIKmPJzs!iHGW1gDzqCNvI5I1Ux}Cb9IpvE96J=kzW&(?c0pHsnRbHQ^tz{Py_eUn zN!+b_6tQfLJ>2V~9CumFiT|hd963i$p#AC{ImZN9?7ByB=7b5lYt}uAQ*o7dq_~+0 zgQCox&R-q;Z>1-e$H(EHhBya@%h;3cIW1=-8Lm97!6DPNA<+dW!QrOSuBmbo&N`{? zmW*^#)qoaKW0J0BWK@AQ6fH4obefJRylm8T(sVQVb-#4tjTWTWI{Jg53VhJIssyrre~4~18&hl z-KK+0;hL6lwJD}IjvIm9UZyXV)+TY4WmScl1zmBHWnWb>P1V!Mi)xJc)eeGD@^kuM zqnPFX)gQdRZ(;w}UMK}a5B%Z#{`P{ueQ|8bDEjx!_I&F0qkYa_s;a&A+Wcz^+ZV-R z)%N>Uy9!miidDO3Pd;eadcUEo(9m_K_s0!K=Dekb&G#F26dHCc8cRdPhG*_K94Is# zC^j6L^OgJ!3yq)nw=5pM-?qQdw*QWP_tn39{cm6YVOM_e^TnfQ@i#6qz3k>1k1l(-K#k>YD&~W2kkT_#n}uV!$&GO^8E#@N z1qkRVn*tO($#JiN`mz8u7bY-z{Zpph&;*19(0&z?bPjQCoI&`QJqBGb>=_ld!=)pta$a_v9MzsL(Z|*ysTvg+`t1+C_-tAP~SVINLFJjwI#D^NRdJwi!2YGKu)7^GGrs zmy`Mns91gdm_aA4p5+#d#qMuTGwZ{3DWT02(y3w7qZ^u<8Z*Yn(1d6CMZ1Evcj7(! z+y-&kVZzTz1?bp-#|64&&bVCzJid^~WZ-urqSl4c+CD0}QABD?C)t~ z`AJ_wC&(_R;8z`lt4%SHGJqw?dz!w!l%ifbp=oK&^t~!4r&O!%Li7h$i%4rdv%JT0+&eBW0FX zcc4>n#k?{rEZ6KQ%UwH*-WTy-KLW2c%YEjrn(Zkyw%$zMNPZ(V@0%0n&X$68*JkEt z-g)bOu)Pq3kG>b|{NBXJ4gcu4xBql0P+#)bE;JPUn^9dBL~rYZ`tXAIq4!9$wOmO^J6Npg75trf^ zaVyRdkK!8fs=g7QDvtOSuj0AFj|67iP>h;WT`cX;4?`WIoPm4>ORyTue{w2mB=)e3 z1au3Q&0@7RDGNESrS)>KH?1bd#$gh4og-TFk+z(|HJ?OSN2v~DTvMrpN?HxTG6)Fm znJG-vV2PmFsaJ9#Kso(vWi}hsT$)C3JO-&}Mc6pYJ%JxN zvByZKlNS=kZs|gLdx3(53hdkHvadlq(;9M^&^k2yygDvl zOr)nYsRLOsLrc%-wn9LSD~cNFL=Mf39H9)WGu?DNMj~U1>Cw?_@(fc3TfLY;X5@s1 zl!|Iht7^*DqmF0yA3PXgXU9}S>cHEo#^{|=R0YbG#=ByAA~_Yqt$IE-rX`e4T^>ai zF@@Vi?}Xf(>>$mWiY#K5EKDC1Jei77;s<0NGJ^!-5z~{lvtFiaJfT3V5#bYzGsW)c zxm&y`#*_Nm3*5gZLzQh^?}R$n^rkS2{)Z@LIXM5OmIs@j`CeP8sm*MEskEhiSrDX` z_=isCalUSubGC<;1Kie~OU{KavQAzaLZ{;y^f|VK9@bVJtX|=&nu2h~;iiXMYioSV zT&>q<0RurhM;yC`hR^gxd$9rQ)kKe~${~SZ4jJ`ma>b`ImSDHex+F$8FnwfO?%e(6j=3 z7~+gdNWD>l3JEU+sX6i6;%$3<7B#pC&2E(DDn|{MJqB4Qdo~Pl9cxi(&Xw~JFA--n zyK_Udge)SH82O+SkeaGwYtUx7L>p;R`k08y3P$y8c%$ij?Rvzm^?^HvJWMakgHv)C zd2&jL#{j0{7m#~XFBuwn7{JHWXgcDx5^rzNNSTWBR18uvjKXwVmxHEVpr$ub=#&d& zn1>36d)Rqq8-GV(*~(v`F&Pw;eX5x|`k<~g|J2E1U2i_n`_sDSh5g04ZFBBVgN>#7 zrhLnue5f-Y==^DO+x_Ofh337bK<)Jd`K`~~op>Y)Ex!5SocnqsRKBM1+FSE)y>ofa z@xbH1=9%~8Yd^O%e9!Z2sV2NoKljDFoyd!bQRdoM5vf|2OwzW4FVoL5jgzV|o<>4| zS^{TI44*v{wQ|Lkw`lk26GLWj;OsEVBM%Jp5BCiYnVyqpqsNE)2BN0tc=zy$-hngc zOn=Xbq2p)z2C11zXMHaI3s^>@;tl-QUqV5-D8JLQ+|0RZ9}0C&--?LZGS!wlxcZ&5 z{iVPjH0e-_LWDDD(4nvu58LUW~)5*Rlu(tM$`Ws8>wDkBN3mB z5Mqw_fr>n4@KhH8qP~=lj~~dMWKW!7xo!qyEPS(D0!%A;ZH}vQ4OVu?t>(B!`2_yK8STG#|VhE;e`Pn~pDw z!c*{@LX97V*I$mRCNKoSm4+YF4M;+wdn~yOr)TZ>Gwcqz`U)h-Rh-^sAt%hba{PO^ z{yz{XUA*c9Cu}E%St?I%HRz^{pS#jgsVEf+tvV+vB1Qk09f=1;u<9R(v<~ulmEI2o z#f3MPy;D3{@U3{?b0|Kv`R(z651dLB-mC37;AS^mJy>?!npdd-eb63X`+=C^mAWjr zQ9?@nM(!BY4Mbg@;XMv~H7enaXq8P@IM6o{eNQC@D37$hbSTX<1E^YvN)wb;aPSo4 zVDl4Z-NI&-e($VX8DHC;xQDj&dl+*g9kj8uwwF29!cWM$wuAreG$XBArpC9Ojpy4@ zrqFggVZNPDf0IWBsuqcB?k~k@o_+Bl3<&6pdtT^#+T_b0h^=IT_sE!bY5xY?=mauXb~H=D~U;@fKvDZkPQGY;bXdvEkbYH_M7RWkdv^K zn#=~M&UAuEycv9=M5;{HvVlsKI(T2i#ga}M0bA{hR8Z#4bW9+i)m}z5;vH~V0z!U|UBI2T7Tm%N`uA{g;7cbmG3ScGopvUs6qc@ESZvAH16J+L z2lhS)He7pq{_O>&7;IxC*AC7fyxx1y-&zW{ES$WNyPjG+hD?9&N0%4V`TG6&(7}A* z;Ahp%3;Pyb?;cqIG>eqVZ%c;?EzjMF=bI1TZU5Nu@wx8@^DhqO2mc@+IA_rW3V?IX z7_+yutMfY@w>s_|y0`1-^}6dl3r#l$@}cdE-G$(ekDq-g@>RV&Tjj$;?;ctlUh4Sh zkBTk(^3BiX1N&*Nt`85sd$7=U>`u?!@L!!SwjC?B9Q&lD?_(_faz5}%X~(Xm1K*k~ z1h?JYOq2Do$?QSB7}Wa7zLUka-eODdCoP}<;r@K{nS5Y~F=;Ot`6yOwc`o0)KOZ=- z9P-ur=A6ro_%y-f(|^yu6*%--|9t<#ktN5{YlV)(`JK<_n~vNI97V>95xsKHFRdZ^ zj`x=L&h|TR6`nm_?C8nwJdtlYc`wjwe`Q^0FV;x;;I_PH8v_c_NMIvms?nEwhE237 zs$Hbw5*0I4&@E-DmXB(yf9R2;S1P`V|M~$GSGZ3d1D`o-9txtM+0KY@?g)QujzhD`i zGNv^A1&apY2auFZkBtEuT43$VXtMBgnrdWBrWXmXeh}g6pg_iw1@MnnUL7={_dfn*y--UK?Jr#W;1K-=^GmC?>-kJtY7I{2i{H35z95a zpnVGy6Jpc9jsgiwp8u44{%72_pK#Sb;X*&-q<`V+KXZj=$L_nr1y?w~>2pQb?pd+q zZ=Ll#atVCHT>KG-pNBrqT|F27ljN-U$ko8tFFGD^_*r>6z}Nh8^JYHut7H5De#=7+ s#jgy$oo{={p;$h{b4`&_c*jGh7#8vz1^P| curve A' = A @ V, plus bias delta B @ (A @ c) + curve -> dense A' = A @ pinv(V), plus bias delta -B @ (A' @ c) + +The reverse direction needs the pseudo-inverse rather than the transpose: the +least-squares ``V`` spans the right subspace but its columns are not +orthonormal, so ``V.T`` would not satisfy ``M @ V == I``. + +The bias delta is emitted as ``.diff_b``; ``comfy.lora.load_lora`` turns that +into a ``("diff",)`` patch on the sibling ``.bias``. Without it the port is +nearly worthless. + +``V`` and ``c`` are recovered by least squares of ``[1 | table]`` against the +silu grid. The fit must use the *target model's own* table: comfy's tables are +mean-centered while some third-party bakes are not, so tables from different +checkpoints are not interchangeable even though each is self-consistent with +its own weights. +""" + +from __future__ import annotations + +import logging +import os + +import torch + +LOG = logging.getLogger(__name__) + +GRID_FILENAME = "h3_silu_temb_grid.safetensors" +_GRID_TENSOR = "silu_t_emb_grid" + +_grid_cache: dict[str, torch.Tensor] = {} +_basis_cache: dict[tuple, tuple[torch.Tensor, torch.Tensor]] = {} + + +def find_silu_grid(extra_dirs=()) -> str: + """Locate ``h3_silu_temb_grid.safetensors`` in the usual places.""" + candidates = list(extra_dirs) + try: + import folder_paths + candidates.append(os.path.join(folder_paths.models_dir, "h3_adaln")) + candidates.append(folder_paths.get_folder_paths("loras")[0]) + candidates.append(os.path.join(folder_paths.models_dir, "diffusion_models")) + candidates.append(folder_paths.get_folder_paths("custom_nodes")[0] + if "custom_nodes" in getattr(folder_paths, "folder_names_and_paths", {}) + else os.path.join(os.path.dirname(folder_paths.models_dir), "custom_nodes")) + except Exception: + pass + for base in candidates: + if not base or not os.path.isdir(base): + continue + direct = os.path.join(base, GRID_FILENAME) + if os.path.isfile(direct): + return direct + # one level down covers custom_nodes//h3_silu_temb_grid.safetensors + try: + for entry in os.listdir(base): + nested = os.path.join(base, entry, GRID_FILENAME) + if os.path.isfile(nested): + return nested + except OSError: + continue + return "" + + +def load_silu_grid(path: str) -> torch.Tensor: + """Load the ``[grid, 2688]`` silu(t_emb) grid, cached by path.""" + cached = _grid_cache.get(path) + if cached is not None: + return cached + from safetensors.torch import load_file + sd = load_file(path) + if _GRID_TENSOR not in sd: + raise ValueError(f"{path} does not contain '{_GRID_TENSOR}'") + grid = sd[_GRID_TENSOR].to(torch.float32) + _grid_cache[path] = grid + return grid + + +def fit_basis(table: torch.Tensor, grid: torch.Tensor): + """Least-squares fit of ``grid ~= 1 c^T + table V^T``. + + ``table`` is the target model's ``adaln_t_table`` ``[G, k]``; ``grid`` is + ``[G, 2688]``. Returns ``(V [2688, k], c [2688])``. + """ + if table.shape[0] != grid.shape[0]: + raise ValueError( + f"adaLN table grid ({table.shape[0]}) and silu grid ({grid.shape[0]}) " + "have different resolutions; they must come from the same bake" + ) + key = (id(table), table.shape, float(table.float().sum()), id(grid)) + cached = _basis_cache.get(key) + if cached is not None: + return cached + + t = table.to(torch.float64) + s = grid.to(torch.float64) + design = torch.cat([torch.ones(t.shape[0], 1, dtype=torch.float64), t], dim=1) # [G, 1+k] + solution = torch.linalg.lstsq(design, s).solution # [1+k, 2688] + c = solution[0].contiguous() # [2688] + v = solution[1:].transpose(0, 1).contiguous() # [2688, k] + + residual = float((design @ solution - s).norm() / s.norm().clamp(min=1e-12)) + # The last two curve directions are near-degenerate (sigma_7 ~ sigma_8), so + # they differ between bakes and a grid from a different build only agrees to + # ~2e-3. That is still ~6x below the int8 quantization floor, and vastly + # better than dropping adaLN entirely, so only a much larger residual is + # worth complaining about. + if residual > 2e-2: + LOG.warning( + "H3 PowerLoraStack: adaLN basis fit residual %.2e is high - the silu grid " + "and the checkpoint's adaln_t_table probably come from different bakes", + residual, + ) + else: + LOG.info("H3 PowerLoraStack: adaLN basis fit residual %.2e", residual) + + out = (v.to(torch.float32), c.to(torch.float32), residual) + _basis_cache[key] = out + return out + + +def fit_table_to_table(source_table: torch.Tensor, target_table: torch.Tensor): + """Map a source curve basis onto a target curve basis. + + Used when a LoRA ships its own ``adaln_t_table``: the two bakes span nearly + the same subspace but with different rotations and centering, so a LoRA + trained against one is wrong on the other even though the shapes match. + Fitting ``source ~= a + target M`` needs no silu grid at all. + + Returns ``(M [k_src, k_tgt], a [k_src], residual)``. + """ + src = source_table.to(torch.float64) + tgt = target_table.to(torch.float64) + if src.shape[0] != tgt.shape[0]: + raise ValueError( + f"adaLN tables have different grid resolutions ({src.shape[0]} vs {tgt.shape[0]})" + ) + design = torch.cat([torch.ones(tgt.shape[0], 1, dtype=torch.float64), tgt], dim=1) + solution = torch.linalg.lstsq(design, src).solution # [1+k_tgt, k_src] + residual = float((design @ solution - src).norm() / src.norm().clamp(min=1e-12)) + a = solution[0].contiguous() # [k_src] + m = solution[1:].transpose(0, 1).contiguous() # [k_src, k_tgt] + return m.to(torch.float32), a.to(torch.float32), residual + + +class AdalnContext: + """Everything needed to move adaLN LoRA pairs between the two bases.""" + + def __init__(self, target_dim: int, table=None, grid_path: str = ""): + self.target_dim = int(target_dim) + self.table = table + self.grid_path = grid_path + self.residual = None + self._basis = None + self._failed = False + + @property + def is_curve(self) -> bool: + return self.table is not None + + def basis(self, curve_table=None): + """``(V, c, residual)`` relating the dense space to a curve basis. + + The fit needs whichever side of the conversion is the *curve* side. On + a pruned target that is the checkpoint's own table; on a dense target + converting a curve-trained LoRA it has to be the table the LoRA shipped, + because nothing else records which bake it was trained against. + """ + table = self.table if self.table is not None else curve_table + if table is None: + if not self._failed: + LOG.warning( + "H3 PowerLoraStack: cannot rebase adaLN onto a dense checkpoint - " + "the LoRA does not carry the adaln_t_table of the curve bake it " + "was trained against" + ) + self._failed = True + return None + if self._basis is not None or self._failed: + return self._basis + try: + if not self.grid_path: + raise ValueError( + f"no {GRID_FILENAME} found; put it in models/h3_adaln/ to enable " + "adaLN porting" + ) + grid = load_silu_grid(self.grid_path) + self._basis = fit_basis(table.detach().to("cpu"), grid) + self.residual = self._basis[2] + except Exception as exc: + LOG.warning("H3 PowerLoraStack: adaLN porting unavailable (%s)", exc) + self._failed = True + self._basis = None + return self._basis + + +def port_adaln_pairs(sd: dict, ctx: AdalnContext, source_table=None): + """Rebase every ``adaln_proj.linear`` LoRA pair onto the target's basis. + + ``source_table`` is the LoRA's own ``adaln_t_table`` if it shipped one. + That is the better route when available: two curve bakes can have the same + width but different rotations and centering, so a LoRA whose adaLN width + already matches may still be wrong on this checkpoint, and only the table + reveals it. It also needs no silu grid. + + Mutates nothing; returns ``(new_sd, stats)``. + """ + stats = {"ported": 0, "skipped": 0, "ok": 0, "rebased": 0, "residual": None} + modules: dict[str, dict] = {} + for key in sd: + if ".adaln_proj.linear." not in key: + continue + body, _, suffix = key.rpartition(".adaln_proj.linear.") + modules.setdefault(body + ".adaln_proj.linear", {})[suffix] = key + + if not modules: + return sd, stats + + # curve -> curve via the LoRA's own table, when it carries one + table_map = None + if source_table is not None and ctx.table is not None: + try: + m, a_const, residual = fit_table_to_table(source_table, ctx.table) + table_map = (m, a_const) + stats["residual"] = residual + LOG.info("H3 PowerLoraStack: adaLN table-to-table fit residual %.2e", residual) + except Exception as exc: + LOG.warning("H3 PowerLoraStack: adaLN table rebase unavailable (%s)", exc) + + out = dict(sd) + basis = None + for module, parts in modules.items(): + a_key = parts.get("lora_A.weight") or parts.get("lora_down.weight") + b_key = parts.get("lora_B.weight") or parts.get("lora_up.weight") + if a_key is None or b_key is None: + continue + a = sd[a_key] + if a.ndim != 2: + continue + source_dim = a.shape[1] + b = sd[b_key] + a32 = a.to(torch.float32) + + if table_map is not None and source_dim == table_map[0].shape[0]: + m, a_const = table_map + a_new = a32 @ m # [r, k_tgt] + const = a32 @ a_const # [r] + sign = 1.0 + kind = "rebased" + elif source_dim == ctx.target_dim: + stats["ok"] += 1 + continue + else: + if basis is None: + basis = ctx.basis(curve_table=source_table) + if basis is None: + for key in parts.values(): + out.pop(key, None) + stats["skipped"] += 1 + continue + v, c, residual = basis # V: [2688, k], c: [2688] + stats["residual"] = residual + if source_dim == v.shape[0] and ctx.target_dim == v.shape[1]: + a_new = a32 @ v # [r, k] + const = a32 @ c # [r] + sign = 1.0 + elif source_dim == v.shape[1] and ctx.target_dim == v.shape[0]: + a_new = a32 @ torch.linalg.pinv(v) # [r, 2688] + const = a_new @ c # [r] + sign = -1.0 + else: + LOG.warning( + "H3 PowerLoraStack: adaLN pair %s has width %d, expected %d or %d - skipped", + module, source_dim, v.shape[0], v.shape[1], + ) + for key in parts.values(): + out.pop(key, None) + stats["skipped"] += 1 + continue + kind = "ported" + + # comfy scales the A/B pair by alpha/rank inside the adapter but applies + # a diff_b patch verbatim, so the factor has to be folded in here + alpha_key = parts.get("alpha") + scale = 1.0 + if alpha_key is not None: + try: + alpha = sd[alpha_key] + scale = float(alpha.item() if hasattr(alpha, "item") else alpha) / a_new.shape[0] + except Exception: + scale = 1.0 + bias_delta = sign * scale * (b.to(torch.float32) @ const) + + out.pop(a_key, None) + out.pop(b_key, None) + out[module + ".lora_A.weight"] = a_new.to(a.dtype) + out[module + ".lora_B.weight"] = b + if alpha_key is not None and alpha_key != module + ".alpha": + out.pop(alpha_key, None) + out[module + ".alpha"] = sd[alpha_key] + out[module + ".diff_b"] = bias_delta.to(b.dtype) + stats[kind] += 1 + + return out, stats + + +def read_target(diffusion_model): + """Inspect a loaded H3 DiT for its adaLN input width and curve table.""" + table = getattr(diffusion_model, "adaln_t_table", None) + dim = None + try: + dim = diffusion_model.blocks[0].adaln_proj.linear.weight.shape[1] + except Exception: + if table is not None: + dim = int(table.shape[1]) + return dim, table diff --git a/h3lora/apply.py b/h3lora/apply.py new file mode 100644 index 0000000..975a0a0 --- /dev/null +++ b/h3lora/apply.py @@ -0,0 +1,197 @@ +"""Stack orchestration: normalise, port, partition, apply.""" + +from __future__ import annotations + +import logging +from collections import OrderedDict + +import comfy.lora +import comfy.utils +from comfy.quant_ops import QuantizedTensor +from comfy.weight_adapter import LoRAAdapter + +from . import adaln as adaln_mod +from . import branch as branch_mod +from . import keymap + +LOG = logging.getLogger("h3.powerlorastack") + +# Layers whose forward is bypassed by a fused kernel, per weight layout. A +# runtime branch on these would be silently dropped, so they must merge. +_FORWARD_BYPASSED = {"TensorWiseINT8Layout": ("mlp.fc2",)} + + +class StackReport: + """Human-readable account of what happened, surfaced on the node.""" + + def __init__(self): + self.lines: list[str] = [] + self.merged = 0 + self.branched = 0 + self.skipped = 0 + self.bank_bytes = 0 + + def add(self, line: str): + self.lines.append(line) + + def text(self) -> str: + return "\n".join(self.lines) + + +def _module_weight(model_patcher, module_path: str): + try: + module = model_patcher.get_model_object(module_path) + except Exception: + return None + return getattr(module, "weight", None) + + +def _is_plain_lora(adapter) -> bool: + """True for a bare rank-decomposition with no locon/dora/reshape extras.""" + return ( + isinstance(adapter, LoRAAdapter) + and adapter.weights[3] is None # locon mid + and adapter.weights[4] is None # dora scale + and adapter.weights[5] is None # reshape / pad + and adapter.weights[0].ndim == 2 + ) + + +def _branchable(weight, module_path: str, mode: str) -> bool: + if weight is None or mode == "merge": + return False + if mode == "auto" and not isinstance(weight, QuantizedTensor): + return False # unquantized merges are exact and free at runtime + layout = getattr(weight, "_layout_cls", None) + for suffix in _FORWARD_BYPASSED.get(layout, ()): + if module_path.endswith(suffix): + return False + return True + + +def detect_quantization(model_patcher) -> str: + """Report the DiT's dominant weight format for the node's status line.""" + layouts: dict[str, int] = {} + plain = 0 + try: + diffusion_model = model_patcher.get_model_object("diffusion_model") + except Exception: + return "unknown" + for module in diffusion_model.modules(): + weight = getattr(module, "weight", None) + if weight is None: + continue + if isinstance(weight, QuantizedTensor): + layout = getattr(weight, "_layout_cls", "quantized") + layouts[layout] = layouts.get(layout, 0) + 1 + else: + plain += 1 + if not layouts: + return "unquantized" + parts = [f"{k.replace('TensorCore', '').replace('TensorWise', '').replace('Layout', '')}" + f" x{v}" for k, v in sorted(layouts.items(), key=lambda kv: -kv[1])] + if plain: + parts.append(f"plain x{plain}") + return ", ".join(parts) + + +def apply_stack(model, entries, mode="auto", adaln_mode="auto", grid_path="", + report: StackReport | None = None): + """Apply a list of ``{'path', 'name', 'strength'}`` LoRAs to an H3 model. + + ``mode`` is ``auto`` (branch quantized layers, merge the rest), ``merge`` + (stock behaviour) or ``branch`` (never touch a weight). + """ + report = report or StackReport() + patcher = model.clone() + + model_sd = patcher.model.state_dict() + index = keymap.build_module_index(model_sd.keys()) + key_map = {k[: -len(".weight")]: k for k in model_sd + if k.startswith("diffusion_model.") and k.endswith(".weight")} + + try: + diffusion_model = patcher.get_model_object("diffusion_model") + except Exception as exc: + raise ValueError("H3 Power LoRA Stack requires a MiniMax H3 model") from exc + target_dim, table = adaln_mod.read_target(diffusion_model) + if adaln_mode == "off": + adaln_ctx = None + else: + if not grid_path: + grid_path = adaln_mod.find_silu_grid() + adaln_ctx = adaln_mod.AdalnContext(target_dim, table, grid_path) + + report.add(f"base: {detect_quantization(patcher)}") + report.add(f"adaLN: {'curve' if table is not None else 'dense'} (input dim {target_dim})") + + per_module: "OrderedDict[str, list]" = OrderedDict() + compute_dtype = model.model_dtype() + + for entry in entries: + name = entry.get("name") or entry["path"] + strength = float(entry.get("strength", 1.0)) + lora_sd = comfy.utils.load_torch_file(entry["path"], safe_load=True) + + normalized, unmatched = keymap.normalize(lora_sd, index) + adaln_note = "" + if adaln_ctx is not None and target_dim: + source_table = normalized.pop("adaln_t_table", None) + normalized, stats = adaln_mod.port_adaln_pairs( + normalized, adaln_ctx, source_table=source_table) + if stats["ported"]: + adaln_note = f", adaLN ported x{stats['ported']}" + if stats["rebased"]: + adaln_note += f", adaLN rebased x{stats['rebased']}" + if stats["skipped"]: + adaln_note += f", adaLN dropped x{stats['skipped']}" + if stats["residual"] is not None: + adaln_note += f" (basis fit {stats['residual']:.1e})" + + loaded = comfy.lora.load_lora(normalized, key_map) + + merge: dict = {} + branched_here = 0 + for weight_key, adapter in loaded.items(): + module_path = weight_key[: -len(".weight")] if weight_key.endswith(".weight") else None + if module_path is None or not _is_plain_lora(adapter): + merge[weight_key] = adapter + continue + weight = _module_weight(patcher, module_path) + if not _branchable(weight, module_path, mode): + merge[weight_key] = adapter + continue + up, down = adapter.weights[0], adapter.weights[1] + if up.shape[0] != weight.shape[0] or down.shape[1] != weight.shape[1]: + LOG.warning("H3 PowerLoraStack: shape mismatch on %s, skipped", weight_key) + report.skipped += 1 + continue + alpha = adapter.weights[2] + scale = strength * (float(alpha) / down.shape[0] if alpha is not None else 1.0) + per_module.setdefault(module_path, []).append((up, down, scale)) + branched_here += 1 + + if merge: + patcher.add_patches(merge, strength) + report.merged += len(merge) + report.branched += branched_here + + detail = f"{len(merge)} merged, {branched_here} branched" + if unmatched: + detail += f", {len(unmatched)} unmatched" + report.add(f"{name} @ {strength:g}: {detail}{adaln_note}") + if not merge and not branched_here: + report.add(f" ! {name} matched no layers on this model") + + if per_module: + report.bank_bytes = branch_mod.bank_bytes(per_module, compute_dtype) + # a chain of stack nodes must not overwrite each other's bank + n = 0 + while patcher.get_additional_models_with_key(f"h3_power_lora_bank_{n}"): + n += 1 + tag = f"h3_power_lora_bank_{n}" + branch_mod.attach(patcher, per_module, compute_dtype, tag) + report.add(f"branch bank: {len(per_module)} layers, " + f"{report.bank_bytes / (1024 ** 2):.0f} MB") + + return patcher, report diff --git a/h3lora/branch.py b/h3lora/branch.py new file mode 100644 index 0000000..e76cf44 --- /dev/null +++ b/h3lora/branch.py @@ -0,0 +1,125 @@ +"""Runtime low-rank branches for quantized MiniMax H3 weights. + +Merging a LoRA into a w4a8 / int4 weight destroys it. ComfyUI's stock path is +dequantize -> add delta -> ``requantize_from_float(scale="recalculate")``, and +that round trip is not idempotent: re-fitting the codebook and re-rounding to +int4 injects ~1.5% relative weight noise, while these deltas are 0.01-0.08% of +the weight. The merge therefore replaces the adapter with noise -- measured +end to end it recovers under a third of the LoRA magnitude at cos 0.12-0.14 +against the correct result. + +Keeping the branch separate is exact, keeps the quantized kernel, and costs +~1.5% extra FLOPs at rank 64. + +Stacking is fused: N LoRAs on one Linear become a single pair by concatenating +along the rank axis, + + sum_i s_i * B_i @ A_i @ x == [s_1 B_1 | ... | s_N B_N] @ [A_1; ...; A_N] @ x + +so a ten-LoRA stack still costs one extra matmul pair per layer rather than +ten, and only one set of factors has to be moved to the GPU. +""" + +from __future__ import annotations + +from collections import OrderedDict + +import torch +import torch.nn as nn +import torch.nn.functional as F + +import comfy.model_management + + +class LoraBank(nn.Module): + """Holds every runtime branch's factors as one movable, accountable module.""" + + def __init__(self, pairs: "OrderedDict[str, tuple[torch.Tensor, torch.Tensor]]"): + super().__init__() + self.index: dict[str, int] = {} + for i, (name, (up, down)) in enumerate(pairs.items()): + self.register_parameter(f"up{i}", nn.Parameter(up, requires_grad=False)) + self.register_parameter(f"down{i}", nn.Parameter(down, requires_grad=False)) + self.index[name] = i + + def get(self, name: str): + i = self.index[name] + return getattr(self, f"up{i}"), getattr(self, f"down{i}") + + +class LoraBranch: + """Object patch for a Linear's ``forward``: ``y = W_q(x) + up @ down @ x``. + + Patches ``.forward`` rather than the module itself -- replacing the module + would shift the memory manager's ``named_modules`` keys. Holds the + *patcher* rather than the bare module so comfy's weakref bookkeeping does + not report a leaked model. + """ + + def __init__(self, bank_patcher, name: str, original): + self.bank_patcher = bank_patcher + self.bank: LoraBank = bank_patcher.model + self.name = name + self.original = original + + def __call__(self, input: torch.Tensor, *args, **kwargs): + out = self.original(input, *args, **kwargs) + up, down = self.bank.get(self.name) + h = F.linear(input, comfy.model_management.cast_to_device(down, input.device, input.dtype)) + delta = F.linear(h, comfy.model_management.cast_to_device(up, input.device, input.dtype)) + return out + delta.to(out.dtype) + + +def fuse(contributions, compute_dtype): + """Concatenate per-LoRA ``(up, down, scale)`` triples into one pair. + + The strength/alpha scale is folded into ``up`` so the forward path stays a + bare pair of matmuls. + """ + ups, downs = [], [] + for up, down, scale in contributions: + ups.append((up.to(torch.float32) * float(scale)).to(compute_dtype)) + downs.append(down.to(compute_dtype)) + if len(ups) == 1: + return ups[0], downs[0] + return torch.cat(ups, dim=1).contiguous(), torch.cat(downs, dim=0).contiguous() + + +def attach(model_patcher, per_module, compute_dtype, tag: str): + """Build the bank, register it for VRAM accounting, patch every forward. + + ``per_module`` maps a module path to a list of ``(up, down, scale)``. + Returns the number of branched layers. + """ + if not per_module: + return 0 + + import comfy.model_patcher + + pairs: "OrderedDict[str, tuple[torch.Tensor, torch.Tensor]]" = OrderedDict() + for module_path, contributions in per_module.items(): + pairs[module_path] = fuse(contributions, compute_dtype) + + bank = LoraBank(pairs) + bank_patcher = comfy.model_patcher.ModelPatcher( + bank, + load_device=comfy.model_management.get_torch_device(), + offload_device=comfy.model_management.unet_offload_device(), + ) + model_patcher.set_additional_models(tag, [bank_patcher]) + + for module_path in pairs: + forward_key = f"{module_path}.forward" + original = model_patcher.get_model_object(forward_key) + model_patcher.add_object_patch(forward_key, LoraBranch(bank_patcher, module_path, original)) + return len(pairs) + + +def bank_bytes(per_module, compute_dtype) -> int: + """Approximate VRAM the bank will occupy, for reporting.""" + itemsize = torch.empty((), dtype=compute_dtype).element_size() + total = 0 + for contributions in per_module.values(): + for up, down, _ in contributions: + total += (up.numel() + down.numel()) * itemsize + return total diff --git a/h3lora/detect.py b/h3lora/detect.py new file mode 100644 index 0000000..d47fc81 --- /dev/null +++ b/h3lora/detect.py @@ -0,0 +1,113 @@ +"""Cheap safetensors introspection for MiniMax H3 LoRA files. + +Reads only the header, so scanning a 2.5 GB LoRA costs a few kilobytes of IO. +Used both for the node's diagnostic report and for deciding, before anything is +loaded, whether a file needs adaLN porting. +""" + +from __future__ import annotations + +import json +import os +import re +import struct + +_cache: dict[tuple, dict] = {} + +_CONVENTIONS = ( + ("kohya", re.compile(r"^lora_unet_")), + ("lycoris", re.compile(r"^lycoris_")), + ("peft", re.compile(r"^(base_model\.model\.|transformer\.)")), + ("comfy", re.compile(r"^diffusion_model\.")), +) + + +def read_header(path: str) -> dict: + with open(path, "rb") as f: + length = struct.unpack(" str: + for name, pattern in _CONVENTIONS: + if any(pattern.match(k) for k in keys): + return name + if any(k.startswith(("blocks.", "token_refiner.", "final_layer.")) for k in keys): + return "bare" + return "unknown" + + +def inspect(path: str) -> dict: + """Summarise one LoRA file. Cached on (path, mtime, size).""" + try: + st = os.stat(path) + except OSError as exc: + return {"error": str(exc), "name": os.path.basename(path)} + key = (path, st.st_mtime_ns, st.st_size) + hit = _cache.get(key) + if hit is not None: + return hit + + info: dict = {"name": os.path.basename(path), "path": path, "size": st.st_size} + try: + header = read_header(path) + except Exception as exc: + info["error"] = f"unreadable header: {exc}" + _cache[key] = info + return info + + metadata = header.pop("__metadata__", {}) or {} + keys = list(header) + info["tensors"] = len(keys) + info["convention"] = _convention(keys) + info["dtypes"] = sorted({v.get("dtype", "?") for v in header.values()}) + info["base_model"] = metadata.get("ss_base_model_version", "") + info["trainer"] = "" + try: + info["trainer"] = json.loads(metadata.get("software", "{}")).get("name", "") + except Exception: + pass + if not info["trainer"] and metadata.get("ss_network_module"): + info["trainer"] = metadata["ss_network_module"] + + ranks: set[int] = set() + adaln_dims: set[int] = set() + modules: set[str] = set() + has_alpha = False + for k, v in header.items(): + if k.endswith(".alpha"): + has_alpha = True + continue + shape = v.get("shape") or [] + is_down = k.endswith(("lora_A.weight", "lora_down.weight")) + if is_down and len(shape) == 2: + ranks.add(int(shape[0])) + if "adaln" in k: + adaln_dims.add(int(shape[1])) + for tag in ("adaln_proj", "attn.qkv_proj", "attn_qkv_proj", "attn.out_proj", + "attn_out_proj", "mlp.fc1", "mlp_fc1", "mlp.fc2", "mlp_fc2", + "token_refiner", "final_layer"): + if tag in k: + modules.add(tag.replace("_proj", "").replace(".", "_").replace("attn_qkv", "qkv") + .replace("attn_out", "out")) + + info["ranks"] = sorted(ranks) + info["alpha"] = has_alpha + info["adaln_dims"] = sorted(adaln_dims) + info["modules"] = sorted(modules) + info["passenger"] = sorted( + k for k in keys + if not re.search(r"(lora_A|lora_B|lora_down|lora_up|alpha|diff|dora|hada|lokr|oft)", k) + )[:8] + _cache[key] = info + return info + + +def summary_line(info: dict) -> str: + if info.get("error"): + return f"{info['name']}: {info['error']}" + ranks = "/".join(str(r) for r in info.get("ranks", [])) or "?" + adaln = info.get("adaln_dims") + adaln_s = f", adaLN dim {'/'.join(str(d) for d in adaln)}" if adaln else ", no adaLN" + return (f"{info['name']}: {info.get('tensors', 0)} tensors, {info.get('convention', '?')} " + f"format, rank {ranks}{adaln_s}") diff --git a/h3lora/keymap.py b/h3lora/keymap.py new file mode 100644 index 0000000..44edc89 --- /dev/null +++ b/h3lora/keymap.py @@ -0,0 +1,126 @@ +"""Key normalisation for MiniMax H3 LoRAs. + +Every H3 LoRA in the wild is one of a handful of naming conventions. Observed +across the local collection: + +* ai-toolkit / diffusers ``diffusion_model.blocks.0.attn.qkv_proj.lora_A.weight`` +* same, no prefix ``blocks.0.attn.qkv_proj.lora_A.weight`` +* kohya / musubi ``lora_unet_blocks_0_attn_qkv_proj.lora_down.weight`` + ``.alpha`` +* lycoris ``lycoris_blocks_0_attn_qkv_proj.*`` +* peft ``base_model.model.blocks.0...`` / ``transformer.blocks.0...`` + +``comfy.lora.model_lora_keys_unet`` already covers the ``diffusion_model.``- +prefixed and ``lora_unet_`` forms, but not the bare form, and it gives us no +handle for rewriting adaLN tensors before they are matched. So we normalise +everything ourselves onto the canonical ``diffusion_model.`` module key, +resolving underscore forms through the model's own state dict rather than +guessing where the word boundaries are. +""" + +from __future__ import annotations + +import re + +# suffixes comfy.lora.load_lora understands, longest first so that +# ``.lora_down.weight`` is not truncated to ``.weight`` +_LORA_SUFFIXES = ( + ".lora_down.weight", ".lora_up.weight", + ".lora_A.weight", ".lora_B.weight", + ".lora_A.default.weight", ".lora_B.default.weight", + ".lora_A", ".lora_B", + ".lora_mid.weight", ".dora_scale", + ".diff_b", ".diff", ".set_weight", ".alpha", + ".hada_w1_a", ".hada_w1_b", ".hada_w2_a", ".hada_w2_b", ".hada_t1", ".hada_t2", + ".lokr_w1", ".lokr_w2", ".lokr_w1_a", ".lokr_w1_b", ".lokr_w2_a", ".lokr_w2_b", + ".lokr_t2", ".oft_blocks", ".boft_blocks", ".rescale", ".w_norm", ".b_norm", +) + +_STRIP_PREFIXES = ( + "diffusion_model.", "transformer.", "base_model.model.", "pipe.dit.", + "model.diffusion_model.", "unet.", +) + +_UNDERSCORE_PREFIXES = ("lora_unet_", "lycoris_", "lora_transformer_") + + +def split_suffix(key: str): + """``blocks.0.mlp.fc1.lora_A.weight`` -> ``('blocks.0.mlp.fc1', '.lora_A.weight')``.""" + for suffix in sorted(_LORA_SUFFIXES, key=len, reverse=True): + if key.endswith(suffix): + return key[: -len(suffix)], suffix + return None, None + + +def build_module_index(model_sd_keys) -> dict: + """Map every spelling of a DiT module path to its canonical form. + + Canonical form is the state-dict key minus ``.weight``, e.g. + ``diffusion_model.blocks.0.attn.qkv_proj``. Both the dotted path and the + underscored path (which is what kohya trainers emit) resolve to it, and the + underscore mapping is derived from the real keys so no heuristic is needed + to decide whether ``qkv_proj`` is one token or two. + """ + index: dict[str, str] = {} + for k in model_sd_keys: + if not k.startswith("diffusion_model.") or not k.endswith(".weight"): + continue + canonical = k[: -len(".weight")] + bare = canonical[len("diffusion_model."):] + index[canonical] = canonical + index[bare] = canonical + index[bare.replace(".", "_")] = canonical + return index + + +def _strip_prefixes(path: str) -> str: + changed = True + while changed: + changed = False + for pre in _STRIP_PREFIXES: + if path.startswith(pre): + path = path[len(pre):] + changed = True + break + for pre in _UNDERSCORE_PREFIXES: + if path.startswith(pre): + path = path[len(pre):] + break + return path + + +def canonical_module(key_body: str, index: dict): + """Resolve one LoRA key body to a canonical module path, or None.""" + body = _strip_prefixes(key_body) + hit = index.get(body) + if hit is not None: + return hit + # kohya writes ``lora_unet__blocks_0_...`` (double underscore) and some + # trainers keep a dotted tail after the underscored head + squashed = body.replace(".", "_") + hit = index.get(squashed) or index.get(re.sub(r"_{2,}", "_", squashed)) + if hit is not None: + return hit + return None + + +def normalize(lora_sd: dict, index: dict): + """Rewrite a LoRA state dict onto canonical module keys. + + Returns ``(normalized_sd, unmatched_keys)``. Non-LoRA passenger tensors + (``adaln_t_table`` and full ``adaln_proj.linear.weight`` replacements found + in "complete pruned" LoRA files) are passed through untouched so the caller + can deal with them. + """ + out: dict = {} + unmatched: list[str] = [] + for key, value in lora_sd.items(): + body, suffix = split_suffix(key) + if body is None: + out[key] = value # passenger tensor, handled upstream + continue + module = canonical_module(body, index) + if module is None: + unmatched.append(key) + continue + out[module + suffix] = value + return out, unmatched diff --git a/h3lora/nodes.py b/h3lora/nodes.py new file mode 100644 index 0000000..b938922 --- /dev/null +++ b/h3lora/nodes.py @@ -0,0 +1,202 @@ +"""Node definitions for the MiniMax H3 Power LoRA Stack.""" + +from __future__ import annotations + +import logging + +import folder_paths + +from . import apply as apply_mod +from . import detect + +LOG = logging.getLogger("h3.powerlorastack") +CATEGORY = "MiniMax-H3/lora" + + +class AnyType(str): + def __ne__(self, other): + return False + + +ANY = AnyType("*") + + +class FlexibleOptionalInputType(dict): + """Accepts the arbitrary ``lora_N`` inputs the frontend adds at runtime. + + ComfyUI validates a prompt against INPUT_TYPES; a stack whose row count is + decided in the browser has no fixed schema, so this dict reports that it + contains every key and hands back a permissive type for the ones it does + not know about. + """ + + def __init__(self, type, data=None): + super().__init__() + self.type = type + self.data = data or {} + self.update(self.data) + + def __getitem__(self, key): + if key in self.data: + return self.data[key] + return (self.type,) + + def __contains__(self, key): + return True + + +def _resolve_lora(name: str): + """Map a stored lora name onto a real file, tolerating separator drift.""" + if not name or name == "None": + return None + path = folder_paths.get_full_path("loras", name) + if path: + return path + wanted = name.replace("\\", "/").lower() + for candidate in folder_paths.get_filename_list("loras"): + normalized = candidate.replace("\\", "/").lower() + if normalized == wanted or normalized.endswith("/" + wanted.rsplit("/", 1)[-1]): + return folder_paths.get_full_path("loras", candidate) + return None + + +def _collect(kwargs): + """Pull enabled rows out of the dynamic ``lora_N`` inputs, in UI order.""" + rows = [] + for key, value in kwargs.items(): + if not key.lower().startswith("lora_") or not isinstance(value, dict): + continue + if "lora" not in value: + continue + try: + order = int(key.split("_", 1)[1]) + except (IndexError, ValueError): + order = 1 << 30 + rows.append((order, value)) + rows.sort(key=lambda item: item[0]) + + entries = [] + for _, value in rows: + if not value.get("on", True): + continue + strength = float(value.get("strength", 1.0)) + if strength == 0.0: + continue + name = value.get("lora") + if not name or name == "None": + continue # a row the user added but has not filled in yet + path = _resolve_lora(name) + if path is None: + LOG.warning("H3 Power LoRA Stack: could not find lora %r, skipping", name) + continue + entries.append({"name": name, "path": path, "strength": strength}) + return entries + + +class H3PowerLoraStack: + """Stacked multi-LoRA loader for MiniMax H3 across every weight format. + + Handles the three things the stock loader and rgthree's Power Lora Loader + get wrong on H3: + + * quantized bases -- w4a8/int4 weights are patched with an exact runtime + branch instead of a lossy dequantize/requantize merge; + * adaLN basis mismatch -- LoRAs trained against a dense checkpoint are + rebased onto a pruned checkpoint's curve, instead of being skipped; + * key conventions -- ai-toolkit, kohya, lycoris, peft and bare-prefix + layouts all resolve against the model's own key set. + """ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": {}, + "optional": FlexibleOptionalInputType(ANY, { + "model": ("MODEL",), + "quantized_layers": (["auto", "branch", "merge"], { + "default": "auto", + "tooltip": "auto: runtime branch for quantized layers, merge for the rest. " + "branch: never modify a weight. merge: stock behaviour " + "(destroys LoRAs on w4a8/int4).", + }), + "adaln_port": (["auto", "off"], { + "default": "auto", + "tooltip": "Rebase adaLN LoRA pairs between dense (2688) and curve (8) " + "checkpoints. Needs h3_silu_temb_grid.safetensors.", + }), + }), + "hidden": {}, + } + + RETURN_TYPES = ("MODEL", "STRING") + RETURN_NAMES = ("MODEL", "report") + OUTPUT_TOOLTIPS = ("Patched model", "Per-LoRA account of what was applied") + FUNCTION = "apply" + CATEGORY = CATEGORY + DESCRIPTION = __doc__ + + def apply(self, model=None, quantized_layers="auto", adaln_port="auto", **kwargs): + if model is None: + raise ValueError("H3 Power LoRA Stack: no model connected") + + entries = _collect(kwargs) + if not entries: + return (model, "no LoRAs enabled") + + patcher, report = apply_mod.apply_stack( + model, entries, mode=quantized_layers, adaln_mode=adaln_port, + ) + text = report.text() + LOG.info("H3 Power LoRA Stack:\n%s", text) + return (patcher, text) + + +class H3LoraInspector: + """Report the format, rank and adaLN basis of a LoRA without loading it.""" + + @classmethod + def INPUT_TYPES(cls): + return {"required": { + "lora_name": (folder_paths.get_filename_list("loras"),), + }} + + RETURN_TYPES = ("STRING",) + RETURN_NAMES = ("info",) + FUNCTION = "inspect" + CATEGORY = CATEGORY + DESCRIPTION = __doc__ + + def inspect(self, lora_name): + path = _resolve_lora(lora_name) + if path is None: + return (f"{lora_name}: not found",) + info = detect.inspect(path) + if info.get("error"): + return (f"{info['name']}: {info['error']}",) + lines = [ + f"file : {info['name']}", + f"size : {info['size'] / (1024 ** 2):.0f} MB, {info['tensors']} tensors", + f"format : {info['convention']}" + + (f" ({info['trainer']})" if info.get("trainer") else ""), + f"dtype : {', '.join(info['dtypes'])}", + f"rank : {'/'.join(str(r) for r in info['ranks']) or '?'}" + + (" (alpha present)" if info["alpha"] else " (no alpha)"), + f"adaLN : {'/'.join(str(d) for d in info['adaln_dims']) or 'none'}", + f"modules : {', '.join(info['modules'])}", + ] + if info.get("base_model"): + lines.append(f"base : {info['base_model']}") + if info.get("passenger"): + lines.append(f"passenger : {', '.join(info['passenger'])}") + return ("\n".join(lines),) + + +NODE_CLASS_MAPPINGS = { + "H3PowerLoraStack": H3PowerLoraStack, + "H3LoraInspector": H3LoraInspector, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "H3PowerLoraStack": "MiniMax H3 Power LoRA Stack", + "H3LoraInspector": "MiniMax H3 LoRA Inspector", +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5805a2f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,14 @@ +[project] +name = "comfyui-h3-powerlorastack" +description = "Stacked multi-LoRA loader for MiniMax H3 across bf16, fp8, int8 and w4a8 weights." +version = "1.0.0" +license = { text = "MIT" } +dependencies = [] + +[project.urls] +Repository = "https://github.com/doggeddalle/ComfyUI-H3-PowerLoraStack" + +[tool.comfy] +PublisherId = "doggeddalle" +DisplayName = "MiniMax H3 Power LoRA Stack" +Icon = "" diff --git a/web/h3_power_lora_stack.js b/web/h3_power_lora_stack.js new file mode 100644 index 0000000..14eee19 --- /dev/null +++ b/web/h3_power_lora_stack.js @@ -0,0 +1,502 @@ +import { app } from "../../scripts/app.js"; +import { api } from "../../scripts/api.js"; + +const NODE_TYPE = "H3PowerLoraStack"; +const ROW_HEIGHT = 22; +const MARGIN = 12; + +// Session cache for the loras folder listing. Deliberately a module global +// rather than graph.extra: anything hung off the graph is serialized into the +// saved workflow, which would bake a stale copy of the whole folder into every +// .json. The promise itself is cached so N rows opening at once share one +// request. +let loraListPromise = null; + +/** + * The stack node declares no ``lora_name`` input of its own -- rows are added + * in the browser -- so the folder listing is borrowed from a node that does. + */ +function fetchLoraList() { + if (!loraListPromise) { + loraListPromise = api + .fetchApi("/object_info/LoraLoaderModelOnly") + .then((res) => res.json()) + .then( + (info) => info?.LoraLoaderModelOnly?.input?.required?.lora_name?.[0] ?? [] + ) + .catch((err) => { + console.warn("[H3PowerLoraStack] could not fetch lora list", err); + loraListPromise = null; // let the next picker retry + return []; + }); + } + return loraListPromise; +} + +function drawToggle(ctx, x, y, size, on) { + ctx.save(); + ctx.beginPath(); + ctx.roundRect(x, y + 3, size * 1.7, size - 6, (size - 6) / 2); + ctx.fillStyle = on ? "#3d8c5a" : "#3a3a3a"; + ctx.fill(); + ctx.beginPath(); + const knob = (size - 8) / 2; + ctx.arc(x + (on ? size * 1.7 - knob - 3 : knob + 3), y + size / 2, knob, 0, Math.PI * 2); + ctx.fillStyle = on ? "#d8f5e4" : "#8a8a8a"; + ctx.fill(); + ctx.restore(); +} + +function shortName(name) { + if (!name || name === "None") return "click to choose"; + const base = String(name).replace(/\\/g, "/").split("/").pop(); + return base.replace(/\.(safetensors|ckpt|pt|bin)$/i, ""); +} + +/** + * One stack row: enable toggle, lora name, strength, remove button. + * Values serialize as {on, lora, strength}, which is what the Python side's + * flexible `lora_N` inputs expect. + */ +function makeLoraWidget(node, name, value) { + const widget = { + name, + type: "H3_LORA", + value: Object.assign({ on: true, lora: "None", strength: 1.0 }, value || {}), + options: { serialize: true }, + hitAreas: {}, + + computeSize() { + return [node.size[0], ROW_HEIGHT]; + }, + + draw(ctx, node_, widgetWidth, y, height) { + const margin = MARGIN; + const left = margin; + const right = widgetWidth - margin; + const midY = y + height / 2; + ctx.save(); + + ctx.beginPath(); + ctx.roundRect(left, y + 1, right - left, height - 2, 6); + ctx.fillStyle = this.value.on ? "#2b2b2b" : "#232323"; + ctx.fill(); + + let cursor = left + 8; + drawToggle(ctx, cursor, y, height, this.value.on); + this.hitAreas.toggle = [cursor, cursor + height * 1.7]; + cursor += height * 1.7 + 10; + + // remove button, laid out from the right edge inward + const removeX = right - 20; + ctx.fillStyle = "#8a8a8a"; + ctx.font = "14px sans-serif"; + ctx.textAlign = "center"; + ctx.textBaseline = "middle"; + ctx.fillText("✕", removeX + 6, midY); + this.hitAreas.remove = [removeX, removeX + 16]; + + const strengthRight = removeX - 10; + const strengthLeft = strengthRight - 74; + ctx.fillStyle = "#1e1e1e"; + ctx.beginPath(); + ctx.roundRect(strengthLeft, y + 3, strengthRight - strengthLeft, height - 6, 4); + ctx.fill(); + ctx.fillStyle = "#b8b8b8"; + ctx.font = "11px sans-serif"; + ctx.fillText("◀", strengthLeft + 9, midY); + ctx.fillText("▶", strengthRight - 9, midY); + ctx.fillStyle = this.value.on ? "#e8e8e8" : "#777"; + ctx.font = "12px sans-serif"; + ctx.fillText( + Number(this.value.strength).toFixed(2), + (strengthLeft + strengthRight) / 2, + midY + ); + this.hitAreas.strengthDown = [strengthLeft, strengthLeft + 18]; + this.hitAreas.strengthUp = [strengthRight - 18, strengthRight]; + this.hitAreas.strengthValue = [strengthLeft + 18, strengthRight - 18]; + + const nameLeft = cursor; + const nameRight = strengthLeft - 10; + ctx.textAlign = "left"; + ctx.fillStyle = this.value.on ? "#e8e8e8" : "#777"; + ctx.font = "12px sans-serif"; + const label = shortName(this.value.lora); + let text = label; + const maxWidth = nameRight - nameLeft; + if (ctx.measureText(text).width > maxWidth) { + while (text.length > 4 && ctx.measureText(text + "…").width > maxWidth) { + text = text.slice(0, -1); + } + text += "…"; + } + ctx.fillText(text, nameLeft, midY); + this.hitAreas.name = [nameLeft, nameRight]; + + ctx.restore(); + }, + + mouse(event, pos, node_) { + if (event.type !== "pointerdown") return false; + const x = pos[0]; + const hit = (area) => area && x >= area[0] && x <= area[1]; + + if (hit(this.hitAreas.toggle)) { + this.value = { ...this.value, on: !this.value.on }; + node_.setDirtyCanvas(true, true); + return true; + } + if (hit(this.hitAreas.remove)) { + removeLoraWidget(node_, this); + return true; + } + if (hit(this.hitAreas.strengthDown)) { + this.value = { ...this.value, strength: round2(this.value.strength - 0.05) }; + node_.setDirtyCanvas(true, true); + return true; + } + if (hit(this.hitAreas.strengthUp)) { + this.value = { ...this.value, strength: round2(this.value.strength + 0.05) }; + node_.setDirtyCanvas(true, true); + return true; + } + if (hit(this.hitAreas.strengthValue)) { + app.canvas.prompt( + "Strength", + this.value.strength, + (v) => { + const parsed = parseFloat(v); + if (!Number.isNaN(parsed)) { + this.value = { ...this.value, strength: parsed }; + node_.setDirtyCanvas(true, true); + } + }, + event + ); + return true; + } + if (hit(this.hitAreas.name)) { + showLoraMenu(node_, this, event); + return true; + } + return false; + }, + + serializeValue() { + return { ...this.value }; + }, + }; + return widget; +} + +function round2(v) { + return Math.round(v * 100) / 100; +} + +/* ---------------------------------------------------------------- picker -- */ + +const PICKER_CSS = ` +.h3pls-picker{position:fixed;z-index:10000;display:flex;flex-direction:column; + width:460px;max-width:92vw;background:#1e1e1e;border:1px solid #4a4a4a; + border-radius:8px;box-shadow:0 12px 32px rgba(0,0,0,.6);overflow:hidden; + font-family:system-ui,-apple-system,sans-serif;} +.h3pls-picker input{margin:8px;padding:7px 9px;background:#111;color:#eee; + border:1px solid #555;border-radius:5px;font-size:13px;outline:none;} +.h3pls-picker input:focus{border-color:#3d8c5a;} +.h3pls-count{padding:0 12px 6px;font-size:11px;color:#777;} +.h3pls-list{overflow-y:auto;max-height:46vh;} +.h3pls-item{padding:6px 12px;font-size:12.5px;color:#ddd;cursor:pointer; + white-space:nowrap;overflow:hidden;text-overflow:ellipsis;} +.h3pls-item:hover{background:#2c2c2c;} +.h3pls-item.sel{background:#31543f;} +.h3pls-item .dir{color:#7a7a7a;} +.h3pls-item .hit{color:#7fd6a0;font-weight:600;} +.h3pls-item.cur::after{content:"current";float:right;color:#666;font-size:10px;} +.h3pls-empty{padding:14px 12px;font-size:12px;color:#888;} +`; + +function ensurePickerCss() { + if (document.getElementById("h3pls-css")) return; + const style = document.createElement("style"); + style.id = "h3pls-css"; + style.textContent = PICKER_CSS; + document.head.appendChild(style); +} + +/** Case-insensitive AND-match of every whitespace-separated term. */ +function matchTerms(name, terms) { + const hay = name.toLowerCase(); + const spans = []; + for (const term of terms) { + const at = hay.indexOf(term); + if (at < 0) return null; + spans.push([at, at + term.length]); + } + return spans; +} + +/** Basename hits and early hits rank first; then shortest name. */ +function score(name, terms, spans) { + if (!terms.length) return 0; + const slash = name.lastIndexOf("/") + 1; + let s = 0; + for (const [start] of spans) { + if (start >= slash) s -= 1000; + s += start; + } + return s + name.length * 0.01; +} + +function highlight(name, spans) { + const slash = name.lastIndexOf("/") + 1; + const merged = []; + for (const [a, b] of [...spans].sort((p, q) => p[0] - q[0])) { + const last = merged[merged.length - 1]; + if (last && a <= last[1]) last[1] = Math.max(last[1], b); + else merged.push([a, b]); + } + const esc = (t) => + t.replace(/&/g, "&").replace(//g, ">"); + let html = ""; + let at = 0; + for (const [a, b] of merged) { + html += wrapDir(esc(name.slice(at, a)), at, slash); + html += `${esc(name.slice(a, b))}`; + at = b; + } + html += wrapDir(esc(name.slice(at)), at, slash); + return html; +} + +function wrapDir(text, offset, slash) { + if (!text) return ""; + if (offset + text.length <= slash) return `${text}`; + if (offset >= slash) return text; + const cut = slash - offset; + return `${text.slice(0, cut)}${text.slice(cut)}`; +} + +/** + * Filterable LoRA picker. LiteGraph's ContextMenu has no search, which is + * unusable once the loras folder holds more than a screenful, so this is a + * self-contained overlay: type to narrow, arrows to move, Enter to take it. + */ +async function showLoraMenu(node, widget, event, { removeOnCancel = false } = {}) { + const names = await fetchLoraList(); + if (!names.length) { + if (removeOnCancel) removeLoraWidget(node, widget); + alert("No LoRAs found in models/loras."); + return; + } + ensurePickerCss(); + + const current = widget.value.lora; + const root = document.createElement("div"); + root.className = "h3pls-picker"; + const input = document.createElement("input"); + input.type = "text"; + input.placeholder = "Type to filter… (space-separated terms all must match)"; + input.spellcheck = false; + const count = document.createElement("div"); + count.className = "h3pls-count"; + const list = document.createElement("div"); + list.className = "h3pls-list"; + root.append(input, count, list); + document.body.appendChild(root); + + // place near the click, clamped into the viewport + const px = event?.clientX ?? window.innerWidth / 2; + const py = event?.clientY ?? window.innerHeight / 2; + root.style.left = `${Math.min(px, window.innerWidth - 470)}px`; + root.style.top = `${Math.min(py, window.innerHeight - 320)}px`; + + let rows = []; + let sel = 0; + + let committed = false; + const close = () => { + document.removeEventListener("pointerdown", onOutside, true); + root.remove(); + // a row opened straight from "Add LoRA" and then dismissed was never + // wanted, so take it back out rather than leaving an empty slot behind + if (removeOnCancel && !committed) removeLoraWidget(node, widget); + }; + const commit = (value) => { + committed = true; + widget.value = { ...widget.value, lora: value }; + node.setDirtyCanvas(true, true); + close(); + }; + const onOutside = (e) => { + if (!root.contains(e.target)) close(); + }; + + const render = () => { + const query = input.value.trim().toLowerCase(); + const terms = query ? query.split(/\s+/) : []; + rows = []; + for (const name of names) { + const spans = matchTerms(name, terms); + if (spans === null) continue; + rows.push({ name, spans, s: score(name, terms, spans) }); + } + if (terms.length) rows.sort((a, b) => a.s - b.s); + sel = 0; + if (!terms.length) { + const at = rows.findIndex((r) => r.name === current); + if (at > 0) sel = at; + } + + count.textContent = `${rows.length} of ${names.length}`; + list.replaceChildren(); + if (!rows.length) { + const empty = document.createElement("div"); + empty.className = "h3pls-empty"; + empty.textContent = "No LoRA matches that filter."; + list.appendChild(empty); + return; + } + rows.forEach((row, i) => { + const item = document.createElement("div"); + item.className = + "h3pls-item" + (i === sel ? " sel" : "") + (row.name === current ? " cur" : ""); + item.innerHTML = highlight(row.name, row.spans); + item.title = row.name; + item.addEventListener("pointerdown", (e) => { + e.preventDefault(); + e.stopPropagation(); + commit(row.name); + }); + list.appendChild(item); + }); + scrollToSel(); + }; + + const scrollToSel = () => { + const el = list.children[sel]; + if (el && el.scrollIntoView) el.scrollIntoView({ block: "nearest" }); + }; + + const move = (delta) => { + if (!rows.length) return; + const prev = list.children[sel]; + if (prev) prev.classList.remove("sel"); + sel = (sel + delta + rows.length) % rows.length; + const next = list.children[sel]; + if (next) next.classList.add("sel"); + scrollToSel(); + }; + + input.addEventListener("input", render); + input.addEventListener("keydown", (e) => { + if (e.key === "ArrowDown") { e.preventDefault(); move(1); } + else if (e.key === "ArrowUp") { e.preventDefault(); move(-1); } + else if (e.key === "PageDown") { e.preventDefault(); move(10); } + else if (e.key === "PageUp") { e.preventDefault(); move(-10); } + else if (e.key === "Enter") { + e.preventDefault(); + if (rows[sel]) commit(rows[sel].name); + } else if (e.key === "Escape") { + e.preventDefault(); + close(); + } + e.stopPropagation(); // keep ComfyUI's global hotkeys out of the box + }); + + render(); + const grabFocus = () => { + if (root.isConnected && document.activeElement !== input) input.focus(); + }; + // Defer by a full task, not a microtask: the pointerdown that opened this is + // still being dispatched, and a capture listener added now would see it and + // close the picker immediately. + setTimeout(() => { + document.addEventListener("pointerdown", onOutside, true); + grabFocus(); + }, 0); + // LiteGraph pulls focus back to the canvas on a delay in some paths; re-assert + // once so the caret really is in the filter box and typing goes there. + setTimeout(grabFocus, 60); +} + +function loraWidgets(node) { + return (node.widgets || []).filter((w) => w.type === "H3_LORA"); +} + +function renumber(node) { + loraWidgets(node).forEach((w, i) => { + w.name = `lora_${i + 1}`; + }); +} + +function addLoraWidget(node, value) { + const widget = makeLoraWidget(node, `lora_${loraWidgets(node).length + 1}`, value); + node.widgets = node.widgets || []; + // keep the "Add LoRA" button last + const buttonIndex = node.widgets.findIndex((w) => w.name === "➕ Add LoRA"); + if (buttonIndex >= 0) node.widgets.splice(buttonIndex, 0, widget); + else node.widgets.push(widget); + renumber(node); + resize(node); + return widget; +} + +function removeLoraWidget(node, widget) { + const i = node.widgets.indexOf(widget); + if (i >= 0) node.widgets.splice(i, 1); + renumber(node); + resize(node); + node.setDirtyCanvas(true, true); +} + +function resize(node) { + const [minWidth, minHeight] = node.computeSize(); + // Width stays the user's choice; height tracks the row count so removing a + // row hands the space back instead of leaving a hole. + node.setSize([Math.max(node.size[0], minWidth), minHeight]); + node.setDirtyCanvas(true, true); +} + +app.registerExtension({ + name: "h3.power.lora.stack", + + async beforeRegisterNodeDef(nodeType, nodeData) { + if (nodeData.name !== NODE_TYPE) return; + + const onCreated = nodeType.prototype.onNodeCreated; + nodeType.prototype.onNodeCreated = function () { + onCreated?.apply(this, arguments); + this.serialize_widgets = true; + + // (value, canvas, node, pos, event) -- the event is the pointerdown that + // started the click, so the picker opens right under the button. + this.addWidget("button", "➕ Add LoRA", null, (_v, _canvas, _node, _pos, event) => { + const widget = addLoraWidget(this); + // open the picker immediately: the row is added to be filled in, so the + // filter box takes the keyboard straight away + showLoraMenu(this, widget, event, { removeOnCancel: true }); + }); + + fetchLoraList(); // warm the cache so the first picker opens instantly + resize(this); + }; + + const onConfigure = nodeType.prototype.onConfigure; + nodeType.prototype.onConfigure = function (info) { + onConfigure?.apply(this, arguments); + // ComfyUI restores widgets_values positionally against the widgets that + // existed before this ran, which is only the fixed combos and the button. + // The stack rows are recovered by shape instead of position, so a slot + // shifting cannot corrupt them. + const values = info?.widgets_values; + if (!Array.isArray(values)) return; + const rows = values.filter( + (v) => v && typeof v === "object" && !Array.isArray(v) && "lora" in v + ); + for (const w of loraWidgets(this)) removeLoraWidget(this, w); + for (const row of rows) addLoraWidget(this, row); + resize(this); + }; + }, +});