From 78beffb29f299571993c73f8fbd748e1577fd192 Mon Sep 17 00:00:00 2001 From: WarpedAnimation Date: Thu, 24 Jul 2025 08:16:24 -0500 Subject: [PATCH] V1.7.1 (#11) * V1.7.1 * V1.7.1 * V1.7.1 * Create deleteme.txt * v1.7.1 * Delete gguf_connector/deleteme.txt * v1.7.1 * V1.7.1 --- README.md | 7 + ...arped Batch Hunyuan LORA T2V Workflow.json | 1037 +++--- ... Hunyuan I2V Using I2V Model Workflow.json | 536 ++- .../Warped Hunyuan I2V Workflow.json | 583 ++-- .../Warped Hunyuan T2V Workflow.json | 399 +-- .../Warped Hunyuan v2v Workflow.json | 577 ++-- .../Warped Midres Upscale Workflow.json | 976 +++--- .../Warped Upscale By Model Workflow.json | 539 ++- Sample_Workflows/Warped WAN I2V Workflow.json | 624 ++-- __init__.py | 23 +- gguf_connector/__init__.py | 192 ++ gguf_connector/b1.py | 630 ++++ gguf_connector/b2.py | 681 ++++ gguf_connector/c.py | 51 + gguf_connector/cg.py | 61 + gguf_connector/const.py | 1137 +++++++ gguf_connector/cp.py | 64 + gguf_connector/cpp.py | 52 + gguf_connector/cs.py | 61 + gguf_connector/d.py | 79 + gguf_connector/d2.py | 61 + gguf_connector/d4.py | 62 + gguf_connector/d5.py | 63 + gguf_connector/d6.py | 185 + gguf_connector/d7.py | 197 ++ gguf_connector/data.json | 186 + gguf_connector/e.py | 48 + gguf_connector/f.py | 54 + gguf_connector/g.py | 29 + gguf_connector/gpp.py | 30 + gguf_connector/i.py | 6 + gguf_connector/i2.py | 42 + gguf_connector/lazy.py | 163 + gguf_connector/logo.png | Bin 0 -> 578 bytes gguf_connector/m.py | 61 + gguf_connector/m2.py | 17 + gguf_connector/ma.py | 25 + gguf_connector/menu.py | 34 + gguf_connector/n.py | 18 + gguf_connector/o.py | 3 + gguf_connector/p.py | 28 + gguf_connector/p1.py | 28 + gguf_connector/p2.py | 18 + gguf_connector/pg.py | 62 + gguf_connector/pp.py | 65 + gguf_connector/ps.py | 62 + gguf_connector/q.py | 43 + gguf_connector/q1.py | 43 + gguf_connector/q2.py | 32 + gguf_connector/quant.py | 755 +++++ gguf_connector/quant2.py | 191 ++ gguf_connector/quant2a.py | 203 ++ gguf_connector/quant2b.py | 21 + gguf_connector/quant2c.py | 287 ++ gguf_connector/quant2d.py | 309 ++ gguf_connector/r.py | 43 + gguf_connector/r2.py | 27 + gguf_connector/r3.py | 30 + gguf_connector/r4.py | 22 + gguf_connector/reader.py | 256 ++ gguf_connector/reader2.py | 107 + gguf_connector/s.py | 44 + gguf_connector/s0.py | 43 + gguf_connector/s1.py | 35 + gguf_connector/s2.py | 195 ++ gguf_connector/s5.py | 35 + gguf_connector/sx.py | 51 + gguf_connector/sy.py | 35 + gguf_connector/t.py | 195 ++ gguf_connector/t0.py | 106 + gguf_connector/t1.py | 118 + gguf_connector/t2.py | 115 + gguf_connector/t3.py | 80 + gguf_connector/t4.py | 198 ++ gguf_connector/t5.py | 116 + gguf_connector/t6.py | 95 + gguf_connector/t7.py | 213 ++ gguf_connector/t8.py | 199 ++ gguf_connector/t9.py | 79 + gguf_connector/tkn.py | 60 + gguf_connector/u.py | 1 + gguf_connector/v.py | 48 + gguf_connector/v2.py | 56 + gguf_connector/vg.py | 52 + gguf_connector/vg2.py | 57 + gguf_connector/w.py | 3 + gguf_connector/writer.py | 801 +++++ gguf_connector/y.py | 1 + gguf_nodes.py | 584 ++++ nodes.py | 2991 ++++++++++++++++- pig.py | 999 ++++++ pyproject.toml | 2 +- version.json | 1 + 93 files changed, 17133 insertions(+), 2700 deletions(-) create mode 100644 gguf_connector/__init__.py create mode 100644 gguf_connector/b1.py create mode 100644 gguf_connector/b2.py create mode 100644 gguf_connector/c.py create mode 100644 gguf_connector/cg.py create mode 100644 gguf_connector/const.py create mode 100644 gguf_connector/cp.py create mode 100644 gguf_connector/cpp.py create mode 100644 gguf_connector/cs.py create mode 100644 gguf_connector/d.py create mode 100644 gguf_connector/d2.py create mode 100644 gguf_connector/d4.py create mode 100644 gguf_connector/d5.py create mode 100644 gguf_connector/d6.py create mode 100644 gguf_connector/d7.py create mode 100644 gguf_connector/data.json create mode 100644 gguf_connector/e.py create mode 100644 gguf_connector/f.py create mode 100644 gguf_connector/g.py create mode 100644 gguf_connector/gpp.py create mode 100644 gguf_connector/i.py create mode 100644 gguf_connector/i2.py create mode 100644 gguf_connector/lazy.py create mode 100644 gguf_connector/logo.png create mode 100644 gguf_connector/m.py create mode 100644 gguf_connector/m2.py create mode 100644 gguf_connector/ma.py create mode 100644 gguf_connector/menu.py create mode 100644 gguf_connector/n.py create mode 100644 gguf_connector/o.py create mode 100644 gguf_connector/p.py create mode 100644 gguf_connector/p1.py create mode 100644 gguf_connector/p2.py create mode 100644 gguf_connector/pg.py create mode 100644 gguf_connector/pp.py create mode 100644 gguf_connector/ps.py create mode 100644 gguf_connector/q.py create mode 100644 gguf_connector/q1.py create mode 100644 gguf_connector/q2.py create mode 100644 gguf_connector/quant.py create mode 100644 gguf_connector/quant2.py create mode 100644 gguf_connector/quant2a.py create mode 100644 gguf_connector/quant2b.py create mode 100644 gguf_connector/quant2c.py create mode 100644 gguf_connector/quant2d.py create mode 100644 gguf_connector/r.py create mode 100644 gguf_connector/r2.py create mode 100644 gguf_connector/r3.py create mode 100644 gguf_connector/r4.py create mode 100644 gguf_connector/reader.py create mode 100644 gguf_connector/reader2.py create mode 100644 gguf_connector/s.py create mode 100644 gguf_connector/s0.py create mode 100644 gguf_connector/s1.py create mode 100644 gguf_connector/s2.py create mode 100644 gguf_connector/s5.py create mode 100644 gguf_connector/sx.py create mode 100644 gguf_connector/sy.py create mode 100644 gguf_connector/t.py create mode 100644 gguf_connector/t0.py create mode 100644 gguf_connector/t1.py create mode 100644 gguf_connector/t2.py create mode 100644 gguf_connector/t3.py create mode 100644 gguf_connector/t4.py create mode 100644 gguf_connector/t5.py create mode 100644 gguf_connector/t6.py create mode 100644 gguf_connector/t7.py create mode 100644 gguf_connector/t8.py create mode 100644 gguf_connector/t9.py create mode 100644 gguf_connector/tkn.py create mode 100644 gguf_connector/u.py create mode 100644 gguf_connector/v.py create mode 100644 gguf_connector/v2.py create mode 100644 gguf_connector/vg.py create mode 100644 gguf_connector/vg2.py create mode 100644 gguf_connector/w.py create mode 100644 gguf_connector/writer.py create mode 100644 gguf_connector/y.py create mode 100644 gguf_nodes.py create mode 100644 pig.py create mode 100644 version.json diff --git a/README.md b/README.md index a1043b9..4c38490 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,13 @@
+# V1.7.1 Updates + +1) Added Fast Loading GGUF Unet, Clip, and Dual Clip Loader Nodes +2) Fixed/Updated Sample Workflows + +
+ # V1.7.0 Overview #### [V1.7.0 Overview Video](https://mega.nz/file/r2YwySbS#a1Fw_Z4ASlkXIVBmLQFHAbEzFxpWGGVmyEMrVNplt6I) diff --git a/Sample_Workflows/Warped Batch Hunyuan LORA T2V Workflow.json b/Sample_Workflows/Warped Batch Hunyuan LORA T2V Workflow.json index 84c7a91..ebe85dc 100644 --- a/Sample_Workflows/Warped Batch Hunyuan LORA T2V Workflow.json +++ b/Sample_Workflows/Warped Batch Hunyuan LORA T2V Workflow.json @@ -1,49 +1,9 @@ { "id": "facb86ea-4076-46ff-921d-e6dcadfb1754", "revision": 0, - "last_node_id": 294, - "last_link_id": 914, + "last_node_id": 298, + "last_link_id": 919, "nodes": [ - { - "id": 113, - "type": "SetNode", - "pos": [ - -7.660725116729736, - 63.07086181640625 - ], - "size": [ - 210, - 58 - ], - "flags": { - "collapsed": true - }, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "VAE", - "type": "VAE", - "link": 270 - } - ], - "outputs": [ - { - "name": "*", - "type": "*", - "links": null - } - ], - "title": "Set_VAE", - "properties": { - "previousName": "VAE" - }, - "widgets_values": [ - "VAE" - ], - "color": "#322", - "bgcolor": "#533" - }, { "id": 152, "type": "PreviewTextNode", @@ -56,15 +16,12 @@ 114.48214721679688 ], "flags": {}, - "order": 20, + "order": 19, "mode": 0, "inputs": [ { "name": "text", "type": "STRING", - "widget": { - "name": "text" - }, "link": 894 } ], @@ -82,7 +39,6 @@ }, "widgets_values": [ "", - "****** WarpedSamplerCustomAdvLatent: Total Images Generated 137 ******", [ false, true @@ -148,7 +104,6 @@ "outputs": [ { "name": "CONDITIONING", - "shape": 3, "type": "CONDITIONING", "slot_index": 0, "links": [ @@ -181,7 +136,7 @@ "flags": { "collapsed": true }, - "order": 17, + "order": 16, "mode": 0, "inputs": [ { @@ -198,7 +153,6 @@ "outputs": [ { "name": "GUIDER", - "shape": 3, "type": "GUIDER", "slot_index": 0, "links": [ @@ -217,225 +171,6 @@ "color": "#222", "bgcolor": "#000" }, - { - "id": 279, - "type": "TeaCache", - "pos": [ - -13, - 9 - ], - "size": [ - 315, - 106 - ], - "flags": { - "collapsed": true - }, - "order": 15, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 883 - } - ], - "outputs": [ - { - "name": "model", - "type": "MODEL", - "links": [ - 884 - ] - } - ], - "properties": { - "cnr_id": "teacache", - "ver": "f0c6c56778d626b23db99ea1fee41736f7f55720", - "Node name for S&R": "TeaCache" - }, - "widgets_values": [ - "hunyuan_video", - 0.4, - 1 - ] - }, - { - "id": 146, - "type": "CLIPTextEncode", - "pos": [ - -6.035816669464111, - 173.69970703125 - ], - "size": [ - 288.2851257324219, - 78.88789367675781 - ], - "flags": { - "collapsed": true - }, - "order": 10, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 869 - }, - { - "name": "text", - "type": "STRING", - "widget": { - "name": "text" - }, - "link": 448 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "slot_index": 0, - "links": [ - 352 - ] - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.27", - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "8itchW4lk, dynamic (real life:1.15) video showing front view of a beautiful stunning gorgeous woman with very long flowing brown hair with blonde highlights and exotic sexy grey eyes, she is wearing (a provocative revealing see-through scientist indiana jones style maroon satin top blouse:1.2) and see-through black thong underwear with an extremely short tight micro miniskirt low on the hips and she is braless, (she is showing her massive ((natural heavy sagging:1.3)) naked breasts:1.15), (her nipples are erect:1.25), (she is also wearing a thin silver military dogtags pendant necklace, she is in an abandonned nuclear facility, she was surprised by the viewer, so now she is walking toward the viewer to seduce him:1.25), (she is walking toward the viewer:1.2), (she is walking in a very sexy manner, smug facial expression with a slight smirk, sexy but dangerous attitude and seductive lips, normal speed, her heavy breasts (((are (bouncing:1.5) and (swaying:1.3) and (jiggling:1.7):1.7)) unrestrained with every step she takes and with ever move she makes:1.15), (((her breasts bounce into each other and recoil away from each other as she walks:1.8))), the viewer keeps her in view all the times, (extremely soft breasts:1.3), (extremely bouncing breasts:1.3), (breasts bouncing up and down side to side:1.5), detailed face, detailed hands, detailed fingers, nail polish, glamour eye makeup, eyeliner, lipstick, mascara, 4k ultra high quality video footage, (realisic jiggle, boucing, wobble, swaying physics, her breasts jiggle and bounce with each staps she take:1.2), dark atmosphere, post apocalyptic nuclear building interior, retro cyberpunk vibe", - [ - false, - true - ] - ] - }, - { - "id": 195, - "type": "Reroute", - "pos": [ - -8.020484924316406, - 198.64976501464844 - ], - "size": [ - 75, - 26 - ], - "flags": {}, - "order": 9, - "mode": 0, - "inputs": [ - { - "name": "", - "type": "*", - "link": 569 - } - ], - "outputs": [ - { - "name": "", - "type": "VAE", - "slot_index": 0, - "links": [ - 850, - 889 - ] - } - ], - "properties": { - "showOutputText": false, - "horizontal": false - } - }, - { - "id": 177, - "type": "BasicScheduler", - "pos": [ - -38.529476165771484, - 761.505126953125 - ], - "size": [ - 315, - 106 - ], - "flags": { - "collapsed": true - }, - "order": 14, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 526 - } - ], - "outputs": [ - { - "name": "SIGMAS", - "type": "SIGMAS", - "slot_index": 0, - "links": [ - 852, - 880, - 892 - ] - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.27", - "Node name for S&R": "BasicScheduler" - }, - "widgets_values": [ - "simple", - 20, - 1 - ] - }, - { - "id": 16, - "type": "KSamplerSelect", - "pos": [ - -39.147926330566406, - 827.6121215820312 - ], - "size": [ - 315, - 58 - ], - "flags": { - "collapsed": true - }, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "SAMPLER", - "shape": 3, - "type": "SAMPLER", - "slot_index": 0, - "links": [ - 851, - 879, - 891 - ] - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.27", - "Node name for S&R": "KSamplerSelect" - }, - "widgets_values": [ - "uni_pc" - ] - }, { "id": 190, "type": "VHS_VideoCombine", @@ -445,10 +180,10 @@ ], "size": [ 708.6593627929688, - 334 + 358 ], "flags": {}, - "order": 19, + "order": 18, "mode": 0, "inputs": [ { @@ -498,6 +233,7 @@ "save_metadata": true, "pingpong": false, "save_output": true, + "save_images": true, "videopreview": { "hidden": false, "paused": false, @@ -514,50 +250,6 @@ } } }, - { - "id": 67, - "type": "ModelSamplingSD3", - "pos": [ - 134.88357543945312, - -20.343273162841797 - ], - "size": [ - 210, - 58 - ], - "flags": { - "collapsed": true - }, - "order": 16, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 884 - } - ], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "slot_index": 0, - "links": [ - 195 - ] - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.27", - "Node name for S&R": "ModelSamplingSD3" - }, - "widgets_values": [ - 12 - ], - "color": "#222", - "bgcolor": "#000" - }, { "id": 284, "type": "WarpedCreateEmptyLatentBatch", @@ -570,7 +262,7 @@ 126 ], "flags": {}, - "order": 2, + "order": 1, "mode": 0, "inputs": [], "outputs": [ @@ -588,9 +280,10 @@ } ], "properties": { - "aux_id": "WarpedAnimation/ComfyUI-WarpedToolset", + "cnr_id": "comfyui-warpedtoolset", "ver": "1f5f5dd1352060b4db60431726812f980926bac4", - "Node name for S&R": "WarpedCreateEmptyLatentBatch" + "Node name for S&R": "WarpedCreateEmptyLatentBatch", + "aux_id": "WarpedAnimation/ComfyUI-WarpedToolset" }, "widgets_values": [ 137, @@ -612,7 +305,7 @@ 26 ], "flags": {}, - "order": 13, + "order": 11, "mode": 0, "inputs": [ { @@ -628,7 +321,7 @@ "slot_index": 0, "links": [ 526, - 883 + 918 ] } ], @@ -637,158 +330,6 @@ "horizontal": false } }, - { - "id": 10, - "type": "VAELoader", - "pos": [ - -436.01348876953125, - 863.2140502929688 - ], - "size": [ - 292.16009521484375, - 69.82014465332031 - ], - "flags": { - "collapsed": true - }, - "order": 3, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "VAE", - "shape": 3, - "type": "VAE", - "slot_index": 0, - "links": [ - 270, - 569 - ] - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.27", - "Node name for S&R": "VAELoader" - }, - "widgets_values": [ - "hunyuan_video_vae_bf16.safetensors" - ] - }, - { - "id": 273, - "type": "DualCLIPLoaderGGUF", - "pos": [ - -435.5494079589844, - 805.490478515625 - ], - "size": [ - 301.0328369140625, - 106 - ], - "flags": { - "collapsed": true - }, - "order": 4, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "CLIP", - "type": "CLIP", - "slot_index": 0, - "links": [ - 869 - ] - } - ], - "properties": { - "cnr_id": "ComfyUI-GGUF", - "ver": "bc5223b0e37e053dbec2ea5e5f52c2fd4b8f712a", - "Node name for S&R": "DualCLIPLoaderGGUF" - }, - "widgets_values": [ - "clip_l.safetensors", - "Kijai\\llava-llama-3-8B-v1_1-Q4_K_M.gguf", - "hunyuan_video" - ] - }, - { - "id": 275, - "type": "UnetLoaderGGUF", - "pos": [ - -431.4759826660156, - 742.2569580078125 - ], - "size": [ - 315, - 58 - ], - "flags": { - "collapsed": true - }, - "order": 5, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 872, - 908, - 909 - ] - } - ], - "properties": { - "cnr_id": "ComfyUI-GGUF", - "ver": "bc5223b0e37e053dbec2ea5e5f52c2fd4b8f712a", - "Node name for S&R": "UnetLoaderGGUF" - }, - "widgets_values": [ - "hunyuan-video-t2v-720p-Q4_K_M.gguf" - ] - }, - { - "id": 170, - "type": "StringConstantMultiline", - "pos": [ - -128.29852294921875, - 280.7070617675781 - ], - "size": [ - 423.325927734375, - 409.93310546875 - ], - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "STRING", - "type": "STRING", - "slot_index": 0, - "links": [ - 448 - ] - } - ], - "properties": { - "cnr_id": "comfyui-kjnodes", - "ver": "e3b4bcf21e03402a2b11927eb5a4e3df500795b7", - "Node name for S&R": "StringConstantMultiline" - }, - "widgets_values": [ - "8itchW4lk, dynamic (real life:1.15) video showing front view of a woman with long flowing auburn_hair and brown_eyes, she is wearing (beige thigh-high stockings and a maroon garter_belt with no panties and she is topless and she is braless:1.2). (she is showing her (large:1.3) ((natural heavy:1.3)) naked breasts:1.15) pointy, large aereolas, (her nipples are erect:1.25), She has small hips and a (small waist:1.4), (she is also wearing a thin diamond choker and navel_piercing, She has a slender fit body. she is in an abandonned nuclear facility, she was surprised by the viewer, so now she is walking toward the viewer to seduce him:1.25), (she is walking toward the viewer:1.2), (she is walking in a very sexy manner, looking at the viewer. Her facial expression goes from puzzled to flirting with the viewer. sexy but dangerous attitude and seductive lips, normal speed, her heavy breasts (((are (bouncing:1.5) and (swaying:1.3) and (jiggling:1.7):1.7)) unrestrained with every step she takes and with ever move she makes:1.15), (((her breasts bounce into each other and recoil away from each other as she walks:1.8))), the viewer keeps her in view all the times, (extremely soft breasts:1.3), (extremely bouncing breasts:1.3), (breasts bouncing up and down side to side:1.5), dark-skinned female, dark skin, detailed face, detailed hands, detailed fingers, nail polish, glamour eye makeup, eyeliner, lipstick, mascara, very petite body, 4k ultra high quality video footage, (realisic jiggle, bouncing, wobble, swaying physics, her breasts jiggle and bounce with each stap she takes:1.2), dark atmosphere, flickering lights shine down from the ceiling, well lit post apocalyptic nuclear building interior with god rays illuminating the room through holes in the ceiling and walls, retro cyberpunk vibe", - true, - [ - false, - true - ] - ] - }, { "id": 280, "type": "WarpedSamplerCustomAdvLatent", @@ -801,7 +342,7 @@ 1387.471923828125 ], "flags": {}, - "order": 18, + "order": 17, "mode": 0, "inputs": [ { @@ -857,9 +398,10 @@ } ], "properties": { - "aux_id": "WarpedAnimation/ComfyUI-WarpedToolset", + "cnr_id": "comfyui-warpedtoolset", "ver": "1f5f5dd1352060b4db60431726812f980926bac4", - "Node name for S&R": "WarpedSamplerCustomAdvLatent" + "Node name for S&R": "WarpedSamplerCustomAdvLatent", + "aux_id": "WarpedAnimation/ComfyUI-WarpedToolset" }, "widgets_values": [ 224176001153222, @@ -888,7 +430,7 @@ 113.97900390625 ], "flags": {}, - "order": 7, + "order": 2, "mode": 0, "inputs": [], "outputs": [ @@ -906,9 +448,10 @@ } ], "properties": { - "aux_id": "WarpedAnimation/ComfyUI-WarpedToolset", + "cnr_id": "comfyui-warpedtoolset", "ver": "1f5f5dd1352060b4db60431726812f980926bac4", - "Node name for S&R": "WarpedLoadLorasBatchByPrefix" + "Node name for S&R": "WarpedLoadLorasBatchByPrefix", + "aux_id": "WarpedAnimation/ComfyUI-WarpedToolset" }, "widgets_values": [ "M:\\Projects\\webui_test\\stable-diffusion-webui-forge\\models\\Lora\\Hunyuan", @@ -929,20 +472,18 @@ 403.67999267578125 ], "flags": {}, - "order": 11, + "order": 8, "mode": 0, "inputs": [ { "name": "model", "type": "MODEL", - "link": 909 + "link": 915 }, { "name": "lora_name", + "shape": 7, "type": "STRING", - "widget": { - "name": "lora_name" - }, "link": 912 } ], @@ -953,12 +494,19 @@ "links": [ 910 ] + }, + { + "name": "lora_metadata", + "shape": 6, + "type": "STRING", + "links": null } ], "properties": { - "aux_id": "WarpedAnimation/ComfyUI-WarpedToolset", + "cnr_id": "comfyui-warpedtoolset", "ver": "1f5f5dd1352060b4db60431726812f980926bac4", - "Node name for S&R": "WarpedHunyuanMultiLoraLoader" + "Node name for S&R": "WarpedHunyuanMultiLoraLoader", + "aux_id": "WarpedAnimation/ComfyUI-WarpedToolset" }, "widgets_values": [ "Hunyuan\\Move_Enhancer_V3_20.safetensors", @@ -973,12 +521,434 @@ "None", 1, "all", - "None", 1, "all" ], "color": "#323", "bgcolor": "#535" + }, + { + "id": 195, + "type": "Reroute", + "pos": [ + -151.76841735839844, + 719.0704956054688 + ], + "size": [ + 75, + 26 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "", + "type": "*", + "link": 917 + } + ], + "outputs": [ + { + "name": "", + "type": "VAE", + "slot_index": 0, + "links": [ + 850, + 889 + ] + } + ], + "properties": { + "showOutputText": false, + "horizontal": false + } + }, + { + "id": 295, + "type": "WarpedLoaderGGUF", + "pos": [ + -443.787109375, + 657.4349975585938 + ], + "size": [ + 398.0114440917969, + 58 + ], + "flags": { + "collapsed": true + }, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 915 + ] + } + ], + "properties": { + "cnr_id": "comfyui-warpedtoolset", + "ver": "1.7.0", + "Node name for S&R": "WarpedLoaderGGUF" + }, + "widgets_values": [ + "hunyuan-video-t2v-720p-Q4_K_M.gguf" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 298, + "type": "MagCache", + "pos": [ + -60.45900344848633, + 9.23701000213623 + ], + "size": [ + 270, + 178 + ], + "flags": { + "collapsed": true + }, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 918 + } + ], + "outputs": [ + { + "name": "model", + "type": "MODEL", + "links": [ + 919 + ] + } + ], + "properties": { + "aux_id": "Zehong-Ma/ComfyUI-MagCache", + "ver": "7d4e982bf7955498afca891c7094c48a70985537", + "Node name for S&R": "MagCache" + }, + "widgets_values": [ + "hunyuan_video", + 0.12000000000000002, + 0.2, + 3, + 0, + -1 + ] + }, + { + "id": 67, + "type": "ModelSamplingSD3", + "pos": [ + 97.61557006835938, + 7.607726097106934 + ], + "size": [ + 210, + 58 + ], + "flags": { + "collapsed": true + }, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 919 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "slot_index": 0, + "links": [ + 195 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.27", + "Node name for S&R": "ModelSamplingSD3" + }, + "widgets_values": [ + 12 + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 177, + "type": "BasicScheduler", + "pos": [ + -38.529476165771484, + 761.505126953125 + ], + "size": [ + 315, + 106 + ], + "flags": { + "collapsed": true + }, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 526 + } + ], + "outputs": [ + { + "name": "SIGMAS", + "type": "SIGMAS", + "slot_index": 0, + "links": [ + 852, + 880, + 892 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.27", + "Node name for S&R": "BasicScheduler" + }, + "widgets_values": [ + "simple", + 25, + 1 + ] + }, + { + "id": 16, + "type": "KSamplerSelect", + "pos": [ + -40.47892761230469, + 814.3021850585938 + ], + "size": [ + 315, + 58 + ], + "flags": { + "collapsed": true + }, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "SAMPLER", + "type": "SAMPLER", + "slot_index": 0, + "links": [ + 851, + 879, + 891 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.27", + "Node name for S&R": "KSamplerSelect" + }, + "widgets_values": [ + "uni_pc" + ] + }, + { + "id": 296, + "type": "WarpedDualClipLoaderGGUF", + "pos": [ + -439.79412841796875, + 722.6533203125 + ], + "size": [ + 300.9039001464844, + 130 + ], + "flags": { + "collapsed": true + }, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 916 + ] + } + ], + "properties": { + "cnr_id": "comfyui-warpedtoolset", + "ver": "1.7.0", + "Node name for S&R": "WarpedDualClipLoaderGGUF" + }, + "widgets_values": [ + "clip_l.safetensors", + "Kijai\\llava-llama-3-8B-v1_1-Q4_K_M.gguf", + "hunyuan_video", + "default" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 297, + "type": "WarpedVAELoader", + "pos": [ + -437.1321716308594, + 789.2032470703125 + ], + "size": [ + 270, + 78 + ], + "flags": { + "collapsed": true + }, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAE", + "links": [ + 917 + ] + }, + { + "name": "vae_state_dict", + "type": "DICT", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-warpedtoolset", + "ver": "1.7.0", + "Node name for S&R": "WarpedVAELoader" + }, + "widgets_values": [ + "hunyuan_video_vae_bf16.safetensors" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 146, + "type": "CLIPTextEncode", + "pos": [ + -20.676578521728516, + 160.389404296875 + ], + "size": [ + 288.2851257324219, + 88 + ], + "flags": { + "collapsed": true + }, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 916 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 448 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 352 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.27", + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "8itchW4lk, dynamic (real life:1.15) video showing front view of a beautiful stunning gorgeous woman with very long flowing brown hair with blonde highlights and exotic sexy grey eyes, she is wearing (a provocative revealing see-through scientist indiana jones style maroon satin top blouse:1.2) and see-through black thong underwear with an extremely short tight micro miniskirt low on the hips and she is braless, (she is showing her massive ((natural heavy sagging:1.3)) naked breasts:1.15), (her nipples are erect:1.25), (she is also wearing a thin silver military dogtags pendant necklace, she is in an abandonned nuclear facility, she was surprised by the viewer, so now she is walking toward the viewer to seduce him:1.25), (she is walking toward the viewer:1.2), (she is walking in a very sexy manner, smug facial expression with a slight smirk, sexy but dangerous attitude and seductive lips, normal speed, her heavy breasts (((are (bouncing:1.5) and (swaying:1.3) and (jiggling:1.7):1.7)) unrestrained with every step she takes and with ever move she makes:1.15), (((her breasts bounce into each other and recoil away from each other as she walks:1.8))), the viewer keeps her in view all the times, (extremely soft breasts:1.3), (extremely bouncing breasts:1.3), (breasts bouncing up and down side to side:1.5), detailed face, detailed hands, detailed fingers, nail polish, glamour eye makeup, eyeliner, lipstick, mascara, 4k ultra high quality video footage, (realisic jiggle, boucing, wobble, swaying physics, her breasts jiggle and bounce with each staps she take:1.2), dark atmosphere, post apocalyptic nuclear building interior, retro cyberpunk vibe", + [ + false, + true + ] + ] + }, + { + "id": 170, + "type": "StringConstantMultiline", + "pos": [ + -126.96752166748047, + 246.10122680664062 + ], + "size": [ + 423.325927734375, + 409.93310546875 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "slot_index": 0, + "links": [ + 448 + ] + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "e3b4bcf21e03402a2b11927eb5a4e3df500795b7", + "Node name for S&R": "StringConstantMultiline" + }, + "widgets_values": [ + "8itchW4lk, dynamic (real life:1.15) video showing front view of a woman with long flowing auburn_hair and brown_eyes, she is wearing (beige thigh-high stockings and a maroon garter_belt with no panties and she is topless and she is braless:1.2). (she is showing her (large:1.3) ((natural heavy:1.3)) naked breasts:1.15) pointy, large aereolas, (her nipples are erect:1.25), She has small hips and a (small waist:1.4), (she is also wearing a thin diamond choker and navel_piercing, She has a slender fit body. she is in an abandonned nuclear facility, she was surprised by the viewer, so now she is walking toward the viewer to seduce him:1.25), (she is walking toward the viewer:1.2), (she is walking in a very sexy manner, looking at the viewer. Her facial expression goes from puzzled to flirting with the viewer. sexy but dangerous attitude and seductive lips, normal speed, her heavy breasts (((are (bouncing:1.5) and (swaying:1.3) and (jiggling:1.7):1.7)) unrestrained with every step she takes and with ever move she makes:1.15), (((her breasts bounce into each other and recoil away from each other as she walks:1.8))), the viewer keeps her in view all the times, (extremely soft breasts:1.3), (extremely bouncing breasts:1.3), (breasts bouncing up and down side to side:1.5), dark-skinned female, dark skin, detailed face, detailed hands, detailed fingers, nail polish, glamour eye makeup, eyeliner, lipstick, mascara, very petite body, 4k ultra high quality video footage, (realisic jiggle, bouncing, wobble, swaying physics, her breasts jiggle and bounce with each stap she takes:1.2), dark atmosphere, flickering lights shine down from the ceiling, well lit post apocalyptic nuclear building interior with god rays illuminating the room through holes in the ceiling and walls, retro cyberpunk vibe", + true, + [ + false, + true + ] + ] } ], "links": [ @@ -998,14 +968,6 @@ 0, "MODEL" ], - [ - 270, - 10, - 0, - 113, - 0, - "*" - ], [ 352, 146, @@ -1030,38 +992,6 @@ 0, "MODEL" ], - [ - 569, - 10, - 0, - 195, - 0, - "*" - ], - [ - 869, - 273, - 0, - 146, - 0, - "CLIP" - ], - [ - 883, - 81, - 0, - 279, - 0, - "MODEL" - ], - [ - 884, - 279, - 0, - 67, - 0, - "MODEL" - ], [ 889, 195, @@ -1118,14 +1048,6 @@ 0, "LATENT" ], - [ - 909, - 275, - 0, - 292, - 0, - "MODEL" - ], [ 910, 292, @@ -1141,6 +1063,46 @@ 292, 1, "STRING" + ], + [ + 915, + 295, + 0, + 292, + 0, + "MODEL" + ], + [ + 916, + 296, + 0, + 146, + 0, + "CLIP" + ], + [ + 917, + 297, + 0, + 195, + 0, + "*" + ], + [ + 918, + 81, + 0, + 298, + 0, + "MODEL" + ], + [ + 919, + 298, + 0, + 67, + 0, + "MODEL" ] ], "groups": [ @@ -1163,14 +1125,15 @@ "ds": { "scale": 0.7513148009015777, "offset": [ - 520.9849999999964, - 78.60899999999866 + 534.2949999999944, + 55.981999999999005 ] }, + "frontendVersion": "1.20.5", "groupNodes": {}, "ue_links": [], "VHS_latentpreview": true, - "VHS_latentpreviewrate": 0, + "VHS_latentpreviewrate": 15, "node_versions": { "comfy-core": "0.3.13", "comfyui-frame-interpolation": "1.0.6", diff --git a/Sample_Workflows/Warped Hunyuan I2V Using I2V Model Workflow.json b/Sample_Workflows/Warped Hunyuan I2V Using I2V Model Workflow.json index abcfe65..8e9e1eb 100644 --- a/Sample_Workflows/Warped Hunyuan I2V Using I2V Model Workflow.json +++ b/Sample_Workflows/Warped Hunyuan I2V Using I2V Model Workflow.json @@ -1,8 +1,8 @@ { "id": "6fee7bc1-689d-4cd1-b4f9-7c20316a44f8", "revision": 0, - "last_node_id": 311, - "last_link_id": 1047, + "last_node_id": 314, + "last_link_id": 1055, "nodes": [ { "id": 152, @@ -144,7 +144,7 @@ "name": "start_image", "shape": 7, "type": "IMAGE", - "link": 1035 + "link": 1051 }, { "name": "width", @@ -152,7 +152,7 @@ "widget": { "name": "width" }, - "link": 1037 + "link": 1049 }, { "name": "height", @@ -160,7 +160,7 @@ "widget": { "name": "height" }, - "link": 1038 + "link": 1050 } ], "outputs": [ @@ -375,7 +375,7 @@ { "name": "clip", "type": "CLIP", - "link": 899 + "link": 1055 }, { "name": "clip_vision_output", @@ -419,13 +419,13 @@ 346.54290771484375 ], "flags": {}, - "order": 9, + "order": 10, "mode": 0, "inputs": [ { "name": "model", "type": "MODEL", - "link": 840 + "link": 1054 } ], "outputs": [ @@ -458,72 +458,6 @@ "all" ] }, - { - "id": 306, - "type": "WarpedImageResize", - "pos": [ - -72.92000579833984, - 1124.219482421875 - ], - "size": [ - 315, - 214 - ], - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "image", - "type": "IMAGE", - "link": 1033 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 1034, - 1035, - 1036 - ] - }, - { - "name": "scale_orig_image", - "type": "IMAGE", - "links": null - }, - { - "name": "width", - "type": "INT", - "links": [ - 1037 - ] - }, - { - "name": "height", - "type": "INT", - "links": [ - 1038 - ] - } - ], - "properties": { - "cnr_id": "ComfyUI-WarpedToolset", - "ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb", - "Node name for S&R": "WarpedImageResize" - }, - "widgets_values": [ - 400, - 720, - "lanczos", - "center", - false - ], - "color": "#323", - "bgcolor": "#535" - }, { "id": 22, "type": "BasicGuider", @@ -584,13 +518,13 @@ 476.8260192871094 ], "flags": {}, - "order": 11, + "order": 12, "mode": 0, "inputs": [ { "name": "images", "type": "IMAGE", - "link": 1034 + "link": 1053 } ], "outputs": [], @@ -624,7 +558,7 @@ "links": [ 1013, 1025, - 1033 + 1048 ] }, { @@ -807,7 +741,7 @@ 26 ], "flags": {}, - "order": 10, + "order": 9, "mode": 0, "inputs": [ { @@ -922,80 +856,6 @@ -1 ] }, - { - "id": 144, - "type": "DualCLIPLoaderGGUF", - "pos": [ - -391.7001037597656, - 433.44525146484375 - ], - "size": [ - 386.0740051269531, - 106 - ], - "flags": { - "collapsed": true - }, - "order": 4, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "CLIP", - "type": "CLIP", - "slot_index": 0, - "links": [ - 899 - ] - } - ], - "properties": { - "cnr_id": "comfyui-gguf", - "ver": "298192ed60f8ca821c6fe5f8030cae23424cada5", - "Node name for S&R": "DualCLIPLoaderGGUF" - }, - "widgets_values": [ - "clip_l.safetensors", - "Kijai\\llava-llama-3-8B-v1_1-Q4_K_M.gguf", - "hunyuan_video" - ] - }, - { - "id": 145, - "type": "UnetLoaderGGUF", - "pos": [ - -390.6521911621094, - 488.76007080078125 - ], - "size": [ - 296.70855712890625, - 65.69999694824219 - ], - "flags": { - "collapsed": true - }, - "order": 5, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "slot_index": 0, - "links": [ - 840 - ] - } - ], - "properties": { - "cnr_id": "comfyui-gguf", - "ver": "298192ed60f8ca821c6fe5f8030cae23424cada5", - "Node name for S&R": "UnetLoaderGGUF" - }, - "widgets_values": [ - "hunyuan-video-i2v-720p-Q5_K_M.gguf" - ] - }, { "id": 309, "type": "WarpedVAELoader", @@ -1010,7 +870,7 @@ "flags": { "collapsed": true }, - "order": 6, + "order": 4, "mode": 0, "inputs": [], "outputs": [ @@ -1041,53 +901,6 @@ "color": "#323", "bgcolor": "#535" }, - { - "id": 280, - "type": "CLIPVisionEncode", - "pos": [ - -386.97509765625, - 593.4584350585938 - ], - "size": [ - 336.4000244140625, - 87.9000015258789 - ], - "flags": { - "collapsed": true - }, - "order": 12, - "mode": 0, - "inputs": [ - { - "name": "clip_vision", - "type": "CLIP_VISION", - "link": 1045 - }, - { - "name": "image", - "type": "IMAGE", - "link": 1036 - } - ], - "outputs": [ - { - "name": "CLIP_VISION_OUTPUT", - "type": "CLIP_VISION_OUTPUT", - "slot_index": 0, - "links": [ - 897 - ] - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.26", - "Node name for S&R": "CLIPVisionEncode" - }, - "widgets_values": [ - "none" - ] - }, { "id": 310, "type": "WarpedCLIPVisionLoader", @@ -1102,7 +915,7 @@ "flags": { "collapsed": true }, - "order": 7, + "order": 5, "mode": 0, "inputs": [], "outputs": [ @@ -1125,6 +938,191 @@ ], "color": "#323", "bgcolor": "#535" + }, + { + "id": 312, + "type": "ImageResizeKJv2", + "pos": [ + -67.84000396728516, + 1105.3377685546875 + ], + "size": [ + 270, + 242 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 1048 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 1051, + 1052, + 1053 + ] + }, + { + "name": "width", + "type": "INT", + "links": [ + 1049 + ] + }, + { + "name": "height", + "type": "INT", + "links": [ + 1050 + ] + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "1.1.0", + "Node name for S&R": "ImageResizeKJv2" + }, + "widgets_values": [ + 400, + 720, + "lanczos", + "resize", + "0, 0, 0", + "center", + 2 + ] + }, + { + "id": 280, + "type": "CLIPVisionEncode", + "pos": [ + -386.97509765625, + 593.4584350585938 + ], + "size": [ + 336.4000244140625, + 87.9000015258789 + ], + "flags": { + "collapsed": true + }, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 1045 + }, + { + "name": "image", + "type": "IMAGE", + "link": 1052 + } + ], + "outputs": [ + { + "name": "CLIP_VISION_OUTPUT", + "type": "CLIP_VISION_OUTPUT", + "slot_index": 0, + "links": [ + 897 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.26", + "Node name for S&R": "CLIPVisionEncode" + }, + "widgets_values": [ + "none" + ] + }, + { + "id": 313, + "type": "WarpedLoaderGGUF", + "pos": [ + -391.27294921875, + 394.58453369140625 + ], + "size": [ + 417.9764404296875, + 64.65499877929688 + ], + "flags": { + "collapsed": true + }, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 1054 + ] + } + ], + "properties": { + "cnr_id": "comfyui-warpedtoolset", + "ver": "1.7.0", + "Node name for S&R": "WarpedLoaderGGUF" + }, + "widgets_values": [ + "hunyuan-video-i2v-720p-Q5_K_M.gguf" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 314, + "type": "WarpedDualClipLoaderGGUF", + "pos": [ + -388.6109924316406, + 462.46478271484375 + ], + "size": [ + 300.9039001464844, + 130 + ], + "flags": { + "collapsed": true + }, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 1055 + ] + } + ], + "properties": { + "cnr_id": "comfyui-warpedtoolset", + "ver": "1.7.0", + "Node name for S&R": "WarpedDualClipLoaderGGUF" + }, + "widgets_values": [ + "clip_l.safetensors", + "Kijai\\llava-llama-3-8B-v1_1-Q4_K_M.gguf", + "hunyuan_video", + "default" + ], + "color": "#323", + "bgcolor": "#535" } ], "links": [ @@ -1136,14 +1134,6 @@ 0, "MODEL" ], - [ - 840, - 145, - 0, - 258, - 0, - "MODEL" - ], [ 897, 280, @@ -1152,14 +1142,6 @@ 1, "CLIP_VISION_OUTPUT" ], - [ - 899, - 144, - 0, - 282, - 0, - "CLIP" - ], [ 987, 282, @@ -1240,54 +1222,6 @@ 0, "STRING" ], - [ - 1033, - 262, - 0, - 306, - 0, - "IMAGE" - ], - [ - 1034, - 306, - 0, - 255, - 0, - "IMAGE" - ], - [ - 1035, - 306, - 0, - 296, - 2, - "IMAGE" - ], - [ - 1036, - 306, - 0, - 280, - 1, - "IMAGE" - ], - [ - 1037, - 306, - 2, - 296, - 3, - "INT" - ], - [ - 1038, - 306, - 3, - 296, - 4, - "INT" - ], [ 1041, 309, @@ -1335,6 +1269,70 @@ 67, 0, "MODEL" + ], + [ + 1048, + 262, + 0, + 312, + 0, + "IMAGE" + ], + [ + 1049, + 312, + 1, + 296, + 3, + "INT" + ], + [ + 1050, + 312, + 2, + 296, + 4, + "INT" + ], + [ + 1051, + 312, + 0, + 296, + 2, + "IMAGE" + ], + [ + 1052, + 312, + 0, + 280, + 1, + "IMAGE" + ], + [ + 1053, + 312, + 0, + 255, + 0, + "IMAGE" + ], + [ + 1054, + 313, + 0, + 258, + 0, + "MODEL" + ], + [ + 1055, + 314, + 0, + 282, + 0, + "CLIP" ] ], "groups": [ @@ -1357,10 +1355,11 @@ "ds": { "scale": 0.7513148009015777, "offset": [ - 670.783000000001, - 269.58499999999214 + 612.2190000000006, + -173.6380000000081 ] }, + "frontendVersion": "1.20.5", "groupNodes": {}, "ue_links": [], "VHS_latentpreview": true, @@ -1375,8 +1374,7 @@ "Hunyuan-Multi-Lora-Loader": "dd009a9d44b47bfaf01a4ffa38d7098f523e2cce" }, "VHS_MetadataImage": true, - "VHS_KeepIntermediate": true, - "frontendVersion": "1.20.5" + "VHS_KeepIntermediate": true }, "version": 0.4 } \ No newline at end of file diff --git a/Sample_Workflows/Warped Hunyuan I2V Workflow.json b/Sample_Workflows/Warped Hunyuan I2V Workflow.json index 3078028..f384d11 100644 --- a/Sample_Workflows/Warped Hunyuan I2V Workflow.json +++ b/Sample_Workflows/Warped Hunyuan I2V Workflow.json @@ -1,8 +1,8 @@ { "id": "dfed12ef-fd2b-4870-b037-155a07e06804", "revision": 0, - "last_node_id": 291, - "last_link_id": 925, + "last_node_id": 294, + "last_link_id": 931, "nodes": [ { "id": 99, @@ -82,44 +82,6 @@ "color": "#222", "bgcolor": "#000" }, - { - "id": 144, - "type": "DualCLIPLoaderGGUF", - "pos": [ - -397.0240783691406, - 445.4241943359375 - ], - "size": [ - 386.0740051269531, - 106 - ], - "flags": { - "collapsed": true - }, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "CLIP", - "type": "CLIP", - "slot_index": 0, - "links": [ - 899 - ] - } - ], - "properties": { - "cnr_id": "ComfyUI-GGUF", - "ver": "bc5223b0e37e053dbec2ea5e5f52c2fd4b8f712a", - "Node name for S&R": "DualCLIPLoaderGGUF" - }, - "widgets_values": [ - "clip_l.safetensors", - "Kijai\\llava-llama-3-8B-v1_1-Q4_K_M.gguf", - "hunyuan_video" - ] - }, { "id": 262, "type": "LoadImage", @@ -132,7 +94,7 @@ 314 ], "flags": {}, - "order": 2, + "order": 1, "mode": 0, "inputs": [], "outputs": [ @@ -142,7 +104,7 @@ "slot_index": 0, "links": [ 900, - 913 + 926 ] }, { @@ -173,7 +135,7 @@ 476.8260192871094 ], "flags": {}, - "order": 15, + "order": 14, "mode": 0, "inputs": [ { @@ -202,13 +164,13 @@ 200 ], "flags": {}, - "order": 13, + "order": 15, "mode": 0, "inputs": [ { "name": "clip", "type": "CLIP", - "link": 899 + "link": 930 }, { "name": "clip_vision_output", @@ -249,13 +211,13 @@ 128.45428466796875 ], "flags": {}, - "order": 11, + "order": 10, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", - "link": 915 + "link": 927 } ], "outputs": [ @@ -303,7 +265,7 @@ "flags": { "collapsed": true }, - "order": 3, + "order": 2, "mode": 0, "inputs": [], "outputs": [ @@ -339,7 +301,7 @@ "flags": { "collapsed": true }, - "order": 14, + "order": 13, "mode": 0, "inputs": [ { @@ -382,7 +344,7 @@ 152.45428466796875 ], "flags": {}, - "order": 17, + "order": 16, "mode": 0, "inputs": [ { @@ -440,13 +402,13 @@ 127.22714233398438 ], "flags": {}, - "order": 12, + "order": 11, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", - "link": 916 + "link": 928 } ], "outputs": [ @@ -601,68 +563,6 @@ } } }, - { - "id": 285, - "type": "WarpedImageResize", - "pos": [ - -64.4500961303711, - 1057.6700439453125 - ], - "size": [ - 315, - 214 - ], - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "image", - "type": "IMAGE", - "link": 913 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 914, - 915, - 916 - ] - }, - { - "name": "scale_orig_image", - "type": "IMAGE", - "links": null - }, - { - "name": "width", - "type": "INT", - "links": null - }, - { - "name": "height", - "type": "INT", - "links": null - } - ], - "properties": { - "cnr_id": "ComfyUI-WarpedToolset", - "ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb", - "Node name for S&R": "WarpedImageResize" - }, - "widgets_values": [ - 544, - 960, - "lanczos", - "center", - false - ], - "color": "#323", - "bgcolor": "#535" - }, { "id": 195, "type": "Reroute", @@ -675,7 +575,7 @@ 26 ], "flags": {}, - "order": 9, + "order": 8, "mode": 0, "inputs": [ { @@ -713,7 +613,7 @@ "flags": { "collapsed": true }, - "order": 16, + "order": 17, "mode": 0, "inputs": [ { @@ -743,42 +643,6 @@ "color": "#232", "bgcolor": "#353" }, - { - "id": 145, - "type": "UnetLoaderGGUF", - "pos": [ - -391.75238037109375, - 494.3700866699219 - ], - "size": [ - 383.6085510253906, - 58 - ], - "flags": { - "collapsed": true - }, - "order": 4, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "slot_index": 0, - "links": [ - 840 - ] - } - ], - "properties": { - "cnr_id": "ComfyUI-GGUF", - "ver": "bc5223b0e37e053dbec2ea5e5f52c2fd4b8f712a", - "Node name for S&R": "UnetLoaderGGUF" - }, - "widgets_values": [ - "hunyuan-video-t2v-720p-Q4_K_M.gguf" - ] - }, { "id": 81, "type": "Reroute", @@ -831,7 +695,7 @@ "flags": { "collapsed": true }, - "order": 5, + "order": 3, "mode": 0, "inputs": [], "outputs": [ @@ -861,91 +725,6 @@ "color": "#323", "bgcolor": "#535" }, - { - "id": 280, - "type": "CLIPVisionEncode", - "pos": [ - -457.8369140625, - 1442.4373779296875 - ], - "size": [ - 380.4000244140625, - 78 - ], - "flags": { - "collapsed": true - }, - "order": 10, - "mode": 0, - "inputs": [ - { - "name": "clip_vision", - "type": "CLIP_VISION", - "link": 921 - }, - { - "name": "image", - "type": "IMAGE", - "link": 914 - } - ], - "outputs": [ - { - "name": "CLIP_VISION_OUTPUT", - "type": "CLIP_VISION_OUTPUT", - "slot_index": 0, - "links": [ - 897 - ] - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.27", - "Node name for S&R": "CLIPVisionEncode" - }, - "widgets_values": [ - "none" - ] - }, - { - "id": 288, - "type": "WarpedCLIPVisionLoader", - "pos": [ - -452.4989013671875, - 1499.314208984375 - ], - "size": [ - 315, - 58 - ], - "flags": { - "collapsed": true - }, - "order": 6, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "clip_vision", - "type": "CLIP_VISION", - "links": [ - 921 - ] - } - ], - "properties": { - "cnr_id": "ComfyUI-WarpedToolset", - "ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb", - "Node name for S&R": "WarpedCLIPVisionLoader", - "widget_ue_connectable": {} - }, - "widgets_values": [ - "clip_vision_h_fp8_e4m3fn.safetensors" - ], - "color": "#323", - "bgcolor": "#535" - }, { "id": 67, "type": "ModelSamplingSD3", @@ -1048,13 +827,13 @@ 346.54290771484375 ], "flags": {}, - "order": 8, + "order": 9, "mode": 0, "inputs": [ { "name": "model", "type": "MODEL", - "link": 840 + "link": 931 } ], "outputs": [ @@ -1348,6 +1127,226 @@ "ver": "1.7.0", "Node name for S&R": "WarpedNumericalConversion" }, + "widgets_values": [], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 280, + "type": "CLIPVisionEncode", + "pos": [ + -445.85797119140625, + 1389.1964111328125 + ], + "size": [ + 380.4000244140625, + 78 + ], + "flags": { + "collapsed": true + }, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 921 + }, + { + "name": "image", + "type": "IMAGE", + "link": 929 + } + ], + "outputs": [ + { + "name": "CLIP_VISION_OUTPUT", + "type": "CLIP_VISION_OUTPUT", + "slot_index": 0, + "links": [ + 897 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.27", + "Node name for S&R": "CLIPVisionEncode" + }, + "widgets_values": [ + "none" + ] + }, + { + "id": 288, + "type": "WarpedCLIPVisionLoader", + "pos": [ + -440.51995849609375, + 1456.720703125 + ], + "size": [ + 315, + 58 + ], + "flags": { + "collapsed": true + }, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "clip_vision", + "type": "CLIP_VISION", + "links": [ + 921 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WarpedToolset", + "ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb", + "Node name for S&R": "WarpedCLIPVisionLoader", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "clip_vision_h_fp8_e4m3fn.safetensors" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 292, + "type": "ImageResizeKJv2", + "pos": [ + -59.854007720947266, + 1054.7607421875 + ], + "size": [ + 270, + 242 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 926 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 927, + 928, + 929 + ] + }, + { + "name": "width", + "type": "INT", + "links": null + }, + { + "name": "height", + "type": "INT", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "1.1.0", + "Node name for S&R": "ImageResizeKJv2" + }, + "widgets_values": [ + 544, + 960, + "lanczos", + "resize", + "0, 0, 0", + "center", + 2 + ] + }, + { + "id": 293, + "type": "WarpedDualClipLoaderGGUF", + "pos": [ + -409.9068603515625, + 435.8446960449219 + ], + "size": [ + 300.9039001464844, + 130 + ], + "flags": { + "collapsed": true + }, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 930 + ] + } + ], + "properties": { + "cnr_id": "comfyui-warpedtoolset", + "ver": "1.7.0", + "Node name for S&R": "WarpedDualClipLoaderGGUF" + }, + "widgets_values": [ + "clip_l.safetensors", + "Kijai\\llava-llama-3-8B-v1_1-Q4_K_M.gguf", + "hunyuan_video", + "default" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 294, + "type": "WarpedLoaderGGUF", + "pos": [ + -396.5967102050781, + 491.7479248046875 + ], + "size": [ + 390.02545166015625, + 61.99300003051758 + ], + "flags": { + "collapsed": true + }, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 931 + ] + } + ], + "properties": { + "cnr_id": "comfyui-warpedtoolset", + "ver": "1.7.0", + "Node name for S&R": "WarpedLoaderGGUF" + }, + "widgets_values": [ + "hunyuan-video-t2v-720p-Q4_K_M.gguf" + ], "color": "#323", "bgcolor": "#535" } @@ -1377,14 +1376,6 @@ 0, "MODEL" ], - [ - 840, - 145, - 0, - 258, - 0, - "MODEL" - ], [ 878, 195, @@ -1497,14 +1488,6 @@ 0, "CONDITIONING" ], - [ - 899, - 144, - 0, - 282, - 0, - "CLIP" - ], [ 907, 284, @@ -1521,38 +1504,6 @@ 0, "IMAGE" ], - [ - 913, - 262, - 0, - 285, - 0, - "IMAGE" - ], - [ - 914, - 285, - 0, - 280, - 1, - "IMAGE" - ], - [ - 915, - 285, - 0, - 274, - 0, - "IMAGE" - ], - [ - 916, - 285, - 0, - 284, - 0, - "IMAGE" - ], [ 919, 287, @@ -1608,6 +1559,54 @@ 290, 0, "STRING" + ], + [ + 926, + 262, + 0, + 292, + 0, + "IMAGE" + ], + [ + 927, + 292, + 0, + 274, + 0, + "IMAGE" + ], + [ + 928, + 292, + 0, + 284, + 0, + "IMAGE" + ], + [ + 929, + 292, + 0, + 280, + 1, + "IMAGE" + ], + [ + 930, + 293, + 0, + 282, + 0, + "CLIP" + ], + [ + 931, + 294, + 0, + 258, + 0, + "MODEL" ] ], "groups": [ @@ -1630,10 +1629,11 @@ "ds": { "scale": 0.7513148009015777, "offset": [ - 608.2260000000005, - -152.3420000000077 + 597.5779999999984, + -401.2390000000077 ] }, + "frontendVersion": "1.20.5", "groupNodes": {}, "ue_links": [], "VHS_latentpreview": true, @@ -1648,8 +1648,7 @@ "comfyui-videohelpersuite": "1.5.2" }, "VHS_MetadataImage": true, - "VHS_KeepIntermediate": true, - "frontendVersion": "1.20.5" + "VHS_KeepIntermediate": true }, "version": 0.4 } \ No newline at end of file diff --git a/Sample_Workflows/Warped Hunyuan T2V Workflow.json b/Sample_Workflows/Warped Hunyuan T2V Workflow.json index f3b0d81..a9bebeb 100644 --- a/Sample_Workflows/Warped Hunyuan T2V Workflow.json +++ b/Sample_Workflows/Warped Hunyuan T2V Workflow.json @@ -1,8 +1,8 @@ { "id": "facb86ea-4076-46ff-921d-e6dcadfb1754", "revision": 0, - "last_node_id": 289, - "last_link_id": 908, + "last_node_id": 291, + "last_link_id": 910, "nodes": [ { "id": 99, @@ -35,41 +35,6 @@ "toggleRestriction": "default" } }, - { - "id": 275, - "type": "UnetLoaderGGUF", - "pos": [ - -446.1169128417969, - 342.95745849609375 - ], - "size": [ - 315, - 58 - ], - "flags": { - "collapsed": true - }, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 872 - ] - } - ], - "properties": { - "cnr_id": "ComfyUI-GGUF", - "ver": "bc5223b0e37e053dbec2ea5e5f52c2fd4b8f712a", - "Node name for S&R": "UnetLoaderGGUF" - }, - "widgets_values": [ - "hunyuan-video-t2v-720p-Q4_K_M.gguf" - ] - }, { "id": 26, "type": "FluxGuidance", @@ -84,7 +49,7 @@ "flags": { "collapsed": true }, - "order": 11, + "order": 10, "mode": 0, "inputs": [ { @@ -163,95 +128,6 @@ "color": "#222", "bgcolor": "#000" }, - { - "id": 273, - "type": "DualCLIPLoaderGGUF", - "pos": [ - -442.2043762207031, - 386.2261047363281 - ], - "size": [ - 301.0328369140625, - 106 - ], - "flags": { - "collapsed": true - }, - "order": 2, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "CLIP", - "type": "CLIP", - "slot_index": 0, - "links": [ - 869 - ] - } - ], - "properties": { - "cnr_id": "ComfyUI-GGUF", - "ver": "bc5223b0e37e053dbec2ea5e5f52c2fd4b8f712a", - "Node name for S&R": "DualCLIPLoaderGGUF" - }, - "widgets_values": [ - "clip_l.safetensors", - "Kijai\\llava-llama-3-8B-v1_1-Q4_K_M.gguf", - "hunyuan_video" - ] - }, - { - "id": 146, - "type": "CLIPTextEncode", - "pos": [ - -6.035816669464111, - 173.69970703125 - ], - "size": [ - 288.2851257324219, - 78.88789367675781 - ], - "flags": { - "collapsed": true - }, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 869 - }, - { - "name": "text", - "type": "STRING", - "widget": { - "name": "text" - }, - "link": 448 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "slot_index": 0, - "links": [ - 352 - ] - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.27", - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "8itchW4lk, dynamic (real life:1.15) video showing front view of a beautiful stunning gorgeous woman with very long flowing brown hair with blonde highlights and exotic sexy grey eyes, she is wearing (a provocative revealing see-through scientist indiana jones style maroon satin top blouse:1.2) and see-through black thong underwear with an extremely short tight micro miniskirt low on the hips and she is braless, (she is showing her massive ((natural heavy sagging:1.3)) naked breasts:1.15), (her nipples are erect:1.25), (she is also wearing a thin silver military dogtags pendant necklace, she is in an abandonned nuclear facility, she was surprised by the viewer, so now she is walking toward the viewer to seduce him:1.25), (she is walking toward the viewer:1.2), (she is walking in a very sexy manner, smug facial expression with a slight smirk, sexy but dangerous attitude and seductive lips, normal speed, her heavy breasts (((are (bouncing:1.5) and (swaying:1.3) and (jiggling:1.7):1.7)) unrestrained with every step she takes and with ever move she makes:1.15), (((her breasts bounce into each other and recoil away from each other as she walks:1.8))), the viewer keeps her in view all the times, (extremely soft breasts:1.3), (extremely bouncing breasts:1.3), (breasts bouncing up and down side to side:1.5), detailed face, detailed hands, detailed fingers, nail polish, glamour eye makeup, eyeliner, lipstick, mascara, 4k ultra high quality video footage, (realisic jiggle, boucing, wobble, swaying physics, her breasts jiggle and bounce with each staps she take:1.2), dark atmosphere, post apocalyptic nuclear building interior, retro cyberpunk vibe", - true - ] - }, { "id": 81, "type": "Reroute", @@ -264,7 +140,7 @@ 26 ], "flags": {}, - "order": 10, + "order": 11, "mode": 0, "inputs": [ { @@ -302,7 +178,7 @@ 26 ], "flags": {}, - "order": 9, + "order": 7, "mode": 0, "inputs": [ { @@ -387,7 +263,7 @@ "flags": { "collapsed": true }, - "order": 3, + "order": 1, "mode": 0, "inputs": [], "outputs": [ @@ -502,7 +378,7 @@ 409.93310546875 ], "flags": {}, - "order": 4, + "order": 2, "mode": 0, "inputs": [], "outputs": [ @@ -526,49 +402,6 @@ true ] }, - { - "id": 286, - "type": "WarpedVAELoader", - "pos": [ - -439.18890380859375, - 438.5068359375 - ], - "size": [ - 315, - 58 - ], - "flags": { - "collapsed": true - }, - "order": 5, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "vae", - "type": "VAE", - "links": [ - 904 - ] - }, - { - "name": "vae_state_dict", - "type": "DICT", - "links": null - } - ], - "properties": { - "cnr_id": "ComfyUI-WarpedToolset", - "ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb", - "Node name for S&R": "WarpedVAELoader", - "widget_ue_connectable": {} - }, - "widgets_values": [ - "hunyuan_video_vae_bf16.safetensors" - ], - "color": "#323", - "bgcolor": "#535" - }, { "id": 67, "type": "ModelSamplingSD3", @@ -671,7 +504,7 @@ 126 ], "flags": {}, - "order": 6, + "order": 3, "mode": 0, "inputs": [], "outputs": [ @@ -939,6 +772,7 @@ "ver": "1.7.0", "Node name for S&R": "WarpedNumericalConversion" }, + "widgets_values": [], "color": "#323", "bgcolor": "#535" }, @@ -954,13 +788,13 @@ 345.9579772949219 ], "flags": {}, - "order": 7, + "order": 9, "mode": 0, "inputs": [ { "name": "model", "type": "MODEL", - "link": 872 + "link": 910 } ], "outputs": [ @@ -993,6 +827,177 @@ 1, "all" ] + }, + { + "id": 146, + "type": "CLIPTextEncode", + "pos": [ + -15.35281753540039, + 169.70672607421875 + ], + "size": [ + 288.2851257324219, + 81 + ], + "flags": { + "collapsed": true + }, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 909 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 448 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 352 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.27", + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "8itchW4lk, dynamic (real life:1.15) video showing front view of a beautiful stunning gorgeous woman with very long flowing brown hair with blonde highlights and exotic sexy grey eyes, she is wearing (a provocative revealing see-through scientist indiana jones style maroon satin top blouse:1.2) and see-through black thong underwear with an extremely short tight micro miniskirt low on the hips and she is braless, (she is showing her massive ((natural heavy sagging:1.3)) naked breasts:1.15), (her nipples are erect:1.25), (she is also wearing a thin silver military dogtags pendant necklace, she is in an abandonned nuclear facility, she was surprised by the viewer, so now she is walking toward the viewer to seduce him:1.25), (she is walking toward the viewer:1.2), (she is walking in a very sexy manner, smug facial expression with a slight smirk, sexy but dangerous attitude and seductive lips, normal speed, her heavy breasts (((are (bouncing:1.5) and (swaying:1.3) and (jiggling:1.7):1.7)) unrestrained with every step she takes and with ever move she makes:1.15), (((her breasts bounce into each other and recoil away from each other as she walks:1.8))), the viewer keeps her in view all the times, (extremely soft breasts:1.3), (extremely bouncing breasts:1.3), (breasts bouncing up and down side to side:1.5), detailed face, detailed hands, detailed fingers, nail polish, glamour eye makeup, eyeliner, lipstick, mascara, 4k ultra high quality video footage, (realisic jiggle, boucing, wobble, swaying physics, her breasts jiggle and bounce with each staps she take:1.2), dark atmosphere, post apocalyptic nuclear building interior, retro cyberpunk vibe", + true + ] + }, + { + "id": 286, + "type": "WarpedVAELoader", + "pos": [ + -436.52691650390625, + 451.8167724609375 + ], + "size": [ + 315, + 58 + ], + "flags": { + "collapsed": true + }, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAE", + "links": [ + 904 + ] + }, + { + "name": "vae_state_dict", + "type": "DICT", + "links": null + } + ], + "properties": { + "cnr_id": "ComfyUI-WarpedToolset", + "ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb", + "Node name for S&R": "WarpedVAELoader", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "hunyuan_video_vae_bf16.safetensors" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 290, + "type": "WarpedDualClipLoaderGGUF", + "pos": [ + -435.19573974609375, + 405.23223876953125 + ], + "size": [ + 300.9039001464844, + 130 + ], + "flags": { + "collapsed": true + }, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 909 + ] + } + ], + "properties": { + "cnr_id": "comfyui-warpedtoolset", + "ver": "1.7.0", + "Node name for S&R": "WarpedDualClipLoaderGGUF" + }, + "widgets_values": [ + "clip_l.safetensors", + "Kijai\\llava-llama-3-8B-v1_1-Q4_K_M.gguf", + "hunyuan_video", + "default" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 291, + "type": "WarpedLoaderGGUF", + "pos": [ + -436.5267639160156, + 354.6539306640625 + ], + "size": [ + 282.2144470214844, + 58 + ], + "flags": { + "collapsed": true + }, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 910 + ] + } + ], + "properties": { + "cnr_id": "comfyui-warpedtoolset", + "ver": "1.7.0", + "Node name for S&R": "WarpedLoaderGGUF" + }, + "widgets_values": [ + "hunyuan-video-t2v-720p-Q4_K_M.gguf" + ], + "color": "#323", + "bgcolor": "#535" } ], "links": [ @@ -1036,22 +1041,6 @@ 0, "MODEL" ], - [ - 869, - 273, - 0, - 146, - 0, - "CLIP" - ], - [ - 872, - 275, - 0, - 258, - 0, - "MODEL" - ], [ 873, 258, @@ -1155,6 +1144,22 @@ 289, 0, "STRING" + ], + [ + 909, + 290, + 0, + 146, + 0, + "CLIP" + ], + [ + 910, + 291, + 0, + 258, + 0, + "MODEL" ] ], "groups": [ @@ -1177,10 +1182,11 @@ "ds": { "scale": 0.7513148009015777, "offset": [ - 541.6760000000013, - 149.79499999999234 + 668.1210000000013, + 107.20299999999239 ] }, + "frontendVersion": "1.20.5", "groupNodes": {}, "ue_links": [], "VHS_latentpreview": true, @@ -1199,8 +1205,7 @@ "Hunyuan-Multi-Lora-Loader": "dd009a9d44b47bfaf01a4ffa38d7098f523e2cce" }, "VHS_MetadataImage": true, - "VHS_KeepIntermediate": true, - "frontendVersion": "1.20.5" + "VHS_KeepIntermediate": true }, "version": 0.4 } \ No newline at end of file diff --git a/Sample_Workflows/Warped Hunyuan v2v Workflow.json b/Sample_Workflows/Warped Hunyuan v2v Workflow.json index d62fde8..81dbf4f 100644 --- a/Sample_Workflows/Warped Hunyuan v2v Workflow.json +++ b/Sample_Workflows/Warped Hunyuan v2v Workflow.json @@ -1,8 +1,8 @@ { "id": "cd4d8b23-c2f1-4f41-b28f-b13b2b63a361", "revision": 0, - "last_node_id": 303, - "last_link_id": 957, + "last_node_id": 306, + "last_link_id": 963, "nodes": [ { "id": 152, @@ -227,79 +227,6 @@ "color": "#222", "bgcolor": "#000" }, - { - "id": 273, - "type": "DualCLIPLoaderGGUF", - "pos": [ - -457.2220153808594, - 416.15118408203125 - ], - "size": [ - 301.0328369140625, - 106 - ], - "flags": { - "collapsed": true - }, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "CLIP", - "type": "CLIP", - "slot_index": 0, - "links": [ - 869 - ] - } - ], - "properties": { - "cnr_id": "ComfyUI-GGUF", - "ver": "bc5223b0e37e053dbec2ea5e5f52c2fd4b8f712a", - "Node name for S&R": "DualCLIPLoaderGGUF" - }, - "widgets_values": [ - "clip_l.safetensors", - "Kijai\\llava-llama-3-8B-v1_1-Q4_K_M.gguf", - "hunyuan_video" - ] - }, - { - "id": 275, - "type": "UnetLoaderGGUF", - "pos": [ - -448.9233093261719, - 363.3001708984375 - ], - "size": [ - 315, - 58 - ], - "flags": { - "collapsed": true - }, - "order": 2, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 872 - ] - } - ], - "properties": { - "cnr_id": "ComfyUI-GGUF", - "ver": "bc5223b0e37e053dbec2ea5e5f52c2fd4b8f712a", - "Node name for S&R": "UnetLoaderGGUF" - }, - "widgets_values": [ - "hunyuan-video-t2v-720p-Q4_K_M.gguf" - ] - }, { "id": 81, "type": "Reroute", @@ -338,93 +265,6 @@ "horizontal": false } }, - { - "id": 146, - "type": "CLIPTextEncode", - "pos": [ - 2.7562851905822754, - 169.0845947265625 - ], - "size": [ - 288.2851257324219, - 78.88789367675781 - ], - "flags": { - "collapsed": true - }, - "order": 30, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 869 - }, - { - "name": "text", - "type": "STRING", - "widget": { - "name": "text" - }, - "link": 448 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "slot_index": 0, - "links": [ - 352 - ] - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.27", - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "8itchW4lk, dynamic (real life:1.15) video showing front view of a beautiful stunning gorgeous woman with very long flowing brown hair with blonde highlights and exotic sexy grey eyes, she is wearing (a provocative revealing see-through scientist indiana jones style maroon satin top blouse:1.2) and see-through black thong underwear with an extremely short tight micro miniskirt low on the hips and she is braless, (she is showing her massive ((natural heavy sagging:1.3)) naked breasts:1.15), (her nipples are erect:1.25), (she is also wearing a thin silver military dogtags pendant necklace, she is in an abandonned nuclear facility, she was surprised by the viewer, so now she is walking toward the viewer to seduce him:1.25), (she is walking toward the viewer:1.2), (she is walking in a very sexy manner, smug facial expression with a slight smirk, sexy but dangerous attitude and seductive lips, normal speed, her heavy breasts (((are (bouncing:1.5) and (swaying:1.3) and (jiggling:1.7):1.7)) unrestrained with every step she takes and with ever move she makes:1.15), (((her breasts bounce into each other and recoil away from each other as she walks:1.8))), the viewer keeps her in view all the times, (extremely soft breasts:1.3), (extremely bouncing breasts:1.3), (breasts bouncing up and down side to side:1.5), detailed face, detailed hands, detailed fingers, nail polish, glamour eye makeup, eyeliner, lipstick, mascara, 4k ultra high quality video footage, (realisic jiggle, boucing, wobble, swaying physics, her breasts jiggle and bounce with each staps she take:1.2), dark atmosphere, post apocalyptic nuclear building interior, retro cyberpunk vibe", - true - ] - }, - { - "id": 195, - "type": "Reroute", - "pos": [ - 20.86060333251953, - 238.17318725585938 - ], - "size": [ - 75, - 26 - ], - "flags": {}, - "order": 10, - "mode": 0, - "inputs": [ - { - "name": "", - "type": "*", - "link": 955 - } - ], - "outputs": [ - { - "name": "", - "type": "VAE", - "slot_index": 0, - "links": [ - 914 - ] - } - ], - "properties": { - "showOutputText": false, - "horizontal": false - } - }, { "id": 289, "type": "WarpedLeapfusionHunyuanI2V", @@ -495,7 +335,7 @@ 281.7099914550781 ], "flags": {}, - "order": 9, + "order": 8, "mode": 0, "inputs": [ { @@ -596,7 +436,7 @@ { "name": "image", "type": "IMAGE", - "link": 953 + "link": 961 }, { "name": "vae", @@ -700,13 +540,13 @@ 476.8260192871094 ], "flags": {}, - "order": 11, + "order": 12, "mode": 0, "inputs": [ { "name": "images", "type": "IMAGE", - "link": 951 + "link": 960 } ], "outputs": [], @@ -984,7 +824,7 @@ "flags": { "collapsed": true }, - "order": 3, + "order": 1, "mode": 0, "inputs": [], "outputs": [ @@ -1120,13 +960,13 @@ 126 ], "flags": {}, - "order": 12, + "order": 11, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", - "link": 952 + "link": 959 } ], "outputs": [ @@ -1230,7 +1070,7 @@ 250 ], "flags": {}, - "order": 4, + "order": 2, "mode": 0, "inputs": [], "outputs": [ @@ -1240,7 +1080,7 @@ "slot_index": 0, "links": [ 940, - 950 + 958 ] }, { @@ -1422,7 +1262,7 @@ "flags": { "collapsed": false }, - "order": 5, + "order": 3, "mode": 0, "inputs": [], "outputs": [ @@ -1603,7 +1443,7 @@ "flags": { "collapsed": true }, - "order": 6, + "order": 4, "mode": 0, "inputs": [], "outputs": [ @@ -1633,68 +1473,6 @@ "color": "#323", "bgcolor": "#535" }, - { - "id": 301, - "type": "WarpedImageResize", - "pos": [ - -457.73272705078125, - 1149.1563720703125 - ], - "size": [ - 315, - 214 - ], - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "image", - "type": "IMAGE", - "link": 950 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 951, - 952, - 953 - ] - }, - { - "name": "scale_orig_image", - "type": "IMAGE", - "links": null - }, - { - "name": "width", - "type": "INT", - "links": null - }, - { - "name": "height", - "type": "INT", - "links": null - } - ], - "properties": { - "cnr_id": "ComfyUI-WarpedToolset", - "ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb", - "Node name for S&R": "WarpedImageResize" - }, - "widgets_values": [ - 512, - 288, - "lanczos", - "center", - false - ], - "color": "#323", - "bgcolor": "#535" - }, { "id": 67, "type": "ModelSamplingSD3", @@ -1797,13 +1575,13 @@ 322 ], "flags": {}, - "order": 7, + "order": 10, "mode": 0, "inputs": [ { "name": "model", "type": "MODEL", - "link": 872 + "link": 963 } ], "outputs": [ @@ -1836,6 +1614,227 @@ 1, "all" ] + }, + { + "id": 304, + "type": "ImageResizeKJv2", + "pos": [ + -453.8300476074219, + 1161.2410888671875 + ], + "size": [ + 270, + 242 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 958 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 959, + 960, + 961 + ] + }, + { + "name": "width", + "type": "INT", + "links": null + }, + { + "name": "height", + "type": "INT", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "1.1.0", + "Node name for S&R": "ImageResizeKJv2" + }, + "widgets_values": [ + 512, + 288, + "lanczos", + "resize", + "0, 0, 0", + "center", + 2 + ] + }, + { + "id": 305, + "type": "WarpedDualClipLoaderGGUF", + "pos": [ + -445.8439636230469, + 415.8797912597656 + ], + "size": [ + 300.9039001464844, + 130 + ], + "flags": { + "collapsed": true + }, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 962 + ] + } + ], + "properties": { + "cnr_id": "comfyui-warpedtoolset", + "ver": "1.7.0", + "Node name for S&R": "WarpedDualClipLoaderGGUF" + }, + "widgets_values": [ + "clip_l.safetensors", + "Kijai\\llava-llama-3-8B-v1_1-Q4_K_M.gguf", + "sdxl", + "default" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 306, + "type": "WarpedLoaderGGUF", + "pos": [ + -449.8374328613281, + 358.64703369140625 + ], + "size": [ + 282.2144470214844, + 58 + ], + "flags": { + "collapsed": true + }, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 963 + ] + } + ], + "properties": { + "cnr_id": "comfyui-warpedtoolset", + "ver": "1.7.0", + "Node name for S&R": "WarpedLoaderGGUF" + }, + "widgets_values": [ + "hunyuan-video-t2v-720p-Q4_K_M.gguf" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 146, + "type": "CLIPTextEncode", + "pos": [ + 2.7562851905822754, + 169.0845947265625 + ], + "size": [ + 288.2851257324219, + 81 + ], + "flags": { + "collapsed": true + }, + "order": 30, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 962 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 448 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 352 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.27", + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "8itchW4lk, dynamic (real life:1.15) video showing front view of a beautiful stunning gorgeous woman with very long flowing brown hair with blonde highlights and exotic sexy grey eyes, she is wearing (a provocative revealing see-through scientist indiana jones style maroon satin top blouse:1.2) and see-through black thong underwear with an extremely short tight micro miniskirt low on the hips and she is braless, (she is showing her massive ((natural heavy sagging:1.3)) naked breasts:1.15), (her nipples are erect:1.25), (she is also wearing a thin silver military dogtags pendant necklace, she is in an abandonned nuclear facility, she was surprised by the viewer, so now she is walking toward the viewer to seduce him:1.25), (she is walking toward the viewer:1.2), (she is walking in a very sexy manner, smug facial expression with a slight smirk, sexy but dangerous attitude and seductive lips, normal speed, her heavy breasts (((are (bouncing:1.5) and (swaying:1.3) and (jiggling:1.7):1.7)) unrestrained with every step she takes and with ever move she makes:1.15), (((her breasts bounce into each other and recoil away from each other as she walks:1.8))), the viewer keeps her in view all the times, (extremely soft breasts:1.3), (extremely bouncing breasts:1.3), (breasts bouncing up and down side to side:1.5), detailed face, detailed hands, detailed fingers, nail polish, glamour eye makeup, eyeliner, lipstick, mascara, 4k ultra high quality video footage, (realisic jiggle, boucing, wobble, swaying physics, her breasts jiggle and bounce with each staps she take:1.2), dark atmosphere, post apocalyptic nuclear building interior, retro cyberpunk vibe", + true + ] + }, + { + "id": 195, + "type": "Reroute", + "pos": [ + 7.55060338973999, + 239.50413513183594 + ], + "size": [ + 75, + 26 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "", + "type": "*", + "link": 955 + } + ], + "outputs": [ + { + "name": "", + "type": "VAE", + "slot_index": 0, + "links": [ + 914 + ] + } + ], + "properties": { + "showOutputText": false, + "horizontal": false + } } ], "links": [ @@ -1879,22 +1878,6 @@ 0, "MODEL" ], - [ - 869, - 273, - 0, - 146, - 0, - "CLIP" - ], - [ - 872, - 275, - 0, - 258, - 0, - "MODEL" - ], [ 890, 282, @@ -2095,38 +2078,6 @@ 0, "IMAGE" ], - [ - 950, - 296, - 0, - 301, - 0, - "IMAGE" - ], - [ - 951, - 301, - 0, - 255, - 0, - "IMAGE" - ], - [ - 952, - 301, - 0, - 290, - 0, - "IMAGE" - ], - [ - 953, - 301, - 0, - 288, - 0, - "IMAGE" - ], [ 954, 302, @@ -2158,6 +2109,54 @@ 67, 0, "MODEL" + ], + [ + 958, + 296, + 0, + 304, + 0, + "IMAGE" + ], + [ + 959, + 304, + 0, + 290, + 0, + "IMAGE" + ], + [ + 960, + 304, + 0, + 255, + 0, + "IMAGE" + ], + [ + 961, + 304, + 0, + 288, + 0, + "IMAGE" + ], + [ + 962, + 305, + 0, + 146, + 0, + "CLIP" + ], + [ + 963, + 306, + 0, + 258, + 0, + "MODEL" ] ], "groups": [ @@ -2180,10 +2179,11 @@ "ds": { "scale": 0.7513148009015777, "offset": [ - 540.3449999999999, - 148.46399999999193 + 568.2959999999991, + 18.025999999991825 ] }, + "frontendVersion": "1.20.5", "groupNodes": {}, "ue_links": [], "VHS_latentpreview": true, @@ -2201,8 +2201,7 @@ "Hunyuan-Multi-Lora-Loader": "dd009a9d44b47bfaf01a4ffa38d7098f523e2cce" }, "VHS_MetadataImage": true, - "VHS_KeepIntermediate": true, - "frontendVersion": "1.20.5" + "VHS_KeepIntermediate": true }, "version": 0.4 } \ No newline at end of file diff --git a/Sample_Workflows/Warped Midres Upscale Workflow.json b/Sample_Workflows/Warped Midres Upscale Workflow.json index af51773..79a96c8 100644 --- a/Sample_Workflows/Warped Midres Upscale Workflow.json +++ b/Sample_Workflows/Warped Midres Upscale Workflow.json @@ -1,8 +1,8 @@ { "id": "0a50bfc6-2def-47da-9774-2630f586b894", "revision": 0, - "last_node_id": 281, - "last_link_id": 907, + "last_node_id": 285, + "last_link_id": 914, "nodes": [ { "id": 152, @@ -22,9 +22,6 @@ { "name": "text", "type": "STRING", - "widget": { - "name": "text" - }, "link": 899 } ], @@ -42,7 +39,6 @@ }, "widgets_values": [ "", - "****** WarpedSamplerCustomAdv: Total Images Generated 129 ******", [ false, true @@ -187,7 +183,6 @@ "outputs": [ { "name": "CONDITIONING", - "shape": 3, "type": "CONDITIONING", "slot_index": 0, "links": [ @@ -206,57 +201,6 @@ "color": "#232", "bgcolor": "#353" }, - { - "id": 146, - "type": "CLIPTextEncode", - "pos": [ - 16.21417999267578, - 237.3597869873047 - ], - "size": [ - 288.2851257324219, - 78.88789367675781 - ], - "flags": { - "collapsed": true - }, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 366 - }, - { - "name": "text", - "type": "STRING", - "widget": { - "name": "text" - }, - "link": 448 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "slot_index": 0, - "links": [ - 352 - ] - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.27", - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "8itchW4lk, dynamic (real life:1.15) video showing front view of a beautiful stunning gorgeous woman with very long flowing brown hair with blonde highlights and exotic sexy grey eyes, she is wearing (a provocative revealing see-through scientist indiana jones style maroon satin top blouse:1.2) and see-through black thong underwear with an extremely short tight micro miniskirt low on the hips and she is braless, (she is showing her massive ((natural heavy sagging:1.3)) naked breasts:1.15), (her nipples are erect:1.25), (she is also wearing a thin silver military dogtags pendant necklace, she is in an abandonned nuclear facility, she was surprised by the viewer, so now she is walking toward the viewer to seduce him:1.25), (she is walking toward the viewer:1.2), (she is walking in a very sexy manner, smug facial expression with a slight smirk, sexy but dangerous attitude and seductive lips, normal speed, her heavy breasts (((are (bouncing:1.5) and (swaying:1.3) and (jiggling:1.7):1.7)) unrestrained with every step she takes and with ever move she makes:1.15), (((her breasts bounce into each other and recoil away from each other as she walks:1.8))), the viewer keeps her in view all the times, (extremely soft breasts:1.3), (extremely bouncing breasts:1.3), (breasts bouncing up and down side to side:1.5), detailed face, detailed hands, detailed fingers, nail polish, glamour eye makeup, eyeliner, lipstick, mascara, 4k ultra high quality video footage, (realisic jiggle, boucing, wobble, swaying physics, her breasts jiggle and bounce with each staps she take:1.2), dark atmosphere, post apocalyptic nuclear building interior, retro cyberpunk vibe", - true - ] - }, { "id": 22, "type": "BasicGuider", @@ -271,7 +215,7 @@ "flags": { "collapsed": true }, - "order": 19, + "order": 20, "mode": 0, "inputs": [ { @@ -288,7 +232,6 @@ "outputs": [ { "name": "GUIDER", - "shape": 3, "type": "GUIDER", "slot_index": 0, "links": [ @@ -305,96 +248,6 @@ "color": "#222", "bgcolor": "#000" }, - { - "id": 67, - "type": "ModelSamplingSD3", - "pos": [ - -4.116424560546875, - 127.65672302246094 - ], - "size": [ - 210, - 58 - ], - "flags": { - "collapsed": true - }, - "order": 17, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 901 - } - ], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "slot_index": 0, - "links": [ - 195 - ] - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.27", - "Node name for S&R": "ModelSamplingSD3" - }, - "widgets_values": [ - 12 - ], - "color": "#222", - "bgcolor": "#000" - }, - { - "id": 278, - "type": "WarpedTeaCache", - "pos": [ - -2, - 75 - ], - "size": [ - 315, - 106 - ], - "flags": { - "collapsed": true - }, - "order": 15, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 900 - } - ], - "outputs": [ - { - "name": "model", - "type": "MODEL", - "slot_index": 0, - "links": [ - 901 - ] - } - ], - "properties": { - "cnr_id": "ComfyUI-WarpedToolset", - "ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb", - "Node name for S&R": "WarpedTeaCache" - }, - "widgets_values": [ - "hunyuan_video", - 0.4, - 1 - ], - "color": "#323", - "bgcolor": "#535" - }, { "id": 258, "type": "HunyuanMultiLoraLoader", @@ -407,13 +260,13 @@ 322 ], "flags": {}, - "order": 7, + "order": 10, "mode": 0, "inputs": [ { "name": "model", "type": "MODEL", - "link": 840 + "link": 914 } ], "outputs": [ @@ -458,7 +311,7 @@ 26 ], "flags": {}, - "order": 11, + "order": 13, "mode": 0, "inputs": [ { @@ -474,7 +327,7 @@ "slot_index": 0, "links": [ 288, - 900 + 908 ] } ], @@ -483,80 +336,6 @@ "horizontal": false } }, - { - "id": 144, - "type": "DualCLIPLoaderGGUF", - "pos": [ - -433.17376708984375, - 369.4688415527344 - ], - "size": [ - 386.0740051269531, - 106 - ], - "flags": { - "collapsed": true - }, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "CLIP", - "type": "CLIP", - "slot_index": 0, - "links": [ - 366 - ] - } - ], - "properties": { - "cnr_id": "ComfyUI-GGUF", - "ver": "bc5223b0e37e053dbec2ea5e5f52c2fd4b8f712a", - "Node name for S&R": "DualCLIPLoaderGGUF" - }, - "widgets_values": [ - "clip_l.safetensors", - "Kijai\\llava-llama-3-8B-v1_1-Q4_K_M.gguf", - "hunyuan_video" - ] - }, - { - "id": 145, - "type": "UnetLoaderGGUF", - "pos": [ - -429.0895080566406, - 425.6887512207031 - ], - "size": [ - 383.6085510253906, - 58 - ], - "flags": { - "collapsed": true - }, - "order": 2, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "slot_index": 0, - "links": [ - 840 - ] - } - ], - "properties": { - "cnr_id": "ComfyUI-GGUF", - "ver": "bc5223b0e37e053dbec2ea5e5f52c2fd4b8f712a", - "Node name for S&R": "UnetLoaderGGUF" - }, - "widgets_values": [ - "hunyuan-video-t2v-720p-Q4_K_M.gguf" - ] - }, { "id": 255, "type": "PreviewImage", @@ -571,13 +350,13 @@ "flags": { "pinned": true }, - "order": 20, + "order": 19, "mode": 0, "inputs": [ { "name": "images", "type": "IMAGE", - "link": 903 + "link": 911 } ], "outputs": [], @@ -602,7 +381,7 @@ "flags": { "pinned": true }, - "order": 3, + "order": 1, "mode": 0, "inputs": [], "outputs": [ @@ -638,7 +417,7 @@ 26 ], "flags": {}, - "order": 10, + "order": 8, "mode": 0, "inputs": [ { @@ -681,7 +460,7 @@ { "name": "image", "type": "IMAGE", - "link": 904 + "link": 912 }, { "name": "vae", @@ -770,7 +549,7 @@ "flags": { "collapsed": false }, - "order": 4, + "order": 2, "mode": 0, "inputs": [], "outputs": [ @@ -815,8 +594,8 @@ "properties": { "cnr_id": "ComfyUI-WarpedToolset", "ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb", - "widget_ue_connectable": {}, - "Node name for S&R": "WarpedBundleAllVideoImages" + "Node name for S&R": "WarpedBundleAllVideoImages", + "widget_ue_connectable": {} }, "widgets_values": [ "", @@ -825,190 +604,6 @@ "color": "#323", "bgcolor": "#535" }, - { - "id": 120, - "type": "BasicScheduler", - "pos": [ - -408.83428955078125, - 1107.899169921875 - ], - "size": [ - 323.0991516113281, - 106 - ], - "flags": { - "collapsed": true - }, - "order": 14, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 288 - } - ], - "outputs": [ - { - "name": "SIGMAS", - "type": "SIGMAS", - "slot_index": 0, - "links": [ - 897 - ] - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.27", - "Node name for S&R": "BasicScheduler" - }, - "widgets_values": [ - "normal", - 16, - 0.75 - ] - }, - { - "id": 16, - "type": "KSamplerSelect", - "pos": [ - -418.24090576171875, - 1165.8077392578125 - ], - "size": [ - 315, - 58 - ], - "flags": { - "collapsed": true - }, - "order": 5, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "SAMPLER", - "shape": 3, - "type": "SAMPLER", - "slot_index": 0, - "links": [ - 896 - ] - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.27", - "Node name for S&R": "KSamplerSelect" - }, - "widgets_values": [ - "euler" - ] - }, - { - "id": 252, - "type": "LatentUpscaleBy", - "pos": [ - -416.2454833984375, - 1223.833740234375 - ], - "size": [ - 315, - 82 - ], - "flags": { - "collapsed": true - }, - "order": 13, - "mode": 0, - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 872 - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "slot_index": 0, - "links": [ - 859 - ] - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.27", - "Node name for S&R": "LatentUpscaleBy" - }, - "widgets_values": [ - "bilinear", - 1.25 - ] - }, - { - "id": 279, - "type": "WarpedImageResize", - "pos": [ - -413.2937316894531, - 853.5418701171875 - ], - "size": [ - 315, - 190 - ], - "flags": {}, - "order": 18, - "mode": 0, - "inputs": [ - { - "name": "image", - "type": "IMAGE", - "link": 902 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 903, - 904 - ] - }, - { - "name": "scale_orig_image", - "type": "IMAGE", - "links": null - }, - { - "name": "width", - "type": "INT", - "links": null - }, - { - "name": "height", - "type": "INT", - "links": null - } - ], - "properties": { - "cnr_id": "ComfyUI-WarpedToolset", - "ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb", - "Node name for S&R": "WarpedImageResize" - }, - "widgets_values": [ - 416, - 640, - "lanczos", - "center" - ], - "color": "#323", - "bgcolor": "#535" - }, { "id": 272, "type": "VAEEncodeTiled", @@ -1023,7 +618,7 @@ "flags": { "collapsed": true }, - "order": 9, + "order": 7, "mode": 0, "inputs": [ { @@ -1073,24 +668,29 @@ "flags": { "collapsed": true }, - "order": 6, + "order": 3, "mode": 0, "inputs": [], "outputs": [ { - "name": "VAE", + "name": "vae", "type": "VAE", "links": [ 906, 907 ] + }, + { + "name": "vae_state_dict", + "type": "DICT", + "links": null } ], "properties": { "cnr_id": "ComfyUI-WarpedToolset", "ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb", - "widget_ue_connectable": {}, - "Node name for S&R": "WarpedVAELoader" + "Node name for S&R": "WarpedVAELoader", + "widget_ue_connectable": {} }, "widgets_values": [ "hunyuan_video_vae_bf16.safetensors" @@ -1098,12 +698,158 @@ "color": "#323", "bgcolor": "#535" }, + { + "id": 282, + "type": "MagCache", + "pos": [ + 23.999088287353516, + 43.20001220703125 + ], + "size": [ + 270, + 178 + ], + "flags": { + "collapsed": true + }, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 908 + } + ], + "outputs": [ + { + "name": "model", + "type": "MODEL", + "links": [ + 909 + ] + } + ], + "properties": { + "aux_id": "Zehong-Ma/ComfyUI-MagCache", + "ver": "7d4e982bf7955498afca891c7094c48a70985537", + "Node name for S&R": "MagCache" + }, + "widgets_values": [ + "hunyuan_video", + 0.12000000000000002, + 0.2, + 3, + 0, + -1 + ] + }, + { + "id": 67, + "type": "ModelSamplingSD3", + "pos": [ + -4.116424560546875, + 127.65672302246094 + ], + "size": [ + 210, + 58 + ], + "flags": { + "collapsed": true + }, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 909 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "slot_index": 0, + "links": [ + 195 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.27", + "Node name for S&R": "ModelSamplingSD3" + }, + "widgets_values": [ + 12 + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 283, + "type": "ImageResizeKJv2", + "pos": [ + -433.86492919921875, + 840.4696044921875 + ], + "size": [ + 270, + 242 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 910 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 911, + 912 + ] + }, + { + "name": "width", + "type": "INT", + "links": null + }, + { + "name": "height", + "type": "INT", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "1.1.0", + "Node name for S&R": "ImageResizeKJv2" + }, + "widgets_values": [ + 416, + 640, + "lanczos", + "resize", + "0, 0, 0", + "center", + 2 + ] + }, { "id": 261, "type": "VAEDecodeTiled", "pos": [ - -408.14068603515625, - 1290.9644775390625 + -412.1336669921875, + 1316.2545166015625 ], "size": [ 317.6997375488281, @@ -1112,7 +858,7 @@ "flags": { "collapsed": true }, - "order": 16, + "order": 14, "mode": 0, "inputs": [ { @@ -1133,7 +879,7 @@ "slot_index": 0, "links": [ 860, - 902 + 910 ] } ], @@ -1148,6 +894,257 @@ 64, 8 ] + }, + { + "id": 252, + "type": "LatentUpscaleBy", + "pos": [ + -414.91448974609375, + 1259.771728515625 + ], + "size": [ + 315, + 82 + ], + "flags": { + "collapsed": true + }, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 872 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "slot_index": 0, + "links": [ + 859 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.27", + "Node name for S&R": "LatentUpscaleBy" + }, + "widgets_values": [ + "bilinear", + 1.25 + ] + }, + { + "id": 16, + "type": "KSamplerSelect", + "pos": [ + -416.909912109375, + 1213.7247314453125 + ], + "size": [ + 315, + 58 + ], + "flags": { + "collapsed": true + }, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "SAMPLER", + "type": "SAMPLER", + "slot_index": 0, + "links": [ + 896 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.27", + "Node name for S&R": "KSamplerSelect" + }, + "widgets_values": [ + "euler" + ] + }, + { + "id": 120, + "type": "BasicScheduler", + "pos": [ + -412.8272705078125, + 1147.8306884765625 + ], + "size": [ + 323.0991516113281, + 106 + ], + "flags": { + "collapsed": true + }, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 288 + } + ], + "outputs": [ + { + "name": "SIGMAS", + "type": "SIGMAS", + "slot_index": 0, + "links": [ + 897 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.27", + "Node name for S&R": "BasicScheduler" + }, + "widgets_values": [ + "normal", + 16, + 0.75 + ] + }, + { + "id": 146, + "type": "CLIPTextEncode", + "pos": [ + 16.21417999267578, + 237.3597869873047 + ], + "size": [ + 288.2851257324219, + 81.00001525878906 + ], + "flags": { + "collapsed": false + }, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 913 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 448 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 352 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.27", + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "8itchW4lk, dynamic (real life:1.15) video showing front view of a beautiful stunning gorgeous woman with very long flowing brown hair with blonde highlights and exotic sexy grey eyes, she is wearing (a provocative revealing see-through scientist indiana jones style maroon satin top blouse:1.2) and see-through black thong underwear with an extremely short tight micro miniskirt low on the hips and she is braless, (she is showing her massive ((natural heavy sagging:1.3)) naked breasts:1.15), (her nipples are erect:1.25), (she is also wearing a thin silver military dogtags pendant necklace, she is in an abandonned nuclear facility, she was surprised by the viewer, so now she is walking toward the viewer to seduce him:1.25), (she is walking toward the viewer:1.2), (she is walking in a very sexy manner, smug facial expression with a slight smirk, sexy but dangerous attitude and seductive lips, normal speed, her heavy breasts (((are (bouncing:1.5) and (swaying:1.3) and (jiggling:1.7):1.7)) unrestrained with every step she takes and with ever move she makes:1.15), (((her breasts bounce into each other and recoil away from each other as she walks:1.8))), the viewer keeps her in view all the times, (extremely soft breasts:1.3), (extremely bouncing breasts:1.3), (breasts bouncing up and down side to side:1.5), detailed face, detailed hands, detailed fingers, nail polish, glamour eye makeup, eyeliner, lipstick, mascara, 4k ultra high quality video footage, (realisic jiggle, boucing, wobble, swaying physics, her breasts jiggle and bounce with each staps she take:1.2), dark atmosphere, post apocalyptic nuclear building interior, retro cyberpunk vibe", + true + ] + }, + { + "id": 284, + "type": "WarpedDualClipLoaderGGUF", + "pos": [ + -425.8790588378906, + 377.28094482421875 + ], + "size": [ + 300.9039001464844, + 130 + ], + "flags": { + "collapsed": true + }, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 913 + ] + } + ], + "properties": { + "cnr_id": "comfyui-warpedtoolset", + "ver": "1.7.0", + "Node name for S&R": "WarpedDualClipLoaderGGUF" + }, + "widgets_values": [ + "clip_l.safetensors", + "Kijai\\llava-llama-3-8B-v1_1-Q4_K_M.gguf", + "hunyuan_video", + "default" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 285, + "type": "WarpedLoaderGGUF", + "pos": [ + -421.885986328125, + 429.1899108886719 + ], + "size": [ + 282.2144470214844, + 58 + ], + "flags": { + "collapsed": true + }, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 914 + ] + } + ], + "properties": { + "cnr_id": "comfyui-warpedtoolset", + "ver": "1.7.0", + "Node name for S&R": "WarpedLoaderGGUF" + }, + "widgets_values": [ + "hunyuan-video-t2v-720p-Q4_K_M.gguf" + ], + "color": "#323", + "bgcolor": "#535" } ], "links": [ @@ -1183,14 +1180,6 @@ 0, "CONDITIONING" ], - [ - 366, - 144, - 0, - 146, - 0, - "CLIP" - ], [ 448, 170, @@ -1215,14 +1204,6 @@ 1, "VAE" ], - [ - 840, - 145, - 0, - 258, - 0, - "MODEL" - ], [ 859, 252, @@ -1287,46 +1268,6 @@ 0, "STRING" ], - [ - 900, - 81, - 0, - 278, - 0, - "MODEL" - ], - [ - 901, - 278, - 0, - 67, - 0, - "MODEL" - ], - [ - 902, - 261, - 0, - 279, - 0, - "IMAGE" - ], - [ - 903, - 279, - 0, - 255, - 0, - "IMAGE" - ], - [ - 904, - 279, - 0, - 277, - 0, - "IMAGE" - ], [ 905, 280, @@ -1350,6 +1291,62 @@ 195, 0, "*" + ], + [ + 908, + 81, + 0, + 282, + 0, + "MODEL" + ], + [ + 909, + 282, + 0, + 67, + 0, + "MODEL" + ], + [ + 910, + 261, + 0, + 283, + 0, + "IMAGE" + ], + [ + 911, + 283, + 0, + 255, + 0, + "IMAGE" + ], + [ + 912, + 283, + 0, + 277, + 0, + "IMAGE" + ], + [ + 913, + 284, + 0, + 146, + 0, + "CLIP" + ], + [ + 914, + 285, + 0, + 258, + 0, + "MODEL" ] ], "groups": [ @@ -1372,14 +1369,14 @@ "ds": { "scale": 0.7513148009015777, "offset": [ - 488.4360000000008, - 26.011999999992337 + 547.0000000000008, + 31.335999999992882 ] }, "groupNodes": {}, "ue_links": [], "VHS_latentpreview": true, - "VHS_latentpreviewrate": 10, + "VHS_latentpreviewrate": 15, "node_versions": { "ComfyUI_Custom_Nodes_AlekPet": "7b3d6f190aeca261422bdfb74b5af37937e5bf68", "comfy-core": "0.3.26", @@ -1389,7 +1386,8 @@ "ComfyUI-GGUF": "298192ed60f8ca821c6fe5f8030cae23424cada5" }, "VHS_MetadataImage": true, - "VHS_KeepIntermediate": true + "VHS_KeepIntermediate": true, + "frontendVersion": "1.20.5" }, "version": 0.4 } \ No newline at end of file diff --git a/Sample_Workflows/Warped Upscale By Model Workflow.json b/Sample_Workflows/Warped Upscale By Model Workflow.json index 4c4f9e3..d51d326 100644 --- a/Sample_Workflows/Warped Upscale By Model Workflow.json +++ b/Sample_Workflows/Warped Upscale By Model Workflow.json @@ -2,8 +2,134 @@ "id": "05a742f7-a4cc-476b-bec2-4dfbaa1009f5", "revision": 0, "last_node_id": 281, - "last_link_id": 895, + "last_link_id": 896, "nodes": [ + { + "id": 279, + "type": "VHS_VideoCombine", + "pos": [ + 1025.608642578125, + 429.6685791015625 + ], + "size": [ + 534.675537109375, + 358 + ], + "flags": {}, + "order": 7, + "mode": 2, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 891 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-videohelpersuite", + "ver": "5e61bcf218fe3bb7c899bbd584bbc99a9d05fb42", + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 30, + "loop_count": 0, + "filename_prefix": "Hunyuan/%date:yyyy_MM_dd%/interpolate/vid", + "format": "video/nvenc_h264-mp4", + "pix_fmt": "yuv420p", + "bitrate": 10, + "megabit": true, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "save_images": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "vid_00001.mp4", + "subfolder": "Hunyuan\\2025_03_19\\interpolate", + "type": "output", + "format": "video/nvenc_h264-mp4", + "frame_rate": 30, + "workflow": "vid_00001.png", + "fullpath": "M:\\Projects\\ComfyUI_windows_portable\\ComfyUI\\output\\Hunyuan\\2025_03_19\\interpolate\\vid_00001.mp4" + } + } + } + }, + { + "id": 278, + "type": "FILM VFI", + "pos": [ + 1029.126953125, + 156.33795166015625 + ], + "size": [ + 443.4000244140625, + 126 + ], + "flags": {}, + "order": 6, + "mode": 2, + "inputs": [ + { + "name": "frames", + "type": "IMAGE", + "link": 890 + }, + { + "name": "optional_interpolation_states", + "shape": 7, + "type": "INTERPOLATION_STATES", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 891 + ] + } + ], + "properties": { + "cnr_id": "comfyui-frame-interpolation", + "ver": "18b7cce5b08290741929e39d4955b88382db4e1d", + "Node name for S&R": "FILM VFI" + }, + "widgets_values": [ + "film_net_fp32.pt", + 20, + 2 + ] + }, { "id": 99, "type": "Fast Groups Muter (rgthree)", @@ -35,12 +161,80 @@ "toggleRestriction": "default" } }, + { + "id": 281, + "type": "WarpedBundleAllVideoImages", + "pos": [ + -179.6438446044922, + 178.9619598388672 + ], + "size": [ + 378, + 202 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "image_batch", + "type": "IMAGE", + "links": [ + 894, + 896 + ] + }, + { + "name": "first_image", + "type": "IMAGE", + "links": null + }, + { + "name": "last_image", + "type": "IMAGE", + "links": null + }, + { + "name": "num_frames", + "type": "INT", + "links": null + }, + { + "name": "width", + "type": "INT", + "links": null + }, + { + "name": "height", + "type": "INT", + "links": null + }, + { + "name": "fps", + "type": "FLOAT", + "links": null + } + ], + "properties": { + "cnr_id": "ComfyUI-WarpedToolset", + "ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb", + "Node name for S&R": "WarpedBundleAllVideoImages", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "N:\\Projects\\ComfyUI_windows_portable\\ComfyUI\\output\\Hunyuan\\2025_07_23\\lowres\\vid_00029.mp4", + true + ], + "color": "#323", + "bgcolor": "#535" + }, { "id": 255, "type": "PreviewImage", "pos": [ - -434.8162536621094, - 537.4317626953125 + -169.94720458984375, + 462.8961181640625 ], "size": [ 542.0654907226562, @@ -68,15 +262,15 @@ "id": 202, "type": "VHS_VideoCombine", "pos": [ - 331.1438293457031, - 451.63348388671875 + 387.04595947265625, + 464.9437255859375 ], "size": [ 534.675537109375, 358 ], "flags": {}, - "order": 6, + "order": 5, "mode": 0, "inputs": [ { @@ -142,53 +336,19 @@ } } }, - { - "id": 201, - "type": "UpscaleModelLoader", - "pos": [ - 341.0927734375, - 334.5395812988281 - ], - "size": [ - 315, - 58 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "UPSCALE_MODEL", - "type": "UPSCALE_MODEL", - "slot_index": 0, - "links": [ - 885 - ] - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.27", - "Node name for S&R": "UpscaleModelLoader" - }, - "widgets_values": [ - "4x_foolhardy_Remacri.pth" - ] - }, { "id": 276, "type": "WarpedUpscaleWithModel", "pos": [ - 345.3873291015625, - 158.17369079589844 + 379.9934997558594, + 160.83570861816406 ], "size": [ 315, 102 ], "flags": {}, - "order": 5, + "order": 4, "mode": 0, "inputs": [ { @@ -199,7 +359,7 @@ { "name": "images", "type": "IMAGE", - "link": 893 + "link": 896 } ], "outputs": [ @@ -226,200 +386,15 @@ "bgcolor": "#535" }, { - "id": 278, - "type": "FILM VFI", + "id": 201, + "type": "UpscaleModelLoader", "pos": [ - 912, - 159 - ], - "size": [ - 443.4000244140625, - 126 - ], - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "frames", - "type": "IMAGE", - "link": 890 - }, - { - "name": "optional_interpolation_states", - "shape": 7, - "type": "INTERPOLATION_STATES", - "link": null - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "slot_index": 0, - "links": [ - 891 - ] - } - ], - "properties": { - "cnr_id": "comfyui-frame-interpolation", - "ver": "18b7cce5b08290741929e39d4955b88382db4e1d", - "Node name for S&R": "FILM VFI" - }, - "widgets_values": [ - "film_net_fp32.pt", - 20, - 2 - ] - }, - { - "id": 279, - "type": "VHS_VideoCombine", - "pos": [ - 911.143798828125, - 449.63348388671875 - ], - "size": [ - 534.675537109375, - 358 - ], - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 891 - }, - { - "name": "audio", - "shape": 7, - "type": "AUDIO", - "link": null - }, - { - "name": "meta_batch", - "shape": 7, - "type": "VHS_BatchManager", - "link": null - }, - { - "name": "vae", - "shape": 7, - "type": "VAE", - "link": null - } - ], - "outputs": [ - { - "name": "Filenames", - "type": "VHS_FILENAMES", - "links": null - } - ], - "properties": { - "cnr_id": "comfyui-videohelpersuite", - "ver": "5e61bcf218fe3bb7c899bbd584bbc99a9d05fb42", - "Node name for S&R": "VHS_VideoCombine" - }, - "widgets_values": { - "frame_rate": 30, - "loop_count": 0, - "filename_prefix": "Hunyuan/%date:yyyy_MM_dd%/interpolate/vid", - "format": "video/nvenc_h264-mp4", - "pix_fmt": "yuv420p", - "bitrate": 10, - "megabit": true, - "save_metadata": true, - "pingpong": false, - "save_output": true, - "save_images": true, - "videopreview": { - "hidden": false, - "paused": false, - "params": { - "filename": "vid_00001.mp4", - "subfolder": "Hunyuan\\2025_03_19\\interpolate", - "type": "output", - "format": "video/nvenc_h264-mp4", - "frame_rate": 30, - "workflow": "vid_00001.png", - "fullpath": "M:\\Projects\\ComfyUI_windows_portable\\ComfyUI\\output\\Hunyuan\\2025_03_19\\interpolate\\vid_00001.mp4" - } - } - } - }, - { - "id": 280, - "type": "WarpedImageResize", - "pos": [ - -24.520009994506836, - 175.57980346679688 + 381.0227966308594, + 339.8635559082031 ], "size": [ 315, - 190 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "image", - "type": "IMAGE", - "link": 895 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 893 - ] - }, - { - "name": "scale_orig_image", - "type": "IMAGE", - "links": null - }, - { - "name": "width", - "type": "INT", - "links": null - }, - { - "name": "height", - "type": "INT", - "links": null - } - ], - "properties": { - "cnr_id": "ComfyUI-WarpedToolset", - "ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb", - "Node name for S&R": "WarpedImageResize" - }, - "widgets_values": [ - 416, - 640, - "lanczos", - "center" - ], - "color": "#223", - "bgcolor": "#335" - }, - { - "id": 281, - "type": "WarpedBundleAllVideoImages", - "pos": [ - -436.52703857421875, - 177.63095092773438 - ], - "size": [ - 378, - 202 + 58 ], "flags": {}, "order": 2, @@ -427,56 +402,22 @@ "inputs": [], "outputs": [ { - "name": "image_batch", - "type": "IMAGE", + "name": "UPSCALE_MODEL", + "type": "UPSCALE_MODEL", + "slot_index": 0, "links": [ - 894, - 895 + 885 ] - }, - { - "name": "first_image", - "type": "IMAGE", - "links": null - }, - { - "name": "last_image", - "type": "IMAGE", - "links": null - }, - { - "name": "num_frames", - "type": "INT", - "links": null - }, - { - "name": "width", - "type": "INT", - "links": null - }, - { - "name": "height", - "type": "INT", - "links": null - }, - { - "name": "fps", - "type": "FLOAT", - "links": null } ], "properties": { - "cnr_id": "ComfyUI-WarpedToolset", - "ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb", - "widget_ue_connectable": {}, - "Node name for S&R": "WarpedBundleAllVideoImages" + "cnr_id": "comfy-core", + "ver": "0.3.27", + "Node name for S&R": "UpscaleModelLoader" }, "widgets_values": [ - "", - true - ], - "color": "#323", - "bgcolor": "#535" + "4x_foolhardy_Remacri.pth" + ] } ], "links": [ @@ -512,14 +453,6 @@ 0, "IMAGE" ], - [ - 893, - 280, - 0, - 276, - 1, - "IMAGE" - ], [ 894, 281, @@ -529,23 +462,36 @@ "IMAGE" ], [ - 895, + 896, 281, 0, - 280, - 0, + 276, + 1, "IMAGE" ] ], "groups": [ { "id": 2, - "title": "Hunyuan High Resolution", + "title": "High Resolution Upscale With Model", "bounding": [ - -479.1864318847656, - -109.82951354980469, - 1971.108642578125, - 1907.701904296875 + -230.2893829345703, + -103.17450714111328, + 1165.8536376953125, + 1875.7579345703125 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Interpolate", + "bounding": [ + 943.718505859375, + -107.20301055908203, + 734.95703125, + 1895.4840087890625 ], "color": "#3f789e", "font_size": 24, @@ -557,14 +503,15 @@ "ds": { "scale": 0.7513148009015777, "offset": [ - 762.6220000000011, - 244.2959999999923 + 455.1610000000002, + -103.0950000000065 ] }, + "frontendVersion": "1.20.5", "groupNodes": {}, "ue_links": [], "VHS_latentpreview": true, - "VHS_latentpreviewrate": 10, + "VHS_latentpreviewrate": 15, "node_versions": { "comfy-core": "0.3.26", "comfyui-kjnodes": "e3b4bcf21e03402a2b11927eb5a4e3df500795b7", diff --git a/Sample_Workflows/Warped WAN I2V Workflow.json b/Sample_Workflows/Warped WAN I2V Workflow.json index f71bb68..98e6f50 100644 --- a/Sample_Workflows/Warped WAN I2V Workflow.json +++ b/Sample_Workflows/Warped WAN I2V Workflow.json @@ -1,79 +1,9 @@ { "id": "2fac44f1-8935-4d17-8233-fe2dcaee9ba8", "revision": 0, - "last_node_id": 140, - "last_link_id": 296, + "last_node_id": 142, + "last_link_id": 303, "nodes": [ - { - "id": 38, - "type": "CLIPLoader", - "pos": [ - -68.24571228027344, - 51.294097900390625 - ], - "size": [ - 376.05950927734375, - 106 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "CLIP", - "type": "CLIP", - "slot_index": 0, - "links": [ - 214 - ] - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.20", - "Node name for S&R": "CLIPLoader" - }, - "widgets_values": [ - "t5xxl_um_fp8_e4m3fn_scaled.safetensors", - "wan", - "default" - ] - }, - { - "id": 55, - "type": "LoaderGGUF", - "pos": [ - -71.74114227294922, - -65.37071228027344 - ], - "size": [ - 380.0403137207031, - 67.37849426269531 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "slot_index": 0, - "links": [ - 213 - ] - } - ], - "properties": { - "cnr_id": "gguf", - "ver": "1.5.8", - "Node name for S&R": "LoaderGGUF" - }, - "widgets_values": [ - "wan2.1-i2v-14b-720p-q5_k_m.gguf" - ] - }, { "id": 7, "type": "CLIPTextEncode", @@ -86,7 +16,7 @@ 90.64099884033203 ], "flags": {}, - "order": 12, + "order": 15, "mode": 0, "inputs": [ { @@ -209,13 +139,13 @@ 134.78460693359375 ], "flags": {}, - "order": 7, + "order": 9, "mode": 0, "inputs": [ { "name": "model", "type": "MODEL", - "link": 213 + "link": 297 }, { "name": "clip", @@ -268,7 +198,7 @@ "flags": { "collapsed": true }, - "order": 2, + "order": 0, "mode": 0, "inputs": [], "outputs": [ @@ -304,7 +234,7 @@ "flags": { "collapsed": true }, - "order": 10, + "order": 13, "mode": 0, "inputs": [ { @@ -346,7 +276,7 @@ 284.9806823730469 ], "flags": {}, - "order": 11, + "order": 14, "mode": 0, "inputs": [ { @@ -621,7 +551,7 @@ 493.4667053222656 ], "flags": {}, - "order": 3, + "order": 1, "mode": 0, "inputs": [], "outputs": [ @@ -631,7 +561,7 @@ "slot_index": 0, "links": [ 186, - 283 + 298 ] }, { @@ -663,13 +593,13 @@ 517.2369384765625 ], "flags": {}, - "order": 14, + "order": 11, "mode": 0, "inputs": [ { "name": "images", "type": "IMAGE", - "link": 287 + "link": 300 } ], "outputs": [], @@ -680,172 +610,6 @@ }, "widgets_values": [] }, - { - "id": 118, - "type": "WarpedImageNoiseAugmentation", - "pos": [ - -390.6877746582031, - 1235.955810546875 - ], - "size": [ - 390.5999755859375, - 126 - ], - "flags": {}, - "order": 13, - "mode": 0, - "inputs": [ - { - "name": "image", - "type": "IMAGE", - "link": 284 - } - ], - "outputs": [ - { - "name": "image", - "type": "IMAGE", - "slot_index": 0, - "links": [ - 267 - ] - }, - { - "name": "seed", - "type": "INT", - "slot_index": 1, - "links": [ - 243, - 276 - ] - } - ], - "properties": { - "cnr_id": "ComfyUI-WarpedToolset", - "ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb", - "Node name for S&R": "WarpedImageNoiseAugmentation" - }, - "widgets_values": [ - 0.3, - 661031532765305, - "fixed" - ], - "color": "#323", - "bgcolor": "#535" - }, - { - "id": 137, - "type": "WarpedImageResize", - "pos": [ - -391.8274230957031, - 970.8604125976562 - ], - "size": [ - 315, - 214 - ], - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "image", - "type": "IMAGE", - "link": 283 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 284, - 287, - 288 - ] - }, - { - "name": "scale_orig_image", - "type": "IMAGE", - "links": null - }, - { - "name": "width", - "type": "INT", - "links": [ - 285 - ] - }, - { - "name": "height", - "type": "INT", - "links": [ - 286 - ] - } - ], - "properties": { - "cnr_id": "ComfyUI-WarpedToolset", - "ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb", - "Node name for S&R": "WarpedImageResize" - }, - "widgets_values": [ - 400, - 720, - "lanczos", - "center", - false - ], - "color": "#323", - "bgcolor": "#535" - }, - { - "id": 51, - "type": "CLIPVisionEncode", - "pos": [ - -72.99646759033203, - 694.0908813476562 - ], - "size": [ - 253.60000610351562, - 78 - ], - "flags": { - "collapsed": true - }, - "order": 15, - "mode": 0, - "inputs": [ - { - "name": "clip_vision", - "type": "CLIP_VISION", - "link": 289 - }, - { - "name": "image", - "type": "IMAGE", - "link": 288 - } - ], - "outputs": [ - { - "name": "CLIP_VISION_OUTPUT", - "type": "CLIP_VISION_OUTPUT", - "slot_index": 0, - "links": [ - 266 - ] - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.20", - "Node name for S&R": "CLIPVisionEncode" - }, - "widgets_values": [ - "none" - ] - }, { "id": 134, "type": "WarpedWanImageToVideo", @@ -894,7 +658,7 @@ "widget": { "name": "width" }, - "link": 285 + "link": 302 }, { "name": "height", @@ -902,7 +666,7 @@ "widget": { "name": "height" }, - "link": 286 + "link": 303 } ], "outputs": [ @@ -966,7 +730,7 @@ "flags": { "collapsed": true }, - "order": 4, + "order": 2, "mode": 0, "inputs": [], "outputs": [ @@ -1004,7 +768,7 @@ "flags": { "collapsed": true }, - "order": 5, + "order": 3, "mode": 0, "inputs": [], "outputs": [ @@ -1048,7 +812,7 @@ "flags": { "collapsed": true }, - "order": 6, + "order": 4, "mode": 0, "inputs": [ { @@ -1094,7 +858,7 @@ "flags": { "collapsed": true }, - "order": 9, + "order": 8, "mode": 0, "inputs": [ { @@ -1121,6 +885,242 @@ "widgets_values": [ 12 ] + }, + { + "id": 141, + "type": "WarpedLoaderGGUF", + "pos": [ + -62.516021728515625, + -49.970008850097656 + ], + "size": [ + 282.2144470214844, + 58 + ], + "flags": { + "collapsed": true + }, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 297 + ] + } + ], + "properties": { + "cnr_id": "comfyui-warpedtoolset", + "ver": "1.7.0", + "Node name for S&R": "WarpedLoaderGGUF" + }, + "widgets_values": [ + "Wan2.1_I2V_14B_FusionX-Q4_K_M.gguf" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 51, + "type": "CLIPVisionEncode", + "pos": [ + -72.99646759033203, + 694.0908813476562 + ], + "size": [ + 253.60000610351562, + 78 + ], + "flags": { + "collapsed": false + }, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 289 + }, + { + "name": "image", + "type": "IMAGE", + "link": 301 + } + ], + "outputs": [ + { + "name": "CLIP_VISION_OUTPUT", + "type": "CLIP_VISION_OUTPUT", + "slot_index": 0, + "links": [ + 266 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.20", + "Node name for S&R": "CLIPVisionEncode" + }, + "widgets_values": [ + "none" + ] + }, + { + "id": 142, + "type": "ImageResizeKJv2", + "pos": [ + -111.76309204101562, + 837.80712890625 + ], + "size": [ + 270, + 242 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 298 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 299, + 300, + 301 + ] + }, + { + "name": "width", + "type": "INT", + "links": [ + 302 + ] + }, + { + "name": "height", + "type": "INT", + "links": [ + 303 + ] + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "1.1.0", + "Node name for S&R": "ImageResizeKJv2" + }, + "widgets_values": [ + 400, + 720, + "lanczos", + "resize", + "0, 0, 0", + "center", + 2 + ] + }, + { + "id": 118, + "type": "WarpedImageNoiseAugmentation", + "pos": [ + -232.29910278320312, + 1134.79833984375 + ], + "size": [ + 390.5999755859375, + 126 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 299 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 267 + ] + }, + { + "name": "seed", + "type": "INT", + "slot_index": 1, + "links": [ + 243, + 276 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WarpedToolset", + "ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb", + "Node name for S&R": "WarpedImageNoiseAugmentation" + }, + "widgets_values": [ + 0.3, + 661031532765305, + "fixed" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 38, + "type": "CLIPLoader", + "pos": [ + -69.57671356201172, + 16.68808937072754 + ], + "size": [ + 376.05950927734375, + 106 + ], + "flags": { + "collapsed": true + }, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "slot_index": 0, + "links": [ + 214 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.20", + "Node name for S&R": "CLIPLoader" + }, + "widgets_values": [ + "t5xxl_um_fp8_e4m3fn_scaled.safetensors", + "wan", + "default" + ] } ], "links": [ @@ -1132,14 +1132,6 @@ 1, "CONDITIONING" ], - [ - 213, - 55, - 0, - 116, - 0, - "MODEL" - ], [ 214, 38, @@ -1292,54 +1284,6 @@ 0, "IMAGE" ], - [ - 283, - 52, - 0, - 137, - 0, - "IMAGE" - ], - [ - 284, - 137, - 0, - 118, - 0, - "IMAGE" - ], - [ - 285, - 137, - 2, - 134, - 5, - "INT" - ], - [ - 286, - 137, - 3, - 134, - 6, - "INT" - ], - [ - 287, - 137, - 0, - 58, - 0, - "IMAGE" - ], - [ - 288, - 137, - 0, - 51, - 1, - "IMAGE" - ], [ 289, 138, @@ -1371,6 +1315,62 @@ 110, 0, "MODEL" + ], + [ + 297, + 141, + 0, + 116, + 0, + "MODEL" + ], + [ + 298, + 52, + 0, + 142, + 0, + "IMAGE" + ], + [ + 299, + 142, + 0, + 118, + 0, + "IMAGE" + ], + [ + 300, + 142, + 0, + 58, + 0, + "IMAGE" + ], + [ + 301, + 142, + 0, + 51, + 1, + "IMAGE" + ], + [ + 302, + 142, + 1, + 134, + 5, + "INT" + ], + [ + 303, + 142, + 2, + 134, + 6, + "INT" ] ], "groups": [], @@ -1379,8 +1379,8 @@ "ds": { "scale": 0.7513148009015777, "offset": [ - 159.67900000000375, - 451.9319999999936 + 473.7950000000054, + -276.1250000000068 ] }, "frontendVersion": "1.20.5", diff --git a/__init__.py b/__init__.py index 8a34381..76cef2c 100644 --- a/__init__.py +++ b/__init__.py @@ -1,4 +1,5 @@ from .nodes import * +from .gguf_nodes import WarpedLoaderGGUF, WarpedClipLoaderGGUF, WarpedDualClipLoaderGGUF NODE_CLASS_MAPPINGS = { "WarpedHunyuanMultiLoraMerge": WarpedHunyuanMultiLoraMerge, @@ -6,13 +7,13 @@ NODE_CLASS_MAPPINGS = { "WarpedHunyuanLoraMerge": WarpedHunyuanLoraMerge, "WarpedHunyuanMultiLoraMixer": WarpedHunyuanMultiLoraMixer, "WarpedHunyuanMultiLoraMixerExt": WarpedHunyuanMultiLoraMixerExt, - # "WarpedHunyuanMultiLoraMixerExt2": WarpedHunyuanMultiLoraMixerExt2, "WarpedHunyuanLoraAvgMerge": WarpedHunyuanLoraAvgMerge, "WarpedHunyuanLoraBatchMerge": WarpedHunyuanLoraBatchMerge, "WarpedWanLoraMerge": WarpedWanLoraMerge, "WarpedSamplerCustomAdv": WarpedSamplerCustomAdv, "WarpedSamplerCustomAdvLatent": WarpedSamplerCustomAdvLatent, - # "WarpedAdvancedSamplerBatch": WarpedAdvancedSamplerBatch, + # "WarpedSamplerCustomBatch": WarpedSamplerCustomBatch, + # "WarpedSamplerCustomScripted": WarpedSamplerCustomScripted, "WarpedFramepackSampler": WarpedFramepackSampler, "WarpedCreateSpecialImageBatch": WarpedCreateSpecialImageBatch, "WarpedCreateEmptyImageBatch": WarpedCreateEmptyImageBatch, @@ -28,7 +29,6 @@ NODE_CLASS_MAPPINGS = { "WarpedLoadVideosBatch": WarpedLoadVideosBatch, "WarpedWanImageToVideo": WarpedWanImageToVideo, "WarpedWanLoadAndEditLoraBlocks": WarpedWanLoadAndEditLoraBlocks, - # "WarpedImageResize": WarpedImageResize, "WarpedImageScaleToSide": WarpedImageScaleToSide, "WarpedLoadLorasBatchByPrefix": WarpedLoadLorasBatchByPrefix, "WarpedHunyuanVideoLoraLoader": WarpedHunyuanVideoLoraLoader, @@ -39,6 +39,7 @@ NODE_CLASS_MAPPINGS = { "WarpedFramepackLoraSelectBatch": WarpedFramepackLoraSelectBatch, "WarpedFramepackMultiLoraSelect": WarpedFramepackMultiLoraSelect, "WarpedFramepackMultiLoraSelectExt": WarpedFramepackMultiLoraSelectExt, + "WarpedCLIPLoader": WarpedCLIPLoader, "WarpedDualCLIPLoader": WarpedDualCLIPLoader, "WarpedDualEncoder": WarpedDualEncoder, "WarpedCLIPVisionLoader": WarpedCLIPVisionLoader, @@ -50,6 +51,11 @@ NODE_CLASS_MAPPINGS = { "WarpedReverseImageBatch": WarpedReverseImageBatch, "WarpedMultiLoraLoader": WarpedMultiLoraLoader, "WarpedHunyuanImageToVideo": WarpedHunyuanImageToVideo, + "WarpedBasicGuider": WarpedBasicGuider, + "WarpedDualGuider": WarpedDualGuider, + "WarpedLoaderGGUF": WarpedLoaderGGUF, + "WarpedClipLoaderGGUF": WarpedClipLoaderGGUF, + "WarpedDualClipLoaderGGUF": WarpedDualClipLoaderGGUF, } NODE_DISPLAY_NAME_MAPPINGS = { @@ -58,13 +64,13 @@ NODE_DISPLAY_NAME_MAPPINGS = { "WarpedHunyuanLoraMerge": "Warped Hunyuan Lora Merge", "WarpedHunyuanMultiLoraMixer": "Warped Hunyuan Multi Lora Mixer", "WarpedHunyuanMultiLoraMixerExt": "Warped Hunyuan Multi Lora Mixer Ext", - # "WarpedHunyuanMultiLoraMixerExt2": "Warped Hunyuan Multi Lora Mixer Ext 2", "WarpedHunyuanLoraAvgMerge": "Warped Hunyuan Lora Avg Merge", "WarpedHunyuanLoraBatchMerge": "Warped Hunyuan Lora Batch Merge", "WarpedWanLoraMerge": "Warped Wan Lora Merge", "WarpedSamplerCustomAdv": "Warped Sampler Custom Advanced (Image)", "WarpedSamplerCustomAdvLatent": "Warped Sampler Custom Advanced (Latent)", - # "WarpedAdvancedSamplerBatch": "Warped Advanced Sampler Batch", + # "WarpedSamplerCustomBatch": "Warped Sampler Custom Batch", + # "WarpedSamplerCustomScripted": "Warped Sampler Custom Scripted", "WarpedFramepackSampler": "Warped Framepack Sampler", "WarpedCreateSpecialImageBatch": "Warped Create Special Image Batch", "WarpedCreateEmptyImageBatch": "Warped Create Empty Image Batch", @@ -80,7 +86,6 @@ NODE_DISPLAY_NAME_MAPPINGS = { "WarpedLoadVideosBatch": "Warped Load Videos Batch", "WarpedWanImageToVideo": "Warped Wan Image To Video", "WarpedWanLoadAndEditLoraBlocks": "Warped Wan Load And Edit Lora Blocks", - # "WarpedImageResize": "Warped Image Resize", "WarpedImageScaleToSide": "Warped Image Scale To Side", "WarpedLoadLorasBatchByPrefix": "Warped Load Loras Batch By Prefix", "WarpedHunyuanVideoLoraLoader": "Warped Hunyuan Video Lora Loader", @@ -91,6 +96,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "WarpedFramepackLoraSelectBatch": "Warped Framepack Lora Select Batch", "WarpedFramepackMultiLoraSelect": "Warped Framepack Multi Lora Select", "WarpedFramepackMultiLoraSelectExt": "Warped Framepack Multi Lora Select Ext", + "WarpedCLIPLoader": "Warped CLIP Loader", "WarpedDualCLIPLoader": "Warped Dual CLIP Loader", "WarpedDualEncoder": "Warped Dual Encoder", "WarpedCLIPVisionLoader": "Warped Load CLIP Vision", @@ -102,6 +108,11 @@ NODE_DISPLAY_NAME_MAPPINGS = { "WarpedReverseImageBatch": "Warped Reverse Image Batch", "WarpedMultiLoraLoader": "Warped Multi Lora Loader", "WarpedHunyuanImageToVideo": "Warped Hunyuan Image To Video", + "WarpedBasicGuider": "Warped Basic Guider Batch", + "WarpedDualGuider": "Warped Dual Guider", + "WarpedLoaderGGUF": "Warped UNET Loader GGUF", + "WarpedClipLoaderGGUF": "Warped Clip Loader GGUF", + "WarpedDualClipLoaderGGUF": "Warped Dual Clip Loader GGUF", } WEB_DIRECTORY = "./web" diff --git a/gguf_connector/__init__.py b/gguf_connector/__init__.py new file mode 100644 index 0000000..5291317 --- /dev/null +++ b/gguf_connector/__init__.py @@ -0,0 +1,192 @@ +__version__ = '1.7.8' +def __init__(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('-v', '--version', action='version', version= + '%(prog)s ' + __version__) + subparsers = parser.add_subparsers(title='subcommands', dest= + 'subcommand', help='choose a subcommand:') + subparsers.add_parser('cpp', help='[cpp] connector cpp') + subparsers.add_parser('gpp', help='[gpp] connector gpp') + subparsers.add_parser('v', help='[v] vision connector') + subparsers.add_parser('g', help='[g] cli connector g') + subparsers.add_parser('c', help='[c] gui connector c') + subparsers.add_parser('m', help='[m] menu') + subparsers.add_parser('o', help='[o] org web mirror') + subparsers.add_parser('i', help='[i] i/o web mirror') + subparsers.add_parser('w', help='[w] page/container') + subparsers.add_parser('y', help='[y] download comfy') + subparsers.add_parser('n', help='[n] clone node') + subparsers.add_parser('u', help='[u] get cutter') + subparsers.add_parser('p', help='[p] take pack') + subparsers.add_parser('p1', help='[p1] take framepack') + subparsers.add_parser('p2', help='[p2] take packpack') + subparsers.add_parser('r', help='[r] metadata reader') + subparsers.add_parser('r2', help='[r2] metadata fast reader') + subparsers.add_parser('r3', help='[r3] tensor reader') + subparsers.add_parser('r4', help='[r4] tensor info reader') + subparsers.add_parser('e', help='[e] weight extractor') + subparsers.add_parser('q', help='[q] tensor quantizor') + subparsers.add_parser('q1', help='[q1] tensor quantizor (cpu)') + subparsers.add_parser('q2', help='[q2] tensor quantizor (upscale)') + subparsers.add_parser('d', help='[d] divider (safetensors)') + subparsers.add_parser('d2', help='[d2] divider (gguf)') + subparsers.add_parser('m2', help='[m2] merger (gguf)') + subparsers.add_parser('ma', help='[ma] merger (safetensors)') + subparsers.add_parser('s1', help='[s1] splitter (uni/multi)') + subparsers.add_parser('s0', help='[s0] splitter (d2 to d5)') + subparsers.add_parser('s5', help='[s5] splitter (d5 else)') + subparsers.add_parser('sx', help='[sx] splitter (d1 to dx)') + subparsers.add_parser('sy', help='[sy] splitter (d2 else)') + subparsers.add_parser('s', help='[s] splitter (checkpoint)') + subparsers.add_parser('f', help='[f] tensor transfer') + subparsers.add_parser('t', help='[t] tensor convertor') + subparsers.add_parser('t0', help='[t0] tensor convertor (zero)') + subparsers.add_parser('t1', help='[t1] tensor convertor (alpha)') + subparsers.add_parser('t2', help='[t2] tensor convertor (beta)') + subparsers.add_parser('t3', help='[t3] tensor convertor (gamma)') + subparsers.add_parser('t4', help='[t4] tensor convertor (delta)') + subparsers.add_parser('t5', help='[t5] tensor convertor (epsilon)') + subparsers.add_parser('t6', help='[t6] tensor convertor (zeta)') + subparsers.add_parser('t7', help='[t7] tensor convertor (eta)') + subparsers.add_parser('t8', help='[t8] tensor convertor (theta)') + subparsers.add_parser('t9', help='[t9] tensor convertor (iota)') + subparsers.add_parser('d5', help='[d5] dimension 5 fixer (t8x)') + subparsers.add_parser('d6', help='[d6] tensor convertor (t8xx)') + subparsers.add_parser('d7', help='[d7] tensor convertor (plus)') + subparsers.add_parser('pp', help='[pp] pdf analyzor pp') + subparsers.add_parser('cp', help='[cp] pdf analyzor cp') + subparsers.add_parser('ps', help='[ps] wav recognizor ps') + subparsers.add_parser('cs', help='[cs] wav recognizor cs') + subparsers.add_parser('cg', help='[cg] wav recognizor cg (api)') + subparsers.add_parser('pg', help='[pg] wav recognizor pg (api)') + subparsers.add_parser('vg', help='[vg] video generator') + subparsers.add_parser('v1', help='[v1] video 1 (i2v)') + subparsers.add_parser('v2', help='[v2] video 2 (t2v)') + subparsers.add_parser('i2', help='[i2] image 2 (t2i)') + subparsers.add_parser('s2', help='[s2] voice 2 (t2s)') + subparsers.add_parser('b1', help='[b1] bagel 1 (old)') + subparsers.add_parser('b2', help='[b2] bagel 2 (a2a)') + args = parser.parse_args() + if args.subcommand == 'm': + from gguf_connector import m + if args.subcommand == 'n': + from gguf_connector import n + if args.subcommand == 'f': + from gguf_connector import f + if args.subcommand == 'p': + from gguf_connector import p + if args.subcommand == 'p1': + from gguf_connector import p1 + if args.subcommand == 'p2': + from gguf_connector import p2 + elif args.subcommand == 'r': + from gguf_connector import r + elif args.subcommand == 'r2': + from gguf_connector import r2 + elif args.subcommand == 'r3': + from gguf_connector import r3 + elif args.subcommand == 'r4': + from gguf_connector import r4 + elif args.subcommand == 'e': + from gguf_connector import e + elif args.subcommand == 's': + from gguf_connector import s + elif args.subcommand == 's1': + from gguf_connector import s1 + elif args.subcommand == 's0': + from gguf_connector import s0 + elif args.subcommand == 's5': + from gguf_connector import s5 + elif args.subcommand == 'sx': + from gguf_connector import sx + elif args.subcommand == 'sy': + from gguf_connector import sy + elif args.subcommand == 'i': + from gguf_connector import i + elif args.subcommand == 'o': + from gguf_connector import o + elif args.subcommand == 'u': + from gguf_connector import u + elif args.subcommand == 'v': + from gguf_connector import v + elif args.subcommand == 'vg': + from gguf_connector import vg + elif args.subcommand == 'v1': + from gguf_connector import vg2 + elif args.subcommand == 'v2': + from gguf_connector import v2 + elif args.subcommand == 'i2': + from gguf_connector import i2 + elif args.subcommand == 's2': + from gguf_connector import s2 + elif args.subcommand == 'b2': + from gguf_connector import b2 + elif args.subcommand == 'b1': + from gguf_connector import b1 + elif args.subcommand == 'w': + from gguf_connector import w + elif args.subcommand == 'y': + from gguf_connector import y + elif args.subcommand == 't': + from gguf_connector import t + elif args.subcommand == 't0': + from gguf_connector import t0 + elif args.subcommand == 't1': + from gguf_connector import t1 + elif args.subcommand == 't2': + from gguf_connector import t2 + elif args.subcommand == 't3': + from gguf_connector import t3 + elif args.subcommand == 't4': + from gguf_connector import t4 + elif args.subcommand == 't5': + from gguf_connector import t5 + elif args.subcommand == 't6': + from gguf_connector import t6 + elif args.subcommand == 't7': + from gguf_connector import t7 + elif args.subcommand == 't8': + from gguf_connector import t8 + elif args.subcommand == 't9': + from gguf_connector import t9 + elif args.subcommand == 'd5': + from gguf_connector import d5 + elif args.subcommand == 'd6': + from gguf_connector import d6 + elif args.subcommand == 'd7': + from gguf_connector import d7 + elif args.subcommand == 'q': + from gguf_connector import q + elif args.subcommand == 'q1': + from gguf_connector import q1 + elif args.subcommand == 'q2': + from gguf_connector import q2 + elif args.subcommand == 'd': + from gguf_connector import d + elif args.subcommand == 'd2': + from gguf_connector import d2 + elif args.subcommand == 'm2': + from gguf_connector import m2 + elif args.subcommand == 'ma': + from gguf_connector import ma + elif args.subcommand == 'cg': + from gguf_connector import cg + elif args.subcommand == 'pg': + from gguf_connector import pg + elif args.subcommand == 'cs': + from gguf_connector import cs + elif args.subcommand == 'ps': + from gguf_connector import ps + elif args.subcommand == 'cp': + from gguf_connector import cp + elif args.subcommand == 'pp': + from gguf_connector import pp + elif args.subcommand == 'c': + from gguf_connector import c + elif args.subcommand == 'cpp': + from gguf_connector import cpp + elif args.subcommand == 'g': + from gguf_connector import g + elif args.subcommand == 'gpp': + from gguf_connector import gpp \ No newline at end of file diff --git a/gguf_connector/b1.py b/gguf_connector/b1.py new file mode 100644 index 0000000..0124ad7 --- /dev/null +++ b/gguf_connector/b1.py @@ -0,0 +1,630 @@ +import os +if not os.path.isfile(os.path.join(os.path.dirname(__file__), + 'models/bagel/config.json')): + from huggingface_hub import snapshot_download + save_dir = os.path.join(os.path.dirname(__file__), 'models/bagel') + repo_id = 'callgg/bagel-decoder' + cache_dir = save_dir + '/cache' + snapshot_download(cache_dir=cache_dir, local_dir=save_dir, repo_id= + repo_id, allow_patterns=['*.json', '*.md', '*.txt']) +import time, psutil, platform, atexit, random +pynvml_available = False +if platform.system() == 'Linux' or platform.system() == 'Windows': + try: + from pynvml import * + nvmlInit() + pynvml_available = True + print( + """pynvml (NVIDIA GPU monitoring library) initialized successfully! +Detecting Safetensors.. +""" + ) + def shutdown_pynvml(): + print('Shutting down pynvml...') + nvmlShutdown() + atexit.register(shutdown_pynvml) + except Exception as e: + print( + f'Warning: pynvml could not be initialized. Detailed GPU stats via pynvml will not be available. Error: {e}' + ) + if 'NVML Shared Library Not Found' in str(e): + print( + 'pynvml error hint: NVML shared library not found. If you have an NVIDIA GPU and drivers, ensure the library is accessible.' + ) + elif 'Driver Not Loaded' in str(e): + print( + 'pynvml error hint: NVIDIA driver is not loaded. Please check your GPU driver installation.' + ) +import torch +import gradio as gr +import numpy as np +from accelerate import infer_auto_device_map, load_checkpoint_and_dispatch, init_empty_weights +from PIL import Image +from bagel2.data.data_utils import add_special_tokens, pil_img2rgb +from bagel2.data.transforms import ImageTransform +from bagel2.inferencer import InterleaveInferencer +from bagel2.modeling.autoencoder import load_ae +from bagel2.modeling.bagel.qwen2_navit import NaiveCache +from bagel2.modeling.bagel import BagelConfig, Bagel, Qwen2Config, Qwen2ForCausalLM, SiglipVisionConfig, SiglipVisionModel +from bagel2.modeling.qwen2 import Qwen2Tokenizer +model_path = os.path.join(os.path.dirname(__file__), 'models/bagel') +llm_config = Qwen2Config.from_json_file(os.path.join(model_path, + 'llm_config.json')) +llm_config.qk_norm = True +llm_config.tie_word_embeddings = False +llm_config.layer_module = 'Qwen2MoTDecoderLayer' +vit_config = SiglipVisionConfig.from_json_file(os.path.join(model_path, + 'vit_config.json')) +vit_config.rope = False +vit_config.num_hidden_layers -= 1 +safetensors_files = [file for file in os.listdir() if file.endswith( + '.safetensors')] +if safetensors_files: + print('Safetensors file(s) available. Select which one for VAE:') + for index, file_name in enumerate(safetensors_files, start=1): + print(f'{index}. {file_name}') + choice1 = input(f'Enter your choice (1 to {len(safetensors_files)}): ') + try: + choice_index = int(choice1) - 1 + selected_file = safetensors_files[choice_index] + print(f'Safetensors file: {selected_file} is selected!\n') + vae_model, vae_config = load_ae(selected_file) + print('Safetensors file(s) available. Select which one for MODEL:') + for index, file_name in enumerate(safetensors_files, start=1): + print(f'{index}. {file_name}') + choice2 = input(f'Enter your choice (1 to {len(safetensors_files)}): ') + try: + choice_index = int(choice2) - 1 + selected_file = safetensors_files[choice_index] + print(f'Model file: {selected_file} is selected!') + checkpoint = selected_file + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No safetensors files are available in the current directory.') + input('--- Press ENTER To Exit ---') +config = BagelConfig(visual_gen=True, visual_und=True, llm_config= + llm_config, vit_config=vit_config, vae_config=vae_config, + vit_max_num_patch_per_side=70, connector_act='gelu_pytorch_tanh', + latent_patch_size=2, max_latent_size=64) +with init_empty_weights(): + language_model = Qwen2ForCausalLM(llm_config) + vit_model = SiglipVisionModel(vit_config) + model = Bagel(language_model, vit_model, config) + model.vit_model.vision_model.embeddings.convert_conv2d_to_linear(vit_config + , meta=True) +tokenizer = Qwen2Tokenizer.from_pretrained(model_path) +tokenizer, new_token_ids, _ = add_special_tokens(tokenizer) +vae_transform = ImageTransform(1024, 512, 16) +vit_transform = ImageTransform(980, 224, 14) +print('Starting model loading and device map configuration...') +def get_gpu_memory_stats_pynvml(device_id=0): + if not pynvml_available: + return f'GPU-{device_id} (pynvml): Not available.' + try: + handle = nvmlDeviceGetHandleByIndex(device_id) + info = nvmlDeviceGetMemoryInfo(handle) + total_gb = info.total / 1024 ** 3 + used_gb = info.used / 1024 ** 3 + return ( + f'GPU-{device_id} (pynvml): Total: {total_gb:.2f} GB, Used (Overall): {used_gb:.2f} GB' + ) + except NVMLError as e: + return f'GPU-{device_id} (pynvml) Error: {e}' +def get_gpu_memory_stats_pytorch(device_id=0): + if not torch.cuda.is_available(): + return 'PyTorch: CUDA not available.' + if device_id < 0 or device_id >= torch.cuda.device_count(): + return f'PyTorch GPU-{device_id}: Invalid device ID.' + allocated_gb = torch.cuda.memory_allocated(device_id) / 1024 ** 3 + reserved_gb = torch.cuda.memory_reserved(device_id) / 1024 ** 3 + total_capacity_str_pt = '' + if pynvml_available: + try: + handle = nvmlDeviceGetHandleByIndex(device_id) + info = nvmlDeviceGetMemoryInfo(handle) + total_gb_pt = info.total / 1024 ** 3 + total_capacity_str_pt = f'Total Capacity: {total_gb_pt:.2f} GB, ' + except NVMLError: + pass + return ( + f'PyTorch GPU-{device_id}: {total_capacity_str_pt}Allocated: {allocated_gb:.2f} GB, Reserved: {reserved_gb:.2f} GB' + ) +def get_system_ram_stats(): + mem = psutil.virtual_memory() + total_gb = mem.total / 1024 ** 3 + available_gb = mem.available / 1024 ** 3 + used_gb = mem.used / 1024 ** 3 + percent_used = mem.percent + return ( + f'System RAM: Total: {total_gb:.2f} GB, Available: {available_gb:.2f} GB, Used (Overall): {used_gb:.2f} GB ({percent_used}%)' + ) +def get_process_ram_stats(): + process = psutil.Process(os.getpid()) + mem_info = process.memory_info() + rss_gb = mem_info.rss / 1024 ** 3 + return f'App Process RAM (RSS): {rss_gb:.2f} GB' +def get_all_memory_stats_for_gradio_display(): + """Prepare the string of memory/video memory statistics for Gradio display""" + stats_lines = [] + if torch.cuda.is_available() and torch.cuda.device_count() > 0: + stats_lines.append('**GPU VRAM Usage:**') + for i in range(torch.cuda.device_count()): + stats_lines.append(get_gpu_memory_stats_pynvml(i)) + stats_lines.append(get_gpu_memory_stats_pytorch(i)) + if i < torch.cuda.device_count() - 1: + stats_lines.append('---') + else: + stats_lines.append( + '**GPU VRAM Usage:** CUDA not available or no GPUs found.') + stats_lines.append('\n**CPU RAM Usage:**') + stats_lines.append(get_system_ram_stats()) + stats_lines.append(get_process_ram_stats()) + return '\n'.join(s for s in stats_lines if s) +cpu_mem_for_offload = '16GiB' +gpu_mem_per_device = '24GiB' +max_memory_config = {i: gpu_mem_per_device for i in range(torch.cuda. + device_count())} +if torch.cuda.device_count() == 0: + max_memory_config['cpu'] = cpu_mem_for_offload +else: + max_memory_config['cpu'] = cpu_mem_for_offload +print(f'Using max_memory_config: {max_memory_config}') +device_map = infer_auto_device_map(model, max_memory=max_memory_config, + no_split_module_classes=['Bagel', 'Qwen2MoTDecoderLayer']) +print('Device map after infer_auto_device_map (with CPU budget):') +for k, v_map in device_map.items(): + print(f' {k}: {v_map}') +same_device_modules = ['language_model.model.embed_tokens', 'time_embedder', + 'latent_pos_embed', 'vae2llm', 'llm2vae', 'connector', 'vit_pos_embed'] +if torch.cuda.device_count() > 0: + first_device_key = same_device_modules[0] + default_target_device = 'cuda:0' + first_module_target_device = device_map.get(first_device_key, + default_target_device) + print( + f'Target device for same_device_modules (based on {first_device_key}): {first_module_target_device}' + ) + for k_module in same_device_modules: + if k_module in device_map: + if device_map[k_module] != first_module_target_device: + print( + f' Moving {k_module} from {device_map[k_module]} to {first_module_target_device} (same_device_modules)' + ) + device_map[k_module] = first_module_target_device + else: + print( + f' Assigning {k_module} (from same_device_modules) to {first_module_target_device} as it was not in initial map.' + ) + device_map[k_module] = first_module_target_device +elif torch.cuda.device_count() == 0 and 'cpu' in max_memory_config: + print('No CUDA devices found. Assigning same_device_modules to CPU.') + for k_module in same_device_modules: + device_map[k_module] = 'cpu' +print('Device map after same_device_modules logic:') +for k, v_map in device_map.items(): + print(f' {k}: {v_map}') +keys_to_change_to_cpu = [] +for module_name, device_assignment in device_map.items(): + if device_assignment == 'disk': + keys_to_change_to_cpu.append(module_name) +if keys_to_change_to_cpu: + print( + f"Manually changing the following layers from 'disk' to 'cpu': {keys_to_change_to_cpu}" + ) + for module_name in keys_to_change_to_cpu: + device_map[module_name] = 'cpu' + print('Final device_map before loading checkpoint (after disk override):') + for k, v_map in device_map.items(): + print(f' {k}: {v_map}') +else: + print( + "No layers assigned to 'disk' by infer_auto_device_map, or they were already handled. Final device_map is as above." + ) +print( + """ +Starting custom device_map modifications to maximize GPU utilization...""" + ) +print('Device map state BEFORE custom modifications:') +for k_map_item, v_map_item in device_map.items(): + print(f' {k_map_item}: {v_map_item}') +NUM_ADDITIONAL_LLM_LAYERS_TO_GPU = 5 +TRY_MOVE_LLM_NORM_HEAD_TO_GPU = True +TRY_MOVE_VIT_MODEL_TO_GPU = False +moved_llm_layers_count = 0 +if NUM_ADDITIONAL_LLM_LAYERS_TO_GPU > 0: + print( + f""" +Attempting to move up to {NUM_ADDITIONAL_LLM_LAYERS_TO_GPU} LLM layers (11 to {10 + NUM_ADDITIONAL_LLM_LAYERS_TO_GPU}) to GPU 0...""" + ) + for i in range(NUM_ADDITIONAL_LLM_LAYERS_TO_GPU): + layer_idx = 11 + i + if layer_idx > 27: + print( + f' Reached max layer index (27). Stopped LLM layer promotion.' + ) + break + layer_name = f'language_model.model.layers.{layer_idx}' + if device_map.get(layer_name) == 'cpu': + print(f" Promoting LLM layer '{layer_name}' from 'cpu' to GPU 0.") + device_map[layer_name] = 0 + moved_llm_layers_count += 1 + elif layer_name in device_map: + print( + f" LLM Layer '{layer_name}' is already on device '{device_map[layer_name]}'. Skipping promotion." + ) + else: + print( + f" Warning: LLM Layer '{layer_name}' not found in device_map. Cannot promote." + ) + print( + f'Successfully promoted {moved_llm_layers_count} LLM layers to GPU 0.') +else: + print( + """ +Skipping promotion of additional LLM layers based on NUM_ADDITIONAL_LLM_LAYERS_TO_GPU setting.""" + ) +if TRY_MOVE_LLM_NORM_HEAD_TO_GPU: + print( + "\nAttempting to move LLM 'norm' and 'lm_head' to GPU 0 (if on CPU)..." + ) + llm_aux_modules = ['language_model.model.norm', + 'language_model.model.lm_head'] + for module_name in llm_aux_modules: + if device_map.get(module_name) == 'cpu': + print(f" Promoting '{module_name}' from 'cpu' to GPU 0.") + device_map[module_name] = 0 + elif module_name in device_map: + print( + f" Module '{module_name}' is already on device '{device_map[module_name]}'. Skipping promotion." + ) + else: + print( + f" Warning: Module '{module_name}' not found in device_map. Cannot promote." + ) +else: + print( + """ +Skipping promotion of LLM 'norm' and 'lm_head' based on TRY_MOVE_LLM_NORM_HEAD_TO_GPU setting.""" + ) +if TRY_MOVE_VIT_MODEL_TO_GPU: + print("\nAttempting to move 'vit_model' to GPU 0 (if on CPU)...") + vit_module_name = 'vit_model' + if device_map.get(vit_module_name) == 'cpu': + print(f" Promoting '{vit_module_name}' from 'cpu' to GPU 0.") + device_map[vit_module_name] = 0 + elif vit_module_name in device_map: + print( + f" Module '{vit_module_name}' is already on device '{device_map[vit_module_name]}'. Skipping promotion." + ) + else: + print( + f" Warning: Module '{vit_module_name}' not found in device_map. Cannot promote." + ) +else: + print( + "\nSkipping promotion of 'vit_model' based on TRY_MOVE_VIT_MODEL_TO_GPU setting." + ) +print(""" +Final device_map after all custom modifications:""") +for k_map_item, v_map_item in device_map.items(): + print(f' {k_map_item}: {v_map_item}') +print('--- End of custom device_map modifications ---') +model = load_checkpoint_and_dispatch(model, checkpoint=checkpoint, + device_map=device_map, offload_buffers=True, offload_folder='offload', + dtype=torch.bfloat16, force_hooks=True).eval() +inferencer = InterleaveInferencer(model=model, vae_model=vae_model, + tokenizer=tokenizer, vae_transform=vae_transform, vit_transform= + vit_transform, new_token_ids=new_token_ids) +def set_seed(seed): + """Set random seeds for reproducibility""" + if seed > 0: + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + return seed +def text_to_image(prompt, show_thinking=False, cfg_text_scale=4.0, + cfg_interval=0.4, timestep_shift=3.0, num_timesteps=50, cfg_renorm_min= + 1.0, cfg_renorm_type='global', max_think_token_n=1024, do_sample=False, + text_temperature=0.3, seed=0, image_ratio='1:1'): + set_seed(seed) + if image_ratio == '1:1': + image_shapes = 1024, 1024 + elif image_ratio == '4:3': + image_shapes = 768, 1024 + elif image_ratio == '3:4': + image_shapes = 1024, 768 + elif image_ratio == '16:9': + image_shapes = 576, 1024 + elif image_ratio == '9:16': + image_shapes = 1024, 576 + inference_hyper = dict(max_think_token_n=max_think_token_n if + show_thinking else 1024, do_sample=do_sample if show_thinking else + False, text_temperature=text_temperature if show_thinking else 0.3, + cfg_text_scale=cfg_text_scale, cfg_interval=[cfg_interval, 1.0], + timestep_shift=timestep_shift, num_timesteps=num_timesteps, + cfg_renorm_min=cfg_renorm_min, cfg_renorm_type=cfg_renorm_type, + image_shapes=image_shapes) + start_time = time.time() + result = inferencer(text=prompt, think=show_thinking, **inference_hyper) + end_time = time.time() + duration = end_time - start_time + duration_str = f'{duration:.2f} seconds' + print(f'The image takes time: {duration_str}') + return result['image'], result.get('text', None), duration_str +def image_understanding(image, prompt, show_thinking=False, do_sample=False, + text_temperature=0.3, max_new_tokens=512): + start_time = time.time() + if image is None: + return 'Please upload an image.' + if isinstance(image, np.ndarray): + image = Image.fromarray(image) + image = pil_img2rgb(image) + inference_hyper = dict(do_sample=do_sample, text_temperature= + text_temperature, max_think_token_n=max_new_tokens) + result = inferencer(image=image, text=prompt, think=show_thinking, + understanding_output=True, **inference_hyper) + end_time = time.time() + duration = end_time - start_time + duration_str = f'{duration:.2f} seconds' + print(f'The image takes time: {duration_str}') + return result['text'], duration_str +def edit_image(image, prompt, show_thinking=False, cfg_text_scale=4.0, + cfg_img_scale=2.0, cfg_interval=0.0, timestep_shift=3.0, num_timesteps= + 50, cfg_renorm_min=1.0, cfg_renorm_type='text_channel', + max_think_token_n=1024, do_sample=False, text_temperature=0.3, seed=0): + set_seed(seed) + start_time = time.time() + if image is None: + return 'Please upload an image.', '' + if isinstance(image, np.ndarray): + image = Image.fromarray(image) + image = pil_img2rgb(image) + inference_hyper = dict(max_think_token_n=max_think_token_n if + show_thinking else 1024, do_sample=do_sample if show_thinking else + False, text_temperature=text_temperature if show_thinking else 0.3, + cfg_text_scale=cfg_text_scale, cfg_img_scale=cfg_img_scale, + cfg_interval=[cfg_interval, 1.0], timestep_shift=timestep_shift, + num_timesteps=num_timesteps, cfg_renorm_min=cfg_renorm_min, + cfg_renorm_type=cfg_renorm_type) + result = inferencer(image=image, text=prompt, think=show_thinking, ** + inference_hyper) + end_time = time.time() + duration = end_time - start_time + duration_str = f'{duration:.2f} seconds' + print(f'The image takes time: {duration_str}') + return result['image'], result.get('text', ''), duration_str +def load_example_image(image_path): + try: + return Image.open(image_path) + except Exception as e: + print(f'Error loading example image: {e}') + return None +block = gr.Blocks(title='gguf').queue() +with block: + with gr.Tab('📝 Text to Image'): + txt_input = gr.Textbox(label='Prompt', value= + 'A female cosplayer portraying an ethereal fairy or elf, wearing a flowing dress made of delicate fabrics in soft, mystical colors like emerald green and silver. She has pointed ears, a gentle, enchanting expression, and her outfit is adorned with sparkling jewels and intricate patterns. The background is a magical forest with glowing plants, mystical creatures, and a serene atmosphere.' + ) + with gr.Row(): + show_thinking = gr.Checkbox(label='Thinking', value=False) + with gr.Accordion('Inference Hyperparameters', open=False): + with gr.Group(): + with gr.Row(): + seed = gr.Slider(minimum=0, maximum=1000000, value=0, + step=1, label='Seed', info= + '0 for random seed, positive for reproducible results') + image_ratio = gr.Dropdown(choices=['1:1', '4:3', '3:4', + '16:9', '9:16'], value='1:1', label='Image Ratio', + info='The longer size is fixed to 1024') + with gr.Row(): + cfg_text_scale = gr.Slider(minimum=1.0, maximum=8.0, + value=4.0, step=0.1, interactive=True, label= + 'CFG Text Scale', info= + 'Controls how strongly the model follows the text prompt (4.0-8.0)' + ) + cfg_interval = gr.Slider(minimum=0.0, maximum=1.0, + value=0.4, step=0.1, label='CFG Interval', info= + 'Start of CFG application interval (end is fixed at 1.0)' + ) + with gr.Row(): + cfg_renorm_type = gr.Dropdown(choices=['global', + 'local', 'text_channel'], value='global', label= + 'CFG Renorm Type', info= + "If the genrated image is blurry, use 'global'") + cfg_renorm_min = gr.Slider(minimum=0.0, maximum=1.0, + value=0.0, step=0.1, interactive=True, label= + 'CFG Renorm Min', info='1.0 disables CFG-Renorm') + with gr.Row(): + num_timesteps = gr.Slider(minimum=10, maximum=100, + value=50, step=5, interactive=True, label= + 'Timesteps', info='Total denoising steps') + timestep_shift = gr.Slider(minimum=1.0, maximum=5.0, + value=3.0, step=0.5, interactive=True, label= + 'Timestep Shift', info= + 'Higher values for layout, lower for details') + thinking_params = gr.Group(visible=False) + with thinking_params: + with gr.Row(): + do_sample = gr.Checkbox(label='Sampling', value= + False, info='Enable sampling for text generation') + max_think_token_n = gr.Slider(minimum=64, maximum= + 4006, value=1024, step=64, interactive=True, + label='Max Think Tokens', info= + 'Maximum number of tokens for thinking') + text_temperature = gr.Slider(minimum=0.1, maximum= + 1.0, value=0.3, step=0.1, interactive=True, + label='Temperature', info= + 'Controls randomness in text generation') + thinking_output = gr.Textbox(label='Thinking Process', visible=False) + img_output = gr.Image(label='Generated Image') + gen_btn = gr.Button('Generate', variant='primary') + generation_time_output = gr.Textbox(label='Processing Time', + interactive=False) + def update_thinking_visibility(show): + return gr.update(visible=show), gr.update(visible=show) + show_thinking.change(fn=update_thinking_visibility, inputs=[ + show_thinking], outputs=[thinking_output, thinking_params]) + def process_text_to_image(prompt, show_thinking, cfg_text_scale, + cfg_interval, timestep_shift, num_timesteps, cfg_renorm_min, + cfg_renorm_type, max_think_token_n, do_sample, text_temperature, + seed, image_ratio): + image, thinking, duration_str = text_to_image(prompt, + show_thinking, cfg_text_scale, cfg_interval, timestep_shift, + num_timesteps, cfg_renorm_min, cfg_renorm_type, + max_think_token_n, do_sample, text_temperature, seed, + image_ratio) + return image, thinking if thinking else '', duration_str + gr.on(triggers=[gen_btn.click, txt_input.submit], fn= + process_text_to_image, inputs=[txt_input, show_thinking, + cfg_text_scale, cfg_interval, timestep_shift, num_timesteps, + cfg_renorm_min, cfg_renorm_type, max_think_token_n, do_sample, + text_temperature, seed, image_ratio], outputs=[img_output, + thinking_output, generation_time_output]) + with gr.Tab('🖌️ Image Edit'): + with gr.Row(): + with gr.Column(scale=1): + edit_image_input = gr.Image(label='Input Image', value= + load_example_image(os.path.join(os.path.dirname( + __file__), '../bagel2/test_images/women.jpg'))) + edit_prompt = gr.Textbox(label='Prompt', value= + 'She boards a modern subway, quietly reading a folded newspaper, wearing the same clothes.' + ) + with gr.Column(scale=1): + edit_image_output = gr.Image(label='Result') + edit_thinking_output = gr.Textbox(label='Thinking Process', + visible=False) + with gr.Row(): + edit_show_thinking = gr.Checkbox(label='Thinking', value=False) + with gr.Accordion('Inference Hyperparameters', open=False): + with gr.Group(): + with gr.Row(): + edit_seed = gr.Slider(minimum=0, maximum=1000000, value + =0, step=1, interactive=True, label='Seed', info= + '0 for random seed, positive for reproducible results') + edit_cfg_text_scale = gr.Slider(minimum=1.0, maximum= + 8.0, value=4.0, step=0.1, interactive=True, label= + 'CFG Text Scale', info= + 'Controls how strongly the model follows the text prompt' + ) + with gr.Row(): + edit_cfg_img_scale = gr.Slider(minimum=1.0, maximum=4.0, + value=2.0, step=0.1, interactive=True, label= + 'CFG Image Scale', info= + 'Controls how much the model preserves input image details' + ) + edit_cfg_interval = gr.Slider(minimum=0.0, maximum=1.0, + value=0.0, step=0.1, interactive=True, label= + 'CFG Interval', info= + 'Start of CFG application interval (end is fixed at 1.0)' + ) + with gr.Row(): + edit_cfg_renorm_type = gr.Dropdown(choices=['global', + 'local', 'text_channel'], value='text_channel', + label='CFG Renorm Type', info= + "If the genrated image is blurry, use 'global") + edit_cfg_renorm_min = gr.Slider(minimum=0.0, maximum= + 1.0, value=0.0, step=0.1, interactive=True, label= + 'CFG Renorm Min', info='1.0 disables CFG-Renorm') + with gr.Row(): + edit_num_timesteps = gr.Slider(minimum=10, maximum=100, + value=50, step=5, interactive=True, label= + 'Timesteps', info='Total denoising steps') + edit_timestep_shift = gr.Slider(minimum=1.0, maximum= + 10.0, value=3.0, step=0.5, interactive=True, label= + 'Timestep Shift', info= + 'Higher values for layout, lower for details') + edit_thinking_params = gr.Group(visible=False) + with edit_thinking_params: + with gr.Row(): + edit_do_sample = gr.Checkbox(label='Sampling', + value=False, info= + 'Enable sampling for text generation') + edit_max_think_token_n = gr.Slider(minimum=64, + maximum=4006, value=1024, step=64, interactive= + True, label='Max Think Tokens', info= + 'Maximum number of tokens for thinking') + edit_text_temperature = gr.Slider(minimum=0.1, + maximum=1.0, value=0.3, step=0.1, interactive= + True, label='Temperature', info= + 'Controls randomness in text generation') + edit_btn = gr.Button('Submit', variant='primary') + edit_generation_time_output = gr.Textbox(label='Processing Time', + interactive=False) + def update_edit_thinking_visibility(show): + return gr.update(visible=show), gr.update(visible=show) + edit_show_thinking.change(fn=update_edit_thinking_visibility, + inputs=[edit_show_thinking], outputs=[edit_thinking_output, + edit_thinking_params]) + def process_edit_image(image, prompt, show_thinking, cfg_text_scale, + cfg_img_scale, cfg_interval, timestep_shift, num_timesteps, + cfg_renorm_min, cfg_renorm_type, max_think_token_n, do_sample, + text_temperature, seed): + edited_image, thinking, duration_str = edit_image(image, prompt, + show_thinking, cfg_text_scale, cfg_img_scale, cfg_interval, + timestep_shift, num_timesteps, cfg_renorm_min, + cfg_renorm_type, max_think_token_n, do_sample, + text_temperature, seed) + return edited_image, thinking if thinking else '', duration_str + gr.on(triggers=[edit_btn.click, edit_prompt.submit], fn= + process_edit_image, inputs=[edit_image_input, edit_prompt, + edit_show_thinking, edit_cfg_text_scale, edit_cfg_img_scale, + edit_cfg_interval, edit_timestep_shift, edit_num_timesteps, + edit_cfg_renorm_min, edit_cfg_renorm_type, + edit_max_think_token_n, edit_do_sample, edit_text_temperature, + edit_seed], outputs=[edit_image_output, edit_thinking_output, + edit_generation_time_output]) + with gr.Tab('🖼️ Image Understanding'): + with gr.Row(): + with gr.Column(scale=1): + img_input = gr.Image(label='Input Image', value= + load_example_image(os.path.join(os.path.dirname( + __file__), '../bagel2/test_images/meme.jpg'))) + understand_prompt = gr.Textbox(label='Prompt', value= + "Can someone explain what's funny about this meme??") + with gr.Column(scale=1): + txt_output = gr.Textbox(label='Result', lines=20) + with gr.Row(): + understand_show_thinking = gr.Checkbox(label='Thinking', value= + False) + with gr.Accordion('Inference Hyperparameters', open=False): + with gr.Row(): + understand_do_sample = gr.Checkbox(label='Sampling', value= + False, info='Enable sampling for text generation') + understand_text_temperature = gr.Slider(minimum=0.0, + maximum=1.0, value=0.3, step=0.05, interactive=True, + label='Temperature', info= + 'Controls randomness in text generation (0=deterministic, 1=creative)' + ) + understand_max_new_tokens = gr.Slider(minimum=64, maximum= + 4096, value=512, step=64, interactive=True, label= + 'Max New Tokens', info= + 'Maximum length of generated text, including potential thinking' + ) + img_understand_btn = gr.Button('Submit', variant='primary') + understand_generation_time_output = gr.Textbox(label= + 'Processing Time', interactive=False) + def process_understanding(image, prompt, show_thinking, do_sample, + text_temperature, max_new_tokens): + result, duration_str = image_understanding(image, prompt, + show_thinking, do_sample, text_temperature, max_new_tokens) + return result, duration_str + gr.on(triggers=[img_understand_btn.click, understand_prompt.submit], + fn=process_understanding, inputs=[img_input, understand_prompt, + understand_show_thinking, understand_do_sample, + understand_text_temperature, understand_max_new_tokens], + outputs=[txt_output, understand_generation_time_output]) + with gr.Tab('📊 System Monitor'): + with gr.Column(): + memory_stats_display = gr.Markdown('Check RAM/VRAM Stats') + refresh_button = gr.Button('🔄 Check RAM/VRAM Stats') + refresh_button.click(fn=get_all_memory_stats_for_gradio_display, + inputs=None, outputs=[memory_stats_display]) +block.launch() \ No newline at end of file diff --git a/gguf_connector/b2.py b/gguf_connector/b2.py new file mode 100644 index 0000000..5caaab9 --- /dev/null +++ b/gguf_connector/b2.py @@ -0,0 +1,681 @@ +import os +if not os.path.isfile(os.path.join(os.path.dirname(__file__), + 'models/bagel/config.json')): + from huggingface_hub import snapshot_download + save_dir = os.path.join(os.path.dirname(__file__), 'models/bagel') + repo_id = 'callgg/bagel-decoder' + cache_dir = save_dir + '/cache' + snapshot_download(cache_dir=cache_dir, local_dir=save_dir, repo_id= + repo_id, allow_patterns=['*.json', '*.md', '*.txt']) +import time, psutil, platform, atexit, random +pynvml_available = False +if platform.system() == 'Linux' or platform.system() == 'Windows': + try: + from pynvml import * + nvmlInit() + pynvml_available = True + print( + """pynvml (NVIDIA GPU monitoring library) initialized successfully! +Detecting GGUF/Safetensors... +""" + ) + def shutdown_pynvml(): + print('Shutting down pynvml...') + nvmlShutdown() + atexit.register(shutdown_pynvml) + except Exception as e: + print( + f'Warning: pynvml could not be initialized. Detailed GPU stats via pynvml will not be available. Error: {e}' + ) + if 'NVML Shared Library Not Found' in str(e): + print( + 'pynvml error hint: NVML shared library not found. If you have an NVIDIA GPU and drivers, ensure the library is accessible.' + ) + elif 'Driver Not Loaded' in str(e): + print( + 'pynvml error hint: NVIDIA driver is not loaded. Please check your GPU driver installation.' + ) +import torch +import gradio as gr +import numpy as np +from accelerate import infer_auto_device_map, load_checkpoint_and_dispatch, init_empty_weights +from PIL import Image +from bagel2.data.data_utils import add_special_tokens, pil_img2rgb +from bagel2.data.transforms import ImageTransform +from bagel2.inferencer import InterleaveInferencer +from bagel2.modeling.autoencoder import load_ae +from bagel2.modeling.bagel.qwen2_navit import NaiveCache +from bagel2.modeling.bagel import BagelConfig, Bagel, Qwen2Config, Qwen2ForCausalLM, SiglipVisionConfig, SiglipVisionModel +from bagel2.modeling.qwen2 import Qwen2Tokenizer +model_path = os.path.join(os.path.dirname(__file__), 'models/bagel') +llm_config = Qwen2Config.from_json_file(os.path.join(model_path, + 'llm_config.json')) +llm_config.qk_norm = True +llm_config.tie_word_embeddings = False +llm_config.layer_module = 'Qwen2MoTDecoderLayer' +vit_config = SiglipVisionConfig.from_json_file(os.path.join(model_path, + 'vit_config.json')) +vit_config.rope = False +vit_config.num_hidden_layers -= 1 +from safetensors.torch import save_file +from typing import Dict, Tuple +from tqdm import tqdm +from .reader import GGUFReader +from .quant import dequantize +def load_gguf_and_extract_metadata(gguf_path): + reader = GGUFReader(gguf_path) + tensors_metadata = [] + for tensor in reader.tensors: + tensor_metadata = {'name': tensor.name, 'shape': tuple(tensor.shape + .tolist()), 'n_elements': tensor.n_elements, 'n_bytes': tensor. + n_bytes, 'data_offset': tensor.data_offset, 'type': tensor. + tensor_type} + tensors_metadata.append(tensor_metadata) + return reader, tensors_metadata +def convert_gguf_to_safetensors(gguf_path, output_path, use_bf16): + reader, tensors_metadata = load_gguf_and_extract_metadata(gguf_path) + print(f'Extracted {len(tensors_metadata)} tensors from GGUF file') + tensors_dict: dict[str, torch.Tensor] = {} + for i, tensor_info in enumerate(tqdm(tensors_metadata, desc= + 'Dequantizing tensors', unit='tensor')): + tensor_name = tensor_info['name'] + tensor_data = reader.get_tensor(i) + weights = dequantize(tensor_data.data, tensor_data.tensor_type).copy() + try: + if use_bf16: + weights_tensor = torch.from_numpy(weights).to(dtype=torch. + float32) + weights_tensor = weights_tensor.to(torch.bfloat16) + else: + weights_tensor = torch.from_numpy(weights).to(dtype=torch. + float32) + weights_hf = weights_tensor + except Exception as e: + print( + f"Error during dequantization for tensor '{tensor_name}': {e}") + weights_tensor = torch.from_numpy(weights.astype(np.float32)).to( + torch.float16) + weights_hf = weights_tensor + tensors_dict[tensor_name] = weights_hf + metadata = {key: str(reader.get_field(key)) for key in reader.fields} + save_file(tensors_dict, output_path, metadata=metadata) + print('Dequantization complete!') +gguf_files = [file for file in os.listdir() if file.endswith('.gguf')] +safetensors_files = [file for file in os.listdir() if file.endswith( + '.safetensors')] +if gguf_files: + print('GGUF file(s) available. Select which one for VAE:') + for index, file_name in enumerate(gguf_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(gguf_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = gguf_files[choice_index] + print(f'VAE file: {selected_file} is selected!') + input_path = selected_file + use_bf16 = False + vae_path = f'{os.path.splitext(input_path)[0]}-f32.safetensors' + if safetensors_files: + print( + '\nSafetensors file(s) available. Select which one for MODEL:') + for index, file_name in enumerate(safetensors_files, start=1): + print(f'{index}. {file_name}') + choice2 = input( + f'Enter your choice (1 to {len(safetensors_files)}): ') + try: + choice_index = int(choice2) - 1 + selected_model_file = safetensors_files[choice_index] + print(f'MODEL file: {selected_model_file} is selected!\n') + convert_gguf_to_safetensors(input_path, vae_path, use_bf16) + vae_model, vae_config = load_ae(vae_path) + checkpoint = selected_model_file + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No GGUF/Safetensors are available in the current directory.') + input('--- Press ENTER To Exit ---') +config = BagelConfig(visual_gen=True, visual_und=True, llm_config= + llm_config, vit_config=vit_config, vae_config=vae_config, + vit_max_num_patch_per_side=70, connector_act='gelu_pytorch_tanh', + latent_patch_size=2, max_latent_size=64) +with init_empty_weights(): + language_model = Qwen2ForCausalLM(llm_config) + vit_model = SiglipVisionModel(vit_config) + model = Bagel(language_model, vit_model, config) + model.vit_model.vision_model.embeddings.convert_conv2d_to_linear(vit_config + , meta=True) +tokenizer = Qwen2Tokenizer.from_pretrained(model_path) +tokenizer, new_token_ids, _ = add_special_tokens(tokenizer) +vae_transform = ImageTransform(1024, 512, 16) +vit_transform = ImageTransform(980, 224, 14) +print('Starting model loading and device map configuration...') +def get_gpu_memory_stats_pynvml(device_id=0): + if not pynvml_available: + return f'GPU-{device_id} (pynvml): Not available.' + try: + handle = nvmlDeviceGetHandleByIndex(device_id) + info = nvmlDeviceGetMemoryInfo(handle) + total_gb = info.total / 1024 ** 3 + used_gb = info.used / 1024 ** 3 + return ( + f'GPU-{device_id} (pynvml): Total: {total_gb:.2f} GB, Used (Overall): {used_gb:.2f} GB' + ) + except NVMLError as e: + return f'GPU-{device_id} (pynvml) Error: {e}' +def get_gpu_memory_stats_pytorch(device_id=0): + if not torch.cuda.is_available(): + return 'PyTorch: CUDA not available.' + if device_id < 0 or device_id >= torch.cuda.device_count(): + return f'PyTorch GPU-{device_id}: Invalid device ID.' + allocated_gb = torch.cuda.memory_allocated(device_id) / 1024 ** 3 + reserved_gb = torch.cuda.memory_reserved(device_id) / 1024 ** 3 + total_capacity_str_pt = '' + if pynvml_available: + try: + handle = nvmlDeviceGetHandleByIndex(device_id) + info = nvmlDeviceGetMemoryInfo(handle) + total_gb_pt = info.total / 1024 ** 3 + total_capacity_str_pt = f'Total Capacity: {total_gb_pt:.2f} GB, ' + except NVMLError: + pass + return ( + f'PyTorch GPU-{device_id}: {total_capacity_str_pt}Allocated: {allocated_gb:.2f} GB, Reserved: {reserved_gb:.2f} GB' + ) +def get_system_ram_stats(): + mem = psutil.virtual_memory() + total_gb = mem.total / 1024 ** 3 + available_gb = mem.available / 1024 ** 3 + used_gb = mem.used / 1024 ** 3 + percent_used = mem.percent + return ( + f'System RAM: Total: {total_gb:.2f} GB, Available: {available_gb:.2f} GB, Used (Overall): {used_gb:.2f} GB ({percent_used}%)' + ) +def get_process_ram_stats(): + process = psutil.Process(os.getpid()) + mem_info = process.memory_info() + rss_gb = mem_info.rss / 1024 ** 3 + return f'App Process RAM (RSS): {rss_gb:.2f} GB' +def get_all_memory_stats_for_gradio_display(): + """Prepare the string of memory/video memory statistics for Gradio display""" + stats_lines = [] + if torch.cuda.is_available() and torch.cuda.device_count() > 0: + stats_lines.append('**GPU VRAM Usage:**') + for i in range(torch.cuda.device_count()): + stats_lines.append(get_gpu_memory_stats_pynvml(i)) + stats_lines.append(get_gpu_memory_stats_pytorch(i)) + if i < torch.cuda.device_count() - 1: + stats_lines.append('---') + else: + stats_lines.append( + '**GPU VRAM Usage:** CUDA not available or no GPUs found.') + stats_lines.append('\n**CPU RAM Usage:**') + stats_lines.append(get_system_ram_stats()) + stats_lines.append(get_process_ram_stats()) + return '\n'.join(s for s in stats_lines if s) +cpu_mem_for_offload = '16GiB' +gpu_mem_per_device = '24GiB' +max_memory_config = {i: gpu_mem_per_device for i in range(torch.cuda. + device_count())} +if torch.cuda.device_count() == 0: + max_memory_config['cpu'] = cpu_mem_for_offload +else: + max_memory_config['cpu'] = cpu_mem_for_offload +print(f'Using max_memory_config: {max_memory_config}') +device_map = infer_auto_device_map(model, max_memory=max_memory_config, + no_split_module_classes=['Bagel', 'Qwen2MoTDecoderLayer']) +print('Device map after infer_auto_device_map (with CPU budget):') +for k, v_map in device_map.items(): + print(f' {k}: {v_map}') +same_device_modules = ['language_model.model.embed_tokens', 'time_embedder', + 'latent_pos_embed', 'vae2llm', 'llm2vae', 'connector', 'vit_pos_embed'] +if torch.cuda.device_count() > 0: + first_device_key = same_device_modules[0] + default_target_device = 'cuda:0' + first_module_target_device = device_map.get(first_device_key, + default_target_device) + print( + f'Target device for same_device_modules (based on {first_device_key}): {first_module_target_device}' + ) + for k_module in same_device_modules: + if k_module in device_map: + if device_map[k_module] != first_module_target_device: + print( + f' Moving {k_module} from {device_map[k_module]} to {first_module_target_device} (same_device_modules)' + ) + device_map[k_module] = first_module_target_device + else: + print( + f' Assigning {k_module} (from same_device_modules) to {first_module_target_device} as it was not in initial map.' + ) + device_map[k_module] = first_module_target_device +elif torch.cuda.device_count() == 0 and 'cpu' in max_memory_config: + print('No CUDA devices found. Assigning same_device_modules to CPU.') + for k_module in same_device_modules: + device_map[k_module] = 'cpu' +print('Device map after same_device_modules logic:') +for k, v_map in device_map.items(): + print(f' {k}: {v_map}') +keys_to_change_to_cpu = [] +for module_name, device_assignment in device_map.items(): + if device_assignment == 'disk': + keys_to_change_to_cpu.append(module_name) +if keys_to_change_to_cpu: + print( + f"Manually changing the following layers from 'disk' to 'cpu': {keys_to_change_to_cpu}" + ) + for module_name in keys_to_change_to_cpu: + device_map[module_name] = 'cpu' + print('Final device_map before loading checkpoint (after disk override):') + for k, v_map in device_map.items(): + print(f' {k}: {v_map}') +else: + print( + "No layers assigned to 'disk' by infer_auto_device_map, or they were already handled. Final device_map is as above." + ) +print( + """ +Starting custom device_map modifications to maximize GPU utilization...""" + ) +print('Device map state BEFORE custom modifications:') +for k_map_item, v_map_item in device_map.items(): + print(f' {k_map_item}: {v_map_item}') +NUM_ADDITIONAL_LLM_LAYERS_TO_GPU = 5 +TRY_MOVE_LLM_NORM_HEAD_TO_GPU = True +TRY_MOVE_VIT_MODEL_TO_GPU = False +moved_llm_layers_count = 0 +if NUM_ADDITIONAL_LLM_LAYERS_TO_GPU > 0: + print( + f""" +Attempting to move up to {NUM_ADDITIONAL_LLM_LAYERS_TO_GPU} LLM layers (11 to {10 + NUM_ADDITIONAL_LLM_LAYERS_TO_GPU}) to GPU 0...""" + ) + for i in range(NUM_ADDITIONAL_LLM_LAYERS_TO_GPU): + layer_idx = 11 + i + if layer_idx > 27: + print( + f' Reached max layer index (27). Stopped LLM layer promotion.' + ) + break + layer_name = f'language_model.model.layers.{layer_idx}' + if device_map.get(layer_name) == 'cpu': + print(f" Promoting LLM layer '{layer_name}' from 'cpu' to GPU 0.") + device_map[layer_name] = 0 + moved_llm_layers_count += 1 + elif layer_name in device_map: + print( + f" LLM Layer '{layer_name}' is already on device '{device_map[layer_name]}'. Skipping promotion." + ) + else: + print( + f" Warning: LLM Layer '{layer_name}' not found in device_map. Cannot promote." + ) + print( + f'Successfully promoted {moved_llm_layers_count} LLM layers to GPU 0.') +else: + print( + """ +Skipping promotion of additional LLM layers based on NUM_ADDITIONAL_LLM_LAYERS_TO_GPU setting.""" + ) +if TRY_MOVE_LLM_NORM_HEAD_TO_GPU: + print( + "\nAttempting to move LLM 'norm' and 'lm_head' to GPU 0 (if on CPU)..." + ) + llm_aux_modules = ['language_model.model.norm', + 'language_model.model.lm_head'] + for module_name in llm_aux_modules: + if device_map.get(module_name) == 'cpu': + print(f" Promoting '{module_name}' from 'cpu' to GPU 0.") + device_map[module_name] = 0 + elif module_name in device_map: + print( + f" Module '{module_name}' is already on device '{device_map[module_name]}'. Skipping promotion." + ) + else: + print( + f" Warning: Module '{module_name}' not found in device_map. Cannot promote." + ) +else: + print( + """ +Skipping promotion of LLM 'norm' and 'lm_head' based on TRY_MOVE_LLM_NORM_HEAD_TO_GPU setting.""" + ) +if TRY_MOVE_VIT_MODEL_TO_GPU: + print("\nAttempting to move 'vit_model' to GPU 0 (if on CPU)...") + vit_module_name = 'vit_model' + if device_map.get(vit_module_name) == 'cpu': + print(f" Promoting '{vit_module_name}' from 'cpu' to GPU 0.") + device_map[vit_module_name] = 0 + elif vit_module_name in device_map: + print( + f" Module '{vit_module_name}' is already on device '{device_map[vit_module_name]}'. Skipping promotion." + ) + else: + print( + f" Warning: Module '{vit_module_name}' not found in device_map. Cannot promote." + ) +else: + print( + "\nSkipping promotion of 'vit_model' based on TRY_MOVE_VIT_MODEL_TO_GPU setting." + ) +print(""" +Final device_map after all custom modifications:""") +for k_map_item, v_map_item in device_map.items(): + print(f' {k_map_item}: {v_map_item}') +print('--- End of custom device_map modifications ---') +model = load_checkpoint_and_dispatch(model, checkpoint=checkpoint, + device_map=device_map, offload_buffers=True, offload_folder='offload', + dtype=torch.bfloat16, force_hooks=True).eval() +inferencer = InterleaveInferencer(model=model, vae_model=vae_model, + tokenizer=tokenizer, vae_transform=vae_transform, vit_transform= + vit_transform, new_token_ids=new_token_ids) +def set_seed(seed): + """Set random seeds for reproducibility""" + if seed > 0: + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + return seed +def text_to_image(prompt, show_thinking=False, cfg_text_scale=4.0, + cfg_interval=0.4, timestep_shift=3.0, num_timesteps=50, cfg_renorm_min= + 1.0, cfg_renorm_type='global', max_think_token_n=1024, do_sample=False, + text_temperature=0.3, seed=0, image_ratio='1:1'): + set_seed(seed) + if image_ratio == '1:1': + image_shapes = 1024, 1024 + elif image_ratio == '4:3': + image_shapes = 768, 1024 + elif image_ratio == '3:4': + image_shapes = 1024, 768 + elif image_ratio == '16:9': + image_shapes = 576, 1024 + elif image_ratio == '9:16': + image_shapes = 1024, 576 + inference_hyper = dict(max_think_token_n=max_think_token_n if + show_thinking else 1024, do_sample=do_sample if show_thinking else + False, text_temperature=text_temperature if show_thinking else 0.3, + cfg_text_scale=cfg_text_scale, cfg_interval=[cfg_interval, 1.0], + timestep_shift=timestep_shift, num_timesteps=num_timesteps, + cfg_renorm_min=cfg_renorm_min, cfg_renorm_type=cfg_renorm_type, + image_shapes=image_shapes) + start_time = time.time() + result = inferencer(text=prompt, think=show_thinking, **inference_hyper) + end_time = time.time() + duration = end_time - start_time + duration_str = f'{duration:.2f} seconds' + print(f'The image takes time: {duration_str}') + return result['image'], result.get('text', None), duration_str +def image_understanding(image, prompt, show_thinking=False, do_sample=False, + text_temperature=0.3, max_new_tokens=512): + start_time = time.time() + if image is None: + return 'Please upload an image.' + if isinstance(image, np.ndarray): + image = Image.fromarray(image) + image = pil_img2rgb(image) + inference_hyper = dict(do_sample=do_sample, text_temperature= + text_temperature, max_think_token_n=max_new_tokens) + result = inferencer(image=image, text=prompt, think=show_thinking, + understanding_output=True, **inference_hyper) + end_time = time.time() + duration = end_time - start_time + duration_str = f'{duration:.2f} seconds' + print(f'The image takes time: {duration_str}') + return result['text'], duration_str +def edit_image(image, prompt, show_thinking=False, cfg_text_scale=4.0, + cfg_img_scale=2.0, cfg_interval=0.0, timestep_shift=3.0, num_timesteps= + 50, cfg_renorm_min=1.0, cfg_renorm_type='text_channel', + max_think_token_n=1024, do_sample=False, text_temperature=0.3, seed=0): + set_seed(seed) + start_time = time.time() + if image is None: + return 'Please upload an image.', '' + if isinstance(image, np.ndarray): + image = Image.fromarray(image) + image = pil_img2rgb(image) + inference_hyper = dict(max_think_token_n=max_think_token_n if + show_thinking else 1024, do_sample=do_sample if show_thinking else + False, text_temperature=text_temperature if show_thinking else 0.3, + cfg_text_scale=cfg_text_scale, cfg_img_scale=cfg_img_scale, + cfg_interval=[cfg_interval, 1.0], timestep_shift=timestep_shift, + num_timesteps=num_timesteps, cfg_renorm_min=cfg_renorm_min, + cfg_renorm_type=cfg_renorm_type) + result = inferencer(image=image, text=prompt, think=show_thinking, ** + inference_hyper) + end_time = time.time() + duration = end_time - start_time + duration_str = f'{duration:.2f} seconds' + print(f'The image takes time: {duration_str}') + return result['image'], result.get('text', ''), duration_str +def load_example_image(image_path): + try: + return Image.open(image_path) + except Exception as e: + print(f'Error loading example image: {e}') + return None +block = gr.Blocks(title='gguf').queue() +with block: + with gr.Tab('📝 Text to Image'): + txt_input = gr.Textbox(label='Prompt', value= + 'A female cosplayer portraying an ethereal fairy or elf, wearing a flowing dress made of delicate fabrics in soft, mystical colors like emerald green and silver. She has pointed ears, a gentle, enchanting expression, and her outfit is adorned with sparkling jewels and intricate patterns. The background is a magical forest with glowing plants, mystical creatures, and a serene atmosphere.' + ) + with gr.Row(): + show_thinking = gr.Checkbox(label='Thinking', value=False) + with gr.Accordion('Inference Hyperparameters', open=False): + with gr.Group(): + with gr.Row(): + seed = gr.Slider(minimum=0, maximum=1000000, value=0, + step=1, label='Seed', info= + '0 for random seed, positive for reproducible results') + image_ratio = gr.Dropdown(choices=['1:1', '4:3', '3:4', + '16:9', '9:16'], value='1:1', label='Image Ratio', + info='The longer size is fixed to 1024') + with gr.Row(): + cfg_text_scale = gr.Slider(minimum=1.0, maximum=8.0, + value=4.0, step=0.1, interactive=True, label= + 'CFG Text Scale', info= + 'Controls how strongly the model follows the text prompt (4.0-8.0)' + ) + cfg_interval = gr.Slider(minimum=0.0, maximum=1.0, + value=0.4, step=0.1, label='CFG Interval', info= + 'Start of CFG application interval (end is fixed at 1.0)' + ) + with gr.Row(): + cfg_renorm_type = gr.Dropdown(choices=['global', + 'local', 'text_channel'], value='global', label= + 'CFG Renorm Type', info= + "If the genrated image is blurry, use 'global'") + cfg_renorm_min = gr.Slider(minimum=0.0, maximum=1.0, + value=0.0, step=0.1, interactive=True, label= + 'CFG Renorm Min', info='1.0 disables CFG-Renorm') + with gr.Row(): + num_timesteps = gr.Slider(minimum=10, maximum=100, + value=50, step=5, interactive=True, label= + 'Timesteps', info='Total denoising steps') + timestep_shift = gr.Slider(minimum=1.0, maximum=5.0, + value=3.0, step=0.5, interactive=True, label= + 'Timestep Shift', info= + 'Higher values for layout, lower for details') + thinking_params = gr.Group(visible=False) + with thinking_params: + with gr.Row(): + do_sample = gr.Checkbox(label='Sampling', value= + False, info='Enable sampling for text generation') + max_think_token_n = gr.Slider(minimum=64, maximum= + 4006, value=1024, step=64, interactive=True, + label='Max Think Tokens', info= + 'Maximum number of tokens for thinking') + text_temperature = gr.Slider(minimum=0.1, maximum= + 1.0, value=0.3, step=0.1, interactive=True, + label='Temperature', info= + 'Controls randomness in text generation') + thinking_output = gr.Textbox(label='Thinking Process', visible=False) + img_output = gr.Image(label='Generated Image') + gen_btn = gr.Button('Generate', variant='primary') + generation_time_output = gr.Textbox(label='Processing Time', + interactive=False) + def update_thinking_visibility(show): + return gr.update(visible=show), gr.update(visible=show) + show_thinking.change(fn=update_thinking_visibility, inputs=[ + show_thinking], outputs=[thinking_output, thinking_params]) + def process_text_to_image(prompt, show_thinking, cfg_text_scale, + cfg_interval, timestep_shift, num_timesteps, cfg_renorm_min, + cfg_renorm_type, max_think_token_n, do_sample, text_temperature, + seed, image_ratio): + image, thinking, duration_str = text_to_image(prompt, + show_thinking, cfg_text_scale, cfg_interval, timestep_shift, + num_timesteps, cfg_renorm_min, cfg_renorm_type, + max_think_token_n, do_sample, text_temperature, seed, + image_ratio) + return image, thinking if thinking else '', duration_str + gr.on(triggers=[gen_btn.click, txt_input.submit], fn= + process_text_to_image, inputs=[txt_input, show_thinking, + cfg_text_scale, cfg_interval, timestep_shift, num_timesteps, + cfg_renorm_min, cfg_renorm_type, max_think_token_n, do_sample, + text_temperature, seed, image_ratio], outputs=[img_output, + thinking_output, generation_time_output]) + with gr.Tab('🖌️ Image Edit'): + with gr.Row(): + with gr.Column(scale=1): + edit_image_input = gr.Image(label='Input Image', value= + load_example_image(os.path.join(os.path.dirname( + __file__), '../bagel2/test_images/women.jpg'))) + edit_prompt = gr.Textbox(label='Prompt', value= + 'She boards a modern subway, quietly reading a folded newspaper, wearing the same clothes.' + ) + with gr.Column(scale=1): + edit_image_output = gr.Image(label='Result') + edit_thinking_output = gr.Textbox(label='Thinking Process', + visible=False) + with gr.Row(): + edit_show_thinking = gr.Checkbox(label='Thinking', value=False) + with gr.Accordion('Inference Hyperparameters', open=False): + with gr.Group(): + with gr.Row(): + edit_seed = gr.Slider(minimum=0, maximum=1000000, value + =0, step=1, interactive=True, label='Seed', info= + '0 for random seed, positive for reproducible results') + edit_cfg_text_scale = gr.Slider(minimum=1.0, maximum= + 8.0, value=4.0, step=0.1, interactive=True, label= + 'CFG Text Scale', info= + 'Controls how strongly the model follows the text prompt' + ) + with gr.Row(): + edit_cfg_img_scale = gr.Slider(minimum=1.0, maximum=4.0, + value=2.0, step=0.1, interactive=True, label= + 'CFG Image Scale', info= + 'Controls how much the model preserves input image details' + ) + edit_cfg_interval = gr.Slider(minimum=0.0, maximum=1.0, + value=0.0, step=0.1, interactive=True, label= + 'CFG Interval', info= + 'Start of CFG application interval (end is fixed at 1.0)' + ) + with gr.Row(): + edit_cfg_renorm_type = gr.Dropdown(choices=['global', + 'local', 'text_channel'], value='text_channel', + label='CFG Renorm Type', info= + "If the genrated image is blurry, use 'global") + edit_cfg_renorm_min = gr.Slider(minimum=0.0, maximum= + 1.0, value=0.0, step=0.1, interactive=True, label= + 'CFG Renorm Min', info='1.0 disables CFG-Renorm') + with gr.Row(): + edit_num_timesteps = gr.Slider(minimum=10, maximum=100, + value=50, step=5, interactive=True, label= + 'Timesteps', info='Total denoising steps') + edit_timestep_shift = gr.Slider(minimum=1.0, maximum= + 10.0, value=3.0, step=0.5, interactive=True, label= + 'Timestep Shift', info= + 'Higher values for layout, lower for details') + edit_thinking_params = gr.Group(visible=False) + with edit_thinking_params: + with gr.Row(): + edit_do_sample = gr.Checkbox(label='Sampling', + value=False, info= + 'Enable sampling for text generation') + edit_max_think_token_n = gr.Slider(minimum=64, + maximum=4006, value=1024, step=64, interactive= + True, label='Max Think Tokens', info= + 'Maximum number of tokens for thinking') + edit_text_temperature = gr.Slider(minimum=0.1, + maximum=1.0, value=0.3, step=0.1, interactive= + True, label='Temperature', info= + 'Controls randomness in text generation') + edit_btn = gr.Button('Submit', variant='primary') + edit_generation_time_output = gr.Textbox(label='Processing Time', + interactive=False) + def update_edit_thinking_visibility(show): + return gr.update(visible=show), gr.update(visible=show) + edit_show_thinking.change(fn=update_edit_thinking_visibility, + inputs=[edit_show_thinking], outputs=[edit_thinking_output, + edit_thinking_params]) + def process_edit_image(image, prompt, show_thinking, cfg_text_scale, + cfg_img_scale, cfg_interval, timestep_shift, num_timesteps, + cfg_renorm_min, cfg_renorm_type, max_think_token_n, do_sample, + text_temperature, seed): + edited_image, thinking, duration_str = edit_image(image, prompt, + show_thinking, cfg_text_scale, cfg_img_scale, cfg_interval, + timestep_shift, num_timesteps, cfg_renorm_min, + cfg_renorm_type, max_think_token_n, do_sample, + text_temperature, seed) + return edited_image, thinking if thinking else '', duration_str + gr.on(triggers=[edit_btn.click, edit_prompt.submit], fn= + process_edit_image, inputs=[edit_image_input, edit_prompt, + edit_show_thinking, edit_cfg_text_scale, edit_cfg_img_scale, + edit_cfg_interval, edit_timestep_shift, edit_num_timesteps, + edit_cfg_renorm_min, edit_cfg_renorm_type, + edit_max_think_token_n, edit_do_sample, edit_text_temperature, + edit_seed], outputs=[edit_image_output, edit_thinking_output, + edit_generation_time_output]) + with gr.Tab('🖼️ Image Understanding'): + with gr.Row(): + with gr.Column(scale=1): + img_input = gr.Image(label='Input Image', value= + load_example_image(os.path.join(os.path.dirname( + __file__), '../bagel2/test_images/meme.jpg'))) + understand_prompt = gr.Textbox(label='Prompt', value= + "Can someone explain what's funny about this meme??") + with gr.Column(scale=1): + txt_output = gr.Textbox(label='Result', lines=20) + with gr.Row(): + understand_show_thinking = gr.Checkbox(label='Thinking', value= + False) + with gr.Accordion('Inference Hyperparameters', open=False): + with gr.Row(): + understand_do_sample = gr.Checkbox(label='Sampling', value= + False, info='Enable sampling for text generation') + understand_text_temperature = gr.Slider(minimum=0.0, + maximum=1.0, value=0.3, step=0.05, interactive=True, + label='Temperature', info= + 'Controls randomness in text generation (0=deterministic, 1=creative)' + ) + understand_max_new_tokens = gr.Slider(minimum=64, maximum= + 4096, value=512, step=64, interactive=True, label= + 'Max New Tokens', info= + 'Maximum length of generated text, including potential thinking' + ) + img_understand_btn = gr.Button('Submit', variant='primary') + understand_generation_time_output = gr.Textbox(label= + 'Processing Time', interactive=False) + def process_understanding(image, prompt, show_thinking, do_sample, + text_temperature, max_new_tokens): + result, duration_str = image_understanding(image, prompt, + show_thinking, do_sample, text_temperature, max_new_tokens) + return result, duration_str + gr.on(triggers=[img_understand_btn.click, understand_prompt.submit], + fn=process_understanding, inputs=[img_input, understand_prompt, + understand_show_thinking, understand_do_sample, + understand_text_temperature, understand_max_new_tokens], + outputs=[txt_output, understand_generation_time_output]) + with gr.Tab('📊 System Monitor'): + with gr.Column(): + memory_stats_display = gr.Markdown('Check RAM/VRAM Stats') + refresh_button = gr.Button('🔄 Check RAM/VRAM Stats') + refresh_button.click(fn=get_all_memory_stats_for_gradio_display, + inputs=None, outputs=[memory_stats_display]) +block.launch() \ No newline at end of file diff --git a/gguf_connector/c.py b/gguf_connector/c.py new file mode 100644 index 0000000..be0eb27 --- /dev/null +++ b/gguf_connector/c.py @@ -0,0 +1,51 @@ +import os +gguf_files = [file for file in os.listdir() if file.endswith('.gguf')] +if gguf_files: + print('GGUF file(s) available. Select which one to use:') + for index, file_name in enumerate(gguf_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(gguf_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = gguf_files[choice_index] + print(f'Model file: {selected_file} is selected!') + ModelPath = selected_file + from ctransformers import AutoModelForCausalLM + llm = AutoModelForCausalLM.from_pretrained(ModelPath) + from tkinter import * + import tkinter.scrolledtext as st + root = Tk() + root.title('chatPIG') + root.columnconfigure([0, 1, 2], minsize=150) + root.rowconfigure(0, weight=2) + root.rowconfigure(1, weight=1) + icon = PhotoImage(file=os.path.join(os.path.dirname(__file__), + 'logo.png')) + root.iconphoto(False, icon) + i = Entry() + o = st.ScrolledText() + def submit(i): + root.title('Processing...') + from rich.console import Console + console = Console() + console.print( + '*note: [green]it might show: (Not Responding) and/or keep spinning; but running in background still; please be patient.' + ) + from rich.progress import Progress + with Progress(transient=True) as progress: + task = progress.add_task('Processing', total=None) + answer = llm(str(i.get())) + print(str(i.get()), answer) + o.insert(INSERT, str(i.get()) + answer + '\n\n') + i.delete(0, END) + root.title('chatPIG') + btn = Button(text='Submit', command=lambda : submit(i)) + i.grid(row=1, columnspan=2, sticky='nsew') + btn.grid(row=1, column=2, sticky='nsew') + o.grid(row=0, columnspan=3, sticky='nsew') + root.mainloop() + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No GGUF files are available in the current directory.') + input('--- Press ENTER To Exit ---') \ No newline at end of file diff --git a/gguf_connector/cg.py b/gguf_connector/cg.py new file mode 100644 index 0000000..b38f8e8 --- /dev/null +++ b/gguf_connector/cg.py @@ -0,0 +1,61 @@ +import os +def wav_handler(): + wav_files = [file for file in os.listdir() if file.endswith('.wav')] + if wav_files: + print('WAV file(s) available. Select which one to use:') + for index, file_name in enumerate(wav_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(wav_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = wav_files[choice_index] + print(f'WAV file: {selected_file} is selected!') + import speech_recognition as sr + r = sr.Recognizer() + with sr.AudioFile(selected_file) as source: + audio = r.record(source) + output = r.recognize_google(audio) + try: + from rich.console import Console + console = Console() + console.print(f'Content recognized as: [green]' + output) + input( + '\n---Enter to prompt the WAV content recognized above into GGUF model---' + ) + print('Processing...') + ans = llm(output) + print(output + ans) + except sr.UnknownValueError: + print('Could not understand audio') + except sr.RequestError as e: + print('Error; {0}'.format(e)) + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') + else: + print('No WAV files are available in the current directory.') + input('--- Press ENTER To Skip ---') +gguf_files = [file for file in os.listdir() if file.endswith('.gguf')] +if gguf_files: + print('GGUF file(s) available. Select which one to use:') + for index, file_name in enumerate(gguf_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(gguf_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = gguf_files[choice_index] + print(f'Model file: {selected_file} is selected!') + ModelPath = selected_file + print('Processing...') + from ctransformers import AutoModelForCausalLM + llm = AutoModelForCausalLM.from_pretrained(ModelPath) + while True: + ask = input('---Enter to select a WAV file (Q for quit)---') + if ask == 'q' or ask == 'Q': + break + wav_handler() + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No GGUF files are available in the current directory.') + input('--- Press ENTER To Exit ---') +print('Goodbye!') \ No newline at end of file diff --git a/gguf_connector/const.py b/gguf_connector/const.py new file mode 100644 index 0000000..5f689c6 --- /dev/null +++ b/gguf_connector/const.py @@ -0,0 +1,1137 @@ +from __future__ import annotations +from enum import Enum, IntEnum, auto +from typing import Any +GGUF_MAGIC = 1179993927 +GGUF_VERSION = 3 +GGUF_DEFAULT_ALIGNMENT = 32 +GGML_QUANT_VERSION = 2 +class Keys: + class General: + TYPE = 'general.type' + ARCHITECTURE = 'general.architecture' + QUANTIZATION_VERSION = 'general.quantization_version' + ALIGNMENT = 'general.alignment' + FILE_TYPE = 'general.file_type' + NAME = 'general.name' + AUTHOR = 'general.author' + VERSION = 'general.version' + ORGANIZATION = 'general.organization' + FINETUNE = 'general.finetune' + BASENAME = 'general.basename' + DESCRIPTION = 'general.description' + QUANTIZED_BY = 'general.quantized_by' + SIZE_LABEL = 'general.size_label' + LICENSE = 'general.license' + LICENSE_NAME = 'general.license.name' + LICENSE_LINK = 'general.license.link' + URL = 'general.url' + DOI = 'general.doi' + UUID = 'general.uuid' + REPO_URL = 'general.repo_url' + SOURCE_URL = 'general.source.url' + SOURCE_DOI = 'general.source.doi' + SOURCE_UUID = 'general.source.uuid' + SOURCE_REPO_URL = 'general.source.repo_url' + BASE_MODEL_COUNT = 'general.base_model.count' + BASE_MODEL_NAME = 'general.base_model.{id}.name' + BASE_MODEL_AUTHOR = 'general.base_model.{id}.author' + BASE_MODEL_VERSION = 'general.base_model.{id}.version' + BASE_MODEL_ORGANIZATION = 'general.base_model.{id}.organization' + BASE_MODEL_DESCRIPTION = 'general.base_model.{id}.description' + BASE_MODEL_URL = 'general.base_model.{id}.url' + BASE_MODEL_DOI = 'general.base_model.{id}.doi' + BASE_MODEL_UUID = 'general.base_model.{id}.uuid' + BASE_MODEL_REPO_URL = 'general.base_model.{id}.repo_url' + DATASET_COUNT = 'general.dataset.count' + DATASET_NAME = 'general.dataset.{id}.name' + DATASET_AUTHOR = 'general.dataset.{id}.author' + DATASET_VERSION = 'general.dataset.{id}.version' + DATASET_ORGANIZATION = 'general.dataset.{id}.organization' + DATASET_DESCRIPTION = 'general.dataset.{id}.description' + DATASET_URL = 'general.dataset.{id}.url' + DATASET_DOI = 'general.dataset.{id}.doi' + DATASET_UUID = 'general.dataset.{id}.uuid' + DATASET_REPO_URL = 'general.dataset.{id}.repo_url' + TAGS = 'general.tags' + LANGUAGES = 'general.languages' + class LLM: + VOCAB_SIZE = '{arch}.vocab_size' + CONTEXT_LENGTH = '{arch}.context_length' + EMBEDDING_LENGTH = '{arch}.embedding_length' + FEATURES_LENGTH = '{arch}.features_length' + BLOCK_COUNT = '{arch}.block_count' + LEADING_DENSE_BLOCK_COUNT = '{arch}.leading_dense_block_count' + FEED_FORWARD_LENGTH = '{arch}.feed_forward_length' + EXPERT_FEED_FORWARD_LENGTH = '{arch}.expert_feed_forward_length' + EXPERT_SHARED_FEED_FORWARD_LENGTH = ( + '{arch}.expert_shared_feed_forward_length') + USE_PARALLEL_RESIDUAL = '{arch}.use_parallel_residual' + TENSOR_DATA_LAYOUT = '{arch}.tensor_data_layout' + EXPERT_COUNT = '{arch}.expert_count' + EXPERT_USED_COUNT = '{arch}.expert_used_count' + EXPERT_SHARED_COUNT = '{arch}.expert_shared_count' + EXPERT_WEIGHTS_SCALE = '{arch}.expert_weights_scale' + EXPERT_WEIGHTS_NORM = '{arch}.expert_weights_norm' + EXPERT_GATING_FUNC = '{arch}.expert_gating_func' + POOLING_TYPE = '{arch}.pooling_type' + LOGIT_SCALE = '{arch}.logit_scale' + DECODER_START_TOKEN_ID = '{arch}.decoder_start_token_id' + ATTN_LOGIT_SOFTCAPPING = '{arch}.attn_logit_softcapping' + FINAL_LOGIT_SOFTCAPPING = '{arch}.final_logit_softcapping' + SWIN_NORM = '{arch}.swin_norm' + RESCALE_EVERY_N_LAYERS = '{arch}.rescale_every_n_layers' + TIME_MIX_EXTRA_DIM = '{arch}.time_mix_extra_dim' + TIME_DECAY_EXTRA_DIM = '{arch}.time_decay_extra_dim' + RESIDUAL_SCALE = '{arch}.residual_scale' + EMBEDDING_SCALE = '{arch}.embedding_scale' + TOKEN_SHIFT_COUNT = '{arch}.token_shift_count' + INTERLEAVE_MOE_LAYER_STEP = '{arch}.interleave_moe_layer_step' + class Attention: + HEAD_COUNT = '{arch}.attention.head_count' + HEAD_COUNT_KV = '{arch}.attention.head_count_kv' + MAX_ALIBI_BIAS = '{arch}.attention.max_alibi_bias' + CLAMP_KQV = '{arch}.attention.clamp_kqv' + KEY_LENGTH = '{arch}.attention.key_length' + VALUE_LENGTH = '{arch}.attention.value_length' + LAYERNORM_EPS = '{arch}.attention.layer_norm_epsilon' + LAYERNORM_RMS_EPS = '{arch}.attention.layer_norm_rms_epsilon' + GROUPNORM_EPS = '{arch}.attention.group_norm_epsilon' + GROUPNORM_GROUPS = '{arch}.attention.group_norm_groups' + CAUSAL = '{arch}.attention.causal' + Q_LORA_RANK = '{arch}.attention.q_lora_rank' + KV_LORA_RANK = '{arch}.attention.kv_lora_rank' + DECAY_LORA_RANK = '{arch}.attention.decay_lora_rank' + ICLR_LORA_RANK = '{arch}.attention.iclr_lora_rank' + VALUE_RESIDUAL_MIX_LORA_RANK = ( + '{arch}.attention.value_residual_mix_lora_rank') + GATE_LORA_RANK = '{arch}.attention.gate_lora_rank' + REL_BUCKETS_COUNT = '{arch}.attention.relative_buckets_count' + SLIDING_WINDOW = '{arch}.attention.sliding_window' + SCALE = '{arch}.attention.scale' + KEY_LENGTH_MLA = '{arch}.attention.key_length_mla' + VALUE_LENGTH_MLA = '{arch}.attention.value_length_mla' + class Rope: + DIMENSION_COUNT = '{arch}.rope.dimension_count' + DIMENSION_SECTIONS = '{arch}.rope.dimension_sections' + FREQ_BASE = '{arch}.rope.freq_base' + SCALING_TYPE = '{arch}.rope.scaling.type' + SCALING_FACTOR = '{arch}.rope.scaling.factor' + SCALING_ATTN_FACTOR = '{arch}.rope.scaling.attn_factor' + SCALING_ORIG_CTX_LEN = '{arch}.rope.scaling.original_context_length' + SCALING_FINETUNED = '{arch}.rope.scaling.finetuned' + SCALING_YARN_LOG_MUL = '{arch}.rope.scaling.yarn_log_multiplier' + class Split: + LLM_KV_SPLIT_NO = 'split.no' + LLM_KV_SPLIT_COUNT = 'split.count' + LLM_KV_SPLIT_TENSORS_COUNT = 'split.tensors.count' + class SSM: + CONV_KERNEL = '{arch}.ssm.conv_kernel' + INNER_SIZE = '{arch}.ssm.inner_size' + STATE_SIZE = '{arch}.ssm.state_size' + TIME_STEP_RANK = '{arch}.ssm.time_step_rank' + DT_B_C_RMS = '{arch}.ssm.dt_b_c_rms' + class WKV: + HEAD_SIZE = '{arch}.wkv.head_size' + class PosNet: + EMBEDDING_LENGTH = '{arch}.posnet.embedding_length' + BLOCK_COUNT = '{arch}.posnet.block_count' + class ConvNext: + EMBEDDING_LENGTH = '{arch}.convnext.embedding_length' + BLOCK_COUNT = '{arch}.convnext.block_count' + class Tokenizer: + MODEL = 'tokenizer.ggml.model' + PRE = 'tokenizer.ggml.pre' + LIST = 'tokenizer.ggml.tokens' + TOKEN_TYPE = 'tokenizer.ggml.token_type' + TOKEN_TYPE_COUNT = 'tokenizer.ggml.token_type_count' + SCORES = 'tokenizer.ggml.scores' + MERGES = 'tokenizer.ggml.merges' + BOS_ID = 'tokenizer.ggml.bos_token_id' + EOS_ID = 'tokenizer.ggml.eos_token_id' + EOT_ID = 'tokenizer.ggml.eot_token_id' + EOM_ID = 'tokenizer.ggml.eom_token_id' + UNK_ID = 'tokenizer.ggml.unknown_token_id' + SEP_ID = 'tokenizer.ggml.seperator_token_id' + PAD_ID = 'tokenizer.ggml.padding_token_id' + MASK_ID = 'tokenizer.ggml.mask_token_id' + ADD_BOS = 'tokenizer.ggml.add_bos_token' + ADD_EOS = 'tokenizer.ggml.add_eos_token' + ADD_PREFIX = 'tokenizer.ggml.add_space_prefix' + REMOVE_EXTRA_WS = 'tokenizer.ggml.remove_extra_whitespaces' + PRECOMPILED_CHARSMAP = 'tokenizer.ggml.precompiled_charsmap' + HF_JSON = 'tokenizer.huggingface.json' + RWKV = 'tokenizer.rwkv.world' + CHAT_TEMPLATE = 'tokenizer.chat_template' + CHAT_TEMPLATE_N = 'tokenizer.chat_template.{name}' + CHAT_TEMPLATES = 'tokenizer.chat_templates' + FIM_PRE_ID = 'tokenizer.ggml.fim_pre_token_id' + FIM_SUF_ID = 'tokenizer.ggml.fim_suf_token_id' + FIM_MID_ID = 'tokenizer.ggml.fim_mid_token_id' + FIM_PAD_ID = 'tokenizer.ggml.fim_pad_token_id' + FIM_REP_ID = 'tokenizer.ggml.fim_rep_token_id' + FIM_SEP_ID = 'tokenizer.ggml.fim_sep_token_id' + PREFIX_ID = 'tokenizer.ggml.prefix_token_id' + SUFFIX_ID = 'tokenizer.ggml.suffix_token_id' + MIDDLE_ID = 'tokenizer.ggml.middle_token_id' + class Adapter: + TYPE = 'adapter.type' + LORA_ALPHA = 'adapter.lora.alpha' +class GGUFType: + MODEL = 'model' + ADAPTER = 'adapter' +class MODEL_ARCH(IntEnum): + LLAMA = auto() + LLAMA4 = auto() + DECI = auto() + FALCON = auto() + BAICHUAN = auto() + GROK = auto() + GPT2 = auto() + GPTJ = auto() + GPTNEOX = auto() + MPT = auto() + STARCODER = auto() + REFACT = auto() + BERT = auto() + NOMIC_BERT = auto() + JINA_BERT_V2 = auto() + BLOOM = auto() + STABLELM = auto() + QWEN = auto() + QWEN2 = auto() + QWEN2MOE = auto() + QWEN2VL = auto() + QWEN3 = auto() + QWEN3MOE = auto() + PHI2 = auto() + PHI3 = auto() + PHIMOE = auto() + PLAMO = auto() + CODESHELL = auto() + ORION = auto() + INTERNLM2 = auto() + MINICPM = auto() + MINICPM3 = auto() + GEMMA = auto() + GEMMA2 = auto() + GEMMA3 = auto() + STARCODER2 = auto() + RWKV6 = auto() + RWKV6QWEN2 = auto() + RWKV7 = auto() + ARWKV7 = auto() + MAMBA = auto() + XVERSE = auto() + COMMAND_R = auto() + COHERE2 = auto() + DBRX = auto() + OLMO = auto() + OLMO2 = auto() + OLMOE = auto() + OPENELM = auto() + ARCTIC = auto() + DEEPSEEK = auto() + DEEPSEEK2 = auto() + CHATGLM = auto() + GLM4 = auto() + BITNET = auto() + T5 = auto() + T5ENCODER = auto() + JAIS = auto() + NEMOTRON = auto() + EXAONE = auto() + GRANITE = auto() + GRANITE_MOE = auto() + CHAMELEON = auto() + WAVTOKENIZER_DEC = auto() + PLM = auto() + BAILINGMOE = auto() +class MODEL_TENSOR(IntEnum): + TOKEN_EMBD = auto() + TOKEN_EMBD_NORM = auto() + TOKEN_TYPES = auto() + POS_EMBD = auto() + OUTPUT = auto() + OUTPUT_NORM = auto() + ROPE_FREQS = auto() + ROPE_FACTORS_LONG = auto() + ROPE_FACTORS_SHORT = auto() + ATTN_Q = auto() + ATTN_K = auto() + ATTN_V = auto() + ATTN_QKV = auto() + ATTN_OUT = auto() + ATTN_NORM = auto() + ATTN_NORM_2 = auto() + ATTN_OUT_NORM = auto() + ATTN_POST_NORM = auto() + ATTN_ROT_EMBD = auto() + FFN_GATE_INP = auto() + FFN_GATE_INP_SHEXP = auto() + FFN_NORM = auto() + FFN_PRE_NORM = auto() + FFN_POST_NORM = auto() + FFN_GATE = auto() + FFN_DOWN = auto() + FFN_UP = auto() + FFN_ACT = auto() + FFN_NORM_EXP = auto() + FFN_GATE_EXP = auto() + FFN_DOWN_EXP = auto() + FFN_UP_EXP = auto() + FFN_GATE_SHEXP = auto() + FFN_DOWN_SHEXP = auto() + FFN_UP_SHEXP = auto() + FFN_EXP_PROBS_B = auto() + ATTN_Q_NORM = auto() + ATTN_K_NORM = auto() + LAYER_OUT_NORM = auto() + SSM_IN = auto() + SSM_CONV1D = auto() + SSM_X = auto() + SSM_DT = auto() + SSM_A = auto() + SSM_D = auto() + SSM_OUT = auto() + TIME_MIX_W0 = auto() + TIME_MIX_W1 = auto() + TIME_MIX_W2 = auto() + TIME_MIX_A0 = auto() + TIME_MIX_A1 = auto() + TIME_MIX_A2 = auto() + TIME_MIX_V0 = auto() + TIME_MIX_V1 = auto() + TIME_MIX_V2 = auto() + TIME_MIX_G1 = auto() + TIME_MIX_G2 = auto() + TIME_MIX_K_K = auto() + TIME_MIX_K_A = auto() + TIME_MIX_R_K = auto() + TIME_MIX_LERP_X = auto() + TIME_MIX_LERP_K = auto() + TIME_MIX_LERP_V = auto() + TIME_MIX_LERP_R = auto() + TIME_MIX_LERP_G = auto() + TIME_MIX_LERP_FUSED = auto() + TIME_MIX_LERP_W = auto() + TIME_MIX_FIRST = auto() + TIME_MIX_DECAY = auto() + TIME_MIX_DECAY_W1 = auto() + TIME_MIX_DECAY_W2 = auto() + TIME_MIX_KEY = auto() + TIME_MIX_VALUE = auto() + TIME_MIX_RECEPTANCE = auto() + TIME_MIX_GATE = auto() + TIME_MIX_LN = auto() + TIME_MIX_OUTPUT = auto() + CHANNEL_MIX_LERP_K = auto() + CHANNEL_MIX_LERP_R = auto() + CHANNEL_MIX_KEY = auto() + CHANNEL_MIX_RECEPTANCE = auto() + CHANNEL_MIX_VALUE = auto() + ATTN_Q_A = auto() + ATTN_Q_B = auto() + ATTN_KV_A_MQA = auto() + ATTN_KV_B = auto() + ATTN_K_B = auto() + ATTN_V_B = auto() + ATTN_Q_A_NORM = auto() + ATTN_KV_A_NORM = auto() + FFN_SUB_NORM = auto() + ATTN_SUB_NORM = auto() + DEC_ATTN_NORM = auto() + DEC_ATTN_Q = auto() + DEC_ATTN_K = auto() + DEC_ATTN_V = auto() + DEC_ATTN_OUT = auto() + DEC_ATTN_REL_B = auto() + DEC_CROSS_ATTN_NORM = auto() + DEC_CROSS_ATTN_Q = auto() + DEC_CROSS_ATTN_K = auto() + DEC_CROSS_ATTN_V = auto() + DEC_CROSS_ATTN_OUT = auto() + DEC_CROSS_ATTN_REL_B = auto() + DEC_FFN_NORM = auto() + DEC_FFN_GATE = auto() + DEC_FFN_DOWN = auto() + DEC_FFN_UP = auto() + DEC_OUTPUT_NORM = auto() + ENC_ATTN_NORM = auto() + ENC_ATTN_Q = auto() + ENC_ATTN_K = auto() + ENC_ATTN_V = auto() + ENC_ATTN_OUT = auto() + ENC_ATTN_REL_B = auto() + ENC_FFN_NORM = auto() + ENC_FFN_GATE = auto() + ENC_FFN_DOWN = auto() + ENC_FFN_UP = auto() + ENC_OUTPUT_NORM = auto() + CLS = auto() + CLS_OUT = auto() + CONV1D = auto() + CONVNEXT_DW = auto() + CONVNEXT_NORM = auto() + CONVNEXT_PW1 = auto() + CONVNEXT_PW2 = auto() + CONVNEXT_GAMMA = auto() + POSNET_CONV1 = auto() + POSNET_CONV2 = auto() + POSNET_NORM = auto() + POSNET_NORM1 = auto() + POSNET_NORM2 = auto() + POSNET_ATTN_NORM = auto() + POSNET_ATTN_Q = auto() + POSNET_ATTN_K = auto() + POSNET_ATTN_V = auto() + POSNET_ATTN_OUT = auto() +MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {MODEL_ARCH.LLAMA: 'llama', + MODEL_ARCH.LLAMA4: 'llama4', MODEL_ARCH.DECI: 'deci', MODEL_ARCH.FALCON: + 'falcon', MODEL_ARCH.BAICHUAN: 'baichuan', MODEL_ARCH.GROK: 'grok', + MODEL_ARCH.GPT2: 'gpt2', MODEL_ARCH.GPTJ: 'gptj', MODEL_ARCH.GPTNEOX: + 'gptneox', MODEL_ARCH.MPT: 'mpt', MODEL_ARCH.STARCODER: 'starcoder', + MODEL_ARCH.REFACT: 'refact', MODEL_ARCH.BERT: 'bert', MODEL_ARCH. + NOMIC_BERT: 'nomic-bert', MODEL_ARCH.JINA_BERT_V2: 'jina-bert-v2', + MODEL_ARCH.BLOOM: 'bloom', MODEL_ARCH.STABLELM: 'stablelm', MODEL_ARCH. + QWEN: 'qwen', MODEL_ARCH.QWEN2: 'qwen2', MODEL_ARCH.QWEN2MOE: + 'qwen2moe', MODEL_ARCH.QWEN2VL: 'qwen2vl', MODEL_ARCH.QWEN3: 'qwen3', + MODEL_ARCH.QWEN3MOE: 'qwen3moe', MODEL_ARCH.PHI2: 'phi2', MODEL_ARCH. + PHI3: 'phi3', MODEL_ARCH.PHIMOE: 'phimoe', MODEL_ARCH.PLAMO: 'plamo', + MODEL_ARCH.CODESHELL: 'codeshell', MODEL_ARCH.ORION: 'orion', + MODEL_ARCH.INTERNLM2: 'internlm2', MODEL_ARCH.MINICPM: 'minicpm', + MODEL_ARCH.MINICPM3: 'minicpm3', MODEL_ARCH.GEMMA: 'gemma', MODEL_ARCH. + GEMMA2: 'gemma2', MODEL_ARCH.GEMMA3: 'gemma3', MODEL_ARCH.STARCODER2: + 'starcoder2', MODEL_ARCH.RWKV6: 'rwkv6', MODEL_ARCH.RWKV6QWEN2: + 'rwkv6qwen2', MODEL_ARCH.RWKV7: 'rwkv7', MODEL_ARCH.ARWKV7: 'arwkv7', + MODEL_ARCH.MAMBA: 'mamba', MODEL_ARCH.XVERSE: 'xverse', MODEL_ARCH. + COMMAND_R: 'command-r', MODEL_ARCH.COHERE2: 'cohere2', MODEL_ARCH.DBRX: + 'dbrx', MODEL_ARCH.OLMO: 'olmo', MODEL_ARCH.OLMO2: 'olmo2', MODEL_ARCH. + OLMOE: 'olmoe', MODEL_ARCH.OPENELM: 'openelm', MODEL_ARCH.ARCTIC: + 'arctic', MODEL_ARCH.DEEPSEEK: 'deepseek', MODEL_ARCH.DEEPSEEK2: + 'deepseek2', MODEL_ARCH.CHATGLM: 'chatglm', MODEL_ARCH.GLM4: 'glm4', + MODEL_ARCH.BITNET: 'bitnet', MODEL_ARCH.T5: 't5', MODEL_ARCH.T5ENCODER: + 't5encoder', MODEL_ARCH.JAIS: 'jais', MODEL_ARCH.NEMOTRON: 'nemotron', + MODEL_ARCH.EXAONE: 'exaone', MODEL_ARCH.GRANITE: 'granite', MODEL_ARCH. + GRANITE_MOE: 'granitemoe', MODEL_ARCH.CHAMELEON: 'chameleon', + MODEL_ARCH.WAVTOKENIZER_DEC: 'wavtokenizer-dec', MODEL_ARCH.PLM: 'plm', + MODEL_ARCH.BAILINGMOE: 'bailingmoe'} +TENSOR_NAMES: dict[MODEL_TENSOR, str] = {MODEL_TENSOR.TOKEN_EMBD: + 'token_embd', MODEL_TENSOR.TOKEN_EMBD_NORM: 'token_embd_norm', + MODEL_TENSOR.TOKEN_TYPES: 'token_types', MODEL_TENSOR.POS_EMBD: + 'position_embd', MODEL_TENSOR.OUTPUT_NORM: 'output_norm', MODEL_TENSOR. + OUTPUT: 'output', MODEL_TENSOR.ROPE_FREQS: 'rope_freqs', MODEL_TENSOR. + ROPE_FACTORS_LONG: 'rope_factors_long', MODEL_TENSOR.ROPE_FACTORS_SHORT: + 'rope_factors_short', MODEL_TENSOR.ATTN_NORM: 'blk.{bid}.attn_norm', + MODEL_TENSOR.ATTN_NORM_2: 'blk.{bid}.attn_norm_2', MODEL_TENSOR. + ATTN_QKV: 'blk.{bid}.attn_qkv', MODEL_TENSOR.ATTN_Q: 'blk.{bid}.attn_q', + MODEL_TENSOR.ATTN_K: 'blk.{bid}.attn_k', MODEL_TENSOR.ATTN_V: + 'blk.{bid}.attn_v', MODEL_TENSOR.ATTN_OUT: 'blk.{bid}.attn_output', + MODEL_TENSOR.ATTN_ROT_EMBD: 'blk.{bid}.attn_rot_embd', MODEL_TENSOR. + ATTN_Q_NORM: 'blk.{bid}.attn_q_norm', MODEL_TENSOR.ATTN_K_NORM: + 'blk.{bid}.attn_k_norm', MODEL_TENSOR.ATTN_OUT_NORM: + 'blk.{bid}.attn_output_norm', MODEL_TENSOR.ATTN_POST_NORM: + 'blk.{bid}.post_attention_norm', MODEL_TENSOR.FFN_GATE_INP: + 'blk.{bid}.ffn_gate_inp', MODEL_TENSOR.FFN_GATE_INP_SHEXP: + 'blk.{bid}.ffn_gate_inp_shexp', MODEL_TENSOR.FFN_NORM: + 'blk.{bid}.ffn_norm', MODEL_TENSOR.FFN_PRE_NORM: 'blk.{bid}.ffn_norm', + MODEL_TENSOR.FFN_POST_NORM: 'blk.{bid}.post_ffw_norm', MODEL_TENSOR. + FFN_GATE: 'blk.{bid}.ffn_gate', MODEL_TENSOR.FFN_DOWN: + 'blk.{bid}.ffn_down', MODEL_TENSOR.FFN_UP: 'blk.{bid}.ffn_up', + MODEL_TENSOR.FFN_GATE_SHEXP: 'blk.{bid}.ffn_gate_shexp', MODEL_TENSOR. + FFN_DOWN_SHEXP: 'blk.{bid}.ffn_down_shexp', MODEL_TENSOR.FFN_UP_SHEXP: + 'blk.{bid}.ffn_up_shexp', MODEL_TENSOR.FFN_ACT: 'blk.{bid}.ffn', + MODEL_TENSOR.FFN_NORM_EXP: 'blk.{bid}.ffn_norm_exps', MODEL_TENSOR. + FFN_GATE_EXP: 'blk.{bid}.ffn_gate_exps', MODEL_TENSOR.FFN_DOWN_EXP: + 'blk.{bid}.ffn_down_exps', MODEL_TENSOR.FFN_UP_EXP: + 'blk.{bid}.ffn_up_exps', MODEL_TENSOR.FFN_EXP_PROBS_B: + 'blk.{bid}.exp_probs_b', MODEL_TENSOR.LAYER_OUT_NORM: + 'blk.{bid}.layer_output_norm', MODEL_TENSOR.SSM_IN: 'blk.{bid}.ssm_in', + MODEL_TENSOR.SSM_CONV1D: 'blk.{bid}.ssm_conv1d', MODEL_TENSOR.SSM_X: + 'blk.{bid}.ssm_x', MODEL_TENSOR.SSM_DT: 'blk.{bid}.ssm_dt', + MODEL_TENSOR.SSM_A: 'blk.{bid}.ssm_a', MODEL_TENSOR.SSM_D: + 'blk.{bid}.ssm_d', MODEL_TENSOR.SSM_OUT: 'blk.{bid}.ssm_out', + MODEL_TENSOR.TIME_MIX_W0: 'blk.{bid}.time_mix_w0', MODEL_TENSOR. + TIME_MIX_W1: 'blk.{bid}.time_mix_w1', MODEL_TENSOR.TIME_MIX_W2: + 'blk.{bid}.time_mix_w2', MODEL_TENSOR.TIME_MIX_A0: + 'blk.{bid}.time_mix_a0', MODEL_TENSOR.TIME_MIX_A1: + 'blk.{bid}.time_mix_a1', MODEL_TENSOR.TIME_MIX_A2: + 'blk.{bid}.time_mix_a2', MODEL_TENSOR.TIME_MIX_V0: + 'blk.{bid}.time_mix_v0', MODEL_TENSOR.TIME_MIX_V1: + 'blk.{bid}.time_mix_v1', MODEL_TENSOR.TIME_MIX_V2: + 'blk.{bid}.time_mix_v2', MODEL_TENSOR.TIME_MIX_G1: + 'blk.{bid}.time_mix_g1', MODEL_TENSOR.TIME_MIX_G2: + 'blk.{bid}.time_mix_g2', MODEL_TENSOR.TIME_MIX_K_K: + 'blk.{bid}.time_mix_k_k', MODEL_TENSOR.TIME_MIX_K_A: + 'blk.{bid}.time_mix_k_a', MODEL_TENSOR.TIME_MIX_R_K: + 'blk.{bid}.time_mix_r_k', MODEL_TENSOR.TIME_MIX_LERP_X: + 'blk.{bid}.time_mix_lerp_x', MODEL_TENSOR.TIME_MIX_LERP_K: + 'blk.{bid}.time_mix_lerp_k', MODEL_TENSOR.TIME_MIX_LERP_V: + 'blk.{bid}.time_mix_lerp_v', MODEL_TENSOR.TIME_MIX_LERP_R: + 'blk.{bid}.time_mix_lerp_r', MODEL_TENSOR.TIME_MIX_LERP_G: + 'blk.{bid}.time_mix_lerp_g', MODEL_TENSOR.TIME_MIX_LERP_FUSED: + 'blk.{bid}.time_mix_lerp_fused', MODEL_TENSOR.TIME_MIX_LERP_W: + 'blk.{bid}.time_mix_lerp_w', MODEL_TENSOR.TIME_MIX_FIRST: + 'blk.{bid}.time_mix_first', MODEL_TENSOR.TIME_MIX_DECAY: + 'blk.{bid}.time_mix_decay', MODEL_TENSOR.TIME_MIX_DECAY_W1: + 'blk.{bid}.time_mix_decay_w1', MODEL_TENSOR.TIME_MIX_DECAY_W2: + 'blk.{bid}.time_mix_decay_w2', MODEL_TENSOR.TIME_MIX_KEY: + 'blk.{bid}.time_mix_key', MODEL_TENSOR.TIME_MIX_VALUE: + 'blk.{bid}.time_mix_value', MODEL_TENSOR.TIME_MIX_RECEPTANCE: + 'blk.{bid}.time_mix_receptance', MODEL_TENSOR.TIME_MIX_GATE: + 'blk.{bid}.time_mix_gate', MODEL_TENSOR.TIME_MIX_LN: + 'blk.{bid}.time_mix_ln', MODEL_TENSOR.TIME_MIX_OUTPUT: + 'blk.{bid}.time_mix_output', MODEL_TENSOR.CHANNEL_MIX_LERP_K: + 'blk.{bid}.channel_mix_lerp_k', MODEL_TENSOR.CHANNEL_MIX_LERP_R: + 'blk.{bid}.channel_mix_lerp_r', MODEL_TENSOR.CHANNEL_MIX_KEY: + 'blk.{bid}.channel_mix_key', MODEL_TENSOR.CHANNEL_MIX_RECEPTANCE: + 'blk.{bid}.channel_mix_receptance', MODEL_TENSOR.CHANNEL_MIX_VALUE: + 'blk.{bid}.channel_mix_value', MODEL_TENSOR.ATTN_Q_A: + 'blk.{bid}.attn_q_a', MODEL_TENSOR.ATTN_Q_B: 'blk.{bid}.attn_q_b', + MODEL_TENSOR.ATTN_KV_A_MQA: 'blk.{bid}.attn_kv_a_mqa', MODEL_TENSOR. + ATTN_KV_B: 'blk.{bid}.attn_kv_b', MODEL_TENSOR.ATTN_K_B: + 'blk.{bid}.attn_k_b', MODEL_TENSOR.ATTN_V_B: 'blk.{bid}.attn_v_b', + MODEL_TENSOR.ATTN_Q_A_NORM: 'blk.{bid}.attn_q_a_norm', MODEL_TENSOR. + ATTN_KV_A_NORM: 'blk.{bid}.attn_kv_a_norm', MODEL_TENSOR.ATTN_SUB_NORM: + 'blk.{bid}.attn_sub_norm', MODEL_TENSOR.FFN_SUB_NORM: + 'blk.{bid}.ffn_sub_norm', MODEL_TENSOR.DEC_ATTN_NORM: + 'dec.blk.{bid}.attn_norm', MODEL_TENSOR.DEC_ATTN_Q: + 'dec.blk.{bid}.attn_q', MODEL_TENSOR.DEC_ATTN_K: 'dec.blk.{bid}.attn_k', + MODEL_TENSOR.DEC_ATTN_V: 'dec.blk.{bid}.attn_v', MODEL_TENSOR. + DEC_ATTN_OUT: 'dec.blk.{bid}.attn_o', MODEL_TENSOR.DEC_ATTN_REL_B: + 'dec.blk.{bid}.attn_rel_b', MODEL_TENSOR.DEC_CROSS_ATTN_NORM: + 'dec.blk.{bid}.cross_attn_norm', MODEL_TENSOR.DEC_CROSS_ATTN_Q: + 'dec.blk.{bid}.cross_attn_q', MODEL_TENSOR.DEC_CROSS_ATTN_K: + 'dec.blk.{bid}.cross_attn_k', MODEL_TENSOR.DEC_CROSS_ATTN_V: + 'dec.blk.{bid}.cross_attn_v', MODEL_TENSOR.DEC_CROSS_ATTN_OUT: + 'dec.blk.{bid}.cross_attn_o', MODEL_TENSOR.DEC_CROSS_ATTN_REL_B: + 'dec.blk.{bid}.cross_attn_rel_b', MODEL_TENSOR.DEC_FFN_NORM: + 'dec.blk.{bid}.ffn_norm', MODEL_TENSOR.DEC_FFN_GATE: + 'dec.blk.{bid}.ffn_gate', MODEL_TENSOR.DEC_FFN_DOWN: + 'dec.blk.{bid}.ffn_down', MODEL_TENSOR.DEC_FFN_UP: + 'dec.blk.{bid}.ffn_up', MODEL_TENSOR.DEC_OUTPUT_NORM: 'dec.output_norm', + MODEL_TENSOR.ENC_ATTN_NORM: 'enc.blk.{bid}.attn_norm', MODEL_TENSOR. + ENC_ATTN_Q: 'enc.blk.{bid}.attn_q', MODEL_TENSOR.ENC_ATTN_K: + 'enc.blk.{bid}.attn_k', MODEL_TENSOR.ENC_ATTN_V: 'enc.blk.{bid}.attn_v', + MODEL_TENSOR.ENC_ATTN_OUT: 'enc.blk.{bid}.attn_o', MODEL_TENSOR. + ENC_ATTN_REL_B: 'enc.blk.{bid}.attn_rel_b', MODEL_TENSOR.ENC_FFN_NORM: + 'enc.blk.{bid}.ffn_norm', MODEL_TENSOR.ENC_FFN_GATE: + 'enc.blk.{bid}.ffn_gate', MODEL_TENSOR.ENC_FFN_DOWN: + 'enc.blk.{bid}.ffn_down', MODEL_TENSOR.ENC_FFN_UP: + 'enc.blk.{bid}.ffn_up', MODEL_TENSOR.ENC_OUTPUT_NORM: 'enc.output_norm', + MODEL_TENSOR.CLS: 'cls', MODEL_TENSOR.CLS_OUT: 'cls.output', + MODEL_TENSOR.CONV1D: 'conv1d', MODEL_TENSOR.CONVNEXT_DW: + 'convnext.{bid}.dw', MODEL_TENSOR.CONVNEXT_NORM: 'convnext.{bid}.norm', + MODEL_TENSOR.CONVNEXT_PW1: 'convnext.{bid}.pw1', MODEL_TENSOR. + CONVNEXT_PW2: 'convnext.{bid}.pw2', MODEL_TENSOR.CONVNEXT_GAMMA: + 'convnext.{bid}.gamma', MODEL_TENSOR.POSNET_CONV1: 'posnet.{bid}.conv1', + MODEL_TENSOR.POSNET_CONV2: 'posnet.{bid}.conv2', MODEL_TENSOR. + POSNET_NORM: 'posnet.{bid}.norm', MODEL_TENSOR.POSNET_NORM1: + 'posnet.{bid}.norm1', MODEL_TENSOR.POSNET_NORM2: 'posnet.{bid}.norm2', + MODEL_TENSOR.POSNET_ATTN_NORM: 'posnet.{bid}.attn_norm', MODEL_TENSOR. + POSNET_ATTN_Q: 'posnet.{bid}.attn_q', MODEL_TENSOR.POSNET_ATTN_K: + 'posnet.{bid}.attn_k', MODEL_TENSOR.POSNET_ATTN_V: + 'posnet.{bid}.attn_v', MODEL_TENSOR.POSNET_ATTN_OUT: + 'posnet.{bid}.attn_output'} +MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {MODEL_ARCH.LLAMA: [ + MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_ROT_EMBD, MODEL_TENSOR.FFN_GATE_INP, MODEL_TENSOR. + FFN_NORM, MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR. + FFN_UP, MODEL_TENSOR.FFN_GATE_EXP, MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP], MODEL_ARCH.LLAMA4: [MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, + MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.ATTN_ROT_EMBD, + MODEL_TENSOR.FFN_GATE_INP, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, MODEL_TENSOR.FFN_UP_EXP, MODEL_TENSOR. + FFN_GATE_SHEXP, MODEL_TENSOR.FFN_DOWN_SHEXP, MODEL_TENSOR.FFN_UP_SHEXP], + MODEL_ARCH.DECI: [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.ATTN_ROT_EMBD, MODEL_TENSOR. + FFN_GATE_INP, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, MODEL_TENSOR.FFN_UP_EXP], MODEL_ARCH.GROK: [ + MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_ROT_EMBD, MODEL_TENSOR.ATTN_OUT_NORM, MODEL_TENSOR. + FFN_GATE_INP, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, MODEL_TENSOR.FFN_UP_EXP, MODEL_TENSOR. + LAYER_OUT_NORM], MODEL_ARCH.GPTNEOX: [MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP], MODEL_ARCH.FALCON: [ + MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_NORM_2, MODEL_TENSOR.ATTN_QKV, + MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP], + MODEL_ARCH.BAICHUAN: [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.ATTN_ROT_EMBD, MODEL_TENSOR. + FFN_NORM, MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR. + FFN_UP], MODEL_ARCH.STARCODER: [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR. + POS_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR. + ATTN_NORM, MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR. + FFN_NORM, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP], MODEL_ARCH.BERT: + [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.TOKEN_EMBD_NORM, MODEL_TENSOR. + TOKEN_TYPES, MODEL_TENSOR.POS_EMBD, MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.ATTN_OUT_NORM, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, MODEL_TENSOR.LAYER_OUT_NORM, MODEL_TENSOR.CLS, + MODEL_TENSOR.CLS_OUT], MODEL_ARCH.NOMIC_BERT: [MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.TOKEN_EMBD_NORM, MODEL_TENSOR.TOKEN_TYPES, MODEL_TENSOR. + POS_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.ATTN_OUT_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, MODEL_TENSOR.LAYER_OUT_NORM + ], MODEL_ARCH.JINA_BERT_V2: [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR. + TOKEN_EMBD_NORM, MODEL_TENSOR.TOKEN_TYPES, MODEL_TENSOR.ATTN_NORM_2, + MODEL_TENSOR.ATTN_OUT_NORM, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR. + ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_K_NORM, + MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR. + LAYER_OUT_NORM, MODEL_TENSOR.CLS], MODEL_ARCH.MPT: [MODEL_TENSOR. + TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR + .ATTN_NORM, MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR. + FFN_NORM, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, MODEL_TENSOR. + FFN_ACT, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K_NORM, + MODEL_TENSOR.POS_EMBD], MODEL_ARCH.GPTJ: [MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP], + MODEL_ARCH.REFACT: [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP], MODEL_ARCH.BLOOM: [MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.TOKEN_EMBD_NORM, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR. + OUTPUT, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR. + ATTN_OUT, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR. + FFN_UP], MODEL_ARCH.STABLELM: [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR. + OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR + .ATTN_NORM, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR. + ATTN_V, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR. + FFN_GATE, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, MODEL_TENSOR. + ATTN_Q_NORM, MODEL_TENSOR.ATTN_K_NORM], MODEL_ARCH.QWEN: [MODEL_TENSOR. + TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR + .ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_QKV, + MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.ATTN_ROT_EMBD, MODEL_TENSOR. + FFN_NORM, MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR. + FFN_UP], MODEL_ARCH.QWEN2: [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR. + OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR + .ATTN_NORM, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR. + ATTN_V, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR. + FFN_GATE, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP], MODEL_ARCH. + QWEN2VL: [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP], MODEL_ARCH.QWEN2MOE: [MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_GATE_EXP, MODEL_TENSOR.FFN_DOWN_EXP, MODEL_TENSOR. + FFN_UP_EXP, MODEL_TENSOR.FFN_GATE_INP_SHEXP, MODEL_TENSOR. + FFN_GATE_SHEXP, MODEL_TENSOR.FFN_DOWN_SHEXP, MODEL_TENSOR.FFN_UP_SHEXP], + MODEL_ARCH.QWEN3: [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_K_NORM, MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP], MODEL_ARCH.QWEN3MOE: [MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_K_NORM, MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_GATE_INP, MODEL_TENSOR. + FFN_GATE_EXP, MODEL_TENSOR.FFN_DOWN_EXP, MODEL_TENSOR.FFN_UP_EXP], + MODEL_ARCH.PLAMO: [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.ATTN_ROT_EMBD, MODEL_TENSOR. + FFN_GATE, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP], MODEL_ARCH.GPT2: + [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.POS_EMBD, MODEL_TENSOR. + OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR. + ATTN_QKV, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR. + FFN_DOWN, MODEL_TENSOR.FFN_UP], MODEL_ARCH.PHI2: [MODEL_TENSOR. + TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR + .ATTN_NORM, MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR. + ATTN_K, MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR. + FFN_NORM, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP], MODEL_ARCH.PHI3: + [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ROPE_FACTORS_LONG, MODEL_TENSOR.ROPE_FACTORS_SHORT, + MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP], + MODEL_ARCH.PHIMOE: [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FACTORS_LONG, MODEL_TENSOR. + ROPE_FACTORS_SHORT, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_QKV, + MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_GATE_EXP, MODEL_TENSOR.FFN_DOWN_EXP, MODEL_TENSOR. + FFN_UP_EXP], MODEL_ARCH.CODESHELL: [MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.POS_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_ROT_EMBD, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR. + FFN_DOWN, MODEL_TENSOR.FFN_UP], MODEL_ARCH.ORION: [MODEL_TENSOR. + TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR + .ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR. + ATTN_K, MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR. + ATTN_ROT_EMBD, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP], MODEL_ARCH.INTERNLM2: [ + MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.ATTN_ROT_EMBD, + MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP], MODEL_ARCH.MINICPM: [MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.ROPE_FREQS, + MODEL_TENSOR.ROPE_FACTORS_LONG, MODEL_TENSOR.ROPE_FACTORS_SHORT, + MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.ATTN_ROT_EMBD, + MODEL_TENSOR.FFN_GATE_INP, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, MODEL_TENSOR.FFN_UP_EXP], MODEL_ARCH. + MINICPM3: [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FACTORS_LONG, MODEL_TENSOR. + ROPE_FACTORS_SHORT, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_Q_A, + MODEL_TENSOR.ATTN_Q_B, MODEL_TENSOR.ATTN_KV_A_MQA, MODEL_TENSOR. + ATTN_KV_B, MODEL_TENSOR.ATTN_Q_A_NORM, MODEL_TENSOR.ATTN_KV_A_NORM, + MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP], MODEL_ARCH.GEMMA: [ + MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR. + ATTN_NORM, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR. + ATTN_V, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_GATE, MODEL_TENSOR. + FFN_DOWN, MODEL_TENSOR.FFN_UP, MODEL_TENSOR.FFN_NORM], MODEL_ARCH. + GEMMA2: [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR. + ATTN_POST_NORM, MODEL_TENSOR.FFN_PRE_NORM, MODEL_TENSOR.FFN_POST_NORM], + MODEL_ARCH.GEMMA3: [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_NORM, + MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_K_NORM, MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR. + ATTN_POST_NORM, MODEL_TENSOR.FFN_PRE_NORM, MODEL_TENSOR.FFN_POST_NORM], + MODEL_ARCH.STARCODER2: [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR. + OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR + .ATTN_NORM, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR. + ATTN_V, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.ATTN_ROT_EMBD, MODEL_TENSOR + .FFN_NORM, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP], MODEL_ARCH. + RWKV6: [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.TOKEN_EMBD_NORM, + MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_NORM_2, MODEL_TENSOR.TIME_MIX_W1, MODEL_TENSOR. + TIME_MIX_W2, MODEL_TENSOR.TIME_MIX_LERP_X, MODEL_TENSOR.TIME_MIX_LERP_K, + MODEL_TENSOR.TIME_MIX_LERP_V, MODEL_TENSOR.TIME_MIX_LERP_R, + MODEL_TENSOR.TIME_MIX_LERP_G, MODEL_TENSOR.TIME_MIX_LERP_W, + MODEL_TENSOR.TIME_MIX_LERP_FUSED, MODEL_TENSOR.TIME_MIX_FIRST, + MODEL_TENSOR.TIME_MIX_DECAY, MODEL_TENSOR.TIME_MIX_DECAY_W1, + MODEL_TENSOR.TIME_MIX_DECAY_W2, MODEL_TENSOR.TIME_MIX_KEY, MODEL_TENSOR + .TIME_MIX_VALUE, MODEL_TENSOR.TIME_MIX_RECEPTANCE, MODEL_TENSOR. + TIME_MIX_GATE, MODEL_TENSOR.TIME_MIX_LN, MODEL_TENSOR.TIME_MIX_OUTPUT, + MODEL_TENSOR.CHANNEL_MIX_LERP_K, MODEL_TENSOR.CHANNEL_MIX_LERP_R, + MODEL_TENSOR.CHANNEL_MIX_KEY, MODEL_TENSOR.CHANNEL_MIX_RECEPTANCE, + MODEL_TENSOR.CHANNEL_MIX_VALUE], MODEL_ARCH.RWKV6QWEN2: [MODEL_TENSOR. + TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR + .ATTN_NORM, MODEL_TENSOR.TIME_MIX_W1, MODEL_TENSOR.TIME_MIX_W2, + MODEL_TENSOR.TIME_MIX_LERP_X, MODEL_TENSOR.TIME_MIX_LERP_K, + MODEL_TENSOR.TIME_MIX_LERP_V, MODEL_TENSOR.TIME_MIX_LERP_R, + MODEL_TENSOR.TIME_MIX_LERP_G, MODEL_TENSOR.TIME_MIX_LERP_W, + MODEL_TENSOR.TIME_MIX_LERP_FUSED, MODEL_TENSOR.TIME_MIX_FIRST, + MODEL_TENSOR.TIME_MIX_DECAY, MODEL_TENSOR.TIME_MIX_DECAY_W1, + MODEL_TENSOR.TIME_MIX_DECAY_W2, MODEL_TENSOR.TIME_MIX_KEY, MODEL_TENSOR + .TIME_MIX_VALUE, MODEL_TENSOR.TIME_MIX_RECEPTANCE, MODEL_TENSOR. + TIME_MIX_GATE, MODEL_TENSOR.TIME_MIX_LN, MODEL_TENSOR.TIME_MIX_OUTPUT, + MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP], MODEL_ARCH.RWKV7: [MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.TOKEN_EMBD_NORM, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR. + OUTPUT, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_NORM_2, MODEL_TENSOR. + TIME_MIX_LERP_FUSED, MODEL_TENSOR.TIME_MIX_W0, MODEL_TENSOR.TIME_MIX_W1, + MODEL_TENSOR.TIME_MIX_W2, MODEL_TENSOR.TIME_MIX_A0, MODEL_TENSOR. + TIME_MIX_A1, MODEL_TENSOR.TIME_MIX_A2, MODEL_TENSOR.TIME_MIX_V0, + MODEL_TENSOR.TIME_MIX_V1, MODEL_TENSOR.TIME_MIX_V2, MODEL_TENSOR. + TIME_MIX_G1, MODEL_TENSOR.TIME_MIX_G2, MODEL_TENSOR.TIME_MIX_K_K, + MODEL_TENSOR.TIME_MIX_K_A, MODEL_TENSOR.TIME_MIX_R_K, MODEL_TENSOR. + TIME_MIX_KEY, MODEL_TENSOR.TIME_MIX_VALUE, MODEL_TENSOR. + TIME_MIX_RECEPTANCE, MODEL_TENSOR.TIME_MIX_LN, MODEL_TENSOR. + TIME_MIX_OUTPUT, MODEL_TENSOR.CHANNEL_MIX_LERP_K, MODEL_TENSOR. + CHANNEL_MIX_KEY, MODEL_TENSOR.CHANNEL_MIX_VALUE], MODEL_ARCH.ARWKV7: [ + MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.TOKEN_EMBD_NORM, MODEL_TENSOR. + OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR. + TIME_MIX_LERP_FUSED, MODEL_TENSOR.TIME_MIX_W0, MODEL_TENSOR.TIME_MIX_W1, + MODEL_TENSOR.TIME_MIX_W2, MODEL_TENSOR.TIME_MIX_A0, MODEL_TENSOR. + TIME_MIX_A1, MODEL_TENSOR.TIME_MIX_A2, MODEL_TENSOR.TIME_MIX_V0, + MODEL_TENSOR.TIME_MIX_V1, MODEL_TENSOR.TIME_MIX_V2, MODEL_TENSOR. + TIME_MIX_G1, MODEL_TENSOR.TIME_MIX_G2, MODEL_TENSOR.TIME_MIX_K_K, + MODEL_TENSOR.TIME_MIX_K_A, MODEL_TENSOR.TIME_MIX_R_K, MODEL_TENSOR. + TIME_MIX_KEY, MODEL_TENSOR.TIME_MIX_VALUE, MODEL_TENSOR. + TIME_MIX_RECEPTANCE, MODEL_TENSOR.TIME_MIX_LN, MODEL_TENSOR. + TIME_MIX_OUTPUT, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP], MODEL_ARCH.MAMBA: [ + MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.SSM_IN, MODEL_TENSOR.SSM_CONV1D, + MODEL_TENSOR.SSM_X, MODEL_TENSOR.SSM_DT, MODEL_TENSOR.SSM_A, + MODEL_TENSOR.SSM_D, MODEL_TENSOR.SSM_OUT], MODEL_ARCH.XVERSE: [ + MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_ROT_EMBD, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR. + FFN_GATE, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP], MODEL_ARCH. + COMMAND_R: [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, MODEL_TENSOR.ATTN_K_NORM, + MODEL_TENSOR.ATTN_Q_NORM], MODEL_ARCH.COHERE2: [MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP], + MODEL_ARCH.DBRX: [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_QKV, + MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.ATTN_OUT_NORM, MODEL_TENSOR. + FFN_GATE_INP, MODEL_TENSOR.FFN_GATE_EXP, MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP], MODEL_ARCH.OLMO: [MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP], MODEL_ARCH.OLMO2: [ + MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.ATTN_POST_NORM, MODEL_TENSOR. + ATTN_Q_NORM, MODEL_TENSOR.ATTN_K_NORM, MODEL_TENSOR.FFN_POST_NORM, + MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP], + MODEL_ARCH.OLMOE: [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K_NORM, MODEL_TENSOR. + FFN_NORM, MODEL_TENSOR.FFN_GATE_INP, MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_UP_EXP, MODEL_TENSOR.FFN_DOWN_EXP], MODEL_ARCH.OPENELM: + [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR. + ATTN_NORM, MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q_NORM, + MODEL_TENSOR.ATTN_K_NORM, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP], + MODEL_ARCH.ARCTIC: [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.ATTN_ROT_EMBD, MODEL_TENSOR. + FFN_GATE_INP, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, MODEL_TENSOR.FFN_NORM_EXP, + MODEL_TENSOR.FFN_GATE_EXP, MODEL_TENSOR.FFN_DOWN_EXP, MODEL_TENSOR. + FFN_UP_EXP], MODEL_ARCH.DEEPSEEK: [MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, + MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.ATTN_ROT_EMBD, + MODEL_TENSOR.FFN_GATE_INP, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, MODEL_TENSOR.FFN_UP_EXP, MODEL_TENSOR. + FFN_GATE_SHEXP, MODEL_TENSOR.FFN_DOWN_SHEXP, MODEL_TENSOR.FFN_UP_SHEXP], + MODEL_ARCH.DEEPSEEK2: [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR. + OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR + .ATTN_NORM, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_A, MODEL_TENSOR. + ATTN_Q_B, MODEL_TENSOR.ATTN_KV_A_MQA, MODEL_TENSOR.ATTN_KV_B, + MODEL_TENSOR.ATTN_K_B, MODEL_TENSOR.ATTN_V_B, MODEL_TENSOR. + ATTN_Q_A_NORM, MODEL_TENSOR.ATTN_KV_A_NORM, MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_ROT_EMBD, MODEL_TENSOR.FFN_GATE_INP, MODEL_TENSOR. + FFN_NORM, MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR. + FFN_UP, MODEL_TENSOR.FFN_GATE_EXP, MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP, MODEL_TENSOR.FFN_GATE_SHEXP, MODEL_TENSOR. + FFN_DOWN_SHEXP, MODEL_TENSOR.FFN_UP_SHEXP, MODEL_TENSOR.FFN_EXP_PROBS_B + ], MODEL_ARCH.PLM: [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_KV_A_MQA, MODEL_TENSOR.ATTN_KV_A_NORM, MODEL_TENSOR. + ATTN_KV_B, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR. + FFN_UP, MODEL_TENSOR.FFN_DOWN], MODEL_ARCH.CHATGLM: [MODEL_TENSOR. + TOKEN_EMBD, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_QKV, + MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP], MODEL_ARCH.GLM4: [MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.ATTN_POST_NORM, MODEL_TENSOR.FFN_POST_NORM], MODEL_ARCH. + BITNET: [MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR. + ATTN_NORM, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR. + FFN_GATE, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, MODEL_TENSOR. + ATTN_SUB_NORM, MODEL_TENSOR.FFN_SUB_NORM], MODEL_ARCH.T5: [MODEL_TENSOR + .TOKEN_EMBD, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.DEC_ATTN_NORM, + MODEL_TENSOR.DEC_ATTN_Q, MODEL_TENSOR.DEC_ATTN_K, MODEL_TENSOR. + DEC_ATTN_V, MODEL_TENSOR.DEC_ATTN_OUT, MODEL_TENSOR.DEC_ATTN_REL_B, + MODEL_TENSOR.DEC_CROSS_ATTN_NORM, MODEL_TENSOR.DEC_CROSS_ATTN_Q, + MODEL_TENSOR.DEC_CROSS_ATTN_K, MODEL_TENSOR.DEC_CROSS_ATTN_V, + MODEL_TENSOR.DEC_CROSS_ATTN_OUT, MODEL_TENSOR.DEC_CROSS_ATTN_REL_B, + MODEL_TENSOR.DEC_FFN_NORM, MODEL_TENSOR.DEC_FFN_GATE, MODEL_TENSOR. + DEC_FFN_DOWN, MODEL_TENSOR.DEC_FFN_UP, MODEL_TENSOR.DEC_OUTPUT_NORM, + MODEL_TENSOR.ENC_ATTN_NORM, MODEL_TENSOR.ENC_ATTN_Q, MODEL_TENSOR. + ENC_ATTN_K, MODEL_TENSOR.ENC_ATTN_V, MODEL_TENSOR.ENC_ATTN_OUT, + MODEL_TENSOR.ENC_ATTN_REL_B, MODEL_TENSOR.ENC_FFN_NORM, MODEL_TENSOR. + ENC_FFN_GATE, MODEL_TENSOR.ENC_FFN_DOWN, MODEL_TENSOR.ENC_FFN_UP, + MODEL_TENSOR.ENC_OUTPUT_NORM], MODEL_ARCH.T5ENCODER: [MODEL_TENSOR. + TOKEN_EMBD, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ENC_ATTN_NORM, + MODEL_TENSOR.ENC_ATTN_Q, MODEL_TENSOR.ENC_ATTN_K, MODEL_TENSOR. + ENC_ATTN_V, MODEL_TENSOR.ENC_ATTN_OUT, MODEL_TENSOR.ENC_ATTN_REL_B, + MODEL_TENSOR.ENC_FFN_NORM, MODEL_TENSOR.ENC_FFN_GATE, MODEL_TENSOR. + ENC_FFN_DOWN, MODEL_TENSOR.ENC_FFN_UP, MODEL_TENSOR.ENC_OUTPUT_NORM], + MODEL_ARCH.JAIS: [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_QKV, + MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_UP], MODEL_ARCH.NEMOTRON: [ + MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_ROT_EMBD, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR. + FFN_DOWN, MODEL_TENSOR.FFN_UP], MODEL_ARCH.EXAONE: [MODEL_TENSOR. + TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR + .ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR. + ATTN_K, MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR. + ATTN_ROT_EMBD, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP], MODEL_ARCH.GRANITE: [ + MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP], + MODEL_ARCH.GRANITE_MOE: [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR. + OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR. + ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_GATE_INP, MODEL_TENSOR. + FFN_GATE_EXP, MODEL_TENSOR.FFN_DOWN_EXP, MODEL_TENSOR.FFN_UP_EXP], + MODEL_ARCH.CHAMELEON: [MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR. + OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR. + ATTN_Q, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, MODEL_TENSOR. + ATTN_K_NORM, MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR. + FFN_NORM, MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR. + FFN_UP], MODEL_ARCH.WAVTOKENIZER_DEC: [MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.TOKEN_EMBD_NORM, MODEL_TENSOR.CONV1D, MODEL_TENSOR. + CONVNEXT_DW, MODEL_TENSOR.CONVNEXT_NORM, MODEL_TENSOR.CONVNEXT_PW1, + MODEL_TENSOR.CONVNEXT_PW2, MODEL_TENSOR.CONVNEXT_GAMMA, MODEL_TENSOR. + OUTPUT, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.POSNET_CONV1, + MODEL_TENSOR.POSNET_CONV2, MODEL_TENSOR.POSNET_NORM, MODEL_TENSOR. + POSNET_NORM1, MODEL_TENSOR.POSNET_NORM2, MODEL_TENSOR.POSNET_ATTN_NORM, + MODEL_TENSOR.POSNET_ATTN_Q, MODEL_TENSOR.POSNET_ATTN_K, MODEL_TENSOR. + POSNET_ATTN_V, MODEL_TENSOR.POSNET_ATTN_OUT], MODEL_ARCH.BAILINGMOE: [ + MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_GATE_INP, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR. + FFN_GATE_EXP, MODEL_TENSOR.FFN_DOWN_EXP, MODEL_TENSOR.FFN_UP_EXP, + MODEL_TENSOR.FFN_GATE_SHEXP, MODEL_TENSOR.FFN_DOWN_SHEXP, MODEL_TENSOR. + FFN_UP_SHEXP]} +MODEL_TENSOR_SKIP: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {MODEL_ARCH.LLAMA: + [MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_ROT_EMBD], MODEL_ARCH.DECI: + [MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_ROT_EMBD], MODEL_ARCH. + BAICHUAN: [MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_ROT_EMBD], + MODEL_ARCH.QWEN: [MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_ROT_EMBD], + MODEL_ARCH.CODESHELL: [MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR. + ATTN_ROT_EMBD], MODEL_ARCH.ORION: [MODEL_TENSOR.ROPE_FREQS, + MODEL_TENSOR.ATTN_ROT_EMBD], MODEL_ARCH.STARCODER2: [MODEL_TENSOR. + ROPE_FREQS, MODEL_TENSOR.ATTN_ROT_EMBD], MODEL_ARCH.XVERSE: [ + MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_ROT_EMBD], MODEL_ARCH. + DEEPSEEK: [MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_ROT_EMBD], + MODEL_ARCH.DEEPSEEK2: [MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR. + ATTN_ROT_EMBD], MODEL_ARCH.CHATGLM: [MODEL_TENSOR.ROPE_FREQS], + MODEL_ARCH.NEMOTRON: [MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR. + ATTN_ROT_EMBD], MODEL_ARCH.BAILINGMOE: [MODEL_TENSOR.ROPE_FREQS]} +class TokenType(IntEnum): + NORMAL = 1 + UNKNOWN = 2 + CONTROL = 3 + USER_DEFINED = 4 + UNUSED = 5 + BYTE = 6 +class RopeScalingType(Enum): + NONE = 'none' + LINEAR = 'linear' + YARN = 'yarn' + LONGROPE = 'longrope' +class PoolingType(IntEnum): + NONE = 0 + MEAN = 1 + CLS = 2 +class GGMLQuantizationType(IntEnum): + F32 = 0 + F16 = 1 + Q4_0 = 2 + Q4_1 = 3 + Q5_0 = 6 + Q5_1 = 7 + Q8_0 = 8 + Q8_1 = 9 + Q2_K = 10 + Q3_K = 11 + Q4_K = 12 + Q5_K = 13 + Q6_K = 14 + Q8_K = 15 + IQ2_XXS = 16 + IQ2_XS = 17 + IQ3_XXS = 18 + IQ1_S = 19 + IQ4_NL = 20 + IQ3_S = 21 + IQ2_S = 22 + IQ4_XS = 23 + I8 = 24 + I16 = 25 + I32 = 26 + I64 = 27 + F64 = 28 + IQ1_M = 29 + BF16 = 30 + TQ1_0 = 34 + TQ2_0 = 35 +class ExpertGatingFuncType(IntEnum): + SOFTMAX = 1 + SIGMOID = 2 +class LlamaFileType(IntEnum): + ALL_F32 = 0 + MOSTLY_F16 = 1 + MOSTLY_Q4_0 = 2 + MOSTLY_Q4_1 = 3 + MOSTLY_Q4_1_SOME_F16 = 4 + MOSTLY_Q8_0 = 7 + MOSTLY_Q5_0 = 8 + MOSTLY_Q5_1 = 9 + MOSTLY_Q2_K = 10 + MOSTLY_Q3_K_S = 11 + MOSTLY_Q3_K_M = 12 + MOSTLY_Q3_K_L = 13 + MOSTLY_Q4_K_S = 14 + MOSTLY_Q4_K_M = 15 + MOSTLY_Q5_K_S = 16 + MOSTLY_Q5_K_M = 17 + MOSTLY_Q6_K = 18 + MOSTLY_IQ2_XXS = 19 + MOSTLY_IQ2_XS = 20 + MOSTLY_Q2_K_S = 21 + MOSTLY_IQ3_XS = 22 + MOSTLY_IQ3_XXS = 23 + MOSTLY_IQ1_S = 24 + MOSTLY_IQ4_NL = 25 + MOSTLY_IQ3_S = 26 + MOSTLY_IQ3_M = 27 + MOSTLY_IQ2_S = 28 + MOSTLY_IQ2_M = 29 + MOSTLY_IQ4_XS = 30 + MOSTLY_IQ1_M = 31 + MOSTLY_BF16 = 32 + MOSTLY_TQ1_0 = 36 + MOSTLY_TQ2_0 = 37 + GUESSED = 1024 +class GGUFEndian(IntEnum): + LITTLE = 0 + BIG = 1 +class GGUFValueType(IntEnum): + UINT8 = 0 + INT8 = 1 + UINT16 = 2 + INT16 = 3 + UINT32 = 4 + INT32 = 5 + FLOAT32 = 6 + BOOL = 7 + STRING = 8 + ARRAY = 9 + UINT64 = 10 + INT64 = 11 + FLOAT64 = 12 + @staticmethod + def get_type(val): + if isinstance(val, (str, bytes, bytearray)): + return GGUFValueType.STRING + elif isinstance(val, list): + return GGUFValueType.ARRAY + elif isinstance(val, float): + return GGUFValueType.FLOAT32 + elif isinstance(val, bool): + return GGUFValueType.BOOL + elif isinstance(val, int): + return GGUFValueType.INT32 + else: + raise ValueError(f'Unknown type: {type(val)}') +QK_K = 256 +GGML_QUANT_SIZES: dict[GGMLQuantizationType, tuple[int, int]] = { + GGMLQuantizationType.F32: (1, 4), GGMLQuantizationType.F16: (1, 2), + GGMLQuantizationType.Q4_0: (32, 2 + 16), GGMLQuantizationType.Q4_1: (32, + 2 + 2 + 16), GGMLQuantizationType.Q5_0: (32, 2 + 4 + 16), + GGMLQuantizationType.Q5_1: (32, 2 + 2 + 4 + 16), GGMLQuantizationType. + Q8_0: (32, 2 + 32), GGMLQuantizationType.Q8_1: (32, 4 + 4 + 32), + GGMLQuantizationType.Q2_K: (256, 2 + 2 + QK_K // 16 + QK_K // 4), + GGMLQuantizationType.Q3_K: (256, 2 + QK_K // 4 + QK_K // 8 + 12), + GGMLQuantizationType.Q4_K: (256, 2 + 2 + QK_K // 2 + 12), + GGMLQuantizationType.Q5_K: (256, 2 + 2 + QK_K // 2 + QK_K // 8 + 12), + GGMLQuantizationType.Q6_K: (256, 2 + QK_K // 2 + QK_K // 4 + QK_K // 16 + ), GGMLQuantizationType.Q8_K: (256, 4 + QK_K + QK_K // 8), + GGMLQuantizationType.IQ2_XXS: (256, 2 + QK_K // 4), + GGMLQuantizationType.IQ2_XS: (256, 2 + QK_K // 4 + QK_K // 32), + GGMLQuantizationType.IQ3_XXS: (256, 2 + QK_K // 4 + QK_K // 8), + GGMLQuantizationType.IQ1_S: (256, 2 + QK_K // 8 + QK_K // 16), + GGMLQuantizationType.IQ4_NL: (32, 2 + 16), GGMLQuantizationType.IQ3_S: + (256, 2 + QK_K // 4 + QK_K // 8 + QK_K // 32 + 4), GGMLQuantizationType + .IQ2_S: (256, 2 + QK_K // 4 + QK_K // 16), GGMLQuantizationType.IQ4_XS: + (256, 2 + 2 + QK_K // 2 + QK_K // 64), GGMLQuantizationType.I8: (1, 1), + GGMLQuantizationType.I16: (1, 2), GGMLQuantizationType.I32: (1, 4), + GGMLQuantizationType.I64: (1, 8), GGMLQuantizationType.F64: (1, 8), + GGMLQuantizationType.IQ1_M: (256, QK_K // 8 + QK_K // 16 + QK_K // 32), + GGMLQuantizationType.BF16: (1, 2), GGMLQuantizationType.TQ1_0: (256, 2 + + 4 * 13), GGMLQuantizationType.TQ2_0: (256, 2 + 64)} +KEY_GENERAL_ARCHITECTURE = Keys.General.ARCHITECTURE +KEY_GENERAL_QUANTIZATION_VERSION = Keys.General.QUANTIZATION_VERSION +KEY_GENERAL_ALIGNMENT = Keys.General.ALIGNMENT +KEY_GENERAL_NAME = Keys.General.NAME +KEY_GENERAL_AUTHOR = Keys.General.AUTHOR +KEY_GENERAL_URL = Keys.General.URL +KEY_GENERAL_DESCRIPTION = Keys.General.DESCRIPTION +KEY_GENERAL_LICENSE = Keys.General.LICENSE +KEY_GENERAL_SOURCE_URL = Keys.General.SOURCE_URL +KEY_GENERAL_FILE_TYPE = Keys.General.FILE_TYPE +KEY_VOCAB_SIZE = Keys.LLM.VOCAB_SIZE +KEY_CONTEXT_LENGTH = Keys.LLM.CONTEXT_LENGTH +KEY_EMBEDDING_LENGTH = Keys.LLM.EMBEDDING_LENGTH +KEY_BLOCK_COUNT = Keys.LLM.BLOCK_COUNT +KEY_FEED_FORWARD_LENGTH = Keys.LLM.FEED_FORWARD_LENGTH +KEY_USE_PARALLEL_RESIDUAL = Keys.LLM.USE_PARALLEL_RESIDUAL +KEY_TENSOR_DATA_LAYOUT = Keys.LLM.TENSOR_DATA_LAYOUT +KEY_ATTENTION_HEAD_COUNT = Keys.Attention.HEAD_COUNT +KEY_ATTENTION_HEAD_COUNT_KV = Keys.Attention.HEAD_COUNT_KV +KEY_ATTENTION_MAX_ALIBI_BIAS = Keys.Attention.MAX_ALIBI_BIAS +KEY_ATTENTION_CLAMP_KQV = Keys.Attention.CLAMP_KQV +KEY_ATTENTION_LAYERNORM_EPS = Keys.Attention.LAYERNORM_EPS +KEY_ATTENTION_LAYERNORM_RMS_EPS = Keys.Attention.LAYERNORM_RMS_EPS +KEY_ROPE_DIMENSION_COUNT = Keys.Rope.DIMENSION_COUNT +KEY_ROPE_FREQ_BASE = Keys.Rope.FREQ_BASE +KEY_ROPE_SCALING_TYPE = Keys.Rope.SCALING_TYPE +KEY_ROPE_SCALING_FACTOR = Keys.Rope.SCALING_FACTOR +KEY_ROPE_SCALING_ORIG_CTX_LEN = Keys.Rope.SCALING_ORIG_CTX_LEN +KEY_ROPE_SCALING_FINETUNED = Keys.Rope.SCALING_FINETUNED +KEY_SSM_CONV_KERNEL = Keys.SSM.CONV_KERNEL +KEY_SSM_INNER_SIZE = Keys.SSM.INNER_SIZE +KEY_SSM_STATE_SIZE = Keys.SSM.STATE_SIZE +KEY_SSM_TIME_STEP_RANK = Keys.SSM.TIME_STEP_RANK +KEY_SSM_DT_B_C_RMS = Keys.SSM.DT_B_C_RMS +KEY_TOKENIZER_MODEL = Keys.Tokenizer.MODEL +KEY_TOKENIZER_PRE = Keys.Tokenizer.PRE +KEY_TOKENIZER_LIST = Keys.Tokenizer.LIST +KEY_TOKENIZER_TOKEN_TYPE = Keys.Tokenizer.TOKEN_TYPE +KEY_TOKENIZER_SCORES = Keys.Tokenizer.SCORES +KEY_TOKENIZER_MERGES = Keys.Tokenizer.MERGES +KEY_TOKENIZER_BOS_ID = Keys.Tokenizer.BOS_ID +KEY_TOKENIZER_EOS_ID = Keys.Tokenizer.EOS_ID +KEY_TOKENIZER_EOT_ID = Keys.Tokenizer.EOT_ID +KEY_TOKENIZER_EOM_ID = Keys.Tokenizer.EOM_ID +KEY_TOKENIZER_UNK_ID = Keys.Tokenizer.UNK_ID +KEY_TOKENIZER_SEP_ID = Keys.Tokenizer.SEP_ID +KEY_TOKENIZER_PAD_ID = Keys.Tokenizer.PAD_ID +KEY_TOKENIZER_MASK_ID = Keys.Tokenizer.MASK_ID +KEY_TOKENIZER_HF_JSON = Keys.Tokenizer.HF_JSON +KEY_TOKENIZER_RWKV = Keys.Tokenizer.RWKV +KEY_TOKENIZER_FIM_PRE_ID = Keys.Tokenizer.FIM_PRE_ID +KEY_TOKENIZER_FIM_SUF_ID = Keys.Tokenizer.FIM_SUF_ID +KEY_TOKENIZER_FIM_MID_ID = Keys.Tokenizer.FIM_MID_ID +KEY_TOKENIZER_FIM_PAD_ID = Keys.Tokenizer.FIM_PAD_ID +KEY_TOKENIZER_FIM_REP_ID = Keys.Tokenizer.FIM_REP_ID +KEY_TOKENIZER_FIM_SEP_ID = Keys.Tokenizer.FIM_SEP_ID +KEY_TOKENIZER_PREFIX_ID = Keys.Tokenizer.PREFIX_ID +KEY_TOKENIZER_SUFFIX_ID = Keys.Tokenizer.SUFFIX_ID +KEY_TOKENIZER_MIDDLE_ID = Keys.Tokenizer.MIDDLE_ID \ No newline at end of file diff --git a/gguf_connector/cp.py b/gguf_connector/cp.py new file mode 100644 index 0000000..037731f --- /dev/null +++ b/gguf_connector/cp.py @@ -0,0 +1,64 @@ +import os +def pdf_handler(): + pdf_files = [file for file in os.listdir() if file.endswith('.pdf')] + def join_text(input_text): + joined_text = ' '.join(input_text.splitlines()) + return joined_text + if pdf_files: + print('PDF file(s) available. Select which one to use:') + for index, file_name in enumerate(pdf_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(pdf_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = pdf_files[choice_index] + print(f'PDF file: {selected_file} is selected!') + from pypdf import PdfReader + reader = PdfReader(selected_file) + text = '' + number_of_pages = len(reader.pages) + for i in range(number_of_pages): + page = reader.pages[i] + text += page.extract_text() + output_text = join_text(text) + inject = f'analyze the content below: ' + output_text + from rich.console import Console + console = Console() + console.print(f'\nPDF cotent extracted as below:\n\n[green]' + text + ) + input('---Enter to analyze the PDF content above---') + from rich.progress import Progress + with Progress(transient=True) as progress: + task = progress.add_task('Processing', total=None) + ans = llm(inject) + print(inject + '\n\nOutput:' + ans) + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') + else: + print('No PDF files are available in the current directory.') + input('--- Press ENTER To Skip ---') +gguf_files = [file for file in os.listdir() if file.endswith('.gguf')] +if gguf_files: + print('GGUF file(s) available. Select which one to use:') + for index, file_name in enumerate(gguf_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(gguf_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = gguf_files[choice_index] + print(f'Model file: {selected_file} is selected!') + ModelPath = selected_file + print('Processing...') + from ctransformers import AutoModelForCausalLM + llm = AutoModelForCausalLM.from_pretrained(ModelPath) + while True: + ask = input('---Enter to select a PDF file (Q for quit)---') + if ask.lower() == 'q': + break + pdf_handler() + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No GGUF files are available in the current directory.') + input('--- Press ENTER To Exit ---') +print('Goodbye!') \ No newline at end of file diff --git a/gguf_connector/cpp.py b/gguf_connector/cpp.py new file mode 100644 index 0000000..3878a12 --- /dev/null +++ b/gguf_connector/cpp.py @@ -0,0 +1,52 @@ +import os +gguf_files = [file for file in os.listdir() if file.endswith('.gguf')] +if gguf_files: + print('GGUF file(s) available. Select which one to use:') + for index, file_name in enumerate(gguf_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(gguf_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = gguf_files[choice_index] + print(f'Model file: {selected_file} is selected!') + ModelPath = selected_file + from llama_cpp import Llama + llm = Llama(model_path=ModelPath) + from tkinter import * + import tkinter.scrolledtext as st + root = Tk() + root.title('chatPIG') + root.columnconfigure([0, 1, 2], minsize=150) + root.rowconfigure(0, weight=2) + root.rowconfigure(1, weight=1) + icon = PhotoImage(file=os.path.join(os.path.dirname(__file__), + 'logo.png')) + root.iconphoto(False, icon) + i = Entry() + o = st.ScrolledText() + def submit(i): + root.title('Processing...') + from rich.console import Console + console = Console() + console.print( + '*note: [green]it might show: (Not Responding) and/or keep spinning; but running in background still; please be patient.' + ) + from rich.progress import Progress + with Progress(transient=True) as progress: + task = progress.add_task('Processing', total=None) + output = llm('Q: ' + str(i.get()), max_tokens=4096, echo=True) + answer = output['choices'][0]['text'] + print(answer) + o.insert(INSERT, answer + '\n\n') + i.delete(0, END) + root.title('chatPIG') + btn = Button(text='Submit', command=lambda : submit(i)) + i.grid(row=1, columnspan=2, sticky='nsew') + btn.grid(row=1, column=2, sticky='nsew') + o.grid(row=0, columnspan=3, sticky='nsew') + root.mainloop() + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No GGUF files are available in the current directory.') + input('--- Press ENTER To Exit ---') \ No newline at end of file diff --git a/gguf_connector/cs.py b/gguf_connector/cs.py new file mode 100644 index 0000000..f68d555 --- /dev/null +++ b/gguf_connector/cs.py @@ -0,0 +1,61 @@ +import os +def wav_handler(): + wav_files = [file for file in os.listdir() if file.endswith('.wav')] + if wav_files: + print('WAV file(s) available. Select which one to use:') + for index, file_name in enumerate(wav_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(wav_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = wav_files[choice_index] + print(f'WAV file: {selected_file} is selected!') + import speech_recognition as sr + r = sr.Recognizer() + with sr.AudioFile(selected_file) as source: + audio = r.record(source) + output = r.recognize_sphinx(audio) + try: + from rich.console import Console + console = Console() + console.print(f'Content recognized as: [green]' + output) + input( + '\n---Enter to prompt the WAV content recognized above into GGUF model---' + ) + print('Processing...') + ans = llm(output) + print(output + ans) + except sr.UnknownValueError: + print('Could not understand audio') + except sr.RequestError as e: + print('Error; {0}'.format(e)) + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') + else: + print('No WAV files are available in the current directory.') + input('--- Press ENTER To Skip ---') +gguf_files = [file for file in os.listdir() if file.endswith('.gguf')] +if gguf_files: + print('GGUF file(s) available. Select which one to use:') + for index, file_name in enumerate(gguf_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(gguf_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = gguf_files[choice_index] + print(f'Model file: {selected_file} is selected!') + ModelPath = selected_file + print('Processing...') + from ctransformers import AutoModelForCausalLM + llm = AutoModelForCausalLM.from_pretrained(ModelPath) + while True: + ask = input('---Enter to select a WAV file (Q for quit)---') + if ask == 'q' or ask == 'Q': + break + wav_handler() + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No GGUF files are available in the current directory.') + input('--- Press ENTER To Exit ---') +print('Goodbye!') \ No newline at end of file diff --git a/gguf_connector/d.py b/gguf_connector/d.py new file mode 100644 index 0000000..fbe2452 --- /dev/null +++ b/gguf_connector/d.py @@ -0,0 +1,79 @@ +import os, json +from safetensors import safe_open +from safetensors.torch import save_file +def list_safetensors(): + files = [f for f in os.listdir() if f.endswith('.safetensors')] + if not files: + print('No .safetensors files found in the current directory.') + return None + print('Available .safetensors files:') + for i, f in enumerate(files, 1): + print(f'{i}. {f}') + while True: + try: + choice = int(input('Select a file by number: ')) + return files[choice - 1] + except (ValueError, IndexError): + print('Invalid selection. Try again.') +def get_cutoff_size(): + while True: + try: + mb = float(input('Enter cutoff size for each part (in MB): ')) + return int(mb * 1024 * 1024) + except ValueError: + print('Please enter a valid number.') +def split_safetensors(file_name, cutoff_size): + with safe_open(file_name, framework='pt') as f: + metadata = f.metadata() + keys = f.keys() + tensors = {key: f.get_tensor(key) for key in keys} + total_size = 0 + tensor_sizes = {} + for k, v in tensors.items(): + size = v.element_size() * v.nelement() + tensor_sizes[k] = size + total_size += size + part_id = 1 + current_size = 0 + current_tensors = {} + weight_map = {} + num_parts = 0 + all_parts = [] + for key, tensor in tensors.items(): + tensor_size = tensor_sizes[key] + if current_size + tensor_size > cutoff_size and current_tensors: + part_name = f'model-{part_id:05d}-of-XXXX.safetensors' + save_file(current_tensors, part_name, metadata={}) + all_parts.append(part_name) + for k in current_tensors: + weight_map[k] = part_name + part_id += 1 + current_tensors = {} + current_size = 0 + current_tensors[key] = tensor + current_size += tensor_size + if current_tensors: + part_name = f'model-{part_id:05d}-of-XXXX.safetensors' + save_file(current_tensors, part_name, metadata={}) + all_parts.append(part_name) + for k in current_tensors: + weight_map[k] = part_name + num_parts = len(all_parts) + final_files = [] + for i, old_name in enumerate(all_parts): + new_name = f'model-{i + 1:05d}-of-{num_parts:05d}.safetensors' + os.rename(old_name, new_name) + for k in weight_map: + if weight_map[k] == old_name: + weight_map[k] = new_name + final_files.append(new_name) + index_json = {'metadata': {'total_size': total_size}, 'weight_map': + weight_map} + with open('model.safetensors.index.json', 'w') as f: + json.dump(index_json, f, indent=2) + print(f'\nSplit complete! {num_parts} parts created.') + print('Index file: model.safetensors.index.json') +file = list_safetensors() +if file: + cutoff = get_cutoff_size() + split_safetensors(file, cutoff) \ No newline at end of file diff --git a/gguf_connector/d2.py b/gguf_connector/d2.py new file mode 100644 index 0000000..bfdcbf6 --- /dev/null +++ b/gguf_connector/d2.py @@ -0,0 +1,61 @@ +import os, json +from tqdm import tqdm +def list_gguf_files(): + files = [f for f in os.listdir() if f.endswith('.gguf')] + if not files: + print('No .gguf files found in the current directory.') + return None + print('Available .gguf files:') + for i, f in enumerate(files, 1): + print(f'{i}. {f}') + while True: + try: + choice = int(input('Select a file by number: ')) + return files[choice - 1] + except (ValueError, IndexError): + print('Invalid selection. Try again.') +def get_cutoff_size(): + while True: + try: + mb = float(input('Enter cutoff size for each part (in MB): ')) + return int(mb * 1024 * 1024) + except ValueError: + print('Please enter a valid number.') +def split_gguf_file(filename, cutoff_size): + with open(filename, 'rb') as f: + data = f.read() + total_size = len(data) + parts = [] + offset = 0 + part_id = 1 + print('\nSplitting file into parts...') + with tqdm(total=total_size, unit='B', unit_scale=True, desc='Splitting' + ) as pbar: + while offset < total_size: + end = min(offset + cutoff_size, total_size) + part_data = data[offset:end] + part_name = f'model-{part_id:05d}-of-XXXX.gguf' + with open(part_name, 'wb') as pf: + pf.write(part_data) + parts.append(part_name) + offset = end + part_id += 1 + pbar.update(len(part_data)) + num_parts = len(parts) + final_files = [] + print('\nRenaming parts with total count info...') + for i, old_name in tqdm(list(enumerate(parts)), desc='Renaming', unit= + 'file'): + new_name = f'model-{i + 1:05d}-of-{num_parts:05d}.gguf' + os.rename(old_name, new_name) + final_files.append(new_name) + index = {'metadata': {'total_size': total_size}, 'file_map': { + f'part_{i + 1}': name for i, name in enumerate(final_files)}} + with open('model.gguf.index.json', 'w') as f: + json.dump(index, f, indent=2) + print(f'\nSplit complete! {num_parts} parts created.') + print('Index file: model.gguf.index.json') +file = list_gguf_files() +if file: + cutoff = get_cutoff_size() + split_gguf_file(file, cutoff) \ No newline at end of file diff --git a/gguf_connector/d4.py b/gguf_connector/d4.py new file mode 100644 index 0000000..b8f882e --- /dev/null +++ b/gguf_connector/d4.py @@ -0,0 +1,62 @@ +import torch +from .const import GGML_QUANT_VERSION, LlamaFileType +from .reader import GGUFReader, GGMLQuantizationType +from .writer import GGUFWriter +from .quant import quantize +from tqdm import tqdm +def get_arch_str(reader): + field = reader.get_field('general.architecture') + return str(field.parts[field.data[-1]], encoding='utf-8') +def get_file_type(reader): + field = reader.get_field('general.file_type') + ft = int(field.parts[field.data[-1]]) + return LlamaFileType(ft) +import os +gguf_files = [file for file in os.listdir() if file.endswith('.gguf')] +if gguf_files: + print('GGUF file(s) available. Select which one to fix:') + for index, file_name in enumerate(gguf_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(gguf_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = gguf_files[choice_index] + print(f'Model file: {selected_file} is selected!') + input_path = selected_file + output_path = f'fixed-{os.path.splitext(input_path)[0]}.gguf' + reader = GGUFReader(input_path) + arch = get_arch_str(reader) + file_type = get_file_type(reader) + print(f"Detected arch: '{arch}' (ftype: {str(file_type)})") + sd5d = torch.load(f'fix_5d_tensors_{arch}.pt', weights_only=False) + print('5D:', sd5d.keys()) + writer = GGUFWriter(path=None, arch=arch) + writer.add_quantization_version(GGML_QUANT_VERSION) + writer.add_file_type(file_type) + added = [] + def add_extra_key(writer, key, data): + data_qtype = GGMLQuantizationType.F32 + data_shape = data.shape + data = quantize(data, data_qtype) + tqdm.write(f'Adding key {key} ({data_shape})') + writer.add_tensor(key, data, raw_dtype=data_qtype) + global added + added.append(key) + for tensor in tqdm(reader.tensors): + writer.add_tensor(tensor.name, tensor.data, raw_dtype=tensor. + tensor_type) + key5d = tensor.name.replace('.bias', '.weight') + if key5d in sd5d.keys(): + add_extra_key(writer, key5d, sd5d[key5d]) + for key, data in sd5d.items(): + if key not in added: + add_extra_key(writer, key, data) + writer.write_header_to_file(path=output_path) + writer.write_kv_data_to_file() + writer.write_tensors_to_file(progress=True) + writer.close() + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No GGUF files are available in the current directory.') + input('--- Press ENTER To Exit ---') \ No newline at end of file diff --git a/gguf_connector/d5.py b/gguf_connector/d5.py new file mode 100644 index 0000000..f14efd9 --- /dev/null +++ b/gguf_connector/d5.py @@ -0,0 +1,63 @@ +import torch +from .const import GGML_QUANT_VERSION, LlamaFileType +from .reader import GGUFReader, GGMLQuantizationType +from .writer import GGUFWriter +from .quant import quantize +from tqdm import tqdm +def get_arch_str(reader): + field = reader.get_field('general.architecture') + return str(field.parts[field.data[-1]], encoding='utf-8') +def get_file_type(reader): + field = reader.get_field('general.file_type') + ft = int(field.parts[field.data[-1]]) + return LlamaFileType(ft) +import os +gguf_files = [file for file in os.listdir() if file.endswith('.gguf')] +if gguf_files: + print('GGUF file(s) available. Select which one to fix:') + for index, file_name in enumerate(gguf_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(gguf_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = gguf_files[choice_index] + print(f'Model file: {selected_file} is selected!') + input_path = selected_file + output_path = f'fixed-{os.path.splitext(input_path)[0]}.gguf' + reader = GGUFReader(input_path) + arch = get_arch_str(reader) + file_type = get_file_type(reader) + print(f"Detected arch: '{arch}' (ftype: {str(file_type)})") + sd5d = torch.load(f'fix_5d_tensors_{arch}.safetensors', + weights_only=False) + sd5d = {k: v.numpy() for k, v in sd5d.items()} + print('5D tensors:', sd5d.keys()) + writer = GGUFWriter(path=None, arch=arch) + writer.add_quantization_version(GGML_QUANT_VERSION) + writer.add_file_type(file_type) + added = [] + def add_extra_key(writer, key, data): + global added + data_qtype = GGMLQuantizationType.F32 + data = quantize(data, data_qtype) + tqdm.write(f'Adding key {key} ({data.shape})') + writer.add_tensor(key, data, raw_dtype=data_qtype) + added.append(key) + for tensor in tqdm(reader.tensors): + writer.add_tensor(tensor.name, tensor.data, raw_dtype=tensor. + tensor_type) + key5d = tensor.name.replace('.bias', '.weight') + if key5d in sd5d.keys(): + add_extra_key(writer, key5d, sd5d[key5d]) + for key, data in sd5d.items(): + if key not in added: + add_extra_key(writer, key, data) + writer.write_header_to_file(path=output_path) + writer.write_kv_data_to_file() + writer.write_tensors_to_file(progress=True) + writer.close() + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No GGUF files are available in the current directory.') + input('--- Press ENTER To Exit ---') \ No newline at end of file diff --git a/gguf_connector/d6.py b/gguf_connector/d6.py new file mode 100644 index 0000000..b36638d --- /dev/null +++ b/gguf_connector/d6.py @@ -0,0 +1,185 @@ +import torch +from .writer import GGUFWriter, GGMLQuantizationType +from .const import GGML_QUANT_VERSION, LlamaFileType +from safetensors.torch import load_file +from tqdm import tqdm +QUANTIZATION_THRESHOLD = 1024 +REARRANGE_THRESHOLD = 512 +MAX_TENSOR_NAME_LENGTH = 127 +MAX_TENSOR_DIMS = 4 +class ModelTemplate: + arch = 'invalid' + shape_fix = False + keys_detect = [] + keys_banned = [] + keys_hiprec = [] +class ModelFlux(ModelTemplate): + arch = 'flux' + keys_detect = [('transformer_blocks.0.attn.norm_added_k.weight',), ( + 'double_blocks.0.img_attn.proj.weight',)] + keys_banned = ['transformer_blocks.0.attn.norm_added_k.weight'] +class ModelSD3(ModelTemplate): + arch = 'sd3' + keys_detect = [('transformer_blocks.0.attn.add_q_proj.weight',), ( + 'joint_blocks.0.x_block.attn.qkv.weight',)] + keys_banned = ['transformer_blocks.0.attn.add_q_proj.weight'] +class ModelAura(ModelTemplate): + arch = 'aura' + keys_detect = [('double_layers.3.modX.1.weight',), ( + 'joint_transformer_blocks.3.ff_context.out_projection.weight',)] + keys_banned = [ + 'joint_transformer_blocks.3.ff_context.out_projection.weight'] +class ModelHiDream(ModelTemplate): + arch = 'hidream' + keys_detect = [('caption_projection.0.linear.weight', + 'double_stream_blocks.0.block.ff_i.shared_experts.w3.weight')] + keys_hiprec = ['.ff_i.gate.weightimg_emb.emb_pos'] +class ModelHyVid(ModelTemplate): + arch = 'hyvid' + keys_detect = [('double_blocks.0.img_attn_proj.weight', + 'txt_in.individual_token_refiner.blocks.1.self_attn_qkv.weight')] +class ModelWan(ModelHyVid): + arch = 'wan' + keys_detect = [('blocks.0.self_attn.norm_q.weight', + 'text_embedding.2.weight', 'head.modulation')] +class ModelLTXV(ModelTemplate): + arch = 'ltxv' + keys_detect = [('adaln_single.emb.timestep_embedder.linear_2.weight', + 'transformer_blocks.27.scale_shift_table', + 'caption_projection.linear_2.weight')] + keys_hiprec = ['scale_shift_table'] +class ModelSDXL(ModelTemplate): + arch = 'sdxl' + shape_fix = True + keys_detect = [('down_blocks.0.downsamplers.0.conv.weight', + 'add_embedding.linear_1.weight'), ('input_blocks.3.0.op.weight', + 'input_blocks.6.0.op.weight', 'output_blocks.2.2.conv.weight', + 'output_blocks.5.2.conv.weight'), ('label_emb.0.0.weight',)] +class ModelSD1(ModelTemplate): + arch = 'sd1' + shape_fix = True + keys_detect = [('down_blocks.0.downsamplers.0.conv.weight',), ( + 'input_blocks.3.0.op.weight', 'input_blocks.6.0.op.weight', + 'input_blocks.9.0.op.weight', 'output_blocks.2.1.conv.weight', + 'output_blocks.5.2.conv.weight', 'output_blocks.8.2.conv.weight')] +arch_list = [ModelFlux, ModelSD3, ModelAura, ModelHiDream, ModelLTXV, + ModelHyVid, ModelWan, ModelSDXL, ModelSD1] +def is_model_arch(model, state_dict): + matched = False + invalid = False + for match_list in model.keys_detect: + if all(key in state_dict for key in match_list): + matched = True + invalid = any(key in state_dict for key in model.keys_banned) + break + assert not invalid, 'Model architecture not allowed for conversion! (i.e. reference VS diffusers format)' + return matched +def detect_arch(state_dict): + model_arch = None + for arch in arch_list: + if is_model_arch(arch, state_dict): + model_arch = arch() + break + assert model_arch is not None, 'Unknown model architecture!' + return model_arch +def load_state_dict(path): + if any(path.endswith(x) for x in ['.ckpt', '.pt', '.bin', '.pth']): + state_dict = torch.load(path, map_location='cpu', weights_only=True) + for subkey in ['model', 'module']: + if subkey in state_dict: + state_dict = state_dict[subkey] + break + if len(state_dict) < 20: + raise RuntimeError(f'pt subkey load failed: {state_dict.keys()}') + else: + state_dict = load_file(path) + prefix = None + for pfx in ['model.diffusion_model.', 'model.']: + if any([x.startswith(pfx) for x in state_dict.keys()]): + prefix = pfx + break + sd = {} + for k, v in state_dict.items(): + if prefix and prefix not in k: + continue + if prefix: + k = k.replace(prefix, '') + sd[k] = v + return sd +def load_model(path): + state_dict = load_state_dict(path) + model_arch = detect_arch(state_dict) + print(f'* Architecture detected from input: {model_arch.arch}') + writer = GGUFWriter(path=None, arch=model_arch.arch) + return writer, state_dict, model_arch +def handle_tensors(args, writer, state_dict, model_arch): + name_lengths = tuple(sorted(((key, len(key)) for key in state_dict.keys + ()), key=lambda item: item[1], reverse=True)) + if not name_lengths: + return + max_name_len = name_lengths[0][1] + if max_name_len > MAX_TENSOR_NAME_LENGTH: + bad_list = ', '.join(f'{key!r} ({namelen})' for key, namelen in + name_lengths if namelen > MAX_TENSOR_NAME_LENGTH) + raise ValueError( + f'Can only handle tensor names up to {MAX_TENSOR_NAME_LENGTH} characters. Tensors exceeding the limit: {bad_list}' + ) + for key, data in tqdm(state_dict.items()): + old_dtype = data.dtype + data = data.to(torch.float32).numpy() + n_dims = len(data.shape) + data_shape = data.shape + data_qtype = getattr(GGMLQuantizationType, 'BF16' if old_dtype == + torch.bfloat16 else 'F16') + if len(data.shape) > MAX_TENSOR_DIMS: + continue + n_params = 1 + for dim_size in data_shape: + n_params *= dim_size + if old_dtype in (torch.float32, torch.bfloat16): + if n_dims == 1: + data_qtype = GGMLQuantizationType.F32 + elif n_params <= QUANTIZATION_THRESHOLD: + data_qtype = GGMLQuantizationType.F32 + elif any(x in key for x in model_arch.keys_hiprec): + data_qtype = GGMLQuantizationType.F32 + new_name = key + data_qtype = GGMLQuantizationType.F32 + shape_str = f"{{{', '.join(str(n) for n in reversed(data.shape))}}}" + tqdm.write( + f"{f'%-{max_name_len + 4}s' % f'{new_name}'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}" + ) + writer.add_tensor(new_name, data, raw_dtype=data_qtype) +import os +safetensors_files = [file for file in os.listdir() if file.endswith( + '.safetensors')] +if safetensors_files: + print('Safetensors file(s) available. Select which one to convert:') + for index, file_name in enumerate(safetensors_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = safetensors_files[choice_index] + print(f'Model file: {selected_file} is selected!') + path = selected_file + writer, state_dict, model_arch = load_model(path) + writer.add_quantization_version(GGML_QUANT_VERSION) + if next(iter(state_dict.values())).dtype == torch.bfloat16: + out_path = f'{os.path.splitext(path)[0]}-f32.gguf' + writer.add_file_type(LlamaFileType.MOSTLY_BF16) + else: + out_path = f'{os.path.splitext(path)[0]}-f32.gguf' + writer.add_file_type(LlamaFileType.MOSTLY_F16) + if os.path.isfile(out_path): + input('Output exists enter to continue or ctrl+c to abort!') + handle_tensors(path, writer, state_dict, model_arch) + writer.write_header_to_file(path=out_path) + writer.write_kv_data_to_file() + writer.write_tensors_to_file(progress=True) + writer.close() + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No safetensors files are available in the current directory.') + input('--- Press ENTER To Exit ---') \ No newline at end of file diff --git a/gguf_connector/d7.py b/gguf_connector/d7.py new file mode 100644 index 0000000..352cfd3 --- /dev/null +++ b/gguf_connector/d7.py @@ -0,0 +1,197 @@ +import torch +from .writer import GGUFWriter, GGMLQuantizationType +from .const import GGML_QUANT_VERSION, LlamaFileType +from safetensors.torch import load_file +from tqdm import tqdm +QUANTIZATION_THRESHOLD = 1024 +REARRANGE_THRESHOLD = 512 +MAX_TENSOR_NAME_LENGTH = 127 +MAX_TENSOR_DIMS = 4 +class ModelTemplate: + arch = 'invalid' + shape_fix = False + keys_detect = [] + keys_banned = [] + keys_hiprec = [] + def handle_nd_tensor(self, key, data): + raise NotImplementedError( + f'Tensor detected that exceeds dims supported by C++ code! ({key} @ {data.shape})' + ) +class ModelFlux(ModelTemplate): + arch = 'flux' + keys_detect = [('transformer_blocks.0.attn.norm_added_k.weight',), ( + 'double_blocks.0.img_attn.proj.weight',)] + keys_banned = ['transformer_blocks.0.attn.norm_added_k.weight'] +class ModelSD3(ModelTemplate): + arch = 'sd3' + keys_detect = [('transformer_blocks.0.attn.add_q_proj.weight',), ( + 'joint_blocks.0.x_block.attn.qkv.weight',)] + keys_banned = ['transformer_blocks.0.attn.add_q_proj.weight'] +class ModelAura(ModelTemplate): + arch = 'aura' + keys_detect = [('double_layers.3.modX.1.weight',), ( + 'joint_transformer_blocks.3.ff_context.out_projection.weight',)] + keys_banned = [ + 'joint_transformer_blocks.3.ff_context.out_projection.weight'] +class ModelHiDream(ModelTemplate): + arch = 'hidream' + keys_detect = [('caption_projection.0.linear.weight', + 'double_stream_blocks.0.block.ff_i.shared_experts.w3.weight')] + keys_hiprec = ['.ff_i.gate.weightimg_emb.emb_pos'] +class ModelHyVid(ModelTemplate): + arch = 'hyvid' + keys_detect = [('double_blocks.0.img_attn_proj.weight', + 'txt_in.individual_token_refiner.blocks.1.self_attn_qkv.weight')] + def handle_nd_tensor(self, key, data): + path = f'./fix_5d_tensors_{self.arch}.safetensors' + fsd = {key: torch.from_numpy(data)} + tqdm.write( + f'5D key found in state dict! Manual fix required! - {key} {data.shape}' + ) + torch.save(fsd, path) +class ModelWan(ModelHyVid): + arch = 'wan' + keys_detect = [('blocks.0.self_attn.norm_q.weight', + 'text_embedding.2.weight', 'head.modulation')] +class ModelLTXV(ModelTemplate): + arch = 'ltxv' + keys_detect = [('adaln_single.emb.timestep_embedder.linear_2.weight', + 'transformer_blocks.27.scale_shift_table', + 'caption_projection.linear_2.weight')] + keys_hiprec = ['scale_shift_table'] +class ModelSDXL(ModelTemplate): + arch = 'sdxl' + shape_fix = True + keys_detect = [('down_blocks.0.downsamplers.0.conv.weight', + 'add_embedding.linear_1.weight'), ('input_blocks.3.0.op.weight', + 'input_blocks.6.0.op.weight', 'output_blocks.2.2.conv.weight', + 'output_blocks.5.2.conv.weight'), ('label_emb.0.0.weight',)] +class ModelSD1(ModelTemplate): + arch = 'sd1' + shape_fix = True + keys_detect = [('down_blocks.0.downsamplers.0.conv.weight',), ( + 'input_blocks.3.0.op.weight', 'input_blocks.6.0.op.weight', + 'input_blocks.9.0.op.weight', 'output_blocks.2.1.conv.weight', + 'output_blocks.5.2.conv.weight', 'output_blocks.8.2.conv.weight')] +arch_list = [ModelFlux, ModelSD3, ModelAura, ModelHiDream, ModelLTXV, + ModelHyVid, ModelWan, ModelSDXL, ModelSD1] +def is_model_arch(model, state_dict): + matched = False + invalid = False + for match_list in model.keys_detect: + if all(key in state_dict for key in match_list): + matched = True + invalid = any(key in state_dict for key in model.keys_banned) + break + assert not invalid, 'Model architecture not allowed for conversion! (i.e. reference VS diffusers format)' + return matched +def detect_arch(state_dict): + model_arch = None + for arch in arch_list: + if is_model_arch(arch, state_dict): + model_arch = arch() + break + assert model_arch is not None, 'Unknown model architecture!' + return model_arch +def load_state_dict(path): + if any(path.endswith(x) for x in ['.ckpt', '.pt', '.bin', '.pth']): + state_dict = torch.load(path, map_location='cpu', weights_only=True) + for subkey in ['model', 'module']: + if subkey in state_dict: + state_dict = state_dict[subkey] + break + if len(state_dict) < 20: + raise RuntimeError(f'pt subkey load failed: {state_dict.keys()}') + else: + state_dict = load_file(path) + prefix = None + for pfx in ['model.diffusion_model.', 'model.']: + if any([x.startswith(pfx) for x in state_dict.keys()]): + prefix = pfx + break + sd = {} + for k, v in state_dict.items(): + if prefix and prefix not in k: + continue + if prefix: + k = k.replace(prefix, '') + sd[k] = v + return sd +def load_model(path): + state_dict = load_state_dict(path) + model_arch = detect_arch(state_dict) + print(f'* Architecture detected from input: {model_arch.arch}') + writer = GGUFWriter(path=None, arch=model_arch.arch) + return writer, state_dict, model_arch +def handle_tensors(args, writer, state_dict, model_arch): + name_lengths = tuple(sorted(((key, len(key)) for key in state_dict.keys + ()), key=lambda item: item[1], reverse=True)) + if not name_lengths: + return + max_name_len = name_lengths[0][1] + if max_name_len > MAX_TENSOR_NAME_LENGTH: + bad_list = ', '.join(f'{key!r} ({namelen})' for key, namelen in + name_lengths if namelen > MAX_TENSOR_NAME_LENGTH) + raise ValueError( + f'Can only handle tensor names up to {MAX_TENSOR_NAME_LENGTH} characters. Tensors exceeding the limit: {bad_list}' + ) + for key, data in tqdm(state_dict.items()): + old_dtype = data.dtype + data = data.to(torch.float32).numpy() + n_dims = len(data.shape) + data_shape = data.shape + data_qtype = getattr(GGMLQuantizationType, 'BF16' if old_dtype == + torch.bfloat16 else 'F16') + if len(data.shape) > MAX_TENSOR_DIMS: + model_arch.handle_nd_tensor(key, data) + continue + n_params = 1 + for dim_size in data_shape: + n_params *= dim_size + if old_dtype in (torch.float32, torch.bfloat16): + if n_dims == 1: + data_qtype = GGMLQuantizationType.F32 + elif n_params <= QUANTIZATION_THRESHOLD: + data_qtype = GGMLQuantizationType.F32 + elif any(x in key for x in model_arch.keys_hiprec): + data_qtype = GGMLQuantizationType.F32 + new_name = key + data_qtype = GGMLQuantizationType.F32 + shape_str = f"{{{', '.join(str(n) for n in reversed(data.shape))}}}" + tqdm.write( + f"{f'%-{max_name_len + 4}s' % f'{new_name}'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}" + ) + writer.add_tensor(new_name, data, raw_dtype=data_qtype) +import os +safetensors_files = [file for file in os.listdir() if file.endswith( + '.safetensors')] +if safetensors_files: + print('Safetensors file(s) available. Select which one to convert:') + for index, file_name in enumerate(safetensors_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = safetensors_files[choice_index] + print(f'Model file: {selected_file} is selected!') + path = selected_file + writer, state_dict, model_arch = load_model(path) + writer.add_quantization_version(GGML_QUANT_VERSION) + if next(iter(state_dict.values())).dtype == torch.bfloat16: + out_path = f'{os.path.splitext(path)[0]}-f32.gguf' + writer.add_file_type(LlamaFileType.MOSTLY_BF16) + else: + out_path = f'{os.path.splitext(path)[0]}-f32.gguf' + writer.add_file_type(LlamaFileType.MOSTLY_F16) + if os.path.isfile(out_path): + input('Output exists enter to continue or ctrl+c to abort!') + handle_tensors(path, writer, state_dict, model_arch) + writer.write_header_to_file(path=out_path) + writer.write_kv_data_to_file() + writer.write_tensors_to_file(progress=True) + writer.close() + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No safetensors files are available in the current directory.') + input('--- Press ENTER To Exit ---') \ No newline at end of file diff --git a/gguf_connector/data.json b/gguf_connector/data.json new file mode 100644 index 0000000..369f4e6 --- /dev/null +++ b/gguf_connector/data.json @@ -0,0 +1,186 @@ +[ + { + "name":" chat.gguf [1.9GB; for general purpose]", + "url":"https://github.com/calcuis/chatgpt-ai-model/releases/download/0.1/chat.gguf" + }, + { + "name":" code.gguf [3.3GB; for coding assistance]", + "url":"https://huggingface.co/calcuis/code_mini/resolve/main/code.gguf" + }, + { + "name":" medi.gguf [3.3GB; for medical/health advice]", + "url":"https://huggingface.co/calcuis/medi_mini/resolve/main/medi.gguf" + }, + { + "name":" law.gguf [3.3GB; for law/legal opinion]", + "url":"https://huggingface.co/calcuis/law_mini/resolve/main/law.gguf" + }, + { + "name":" finance.gguf [3.3GB; for wealth/financial plan]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/finance.gguf" + }, + { + "name":" coder.gguf [1.4GB; for simple coding issue]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/coder.gguf" + }, + { + "name":" phi2.gguf [1.4GB; for technical problem]", + "url":"https://huggingface.co/calcuis/phi2_mini/resolve/main/phi2.gguf" + }, + { + "name":" lam2.gguf [3.3GB; for long/detailed response]", + "url":"https://huggingface.co/calcuis/lam2_mini/resolve/main/lam2.gguf" + }, + { + "name":" tiny2.gguf [783MB; for fast/rapid response]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/tiny2.gguf" + }, + { + "name":"stable.gguf [1.7GB; for code review]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/stable.gguf" + }, + { + "name":"hermes.gguf [3.5GB; for general/basic consultation]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/hermes.gguf" + }, + { + "name":"samantha.gguf [2.9GB; for medical student]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/samantha.gguf" + }, + { + "name":"doublee.gguf [1.4GB; for electrical engineering]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/doublee.gguf" + }, + { + "name":"math.gguf [3.1GB; for math/calculation]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/math.gguf" + }, + { + "name":"math2.gguf [3.6GB; for math/calculation (2nd)]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/math2.gguf" + }, + { + "name":"python.gguf [3.3GB; for coding python]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/python.gguf" + }, + { + "name":"bloom.gguf [1.6GB; for quick response]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/bloom.gguf" + }, + { + "name":"yarn.gguf [3.1GB; for problem solving]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/yarn.gguf" + }, + { + "name":"dolphin.gguf [3.0GB; for solving problem]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/dolphin.gguf" + }, + { + "name":"karen.gguf [2.7GB; for drafting letter/paper]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/karen.gguf" + }, + { + "name":"beagle.gguf [3.5GB; for genearl response]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/beagle.gguf" + }, + { + "name":"garrulus.gguf [2.7GB; for specific response]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/garrulus.gguf" + }, + { + "name":"westlake.gguf [2.7GB; for common information]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/westlake.gguf" + }, + { + "name":"openchat.gguf [3.0GB; for most purpose]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/openchat.gguf" + }, + { + "name":"zephyr.gguf [3.0GB; for multilayer response]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/zephyr.gguf" + }, + { + "name":"mamba.gguf [2.1GB; for quick/fast enquiry]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/mamba.gguf" + }, + { + "name":"deep.gguf [3.1GB; for lengthy response]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/deep.gguf" + }, + { + "name":"orca.gguf [4.1GB; for multiple response]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/orca.gguf" + }, + { + "name":"claude.gguf [3.7GB; for comprehensive response]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/claude.gguf" + }, + { + "name":"mistral.gguf [3.1GB; for instructional enquiry]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/mistral.gguf" + }, + { + "name":"gpt.gguf [330MB; for general response]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/gpt.gguf" + }, + { + "name":"gpt2.gguf [1.6GB; for general response (2nd)]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/gpt2.gguf" + }, + { + "name":"gemma2.gguf [3.4GB; for specific response]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/gemma2.gguf" + }, + { + "name":"gemma3-1b-it [687MB; for great summary; q4_0]", + "url":"https://huggingface.co/gguf-org/gemma-3-1b-it-gguf/resolve/main/gemma-3-1b-it-q4_0.gguf" + }, + { + "name":"gemma3-1b-pt [1.0GB; for list response; q8_0]", + "url":"https://huggingface.co/gguf-org/gemma-3-1b-pt-gguf/resolve/main/gemma-3-1b-pt-q8_0.gguf" + }, + { + "name":"phi3.gguf [2.3GB; for technical problem (3rd)]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/phi3.gguf" + }, + { + "name":"phi3.5.gguf [2.0GB; for technical problem (3.5)]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/phi3.5.gguf" + }, + { + "name":"phi4.gguf [5.6GB; for technical problem (4th)]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/phi4.gguf" + }, + { + "name":"llama2.gguf [2.8GB; for comprehensive response]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/llama2.gguf" + }, + { + "name":"llama3.gguf [3.6GB; for comprehensive response (3rd)]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/llama3.gguf" + }, + { + "name":"llama3.2.gguf [1.7GB; for uncensored content]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/llama3.2.gguf" + }, + { + "name":"guanaco.gguf [2.8GB; for uncensored content]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/guanaco.gguf" + }, + { + "name":"wizard.gguf [3.3GB; for uncensored content]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/wizard.gguf" + }, + { + "name":"solar.gguf [4.5GB; for uncensored content]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/solar.gguf" + }, + { + "name":"maid.gguf [3.1GB; for adult/18+ nsfw]", + "url":"https://huggingface.co/calcuis/gguf/resolve/main/maid.gguf" + }, + { + "name":"tiny.gguf [551MB; for testing pdf analyzor]", + "url":"https://github.com/calcuis/chatgpt-ai-model/releases/download/0.2/tiny.gguf" + } +] \ No newline at end of file diff --git a/gguf_connector/e.py b/gguf_connector/e.py new file mode 100644 index 0000000..5315ba8 --- /dev/null +++ b/gguf_connector/e.py @@ -0,0 +1,48 @@ +from .const import GGML_QUANT_VERSION, LlamaFileType +from .reader import GGUFReader +from .writer import GGUFWriter +def get_arch_str(reader): + field = reader.get_field('general.architecture') + return str(field.parts[field.data[-1]], encoding='utf-8') +def get_file_type(reader): + field = reader.get_field('general.file_type') + ft = int(field.parts[field.data[-1]]) + return LlamaFileType(ft) +def extract_weight_tensors(input_path, output_path): + with open(input_path, 'rb') as f1: + reader1 = GGUFReader(f1) + arch = get_arch_str(reader1) + file_type = get_file_type(reader1) + print(f"Detected arch: '{arch}' (ftype: {str(file_type)})") + writer = GGUFWriter(path=None, arch=arch) + writer.add_quantization_version(GGML_QUANT_VERSION) + writer.add_file_type(file_type) + for tensor in reader1.tensors: + if tensor.name.endswith('.weight'): + writer.add_tensor(tensor.name, tensor.data, raw_dtype= + tensor.tensor_type) + with open(output_path, 'wb'): + writer.write_header_to_file(path=output_path) + writer.write_kv_data_to_file() + writer.write_tensors_to_file(progress=True) + writer.close() +import os +gguf_files = [file for file in os.listdir() if file.endswith('.gguf')] +if gguf_files: + print('GGUF file(s) available. Select which one to extract:') + for index, file_name in enumerate(gguf_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(gguf_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = gguf_files[choice_index] + print(f'Model file: {selected_file} is selected!') + input_path = selected_file + output_path = ( + f'{os.path.splitext(input_path)[0]}-weight-extracted.gguf') + extract_weight_tensors(input_path, output_path) + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No GGUF files are available in the current directory.') + input('--- Press ENTER To Exit ---') \ No newline at end of file diff --git a/gguf_connector/f.py b/gguf_connector/f.py new file mode 100644 index 0000000..84c560d --- /dev/null +++ b/gguf_connector/f.py @@ -0,0 +1,54 @@ +from .const import GGML_QUANT_VERSION, LlamaFileType +from .reader import GGUFReader +from .writer import GGUFWriter +from glob import glob +def get_arch_str(reader): + field = reader.get_field('general.architecture') + return str(field.parts[field.data[-1]], encoding='utf-8') +def get_file_type(reader): + field = reader.get_field('general.file_type') + ft = int(field.parts[field.data[-1]]) + return LlamaFileType(ft) +def merge_gguf_files(master_file, output_file): + with open(master_file, 'rb') as f_master: + reader_master = GGUFReader(f_master) + arch = get_arch_str(reader_master) + file_type = get_file_type(reader_master) + print(f'Using master file: {master_file}') + print(f"Detected arch: '{arch}' (ftype: {str(file_type)})") + writer = GGUFWriter(path=None, arch=arch) + writer.add_quantization_version(GGML_QUANT_VERSION) + writer.add_file_type(file_type) + for tensor in reader_master.tensors: + writer.add_tensor(tensor.name, tensor.data, raw_dtype=tensor. + tensor_type) + for file in glob('*.gguf'): + if file == master_file: + continue + print(f'Merging from: {file}') + with open(file, 'rb') as f: + reader = GGUFReader(f) + for tensor in reader.tensors: + writer.add_tensor(tensor.name, tensor.data, raw_dtype= + tensor.tensor_type) + with open(output_file, 'wb'): + writer.write_header_to_file(path=output_file) + writer.write_kv_data_to_file() + writer.write_tensors_to_file(progress=True) + writer.close() + print(f'Saved merged file as: {output_file}') +all_gguf_files = sorted(glob('*.gguf')) +if not all_gguf_files: + print('No .gguf files found in the current directory.') +else: + print('Available .gguf files:') + for idx, fname in enumerate(all_gguf_files): + print(f'{idx + 1}: {fname}') + selected = int(input('Select the master file by number: ')) - 1 + master_file = all_gguf_files[selected] + ask = input('Assign a name other than output.gguf (Y/n)? ') + if ask.lower() == 'y': + given = input('Enter a file name: ') + else: + given = 'output.gguf' + merge_gguf_files(master_file, given) \ No newline at end of file diff --git a/gguf_connector/g.py b/gguf_connector/g.py new file mode 100644 index 0000000..0ee1686 --- /dev/null +++ b/gguf_connector/g.py @@ -0,0 +1,29 @@ +import os +gguf_files = [file for file in os.listdir() if file.endswith('.gguf')] +if gguf_files: + print('GGUF file(s) available. Select which one to use:') + for index, file_name in enumerate(gguf_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(gguf_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = gguf_files[choice_index] + print(f'Model file: {selected_file} is selected!') + ModelPath = selected_file + from ctransformers import AutoModelForCausalLM + llm = AutoModelForCausalLM.from_pretrained(ModelPath) + while True: + ask = input('Enter a Question (Q for quit): ') + if ask.lower() == 'q': + break + from rich.progress import Progress + with Progress(transient=True) as progress: + task = progress.add_task('Processing', total=None) + ans = llm(ask) + print(ask + ans) + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No GGUF files are available in the current directory.') + input('--- Press ENTER To Exit ---') +print('Goodbye!') \ No newline at end of file diff --git a/gguf_connector/gpp.py b/gguf_connector/gpp.py new file mode 100644 index 0000000..12c9fcc --- /dev/null +++ b/gguf_connector/gpp.py @@ -0,0 +1,30 @@ +import os +gguf_files = [file for file in os.listdir() if file.endswith('.gguf')] +if gguf_files: + print('GGUF file(s) available. Select which one to use:') + for index, file_name in enumerate(gguf_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(gguf_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = gguf_files[choice_index] + print(f'Model file: {selected_file} is selected!') + ModelPath = selected_file + from llama_cpp import Llama + llm = Llama(model_path=ModelPath) + while True: + ask = input('Enter a Question (Q for quit): ') + if ask.lower() == 'q': + break + from rich.progress import Progress + with Progress(transient=True) as progress: + task = progress.add_task('Processing', total=None) + output = llm('Q: ' + ask, max_tokens=4096, echo=True) + answer = output['choices'][0]['text'] + print(answer + '\n') + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No GGUF files are available in the current directory.') + input('--- Press ENTER To Exit ---') +print('Goodbye!') \ No newline at end of file diff --git a/gguf_connector/i.py b/gguf_connector/i.py new file mode 100644 index 0000000..752507b --- /dev/null +++ b/gguf_connector/i.py @@ -0,0 +1,6 @@ +import webbrowser +print('activating the default browser...') +webbrowser.open('https://gguf.io') +print( + 'note: if you are encountering restricted access problem with this domain; visit https://gguf.us instead; same content' + ) \ No newline at end of file diff --git a/gguf_connector/i2.py b/gguf_connector/i2.py new file mode 100644 index 0000000..9c7b246 --- /dev/null +++ b/gguf_connector/i2.py @@ -0,0 +1,42 @@ +import torch +import gradio as gr +from transformers import T5EncoderModel +from diffusers import PixArtSigmaPipeline +text_encoder = T5EncoderModel.from_pretrained('calcuis/ltxv-gguf', + gguf_file='t5xxl_fp16-q4_0.gguf', torch_dtype=torch.float16) +pipe = PixArtSigmaPipeline.from_pretrained('callgg/pixart-decoder', + text_encoder=text_encoder, torch_dtype=torch.float16) +pipe.enable_model_cpu_offload() +def generate_image(prompt, negative_prompt, width, height, guidance_scale, + num_inference_steps): + image = pipe(prompt=prompt, negative_prompt=negative_prompt, width= + width, height=height, guidance_scale=guidance_scale, + num_inference_steps=num_inference_steps).images[0] + image.save('output.png') + return 'output.png' +sample_prompts = ['close-up portrait of girl', + 'close-up portrait of anime pig', 'close-up portrait of young lady'] +sample_prompts = [[x] for x in sample_prompts] +block = gr.Blocks(title='gguf').queue() +with block: + gr.Markdown('## 📷 Image 2') + with gr.Row(): + prompt_input = gr.Textbox(label='Prompt', placeholder= + 'Enter your prompt here (or click Sample Prompt)', value='') + neg_prompt_input = gr.Textbox(label='Negative Prompt', value= + 'blurry, cropped, ugly', visible=False) + quick_prompts = gr.Dataset(samples=sample_prompts, label= + 'Sample Prompt', samples_per_page=1000, components=[prompt_input]) + quick_prompts.click(lambda x: x[0], inputs=[quick_prompts], outputs + =prompt_input, show_progress=False, queue=False) + with gr.Row(): + width_input = gr.Number(label='Width', value=1024) + height_input = gr.Number(label='Height', value=1024) + guidance_scale = gr.Number(label='Guidance Scale', value=3.5) + num_inference_steps = gr.Number(label='Inference Steps', value=25) + generate_btn = gr.Button('Generate') + output_image = gr.Image(label='Generated Picture') + generate_btn.click(fn=generate_image, inputs=[prompt_input, + neg_prompt_input, width_input, height_input, guidance_scale, + num_inference_steps], outputs=output_image) +block.launch() \ No newline at end of file diff --git a/gguf_connector/lazy.py b/gguf_connector/lazy.py new file mode 100644 index 0000000..8a8abe2 --- /dev/null +++ b/gguf_connector/lazy.py @@ -0,0 +1,163 @@ +from __future__ import annotations +from abc import ABC, ABCMeta, abstractmethod +import logging +from typing import Any, Callable +import numpy as np +from numpy.typing import DTypeLike +logger = logging.getLogger(__name__) +class LazyMeta(ABCMeta): + def __new__(cls, name, bases, namespace, **kwargs): + def __getattr__(self, name): + meta_attr = getattr(self._meta, name) + if callable(meta_attr): + return type(self)._wrap_fn(lambda s, *args, **kwargs: + getattr(s, name)(*args, **kwargs), use_self=self) + elif isinstance(meta_attr, self._tensor_type): + return type(self)._wrap_fn(lambda s: getattr(s, name))(self) + else: + return meta_attr + namespace['__getattr__'] = __getattr__ + def mk_wrap(op_name, *, meta_noop: bool=False): + def wrapped_special_op(self, *args, **kwargs): + return type(self)._wrap_fn(getattr(type(self)._tensor_type, + op_name), meta_noop=meta_noop)(self, *args, **kwargs) + return wrapped_special_op + for binary_op in ('lt', 'le', 'eq', 'ne', 'ge', 'gt', 'notabs', + 'add', 'and', 'floordiv', 'invert', 'lshift', 'mod', 'mul', + 'matmul', 'neg', 'or', 'pos', 'pow', 'rshift', 'sub', 'truediv', + 'xor', 'iadd', 'iand', 'ifloordiv', 'ilshift', 'imod', 'imul', + 'ior', 'irshift', 'isub', 'ixor', 'radd', 'rand', 'rfloordiv', + 'rmul', 'ror', 'rpow', 'rsub', 'rtruediv', 'rxor'): + attr_name = f'__{binary_op}__' + namespace[attr_name] = mk_wrap(attr_name, meta_noop=True) + for special_op in ('getitem', 'setitem', 'len'): + attr_name = f'__{special_op}__' + namespace[attr_name] = mk_wrap(attr_name, meta_noop=False) + return super().__new__(cls, name, bases, namespace, **kwargs) +class LazyBase(ABC, metaclass=LazyMeta): + _tensor_type: type + _meta: Any + _data: Any | None + _args: tuple + _kwargs: dict[str, Any] + _func: Callable[[Any], Any] | None + def __init__(self, *, meta: Any, data: (Any | None)=None, args: tuple=( + ), kwargs: (dict[str, Any] | None)=None, func: (Callable[[Any], Any + ] | None)=None): + super().__init__() + self._meta = meta + self._data = data + self._args = args + self._kwargs = kwargs if kwargs is not None else {} + self._func = func + assert self._func is not None or self._data is not None + def __init_subclass__(cls): + if '_tensor_type' not in cls.__dict__: + raise TypeError( + f"property '_tensor_type' must be defined for {cls!r}") + return super().__init_subclass__() + @staticmethod + def _recurse_apply(o, fn): + if isinstance(o, (list, tuple)): + L = [] + for item in o: + L.append(LazyBase._recurse_apply(item, fn)) + if isinstance(o, tuple): + L = tuple(L) + return L + elif isinstance(o, LazyBase): + return fn(o) + else: + return o + @classmethod + def _wrap_fn(cls, fn, *, use_self: (LazyBase | None)=None, meta_noop: ( + bool | DTypeLike | tuple[DTypeLike, Callable[[tuple[int, ...]], + tuple[int, ...]]])=False): + def wrapped_fn(*args, **kwargs): + if kwargs is None: + kwargs = {} + args = ((use_self,) if use_self is not None else ()) + args + meta_args = LazyBase._recurse_apply(args, lambda t: t._meta) + if isinstance(meta_noop, bool) and not meta_noop: + try: + res = fn(*meta_args, **kwargs) + except NotImplementedError: + res = None + else: + assert len(args) > 0 + res = args[0] + assert isinstance(res, cls) + res = res._meta + if meta_noop is not True: + if isinstance(meta_noop, tuple): + dtype, shape = meta_noop + assert callable(shape) + res = cls.meta_with_dtype_and_shape(dtype, shape( + res.shape)) + else: + res = cls.meta_with_dtype_and_shape(meta_noop, res. + shape) + if isinstance(res, cls._tensor_type): + return cls(meta=cls.eager_to_meta(res), args=args, kwargs= + kwargs, func=fn) + elif isinstance(res, tuple) and all(isinstance(t, cls. + _tensor_type) for t in res): + shared_args: list = [args, None] + def eager_tuple_element(a: list[Any], i: int=0, /, **kw): + assert len(a) == 2 + if a[1] is None: + a[1] = fn(*a[0], **kw) + return a[1][i] + return tuple(cls(meta=cls.eager_to_meta(res[i]), args=( + shared_args, i), kwargs=kwargs, func= + eager_tuple_element) for i in range(len(res))) + else: + del res + eager_args = cls.to_eager(args) + return fn(*eager_args, **kwargs) + return wrapped_fn + @classmethod + def to_eager(cls, t): + def simple_to_eager(_t): + if _t._data is not None: + return _t._data + assert _t._func is not None + _t._args = cls._recurse_apply(_t._args, simple_to_eager) + _t._data = _t._func(*_t._args, **_t._kwargs) + assert _t._data is not None + assert _t._data.dtype == _t._meta.dtype + assert _t._data.shape == _t._meta.shape + return _t._data + return cls._recurse_apply(t, simple_to_eager) + @classmethod + def eager_to_meta(cls, t): + return cls.meta_with_dtype_and_shape(t.dtype, t.shape) + @classmethod + @abstractmethod + def meta_with_dtype_and_shape(cls, dtype, shape): + pass + @classmethod + def from_eager(cls, t): + if type(t) is cls: + return t + elif isinstance(t, cls._tensor_type): + return cls(meta=cls.eager_to_meta(t), data=t) + else: + return TypeError( + f'{type(t)!r} is not compatible with {cls._tensor_type!r}') +class LazyNumpyTensor(LazyBase): + _tensor_type = np.ndarray + shape: tuple[int, ...] + @classmethod + def meta_with_dtype_and_shape(cls, dtype, shape): + cheat = np.zeros(1, dtype) + return np.lib.stride_tricks.as_strided(cheat, shape, (0 for _ in shape) + ) + def astype(self, dtype, *args, **kwargs): + meta = type(self).meta_with_dtype_and_shape(dtype, self._meta.shape) + full_args = (self, dtype) + args + return type(self)(meta=meta, args=full_args, kwargs=kwargs, func=lambda + a, *args, **kwargs: a.astype(*args, **kwargs)) + def tofile(self, *args, **kwargs): + eager = LazyNumpyTensor.to_eager(self) + return eager.tofile(*args, **kwargs) \ No newline at end of file diff --git a/gguf_connector/logo.png b/gguf_connector/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..64a091c31782f5a9b74ac7d0b42ce0a8e19cc11e GIT binary patch literal 578 zcmV-I0=@l-P)Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D0oO@HK~z{r-Ic#i z!!QuWol;m3bm$BcNT?gE44sf*fQ{$i1(*;M@4~dXX<|UXeEsQAZ;0co zuDUxPs;|8V*aqLUO_W@(VC(d&k3q4bGSN;@xEgwj9)!T&_D#GuBM>RgHAE1!|8($v)MmHb0n&N83 zMd%O2%h+zHQ|#12x?KipF07*qoM6N<$f&m8gw*UYD literal 0 HcmV?d00001 diff --git a/gguf_connector/m.py b/gguf_connector/m.py new file mode 100644 index 0000000..7772f0f --- /dev/null +++ b/gguf_connector/m.py @@ -0,0 +1,61 @@ +import json, os.path, urllib.request +def get_file_size(url): + with urllib.request.urlopen(url) as response: + size = int(response.headers['Content-Length']) + return size +from tqdm import tqdm +def clone_file(url): + try: + file_size = get_file_size(url) + filename = os.path.basename(url) + with urllib.request.urlopen(url) as response, open(filename, 'wb' + ) as file, tqdm(total=file_size, unit='B', unit_scale=True, + unit_divisor=1024, desc=f'Downloading {filename}') as pbar: + chunk_size = 1024 + while True: + chunk = response.read(chunk_size) + if not chunk: + break + file.write(chunk) + pbar.update(len(chunk)) + print( + f"File cloned successfully and saved as '{filename}' in the current directory." + ) + except Exception as e: + print(f'Error: {e}') +def read_json_file(file_path): + with open(file_path, 'r') as file: + data = json.load(file) + return data +def extract_names(data): + for idx, entry in enumerate(data, start=1): + print(f"{idx}. {entry['name']}") +def handle_user_input(data): + while True: + user_choice = input( + f"Enter your choice (1 to {len(data)}) or 'q' to quit: ") + if user_choice.lower() == 'q': + break + try: + index = int(user_choice) + if 1 <= index <= len(data): + source_url = data[index - 1]['url'] + clone_file(source_url) + break + else: + print('Invalid selection. Please enter a valid number.') + except ValueError: + print('Invalid input. Please enter a number.') +print("""Please select an option: +1. choose a connector +2. download model(s)""" + ) +choice = input('Enter your choice (1 to 2): ') +if choice == '1': + from gguf_connector import menu +elif choice == '2': + file_path = os.path.join(os.path.dirname(__file__), 'data.json') + json_data = read_json_file(file_path) + print('Please select a GGUF file to download:') + extract_names(json_data) + handle_user_input(json_data) \ No newline at end of file diff --git a/gguf_connector/m2.py b/gguf_connector/m2.py new file mode 100644 index 0000000..bba7443 --- /dev/null +++ b/gguf_connector/m2.py @@ -0,0 +1,17 @@ +import os +from tqdm import tqdm +def merge_gguf_files(): + gguf_files = [f for f in os.listdir('.') if f.endswith('.gguf')] + if not gguf_files: + print('No .gguf files found in the current directory.') + return + filename = input('Enter the output file name (without .gguf): ').strip() + if not filename: + filename = 'model' + output_file = f'{filename}.gguf' + with open(output_file, 'wb') as outfile: + for fname in tqdm(gguf_files, desc='Merging files', unit='file'): + with open(fname, 'rb') as infile: + outfile.write(infile.read()) + print(f'\nAll files merged into: {output_file}') +merge_gguf_files() \ No newline at end of file diff --git a/gguf_connector/ma.py b/gguf_connector/ma.py new file mode 100644 index 0000000..c058551 --- /dev/null +++ b/gguf_connector/ma.py @@ -0,0 +1,25 @@ +import os, safetensors.torch +def merge_safetensors(output): + files = [f for f in os.listdir() if f.endswith('.safetensors') and f != + output] + if not files: + print('No safetensors files found.') + return + merged_data = {} + for file in files: + print(f'Loading {file}...') + data = safetensors.torch.load_file(file) + for key, value in data.items(): + if key in merged_data: + print(f"Warning: Duplicate key '{key}' in {file}. Skipping.") + else: + merged_data[key] = value + print(f'Saving merged data to {output}...') + safetensors.torch.save_file(merged_data, output) + print('Merge completed successfully.') +ask = input('Assign a name other than model.safetensors (Y/n)? ') +if ask.lower() == 'y': + given = input('Enter a file name: ') +else: + given = 'model.safetensors' +merge_safetensors(given) \ No newline at end of file diff --git a/gguf_connector/menu.py b/gguf_connector/menu.py new file mode 100644 index 0000000..025ad74 --- /dev/null +++ b/gguf_connector/menu.py @@ -0,0 +1,34 @@ +print("""Please select a connector: +1. llama.cpp +2. ctransformers""") +choice = input('Enter your choice (1 to 2): ') +if choice == '1': + print( + """Connector: llama.cpp is selected! +Please select an interface: +1. graphical +2. command-line""" + ) + option = input('Enter your choice (1 to 2): ') + if option == '1': + from .cpp import * + elif option == '2': + from .gpp import * + else: + print('Not a valid number.') +elif choice == '2': + print( + """Connector: ctransformers is selected! +Please select an interface: +1. graphical +2. command-line""" + ) + option = input('Enter your choice (1 to 2): ') + if option == '1': + from .c import * + elif option == '2': + from .g import * + else: + print('Not a valid number.') +else: + print('Not a valid number.') \ No newline at end of file diff --git a/gguf_connector/n.py b/gguf_connector/n.py new file mode 100644 index 0000000..df50d8b --- /dev/null +++ b/gguf_connector/n.py @@ -0,0 +1,18 @@ +import os, subprocess +def clone_github_repo(repo_url): + try: + repo_name = repo_url.rstrip('/').split('/')[-1].replace('.git', '') + if os.path.exists(repo_name): + print( + f"Error: A folder named '{repo_name}' already exists in the current directory." + ) + return + print(f"Cloning repository '{repo_url}'...") + subprocess.run(['git', 'clone', repo_url], check=True) + print(f"Repository '{repo_name}' cloned successfully.") + except subprocess.CalledProcessError as e: + print(f'Error: Failed to clone the repository. {e}') + except Exception as e: + print(f'Unexpected error: {e}') +repo_url = 'https://github.com/calcuis/gguf' +clone_github_repo(repo_url) \ No newline at end of file diff --git a/gguf_connector/o.py b/gguf_connector/o.py new file mode 100644 index 0000000..30afba7 --- /dev/null +++ b/gguf_connector/o.py @@ -0,0 +1,3 @@ +import webbrowser +print('activating the default browser...') +webbrowser.open('https://gguf.org') \ No newline at end of file diff --git a/gguf_connector/p.py b/gguf_connector/p.py new file mode 100644 index 0000000..f0f13b2 --- /dev/null +++ b/gguf_connector/p.py @@ -0,0 +1,28 @@ +import json, requests, urllib.request +from tqdm import tqdm +def read_json_file(file_path): + response = urllib.request.urlopen(file_path) + data = json.loads(response.read()) + return data +def download_7z_file(url, output_filename): + response = requests.get(url, stream=True) + if response.status_code == 200: + total_size = int(response.headers.get('content-length', 0)) + with open(output_filename, 'wb') as file, tqdm(desc= + f'Downloading {output_filename}', total=total_size, unit='B', + unit_scale=True, unit_divisor=1024) as progress_bar: + for chunk in response.iter_content(chunk_size=1024): + file.write(chunk) + progress_bar.update(len(chunk)) + print(f'Downloaded {output_filename} to the current directory.') + else: + print( + 'Failed to download the file. Seems encountering a connection problem.' + ) +ver = 'https://raw.githubusercontent.com/calcuis/gguf-pack/main/version.json' +v = read_json_file(ver) +url = ( + f"https://github.com/calcuis/gguf-pack/releases/download/{v[0]['version']}/GGUF_windows_portable.7z" + ) +output_filename = 'gguf.7z' +download_7z_file(url, output_filename) \ No newline at end of file diff --git a/gguf_connector/p1.py b/gguf_connector/p1.py new file mode 100644 index 0000000..c79dc81 --- /dev/null +++ b/gguf_connector/p1.py @@ -0,0 +1,28 @@ +import json, requests, urllib.request +from tqdm import tqdm +def read_json_file(file_path): + response = urllib.request.urlopen(file_path) + data = json.loads(response.read()) + return data +def download_7z_file(url, output_filename): + response = requests.get(url, stream=True) + if response.status_code == 200: + total_size = int(response.headers.get('content-length', 0)) + with open(output_filename, 'wb') as file, tqdm(desc= + f'Downloading {output_filename}', total=total_size, unit='B', + unit_scale=True, unit_divisor=1024) as progress_bar: + for chunk in response.iter_content(chunk_size=1024): + file.write(chunk) + progress_bar.update(len(chunk)) + print(f'Downloaded {output_filename} to the current directory.') + else: + print( + 'Failed to download the file. Seems encountering a connection problem.' + ) +ver = 'https://raw.githubusercontent.com/calcuis/packpack/main/version.json' +v = read_json_file(ver) +url = ( + f"https://github.com/calcuis/packpack/releases/download/{v[0]['version']}/packpack.7z" + ) +output_filename = 'framepack.7z' +download_7z_file(url, output_filename) \ No newline at end of file diff --git a/gguf_connector/p2.py b/gguf_connector/p2.py new file mode 100644 index 0000000..7d80b3b --- /dev/null +++ b/gguf_connector/p2.py @@ -0,0 +1,18 @@ +import os, subprocess +def clone_github_repo(repo_url): + try: + repo_name = repo_url.rstrip('/').split('/')[-1].replace('.git', '') + if os.path.exists(repo_name): + print( + f"Error: A folder named '{repo_name}' already exists in the current directory." + ) + return + print(f"Cloning repository '{repo_url}'...") + subprocess.run(['git', 'clone', repo_url], check=True) + print(f"Repository '{repo_name}' cloned successfully.") + except subprocess.CalledProcessError as e: + print(f'Error: Failed to clone the repository. {e}') + except Exception as e: + print(f'Unexpected error: {e}') +repo_url = 'https://github.com/calcuis/packpack' +clone_github_repo(repo_url) \ No newline at end of file diff --git a/gguf_connector/pg.py b/gguf_connector/pg.py new file mode 100644 index 0000000..0e8fbad --- /dev/null +++ b/gguf_connector/pg.py @@ -0,0 +1,62 @@ +import os +def wav_handler(): + wav_files = [file for file in os.listdir() if file.endswith('.wav')] + if wav_files: + print('WAV file(s) available. Select which one to use:') + for index, file_name in enumerate(wav_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(wav_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = wav_files[choice_index] + print(f'WAV file: {selected_file} is selected!') + import speech_recognition as sr + r = sr.Recognizer() + with sr.AudioFile(selected_file) as source: + audio = r.record(source) + text = r.recognize_google(audio) + try: + from rich.console import Console + console = Console() + console.print(f'Content recognized as: [green]' + text) + input( + '\n---Enter to prompt the WAV content recognized above into GGUF model---' + ) + print('Processing...') + output = llm('Q: ' + text, max_tokens=4096, echo=True) + ans = output['choices'][0]['text'] + print(ans + '\n') + except sr.UnknownValueError: + print('Could not understand audio') + except sr.RequestError as e: + print('Error; {0}'.format(e)) + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') + else: + print('No WAV files are available in the current directory.') + input('--- Press ENTER To Skip ---') +gguf_files = [file for file in os.listdir() if file.endswith('.gguf')] +if gguf_files: + print('GGUF file(s) available. Select which one to use:') + for index, file_name in enumerate(gguf_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(gguf_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = gguf_files[choice_index] + print(f'Model file: {selected_file} is selected!') + ModelPath = selected_file + print('Processing...') + from llama_cpp import Llama + llm = Llama(model_path=ModelPath) + while True: + ask = input('---Enter to select a WAV file (Q for quit)---') + if ask == 'q' or ask == 'Q': + break + wav_handler() + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No GGUF files are available in the current directory.') + input('--- Press ENTER To Exit ---') +print('Goodbye!') \ No newline at end of file diff --git a/gguf_connector/pp.py b/gguf_connector/pp.py new file mode 100644 index 0000000..af30e88 --- /dev/null +++ b/gguf_connector/pp.py @@ -0,0 +1,65 @@ +import os +def pdf_handler(): + pdf_files = [file for file in os.listdir() if file.endswith('.pdf')] + def join_text(input_text): + joined_text = ' '.join(input_text.splitlines()) + return joined_text + if pdf_files: + print('PDF file(s) available. Select which one to use:') + for index, file_name in enumerate(pdf_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(pdf_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = pdf_files[choice_index] + print(f'PDF file: {selected_file} is selected!') + from pypdf import PdfReader + reader = PdfReader(selected_file) + text = '' + number_of_pages = len(reader.pages) + for i in range(number_of_pages): + page = reader.pages[i] + text += page.extract_text() + output_text = join_text(text) + inject = f'analyze the content below: ' + output_text + from rich.console import Console + console = Console() + console.print(f'\nPDF cotent extracted as below:\n\n[green]' + text + ) + input('---Enter to analyze the PDF content above---') + from rich.progress import Progress + with Progress(transient=True) as progress: + task = progress.add_task('Processing', total=None) + output = llm('Q: ' + inject, max_tokens=32768, echo=False) + ans = output['choices'][0]['text'] + print(inject + '\n\nOutput:' + ans) + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') + else: + print('No PDF files are available in the current directory.') + input('--- Press ENTER To Skip ---') +gguf_files = [file for file in os.listdir() if file.endswith('.gguf')] +if gguf_files: + print('GGUF file(s) available. Select which one to use:') + for index, file_name in enumerate(gguf_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(gguf_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = gguf_files[choice_index] + print(f'Model file: {selected_file} is selected!') + ModelPath = selected_file + print('Processing...') + from llama_cpp import Llama + llm = Llama(model_path=ModelPath) + while True: + ask = input('---Enter to select a PDF file (Q for quit)---') + if ask.lower() == 'q': + break + pdf_handler() + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No GGUF files are available in the current directory.') + input('--- Press ENTER To Exit ---') +print('Goodbye!') \ No newline at end of file diff --git a/gguf_connector/ps.py b/gguf_connector/ps.py new file mode 100644 index 0000000..21eceac --- /dev/null +++ b/gguf_connector/ps.py @@ -0,0 +1,62 @@ +import os +def wav_handler(): + wav_files = [file for file in os.listdir() if file.endswith('.wav')] + if wav_files: + print('WAV file(s) available. Select which one to use:') + for index, file_name in enumerate(wav_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(wav_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = wav_files[choice_index] + print(f'WAV file: {selected_file} is selected!') + import speech_recognition as sr + r = sr.Recognizer() + with sr.AudioFile(selected_file) as source: + audio = r.record(source) + text = r.recognize_sphinx(audio) + try: + from rich.console import Console + console = Console() + console.print(f'Content recognized as: [green]' + text) + input( + '\n---Enter to prompt the WAV content recognized above into GGUF model---' + ) + print('Processing...') + output = llm('Q: ' + text, max_tokens=4096, echo=True) + ans = output['choices'][0]['text'] + print(ans + '\n') + except sr.UnknownValueError: + print('Could not understand audio') + except sr.RequestError as e: + print('Error; {0}'.format(e)) + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') + else: + print('No WAV files are available in the current directory.') + input('--- Press ENTER To Skip ---') +gguf_files = [file for file in os.listdir() if file.endswith('.gguf')] +if gguf_files: + print('GGUF file(s) available. Select which one to use:') + for index, file_name in enumerate(gguf_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(gguf_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = gguf_files[choice_index] + print(f'Model file: {selected_file} is selected!') + ModelPath = selected_file + print('Processing...') + from llama_cpp import Llama + llm = Llama(model_path=ModelPath) + while True: + ask = input('---Enter to select a WAV file (Q for quit)---') + if ask == 'q' or ask == 'Q': + break + wav_handler() + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No GGUF files are available in the current directory.') + input('--- Press ENTER To Exit ---') +print('Goodbye!') \ No newline at end of file diff --git a/gguf_connector/q.py b/gguf_connector/q.py new file mode 100644 index 0000000..c399073 --- /dev/null +++ b/gguf_connector/q.py @@ -0,0 +1,43 @@ +import torch +from safetensors.torch import load_file, save_file +from tqdm import tqdm +def quantize_to_fp8(tensor): + if tensor.dtype != torch.bfloat16: + raise ValueError('Input tensor must be in BF16 format.') + tensor = tensor.to(torch.float16) + fp8_max = 240.0 + fp8_min = -fp8_max + clamped_tensor = tensor.clamp(min=fp8_min, max=fp8_max) + scale = fp8_max / torch.max(torch.abs(clamped_tensor)) + quantized_tensor = torch.round(clamped_tensor * scale) / scale + return quantized_tensor.to(torch.float8_e4m3fn) +import os +safetensors_files = [file for file in os.listdir() if file.endswith( + '.safetensors')] +if safetensors_files: + print('Safetensors file(s) available. Select which one to convert:') + for index, file_name in enumerate(safetensors_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = safetensors_files[choice_index] + print(f'Model file: {selected_file} is selected!') + input_file = selected_file + output_file = ( + f'{os.path.splitext(input_file)[0]}_fp8_e4m3fn.safetensors') + data = load_file(input_file) + quantized_data = {} + print('Starting quantization process...') + for key, tensor in tqdm(data.items(), desc='Quantizing tensors', + unit='tensor'): + tensor = tensor.to(dtype=torch.bfloat16, device='cuda') + quantized_tensor = quantize_to_fp8(tensor) + quantized_data[key] = quantized_tensor.cpu() + save_file(quantized_data, output_file) + print(f'Quantized safetensors saved to {output_file}.') + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No safetensors files are available in the current directory.') + input('--- Press ENTER To Exit ---') \ No newline at end of file diff --git a/gguf_connector/q1.py b/gguf_connector/q1.py new file mode 100644 index 0000000..bd2318c --- /dev/null +++ b/gguf_connector/q1.py @@ -0,0 +1,43 @@ +import torch +from safetensors.torch import load_file, save_file +from tqdm import tqdm +def quantize_to_fp8(tensor): + if tensor.dtype != torch.bfloat16: + raise ValueError('Input tensor must be in BF16 format.') + tensor = tensor.to(torch.float16) + fp8_max = 240.0 + fp8_min = -fp8_max + clamped_tensor = tensor.clamp(min=fp8_min, max=fp8_max) + scale = fp8_max / torch.max(torch.abs(clamped_tensor)) + quantized_tensor = torch.round(clamped_tensor * scale) / scale + return quantized_tensor.to(torch.float8_e4m3fn) +import os +safetensors_files = [file for file in os.listdir() if file.endswith( + '.safetensors')] +if safetensors_files: + print('Safetensors file(s) available. Select which one to convert:') + for index, file_name in enumerate(safetensors_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = safetensors_files[choice_index] + print(f'Model file: {selected_file} is selected!') + input_file = selected_file + output_file = ( + f'{os.path.splitext(input_file)[0]}_fp8_e4m3fn.safetensors') + data = load_file(input_file) + quantized_data = {} + print('Starting quantization process...') + for key, tensor in tqdm(data.items(), desc='Quantizing tensors', + unit='tensor'): + tensor = tensor.to(dtype=torch.bfloat16, device='cpu') + quantized_tensor = quantize_to_fp8(tensor) + quantized_data[key] = quantized_tensor.cpu() + save_file(quantized_data, output_file) + print(f'Quantized safetensors saved to {output_file}.') + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No safetensors files are available in the current directory.') + input('--- Press ENTER To Exit ---') \ No newline at end of file diff --git a/gguf_connector/q2.py b/gguf_connector/q2.py new file mode 100644 index 0000000..de531f2 --- /dev/null +++ b/gguf_connector/q2.py @@ -0,0 +1,32 @@ +import torch +from safetensors.torch import load_file, save_file +from tqdm import tqdm +def convert_safetensors_to_fp32(file_path): + print(f'Loading: {file_path}') + tensors = load_file(file_path) + converted_tensors = {} + for key, tensor in tqdm(tensors.items(), desc='Converting tensors', + unit='tensor'): + converted_tensors[key] = tensor.to(torch.float32) + output_file = file_path.replace('.safetensors', '_fp32.safetensors') + save_file(converted_tensors, output_file) + print(f'Converted and saved to: {output_file}') +def upscale_to_fp32(): + import os + safetensors_files = [f for f in os.listdir('.') if f.endswith( + '.safetensors')] + if not safetensors_files: + print('No .safetensors files found in the current directory.') + return + print('Select a file to convert:') + for idx, file in enumerate(safetensors_files, 1): + print(f'{idx}. {file}') + try: + choice = int(input('Enter the number of the file: ')) - 1 + if choice < 0 or choice >= len(safetensors_files): + print('Invalid selection.') + return + convert_safetensors_to_fp32(safetensors_files[choice]) + except ValueError: + print('Please enter a valid number.') +upscale_to_fp32() \ No newline at end of file diff --git a/gguf_connector/quant.py b/gguf_connector/quant.py new file mode 100644 index 0000000..55f4bae --- /dev/null +++ b/gguf_connector/quant.py @@ -0,0 +1,755 @@ +from __future__ import annotations +from abc import ABC, abstractmethod +from typing import Any, Callable, Sequence +from math import log2, ceil +from numpy.typing import DTypeLike +import numpy as np +from .const import GGML_QUANT_SIZES, GGMLQuantizationType, QK_K +from .lazy import LazyNumpyTensor +def quant_shape_to_byte_shape(shape, quant_type): + block_size, type_size = GGML_QUANT_SIZES[quant_type] + if shape[-1] % block_size != 0: + raise ValueError( + f'Quantized tensor row size ({shape[-1]}) is not a multiple of {quant_type.name} block size ({block_size})' + ) + return *shape[:-1], shape[-1] // block_size * type_size +def quant_shape_from_byte_shape(shape, quant_type): + block_size, type_size = GGML_QUANT_SIZES[quant_type] + if shape[-1] % type_size != 0: + raise ValueError( + f'Quantized tensor bytes per row ({shape[-1]}) is not a multiple of {quant_type.name} type size ({type_size})' + ) + return *shape[:-1], shape[-1] // type_size * block_size +def _apply_over_grouped_rows(func, arr, otype, oshape): + rows = arr.reshape((-1, arr.shape[-1])) + osize = 1 + for dim in oshape: + osize *= dim + out = np.empty(shape=osize, dtype=otype) + n_groups = rows.shape[0] // 16 or 1 + np.concatenate([func(group).ravel() for group in np.array_split(rows, + n_groups)], axis=0, out=out) + return out.reshape(oshape) +def np_roundf(n): + a = abs(n) + floored = np.floor(a) + b = floored + np.floor(2 * (a - floored)) + return np.sign(n) * b +class QuantError(Exception): + ... +_type_traits: dict[GGMLQuantizationType, type[__Quant]] = {} +def quantize(data, qtype): + if qtype == GGMLQuantizationType.F32: + return data.astype(np.float32, copy=False) + elif qtype == GGMLQuantizationType.F16: + return data.astype(np.float16, copy=False) + elif (q := _type_traits.get(qtype)) is not None: + return q.quantize(data) + else: + raise NotImplementedError( + f'Quantization for {qtype.name} is not yet implemented') +def dequantize(data, qtype): + if qtype == GGMLQuantizationType.F32: + return data.view(np.float32) + elif qtype == GGMLQuantizationType.F16: + return data.view(np.float16).astype(np.float32) + elif (q := _type_traits.get(qtype)) is not None: + return q.dequantize(data) + else: + raise NotImplementedError( + f'Dequantization for {qtype.name} is not yet implemented') +class __Quant(ABC): + qtype: GGMLQuantizationType + block_size: int + type_size: int + grid: np.ndarray[Any, np.dtype[np.float32]] | None = None + grid_shape: tuple[int, int] = (0, 0) + grid_map: tuple[int | float, ...] = () + grid_hex: bytes | None = None + def __init__(self): + return TypeError("Quant conversion classes can't have instances") + def __init_subclass__(cls, qtype): + cls.qtype = qtype + cls.block_size, cls.type_size = GGML_QUANT_SIZES[qtype] + cls.__quantize_lazy = LazyNumpyTensor._wrap_fn(cls.__quantize_array, + meta_noop=(np.uint8, cls.__shape_to_bytes)) + cls.__dequantize_lazy = LazyNumpyTensor._wrap_fn(cls. + __dequantize_array, meta_noop=(np.float32, cls.__shape_from_bytes)) + assert qtype not in _type_traits + _type_traits[qtype] = cls + @classmethod + def init_grid(cls): + if cls.grid is not None or cls.grid_hex is None: + return + bits_per_elem = ceil(log2(len(cls.grid_map))) + assert bits_per_elem != 0, cls.qtype.name + elems_per_byte = 8 // bits_per_elem + grid = np.frombuffer(cls.grid_hex, dtype=np.uint8) + grid = grid.reshape((-1, 2)) + grid = (np.where(grid > 64, grid + 9, grid) & 15) << np.array([4, 0 + ], dtype=np.uint8).reshape((1, 2)) + grid = grid[..., 0] | grid[..., 1] + grid = grid.reshape((-1, 1)) >> np.array([i for i in range(0, 8, 8 // + elems_per_byte)], dtype=np.uint8).reshape((1, elems_per_byte)) + grid = (grid & (1 << bits_per_elem) - 1).reshape((-1, 1)) + grid_map = np.array(cls.grid_map, dtype=np.float32).reshape((1, -1)) + grid = np.take_along_axis(grid_map, grid, axis=-1) + cls.grid = grid.reshape((1, 1, *cls.grid_shape)) + @classmethod + @abstractmethod + def quantize_blocks(cls, blocks): + raise NotImplementedError + @classmethod + @abstractmethod + def dequantize_blocks(cls, blocks): + raise NotImplementedError + @classmethod + def quantize_rows(cls, rows): + rows = rows.astype(np.float32, copy=False) + shape = rows.shape + n_blocks = rows.size // cls.block_size + blocks = rows.reshape((n_blocks, cls.block_size)) + blocks = cls.quantize_blocks(blocks) + assert blocks.dtype == np.uint8 + assert blocks.shape[-1] == cls.type_size + return blocks.reshape(cls.__shape_to_bytes(shape)) + @classmethod + def dequantize_rows(cls, rows): + rows = rows.view(np.uint8) + shape = rows.shape + n_blocks = rows.size // cls.type_size + blocks = rows.reshape((n_blocks, cls.type_size)) + blocks = cls.dequantize_blocks(blocks) + assert blocks.dtype == np.float32 + assert blocks.shape[-1] == cls.block_size + return blocks.reshape(cls.__shape_from_bytes(shape)) + @classmethod + def __shape_to_bytes(cls, shape): + return quant_shape_to_byte_shape(shape, cls.qtype) + @classmethod + def __shape_from_bytes(cls, shape): + return quant_shape_from_byte_shape(shape, cls.qtype) + @classmethod + def __quantize_array(cls, array): + return _apply_over_grouped_rows(cls.quantize_rows, arr=array, otype + =np.uint8, oshape=cls.__shape_to_bytes(array.shape)) + @classmethod + def __dequantize_array(cls, array): + cls.init_grid() + return _apply_over_grouped_rows(cls.dequantize_rows, arr=array, + otype=np.float32, oshape=cls.__shape_from_bytes(array.shape)) + @classmethod + def __quantize_lazy(cls, lazy_tensor: LazyNumpyTensor, /): + pass + @classmethod + def __dequantize_lazy(cls, lazy_tensor: LazyNumpyTensor, /): + pass + @classmethod + def can_quantize(cls, tensor): + return tensor.shape[-1] % cls.block_size == 0 + @classmethod + def quantize(cls, tensor): + if not cls.can_quantize(tensor): + raise QuantError( + f"Can't quantize tensor with shape {tensor.shape} to {cls.qtype.name}" + ) + if isinstance(tensor, LazyNumpyTensor): + return cls.__quantize_lazy(tensor) + else: + return cls.__quantize_array(tensor) + @classmethod + def dequantize(cls, tensor): + if isinstance(tensor, LazyNumpyTensor): + return cls.__dequantize_lazy(tensor) + else: + return cls.__dequantize_array(tensor) +class BF16(__Quant, qtype=GGMLQuantizationType.BF16): + @classmethod + def quantize_blocks(cls, blocks): + n = blocks.view(np.uint32) + n = np.where(n & 2147483647 > 2139095040, n & np.uint32(4294901760) | + np.uint32(64 << 16), n) + n = np.uint64(n) + (32767 + (n >> 16 & 1)) >> 16 + return n.astype(np.uint16).view(np.uint8) + @classmethod + def dequantize_blocks(cls, blocks): + return (blocks.view(np.int16).astype(np.int32) << 16).view(np.float32) +class Q4_0(__Quant, qtype=GGMLQuantizationType.Q4_0): + @classmethod + def quantize_blocks(cls, blocks): + n_blocks = blocks.shape[0] + imax = abs(blocks).argmax(axis=-1, keepdims=True) + max = np.take_along_axis(blocks, imax, axis=-1) + d = max / -8 + with np.errstate(divide='ignore'): + id = np.where(d == 0, 0, 1 / d) + qs = np.trunc(np.float64(blocks) * np.float64(id) + np.float64(8.5), + dtype=np.float32).astype(np.uint8).clip(0, 15) + qs = qs.reshape((n_blocks, 2, cls.block_size // 2)) + qs = qs[..., 0, :] | qs[..., 1, :] << np.uint8(4) + d = d.astype(np.float16).view(np.uint8) + return np.concatenate([d, qs], axis=-1) + @classmethod + def dequantize_blocks(cls, blocks): + n_blocks = blocks.shape[0] + d, qs = np.hsplit(blocks, [2]) + d = d.view(np.float16).astype(np.float32) + qs = qs.reshape((n_blocks, -1, 1, cls.block_size // 2)) >> np.array([ + 0, 4], dtype=np.uint8).reshape((1, 1, 2, 1)) + qs = (qs & np.uint8(15)).reshape((n_blocks, -1)).astype(np.int8 + ) - np.int8(8) + return d * qs.astype(np.float32) +class Q4_1(__Quant, qtype=GGMLQuantizationType.Q4_1): + @classmethod + def quantize_blocks(cls, blocks): + n_blocks = blocks.shape[0] + max = blocks.max(axis=-1, keepdims=True) + min = blocks.min(axis=-1, keepdims=True) + d = (max - min) / 15 + with np.errstate(divide='ignore'): + id = np.where(d == 0, 0, 1 / d) + qs = np.trunc((blocks - min) * id + np.float32(0.5), dtype=np.float32 + ).astype(np.uint8).clip(0, 15) + qs = qs.reshape((n_blocks, 2, cls.block_size // 2)) + qs = qs[..., 0, :] | qs[..., 1, :] << np.uint8(4) + d = d.astype(np.float16).view(np.uint8) + m = min.astype(np.float16).view(np.uint8) + return np.concatenate([d, m, qs], axis=-1) + @classmethod + def dequantize_blocks(cls, blocks): + n_blocks = blocks.shape[0] + d, rest = np.hsplit(blocks, [2]) + m, qs = np.hsplit(rest, [2]) + d = d.view(np.float16).astype(np.float32) + m = m.view(np.float16).astype(np.float32) + qs = qs.reshape((n_blocks, -1, 1, cls.block_size // 2)) >> np.array([ + 0, 4], dtype=np.uint8).reshape((1, 1, 2, 1)) + qs = (qs & np.uint8(15)).reshape((n_blocks, -1)).astype(np.float32) + return d * qs + m +class Q5_0(__Quant, qtype=GGMLQuantizationType.Q5_0): + @classmethod + def quantize_blocks(cls, blocks): + n_blocks = blocks.shape[0] + imax = abs(blocks).argmax(axis=-1, keepdims=True) + max = np.take_along_axis(blocks, imax, axis=-1) + d = max / -16 + with np.errstate(divide='ignore'): + id = np.where(d == 0, 0, 1 / d) + q = np.trunc(np.float64(blocks) * np.float64(id) + np.float64(16.5), + dtype=np.float32).astype(np.uint8).clip(0, 31) + qs = q.reshape((n_blocks, 2, cls.block_size // 2)) + qs = qs[..., 0, :] & np.uint8(15) | qs[..., 1, :] << np.uint8(4) + qh = np.packbits(q.reshape((n_blocks, 1, 32)) >> np.uint8(4), axis= + -1, bitorder='little').reshape(n_blocks, 4) + d = d.astype(np.float16).view(np.uint8) + return np.concatenate([d, qh, qs], axis=-1) + @classmethod + def dequantize_blocks(cls, blocks): + n_blocks = blocks.shape[0] + d, rest = np.hsplit(blocks, [2]) + qh, qs = np.hsplit(rest, [4]) + d = d.view(np.float16).astype(np.float32) + qh = qh.view(np.uint32) + qh = qh.reshape((n_blocks, 1)) >> np.array([i for i in range(32)], + dtype=np.uint32).reshape((1, 32)) + ql = qs.reshape((n_blocks, -1, 1, cls.block_size // 2)) >> np.array([ + 0, 4], dtype=np.uint8).reshape((1, 1, 2, 1)) + qh = (qh & np.uint32(1)).astype(np.uint8) + ql = (ql & np.uint8(15)).reshape((n_blocks, -1)) + qs = (ql | qh << np.uint8(4)).astype(np.int8) - np.int8(16) + return d * qs.astype(np.float32) +class Q5_1(__Quant, qtype=GGMLQuantizationType.Q5_1): + @classmethod + def quantize_blocks(cls, blocks): + n_blocks = blocks.shape[0] + max = blocks.max(axis=-1, keepdims=True) + min = blocks.min(axis=-1, keepdims=True) + d = (max - min) / 31 + with np.errstate(divide='ignore'): + id = np.where(d == 0, 0, 1 / d) + q = np.trunc((blocks - min) * id + np.float32(0.5), dtype=np.float32 + ).astype(np.uint8).clip(0, 31) + qs = q.reshape((n_blocks, 2, cls.block_size // 2)) + qs = qs[..., 0, :] & np.uint8(15) | qs[..., 1, :] << np.uint8(4) + qh = np.packbits(q.reshape((n_blocks, 1, 32)) >> np.uint8(4), axis= + -1, bitorder='little').reshape(n_blocks, 4) + d = d.astype(np.float16).view(np.uint8) + m = min.astype(np.float16).view(np.uint8) + return np.concatenate([d, m, qh, qs], axis=-1) + @classmethod + def dequantize_blocks(cls, blocks): + n_blocks = blocks.shape[0] + d, rest = np.hsplit(blocks, [2]) + m, rest = np.hsplit(rest, [2]) + qh, qs = np.hsplit(rest, [4]) + d = d.view(np.float16).astype(np.float32) + m = m.view(np.float16).astype(np.float32) + qh = qh.view(np.uint32) + qh = qh.reshape((n_blocks, 1)) >> np.array([i for i in range(32)], + dtype=np.uint32).reshape((1, 32)) + ql = qs.reshape((n_blocks, -1, 1, cls.block_size // 2)) >> np.array([ + 0, 4], dtype=np.uint8).reshape((1, 1, 2, 1)) + qh = (qh & np.uint32(1)).astype(np.uint8) + ql = (ql & np.uint8(15)).reshape((n_blocks, -1)) + qs = (ql | qh << np.uint8(4)).astype(np.float32) + return d * qs + m +class Q8_0(__Quant, qtype=GGMLQuantizationType.Q8_0): + @classmethod + def quantize_blocks(cls, blocks): + d = abs(blocks).max(axis=1, keepdims=True) / 127 + with np.errstate(divide='ignore'): + id = np.where(d == 0, 0, 1 / d) + qs = np_roundf(blocks * id) + d = d.astype(np.float16).view(np.uint8) + qs = qs.astype(np.int8).view(np.uint8) + return np.concatenate([d, qs], axis=1) + @classmethod + def dequantize_blocks(cls, blocks): + d, x = np.split(blocks, [2], axis=1) + d = d.view(np.float16).astype(np.float32) + x = x.view(np.int8).astype(np.float32) + return x * d +class Q2_K(__Quant, qtype=GGMLQuantizationType.Q2_K): + @classmethod + def dequantize_blocks(cls, blocks): + n_blocks = blocks.shape[0] + scales, rest = np.hsplit(blocks, [QK_K // 16]) + qs, rest = np.hsplit(rest, [QK_K // 4]) + d, dmin = np.hsplit(rest, [2]) + d = d.view(np.float16).astype(np.float32) + dmin = dmin.view(np.float16).astype(np.float32) + dl = (d * (scales & 15).astype(np.float32)).reshape((n_blocks, QK_K // + 16, 1)) + ml = (dmin * (scales >> 4).astype(np.float32)).reshape((n_blocks, + QK_K // 16, 1)) + shift = np.array([0, 2, 4, 6], dtype=np.uint8).reshape((1, 1, 4, 1)) + qs = qs.reshape((n_blocks, -1, 1, 32)) >> shift & np.uint8(3) + qs = qs.reshape((n_blocks, QK_K // 16, 16)).astype(np.float32) + qs = dl * qs - ml + return qs.reshape((n_blocks, -1)) +class Q3_K(__Quant, qtype=GGMLQuantizationType.Q3_K): + @classmethod + def dequantize_blocks(cls, blocks): + n_blocks = blocks.shape[0] + hmask, rest = np.hsplit(blocks, [QK_K // 8]) + qs, rest = np.hsplit(rest, [QK_K // 4]) + scales, d = np.hsplit(rest, [12]) + d = d.view(np.float16).astype(np.float32) + lscales, hscales = np.hsplit(scales, [8]) + lscales = lscales.reshape((n_blocks, 1, 8)) >> np.array([0, 4], + dtype=np.uint8).reshape((1, 2, 1)) + lscales = lscales.reshape((n_blocks, 16)) + hscales = hscales.reshape((n_blocks, 1, 4)) >> np.array([0, 2, 4, 6 + ], dtype=np.uint8).reshape((1, 4, 1)) + hscales = hscales.reshape((n_blocks, 16)) + scales = lscales & np.uint8(15) | (hscales & np.uint8(3)) << np.uint8(4 + ) + scales = (scales.astype(np.int8) - np.int8(32)).astype(np.float32) + dl = (d * scales).reshape((n_blocks, 16, 1)) + ql = qs.reshape((n_blocks, -1, 1, 32)) >> np.array([0, 2, 4, 6], + dtype=np.uint8).reshape((1, 1, 4, 1)) + qh = hmask.reshape(n_blocks, -1, 1, 32) >> np.array([i for i in + range(8)], dtype=np.uint8).reshape((1, 1, 8, 1)) + ql = ql.reshape((n_blocks, 16, QK_K // 16)) & np.uint8(3) + qh = qh.reshape((n_blocks, 16, QK_K // 16)) & np.uint8(1) + qh = qh ^ np.uint8(1) + q = (ql.astype(np.int8) - (qh << np.uint8(2)).astype(np.int8)).astype( + np.float32) + return (dl * q).reshape((n_blocks, QK_K)) +class Q4_K(__Quant, qtype=GGMLQuantizationType.Q4_K): + K_SCALE_SIZE = 12 + @staticmethod + def get_scale_min(scales): + n_blocks = scales.shape[0] + scales = scales.view(np.uint8) + scales = scales.reshape((n_blocks, 3, 4)) + d, m, m_d = np.split(scales, 3, axis=-2) + sc = np.concatenate([d & 63, m_d & 15 | d >> 2 & 48], axis=-1) + min = np.concatenate([m & 63, m_d >> 4 | m >> 2 & 48], axis=-1) + return sc.reshape((n_blocks, 8)), min.reshape((n_blocks, 8)) + @classmethod + def dequantize_blocks(cls, blocks): + n_blocks = blocks.shape[0] + d, rest = np.hsplit(blocks, [2]) + dmin, rest = np.hsplit(rest, [2]) + scales, qs = np.hsplit(rest, [cls.K_SCALE_SIZE]) + d = d.view(np.float16).astype(np.float32) + dmin = dmin.view(np.float16).astype(np.float32) + sc, m = Q4_K.get_scale_min(scales) + d = (d * sc.astype(np.float32)).reshape((n_blocks, -1, 1)) + dm = (dmin * m.astype(np.float32)).reshape((n_blocks, -1, 1)) + qs = qs.reshape((n_blocks, -1, 1, 32)) >> np.array([0, 4], dtype=np + .uint8).reshape((1, 1, 2, 1)) + qs = (qs & np.uint8(15)).reshape((n_blocks, -1, 32)).astype(np.float32) + return (d * qs - dm).reshape((n_blocks, QK_K)) +class Q5_K(__Quant, qtype=GGMLQuantizationType.Q5_K): + @classmethod + def dequantize_blocks(cls, blocks): + n_blocks = blocks.shape[0] + d, rest = np.hsplit(blocks, [2]) + dmin, rest = np.hsplit(rest, [2]) + scales, rest = np.hsplit(rest, [Q4_K.K_SCALE_SIZE]) + qh, qs = np.hsplit(rest, [QK_K // 8]) + d = d.view(np.float16).astype(np.float32) + dmin = dmin.view(np.float16).astype(np.float32) + sc, m = Q4_K.get_scale_min(scales) + d = (d * sc.astype(np.float32)).reshape((n_blocks, -1, 1)) + dm = (dmin * m.astype(np.float32)).reshape((n_blocks, -1, 1)) + ql = qs.reshape((n_blocks, -1, 1, 32)) >> np.array([0, 4], dtype=np + .uint8).reshape((1, 1, 2, 1)) + qh = qh.reshape((n_blocks, -1, 1, 32)) >> np.array([i for i in + range(8)], dtype=np.uint8).reshape((1, 1, 8, 1)) + ql = (ql & np.uint8(15)).reshape((n_blocks, -1, 32)) + qh = (qh & np.uint8(1)).reshape((n_blocks, -1, 32)) + q = (ql | qh << np.uint8(4)).astype(np.float32) + return (d * q - dm).reshape((n_blocks, QK_K)) +class Q6_K(__Quant, qtype=GGMLQuantizationType.Q6_K): + @classmethod + def dequantize_blocks(cls, blocks): + n_blocks = blocks.shape[0] + ql, rest = np.hsplit(blocks, [QK_K // 2]) + qh, rest = np.hsplit(rest, [QK_K // 4]) + scales, d = np.hsplit(rest, [QK_K // 16]) + scales = scales.view(np.int8).astype(np.float32) + d = d.view(np.float16).astype(np.float32) + d = (d * scales).reshape((n_blocks, QK_K // 16, 1)) + ql = ql.reshape((n_blocks, -1, 1, 64)) >> np.array([0, 4], dtype=np + .uint8).reshape((1, 1, 2, 1)) + ql = (ql & np.uint8(15)).reshape((n_blocks, -1, 32)) + qh = qh.reshape((n_blocks, -1, 1, 32)) >> np.array([0, 2, 4, 6], + dtype=np.uint8).reshape((1, 1, 4, 1)) + qh = (qh & np.uint8(3)).reshape((n_blocks, -1, 32)) + q = (ql | qh << np.uint8(4)).astype(np.int8) - np.int8(32) + q = q.reshape((n_blocks, QK_K // 16, -1)).astype(np.float32) + return (d * q).reshape((n_blocks, QK_K)) +class TQ1_0(__Quant, qtype=GGMLQuantizationType.TQ1_0): + @classmethod + def quantize_blocks(cls, blocks): + n_blocks = blocks.shape[0] + d = abs(blocks).max(axis=-1, keepdims=True) + with np.errstate(divide='ignore'): + id = np.where(d == 0, 0, 1 / d) + qs = np_roundf(blocks * id) + qs = (qs.astype(np.int8) + np.int8(1)).astype(np.uint8) + qs0, qs1, qh = qs[..., :32 * 5], qs[..., 32 * 5:48 * 5], qs[..., 48 * + 5:] + qs0 = qs0.reshape((n_blocks, -1, 5, 32)) * np.array([81, 27, 9, 3, + 1], dtype=np.uint8).reshape((1, 1, 5, 1)) + qs0 = np.sum(qs0, axis=-2).reshape((n_blocks, -1)) + qs1 = qs1.reshape((n_blocks, -1, 5, 16)) * np.array([81, 27, 9, 3, + 1], dtype=np.uint8).reshape((1, 1, 5, 1)) + qs1 = np.sum(qs1, axis=-2).reshape((n_blocks, -1)) + qh = qh.reshape((n_blocks, -1, 4, 4)) * np.array([81, 27, 9, 3], + dtype=np.uint8).reshape((1, 1, 4, 1)) + qh = np.sum(qh, axis=-2).reshape((n_blocks, -1)) + qs = np.concatenate([qs0, qs1, qh], axis=-1) + qs = (qs.astype(np.uint16) * 256 + (243 - 1)) // 243 + qs = qs.astype(np.uint8) + d = d.astype(np.float16).view(np.uint8) + return np.concatenate([qs, d], axis=-1) + @classmethod + def dequantize_blocks(cls, blocks): + n_blocks = blocks.shape[0] + qs, rest = np.hsplit(blocks, [(QK_K - 4 * QK_K // 64) // 5]) + qh, d = np.hsplit(rest, [QK_K // 64]) + d = d.view(np.float16).astype(np.float32) + qs0, qs1 = qs[..., :32], qs[..., 32:] + qs0 = qs0.reshape((n_blocks, -1, 1, 32)) * np.array([1, 3, 9, 27, + 81], dtype=np.uint8).reshape((1, 1, 5, 1)) + qs0 = qs0.reshape((n_blocks, -1)) + qs1 = qs1.reshape((n_blocks, -1, 1, 16)) * np.array([1, 3, 9, 27, + 81], dtype=np.uint8).reshape((1, 1, 5, 1)) + qs1 = qs1.reshape((n_blocks, -1)) + qh = qh.reshape((n_blocks, -1, 1, 4)) * np.array([1, 3, 9, 27], + dtype=np.uint8).reshape((1, 1, 4, 1)) + qh = qh.reshape((n_blocks, -1)) + qs = np.concatenate([qs0, qs1, qh], axis=-1) + qs = (qs.astype(np.uint16) * 3 >> 8).astype(np.int8) - np.int8(1) + return d * qs.astype(np.float32) +class TQ2_0(__Quant, qtype=GGMLQuantizationType.TQ2_0): + @classmethod + def quantize_blocks(cls, blocks): + n_blocks = blocks.shape[0] + d = abs(blocks).max(axis=-1, keepdims=True) + with np.errstate(divide='ignore'): + id = np.where(d == 0, 0, 1 / d) + qs = np_roundf(blocks * id) + qs = (qs.astype(np.int8) + np.int8(1)).astype(np.uint8) + qs = qs.reshape((n_blocks, -1, 4, 32)) << np.array([0, 2, 4, 6], + dtype=np.uint8).reshape((1, 1, 4, 1)) + qs = qs[..., 0, :] | qs[..., 1, :] | qs[..., 2, :] | qs[..., 3, :] + qs = qs.reshape((n_blocks, -1)) + d = d.astype(np.float16).view(np.uint8) + return np.concatenate([qs, d], axis=-1) + @classmethod + def dequantize_blocks(cls, blocks): + n_blocks = blocks.shape[0] + qs, d = np.hsplit(blocks, [QK_K // 4]) + d = d.view(np.float16).astype(np.float32) + qs = qs.reshape((n_blocks, -1, 1, 32)) >> np.array([0, 2, 4, 6], + dtype=np.uint8).reshape((1, 1, 4, 1)) + qs = (qs & 3).reshape((n_blocks, -1)).astype(np.int8) - np.int8(1) + return d * qs.astype(np.float32) +class IQ2_XXS(__Quant, qtype=GGMLQuantizationType.IQ2_XXS): + ksigns: bytes = ( + b'\x00\x81\x82\x03\x84\x05\x06\x87\x88\t\n\x8b\x0c\x8d\x8e\x0f\x90\x11\x12\x93\x14\x95\x96\x17\x18\x99\x9a\x1b\x9c\x1d\x1e\x9f\xa0!"\xa3$\xa5\xa6\'(\xa9\xaa+\xac-.\xaf0\xb1\xb23\xb456\xb7\xb89:\xbb<\xbd\xbe?\xc0AB\xc3D\xc5\xc6GH\xc9\xcaK\xccMN\xcfP\xd1\xd2S\xd4UV\xd7\xd8YZ\xdb\\\xdd\xde_`\xe1\xe2c\xe4ef\xe7\xe8ij\xebl\xed\xeeo\xf0qr\xf3t\xf5\xf6wx\xf9\xfa{\xfc}~\xff' + ) + grid_shape = 256, 8 + grid_map = 8, 25, 43 + grid_hex = ( + b'00000200050008000a00110014002000220028002a00410044005000580061006400800082008a00a20001010401100115014001840198010002020222028202010404041004210424044004420448046004810484049004a404000502050805200546056905800591050906100640068406a406000805080808140828084108440850085208880804094009020a140a01100410101021104010601084109010951000110811201150115a1180112412451200140814201425144914801418156215001616160118041810184018811800190519a019511a002002200a2044206120802082202921482100220222012404241024402456240025412564259026082820289428442a0140044010401840214024404040484056406040814084409040004120416141804185410142104248425642684200440844204480449944124524450046014804481048404845480049584961498249454a904a005008501150195020508050885004514251a4519152905492540a550156545600581158195864584059085a04601060406068600061556118626062006405641065126584654268008002800a8041808280048118814081118201840484108415844084608400854685948509864086608602880489118a0490109024904090a19016918091459200942294449451958198209902a050a085a009a100a218a450a804a9' + ) + @classmethod + def dequantize_blocks(cls, blocks): + n_blocks = blocks.shape[0] + d, qs = np.hsplit(blocks, [2]) + d = d.view(np.float16).astype(np.float32) + qs = qs.view(np.uint32).reshape(n_blocks, -1, 2) + db = d * (np.float32(0.5) + (qs[..., 1] >> 28).astype(np.float32) + ) * np.float32(0.25) + db = db.reshape((n_blocks, -1, 1, 1)) + signs = qs[..., 1].reshape((n_blocks, -1, 1)) >> np.array([0, 7, 14, + 21], dtype=np.uint32).reshape((1, 1, 4)) + ksigns = np.frombuffer(cls.ksigns, dtype=np.uint8).reshape((1, 1, 1, + 128)) + signs = (signs & np.uint32(127)).reshape((n_blocks, -1, 4, 1)) + signs = np.take_along_axis(ksigns, signs, axis=-1) + signs = signs.reshape((n_blocks, -1, 4, 1)) >> np.array([i for i in + range(8)], dtype=np.uint8).reshape((1, 1, 1, 8)) + signs = signs & np.uint8(1) + signs = np.where(signs == 0, np.float32(1), np.float32(-1)) + signs = signs.reshape((n_blocks, -1, 4, 8)) + assert cls.grid is not None + grid = np.take_along_axis(cls.grid, qs[..., 0].copy().view(np.uint8 + ).reshape((n_blocks, -1, 1, 1)), axis=-2) + grid = grid.reshape((n_blocks, -1, 4, 8)) + return (db * grid * signs).reshape((n_blocks, -1)) +class IQ2_XS(__Quant, qtype=GGMLQuantizationType.IQ2_XS): + grid_shape = 512, 8 + grid_map = 8, 25, 43 + grid_hex = ( + b'00000200050008000a001100140016001900200022002500280041004400460049005000520055005800610064008000820085008800910094009900a000010104010601090110011201150118011a0121012401400142014501480151015401600168018101840190010002020205020802110214022002410244025002550280028a02010404040604090410041204150418042104240440044204450448045104540456046004810484049004000502050505080511051405200541054405500561058005010604061006260640064206840600080208050808080a08110814082008250841084408500858088008a008aa08010904091009400981098909000a200a280a960aa00a011004100610091010101210151018102110241040104210451048105110541060106a108110841090100011021105110811111114112011411144115011801194119611011204120612101240126012001402140514081411141414201441144414491450146414801401150415101540150016141649160118041810181218401854188618001905196619511aa91a00200220052008200a201120142020204120442050208020a020012104211021402148216521002222228022a82201240424102429244024002541255225992501261a26a626002808280a28202855288828a22868299029082a202a822a882a8a2a014004400640094010401240154018402140244040404240454048404a405140544060406540814084409040004102410541084111411441204141414441504180418541a2410142044210421242294240420044024405440844114414441944204441444444504480449444014504451045244540459a4500460a4644465046014804481048404845485448624800491149444950496949044a00500250055008501150145020502850415044505050805001510451105115514051425100524452aa520154045410542154405460548154a154005508558055885521566856a156005814584158505899581a5940594259855a0160046010604060546062608660a960006124624a62926200641664106540654565a46501686a682569066a546a626a00800280058008801180148020802a8041804480508080808280a880aa8001810481068110814081518159810082208280828282a082a8820184048410841284158440846084898400854485a58518866a860088088825885a8880888288a8880689228a808a888a968aa88a019004901090409056908490009122916491569289920094059444945094589429959095929541965198a6984999159a609a00a002a008a00aa020a02aa0a0a051a159a1a6a100a202a208a22aa280a2a0a240a495a465a698a60aa820a822a828a8a0a8a8a804a984a986a928aa2aaa91aaaaaa' + ) + @classmethod + def dequantize_blocks(cls, blocks): + n_blocks = blocks.shape[0] + d, rest = np.hsplit(blocks, [2]) + qs, scales = np.hsplit(rest, [2 * QK_K // 8]) + d = d.view(np.float16).astype(np.float32) + qs = qs.view(np.uint16) + scales = scales.reshape((n_blocks, -1, 1)) >> np.array([0, 4], + dtype=np.uint8).reshape((1, 1, 2)) + scales = (scales & 15).reshape((n_blocks, -1)) + db = d * (np.float32(0.5) + scales) * np.float32(0.25) + db = db.reshape((n_blocks, -1, 1, 1)) + signs = np.frombuffer(IQ2_XXS.ksigns, dtype=np.uint8).reshape(1, 1, 128 + ) + signs = np.take_along_axis(signs, (qs >> 9).reshape((n_blocks, -1, + 1)), axis=-1) + signs = signs.reshape((n_blocks, -1, 1)) >> np.array([i for i in + range(8)], dtype=np.uint8).reshape((1, 1, 8)) + signs = signs & np.uint8(1) + signs = np.where(signs == 0, np.float32(1), np.float32(-1)) + signs = signs.reshape((n_blocks, -1, 2, 8)) + assert cls.grid is not None + grid = np.take_along_axis(cls.grid, (qs & np.uint16(511)).reshape(( + n_blocks, -1, 1, 1)), axis=-2) + grid = grid.reshape((n_blocks, -1, 2, 8)) + return (db * grid * signs).reshape((n_blocks, -1)) +class IQ2_S(__Quant, qtype=GGMLQuantizationType.IQ2_S): + grid_shape = 1024, 8 + grid_map = 8, 25, 43 + grid_hex = ( + b'00000200050008000a0011001400160019002000220025002800410044004600490050005200550058006100640066006900800082008500880091009400a000a500aa0001010401060109011001120115011801210124014001420145014801510154015601590160016501680181018401900192019501a101a40100020202050208021102140220022a02410244024602490250025502800285028a029402a20201040404060409041004120415041804210424042604290440044204450448044a0451045404560459046004620465048104840486048904900495049804a104a40400050205050508050a0511051405160519052005250528054105440546054905500552055505580561056405800582058505880591059405a00501060406060609061006150640064506480651065406600681068406900600080208050808081108140816081908200825082a084108440846084908500852085508580861086408800885089408aa08010904091009120915091809210940094509480951095409600981099009000a110a140a220a280a2a0a500a990a01100410061009101010121015101810211024102610401042104510481051105410561059106010621065106810811084108610901095109810a110a41000110211051108110a1111111411161119112011221125112811411144114611491150115211551158116111641180118211851188119111941101120412091210121512211224124012451251125412811284129012001402140514081411141414161419142014251428144114441446144914501452145514581461146414801482148514881491149414a014011504150615091510151215151518152115241540154215451548155115541560158115841590150016051608161116141620164116441650168016aa160118041806180918101815181818211840184218451848185118541860188118841800190219051908191119141920194119441950196919a219041a101a401a561a00200220052008201120142016201920202025202a2041204420502052205520642080208a209420aa2001210421102112211521212140214221452151215421602181218421902100220a22222228222a224422502288228a22a8220124042406240924102415241824212424244024422445244824512454246024812484249024002505250825112514252025412544255025662580250126042610264026592600280528112814284128442850288a28aa2801290429102995290a2a222a642a882a8a2a014004400640094010401240154018401a4021402440264040404240454048404a40514054405640594060406240654081408440904095409840a140a440004102410541084111411441164119412041224125414141444146414941504152415541584161416441804182418541884191419441a04101420442104212421542184224424042454248425142544260428142844200440244054408440a44114414441644194420442244254428444144444446444944504452445544584461446444804482448544884491449444a044014504450645094510451245154518452145244540454245454548455145544560456a4581458445904500460246054608461146144620464146444650468046a546014804480948104812481548184821482448404842484548484851485448604884489048004902490549084911491449204941494449504980499649014a044a104a404a0050025005500850115014501650195020502250255028504150445046504950505052505550585061506450805082508550885091509450015104510651095110511251155118512151245140514251455148515151545160518151845190510052055208521152145220524152445250526952805201540454065409541054125415541854215424544054425445544854515454546054815484549054005502550555085511551455205541554455505580550156045610562656405600580258055808581158145820584158445850585a5880580159045910594059005a195a855aa85a016004600660106012601560186021602460406045604860516054606060846090600061026105610861116114612061416144615061806199610462106240625662a162006405640864116414642064416444645064806401650465106540654a6568659265006694660168046810686568986800692a69426aa16a00800280058008801180148019802080258041804480508052805580588061808080858091809480018104810981108112811581188121812481408142814581488151815481818184819081a981008205820a82118214824182448250820184048406840984108412841584188421844084428445844884518454846084818484849084008502850585088511851485208541854485508580858a85018604861086298640860088058811881488418844885088a2880189048940896589228a588a5a8a828aa28a01900490099010901290159018902490409042904590489051905490609081908490909000910591119114914191449150915a910192049210924092a69200940294059408941194149420944194449450948094969401950495109540959895a19500964696649601980498109826984098a998009949995299909a00a005a00aa014a022a02aa041a044a050a0a2a0aaa040a165a102a20aa222a228a22aa282a288a28aa2a8a201a404a410a440a489a4a4a400a519a551a60aa828a8a2a854a986a908aa0aaa20aa22aa28aa88aaaaaa' + ) + @classmethod + def dequantize_blocks(cls, blocks): + n_blocks = blocks.shape[0] + d, rest = np.hsplit(blocks, [2]) + qs, rest = np.hsplit(rest, [QK_K // 8]) + signs, rest = np.hsplit(rest, [QK_K // 8]) + qh, scales = np.hsplit(rest, [QK_K // 32]) + d = d.view(np.float16).astype(np.float32) + scales = scales.reshape((n_blocks, -1, 1)) >> np.array([0, 4], + dtype=np.uint8).reshape((1, 1, 2)) + scales = (scales & 15).reshape((n_blocks, -1)) + db = d * (np.float32(0.5) + scales) * np.float32(0.25) + db = db.reshape((n_blocks, -1, 1, 1)) + signs = signs.reshape((n_blocks, -1, 1)) >> np.array([i for i in + range(8)], dtype=np.uint8).reshape((1, 1, 8)) + signs = signs & np.uint8(1) + signs = np.where(signs == 0, np.float32(1), np.float32(-1)) + signs = signs.reshape((n_blocks, -1, 2, 8)) + qh = qh.reshape((n_blocks, -1, 1)) >> np.array([0, 2, 4, 6], dtype= + np.uint8).reshape((1, 1, 4)) + qs = qs.astype(np.uint16) | ((qh & 3).astype(np.uint16) << 8).reshape(( + n_blocks, -1)) + assert cls.grid is not None + grid = np.take_along_axis(cls.grid, qs.reshape((n_blocks, -1, 1, 1) + ), axis=-2) + grid = grid.reshape((n_blocks, -1, 2, 8)) + return (db * grid * signs).reshape((n_blocks, -1)) +class IQ3_XXS(__Quant, qtype=GGMLQuantizationType.IQ3_XXS): + grid_shape = 256, 4 + grid_map = 4, 12, 20, 28, 36, 44, 52, 62 + grid_hex = ( + b'0000020004001100130017002000220031004200730075000101030110011201210125013001320141015401700100020202040211022002220231023302370251025702750201030703100312032503700313043704440457047304750401050705320552053506640610071407160743076107011003101010121021102310301032103410471050100011021111112011221101120312101212122112301272120013021320133113461366130114051450142015241546157115051622174017002002201120132020202220262031204220012103210521102112212121302163216721702100220222112217222022222237224022552201231023142370237423352453240325272541257425012703271627452701301030123021302330503065307230003102312031313144314631013203321032253252327232113333333034473472340035063522355535143636366336333760370440174035403740534057407441204237424042604266420743454304445144644425454345704505471047124730471250415070500051065126515551145232527252025353531054235427547254025531555056245742572460446046606460216161611762646230633663446405655265336603672167037005700770107032705270267140711272457252720073157333736073217441740075027524753076' + ) + @classmethod + def dequantize_blocks(cls, blocks): + n_blocks = blocks.shape[0] + d, rest = np.hsplit(blocks, [2]) + qs, scales = np.hsplit(rest, [QK_K // 4]) + d = d.view(np.float16).astype(np.float32) + scales = scales.view(np.uint32) + db = d * (np.float32(0.5) + (scales >> 28).astype(np.float32) + ) * np.float32(0.5) + db = db.reshape((n_blocks, -1, 1, 1)) + signs = scales.reshape((n_blocks, -1, 1)) >> np.array([0, 7, 14, 21 + ], dtype=np.uint32).reshape((1, 1, 4)) + ksigns = np.frombuffer(IQ2_XXS.ksigns, dtype=np.uint8).reshape((1, + 1, 1, 128)) + signs = (signs & np.uint32(127)).reshape((n_blocks, -1, 4, 1)) + signs = np.take_along_axis(ksigns, signs, axis=-1) + signs = signs.reshape((n_blocks, -1, 4, 1)) >> np.array([i for i in + range(8)], dtype=np.uint8).reshape((1, 1, 1, 8)) + signs = signs & np.uint8(1) + signs = np.where(signs == 0, np.float32(1), np.float32(-1)) + signs = signs.reshape((n_blocks, -1, 4, 8)) + assert cls.grid is not None + grid = np.take_along_axis(cls.grid, qs.reshape((n_blocks, -1, 1, 1) + ), axis=-2) + grid = grid.reshape((n_blocks, -1, 4, 8)) + return (db * grid * signs).reshape((n_blocks, -1)) +class IQ3_S(__Quant, qtype=GGMLQuantizationType.IQ3_S): + grid_shape = 512, 4 + grid_map = 1, 3, 5, 7, 9, 11, 13, 15 + grid_hex = ( + b'00000100020005000700100011001200140016002000210025003300400042004500470051005300600062007100740077000001010102010401100111011501200123012701310135014401610165017201000201020502070210021302160221022502300234024202450247025102530270027302030311031503200322033103330336034403500352036703710375030004130417042104240432044004430451047004020504052005220526053305410545054705660573050606110613063106520671060007020704072007220726073307500754070010011002100410101011101310151017102010221031103410361054105610611072100011011103110611101114112111301133114111501152117011761100121212151217122012241232124012431255126012721201130413071310131313211327133013341341136213701303140514121414143114331442144614501454140115101513152115301532155115201624162716441646160117031710171217211735174117621770170020012003200520072010201220142016202120232027203020322041204320452050205220672070207320752000210221102113211721222125213121342142215121012204220722212223223022372241225322572271227422002302230523112322232423312333234223502366230124072420242324322435244124722475240425112522253725402553257025002602260726212655266126052711272627302743275027023011301330153017302230313033303530423044304730513063307130013103310531143121312331403160317231763100321232203232323432503201331033143321332333273330334133433347335533733303341134163422343134523460346434013510351235253532354435563573351636413601370337203722373537004004401240204024402740324041405040704002410741114113412241304135414341514155410142034210421542214233424042574262427042044311431343204322433143354300440244244437444044714405450745214562451346344660461047154730474347514702501050145022504050445047505250665074500151035105511251215132517251005211522352305236525352025307531053275344535153655373530154045420543254465412552655515553554256025704572257116013601560316033606060006120612761646112623462426255626262706200631463216340632564436462640065036534656065056640661167136700700470077020702270367040705470627002711171247143714571017204721072167221723072517202733273357353730174057413742074507422754275027631760077' + ) + @classmethod + def dequantize_blocks(cls, blocks): + n_blocks = blocks.shape[0] + d, rest = np.hsplit(blocks, [2]) + qs, rest = np.hsplit(rest, [QK_K // 4]) + qh, rest = np.hsplit(rest, [QK_K // 32]) + signs, scales = np.hsplit(rest, [QK_K // 8]) + d = d.view(np.float16).astype(np.float32) + scales = scales.reshape((n_blocks, -1, 1)) >> np.array([0, 4], + dtype=np.uint8).reshape((1, 1, 2)) + scales = (scales & 15).reshape((n_blocks, -1)) + db = d * (1 + 2 * scales) + db = db.reshape((n_blocks, -1, 1, 1)) + signs = signs.reshape((n_blocks, -1, 1)) >> np.array([i for i in + range(8)], dtype=np.uint8).reshape((1, 1, 8)) + signs = signs & np.uint8(1) + signs = np.where(signs == 0, np.float32(1), np.float32(-1)) + signs = signs.reshape((n_blocks, -1, 4, 8)) + qh = qh.reshape((n_blocks, -1, 1)) >> np.array([i for i in range(8) + ], dtype=np.uint8) + qh = (qh & 1).astype(np.uint16).reshape((n_blocks, -1)) + qs = qs.astype(np.uint16) | qh << 8 + assert cls.grid is not None + grid = np.take_along_axis(cls.grid, qs.reshape((n_blocks, -1, 1, 1) + ), axis=-2) + grid = grid.reshape((n_blocks, -1, 4, 8)) + return (db * grid * signs).reshape((n_blocks, -1)) +class IQ1_S(__Quant, qtype=GGMLQuantizationType.IQ1_S): + grid_shape = 2048, 8 + grid_map = -1, 0, 1 + grid_hex = ( + b'00000200050008000a00110015002000220028002a00450051005400560065008000820088008a009500a000a200a800aa000401050111011401160119011a012501410146014901520155015a0161016401660168018501910194019601a5010002020208020a0215022002220228022a02450251025902640269028002820288028a02910295029902a002a202a802aa0211041404160425044104490455045a046404650491049904a5040105040505050605150518051a052905400545054a0550055105540555055605590560056205650568056a0581059105950598059a05a105a405a505a605a90514061906410644065006520655065806600661066606690685069106940699060008020808080a0815082008220828082a0845085108560865088008820888088a089508a008a208a808aa0805091109140919092409250941095009510955096109640969099109940996099909a509000a020a080a0a0a150a200a220a280a2a0a450a510a590a610a650a800a820a850a880a8a0a950aa00aa20aa80aaa0a101011101410191024102510411044105010551058106110641065106910911094109610a110a5100111041106110911101112111511181121112411291145114a11501151115211541155115611591160116511841192119511a111a41111121412161225124012461249125212551258125a12641266128512911294129612a51201140614091414141514181419142114261441144514461448144a145114541455145614591462146514681484148914901494149514981499149a14a114a414a514a914021505150a151115141515151615191520152215251528152a1541154415451546155115521554155515561559155a1561156415651566156915801582158415851588158a159015911594159515961599159a15a015a215a51501160416051606161516161618161a1621162616401642164416451648164a1651165516561658165916611664166516681669166a1686168a1692169516a416a91611181618251841184418461849185018551858185a1860186118641866186918851891189418a5181019121915191a19211925194219441945194819511954195519561959195a19601965196a1989199119921995199819a119a619a919091a161a241a261a441a461a491a501a521a551a581a611a661a691a851a911a961a9a1a0020022008200a20152020202220252028202a20452051205920612065208020822088208a209520a020a220a520a820aa2005211121142119212521422144214921552158215a2161216421652166218521902196219921a521012208220a22112215222022222228222a2245225122562259226522812288228a2291229522a022a222a822aa220524142416241924252444244524462449245224552458245a2466248524912494249924a124a52409251525212529254025452548255125542555255925622565256825892590259425952598259a25a125a425a625a925052610261226192625264126492655266026612669268426862690269a260028022808280a2815282028222828282a2845285128542865288028822888288a28a028a228a828aa2809291129142919292529462949295229552961296429662969298529902996299929a429a529002a022a082a0a2a202a222a282a2a2a452a512a562a592a652a802a822a882a8a2a952aa02aa22aa82aaa2a054011401640254049405240554058405a4061406440664094409940a140a6400041014104410641094112411541164118411a41214126412941454148414a41514154415541564159415a41654168416a41814184418641904192419541a041a141a241054211421442164225424142524255425a426442694289429442a5420144154419442944454448444a44514454445544564461446244654468446a44814486448944904492449544a044a144a9440145024505450a4511451445154516451945204525452a45414544454545464549455045514554455545564558455945614564456545664569458245844585458845914594459545964599459a45a545a845aa450146054609461446154618461a462146244629464046424645464846504651465246554656465946624665466846814685468a4694469546a146a446a6460548114815481a48254842484948504855485848614864486648694885489148944896489948a5480149054906490a491049144915491849214924492649404945494a4951495249544955495649594960496249654966496a49864989499249954996499849a149a449a649a949164a444a464a494a554a584a5a4a644a694a944aa54a01500450055006500950125015501a5021502450295040504550485051505450555056505950655068508650895095509850a050a150a650a9500551085109510a5111511451155116511851195120512551265128512a5141514451455146514951505151515251545155515651585159515a51615164516551665169518251855191519451955196519951a051a551aa5101520652125215521a5221522452425245524a525152545255525652595262526552855290529252955299529a52a452045405541154145415541654185419542154255428542a54415444544554465449544a5450545154545455545654585459545a54615462546454655466546954805488548a5491549454955496549954a154a454a554aa5401550255045505550655095510551155125514551555165519551a552155245525552655295540554155425544554555465548554955505551555255545555555655585559555a5560556155645565556655685569556a5581558455855589558a559055915594559555965598559955a155a455a555a655a95500560156025604560656085609561156145615561856195620562156225624562556265628562956415645564656485649564a56505651565256545655565656585659565a566156645665566956825685568656885689568a56915695569a56a256a556a656a856a956045805580658095810581558185821582a58455848584a585158545855585658585859586058625864586558825889589058925895589858a158a9580159025905590a5911591459155916591959255941594459455946594959505951595259545955595659585959595a5961596459655966596959815985598959915994599559965998599959a559045a085a155a1a5a205a255a265a295a455a485a495a515a555a565a585a595a625a655a685a6a5a815a8a5a925a955a965a985a9a5aa15a05601460166019602560446050605560566058605a60616064606660696081609660a5600161046106610961126115612161226126612961456149615161556156615961656166616a6184618a6192619561a161a661a9611162166219624062416246625562566258626062856291629662a56211641264156416641a6421642664296440644264456448644a64516454645564566459645a646064626465648464856489649064926494649564966498649a64a164a464a964056508650a651165156516651965446545654665496550655165546555655665596561656465656566656965866589658a6591659565966599659a65a265a565a665a86502660966156620662666286629664066456648664a66516654665566566658665a666066656668668066826685668a669466966698669966a066a466a666aa661668196825684168526855685a6861686968856891689868a668016904691069156921692469266929694069416945694669486951695469556956695969606965696a69826984698a699569a169a469a569a969116a166a186a416a446a496a506a556a586a5a6a646a656a696a866a946a986a9a6aa66a0080028008800a802080228028802a8045805080518054805680598065808080828088808a809580a080a280a880aa8005811181148116811981258141814481498150815281558156815881598164816681698185818981948196819981a5810082028208820a8215822082228228822a8251825482598265828082828288828a829582a082a282a882aa821484198441844484518455845a846184648469849484998401850985128515851a85268529854085418545854885518554855585568559855a856585668568856a8581858485868589859085928595859885a68511861686198625864186448649864a865086558659865a86618666866a86858691869a86a4860088028808880a8815882088228828882a8841884588518854885988658869888088828888888a889588a088a288a888aa8805890689118914891689258941894489468949895089528955895a8961896489858996899989a589008a028a088a0a8a158a208a228a288a2a8a458a518a548a568a808a828a888a8a8a958aa08aa28aa88aaa8a059011901690189019902590419046904990559058905a9069906a9085909190949096909990a59001910491069109911091159118911a912191249126912991409145915091519154915591569159916291659184918691929195919891a191a491a691a99105921192149219922592449246924992509252925592589266926992859294929692a992019404940694109415941894269440944a945194549455945694589459946094619462946594849486949294949495949894a194a9940095059508950a951095119514951595169519952195259529952a9541954495459546954995509551955295549555955695589559955a95619564956595669569958195859588959195929594959595969599959a95a095a295a595a895aa950196049610961596199620962696299645964896499651965296559656965996659668968296849689968a96929694969596a496a696a99605981698199825984198469850985298559856985a98649865988598919896989998a59804990699099910991299159918991a99209921992499269940994299459948994a99519954995599569959996299659966996a99819984999099929995999a99a199a699059a159a259a449a469a499a509a559a589a619a859a919a949a959a969a00a002a008a00aa015a020a022a028a02aa045a051a054a056a059a080a082a088a08aa095a0a0a0a2a0a8a0aaa005a109a111a114a116a119a11aa146a149a151a155a158a15aa161a164a185a190a192a196a199a102a208a20aa210a219a222a228a22aa245a251a256a259a265a280a282a288a28aa295a2a0a2a2a2a8a2aaa219a425a441a444a450a454a455a458a45aa461a465a466a468a469a485a406a509a510a512a515a518a526a529a542a545a551a554a555a556a559a565a56aa581a584a585a586a589a592a595a598a505a611a616a61aa621a625a644a646a64aa652a655a656a658a660a662a686a690a695a696a699a6a1a6a4a6a6a600a802a808a80aa820a822a828a82aa851a854a856a859a880a882a888a88aa895a8a0a8a2a8a8a8aaa805a914a919a921a925a941a950a955a95aa961a966a969a990a996a900aa02aa08aa0aaa20aa22aa28aa2aaa51aa54aa56aa80aa82aa88aa8aaa95aaa0aaa2aaa8aaaaaa' + ) + delta = np.float32(0.125) + @classmethod + def dequantize_blocks(cls, blocks): + n_blocks = blocks.shape[0] + d, rest = np.hsplit(blocks, [2]) + qs, qh = np.hsplit(rest, [QK_K // 8]) + d = d.view(np.float16).astype(np.float32) + qh = qh.view(np.uint16) + dl = d * (2 * (qh >> 12 & 7) + 1) + dl = dl.reshape((n_blocks, -1, 1, 1)) + delta = np.where(qh & np.uint16(32768) == 0, cls.delta, -cls.delta) + delta = delta.reshape((n_blocks, -1, 1, 1)) + qh = qh.reshape((n_blocks, -1, 1)) >> np.array([0, 3, 6, 9], dtype= + np.uint16).reshape((1, 1, 4)) + qs = qs.astype(np.uint16) | ((qh & 7) << 8).reshape((n_blocks, -1)) + assert cls.grid is not None + grid = np.take_along_axis(cls.grid, qs.reshape((n_blocks, -1, 1, 1) + ), axis=-2) + grid = grid.reshape((n_blocks, -1, 4, 8)) + return (dl * (grid + delta)).reshape((n_blocks, -1)) +class IQ1_M(__Quant, qtype=GGMLQuantizationType.IQ1_M): + grid_shape = IQ1_S.grid_shape + grid_map = IQ1_S.grid_map + grid_hex = IQ1_S.grid_hex + delta = IQ1_S.delta + @classmethod + def dequantize_blocks(cls, blocks): + n_blocks = blocks.shape[0] + qs, rest = np.hsplit(blocks, [QK_K // 8]) + qh, scales = np.hsplit(rest, [QK_K // 16]) + scales = scales.view(np.uint16) + d = (scales.reshape((n_blocks, 4)) & np.uint16(61440)) >> np.array([ + 12, 8, 4, 0], dtype=np.uint16).reshape((1, 4)) + d = d[..., 0] | d[..., 1] | d[..., 2] | d[..., 3] + d = d.view(np.float16).astype(np.float32).reshape((n_blocks, 1)) + scales = scales.reshape(n_blocks, -1, 1) >> np.array([0, 3, 6, 9], + dtype=np.uint16).reshape((1, 1, 4)) + scales = (scales & 7).reshape((n_blocks, -1)) + dl = d * (2 * scales + 1) + dl = dl.reshape((n_blocks, -1, 2, 1, 1)) + qh = qh.reshape((n_blocks, -1, 1)) >> np.array([0, 4], dtype=np.uint8 + ).reshape((1, 1, 2)) + qs = qs.astype(np.uint16) | ((qh & 7).astype(np.uint16) << 8).reshape(( + n_blocks, -1)) + delta = np.where(qh & 8 == 0, cls.delta, -cls.delta) + delta = delta.reshape((n_blocks, -1, 2, 2, 1)) + assert cls.grid is not None + grid = np.take_along_axis(cls.grid, qs.reshape((n_blocks, -1, 1, 1) + ), axis=-2) + grid = grid.reshape((n_blocks, -1, 2, 2, 8)) + return (dl * (grid + delta)).reshape((n_blocks, -1)) +class IQ4_NL(__Quant, qtype=GGMLQuantizationType.IQ4_NL): + kvalues = (-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, + 69, 89, 113) + @classmethod + def dequantize_blocks(cls, blocks): + n_blocks = blocks.shape[0] + d, qs = np.hsplit(blocks, [2]) + d = d.view(np.float16).astype(np.float32) + qs = qs.reshape((n_blocks, -1, 1, cls.block_size // 2)) >> np.array([ + 0, 4], dtype=np.uint8).reshape((1, 1, 2, 1)) + qs = (qs & np.uint8(15)).reshape((n_blocks, -1, 1)) + kvalues = np.array(cls.kvalues, dtype=np.int8).reshape(1, 1, 16) + qs = np.take_along_axis(kvalues, qs, axis=-1).astype(np.float32 + ).reshape((n_blocks, -1)) + return d * qs +class IQ4_XS(__Quant, qtype=GGMLQuantizationType.IQ4_XS): + @classmethod + def dequantize_blocks(cls, blocks): + n_blocks = blocks.shape[0] + d, rest = np.hsplit(blocks, [2]) + scales_h, rest = np.hsplit(rest, [2]) + scales_l, qs = np.hsplit(rest, [QK_K // 64]) + d = d.view(np.float16).astype(np.float32) + scales_h = scales_h.view(np.uint16) + scales_l = scales_l.reshape((n_blocks, -1, 1)) >> np.array([0, 4], + dtype=np.uint8).reshape((1, 1, 2)) + scales_h = scales_h.reshape((n_blocks, 1, -1)) >> np.array([(2 * i) for + i in range(QK_K // 32)], dtype=np.uint16).reshape((1, -1, 1)) + scales_l = scales_l.reshape((n_blocks, -1)) & np.uint8(15) + scales_h = scales_h.reshape((n_blocks, -1)).astype(np.uint8 + ) & np.uint8(3) + scales = (scales_l | scales_h << np.uint8(4)).astype(np.int8 + ) - np.int8(32) + dl = (d * scales.astype(np.float32)).reshape((n_blocks, -1, 1)) + qs = qs.reshape((n_blocks, -1, 1, 16)) >> np.array([0, 4], dtype=np + .uint8).reshape((1, 1, 2, 1)) + qs = qs.reshape((n_blocks, -1, 32, 1)) & np.uint8(15) + kvalues = np.array(IQ4_NL.kvalues, dtype=np.int8).reshape((1, 1, 1, -1) + ) + qs = np.take_along_axis(kvalues, qs, axis=-1).astype(np.float32 + ).reshape((n_blocks, -1, 32)) + return (dl * qs).reshape((n_blocks, -1)) \ No newline at end of file diff --git a/gguf_connector/quant2.py b/gguf_connector/quant2.py new file mode 100644 index 0000000..726722f --- /dev/null +++ b/gguf_connector/quant2.py @@ -0,0 +1,191 @@ +import torch +from .quant import dequantize as gq +from .reader import GGMLQuantizationType, GGML_QUANT_SIZES +from tqdm import tqdm +TORCH_COMPATIBLE_QTYPES = {None, GGMLQuantizationType.F32, + GGMLQuantizationType.F16} +def is_torch_compatible(tensor): + return tensor is None or getattr(tensor, 'tensor_type', None + ) in TORCH_COMPATIBLE_QTYPES +def is_quantized(tensor): + return not is_torch_compatible(tensor) +def dequantize_tensor(tensor, dtype=None, dequant_dtype=None): + qtype = getattr(tensor, 'tensor_type', None) + oshape = getattr(tensor, 'tensor_shape', tensor.shape) + if qtype in TORCH_COMPATIBLE_QTYPES: + return tensor.to(dtype) + elif qtype in dequantize_functions: + dequant_dtype = dtype if dequant_dtype == 'target' else dequant_dtype + return dequantize(tensor.data, qtype, oshape, dtype=dequant_dtype).to( + dtype) + else: + tqdm.write(f'Pushing back to numpy dequant for qtype: {qtype}') + new = gq(tensor.cpu().numpy(), qtype) + return torch.from_numpy(new).to(tensor.device, dtype=dtype) +def dequantize(data, qtype, oshape, dtype=None): + block_size, type_size = GGML_QUANT_SIZES[qtype] + dequantize_blocks = dequantize_functions[qtype] + rows = data.reshape((-1, data.shape[-1])).view(torch.uint8) + n_blocks = rows.numel() // type_size + blocks = rows.reshape((n_blocks, type_size)) + blocks = dequantize_blocks(blocks, block_size, type_size, dtype) + return blocks.reshape(oshape) +def to_uint32(x): + x = x.view(torch.uint8).to(torch.int32) + return (x[:, 0] | x[:, 1] << 8 | x[:, 2] << 16 | x[:, 3] << 24).unsqueeze(1 + ) +def split_block_dims(blocks, *args): + n_max = blocks.shape[1] + dims = list(args) + [n_max - sum(args)] + return torch.split(blocks, dims, dim=1) +def dequantize_blocks_BF16(blocks, block_size, type_size, dtype=None): + return (blocks.view(torch.int16).to(torch.int32) << 16).view(torch.float32) +def dequantize_blocks_Q8_0(blocks, block_size, type_size, dtype=None): + d, x = split_block_dims(blocks, 2) + d = d.view(torch.float16).to(dtype) + x = x.view(torch.int8) + return d * x +def dequantize_blocks_Q5_1(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d, m, qh, qs = split_block_dims(blocks, 2, 2, 4) + d = d.view(torch.float16).to(dtype) + m = m.view(torch.float16).to(dtype) + qh = to_uint32(qh) + qh = qh.reshape((n_blocks, 1)) >> torch.arange(32, device=d.device, + dtype=torch.int32).reshape(1, 32) + ql = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0, + 4], device=d.device, dtype=torch.uint8).reshape(1, 1, 2, 1) + qh = (qh & 1).to(torch.uint8) + ql = (ql & 15).reshape((n_blocks, -1)) + qs = ql | qh << 4 + return d * qs + m +def dequantize_blocks_Q5_0(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d, qh, qs = split_block_dims(blocks, 2, 4) + d = d.view(torch.float16).to(dtype) + qh = to_uint32(qh) + qh = qh.reshape(n_blocks, 1) >> torch.arange(32, device=d.device, dtype + =torch.int32).reshape(1, 32) + ql = qs.reshape(n_blocks, -1, 1, block_size // 2) >> torch.tensor([0, 4 + ], device=d.device, dtype=torch.uint8).reshape(1, 1, 2, 1) + qh = (qh & 1).to(torch.uint8) + ql = (ql & 15).reshape(n_blocks, -1) + qs = (ql | qh << 4).to(torch.int8) - 16 + return d * qs +def dequantize_blocks_Q4_1(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d, m, qs = split_block_dims(blocks, 2, 2) + d = d.view(torch.float16).to(dtype) + m = m.view(torch.float16).to(dtype) + qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0, + 4], device=d.device, dtype=torch.uint8).reshape(1, 1, 2, 1) + qs = (qs & 15).reshape(n_blocks, -1) + return d * qs + m +def dequantize_blocks_Q4_0(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d, qs = split_block_dims(blocks, 2) + d = d.view(torch.float16).to(dtype) + qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0, + 4], device=d.device, dtype=torch.uint8).reshape((1, 1, 2, 1)) + qs = (qs & 15).reshape((n_blocks, -1)).to(torch.int8) - 8 + return d * qs +QK_K = 256 +K_SCALE_SIZE = 12 +def get_scale_min(scales): + n_blocks = scales.shape[0] + scales = scales.view(torch.uint8) + scales = scales.reshape((n_blocks, 3, 4)) + d, m, m_d = torch.split(scales, scales.shape[-2] // 3, dim=-2) + sc = torch.cat([d & 63, m_d & 15 | d >> 2 & 48], dim=-1) + min = torch.cat([m & 63, m_d >> 4 | m >> 2 & 48], dim=-1) + return sc.reshape((n_blocks, 8)), min.reshape((n_blocks, 8)) +def dequantize_blocks_Q6_K(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + ql, qh, scales, d = split_block_dims(blocks, QK_K // 2, QK_K // 4, QK_K // + 16) + scales = scales.view(torch.int8).to(dtype) + d = d.view(torch.float16).to(dtype) + d = (d * scales).reshape((n_blocks, QK_K // 16, 1)) + ql = ql.reshape((n_blocks, -1, 1, 64)) >> torch.tensor([0, 4], device=d + .device, dtype=torch.uint8).reshape((1, 1, 2, 1)) + ql = (ql & 15).reshape((n_blocks, -1, 32)) + qh = qh.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 2, 4, 6], + device=d.device, dtype=torch.uint8).reshape((1, 1, 4, 1)) + qh = (qh & 3).reshape((n_blocks, -1, 32)) + q = (ql | qh << 4).to(torch.int8) - 32 + q = q.reshape((n_blocks, QK_K // 16, -1)) + return (d * q).reshape((n_blocks, QK_K)) +def dequantize_blocks_Q5_K(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d, dmin, scales, qh, qs = split_block_dims(blocks, 2, 2, K_SCALE_SIZE, + QK_K // 8) + d = d.view(torch.float16).to(dtype) + dmin = dmin.view(torch.float16).to(dtype) + sc, m = get_scale_min(scales) + d = (d * sc).reshape((n_blocks, -1, 1)) + dm = (dmin * m).reshape((n_blocks, -1, 1)) + ql = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 4], device=d + .device, dtype=torch.uint8).reshape((1, 1, 2, 1)) + qh = qh.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([i for i in + range(8)], device=d.device, dtype=torch.uint8).reshape((1, 1, 8, 1)) + ql = (ql & 15).reshape((n_blocks, -1, 32)) + qh = (qh & 1).reshape((n_blocks, -1, 32)) + q = ql | qh << 4 + return (d * q - dm).reshape((n_blocks, QK_K)) +def dequantize_blocks_Q4_K(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d, dmin, scales, qs = split_block_dims(blocks, 2, 2, K_SCALE_SIZE) + d = d.view(torch.float16).to(dtype) + dmin = dmin.view(torch.float16).to(dtype) + sc, m = get_scale_min(scales) + d = (d * sc).reshape((n_blocks, -1, 1)) + dm = (dmin * m).reshape((n_blocks, -1, 1)) + qs = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 4], device=d + .device, dtype=torch.uint8).reshape((1, 1, 2, 1)) + qs = (qs & 15).reshape((n_blocks, -1, 32)) + return (d * qs - dm).reshape((n_blocks, QK_K)) +def dequantize_blocks_Q3_K(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + hmask, qs, scales, d = split_block_dims(blocks, QK_K // 8, QK_K // 4, 12) + d = d.view(torch.float16).to(dtype) + lscales, hscales = scales[:, :8], scales[:, 8:] + lscales = lscales.reshape((n_blocks, 1, 8)) >> torch.tensor([0, 4], + device=d.device, dtype=torch.uint8).reshape((1, 2, 1)) + lscales = lscales.reshape((n_blocks, 16)) + hscales = hscales.reshape((n_blocks, 1, 4)) >> torch.tensor([0, 2, 4, 6 + ], device=d.device, dtype=torch.uint8).reshape((1, 4, 1)) + hscales = hscales.reshape((n_blocks, 16)) + scales = lscales & 15 | (hscales & 3) << 4 + scales = scales.to(torch.int8) - 32 + dl = (d * scales).reshape((n_blocks, 16, 1)) + ql = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 2, 4, 6], + device=d.device, dtype=torch.uint8).reshape((1, 1, 4, 1)) + qh = hmask.reshape(n_blocks, -1, 1, 32) >> torch.tensor([i for i in + range(8)], device=d.device, dtype=torch.uint8).reshape((1, 1, 8, 1)) + ql = ql.reshape((n_blocks, 16, QK_K // 16)) & 3 + qh = qh.reshape((n_blocks, 16, QK_K // 16)) & 1 ^ 1 + q = ql.to(torch.int8) - (qh << 2).to(torch.int8) + return (dl * q).reshape((n_blocks, QK_K)) +def dequantize_blocks_Q2_K(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + scales, qs, d, dmin = split_block_dims(blocks, QK_K // 16, QK_K // 4, 2) + d = d.view(torch.float16).to(dtype) + dmin = dmin.view(torch.float16).to(dtype) + dl = (d * (scales & 15)).reshape((n_blocks, QK_K // 16, 1)) + ml = (dmin * (scales >> 4)).reshape((n_blocks, QK_K // 16, 1)) + shift = torch.tensor([0, 2, 4, 6], device=d.device, dtype=torch.uint8 + ).reshape((1, 1, 4, 1)) + qs = qs.reshape((n_blocks, -1, 1, 32)) >> shift & 3 + qs = qs.reshape((n_blocks, QK_K // 16, 16)) + qs = dl * qs - ml + return qs.reshape((n_blocks, -1)) +dequantize_functions = {GGMLQuantizationType.BF16: dequantize_blocks_BF16, + GGMLQuantizationType.Q8_0: dequantize_blocks_Q8_0, GGMLQuantizationType + .Q5_1: dequantize_blocks_Q5_1, GGMLQuantizationType.Q5_0: + dequantize_blocks_Q5_0, GGMLQuantizationType.Q4_1: + dequantize_blocks_Q4_1, GGMLQuantizationType.Q4_0: + dequantize_blocks_Q4_0, GGMLQuantizationType.Q6_K: + dequantize_blocks_Q6_K, GGMLQuantizationType.Q5_K: + dequantize_blocks_Q5_K, GGMLQuantizationType.Q4_K: + dequantize_blocks_Q4_K, GGMLQuantizationType.Q3_K: + dequantize_blocks_Q3_K, GGMLQuantizationType.Q2_K: dequantize_blocks_Q2_K} \ No newline at end of file diff --git a/gguf_connector/quant2a.py b/gguf_connector/quant2a.py new file mode 100644 index 0000000..e158ea6 --- /dev/null +++ b/gguf_connector/quant2a.py @@ -0,0 +1,203 @@ +import torch +from .quant import dequantize as gq +from .reader import GGMLQuantizationType, GGML_QUANT_SIZES +from tqdm import tqdm +TORCH_COMPATIBLE_QTYPES = {None, GGMLQuantizationType.F32, + GGMLQuantizationType.F16} +def is_torch_compatible(tensor): + return tensor is None or getattr(tensor, 'tensor_type', None + ) in TORCH_COMPATIBLE_QTYPES +def is_quantized(tensor): + return not is_torch_compatible(tensor) +def dequantize(data, qtype, oshape, dtype=None): + block_size, type_size = GGML_QUANT_SIZES[qtype] + dequantize_blocks = dequantize_functions[qtype] + rows = data.reshape((-1, data.shape[-1])).view(torch.uint8) + n_blocks = rows.numel() // type_size + blocks = rows.reshape((n_blocks, type_size)) + blocks = dequantize_blocks(blocks, block_size, type_size, dtype) + return blocks.reshape(oshape) +def dequantize_tensor(tensor, dtype=None, dequant_dtype=None): + qtype = getattr(tensor, 'tensor_type', None) + oshape = getattr(tensor, 'tensor_shape', tensor.shape) + if qtype in TORCH_COMPATIBLE_QTYPES: + return tensor.to(dtype) + elif qtype in dequantize_functions: + dequant_dtype = dtype if dequant_dtype == 'target' else dequant_dtype + return dequantize(tensor.data, qtype, oshape, dtype=dequant_dtype).to( + dtype) + else: + tqdm.write( + f'Pushing back to numpy dequant (cpu offload) for qtype: {qtype}') + new = gq(tensor.cpu().numpy(), qtype) + return torch.from_numpy(new).to(tensor.device, dtype=dtype) +def to_uint32(x): + x = x.view(torch.uint8).to(torch.int32) + return (x[:, 0] | x[:, 1] << 8 | x[:, 2] << 16 | x[:, 3] << 24).unsqueeze(1 + ) +def dequantize_blocks_BF16(blocks, block_size, type_size, dtype=None): + return (blocks.view(torch.int16).to(torch.int32) << 16).view(torch.float32) +def dequantize_blocks_Q8_0(blocks, block_size, type_size, dtype=None): + d = blocks[:, :2].view(torch.float16).to(dtype) + x = blocks[:, 2:].view(torch.int8) + return d * x +def dequantize_blocks_Q5_0(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d = blocks[:, :2].view(torch.float16).to(dtype) + qh = blocks[:, 2:6] + qs = blocks[:, 6:] + qh = to_uint32(qh) + qh = qh.reshape(n_blocks, 1) >> torch.arange(32, device=d.device, dtype + =torch.int32).reshape(1, 32) + ql = qs.reshape(n_blocks, -1, 1, block_size // 2) >> torch.tensor([0, 4 + ], device=d.device, dtype=torch.uint8).reshape(1, 1, 2, 1) + qh = (qh & 1).to(torch.uint8) + ql = (ql & 15).reshape(n_blocks, -1) + qs = (ql | qh << 4).to(torch.int8) - 16 + return d * qs +def dequantize_blocks_Q4_0(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d = blocks[:, :2].view(torch.float16).to(dtype) + qs = blocks[:, 2:] + qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0, + 4], device=d.device, dtype=torch.uint8).reshape((1, 1, 2, 1)) + qs = (qs & 15).reshape((n_blocks, -1)).to(torch.int8) - 8 + return d * qs +def dequantize_blocks_Q5_1(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d = blocks[:, :2].view(torch.float16).to(dtype) + m = blocks[:, 2:4].view(torch.float16).to(dtype) + qh = blocks[:, 4:8] + qs = blocks[:, 8:] + qh = to_uint32(qh) + qh = qh.reshape((n_blocks, 1)) >> torch.arange(32, device=d.device, + dtype=torch.int32).reshape(1, 32) + ql = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0, + 4], device=d.device, dtype=torch.uint8).reshape(1, 1, 2, 1) + qh = (qh & 1).to(torch.uint8) + ql = (ql & 15).reshape((n_blocks, -1)) + qs = ql | qh << 4 + return d * qs + m +def dequantize_blocks_Q4_1(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d = blocks[:, :2].view(torch.float16).to(dtype) + m = blocks[:, 2:4].view(torch.float16).to(dtype) + qs = blocks[:, 4:] + qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0, + 4], device=d.device, dtype=torch.uint8).reshape(1, 1, 2, 1) + qs = (qs & 15).reshape(n_blocks, -1) + return d * qs + m +QK_K = 256 +K_SCALE = 12 +def split_block_dims(blocks, *args): + n_max = blocks.shape[1] + dims = list(args) + [n_max - sum(args)] + return torch.split(blocks, dims, dim=1) +def dequantize_blocks_Q6_K(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + ql, qh, scales, d = split_block_dims(blocks, QK_K // 2, QK_K // 4, QK_K // + 16) + scales = scales.view(torch.int8).to(dtype) + d = d.view(torch.float16).to(dtype) + d = (d * scales).reshape((n_blocks, QK_K // 16, 1)) + ql = ql.reshape((n_blocks, -1, 1, 64)) >> torch.tensor([0, 4], device=d + .device, dtype=torch.uint8).reshape((1, 1, 2, 1)) + ql = (ql & 15).reshape((n_blocks, -1, 32)) + qh = qh.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 2, 4, 6], + device=d.device, dtype=torch.uint8).reshape((1, 1, 4, 1)) + qh = (qh & 3).reshape((n_blocks, -1, 32)) + q = (ql | qh << 4).to(torch.int8) - 32 + q = q.reshape((n_blocks, QK_K // 16, -1)) + return (d * q).reshape((n_blocks, QK_K)) +def dequantize_blocks_Q3_K(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + hmask, qs, scales, d = split_block_dims(blocks, QK_K // 8, QK_K // 4, 12) + d = d.view(torch.float16).to(dtype) + lscales, hscales = scales[:, :8], scales[:, 8:] + lscales = lscales.reshape((n_blocks, 1, 8)) >> torch.tensor([0, 4], + device=d.device, dtype=torch.uint8).reshape((1, 2, 1)) + lscales = lscales.reshape((n_blocks, 16)) + hscales = hscales.reshape((n_blocks, 1, 4)) >> torch.tensor([0, 2, 4, 6 + ], device=d.device, dtype=torch.uint8).reshape((1, 4, 1)) + hscales = hscales.reshape((n_blocks, 16)) + scales = lscales & 15 | (hscales & 3) << 4 + scales = scales.to(torch.int8) - 32 + dl = (d * scales).reshape((n_blocks, 16, 1)) + ql = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 2, 4, 6], + device=d.device, dtype=torch.uint8).reshape((1, 1, 4, 1)) + qh = hmask.reshape(n_blocks, -1, 1, 32) >> torch.tensor([i for i in + range(8)], device=d.device, dtype=torch.uint8).reshape((1, 1, 8, 1)) + ql = ql.reshape((n_blocks, 16, QK_K // 16)) & 3 + qh = qh.reshape((n_blocks, 16, QK_K // 16)) & 1 ^ 1 + q = ql.to(torch.int8) - (qh << 2).to(torch.int8) + return (dl * q).reshape((n_blocks, QK_K)) +def dequantize_blocks_Q2_K(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + scales, qs, d, dmin = split_block_dims(blocks, QK_K // 16, QK_K // 4, 2) + d = d.view(torch.float16).to(dtype) + dmin = dmin.view(torch.float16).to(dtype) + dl = (d * (scales & 15)).reshape((n_blocks, QK_K // 16, 1)) + ml = (dmin * (scales >> 4)).reshape((n_blocks, QK_K // 16, 1)) + shift = torch.tensor([0, 2, 4, 6], device=d.device, dtype=torch.uint8 + ).reshape((1, 1, 4, 1)) + qs = qs.reshape((n_blocks, -1, 1, 32)) >> shift & 3 + qs = qs.reshape((n_blocks, QK_K // 16, 16)) + qs = dl * qs - ml + return qs.reshape((n_blocks, -1)) +def get_scale_min(scales): + n_blocks = scales.shape[0] + scales = scales.view(torch.uint8) + scales = scales.reshape((n_blocks, 3, 4)) + d, m, m_d = torch.split(scales, scales.shape[-2] // 3, dim=-2) + sc = torch.cat([d & 63, m_d & 15 | d >> 2 & 48], dim=-1) + min = torch.cat([m & 63, m_d >> 4 | m >> 2 & 48], dim=-1) + return sc.reshape((n_blocks, 8)), min.reshape((n_blocks, 8)) +def dequantize_blocks_Q5_K(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d, dmin, scales, qh, qs = split_block_dims(blocks, 2, 2, K_SCALE, QK_K // 8 + ) + d = d.view(torch.float16).to(dtype) + dmin = dmin.view(torch.float16).to(dtype) + sc, m = get_scale_min(scales) + d = (d * sc).reshape((n_blocks, -1, 1)) + dm = (dmin * m).reshape((n_blocks, -1, 1)) + ql = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 4], device=d + .device, dtype=torch.uint8).reshape((1, 1, 2, 1)) + qh = qh.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([i for i in + range(8)], device=d.device, dtype=torch.uint8).reshape((1, 1, 8, 1)) + ql = (ql & 15).reshape((n_blocks, -1, 32)) + qh = (qh & 1).reshape((n_blocks, -1, 32)) + q = ql | qh << 4 + return (d * q - dm).reshape((n_blocks, QK_K)) +def dequantize_blocks_Q4_K(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d, dmin, scales, qs = split_block_dims(blocks, 2, 2, K_SCALE) + d = d.view(torch.float16).to(dtype) + dmin = dmin.view(torch.float16).to(dtype) + sc, m = get_scale_min(scales) + d = (d * sc).reshape((n_blocks, -1, 1)) + dm = (dmin * m).reshape((n_blocks, -1, 1)) + qs = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 4], device=d + .device, dtype=torch.uint8).reshape((1, 1, 2, 1)) + qs = (qs & 15).reshape((n_blocks, -1, 32)) + return (d * qs - dm).reshape((n_blocks, QK_K)) +def dequantize_blocks_IQ4_NL(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d, qs = split_block_dims(blocks, 2) + d = d.view(torch.float16).to(dtype) + qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0, + 4], device=qs.device, dtype=torch.uint8).reshape((1, 1, 2, 1)) + qs = (qs & 15).reshape((n_blocks, -1)).to(torch.int8) - 8 + return d * qs +dequantize_functions = {GGMLQuantizationType.BF16: dequantize_blocks_BF16, + GGMLQuantizationType.Q8_0: dequantize_blocks_Q8_0, GGMLQuantizationType + .Q5_1: dequantize_blocks_Q5_1, GGMLQuantizationType.Q5_0: + dequantize_blocks_Q5_0, GGMLQuantizationType.Q4_1: + dequantize_blocks_Q4_1, GGMLQuantizationType.Q4_0: + dequantize_blocks_Q4_0, GGMLQuantizationType.Q6_K: + dequantize_blocks_Q6_K, GGMLQuantizationType.Q5_K: + dequantize_blocks_Q5_K, GGMLQuantizationType.Q4_K: + dequantize_blocks_Q4_K, GGMLQuantizationType.Q3_K: + dequantize_blocks_Q3_K, GGMLQuantizationType.Q2_K: + dequantize_blocks_Q2_K, GGMLQuantizationType.IQ4_NL: + dequantize_blocks_IQ4_NL} diff --git a/gguf_connector/quant2b.py b/gguf_connector/quant2b.py new file mode 100644 index 0000000..71e2744 --- /dev/null +++ b/gguf_connector/quant2b.py @@ -0,0 +1,21 @@ +import torch +def split_block_dims(blocks, *args): + n_max = blocks.shape[1] + dims = list(args) + [n_max - sum(args)] + return torch.split(blocks, dims, dim=1) +def to_uint32(x): + x = x.view(torch.uint8).to(torch.int32) + return (x[:, 0] | x[:, 1] << 8 | x[:, 2] << 16 | x[:, 3] << 24).unsqueeze(1) +def get_scale_min(scales): + n_blocks = scales.shape[0] + scales = scales.view(torch.uint8) + scales = scales.reshape((n_blocks, 3, 4)) + d, m, m_d = torch.split(scales, scales.shape[-2] // 3, dim=-2) + sc = torch.cat([d & 63, m_d & 15 | d >> 2 & 48], dim=-1) + min = torch.cat([m & 63, m_d >> 4 | m >> 2 & 48], dim=-1) + return sc.reshape((n_blocks, 8)), min.reshape((n_blocks, 8)) +def load_grid_tensor(grid_shape, grid_hex): + grid_bytes = torch.tensor(list(grid_hex)) + grid_words = grid_bytes.view(-1, 2).flip(1) + grid = grid_words.contiguous().view(-1).to(torch.int16).view(*grid_shape) + return grid \ No newline at end of file diff --git a/gguf_connector/quant2c.py b/gguf_connector/quant2c.py new file mode 100644 index 0000000..72cceee --- /dev/null +++ b/gguf_connector/quant2c.py @@ -0,0 +1,287 @@ +import torch +from .quant import dequantize as gq +from .reader import GGMLQuantizationType, GGML_QUANT_SIZES +from tqdm import tqdm +TORCH_COMPATIBLE_QTYPES = {None, GGMLQuantizationType.F32, + GGMLQuantizationType.F16} +def is_torch_compatible(tensor): + return tensor is None or getattr(tensor, 'tensor_type', None + ) in TORCH_COMPATIBLE_QTYPES +def is_quantized(tensor): + return not is_torch_compatible(tensor) +def dequantize(data, qtype, oshape, dtype=None): + block_size, type_size = GGML_QUANT_SIZES[qtype] + dequantize_blocks = dequantize_functions[qtype] + rows = data.reshape((-1, data.shape[-1])).view(torch.uint8) + n_blocks = rows.numel() // type_size + blocks = rows.reshape((n_blocks, type_size)) + blocks = dequantize_blocks(blocks, block_size, type_size, dtype) + return blocks.reshape(oshape) +def dequantize_tensor(tensor, dtype=None, dequant_dtype=None): + qtype = getattr(tensor, 'tensor_type', None) + oshape = getattr(tensor, 'tensor_shape', tensor.shape) + if qtype in TORCH_COMPATIBLE_QTYPES: + return tensor.to(dtype) + elif qtype in dequantize_functions: + dequant_dtype = dtype if dequant_dtype == 'target' else dequant_dtype + return dequantize(tensor.data, qtype, oshape, dtype=dequant_dtype).to( + dtype) + else: + tqdm.write(f'Processing tensor: qtype: {qtype}, {oshape}') + new = gq(tensor.cpu().numpy(), qtype) + return torch.from_numpy(new).to(tensor.device, dtype=dtype) +def dequantize_blocks_BF16(blocks, block_size, type_size, dtype=None): + return (blocks.view(torch.int16).to(torch.int32) << 16).view(torch.float32) +def dequantize_blocks_Q8_0(blocks, block_size, type_size, dtype=None): + d = blocks[:, :2].view(torch.float16).to(dtype) + x = blocks[:, 2:].view(torch.int8) + return x * d +def dequantize_blocks_Q4_0(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d = blocks[:, :2].view(torch.float16).to(dtype) + qs = blocks[:, 2:] + qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0, + 4], device=d.device, dtype=torch.uint8).reshape((1, 1, 2, 1)) + qs = (qs & 15).reshape((n_blocks, -1)).to(torch.int8) - 8 + return d * qs +def dequantize_blocks_Q4_1(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d = blocks[:, :2].view(torch.float16).to(dtype) + m = blocks[:, 2:4].view(torch.float16).to(dtype) + qs = blocks[:, 4:] + qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0, + 4], device=d.device, dtype=torch.uint8).reshape(1, 1, 2, 1) + qs = (qs & 15).reshape(n_blocks, -1) + return d * qs + m +from .quant2b import to_uint32 +def dequantize_blocks_Q5_0(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d = blocks[:, :2].view(torch.float16).to(dtype) + qh = blocks[:, 2:6] + qs = blocks[:, 6:] + qh = to_uint32(qh) + qh = qh.reshape(n_blocks, 1) >> torch.arange(32, device=d.device, dtype + =torch.int32).reshape(1, 32) + ql = qs.reshape(n_blocks, -1, 1, block_size // 2) >> torch.tensor([0, 4 + ], device=d.device, dtype=torch.uint8).reshape(1, 1, 2, 1) + qh = (qh & 1).to(torch.uint8) + ql = (ql & 15).reshape(n_blocks, -1) + qs = (ql | qh << 4).to(torch.int8) - 16 + return d * qs +def dequantize_blocks_Q5_1(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d = blocks[:, :2].view(torch.float16).to(dtype) + m = blocks[:, 2:4].view(torch.float16).to(dtype) + qh = blocks[:, 4:8] + qs = blocks[:, 8:] + qh = to_uint32(qh) + qh = qh.reshape((n_blocks, 1)) >> torch.arange(32, device=d.device, + dtype=torch.int32).reshape(1, 32) + ql = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0, + 4], device=d.device, dtype=torch.uint8).reshape(1, 1, 2, 1) + qh = (qh & 1).to(torch.uint8) + ql = (ql & 15).reshape((n_blocks, -1)) + qs = ql | qh << 4 + return d * qs + m +QK_K = 256 +from .quant2b import split_block_dims +def dequantize_blocks_Q6_K(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + ql, qh, scales, d = split_block_dims(blocks, QK_K // 2, QK_K // 4, QK_K // + 16) + scales = scales.view(torch.int8).to(dtype) + d = d.view(torch.float16).to(dtype) + d = (d * scales).reshape((n_blocks, QK_K // 16, 1)) + ql = ql.reshape((n_blocks, -1, 1, 64)) >> torch.tensor([0, 4], device=d + .device, dtype=torch.uint8).reshape((1, 1, 2, 1)) + ql = (ql & 15).reshape((n_blocks, -1, 32)) + qh = qh.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 2, 4, 6], + device=d.device, dtype=torch.uint8).reshape((1, 1, 4, 1)) + qh = (qh & 3).reshape((n_blocks, -1, 32)) + q = (ql | qh << 4).to(torch.int8) - 32 + q = q.reshape((n_blocks, QK_K // 16, -1)) + return (d * q).reshape((n_blocks, QK_K)) +def dequantize_blocks_Q3_K(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + hmask, qs, scales, d = split_block_dims(blocks, QK_K // 8, QK_K // 4, 12) + d = d.view(torch.float16).to(dtype) + lscales, hscales = scales[:, :8], scales[:, 8:] + lscales = lscales.reshape((n_blocks, 1, 8)) >> torch.tensor([0, 4], + device=d.device, dtype=torch.uint8).reshape((1, 2, 1)) + lscales = lscales.reshape((n_blocks, 16)) + hscales = hscales.reshape((n_blocks, 1, 4)) >> torch.tensor([0, 2, 4, 6 + ], device=d.device, dtype=torch.uint8).reshape((1, 4, 1)) + hscales = hscales.reshape((n_blocks, 16)) + scales = lscales & 15 | (hscales & 3) << 4 + scales = scales.to(torch.int8) - 32 + dl = (d * scales).reshape((n_blocks, 16, 1)) + ql = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 2, 4, 6], + device=d.device, dtype=torch.uint8).reshape((1, 1, 4, 1)) + qh = hmask.reshape(n_blocks, -1, 1, 32) >> torch.tensor([i for i in + range(8)], device=d.device, dtype=torch.uint8).reshape((1, 1, 8, 1)) + ql = ql.reshape((n_blocks, 16, QK_K // 16)) & 3 + qh = qh.reshape((n_blocks, 16, QK_K // 16)) & 1 ^ 1 + q = ql.to(torch.int8) - (qh << 2).to(torch.int8) + return (dl * q).reshape((n_blocks, QK_K)) +def dequantize_blocks_Q2_K(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + scales, qs, d, dmin = split_block_dims(blocks, QK_K // 16, QK_K // 4, 2) + d = d.view(torch.float16).to(dtype) + dmin = dmin.view(torch.float16).to(dtype) + dl = (d * (scales & 15)).reshape((n_blocks, QK_K // 16, 1)) + ml = (dmin * (scales >> 4)).reshape((n_blocks, QK_K // 16, 1)) + shift = torch.tensor([0, 2, 4, 6], device=d.device, dtype=torch.uint8 + ).reshape((1, 1, 4, 1)) + qs = qs.reshape((n_blocks, -1, 1, 32)) >> shift & 3 + qs = qs.reshape((n_blocks, QK_K // 16, 16)) + qs = dl * qs - ml + return qs.reshape((n_blocks, -1)) +K_SCALE_SIZE = 12 +from .quant2b import get_scale_min +def dequantize_blocks_Q5_K(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d, dmin, scales, qh, qs = split_block_dims(blocks, 2, 2, K_SCALE_SIZE, + QK_K // 8) + d = d.view(torch.float16).to(dtype) + dmin = dmin.view(torch.float16).to(dtype) + sc, m = get_scale_min(scales) + d = (d * sc).reshape((n_blocks, -1, 1)) + dm = (dmin * m).reshape((n_blocks, -1, 1)) + ql = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 4], device=d + .device, dtype=torch.uint8).reshape((1, 1, 2, 1)) + qh = qh.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([i for i in + range(8)], device=d.device, dtype=torch.uint8).reshape((1, 1, 8, 1)) + ql = (ql & 15).reshape((n_blocks, -1, 32)) + qh = (qh & 1).reshape((n_blocks, -1, 32)) + q = ql | qh << 4 + return (d * q - dm).reshape((n_blocks, QK_K)) +def dequantize_blocks_Q4_K(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d, dmin, scales, qs = split_block_dims(blocks, 2, 2, K_SCALE_SIZE) + d = d.view(torch.float16).to(dtype) + dmin = dmin.view(torch.float16).to(dtype) + sc, m = get_scale_min(scales) + d = (d * sc).reshape((n_blocks, -1, 1)) + dm = (dmin * m).reshape((n_blocks, -1, 1)) + qs = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 4], device=d + .device, dtype=torch.uint8).reshape((1, 1, 2, 1)) + qs = (qs & 15).reshape((n_blocks, -1, 32)) + return (d * qs - dm).reshape((n_blocks, QK_K)) +def dequantize_blocks_IQ4_NL(blocks, block_size, type_size, dtype=None): + kvalues = torch.tensor([-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, + 25, 38, 53, 69, 89, 113], dtype=torch.float32, device=blocks.device) + n_blocks = blocks.shape[0] + d, qs = split_block_dims(blocks, 2) + d = d.view(torch.float16).to(dtype) + qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0, + 4], device=blocks.device, dtype=torch.uint8).reshape((1, 1, 2, 1)) + qs = (qs & 15).reshape((n_blocks, -1)).to(torch.int64) + kvalues = kvalues.view(1, 1, 16) + qs = qs.unsqueeze(-1) + qs = torch.gather(kvalues.expand(qs.shape[0], qs.shape[1], 16), 2, qs) + qs = qs.squeeze(-1).to(dtype) + return d * qs +def dequantize_blocks_IQ4_XS(blocks, block_size, type_size, dtype=None): + kvalues = torch.tensor([-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, + 25, 38, 53, 69, 89, 113], dtype=torch.float32, device=blocks.device) + n_blocks = blocks.shape[0] + d, scales_h, scales_l, qs = split_block_dims(blocks, 2, 2, QK_K // 64) + d = d.view(torch.float16).to(dtype) + scales_h = scales_h.view(torch.int16) + scales_l = scales_l.reshape((n_blocks, -1, 1)) >> torch.tensor( + [0, 4], device=blocks.device, dtype=torch.uint8).reshape((1, 1, 2)) + scales_h = scales_h.reshape((n_blocks, 1, -1)) >> torch.tensor( + [2 * i for i in range(QK_K // 32)], + device=blocks.device, dtype=torch.uint8).reshape((1, -1, 1)) + scales_l = scales_l.reshape((n_blocks, -1)) & 0x0F + scales_h = scales_h.reshape((n_blocks, -1)) & 0x03 + scales = (scales_l | (scales_h << 4)) - 32 + dl = (d * scales.to(dtype)).reshape((n_blocks, -1, 1)) + shifts_q = torch.tensor( + [0, 4], device=blocks.device, dtype=torch.uint8).reshape(1, 1, 2, 1) + qs = qs.reshape((n_blocks, -1, 1, 16)) >> shifts_q + qs = (qs & 15).reshape((n_blocks, -1, 32)).to(torch.int64) + kvalues = kvalues.view(1, 1, 1, 16) + qs = qs.unsqueeze(-1) + qs = torch.gather(kvalues.expand(qs.shape[0], qs.shape[1], qs.shape[2], 16), 3, qs) + qs = qs.squeeze(-1).to(dtype) + return (dl * qs).reshape(n_blocks, -1) +def dequantize_blocks_TQ2_0(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + qs, d = split_block_dims(blocks, QK_K // 4) + d = d.view(torch.float16).to(dtype) + qs = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 2, 4, 6], + device=d.device, dtype=torch.uint8).reshape((1, 1, 4, 1)) + qs = (qs & 3).reshape((n_blocks, -1)) - 1 + return d * qs +def dequantize_blocks_TQ1_0(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + qs, qh, d = split_block_dims(blocks, (QK_K - 4 * QK_K // 64) // 5, QK_K // + 64) + d = d.view(torch.float16).to(dtype) + qs0, qs1 = qs[..., :32], qs[..., 32:] + qs0 = qs0.reshape((n_blocks, -1, 1, 32)) * torch.tensor([1, 3, 9, 27, + 81], device=d.device, dtype=torch.uint8).reshape((1, 1, 5, 1)) + qs0 = qs0.reshape((n_blocks, -1)) + qs1 = qs1.reshape((n_blocks, -1, 1, 16)) * torch.tensor([1, 3, 9, 27, + 81], device=d.device, dtype=torch.uint8).reshape((1, 1, 5, 1)) + qs1 = qs1.reshape((n_blocks, -1)) + qh = qh.reshape((n_blocks, -1, 1, 4)) * torch.tensor([1, 3, 9, 27], + device=d.device, dtype=torch.uint8).reshape((1, 1, 4, 1)) + qh = qh.reshape((n_blocks, -1)) + qs = torch.cat([qs0, qs1, qh], dim=-1) + qs = (qs * 3 >> 8) - 1 + return d * qs +def dequantize_blocks_IQ3_S(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d, qs, qh, signs, scales = split_block_dims(blocks, 2, QK_K // 4, QK_K // 32, QK_K // 8) + d = d.view(torch.float16).to(dtype) + scales = scales.reshape((n_blocks, -1, 1)) >> torch.tensor( + [0, 4], device=blocks.device, dtype=torch.uint8).reshape((1, 1, 2)) + scales = (scales & 15).reshape((n_blocks, -1)) + db = d * (1 + 2 * scales) + db = db.reshape((n_blocks, -1, 1, 1)) + signs = signs.reshape((n_blocks, -1, 1)) >> torch.tensor( + [i for i in range(8)], device=blocks.device, dtype=torch.uint8).reshape((1, 1, 8)) + signs = signs & 1 + signs = torch.where(signs == 0, torch.tensor(1.0, device=blocks.device), torch.tensor(-1.0, device=blocks.device)) + signs = signs.reshape((n_blocks, -1, 4, 8)) + return (db * signs).reshape((n_blocks, -1)) +def dequantize_blocks_IQ3_XXS(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d, qs, scales = split_block_dims(blocks, 2, QK_K // 4) + d = d.view(torch.float16).to(dtype) + scales = to_uint32(scales) + db = d * (0.5 + (scales >> 28)) * 0.5 + db = db.reshape((n_blocks, -1, 1, 1)) + bit_shifts = torch.tensor( + [0, 7, 14, 21], device=blocks.device, dtype=torch.uint8).reshape((1, 1, 4)) + signs = scales.reshape((n_blocks, -1, 1)) >> bit_shifts + db = db.reshape((n_blocks, -1, 1, 1)) + signs = signs.reshape((n_blocks, -1, 1)) >> torch.tensor( + [i for i in range(8)], device=blocks.device, dtype=torch.uint8).reshape((1, 1, 8)) + signs = signs & 1 + db = db.reshape((n_blocks, -1, 1, 1)) + sign_shifts = torch.arange(8, device=blocks.device, dtype=torch.uint8).view(1, 1, 8) + signs = signs.reshape((n_blocks, -1, 1)) >> sign_shifts + signs = (signs & 1).float() + signs = torch.where(signs == 0, torch.tensor(1.0, device=blocks.device), torch.tensor(-1.0, device=blocks.device)) + signs = signs.reshape((n_blocks, -1, 4, 8)) + return (db * signs).reshape((n_blocks, -1)) +dequantize_functions = {GGMLQuantizationType.BF16: dequantize_blocks_BF16, + GGMLQuantizationType.Q8_0: dequantize_blocks_Q8_0, GGMLQuantizationType + .Q5_1: dequantize_blocks_Q5_1, GGMLQuantizationType.Q5_0: + dequantize_blocks_Q5_0, GGMLQuantizationType.Q4_1: + dequantize_blocks_Q4_1, GGMLQuantizationType.Q4_0: + dequantize_blocks_Q4_0, GGMLQuantizationType.Q6_K: + dequantize_blocks_Q6_K, GGMLQuantizationType.Q5_K: + dequantize_blocks_Q5_K, GGMLQuantizationType.Q4_K: + dequantize_blocks_Q4_K, GGMLQuantizationType.Q3_K: + dequantize_blocks_Q3_K, GGMLQuantizationType.Q2_K: + dequantize_blocks_Q2_K, GGMLQuantizationType.IQ4_NL: + dequantize_blocks_IQ4_NL, GGMLQuantizationType.IQ4_XS: + dequantize_blocks_IQ4_XS, GGMLQuantizationType.IQ3_S: + dequantize_blocks_IQ3_S, GGMLQuantizationType.IQ3_XXS: + dequantize_blocks_IQ3_XXS, GGMLQuantizationType.TQ2_0: + dequantize_blocks_TQ2_0, GGMLQuantizationType.TQ1_0: + dequantize_blocks_TQ1_0} diff --git a/gguf_connector/quant2d.py b/gguf_connector/quant2d.py new file mode 100644 index 0000000..4f08156 --- /dev/null +++ b/gguf_connector/quant2d.py @@ -0,0 +1,309 @@ +import torch +from .quant import dequantize as gq +from .reader import GGMLQuantizationType, GGML_QUANT_SIZES +from tqdm import tqdm +TORCH_COMPATIBLE_QTYPES = {None, GGMLQuantizationType.F32, + GGMLQuantizationType.F16} +def is_torch_compatible(tensor): + return tensor is None or getattr(tensor, 'tensor_type', None + ) in TORCH_COMPATIBLE_QTYPES +def is_quantized(tensor): + return not is_torch_compatible(tensor) +def dequantize(data, qtype, oshape, dtype=None): + block_size, type_size = GGML_QUANT_SIZES[qtype] + dequantize_blocks = dequantize_functions[qtype] + rows = data.reshape((-1, data.shape[-1])).view(torch.uint8) + n_blocks = rows.numel() // type_size + blocks = rows.reshape((n_blocks, type_size)) + blocks = dequantize_blocks(blocks, block_size, type_size, dtype) + return blocks.reshape(oshape) +def dequantize_tensor(tensor, dtype=None, dequant_dtype=None): + qtype = getattr(tensor, 'tensor_type', None) + oshape = getattr(tensor, 'tensor_shape', tensor.shape) + if qtype in TORCH_COMPATIBLE_QTYPES: + return tensor.to(dtype) + elif qtype in dequantize_functions: + dequant_dtype = dtype if dequant_dtype == 'target' else dequant_dtype + return dequantize(tensor.data, qtype, oshape, dtype=dequant_dtype).to( + dtype) + else: + tqdm.write(f'Processing tensor: qtype: {qtype}, {oshape}') + new = gq(tensor.cpu().numpy(), qtype) + return torch.from_numpy(new).to(tensor.device, dtype=dtype) +def dequantize_blocks_BF16(blocks, block_size, type_size, dtype=None): + return (blocks.view(torch.int16).to(torch.int32) << 16).view(torch.float32) +def dequantize_blocks_Q8_0(blocks, block_size, type_size, dtype=None): + d = blocks[:, :2].view(torch.float16).to(dtype) + x = blocks[:, 2:].view(torch.int8) + return x * d +def dequantize_blocks_Q4_0(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d = blocks[:, :2].view(torch.float16).to(dtype) + qs = blocks[:, 2:] + qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0, + 4], device=d.device, dtype=torch.uint8).reshape((1, 1, 2, 1)) + qs = (qs & 15).reshape((n_blocks, -1)).to(torch.int8) - 8 + return d * qs +def dequantize_blocks_Q4_1(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d = blocks[:, :2].view(torch.float16).to(dtype) + m = blocks[:, 2:4].view(torch.float16).to(dtype) + qs = blocks[:, 4:] + qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0, + 4], device=d.device, dtype=torch.uint8).reshape(1, 1, 2, 1) + qs = (qs & 15).reshape(n_blocks, -1) + return d * qs + m +from .quant2b import to_uint32 +def dequantize_blocks_Q5_0(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d = blocks[:, :2].view(torch.float16).to(dtype) + qh = blocks[:, 2:6] + qs = blocks[:, 6:] + qh = to_uint32(qh) + qh = qh.reshape(n_blocks, 1) >> torch.arange(32, device=d.device, dtype + =torch.int32).reshape(1, 32) + ql = qs.reshape(n_blocks, -1, 1, block_size // 2) >> torch.tensor([0, 4 + ], device=d.device, dtype=torch.uint8).reshape(1, 1, 2, 1) + qh = (qh & 1).to(torch.uint8) + ql = (ql & 15).reshape(n_blocks, -1) + qs = (ql | qh << 4).to(torch.int8) - 16 + return d * qs +def dequantize_blocks_Q5_1(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d = blocks[:, :2].view(torch.float16).to(dtype) + m = blocks[:, 2:4].view(torch.float16).to(dtype) + qh = blocks[:, 4:8] + qs = blocks[:, 8:] + qh = to_uint32(qh) + qh = qh.reshape((n_blocks, 1)) >> torch.arange(32, device=d.device, + dtype=torch.int32).reshape(1, 32) + ql = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0, + 4], device=d.device, dtype=torch.uint8).reshape(1, 1, 2, 1) + qh = (qh & 1).to(torch.uint8) + ql = (ql & 15).reshape((n_blocks, -1)) + qs = ql | qh << 4 + return d * qs + m +QK_K = 256 +from .quant2b import split_block_dims +def dequantize_blocks_Q6_K(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + ql, qh, scales, d = split_block_dims(blocks, QK_K // 2, QK_K // 4, QK_K // + 16) + scales = scales.view(torch.int8).to(dtype) + d = d.view(torch.float16).to(dtype) + d = (d * scales).reshape((n_blocks, QK_K // 16, 1)) + ql = ql.reshape((n_blocks, -1, 1, 64)) >> torch.tensor([0, 4], device=d + .device, dtype=torch.uint8).reshape((1, 1, 2, 1)) + ql = (ql & 15).reshape((n_blocks, -1, 32)) + qh = qh.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 2, 4, 6], + device=d.device, dtype=torch.uint8).reshape((1, 1, 4, 1)) + qh = (qh & 3).reshape((n_blocks, -1, 32)) + q = (ql | qh << 4).to(torch.int8) - 32 + q = q.reshape((n_blocks, QK_K // 16, -1)) + return (d * q).reshape((n_blocks, QK_K)) +def dequantize_blocks_Q3_K(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + hmask, qs, scales, d = split_block_dims(blocks, QK_K // 8, QK_K // 4, 12) + d = d.view(torch.float16).to(dtype) + lscales, hscales = scales[:, :8], scales[:, 8:] + lscales = lscales.reshape((n_blocks, 1, 8)) >> torch.tensor([0, 4], + device=d.device, dtype=torch.uint8).reshape((1, 2, 1)) + lscales = lscales.reshape((n_blocks, 16)) + hscales = hscales.reshape((n_blocks, 1, 4)) >> torch.tensor([0, 2, 4, 6 + ], device=d.device, dtype=torch.uint8).reshape((1, 4, 1)) + hscales = hscales.reshape((n_blocks, 16)) + scales = lscales & 15 | (hscales & 3) << 4 + scales = scales.to(torch.int8) - 32 + dl = (d * scales).reshape((n_blocks, 16, 1)) + ql = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 2, 4, 6], + device=d.device, dtype=torch.uint8).reshape((1, 1, 4, 1)) + qh = hmask.reshape(n_blocks, -1, 1, 32) >> torch.tensor([i for i in + range(8)], device=d.device, dtype=torch.uint8).reshape((1, 1, 8, 1)) + ql = ql.reshape((n_blocks, 16, QK_K // 16)) & 3 + qh = qh.reshape((n_blocks, 16, QK_K // 16)) & 1 ^ 1 + q = ql.to(torch.int8) - (qh << 2).to(torch.int8) + return (dl * q).reshape((n_blocks, QK_K)) +def dequantize_blocks_Q2_K(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + scales, qs, d, dmin = split_block_dims(blocks, QK_K // 16, QK_K // 4, 2) + d = d.view(torch.float16).to(dtype) + dmin = dmin.view(torch.float16).to(dtype) + dl = (d * (scales & 15)).reshape((n_blocks, QK_K // 16, 1)) + ml = (dmin * (scales >> 4)).reshape((n_blocks, QK_K // 16, 1)) + shift = torch.tensor([0, 2, 4, 6], device=d.device, dtype=torch.uint8 + ).reshape((1, 1, 4, 1)) + qs = qs.reshape((n_blocks, -1, 1, 32)) >> shift & 3 + qs = qs.reshape((n_blocks, QK_K // 16, 16)) + qs = dl * qs - ml + return qs.reshape((n_blocks, -1)) +K_SCALE_SIZE = 12 +from .quant2b import get_scale_min +def dequantize_blocks_Q5_K(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d, dmin, scales, qh, qs = split_block_dims(blocks, 2, 2, K_SCALE_SIZE, + QK_K // 8) + d = d.view(torch.float16).to(dtype) + dmin = dmin.view(torch.float16).to(dtype) + sc, m = get_scale_min(scales) + d = (d * sc).reshape((n_blocks, -1, 1)) + dm = (dmin * m).reshape((n_blocks, -1, 1)) + ql = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 4], device=d + .device, dtype=torch.uint8).reshape((1, 1, 2, 1)) + qh = qh.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([i for i in + range(8)], device=d.device, dtype=torch.uint8).reshape((1, 1, 8, 1)) + ql = (ql & 15).reshape((n_blocks, -1, 32)) + qh = (qh & 1).reshape((n_blocks, -1, 32)) + q = ql | qh << 4 + return (d * q - dm).reshape((n_blocks, QK_K)) +def dequantize_blocks_Q4_K(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + d, dmin, scales, qs = split_block_dims(blocks, 2, 2, K_SCALE_SIZE) + d = d.view(torch.float16).to(dtype) + dmin = dmin.view(torch.float16).to(dtype) + sc, m = get_scale_min(scales) + d = (d * sc).reshape((n_blocks, -1, 1)) + dm = (dmin * m).reshape((n_blocks, -1, 1)) + qs = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 4], device=d + .device, dtype=torch.uint8).reshape((1, 1, 2, 1)) + qs = (qs & 15).reshape((n_blocks, -1, 32)) + return (d * qs - dm).reshape((n_blocks, QK_K)) +def dequantize_blocks_TQ2_0(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + qs, d = split_block_dims(blocks, QK_K // 4) + d = d.view(torch.float16).to(dtype) + qs = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 2, 4, 6], + device=d.device, dtype=torch.uint8).reshape((1, 1, 4, 1)) + qs = (qs & 3).reshape((n_blocks, -1)) - 1 + return d * qs +def dequantize_blocks_TQ1_0(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + qs, qh, d = split_block_dims(blocks, (QK_K - 4 * QK_K // 64) // 5, QK_K // + 64) + d = d.view(torch.float16).to(dtype) + qs0, qs1 = qs[..., :32], qs[..., 32:] + qs0 = qs0.reshape((n_blocks, -1, 1, 32)) * torch.tensor([1, 3, 9, 27, + 81], device=d.device, dtype=torch.uint8).reshape((1, 1, 5, 1)) + qs0 = qs0.reshape((n_blocks, -1)) + qs1 = qs1.reshape((n_blocks, -1, 1, 16)) * torch.tensor([1, 3, 9, 27, + 81], device=d.device, dtype=torch.uint8).reshape((1, 1, 5, 1)) + qs1 = qs1.reshape((n_blocks, -1)) + qh = qh.reshape((n_blocks, -1, 1, 4)) * torch.tensor([1, 3, 9, 27], + device=d.device, dtype=torch.uint8).reshape((1, 1, 4, 1)) + qh = qh.reshape((n_blocks, -1)) + qs = torch.cat([qs0, qs1, qh], dim=-1) + qs = (qs * 3 >> 8) - 1 + return d * qs +def dequantize_blocks_IQ4_NL(blocks, block_size, type_size, dtype=None): + kvalues = torch.tensor([-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, + 25, 38, 53, 69, 89, 113], dtype=torch.float32, device=blocks.device) + n_blocks = blocks.shape[0] + d, qs = split_block_dims(blocks, 2) + d = d.view(torch.float16).to(dtype) + qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0, + 4], device=blocks.device, dtype=torch.uint8).reshape((1, 1, 2, 1)) + qs = (qs & 15).reshape((n_blocks, -1)).to(torch.int64) + kvalues = kvalues.view(1, 1, 16) + qs = qs.unsqueeze(-1) + qs = torch.gather(kvalues.expand(qs.shape[0], qs.shape[1], 16), 2, qs) + qs = qs.squeeze(-1).to(dtype) + return d * qs +def dequantize_blocks_IQ4_XS(blocks, block_size, type_size, dtype=None): + kvalues = torch.tensor([-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, + 25, 38, 53, 69, 89, 113], dtype=torch.float32, device=blocks.device) + n_blocks = blocks.shape[0] + d, scales_h, scales_l, qs = split_block_dims(blocks, 2, 2, QK_K // 64) + d = d.view(torch.float16).to(dtype) + scales_h = scales_h.view(torch.int16) + scales_l = scales_l.reshape((n_blocks, -1, 1)) >> torch.tensor([0, 4], + device=blocks.device, dtype=torch.uint8).reshape((1, 1, 2)) + scales_h = scales_h.reshape((n_blocks, 1, -1)) >> torch.tensor([(2 * i) for + i in range(QK_K // 32)], device=blocks.device, dtype=torch.uint8 + ).reshape((1, -1, 1)) + scales_l = scales_l.reshape((n_blocks, -1)) & 15 + scales_h = scales_h.reshape((n_blocks, -1)) & 3 + scales = (scales_l | scales_h << 4) - 32 + dl = (d * scales.to(dtype)).reshape((n_blocks, -1, 1)) + shifts_q = torch.tensor([0, 4], device=blocks.device, dtype=torch.uint8 + ).reshape(1, 1, 2, 1) + qs = qs.reshape((n_blocks, -1, 1, 16)) >> shifts_q + qs = (qs & 15).reshape((n_blocks, -1, 32)).to(torch.int64) + kvalues = kvalues.view(1, 1, 1, 16) + qs = qs.unsqueeze(-1) + qs = torch.gather(kvalues.expand(qs.shape[0], qs.shape[1], qs.shape[2], + 16), 3, qs) + qs = qs.squeeze(-1).to(dtype) + return (dl * qs).reshape(n_blocks, -1) +from .quant2b import load_grid_tensor +def dequantize_blocks_IQ3_S(blocks, block_size, type_size, dtype=None): + grid_shape = 512, 4 + grid_hex = ( + b'00000100020005000700100011001200140016002000210025003300400042004500470051005300600062007100740077000001010102010401100111011501200123012701310135014401610165017201000201020502070210021302160221022502300234024202450247025102530270027302030311031503200322033103330336034403500352036703710375030004130417042104240432044004430451047004020504052005220526053305410545054705660573050606110613063106520671060007020704072007220726073307500754070010011002100410101011101310151017102010221031103410361054105610611072100011011103110611101114112111301133114111501152117011761100121212151217122012241232124012431255126012721201130413071310131313211327133013341341136213701303140514121414143114331442144614501454140115101513152115301532155115201624162716441646160117031710171217211735174117621770170020012003200520072010201220142016202120232027203020322041204320452050205220672070207320752000210221102113211721222125213121342142215121012204220722212223223022372241225322572271227422002302230523112322232423312333234223502366230124072420242324322435244124722475240425112522253725402553257025002602260726212655266126052711272627302743275027023011301330153017302230313033303530423044304730513063307130013103310531143121312331403160317231763100321232203232323432503201331033143321332333273330334133433347335533733303341134163422343134523460346434013510351235253532354435563573351636413601370337203722373537004004401240204024402740324041405040704002410741114113412241304135414341514155410142034210421542214233424042574262427042044311431343204322433143354300440244244437444044714405450745214562451346344660461047154730474347514702501050145022504050445047505250665074500151035105511251215132517251005211522352305236525352025307531053275344535153655373530154045420543254465412552655515553554256025704572257116013601560316033606060006120612761646112623462426255626262706200631463216340632564436462640065036534656065056640661167136700700470077020702270367040705470627002711171247143714571017204721072167221723072517202733273357353730174057413742074507422754275027631760077' + ) + n_blocks = blocks.shape[0] + d, qs, qh, signs, scales = split_block_dims(blocks, 2, QK_K // 4, QK_K // + 32, QK_K // 8) + d = d.view(torch.float16).to(dtype) + scales = scales.reshape((n_blocks, -1, 1)) >> torch.tensor([0, 4], + device=blocks.device, dtype=torch.uint8).reshape((1, 1, 2)) + scales = (scales & 15).reshape((n_blocks, -1)) + db = d * (1 + 2 * scales.to(dtype)) + db = db.reshape((n_blocks, -1, 1, 1)) + signs = signs.reshape((n_blocks, -1, 1)) >> torch.arange(8, device= + blocks.device, dtype=torch.uint8).reshape((1, 1, 8)) + signs = signs & 1 + signs = signs.reshape((n_blocks, -1, 4, 8)) + qh_shifts = torch.arange(8, device=blocks.device, dtype=torch.uint8).view( + 1, 1, 8) + qh = qh.reshape((n_blocks, -1, 1)) >> qh_shifts & 1 + qh = qh.reshape((n_blocks, -1)).to(torch.int16) + qs = qs.to(torch.int64) | qh << 8 + grid = load_grid_tensor(grid_shape, grid_hex) + grid = grid.unsqueeze(0).expand(n_blocks, -1, -1) + return (db * signs).reshape(n_blocks, -1) +def dequantize_blocks_IQ3_XXS(blocks, block_size, type_size, dtype=None): + grid_shape = 256, 4 + grid_hex = ( + b'0000020004001100130017002000220031004200730075000101030110011201210125013001320141015401700100020202040211022002220231023302370251025702750201030703100312032503700313043704440457047304750401050705320552053506640610071407160743076107011003101010121021102310301032103410471050100011021111112011221101120312101212122112301272120013021320133113461366130114051450142015241546157115051622174017002002201120132020202220262031204220012103210521102112212121302163216721702100220222112217222022222237224022552201231023142370237423352453240325272541257425012703271627452701301030123021302330503065307230003102312031313144314631013203321032253252327232113333333034473472340035063522355535143636366336333760370440174035403740534057407441204237424042604266420743454304445144644425454345704505471047124730471250415070500051065126515551145232527252025353531054235427547254025531555056245742572460446046606460216161611762646230633663446405655265336603672167037005700770107032705270267140711272457252720073157333736073217441740075027524753076' + ) + n_blocks = blocks.shape[0] + d, qs, scales = split_block_dims(blocks, 2, QK_K // 4) + d = d.view(torch.float16).to(dtype) + scales = to_uint32(scales) + db = d * (0.5 + (scales >> 28)) * 0.5 + db = db.reshape((n_blocks, -1, 1, 1)) + bit_shifts = torch.tensor([0, 7, 14, 21], device=blocks.device, dtype= + torch.uint8).reshape((1, 1, 4)) + signs = scales.reshape((n_blocks, -1, 1)) >> bit_shifts + db = db.reshape((n_blocks, -1, 1, 1)) + signs = signs.reshape((n_blocks, -1, 1)) >> torch.tensor([i for i in + range(8)], device=blocks.device, dtype=torch.uint8).reshape((1, 1, 8)) + signs = signs & 1 + db = db.reshape((n_blocks, -1, 1, 1)) + sign_shifts = torch.arange(8, device=blocks.device, dtype=torch.uint8 + ).view(1, 1, 8) + signs = signs.reshape((n_blocks, -1, 1)) >> sign_shifts + signs = (signs & 1).float() + signs = torch.where(signs == 0, torch.tensor(1.0, device=blocks.device), + torch.tensor(-1.0, device=blocks.device)) + signs = signs.reshape(n_blocks, -1, 4, 8) + qs = qs.reshape(n_blocks, -1, 1, 1) + grid = load_grid_tensor(grid_shape, grid_hex) + grid = grid.unsqueeze(0).expand(n_blocks, -1, -1) + return (db * signs).reshape(n_blocks, -1) +dequantize_functions = {GGMLQuantizationType.BF16: dequantize_blocks_BF16, + GGMLQuantizationType.Q8_0: dequantize_blocks_Q8_0, GGMLQuantizationType + .Q5_1: dequantize_blocks_Q5_1, GGMLQuantizationType.Q5_0: + dequantize_blocks_Q5_0, GGMLQuantizationType.Q4_1: + dequantize_blocks_Q4_1, GGMLQuantizationType.Q4_0: + dequantize_blocks_Q4_0, GGMLQuantizationType.Q6_K: + dequantize_blocks_Q6_K, GGMLQuantizationType.Q5_K: + dequantize_blocks_Q5_K, GGMLQuantizationType.Q4_K: + dequantize_blocks_Q4_K, GGMLQuantizationType.Q3_K: + dequantize_blocks_Q3_K, GGMLQuantizationType.Q2_K: + dequantize_blocks_Q2_K, GGMLQuantizationType.TQ2_0: + dequantize_blocks_TQ2_0, GGMLQuantizationType.TQ1_0: + dequantize_blocks_TQ1_0, GGMLQuantizationType.IQ4_NL: + dequantize_blocks_IQ4_NL, GGMLQuantizationType.IQ4_XS: + dequantize_blocks_IQ4_XS, GGMLQuantizationType.IQ3_S: + dequantize_blocks_IQ3_S, GGMLQuantizationType.IQ3_XXS: + dequantize_blocks_IQ3_XXS} \ No newline at end of file diff --git a/gguf_connector/r.py b/gguf_connector/r.py new file mode 100644 index 0000000..77b024c --- /dev/null +++ b/gguf_connector/r.py @@ -0,0 +1,43 @@ +def read_gguf_file(gguf_file_path): + from gguf_connector.reader import GGUFReader + reader = GGUFReader(gguf_file_path) + print('-' * 105) + print('Key-Value Pairs:') + max_key_length = max(len(key) for key in reader.fields.keys()) + for key, field in reader.fields.items(): + value = field.parts[field.data[0]] + print(f'{key:{max_key_length}} : {value}') + print('-' * 105) + print('Tensors:') + tensor_info_format = ( + '{:<30} | Shape: {:<15} | Size: {:<12} | Quantization: {}') + print(tensor_info_format.format('Tensor Name', 'Shape', 'Size', + 'Quantization')) + print('-' * 105) + for tensor in reader.tensors: + shape_str = 'x'.join(map(str, tensor.shape)) + size_str = str(tensor.n_elements) + quantization_str = tensor.tensor_type.name + print(tensor_info_format.format(tensor.name, shape_str, size_str, + quantization_str)) +import os +gguf_files = [file for file in os.listdir() if file.endswith('.gguf')] +if gguf_files: + print('GGUF file(s) available. Select which one to read:') + for index, file_name in enumerate(gguf_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(gguf_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = gguf_files[choice_index] + print(f'Model file: {selected_file} is selected!') + ModelPath = selected_file + from rich.progress import Progress + with Progress(transient=True) as progress: + task = progress.add_task('Processing', total=None) + read_gguf_file(ModelPath) + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No GGUF files are available in the current directory.') + input('--- Press ENTER To Exit ---') \ No newline at end of file diff --git a/gguf_connector/r2.py b/gguf_connector/r2.py new file mode 100644 index 0000000..0dffee1 --- /dev/null +++ b/gguf_connector/r2.py @@ -0,0 +1,27 @@ +def read_gguf_file(gguf_file_path): + from gguf_connector.reader2 import GGUFReader, ReaderError + reader = GGUFReader(gguf_file_path) + try: + reader.read() + except ReaderError as e: + print(f'Error: {e}') + else: + reader.print() +import os +gguf_files = [file for file in os.listdir() if file.endswith('.gguf')] +if gguf_files: + print('GGUF file(s) available. Select which one to read:') + for index, file_name in enumerate(gguf_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(gguf_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = gguf_files[choice_index] + print(f'Model file: {selected_file} is selected!') + ModelPath = selected_file + read_gguf_file(ModelPath) + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No GGUF files are available in the current directory.') + input('--- Press ENTER To Exit ---') \ No newline at end of file diff --git a/gguf_connector/r3.py b/gguf_connector/r3.py new file mode 100644 index 0000000..0d5b1f0 --- /dev/null +++ b/gguf_connector/r3.py @@ -0,0 +1,30 @@ +from safetensors.torch import load_file +def tensor_reader(file_path): + try: + tensors = load_file(file_path) + tensor_info = {name: tensor.shape for name, tensor in tensors.items()} + return tensor_info + except Exception as e: + return f'Error reading tensor: {e}', {} +import os +files = [file for file in os.listdir() if file.endswith('.safetensors')] +if files: + print('Available .safetensors files:') + for idx, file in enumerate(files, 1): + print(f'{idx}. {file}') + choice = input('Enter the number of the file to read tensor info from: ') + try: + choice = int(choice) + if 1 <= choice <= len(files): + file_path = files[choice - 1] + tensor_info = tensor_reader(file_path) + print('Tensors:') + for name, shape in tensor_info.items(): + print(f' {name}: {shape}') + else: + print('Invalid selection.') + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No safetensors files are available in the current directory.') + input('--- Press ENTER To Exit ---') \ No newline at end of file diff --git a/gguf_connector/r4.py b/gguf_connector/r4.py new file mode 100644 index 0000000..e75e60d --- /dev/null +++ b/gguf_connector/r4.py @@ -0,0 +1,22 @@ +from safetensors import safe_open +import os +def list_safetensors_files(): + files = [f for f in os.listdir() if f.endswith('.safetensors')] + if not files: + print('No .safetensors files found in the current directory.') + return None + print('Select a .safetensors file to split:') + for idx, file in enumerate(files, 1): + print(f'{idx}: {file}') + while True: + try: + choice = int(input('Enter number: ')) + if 1 <= choice <= len(files): + return files[choice - 1] + except ValueError: + pass + print('Invalid selection. Try again.') +selected_file = list_safetensors_files() +if selected_file: + with safe_open(selected_file, framework='pt') as f: + print(f.metadata()) \ No newline at end of file diff --git a/gguf_connector/reader.py b/gguf_connector/reader.py new file mode 100644 index 0000000..c924fbc --- /dev/null +++ b/gguf_connector/reader.py @@ -0,0 +1,256 @@ +from __future__ import annotations +import logging, os, sys +from collections import OrderedDict +from typing import Any, Literal, NamedTuple, TypeVar, Union +import numpy as np +import numpy.typing as npt +import io +from .quant import quant_shape_to_byte_shape + + +if __name__ == '__main__': + from pathlib import Path + sys.path.insert(0, str(Path(__file__).parent.parent)) +from .const import GGML_QUANT_SIZES, GGUF_DEFAULT_ALIGNMENT, GGUF_MAGIC, GGUF_VERSION, GGMLQuantizationType, GGUFValueType, GGUFEndian +logger = logging.getLogger(__name__) +READER_SUPPORTED_VERSIONS = [2, GGUF_VERSION] +class ReaderField(NamedTuple): + offset: int + name: str + parts: list[npt.NDArray[Any]] = [] + data: list[int] = [-1] + types: list[GGUFValueType] = [] + def contents(self, index_or_slice=slice(None)): + if self.types: + to_string = lambda x: str(x.tobytes(), encoding='utf-8') + main_type = self.types[0] + if main_type == GGUFValueType.ARRAY: + sub_type = self.types[-1] + if sub_type == GGUFValueType.STRING: + indices = self.data[index_or_slice] + if isinstance(index_or_slice, int): + return to_string(self.parts[indices]) + else: + return [to_string(self.parts[idx]) for idx in indices] + elif isinstance(index_or_slice, int): + return self.parts[self.data[index_or_slice]].tolist()[0] + else: + return [pv for idx in self.data[index_or_slice] for pv in + self.parts[idx].tolist()] + if main_type == GGUFValueType.STRING: + return to_string(self.parts[-1]) + else: + return self.parts[-1].tolist()[0] + return None +class ReaderTensor(NamedTuple): + name: str + tensor_type: GGMLQuantizationType + shape: npt.NDArray[np.uint32] + n_elements: int + n_bytes: int + data_offset: int + data: npt.NDArray[Any] + field: ReaderField +class GGUFReader: + byte_order: Literal['I', 'S'] = 'I' + alignment: int = GGUF_DEFAULT_ALIGNMENT + data_offset: int + gguf_scalar_to_np: dict[GGUFValueType, type[np.generic]] = {GGUFValueType + .UINT8: np.uint8, GGUFValueType.INT8: np.int8, GGUFValueType.UINT16: + np.uint16, GGUFValueType.INT16: np.int16, GGUFValueType.UINT32: np. + uint32, GGUFValueType.INT32: np.int32, GGUFValueType.FLOAT32: np. + float32, GGUFValueType.UINT64: np.uint64, GGUFValueType.INT64: np. + int64, GGUFValueType.FLOAT64: np.float64, GGUFValueType.BOOL: np.bool_} + def __init__(self, path, mode='r'): + if isinstance(path, io.BytesIO): + path.seek(0) + # print("bytes length: {}".format(len(path))) + self.data = np.frombuffer(path.read(), dtype=np.byte) + else: + self.data = np.memmap(path, mode=mode) + + offs = 0 + if self._get(offs, np.uint32, override_order='<')[0] != GGUF_MAGIC: + raise ValueError('GGUF magic invalid') + offs += 4 + + temp_version = self._get(offs, np.uint32) + + if temp_version[0] & 65535 == 0: + self.byte_order = 'S' + temp_version = temp_version.newbyteorder(self.byte_order) + + version = temp_version[0] + if version not in READER_SUPPORTED_VERSIONS: + raise ValueError( + f'Sorry, file appears to be version {version} which we cannot handle' + ) + if sys.byteorder == 'little': + host_endian = GGUFEndian.LITTLE + swapped_endian = GGUFEndian.BIG + else: + host_endian = GGUFEndian.BIG + swapped_endian = GGUFEndian.LITTLE + self.endianess = (swapped_endian if self.byte_order == 'S' else + host_endian) + self.fields: OrderedDict[str, ReaderField] = OrderedDict() + self.tensors: list[ReaderTensor] = [] + offs += self._push_field(ReaderField(offs, 'GGUF.version', [ + temp_version], [0], [GGUFValueType.UINT32])) + temp_counts = self._get(offs, np.uint64, 2) + offs += self._push_field(ReaderField(offs, 'GGUF.tensor_count', [ + temp_counts[:1]], [0], [GGUFValueType.UINT64])) + offs += self._push_field(ReaderField(offs, 'GGUF.kv_count', [ + temp_counts[1:]], [0], [GGUFValueType.UINT64])) + tensor_count, kv_count = temp_counts + offs = self._build_fields(offs, kv_count) + offs, tensors_fields = self._build_tensor_info(offs, tensor_count) + new_align = self.fields.get('general.alignment') + if new_align is not None: + if new_align.types != [GGUFValueType.UINT32]: + raise ValueError('Bad type for general.alignment field') + self.alignment = new_align.parts[-1][0] + padding = offs % self.alignment + if padding != 0: + offs += self.alignment - padding + self.data_offset = offs + self._build_tensors(offs, tensors_fields) + _DT = TypeVar('_DT', bound=npt.DTypeLike) + def get_field(self, key): + return self.fields.get(key, None) + def get_tensor(self, idx): + return self.tensors[idx] + def _get(self, offset, dtype, count=1, override_order=None): + count = int(count) + itemsize = int(np.empty([], dtype=dtype).itemsize) + end_offs = offset + itemsize * count + return self.data[offset:end_offs].view(dtype=dtype)[:count] + def _push_field(self, field, skip_sum=False): + if field.name in self.fields: + logger.warning( + f'Duplicate key {field.name} at offset {field.offset}') + self.fields[field.name + '_{}'.format(field.offset)] = field + else: + self.fields[field.name] = field + return 0 if skip_sum else sum(int(part.nbytes) for part in field.parts) + def _get_str(self, offset): + slen = self._get(offset, np.uint64) + return slen, self._get(offset + 8, np.uint8, slen[0]) + def _get_field_parts(self, orig_offs, raw_type): + offs = orig_offs + types: list[GGUFValueType] = [] + gtype = GGUFValueType(raw_type) + types.append(gtype) + if gtype == GGUFValueType.STRING: + sparts: list[npt.NDArray[Any]] = list(self._get_str(offs)) + size = sum(int(part.nbytes) for part in sparts) + return size, sparts, [1], types + nptype = self.gguf_scalar_to_np.get(gtype) + if nptype is not None: + val = self._get(offs, nptype) + return int(val.nbytes), [val], [0], types + if gtype == GGUFValueType.ARRAY: + raw_itype = self._get(offs, np.uint32) + offs += int(raw_itype.nbytes) + alen = self._get(offs, np.uint64) + offs += int(alen.nbytes) + aparts: list[npt.NDArray[Any]] = [raw_itype, alen] + data_idxs: list[int] = [] + for idx in range(alen[0]): + curr_size, curr_parts, curr_idxs, curr_types = (self. + _get_field_parts(offs, raw_itype[0])) + if idx == 0: + types += curr_types + idxs_offs = len(aparts) + aparts += curr_parts + data_idxs += (idx + idxs_offs for idx in curr_idxs) + offs += curr_size + return offs - orig_offs, aparts, data_idxs, types + raise ValueError('Unknown/unhandled field type {gtype}') + def _get_tensor_info_field(self, orig_offs): + offs = orig_offs + name_len, name_data = self._get_str(offs) + offs += int(name_len.nbytes + name_data.nbytes) + n_dims = self._get(offs, np.uint32) + offs += int(n_dims.nbytes) + dims = self._get(offs, np.uint64, n_dims[0]) + offs += int(dims.nbytes) + raw_dtype = self._get(offs, np.uint32) + offs += int(raw_dtype.nbytes) + offset_tensor = self._get(offs, np.uint64) + offs += int(offset_tensor.nbytes) + return ReaderField(orig_offs, str(bytes(name_data), encoding= + 'utf-8'), [name_len, name_data, n_dims, dims, raw_dtype, + offset_tensor], [1, 3, 4, 5]) + def _build_fields(self, offs, count): + for _ in range(count): + orig_offs = offs + kv_klen, kv_kdata = self._get_str(offs) + offs += int(kv_klen.nbytes + kv_kdata.nbytes) + raw_kv_type = self._get(offs, np.uint32) + offs += int(raw_kv_type.nbytes) + parts: list[npt.NDArray[Any]] = [kv_klen, kv_kdata, raw_kv_type] + idxs_offs = len(parts) + field_size, field_parts, field_idxs, field_types = (self. + _get_field_parts(offs, raw_kv_type[0])) + parts += field_parts + self._push_field(ReaderField(orig_offs, str(bytes(kv_kdata), + encoding='utf-8'), parts, [(idx + idxs_offs) for idx in + field_idxs], field_types), skip_sum=True) + offs += field_size + return offs + def _build_tensor_info(self, offs, count): + tensor_fields = [] + for _ in range(count): + field = self._get_tensor_info_field(offs) + offs += sum(int(part.nbytes) for part in field.parts) + tensor_fields.append(field) + return offs, tensor_fields + def _build_tensors(self, start_offs, fields): + tensors = [] + tensor_names = set() + for field in fields: + (_name_len, name_data, _n_dims, dims, raw_dtype, offset_tensor + ) = field.parts + tensor_name = str(bytes(name_data), encoding='utf-8') + if tensor_name in tensor_names: + raise ValueError( + f'Found duplicated tensor with name {tensor_name}') + tensor_names.add(tensor_name) + ggml_type = GGMLQuantizationType(raw_dtype[0]) + n_elems = int(np.prod(dims)) + np_dims = tuple(reversed(dims.tolist())) + block_size, type_size = GGML_QUANT_SIZES[ggml_type] + n_bytes = n_elems * type_size // block_size + data_offs = int(start_offs + offset_tensor[0]) + item_type: npt.DTypeLike + if ggml_type == GGMLQuantizationType.F16: + item_count = n_elems + item_type = np.float16 + elif ggml_type == GGMLQuantizationType.F32: + item_count = n_elems + item_type = np.float32 + elif ggml_type == GGMLQuantizationType.F64: + item_count = n_elems + item_type = np.float64 + elif ggml_type == GGMLQuantizationType.I8: + item_count = n_elems + item_type = np.int8 + elif ggml_type == GGMLQuantizationType.I16: + item_count = n_elems + item_type = np.int16 + elif ggml_type == GGMLQuantizationType.I32: + item_count = n_elems + item_type = np.int32 + elif ggml_type == GGMLQuantizationType.I64: + item_count = n_elems + item_type = np.int64 + else: + item_count = n_bytes + item_type = np.uint8 + np_dims = quant_shape_to_byte_shape(np_dims, ggml_type) + tensors.append(ReaderTensor(name=tensor_name, tensor_type= + ggml_type, shape=dims, n_elements=n_elems, n_bytes=n_bytes, + data_offset=data_offs, data=self._get(data_offs, item_type, + item_count).reshape(np_dims), field=field)) + self.tensors = tensors diff --git a/gguf_connector/reader2.py b/gguf_connector/reader2.py new file mode 100644 index 0000000..7bc7873 --- /dev/null +++ b/gguf_connector/reader2.py @@ -0,0 +1,107 @@ +import struct +class ReaderError(Exception): + """Exception raised for errors in GGUF reader""" +class GGUFReader: + GGUF_FORMAT = b'GGUF' + VALUE_FORMATS = {(0): 'B', (1): 'b', (2): 'H', (3): 'h', (4): 'I', (5): + 'i', (6): 'f', (7): '?', (10): 'Q', (11): 'q', (12): 'd'} + TENSOR_TYPES = {(0): 'GGML_TYPE_F32', (1): 'GGML_TYPE_F16', (2): + 'GGML_TYPE_Q4_0', (3): 'GGML_TYPE_Q4_1', (4): + 'GGML_TYPE_Q4_1_SOME_F16', (5): 'GGML_TYPE_TYPE_Q4_2', (6): + 'GGML_TYPE_Q5_0', (7): 'GGML_TYPE_Q5_1', (8): 'GGML_TYPE_Q8_0', (9): + 'GGML_TYPE_Q8_1', (10): 'GGML_TYPE_Q2_K', (11): 'GGML_TYPE_Q3_K', ( + 12): 'GGML_TYPE_Q4_K', (13): 'GGML_TYPE_Q5_K', (14): + 'GGML_TYPE_Q6_K', (15): 'GGML_TYPE_Q8_K', (16): 'GGML_TYPE_IQ2_XXS', + (17): 'GGML_TYPE_IQ2_XS', (18): 'GGML_TYPE_IQ3_XXS', (19): + 'GGML_TYPE_IQ1_S', (20): 'GGML_TYPE_IQ4_NL', (21): + 'GGML_TYPE_IQ3_S', (22): 'GGML_TYPE_IQ2_S', (23): + 'GGML_TYPE_IQ4_XS', (24): 'GGML_TYPE_I8', (25): 'GGML_TYPE_I16', ( + 26): 'GGML_TYPE_I32', (27): 'GGML_TYPE_I64', (28): 'GGML_TYPE_F64', + (29): 'GGML_TYPE_IQ1_M', (30): 'GGML_TYPE_BF16', (32): + 'GGML_TYPE_BF16', (34): 'GGML_TYPE__TQ1_0', (35): 'GGML_TYPE__TQ2_0'} + def __init__(self, file_path): + """Initialize the GGUF reader""" + self.file_path = file_path + self.version = None + self.format = None + self.tensors_info = None + self.metadata = None + self.alignment = None + def read(self): + """Read the GGUF file.""" + with open(self.file_path, 'rb') as f: + self.format = f.read(4) + if self.format != self.GGUF_FORMAT: + raise ReaderError('Invalid format') + self.version = struct.unpack('I', f.read(4))[0] + tensor_count = struct.unpack('Q', f.read(8))[0] + metadata_kv_count = struct.unpack('Q', f.read(8))[0] + self.metadata = {} + for _ in range(metadata_kv_count): + key, value = self._read_metadata_kv(f) + self.metadata[key] = value + self.alignment = self.metadata.get('general.alignment', 1) + self.tensors_info = [] + for _ in range(tensor_count): + tensor_info = self._read_tensor_info(f) + self.tensors_info.append(tensor_info) + def _read_string(self, f): + """Read a string from the file""" + length = struct.unpack('Q', f.read(8))[0] + return f.read(length).decode('utf-8') + def _read_metadata_kv(self, f): + """Read a metadata key-value pair from the file""" + key = self._read_string(f) + value_type = struct.unpack('I', f.read(4))[0] + value = self._read_value(f, value_type) + return key, value + def _read_value(self, f, value_type): + """Read a value of the given type from the file""" + if value_type in self.VALUE_FORMATS: + return struct.unpack(self.VALUE_FORMATS[value_type], f.read( + struct.calcsize(self.VALUE_FORMATS[value_type])))[0] + if value_type == 8: + return self._read_string(f) + if value_type == 9: + array_type = struct.unpack('I', f.read(4))[0] + array_len = struct.unpack('Q', f.read(8))[0] + return [self._read_value(f, array_type) for _ in range(array_len)] + raise ReaderError('Unsupported value type') + def _read_tensor_info(self, f): + """Read tensor information from the file""" + name = self._read_string(f) + n_dimensions = struct.unpack('I', f.read(4))[0] + dimensions = struct.unpack(f'{n_dimensions}Q', f.read(8 * n_dimensions) + ) + tensor_type = struct.unpack('I', f.read(4))[0] + offset = struct.unpack('Q', f.read(8))[0] + return {'name': name, 'n_dimensions': n_dimensions, 'dimensions': + dimensions, 'type': tensor_type, 'offset': offset} + def load_tensors(self): + """Load the tensors from the file""" + tensors = [] + with open(self.file_path, 'rb') as f: + for tensor_info in self.tensors_info: + f.seek(tensor_info['offset']) + tensor_data = f.read(tensor_info['n_dimensions']) + if f.tell() % self.alignment != 0: + f.read(self.alignment - f.tell() % self.alignment) + tensors.append(tensor_data) + return tensors + def print(self): + """Print the file details""" + print(f'Version: {self.version}') + print(f'Format: {self.format}') + print('Tensors Info:') + for tensor_info in self.tensors_info: + print( + f" Name: {tensor_info['name']},\tShape: {tensor_info['dimensions']},\tType: {self.TENSOR_TYPES[tensor_info['type']]},\tOffset: {tensor_info['offset']}" + ) + print('Metadata:') + for key, value in self.metadata.items(): + if isinstance(value, list) and len(value) > 50: + print( + f' {key}: {value[:50]}... ({len(value) - 50} more elements)' + ) + else: + print(f' {key}: {value}') \ No newline at end of file diff --git a/gguf_connector/s.py b/gguf_connector/s.py new file mode 100644 index 0000000..abc09bf --- /dev/null +++ b/gguf_connector/s.py @@ -0,0 +1,44 @@ +from safetensors.torch import load_file, save_file +import os +def split_safetensors_file(input_path, output_dir): + if not os.path.exists(output_dir): + os.makedirs(output_dir) + tensors = load_file(input_path) + components = {'model': {}, 'encoder': {}, 'vae': {}} + print(f'Splitting {input_path} into components...') + for key, tensor in tensors.items(): + for comp in components: + if key.startswith(comp): + if comp == 'vae': + new_key = key[len('vae.'):] if key.startswith('vae.' + ) else key + components[comp][new_key] = tensors[key] + else: + components[comp][key] = tensors[key] + break + for comp_name, comp_tensors in components.items(): + if comp_tensors: + output_path = os.path.join(output_dir, f'{comp_name}.safetensors') + save_file(comp_tensors, output_path) + print( + f'✅ Saved {comp_name} to {output_path} ({len(comp_tensors)} tensors)' + ) + else: + print(f'⚠️ No tensors found for component: {comp_name}') +safetensors_files = [file for file in os.listdir() if file.endswith( + '.safetensors')] +if safetensors_files: + print('Safetensors file(s) available. Select which one to split:') + for index, file_name in enumerate(safetensors_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = safetensors_files[choice_index] + print(f'Model file: {selected_file} is selected!') + split_safetensors_file(selected_file, 'split_output') + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No safetensors files are available in the current directory.') + input('--- Press ENTER To Exit ---') \ No newline at end of file diff --git a/gguf_connector/s0.py b/gguf_connector/s0.py new file mode 100644 index 0000000..39df53e --- /dev/null +++ b/gguf_connector/s0.py @@ -0,0 +1,43 @@ +from safetensors.torch import load_file, save_file +from tqdm import tqdm +import os +def list_safetensors_files(): + files = [f for f in os.listdir() if f.endswith('.safetensors')] + if not files: + print('No .safetensors files found in the current directory.') + return None + print('Select a .safetensors file to split:') + for idx, file in enumerate(files, 1): + print(f'{idx}: {file}') + while True: + try: + choice = int(input('Enter number: ')) + if 1 <= choice <= len(files): + return files[choice - 1] + except ValueError: + pass + print('Invalid selection. Try again.') +def split_safetensors(file_path): + tensors = load_file(file_path) + dim2 = {} + dim3 = {} + dim4 = {} + dim5 = {} + for name in tqdm(tensors, desc='Splitting tensors'): + tensor = tensors[name] + if tensor.dim() <= 2: + dim2[name] = tensor + elif tensor.dim() == 3: + dim3[name] = tensor + elif tensor.dim() == 4: + dim4[name] = tensor + else: + dim5[name] = tensor + save_file(dim2, '2d.safetensors') + save_file(dim3, '3d.safetensors') + save_file(dim4, '4d.safetensors') + save_file(dim5, '5d.safetensors') + print('Saved: 2d, 3d, 4d and 5d.safetensors') +selected_file = list_safetensors_files() +if selected_file: + split_safetensors(selected_file) \ No newline at end of file diff --git a/gguf_connector/s1.py b/gguf_connector/s1.py new file mode 100644 index 0000000..ce08f10 --- /dev/null +++ b/gguf_connector/s1.py @@ -0,0 +1,35 @@ +from safetensors.torch import load_file, save_file +from tqdm import tqdm +import os +def list_safetensors_files(): + files = [f for f in os.listdir() if f.endswith('.safetensors')] + if not files: + print('No .safetensors files found in the current directory.') + return None + print('Select a .safetensors file to split:') + for idx, file in enumerate(files, 1): + print(f'{idx}: {file}') + while True: + try: + choice = int(input('Enter number: ')) + if 1 <= choice <= len(files): + return files[choice - 1] + except ValueError: + pass + print('Invalid selection. Try again.') +def split_safetensors(file_path): + tensors = load_file(file_path) + unidim = {} + multidim = {} + for name in tqdm(tensors, desc='Splitting tensors'): + tensor = tensors[name] + if tensor.dim() == 1: + unidim[name] = tensor + else: + multidim[name] = tensor + save_file(unidim, 'unidimension.safetensors') + save_file(multidim, 'multidimension.safetensors') + print('Saved: unidimension.safetensors and multidimension.safetensors') +selected_file = list_safetensors_files() +if selected_file: + split_safetensors(selected_file) \ No newline at end of file diff --git a/gguf_connector/s2.py b/gguf_connector/s2.py new file mode 100644 index 0000000..df9008f --- /dev/null +++ b/gguf_connector/s2.py @@ -0,0 +1,195 @@ +import torch +import tempfile, time +import gradio as gr +import numpy as np +import soundfile as sf +from pathlib import Path +from typing import Optional, Tuple +from diao.dia.model import Dia +if torch.cuda.is_available(): + device = torch.device('cuda') +elif hasattr(torch.backends, 'mps') and torch.backends.mps.is_available(): + device = torch.device('mps') +else: + device = torch.device('cpu') +print(f'Using device: {device}') +print('Loading model...') +try: + model = Dia.from_pretrained('callgg/dia-f16', compute_dtype='float16', + device=device) +except Exception as e: + print(f'Error loading Nari model: {e}') + raise +def run_inference(text_input, audio_prompt_input, max_new_tokens, cfg_scale, + temperature, top_p, cfg_filter_top_k, speed_factor): + """ + Runs inference using the globally loaded model and provided inputs. + """ + global model, device + if not text_input or text_input.isspace(): + raise gr.Error('Text input cannot be empty.') + temp_txt_file_path = None + temp_audio_prompt_path = None + output_audio = 44100, np.zeros(1, dtype=np.float32) + try: + prompt_path_for_generate = None + if audio_prompt_input is not None: + sr, audio_data = audio_prompt_input + if audio_data is None or audio_data.size == 0 or audio_data.max( + ) == 0: + gr.Warning( + 'Audio prompt seems empty or silent, ignoring prompt.') + else: + with tempfile.NamedTemporaryFile(mode='wb', suffix='.wav', + delete=False) as f_audio: + temp_audio_prompt_path = f_audio.name + if np.issubdtype(audio_data.dtype, np.integer): + max_val = np.iinfo(audio_data.dtype).max + audio_data = audio_data.astype(np.float32) / max_val + elif not np.issubdtype(audio_data.dtype, np.floating): + gr.Warning( + f'Unsupported audio prompt dtype {audio_data.dtype}, attempting conversion.' + ) + try: + audio_data = audio_data.astype(np.float32) + except Exception as conv_e: + raise gr.Error( + f'Failed to convert audio prompt to float32: {conv_e}' + ) + if audio_data.ndim > 1: + if audio_data.shape[0] == 2: + audio_data = np.mean(audio_data, axis=0) + elif audio_data.shape[1] == 2: + audio_data = np.mean(audio_data, axis=1) + else: + gr.Warning( + f'Audio prompt has unexpected shape {audio_data.shape}, taking first channel/axis.' + ) + audio_data = audio_data[0] if audio_data.shape[0 + ] < audio_data.shape[1] else audio_data[:, 0] + audio_data = np.ascontiguousarray(audio_data) + try: + sf.write(temp_audio_prompt_path, audio_data, sr, + subtype='FLOAT') + prompt_path_for_generate = temp_audio_prompt_path + print( + f'Created temporary audio prompt file: {temp_audio_prompt_path} (orig sr: {sr})' + ) + except Exception as write_e: + print(f'Error writing temporary audio file: {write_e}') + raise gr.Error( + f'Failed to save audio prompt: {write_e}') + start_time = time.time() + with torch.inference_mode(): + output_audio_np = model.generate(text_input, max_tokens= + max_new_tokens, cfg_scale=cfg_scale, temperature= + temperature, top_p=top_p, cfg_filter_top_k=cfg_filter_top_k, + use_torch_compile=False, audio_prompt=prompt_path_for_generate) + end_time = time.time() + print(f'Generation finished in {end_time - start_time:.2f} seconds.') + if output_audio_np is not None: + output_sr = 44100 + original_len = len(output_audio_np) + speed_factor = max(0.1, min(speed_factor, 5.0)) + target_len = int(original_len / speed_factor) + if target_len != original_len and target_len > 0: + x_original = np.arange(original_len) + x_resampled = np.linspace(0, original_len - 1, target_len) + resampled_audio_np = np.interp(x_resampled, x_original, + output_audio_np) + output_audio = output_sr, resampled_audio_np.astype(np.float32) + print( + f'Resampled audio from {original_len} to {target_len} samples for {speed_factor:.2f}x speed.' + ) + else: + output_audio = output_sr, output_audio_np + print( + f'Skipping audio speed adjustment (factor: {speed_factor:.2f}).' + ) + print( + f'Audio conversion successful. Final shape: {output_audio[1].shape}, Sample Rate: {output_sr}' + ) + if output_audio[1].dtype == np.float32 or output_audio[1 + ].dtype == np.float64: + audio_for_gradio = np.clip(output_audio[1], -1.0, 1.0) + audio_for_gradio = (audio_for_gradio * 32767).astype(np.int16) + output_audio = output_sr, audio_for_gradio + print('Converted audio to int16 for Gradio output.') + else: + print('\nGeneration finished, but no valid tokens were produced.') + gr.Warning('Generation produced no output.') + except Exception as e: + print(f'Error during inference: {e}') + import traceback + traceback.print_exc() + raise gr.Error(f'Inference failed: {e}') + finally: + if temp_txt_file_path and Path(temp_txt_file_path).exists(): + try: + Path(temp_txt_file_path).unlink() + print(f'Deleted temporary text file: {temp_txt_file_path}') + except OSError as e: + print( + f'Warning: Error deleting temporary text file {temp_txt_file_path}: {e}' + ) + if temp_audio_prompt_path and Path(temp_audio_prompt_path).exists(): + try: + Path(temp_audio_prompt_path).unlink() + print( + f'Deleted temporary audio prompt file: {temp_audio_prompt_path}' + ) + except OSError as e: + print( + f'Warning: Error deleting temporary audio prompt file {temp_audio_prompt_path}: {e}' + ) + return output_audio +css = """ +""" +default_text = """[S1] This is an open weights text to dialogue model. +[S2] You get full control over scripts and voices. +[S1] Wow. Amazing. (laughs) +[S2] Try it now on Git hub or Hugging Face.""" +block = gr.Blocks(title='gguf', css=css).queue() +with block: + gr.Markdown('## 🎤 Text-to-Speech Synthesis') + with gr.Row(equal_height=False): + with gr.Column(scale=1): + text_input = gr.Textbox(label='Input Text', placeholder= + 'Enter text here...', value=default_text, lines=5) + audio_prompt_input = gr.Audio(label='Audio Prompt (Optional)', + show_label=True, sources=['upload', 'microphone'], type='numpy' + ) + with gr.Accordion('Generation Parameters', open=False): + max_new_tokens = gr.Slider(label= + 'Max New Tokens (Audio Length)', minimum=860, maximum= + 3072, value=model.config.data.audio_length, step=50, + info= + 'Controls the maximum length of the generated audio (more tokens = longer audio).' + ) + cfg_scale = gr.Slider(label='CFG Scale (Guidance Strength)', + minimum=1.0, maximum=5.0, value=3.0, step=0.1, info= + 'Higher values increase adherence to the text prompt.') + temperature = gr.Slider(label='Temperature (Randomness)', + minimum=1.0, maximum=1.5, value=1.3, step=0.05, info= + 'Lower values make the output more deterministic, higher values increase randomness.' + ) + top_p = gr.Slider(label='Top P (Nucleus Sampling)', minimum + =0.8, maximum=1.0, value=0.95, step=0.01, info= + 'Filters vocabulary to the most likely tokens cumulatively reaching probability P.' + ) + cfg_filter_top_k = gr.Slider(label='CFG Filter Top K', + minimum=15, maximum=50, value=30, step=1, info= + 'Top k filter for CFG guidance.') + speed_factor_slider = gr.Slider(label='Speed Factor', + minimum=0.8, maximum=1.0, value=0.94, step=0.02, info= + 'Adjusts the speed of the generated audio (1.0 = original speed).' + ) + run_button = gr.Button('Generate Audio', variant='primary') + with gr.Column(scale=1): + audio_output = gr.Audio(label='Generated Audio', type='numpy', + autoplay=False) + run_button.click(fn=run_inference, inputs=[text_input, + audio_prompt_input, max_new_tokens, cfg_scale, temperature, top_p, + cfg_filter_top_k, speed_factor_slider], outputs=[audio_output], + api_name='generate_audio') +block.launch() \ No newline at end of file diff --git a/gguf_connector/s5.py b/gguf_connector/s5.py new file mode 100644 index 0000000..8d8d8cd --- /dev/null +++ b/gguf_connector/s5.py @@ -0,0 +1,35 @@ +from safetensors.torch import load_file, save_file +from tqdm import tqdm +import os +def list_safetensors_files(): + files = [f for f in os.listdir() if f.endswith('.safetensors')] + if not files: + print('No .safetensors files found in the current directory.') + return None + print('Select a .safetensors file to split:') + for idx, file in enumerate(files, 1): + print(f'{idx}: {file}') + while True: + try: + choice = int(input('Enter number: ')) + if 1 <= choice <= len(files): + return files[choice - 1] + except ValueError: + pass + print('Invalid selection. Try again.') +def split_safetensors(file_path): + tensors = load_file(file_path) + unidim = {} + multidim = {} + for name in tqdm(tensors, desc='Splitting tensors'): + tensor = tensors[name] + if tensor.dim() <= 4: + unidim[name] = tensor + else: + multidim[name] = tensor + save_file(unidim, 'except5d.safetensors') + save_file(multidim, '5d.safetensors') + print('Saved: except5d.safetensors and 5d.safetensors') +selected_file = list_safetensors_files() +if selected_file: + split_safetensors(selected_file) \ No newline at end of file diff --git a/gguf_connector/sx.py b/gguf_connector/sx.py new file mode 100644 index 0000000..2d01b57 --- /dev/null +++ b/gguf_connector/sx.py @@ -0,0 +1,51 @@ +from safetensors.torch import load_file, save_file +from tqdm import tqdm +import os +def list_safetensors_files(): + files = [f for f in os.listdir() if f.endswith('.safetensors')] + if not files: + print('No .safetensors files found in the current directory.') + return None + print('Select a .safetensors file to split:') + for idx, file in enumerate(files, 1): + print(f'{idx}: {file}') + while True: + try: + choice = int(input('Enter number: ')) + if 1 <= choice <= len(files): + return files[choice - 1] + except ValueError: + pass + print('Invalid selection. Try again.') +def split_safetensors(file_path): + tensors = load_file(file_path) + dim1 = {} + dim2 = {} + dim3 = {} + dim4 = {} + dim5 = {} + dimx = {} + for name in tqdm(tensors, desc='Splitting tensors'): + tensor = tensors[name] + if tensor.dim() == 1: + dim1[name] = tensor + elif tensor.dim() == 2: + dim2[name] = tensor + elif tensor.dim() == 3: + dim3[name] = tensor + elif tensor.dim() == 4: + dim4[name] = tensor + elif tensor.dim() == 5: + dim5[name] = tensor + else: + dimx[name] = tensor + save_file(dim1, '1d.safetensors') + save_file(dim2, '2d.safetensors') + save_file(dim3, '3d.safetensors') + save_file(dim4, '4d.safetensors') + save_file(dim5, '5d.safetensors') + save_file(dimx, 'xd.safetensors') + print('Saved: 1d, 2d, 3d, 4d, 5d and xd.safetensors') +selected_file = list_safetensors_files() +if selected_file: + split_safetensors(selected_file) \ No newline at end of file diff --git a/gguf_connector/sy.py b/gguf_connector/sy.py new file mode 100644 index 0000000..7e07d67 --- /dev/null +++ b/gguf_connector/sy.py @@ -0,0 +1,35 @@ +from safetensors.torch import load_file, save_file +from tqdm import tqdm +import os +def list_safetensors_files(): + files = [f for f in os.listdir() if f.endswith('.safetensors')] + if not files: + print('No .safetensors files found in the current directory.') + return None + print('Select a .safetensors file to split:') + for idx, file in enumerate(files, 1): + print(f'{idx}: {file}') + while True: + try: + choice = int(input('Enter number: ')) + if 1 <= choice <= len(files): + return files[choice - 1] + except ValueError: + pass + print('Invalid selection. Try again.') +def split_safetensors(file_path): + tensors = load_file(file_path) + dim2 = {} + dimy = {} + for name in tqdm(tensors, desc='Splitting tensors'): + tensor = tensors[name] + if tensor.dim() == 2: + dim2[name] = tensor + else: + dimy[name] = tensor + save_file(dim2, 'tensor2d.safetensors') + save_file(dimy, 'except2d.safetensors') + print('Saved: except2d.safetensors and tensor2d.safetensors') +selected_file = list_safetensors_files() +if selected_file: + split_safetensors(selected_file) \ No newline at end of file diff --git a/gguf_connector/t.py b/gguf_connector/t.py new file mode 100644 index 0000000..6c529dd --- /dev/null +++ b/gguf_connector/t.py @@ -0,0 +1,195 @@ +import torch +from .writer import GGUFWriter, GGMLQuantizationType +from .quant import quantize, QuantError +from .const import GGML_QUANT_VERSION, LlamaFileType +from safetensors.torch import load_file +from tqdm import tqdm +QUANTIZATION_THRESHOLD = 1024 +REARRANGE_THRESHOLD = 512 +MAX_TENSOR_NAME_LENGTH = 127 +class ModelTemplate: + arch = 'invalid' + shape_fix = False + keys_detect = [] + keys_banned = [] +class ModelSDXL(ModelTemplate): + arch = 'sdxl' + shape_fix = True + keys_detect = [('down_blocks.0.downsamplers.0.conv.weight', + 'add_embedding.linear_1.weight'), ('input_blocks.3.0.op.weight', + 'input_blocks.6.0.op.weight', 'output_blocks.2.2.conv.weight', + 'output_blocks.5.2.conv.weight'), ('label_emb.0.0.weight',)] +class ModelSD1(ModelTemplate): + arch = 'sd1' + shape_fix = True + keys_detect = [('down_blocks.0.downsamplers.0.conv.weight',), ( + 'input_blocks.3.0.op.weight', 'input_blocks.6.0.op.weight', + 'input_blocks.9.0.op.weight', 'output_blocks.2.1.conv.weight', + 'output_blocks.5.2.conv.weight', 'output_blocks.8.2.conv.weight')] +class ModelSD3(ModelTemplate): + arch = 'sd3' + keys_detect = [('transformer_blocks.0.attn.add_q_proj.weight',), ( + 'joint_blocks.0.x_block.attn.qkv.weight',)] + keys_banned = ['transformer_blocks.0.attn.add_q_proj.weight'] +class ModelFlux(ModelTemplate): + arch = 'flux' + keys_detect = [('transformer_blocks.0.attn.norm_added_k.weight',), ( + 'double_blocks.0.img_attn.proj.weight',)] + keys_banned = ['transformer_blocks.0.attn.norm_added_k.weight'] +class ModelAura(ModelTemplate): + arch = 'aura' + keys_detect = [('double_layers.3.modX.1.weight',), ( + 'joint_transformer_blocks.3.ff_context.out_projection.weight',)] + keys_banned = [ + 'joint_transformer_blocks.3.ff_context.out_projection.weight'] +class ModelHYVID(ModelTemplate): + arch = 'hyvid' + keys_detect = [('txt_in.individual_token_refiner.blocks.0.norm1.weight',)] +class ModelCosmos(ModelTemplate): + arch = 'cosmos' + keys_detect = [('blocks.block0.blocks.0.block.attn.to_q.0.weight',)] +class ModelPixArt(ModelTemplate): + arch = 'pixart' + keys_detect = [('transformer_blocks.27.scale_shift_table',)] +class ModelMochi(ModelTemplate): + arch = 'mochi' + keys_detect = [('t5_yproj.weight',)] +class ModelLumina(ModelTemplate): + arch = 'lumina' + keys_detect = [('cap_embedder.0.weight',)] +class ModelLTXV(ModelTemplate): + arch = 'ltxv' + keys_detect = [('adaln_single.emb.timestep_embedder.linear_2.weight', + 'transformer_blocks.27.scale_shift_table', + 'caption_projection.linear_2.weight')] +arch_list = [ModelSDXL, ModelSD1, ModelSD3, ModelAura, ModelFlux, + ModelHYVID, ModelCosmos, ModelPixArt, ModelMochi, ModelLumina, ModelLTXV] +def is_model_arch(model, state_dict): + matched = False + invalid = False + for match_list in model.keys_detect: + if all(key in state_dict for key in match_list): + matched = True + invalid = any(key in state_dict for key in model.keys_banned) + break + assert not invalid, 'Model architecture not allowed for conversion! (i.e. reference VS diffusers format)' + return matched +def detect_arch(state_dict): + model_arch = None + for arch in arch_list: + if is_model_arch(arch, state_dict): + model_arch = arch + break + assert model_arch is not None, 'Unknown model architecture!' + return model_arch +def load_state_dict(path): + if any(path.endswith(x) for x in ['.ckpt', '.pt', '.bin', '.pth']): + state_dict = torch.load(path, map_location='cpu', weights_only=True) + state_dict = state_dict.get('model', state_dict) + else: + state_dict = load_file(path) + prefix = None + for pfx in ['model.diffusion_model.', 'model.']: + if any([x.startswith(pfx) for x in state_dict.keys()]): + prefix = pfx + break + sd = {} + for k, v in state_dict.items(): + if prefix and prefix not in k: + continue + if prefix: + k = k.replace(prefix, '') + sd[k] = v + return sd +def load_model(path): + state_dict = load_state_dict(path) + model_arch = detect_arch(state_dict) + print(f'* Architecture detected from input: {model_arch.arch}') + writer = GGUFWriter(path=None, arch=model_arch.arch) + return writer, state_dict, model_arch +def handle_tensors(args, writer, state_dict, model_arch): + name_lengths = tuple(sorted(((key, len(key)) for key in state_dict.keys + ()), key=lambda item: item[1], reverse=True)) + if not name_lengths: + return + max_name_len = name_lengths[0][1] + if max_name_len > MAX_TENSOR_NAME_LENGTH: + bad_list = ', '.join(f'{key!r} ({namelen})' for key, namelen in + name_lengths if namelen > MAX_TENSOR_NAME_LENGTH) + raise ValueError( + f'Can only handle tensor names up to {MAX_TENSOR_NAME_LENGTH} characters. Tensors exceeding the limit: {bad_list}' + ) + for key, data in tqdm(state_dict.items()): + old_dtype = data.dtype + if data.dtype == torch.bfloat16: + data = data.to(torch.float32).numpy() + elif data.dtype in [getattr(torch, 'float8_e4m3fn', '_invalid'), + getattr(torch, 'float8_e5m2', '_invalid')]: + data = data.to(torch.float16).numpy() + else: + data = data.numpy() + n_dims = len(data.shape) + data_shape = data.shape + data_qtype = getattr(GGMLQuantizationType, 'BF16' if old_dtype == + torch.bfloat16 else 'F16') + n_params = 1 + for dim_size in data_shape: + n_params *= dim_size + blacklist = {'time_embedding.', 'add_embedding.', 'time_in.', + 'txt_in.', 'vector_in.', 'img_in.', 'guidance_in.', 'final_layer.'} + if old_dtype in (torch.float32, torch.bfloat16): + if n_dims == 1: + data_qtype = GGMLQuantizationType.F32 + elif n_params <= QUANTIZATION_THRESHOLD: + data_qtype = GGMLQuantizationType.F32 + elif '.weight' in key and any(x in key for x in blacklist): + data_qtype = GGMLQuantizationType.F32 + if (model_arch.shape_fix and n_dims > 1 and n_params >= + REARRANGE_THRESHOLD and (n_params / 256).is_integer() and not ( + data.shape[-1] / 256).is_integer()): + data = data.reshape(n_params // 256, 256) + try: + data = quantize(data, data_qtype) + except (AttributeError, QuantError) as e: + tqdm.write(f'falling back to F16: {e}') + data_qtype = GGMLQuantizationType.F16 + data = quantize(data, data_qtype) + new_name = key + shape_str = f"{{{', '.join(str(n) for n in reversed(data.shape))}}}" + tqdm.write( + f"{f'%-{max_name_len + 4}s' % f'{new_name}'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}" + ) + writer.add_tensor(new_name, data, raw_dtype=data_qtype) +import os +safetensors_files = [file for file in os.listdir() if file.endswith( + '.safetensors')] +if safetensors_files: + print('Safetensors file(s) available. Select which one to convert:') + for index, file_name in enumerate(safetensors_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = safetensors_files[choice_index] + print(f'Model file: {selected_file} is selected!') + path = selected_file + writer, state_dict, model_arch = load_model(path) + writer.add_quantization_version(GGML_QUANT_VERSION) + if next(iter(state_dict.values())).dtype == torch.bfloat16: + out_path = f'{os.path.splitext(path)[0]}-bf16.gguf' + writer.add_file_type(LlamaFileType.MOSTLY_BF16) + else: + out_path = f'{os.path.splitext(path)[0]}-f16.gguf' + writer.add_file_type(LlamaFileType.MOSTLY_F16) + if os.path.isfile(out_path): + input('Output exists enter to continue or ctrl+c to abort!') + handle_tensors(path, writer, state_dict, model_arch) + writer.write_header_to_file(path=out_path) + writer.write_kv_data_to_file() + writer.write_tensors_to_file(progress=True) + writer.close() + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No safetensors files are available in the current directory.') + input('--- Press ENTER To Exit ---') \ No newline at end of file diff --git a/gguf_connector/t0.py b/gguf_connector/t0.py new file mode 100644 index 0000000..db55199 --- /dev/null +++ b/gguf_connector/t0.py @@ -0,0 +1,106 @@ +import torch +from .writer import GGUFWriter, GGMLQuantizationType +from .quant import quantize, QuantError +from .const import GGML_QUANT_VERSION, LlamaFileType +from safetensors.torch import load_file +from tqdm import tqdm +QUANTIZATION_THRESHOLD = 1024 +REARRANGE_THRESHOLD = 512 +MAX_TENSOR_NAME_LENGTH = 127 +def load_state_dict(path): + state_dict = load_file(path) + sd = {} + for k, v in state_dict.items(): + sd[k] = v + return sd +def load_model(path, model_arch): + state_dict = load_state_dict(path) + writer = GGUFWriter(path=None, arch=model_arch) + return writer, state_dict, model_arch +def handle_tensors(args, writer, state_dict, model_arch): + name_lengths = tuple(sorted(((key, len(key)) for key in state_dict.keys + ()), key=lambda item: item[1], reverse=True)) + if not name_lengths: + return + max_name_len = name_lengths[0][1] + if max_name_len > MAX_TENSOR_NAME_LENGTH: + bad_list = ', '.join(f'{key!r} ({namelen})' for key, namelen in + name_lengths if namelen > MAX_TENSOR_NAME_LENGTH) + raise ValueError( + f'Can only handle tensor names up to {MAX_TENSOR_NAME_LENGTH} characters. Tensors exceeding the limit: {bad_list}' + ) + for key, data in tqdm(state_dict.items()): + old_dtype = data.dtype + if data.dtype == torch.bfloat16: + data = data.to(torch.float32).numpy() + elif data.dtype in [getattr(torch, 'float8_e4m3fn', '_invalid'), + getattr(torch, 'float8_e5m2', '_invalid')]: + data = data.to(torch.float16).numpy() + else: + data = data.numpy() + n_dims = len(data.shape) + data_shape = data.shape + data_qtype = getattr(GGMLQuantizationType, 'BF16' if old_dtype == + torch.bfloat16 else 'F16') + n_params = 1 + for dim_size in data_shape: + n_params *= dim_size + blacklist = {'time_embedding.', 'add_embedding.', 'time_in.', + 'txt_in.', 'vector_in.', 'img_in.', 'guidance_in.', 'final_layer.'} + if old_dtype in (torch.float32, torch.bfloat16): + if n_dims == 1: + data_qtype = GGMLQuantizationType.F32 + elif n_params <= QUANTIZATION_THRESHOLD: + data_qtype = GGMLQuantizationType.F32 + elif '.weight' in key and any(x in key for x in blacklist): + data_qtype = GGMLQuantizationType.F32 + try: + data = quantize(data, data_qtype) + except (AttributeError, QuantError) as e: + tqdm.write(f'falling back to F16: {e}') + data_qtype = GGMLQuantizationType.F16 + data = quantize(data, data_qtype) + new_name = key + shape_str = f"{{{', '.join(str(n) for n in reversed(data.shape))}}}" + tqdm.write( + f"{f'%-{max_name_len + 4}s' % f'{new_name}'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}" + ) + writer.add_tensor(new_name, data, raw_dtype=data_qtype) +import os +safetensors_files = [file for file in os.listdir() if file.endswith( + '.safetensors')] +if safetensors_files: + print('Safetensors file(s) available. Select which one to convert:') + for index, file_name in enumerate(safetensors_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = safetensors_files[choice_index] + print(f'Model file: {selected_file} is selected!') + path = selected_file + ask = input('Assign a name for the model (Y/n)? ') + if ask.lower() == 'y': + given = input('Enter a model name: ') + else: + given = None + writer, state_dict, model_arch = load_model(path, given) + writer.add_quantization_version(GGML_QUANT_VERSION) + if next(iter(state_dict.values())).dtype == torch.bfloat16: + out_path = f'{os.path.splitext(path)[0]}-bf16.gguf' + writer.add_file_type(LlamaFileType.MOSTLY_BF16) + else: + out_path = f'{os.path.splitext(path)[0]}-f16.gguf' + writer.add_file_type(LlamaFileType.MOSTLY_F16) + if os.path.isfile(out_path): + input('Output exists enter to continue or ctrl+c to abort!') + handle_tensors(path, writer, state_dict, model_arch) + writer.write_header_to_file(path=out_path) + writer.write_kv_data_to_file() + writer.write_tensors_to_file(progress=True) + writer.close() + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No safetensors files are available in the current directory.') + input('--- Press ENTER To Exit ---') \ No newline at end of file diff --git a/gguf_connector/t1.py b/gguf_connector/t1.py new file mode 100644 index 0000000..c3bffa3 --- /dev/null +++ b/gguf_connector/t1.py @@ -0,0 +1,118 @@ +import torch +from .writer import GGUFWriter, GGMLQuantizationType +from .quant import quantize, QuantError +from .const import GGML_QUANT_VERSION, LlamaFileType +from safetensors.torch import load_file +from tqdm import tqdm +QUANTIZATION_THRESHOLD = 1024 +REARRANGE_THRESHOLD = 512 +MAX_TENSOR_NAME_LENGTH = 127 +def load_state_dict(path): + state_dict = load_file(path) + prefix = None + for pfx in ['model.diffusion_model.', 'model.']: + if any([x.startswith(pfx) for x in state_dict.keys()]): + prefix = pfx + break + sd = {} + for k, v in state_dict.items(): + if prefix and prefix not in k: + continue + if prefix: + k = k.replace(prefix, '') + sd[k] = v + return sd +def load_model(path, model_arch): + state_dict = load_state_dict(path) + writer = GGUFWriter(path=None, arch=model_arch) + return writer, state_dict, model_arch +def handle_tensors(args, writer, state_dict, model_arch): + name_lengths = tuple(sorted(((key, len(key)) for key in state_dict.keys + ()), key=lambda item: item[1], reverse=True)) + if not name_lengths: + return + max_name_len = name_lengths[0][1] + if max_name_len > MAX_TENSOR_NAME_LENGTH: + bad_list = ', '.join(f'{key!r} ({namelen})' for key, namelen in + name_lengths if namelen > MAX_TENSOR_NAME_LENGTH) + raise ValueError( + f'Can only handle tensor names up to {MAX_TENSOR_NAME_LENGTH} characters. Tensors exceeding the limit: {bad_list}' + ) + for key, data in tqdm(state_dict.items()): + old_dtype = data.dtype + if data.dtype == torch.bfloat16: + data = data.to(torch.float32).numpy() + elif data.dtype in [getattr(torch, 'float8_e4m3fn', '_invalid'), + getattr(torch, 'float8_e5m2', '_invalid')]: + data = data.to(torch.float16).numpy() + else: + data = data.numpy() + n_dims = len(data.shape) + data_shape = data.shape + data_qtype = getattr(GGMLQuantizationType, 'BF16' if old_dtype == + torch.bfloat16 else 'F16') + n_params = 1 + for dim_size in data_shape: + n_params *= dim_size + blacklist = {'time_embedding.', 'add_embedding.', 'time_in.', + 'txt_in.', 'vector_in.', 'img_in.', 'guidance_in.', 'final_layer.'} + if old_dtype in (torch.float32, torch.bfloat16): + if n_dims == 1: + data_qtype = GGMLQuantizationType.F32 + elif n_params <= QUANTIZATION_THRESHOLD: + data_qtype = GGMLQuantizationType.F32 + elif '.weight' in key and any(x in key for x in blacklist): + data_qtype = GGMLQuantizationType.F32 + try: + data = quantize(data, data_qtype) + except (AttributeError, QuantError) as e: + tqdm.write(f'falling back to F16: {e}') + data_qtype = GGMLQuantizationType.F16 + data = quantize(data, data_qtype) + orig_shape = data.shape + writer.add_array(f'comfy.gguf.orig_shape.{key}', tuple(int(dim) for + dim in orig_shape)) + new_name = key + shape_str = f"{{{', '.join(str(n) for n in reversed(data.shape))}}}" + tqdm.write( + f"{f'%-{max_name_len + 4}s' % f'{new_name}'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}" + ) + writer.add_tensor(new_name, data, raw_dtype=data_qtype) +import os +safetensors_files = [file for file in os.listdir() if file.endswith( + '.safetensors')] +if safetensors_files: + print('Safetensors file(s) available. Select which one to convert:') + for index, file_name in enumerate(safetensors_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = safetensors_files[choice_index] + print(f'Model file: {selected_file} is selected!') + path = selected_file + ask = input('Assign a name for the model (Y/n)? ') + if ask.lower() == 'y': + given = input('Enter a model name: ') + else: + given = None + writer, state_dict, model_arch = load_model(path, given) + writer.add_quantization_version(GGML_QUANT_VERSION) + if next(iter(state_dict.values())).dtype == torch.bfloat16: + out_path = f'{os.path.splitext(path)[0]}-bf16.gguf' + writer.add_file_type(LlamaFileType.MOSTLY_BF16) + else: + out_path = f'{os.path.splitext(path)[0]}-f16.gguf' + writer.add_file_type(LlamaFileType.MOSTLY_F16) + if os.path.isfile(out_path): + input('Output exists enter to continue or ctrl+c to abort!') + handle_tensors(path, writer, state_dict, model_arch) + writer.write_header_to_file(path=out_path) + writer.write_kv_data_to_file() + writer.write_tensors_to_file(progress=True) + writer.close() + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No safetensors files are available in the current directory.') + input('--- Press ENTER To Exit ---') \ No newline at end of file diff --git a/gguf_connector/t2.py b/gguf_connector/t2.py new file mode 100644 index 0000000..7065e94 --- /dev/null +++ b/gguf_connector/t2.py @@ -0,0 +1,115 @@ +import torch +from .writer import GGUFWriter, GGMLQuantizationType +from .quant import quantize, QuantError +from .const import GGML_QUANT_VERSION, LlamaFileType +from safetensors.torch import load_file +from tqdm import tqdm +QUANTIZATION_THRESHOLD = 1024 +REARRANGE_THRESHOLD = 512 +MAX_TENSOR_NAME_LENGTH = 127 +def load_state_dict(path): + state_dict = load_file(path) + prefix = None + for pfx in ['model.diffusion_model.', 'model.']: + if any([x.startswith(pfx) for x in state_dict.keys()]): + prefix = pfx + break + sd = {} + for k, v in state_dict.items(): + if prefix and prefix not in k: + continue + if prefix: + k = k.replace(prefix, '') + sd[k] = v + return sd +def load_model(path, model_arch): + state_dict = load_state_dict(path) + writer = GGUFWriter(path=None, arch=model_arch) + return writer, state_dict, model_arch +def handle_tensors(args, writer, state_dict, model_arch): + name_lengths = tuple(sorted(((key, len(key)) for key in state_dict.keys + ()), key=lambda item: item[1], reverse=True)) + if not name_lengths: + return + max_name_len = name_lengths[0][1] + if max_name_len > MAX_TENSOR_NAME_LENGTH: + bad_list = ', '.join(f'{key!r} ({namelen})' for key, namelen in + name_lengths if namelen > MAX_TENSOR_NAME_LENGTH) + raise ValueError( + f'Can only handle tensor names up to {MAX_TENSOR_NAME_LENGTH} characters. Tensors exceeding the limit: {bad_list}' + ) + for key, data in tqdm(state_dict.items()): + old_dtype = data.dtype + if data.dtype == torch.bfloat16: + data = data.to(torch.float32).numpy() + elif data.dtype in [getattr(torch, 'float8_e4m3fn', '_invalid'), + getattr(torch, 'float8_e5m2', '_invalid')]: + data = data.to(torch.float16).numpy() + else: + data = data.numpy() + n_dims = len(data.shape) + data_shape = data.shape + data_qtype = getattr(GGMLQuantizationType, 'BF16' if old_dtype == + torch.bfloat16 else 'F16') + n_params = 1 + for dim_size in data_shape: + n_params *= dim_size + blacklist = {'time_embedding.', 'add_embedding.', 'time_in.', + 'txt_in.', 'vector_in.', 'img_in.', 'guidance_in.', 'final_layer.'} + if old_dtype in (torch.float32, torch.bfloat16): + if n_dims == 1: + data_qtype = GGMLQuantizationType.F32 + elif n_params <= QUANTIZATION_THRESHOLD: + data_qtype = GGMLQuantizationType.F32 + elif '.weight' in key and any(x in key for x in blacklist): + data_qtype = GGMLQuantizationType.F32 + try: + data = quantize(data, data_qtype) + except (AttributeError, QuantError) as e: + tqdm.write(f'falling back to F16: {e}') + data_qtype = GGMLQuantizationType.F16 + data = quantize(data, data_qtype) + new_name = key + shape_str = f"{{{', '.join(str(n) for n in reversed(data.shape))}}}" + tqdm.write( + f"{f'%-{max_name_len + 4}s' % f'{new_name}'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}" + ) + writer.add_tensor(new_name, data, raw_dtype=data_qtype) +import os +safetensors_files = [file for file in os.listdir() if file.endswith( + '.safetensors')] +if safetensors_files: + print('Safetensors file(s) available. Select which one to convert:') + for index, file_name in enumerate(safetensors_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = safetensors_files[choice_index] + print(f'Model file: {selected_file} is selected!') + path = selected_file + ask = input('Assign a name for the model (Y/n)? ') + if ask.lower() == 'y': + given = input('Enter a model name: ') + else: + given = None + writer, state_dict, model_arch = load_model(path, given) + writer.add_quantization_version(GGML_QUANT_VERSION) + if next(iter(state_dict.values())).dtype == torch.bfloat16: + out_path = f'{os.path.splitext(path)[0]}-bf16.gguf' + writer.add_file_type(LlamaFileType.MOSTLY_BF16) + else: + out_path = f'{os.path.splitext(path)[0]}-f16.gguf' + writer.add_file_type(LlamaFileType.MOSTLY_F16) + if os.path.isfile(out_path): + input('Output exists enter to continue or ctrl+c to abort!') + handle_tensors(path, writer, state_dict, model_arch) + writer.write_header_to_file(path=out_path) + writer.write_kv_data_to_file() + writer.write_tensors_to_file(progress=True) + writer.close() + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No safetensors files are available in the current directory.') + input('--- Press ENTER To Exit ---') \ No newline at end of file diff --git a/gguf_connector/t3.py b/gguf_connector/t3.py new file mode 100644 index 0000000..84e86af --- /dev/null +++ b/gguf_connector/t3.py @@ -0,0 +1,80 @@ + +import torch # optional (need torch to work; pip install torch) +import numpy as np +from safetensors.torch import save_file +from typing import Dict, Tuple +from .quant import dequantize +from .reader import GGUFReader +from tqdm import tqdm + +def load_gguf_and_extract_metadata(gguf_path): + reader = GGUFReader(gguf_path) + tensors_metadata = [] + for tensor in reader.tensors: + tensor_metadata = { + 'name': tensor.name, + 'shape': tuple(tensor.shape.tolist()), + 'n_elements': tensor.n_elements, + 'n_bytes': tensor.n_bytes, + 'data_offset': tensor.data_offset, + 'type': tensor.tensor_type, + } + tensors_metadata.append(tensor_metadata) + return reader, tensors_metadata + +def convert_gguf_to_safetensors(gguf_path, output_path, use_bf16): + reader, tensors_metadata = load_gguf_and_extract_metadata(gguf_path) + print(f'Extracted {len(tensors_metadata)} tensors from GGUF file') + tensors_dict: dict[str, torch.Tensor] = {} + for i, tensor_info in enumerate(tqdm(tensors_metadata, desc= + 'Converting tensors', unit='tensor')): + tensor_name = tensor_info['name'] + tensor_data = reader.get_tensor(i) + weights = dequantize(tensor_data.data, tensor_data.tensor_type).copy() + try: + if use_bf16: + weights_tensor = torch.from_numpy(weights).to(dtype=torch. + float32) + weights_tensor = weights_tensor.to(torch.bfloat16) + else: + weights_tensor = torch.from_numpy(weights).to(dtype=torch. + float16) + weights_hf = weights_tensor + except Exception as e: + print( + f"Error during BF16 conversion for tensor '{tensor_name}': {e}" + ) + weights_tensor = torch.from_numpy(weights.astype(np.float32)).to( + torch.float16) + weights_hf = weights_tensor + tensors_dict[tensor_name] = weights_hf + metadata = {key: str(reader.get_field(key)) for key in reader.fields} + save_file(tensors_dict, output_path, metadata=metadata) + print('Conversion complete!') + +import os +gguf_files = [file for file in os.listdir() if file.endswith('.gguf')] + +if gguf_files: + print('GGUF file(s) available. Select which one to convert:') + for index, file_name in enumerate(gguf_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(gguf_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = gguf_files[choice_index] + print(f'Model file: {selected_file} is selected!') + input_path = selected_file + ask = input('Convert tensors to BF16 format instead of FP16 (Y/n)? ') + if ask.lower() == 'y': + use_bf16 = True + out_path = f'{os.path.splitext(input_path)[0]}-bf16.safetensors' + else: + use_bf16 = False + out_path = f'{os.path.splitext(input_path)[0]}-fp16.safetensors' + convert_gguf_to_safetensors(input_path, out_path, use_bf16) + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No GGUF files are available in the current directory.') + input('--- Press ENTER To Exit ---') diff --git a/gguf_connector/t4.py b/gguf_connector/t4.py new file mode 100644 index 0000000..3d96446 --- /dev/null +++ b/gguf_connector/t4.py @@ -0,0 +1,198 @@ +import torch +from .writer import GGUFWriter, GGMLQuantizationType +from .quant import quantize, QuantError +from .const import GGML_QUANT_VERSION, LlamaFileType +from safetensors.torch import load_file +from tqdm import tqdm +QUANTIZATION_THRESHOLD = 1024 +REARRANGE_THRESHOLD = 512 +MAX_TENSOR_NAME_LENGTH = 127 +class ModelTemplate: + arch = 'invalid' + shape_fix = False + keys_detect = [] + keys_banned = [] +class ModelSDXL(ModelTemplate): + arch = 'sdxl' + shape_fix = True + keys_detect = [('down_blocks.0.downsamplers.0.conv.weight', + 'add_embedding.linear_1.weight'), ('input_blocks.3.0.op.weight', + 'input_blocks.6.0.op.weight', 'output_blocks.2.2.conv.weight', + 'output_blocks.5.2.conv.weight'), ('label_emb.0.0.weight',)] +class ModelSD1(ModelTemplate): + arch = 'sd1' + shape_fix = True + keys_detect = [('down_blocks.0.downsamplers.0.conv.weight',), ( + 'input_blocks.3.0.op.weight', 'input_blocks.6.0.op.weight', + 'input_blocks.9.0.op.weight', 'output_blocks.2.1.conv.weight', + 'output_blocks.5.2.conv.weight', 'output_blocks.8.2.conv.weight')] +class ModelSD3(ModelTemplate): + arch = 'sd3' + keys_detect = [('transformer_blocks.0.attn.add_q_proj.weight',), ( + 'joint_blocks.0.x_block.attn.qkv.weight',)] + keys_banned = ['transformer_blocks.0.attn.add_q_proj.weight'] +class ModelAura(ModelTemplate): + arch = 'aura' + keys_detect = [('double_layers.3.modX.1.weight',), ( + 'joint_transformer_blocks.3.ff_context.out_projection.weight',)] + keys_banned = [ + 'joint_transformer_blocks.3.ff_context.out_projection.weight'] +class ModelPixArt(ModelTemplate): + arch = 'pixart' + keys_detect = [('transformer_blocks.27.scale_shift_table',)] +class ModelFlux(ModelTemplate): + arch = 'flux' + keys_detect = [('transformer_blocks.0.attn.norm_added_k.weight',), ( + 'double_blocks.0.img_attn.proj.weight',)] + keys_banned = ['transformer_blocks.0.attn.norm_added_k.weight'] +class ModelLumina(ModelTemplate): + arch = 'lumina' + keys_detect = [('cap_embedder.0.weight',)] +class ModelHYVID(ModelTemplate): + arch = 'hyvid' + keys_detect = [('txt_in.individual_token_refiner.blocks.0.norm1.weight',)] +class ModelCosmos(ModelTemplate): + arch = 'cosmos' + keys_detect = [('blocks.block0.blocks.0.block.attn.to_q.0.weight',)] +class ModelMochi(ModelTemplate): + arch = 'mochi' + keys_detect = [('t5_yproj.weight',)] +class ModelLTXV(ModelTemplate): + arch = 'ltxv' + keys_detect = [('adaln_single.emb.timestep_embedder.linear_2.weight', + 'transformer_blocks.27.scale_shift_table', + 'caption_projection.linear_2.weight')] +arch_list = [ModelFlux, ModelSD3, ModelAura, ModelPixArt, ModelHYVID, + ModelSDXL, ModelSD1, ModelCosmos, ModelMochi, ModelLumina, ModelLTXV] +def is_model_arch(model, state_dict): + matched = False + invalid = False + for match_list in model.keys_detect: + if all(key in state_dict for key in match_list): + matched = True + invalid = any(key in state_dict for key in model.keys_banned) + break + assert not invalid, 'Model architecture not allowed for conversion! (i.e. reference VS diffusers format)' + return matched +def detect_arch(state_dict): + model_arch = None + for arch in arch_list: + if is_model_arch(arch, state_dict): + model_arch = arch + break + assert model_arch is not None, 'Unknown model architecture!' + return model_arch +def load_state_dict(path): + if any(path.endswith(x) for x in ['.ckpt', '.pt', '.bin', '.pth']): + state_dict = torch.load(path, map_location='cpu', weights_only=True) + state_dict = state_dict.get('model', state_dict) + else: + state_dict = load_file(path) + prefix = None + for pfx in ['model.diffusion_model.', 'model.']: + if any([x.startswith(pfx) for x in state_dict.keys()]): + prefix = pfx + break + sd = {} + for k, v in state_dict.items(): + if prefix and prefix not in k: + continue + if prefix: + k = k.replace(prefix, '') + sd[k] = v + return sd +def load_model(path): + state_dict = load_state_dict(path) + model_arch = detect_arch(state_dict) + print(f'* Architecture detected from input: {model_arch.arch}') + writer = GGUFWriter(path=None, arch=model_arch.arch) + return writer, state_dict, model_arch +def handle_tensors(args, writer, state_dict, model_arch): + name_lengths = tuple(sorted(((key, len(key)) for key in state_dict.keys + ()), key=lambda item: item[1], reverse=True)) + if not name_lengths: + return + max_name_len = name_lengths[0][1] + if max_name_len > MAX_TENSOR_NAME_LENGTH: + bad_list = ', '.join(f'{key!r} ({namelen})' for key, namelen in + name_lengths if namelen > MAX_TENSOR_NAME_LENGTH) + raise ValueError( + f'Can only handle tensor names up to {MAX_TENSOR_NAME_LENGTH} characters. Tensors exceeding the limit: {bad_list}' + ) + for key, data in tqdm(state_dict.items()): + old_dtype = data.dtype + if data.dtype == torch.bfloat16: + data = data.to(torch.float32).numpy() + elif data.dtype in [getattr(torch, 'float8_e4m3fn', '_invalid'), + getattr(torch, 'float8_e5m2', '_invalid')]: + data = data.to(torch.float16).numpy() + else: + data = data.numpy() + n_dims = len(data.shape) + data_shape = data.shape + data_qtype = getattr(GGMLQuantizationType, 'BF16' if old_dtype == + torch.bfloat16 else 'F16') + n_params = 1 + for dim_size in data_shape: + n_params *= dim_size + blacklist = {'time_embedding.', 'add_embedding.', 'time_in.', + 'txt_in.', 'vector_in.', 'img_in.', 'guidance_in.', 'final_layer.'} + if old_dtype in (torch.float32, torch.bfloat16): + if n_dims == 1: + data_qtype = GGMLQuantizationType.F32 + elif n_params <= QUANTIZATION_THRESHOLD: + data_qtype = GGMLQuantizationType.F32 + elif '.weight' in key and any(x in key for x in blacklist): + data_qtype = GGMLQuantizationType.F32 + if (model_arch.shape_fix and n_dims > 1 and n_params >= + REARRANGE_THRESHOLD and (n_params / 256).is_integer() and not ( + data.shape[-1] / 256).is_integer()): + orig_shape = data.shape + data = data.reshape(n_params // 256, 256) + writer.add_array(f'comfy.gguf.orig_shape.{key}', tuple(int(dim) for + dim in orig_shape)) + try: + data = quantize(data, data_qtype) + except (AttributeError, QuantError) as e: + tqdm.write(f'falling back to F16: {e}') + data_qtype = GGMLQuantizationType.F16 + data = quantize(data, data_qtype) + new_name = key + shape_str = f"{{{', '.join(str(n) for n in reversed(data.shape))}}}" + tqdm.write( + f"{f'%-{max_name_len + 4}s' % f'{new_name}'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}" + ) + writer.add_tensor(new_name, data, raw_dtype=data_qtype) +import os +safetensors_files = [file for file in os.listdir() if file.endswith( + '.safetensors')] +if safetensors_files: + print('Safetensors file(s) available. Select which one to convert:') + for index, file_name in enumerate(safetensors_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = safetensors_files[choice_index] + print(f'Model file: {selected_file} is selected!') + path = selected_file + writer, state_dict, model_arch = load_model(path) + writer.add_quantization_version(GGML_QUANT_VERSION) + if next(iter(state_dict.values())).dtype == torch.bfloat16: + out_path = f'{os.path.splitext(path)[0]}-bf16.gguf' + writer.add_file_type(LlamaFileType.MOSTLY_BF16) + else: + out_path = f'{os.path.splitext(path)[0]}-f16.gguf' + writer.add_file_type(LlamaFileType.MOSTLY_F16) + if os.path.isfile(out_path): + input('Output exists enter to continue or ctrl+c to abort!') + handle_tensors(path, writer, state_dict, model_arch) + writer.write_header_to_file(path=out_path) + writer.write_kv_data_to_file() + writer.write_tensors_to_file(progress=True) + writer.close() + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No safetensors files are available in the current directory.') + input('--- Press ENTER To Exit ---') \ No newline at end of file diff --git a/gguf_connector/t5.py b/gguf_connector/t5.py new file mode 100644 index 0000000..773e885 --- /dev/null +++ b/gguf_connector/t5.py @@ -0,0 +1,116 @@ +import torch +from .writer import GGUFWriter, GGMLQuantizationType +from .quant import quantize, QuantError +from .const import GGML_QUANT_VERSION, LlamaFileType +from safetensors.torch import load_file +from tqdm import tqdm +QUANTIZATION_THRESHOLD = 1024 +REARRANGE_THRESHOLD = 512 +MAX_TENSOR_NAME_LENGTH = 127 +T5_MAP = {'encoder.': 'enc.', '.block.': '.blk.', 'shared': 'token_embd', + 'final_layer_norm': 'output_norm', 'layer.0.SelfAttention.q': 'attn_q', + 'layer.0.SelfAttention.k': 'attn_k', 'layer.0.SelfAttention.v': + 'attn_v', 'layer.0.SelfAttention.o': 'attn_o', 'layer.0.layer_norm': + 'attn_norm', 'layer.0.SelfAttention.relative_attention_bias': + 'attn_rel_b', 'layer.1.DenseReluDense.wi_1': 'ffn_up', + 'layer.1.DenseReluDense.wo': 'ffn_down', 'layer.1.DenseReluDense.wi_0': + 'ffn_gate', 'layer.1.layer_norm': 'ffn_norm'} +def load_state_dict(path, key_map): + state_dict = load_file(path) + sd = {} + for k, v in state_dict.items(): + for s, d in key_map.items(): + k = k.replace(s, d) + sd[k] = v + return sd +def load_model(path, model_arch): + state_dict = load_state_dict(path, T5_MAP) + writer = GGUFWriter(path=None, arch=model_arch) + return writer, state_dict, model_arch +def handle_tensors(args, writer, state_dict, model_arch): + name_lengths = tuple(sorted(((key, len(key)) for key in state_dict.keys + ()), key=lambda item: item[1], reverse=True)) + if not name_lengths: + return + max_name_len = name_lengths[0][1] + if max_name_len > MAX_TENSOR_NAME_LENGTH: + bad_list = ', '.join(f'{key!r} ({namelen})' for key, namelen in + name_lengths if namelen > MAX_TENSOR_NAME_LENGTH) + raise ValueError( + f'Can only handle tensor names up to {MAX_TENSOR_NAME_LENGTH} characters. Tensors exceeding the limit: {bad_list}' + ) + for key, data in tqdm(state_dict.items()): + old_dtype = data.dtype + if data.dtype == torch.bfloat16: + data = data.to(torch.float32).numpy() + elif data.dtype in [getattr(torch, 'float8_e4m3fn', '_invalid'), + getattr(torch, 'float8_e5m2', '_invalid')]: + data = data.to(torch.float16).numpy() + else: + data = data.numpy() + n_dims = len(data.shape) + data_shape = data.shape + data_qtype = getattr(GGMLQuantizationType, 'BF16' if old_dtype == + torch.bfloat16 else 'F16') + n_params = 1 + for dim_size in data_shape: + n_params *= dim_size + blacklist = {'time_embedding.', 'add_embedding.', 'time_in.', + 'txt_in.', 'vector_in.', 'img_in.', 'guidance_in.', 'final_layer.'} + if old_dtype in (torch.float32, torch.bfloat16): + if n_dims == 1: + data_qtype = GGMLQuantizationType.F32 + elif n_params <= QUANTIZATION_THRESHOLD: + data_qtype = GGMLQuantizationType.F32 + elif '.weight' in key and any(x in key for x in blacklist): + data_qtype = GGMLQuantizationType.F32 + try: + data = quantize(data, data_qtype) + except (AttributeError, QuantError) as e: + tqdm.write(f'falling back to F16: {e}') + data_qtype = GGMLQuantizationType.F16 + data = quantize(data, data_qtype) + new_name = key + shape_str = f"{{{', '.join(str(n) for n in reversed(data.shape))}}}" + tqdm.write( + f"{f'%-{max_name_len + 4}s' % f'{new_name}'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}" + ) + writer.add_tensor(new_name, data, raw_dtype=data_qtype) +import os +safetensors_files = [file for file in os.listdir() if file.endswith( + '.safetensors')] +if safetensors_files: + print('Safetensors file(s) available. Select which one to convert:') + for index, file_name in enumerate(safetensors_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = safetensors_files[choice_index] + print(f'Model file: {selected_file} is selected!') + path = selected_file + ask = input('Assign a name for the model (Y/n)? ') + if ask.lower() == 'y': + given = input('Enter a model name: ') + else: + given = 't5' + writer, state_dict, model_arch = load_model(path, given) + writer.add_quantization_version(GGML_QUANT_VERSION) + if next(iter(state_dict.values())).dtype == torch.bfloat16: + out_path = f'{os.path.splitext(path)[0]}-bf16.gguf' + writer.add_file_type(LlamaFileType.MOSTLY_BF16) + else: + out_path = f'{os.path.splitext(path)[0]}-f16.gguf' + writer.add_file_type(LlamaFileType.MOSTLY_F16) + if os.path.isfile(out_path): + input('Output exists enter to continue or ctrl+c to abort!') + handle_tensors(path, writer, state_dict, model_arch) + writer.write_header_to_file(path=out_path) + writer.write_kv_data_to_file() + writer.write_tensors_to_file(progress=True) + writer.close() + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No safetensors files are available in the current directory.') + input('--- Press ENTER To Exit ---') \ No newline at end of file diff --git a/gguf_connector/t6.py b/gguf_connector/t6.py new file mode 100644 index 0000000..5e146eb --- /dev/null +++ b/gguf_connector/t6.py @@ -0,0 +1,95 @@ +import torch +from .writer import GGUFWriter, GGMLQuantizationType +from .quant import quantize, QuantError +from .const import GGML_QUANT_VERSION, LlamaFileType +from safetensors.torch import load_file +from tqdm import tqdm +import numpy as np +QUANTIZATION_THRESHOLD = 1024 +MAX_TENSOR_NAME_LENGTH = 127 +def load_state_dict(path): + state_dict = load_file(path) + return {k: v for k, v in state_dict.items()} +def load_model(path, model_arch): + state_dict = load_state_dict(path) + writer = GGUFWriter(path=None, arch=model_arch) + return writer, state_dict, model_arch +def validate_tensor_data(data, key): + if np.any(np.isnan(data)) or np.any(np.isinf(data)): + print(f"Warning: Tensor '{key}' contains NaN or Inf values. Skipping.") + return None + return data +def handle_tensors(writer, state_dict): + name_lengths = [(key, len(key)) for key in state_dict.keys()] + if not name_lengths: + return + max_name_len = max(name_lengths, key=lambda x: x[1])[1] + for key, data in tqdm(state_dict.items(), desc='Processing Tensors'): + old_dtype = data.dtype + if old_dtype == torch.bfloat16: + data = data.to(torch.float32).numpy() + elif old_dtype in [getattr(torch, 'float8_e4m3fn', '_invalid'), + getattr(torch, 'float8_e5m2', '_invalid')]: + data = data.to(torch.float16).numpy() + else: + data = data.numpy() + data = validate_tensor_data(data, key) + if data is None: + continue + data_qtype = (GGMLQuantizationType.BF16 if old_dtype == torch. + bfloat16 else GGMLQuantizationType.F16) + n_params = data.size + if old_dtype in (torch.float32, torch.bfloat16): + if len(data.shape) == 1 or n_params <= QUANTIZATION_THRESHOLD: + data_qtype = GGMLQuantizationType.F32 + try: + data = quantize(data, data_qtype) + except (AttributeError, QuantError) as e: + tqdm.write(f'Quantization error ({e}), falling back to F16') + data_qtype = GGMLQuantizationType.F16 + data = quantize(data, data_qtype) + shape_str = f"{{{', '.join(map(str, reversed(data.shape)))}}}" + tqdm.write( + f'{key.ljust(max_name_len)} {old_dtype} --> {data_qtype.name}, shape = {shape_str}' + ) + writer.add_tensor(key, data, raw_dtype=data_qtype) +import os +safetensors_files = [file for file in os.listdir() if file.endswith( + '.safetensors')] +if safetensors_files: + print('Safetensors file(s) available. Select which one to convert:') + for index, file_name in enumerate(safetensors_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = safetensors_files[choice_index] + print(f'Model file: {selected_file} is selected!') + path = selected_file + ask = input('Assign a name for the model (Y/n)? ') + if ask.lower() == 'y': + given = input('Enter a model name: ') + else: + given = None + writer, state_dict, _ = load_model(path, given) + writer.add_quantization_version(GGML_QUANT_VERSION) + first_tensor_dtype = next(iter(state_dict.values())).dtype + file_type = (LlamaFileType.MOSTLY_BF16 if first_tensor_dtype == + torch.bfloat16 else LlamaFileType.MOSTLY_F16) + out_path = f'{os.path.splitext(path)[0]}-{file_type.name.lower()}.gguf' + writer.add_file_type(file_type) + if os.path.isfile(out_path): + input( + 'Output file exists. Press Enter to overwrite or Ctrl+C to abort.' + ) + handle_tensors(writer, state_dict) + writer.write_header_to_file(path=out_path) + writer.write_kv_data_to_file() + writer.write_tensors_to_file(progress=True) + writer.close() + print(f'Conversion completed: {out_path}') + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No safetensors files are available in the current directory.') + input('--- Press ENTER To Exit ---') \ No newline at end of file diff --git a/gguf_connector/t7.py b/gguf_connector/t7.py new file mode 100644 index 0000000..154b157 --- /dev/null +++ b/gguf_connector/t7.py @@ -0,0 +1,213 @@ +import torch +from .writer import GGUFWriter, GGMLQuantizationType +from .quant import quantize, QuantError +from .const import GGML_QUANT_VERSION, LlamaFileType +from safetensors.torch import load_file +from tqdm import tqdm +QUANTIZATION_THRESHOLD = 1024 +REARRANGE_THRESHOLD = 512 +MAX_TENSOR_NAME_LENGTH = 127 +MAX_TENSOR_DIMS = 4 +class ModelTemplate: + arch = 'invalid' + shape_fix = False + keys_detect = [] + keys_banned = [] + keys_hiprec = [] + def handle_nd_tensor(self, key, data): + raise NotImplementedError( + f'Tensor detected that exceeds dims supported by C++ code! ({key} @ {data.shape})' + ) +class ModelFlux(ModelTemplate): + arch = 'flux' + keys_detect = [('transformer_blocks.0.attn.norm_added_k.weight',), ( + 'double_blocks.0.img_attn.proj.weight',)] + keys_banned = ['transformer_blocks.0.attn.norm_added_k.weight'] +class ModelSD3(ModelTemplate): + arch = 'sd3' + keys_detect = [('transformer_blocks.0.attn.add_q_proj.weight',), ( + 'joint_blocks.0.x_block.attn.qkv.weight',)] + keys_banned = ['transformer_blocks.0.attn.add_q_proj.weight'] +class ModelAura(ModelTemplate): + arch = 'aura' + keys_detect = [('double_layers.3.modX.1.weight',), ( + 'joint_transformer_blocks.3.ff_context.out_projection.weight',)] + keys_banned = [ + 'joint_transformer_blocks.3.ff_context.out_projection.weight'] +class ModelHyVid(ModelTemplate): + arch = 'hyvid' + keys_detect = [('double_blocks.0.img_attn_proj.weight', + 'txt_in.individual_token_refiner.blocks.1.self_attn_qkv.weight')] + def handle_nd_tensor(self, key, data): + path = f'./fix_5d_tensors_{self.arch}.pt' + if os.path.isfile(path): + raise RuntimeError(f'5D tensor fix file already exists! {path}') + fsd = {key: data} + tqdm.write( + f'5D key found in state dict! Manual fix required! - {key} {data.shape}' + ) + torch.save(fsd, path) +class ModelWan(ModelHyVid): + arch = 'wan' + keys_detect = [('blocks.0.self_attn.norm_q.weight', + 'text_embedding.2.weight', 'head.modulation')] +class ModelLTXV(ModelTemplate): + arch = 'ltxv' + keys_detect = [('adaln_single.emb.timestep_embedder.linear_2.weight', + 'transformer_blocks.27.scale_shift_table', + 'caption_projection.linear_2.weight')] + keys_hiprec = ['scale_shift_table'] +class ModelSDXL(ModelTemplate): + arch = 'sdxl' + shape_fix = True + keys_detect = [('down_blocks.0.downsamplers.0.conv.weight', + 'add_embedding.linear_1.weight'), ('input_blocks.3.0.op.weight', + 'input_blocks.6.0.op.weight', 'output_blocks.2.2.conv.weight', + 'output_blocks.5.2.conv.weight'), ('label_emb.0.0.weight',)] +class ModelSD1(ModelTemplate): + arch = 'sd1' + shape_fix = True + keys_detect = [('down_blocks.0.downsamplers.0.conv.weight',), ( + 'input_blocks.3.0.op.weight', 'input_blocks.6.0.op.weight', + 'input_blocks.9.0.op.weight', 'output_blocks.2.1.conv.weight', + 'output_blocks.5.2.conv.weight', 'output_blocks.8.2.conv.weight')] +arch_list = [ModelFlux, ModelSD3, ModelAura, ModelLTXV, ModelHyVid, + ModelWan, ModelSDXL, ModelSD1] +def is_model_arch(model, state_dict): + matched = False + invalid = False + for match_list in model.keys_detect: + if all(key in state_dict for key in match_list): + matched = True + invalid = any(key in state_dict for key in model.keys_banned) + break + assert not invalid, 'Model architecture not allowed for conversion! (i.e. reference VS diffusers format)' + return matched +def detect_arch(state_dict): + model_arch = None + for arch in arch_list: + if is_model_arch(arch, state_dict): + model_arch = arch() + break + assert model_arch is not None, 'Unknown model architecture!' + return model_arch +def load_state_dict(path): + if any(path.endswith(x) for x in ['.ckpt', '.pt', '.bin', '.pth']): + state_dict = torch.load(path, map_location='cpu', weights_only=True) + for subkey in ['model', 'module']: + if subkey in state_dict: + state_dict = state_dict[subkey] + break + if len(state_dict) < 20: + raise RuntimeError(f'pt subkey load failed: {state_dict.keys()}') + else: + state_dict = load_file(path) + prefix = None + for pfx in ['model.diffusion_model.', 'model.']: + if any([x.startswith(pfx) for x in state_dict.keys()]): + prefix = pfx + break + sd = {} + for k, v in state_dict.items(): + if prefix and prefix not in k: + continue + if prefix: + k = k.replace(prefix, '') + sd[k] = v + return sd +def load_model(path): + state_dict = load_state_dict(path) + model_arch = detect_arch(state_dict) + print(f'* Architecture detected from input: {model_arch.arch}') + writer = GGUFWriter(path=None, arch=model_arch.arch) + return writer, state_dict, model_arch +def handle_tensors(args, writer, state_dict, model_arch): + name_lengths = tuple(sorted(((key, len(key)) for key in state_dict.keys + ()), key=lambda item: item[1], reverse=True)) + if not name_lengths: + return + max_name_len = name_lengths[0][1] + if max_name_len > MAX_TENSOR_NAME_LENGTH: + bad_list = ', '.join(f'{key!r} ({namelen})' for key, namelen in + name_lengths if namelen > MAX_TENSOR_NAME_LENGTH) + raise ValueError( + f'Can only handle tensor names up to {MAX_TENSOR_NAME_LENGTH} characters. Tensors exceeding the limit: {bad_list}' + ) + for key, data in tqdm(state_dict.items()): + old_dtype = data.dtype + if data.dtype == torch.bfloat16: + data = data.to(torch.float32).numpy() + elif data.dtype in [getattr(torch, 'float8_e4m3fn', '_invalid'), + getattr(torch, 'float8_e5m2', '_invalid')]: + data = data.to(torch.float16).numpy() + else: + data = data.numpy() + n_dims = len(data.shape) + data_shape = data.shape + data_qtype = getattr(GGMLQuantizationType, 'BF16' if old_dtype == + torch.bfloat16 else 'F16') + if len(data.shape) > MAX_TENSOR_DIMS: + model_arch.handle_nd_tensor(key, data) + continue + n_params = 1 + for dim_size in data_shape: + n_params *= dim_size + if old_dtype in (torch.float32, torch.bfloat16): + if n_dims == 1: + data_qtype = GGMLQuantizationType.F32 + elif n_params <= QUANTIZATION_THRESHOLD: + data_qtype = GGMLQuantizationType.F32 + elif any(x in key for x in model_arch.keys_hiprec): + data_qtype = GGMLQuantizationType.F32 + if (model_arch.shape_fix and n_dims > 1 and n_params >= + REARRANGE_THRESHOLD and (n_params / 256).is_integer() and not ( + data.shape[-1] / 256).is_integer()): + orig_shape = data.shape + data = data.reshape(n_params // 256, 256) + writer.add_array(f'comfy.gguf.orig_shape.{key}', tuple(int(dim) for + dim in orig_shape)) + try: + data = quantize(data, data_qtype) + except (AttributeError, QuantError) as e: + tqdm.write(f'falling back to F16: {e}') + data_qtype = GGMLQuantizationType.F16 + data = quantize(data, data_qtype) + new_name = key + shape_str = f"{{{', '.join(str(n) for n in reversed(data.shape))}}}" + tqdm.write( + f"{f'%-{max_name_len + 4}s' % f'{new_name}'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}" + ) + writer.add_tensor(new_name, data, raw_dtype=data_qtype) +import os +safetensors_files = [file for file in os.listdir() if file.endswith( + '.safetensors')] +if safetensors_files: + print('Safetensors file(s) available. Select which one to convert:') + for index, file_name in enumerate(safetensors_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = safetensors_files[choice_index] + print(f'Model file: {selected_file} is selected!') + path = selected_file + writer, state_dict, model_arch = load_model(path) + writer.add_quantization_version(GGML_QUANT_VERSION) + if next(iter(state_dict.values())).dtype == torch.bfloat16: + out_path = f'{os.path.splitext(path)[0]}-bf16.gguf' + writer.add_file_type(LlamaFileType.MOSTLY_BF16) + else: + out_path = f'{os.path.splitext(path)[0]}-f16.gguf' + writer.add_file_type(LlamaFileType.MOSTLY_F16) + if os.path.isfile(out_path): + input('Output exists enter to continue or ctrl+c to abort!') + handle_tensors(path, writer, state_dict, model_arch) + writer.write_header_to_file(path=out_path) + writer.write_kv_data_to_file() + writer.write_tensors_to_file(progress=True) + writer.close() + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No safetensors files are available in the current directory.') + input('--- Press ENTER To Exit ---') \ No newline at end of file diff --git a/gguf_connector/t8.py b/gguf_connector/t8.py new file mode 100644 index 0000000..f794f91 --- /dev/null +++ b/gguf_connector/t8.py @@ -0,0 +1,199 @@ +import torch +from .writer import GGUFWriter, GGMLQuantizationType +from .const import GGML_QUANT_VERSION, LlamaFileType +from safetensors.torch import load_file +from tqdm import tqdm +QUANTIZATION_THRESHOLD = 1024 +REARRANGE_THRESHOLD = 512 +MAX_TENSOR_NAME_LENGTH = 127 +MAX_TENSOR_DIMS = 4 +class ModelTemplate: + arch = 'invalid' + shape_fix = False + keys_detect = [] + keys_banned = [] + keys_hiprec = [] + def handle_nd_tensor(self, key, data): + raise NotImplementedError( + f'Tensor detected that exceeds dims supported by C++ code! ({key} @ {data.shape})' + ) +class ModelFlux(ModelTemplate): + arch = 'flux' + keys_detect = [('transformer_blocks.0.attn.norm_added_k.weight',), ( + 'double_blocks.0.img_attn.proj.weight',)] + keys_banned = ['transformer_blocks.0.attn.norm_added_k.weight'] +class ModelSD3(ModelTemplate): + arch = 'sd3' + keys_detect = [('transformer_blocks.0.attn.add_q_proj.weight',), ( + 'joint_blocks.0.x_block.attn.qkv.weight',)] + keys_banned = ['transformer_blocks.0.attn.add_q_proj.weight'] +class ModelAura(ModelTemplate): + arch = 'aura' + keys_detect = [('double_layers.3.modX.1.weight',), ( + 'joint_transformer_blocks.3.ff_context.out_projection.weight',)] + keys_banned = [ + 'joint_transformer_blocks.3.ff_context.out_projection.weight'] +class ModelHiDream(ModelTemplate): + arch = 'hidream' + keys_detect = [('caption_projection.0.linear.weight', + 'double_stream_blocks.0.block.ff_i.shared_experts.w3.weight')] + keys_hiprec = ['.ff_i.gate.weightimg_emb.emb_pos'] +class ModelHyVid(ModelTemplate): + arch = 'hyvid' + keys_detect = [('double_blocks.0.img_attn_proj.weight', + 'txt_in.individual_token_refiner.blocks.1.self_attn_qkv.weight')] + def handle_nd_tensor(self, key, data): + path = f'./fix_5d_tensors_{self.arch}.safetensors' + if os.path.isfile(path): + raise RuntimeError(f'5D tensor fix file already exists! {path}') + fsd = {key: torch.from_numpy(data)} + tqdm.write( + f'5D key found in state dict! Manual fix required! - {key} {data.shape}' + ) + torch.save(fsd, path) +class ModelWan(ModelHyVid): + arch = 'wan' + keys_detect = [('blocks.0.self_attn.norm_q.weight', + 'text_embedding.2.weight', 'head.modulation')] +class ModelLTXV(ModelTemplate): + arch = 'ltxv' + keys_detect = [('adaln_single.emb.timestep_embedder.linear_2.weight', + 'transformer_blocks.27.scale_shift_table', + 'caption_projection.linear_2.weight')] + keys_hiprec = ['scale_shift_table'] +class ModelSDXL(ModelTemplate): + arch = 'sdxl' + shape_fix = True + keys_detect = [('down_blocks.0.downsamplers.0.conv.weight', + 'add_embedding.linear_1.weight'), ('input_blocks.3.0.op.weight', + 'input_blocks.6.0.op.weight', 'output_blocks.2.2.conv.weight', + 'output_blocks.5.2.conv.weight'), ('label_emb.0.0.weight',)] +class ModelSD1(ModelTemplate): + arch = 'sd1' + shape_fix = True + keys_detect = [('down_blocks.0.downsamplers.0.conv.weight',), ( + 'input_blocks.3.0.op.weight', 'input_blocks.6.0.op.weight', + 'input_blocks.9.0.op.weight', 'output_blocks.2.1.conv.weight', + 'output_blocks.5.2.conv.weight', 'output_blocks.8.2.conv.weight')] +arch_list = [ModelFlux, ModelSD3, ModelAura, ModelHiDream, ModelLTXV, + ModelHyVid, ModelWan, ModelSDXL, ModelSD1] +def is_model_arch(model, state_dict): + matched = False + invalid = False + for match_list in model.keys_detect: + if all(key in state_dict for key in match_list): + matched = True + invalid = any(key in state_dict for key in model.keys_banned) + break + assert not invalid, 'Model architecture not allowed for conversion! (i.e. reference VS diffusers format)' + return matched +def detect_arch(state_dict): + model_arch = None + for arch in arch_list: + if is_model_arch(arch, state_dict): + model_arch = arch() + break + assert model_arch is not None, 'Unknown model architecture!' + return model_arch +def load_state_dict(path): + if any(path.endswith(x) for x in ['.ckpt', '.pt', '.bin', '.pth']): + state_dict = torch.load(path, map_location='cpu', weights_only=True) + for subkey in ['model', 'module']: + if subkey in state_dict: + state_dict = state_dict[subkey] + break + if len(state_dict) < 20: + raise RuntimeError(f'pt subkey load failed: {state_dict.keys()}') + else: + state_dict = load_file(path) + prefix = None + for pfx in ['model.diffusion_model.', 'model.']: + if any([x.startswith(pfx) for x in state_dict.keys()]): + prefix = pfx + break + sd = {} + for k, v in state_dict.items(): + if prefix and prefix not in k: + continue + if prefix: + k = k.replace(prefix, '') + sd[k] = v + return sd +def load_model(path): + state_dict = load_state_dict(path) + model_arch = detect_arch(state_dict) + print(f'* Architecture detected from input: {model_arch.arch}') + writer = GGUFWriter(path=None, arch=model_arch.arch) + return writer, state_dict, model_arch +def handle_tensors(args, writer, state_dict, model_arch): + name_lengths = tuple(sorted(((key, len(key)) for key in state_dict.keys + ()), key=lambda item: item[1], reverse=True)) + if not name_lengths: + return + max_name_len = name_lengths[0][1] + if max_name_len > MAX_TENSOR_NAME_LENGTH: + bad_list = ', '.join(f'{key!r} ({namelen})' for key, namelen in + name_lengths if namelen > MAX_TENSOR_NAME_LENGTH) + raise ValueError( + f'Can only handle tensor names up to {MAX_TENSOR_NAME_LENGTH} characters. Tensors exceeding the limit: {bad_list}' + ) + for key, data in tqdm(state_dict.items()): + old_dtype = data.dtype + data = data.to(torch.float32).numpy() + n_dims = len(data.shape) + data_shape = data.shape + data_qtype = getattr(GGMLQuantizationType, 'BF16' if old_dtype == + torch.bfloat16 else 'F16') + if len(data.shape) > MAX_TENSOR_DIMS: + model_arch.handle_nd_tensor(key, data) + continue + n_params = 1 + for dim_size in data_shape: + n_params *= dim_size + if old_dtype in (torch.float32, torch.bfloat16): + if n_dims == 1: + data_qtype = GGMLQuantizationType.F32 + elif n_params <= QUANTIZATION_THRESHOLD: + data_qtype = GGMLQuantizationType.F32 + elif any(x in key for x in model_arch.keys_hiprec): + data_qtype = GGMLQuantizationType.F32 + new_name = key + data_qtype = GGMLQuantizationType.F32 + shape_str = f"{{{', '.join(str(n) for n in reversed(data.shape))}}}" + tqdm.write( + f"{f'%-{max_name_len + 4}s' % f'{new_name}'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}" + ) + writer.add_tensor(new_name, data, raw_dtype=data_qtype) +import os +safetensors_files = [file for file in os.listdir() if file.endswith( + '.safetensors')] +if safetensors_files: + print('Safetensors file(s) available. Select which one to convert:') + for index, file_name in enumerate(safetensors_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = safetensors_files[choice_index] + print(f'Model file: {selected_file} is selected!') + path = selected_file + writer, state_dict, model_arch = load_model(path) + writer.add_quantization_version(GGML_QUANT_VERSION) + if next(iter(state_dict.values())).dtype == torch.bfloat16: + out_path = f'{os.path.splitext(path)[0]}-f32.gguf' + writer.add_file_type(LlamaFileType.MOSTLY_BF16) + else: + out_path = f'{os.path.splitext(path)[0]}-f32.gguf' + writer.add_file_type(LlamaFileType.MOSTLY_F16) + if os.path.isfile(out_path): + input('Output exists enter to continue or ctrl+c to abort!') + handle_tensors(path, writer, state_dict, model_arch) + writer.write_header_to_file(path=out_path) + writer.write_kv_data_to_file() + writer.write_tensors_to_file(progress=True) + writer.close() + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No safetensors files are available in the current directory.') + input('--- Press ENTER To Exit ---') \ No newline at end of file diff --git a/gguf_connector/t9.py b/gguf_connector/t9.py new file mode 100644 index 0000000..57ce856 --- /dev/null +++ b/gguf_connector/t9.py @@ -0,0 +1,79 @@ +import torch +from .writer import GGUFWriter, GGMLQuantizationType +from .const import GGML_QUANT_VERSION, LlamaFileType +from safetensors.torch import load_file +from tqdm import tqdm +import numpy as np +MAX_TENSOR_NAME_LENGTH = 127 +def load_state_dict(path): + state_dict = load_file(path) + return {k: v for k, v in state_dict.items()} +def load_model(path, model_arch): + state_dict = load_state_dict(path) + writer = GGUFWriter(path=None, arch=model_arch) + return writer, state_dict, model_arch +def is_tensor_valid(data, key): + if data is None: + print(f"[WARNING] Skipping tensor '{key}': Empty or NULL tensor.") + return False + if np.any(np.isnan(data)) or np.any(np.isinf(data)): + print(f"[WARNING] Skipping tensor '{key}': Contains NaN or Inf values." + ) + return False + return True +def handle_tensors(writer, state_dict): + name_lengths = [(key, len(key)) for key in state_dict.keys()] + if not name_lengths: + return + max_name_len = max(name_lengths, key=lambda x: x[1])[1] + for key, data in tqdm(state_dict.items(), desc='Processing Tensors'): + old_dtype = data.dtype + print( + f'[INFO] Processing: {key} | Original dtype: {old_dtype} | Shape: {data.shape}' + ) + data = data.to(torch.float32).numpy() + if not is_tensor_valid(data, key): + continue + data_qtype = GGMLQuantizationType.F32 + shape_str = f"{{{', '.join(map(str, reversed(data.shape)))}}}" + print( + f'[INFO] Writing: {key.ljust(max_name_len)} | {old_dtype} -> {data_qtype.name} | Shape: {shape_str}' + ) + writer.add_tensor(key, data, raw_dtype=data_qtype) +import os +safetensors_files = [file for file in os.listdir() if file.endswith( + '.safetensors')] +if safetensors_files: + print('Safetensors file(s) available. Select which one to convert:') + for index, file_name in enumerate(safetensors_files, start=1): + print(f'{index}. {file_name}') + choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ') + try: + choice_index = int(choice) - 1 + selected_file = safetensors_files[choice_index] + print(f'Model file: {selected_file} is selected!') + path = selected_file + ask = input('Assign a name for the model (Y/n)? ') + if ask.lower() == 'y': + given = input('Enter a model name: ') + else: + given = None + writer, state_dict, _ = load_model(path, given) + writer.add_quantization_version(GGML_QUANT_VERSION) + out_path = f'{os.path.splitext(path)[0]}-f32.gguf' + writer.add_file_type(LlamaFileType.MOSTLY_F16) + if os.path.isfile(out_path): + input( + 'Output file exists. Press Enter to overwrite or Ctrl+C to abort.' + ) + handle_tensors(writer, state_dict) + writer.write_header_to_file(path=out_path) + writer.write_kv_data_to_file() + writer.write_tensors_to_file(progress=True) + writer.close() + print(f'Conversion completed: {out_path}') + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No safetensors files are available in the current directory.') + input('--- Press ENTER To Exit ---') \ No newline at end of file diff --git a/gguf_connector/tkn.py b/gguf_connector/tkn.py new file mode 100644 index 0000000..6a6643b --- /dev/null +++ b/gguf_connector/tkn.py @@ -0,0 +1,60 @@ +import torch +from .reader import GGUFReader, GGUFValueType +def get_field(reader, field_name, field_type): + field = reader.get_field(field_name) + if field is None: + return None + elif field_type == str: + if len(field.types) != 1 or field.types[0] != GGUFValueType.STRING: + raise TypeError( + f'Bad type for GGUF {field_name} key: expected string, got {field.types!r}' + ) + return str(field.parts[field.data[-1]], encoding='utf-8') + elif field_type in [int, float, bool]: + return field_type(field.parts[field.data[-1]]) + else: + raise TypeError(f'Unknown field type {field_type}') +def get_list_field(reader, field_name, field_type): + field = reader.get_field(field_name) + if field is None: + return None + elif field_type == str: + return tuple(str(field.parts[part_idx], encoding='utf-8') for + part_idx in field.data) + elif field_type in [int, float, bool]: + return tuple(field_type(field.parts[part_idx][0]) for part_idx in + field.data) + else: + raise TypeError(f'Unknown field type {field_type}') +def tokenizer_builder(path): + print(f'Attempting to rebuild sentencepiece tokenizer from metadata..') + try: + from sentencepiece import sentencepiece_model_pb2 as model + except ImportError: + raise ImportError('protobuf is required; pip install protobuf') + spm = model.ModelProto() + reader = GGUFReader(path) + spm.trainer_spec.model_type == 1 + spm.normalizer_spec.add_dummy_prefix = get_field(reader, + 'tokenizer.ggml.add_space_prefix', bool) + tokens = get_list_field(reader, 'tokenizer.ggml.tokens', str) + scores = get_list_field(reader, 'tokenizer.ggml.scores', float) + toktypes = get_list_field(reader, 'tokenizer.ggml.token_type', int) + for idx, (token, score, toktype) in enumerate(zip(tokens, scores, toktypes) + ): + piece = spm.SentencePiece() + piece.piece = token + piece.score = score + piece.type = toktype + spm.pieces.append(piece) + spm.trainer_spec.byte_fallback = True + spm.trainer_spec.vocab_size = len(tokens) + spm.trainer_spec.max_sentence_length = 4096 + spm.trainer_spec.eos_id = get_field(reader, + 'tokenizer.ggml.eos_token_id', int) + spm.trainer_spec.pad_id = get_field(reader, + 'tokenizer.ggml.padding_token_id', int) + print( + f'Rebuilt tokenizer successfully with vocab size of {len(spm.pieces)}') + del reader + return torch.ByteTensor(list(spm.SerializeToString())) \ No newline at end of file diff --git a/gguf_connector/u.py b/gguf_connector/u.py new file mode 100644 index 0000000..3a18347 --- /dev/null +++ b/gguf_connector/u.py @@ -0,0 +1 @@ +from gguf_cutter import downloader \ No newline at end of file diff --git a/gguf_connector/v.py b/gguf_connector/v.py new file mode 100644 index 0000000..68ba606 --- /dev/null +++ b/gguf_connector/v.py @@ -0,0 +1,48 @@ +import os +gguf_files = [file for file in os.listdir() if file.endswith('.gguf')] +if gguf_files: + print('GGUF file(s) available. Select which one to use as Clip Handler:') + for index, file_name in enumerate(gguf_files, start=1): + print(f'{index}. {file_name}') + choice1 = input(f'Enter your choice (1 to {len(gguf_files)}): ') + try: + choice_index = int(choice1) - 1 + selected_file = gguf_files[choice_index] + print(f'Model file: {selected_file} is selected!') + clip_model_path = selected_file + from llama_cpp.llama_chat_format import Llava15ChatHandler + chat_handler = Llava15ChatHandler(clip_model_path) + print( + 'GGUF file(s) available. Select which one to use as Vision Model:') + for index, file_name in enumerate(gguf_files, start=1): + print(f'{index}. {file_name}') + choice2 = input(f'Enter your choice (1 to {len(gguf_files)}): ') + try: + choice_index = int(choice2) - 1 + selected_file = gguf_files[choice_index] + print(f'Model file: {selected_file} is selected!') + model_path = selected_file + from llama_cpp import Llama + llm = Llama(model_path=model_path, chat_handler=chat_handler, + n_ctx=2048) + while True: + ask = input('Provide a picture URL (Q for quit): ') + if ask.lower() == 'q': + break + from rich.progress import Progress + with Progress(transient=True) as progress: + task = progress.add_task('Processing', total=None) + response = llm.create_chat_completion(messages=[{'role': + 'user', 'content': [{'type': 'text', 'text': + "What's in this image?"}, {'type': 'image_url', + 'image_url': {'url': ask}}]}]) + print('\nOutput:' + response['choices'][0]['message'][ + 'content'] + '\n') + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') + except (ValueError, IndexError): + print('Invalid choice. Please enter a valid number.') +else: + print('No GGUF files are available in the current directory.') + input('--- Press ENTER To Exit ---') +print('Goodbye!') \ No newline at end of file diff --git a/gguf_connector/v2.py b/gguf_connector/v2.py new file mode 100644 index 0000000..733bf02 --- /dev/null +++ b/gguf_connector/v2.py @@ -0,0 +1,56 @@ +import torch +import gradio as gr +from transformers import UMT5EncoderModel +from diffusers import WanPipeline, AutoencoderKLWan, WanTransformer3DModel, GGUFQuantizationConfig +from diffusers.utils import export_to_video +model_path = ( + 'https://huggingface.co/calcuis/wan-gguf/blob/main/wan2.1_t2v_1.3b-q8_0.gguf' + ) +transformer = WanTransformer3DModel.from_single_file(model_path, + quantization_config=GGUFQuantizationConfig(compute_dtype=torch.bfloat16 + ), torch_dtype=torch.bfloat16) +text_encoder = UMT5EncoderModel.from_pretrained('chatpig/umt5xxl-encoder-gguf', + gguf_file='umt5xxl-encoder-q4_0.gguf', torch_dtype=torch.bfloat16) +vae = AutoencoderKLWan.from_pretrained('callgg/t2v-decoder', subfolder= + 'vae', torch_dtype=torch.float32) +pipe = WanPipeline.from_pretrained('callgg/t2v-decoder', transformer= + transformer, text_encoder=text_encoder, vae=vae, torch_dtype=torch.bfloat16 + ) +pipe.enable_model_cpu_offload() +def generate_video(prompt, negative_prompt, width, height, num_frames, + num_inference_steps, fps): + video = pipe(prompt=prompt, negative_prompt=negative_prompt, width= + width, height=height, num_frames=num_frames, num_inference_steps= + num_inference_steps).frames[0] + export_to_video(video, 'output.mp4', fps=fps) + return 'output.mp4' +sample_prompts = [ + 'A cat and a dog baking a cake together in a kitchen. The cat is carefully measuring flour, while the dog is stirring the batter with a wooden spoon. The kitchen is cozy, with sunlight streaming through the window.' + , + 'A pig moving quickly in a beautiful winter scenery nature trees sunset tracking camera.' + ] +sample_prompts = [[x] for x in sample_prompts] +block = gr.Blocks(title='gguf').queue() +with block: + gr.Markdown('## 🎥 Video 2') + with gr.Row(): + prompt_input = gr.Textbox(label='Prompt', placeholder= + 'Enter your prompt here (or click Sample Prompt)', value='') + neg_prompt_input = gr.Textbox(label='Negative Prompt', value= + 'blurry ugly bad', visible=False) + quick_prompts = gr.Dataset(samples=sample_prompts, label= + 'Sample Prompt', samples_per_page=1000, components=[prompt_input]) + quick_prompts.click(lambda x: x[0], inputs=[quick_prompts], outputs + =prompt_input, show_progress=False, queue=False) + with gr.Row(): + width_input = gr.Number(label='Width', value=512) + height_input = gr.Number(label='Height', value=512) + num_frames_input = gr.Number(label='Number of Frames', value=25) + num_steps_input = gr.Number(label='Inference Steps', value=25) + fps_input = gr.Number(label='FPS', value=16) + generate_btn = gr.Button('Generate Video') + output_video = gr.Video(label='Generated Video') + generate_btn.click(fn=generate_video, inputs=[prompt_input, + neg_prompt_input, width_input, height_input, num_frames_input, + num_steps_input, fps_input], outputs=output_video) +block.launch() \ No newline at end of file diff --git a/gguf_connector/vg.py b/gguf_connector/vg.py new file mode 100644 index 0000000..5981f0c --- /dev/null +++ b/gguf_connector/vg.py @@ -0,0 +1,52 @@ +import torch +import gradio as gr +from transformers import T5EncoderModel +from diffusers import LTXPipeline, GGUFQuantizationConfig, LTXVideoTransformer3DModel +from diffusers.utils import export_to_video +model_path = ( + 'https://huggingface.co/calcuis/ltxv-gguf/blob/main/ltxv-2b-0.9.6-distilled-fp32-q8_0.gguf' + ) +transformer = LTXVideoTransformer3DModel.from_single_file(model_path, + quantization_config=GGUFQuantizationConfig(compute_dtype=torch.bfloat16 + ), torch_dtype=torch.bfloat16) +text_encoder = T5EncoderModel.from_pretrained('calcuis/ltxv-gguf', + gguf_file='t5xxl_fp16-q4_0.gguf', torch_dtype=torch.bfloat16) +pipe = LTXPipeline.from_pretrained('callgg/ltxv0.9.6-decoder', text_encoder + =text_encoder, transformer=transformer, torch_dtype=torch.bfloat16).to( + 'cuda') +def generate_video(prompt, negative_prompt, width, height, num_frames, + num_inference_steps, fps): + video = pipe(prompt=prompt, negative_prompt=negative_prompt, width= + width, height=height, num_frames=num_frames, num_inference_steps= + num_inference_steps).frames[0] + export_to_video(video, 'output.mp4', fps=fps) + return 'output.mp4' +sample_prompts = [ + 'A drone quickly rises through a bank of morning fog, revealing a pristine alpine lake surrounded by snow-capped mountains. The camera glides forward over the glassy water, capturing perfect reflections of the peaks. As it continues, the perspective shifts to reveal a lone wooden cabin with a curl of smoke from its chimney, nestled among tall pines at the lake edge. The final shot tracks upward rapidly, transitioning from intimate to epic as the full mountain range comes into view, bathed in the golden light of sunrise breaking through scattered clouds.' + ] +sample_prompts = [[x] for x in sample_prompts] +block = gr.Blocks(title='gguf').queue() +with block: + gr.Markdown('## 🎥 Video Generator (t2v)') + with gr.Row(): + prompt_input = gr.Textbox(label='Prompt', placeholder= + 'Enter your prompt here (or click Sample Prompt)', value='') + neg_prompt_input = gr.Textbox(label='Negative Prompt', value= + 'low quality, worst quality, deformed, distorted, disfigured, motion smear, motion artifacts, fused fingers, bad anatomy, weird hand, ugly' + , visible=False) + quick_prompts = gr.Dataset(samples=sample_prompts, label= + 'Sample Prompt', samples_per_page=1000, components=[prompt_input]) + quick_prompts.click(lambda x: x[0], inputs=[quick_prompts], outputs + =prompt_input, show_progress=False, queue=False) + with gr.Row(): + width_input = gr.Number(label='Width', value=512) + height_input = gr.Number(label='Height', value=512) + num_frames_input = gr.Number(label='Number of Frames', value=25) + num_steps_input = gr.Number(label='Inference Steps', value=25) + fps_input = gr.Number(label='FPS', value=24) + generate_btn = gr.Button('Generate Video') + output_video = gr.Video(label='Generated Video') + generate_btn.click(fn=generate_video, inputs=[prompt_input, + neg_prompt_input, width_input, height_input, num_frames_input, + num_steps_input, fps_input], outputs=output_video) +block.launch() \ No newline at end of file diff --git a/gguf_connector/vg2.py b/gguf_connector/vg2.py new file mode 100644 index 0000000..859473e --- /dev/null +++ b/gguf_connector/vg2.py @@ -0,0 +1,57 @@ +import torch +import gradio as gr +from diffusers.pipelines.ltx.pipeline_ltx_condition import LTXConditionPipeline, LTXVideoCondition +from diffusers import GGUFQuantizationConfig, LTXVideoTransformer3DModel +from diffusers.utils import export_to_video +from transformers import T5EncoderModel +model_path = ( + 'https://huggingface.co/calcuis/ltxv-gguf/blob/main/ltxv-2b-0.9.6-distilled-fp32-q8_0.gguf' + ) +transformer = LTXVideoTransformer3DModel.from_single_file(model_path, + quantization_config=GGUFQuantizationConfig(compute_dtype=torch.bfloat16 + ), torch_dtype=torch.bfloat16) +text_encoder = T5EncoderModel.from_pretrained('calcuis/ltxv-gguf', + gguf_file='t5xxl_fp16-q4_0.gguf', torch_dtype=torch.bfloat16) +pipe = LTXConditionPipeline.from_pretrained('callgg/ltxv0.9.6-decoder', + transformer=transformer, text_encoder=text_encoder, torch_dtype=torch. + bfloat16).to('cuda') +def generate_video(input_image, prompt, negative_prompt, width, height, + num_frames, num_inference_steps, fps): + image = input_image.convert('RGB') + condition1 = LTXVideoCondition(image=image, frame_index=0) + generator = torch.Generator('cuda').manual_seed(0) + video = pipe(conditions=[condition1], prompt=prompt, negative_prompt= + negative_prompt, width=width, height=height, num_frames=num_frames, + num_inference_steps=num_inference_steps, generator=generator).frames[0] + export_to_video(video, 'output.mp4', fps=fps) + return 'output.mp4' +sample_prompts = [ + 'The girl dances gracefully, with clear movements, full of charm.', + 'A character doing some simple body movements.'] +sample_prompts = [[x] for x in sample_prompts] +block = gr.Blocks(title='gguf').queue() +with block: + gr.Markdown('## 🎥 Video Generator (i2v)') + with gr.Row(): + input_image = gr.Image(type='pil', label='Upload Image') + prompt = gr.Textbox(label='Prompt', placeholder= + 'Enter your prompt here (or click Sample Prompt)', value='') + negative_prompt = gr.Textbox(label='Negative Prompt', value= + 'worst quality, inconsistent motion, blurry, jittery, distorted', + visible=False) + quick_prompts = gr.Dataset(samples=sample_prompts, label= + 'Sample Prompt', samples_per_page=1000, components=[prompt]) + quick_prompts.click(lambda x: x[0], inputs=[quick_prompts], outputs= + prompt, show_progress=False, queue=False) + with gr.Row(): + width = gr.Number(label='Width', value=512) + height = gr.Number(label='Height', value=512) + num_frames = gr.Number(label='Num Frames', value=25) + num_inference_steps = gr.Number(label='Num Inference Steps', value=25) + fps = gr.Number(label='FPS', value=24) + generate_btn = gr.Button('Generate') + output_video = gr.Video(label='Generated Video') + generate_btn.click(fn=generate_video, inputs=[input_image, prompt, + negative_prompt, width, height, num_frames, num_inference_steps, + fps], outputs=output_video) +block.launch() \ No newline at end of file diff --git a/gguf_connector/w.py b/gguf_connector/w.py new file mode 100644 index 0000000..af9b50a --- /dev/null +++ b/gguf_connector/w.py @@ -0,0 +1,3 @@ +import webbrowser +print('activating the default browser...') +webbrowser.open('https://gguf.us') \ No newline at end of file diff --git a/gguf_connector/writer.py b/gguf_connector/writer.py new file mode 100644 index 0000000..e5ff0aa --- /dev/null +++ b/gguf_connector/writer.py @@ -0,0 +1,801 @@ +from __future__ import annotations +import logging, os, shutil, struct, tempfile +from dataclasses import dataclass +from enum import Enum, auto +from math import prod +from pathlib import Path +from io import BufferedWriter +from typing import IO, Any, Sequence, Mapping +from string import ascii_letters, digits +import numpy as np +from .const import GGUF_DEFAULT_ALIGNMENT, GGUF_MAGIC, GGUF_VERSION, GGMLQuantizationType, GGUFEndian, GGUFValueType, Keys, RopeScalingType, PoolingType, TokenType, ExpertGatingFuncType +from .quant import quant_shape_from_byte_shape +logger = logging.getLogger(__name__) +SHARD_NAME_FORMAT = '{:s}-{:05d}-of-{:05d}.gguf' +@dataclass +class TensorInfo: + shape: Sequence[int] + dtype: GGMLQuantizationType + nbytes: int + tensor: np.ndarray[Any, Any] | None = None +@dataclass +class GGUFValue: + value: Any + type: GGUFValueType +class WriterState(Enum): + NO_FILE = auto() + EMPTY = auto() + HEADER = auto() + KV_DATA = auto() + TI_DATA = auto() + WEIGHTS = auto() +class GGUFWriter: + fout: list[BufferedWriter] | None + path: Path | None + temp_file: tempfile.SpooledTemporaryFile[bytes] | None + tensors: list[dict[str, TensorInfo]] + kv_data: list[dict[str, GGUFValue]] + state: WriterState + _simple_value_packing = {GGUFValueType.UINT8: 'B', GGUFValueType.INT8: + 'b', GGUFValueType.UINT16: 'H', GGUFValueType.INT16: 'h', + GGUFValueType.UINT32: 'I', GGUFValueType.INT32: 'i', GGUFValueType. + FLOAT32: 'f', GGUFValueType.UINT64: 'Q', GGUFValueType.INT64: 'q', + GGUFValueType.FLOAT64: 'd', GGUFValueType.BOOL: '?'} + def __init__(self, path, arch, use_temp_file=False, endianess= + GGUFEndian.LITTLE, split_max_tensors=0, split_max_size=0, dry_run= + False, small_first_shard=False): + self.fout = None + self.path = Path(path) if path else None + self.arch = arch + self.endianess = endianess + self.data_alignment = GGUF_DEFAULT_ALIGNMENT + self.use_temp_file = use_temp_file + self.temp_file = None + self.tensors = [{}] + self.kv_data = [{}] + self.split_max_tensors = split_max_tensors + self.split_max_size = split_max_size + self.dry_run = dry_run + self.small_first_shard = small_first_shard + logger.info('gguf: This GGUF file is for {0} Endian only'.format( + 'Big' if self.endianess == GGUFEndian.BIG else 'Little')) + self.state = WriterState.NO_FILE + if self.small_first_shard: + self.tensors.append({}) + self.add_architecture() + def get_total_parameter_count(self): + total_params = 0 + shared_params = 0 + expert_params = 0 + expert_sum = 0 + n_expert_tensors = 0 + last_lora_a: tuple[str, TensorInfo] | None = None + for tensors in self.tensors: + for name, info in tensors.items(): + shape = info.shape + if name.endswith('.lora_a'): + last_lora_a = name, info + continue + elif name.endswith('.lora_b'): + if last_lora_a is None or last_lora_a[0] != name[:-1 + ] + 'a': + logger.warning( + "can't measure LoRA size correctly, tensor order is unusual" + ) + return 0, 0, 0, 0 + else: + shape = *shape[:-1], last_lora_a[1].shape[-1] + size = prod(shape) + if '_exps.' in name: + expert_params += size // shape[-3] + expert_sum += shape[-3] + n_expert_tensors += 1 + else: + shared_params += size + total_params += size + expert_count = (expert_sum // n_expert_tensors if n_expert_tensors > + 0 else 0) + if last_lora_a is not None: + total_params = -total_params + return total_params, shared_params, expert_params, expert_count + def format_shard_names(self, path): + if len(self.tensors) == 1: + return [path] + return [path.with_name(SHARD_NAME_FORMAT.format(path.stem, i + 1, + len(self.tensors))) for i in range(len(self.tensors))] + def open_output_file(self, path=None): + if self.state is WriterState.EMPTY and self.fout is not None and ( + path is None or path == self.path): + return + if self.state is not WriterState.NO_FILE: + raise ValueError( + f'Expected output file to be not yet opened, got {self.state}') + if path is not None: + self.path = path + if self.path is not None: + filenames = self.print_plan() + self.fout = [open(filename, 'wb') for filename in filenames] + self.state = WriterState.EMPTY + def print_plan(self): + logger.info('Writing the following files:') + assert self.path is not None + filenames = self.format_shard_names(self.path) + assert len(filenames) == len(self.tensors) + for name, tensors in zip(filenames, self.tensors): + logger.info( + f'{name}: n_tensors = {len(tensors)}, total_size = {GGUFWriter.format_n_bytes_to_str(sum(ti.nbytes for ti in tensors.values()))}' + ) + if self.dry_run: + logger.info('Dry run, not writing files') + for name in filenames: + print(name) + exit() + return filenames + def add_shard_kv_data(self): + if len(self.tensors) == 1: + return + total_tensors = sum(len(t) for t in self.tensors) + assert self.fout is not None + total_splits = len(self.fout) + self.kv_data.extend({} for _ in range(len(self.kv_data), total_splits)) + for i, kv_data in enumerate(self.kv_data): + kv_data[Keys.Split.LLM_KV_SPLIT_NO] = GGUFValue(i, + GGUFValueType.UINT16) + kv_data[Keys.Split.LLM_KV_SPLIT_COUNT] = GGUFValue(total_splits, + GGUFValueType.UINT16) + kv_data[Keys.Split.LLM_KV_SPLIT_TENSORS_COUNT] = GGUFValue( + total_tensors, GGUFValueType.INT32) + def write_header_to_file(self, path=None): + if len(self.tensors) == 1 and (self.split_max_tensors != 0 or self. + split_max_size != 0): + logger.warning('Model fails split requirements, not splitting') + self.open_output_file(path) + if self.state is not WriterState.EMPTY: + raise ValueError( + f'Expected output file to be empty, got {self.state}') + assert self.fout is not None + assert len(self.fout) == len(self.tensors) + assert len(self.kv_data) == 1 + self.add_shard_kv_data() + for fout, tensors, kv_data in zip(self.fout, self.tensors, self.kv_data + ): + fout.write(self._pack(' 0: + if self.split_max_tensors != 0 and len(self.tensors[-1] + ) >= self.split_max_tensors or self.split_max_size != 0 and sum( + ti.nbytes for ti in self.tensors[-1].values() + ) + tensor_nbytes > self.split_max_size: + self.tensors.append({}) + self.tensors[-1][name] = TensorInfo(shape=tensor_shape, dtype=dtype, + nbytes=tensor_nbytes) + def add_tensor(self, name, tensor, raw_shape=None, raw_dtype=None): + if self.endianess == GGUFEndian.BIG: + tensor.byteswap(inplace=True) + if self.use_temp_file and self.temp_file is None: + fp = tempfile.SpooledTemporaryFile(mode='w+b', max_size=256 * + 1024 * 1024) + fp.seek(0) + self.temp_file = fp + shape: Sequence[int + ] = raw_shape if raw_shape is not None else tensor.shape + self.add_tensor_info(name, shape, tensor.dtype, tensor.nbytes, + raw_dtype=raw_dtype) + if self.temp_file is None: + self.tensors[-1][name].tensor = tensor + return + tensor.tofile(self.temp_file) + self.write_padding(self.temp_file, tensor.nbytes) + def write_padding(self, fp, n, align=None): + pad = GGUFWriter.ggml_pad(n, align if align is not None else self. + data_alignment) - n + if pad != 0: + fp.write(bytes([0] * pad)) + def write_tensor_data(self, tensor): + if (self.state is not WriterState.TI_DATA and self.state is not + WriterState.WEIGHTS): + raise ValueError( + f'Expected output file to contain tensor info or weights, got {self.state}' + ) + assert self.fout is not None + if self.endianess == GGUFEndian.BIG: + tensor.byteswap(inplace=True) + file_id = -1 + for i, tensors in enumerate(self.tensors): + if len(tensors) > 0: + file_id = i + break + fout = self.fout[file_id] + first_tensor_name = [name for name, _ in zip(self.tensors[file_id]. + keys(), range(1))][0] + ti = self.tensors[file_id].pop(first_tensor_name) + assert ti.nbytes == tensor.nbytes + self.write_padding(fout, fout.tell()) + tensor.tofile(fout) + self.write_padding(fout, tensor.nbytes) + self.state = WriterState.WEIGHTS + def write_tensors_to_file(self, *, progress: bool=False): + self.write_ti_data_to_file() + assert self.fout is not None + for fout in self.fout: + self.write_padding(fout, fout.tell()) + if self.temp_file is None: + shard_bar = None + bar = None + if progress: + from tqdm import tqdm + total_bytes = sum(ti.nbytes for t in self.tensors for ti in + t.values()) + if len(self.fout) > 1: + shard_bar = tqdm(desc=f'Shard (0/{len(self.fout)})', + total=None, unit='byte', unit_scale=True) + bar = tqdm(desc='Writing', total=total_bytes, unit='byte', + unit_scale=True) + for i, (fout, tensors) in enumerate(zip(self.fout, self.tensors)): + if shard_bar is not None: + shard_bar.set_description( + f'Shard ({i + 1}/{len(self.fout)})') + total = sum(ti.nbytes for ti in tensors.values()) + shard_bar.reset(total=total if total > 0 else None) + for ti in tensors.values(): + assert ti.tensor is not None + assert ti.tensor.nbytes == ti.nbytes + ti.tensor.tofile(fout) + if shard_bar is not None: + shard_bar.update(ti.nbytes) + if bar is not None: + bar.update(ti.nbytes) + self.write_padding(fout, ti.nbytes) + ti.tensor = None + else: + self.temp_file.seek(0) + shutil.copyfileobj(self.temp_file, self.fout[0 if not self. + small_first_shard else 1]) + self.flush() + self.temp_file.close() + self.state = WriterState.WEIGHTS + def flush(self): + assert self.fout is not None + for fout in self.fout: + fout.flush() + def close(self): + if self.fout is not None: + for fout in self.fout: + fout.close() + self.fout = None + def add_type(self, type_name): + self.add_string(Keys.General.TYPE, type_name) + def add_architecture(self): + self.add_string(Keys.General.ARCHITECTURE, self.arch) + def add_quantization_version(self, quantization_version): + self.add_uint32(Keys.General.QUANTIZATION_VERSION, quantization_version + ) + def add_custom_alignment(self, alignment): + self.data_alignment = alignment + self.add_uint32(Keys.General.ALIGNMENT, alignment) + def add_file_type(self, ftype): + self.add_uint32(Keys.General.FILE_TYPE, ftype) + def add_name(self, name): + self.add_string(Keys.General.NAME, name) + def add_author(self, author): + self.add_string(Keys.General.AUTHOR, author) + def add_version(self, version): + self.add_string(Keys.General.VERSION, version) + def add_organization(self, organization): + self.add_string(Keys.General.ORGANIZATION, organization) + def add_finetune(self, finetune): + self.add_string(Keys.General.FINETUNE, finetune) + def add_basename(self, basename): + self.add_string(Keys.General.BASENAME, basename) + def add_description(self, description): + self.add_string(Keys.General.DESCRIPTION, description) + def add_quantized_by(self, quantized): + self.add_string(Keys.General.QUANTIZED_BY, quantized) + def add_size_label(self, size_label): + self.add_string(Keys.General.SIZE_LABEL, size_label) + def add_license(self, license): + self.add_string(Keys.General.LICENSE, license) + def add_license_name(self, license): + self.add_string(Keys.General.LICENSE_NAME, license) + def add_license_link(self, license): + self.add_string(Keys.General.LICENSE_LINK, license) + def add_url(self, url): + self.add_string(Keys.General.URL, url) + def add_doi(self, doi): + self.add_string(Keys.General.DOI, doi) + def add_uuid(self, uuid): + self.add_string(Keys.General.UUID, uuid) + def add_repo_url(self, repo_url): + self.add_string(Keys.General.REPO_URL, repo_url) + def add_source_url(self, url): + self.add_string(Keys.General.SOURCE_URL, url) + def add_source_doi(self, doi): + self.add_string(Keys.General.SOURCE_DOI, doi) + def add_source_uuid(self, uuid): + self.add_string(Keys.General.SOURCE_UUID, uuid) + def add_source_repo_url(self, repo_url): + self.add_string(Keys.General.SOURCE_REPO_URL, repo_url) + def add_base_model_count(self, source_count): + self.add_uint32(Keys.General.BASE_MODEL_COUNT, source_count) + def add_base_model_name(self, source_id, name): + self.add_string(Keys.General.BASE_MODEL_NAME.format(id=source_id), name + ) + def add_base_model_author(self, source_id, author): + self.add_string(Keys.General.BASE_MODEL_AUTHOR.format(id=source_id), + author) + def add_base_model_version(self, source_id, version): + self.add_string(Keys.General.BASE_MODEL_VERSION.format(id=source_id + ), version) + def add_base_model_organization(self, source_id, organization): + self.add_string(Keys.General.BASE_MODEL_ORGANIZATION.format(id= + source_id), organization) + def add_base_model_description(self, source_id, description): + self.add_string(Keys.General.BASE_MODEL_DESCRIPTION.format(id= + source_id), description) + def add_base_model_url(self, source_id, url): + self.add_string(Keys.General.BASE_MODEL_URL.format(id=source_id), url) + def add_base_model_doi(self, source_id, doi): + self.add_string(Keys.General.BASE_MODEL_DOI.format(id=source_id), doi) + def add_base_model_uuid(self, source_id, uuid): + self.add_string(Keys.General.BASE_MODEL_UUID.format(id=source_id), uuid + ) + def add_base_model_repo_url(self, source_id, repo_url): + self.add_string(Keys.General.BASE_MODEL_REPO_URL.format(id= + source_id), repo_url) + def add_dataset_count(self, source_count): + self.add_uint32(Keys.General.DATASET_COUNT, source_count) + def add_dataset_name(self, source_id, name): + self.add_string(Keys.General.DATASET_NAME.format(id=source_id), name) + def add_dataset_author(self, source_id, author): + self.add_string(Keys.General.DATASET_AUTHOR.format(id=source_id), + author) + def add_dataset_version(self, source_id, version): + self.add_string(Keys.General.DATASET_VERSION.format(id=source_id), + version) + def add_dataset_organization(self, source_id, organization): + self.add_string(Keys.General.DATASET_ORGANIZATION.format(id= + source_id), organization) + def add_dataset_description(self, source_id, description): + self.add_string(Keys.General.DATASET_DESCRIPTION.format(id= + source_id), description) + def add_dataset_url(self, source_id, url): + self.add_string(Keys.General.DATASET_URL.format(id=source_id), url) + def add_dataset_doi(self, source_id, doi): + self.add_string(Keys.General.DATASET_DOI.format(id=source_id), doi) + def add_dataset_uuid(self, source_id, uuid): + self.add_string(Keys.General.DATASET_UUID.format(id=source_id), uuid) + def add_dataset_repo_url(self, source_id, repo_url): + self.add_string(Keys.General.DATASET_REPO_URL.format(id=source_id), + repo_url) + def add_tags(self, tags): + self.add_array(Keys.General.TAGS, tags) + def add_languages(self, languages): + self.add_array(Keys.General.LANGUAGES, languages) + def add_tensor_data_layout(self, layout): + self.add_string(Keys.LLM.TENSOR_DATA_LAYOUT.format(arch=self.arch), + layout) + def add_vocab_size(self, size): + self.add_uint32(Keys.LLM.VOCAB_SIZE.format(arch=self.arch), size) + def add_context_length(self, length): + self.add_uint32(Keys.LLM.CONTEXT_LENGTH.format(arch=self.arch), length) + def add_embedding_length(self, length): + self.add_uint32(Keys.LLM.EMBEDDING_LENGTH.format(arch=self.arch), + length) + def add_features_length(self, length): + self.add_uint32(Keys.LLM.FEATURES_LENGTH.format(arch=self.arch), length + ) + def add_posnet_embedding_length(self, length): + self.add_uint32(Keys.PosNet.EMBEDDING_LENGTH.format(arch=self.arch), + length) + def add_posnet_block_count(self, length): + self.add_uint32(Keys.PosNet.BLOCK_COUNT.format(arch=self.arch), length) + def add_convnext_embedding_length(self, length): + self.add_uint32(Keys.ConvNext.EMBEDDING_LENGTH.format(arch=self. + arch), length) + def add_convnext_block_count(self, length): + self.add_uint32(Keys.ConvNext.BLOCK_COUNT.format(arch=self.arch), + length) + def add_block_count(self, length): + self.add_uint32(Keys.LLM.BLOCK_COUNT.format(arch=self.arch), length) + def add_leading_dense_block_count(self, length): + self.add_uint32(Keys.LLM.LEADING_DENSE_BLOCK_COUNT.format(arch=self + .arch), length) + def add_feed_forward_length(self, length): + if isinstance(length, int): + self.add_uint32(Keys.LLM.FEED_FORWARD_LENGTH.format(arch=self. + arch), length) + else: + self.add_array(Keys.LLM.FEED_FORWARD_LENGTH.format(arch=self. + arch), length) + def add_expert_feed_forward_length(self, length): + self.add_uint32(Keys.LLM.EXPERT_FEED_FORWARD_LENGTH.format(arch= + self.arch), length) + def add_expert_shared_feed_forward_length(self, length): + self.add_uint32(Keys.LLM.EXPERT_SHARED_FEED_FORWARD_LENGTH.format( + arch=self.arch), length) + def add_parallel_residual(self, use): + self.add_bool(Keys.LLM.USE_PARALLEL_RESIDUAL.format(arch=self.arch), + use) + def add_decoder_start_token_id(self, id): + self.add_uint32(Keys.LLM.DECODER_START_TOKEN_ID.format(arch=self. + arch), id) + def add_head_count(self, count): + if isinstance(count, int): + self.add_uint32(Keys.Attention.HEAD_COUNT.format(arch=self.arch + ), count) + else: + self.add_array(Keys.Attention.HEAD_COUNT.format(arch=self.arch), + count) + def add_head_count_kv(self, count): + if isinstance(count, int): + self.add_uint32(Keys.Attention.HEAD_COUNT_KV.format(arch=self. + arch), count) + else: + self.add_array(Keys.Attention.HEAD_COUNT_KV.format(arch=self. + arch), count) + def add_key_length(self, length): + self.add_uint32(Keys.Attention.KEY_LENGTH.format(arch=self.arch), + length) + def add_value_length(self, length): + self.add_uint32(Keys.Attention.VALUE_LENGTH.format(arch=self.arch), + length) + def add_key_length_mla(self, length): + self.add_uint32(Keys.Attention.KEY_LENGTH_MLA.format(arch=self.arch + ), length) + def add_value_length_mla(self, length): + self.add_uint32(Keys.Attention.VALUE_LENGTH_MLA.format(arch=self. + arch), length) + def add_max_alibi_bias(self, bias): + self.add_float32(Keys.Attention.MAX_ALIBI_BIAS.format(arch=self. + arch), bias) + def add_clamp_kqv(self, value): + self.add_float32(Keys.Attention.CLAMP_KQV.format(arch=self.arch), value + ) + def add_logit_scale(self, value): + self.add_float32(Keys.LLM.LOGIT_SCALE.format(arch=self.arch), value) + def add_attn_logit_softcapping(self, value): + self.add_float32(Keys.LLM.ATTN_LOGIT_SOFTCAPPING.format(arch=self. + arch), value) + def add_final_logit_softcapping(self, value): + self.add_float32(Keys.LLM.FINAL_LOGIT_SOFTCAPPING.format(arch=self. + arch), value) + def add_expert_count(self, count): + self.add_uint32(Keys.LLM.EXPERT_COUNT.format(arch=self.arch), count) + def add_expert_used_count(self, count): + self.add_uint32(Keys.LLM.EXPERT_USED_COUNT.format(arch=self.arch), + count) + def add_expert_shared_count(self, count): + self.add_uint32(Keys.LLM.EXPERT_SHARED_COUNT.format(arch=self.arch), + count) + def add_expert_weights_scale(self, value): + self.add_float32(Keys.LLM.EXPERT_WEIGHTS_SCALE.format(arch=self. + arch), value) + def add_expert_weights_norm(self, value): + self.add_bool(Keys.LLM.EXPERT_WEIGHTS_NORM.format(arch=self.arch), + value) + def add_expert_gating_func(self, value): + self.add_uint32(Keys.LLM.EXPERT_GATING_FUNC.format(arch=self.arch), + value.value) + def add_swin_norm(self, value): + self.add_bool(Keys.LLM.SWIN_NORM.format(arch=self.arch), value) + def add_rescale_every_n_layers(self, count): + self.add_uint32(Keys.LLM.RESCALE_EVERY_N_LAYERS.format(arch=self. + arch), count) + def add_time_mix_extra_dim(self, dim): + self.add_uint32(Keys.LLM.TIME_MIX_EXTRA_DIM.format(arch=self.arch), dim + ) + def add_time_decay_extra_dim(self, dim): + self.add_uint32(Keys.LLM.TIME_DECAY_EXTRA_DIM.format(arch=self.arch + ), dim) + def add_residual_scale(self, value): + self.add_float32(Keys.LLM.RESIDUAL_SCALE.format(arch=self.arch), value) + def add_embedding_scale(self, value): + self.add_float32(Keys.LLM.EMBEDDING_SCALE.format(arch=self.arch), value + ) + def add_wkv_head_size(self, size): + self.add_uint32(Keys.WKV.HEAD_SIZE.format(arch=self.arch), size) + def add_token_shift_count(self, count): + self.add_uint32(Keys.LLM.TOKEN_SHIFT_COUNT.format(arch=self.arch), + count) + def add_interleave_moe_layer_step(self, value): + self.add_uint32(Keys.LLM.INTERLEAVE_MOE_LAYER_STEP.format(arch=self + .arch), value) + def add_layer_norm_eps(self, value): + self.add_float32(Keys.Attention.LAYERNORM_EPS.format(arch=self.arch + ), value) + def add_layer_norm_rms_eps(self, value): + self.add_float32(Keys.Attention.LAYERNORM_RMS_EPS.format(arch=self. + arch), value) + def add_group_norm_eps(self, value): + self.add_float32(Keys.Attention.GROUPNORM_EPS.format(arch=self.arch + ), value) + def add_group_norm_groups(self, value): + self.add_uint32(Keys.Attention.GROUPNORM_GROUPS.format(arch=self. + arch), value) + def add_causal_attention(self, value): + self.add_bool(Keys.Attention.CAUSAL.format(arch=self.arch), value) + def add_q_lora_rank(self, length): + self.add_uint32(Keys.Attention.Q_LORA_RANK.format(arch=self.arch), + length) + def add_kv_lora_rank(self, length): + self.add_uint32(Keys.Attention.KV_LORA_RANK.format(arch=self.arch), + length) + def add_decay_lora_rank(self, length): + self.add_uint32(Keys.Attention.DECAY_LORA_RANK.format(arch=self. + arch), length) + def add_iclr_lora_rank(self, length): + self.add_uint32(Keys.Attention.ICLR_LORA_RANK.format(arch=self.arch + ), length) + def add_value_residual_mix_lora_rank(self, length): + self.add_uint32(Keys.Attention.VALUE_RESIDUAL_MIX_LORA_RANK.format( + arch=self.arch), length) + def add_gate_lora_rank(self, length): + self.add_uint32(Keys.Attention.GATE_LORA_RANK.format(arch=self.arch + ), length) + def add_relative_attn_buckets_count(self, value): + self.add_uint32(Keys.Attention.REL_BUCKETS_COUNT.format(arch=self. + arch), value) + def add_sliding_window(self, value): + self.add_uint32(Keys.Attention.SLIDING_WINDOW.format(arch=self.arch + ), value) + def add_attention_scale(self, value): + self.add_float32(Keys.Attention.SCALE.format(arch=self.arch), value) + def add_pooling_type(self, value): + self.add_uint32(Keys.LLM.POOLING_TYPE.format(arch=self.arch), value + .value) + def add_rope_dimension_count(self, count): + self.add_uint32(Keys.Rope.DIMENSION_COUNT.format(arch=self.arch), count + ) + def add_rope_dimension_sections(self, dims): + self.add_array(Keys.Rope.DIMENSION_SECTIONS.format(arch=self.arch), + dims) + def add_rope_freq_base(self, value): + self.add_float32(Keys.Rope.FREQ_BASE.format(arch=self.arch), value) + def add_rope_scaling_type(self, value): + self.add_string(Keys.Rope.SCALING_TYPE.format(arch=self.arch), + value.value) + def add_rope_scaling_factor(self, value): + self.add_float32(Keys.Rope.SCALING_FACTOR.format(arch=self.arch), value + ) + def add_rope_scaling_attn_factors(self, value): + self.add_float32(Keys.Rope.SCALING_ATTN_FACTOR.format(arch=self. + arch), value) + def add_rope_scaling_orig_ctx_len(self, value): + self.add_uint32(Keys.Rope.SCALING_ORIG_CTX_LEN.format(arch=self. + arch), value) + def add_rope_scaling_finetuned(self, value): + self.add_bool(Keys.Rope.SCALING_FINETUNED.format(arch=self.arch), value + ) + def add_rope_scaling_yarn_log_mul(self, value): + self.add_float32(Keys.Rope.SCALING_YARN_LOG_MUL.format(arch=self. + arch), value) + def add_ssm_conv_kernel(self, value): + self.add_uint32(Keys.SSM.CONV_KERNEL.format(arch=self.arch), value) + def add_ssm_inner_size(self, value): + self.add_uint32(Keys.SSM.INNER_SIZE.format(arch=self.arch), value) + def add_ssm_state_size(self, value): + self.add_uint32(Keys.SSM.STATE_SIZE.format(arch=self.arch), value) + def add_ssm_time_step_rank(self, value): + self.add_uint32(Keys.SSM.TIME_STEP_RANK.format(arch=self.arch), value) + def add_ssm_dt_b_c_rms(self, value): + self.add_bool(Keys.SSM.DT_B_C_RMS.format(arch=self.arch), value) + def add_tokenizer_model(self, model): + self.add_string(Keys.Tokenizer.MODEL, model) + def add_tokenizer_pre(self, pre): + self.add_string(Keys.Tokenizer.PRE, pre) + def add_token_list(self, tokens): + self.add_array(Keys.Tokenizer.LIST, tokens) + def add_token_merges(self, merges): + self.add_array(Keys.Tokenizer.MERGES, merges) + def add_token_types(self, types): + self.add_array(Keys.Tokenizer.TOKEN_TYPE, types) + def add_token_type_count(self, value): + self.add_uint32(Keys.Tokenizer.TOKEN_TYPE_COUNT, value) + def add_token_scores(self, scores): + self.add_array(Keys.Tokenizer.SCORES, scores) + def add_bos_token_id(self, id): + self.add_uint32(Keys.Tokenizer.BOS_ID, id) + def add_eos_token_id(self, id): + self.add_uint32(Keys.Tokenizer.EOS_ID, id) + def add_unk_token_id(self, id): + self.add_uint32(Keys.Tokenizer.UNK_ID, id) + def add_sep_token_id(self, id): + self.add_uint32(Keys.Tokenizer.SEP_ID, id) + def add_pad_token_id(self, id): + self.add_uint32(Keys.Tokenizer.PAD_ID, id) + def add_mask_token_id(self, id): + self.add_uint32(Keys.Tokenizer.MASK_ID, id) + def add_add_bos_token(self, value): + self.add_bool(Keys.Tokenizer.ADD_BOS, value) + def add_add_eos_token(self, value): + self.add_bool(Keys.Tokenizer.ADD_EOS, value) + def add_add_space_prefix(self, value): + self.add_bool(Keys.Tokenizer.ADD_PREFIX, value) + def add_remove_extra_whitespaces(self, value): + self.add_bool(Keys.Tokenizer.REMOVE_EXTRA_WS, value) + def add_precompiled_charsmap(self, charsmap): + self.add_array(Keys.Tokenizer.PRECOMPILED_CHARSMAP, charsmap) + def add_chat_template(self, value): + if not isinstance(value, str): + template_default = None + template_names = set() + for choice in value: + name = choice.get('name', '') + template = choice.get('template') + name = ''.join(c if c in ascii_letters + digits else '_' for + c in name) + if name and template is not None: + if name == 'default': + template_default = template + else: + template_names.add(name) + self.add_string(Keys.Tokenizer.CHAT_TEMPLATE_N. + format(name=name), template) + if template_names: + self.add_array(Keys.Tokenizer.CHAT_TEMPLATES, list( + template_names)) + if template_default is None: + return + value = template_default + self.add_string(Keys.Tokenizer.CHAT_TEMPLATE, value) + def add_eot_token_id(self, id): + self.add_uint32(Keys.Tokenizer.EOT_ID, id) + def add_eom_token_id(self, id): + self.add_uint32(Keys.Tokenizer.EOM_ID, id) + def _pack(self, fmt, value, skip_pack_prefix=False): + pack_prefix = '' + if not skip_pack_prefix: + pack_prefix = '<' if self.endianess == GGUFEndian.LITTLE else '>' + return struct.pack(f'{pack_prefix}{fmt}', value) + def _pack_val(self, val, vtype, add_vtype): + kv_data = bytearray() + if add_vtype: + kv_data += self._pack('I', vtype) + pack_fmt = self._simple_value_packing.get(vtype) + if pack_fmt is not None: + kv_data += self._pack(pack_fmt, val, skip_pack_prefix=vtype == + GGUFValueType.BOOL) + elif vtype == GGUFValueType.STRING: + encoded_val = val.encode('utf-8') if isinstance(val, str) else val + kv_data += self._pack('Q', len(encoded_val)) + kv_data += encoded_val + elif vtype == GGUFValueType.ARRAY: + if not isinstance(val, Sequence): + raise ValueError( + 'Invalid GGUF metadata array, expecting sequence') + if len(val) == 0: + raise ValueError('Invalid GGUF metadata array. Empty array') + if isinstance(val, bytes): + ltype = GGUFValueType.UINT8 + else: + ltype = GGUFValueType.get_type(val[0]) + if not all(GGUFValueType.get_type(i) is ltype for i in val[1:] + ): + raise ValueError( + 'All items in a GGUF array should be of the same type') + kv_data += self._pack('I', ltype) + kv_data += self._pack('Q', len(val)) + for item in val: + kv_data += self._pack_val(item, ltype, add_vtype=False) + else: + raise ValueError('Invalid GGUF metadata value type or value') + return kv_data + @staticmethod + def format_n_bytes_to_str(num): + if num == 0: + return 'negligible - metadata only' + fnum = float(num) + for unit in ('', 'K', 'M', 'G'): + if abs(fnum) < 1000.0: + return f'{fnum:3.1f}{unit}' + fnum /= 1000.0 + return f'{fnum:.1f}T - over 1TB, split recommended' \ No newline at end of file diff --git a/gguf_connector/y.py b/gguf_connector/y.py new file mode 100644 index 0000000..e403922 --- /dev/null +++ b/gguf_connector/y.py @@ -0,0 +1 @@ +from gguf_comfy import downloader \ No newline at end of file diff --git a/gguf_nodes.py b/gguf_nodes.py new file mode 100644 index 0000000..e5a402c --- /dev/null +++ b/gguf_nodes.py @@ -0,0 +1,584 @@ +import os +import gc +import time +import comfy +import io +import torch +import json +import nodes +import folder_paths + +from .gguf_connector import reader as gr +from .gguf_connector.writer import GGUFWriter, GGMLQuantizationType +from .gguf_connector.const import GGML_QUANT_VERSION, LlamaFileType +from .gguf_connector.quant import quantize, dequantize, QuantError +from .gguf_connector.quant2d import dequantize_tensor, is_quantized, is_torch_compatible +from .gguf_connector.tkn import get_field, tokenizer_builder + +if hasattr(torch, 'compiler') and hasattr(torch.compiler, 'disable'): + torch_compiler_disable = torch.compiler.disable +else: + def torch_compiler_disable(*args, **kwargs): + def noop(x): + return x + return noop + +def get_clip_type(type): + clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION) + return clip_type + +def get_device(device): + model_options = {} + if device == "cpu": + model_options["load_device"] = model_options["offload_device"] = torch.device("cpu") + return model_options + +def tensor_swap(raw_sd, key_map): + sd = {} + for k, v in raw_sd.items(): + for s, d in key_map.items(): + k = k.replace(s, d) + sd[k] = v + return sd + +def llama_permute(raw_sd, n_head, n_head_kv): + sd = {} + permute = lambda x, h: x.reshape(h, x.shape[0] // h // 2, 2, *x.shape[1:] + ).swapaxes(1, 2).reshape(x.shape) + for k, v in raw_sd.items(): + if k.endswith(('q_proj.weight', 'q_proj.bias')): + v.data = permute(v.data, n_head) + if k.endswith(('k_proj.weight', 'k_proj.bias')): + v.data = permute(v.data, n_head_kv) + sd[k] = v + return sd + +def load_patch_to_device(item, device): + if isinstance(item, torch.Tensor): + return item.to(device, non_blocking=True) + elif isinstance(item, tuple): + return tuple(load_patch_to_device(x, device) for x in item) + elif isinstance(item, list): + return [load_patch_to_device(x, device) for x in item] + else: + return item + +class GGMLTensor(torch.Tensor): + def __init__(self, *args, tensor_type, tensor_shape, patches=[], **kwargs): + super().__init__() + self.tensor_type = tensor_type + self.tensor_shape = tensor_shape + self.patches = patches + def __new__(cls, *args, tensor_type, tensor_shape, patches=[], **kwargs): + return super().__new__(cls, *args, **kwargs) + def to(self, *args, **kwargs): + new = super().to(*args, **kwargs) + new.tensor_type = getattr(self, 'tensor_type', None) + new.tensor_shape = getattr(self, 'tensor_shape', new.data.shape) + new.patches = getattr(self, 'patches', []).copy() + return new + def clone(self, *args, **kwargs): + return self + def detach(self, *args, **kwargs): + return self + def copy_(self, *args, **kwargs): + try: + return super().copy_(*args, **kwargs) + except Exception as e: + print(f"ignoring 'copy_' on tensor: {e}") + def empty_(self, size, *args, **kwargs): + new_tensor = super().empty_(size, *args, **kwargs) + return GGMLTensor(new_tensor, tensor_type=getattr(self, + 'tensor_type', None), tensor_shape=size, patches=getattr(self, + 'patches', []).copy()) + @property + def shape(self): + if not hasattr(self, 'tensor_shape'): + self.tensor_shape = self.size() + return self.tensor_shape + +class GGMLLayer(torch.nn.Module): + comfy_cast_weights = True + dequant_dtype = None + patch_dtype = None + largest_layer = False + torch_compatible_tensor_types = {None, gr.GGMLQuantizationType.F32, gr. + GGMLQuantizationType.F16} + def is_ggml_quantized(self, *, weight=None, bias=None): + if weight is None: + weight = self.weight + if bias is None: + bias = self.bias + return is_quantized(weight) or is_quantized(bias) + def _load_from_state_dict(self, state_dict, prefix, *args, **kwargs): + weight, bias = state_dict.get(f'{prefix}weight'), state_dict.get( + f'{prefix}bias') + if self.is_ggml_quantized(weight=weight, bias=bias) or isinstance(self, + torch.nn.Linear): + return self.ggml_load_from_state_dict(state_dict, prefix, *args, + **kwargs) + return super()._load_from_state_dict(state_dict, prefix, *args, ** + kwargs) + def ggml_load_from_state_dict(self, state_dict, prefix, local_metadata, + strict, missing_keys, unexpected_keys, error_msgs): + prefix_len = len(prefix) + for k, v in state_dict.items(): + if k[prefix_len:] == 'weight': + self.weight = torch.nn.Parameter(v, requires_grad=False) + elif k[prefix_len:] == 'bias' and v is not None: + self.bias = torch.nn.Parameter(v, requires_grad=False) + else: + missing_keys.append(k) + if self.weight is None and isinstance(self, torch.nn.Linear): + v = torch.zeros(self.in_features, self.out_features) + self.weight = torch.nn.Parameter(v, requires_grad=False) + missing_keys.append(prefix + 'weight') + if getattr(self.weight, 'is_largest_weight', False): + self.largest_layer = True + def _save_to_state_dict(self, *args, **kwargs): + if self.is_ggml_quantized(): + return self.ggml_save_to_state_dict(*args, **kwargs) + return super()._save_to_state_dict(*args, **kwargs) + def ggml_save_to_state_dict(self, destination, prefix, keep_vars): + weight = torch.zeros_like(self.weight, device=torch.device('meta')) + destination[prefix + 'weight'] = weight + if self.bias is not None: + bias = torch.zeros_like(self.bias, device=torch.device('meta')) + destination[prefix + 'bias'] = bias + if self.largest_layer: + shape = getattr(self.weight, 'tensor_shape', self.weight.shape) + dtype = self.dequant_dtype or torch.float16 + temp = torch.empty(*shape, device=torch.device('meta'), dtype=dtype + ) + destination[prefix + 'temp.weight'] = temp + return + destination[prefix + 'weight'] = self.get_weight(self.weight) + if bias is not None: + destination[prefix + 'bias'] = self.get_weight(self.bias) + def get_weight(self, tensor, dtype): + if tensor is None: + return + patch_list = [] + device = tensor.device + for function, patches, key in getattr(tensor, 'patches', []): + patch_list += load_patch_to_device(patches, device) + weight = dequantize_tensor(tensor, dtype, self.dequant_dtype) + if isinstance(weight, GGMLTensor): + weight = torch.Tensor(weight) + if patch_list: + if self.patch_dtype is None: + weight = function(patch_list, weight, key) + else: + patch_dtype = (dtype if self.patch_dtype == 'target' else + self.patch_dtype) + weight = function(patch_list, weight, key, patch_dtype) + return weight + @torch_compiler_disable() + def cast_bias_weight(s, input=None, dtype=None, device=None, bias_dtype + =None): + if input is not None: + if dtype is None: + dtype = getattr(input, 'dtype', torch.float32) + if bias_dtype is None: + bias_dtype = dtype + if device is None: + device = input.device + bias = None + non_blocking = comfy.model_management.device_supports_non_blocking( + device) + if s.bias is not None: + bias = s.get_weight(s.bias.to(device), dtype) + bias = comfy.ops.cast_to(bias, bias_dtype, device, non_blocking + =non_blocking, copy=False) + weight = s.get_weight(s.weight.to(device), dtype) + weight = comfy.ops.cast_to(weight, dtype, device, non_blocking= + non_blocking, copy=False) + return weight, bias + def forward_comfy_cast_weights(self, input, *args, **kwargs): + if self.is_ggml_quantized(): + out = self.forward_ggml_cast_weights(input, *args, **kwargs) + else: + out = super().forward_comfy_cast_weights(input, *args, **kwargs) + if isinstance(out, GGMLTensor): + out = torch.Tensor(out) + return out + def forward_ggml_cast_weights(self, input): + raise NotImplementedError + +class GGMLOps(comfy.ops.manual_cast): + class Linear(GGMLLayer, comfy.ops.manual_cast.Linear): + def __init__(self, in_features, out_features, bias=True, device= + None, dtype=None): + torch.nn.Module.__init__(self) + self.in_features = in_features + self.out_features = out_features + self.weight = None + self.bias = None + def forward_ggml_cast_weights(self, input): + weight, bias = self.cast_bias_weight(input) + return torch.nn.functional.linear(input, weight, bias) + class Conv2d(GGMLLayer, comfy.ops.manual_cast.Conv2d): + def forward_ggml_cast_weights(self, input): + weight, bias = self.cast_bias_weight(input) + return self._conv_forward(input, weight, bias) + class Embedding(GGMLLayer, comfy.ops.manual_cast.Embedding): + def forward_ggml_cast_weights(self, input, out_dtype=None): + output_dtype = out_dtype + if (self.weight.dtype == torch.float16 or self.weight.dtype == + torch.bfloat16): + out_dtype = None + weight, _bias = self.cast_bias_weight(self, device=input.device, + dtype=out_dtype) + return torch.nn.functional.embedding(input, weight, self. + padding_idx, self.max_norm, self.norm_type, self. + scale_grad_by_freq, self.sparse).to(dtype=output_dtype) + class LayerNorm(GGMLLayer, comfy.ops.manual_cast.LayerNorm): + def forward_ggml_cast_weights(self, input): + if self.weight is None: + return super().forward_comfy_cast_weights(input) + weight, bias = self.cast_bias_weight(input) + return torch.nn.functional.layer_norm(input, self. + normalized_shape, weight, bias, self.eps) + class GroupNorm(GGMLLayer, comfy.ops.manual_cast.GroupNorm): + def forward_ggml_cast_weights(self, input): + weight, bias = self.cast_bias_weight(input) + return torch.nn.functional.group_norm(input, self.num_groups, + weight, bias, self.eps) + +pig = os.path.join(os.path.dirname(__file__), 'version.json') +with open(pig, 'r') as file: + data = json.load(file) +arrays = {} +for key, value in data[0].items(): + arrays[key] = value + +def get_orig_shape(reader, tensor_name): + field_key = f'comfy.gguf.orig_shape.{tensor_name}' + field = reader.get_field(field_key) + if field is None: + return None + if len(field.types) != 2 or field.types[0 + ] != gr.GGUFValueType.ARRAY or field.types[1 + ] != gr.GGUFValueType.INT32: + raise TypeError( + f'Bad original shape metadata for {field_key}: Expected ARRAY of INT32, got {field.types}' + ) + return torch.Size(tuple(int(field.parts[part_idx][0]) for part_idx in + field.data)) + +class GGUFModelPatcher(comfy.model_patcher.ModelPatcher): + patch_on_device = False + def patch_weight_to_device(self, key, device_to=None, inplace_update=False): + if key not in self.patches: + return + weight = comfy.utils.get_attr(self.model, key) + try: + from comfy.lora import calculate_weight + except Exception: + calculate_weight = self.calculate_weight + patches = self.patches[key] + if is_quantized(weight): + out_weight = weight.to(device_to) + patches = load_patch_to_device(patches, self.load_device if + self.patch_on_device else self.offload_device) + out_weight.patches = [(calculate_weight, patches, key)] + else: + inplace_update = self.weight_inplace_update or inplace_update + if key not in self.backup: + self.backup[key] = collections.namedtuple('Dimension', [ + 'weight', 'inplace_update'])(weight.to(device=self. + offload_device, copy=inplace_update), inplace_update) + if device_to is not None: + temp_weight = comfy.model_management.cast_to_device(weight, + device_to, torch.float32, copy=True) + else: + temp_weight = weight.to(torch.float32, copy=True) + out_weight = calculate_weight(patches, temp_weight, key) + out_weight = comfy.float.stochastic_rounding(out_weight, weight + .dtype) + if inplace_update: + comfy.utils.copy_to_param(self.model, key, out_weight) + else: + comfy.utils.set_attr_param(self.model, key, out_weight) + def unpatch_model(self, device_to=None, unpatch_weights=True): + if unpatch_weights: + for p in self.model.parameters(): + if is_torch_compatible(p): + continue + patches = getattr(p, 'patches', []) + if len(patches) > 0: + p.patches = [] + return super().unpatch_model(device_to=device_to, unpatch_weights= + unpatch_weights) + mmap_released = False + def load(self, *args, force_patch_weights=False, **kwargs): + super().load(*args, force_patch_weights=True, **kwargs) + if not self.mmap_released: + linked = [] + if kwargs.get('lowvram_model_memory', 0) > 0: + for n, m in self.model.named_modules(): + if hasattr(m, 'weight'): + device = getattr(m.weight, 'device', None) + if device == self.offload_device: + linked.append((n, m)) + continue + if hasattr(m, 'bias'): + device = getattr(m.bias, 'device', None) + if device == self.offload_device: + linked.append((n, m)) + continue + if linked: + print(f'Attempting to release mmap ({len(linked)})') + for n, m in linked: + m.to(self.load_device).to(self.offload_device) + self.mmap_released = True + def clone(self, *args, **kwargs): + src_cls = self.__class__ + self.__class__ = GGUFModelPatcher + n = super().clone(*args, **kwargs) + n.__class__ = GGUFModelPatcher + self.__class__ = src_cls + n.patch_on_device = getattr(self, 'patch_on_device', False) + return n + +def load_gguf_sd(path, handle_prefix='model.diffusion_model.', return_arch= + False): + reader = gr.GGUFReader(path) + has_prefix = False + if handle_prefix is not None: + prefix_len = len(handle_prefix) + tensor_names = set(tensor.name for tensor in reader.tensors) + has_prefix = any(s.startswith(handle_prefix) for s in tensor_names) + tensors = [] + for tensor in reader.tensors: + sd_key = tensor_name = tensor.name + if has_prefix: + if not tensor_name.startswith(handle_prefix): + continue + sd_key = tensor_name[prefix_len:] + tensors.append((sd_key, tensor)) + + # print(arrays) + + compat = None + arch_str = get_field(reader, 'general.architecture', str) + + # print("arch_str : {}".format(arch_str)) + + if arch_str is None: + compat = 'sd.cpp' + elif arch_str not in arrays['PIG_ARCH_LIST'] and arch_str not in arrays['TXT_ARCH_LIST']: + raise ValueError(f"Unknown architecture: {arch_str!r}") + state_dict, qtype_dict = {}, {} + for sd_key, tensor in tensors: + tensor_name = tensor.name + torch_tensor = torch.from_numpy(tensor.data) + shape = get_orig_shape(reader, tensor_name) + if shape is None: + shape = torch.Size(tuple(int(v) for v in reversed(tensor.shape))) + if compat == 'sd.cpp' and arch_str == 'sdxl': + if any([tensor_name.endswith(x) for x in ('.proj_in.weight', + '.proj_out.weight')]): + while len(shape) > 2 and shape[-1] == 1: + shape = shape[:-1] + if tensor.tensor_type in {gr.GGMLQuantizationType.F32, gr. + GGMLQuantizationType.F16}: + torch_tensor = torch_tensor.view(*shape) + state_dict[sd_key] = GGMLTensor(torch_tensor, tensor_type=tensor. + tensor_type, tensor_shape=shape) + tensor_type_str = getattr(tensor.tensor_type, 'name', repr(tensor.tensor_type)) + qtype_dict[tensor_type_str] = qtype_dict.get(tensor_type_str, 0) + 1 + print('gguf qtypes: ' + ', '.join(f'{k} ({v})' for k, v in qtype_dict.items())) + qsd = {k: v for k, v in state_dict.items() if is_quantized(v)} + if len(qsd) > 0: + max_key = max(qsd.keys(), key=lambda k: qsd[k].numel()) + state_dict[max_key].is_largest_weight = True + if return_arch: + return state_dict, arch_str + return state_dict + +class WarpedLoaderGGUF: + @classmethod + def INPUT_TYPES(s): + gguf_names = [x for x in folder_paths.get_filename_list('model_gguf')] + return {'required': {'gguf_name': (gguf_names,)}} + RETURN_TYPES = 'MODEL', + FUNCTION = 'load_model' + CATEGORY = 'Warped/GGUF/Loaders' + + def load_model(self, gguf_name, dequant_dtype=None, patch_dtype=None, + patch_on_device=None): + ops = GGMLOps() + if dequant_dtype in ('default', None): + ops.Linear.dequant_dtype = None + elif dequant_dtype in ['target']: + ops.Linear.dequant_dtype = dequant_dtype + else: + ops.Linear.dequant_dtype = getattr(torch, dequant_dtype) + if patch_dtype in ('default', None): + ops.Linear.patch_dtype = None + elif patch_dtype in ['target']: + ops.Linear.patch_dtype = patch_dtype + else: + ops.Linear.patch_dtype = getattr(torch, patch_dtype) + model_path = folder_paths.get_full_path('unet', gguf_name) + + print("Reading Checkpoint: {}...".format(model_path)) + with open(model_path, "rb") as file: + checkpoint_temp = file.read() + print("Reading Checkpoint: {}...Done!".format(model_path)) + + print("Loading Checkpoint...") + + temp_io = io.BytesIO(checkpoint_temp) + + sd = load_gguf_sd(temp_io) + print("Loading Checkpoint...Done!") + + model = comfy.sd.load_diffusion_model_state_dict(sd, model_options= + {'custom_operations': ops}) + if model is None: + logging.error('ERROR UNSUPPORTED MODEL {}'.format(model_path)) + raise RuntimeError('ERROR: Could not detect model type of: {}'. + format(model_path)) + model = GGUFModelPatcher.clone(model) + model.patch_on_device = patch_on_device + + temp_io = None + checkpoint_temp = None + gc.collect() + time.sleep(1) + + return model, + +def warped_load_torch_file(ckpt, return_metadata=False): + from safetensors.torch import load as safeload + + metadata = None + + try: + f = safeload(ckpt) + + sd = {} + for k in f.keys(): + if k != "__metadata__": + sd[k] = f[k] + elif return_metadata: + metadata = f[k] + except Exception as e: + if len(e.args) > 0: + message = e.args[0] + if "HeaderTooLarge" in message: + raise ValueError("{}\n\nFile path: {}\n\nThe safetensors file is corrupt or invalid. Make sure this is actually a safetensors file and not a ckpt or pt or other filetype.".format(message, ckpt)) + if "MetadataIncompleteBuffer" in message: + raise ValueError("{}\n\nFile path: {}\n\nThe safetensors file is corrupt/incomplete. Check the file size and make sure you have copied/downloaded it correctly.".format(message, ckpt)) + raise e + + return (sd, metadata) if return_metadata else sd + +def load_gguf_clip(path): + print("Reading Clip: {}...".format(path)) + with open(path, "rb") as file: + checkpoint_temp = file.read() + print("Reading Clip: {}...Done!".format(path)) + + print("Loading Clip...") + # temp = warped_load_torch_file(checkpoint_temp) + temp_io = io.BytesIO(checkpoint_temp) + # print("temp_io Type: {}".format(type(temp_io))) + + sd, arch = load_gguf_sd(temp_io, return_arch=True) + print("Loading Clip...Done!") + + if arch in {'t5', 't5encoder'}: + temb_key = 'token_embd.weight' + if temb_key in sd and (sd[temb_key].shape == (256384, 4096) or sd[temb_key].shape == (256384, 768)): + sd['spiece_model'] = tokenizer_builder(path) + sd = tensor_swap(sd, arrays['T5']) + elif temb_key in sd and sd[temb_key].shape == (32128, 768): + sd = tensor_swap(sd, arrays['B5']) + else: + sd = tensor_swap(sd, arrays['T5']) + elif arch in {'llama'}: + sd = tensor_swap(sd, arrays['L3']) + sd = llama_permute(sd, 32, 8) + elif arch in {'gemma2'}: + sd["spiece_model"] = tokenizer_builder(path) + sd = tensor_swap(sd, arrays['G2']) + elif arch in {'pig'}: + sd = pig_work(sd) + else: + pass + + temp_io = None + + return sd + +class WarpedClipLoaderGGUF: + @classmethod + def INPUT_TYPES(s): + base = nodes.CLIPLoader.INPUT_TYPES() + return {'required': {'clip_name': (s.get_filename_list(),), 'type': + base['required']['type']}, + 'optional':{'device':(['default','cpu'],{'advanced':True}),}} + RETURN_TYPES = 'CLIP', + FUNCTION = 'load_clip' + CATEGORY = 'Warped/GGUF/Loaders' + + @classmethod + def get_filename_list(s): + files = [] + files += folder_paths.get_filename_list('clip') + files += folder_paths.get_filename_list('clip_gguf') + return sorted(files) + + def load_data(self, ckpt_paths): + clip_data = [] + for p in ckpt_paths: + if p.endswith('.gguf'): + sd = load_gguf_clip(p) + else: + print("Reading Clip: {}...".format(p)) + with open(p, "rb") as file: + checkpoint_temp = file.read() + print("Reading Clip: {}...Done!".format(p)) + sd = warped_load_torch_file(checkpoint_temp) + + clip_data.append(sd) + return clip_data + + def load_patcher(self, clip_paths, clip_type, clip_data): + clip = comfy.sd.load_text_encoder_state_dicts(clip_type=clip_type, + state_dicts=clip_data, model_options={'custom_operations': + GGMLOps, 'initial_device': comfy.model_management. + text_encoder_offload_device()}, embedding_directory= + folder_paths.get_folder_paths('embeddings')) + clip.patcher = GGUFModelPatcher.clone(clip.patcher) + return clip + + def load_clip(self, clip_name, type='stable_diffusion', device='default'): + clip_path = folder_paths.get_full_path('clip', clip_name) + if clip_name.endswith('.safetensors') and device != 'default': + clip = comfy.sd.load_clip(ckpt_paths=[clip_path], embedding_directory=folder_paths.get_folder_paths("embeddings"), clip_type=get_clip_type(type), model_options=get_device(device)) + return (clip,) + else: + return (self.load_patcher([clip_path], get_clip_type(type), self.load_data([clip_path])), get_device(device)) + +class WarpedDualClipLoaderGGUF(WarpedClipLoaderGGUF): + @classmethod + def INPUT_TYPES(s): + base = nodes.DualCLIPLoader.INPUT_TYPES() + file_options = s.get_filename_list(), + return {'required': {'clip_name1':file_options, 'clip_name2':file_options, 'type': + base['required']['type']}, + 'optional':{'device':(['default','cpu'],{'advanced':True}),}} + + def load_clip(self, clip_name1, clip_name2, type, device='default'): + clip_path1 = folder_paths.get_full_path('clip', clip_name1) + clip_path2 = folder_paths.get_full_path('clip', clip_name2) + clip_paths = [clip_path1, clip_path2] + if device != 'default' and (clip_name1.endswith('.safetensors') and clip_name2.endswith('.safetensors')): + clip = comfy.sd.load_clip(ckpt_paths=clip_paths, embedding_directory=folder_paths.get_folder_paths("embeddings"), clip_type=get_clip_type(type), model_options=get_device(device)) + return (clip,) + else: + return (self.load_patcher(clip_paths, get_clip_type(type), self.load_data(clip_paths)), get_device(device)) diff --git a/nodes.py b/nodes.py index ed5c26f..2e430db 100644 --- a/nodes.py +++ b/nodes.py @@ -66,6 +66,7 @@ import copy import safetensors import einops +import io import logging logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') @@ -73,6 +74,8 @@ logger = logging.getLogger(__name__) ALLOWED_VIDEO_EXT = ('mp4', 'flv', 'mov', 'avi', 'mpg', 'webm', 'mkv') +vae_scaling_factor = 0.476986 + class VideoGenerationType(Enum): T2V = "t2v" I2V = "i2v" @@ -2925,6 +2928,2821 @@ class WarpedSamplerCustomAdvLatent: return noise_latents_full +class WarpedWanImageToVideo: + @classmethod + def INPUT_TYPES(s): + return {"required": {"positive": ("CONDITIONING", ), + "negative": ("CONDITIONING", ), + "vae": ("VAE", ), + "width": ("INT", {"default": 832, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}), + "height": ("INT", {"default": 480, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}), + "length": ("INT", {"default": 81, "min": 1, "max": nodes.MAX_RESOLUTION, "step": 4}), + "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}), + }, + "optional": {"clip_vision_output": ("CLIP_VISION_OUTPUT", ), + "start_image": ("IMAGE", ), + }} + + RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT", "INT", ) + RETURN_NAMES = ("positive", "negative", "latent", "num_frames", ) + FUNCTION = "encode" + + CATEGORY = "Warped/Wan/Conditioning" + + def encode(self, positive, negative, vae, width, height, length, batch_size, start_image=None, clip_vision_output=None): + latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device()) + if start_image is not None: + start_image = comfy.utils.common_upscale(start_image[:length].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1) + image = torch.mul(torch.ones((length, height, width, start_image.shape[-1]), device=start_image.device, dtype=start_image.dtype), 0.5) + image[:start_image.shape[0]] = start_image + + concat_latent_image = vae.encode(image[:, :, :, :3]) + + mask = torch.ones((1, 1, latent.shape[2], concat_latent_image.shape[-2], concat_latent_image.shape[-1]), device=start_image.device, dtype=start_image.dtype) + mask[:, :, :((start_image.shape[0] - 1) // 4) + 1] = 0.0 + + positive = node_helpers.conditioning_set_values(positive, {"concat_latent_image": concat_latent_image, "concat_mask": mask}) + negative = node_helpers.conditioning_set_values(negative, {"concat_latent_image": concat_latent_image, "concat_mask": mask}) + + if clip_vision_output is not None: + positive = node_helpers.conditioning_set_values(positive, {"clip_vision_output": clip_vision_output}) + negative = node_helpers.conditioning_set_values(negative, {"clip_vision_output": clip_vision_output}) + + out_latent = {} + out_latent["samples"] = latent + return (positive, negative, out_latent, length, ) + +PROMPT_TEMPLATE_ENCODE_VIDEO_I2V = ( + "<|start_header_id|>system<|end_header_id|>\n\n\nDescribe the video by detailing the following aspects according to the reference image: " + "1. The main content and theme of the video." + "2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects." + "3. Actions, events, behaviors temporal relationships, physical movement changes of the objects." + "4. background environment, light, style and atmosphere." + "5. camera angles, movements, and transitions used in the video:<|eot_id|>\n\n" + "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|>" + "<|start_header_id|>assistant<|end_header_id|>\n\n" +) + +# class WarpedSamplerCustomBatch: +# @classmethod +# def INPUT_TYPES(s): +# return {"required": +# { +# "positive": ("STRING", ), +# "negative": ("STRING", {"default": None}), +# "clip": ("CLIP", ), +# "preferred_frame_count": ("INT", {"default": 81, "min": 17, "max": nodes.MAX_RESOLUTION, "step": 4}), +# "preferred_batch_size": ("INT", {"default": 17, "min": 17, "max": 161, "step": 4}), +# "use_batch_size": (["next_lowest", "next_highest", "closest", "exact"], {"default": "next_lowest", "tooltip": "Number of frames generated may be impacted by choice."}), +# "vae": ("VAE", ), +# "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), +# "model": ("MODEL", ), +# "sampler": ("SAMPLER", ), +# "sigmas": ("SIGMAS", ), +# "dec_tile_size": ("INT", {"default": 256, "min": 64, "max": 4096, "step": 32}), +# "dec_overlap": ("INT", {"default": 64, "min": 0, "max": 4096, "step": 32}), +# "dec_temporal_size": ("INT", {"default": 64, "min": 8, "max": 4096, "step": 4, "tooltip": "Only used for video VAEs: Amount of frames to decode at a time."}), +# "dec_temporal_overlap": ("INT", {"default": 8, "min": 4, "max": 4096, "step": 4, "tooltip": "Only used for video VAEs: Amount of frames to overlap."}), +# "skip_frames": ("INT", {"default": 0, "min": 0, "max": 32, "step": 4}), +# "noise_scale": ("FLOAT", {"default": 1.00, "min": 0.00, "max": 1.00, "step": 0.01}), +# "mode": (["Hunyuan", "Wan"], {"default": "Hunyuan"}), +# "hunyuan_guidance_type": (["v1 (concat)", "v2 (replace)", "custom"], {"default": "v2 (replace)", "tooltip": "Only used if mode is \"Hunyuan\"."}), +# "hunyuan_image_interleave": ("INT", {"default": 1, "min": 1, "max": 512, "tooltip": "How much the image influences things vs the text prompt. Higher number means more influence from the text prompt."}), +# }, +# "optional": +# {"noise_strength": ("FLOAT", {"default": 1.00, "min": 0.10, "max": 1.00, "step": 0.01}), +# "blend_frames": ("INT", {"default":5, "min":0, "max": 16, "step": 1}), +# "t2v_width": ("INT", {"default":480, "min":256, "max": 1280, "step": 16}), +# "t2v_height": ("INT", {"default":720, "min":256, "max": 1280, "step": 16}), +# "scaling_strength": ("FLOAT", {"default": 1.0}), +# "clip_vision_model": ("CLIP_VISION", ), +# "start_image": ("IMAGE", ), +# "flux_guidance": ("FLOAT", {"default": 3.5, "min": 0.0, "max": 100.0, "step": 0.1}), +# "use_flux_guidance": ("BOOLEAN", {"default": False}), +# "dummy_frames": ("INT", {"default":17, "min":17, "max": 121, "step": 4, "tooltip": "Number of frames to generate in dummy batch."}), +# "gen_dummy": ("BOOLEAN", {"default": False, "tooltip": "For t2v or i2v only. Will generate a dummy batch to obtain a starting image for main generation."}), +# "gen_dummy_only": ("BOOLEAN", {"default": False, "tooltip": "Will generate dummy batch only."}), +# "use_dummy_image": (["first", "middle", "last", "random", "all"], {"default": "last", "tooltip": "Which dummy batch image to start main generation."}), +# "output_latents": ("BOOLEAN", {"default": False}), +# "verbose_messaging": ("BOOLEAN", {"default": False}), +# "i2v_sigmas": ("SIGMAS", {"default": None}), +# "i2v_model": ("MODEL", {"default": None}), +# } +# } +# +# RETURN_TYPES = ("IMAGE", "LATENT", "INT", "STRING", "IMAGE", ) +# RETURN_NAMES = ("images", "latents", "seed", "generation_status", "image_used",) +# +# FUNCTION = "sample" +# +# CATEGORY = "Warped/General/Sampling" +# +# def sample(self, positive, negative, clip, preferred_frame_count, preferred_batch_size, use_batch_size, vae, seed, model, sampler, sigmas, dec_tile_size, dec_overlap, dec_temporal_size, dec_temporal_overlap, +# skip_frames, noise_scale, mode="Hunyuan", hunyuan_guidance_type="v2 (replace)", hunyuan_image_interleave=2, noise_strength=1.0, blend_frames=5, t2v_width=480, t2v_height=720, scaling_strength=1.0, +# clip_vision_model=None, start_image=None, flux_guidance=3.5, dummy_frames=17, gen_dummy=False, gen_dummy_only=False, use_dummy_image="last", use_flux_guidance=False, output_latents=False, verbose_messaging=False, +# i2v_sigmas=None, i2v_model=None,): +# self.device = mm.get_torch_device() +# self.offload_device = get_offload_device() +# mm.unload_all_models() +# gc.collect() +# time.sleep(1) +# +# self.positive = positive +# self.negative = negative +# self.clip = clip +# self.use_clip = clip.clone() +# self.preferred_frame_count = preferred_frame_count +# self.preferred_batch_size = preferred_batch_size +# self.vae = vae +# self.seed = seed +# self.sampler = sampler +# self.sigmas = sigmas +# self.i2v_sigmas = i2v_sigmas +# self.noise_scale = noise_scale +# self.mode = mode +# self.dec_tile_size = dec_tile_size +# self.dec_overlap = dec_overlap +# self.dec_temporal_size = dec_temporal_size +# self.dec_temporal_overlap = dec_temporal_overlap +# self.clip_vision = clip_vision_model +# self.noise_strength = noise_strength +# self.blend_frames = blend_frames +# self.t2v_width = t2v_width +# self.t2v_height = t2v_height +# self.dummy_frames = dummy_frames +# self.dummy_latents = int((dummy_frames - 1) // 4) + 1 +# self.gen_dummy = gen_dummy +# self.gen_dummy_only = gen_dummy_only +# self.use_dummy_image = use_dummy_image +# self.skip_frames = skip_frames +# self.verbose_messaging = verbose_messaging +# self.model = model +# self.i2v_model = i2v_model +# self.g_output = {} +# +# disable_pbar = not utils.PROGRESS_BAR_ENABLED +# +# self.use_sigmas = self.sigmas +# +# self.guider = WarpedGuider_Basic() +# self.guider.set_model(self.model) +# callback = self.setup_callbacks() +# +# self.latent_window_size, self.batch_count, self.truncated_frame_count = self.get_latent_window_size(preferred_batch_size, self.preferred_frame_count, use_batch_size=use_batch_size) +# +# is_i2v = True +# +# if start_image is None: +# start_image = torch.zeros([1, self.t2v_height, self.t2v_width, 3], dtype=torch.float32, device=self.offload_device) +# is_i2v = False +# +# print("start_image Shape: {}".format(start_image.shape)) +# +# self.width = int(start_image.shape[2] // 8) +# self.height = int(start_image.shape[1] // 8) +# +# noise, dummy_noise = self.setup_latent_noise() +# +# output_images = [] +# output_images_latents = None +# interrupted = False +# generation_status = "" +# total_images_generated = 0 +# is_dummy_section = False +# is_first_section = False +# is_last_section = False +# switch_guider = True +# last_image = None +# +# if not self.gen_dummy: +# print("Generating {} Batches".format(self.batch_count)) +# else: +# print("Generating Dummy Batch Plus {} Batches".format(self.batch_count)) +# +# generation_batches = list(range(self.batch_count)) +# +# if self.gen_dummy and (not self.gen_dummy_only): +# generation_batches = [0] + generation_batches +# is_dummy_section = True +# else: +# generation_batches = [0] +# is_dummy_section = True +# +# print("generation_batches: {}".format(generation_batches)) +# +# try: +# for i, generation_batch in enumerate(generation_batches): +# batch_number = generation_batches[i] +# # batch_key = "{}".format(batch_number) +# +# if is_dummy_section: +# print("Generating Dummy Batch.") +# is_first_section = False +# +# if not self.gen_dummy_only: +# is_last_section = False +# else: +# is_last_section = True +# else: +# print("Generating Batch {}.".format(batch_number)) +# +# is_first_section = batch_number == min(generation_batches) +# is_last_section = batch_number == max(generation_batches) +# +# print("is_first_section: {} | is_last_section: {} | is_dummy_section: {}".format(is_first_section, is_last_section, is_dummy_section)) +# +# mm.soft_empty_cache() +# gc.collect() +# time.sleep(0.1) +# +# image_embeds = None +# +# print("start_image Shape: {}".format(start_image.shape)) +# +# if is_i2v: +# if len(start_image.shape) < 4: +# start_image = start_image.unsqueeze(0) +# +# print("Performing clip_vision_encode.") +# image_embeds = self.clip_vision_encode(start_image.clone().detach()) +# +# if not is_dummy_section: +# noise_latents = noise[batch_number] +# else: +# noise_latents = dummy_noise +# +# print("noise_latents Shape: {}".format(noise_latents.shape)) +# +# noise_mask = None +# +# if mode == "Hunyuan": +# if is_i2v: +# if (not self.i2v_model is None) and switch_guider: +# switch_guider = False +# callback = None +# self.guider = WarpedGuider_Basic() +# self.guider.set_model(self.i2v_model) +# callback = self.setup_callbacks() +# +# self.use_clip = self.clip.clone() +# temp_cond = self.hunyuan_text_encode(self.use_clip, image_embeds, positive, hunyuan_image_interleave) +# positive_cond, latents = self.hunyuan_encode(temp_cond, self.vae, start_image.shape[2], start_image.shape[1], self.truncated_frame_count, 1, hunyuan_guidance_type, start_image=start_image) +# +# if "noise_mask" in latents: +# noise_mask = latents["noise_mask"] +# +# latents = latents["samples"] +# +# # latents = latents["samples"] * (scaling_strength * vae_scaling_factor) +# else: +# positive_cond = self.do_text_encode(self.use_clip, positive) +# latents = torch.zeros([1, 16, noise_latents.shape[2], self.height, self.width], dtype=torch.float32, device=self.offload_device) +# +# latents = latents * scaling_strength # scaling_strength +# latents = latents * (scaling_strength * vae_scaling_factor) +# +# if use_flux_guidance: +# positive_cond = self.apply_flux_guidance(positive_cond, flux_guidance) +# +# self.guider.set_conds(positive_cond) +# # else: +# # positive_cond, negative_cond = wan_text_encode(self.use_clip, positive_text=positive, negative_text=negative) +# # positive_cond, negative_cond, latents = self.wan_encode(positive_cond, negative_cond, self.vae, start_image.shape[3], start_image.shape[2], ((noise_latents.shape[2] - 1) * 4) + 1, 1, start_image=start_image, clip_vision_output=image_embeds) +# +# # self.guider.set_conds(positive_cond, negative_cond) +# +# print("latents Shape: {}".format(latents.shape)) +# +# num_frames = int(((latents.shape[2] - 1) * 4) + 1) +# +# print("\nDecoded Width is {} | Decoded Height is {}".format(int(self.width * 8), int(self.height * 8))) +# +# print("-------------------------------------------------------------------------------------------") +# print("WarpedSamplerCustomBatch: Latents Shape: {} | Noise Latents Shape: {}".format(latents.shape, noise_latents.shape)) +# print("-------------------------------------------------------------------------------------------") +# +# print("Noise Shape: {} | Latents Shape: {}".format(noise_latents.shape, latents.shape)) +# print("WarpedSamplerCustomBatch: Generating {} Frames in {} Latents....".format(num_frames, latents.shape[2])) +# +# samples = self.guider.sample(noise_latents, latents, self.sampler, self.use_sigmas, denoise_mask=noise_mask, callback=callback, disable_pbar=disable_pbar, seed=self.seed) +# samples = samples.to(mm.intermediate_device()) +# +# mm.unload_all_models() +# mm.soft_empty_cache() +# gc.collect() +# time.sleep(1) +# +# if len(samples.shape) < 5: +# samples = samples.unsqueeze(0) +# +# samples = samples.clone().detach() / scaling_strength # (scaling_strength * vae_scaling_factor) +# +# print("WarpedSamplerCustomBatch: Decoding Latents...") +# decoded_images = self.decode_tiled(samples) +# +# if not is_dummy_section or (is_dummy_section and ((self.use_dummy_image == "all")) or self.gen_dummy_only): +# output_images.append(decoded_images) +# total_images_generated += decoded_images.shape[0] +# +# samples = None +# +# # output_images = self.process_skip_images(output_images, skip_frames) +# print("WarpedSamplerCustomBatch: Decoded Images Shape: {}".format(decoded_images.shape[0])) +# +# # if output_latents: +# # output_images_latents = self.encode_tiled(output_images) +# +# print("WarpedSamplerCustomBatch: Generating {} frames in {} latents...Done.".format(num_frames, latents.shape[2])) +# +# if not is_i2v: +# self.guider.model_patcher.model.to(get_offload_device()) +# # guider.set_i2v_model() +# +# if not self.i2v_sigmas is None: +# self.use_sigmas = self.i2v_sigmas +# +# if not is_last_section: +# print("HERE 9090000") +# decoded_tuple = torch.split(decoded_images, 1, dim=0) +# decoded_split = [item for item in decoded_tuple] +# decoded_images = decoded_images.to(get_offload_device()) +# decoded_images = None +# +# if (not is_dummy_section) or (is_dummy_section and ((self.use_dummy_image == "last") or (self.use_dummy_image == "all"))): +# print("HERE 9090000a") +# start_image = decoded_split[len(decoded_split) - 1].clone().detach() +# elif is_dummy_section: +# print("HERE 9090000b") +# if self.use_dummy_image == "first": +# print("HERE 9090000c") +# start_image = decoded_split[0].clone().detach() +# elif self.use_dummy_image == "middle": +# print("HERE 9090000d") +# start_image = decoded_split[int(len(decoded_split) // 2)].clone().detach() +# else: +# print("HERE 9090000e") +# random.seed(self.seed) +# start_image = dummy_split[random.randrange(0, len(decoded_split) - 1, 1)].clone().detach() +# +# is_i2v = True +# +# if last_image is None: +# last_image = start_image.clone().detach() +# +# print("HERE 9090001") +# +# samples = None +# latents = None +# +# mm.soft_empty_cache() +# gc.collect() +# time.sleep(0.1) +# +# print("*******************************************") +# print("****** WarpedSamplerCustomBatch: Total Images Generated {}".format(total_images_generated)) +# print("*******************************************\n") +# +# generation_status = "****** WarpedSamplerCustomBatch: Total Images Generated {} ******".format(total_images_generated) +# +# interrupted = False +# +# print("HERE 9090002") +# +# if is_dummy_section: +# print("HERE 9090003") +# +# if self.gen_dummy_only: +# print("HERE 9090004") +# break +# +# print("HERE 9090005") +# +# is_dummy_section = False +# +# print("HERE 9090006") +# except mm.InterruptProcessingException as ie: +# interrupted = True +# print(f"\nWarpedSamplerCustomBatch: Processing Interrupted.") +# print("WarpedSamplerCustomBatch: Returning only partial results (if any).\n If zero images generated, a blank yellow image will be returned.\n") +# mm.unload_all_models() +# mm.soft_empty_cache() +# gc.collect() +# time.sleep(1) +# +# generation_status = f"\nWarpedSamplerCustomBatch: Processing Interrupted." +# +# traceback.print_tb(ie.__traceback__, limit=99, file=sys.stdout) +# +# raise mm.InterruptProcessingException(f"WarpedSamplerCustomBatch: Processing Interrupted.") +# +# except BaseException as e: +# print(f"\nWarpedSamplerCustomBatch: Exception During Processing: {str(e)}") +# print("WarpedSamplerCustomBatch: Returning only partial results (if any).\n If zero images generated, a blank red image will be returned.\n") +# mm.unload_all_models() +# mm.soft_empty_cache() +# gc.collect() +# time.sleep(1) +# +# generation_status = f"WarpedSamplerCustomBatch: Exception During Processing: {str(e)}" +# generation_status = "{}{}".format(generation_status, "WarpedSamplerCustomBatch: Returning only partial results (if any).\nIf zero images generated, a blank red image will be returned.") +# +# traceback.print_tb(e.__traceback__, limit=99, file=sys.stdout) +# +# pass +# +# callback = None +# self.guider.model_patcher.model.to(get_offload_device()) +# +# latent = None +# latent_image = None +# noise_mask = None +# samples = None +# +# torch.cuda.empty_cache() +# torch.cuda.ipc_collect() +# gc.collect() +# time.sleep(1) +# +# mm.unload_all_models() +# mm.soft_empty_cache() +# gc.collect() +# time.sleep(1) +# +# image = None +# +# if interrupted and (len(output_images) < 1): +# temp_image = Image.new('RGB', (self.width * 8, self.height * 8), color = 'yellow') +# image = pil2tensorSwap(temp_image) +# final_images = image +# elif (len(output_images) < 1): +# temp_image = Image.new('RGB', (self.width * 8, self.height * 8), color = 'red') +# image = pil2tensorSwap(temp_image) +# final_images = image +# else: +# final_images = self.assemble_final_result(output_images) +# +# if output_images_latents is None: +# output_images_latents = torch.zeros([1, 16, 1, self.height, self.width], dtype=torch.float32, device=self.offload_device) +# +# return (final_images, {"samples": output_images_latents}, self.seed, generation_status, last_image,) +# +# def assemble_final_result(self, image_batches): +# if self.blend_frames < 1: +# resulting_images = None +# for entry in image_batches: +# if not resulting_images is None: +# resulting_images = torch.cat((resulting_images, entry), 0) +# else: +# resulting_images = entry +# +# entry.to(device=self.offload_device) +# entry = None +# else: +# blend_value = 1.0 / self.blend_frames +# i = 0 +# while i < (len(image_batches) - 1): +# alpha_blend_val = blend_value +# blend_count = self.blend_frames +# +# image_batches_tuple = torch.split(image_batches[i], 1, dim=0) +# image_batches_split = [item for item in image_batches_tuple] +# image1 = image_batches_split[len(image_batches_split) - 1] +# # image1 = image_batches_split[len(image_batches_split) - 4] +# image_batches_tuple = None +# image_batches_split = None +# +# image_batches_tuple = torch.split(image_batches[i + 1], 1, dim=0) +# image_batches_split = [item for item in image_batches_tuple] +# image2 = image_batches_split[0] +# image_batches_tuple = None +# image_batches_split = None +# +# image1 = tensor2pilSwap(image1)[0] +# image2 = tensor2pilSwap(image2)[0] +# +# blend_latents = None +# +# while blend_count > 0: +# blended_image = Image.blend(image1, image2, alpha_blend_val) +# temp_latent = pil2tensorSwap(blended_image) +# blended_image = None +# +# if len(temp_latent.shape) < 4: +# temp_latent = temp_latent.unsqueeze(0) +# +# if not blend_latents is None: +# blend_latents = torch.cat((blend_latents, temp_latent), 0) +# else: +# blend_latents = temp_latent +# +# alpha_blend_val += blend_value +# blend_count -= 1 +# +# image_batches_tuple = torch.split(image_batches[i], image_batches[i].shape[0] - 3, dim=0) +# image_batches_split = [item for item in image_batches_tuple] +# image_batches_tuple = None +# +# image_batches[i] = torch.cat((image_batches_split[0], blend_latents), 0) +# blend_latents = None +# image_batches_split = None +# +# self.cleanup(unload_models=False, cleanup_models=False, cleanup_cuda=True) +# +# i += 1 +# +# resulting_images = None +# for entry in image_batches: +# if not resulting_images is None: +# resulting_images = torch.cat((resulting_images, entry), 0) +# else: +# resulting_images = entry +# +# entry.to(device=self.offload_device) +# +# image_batches = None +# +# self.cleanup(unload_models=False, cleanup_models=False, cleanup_cuda=True) +# +# +# print("assemble_final_result: Full decoded images count: {}".format(resulting_images.shape[0])) +# +# if self.skip_frames < 1: +# return resulting_images +# +# skipped_frames = 1 +# +# image_batches_tuple = torch.split(resulting_images, 1, dim=0) +# image_batches_split = [item for item in image_batches_tuple] +# +# resulting_images = None +# +# for image in image_batches_split: +# if skipped_frames <= self.skip_frames: +# skipped_frames += 1 +# continue +# +# if not resulting_images is None: +# resulting_images = torch.cat((resulting_images, image), 0) +# else: +# resulting_images = image +# +# print("assemble_final_result: Final decoded images count: {}".format(resulting_images.shape[0])) +# +# return resulting_images +# +# def cleanup(self, unload_models=False, cleanup_models=False, cleanup_cuda=False): +# if unload_models: +# mm.unload_all_models() +# +# if cleanup_models: +# mm.cleanup_models() +# +# if cleanup_cuda: +# mm.soft_empty_cache() +# +# gc.collect() +# time.sleep(1) +# +# return +# +# def generate_noise(self, input_latent, generator=None): +# latent_image = input_latent["samples"] +# return warped_prepare_noise(latent_image, self.seed, generator=generator) +# +# def setup_latent_noise(self): +# dummy_noise = None +# +# if self.gen_dummy: +# rnd = torch.Generator("cpu").manual_seed(self.seed) +# dummy_noise = torch.randn((1, 16, self.dummy_latents, self.height, self.width), generator=rnd, device=rnd.device).to(device=rnd.device, dtype=torch.float32) +# +# noise = [] +# +# for i in range(self.batch_count): +# rnd = torch.Generator("cpu").manual_seed(self.seed) +# temp_noise = torch.randn((1, 16, self.latent_window_size, self.height, self.width), generator=rnd, device=rnd.device).to(device=rnd.device, dtype=torch.float32) +# +# noise.append(temp_noise) +# temp_noise = None +# +# if self.verbose_messaging: +# if not dummy_noise is None: +# print("\nsetup_latent_noise: dummy_noise Shape: {}".format(dummy_noise.shape)) +# +# i = 0 +# for entry in noise: +# print("setup_latent_noise: Batch: {} | noise Shape: {}".format(i, entry.shape)) +# i += 1 +# +# print("\n") +# +# return noise, dummy_noise +# +# def get_latent_window_size(self, preferred_batch_size, frame_count, use_batch_size="next_lowest"): +# latent_size_factor = 4 +# +# if self.verbose_messaging: +# print("get_latent_window_size: preferred_batch_size: {}".format(preferred_batch_size)) +# print("get_latent_window_size: frame_count: {}".format(frame_count)) +# print("get_latent_window_size: use_batch_size: {}".format(use_batch_size)) +# +# num_frames = int(((frame_count - 1) // 4) * 4) + 1 +# +# if num_frames != frame_count: +# print(f"Truncating video from {frame_count} to {num_frames} an because odd number of frames is not allowed.") +# +# if ((num_frames - 1) % (preferred_batch_size - 1)) == 0: +# print("(1) latent_window_size set to: {}".format(self.decoded_to_encoded_length(preferred_batch_size))) +# print("(1) batch_count set to: {}".format(int((num_frames - 1) / (preferred_batch_size - 1)))) +# return self.decoded_to_encoded_length(preferred_batch_size), int((num_frames - 1) / (preferred_batch_size - 1)), num_frames +# +# if use_batch_size == "exact": +# num_frames_final = int(((num_frames - 1) // (preferred_batch_size - 1)) + 1) * (preferred_batch_size - 1) +# +# if num_frames_final != frame_count: +# print(f"Truncating video from {num_frames} to {num_frames_final} frames for preferred_batch_size compatibility.") +# +# print("(2) latent_window_size set to: {}".format(self.decoded_to_encoded_length(preferred_batch_size + 1))) +# print("(2) batch_count set to: {}".format(int((num_frames_final - 1) / (preferred_batch_size - 1)))) +# return self.decoded_to_encoded_length(preferred_batch_size), int((num_frames_final - 1) / (preferred_batch_size - 1)), num_frames_final +# +# next_lowest_found = False +# next_highest_found = False +# next_lowest = preferred_batch_size - 1 +# next_highest = preferred_batch_size - 1 +# +# if self.verbose_messaging: +# print("get_latent_window_size: Next Lowest Initialized To: {}".format(next_lowest)) +# print("get_latent_window_size: Next Highest Initialized To: {}".format(next_highest)) +# +# num_frames_final = int(((num_frames - 1) // 4) * 4) + 1 +# +# if num_frames != num_frames_final: +# print(f"Truncating video from {num_frames} to {num_frames_final} frames for latent_window_size compatibility.") +# +# if (use_batch_size == "closest") or (use_batch_size == "next_lowest"): +# while next_lowest >= 12: +# next_lowest -= 4 +# +# if (int((num_frames_final - 1) // 4) % next_lowest) == 0: +# next_lowest_found = True +# break +# +# next_lowest += 1 +# +# if next_lowest_found and (use_batch_size == "next_lowest"): +# print("(3) latent_window_size set to: {}".format(self.decoded_to_encoded_length(next_lowest + 1))) +# print("(3) batch_count set to: {}".format(int((num_frames_final - 1) / next_lowest))) +# return self.decoded_to_encoded_length(next_lowest + 1), int((num_frames_final - 1) / next_lowest), num_frames_final +# +# while next_highest <= 156: +# next_highest += 4 +# +# if (int((num_frames_final - 1) // 4) % next_highest) == 0: +# next_highest_found = True +# break +# +# if next_highest_found and (use_batch_size == "next_highest"): +# print("(4) latent_window_size set to: {}".format(self.decoded_to_encoded_length(next_highest + 1))) +# print("(4) batch_count set to: {}".format(int((num_frames_final - 1) / next_highest))) +# return self.decoded_to_encoded_length(next_highest + 1), int((num_frames_final - 1) / next_highest), num_frames_final +# +# if next_highest_found and next_lowest_found: +# if (preferred_batch_size - next_lowest) <= (next_highest - preferred_batch_size): +# print("(5) latent_window_size set to: {}".format(self.decoded_to_encoded_length(next_lowest + 1))) +# print("(5) batch_count set to: {}".format(int((num_frames_final - 1) / next_lowest))) +# return self.decoded_to_encoded_length(next_lowest + 1), int((num_frames_final - 1) / next_lowest), num_frames_final +# elif (next_highest - preferred_batch_size) < (preferred_batch_size - next_lowest): +# print("(6) latent_window_size set to: {}".format(self.decoded_to_encoded_length(next_highest + 1))) +# print("(6) batch_count set to: {}".format(int((num_frames_final - 1) / next_highest))) +# return self.decoded_to_encoded_length(next_highest + 1), int((num_frames_final - 1) / next_highest), num_frames_final +# +# print("Unable to find a compatible latent_window_size for number of frames = {} and preferred_batch_size = {}.".format(frame_count, preferred_batch_size)) +# print("Recalculating Number Of Frames Based On preferred_batch_size of: {}".format(preferred_batch_size)) +# +# return self.calculate_new_number_of_frames(preferred_batch_size, (((frame_count - 1) // 4) * 4) + 1, use_batch_size) +# +# def calculate_new_number_of_frames(self, preferred_batch_size, frame_count, use_batch_size): +# working_batch_size = preferred_batch_size - 1 +# working_frame_count = frame_count - 1 +# +# next_lowest = next_highest = working_frame_count +# next_lowest_found = False +# next_highest_found = False +# +# while next_lowest > 37: +# next_lowest -= 4 +# +# if int(next_lowest % working_batch_size) == 0: +# next_lowest_found = True +# break +# +# if next_lowest_found and (use_batch_size == "next_lowest"): +# return self.decoded_to_encoded_length(preferred_batch_size), int(next_lowest // working_batch_size), next_lowest + 1 +# +# while next_highest < 999997: +# next_highest += 4 +# +# if int(next_highest % working_batch_size) == 0: +# next_highest_found = True +# break +# +# if next_highest_found and (use_batch_size == "next_highest"): +# return self.decoded_to_encoded_length(preferred_batch_size), int(next_highest // working_batch_size), next_highest + 1 +# +# if next_lowest_found and next_highest_found: +# if (working_frame_count - next_lowest) <= (next_highest - working_frame_count): +# return self.decoded_to_encoded_length(preferred_batch_size), int(next_lowest // working_batch_size), next_lowest + 1 +# +# return self.decoded_to_encoded_length(preferred_batch_size), int(next_highest // working_batch_size), next_highest + 1 +# +# if next_lowest_found: +# return self.decoded_to_encoded_length(preferred_batch_size), int(next_lowest // working_batch_size), next_lowest + 1 +# +# if next_highest_found: +# return self.decoded_to_encoded_length(preferred_batch_size), int(next_highest // working_batch_size), next_highest + 1 +# +# raise ValueError("Unable to find a compatible latent_window_size for number of frames = {} and preferred_batch_size = {}.".format(frame_count, preferred_batch_size)) +# +# def clip_vision_encode(self, image, crop="center"): +# crop_image = False +# if crop != "center": +# crop_image = False +# output = self.clip_vision.encode_image(image, crop=crop_image) +# return output +# +# def encoded_to_decoded_length(self, latent_length): +# if latent_length <= 0: +# return 0 +# +# result_length = ((latent_length - 1) * 4) + 1 +# +# return result_length +# +# def decoded_to_encoded_length(self, image_length): +# if image_length <= 0: +# return 0 +# +# result_length = int(((image_length - 1) / 4) + 1) +# +# return result_length +# +# def apply_flux_guidance(self, conditioning, guidance): +# c = node_helpers.conditioning_set_values(conditioning, {"guidance": guidance}) +# return c +# +# def hunyuan_text_encode(self, clip, clip_vision_output, prompt, image_interleave): +# tokens = clip.tokenize(prompt, llama_template=PROMPT_TEMPLATE_ENCODE_VIDEO_I2V, image_embeds=clip_vision_output.mm_projected, image_interleave=image_interleave) +# return clip.encode_from_tokens_scheduled(tokens) +# +# def hunyuan_encode(self, positive, vae, width, height, length, batch_size, guidance_type, start_image=None): +# latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device()) +# out_latent = {} +# +# if start_image is not None: +# start_image = comfy.utils.common_upscale(start_image[:length, :, :, :3].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1) +# +# concat_latent_image = vae.encode(start_image) +# mask = torch.ones((1, 1, latent.shape[2], concat_latent_image.shape[-2], concat_latent_image.shape[-1]), device=start_image.device, dtype=start_image.dtype) +# mask[:, :, :((start_image.shape[0] - 1) // 4) + 1] = 0.0 +# +# if guidance_type == "v1 (concat)": +# cond = {"concat_latent_image": concat_latent_image, "concat_mask": mask} +# elif guidance_type == "v2 (replace)": +# cond = {'guiding_frame_index': 0} +# latent[:, :, :concat_latent_image.shape[2]] = concat_latent_image +# out_latent["noise_mask"] = mask +# elif guidance_type == "custom": +# cond = {"ref_latent": concat_latent_image} +# +# positive = node_helpers.conditioning_set_values(positive, cond) +# +# out_latent["samples"] = latent +# return positive, out_latent +# +# def wan_text_encode(self, clip, positive_text="", negative_text=""): +# print("wan_text_encode: Loading clip model to device: {}".format(clip.patcher.load_device)) +# clip.patcher.model.to(device=clip.patcher.load_device) +# +# print("wan_text_encode: Encoding Prompts...") +# positive_conditioning = self.do_text_encode(clip, positive_text) +# negative_conditioning = self.do_text_encode(clip, negative_text) +# print("wan_text_encode: Encoding Prompts...Done.") +# +# print("wan_text_encode: Unloading clip model to device: {}".format(get_offload_device())) +# clip.patcher.model.to(device=get_offload_device()) +# +# if torch.cuda.is_available(): +# torch.cuda.empty_cache() +# +# gc.collect() +# time.sleep(1) +# +# return positive_conditioning, negative_conditioning +# +# def encode(self, clip, text): +# if clip is None: +# raise RuntimeError("ERROR: clip input is invalid: None\n\nIf the clip is from a checkpoint loader node your checkpoint does not contain a valid clip or text encoder model.") +# tokens = clip.tokenize(text) +# return clip.encode_from_tokens_scheduled(tokens) +# +# def do_text_encode(self, clip, text): +# +# if clip is None: +# raise RuntimeError("ERROR: clip input is invalid: None\n\nIf the clip is from a checkpoint loader node your checkpoint does not contain a valid clip or text encoder model.") +# +# tokens = clip.tokenize(text) +# return_encoding = clip.encode_from_tokens_scheduled(tokens) +# +# return return_encoding +# +# def wan_encode(self, positive, negative, vae, width, height, length, batch_size, start_image=None, clip_vision_output=None): +# latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device()) +# if start_image is not None: +# start_image = comfy.utils.common_upscale(start_image[:length].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1) +# image = torch.mul(torch.ones((length, height, width, start_image.shape[-1]), device=start_image.device, dtype=start_image.dtype), 0.5) +# image[:start_image.shape[0]] = start_image +# +# concat_latent_image = vae.encode(image[:, :, :, :3]) +# +# mask = torch.ones((1, 1, latent.shape[2], concat_latent_image.shape[-2], concat_latent_image.shape[-1]), device=start_image.device, dtype=start_image.dtype) +# mask[:, :, :((start_image.shape[0] - 1) // 4) + 1] = 0.0 +# +# positive = node_helpers.conditioning_set_values(positive, {"concat_latent_image": concat_latent_image, "concat_mask": mask}) +# negative = node_helpers.conditioning_set_values(negative, {"concat_latent_image": concat_latent_image, "concat_mask": mask}) +# +# if clip_vision_output is not None: +# positive = node_helpers.conditioning_set_values(positive, {"clip_vision_output": clip_vision_output}) +# negative = node_helpers.conditioning_set_values(negative, {"clip_vision_output": clip_vision_output}) +# +# out_latent = {} +# out_latent["samples"] = latent +# return positive, negative, out_latent +# +# def process_skip_images(self, frames, skip_count): +# if len(frames.shape) < 4: +# frames = frames.unsqueeze(0) +# +# num_frames = frames.shape[0] +# +# image_batches_tuple = torch.split(frames, 1, dim=0) +# image_batches_split = [item for item in image_batches_tuple] +# +# new_video = None +# i = 0 +# +# while i < len(image_batches_split): +# if i < skip_count: +# i += 1 +# continue +# +# if not new_video is None: +# new_video = torch.cat((new_video, image_batches_split[i]), 0) +# else: +# new_video = image_batches_split[i] +# +# i += 1 +# +# return new_video +# +# def get_blank_image(self, length=1): +# new_image = torch.zeros([length, self.height, self.width, 3], dtype=torch.float32, device=self.offload_device) +# return new_image +# +# def get_new_noise(self, length): +# new_noise = torch.zeros([length, self.height, self.width, 3], dtype=torch.float32, device=self.offload_device) +# +# new_noise = self.encode_tiled(new_noise) +# +# new_noise = comfy.sample.fix_empty_latent_channels(self.guider.model_patcher, new_noise) +# +# if len(new_noise) < 5: +# new_latent = new_noise.unsqueeze(0) +# +# new_noise = self.generate_noise({"samples": new_noise}) +# +# return new_noise +# +# def pad_noise(self, latent, num_frames=1): +# pad_frames = torch.zeros([1, 16, num_frames, self.height, self.width], dtype=torch.float32, device=self.offload_device) +# pad_frames = torch.cat((latent, pad_frames), 2) +# +# return pad_frames +# +# def setup_callbacks(self): +# callback = latent_preview.prepare_callback(self.guider.model_patcher, self.sigmas.shape[-1] - 1, self.g_output) +# +# return callback +# +# def decode_tiled(self, latents): +# decoded_data = partial_decode_tiled(self.vae, latents, self.dec_tile_size, self.dec_overlap, self.dec_temporal_size, self.dec_temporal_overlap) +# +# if len(decoded_data.shape) < 4: +# decoded_data.unsqueeze(0) +# +# return decoded_data +# +# def initialize_noise(self, frame_count, clear_cache=True): +# noise_latents_full = torch.zeros([1, 16, int(frame_count), self.height, self.width], dtype=torch.float32, device=self.offload_device) +# print("WarpedSamplerCustomBatch: Encoded noise_latents_full Shape: {}".format(noise_latents_full.shape)) +# +# if Decimal(self.noise_scale).compare(Decimal(0.00)) != 0: +# noise_latents_full = warped_prepare_noise(noise_latents_full, self.seed) +# print("WarpedSamplerCustomBatch: noise_latents_full Shape: {}".format(noise_latents_full.shape)) +# +# noise_latents_full = torch.mul(noise_latents_full, self.noise_scale) +# +# if len(noise_latents_full.shape) < 5: +# noise_latents_full.unsqueeze(0) +# +# noise_latents_full = noise_latents_full.to(dtype=torch.float32, device=self.offload_device) +# +# if clear_cache: +# mm.soft_empty_cache() +# gc.collect() +# time.sleep(0.1) +# +# return noise_latents_full +# +# def initialize_frames(self, latents): +# if len(latents.shape) < 5: +# latents = latents.unsqueeze(0) +# +# print("WarpedSamplerCustomBatch: Encoded latents_full Shape: {}".format(latents.shape)) +# latents_full = latents.clone().detach() +# latents_full = latents_full.to(dtype=torch.float32, device=self.offload_device) +# +# noise_latents_full = self.initialize_noise(latents_full.shape[2]) +# +# mm.soft_empty_cache() +# gc.collect() +# time.sleep(0.1) +# +# return noise_latents_full +# +# class WarpedSamplerCustomScripted: +# @classmethod +# def INPUT_TYPES(s): +# return {"required": +# { +# "positive": ("STRING", {"default": None} ), +# "negative": ("STRING", {"default": None}), +# "clip": ("CLIP", ), +# "preferred_frame_count": ("INT", {"default": 81, "min": 17, "max": nodes.MAX_RESOLUTION, "step": 4}), +# "preferred_batch_size": ("INT", {"default": 17, "min": 17, "max": 161, "step": 4}), +# "use_batch_size": (["next_lowest", "next_highest", "closest", "exact"], {"default": "next_lowest", "tooltip": "Number of frames generated may be impacted by choice."}), +# "vae": ("VAE", ), +# "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), +# "model": ("MODEL", ), +# "sampler": ("SAMPLER", ), +# "sigmas": ("SIGMAS", ), +# "dec_tile_size": ("INT", {"default": 256, "min": 64, "max": 4096, "step": 32}), +# "dec_overlap": ("INT", {"default": 64, "min": 0, "max": 4096, "step": 32}), +# "dec_temporal_size": ("INT", {"default": 64, "min": 8, "max": 4096, "step": 4, "tooltip": "Only used for video VAEs: Amount of frames to decode at a time."}), +# "dec_temporal_overlap": ("INT", {"default": 8, "min": 4, "max": 4096, "step": 4, "tooltip": "Only used for video VAEs: Amount of frames to overlap."}), +# "skip_frames": ("INT", {"default": 0, "min": 0, "max": 32, "step": 4}), +# "noise_scale": ("FLOAT", {"default": 1.00, "min": 0.00, "max": 1.00, "step": 0.01}), +# "mode": (["Hunyuan", "Wan"], {"default": "Hunyuan"}), +# "hunyuan_guidance_type": (["v1 (concat)", "v2 (replace)", "custom"], {"default": "v2 (replace)", "tooltip": "Only used if mode is \"Hunyuan\"."}), +# "hunyuan_image_interleave": ("INT", {"default": 1, "min": 1, "max": 512, "tooltip": "How much the image influences things vs the text prompt. Higher number means more influence from the text prompt."}), +# }, +# "optional": +# {"noise_strength": ("FLOAT", {"default": 1.00, "min": 0.10, "max": 1.00, "step": 0.01}), +# "blend_frames": ("INT", {"default":5, "min":0, "max": 16, "step": 1}), +# "t2v_width": ("INT", {"default":480, "min":256, "max": 1280, "step": 16}), +# "t2v_height": ("INT", {"default":720, "min":256, "max": 1280, "step": 16}), +# "scaling_strength": ("FLOAT", {"default": 1.0}), +# "clip_vision_model": ("CLIP_VISION", ), +# "start_image": ("IMAGE", ), +# "flux_guidance": ("FLOAT", {"default": 3.5, "min": 0.0, "max": 100.0, "step": 0.1}), +# "use_flux_guidance": ("BOOLEAN", {"default": False}), +# "dummy_frames": ("INT", {"default":17, "min":17, "max": 121, "step": 4, "tooltip": "Number of frames to generate in dummy batch."}), +# "gen_dummy": ("BOOLEAN", {"default": False, "tooltip": "For t2v or i2v only. Will generate a dummy batch to obtain a starting image for main generation."}), +# "gen_dummy_only": ("BOOLEAN", {"default": False, "tooltip": "Will generate dummy batch only."}), +# "use_dummy_image": (["first", "middle", "last", "random", "all"], {"default": "last", "tooltip": "Which dummy batch image to start main generation."}), +# "output_latents": ("BOOLEAN", {"default": False}), +# "verbose_messaging": ("BOOLEAN", {"default": False}), +# "i2v_sigmas": ("SIGMAS", {"default": None}), +# "i2v_model": ("MODEL", {"default": None}), +# "batches_script": ("STRING", {"default": None}), +# } +# } +# +# RETURN_TYPES = ("IMAGE", "LATENT", "INT", "STRING", "IMAGE", ) +# RETURN_NAMES = ("images", "latents", "seed", "generation_status", "image_used",) +# +# FUNCTION = "sample" +# +# CATEGORY = "Warped/General/Sampling" +# +# def sample(self, positive, negative, clip, preferred_frame_count, preferred_batch_size, use_batch_size, vae, seed, model, sampler, sigmas, dec_tile_size, dec_overlap, dec_temporal_size, dec_temporal_overlap, +# skip_frames, noise_scale, mode="Hunyuan", hunyuan_guidance_type="v2 (replace)", hunyuan_image_interleave=2, noise_strength=1.0, blend_frames=5, t2v_width=480, t2v_height=720, scaling_strength=1.0, +# clip_vision_model=None, start_image=None, flux_guidance=3.5, dummy_frames=17, gen_dummy=False, gen_dummy_only=False, use_dummy_image="last", use_flux_guidance=False, output_latents=False, verbose_messaging=False, +# i2v_sigmas=None, i2v_model=None, batches_script=None): +# +# batch_scripts = None +# +# if (positive is None) and (batches_script is None): +# raise ValueError("positive and batches_script cannot both be None.") +# elif not batches_script is None: +# if verbose_messaging: +# print("\n{}\n".format(batches_script)) +# +# batch_scripts = {} +# temp_strings = batches_script.split("||") +# +# for entry in temp_strings: +# temp_script = entry.strip().split("|") +# batch_scripts[temp_script[0].strip()] = temp_script[1].strip() +# +# if verbose_messaging: +# print("\n{}\n".format(batch_scripts)) +# +# self.device = mm.get_torch_device() +# self.offload_device = get_offload_device() +# mm.unload_all_models() +# gc.collect() +# time.sleep(1) +# +# self.positive = positive +# self.negative = negative +# self.clip = clip +# self.use_clip = clip.clone() +# self.preferred_frame_count = preferred_frame_count +# self.preferred_batch_size = preferred_batch_size +# self.vae = vae +# self.seed = seed +# self.sampler = sampler +# self.sigmas = sigmas +# self.i2v_sigmas = i2v_sigmas +# self.noise_scale = noise_scale +# self.mode = mode +# self.dec_tile_size = dec_tile_size +# self.dec_overlap = dec_overlap +# self.dec_temporal_size = dec_temporal_size +# self.dec_temporal_overlap = dec_temporal_overlap +# self.clip_vision = clip_vision_model +# self.noise_strength = noise_strength +# self.blend_frames = blend_frames +# self.t2v_width = t2v_width +# self.t2v_height = t2v_height +# self.dummy_frames = dummy_frames +# self.dummy_latents = int((dummy_frames - 1) // 4) + 1 +# self.gen_dummy = gen_dummy +# self.gen_dummy_only = gen_dummy_only +# self.use_dummy_image = use_dummy_image +# self.skip_frames = skip_frames +# self.verbose_messaging = verbose_messaging +# self.model = model +# self.i2v_model = i2v_model +# self.batches_script = batches_script +# self.g_output = {} +# +# disable_pbar = not utils.PROGRESS_BAR_ENABLED +# +# self.use_sigmas = self.sigmas +# +# self.guider = WarpedGuider_Basic() +# self.guider.set_model(self.model) +# callback = self.setup_callbacks() +# +# self.latent_window_size, self.batch_count, self.truncated_frame_count = self.get_latent_window_size(preferred_batch_size, self.preferred_frame_count, use_batch_size=use_batch_size) +# self.batch_frame_count = int(((self.truncated_frame_count - 1) // self.batch_count) + 1) +# print("latent_window_size: {} | batch_count: {} | truncated_frame_count: {} | batch_frame_count: {}".format(self.latent_window_size, self.batch_count, self.truncated_frame_count, self.batch_frame_count)) +# +# is_i2v = True +# +# if start_image is None: +# start_image = torch.zeros([1, self.t2v_height, self.t2v_width, 3], dtype=torch.float32, device=self.offload_device) +# is_i2v = False +# +# print("start_image Shape: {}".format(start_image.shape)) +# +# self.width = int(start_image.shape[2] // 8) +# self.height = int(start_image.shape[1] // 8) +# +# noise, dummy_noise = self.setup_latent_noise() +# +# output_images = [] +# output_images_latents = None +# interrupted = False +# generation_status = "" +# total_images_generated = 0 +# is_dummy_section = False +# is_first_section = False +# is_last_section = False +# switch_guider = True +# last_image = None +# positive_prompt = positive +# +# if not self.gen_dummy: +# print("Generating {} Batches".format(self.batch_count)) +# else: +# print("Generating Dummy Batch Plus {} Batches".format(self.batch_count)) +# +# generation_batches = list(range(self.batch_count)) +# +# if self.gen_dummy and (not self.gen_dummy_only): +# generation_batches = [0] + generation_batches +# is_dummy_section = True +# else: +# generation_batches = [0] +# is_dummy_section = True +# +# print("generation_batches: {}".format(generation_batches)) +# +# try: +# for i, generation_batch in enumerate(generation_batches): +# batch_number = generation_batches[i] +# batch_key = "{}".format(batch_number) +# +# if is_dummy_section: +# print("Generating Dummy Batch.") +# is_first_section = False +# +# if not self.gen_dummy_only: +# is_last_section = False +# else: +# is_last_section = True +# +# if not batch_scripts is None: +# positive_prompt = batch_scripts["dummy"] +# print("\nDummy Section Positive Prompt: {}\n".format(positive_prompt)) +# last_prompt = positive_prompt +# else: +# print("Generating Batch {}.".format(batch_number)) +# +# is_first_section = batch_number == min(generation_batches) +# is_last_section = batch_number == max(generation_batches) +# +# if not batch_scripts is None: +# if batch_key in batch_scripts.keys(): +# positive_prompt = batch_scripts[batch_key] +# last_prompt = positive_prompt +# else: +# positive_prompt = last_prompt +# +# print("\nBatch: {} Positive Prompt: {}\n".format(batch_number, positive_prompt)) +# +# print("is_first_section: {} | is_last_section: {} | is_dummy_section: {}".format(is_first_section, is_last_section, is_dummy_section)) +# +# mm.soft_empty_cache() +# gc.collect() +# time.sleep(0.1) +# +# image_embeds = None +# +# print("start_image Shape: {}".format(start_image.shape)) +# +# if is_i2v: +# if len(start_image.shape) < 4: +# start_image = start_image.unsqueeze(0) +# +# print("Performing clip_vision_encode.") +# image_embeds = self.clip_vision_encode(start_image.clone().detach()) +# +# if not is_dummy_section: +# noise_latents = noise[batch_number] +# else: +# noise_latents = dummy_noise +# +# print("noise_latents Shape: {}".format(noise_latents.shape)) +# +# noise_mask = None +# +# if mode == "Hunyuan": +# if is_i2v: +# if (not self.i2v_model is None) and switch_guider: +# switch_guider = False +# callback = None +# self.guider = WarpedGuider_Basic() +# self.guider.set_model(self.i2v_model) +# callback = self.setup_callbacks() +# +# self.use_clip = self.clip.clone() +# temp_cond = self.hunyuan_text_encode(self.use_clip, image_embeds, positive_prompt, hunyuan_image_interleave) +# positive_cond, latents = self.hunyuan_encode(temp_cond, self.vae, start_image.shape[2], start_image.shape[1], self.batch_frame_count, 1, hunyuan_guidance_type, start_image=start_image) +# +# if "noise_mask" in latents: +# noise_mask = latents["noise_mask"] * scaling_strength +# +# latents = latents["samples"] +# +# # latents = latents["samples"] * (scaling_strength * vae_scaling_factor) +# else: +# positive_cond = self.do_text_encode(self.use_clip, positive_prompt) +# latents = torch.zeros([1, 16, noise_latents.shape[2], self.height, self.width], dtype=torch.float32, device=self.offload_device) +# +# latents = latents * scaling_strength # scaling_strength +# +# if use_flux_guidance: +# positive_cond = self.apply_flux_guidance(positive_cond, flux_guidance) +# +# self.guider.set_conds_single(positive_cond) +# else: +# # if is_i2v: +# # if (not self.i2v_model is None) and switch_guider: +# # switch_guider = False +# # callback = None +# # self.guider = WarpedGuider_Basic() +# # self.guider.set_model(self.i2v_model) +# # callback = self.setup_callbacks() +# if not is_i2v: +# positive_cond, negative_cond = self.wan_text_encode(self.use_clip, positive_text=positive_prompt, negative_text=negative) +# positive_cond, negative_cond, latents = self.wan_encode(positive_cond, negative_cond, self.vae, start_image.shape[2], start_image.shape[1], ((noise_latents.shape[2] - 1) * 4) + 1, 1, start_image=None, clip_vision_output=None) +# else: +# positive_cond, negative_cond = self.wan_text_encode(self.use_clip, positive_text=positive_prompt, negative_text=negative) +# positive_cond, negative_cond, latents = self.wan_encode(positive_cond, negative_cond, self.vae, start_image.shape[2], start_image.shape[1], ((noise_latents.shape[2] - 1) * 4) + 1, 1, start_image=start_image, clip_vision_output=image_embeds) +# +# latents = latents["samples"] * scaling_strength +# +# self.guider.set_conds_both(positive_cond, negative_cond) +# +# print("latents Shape: {}".format(latents.shape)) +# +# num_frames = int(((latents.shape[2] - 1) * 4) + 1) +# +# print("\nDecoded Width is {} | Decoded Height is {}".format(int(self.width * 8), int(self.height * 8))) +# +# print("-------------------------------------------------------------------------------------------") +# print("WarpedSamplerCustomScripted: Latents Shape: {} | Noise Latents Shape: {}".format(latents.shape, noise_latents.shape)) +# print("-------------------------------------------------------------------------------------------") +# +# print("Noise Shape: {} | Latents Shape: {}".format(noise_latents.shape, latents.shape)) +# print("WarpedSamplerCustomScripted: Generating {} Frames in {} Latents....".format(num_frames, latents.shape[2])) +# +# samples = self.guider.sample(noise_latents, latents, self.sampler, self.use_sigmas, denoise_mask=noise_mask, callback=callback, disable_pbar=disable_pbar, seed=self.seed) +# samples = samples.to(mm.intermediate_device()) +# +# mm.unload_all_models() +# mm.soft_empty_cache() +# gc.collect() +# time.sleep(1) +# +# if len(samples.shape) < 5: +# samples = samples.unsqueeze(0) +# +# samples = samples.clone().detach() / scaling_strength # (scaling_strength * vae_scaling_factor) +# +# print("WarpedSamplerCustomScripted: Decoding Latents...") +# decoded_images = self.decode_tiled(samples) +# +# if not is_dummy_section or (is_dummy_section and ((self.use_dummy_image == "all") or self.gen_dummy_only)): +# output_images.append(decoded_images) +# total_images_generated += decoded_images.shape[0] +# +# samples = None +# +# # output_images = self.process_skip_images(output_images, skip_frames) +# print("WarpedSamplerCustomScripted: Decoded Images Shape: {}".format(decoded_images.shape[0])) +# +# # if output_latents: +# # output_images_latents = self.encode_tiled(output_images) +# +# print("WarpedSamplerCustomScripted: Generating {} frames in {} latents...Done.".format(num_frames, latents.shape[2])) +# +# if not is_i2v: +# self.guider.model_patcher.model.to(get_offload_device()) +# # guider.set_i2v_model() +# +# if not self.i2v_sigmas is None: +# self.use_sigmas = self.i2v_sigmas +# +# if not is_last_section: +# print("HERE 9090000") +# decoded_tuple = torch.split(decoded_images, 1, dim=0) +# decoded_split = [item for item in decoded_tuple] +# decoded_images = decoded_images.to(get_offload_device()) +# decoded_images = None +# +# if (not is_dummy_section) or (is_dummy_section and ((self.use_dummy_image == "last") or (self.use_dummy_image == "all"))): +# print("HERE 9090000a") +# start_image = decoded_split[len(decoded_split) - 1].clone().detach() +# elif is_dummy_section: +# print("HERE 9090000b") +# if self.use_dummy_image == "first": +# print("HERE 9090000c") +# start_image = decoded_split[0].clone().detach() +# elif self.use_dummy_image == "middle": +# print("HERE 9090000d") +# start_image = decoded_split[int(len(decoded_split) // 2)].clone().detach() +# else: +# print("HERE 9090000e") +# random.seed(self.seed) +# start_image = dummy_split[random.randrange(0, len(decoded_split) - 1, 1)].clone().detach() +# +# is_i2v = True +# +# if last_image is None: +# last_image = start_image.clone().detach() +# +# print("HERE 9090001") +# +# samples = None +# latents = None +# +# mm.soft_empty_cache() +# gc.collect() +# time.sleep(0.1) +# +# print("*******************************************") +# print("****** WarpedSamplerCustomScripted: Total Images Generated {}".format(total_images_generated)) +# print("*******************************************\n") +# +# generation_status = "****** WarpedSamplerCustomScripted: Total Images Generated {} ******".format(total_images_generated) +# +# interrupted = False +# +# print("HERE 9090002") +# +# if is_dummy_section: +# print("HERE 9090003") +# +# if self.gen_dummy_only: +# print("HERE 9090004") +# break +# +# print("HERE 9090005") +# +# is_dummy_section = False +# +# print("HERE 9090006") +# except mm.InterruptProcessingException as ie: +# interrupted = True +# print(f"\nWarpedSamplerCustomScripted: Processing Interrupted.") +# print("WarpedSamplerCustomScripted: Returning only partial results (if any).\n If zero images generated, a blank yellow image will be returned.\n") +# mm.unload_all_models() +# mm.soft_empty_cache() +# gc.collect() +# time.sleep(1) +# +# generation_status = f"\nWarpedSamplerCustomScripted: Processing Interrupted." +# +# traceback.print_tb(ie.__traceback__, limit=99, file=sys.stdout) +# +# raise mm.InterruptProcessingException(f"WarpedSamplerCustomScripted: Processing Interrupted.") +# +# except BaseException as e: +# print(f"\nWarpedSamplerCustomScripted: Exception During Processing: {str(e)}") +# print("WarpedSamplerCustomScripted: Returning only partial results (if any).\n If zero images generated, a blank red image will be returned.\n") +# mm.unload_all_models() +# mm.soft_empty_cache() +# gc.collect() +# time.sleep(1) +# +# generation_status = f"WarpedSamplerCustomScripted: Exception During Processing: {str(e)}" +# generation_status = "{}{}".format(generation_status, "WarpedSamplerCustomScripted: Returning only partial results (if any).\nIf zero images generated, a blank red image will be returned.") +# +# traceback.print_tb(e.__traceback__, limit=99, file=sys.stdout) +# +# pass +# +# callback = None +# self.guider.model_patcher.model.to(get_offload_device()) +# +# latent = None +# latent_image = None +# noise_mask = None +# samples = None +# +# torch.cuda.empty_cache() +# torch.cuda.ipc_collect() +# gc.collect() +# time.sleep(1) +# +# mm.unload_all_models() +# mm.soft_empty_cache() +# gc.collect() +# time.sleep(1) +# +# image = None +# +# if interrupted and (len(output_images) < 1): +# temp_image = Image.new('RGB', (self.width * 8, self.height * 8), color = 'yellow') +# image = pil2tensorSwap(temp_image) +# final_images = image +# elif (len(output_images) < 1): +# temp_image = Image.new('RGB', (self.width * 8, self.height * 8), color = 'red') +# image = pil2tensorSwap(temp_image) +# final_images = image +# else: +# final_images = self.assemble_final_result(output_images) +# +# if output_images_latents is None: +# output_images_latents = torch.zeros([1, 16, 1, self.height, self.width], dtype=torch.float32, device=self.offload_device) +# +# return (final_images, {"samples": output_images_latents}, self.seed, generation_status, last_image,) +# +# def assemble_final_result(self, image_batches): +# if self.blend_frames < 1: +# resulting_images = None +# for entry in image_batches: +# if not resulting_images is None: +# resulting_images = torch.cat((resulting_images, entry), 0) +# else: +# resulting_images = entry +# +# entry.to(device=self.offload_device) +# entry = None +# else: +# blend_value = 1.0 / self.blend_frames +# i = 0 +# while i < (len(image_batches) - 1): +# alpha_blend_val = blend_value +# blend_count = self.blend_frames +# +# image_batches_tuple = torch.split(image_batches[i], 1, dim=0) +# image_batches_split = [item for item in image_batches_tuple] +# image1 = image_batches_split[len(image_batches_split) - 1] +# # image1 = image_batches_split[len(image_batches_split) - 4] +# image_batches_tuple = None +# image_batches_split = None +# +# image_batches_tuple = torch.split(image_batches[i + 1], 1, dim=0) +# image_batches_split = [item for item in image_batches_tuple] +# image2 = image_batches_split[0] +# image_batches_tuple = None +# image_batches_split = None +# +# image1 = tensor2pilSwap(image1)[0] +# image2 = tensor2pilSwap(image2)[0] +# +# blend_latents = None +# +# while blend_count > 0: +# blended_image = Image.blend(image1, image2, alpha_blend_val) +# temp_latent = pil2tensorSwap(blended_image) +# blended_image = None +# +# if len(temp_latent.shape) < 4: +# temp_latent = temp_latent.unsqueeze(0) +# +# if not blend_latents is None: +# blend_latents = torch.cat((blend_latents, temp_latent), 0) +# else: +# blend_latents = temp_latent +# +# alpha_blend_val += blend_value +# blend_count -= 1 +# +# image_batches_tuple = torch.split(image_batches[i], image_batches[i].shape[0] - 3, dim=0) +# image_batches_split = [item for item in image_batches_tuple] +# image_batches_tuple = None +# +# image_batches[i] = torch.cat((image_batches_split[0], blend_latents), 0) +# blend_latents = None +# image_batches_split = None +# +# self.cleanup(unload_models=False, cleanup_models=False, cleanup_cuda=True) +# +# i += 1 +# +# resulting_images = None +# for entry in image_batches: +# if not resulting_images is None: +# resulting_images = torch.cat((resulting_images, entry), 0) +# else: +# resulting_images = entry +# +# entry.to(device=self.offload_device) +# +# image_batches = None +# +# self.cleanup(unload_models=False, cleanup_models=False, cleanup_cuda=True) +# +# +# print("assemble_final_result: Full decoded images count: {}".format(resulting_images.shape[0])) +# +# if self.skip_frames < 1: +# return resulting_images +# +# skipped_frames = 1 +# +# image_batches_tuple = torch.split(resulting_images, 1, dim=0) +# image_batches_split = [item for item in image_batches_tuple] +# +# resulting_images = None +# +# for image in image_batches_split: +# if skipped_frames <= self.skip_frames: +# skipped_frames += 1 +# continue +# +# if not resulting_images is None: +# resulting_images = torch.cat((resulting_images, image), 0) +# else: +# resulting_images = image +# +# print("assemble_final_result: Final decoded images count: {}".format(resulting_images.shape[0])) +# +# return resulting_images +# +# def cleanup(self, unload_models=False, cleanup_models=False, cleanup_cuda=False): +# if unload_models: +# mm.unload_all_models() +# +# if cleanup_models: +# mm.cleanup_models() +# +# if cleanup_cuda: +# mm.soft_empty_cache() +# +# gc.collect() +# time.sleep(1) +# +# return +# +# def generate_noise(self, input_latent, generator=None): +# latent_image = input_latent["samples"] +# return warped_prepare_noise(latent_image, self.seed, generator=generator) +# +# def setup_latent_noise(self): +# dummy_noise = None +# +# if self.gen_dummy: +# rnd = torch.Generator("cpu").manual_seed(self.seed) +# dummy_noise = torch.randn((1, 16, self.dummy_latents, self.height, self.width), generator=rnd, device=rnd.device).to(device=rnd.device, dtype=torch.float32) +# +# noise = [] +# +# for i in range(self.batch_count): +# rnd = torch.Generator("cpu").manual_seed(self.seed) +# temp_noise = torch.randn((1, 16, self.latent_window_size, self.height, self.width), generator=rnd, device=rnd.device).to(device=rnd.device, dtype=torch.float32) +# +# noise.append(temp_noise) +# temp_noise = None +# +# if self.verbose_messaging: +# if not dummy_noise is None: +# print("\nsetup_latent_noise: dummy_noise Shape: {}".format(dummy_noise.shape)) +# +# i = 0 +# for entry in noise: +# print("setup_latent_noise: Batch: {} | noise Shape: {}".format(i, entry.shape)) +# i += 1 +# +# print("\n") +# +# return noise, dummy_noise +# +# def get_latent_window_size(self, preferred_batch_size, frame_count, use_batch_size="next_lowest"): +# latent_size_factor = 4 +# +# if self.verbose_messaging: +# print("get_latent_window_size: preferred_batch_size: {}".format(preferred_batch_size)) +# print("get_latent_window_size: frame_count: {}".format(frame_count)) +# print("get_latent_window_size: use_batch_size: {}".format(use_batch_size)) +# +# num_frames = int(((frame_count - 1) // 4) * 4) + 1 +# +# if num_frames != frame_count: +# print(f"Truncating video from {frame_count} to {num_frames} an because odd number of frames is not allowed.") +# +# if ((num_frames - 1) % (preferred_batch_size - 1)) == 0: +# print("(1) latent_window_size set to: {}".format(self.decoded_to_encoded_length(preferred_batch_size))) +# print("(1) batch_count set to: {}".format(int((num_frames - 1) / (preferred_batch_size - 1)))) +# return self.decoded_to_encoded_length(preferred_batch_size), int((num_frames - 1) / (preferred_batch_size - 1)), num_frames +# +# if use_batch_size == "exact": +# num_frames_final = int(((num_frames - 1) // (preferred_batch_size - 1)) + 1) * (preferred_batch_size - 1) +# +# if num_frames_final != frame_count: +# print(f"Truncating video from {num_frames} to {num_frames_final} frames for preferred_batch_size compatibility.") +# +# print("(2) latent_window_size set to: {}".format(self.decoded_to_encoded_length(preferred_batch_size + 1))) +# print("(2) batch_count set to: {}".format(int((num_frames_final - 1) / (preferred_batch_size - 1)))) +# return self.decoded_to_encoded_length(preferred_batch_size), int((num_frames_final - 1) / (preferred_batch_size - 1)), num_frames_final +# +# next_lowest_found = False +# next_highest_found = False +# next_lowest = preferred_batch_size - 1 +# next_highest = preferred_batch_size - 1 +# +# if self.verbose_messaging: +# print("get_latent_window_size: Next Lowest Initialized To: {}".format(next_lowest)) +# print("get_latent_window_size: Next Highest Initialized To: {}".format(next_highest)) +# +# num_frames_final = int(((num_frames - 1) // 4) * 4) + 1 +# +# if num_frames != num_frames_final: +# print(f"Truncating video from {num_frames} to {num_frames_final} frames for latent_window_size compatibility.") +# +# if (use_batch_size == "closest") or (use_batch_size == "next_lowest"): +# while next_lowest >= 12: +# next_lowest -= 4 +# +# if (int((num_frames_final - 1) // 4) % next_lowest) == 0: +# next_lowest_found = True +# break +# +# next_lowest += 1 +# +# if next_lowest_found and (use_batch_size == "next_lowest"): +# print("(3) latent_window_size set to: {}".format(self.decoded_to_encoded_length(next_lowest + 1))) +# print("(3) batch_count set to: {}".format(int((num_frames_final - 1) / next_lowest))) +# return self.decoded_to_encoded_length(next_lowest + 1), int((num_frames_final - 1) / next_lowest), num_frames_final +# +# while next_highest <= 156: +# next_highest += 4 +# +# if (int((num_frames_final - 1) // 4) % next_highest) == 0: +# next_highest_found = True +# break +# +# if next_highest_found and (use_batch_size == "next_highest"): +# print("(4) latent_window_size set to: {}".format(self.decoded_to_encoded_length(next_highest + 1))) +# print("(4) batch_count set to: {}".format(int((num_frames_final - 1) / next_highest))) +# return self.decoded_to_encoded_length(next_highest + 1), int((num_frames_final - 1) / next_highest), num_frames_final +# +# if next_highest_found and next_lowest_found: +# if (preferred_batch_size - next_lowest) <= (next_highest - preferred_batch_size): +# print("(5) latent_window_size set to: {}".format(self.decoded_to_encoded_length(next_lowest + 1))) +# print("(5) batch_count set to: {}".format(int((num_frames_final - 1) / next_lowest))) +# return self.decoded_to_encoded_length(next_lowest + 1), int((num_frames_final - 1) / next_lowest), num_frames_final +# elif (next_highest - preferred_batch_size) < (preferred_batch_size - next_lowest): +# print("(6) latent_window_size set to: {}".format(self.decoded_to_encoded_length(next_highest + 1))) +# print("(6) batch_count set to: {}".format(int((num_frames_final - 1) / next_highest))) +# return self.decoded_to_encoded_length(next_highest + 1), int((num_frames_final - 1) / next_highest), num_frames_final +# +# print("Unable to find a compatible latent_window_size for number of frames = {} and preferred_batch_size = {}.".format(frame_count, preferred_batch_size)) +# print("Recalculating Number Of Frames Based On preferred_batch_size of: {}".format(preferred_batch_size)) +# +# return self.calculate_new_number_of_frames(preferred_batch_size, (((frame_count - 1) // 4) * 4) + 1, use_batch_size) +# +# def calculate_new_number_of_frames(self, preferred_batch_size, frame_count, use_batch_size): +# working_batch_size = preferred_batch_size - 1 +# working_frame_count = frame_count - 1 +# +# next_lowest = next_highest = working_frame_count +# next_lowest_found = False +# next_highest_found = False +# +# while next_lowest > 37: +# next_lowest -= 4 +# +# if int(next_lowest % working_batch_size) == 0: +# next_lowest_found = True +# break +# +# if next_lowest_found and (use_batch_size == "next_lowest"): +# return self.decoded_to_encoded_length(preferred_batch_size), int(next_lowest // working_batch_size), next_lowest + 1 +# +# while next_highest < 999997: +# next_highest += 4 +# +# if int(next_highest % working_batch_size) == 0: +# next_highest_found = True +# break +# +# if next_highest_found and (use_batch_size == "next_highest"): +# return self.decoded_to_encoded_length(preferred_batch_size), int(next_highest // working_batch_size), next_highest + 1 +# +# if next_lowest_found and next_highest_found: +# if (working_frame_count - next_lowest) <= (next_highest - working_frame_count): +# return self.decoded_to_encoded_length(preferred_batch_size), int(next_lowest // working_batch_size), next_lowest + 1 +# +# return self.decoded_to_encoded_length(preferred_batch_size), int(next_highest // working_batch_size), next_highest + 1 +# +# if next_lowest_found: +# return self.decoded_to_encoded_length(preferred_batch_size), int(next_lowest // working_batch_size), next_lowest + 1 +# +# if next_highest_found: +# return self.decoded_to_encoded_length(preferred_batch_size), int(next_highest // working_batch_size), next_highest + 1 +# +# raise ValueError("Unable to find a compatible latent_window_size for number of frames = {} and preferred_batch_size = {}.".format(frame_count, preferred_batch_size)) +# +# def clip_vision_encode(self, image, crop="center"): +# crop_image = False +# if crop != "center": +# crop_image = False +# output = self.clip_vision.encode_image(image, crop=crop_image) +# return output +# +# def encoded_to_decoded_length(self, latent_length): +# if latent_length <= 0: +# return 0 +# +# result_length = ((latent_length - 1) * 4) + 1 +# +# return result_length +# +# def decoded_to_encoded_length(self, image_length): +# if image_length <= 0: +# return 0 +# +# result_length = int(((image_length - 1) / 4) + 1) +# +# return result_length +# +# def apply_flux_guidance(self, conditioning, guidance): +# c = node_helpers.conditioning_set_values(conditioning, {"guidance": guidance}) +# return c +# +# def hunyuan_text_encode(self, clip, clip_vision_output, prompt, image_interleave): +# tokens = clip.tokenize(prompt, llama_template=PROMPT_TEMPLATE_ENCODE_VIDEO_I2V, image_embeds=clip_vision_output.mm_projected, image_interleave=image_interleave) +# return clip.encode_from_tokens_scheduled(tokens) +# +# def hunyuan_encode(self, positive, vae, width, height, length, batch_size, guidance_type, start_image=None): +# latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device()) +# out_latent = {} +# +# if start_image is not None: +# start_image = comfy.utils.common_upscale(start_image[:length, :, :, :3].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1) +# +# concat_latent_image = vae.encode(start_image) +# mask = torch.ones((1, 1, latent.shape[2], concat_latent_image.shape[-2], concat_latent_image.shape[-1]), device=start_image.device, dtype=start_image.dtype) +# mask[:, :, :((start_image.shape[0] - 1) // 4) + 1] = 0.0 +# +# if guidance_type == "v1 (concat)": +# cond = {"concat_latent_image": concat_latent_image, "concat_mask": mask} +# elif guidance_type == "v2 (replace)": +# cond = {'guiding_frame_index': 0} +# latent[:, :, :concat_latent_image.shape[2]] = concat_latent_image +# out_latent["noise_mask"] = mask +# elif guidance_type == "custom": +# cond = {"ref_latent": concat_latent_image} +# +# positive = node_helpers.conditioning_set_values(positive, cond) +# +# out_latent["samples"] = latent +# return positive, out_latent +# +# def wan_text_encode(self, clip, positive_text="", negative_text=""): +# print("wan_text_encode: Loading clip model to device: {}".format(clip.patcher.load_device)) +# clip.patcher.model.to(device=clip.patcher.load_device) +# +# print("wan_text_encode: Encoding Prompts...") +# positive_conditioning = self.do_text_encode(clip, positive_text) +# negative_conditioning = self.do_text_encode(clip, negative_text) +# print("wan_text_encode: Encoding Prompts...Done.") +# +# print("wan_text_encode: Unloading clip model to device: {}".format(get_offload_device())) +# clip.patcher.model.to(device=get_offload_device()) +# +# if torch.cuda.is_available(): +# torch.cuda.empty_cache() +# +# gc.collect() +# time.sleep(1) +# +# return positive_conditioning, negative_conditioning +# +# def encode(self, clip, text): +# if clip is None: +# raise RuntimeError("ERROR: clip input is invalid: None\n\nIf the clip is from a checkpoint loader node your checkpoint does not contain a valid clip or text encoder model.") +# tokens = clip.tokenize(text) +# return clip.encode_from_tokens_scheduled(tokens) +# +# def do_text_encode(self, clip, text): +# +# if clip is None: +# raise RuntimeError("ERROR: clip input is invalid: None\n\nIf the clip is from a checkpoint loader node your checkpoint does not contain a valid clip or text encoder model.") +# +# tokens = clip.tokenize(text) +# return_encoding = clip.encode_from_tokens_scheduled(tokens) +# +# return return_encoding +# +# def wan_encode(self, positive, negative, vae, width, height, length, batch_size, start_image=None, clip_vision_output=None): +# latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device()) +# if start_image is not None: +# start_image = comfy.utils.common_upscale(start_image[:length].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1) +# image = torch.mul(torch.ones((length, height, width, start_image.shape[-1]), device=start_image.device, dtype=start_image.dtype), 0.5) +# image[:start_image.shape[0]] = start_image +# +# concat_latent_image = vae.encode(image[:, :, :, :3]) +# +# mask = torch.ones((1, 1, latent.shape[2], concat_latent_image.shape[-2], concat_latent_image.shape[-1]), device=start_image.device, dtype=start_image.dtype) +# mask[:, :, :((start_image.shape[0] - 1) // 4) + 1] = 0.0 +# +# positive = node_helpers.conditioning_set_values(positive, {"concat_latent_image": concat_latent_image, "concat_mask": mask}) +# negative = node_helpers.conditioning_set_values(negative, {"concat_latent_image": concat_latent_image, "concat_mask": mask}) +# +# if clip_vision_output is not None: +# positive = node_helpers.conditioning_set_values(positive, {"clip_vision_output": clip_vision_output}) +# negative = node_helpers.conditioning_set_values(negative, {"clip_vision_output": clip_vision_output}) +# +# out_latent = {} +# out_latent["samples"] = latent +# return positive, negative, out_latent +# +# def process_skip_images(self, frames, skip_count): +# if len(frames.shape) < 4: +# frames = frames.unsqueeze(0) +# +# num_frames = frames.shape[0] +# +# image_batches_tuple = torch.split(frames, 1, dim=0) +# image_batches_split = [item for item in image_batches_tuple] +# +# new_video = None +# i = 0 +# +# while i < len(image_batches_split): +# if i < skip_count: +# i += 1 +# continue +# +# if not new_video is None: +# new_video = torch.cat((new_video, image_batches_split[i]), 0) +# else: +# new_video = image_batches_split[i] +# +# i += 1 +# +# return new_video +# +# def get_blank_image(self, length=1): +# new_image = torch.zeros([length, self.height, self.width, 3], dtype=torch.float32, device=self.offload_device) +# return new_image +# +# def get_new_noise(self, length): +# new_noise = torch.zeros([length, self.height, self.width, 3], dtype=torch.float32, device=self.offload_device) +# +# new_noise = self.encode_tiled(new_noise) +# +# new_noise = comfy.sample.fix_empty_latent_channels(self.guider.model_patcher, new_noise) +# +# if len(new_noise) < 5: +# new_latent = new_noise.unsqueeze(0) +# +# new_noise = self.generate_noise({"samples": new_noise}) +# +# return new_noise +# +# def pad_noise(self, latent, num_frames=1): +# pad_frames = torch.zeros([1, 16, num_frames, self.height, self.width], dtype=torch.float32, device=self.offload_device) +# pad_frames = torch.cat((latent, pad_frames), 2) +# +# return pad_frames +# +# def setup_callbacks(self): +# callback = latent_preview.prepare_callback(self.guider.model_patcher, self.sigmas.shape[-1] - 1, self.g_output) +# +# return callback +# +# def decode_tiled(self, latents): +# decoded_data = partial_decode_tiled(self.vae, latents, self.dec_tile_size, self.dec_overlap, self.dec_temporal_size, self.dec_temporal_overlap) +# +# if len(decoded_data.shape) < 4: +# decoded_data.unsqueeze(0) +# +# return decoded_data +# +# def initialize_noise(self, frame_count, clear_cache=True): +# noise_latents_full = torch.zeros([1, 16, int(frame_count), self.height, self.width], dtype=torch.float32, device=self.offload_device) +# print("WarpedSamplerCustomScripted: Encoded noise_latents_full Shape: {}".format(noise_latents_full.shape)) +# +# if Decimal(self.noise_scale).compare(Decimal(0.00)) != 0: +# noise_latents_full = warped_prepare_noise(noise_latents_full, self.seed) +# print("WarpedSamplerCustomScripted: noise_latents_full Shape: {}".format(noise_latents_full.shape)) +# +# noise_latents_full = torch.mul(noise_latents_full, self.noise_scale) +# +# if len(noise_latents_full.shape) < 5: +# noise_latents_full.unsqueeze(0) +# +# noise_latents_full = noise_latents_full.to(dtype=torch.float32, device=self.offload_device) +# +# if clear_cache: +# mm.soft_empty_cache() +# gc.collect() +# time.sleep(0.1) +# +# return noise_latents_full +# +# def initialize_frames(self, latents): +# if len(latents.shape) < 5: +# latents = latents.unsqueeze(0) +# +# print("WarpedSamplerCustomScripted: Encoded latents_full Shape: {}".format(latents.shape)) +# latents_full = latents.clone().detach() +# latents_full = latents_full.to(dtype=torch.float32, device=self.offload_device) +# +# noise_latents_full = self.initialize_noise(latents_full.shape[2]) +# +# mm.soft_empty_cache() +# gc.collect() +# time.sleep(0.1) +# +# return noise_latents_full + + +# class WarpedSamplerCustomBatch: +# @classmethod +# def INPUT_TYPES(s): +# return {"required": +# { +# "positive": ("STRING", ), +# "negative": ("STRING", {"default": None}), +# "clip": ("CLIP", ), +# "preferred_frame_count": ("INT", {"default": 81, "min": 17, "max": nodes.MAX_RESOLUTION, "step": 4}), +# "preferred_batch_size": ("INT", {"default": 17, "min": 17, "max": 161, "step": 4}), +# "use_batch_size": (["next_lowest", "next_highest", "closest", "exact"], {"default": "next_lowest", "tooltip": "Number of frames generated may be impacted by choice."}), +# "vae": ("VAE", ), +# "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), +# "guider": ("GUIDER", ), +# "sampler": ("SAMPLER", ), +# "sigmas": ("SIGMAS", ), +# "dec_tile_size": ("INT", {"default": 256, "min": 64, "max": 4096, "step": 32}), +# "dec_overlap": ("INT", {"default": 64, "min": 0, "max": 4096, "step": 32}), +# "dec_temporal_size": ("INT", {"default": 64, "min": 8, "max": 4096, "step": 4, "tooltip": "Only used for video VAEs: Amount of frames to decode at a time."}), +# "dec_temporal_overlap": ("INT", {"default": 8, "min": 4, "max": 4096, "step": 4, "tooltip": "Only used for video VAEs: Amount of frames to overlap."}), +# "skip_frames": ("INT", {"default": 0, "min": 0, "max": 32, "step": 4}), +# "noise_scale": ("FLOAT", {"default": 1.00, "min": 0.00, "max": 1.00, "step": 0.01}), +# "mode": (["Hunyuan", "Wan"], {"default": "Hunyuan"}), +# "hunyuan_guidance_type": (["v1 (concat)", "v2 (replace)", "custom"], {"default": "v2 (replace)", "tooltip": "Only used if mode is \"Hunyuan\"."}), +# "hunyuan_image_interleave": ("INT", {"default": 1, "min": 1, "max": 512, "tooltip": "How much the image influences things vs the text prompt. Higher number means more influence from the text prompt."}), +# }, +# "optional": +# {"noise_strength": ("FLOAT", {"default": 1.00, "min": 0.10, "max": 1.00, "step": 0.01}), +# "blend_frames": ("INT", {"default":5, "min":0, "max": 16, "step": 1}), +# "t2v_width": ("INT", {"default":480, "min":256, "max": 1280, "step": 16}), +# "t2v_height": ("INT", {"default":720, "min":256, "max": 1280, "step": 16}), +# "scaling_strength": ("FLOAT", {"default": 1.0}), +# "clip_vision_model": ("CLIP_VISION", ), +# "start_image": ("IMAGE", ), +# "flux_guidance": ("FLOAT", {"default": 3.5, "min": 0.0, "max": 100.0, "step": 0.1}), +# "use_flux_guidance": ("BOOLEAN", {"default": False}), +# "dummy_frames": ("INT", {"default":17, "min":17, "max": 121, "step": 4, "tooltip": "Number of frames to generate in dummy batch."}), +# "gen_dummy": ("BOOLEAN", {"default": False, "tooltip": "For t2v or i2v only. Will generate a dummy batch to obtain a starting image for main generation."}), +# "gen_dummy_only": ("BOOLEAN", {"default": False, "tooltip": "Will generate dummy batch only."}), +# "use_dummy_image": (["first", "middle", "last", "random", "all"], {"default": "last", "tooltip": "Which dummy batch image to start main generation."}), +# "output_latents": ("BOOLEAN", {"default": False}), +# "verbose_messaging": ("BOOLEAN", {"default": False}), +# "i2v_sigmas": ("SIGMAS", {"default": None}), +# "i2v_model": ("MODEL", {"default": None}), +# "temp_cond": ("CONDITIONING", ), +# "temp_latent": ("LATENT", ), +# } +# } +# +# RETURN_TYPES = ("IMAGE", "LATENT", "INT", "STRING", "IMAGE", ) +# RETURN_NAMES = ("images", "latents", "seed", "generation_status", "last_image",) +# +# FUNCTION = "sample" +# +# CATEGORY = "Warped/General/Sampling" +# +# def sample(self, positive, negative, clip, preferred_frame_count, preferred_batch_size, use_batch_size, vae, seed, guider, sampler, sigmas, dec_tile_size, dec_overlap, dec_temporal_size, dec_temporal_overlap, +# skip_frames, noise_scale, mode="Hunyuan", hunyuan_guidance_type="v2 (replace)", hunyuan_image_interleave=2, noise_strength=1.0, blend_frames=5, t2v_width=480, t2v_height=720, scaling_strength=1.0, +# clip_vision_model=None, start_image=None, flux_guidance=3.5, dummy_frames=17, gen_dummy=False, gen_dummy_only=False, use_dummy_image="last", use_flux_guidance=False, output_latents=False, verbose_messaging=False, +# i2v_sigmas=None, i2v_model=None, temp_cond=None, temp_latent=None): +# self.device = mm.get_torch_device() +# self.offload_device = get_offload_device() +# mm.unload_all_models() +# gc.collect() +# time.sleep(1) +# +# self.positive = positive +# self.negative = negative +# self.clip = clip +# self.use_clip = clip.clone() +# self.preferred_frame_count = preferred_frame_count +# self.preferred_batch_size = preferred_batch_size +# self.vae = vae +# self.seed = seed +# self.sampler = sampler +# self.sigmas = sigmas +# self.i2v_sigmas = i2v_sigmas +# self.noise_scale = noise_scale +# self.mode = mode +# self.dec_tile_size = dec_tile_size +# self.dec_overlap = dec_overlap +# self.dec_temporal_size = dec_temporal_size +# self.dec_temporal_overlap = dec_temporal_overlap +# self.clip_vision = clip_vision_model +# self.noise_strength = noise_strength +# self.blend_frames = blend_frames +# self.t2v_width = t2v_width +# self.t2v_height = t2v_height +# self.dummy_frames = dummy_frames +# self.dummy_latents = int((dummy_frames - 1) // 4) + 1 +# self.gen_dummy = gen_dummy +# self.gen_dummy_only = gen_dummy_only +# self.use_dummy_image = use_dummy_image +# self.skip_frames = skip_frames +# self.verbose_messaging = verbose_messaging +# self.guider = guider +# self.i2v_model = i2v_model +# self.g_output = {} +# +# disable_pbar = not utils.PROGRESS_BAR_ENABLED +# +# self.use_sigmas = self.sigmas +# +# # self.guider = WarpedGuider_Basic() +# # self.guider.set_model(self.model) +# callback = self.setup_callbacks() +# +# self.latent_window_size, self.batch_count, self.truncated_frame_count = self.get_latent_window_size(preferred_batch_size, self.preferred_frame_count, use_batch_size=use_batch_size) +# +# # if not self.start_image is None: +# # start_latent = partial_encode_basic(self.vae, start_image) +# is_i2v = True +# +# if start_image is None: +# start_image = torch.zeros([1, self.t2v_height, self.t2v_width, 3], dtype=torch.float32, device=self.offload_device) +# is_i2v = False +# +# print("start_image Shape: {}".format(start_image.shape)) +# +# self.width = int(start_image.shape[2] // 8) +# self.height = int(start_image.shape[1] // 8) +# +# noise, dummy_noise = self.setup_latent_noise() +# +# output_images = [] +# output_images_latents = None +# interrupted = False +# generation_status = "" +# total_images_generated = 0 +# is_dummy_section = False +# is_first_section = False +# is_last_section = False +# switch_guider = True +# last_image = None +# +# if not self.gen_dummy: +# print("Generating {} Batches".format(self.batch_count)) +# else: +# print("Generating Dummy Batch Plus {} Batches".format(self.batch_count)) +# +# generation_batches = list(range(self.batch_count)) +# +# if self.gen_dummy and (not self.gen_dummy_only): +# generation_batches = [0] + generation_batches +# is_dummy_section = True +# else: +# generation_batches = [0] +# is_dummy_section = True +# +# print("generation_batches: {}".format(generation_batches)) +# +# try: +# # if not self.gen_dummy: +# # i = -1 +# # else: +# # i = -2 +# # is_dummy_section = True +# +# # for batch_number in range(self.batch_count): +# for i, generation_batch in enumerate(generation_batches): +# batch_number = generation_batches[i] +# # batch_key = "{}".format(batch_number) +# +# if is_dummy_section: +# print("Generating Dummy Batch.") +# is_first_section = False +# +# if not self.gen_dummy_only: +# is_last_section = False +# else: +# is_last_section = True +# else: +# print("Generating Batch {}.".format(batch_number)) +# +# is_first_section = batch_number == min(generation_batches) +# is_last_section = batch_number == max(generation_batches) +# +# print("is_first_section: {} | is_last_section: {} | is_dummy_section: {}".format(is_first_section, is_last_section, is_dummy_section)) +# +# mm.soft_empty_cache() +# gc.collect() +# time.sleep(0.1) +# +# image_embeds = None +# +# print("start_image Shape: {}".format(start_image.shape)) +# +# if is_i2v: +# if len(start_image.shape) < 4: +# start_image = start_image.unsqueeze(0) +# +# # print("Performing clip_vision_encode.") +# # image_embeds = self.clip_vision_encode(start_image.clone().detach()) +# +# if not is_dummy_section: +# noise_latents = noise[batch_number] +# else: +# noise_latents = dummy_noise +# +# print("noise_latents Shape: {}".format(noise_latents.shape)) +# +# if mode == "Hunyuan": +# if is_i2v: +# # if (not self.i2v_model is None) and switch_guider: +# # switch_guider = False +# # callback = None +# # self.guider = WarpedGuider_Basic() +# # self.guider.set_model(self.i2v_model) +# # callback = self.setup_callbacks() +# # +# # self.use_clip = self.clip.clone() +# # hunyuan_text_encode(self, clip, clip_vision_output, prompt, image_interleave) +# # temp_cond = self.hunyuan_text_encode(self.use_clip, image_embeds, positive, hunyuan_image_interleave) +# # positive_cond, latents = self.hunyuan_encode(temp_cond, self.vae, start_image.shape[2], start_image.shape[1], self.truncated_frame_count, 1, hunyuan_guidance_type, start_image=start_image) +# +# print("HERE 90210") +# positive_cond = temp_cond +# latents = temp_latent +# +# noise_mask = None +# if "noise_mask" in latents: +# noise_mask = latents["noise_mask"] +# +# latents = latents["samples"] # * scaling_strength +# +# if len(latents.shape) < 5: +# latents = latents.unsqueeze(0) +# +# # latents = latents["samples"] * (scaling_strength * vae_scaling_factor) +# # else: +# # positive_cond = self.do_text_encode(self.use_clip, positive) +# # latents = torch.zeros([1, 16, noise_latents.shape[2], self.height, self.width], dtype=torch.float32, device=self.offload_device) +# # +# # latents = latents * scaling_strength # scaling_strength +# # latents = latents * (scaling_strength * vae_scaling_factor) +# # +# # if use_flux_guidance: +# # positive_cond = self.apply_flux_guidance(positive_cond, flux_guidance) +# +# # self.guider.set_conds(positive_cond) +# # else: +# # positive_cond, negative_cond = wan_text_encode(self.use_clip, positive_text=positive, negative_text=negative) +# # positive_cond, negative_cond, latents = self.wan_encode(positive_cond, negative_cond, self.vae, start_image.shape[3], start_image.shape[2], ((noise_latents.shape[2] - 1) * 4) + 1, 1, start_image=start_image, clip_vision_output=image_embeds) +# +# # self.guider.set_conds(positive_cond, negative_cond) +# +# print("latents Shape: {}".format(latents.shape)) +# +# num_frames = int(((latents.shape[2] - 1) * 4) + 1) +# +# print("\nDecoded Width is {} | Decoded Height is {}".format(int(self.width * 8), int(self.height * 8))) +# +# print("-------------------------------------------------------------------------------------------") +# print("WarpedSamplerCustomBatch: Latents Shape: {} | Noise Latents Shape: {}".format(latents.shape, noise_latents.shape)) +# print("-------------------------------------------------------------------------------------------") +# +# print("Noise Shape: {} | Latents Shape: {}".format(noise_latents.shape, latents.shape)) +# print("WarpedSamplerCustomBatch: Generating {} Frames in {} Latents....".format(num_frames, latents.shape[2])) +# +# samples = self.guider.sample(noise_latents, latents, self.sampler, self.sigmas, denoise_mask=noise_mask, callback=callback, disable_pbar=disable_pbar, seed=self.seed) +# samples = samples.to(mm.intermediate_device()) +# +# mm.unload_all_models() +# mm.soft_empty_cache() +# gc.collect() +# time.sleep(1) +# +# if len(samples.shape) < 5: +# samples = samples.unsqueeze(0) +# +# samples = samples.clone().detach() / scaling_strength # (scaling_strength * vae_scaling_factor) +# +# print("WarpedSamplerCustomBatch: Decoding Latents...") +# decoded_images = self.decode_tiled(samples) +# +# if not is_dummy_section or (is_dummy_section and ((self.use_dummy_image == "all")) or self.gen_dummy_only): +# output_images.append(decoded_images) +# total_images_generated += decoded_images.shape[0] +# +# samples = None +# +# # output_images = self.process_skip_images(output_images, skip_frames) +# print("WarpedSamplerCustomBatch: Decoded Images Shape: {}".format(decoded_images.shape[0])) +# +# # if output_latents: +# # output_images_latents = self.encode_tiled(output_images) +# +# print("WarpedSamplerCustomBatch: Generating {} frames in {} latents...Done.".format(num_frames, latents.shape[2])) +# +# # if not is_i2v: +# # self.guider.model_patcher.model.to(get_offload_device()) +# # # guider.set_i2v_model() +# # +# # if not self.i2v_sigmas is None: +# # self.use_sigmas = self.i2v_sigmas +# +# if not is_last_section: +# print("HERE 9090000") +# decoded_tuple = torch.split(decoded_images, 1, dim=0) +# decoded_split = [item for item in decoded_tuple] +# decoded_images = decoded_images.to(get_offload_device()) +# decoded_images = None +# +# if (not is_dummy_section) or (is_dummy_section and ((self.use_dummy_image == "last") or (self.use_dummy_image == "all"))): +# print("HERE 9090000a") +# start_image = decoded_split[len(decoded_split) - 1].clone().detach() +# elif is_dummy_section: +# print("HERE 9090000b") +# if self.use_dummy_image == "first": +# print("HERE 9090000c") +# start_image = decoded_split[0].clone().detach() +# elif self.use_dummy_image == "middle": +# print("HERE 9090000d") +# start_image = decoded_split[int(len(decoded_split) // 2)].clone().detach() +# else: +# print("HERE 9090000e") +# random.seed(self.seed) +# start_image = dummy_split[random.randrange(0, len(decoded_split) - 1, 1)].clone().detach() +# +# is_i2v = True +# +# if last_image is None: +# last_image = start_image.clone().detach() +# +# print("HERE 9090001") +# +# samples = None +# latents = None +# +# mm.soft_empty_cache() +# gc.collect() +# time.sleep(0.1) +# +# print("*******************************************") +# print("****** WarpedSamplerCustomBatch: Total Images Generated {}".format(total_images_generated)) +# print("*******************************************\n") +# +# generation_status = "****** WarpedSamplerCustomBatch: Total Images Generated {} ******".format(total_images_generated) +# +# interrupted = False +# +# print("HERE 9090002") +# +# if is_dummy_section: +# print("HERE 9090003") +# +# if self.gen_dummy_only: +# print("HERE 9090004") +# break +# +# print("HERE 9090005") +# +# is_dummy_section = False +# +# print("HERE 9090006") +# except mm.InterruptProcessingException as ie: +# interrupted = True +# print(f"\nWarpedSamplerCustomBatch: Processing Interrupted.") +# print("WarpedSamplerCustomBatch: Returning only partial results (if any).\n If zero images generated, a blank yellow image will be returned.\n") +# mm.unload_all_models() +# mm.soft_empty_cache() +# gc.collect() +# time.sleep(1) +# +# generation_status = f"\nWarpedSamplerCustomBatch: Processing Interrupted." +# +# traceback.print_tb(ie.__traceback__, limit=99, file=sys.stdout) +# +# raise mm.InterruptProcessingException(f"WarpedSamplerCustomBatch: Processing Interrupted.") +# +# except BaseException as e: +# print(f"\nWarpedSamplerCustomBatch: Exception During Processing: {str(e)}") +# print("WarpedSamplerCustomBatch: Returning only partial results (if any).\n If zero images generated, a blank red image will be returned.\n") +# mm.unload_all_models() +# mm.soft_empty_cache() +# gc.collect() +# time.sleep(1) +# +# generation_status = f"WarpedSamplerCustomBatch: Exception During Processing: {str(e)}" +# generation_status = "{}{}".format(generation_status, "WarpedSamplerCustomBatch: Returning only partial results (if any).\nIf zero images generated, a blank red image will be returned.") +# +# traceback.print_tb(e.__traceback__, limit=99, file=sys.stdout) +# +# pass +# +# callback = None +# self.guider.model_patcher.model.to(get_offload_device()) +# +# latent = None +# latent_image = None +# noise_mask = None +# samples = None +# +# torch.cuda.empty_cache() +# torch.cuda.ipc_collect() +# gc.collect() +# time.sleep(1) +# +# mm.unload_all_models() +# mm.soft_empty_cache() +# gc.collect() +# time.sleep(1) +# +# image = None +# +# if interrupted and (len(output_images) < 1): +# temp_image = Image.new('RGB', (self.width * 8, self.height * 8), color = 'yellow') +# image = pil2tensorSwap(temp_image) +# final_images = image +# elif (len(output_images) < 1): +# temp_image = Image.new('RGB', (self.width * 8, self.height * 8), color = 'red') +# image = pil2tensorSwap(temp_image) +# final_images = image +# else: +# final_images = self.assemble_final_result(output_images) +# +# if output_images_latents is None: +# output_images_latents = torch.zeros([1, 16, 1, self.height, self.width], dtype=torch.float32, device=self.offload_device) +# +# return (final_images, {"samples": output_images_latents}, self.seed, generation_status, last_image,) +# +# def assemble_final_result(self, image_batches): +# if self.blend_frames < 1: +# resulting_images = None +# for entry in image_batches: +# if not resulting_images is None: +# resulting_images = torch.cat((resulting_images, entry), 0) +# else: +# resulting_images = entry +# +# entry.to(device=self.offload_device) +# entry = None +# else: +# blend_value = 1.0 / self.blend_frames +# i = 0 +# while i < (len(image_batches) - 1): +# alpha_blend_val = blend_value +# blend_count = self.blend_frames +# +# image_batches_tuple = torch.split(image_batches[i], 1, dim=0) +# image_batches_split = [item for item in image_batches_tuple] +# image1 = image_batches_split[len(image_batches_split) - 1] +# # image1 = image_batches_split[len(image_batches_split) - 4] +# image_batches_tuple = None +# image_batches_split = None +# +# image_batches_tuple = torch.split(image_batches[i + 1], 1, dim=0) +# image_batches_split = [item for item in image_batches_tuple] +# image2 = image_batches_split[0] +# image_batches_tuple = None +# image_batches_split = None +# +# image1 = tensor2pilSwap(image1)[0] +# image2 = tensor2pilSwap(image2)[0] +# +# blend_latents = None +# +# while blend_count > 0: +# blended_image = Image.blend(image1, image2, alpha_blend_val) +# temp_latent = pil2tensorSwap(blended_image) +# blended_image = None +# +# if len(temp_latent.shape) < 4: +# temp_latent = temp_latent.unsqueeze(0) +# +# if not blend_latents is None: +# blend_latents = torch.cat((blend_latents, temp_latent), 0) +# else: +# blend_latents = temp_latent +# +# alpha_blend_val += blend_value +# blend_count -= 1 +# +# image_batches_tuple = torch.split(image_batches[i], image_batches[i].shape[0] - 3, dim=0) +# image_batches_split = [item for item in image_batches_tuple] +# image_batches_tuple = None +# +# image_batches[i] = torch.cat((image_batches_split[0], blend_latents), 0) +# blend_latents = None +# image_batches_split = None +# +# self.cleanup(unload_models=False, cleanup_models=False, cleanup_cuda=True) +# +# i += 1 +# +# resulting_images = None +# for entry in image_batches: +# if not resulting_images is None: +# resulting_images = torch.cat((resulting_images, entry), 0) +# else: +# resulting_images = entry +# +# entry.to(device=self.offload_device) +# +# image_batches = None +# +# self.cleanup(unload_models=False, cleanup_models=False, cleanup_cuda=True) +# +# +# print("assemble_final_result: Full decoded images count: {}".format(resulting_images.shape[0])) +# +# if self.skip_frames < 1: +# return resulting_images +# +# skipped_frames = 1 +# +# image_batches_tuple = torch.split(resulting_images, 1, dim=0) +# image_batches_split = [item for item in image_batches_tuple] +# +# resulting_images = None +# +# for image in image_batches_split: +# if skipped_frames <= self.skip_frames: +# skipped_frames += 1 +# continue +# +# if not resulting_images is None: +# resulting_images = torch.cat((resulting_images, image), 0) +# else: +# resulting_images = image +# +# print("assemble_final_result: Final decoded images count: {}".format(resulting_images.shape[0])) +# +# return resulting_images +# +# def cleanup(self, unload_models=False, cleanup_models=False, cleanup_cuda=False): +# if unload_models: +# mm.unload_all_models() +# +# if cleanup_models: +# mm.cleanup_models() +# +# if cleanup_cuda: +# mm.soft_empty_cache() +# +# gc.collect() +# time.sleep(1) +# +# return +# +# def generate_noise(self, input_latent, generator=None): +# latent_image = input_latent["samples"] +# return warped_prepare_noise(latent_image, self.seed, generator=generator) +# +# def setup_latent_noise(self): +# dummy_noise = None +# +# if self.gen_dummy: +# rnd = torch.Generator("cpu").manual_seed(self.seed) +# dummy_noise = torch.randn((1, 16, self.dummy_latents, self.height, self.width), generator=rnd, device=rnd.device).to(device=rnd.device, dtype=torch.float32) +# +# noise = [] +# +# for i in range(self.batch_count): +# rnd = torch.Generator("cpu").manual_seed(self.seed) +# temp_noise = torch.randn((1, 16, self.latent_window_size, self.height, self.width), generator=rnd, device=rnd.device).to(device=rnd.device, dtype=torch.float32) +# +# noise.append(temp_noise) +# temp_noise = None +# +# if self.verbose_messaging: +# if not dummy_noise is None: +# print("\nsetup_latent_noise: dummy_noise Shape: {}".format(dummy_noise.shape)) +# +# i = 0 +# for entry in noise: +# print("setup_latent_noise: Batch: {} | noise Shape: {}".format(i, entry.shape)) +# i += 1 +# +# print("\n") +# +# return noise, dummy_noise +# +# def get_latent_window_size(self, preferred_batch_size, frame_count, use_batch_size="next_lowest"): +# latent_size_factor = 4 +# +# if self.verbose_messaging: +# print("get_latent_window_size: preferred_batch_size: {}".format(preferred_batch_size)) +# print("get_latent_window_size: frame_count: {}".format(frame_count)) +# print("get_latent_window_size: use_batch_size: {}".format(use_batch_size)) +# +# num_frames = int(((frame_count - 1) // 4) * 4) + 1 +# +# if num_frames != frame_count: +# print(f"Truncating video from {frame_count} to {num_frames} an because odd number of frames is not allowed.") +# +# if ((num_frames - 1) % (preferred_batch_size - 1)) == 0: +# print("(1) latent_window_size set to: {}".format(self.decoded_to_encoded_length(preferred_batch_size))) +# print("(1) batch_count set to: {}".format(int((num_frames - 1) / (preferred_batch_size - 1)))) +# return self.decoded_to_encoded_length(preferred_batch_size), int((num_frames - 1) / (preferred_batch_size - 1)), num_frames +# +# if use_batch_size == "exact": +# num_frames_final = int(((num_frames - 1) // (preferred_batch_size - 1)) + 1) * (preferred_batch_size - 1) +# +# if num_frames_final != frame_count: +# print(f"Truncating video from {num_frames} to {num_frames_final} frames for preferred_batch_size compatibility.") +# +# print("(2) latent_window_size set to: {}".format(self.decoded_to_encoded_length(preferred_batch_size + 1))) +# print("(2) batch_count set to: {}".format(int((num_frames_final - 1) / (preferred_batch_size - 1)))) +# return self.decoded_to_encoded_length(preferred_batch_size), int((num_frames_final - 1) / (preferred_batch_size - 1)), num_frames_final +# +# next_lowest_found = False +# next_highest_found = False +# next_lowest = preferred_batch_size - 1 +# next_highest = preferred_batch_size - 1 +# +# if self.verbose_messaging: +# print("get_latent_window_size: Next Lowest Initialized To: {}".format(next_lowest)) +# print("get_latent_window_size: Next Highest Initialized To: {}".format(next_highest)) +# +# num_frames_final = int(((num_frames - 1) // 4) * 4) + 1 +# +# if num_frames != num_frames_final: +# print(f"Truncating video from {num_frames} to {num_frames_final} frames for latent_window_size compatibility.") +# +# if (use_batch_size == "closest") or (use_batch_size == "next_lowest"): +# while next_lowest >= 12: +# next_lowest -= 4 +# +# if (int((num_frames_final - 1) // 4) % next_lowest) == 0: +# next_lowest_found = True +# break +# +# next_lowest += 1 +# +# if next_lowest_found and (use_batch_size == "next_lowest"): +# print("(3) latent_window_size set to: {}".format(self.decoded_to_encoded_length(next_lowest + 1))) +# print("(3) batch_count set to: {}".format(int((num_frames_final - 1) / next_lowest))) +# return self.decoded_to_encoded_length(next_lowest + 1), int((num_frames_final - 1) / next_lowest), num_frames_final +# +# while next_highest <= 156: +# next_highest += 4 +# +# if (int((num_frames_final - 1) // 4) % next_highest) == 0: +# next_highest_found = True +# break +# +# if next_highest_found and (use_batch_size == "next_highest"): +# print("(4) latent_window_size set to: {}".format(self.decoded_to_encoded_length(next_highest + 1))) +# print("(4) batch_count set to: {}".format(int((num_frames_final - 1) / next_highest))) +# return self.decoded_to_encoded_length(next_highest + 1), int((num_frames_final - 1) / next_highest), num_frames_final +# +# if next_highest_found and next_lowest_found: +# if (preferred_batch_size - next_lowest) <= (next_highest - preferred_batch_size): +# print("(5) latent_window_size set to: {}".format(self.decoded_to_encoded_length(next_lowest + 1))) +# print("(5) batch_count set to: {}".format(int((num_frames_final - 1) / next_lowest))) +# return self.decoded_to_encoded_length(next_lowest + 1), int((num_frames_final - 1) / next_lowest), num_frames_final +# elif (next_highest - preferred_batch_size) < (preferred_batch_size - next_lowest): +# print("(6) latent_window_size set to: {}".format(self.decoded_to_encoded_length(next_highest + 1))) +# print("(6) batch_count set to: {}".format(int((num_frames_final - 1) / next_highest))) +# return self.decoded_to_encoded_length(next_highest + 1), int((num_frames_final - 1) / next_highest), num_frames_final +# +# print("Unable to find a compatible latent_window_size for number of frames = {} and preferred_batch_size = {}.".format(frame_count, preferred_batch_size)) +# print("Recalculating Number Of Frames Based On preferred_batch_size of: {}".format(preferred_batch_size)) +# +# return self.calculate_new_number_of_frames(preferred_batch_size, (((frame_count - 1) // 4) * 4) + 1, use_batch_size) +# +# def calculate_new_number_of_frames(self, preferred_batch_size, frame_count, use_batch_size): +# working_batch_size = preferred_batch_size - 1 +# working_frame_count = frame_count - 1 +# +# next_lowest = next_highest = working_frame_count +# next_lowest_found = False +# next_highest_found = False +# +# while next_lowest > 37: +# next_lowest -= 4 +# +# if int(next_lowest % working_batch_size) == 0: +# next_lowest_found = True +# break +# +# if next_lowest_found and (use_batch_size == "next_lowest"): +# return self.decoded_to_encoded_length(preferred_batch_size), int(next_lowest // working_batch_size), next_lowest + 1 +# +# while next_highest < 999997: +# next_highest += 4 +# +# if int(next_highest % working_batch_size) == 0: +# next_highest_found = True +# break +# +# if next_highest_found and (use_batch_size == "next_highest"): +# return self.decoded_to_encoded_length(preferred_batch_size), int(next_highest // working_batch_size), next_highest + 1 +# +# if next_lowest_found and next_highest_found: +# if (working_frame_count - next_lowest) <= (next_highest - working_frame_count): +# return self.decoded_to_encoded_length(preferred_batch_size), int(next_lowest // working_batch_size), next_lowest + 1 +# +# return self.decoded_to_encoded_length(preferred_batch_size), int(next_highest // working_batch_size), next_highest + 1 +# +# if next_lowest_found: +# return self.decoded_to_encoded_length(preferred_batch_size), int(next_lowest // working_batch_size), next_lowest + 1 +# +# if next_highest_found: +# return self.decoded_to_encoded_length(preferred_batch_size), int(next_highest // working_batch_size), next_highest + 1 +# +# raise ValueError("Unable to find a compatible latent_window_size for number of frames = {} and preferred_batch_size = {}.".format(frame_count, preferred_batch_size)) +# +# def clip_vision_encode(self, image, crop="center"): +# crop_image = False +# if crop != "center": +# crop_image = False +# output = self.clip_vision.encode_image(image, crop=crop_image) +# return output +# +# def encoded_to_decoded_length(self, latent_length): +# if latent_length <= 0: +# return 0 +# +# result_length = ((latent_length - 1) * 4) + 1 +# +# return result_length +# +# def decoded_to_encoded_length(self, image_length): +# if image_length <= 0: +# return 0 +# +# result_length = int(((image_length - 1) / 4) + 1) +# +# return result_length +# +# def apply_flux_guidance(self, conditioning, guidance): +# c = node_helpers.conditioning_set_values(conditioning, {"guidance": guidance}) +# return c +# +# def hunyuan_text_encode(self, clip, clip_vision_output, prompt, image_interleave): +# tokens = clip.tokenize(prompt, llama_template=PROMPT_TEMPLATE_ENCODE_VIDEO_I2V, image_embeds=clip_vision_output.mm_projected, image_interleave=image_interleave) +# return clip.encode_from_tokens_scheduled(tokens) +# +# def hunyuan_encode(self, positive, vae, width, height, length, batch_size, guidance_type, start_image=None): +# latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device()) +# out_latent = {} +# +# if start_image is not None: +# start_image = comfy.utils.common_upscale(start_image[:length, :, :, :3].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1) +# +# concat_latent_image = vae.encode(start_image) +# mask = torch.ones((1, 1, latent.shape[2], concat_latent_image.shape[-2], concat_latent_image.shape[-1]), device=start_image.device, dtype=start_image.dtype) +# mask[:, :, :((start_image.shape[0] - 1) // 4) + 1] = 0.0 +# +# if guidance_type == "v1 (concat)": +# cond = {"concat_latent_image": concat_latent_image, "concat_mask": mask} +# elif guidance_type == "v2 (replace)": +# cond = {'guiding_frame_index': 0} +# latent[:, :, :concat_latent_image.shape[2]] = concat_latent_image +# out_latent["noise_mask"] = mask +# elif guidance_type == "custom": +# cond = {"ref_latent": concat_latent_image} +# +# positive = node_helpers.conditioning_set_values(positive, cond) +# +# out_latent["samples"] = latent +# return positive, out_latent +# +# def wan_text_encode(self, clip, positive_text="", negative_text=""): +# print("wan_text_encode: Loading clip model to device: {}".format(clip.patcher.load_device)) +# clip.patcher.model.to(device=clip.patcher.load_device) +# +# print("wan_text_encode: Encoding Prompts...") +# positive_conditioning = self.do_text_encode(clip, positive_text) +# negative_conditioning = self.do_text_encode(clip, negative_text) +# print("wan_text_encode: Encoding Prompts...Done.") +# +# print("wan_text_encode: Unloading clip model to device: {}".format(get_offload_device())) +# clip.patcher.model.to(device=get_offload_device()) +# +# if torch.cuda.is_available(): +# torch.cuda.empty_cache() +# +# gc.collect() +# time.sleep(1) +# +# return positive_conditioning, negative_conditioning +# +# def encode(self, clip, text): +# if clip is None: +# raise RuntimeError("ERROR: clip input is invalid: None\n\nIf the clip is from a checkpoint loader node your checkpoint does not contain a valid clip or text encoder model.") +# tokens = clip.tokenize(text) +# return clip.encode_from_tokens_scheduled(tokens) +# +# def do_text_encode(self, clip, text): +# +# if clip is None: +# raise RuntimeError("ERROR: clip input is invalid: None\n\nIf the clip is from a checkpoint loader node your checkpoint does not contain a valid clip or text encoder model.") +# +# tokens = clip.tokenize(text) +# return_encoding = clip.encode_from_tokens_scheduled(tokens) +# +# return return_encoding +# +# def wan_encode(self, positive, negative, vae, width, height, length, batch_size, start_image=None, clip_vision_output=None): +# latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device()) +# if start_image is not None: +# start_image = comfy.utils.common_upscale(start_image[:length].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1) +# image = torch.mul(torch.ones((length, height, width, start_image.shape[-1]), device=start_image.device, dtype=start_image.dtype), 0.5) +# image[:start_image.shape[0]] = start_image +# +# concat_latent_image = vae.encode(image[:, :, :, :3]) +# +# mask = torch.ones((1, 1, latent.shape[2], concat_latent_image.shape[-2], concat_latent_image.shape[-1]), device=start_image.device, dtype=start_image.dtype) +# mask[:, :, :((start_image.shape[0] - 1) // 4) + 1] = 0.0 +# +# positive = node_helpers.conditioning_set_values(positive, {"concat_latent_image": concat_latent_image, "concat_mask": mask}) +# negative = node_helpers.conditioning_set_values(negative, {"concat_latent_image": concat_latent_image, "concat_mask": mask}) +# +# if clip_vision_output is not None: +# positive = node_helpers.conditioning_set_values(positive, {"clip_vision_output": clip_vision_output}) +# negative = node_helpers.conditioning_set_values(negative, {"clip_vision_output": clip_vision_output}) +# +# out_latent = {} +# out_latent["samples"] = latent +# return positive, negative, out_latent +# +# def process_skip_images(self, frames, skip_count): +# if len(frames.shape) < 4: +# frames = frames.unsqueeze(0) +# +# num_frames = frames.shape[0] +# +# image_batches_tuple = torch.split(frames, 1, dim=0) +# image_batches_split = [item for item in image_batches_tuple] +# +# new_video = None +# i = 0 +# +# while i < len(image_batches_split): +# if i < skip_count: +# i += 1 +# continue +# +# if not new_video is None: +# new_video = torch.cat((new_video, image_batches_split[i]), 0) +# else: +# new_video = image_batches_split[i] +# +# i += 1 +# +# return new_video +# +# def get_blank_image(self, length=1): +# new_image = torch.zeros([length, self.height, self.width, 3], dtype=torch.float32, device=self.offload_device) +# return new_image +# +# def get_new_noise(self, length): +# new_noise = torch.zeros([length, self.height, self.width, 3], dtype=torch.float32, device=self.offload_device) +# +# new_noise = self.encode_tiled(new_noise) +# +# new_noise = comfy.sample.fix_empty_latent_channels(self.guider.model_patcher, new_noise) +# +# if len(new_noise) < 5: +# new_latent = new_noise.unsqueeze(0) +# +# new_noise = self.generate_noise({"samples": new_noise}) +# +# return new_noise +# +# def pad_noise(self, latent, num_frames=1): +# pad_frames = torch.zeros([1, 16, num_frames, self.height, self.width], dtype=torch.float32, device=self.offload_device) +# pad_frames = torch.cat((latent, pad_frames), 2) +# +# return pad_frames +# +# def setup_callbacks(self): +# callback = latent_preview.prepare_callback(self.guider.model_patcher, self.sigmas.shape[-1] - 1, self.g_output) +# +# return callback +# +# def decode_tiled(self, latents): +# decoded_data = partial_decode_tiled(self.vae, latents, self.dec_tile_size, self.dec_overlap, self.dec_temporal_size, self.dec_temporal_overlap) +# +# if len(decoded_data.shape) < 4: +# decoded_data.unsqueeze(0) +# +# return decoded_data +# +# def initialize_noise(self, frame_count, clear_cache=True): +# noise_latents_full = torch.zeros([1, 16, int(frame_count), self.height, self.width], dtype=torch.float32, device=self.offload_device) +# print("WarpedSamplerCustomBatch: Encoded noise_latents_full Shape: {}".format(noise_latents_full.shape)) +# +# if Decimal(self.noise_scale).compare(Decimal(0.00)) != 0: +# noise_latents_full = warped_prepare_noise(noise_latents_full, self.seed) +# print("WarpedSamplerCustomBatch: noise_latents_full Shape: {}".format(noise_latents_full.shape)) +# +# noise_latents_full = torch.mul(noise_latents_full, self.noise_scale) +# +# if len(noise_latents_full.shape) < 5: +# noise_latents_full.unsqueeze(0) +# +# noise_latents_full = noise_latents_full.to(dtype=torch.float32, device=self.offload_device) +# +# if clear_cache: +# mm.soft_empty_cache() +# gc.collect() +# time.sleep(0.1) +# +# return noise_latents_full +# +# def initialize_frames(self, latents): +# if len(latents.shape) < 5: +# latents = latents.unsqueeze(0) +# +# print("WarpedSamplerCustomBatch: Encoded latents_full Shape: {}".format(latents.shape)) +# latents_full = latents.clone().detach() +# latents_full = latents_full.to(dtype=torch.float32, device=self.offload_device) +# +# noise_latents_full = self.initialize_noise(latents_full.shape[2]) +# +# mm.soft_empty_cache() +# gc.collect() +# time.sleep(0.1) +# +# return noise_latents_full + def sd3_patch(model, shift, multiplier=1000): m = model.clone() @@ -3061,6 +5879,8 @@ class WarpedBundleAllVideoImages: fps = round(fps) print('WarpedBundleAllVideoImages: fps = %d' % fps) + print("WarpedBundleAllVideoImages: Width: {} | Height: {}".format(width, height)) + batched_images = None num_frames = 0 last_image = None @@ -3280,6 +6100,7 @@ class WarpedCFGGuider: self.original_conds = {} self.cfg = 1.0 self.inner_executor = None + self.i2v_model = None def set_conds(self, positive, negative): self.inner_set_conds({"positive": positive, "negative": negative}) @@ -3345,6 +6166,22 @@ class WarpedCFGGuider: return + def set_models(self, t2v_model, i2v_model): + self.model_patcher = t2v_model + self.model_options = t2v_model.model_options + self.inner_executor = None + self.i2v_model = i2v_model + + return + + def set_i2v_model(self): + if not self.i2v_model is None: + self.model_patcher = self.i2v_model + self.model_options = self.i2v_model.model_options + self.inner_executor = None + + return + def sample(self, noise, latent_image, sampler, sigmas, denoise_mask=None, callback=None, disable_pbar=False, seed=None): if sigmas.shape[-1] == 0: return latent_image @@ -3379,26 +6216,57 @@ class WarpedCFGGuider: return output class WarpedGuider_Basic(WarpedCFGGuider): - def set_conds(self, positive): + def set_conds_single(self, positive): self.inner_set_conds({"positive": positive}) + def set_conds_both(self, positive, negative): + self.inner_set_conds({"positive": positive, "negative": negative}) + class WarpedBasicGuider: + def __init__(self): + self.guider = None + @classmethod def INPUT_TYPES(s): return {"required": - {"conditioning": ("CONDITIONING", ), + {"model": ("MODEL", ), } } - RETURN_TYPES = ("GUIDER",) + RETURN_TYPES = ("WARPED_GUIDER",) + RETURN_NAMES = ("guider",) FUNCTION = "get_guider" CATEGORY = "Warped/General/Sampling/Guiders" - def get_guider(self, conditioning): - guider = WarpedGuider_Basic() - guider.set_conds(conditioning) - return (guider,) + def get_guider(self, model): + self.guider = WarpedGuider_Basic() + self.guider.set_model(model) + return (self.guider,) + +class WarpedDualGuider: + def __init__(self): + self.guider = None + + @classmethod + def INPUT_TYPES(s): + return {"required": + {"t2v_model": ("MODEL", ), + "i2v_model": ("MODEL", ), + } + } + + RETURN_TYPES = ("WARPED_GUIDER",) + RETURN_NAMES = ("guider",) + + FUNCTION = "get_guider" + CATEGORY = "Warped/General/Sampling/Guiders" + + def get_guider(self, t2v_model, i2v_model): + self.guider = WarpedGuider_Basic() + self.guider.set_models(t2v_model, i2v_model) + + return (self.guider,) class WarpedUpscaleWithModel: rescale_methods = ["nearest-exact", "bilinear", "area", "bicubic", "lanczos"] @@ -5559,8 +8427,6 @@ class WarpedHunyuanLoraConvert: return {"ui": {"tags": [save_message]}} -vae_scaling_factor = 0.476986 - class WarpedFramepackSampler: def __init__(self): self.clip_vision = None @@ -7316,6 +10182,107 @@ class WarpedDualEncoder(ComfyNodeABC): return return_encoding +# class CLIPLoader: +# @classmethod +# def INPUT_TYPES(s): +# return {"required": { "clip_name": (folder_paths.get_filename_list("text_encoders"), ), +# "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace"], ), +# }, +# "optional": { +# "device": (["default", "cpu"], {"advanced": True}), +# }} +# RETURN_TYPES = ("CLIP",) +# FUNCTION = "load_clip" +# +# CATEGORY = "advanced/loaders" +# +# DESCRIPTION = "[Recipes]\n\nstable_diffusion: clip-l\nstable_cascade: clip-g\nsd3: t5 xxl/ clip-g / clip-l\nstable_audio: t5 base\nmochi: t5 xxl\ncosmos: old t5 xxl\nlumina2: gemma 2 2B\nwan: umt5 xxl\n hidream: llama-3.1 (Recommend) or t5" +# +# def load_clip(self, clip_name, type="stable_diffusion", device="default"): +# clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION) +# +# model_options = {} +# if device == "cpu": +# model_options["load_device"] = model_options["offload_device"] = torch.device("cpu") +# +# clip_path = folder_paths.get_full_path_or_raise("text_encoders", clip_name) +# clip = comfy.sd.load_clip(ckpt_paths=[clip_path], embedding_directory=folder_paths.get_folder_paths("embeddings"), clip_type=clip_type, model_options=model_options) +# return (clip,) + +class WarpedCLIPLoader: + @classmethod + def INPUT_TYPES(s): + return {"required": { "clip_name": (folder_paths.get_filename_list("text_encoders"), ), + "clip_type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace"], ), + }, + "optional": { + "device": (get_available_devices(), {"default": "cpu"}), + }} + RETURN_TYPES = ("CLIP",) + FUNCTION = "load_clip" + + CATEGORY = "Warped/General/Loaders" + + DESCRIPTION = "[Recipes]\n\nstable_diffusion: clip-l\nstable_cascade: clip-g\nsd3: t5 xxl/ clip-g / clip-l\nstable_audio: t5 base\nmochi: t5 xxl\ncosmos: old t5 xxl\nlumina2: gemma 2 2B\nwan: umt5 xxl\n hidream: llama-3.1 (Recommend) or t5" + + def load_clip(self, clip_name, clip_type="stable_diffusion", device="cpu"): + clip_type_attr = getattr(comfy.sd.CLIPType, clip_type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION) + + print("WarpedCLIPLoader: Clip Type: {} | {}".format(clip_type, clip_type_attr)) + + model_options = {} + if device == "cpu": + model_options["load_device"] = model_options["offload_device"] = torch.device("cpu") + else: + model_options["load_device"] = model_options["offload_device"] = torch.device("cpu") + + if torch.cuda.is_available(): + if device == "cuda": + model_options["load_device"] = torch.cuda.current_device() + else: + temp_device = device.split(':') + device_number = int(temp_device[len(temp_device) - 1]) + model_options["load_device"] = torch.device(device_number) + + print("WarpedCLIPLoader: {}".format(model_options)) + + clip_path = folder_paths.get_full_path_or_raise("text_encoders", clip_name) + clip = self.sd_load_clip(ckpt_paths=[clip_path], embedding_directory=folder_paths.get_folder_paths("embeddings"), clip_type=clip_type_attr, model_options=model_options) + return (clip,) + + def sd_load_clip(self, ckpt_paths, embedding_directory=None, clip_type=comfy.sd.CLIPType.STABLE_DIFFUSION, model_options={"offload_device": get_offload_device()}): + checkpoint_temp = [] + for p in ckpt_paths: + print("Reading: {}".format(p)) + with open(p, "rb") as file: + content = file.read() + + checkpoint_temp.append(content) + + clip_data = [] + + for p in checkpoint_temp: + print("Loading Clip...") + clip_data.append(warped_load_torch_file(p)) + print("Loading Clip...Done") + + return_clip = comfy.sd.load_text_encoder_state_dicts(clip_data, embedding_directory=embedding_directory, clip_type=clip_type, model_options=model_options) + + checkpoint_temp = None + clip_data = None + + return return_clip + + def encode(self, clip, text): + + if clip is None: + raise RuntimeError("ERROR: clip input is invalid: None\n\nIf the clip is from a checkpoint loader node your checkpoint does not contain a valid clip or text encoder model.") + + tokens = clip.tokenize(text) + return_encoding = clip.encode_from_tokens_scheduled(tokens) + + return return_encoding + class WarpedDualCLIPLoader: @classmethod def INPUT_TYPES(self): @@ -7333,7 +10300,7 @@ class WarpedDualCLIPLoader: DESCRIPTION = "[Recipes]\n\nsdxl: clip-l, clip-g\nsd3: clip-l, clip-g / clip-l, t5 / clip-g, t5\nflux: clip-l, t5" - def load_clip(self, clip_name1, clip_name2, type, device="default"): + def load_clip(self, clip_name1, clip_name2, type, device="cpu"): clip_path1 = folder_paths.get_full_path_or_raise("text_encoders", clip_name1) clip_path2 = folder_paths.get_full_path_or_raise("text_encoders", clip_name2) if type == "sdxl": @@ -7359,7 +10326,7 @@ class WarpedDualCLIPLoader: device_number = int(temp_device[len(temp_device) - 1]) model_options["load_device"] = torch.device(device_number) - print("WarpedDualCLIPLoader: {}".format(model_options)) + print("WarpedDualCLIPLoader: {}".format(model_options)) clip = self.sd_load_clip(ckpt_paths=[clip_path1, clip_path2], embedding_directory=folder_paths.get_folder_paths("embeddings"), clip_type=clip_type, model_options=model_options) @@ -7768,7 +10735,7 @@ class WarpedNumericalConversion: def int_to_number(self, int_value=None, float_value=None, number_value=None): value = None - if (int_value == None) and (float_value == None) and (number == None): + if (int_value == None) and (float_value == None) and (number_value == None): raise ValueError("WarpedNumericalConversion: All inputs are None. Nothing to convert.") i = 0 diff --git a/pig.py b/pig.py new file mode 100644 index 0000000..0acd317 --- /dev/null +++ b/pig.py @@ -0,0 +1,999 @@ +import comfy.sd +import comfy.ops +import comfy.utils +import comfy.model_patcher +import comfy.model_management +import torch, numpy, os, json, logging, collections, nodes, folder_paths +from safetensors.torch import load_file, save_file +from typing import Dict, Tuple +from tqdm import tqdm as loading +from .gguf_connector import reader as gr +from .gguf_connector.writer import GGUFWriter, GGMLQuantizationType +from .gguf_connector.const import GGML_QUANT_VERSION, LlamaFileType +from .gguf_connector.quant import quantize, dequantize, QuantError +from .gguf_connector.quant2c import dequantize_tensor, is_quantized, is_torch_compatible +from .gguf_connector.tkn import get_field, tokenizer_builder +pig = os.path.join(os.path.dirname(__file__), 'version.json') +with open(pig, 'r') as file: + data = json.load(file) +arrays = {} +for key, value in data[0].items(): + arrays[key] = value +class GGUFModelPatcher(comfy.model_patcher.ModelPatcher): + patch_on_device = False + def patch_weight_to_device(self, key, device_to=None, inplace_update=False): + if key not in self.patches: + return + weight = comfy.utils.get_attr(self.model, key) + try: + from comfy.lora import calculate_weight + except Exception: + calculate_weight = self.calculate_weight + patches = self.patches[key] + if is_quantized(weight): + out_weight = weight.to(device_to) + patches = load_patch_to_device(patches, self.load_device if + self.patch_on_device else self.offload_device) + out_weight.patches = [(calculate_weight, patches, key)] + else: + inplace_update = self.weight_inplace_update or inplace_update + if key not in self.backup: + self.backup[key] = collections.namedtuple('Dimension', [ + 'weight', 'inplace_update'])(weight.to(device=self. + offload_device, copy=inplace_update), inplace_update) + if device_to is not None: + temp_weight = comfy.model_management.cast_to_device(weight, + device_to, torch.float32, copy=True) + else: + temp_weight = weight.to(torch.float32, copy=True) + out_weight = calculate_weight(patches, temp_weight, key) + out_weight = comfy.float.stochastic_rounding(out_weight, weight + .dtype) + if inplace_update: + comfy.utils.copy_to_param(self.model, key, out_weight) + else: + comfy.utils.set_attr_param(self.model, key, out_weight) + def unpatch_model(self, device_to=None, unpatch_weights=True): + if unpatch_weights: + for p in self.model.parameters(): + if is_torch_compatible(p): + continue + patches = getattr(p, 'patches', []) + if len(patches) > 0: + p.patches = [] + return super().unpatch_model(device_to=device_to, unpatch_weights= + unpatch_weights) + mmap_released = False + def load(self, *args, force_patch_weights=False, **kwargs): + super().load(*args, force_patch_weights=True, **kwargs) + if not self.mmap_released: + linked = [] + if kwargs.get('lowvram_model_memory', 0) > 0: + for n, m in self.model.named_modules(): + if hasattr(m, 'weight'): + device = getattr(m.weight, 'device', None) + if device == self.offload_device: + linked.append((n, m)) + continue + if hasattr(m, 'bias'): + device = getattr(m.bias, 'device', None) + if device == self.offload_device: + linked.append((n, m)) + continue + if linked: + print(f'Attempting to release mmap ({len(linked)})') + for n, m in linked: + m.to(self.load_device).to(self.offload_device) + self.mmap_released = True + def clone(self, *args, **kwargs): + src_cls = self.__class__ + self.__class__ = GGUFModelPatcher + n = super().clone(*args, **kwargs) + n.__class__ = GGUFModelPatcher + self.__class__ = src_cls + n.patch_on_device = getattr(self, 'patch_on_device', False) + return n +class GGMLTensor(torch.Tensor): + def __init__(self, *args, tensor_type, tensor_shape, patches=[], **kwargs): + super().__init__() + self.tensor_type = tensor_type + self.tensor_shape = tensor_shape + self.patches = patches + def __new__(cls, *args, tensor_type, tensor_shape, patches=[], **kwargs): + return super().__new__(cls, *args, **kwargs) + def to(self, *args, **kwargs): + new = super().to(*args, **kwargs) + new.tensor_type = getattr(self, 'tensor_type', None) + new.tensor_shape = getattr(self, 'tensor_shape', new.data.shape) + new.patches = getattr(self, 'patches', []).copy() + return new + def clone(self, *args, **kwargs): + return self + def detach(self, *args, **kwargs): + return self + def copy_(self, *args, **kwargs): + try: + return super().copy_(*args, **kwargs) + except Exception as e: + print(f"ignoring 'copy_' on tensor: {e}") + def empty_(self, size, *args, **kwargs): + new_tensor = super().empty_(size, *args, **kwargs) + return GGMLTensor(new_tensor, tensor_type=getattr(self, + 'tensor_type', None), tensor_shape=size, patches=getattr(self, + 'patches', []).copy()) + @property + def shape(self): + if not hasattr(self, 'tensor_shape'): + self.tensor_shape = self.size() + return self.tensor_shape +if hasattr(torch, 'compiler') and hasattr(torch.compiler, 'disable'): + torch_compiler_disable = torch.compiler.disable +else: + def torch_compiler_disable(*args, **kwargs): + def noop(x): + return x + return noop +class GGMLLayer(torch.nn.Module): + comfy_cast_weights = True + dequant_dtype = None + patch_dtype = None + largest_layer = False + torch_compatible_tensor_types = {None, gr.GGMLQuantizationType.F32, gr. + GGMLQuantizationType.F16} + def is_ggml_quantized(self, *, weight=None, bias=None): + if weight is None: + weight = self.weight + if bias is None: + bias = self.bias + return is_quantized(weight) or is_quantized(bias) + def _load_from_state_dict(self, state_dict, prefix, *args, **kwargs): + weight, bias = state_dict.get(f'{prefix}weight'), state_dict.get( + f'{prefix}bias') + if self.is_ggml_quantized(weight=weight, bias=bias) or isinstance(self, + torch.nn.Linear): + return self.ggml_load_from_state_dict(state_dict, prefix, *args, + **kwargs) + return super()._load_from_state_dict(state_dict, prefix, *args, ** + kwargs) + def ggml_load_from_state_dict(self, state_dict, prefix, local_metadata, + strict, missing_keys, unexpected_keys, error_msgs): + prefix_len = len(prefix) + for k, v in state_dict.items(): + if k[prefix_len:] == 'weight': + self.weight = torch.nn.Parameter(v, requires_grad=False) + elif k[prefix_len:] == 'bias' and v is not None: + self.bias = torch.nn.Parameter(v, requires_grad=False) + else: + missing_keys.append(k) + if self.weight is None and isinstance(self, torch.nn.Linear): + v = torch.zeros(self.in_features, self.out_features) + self.weight = torch.nn.Parameter(v, requires_grad=False) + missing_keys.append(prefix + 'weight') + if getattr(self.weight, 'is_largest_weight', False): + self.largest_layer = True + def _save_to_state_dict(self, *args, **kwargs): + if self.is_ggml_quantized(): + return self.ggml_save_to_state_dict(*args, **kwargs) + return super()._save_to_state_dict(*args, **kwargs) + def ggml_save_to_state_dict(self, destination, prefix, keep_vars): + weight = torch.zeros_like(self.weight, device=torch.device('meta')) + destination[prefix + 'weight'] = weight + if self.bias is not None: + bias = torch.zeros_like(self.bias, device=torch.device('meta')) + destination[prefix + 'bias'] = bias + if self.largest_layer: + shape = getattr(self.weight, 'tensor_shape', self.weight.shape) + dtype = self.dequant_dtype or torch.float16 + temp = torch.empty(*shape, device=torch.device('meta'), dtype=dtype + ) + destination[prefix + 'temp.weight'] = temp + return + destination[prefix + 'weight'] = self.get_weight(self.weight) + if bias is not None: + destination[prefix + 'bias'] = self.get_weight(self.bias) + def get_weight(self, tensor, dtype): + if tensor is None: + return + patch_list = [] + device = tensor.device + for function, patches, key in getattr(tensor, 'patches', []): + patch_list += load_patch_to_device(patches, device) + weight = dequantize_tensor(tensor, dtype, self.dequant_dtype) + if isinstance(weight, GGMLTensor): + weight = torch.Tensor(weight) + if patch_list: + if self.patch_dtype is None: + weight = function(patch_list, weight, key) + else: + patch_dtype = (dtype if self.patch_dtype == 'target' else + self.patch_dtype) + weight = function(patch_list, weight, key, patch_dtype) + return weight + @torch_compiler_disable() + def cast_bias_weight(s, input=None, dtype=None, device=None, bias_dtype + =None): + if input is not None: + if dtype is None: + dtype = getattr(input, 'dtype', torch.float32) + if bias_dtype is None: + bias_dtype = dtype + if device is None: + device = input.device + bias = None + non_blocking = comfy.model_management.device_supports_non_blocking( + device) + if s.bias is not None: + bias = s.get_weight(s.bias.to(device), dtype) + bias = comfy.ops.cast_to(bias, bias_dtype, device, non_blocking + =non_blocking, copy=False) + weight = s.get_weight(s.weight.to(device), dtype) + weight = comfy.ops.cast_to(weight, dtype, device, non_blocking= + non_blocking, copy=False) + return weight, bias + def forward_comfy_cast_weights(self, input, *args, **kwargs): + if self.is_ggml_quantized(): + out = self.forward_ggml_cast_weights(input, *args, **kwargs) + else: + out = super().forward_comfy_cast_weights(input, *args, **kwargs) + if isinstance(out, GGMLTensor): + out = torch.Tensor(out) + return out + def forward_ggml_cast_weights(self, input): + raise NotImplementedError +class GGMLOps(comfy.ops.manual_cast): + class Linear(GGMLLayer, comfy.ops.manual_cast.Linear): + def __init__(self, in_features, out_features, bias=True, device= + None, dtype=None): + torch.nn.Module.__init__(self) + self.in_features = in_features + self.out_features = out_features + self.weight = None + self.bias = None + def forward_ggml_cast_weights(self, input): + weight, bias = self.cast_bias_weight(input) + return torch.nn.functional.linear(input, weight, bias) + class Conv2d(GGMLLayer, comfy.ops.manual_cast.Conv2d): + def forward_ggml_cast_weights(self, input): + weight, bias = self.cast_bias_weight(input) + return self._conv_forward(input, weight, bias) + class Embedding(GGMLLayer, comfy.ops.manual_cast.Embedding): + def forward_ggml_cast_weights(self, input, out_dtype=None): + output_dtype = out_dtype + if (self.weight.dtype == torch.float16 or self.weight.dtype == + torch.bfloat16): + out_dtype = None + weight, _bias = self.cast_bias_weight(self, device=input.device, + dtype=out_dtype) + return torch.nn.functional.embedding(input, weight, self. + padding_idx, self.max_norm, self.norm_type, self. + scale_grad_by_freq, self.sparse).to(dtype=output_dtype) + class LayerNorm(GGMLLayer, comfy.ops.manual_cast.LayerNorm): + def forward_ggml_cast_weights(self, input): + if self.weight is None: + return super().forward_comfy_cast_weights(input) + weight, bias = self.cast_bias_weight(input) + return torch.nn.functional.layer_norm(input, self. + normalized_shape, weight, bias, self.eps) + class GroupNorm(GGMLLayer, comfy.ops.manual_cast.GroupNorm): + def forward_ggml_cast_weights(self, input): + weight, bias = self.cast_bias_weight(input) + return torch.nn.functional.group_norm(input, self.num_groups, + weight, bias, self.eps) +def load_patch_to_device(item, device): + if isinstance(item, torch.Tensor): + return item.to(device, non_blocking=True) + elif isinstance(item, tuple): + return tuple(load_patch_to_device(x, device) for x in item) + elif isinstance(item, list): + return [load_patch_to_device(x, device) for x in item] + else: + return item +def get_folder_names_and_paths(key, targets=[]): + base = folder_paths.folder_names_and_paths.get(key, ([], {})) + base = base[0] if isinstance(base[0], (list, set, tuple)) else [] + target = next((x for x in targets if x in folder_paths. + folder_names_and_paths), targets[0]) + orig, _ = folder_paths.folder_names_and_paths.get(target, ([], {})) + folder_paths.folder_names_and_paths[key] = orig or base, {'.gguf'} + if base and base != orig: + logging.warning( + f'Unknown file list already present on key {key}: {base}') +get_folder_names_and_paths('model_gguf', ['diffusion_models', 'unet']) +get_folder_names_and_paths('clip_gguf', ['text_encoders', 'clip']) +get_folder_names_and_paths('vae_gguf', ['vae']) +def get_orig_shape(reader, tensor_name): + field_key = f'comfy.gguf.orig_shape.{tensor_name}' + field = reader.get_field(field_key) + if field is None: + return None + if len(field.types) != 2 or field.types[0 + ] != gr.GGUFValueType.ARRAY or field.types[1 + ] != gr.GGUFValueType.INT32: + raise TypeError( + f'Bad original shape metadata for {field_key}: Expected ARRAY of INT32, got {field.types}' + ) + return torch.Size(tuple(int(field.parts[part_idx][0]) for part_idx in + field.data)) +def load_gguf_sd(path, handle_prefix='model.diffusion_model.', return_arch= + False): + reader = gr.GGUFReader(path) + has_prefix = False + if handle_prefix is not None: + prefix_len = len(handle_prefix) + tensor_names = set(tensor.name for tensor in reader.tensors) + has_prefix = any(s.startswith(handle_prefix) for s in tensor_names) + tensors = [] + for tensor in reader.tensors: + sd_key = tensor_name = tensor.name + if has_prefix: + if not tensor_name.startswith(handle_prefix): + continue + sd_key = tensor_name[prefix_len:] + tensors.append((sd_key, tensor)) + compat = None + arch_str = get_field(reader, 'general.architecture', str) + if arch_str is None: + compat = 'sd.cpp' + elif arch_str not in arrays['PIG_ARCH_LIST'] and arch_str not in arrays['TXT_ARCH_LIST']: + raise ValueError(f"Unknown architecture: {arch_str!r}") + state_dict, qtype_dict = {}, {} + for sd_key, tensor in tensors: + tensor_name = tensor.name + torch_tensor = torch.from_numpy(tensor.data) + shape = get_orig_shape(reader, tensor_name) + if shape is None: + shape = torch.Size(tuple(int(v) for v in reversed(tensor.shape))) + if compat == 'sd.cpp' and arch_str == 'sdxl': + if any([tensor_name.endswith(x) for x in ('.proj_in.weight', + '.proj_out.weight')]): + while len(shape) > 2 and shape[-1] == 1: + shape = shape[:-1] + if tensor.tensor_type in {gr.GGMLQuantizationType.F32, gr. + GGMLQuantizationType.F16}: + torch_tensor = torch_tensor.view(*shape) + state_dict[sd_key] = GGMLTensor(torch_tensor, tensor_type=tensor. + tensor_type, tensor_shape=shape) + tensor_type_str = getattr(tensor.tensor_type, 'name', repr(tensor.tensor_type)) + qtype_dict[tensor_type_str] = qtype_dict.get(tensor_type_str, 0) + 1 + print('gguf qtypes: ' + ', '.join(f'{k} ({v})' for k, v in qtype_dict.items())) + qsd = {k: v for k, v in state_dict.items() if is_quantized(v)} + if len(qsd) > 0: + max_key = max(qsd.keys(), key=lambda k: qsd[k].numel()) + state_dict[max_key].is_largest_weight = True + if return_arch: + return state_dict, arch_str + return state_dict +def tensor_swap(raw_sd, key_map): + sd = {} + for k, v in raw_sd.items(): + for s, d in key_map.items(): + k = k.replace(s, d) + sd[k] = v + return sd +def pig_work(raw_sd): + sd = {} + for k, v in raw_sd.items(): + sd[k] = v + return sd +def llama_permute(raw_sd, n_head, n_head_kv): + sd = {} + permute = lambda x, h: x.reshape(h, x.shape[0] // h // 2, 2, *x.shape[1:] + ).swapaxes(1, 2).reshape(x.shape) + for k, v in raw_sd.items(): + if k.endswith(('q_proj.weight', 'q_proj.bias')): + v.data = permute(v.data, n_head) + if k.endswith(('k_proj.weight', 'k_proj.bias')): + v.data = permute(v.data, n_head_kv) + sd[k] = v + return sd +def load_gguf_clip(path): + sd, arch = load_gguf_sd(path, return_arch=True) + if arch in {'t5', 't5encoder'}: + temb_key = 'token_embd.weight' + if temb_key in sd and (sd[temb_key].shape == (256384, 4096) or sd[temb_key].shape == (256384, 768)): + sd['spiece_model'] = tokenizer_builder(path) + sd = tensor_swap(sd, arrays['T5']) + elif temb_key in sd and sd[temb_key].shape == (32128, 768): + sd = tensor_swap(sd, arrays['B5']) + else: + sd = tensor_swap(sd, arrays['T5']) + elif arch in {'llama'}: + sd = tensor_swap(sd, arrays['L3']) + sd = llama_permute(sd, 32, 8) + elif arch in {'gemma2'}: + sd["spiece_model"] = tokenizer_builder(path) + sd = tensor_swap(sd, arrays['G2']) + elif arch in {'pig'}: + sd = pig_work(sd) + else: + pass + return sd +class LoaderGGUF: + @classmethod + def INPUT_TYPES(s): + gguf_names = [x for x in folder_paths.get_filename_list('model_gguf')] + return {'required': {'gguf_name': (gguf_names,)}} + RETURN_TYPES = 'MODEL', + FUNCTION = 'load_model' + CATEGORY = 'gguf' + TITLE = 'GGUF Loader' + def load_model(self, gguf_name, dequant_dtype=None, patch_dtype=None, + patch_on_device=None): + ops = GGMLOps() + if dequant_dtype in ('default', None): + ops.Linear.dequant_dtype = None + elif dequant_dtype in ['target']: + ops.Linear.dequant_dtype = dequant_dtype + else: + ops.Linear.dequant_dtype = getattr(torch, dequant_dtype) + if patch_dtype in ('default', None): + ops.Linear.patch_dtype = None + elif patch_dtype in ['target']: + ops.Linear.patch_dtype = patch_dtype + else: + ops.Linear.patch_dtype = getattr(torch, patch_dtype) + model_path = folder_paths.get_full_path('unet', gguf_name) + sd = load_gguf_sd(model_path) + model = comfy.sd.load_diffusion_model_state_dict(sd, model_options= + {'custom_operations': ops}) + if model is None: + logging.error('ERROR UNSUPPORTED MODEL {}'.format(model_path)) + raise RuntimeError('ERROR: Could not detect model type of: {}'. + format(model_path)) + model = GGUFModelPatcher.clone(model) + model.patch_on_device = patch_on_device + return model, +class LoaderGGUFAdvanced(LoaderGGUF): + @classmethod + def INPUT_TYPES(s): + model_names = [x for x in folder_paths.get_filename_list('model_gguf')] + return {'required': {'gguf_name': (model_names,), 'dequant_dtype': + (['default', 'target', 'float32', 'float16', 'bfloat16'], { + 'default': 'default'}), 'patch_dtype': (['default', 'target', + 'float32', 'float16', 'bfloat16'], {'default': 'default'}), + 'patch_on_device': ('BOOLEAN', {'default': False})}} + TITLE = 'GGUF Loader (Advanced)' +def get_clip_type(type): + clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION) + return clip_type +def get_device(device): + model_options = {} + if device == "cpu": + model_options["load_device"] = model_options["offload_device"] = torch.device("cpu") + return model_options +class ClipLoaderGGUF: + @classmethod + def INPUT_TYPES(s): + base = nodes.CLIPLoader.INPUT_TYPES() + return {'required': {'clip_name': (s.get_filename_list(),), 'type': + base['required']['type']}, + 'optional':{'device':(['default','cpu'],{'advanced':True}),}} + RETURN_TYPES = 'CLIP', + FUNCTION = 'load_clip' + CATEGORY = 'gguf' + TITLE = 'GGUF CLIP Loader' + @classmethod + def get_filename_list(s): + files = [] + files += folder_paths.get_filename_list('clip') + files += folder_paths.get_filename_list('clip_gguf') + return sorted(files) + def load_data(self, ckpt_paths): + clip_data = [] + for p in ckpt_paths: + if p.endswith('.gguf'): + sd = load_gguf_clip(p) + else: + sd = comfy.utils.load_torch_file(p, safe_load=True) + clip_data.append(sd) + return clip_data + def load_patcher(self, clip_paths, clip_type, clip_data): + clip = comfy.sd.load_text_encoder_state_dicts(clip_type=clip_type, + state_dicts=clip_data, model_options={'custom_operations': + GGMLOps, 'initial_device': comfy.model_management. + text_encoder_offload_device()}, embedding_directory= + folder_paths.get_folder_paths('embeddings')) + clip.patcher = GGUFModelPatcher.clone(clip.patcher) + return clip + def load_clip(self, clip_name, type='stable_diffusion', device='default'): + clip_path = folder_paths.get_full_path('clip', clip_name) + if clip_name.endswith('.safetensors') and device != 'default': + clip = comfy.sd.load_clip(ckpt_paths=[clip_path], embedding_directory=folder_paths.get_folder_paths("embeddings"), clip_type=get_clip_type(type), model_options=get_device(device)) + return (clip,) + else: + return (self.load_patcher([clip_path], get_clip_type(type), self.load_data([clip_path])), get_device(device)) +class DualClipLoaderGGUF(ClipLoaderGGUF): + @classmethod + def INPUT_TYPES(s): + base = nodes.DualCLIPLoader.INPUT_TYPES() + file_options = s.get_filename_list(), + return {'required': {'clip_name1':file_options, 'clip_name2':file_options, 'type': + base['required']['type']}, + 'optional':{'device':(['default','cpu'],{'advanced':True}),}} + TITLE = 'GGUF DualCLIP Loader' + def load_clip(self, clip_name1, clip_name2, type, device='default'): + clip_path1 = folder_paths.get_full_path('clip', clip_name1) + clip_path2 = folder_paths.get_full_path('clip', clip_name2) + clip_paths = clip_path1, clip_path2 + if device != 'default' and (clip_name1.endswith('.safetensors') and clip_name2.endswith('.safetensors')): + clip = comfy.sd.load_clip(ckpt_paths=clip_paths, embedding_directory=folder_paths.get_folder_paths("embeddings"), clip_type=get_clip_type(type), model_options=get_device(device)) + return (clip,) + else: + return (self.load_patcher(clip_paths, get_clip_type(type), self.load_data(clip_paths)), get_device(device)) +class TripleClipLoaderGGUF(ClipLoaderGGUF): + @classmethod + def INPUT_TYPES(s): + file_options = s.get_filename_list(), + return {'required': {'clip_name1': file_options, 'clip_name2': + file_options, 'clip_name3': file_options}} + TITLE = 'GGUF TripleCLIP Loader' + def load_clip(self, clip_name1, clip_name2, clip_name3, type='sd3'): + clip_path1 = folder_paths.get_full_path('clip', clip_name1) + clip_path2 = folder_paths.get_full_path('clip', clip_name2) + clip_path3 = folder_paths.get_full_path('clip', clip_name3) + clip_paths = clip_path1, clip_path2, clip_path3 + return (self.load_patcher(clip_paths, get_clip_type(type), self.load_data(clip_paths)),) +class QuadrupleClipLoaderGGUF(ClipLoaderGGUF): + @classmethod + def INPUT_TYPES(s): + file_options = s.get_filename_list(), + return {'required': {'clip_name1': file_options, 'clip_name2': + file_options, 'clip_name3': file_options, 'clip_name4': file_options}} + TITLE = 'GGUF QuadrupleCLIP Loader' + def load_clip(self, clip_name1, clip_name2, clip_name3, clip_name4, type='hunyuan_video'): + clip_path1 = folder_paths.get_full_path('clip', clip_name1) + clip_path2 = folder_paths.get_full_path('clip', clip_name2) + clip_path3 = folder_paths.get_full_path('clip', clip_name3) + clip_path4 = folder_paths.get_full_path('clip', clip_name4) + clip_paths = clip_path1, clip_path2, clip_path3, clip_path4 + return (self.load_patcher(clip_paths, get_clip_type(type), self.load_data(clip_paths)),) +QUANTIZATION_THRESHOLD = 1024 +REARRANGE_THRESHOLD = 512 +MAX_TENSOR_NAME_LENGTH = 127 +class ModelTemplate: + arch = 'invalid' + keys_detect = [] + keys_banned = [] +class ModelFlux(ModelTemplate): + arch = 'flux' + keys_detect = [('transformer_blocks.0.attn.norm_added_k.weight',), ( + 'double_blocks.0.img_attn.proj.weight',)] + keys_banned = ['transformer_blocks.0.attn.norm_added_k.weight'] +class ModelSD3(ModelTemplate): + arch = 'sd3' + keys_detect = [('transformer_blocks.0.attn.add_q_proj.weight',), ( + 'joint_blocks.0.x_block.attn.qkv.weight',)] + keys_banned = ['transformer_blocks.0.attn.add_q_proj.weight'] +class ModelSDXL(ModelTemplate): + arch = 'sdxl' + keys_detect = [('down_blocks.0.downsamplers.0.conv.weight', + 'add_embedding.linear_1.weight'), ('input_blocks.3.0.op.weight', + 'input_blocks.6.0.op.weight', 'output_blocks.2.2.conv.weight', + 'output_blocks.5.2.conv.weight'), ('label_emb.0.0.weight',)] +class ModelSD1(ModelTemplate): + arch = 'sd1' + keys_detect = [('down_blocks.0.downsamplers.0.conv.weight',), ( + 'input_blocks.3.0.op.weight', 'input_blocks.6.0.op.weight', + 'input_blocks.9.0.op.weight', 'output_blocks.2.1.conv.weight', + 'output_blocks.5.2.conv.weight', 'output_blocks.8.2.conv.weight')] +arch_list = [ModelFlux, ModelSD3, ModelSDXL, ModelSD1] +def is_model_arch(model, state_dict): + matched = False + invalid = False + for match_list in model.keys_detect: + if all(key in state_dict for key in match_list): + matched = True + invalid = any(key in state_dict for key in model.keys_banned) + break + assert not invalid, 'Model architecture not supported for alpha (please opt to use zero)' + return matched +def detect_arch(state_dict): + model_arch = None + for arch in arch_list: + if is_model_arch(arch, state_dict): + model_arch = arch + break + assert model_arch is not None, 'Unknown model architecture detected!' + return model_arch +def load_state_dict(path): + state_dict = load_file(path) + prefix = None + for pfx in ['model.diffusion_model.', 'model.', 'net.']: + if any([x.startswith(pfx) for x in state_dict.keys()]): + prefix = pfx + break + sd = {} + for k, v in state_dict.items(): + if prefix and prefix not in k: + continue + if prefix: + k = k.replace(prefix, '') + sd[k] = v + return sd +def load_model(path): + state_dict = load_state_dict(path) + model_arch = detect_arch(state_dict) + print(f'* Architecture detected from input: {model_arch.arch}') + writer = GGUFWriter(path=None, arch=model_arch.arch) + return writer, state_dict, model_arch +def load_pig_state(path): + pig_state = load_file(path) + sd = {} + for k, v in pig_state.items(): + sd[k] = v + return sd +def load_pig(path): + state_dict = load_pig_state(path) + model_arch = arrays['TXT_ARCH_LIST'][0] + writer = GGUFWriter(path=None, arch=model_arch) + return writer, state_dict, model_arch +def handle_tensors(args, writer, state_dict, model_arch): + name_lengths = tuple(sorted(((key, len(key)) for key in state_dict.keys + ()), key=lambda item: item[1], reverse=True)) + if not name_lengths: + return + max_name_len = name_lengths[0][1] + if max_name_len > MAX_TENSOR_NAME_LENGTH: + bad_list = ', '.join(f'{key!r} ({namelen})' for key, namelen in + name_lengths if namelen > MAX_TENSOR_NAME_LENGTH) + raise ValueError( + f'Can only handle tensor names up to {MAX_TENSOR_NAME_LENGTH} characters. Tensors exceeding the limit: {bad_list}' + ) + for key, data in loading(state_dict.items()): + old_dtype = data.dtype + if data.dtype == torch.bfloat16: + data = data.to(torch.float32).numpy() + elif data.dtype in [getattr(torch, 'float8_e4m3fn', '_invalid'), + getattr(torch, 'float8_e5m2', '_invalid')]: + data = data.to(torch.float16).numpy() + else: + data = data.numpy() + n_dims = len(data.shape) + data_shape = data.shape + data_qtype = getattr(GGMLQuantizationType, 'BF16' if old_dtype == + torch.bfloat16 else 'F16') + n_params = 1 + for dim_size in data_shape: + n_params *= dim_size + blacklist = {'time_embedding.', 'add_embedding.', 'time_in.', + 'txt_in.', 'vector_in.', 'img_in.', 'guidance_in.', 'final_layer.'} + if old_dtype in (torch.float32, torch.bfloat16): + if n_dims == 1: + data_qtype = GGMLQuantizationType.F32 + elif n_params <= QUANTIZATION_THRESHOLD: + data_qtype = GGMLQuantizationType.F32 + elif '.weight' in key and any(x in key for x in blacklist): + data_qtype = GGMLQuantizationType.F32 + try: + data = quantize(data, data_qtype) + except (AttributeError, QuantError) as e: + loading.write(f'falling back to F16: {e}') + data_qtype = GGMLQuantizationType.F16 + data = quantize(data, data_qtype) + new_name = key + shape_str = f"{{{', '.join(str(n) for n in reversed(data.shape))}}}" + loading.write( + f"{f'%-{max_name_len + 4}s' % f'{new_name}'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}" + ) + writer.add_tensor(new_name, data, raw_dtype=data_qtype) +if 'select_safetensors' not in folder_paths.folder_names_and_paths: + orig = folder_paths.folder_names_and_paths.get('diffusion_models', + folder_paths.folder_names_and_paths.get('checkpoints', [[], set()])) + folder_paths.folder_names_and_paths['select_safetensors'] = orig[0], { + '.safetensors'} +class GGUFSave: + def __init__(self): + self.output_dir = folder_paths.get_output_directory() + @classmethod + def INPUT_TYPES(s): + return {'required': {'select_safetensors': (s.get_filename_list(),)}} + RETURN_TYPES = () + FUNCTION = 'save' + OUTPUT_NODE = True + CATEGORY = 'gguf' + TITLE = 'GGUF Convertor (Alpha)' + @classmethod + def get_filename_list(s): + files = [] + files += folder_paths.get_filename_list('select_safetensors') + return sorted(files) + def save(self, select_safetensors): + path = folder_paths.get_full_path('select_safetensors', + select_safetensors) + writer, state_dict, model_arch = load_model(path) + writer.add_quantization_version(GGML_QUANT_VERSION) + if next(iter(state_dict.values())).dtype == torch.bfloat16: + output_path = ( + f'{self.output_dir}/{os.path.splitext(select_safetensors)[0]}-bf16.gguf' + ) + writer.add_file_type(LlamaFileType.MOSTLY_BF16) + else: + output_path = ( + f'{self.output_dir}/{os.path.splitext(select_safetensors)[0]}-f16.gguf' + ) + writer.add_file_type(LlamaFileType.MOSTLY_F16) + if os.path.isfile(output_path): + input('Output exists enter to continue or ctrl+c to abort!') + handle_tensors(output_path, writer, state_dict, model_arch) + writer.write_header_to_file(path=output_path) + writer.write_kv_data_to_file() + writer.write_tensors_to_file(progress=True) + writer.close() + return {} +class GGUFRun: + def __init__(self): + self.output_dir = folder_paths.get_output_directory() + @classmethod + def INPUT_TYPES(s): + return {'required': {'select_safetensors': (s.get_filename_list(),)}} + RETURN_TYPES = () + FUNCTION = 'run' + OUTPUT_NODE = True + CATEGORY = 'gguf' + TITLE = 'GGUF Convertor (Zero)' + @classmethod + def get_filename_list(s): + files = [] + files += folder_paths.get_filename_list('select_safetensors') + return sorted(files) + def run(self, select_safetensors): + path = folder_paths.get_full_path('select_safetensors', + select_safetensors) + writer, state_dict, model_arch = load_pig(path) + writer.add_quantization_version(GGML_QUANT_VERSION) + if next(iter(state_dict.values())).dtype == torch.bfloat16: + output_path = ( + f'{self.output_dir}/{os.path.splitext(select_safetensors)[0]}-bf16.gguf' + ) + writer.add_file_type(LlamaFileType.MOSTLY_BF16) + else: + output_path = ( + f'{self.output_dir}/{os.path.splitext(select_safetensors)[0]}-f16.gguf' + ) + writer.add_file_type(LlamaFileType.MOSTLY_F16) + if os.path.isfile(output_path): + input('Output exists enter to continue or ctrl+c to abort!') + handle_tensors(output_path, writer, state_dict, model_arch) + writer.write_header_to_file(path=output_path) + writer.write_kv_data_to_file() + writer.write_tensors_to_file(progress=True) + writer.close() + return {} +def quantize_to_fp8(tensor): + if tensor.dtype != torch.bfloat16: + raise ValueError('Input tensor must be in BF16 format.') + tensor = tensor.to(torch.float16) + fp8_max = 240.0 + fp8_min = -fp8_max + clamped_tensor = tensor.clamp(min=fp8_min, max=fp8_max) + scale = fp8_max / torch.max(torch.abs(clamped_tensor)) + quantized_tensor = torch.round(clamped_tensor * scale) / scale + return quantized_tensor.to(torch.float8_e4m3fn) +class TENSORCut: + def __init__(self): + self.output_dir = folder_paths.get_output_directory() + @classmethod + def INPUT_TYPES(s): + return {'required': {'select_safetensors': (s.get_filename_list(),)}} + RETURN_TYPES = () + FUNCTION = 'cut' + OUTPUT_NODE = True + CATEGORY = 'gguf' + TITLE = 'TENSOR Cutter (Beta)' + @classmethod + def get_filename_list(s): + files = [] + files += folder_paths.get_filename_list('select_safetensors') + return sorted(files) + def cut(self, select_safetensors): + input_file = folder_paths.get_full_path('select_safetensors', + select_safetensors) + output_file = ( + f'{self.output_dir}/{os.path.splitext(select_safetensors)[0]}_fp8_e4m3fn.safetensors' + ) + data = load_file(input_file) + quantized_data = {} + print('Starting quantization process...') + for key, tensor in loading(data.items(), desc='Quantizing tensors', + unit='tensor'): + tensor = tensor.to(dtype=torch.bfloat16, device='cuda') + quantized_tensor = quantize_to_fp8(tensor) + quantized_data[key] = quantized_tensor.cpu() + save_file(quantized_data, output_file) + print(f'Quantized safetensors saved to {output_file}.') + return {} +class TENSORBoost: + def __init__(self): + self.output_dir = folder_paths.get_output_directory() + @classmethod + def INPUT_TYPES(s): + return {'required': {'select_safetensors': (s.get_filename_list(),)}} + RETURN_TYPES = () + FUNCTION = 'boost' + OUTPUT_NODE = True + CATEGORY = 'gguf' + TITLE = 'TENSOR Booster' + @classmethod + def get_filename_list(s): + files = [] + files += folder_paths.get_filename_list('select_safetensors') + return sorted(files) + def boost(self, select_safetensors): + input_file = folder_paths.get_full_path('select_safetensors', + select_safetensors) + output_file = ( + f'{self.output_dir}/{os.path.splitext(select_safetensors)[0]}_fp32.safetensors' + ) + data = load_file(input_file) + quantized_data = {} + print('Starting quantization process...') + for key, tensor in loading(data.items(), desc="Converting tensors", unit="tensor"): + quantized_data[key] = tensor.to(torch.float32) + save_file(quantized_data, output_file) + print(f'Quantized safetensors saved to {output_file}.') + return {} +def load_gguf_and_extract_metadata(gguf_path): + reader = gr.GGUFReader(gguf_path) + tensors_metadata = [] + for tensor in reader.tensors: + tensor_metadata = {'name': tensor.name, 'shape': tuple(tensor.shape + .tolist()), 'n_elements': tensor.n_elements, 'n_bytes': tensor. + n_bytes, 'data_offset': tensor.data_offset, 'type': tensor. + tensor_type} + tensors_metadata.append(tensor_metadata) + return reader, tensors_metadata +def convert_gguf_to_safetensors(gguf_path, output_path, use_bf16): + reader, tensors_metadata = load_gguf_and_extract_metadata(gguf_path) + print(f'Extracted {len(tensors_metadata)} tensors from GGUF file') + tensors_dict: dict[str, torch.Tensor] = {} + for i, tensor_info in enumerate(loading(tensors_metadata, desc= + 'Converting tensors', unit='tensor')): + tensor_name = tensor_info['name'] + tensor_data = reader.get_tensor(i) + weights = dequantize(tensor_data.data, tensor_data.tensor_type).copy() + try: + if use_bf16: + weights_tensor = torch.from_numpy(weights).to(dtype=torch. + float32) + weights_tensor = weights_tensor.to(torch.bfloat16) + else: + weights_tensor = torch.from_numpy(weights).to(dtype=torch. + float16) + weights_hf = weights_tensor + except Exception as e: + print( + f"Error during BF16 conversion for tensor '{tensor_name}': {e}" + ) + weights_tensor = torch.from_numpy(weights.astype(numpy.float32) + ).to(torch.float16) + weights_hf = weights_tensor + tensors_dict[tensor_name] = weights_hf + metadata = {key: str(reader.get_field(key)) for key in reader.fields} + save_file(tensors_dict, output_path, metadata=metadata) + print('Conversion complete!') +if 'select_gguf' not in folder_paths.folder_names_and_paths: + orig = folder_paths.folder_names_and_paths.get('diffusion_models', + folder_paths.folder_names_and_paths.get('unet', [[], set()])) + folder_paths.folder_names_and_paths['select_gguf'] = orig[0], {'.gguf'} +class GGUFUndo: + def __init__(self): + self.output_dir = folder_paths.get_output_directory() + @classmethod + def INPUT_TYPES(s): + return {'required': {'select_gguf': (s.get_filename_list(),)}} + RETURN_TYPES = () + FUNCTION = 'undo' + OUTPUT_NODE = True + CATEGORY = 'gguf' + TITLE = 'GGUF Convertor (Reverse)' + @classmethod + def get_filename_list(s): + files = [] + files += folder_paths.get_filename_list('select_gguf') + return sorted(files) + def undo(self, select_gguf): + in_file = folder_paths.get_full_path('select_gguf', select_gguf) + out_file = (f'{self.output_dir}/{os.path.splitext(select_gguf)[0]}_fp16.safetensors') + use_bf16 = False + convert_gguf_to_safetensors(in_file, out_file, use_bf16) + return {} +class VaeGGUF: + @staticmethod + def vae_list(): + vaes = [] + vaes += folder_paths.get_filename_list('vae') + vaes += folder_paths.get_filename_list('vae_gguf') + approx_vaes = folder_paths.get_filename_list('vae_approx') + sdxl_taesd_enc = False + sdxl_taesd_dec = False + sd1_taesd_enc = False + sd1_taesd_dec = False + sd3_taesd_enc = False + sd3_taesd_dec = False + f1_taesd_enc = False + f1_taesd_dec = False + for v in approx_vaes: + if v.startswith('taesd_decoder.'): + sd1_taesd_dec = True + elif v.startswith('taesd_encoder.'): + sd1_taesd_enc = True + elif v.startswith('taesdxl_decoder.'): + sdxl_taesd_dec = True + elif v.startswith('taesdxl_encoder.'): + sdxl_taesd_enc = True + elif v.startswith('taesd3_decoder.'): + sd3_taesd_dec = True + elif v.startswith('taesd3_encoder.'): + sd3_taesd_enc = True + elif v.startswith('taef1_encoder.'): + f1_taesd_dec = True + elif v.startswith('taef1_decoder.'): + f1_taesd_enc = True + if sd1_taesd_dec and sd1_taesd_enc: + vaes.append('taesd') + if sdxl_taesd_dec and sdxl_taesd_enc: + vaes.append('taesdxl') + if sd3_taesd_dec and sd3_taesd_enc: + vaes.append('taesd3') + if f1_taesd_dec and f1_taesd_enc: + vaes.append('taef1') + return vaes + @staticmethod + def load_taesd(name): + sd = {} + approx_vaes = folder_paths.get_filename_list('vae_approx') + encoder = next(filter(lambda a: a.startswith('{}_encoder.'.format(name)), approx_vaes)) + decoder = next(filter(lambda a: a.startswith('{}_decoder.'.format(name)), approx_vaes)) + enc = comfy.utils.load_torch_file(folder_paths. + get_full_path_or_raise('vae_approx', encoder)) + for k in enc: + sd['taesd_encoder.{}'.format(k)] = enc[k] + dec = comfy.utils.load_torch_file(folder_paths. + get_full_path_or_raise('vae_approx', decoder)) + for k in dec: + sd['taesd_decoder.{}'.format(k)] = dec[k] + if name == 'taesd': + sd['vae_scale'] = torch.tensor(0.18215) + sd['vae_shift'] = torch.tensor(0.0) + elif name == 'taesdxl': + sd['vae_scale'] = torch.tensor(0.13025) + sd['vae_shift'] = torch.tensor(0.0) + elif name == 'taesd3': + sd['vae_scale'] = torch.tensor(1.5305) + sd['vae_shift'] = torch.tensor(0.0609) + elif name == 'taef1': + sd['vae_scale'] = torch.tensor(0.3611) + sd['vae_shift'] = torch.tensor(0.1159) + return sd + @classmethod + def INPUT_TYPES(s): + return {'required': {'vae_name': (s.vae_list(),)}} + RETURN_TYPES = 'VAE', + FUNCTION = 'load_vae' + CATEGORY = 'gguf' + TITLE = 'GGUF VAE Loader' + def load_vae(self, vae_name): + if vae_name.endswith('.gguf'): + vae_path = folder_paths.get_full_path_or_raise('vae_gguf', vae_name) + sd = load_gguf_clip(vae_path) + elif vae_name in ['taesd', 'taesdxl', 'taesd3', 'taef1']: + sd = self.load_taesd(vae_name) + else: + vae_path = folder_paths.get_full_path_or_raise('vae', vae_name) + sd = comfy.utils.load_torch_file(vae_path) + vae = comfy.sd.VAE(sd=sd) + return vae, +NODE_CLASS_MAPPINGS = { + "LoaderGGUF": LoaderGGUF, + "ClipLoaderGGUF": ClipLoaderGGUF, + "DualClipLoaderGGUF": DualClipLoaderGGUF, + "TripleClipLoaderGGUF": TripleClipLoaderGGUF, + "QuadrupleClipLoaderGGUF": QuadrupleClipLoaderGGUF, + "LoaderGGUFAdvanced": LoaderGGUFAdvanced, + "VaeGGUF": VaeGGUF, + "GGUFUndo": GGUFUndo, + "GGUFSave": GGUFSave, + "GGUFRun": GGUFRun, + "TENSORCut": TENSORCut, + "TENSORBoost": TENSORBoost, +} diff --git a/pyproject.toml b/pyproject.toml index 83dc1fb..90f42ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "ComfyUI-WarpedToolset" description = "A toolset for Hunyuan Video (mainly), with some additional nodes applicable to Framepack Video and WAN Video." -version = "1.7.0" +version = "1.7.1" license = { file = "LICENSE" } [project.urls] diff --git a/version.json b/version.json new file mode 100644 index 0000000..6341f66 --- /dev/null +++ b/version.json @@ -0,0 +1 @@ +[{"version":"0.0.58","PIG_ARCH_LIST":["pig","fatfat","piggy","piglet","gguf","frame","sd1","sdxl","sd3","flux","aura","mochi","ltxv","hyvid","cosmos","pixart","lumina","wan","hidream"],"TXT_ARCH_LIST":["pig","t5","t5encoder","llama","gemma2"],"L3":{"blk.":"model.layers.","attn_norm":"input_layernorm","attn_q":"self_attn.q_proj","attn_k":"self_attn.k_proj","attn_v":"self_attn.v_proj","attn_output":"self_attn.o_proj","ffn_up":"mlp.up_proj","ffn_down":"mlp.down_proj","ffn_gate":"mlp.gate_proj","ffn_norm":"post_attention_layernorm","token_embd":"model.embed_tokens","output_norm":"model.norm","output.weight":"lm_head.weight"},"T5":{"enc.":"encoder.",".blk.":".block.","token_embd":"shared","output_norm":"final_layer_norm","attn_q":"layer.0.SelfAttention.q","attn_k":"layer.0.SelfAttention.k","attn_v":"layer.0.SelfAttention.v","attn_o":"layer.0.SelfAttention.o","attn_norm":"layer.0.layer_norm","attn_rel_b":"layer.0.SelfAttention.relative_attention_bias","ffn_up":"layer.1.DenseReluDense.wi_1","ffn_down":"layer.1.DenseReluDense.wo","ffn_gate":"layer.1.DenseReluDense.wi_0","ffn_norm":"layer.1.layer_norm"},"B5":{"enc.":"encoder.",".blk.":".block.","token_embd":"shared","output_norm":"final_layer_norm","attn_q":"layer.0.SelfAttention.q","attn_k":"layer.0.SelfAttention.k","attn_v":"layer.0.SelfAttention.v","attn_o":"layer.0.SelfAttention.o","attn_norm":"layer.0.layer_norm","attn_rel_b":"layer.0.SelfAttention.relative_attention_bias","ffn_up":"layer.1.DenseReluDense.wi","ffn_down":"layer.1.DenseReluDense.wo","ffn_norm":"layer.1.layer_norm"},"G2":{"gemma2_2b.transformer.":""}}]