nodes/WarpedAnimation-ComfyUI-WarpedToolset
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

V1.7.1 (#11)

Browse Source 
* V1.7.1

* V1.7.1

* V1.7.1

* Create deleteme.txt

* v1.7.1

* Delete gguf_connector/deleteme.txt

* v1.7.1

* V1.7.1
This commit is contained in:
WarpedAnimation
2025-07-24 08:16:24 -05:00
committed by GitHub
parent 820bc62198
commit 78beffb29f
93 changed files with 17133 additions and 2700 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+7
View File
@@ -25,6 +25,13 @@
<div align="left">
# V1.7.1 Updates
1) Added Fast Loading GGUF Unet, Clip, and Dual Clip Loader Nodes
2) Fixed/Updated Sample Workflows
<div align="left">
# V1.7.0 Overview
#### [V1.7.0 Overview Video](https://mega.nz/file/r2YwySbS#a1Fw_Z4ASlkXIVBmLQFHAbEzFxpWGGVmyEMrVNplt6I)
Sample_Workflows/Warped Batch Hunyuan LORA T2V Workflow.json
+500 -537
View File
File diff suppressed because it is too large Load Diff
Sample_Workflows/Warped Hunyuan I2V Using I2V Model Workflow.json
+267 -269
View File
@@ -1,8 +1,8 @@
{
"id": "6fee7bc1-689d-4cd1-b4f9-7c20316a44f8",
"revision": 0,
"last_node_id": 311,
"last_link_id": 1047,
"last_node_id": 314,
"last_link_id": 1055,
"nodes": [
{
"id": 152,
@@ -144,7 +144,7 @@
"name": "start_image",
"shape": 7,
"type": "IMAGE",
"link": 1035
"link": 1051
},
{
"name": "width",
@@ -152,7 +152,7 @@
"widget": {
"name": "width"
},
"link": 1037
"link": 1049
},
{
"name": "height",
@@ -160,7 +160,7 @@
"widget": {
"name": "height"
},
"link": 1038
"link": 1050
}
],
"outputs": [
@@ -375,7 +375,7 @@
{
"name": "clip",
"type": "CLIP",
"link": 899
"link": 1055
},
{
"name": "clip_vision_output",
@@ -419,13 +419,13 @@
346.54290771484375
],
"flags": {},
"order": 9,
"order": 10,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "MODEL",
"link": 840
"link": 1054
}
],
"outputs": [
@@ -458,72 +458,6 @@
"all"
]
},
{
"id": 306,
"type": "WarpedImageResize",
"pos": [
-72.92000579833984,
1124.219482421875
],
"size": [
315,
214
],
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 1033
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
1034,
1035,
1036
]
},
{
"name": "scale_orig_image",
"type": "IMAGE",
"links": null
},
{
"name": "width",
"type": "INT",
"links": [
1037
]
},
{
"name": "height",
"type": "INT",
"links": [
1038
]
}
],
"properties": {
"cnr_id": "ComfyUI-WarpedToolset",
"ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb",
"Node name for S&R": "WarpedImageResize"
},
"widgets_values": [
400,
720,
"lanczos",
"center",
false
],
"color": "#323",
"bgcolor": "#535"
},
{
"id": 22,
"type": "BasicGuider",
@@ -584,13 +518,13 @@
476.8260192871094
],
"flags": {},
"order": 11,
"order": 12,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 1034
"link": 1053
}
],
"outputs": [],
@@ -624,7 +558,7 @@
"links": [
1013,
1025,
1033
1048
]
},
{
@@ -807,7 +741,7 @@
26
],
"flags": {},
"order": 10,
"order": 9,
"mode": 0,
"inputs": [
{
@@ -922,80 +856,6 @@
-1
]
},
{
"id": 144,
"type": "DualCLIPLoaderGGUF",
"pos": [
-391.7001037597656,
433.44525146484375
],
"size": [
386.0740051269531,
106
],
"flags": {
"collapsed": true
},
"order": 4,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "CLIP",
"type": "CLIP",
"slot_index": 0,
"links": [
899
]
}
],
"properties": {
"cnr_id": "comfyui-gguf",
"ver": "298192ed60f8ca821c6fe5f8030cae23424cada5",
"Node name for S&R": "DualCLIPLoaderGGUF"
},
"widgets_values": [
"clip_l.safetensors",
"Kijai\\llava-llama-3-8B-v1_1-Q4_K_M.gguf",
"hunyuan_video"
]
},
{
"id": 145,
"type": "UnetLoaderGGUF",
"pos": [
-390.6521911621094,
488.76007080078125
],
"size": [
296.70855712890625,
65.69999694824219
],
"flags": {
"collapsed": true
},
"order": 5,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"slot_index": 0,
"links": [
840
]
}
],
"properties": {
"cnr_id": "comfyui-gguf",
"ver": "298192ed60f8ca821c6fe5f8030cae23424cada5",
"Node name for S&R": "UnetLoaderGGUF"
},
"widgets_values": [
"hunyuan-video-i2v-720p-Q5_K_M.gguf"
]
},
{
"id": 309,
"type": "WarpedVAELoader",
@@ -1010,7 +870,7 @@
"flags": {
"collapsed": true
},
"order": 6,
"order": 4,
"mode": 0,
"inputs": [],
"outputs": [
@@ -1041,53 +901,6 @@
"color": "#323",
"bgcolor": "#535"
},
{
"id": 280,
"type": "CLIPVisionEncode",
"pos": [
-386.97509765625,
593.4584350585938
],
"size": [
336.4000244140625,
87.9000015258789
],
"flags": {
"collapsed": true
},
"order": 12,
"mode": 0,
"inputs": [
{
"name": "clip_vision",
"type": "CLIP_VISION",
"link": 1045
},
{
"name": "image",
"type": "IMAGE",
"link": 1036
}
],
"outputs": [
{
"name": "CLIP_VISION_OUTPUT",
"type": "CLIP_VISION_OUTPUT",
"slot_index": 0,
"links": [
897
]
}
],
"properties": {
"cnr_id": "comfy-core",
"ver": "0.3.26",
"Node name for S&R": "CLIPVisionEncode"
},
"widgets_values": [
"none"
]
},
{
"id": 310,
"type": "WarpedCLIPVisionLoader",
@@ -1102,7 +915,7 @@
"flags": {
"collapsed": true
},
"order": 7,
"order": 5,
"mode": 0,
"inputs": [],
"outputs": [
@@ -1125,6 +938,191 @@
],
"color": "#323",
"bgcolor": "#535"
},
{
"id": 312,
"type": "ImageResizeKJv2",
"pos": [
-67.84000396728516,
1105.3377685546875
],
"size": [
270,
242
],
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 1048
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
1051,
1052,
1053
]
},
{
"name": "width",
"type": "INT",
"links": [
1049
]
},
{
"name": "height",
"type": "INT",
"links": [
1050
]
}
],
"properties": {
"cnr_id": "comfyui-kjnodes",
"ver": "1.1.0",
"Node name for S&R": "ImageResizeKJv2"
},
"widgets_values": [
400,
720,
"lanczos",
"resize",
"0, 0, 0",
"center",
2
]
},
{
"id": 280,
"type": "CLIPVisionEncode",
"pos": [
-386.97509765625,
593.4584350585938
],
"size": [
336.4000244140625,
87.9000015258789
],
"flags": {
"collapsed": true
},
"order": 11,
"mode": 0,
"inputs": [
{
"name": "clip_vision",
"type": "CLIP_VISION",
"link": 1045
},
{
"name": "image",
"type": "IMAGE",
"link": 1052
}
],
"outputs": [
{
"name": "CLIP_VISION_OUTPUT",
"type": "CLIP_VISION_OUTPUT",
"slot_index": 0,
"links": [
897
]
}
],
"properties": {
"cnr_id": "comfy-core",
"ver": "0.3.26",
"Node name for S&R": "CLIPVisionEncode"
},
"widgets_values": [
"none"
]
},
{
"id": 313,
"type": "WarpedLoaderGGUF",
"pos": [
-391.27294921875,
394.58453369140625
],
"size": [
417.9764404296875,
64.65499877929688
],
"flags": {
"collapsed": true
},
"order": 6,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
1054
]
}
],
"properties": {
"cnr_id": "comfyui-warpedtoolset",
"ver": "1.7.0",
"Node name for S&R": "WarpedLoaderGGUF"
},
"widgets_values": [
"hunyuan-video-i2v-720p-Q5_K_M.gguf"
],
"color": "#323",
"bgcolor": "#535"
},
{
"id": 314,
"type": "WarpedDualClipLoaderGGUF",
"pos": [
-388.6109924316406,
462.46478271484375
],
"size": [
300.9039001464844,
130
],
"flags": {
"collapsed": true
},
"order": 7,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "CLIP",
"type": "CLIP",
"links": [
1055
]
}
],
"properties": {
"cnr_id": "comfyui-warpedtoolset",
"ver": "1.7.0",
"Node name for S&R": "WarpedDualClipLoaderGGUF"
},
"widgets_values": [
"clip_l.safetensors",
"Kijai\\llava-llama-3-8B-v1_1-Q4_K_M.gguf",
"hunyuan_video",
"default"
],
"color": "#323",
"bgcolor": "#535"
}
],
"links": [
@@ -1136,14 +1134,6 @@
0,
"MODEL"
],
[
840,
145,
0,
258,
0,
"MODEL"
],
[
897,
280,
@@ -1152,14 +1142,6 @@
1,
"CLIP_VISION_OUTPUT"
],
[
899,
144,
0,
282,
0,
"CLIP"
],
[
987,
282,
@@ -1240,54 +1222,6 @@
0,
"STRING"
],
[
1033,
262,
0,
306,
0,
"IMAGE"
],
[
1034,
306,
0,
255,
0,
"IMAGE"
],
[
1035,
306,
0,
296,
2,
"IMAGE"
],
[
1036,
306,
0,
280,
1,
"IMAGE"
],
[
1037,
306,
2,
296,
3,
"INT"
],
[
1038,
306,
3,
296,
4,
"INT"
],
[
1041,
309,
@@ -1335,6 +1269,70 @@
67,
0,
"MODEL"
],
[
1048,
262,
0,
312,
0,
"IMAGE"
],
[
1049,
312,
1,
296,
3,
"INT"
],
[
1050,
312,
2,
296,
4,
"INT"
],
[
1051,
312,
0,
296,
2,
"IMAGE"
],
[
1052,
312,
0,
280,
1,
"IMAGE"
],
[
1053,
312,
0,
255,
0,
"IMAGE"
],
[
1054,
313,
0,
258,
0,
"MODEL"
],
[
1055,
314,
0,
282,
0,
"CLIP"
]
],
"groups": [
@@ -1357,10 +1355,11 @@
"ds": {
"scale": 0.7513148009015777,
"offset": [
670.783000000001,
269.58499999999214
612.2190000000006,
-173.6380000000081
]
},
"frontendVersion": "1.20.5",
"groupNodes": {},
"ue_links": [],
"VHS_latentpreview": true,
@@ -1375,8 +1374,7 @@
"Hunyuan-Multi-Lora-Loader": "dd009a9d44b47bfaf01a4ffa38d7098f523e2cce"
},
"VHS_MetadataImage": true,
"VHS_KeepIntermediate": true,
"frontendVersion": "1.20.5"
"VHS_KeepIntermediate": true
},
"version": 0.4
}
Sample_Workflows/Warped Hunyuan I2V Workflow.json
+291 -292
View File
@@ -1,8 +1,8 @@
{
"id": "dfed12ef-fd2b-4870-b037-155a07e06804",
"revision": 0,
"last_node_id": 291,
"last_link_id": 925,
"last_node_id": 294,
"last_link_id": 931,
"nodes": [
{
"id": 99,
@@ -82,44 +82,6 @@
"color": "#222",
"bgcolor": "#000"
},
{
"id": 144,
"type": "DualCLIPLoaderGGUF",
"pos": [
-397.0240783691406,
445.4241943359375
],
"size": [
386.0740051269531,
106
],
"flags": {
"collapsed": true
},
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "CLIP",
"type": "CLIP",
"slot_index": 0,
"links": [
899
]
}
],
"properties": {
"cnr_id": "ComfyUI-GGUF",
"ver": "bc5223b0e37e053dbec2ea5e5f52c2fd4b8f712a",
"Node name for S&R": "DualCLIPLoaderGGUF"
},
"widgets_values": [
"clip_l.safetensors",
"Kijai\\llava-llama-3-8B-v1_1-Q4_K_M.gguf",
"hunyuan_video"
]
},
{
"id": 262,
"type": "LoadImage",
@@ -132,7 +94,7 @@
314
],
"flags": {},
"order": 2,
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
@@ -142,7 +104,7 @@
"slot_index": 0,
"links": [
900,
913
926
]
},
{
@@ -173,7 +135,7 @@
476.8260192871094
],
"flags": {},
"order": 15,
"order": 14,
"mode": 0,
"inputs": [
{
@@ -202,13 +164,13 @@
200
],
"flags": {},
"order": 13,
"order": 15,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 899
"link": 930
},
{
"name": "clip_vision_output",
@@ -249,13 +211,13 @@
128.45428466796875
],
"flags": {},
"order": 11,
"order": 10,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 915
"link": 927
}
],
"outputs": [
@@ -303,7 +265,7 @@
"flags": {
"collapsed": true
},
"order": 3,
"order": 2,
"mode": 0,
"inputs": [],
"outputs": [
@@ -339,7 +301,7 @@
"flags": {
"collapsed": true
},
"order": 14,
"order": 13,
"mode": 0,
"inputs": [
{
@@ -382,7 +344,7 @@
152.45428466796875
],
"flags": {},
"order": 17,
"order": 16,
"mode": 0,
"inputs": [
{
@@ -440,13 +402,13 @@
127.22714233398438
],
"flags": {},
"order": 12,
"order": 11,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 916
"link": 928
}
],
"outputs": [
@@ -601,68 +563,6 @@
}
}
},
{
"id": 285,
"type": "WarpedImageResize",
"pos": [
-64.4500961303711,
1057.6700439453125
],
"size": [
315,
214
],
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 913
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
914,
915,
916
]
},
{
"name": "scale_orig_image",
"type": "IMAGE",
"links": null
},
{
"name": "width",
"type": "INT",
"links": null
},
{
"name": "height",
"type": "INT",
"links": null
}
],
"properties": {
"cnr_id": "ComfyUI-WarpedToolset",
"ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb",
"Node name for S&R": "WarpedImageResize"
},
"widgets_values": [
544,
960,
"lanczos",
"center",
false
],
"color": "#323",
"bgcolor": "#535"
},
{
"id": 195,
"type": "Reroute",
@@ -675,7 +575,7 @@
26
],
"flags": {},
"order": 9,
"order": 8,
"mode": 0,
"inputs": [
{
@@ -713,7 +613,7 @@
"flags": {
"collapsed": true
},
"order": 16,
"order": 17,
"mode": 0,
"inputs": [
{
@@ -743,42 +643,6 @@
"color": "#232",
"bgcolor": "#353"
},
{
"id": 145,
"type": "UnetLoaderGGUF",
"pos": [
-391.75238037109375,
494.3700866699219
],
"size": [
383.6085510253906,
58
],
"flags": {
"collapsed": true
},
"order": 4,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"slot_index": 0,
"links": [
840
]
}
],
"properties": {
"cnr_id": "ComfyUI-GGUF",
"ver": "bc5223b0e37e053dbec2ea5e5f52c2fd4b8f712a",
"Node name for S&R": "UnetLoaderGGUF"
},
"widgets_values": [
"hunyuan-video-t2v-720p-Q4_K_M.gguf"
]
},
{
"id": 81,
"type": "Reroute",
@@ -831,7 +695,7 @@
"flags": {
"collapsed": true
},
"order": 5,
"order": 3,
"mode": 0,
"inputs": [],
"outputs": [
@@ -861,91 +725,6 @@
"color": "#323",
"bgcolor": "#535"
},
{
"id": 280,
"type": "CLIPVisionEncode",
"pos": [
-457.8369140625,
1442.4373779296875
],
"size": [
380.4000244140625,
78
],
"flags": {
"collapsed": true
},
"order": 10,
"mode": 0,
"inputs": [
{
"name": "clip_vision",
"type": "CLIP_VISION",
"link": 921
},
{
"name": "image",
"type": "IMAGE",
"link": 914
}
],
"outputs": [
{
"name": "CLIP_VISION_OUTPUT",
"type": "CLIP_VISION_OUTPUT",
"slot_index": 0,
"links": [
897
]
}
],
"properties": {
"cnr_id": "comfy-core",
"ver": "0.3.27",
"Node name for S&R": "CLIPVisionEncode"
},
"widgets_values": [
"none"
]
},
{
"id": 288,
"type": "WarpedCLIPVisionLoader",
"pos": [
-452.4989013671875,
1499.314208984375
],
"size": [
315,
58
],
"flags": {
"collapsed": true
},
"order": 6,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "clip_vision",
"type": "CLIP_VISION",
"links": [
921
]
}
],
"properties": {
"cnr_id": "ComfyUI-WarpedToolset",
"ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb",
"Node name for S&R": "WarpedCLIPVisionLoader",
"widget_ue_connectable": {}
},
"widgets_values": [
"clip_vision_h_fp8_e4m3fn.safetensors"
],
"color": "#323",
"bgcolor": "#535"
},
{
"id": 67,
"type": "ModelSamplingSD3",
@@ -1048,13 +827,13 @@
346.54290771484375
],
"flags": {},
"order": 8,
"order": 9,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "MODEL",
"link": 840
"link": 931
}
],
"outputs": [
@@ -1348,6 +1127,226 @@
"ver": "1.7.0",
"Node name for S&R": "WarpedNumericalConversion"
},
"widgets_values": [],
"color": "#323",
"bgcolor": "#535"
},
{
"id": 280,
"type": "CLIPVisionEncode",
"pos": [
-445.85797119140625,
1389.1964111328125
],
"size": [
380.4000244140625,
78
],
"flags": {
"collapsed": true
},
"order": 12,
"mode": 0,
"inputs": [
{
"name": "clip_vision",
"type": "CLIP_VISION",
"link": 921
},
{
"name": "image",
"type": "IMAGE",
"link": 929
}
],
"outputs": [
{
"name": "CLIP_VISION_OUTPUT",
"type": "CLIP_VISION_OUTPUT",
"slot_index": 0,
"links": [
897
]
}
],
"properties": {
"cnr_id": "comfy-core",
"ver": "0.3.27",
"Node name for S&R": "CLIPVisionEncode"
},
"widgets_values": [
"none"
]
},
{
"id": 288,
"type": "WarpedCLIPVisionLoader",
"pos": [
-440.51995849609375,
1456.720703125
],
"size": [
315,
58
],
"flags": {
"collapsed": true
},
"order": 4,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "clip_vision",
"type": "CLIP_VISION",
"links": [
921
]
}
],
"properties": {
"cnr_id": "ComfyUI-WarpedToolset",
"ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb",
"Node name for S&R": "WarpedCLIPVisionLoader",
"widget_ue_connectable": {}
},
"widgets_values": [
"clip_vision_h_fp8_e4m3fn.safetensors"
],
"color": "#323",
"bgcolor": "#535"
},
{
"id": 292,
"type": "ImageResizeKJv2",
"pos": [
-59.854007720947266,
1054.7607421875
],
"size": [
270,
242
],
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 926
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
927,
928,
929
]
},
{
"name": "width",
"type": "INT",
"links": null
},
{
"name": "height",
"type": "INT",
"links": null
}
],
"properties": {
"cnr_id": "comfyui-kjnodes",
"ver": "1.1.0",
"Node name for S&R": "ImageResizeKJv2"
},
"widgets_values": [
544,
960,
"lanczos",
"resize",
"0, 0, 0",
"center",
2
]
},
{
"id": 293,
"type": "WarpedDualClipLoaderGGUF",
"pos": [
-409.9068603515625,
435.8446960449219
],
"size": [
300.9039001464844,
130
],
"flags": {
"collapsed": true
},
"order": 5,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "CLIP",
"type": "CLIP",
"links": [
930
]
}
],
"properties": {
"cnr_id": "comfyui-warpedtoolset",
"ver": "1.7.0",
"Node name for S&R": "WarpedDualClipLoaderGGUF"
},
"widgets_values": [
"clip_l.safetensors",
"Kijai\\llava-llama-3-8B-v1_1-Q4_K_M.gguf",
"hunyuan_video",
"default"
],
"color": "#323",
"bgcolor": "#535"
},
{
"id": 294,
"type": "WarpedLoaderGGUF",
"pos": [
-396.5967102050781,
491.7479248046875
],
"size": [
390.02545166015625,
61.99300003051758
],
"flags": {
"collapsed": true
},
"order": 6,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
931
]
}
],
"properties": {
"cnr_id": "comfyui-warpedtoolset",
"ver": "1.7.0",
"Node name for S&R": "WarpedLoaderGGUF"
},
"widgets_values": [
"hunyuan-video-t2v-720p-Q4_K_M.gguf"
],
"color": "#323",
"bgcolor": "#535"
}
@@ -1377,14 +1376,6 @@
0,
"MODEL"
],
[
840,
145,
0,
258,
0,
"MODEL"
],
[
878,
195,
@@ -1497,14 +1488,6 @@
0,
"CONDITIONING"
],
[
899,
144,
0,
282,
0,
"CLIP"
],
[
907,
284,
@@ -1521,38 +1504,6 @@
0,
"IMAGE"
],
[
913,
262,
0,
285,
0,
"IMAGE"
],
[
914,
285,
0,
280,
1,
"IMAGE"
],
[
915,
285,
0,
274,
0,
"IMAGE"
],
[
916,
285,
0,
284,
0,
"IMAGE"
],
[
919,
287,
@@ -1608,6 +1559,54 @@
290,
0,
"STRING"
],
[
926,
262,
0,
292,
0,
"IMAGE"
],
[
927,
292,
0,
274,
0,
"IMAGE"
],
[
928,
292,
0,
284,
0,
"IMAGE"
],
[
929,
292,
0,
280,
1,
"IMAGE"
],
[
930,
293,
0,
282,
0,
"CLIP"
],
[
931,
294,
0,
258,
0,
"MODEL"
]
],
"groups": [
@@ -1630,10 +1629,11 @@
"ds": {
"scale": 0.7513148009015777,
"offset": [
608.2260000000005,
-152.3420000000077
597.5779999999984,
-401.2390000000077
]
},
"frontendVersion": "1.20.5",
"groupNodes": {},
"ue_links": [],
"VHS_latentpreview": true,
@@ -1648,8 +1648,7 @@
"comfyui-videohelpersuite": "1.5.2"
},
"VHS_MetadataImage": true,
"VHS_KeepIntermediate": true,
"frontendVersion": "1.20.5"
"VHS_KeepIntermediate": true
},
"version": 0.4
}
Sample_Workflows/Warped Hunyuan T2V Workflow.json
+202 -197
View File
@@ -1,8 +1,8 @@
{
"id": "facb86ea-4076-46ff-921d-e6dcadfb1754",
"revision": 0,
"last_node_id": 289,
"last_link_id": 908,
"last_node_id": 291,
"last_link_id": 910,
"nodes": [
{
"id": 99,
@@ -35,41 +35,6 @@
"toggleRestriction": "default"
}
},
{
"id": 275,
"type": "UnetLoaderGGUF",
"pos": [
-446.1169128417969,
342.95745849609375
],
"size": [
315,
58
],
"flags": {
"collapsed": true
},
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
872
]
}
],
"properties": {
"cnr_id": "ComfyUI-GGUF",
"ver": "bc5223b0e37e053dbec2ea5e5f52c2fd4b8f712a",
"Node name for S&R": "UnetLoaderGGUF"
},
"widgets_values": [
"hunyuan-video-t2v-720p-Q4_K_M.gguf"
]
},
{
"id": 26,
"type": "FluxGuidance",
@@ -84,7 +49,7 @@
"flags": {
"collapsed": true
},
"order": 11,
"order": 10,
"mode": 0,
"inputs": [
{
@@ -163,95 +128,6 @@
"color": "#222",
"bgcolor": "#000"
},
{
"id": 273,
"type": "DualCLIPLoaderGGUF",
"pos": [
-442.2043762207031,
386.2261047363281
],
"size": [
301.0328369140625,
106
],
"flags": {
"collapsed": true
},
"order": 2,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "CLIP",
"type": "CLIP",
"slot_index": 0,
"links": [
869
]
}
],
"properties": {
"cnr_id": "ComfyUI-GGUF",
"ver": "bc5223b0e37e053dbec2ea5e5f52c2fd4b8f712a",
"Node name for S&R": "DualCLIPLoaderGGUF"
},
"widgets_values": [
"clip_l.safetensors",
"Kijai\\llava-llama-3-8B-v1_1-Q4_K_M.gguf",
"hunyuan_video"
]
},
{
"id": 146,
"type": "CLIPTextEncode",
"pos": [
-6.035816669464111,
173.69970703125
],
"size": [
288.2851257324219,
78.88789367675781
],
"flags": {
"collapsed": true
},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 869
},
{
"name": "text",
"type": "STRING",
"widget": {
"name": "text"
},
"link": 448
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"slot_index": 0,
"links": [
352
]
}
],
"properties": {
"cnr_id": "comfy-core",
"ver": "0.3.27",
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
"8itchW4lk, dynamic (real life:1.15) video showing front view of a beautiful stunning gorgeous woman with very long flowing brown hair with blonde highlights and exotic sexy grey eyes, she is wearing (a provocative revealing see-through scientist indiana jones style maroon satin top blouse:1.2) and see-through black thong underwear with an extremely short tight micro miniskirt low on the hips and she is braless, (she is showing her massive ((natural heavy sagging:1.3)) naked breasts:1.15), (her nipples are erect:1.25), (she is also wearing a thin silver military dogtags pendant necklace, she is in an abandonned nuclear facility, she was surprised by the viewer, so now she is walking toward the viewer to seduce him:1.25), (she is walking toward the viewer:1.2), (she is walking in a very sexy manner, smug facial expression with a slight smirk, sexy but dangerous attitude and seductive lips, normal speed, her heavy breasts (((are (bouncing:1.5) and (swaying:1.3) and (jiggling:1.7):1.7)) unrestrained with every step she takes and with ever move she makes:1.15), (((her breasts bounce into each other and recoil away from each other as she walks:1.8))), the viewer keeps her in view all the times, (extremely soft breasts:1.3), (extremely bouncing breasts:1.3), (breasts bouncing up and down side to side:1.5), detailed face, detailed hands, detailed fingers, nail polish, glamour eye makeup, eyeliner, lipstick, mascara, 4k ultra high quality video footage, (realisic jiggle, boucing, wobble, swaying physics, her breasts jiggle and bounce with each staps she take:1.2), dark atmosphere, post apocalyptic nuclear building interior, retro cyberpunk vibe",
true
]
},
{
"id": 81,
"type": "Reroute",
@@ -264,7 +140,7 @@
26
],
"flags": {},
"order": 10,
"order": 11,
"mode": 0,
"inputs": [
{
@@ -302,7 +178,7 @@
26
],
"flags": {},
"order": 9,
"order": 7,
"mode": 0,
"inputs": [
{
@@ -387,7 +263,7 @@
"flags": {
"collapsed": true
},
"order": 3,
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
@@ -502,7 +378,7 @@
409.93310546875
],
"flags": {},
"order": 4,
"order": 2,
"mode": 0,
"inputs": [],
"outputs": [
@@ -526,49 +402,6 @@
true
]
},
{
"id": 286,
"type": "WarpedVAELoader",
"pos": [
-439.18890380859375,
438.5068359375
],
"size": [
315,
58
],
"flags": {
"collapsed": true
},
"order": 5,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "vae",
"type": "VAE",
"links": [
904
]
},
{
"name": "vae_state_dict",
"type": "DICT",
"links": null
}
],
"properties": {
"cnr_id": "ComfyUI-WarpedToolset",
"ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb",
"Node name for S&R": "WarpedVAELoader",
"widget_ue_connectable": {}
},
"widgets_values": [
"hunyuan_video_vae_bf16.safetensors"
],
"color": "#323",
"bgcolor": "#535"
},
{
"id": 67,
"type": "ModelSamplingSD3",
@@ -671,7 +504,7 @@
126
],
"flags": {},
"order": 6,
"order": 3,
"mode": 0,
"inputs": [],
"outputs": [
@@ -939,6 +772,7 @@
"ver": "1.7.0",
"Node name for S&R": "WarpedNumericalConversion"
},
"widgets_values": [],
"color": "#323",
"bgcolor": "#535"
},
@@ -954,13 +788,13 @@
345.9579772949219
],
"flags": {},
"order": 7,
"order": 9,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "MODEL",
"link": 872
"link": 910
}
],
"outputs": [
@@ -993,6 +827,177 @@
1,
"all"
]
},
{
"id": 146,
"type": "CLIPTextEncode",
"pos": [
-15.35281753540039,
169.70672607421875
],
"size": [
288.2851257324219,
81
],
"flags": {
"collapsed": true
},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 909
},
{
"name": "text",
"type": "STRING",
"widget": {
"name": "text"
},
"link": 448
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"slot_index": 0,
"links": [
352
]
}
],
"properties": {
"cnr_id": "comfy-core",
"ver": "0.3.27",
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
"8itchW4lk, dynamic (real life:1.15) video showing front view of a beautiful stunning gorgeous woman with very long flowing brown hair with blonde highlights and exotic sexy grey eyes, she is wearing (a provocative revealing see-through scientist indiana jones style maroon satin top blouse:1.2) and see-through black thong underwear with an extremely short tight micro miniskirt low on the hips and she is braless, (she is showing her massive ((natural heavy sagging:1.3)) naked breasts:1.15), (her nipples are erect:1.25), (she is also wearing a thin silver military dogtags pendant necklace, she is in an abandonned nuclear facility, she was surprised by the viewer, so now she is walking toward the viewer to seduce him:1.25), (she is walking toward the viewer:1.2), (she is walking in a very sexy manner, smug facial expression with a slight smirk, sexy but dangerous attitude and seductive lips, normal speed, her heavy breasts (((are (bouncing:1.5) and (swaying:1.3) and (jiggling:1.7):1.7)) unrestrained with every step she takes and with ever move she makes:1.15), (((her breasts bounce into each other and recoil away from each other as she walks:1.8))), the viewer keeps her in view all the times, (extremely soft breasts:1.3), (extremely bouncing breasts:1.3), (breasts bouncing up and down side to side:1.5), detailed face, detailed hands, detailed fingers, nail polish, glamour eye makeup, eyeliner, lipstick, mascara, 4k ultra high quality video footage, (realisic jiggle, boucing, wobble, swaying physics, her breasts jiggle and bounce with each staps she take:1.2), dark atmosphere, post apocalyptic nuclear building interior, retro cyberpunk vibe",
true
]
},
{
"id": 286,
"type": "WarpedVAELoader",
"pos": [
-436.52691650390625,
451.8167724609375
],
"size": [
315,
58
],
"flags": {
"collapsed": true
},
"order": 4,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "vae",
"type": "VAE",
"links": [
904
]
},
{
"name": "vae_state_dict",
"type": "DICT",
"links": null
}
],
"properties": {
"cnr_id": "ComfyUI-WarpedToolset",
"ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb",
"Node name for S&R": "WarpedVAELoader",
"widget_ue_connectable": {}
},
"widgets_values": [
"hunyuan_video_vae_bf16.safetensors"
],
"color": "#323",
"bgcolor": "#535"
},
{
"id": 290,
"type": "WarpedDualClipLoaderGGUF",
"pos": [
-435.19573974609375,
405.23223876953125
],
"size": [
300.9039001464844,
130
],
"flags": {
"collapsed": true
},
"order": 5,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "CLIP",
"type": "CLIP",
"links": [
909
]
}
],
"properties": {
"cnr_id": "comfyui-warpedtoolset",
"ver": "1.7.0",
"Node name for S&R": "WarpedDualClipLoaderGGUF"
},
"widgets_values": [
"clip_l.safetensors",
"Kijai\\llava-llama-3-8B-v1_1-Q4_K_M.gguf",
"hunyuan_video",
"default"
],
"color": "#323",
"bgcolor": "#535"
},
{
"id": 291,
"type": "WarpedLoaderGGUF",
"pos": [
-436.5267639160156,
354.6539306640625
],
"size": [
282.2144470214844,
58
],
"flags": {
"collapsed": true
},
"order": 6,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
910
]
}
],
"properties": {
"cnr_id": "comfyui-warpedtoolset",
"ver": "1.7.0",
"Node name for S&R": "WarpedLoaderGGUF"
},
"widgets_values": [
"hunyuan-video-t2v-720p-Q4_K_M.gguf"
],
"color": "#323",
"bgcolor": "#535"
}
],
"links": [
@@ -1036,22 +1041,6 @@
0,
"MODEL"
],
[
869,
273,
0,
146,
0,
"CLIP"
],
[
872,
275,
0,
258,
0,
"MODEL"
],
[
873,
258,
@@ -1155,6 +1144,22 @@
289,
0,
"STRING"
],
[
909,
290,
0,
146,
0,
"CLIP"
],
[
910,
291,
0,
258,
0,
"MODEL"
]
],
"groups": [
@@ -1177,10 +1182,11 @@
"ds": {
"scale": 0.7513148009015777,
"offset": [
541.6760000000013,
149.79499999999234
668.1210000000013,
107.20299999999239
]
},
"frontendVersion": "1.20.5",
"groupNodes": {},
"ue_links": [],
"VHS_latentpreview": true,
@@ -1199,8 +1205,7 @@
"Hunyuan-Multi-Lora-Loader": "dd009a9d44b47bfaf01a4ffa38d7098f523e2cce"
},
"VHS_MetadataImage": true,
"VHS_KeepIntermediate": true,
"frontendVersion": "1.20.5"
"VHS_KeepIntermediate": true
},
"version": 0.4
}
Sample_Workflows/Warped Hunyuan v2v Workflow.json
+288 -289
View File
@@ -1,8 +1,8 @@
{
"id": "cd4d8b23-c2f1-4f41-b28f-b13b2b63a361",
"revision": 0,
"last_node_id": 303,
"last_link_id": 957,
"last_node_id": 306,
"last_link_id": 963,
"nodes": [
{
"id": 152,
@@ -227,79 +227,6 @@
"color": "#222",
"bgcolor": "#000"
},
{
"id": 273,
"type": "DualCLIPLoaderGGUF",
"pos": [
-457.2220153808594,
416.15118408203125
],
"size": [
301.0328369140625,
106
],
"flags": {
"collapsed": true
},
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "CLIP",
"type": "CLIP",
"slot_index": 0,
"links": [
869
]
}
],
"properties": {
"cnr_id": "ComfyUI-GGUF",
"ver": "bc5223b0e37e053dbec2ea5e5f52c2fd4b8f712a",
"Node name for S&R": "DualCLIPLoaderGGUF"
},
"widgets_values": [
"clip_l.safetensors",
"Kijai\\llava-llama-3-8B-v1_1-Q4_K_M.gguf",
"hunyuan_video"
]
},
{
"id": 275,
"type": "UnetLoaderGGUF",
"pos": [
-448.9233093261719,
363.3001708984375
],
"size": [
315,
58
],
"flags": {
"collapsed": true
},
"order": 2,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
872
]
}
],
"properties": {
"cnr_id": "ComfyUI-GGUF",
"ver": "bc5223b0e37e053dbec2ea5e5f52c2fd4b8f712a",
"Node name for S&R": "UnetLoaderGGUF"
},
"widgets_values": [
"hunyuan-video-t2v-720p-Q4_K_M.gguf"
]
},
{
"id": 81,
"type": "Reroute",
@@ -338,93 +265,6 @@
"horizontal": false
}
},
{
"id": 146,
"type": "CLIPTextEncode",
"pos": [
2.7562851905822754,
169.0845947265625
],
"size": [
288.2851257324219,
78.88789367675781
],
"flags": {
"collapsed": true
},
"order": 30,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 869
},
{
"name": "text",
"type": "STRING",
"widget": {
"name": "text"
},
"link": 448
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"slot_index": 0,
"links": [
352
]
}
],
"properties": {
"cnr_id": "comfy-core",
"ver": "0.3.27",
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
"8itchW4lk, dynamic (real life:1.15) video showing front view of a beautiful stunning gorgeous woman with very long flowing brown hair with blonde highlights and exotic sexy grey eyes, she is wearing (a provocative revealing see-through scientist indiana jones style maroon satin top blouse:1.2) and see-through black thong underwear with an extremely short tight micro miniskirt low on the hips and she is braless, (she is showing her massive ((natural heavy sagging:1.3)) naked breasts:1.15), (her nipples are erect:1.25), (she is also wearing a thin silver military dogtags pendant necklace, she is in an abandonned nuclear facility, she was surprised by the viewer, so now she is walking toward the viewer to seduce him:1.25), (she is walking toward the viewer:1.2), (she is walking in a very sexy manner, smug facial expression with a slight smirk, sexy but dangerous attitude and seductive lips, normal speed, her heavy breasts (((are (bouncing:1.5) and (swaying:1.3) and (jiggling:1.7):1.7)) unrestrained with every step she takes and with ever move she makes:1.15), (((her breasts bounce into each other and recoil away from each other as she walks:1.8))), the viewer keeps her in view all the times, (extremely soft breasts:1.3), (extremely bouncing breasts:1.3), (breasts bouncing up and down side to side:1.5), detailed face, detailed hands, detailed fingers, nail polish, glamour eye makeup, eyeliner, lipstick, mascara, 4k ultra high quality video footage, (realisic jiggle, boucing, wobble, swaying physics, her breasts jiggle and bounce with each staps she take:1.2), dark atmosphere, post apocalyptic nuclear building interior, retro cyberpunk vibe",
true
]
},
{
"id": 195,
"type": "Reroute",
"pos": [
20.86060333251953,
238.17318725585938
],
"size": [
75,
26
],
"flags": {},
"order": 10,
"mode": 0,
"inputs": [
{
"name": "",
"type": "*",
"link": 955
}
],
"outputs": [
{
"name": "",
"type": "VAE",
"slot_index": 0,
"links": [
914
]
}
],
"properties": {
"showOutputText": false,
"horizontal": false
}
},
{
"id": 289,
"type": "WarpedLeapfusionHunyuanI2V",
@@ -495,7 +335,7 @@
281.7099914550781
],
"flags": {},
"order": 9,
"order": 8,
"mode": 0,
"inputs": [
{
@@ -596,7 +436,7 @@
{
"name": "image",
"type": "IMAGE",
"link": 953
"link": 961
},
{
"name": "vae",
@@ -700,13 +540,13 @@
476.8260192871094
],
"flags": {},
"order": 11,
"order": 12,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 951
"link": 960
}
],
"outputs": [],
@@ -984,7 +824,7 @@
"flags": {
"collapsed": true
},
"order": 3,
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
@@ -1120,13 +960,13 @@
126
],
"flags": {},
"order": 12,
"order": 11,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 952
"link": 959
}
],
"outputs": [
@@ -1230,7 +1070,7 @@
250
],
"flags": {},
"order": 4,
"order": 2,
"mode": 0,
"inputs": [],
"outputs": [
@@ -1240,7 +1080,7 @@
"slot_index": 0,
"links": [
940,
950
958
]
},
{
@@ -1422,7 +1262,7 @@
"flags": {
"collapsed": false
},
"order": 5,
"order": 3,
"mode": 0,
"inputs": [],
"outputs": [
@@ -1603,7 +1443,7 @@
"flags": {
"collapsed": true
},
"order": 6,
"order": 4,
"mode": 0,
"inputs": [],
"outputs": [
@@ -1633,68 +1473,6 @@
"color": "#323",
"bgcolor": "#535"
},
{
"id": 301,
"type": "WarpedImageResize",
"pos": [
-457.73272705078125,
1149.1563720703125
],
"size": [
315,
214
],
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 950
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
951,
952,
953
]
},
{
"name": "scale_orig_image",
"type": "IMAGE",
"links": null
},
{
"name": "width",
"type": "INT",
"links": null
},
{
"name": "height",
"type": "INT",
"links": null
}
],
"properties": {
"cnr_id": "ComfyUI-WarpedToolset",
"ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb",
"Node name for S&R": "WarpedImageResize"
},
"widgets_values": [
512,
288,
"lanczos",
"center",
false
],
"color": "#323",
"bgcolor": "#535"
},
{
"id": 67,
"type": "ModelSamplingSD3",
@@ -1797,13 +1575,13 @@
322
],
"flags": {},
"order": 7,
"order": 10,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "MODEL",
"link": 872
"link": 963
}
],
"outputs": [
@@ -1836,6 +1614,227 @@
1,
"all"
]
},
{
"id": 304,
"type": "ImageResizeKJv2",
"pos": [
-453.8300476074219,
1161.2410888671875
],
"size": [
270,
242
],
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 958
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
959,
960,
961
]
},
{
"name": "width",
"type": "INT",
"links": null
},
{
"name": "height",
"type": "INT",
"links": null
}
],
"properties": {
"cnr_id": "comfyui-kjnodes",
"ver": "1.1.0",
"Node name for S&R": "ImageResizeKJv2"
},
"widgets_values": [
512,
288,
"lanczos",
"resize",
"0, 0, 0",
"center",
2
]
},
{
"id": 305,
"type": "WarpedDualClipLoaderGGUF",
"pos": [
-445.8439636230469,
415.8797912597656
],
"size": [
300.9039001464844,
130
],
"flags": {
"collapsed": true
},
"order": 5,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "CLIP",
"type": "CLIP",
"links": [
962
]
}
],
"properties": {
"cnr_id": "comfyui-warpedtoolset",
"ver": "1.7.0",
"Node name for S&R": "WarpedDualClipLoaderGGUF"
},
"widgets_values": [
"clip_l.safetensors",
"Kijai\\llava-llama-3-8B-v1_1-Q4_K_M.gguf",
"sdxl",
"default"
],
"color": "#323",
"bgcolor": "#535"
},
{
"id": 306,
"type": "WarpedLoaderGGUF",
"pos": [
-449.8374328613281,
358.64703369140625
],
"size": [
282.2144470214844,
58
],
"flags": {
"collapsed": true
},
"order": 6,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
963
]
}
],
"properties": {
"cnr_id": "comfyui-warpedtoolset",
"ver": "1.7.0",
"Node name for S&R": "WarpedLoaderGGUF"
},
"widgets_values": [
"hunyuan-video-t2v-720p-Q4_K_M.gguf"
],
"color": "#323",
"bgcolor": "#535"
},
{
"id": 146,
"type": "CLIPTextEncode",
"pos": [
2.7562851905822754,
169.0845947265625
],
"size": [
288.2851257324219,
81
],
"flags": {
"collapsed": true
},
"order": 30,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 962
},
{
"name": "text",
"type": "STRING",
"widget": {
"name": "text"
},
"link": 448
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"slot_index": 0,
"links": [
352
]
}
],
"properties": {
"cnr_id": "comfy-core",
"ver": "0.3.27",
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
"8itchW4lk, dynamic (real life:1.15) video showing front view of a beautiful stunning gorgeous woman with very long flowing brown hair with blonde highlights and exotic sexy grey eyes, she is wearing (a provocative revealing see-through scientist indiana jones style maroon satin top blouse:1.2) and see-through black thong underwear with an extremely short tight micro miniskirt low on the hips and she is braless, (she is showing her massive ((natural heavy sagging:1.3)) naked breasts:1.15), (her nipples are erect:1.25), (she is also wearing a thin silver military dogtags pendant necklace, she is in an abandonned nuclear facility, she was surprised by the viewer, so now she is walking toward the viewer to seduce him:1.25), (she is walking toward the viewer:1.2), (she is walking in a very sexy manner, smug facial expression with a slight smirk, sexy but dangerous attitude and seductive lips, normal speed, her heavy breasts (((are (bouncing:1.5) and (swaying:1.3) and (jiggling:1.7):1.7)) unrestrained with every step she takes and with ever move she makes:1.15), (((her breasts bounce into each other and recoil away from each other as she walks:1.8))), the viewer keeps her in view all the times, (extremely soft breasts:1.3), (extremely bouncing breasts:1.3), (breasts bouncing up and down side to side:1.5), detailed face, detailed hands, detailed fingers, nail polish, glamour eye makeup, eyeliner, lipstick, mascara, 4k ultra high quality video footage, (realisic jiggle, boucing, wobble, swaying physics, her breasts jiggle and bounce with each staps she take:1.2), dark atmosphere, post apocalyptic nuclear building interior, retro cyberpunk vibe",
true
]
},
{
"id": 195,
"type": "Reroute",
"pos": [
7.55060338973999,
239.50413513183594
],
"size": [
75,
26
],
"flags": {},
"order": 9,
"mode": 0,
"inputs": [
{
"name": "",
"type": "*",
"link": 955
}
],
"outputs": [
{
"name": "",
"type": "VAE",
"slot_index": 0,
"links": [
914
]
}
],
"properties": {
"showOutputText": false,
"horizontal": false
}
}
],
"links": [
@@ -1879,22 +1878,6 @@
0,
"MODEL"
],
[
869,
273,
0,
146,
0,
"CLIP"
],
[
872,
275,
0,
258,
0,
"MODEL"
],
[
890,
282,
@@ -2095,38 +2078,6 @@
0,
"IMAGE"
],
[
950,
296,
0,
301,
0,
"IMAGE"
],
[
951,
301,
0,
255,
0,
"IMAGE"
],
[
952,
301,
0,
290,
0,
"IMAGE"
],
[
953,
301,
0,
288,
0,
"IMAGE"
],
[
954,
302,
@@ -2158,6 +2109,54 @@
67,
0,
"MODEL"
],
[
958,
296,
0,
304,
0,
"IMAGE"
],
[
959,
304,
0,
290,
0,
"IMAGE"
],
[
960,
304,
0,
255,
0,
"IMAGE"
],
[
961,
304,
0,
288,
0,
"IMAGE"
],
[
962,
305,
0,
146,
0,
"CLIP"
],
[
963,
306,
0,
258,
0,
"MODEL"
]
],
"groups": [
@@ -2180,10 +2179,11 @@
"ds": {
"scale": 0.7513148009015777,
"offset": [
540.3449999999999,
148.46399999999193
568.2959999999991,
18.025999999991825
]
},
"frontendVersion": "1.20.5",
"groupNodes": {},
"ue_links": [],
"VHS_latentpreview": true,
@@ -2201,8 +2201,7 @@
"Hunyuan-Multi-Lora-Loader": "dd009a9d44b47bfaf01a4ffa38d7098f523e2cce"
},
"VHS_MetadataImage": true,
"VHS_KeepIntermediate": true,
"frontendVersion": "1.20.5"
"VHS_KeepIntermediate": true
},
"version": 0.4
}
Sample_Workflows/Warped Midres Upscale Workflow.json
+487 -489
View File
File diff suppressed because it is too large Load Diff
Sample_Workflows/Warped Upscale By Model Workflow.json
+243 -296
View File
@@ -2,8 +2,134 @@
"id": "05a742f7-a4cc-476b-bec2-4dfbaa1009f5",
"revision": 0,
"last_node_id": 281,
"last_link_id": 895,
"last_link_id": 896,
"nodes": [
{
"id": 279,
"type": "VHS_VideoCombine",
"pos": [
1025.608642578125,
429.6685791015625
],
"size": [
534.675537109375,
358
],
"flags": {},
"order": 7,
"mode": 2,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 891
},
{
"name": "audio",
"shape": 7,
"type": "AUDIO",
"link": null
},
{
"name": "meta_batch",
"shape": 7,
"type": "VHS_BatchManager",
"link": null
},
{
"name": "vae",
"shape": 7,
"type": "VAE",
"link": null
}
],
"outputs": [
{
"name": "Filenames",
"type": "VHS_FILENAMES",
"links": null
}
],
"properties": {
"cnr_id": "comfyui-videohelpersuite",
"ver": "5e61bcf218fe3bb7c899bbd584bbc99a9d05fb42",
"Node name for S&R": "VHS_VideoCombine"
},
"widgets_values": {
"frame_rate": 30,
"loop_count": 0,
"filename_prefix": "Hunyuan/%date:yyyy_MM_dd%/interpolate/vid",
"format": "video/nvenc_h264-mp4",
"pix_fmt": "yuv420p",
"bitrate": 10,
"megabit": true,
"save_metadata": true,
"pingpong": false,
"save_output": true,
"save_images": true,
"videopreview": {
"hidden": false,
"paused": false,
"params": {
"filename": "vid_00001.mp4",
"subfolder": "Hunyuan\\2025_03_19\\interpolate",
"type": "output",
"format": "video/nvenc_h264-mp4",
"frame_rate": 30,
"workflow": "vid_00001.png",
"fullpath": "M:\\Projects\\ComfyUI_windows_portable\\ComfyUI\\output\\Hunyuan\\2025_03_19\\interpolate\\vid_00001.mp4"
}
}
}
},
{
"id": 278,
"type": "FILM VFI",
"pos": [
1029.126953125,
156.33795166015625
],
"size": [
443.4000244140625,
126
],
"flags": {},
"order": 6,
"mode": 2,
"inputs": [
{
"name": "frames",
"type": "IMAGE",
"link": 890
},
{
"name": "optional_interpolation_states",
"shape": 7,
"type": "INTERPOLATION_STATES",
"link": null
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"slot_index": 0,
"links": [
891
]
}
],
"properties": {
"cnr_id": "comfyui-frame-interpolation",
"ver": "18b7cce5b08290741929e39d4955b88382db4e1d",
"Node name for S&R": "FILM VFI"
},
"widgets_values": [
"film_net_fp32.pt",
20,
2
]
},
{
"id": 99,
"type": "Fast Groups Muter (rgthree)",
@@ -35,12 +161,80 @@
"toggleRestriction": "default"
}
},
{
"id": 281,
"type": "WarpedBundleAllVideoImages",
"pos": [
-179.6438446044922,
178.9619598388672
],
"size": [
378,
202
],
"flags": {},
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "image_batch",
"type": "IMAGE",
"links": [
894,
896
]
},
{
"name": "first_image",
"type": "IMAGE",
"links": null
},
{
"name": "last_image",
"type": "IMAGE",
"links": null
},
{
"name": "num_frames",
"type": "INT",
"links": null
},
{
"name": "width",
"type": "INT",
"links": null
},
{
"name": "height",
"type": "INT",
"links": null
},
{
"name": "fps",
"type": "FLOAT",
"links": null
}
],
"properties": {
"cnr_id": "ComfyUI-WarpedToolset",
"ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb",
"Node name for S&R": "WarpedBundleAllVideoImages",
"widget_ue_connectable": {}
},
"widgets_values": [
"N:\\Projects\\ComfyUI_windows_portable\\ComfyUI\\output\\Hunyuan\\2025_07_23\\lowres\\vid_00029.mp4",
true
],
"color": "#323",
"bgcolor": "#535"
},
{
"id": 255,
"type": "PreviewImage",
"pos": [
-434.8162536621094,
537.4317626953125
-169.94720458984375,
462.8961181640625
],
"size": [
542.0654907226562,
@@ -68,15 +262,15 @@
"id": 202,
"type": "VHS_VideoCombine",
"pos": [
331.1438293457031,
451.63348388671875
387.04595947265625,
464.9437255859375
],
"size": [
534.675537109375,
358
],
"flags": {},
"order": 6,
"order": 5,
"mode": 0,
"inputs": [
{
@@ -142,53 +336,19 @@
}
}
},
{
"id": 201,
"type": "UpscaleModelLoader",
"pos": [
341.0927734375,
334.5395812988281
],
"size": [
315,
58
],
"flags": {},
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "UPSCALE_MODEL",
"type": "UPSCALE_MODEL",
"slot_index": 0,
"links": [
885
]
}
],
"properties": {
"cnr_id": "comfy-core",
"ver": "0.3.27",
"Node name for S&R": "UpscaleModelLoader"
},
"widgets_values": [
"4x_foolhardy_Remacri.pth"
]
},
{
"id": 276,
"type": "WarpedUpscaleWithModel",
"pos": [
345.3873291015625,
158.17369079589844
379.9934997558594,
160.83570861816406
],
"size": [
315,
102
],
"flags": {},
"order": 5,
"order": 4,
"mode": 0,
"inputs": [
{
@@ -199,7 +359,7 @@
{
"name": "images",
"type": "IMAGE",
"link": 893
"link": 896
}
],
"outputs": [
@@ -226,200 +386,15 @@
"bgcolor": "#535"
},
{
"id": 278,
"type": "FILM VFI",
"id": 201,
"type": "UpscaleModelLoader",
"pos": [
912,
159
],
"size": [
443.4000244140625,
126
],
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "frames",
"type": "IMAGE",
"link": 890
},
{
"name": "optional_interpolation_states",
"shape": 7,
"type": "INTERPOLATION_STATES",
"link": null
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"slot_index": 0,
"links": [
891
]
}
],
"properties": {
"cnr_id": "comfyui-frame-interpolation",
"ver": "18b7cce5b08290741929e39d4955b88382db4e1d",
"Node name for S&R": "FILM VFI"
},
"widgets_values": [
"film_net_fp32.pt",
20,
2
]
},
{
"id": 279,
"type": "VHS_VideoCombine",
"pos": [
911.143798828125,
449.63348388671875
],
"size": [
534.675537109375,
358
],
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 891
},
{
"name": "audio",
"shape": 7,
"type": "AUDIO",
"link": null
},
{
"name": "meta_batch",
"shape": 7,
"type": "VHS_BatchManager",
"link": null
},
{
"name": "vae",
"shape": 7,
"type": "VAE",
"link": null
}
],
"outputs": [
{
"name": "Filenames",
"type": "VHS_FILENAMES",
"links": null
}
],
"properties": {
"cnr_id": "comfyui-videohelpersuite",
"ver": "5e61bcf218fe3bb7c899bbd584bbc99a9d05fb42",
"Node name for S&R": "VHS_VideoCombine"
},
"widgets_values": {
"frame_rate": 30,
"loop_count": 0,
"filename_prefix": "Hunyuan/%date:yyyy_MM_dd%/interpolate/vid",
"format": "video/nvenc_h264-mp4",
"pix_fmt": "yuv420p",
"bitrate": 10,
"megabit": true,
"save_metadata": true,
"pingpong": false,
"save_output": true,
"save_images": true,
"videopreview": {
"hidden": false,
"paused": false,
"params": {
"filename": "vid_00001.mp4",
"subfolder": "Hunyuan\\2025_03_19\\interpolate",
"type": "output",
"format": "video/nvenc_h264-mp4",
"frame_rate": 30,
"workflow": "vid_00001.png",
"fullpath": "M:\\Projects\\ComfyUI_windows_portable\\ComfyUI\\output\\Hunyuan\\2025_03_19\\interpolate\\vid_00001.mp4"
}
}
}
},
{
"id": 280,
"type": "WarpedImageResize",
"pos": [
-24.520009994506836,
175.57980346679688
381.0227966308594,
339.8635559082031
],
"size": [
315,
190
],
"flags": {},
"order": 4,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 895
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
893
]
},
{
"name": "scale_orig_image",
"type": "IMAGE",
"links": null
},
{
"name": "width",
"type": "INT",
"links": null
},
{
"name": "height",
"type": "INT",
"links": null
}
],
"properties": {
"cnr_id": "ComfyUI-WarpedToolset",
"ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb",
"Node name for S&R": "WarpedImageResize"
},
"widgets_values": [
416,
640,
"lanczos",
"center"
],
"color": "#223",
"bgcolor": "#335"
},
{
"id": 281,
"type": "WarpedBundleAllVideoImages",
"pos": [
-436.52703857421875,
177.63095092773438
],
"size": [
378,
202
58
],
"flags": {},
"order": 2,
@@ -427,56 +402,22 @@
"inputs": [],
"outputs": [
{
"name": "image_batch",
"type": "IMAGE",
"name": "UPSCALE_MODEL",
"type": "UPSCALE_MODEL",
"slot_index": 0,
"links": [
894,
895
885
]
},
{
"name": "first_image",
"type": "IMAGE",
"links": null
},
{
"name": "last_image",
"type": "IMAGE",
"links": null
},
{
"name": "num_frames",
"type": "INT",
"links": null
},
{
"name": "width",
"type": "INT",
"links": null
},
{
"name": "height",
"type": "INT",
"links": null
},
{
"name": "fps",
"type": "FLOAT",
"links": null
}
],
"properties": {
"cnr_id": "ComfyUI-WarpedToolset",
"ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb",
"widget_ue_connectable": {},
"Node name for S&R": "WarpedBundleAllVideoImages"
"cnr_id": "comfy-core",
"ver": "0.3.27",
"Node name for S&R": "UpscaleModelLoader"
},
"widgets_values": [
"",
true
],
"color": "#323",
"bgcolor": "#535"
"4x_foolhardy_Remacri.pth"
]
}
],
"links": [
@@ -512,14 +453,6 @@
0,
"IMAGE"
],
[
893,
280,
0,
276,
1,
"IMAGE"
],
[
894,
281,
@@ -529,23 +462,36 @@
"IMAGE"
],
[
895,
896,
281,
0,
280,
0,
276,
1,
"IMAGE"
]
],
"groups": [
{
"id": 2,
"title": "Hunyuan High Resolution",
"title": "High Resolution Upscale With Model",
"bounding": [
-479.1864318847656,
-109.82951354980469,
1971.108642578125,
1907.701904296875
-230.2893829345703,
-103.17450714111328,
1165.8536376953125,
1875.7579345703125
],
"color": "#3f789e",
"font_size": 24,
"flags": {}
},
{
"id": 3,
"title": "Interpolate",
"bounding": [
943.718505859375,
-107.20301055908203,
734.95703125,
1895.4840087890625
],
"color": "#3f789e",
"font_size": 24,
@@ -557,14 +503,15 @@
"ds": {
"scale": 0.7513148009015777,
"offset": [
762.6220000000011,
244.2959999999923
455.1610000000002,
-103.0950000000065
]
},
"frontendVersion": "1.20.5",
"groupNodes": {},
"ue_links": [],
"VHS_latentpreview": true,
"VHS_latentpreviewrate": 10,
"VHS_latentpreviewrate": 15,
"node_versions": {
"comfy-core": "0.3.26",
"comfyui-kjnodes": "e3b4bcf21e03402a2b11927eb5a4e3df500795b7",
Sample_Workflows/Warped WAN I2V Workflow.json
+312 -312
View File
@@ -1,79 +1,9 @@
{
"id": "2fac44f1-8935-4d17-8233-fe2dcaee9ba8",
"revision": 0,
"last_node_id": 140,
"last_link_id": 296,
"last_node_id": 142,
"last_link_id": 303,
"nodes": [
{
"id": 38,
"type": "CLIPLoader",
"pos": [
-68.24571228027344,
51.294097900390625
],
"size": [
376.05950927734375,
106
],
"flags": {},
"order": 0,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "CLIP",
"type": "CLIP",
"slot_index": 0,
"links": [
214
]
}
],
"properties": {
"cnr_id": "comfy-core",
"ver": "0.3.20",
"Node name for S&R": "CLIPLoader"
},
"widgets_values": [
"t5xxl_um_fp8_e4m3fn_scaled.safetensors",
"wan",
"default"
]
},
{
"id": 55,
"type": "LoaderGGUF",
"pos": [
-71.74114227294922,
-65.37071228027344
],
"size": [
380.0403137207031,
67.37849426269531
],
"flags": {},
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"slot_index": 0,
"links": [
213
]
}
],
"properties": {
"cnr_id": "gguf",
"ver": "1.5.8",
"Node name for S&R": "LoaderGGUF"
},
"widgets_values": [
"wan2.1-i2v-14b-720p-q5_k_m.gguf"
]
},
{
"id": 7,
"type": "CLIPTextEncode",
@@ -86,7 +16,7 @@
90.64099884033203
],
"flags": {},
"order": 12,
"order": 15,
"mode": 0,
"inputs": [
{
@@ -209,13 +139,13 @@
134.78460693359375
],
"flags": {},
"order": 7,
"order": 9,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "MODEL",
"link": 213
"link": 297
},
{
"name": "clip",
@@ -268,7 +198,7 @@
"flags": {
"collapsed": true
},
"order": 2,
"order": 0,
"mode": 0,
"inputs": [],
"outputs": [
@@ -304,7 +234,7 @@
"flags": {
"collapsed": true
},
"order": 10,
"order": 13,
"mode": 0,
"inputs": [
{
@@ -346,7 +276,7 @@
284.9806823730469
],
"flags": {},
"order": 11,
"order": 14,
"mode": 0,
"inputs": [
{
@@ -621,7 +551,7 @@
493.4667053222656
],
"flags": {},
"order": 3,
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
@@ -631,7 +561,7 @@
"slot_index": 0,
"links": [
186,
283
298
]
},
{
@@ -663,13 +593,13 @@
517.2369384765625
],
"flags": {},
"order": 14,
"order": 11,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 287
"link": 300
}
],
"outputs": [],
@@ -680,172 +610,6 @@
},
"widgets_values": []
},
{
"id": 118,
"type": "WarpedImageNoiseAugmentation",
"pos": [
-390.6877746582031,
1235.955810546875
],
"size": [
390.5999755859375,
126
],
"flags": {},
"order": 13,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 284
}
],
"outputs": [
{
"name": "image",
"type": "IMAGE",
"slot_index": 0,
"links": [
267
]
},
{
"name": "seed",
"type": "INT",
"slot_index": 1,
"links": [
243,
276
]
}
],
"properties": {
"cnr_id": "ComfyUI-WarpedToolset",
"ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb",
"Node name for S&R": "WarpedImageNoiseAugmentation"
},
"widgets_values": [
0.3,
661031532765305,
"fixed"
],
"color": "#323",
"bgcolor": "#535"
},
{
"id": 137,
"type": "WarpedImageResize",
"pos": [
-391.8274230957031,
970.8604125976562
],
"size": [
315,
214
],
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 283
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
284,
287,
288
]
},
{
"name": "scale_orig_image",
"type": "IMAGE",
"links": null
},
{
"name": "width",
"type": "INT",
"links": [
285
]
},
{
"name": "height",
"type": "INT",
"links": [
286
]
}
],
"properties": {
"cnr_id": "ComfyUI-WarpedToolset",
"ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb",
"Node name for S&R": "WarpedImageResize"
},
"widgets_values": [
400,
720,
"lanczos",
"center",
false
],
"color": "#323",
"bgcolor": "#535"
},
{
"id": 51,
"type": "CLIPVisionEncode",
"pos": [
-72.99646759033203,
694.0908813476562
],
"size": [
253.60000610351562,
78
],
"flags": {
"collapsed": true
},
"order": 15,
"mode": 0,
"inputs": [
{
"name": "clip_vision",
"type": "CLIP_VISION",
"link": 289
},
{
"name": "image",
"type": "IMAGE",
"link": 288
}
],
"outputs": [
{
"name": "CLIP_VISION_OUTPUT",
"type": "CLIP_VISION_OUTPUT",
"slot_index": 0,
"links": [
266
]
}
],
"properties": {
"cnr_id": "comfy-core",
"ver": "0.3.20",
"Node name for S&R": "CLIPVisionEncode"
},
"widgets_values": [
"none"
]
},
{
"id": 134,
"type": "WarpedWanImageToVideo",
@@ -894,7 +658,7 @@
"widget": {
"name": "width"
},
"link": 285
"link": 302
},
{
"name": "height",
@@ -902,7 +666,7 @@
"widget": {
"name": "height"
},
"link": 286
"link": 303
}
],
"outputs": [
@@ -966,7 +730,7 @@
"flags": {
"collapsed": true
},
"order": 4,
"order": 2,
"mode": 0,
"inputs": [],
"outputs": [
@@ -1004,7 +768,7 @@
"flags": {
"collapsed": true
},
"order": 5,
"order": 3,
"mode": 0,
"inputs": [],
"outputs": [
@@ -1048,7 +812,7 @@
"flags": {
"collapsed": true
},
"order": 6,
"order": 4,
"mode": 0,
"inputs": [
{
@@ -1094,7 +858,7 @@
"flags": {
"collapsed": true
},
"order": 9,
"order": 8,
"mode": 0,
"inputs": [
{
@@ -1121,6 +885,242 @@
"widgets_values": [
12
]
},
{
"id": 141,
"type": "WarpedLoaderGGUF",
"pos": [
-62.516021728515625,
-49.970008850097656
],
"size": [
282.2144470214844,
58
],
"flags": {
"collapsed": true
},
"order": 5,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
297
]
}
],
"properties": {
"cnr_id": "comfyui-warpedtoolset",
"ver": "1.7.0",
"Node name for S&R": "WarpedLoaderGGUF"
},
"widgets_values": [
"Wan2.1_I2V_14B_FusionX-Q4_K_M.gguf"
],
"color": "#323",
"bgcolor": "#535"
},
{
"id": 51,
"type": "CLIPVisionEncode",
"pos": [
-72.99646759033203,
694.0908813476562
],
"size": [
253.60000610351562,
78
],
"flags": {
"collapsed": false
},
"order": 12,
"mode": 0,
"inputs": [
{
"name": "clip_vision",
"type": "CLIP_VISION",
"link": 289
},
{
"name": "image",
"type": "IMAGE",
"link": 301
}
],
"outputs": [
{
"name": "CLIP_VISION_OUTPUT",
"type": "CLIP_VISION_OUTPUT",
"slot_index": 0,
"links": [
266
]
}
],
"properties": {
"cnr_id": "comfy-core",
"ver": "0.3.20",
"Node name for S&R": "CLIPVisionEncode"
},
"widgets_values": [
"none"
]
},
{
"id": 142,
"type": "ImageResizeKJv2",
"pos": [
-111.76309204101562,
837.80712890625
],
"size": [
270,
242
],
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 298
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
299,
300,
301
]
},
{
"name": "width",
"type": "INT",
"links": [
302
]
},
{
"name": "height",
"type": "INT",
"links": [
303
]
}
],
"properties": {
"cnr_id": "comfyui-kjnodes",
"ver": "1.1.0",
"Node name for S&R": "ImageResizeKJv2"
},
"widgets_values": [
400,
720,
"lanczos",
"resize",
"0, 0, 0",
"center",
2
]
},
{
"id": 118,
"type": "WarpedImageNoiseAugmentation",
"pos": [
-232.29910278320312,
1134.79833984375
],
"size": [
390.5999755859375,
126
],
"flags": {},
"order": 10,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 299
}
],
"outputs": [
{
"name": "image",
"type": "IMAGE",
"slot_index": 0,
"links": [
267
]
},
{
"name": "seed",
"type": "INT",
"slot_index": 1,
"links": [
243,
276
]
}
],
"properties": {
"cnr_id": "ComfyUI-WarpedToolset",
"ver": "4e08ee52f8f8f1b9b672937866812d63bbb520eb",
"Node name for S&R": "WarpedImageNoiseAugmentation"
},
"widgets_values": [
0.3,
661031532765305,
"fixed"
],
"color": "#323",
"bgcolor": "#535"
},
{
"id": 38,
"type": "CLIPLoader",
"pos": [
-69.57671356201172,
16.68808937072754
],
"size": [
376.05950927734375,
106
],
"flags": {
"collapsed": true
},
"order": 6,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "CLIP",
"type": "CLIP",
"slot_index": 0,
"links": [
214
]
}
],
"properties": {
"cnr_id": "comfy-core",
"ver": "0.3.20",
"Node name for S&R": "CLIPLoader"
},
"widgets_values": [
"t5xxl_um_fp8_e4m3fn_scaled.safetensors",
"wan",
"default"
]
}
],
"links": [
@@ -1132,14 +1132,6 @@
1,
"CONDITIONING"
],
[
213,
55,
0,
116,
0,
"MODEL"
],
[
214,
38,
@@ -1292,54 +1284,6 @@
0,
"IMAGE"
],
[
283,
52,
0,
137,
0,
"IMAGE"
],
[
284,
137,
0,
118,
0,
"IMAGE"
],
[
285,
137,
2,
134,
5,
"INT"
],
[
286,
137,
3,
134,
6,
"INT"
],
[
287,
137,
0,
58,
0,
"IMAGE"
],
[
288,
137,
0,
51,
1,
"IMAGE"
],
[
289,
138,
@@ -1371,6 +1315,62 @@
110,
0,
"MODEL"
],
[
297,
141,
0,
116,
0,
"MODEL"
],
[
298,
52,
0,
142,
0,
"IMAGE"
],
[
299,
142,
0,
118,
0,
"IMAGE"
],
[
300,
142,
0,
58,
0,
"IMAGE"
],
[
301,
142,
0,
51,
1,
"IMAGE"
],
[
302,
142,
1,
134,
5,
"INT"
],
[
303,
142,
2,
134,
6,
"INT"
]
],
"groups": [],
@@ -1379,8 +1379,8 @@
"ds": {
"scale": 0.7513148009015777,
"offset": [
159.67900000000375,
451.9319999999936
473.7950000000054,
-276.1250000000068
]
},
"frontendVersion": "1.20.5",
__init__.py
+17 -6
View File
@@ -1,4 +1,5 @@
from .nodes import *
from .gguf_nodes import WarpedLoaderGGUF, WarpedClipLoaderGGUF, WarpedDualClipLoaderGGUF
NODE_CLASS_MAPPINGS = {
"WarpedHunyuanMultiLoraMerge": WarpedHunyuanMultiLoraMerge,
@@ -6,13 +7,13 @@ NODE_CLASS_MAPPINGS = {
"WarpedHunyuanLoraMerge": WarpedHunyuanLoraMerge,
"WarpedHunyuanMultiLoraMixer": WarpedHunyuanMultiLoraMixer,
"WarpedHunyuanMultiLoraMixerExt": WarpedHunyuanMultiLoraMixerExt,
# "WarpedHunyuanMultiLoraMixerExt2": WarpedHunyuanMultiLoraMixerExt2,
"WarpedHunyuanLoraAvgMerge": WarpedHunyuanLoraAvgMerge,
"WarpedHunyuanLoraBatchMerge": WarpedHunyuanLoraBatchMerge,
"WarpedWanLoraMerge": WarpedWanLoraMerge,
"WarpedSamplerCustomAdv": WarpedSamplerCustomAdv,
"WarpedSamplerCustomAdvLatent": WarpedSamplerCustomAdvLatent,
# "WarpedAdvancedSamplerBatch": WarpedAdvancedSamplerBatch,
# "WarpedSamplerCustomBatch": WarpedSamplerCustomBatch,
# "WarpedSamplerCustomScripted": WarpedSamplerCustomScripted,
"WarpedFramepackSampler": WarpedFramepackSampler,
"WarpedCreateSpecialImageBatch": WarpedCreateSpecialImageBatch,
"WarpedCreateEmptyImageBatch": WarpedCreateEmptyImageBatch,
@@ -28,7 +29,6 @@ NODE_CLASS_MAPPINGS = {
"WarpedLoadVideosBatch": WarpedLoadVideosBatch,
"WarpedWanImageToVideo": WarpedWanImageToVideo,
"WarpedWanLoadAndEditLoraBlocks": WarpedWanLoadAndEditLoraBlocks,
# "WarpedImageResize": WarpedImageResize,
"WarpedImageScaleToSide": WarpedImageScaleToSide,
"WarpedLoadLorasBatchByPrefix": WarpedLoadLorasBatchByPrefix,
"WarpedHunyuanVideoLoraLoader": WarpedHunyuanVideoLoraLoader,
@@ -39,6 +39,7 @@ NODE_CLASS_MAPPINGS = {
"WarpedFramepackLoraSelectBatch": WarpedFramepackLoraSelectBatch,
"WarpedFramepackMultiLoraSelect": WarpedFramepackMultiLoraSelect,
"WarpedFramepackMultiLoraSelectExt": WarpedFramepackMultiLoraSelectExt,
"WarpedCLIPLoader": WarpedCLIPLoader,
"WarpedDualCLIPLoader": WarpedDualCLIPLoader,
"WarpedDualEncoder": WarpedDualEncoder,
"WarpedCLIPVisionLoader": WarpedCLIPVisionLoader,
@@ -50,6 +51,11 @@ NODE_CLASS_MAPPINGS = {
"WarpedReverseImageBatch": WarpedReverseImageBatch,
"WarpedMultiLoraLoader": WarpedMultiLoraLoader,
"WarpedHunyuanImageToVideo": WarpedHunyuanImageToVideo,
"WarpedBasicGuider": WarpedBasicGuider,
"WarpedDualGuider": WarpedDualGuider,
"WarpedLoaderGGUF": WarpedLoaderGGUF,
"WarpedClipLoaderGGUF": WarpedClipLoaderGGUF,
"WarpedDualClipLoaderGGUF": WarpedDualClipLoaderGGUF,
}
NODE_DISPLAY_NAME_MAPPINGS = {
@@ -58,13 +64,13 @@ NODE_DISPLAY_NAME_MAPPINGS = {
"WarpedHunyuanLoraMerge": "Warped Hunyuan Lora Merge",
"WarpedHunyuanMultiLoraMixer": "Warped Hunyuan Multi Lora Mixer",
"WarpedHunyuanMultiLoraMixerExt": "Warped Hunyuan Multi Lora Mixer Ext",
# "WarpedHunyuanMultiLoraMixerExt2": "Warped Hunyuan Multi Lora Mixer Ext 2",
"WarpedHunyuanLoraAvgMerge": "Warped Hunyuan Lora Avg Merge",
"WarpedHunyuanLoraBatchMerge": "Warped Hunyuan Lora Batch Merge",
"WarpedWanLoraMerge": "Warped Wan Lora Merge",
"WarpedSamplerCustomAdv": "Warped Sampler Custom Advanced (Image)",
"WarpedSamplerCustomAdvLatent": "Warped Sampler Custom Advanced (Latent)",
# "WarpedAdvancedSamplerBatch": "Warped Advanced Sampler Batch",
# "WarpedSamplerCustomBatch": "Warped Sampler Custom Batch",
# "WarpedSamplerCustomScripted": "Warped Sampler Custom Scripted",
"WarpedFramepackSampler": "Warped Framepack Sampler",
"WarpedCreateSpecialImageBatch": "Warped Create Special Image Batch",
"WarpedCreateEmptyImageBatch": "Warped Create Empty Image Batch",
@@ -80,7 +86,6 @@ NODE_DISPLAY_NAME_MAPPINGS = {
"WarpedLoadVideosBatch": "Warped Load Videos Batch",
"WarpedWanImageToVideo": "Warped Wan Image To Video",
"WarpedWanLoadAndEditLoraBlocks": "Warped Wan Load And Edit Lora Blocks",
# "WarpedImageResize": "Warped Image Resize",
"WarpedImageScaleToSide": "Warped Image Scale To Side",
"WarpedLoadLorasBatchByPrefix": "Warped Load Loras Batch By Prefix",
"WarpedHunyuanVideoLoraLoader": "Warped Hunyuan Video Lora Loader",
@@ -91,6 +96,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
"WarpedFramepackLoraSelectBatch": "Warped Framepack Lora Select Batch",
"WarpedFramepackMultiLoraSelect": "Warped Framepack Multi Lora Select",
"WarpedFramepackMultiLoraSelectExt": "Warped Framepack Multi Lora Select Ext",
"WarpedCLIPLoader": "Warped CLIP Loader",
"WarpedDualCLIPLoader": "Warped Dual CLIP Loader",
"WarpedDualEncoder": "Warped Dual Encoder",
"WarpedCLIPVisionLoader": "Warped Load CLIP Vision",
@@ -102,6 +108,11 @@ NODE_DISPLAY_NAME_MAPPINGS = {
"WarpedReverseImageBatch": "Warped Reverse Image Batch",
"WarpedMultiLoraLoader": "Warped Multi Lora Loader",
"WarpedHunyuanImageToVideo": "Warped Hunyuan Image To Video",
"WarpedBasicGuider": "Warped Basic Guider Batch",
"WarpedDualGuider": "Warped Dual Guider",
"WarpedLoaderGGUF": "Warped UNET Loader GGUF",
"WarpedClipLoaderGGUF": "Warped Clip Loader GGUF",
"WarpedDualClipLoaderGGUF": "Warped Dual Clip Loader GGUF",
}
WEB_DIRECTORY = "./web"
gguf_connector/__init__.py
+192
View File
@@ -0,0 +1,192 @@
__version__ = '1.7.8'
def __init__():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('-v', '--version', action='version', version=
'%(prog)s ' + __version__)
subparsers = parser.add_subparsers(title='subcommands', dest=
'subcommand', help='choose a subcommand:')
subparsers.add_parser('cpp', help='[cpp] connector cpp')
subparsers.add_parser('gpp', help='[gpp] connector gpp')
subparsers.add_parser('v', help='[v] vision connector')
subparsers.add_parser('g', help='[g] cli connector g')
subparsers.add_parser('c', help='[c] gui connector c')
subparsers.add_parser('m', help='[m] menu')
subparsers.add_parser('o', help='[o] org web mirror')
subparsers.add_parser('i', help='[i] i/o web mirror')
subparsers.add_parser('w', help='[w] page/container')
subparsers.add_parser('y', help='[y] download comfy')
subparsers.add_parser('n', help='[n] clone node')
subparsers.add_parser('u', help='[u] get cutter')
subparsers.add_parser('p', help='[p] take pack')
subparsers.add_parser('p1', help='[p1] take framepack')
subparsers.add_parser('p2', help='[p2] take packpack')
subparsers.add_parser('r', help='[r] metadata reader')
subparsers.add_parser('r2', help='[r2] metadata fast reader')
subparsers.add_parser('r3', help='[r3] tensor reader')
subparsers.add_parser('r4', help='[r4] tensor info reader')
subparsers.add_parser('e', help='[e] weight extractor')
subparsers.add_parser('q', help='[q] tensor quantizor')
subparsers.add_parser('q1', help='[q1] tensor quantizor (cpu)')
subparsers.add_parser('q2', help='[q2] tensor quantizor (upscale)')
subparsers.add_parser('d', help='[d] divider (safetensors)')
subparsers.add_parser('d2', help='[d2] divider (gguf)')
subparsers.add_parser('m2', help='[m2] merger (gguf)')
subparsers.add_parser('ma', help='[ma] merger (safetensors)')
subparsers.add_parser('s1', help='[s1] splitter (uni/multi)')
subparsers.add_parser('s0', help='[s0] splitter (d2 to d5)')
subparsers.add_parser('s5', help='[s5] splitter (d5 else)')
subparsers.add_parser('sx', help='[sx] splitter (d1 to dx)')
subparsers.add_parser('sy', help='[sy] splitter (d2 else)')
subparsers.add_parser('s', help='[s] splitter (checkpoint)')
subparsers.add_parser('f', help='[f] tensor transfer')
subparsers.add_parser('t', help='[t] tensor convertor')
subparsers.add_parser('t0', help='[t0] tensor convertor (zero)')
subparsers.add_parser('t1', help='[t1] tensor convertor (alpha)')
subparsers.add_parser('t2', help='[t2] tensor convertor (beta)')
subparsers.add_parser('t3', help='[t3] tensor convertor (gamma)')
subparsers.add_parser('t4', help='[t4] tensor convertor (delta)')
subparsers.add_parser('t5', help='[t5] tensor convertor (epsilon)')
subparsers.add_parser('t6', help='[t6] tensor convertor (zeta)')
subparsers.add_parser('t7', help='[t7] tensor convertor (eta)')
subparsers.add_parser('t8', help='[t8] tensor convertor (theta)')
subparsers.add_parser('t9', help='[t9] tensor convertor (iota)')
subparsers.add_parser('d5', help='[d5] dimension 5 fixer (t8x)')
subparsers.add_parser('d6', help='[d6] tensor convertor (t8xx)')
subparsers.add_parser('d7', help='[d7] tensor convertor (plus)')
subparsers.add_parser('pp', help='[pp] pdf analyzor pp')
subparsers.add_parser('cp', help='[cp] pdf analyzor cp')
subparsers.add_parser('ps', help='[ps] wav recognizor ps')
subparsers.add_parser('cs', help='[cs] wav recognizor cs')
subparsers.add_parser('cg', help='[cg] wav recognizor cg (api)')
subparsers.add_parser('pg', help='[pg] wav recognizor pg (api)')
subparsers.add_parser('vg', help='[vg] video generator')
subparsers.add_parser('v1', help='[v1] video 1 (i2v)')
subparsers.add_parser('v2', help='[v2] video 2 (t2v)')
subparsers.add_parser('i2', help='[i2] image 2 (t2i)')
subparsers.add_parser('s2', help='[s2] voice 2 (t2s)')
subparsers.add_parser('b1', help='[b1] bagel 1 (old)')
subparsers.add_parser('b2', help='[b2] bagel 2 (a2a)')
args = parser.parse_args()
if args.subcommand == 'm':
from gguf_connector import m
if args.subcommand == 'n':
from gguf_connector import n
if args.subcommand == 'f':
from gguf_connector import f
if args.subcommand == 'p':
from gguf_connector import p
if args.subcommand == 'p1':
from gguf_connector import p1
if args.subcommand == 'p2':
from gguf_connector import p2
elif args.subcommand == 'r':
from gguf_connector import r
elif args.subcommand == 'r2':
from gguf_connector import r2
elif args.subcommand == 'r3':
from gguf_connector import r3
elif args.subcommand == 'r4':
from gguf_connector import r4
elif args.subcommand == 'e':
from gguf_connector import e
elif args.subcommand == 's':
from gguf_connector import s
elif args.subcommand == 's1':
from gguf_connector import s1
elif args.subcommand == 's0':
from gguf_connector import s0
elif args.subcommand == 's5':
from gguf_connector import s5
elif args.subcommand == 'sx':
from gguf_connector import sx
elif args.subcommand == 'sy':
from gguf_connector import sy
elif args.subcommand == 'i':
from gguf_connector import i
elif args.subcommand == 'o':
from gguf_connector import o
elif args.subcommand == 'u':
from gguf_connector import u
elif args.subcommand == 'v':
from gguf_connector import v
elif args.subcommand == 'vg':
from gguf_connector import vg
elif args.subcommand == 'v1':
from gguf_connector import vg2
elif args.subcommand == 'v2':
from gguf_connector import v2
elif args.subcommand == 'i2':
from gguf_connector import i2
elif args.subcommand == 's2':
from gguf_connector import s2
elif args.subcommand == 'b2':
from gguf_connector import b2
elif args.subcommand == 'b1':
from gguf_connector import b1
elif args.subcommand == 'w':
from gguf_connector import w
elif args.subcommand == 'y':
from gguf_connector import y
elif args.subcommand == 't':
from gguf_connector import t
elif args.subcommand == 't0':
from gguf_connector import t0
elif args.subcommand == 't1':
from gguf_connector import t1
elif args.subcommand == 't2':
from gguf_connector import t2
elif args.subcommand == 't3':
from gguf_connector import t3
elif args.subcommand == 't4':
from gguf_connector import t4
elif args.subcommand == 't5':
from gguf_connector import t5
elif args.subcommand == 't6':
from gguf_connector import t6
elif args.subcommand == 't7':
from gguf_connector import t7
elif args.subcommand == 't8':
from gguf_connector import t8
elif args.subcommand == 't9':
from gguf_connector import t9
elif args.subcommand == 'd5':
from gguf_connector import d5
elif args.subcommand == 'd6':
from gguf_connector import d6
elif args.subcommand == 'd7':
from gguf_connector import d7
elif args.subcommand == 'q':
from gguf_connector import q
elif args.subcommand == 'q1':
from gguf_connector import q1
elif args.subcommand == 'q2':
from gguf_connector import q2
elif args.subcommand == 'd':
from gguf_connector import d
elif args.subcommand == 'd2':
from gguf_connector import d2
elif args.subcommand == 'm2':
from gguf_connector import m2
elif args.subcommand == 'ma':
from gguf_connector import ma
elif args.subcommand == 'cg':
from gguf_connector import cg
elif args.subcommand == 'pg':
from gguf_connector import pg
elif args.subcommand == 'cs':
from gguf_connector import cs
elif args.subcommand == 'ps':
from gguf_connector import ps
elif args.subcommand == 'cp':
from gguf_connector import cp
elif args.subcommand == 'pp':
from gguf_connector import pp
elif args.subcommand == 'c':
from gguf_connector import c
elif args.subcommand == 'cpp':
from gguf_connector import cpp
elif args.subcommand == 'g':
from gguf_connector import g
elif args.subcommand == 'gpp':
from gguf_connector import gpp
gguf_connector/b1.py
+630
View File
@@ -0,0 +1,630 @@
import os
if not os.path.isfile(os.path.join(os.path.dirname(__file__),
'models/bagel/config.json')):
from huggingface_hub import snapshot_download
save_dir = os.path.join(os.path.dirname(__file__), 'models/bagel')
repo_id = 'callgg/bagel-decoder'
cache_dir = save_dir + '/cache'
snapshot_download(cache_dir=cache_dir, local_dir=save_dir, repo_id=
repo_id, allow_patterns=['*.json', '*.md', '*.txt'])
import time, psutil, platform, atexit, random
pynvml_available = False
if platform.system() == 'Linux' or platform.system() == 'Windows':
try:
from pynvml import *
nvmlInit()
pynvml_available = True
print(
"""pynvml (NVIDIA GPU monitoring library) initialized successfully!
Detecting Safetensors..
"""
)
def shutdown_pynvml():
print('Shutting down pynvml...')
nvmlShutdown()
atexit.register(shutdown_pynvml)
except Exception as e:
print(
f'Warning: pynvml could not be initialized. Detailed GPU stats via pynvml will not be available. Error: {e}'
)
if 'NVML Shared Library Not Found' in str(e):
print(
'pynvml error hint: NVML shared library not found. If you have an NVIDIA GPU and drivers, ensure the library is accessible.'
)
elif 'Driver Not Loaded' in str(e):
print(
'pynvml error hint: NVIDIA driver is not loaded. Please check your GPU driver installation.'
)
import torch
import gradio as gr
import numpy as np
from accelerate import infer_auto_device_map, load_checkpoint_and_dispatch, init_empty_weights
from PIL import Image
from bagel2.data.data_utils import add_special_tokens, pil_img2rgb
from bagel2.data.transforms import ImageTransform
from bagel2.inferencer import InterleaveInferencer
from bagel2.modeling.autoencoder import load_ae
from bagel2.modeling.bagel.qwen2_navit import NaiveCache
from bagel2.modeling.bagel import BagelConfig, Bagel, Qwen2Config, Qwen2ForCausalLM, SiglipVisionConfig, SiglipVisionModel
from bagel2.modeling.qwen2 import Qwen2Tokenizer
model_path = os.path.join(os.path.dirname(__file__), 'models/bagel')
llm_config = Qwen2Config.from_json_file(os.path.join(model_path,
'llm_config.json'))
llm_config.qk_norm = True
llm_config.tie_word_embeddings = False
llm_config.layer_module = 'Qwen2MoTDecoderLayer'
vit_config = SiglipVisionConfig.from_json_file(os.path.join(model_path,
'vit_config.json'))
vit_config.rope = False
vit_config.num_hidden_layers -= 1
safetensors_files = [file for file in os.listdir() if file.endswith(
'.safetensors')]
if safetensors_files:
print('Safetensors file(s) available. Select which one for VAE:')
for index, file_name in enumerate(safetensors_files, start=1):
print(f'{index}. {file_name}')
choice1 = input(f'Enter your choice (1 to {len(safetensors_files)}): ')
try:
choice_index = int(choice1) - 1
selected_file = safetensors_files[choice_index]
print(f'Safetensors file: {selected_file} is selected!\n')
vae_model, vae_config = load_ae(selected_file)
print('Safetensors file(s) available. Select which one for MODEL:')
for index, file_name in enumerate(safetensors_files, start=1):
print(f'{index}. {file_name}')
choice2 = input(f'Enter your choice (1 to {len(safetensors_files)}): ')
try:
choice_index = int(choice2) - 1
selected_file = safetensors_files[choice_index]
print(f'Model file: {selected_file} is selected!')
checkpoint = selected_file
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No safetensors files are available in the current directory.')
input('--- Press ENTER To Exit ---')
config = BagelConfig(visual_gen=True, visual_und=True, llm_config=
llm_config, vit_config=vit_config, vae_config=vae_config,
vit_max_num_patch_per_side=70, connector_act='gelu_pytorch_tanh',
latent_patch_size=2, max_latent_size=64)
with init_empty_weights():
language_model = Qwen2ForCausalLM(llm_config)
vit_model = SiglipVisionModel(vit_config)
model = Bagel(language_model, vit_model, config)
model.vit_model.vision_model.embeddings.convert_conv2d_to_linear(vit_config
, meta=True)
tokenizer = Qwen2Tokenizer.from_pretrained(model_path)
tokenizer, new_token_ids, _ = add_special_tokens(tokenizer)
vae_transform = ImageTransform(1024, 512, 16)
vit_transform = ImageTransform(980, 224, 14)
print('Starting model loading and device map configuration...')
def get_gpu_memory_stats_pynvml(device_id=0):
if not pynvml_available:
return f'GPU-{device_id} (pynvml): Not available.'
try:
handle = nvmlDeviceGetHandleByIndex(device_id)
info = nvmlDeviceGetMemoryInfo(handle)
total_gb = info.total / 1024 ** 3
used_gb = info.used / 1024 ** 3
return (
f'GPU-{device_id} (pynvml): Total: {total_gb:.2f} GB, Used (Overall): {used_gb:.2f} GB'
)
except NVMLError as e:
return f'GPU-{device_id} (pynvml) Error: {e}'
def get_gpu_memory_stats_pytorch(device_id=0):
if not torch.cuda.is_available():
return 'PyTorch: CUDA not available.'
if device_id < 0 or device_id >= torch.cuda.device_count():
return f'PyTorch GPU-{device_id}: Invalid device ID.'
allocated_gb = torch.cuda.memory_allocated(device_id) / 1024 ** 3
reserved_gb = torch.cuda.memory_reserved(device_id) / 1024 ** 3
total_capacity_str_pt = ''
if pynvml_available:
try:
handle = nvmlDeviceGetHandleByIndex(device_id)
info = nvmlDeviceGetMemoryInfo(handle)
total_gb_pt = info.total / 1024 ** 3
total_capacity_str_pt = f'Total Capacity: {total_gb_pt:.2f} GB, '
except NVMLError:
pass
return (
f'PyTorch GPU-{device_id}: {total_capacity_str_pt}Allocated: {allocated_gb:.2f} GB, Reserved: {reserved_gb:.2f} GB'
)
def get_system_ram_stats():
mem = psutil.virtual_memory()
total_gb = mem.total / 1024 ** 3
available_gb = mem.available / 1024 ** 3
used_gb = mem.used / 1024 ** 3
percent_used = mem.percent
return (
f'System RAM: Total: {total_gb:.2f} GB, Available: {available_gb:.2f} GB, Used (Overall): {used_gb:.2f} GB ({percent_used}%)'
)
def get_process_ram_stats():
process = psutil.Process(os.getpid())
mem_info = process.memory_info()
rss_gb = mem_info.rss / 1024 ** 3
return f'App Process RAM (RSS): {rss_gb:.2f} GB'
def get_all_memory_stats_for_gradio_display():
"""Prepare the string of memory/video memory statistics for Gradio display"""
stats_lines = []
if torch.cuda.is_available() and torch.cuda.device_count() > 0:
stats_lines.append('**GPU VRAM Usage:**')
for i in range(torch.cuda.device_count()):
stats_lines.append(get_gpu_memory_stats_pynvml(i))
stats_lines.append(get_gpu_memory_stats_pytorch(i))
if i < torch.cuda.device_count() - 1:
stats_lines.append('---')
else:
stats_lines.append(
'**GPU VRAM Usage:** CUDA not available or no GPUs found.')
stats_lines.append('\n**CPU RAM Usage:**')
stats_lines.append(get_system_ram_stats())
stats_lines.append(get_process_ram_stats())
return '\n'.join(s for s in stats_lines if s)
cpu_mem_for_offload = '16GiB'
gpu_mem_per_device = '24GiB'
max_memory_config = {i: gpu_mem_per_device for i in range(torch.cuda.
device_count())}
if torch.cuda.device_count() == 0:
max_memory_config['cpu'] = cpu_mem_for_offload
else:
max_memory_config['cpu'] = cpu_mem_for_offload
print(f'Using max_memory_config: {max_memory_config}')
device_map = infer_auto_device_map(model, max_memory=max_memory_config,
no_split_module_classes=['Bagel', 'Qwen2MoTDecoderLayer'])
print('Device map after infer_auto_device_map (with CPU budget):')
for k, v_map in device_map.items():
print(f' {k}: {v_map}')
same_device_modules = ['language_model.model.embed_tokens', 'time_embedder',
'latent_pos_embed', 'vae2llm', 'llm2vae', 'connector', 'vit_pos_embed']
if torch.cuda.device_count() > 0:
first_device_key = same_device_modules[0]
default_target_device = 'cuda:0'
first_module_target_device = device_map.get(first_device_key,
default_target_device)
print(
f'Target device for same_device_modules (based on {first_device_key}): {first_module_target_device}'
)
for k_module in same_device_modules:
if k_module in device_map:
if device_map[k_module] != first_module_target_device:
print(
f' Moving {k_module} from {device_map[k_module]} to {first_module_target_device} (same_device_modules)'
)
device_map[k_module] = first_module_target_device
else:
print(
f' Assigning {k_module} (from same_device_modules) to {first_module_target_device} as it was not in initial map.'
)
device_map[k_module] = first_module_target_device
elif torch.cuda.device_count() == 0 and 'cpu' in max_memory_config:
print('No CUDA devices found. Assigning same_device_modules to CPU.')
for k_module in same_device_modules:
device_map[k_module] = 'cpu'
print('Device map after same_device_modules logic:')
for k, v_map in device_map.items():
print(f' {k}: {v_map}')
keys_to_change_to_cpu = []
for module_name, device_assignment in device_map.items():
if device_assignment == 'disk':
keys_to_change_to_cpu.append(module_name)
if keys_to_change_to_cpu:
print(
f"Manually changing the following layers from 'disk' to 'cpu': {keys_to_change_to_cpu}"
)
for module_name in keys_to_change_to_cpu:
device_map[module_name] = 'cpu'
print('Final device_map before loading checkpoint (after disk override):')
for k, v_map in device_map.items():
print(f' {k}: {v_map}')
else:
print(
"No layers assigned to 'disk' by infer_auto_device_map, or they were already handled. Final device_map is as above."
)
print(
"""
Starting custom device_map modifications to maximize GPU utilization..."""
)
print('Device map state BEFORE custom modifications:')
for k_map_item, v_map_item in device_map.items():
print(f' {k_map_item}: {v_map_item}')
NUM_ADDITIONAL_LLM_LAYERS_TO_GPU = 5
TRY_MOVE_LLM_NORM_HEAD_TO_GPU = True
TRY_MOVE_VIT_MODEL_TO_GPU = False
moved_llm_layers_count = 0
if NUM_ADDITIONAL_LLM_LAYERS_TO_GPU > 0:
print(
f"""
Attempting to move up to {NUM_ADDITIONAL_LLM_LAYERS_TO_GPU} LLM layers (11 to {10 + NUM_ADDITIONAL_LLM_LAYERS_TO_GPU}) to GPU 0..."""
)
for i in range(NUM_ADDITIONAL_LLM_LAYERS_TO_GPU):
layer_idx = 11 + i
if layer_idx > 27:
print(
f' Reached max layer index (27). Stopped LLM layer promotion.'
)
break
layer_name = f'language_model.model.layers.{layer_idx}'
if device_map.get(layer_name) == 'cpu':
print(f" Promoting LLM layer '{layer_name}' from 'cpu' to GPU 0.")
device_map[layer_name] = 0
moved_llm_layers_count += 1
elif layer_name in device_map:
print(
f" LLM Layer '{layer_name}' is already on device '{device_map[layer_name]}'. Skipping promotion."
)
else:
print(
f" Warning: LLM Layer '{layer_name}' not found in device_map. Cannot promote."
)
print(
f'Successfully promoted {moved_llm_layers_count} LLM layers to GPU 0.')
else:
print(
"""
Skipping promotion of additional LLM layers based on NUM_ADDITIONAL_LLM_LAYERS_TO_GPU setting."""
)
if TRY_MOVE_LLM_NORM_HEAD_TO_GPU:
print(
"\nAttempting to move LLM 'norm' and 'lm_head' to GPU 0 (if on CPU)..."
)
llm_aux_modules = ['language_model.model.norm',
'language_model.model.lm_head']
for module_name in llm_aux_modules:
if device_map.get(module_name) == 'cpu':
print(f" Promoting '{module_name}' from 'cpu' to GPU 0.")
device_map[module_name] = 0
elif module_name in device_map:
print(
f" Module '{module_name}' is already on device '{device_map[module_name]}'. Skipping promotion."
)
else:
print(
f" Warning: Module '{module_name}' not found in device_map. Cannot promote."
)
else:
print(
"""
Skipping promotion of LLM 'norm' and 'lm_head' based on TRY_MOVE_LLM_NORM_HEAD_TO_GPU setting."""
)
if TRY_MOVE_VIT_MODEL_TO_GPU:
print("\nAttempting to move 'vit_model' to GPU 0 (if on CPU)...")
vit_module_name = 'vit_model'
if device_map.get(vit_module_name) == 'cpu':
print(f" Promoting '{vit_module_name}' from 'cpu' to GPU 0.")
device_map[vit_module_name] = 0
elif vit_module_name in device_map:
print(
f" Module '{vit_module_name}' is already on device '{device_map[vit_module_name]}'. Skipping promotion."
)
else:
print(
f" Warning: Module '{vit_module_name}' not found in device_map. Cannot promote."
)
else:
print(
"\nSkipping promotion of 'vit_model' based on TRY_MOVE_VIT_MODEL_TO_GPU setting."
)
print("""
Final device_map after all custom modifications:""")
for k_map_item, v_map_item in device_map.items():
print(f' {k_map_item}: {v_map_item}')
print('--- End of custom device_map modifications ---')
model = load_checkpoint_and_dispatch(model, checkpoint=checkpoint,
device_map=device_map, offload_buffers=True, offload_folder='offload',
dtype=torch.bfloat16, force_hooks=True).eval()
inferencer = InterleaveInferencer(model=model, vae_model=vae_model,
tokenizer=tokenizer, vae_transform=vae_transform, vit_transform=
vit_transform, new_token_ids=new_token_ids)
def set_seed(seed):
"""Set random seeds for reproducibility"""
if seed > 0:
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
return seed
def text_to_image(prompt, show_thinking=False, cfg_text_scale=4.0,
cfg_interval=0.4, timestep_shift=3.0, num_timesteps=50, cfg_renorm_min=
1.0, cfg_renorm_type='global', max_think_token_n=1024, do_sample=False,
text_temperature=0.3, seed=0, image_ratio='1:1'):
set_seed(seed)
if image_ratio == '1:1':
image_shapes = 1024, 1024
elif image_ratio == '4:3':
image_shapes = 768, 1024
elif image_ratio == '3:4':
image_shapes = 1024, 768
elif image_ratio == '16:9':
image_shapes = 576, 1024
elif image_ratio == '9:16':
image_shapes = 1024, 576
inference_hyper = dict(max_think_token_n=max_think_token_n if
show_thinking else 1024, do_sample=do_sample if show_thinking else
False, text_temperature=text_temperature if show_thinking else 0.3,
cfg_text_scale=cfg_text_scale, cfg_interval=[cfg_interval, 1.0],
timestep_shift=timestep_shift, num_timesteps=num_timesteps,
cfg_renorm_min=cfg_renorm_min, cfg_renorm_type=cfg_renorm_type,
image_shapes=image_shapes)
start_time = time.time()
result = inferencer(text=prompt, think=show_thinking, **inference_hyper)
end_time = time.time()
duration = end_time - start_time
duration_str = f'{duration:.2f} seconds'
print(f'The image takes time: {duration_str}')
return result['image'], result.get('text', None), duration_str
def image_understanding(image, prompt, show_thinking=False, do_sample=False,
text_temperature=0.3, max_new_tokens=512):
start_time = time.time()
if image is None:
return 'Please upload an image.'
if isinstance(image, np.ndarray):
image = Image.fromarray(image)
image = pil_img2rgb(image)
inference_hyper = dict(do_sample=do_sample, text_temperature=
text_temperature, max_think_token_n=max_new_tokens)
result = inferencer(image=image, text=prompt, think=show_thinking,
understanding_output=True, **inference_hyper)
end_time = time.time()
duration = end_time - start_time
duration_str = f'{duration:.2f} seconds'
print(f'The image takes time: {duration_str}')
return result['text'], duration_str
def edit_image(image, prompt, show_thinking=False, cfg_text_scale=4.0,
cfg_img_scale=2.0, cfg_interval=0.0, timestep_shift=3.0, num_timesteps=
50, cfg_renorm_min=1.0, cfg_renorm_type='text_channel',
max_think_token_n=1024, do_sample=False, text_temperature=0.3, seed=0):
set_seed(seed)
start_time = time.time()
if image is None:
return 'Please upload an image.', ''
if isinstance(image, np.ndarray):
image = Image.fromarray(image)
image = pil_img2rgb(image)
inference_hyper = dict(max_think_token_n=max_think_token_n if
show_thinking else 1024, do_sample=do_sample if show_thinking else
False, text_temperature=text_temperature if show_thinking else 0.3,
cfg_text_scale=cfg_text_scale, cfg_img_scale=cfg_img_scale,
cfg_interval=[cfg_interval, 1.0], timestep_shift=timestep_shift,
num_timesteps=num_timesteps, cfg_renorm_min=cfg_renorm_min,
cfg_renorm_type=cfg_renorm_type)
result = inferencer(image=image, text=prompt, think=show_thinking, **
inference_hyper)
end_time = time.time()
duration = end_time - start_time
duration_str = f'{duration:.2f} seconds'
print(f'The image takes time: {duration_str}')
return result['image'], result.get('text', ''), duration_str
def load_example_image(image_path):
try:
return Image.open(image_path)
except Exception as e:
print(f'Error loading example image: {e}')
return None
block = gr.Blocks(title='gguf').queue()
with block:
with gr.Tab('📝 Text to Image'):
txt_input = gr.Textbox(label='Prompt', value=
'A female cosplayer portraying an ethereal fairy or elf, wearing a flowing dress made of delicate fabrics in soft, mystical colors like emerald green and silver. She has pointed ears, a gentle, enchanting expression, and her outfit is adorned with sparkling jewels and intricate patterns. The background is a magical forest with glowing plants, mystical creatures, and a serene atmosphere.'
)
with gr.Row():
show_thinking = gr.Checkbox(label='Thinking', value=False)
with gr.Accordion('Inference Hyperparameters', open=False):
with gr.Group():
with gr.Row():
seed = gr.Slider(minimum=0, maximum=1000000, value=0,
step=1, label='Seed', info=
'0 for random seed, positive for reproducible results')
image_ratio = gr.Dropdown(choices=['1:1', '4:3', '3:4',
'16:9', '9:16'], value='1:1', label='Image Ratio',
info='The longer size is fixed to 1024')
with gr.Row():
cfg_text_scale = gr.Slider(minimum=1.0, maximum=8.0,
value=4.0, step=0.1, interactive=True, label=
'CFG Text Scale', info=
'Controls how strongly the model follows the text prompt (4.0-8.0)'
)
cfg_interval = gr.Slider(minimum=0.0, maximum=1.0,
value=0.4, step=0.1, label='CFG Interval', info=
'Start of CFG application interval (end is fixed at 1.0)'
)
with gr.Row():
cfg_renorm_type = gr.Dropdown(choices=['global',
'local', 'text_channel'], value='global', label=
'CFG Renorm Type', info=
"If the genrated image is blurry, use 'global'")
cfg_renorm_min = gr.Slider(minimum=0.0, maximum=1.0,
value=0.0, step=0.1, interactive=True, label=
'CFG Renorm Min', info='1.0 disables CFG-Renorm')
with gr.Row():
num_timesteps = gr.Slider(minimum=10, maximum=100,
value=50, step=5, interactive=True, label=
'Timesteps', info='Total denoising steps')
timestep_shift = gr.Slider(minimum=1.0, maximum=5.0,
value=3.0, step=0.5, interactive=True, label=
'Timestep Shift', info=
'Higher values for layout, lower for details')
thinking_params = gr.Group(visible=False)
with thinking_params:
with gr.Row():
do_sample = gr.Checkbox(label='Sampling', value=
False, info='Enable sampling for text generation')
max_think_token_n = gr.Slider(minimum=64, maximum=
4006, value=1024, step=64, interactive=True,
label='Max Think Tokens', info=
'Maximum number of tokens for thinking')
text_temperature = gr.Slider(minimum=0.1, maximum=
1.0, value=0.3, step=0.1, interactive=True,
label='Temperature', info=
'Controls randomness in text generation')
thinking_output = gr.Textbox(label='Thinking Process', visible=False)
img_output = gr.Image(label='Generated Image')
gen_btn = gr.Button('Generate', variant='primary')
generation_time_output = gr.Textbox(label='Processing Time',
interactive=False)
def update_thinking_visibility(show):
return gr.update(visible=show), gr.update(visible=show)
show_thinking.change(fn=update_thinking_visibility, inputs=[
show_thinking], outputs=[thinking_output, thinking_params])
def process_text_to_image(prompt, show_thinking, cfg_text_scale,
cfg_interval, timestep_shift, num_timesteps, cfg_renorm_min,
cfg_renorm_type, max_think_token_n, do_sample, text_temperature,
seed, image_ratio):
image, thinking, duration_str = text_to_image(prompt,
show_thinking, cfg_text_scale, cfg_interval, timestep_shift,
num_timesteps, cfg_renorm_min, cfg_renorm_type,
max_think_token_n, do_sample, text_temperature, seed,
image_ratio)
return image, thinking if thinking else '', duration_str
gr.on(triggers=[gen_btn.click, txt_input.submit], fn=
process_text_to_image, inputs=[txt_input, show_thinking,
cfg_text_scale, cfg_interval, timestep_shift, num_timesteps,
cfg_renorm_min, cfg_renorm_type, max_think_token_n, do_sample,
text_temperature, seed, image_ratio], outputs=[img_output,
thinking_output, generation_time_output])
with gr.Tab('🖌️ Image Edit'):
with gr.Row():
with gr.Column(scale=1):
edit_image_input = gr.Image(label='Input Image', value=
load_example_image(os.path.join(os.path.dirname(
__file__), '../bagel2/test_images/women.jpg')))
edit_prompt = gr.Textbox(label='Prompt', value=
'She boards a modern subway, quietly reading a folded newspaper, wearing the same clothes.'
)
with gr.Column(scale=1):
edit_image_output = gr.Image(label='Result')
edit_thinking_output = gr.Textbox(label='Thinking Process',
visible=False)
with gr.Row():
edit_show_thinking = gr.Checkbox(label='Thinking', value=False)
with gr.Accordion('Inference Hyperparameters', open=False):
with gr.Group():
with gr.Row():
edit_seed = gr.Slider(minimum=0, maximum=1000000, value
=0, step=1, interactive=True, label='Seed', info=
'0 for random seed, positive for reproducible results')
edit_cfg_text_scale = gr.Slider(minimum=1.0, maximum=
8.0, value=4.0, step=0.1, interactive=True, label=
'CFG Text Scale', info=
'Controls how strongly the model follows the text prompt'
)
with gr.Row():
edit_cfg_img_scale = gr.Slider(minimum=1.0, maximum=4.0,
value=2.0, step=0.1, interactive=True, label=
'CFG Image Scale', info=
'Controls how much the model preserves input image details'
)
edit_cfg_interval = gr.Slider(minimum=0.0, maximum=1.0,
value=0.0, step=0.1, interactive=True, label=
'CFG Interval', info=
'Start of CFG application interval (end is fixed at 1.0)'
)
with gr.Row():
edit_cfg_renorm_type = gr.Dropdown(choices=['global',
'local', 'text_channel'], value='text_channel',
label='CFG Renorm Type', info=
"If the genrated image is blurry, use 'global")
edit_cfg_renorm_min = gr.Slider(minimum=0.0, maximum=
1.0, value=0.0, step=0.1, interactive=True, label=
'CFG Renorm Min', info='1.0 disables CFG-Renorm')
with gr.Row():
edit_num_timesteps = gr.Slider(minimum=10, maximum=100,
value=50, step=5, interactive=True, label=
'Timesteps', info='Total denoising steps')
edit_timestep_shift = gr.Slider(minimum=1.0, maximum=
10.0, value=3.0, step=0.5, interactive=True, label=
'Timestep Shift', info=
'Higher values for layout, lower for details')
edit_thinking_params = gr.Group(visible=False)
with edit_thinking_params:
with gr.Row():
edit_do_sample = gr.Checkbox(label='Sampling',
value=False, info=
'Enable sampling for text generation')
edit_max_think_token_n = gr.Slider(minimum=64,
maximum=4006, value=1024, step=64, interactive=
True, label='Max Think Tokens', info=
'Maximum number of tokens for thinking')
edit_text_temperature = gr.Slider(minimum=0.1,
maximum=1.0, value=0.3, step=0.1, interactive=
True, label='Temperature', info=
'Controls randomness in text generation')
edit_btn = gr.Button('Submit', variant='primary')
edit_generation_time_output = gr.Textbox(label='Processing Time',
interactive=False)
def update_edit_thinking_visibility(show):
return gr.update(visible=show), gr.update(visible=show)
edit_show_thinking.change(fn=update_edit_thinking_visibility,
inputs=[edit_show_thinking], outputs=[edit_thinking_output,
edit_thinking_params])
def process_edit_image(image, prompt, show_thinking, cfg_text_scale,
cfg_img_scale, cfg_interval, timestep_shift, num_timesteps,
cfg_renorm_min, cfg_renorm_type, max_think_token_n, do_sample,
text_temperature, seed):
edited_image, thinking, duration_str = edit_image(image, prompt,
show_thinking, cfg_text_scale, cfg_img_scale, cfg_interval,
timestep_shift, num_timesteps, cfg_renorm_min,
cfg_renorm_type, max_think_token_n, do_sample,
text_temperature, seed)
return edited_image, thinking if thinking else '', duration_str
gr.on(triggers=[edit_btn.click, edit_prompt.submit], fn=
process_edit_image, inputs=[edit_image_input, edit_prompt,
edit_show_thinking, edit_cfg_text_scale, edit_cfg_img_scale,
edit_cfg_interval, edit_timestep_shift, edit_num_timesteps,
edit_cfg_renorm_min, edit_cfg_renorm_type,
edit_max_think_token_n, edit_do_sample, edit_text_temperature,
edit_seed], outputs=[edit_image_output, edit_thinking_output,
edit_generation_time_output])
with gr.Tab('🖼️ Image Understanding'):
with gr.Row():
with gr.Column(scale=1):
img_input = gr.Image(label='Input Image', value=
load_example_image(os.path.join(os.path.dirname(
__file__), '../bagel2/test_images/meme.jpg')))
understand_prompt = gr.Textbox(label='Prompt', value=
"Can someone explain what's funny about this meme??")
with gr.Column(scale=1):
txt_output = gr.Textbox(label='Result', lines=20)
with gr.Row():
understand_show_thinking = gr.Checkbox(label='Thinking', value=
False)
with gr.Accordion('Inference Hyperparameters', open=False):
with gr.Row():
understand_do_sample = gr.Checkbox(label='Sampling', value=
False, info='Enable sampling for text generation')
understand_text_temperature = gr.Slider(minimum=0.0,
maximum=1.0, value=0.3, step=0.05, interactive=True,
label='Temperature', info=
'Controls randomness in text generation (0=deterministic, 1=creative)'
)
understand_max_new_tokens = gr.Slider(minimum=64, maximum=
4096, value=512, step=64, interactive=True, label=
'Max New Tokens', info=
'Maximum length of generated text, including potential thinking'
)
img_understand_btn = gr.Button('Submit', variant='primary')
understand_generation_time_output = gr.Textbox(label=
'Processing Time', interactive=False)
def process_understanding(image, prompt, show_thinking, do_sample,
text_temperature, max_new_tokens):
result, duration_str = image_understanding(image, prompt,
show_thinking, do_sample, text_temperature, max_new_tokens)
return result, duration_str
gr.on(triggers=[img_understand_btn.click, understand_prompt.submit],
fn=process_understanding, inputs=[img_input, understand_prompt,
understand_show_thinking, understand_do_sample,
understand_text_temperature, understand_max_new_tokens],
outputs=[txt_output, understand_generation_time_output])
with gr.Tab('📊 System Monitor'):
with gr.Column():
memory_stats_display = gr.Markdown('Check RAM/VRAM Stats')
refresh_button = gr.Button('🔄 Check RAM/VRAM Stats')
refresh_button.click(fn=get_all_memory_stats_for_gradio_display,
inputs=None, outputs=[memory_stats_display])
block.launch()
gguf_connector/b2.py
+681
View File
@@ -0,0 +1,681 @@
import os
if not os.path.isfile(os.path.join(os.path.dirname(__file__),
'models/bagel/config.json')):
from huggingface_hub import snapshot_download
save_dir = os.path.join(os.path.dirname(__file__), 'models/bagel')
repo_id = 'callgg/bagel-decoder'
cache_dir = save_dir + '/cache'
snapshot_download(cache_dir=cache_dir, local_dir=save_dir, repo_id=
repo_id, allow_patterns=['*.json', '*.md', '*.txt'])
import time, psutil, platform, atexit, random
pynvml_available = False
if platform.system() == 'Linux' or platform.system() == 'Windows':
try:
from pynvml import *
nvmlInit()
pynvml_available = True
print(
"""pynvml (NVIDIA GPU monitoring library) initialized successfully!
Detecting GGUF/Safetensors...
"""
)
def shutdown_pynvml():
print('Shutting down pynvml...')
nvmlShutdown()
atexit.register(shutdown_pynvml)
except Exception as e:
print(
f'Warning: pynvml could not be initialized. Detailed GPU stats via pynvml will not be available. Error: {e}'
)
if 'NVML Shared Library Not Found' in str(e):
print(
'pynvml error hint: NVML shared library not found. If you have an NVIDIA GPU and drivers, ensure the library is accessible.'
)
elif 'Driver Not Loaded' in str(e):
print(
'pynvml error hint: NVIDIA driver is not loaded. Please check your GPU driver installation.'
)
import torch
import gradio as gr
import numpy as np
from accelerate import infer_auto_device_map, load_checkpoint_and_dispatch, init_empty_weights
from PIL import Image
from bagel2.data.data_utils import add_special_tokens, pil_img2rgb
from bagel2.data.transforms import ImageTransform
from bagel2.inferencer import InterleaveInferencer
from bagel2.modeling.autoencoder import load_ae
from bagel2.modeling.bagel.qwen2_navit import NaiveCache
from bagel2.modeling.bagel import BagelConfig, Bagel, Qwen2Config, Qwen2ForCausalLM, SiglipVisionConfig, SiglipVisionModel
from bagel2.modeling.qwen2 import Qwen2Tokenizer
model_path = os.path.join(os.path.dirname(__file__), 'models/bagel')
llm_config = Qwen2Config.from_json_file(os.path.join(model_path,
'llm_config.json'))
llm_config.qk_norm = True
llm_config.tie_word_embeddings = False
llm_config.layer_module = 'Qwen2MoTDecoderLayer'
vit_config = SiglipVisionConfig.from_json_file(os.path.join(model_path,
'vit_config.json'))
vit_config.rope = False
vit_config.num_hidden_layers -= 1
from safetensors.torch import save_file
from typing import Dict, Tuple
from tqdm import tqdm
from .reader import GGUFReader
from .quant import dequantize
def load_gguf_and_extract_metadata(gguf_path):
reader = GGUFReader(gguf_path)
tensors_metadata = []
for tensor in reader.tensors:
tensor_metadata = {'name': tensor.name, 'shape': tuple(tensor.shape
.tolist()), 'n_elements': tensor.n_elements, 'n_bytes': tensor.
n_bytes, 'data_offset': tensor.data_offset, 'type': tensor.
tensor_type}
tensors_metadata.append(tensor_metadata)
return reader, tensors_metadata
def convert_gguf_to_safetensors(gguf_path, output_path, use_bf16):
reader, tensors_metadata = load_gguf_and_extract_metadata(gguf_path)
print(f'Extracted {len(tensors_metadata)} tensors from GGUF file')
tensors_dict: dict[str, torch.Tensor] = {}
for i, tensor_info in enumerate(tqdm(tensors_metadata, desc=
'Dequantizing tensors', unit='tensor')):
tensor_name = tensor_info['name']
tensor_data = reader.get_tensor(i)
weights = dequantize(tensor_data.data, tensor_data.tensor_type).copy()
try:
if use_bf16:
weights_tensor = torch.from_numpy(weights).to(dtype=torch.
float32)
weights_tensor = weights_tensor.to(torch.bfloat16)
else:
weights_tensor = torch.from_numpy(weights).to(dtype=torch.
float32)
weights_hf = weights_tensor
except Exception as e:
print(
f"Error during dequantization for tensor '{tensor_name}': {e}")
weights_tensor = torch.from_numpy(weights.astype(np.float32)).to(
torch.float16)
weights_hf = weights_tensor
tensors_dict[tensor_name] = weights_hf
metadata = {key: str(reader.get_field(key)) for key in reader.fields}
save_file(tensors_dict, output_path, metadata=metadata)
print('Dequantization complete!')
gguf_files = [file for file in os.listdir() if file.endswith('.gguf')]
safetensors_files = [file for file in os.listdir() if file.endswith(
'.safetensors')]
if gguf_files:
print('GGUF file(s) available. Select which one for VAE:')
for index, file_name in enumerate(gguf_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(gguf_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = gguf_files[choice_index]
print(f'VAE file: {selected_file} is selected!')
input_path = selected_file
use_bf16 = False
vae_path = f'{os.path.splitext(input_path)[0]}-f32.safetensors'
if safetensors_files:
print(
'\nSafetensors file(s) available. Select which one for MODEL:')
for index, file_name in enumerate(safetensors_files, start=1):
print(f'{index}. {file_name}')
choice2 = input(
f'Enter your choice (1 to {len(safetensors_files)}): ')
try:
choice_index = int(choice2) - 1
selected_model_file = safetensors_files[choice_index]
print(f'MODEL file: {selected_model_file} is selected!\n')
convert_gguf_to_safetensors(input_path, vae_path, use_bf16)
vae_model, vae_config = load_ae(vae_path)
checkpoint = selected_model_file
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No GGUF/Safetensors are available in the current directory.')
input('--- Press ENTER To Exit ---')
config = BagelConfig(visual_gen=True, visual_und=True, llm_config=
llm_config, vit_config=vit_config, vae_config=vae_config,
vit_max_num_patch_per_side=70, connector_act='gelu_pytorch_tanh',
latent_patch_size=2, max_latent_size=64)
with init_empty_weights():
language_model = Qwen2ForCausalLM(llm_config)
vit_model = SiglipVisionModel(vit_config)
model = Bagel(language_model, vit_model, config)
model.vit_model.vision_model.embeddings.convert_conv2d_to_linear(vit_config
, meta=True)
tokenizer = Qwen2Tokenizer.from_pretrained(model_path)
tokenizer, new_token_ids, _ = add_special_tokens(tokenizer)
vae_transform = ImageTransform(1024, 512, 16)
vit_transform = ImageTransform(980, 224, 14)
print('Starting model loading and device map configuration...')
def get_gpu_memory_stats_pynvml(device_id=0):
if not pynvml_available:
return f'GPU-{device_id} (pynvml): Not available.'
try:
handle = nvmlDeviceGetHandleByIndex(device_id)
info = nvmlDeviceGetMemoryInfo(handle)
total_gb = info.total / 1024 ** 3
used_gb = info.used / 1024 ** 3
return (
f'GPU-{device_id} (pynvml): Total: {total_gb:.2f} GB, Used (Overall): {used_gb:.2f} GB'
)
except NVMLError as e:
return f'GPU-{device_id} (pynvml) Error: {e}'
def get_gpu_memory_stats_pytorch(device_id=0):
if not torch.cuda.is_available():
return 'PyTorch: CUDA not available.'
if device_id < 0 or device_id >= torch.cuda.device_count():
return f'PyTorch GPU-{device_id}: Invalid device ID.'
allocated_gb = torch.cuda.memory_allocated(device_id) / 1024 ** 3
reserved_gb = torch.cuda.memory_reserved(device_id) / 1024 ** 3
total_capacity_str_pt = ''
if pynvml_available:
try:
handle = nvmlDeviceGetHandleByIndex(device_id)
info = nvmlDeviceGetMemoryInfo(handle)
total_gb_pt = info.total / 1024 ** 3
total_capacity_str_pt = f'Total Capacity: {total_gb_pt:.2f} GB, '
except NVMLError:
pass
return (
f'PyTorch GPU-{device_id}: {total_capacity_str_pt}Allocated: {allocated_gb:.2f} GB, Reserved: {reserved_gb:.2f} GB'
)
def get_system_ram_stats():
mem = psutil.virtual_memory()
total_gb = mem.total / 1024 ** 3
available_gb = mem.available / 1024 ** 3
used_gb = mem.used / 1024 ** 3
percent_used = mem.percent
return (
f'System RAM: Total: {total_gb:.2f} GB, Available: {available_gb:.2f} GB, Used (Overall): {used_gb:.2f} GB ({percent_used}%)'
)
def get_process_ram_stats():
process = psutil.Process(os.getpid())
mem_info = process.memory_info()
rss_gb = mem_info.rss / 1024 ** 3
return f'App Process RAM (RSS): {rss_gb:.2f} GB'
def get_all_memory_stats_for_gradio_display():
"""Prepare the string of memory/video memory statistics for Gradio display"""
stats_lines = []
if torch.cuda.is_available() and torch.cuda.device_count() > 0:
stats_lines.append('**GPU VRAM Usage:**')
for i in range(torch.cuda.device_count()):
stats_lines.append(get_gpu_memory_stats_pynvml(i))
stats_lines.append(get_gpu_memory_stats_pytorch(i))
if i < torch.cuda.device_count() - 1:
stats_lines.append('---')
else:
stats_lines.append(
'**GPU VRAM Usage:** CUDA not available or no GPUs found.')
stats_lines.append('\n**CPU RAM Usage:**')
stats_lines.append(get_system_ram_stats())
stats_lines.append(get_process_ram_stats())
return '\n'.join(s for s in stats_lines if s)
cpu_mem_for_offload = '16GiB'
gpu_mem_per_device = '24GiB'
max_memory_config = {i: gpu_mem_per_device for i in range(torch.cuda.
device_count())}
if torch.cuda.device_count() == 0:
max_memory_config['cpu'] = cpu_mem_for_offload
else:
max_memory_config['cpu'] = cpu_mem_for_offload
print(f'Using max_memory_config: {max_memory_config}')
device_map = infer_auto_device_map(model, max_memory=max_memory_config,
no_split_module_classes=['Bagel', 'Qwen2MoTDecoderLayer'])
print('Device map after infer_auto_device_map (with CPU budget):')
for k, v_map in device_map.items():
print(f' {k}: {v_map}')
same_device_modules = ['language_model.model.embed_tokens', 'time_embedder',
'latent_pos_embed', 'vae2llm', 'llm2vae', 'connector', 'vit_pos_embed']
if torch.cuda.device_count() > 0:
first_device_key = same_device_modules[0]
default_target_device = 'cuda:0'
first_module_target_device = device_map.get(first_device_key,
default_target_device)
print(
f'Target device for same_device_modules (based on {first_device_key}): {first_module_target_device}'
)
for k_module in same_device_modules:
if k_module in device_map:
if device_map[k_module] != first_module_target_device:
print(
f' Moving {k_module} from {device_map[k_module]} to {first_module_target_device} (same_device_modules)'
)
device_map[k_module] = first_module_target_device
else:
print(
f' Assigning {k_module} (from same_device_modules) to {first_module_target_device} as it was not in initial map.'
)
device_map[k_module] = first_module_target_device
elif torch.cuda.device_count() == 0 and 'cpu' in max_memory_config:
print('No CUDA devices found. Assigning same_device_modules to CPU.')
for k_module in same_device_modules:
device_map[k_module] = 'cpu'
print('Device map after same_device_modules logic:')
for k, v_map in device_map.items():
print(f' {k}: {v_map}')
keys_to_change_to_cpu = []
for module_name, device_assignment in device_map.items():
if device_assignment == 'disk':
keys_to_change_to_cpu.append(module_name)
if keys_to_change_to_cpu:
print(
f"Manually changing the following layers from 'disk' to 'cpu': {keys_to_change_to_cpu}"
)
for module_name in keys_to_change_to_cpu:
device_map[module_name] = 'cpu'
print('Final device_map before loading checkpoint (after disk override):')
for k, v_map in device_map.items():
print(f' {k}: {v_map}')
else:
print(
"No layers assigned to 'disk' by infer_auto_device_map, or they were already handled. Final device_map is as above."
)
print(
"""
Starting custom device_map modifications to maximize GPU utilization..."""
)
print('Device map state BEFORE custom modifications:')
for k_map_item, v_map_item in device_map.items():
print(f' {k_map_item}: {v_map_item}')
NUM_ADDITIONAL_LLM_LAYERS_TO_GPU = 5
TRY_MOVE_LLM_NORM_HEAD_TO_GPU = True
TRY_MOVE_VIT_MODEL_TO_GPU = False
moved_llm_layers_count = 0
if NUM_ADDITIONAL_LLM_LAYERS_TO_GPU > 0:
print(
f"""
Attempting to move up to {NUM_ADDITIONAL_LLM_LAYERS_TO_GPU} LLM layers (11 to {10 + NUM_ADDITIONAL_LLM_LAYERS_TO_GPU}) to GPU 0..."""
)
for i in range(NUM_ADDITIONAL_LLM_LAYERS_TO_GPU):
layer_idx = 11 + i
if layer_idx > 27:
print(
f' Reached max layer index (27). Stopped LLM layer promotion.'
)
break
layer_name = f'language_model.model.layers.{layer_idx}'
if device_map.get(layer_name) == 'cpu':
print(f" Promoting LLM layer '{layer_name}' from 'cpu' to GPU 0.")
device_map[layer_name] = 0
moved_llm_layers_count += 1
elif layer_name in device_map:
print(
f" LLM Layer '{layer_name}' is already on device '{device_map[layer_name]}'. Skipping promotion."
)
else:
print(
f" Warning: LLM Layer '{layer_name}' not found in device_map. Cannot promote."
)
print(
f'Successfully promoted {moved_llm_layers_count} LLM layers to GPU 0.')
else:
print(
"""
Skipping promotion of additional LLM layers based on NUM_ADDITIONAL_LLM_LAYERS_TO_GPU setting."""
)
if TRY_MOVE_LLM_NORM_HEAD_TO_GPU:
print(
"\nAttempting to move LLM 'norm' and 'lm_head' to GPU 0 (if on CPU)..."
)
llm_aux_modules = ['language_model.model.norm',
'language_model.model.lm_head']
for module_name in llm_aux_modules:
if device_map.get(module_name) == 'cpu':
print(f" Promoting '{module_name}' from 'cpu' to GPU 0.")
device_map[module_name] = 0
elif module_name in device_map:
print(
f" Module '{module_name}' is already on device '{device_map[module_name]}'. Skipping promotion."
)
else:
print(
f" Warning: Module '{module_name}' not found in device_map. Cannot promote."
)
else:
print(
"""
Skipping promotion of LLM 'norm' and 'lm_head' based on TRY_MOVE_LLM_NORM_HEAD_TO_GPU setting."""
)
if TRY_MOVE_VIT_MODEL_TO_GPU:
print("\nAttempting to move 'vit_model' to GPU 0 (if on CPU)...")
vit_module_name = 'vit_model'
if device_map.get(vit_module_name) == 'cpu':
print(f" Promoting '{vit_module_name}' from 'cpu' to GPU 0.")
device_map[vit_module_name] = 0
elif vit_module_name in device_map:
print(
f" Module '{vit_module_name}' is already on device '{device_map[vit_module_name]}'. Skipping promotion."
)
else:
print(
f" Warning: Module '{vit_module_name}' not found in device_map. Cannot promote."
)
else:
print(
"\nSkipping promotion of 'vit_model' based on TRY_MOVE_VIT_MODEL_TO_GPU setting."
)
print("""
Final device_map after all custom modifications:""")
for k_map_item, v_map_item in device_map.items():
print(f' {k_map_item}: {v_map_item}')
print('--- End of custom device_map modifications ---')
model = load_checkpoint_and_dispatch(model, checkpoint=checkpoint,
device_map=device_map, offload_buffers=True, offload_folder='offload',
dtype=torch.bfloat16, force_hooks=True).eval()
inferencer = InterleaveInferencer(model=model, vae_model=vae_model,
tokenizer=tokenizer, vae_transform=vae_transform, vit_transform=
vit_transform, new_token_ids=new_token_ids)
def set_seed(seed):
"""Set random seeds for reproducibility"""
if seed > 0:
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
return seed
def text_to_image(prompt, show_thinking=False, cfg_text_scale=4.0,
cfg_interval=0.4, timestep_shift=3.0, num_timesteps=50, cfg_renorm_min=
1.0, cfg_renorm_type='global', max_think_token_n=1024, do_sample=False,
text_temperature=0.3, seed=0, image_ratio='1:1'):
set_seed(seed)
if image_ratio == '1:1':
image_shapes = 1024, 1024
elif image_ratio == '4:3':
image_shapes = 768, 1024
elif image_ratio == '3:4':
image_shapes = 1024, 768
elif image_ratio == '16:9':
image_shapes = 576, 1024
elif image_ratio == '9:16':
image_shapes = 1024, 576
inference_hyper = dict(max_think_token_n=max_think_token_n if
show_thinking else 1024, do_sample=do_sample if show_thinking else
False, text_temperature=text_temperature if show_thinking else 0.3,
cfg_text_scale=cfg_text_scale, cfg_interval=[cfg_interval, 1.0],
timestep_shift=timestep_shift, num_timesteps=num_timesteps,
cfg_renorm_min=cfg_renorm_min, cfg_renorm_type=cfg_renorm_type,
image_shapes=image_shapes)
start_time = time.time()
result = inferencer(text=prompt, think=show_thinking, **inference_hyper)
end_time = time.time()
duration = end_time - start_time
duration_str = f'{duration:.2f} seconds'
print(f'The image takes time: {duration_str}')
return result['image'], result.get('text', None), duration_str
def image_understanding(image, prompt, show_thinking=False, do_sample=False,
text_temperature=0.3, max_new_tokens=512):
start_time = time.time()
if image is None:
return 'Please upload an image.'
if isinstance(image, np.ndarray):
image = Image.fromarray(image)
image = pil_img2rgb(image)
inference_hyper = dict(do_sample=do_sample, text_temperature=
text_temperature, max_think_token_n=max_new_tokens)
result = inferencer(image=image, text=prompt, think=show_thinking,
understanding_output=True, **inference_hyper)
end_time = time.time()
duration = end_time - start_time
duration_str = f'{duration:.2f} seconds'
print(f'The image takes time: {duration_str}')
return result['text'], duration_str
def edit_image(image, prompt, show_thinking=False, cfg_text_scale=4.0,
cfg_img_scale=2.0, cfg_interval=0.0, timestep_shift=3.0, num_timesteps=
50, cfg_renorm_min=1.0, cfg_renorm_type='text_channel',
max_think_token_n=1024, do_sample=False, text_temperature=0.3, seed=0):
set_seed(seed)
start_time = time.time()
if image is None:
return 'Please upload an image.', ''
if isinstance(image, np.ndarray):
image = Image.fromarray(image)
image = pil_img2rgb(image)
inference_hyper = dict(max_think_token_n=max_think_token_n if
show_thinking else 1024, do_sample=do_sample if show_thinking else
False, text_temperature=text_temperature if show_thinking else 0.3,
cfg_text_scale=cfg_text_scale, cfg_img_scale=cfg_img_scale,
cfg_interval=[cfg_interval, 1.0], timestep_shift=timestep_shift,
num_timesteps=num_timesteps, cfg_renorm_min=cfg_renorm_min,
cfg_renorm_type=cfg_renorm_type)
result = inferencer(image=image, text=prompt, think=show_thinking, **
inference_hyper)
end_time = time.time()
duration = end_time - start_time
duration_str = f'{duration:.2f} seconds'
print(f'The image takes time: {duration_str}')
return result['image'], result.get('text', ''), duration_str
def load_example_image(image_path):
try:
return Image.open(image_path)
except Exception as e:
print(f'Error loading example image: {e}')
return None
block = gr.Blocks(title='gguf').queue()
with block:
with gr.Tab('📝 Text to Image'):
txt_input = gr.Textbox(label='Prompt', value=
'A female cosplayer portraying an ethereal fairy or elf, wearing a flowing dress made of delicate fabrics in soft, mystical colors like emerald green and silver. She has pointed ears, a gentle, enchanting expression, and her outfit is adorned with sparkling jewels and intricate patterns. The background is a magical forest with glowing plants, mystical creatures, and a serene atmosphere.'
)
with gr.Row():
show_thinking = gr.Checkbox(label='Thinking', value=False)
with gr.Accordion('Inference Hyperparameters', open=False):
with gr.Group():
with gr.Row():
seed = gr.Slider(minimum=0, maximum=1000000, value=0,
step=1, label='Seed', info=
'0 for random seed, positive for reproducible results')
image_ratio = gr.Dropdown(choices=['1:1', '4:3', '3:4',
'16:9', '9:16'], value='1:1', label='Image Ratio',
info='The longer size is fixed to 1024')
with gr.Row():
cfg_text_scale = gr.Slider(minimum=1.0, maximum=8.0,
value=4.0, step=0.1, interactive=True, label=
'CFG Text Scale', info=
'Controls how strongly the model follows the text prompt (4.0-8.0)'
)
cfg_interval = gr.Slider(minimum=0.0, maximum=1.0,
value=0.4, step=0.1, label='CFG Interval', info=
'Start of CFG application interval (end is fixed at 1.0)'
)
with gr.Row():
cfg_renorm_type = gr.Dropdown(choices=['global',
'local', 'text_channel'], value='global', label=
'CFG Renorm Type', info=
"If the genrated image is blurry, use 'global'")
cfg_renorm_min = gr.Slider(minimum=0.0, maximum=1.0,
value=0.0, step=0.1, interactive=True, label=
'CFG Renorm Min', info='1.0 disables CFG-Renorm')
with gr.Row():
num_timesteps = gr.Slider(minimum=10, maximum=100,
value=50, step=5, interactive=True, label=
'Timesteps', info='Total denoising steps')
timestep_shift = gr.Slider(minimum=1.0, maximum=5.0,
value=3.0, step=0.5, interactive=True, label=
'Timestep Shift', info=
'Higher values for layout, lower for details')
thinking_params = gr.Group(visible=False)
with thinking_params:
with gr.Row():
do_sample = gr.Checkbox(label='Sampling', value=
False, info='Enable sampling for text generation')
max_think_token_n = gr.Slider(minimum=64, maximum=
4006, value=1024, step=64, interactive=True,
label='Max Think Tokens', info=
'Maximum number of tokens for thinking')
text_temperature = gr.Slider(minimum=0.1, maximum=
1.0, value=0.3, step=0.1, interactive=True,
label='Temperature', info=
'Controls randomness in text generation')
thinking_output = gr.Textbox(label='Thinking Process', visible=False)
img_output = gr.Image(label='Generated Image')
gen_btn = gr.Button('Generate', variant='primary')
generation_time_output = gr.Textbox(label='Processing Time',
interactive=False)
def update_thinking_visibility(show):
return gr.update(visible=show), gr.update(visible=show)
show_thinking.change(fn=update_thinking_visibility, inputs=[
show_thinking], outputs=[thinking_output, thinking_params])
def process_text_to_image(prompt, show_thinking, cfg_text_scale,
cfg_interval, timestep_shift, num_timesteps, cfg_renorm_min,
cfg_renorm_type, max_think_token_n, do_sample, text_temperature,
seed, image_ratio):
image, thinking, duration_str = text_to_image(prompt,
show_thinking, cfg_text_scale, cfg_interval, timestep_shift,
num_timesteps, cfg_renorm_min, cfg_renorm_type,
max_think_token_n, do_sample, text_temperature, seed,
image_ratio)
return image, thinking if thinking else '', duration_str
gr.on(triggers=[gen_btn.click, txt_input.submit], fn=
process_text_to_image, inputs=[txt_input, show_thinking,
cfg_text_scale, cfg_interval, timestep_shift, num_timesteps,
cfg_renorm_min, cfg_renorm_type, max_think_token_n, do_sample,
text_temperature, seed, image_ratio], outputs=[img_output,
thinking_output, generation_time_output])
with gr.Tab('🖌️ Image Edit'):
with gr.Row():
with gr.Column(scale=1):
edit_image_input = gr.Image(label='Input Image', value=
load_example_image(os.path.join(os.path.dirname(
__file__), '../bagel2/test_images/women.jpg')))
edit_prompt = gr.Textbox(label='Prompt', value=
'She boards a modern subway, quietly reading a folded newspaper, wearing the same clothes.'
)
with gr.Column(scale=1):
edit_image_output = gr.Image(label='Result')
edit_thinking_output = gr.Textbox(label='Thinking Process',
visible=False)
with gr.Row():
edit_show_thinking = gr.Checkbox(label='Thinking', value=False)
with gr.Accordion('Inference Hyperparameters', open=False):
with gr.Group():
with gr.Row():
edit_seed = gr.Slider(minimum=0, maximum=1000000, value
=0, step=1, interactive=True, label='Seed', info=
'0 for random seed, positive for reproducible results')
edit_cfg_text_scale = gr.Slider(minimum=1.0, maximum=
8.0, value=4.0, step=0.1, interactive=True, label=
'CFG Text Scale', info=
'Controls how strongly the model follows the text prompt'
)
with gr.Row():
edit_cfg_img_scale = gr.Slider(minimum=1.0, maximum=4.0,
value=2.0, step=0.1, interactive=True, label=
'CFG Image Scale', info=
'Controls how much the model preserves input image details'
)
edit_cfg_interval = gr.Slider(minimum=0.0, maximum=1.0,
value=0.0, step=0.1, interactive=True, label=
'CFG Interval', info=
'Start of CFG application interval (end is fixed at 1.0)'
)
with gr.Row():
edit_cfg_renorm_type = gr.Dropdown(choices=['global',
'local', 'text_channel'], value='text_channel',
label='CFG Renorm Type', info=
"If the genrated image is blurry, use 'global")
edit_cfg_renorm_min = gr.Slider(minimum=0.0, maximum=
1.0, value=0.0, step=0.1, interactive=True, label=
'CFG Renorm Min', info='1.0 disables CFG-Renorm')
with gr.Row():
edit_num_timesteps = gr.Slider(minimum=10, maximum=100,
value=50, step=5, interactive=True, label=
'Timesteps', info='Total denoising steps')
edit_timestep_shift = gr.Slider(minimum=1.0, maximum=
10.0, value=3.0, step=0.5, interactive=True, label=
'Timestep Shift', info=
'Higher values for layout, lower for details')
edit_thinking_params = gr.Group(visible=False)
with edit_thinking_params:
with gr.Row():
edit_do_sample = gr.Checkbox(label='Sampling',
value=False, info=
'Enable sampling for text generation')
edit_max_think_token_n = gr.Slider(minimum=64,
maximum=4006, value=1024, step=64, interactive=
True, label='Max Think Tokens', info=
'Maximum number of tokens for thinking')
edit_text_temperature = gr.Slider(minimum=0.1,
maximum=1.0, value=0.3, step=0.1, interactive=
True, label='Temperature', info=
'Controls randomness in text generation')
edit_btn = gr.Button('Submit', variant='primary')
edit_generation_time_output = gr.Textbox(label='Processing Time',
interactive=False)
def update_edit_thinking_visibility(show):
return gr.update(visible=show), gr.update(visible=show)
edit_show_thinking.change(fn=update_edit_thinking_visibility,
inputs=[edit_show_thinking], outputs=[edit_thinking_output,
edit_thinking_params])
def process_edit_image(image, prompt, show_thinking, cfg_text_scale,
cfg_img_scale, cfg_interval, timestep_shift, num_timesteps,
cfg_renorm_min, cfg_renorm_type, max_think_token_n, do_sample,
text_temperature, seed):
edited_image, thinking, duration_str = edit_image(image, prompt,
show_thinking, cfg_text_scale, cfg_img_scale, cfg_interval,
timestep_shift, num_timesteps, cfg_renorm_min,
cfg_renorm_type, max_think_token_n, do_sample,
text_temperature, seed)
return edited_image, thinking if thinking else '', duration_str
gr.on(triggers=[edit_btn.click, edit_prompt.submit], fn=
process_edit_image, inputs=[edit_image_input, edit_prompt,
edit_show_thinking, edit_cfg_text_scale, edit_cfg_img_scale,
edit_cfg_interval, edit_timestep_shift, edit_num_timesteps,
edit_cfg_renorm_min, edit_cfg_renorm_type,
edit_max_think_token_n, edit_do_sample, edit_text_temperature,
edit_seed], outputs=[edit_image_output, edit_thinking_output,
edit_generation_time_output])
with gr.Tab('🖼️ Image Understanding'):
with gr.Row():
with gr.Column(scale=1):
img_input = gr.Image(label='Input Image', value=
load_example_image(os.path.join(os.path.dirname(
__file__), '../bagel2/test_images/meme.jpg')))
understand_prompt = gr.Textbox(label='Prompt', value=
"Can someone explain what's funny about this meme??")
with gr.Column(scale=1):
txt_output = gr.Textbox(label='Result', lines=20)
with gr.Row():
understand_show_thinking = gr.Checkbox(label='Thinking', value=
False)
with gr.Accordion('Inference Hyperparameters', open=False):
with gr.Row():
understand_do_sample = gr.Checkbox(label='Sampling', value=
False, info='Enable sampling for text generation')
understand_text_temperature = gr.Slider(minimum=0.0,
maximum=1.0, value=0.3, step=0.05, interactive=True,
label='Temperature', info=
'Controls randomness in text generation (0=deterministic, 1=creative)'
)
understand_max_new_tokens = gr.Slider(minimum=64, maximum=
4096, value=512, step=64, interactive=True, label=
'Max New Tokens', info=
'Maximum length of generated text, including potential thinking'
)
img_understand_btn = gr.Button('Submit', variant='primary')
understand_generation_time_output = gr.Textbox(label=
'Processing Time', interactive=False)
def process_understanding(image, prompt, show_thinking, do_sample,
text_temperature, max_new_tokens):
result, duration_str = image_understanding(image, prompt,
show_thinking, do_sample, text_temperature, max_new_tokens)
return result, duration_str
gr.on(triggers=[img_understand_btn.click, understand_prompt.submit],
fn=process_understanding, inputs=[img_input, understand_prompt,
understand_show_thinking, understand_do_sample,
understand_text_temperature, understand_max_new_tokens],
outputs=[txt_output, understand_generation_time_output])
with gr.Tab('📊 System Monitor'):
with gr.Column():
memory_stats_display = gr.Markdown('Check RAM/VRAM Stats')
refresh_button = gr.Button('🔄 Check RAM/VRAM Stats')
refresh_button.click(fn=get_all_memory_stats_for_gradio_display,
inputs=None, outputs=[memory_stats_display])
block.launch()
gguf_connector/c.py
+51
View File
@@ -0,0 +1,51 @@
import os
gguf_files = [file for file in os.listdir() if file.endswith('.gguf')]
if gguf_files:
print('GGUF file(s) available. Select which one to use:')
for index, file_name in enumerate(gguf_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(gguf_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = gguf_files[choice_index]
print(f'Model file: {selected_file} is selected!')
ModelPath = selected_file
from ctransformers import AutoModelForCausalLM
llm = AutoModelForCausalLM.from_pretrained(ModelPath)
from tkinter import *
import tkinter.scrolledtext as st
root = Tk()
root.title('chatPIG')
root.columnconfigure([0, 1, 2], minsize=150)
root.rowconfigure(0, weight=2)
root.rowconfigure(1, weight=1)
icon = PhotoImage(file=os.path.join(os.path.dirname(__file__),
'logo.png'))
root.iconphoto(False, icon)
i = Entry()
o = st.ScrolledText()
def submit(i):
root.title('Processing...')
from rich.console import Console
console = Console()
console.print(
'*note: [green]it might show: (Not Responding) and/or keep spinning; but running in background still; please be patient.'
)
from rich.progress import Progress
with Progress(transient=True) as progress:
task = progress.add_task('Processing', total=None)
answer = llm(str(i.get()))
print(str(i.get()), answer)
o.insert(INSERT, str(i.get()) + answer + '\n\n')
i.delete(0, END)
root.title('chatPIG')
btn = Button(text='Submit', command=lambda : submit(i))
i.grid(row=1, columnspan=2, sticky='nsew')
btn.grid(row=1, column=2, sticky='nsew')
o.grid(row=0, columnspan=3, sticky='nsew')
root.mainloop()
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No GGUF files are available in the current directory.')
input('--- Press ENTER To Exit ---')
gguf_connector/cg.py
+61
View File
@@ -0,0 +1,61 @@
import os
def wav_handler():
wav_files = [file for file in os.listdir() if file.endswith('.wav')]
if wav_files:
print('WAV file(s) available. Select which one to use:')
for index, file_name in enumerate(wav_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(wav_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = wav_files[choice_index]
print(f'WAV file: {selected_file} is selected!')
import speech_recognition as sr
r = sr.Recognizer()
with sr.AudioFile(selected_file) as source:
audio = r.record(source)
output = r.recognize_google(audio)
try:
from rich.console import Console
console = Console()
console.print(f'Content recognized as: [green]' + output)
input(
'\n---Enter to prompt the WAV content recognized above into GGUF model---'
)
print('Processing...')
ans = llm(output)
print(output + ans)
except sr.UnknownValueError:
print('Could not understand audio')
except sr.RequestError as e:
print('Error; {0}'.format(e))
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No WAV files are available in the current directory.')
input('--- Press ENTER To Skip ---')
gguf_files = [file for file in os.listdir() if file.endswith('.gguf')]
if gguf_files:
print('GGUF file(s) available. Select which one to use:')
for index, file_name in enumerate(gguf_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(gguf_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = gguf_files[choice_index]
print(f'Model file: {selected_file} is selected!')
ModelPath = selected_file
print('Processing...')
from ctransformers import AutoModelForCausalLM
llm = AutoModelForCausalLM.from_pretrained(ModelPath)
while True:
ask = input('---Enter to select a WAV file (Q for quit)---')
if ask == 'q' or ask == 'Q':
break
wav_handler()
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No GGUF files are available in the current directory.')
input('--- Press ENTER To Exit ---')
print('Goodbye!')
gguf_connector/const.py
+1137
View File
File diff suppressed because it is too large Load Diff
gguf_connector/cp.py
+64
View File
@@ -0,0 +1,64 @@
import os
def pdf_handler():
pdf_files = [file for file in os.listdir() if file.endswith('.pdf')]
def join_text(input_text):
joined_text = ' '.join(input_text.splitlines())
return joined_text
if pdf_files:
print('PDF file(s) available. Select which one to use:')
for index, file_name in enumerate(pdf_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(pdf_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = pdf_files[choice_index]
print(f'PDF file: {selected_file} is selected!')
from pypdf import PdfReader
reader = PdfReader(selected_file)
text = ''
number_of_pages = len(reader.pages)
for i in range(number_of_pages):
page = reader.pages[i]
text += page.extract_text()
output_text = join_text(text)
inject = f'analyze the content below: ' + output_text
from rich.console import Console
console = Console()
console.print(f'\nPDF cotent extracted as below:\n\n[green]' + text
)
input('---Enter to analyze the PDF content above---')
from rich.progress import Progress
with Progress(transient=True) as progress:
task = progress.add_task('Processing', total=None)
ans = llm(inject)
print(inject + '\n\nOutput:' + ans)
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No PDF files are available in the current directory.')
input('--- Press ENTER To Skip ---')
gguf_files = [file for file in os.listdir() if file.endswith('.gguf')]
if gguf_files:
print('GGUF file(s) available. Select which one to use:')
for index, file_name in enumerate(gguf_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(gguf_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = gguf_files[choice_index]
print(f'Model file: {selected_file} is selected!')
ModelPath = selected_file
print('Processing...')
from ctransformers import AutoModelForCausalLM
llm = AutoModelForCausalLM.from_pretrained(ModelPath)
while True:
ask = input('---Enter to select a PDF file (Q for quit)---')
if ask.lower() == 'q':
break
pdf_handler()
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No GGUF files are available in the current directory.')
input('--- Press ENTER To Exit ---')
print('Goodbye!')
gguf_connector/cpp.py
+52
View File
@@ -0,0 +1,52 @@
import os
gguf_files = [file for file in os.listdir() if file.endswith('.gguf')]
if gguf_files:
print('GGUF file(s) available. Select which one to use:')
for index, file_name in enumerate(gguf_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(gguf_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = gguf_files[choice_index]
print(f'Model file: {selected_file} is selected!')
ModelPath = selected_file
from llama_cpp import Llama
llm = Llama(model_path=ModelPath)
from tkinter import *
import tkinter.scrolledtext as st
root = Tk()
root.title('chatPIG')
root.columnconfigure([0, 1, 2], minsize=150)
root.rowconfigure(0, weight=2)
root.rowconfigure(1, weight=1)
icon = PhotoImage(file=os.path.join(os.path.dirname(__file__),
'logo.png'))
root.iconphoto(False, icon)
i = Entry()
o = st.ScrolledText()
def submit(i):
root.title('Processing...')
from rich.console import Console
console = Console()
console.print(
'*note: [green]it might show: (Not Responding) and/or keep spinning; but running in background still; please be patient.'
)
from rich.progress import Progress
with Progress(transient=True) as progress:
task = progress.add_task('Processing', total=None)
output = llm('Q: ' + str(i.get()), max_tokens=4096, echo=True)
answer = output['choices'][0]['text']
print(answer)
o.insert(INSERT, answer + '\n\n')
i.delete(0, END)
root.title('chatPIG')
btn = Button(text='Submit', command=lambda : submit(i))
i.grid(row=1, columnspan=2, sticky='nsew')
btn.grid(row=1, column=2, sticky='nsew')
o.grid(row=0, columnspan=3, sticky='nsew')
root.mainloop()
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No GGUF files are available in the current directory.')
input('--- Press ENTER To Exit ---')
gguf_connector/cs.py
+61
View File
@@ -0,0 +1,61 @@
import os
def wav_handler():
wav_files = [file for file in os.listdir() if file.endswith('.wav')]
if wav_files:
print('WAV file(s) available. Select which one to use:')
for index, file_name in enumerate(wav_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(wav_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = wav_files[choice_index]
print(f'WAV file: {selected_file} is selected!')
import speech_recognition as sr
r = sr.Recognizer()
with sr.AudioFile(selected_file) as source:
audio = r.record(source)
output = r.recognize_sphinx(audio)
try:
from rich.console import Console
console = Console()
console.print(f'Content recognized as: [green]' + output)
input(
'\n---Enter to prompt the WAV content recognized above into GGUF model---'
)
print('Processing...')
ans = llm(output)
print(output + ans)
except sr.UnknownValueError:
print('Could not understand audio')
except sr.RequestError as e:
print('Error; {0}'.format(e))
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No WAV files are available in the current directory.')
input('--- Press ENTER To Skip ---')
gguf_files = [file for file in os.listdir() if file.endswith('.gguf')]
if gguf_files:
print('GGUF file(s) available. Select which one to use:')
for index, file_name in enumerate(gguf_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(gguf_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = gguf_files[choice_index]
print(f'Model file: {selected_file} is selected!')
ModelPath = selected_file
print('Processing...')
from ctransformers import AutoModelForCausalLM
llm = AutoModelForCausalLM.from_pretrained(ModelPath)
while True:
ask = input('---Enter to select a WAV file (Q for quit)---')
if ask == 'q' or ask == 'Q':
break
wav_handler()
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No GGUF files are available in the current directory.')
input('--- Press ENTER To Exit ---')
print('Goodbye!')
gguf_connector/d.py
+79
View File
@@ -0,0 +1,79 @@
import os, json
from safetensors import safe_open
from safetensors.torch import save_file
def list_safetensors():
files = [f for f in os.listdir() if f.endswith('.safetensors')]
if not files:
print('No .safetensors files found in the current directory.')
return None
print('Available .safetensors files:')
for i, f in enumerate(files, 1):
print(f'{i}. {f}')
while True:
try:
choice = int(input('Select a file by number: '))
return files[choice - 1]
except (ValueError, IndexError):
print('Invalid selection. Try again.')
def get_cutoff_size():
while True:
try:
mb = float(input('Enter cutoff size for each part (in MB): '))
return int(mb * 1024 * 1024)
except ValueError:
print('Please enter a valid number.')
def split_safetensors(file_name, cutoff_size):
with safe_open(file_name, framework='pt') as f:
metadata = f.metadata()
keys = f.keys()
tensors = {key: f.get_tensor(key) for key in keys}
total_size = 0
tensor_sizes = {}
for k, v in tensors.items():
size = v.element_size() * v.nelement()
tensor_sizes[k] = size
total_size += size
part_id = 1
current_size = 0
current_tensors = {}
weight_map = {}
num_parts = 0
all_parts = []
for key, tensor in tensors.items():
tensor_size = tensor_sizes[key]
if current_size + tensor_size > cutoff_size and current_tensors:
part_name = f'model-{part_id:05d}-of-XXXX.safetensors'
save_file(current_tensors, part_name, metadata={})
all_parts.append(part_name)
for k in current_tensors:
weight_map[k] = part_name
part_id += 1
current_tensors = {}
current_size = 0
current_tensors[key] = tensor
current_size += tensor_size
if current_tensors:
part_name = f'model-{part_id:05d}-of-XXXX.safetensors'
save_file(current_tensors, part_name, metadata={})
all_parts.append(part_name)
for k in current_tensors:
weight_map[k] = part_name
num_parts = len(all_parts)
final_files = []
for i, old_name in enumerate(all_parts):
new_name = f'model-{i + 1:05d}-of-{num_parts:05d}.safetensors'
os.rename(old_name, new_name)
for k in weight_map:
if weight_map[k] == old_name:
weight_map[k] = new_name
final_files.append(new_name)
index_json = {'metadata': {'total_size': total_size}, 'weight_map':
weight_map}
with open('model.safetensors.index.json', 'w') as f:
json.dump(index_json, f, indent=2)
print(f'\nSplit complete! {num_parts} parts created.')
print('Index file: model.safetensors.index.json')
file = list_safetensors()
if file:
cutoff = get_cutoff_size()
split_safetensors(file, cutoff)
gguf_connector/d2.py
+61
View File
@@ -0,0 +1,61 @@
import os, json
from tqdm import tqdm
def list_gguf_files():
files = [f for f in os.listdir() if f.endswith('.gguf')]
if not files:
print('No .gguf files found in the current directory.')
return None
print('Available .gguf files:')
for i, f in enumerate(files, 1):
print(f'{i}. {f}')
while True:
try:
choice = int(input('Select a file by number: '))
return files[choice - 1]
except (ValueError, IndexError):
print('Invalid selection. Try again.')
def get_cutoff_size():
while True:
try:
mb = float(input('Enter cutoff size for each part (in MB): '))
return int(mb * 1024 * 1024)
except ValueError:
print('Please enter a valid number.')
def split_gguf_file(filename, cutoff_size):
with open(filename, 'rb') as f:
data = f.read()
total_size = len(data)
parts = []
offset = 0
part_id = 1
print('\nSplitting file into parts...')
with tqdm(total=total_size, unit='B', unit_scale=True, desc='Splitting'
) as pbar:
while offset < total_size:
end = min(offset + cutoff_size, total_size)
part_data = data[offset:end]
part_name = f'model-{part_id:05d}-of-XXXX.gguf'
with open(part_name, 'wb') as pf:
pf.write(part_data)
parts.append(part_name)
offset = end
part_id += 1
pbar.update(len(part_data))
num_parts = len(parts)
final_files = []
print('\nRenaming parts with total count info...')
for i, old_name in tqdm(list(enumerate(parts)), desc='Renaming', unit=
'file'):
new_name = f'model-{i + 1:05d}-of-{num_parts:05d}.gguf'
os.rename(old_name, new_name)
final_files.append(new_name)
index = {'metadata': {'total_size': total_size}, 'file_map': {
f'part_{i + 1}': name for i, name in enumerate(final_files)}}
with open('model.gguf.index.json', 'w') as f:
json.dump(index, f, indent=2)
print(f'\nSplit complete! {num_parts} parts created.')
print('Index file: model.gguf.index.json')
file = list_gguf_files()
if file:
cutoff = get_cutoff_size()
split_gguf_file(file, cutoff)
gguf_connector/d4.py
+62
View File
@@ -0,0 +1,62 @@
import torch
from .const import GGML_QUANT_VERSION, LlamaFileType
from .reader import GGUFReader, GGMLQuantizationType
from .writer import GGUFWriter
from .quant import quantize
from tqdm import tqdm
def get_arch_str(reader):
field = reader.get_field('general.architecture')
return str(field.parts[field.data[-1]], encoding='utf-8')
def get_file_type(reader):
field = reader.get_field('general.file_type')
ft = int(field.parts[field.data[-1]])
return LlamaFileType(ft)
import os
gguf_files = [file for file in os.listdir() if file.endswith('.gguf')]
if gguf_files:
print('GGUF file(s) available. Select which one to fix:')
for index, file_name in enumerate(gguf_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(gguf_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = gguf_files[choice_index]
print(f'Model file: {selected_file} is selected!')
input_path = selected_file
output_path = f'fixed-{os.path.splitext(input_path)[0]}.gguf'
reader = GGUFReader(input_path)
arch = get_arch_str(reader)
file_type = get_file_type(reader)
print(f"Detected arch: '{arch}' (ftype: {str(file_type)})")
sd5d = torch.load(f'fix_5d_tensors_{arch}.pt', weights_only=False)
print('5D:', sd5d.keys())
writer = GGUFWriter(path=None, arch=arch)
writer.add_quantization_version(GGML_QUANT_VERSION)
writer.add_file_type(file_type)
added = []
def add_extra_key(writer, key, data):
data_qtype = GGMLQuantizationType.F32
data_shape = data.shape
data = quantize(data, data_qtype)
tqdm.write(f'Adding key {key} ({data_shape})')
writer.add_tensor(key, data, raw_dtype=data_qtype)
global added
added.append(key)
for tensor in tqdm(reader.tensors):
writer.add_tensor(tensor.name, tensor.data, raw_dtype=tensor.
tensor_type)
key5d = tensor.name.replace('.bias', '.weight')
if key5d in sd5d.keys():
add_extra_key(writer, key5d, sd5d[key5d])
for key, data in sd5d.items():
if key not in added:
add_extra_key(writer, key, data)
writer.write_header_to_file(path=output_path)
writer.write_kv_data_to_file()
writer.write_tensors_to_file(progress=True)
writer.close()
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No GGUF files are available in the current directory.')
input('--- Press ENTER To Exit ---')
gguf_connector/d5.py
+63
View File
@@ -0,0 +1,63 @@
import torch
from .const import GGML_QUANT_VERSION, LlamaFileType
from .reader import GGUFReader, GGMLQuantizationType
from .writer import GGUFWriter
from .quant import quantize
from tqdm import tqdm
def get_arch_str(reader):
field = reader.get_field('general.architecture')
return str(field.parts[field.data[-1]], encoding='utf-8')
def get_file_type(reader):
field = reader.get_field('general.file_type')
ft = int(field.parts[field.data[-1]])
return LlamaFileType(ft)
import os
gguf_files = [file for file in os.listdir() if file.endswith('.gguf')]
if gguf_files:
print('GGUF file(s) available. Select which one to fix:')
for index, file_name in enumerate(gguf_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(gguf_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = gguf_files[choice_index]
print(f'Model file: {selected_file} is selected!')
input_path = selected_file
output_path = f'fixed-{os.path.splitext(input_path)[0]}.gguf'
reader = GGUFReader(input_path)
arch = get_arch_str(reader)
file_type = get_file_type(reader)
print(f"Detected arch: '{arch}' (ftype: {str(file_type)})")
sd5d = torch.load(f'fix_5d_tensors_{arch}.safetensors',
weights_only=False)
sd5d = {k: v.numpy() for k, v in sd5d.items()}
print('5D tensors:', sd5d.keys())
writer = GGUFWriter(path=None, arch=arch)
writer.add_quantization_version(GGML_QUANT_VERSION)
writer.add_file_type(file_type)
added = []
def add_extra_key(writer, key, data):
global added
data_qtype = GGMLQuantizationType.F32
data = quantize(data, data_qtype)
tqdm.write(f'Adding key {key} ({data.shape})')
writer.add_tensor(key, data, raw_dtype=data_qtype)
added.append(key)
for tensor in tqdm(reader.tensors):
writer.add_tensor(tensor.name, tensor.data, raw_dtype=tensor.
tensor_type)
key5d = tensor.name.replace('.bias', '.weight')
if key5d in sd5d.keys():
add_extra_key(writer, key5d, sd5d[key5d])
for key, data in sd5d.items():
if key not in added:
add_extra_key(writer, key, data)
writer.write_header_to_file(path=output_path)
writer.write_kv_data_to_file()
writer.write_tensors_to_file(progress=True)
writer.close()
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No GGUF files are available in the current directory.')
input('--- Press ENTER To Exit ---')
gguf_connector/d6.py
+185
View File
@@ -0,0 +1,185 @@
import torch
from .writer import GGUFWriter, GGMLQuantizationType
from .const import GGML_QUANT_VERSION, LlamaFileType
from safetensors.torch import load_file
from tqdm import tqdm
QUANTIZATION_THRESHOLD = 1024
REARRANGE_THRESHOLD = 512
MAX_TENSOR_NAME_LENGTH = 127
MAX_TENSOR_DIMS = 4
class ModelTemplate:
arch = 'invalid'
shape_fix = False
keys_detect = []
keys_banned = []
keys_hiprec = []
class ModelFlux(ModelTemplate):
arch = 'flux'
keys_detect = [('transformer_blocks.0.attn.norm_added_k.weight',), (
'double_blocks.0.img_attn.proj.weight',)]
keys_banned = ['transformer_blocks.0.attn.norm_added_k.weight']
class ModelSD3(ModelTemplate):
arch = 'sd3'
keys_detect = [('transformer_blocks.0.attn.add_q_proj.weight',), (
'joint_blocks.0.x_block.attn.qkv.weight',)]
keys_banned = ['transformer_blocks.0.attn.add_q_proj.weight']
class ModelAura(ModelTemplate):
arch = 'aura'
keys_detect = [('double_layers.3.modX.1.weight',), (
'joint_transformer_blocks.3.ff_context.out_projection.weight',)]
keys_banned = [
'joint_transformer_blocks.3.ff_context.out_projection.weight']
class ModelHiDream(ModelTemplate):
arch = 'hidream'
keys_detect = [('caption_projection.0.linear.weight',
'double_stream_blocks.0.block.ff_i.shared_experts.w3.weight')]
keys_hiprec = ['.ff_i.gate.weightimg_emb.emb_pos']
class ModelHyVid(ModelTemplate):
arch = 'hyvid'
keys_detect = [('double_blocks.0.img_attn_proj.weight',
'txt_in.individual_token_refiner.blocks.1.self_attn_qkv.weight')]
class ModelWan(ModelHyVid):
arch = 'wan'
keys_detect = [('blocks.0.self_attn.norm_q.weight',
'text_embedding.2.weight', 'head.modulation')]
class ModelLTXV(ModelTemplate):
arch = 'ltxv'
keys_detect = [('adaln_single.emb.timestep_embedder.linear_2.weight',
'transformer_blocks.27.scale_shift_table',
'caption_projection.linear_2.weight')]
keys_hiprec = ['scale_shift_table']
class ModelSDXL(ModelTemplate):
arch = 'sdxl'
shape_fix = True
keys_detect = [('down_blocks.0.downsamplers.0.conv.weight',
'add_embedding.linear_1.weight'), ('input_blocks.3.0.op.weight',
'input_blocks.6.0.op.weight', 'output_blocks.2.2.conv.weight',
'output_blocks.5.2.conv.weight'), ('label_emb.0.0.weight',)]
class ModelSD1(ModelTemplate):
arch = 'sd1'
shape_fix = True
keys_detect = [('down_blocks.0.downsamplers.0.conv.weight',), (
'input_blocks.3.0.op.weight', 'input_blocks.6.0.op.weight',
'input_blocks.9.0.op.weight', 'output_blocks.2.1.conv.weight',
'output_blocks.5.2.conv.weight', 'output_blocks.8.2.conv.weight')]
arch_list = [ModelFlux, ModelSD3, ModelAura, ModelHiDream, ModelLTXV,
ModelHyVid, ModelWan, ModelSDXL, ModelSD1]
def is_model_arch(model, state_dict):
matched = False
invalid = False
for match_list in model.keys_detect:
if all(key in state_dict for key in match_list):
matched = True
invalid = any(key in state_dict for key in model.keys_banned)
break
assert not invalid, 'Model architecture not allowed for conversion! (i.e. reference VS diffusers format)'
return matched
def detect_arch(state_dict):
model_arch = None
for arch in arch_list:
if is_model_arch(arch, state_dict):
model_arch = arch()
break
assert model_arch is not None, 'Unknown model architecture!'
return model_arch
def load_state_dict(path):
if any(path.endswith(x) for x in ['.ckpt', '.pt', '.bin', '.pth']):
state_dict = torch.load(path, map_location='cpu', weights_only=True)
for subkey in ['model', 'module']:
if subkey in state_dict:
state_dict = state_dict[subkey]
break
if len(state_dict) < 20:
raise RuntimeError(f'pt subkey load failed: {state_dict.keys()}')
else:
state_dict = load_file(path)
prefix = None
for pfx in ['model.diffusion_model.', 'model.']:
if any([x.startswith(pfx) for x in state_dict.keys()]):
prefix = pfx
break
sd = {}
for k, v in state_dict.items():
if prefix and prefix not in k:
continue
if prefix:
k = k.replace(prefix, '')
sd[k] = v
return sd
def load_model(path):
state_dict = load_state_dict(path)
model_arch = detect_arch(state_dict)
print(f'* Architecture detected from input: {model_arch.arch}')
writer = GGUFWriter(path=None, arch=model_arch.arch)
return writer, state_dict, model_arch
def handle_tensors(args, writer, state_dict, model_arch):
name_lengths = tuple(sorted(((key, len(key)) for key in state_dict.keys
()), key=lambda item: item[1], reverse=True))
if not name_lengths:
return
max_name_len = name_lengths[0][1]
if max_name_len > MAX_TENSOR_NAME_LENGTH:
bad_list = ', '.join(f'{key!r} ({namelen})' for key, namelen in
name_lengths if namelen > MAX_TENSOR_NAME_LENGTH)
raise ValueError(
f'Can only handle tensor names up to {MAX_TENSOR_NAME_LENGTH} characters. Tensors exceeding the limit: {bad_list}'
)
for key, data in tqdm(state_dict.items()):
old_dtype = data.dtype
data = data.to(torch.float32).numpy()
n_dims = len(data.shape)
data_shape = data.shape
data_qtype = getattr(GGMLQuantizationType, 'BF16' if old_dtype ==
torch.bfloat16 else 'F16')
if len(data.shape) > MAX_TENSOR_DIMS:
continue
n_params = 1
for dim_size in data_shape:
n_params *= dim_size
if old_dtype in (torch.float32, torch.bfloat16):
if n_dims == 1:
data_qtype = GGMLQuantizationType.F32
elif n_params <= QUANTIZATION_THRESHOLD:
data_qtype = GGMLQuantizationType.F32
elif any(x in key for x in model_arch.keys_hiprec):
data_qtype = GGMLQuantizationType.F32
new_name = key
data_qtype = GGMLQuantizationType.F32
shape_str = f"{{{', '.join(str(n) for n in reversed(data.shape))}}}"
tqdm.write(
f"{f'%-{max_name_len + 4}s' % f'{new_name}'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}"
)
writer.add_tensor(new_name, data, raw_dtype=data_qtype)
import os
safetensors_files = [file for file in os.listdir() if file.endswith(
'.safetensors')]
if safetensors_files:
print('Safetensors file(s) available. Select which one to convert:')
for index, file_name in enumerate(safetensors_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = safetensors_files[choice_index]
print(f'Model file: {selected_file} is selected!')
path = selected_file
writer, state_dict, model_arch = load_model(path)
writer.add_quantization_version(GGML_QUANT_VERSION)
if next(iter(state_dict.values())).dtype == torch.bfloat16:
out_path = f'{os.path.splitext(path)[0]}-f32.gguf'
writer.add_file_type(LlamaFileType.MOSTLY_BF16)
else:
out_path = f'{os.path.splitext(path)[0]}-f32.gguf'
writer.add_file_type(LlamaFileType.MOSTLY_F16)
if os.path.isfile(out_path):
input('Output exists enter to continue or ctrl+c to abort!')
handle_tensors(path, writer, state_dict, model_arch)
writer.write_header_to_file(path=out_path)
writer.write_kv_data_to_file()
writer.write_tensors_to_file(progress=True)
writer.close()
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No safetensors files are available in the current directory.')
input('--- Press ENTER To Exit ---')
gguf_connector/d7.py
+197
View File
@@ -0,0 +1,197 @@
import torch
from .writer import GGUFWriter, GGMLQuantizationType
from .const import GGML_QUANT_VERSION, LlamaFileType
from safetensors.torch import load_file
from tqdm import tqdm
QUANTIZATION_THRESHOLD = 1024
REARRANGE_THRESHOLD = 512
MAX_TENSOR_NAME_LENGTH = 127
MAX_TENSOR_DIMS = 4
class ModelTemplate:
arch = 'invalid'
shape_fix = False
keys_detect = []
keys_banned = []
keys_hiprec = []
def handle_nd_tensor(self, key, data):
raise NotImplementedError(
f'Tensor detected that exceeds dims supported by C++ code! ({key} @ {data.shape})'
)
class ModelFlux(ModelTemplate):
arch = 'flux'
keys_detect = [('transformer_blocks.0.attn.norm_added_k.weight',), (
'double_blocks.0.img_attn.proj.weight',)]
keys_banned = ['transformer_blocks.0.attn.norm_added_k.weight']
class ModelSD3(ModelTemplate):
arch = 'sd3'
keys_detect = [('transformer_blocks.0.attn.add_q_proj.weight',), (
'joint_blocks.0.x_block.attn.qkv.weight',)]
keys_banned = ['transformer_blocks.0.attn.add_q_proj.weight']
class ModelAura(ModelTemplate):
arch = 'aura'
keys_detect = [('double_layers.3.modX.1.weight',), (
'joint_transformer_blocks.3.ff_context.out_projection.weight',)]
keys_banned = [
'joint_transformer_blocks.3.ff_context.out_projection.weight']
class ModelHiDream(ModelTemplate):
arch = 'hidream'
keys_detect = [('caption_projection.0.linear.weight',
'double_stream_blocks.0.block.ff_i.shared_experts.w3.weight')]
keys_hiprec = ['.ff_i.gate.weightimg_emb.emb_pos']
class ModelHyVid(ModelTemplate):
arch = 'hyvid'
keys_detect = [('double_blocks.0.img_attn_proj.weight',
'txt_in.individual_token_refiner.blocks.1.self_attn_qkv.weight')]
def handle_nd_tensor(self, key, data):
path = f'./fix_5d_tensors_{self.arch}.safetensors'
fsd = {key: torch.from_numpy(data)}
tqdm.write(
f'5D key found in state dict! Manual fix required! - {key} {data.shape}'
)
torch.save(fsd, path)
class ModelWan(ModelHyVid):
arch = 'wan'
keys_detect = [('blocks.0.self_attn.norm_q.weight',
'text_embedding.2.weight', 'head.modulation')]
class ModelLTXV(ModelTemplate):
arch = 'ltxv'
keys_detect = [('adaln_single.emb.timestep_embedder.linear_2.weight',
'transformer_blocks.27.scale_shift_table',
'caption_projection.linear_2.weight')]
keys_hiprec = ['scale_shift_table']
class ModelSDXL(ModelTemplate):
arch = 'sdxl'
shape_fix = True
keys_detect = [('down_blocks.0.downsamplers.0.conv.weight',
'add_embedding.linear_1.weight'), ('input_blocks.3.0.op.weight',
'input_blocks.6.0.op.weight', 'output_blocks.2.2.conv.weight',
'output_blocks.5.2.conv.weight'), ('label_emb.0.0.weight',)]
class ModelSD1(ModelTemplate):
arch = 'sd1'
shape_fix = True
keys_detect = [('down_blocks.0.downsamplers.0.conv.weight',), (
'input_blocks.3.0.op.weight', 'input_blocks.6.0.op.weight',
'input_blocks.9.0.op.weight', 'output_blocks.2.1.conv.weight',
'output_blocks.5.2.conv.weight', 'output_blocks.8.2.conv.weight')]
arch_list = [ModelFlux, ModelSD3, ModelAura, ModelHiDream, ModelLTXV,
ModelHyVid, ModelWan, ModelSDXL, ModelSD1]
def is_model_arch(model, state_dict):
matched = False
invalid = False
for match_list in model.keys_detect:
if all(key in state_dict for key in match_list):
matched = True
invalid = any(key in state_dict for key in model.keys_banned)
break
assert not invalid, 'Model architecture not allowed for conversion! (i.e. reference VS diffusers format)'
return matched
def detect_arch(state_dict):
model_arch = None
for arch in arch_list:
if is_model_arch(arch, state_dict):
model_arch = arch()
break
assert model_arch is not None, 'Unknown model architecture!'
return model_arch
def load_state_dict(path):
if any(path.endswith(x) for x in ['.ckpt', '.pt', '.bin', '.pth']):
state_dict = torch.load(path, map_location='cpu', weights_only=True)
for subkey in ['model', 'module']:
if subkey in state_dict:
state_dict = state_dict[subkey]
break
if len(state_dict) < 20:
raise RuntimeError(f'pt subkey load failed: {state_dict.keys()}')
else:
state_dict = load_file(path)
prefix = None
for pfx in ['model.diffusion_model.', 'model.']:
if any([x.startswith(pfx) for x in state_dict.keys()]):
prefix = pfx
break
sd = {}
for k, v in state_dict.items():
if prefix and prefix not in k:
continue
if prefix:
k = k.replace(prefix, '')
sd[k] = v
return sd
def load_model(path):
state_dict = load_state_dict(path)
model_arch = detect_arch(state_dict)
print(f'* Architecture detected from input: {model_arch.arch}')
writer = GGUFWriter(path=None, arch=model_arch.arch)
return writer, state_dict, model_arch
def handle_tensors(args, writer, state_dict, model_arch):
name_lengths = tuple(sorted(((key, len(key)) for key in state_dict.keys
()), key=lambda item: item[1], reverse=True))
if not name_lengths:
return
max_name_len = name_lengths[0][1]
if max_name_len > MAX_TENSOR_NAME_LENGTH:
bad_list = ', '.join(f'{key!r} ({namelen})' for key, namelen in
name_lengths if namelen > MAX_TENSOR_NAME_LENGTH)
raise ValueError(
f'Can only handle tensor names up to {MAX_TENSOR_NAME_LENGTH} characters. Tensors exceeding the limit: {bad_list}'
)
for key, data in tqdm(state_dict.items()):
old_dtype = data.dtype
data = data.to(torch.float32).numpy()
n_dims = len(data.shape)
data_shape = data.shape
data_qtype = getattr(GGMLQuantizationType, 'BF16' if old_dtype ==
torch.bfloat16 else 'F16')
if len(data.shape) > MAX_TENSOR_DIMS:
model_arch.handle_nd_tensor(key, data)
continue
n_params = 1
for dim_size in data_shape:
n_params *= dim_size
if old_dtype in (torch.float32, torch.bfloat16):
if n_dims == 1:
data_qtype = GGMLQuantizationType.F32
elif n_params <= QUANTIZATION_THRESHOLD:
data_qtype = GGMLQuantizationType.F32
elif any(x in key for x in model_arch.keys_hiprec):
data_qtype = GGMLQuantizationType.F32
new_name = key
data_qtype = GGMLQuantizationType.F32
shape_str = f"{{{', '.join(str(n) for n in reversed(data.shape))}}}"
tqdm.write(
f"{f'%-{max_name_len + 4}s' % f'{new_name}'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}"
)
writer.add_tensor(new_name, data, raw_dtype=data_qtype)
import os
safetensors_files = [file for file in os.listdir() if file.endswith(
'.safetensors')]
if safetensors_files:
print('Safetensors file(s) available. Select which one to convert:')
for index, file_name in enumerate(safetensors_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = safetensors_files[choice_index]
print(f'Model file: {selected_file} is selected!')
path = selected_file
writer, state_dict, model_arch = load_model(path)
writer.add_quantization_version(GGML_QUANT_VERSION)
if next(iter(state_dict.values())).dtype == torch.bfloat16:
out_path = f'{os.path.splitext(path)[0]}-f32.gguf'
writer.add_file_type(LlamaFileType.MOSTLY_BF16)
else:
out_path = f'{os.path.splitext(path)[0]}-f32.gguf'
writer.add_file_type(LlamaFileType.MOSTLY_F16)
if os.path.isfile(out_path):
input('Output exists enter to continue or ctrl+c to abort!')
handle_tensors(path, writer, state_dict, model_arch)
writer.write_header_to_file(path=out_path)
writer.write_kv_data_to_file()
writer.write_tensors_to_file(progress=True)
writer.close()
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No safetensors files are available in the current directory.')
input('--- Press ENTER To Exit ---')
gguf_connector/data.json
+186
View File
@@ -0,0 +1,186 @@
[
{
"name":" chat.gguf [1.9GB; for general purpose]",
"url":"https://github.com/calcuis/chatgpt-ai-model/releases/download/0.1/chat.gguf"
},
{
"name":" code.gguf [3.3GB; for coding assistance]",
"url":"https://huggingface.co/calcuis/code_mini/resolve/main/code.gguf"
},
{
"name":" medi.gguf [3.3GB; for medical/health advice]",
"url":"https://huggingface.co/calcuis/medi_mini/resolve/main/medi.gguf"
},
{
"name":" law.gguf [3.3GB; for law/legal opinion]",
"url":"https://huggingface.co/calcuis/law_mini/resolve/main/law.gguf"
},
{
"name":" finance.gguf [3.3GB; for wealth/financial plan]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/finance.gguf"
},
{
"name":" coder.gguf [1.4GB; for simple coding issue]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/coder.gguf"
},
{
"name":" phi2.gguf [1.4GB; for technical problem]",
"url":"https://huggingface.co/calcuis/phi2_mini/resolve/main/phi2.gguf"
},
{
"name":" lam2.gguf [3.3GB; for long/detailed response]",
"url":"https://huggingface.co/calcuis/lam2_mini/resolve/main/lam2.gguf"
},
{
"name":" tiny2.gguf [783MB; for fast/rapid response]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/tiny2.gguf"
},
{
"name":"stable.gguf [1.7GB; for code review]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/stable.gguf"
},
{
"name":"hermes.gguf [3.5GB; for general/basic consultation]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/hermes.gguf"
},
{
"name":"samantha.gguf [2.9GB; for medical student]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/samantha.gguf"
},
{
"name":"doublee.gguf [1.4GB; for electrical engineering]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/doublee.gguf"
},
{
"name":"math.gguf [3.1GB; for math/calculation]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/math.gguf"
},
{
"name":"math2.gguf [3.6GB; for math/calculation (2nd)]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/math2.gguf"
},
{
"name":"python.gguf [3.3GB; for coding python]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/python.gguf"
},
{
"name":"bloom.gguf [1.6GB; for quick response]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/bloom.gguf"
},
{
"name":"yarn.gguf [3.1GB; for problem solving]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/yarn.gguf"
},
{
"name":"dolphin.gguf [3.0GB; for solving problem]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/dolphin.gguf"
},
{
"name":"karen.gguf [2.7GB; for drafting letter/paper]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/karen.gguf"
},
{
"name":"beagle.gguf [3.5GB; for genearl response]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/beagle.gguf"
},
{
"name":"garrulus.gguf [2.7GB; for specific response]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/garrulus.gguf"
},
{
"name":"westlake.gguf [2.7GB; for common information]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/westlake.gguf"
},
{
"name":"openchat.gguf [3.0GB; for most purpose]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/openchat.gguf"
},
{
"name":"zephyr.gguf [3.0GB; for multilayer response]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/zephyr.gguf"
},
{
"name":"mamba.gguf [2.1GB; for quick/fast enquiry]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/mamba.gguf"
},
{
"name":"deep.gguf [3.1GB; for lengthy response]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/deep.gguf"
},
{
"name":"orca.gguf [4.1GB; for multiple response]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/orca.gguf"
},
{
"name":"claude.gguf [3.7GB; for comprehensive response]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/claude.gguf"
},
{
"name":"mistral.gguf [3.1GB; for instructional enquiry]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/mistral.gguf"
},
{
"name":"gpt.gguf [330MB; for general response]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/gpt.gguf"
},
{
"name":"gpt2.gguf [1.6GB; for general response (2nd)]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/gpt2.gguf"
},
{
"name":"gemma2.gguf [3.4GB; for specific response]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/gemma2.gguf"
},
{
"name":"gemma3-1b-it [687MB; for great summary; q4_0]",
"url":"https://huggingface.co/gguf-org/gemma-3-1b-it-gguf/resolve/main/gemma-3-1b-it-q4_0.gguf"
},
{
"name":"gemma3-1b-pt [1.0GB; for list response; q8_0]",
"url":"https://huggingface.co/gguf-org/gemma-3-1b-pt-gguf/resolve/main/gemma-3-1b-pt-q8_0.gguf"
},
{
"name":"phi3.gguf [2.3GB; for technical problem (3rd)]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/phi3.gguf"
},
{
"name":"phi3.5.gguf [2.0GB; for technical problem (3.5)]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/phi3.5.gguf"
},
{
"name":"phi4.gguf [5.6GB; for technical problem (4th)]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/phi4.gguf"
},
{
"name":"llama2.gguf [2.8GB; for comprehensive response]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/llama2.gguf"
},
{
"name":"llama3.gguf [3.6GB; for comprehensive response (3rd)]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/llama3.gguf"
},
{
"name":"llama3.2.gguf [1.7GB; for uncensored content]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/llama3.2.gguf"
},
{
"name":"guanaco.gguf [2.8GB; for uncensored content]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/guanaco.gguf"
},
{
"name":"wizard.gguf [3.3GB; for uncensored content]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/wizard.gguf"
},
{
"name":"solar.gguf [4.5GB; for uncensored content]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/solar.gguf"
},
{
"name":"maid.gguf [3.1GB; for adult/18+ nsfw]",
"url":"https://huggingface.co/calcuis/gguf/resolve/main/maid.gguf"
},
{
"name":"tiny.gguf [551MB; for testing pdf analyzor]",
"url":"https://github.com/calcuis/chatgpt-ai-model/releases/download/0.2/tiny.gguf"
}
]
gguf_connector/e.py
+48
View File
@@ -0,0 +1,48 @@
from .const import GGML_QUANT_VERSION, LlamaFileType
from .reader import GGUFReader
from .writer import GGUFWriter
def get_arch_str(reader):
field = reader.get_field('general.architecture')
return str(field.parts[field.data[-1]], encoding='utf-8')
def get_file_type(reader):
field = reader.get_field('general.file_type')
ft = int(field.parts[field.data[-1]])
return LlamaFileType(ft)
def extract_weight_tensors(input_path, output_path):
with open(input_path, 'rb') as f1:
reader1 = GGUFReader(f1)
arch = get_arch_str(reader1)
file_type = get_file_type(reader1)
print(f"Detected arch: '{arch}' (ftype: {str(file_type)})")
writer = GGUFWriter(path=None, arch=arch)
writer.add_quantization_version(GGML_QUANT_VERSION)
writer.add_file_type(file_type)
for tensor in reader1.tensors:
if tensor.name.endswith('.weight'):
writer.add_tensor(tensor.name, tensor.data, raw_dtype=
tensor.tensor_type)
with open(output_path, 'wb'):
writer.write_header_to_file(path=output_path)
writer.write_kv_data_to_file()
writer.write_tensors_to_file(progress=True)
writer.close()
import os
gguf_files = [file for file in os.listdir() if file.endswith('.gguf')]
if gguf_files:
print('GGUF file(s) available. Select which one to extract:')
for index, file_name in enumerate(gguf_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(gguf_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = gguf_files[choice_index]
print(f'Model file: {selected_file} is selected!')
input_path = selected_file
output_path = (
f'{os.path.splitext(input_path)[0]}-weight-extracted.gguf')
extract_weight_tensors(input_path, output_path)
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No GGUF files are available in the current directory.')
input('--- Press ENTER To Exit ---')
gguf_connector/f.py
+54
View File
@@ -0,0 +1,54 @@
from .const import GGML_QUANT_VERSION, LlamaFileType
from .reader import GGUFReader
from .writer import GGUFWriter
from glob import glob
def get_arch_str(reader):
field = reader.get_field('general.architecture')
return str(field.parts[field.data[-1]], encoding='utf-8')
def get_file_type(reader):
field = reader.get_field('general.file_type')
ft = int(field.parts[field.data[-1]])
return LlamaFileType(ft)
def merge_gguf_files(master_file, output_file):
with open(master_file, 'rb') as f_master:
reader_master = GGUFReader(f_master)
arch = get_arch_str(reader_master)
file_type = get_file_type(reader_master)
print(f'Using master file: {master_file}')
print(f"Detected arch: '{arch}' (ftype: {str(file_type)})")
writer = GGUFWriter(path=None, arch=arch)
writer.add_quantization_version(GGML_QUANT_VERSION)
writer.add_file_type(file_type)
for tensor in reader_master.tensors:
writer.add_tensor(tensor.name, tensor.data, raw_dtype=tensor.
tensor_type)
for file in glob('*.gguf'):
if file == master_file:
continue
print(f'Merging from: {file}')
with open(file, 'rb') as f:
reader = GGUFReader(f)
for tensor in reader.tensors:
writer.add_tensor(tensor.name, tensor.data, raw_dtype=
tensor.tensor_type)
with open(output_file, 'wb'):
writer.write_header_to_file(path=output_file)
writer.write_kv_data_to_file()
writer.write_tensors_to_file(progress=True)
writer.close()
print(f'Saved merged file as: {output_file}')
all_gguf_files = sorted(glob('*.gguf'))
if not all_gguf_files:
print('No .gguf files found in the current directory.')
else:
print('Available .gguf files:')
for idx, fname in enumerate(all_gguf_files):
print(f'{idx + 1}: {fname}')
selected = int(input('Select the master file by number: ')) - 1
master_file = all_gguf_files[selected]
ask = input('Assign a name other than output.gguf (Y/n)? ')
if ask.lower() == 'y':
given = input('Enter a file name: ')
else:
given = 'output.gguf'
merge_gguf_files(master_file, given)
gguf_connector/g.py
+29
View File
@@ -0,0 +1,29 @@
import os
gguf_files = [file for file in os.listdir() if file.endswith('.gguf')]
if gguf_files:
print('GGUF file(s) available. Select which one to use:')
for index, file_name in enumerate(gguf_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(gguf_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = gguf_files[choice_index]
print(f'Model file: {selected_file} is selected!')
ModelPath = selected_file
from ctransformers import AutoModelForCausalLM
llm = AutoModelForCausalLM.from_pretrained(ModelPath)
while True:
ask = input('Enter a Question (Q for quit): ')
if ask.lower() == 'q':
break
from rich.progress import Progress
with Progress(transient=True) as progress:
task = progress.add_task('Processing', total=None)
ans = llm(ask)
print(ask + ans)
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No GGUF files are available in the current directory.')
input('--- Press ENTER To Exit ---')
print('Goodbye!')
gguf_connector/gpp.py
+30
View File
@@ -0,0 +1,30 @@
import os
gguf_files = [file for file in os.listdir() if file.endswith('.gguf')]
if gguf_files:
print('GGUF file(s) available. Select which one to use:')
for index, file_name in enumerate(gguf_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(gguf_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = gguf_files[choice_index]
print(f'Model file: {selected_file} is selected!')
ModelPath = selected_file
from llama_cpp import Llama
llm = Llama(model_path=ModelPath)
while True:
ask = input('Enter a Question (Q for quit): ')
if ask.lower() == 'q':
break
from rich.progress import Progress
with Progress(transient=True) as progress:
task = progress.add_task('Processing', total=None)
output = llm('Q: ' + ask, max_tokens=4096, echo=True)
answer = output['choices'][0]['text']
print(answer + '\n')
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No GGUF files are available in the current directory.')
input('--- Press ENTER To Exit ---')
print('Goodbye!')
gguf_connector/i.py
+6
View File
@@ -0,0 +1,6 @@
import webbrowser
print('activating the default browser...')
webbrowser.open('https://gguf.io')
print(
'note: if you are encountering restricted access problem with this domain; visit https://gguf.us instead; same content'
)
gguf_connector/i2.py
+42
View File
@@ -0,0 +1,42 @@
import torch
import gradio as gr
from transformers import T5EncoderModel
from diffusers import PixArtSigmaPipeline
text_encoder = T5EncoderModel.from_pretrained('calcuis/ltxv-gguf',
gguf_file='t5xxl_fp16-q4_0.gguf', torch_dtype=torch.float16)
pipe = PixArtSigmaPipeline.from_pretrained('callgg/pixart-decoder',
text_encoder=text_encoder, torch_dtype=torch.float16)
pipe.enable_model_cpu_offload()
def generate_image(prompt, negative_prompt, width, height, guidance_scale,
num_inference_steps):
image = pipe(prompt=prompt, negative_prompt=negative_prompt, width=
width, height=height, guidance_scale=guidance_scale,
num_inference_steps=num_inference_steps).images[0]
image.save('output.png')
return 'output.png'
sample_prompts = ['close-up portrait of girl',
'close-up portrait of anime pig', 'close-up portrait of young lady']
sample_prompts = [[x] for x in sample_prompts]
block = gr.Blocks(title='gguf').queue()
with block:
gr.Markdown('## 📷 Image 2')
with gr.Row():
prompt_input = gr.Textbox(label='Prompt', placeholder=
'Enter your prompt here (or click Sample Prompt)', value='')
neg_prompt_input = gr.Textbox(label='Negative Prompt', value=
'blurry, cropped, ugly', visible=False)
quick_prompts = gr.Dataset(samples=sample_prompts, label=
'Sample Prompt', samples_per_page=1000, components=[prompt_input])
quick_prompts.click(lambda x: x[0], inputs=[quick_prompts], outputs
=prompt_input, show_progress=False, queue=False)
with gr.Row():
width_input = gr.Number(label='Width', value=1024)
height_input = gr.Number(label='Height', value=1024)
guidance_scale = gr.Number(label='Guidance Scale', value=3.5)
num_inference_steps = gr.Number(label='Inference Steps', value=25)
generate_btn = gr.Button('Generate')
output_image = gr.Image(label='Generated Picture')
generate_btn.click(fn=generate_image, inputs=[prompt_input,
neg_prompt_input, width_input, height_input, guidance_scale,
num_inference_steps], outputs=output_image)
block.launch()
gguf_connector/lazy.py
+163
View File
@@ -0,0 +1,163 @@
from __future__ import annotations
from abc import ABC, ABCMeta, abstractmethod
import logging
from typing import Any, Callable
import numpy as np
from numpy.typing import DTypeLike
logger = logging.getLogger(__name__)
class LazyMeta(ABCMeta):
def __new__(cls, name, bases, namespace, **kwargs):
def __getattr__(self, name):
meta_attr = getattr(self._meta, name)
if callable(meta_attr):
return type(self)._wrap_fn(lambda s, *args, **kwargs:
getattr(s, name)(*args, **kwargs), use_self=self)
elif isinstance(meta_attr, self._tensor_type):
return type(self)._wrap_fn(lambda s: getattr(s, name))(self)
else:
return meta_attr
namespace['__getattr__'] = __getattr__
def mk_wrap(op_name, *, meta_noop: bool=False):
def wrapped_special_op(self, *args, **kwargs):
return type(self)._wrap_fn(getattr(type(self)._tensor_type,
op_name), meta_noop=meta_noop)(self, *args, **kwargs)
return wrapped_special_op
for binary_op in ('lt', 'le', 'eq', 'ne', 'ge', 'gt', 'notabs',
'add', 'and', 'floordiv', 'invert', 'lshift', 'mod', 'mul',
'matmul', 'neg', 'or', 'pos', 'pow', 'rshift', 'sub', 'truediv',
'xor', 'iadd', 'iand', 'ifloordiv', 'ilshift', 'imod', 'imul',
'ior', 'irshift', 'isub', 'ixor', 'radd', 'rand', 'rfloordiv',
'rmul', 'ror', 'rpow', 'rsub', 'rtruediv', 'rxor'):
attr_name = f'__{binary_op}__'
namespace[attr_name] = mk_wrap(attr_name, meta_noop=True)
for special_op in ('getitem', 'setitem', 'len'):
attr_name = f'__{special_op}__'
namespace[attr_name] = mk_wrap(attr_name, meta_noop=False)
return super().__new__(cls, name, bases, namespace, **kwargs)
class LazyBase(ABC, metaclass=LazyMeta):
_tensor_type: type
_meta: Any
_data: Any | None
_args: tuple
_kwargs: dict[str, Any]
_func: Callable[[Any], Any] | None
def __init__(self, *, meta: Any, data: (Any | None)=None, args: tuple=(
), kwargs: (dict[str, Any] | None)=None, func: (Callable[[Any], Any
] | None)=None):
super().__init__()
self._meta = meta
self._data = data
self._args = args
self._kwargs = kwargs if kwargs is not None else {}
self._func = func
assert self._func is not None or self._data is not None
def __init_subclass__(cls):
if '_tensor_type' not in cls.__dict__:
raise TypeError(
f"property '_tensor_type' must be defined for {cls!r}")
return super().__init_subclass__()
@staticmethod
def _recurse_apply(o, fn):
if isinstance(o, (list, tuple)):
L = []
for item in o:
L.append(LazyBase._recurse_apply(item, fn))
if isinstance(o, tuple):
L = tuple(L)
return L
elif isinstance(o, LazyBase):
return fn(o)
else:
return o
@classmethod
def _wrap_fn(cls, fn, *, use_self: (LazyBase | None)=None, meta_noop: (
bool | DTypeLike | tuple[DTypeLike, Callable[[tuple[int, ...]],
tuple[int, ...]]])=False):
def wrapped_fn(*args, **kwargs):
if kwargs is None:
kwargs = {}
args = ((use_self,) if use_self is not None else ()) + args
meta_args = LazyBase._recurse_apply(args, lambda t: t._meta)
if isinstance(meta_noop, bool) and not meta_noop:
try:
res = fn(*meta_args, **kwargs)
except NotImplementedError:
res = None
else:
assert len(args) > 0
res = args[0]
assert isinstance(res, cls)
res = res._meta
if meta_noop is not True:
if isinstance(meta_noop, tuple):
dtype, shape = meta_noop
assert callable(shape)
res = cls.meta_with_dtype_and_shape(dtype, shape(
res.shape))
else:
res = cls.meta_with_dtype_and_shape(meta_noop, res.
shape)
if isinstance(res, cls._tensor_type):
return cls(meta=cls.eager_to_meta(res), args=args, kwargs=
kwargs, func=fn)
elif isinstance(res, tuple) and all(isinstance(t, cls.
_tensor_type) for t in res):
shared_args: list = [args, None]
def eager_tuple_element(a: list[Any], i: int=0, /, **kw):
assert len(a) == 2
if a[1] is None:
a[1] = fn(*a[0], **kw)
return a[1][i]
return tuple(cls(meta=cls.eager_to_meta(res[i]), args=(
shared_args, i), kwargs=kwargs, func=
eager_tuple_element) for i in range(len(res)))
else:
del res
eager_args = cls.to_eager(args)
return fn(*eager_args, **kwargs)
return wrapped_fn
@classmethod
def to_eager(cls, t):
def simple_to_eager(_t):
if _t._data is not None:
return _t._data
assert _t._func is not None
_t._args = cls._recurse_apply(_t._args, simple_to_eager)
_t._data = _t._func(*_t._args, **_t._kwargs)
assert _t._data is not None
assert _t._data.dtype == _t._meta.dtype
assert _t._data.shape == _t._meta.shape
return _t._data
return cls._recurse_apply(t, simple_to_eager)
@classmethod
def eager_to_meta(cls, t):
return cls.meta_with_dtype_and_shape(t.dtype, t.shape)
@classmethod
@abstractmethod
def meta_with_dtype_and_shape(cls, dtype, shape):
pass
@classmethod
def from_eager(cls, t):
if type(t) is cls:
return t
elif isinstance(t, cls._tensor_type):
return cls(meta=cls.eager_to_meta(t), data=t)
else:
return TypeError(
f'{type(t)!r} is not compatible with {cls._tensor_type!r}')
class LazyNumpyTensor(LazyBase):
_tensor_type = np.ndarray
shape: tuple[int, ...]
@classmethod
def meta_with_dtype_and_shape(cls, dtype, shape):
cheat = np.zeros(1, dtype)
return np.lib.stride_tricks.as_strided(cheat, shape, (0 for _ in shape)
)
def astype(self, dtype, *args, **kwargs):
meta = type(self).meta_with_dtype_and_shape(dtype, self._meta.shape)
full_args = (self, dtype) + args
return type(self)(meta=meta, args=full_args, kwargs=kwargs, func=lambda
a, *args, **kwargs: a.astype(*args, **kwargs))
def tofile(self, *args, **kwargs):
eager = LazyNumpyTensor.to_eager(self)
return eager.tofile(*args, **kwargs)
gguf_connector/logo.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 578 B

gguf_connector/m.py
+61
View File
@@ -0,0 +1,61 @@
import json, os.path, urllib.request
def get_file_size(url):
with urllib.request.urlopen(url) as response:
size = int(response.headers['Content-Length'])
return size
from tqdm import tqdm
def clone_file(url):
try:
file_size = get_file_size(url)
filename = os.path.basename(url)
with urllib.request.urlopen(url) as response, open(filename, 'wb'
) as file, tqdm(total=file_size, unit='B', unit_scale=True,
unit_divisor=1024, desc=f'Downloading {filename}') as pbar:
chunk_size = 1024
while True:
chunk = response.read(chunk_size)
if not chunk:
break
file.write(chunk)
pbar.update(len(chunk))
print(
f"File cloned successfully and saved as '{filename}' in the current directory."
)
except Exception as e:
print(f'Error: {e}')
def read_json_file(file_path):
with open(file_path, 'r') as file:
data = json.load(file)
return data
def extract_names(data):
for idx, entry in enumerate(data, start=1):
print(f"{idx}. {entry['name']}")
def handle_user_input(data):
while True:
user_choice = input(
f"Enter your choice (1 to {len(data)}) or 'q' to quit: ")
if user_choice.lower() == 'q':
break
try:
index = int(user_choice)
if 1 <= index <= len(data):
source_url = data[index - 1]['url']
clone_file(source_url)
break
else:
print('Invalid selection. Please enter a valid number.')
except ValueError:
print('Invalid input. Please enter a number.')
print("""Please select an option:
1. choose a connector
2. download model(s)"""
)
choice = input('Enter your choice (1 to 2): ')
if choice == '1':
from gguf_connector import menu
elif choice == '2':
file_path = os.path.join(os.path.dirname(__file__), 'data.json')
json_data = read_json_file(file_path)
print('Please select a GGUF file to download:')
extract_names(json_data)
handle_user_input(json_data)
gguf_connector/m2.py
+17
View File
@@ -0,0 +1,17 @@
import os
from tqdm import tqdm
def merge_gguf_files():
gguf_files = [f for f in os.listdir('.') if f.endswith('.gguf')]
if not gguf_files:
print('No .gguf files found in the current directory.')
return
filename = input('Enter the output file name (without .gguf): ').strip()
if not filename:
filename = 'model'
output_file = f'{filename}.gguf'
with open(output_file, 'wb') as outfile:
for fname in tqdm(gguf_files, desc='Merging files', unit='file'):
with open(fname, 'rb') as infile:
outfile.write(infile.read())
print(f'\nAll files merged into: {output_file}')
merge_gguf_files()
gguf_connector/ma.py
+25
View File
@@ -0,0 +1,25 @@
import os, safetensors.torch
def merge_safetensors(output):
files = [f for f in os.listdir() if f.endswith('.safetensors') and f !=
output]
if not files:
print('No safetensors files found.')
return
merged_data = {}
for file in files:
print(f'Loading {file}...')
data = safetensors.torch.load_file(file)
for key, value in data.items():
if key in merged_data:
print(f"Warning: Duplicate key '{key}' in {file}. Skipping.")
else:
merged_data[key] = value
print(f'Saving merged data to {output}...')
safetensors.torch.save_file(merged_data, output)
print('Merge completed successfully.')
ask = input('Assign a name other than model.safetensors (Y/n)? ')
if ask.lower() == 'y':
given = input('Enter a file name: ')
else:
given = 'model.safetensors'
merge_safetensors(given)
gguf_connector/menu.py
+34
View File
@@ -0,0 +1,34 @@
print("""Please select a connector:
1. llama.cpp
2. ctransformers""")
choice = input('Enter your choice (1 to 2): ')
if choice == '1':
print(
"""Connector: llama.cpp is selected!
Please select an interface:
1. graphical
2. command-line"""
)
option = input('Enter your choice (1 to 2): ')
if option == '1':
from .cpp import *
elif option == '2':
from .gpp import *
else:
print('Not a valid number.')
elif choice == '2':
print(
"""Connector: ctransformers is selected!
Please select an interface:
1. graphical
2. command-line"""
)
option = input('Enter your choice (1 to 2): ')
if option == '1':
from .c import *
elif option == '2':
from .g import *
else:
print('Not a valid number.')
else:
print('Not a valid number.')
gguf_connector/n.py
+18
View File
@@ -0,0 +1,18 @@
import os, subprocess
def clone_github_repo(repo_url):
try:
repo_name = repo_url.rstrip('/').split('/')[-1].replace('.git', '')
if os.path.exists(repo_name):
print(
f"Error: A folder named '{repo_name}' already exists in the current directory."
)
return
print(f"Cloning repository '{repo_url}'...")
subprocess.run(['git', 'clone', repo_url], check=True)
print(f"Repository '{repo_name}' cloned successfully.")
except subprocess.CalledProcessError as e:
print(f'Error: Failed to clone the repository. {e}')
except Exception as e:
print(f'Unexpected error: {e}')
repo_url = 'https://github.com/calcuis/gguf'
clone_github_repo(repo_url)
gguf_connector/o.py
+3
View File
@@ -0,0 +1,3 @@
import webbrowser
print('activating the default browser...')
webbrowser.open('https://gguf.org')
gguf_connector/p.py
+28
View File
@@ -0,0 +1,28 @@
import json, requests, urllib.request
from tqdm import tqdm
def read_json_file(file_path):
response = urllib.request.urlopen(file_path)
data = json.loads(response.read())
return data
def download_7z_file(url, output_filename):
response = requests.get(url, stream=True)
if response.status_code == 200:
total_size = int(response.headers.get('content-length', 0))
with open(output_filename, 'wb') as file, tqdm(desc=
f'Downloading {output_filename}', total=total_size, unit='B',
unit_scale=True, unit_divisor=1024) as progress_bar:
for chunk in response.iter_content(chunk_size=1024):
file.write(chunk)
progress_bar.update(len(chunk))
print(f'Downloaded {output_filename} to the current directory.')
else:
print(
'Failed to download the file. Seems encountering a connection problem.'
)
ver = 'https://raw.githubusercontent.com/calcuis/gguf-pack/main/version.json'
v = read_json_file(ver)
url = (
f"https://github.com/calcuis/gguf-pack/releases/download/{v[0]['version']}/GGUF_windows_portable.7z"
)
output_filename = 'gguf.7z'
download_7z_file(url, output_filename)
gguf_connector/p1.py
+28
View File
@@ -0,0 +1,28 @@
import json, requests, urllib.request
from tqdm import tqdm
def read_json_file(file_path):
response = urllib.request.urlopen(file_path)
data = json.loads(response.read())
return data
def download_7z_file(url, output_filename):
response = requests.get(url, stream=True)
if response.status_code == 200:
total_size = int(response.headers.get('content-length', 0))
with open(output_filename, 'wb') as file, tqdm(desc=
f'Downloading {output_filename}', total=total_size, unit='B',
unit_scale=True, unit_divisor=1024) as progress_bar:
for chunk in response.iter_content(chunk_size=1024):
file.write(chunk)
progress_bar.update(len(chunk))
print(f'Downloaded {output_filename} to the current directory.')
else:
print(
'Failed to download the file. Seems encountering a connection problem.'
)
ver = 'https://raw.githubusercontent.com/calcuis/packpack/main/version.json'
v = read_json_file(ver)
url = (
f"https://github.com/calcuis/packpack/releases/download/{v[0]['version']}/packpack.7z"
)
output_filename = 'framepack.7z'
download_7z_file(url, output_filename)
gguf_connector/p2.py
+18
View File
@@ -0,0 +1,18 @@
import os, subprocess
def clone_github_repo(repo_url):
try:
repo_name = repo_url.rstrip('/').split('/')[-1].replace('.git', '')
if os.path.exists(repo_name):
print(
f"Error: A folder named '{repo_name}' already exists in the current directory."
)
return
print(f"Cloning repository '{repo_url}'...")
subprocess.run(['git', 'clone', repo_url], check=True)
print(f"Repository '{repo_name}' cloned successfully.")
except subprocess.CalledProcessError as e:
print(f'Error: Failed to clone the repository. {e}')
except Exception as e:
print(f'Unexpected error: {e}')
repo_url = 'https://github.com/calcuis/packpack'
clone_github_repo(repo_url)
gguf_connector/pg.py
+62
View File
@@ -0,0 +1,62 @@
import os
def wav_handler():
wav_files = [file for file in os.listdir() if file.endswith('.wav')]
if wav_files:
print('WAV file(s) available. Select which one to use:')
for index, file_name in enumerate(wav_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(wav_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = wav_files[choice_index]
print(f'WAV file: {selected_file} is selected!')
import speech_recognition as sr
r = sr.Recognizer()
with sr.AudioFile(selected_file) as source:
audio = r.record(source)
text = r.recognize_google(audio)
try:
from rich.console import Console
console = Console()
console.print(f'Content recognized as: [green]' + text)
input(
'\n---Enter to prompt the WAV content recognized above into GGUF model---'
)
print('Processing...')
output = llm('Q: ' + text, max_tokens=4096, echo=True)
ans = output['choices'][0]['text']
print(ans + '\n')
except sr.UnknownValueError:
print('Could not understand audio')
except sr.RequestError as e:
print('Error; {0}'.format(e))
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No WAV files are available in the current directory.')
input('--- Press ENTER To Skip ---')
gguf_files = [file for file in os.listdir() if file.endswith('.gguf')]
if gguf_files:
print('GGUF file(s) available. Select which one to use:')
for index, file_name in enumerate(gguf_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(gguf_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = gguf_files[choice_index]
print(f'Model file: {selected_file} is selected!')
ModelPath = selected_file
print('Processing...')
from llama_cpp import Llama
llm = Llama(model_path=ModelPath)
while True:
ask = input('---Enter to select a WAV file (Q for quit)---')
if ask == 'q' or ask == 'Q':
break
wav_handler()
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No GGUF files are available in the current directory.')
input('--- Press ENTER To Exit ---')
print('Goodbye!')
gguf_connector/pp.py
+65
View File
@@ -0,0 +1,65 @@
import os
def pdf_handler():
pdf_files = [file for file in os.listdir() if file.endswith('.pdf')]
def join_text(input_text):
joined_text = ' '.join(input_text.splitlines())
return joined_text
if pdf_files:
print('PDF file(s) available. Select which one to use:')
for index, file_name in enumerate(pdf_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(pdf_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = pdf_files[choice_index]
print(f'PDF file: {selected_file} is selected!')
from pypdf import PdfReader
reader = PdfReader(selected_file)
text = ''
number_of_pages = len(reader.pages)
for i in range(number_of_pages):
page = reader.pages[i]
text += page.extract_text()
output_text = join_text(text)
inject = f'analyze the content below: ' + output_text
from rich.console import Console
console = Console()
console.print(f'\nPDF cotent extracted as below:\n\n[green]' + text
)
input('---Enter to analyze the PDF content above---')
from rich.progress import Progress
with Progress(transient=True) as progress:
task = progress.add_task('Processing', total=None)
output = llm('Q: ' + inject, max_tokens=32768, echo=False)
ans = output['choices'][0]['text']
print(inject + '\n\nOutput:' + ans)
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No PDF files are available in the current directory.')
input('--- Press ENTER To Skip ---')
gguf_files = [file for file in os.listdir() if file.endswith('.gguf')]
if gguf_files:
print('GGUF file(s) available. Select which one to use:')
for index, file_name in enumerate(gguf_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(gguf_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = gguf_files[choice_index]
print(f'Model file: {selected_file} is selected!')
ModelPath = selected_file
print('Processing...')
from llama_cpp import Llama
llm = Llama(model_path=ModelPath)
while True:
ask = input('---Enter to select a PDF file (Q for quit)---')
if ask.lower() == 'q':
break
pdf_handler()
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No GGUF files are available in the current directory.')
input('--- Press ENTER To Exit ---')
print('Goodbye!')
gguf_connector/ps.py
+62
View File
@@ -0,0 +1,62 @@
import os
def wav_handler():
wav_files = [file for file in os.listdir() if file.endswith('.wav')]
if wav_files:
print('WAV file(s) available. Select which one to use:')
for index, file_name in enumerate(wav_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(wav_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = wav_files[choice_index]
print(f'WAV file: {selected_file} is selected!')
import speech_recognition as sr
r = sr.Recognizer()
with sr.AudioFile(selected_file) as source:
audio = r.record(source)
text = r.recognize_sphinx(audio)
try:
from rich.console import Console
console = Console()
console.print(f'Content recognized as: [green]' + text)
input(
'\n---Enter to prompt the WAV content recognized above into GGUF model---'
)
print('Processing...')
output = llm('Q: ' + text, max_tokens=4096, echo=True)
ans = output['choices'][0]['text']
print(ans + '\n')
except sr.UnknownValueError:
print('Could not understand audio')
except sr.RequestError as e:
print('Error; {0}'.format(e))
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No WAV files are available in the current directory.')
input('--- Press ENTER To Skip ---')
gguf_files = [file for file in os.listdir() if file.endswith('.gguf')]
if gguf_files:
print('GGUF file(s) available. Select which one to use:')
for index, file_name in enumerate(gguf_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(gguf_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = gguf_files[choice_index]
print(f'Model file: {selected_file} is selected!')
ModelPath = selected_file
print('Processing...')
from llama_cpp import Llama
llm = Llama(model_path=ModelPath)
while True:
ask = input('---Enter to select a WAV file (Q for quit)---')
if ask == 'q' or ask == 'Q':
break
wav_handler()
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No GGUF files are available in the current directory.')
input('--- Press ENTER To Exit ---')
print('Goodbye!')
gguf_connector/q.py
+43
View File
@@ -0,0 +1,43 @@
import torch
from safetensors.torch import load_file, save_file
from tqdm import tqdm
def quantize_to_fp8(tensor):
if tensor.dtype != torch.bfloat16:
raise ValueError('Input tensor must be in BF16 format.')
tensor = tensor.to(torch.float16)
fp8_max = 240.0
fp8_min = -fp8_max
clamped_tensor = tensor.clamp(min=fp8_min, max=fp8_max)
scale = fp8_max / torch.max(torch.abs(clamped_tensor))
quantized_tensor = torch.round(clamped_tensor * scale) / scale
return quantized_tensor.to(torch.float8_e4m3fn)
import os
safetensors_files = [file for file in os.listdir() if file.endswith(
'.safetensors')]
if safetensors_files:
print('Safetensors file(s) available. Select which one to convert:')
for index, file_name in enumerate(safetensors_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = safetensors_files[choice_index]
print(f'Model file: {selected_file} is selected!')
input_file = selected_file
output_file = (
f'{os.path.splitext(input_file)[0]}_fp8_e4m3fn.safetensors')
data = load_file(input_file)
quantized_data = {}
print('Starting quantization process...')
for key, tensor in tqdm(data.items(), desc='Quantizing tensors',
unit='tensor'):
tensor = tensor.to(dtype=torch.bfloat16, device='cuda')
quantized_tensor = quantize_to_fp8(tensor)
quantized_data[key] = quantized_tensor.cpu()
save_file(quantized_data, output_file)
print(f'Quantized safetensors saved to {output_file}.')
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No safetensors files are available in the current directory.')
input('--- Press ENTER To Exit ---')
gguf_connector/q1.py
+43
View File
@@ -0,0 +1,43 @@
import torch
from safetensors.torch import load_file, save_file
from tqdm import tqdm
def quantize_to_fp8(tensor):
if tensor.dtype != torch.bfloat16:
raise ValueError('Input tensor must be in BF16 format.')
tensor = tensor.to(torch.float16)
fp8_max = 240.0
fp8_min = -fp8_max
clamped_tensor = tensor.clamp(min=fp8_min, max=fp8_max)
scale = fp8_max / torch.max(torch.abs(clamped_tensor))
quantized_tensor = torch.round(clamped_tensor * scale) / scale
return quantized_tensor.to(torch.float8_e4m3fn)
import os
safetensors_files = [file for file in os.listdir() if file.endswith(
'.safetensors')]
if safetensors_files:
print('Safetensors file(s) available. Select which one to convert:')
for index, file_name in enumerate(safetensors_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = safetensors_files[choice_index]
print(f'Model file: {selected_file} is selected!')
input_file = selected_file
output_file = (
f'{os.path.splitext(input_file)[0]}_fp8_e4m3fn.safetensors')
data = load_file(input_file)
quantized_data = {}
print('Starting quantization process...')
for key, tensor in tqdm(data.items(), desc='Quantizing tensors',
unit='tensor'):
tensor = tensor.to(dtype=torch.bfloat16, device='cpu')
quantized_tensor = quantize_to_fp8(tensor)
quantized_data[key] = quantized_tensor.cpu()
save_file(quantized_data, output_file)
print(f'Quantized safetensors saved to {output_file}.')
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No safetensors files are available in the current directory.')
input('--- Press ENTER To Exit ---')
gguf_connector/q2.py
+32
View File
@@ -0,0 +1,32 @@
import torch
from safetensors.torch import load_file, save_file
from tqdm import tqdm
def convert_safetensors_to_fp32(file_path):
print(f'Loading: {file_path}')
tensors = load_file(file_path)
converted_tensors = {}
for key, tensor in tqdm(tensors.items(), desc='Converting tensors',
unit='tensor'):
converted_tensors[key] = tensor.to(torch.float32)
output_file = file_path.replace('.safetensors', '_fp32.safetensors')
save_file(converted_tensors, output_file)
print(f'Converted and saved to: {output_file}')
def upscale_to_fp32():
import os
safetensors_files = [f for f in os.listdir('.') if f.endswith(
'.safetensors')]
if not safetensors_files:
print('No .safetensors files found in the current directory.')
return
print('Select a file to convert:')
for idx, file in enumerate(safetensors_files, 1):
print(f'{idx}. {file}')
try:
choice = int(input('Enter the number of the file: ')) - 1
if choice < 0 or choice >= len(safetensors_files):
print('Invalid selection.')
return
convert_safetensors_to_fp32(safetensors_files[choice])
except ValueError:
print('Please enter a valid number.')
upscale_to_fp32()
gguf_connector/quant.py
+755
View File
File diff suppressed because one or more lines are too long
gguf_connector/quant2.py
+191
View File
@@ -0,0 +1,191 @@
import torch
from .quant import dequantize as gq
from .reader import GGMLQuantizationType, GGML_QUANT_SIZES
from tqdm import tqdm
TORCH_COMPATIBLE_QTYPES = {None, GGMLQuantizationType.F32,
GGMLQuantizationType.F16}
def is_torch_compatible(tensor):
return tensor is None or getattr(tensor, 'tensor_type', None
) in TORCH_COMPATIBLE_QTYPES
def is_quantized(tensor):
return not is_torch_compatible(tensor)
def dequantize_tensor(tensor, dtype=None, dequant_dtype=None):
qtype = getattr(tensor, 'tensor_type', None)
oshape = getattr(tensor, 'tensor_shape', tensor.shape)
if qtype in TORCH_COMPATIBLE_QTYPES:
return tensor.to(dtype)
elif qtype in dequantize_functions:
dequant_dtype = dtype if dequant_dtype == 'target' else dequant_dtype
return dequantize(tensor.data, qtype, oshape, dtype=dequant_dtype).to(
dtype)
else:
tqdm.write(f'Pushing back to numpy dequant for qtype: {qtype}')
new = gq(tensor.cpu().numpy(), qtype)
return torch.from_numpy(new).to(tensor.device, dtype=dtype)
def dequantize(data, qtype, oshape, dtype=None):
block_size, type_size = GGML_QUANT_SIZES[qtype]
dequantize_blocks = dequantize_functions[qtype]
rows = data.reshape((-1, data.shape[-1])).view(torch.uint8)
n_blocks = rows.numel() // type_size
blocks = rows.reshape((n_blocks, type_size))
blocks = dequantize_blocks(blocks, block_size, type_size, dtype)
return blocks.reshape(oshape)
def to_uint32(x):
x = x.view(torch.uint8).to(torch.int32)
return (x[:, 0] | x[:, 1] << 8 | x[:, 2] << 16 | x[:, 3] << 24).unsqueeze(1
)
def split_block_dims(blocks, *args):
n_max = blocks.shape[1]
dims = list(args) + [n_max - sum(args)]
return torch.split(blocks, dims, dim=1)
def dequantize_blocks_BF16(blocks, block_size, type_size, dtype=None):
return (blocks.view(torch.int16).to(torch.int32) << 16).view(torch.float32)
def dequantize_blocks_Q8_0(blocks, block_size, type_size, dtype=None):
d, x = split_block_dims(blocks, 2)
d = d.view(torch.float16).to(dtype)
x = x.view(torch.int8)
return d * x
def dequantize_blocks_Q5_1(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d, m, qh, qs = split_block_dims(blocks, 2, 2, 4)
d = d.view(torch.float16).to(dtype)
m = m.view(torch.float16).to(dtype)
qh = to_uint32(qh)
qh = qh.reshape((n_blocks, 1)) >> torch.arange(32, device=d.device,
dtype=torch.int32).reshape(1, 32)
ql = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0,
4], device=d.device, dtype=torch.uint8).reshape(1, 1, 2, 1)
qh = (qh & 1).to(torch.uint8)
ql = (ql & 15).reshape((n_blocks, -1))
qs = ql | qh << 4
return d * qs + m
def dequantize_blocks_Q5_0(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d, qh, qs = split_block_dims(blocks, 2, 4)
d = d.view(torch.float16).to(dtype)
qh = to_uint32(qh)
qh = qh.reshape(n_blocks, 1) >> torch.arange(32, device=d.device, dtype
=torch.int32).reshape(1, 32)
ql = qs.reshape(n_blocks, -1, 1, block_size // 2) >> torch.tensor([0, 4
], device=d.device, dtype=torch.uint8).reshape(1, 1, 2, 1)
qh = (qh & 1).to(torch.uint8)
ql = (ql & 15).reshape(n_blocks, -1)
qs = (ql | qh << 4).to(torch.int8) - 16
return d * qs
def dequantize_blocks_Q4_1(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d, m, qs = split_block_dims(blocks, 2, 2)
d = d.view(torch.float16).to(dtype)
m = m.view(torch.float16).to(dtype)
qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0,
4], device=d.device, dtype=torch.uint8).reshape(1, 1, 2, 1)
qs = (qs & 15).reshape(n_blocks, -1)
return d * qs + m
def dequantize_blocks_Q4_0(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d, qs = split_block_dims(blocks, 2)
d = d.view(torch.float16).to(dtype)
qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0,
4], device=d.device, dtype=torch.uint8).reshape((1, 1, 2, 1))
qs = (qs & 15).reshape((n_blocks, -1)).to(torch.int8) - 8
return d * qs
QK_K = 256
K_SCALE_SIZE = 12
def get_scale_min(scales):
n_blocks = scales.shape[0]
scales = scales.view(torch.uint8)
scales = scales.reshape((n_blocks, 3, 4))
d, m, m_d = torch.split(scales, scales.shape[-2] // 3, dim=-2)
sc = torch.cat([d & 63, m_d & 15 | d >> 2 & 48], dim=-1)
min = torch.cat([m & 63, m_d >> 4 | m >> 2 & 48], dim=-1)
return sc.reshape((n_blocks, 8)), min.reshape((n_blocks, 8))
def dequantize_blocks_Q6_K(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
ql, qh, scales, d = split_block_dims(blocks, QK_K // 2, QK_K // 4, QK_K //
16)
scales = scales.view(torch.int8).to(dtype)
d = d.view(torch.float16).to(dtype)
d = (d * scales).reshape((n_blocks, QK_K // 16, 1))
ql = ql.reshape((n_blocks, -1, 1, 64)) >> torch.tensor([0, 4], device=d
.device, dtype=torch.uint8).reshape((1, 1, 2, 1))
ql = (ql & 15).reshape((n_blocks, -1, 32))
qh = qh.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 2, 4, 6],
device=d.device, dtype=torch.uint8).reshape((1, 1, 4, 1))
qh = (qh & 3).reshape((n_blocks, -1, 32))
q = (ql | qh << 4).to(torch.int8) - 32
q = q.reshape((n_blocks, QK_K // 16, -1))
return (d * q).reshape((n_blocks, QK_K))
def dequantize_blocks_Q5_K(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d, dmin, scales, qh, qs = split_block_dims(blocks, 2, 2, K_SCALE_SIZE,
QK_K // 8)
d = d.view(torch.float16).to(dtype)
dmin = dmin.view(torch.float16).to(dtype)
sc, m = get_scale_min(scales)
d = (d * sc).reshape((n_blocks, -1, 1))
dm = (dmin * m).reshape((n_blocks, -1, 1))
ql = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 4], device=d
.device, dtype=torch.uint8).reshape((1, 1, 2, 1))
qh = qh.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([i for i in
range(8)], device=d.device, dtype=torch.uint8).reshape((1, 1, 8, 1))
ql = (ql & 15).reshape((n_blocks, -1, 32))
qh = (qh & 1).reshape((n_blocks, -1, 32))
q = ql | qh << 4
return (d * q - dm).reshape((n_blocks, QK_K))
def dequantize_blocks_Q4_K(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d, dmin, scales, qs = split_block_dims(blocks, 2, 2, K_SCALE_SIZE)
d = d.view(torch.float16).to(dtype)
dmin = dmin.view(torch.float16).to(dtype)
sc, m = get_scale_min(scales)
d = (d * sc).reshape((n_blocks, -1, 1))
dm = (dmin * m).reshape((n_blocks, -1, 1))
qs = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 4], device=d
.device, dtype=torch.uint8).reshape((1, 1, 2, 1))
qs = (qs & 15).reshape((n_blocks, -1, 32))
return (d * qs - dm).reshape((n_blocks, QK_K))
def dequantize_blocks_Q3_K(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
hmask, qs, scales, d = split_block_dims(blocks, QK_K // 8, QK_K // 4, 12)
d = d.view(torch.float16).to(dtype)
lscales, hscales = scales[:, :8], scales[:, 8:]
lscales = lscales.reshape((n_blocks, 1, 8)) >> torch.tensor([0, 4],
device=d.device, dtype=torch.uint8).reshape((1, 2, 1))
lscales = lscales.reshape((n_blocks, 16))
hscales = hscales.reshape((n_blocks, 1, 4)) >> torch.tensor([0, 2, 4, 6
], device=d.device, dtype=torch.uint8).reshape((1, 4, 1))
hscales = hscales.reshape((n_blocks, 16))
scales = lscales & 15 | (hscales & 3) << 4
scales = scales.to(torch.int8) - 32
dl = (d * scales).reshape((n_blocks, 16, 1))
ql = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 2, 4, 6],
device=d.device, dtype=torch.uint8).reshape((1, 1, 4, 1))
qh = hmask.reshape(n_blocks, -1, 1, 32) >> torch.tensor([i for i in
range(8)], device=d.device, dtype=torch.uint8).reshape((1, 1, 8, 1))
ql = ql.reshape((n_blocks, 16, QK_K // 16)) & 3
qh = qh.reshape((n_blocks, 16, QK_K // 16)) & 1 ^ 1
q = ql.to(torch.int8) - (qh << 2).to(torch.int8)
return (dl * q).reshape((n_blocks, QK_K))
def dequantize_blocks_Q2_K(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
scales, qs, d, dmin = split_block_dims(blocks, QK_K // 16, QK_K // 4, 2)
d = d.view(torch.float16).to(dtype)
dmin = dmin.view(torch.float16).to(dtype)
dl = (d * (scales & 15)).reshape((n_blocks, QK_K // 16, 1))
ml = (dmin * (scales >> 4)).reshape((n_blocks, QK_K // 16, 1))
shift = torch.tensor([0, 2, 4, 6], device=d.device, dtype=torch.uint8
).reshape((1, 1, 4, 1))
qs = qs.reshape((n_blocks, -1, 1, 32)) >> shift & 3
qs = qs.reshape((n_blocks, QK_K // 16, 16))
qs = dl * qs - ml
return qs.reshape((n_blocks, -1))
dequantize_functions = {GGMLQuantizationType.BF16: dequantize_blocks_BF16,
GGMLQuantizationType.Q8_0: dequantize_blocks_Q8_0, GGMLQuantizationType
.Q5_1: dequantize_blocks_Q5_1, GGMLQuantizationType.Q5_0:
dequantize_blocks_Q5_0, GGMLQuantizationType.Q4_1:
dequantize_blocks_Q4_1, GGMLQuantizationType.Q4_0:
dequantize_blocks_Q4_0, GGMLQuantizationType.Q6_K:
dequantize_blocks_Q6_K, GGMLQuantizationType.Q5_K:
dequantize_blocks_Q5_K, GGMLQuantizationType.Q4_K:
dequantize_blocks_Q4_K, GGMLQuantizationType.Q3_K:
dequantize_blocks_Q3_K, GGMLQuantizationType.Q2_K: dequantize_blocks_Q2_K}
gguf_connector/quant2a.py
+203
View File
@@ -0,0 +1,203 @@
import torch
from .quant import dequantize as gq
from .reader import GGMLQuantizationType, GGML_QUANT_SIZES
from tqdm import tqdm
TORCH_COMPATIBLE_QTYPES = {None, GGMLQuantizationType.F32,
GGMLQuantizationType.F16}
def is_torch_compatible(tensor):
return tensor is None or getattr(tensor, 'tensor_type', None
) in TORCH_COMPATIBLE_QTYPES
def is_quantized(tensor):
return not is_torch_compatible(tensor)
def dequantize(data, qtype, oshape, dtype=None):
block_size, type_size = GGML_QUANT_SIZES[qtype]
dequantize_blocks = dequantize_functions[qtype]
rows = data.reshape((-1, data.shape[-1])).view(torch.uint8)
n_blocks = rows.numel() // type_size
blocks = rows.reshape((n_blocks, type_size))
blocks = dequantize_blocks(blocks, block_size, type_size, dtype)
return blocks.reshape(oshape)
def dequantize_tensor(tensor, dtype=None, dequant_dtype=None):
qtype = getattr(tensor, 'tensor_type', None)
oshape = getattr(tensor, 'tensor_shape', tensor.shape)
if qtype in TORCH_COMPATIBLE_QTYPES:
return tensor.to(dtype)
elif qtype in dequantize_functions:
dequant_dtype = dtype if dequant_dtype == 'target' else dequant_dtype
return dequantize(tensor.data, qtype, oshape, dtype=dequant_dtype).to(
dtype)
else:
tqdm.write(
f'Pushing back to numpy dequant (cpu offload) for qtype: {qtype}')
new = gq(tensor.cpu().numpy(), qtype)
return torch.from_numpy(new).to(tensor.device, dtype=dtype)
def to_uint32(x):
x = x.view(torch.uint8).to(torch.int32)
return (x[:, 0] | x[:, 1] << 8 | x[:, 2] << 16 | x[:, 3] << 24).unsqueeze(1
)
def dequantize_blocks_BF16(blocks, block_size, type_size, dtype=None):
return (blocks.view(torch.int16).to(torch.int32) << 16).view(torch.float32)
def dequantize_blocks_Q8_0(blocks, block_size, type_size, dtype=None):
d = blocks[:, :2].view(torch.float16).to(dtype)
x = blocks[:, 2:].view(torch.int8)
return d * x
def dequantize_blocks_Q5_0(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d = blocks[:, :2].view(torch.float16).to(dtype)
qh = blocks[:, 2:6]
qs = blocks[:, 6:]
qh = to_uint32(qh)
qh = qh.reshape(n_blocks, 1) >> torch.arange(32, device=d.device, dtype
=torch.int32).reshape(1, 32)
ql = qs.reshape(n_blocks, -1, 1, block_size // 2) >> torch.tensor([0, 4
], device=d.device, dtype=torch.uint8).reshape(1, 1, 2, 1)
qh = (qh & 1).to(torch.uint8)
ql = (ql & 15).reshape(n_blocks, -1)
qs = (ql | qh << 4).to(torch.int8) - 16
return d * qs
def dequantize_blocks_Q4_0(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d = blocks[:, :2].view(torch.float16).to(dtype)
qs = blocks[:, 2:]
qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0,
4], device=d.device, dtype=torch.uint8).reshape((1, 1, 2, 1))
qs = (qs & 15).reshape((n_blocks, -1)).to(torch.int8) - 8
return d * qs
def dequantize_blocks_Q5_1(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d = blocks[:, :2].view(torch.float16).to(dtype)
m = blocks[:, 2:4].view(torch.float16).to(dtype)
qh = blocks[:, 4:8]
qs = blocks[:, 8:]
qh = to_uint32(qh)
qh = qh.reshape((n_blocks, 1)) >> torch.arange(32, device=d.device,
dtype=torch.int32).reshape(1, 32)
ql = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0,
4], device=d.device, dtype=torch.uint8).reshape(1, 1, 2, 1)
qh = (qh & 1).to(torch.uint8)
ql = (ql & 15).reshape((n_blocks, -1))
qs = ql | qh << 4
return d * qs + m
def dequantize_blocks_Q4_1(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d = blocks[:, :2].view(torch.float16).to(dtype)
m = blocks[:, 2:4].view(torch.float16).to(dtype)
qs = blocks[:, 4:]
qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0,
4], device=d.device, dtype=torch.uint8).reshape(1, 1, 2, 1)
qs = (qs & 15).reshape(n_blocks, -1)
return d * qs + m
QK_K = 256
K_SCALE = 12
def split_block_dims(blocks, *args):
n_max = blocks.shape[1]
dims = list(args) + [n_max - sum(args)]
return torch.split(blocks, dims, dim=1)
def dequantize_blocks_Q6_K(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
ql, qh, scales, d = split_block_dims(blocks, QK_K // 2, QK_K // 4, QK_K //
16)
scales = scales.view(torch.int8).to(dtype)
d = d.view(torch.float16).to(dtype)
d = (d * scales).reshape((n_blocks, QK_K // 16, 1))
ql = ql.reshape((n_blocks, -1, 1, 64)) >> torch.tensor([0, 4], device=d
.device, dtype=torch.uint8).reshape((1, 1, 2, 1))
ql = (ql & 15).reshape((n_blocks, -1, 32))
qh = qh.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 2, 4, 6],
device=d.device, dtype=torch.uint8).reshape((1, 1, 4, 1))
qh = (qh & 3).reshape((n_blocks, -1, 32))
q = (ql | qh << 4).to(torch.int8) - 32
q = q.reshape((n_blocks, QK_K // 16, -1))
return (d * q).reshape((n_blocks, QK_K))
def dequantize_blocks_Q3_K(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
hmask, qs, scales, d = split_block_dims(blocks, QK_K // 8, QK_K // 4, 12)
d = d.view(torch.float16).to(dtype)
lscales, hscales = scales[:, :8], scales[:, 8:]
lscales = lscales.reshape((n_blocks, 1, 8)) >> torch.tensor([0, 4],
device=d.device, dtype=torch.uint8).reshape((1, 2, 1))
lscales = lscales.reshape((n_blocks, 16))
hscales = hscales.reshape((n_blocks, 1, 4)) >> torch.tensor([0, 2, 4, 6
], device=d.device, dtype=torch.uint8).reshape((1, 4, 1))
hscales = hscales.reshape((n_blocks, 16))
scales = lscales & 15 | (hscales & 3) << 4
scales = scales.to(torch.int8) - 32
dl = (d * scales).reshape((n_blocks, 16, 1))
ql = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 2, 4, 6],
device=d.device, dtype=torch.uint8).reshape((1, 1, 4, 1))
qh = hmask.reshape(n_blocks, -1, 1, 32) >> torch.tensor([i for i in
range(8)], device=d.device, dtype=torch.uint8).reshape((1, 1, 8, 1))
ql = ql.reshape((n_blocks, 16, QK_K // 16)) & 3
qh = qh.reshape((n_blocks, 16, QK_K // 16)) & 1 ^ 1
q = ql.to(torch.int8) - (qh << 2).to(torch.int8)
return (dl * q).reshape((n_blocks, QK_K))
def dequantize_blocks_Q2_K(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
scales, qs, d, dmin = split_block_dims(blocks, QK_K // 16, QK_K // 4, 2)
d = d.view(torch.float16).to(dtype)
dmin = dmin.view(torch.float16).to(dtype)
dl = (d * (scales & 15)).reshape((n_blocks, QK_K // 16, 1))
ml = (dmin * (scales >> 4)).reshape((n_blocks, QK_K // 16, 1))
shift = torch.tensor([0, 2, 4, 6], device=d.device, dtype=torch.uint8
).reshape((1, 1, 4, 1))
qs = qs.reshape((n_blocks, -1, 1, 32)) >> shift & 3
qs = qs.reshape((n_blocks, QK_K // 16, 16))
qs = dl * qs - ml
return qs.reshape((n_blocks, -1))
def get_scale_min(scales):
n_blocks = scales.shape[0]
scales = scales.view(torch.uint8)
scales = scales.reshape((n_blocks, 3, 4))
d, m, m_d = torch.split(scales, scales.shape[-2] // 3, dim=-2)
sc = torch.cat([d & 63, m_d & 15 | d >> 2 & 48], dim=-1)
min = torch.cat([m & 63, m_d >> 4 | m >> 2 & 48], dim=-1)
return sc.reshape((n_blocks, 8)), min.reshape((n_blocks, 8))
def dequantize_blocks_Q5_K(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d, dmin, scales, qh, qs = split_block_dims(blocks, 2, 2, K_SCALE, QK_K // 8
)
d = d.view(torch.float16).to(dtype)
dmin = dmin.view(torch.float16).to(dtype)
sc, m = get_scale_min(scales)
d = (d * sc).reshape((n_blocks, -1, 1))
dm = (dmin * m).reshape((n_blocks, -1, 1))
ql = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 4], device=d
.device, dtype=torch.uint8).reshape((1, 1, 2, 1))
qh = qh.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([i for i in
range(8)], device=d.device, dtype=torch.uint8).reshape((1, 1, 8, 1))
ql = (ql & 15).reshape((n_blocks, -1, 32))
qh = (qh & 1).reshape((n_blocks, -1, 32))
q = ql | qh << 4
return (d * q - dm).reshape((n_blocks, QK_K))
def dequantize_blocks_Q4_K(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d, dmin, scales, qs = split_block_dims(blocks, 2, 2, K_SCALE)
d = d.view(torch.float16).to(dtype)
dmin = dmin.view(torch.float16).to(dtype)
sc, m = get_scale_min(scales)
d = (d * sc).reshape((n_blocks, -1, 1))
dm = (dmin * m).reshape((n_blocks, -1, 1))
qs = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 4], device=d
.device, dtype=torch.uint8).reshape((1, 1, 2, 1))
qs = (qs & 15).reshape((n_blocks, -1, 32))
return (d * qs - dm).reshape((n_blocks, QK_K))
def dequantize_blocks_IQ4_NL(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d, qs = split_block_dims(blocks, 2)
d = d.view(torch.float16).to(dtype)
qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0,
4], device=qs.device, dtype=torch.uint8).reshape((1, 1, 2, 1))
qs = (qs & 15).reshape((n_blocks, -1)).to(torch.int8) - 8
return d * qs
dequantize_functions = {GGMLQuantizationType.BF16: dequantize_blocks_BF16,
GGMLQuantizationType.Q8_0: dequantize_blocks_Q8_0, GGMLQuantizationType
.Q5_1: dequantize_blocks_Q5_1, GGMLQuantizationType.Q5_0:
dequantize_blocks_Q5_0, GGMLQuantizationType.Q4_1:
dequantize_blocks_Q4_1, GGMLQuantizationType.Q4_0:
dequantize_blocks_Q4_0, GGMLQuantizationType.Q6_K:
dequantize_blocks_Q6_K, GGMLQuantizationType.Q5_K:
dequantize_blocks_Q5_K, GGMLQuantizationType.Q4_K:
dequantize_blocks_Q4_K, GGMLQuantizationType.Q3_K:
dequantize_blocks_Q3_K, GGMLQuantizationType.Q2_K:
dequantize_blocks_Q2_K, GGMLQuantizationType.IQ4_NL:
dequantize_blocks_IQ4_NL}
gguf_connector/quant2b.py
+21
View File
@@ -0,0 +1,21 @@
import torch
def split_block_dims(blocks, *args):
n_max = blocks.shape[1]
dims = list(args) + [n_max - sum(args)]
return torch.split(blocks, dims, dim=1)
def to_uint32(x):
x = x.view(torch.uint8).to(torch.int32)
return (x[:, 0] | x[:, 1] << 8 | x[:, 2] << 16 | x[:, 3] << 24).unsqueeze(1)
def get_scale_min(scales):
n_blocks = scales.shape[0]
scales = scales.view(torch.uint8)
scales = scales.reshape((n_blocks, 3, 4))
d, m, m_d = torch.split(scales, scales.shape[-2] // 3, dim=-2)
sc = torch.cat([d & 63, m_d & 15 | d >> 2 & 48], dim=-1)
min = torch.cat([m & 63, m_d >> 4 | m >> 2 & 48], dim=-1)
return sc.reshape((n_blocks, 8)), min.reshape((n_blocks, 8))
def load_grid_tensor(grid_shape, grid_hex):
grid_bytes = torch.tensor(list(grid_hex))
grid_words = grid_bytes.view(-1, 2).flip(1)
grid = grid_words.contiguous().view(-1).to(torch.int16).view(*grid_shape)
return grid
gguf_connector/quant2c.py
+287
View File
@@ -0,0 +1,287 @@
import torch
from .quant import dequantize as gq
from .reader import GGMLQuantizationType, GGML_QUANT_SIZES
from tqdm import tqdm
TORCH_COMPATIBLE_QTYPES = {None, GGMLQuantizationType.F32,
GGMLQuantizationType.F16}
def is_torch_compatible(tensor):
return tensor is None or getattr(tensor, 'tensor_type', None
) in TORCH_COMPATIBLE_QTYPES
def is_quantized(tensor):
return not is_torch_compatible(tensor)
def dequantize(data, qtype, oshape, dtype=None):
block_size, type_size = GGML_QUANT_SIZES[qtype]
dequantize_blocks = dequantize_functions[qtype]
rows = data.reshape((-1, data.shape[-1])).view(torch.uint8)
n_blocks = rows.numel() // type_size
blocks = rows.reshape((n_blocks, type_size))
blocks = dequantize_blocks(blocks, block_size, type_size, dtype)
return blocks.reshape(oshape)
def dequantize_tensor(tensor, dtype=None, dequant_dtype=None):
qtype = getattr(tensor, 'tensor_type', None)
oshape = getattr(tensor, 'tensor_shape', tensor.shape)
if qtype in TORCH_COMPATIBLE_QTYPES:
return tensor.to(dtype)
elif qtype in dequantize_functions:
dequant_dtype = dtype if dequant_dtype == 'target' else dequant_dtype
return dequantize(tensor.data, qtype, oshape, dtype=dequant_dtype).to(
dtype)
else:
tqdm.write(f'Processing tensor: qtype: {qtype}, {oshape}')
new = gq(tensor.cpu().numpy(), qtype)
return torch.from_numpy(new).to(tensor.device, dtype=dtype)
def dequantize_blocks_BF16(blocks, block_size, type_size, dtype=None):
return (blocks.view(torch.int16).to(torch.int32) << 16).view(torch.float32)
def dequantize_blocks_Q8_0(blocks, block_size, type_size, dtype=None):
d = blocks[:, :2].view(torch.float16).to(dtype)
x = blocks[:, 2:].view(torch.int8)
return x * d
def dequantize_blocks_Q4_0(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d = blocks[:, :2].view(torch.float16).to(dtype)
qs = blocks[:, 2:]
qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0,
4], device=d.device, dtype=torch.uint8).reshape((1, 1, 2, 1))
qs = (qs & 15).reshape((n_blocks, -1)).to(torch.int8) - 8
return d * qs
def dequantize_blocks_Q4_1(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d = blocks[:, :2].view(torch.float16).to(dtype)
m = blocks[:, 2:4].view(torch.float16).to(dtype)
qs = blocks[:, 4:]
qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0,
4], device=d.device, dtype=torch.uint8).reshape(1, 1, 2, 1)
qs = (qs & 15).reshape(n_blocks, -1)
return d * qs + m
from .quant2b import to_uint32
def dequantize_blocks_Q5_0(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d = blocks[:, :2].view(torch.float16).to(dtype)
qh = blocks[:, 2:6]
qs = blocks[:, 6:]
qh = to_uint32(qh)
qh = qh.reshape(n_blocks, 1) >> torch.arange(32, device=d.device, dtype
=torch.int32).reshape(1, 32)
ql = qs.reshape(n_blocks, -1, 1, block_size // 2) >> torch.tensor([0, 4
], device=d.device, dtype=torch.uint8).reshape(1, 1, 2, 1)
qh = (qh & 1).to(torch.uint8)
ql = (ql & 15).reshape(n_blocks, -1)
qs = (ql | qh << 4).to(torch.int8) - 16
return d * qs
def dequantize_blocks_Q5_1(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d = blocks[:, :2].view(torch.float16).to(dtype)
m = blocks[:, 2:4].view(torch.float16).to(dtype)
qh = blocks[:, 4:8]
qs = blocks[:, 8:]
qh = to_uint32(qh)
qh = qh.reshape((n_blocks, 1)) >> torch.arange(32, device=d.device,
dtype=torch.int32).reshape(1, 32)
ql = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0,
4], device=d.device, dtype=torch.uint8).reshape(1, 1, 2, 1)
qh = (qh & 1).to(torch.uint8)
ql = (ql & 15).reshape((n_blocks, -1))
qs = ql | qh << 4
return d * qs + m
QK_K = 256
from .quant2b import split_block_dims
def dequantize_blocks_Q6_K(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
ql, qh, scales, d = split_block_dims(blocks, QK_K // 2, QK_K // 4, QK_K //
16)
scales = scales.view(torch.int8).to(dtype)
d = d.view(torch.float16).to(dtype)
d = (d * scales).reshape((n_blocks, QK_K // 16, 1))
ql = ql.reshape((n_blocks, -1, 1, 64)) >> torch.tensor([0, 4], device=d
.device, dtype=torch.uint8).reshape((1, 1, 2, 1))
ql = (ql & 15).reshape((n_blocks, -1, 32))
qh = qh.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 2, 4, 6],
device=d.device, dtype=torch.uint8).reshape((1, 1, 4, 1))
qh = (qh & 3).reshape((n_blocks, -1, 32))
q = (ql | qh << 4).to(torch.int8) - 32
q = q.reshape((n_blocks, QK_K // 16, -1))
return (d * q).reshape((n_blocks, QK_K))
def dequantize_blocks_Q3_K(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
hmask, qs, scales, d = split_block_dims(blocks, QK_K // 8, QK_K // 4, 12)
d = d.view(torch.float16).to(dtype)
lscales, hscales = scales[:, :8], scales[:, 8:]
lscales = lscales.reshape((n_blocks, 1, 8)) >> torch.tensor([0, 4],
device=d.device, dtype=torch.uint8).reshape((1, 2, 1))
lscales = lscales.reshape((n_blocks, 16))
hscales = hscales.reshape((n_blocks, 1, 4)) >> torch.tensor([0, 2, 4, 6
], device=d.device, dtype=torch.uint8).reshape((1, 4, 1))
hscales = hscales.reshape((n_blocks, 16))
scales = lscales & 15 | (hscales & 3) << 4
scales = scales.to(torch.int8) - 32
dl = (d * scales).reshape((n_blocks, 16, 1))
ql = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 2, 4, 6],
device=d.device, dtype=torch.uint8).reshape((1, 1, 4, 1))
qh = hmask.reshape(n_blocks, -1, 1, 32) >> torch.tensor([i for i in
range(8)], device=d.device, dtype=torch.uint8).reshape((1, 1, 8, 1))
ql = ql.reshape((n_blocks, 16, QK_K // 16)) & 3
qh = qh.reshape((n_blocks, 16, QK_K // 16)) & 1 ^ 1
q = ql.to(torch.int8) - (qh << 2).to(torch.int8)
return (dl * q).reshape((n_blocks, QK_K))
def dequantize_blocks_Q2_K(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
scales, qs, d, dmin = split_block_dims(blocks, QK_K // 16, QK_K // 4, 2)
d = d.view(torch.float16).to(dtype)
dmin = dmin.view(torch.float16).to(dtype)
dl = (d * (scales & 15)).reshape((n_blocks, QK_K // 16, 1))
ml = (dmin * (scales >> 4)).reshape((n_blocks, QK_K // 16, 1))
shift = torch.tensor([0, 2, 4, 6], device=d.device, dtype=torch.uint8
).reshape((1, 1, 4, 1))
qs = qs.reshape((n_blocks, -1, 1, 32)) >> shift & 3
qs = qs.reshape((n_blocks, QK_K // 16, 16))
qs = dl * qs - ml
return qs.reshape((n_blocks, -1))
K_SCALE_SIZE = 12
from .quant2b import get_scale_min
def dequantize_blocks_Q5_K(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d, dmin, scales, qh, qs = split_block_dims(blocks, 2, 2, K_SCALE_SIZE,
QK_K // 8)
d = d.view(torch.float16).to(dtype)
dmin = dmin.view(torch.float16).to(dtype)
sc, m = get_scale_min(scales)
d = (d * sc).reshape((n_blocks, -1, 1))
dm = (dmin * m).reshape((n_blocks, -1, 1))
ql = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 4], device=d
.device, dtype=torch.uint8).reshape((1, 1, 2, 1))
qh = qh.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([i for i in
range(8)], device=d.device, dtype=torch.uint8).reshape((1, 1, 8, 1))
ql = (ql & 15).reshape((n_blocks, -1, 32))
qh = (qh & 1).reshape((n_blocks, -1, 32))
q = ql | qh << 4
return (d * q - dm).reshape((n_blocks, QK_K))
def dequantize_blocks_Q4_K(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d, dmin, scales, qs = split_block_dims(blocks, 2, 2, K_SCALE_SIZE)
d = d.view(torch.float16).to(dtype)
dmin = dmin.view(torch.float16).to(dtype)
sc, m = get_scale_min(scales)
d = (d * sc).reshape((n_blocks, -1, 1))
dm = (dmin * m).reshape((n_blocks, -1, 1))
qs = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 4], device=d
.device, dtype=torch.uint8).reshape((1, 1, 2, 1))
qs = (qs & 15).reshape((n_blocks, -1, 32))
return (d * qs - dm).reshape((n_blocks, QK_K))
def dequantize_blocks_IQ4_NL(blocks, block_size, type_size, dtype=None):
kvalues = torch.tensor([-127, -104, -83, -65, -49, -35, -22, -10, 1, 13,
25, 38, 53, 69, 89, 113], dtype=torch.float32, device=blocks.device)
n_blocks = blocks.shape[0]
d, qs = split_block_dims(blocks, 2)
d = d.view(torch.float16).to(dtype)
qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0,
4], device=blocks.device, dtype=torch.uint8).reshape((1, 1, 2, 1))
qs = (qs & 15).reshape((n_blocks, -1)).to(torch.int64)
kvalues = kvalues.view(1, 1, 16)
qs = qs.unsqueeze(-1)
qs = torch.gather(kvalues.expand(qs.shape[0], qs.shape[1], 16), 2, qs)
qs = qs.squeeze(-1).to(dtype)
return d * qs
def dequantize_blocks_IQ4_XS(blocks, block_size, type_size, dtype=None):
kvalues = torch.tensor([-127, -104, -83, -65, -49, -35, -22, -10, 1, 13,
25, 38, 53, 69, 89, 113], dtype=torch.float32, device=blocks.device)
n_blocks = blocks.shape[0]
d, scales_h, scales_l, qs = split_block_dims(blocks, 2, 2, QK_K // 64)
d = d.view(torch.float16).to(dtype)
scales_h = scales_h.view(torch.int16)
scales_l = scales_l.reshape((n_blocks, -1, 1)) >> torch.tensor(
[0, 4], device=blocks.device, dtype=torch.uint8).reshape((1, 1, 2))
scales_h = scales_h.reshape((n_blocks, 1, -1)) >> torch.tensor(
[2 * i for i in range(QK_K // 32)],
device=blocks.device, dtype=torch.uint8).reshape((1, -1, 1))
scales_l = scales_l.reshape((n_blocks, -1)) & 0x0F
scales_h = scales_h.reshape((n_blocks, -1)) & 0x03
scales = (scales_l | (scales_h << 4)) - 32
dl = (d * scales.to(dtype)).reshape((n_blocks, -1, 1))
shifts_q = torch.tensor(
[0, 4], device=blocks.device, dtype=torch.uint8).reshape(1, 1, 2, 1)
qs = qs.reshape((n_blocks, -1, 1, 16)) >> shifts_q
qs = (qs & 15).reshape((n_blocks, -1, 32)).to(torch.int64)
kvalues = kvalues.view(1, 1, 1, 16)
qs = qs.unsqueeze(-1)
qs = torch.gather(kvalues.expand(qs.shape[0], qs.shape[1], qs.shape[2], 16), 3, qs)
qs = qs.squeeze(-1).to(dtype)
return (dl * qs).reshape(n_blocks, -1)
def dequantize_blocks_TQ2_0(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
qs, d = split_block_dims(blocks, QK_K // 4)
d = d.view(torch.float16).to(dtype)
qs = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 2, 4, 6],
device=d.device, dtype=torch.uint8).reshape((1, 1, 4, 1))
qs = (qs & 3).reshape((n_blocks, -1)) - 1
return d * qs
def dequantize_blocks_TQ1_0(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
qs, qh, d = split_block_dims(blocks, (QK_K - 4 * QK_K // 64) // 5, QK_K //
64)
d = d.view(torch.float16).to(dtype)
qs0, qs1 = qs[..., :32], qs[..., 32:]
qs0 = qs0.reshape((n_blocks, -1, 1, 32)) * torch.tensor([1, 3, 9, 27,
81], device=d.device, dtype=torch.uint8).reshape((1, 1, 5, 1))
qs0 = qs0.reshape((n_blocks, -1))
qs1 = qs1.reshape((n_blocks, -1, 1, 16)) * torch.tensor([1, 3, 9, 27,
81], device=d.device, dtype=torch.uint8).reshape((1, 1, 5, 1))
qs1 = qs1.reshape((n_blocks, -1))
qh = qh.reshape((n_blocks, -1, 1, 4)) * torch.tensor([1, 3, 9, 27],
device=d.device, dtype=torch.uint8).reshape((1, 1, 4, 1))
qh = qh.reshape((n_blocks, -1))
qs = torch.cat([qs0, qs1, qh], dim=-1)
qs = (qs * 3 >> 8) - 1
return d * qs
def dequantize_blocks_IQ3_S(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d, qs, qh, signs, scales = split_block_dims(blocks, 2, QK_K // 4, QK_K // 32, QK_K // 8)
d = d.view(torch.float16).to(dtype)
scales = scales.reshape((n_blocks, -1, 1)) >> torch.tensor(
[0, 4], device=blocks.device, dtype=torch.uint8).reshape((1, 1, 2))
scales = (scales & 15).reshape((n_blocks, -1))
db = d * (1 + 2 * scales)
db = db.reshape((n_blocks, -1, 1, 1))
signs = signs.reshape((n_blocks, -1, 1)) >> torch.tensor(
[i for i in range(8)], device=blocks.device, dtype=torch.uint8).reshape((1, 1, 8))
signs = signs & 1
signs = torch.where(signs == 0, torch.tensor(1.0, device=blocks.device), torch.tensor(-1.0, device=blocks.device))
signs = signs.reshape((n_blocks, -1, 4, 8))
return (db * signs).reshape((n_blocks, -1))
def dequantize_blocks_IQ3_XXS(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d, qs, scales = split_block_dims(blocks, 2, QK_K // 4)
d = d.view(torch.float16).to(dtype)
scales = to_uint32(scales)
db = d * (0.5 + (scales >> 28)) * 0.5
db = db.reshape((n_blocks, -1, 1, 1))
bit_shifts = torch.tensor(
[0, 7, 14, 21], device=blocks.device, dtype=torch.uint8).reshape((1, 1, 4))
signs = scales.reshape((n_blocks, -1, 1)) >> bit_shifts
db = db.reshape((n_blocks, -1, 1, 1))
signs = signs.reshape((n_blocks, -1, 1)) >> torch.tensor(
[i for i in range(8)], device=blocks.device, dtype=torch.uint8).reshape((1, 1, 8))
signs = signs & 1
db = db.reshape((n_blocks, -1, 1, 1))
sign_shifts = torch.arange(8, device=blocks.device, dtype=torch.uint8).view(1, 1, 8)
signs = signs.reshape((n_blocks, -1, 1)) >> sign_shifts
signs = (signs & 1).float()
signs = torch.where(signs == 0, torch.tensor(1.0, device=blocks.device), torch.tensor(-1.0, device=blocks.device))
signs = signs.reshape((n_blocks, -1, 4, 8))
return (db * signs).reshape((n_blocks, -1))
dequantize_functions = {GGMLQuantizationType.BF16: dequantize_blocks_BF16,
GGMLQuantizationType.Q8_0: dequantize_blocks_Q8_0, GGMLQuantizationType
.Q5_1: dequantize_blocks_Q5_1, GGMLQuantizationType.Q5_0:
dequantize_blocks_Q5_0, GGMLQuantizationType.Q4_1:
dequantize_blocks_Q4_1, GGMLQuantizationType.Q4_0:
dequantize_blocks_Q4_0, GGMLQuantizationType.Q6_K:
dequantize_blocks_Q6_K, GGMLQuantizationType.Q5_K:
dequantize_blocks_Q5_K, GGMLQuantizationType.Q4_K:
dequantize_blocks_Q4_K, GGMLQuantizationType.Q3_K:
dequantize_blocks_Q3_K, GGMLQuantizationType.Q2_K:
dequantize_blocks_Q2_K, GGMLQuantizationType.IQ4_NL:
dequantize_blocks_IQ4_NL, GGMLQuantizationType.IQ4_XS:
dequantize_blocks_IQ4_XS, GGMLQuantizationType.IQ3_S:
dequantize_blocks_IQ3_S, GGMLQuantizationType.IQ3_XXS:
dequantize_blocks_IQ3_XXS, GGMLQuantizationType.TQ2_0:
dequantize_blocks_TQ2_0, GGMLQuantizationType.TQ1_0:
dequantize_blocks_TQ1_0}
gguf_connector/quant2d.py
+309
View File
@@ -0,0 +1,309 @@
import torch
from .quant import dequantize as gq
from .reader import GGMLQuantizationType, GGML_QUANT_SIZES
from tqdm import tqdm
TORCH_COMPATIBLE_QTYPES = {None, GGMLQuantizationType.F32,
GGMLQuantizationType.F16}
def is_torch_compatible(tensor):
return tensor is None or getattr(tensor, 'tensor_type', None
) in TORCH_COMPATIBLE_QTYPES
def is_quantized(tensor):
return not is_torch_compatible(tensor)
def dequantize(data, qtype, oshape, dtype=None):
block_size, type_size = GGML_QUANT_SIZES[qtype]
dequantize_blocks = dequantize_functions[qtype]
rows = data.reshape((-1, data.shape[-1])).view(torch.uint8)
n_blocks = rows.numel() // type_size
blocks = rows.reshape((n_blocks, type_size))
blocks = dequantize_blocks(blocks, block_size, type_size, dtype)
return blocks.reshape(oshape)
def dequantize_tensor(tensor, dtype=None, dequant_dtype=None):
qtype = getattr(tensor, 'tensor_type', None)
oshape = getattr(tensor, 'tensor_shape', tensor.shape)
if qtype in TORCH_COMPATIBLE_QTYPES:
return tensor.to(dtype)
elif qtype in dequantize_functions:
dequant_dtype = dtype if dequant_dtype == 'target' else dequant_dtype
return dequantize(tensor.data, qtype, oshape, dtype=dequant_dtype).to(
dtype)
else:
tqdm.write(f'Processing tensor: qtype: {qtype}, {oshape}')
new = gq(tensor.cpu().numpy(), qtype)
return torch.from_numpy(new).to(tensor.device, dtype=dtype)
def dequantize_blocks_BF16(blocks, block_size, type_size, dtype=None):
return (blocks.view(torch.int16).to(torch.int32) << 16).view(torch.float32)
def dequantize_blocks_Q8_0(blocks, block_size, type_size, dtype=None):
d = blocks[:, :2].view(torch.float16).to(dtype)
x = blocks[:, 2:].view(torch.int8)
return x * d
def dequantize_blocks_Q4_0(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d = blocks[:, :2].view(torch.float16).to(dtype)
qs = blocks[:, 2:]
qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0,
4], device=d.device, dtype=torch.uint8).reshape((1, 1, 2, 1))
qs = (qs & 15).reshape((n_blocks, -1)).to(torch.int8) - 8
return d * qs
def dequantize_blocks_Q4_1(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d = blocks[:, :2].view(torch.float16).to(dtype)
m = blocks[:, 2:4].view(torch.float16).to(dtype)
qs = blocks[:, 4:]
qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0,
4], device=d.device, dtype=torch.uint8).reshape(1, 1, 2, 1)
qs = (qs & 15).reshape(n_blocks, -1)
return d * qs + m
from .quant2b import to_uint32
def dequantize_blocks_Q5_0(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d = blocks[:, :2].view(torch.float16).to(dtype)
qh = blocks[:, 2:6]
qs = blocks[:, 6:]
qh = to_uint32(qh)
qh = qh.reshape(n_blocks, 1) >> torch.arange(32, device=d.device, dtype
=torch.int32).reshape(1, 32)
ql = qs.reshape(n_blocks, -1, 1, block_size // 2) >> torch.tensor([0, 4
], device=d.device, dtype=torch.uint8).reshape(1, 1, 2, 1)
qh = (qh & 1).to(torch.uint8)
ql = (ql & 15).reshape(n_blocks, -1)
qs = (ql | qh << 4).to(torch.int8) - 16
return d * qs
def dequantize_blocks_Q5_1(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d = blocks[:, :2].view(torch.float16).to(dtype)
m = blocks[:, 2:4].view(torch.float16).to(dtype)
qh = blocks[:, 4:8]
qs = blocks[:, 8:]
qh = to_uint32(qh)
qh = qh.reshape((n_blocks, 1)) >> torch.arange(32, device=d.device,
dtype=torch.int32).reshape(1, 32)
ql = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0,
4], device=d.device, dtype=torch.uint8).reshape(1, 1, 2, 1)
qh = (qh & 1).to(torch.uint8)
ql = (ql & 15).reshape((n_blocks, -1))
qs = ql | qh << 4
return d * qs + m
QK_K = 256
from .quant2b import split_block_dims
def dequantize_blocks_Q6_K(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
ql, qh, scales, d = split_block_dims(blocks, QK_K // 2, QK_K // 4, QK_K //
16)
scales = scales.view(torch.int8).to(dtype)
d = d.view(torch.float16).to(dtype)
d = (d * scales).reshape((n_blocks, QK_K // 16, 1))
ql = ql.reshape((n_blocks, -1, 1, 64)) >> torch.tensor([0, 4], device=d
.device, dtype=torch.uint8).reshape((1, 1, 2, 1))
ql = (ql & 15).reshape((n_blocks, -1, 32))
qh = qh.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 2, 4, 6],
device=d.device, dtype=torch.uint8).reshape((1, 1, 4, 1))
qh = (qh & 3).reshape((n_blocks, -1, 32))
q = (ql | qh << 4).to(torch.int8) - 32
q = q.reshape((n_blocks, QK_K // 16, -1))
return (d * q).reshape((n_blocks, QK_K))
def dequantize_blocks_Q3_K(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
hmask, qs, scales, d = split_block_dims(blocks, QK_K // 8, QK_K // 4, 12)
d = d.view(torch.float16).to(dtype)
lscales, hscales = scales[:, :8], scales[:, 8:]
lscales = lscales.reshape((n_blocks, 1, 8)) >> torch.tensor([0, 4],
device=d.device, dtype=torch.uint8).reshape((1, 2, 1))
lscales = lscales.reshape((n_blocks, 16))
hscales = hscales.reshape((n_blocks, 1, 4)) >> torch.tensor([0, 2, 4, 6
], device=d.device, dtype=torch.uint8).reshape((1, 4, 1))
hscales = hscales.reshape((n_blocks, 16))
scales = lscales & 15 | (hscales & 3) << 4
scales = scales.to(torch.int8) - 32
dl = (d * scales).reshape((n_blocks, 16, 1))
ql = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 2, 4, 6],
device=d.device, dtype=torch.uint8).reshape((1, 1, 4, 1))
qh = hmask.reshape(n_blocks, -1, 1, 32) >> torch.tensor([i for i in
range(8)], device=d.device, dtype=torch.uint8).reshape((1, 1, 8, 1))
ql = ql.reshape((n_blocks, 16, QK_K // 16)) & 3
qh = qh.reshape((n_blocks, 16, QK_K // 16)) & 1 ^ 1
q = ql.to(torch.int8) - (qh << 2).to(torch.int8)
return (dl * q).reshape((n_blocks, QK_K))
def dequantize_blocks_Q2_K(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
scales, qs, d, dmin = split_block_dims(blocks, QK_K // 16, QK_K // 4, 2)
d = d.view(torch.float16).to(dtype)
dmin = dmin.view(torch.float16).to(dtype)
dl = (d * (scales & 15)).reshape((n_blocks, QK_K // 16, 1))
ml = (dmin * (scales >> 4)).reshape((n_blocks, QK_K // 16, 1))
shift = torch.tensor([0, 2, 4, 6], device=d.device, dtype=torch.uint8
).reshape((1, 1, 4, 1))
qs = qs.reshape((n_blocks, -1, 1, 32)) >> shift & 3
qs = qs.reshape((n_blocks, QK_K // 16, 16))
qs = dl * qs - ml
return qs.reshape((n_blocks, -1))
K_SCALE_SIZE = 12
from .quant2b import get_scale_min
def dequantize_blocks_Q5_K(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d, dmin, scales, qh, qs = split_block_dims(blocks, 2, 2, K_SCALE_SIZE,
QK_K // 8)
d = d.view(torch.float16).to(dtype)
dmin = dmin.view(torch.float16).to(dtype)
sc, m = get_scale_min(scales)
d = (d * sc).reshape((n_blocks, -1, 1))
dm = (dmin * m).reshape((n_blocks, -1, 1))
ql = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 4], device=d
.device, dtype=torch.uint8).reshape((1, 1, 2, 1))
qh = qh.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([i for i in
range(8)], device=d.device, dtype=torch.uint8).reshape((1, 1, 8, 1))
ql = (ql & 15).reshape((n_blocks, -1, 32))
qh = (qh & 1).reshape((n_blocks, -1, 32))
q = ql | qh << 4
return (d * q - dm).reshape((n_blocks, QK_K))
def dequantize_blocks_Q4_K(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
d, dmin, scales, qs = split_block_dims(blocks, 2, 2, K_SCALE_SIZE)
d = d.view(torch.float16).to(dtype)
dmin = dmin.view(torch.float16).to(dtype)
sc, m = get_scale_min(scales)
d = (d * sc).reshape((n_blocks, -1, 1))
dm = (dmin * m).reshape((n_blocks, -1, 1))
qs = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 4], device=d
.device, dtype=torch.uint8).reshape((1, 1, 2, 1))
qs = (qs & 15).reshape((n_blocks, -1, 32))
return (d * qs - dm).reshape((n_blocks, QK_K))
def dequantize_blocks_TQ2_0(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
qs, d = split_block_dims(blocks, QK_K // 4)
d = d.view(torch.float16).to(dtype)
qs = qs.reshape((n_blocks, -1, 1, 32)) >> torch.tensor([0, 2, 4, 6],
device=d.device, dtype=torch.uint8).reshape((1, 1, 4, 1))
qs = (qs & 3).reshape((n_blocks, -1)) - 1
return d * qs
def dequantize_blocks_TQ1_0(blocks, block_size, type_size, dtype=None):
n_blocks = blocks.shape[0]
qs, qh, d = split_block_dims(blocks, (QK_K - 4 * QK_K // 64) // 5, QK_K //
64)
d = d.view(torch.float16).to(dtype)
qs0, qs1 = qs[..., :32], qs[..., 32:]
qs0 = qs0.reshape((n_blocks, -1, 1, 32)) * torch.tensor([1, 3, 9, 27,
81], device=d.device, dtype=torch.uint8).reshape((1, 1, 5, 1))
qs0 = qs0.reshape((n_blocks, -1))
qs1 = qs1.reshape((n_blocks, -1, 1, 16)) * torch.tensor([1, 3, 9, 27,
81], device=d.device, dtype=torch.uint8).reshape((1, 1, 5, 1))
qs1 = qs1.reshape((n_blocks, -1))
qh = qh.reshape((n_blocks, -1, 1, 4)) * torch.tensor([1, 3, 9, 27],
device=d.device, dtype=torch.uint8).reshape((1, 1, 4, 1))
qh = qh.reshape((n_blocks, -1))
qs = torch.cat([qs0, qs1, qh], dim=-1)
qs = (qs * 3 >> 8) - 1
return d * qs
def dequantize_blocks_IQ4_NL(blocks, block_size, type_size, dtype=None):
kvalues = torch.tensor([-127, -104, -83, -65, -49, -35, -22, -10, 1, 13,
25, 38, 53, 69, 89, 113], dtype=torch.float32, device=blocks.device)
n_blocks = blocks.shape[0]
d, qs = split_block_dims(blocks, 2)
d = d.view(torch.float16).to(dtype)
qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> torch.tensor([0,
4], device=blocks.device, dtype=torch.uint8).reshape((1, 1, 2, 1))
qs = (qs & 15).reshape((n_blocks, -1)).to(torch.int64)
kvalues = kvalues.view(1, 1, 16)
qs = qs.unsqueeze(-1)
qs = torch.gather(kvalues.expand(qs.shape[0], qs.shape[1], 16), 2, qs)
qs = qs.squeeze(-1).to(dtype)
return d * qs
def dequantize_blocks_IQ4_XS(blocks, block_size, type_size, dtype=None):
kvalues = torch.tensor([-127, -104, -83, -65, -49, -35, -22, -10, 1, 13,
25, 38, 53, 69, 89, 113], dtype=torch.float32, device=blocks.device)
n_blocks = blocks.shape[0]
d, scales_h, scales_l, qs = split_block_dims(blocks, 2, 2, QK_K // 64)
d = d.view(torch.float16).to(dtype)
scales_h = scales_h.view(torch.int16)
scales_l = scales_l.reshape((n_blocks, -1, 1)) >> torch.tensor([0, 4],
device=blocks.device, dtype=torch.uint8).reshape((1, 1, 2))
scales_h = scales_h.reshape((n_blocks, 1, -1)) >> torch.tensor([(2 * i) for
i in range(QK_K // 32)], device=blocks.device, dtype=torch.uint8
).reshape((1, -1, 1))
scales_l = scales_l.reshape((n_blocks, -1)) & 15
scales_h = scales_h.reshape((n_blocks, -1)) & 3
scales = (scales_l | scales_h << 4) - 32
dl = (d * scales.to(dtype)).reshape((n_blocks, -1, 1))
shifts_q = torch.tensor([0, 4], device=blocks.device, dtype=torch.uint8
).reshape(1, 1, 2, 1)
qs = qs.reshape((n_blocks, -1, 1, 16)) >> shifts_q
qs = (qs & 15).reshape((n_blocks, -1, 32)).to(torch.int64)
kvalues = kvalues.view(1, 1, 1, 16)
qs = qs.unsqueeze(-1)
qs = torch.gather(kvalues.expand(qs.shape[0], qs.shape[1], qs.shape[2],
16), 3, qs)
qs = qs.squeeze(-1).to(dtype)
return (dl * qs).reshape(n_blocks, -1)
from .quant2b import load_grid_tensor
def dequantize_blocks_IQ3_S(blocks, block_size, type_size, dtype=None):
grid_shape = 512, 4
grid_hex = (
b'00000100020005000700100011001200140016002000210025003300400042004500470051005300600062007100740077000001010102010401100111011501200123012701310135014401610165017201000201020502070210021302160221022502300234024202450247025102530270027302030311031503200322033103330336034403500352036703710375030004130417042104240432044004430451047004020504052005220526053305410545054705660573050606110613063106520671060007020704072007220726073307500754070010011002100410101011101310151017102010221031103410361054105610611072100011011103110611101114112111301133114111501152117011761100121212151217122012241232124012431255126012721201130413071310131313211327133013341341136213701303140514121414143114331442144614501454140115101513152115301532155115201624162716441646160117031710171217211735174117621770170020012003200520072010201220142016202120232027203020322041204320452050205220672070207320752000210221102113211721222125213121342142215121012204220722212223223022372241225322572271227422002302230523112322232423312333234223502366230124072420242324322435244124722475240425112522253725402553257025002602260726212655266126052711272627302743275027023011301330153017302230313033303530423044304730513063307130013103310531143121312331403160317231763100321232203232323432503201331033143321332333273330334133433347335533733303341134163422343134523460346434013510351235253532354435563573351636413601370337203722373537004004401240204024402740324041405040704002410741114113412241304135414341514155410142034210421542214233424042574262427042044311431343204322433143354300440244244437444044714405450745214562451346344660461047154730474347514702501050145022504050445047505250665074500151035105511251215132517251005211522352305236525352025307531053275344535153655373530154045420543254465412552655515553554256025704572257116013601560316033606060006120612761646112623462426255626262706200631463216340632564436462640065036534656065056640661167136700700470077020702270367040705470627002711171247143714571017204721072167221723072517202733273357353730174057413742074507422754275027631760077'
)
n_blocks = blocks.shape[0]
d, qs, qh, signs, scales = split_block_dims(blocks, 2, QK_K // 4, QK_K //
32, QK_K // 8)
d = d.view(torch.float16).to(dtype)
scales = scales.reshape((n_blocks, -1, 1)) >> torch.tensor([0, 4],
device=blocks.device, dtype=torch.uint8).reshape((1, 1, 2))
scales = (scales & 15).reshape((n_blocks, -1))
db = d * (1 + 2 * scales.to(dtype))
db = db.reshape((n_blocks, -1, 1, 1))
signs = signs.reshape((n_blocks, -1, 1)) >> torch.arange(8, device=
blocks.device, dtype=torch.uint8).reshape((1, 1, 8))
signs = signs & 1
signs = signs.reshape((n_blocks, -1, 4, 8))
qh_shifts = torch.arange(8, device=blocks.device, dtype=torch.uint8).view(
1, 1, 8)
qh = qh.reshape((n_blocks, -1, 1)) >> qh_shifts & 1
qh = qh.reshape((n_blocks, -1)).to(torch.int16)
qs = qs.to(torch.int64) | qh << 8
grid = load_grid_tensor(grid_shape, grid_hex)
grid = grid.unsqueeze(0).expand(n_blocks, -1, -1)
return (db * signs).reshape(n_blocks, -1)
def dequantize_blocks_IQ3_XXS(blocks, block_size, type_size, dtype=None):
grid_shape = 256, 4
grid_hex = (
b'0000020004001100130017002000220031004200730075000101030110011201210125013001320141015401700100020202040211022002220231023302370251025702750201030703100312032503700313043704440457047304750401050705320552053506640610071407160743076107011003101010121021102310301032103410471050100011021111112011221101120312101212122112301272120013021320133113461366130114051450142015241546157115051622174017002002201120132020202220262031204220012103210521102112212121302163216721702100220222112217222022222237224022552201231023142370237423352453240325272541257425012703271627452701301030123021302330503065307230003102312031313144314631013203321032253252327232113333333034473472340035063522355535143636366336333760370440174035403740534057407441204237424042604266420743454304445144644425454345704505471047124730471250415070500051065126515551145232527252025353531054235427547254025531555056245742572460446046606460216161611762646230633663446405655265336603672167037005700770107032705270267140711272457252720073157333736073217441740075027524753076'
)
n_blocks = blocks.shape[0]
d, qs, scales = split_block_dims(blocks, 2, QK_K // 4)
d = d.view(torch.float16).to(dtype)
scales = to_uint32(scales)
db = d * (0.5 + (scales >> 28)) * 0.5
db = db.reshape((n_blocks, -1, 1, 1))
bit_shifts = torch.tensor([0, 7, 14, 21], device=blocks.device, dtype=
torch.uint8).reshape((1, 1, 4))
signs = scales.reshape((n_blocks, -1, 1)) >> bit_shifts
db = db.reshape((n_blocks, -1, 1, 1))
signs = signs.reshape((n_blocks, -1, 1)) >> torch.tensor([i for i in
range(8)], device=blocks.device, dtype=torch.uint8).reshape((1, 1, 8))
signs = signs & 1
db = db.reshape((n_blocks, -1, 1, 1))
sign_shifts = torch.arange(8, device=blocks.device, dtype=torch.uint8
).view(1, 1, 8)
signs = signs.reshape((n_blocks, -1, 1)) >> sign_shifts
signs = (signs & 1).float()
signs = torch.where(signs == 0, torch.tensor(1.0, device=blocks.device),
torch.tensor(-1.0, device=blocks.device))
signs = signs.reshape(n_blocks, -1, 4, 8)
qs = qs.reshape(n_blocks, -1, 1, 1)
grid = load_grid_tensor(grid_shape, grid_hex)
grid = grid.unsqueeze(0).expand(n_blocks, -1, -1)
return (db * signs).reshape(n_blocks, -1)
dequantize_functions = {GGMLQuantizationType.BF16: dequantize_blocks_BF16,
GGMLQuantizationType.Q8_0: dequantize_blocks_Q8_0, GGMLQuantizationType
.Q5_1: dequantize_blocks_Q5_1, GGMLQuantizationType.Q5_0:
dequantize_blocks_Q5_0, GGMLQuantizationType.Q4_1:
dequantize_blocks_Q4_1, GGMLQuantizationType.Q4_0:
dequantize_blocks_Q4_0, GGMLQuantizationType.Q6_K:
dequantize_blocks_Q6_K, GGMLQuantizationType.Q5_K:
dequantize_blocks_Q5_K, GGMLQuantizationType.Q4_K:
dequantize_blocks_Q4_K, GGMLQuantizationType.Q3_K:
dequantize_blocks_Q3_K, GGMLQuantizationType.Q2_K:
dequantize_blocks_Q2_K, GGMLQuantizationType.TQ2_0:
dequantize_blocks_TQ2_0, GGMLQuantizationType.TQ1_0:
dequantize_blocks_TQ1_0, GGMLQuantizationType.IQ4_NL:
dequantize_blocks_IQ4_NL, GGMLQuantizationType.IQ4_XS:
dequantize_blocks_IQ4_XS, GGMLQuantizationType.IQ3_S:
dequantize_blocks_IQ3_S, GGMLQuantizationType.IQ3_XXS:
dequantize_blocks_IQ3_XXS}
gguf_connector/r.py
+43
View File
@@ -0,0 +1,43 @@
def read_gguf_file(gguf_file_path):
from gguf_connector.reader import GGUFReader
reader = GGUFReader(gguf_file_path)
print('-' * 105)
print('Key-Value Pairs:')
max_key_length = max(len(key) for key in reader.fields.keys())
for key, field in reader.fields.items():
value = field.parts[field.data[0]]
print(f'{key:{max_key_length}} : {value}')
print('-' * 105)
print('Tensors:')
tensor_info_format = (
'{:<30} | Shape: {:<15} | Size: {:<12} | Quantization: {}')
print(tensor_info_format.format('Tensor Name', 'Shape', 'Size',
'Quantization'))
print('-' * 105)
for tensor in reader.tensors:
shape_str = 'x'.join(map(str, tensor.shape))
size_str = str(tensor.n_elements)
quantization_str = tensor.tensor_type.name
print(tensor_info_format.format(tensor.name, shape_str, size_str,
quantization_str))
import os
gguf_files = [file for file in os.listdir() if file.endswith('.gguf')]
if gguf_files:
print('GGUF file(s) available. Select which one to read:')
for index, file_name in enumerate(gguf_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(gguf_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = gguf_files[choice_index]
print(f'Model file: {selected_file} is selected!')
ModelPath = selected_file
from rich.progress import Progress
with Progress(transient=True) as progress:
task = progress.add_task('Processing', total=None)
read_gguf_file(ModelPath)
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No GGUF files are available in the current directory.')
input('--- Press ENTER To Exit ---')
gguf_connector/r2.py
+27
View File
@@ -0,0 +1,27 @@
def read_gguf_file(gguf_file_path):
from gguf_connector.reader2 import GGUFReader, ReaderError
reader = GGUFReader(gguf_file_path)
try:
reader.read()
except ReaderError as e:
print(f'Error: {e}')
else:
reader.print()
import os
gguf_files = [file for file in os.listdir() if file.endswith('.gguf')]
if gguf_files:
print('GGUF file(s) available. Select which one to read:')
for index, file_name in enumerate(gguf_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(gguf_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = gguf_files[choice_index]
print(f'Model file: {selected_file} is selected!')
ModelPath = selected_file
read_gguf_file(ModelPath)
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No GGUF files are available in the current directory.')
input('--- Press ENTER To Exit ---')
gguf_connector/r3.py
+30
View File
@@ -0,0 +1,30 @@
from safetensors.torch import load_file
def tensor_reader(file_path):
try:
tensors = load_file(file_path)
tensor_info = {name: tensor.shape for name, tensor in tensors.items()}
return tensor_info
except Exception as e:
return f'Error reading tensor: {e}', {}
import os
files = [file for file in os.listdir() if file.endswith('.safetensors')]
if files:
print('Available .safetensors files:')
for idx, file in enumerate(files, 1):
print(f'{idx}. {file}')
choice = input('Enter the number of the file to read tensor info from: ')
try:
choice = int(choice)
if 1 <= choice <= len(files):
file_path = files[choice - 1]
tensor_info = tensor_reader(file_path)
print('Tensors:')
for name, shape in tensor_info.items():
print(f' {name}: {shape}')
else:
print('Invalid selection.')
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No safetensors files are available in the current directory.')
input('--- Press ENTER To Exit ---')
gguf_connector/r4.py
+22
View File
@@ -0,0 +1,22 @@
from safetensors import safe_open
import os
def list_safetensors_files():
files = [f for f in os.listdir() if f.endswith('.safetensors')]
if not files:
print('No .safetensors files found in the current directory.')
return None
print('Select a .safetensors file to split:')
for idx, file in enumerate(files, 1):
print(f'{idx}: {file}')
while True:
try:
choice = int(input('Enter number: '))
if 1 <= choice <= len(files):
return files[choice - 1]
except ValueError:
pass
print('Invalid selection. Try again.')
selected_file = list_safetensors_files()
if selected_file:
with safe_open(selected_file, framework='pt') as f:
print(f.metadata())
gguf_connector/reader.py
+256
View File
@@ -0,0 +1,256 @@
from __future__ import annotations
import logging, os, sys
from collections import OrderedDict
from typing import Any, Literal, NamedTuple, TypeVar, Union
import numpy as np
import numpy.typing as npt
import io
from .quant import quant_shape_to_byte_shape
if __name__ == '__main__':
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent))
from .const import GGML_QUANT_SIZES, GGUF_DEFAULT_ALIGNMENT, GGUF_MAGIC, GGUF_VERSION, GGMLQuantizationType, GGUFValueType, GGUFEndian
logger = logging.getLogger(__name__)
READER_SUPPORTED_VERSIONS = [2, GGUF_VERSION]
class ReaderField(NamedTuple):
offset: int
name: str
parts: list[npt.NDArray[Any]] = []
data: list[int] = [-1]
types: list[GGUFValueType] = []
def contents(self, index_or_slice=slice(None)):
if self.types:
to_string = lambda x: str(x.tobytes(), encoding='utf-8')
main_type = self.types[0]
if main_type == GGUFValueType.ARRAY:
sub_type = self.types[-1]
if sub_type == GGUFValueType.STRING:
indices = self.data[index_or_slice]
if isinstance(index_or_slice, int):
return to_string(self.parts[indices])
else:
return [to_string(self.parts[idx]) for idx in indices]
elif isinstance(index_or_slice, int):
return self.parts[self.data[index_or_slice]].tolist()[0]
else:
return [pv for idx in self.data[index_or_slice] for pv in
self.parts[idx].tolist()]
if main_type == GGUFValueType.STRING:
return to_string(self.parts[-1])
else:
return self.parts[-1].tolist()[0]
return None
class ReaderTensor(NamedTuple):
name: str
tensor_type: GGMLQuantizationType
shape: npt.NDArray[np.uint32]
n_elements: int
n_bytes: int
data_offset: int
data: npt.NDArray[Any]
field: ReaderField
class GGUFReader:
byte_order: Literal['I', 'S'] = 'I'
alignment: int = GGUF_DEFAULT_ALIGNMENT
data_offset: int
gguf_scalar_to_np: dict[GGUFValueType, type[np.generic]] = {GGUFValueType
.UINT8: np.uint8, GGUFValueType.INT8: np.int8, GGUFValueType.UINT16:
np.uint16, GGUFValueType.INT16: np.int16, GGUFValueType.UINT32: np.
uint32, GGUFValueType.INT32: np.int32, GGUFValueType.FLOAT32: np.
float32, GGUFValueType.UINT64: np.uint64, GGUFValueType.INT64: np.
int64, GGUFValueType.FLOAT64: np.float64, GGUFValueType.BOOL: np.bool_}
def __init__(self, path, mode='r'):
if isinstance(path, io.BytesIO):
path.seek(0)
# print("bytes length: {}".format(len(path)))
self.data = np.frombuffer(path.read(), dtype=np.byte)
else:
self.data = np.memmap(path, mode=mode)
offs = 0
if self._get(offs, np.uint32, override_order='<')[0] != GGUF_MAGIC:
raise ValueError('GGUF magic invalid')
offs += 4
temp_version = self._get(offs, np.uint32)
if temp_version[0] & 65535 == 0:
self.byte_order = 'S'
temp_version = temp_version.newbyteorder(self.byte_order)
version = temp_version[0]
if version not in READER_SUPPORTED_VERSIONS:
raise ValueError(
f'Sorry, file appears to be version {version} which we cannot handle'
)
if sys.byteorder == 'little':
host_endian = GGUFEndian.LITTLE
swapped_endian = GGUFEndian.BIG
else:
host_endian = GGUFEndian.BIG
swapped_endian = GGUFEndian.LITTLE
self.endianess = (swapped_endian if self.byte_order == 'S' else
host_endian)
self.fields: OrderedDict[str, ReaderField] = OrderedDict()
self.tensors: list[ReaderTensor] = []
offs += self._push_field(ReaderField(offs, 'GGUF.version', [
temp_version], [0], [GGUFValueType.UINT32]))
temp_counts = self._get(offs, np.uint64, 2)
offs += self._push_field(ReaderField(offs, 'GGUF.tensor_count', [
temp_counts[:1]], [0], [GGUFValueType.UINT64]))
offs += self._push_field(ReaderField(offs, 'GGUF.kv_count', [
temp_counts[1:]], [0], [GGUFValueType.UINT64]))
tensor_count, kv_count = temp_counts
offs = self._build_fields(offs, kv_count)
offs, tensors_fields = self._build_tensor_info(offs, tensor_count)
new_align = self.fields.get('general.alignment')
if new_align is not None:
if new_align.types != [GGUFValueType.UINT32]:
raise ValueError('Bad type for general.alignment field')
self.alignment = new_align.parts[-1][0]
padding = offs % self.alignment
if padding != 0:
offs += self.alignment - padding
self.data_offset = offs
self._build_tensors(offs, tensors_fields)
_DT = TypeVar('_DT', bound=npt.DTypeLike)
def get_field(self, key):
return self.fields.get(key, None)
def get_tensor(self, idx):
return self.tensors[idx]
def _get(self, offset, dtype, count=1, override_order=None):
count = int(count)
itemsize = int(np.empty([], dtype=dtype).itemsize)
end_offs = offset + itemsize * count
return self.data[offset:end_offs].view(dtype=dtype)[:count]
def _push_field(self, field, skip_sum=False):
if field.name in self.fields:
logger.warning(
f'Duplicate key {field.name} at offset {field.offset}')
self.fields[field.name + '_{}'.format(field.offset)] = field
else:
self.fields[field.name] = field
return 0 if skip_sum else sum(int(part.nbytes) for part in field.parts)
def _get_str(self, offset):
slen = self._get(offset, np.uint64)
return slen, self._get(offset + 8, np.uint8, slen[0])
def _get_field_parts(self, orig_offs, raw_type):
offs = orig_offs
types: list[GGUFValueType] = []
gtype = GGUFValueType(raw_type)
types.append(gtype)
if gtype == GGUFValueType.STRING:
sparts: list[npt.NDArray[Any]] = list(self._get_str(offs))
size = sum(int(part.nbytes) for part in sparts)
return size, sparts, [1], types
nptype = self.gguf_scalar_to_np.get(gtype)
if nptype is not None:
val = self._get(offs, nptype)
return int(val.nbytes), [val], [0], types
if gtype == GGUFValueType.ARRAY:
raw_itype = self._get(offs, np.uint32)
offs += int(raw_itype.nbytes)
alen = self._get(offs, np.uint64)
offs += int(alen.nbytes)
aparts: list[npt.NDArray[Any]] = [raw_itype, alen]
data_idxs: list[int] = []
for idx in range(alen[0]):
curr_size, curr_parts, curr_idxs, curr_types = (self.
_get_field_parts(offs, raw_itype[0]))
if idx == 0:
types += curr_types
idxs_offs = len(aparts)
aparts += curr_parts
data_idxs += (idx + idxs_offs for idx in curr_idxs)
offs += curr_size
return offs - orig_offs, aparts, data_idxs, types
raise ValueError('Unknown/unhandled field type {gtype}')
def _get_tensor_info_field(self, orig_offs):
offs = orig_offs
name_len, name_data = self._get_str(offs)
offs += int(name_len.nbytes + name_data.nbytes)
n_dims = self._get(offs, np.uint32)
offs += int(n_dims.nbytes)
dims = self._get(offs, np.uint64, n_dims[0])
offs += int(dims.nbytes)
raw_dtype = self._get(offs, np.uint32)
offs += int(raw_dtype.nbytes)
offset_tensor = self._get(offs, np.uint64)
offs += int(offset_tensor.nbytes)
return ReaderField(orig_offs, str(bytes(name_data), encoding=
'utf-8'), [name_len, name_data, n_dims, dims, raw_dtype,
offset_tensor], [1, 3, 4, 5])
def _build_fields(self, offs, count):
for _ in range(count):
orig_offs = offs
kv_klen, kv_kdata = self._get_str(offs)
offs += int(kv_klen.nbytes + kv_kdata.nbytes)
raw_kv_type = self._get(offs, np.uint32)
offs += int(raw_kv_type.nbytes)
parts: list[npt.NDArray[Any]] = [kv_klen, kv_kdata, raw_kv_type]
idxs_offs = len(parts)
field_size, field_parts, field_idxs, field_types = (self.
_get_field_parts(offs, raw_kv_type[0]))
parts += field_parts
self._push_field(ReaderField(orig_offs, str(bytes(kv_kdata),
encoding='utf-8'), parts, [(idx + idxs_offs) for idx in
field_idxs], field_types), skip_sum=True)
offs += field_size
return offs
def _build_tensor_info(self, offs, count):
tensor_fields = []
for _ in range(count):
field = self._get_tensor_info_field(offs)
offs += sum(int(part.nbytes) for part in field.parts)
tensor_fields.append(field)
return offs, tensor_fields
def _build_tensors(self, start_offs, fields):
tensors = []
tensor_names = set()
for field in fields:
(_name_len, name_data, _n_dims, dims, raw_dtype, offset_tensor
) = field.parts
tensor_name = str(bytes(name_data), encoding='utf-8')
if tensor_name in tensor_names:
raise ValueError(
f'Found duplicated tensor with name {tensor_name}')
tensor_names.add(tensor_name)
ggml_type = GGMLQuantizationType(raw_dtype[0])
n_elems = int(np.prod(dims))
np_dims = tuple(reversed(dims.tolist()))
block_size, type_size = GGML_QUANT_SIZES[ggml_type]
n_bytes = n_elems * type_size // block_size
data_offs = int(start_offs + offset_tensor[0])
item_type: npt.DTypeLike
if ggml_type == GGMLQuantizationType.F16:
item_count = n_elems
item_type = np.float16
elif ggml_type == GGMLQuantizationType.F32:
item_count = n_elems
item_type = np.float32
elif ggml_type == GGMLQuantizationType.F64:
item_count = n_elems
item_type = np.float64
elif ggml_type == GGMLQuantizationType.I8:
item_count = n_elems
item_type = np.int8
elif ggml_type == GGMLQuantizationType.I16:
item_count = n_elems
item_type = np.int16
elif ggml_type == GGMLQuantizationType.I32:
item_count = n_elems
item_type = np.int32
elif ggml_type == GGMLQuantizationType.I64:
item_count = n_elems
item_type = np.int64
else:
item_count = n_bytes
item_type = np.uint8
np_dims = quant_shape_to_byte_shape(np_dims, ggml_type)
tensors.append(ReaderTensor(name=tensor_name, tensor_type=
ggml_type, shape=dims, n_elements=n_elems, n_bytes=n_bytes,
data_offset=data_offs, data=self._get(data_offs, item_type,
item_count).reshape(np_dims), field=field))
self.tensors = tensors
gguf_connector/reader2.py
+107
View File
@@ -0,0 +1,107 @@
import struct
class ReaderError(Exception):
"""Exception raised for errors in GGUF reader"""
class GGUFReader:
GGUF_FORMAT = b'GGUF'
VALUE_FORMATS = {(0): 'B', (1): 'b', (2): 'H', (3): 'h', (4): 'I', (5):
'i', (6): 'f', (7): '?', (10): 'Q', (11): 'q', (12): 'd'}
TENSOR_TYPES = {(0): 'GGML_TYPE_F32', (1): 'GGML_TYPE_F16', (2):
'GGML_TYPE_Q4_0', (3): 'GGML_TYPE_Q4_1', (4):
'GGML_TYPE_Q4_1_SOME_F16', (5): 'GGML_TYPE_TYPE_Q4_2', (6):
'GGML_TYPE_Q5_0', (7): 'GGML_TYPE_Q5_1', (8): 'GGML_TYPE_Q8_0', (9):
'GGML_TYPE_Q8_1', (10): 'GGML_TYPE_Q2_K', (11): 'GGML_TYPE_Q3_K', (
12): 'GGML_TYPE_Q4_K', (13): 'GGML_TYPE_Q5_K', (14):
'GGML_TYPE_Q6_K', (15): 'GGML_TYPE_Q8_K', (16): 'GGML_TYPE_IQ2_XXS',
(17): 'GGML_TYPE_IQ2_XS', (18): 'GGML_TYPE_IQ3_XXS', (19):
'GGML_TYPE_IQ1_S', (20): 'GGML_TYPE_IQ4_NL', (21):
'GGML_TYPE_IQ3_S', (22): 'GGML_TYPE_IQ2_S', (23):
'GGML_TYPE_IQ4_XS', (24): 'GGML_TYPE_I8', (25): 'GGML_TYPE_I16', (
26): 'GGML_TYPE_I32', (27): 'GGML_TYPE_I64', (28): 'GGML_TYPE_F64',
(29): 'GGML_TYPE_IQ1_M', (30): 'GGML_TYPE_BF16', (32):
'GGML_TYPE_BF16', (34): 'GGML_TYPE__TQ1_0', (35): 'GGML_TYPE__TQ2_0'}
def __init__(self, file_path):
"""Initialize the GGUF reader"""
self.file_path = file_path
self.version = None
self.format = None
self.tensors_info = None
self.metadata = None
self.alignment = None
def read(self):
"""Read the GGUF file."""
with open(self.file_path, 'rb') as f:
self.format = f.read(4)
if self.format != self.GGUF_FORMAT:
raise ReaderError('Invalid format')
self.version = struct.unpack('I', f.read(4))[0]
tensor_count = struct.unpack('Q', f.read(8))[0]
metadata_kv_count = struct.unpack('Q', f.read(8))[0]
self.metadata = {}
for _ in range(metadata_kv_count):
key, value = self._read_metadata_kv(f)
self.metadata[key] = value
self.alignment = self.metadata.get('general.alignment', 1)
self.tensors_info = []
for _ in range(tensor_count):
tensor_info = self._read_tensor_info(f)
self.tensors_info.append(tensor_info)
def _read_string(self, f):
"""Read a string from the file"""
length = struct.unpack('Q', f.read(8))[0]
return f.read(length).decode('utf-8')
def _read_metadata_kv(self, f):
"""Read a metadata key-value pair from the file"""
key = self._read_string(f)
value_type = struct.unpack('I', f.read(4))[0]
value = self._read_value(f, value_type)
return key, value
def _read_value(self, f, value_type):
"""Read a value of the given type from the file"""
if value_type in self.VALUE_FORMATS:
return struct.unpack(self.VALUE_FORMATS[value_type], f.read(
struct.calcsize(self.VALUE_FORMATS[value_type])))[0]
if value_type == 8:
return self._read_string(f)
if value_type == 9:
array_type = struct.unpack('I', f.read(4))[0]
array_len = struct.unpack('Q', f.read(8))[0]
return [self._read_value(f, array_type) for _ in range(array_len)]
raise ReaderError('Unsupported value type')
def _read_tensor_info(self, f):
"""Read tensor information from the file"""
name = self._read_string(f)
n_dimensions = struct.unpack('I', f.read(4))[0]
dimensions = struct.unpack(f'{n_dimensions}Q', f.read(8 * n_dimensions)
)
tensor_type = struct.unpack('I', f.read(4))[0]
offset = struct.unpack('Q', f.read(8))[0]
return {'name': name, 'n_dimensions': n_dimensions, 'dimensions':
dimensions, 'type': tensor_type, 'offset': offset}
def load_tensors(self):
"""Load the tensors from the file"""
tensors = []
with open(self.file_path, 'rb') as f:
for tensor_info in self.tensors_info:
f.seek(tensor_info['offset'])
tensor_data = f.read(tensor_info['n_dimensions'])
if f.tell() % self.alignment != 0:
f.read(self.alignment - f.tell() % self.alignment)
tensors.append(tensor_data)
return tensors
def print(self):
"""Print the file details"""
print(f'Version: {self.version}')
print(f'Format: {self.format}')
print('Tensors Info:')
for tensor_info in self.tensors_info:
print(
f" Name: {tensor_info['name']},\tShape: {tensor_info['dimensions']},\tType: {self.TENSOR_TYPES[tensor_info['type']]},\tOffset: {tensor_info['offset']}"
)
print('Metadata:')
for key, value in self.metadata.items():
if isinstance(value, list) and len(value) > 50:
print(
f' {key}: {value[:50]}... ({len(value) - 50} more elements)'
)
else:
print(f' {key}: {value}')
gguf_connector/s.py
+44
View File
@@ -0,0 +1,44 @@
from safetensors.torch import load_file, save_file
import os
def split_safetensors_file(input_path, output_dir):
if not os.path.exists(output_dir):
os.makedirs(output_dir)
tensors = load_file(input_path)
components = {'model': {}, 'encoder': {}, 'vae': {}}
print(f'Splitting {input_path} into components...')
for key, tensor in tensors.items():
for comp in components:
if key.startswith(comp):
if comp == 'vae':
new_key = key[len('vae.'):] if key.startswith('vae.'
) else key
components[comp][new_key] = tensors[key]
else:
components[comp][key] = tensors[key]
break
for comp_name, comp_tensors in components.items():
if comp_tensors:
output_path = os.path.join(output_dir, f'{comp_name}.safetensors')
save_file(comp_tensors, output_path)
print(
f'✅ Saved {comp_name} to {output_path} ({len(comp_tensors)} tensors)'
)
else:
print(f'⚠️ No tensors found for component: {comp_name}')
safetensors_files = [file for file in os.listdir() if file.endswith(
'.safetensors')]
if safetensors_files:
print('Safetensors file(s) available. Select which one to split:')
for index, file_name in enumerate(safetensors_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = safetensors_files[choice_index]
print(f'Model file: {selected_file} is selected!')
split_safetensors_file(selected_file, 'split_output')
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No safetensors files are available in the current directory.')
input('--- Press ENTER To Exit ---')
gguf_connector/s0.py
+43
View File
@@ -0,0 +1,43 @@
from safetensors.torch import load_file, save_file
from tqdm import tqdm
import os
def list_safetensors_files():
files = [f for f in os.listdir() if f.endswith('.safetensors')]
if not files:
print('No .safetensors files found in the current directory.')
return None
print('Select a .safetensors file to split:')
for idx, file in enumerate(files, 1):
print(f'{idx}: {file}')
while True:
try:
choice = int(input('Enter number: '))
if 1 <= choice <= len(files):
return files[choice - 1]
except ValueError:
pass
print('Invalid selection. Try again.')
def split_safetensors(file_path):
tensors = load_file(file_path)
dim2 = {}
dim3 = {}
dim4 = {}
dim5 = {}
for name in tqdm(tensors, desc='Splitting tensors'):
tensor = tensors[name]
if tensor.dim() <= 2:
dim2[name] = tensor
elif tensor.dim() == 3:
dim3[name] = tensor
elif tensor.dim() == 4:
dim4[name] = tensor
else:
dim5[name] = tensor
save_file(dim2, '2d.safetensors')
save_file(dim3, '3d.safetensors')
save_file(dim4, '4d.safetensors')
save_file(dim5, '5d.safetensors')
print('Saved: 2d, 3d, 4d and 5d.safetensors')
selected_file = list_safetensors_files()
if selected_file:
split_safetensors(selected_file)
gguf_connector/s1.py
+35
View File
@@ -0,0 +1,35 @@
from safetensors.torch import load_file, save_file
from tqdm import tqdm
import os
def list_safetensors_files():
files = [f for f in os.listdir() if f.endswith('.safetensors')]
if not files:
print('No .safetensors files found in the current directory.')
return None
print('Select a .safetensors file to split:')
for idx, file in enumerate(files, 1):
print(f'{idx}: {file}')
while True:
try:
choice = int(input('Enter number: '))
if 1 <= choice <= len(files):
return files[choice - 1]
except ValueError:
pass
print('Invalid selection. Try again.')
def split_safetensors(file_path):
tensors = load_file(file_path)
unidim = {}
multidim = {}
for name in tqdm(tensors, desc='Splitting tensors'):
tensor = tensors[name]
if tensor.dim() == 1:
unidim[name] = tensor
else:
multidim[name] = tensor
save_file(unidim, 'unidimension.safetensors')
save_file(multidim, 'multidimension.safetensors')
print('Saved: unidimension.safetensors and multidimension.safetensors')
selected_file = list_safetensors_files()
if selected_file:
split_safetensors(selected_file)
gguf_connector/s2.py
+195
View File
@@ -0,0 +1,195 @@
import torch
import tempfile, time
import gradio as gr
import numpy as np
import soundfile as sf
from pathlib import Path
from typing import Optional, Tuple
from diao.dia.model import Dia
if torch.cuda.is_available():
device = torch.device('cuda')
elif hasattr(torch.backends, 'mps') and torch.backends.mps.is_available():
device = torch.device('mps')
else:
device = torch.device('cpu')
print(f'Using device: {device}')
print('Loading model...')
try:
model = Dia.from_pretrained('callgg/dia-f16', compute_dtype='float16',
device=device)
except Exception as e:
print(f'Error loading Nari model: {e}')
raise
def run_inference(text_input, audio_prompt_input, max_new_tokens, cfg_scale,
temperature, top_p, cfg_filter_top_k, speed_factor):
"""
Runs inference using the globally loaded model and provided inputs.
"""
global model, device
if not text_input or text_input.isspace():
raise gr.Error('Text input cannot be empty.')
temp_txt_file_path = None
temp_audio_prompt_path = None
output_audio = 44100, np.zeros(1, dtype=np.float32)
try:
prompt_path_for_generate = None
if audio_prompt_input is not None:
sr, audio_data = audio_prompt_input
if audio_data is None or audio_data.size == 0 or audio_data.max(
) == 0:
gr.Warning(
'Audio prompt seems empty or silent, ignoring prompt.')
else:
with tempfile.NamedTemporaryFile(mode='wb', suffix='.wav',
delete=False) as f_audio:
temp_audio_prompt_path = f_audio.name
if np.issubdtype(audio_data.dtype, np.integer):
max_val = np.iinfo(audio_data.dtype).max
audio_data = audio_data.astype(np.float32) / max_val
elif not np.issubdtype(audio_data.dtype, np.floating):
gr.Warning(
f'Unsupported audio prompt dtype {audio_data.dtype}, attempting conversion.'
)
try:
audio_data = audio_data.astype(np.float32)
except Exception as conv_e:
raise gr.Error(
f'Failed to convert audio prompt to float32: {conv_e}'
)
if audio_data.ndim > 1:
if audio_data.shape[0] == 2:
audio_data = np.mean(audio_data, axis=0)
elif audio_data.shape[1] == 2:
audio_data = np.mean(audio_data, axis=1)
else:
gr.Warning(
f'Audio prompt has unexpected shape {audio_data.shape}, taking first channel/axis.'
)
audio_data = audio_data[0] if audio_data.shape[0
] < audio_data.shape[1] else audio_data[:, 0]
audio_data = np.ascontiguousarray(audio_data)
try:
sf.write(temp_audio_prompt_path, audio_data, sr,
subtype='FLOAT')
prompt_path_for_generate = temp_audio_prompt_path
print(
f'Created temporary audio prompt file: {temp_audio_prompt_path} (orig sr: {sr})'
)
except Exception as write_e:
print(f'Error writing temporary audio file: {write_e}')
raise gr.Error(
f'Failed to save audio prompt: {write_e}')
start_time = time.time()
with torch.inference_mode():
output_audio_np = model.generate(text_input, max_tokens=
max_new_tokens, cfg_scale=cfg_scale, temperature=
temperature, top_p=top_p, cfg_filter_top_k=cfg_filter_top_k,
use_torch_compile=False, audio_prompt=prompt_path_for_generate)
end_time = time.time()
print(f'Generation finished in {end_time - start_time:.2f} seconds.')
if output_audio_np is not None:
output_sr = 44100
original_len = len(output_audio_np)
speed_factor = max(0.1, min(speed_factor, 5.0))
target_len = int(original_len / speed_factor)
if target_len != original_len and target_len > 0:
x_original = np.arange(original_len)
x_resampled = np.linspace(0, original_len - 1, target_len)
resampled_audio_np = np.interp(x_resampled, x_original,
output_audio_np)
output_audio = output_sr, resampled_audio_np.astype(np.float32)
print(
f'Resampled audio from {original_len} to {target_len} samples for {speed_factor:.2f}x speed.'
)
else:
output_audio = output_sr, output_audio_np
print(
f'Skipping audio speed adjustment (factor: {speed_factor:.2f}).'
)
print(
f'Audio conversion successful. Final shape: {output_audio[1].shape}, Sample Rate: {output_sr}'
)
if output_audio[1].dtype == np.float32 or output_audio[1
].dtype == np.float64:
audio_for_gradio = np.clip(output_audio[1], -1.0, 1.0)
audio_for_gradio = (audio_for_gradio * 32767).astype(np.int16)
output_audio = output_sr, audio_for_gradio
print('Converted audio to int16 for Gradio output.')
else:
print('\nGeneration finished, but no valid tokens were produced.')
gr.Warning('Generation produced no output.')
except Exception as e:
print(f'Error during inference: {e}')
import traceback
traceback.print_exc()
raise gr.Error(f'Inference failed: {e}')
finally:
if temp_txt_file_path and Path(temp_txt_file_path).exists():
try:
Path(temp_txt_file_path).unlink()
print(f'Deleted temporary text file: {temp_txt_file_path}')
except OSError as e:
print(
f'Warning: Error deleting temporary text file {temp_txt_file_path}: {e}'
)
if temp_audio_prompt_path and Path(temp_audio_prompt_path).exists():
try:
Path(temp_audio_prompt_path).unlink()
print(
f'Deleted temporary audio prompt file: {temp_audio_prompt_path}'
)
except OSError as e:
print(
f'Warning: Error deleting temporary audio prompt file {temp_audio_prompt_path}: {e}'
)
return output_audio
css = """
"""
default_text = """[S1] This is an open weights text to dialogue model.
[S2] You get full control over scripts and voices.
[S1] Wow. Amazing. (laughs)
[S2] Try it now on Git hub or Hugging Face."""
block = gr.Blocks(title='gguf', css=css).queue()
with block:
gr.Markdown('## 🎤 Text-to-Speech Synthesis')
with gr.Row(equal_height=False):
with gr.Column(scale=1):
text_input = gr.Textbox(label='Input Text', placeholder=
'Enter text here...', value=default_text, lines=5)
audio_prompt_input = gr.Audio(label='Audio Prompt (Optional)',
show_label=True, sources=['upload', 'microphone'], type='numpy'
)
with gr.Accordion('Generation Parameters', open=False):
max_new_tokens = gr.Slider(label=
'Max New Tokens (Audio Length)', minimum=860, maximum=
3072, value=model.config.data.audio_length, step=50,
info=
'Controls the maximum length of the generated audio (more tokens = longer audio).'
)
cfg_scale = gr.Slider(label='CFG Scale (Guidance Strength)',
minimum=1.0, maximum=5.0, value=3.0, step=0.1, info=
'Higher values increase adherence to the text prompt.')
temperature = gr.Slider(label='Temperature (Randomness)',
minimum=1.0, maximum=1.5, value=1.3, step=0.05, info=
'Lower values make the output more deterministic, higher values increase randomness.'
)
top_p = gr.Slider(label='Top P (Nucleus Sampling)', minimum
=0.8, maximum=1.0, value=0.95, step=0.01, info=
'Filters vocabulary to the most likely tokens cumulatively reaching probability P.'
)
cfg_filter_top_k = gr.Slider(label='CFG Filter Top K',
minimum=15, maximum=50, value=30, step=1, info=
'Top k filter for CFG guidance.')
speed_factor_slider = gr.Slider(label='Speed Factor',
minimum=0.8, maximum=1.0, value=0.94, step=0.02, info=
'Adjusts the speed of the generated audio (1.0 = original speed).'
)
run_button = gr.Button('Generate Audio', variant='primary')
with gr.Column(scale=1):
audio_output = gr.Audio(label='Generated Audio', type='numpy',
autoplay=False)
run_button.click(fn=run_inference, inputs=[text_input,
audio_prompt_input, max_new_tokens, cfg_scale, temperature, top_p,
cfg_filter_top_k, speed_factor_slider], outputs=[audio_output],
api_name='generate_audio')
block.launch()
gguf_connector/s5.py
+35
View File
@@ -0,0 +1,35 @@
from safetensors.torch import load_file, save_file
from tqdm import tqdm
import os
def list_safetensors_files():
files = [f for f in os.listdir() if f.endswith('.safetensors')]
if not files:
print('No .safetensors files found in the current directory.')
return None
print('Select a .safetensors file to split:')
for idx, file in enumerate(files, 1):
print(f'{idx}: {file}')
while True:
try:
choice = int(input('Enter number: '))
if 1 <= choice <= len(files):
return files[choice - 1]
except ValueError:
pass
print('Invalid selection. Try again.')
def split_safetensors(file_path):
tensors = load_file(file_path)
unidim = {}
multidim = {}
for name in tqdm(tensors, desc='Splitting tensors'):
tensor = tensors[name]
if tensor.dim() <= 4:
unidim[name] = tensor
else:
multidim[name] = tensor
save_file(unidim, 'except5d.safetensors')
save_file(multidim, '5d.safetensors')
print('Saved: except5d.safetensors and 5d.safetensors')
selected_file = list_safetensors_files()
if selected_file:
split_safetensors(selected_file)
gguf_connector/sx.py
+51
View File
@@ -0,0 +1,51 @@
from safetensors.torch import load_file, save_file
from tqdm import tqdm
import os
def list_safetensors_files():
files = [f for f in os.listdir() if f.endswith('.safetensors')]
if not files:
print('No .safetensors files found in the current directory.')
return None
print('Select a .safetensors file to split:')
for idx, file in enumerate(files, 1):
print(f'{idx}: {file}')
while True:
try:
choice = int(input('Enter number: '))
if 1 <= choice <= len(files):
return files[choice - 1]
except ValueError:
pass
print('Invalid selection. Try again.')
def split_safetensors(file_path):
tensors = load_file(file_path)
dim1 = {}
dim2 = {}
dim3 = {}
dim4 = {}
dim5 = {}
dimx = {}
for name in tqdm(tensors, desc='Splitting tensors'):
tensor = tensors[name]
if tensor.dim() == 1:
dim1[name] = tensor
elif tensor.dim() == 2:
dim2[name] = tensor
elif tensor.dim() == 3:
dim3[name] = tensor
elif tensor.dim() == 4:
dim4[name] = tensor
elif tensor.dim() == 5:
dim5[name] = tensor
else:
dimx[name] = tensor
save_file(dim1, '1d.safetensors')
save_file(dim2, '2d.safetensors')
save_file(dim3, '3d.safetensors')
save_file(dim4, '4d.safetensors')
save_file(dim5, '5d.safetensors')
save_file(dimx, 'xd.safetensors')
print('Saved: 1d, 2d, 3d, 4d, 5d and xd.safetensors')
selected_file = list_safetensors_files()
if selected_file:
split_safetensors(selected_file)
gguf_connector/sy.py
+35
View File
@@ -0,0 +1,35 @@
from safetensors.torch import load_file, save_file
from tqdm import tqdm
import os
def list_safetensors_files():
files = [f for f in os.listdir() if f.endswith('.safetensors')]
if not files:
print('No .safetensors files found in the current directory.')
return None
print('Select a .safetensors file to split:')
for idx, file in enumerate(files, 1):
print(f'{idx}: {file}')
while True:
try:
choice = int(input('Enter number: '))
if 1 <= choice <= len(files):
return files[choice - 1]
except ValueError:
pass
print('Invalid selection. Try again.')
def split_safetensors(file_path):
tensors = load_file(file_path)
dim2 = {}
dimy = {}
for name in tqdm(tensors, desc='Splitting tensors'):
tensor = tensors[name]
if tensor.dim() == 2:
dim2[name] = tensor
else:
dimy[name] = tensor
save_file(dim2, 'tensor2d.safetensors')
save_file(dimy, 'except2d.safetensors')
print('Saved: except2d.safetensors and tensor2d.safetensors')
selected_file = list_safetensors_files()
if selected_file:
split_safetensors(selected_file)
gguf_connector/t.py
+195
View File
@@ -0,0 +1,195 @@
import torch
from .writer import GGUFWriter, GGMLQuantizationType
from .quant import quantize, QuantError
from .const import GGML_QUANT_VERSION, LlamaFileType
from safetensors.torch import load_file
from tqdm import tqdm
QUANTIZATION_THRESHOLD = 1024
REARRANGE_THRESHOLD = 512
MAX_TENSOR_NAME_LENGTH = 127
class ModelTemplate:
arch = 'invalid'
shape_fix = False
keys_detect = []
keys_banned = []
class ModelSDXL(ModelTemplate):
arch = 'sdxl'
shape_fix = True
keys_detect = [('down_blocks.0.downsamplers.0.conv.weight',
'add_embedding.linear_1.weight'), ('input_blocks.3.0.op.weight',
'input_blocks.6.0.op.weight', 'output_blocks.2.2.conv.weight',
'output_blocks.5.2.conv.weight'), ('label_emb.0.0.weight',)]
class ModelSD1(ModelTemplate):
arch = 'sd1'
shape_fix = True
keys_detect = [('down_blocks.0.downsamplers.0.conv.weight',), (
'input_blocks.3.0.op.weight', 'input_blocks.6.0.op.weight',
'input_blocks.9.0.op.weight', 'output_blocks.2.1.conv.weight',
'output_blocks.5.2.conv.weight', 'output_blocks.8.2.conv.weight')]
class ModelSD3(ModelTemplate):
arch = 'sd3'
keys_detect = [('transformer_blocks.0.attn.add_q_proj.weight',), (
'joint_blocks.0.x_block.attn.qkv.weight',)]
keys_banned = ['transformer_blocks.0.attn.add_q_proj.weight']
class ModelFlux(ModelTemplate):
arch = 'flux'
keys_detect = [('transformer_blocks.0.attn.norm_added_k.weight',), (
'double_blocks.0.img_attn.proj.weight',)]
keys_banned = ['transformer_blocks.0.attn.norm_added_k.weight']
class ModelAura(ModelTemplate):
arch = 'aura'
keys_detect = [('double_layers.3.modX.1.weight',), (
'joint_transformer_blocks.3.ff_context.out_projection.weight',)]
keys_banned = [
'joint_transformer_blocks.3.ff_context.out_projection.weight']
class ModelHYVID(ModelTemplate):
arch = 'hyvid'
keys_detect = [('txt_in.individual_token_refiner.blocks.0.norm1.weight',)]
class ModelCosmos(ModelTemplate):
arch = 'cosmos'
keys_detect = [('blocks.block0.blocks.0.block.attn.to_q.0.weight',)]
class ModelPixArt(ModelTemplate):
arch = 'pixart'
keys_detect = [('transformer_blocks.27.scale_shift_table',)]
class ModelMochi(ModelTemplate):
arch = 'mochi'
keys_detect = [('t5_yproj.weight',)]
class ModelLumina(ModelTemplate):
arch = 'lumina'
keys_detect = [('cap_embedder.0.weight',)]
class ModelLTXV(ModelTemplate):
arch = 'ltxv'
keys_detect = [('adaln_single.emb.timestep_embedder.linear_2.weight',
'transformer_blocks.27.scale_shift_table',
'caption_projection.linear_2.weight')]
arch_list = [ModelSDXL, ModelSD1, ModelSD3, ModelAura, ModelFlux,
ModelHYVID, ModelCosmos, ModelPixArt, ModelMochi, ModelLumina, ModelLTXV]
def is_model_arch(model, state_dict):
matched = False
invalid = False
for match_list in model.keys_detect:
if all(key in state_dict for key in match_list):
matched = True
invalid = any(key in state_dict for key in model.keys_banned)
break
assert not invalid, 'Model architecture not allowed for conversion! (i.e. reference VS diffusers format)'
return matched
def detect_arch(state_dict):
model_arch = None
for arch in arch_list:
if is_model_arch(arch, state_dict):
model_arch = arch
break
assert model_arch is not None, 'Unknown model architecture!'
return model_arch
def load_state_dict(path):
if any(path.endswith(x) for x in ['.ckpt', '.pt', '.bin', '.pth']):
state_dict = torch.load(path, map_location='cpu', weights_only=True)
state_dict = state_dict.get('model', state_dict)
else:
state_dict = load_file(path)
prefix = None
for pfx in ['model.diffusion_model.', 'model.']:
if any([x.startswith(pfx) for x in state_dict.keys()]):
prefix = pfx
break
sd = {}
for k, v in state_dict.items():
if prefix and prefix not in k:
continue
if prefix:
k = k.replace(prefix, '')
sd[k] = v
return sd
def load_model(path):
state_dict = load_state_dict(path)
model_arch = detect_arch(state_dict)
print(f'* Architecture detected from input: {model_arch.arch}')
writer = GGUFWriter(path=None, arch=model_arch.arch)
return writer, state_dict, model_arch
def handle_tensors(args, writer, state_dict, model_arch):
name_lengths = tuple(sorted(((key, len(key)) for key in state_dict.keys
()), key=lambda item: item[1], reverse=True))
if not name_lengths:
return
max_name_len = name_lengths[0][1]
if max_name_len > MAX_TENSOR_NAME_LENGTH:
bad_list = ', '.join(f'{key!r} ({namelen})' for key, namelen in
name_lengths if namelen > MAX_TENSOR_NAME_LENGTH)
raise ValueError(
f'Can only handle tensor names up to {MAX_TENSOR_NAME_LENGTH} characters. Tensors exceeding the limit: {bad_list}'
)
for key, data in tqdm(state_dict.items()):
old_dtype = data.dtype
if data.dtype == torch.bfloat16:
data = data.to(torch.float32).numpy()
elif data.dtype in [getattr(torch, 'float8_e4m3fn', '_invalid'),
getattr(torch, 'float8_e5m2', '_invalid')]:
data = data.to(torch.float16).numpy()
else:
data = data.numpy()
n_dims = len(data.shape)
data_shape = data.shape
data_qtype = getattr(GGMLQuantizationType, 'BF16' if old_dtype ==
torch.bfloat16 else 'F16')
n_params = 1
for dim_size in data_shape:
n_params *= dim_size
blacklist = {'time_embedding.', 'add_embedding.', 'time_in.',
'txt_in.', 'vector_in.', 'img_in.', 'guidance_in.', 'final_layer.'}
if old_dtype in (torch.float32, torch.bfloat16):
if n_dims == 1:
data_qtype = GGMLQuantizationType.F32
elif n_params <= QUANTIZATION_THRESHOLD:
data_qtype = GGMLQuantizationType.F32
elif '.weight' in key and any(x in key for x in blacklist):
data_qtype = GGMLQuantizationType.F32
if (model_arch.shape_fix and n_dims > 1 and n_params >=
REARRANGE_THRESHOLD and (n_params / 256).is_integer() and not (
data.shape[-1] / 256).is_integer()):
data = data.reshape(n_params // 256, 256)
try:
data = quantize(data, data_qtype)
except (AttributeError, QuantError) as e:
tqdm.write(f'falling back to F16: {e}')
data_qtype = GGMLQuantizationType.F16
data = quantize(data, data_qtype)
new_name = key
shape_str = f"{{{', '.join(str(n) for n in reversed(data.shape))}}}"
tqdm.write(
f"{f'%-{max_name_len + 4}s' % f'{new_name}'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}"
)
writer.add_tensor(new_name, data, raw_dtype=data_qtype)
import os
safetensors_files = [file for file in os.listdir() if file.endswith(
'.safetensors')]
if safetensors_files:
print('Safetensors file(s) available. Select which one to convert:')
for index, file_name in enumerate(safetensors_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = safetensors_files[choice_index]
print(f'Model file: {selected_file} is selected!')
path = selected_file
writer, state_dict, model_arch = load_model(path)
writer.add_quantization_version(GGML_QUANT_VERSION)
if next(iter(state_dict.values())).dtype == torch.bfloat16:
out_path = f'{os.path.splitext(path)[0]}-bf16.gguf'
writer.add_file_type(LlamaFileType.MOSTLY_BF16)
else:
out_path = f'{os.path.splitext(path)[0]}-f16.gguf'
writer.add_file_type(LlamaFileType.MOSTLY_F16)
if os.path.isfile(out_path):
input('Output exists enter to continue or ctrl+c to abort!')
handle_tensors(path, writer, state_dict, model_arch)
writer.write_header_to_file(path=out_path)
writer.write_kv_data_to_file()
writer.write_tensors_to_file(progress=True)
writer.close()
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No safetensors files are available in the current directory.')
input('--- Press ENTER To Exit ---')
gguf_connector/t0.py
+106
View File
@@ -0,0 +1,106 @@
import torch
from .writer import GGUFWriter, GGMLQuantizationType
from .quant import quantize, QuantError
from .const import GGML_QUANT_VERSION, LlamaFileType
from safetensors.torch import load_file
from tqdm import tqdm
QUANTIZATION_THRESHOLD = 1024
REARRANGE_THRESHOLD = 512
MAX_TENSOR_NAME_LENGTH = 127
def load_state_dict(path):
state_dict = load_file(path)
sd = {}
for k, v in state_dict.items():
sd[k] = v
return sd
def load_model(path, model_arch):
state_dict = load_state_dict(path)
writer = GGUFWriter(path=None, arch=model_arch)
return writer, state_dict, model_arch
def handle_tensors(args, writer, state_dict, model_arch):
name_lengths = tuple(sorted(((key, len(key)) for key in state_dict.keys
()), key=lambda item: item[1], reverse=True))
if not name_lengths:
return
max_name_len = name_lengths[0][1]
if max_name_len > MAX_TENSOR_NAME_LENGTH:
bad_list = ', '.join(f'{key!r} ({namelen})' for key, namelen in
name_lengths if namelen > MAX_TENSOR_NAME_LENGTH)
raise ValueError(
f'Can only handle tensor names up to {MAX_TENSOR_NAME_LENGTH} characters. Tensors exceeding the limit: {bad_list}'
)
for key, data in tqdm(state_dict.items()):
old_dtype = data.dtype
if data.dtype == torch.bfloat16:
data = data.to(torch.float32).numpy()
elif data.dtype in [getattr(torch, 'float8_e4m3fn', '_invalid'),
getattr(torch, 'float8_e5m2', '_invalid')]:
data = data.to(torch.float16).numpy()
else:
data = data.numpy()
n_dims = len(data.shape)
data_shape = data.shape
data_qtype = getattr(GGMLQuantizationType, 'BF16' if old_dtype ==
torch.bfloat16 else 'F16')
n_params = 1
for dim_size in data_shape:
n_params *= dim_size
blacklist = {'time_embedding.', 'add_embedding.', 'time_in.',
'txt_in.', 'vector_in.', 'img_in.', 'guidance_in.', 'final_layer.'}
if old_dtype in (torch.float32, torch.bfloat16):
if n_dims == 1:
data_qtype = GGMLQuantizationType.F32
elif n_params <= QUANTIZATION_THRESHOLD:
data_qtype = GGMLQuantizationType.F32
elif '.weight' in key and any(x in key for x in blacklist):
data_qtype = GGMLQuantizationType.F32
try:
data = quantize(data, data_qtype)
except (AttributeError, QuantError) as e:
tqdm.write(f'falling back to F16: {e}')
data_qtype = GGMLQuantizationType.F16
data = quantize(data, data_qtype)
new_name = key
shape_str = f"{{{', '.join(str(n) for n in reversed(data.shape))}}}"
tqdm.write(
f"{f'%-{max_name_len + 4}s' % f'{new_name}'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}"
)
writer.add_tensor(new_name, data, raw_dtype=data_qtype)
import os
safetensors_files = [file for file in os.listdir() if file.endswith(
'.safetensors')]
if safetensors_files:
print('Safetensors file(s) available. Select which one to convert:')
for index, file_name in enumerate(safetensors_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = safetensors_files[choice_index]
print(f'Model file: {selected_file} is selected!')
path = selected_file
ask = input('Assign a name for the model (Y/n)? ')
if ask.lower() == 'y':
given = input('Enter a model name: ')
else:
given = None
writer, state_dict, model_arch = load_model(path, given)
writer.add_quantization_version(GGML_QUANT_VERSION)
if next(iter(state_dict.values())).dtype == torch.bfloat16:
out_path = f'{os.path.splitext(path)[0]}-bf16.gguf'
writer.add_file_type(LlamaFileType.MOSTLY_BF16)
else:
out_path = f'{os.path.splitext(path)[0]}-f16.gguf'
writer.add_file_type(LlamaFileType.MOSTLY_F16)
if os.path.isfile(out_path):
input('Output exists enter to continue or ctrl+c to abort!')
handle_tensors(path, writer, state_dict, model_arch)
writer.write_header_to_file(path=out_path)
writer.write_kv_data_to_file()
writer.write_tensors_to_file(progress=True)
writer.close()
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No safetensors files are available in the current directory.')
input('--- Press ENTER To Exit ---')
gguf_connector/t1.py
+118
View File
@@ -0,0 +1,118 @@
import torch
from .writer import GGUFWriter, GGMLQuantizationType
from .quant import quantize, QuantError
from .const import GGML_QUANT_VERSION, LlamaFileType
from safetensors.torch import load_file
from tqdm import tqdm
QUANTIZATION_THRESHOLD = 1024
REARRANGE_THRESHOLD = 512
MAX_TENSOR_NAME_LENGTH = 127
def load_state_dict(path):
state_dict = load_file(path)
prefix = None
for pfx in ['model.diffusion_model.', 'model.']:
if any([x.startswith(pfx) for x in state_dict.keys()]):
prefix = pfx
break
sd = {}
for k, v in state_dict.items():
if prefix and prefix not in k:
continue
if prefix:
k = k.replace(prefix, '')
sd[k] = v
return sd
def load_model(path, model_arch):
state_dict = load_state_dict(path)
writer = GGUFWriter(path=None, arch=model_arch)
return writer, state_dict, model_arch
def handle_tensors(args, writer, state_dict, model_arch):
name_lengths = tuple(sorted(((key, len(key)) for key in state_dict.keys
()), key=lambda item: item[1], reverse=True))
if not name_lengths:
return
max_name_len = name_lengths[0][1]
if max_name_len > MAX_TENSOR_NAME_LENGTH:
bad_list = ', '.join(f'{key!r} ({namelen})' for key, namelen in
name_lengths if namelen > MAX_TENSOR_NAME_LENGTH)
raise ValueError(
f'Can only handle tensor names up to {MAX_TENSOR_NAME_LENGTH} characters. Tensors exceeding the limit: {bad_list}'
)
for key, data in tqdm(state_dict.items()):
old_dtype = data.dtype
if data.dtype == torch.bfloat16:
data = data.to(torch.float32).numpy()
elif data.dtype in [getattr(torch, 'float8_e4m3fn', '_invalid'),
getattr(torch, 'float8_e5m2', '_invalid')]:
data = data.to(torch.float16).numpy()
else:
data = data.numpy()
n_dims = len(data.shape)
data_shape = data.shape
data_qtype = getattr(GGMLQuantizationType, 'BF16' if old_dtype ==
torch.bfloat16 else 'F16')
n_params = 1
for dim_size in data_shape:
n_params *= dim_size
blacklist = {'time_embedding.', 'add_embedding.', 'time_in.',
'txt_in.', 'vector_in.', 'img_in.', 'guidance_in.', 'final_layer.'}
if old_dtype in (torch.float32, torch.bfloat16):
if n_dims == 1:
data_qtype = GGMLQuantizationType.F32
elif n_params <= QUANTIZATION_THRESHOLD:
data_qtype = GGMLQuantizationType.F32
elif '.weight' in key and any(x in key for x in blacklist):
data_qtype = GGMLQuantizationType.F32
try:
data = quantize(data, data_qtype)
except (AttributeError, QuantError) as e:
tqdm.write(f'falling back to F16: {e}')
data_qtype = GGMLQuantizationType.F16
data = quantize(data, data_qtype)
orig_shape = data.shape
writer.add_array(f'comfy.gguf.orig_shape.{key}', tuple(int(dim) for
dim in orig_shape))
new_name = key
shape_str = f"{{{', '.join(str(n) for n in reversed(data.shape))}}}"
tqdm.write(
f"{f'%-{max_name_len + 4}s' % f'{new_name}'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}"
)
writer.add_tensor(new_name, data, raw_dtype=data_qtype)
import os
safetensors_files = [file for file in os.listdir() if file.endswith(
'.safetensors')]
if safetensors_files:
print('Safetensors file(s) available. Select which one to convert:')
for index, file_name in enumerate(safetensors_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = safetensors_files[choice_index]
print(f'Model file: {selected_file} is selected!')
path = selected_file
ask = input('Assign a name for the model (Y/n)? ')
if ask.lower() == 'y':
given = input('Enter a model name: ')
else:
given = None
writer, state_dict, model_arch = load_model(path, given)
writer.add_quantization_version(GGML_QUANT_VERSION)
if next(iter(state_dict.values())).dtype == torch.bfloat16:
out_path = f'{os.path.splitext(path)[0]}-bf16.gguf'
writer.add_file_type(LlamaFileType.MOSTLY_BF16)
else:
out_path = f'{os.path.splitext(path)[0]}-f16.gguf'
writer.add_file_type(LlamaFileType.MOSTLY_F16)
if os.path.isfile(out_path):
input('Output exists enter to continue or ctrl+c to abort!')
handle_tensors(path, writer, state_dict, model_arch)
writer.write_header_to_file(path=out_path)
writer.write_kv_data_to_file()
writer.write_tensors_to_file(progress=True)
writer.close()
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No safetensors files are available in the current directory.')
input('--- Press ENTER To Exit ---')
gguf_connector/t2.py
+115
View File
@@ -0,0 +1,115 @@
import torch
from .writer import GGUFWriter, GGMLQuantizationType
from .quant import quantize, QuantError
from .const import GGML_QUANT_VERSION, LlamaFileType
from safetensors.torch import load_file
from tqdm import tqdm
QUANTIZATION_THRESHOLD = 1024
REARRANGE_THRESHOLD = 512
MAX_TENSOR_NAME_LENGTH = 127
def load_state_dict(path):
state_dict = load_file(path)
prefix = None
for pfx in ['model.diffusion_model.', 'model.']:
if any([x.startswith(pfx) for x in state_dict.keys()]):
prefix = pfx
break
sd = {}
for k, v in state_dict.items():
if prefix and prefix not in k:
continue
if prefix:
k = k.replace(prefix, '')
sd[k] = v
return sd
def load_model(path, model_arch):
state_dict = load_state_dict(path)
writer = GGUFWriter(path=None, arch=model_arch)
return writer, state_dict, model_arch
def handle_tensors(args, writer, state_dict, model_arch):
name_lengths = tuple(sorted(((key, len(key)) for key in state_dict.keys
()), key=lambda item: item[1], reverse=True))
if not name_lengths:
return
max_name_len = name_lengths[0][1]
if max_name_len > MAX_TENSOR_NAME_LENGTH:
bad_list = ', '.join(f'{key!r} ({namelen})' for key, namelen in
name_lengths if namelen > MAX_TENSOR_NAME_LENGTH)
raise ValueError(
f'Can only handle tensor names up to {MAX_TENSOR_NAME_LENGTH} characters. Tensors exceeding the limit: {bad_list}'
)
for key, data in tqdm(state_dict.items()):
old_dtype = data.dtype
if data.dtype == torch.bfloat16:
data = data.to(torch.float32).numpy()
elif data.dtype in [getattr(torch, 'float8_e4m3fn', '_invalid'),
getattr(torch, 'float8_e5m2', '_invalid')]:
data = data.to(torch.float16).numpy()
else:
data = data.numpy()
n_dims = len(data.shape)
data_shape = data.shape
data_qtype = getattr(GGMLQuantizationType, 'BF16' if old_dtype ==
torch.bfloat16 else 'F16')
n_params = 1
for dim_size in data_shape:
n_params *= dim_size
blacklist = {'time_embedding.', 'add_embedding.', 'time_in.',
'txt_in.', 'vector_in.', 'img_in.', 'guidance_in.', 'final_layer.'}
if old_dtype in (torch.float32, torch.bfloat16):
if n_dims == 1:
data_qtype = GGMLQuantizationType.F32
elif n_params <= QUANTIZATION_THRESHOLD:
data_qtype = GGMLQuantizationType.F32
elif '.weight' in key and any(x in key for x in blacklist):
data_qtype = GGMLQuantizationType.F32
try:
data = quantize(data, data_qtype)
except (AttributeError, QuantError) as e:
tqdm.write(f'falling back to F16: {e}')
data_qtype = GGMLQuantizationType.F16
data = quantize(data, data_qtype)
new_name = key
shape_str = f"{{{', '.join(str(n) for n in reversed(data.shape))}}}"
tqdm.write(
f"{f'%-{max_name_len + 4}s' % f'{new_name}'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}"
)
writer.add_tensor(new_name, data, raw_dtype=data_qtype)
import os
safetensors_files = [file for file in os.listdir() if file.endswith(
'.safetensors')]
if safetensors_files:
print('Safetensors file(s) available. Select which one to convert:')
for index, file_name in enumerate(safetensors_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = safetensors_files[choice_index]
print(f'Model file: {selected_file} is selected!')
path = selected_file
ask = input('Assign a name for the model (Y/n)? ')
if ask.lower() == 'y':
given = input('Enter a model name: ')
else:
given = None
writer, state_dict, model_arch = load_model(path, given)
writer.add_quantization_version(GGML_QUANT_VERSION)
if next(iter(state_dict.values())).dtype == torch.bfloat16:
out_path = f'{os.path.splitext(path)[0]}-bf16.gguf'
writer.add_file_type(LlamaFileType.MOSTLY_BF16)
else:
out_path = f'{os.path.splitext(path)[0]}-f16.gguf'
writer.add_file_type(LlamaFileType.MOSTLY_F16)
if os.path.isfile(out_path):
input('Output exists enter to continue or ctrl+c to abort!')
handle_tensors(path, writer, state_dict, model_arch)
writer.write_header_to_file(path=out_path)
writer.write_kv_data_to_file()
writer.write_tensors_to_file(progress=True)
writer.close()
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No safetensors files are available in the current directory.')
input('--- Press ENTER To Exit ---')
gguf_connector/t3.py
+80
View File
@@ -0,0 +1,80 @@
import torch # optional (need torch to work; pip install torch)
import numpy as np
from safetensors.torch import save_file
from typing import Dict, Tuple
from .quant import dequantize
from .reader import GGUFReader
from tqdm import tqdm
def load_gguf_and_extract_metadata(gguf_path):
reader = GGUFReader(gguf_path)
tensors_metadata = []
for tensor in reader.tensors:
tensor_metadata = {
'name': tensor.name,
'shape': tuple(tensor.shape.tolist()),
'n_elements': tensor.n_elements,
'n_bytes': tensor.n_bytes,
'data_offset': tensor.data_offset,
'type': tensor.tensor_type,
}
tensors_metadata.append(tensor_metadata)
return reader, tensors_metadata
def convert_gguf_to_safetensors(gguf_path, output_path, use_bf16):
reader, tensors_metadata = load_gguf_and_extract_metadata(gguf_path)
print(f'Extracted {len(tensors_metadata)} tensors from GGUF file')
tensors_dict: dict[str, torch.Tensor] = {}
for i, tensor_info in enumerate(tqdm(tensors_metadata, desc=
'Converting tensors', unit='tensor')):
tensor_name = tensor_info['name']
tensor_data = reader.get_tensor(i)
weights = dequantize(tensor_data.data, tensor_data.tensor_type).copy()
try:
if use_bf16:
weights_tensor = torch.from_numpy(weights).to(dtype=torch.
float32)
weights_tensor = weights_tensor.to(torch.bfloat16)
else:
weights_tensor = torch.from_numpy(weights).to(dtype=torch.
float16)
weights_hf = weights_tensor
except Exception as e:
print(
f"Error during BF16 conversion for tensor '{tensor_name}': {e}"
)
weights_tensor = torch.from_numpy(weights.astype(np.float32)).to(
torch.float16)
weights_hf = weights_tensor
tensors_dict[tensor_name] = weights_hf
metadata = {key: str(reader.get_field(key)) for key in reader.fields}
save_file(tensors_dict, output_path, metadata=metadata)
print('Conversion complete!')
import os
gguf_files = [file for file in os.listdir() if file.endswith('.gguf')]
if gguf_files:
print('GGUF file(s) available. Select which one to convert:')
for index, file_name in enumerate(gguf_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(gguf_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = gguf_files[choice_index]
print(f'Model file: {selected_file} is selected!')
input_path = selected_file
ask = input('Convert tensors to BF16 format instead of FP16 (Y/n)? ')
if ask.lower() == 'y':
use_bf16 = True
out_path = f'{os.path.splitext(input_path)[0]}-bf16.safetensors'
else:
use_bf16 = False
out_path = f'{os.path.splitext(input_path)[0]}-fp16.safetensors'
convert_gguf_to_safetensors(input_path, out_path, use_bf16)
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No GGUF files are available in the current directory.')
input('--- Press ENTER To Exit ---')
gguf_connector/t4.py
+198
View File
@@ -0,0 +1,198 @@
import torch
from .writer import GGUFWriter, GGMLQuantizationType
from .quant import quantize, QuantError
from .const import GGML_QUANT_VERSION, LlamaFileType
from safetensors.torch import load_file
from tqdm import tqdm
QUANTIZATION_THRESHOLD = 1024
REARRANGE_THRESHOLD = 512
MAX_TENSOR_NAME_LENGTH = 127
class ModelTemplate:
arch = 'invalid'
shape_fix = False
keys_detect = []
keys_banned = []
class ModelSDXL(ModelTemplate):
arch = 'sdxl'
shape_fix = True
keys_detect = [('down_blocks.0.downsamplers.0.conv.weight',
'add_embedding.linear_1.weight'), ('input_blocks.3.0.op.weight',
'input_blocks.6.0.op.weight', 'output_blocks.2.2.conv.weight',
'output_blocks.5.2.conv.weight'), ('label_emb.0.0.weight',)]
class ModelSD1(ModelTemplate):
arch = 'sd1'
shape_fix = True
keys_detect = [('down_blocks.0.downsamplers.0.conv.weight',), (
'input_blocks.3.0.op.weight', 'input_blocks.6.0.op.weight',
'input_blocks.9.0.op.weight', 'output_blocks.2.1.conv.weight',
'output_blocks.5.2.conv.weight', 'output_blocks.8.2.conv.weight')]
class ModelSD3(ModelTemplate):
arch = 'sd3'
keys_detect = [('transformer_blocks.0.attn.add_q_proj.weight',), (
'joint_blocks.0.x_block.attn.qkv.weight',)]
keys_banned = ['transformer_blocks.0.attn.add_q_proj.weight']
class ModelAura(ModelTemplate):
arch = 'aura'
keys_detect = [('double_layers.3.modX.1.weight',), (
'joint_transformer_blocks.3.ff_context.out_projection.weight',)]
keys_banned = [
'joint_transformer_blocks.3.ff_context.out_projection.weight']
class ModelPixArt(ModelTemplate):
arch = 'pixart'
keys_detect = [('transformer_blocks.27.scale_shift_table',)]
class ModelFlux(ModelTemplate):
arch = 'flux'
keys_detect = [('transformer_blocks.0.attn.norm_added_k.weight',), (
'double_blocks.0.img_attn.proj.weight',)]
keys_banned = ['transformer_blocks.0.attn.norm_added_k.weight']
class ModelLumina(ModelTemplate):
arch = 'lumina'
keys_detect = [('cap_embedder.0.weight',)]
class ModelHYVID(ModelTemplate):
arch = 'hyvid'
keys_detect = [('txt_in.individual_token_refiner.blocks.0.norm1.weight',)]
class ModelCosmos(ModelTemplate):
arch = 'cosmos'
keys_detect = [('blocks.block0.blocks.0.block.attn.to_q.0.weight',)]
class ModelMochi(ModelTemplate):
arch = 'mochi'
keys_detect = [('t5_yproj.weight',)]
class ModelLTXV(ModelTemplate):
arch = 'ltxv'
keys_detect = [('adaln_single.emb.timestep_embedder.linear_2.weight',
'transformer_blocks.27.scale_shift_table',
'caption_projection.linear_2.weight')]
arch_list = [ModelFlux, ModelSD3, ModelAura, ModelPixArt, ModelHYVID,
ModelSDXL, ModelSD1, ModelCosmos, ModelMochi, ModelLumina, ModelLTXV]
def is_model_arch(model, state_dict):
matched = False
invalid = False
for match_list in model.keys_detect:
if all(key in state_dict for key in match_list):
matched = True
invalid = any(key in state_dict for key in model.keys_banned)
break
assert not invalid, 'Model architecture not allowed for conversion! (i.e. reference VS diffusers format)'
return matched
def detect_arch(state_dict):
model_arch = None
for arch in arch_list:
if is_model_arch(arch, state_dict):
model_arch = arch
break
assert model_arch is not None, 'Unknown model architecture!'
return model_arch
def load_state_dict(path):
if any(path.endswith(x) for x in ['.ckpt', '.pt', '.bin', '.pth']):
state_dict = torch.load(path, map_location='cpu', weights_only=True)
state_dict = state_dict.get('model', state_dict)
else:
state_dict = load_file(path)
prefix = None
for pfx in ['model.diffusion_model.', 'model.']:
if any([x.startswith(pfx) for x in state_dict.keys()]):
prefix = pfx
break
sd = {}
for k, v in state_dict.items():
if prefix and prefix not in k:
continue
if prefix:
k = k.replace(prefix, '')
sd[k] = v
return sd
def load_model(path):
state_dict = load_state_dict(path)
model_arch = detect_arch(state_dict)
print(f'* Architecture detected from input: {model_arch.arch}')
writer = GGUFWriter(path=None, arch=model_arch.arch)
return writer, state_dict, model_arch
def handle_tensors(args, writer, state_dict, model_arch):
name_lengths = tuple(sorted(((key, len(key)) for key in state_dict.keys
()), key=lambda item: item[1], reverse=True))
if not name_lengths:
return
max_name_len = name_lengths[0][1]
if max_name_len > MAX_TENSOR_NAME_LENGTH:
bad_list = ', '.join(f'{key!r} ({namelen})' for key, namelen in
name_lengths if namelen > MAX_TENSOR_NAME_LENGTH)
raise ValueError(
f'Can only handle tensor names up to {MAX_TENSOR_NAME_LENGTH} characters. Tensors exceeding the limit: {bad_list}'
)
for key, data in tqdm(state_dict.items()):
old_dtype = data.dtype
if data.dtype == torch.bfloat16:
data = data.to(torch.float32).numpy()
elif data.dtype in [getattr(torch, 'float8_e4m3fn', '_invalid'),
getattr(torch, 'float8_e5m2', '_invalid')]:
data = data.to(torch.float16).numpy()
else:
data = data.numpy()
n_dims = len(data.shape)
data_shape = data.shape
data_qtype = getattr(GGMLQuantizationType, 'BF16' if old_dtype ==
torch.bfloat16 else 'F16')
n_params = 1
for dim_size in data_shape:
n_params *= dim_size
blacklist = {'time_embedding.', 'add_embedding.', 'time_in.',
'txt_in.', 'vector_in.', 'img_in.', 'guidance_in.', 'final_layer.'}
if old_dtype in (torch.float32, torch.bfloat16):
if n_dims == 1:
data_qtype = GGMLQuantizationType.F32
elif n_params <= QUANTIZATION_THRESHOLD:
data_qtype = GGMLQuantizationType.F32
elif '.weight' in key and any(x in key for x in blacklist):
data_qtype = GGMLQuantizationType.F32
if (model_arch.shape_fix and n_dims > 1 and n_params >=
REARRANGE_THRESHOLD and (n_params / 256).is_integer() and not (
data.shape[-1] / 256).is_integer()):
orig_shape = data.shape
data = data.reshape(n_params // 256, 256)
writer.add_array(f'comfy.gguf.orig_shape.{key}', tuple(int(dim) for
dim in orig_shape))
try:
data = quantize(data, data_qtype)
except (AttributeError, QuantError) as e:
tqdm.write(f'falling back to F16: {e}')
data_qtype = GGMLQuantizationType.F16
data = quantize(data, data_qtype)
new_name = key
shape_str = f"{{{', '.join(str(n) for n in reversed(data.shape))}}}"
tqdm.write(
f"{f'%-{max_name_len + 4}s' % f'{new_name}'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}"
)
writer.add_tensor(new_name, data, raw_dtype=data_qtype)
import os
safetensors_files = [file for file in os.listdir() if file.endswith(
'.safetensors')]
if safetensors_files:
print('Safetensors file(s) available. Select which one to convert:')
for index, file_name in enumerate(safetensors_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = safetensors_files[choice_index]
print(f'Model file: {selected_file} is selected!')
path = selected_file
writer, state_dict, model_arch = load_model(path)
writer.add_quantization_version(GGML_QUANT_VERSION)
if next(iter(state_dict.values())).dtype == torch.bfloat16:
out_path = f'{os.path.splitext(path)[0]}-bf16.gguf'
writer.add_file_type(LlamaFileType.MOSTLY_BF16)
else:
out_path = f'{os.path.splitext(path)[0]}-f16.gguf'
writer.add_file_type(LlamaFileType.MOSTLY_F16)
if os.path.isfile(out_path):
input('Output exists enter to continue or ctrl+c to abort!')
handle_tensors(path, writer, state_dict, model_arch)
writer.write_header_to_file(path=out_path)
writer.write_kv_data_to_file()
writer.write_tensors_to_file(progress=True)
writer.close()
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No safetensors files are available in the current directory.')
input('--- Press ENTER To Exit ---')
gguf_connector/t5.py
+116
View File
@@ -0,0 +1,116 @@
import torch
from .writer import GGUFWriter, GGMLQuantizationType
from .quant import quantize, QuantError
from .const import GGML_QUANT_VERSION, LlamaFileType
from safetensors.torch import load_file
from tqdm import tqdm
QUANTIZATION_THRESHOLD = 1024
REARRANGE_THRESHOLD = 512
MAX_TENSOR_NAME_LENGTH = 127
T5_MAP = {'encoder.': 'enc.', '.block.': '.blk.', 'shared': 'token_embd',
'final_layer_norm': 'output_norm', 'layer.0.SelfAttention.q': 'attn_q',
'layer.0.SelfAttention.k': 'attn_k', 'layer.0.SelfAttention.v':
'attn_v', 'layer.0.SelfAttention.o': 'attn_o', 'layer.0.layer_norm':
'attn_norm', 'layer.0.SelfAttention.relative_attention_bias':
'attn_rel_b', 'layer.1.DenseReluDense.wi_1': 'ffn_up',
'layer.1.DenseReluDense.wo': 'ffn_down', 'layer.1.DenseReluDense.wi_0':
'ffn_gate', 'layer.1.layer_norm': 'ffn_norm'}
def load_state_dict(path, key_map):
state_dict = load_file(path)
sd = {}
for k, v in state_dict.items():
for s, d in key_map.items():
k = k.replace(s, d)
sd[k] = v
return sd
def load_model(path, model_arch):
state_dict = load_state_dict(path, T5_MAP)
writer = GGUFWriter(path=None, arch=model_arch)
return writer, state_dict, model_arch
def handle_tensors(args, writer, state_dict, model_arch):
name_lengths = tuple(sorted(((key, len(key)) for key in state_dict.keys
()), key=lambda item: item[1], reverse=True))
if not name_lengths:
return
max_name_len = name_lengths[0][1]
if max_name_len > MAX_TENSOR_NAME_LENGTH:
bad_list = ', '.join(f'{key!r} ({namelen})' for key, namelen in
name_lengths if namelen > MAX_TENSOR_NAME_LENGTH)
raise ValueError(
f'Can only handle tensor names up to {MAX_TENSOR_NAME_LENGTH} characters. Tensors exceeding the limit: {bad_list}'
)
for key, data in tqdm(state_dict.items()):
old_dtype = data.dtype
if data.dtype == torch.bfloat16:
data = data.to(torch.float32).numpy()
elif data.dtype in [getattr(torch, 'float8_e4m3fn', '_invalid'),
getattr(torch, 'float8_e5m2', '_invalid')]:
data = data.to(torch.float16).numpy()
else:
data = data.numpy()
n_dims = len(data.shape)
data_shape = data.shape
data_qtype = getattr(GGMLQuantizationType, 'BF16' if old_dtype ==
torch.bfloat16 else 'F16')
n_params = 1
for dim_size in data_shape:
n_params *= dim_size
blacklist = {'time_embedding.', 'add_embedding.', 'time_in.',
'txt_in.', 'vector_in.', 'img_in.', 'guidance_in.', 'final_layer.'}
if old_dtype in (torch.float32, torch.bfloat16):
if n_dims == 1:
data_qtype = GGMLQuantizationType.F32
elif n_params <= QUANTIZATION_THRESHOLD:
data_qtype = GGMLQuantizationType.F32
elif '.weight' in key and any(x in key for x in blacklist):
data_qtype = GGMLQuantizationType.F32
try:
data = quantize(data, data_qtype)
except (AttributeError, QuantError) as e:
tqdm.write(f'falling back to F16: {e}')
data_qtype = GGMLQuantizationType.F16
data = quantize(data, data_qtype)
new_name = key
shape_str = f"{{{', '.join(str(n) for n in reversed(data.shape))}}}"
tqdm.write(
f"{f'%-{max_name_len + 4}s' % f'{new_name}'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}"
)
writer.add_tensor(new_name, data, raw_dtype=data_qtype)
import os
safetensors_files = [file for file in os.listdir() if file.endswith(
'.safetensors')]
if safetensors_files:
print('Safetensors file(s) available. Select which one to convert:')
for index, file_name in enumerate(safetensors_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = safetensors_files[choice_index]
print(f'Model file: {selected_file} is selected!')
path = selected_file
ask = input('Assign a name for the model (Y/n)? ')
if ask.lower() == 'y':
given = input('Enter a model name: ')
else:
given = 't5'
writer, state_dict, model_arch = load_model(path, given)
writer.add_quantization_version(GGML_QUANT_VERSION)
if next(iter(state_dict.values())).dtype == torch.bfloat16:
out_path = f'{os.path.splitext(path)[0]}-bf16.gguf'
writer.add_file_type(LlamaFileType.MOSTLY_BF16)
else:
out_path = f'{os.path.splitext(path)[0]}-f16.gguf'
writer.add_file_type(LlamaFileType.MOSTLY_F16)
if os.path.isfile(out_path):
input('Output exists enter to continue or ctrl+c to abort!')
handle_tensors(path, writer, state_dict, model_arch)
writer.write_header_to_file(path=out_path)
writer.write_kv_data_to_file()
writer.write_tensors_to_file(progress=True)
writer.close()
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No safetensors files are available in the current directory.')
input('--- Press ENTER To Exit ---')
gguf_connector/t6.py
+95
View File
@@ -0,0 +1,95 @@
import torch
from .writer import GGUFWriter, GGMLQuantizationType
from .quant import quantize, QuantError
from .const import GGML_QUANT_VERSION, LlamaFileType
from safetensors.torch import load_file
from tqdm import tqdm
import numpy as np
QUANTIZATION_THRESHOLD = 1024
MAX_TENSOR_NAME_LENGTH = 127
def load_state_dict(path):
state_dict = load_file(path)
return {k: v for k, v in state_dict.items()}
def load_model(path, model_arch):
state_dict = load_state_dict(path)
writer = GGUFWriter(path=None, arch=model_arch)
return writer, state_dict, model_arch
def validate_tensor_data(data, key):
if np.any(np.isnan(data)) or np.any(np.isinf(data)):
print(f"Warning: Tensor '{key}' contains NaN or Inf values. Skipping.")
return None
return data
def handle_tensors(writer, state_dict):
name_lengths = [(key, len(key)) for key in state_dict.keys()]
if not name_lengths:
return
max_name_len = max(name_lengths, key=lambda x: x[1])[1]
for key, data in tqdm(state_dict.items(), desc='Processing Tensors'):
old_dtype = data.dtype
if old_dtype == torch.bfloat16:
data = data.to(torch.float32).numpy()
elif old_dtype in [getattr(torch, 'float8_e4m3fn', '_invalid'),
getattr(torch, 'float8_e5m2', '_invalid')]:
data = data.to(torch.float16).numpy()
else:
data = data.numpy()
data = validate_tensor_data(data, key)
if data is None:
continue
data_qtype = (GGMLQuantizationType.BF16 if old_dtype == torch.
bfloat16 else GGMLQuantizationType.F16)
n_params = data.size
if old_dtype in (torch.float32, torch.bfloat16):
if len(data.shape) == 1 or n_params <= QUANTIZATION_THRESHOLD:
data_qtype = GGMLQuantizationType.F32
try:
data = quantize(data, data_qtype)
except (AttributeError, QuantError) as e:
tqdm.write(f'Quantization error ({e}), falling back to F16')
data_qtype = GGMLQuantizationType.F16
data = quantize(data, data_qtype)
shape_str = f"{{{', '.join(map(str, reversed(data.shape)))}}}"
tqdm.write(
f'{key.ljust(max_name_len)} {old_dtype} --> {data_qtype.name}, shape = {shape_str}'
)
writer.add_tensor(key, data, raw_dtype=data_qtype)
import os
safetensors_files = [file for file in os.listdir() if file.endswith(
'.safetensors')]
if safetensors_files:
print('Safetensors file(s) available. Select which one to convert:')
for index, file_name in enumerate(safetensors_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = safetensors_files[choice_index]
print(f'Model file: {selected_file} is selected!')
path = selected_file
ask = input('Assign a name for the model (Y/n)? ')
if ask.lower() == 'y':
given = input('Enter a model name: ')
else:
given = None
writer, state_dict, _ = load_model(path, given)
writer.add_quantization_version(GGML_QUANT_VERSION)
first_tensor_dtype = next(iter(state_dict.values())).dtype
file_type = (LlamaFileType.MOSTLY_BF16 if first_tensor_dtype ==
torch.bfloat16 else LlamaFileType.MOSTLY_F16)
out_path = f'{os.path.splitext(path)[0]}-{file_type.name.lower()}.gguf'
writer.add_file_type(file_type)
if os.path.isfile(out_path):
input(
'Output file exists. Press Enter to overwrite or Ctrl+C to abort.'
)
handle_tensors(writer, state_dict)
writer.write_header_to_file(path=out_path)
writer.write_kv_data_to_file()
writer.write_tensors_to_file(progress=True)
writer.close()
print(f'Conversion completed: {out_path}')
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No safetensors files are available in the current directory.')
input('--- Press ENTER To Exit ---')
gguf_connector/t7.py
+213
View File
@@ -0,0 +1,213 @@
import torch
from .writer import GGUFWriter, GGMLQuantizationType
from .quant import quantize, QuantError
from .const import GGML_QUANT_VERSION, LlamaFileType
from safetensors.torch import load_file
from tqdm import tqdm
QUANTIZATION_THRESHOLD = 1024
REARRANGE_THRESHOLD = 512
MAX_TENSOR_NAME_LENGTH = 127
MAX_TENSOR_DIMS = 4
class ModelTemplate:
arch = 'invalid'
shape_fix = False
keys_detect = []
keys_banned = []
keys_hiprec = []
def handle_nd_tensor(self, key, data):
raise NotImplementedError(
f'Tensor detected that exceeds dims supported by C++ code! ({key} @ {data.shape})'
)
class ModelFlux(ModelTemplate):
arch = 'flux'
keys_detect = [('transformer_blocks.0.attn.norm_added_k.weight',), (
'double_blocks.0.img_attn.proj.weight',)]
keys_banned = ['transformer_blocks.0.attn.norm_added_k.weight']
class ModelSD3(ModelTemplate):
arch = 'sd3'
keys_detect = [('transformer_blocks.0.attn.add_q_proj.weight',), (
'joint_blocks.0.x_block.attn.qkv.weight',)]
keys_banned = ['transformer_blocks.0.attn.add_q_proj.weight']
class ModelAura(ModelTemplate):
arch = 'aura'
keys_detect = [('double_layers.3.modX.1.weight',), (
'joint_transformer_blocks.3.ff_context.out_projection.weight',)]
keys_banned = [
'joint_transformer_blocks.3.ff_context.out_projection.weight']
class ModelHyVid(ModelTemplate):
arch = 'hyvid'
keys_detect = [('double_blocks.0.img_attn_proj.weight',
'txt_in.individual_token_refiner.blocks.1.self_attn_qkv.weight')]
def handle_nd_tensor(self, key, data):
path = f'./fix_5d_tensors_{self.arch}.pt'
if os.path.isfile(path):
raise RuntimeError(f'5D tensor fix file already exists! {path}')
fsd = {key: data}
tqdm.write(
f'5D key found in state dict! Manual fix required! - {key} {data.shape}'
)
torch.save(fsd, path)
class ModelWan(ModelHyVid):
arch = 'wan'
keys_detect = [('blocks.0.self_attn.norm_q.weight',
'text_embedding.2.weight', 'head.modulation')]
class ModelLTXV(ModelTemplate):
arch = 'ltxv'
keys_detect = [('adaln_single.emb.timestep_embedder.linear_2.weight',
'transformer_blocks.27.scale_shift_table',
'caption_projection.linear_2.weight')]
keys_hiprec = ['scale_shift_table']
class ModelSDXL(ModelTemplate):
arch = 'sdxl'
shape_fix = True
keys_detect = [('down_blocks.0.downsamplers.0.conv.weight',
'add_embedding.linear_1.weight'), ('input_blocks.3.0.op.weight',
'input_blocks.6.0.op.weight', 'output_blocks.2.2.conv.weight',
'output_blocks.5.2.conv.weight'), ('label_emb.0.0.weight',)]
class ModelSD1(ModelTemplate):
arch = 'sd1'
shape_fix = True
keys_detect = [('down_blocks.0.downsamplers.0.conv.weight',), (
'input_blocks.3.0.op.weight', 'input_blocks.6.0.op.weight',
'input_blocks.9.0.op.weight', 'output_blocks.2.1.conv.weight',
'output_blocks.5.2.conv.weight', 'output_blocks.8.2.conv.weight')]
arch_list = [ModelFlux, ModelSD3, ModelAura, ModelLTXV, ModelHyVid,
ModelWan, ModelSDXL, ModelSD1]
def is_model_arch(model, state_dict):
matched = False
invalid = False
for match_list in model.keys_detect:
if all(key in state_dict for key in match_list):
matched = True
invalid = any(key in state_dict for key in model.keys_banned)
break
assert not invalid, 'Model architecture not allowed for conversion! (i.e. reference VS diffusers format)'
return matched
def detect_arch(state_dict):
model_arch = None
for arch in arch_list:
if is_model_arch(arch, state_dict):
model_arch = arch()
break
assert model_arch is not None, 'Unknown model architecture!'
return model_arch
def load_state_dict(path):
if any(path.endswith(x) for x in ['.ckpt', '.pt', '.bin', '.pth']):
state_dict = torch.load(path, map_location='cpu', weights_only=True)
for subkey in ['model', 'module']:
if subkey in state_dict:
state_dict = state_dict[subkey]
break
if len(state_dict) < 20:
raise RuntimeError(f'pt subkey load failed: {state_dict.keys()}')
else:
state_dict = load_file(path)
prefix = None
for pfx in ['model.diffusion_model.', 'model.']:
if any([x.startswith(pfx) for x in state_dict.keys()]):
prefix = pfx
break
sd = {}
for k, v in state_dict.items():
if prefix and prefix not in k:
continue
if prefix:
k = k.replace(prefix, '')
sd[k] = v
return sd
def load_model(path):
state_dict = load_state_dict(path)
model_arch = detect_arch(state_dict)
print(f'* Architecture detected from input: {model_arch.arch}')
writer = GGUFWriter(path=None, arch=model_arch.arch)
return writer, state_dict, model_arch
def handle_tensors(args, writer, state_dict, model_arch):
name_lengths = tuple(sorted(((key, len(key)) for key in state_dict.keys
()), key=lambda item: item[1], reverse=True))
if not name_lengths:
return
max_name_len = name_lengths[0][1]
if max_name_len > MAX_TENSOR_NAME_LENGTH:
bad_list = ', '.join(f'{key!r} ({namelen})' for key, namelen in
name_lengths if namelen > MAX_TENSOR_NAME_LENGTH)
raise ValueError(
f'Can only handle tensor names up to {MAX_TENSOR_NAME_LENGTH} characters. Tensors exceeding the limit: {bad_list}'
)
for key, data in tqdm(state_dict.items()):
old_dtype = data.dtype
if data.dtype == torch.bfloat16:
data = data.to(torch.float32).numpy()
elif data.dtype in [getattr(torch, 'float8_e4m3fn', '_invalid'),
getattr(torch, 'float8_e5m2', '_invalid')]:
data = data.to(torch.float16).numpy()
else:
data = data.numpy()
n_dims = len(data.shape)
data_shape = data.shape
data_qtype = getattr(GGMLQuantizationType, 'BF16' if old_dtype ==
torch.bfloat16 else 'F16')
if len(data.shape) > MAX_TENSOR_DIMS:
model_arch.handle_nd_tensor(key, data)
continue
n_params = 1
for dim_size in data_shape:
n_params *= dim_size
if old_dtype in (torch.float32, torch.bfloat16):
if n_dims == 1:
data_qtype = GGMLQuantizationType.F32
elif n_params <= QUANTIZATION_THRESHOLD:
data_qtype = GGMLQuantizationType.F32
elif any(x in key for x in model_arch.keys_hiprec):
data_qtype = GGMLQuantizationType.F32
if (model_arch.shape_fix and n_dims > 1 and n_params >=
REARRANGE_THRESHOLD and (n_params / 256).is_integer() and not (
data.shape[-1] / 256).is_integer()):
orig_shape = data.shape
data = data.reshape(n_params // 256, 256)
writer.add_array(f'comfy.gguf.orig_shape.{key}', tuple(int(dim) for
dim in orig_shape))
try:
data = quantize(data, data_qtype)
except (AttributeError, QuantError) as e:
tqdm.write(f'falling back to F16: {e}')
data_qtype = GGMLQuantizationType.F16
data = quantize(data, data_qtype)
new_name = key
shape_str = f"{{{', '.join(str(n) for n in reversed(data.shape))}}}"
tqdm.write(
f"{f'%-{max_name_len + 4}s' % f'{new_name}'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}"
)
writer.add_tensor(new_name, data, raw_dtype=data_qtype)
import os
safetensors_files = [file for file in os.listdir() if file.endswith(
'.safetensors')]
if safetensors_files:
print('Safetensors file(s) available. Select which one to convert:')
for index, file_name in enumerate(safetensors_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = safetensors_files[choice_index]
print(f'Model file: {selected_file} is selected!')
path = selected_file
writer, state_dict, model_arch = load_model(path)
writer.add_quantization_version(GGML_QUANT_VERSION)
if next(iter(state_dict.values())).dtype == torch.bfloat16:
out_path = f'{os.path.splitext(path)[0]}-bf16.gguf'
writer.add_file_type(LlamaFileType.MOSTLY_BF16)
else:
out_path = f'{os.path.splitext(path)[0]}-f16.gguf'
writer.add_file_type(LlamaFileType.MOSTLY_F16)
if os.path.isfile(out_path):
input('Output exists enter to continue or ctrl+c to abort!')
handle_tensors(path, writer, state_dict, model_arch)
writer.write_header_to_file(path=out_path)
writer.write_kv_data_to_file()
writer.write_tensors_to_file(progress=True)
writer.close()
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No safetensors files are available in the current directory.')
input('--- Press ENTER To Exit ---')
gguf_connector/t8.py
+199
View File
@@ -0,0 +1,199 @@
import torch
from .writer import GGUFWriter, GGMLQuantizationType
from .const import GGML_QUANT_VERSION, LlamaFileType
from safetensors.torch import load_file
from tqdm import tqdm
QUANTIZATION_THRESHOLD = 1024
REARRANGE_THRESHOLD = 512
MAX_TENSOR_NAME_LENGTH = 127
MAX_TENSOR_DIMS = 4
class ModelTemplate:
arch = 'invalid'
shape_fix = False
keys_detect = []
keys_banned = []
keys_hiprec = []
def handle_nd_tensor(self, key, data):
raise NotImplementedError(
f'Tensor detected that exceeds dims supported by C++ code! ({key} @ {data.shape})'
)
class ModelFlux(ModelTemplate):
arch = 'flux'
keys_detect = [('transformer_blocks.0.attn.norm_added_k.weight',), (
'double_blocks.0.img_attn.proj.weight',)]
keys_banned = ['transformer_blocks.0.attn.norm_added_k.weight']
class ModelSD3(ModelTemplate):
arch = 'sd3'
keys_detect = [('transformer_blocks.0.attn.add_q_proj.weight',), (
'joint_blocks.0.x_block.attn.qkv.weight',)]
keys_banned = ['transformer_blocks.0.attn.add_q_proj.weight']
class ModelAura(ModelTemplate):
arch = 'aura'
keys_detect = [('double_layers.3.modX.1.weight',), (
'joint_transformer_blocks.3.ff_context.out_projection.weight',)]
keys_banned = [
'joint_transformer_blocks.3.ff_context.out_projection.weight']
class ModelHiDream(ModelTemplate):
arch = 'hidream'
keys_detect = [('caption_projection.0.linear.weight',
'double_stream_blocks.0.block.ff_i.shared_experts.w3.weight')]
keys_hiprec = ['.ff_i.gate.weightimg_emb.emb_pos']
class ModelHyVid(ModelTemplate):
arch = 'hyvid'
keys_detect = [('double_blocks.0.img_attn_proj.weight',
'txt_in.individual_token_refiner.blocks.1.self_attn_qkv.weight')]
def handle_nd_tensor(self, key, data):
path = f'./fix_5d_tensors_{self.arch}.safetensors'
if os.path.isfile(path):
raise RuntimeError(f'5D tensor fix file already exists! {path}')
fsd = {key: torch.from_numpy(data)}
tqdm.write(
f'5D key found in state dict! Manual fix required! - {key} {data.shape}'
)
torch.save(fsd, path)
class ModelWan(ModelHyVid):
arch = 'wan'
keys_detect = [('blocks.0.self_attn.norm_q.weight',
'text_embedding.2.weight', 'head.modulation')]
class ModelLTXV(ModelTemplate):
arch = 'ltxv'
keys_detect = [('adaln_single.emb.timestep_embedder.linear_2.weight',
'transformer_blocks.27.scale_shift_table',
'caption_projection.linear_2.weight')]
keys_hiprec = ['scale_shift_table']
class ModelSDXL(ModelTemplate):
arch = 'sdxl'
shape_fix = True
keys_detect = [('down_blocks.0.downsamplers.0.conv.weight',
'add_embedding.linear_1.weight'), ('input_blocks.3.0.op.weight',
'input_blocks.6.0.op.weight', 'output_blocks.2.2.conv.weight',
'output_blocks.5.2.conv.weight'), ('label_emb.0.0.weight',)]
class ModelSD1(ModelTemplate):
arch = 'sd1'
shape_fix = True
keys_detect = [('down_blocks.0.downsamplers.0.conv.weight',), (
'input_blocks.3.0.op.weight', 'input_blocks.6.0.op.weight',
'input_blocks.9.0.op.weight', 'output_blocks.2.1.conv.weight',
'output_blocks.5.2.conv.weight', 'output_blocks.8.2.conv.weight')]
arch_list = [ModelFlux, ModelSD3, ModelAura, ModelHiDream, ModelLTXV,
ModelHyVid, ModelWan, ModelSDXL, ModelSD1]
def is_model_arch(model, state_dict):
matched = False
invalid = False
for match_list in model.keys_detect:
if all(key in state_dict for key in match_list):
matched = True
invalid = any(key in state_dict for key in model.keys_banned)
break
assert not invalid, 'Model architecture not allowed for conversion! (i.e. reference VS diffusers format)'
return matched
def detect_arch(state_dict):
model_arch = None
for arch in arch_list:
if is_model_arch(arch, state_dict):
model_arch = arch()
break
assert model_arch is not None, 'Unknown model architecture!'
return model_arch
def load_state_dict(path):
if any(path.endswith(x) for x in ['.ckpt', '.pt', '.bin', '.pth']):
state_dict = torch.load(path, map_location='cpu', weights_only=True)
for subkey in ['model', 'module']:
if subkey in state_dict:
state_dict = state_dict[subkey]
break
if len(state_dict) < 20:
raise RuntimeError(f'pt subkey load failed: {state_dict.keys()}')
else:
state_dict = load_file(path)
prefix = None
for pfx in ['model.diffusion_model.', 'model.']:
if any([x.startswith(pfx) for x in state_dict.keys()]):
prefix = pfx
break
sd = {}
for k, v in state_dict.items():
if prefix and prefix not in k:
continue
if prefix:
k = k.replace(prefix, '')
sd[k] = v
return sd
def load_model(path):
state_dict = load_state_dict(path)
model_arch = detect_arch(state_dict)
print(f'* Architecture detected from input: {model_arch.arch}')
writer = GGUFWriter(path=None, arch=model_arch.arch)
return writer, state_dict, model_arch
def handle_tensors(args, writer, state_dict, model_arch):
name_lengths = tuple(sorted(((key, len(key)) for key in state_dict.keys
()), key=lambda item: item[1], reverse=True))
if not name_lengths:
return
max_name_len = name_lengths[0][1]
if max_name_len > MAX_TENSOR_NAME_LENGTH:
bad_list = ', '.join(f'{key!r} ({namelen})' for key, namelen in
name_lengths if namelen > MAX_TENSOR_NAME_LENGTH)
raise ValueError(
f'Can only handle tensor names up to {MAX_TENSOR_NAME_LENGTH} characters. Tensors exceeding the limit: {bad_list}'
)
for key, data in tqdm(state_dict.items()):
old_dtype = data.dtype
data = data.to(torch.float32).numpy()
n_dims = len(data.shape)
data_shape = data.shape
data_qtype = getattr(GGMLQuantizationType, 'BF16' if old_dtype ==
torch.bfloat16 else 'F16')
if len(data.shape) > MAX_TENSOR_DIMS:
model_arch.handle_nd_tensor(key, data)
continue
n_params = 1
for dim_size in data_shape:
n_params *= dim_size
if old_dtype in (torch.float32, torch.bfloat16):
if n_dims == 1:
data_qtype = GGMLQuantizationType.F32
elif n_params <= QUANTIZATION_THRESHOLD:
data_qtype = GGMLQuantizationType.F32
elif any(x in key for x in model_arch.keys_hiprec):
data_qtype = GGMLQuantizationType.F32
new_name = key
data_qtype = GGMLQuantizationType.F32
shape_str = f"{{{', '.join(str(n) for n in reversed(data.shape))}}}"
tqdm.write(
f"{f'%-{max_name_len + 4}s' % f'{new_name}'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}"
)
writer.add_tensor(new_name, data, raw_dtype=data_qtype)
import os
safetensors_files = [file for file in os.listdir() if file.endswith(
'.safetensors')]
if safetensors_files:
print('Safetensors file(s) available. Select which one to convert:')
for index, file_name in enumerate(safetensors_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = safetensors_files[choice_index]
print(f'Model file: {selected_file} is selected!')
path = selected_file
writer, state_dict, model_arch = load_model(path)
writer.add_quantization_version(GGML_QUANT_VERSION)
if next(iter(state_dict.values())).dtype == torch.bfloat16:
out_path = f'{os.path.splitext(path)[0]}-f32.gguf'
writer.add_file_type(LlamaFileType.MOSTLY_BF16)
else:
out_path = f'{os.path.splitext(path)[0]}-f32.gguf'
writer.add_file_type(LlamaFileType.MOSTLY_F16)
if os.path.isfile(out_path):
input('Output exists enter to continue or ctrl+c to abort!')
handle_tensors(path, writer, state_dict, model_arch)
writer.write_header_to_file(path=out_path)
writer.write_kv_data_to_file()
writer.write_tensors_to_file(progress=True)
writer.close()
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No safetensors files are available in the current directory.')
input('--- Press ENTER To Exit ---')
gguf_connector/t9.py
+79
View File
@@ -0,0 +1,79 @@
import torch
from .writer import GGUFWriter, GGMLQuantizationType
from .const import GGML_QUANT_VERSION, LlamaFileType
from safetensors.torch import load_file
from tqdm import tqdm
import numpy as np
MAX_TENSOR_NAME_LENGTH = 127
def load_state_dict(path):
state_dict = load_file(path)
return {k: v for k, v in state_dict.items()}
def load_model(path, model_arch):
state_dict = load_state_dict(path)
writer = GGUFWriter(path=None, arch=model_arch)
return writer, state_dict, model_arch
def is_tensor_valid(data, key):
if data is None:
print(f"[WARNING] Skipping tensor '{key}': Empty or NULL tensor.")
return False
if np.any(np.isnan(data)) or np.any(np.isinf(data)):
print(f"[WARNING] Skipping tensor '{key}': Contains NaN or Inf values."
)
return False
return True
def handle_tensors(writer, state_dict):
name_lengths = [(key, len(key)) for key in state_dict.keys()]
if not name_lengths:
return
max_name_len = max(name_lengths, key=lambda x: x[1])[1]
for key, data in tqdm(state_dict.items(), desc='Processing Tensors'):
old_dtype = data.dtype
print(
f'[INFO] Processing: {key} | Original dtype: {old_dtype} | Shape: {data.shape}'
)
data = data.to(torch.float32).numpy()
if not is_tensor_valid(data, key):
continue
data_qtype = GGMLQuantizationType.F32
shape_str = f"{{{', '.join(map(str, reversed(data.shape)))}}}"
print(
f'[INFO] Writing: {key.ljust(max_name_len)} | {old_dtype} -> {data_qtype.name} | Shape: {shape_str}'
)
writer.add_tensor(key, data, raw_dtype=data_qtype)
import os
safetensors_files = [file for file in os.listdir() if file.endswith(
'.safetensors')]
if safetensors_files:
print('Safetensors file(s) available. Select which one to convert:')
for index, file_name in enumerate(safetensors_files, start=1):
print(f'{index}. {file_name}')
choice = input(f'Enter your choice (1 to {len(safetensors_files)}): ')
try:
choice_index = int(choice) - 1
selected_file = safetensors_files[choice_index]
print(f'Model file: {selected_file} is selected!')
path = selected_file
ask = input('Assign a name for the model (Y/n)? ')
if ask.lower() == 'y':
given = input('Enter a model name: ')
else:
given = None
writer, state_dict, _ = load_model(path, given)
writer.add_quantization_version(GGML_QUANT_VERSION)
out_path = f'{os.path.splitext(path)[0]}-f32.gguf'
writer.add_file_type(LlamaFileType.MOSTLY_F16)
if os.path.isfile(out_path):
input(
'Output file exists. Press Enter to overwrite or Ctrl+C to abort.'
)
handle_tensors(writer, state_dict)
writer.write_header_to_file(path=out_path)
writer.write_kv_data_to_file()
writer.write_tensors_to_file(progress=True)
writer.close()
print(f'Conversion completed: {out_path}')
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No safetensors files are available in the current directory.')
input('--- Press ENTER To Exit ---')
gguf_connector/tkn.py
+60
View File
@@ -0,0 +1,60 @@
import torch
from .reader import GGUFReader, GGUFValueType
def get_field(reader, field_name, field_type):
field = reader.get_field(field_name)
if field is None:
return None
elif field_type == str:
if len(field.types) != 1 or field.types[0] != GGUFValueType.STRING:
raise TypeError(
f'Bad type for GGUF {field_name} key: expected string, got {field.types!r}'
)
return str(field.parts[field.data[-1]], encoding='utf-8')
elif field_type in [int, float, bool]:
return field_type(field.parts[field.data[-1]])
else:
raise TypeError(f'Unknown field type {field_type}')
def get_list_field(reader, field_name, field_type):
field = reader.get_field(field_name)
if field is None:
return None
elif field_type == str:
return tuple(str(field.parts[part_idx], encoding='utf-8') for
part_idx in field.data)
elif field_type in [int, float, bool]:
return tuple(field_type(field.parts[part_idx][0]) for part_idx in
field.data)
else:
raise TypeError(f'Unknown field type {field_type}')
def tokenizer_builder(path):
print(f'Attempting to rebuild sentencepiece tokenizer from metadata..')
try:
from sentencepiece import sentencepiece_model_pb2 as model
except ImportError:
raise ImportError('protobuf is required; pip install protobuf')
spm = model.ModelProto()
reader = GGUFReader(path)
spm.trainer_spec.model_type == 1
spm.normalizer_spec.add_dummy_prefix = get_field(reader,
'tokenizer.ggml.add_space_prefix', bool)
tokens = get_list_field(reader, 'tokenizer.ggml.tokens', str)
scores = get_list_field(reader, 'tokenizer.ggml.scores', float)
toktypes = get_list_field(reader, 'tokenizer.ggml.token_type', int)
for idx, (token, score, toktype) in enumerate(zip(tokens, scores, toktypes)
):
piece = spm.SentencePiece()
piece.piece = token
piece.score = score
piece.type = toktype
spm.pieces.append(piece)
spm.trainer_spec.byte_fallback = True
spm.trainer_spec.vocab_size = len(tokens)
spm.trainer_spec.max_sentence_length = 4096
spm.trainer_spec.eos_id = get_field(reader,
'tokenizer.ggml.eos_token_id', int)
spm.trainer_spec.pad_id = get_field(reader,
'tokenizer.ggml.padding_token_id', int)
print(
f'Rebuilt tokenizer successfully with vocab size of {len(spm.pieces)}')
del reader
return torch.ByteTensor(list(spm.SerializeToString()))
gguf_connector/u.py
+1
View File
@@ -0,0 +1 @@
from gguf_cutter import downloader
gguf_connector/v.py
+48
View File
@@ -0,0 +1,48 @@
import os
gguf_files = [file for file in os.listdir() if file.endswith('.gguf')]
if gguf_files:
print('GGUF file(s) available. Select which one to use as Clip Handler:')
for index, file_name in enumerate(gguf_files, start=1):
print(f'{index}. {file_name}')
choice1 = input(f'Enter your choice (1 to {len(gguf_files)}): ')
try:
choice_index = int(choice1) - 1
selected_file = gguf_files[choice_index]
print(f'Model file: {selected_file} is selected!')
clip_model_path = selected_file
from llama_cpp.llama_chat_format import Llava15ChatHandler
chat_handler = Llava15ChatHandler(clip_model_path)
print(
'GGUF file(s) available. Select which one to use as Vision Model:')
for index, file_name in enumerate(gguf_files, start=1):
print(f'{index}. {file_name}')
choice2 = input(f'Enter your choice (1 to {len(gguf_files)}): ')
try:
choice_index = int(choice2) - 1
selected_file = gguf_files[choice_index]
print(f'Model file: {selected_file} is selected!')
model_path = selected_file
from llama_cpp import Llama
llm = Llama(model_path=model_path, chat_handler=chat_handler,
n_ctx=2048)
while True:
ask = input('Provide a picture URL (Q for quit): ')
if ask.lower() == 'q':
break
from rich.progress import Progress
with Progress(transient=True) as progress:
task = progress.add_task('Processing', total=None)
response = llm.create_chat_completion(messages=[{'role':
'user', 'content': [{'type': 'text', 'text':
"What's in this image?"}, {'type': 'image_url',
'image_url': {'url': ask}}]}])
print('\nOutput:' + response['choices'][0]['message'][
'content'] + '\n')
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
except (ValueError, IndexError):
print('Invalid choice. Please enter a valid number.')
else:
print('No GGUF files are available in the current directory.')
input('--- Press ENTER To Exit ---')
print('Goodbye!')
gguf_connector/v2.py
+56
View File
@@ -0,0 +1,56 @@
import torch
import gradio as gr
from transformers import UMT5EncoderModel
from diffusers import WanPipeline, AutoencoderKLWan, WanTransformer3DModel, GGUFQuantizationConfig
from diffusers.utils import export_to_video
model_path = (
'https://huggingface.co/calcuis/wan-gguf/blob/main/wan2.1_t2v_1.3b-q8_0.gguf'
)
transformer = WanTransformer3DModel.from_single_file(model_path,
quantization_config=GGUFQuantizationConfig(compute_dtype=torch.bfloat16
), torch_dtype=torch.bfloat16)
text_encoder = UMT5EncoderModel.from_pretrained('chatpig/umt5xxl-encoder-gguf',
gguf_file='umt5xxl-encoder-q4_0.gguf', torch_dtype=torch.bfloat16)
vae = AutoencoderKLWan.from_pretrained('callgg/t2v-decoder', subfolder=
'vae', torch_dtype=torch.float32)
pipe = WanPipeline.from_pretrained('callgg/t2v-decoder', transformer=
transformer, text_encoder=text_encoder, vae=vae, torch_dtype=torch.bfloat16
)
pipe.enable_model_cpu_offload()
def generate_video(prompt, negative_prompt, width, height, num_frames,
num_inference_steps, fps):
video = pipe(prompt=prompt, negative_prompt=negative_prompt, width=
width, height=height, num_frames=num_frames, num_inference_steps=
num_inference_steps).frames[0]
export_to_video(video, 'output.mp4', fps=fps)
return 'output.mp4'
sample_prompts = [
'A cat and a dog baking a cake together in a kitchen. The cat is carefully measuring flour, while the dog is stirring the batter with a wooden spoon. The kitchen is cozy, with sunlight streaming through the window.'
,
'A pig moving quickly in a beautiful winter scenery nature trees sunset tracking camera.'
]
sample_prompts = [[x] for x in sample_prompts]
block = gr.Blocks(title='gguf').queue()
with block:
gr.Markdown('## 🎥 Video 2')
with gr.Row():
prompt_input = gr.Textbox(label='Prompt', placeholder=
'Enter your prompt here (or click Sample Prompt)', value='')
neg_prompt_input = gr.Textbox(label='Negative Prompt', value=
'blurry ugly bad', visible=False)
quick_prompts = gr.Dataset(samples=sample_prompts, label=
'Sample Prompt', samples_per_page=1000, components=[prompt_input])
quick_prompts.click(lambda x: x[0], inputs=[quick_prompts], outputs
=prompt_input, show_progress=False, queue=False)
with gr.Row():
width_input = gr.Number(label='Width', value=512)
height_input = gr.Number(label='Height', value=512)
num_frames_input = gr.Number(label='Number of Frames', value=25)
num_steps_input = gr.Number(label='Inference Steps', value=25)
fps_input = gr.Number(label='FPS', value=16)
generate_btn = gr.Button('Generate Video')
output_video = gr.Video(label='Generated Video')
generate_btn.click(fn=generate_video, inputs=[prompt_input,
neg_prompt_input, width_input, height_input, num_frames_input,
num_steps_input, fps_input], outputs=output_video)
block.launch()
gguf_connector/vg.py
+52
View File
@@ -0,0 +1,52 @@
import torch
import gradio as gr
from transformers import T5EncoderModel
from diffusers import LTXPipeline, GGUFQuantizationConfig, LTXVideoTransformer3DModel
from diffusers.utils import export_to_video
model_path = (
'https://huggingface.co/calcuis/ltxv-gguf/blob/main/ltxv-2b-0.9.6-distilled-fp32-q8_0.gguf'
)
transformer = LTXVideoTransformer3DModel.from_single_file(model_path,
quantization_config=GGUFQuantizationConfig(compute_dtype=torch.bfloat16
), torch_dtype=torch.bfloat16)
text_encoder = T5EncoderModel.from_pretrained('calcuis/ltxv-gguf',
gguf_file='t5xxl_fp16-q4_0.gguf', torch_dtype=torch.bfloat16)
pipe = LTXPipeline.from_pretrained('callgg/ltxv0.9.6-decoder', text_encoder
=text_encoder, transformer=transformer, torch_dtype=torch.bfloat16).to(
'cuda')
def generate_video(prompt, negative_prompt, width, height, num_frames,
num_inference_steps, fps):
video = pipe(prompt=prompt, negative_prompt=negative_prompt, width=
width, height=height, num_frames=num_frames, num_inference_steps=
num_inference_steps).frames[0]
export_to_video(video, 'output.mp4', fps=fps)
return 'output.mp4'
sample_prompts = [
'A drone quickly rises through a bank of morning fog, revealing a pristine alpine lake surrounded by snow-capped mountains. The camera glides forward over the glassy water, capturing perfect reflections of the peaks. As it continues, the perspective shifts to reveal a lone wooden cabin with a curl of smoke from its chimney, nestled among tall pines at the lake edge. The final shot tracks upward rapidly, transitioning from intimate to epic as the full mountain range comes into view, bathed in the golden light of sunrise breaking through scattered clouds.'
]
sample_prompts = [[x] for x in sample_prompts]
block = gr.Blocks(title='gguf').queue()
with block:
gr.Markdown('## 🎥 Video Generator (t2v)')
with gr.Row():
prompt_input = gr.Textbox(label='Prompt', placeholder=
'Enter your prompt here (or click Sample Prompt)', value='')
neg_prompt_input = gr.Textbox(label='Negative Prompt', value=
'low quality, worst quality, deformed, distorted, disfigured, motion smear, motion artifacts, fused fingers, bad anatomy, weird hand, ugly'
, visible=False)
quick_prompts = gr.Dataset(samples=sample_prompts, label=
'Sample Prompt', samples_per_page=1000, components=[prompt_input])
quick_prompts.click(lambda x: x[0], inputs=[quick_prompts], outputs
=prompt_input, show_progress=False, queue=False)
with gr.Row():
width_input = gr.Number(label='Width', value=512)
height_input = gr.Number(label='Height', value=512)
num_frames_input = gr.Number(label='Number of Frames', value=25)
num_steps_input = gr.Number(label='Inference Steps', value=25)
fps_input = gr.Number(label='FPS', value=24)
generate_btn = gr.Button('Generate Video')
output_video = gr.Video(label='Generated Video')
generate_btn.click(fn=generate_video, inputs=[prompt_input,
neg_prompt_input, width_input, height_input, num_frames_input,
num_steps_input, fps_input], outputs=output_video)
block.launch()
gguf_connector/vg2.py
+57
View File
@@ -0,0 +1,57 @@
import torch
import gradio as gr
from diffusers.pipelines.ltx.pipeline_ltx_condition import LTXConditionPipeline, LTXVideoCondition
from diffusers import GGUFQuantizationConfig, LTXVideoTransformer3DModel
from diffusers.utils import export_to_video
from transformers import T5EncoderModel
model_path = (
'https://huggingface.co/calcuis/ltxv-gguf/blob/main/ltxv-2b-0.9.6-distilled-fp32-q8_0.gguf'
)
transformer = LTXVideoTransformer3DModel.from_single_file(model_path,
quantization_config=GGUFQuantizationConfig(compute_dtype=torch.bfloat16
), torch_dtype=torch.bfloat16)
text_encoder = T5EncoderModel.from_pretrained('calcuis/ltxv-gguf',
gguf_file='t5xxl_fp16-q4_0.gguf', torch_dtype=torch.bfloat16)
pipe = LTXConditionPipeline.from_pretrained('callgg/ltxv0.9.6-decoder',
transformer=transformer, text_encoder=text_encoder, torch_dtype=torch.
bfloat16).to('cuda')
def generate_video(input_image, prompt, negative_prompt, width, height,
num_frames, num_inference_steps, fps):
image = input_image.convert('RGB')
condition1 = LTXVideoCondition(image=image, frame_index=0)
generator = torch.Generator('cuda').manual_seed(0)
video = pipe(conditions=[condition1], prompt=prompt, negative_prompt=
negative_prompt, width=width, height=height, num_frames=num_frames,
num_inference_steps=num_inference_steps, generator=generator).frames[0]
export_to_video(video, 'output.mp4', fps=fps)
return 'output.mp4'
sample_prompts = [
'The girl dances gracefully, with clear movements, full of charm.',
'A character doing some simple body movements.']
sample_prompts = [[x] for x in sample_prompts]
block = gr.Blocks(title='gguf').queue()
with block:
gr.Markdown('## 🎥 Video Generator (i2v)')
with gr.Row():
input_image = gr.Image(type='pil', label='Upload Image')
prompt = gr.Textbox(label='Prompt', placeholder=
'Enter your prompt here (or click Sample Prompt)', value='')
negative_prompt = gr.Textbox(label='Negative Prompt', value=
'worst quality, inconsistent motion, blurry, jittery, distorted',
visible=False)
quick_prompts = gr.Dataset(samples=sample_prompts, label=
'Sample Prompt', samples_per_page=1000, components=[prompt])
quick_prompts.click(lambda x: x[0], inputs=[quick_prompts], outputs=
prompt, show_progress=False, queue=False)
with gr.Row():
width = gr.Number(label='Width', value=512)
height = gr.Number(label='Height', value=512)
num_frames = gr.Number(label='Num Frames', value=25)
num_inference_steps = gr.Number(label='Num Inference Steps', value=25)
fps = gr.Number(label='FPS', value=24)
generate_btn = gr.Button('Generate')
output_video = gr.Video(label='Generated Video')
generate_btn.click(fn=generate_video, inputs=[input_image, prompt,
negative_prompt, width, height, num_frames, num_inference_steps,
fps], outputs=output_video)
block.launch()
gguf_connector/w.py
+3
View File
@@ -0,0 +1,3 @@
import webbrowser
print('activating the default browser...')
webbrowser.open('https://gguf.us')
gguf_connector/writer.py
+801
View File
@@ -0,0 +1,801 @@
from __future__ import annotations
import logging, os, shutil, struct, tempfile
from dataclasses import dataclass
from enum import Enum, auto
from math import prod
from pathlib import Path
from io import BufferedWriter
from typing import IO, Any, Sequence, Mapping
from string import ascii_letters, digits
import numpy as np
from .const import GGUF_DEFAULT_ALIGNMENT, GGUF_MAGIC, GGUF_VERSION, GGMLQuantizationType, GGUFEndian, GGUFValueType, Keys, RopeScalingType, PoolingType, TokenType, ExpertGatingFuncType
from .quant import quant_shape_from_byte_shape
logger = logging.getLogger(__name__)
SHARD_NAME_FORMAT = '{:s}-{:05d}-of-{:05d}.gguf'
@dataclass
class TensorInfo:
shape: Sequence[int]
dtype: GGMLQuantizationType
nbytes: int
tensor: np.ndarray[Any, Any] | None = None
@dataclass
class GGUFValue:
value: Any
type: GGUFValueType
class WriterState(Enum):
NO_FILE = auto()
EMPTY = auto()
HEADER = auto()
KV_DATA = auto()
TI_DATA = auto()
WEIGHTS = auto()
class GGUFWriter:
fout: list[BufferedWriter] | None
path: Path | None
temp_file: tempfile.SpooledTemporaryFile[bytes] | None
tensors: list[dict[str, TensorInfo]]
kv_data: list[dict[str, GGUFValue]]
state: WriterState
_simple_value_packing = {GGUFValueType.UINT8: 'B', GGUFValueType.INT8:
'b', GGUFValueType.UINT16: 'H', GGUFValueType.INT16: 'h',
GGUFValueType.UINT32: 'I', GGUFValueType.INT32: 'i', GGUFValueType.
FLOAT32: 'f', GGUFValueType.UINT64: 'Q', GGUFValueType.INT64: 'q',
GGUFValueType.FLOAT64: 'd', GGUFValueType.BOOL: '?'}
def __init__(self, path, arch, use_temp_file=False, endianess=
GGUFEndian.LITTLE, split_max_tensors=0, split_max_size=0, dry_run=
False, small_first_shard=False):
self.fout = None
self.path = Path(path) if path else None
self.arch = arch
self.endianess = endianess
self.data_alignment = GGUF_DEFAULT_ALIGNMENT
self.use_temp_file = use_temp_file
self.temp_file = None
self.tensors = [{}]
self.kv_data = [{}]
self.split_max_tensors = split_max_tensors
self.split_max_size = split_max_size
self.dry_run = dry_run
self.small_first_shard = small_first_shard
logger.info('gguf: This GGUF file is for {0} Endian only'.format(
'Big' if self.endianess == GGUFEndian.BIG else 'Little'))
self.state = WriterState.NO_FILE
if self.small_first_shard:
self.tensors.append({})
self.add_architecture()
def get_total_parameter_count(self):
total_params = 0
shared_params = 0
expert_params = 0
expert_sum = 0
n_expert_tensors = 0
last_lora_a: tuple[str, TensorInfo] | None = None
for tensors in self.tensors:
for name, info in tensors.items():
shape = info.shape
if name.endswith('.lora_a'):
last_lora_a = name, info
continue
elif name.endswith('.lora_b'):
if last_lora_a is None or last_lora_a[0] != name[:-1
] + 'a':
logger.warning(
"can't measure LoRA size correctly, tensor order is unusual"
)
return 0, 0, 0, 0
else:
shape = *shape[:-1], last_lora_a[1].shape[-1]
size = prod(shape)
if '_exps.' in name:
expert_params += size // shape[-3]
expert_sum += shape[-3]
n_expert_tensors += 1
else:
shared_params += size
total_params += size
expert_count = (expert_sum // n_expert_tensors if n_expert_tensors >
0 else 0)
if last_lora_a is not None:
total_params = -total_params
return total_params, shared_params, expert_params, expert_count
def format_shard_names(self, path):
if len(self.tensors) == 1:
return [path]
return [path.with_name(SHARD_NAME_FORMAT.format(path.stem, i + 1,
len(self.tensors))) for i in range(len(self.tensors))]
def open_output_file(self, path=None):
if self.state is WriterState.EMPTY and self.fout is not None and (
path is None or path == self.path):
return
if self.state is not WriterState.NO_FILE:
raise ValueError(
f'Expected output file to be not yet opened, got {self.state}')
if path is not None:
self.path = path
if self.path is not None:
filenames = self.print_plan()
self.fout = [open(filename, 'wb') for filename in filenames]
self.state = WriterState.EMPTY
def print_plan(self):
logger.info('Writing the following files:')
assert self.path is not None
filenames = self.format_shard_names(self.path)
assert len(filenames) == len(self.tensors)
for name, tensors in zip(filenames, self.tensors):
logger.info(
f'{name}: n_tensors = {len(tensors)}, total_size = {GGUFWriter.format_n_bytes_to_str(sum(ti.nbytes for ti in tensors.values()))}'
)
if self.dry_run:
logger.info('Dry run, not writing files')
for name in filenames:
print(name)
exit()
return filenames
def add_shard_kv_data(self):
if len(self.tensors) == 1:
return
total_tensors = sum(len(t) for t in self.tensors)
assert self.fout is not None
total_splits = len(self.fout)
self.kv_data.extend({} for _ in range(len(self.kv_data), total_splits))
for i, kv_data in enumerate(self.kv_data):
kv_data[Keys.Split.LLM_KV_SPLIT_NO] = GGUFValue(i,
GGUFValueType.UINT16)
kv_data[Keys.Split.LLM_KV_SPLIT_COUNT] = GGUFValue(total_splits,
GGUFValueType.UINT16)
kv_data[Keys.Split.LLM_KV_SPLIT_TENSORS_COUNT] = GGUFValue(
total_tensors, GGUFValueType.INT32)
def write_header_to_file(self, path=None):
if len(self.tensors) == 1 and (self.split_max_tensors != 0 or self.
split_max_size != 0):
logger.warning('Model fails split requirements, not splitting')
self.open_output_file(path)
if self.state is not WriterState.EMPTY:
raise ValueError(
f'Expected output file to be empty, got {self.state}')
assert self.fout is not None
assert len(self.fout) == len(self.tensors)
assert len(self.kv_data) == 1
self.add_shard_kv_data()
for fout, tensors, kv_data in zip(self.fout, self.tensors, self.kv_data
):
fout.write(self._pack('<I', GGUF_MAGIC, skip_pack_prefix=True))
fout.write(self._pack('I', GGUF_VERSION))
fout.write(self._pack('Q', len(tensors)))
fout.write(self._pack('Q', len(kv_data)))
fout.flush()
self.state = WriterState.HEADER
def write_kv_data_to_file(self):
if self.state is not WriterState.HEADER:
raise ValueError(
f'Expected output file to contain the header, got {self.state}'
)
assert self.fout is not None
for fout, kv_data in zip(self.fout, self.kv_data):
kv_bytes = bytearray()
for key, val in kv_data.items():
kv_bytes += self._pack_val(key, GGUFValueType.STRING,
add_vtype=False)
kv_bytes += self._pack_val(val.value, val.type, add_vtype=True)
fout.write(kv_bytes)
self.flush()
self.state = WriterState.KV_DATA
def write_ti_data_to_file(self):
if self.state is not WriterState.KV_DATA:
raise ValueError(
f'Expected output file to contain KV data, got {self.state}')
assert self.fout is not None
for fout, tensors in zip(self.fout, self.tensors):
ti_data = bytearray()
offset_tensor = 0
for name, ti in tensors.items():
ti_data += self._pack_val(name, GGUFValueType.STRING,
add_vtype=False)
n_dims = len(ti.shape)
ti_data += self._pack('I', n_dims)
for j in range(n_dims):
ti_data += self._pack('Q', ti.shape[n_dims - 1 - j])
ti_data += self._pack('I', ti.dtype)
ti_data += self._pack('Q', offset_tensor)
offset_tensor += GGUFWriter.ggml_pad(ti.nbytes, self.
data_alignment)
fout.write(ti_data)
fout.flush()
self.state = WriterState.TI_DATA
def add_key_value(self, key, val, vtype):
if any(key in kv_data for kv_data in self.kv_data):
raise ValueError(f'Duplicated key name {key!r}')
self.kv_data[0][key] = GGUFValue(value=val, type=vtype)
def add_uint8(self, key, val):
self.add_key_value(key, val, GGUFValueType.UINT8)
def add_int8(self, key, val):
self.add_key_value(key, val, GGUFValueType.INT8)
def add_uint16(self, key, val):
self.add_key_value(key, val, GGUFValueType.UINT16)
def add_int16(self, key, val):
self.add_key_value(key, val, GGUFValueType.INT16)
def add_uint32(self, key, val):
self.add_key_value(key, val, GGUFValueType.UINT32)
def add_int32(self, key, val):
self.add_key_value(key, val, GGUFValueType.INT32)
def add_float32(self, key, val):
self.add_key_value(key, val, GGUFValueType.FLOAT32)
def add_uint64(self, key, val):
self.add_key_value(key, val, GGUFValueType.UINT64)
def add_int64(self, key, val):
self.add_key_value(key, val, GGUFValueType.INT64)
def add_float64(self, key, val):
self.add_key_value(key, val, GGUFValueType.FLOAT64)
def add_bool(self, key, val):
self.add_key_value(key, val, GGUFValueType.BOOL)
def add_string(self, key, val):
if not val:
return
self.add_key_value(key, val, GGUFValueType.STRING)
def add_array(self, key, val):
if len(val) == 0:
return
self.add_key_value(key, val, GGUFValueType.ARRAY)
@staticmethod
def ggml_pad(x, n):
return (x + n - 1) // n * n
def add_tensor_info(self, name, tensor_shape, tensor_dtype,
tensor_nbytes, raw_dtype=None):
if self.state is not WriterState.NO_FILE:
raise ValueError(
f'Expected output file to be not yet opened, got {self.state}')
if any(name in tensors for tensors in self.tensors):
raise ValueError(f'Duplicated tensor name {name!r}')
if raw_dtype is None:
if tensor_dtype == np.float16:
dtype = GGMLQuantizationType.F16
elif tensor_dtype == np.float32:
dtype = GGMLQuantizationType.F32
elif tensor_dtype == np.float64:
dtype = GGMLQuantizationType.F64
elif tensor_dtype == np.int8:
dtype = GGMLQuantizationType.I8
elif tensor_dtype == np.int16:
dtype = GGMLQuantizationType.I16
elif tensor_dtype == np.int32:
dtype = GGMLQuantizationType.I32
elif tensor_dtype == np.int64:
dtype = GGMLQuantizationType.I64
else:
raise ValueError(
'Only F16, F32, F64, I8, I16, I32, I64 tensors are supported for now'
)
else:
dtype = raw_dtype
if tensor_dtype == np.uint8:
tensor_shape = quant_shape_from_byte_shape(tensor_shape,
raw_dtype)
if len(self.tensors[-1]) > 0:
if self.split_max_tensors != 0 and len(self.tensors[-1]
) >= self.split_max_tensors or self.split_max_size != 0 and sum(
ti.nbytes for ti in self.tensors[-1].values()
) + tensor_nbytes > self.split_max_size:
self.tensors.append({})
self.tensors[-1][name] = TensorInfo(shape=tensor_shape, dtype=dtype,
nbytes=tensor_nbytes)
def add_tensor(self, name, tensor, raw_shape=None, raw_dtype=None):
if self.endianess == GGUFEndian.BIG:
tensor.byteswap(inplace=True)
if self.use_temp_file and self.temp_file is None:
fp = tempfile.SpooledTemporaryFile(mode='w+b', max_size=256 *
1024 * 1024)
fp.seek(0)
self.temp_file = fp
shape: Sequence[int
] = raw_shape if raw_shape is not None else tensor.shape
self.add_tensor_info(name, shape, tensor.dtype, tensor.nbytes,
raw_dtype=raw_dtype)
if self.temp_file is None:
self.tensors[-1][name].tensor = tensor
return
tensor.tofile(self.temp_file)
self.write_padding(self.temp_file, tensor.nbytes)
def write_padding(self, fp, n, align=None):
pad = GGUFWriter.ggml_pad(n, align if align is not None else self.
data_alignment) - n
if pad != 0:
fp.write(bytes([0] * pad))
def write_tensor_data(self, tensor):
if (self.state is not WriterState.TI_DATA and self.state is not
WriterState.WEIGHTS):
raise ValueError(
f'Expected output file to contain tensor info or weights, got {self.state}'
)
assert self.fout is not None
if self.endianess == GGUFEndian.BIG:
tensor.byteswap(inplace=True)
file_id = -1
for i, tensors in enumerate(self.tensors):
if len(tensors) > 0:
file_id = i
break
fout = self.fout[file_id]
first_tensor_name = [name for name, _ in zip(self.tensors[file_id].
keys(), range(1))][0]
ti = self.tensors[file_id].pop(first_tensor_name)
assert ti.nbytes == tensor.nbytes
self.write_padding(fout, fout.tell())
tensor.tofile(fout)
self.write_padding(fout, tensor.nbytes)
self.state = WriterState.WEIGHTS
def write_tensors_to_file(self, *, progress: bool=False):
self.write_ti_data_to_file()
assert self.fout is not None
for fout in self.fout:
self.write_padding(fout, fout.tell())
if self.temp_file is None:
shard_bar = None
bar = None
if progress:
from tqdm import tqdm
total_bytes = sum(ti.nbytes for t in self.tensors for ti in
t.values())
if len(self.fout) > 1:
shard_bar = tqdm(desc=f'Shard (0/{len(self.fout)})',
total=None, unit='byte', unit_scale=True)
bar = tqdm(desc='Writing', total=total_bytes, unit='byte',
unit_scale=True)
for i, (fout, tensors) in enumerate(zip(self.fout, self.tensors)):
if shard_bar is not None:
shard_bar.set_description(
f'Shard ({i + 1}/{len(self.fout)})')
total = sum(ti.nbytes for ti in tensors.values())
shard_bar.reset(total=total if total > 0 else None)
for ti in tensors.values():
assert ti.tensor is not None
assert ti.tensor.nbytes == ti.nbytes
ti.tensor.tofile(fout)
if shard_bar is not None:
shard_bar.update(ti.nbytes)
if bar is not None:
bar.update(ti.nbytes)
self.write_padding(fout, ti.nbytes)
ti.tensor = None
else:
self.temp_file.seek(0)
shutil.copyfileobj(self.temp_file, self.fout[0 if not self.
small_first_shard else 1])
self.flush()
self.temp_file.close()
self.state = WriterState.WEIGHTS
def flush(self):
assert self.fout is not None
for fout in self.fout:
fout.flush()
def close(self):
if self.fout is not None:
for fout in self.fout:
fout.close()
self.fout = None
def add_type(self, type_name):
self.add_string(Keys.General.TYPE, type_name)
def add_architecture(self):
self.add_string(Keys.General.ARCHITECTURE, self.arch)
def add_quantization_version(self, quantization_version):
self.add_uint32(Keys.General.QUANTIZATION_VERSION, quantization_version
)
def add_custom_alignment(self, alignment):
self.data_alignment = alignment
self.add_uint32(Keys.General.ALIGNMENT, alignment)
def add_file_type(self, ftype):
self.add_uint32(Keys.General.FILE_TYPE, ftype)
def add_name(self, name):
self.add_string(Keys.General.NAME, name)
def add_author(self, author):
self.add_string(Keys.General.AUTHOR, author)
def add_version(self, version):
self.add_string(Keys.General.VERSION, version)
def add_organization(self, organization):
self.add_string(Keys.General.ORGANIZATION, organization)
def add_finetune(self, finetune):
self.add_string(Keys.General.FINETUNE, finetune)
def add_basename(self, basename):
self.add_string(Keys.General.BASENAME, basename)
def add_description(self, description):
self.add_string(Keys.General.DESCRIPTION, description)
def add_quantized_by(self, quantized):
self.add_string(Keys.General.QUANTIZED_BY, quantized)
def add_size_label(self, size_label):
self.add_string(Keys.General.SIZE_LABEL, size_label)
def add_license(self, license):
self.add_string(Keys.General.LICENSE, license)
def add_license_name(self, license):
self.add_string(Keys.General.LICENSE_NAME, license)
def add_license_link(self, license):
self.add_string(Keys.General.LICENSE_LINK, license)
def add_url(self, url):
self.add_string(Keys.General.URL, url)
def add_doi(self, doi):
self.add_string(Keys.General.DOI, doi)
def add_uuid(self, uuid):
self.add_string(Keys.General.UUID, uuid)
def add_repo_url(self, repo_url):
self.add_string(Keys.General.REPO_URL, repo_url)
def add_source_url(self, url):
self.add_string(Keys.General.SOURCE_URL, url)
def add_source_doi(self, doi):
self.add_string(Keys.General.SOURCE_DOI, doi)
def add_source_uuid(self, uuid):
self.add_string(Keys.General.SOURCE_UUID, uuid)
def add_source_repo_url(self, repo_url):
self.add_string(Keys.General.SOURCE_REPO_URL, repo_url)
def add_base_model_count(self, source_count):
self.add_uint32(Keys.General.BASE_MODEL_COUNT, source_count)
def add_base_model_name(self, source_id, name):
self.add_string(Keys.General.BASE_MODEL_NAME.format(id=source_id), name
)
def add_base_model_author(self, source_id, author):
self.add_string(Keys.General.BASE_MODEL_AUTHOR.format(id=source_id),
author)
def add_base_model_version(self, source_id, version):
self.add_string(Keys.General.BASE_MODEL_VERSION.format(id=source_id
), version)
def add_base_model_organization(self, source_id, organization):
self.add_string(Keys.General.BASE_MODEL_ORGANIZATION.format(id=
source_id), organization)
def add_base_model_description(self, source_id, description):
self.add_string(Keys.General.BASE_MODEL_DESCRIPTION.format(id=
source_id), description)
def add_base_model_url(self, source_id, url):
self.add_string(Keys.General.BASE_MODEL_URL.format(id=source_id), url)
def add_base_model_doi(self, source_id, doi):
self.add_string(Keys.General.BASE_MODEL_DOI.format(id=source_id), doi)
def add_base_model_uuid(self, source_id, uuid):
self.add_string(Keys.General.BASE_MODEL_UUID.format(id=source_id), uuid
)
def add_base_model_repo_url(self, source_id, repo_url):
self.add_string(Keys.General.BASE_MODEL_REPO_URL.format(id=
source_id), repo_url)
def add_dataset_count(self, source_count):
self.add_uint32(Keys.General.DATASET_COUNT, source_count)
def add_dataset_name(self, source_id, name):
self.add_string(Keys.General.DATASET_NAME.format(id=source_id), name)
def add_dataset_author(self, source_id, author):
self.add_string(Keys.General.DATASET_AUTHOR.format(id=source_id),
author)
def add_dataset_version(self, source_id, version):
self.add_string(Keys.General.DATASET_VERSION.format(id=source_id),
version)
def add_dataset_organization(self, source_id, organization):
self.add_string(Keys.General.DATASET_ORGANIZATION.format(id=
source_id), organization)
def add_dataset_description(self, source_id, description):
self.add_string(Keys.General.DATASET_DESCRIPTION.format(id=
source_id), description)
def add_dataset_url(self, source_id, url):
self.add_string(Keys.General.DATASET_URL.format(id=source_id), url)
def add_dataset_doi(self, source_id, doi):
self.add_string(Keys.General.DATASET_DOI.format(id=source_id), doi)
def add_dataset_uuid(self, source_id, uuid):
self.add_string(Keys.General.DATASET_UUID.format(id=source_id), uuid)
def add_dataset_repo_url(self, source_id, repo_url):
self.add_string(Keys.General.DATASET_REPO_URL.format(id=source_id),
repo_url)
def add_tags(self, tags):
self.add_array(Keys.General.TAGS, tags)
def add_languages(self, languages):
self.add_array(Keys.General.LANGUAGES, languages)
def add_tensor_data_layout(self, layout):
self.add_string(Keys.LLM.TENSOR_DATA_LAYOUT.format(arch=self.arch),
layout)
def add_vocab_size(self, size):
self.add_uint32(Keys.LLM.VOCAB_SIZE.format(arch=self.arch), size)
def add_context_length(self, length):
self.add_uint32(Keys.LLM.CONTEXT_LENGTH.format(arch=self.arch), length)
def add_embedding_length(self, length):
self.add_uint32(Keys.LLM.EMBEDDING_LENGTH.format(arch=self.arch),
length)
def add_features_length(self, length):
self.add_uint32(Keys.LLM.FEATURES_LENGTH.format(arch=self.arch), length
)
def add_posnet_embedding_length(self, length):
self.add_uint32(Keys.PosNet.EMBEDDING_LENGTH.format(arch=self.arch),
length)
def add_posnet_block_count(self, length):
self.add_uint32(Keys.PosNet.BLOCK_COUNT.format(arch=self.arch), length)
def add_convnext_embedding_length(self, length):
self.add_uint32(Keys.ConvNext.EMBEDDING_LENGTH.format(arch=self.
arch), length)
def add_convnext_block_count(self, length):
self.add_uint32(Keys.ConvNext.BLOCK_COUNT.format(arch=self.arch),
length)
def add_block_count(self, length):
self.add_uint32(Keys.LLM.BLOCK_COUNT.format(arch=self.arch), length)
def add_leading_dense_block_count(self, length):
self.add_uint32(Keys.LLM.LEADING_DENSE_BLOCK_COUNT.format(arch=self
.arch), length)
def add_feed_forward_length(self, length):
if isinstance(length, int):
self.add_uint32(Keys.LLM.FEED_FORWARD_LENGTH.format(arch=self.
arch), length)
else:
self.add_array(Keys.LLM.FEED_FORWARD_LENGTH.format(arch=self.
arch), length)
def add_expert_feed_forward_length(self, length):
self.add_uint32(Keys.LLM.EXPERT_FEED_FORWARD_LENGTH.format(arch=
self.arch), length)
def add_expert_shared_feed_forward_length(self, length):
self.add_uint32(Keys.LLM.EXPERT_SHARED_FEED_FORWARD_LENGTH.format(
arch=self.arch), length)
def add_parallel_residual(self, use):
self.add_bool(Keys.LLM.USE_PARALLEL_RESIDUAL.format(arch=self.arch),
use)
def add_decoder_start_token_id(self, id):
self.add_uint32(Keys.LLM.DECODER_START_TOKEN_ID.format(arch=self.
arch), id)
def add_head_count(self, count):
if isinstance(count, int):
self.add_uint32(Keys.Attention.HEAD_COUNT.format(arch=self.arch
), count)
else:
self.add_array(Keys.Attention.HEAD_COUNT.format(arch=self.arch),
count)
def add_head_count_kv(self, count):
if isinstance(count, int):
self.add_uint32(Keys.Attention.HEAD_COUNT_KV.format(arch=self.
arch), count)
else:
self.add_array(Keys.Attention.HEAD_COUNT_KV.format(arch=self.
arch), count)
def add_key_length(self, length):
self.add_uint32(Keys.Attention.KEY_LENGTH.format(arch=self.arch),
length)
def add_value_length(self, length):
self.add_uint32(Keys.Attention.VALUE_LENGTH.format(arch=self.arch),
length)
def add_key_length_mla(self, length):
self.add_uint32(Keys.Attention.KEY_LENGTH_MLA.format(arch=self.arch
), length)
def add_value_length_mla(self, length):
self.add_uint32(Keys.Attention.VALUE_LENGTH_MLA.format(arch=self.
arch), length)
def add_max_alibi_bias(self, bias):
self.add_float32(Keys.Attention.MAX_ALIBI_BIAS.format(arch=self.
arch), bias)
def add_clamp_kqv(self, value):
self.add_float32(Keys.Attention.CLAMP_KQV.format(arch=self.arch), value
)
def add_logit_scale(self, value):
self.add_float32(Keys.LLM.LOGIT_SCALE.format(arch=self.arch), value)
def add_attn_logit_softcapping(self, value):
self.add_float32(Keys.LLM.ATTN_LOGIT_SOFTCAPPING.format(arch=self.
arch), value)
def add_final_logit_softcapping(self, value):
self.add_float32(Keys.LLM.FINAL_LOGIT_SOFTCAPPING.format(arch=self.
arch), value)
def add_expert_count(self, count):
self.add_uint32(Keys.LLM.EXPERT_COUNT.format(arch=self.arch), count)
def add_expert_used_count(self, count):
self.add_uint32(Keys.LLM.EXPERT_USED_COUNT.format(arch=self.arch),
count)
def add_expert_shared_count(self, count):
self.add_uint32(Keys.LLM.EXPERT_SHARED_COUNT.format(arch=self.arch),
count)
def add_expert_weights_scale(self, value):
self.add_float32(Keys.LLM.EXPERT_WEIGHTS_SCALE.format(arch=self.
arch), value)
def add_expert_weights_norm(self, value):
self.add_bool(Keys.LLM.EXPERT_WEIGHTS_NORM.format(arch=self.arch),
value)
def add_expert_gating_func(self, value):
self.add_uint32(Keys.LLM.EXPERT_GATING_FUNC.format(arch=self.arch),
value.value)
def add_swin_norm(self, value):
self.add_bool(Keys.LLM.SWIN_NORM.format(arch=self.arch), value)
def add_rescale_every_n_layers(self, count):
self.add_uint32(Keys.LLM.RESCALE_EVERY_N_LAYERS.format(arch=self.
arch), count)
def add_time_mix_extra_dim(self, dim):
self.add_uint32(Keys.LLM.TIME_MIX_EXTRA_DIM.format(arch=self.arch), dim
)
def add_time_decay_extra_dim(self, dim):
self.add_uint32(Keys.LLM.TIME_DECAY_EXTRA_DIM.format(arch=self.arch
), dim)
def add_residual_scale(self, value):
self.add_float32(Keys.LLM.RESIDUAL_SCALE.format(arch=self.arch), value)
def add_embedding_scale(self, value):
self.add_float32(Keys.LLM.EMBEDDING_SCALE.format(arch=self.arch), value
)
def add_wkv_head_size(self, size):
self.add_uint32(Keys.WKV.HEAD_SIZE.format(arch=self.arch), size)
def add_token_shift_count(self, count):
self.add_uint32(Keys.LLM.TOKEN_SHIFT_COUNT.format(arch=self.arch),
count)
def add_interleave_moe_layer_step(self, value):
self.add_uint32(Keys.LLM.INTERLEAVE_MOE_LAYER_STEP.format(arch=self
.arch), value)
def add_layer_norm_eps(self, value):
self.add_float32(Keys.Attention.LAYERNORM_EPS.format(arch=self.arch
), value)
def add_layer_norm_rms_eps(self, value):
self.add_float32(Keys.Attention.LAYERNORM_RMS_EPS.format(arch=self.
arch), value)
def add_group_norm_eps(self, value):
self.add_float32(Keys.Attention.GROUPNORM_EPS.format(arch=self.arch
), value)
def add_group_norm_groups(self, value):
self.add_uint32(Keys.Attention.GROUPNORM_GROUPS.format(arch=self.
arch), value)
def add_causal_attention(self, value):
self.add_bool(Keys.Attention.CAUSAL.format(arch=self.arch), value)
def add_q_lora_rank(self, length):
self.add_uint32(Keys.Attention.Q_LORA_RANK.format(arch=self.arch),
length)
def add_kv_lora_rank(self, length):
self.add_uint32(Keys.Attention.KV_LORA_RANK.format(arch=self.arch),
length)
def add_decay_lora_rank(self, length):
self.add_uint32(Keys.Attention.DECAY_LORA_RANK.format(arch=self.
arch), length)
def add_iclr_lora_rank(self, length):
self.add_uint32(Keys.Attention.ICLR_LORA_RANK.format(arch=self.arch
), length)
def add_value_residual_mix_lora_rank(self, length):
self.add_uint32(Keys.Attention.VALUE_RESIDUAL_MIX_LORA_RANK.format(
arch=self.arch), length)
def add_gate_lora_rank(self, length):
self.add_uint32(Keys.Attention.GATE_LORA_RANK.format(arch=self.arch
), length)
def add_relative_attn_buckets_count(self, value):
self.add_uint32(Keys.Attention.REL_BUCKETS_COUNT.format(arch=self.
arch), value)
def add_sliding_window(self, value):
self.add_uint32(Keys.Attention.SLIDING_WINDOW.format(arch=self.arch
), value)
def add_attention_scale(self, value):
self.add_float32(Keys.Attention.SCALE.format(arch=self.arch), value)
def add_pooling_type(self, value):
self.add_uint32(Keys.LLM.POOLING_TYPE.format(arch=self.arch), value
.value)
def add_rope_dimension_count(self, count):
self.add_uint32(Keys.Rope.DIMENSION_COUNT.format(arch=self.arch), count
)
def add_rope_dimension_sections(self, dims):
self.add_array(Keys.Rope.DIMENSION_SECTIONS.format(arch=self.arch),
dims)
def add_rope_freq_base(self, value):
self.add_float32(Keys.Rope.FREQ_BASE.format(arch=self.arch), value)
def add_rope_scaling_type(self, value):
self.add_string(Keys.Rope.SCALING_TYPE.format(arch=self.arch),
value.value)
def add_rope_scaling_factor(self, value):
self.add_float32(Keys.Rope.SCALING_FACTOR.format(arch=self.arch), value
)
def add_rope_scaling_attn_factors(self, value):
self.add_float32(Keys.Rope.SCALING_ATTN_FACTOR.format(arch=self.
arch), value)
def add_rope_scaling_orig_ctx_len(self, value):
self.add_uint32(Keys.Rope.SCALING_ORIG_CTX_LEN.format(arch=self.
arch), value)
def add_rope_scaling_finetuned(self, value):
self.add_bool(Keys.Rope.SCALING_FINETUNED.format(arch=self.arch), value
)
def add_rope_scaling_yarn_log_mul(self, value):
self.add_float32(Keys.Rope.SCALING_YARN_LOG_MUL.format(arch=self.
arch), value)
def add_ssm_conv_kernel(self, value):
self.add_uint32(Keys.SSM.CONV_KERNEL.format(arch=self.arch), value)
def add_ssm_inner_size(self, value):
self.add_uint32(Keys.SSM.INNER_SIZE.format(arch=self.arch), value)
def add_ssm_state_size(self, value):
self.add_uint32(Keys.SSM.STATE_SIZE.format(arch=self.arch), value)
def add_ssm_time_step_rank(self, value):
self.add_uint32(Keys.SSM.TIME_STEP_RANK.format(arch=self.arch), value)
def add_ssm_dt_b_c_rms(self, value):
self.add_bool(Keys.SSM.DT_B_C_RMS.format(arch=self.arch), value)
def add_tokenizer_model(self, model):
self.add_string(Keys.Tokenizer.MODEL, model)
def add_tokenizer_pre(self, pre):
self.add_string(Keys.Tokenizer.PRE, pre)
def add_token_list(self, tokens):
self.add_array(Keys.Tokenizer.LIST, tokens)
def add_token_merges(self, merges):
self.add_array(Keys.Tokenizer.MERGES, merges)
def add_token_types(self, types):
self.add_array(Keys.Tokenizer.TOKEN_TYPE, types)
def add_token_type_count(self, value):
self.add_uint32(Keys.Tokenizer.TOKEN_TYPE_COUNT, value)
def add_token_scores(self, scores):
self.add_array(Keys.Tokenizer.SCORES, scores)
def add_bos_token_id(self, id):
self.add_uint32(Keys.Tokenizer.BOS_ID, id)
def add_eos_token_id(self, id):
self.add_uint32(Keys.Tokenizer.EOS_ID, id)
def add_unk_token_id(self, id):
self.add_uint32(Keys.Tokenizer.UNK_ID, id)
def add_sep_token_id(self, id):
self.add_uint32(Keys.Tokenizer.SEP_ID, id)
def add_pad_token_id(self, id):
self.add_uint32(Keys.Tokenizer.PAD_ID, id)
def add_mask_token_id(self, id):
self.add_uint32(Keys.Tokenizer.MASK_ID, id)
def add_add_bos_token(self, value):
self.add_bool(Keys.Tokenizer.ADD_BOS, value)
def add_add_eos_token(self, value):
self.add_bool(Keys.Tokenizer.ADD_EOS, value)
def add_add_space_prefix(self, value):
self.add_bool(Keys.Tokenizer.ADD_PREFIX, value)
def add_remove_extra_whitespaces(self, value):
self.add_bool(Keys.Tokenizer.REMOVE_EXTRA_WS, value)
def add_precompiled_charsmap(self, charsmap):
self.add_array(Keys.Tokenizer.PRECOMPILED_CHARSMAP, charsmap)
def add_chat_template(self, value):
if not isinstance(value, str):
template_default = None
template_names = set()
for choice in value:
name = choice.get('name', '')
template = choice.get('template')
name = ''.join(c if c in ascii_letters + digits else '_' for
c in name)
if name and template is not None:
if name == 'default':
template_default = template
else:
template_names.add(name)
self.add_string(Keys.Tokenizer.CHAT_TEMPLATE_N.
format(name=name), template)
if template_names:
self.add_array(Keys.Tokenizer.CHAT_TEMPLATES, list(
template_names))
if template_default is None:
return
value = template_default
self.add_string(Keys.Tokenizer.CHAT_TEMPLATE, value)
def add_eot_token_id(self, id):
self.add_uint32(Keys.Tokenizer.EOT_ID, id)
def add_eom_token_id(self, id):
self.add_uint32(Keys.Tokenizer.EOM_ID, id)
def _pack(self, fmt, value, skip_pack_prefix=False):
pack_prefix = ''
if not skip_pack_prefix:
pack_prefix = '<' if self.endianess == GGUFEndian.LITTLE else '>'
return struct.pack(f'{pack_prefix}{fmt}', value)
def _pack_val(self, val, vtype, add_vtype):
kv_data = bytearray()
if add_vtype:
kv_data += self._pack('I', vtype)
pack_fmt = self._simple_value_packing.get(vtype)
if pack_fmt is not None:
kv_data += self._pack(pack_fmt, val, skip_pack_prefix=vtype ==
GGUFValueType.BOOL)
elif vtype == GGUFValueType.STRING:
encoded_val = val.encode('utf-8') if isinstance(val, str) else val
kv_data += self._pack('Q', len(encoded_val))
kv_data += encoded_val
elif vtype == GGUFValueType.ARRAY:
if not isinstance(val, Sequence):
raise ValueError(
'Invalid GGUF metadata array, expecting sequence')
if len(val) == 0:
raise ValueError('Invalid GGUF metadata array. Empty array')
if isinstance(val, bytes):
ltype = GGUFValueType.UINT8
else:
ltype = GGUFValueType.get_type(val[0])
if not all(GGUFValueType.get_type(i) is ltype for i in val[1:]
):
raise ValueError(
'All items in a GGUF array should be of the same type')
kv_data += self._pack('I', ltype)
kv_data += self._pack('Q', len(val))
for item in val:
kv_data += self._pack_val(item, ltype, add_vtype=False)
else:
raise ValueError('Invalid GGUF metadata value type or value')
return kv_data
@staticmethod
def format_n_bytes_to_str(num):
if num == 0:
return 'negligible - metadata only'
fnum = float(num)
for unit in ('', 'K', 'M', 'G'):
if abs(fnum) < 1000.0:
return f'{fnum:3.1f}{unit}'
fnum /= 1000.0
return f'{fnum:.1f}T - over 1TB, split recommended'
gguf_connector/y.py
+1
View File
@@ -0,0 +1 @@
from gguf_comfy import downloader
gguf_nodes.py
+584
View File
@@ -0,0 +1,584 @@
import os
import gc
import time
import comfy
import io
import torch
import json
import nodes
import folder_paths
from .gguf_connector import reader as gr
from .gguf_connector.writer import GGUFWriter, GGMLQuantizationType
from .gguf_connector.const import GGML_QUANT_VERSION, LlamaFileType
from .gguf_connector.quant import quantize, dequantize, QuantError
from .gguf_connector.quant2d import dequantize_tensor, is_quantized, is_torch_compatible
from .gguf_connector.tkn import get_field, tokenizer_builder
if hasattr(torch, 'compiler') and hasattr(torch.compiler, 'disable'):
torch_compiler_disable = torch.compiler.disable
else:
def torch_compiler_disable(*args, **kwargs):
def noop(x):
return x
return noop
def get_clip_type(type):
clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION)
return clip_type
def get_device(device):
model_options = {}
if device == "cpu":
model_options["load_device"] = model_options["offload_device"] = torch.device("cpu")
return model_options
def tensor_swap(raw_sd, key_map):
sd = {}
for k, v in raw_sd.items():
for s, d in key_map.items():
k = k.replace(s, d)
sd[k] = v
return sd
def llama_permute(raw_sd, n_head, n_head_kv):
sd = {}
permute = lambda x, h: x.reshape(h, x.shape[0] // h // 2, 2, *x.shape[1:]
).swapaxes(1, 2).reshape(x.shape)
for k, v in raw_sd.items():
if k.endswith(('q_proj.weight', 'q_proj.bias')):
v.data = permute(v.data, n_head)
if k.endswith(('k_proj.weight', 'k_proj.bias')):
v.data = permute(v.data, n_head_kv)
sd[k] = v
return sd
def load_patch_to_device(item, device):
if isinstance(item, torch.Tensor):
return item.to(device, non_blocking=True)
elif isinstance(item, tuple):
return tuple(load_patch_to_device(x, device) for x in item)
elif isinstance(item, list):
return [load_patch_to_device(x, device) for x in item]
else:
return item
class GGMLTensor(torch.Tensor):
def __init__(self, *args, tensor_type, tensor_shape, patches=[], **kwargs):
super().__init__()
self.tensor_type = tensor_type
self.tensor_shape = tensor_shape
self.patches = patches
def __new__(cls, *args, tensor_type, tensor_shape, patches=[], **kwargs):
return super().__new__(cls, *args, **kwargs)
def to(self, *args, **kwargs):
new = super().to(*args, **kwargs)
new.tensor_type = getattr(self, 'tensor_type', None)
new.tensor_shape = getattr(self, 'tensor_shape', new.data.shape)
new.patches = getattr(self, 'patches', []).copy()
return new
def clone(self, *args, **kwargs):
return self
def detach(self, *args, **kwargs):
return self
def copy_(self, *args, **kwargs):
try:
return super().copy_(*args, **kwargs)
except Exception as e:
print(f"ignoring 'copy_' on tensor: {e}")
def empty_(self, size, *args, **kwargs):
new_tensor = super().empty_(size, *args, **kwargs)
return GGMLTensor(new_tensor, tensor_type=getattr(self,
'tensor_type', None), tensor_shape=size, patches=getattr(self,
'patches', []).copy())
@property
def shape(self):
if not hasattr(self, 'tensor_shape'):
self.tensor_shape = self.size()
return self.tensor_shape
class GGMLLayer(torch.nn.Module):
comfy_cast_weights = True
dequant_dtype = None
patch_dtype = None
largest_layer = False
torch_compatible_tensor_types = {None, gr.GGMLQuantizationType.F32, gr.
GGMLQuantizationType.F16}
def is_ggml_quantized(self, *, weight=None, bias=None):
if weight is None:
weight = self.weight
if bias is None:
bias = self.bias
return is_quantized(weight) or is_quantized(bias)
def _load_from_state_dict(self, state_dict, prefix, *args, **kwargs):
weight, bias = state_dict.get(f'{prefix}weight'), state_dict.get(
f'{prefix}bias')
if self.is_ggml_quantized(weight=weight, bias=bias) or isinstance(self,
torch.nn.Linear):
return self.ggml_load_from_state_dict(state_dict, prefix, *args,
**kwargs)
return super()._load_from_state_dict(state_dict, prefix, *args, **
kwargs)
def ggml_load_from_state_dict(self, state_dict, prefix, local_metadata,
strict, missing_keys, unexpected_keys, error_msgs):
prefix_len = len(prefix)
for k, v in state_dict.items():
if k[prefix_len:] == 'weight':
self.weight = torch.nn.Parameter(v, requires_grad=False)
elif k[prefix_len:] == 'bias' and v is not None:
self.bias = torch.nn.Parameter(v, requires_grad=False)
else:
missing_keys.append(k)
if self.weight is None and isinstance(self, torch.nn.Linear):
v = torch.zeros(self.in_features, self.out_features)
self.weight = torch.nn.Parameter(v, requires_grad=False)
missing_keys.append(prefix + 'weight')
if getattr(self.weight, 'is_largest_weight', False):
self.largest_layer = True
def _save_to_state_dict(self, *args, **kwargs):
if self.is_ggml_quantized():
return self.ggml_save_to_state_dict(*args, **kwargs)
return super()._save_to_state_dict(*args, **kwargs)
def ggml_save_to_state_dict(self, destination, prefix, keep_vars):
weight = torch.zeros_like(self.weight, device=torch.device('meta'))
destination[prefix + 'weight'] = weight
if self.bias is not None:
bias = torch.zeros_like(self.bias, device=torch.device('meta'))
destination[prefix + 'bias'] = bias
if self.largest_layer:
shape = getattr(self.weight, 'tensor_shape', self.weight.shape)
dtype = self.dequant_dtype or torch.float16
temp = torch.empty(*shape, device=torch.device('meta'), dtype=dtype
)
destination[prefix + 'temp.weight'] = temp
return
destination[prefix + 'weight'] = self.get_weight(self.weight)
if bias is not None:
destination[prefix + 'bias'] = self.get_weight(self.bias)
def get_weight(self, tensor, dtype):
if tensor is None:
return
patch_list = []
device = tensor.device
for function, patches, key in getattr(tensor, 'patches', []):
patch_list += load_patch_to_device(patches, device)
weight = dequantize_tensor(tensor, dtype, self.dequant_dtype)
if isinstance(weight, GGMLTensor):
weight = torch.Tensor(weight)
if patch_list:
if self.patch_dtype is None:
weight = function(patch_list, weight, key)
else:
patch_dtype = (dtype if self.patch_dtype == 'target' else
self.patch_dtype)
weight = function(patch_list, weight, key, patch_dtype)
return weight
@torch_compiler_disable()
def cast_bias_weight(s, input=None, dtype=None, device=None, bias_dtype
=None):
if input is not None:
if dtype is None:
dtype = getattr(input, 'dtype', torch.float32)
if bias_dtype is None:
bias_dtype = dtype
if device is None:
device = input.device
bias = None
non_blocking = comfy.model_management.device_supports_non_blocking(
device)
if s.bias is not None:
bias = s.get_weight(s.bias.to(device), dtype)
bias = comfy.ops.cast_to(bias, bias_dtype, device, non_blocking
=non_blocking, copy=False)
weight = s.get_weight(s.weight.to(device), dtype)
weight = comfy.ops.cast_to(weight, dtype, device, non_blocking=
non_blocking, copy=False)
return weight, bias
def forward_comfy_cast_weights(self, input, *args, **kwargs):
if self.is_ggml_quantized():
out = self.forward_ggml_cast_weights(input, *args, **kwargs)
else:
out = super().forward_comfy_cast_weights(input, *args, **kwargs)
if isinstance(out, GGMLTensor):
out = torch.Tensor(out)
return out
def forward_ggml_cast_weights(self, input):
raise NotImplementedError
class GGMLOps(comfy.ops.manual_cast):
class Linear(GGMLLayer, comfy.ops.manual_cast.Linear):
def __init__(self, in_features, out_features, bias=True, device=
None, dtype=None):
torch.nn.Module.__init__(self)
self.in_features = in_features
self.out_features = out_features
self.weight = None
self.bias = None
def forward_ggml_cast_weights(self, input):
weight, bias = self.cast_bias_weight(input)
return torch.nn.functional.linear(input, weight, bias)
class Conv2d(GGMLLayer, comfy.ops.manual_cast.Conv2d):
def forward_ggml_cast_weights(self, input):
weight, bias = self.cast_bias_weight(input)
return self._conv_forward(input, weight, bias)
class Embedding(GGMLLayer, comfy.ops.manual_cast.Embedding):
def forward_ggml_cast_weights(self, input, out_dtype=None):
output_dtype = out_dtype
if (self.weight.dtype == torch.float16 or self.weight.dtype ==
torch.bfloat16):
out_dtype = None
weight, _bias = self.cast_bias_weight(self, device=input.device,
dtype=out_dtype)
return torch.nn.functional.embedding(input, weight, self.
padding_idx, self.max_norm, self.norm_type, self.
scale_grad_by_freq, self.sparse).to(dtype=output_dtype)
class LayerNorm(GGMLLayer, comfy.ops.manual_cast.LayerNorm):
def forward_ggml_cast_weights(self, input):
if self.weight is None:
return super().forward_comfy_cast_weights(input)
weight, bias = self.cast_bias_weight(input)
return torch.nn.functional.layer_norm(input, self.
normalized_shape, weight, bias, self.eps)
class GroupNorm(GGMLLayer, comfy.ops.manual_cast.GroupNorm):
def forward_ggml_cast_weights(self, input):
weight, bias = self.cast_bias_weight(input)
return torch.nn.functional.group_norm(input, self.num_groups,
weight, bias, self.eps)
pig = os.path.join(os.path.dirname(__file__), 'version.json')
with open(pig, 'r') as file:
data = json.load(file)
arrays = {}
for key, value in data[0].items():
arrays[key] = value
def get_orig_shape(reader, tensor_name):
field_key = f'comfy.gguf.orig_shape.{tensor_name}'
field = reader.get_field(field_key)
if field is None:
return None
if len(field.types) != 2 or field.types[0
] != gr.GGUFValueType.ARRAY or field.types[1
] != gr.GGUFValueType.INT32:
raise TypeError(
f'Bad original shape metadata for {field_key}: Expected ARRAY of INT32, got {field.types}'
)
return torch.Size(tuple(int(field.parts[part_idx][0]) for part_idx in
field.data))
class GGUFModelPatcher(comfy.model_patcher.ModelPatcher):
patch_on_device = False
def patch_weight_to_device(self, key, device_to=None, inplace_update=False):
if key not in self.patches:
return
weight = comfy.utils.get_attr(self.model, key)
try:
from comfy.lora import calculate_weight
except Exception:
calculate_weight = self.calculate_weight
patches = self.patches[key]
if is_quantized(weight):
out_weight = weight.to(device_to)
patches = load_patch_to_device(patches, self.load_device if
self.patch_on_device else self.offload_device)
out_weight.patches = [(calculate_weight, patches, key)]
else:
inplace_update = self.weight_inplace_update or inplace_update
if key not in self.backup:
self.backup[key] = collections.namedtuple('Dimension', [
'weight', 'inplace_update'])(weight.to(device=self.
offload_device, copy=inplace_update), inplace_update)
if device_to is not None:
temp_weight = comfy.model_management.cast_to_device(weight,
device_to, torch.float32, copy=True)
else:
temp_weight = weight.to(torch.float32, copy=True)
out_weight = calculate_weight(patches, temp_weight, key)
out_weight = comfy.float.stochastic_rounding(out_weight, weight
.dtype)
if inplace_update:
comfy.utils.copy_to_param(self.model, key, out_weight)
else:
comfy.utils.set_attr_param(self.model, key, out_weight)
def unpatch_model(self, device_to=None, unpatch_weights=True):
if unpatch_weights:
for p in self.model.parameters():
if is_torch_compatible(p):
continue
patches = getattr(p, 'patches', [])
if len(patches) > 0:
p.patches = []
return super().unpatch_model(device_to=device_to, unpatch_weights=
unpatch_weights)
mmap_released = False
def load(self, *args, force_patch_weights=False, **kwargs):
super().load(*args, force_patch_weights=True, **kwargs)
if not self.mmap_released:
linked = []
if kwargs.get('lowvram_model_memory', 0) > 0:
for n, m in self.model.named_modules():
if hasattr(m, 'weight'):
device = getattr(m.weight, 'device', None)
if device == self.offload_device:
linked.append((n, m))
continue
if hasattr(m, 'bias'):
device = getattr(m.bias, 'device', None)
if device == self.offload_device:
linked.append((n, m))
continue
if linked:
print(f'Attempting to release mmap ({len(linked)})')
for n, m in linked:
m.to(self.load_device).to(self.offload_device)
self.mmap_released = True
def clone(self, *args, **kwargs):
src_cls = self.__class__
self.__class__ = GGUFModelPatcher
n = super().clone(*args, **kwargs)
n.__class__ = GGUFModelPatcher
self.__class__ = src_cls
n.patch_on_device = getattr(self, 'patch_on_device', False)
return n
def load_gguf_sd(path, handle_prefix='model.diffusion_model.', return_arch=
False):
reader = gr.GGUFReader(path)
has_prefix = False
if handle_prefix is not None:
prefix_len = len(handle_prefix)
tensor_names = set(tensor.name for tensor in reader.tensors)
has_prefix = any(s.startswith(handle_prefix) for s in tensor_names)
tensors = []
for tensor in reader.tensors:
sd_key = tensor_name = tensor.name
if has_prefix:
if not tensor_name.startswith(handle_prefix):
continue
sd_key = tensor_name[prefix_len:]
tensors.append((sd_key, tensor))
# print(arrays)
compat = None
arch_str = get_field(reader, 'general.architecture', str)
# print("arch_str : {}".format(arch_str))
if arch_str is None:
compat = 'sd.cpp'
elif arch_str not in arrays['PIG_ARCH_LIST'] and arch_str not in arrays['TXT_ARCH_LIST']:
raise ValueError(f"Unknown architecture: {arch_str!r}")
state_dict, qtype_dict = {}, {}
for sd_key, tensor in tensors:
tensor_name = tensor.name
torch_tensor = torch.from_numpy(tensor.data)
shape = get_orig_shape(reader, tensor_name)
if shape is None:
shape = torch.Size(tuple(int(v) for v in reversed(tensor.shape)))
if compat == 'sd.cpp' and arch_str == 'sdxl':
if any([tensor_name.endswith(x) for x in ('.proj_in.weight',
'.proj_out.weight')]):
while len(shape) > 2 and shape[-1] == 1:
shape = shape[:-1]
if tensor.tensor_type in {gr.GGMLQuantizationType.F32, gr.
GGMLQuantizationType.F16}:
torch_tensor = torch_tensor.view(*shape)
state_dict[sd_key] = GGMLTensor(torch_tensor, tensor_type=tensor.
tensor_type, tensor_shape=shape)
tensor_type_str = getattr(tensor.tensor_type, 'name', repr(tensor.tensor_type))
qtype_dict[tensor_type_str] = qtype_dict.get(tensor_type_str, 0) + 1
print('gguf qtypes: ' + ', '.join(f'{k} ({v})' for k, v in qtype_dict.items()))
qsd = {k: v for k, v in state_dict.items() if is_quantized(v)}
if len(qsd) > 0:
max_key = max(qsd.keys(), key=lambda k: qsd[k].numel())
state_dict[max_key].is_largest_weight = True
if return_arch:
return state_dict, arch_str
return state_dict
class WarpedLoaderGGUF:
@classmethod
def INPUT_TYPES(s):
gguf_names = [x for x in folder_paths.get_filename_list('model_gguf')]
return {'required': {'gguf_name': (gguf_names,)}}
RETURN_TYPES = 'MODEL',
FUNCTION = 'load_model'
CATEGORY = 'Warped/GGUF/Loaders'
def load_model(self, gguf_name, dequant_dtype=None, patch_dtype=None,
patch_on_device=None):
ops = GGMLOps()
if dequant_dtype in ('default', None):
ops.Linear.dequant_dtype = None
elif dequant_dtype in ['target']:
ops.Linear.dequant_dtype = dequant_dtype
else:
ops.Linear.dequant_dtype = getattr(torch, dequant_dtype)
if patch_dtype in ('default', None):
ops.Linear.patch_dtype = None
elif patch_dtype in ['target']:
ops.Linear.patch_dtype = patch_dtype
else:
ops.Linear.patch_dtype = getattr(torch, patch_dtype)
model_path = folder_paths.get_full_path('unet', gguf_name)
print("Reading Checkpoint: {}...".format(model_path))
with open(model_path, "rb") as file:
checkpoint_temp = file.read()
print("Reading Checkpoint: {}...Done!".format(model_path))
print("Loading Checkpoint...")
temp_io = io.BytesIO(checkpoint_temp)
sd = load_gguf_sd(temp_io)
print("Loading Checkpoint...Done!")
model = comfy.sd.load_diffusion_model_state_dict(sd, model_options=
{'custom_operations': ops})
if model is None:
logging.error('ERROR UNSUPPORTED MODEL {}'.format(model_path))
raise RuntimeError('ERROR: Could not detect model type of: {}'.
format(model_path))
model = GGUFModelPatcher.clone(model)
model.patch_on_device = patch_on_device
temp_io = None
checkpoint_temp = None
gc.collect()
time.sleep(1)
return model,
def warped_load_torch_file(ckpt, return_metadata=False):
from safetensors.torch import load as safeload
metadata = None
try:
f = safeload(ckpt)
sd = {}
for k in f.keys():
if k != "__metadata__":
sd[k] = f[k]
elif return_metadata:
metadata = f[k]
except Exception as e:
if len(e.args) > 0:
message = e.args[0]
if "HeaderTooLarge" in message:
raise ValueError("{}\n\nFile path: {}\n\nThe safetensors file is corrupt or invalid. Make sure this is actually a safetensors file and not a ckpt or pt or other filetype.".format(message, ckpt))
if "MetadataIncompleteBuffer" in message:
raise ValueError("{}\n\nFile path: {}\n\nThe safetensors file is corrupt/incomplete. Check the file size and make sure you have copied/downloaded it correctly.".format(message, ckpt))
raise e
return (sd, metadata) if return_metadata else sd
def load_gguf_clip(path):
print("Reading Clip: {}...".format(path))
with open(path, "rb") as file:
checkpoint_temp = file.read()
print("Reading Clip: {}...Done!".format(path))
print("Loading Clip...")
# temp = warped_load_torch_file(checkpoint_temp)
temp_io = io.BytesIO(checkpoint_temp)
# print("temp_io Type: {}".format(type(temp_io)))
sd, arch = load_gguf_sd(temp_io, return_arch=True)
print("Loading Clip...Done!")
if arch in {'t5', 't5encoder'}:
temb_key = 'token_embd.weight'
if temb_key in sd and (sd[temb_key].shape == (256384, 4096) or sd[temb_key].shape == (256384, 768)):
sd['spiece_model'] = tokenizer_builder(path)
sd = tensor_swap(sd, arrays['T5'])
elif temb_key in sd and sd[temb_key].shape == (32128, 768):
sd = tensor_swap(sd, arrays['B5'])
else:
sd = tensor_swap(sd, arrays['T5'])
elif arch in {'llama'}:
sd = tensor_swap(sd, arrays['L3'])
sd = llama_permute(sd, 32, 8)
elif arch in {'gemma2'}:
sd["spiece_model"] = tokenizer_builder(path)
sd = tensor_swap(sd, arrays['G2'])
elif arch in {'pig'}:
sd = pig_work(sd)
else:
pass
temp_io = None
return sd
class WarpedClipLoaderGGUF:
@classmethod
def INPUT_TYPES(s):
base = nodes.CLIPLoader.INPUT_TYPES()
return {'required': {'clip_name': (s.get_filename_list(),), 'type':
base['required']['type']},
'optional':{'device':(['default','cpu'],{'advanced':True}),}}
RETURN_TYPES = 'CLIP',
FUNCTION = 'load_clip'
CATEGORY = 'Warped/GGUF/Loaders'
@classmethod
def get_filename_list(s):
files = []
files += folder_paths.get_filename_list('clip')
files += folder_paths.get_filename_list('clip_gguf')
return sorted(files)
def load_data(self, ckpt_paths):
clip_data = []
for p in ckpt_paths:
if p.endswith('.gguf'):
sd = load_gguf_clip(p)
else:
print("Reading Clip: {}...".format(p))
with open(p, "rb") as file:
checkpoint_temp = file.read()
print("Reading Clip: {}...Done!".format(p))
sd = warped_load_torch_file(checkpoint_temp)
clip_data.append(sd)
return clip_data
def load_patcher(self, clip_paths, clip_type, clip_data):
clip = comfy.sd.load_text_encoder_state_dicts(clip_type=clip_type,
state_dicts=clip_data, model_options={'custom_operations':
GGMLOps, 'initial_device': comfy.model_management.
text_encoder_offload_device()}, embedding_directory=
folder_paths.get_folder_paths('embeddings'))
clip.patcher = GGUFModelPatcher.clone(clip.patcher)
return clip
def load_clip(self, clip_name, type='stable_diffusion', device='default'):
clip_path = folder_paths.get_full_path('clip', clip_name)
if clip_name.endswith('.safetensors') and device != 'default':
clip = comfy.sd.load_clip(ckpt_paths=[clip_path], embedding_directory=folder_paths.get_folder_paths("embeddings"), clip_type=get_clip_type(type), model_options=get_device(device))
return (clip,)
else:
return (self.load_patcher([clip_path], get_clip_type(type), self.load_data([clip_path])), get_device(device))
class WarpedDualClipLoaderGGUF(WarpedClipLoaderGGUF):
@classmethod
def INPUT_TYPES(s):
base = nodes.DualCLIPLoader.INPUT_TYPES()
file_options = s.get_filename_list(),
return {'required': {'clip_name1':file_options, 'clip_name2':file_options, 'type':
base['required']['type']},
'optional':{'device':(['default','cpu'],{'advanced':True}),}}
def load_clip(self, clip_name1, clip_name2, type, device='default'):
clip_path1 = folder_paths.get_full_path('clip', clip_name1)
clip_path2 = folder_paths.get_full_path('clip', clip_name2)
clip_paths = [clip_path1, clip_path2]
if device != 'default' and (clip_name1.endswith('.safetensors') and clip_name2.endswith('.safetensors')):
clip = comfy.sd.load_clip(ckpt_paths=clip_paths, embedding_directory=folder_paths.get_folder_paths("embeddings"), clip_type=get_clip_type(type), model_options=get_device(device))
return (clip,)
else:
return (self.load_patcher(clip_paths, get_clip_type(type), self.load_data(clip_paths)), get_device(device))
nodes.py
+2979 -12
View File
File diff suppressed because it is too large Load Diff
pig.py
+999
View File
@@ -0,0 +1,999 @@
import comfy.sd
import comfy.ops
import comfy.utils
import comfy.model_patcher
import comfy.model_management
import torch, numpy, os, json, logging, collections, nodes, folder_paths
from safetensors.torch import load_file, save_file
from typing import Dict, Tuple
from tqdm import tqdm as loading
from .gguf_connector import reader as gr
from .gguf_connector.writer import GGUFWriter, GGMLQuantizationType
from .gguf_connector.const import GGML_QUANT_VERSION, LlamaFileType
from .gguf_connector.quant import quantize, dequantize, QuantError
from .gguf_connector.quant2c import dequantize_tensor, is_quantized, is_torch_compatible
from .gguf_connector.tkn import get_field, tokenizer_builder
pig = os.path.join(os.path.dirname(__file__), 'version.json')
with open(pig, 'r') as file:
data = json.load(file)
arrays = {}
for key, value in data[0].items():
arrays[key] = value
class GGUFModelPatcher(comfy.model_patcher.ModelPatcher):
patch_on_device = False
def patch_weight_to_device(self, key, device_to=None, inplace_update=False):
if key not in self.patches:
return
weight = comfy.utils.get_attr(self.model, key)
try:
from comfy.lora import calculate_weight
except Exception:
calculate_weight = self.calculate_weight
patches = self.patches[key]
if is_quantized(weight):
out_weight = weight.to(device_to)
patches = load_patch_to_device(patches, self.load_device if
self.patch_on_device else self.offload_device)
out_weight.patches = [(calculate_weight, patches, key)]
else:
inplace_update = self.weight_inplace_update or inplace_update
if key not in self.backup:
self.backup[key] = collections.namedtuple('Dimension', [
'weight', 'inplace_update'])(weight.to(device=self.
offload_device, copy=inplace_update), inplace_update)
if device_to is not None:
temp_weight = comfy.model_management.cast_to_device(weight,
device_to, torch.float32, copy=True)
else:
temp_weight = weight.to(torch.float32, copy=True)
out_weight = calculate_weight(patches, temp_weight, key)
out_weight = comfy.float.stochastic_rounding(out_weight, weight
.dtype)
if inplace_update:
comfy.utils.copy_to_param(self.model, key, out_weight)
else:
comfy.utils.set_attr_param(self.model, key, out_weight)
def unpatch_model(self, device_to=None, unpatch_weights=True):
if unpatch_weights:
for p in self.model.parameters():
if is_torch_compatible(p):
continue
patches = getattr(p, 'patches', [])
if len(patches) > 0:
p.patches = []
return super().unpatch_model(device_to=device_to, unpatch_weights=
unpatch_weights)
mmap_released = False
def load(self, *args, force_patch_weights=False, **kwargs):
super().load(*args, force_patch_weights=True, **kwargs)
if not self.mmap_released:
linked = []
if kwargs.get('lowvram_model_memory', 0) > 0:
for n, m in self.model.named_modules():
if hasattr(m, 'weight'):
device = getattr(m.weight, 'device', None)
if device == self.offload_device:
linked.append((n, m))
continue
if hasattr(m, 'bias'):
device = getattr(m.bias, 'device', None)
if device == self.offload_device:
linked.append((n, m))
continue
if linked:
print(f'Attempting to release mmap ({len(linked)})')
for n, m in linked:
m.to(self.load_device).to(self.offload_device)
self.mmap_released = True
def clone(self, *args, **kwargs):
src_cls = self.__class__
self.__class__ = GGUFModelPatcher
n = super().clone(*args, **kwargs)
n.__class__ = GGUFModelPatcher
self.__class__ = src_cls
n.patch_on_device = getattr(self, 'patch_on_device', False)
return n
class GGMLTensor(torch.Tensor):
def __init__(self, *args, tensor_type, tensor_shape, patches=[], **kwargs):
super().__init__()
self.tensor_type = tensor_type
self.tensor_shape = tensor_shape
self.patches = patches
def __new__(cls, *args, tensor_type, tensor_shape, patches=[], **kwargs):
return super().__new__(cls, *args, **kwargs)
def to(self, *args, **kwargs):
new = super().to(*args, **kwargs)
new.tensor_type = getattr(self, 'tensor_type', None)
new.tensor_shape = getattr(self, 'tensor_shape', new.data.shape)
new.patches = getattr(self, 'patches', []).copy()
return new
def clone(self, *args, **kwargs):
return self
def detach(self, *args, **kwargs):
return self
def copy_(self, *args, **kwargs):
try:
return super().copy_(*args, **kwargs)
except Exception as e:
print(f"ignoring 'copy_' on tensor: {e}")
def empty_(self, size, *args, **kwargs):
new_tensor = super().empty_(size, *args, **kwargs)
return GGMLTensor(new_tensor, tensor_type=getattr(self,
'tensor_type', None), tensor_shape=size, patches=getattr(self,
'patches', []).copy())
@property
def shape(self):
if not hasattr(self, 'tensor_shape'):
self.tensor_shape = self.size()
return self.tensor_shape
if hasattr(torch, 'compiler') and hasattr(torch.compiler, 'disable'):
torch_compiler_disable = torch.compiler.disable
else:
def torch_compiler_disable(*args, **kwargs):
def noop(x):
return x
return noop
class GGMLLayer(torch.nn.Module):
comfy_cast_weights = True
dequant_dtype = None
patch_dtype = None
largest_layer = False
torch_compatible_tensor_types = {None, gr.GGMLQuantizationType.F32, gr.
GGMLQuantizationType.F16}
def is_ggml_quantized(self, *, weight=None, bias=None):
if weight is None:
weight = self.weight
if bias is None:
bias = self.bias
return is_quantized(weight) or is_quantized(bias)
def _load_from_state_dict(self, state_dict, prefix, *args, **kwargs):
weight, bias = state_dict.get(f'{prefix}weight'), state_dict.get(
f'{prefix}bias')
if self.is_ggml_quantized(weight=weight, bias=bias) or isinstance(self,
torch.nn.Linear):
return self.ggml_load_from_state_dict(state_dict, prefix, *args,
**kwargs)
return super()._load_from_state_dict(state_dict, prefix, *args, **
kwargs)
def ggml_load_from_state_dict(self, state_dict, prefix, local_metadata,
strict, missing_keys, unexpected_keys, error_msgs):
prefix_len = len(prefix)
for k, v in state_dict.items():
if k[prefix_len:] == 'weight':
self.weight = torch.nn.Parameter(v, requires_grad=False)
elif k[prefix_len:] == 'bias' and v is not None:
self.bias = torch.nn.Parameter(v, requires_grad=False)
else:
missing_keys.append(k)
if self.weight is None and isinstance(self, torch.nn.Linear):
v = torch.zeros(self.in_features, self.out_features)
self.weight = torch.nn.Parameter(v, requires_grad=False)
missing_keys.append(prefix + 'weight')
if getattr(self.weight, 'is_largest_weight', False):
self.largest_layer = True
def _save_to_state_dict(self, *args, **kwargs):
if self.is_ggml_quantized():
return self.ggml_save_to_state_dict(*args, **kwargs)
return super()._save_to_state_dict(*args, **kwargs)
def ggml_save_to_state_dict(self, destination, prefix, keep_vars):
weight = torch.zeros_like(self.weight, device=torch.device('meta'))
destination[prefix + 'weight'] = weight
if self.bias is not None:
bias = torch.zeros_like(self.bias, device=torch.device('meta'))
destination[prefix + 'bias'] = bias
if self.largest_layer:
shape = getattr(self.weight, 'tensor_shape', self.weight.shape)
dtype = self.dequant_dtype or torch.float16
temp = torch.empty(*shape, device=torch.device('meta'), dtype=dtype
)
destination[prefix + 'temp.weight'] = temp
return
destination[prefix + 'weight'] = self.get_weight(self.weight)
if bias is not None:
destination[prefix + 'bias'] = self.get_weight(self.bias)
def get_weight(self, tensor, dtype):
if tensor is None:
return
patch_list = []
device = tensor.device
for function, patches, key in getattr(tensor, 'patches', []):
patch_list += load_patch_to_device(patches, device)
weight = dequantize_tensor(tensor, dtype, self.dequant_dtype)
if isinstance(weight, GGMLTensor):
weight = torch.Tensor(weight)
if patch_list:
if self.patch_dtype is None:
weight = function(patch_list, weight, key)
else:
patch_dtype = (dtype if self.patch_dtype == 'target' else
self.patch_dtype)
weight = function(patch_list, weight, key, patch_dtype)
return weight
@torch_compiler_disable()
def cast_bias_weight(s, input=None, dtype=None, device=None, bias_dtype
=None):
if input is not None:
if dtype is None:
dtype = getattr(input, 'dtype', torch.float32)
if bias_dtype is None:
bias_dtype = dtype
if device is None:
device = input.device
bias = None
non_blocking = comfy.model_management.device_supports_non_blocking(
device)
if s.bias is not None:
bias = s.get_weight(s.bias.to(device), dtype)
bias = comfy.ops.cast_to(bias, bias_dtype, device, non_blocking
=non_blocking, copy=False)
weight = s.get_weight(s.weight.to(device), dtype)
weight = comfy.ops.cast_to(weight, dtype, device, non_blocking=
non_blocking, copy=False)
return weight, bias
def forward_comfy_cast_weights(self, input, *args, **kwargs):
if self.is_ggml_quantized():
out = self.forward_ggml_cast_weights(input, *args, **kwargs)
else:
out = super().forward_comfy_cast_weights(input, *args, **kwargs)
if isinstance(out, GGMLTensor):
out = torch.Tensor(out)
return out
def forward_ggml_cast_weights(self, input):
raise NotImplementedError
class GGMLOps(comfy.ops.manual_cast):
class Linear(GGMLLayer, comfy.ops.manual_cast.Linear):
def __init__(self, in_features, out_features, bias=True, device=
None, dtype=None):
torch.nn.Module.__init__(self)
self.in_features = in_features
self.out_features = out_features
self.weight = None
self.bias = None
def forward_ggml_cast_weights(self, input):
weight, bias = self.cast_bias_weight(input)
return torch.nn.functional.linear(input, weight, bias)
class Conv2d(GGMLLayer, comfy.ops.manual_cast.Conv2d):
def forward_ggml_cast_weights(self, input):
weight, bias = self.cast_bias_weight(input)
return self._conv_forward(input, weight, bias)
class Embedding(GGMLLayer, comfy.ops.manual_cast.Embedding):
def forward_ggml_cast_weights(self, input, out_dtype=None):
output_dtype = out_dtype
if (self.weight.dtype == torch.float16 or self.weight.dtype ==
torch.bfloat16):
out_dtype = None
weight, _bias = self.cast_bias_weight(self, device=input.device,
dtype=out_dtype)
return torch.nn.functional.embedding(input, weight, self.
padding_idx, self.max_norm, self.norm_type, self.
scale_grad_by_freq, self.sparse).to(dtype=output_dtype)
class LayerNorm(GGMLLayer, comfy.ops.manual_cast.LayerNorm):
def forward_ggml_cast_weights(self, input):
if self.weight is None:
return super().forward_comfy_cast_weights(input)
weight, bias = self.cast_bias_weight(input)
return torch.nn.functional.layer_norm(input, self.
normalized_shape, weight, bias, self.eps)
class GroupNorm(GGMLLayer, comfy.ops.manual_cast.GroupNorm):
def forward_ggml_cast_weights(self, input):
weight, bias = self.cast_bias_weight(input)
return torch.nn.functional.group_norm(input, self.num_groups,
weight, bias, self.eps)
def load_patch_to_device(item, device):
if isinstance(item, torch.Tensor):
return item.to(device, non_blocking=True)
elif isinstance(item, tuple):
return tuple(load_patch_to_device(x, device) for x in item)
elif isinstance(item, list):
return [load_patch_to_device(x, device) for x in item]
else:
return item
def get_folder_names_and_paths(key, targets=[]):
base = folder_paths.folder_names_and_paths.get(key, ([], {}))
base = base[0] if isinstance(base[0], (list, set, tuple)) else []
target = next((x for x in targets if x in folder_paths.
folder_names_and_paths), targets[0])
orig, _ = folder_paths.folder_names_and_paths.get(target, ([], {}))
folder_paths.folder_names_and_paths[key] = orig or base, {'.gguf'}
if base and base != orig:
logging.warning(
f'Unknown file list already present on key {key}: {base}')
get_folder_names_and_paths('model_gguf', ['diffusion_models', 'unet'])
get_folder_names_and_paths('clip_gguf', ['text_encoders', 'clip'])
get_folder_names_and_paths('vae_gguf', ['vae'])
def get_orig_shape(reader, tensor_name):
field_key = f'comfy.gguf.orig_shape.{tensor_name}'
field = reader.get_field(field_key)
if field is None:
return None
if len(field.types) != 2 or field.types[0
] != gr.GGUFValueType.ARRAY or field.types[1
] != gr.GGUFValueType.INT32:
raise TypeError(
f'Bad original shape metadata for {field_key}: Expected ARRAY of INT32, got {field.types}'
)
return torch.Size(tuple(int(field.parts[part_idx][0]) for part_idx in
field.data))
def load_gguf_sd(path, handle_prefix='model.diffusion_model.', return_arch=
False):
reader = gr.GGUFReader(path)
has_prefix = False
if handle_prefix is not None:
prefix_len = len(handle_prefix)
tensor_names = set(tensor.name for tensor in reader.tensors)
has_prefix = any(s.startswith(handle_prefix) for s in tensor_names)
tensors = []
for tensor in reader.tensors:
sd_key = tensor_name = tensor.name
if has_prefix:
if not tensor_name.startswith(handle_prefix):
continue
sd_key = tensor_name[prefix_len:]
tensors.append((sd_key, tensor))
compat = None
arch_str = get_field(reader, 'general.architecture', str)
if arch_str is None:
compat = 'sd.cpp'
elif arch_str not in arrays['PIG_ARCH_LIST'] and arch_str not in arrays['TXT_ARCH_LIST']:
raise ValueError(f"Unknown architecture: {arch_str!r}")
state_dict, qtype_dict = {}, {}
for sd_key, tensor in tensors:
tensor_name = tensor.name
torch_tensor = torch.from_numpy(tensor.data)
shape = get_orig_shape(reader, tensor_name)
if shape is None:
shape = torch.Size(tuple(int(v) for v in reversed(tensor.shape)))
if compat == 'sd.cpp' and arch_str == 'sdxl':
if any([tensor_name.endswith(x) for x in ('.proj_in.weight',
'.proj_out.weight')]):
while len(shape) > 2 and shape[-1] == 1:
shape = shape[:-1]
if tensor.tensor_type in {gr.GGMLQuantizationType.F32, gr.
GGMLQuantizationType.F16}:
torch_tensor = torch_tensor.view(*shape)
state_dict[sd_key] = GGMLTensor(torch_tensor, tensor_type=tensor.
tensor_type, tensor_shape=shape)
tensor_type_str = getattr(tensor.tensor_type, 'name', repr(tensor.tensor_type))
qtype_dict[tensor_type_str] = qtype_dict.get(tensor_type_str, 0) + 1
print('gguf qtypes: ' + ', '.join(f'{k} ({v})' for k, v in qtype_dict.items()))
qsd = {k: v for k, v in state_dict.items() if is_quantized(v)}
if len(qsd) > 0:
max_key = max(qsd.keys(), key=lambda k: qsd[k].numel())
state_dict[max_key].is_largest_weight = True
if return_arch:
return state_dict, arch_str
return state_dict
def tensor_swap(raw_sd, key_map):
sd = {}
for k, v in raw_sd.items():
for s, d in key_map.items():
k = k.replace(s, d)
sd[k] = v
return sd
def pig_work(raw_sd):
sd = {}
for k, v in raw_sd.items():
sd[k] = v
return sd
def llama_permute(raw_sd, n_head, n_head_kv):
sd = {}
permute = lambda x, h: x.reshape(h, x.shape[0] // h // 2, 2, *x.shape[1:]
).swapaxes(1, 2).reshape(x.shape)
for k, v in raw_sd.items():
if k.endswith(('q_proj.weight', 'q_proj.bias')):
v.data = permute(v.data, n_head)
if k.endswith(('k_proj.weight', 'k_proj.bias')):
v.data = permute(v.data, n_head_kv)
sd[k] = v
return sd
def load_gguf_clip(path):
sd, arch = load_gguf_sd(path, return_arch=True)
if arch in {'t5', 't5encoder'}:
temb_key = 'token_embd.weight'
if temb_key in sd and (sd[temb_key].shape == (256384, 4096) or sd[temb_key].shape == (256384, 768)):
sd['spiece_model'] = tokenizer_builder(path)
sd = tensor_swap(sd, arrays['T5'])
elif temb_key in sd and sd[temb_key].shape == (32128, 768):
sd = tensor_swap(sd, arrays['B5'])
else:
sd = tensor_swap(sd, arrays['T5'])
elif arch in {'llama'}:
sd = tensor_swap(sd, arrays['L3'])
sd = llama_permute(sd, 32, 8)
elif arch in {'gemma2'}:
sd["spiece_model"] = tokenizer_builder(path)
sd = tensor_swap(sd, arrays['G2'])
elif arch in {'pig'}:
sd = pig_work(sd)
else:
pass
return sd
class LoaderGGUF:
@classmethod
def INPUT_TYPES(s):
gguf_names = [x for x in folder_paths.get_filename_list('model_gguf')]
return {'required': {'gguf_name': (gguf_names,)}}
RETURN_TYPES = 'MODEL',
FUNCTION = 'load_model'
CATEGORY = 'gguf'
TITLE = 'GGUF Loader'
def load_model(self, gguf_name, dequant_dtype=None, patch_dtype=None,
patch_on_device=None):
ops = GGMLOps()
if dequant_dtype in ('default', None):
ops.Linear.dequant_dtype = None
elif dequant_dtype in ['target']:
ops.Linear.dequant_dtype = dequant_dtype
else:
ops.Linear.dequant_dtype = getattr(torch, dequant_dtype)
if patch_dtype in ('default', None):
ops.Linear.patch_dtype = None
elif patch_dtype in ['target']:
ops.Linear.patch_dtype = patch_dtype
else:
ops.Linear.patch_dtype = getattr(torch, patch_dtype)
model_path = folder_paths.get_full_path('unet', gguf_name)
sd = load_gguf_sd(model_path)
model = comfy.sd.load_diffusion_model_state_dict(sd, model_options=
{'custom_operations': ops})
if model is None:
logging.error('ERROR UNSUPPORTED MODEL {}'.format(model_path))
raise RuntimeError('ERROR: Could not detect model type of: {}'.
format(model_path))
model = GGUFModelPatcher.clone(model)
model.patch_on_device = patch_on_device
return model,
class LoaderGGUFAdvanced(LoaderGGUF):
@classmethod
def INPUT_TYPES(s):
model_names = [x for x in folder_paths.get_filename_list('model_gguf')]
return {'required': {'gguf_name': (model_names,), 'dequant_dtype':
(['default', 'target', 'float32', 'float16', 'bfloat16'], {
'default': 'default'}), 'patch_dtype': (['default', 'target',
'float32', 'float16', 'bfloat16'], {'default': 'default'}),
'patch_on_device': ('BOOLEAN', {'default': False})}}
TITLE = 'GGUF Loader (Advanced)'
def get_clip_type(type):
clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION)
return clip_type
def get_device(device):
model_options = {}
if device == "cpu":
model_options["load_device"] = model_options["offload_device"] = torch.device("cpu")
return model_options
class ClipLoaderGGUF:
@classmethod
def INPUT_TYPES(s):
base = nodes.CLIPLoader.INPUT_TYPES()
return {'required': {'clip_name': (s.get_filename_list(),), 'type':
base['required']['type']},
'optional':{'device':(['default','cpu'],{'advanced':True}),}}
RETURN_TYPES = 'CLIP',
FUNCTION = 'load_clip'
CATEGORY = 'gguf'
TITLE = 'GGUF CLIP Loader'
@classmethod
def get_filename_list(s):
files = []
files += folder_paths.get_filename_list('clip')
files += folder_paths.get_filename_list('clip_gguf')
return sorted(files)
def load_data(self, ckpt_paths):
clip_data = []
for p in ckpt_paths:
if p.endswith('.gguf'):
sd = load_gguf_clip(p)
else:
sd = comfy.utils.load_torch_file(p, safe_load=True)
clip_data.append(sd)
return clip_data
def load_patcher(self, clip_paths, clip_type, clip_data):
clip = comfy.sd.load_text_encoder_state_dicts(clip_type=clip_type,
state_dicts=clip_data, model_options={'custom_operations':
GGMLOps, 'initial_device': comfy.model_management.
text_encoder_offload_device()}, embedding_directory=
folder_paths.get_folder_paths('embeddings'))
clip.patcher = GGUFModelPatcher.clone(clip.patcher)
return clip
def load_clip(self, clip_name, type='stable_diffusion', device='default'):
clip_path = folder_paths.get_full_path('clip', clip_name)
if clip_name.endswith('.safetensors') and device != 'default':
clip = comfy.sd.load_clip(ckpt_paths=[clip_path], embedding_directory=folder_paths.get_folder_paths("embeddings"), clip_type=get_clip_type(type), model_options=get_device(device))
return (clip,)
else:
return (self.load_patcher([clip_path], get_clip_type(type), self.load_data([clip_path])), get_device(device))
class DualClipLoaderGGUF(ClipLoaderGGUF):
@classmethod
def INPUT_TYPES(s):
base = nodes.DualCLIPLoader.INPUT_TYPES()
file_options = s.get_filename_list(),
return {'required': {'clip_name1':file_options, 'clip_name2':file_options, 'type':
base['required']['type']},
'optional':{'device':(['default','cpu'],{'advanced':True}),}}
TITLE = 'GGUF DualCLIP Loader'
def load_clip(self, clip_name1, clip_name2, type, device='default'):
clip_path1 = folder_paths.get_full_path('clip', clip_name1)
clip_path2 = folder_paths.get_full_path('clip', clip_name2)
clip_paths = clip_path1, clip_path2
if device != 'default' and (clip_name1.endswith('.safetensors') and clip_name2.endswith('.safetensors')):
clip = comfy.sd.load_clip(ckpt_paths=clip_paths, embedding_directory=folder_paths.get_folder_paths("embeddings"), clip_type=get_clip_type(type), model_options=get_device(device))
return (clip,)
else:
return (self.load_patcher(clip_paths, get_clip_type(type), self.load_data(clip_paths)), get_device(device))
class TripleClipLoaderGGUF(ClipLoaderGGUF):
@classmethod
def INPUT_TYPES(s):
file_options = s.get_filename_list(),
return {'required': {'clip_name1': file_options, 'clip_name2':
file_options, 'clip_name3': file_options}}
TITLE = 'GGUF TripleCLIP Loader'
def load_clip(self, clip_name1, clip_name2, clip_name3, type='sd3'):
clip_path1 = folder_paths.get_full_path('clip', clip_name1)
clip_path2 = folder_paths.get_full_path('clip', clip_name2)
clip_path3 = folder_paths.get_full_path('clip', clip_name3)
clip_paths = clip_path1, clip_path2, clip_path3
return (self.load_patcher(clip_paths, get_clip_type(type), self.load_data(clip_paths)),)
class QuadrupleClipLoaderGGUF(ClipLoaderGGUF):
@classmethod
def INPUT_TYPES(s):
file_options = s.get_filename_list(),
return {'required': {'clip_name1': file_options, 'clip_name2':
file_options, 'clip_name3': file_options, 'clip_name4': file_options}}
TITLE = 'GGUF QuadrupleCLIP Loader'
def load_clip(self, clip_name1, clip_name2, clip_name3, clip_name4, type='hunyuan_video'):
clip_path1 = folder_paths.get_full_path('clip', clip_name1)
clip_path2 = folder_paths.get_full_path('clip', clip_name2)
clip_path3 = folder_paths.get_full_path('clip', clip_name3)
clip_path4 = folder_paths.get_full_path('clip', clip_name4)
clip_paths = clip_path1, clip_path2, clip_path3, clip_path4
return (self.load_patcher(clip_paths, get_clip_type(type), self.load_data(clip_paths)),)
QUANTIZATION_THRESHOLD = 1024
REARRANGE_THRESHOLD = 512
MAX_TENSOR_NAME_LENGTH = 127
class ModelTemplate:
arch = 'invalid'
keys_detect = []
keys_banned = []
class ModelFlux(ModelTemplate):
arch = 'flux'
keys_detect = [('transformer_blocks.0.attn.norm_added_k.weight',), (
'double_blocks.0.img_attn.proj.weight',)]
keys_banned = ['transformer_blocks.0.attn.norm_added_k.weight']
class ModelSD3(ModelTemplate):
arch = 'sd3'
keys_detect = [('transformer_blocks.0.attn.add_q_proj.weight',), (
'joint_blocks.0.x_block.attn.qkv.weight',)]
keys_banned = ['transformer_blocks.0.attn.add_q_proj.weight']
class ModelSDXL(ModelTemplate):
arch = 'sdxl'
keys_detect = [('down_blocks.0.downsamplers.0.conv.weight',
'add_embedding.linear_1.weight'), ('input_blocks.3.0.op.weight',
'input_blocks.6.0.op.weight', 'output_blocks.2.2.conv.weight',
'output_blocks.5.2.conv.weight'), ('label_emb.0.0.weight',)]
class ModelSD1(ModelTemplate):
arch = 'sd1'
keys_detect = [('down_blocks.0.downsamplers.0.conv.weight',), (
'input_blocks.3.0.op.weight', 'input_blocks.6.0.op.weight',
'input_blocks.9.0.op.weight', 'output_blocks.2.1.conv.weight',
'output_blocks.5.2.conv.weight', 'output_blocks.8.2.conv.weight')]
arch_list = [ModelFlux, ModelSD3, ModelSDXL, ModelSD1]
def is_model_arch(model, state_dict):
matched = False
invalid = False
for match_list in model.keys_detect:
if all(key in state_dict for key in match_list):
matched = True
invalid = any(key in state_dict for key in model.keys_banned)
break
assert not invalid, 'Model architecture not supported for alpha (please opt to use zero)'
return matched
def detect_arch(state_dict):
model_arch = None
for arch in arch_list:
if is_model_arch(arch, state_dict):
model_arch = arch
break
assert model_arch is not None, 'Unknown model architecture detected!'
return model_arch
def load_state_dict(path):
state_dict = load_file(path)
prefix = None
for pfx in ['model.diffusion_model.', 'model.', 'net.']:
if any([x.startswith(pfx) for x in state_dict.keys()]):
prefix = pfx
break
sd = {}
for k, v in state_dict.items():
if prefix and prefix not in k:
continue
if prefix:
k = k.replace(prefix, '')
sd[k] = v
return sd
def load_model(path):
state_dict = load_state_dict(path)
model_arch = detect_arch(state_dict)
print(f'* Architecture detected from input: {model_arch.arch}')
writer = GGUFWriter(path=None, arch=model_arch.arch)
return writer, state_dict, model_arch
def load_pig_state(path):
pig_state = load_file(path)
sd = {}
for k, v in pig_state.items():
sd[k] = v
return sd
def load_pig(path):
state_dict = load_pig_state(path)
model_arch = arrays['TXT_ARCH_LIST'][0]
writer = GGUFWriter(path=None, arch=model_arch)
return writer, state_dict, model_arch
def handle_tensors(args, writer, state_dict, model_arch):
name_lengths = tuple(sorted(((key, len(key)) for key in state_dict.keys
()), key=lambda item: item[1], reverse=True))
if not name_lengths:
return
max_name_len = name_lengths[0][1]
if max_name_len > MAX_TENSOR_NAME_LENGTH:
bad_list = ', '.join(f'{key!r} ({namelen})' for key, namelen in
name_lengths if namelen > MAX_TENSOR_NAME_LENGTH)
raise ValueError(
f'Can only handle tensor names up to {MAX_TENSOR_NAME_LENGTH} characters. Tensors exceeding the limit: {bad_list}'
)
for key, data in loading(state_dict.items()):
old_dtype = data.dtype
if data.dtype == torch.bfloat16:
data = data.to(torch.float32).numpy()
elif data.dtype in [getattr(torch, 'float8_e4m3fn', '_invalid'),
getattr(torch, 'float8_e5m2', '_invalid')]:
data = data.to(torch.float16).numpy()
else:
data = data.numpy()
n_dims = len(data.shape)
data_shape = data.shape
data_qtype = getattr(GGMLQuantizationType, 'BF16' if old_dtype ==
torch.bfloat16 else 'F16')
n_params = 1
for dim_size in data_shape:
n_params *= dim_size
blacklist = {'time_embedding.', 'add_embedding.', 'time_in.',
'txt_in.', 'vector_in.', 'img_in.', 'guidance_in.', 'final_layer.'}
if old_dtype in (torch.float32, torch.bfloat16):
if n_dims == 1:
data_qtype = GGMLQuantizationType.F32
elif n_params <= QUANTIZATION_THRESHOLD:
data_qtype = GGMLQuantizationType.F32
elif '.weight' in key and any(x in key for x in blacklist):
data_qtype = GGMLQuantizationType.F32
try:
data = quantize(data, data_qtype)
except (AttributeError, QuantError) as e:
loading.write(f'falling back to F16: {e}')
data_qtype = GGMLQuantizationType.F16
data = quantize(data, data_qtype)
new_name = key
shape_str = f"{{{', '.join(str(n) for n in reversed(data.shape))}}}"
loading.write(
f"{f'%-{max_name_len + 4}s' % f'{new_name}'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}"
)
writer.add_tensor(new_name, data, raw_dtype=data_qtype)
if 'select_safetensors' not in folder_paths.folder_names_and_paths:
orig = folder_paths.folder_names_and_paths.get('diffusion_models',
folder_paths.folder_names_and_paths.get('checkpoints', [[], set()]))
folder_paths.folder_names_and_paths['select_safetensors'] = orig[0], {
'.safetensors'}
class GGUFSave:
def __init__(self):
self.output_dir = folder_paths.get_output_directory()
@classmethod
def INPUT_TYPES(s):
return {'required': {'select_safetensors': (s.get_filename_list(),)}}
RETURN_TYPES = ()
FUNCTION = 'save'
OUTPUT_NODE = True
CATEGORY = 'gguf'
TITLE = 'GGUF Convertor (Alpha)'
@classmethod
def get_filename_list(s):
files = []
files += folder_paths.get_filename_list('select_safetensors')
return sorted(files)
def save(self, select_safetensors):
path = folder_paths.get_full_path('select_safetensors',
select_safetensors)
writer, state_dict, model_arch = load_model(path)
writer.add_quantization_version(GGML_QUANT_VERSION)
if next(iter(state_dict.values())).dtype == torch.bfloat16:
output_path = (
f'{self.output_dir}/{os.path.splitext(select_safetensors)[0]}-bf16.gguf'
)
writer.add_file_type(LlamaFileType.MOSTLY_BF16)
else:
output_path = (
f'{self.output_dir}/{os.path.splitext(select_safetensors)[0]}-f16.gguf'
)
writer.add_file_type(LlamaFileType.MOSTLY_F16)
if os.path.isfile(output_path):
input('Output exists enter to continue or ctrl+c to abort!')
handle_tensors(output_path, writer, state_dict, model_arch)
writer.write_header_to_file(path=output_path)
writer.write_kv_data_to_file()
writer.write_tensors_to_file(progress=True)
writer.close()
return {}
class GGUFRun:
def __init__(self):
self.output_dir = folder_paths.get_output_directory()
@classmethod
def INPUT_TYPES(s):
return {'required': {'select_safetensors': (s.get_filename_list(),)}}
RETURN_TYPES = ()
FUNCTION = 'run'
OUTPUT_NODE = True
CATEGORY = 'gguf'
TITLE = 'GGUF Convertor (Zero)'
@classmethod
def get_filename_list(s):
files = []
files += folder_paths.get_filename_list('select_safetensors')
return sorted(files)
def run(self, select_safetensors):
path = folder_paths.get_full_path('select_safetensors',
select_safetensors)
writer, state_dict, model_arch = load_pig(path)
writer.add_quantization_version(GGML_QUANT_VERSION)
if next(iter(state_dict.values())).dtype == torch.bfloat16:
output_path = (
f'{self.output_dir}/{os.path.splitext(select_safetensors)[0]}-bf16.gguf'
)
writer.add_file_type(LlamaFileType.MOSTLY_BF16)
else:
output_path = (
f'{self.output_dir}/{os.path.splitext(select_safetensors)[0]}-f16.gguf'
)
writer.add_file_type(LlamaFileType.MOSTLY_F16)
if os.path.isfile(output_path):
input('Output exists enter to continue or ctrl+c to abort!')
handle_tensors(output_path, writer, state_dict, model_arch)
writer.write_header_to_file(path=output_path)
writer.write_kv_data_to_file()
writer.write_tensors_to_file(progress=True)
writer.close()
return {}
def quantize_to_fp8(tensor):
if tensor.dtype != torch.bfloat16:
raise ValueError('Input tensor must be in BF16 format.')
tensor = tensor.to(torch.float16)
fp8_max = 240.0
fp8_min = -fp8_max
clamped_tensor = tensor.clamp(min=fp8_min, max=fp8_max)
scale = fp8_max / torch.max(torch.abs(clamped_tensor))
quantized_tensor = torch.round(clamped_tensor * scale) / scale
return quantized_tensor.to(torch.float8_e4m3fn)
class TENSORCut:
def __init__(self):
self.output_dir = folder_paths.get_output_directory()
@classmethod
def INPUT_TYPES(s):
return {'required': {'select_safetensors': (s.get_filename_list(),)}}
RETURN_TYPES = ()
FUNCTION = 'cut'
OUTPUT_NODE = True
CATEGORY = 'gguf'
TITLE = 'TENSOR Cutter (Beta)'
@classmethod
def get_filename_list(s):
files = []
files += folder_paths.get_filename_list('select_safetensors')
return sorted(files)
def cut(self, select_safetensors):
input_file = folder_paths.get_full_path('select_safetensors',
select_safetensors)
output_file = (
f'{self.output_dir}/{os.path.splitext(select_safetensors)[0]}_fp8_e4m3fn.safetensors'
)
data = load_file(input_file)
quantized_data = {}
print('Starting quantization process...')
for key, tensor in loading(data.items(), desc='Quantizing tensors',
unit='tensor'):
tensor = tensor.to(dtype=torch.bfloat16, device='cuda')
quantized_tensor = quantize_to_fp8(tensor)
quantized_data[key] = quantized_tensor.cpu()
save_file(quantized_data, output_file)
print(f'Quantized safetensors saved to {output_file}.')
return {}
class TENSORBoost:
def __init__(self):
self.output_dir = folder_paths.get_output_directory()
@classmethod
def INPUT_TYPES(s):
return {'required': {'select_safetensors': (s.get_filename_list(),)}}
RETURN_TYPES = ()
FUNCTION = 'boost'
OUTPUT_NODE = True
CATEGORY = 'gguf'
TITLE = 'TENSOR Booster'
@classmethod
def get_filename_list(s):
files = []
files += folder_paths.get_filename_list('select_safetensors')
return sorted(files)
def boost(self, select_safetensors):
input_file = folder_paths.get_full_path('select_safetensors',
select_safetensors)
output_file = (
f'{self.output_dir}/{os.path.splitext(select_safetensors)[0]}_fp32.safetensors'
)
data = load_file(input_file)
quantized_data = {}
print('Starting quantization process...')
for key, tensor in loading(data.items(), desc="Converting tensors", unit="tensor"):
quantized_data[key] = tensor.to(torch.float32)
save_file(quantized_data, output_file)
print(f'Quantized safetensors saved to {output_file}.')
return {}
def load_gguf_and_extract_metadata(gguf_path):
reader = gr.GGUFReader(gguf_path)
tensors_metadata = []
for tensor in reader.tensors:
tensor_metadata = {'name': tensor.name, 'shape': tuple(tensor.shape
.tolist()), 'n_elements': tensor.n_elements, 'n_bytes': tensor.
n_bytes, 'data_offset': tensor.data_offset, 'type': tensor.
tensor_type}
tensors_metadata.append(tensor_metadata)
return reader, tensors_metadata
def convert_gguf_to_safetensors(gguf_path, output_path, use_bf16):
reader, tensors_metadata = load_gguf_and_extract_metadata(gguf_path)
print(f'Extracted {len(tensors_metadata)} tensors from GGUF file')
tensors_dict: dict[str, torch.Tensor] = {}
for i, tensor_info in enumerate(loading(tensors_metadata, desc=
'Converting tensors', unit='tensor')):
tensor_name = tensor_info['name']
tensor_data = reader.get_tensor(i)
weights = dequantize(tensor_data.data, tensor_data.tensor_type).copy()
try:
if use_bf16:
weights_tensor = torch.from_numpy(weights).to(dtype=torch.
float32)
weights_tensor = weights_tensor.to(torch.bfloat16)
else:
weights_tensor = torch.from_numpy(weights).to(dtype=torch.
float16)
weights_hf = weights_tensor
except Exception as e:
print(
f"Error during BF16 conversion for tensor '{tensor_name}': {e}"
)
weights_tensor = torch.from_numpy(weights.astype(numpy.float32)
).to(torch.float16)
weights_hf = weights_tensor
tensors_dict[tensor_name] = weights_hf
metadata = {key: str(reader.get_field(key)) for key in reader.fields}
save_file(tensors_dict, output_path, metadata=metadata)
print('Conversion complete!')
if 'select_gguf' not in folder_paths.folder_names_and_paths:
orig = folder_paths.folder_names_and_paths.get('diffusion_models',
folder_paths.folder_names_and_paths.get('unet', [[], set()]))
folder_paths.folder_names_and_paths['select_gguf'] = orig[0], {'.gguf'}
class GGUFUndo:
def __init__(self):
self.output_dir = folder_paths.get_output_directory()
@classmethod
def INPUT_TYPES(s):
return {'required': {'select_gguf': (s.get_filename_list(),)}}
RETURN_TYPES = ()
FUNCTION = 'undo'
OUTPUT_NODE = True
CATEGORY = 'gguf'
TITLE = 'GGUF Convertor (Reverse)'
@classmethod
def get_filename_list(s):
files = []
files += folder_paths.get_filename_list('select_gguf')
return sorted(files)
def undo(self, select_gguf):
in_file = folder_paths.get_full_path('select_gguf', select_gguf)
out_file = (f'{self.output_dir}/{os.path.splitext(select_gguf)[0]}_fp16.safetensors')
use_bf16 = False
convert_gguf_to_safetensors(in_file, out_file, use_bf16)
return {}
class VaeGGUF:
@staticmethod
def vae_list():
vaes = []
vaes += folder_paths.get_filename_list('vae')
vaes += folder_paths.get_filename_list('vae_gguf')
approx_vaes = folder_paths.get_filename_list('vae_approx')
sdxl_taesd_enc = False
sdxl_taesd_dec = False
sd1_taesd_enc = False
sd1_taesd_dec = False
sd3_taesd_enc = False
sd3_taesd_dec = False
f1_taesd_enc = False
f1_taesd_dec = False
for v in approx_vaes:
if v.startswith('taesd_decoder.'):
sd1_taesd_dec = True
elif v.startswith('taesd_encoder.'):
sd1_taesd_enc = True
elif v.startswith('taesdxl_decoder.'):
sdxl_taesd_dec = True
elif v.startswith('taesdxl_encoder.'):
sdxl_taesd_enc = True
elif v.startswith('taesd3_decoder.'):
sd3_taesd_dec = True
elif v.startswith('taesd3_encoder.'):
sd3_taesd_enc = True
elif v.startswith('taef1_encoder.'):
f1_taesd_dec = True
elif v.startswith('taef1_decoder.'):
f1_taesd_enc = True
if sd1_taesd_dec and sd1_taesd_enc:
vaes.append('taesd')
if sdxl_taesd_dec and sdxl_taesd_enc:
vaes.append('taesdxl')
if sd3_taesd_dec and sd3_taesd_enc:
vaes.append('taesd3')
if f1_taesd_dec and f1_taesd_enc:
vaes.append('taef1')
return vaes
@staticmethod
def load_taesd(name):
sd = {}
approx_vaes = folder_paths.get_filename_list('vae_approx')
encoder = next(filter(lambda a: a.startswith('{}_encoder.'.format(name)), approx_vaes))
decoder = next(filter(lambda a: a.startswith('{}_decoder.'.format(name)), approx_vaes))
enc = comfy.utils.load_torch_file(folder_paths.
get_full_path_or_raise('vae_approx', encoder))
for k in enc:
sd['taesd_encoder.{}'.format(k)] = enc[k]
dec = comfy.utils.load_torch_file(folder_paths.
get_full_path_or_raise('vae_approx', decoder))
for k in dec:
sd['taesd_decoder.{}'.format(k)] = dec[k]
if name == 'taesd':
sd['vae_scale'] = torch.tensor(0.18215)
sd['vae_shift'] = torch.tensor(0.0)
elif name == 'taesdxl':
sd['vae_scale'] = torch.tensor(0.13025)
sd['vae_shift'] = torch.tensor(0.0)
elif name == 'taesd3':
sd['vae_scale'] = torch.tensor(1.5305)
sd['vae_shift'] = torch.tensor(0.0609)
elif name == 'taef1':
sd['vae_scale'] = torch.tensor(0.3611)
sd['vae_shift'] = torch.tensor(0.1159)
return sd
@classmethod
def INPUT_TYPES(s):
return {'required': {'vae_name': (s.vae_list(),)}}
RETURN_TYPES = 'VAE',
FUNCTION = 'load_vae'
CATEGORY = 'gguf'
TITLE = 'GGUF VAE Loader'
def load_vae(self, vae_name):
if vae_name.endswith('.gguf'):
vae_path = folder_paths.get_full_path_or_raise('vae_gguf', vae_name)
sd = load_gguf_clip(vae_path)
elif vae_name in ['taesd', 'taesdxl', 'taesd3', 'taef1']:
sd = self.load_taesd(vae_name)
else:
vae_path = folder_paths.get_full_path_or_raise('vae', vae_name)
sd = comfy.utils.load_torch_file(vae_path)
vae = comfy.sd.VAE(sd=sd)
return vae,
NODE_CLASS_MAPPINGS = {
"LoaderGGUF": LoaderGGUF,
"ClipLoaderGGUF": ClipLoaderGGUF,
"DualClipLoaderGGUF": DualClipLoaderGGUF,
"TripleClipLoaderGGUF": TripleClipLoaderGGUF,
"QuadrupleClipLoaderGGUF": QuadrupleClipLoaderGGUF,
"LoaderGGUFAdvanced": LoaderGGUFAdvanced,
"VaeGGUF": VaeGGUF,
"GGUFUndo": GGUFUndo,
"GGUFSave": GGUFSave,
"GGUFRun": GGUFRun,
"TENSORCut": TENSORCut,
"TENSORBoost": TENSORBoost,
}
pyproject.toml
+1 -1
View File
@@ -1,7 +1,7 @@
[project]
name = "ComfyUI-WarpedToolset"
description = "A toolset for Hunyuan Video (mainly), with some additional nodes applicable to Framepack Video and WAN Video."
version = "1.7.0"
version = "1.7.1"
license = { file = "LICENSE" }
[project.urls]
version.json
+1
View File
@@ -0,0 +1 @@
[{"version":"0.0.58","PIG_ARCH_LIST":["pig","fatfat","piggy","piglet","gguf","frame","sd1","sdxl","sd3","flux","aura","mochi","ltxv","hyvid","cosmos","pixart","lumina","wan","hidream"],"TXT_ARCH_LIST":["pig","t5","t5encoder","llama","gemma2"],"L3":{"blk.":"model.layers.","attn_norm":"input_layernorm","attn_q":"self_attn.q_proj","attn_k":"self_attn.k_proj","attn_v":"self_attn.v_proj","attn_output":"self_attn.o_proj","ffn_up":"mlp.up_proj","ffn_down":"mlp.down_proj","ffn_gate":"mlp.gate_proj","ffn_norm":"post_attention_layernorm","token_embd":"model.embed_tokens","output_norm":"model.norm","output.weight":"lm_head.weight"},"T5":{"enc.":"encoder.",".blk.":".block.","token_embd":"shared","output_norm":"final_layer_norm","attn_q":"layer.0.SelfAttention.q","attn_k":"layer.0.SelfAttention.k","attn_v":"layer.0.SelfAttention.v","attn_o":"layer.0.SelfAttention.o","attn_norm":"layer.0.layer_norm","attn_rel_b":"layer.0.SelfAttention.relative_attention_bias","ffn_up":"layer.1.DenseReluDense.wi_1","ffn_down":"layer.1.DenseReluDense.wo","ffn_gate":"layer.1.DenseReluDense.wi_0","ffn_norm":"layer.1.layer_norm"},"B5":{"enc.":"encoder.",".blk.":".block.","token_embd":"shared","output_norm":"final_layer_norm","attn_q":"layer.0.SelfAttention.q","attn_k":"layer.0.SelfAttention.k","attn_v":"layer.0.SelfAttention.v","attn_o":"layer.0.SelfAttention.o","attn_norm":"layer.0.layer_norm","attn_rel_b":"layer.0.SelfAttention.relative_attention_bias","ffn_up":"layer.1.DenseReluDense.wi","ffn_down":"layer.1.DenseReluDense.wo","ffn_norm":"layer.1.layer_norm"},"G2":{"gemma2_2b.transformer.":""}}]