From 8d313a677a4551414989010fc0a4d916650c8c55 Mon Sep 17 00:00:00 2001 From: City <125218114+city96@users.noreply.github.com> Date: Wed, 8 Jan 2025 17:17:49 +0100 Subject: [PATCH 01/15] Unused/old logic --- tools/convert.py | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/tools/convert.py b/tools/convert.py index aa6c961..3015abf 100644 --- a/tools/convert.py +++ b/tools/convert.py @@ -140,7 +140,7 @@ def load_model(path): writer = gguf.GGUFWriter(path=None, arch=model_arch.arch) return (writer, state_dict, model_arch) -def handle_tensors(args, writer, state_dict, model_arch): +def handle_tensors(writer, state_dict, model_arch): name_lengths = tuple(sorted( ((key, len(key)) for key in state_dict.keys()), key=lambda item: item[1], @@ -175,18 +175,6 @@ def handle_tensors(args, writer, state_dict, model_arch): for dim_size in data_shape: n_params *= dim_size - # keys to keep as max precision - blacklist = { - "time_embedding.", - "add_embedding.", - "time_in.", - "txt_in.", - "vector_in.", - "img_in.", - "guidance_in.", - "final_layer.", - } - if old_dtype in (torch.float32, torch.bfloat16): if n_dims == 1: # one-dimensional tensors should be kept in F32 @@ -197,9 +185,6 @@ def handle_tensors(args, writer, state_dict, model_arch): # very small tensors data_qtype = gguf.GGMLQuantizationType.F32 - elif ".weight" in key and any(x in key for x in blacklist): - data_qtype = gguf.GGMLQuantizationType.F32 - if (model_arch.shape_fix # NEVER reshape for models such as flux and n_dims > 1 # Skip one-dimensional tensors and n_params >= REARRANGE_THRESHOLD # Only rearrange tensors meeting the size requirement @@ -241,7 +226,7 @@ if __name__ == "__main__": if os.path.isfile(out_path): input("Output exists enter to continue or ctrl+c to abort!") - handle_tensors(path, writer, state_dict, model_arch) + handle_tensors(writer, state_dict, model_arch) writer.write_header_to_file(path=out_path) writer.write_kv_data_to_file() writer.write_tensors_to_file(progress=True) From 2d46bcd61b0c4fd612c657c9e3b7b4f0448ced1d Mon Sep 17 00:00:00 2001 From: City <125218114+city96@users.noreply.github.com> Date: Wed, 8 Jan 2025 17:45:37 +0100 Subject: [PATCH 02/15] Sanity checks in conversion code --- tools/convert.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tools/convert.py b/tools/convert.py index 3015abf..274f01d 100644 --- a/tools/convert.py +++ b/tools/convert.py @@ -10,6 +10,7 @@ from safetensors.torch import load_file QUANTIZATION_THRESHOLD = 1024 REARRANGE_THRESHOLD = 512 MAX_TENSOR_NAME_LENGTH = 127 +MAX_TENSOR_DIMS = 4 class ModelTemplate: arch = "invalid" # string describing architecture @@ -17,6 +18,9 @@ class ModelTemplate: keys_detect = [] # list of lists to match in state dict keys_banned = [] # list of keys that should mark model as invalid for conversion + def handle_nd_tensor(self, key, data): + raise NotImplementedError(f"Tensor detected that exceeds dims supported by C++ code! ({key} @ {data.shape})") + class ModelFlux(ModelTemplate): arch = "flux" keys_detect = [ @@ -93,7 +97,7 @@ def detect_arch(state_dict): model_arch = None for arch in arch_list: if is_model_arch(arch, state_dict): - model_arch = arch + model_arch = arch() break assert model_arch is not None, "Unknown model architecture!" return model_arch @@ -113,6 +117,8 @@ def load_state_dict(path): if any(path.endswith(x) for x in [".ckpt", ".pt", ".bin", ".pth"]): state_dict = torch.load(path, map_location="cpu", weights_only=True) state_dict = state_dict.get("model", state_dict) + if len(state_dict) < 20: + raise RuntimeError(f"pt subkey load failed: {state_dict.keys()}") else: state_dict = load_file(path) @@ -170,6 +176,11 @@ def handle_tensors(writer, state_dict, model_arch): "BF16" if old_dtype == torch.bfloat16 else "F16" ) + # The max no. of dimensions that can be handled by the quantization code is 4 + if len(data.shape) > MAX_TENSOR_DIMS: + model_arch.handle_nd_tensor(key, data) + continue # needs to be added back later + # get number of parameters (AKA elements) in this tensor n_params = 1 for dim_size in data_shape: From abdfec54c4b929eeca5cff3bf99545969f0d3026 Mon Sep 17 00:00:00 2001 From: City <125218114+city96@users.noreply.github.com> Date: Thu, 20 Feb 2025 05:10:27 +0100 Subject: [PATCH 03/15] Hunyuan Jank --- tools/README.md | 7 +- tools/convert.py | 20 ++- tools/fix_5d_tensors.py | 46 ++++++ tools/lcpp.patch | 298 +++++++++++++++++++++++++++--------- tools/lcpp_sd3.patch | 324 ---------------------------------------- 5 files changed, 294 insertions(+), 401 deletions(-) create mode 100644 tools/fix_5d_tensors.py delete mode 100644 tools/lcpp_sd3.patch diff --git a/tools/README.md b/tools/README.md index fb962c3..5b040ba 100644 --- a/tools/README.md +++ b/tools/README.md @@ -15,12 +15,10 @@ python convert.py --src E:\models\unet\flux1-dev.safetensors To quantize the model, first apply the provided patch to the llama.cpp repo you've just cloned. If you get a "corrupt patch" error, you may have to [change the line endings in the patch file](https://github.com/city96/ComfyUI-GGUF/issues/90#issuecomment-2323011648). ``` cd llama.cpp -git checkout tags/b3600 +git checkout tags/b3962 git apply ..\lcpp.patch ``` -If you wish to quantize **SD3** or **AuraFlow** models, you should use the patch named `lcpp_sd3.patch`, which has slightly modified logic for quantizing. For this you'll want to target `tags/b3962` instead. - Then, compile the llama-quantize binary. This example uses cmake, on linux you can just use make. ``` @@ -41,6 +39,9 @@ llama.cpp\build\bin\Debug\llama-quantize.exe E:\models\unet\flux1-dev-BF16.gguf You can extract the patch again with `git diff src\llama.cpp > lcpp.patch` if you wish to change something and contribute back. +> [!WARNING] +>For hunyuan video, you will have to uncomment the block in convert.py that deals with 5D tensors. This will save a **non functional** model to disk first, that you can quantize. After quantization, run `fix_5d_tensor.py` to add back the missing key that was saved by the conversion code. You will have to edit this file to set the correct paths/architecture. This may change in the future. + > [!WARNING] > Do not use the diffusers UNET for flux, it won't work, use the default/reference checkpoint format. This is due to q/k/v being merged into one qkv key. You can convert it by loading it in ComfyUI and saving it using the built-in "ModelSave" node. diff --git a/tools/convert.py b/tools/convert.py index 274f01d..61e1592 100644 --- a/tools/convert.py +++ b/tools/convert.py @@ -45,6 +45,24 @@ class ModelAura(ModelTemplate): ] keys_banned = ["joint_transformer_blocks.3.ff_context.out_projection.weight",] +class ModelHyVid(ModelTemplate): + arch = "hyvid" + keys_detect = [ + ( + "double_blocks.0.img_attn_proj.weight", + "txt_in.individual_token_refiner.blocks.1.self_attn_qkv.weight", + ) + ] + + # def handle_nd_tensor(self, key, data): + # # hacky but this is the only arch that uses it + # path = f"./fix_5d_tensors_{self.arch}.pt" + # if os.path.isfile(path): + # raise RuntimeError(f"5D tensor fix file already exists! {path}") + # fsd = {key: data} + # tqdm.write(f"5D key found in state dict! Manual fix required! - {key} {data.shape}") + # torch.save(fsd, path) + class ModelLTXV(ModelTemplate): arch = "ltxv" keys_detect = [ @@ -79,7 +97,7 @@ class ModelSD1(ModelTemplate): ] # The architectures are checked in order and the first successful match terminates the search. -arch_list = [ModelFlux, ModelSD3, ModelAura, ModelLTXV, ModelSDXL, ModelSD1] +arch_list = [ModelFlux, ModelSD3, ModelAura, ModelLTXV, ModelHyVid, ModelSDXL, ModelSD1] def is_model_arch(model, state_dict): # check if model is correct diff --git a/tools/fix_5d_tensors.py b/tools/fix_5d_tensors.py new file mode 100644 index 0000000..7465cb4 --- /dev/null +++ b/tools/fix_5d_tensors.py @@ -0,0 +1,46 @@ +import gguf +import torch +from tqdm import tqdm + +arch = "hyvid" # TODO: Really should autodetect this +prec = "Q4_K_S" # edit manually for each step +src = fr"raw/hunyuan-video-t2v-720p-{prec}.gguf" +dst = fr"hunyuan-video-t2v-720p-{prec}.gguf" + +sd5d = torch.load(f"./fix_5d_tensors_{arch}.pt") +print("5D:", sd5d.keys()) + +reader = gguf.GGUFReader(src) +writer = gguf.GGUFWriter(path=None, arch=arch) + +writer.add_quantization_version(gguf.GGML_QUANT_VERSION) +writer.add_file_type(getattr(gguf.LlamaFileType, f"MOSTLY_{prec}")) # TODO: also autodetect + +added = [] +def add_extra_key(writer, key, data): + old_dtype = data.dtype + data_qtype = gguf.GGMLQuantizationType.F32 + n_dims = len(data.shape) + data_shape = data.shape + data = gguf.quants.quantize(data, data_qtype) + tqdm.write(f"Adding key {key} ({data_shape})") + writer.add_tensor(key, data, raw_dtype=data_qtype) + global added + added.append(key) + +# main loop to add missing +for tensor in tqdm(reader.tensors): + writer.add_tensor(tensor.name, tensor.data, raw_dtype=tensor.tensor_type) + key5d = tensor.name.replace(".bias", ".weight") + if key5d in sd5d.keys(): + add_extra_key(writer, key5d, sd5d[key5d]) + +# brute force for any missed +for key, data in sd5d.items(): + if key not in added: + add_extra_key(writer, key, data) + +writer.write_header_to_file(path=dst) +writer.write_kv_data_to_file() +writer.write_tensors_to_file(progress=True) +writer.close() diff --git a/tools/lcpp.patch b/tools/lcpp.patch index e7bfbff..2706a51 100644 --- a/tools/lcpp.patch +++ b/tools/lcpp.patch @@ -1,67 +1,115 @@ diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h -index 1d2a3540..b1a9ee96 100644 +index de3c706f..0267c1fa 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h -@@ -230,7 +230,7 @@ - #define GGML_MAX_CONTEXTS 64 - #define GGML_MAX_SRC 10 - #ifndef GGML_MAX_NAME --#define GGML_MAX_NAME 64 -+#define GGML_MAX_NAME 128 - #endif +@@ -223,7 +223,7 @@ #define GGML_MAX_OP_PARAMS 64 + + #ifndef GGML_MAX_NAME +-# define GGML_MAX_NAME 64 ++# define GGML_MAX_NAME 128 + #endif + #define GGML_DEFAULT_N_THREADS 4 +@@ -2449,6 +2449,7 @@ extern "C" { + + // manage tensor info + GGML_API void gguf_add_tensor(struct gguf_context * ctx, const struct ggml_tensor * tensor); ++ GGML_API void gguf_set_tensor_ndim(struct gguf_context * ctx, const char * name, int n_dim); + GGML_API void gguf_set_tensor_type(struct gguf_context * ctx, const char * name, enum ggml_type type); + GGML_API void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const void * data, size_t size); + +diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c +index b16c462f..6d1568f1 100644 +--- a/ggml/src/ggml.c ++++ b/ggml/src/ggml.c +@@ -22960,6 +22960,14 @@ void gguf_add_tensor( + ctx->header.n_tensors++; + } + ++void gguf_set_tensor_ndim(struct gguf_context * ctx, const char * name, const int n_dim) { ++ const int idx = gguf_find_tensor(ctx, name); ++ if (idx < 0) { ++ GGML_ABORT("tensor not found"); ++ } ++ ctx->infos[idx].n_dims = n_dim; ++} ++ + void gguf_set_tensor_type(struct gguf_context * ctx, const char * name, enum ggml_type type) { + const int idx = gguf_find_tensor(ctx, name); + if (idx < 0) { diff --git a/src/llama.cpp b/src/llama.cpp -index 5ab65ea9..35580d9d 100644 +index 24e1f1f0..c2142c13 100644 --- a/src/llama.cpp +++ b/src/llama.cpp -@@ -212,6 +212,9 @@ enum llm_arch { - LLM_ARCH_JAIS, - LLM_ARCH_NEMOTRON, - LLM_ARCH_EXAONE, +@@ -205,6 +205,13 @@ enum llm_arch { + LLM_ARCH_GRANITE, + LLM_ARCH_GRANITE_MOE, + LLM_ARCH_CHAMELEON, + LLM_ARCH_FLUX, + LLM_ARCH_SD1, + LLM_ARCH_SDXL, ++ LLM_ARCH_SD3, ++ LLM_ARCH_AURA, ++ LLM_ARCH_LTXV, ++ LLM_ARCH_HYVID, LLM_ARCH_UNKNOWN, }; - -@@ -259,6 +262,9 @@ static const std::map LLM_ARCH_NAMES = { - { LLM_ARCH_JAIS, "jais" }, - { LLM_ARCH_NEMOTRON, "nemotron" }, - { LLM_ARCH_EXAONE, "exaone" }, + +@@ -258,6 +265,13 @@ static const std::map LLM_ARCH_NAMES = { + { LLM_ARCH_GRANITE, "granite" }, + { LLM_ARCH_GRANITE_MOE, "granitemoe" }, + { LLM_ARCH_CHAMELEON, "chameleon" }, + { LLM_ARCH_FLUX, "flux" }, + { LLM_ARCH_SD1, "sd1" }, + { LLM_ARCH_SDXL, "sdxl" }, ++ { LLM_ARCH_SD3, "sd3" }, ++ { LLM_ARCH_AURA, "aura" }, ++ { LLM_ARCH_LTXV, "ltxv" }, ++ { LLM_ARCH_HYVID, "hyvid" }, { LLM_ARCH_UNKNOWN, "(unknown)" }, }; - -@@ -1337,6 +1343,9 @@ static const std::map> LLM_TENSOR_NA - { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, + +@@ -1531,6 +1545,13 @@ static const std::map> LLM_TENSOR_N + { LLM_TENSOR_ATTN_K_NORM, "blk.%d.attn_k_norm" }, }, }, + { LLM_ARCH_FLUX, {}}, + { LLM_ARCH_SD1, {}}, + { LLM_ARCH_SDXL, {}}, ++ { LLM_ARCH_SD3, {}}, ++ { LLM_ARCH_AURA, {}}, ++ { LLM_ARCH_LTXV, {}}, ++ { LLM_ARCH_HYVID,{}}, { LLM_ARCH_UNKNOWN, { -@@ -4629,6 +4638,12 @@ static void llm_load_hparams( +@@ -5403,6 +5424,21 @@ static void llm_load_hparams( // get general kv ml.get_key(LLM_KV_GENERAL_NAME, model.name, false); - + + // Disable LLM metadata for image models -+ if (model.arch == LLM_ARCH_FLUX || model.arch == LLM_ARCH_SD1 || model.arch == LLM_ARCH_SDXL) { -+ model.ftype = ml.ftype; -+ return; ++ switch (model.arch) { ++ case LLM_ARCH_FLUX: ++ case LLM_ARCH_SD1: ++ case LLM_ARCH_SDXL: ++ case LLM_ARCH_SD3: ++ case LLM_ARCH_AURA: ++ case LLM_ARCH_LTXV: ++ case LLM_ARCH_HYVID: ++ model.ftype = ml.ftype; ++ return; ++ default: ++ break; + } + // get hparams kv ml.get_key(LLM_KV_VOCAB_SIZE, hparams.n_vocab, false) || ml.get_arr_n(LLM_KV_TOKENIZER_LIST, hparams.n_vocab); - -@@ -15827,11 +15842,162 @@ static void llama_tensor_dequantize_internal( + +@@ -18016,6 +18052,126 @@ static void llama_tensor_dequantize_internal( workers.clear(); } - + +static ggml_type img_tensor_get_type(quantize_state_internal & qs, ggml_type new_type, const ggml_tensor * tensor, llama_ftype ftype) { + // Special function for quantizing image model tensors + const std::string name = ggml_get_name(tensor); @@ -71,7 +119,8 @@ index 5ab65ea9..35580d9d 100644 + if ( + (name.find("model.diffusion_model.") != std::string::npos) || + (name.find("first_stage_model.") != std::string::npos) || -+ (name.find("single_transformer_blocks.") != std::string::npos) ++ (name.find("single_transformer_blocks.") != std::string::npos) || ++ (name.find("joint_transformer_blocks.") != std::string::npos) + ) { + throw std::runtime_error("Invalid input GGUF file. This is not a supported UNET model"); + } @@ -87,41 +136,11 @@ index 5ab65ea9..35580d9d 100644 + throw std::runtime_error("Invalid quantization type for image model (Not supported)"); + } + -+ if ( // Tensors to keep in FP32 precision -+ (arch == LLM_ARCH_FLUX) && ( -+ (name.find("img_in.") != std::string::npos) || -+ (name.find("time_in.in_layer.") != std::string::npos) || -+ (name.find("vector_in.in_layer.") != std::string::npos) || -+ (name.find("guidance_in.in_layer.") != std::string::npos) || -+ (name.find("final_layer.linear.") != std::string::npos) -+ ) || (arch == LLM_ARCH_SD1 || arch == LLM_ARCH_SDXL) && ( -+ (name.find("conv_in.") != std::string::npos) || -+ (name.find("conv_out.") != std::string::npos) || -+ (name == "input_blocks.0.0.weight") || -+ (name == "out.2.weight") -+ )) { -+ new_type = GGML_TYPE_F32; -+ } else if ( // Tensors to keep in FP16 precision -+ (arch == LLM_ARCH_FLUX) && ( -+ (name.find("txt_in.") != std::string::npos) || -+ (name.find("time_in.") != std::string::npos) || -+ (name.find("vector_in.") != std::string::npos) || -+ (name.find("guidance_in.") != std::string::npos) || -+ (name.find("final_layer.") != std::string::npos) -+ ) || (arch == LLM_ARCH_SD1 || arch == LLM_ARCH_SDXL) && ( -+ (name.find("class_embedding.") != std::string::npos) || -+ (name.find("time_embedding.") != std::string::npos) || -+ (name.find("add_embedding.") != std::string::npos) || -+ (name.find("time_embed.") != std::string::npos) || -+ (name.find("label_emb.") != std::string::npos) || -+ (name.find("proj_in.") != std::string::npos) || -+ (name.find("proj_out.") != std::string::npos) -+ // (name.find("conv_shortcut.") != std::string::npos) // marginal improvement -+ )) { -+ new_type = GGML_TYPE_F16; -+ } else if ( // Rules for to_v attention ++ if ( // Rules for to_v attention + (name.find("attn_v.weight") != std::string::npos) || -+ (name.find(".to_v.weight") != std::string::npos) ++ (name.find(".to_v.weight") != std::string::npos) || ++ (name.find(".attn.w1v.weight") != std::string::npos) || ++ (name.find(".attn.w2v.weight") != std::string::npos) + ){ + if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) { + new_type = GGML_TYPE_Q3_K; @@ -154,7 +173,7 @@ index 5ab65ea9..35580d9d 100644 + } + } else if ( // Rules for ffn + (name.find("ffn_down") != std::string::npos) || -+ (name.find("DenseReluDense.wo") != std::string::npos) ++ (name.find(".ff.net.2.weight") != std::string::npos) + ) { + // TODO: add back `layer_info` with some model specific logic + logic further down + if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) { @@ -210,14 +229,147 @@ index 5ab65ea9..35580d9d 100644 + } + return new_type; +} -+ + static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type new_type, const ggml_tensor * tensor, llama_ftype ftype) { const std::string name = ggml_get_name(tensor); - - // TODO: avoid hardcoded tensor names - use the TN_* constants - const llm_arch arch = qs.model.arch; -+ if (arch == LLM_ARCH_FLUX || arch == LLM_ARCH_SD1 || arch == LLM_ARCH_SDXL) { return img_tensor_get_type(qs, new_type, tensor, ftype); }; - const auto tn = LLM_TN(arch); - - auto use_more_bits = [](int i_layer, int n_layers) -> bool { + +@@ -18513,7 +18669,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s + if (llama_model_has_encoder(&model)) { + n_attn_layer *= 3; + } +- GGML_ASSERT((qs.n_attention_wv == n_attn_layer) && "n_attention_wv is unexpected"); ++ if (model.arch != LLM_ARCH_HYVID) { // TODO: Check why this fails ++ GGML_ASSERT((qs.n_attention_wv == n_attn_layer) && "n_attention_wv is unexpected"); ++ } + } + + size_t total_size_org = 0; +@@ -18547,6 +18705,36 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s + ctx_outs[i_split] = gguf_init_empty(); + } + gguf_add_tensor(ctx_outs[i_split], tensor); ++ // SD3 pos_embed needs special fix as first dim is 1, which gets truncated here ++ if (model.arch == LLM_ARCH_SD3) { ++ const std::string name = ggml_get_name(tensor); ++ if (name == "pos_embed" && tensor->ne[2] == 1) { ++ const int n_dim = 3; ++ gguf_set_tensor_ndim(ctx_outs[i_split], "pos_embed", n_dim); ++ LLAMA_LOG_INFO("\n%s: Correcting pos_embed shape for SD3: [key:%s]\n", __func__, tensor->name); ++ } ++ } ++ // same goes for auraflow ++ if (model.arch == LLM_ARCH_AURA) { ++ const std::string name = ggml_get_name(tensor); ++ if (name == "positional_encoding" && tensor->ne[2] == 1) { ++ const int n_dim = 3; ++ gguf_set_tensor_ndim(ctx_outs[i_split], "positional_encoding", n_dim); ++ LLAMA_LOG_INFO("\n%s: Correcting positional_encoding shape for AuraFlow: [key:%s]\n", __func__, tensor->name); ++ } ++ if (name == "register_tokens" && tensor->ne[2] == 1) { ++ const int n_dim = 3; ++ gguf_set_tensor_ndim(ctx_outs[i_split], "register_tokens", n_dim); ++ LLAMA_LOG_INFO("\n%s: Correcting register_tokens shape for AuraFlow: [key:%s]\n", __func__, tensor->name); ++ } ++ } ++ // conv3d fails due to max dims - unsure what to do here as we never even reach this check ++ if (model.arch == LLM_ARCH_HYVID) { ++ const std::string name = ggml_get_name(tensor); ++ if (name == "img_in.proj.weight" && tensor->ne[5] != 1 ) { ++ throw std::runtime_error("img_in.proj.weight size failed for HyVid"); ++ } ++ } + } + + // Set split info if needed +@@ -18647,6 +18835,73 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s + // do not quantize relative position bias (T5) + quantize &= name.find("attn_rel_b.weight") == std::string::npos; + ++ // rules for image models ++ bool image_model = false; ++ if (model.arch == LLM_ARCH_FLUX) { ++ image_model = true; ++ quantize &= name.find("txt_in.") == std::string::npos; ++ quantize &= name.find("img_in.") == std::string::npos; ++ quantize &= name.find("time_in.") == std::string::npos; ++ quantize &= name.find("vector_in.") == std::string::npos; ++ quantize &= name.find("guidance_in.") == std::string::npos; ++ quantize &= name.find("final_layer.") == std::string::npos; ++ } ++ if (model.arch == LLM_ARCH_SD1 || model.arch == LLM_ARCH_SDXL) { ++ image_model = true; ++ quantize &= name.find("class_embedding.") == std::string::npos; ++ quantize &= name.find("time_embedding.") == std::string::npos; ++ quantize &= name.find("add_embedding.") == std::string::npos; ++ quantize &= name.find("time_embed.") == std::string::npos; ++ quantize &= name.find("label_emb.") == std::string::npos; ++ quantize &= name.find("conv_in.") == std::string::npos; ++ quantize &= name.find("conv_out.") == std::string::npos; ++ quantize &= name != "input_blocks.0.0.weight"; ++ quantize &= name != "out.2.weight"; ++ } ++ if (model.arch == LLM_ARCH_SD3) { ++ image_model = true; ++ quantize &= name.find("final_layer.") == std::string::npos; ++ quantize &= name.find("time_text_embed.") == std::string::npos; ++ quantize &= name.find("context_embedder.") == std::string::npos; ++ quantize &= name.find("t_embedder.") == std::string::npos; ++ quantize &= name.find("y_embedder.") == std::string::npos; ++ quantize &= name.find("x_embedder.") == std::string::npos; ++ quantize &= name != "proj_out.weight"; ++ quantize &= name != "pos_embed"; ++ } ++ if (model.arch == LLM_ARCH_AURA) { ++ image_model = true; ++ quantize &= name.find("t_embedder.") == std::string::npos; ++ quantize &= name.find("init_x_linear.") == std::string::npos; ++ quantize &= name != "modF.1.weight"; ++ quantize &= name != "cond_seq_linear.weight"; ++ quantize &= name != "final_linear.weight"; ++ quantize &= name != "final_linear.weight"; ++ quantize &= name != "positional_encoding"; ++ quantize &= name != "register_tokens"; ++ } ++ if (model.arch == LLM_ARCH_LTXV) { ++ image_model = true; ++ quantize &= name.find("adaln_single.") == std::string::npos; ++ quantize &= name.find("caption_projection.") == std::string::npos; ++ quantize &= name.find("patchify_proj.") == std::string::npos; ++ quantize &= name.find("proj_out.") == std::string::npos; ++ quantize &= name.find("scale_shift_table") == std::string::npos; // last block too ++ } ++ if (model.arch == LLM_ARCH_HYVID) { ++ image_model = true; ++ quantize &= name.find("txt_in.") == std::string::npos; ++ quantize &= name.find("img_in.") == std::string::npos; ++ quantize &= name.find("time_in.") == std::string::npos; ++ quantize &= name.find("vector_in.") == std::string::npos; ++ quantize &= name.find("guidance_in.") == std::string::npos; ++ quantize &= name.find("final_layer.") == std::string::npos; ++ } ++ // ignore 3D/4D tensors for image models as the code was never meant to handle these ++ if (image_model) { ++ quantize &= ggml_n_dims(tensor) == 2; ++ } ++ + enum ggml_type new_type; + void * new_data; + size_t new_size; +@@ -18655,6 +18910,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s + new_type = default_type; + + // get more optimal quantization type based on the tensor shape, layer, etc. ++ if (image_model) { ++ new_type = img_tensor_get_type(qs, new_type, tensor, ftype); ++ } else { + if (!params->pure && ggml_is_quantized(default_type)) { + new_type = llama_tensor_get_type(qs, new_type, tensor, ftype); + } +@@ -18664,6 +18922,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s + if (params->output_tensor_type < GGML_TYPE_COUNT && strcmp(tensor->name, "output.weight") == 0) { + new_type = params->output_tensor_type; + } ++ } + + // If we've decided to quantize to the same type the tensor is already + // in then there's nothing to do. diff --git a/tools/lcpp_sd3.patch b/tools/lcpp_sd3.patch deleted file mode 100644 index 2367944..0000000 --- a/tools/lcpp_sd3.patch +++ /dev/null @@ -1,324 +0,0 @@ -diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h -index de3c706f..0267c1fa 100644 ---- a/ggml/include/ggml.h -+++ b/ggml/include/ggml.h -@@ -223,7 +223,7 @@ - #define GGML_MAX_OP_PARAMS 64 - - #ifndef GGML_MAX_NAME --# define GGML_MAX_NAME 64 -+# define GGML_MAX_NAME 128 - #endif - - #define GGML_DEFAULT_N_THREADS 4 -@@ -2449,6 +2449,7 @@ extern "C" { - - // manage tensor info - GGML_API void gguf_add_tensor(struct gguf_context * ctx, const struct ggml_tensor * tensor); -+ GGML_API void gguf_set_tensor_ndim(struct gguf_context * ctx, const char * name, int n_dim); - GGML_API void gguf_set_tensor_type(struct gguf_context * ctx, const char * name, enum ggml_type type); - GGML_API void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const void * data, size_t size); - -diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c -index b16c462f..6d1568f1 100644 ---- a/ggml/src/ggml.c -+++ b/ggml/src/ggml.c -@@ -22960,6 +22960,14 @@ void gguf_add_tensor( - ctx->header.n_tensors++; - } - -+void gguf_set_tensor_ndim(struct gguf_context * ctx, const char * name, const int n_dim) { -+ const int idx = gguf_find_tensor(ctx, name); -+ if (idx < 0) { -+ GGML_ABORT("tensor not found"); -+ } -+ ctx->infos[idx].n_dims = n_dim; -+} -+ - void gguf_set_tensor_type(struct gguf_context * ctx, const char * name, enum ggml_type type) { - const int idx = gguf_find_tensor(ctx, name); - if (idx < 0) { -diff --git a/src/llama.cpp b/src/llama.cpp -index 24e1f1f0..aeccc173 100644 ---- a/src/llama.cpp -+++ b/src/llama.cpp -@@ -205,6 +205,11 @@ enum llm_arch { - LLM_ARCH_GRANITE, - LLM_ARCH_GRANITE_MOE, - LLM_ARCH_CHAMELEON, -+ LLM_ARCH_FLUX, -+ LLM_ARCH_SD1, -+ LLM_ARCH_SDXL, -+ LLM_ARCH_SD3, -+ LLM_ARCH_AURA, - LLM_ARCH_UNKNOWN, - }; - -@@ -258,6 +263,11 @@ static const std::map LLM_ARCH_NAMES = { - { LLM_ARCH_GRANITE, "granite" }, - { LLM_ARCH_GRANITE_MOE, "granitemoe" }, - { LLM_ARCH_CHAMELEON, "chameleon" }, -+ { LLM_ARCH_FLUX, "flux" }, -+ { LLM_ARCH_SD1, "sd1" }, -+ { LLM_ARCH_SDXL, "sdxl" }, -+ { LLM_ARCH_SD3, "sd3" }, -+ { LLM_ARCH_AURA, "aura" }, - { LLM_ARCH_UNKNOWN, "(unknown)" }, - }; - -@@ -1531,6 +1541,11 @@ static const std::map> LLM_TENSOR_N - { LLM_TENSOR_ATTN_K_NORM, "blk.%d.attn_k_norm" }, - }, - }, -+ { LLM_ARCH_FLUX, {}}, -+ { LLM_ARCH_SD1, {}}, -+ { LLM_ARCH_SDXL, {}}, -+ { LLM_ARCH_SD3, {}}, -+ { LLM_ARCH_AURA, {}}, - { - LLM_ARCH_UNKNOWN, - { -@@ -5403,6 +5418,12 @@ static void llm_load_hparams( - // get general kv - ml.get_key(LLM_KV_GENERAL_NAME, model.name, false); - -+ // Disable LLM metadata for image models -+ if (model.arch == LLM_ARCH_FLUX || model.arch == LLM_ARCH_SD1 || model.arch == LLM_ARCH_SDXL || model.arch == LLM_ARCH_SD3 || model.arch == LLM_ARCH_AURA) { -+ model.ftype = ml.ftype; -+ return; -+ } -+ - // get hparams kv - ml.get_key(LLM_KV_VOCAB_SIZE, hparams.n_vocab, false) || ml.get_arr_n(LLM_KV_TOKENIZER_LIST, hparams.n_vocab); - -@@ -18016,6 +18037,125 @@ static void llama_tensor_dequantize_internal( - workers.clear(); - } - -+static ggml_type img_tensor_get_type(quantize_state_internal & qs, ggml_type new_type, const ggml_tensor * tensor, llama_ftype ftype) { -+ // Special function for quantizing image model tensors -+ const std::string name = ggml_get_name(tensor); -+ const llm_arch arch = qs.model.arch; -+ -+ // Sanity check -+ if ( -+ (name.find("model.diffusion_model.") != std::string::npos) || -+ (name.find("first_stage_model.") != std::string::npos) || -+ (name.find("single_transformer_blocks.") != std::string::npos) || -+ (name.find("joint_transformer_blocks.") != std::string::npos) -+ ) { -+ throw std::runtime_error("Invalid input GGUF file. This is not a supported UNET model"); -+ } -+ -+ // Unsupported quant types - exclude all IQ quants for now -+ if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || -+ ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M || -+ ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || -+ ftype == LLAMA_FTYPE_MOSTLY_IQ1_M || ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL || -+ ftype == LLAMA_FTYPE_MOSTLY_IQ4_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_S || -+ ftype == LLAMA_FTYPE_MOSTLY_IQ3_M || ftype == LLAMA_FTYPE_MOSTLY_Q4_0_4_4 || -+ ftype == LLAMA_FTYPE_MOSTLY_Q4_0_4_8 || ftype == LLAMA_FTYPE_MOSTLY_Q4_0_8_8) { -+ throw std::runtime_error("Invalid quantization type for image model (Not supported)"); -+ } -+ -+ if ( // Rules for to_v attention -+ (name.find("attn_v.weight") != std::string::npos) || -+ (name.find(".to_v.weight") != std::string::npos) || -+ (name.find(".attn.w1v.weight") != std::string::npos) || -+ (name.find(".attn.w2v.weight") != std::string::npos) -+ ){ -+ if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) { -+ new_type = GGML_TYPE_Q3_K; -+ } -+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) { -+ new_type = qs.i_attention_wv < 2 ? GGML_TYPE_Q5_K : GGML_TYPE_Q4_K; -+ } -+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) { -+ new_type = GGML_TYPE_Q5_K; -+ } -+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) { -+ new_type = GGML_TYPE_Q6_K; -+ } -+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S && qs.i_attention_wv < 4) { -+ new_type = GGML_TYPE_Q5_K; -+ } -+ ++qs.i_attention_wv; -+ } else if ( // Rules for fused qkv attention -+ (name.find("attn_qkv.weight") != std::string::npos) || -+ (name.find("attn.qkv.weight") != std::string::npos) -+ ) { -+ if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) { -+ new_type = GGML_TYPE_Q4_K; -+ } -+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) { -+ new_type = GGML_TYPE_Q5_K; -+ } -+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) { -+ new_type = GGML_TYPE_Q6_K; -+ } -+ } else if ( // Rules for ffn -+ (name.find("ffn_down") != std::string::npos) -+ ) { -+ // TODO: add back `layer_info` with some model specific logic + logic further down -+ if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) { -+ new_type = GGML_TYPE_Q4_K; -+ } -+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) { -+ new_type = GGML_TYPE_Q5_K; -+ } -+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S) { -+ new_type = GGML_TYPE_Q5_K; -+ } -+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) { -+ new_type = GGML_TYPE_Q6_K; -+ } -+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) { -+ new_type = GGML_TYPE_Q6_K; -+ } -+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_0) { -+ new_type = GGML_TYPE_Q4_1; -+ } -+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q5_0) { -+ new_type = GGML_TYPE_Q5_1; -+ } -+ ++qs.i_ffn_down; -+ } -+ -+ // Sanity check for row shape -+ bool convert_incompatible_tensor = false; -+ if (new_type == GGML_TYPE_Q2_K || new_type == GGML_TYPE_Q3_K || new_type == GGML_TYPE_Q4_K || -+ new_type == GGML_TYPE_Q5_K || new_type == GGML_TYPE_Q6_K) { -+ int nx = tensor->ne[0]; -+ int ny = tensor->ne[1]; -+ if (nx % QK_K != 0) { -+ LLAMA_LOG_WARN("\n\n%s : tensor cols %d x %d are not divisible by %d, required for %s", __func__, nx, ny, QK_K, ggml_type_name(new_type)); -+ convert_incompatible_tensor = true; -+ } else { -+ ++qs.n_k_quantized; -+ } -+ } -+ if (convert_incompatible_tensor) { -+ // TODO: Possibly reenable this in the future -+ // switch (new_type) { -+ // case GGML_TYPE_Q2_K: -+ // case GGML_TYPE_Q3_K: -+ // case GGML_TYPE_Q4_K: new_type = GGML_TYPE_Q5_0; break; -+ // case GGML_TYPE_Q5_K: new_type = GGML_TYPE_Q5_1; break; -+ // case GGML_TYPE_Q6_K: new_type = GGML_TYPE_Q8_0; break; -+ // default: throw std::runtime_error("\nUnsupported tensor size encountered\n"); -+ // } -+ new_type = GGML_TYPE_F16; -+ LLAMA_LOG_WARN(" - using fallback quantization %s\n", ggml_type_name(new_type)); -+ ++qs.n_fallback; -+ } -+ return new_type; -+} -+ - static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type new_type, const ggml_tensor * tensor, llama_ftype ftype) { - const std::string name = ggml_get_name(tensor); - -@@ -18547,6 +18687,29 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s - ctx_outs[i_split] = gguf_init_empty(); - } - gguf_add_tensor(ctx_outs[i_split], tensor); -+ // SD3 pos_embed needs special fix as first dim is 1, which gets truncated here -+ if (model.arch == LLM_ARCH_SD3) { -+ const std::string name = ggml_get_name(tensor); -+ if (name == "pos_embed" && tensor->ne[2] == 1) { -+ const int n_dim = 3; -+ gguf_set_tensor_ndim(ctx_outs[i_split], "pos_embed", n_dim); -+ LLAMA_LOG_INFO("\n%s: Correcting pos_embed shape for SD3: [key:%s]\n", __func__, tensor->name); -+ } -+ } -+ // same goes for auraflow -+ if (model.arch == LLM_ARCH_AURA) { -+ const std::string name = ggml_get_name(tensor); -+ if (name == "positional_encoding" && tensor->ne[2] == 1) { -+ const int n_dim = 3; -+ gguf_set_tensor_ndim(ctx_outs[i_split], "positional_encoding", n_dim); -+ LLAMA_LOG_INFO("\n%s: Correcting positional_encoding shape for AuraFlow: [key:%s]\n", __func__, tensor->name); -+ } -+ if (name == "register_tokens" && tensor->ne[2] == 1) { -+ const int n_dim = 3; -+ gguf_set_tensor_ndim(ctx_outs[i_split], "register_tokens", n_dim); -+ LLAMA_LOG_INFO("\n%s: Correcting register_tokens shape for AuraFlow: [key:%s]\n", __func__, tensor->name); -+ } -+ } - } - - // Set split info if needed -@@ -18647,6 +18810,56 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s - // do not quantize relative position bias (T5) - quantize &= name.find("attn_rel_b.weight") == std::string::npos; - -+ // rules for image models -+ bool image_model = false; -+ if (model.arch == LLM_ARCH_FLUX) { -+ image_model = true; -+ quantize &= name.find("txt_in.") == std::string::npos; -+ quantize &= name.find("img_in.") == std::string::npos; -+ quantize &= name.find("time_in.") == std::string::npos; -+ quantize &= name.find("vector_in.") == std::string::npos; -+ quantize &= name.find("guidance_in.") == std::string::npos; -+ quantize &= name.find("final_layer.") == std::string::npos; -+ } -+ if (model.arch == LLM_ARCH_SD1 || model.arch == LLM_ARCH_SDXL) { -+ image_model = true; -+ quantize &= name.find("class_embedding.") == std::string::npos; -+ quantize &= name.find("time_embedding.") == std::string::npos; -+ quantize &= name.find("add_embedding.") == std::string::npos; -+ quantize &= name.find("time_embed.") == std::string::npos; -+ quantize &= name.find("label_emb.") == std::string::npos; -+ quantize &= name.find("conv_in.") == std::string::npos; -+ quantize &= name.find("conv_out.") == std::string::npos; -+ quantize &= name != "input_blocks.0.0.weight"; -+ quantize &= name != "out.2.weight"; -+ } -+ if (model.arch == LLM_ARCH_SD3) { -+ image_model = true; -+ quantize &= name.find("final_layer.") == std::string::npos; -+ quantize &= name.find("time_text_embed.") == std::string::npos; -+ quantize &= name.find("context_embedder.") == std::string::npos; -+ quantize &= name.find("t_embedder.") == std::string::npos; -+ quantize &= name.find("y_embedder.") == std::string::npos; -+ quantize &= name.find("x_embedder.") == std::string::npos; -+ quantize &= name != "proj_out.weight"; -+ quantize &= name != "pos_embed"; -+ } -+ if (model.arch == LLM_ARCH_AURA) { -+ image_model = true; -+ quantize &= name.find("t_embedder.") == std::string::npos; -+ quantize &= name.find("init_x_linear.") == std::string::npos; -+ quantize &= name != "modF.1.weight"; -+ quantize &= name != "cond_seq_linear.weight"; -+ quantize &= name != "final_linear.weight"; -+ quantize &= name != "final_linear.weight"; -+ quantize &= name != "positional_encoding"; -+ quantize &= name != "register_tokens"; -+ } -+ // ignore 3D/4D tensors for image models as the code was never meant to handle these -+ if (image_model) { -+ quantize &= ggml_n_dims(tensor) == 2; -+ } -+ - enum ggml_type new_type; - void * new_data; - size_t new_size; -@@ -18655,6 +18868,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s - new_type = default_type; - - // get more optimal quantization type based on the tensor shape, layer, etc. -+ if (image_model) { -+ new_type = img_tensor_get_type(qs, new_type, tensor, ftype); -+ } else { - if (!params->pure && ggml_is_quantized(default_type)) { - new_type = llama_tensor_get_type(qs, new_type, tensor, ftype); - } -@@ -18664,6 +18880,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s - if (params->output_tensor_type < GGML_TYPE_COUNT && strcmp(tensor->name, "output.weight") == 0) { - new_type = params->output_tensor_type; - } -+ } - - // If we've decided to quantize to the same type the tensor is already - // in then there's nothing to do. From 355b43228ced543c361b67f77b1238ed884de0d8 Mon Sep 17 00:00:00 2001 From: City <125218114+city96@users.noreply.github.com> Date: Wed, 26 Feb 2025 07:06:54 +0100 Subject: [PATCH 04/15] Wan2.1 convert logic --- tools/convert.py | 14 ++++++++++++-- tools/lcpp.patch | 46 +++++++++++++++++++++++++++++++++++----------- 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/tools/convert.py b/tools/convert.py index 61e1592..001cd86 100644 --- a/tools/convert.py +++ b/tools/convert.py @@ -55,7 +55,7 @@ class ModelHyVid(ModelTemplate): ] # def handle_nd_tensor(self, key, data): - # # hacky but this is the only arch that uses it + # # hacky but don't have any better ideas # path = f"./fix_5d_tensors_{self.arch}.pt" # if os.path.isfile(path): # raise RuntimeError(f"5D tensor fix file already exists! {path}") @@ -63,6 +63,16 @@ class ModelHyVid(ModelTemplate): # tqdm.write(f"5D key found in state dict! Manual fix required! - {key} {data.shape}") # torch.save(fsd, path) +class ModelWan(ModelHyVid): + arch = "wan" + keys_detect = [ + ( + "blocks.0.self_attn.norm_q.weight", + "text_embedding.2.weight", + "head.modulation", + ) + ] + class ModelLTXV(ModelTemplate): arch = "ltxv" keys_detect = [ @@ -97,7 +107,7 @@ class ModelSD1(ModelTemplate): ] # The architectures are checked in order and the first successful match terminates the search. -arch_list = [ModelFlux, ModelSD3, ModelAura, ModelLTXV, ModelHyVid, ModelSDXL, ModelSD1] +arch_list = [ModelFlux, ModelSD3, ModelAura, ModelLTXV, ModelHyVid, ModelWan, ModelSDXL, ModelSD1] def is_model_arch(model, state_dict): # check if model is correct diff --git a/tools/lcpp.patch b/tools/lcpp.patch index 2706a51..0ad92d1 100644 --- a/tools/lcpp.patch +++ b/tools/lcpp.patch @@ -39,10 +39,10 @@ index b16c462f..6d1568f1 100644 const int idx = gguf_find_tensor(ctx, name); if (idx < 0) { diff --git a/src/llama.cpp b/src/llama.cpp -index 24e1f1f0..c2142c13 100644 +index 24e1f1f0..f90738cf 100644 --- a/src/llama.cpp +++ b/src/llama.cpp -@@ -205,6 +205,13 @@ enum llm_arch { +@@ -205,6 +205,14 @@ enum llm_arch { LLM_ARCH_GRANITE, LLM_ARCH_GRANITE_MOE, LLM_ARCH_CHAMELEON, @@ -53,10 +53,11 @@ index 24e1f1f0..c2142c13 100644 + LLM_ARCH_AURA, + LLM_ARCH_LTXV, + LLM_ARCH_HYVID, ++ LLM_ARCH_WAN, LLM_ARCH_UNKNOWN, }; -@@ -258,6 +265,13 @@ static const std::map LLM_ARCH_NAMES = { +@@ -258,6 +266,14 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_GRANITE, "granite" }, { LLM_ARCH_GRANITE_MOE, "granitemoe" }, { LLM_ARCH_CHAMELEON, "chameleon" }, @@ -67,10 +68,11 @@ index 24e1f1f0..c2142c13 100644 + { LLM_ARCH_AURA, "aura" }, + { LLM_ARCH_LTXV, "ltxv" }, + { LLM_ARCH_HYVID, "hyvid" }, ++ { LLM_ARCH_WAN, "wan" }, { LLM_ARCH_UNKNOWN, "(unknown)" }, }; -@@ -1531,6 +1545,13 @@ static const std::map> LLM_TENSOR_N +@@ -1531,6 +1547,14 @@ static const std::map> LLM_TENSOR_N { LLM_TENSOR_ATTN_K_NORM, "blk.%d.attn_k_norm" }, }, }, @@ -81,10 +83,11 @@ index 24e1f1f0..c2142c13 100644 + { LLM_ARCH_AURA, {}}, + { LLM_ARCH_LTXV, {}}, + { LLM_ARCH_HYVID,{}}, ++ { LLM_ARCH_WAN,{}}, { LLM_ARCH_UNKNOWN, { -@@ -5403,6 +5424,21 @@ static void llm_load_hparams( +@@ -5403,6 +5427,22 @@ static void llm_load_hparams( // get general kv ml.get_key(LLM_KV_GENERAL_NAME, model.name, false); @@ -97,6 +100,7 @@ index 24e1f1f0..c2142c13 100644 + case LLM_ARCH_AURA: + case LLM_ARCH_LTXV: + case LLM_ARCH_HYVID: ++ case LLM_ARCH_WAN: + model.ftype = ml.ftype; + return; + default: @@ -106,7 +110,7 @@ index 24e1f1f0..c2142c13 100644 // get hparams kv ml.get_key(LLM_KV_VOCAB_SIZE, hparams.n_vocab, false) || ml.get_arr_n(LLM_KV_TOKENIZER_LIST, hparams.n_vocab); -@@ -18016,6 +18052,126 @@ static void llama_tensor_dequantize_internal( +@@ -18016,6 +18056,128 @@ static void llama_tensor_dequantize_internal( workers.clear(); } @@ -139,6 +143,7 @@ index 24e1f1f0..c2142c13 100644 + if ( // Rules for to_v attention + (name.find("attn_v.weight") != std::string::npos) || + (name.find(".to_v.weight") != std::string::npos) || ++ (name.find(".v.weight") != std::string::npos) || + (name.find(".attn.w1v.weight") != std::string::npos) || + (name.find(".attn.w2v.weight") != std::string::npos) + ){ @@ -173,6 +178,7 @@ index 24e1f1f0..c2142c13 100644 + } + } else if ( // Rules for ffn + (name.find("ffn_down") != std::string::npos) || ++ (name.find(".ffn.2.weight") != std::string::npos) || // is this even the right way around? + (name.find(".ff.net.2.weight") != std::string::npos) + ) { + // TODO: add back `layer_info` with some model specific logic + logic further down @@ -233,7 +239,7 @@ index 24e1f1f0..c2142c13 100644 static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type new_type, const ggml_tensor * tensor, llama_ftype ftype) { const std::string name = ggml_get_name(tensor); -@@ -18513,7 +18669,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s +@@ -18513,7 +18675,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s if (llama_model_has_encoder(&model)) { n_attn_layer *= 3; } @@ -244,7 +250,7 @@ index 24e1f1f0..c2142c13 100644 } size_t total_size_org = 0; -@@ -18547,6 +18705,36 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s +@@ -18547,6 +18711,45 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s ctx_outs[i_split] = gguf_init_empty(); } gguf_add_tensor(ctx_outs[i_split], tensor); @@ -277,11 +283,20 @@ index 24e1f1f0..c2142c13 100644 + if (name == "img_in.proj.weight" && tensor->ne[5] != 1 ) { + throw std::runtime_error("img_in.proj.weight size failed for HyVid"); + } ++ } ++ // All the modulation layers also have dim1, and I think conv3d fails here too but we segfaul way before that... ++ if (model.arch == LLM_ARCH_WAN) { ++ const std::string name = ggml_get_name(tensor); ++ if (name.find(".modulation") != std::string::npos && tensor->ne[2] == 1) { ++ const int n_dim = 3; ++ gguf_set_tensor_ndim(ctx_outs[i_split], tensor->name, n_dim); ++ LLAMA_LOG_INFO("\n%s: Correcting shape for Wan: [key:%s]\n", __func__, tensor->name); ++ } + } } // Set split info if needed -@@ -18647,6 +18835,73 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s +@@ -18647,6 +18850,82 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s // do not quantize relative position bias (T5) quantize &= name.find("attn_rel_b.weight") == std::string::npos; @@ -347,6 +362,15 @@ index 24e1f1f0..c2142c13 100644 + quantize &= name.find("guidance_in.") == std::string::npos; + quantize &= name.find("final_layer.") == std::string::npos; + } ++ if (model.arch == LLM_ARCH_WAN) { ++ image_model = true; ++ quantize &= name.find("modulation.") == std::string::npos; ++ quantize &= name.find("patch_embedding.") == std::string::npos; ++ quantize &= name.find("text_embedding.") == std::string::npos; ++ quantize &= name.find("time_projection.") == std::string::npos; ++ quantize &= name.find("time_embedding.") == std::string::npos; ++ quantize &= name.find("head.") == std::string::npos; ++ } + // ignore 3D/4D tensors for image models as the code was never meant to handle these + if (image_model) { + quantize &= ggml_n_dims(tensor) == 2; @@ -355,7 +379,7 @@ index 24e1f1f0..c2142c13 100644 enum ggml_type new_type; void * new_data; size_t new_size; -@@ -18655,6 +18910,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s +@@ -18655,6 +18934,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s new_type = default_type; // get more optimal quantization type based on the tensor shape, layer, etc. @@ -365,7 +389,7 @@ index 24e1f1f0..c2142c13 100644 if (!params->pure && ggml_is_quantized(default_type)) { new_type = llama_tensor_get_type(qs, new_type, tensor, ftype); } -@@ -18664,6 +18922,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s +@@ -18664,6 +18946,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s if (params->output_tensor_type < GGML_TYPE_COUNT && strcmp(tensor->name, "output.weight") == 0) { new_type = params->output_tensor_type; } From 929a9e5f0dc78a0c97ce36c2fb5741b0486a875e Mon Sep 17 00:00:00 2001 From: City <125218114+city96@users.noreply.github.com> Date: Thu, 27 Feb 2025 05:58:01 +0100 Subject: [PATCH 05/15] Keep img_emb in F32 for wan i2v --- tools/lcpp.patch | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/lcpp.patch b/tools/lcpp.patch index 0ad92d1..9c522ce 100644 --- a/tools/lcpp.patch +++ b/tools/lcpp.patch @@ -39,7 +39,7 @@ index b16c462f..6d1568f1 100644 const int idx = gguf_find_tensor(ctx, name); if (idx < 0) { diff --git a/src/llama.cpp b/src/llama.cpp -index 24e1f1f0..f90738cf 100644 +index 24e1f1f0..d3ca4756 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -205,6 +205,14 @@ enum llm_arch { @@ -296,7 +296,7 @@ index 24e1f1f0..f90738cf 100644 } // Set split info if needed -@@ -18647,6 +18850,82 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s +@@ -18647,6 +18850,83 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s // do not quantize relative position bias (T5) quantize &= name.find("attn_rel_b.weight") == std::string::npos; @@ -369,6 +369,7 @@ index 24e1f1f0..f90738cf 100644 + quantize &= name.find("text_embedding.") == std::string::npos; + quantize &= name.find("time_projection.") == std::string::npos; + quantize &= name.find("time_embedding.") == std::string::npos; ++ quantize &= name.find("img_emb.") == std::string::npos; + quantize &= name.find("head.") == std::string::npos; + } + // ignore 3D/4D tensors for image models as the code was never meant to handle these @@ -379,7 +380,7 @@ index 24e1f1f0..f90738cf 100644 enum ggml_type new_type; void * new_data; size_t new_size; -@@ -18655,6 +18934,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s +@@ -18655,6 +18935,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s new_type = default_type; // get more optimal quantization type based on the tensor shape, layer, etc. @@ -389,7 +390,7 @@ index 24e1f1f0..f90738cf 100644 if (!params->pure && ggml_is_quantized(default_type)) { new_type = llama_tensor_get_type(qs, new_type, tensor, ftype); } -@@ -18664,6 +18946,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s +@@ -18664,6 +18947,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s if (params->output_tensor_type < GGML_TYPE_COUNT && strcmp(tensor->name, "output.weight") == 0) { new_type = params->output_tensor_type; } From c3a6103af54a43a010f375a8a56eac6489d79fb8 Mon Sep 17 00:00:00 2001 From: City <125218114+city96@users.noreply.github.com> Date: Fri, 7 Mar 2025 22:36:51 +0100 Subject: [PATCH 06/15] hyvid subkey load fix --- tools/convert.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/convert.py b/tools/convert.py index 001cd86..6a90960 100644 --- a/tools/convert.py +++ b/tools/convert.py @@ -144,7 +144,10 @@ def parse_args(): def load_state_dict(path): if any(path.endswith(x) for x in [".ckpt", ".pt", ".bin", ".pth"]): state_dict = torch.load(path, map_location="cpu", weights_only=True) - state_dict = state_dict.get("model", state_dict) + for subkey in ["model", "module"]: + if subkey in state_dict: + state_dict = state_dict[subkey] + break if len(state_dict) < 20: raise RuntimeError(f"pt subkey load failed: {state_dict.keys()}") else: From 9d0a0e317e999f79d3758b832ef04be0bb8a1346 Mon Sep 17 00:00:00 2001 From: City <125218114+city96@users.noreply.github.com> Date: Fri, 7 Mar 2025 22:37:17 +0100 Subject: [PATCH 07/15] ltxv bf16 fix scale shift table loads into nn.parameter, which can't work with BF16 --- tools/convert.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/convert.py b/tools/convert.py index 6a90960..480de37 100644 --- a/tools/convert.py +++ b/tools/convert.py @@ -17,6 +17,7 @@ class ModelTemplate: shape_fix = False # whether to reshape tensors keys_detect = [] # list of lists to match in state dict keys_banned = [] # list of keys that should mark model as invalid for conversion + keys_hiprec = [] # list of keys that need to be kept in fp32 for some reason def handle_nd_tensor(self, key, data): raise NotImplementedError(f"Tensor detected that exceeds dims supported by C++ code! ({key} @ {data.shape})") @@ -82,6 +83,9 @@ class ModelLTXV(ModelTemplate): "caption_projection.linear_2.weight", ) ] + keys_hiprec = [ + "scale_shift_table" # nn.parameter, can't load from BF16 base quant + ] class ModelSDXL(ModelTemplate): arch = "sdxl" @@ -227,6 +231,10 @@ def handle_tensors(writer, state_dict, model_arch): # very small tensors data_qtype = gguf.GGMLQuantizationType.F32 + elif any(x in key for x in model_arch.keys_hiprec): + # tensors that require max precision + data_qtype = gguf.GGMLQuantizationType.F32 + if (model_arch.shape_fix # NEVER reshape for models such as flux and n_dims > 1 # Skip one-dimensional tensors and n_params >= REARRANGE_THRESHOLD # Only rearrange tensors meeting the size requirement From d40e9bdcfec93b5b6343261b7d3121faa396d96f Mon Sep 17 00:00:00 2001 From: City <125218114+city96@users.noreply.github.com> Date: Fri, 14 Mar 2025 16:48:01 +0100 Subject: [PATCH 08/15] Use args for 5d fix --- tools/fix_5d_tensors.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tools/fix_5d_tensors.py b/tools/fix_5d_tensors.py index 7465cb4..8ded65f 100644 --- a/tools/fix_5d_tensors.py +++ b/tools/fix_5d_tensors.py @@ -2,10 +2,16 @@ import gguf import torch from tqdm import tqdm +import argparse +parser = argparse.ArgumentParser() +parser.add_argument("prec") +parser.add_argument("--root") +args = parser.parse_args() + arch = "hyvid" # TODO: Really should autodetect this -prec = "Q4_K_S" # edit manually for each step -src = fr"raw/hunyuan-video-t2v-720p-{prec}.gguf" -dst = fr"hunyuan-video-t2v-720p-{prec}.gguf" +prec = args.prec #"Q4_K_S" # edit manually for each step +src = fr"{args.root}/raw/hunyuan-video-t2v-720p-{prec}.gguf" +dst = fr"{args.root}/hunyuan-video-t2v-720p-{prec}.gguf" sd5d = torch.load(f"./fix_5d_tensors_{arch}.pt") print("5D:", sd5d.keys()) From 1c873edeb61d45fa68b5fabdac1d9975d9a5796b Mon Sep 17 00:00:00 2001 From: City <125218114+city96@users.noreply.github.com> Date: Sat, 22 Mar 2025 20:40:24 +0100 Subject: [PATCH 09/15] Expose strip prefix as function For external convert/detection logic --- tools/convert.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/tools/convert.py b/tools/convert.py index 480de37..80ff3fd 100644 --- a/tools/convert.py +++ b/tools/convert.py @@ -145,18 +145,7 @@ def parse_args(): return args -def load_state_dict(path): - if any(path.endswith(x) for x in [".ckpt", ".pt", ".bin", ".pth"]): - state_dict = torch.load(path, map_location="cpu", weights_only=True) - for subkey in ["model", "module"]: - if subkey in state_dict: - state_dict = state_dict[subkey] - break - if len(state_dict) < 20: - raise RuntimeError(f"pt subkey load failed: {state_dict.keys()}") - else: - state_dict = load_file(path) - +def strip_prefix(state_dict): # only keep unet with no prefix! prefix = None for pfx in ["model.diffusion_model.", "model."]: @@ -174,6 +163,20 @@ def load_state_dict(path): return sd +def load_state_dict(path): + if any(path.endswith(x) for x in [".ckpt", ".pt", ".bin", ".pth"]): + state_dict = torch.load(path, map_location="cpu", weights_only=True) + for subkey in ["model", "module"]: + if subkey in state_dict: + state_dict = state_dict[subkey] + break + if len(state_dict) < 20: + raise RuntimeError(f"pt subkey load failed: {state_dict.keys()}") + else: + state_dict = load_file(path) + + return strip_prefix(state_dict) + def load_model(path): state_dict = load_state_dict(path) model_arch = detect_arch(state_dict) From 65a2bc20bb924d8dff391fe3dd55830b0c9721aa Mon Sep 17 00:00:00 2001 From: City <125218114+city96@users.noreply.github.com> Date: Fri, 28 Mar 2025 04:21:26 +0100 Subject: [PATCH 10/15] wan bf16 fix --- tools/convert.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/convert.py b/tools/convert.py index 80ff3fd..52d1800 100644 --- a/tools/convert.py +++ b/tools/convert.py @@ -73,6 +73,9 @@ class ModelWan(ModelHyVid): "head.modulation", ) ] + keys_hiprec = [ + ".modulation" # nn.parameter, can't load from BF16 ver + ] class ModelLTXV(ModelTemplate): arch = "ltxv" From bbd89ae5471d7221cd945c46d000809727994d7b Mon Sep 17 00:00:00 2001 From: City <125218114+city96@users.noreply.github.com> Date: Sun, 30 Mar 2025 18:42:17 +0200 Subject: [PATCH 11/15] Update .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 691fb41..44fe82f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ *.bin *.gguf *.safetensors -tools/llama.cpp/ +tools/llama.cpp* # Byte-compiled / optimized / DLL files __pycache__/ From c36b7236e65944ae7e76b6e586785fce7ee190f9 Mon Sep 17 00:00:00 2001 From: City <125218114+city96@users.noreply.github.com> Date: Sun, 30 Mar 2025 18:46:09 +0200 Subject: [PATCH 12/15] Update README.md --- tools/README.md | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/tools/README.md b/tools/README.md index 2bee67b..6d1ee90 100644 --- a/tools/README.md +++ b/tools/README.md @@ -29,14 +29,6 @@ git checkout tags/b3962 git apply ..\lcpp.patch ``` -- To quantize **SD3** or **AuraFlow** models, you should use the patch `lcpp_sd3.patch` and target `tags/b3962` instead. -- There is a [WIP PR for other model architectures](https://github.com/city96/ComfyUI-GGUF/pull/216) -``` -cd llama.cpp -git checkout tags/b3962 -git apply ..\lcpp_sd3.patch -``` - ## STEP 5 (Compile llama-quantize binary) @@ -60,7 +52,7 @@ llama.cpp\build\bin\Debug\llama-quantize.exe E:\models\unet\flux1-dev-BF16.gguf You can extract the patch again with `git diff src\llama.cpp > lcpp.patch` if you wish to change something and contribute back. > [!WARNING] ->For hunyuan video, you will have to uncomment the block in convert.py that deals with 5D tensors. This will save a **non functional** model to disk first, that you can quantize. After quantization, run `fix_5d_tensor.py` to add back the missing key that was saved by the conversion code. You will have to edit this file to set the correct paths/architecture. This may change in the future. +>For hunyuan video/wan 2.1, you will have to uncomment the block in convert.py that deals with 5D tensors. This will save a **non functional** model to disk first, that you can quantize. After quantization, run `fix_5d_tensor.py` to add back the missing key that was saved by the conversion code. You will have to edit this file to set the correct paths/architecture. This may change in the future. > [!WARNING] From 862e4147c4a269445823ed2fe22bc6cea7cc216e Mon Sep 17 00:00:00 2001 From: City <125218114+city96@users.noreply.github.com> Date: Thu, 3 Apr 2025 00:38:46 +0200 Subject: [PATCH 13/15] Make 5D tensor fix less fragile --- tools/fix_5d_tensors.py | 107 +++++++++++++++++++++++++--------------- 1 file changed, 67 insertions(+), 40 deletions(-) diff --git a/tools/fix_5d_tensors.py b/tools/fix_5d_tensors.py index 8ded65f..e59794e 100644 --- a/tools/fix_5d_tensors.py +++ b/tools/fix_5d_tensors.py @@ -1,52 +1,79 @@ +import os import gguf import torch +import argparse from tqdm import tqdm -import argparse -parser = argparse.ArgumentParser() -parser.add_argument("prec") -parser.add_argument("--root") -args = parser.parse_args() +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--src", required=True) + parser.add_argument("--dst", required=True) + parser.add_argument("--fix", required=False, help="Defaults to ./fix_5d_tensors_[arch].pt") + parser.add_argument("--overwrite", action="store_true") + args = parser.parse_args() -arch = "hyvid" # TODO: Really should autodetect this -prec = args.prec #"Q4_K_S" # edit manually for each step -src = fr"{args.root}/raw/hunyuan-video-t2v-720p-{prec}.gguf" -dst = fr"{args.root}/hunyuan-video-t2v-720p-{prec}.gguf" + if not os.path.isfile(args.src): + parser.error(f"Invalid source file '{args.src}'") + if not args.overwrite and os.path.exists(args.dst): + parser.error(f"Output exists, use '--overwrite' ({args.dst})") -sd5d = torch.load(f"./fix_5d_tensors_{arch}.pt") -print("5D:", sd5d.keys()) + return args -reader = gguf.GGUFReader(src) -writer = gguf.GGUFWriter(path=None, arch=arch) +def get_arch_str(reader): + field = reader.get_field("general.architecture") + return str(field.parts[field.data[-1]], encoding="utf-8") -writer.add_quantization_version(gguf.GGML_QUANT_VERSION) -writer.add_file_type(getattr(gguf.LlamaFileType, f"MOSTLY_{prec}")) # TODO: also autodetect +def get_file_type(reader): + field = reader.get_field("general.file_type") + ft = int(field.parts[field.data[-1]]) + return gguf.LlamaFileType(ft) -added = [] -def add_extra_key(writer, key, data): - old_dtype = data.dtype - data_qtype = gguf.GGMLQuantizationType.F32 - n_dims = len(data.shape) - data_shape = data.shape - data = gguf.quants.quantize(data, data_qtype) - tqdm.write(f"Adding key {key} ({data_shape})") - writer.add_tensor(key, data, raw_dtype=data_qtype) - global added - added.append(key) +if __name__ == "__main__": + args = get_args() -# main loop to add missing -for tensor in tqdm(reader.tensors): - writer.add_tensor(tensor.name, tensor.data, raw_dtype=tensor.tensor_type) - key5d = tensor.name.replace(".bias", ".weight") - if key5d in sd5d.keys(): - add_extra_key(writer, key5d, sd5d[key5d]) + # read existing + reader = gguf.GGUFReader(args.src) + arch = get_arch_str(reader) + file_type = get_file_type(reader) + print(f"Detected arch: '{arch}' (ftype: {str(file_type)})") -# brute force for any missed -for key, data in sd5d.items(): - if key not in added: - add_extra_key(writer, key, data) + # prep fix + if args.fix is None: + args.fix = f"./fix_5d_tensors_{arch}.pt" + + if not os.path.isfile(args.fix): + raise OSError(f"No 5D tensor fix file: {args.fix}") -writer.write_header_to_file(path=dst) -writer.write_kv_data_to_file() -writer.write_tensors_to_file(progress=True) -writer.close() + sd5d = torch.load(args.fix) + print("5D tensors:", sd5d.keys()) + + # prep output + writer = gguf.GGUFWriter(path=None, arch=arch) + writer.add_quantization_version(gguf.GGML_QUANT_VERSION) + writer.add_file_type(file_type) + + added = [] + def add_extra_key(writer, key, data): + global added + data_qtype = gguf.GGMLQuantizationType.F32 + data = gguf.quants.quantize(data, data_qtype) + tqdm.write(f"Adding key {key} ({data.shape})") + writer.add_tensor(key, data, raw_dtype=data_qtype) + added.append(key) + + # main loop to add missing 5D tensor(s) + for tensor in tqdm(reader.tensors): + writer.add_tensor(tensor.name, tensor.data, raw_dtype=tensor.tensor_type) + key5d = tensor.name.replace(".bias", ".weight") + if key5d in sd5d.keys(): + add_extra_key(writer, key5d, sd5d[key5d]) + + # brute force for any missed + for key, data in sd5d.items(): + if key not in added: + add_extra_key(writer, key, data) + + writer.write_header_to_file(path=args.dst) + writer.write_kv_data_to_file() + writer.write_tensors_to_file(progress=True) + writer.close() From 624687939f1d361b59770f0a0b5bf75b7d5892e9 Mon Sep 17 00:00:00 2001 From: City <125218114+city96@users.noreply.github.com> Date: Thu, 10 Apr 2025 07:12:10 +0200 Subject: [PATCH 14/15] Refactor 5D patch and save loop --- tools/convert.py | 98 ++++++++++++++++++++++++++--------------- tools/fix_5d_tensors.py | 7 ++- 2 files changed, 67 insertions(+), 38 deletions(-) diff --git a/tools/convert.py b/tools/convert.py index 52d1800..6c5669d 100644 --- a/tools/convert.py +++ b/tools/convert.py @@ -1,11 +1,10 @@ # (c) City96 || Apache-2.0 (apache.org/licenses/LICENSE-2.0) import os +import gguf import torch -import gguf # This needs to be the llama.cpp one specifically! import argparse from tqdm import tqdm - -from safetensors.torch import load_file +from safetensors.torch import load_file, save_file QUANTIZATION_THRESHOLD = 1024 REARRANGE_THRESHOLD = 512 @@ -55,14 +54,14 @@ class ModelHyVid(ModelTemplate): ) ] - # def handle_nd_tensor(self, key, data): - # # hacky but don't have any better ideas - # path = f"./fix_5d_tensors_{self.arch}.pt" - # if os.path.isfile(path): - # raise RuntimeError(f"5D tensor fix file already exists! {path}") - # fsd = {key: data} - # tqdm.write(f"5D key found in state dict! Manual fix required! - {key} {data.shape}") - # torch.save(fsd, path) + def handle_nd_tensor(self, key, data): + # hacky but don't have any better ideas + path = f"./fix_5d_tensors_{self.arch}.safetensors" # TODO: somehow get a path here?? + if os.path.isfile(path): + raise RuntimeError(f"5D tensor fix file already exists! {path}") + fsd = {key: torch.from_numpy(data)} + tqdm.write(f"5D key found in state dict! Manual fix required! - {key} {data.shape}") + save_file(fsd, path) class ModelWan(ModelHyVid): arch = "wan" @@ -180,13 +179,6 @@ def load_state_dict(path): return strip_prefix(state_dict) -def load_model(path): - state_dict = load_state_dict(path) - model_arch = detect_arch(state_dict) - print(f"* Architecture detected from input: {model_arch.arch}") - writer = gguf.GGUFWriter(path=None, arch=model_arch.arch) - return (writer, state_dict, model_arch) - def handle_tensors(writer, state_dict, model_arch): name_lengths = tuple(sorted( ((key, len(key)) for key in state_dict.keys()), @@ -212,10 +204,12 @@ def handle_tensors(writer, state_dict, model_arch): n_dims = len(data.shape) data_shape = data.shape - data_qtype = getattr( - gguf.GGMLQuantizationType, - "BF16" if old_dtype == torch.bfloat16 else "F16" - ) + if old_dtype == torch.bfloat16: + data_qtype = gguf.GGMLQuantizationType.BF16 + # elif old_dtype == torch.float32: + # data_qtype = gguf.GGMLQuantizationType.F32 + else: + data_qtype = gguf.GGMLQuantizationType.F16 # The max no. of dimensions that can be handled by the quantization code is 4 if len(data.shape) > MAX_TENSOR_DIMS: @@ -265,25 +259,57 @@ def handle_tensors(writer, state_dict, model_arch): writer.add_tensor(new_name, data, raw_dtype=data_qtype) -if __name__ == "__main__": - args = parse_args() - path = args.src - writer, state_dict, model_arch = load_model(path) +def convert_file(path, dst_path=None, interact=True, overwrite=False): + # load & run model detection logic + state_dict = load_state_dict(path) + model_arch = detect_arch(state_dict) + print(f"* Architecture detected from input: {model_arch.arch}") - writer.add_quantization_version(gguf.GGML_QUANT_VERSION) - if next(iter(state_dict.values())).dtype == torch.bfloat16: - out_path = f"{os.path.splitext(path)[0]}-BF16.gguf" - writer.add_file_type(gguf.LlamaFileType.MOSTLY_BF16) + # detect & set dtype for output file + dtypes = [x.dtype for x in state_dict.values()] + dtypes = {x:dtypes.count(x) for x in set(dtypes)} + main_dtype = max(dtypes, key=dtypes.get) + + if main_dtype == torch.bfloat16: + ftype_name = "BF16" + ftype_gguf = gguf.LlamaFileType.MOSTLY_BF16 + # elif main_dtype == torch.float32: + # ftype_name = "F32" + # ftype_gguf = None else: - out_path = f"{os.path.splitext(path)[0]}-F16.gguf" - writer.add_file_type(gguf.LlamaFileType.MOSTLY_F16) + ftype_name = "F16" + ftype_gguf = gguf.LlamaFileType.MOSTLY_F16 - out_path = args.dst or out_path - if os.path.isfile(out_path): - input("Output exists enter to continue or ctrl+c to abort!") + if dst_path is None: + dst_path = f"{os.path.splitext(path)[0]}-{ftype_name}.gguf" + elif "{ftype}" in dst_path: # lcpp logic + dst_path = dst_path.replace("{ftype}", ftype_name) + + if os.path.isfile(dst_path) and not overwrite: + if interact: + input("Output exists enter to continue or ctrl+c to abort!") + else: + raise OSError("Output exists and overwriting is disabled!") + + # handle actual file + writer = gguf.GGUFWriter(path=None, arch=model_arch.arch) + writer.add_quantization_version(gguf.GGML_QUANT_VERSION) + if ftype_gguf is not None: + writer.add_file_type(ftype_gguf) handle_tensors(writer, state_dict, model_arch) - writer.write_header_to_file(path=out_path) + writer.write_header_to_file(path=dst_path) writer.write_kv_data_to_file() writer.write_tensors_to_file(progress=True) writer.close() + + fix = f"./fix_5d_tensors_{model_arch.arch}.safetensors" + if os.path.isfile(fix): + print(f"\n### Warning! Fix file found at '{fix}'") + print(f" you most likely need to run 'fix_5d_tensors.py' after quantization.") + + return dst_path, model_arch + +if __name__ == "__main__": + args = parse_args() + convert_file(args.src, args.dst) diff --git a/tools/fix_5d_tensors.py b/tools/fix_5d_tensors.py index e59794e..0e61d1c 100644 --- a/tools/fix_5d_tensors.py +++ b/tools/fix_5d_tensors.py @@ -1,8 +1,10 @@ +# (c) City96 || Apache-2.0 (apache.org/licenses/LICENSE-2.0) import os import gguf import torch import argparse from tqdm import tqdm +from safetensors.torch import load_file def get_args(): parser = argparse.ArgumentParser() @@ -39,12 +41,13 @@ if __name__ == "__main__": # prep fix if args.fix is None: - args.fix = f"./fix_5d_tensors_{arch}.pt" + args.fix = f"./fix_5d_tensors_{arch}.safetensors" if not os.path.isfile(args.fix): raise OSError(f"No 5D tensor fix file: {args.fix}") - sd5d = torch.load(args.fix) + sd5d = load_file(args.fix) + sd5d = {k:v.numpy() for k,v in sd5d.items()} print("5D tensors:", sd5d.keys()) # prep output From ca97e050b3ad621dd3b2a5f05249f31164cf3377 Mon Sep 17 00:00:00 2001 From: City <125218114+city96@users.noreply.github.com> Date: Thu, 10 Apr 2025 08:04:17 +0200 Subject: [PATCH 15/15] Update README.md --- tools/README.md | 62 +++++++++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/tools/README.md b/tools/README.md index 6d1ee90..ec98261 100644 --- a/tools/README.md +++ b/tools/README.md @@ -1,64 +1,66 @@ -## STEP 1 (Patch files with Unix (LF) line endings +## Converting initial model -Solution to fix lines endings of the patch files from Windows (CRLF) to Unix (LF) +To convert your initial safetensors/ckpt model to FP16/BF16 GGUF, run the following command: + +``` +python convert.py --src E:\models\unet\flux1-dev.safetensors +``` +Make sure `gguf>=0.13.0` is installed for this step. Optionally, specify the output gguf file with the `--dst` arg. + +> [!NOTE] +> Do not use the diffusers UNET format for flux, it won't work, use the default/reference checkpoint key format. This is due to q/k/v being merged into one qkv key. +> You can convert it by loading it in ComfyUI and saving it using the built-in "ModelSave" node. + +> [!WARNING] +> For hunyuan video/wan 2.1, you will see a warning about 5D tensors. This means the script will save a **non functional** model to disk first, that you can quantize. I recommend saving these in a separate `raw` folder to avoid confusion. +> +> After quantization, you will have to run `fix_5d_tensor.py` manually to add back the missing key that was saved by the conversion code. + +## Quantizing using custom llama.cpp + +Depending on your git settings, you may need to run the following script first in order to make sure the patch file is valid. It will convert Windows (CRLF) line endings to Unix (LF) ones. ``` python fix_lines_ending.py ``` -## STEP 2 (Clone llama.cpp version of gguf-py) -Git clone llama.cpp into the current folder. You may also install gguf-py from the llama.cpp repo directly, though the one specified in `requirements.txt` should also work on recent versions. +Git clone llama.cpp into the current folder: ``` git clone https://github.com/ggerganov/llama.cpp -pip install llama.cpp/gguf-py ``` -## STEP 3 (Convert to FP16 or BF16) +Check out the correct branch, then apply the custom patch needed to add image model support to the repo you just cloned. -To convert your initial source model to FP16 (or BF16), run the following command: -``` -python convert.py --src E:\models\unet\flux1-dev.safetensors -``` -## STEP 4 (Patch llama.cpp) - -- To quantize the model, first apply the provided patch to the llama.cpp repo you've just cloned. ``` cd llama.cpp git checkout tags/b3962 git apply ..\lcpp.patch ``` +Compile the llama-quantize binary. This example uses cmake, on linux you can just use make. -## STEP 5 (Compile llama-quantize binary) - -Then, compile the llama-quantize binary. This example uses cmake, on linux you can just use make. ``` mkdir build -cd build -cmake .. -cmake --build . --config Debug -j10 --target llama-quantize -cd .. +cmake -B build +cmake --build build --config Debug -j10 --target llama-quantize cd .. ``` -## STEP 6 (Quantization) Now you can use the newly build binary to quantize your model to the desired format: ``` llama.cpp\build\bin\Debug\llama-quantize.exe E:\models\unet\flux1-dev-BF16.gguf E:\models\unet\flux1-dev-Q4_K_S.gguf Q4_K_S ``` - You can extract the patch again with `git diff src\llama.cpp > lcpp.patch` if you wish to change something and contribute back. > [!WARNING] ->For hunyuan video/wan 2.1, you will have to uncomment the block in convert.py that deals with 5D tensors. This will save a **non functional** model to disk first, that you can quantize. After quantization, run `fix_5d_tensor.py` to add back the missing key that was saved by the conversion code. You will have to edit this file to set the correct paths/architecture. This may change in the future. +> For hunyuan video/wan 2.1, you will have to run `fix_5d_tensor.py` after the quantization step is done. +> +> Example usage: `fix_5d_tensors.py --src E:\models\video\raw\wan2.1-t2v-1.3b-Q8_0.gguf --dst E:\models\video\wan2.1-t2v-1.3b-Q8_0.gguf` +> +> By default, this also saves a `fix_5d_tensors_[arch].safetensors` file in the `ComfyUI-GGUF/tools` folder, it's recommended to delete this after all models have been converted. - -> [!WARNING] -> Do not use the diffusers UNET for flux, it won't work, use the default/reference checkpoint format. This is due to q/k/v being merged into one qkv key. You can convert it by loading it in ComfyUI and saving it using the built-in "ModelSave" node. - - -> [!WARNING] -> Do not quantize SDXL / SD1 / other Conv2D heavy models. There's little to no benefit with these models. If you do, make sure to **extract the UNET model first**. +> [!NOTE] +> Do not quantize SDXL / SD1 / other Conv2D heavy models. If you do, make sure to **extract the UNET model first**. >This should be obvious, but also don't use the resulting llama-quantize binary with LLMs.