diff --git a/tools/convert.py b/tools/convert.py index 490f48f..a62403c 100644 --- a/tools/convert.py +++ b/tools/convert.py @@ -24,18 +24,18 @@ class ModelTemplate: class ModelFlux(ModelTemplate): arch = "flux" keys_detect = [ - ("transformer_blocks.0.attn.norm_added_k.weight",), + ("single_transformer_blocks.0.attn.norm_k.weight",), ("double_blocks.0.img_attn.proj.weight",), ] - keys_banned = ["transformer_blocks.0.attn.norm_added_k.weight",] + keys_banned = ["single_transformer_blocks.0.attn.norm_k.weight",] class ModelSD3(ModelTemplate): arch = "sd3" keys_detect = [ - ("transformer_blocks.0.attn.add_q_proj.weight",), + ("transformer_blocks.0.ff_context.net.0.proj.weight",), ("joint_blocks.0.x_block.attn.qkv.weight",), ] - keys_banned = ["transformer_blocks.0.attn.add_q_proj.weight",] + keys_banned = ["transformer_blocks.0.ff_context.net.0.proj.weight",] class ModelAura(ModelTemplate): arch = "aura" @@ -59,7 +59,7 @@ class ModelHiDream(ModelTemplate): "img_emb.emb_pos" ] -class CosmosPredict2(ModelTemplate): +class ModelCosmosPredict2(ModelTemplate): arch = "cosmos" keys_detect = [ ( @@ -70,6 +70,16 @@ class CosmosPredict2(ModelTemplate): keys_hiprec = ["pos_embedder"] keys_ignore = ["_extra_state", "accum_"] +class ModelQwenImage(ModelTemplate): + arch = "qwen_image" + keys_detect = [ + ( + "time_text_embed.timestep_embedder.linear_2.weight", + "transformer_blocks.0.attn.norm_added_q.weight", + "transformer_blocks.0.img_mlp.net.0.proj.weight", + ) + ] + class ModelHyVid(ModelTemplate): arch = "hyvid" ndims_fix = True @@ -136,8 +146,11 @@ class ModelLumina2(ModelTemplate): ("cap_embedder.1.weight", "context_refiner.0.attention.qkv.weight") ] -arch_list = [ModelFlux, ModelSD3, ModelAura, ModelHiDream, CosmosPredict2, - ModelLTXV, ModelHyVid, ModelWan, ModelSDXL, ModelSD1, ModelLumina2] +# The architectures are checked in order and the first successful match terminates the search. +arch_list = [ + ModelFlux, ModelSD3, ModelAura, ModelHiDream, ModelCosmosPredict2, ModelQwenImage, + ModelLTXV, ModelHyVid, ModelWan, ModelSDXL, ModelSD1, ModelLumina2 +] def is_model_arch(model, state_dict): # check if model is correct @@ -148,7 +161,7 @@ def is_model_arch(model, state_dict): matched = True invalid = any(key in state_dict for key in model.keys_banned) break - assert not invalid, "Model architecture not allowed for conversion! (i.e. reference VS diffusers format)" + assert not invalid, f"Model architecture not allowed for conversion! (i.e. reference VS diffusers format) [arch:{model.arch}]" return matched def detect_arch(state_dict): @@ -377,4 +390,3 @@ def convert_file(path, dst_path=None, interact=True, overwrite=False, allow_fp32 if __name__ == "__main__": args = parse_args() convert_file(args.src, args.dst) - diff --git a/tools/lcpp.patch b/tools/lcpp.patch index 92396e1..a0c3768 100644 --- a/tools/lcpp.patch +++ b/tools/lcpp.patch @@ -39,10 +39,10 @@ index b16c462f..6d1568f1 100644 const int idx = gguf_find_tensor(ctx, name); if (idx < 0) { diff --git a/src/llama.cpp b/src/llama.cpp -index 24e1f1f0..25db4c69 100644 +index 24e1f1f0..5d191f92 100644 --- a/src/llama.cpp +++ b/src/llama.cpp -@@ -205,6 +205,17 @@ enum llm_arch { +@@ -205,6 +205,18 @@ enum llm_arch { LLM_ARCH_GRANITE, LLM_ARCH_GRANITE_MOE, LLM_ARCH_CHAMELEON, @@ -57,10 +57,11 @@ index 24e1f1f0..25db4c69 100644 + LLM_ARCH_HIDREAM, + LLM_ARCH_COSMOS, + LLM_ARCH_LUMINA2, ++ LLM_ARCH_QWEN_IMAGE, LLM_ARCH_UNKNOWN, }; -@@ -258,6 +269,17 @@ static const std::map LLM_ARCH_NAMES = { +@@ -258,6 +270,18 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_GRANITE, "granite" }, { LLM_ARCH_GRANITE_MOE, "granitemoe" }, { LLM_ARCH_CHAMELEON, "chameleon" }, @@ -75,28 +76,30 @@ index 24e1f1f0..25db4c69 100644 + { LLM_ARCH_HIDREAM, "hidream" }, + { LLM_ARCH_COSMOS, "cosmos" }, + { LLM_ARCH_LUMINA2, "lumina2" }, ++ { LLM_ARCH_QWEN_IMAGE, "qwen_image" }, { LLM_ARCH_UNKNOWN, "(unknown)" }, }; -@@ -1531,6 +1553,17 @@ static const std::map> LLM_TENSOR_N +@@ -1531,6 +1555,18 @@ static const std::map> LLM_TENSOR_N { LLM_TENSOR_ATTN_K_NORM, "blk.%d.attn_k_norm" }, }, }, -+ { LLM_ARCH_FLUX, {}}, -+ { LLM_ARCH_SD1, {}}, -+ { LLM_ARCH_SDXL, {}}, -+ { LLM_ARCH_SD3, {}}, -+ { LLM_ARCH_AURA, {}}, -+ { LLM_ARCH_LTXV, {}}, -+ { LLM_ARCH_HYVID, {}}, -+ { LLM_ARCH_WAN, {}}, -+ { LLM_ARCH_HIDREAM, {}}, -+ { LLM_ARCH_COSMOS, {}}, -+ { LLM_ARCH_LUMINA2, {}}, ++ { LLM_ARCH_FLUX, {}}, ++ { LLM_ARCH_SD1, {}}, ++ { LLM_ARCH_SDXL, {}}, ++ { LLM_ARCH_SD3, {}}, ++ { LLM_ARCH_AURA, {}}, ++ { LLM_ARCH_LTXV, {}}, ++ { LLM_ARCH_HYVID, {}}, ++ { LLM_ARCH_WAN, {}}, ++ { LLM_ARCH_HIDREAM, {}}, ++ { LLM_ARCH_COSMOS, {}}, ++ { LLM_ARCH_LUMINA2, {}}, ++ { LLM_ARCH_QWEN_IMAGE, {}}, { LLM_ARCH_UNKNOWN, { -@@ -5403,6 +5436,25 @@ static void llm_load_hparams( +@@ -5403,6 +5439,26 @@ static void llm_load_hparams( // get general kv ml.get_key(LLM_KV_GENERAL_NAME, model.name, false); @@ -113,6 +116,7 @@ index 24e1f1f0..25db4c69 100644 + case LLM_ARCH_HIDREAM: + case LLM_ARCH_COSMOS: + case LLM_ARCH_LUMINA2: ++ case LLM_ARCH_QWEN_IMAGE: + model.ftype = ml.ftype; + return; + default: @@ -122,7 +126,7 @@ index 24e1f1f0..25db4c69 100644 // get hparams kv ml.get_key(LLM_KV_VOCAB_SIZE, hparams.n_vocab, false) || ml.get_arr_n(LLM_KV_TOKENIZER_LIST, hparams.n_vocab); -@@ -18016,6 +18068,134 @@ static void llama_tensor_dequantize_internal( +@@ -18016,6 +18072,137 @@ static void llama_tensor_dequantize_internal( workers.clear(); } @@ -158,6 +162,7 @@ index 24e1f1f0..25db4c69 100644 + (name.find(".v.weight") != std::string::npos) || + (name.find(".attn.w1v.weight") != std::string::npos) || + (name.find(".attn.w2v.weight") != std::string::npos) || ++ (name.find(".add_v_proj.weight") != std::string::npos) || + (name.find("_attn.v_proj.weight") != std::string::npos) + ){ + if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) { @@ -197,7 +202,9 @@ index 24e1f1f0..25db4c69 100644 + (name.find(".ff.net.2.weight") != std::string::npos) || + (name.find(".mlp.layer2.weight") != std::string::npos) || + (name.find(".adaln_modulation_mlp.2.weight") != std::string::npos) || -+ (name.find(".feed_forward.w2.weight") != std::string::npos) ++ (name.find(".feed_forward.w2.weight") != std::string::npos) || ++ (name.find(".img_mlp.net.2.weight") != std::string::npos) || ++ (name.find(".txt_mlp.net.0.proj.weight") != std::string::npos) + ) { + // TODO: add back `layer_info` with some model specific logic + logic further down + if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) { @@ -257,7 +264,7 @@ index 24e1f1f0..25db4c69 100644 static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type new_type, const ggml_tensor * tensor, llama_ftype ftype) { const std::string name = ggml_get_name(tensor); -@@ -18513,7 +18693,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s +@@ -18513,7 +18700,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s if (llama_model_has_encoder(&model)) { n_attn_layer *= 3; } @@ -268,7 +275,7 @@ index 24e1f1f0..25db4c69 100644 } size_t total_size_org = 0; -@@ -18547,6 +18729,51 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s +@@ -18547,6 +18736,51 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s ctx_outs[i_split] = gguf_init_empty(); } gguf_add_tensor(ctx_outs[i_split], tensor); @@ -320,7 +327,7 @@ index 24e1f1f0..25db4c69 100644 } // Set split info if needed -@@ -18647,6 +18874,110 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s +@@ -18647,6 +18881,118 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s // do not quantize relative position bias (T5) quantize &= name.find("attn_rel_b.weight") == std::string::npos; @@ -423,6 +430,14 @@ index 24e1f1f0..25db4c69 100644 + quantize &= name.find("context_refiner.") == std::string::npos; + quantize &= name.find("noise_refiner.") == std::string::npos; + } ++ if (model.arch == LLM_ARCH_QWEN_IMAGE) { ++ image_model = true; ++ quantize &= name.find("img_in.") == std::string::npos; ++ quantize &= name.find("txt_in.") == std::string::npos; ++ quantize &= name.find("time_text_embed.") == std::string::npos; ++ quantize &= name.find("proj_out.") == std::string::npos; ++ quantize &= name.find("norm_out.") == std::string::npos; ++ } + // ignore 3D/4D tensors for image models as the code was never meant to handle these + if (image_model) { + quantize &= ggml_n_dims(tensor) == 2; @@ -431,7 +446,7 @@ index 24e1f1f0..25db4c69 100644 enum ggml_type new_type; void * new_data; size_t new_size; -@@ -18655,6 +18986,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s +@@ -18655,6 +19001,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s new_type = default_type; // get more optimal quantization type based on the tensor shape, layer, etc. @@ -441,7 +456,7 @@ index 24e1f1f0..25db4c69 100644 if (!params->pure && ggml_is_quantized(default_type)) { new_type = llama_tensor_get_type(qs, new_type, tensor, ftype); } -@@ -18664,6 +18998,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s +@@ -18664,6 +19013,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s if (params->output_tensor_type < GGML_TYPE_COUNT && strcmp(tensor->name, "output.weight") == 0) { new_type = params->output_tensor_type; }