nodes/city96-ComfyUI-GGUF
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Convert code for qwen image

Browse Source
This commit is contained in:
City
2025-08-05 14:27:52 +02:00
parent 324c45ab0f
commit 16341ccfc0
2 changed files with 59 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/convert.py
+21 -9
View File
@@ -24,18 +24,18 @@ class ModelTemplate:
class ModelFlux(ModelTemplate):
arch = "flux"
keys_detect = [
("transformer_blocks.0.attn.norm_added_k.weight",),
("single_transformer_blocks.0.attn.norm_k.weight",),
("double_blocks.0.img_attn.proj.weight",),
]
keys_banned = ["transformer_blocks.0.attn.norm_added_k.weight",]
keys_banned = ["single_transformer_blocks.0.attn.norm_k.weight",]
class ModelSD3(ModelTemplate):
arch = "sd3"
keys_detect = [
("transformer_blocks.0.attn.add_q_proj.weight",),
("transformer_blocks.0.ff_context.net.0.proj.weight",),
("joint_blocks.0.x_block.attn.qkv.weight",),
]
keys_banned = ["transformer_blocks.0.attn.add_q_proj.weight",]
keys_banned = ["transformer_blocks.0.ff_context.net.0.proj.weight",]
class ModelAura(ModelTemplate):
arch = "aura"
@@ -59,7 +59,7 @@ class ModelHiDream(ModelTemplate):
"img_emb.emb_pos"
]
class CosmosPredict2(ModelTemplate):
class ModelCosmosPredict2(ModelTemplate):
arch = "cosmos"
keys_detect = [
(
@@ -70,6 +70,16 @@ class CosmosPredict2(ModelTemplate):
keys_hiprec = ["pos_embedder"]
keys_ignore = ["_extra_state", "accum_"]
class ModelQwenImage(ModelTemplate):
arch = "qwen_image"
keys_detect = [
(
"time_text_embed.timestep_embedder.linear_2.weight",
"transformer_blocks.0.attn.norm_added_q.weight",
"transformer_blocks.0.img_mlp.net.0.proj.weight",
)
]
class ModelHyVid(ModelTemplate):
arch = "hyvid"
ndims_fix = True
@@ -136,8 +146,11 @@ class ModelLumina2(ModelTemplate):
("cap_embedder.1.weight", "context_refiner.0.attention.qkv.weight")
]
arch_list = [ModelFlux, ModelSD3, ModelAura, ModelHiDream, CosmosPredict2,
ModelLTXV, ModelHyVid, ModelWan, ModelSDXL, ModelSD1, ModelLumina2]
# The architectures are checked in order and the first successful match terminates the search.
arch_list = [
ModelFlux, ModelSD3, ModelAura, ModelHiDream, ModelCosmosPredict2, ModelQwenImage,
ModelLTXV, ModelHyVid, ModelWan, ModelSDXL, ModelSD1, ModelLumina2
]
def is_model_arch(model, state_dict):
# check if model is correct
@@ -148,7 +161,7 @@ def is_model_arch(model, state_dict):
matched = True
invalid = any(key in state_dict for key in model.keys_banned)
break
assert not invalid, "Model architecture not allowed for conversion! (i.e. reference VS diffusers format)"
assert not invalid, f"Model architecture not allowed for conversion! (i.e. reference VS diffusers format) [arch:{model.arch}]"
return matched
def detect_arch(state_dict):
@@ -377,4 +390,3 @@ def convert_file(path, dst_path=None, interact=True, overwrite=False, allow_fp32
if __name__ == "__main__":
args = parse_args()
convert_file(args.src, args.dst)
tools/lcpp.patch
+38 -23
View File
@@ -39,10 +39,10 @@ index b16c462f..6d1568f1 100644
const int idx = gguf_find_tensor(ctx, name);
if (idx < 0) {
diff --git a/src/llama.cpp b/src/llama.cpp
index 24e1f1f0..25db4c69 100644
index 24e1f1f0..5d191f92 100644
--- a/src/llama.cpp
+++ b/src/llama.cpp
@@ -205,6 +205,17 @@ enum llm_arch {
@@ -205,6 +205,18 @@ enum llm_arch {
LLM_ARCH_GRANITE,
LLM_ARCH_GRANITE_MOE,
LLM_ARCH_CHAMELEON,
@@ -57,10 +57,11 @@ index 24e1f1f0..25db4c69 100644
+ LLM_ARCH_HIDREAM,
+ LLM_ARCH_COSMOS,
+ LLM_ARCH_LUMINA2,
+ LLM_ARCH_QWEN_IMAGE,
LLM_ARCH_UNKNOWN,
};
@@ -258,6 +269,17 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
@@ -258,6 +270,18 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
{ LLM_ARCH_GRANITE, "granite" },
{ LLM_ARCH_GRANITE_MOE, "granitemoe" },
{ LLM_ARCH_CHAMELEON, "chameleon" },
@@ -75,28 +76,30 @@ index 24e1f1f0..25db4c69 100644
+ { LLM_ARCH_HIDREAM, "hidream" },
+ { LLM_ARCH_COSMOS, "cosmos" },
+ { LLM_ARCH_LUMINA2, "lumina2" },
+ { LLM_ARCH_QWEN_IMAGE, "qwen_image" },
{ LLM_ARCH_UNKNOWN, "(unknown)" },
};
@@ -1531,6 +1553,17 @@ static const std::map<llm_arch, std::map<llm_tensor, const char *>> LLM_TENSOR_N
@@ -1531,6 +1555,18 @@ static const std::map<llm_arch, std::map<llm_tensor, const char *>> LLM_TENSOR_N
{ LLM_TENSOR_ATTN_K_NORM, "blk.%d.attn_k_norm" },
},
},
+ { LLM_ARCH_FLUX, {}},
+ { LLM_ARCH_SD1, {}},
+ { LLM_ARCH_SDXL, {}},
+ { LLM_ARCH_SD3, {}},
+ { LLM_ARCH_AURA, {}},
+ { LLM_ARCH_LTXV, {}},
+ { LLM_ARCH_HYVID, {}},
+ { LLM_ARCH_WAN, {}},
+ { LLM_ARCH_HIDREAM, {}},
+ { LLM_ARCH_COSMOS, {}},
+ { LLM_ARCH_LUMINA2, {}},
+ { LLM_ARCH_FLUX, {}},
+ { LLM_ARCH_SD1, {}},
+ { LLM_ARCH_SDXL, {}},
+ { LLM_ARCH_SD3, {}},
+ { LLM_ARCH_AURA, {}},
+ { LLM_ARCH_LTXV, {}},
+ { LLM_ARCH_HYVID, {}},
+ { LLM_ARCH_WAN, {}},
+ { LLM_ARCH_HIDREAM, {}},
+ { LLM_ARCH_COSMOS, {}},
+ { LLM_ARCH_LUMINA2, {}},
+ { LLM_ARCH_QWEN_IMAGE, {}},
{
LLM_ARCH_UNKNOWN,
{
@@ -5403,6 +5436,25 @@ static void llm_load_hparams(
@@ -5403,6 +5439,26 @@ static void llm_load_hparams(
// get general kv
ml.get_key(LLM_KV_GENERAL_NAME, model.name, false);
@@ -113,6 +116,7 @@ index 24e1f1f0..25db4c69 100644
+ case LLM_ARCH_HIDREAM:
+ case LLM_ARCH_COSMOS:
+ case LLM_ARCH_LUMINA2:
+ case LLM_ARCH_QWEN_IMAGE:
+ model.ftype = ml.ftype;
+ return;
+ default:
@@ -122,7 +126,7 @@ index 24e1f1f0..25db4c69 100644
// get hparams kv
ml.get_key(LLM_KV_VOCAB_SIZE, hparams.n_vocab, false) || ml.get_arr_n(LLM_KV_TOKENIZER_LIST, hparams.n_vocab);
@@ -18016,6 +18068,134 @@ static void llama_tensor_dequantize_internal(
@@ -18016,6 +18072,137 @@ static void llama_tensor_dequantize_internal(
workers.clear();
}
@@ -158,6 +162,7 @@ index 24e1f1f0..25db4c69 100644
+ (name.find(".v.weight") != std::string::npos) ||
+ (name.find(".attn.w1v.weight") != std::string::npos) ||
+ (name.find(".attn.w2v.weight") != std::string::npos) ||
+ (name.find(".add_v_proj.weight") != std::string::npos) ||
+ (name.find("_attn.v_proj.weight") != std::string::npos)
+ ){
+ if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) {
@@ -197,7 +202,9 @@ index 24e1f1f0..25db4c69 100644
+ (name.find(".ff.net.2.weight") != std::string::npos) ||
+ (name.find(".mlp.layer2.weight") != std::string::npos) ||
+ (name.find(".adaln_modulation_mlp.2.weight") != std::string::npos) ||
+ (name.find(".feed_forward.w2.weight") != std::string::npos)
+ (name.find(".feed_forward.w2.weight") != std::string::npos) ||
+ (name.find(".img_mlp.net.2.weight") != std::string::npos) ||
+ (name.find(".txt_mlp.net.0.proj.weight") != std::string::npos)
+ ) {
+ // TODO: add back `layer_info` with some model specific logic + logic further down
+ if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) {
@@ -257,7 +264,7 @@ index 24e1f1f0..25db4c69 100644
static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type new_type, const ggml_tensor * tensor, llama_ftype ftype) {
const std::string name = ggml_get_name(tensor);
@@ -18513,7 +18693,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
@@ -18513,7 +18700,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
if (llama_model_has_encoder(&model)) {
n_attn_layer *= 3;
}
@@ -268,7 +275,7 @@ index 24e1f1f0..25db4c69 100644
}
size_t total_size_org = 0;
@@ -18547,6 +18729,51 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
@@ -18547,6 +18736,51 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
ctx_outs[i_split] = gguf_init_empty();
}
gguf_add_tensor(ctx_outs[i_split], tensor);
@@ -320,7 +327,7 @@ index 24e1f1f0..25db4c69 100644
}
// Set split info if needed
@@ -18647,6 +18874,110 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
@@ -18647,6 +18881,118 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
// do not quantize relative position bias (T5)
quantize &= name.find("attn_rel_b.weight") == std::string::npos;
@@ -423,6 +430,14 @@ index 24e1f1f0..25db4c69 100644
+ quantize &= name.find("context_refiner.") == std::string::npos;
+ quantize &= name.find("noise_refiner.") == std::string::npos;
+ }
+ if (model.arch == LLM_ARCH_QWEN_IMAGE) {
+ image_model = true;
+ quantize &= name.find("img_in.") == std::string::npos;
+ quantize &= name.find("txt_in.") == std::string::npos;
+ quantize &= name.find("time_text_embed.") == std::string::npos;
+ quantize &= name.find("proj_out.") == std::string::npos;
+ quantize &= name.find("norm_out.") == std::string::npos;
+ }
+ // ignore 3D/4D tensors for image models as the code was never meant to handle these
+ if (image_model) {
+ quantize &= ggml_n_dims(tensor) == 2;
@@ -431,7 +446,7 @@ index 24e1f1f0..25db4c69 100644
enum ggml_type new_type;
void * new_data;
size_t new_size;
@@ -18655,6 +18986,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
@@ -18655,6 +19001,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
new_type = default_type;
// get more optimal quantization type based on the tensor shape, layer, etc.
@@ -441,7 +456,7 @@ index 24e1f1f0..25db4c69 100644
if (!params->pure && ggml_is_quantized(default_type)) {
new_type = llama_tensor_get_type(qs, new_type, tensor, ftype);
}
@@ -18664,6 +18998,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
@@ -18664,6 +19013,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
if (params->output_tensor_type < GGML_TYPE_COUNT && strcmp(tensor->name, "output.weight") == 0) {
new_type = params->output_tensor_type;
}