nodes/city96-ComfyUI-GGUF
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Keep first/last block in high precision for qwen image

Browse Source
This commit is contained in:
City
2025-08-05 16:17:51 +02:00
parent 001ffcf0b5
commit 7c2ceff6ed
1 changed files with 28 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/lcpp.patch
+28 -7
View File
@@ -39,7 +39,7 @@ index b16c462f..6d1568f1 100644
const int idx = gguf_find_tensor(ctx, name);
if (idx < 0) {
diff --git a/src/llama.cpp b/src/llama.cpp
index 24e1f1f0..5d191f92 100644
index 24e1f1f0..19fdc5df 100644
--- a/src/llama.cpp
+++ b/src/llama.cpp
@@ -205,6 +205,18 @@ enum llm_arch {
@@ -126,7 +126,7 @@ index 24e1f1f0..5d191f92 100644
// get hparams kv
ml.get_key(LLM_KV_VOCAB_SIZE, hparams.n_vocab, false) || ml.get_arr_n(LLM_KV_TOKENIZER_LIST, hparams.n_vocab);
@@ -18016,6 +18072,137 @@ static void llama_tensor_dequantize_internal(
@@ -18016,6 +18072,158 @@ static void llama_tensor_dequantize_internal(
workers.clear();
}
@@ -231,6 +231,27 @@ index 24e1f1f0..5d191f92 100644
+ ++qs.i_ffn_down;
+ }
+
+ // first/last block high precision test
+ if (arch == LLM_ARCH_QWEN_IMAGE){
+ if (
+ (name.find("transformer_blocks.0.") != std::string::npos) ||
+ (name.find("transformer_blocks.59.") != std::string::npos) // this should be dynamic
+ ) {
+ if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q2_K) {
+ new_type = GGML_TYPE_Q4_K;
+ }
+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) {
+ new_type = GGML_TYPE_Q4_K;
+ }
+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) {
+ new_type = GGML_TYPE_Q5_K;
+ }
+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) {
+ new_type = GGML_TYPE_Q6_K;
+ }
+ }
+ }
+
+ // Sanity check for row shape
+ bool convert_incompatible_tensor = false;
+ if (new_type == GGML_TYPE_Q2_K || new_type == GGML_TYPE_Q3_K || new_type == GGML_TYPE_Q4_K ||
@@ -264,7 +285,7 @@ index 24e1f1f0..5d191f92 100644
static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type new_type, const ggml_tensor * tensor, llama_ftype ftype) {
const std::string name = ggml_get_name(tensor);
@@ -18513,7 +18700,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
@@ -18513,7 +18721,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
if (llama_model_has_encoder(&model)) {
n_attn_layer *= 3;
}
@@ -275,7 +296,7 @@ index 24e1f1f0..5d191f92 100644
}
size_t total_size_org = 0;
@@ -18547,6 +18736,51 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
@@ -18547,6 +18757,51 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
ctx_outs[i_split] = gguf_init_empty();
}
gguf_add_tensor(ctx_outs[i_split], tensor);
@@ -327,7 +348,7 @@ index 24e1f1f0..5d191f92 100644
}
// Set split info if needed
@@ -18647,6 +18881,118 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
@@ -18647,6 +18902,118 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
// do not quantize relative position bias (T5)
quantize &= name.find("attn_rel_b.weight") == std::string::npos;
@@ -446,7 +467,7 @@ index 24e1f1f0..5d191f92 100644
enum ggml_type new_type;
void * new_data;
size_t new_size;
@@ -18655,6 +19001,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
@@ -18655,6 +19022,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
new_type = default_type;
// get more optimal quantization type based on the tensor shape, layer, etc.
@@ -456,7 +477,7 @@ index 24e1f1f0..5d191f92 100644
if (!params->pure && ggml_is_quantized(default_type)) {
new_type = llama_tensor_get_type(qs, new_type, tensor, ftype);
}
@@ -18664,6 +19013,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
@@ -18664,6 +19034,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
if (params->output_tensor_type < GGML_TYPE_COUNT && strcmp(tensor->name, "output.weight") == 0) {
new_type = params->output_tensor_type;
}