diff --git a/tools/lcpp.patch b/tools/lcpp.patch index 86dd622..49e5971 100644 --- a/tools/lcpp.patch +++ b/tools/lcpp.patch @@ -39,7 +39,7 @@ index b16c462f..6d1568f1 100644 const int idx = gguf_find_tensor(ctx, name); if (idx < 0) { diff --git a/src/llama.cpp b/src/llama.cpp -index 24e1f1f0..5d191f92 100644 +index 24e1f1f0..19fdc5df 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -205,6 +205,18 @@ enum llm_arch { @@ -126,7 +126,7 @@ index 24e1f1f0..5d191f92 100644 // get hparams kv ml.get_key(LLM_KV_VOCAB_SIZE, hparams.n_vocab, false) || ml.get_arr_n(LLM_KV_TOKENIZER_LIST, hparams.n_vocab); -@@ -18016,6 +18072,137 @@ static void llama_tensor_dequantize_internal( +@@ -18016,6 +18072,158 @@ static void llama_tensor_dequantize_internal( workers.clear(); } @@ -231,6 +231,27 @@ index 24e1f1f0..5d191f92 100644 + ++qs.i_ffn_down; + } + ++ // first/last block high precision test ++ if (arch == LLM_ARCH_QWEN_IMAGE){ ++ if ( ++ (name.find("transformer_blocks.0.") != std::string::npos) || ++ (name.find("transformer_blocks.59.") != std::string::npos) // this should be dynamic ++ ) { ++ if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q2_K) { ++ new_type = GGML_TYPE_Q4_K; ++ } ++ else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) { ++ new_type = GGML_TYPE_Q4_K; ++ } ++ else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) { ++ new_type = GGML_TYPE_Q5_K; ++ } ++ else if (ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) { ++ new_type = GGML_TYPE_Q6_K; ++ } ++ } ++ } ++ + // Sanity check for row shape + bool convert_incompatible_tensor = false; + if (new_type == GGML_TYPE_Q2_K || new_type == GGML_TYPE_Q3_K || new_type == GGML_TYPE_Q4_K || @@ -264,7 +285,7 @@ index 24e1f1f0..5d191f92 100644 static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type new_type, const ggml_tensor * tensor, llama_ftype ftype) { const std::string name = ggml_get_name(tensor); -@@ -18513,7 +18700,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s +@@ -18513,7 +18721,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s if (llama_model_has_encoder(&model)) { n_attn_layer *= 3; } @@ -275,7 +296,7 @@ index 24e1f1f0..5d191f92 100644 } size_t total_size_org = 0; -@@ -18547,6 +18736,51 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s +@@ -18547,6 +18757,51 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s ctx_outs[i_split] = gguf_init_empty(); } gguf_add_tensor(ctx_outs[i_split], tensor); @@ -327,7 +348,7 @@ index 24e1f1f0..5d191f92 100644 } // Set split info if needed -@@ -18647,6 +18881,118 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s +@@ -18647,6 +18902,118 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s // do not quantize relative position bias (T5) quantize &= name.find("attn_rel_b.weight") == std::string::npos; @@ -446,7 +467,7 @@ index 24e1f1f0..5d191f92 100644 enum ggml_type new_type; void * new_data; size_t new_size; -@@ -18655,6 +19001,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s +@@ -18655,6 +19022,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s new_type = default_type; // get more optimal quantization type based on the tensor shape, layer, etc. @@ -456,7 +477,7 @@ index 24e1f1f0..5d191f92 100644 if (!params->pure && ggml_is_quantized(default_type)) { new_type = llama_tensor_get_type(qs, new_type, tensor, ftype); } -@@ -18664,6 +19013,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s +@@ -18664,6 +19034,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s if (params->output_tensor_type < GGML_TYPE_COUNT && strcmp(tensor->name, "output.weight") == 0) { new_type = params->output_tensor_type; }