From bd608c22bd0fb2f3454d630339035edf824b30c0 Mon Sep 17 00:00:00 2001 From: wailovet Date: Tue, 23 Apr 2024 01:28:55 +0800 Subject: [PATCH] 'update' --- mz_llama_cpp.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/mz_llama_cpp.py b/mz_llama_cpp.py index 5ab3e74..ae6e841 100644 --- a/mz_llama_cpp.py +++ b/mz_llama_cpp.py @@ -69,8 +69,7 @@ def check_llama_cpp_requirements(): def freed_gpu_memory(model_file): - check_llama_cpp_requirements() - import llama_cpp + check_llama_cpp_requirements() model_and_opt = mz_utils.Utils.cache_get(f"llama_cpp_model_and_opt_{model_file}") @@ -79,7 +78,8 @@ def freed_gpu_memory(model_file): model = model_and_opt.get("model") - llama_cpp.llama_free(model.ctx) + del model + torch.cuda.empty_cache() mz_utils.Utils.cache_set(f"llama_cpp_model_and_opt_{model_file}", None) @@ -91,8 +91,6 @@ def llama_cpp_messages(model_file, n_gpu_layers, chat_handler=None, messages=[], from llama_cpp import Llama - - model_and_opt = mz_utils.Utils.cache_get(f"llama_cpp_model_and_opt_{model_file}") # compared