'update'

2024-04-23 01:28:55 +08:00
parent eeb198c088
commit bd608c22bd
1 changed files with 3 additions and 5 deletions
@@ -69,8 +69,7 @@ def check_llama_cpp_requirements():


 def freed_gpu_memory(model_file):
-    check_llama_cpp_requirements() 
-    import llama_cpp
+    check_llama_cpp_requirements()  
    
    model_and_opt = mz_utils.Utils.cache_get(f"llama_cpp_model_and_opt_{model_file}")

@@ -79,7 +78,8 @@ def freed_gpu_memory(model_file):
    
    model = model_and_opt.get("model")

-    llama_cpp.llama_free(model.ctx)
+    del model
+    torch.cuda.empty_cache()

    mz_utils.Utils.cache_set(f"llama_cpp_model_and_opt_{model_file}", None)

@@ -91,8 +91,6 @@ def llama_cpp_messages(model_file, n_gpu_layers, chat_handler=None, messages=[],

    from llama_cpp import Llama

-
-
    model_and_opt = mz_utils.Utils.cache_get(f"llama_cpp_model_and_opt_{model_file}")
    
    # compared