From 9fbb8b720ced863fbbf6f7c277aaa70d4790469d Mon Sep 17 00:00:00 2001 From: wailovet Date: Tue, 23 Apr 2024 00:51:22 +0800 Subject: [PATCH] 'update' --- __init__.py | 131 ++++-------------------------------------------- mz_llama3.py | 3 +- mz_llama_cpp.py | 21 ++++---- mz_llava.py | 3 +- 4 files changed, 24 insertions(+), 134 deletions(-) diff --git a/__init__.py b/__init__.py index 60db416..7487ad8 100644 --- a/__init__.py +++ b/__init__.py @@ -74,6 +74,7 @@ class MZ_LLama3CLIPTextEncode: ), "text": ("STRING", {"multiline": True,}), "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), + "n_gpu_layers": ("INT", {"default": 40, "min": -1, "max": 0xffffffffffffffff}), }, "optional": { "clip": ("CLIP", ), @@ -82,12 +83,12 @@ class MZ_LLama3CLIPTextEncode: RETURN_TYPES = ("STRING", "CONDITIONING",) FUNCTION = "encode" CATEGORY = CATEGORY_NAME - def encode(self, text, llama3_model, style_presets, clip=None, download_source=None, seed=0): + def encode(self, text, llama3_model, style_presets, clip=None, download_source=None, seed=0, n_gpu_layers=40): importlib.reload(mz_llama3) llama3_model = llama3_model.replace("[downloaded]", "") - text = mz_llama3.query_beautify_prompt_text(llama3_model, text, style_presets, download_source,) + text = mz_llama3.query_beautify_prompt_text(llama3_model, n_gpu_layers, text, style_presets, download_source,) conditionings = None if clip is not None: tokens = clip.tokenize(text) @@ -145,6 +146,7 @@ class MZ_LLavaImageInterrogator: "resolution": ("INT", {"default": 512, "min": 128, "max": 2048}), "prefix" : ("STRING", {"default": "(masterpiece)", "multiline": True, }), "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), + "n_gpu_layers": ("INT", {"default": 40, "min": -1, "max": 0xffffffffffffffff}), }, "optional": { "clip": ("CLIP",), @@ -152,7 +154,7 @@ class MZ_LLavaImageInterrogator: RETURN_TYPES = ("STRING", "CONDITIONING",) FUNCTION = "interrogate" CATEGORY = CATEGORY_NAME - def interrogate(self, llava_model, mmproj_model, image, resolution, prefix, download_source=None, seed=0, clip=None): + def interrogate(self, llava_model, mmproj_model, image, resolution, prefix, download_source=None, seed=0, clip=None, n_gpu_layers=40): importlib.reload(mz_llava) llava_model = llava_model.replace("[downloaded]", "") @@ -166,6 +168,7 @@ class MZ_LLavaImageInterrogator: response = mz_llava.image_interrogator( llava_model, mmproj_model, + n_gpu_layers, image_pil, resolution, download_source, @@ -180,125 +183,7 @@ class MZ_LLavaImageInterrogator: NODE_CLASS_MAPPINGS["MZ_LLavaImageInterrogator"] = MZ_LLavaImageInterrogator NODE_DISPLAY_NAME_MAPPINGS["MZ_LLavaImageInterrogator"] = f"{AUTHOR_NAME} - LLavaImageInterrogator" - -qwen_models = [ - "Qwen1.5-1.8B-Chat-GPTQ-Int4", - "Qwen1.5-7B-Chat-GPTQ-Int4", - "Qwen1.5-14B-Chat-GPTQ-Int4", - "Qwen1.5-32B-Chat-GPTQ-Int4", - "Qwen1.5-72B-Chat-GPTQ-Int4", -] - -class MZ_QWenOptionalArgs: - model_id = "" - download_source = "" - customize_prompt = "" - -class MZ_QWenOptional: - @classmethod - def INPUT_TYPES(s): - # qwen_model - return { - "required": { - "model_id": ("STRING", {"default": ""}), - "customize_prompt": ("STRING", {"multiline": True, "default": ""}), - }, - } - RETURN_TYPES = ("MZ_QWenOptionalArgs",) - RETURN_NAMES = ("qwen_optional",) - FUNCTION = "optional" - CATEGORY = CATEGORY_NAME - def optional(self, model_id, download_source, customize_prompt=""): - result = MZ_QWenOptionalArgs() - result.model_id = model_id - result.download_source = download_source - result.customize_prompt = customize_prompt - return result - -# NODE_CLASS_MAPPINGS["MZ_QWenOptional"] = MZ_QWenOptional -# NODE_DISPLAY_NAME_MAPPINGS["MZ_QWenOptional"] = f"{AUTHOR_NAME} - QWenOptionalParameter" - - - -class MZ_QWenCLIPTextEncode: - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "qwen_model": (qwen_models, {"default": "Qwen1.5-1.8B-Chat-GPTQ-Int4"}), - "download_source": ( - ["none", "modelscope", "https://hf-mirror.com",], - {"default": "none"} - ), - "text": ("STRING", {"multiline": True,}), - "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), - }, - "optional": { - "clip": ("CLIP", ), - "qwen_optional": ("MZ_QWenOptionalArgs",), - }, - } - RETURN_TYPES = ("STRING", "CONDITIONING",) - FUNCTION = "encode" - CATEGORY = CATEGORY_NAME - def encode(self, clip, text, qwen_model, download_source=None, seed=0, qwen_optional=None): - importlib.reload(mz_qwen) - text = mz_qwen.auto_prompt_text(qwen_model, text, download_source, qwen_optional=qwen_optional,) - conditionings = None - if clip is not None: - tokens = clip.tokenize(text) - cond, pooled = clip.encode_from_tokens(tokens, return_pooled=True,) - conditionings = [[cond, {"pooled_output": pooled}]] - return (text, conditionings) - - -# NODE_CLASS_MAPPINGS["MZ_QWenCLIPTextEncode"] = MZ_QWenCLIPTextEncode -# NODE_DISPLAY_NAME_MAPPINGS["MZ_QWenCLIPTextEncode"] = f"{AUTHOR_NAME} - QWenCLIPTextEncode" - - - -class MZ_QWenImageInterrogator: - @classmethod - def INPUT_TYPES(s): - return {"required": { - "image": ("IMAGE",), - "resolution": ("INT", {"default": 512, "min": 128, "max": 2048}), - "prefix" : ("STRING", {"default": "(masterpiece)", "multiline": True, }), - "download_source": ( - ["none", "modelscope", "https://hf-mirror.com",], - {"default": "none"} - ), - "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), - }, - "optional": { - "clip": ("CLIP",), - "qwen_optional": ("MZ_QWenOptionalArgs",), - }} - RETURN_TYPES = ("STRING", "CONDITIONING",) - FUNCTION = "interrogate" - CATEGORY = CATEGORY_NAME - def interrogate(self, image, resolution, prefix, download_source=None, seed=0, clip=None, qwen_optional=None): - importlib.reload(mz_qwen) - prefix = Utils.prompt_zh_to_en(prefix) - if not prefix.endswith(","): - prefix += "," - response = prefix + mz_qwen.auto_image_to_text(image, resolution, download_source, qwen_optional=qwen_optional,) - - conditionings = None - if clip is not None: - tokens = clip.tokenize(response) - cond, pooled = clip.encode_from_tokens(tokens, return_pooled=True) - conditionings = [[cond, {"pooled_output": pooled}]] - return (response, conditionings) - -# NODE_CLASS_MAPPINGS["MZ_QWenImageInterrogator"] = MZ_QWenImageInterrogator -# NODE_DISPLAY_NAME_MAPPINGS["MZ_QWenImageInterrogator"] = f"{AUTHOR_NAME} - QWenImageInterrogator" - - - - - class MZ_LLamaCPPInterrogator: @classmethod def INPUT_TYPES(s): @@ -308,15 +193,17 @@ class MZ_LLamaCPPInterrogator: "use_system": ([True, False], {"default": True}), "system": ("STRING", {"multiline": True, "default": "You are a helpful assistant."}), "prompt": ("STRING", {"multiline": True,}), + "n_gpu_layers": ("INT", {"default": 40, "min": -1, "max": 0xffffffffffffffff}), }, } RETURN_TYPES = ("STRING",) FUNCTION = "simple_interrogator" CATEGORY = CATEGORY_NAME - def simple_interrogator(self, model_file, prompt, use_system=True, system="You are a helpful assistant."): + def simple_interrogator(self, model_file, prompt, use_system=True, system="You are a helpful assistant.", n_gpu_layers=40): importlib.reload(mz_llama_cpp) result = mz_llama_cpp.llama_cpp_simple_interrogator( model_file, + n_gpu_layers=n_gpu_layers, use_system=use_system, system=system, question=prompt, diff --git a/mz_llama3.py b/mz_llama3.py index 8d95ef1..74bc143 100644 --- a/mz_llama3.py +++ b/mz_llama3.py @@ -36,7 +36,7 @@ def get_style_presets(): "illustration", ] -def query_beautify_prompt_text(model_name, text, style_presets, download_source=None, ): +def query_beautify_prompt_text(model_name, n_gpu_layers, text, style_presets, download_source=None): import mz_prompts importlib.reload(mz_prompts) importlib.reload(mz_llama_cpp) @@ -83,6 +83,7 @@ def query_beautify_prompt_text(model_name, text, style_presets, download_source= response_json = mz_llama_cpp.llama_cpp_simple_interrogator_to_json( model_file=model_file, + n_gpu_layers=n_gpu_layers, system=mz_prompts.Beautify_Prompt, question=f"IDEA: {style_presets},{text}", schema=schema, diff --git a/mz_llama_cpp.py b/mz_llama_cpp.py index 0f9ad24..415af5d 100644 --- a/mz_llama_cpp.py +++ b/mz_llama_cpp.py @@ -70,7 +70,7 @@ def check_llama_cpp_requirements(): -def llama_cpp_messages(model_file, chat_handler=None, messages=[], options={}): +def llama_cpp_messages(model_file, n_gpu_layers, chat_handler=None, messages=[], options={}): print(f"Find local model file: {model_file}") check_llama_cpp_requirements() @@ -97,6 +97,7 @@ def llama_cpp_messages(model_file, chat_handler=None, messages=[], options={}): model = Llama( model_path=model_file, + n_gpu_layers=n_gpu_layers, n_ctx=options.get("n_ctx", 1024), logits_all=options.get("logits_all", False), chat_handler=chat_handler, @@ -158,7 +159,7 @@ def get_schema_array(item_type="string"): -def llama_cpp_simple_interrogator_to_json(model_file, use_system=True, system=None, question="", schema={}, max_tokens=1024, temperature=0.8): +def llama_cpp_simple_interrogator_to_json(model_file, n_gpu_layers, use_system=True, system=None, question="", schema={}, max_tokens=1024, temperature=0.8): if system is None: system = "" @@ -201,14 +202,14 @@ def llama_cpp_simple_interrogator_to_json(model_file, use_system=True, system=No } - return llama_cpp_messages(model_file, None, messages, options={ + return llama_cpp_messages(model_file, n_gpu_layers, None, messages, options={ "response_format": response_format, "chat_format":"chatml", "temperature": temperature, - "max_tokens": max_tokens, + "max_tokens": max_tokens, }) -def llama_cpp_simple_interrogator(model_file, use_system=True, system=None, question=""): +def llama_cpp_simple_interrogator(model_file, n_gpu_layers, use_system=True, system=None, question=""): if system is None: system = "" messages = [ @@ -243,17 +244,17 @@ def llama_cpp_simple_interrogator(model_file, use_system=True, system=None, ques "content": question }, ] - return llama_cpp_messages(model_file, None, messages) + return llama_cpp_messages(model_file, n_gpu_layers, None, messages) -def llava_cpp_messages(model_file, chat_handler, messages, options={}): +def llava_cpp_messages(model_file, n_gpu_layers, chat_handler, messages, options={}): options["logits_all"] = True options["n_ctx"] = max(2048, options.get("n_ctx", 2048)) - return llama_cpp_messages(model_file, chat_handler, messages, options) + return llama_cpp_messages(model_file, n_gpu_layers, chat_handler, messages, options) -def llava_cpp_simple_interrogator(model_file, mmproj_file, system="You are an assistant who perfectly describes images.", question="Describe this image in detail please.", image=None): +def llava_cpp_simple_interrogator(model_file, mmproj_file, n_gpu_layers, system="You are an assistant who perfectly describes images.", question="Describe this image in detail please.", image=None): check_llama_cpp_requirements() content = [] @@ -268,7 +269,7 @@ def llava_cpp_simple_interrogator(model_file, mmproj_file, system="You are an as if mmproj_file is not None: chat_handler = Llava15ChatHandler(clip_model_path=mmproj_file) - return llava_cpp_messages(model_file, chat_handler, [ + return llava_cpp_messages(model_file, n_gpu_layers, chat_handler, [ { "role": "system", "content": system, diff --git a/mz_llava.py b/mz_llava.py index 31353c9..6e0df7c 100644 --- a/mz_llava.py +++ b/mz_llava.py @@ -45,7 +45,7 @@ def get_exist_model(model_name): return None -def image_interrogator(model_name, mmproj_name, image, resolution, download_source=None): +def image_interrogator(model_name, mmproj_name, n_gpu_layers, image, resolution, download_source=None): image = mz_utils.Utils.resize_max(image, resolution, resolution) model_file = get_exist_model(model_name) mmproj_file = get_exist_model(mmproj_name) @@ -93,6 +93,7 @@ def image_interrogator(model_name, mmproj_name, image, resolution, download_sour response = mz_llama_cpp.llava_cpp_simple_interrogator( model_file=model_file, mmproj_file=mmproj_file, + n_gpu_layers=n_gpu_layers, image=image, )