diff --git a/__init__.py b/__init__.py index 7487ad8..fa90f03 100644 --- a/__init__.py +++ b/__init__.py @@ -50,6 +50,43 @@ llama3_models = [ "Meta-Llama-3-8B-Instruct.Q8_0.gguf", ] +class MZ_LLamaCPPOptions: + @classmethod + def INPUT_TYPES(s): + opt = mz_llama_cpp.LlamaCppOptions() + value = opt.value + result = {} + + for key in value: + if type(value[key]) == bool: + result[key] = ([True, False], {"default": value[key]}) + elif type(value[key]) == int: + result[key] = ("INT", {"default": value[key]}) + elif type(value[key]) == float: + result[key] = ("FLOAT", {"default": value[key]}) + elif type(value[key]) == str: + result[key] = ("STRING", {"default": value[key]}) + else: + raise Exception(f"Unknown type: {type(value[key])}") + + return { + "required": result, + } + + + RETURN_TYPES = ("LLamaCPPOptions",) + FUNCTION = "create" + CATEGORY = CATEGORY_NAME + def create(self, **kwargs): + importlib.reload(mz_llama_cpp) + opt = mz_llama_cpp.LlamaCppOptions() + for key in kwargs: + opt.value[key] = kwargs[key] + return (opt,) + +NODE_CLASS_MAPPINGS["MZ_LLamaCPPOptions"] = MZ_LLamaCPPOptions +NODE_DISPLAY_NAME_MAPPINGS["MZ_LLamaCPPOptions"] = f"{AUTHOR_NAME} - LLamaCPPOptions" + class MZ_LLama3CLIPTextEncode: @classmethod def INPUT_TYPES(s): @@ -64,7 +101,7 @@ class MZ_LLama3CLIPTextEncode: return { "required": { - "llama3_model": (m_llama3_models, {"default": m_llama3_models[0]}), + "llama_cpp_model": (m_llama3_models, {"default": m_llama3_models[0]}), "download_source": ( ["none", "modelscope", "hf-mirror.com",], {"default": "none"} @@ -78,17 +115,18 @@ class MZ_LLama3CLIPTextEncode: }, "optional": { "clip": ("CLIP", ), + "options": ("LLamaCPPOptions", ), }, } RETURN_TYPES = ("STRING", "CONDITIONING",) FUNCTION = "encode" CATEGORY = CATEGORY_NAME - def encode(self, text, llama3_model, style_presets, clip=None, download_source=None, seed=0, n_gpu_layers=40): + def encode(self, text, llama_cpp_model, style_presets, clip=None, download_source=None, seed=0, n_gpu_layers=40): importlib.reload(mz_llama3) - llama3_model = llama3_model.replace("[downloaded]", "") + llama_cpp_model = llama_cpp_model.replace("[downloaded]", "") - text = mz_llama3.query_beautify_prompt_text(llama3_model, n_gpu_layers, text, style_presets, download_source,) + text = mz_llama3.query_beautify_prompt_text(llama_cpp_model, n_gpu_layers, text, style_presets, download_source,) conditionings = None if clip is not None: tokens = clip.tokenize(text) @@ -132,7 +170,7 @@ class MZ_LLavaImageInterrogator: return {"required": { - "llava_model": (m_llava_models, {"default": m_llava_models[0]}), + "llama_cpp_model": (m_llava_models, {"default": m_llava_models[0]}), "mmproj_model": (m_llava_mmproj_models, {"default": m_llava_mmproj_models[0]}), "download_source": ( [ @@ -149,15 +187,16 @@ class MZ_LLavaImageInterrogator: "n_gpu_layers": ("INT", {"default": 40, "min": -1, "max": 0xffffffffffffffff}), }, "optional": { - "clip": ("CLIP",), + "clip": ("CLIP", ), + "options": ("LLamaCPPOptions", ), }} RETURN_TYPES = ("STRING", "CONDITIONING",) FUNCTION = "interrogate" CATEGORY = CATEGORY_NAME - def interrogate(self, llava_model, mmproj_model, image, resolution, prefix, download_source=None, seed=0, clip=None, n_gpu_layers=40): + def interrogate(self, llama_cpp_model, mmproj_model, image, resolution, prefix, download_source=None, seed=0, clip=None, n_gpu_layers=40, options={}): importlib.reload(mz_llava) - llava_model = llava_model.replace("[downloaded]", "") + llama_cpp_model = llama_cpp_model.replace("[downloaded]", "") mmproj_model = mmproj_model.replace("[downloaded]", "") prefix = Utils.prompt_zh_to_en(prefix) @@ -166,12 +205,13 @@ class MZ_LLavaImageInterrogator: image_pil = Utils.tensor2pil(image) response = mz_llava.image_interrogator( - llava_model, + llama_cpp_model, mmproj_model, n_gpu_layers, image_pil, resolution, download_source, + options, ) conditionings = None if clip is not None: diff --git a/mz_llama3.py b/mz_llama3.py index fe52308..f2a61d6 100644 --- a/mz_llama3.py +++ b/mz_llama3.py @@ -36,7 +36,9 @@ def get_style_presets(): "illustration", ] -def query_beautify_prompt_text(model_name, n_gpu_layers, text, style_presets, download_source=None): +def query_beautify_prompt_text(model_name, n_gpu_layers, text, style_presets, download_source=None, options={}): + if options is None: + options = {} import mz_prompts importlib.reload(mz_prompts) importlib.reload(mz_llama_cpp) @@ -81,14 +83,17 @@ def query_beautify_prompt_text(model_name, n_gpu_layers, text, style_presets, do ] ) + options["max_tokens"] = options.get("max_tokens", 2048) + options["temperature"] = options.get("temperature", 1.6) + + response_json = mz_llama_cpp.llama_cpp_simple_interrogator_to_json( model_file=model_file, n_gpu_layers=n_gpu_layers, system=mz_prompts.Beautify_Prompt, question=f"IDEA: {style_presets},{text}", schema=schema, - temperature=1.6, - max_tokens=2048, + options=options, ) mz_utils.Utils.print_log(f"response_json: {response_json}") response = json.loads(response_json) diff --git a/mz_llama_cpp.py b/mz_llama_cpp.py index ae6e841..b94c878 100644 --- a/mz_llama_cpp.py +++ b/mz_llama_cpp.py @@ -8,6 +8,13 @@ try: except ImportError: pass + +class LlamaCppOptions: + value = { + "max_tokens": 2048, + "temperature": 0.8, + } + def check_llama_cpp_requirements(): last_version = "0.2.63" try: @@ -84,6 +91,8 @@ def freed_gpu_memory(model_file): mz_utils.Utils.cache_set(f"llama_cpp_model_and_opt_{model_file}", None) def llama_cpp_messages(model_file, n_gpu_layers, chat_handler=None, messages=[], options={}): + if options is None: + options = {} print(f"Find local model file: {model_file}") init_opts = ["n_ctx", "logits_all", "chat_format",] @@ -176,7 +185,7 @@ def get_schema_array(item_type="string"): -def llama_cpp_simple_interrogator_to_json(model_file, n_gpu_layers, use_system=True, system=None, question="", schema={}, max_tokens=1024, temperature=0.8): +def llama_cpp_simple_interrogator_to_json(model_file, n_gpu_layers, use_system=True, system=None, question="", schema={}, options={}): if system is None: system = "" messages = [ @@ -218,14 +227,18 @@ def llama_cpp_simple_interrogator_to_json(model_file, n_gpu_layers, use_system=T } - return llama_cpp_messages(model_file, n_gpu_layers, None, messages, options={ - "response_format": response_format, - "chat_format":"chatml", - "temperature": temperature, - "max_tokens": max_tokens, - }) + options["response_format"] = response_format + options["chat_format"] = "chatml" + if options.get("temperature", None) is None: + options["temperature"] = 0.8 + if options.get("max_tokens", None) is None: + options["max_tokens"] = 2048 + + return llama_cpp_messages(model_file, n_gpu_layers, None, messages, options=options) -def llama_cpp_simple_interrogator(model_file, n_gpu_layers, use_system=True, system=None, question=""): +def llama_cpp_simple_interrogator(model_file, n_gpu_layers, use_system=True, system=None, question="", options={}): + if options is None: + options = {} if system is None: system = "" messages = [ @@ -260,17 +273,25 @@ def llama_cpp_simple_interrogator(model_file, n_gpu_layers, use_system=True, sys "content": question }, ] - return llama_cpp_messages(model_file, n_gpu_layers, None, messages) + return llama_cpp_messages(model_file, n_gpu_layers, None, messages, options={}) def llava_cpp_messages(model_file, n_gpu_layers, chat_handler, messages, options={}): + if options is None: + options = {} options["logits_all"] = True options["n_ctx"] = max(2048, options.get("n_ctx", 2048)) return llama_cpp_messages(model_file, n_gpu_layers, chat_handler, messages, options) -def llava_cpp_simple_interrogator(model_file, mmproj_file, n_gpu_layers, system="You are an assistant who perfectly describes images.", question="Describe this image in detail please.", image=None): +def llava_cpp_simple_interrogator( + model_file, mmproj_file, n_gpu_layers, system="You are an assistant who perfectly describes images.", question="Describe this image in detail please.", + image=None, options={}): + if options is None: + options = {} + + check_llama_cpp_requirements() content = [] @@ -294,7 +315,7 @@ def llava_cpp_simple_interrogator(model_file, mmproj_file, n_gpu_layers, system= "role": "user", "content": content, }, - ]) + ], options=options) diff --git a/mz_llava.py b/mz_llava.py index 64071af..8487728 100644 --- a/mz_llava.py +++ b/mz_llava.py @@ -45,7 +45,9 @@ def get_exist_model(model_name): return None -def image_interrogator(model_name, mmproj_name, n_gpu_layers, image, resolution, download_source=None): +def image_interrogator(model_name, mmproj_name, n_gpu_layers, image, resolution, download_source=None, options={}): + if options is None: + options = {} image = mz_utils.Utils.resize_max(image, resolution, resolution) model_file = get_exist_model(model_name) mmproj_file = get_exist_model(mmproj_name) @@ -95,6 +97,7 @@ def image_interrogator(model_name, mmproj_name, n_gpu_layers, image, resolution, mmproj_file=mmproj_file, n_gpu_layers=n_gpu_layers, image=image, + options=options, )