diff --git a/__init__.py b/__init__.py index aaa25ba..32e3401 100644 --- a/__init__.py +++ b/__init__.py @@ -96,8 +96,8 @@ class MZ_LLama3CLIPTextEncode: m_llama3_models[i] += "[downloaded]" - importlib.reload(mz_llama3) - style_presets = mz_llama3.get_style_presets() + importlib.reload(mz_llama_cpp) + style_presets = mz_llama_cpp.get_style_presets() return { "required": { @@ -260,4 +260,50 @@ class MZ_LLamaCPPInterrogator: return (result,) NODE_CLASS_MAPPINGS["MZ_LLamaCPPInterrogator"] = MZ_LLamaCPPInterrogator -NODE_DISPLAY_NAME_MAPPINGS["MZ_LLamaCPPInterrogator"] = f"{AUTHOR_NAME} - LLamaCPP simple interrogator" \ No newline at end of file +NODE_DISPLAY_NAME_MAPPINGS["MZ_LLamaCPPInterrogator"] = f"{AUTHOR_NAME} - LLamaCPP simple interrogator" + + + + +class MZ_BaseLLamaCPPCLIPTextEncode: + @classmethod + def INPUT_TYPES(s): + + importlib.reload(mz_llama_cpp) + style_presets = mz_llama_cpp.get_style_presets() + return { + "required": { + "llama_cpp_model": ("STRING", {"default": "", "placeholder": "model_path"}), + "style_presets": ( + style_presets, {"default": style_presets[0]} + ), + "text": ("STRING", {"multiline": True,}), + "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), + "n_gpu_layers": ("INT", {"default": 40, "min": -1, "max": 0xffffffffffffffff}), + }, + "optional": { + "clip": ("CLIP", ), + "llama_cpp_options": ("LLamaCPPOptions", ), + }, + } + RETURN_TYPES = ("STRING", "CONDITIONING",) + FUNCTION = "encode" + CATEGORY = CATEGORY_NAME + def encode(self, text, llama_cpp_model, style_presets, clip=None, seed=0, n_gpu_layers=40, llama_cpp_options=None): + importlib.reload(mz_llama3) + + + options = {} + if llama_cpp_options is not None: + options = llama_cpp_options.value + text = mz_llama_cpp.base_query_beautify_prompt_text(llama_cpp_model, n_gpu_layers, text, style_presets, options) + conditionings = None + if clip is not None: + tokens = clip.tokenize(text) + cond, pooled = clip.encode_from_tokens(tokens, return_pooled=True,) + conditionings = [[cond, {"pooled_output": pooled}]] + + return (text, conditionings) + +NODE_CLASS_MAPPINGS["MZ_BaseLLamaCPPCLIPTextEncode"] = MZ_BaseLLamaCPPCLIPTextEncode +NODE_DISPLAY_NAME_MAPPINGS["MZ_BaseLLamaCPPCLIPTextEncode"] = f"{AUTHOR_NAME} - BaseLLamaCPPCLIPTextEncode" diff --git a/mz_llama3.py b/mz_llama3.py index 252e0ad..8b07bd0 100644 --- a/mz_llama3.py +++ b/mz_llama3.py @@ -23,18 +23,6 @@ def get_exist_model(model_name): return None -high_quality_prompt = "((high quality:1.4), (best quality:1.4), (masterpiece:1.4), (8K resolution), (2k wallpaper))" -style_presets_prompt = { - "high_quality": high_quality_prompt, - "photography": f"{high_quality_prompt}, (RAW photo, best quality), (realistic, photo-realistic:1.2), (bokeh, cinematic shot, dynamic composition, incredibly detailed, sharpen, details, intricate detail, professional lighting, film lighting, 35mm, anamorphic, lightroom, cinematography, bokeh, lens flare, film grain, HDR10, 8K)", - "illustration": f"{high_quality_prompt}, ((detailed matte painting, intricate detail, splash screen, complementary colors), (detailed),(intricate details),illustration,an extremely delicate and beautiful,ultra-detailed,highres,extremely detailed)", -} -def get_style_presets(): - return [ - "high_quality", - "photography", - "illustration", - ] def query_beautify_prompt_text(model_name, n_gpu_layers, text, style_presets, download_source=None, options={}): if options is None: @@ -46,6 +34,12 @@ def query_beautify_prompt_text(model_name, n_gpu_layers, text, style_presets, do try: + + model_file = None + if options.get("customize_model_file", None) is not None: + model_file = options.get("customize_model_file") + + model_file = get_exist_model(model_name) if model_file is None: @@ -151,7 +145,7 @@ def query_beautify_prompt_text(model_name, n_gpu_layers, text, style_presets, do full_response = mz_prompt_utils.Utils.prompt_zh_to_en(full_response) - style_presets_prompt_text = style_presets_prompt.get(style_presets, "") + style_presets_prompt_text = mz_llama_cpp.style_presets_prompt.get(style_presets, "") full_response = f"{style_presets_prompt_text}, {full_response}" return full_response diff --git a/mz_llama_cpp.py b/mz_llama_cpp.py index 4525ea4..7e57325 100644 --- a/mz_llama_cpp.py +++ b/mz_llama_cpp.py @@ -1,3 +1,5 @@ +import importlib +import json import os import shutil import subprocess @@ -124,7 +126,7 @@ def llama_cpp_messages(model_file, n_gpu_layers, chat_handler=None, messages=[], model = Llama( model_path=model_file, n_gpu_layers=n_gpu_layers, - n_ctx=options.get("n_ctx", 1024), + n_ctx=options.get("n_ctx", 2048), logits_all=options.get("logits_all", False), chat_handler=chat_handler, chat_format=options.get("chat_format", None), @@ -321,6 +323,127 @@ def llava_cpp_simple_interrogator( +high_quality_prompt = "((high quality:1.4), (best quality:1.4), (masterpiece:1.4), (8K resolution), (2k wallpaper))" +style_presets_prompt = { + "high_quality": high_quality_prompt, + "photography": f"{high_quality_prompt}, (RAW photo, best quality), (realistic, photo-realistic:1.2), (bokeh, cinematic shot, dynamic composition, incredibly detailed, sharpen, details, intricate detail, professional lighting, film lighting, 35mm, anamorphic, lightroom, cinematography, bokeh, lens flare, film grain, HDR10, 8K)", + "illustration": f"{high_quality_prompt}, ((detailed matte painting, intricate detail, splash screen, complementary colors), (detailed),(intricate details),illustration,an extremely delicate and beautiful,ultra-detailed,highres,extremely detailed)", +} +def get_style_presets(): + return [ + "high_quality", + "photography", + "illustration", + ] + + +def base_query_beautify_prompt_text(model_file, n_gpu_layers, text, style_presets, options={}): + if options is None: + options = {} + import mz_prompts + importlib.reload(mz_prompts) + + + try: + schema = get_schema_obj( + keys_type={ + "description": get_schema_base_type("string"), + "long_prompt": get_schema_base_type("string"), + "main_color_word": get_schema_base_type("string"), + "camera_angle_word": get_schema_base_type("string"), + "style_words": get_schema_array("string"), + "subject_words": get_schema_array("string"), + "light_words": get_schema_array("string"), + "environment_words": get_schema_array("string"), + }, + required=[ + "description", + "long_prompt", + "main_color_word", + "camera_angle_word", + "style_words", + "subject_words", + "light_words", + "environment_words", + ] + ) + + options["max_tokens"] = options.get("max_tokens", 2048) + options["temperature"] = options.get("temperature", 1.6) + + + response_json = llama_cpp_simple_interrogator_to_json( + model_file=model_file, + n_gpu_layers=n_gpu_layers, + system=mz_prompts.Beautify_Prompt, + question=f"IDEA: {style_presets},{text}", + schema=schema, + options=options, + ) + mz_prompt_utils.Utils.print_log(f"response_json: {response_json}") + response = json.loads(response_json) + freed_gpu_memory(model_file=model_file) + + + full_responses = [] + + if response["description"] != "": + full_responses.append(f"({response['description']})") + if response["long_prompt"] != "": + full_responses.append(f"({response['long_prompt']})") + if response["main_color_word"] != "": + full_responses.append(f"({response['main_color_word']})") + if response["camera_angle_word"] != "": + full_responses.append(f"({response['camera_angle_word']})") + + + response["style_words"] = [x for x in response["style_words"] if x != ""] + if len(response["style_words"]) > 0: + full_responses.append(f"({', '.join(response['style_words'])})") + + + response["subject_words"] = [x for x in response["subject_words"] if x != ""] + if len(response["subject_words"]) > 0: + full_responses.append(f"({', '.join(response['subject_words'])})") + + response["light_words"] = [x for x in response["light_words"] if x != ""] + if len(response["light_words"]) > 0: + full_responses.append(f"({', '.join(response['light_words'])})") + + + response["environment_words"] = [x for x in response["environment_words"] if x != ""] + if len(response["environment_words"]) > 0: + full_responses.append(f"({', '.join(response['environment_words'])})") + + full_response = ", ".join(full_responses) + + + + # 去除换行 + while full_response.find("\n") != -1: + full_response = full_response.replace("\n", " ") + + # 句号换成逗号 + while full_response.find(".") != -1: + full_response = full_response.replace(".", ",") + + # 去除多余逗号 + while full_response.find(",,") != -1: + full_response = full_response.replace(",,", ",") + while full_response.find(", ,") != -1: + full_response = full_response.replace(", ,", ",") + + full_response = mz_prompt_utils.Utils.prompt_zh_to_en(full_response) + + + style_presets_prompt_text = style_presets_prompt.get(style_presets, "") + full_response = f"{style_presets_prompt_text}, {full_response}" + return full_response + + except Exception as e: + freed_gpu_memory(model_file=model_file) + # mz_utils.Utils.print_log(f"Error in auto_prompt_text: {e}") + raise e if __name__ == "__main__": diff --git a/mz_llava.py b/mz_llava.py index f29216b..c6af876 100644 --- a/mz_llava.py +++ b/mz_llava.py @@ -49,8 +49,21 @@ def image_interrogator(model_name, mmproj_name, n_gpu_layers, image, resolution, if options is None: options = {} image = mz_prompt_utils.Utils.resize_max(image, resolution, resolution) - model_file = get_exist_model(model_name) - mmproj_file = get_exist_model(mmproj_name) + + model_file = None + mmproj_file = None + if options.get("customize_model_file", None) is not None: + model_file = options.get("customize_model_file") + + if options.get("customize_mmproj_file", None) is not None: + mmproj_file = options.get("customize_mmproj_file") + + + + if model_file is None: + model_file = get_exist_model(model_name) + if mmproj_file is None: + mmproj_file = get_exist_model(mmproj_name) if model_file is None or mmproj_file is None: if download_source == "modelscope":