From 9bb6e8919d2aed52d30335a864d1ef109a890898 Mon Sep 17 00:00:00 2001 From: wailovet Date: Sat, 1 Jun 2024 03:42:52 +0800 Subject: [PATCH] 'update' --- __init__.py | 242 ++++++++----------------- mz_gen_translate.py | 1 + mz_llama_core_nodes.py | 216 +++++++++++++++++++++++ mz_llama_cpp.py | 302 +++----------------------------- mz_llava.py | 8 +- mz_openaiapi.py | 6 +- mz_prompt_utils.py | 6 + mz_prompt_webserver.py | 2 +- mz_prompts.py | 15 ++ v1/init.py | 195 +++++++++++++++++++++ v1/mz_deprecated.py | 160 +++++++++++++++++ mz_llama3.py => v1/mz_llama3.py | 12 +- mz_phi3.py => v1/mz_phi3.py | 11 +- web/prompt_mz.js | 4 - 14 files changed, 708 insertions(+), 472 deletions(-) create mode 100644 mz_llama_core_nodes.py create mode 100644 v1/init.py create mode 100644 v1/mz_deprecated.py rename mz_llama3.py => v1/mz_llama3.py (92%) rename mz_phi3.py => v1/mz_phi3.py (92%) diff --git a/__init__.py b/__init__.py index 159e56e..1744806 100644 --- a/__init__.py +++ b/__init__.py @@ -8,6 +8,7 @@ from nodes import MAX_RESOLUTION import comfy.utils import shutil import comfy.samplers +import folder_paths WEB_DIRECTORY = "./web" @@ -15,12 +16,12 @@ WEB_DIRECTORY = "./web" AUTHOR_NAME = u"MinusZone" CATEGORY_NAME = f"{AUTHOR_NAME} - Prompt" -sys.path.append(os.path.join(os.path.dirname(__file__))) +# sys.path.append(os.path.join(os.path.dirname(__file__))) import importlib -import mz_prompt_webserver -mz_prompt_webserver.start_server() +# import mz_prompt_webserver +# mz_prompt_webserver.start_server() NODE_CLASS_MAPPINGS = { } @@ -29,11 +30,64 @@ NODE_CLASS_MAPPINGS = { NODE_DISPLAY_NAME_MAPPINGS = { } + +from . import mz_llama_cpp +from . import mz_llava -import mz_llama3 -import mz_phi3 -import mz_llama_cpp -import mz_llava + +class MZ_LLamaCPPModelConfig_ManualSelect: + @classmethod + def INPUT_TYPES(s): + gguf_files = folder_paths.get_filename_list("gguf") + return { + "required": { + "llama_cpp_model": (gguf_files,), + }, + "optional": { + }, + } + + RETURN_TYPES = ("LLamaCPPModelConfig",) + RETURN_NAMES = ("llama_cpp_model_config",) + + FUNCTION = "create" + CATEGORY = CATEGORY_NAME + + def create(self, **kwargs): + return (kwargs,) + + +class MZ_LLamaCPPCLIPTextEncode: + @classmethod + def INPUT_TYPES(s): + importlib.reload(mz_llama_cpp) + + result = { + "required": { + }, + "optional": { + "llama_cpp_model": ("LLamaCPPModelConfig",), + }, + } + + common_input = getCommonCLIPTextEncodeInput() + for key in common_input["required"]: + result["required"][key] = common_input["required"][key] + for key in common_input["optional"]: + result["optional"][key] = common_input["optional"][key] + + return result + RETURN_TYPES = ("STRING", "CONDITIONING",) + RETURN_NAMES = ("text", "conditioning",) + OUTPUT_NODE = True + FUNCTION = "encode" + CATEGORY = CATEGORY_NAME + + def encode(self, **kwargs): + from . import mz_llama_core_nodes + importlib.reload(mz_llama_core_nodes) + + return mz_llama_core_nodes.llama_cpp_node_encode(kwargs) class MZ_LLamaCPPOptions: @@ -83,13 +137,12 @@ NODE_DISPLAY_NAME_MAPPINGS["MZ_LLamaCPPOptions"] = f"{AUTHOR_NAME} - LLamaCPPOpt class MZ_CustomizeInstruct: @classmethod def INPUT_TYPES(s): - import mz_prompts + from . import mz_prompts + return { "required": { "system": ("STRING", {"multiline": True, "default": mz_prompts.Long_prompt}), "instruct": ("STRING", {"multiline": True, "default": "Short: %text%"}), - "start_str": ("STRING", {"default": "Long: "}), - "end_str": ("STRING", {"default": ""}), }, } @@ -107,7 +160,8 @@ NODE_DISPLAY_NAME_MAPPINGS["MZ_CustomizeInstruct"] = f"{AUTHOR_NAME} - Customize def getCommonCLIPTextEncodeInput(): - style_presets = mz_llama_cpp.get_style_presets() + from . import mz_llama_core_nodes + style_presets = mz_llama_core_nodes.get_style_presets() CommonCLIPTextEncodeInput = { "required": { "prompt_version": (["v1"], {"default": "v1"}), @@ -129,162 +183,6 @@ def getCommonCLIPTextEncodeInput(): return CommonCLIPTextEncodeInput -class MZ_LLama3CLIPTextEncode: - @classmethod - def INPUT_TYPES(s): - m_models = mz_llama3.llama3_models.copy() - for i in range(len(m_models)): - if mz_llama3.get_exist_model(m_models[i]) is not None: - m_models[i] += "[downloaded]" - - importlib.reload(mz_llama_cpp) - - result = { - "required": { - "llama_cpp_model": (m_models, {"default": m_models[0]}), - "download_source": ( - ["none", "modelscope", "hf-mirror.com",], - {"default": "none"} - ), - }, - "optional": {}, - } - - common_input = getCommonCLIPTextEncodeInput() - for key in common_input["required"]: - result["required"][key] = common_input["required"][key] - for key in common_input["optional"]: - result["optional"][key] = common_input["optional"][key] - - return result - - RETURN_TYPES = ("STRING", "CONDITIONING",) - RETURN_NAMES = ("text", "conditioning",) - OUTPUT_NODE = True - FUNCTION = "encode" - CATEGORY = CATEGORY_NAME - - def encode(self, **kwargs): - importlib.reload(mz_llama3) - - kwargs["llama_cpp_model"] = kwargs.get( - "llama_cpp_model", "").replace("[downloaded]", "") - - text = mz_llama3.query_beautify_prompt_text(kwargs) - conditionings = None - clip = kwargs.get("clip", None) - if clip is not None: - conditionings = Utils.a1111_clip_text_encode(clip, text, ) - - return {"ui": {"string": [text,]}, "result": (text, conditionings)} - - -NODE_CLASS_MAPPINGS["MZ_LLama3CLIPTextEncode"] = MZ_LLama3CLIPTextEncode -NODE_DISPLAY_NAME_MAPPINGS[ - "MZ_LLama3CLIPTextEncode"] = f"{AUTHOR_NAME} - CLIPTextEncode(LLama3)" - - -class MZ_Phi3CLIPTextEncode: - @classmethod - def INPUT_TYPES(s): - m_models = mz_phi3.phi3_models.copy() - for i in range(len(m_models)): - if mz_llama3.get_exist_model(m_models[i]) is not None: - m_models[i] += "[downloaded]" - - importlib.reload(mz_phi3) - - result = { - "required": { - "llama_cpp_model": (m_models, {"default": m_models[0]}), - "download_source": ( - ["none", "modelscope", "hf-mirror.com",], - {"default": "none"} - ), - }, - "optional": {}, - } - - common_input = getCommonCLIPTextEncodeInput() - for key in common_input["required"]: - result["required"][key] = common_input["required"][key] - for key in common_input["optional"]: - result["optional"][key] = common_input["optional"][key] - - return result - - RETURN_TYPES = ("STRING", "CONDITIONING",) - RETURN_NAMES = ("text", "conditioning",) - OUTPUT_NODE = True - FUNCTION = "encode" - CATEGORY = CATEGORY_NAME - - def encode(self, **kwargs): - importlib.reload(mz_llama3) - - kwargs["llama_cpp_model"] = kwargs.get( - "llama_cpp_model", "").replace("[downloaded]", "") - - text = mz_phi3.query_beautify_prompt_text(kwargs) - conditionings = None - clip = kwargs.get("clip", None) - if clip is not None: - conditionings = Utils.a1111_clip_text_encode(clip, text, ) - - return {"ui": {"string": [text,]}, "result": (text, conditionings)} - - -NODE_CLASS_MAPPINGS["MZ_Phi3CLIPTextEncode"] = MZ_Phi3CLIPTextEncode -NODE_DISPLAY_NAME_MAPPINGS[ - "MZ_Phi3CLIPTextEncode"] = f"{AUTHOR_NAME} - CLIPTextEncode(Phi3)" - - -class MZ_BaseLLamaCPPCLIPTextEncode: - @classmethod - def INPUT_TYPES(s): - importlib.reload(mz_llama_cpp) - - result = { - "required": { - "llama_cpp_model": ("STRING", {"default": "", "placeholder": "model_path"}), - }, - "optional": { - }, - } - - common_input = getCommonCLIPTextEncodeInput() - for key in common_input["required"]: - result["required"][key] = common_input["required"][key] - for key in common_input["optional"]: - result["optional"][key] = common_input["optional"][key] - - return result - RETURN_TYPES = ("STRING", "CONDITIONING",) - RETURN_NAMES = ("text", "conditioning",) - OUTPUT_NODE = True - FUNCTION = "encode" - CATEGORY = CATEGORY_NAME - - def encode(self, **kwargs): - importlib.reload(mz_llama3) - - kwargs["llama_cpp_model"] = kwargs.get( - "llama_cpp_model", "").replace("[downloaded]", "") - - text = mz_llama_cpp.base_query_beautify_prompt_text(kwargs) - conditionings = None - clip = kwargs.get("clip", None) - if clip is not None: - conditionings = Utils.a1111_clip_text_encode(clip, text, ) - - return {"ui": {"string": [text,]}, "result": (text, conditionings)} - - -NODE_CLASS_MAPPINGS["MZ_BaseLLamaCPPCLIPTextEncode"] = MZ_BaseLLamaCPPCLIPTextEncode -NODE_DISPLAY_NAME_MAPPINGS[ - "MZ_BaseLLamaCPPCLIPTextEncode"] = f"{AUTHOR_NAME} - CLIPTextEncode(BaseLLamaCPP)" - - class MZ_LLavaImageInterrogator: @classmethod def INPUT_TYPES(s): @@ -518,6 +416,12 @@ NODE_CLASS_MAPPINGS["MZ_OpenAIApiCLIPTextEncode"] = MZ_OpenAIApiCLIPTextEncode NODE_DISPLAY_NAME_MAPPINGS[ "MZ_OpenAIApiCLIPTextEncode"] = f"{AUTHOR_NAME} - CLIPTextEncode(OpenAIApi)" - -import mz_gen_translate +from . import mz_gen_translate mz_gen_translate.gen_translate(NODE_DISPLAY_NAME_MAPPINGS, NODE_CLASS_MAPPINGS) + + +from .v1.init import NODE_CLASS_MAPPINGS as DEPRECATED_NODE_CLASS_MAPPINGS +from .v1.init import NODE_DISPLAY_NAME_MAPPINGS as DEPRECATED_NODE_DISPLAY_NAME_MAPPINGS + +NODE_CLASS_MAPPINGS.update(DEPRECATED_NODE_CLASS_MAPPINGS) +NODE_DISPLAY_NAME_MAPPINGS.update(DEPRECATED_NODE_DISPLAY_NAME_MAPPINGS) diff --git a/mz_gen_translate.py b/mz_gen_translate.py index 0baac8b..e395c7f 100644 --- a/mz_gen_translate.py +++ b/mz_gen_translate.py @@ -26,6 +26,7 @@ ZH_Replace_Map = { "ImageInterrogator": "图像反推", "customize_instruct": "自定义指令", "CustomizeInstruct": "自定义指令", + "deprecated": "已废弃", } diff --git a/mz_llama_core_nodes.py b/mz_llama_core_nodes.py new file mode 100644 index 0000000..c3b3308 --- /dev/null +++ b/mz_llama_core_nodes.py @@ -0,0 +1,216 @@ + +import importlib +import json +from . import mz_prompt_utils +from . import mz_llama_cpp +from . import mz_prompts + + +def get_schema_base_type(t): + return { + "type": t, + } + + +def get_schema_obj(keys_type={}, required=[]): + item = {} + for key, value in keys_type.items(): + if type(value) == str: + value = get_schema_base_type(value) + item[key] = value + return { + "type": "object", + "properties": item, + "required": required + } + + +def get_schema_array(item_type="string"): + if type(item_type) == str: + item_type = get_schema_base_type(item_type) + return { + "type": "array", + "items": item_type, + } + + +high_quality_prompt = "((high quality:1.4), (best quality:1.4), (masterpiece:1.4), (8K resolution), (2k wallpaper))" +style_presets_prompt = { + "none": "", + "high_quality": high_quality_prompt, + "photography": f"{high_quality_prompt}, (RAW photo, best quality), (realistic, photo-realistic:1.2), (bokeh, cinematic shot, dynamic composition, incredibly detailed, sharpen, details, intricate detail, professional lighting, film lighting, 35mm, anamorphic, lightroom, cinematography, bokeh, lens flare, film grain, HDR10, 8K)", + "illustration": f"{high_quality_prompt}, ((detailed matte painting, intricate detail, splash screen, complementary colors), (detailed),(intricate details),illustration,an extremely delicate and beautiful,ultra-detailed,highres,extremely detailed)", +} + + +def get_style_presets(): + return [ + "none", + "high_quality", + "photography", + "illustration", + ] + + +def llama_cpp_node_encode(args_dict): + importlib.reload(mz_prompts) + importlib.reload(mz_llama_cpp) + + model_file = args_dict.get("llama_cpp_model", "") + text = args_dict.get("text", "") + style_presets = args_dict.get("style_presets", "") + options = args_dict.get("llama_cpp_options", {}) + keep_device = args_dict.get("keep_device", False) + seed = args_dict.get("seed", -1) + options["seed"] = seed + + customize_instruct = args_dict.get("customize_instruct", None) + mz_prompt_utils.Utils.print_log( + f"customize_instruct: {customize_instruct}") + try: + schema = None + if customize_instruct is None: + schema = get_schema_obj( + keys_type={ + "description": get_schema_base_type("string"), + "long_prompt": get_schema_base_type("string"), + "main_color_word": get_schema_base_type("string"), + "camera_angle_word": get_schema_base_type("string"), + "style_words": get_schema_array("string"), + "subject_words": get_schema_array("string"), + "light_words": get_schema_array("string"), + "environment_words": get_schema_array("string"), + }, + required=[ + "description", + "long_prompt", + "main_color_word", + "camera_angle_word", + "style_words", + "subject_words", + "light_words", + "environment_words", + ] + ) + + question = f"IDEA: {style_presets},{text}" + if style_presets == "none": + question = f"IDEA: {text}" + + system_prompt = mz_prompts.Beautify_Prompt + mz_prompts.Long_prompt + "\n" + + else: + + system_prompt = customize_instruct.get("system", "") + question = customize_instruct.get("instruct", "%text%") + + system_prompt = system_prompt.replace("%text%", text) + question = question.replace("%text%", text) + + mz_prompt_utils.Utils.print_log(f"system_prompt: {system_prompt}") + mz_prompt_utils.Utils.print_log(f"question: {question}") + + if schema is not None: + response_json = mz_llama_cpp.llama_cpp_simple_interrogator_to_json( + model_file=model_file, + system=system_prompt, + question=question, + schema=schema, + options=options, + ) + mz_prompt_utils.Utils.print_log(f"response_json: {response_json}") + + response = json.loads(response_json) + full_responses = [] + + if response["description"] != "": + full_responses.append(f"({response['description']})") + if response["long_prompt"] != "": + full_responses.append(f"({response['long_prompt']})") + if response["main_color_word"] != "": + full_responses.append(f"({response['main_color_word']})") + if response["camera_angle_word"] != "": + full_responses.append(f"({response['camera_angle_word']})") + + response["style_words"] = [ + x for x in response["style_words"] if x != ""] + if len(response["style_words"]) > 0: + full_responses.append( + f"({', '.join(response['style_words'])})") + + response["subject_words"] = [ + x for x in response["subject_words"] if x != ""] + if len(response["subject_words"]) > 0: + full_responses.append( + f"({', '.join(response['subject_words'])})") + + response["light_words"] = [ + x for x in response["light_words"] if x != ""] + if len(response["light_words"]) > 0: + full_responses.append( + f"({', '.join(response['light_words'])})") + + response["environment_words"] = [ + x for x in response["environment_words"] if x != ""] + if len(response["environment_words"]) > 0: + full_responses.append( + f"({', '.join(response['environment_words'])})") + + full_response = ", ".join(full_responses) + else: + full_response = mz_llama_cpp.llama_cpp_simple_interrogator( + model_file=model_file, + system=system_prompt, + question=question, + options=options, + ) + + start_str = customize_instruct.get("start_str", "") + if start_str != "" and full_response.find(start_str) != -1: + full_response_list = full_response.split(start_str) + # 删除第一个元素 + full_response_list.pop(0) + full_response = start_str.join(full_response_list) + + end_str = customize_instruct.get("end_str", "") + if end_str != "" and full_response.find(end_str) != -1: + full_response_list = full_response.split(end_str) + # 删除最后一个元素 + full_response_list.pop() + full_response = end_str.join(full_response_list) + + if keep_device is False: + mz_llama_cpp.freed_gpu_memory(model_file=model_file) + + # 去除换行 + while full_response.find("\n") != -1: + full_response = full_response.replace("\n", " ") + + # 句号换成逗号 + while full_response.find(".") != -1: + full_response = full_response.replace(".", ",") + + # 去除多余逗号 + while full_response.find(",,") != -1: + full_response = full_response.replace(",,", ",") + while full_response.find(", ,") != -1: + full_response = full_response.replace(", ,", ",") + + full_response = mz_prompt_utils.Utils.prompt_zh_to_en(full_response) + + style_presets_prompt_text = style_presets_prompt.get(style_presets, "") + + if style_presets_prompt_text != "": + full_response = f"{style_presets_prompt_text}, {full_response}" + + except Exception as e: + mz_llama_cpp.freed_gpu_memory(model_file=model_file) + raise e + + conditionings = None + clip = args_dict.get("clip", None) + if clip is not None: + conditionings = mz_prompt_utils.Utils.a1111_clip_text_encode( + clip, full_response, ) + + return {"ui": {"string": [full_response,]}, "result": (full_response, conditionings)} diff --git a/mz_llama_cpp.py b/mz_llama_cpp.py index 3a33df4..84a44e8 100644 --- a/mz_llama_cpp.py +++ b/mz_llama_cpp.py @@ -6,40 +6,12 @@ import subprocess import sys import torch try: - import mz_prompt_utils - import mz_prompt_webserver + from . import mz_prompt_utils + from . import mz_prompt_webserver except ImportError: pass -def LlamaCppOptions(): - return { - "n_ctx": 2048, - "n_batch": 2048, - "n_threads": 0, - "n_threads_batch": 0, - "split_mode": ["LLAMA_SPLIT_MODE_NONE", "LLAMA_SPLIT_MODE_LAYER", "LLAMA_SPLIT_MODE_ROW",], - "main_gpu": 0, - "n_gpu_layers": -1, - - - "max_tokens": 4096, - "temperature": 1.6, - "top_p": 0.95, - "min_p": 0.05, - "typical_p": 1.0, - "stop": "", - "frequency_penalty": 0.0, - "presence_penalty": 0.0, - "repeat_penalty": 1.1, - "top_k": 50, - "tfs_z": 1.0, - "mirostat_mode": ["none", "mirostat", "mirostat_v2"], - "mirostat_tau": 5.0, - "mirostat_eta": 0.1, - } - - def check_llama_cpp_requirements(): last_version = "0.2.63" try: @@ -103,6 +75,32 @@ def check_llama_cpp_requirements(): print("llama_cpp installed successfully. (llama_cpp安装成功)") +def LlamaCppOptions(): + return { + "n_ctx": 2048, + "n_batch": 2048, + "n_threads": 0, + "n_threads_batch": 0, + "split_mode": ["LLAMA_SPLIT_MODE_NONE", "LLAMA_SPLIT_MODE_LAYER", "LLAMA_SPLIT_MODE_ROW",], + "main_gpu": 0, + "n_gpu_layers": -1, + "max_tokens": 4096, + "temperature": 1.6, + "top_p": 0.95, + "min_p": 0.05, + "typical_p": 1.0, + "stop": "", + "frequency_penalty": 0.0, + "presence_penalty": 0.0, + "repeat_penalty": 1.1, + "top_k": 50, + "tfs_z": 1.0, + "mirostat_mode": ["none", "mirostat", "mirostat_v2"], + "mirostat_tau": 5.0, + "mirostat_eta": 0.1, + } + + def freed_gpu_memory(model_file): check_llama_cpp_requirements() @@ -237,33 +235,6 @@ def llama_cpp_messages(model_file, chat_handler=None, messages=[], options={}): return result -def get_schema_base_type(t): - return { - "type": t, - } - - -def get_schema_obj(keys_type={}, required=[]): - item = {} - for key, value in keys_type.items(): - if type(value) == str: - value = get_schema_base_type(value) - item[key] = value - return { - "type": "object", - "properties": item, - "required": required - } - - -def get_schema_array(item_type="string"): - if type(item_type) == str: - item_type = get_schema_base_type(item_type) - return { - "type": "array", - "items": item_type, - } - def llama_cpp_simple_interrogator_to_json(model_file, use_system=True, system=None, question="", schema={}, options={}): if system is None: @@ -409,220 +380,3 @@ def llava_cpp_simple_interrogator_to_json( model_file, mmproj_file, system=system, question=question, image=image, options=options) return result - -high_quality_prompt = "((high quality:1.4), (best quality:1.4), (masterpiece:1.4), (8K resolution), (2k wallpaper))" -style_presets_prompt = { - "none": "", - "high_quality": high_quality_prompt, - "photography": f"{high_quality_prompt}, (RAW photo, best quality), (realistic, photo-realistic:1.2), (bokeh, cinematic shot, dynamic composition, incredibly detailed, sharpen, details, intricate detail, professional lighting, film lighting, 35mm, anamorphic, lightroom, cinematography, bokeh, lens flare, film grain, HDR10, 8K)", - "illustration": f"{high_quality_prompt}, ((detailed matte painting, intricate detail, splash screen, complementary colors), (detailed),(intricate details),illustration,an extremely delicate and beautiful,ultra-detailed,highres,extremely detailed)", -} - - -def get_style_presets(): - return [ - "none", - "high_quality", - "photography", - "illustration", - ] - - -def base_query_beautify_prompt_text(args_dict): - model_file = args_dict.get("llama_cpp_model", "") - - text = args_dict.get("text", "") - style_presets = args_dict.get("style_presets", "") - options = args_dict.get("llama_cpp_options", {}) - keep_device = args_dict.get("keep_device", False) - seed = args_dict.get("seed", -1) - options["seed"] = seed - - import mz_prompts - importlib.reload(mz_prompts) - - customize_instruct = args_dict.get("customize_instruct", None) - mz_prompt_utils.Utils.print_log( - f"customize_instruct: {customize_instruct}") - try: - schema = None - if customize_instruct is None: - schema = get_schema_obj( - keys_type={ - "description": get_schema_base_type("string"), - "long_prompt": get_schema_base_type("string"), - "main_color_word": get_schema_base_type("string"), - "camera_angle_word": get_schema_base_type("string"), - "style_words": get_schema_array("string"), - "subject_words": get_schema_array("string"), - "light_words": get_schema_array("string"), - "environment_words": get_schema_array("string"), - }, - required=[ - "description", - "long_prompt", - "main_color_word", - "camera_angle_word", - "style_words", - "subject_words", - "light_words", - "environment_words", - ] - ) - - question = f"IDEA: {style_presets},{text}" - if style_presets == "none": - question = f"IDEA: {text}" - - system_prompt = mz_prompts.Beautify_Prompt + mz_prompts.Long_prompt + "\n" - - else: - - system_prompt = customize_instruct.get("system", "") - question = customize_instruct.get("instruct", "%text%") - - system_prompt = system_prompt.replace("%text%", text) - question = question.replace("%text%", text) - - mz_prompt_utils.Utils.print_log(f"system_prompt: {system_prompt}") - mz_prompt_utils.Utils.print_log(f"question: {question}") - - if schema is not None: - response_json = llama_cpp_simple_interrogator_to_json( - model_file=model_file, - system=system_prompt, - question=question, - schema=schema, - options=options, - ) - mz_prompt_utils.Utils.print_log(f"response_json: {response_json}") - - response = json.loads(response_json) - full_responses = [] - - if response["description"] != "": - full_responses.append(f"({response['description']})") - if response["long_prompt"] != "": - full_responses.append(f"({response['long_prompt']})") - if response["main_color_word"] != "": - full_responses.append(f"({response['main_color_word']})") - if response["camera_angle_word"] != "": - full_responses.append(f"({response['camera_angle_word']})") - - response["style_words"] = [ - x for x in response["style_words"] if x != ""] - if len(response["style_words"]) > 0: - full_responses.append( - f"({', '.join(response['style_words'])})") - - response["subject_words"] = [ - x for x in response["subject_words"] if x != ""] - if len(response["subject_words"]) > 0: - full_responses.append( - f"({', '.join(response['subject_words'])})") - - response["light_words"] = [ - x for x in response["light_words"] if x != ""] - if len(response["light_words"]) > 0: - full_responses.append( - f"({', '.join(response['light_words'])})") - - response["environment_words"] = [ - x for x in response["environment_words"] if x != ""] - if len(response["environment_words"]) > 0: - full_responses.append( - f"({', '.join(response['environment_words'])})") - - full_response = ", ".join(full_responses) - else: - full_response = llama_cpp_simple_interrogator( - model_file=model_file, - system=system_prompt, - question=question, - options=options, - ) - - start_str = customize_instruct.get("start_str", "") - if start_str != "" and full_response.find(start_str) != -1: - full_response_list = full_response.split(start_str) - # 删除第一个元素 - full_response_list.pop(0) - full_response = start_str.join(full_response_list) - - end_str = customize_instruct.get("end_str", "") - if end_str != "" and full_response.find(end_str) != -1: - full_response_list = full_response.split(end_str) - # 删除最后一个元素 - full_response_list.pop() - full_response = end_str.join(full_response_list) - - if keep_device is False: - freed_gpu_memory(model_file=model_file) - - # 去除换行 - while full_response.find("\n") != -1: - full_response = full_response.replace("\n", " ") - - # 句号换成逗号 - while full_response.find(".") != -1: - full_response = full_response.replace(".", ",") - - # 去除多余逗号 - while full_response.find(",,") != -1: - full_response = full_response.replace(",,", ",") - while full_response.find(", ,") != -1: - full_response = full_response.replace(", ,", ",") - - full_response = mz_prompt_utils.Utils.prompt_zh_to_en(full_response) - - style_presets_prompt_text = style_presets_prompt.get(style_presets, "") - - if style_presets_prompt_text != "": - full_response = f"{style_presets_prompt_text}, {full_response}" - - return full_response - - except Exception as e: - freed_gpu_memory(model_file=model_file) - # mz_utils.Utils.print_log(f"Error in auto_prompt_text: {e}") - raise e - - -def get_image_composition_json_schema(): - system = "Please make a specific composition of the proposed content. You need to have the width and height of the overall picture and a list of specific element positions." - return system, { - "type": "object", - "properties": { - "width": { - "type": "number" - }, - "height": { - "type": "number" - }, - "bbox": { - "type": "array", - "items": { - "type": "object", - "properties": { - "english_name": { - "type": "string" - }, - "left": { - "type": "number" - }, - "top": { - "type": "number" - }, - "width": { - "type": "number" - }, - "height": { - "type": "number" - } - }, - "required": ["english_name", "left", "top", "width", "height"] - } - } - }, - "required": ["width", "height", "bbox"] - } diff --git a/mz_llava.py b/mz_llava.py index 7aefab3..7e43491 100644 --- a/mz_llava.py +++ b/mz_llava.py @@ -1,12 +1,12 @@ import json import os -import mz_prompt_utils -import mz_llama_cpp +from . import mz_prompt_utils +from . import mz_llama_cpp +from . import mz_prompts +from . import mz_prompt_webserver import importlib -import mz_prompts -import mz_prompt_webserver LLava_models = [ "llava-1.6-mistral-7b-gguf/llava-v1.6-mistral-7b.Q5_K_M.gguf", diff --git a/mz_openaiapi.py b/mz_openaiapi.py index bdbe9c2..3176d94 100644 --- a/mz_openaiapi.py +++ b/mz_openaiapi.py @@ -3,9 +3,9 @@ import sys import json import subprocess -import mz_prompt_utils -import mz_llama_cpp -import mz_prompts +from . import mz_prompt_utils +from . import mz_llama_cpp +from . import mz_prompts def zhipu_json_fix(input_data): diff --git a/mz_prompt_utils.py b/mz_prompt_utils.py index b061291..d0b3527 100644 --- a/mz_prompt_utils.py +++ b/mz_prompt_utils.py @@ -165,6 +165,12 @@ class Utils: os.makedirs(models_path, exist_ok=True) return models_path + def get_gguf_models_path(): + models_path = os.path.join( + folder_paths.models_dir, "gguf") + os.makedirs(models_path, exist_ok=True) + return models_path + def translate_text(text, from_code, to_code): try: import argostranslate diff --git a/mz_prompt_webserver.py b/mz_prompt_webserver.py index b4e5758..ae0673b 100644 --- a/mz_prompt_webserver.py +++ b/mz_prompt_webserver.py @@ -2,7 +2,7 @@ import asyncio import uuid -import mz_prompt_utils +from . import mz_prompt_utils web_msg_pool = { diff --git a/mz_prompts.py b/mz_prompts.py index c782800..6080cc7 100644 --- a/mz_prompts.py +++ b/mz_prompts.py @@ -18,3 +18,18 @@ Standardize_Prompt = """ Extract the content about Stable Diffusion style from the following input and combine it into a json array. Note that the output will be directly used in the program. Please output the standardized json content. """ + + +# 来自https://github.com/jiayev/GPT4V-Image-Captioner +GPT4VImageCaptioner_Prompt = """ +As an AI image tagging expert, please provide precise tags for these images to enhance CLIP model's understanding of the content. +Employ succinct keywords or phrases, steering clear of elaborate sentences and extraneous conjunctions. +Prioritize the tags by relevance. +Your tags should capture key elements such as the main subject, setting, artistic style, composition, image quality, color tone, filter, and camera specifications, and any other tags crucial for the image. +When tagging photos of people, include specific details like gender, nationality, attire, actions, pose, expressions, accessories, makeup, composition type, age, etc. +For other image categories, apply appropriate and common descriptive tags as well. +Recognize and tag any celebrities, well-known landmark or IPs if clearly featured in the image. +Your tags should be accurate, non-duplicative, and within a 20-75 word count range. +These tags will use for image re-creation, so the closer the resemblance to the original image, the better the tag quality. +Tags should be comma-separated. +""" \ No newline at end of file diff --git a/v1/init.py b/v1/init.py new file mode 100644 index 0000000..3005954 --- /dev/null +++ b/v1/init.py @@ -0,0 +1,195 @@ +from ..mz_prompt_utils import Utils + +NODE_CLASS_MAPPINGS = { +} + + +NODE_DISPLAY_NAME_MAPPINGS = { +} + + +import importlib +from . import mz_llama3 +from . import mz_phi3 +from .. import mz_llama_cpp +from .. import mz_llama_core_nodes + + +AUTHOR_NAME = u"MinusZone" +CATEGORY_NAME = f"{AUTHOR_NAME} - Prompt/deprecated" + + +def getCommonCLIPTextEncodeInput(): + style_presets = mz_llama_core_nodes.get_style_presets() + CommonCLIPTextEncodeInput = { + "required": { + "prompt_version": (["v1"], {"default": "v1"}), + "style_presets": ( + style_presets, {"default": style_presets[1]} + ), + "text": ("STRING", {"multiline": True, }), + "keep_device": ([False, True], {"default": False}), + "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), + }, + "optional": { + "clip": ("CLIP", ), + "llama_cpp_options": ("LLamaCPPOptions", ), + "customize_instruct": ("CustomizeInstruct", ), + # "customize_json_schema": ("STRING", ), + } + } + + return CommonCLIPTextEncodeInput + + +class MZ_LLama3CLIPTextEncode: + @classmethod + def INPUT_TYPES(s): + m_models = mz_llama3.llama3_models.copy() + for i in range(len(m_models)): + if mz_llama3.get_exist_model(m_models[i]) is not None: + m_models[i] += "[downloaded]" + + result = { + "required": { + "llama_cpp_model": (m_models, {"default": m_models[0]}), + "download_source": ( + ["none", "modelscope", "hf-mirror.com",], + {"default": "none"} + ), + }, + "optional": {}, + } + + common_input = getCommonCLIPTextEncodeInput() + for key in common_input["required"]: + result["required"][key] = common_input["required"][key] + for key in common_input["optional"]: + result["optional"][key] = common_input["optional"][key] + + return result + + RETURN_TYPES = ("STRING", "CONDITIONING",) + RETURN_NAMES = ("text", "conditioning",) + OUTPUT_NODE = True + FUNCTION = "encode" + CATEGORY = CATEGORY_NAME + + def encode(self, **kwargs): + + kwargs["llama_cpp_model"] = kwargs.get( + "llama_cpp_model", "").replace("[downloaded]", "") + + text = mz_llama3.query_beautify_prompt_text(kwargs) + conditionings = None + clip = kwargs.get("clip", None) + if clip is not None: + conditionings = Utils.a1111_clip_text_encode(clip, text, ) + + return {"ui": {"string": [text,]}, "result": (text, conditionings)} + + +NODE_CLASS_MAPPINGS["MZ_LLama3CLIPTextEncode"] = MZ_LLama3CLIPTextEncode +NODE_DISPLAY_NAME_MAPPINGS[ + "MZ_LLama3CLIPTextEncode"] = f"{AUTHOR_NAME} - deprecated - CLIPTextEncode(LLama3)" + + +class MZ_Phi3CLIPTextEncode: + @classmethod + def INPUT_TYPES(s): + m_models = mz_phi3.phi3_models.copy() + for i in range(len(m_models)): + if mz_llama3.get_exist_model(m_models[i]) is not None: + m_models[i] += "[downloaded]" + + importlib.reload(mz_phi3) + + result = { + "required": { + "llama_cpp_model": (m_models, {"default": m_models[0]}), + "download_source": ( + ["none", "modelscope", "hf-mirror.com",], + {"default": "none"} + ), + }, + "optional": {}, + } + + common_input = getCommonCLIPTextEncodeInput() + for key in common_input["required"]: + result["required"][key] = common_input["required"][key] + for key in common_input["optional"]: + result["optional"][key] = common_input["optional"][key] + + return result + + RETURN_TYPES = ("STRING", "CONDITIONING",) + RETURN_NAMES = ("text", "conditioning",) + OUTPUT_NODE = True + FUNCTION = "encode" + CATEGORY = CATEGORY_NAME + + def encode(self, **kwargs): + importlib.reload(mz_llama3) + + kwargs["llama_cpp_model"] = kwargs.get( + "llama_cpp_model", "").replace("[downloaded]", "") + + text = mz_phi3.query_beautify_prompt_text(kwargs) + conditionings = None + clip = kwargs.get("clip", None) + if clip is not None: + conditionings = Utils.a1111_clip_text_encode(clip, text, ) + + return {"ui": {"string": [text,]}, "result": (text, conditionings)} + + +NODE_CLASS_MAPPINGS["MZ_Phi3CLIPTextEncode"] = MZ_Phi3CLIPTextEncode +NODE_DISPLAY_NAME_MAPPINGS[ + "MZ_Phi3CLIPTextEncode"] = f"{AUTHOR_NAME} - deprecated - CLIPTextEncode(Phi3)" + + +class MZ_BaseLLamaCPPCLIPTextEncode: + @classmethod + def INPUT_TYPES(s): + importlib.reload(mz_llama_cpp) + + result = { + "required": { + "llama_cpp_model": ("STRING", {"default": "", "placeholder": "model_path"}), + }, + "optional": { + }, + } + + common_input = getCommonCLIPTextEncodeInput() + for key in common_input["required"]: + result["required"][key] = common_input["required"][key] + for key in common_input["optional"]: + result["optional"][key] = common_input["optional"][key] + + return result + RETURN_TYPES = ("STRING", "CONDITIONING",) + RETURN_NAMES = ("text", "conditioning",) + OUTPUT_NODE = True + FUNCTION = "encode" + CATEGORY = CATEGORY_NAME + + def encode(self, **kwargs): + importlib.reload(mz_llama3) + + kwargs["llama_cpp_model"] = kwargs.get( + "llama_cpp_model", "").replace("[downloaded]", "") + + text = mz_llama_cpp.base_query_beautify_prompt_text(kwargs) + conditionings = None + clip = kwargs.get("clip", None) + if clip is not None: + conditionings = Utils.a1111_clip_text_encode(clip, text, ) + + return {"ui": {"string": [text,]}, "result": (text, conditionings)} + + +NODE_CLASS_MAPPINGS["MZ_BaseLLamaCPPCLIPTextEncode"] = MZ_BaseLLamaCPPCLIPTextEncode +NODE_DISPLAY_NAME_MAPPINGS[ + "MZ_BaseLLamaCPPCLIPTextEncode"] = f"{AUTHOR_NAME} - deprecated - CLIPTextEncode(BaseLLamaCPP)" diff --git a/v1/mz_deprecated.py b/v1/mz_deprecated.py new file mode 100644 index 0000000..0d1dbd5 --- /dev/null +++ b/v1/mz_deprecated.py @@ -0,0 +1,160 @@ +from ..mz_prompt_utils import Utils +from ..mz_llama_cpp import * +from ..mz_llama_core_nodes import * +from ..mz_prompts import * + +def base_query_beautify_prompt_text(args_dict): + model_file = args_dict.get("llama_cpp_model", "") + text = args_dict.get("text", "") + style_presets = args_dict.get("style_presets", "") + options = args_dict.get("llama_cpp_options", {}) + keep_device = args_dict.get("keep_device", False) + seed = args_dict.get("seed", -1) + options["seed"] = seed + + + customize_instruct = args_dict.get("customize_instruct", None) + Utils.print_log( + f"customize_instruct: {customize_instruct}") + try: + schema = None + if customize_instruct is None: + schema = get_schema_obj( + keys_type={ + "description": get_schema_base_type("string"), + "long_prompt": get_schema_base_type("string"), + "main_color_word": get_schema_base_type("string"), + "camera_angle_word": get_schema_base_type("string"), + "style_words": get_schema_array("string"), + "subject_words": get_schema_array("string"), + "light_words": get_schema_array("string"), + "environment_words": get_schema_array("string"), + }, + required=[ + "description", + "long_prompt", + "main_color_word", + "camera_angle_word", + "style_words", + "subject_words", + "light_words", + "environment_words", + ] + ) + + question = f"IDEA: {style_presets},{text}" + if style_presets == "none": + question = f"IDEA: {text}" + + system_prompt = mz_prompts.Beautify_Prompt + mz_prompts.Long_prompt + "\n" + + else: + + system_prompt = customize_instruct.get("system", "") + question = customize_instruct.get("instruct", "%text%") + + system_prompt = system_prompt.replace("%text%", text) + question = question.replace("%text%", text) + + Utils.print_log(f"system_prompt: {system_prompt}") + Utils.print_log(f"question: {question}") + + if schema is not None: + response_json = llama_cpp_simple_interrogator_to_json( + model_file=model_file, + system=system_prompt, + question=question, + schema=schema, + options=options, + ) + Utils.print_log(f"response_json: {response_json}") + + response = json.loads(response_json) + full_responses = [] + + if response["description"] != "": + full_responses.append(f"({response['description']})") + if response["long_prompt"] != "": + full_responses.append(f"({response['long_prompt']})") + if response["main_color_word"] != "": + full_responses.append(f"({response['main_color_word']})") + if response["camera_angle_word"] != "": + full_responses.append(f"({response['camera_angle_word']})") + + response["style_words"] = [ + x for x in response["style_words"] if x != ""] + if len(response["style_words"]) > 0: + full_responses.append( + f"({', '.join(response['style_words'])})") + + response["subject_words"] = [ + x for x in response["subject_words"] if x != ""] + if len(response["subject_words"]) > 0: + full_responses.append( + f"({', '.join(response['subject_words'])})") + + response["light_words"] = [ + x for x in response["light_words"] if x != ""] + if len(response["light_words"]) > 0: + full_responses.append( + f"({', '.join(response['light_words'])})") + + response["environment_words"] = [ + x for x in response["environment_words"] if x != ""] + if len(response["environment_words"]) > 0: + full_responses.append( + f"({', '.join(response['environment_words'])})") + + full_response = ", ".join(full_responses) + else: + full_response = llama_cpp_simple_interrogator( + model_file=model_file, + system=system_prompt, + question=question, + options=options, + ) + + start_str = customize_instruct.get("start_str", "") + if start_str != "" and full_response.find(start_str) != -1: + full_response_list = full_response.split(start_str) + # 删除第一个元素 + full_response_list.pop(0) + full_response = start_str.join(full_response_list) + + end_str = customize_instruct.get("end_str", "") + if end_str != "" and full_response.find(end_str) != -1: + full_response_list = full_response.split(end_str) + # 删除最后一个元素 + full_response_list.pop() + full_response = end_str.join(full_response_list) + + if keep_device is False: + freed_gpu_memory(model_file=model_file) + + # 去除换行 + while full_response.find("\n") != -1: + full_response = full_response.replace("\n", " ") + + # 句号换成逗号 + while full_response.find(".") != -1: + full_response = full_response.replace(".", ",") + + # 去除多余逗号 + while full_response.find(",,") != -1: + full_response = full_response.replace(",,", ",") + while full_response.find(", ,") != -1: + full_response = full_response.replace(", ,", ",") + + full_response = Utils.prompt_zh_to_en(full_response) + + style_presets_prompt_text = style_presets_prompt.get(style_presets, "") + + if style_presets_prompt_text != "": + full_response = f"{style_presets_prompt_text}, {full_response}" + + return full_response + + except Exception as e: + freed_gpu_memory(model_file=model_file) + # mz_utils.Utils.print_log(f"Error in auto_prompt_text: {e}") + raise e diff --git a/mz_llama3.py b/v1/mz_llama3.py similarity index 92% rename from mz_llama3.py rename to v1/mz_llama3.py index 6127dd2..06c6005 100644 --- a/mz_llama3.py +++ b/v1/mz_llama3.py @@ -1,7 +1,7 @@ import json -import mz_prompt_utils -import mz_llama_cpp - +from .. import mz_prompt_utils +from .. import mz_llama_cpp +from . import mz_deprecated import importlib @@ -46,10 +46,6 @@ def query_beautify_prompt_text(args_dict): model_name = args_dict.get("llama_cpp_model", "") download_source = args_dict.get("download_source", None) - import mz_prompts - importlib.reload(mz_prompts) - importlib.reload(mz_llama_cpp) - try: model_file = get_exist_model(model_name) @@ -66,7 +62,7 @@ def query_beautify_prompt_text(args_dict): model_file = mz_prompt_utils.Utils.hf_download_model(model_url) args_dict["llama_cpp_model"] = model_file - full_response = mz_llama_cpp.base_query_beautify_prompt_text( + full_response = mz_deprecated.base_query_beautify_prompt_text( args_dict=args_dict) return full_response diff --git a/mz_phi3.py b/v1/mz_phi3.py similarity index 92% rename from mz_phi3.py rename to v1/mz_phi3.py index bb18ce4..1d904a2 100644 --- a/mz_phi3.py +++ b/v1/mz_phi3.py @@ -1,9 +1,6 @@ import json -import mz_prompt_utils -import mz_llama_cpp - -import importlib - +from .. import mz_prompt_utils +from .. import mz_llama_cpp phi3_models = [ "Phi-3-mini-4k-instruct-q4.gguf" @@ -33,10 +30,6 @@ def query_beautify_prompt_text(args_dict): model_name = args_dict.get("llama_cpp_model", "") download_source = args_dict.get("download_source", None) - import mz_prompts - importlib.reload(mz_prompts) - importlib.reload(mz_llama_cpp) - try: model_file = get_exist_model(model_name) diff --git a/web/prompt_mz.js b/web/prompt_mz.js index cf5fd87..7055a59 100644 --- a/web/prompt_mz.js +++ b/web/prompt_mz.js @@ -41,14 +41,10 @@ async function waitMessage() { }; websocket.onclose = async (event) => { console.log("Connection closed", event); - await sleep(1000); - await waitMessage(); }; websocket.onerror = async (event) => { console.log("Connection error", event); - await sleep(1000); - await waitMessage(); }; for (;;) {