diff --git a/__init__.py b/__init__.py index c8ff7ff..f7b2cbb 100644 --- a/__init__.py +++ b/__init__.py @@ -32,22 +32,34 @@ NODE_DISPLAY_NAME_MAPPINGS = { from . import mz_llama_cpp -from . import mz_llava + + +def getCommonCLIPTextEncodeInput(): + from . import mz_llama_core_nodes + style_presets = mz_llama_core_nodes.get_style_presets() + CommonCLIPTextEncodeInput = { + "required": { + "style_presets": ( + style_presets, {"default": style_presets[1]} + ), + "text": ("STRING", {"multiline": True, }), + "keep_device": ([False, True], {"default": False}), + "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), + }, + "optional": { + "clip": ("CLIP", ), + "llama_cpp_options": ("LLamaCPPOptions", ), + "customize_instruct": ("CustomizeInstruct", ), + } + } + + return CommonCLIPTextEncodeInput class MZ_LLamaCPPModelConfig_ManualSelect: @classmethod def INPUT_TYPES(s): - gguf_dir = Utils.get_gguf_models_path() - if not os.path.exists(gguf_dir): - os.makedirs(gguf_dir) - gguf_files = [] - # walk gguf_dir - for root, dirs, files in os.walk(gguf_dir): - for file in files: - if file.endswith(".gguf"): - gguf_files.append( - os.path.relpath(os.path.join(root, file), gguf_dir)) + gguf_files = Utils.get_gguf_files() return { "required": { "llama_cpp_model": (gguf_files,), @@ -60,7 +72,7 @@ class MZ_LLamaCPPModelConfig_ManualSelect: RETURN_NAMES = ("llama_cpp_model_config",) FUNCTION = "create" - CATEGORY = CATEGORY_NAME + CATEGORY = f"{CATEGORY_NAME}/others" def create(self, **kwargs): kwargs = kwargs.copy() @@ -78,7 +90,50 @@ class MZ_LLamaCPPModelConfig_ManualSelect: NODE_CLASS_MAPPINGS["MZ_LLamaCPPModelConfig_ManualSelect"] = MZ_LLamaCPPModelConfig_ManualSelect NODE_DISPLAY_NAME_MAPPINGS[ - "MZ_LLamaCPPModelConfig_ManualSelect"] = f"{AUTHOR_NAME} - ModelConfigManualSelect" + "MZ_LLamaCPPModelConfig_ManualSelect"] = f"{AUTHOR_NAME} - ModelConfigManualSelect(LLamaCPP)" + + +class MZ_LLamaCPPModelConfig_DownloaderSelect: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model_name": ([ + "Meta-Llama-3-8B-Instruct-Q4_K_M", + "Phi-3-mini-4k-instruct-q4", + "Meta-Llama-3-8B-Q4_K_M", + "llama3_8b_instruct_dpo_zh-Q4_K_M", + "llama3-zh.Q4_K_M", + "qwen1_5-14b-chat-q4_k_m", + "qwen1_5-7b-chat-q4_k_m", + "qwen1_5-4b-chat-q4_k_m", + "qwen1_5-1_8b-chat-q4_k_m", + "qwen1_5-0_5b-chat-q4_k_m", + ],), + }, + "optional": { + }, + } + + RETURN_TYPES = ("LLamaCPPModelConfig",) + RETURN_NAMES = ("llama_cpp_model_config",) + + FUNCTION = "create" + CATEGORY = f"{CATEGORY_NAME}/others" + + def create(self, **kwargs): + kwargs = kwargs.copy() + + model_name = kwargs.get("model_name", "") + return ({ + "type": "DownloaderSelect", + "model_name": model_name, + },) + + +NODE_CLASS_MAPPINGS["MZ_LLamaCPPModelConfig_DownloaderSelect"] = MZ_LLamaCPPModelConfig_DownloaderSelect +NODE_DISPLAY_NAME_MAPPINGS[ + "MZ_LLamaCPPModelConfig_DownloaderSelect"] = f"{AUTHOR_NAME} - ModelConfigDownloaderSelect(LLamaCPP)" class MZ_LLamaCPPCLIPTextEncode: @@ -109,7 +164,7 @@ class MZ_LLamaCPPCLIPTextEncode: CATEGORY = CATEGORY_NAME DESCRIPTION = """ -llama_cpp_model不设置时，将使用默认模型: Meta-Llama-3-8B-Instruct.Q4_K_M.gguf +llama_cpp_model不设置时，将使用默认模型: Meta-Llama-3-8B-Instruct-Q4_K_M.gguf """ def encode(self, **kwargs): @@ -155,7 +210,7 @@ class MZ_LLamaCPPOptions: RETURN_NAMES = ("llama_cpp_options",) FUNCTION = "create" - CATEGORY = CATEGORY_NAME + CATEGORY = f"{CATEGORY_NAME}/others" def create(self, **kwargs): kwargs = kwargs.copy() @@ -178,18 +233,18 @@ class MZ_CustomizeInstruct: return { "required": { "system": ("STRING", {"multiline": True, "default": mz_prompts.Long_prompt}), - "instruct": ("STRING", {"multiline": True, "default": "Short: %text%"}), + "instruct": ("STRING", {"multiline": True, "default": ""}), }, } RETURN_TYPES = ("CustomizeInstruct",) RETURN_NAMES = ("customize_instruct",) FUNCTION = "create" - CATEGORY = CATEGORY_NAME + CATEGORY = f"{CATEGORY_NAME}/others" def create(self, **kwargs): kwargs = kwargs.copy() - + return (kwargs,) @@ -197,154 +252,6 @@ NODE_CLASS_MAPPINGS["MZ_CustomizeInstruct"] = MZ_CustomizeInstruct NODE_DISPLAY_NAME_MAPPINGS["MZ_CustomizeInstruct"] = f"{AUTHOR_NAME} - CustomizeInstruct" -def getCommonCLIPTextEncodeInput(): - from . import mz_llama_core_nodes - style_presets = mz_llama_core_nodes.get_style_presets() - CommonCLIPTextEncodeInput = { - "required": { - "prompt_version": (["v1"], {"default": "v1"}), - "style_presets": ( - style_presets, {"default": style_presets[1]} - ), - "text": ("STRING", {"multiline": True, }), - "keep_device": ([False, True], {"default": False}), - "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), - }, - "optional": { - "clip": ("CLIP", ), - "llama_cpp_options": ("LLamaCPPOptions", ), - "customize_instruct": ("CustomizeInstruct", ), - # "customize_json_schema": ("STRING", ), - } - } - - return CommonCLIPTextEncodeInput - - -class MZ_LLavaImageInterrogator: - @classmethod - def INPUT_TYPES(s): - importlib.reload(mz_llava) - m_llava_models = mz_llava.LLava_models.copy() - for i in range(len(m_llava_models)): - if mz_llava.get_exist_model(m_llava_models[i]) is not None: - m_llava_models[i] += "[downloaded]" - - m_llava_mmproj_models = mz_llava.LLava_mmproj_models.copy() - for i in range(len(m_llava_mmproj_models)): - if mz_llava.get_exist_model(m_llava_mmproj_models[i]) is not None: - m_llava_mmproj_models[i] += "[downloaded]" - - return { - "required": { - "llama_cpp_model": (m_llava_models, {"default": m_llava_models[0]}), - "mmproj_model": (m_llava_mmproj_models, {"default": m_llava_mmproj_models[0]}), - "download_source": ( - [ - "none", - "modelscope", - "hf-mirror.com", - ], - {"default": "none"} - ), - "resolution": ("INT", {"default": 512, "min": 128, "max": 2048}), - "sd_format": (["none", "v1", ], {"default": "none"}), - "keep_device": ([False, True], {"default": False}), - "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), - }, - "optional": { - "image": ("IMAGE",), - "clip": ("CLIP", ), - "llama_cpp_options": ("LLamaCPPOptions", ), - "customize_instruct": ("CustomizeInstruct", ), - "captioner_config": ("ImageCaptionerConfig", ), - }, - } - RETURN_TYPES = ("STRING", "CONDITIONING",) - RETURN_NAMES = ("text", "conditioning",) - OUTPUT_NODE = True - FUNCTION = "interrogate" - CATEGORY = CATEGORY_NAME - - def interrogate(self, **kwargs): - kwargs = kwargs.copy() - - importlib.reload(mz_llava) - - kwargs["llama_cpp_model"] = kwargs.get( - "llama_cpp_model", "").replace("[downloaded]", "") - kwargs["mmproj_model"] = kwargs.get( - "mmproj_model", "").replace("[downloaded]", "") - - if kwargs.get("image", None) is not None: - kwargs["image"] = Utils.tensor2pil(kwargs["image"]) - else: - kwargs["image"] = None - - text = mz_llava.image_interrogator(kwargs) - conditionings = None - clip = kwargs.get("clip", None) - if clip is not None: - conditionings = Utils.a1111_clip_text_encode(clip, text, ) - - return {"ui": {"string": [Utils.to_debug_prompt(text),]}, "result": (text, conditionings)} - - -NODE_CLASS_MAPPINGS["MZ_LLavaImageInterrogator"] = MZ_LLavaImageInterrogator -NODE_DISPLAY_NAME_MAPPINGS[ - "MZ_LLavaImageInterrogator"] = f"{AUTHOR_NAME} - ImageInterrogator(LLava)" - - -class MZ_BaseLLavaImageInterrogator: - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "llama_cpp_model": ("STRING", {"default": ""}), - "mmproj_model": ("STRING", {"default": ""}), - "resolution": ("INT", {"default": 512, "min": 128, "max": 2048}), - "sd_format": (["none", "v1"], {"default": "none"}), - "keep_device": ([False, True], {"default": False}), - "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), - }, - "optional": { - "image": ("IMAGE",), - "clip": ("CLIP", ), - "llama_cpp_options": ("LLamaCPPOptions", ), - "customize_instruct": ("CustomizeInstruct", ), - "captioner_config": ("ImageCaptionerConfig", ), - }, - } - RETURN_TYPES = ("STRING", "CONDITIONING",) - RETURN_NAMES = ("text", "conditioning",) - OUTPUT_NODE = True - FUNCTION = "interrogate" - CATEGORY = CATEGORY_NAME - - def interrogate(self, **kwargs): - kwargs = kwargs.copy() - - importlib.reload(mz_llava) - - if kwargs.get("image", None) is not None: - kwargs["image"] = Utils.tensor2pil(kwargs["image"]) - else: - kwargs["image"] = None - - text = mz_llava.base_image_interrogator(kwargs) - conditionings = None - clip = kwargs.get("clip", None) - if clip is not None: - conditionings = Utils.a1111_clip_text_encode(clip, text, ) - - return {"ui": {"string": [Utils.to_debug_prompt(text),]}, "result": (text, conditionings)} - - -NODE_CLASS_MAPPINGS["MZ_BaseLLavaImageInterrogator"] = MZ_BaseLLavaImageInterrogator -NODE_DISPLAY_NAME_MAPPINGS[ - "MZ_BaseLLavaImageInterrogator"] = f"{AUTHOR_NAME} - ImageInterrogator(BaseLLava)" - - class MZ_ImageCaptionerConfig: @classmethod def INPUT_TYPES(s): @@ -364,11 +271,11 @@ class MZ_ImageCaptionerConfig: RETURN_NAMES = ("captioner_config", ) FUNCTION = "interrogate_batch" - CATEGORY = CATEGORY_NAME + CATEGORY = f"{CATEGORY_NAME}/others" def interrogate_batch(self, **kwargs): kwargs = kwargs.copy() - + return (kwargs, ) @@ -427,8 +334,8 @@ class MZ_OpenAIApiCLIPTextEncode: def encode(self, **kwargs): kwargs = kwargs.copy() - - import mz_openaiapi + + from . import mz_openaiapi importlib.reload(mz_openaiapi) if kwargs.get("api_key", "").endswith("******"): @@ -463,6 +370,132 @@ NODE_CLASS_MAPPINGS["MZ_OpenAIApiCLIPTextEncode"] = MZ_OpenAIApiCLIPTextEncode NODE_DISPLAY_NAME_MAPPINGS[ "MZ_OpenAIApiCLIPTextEncode"] = f"{AUTHOR_NAME} - CLIPTextEncode(OpenAIApi)" + +class MZ_ImageInterrogatorCLIPTextEncode: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "resolution": ("INT", {"default": 512, "min": 128, "max": 0xffffffffffffffff}), + "keep_device": ([False, True], {"default": False}), + "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), + }, + "optional": { + "image_interrogator_model": ("ImageInterrogatorModelConfig", ), + "image": ("IMAGE",), + "clip": ("CLIP", ), + "llama_cpp_options": ("LLamaCPPOptions", ), + "customize_instruct": ("CustomizeInstruct", ), + "captioner_config": ("ImageCaptionerConfig", ), + }, + } + + RETURN_TYPES = ("STRING", "CONDITIONING",) + RETURN_NAMES = ("text", "conditioning",) + OUTPUT_NODE = True + FUNCTION = "encode" + CATEGORY = CATEGORY_NAME + + def encode(self, **kwargs): + kwargs = kwargs.copy() + from . import mz_llama_core_nodes + importlib.reload(mz_llama_core_nodes) + + return mz_llama_core_nodes.image_interrogator_node_encode(kwargs) + + +NODE_CLASS_MAPPINGS["MZ_ImageInterrogatorCLIPTextEncode"] = MZ_ImageInterrogatorCLIPTextEncode +NODE_DISPLAY_NAME_MAPPINGS[ + "MZ_ImageInterrogatorCLIPTextEncode"] = f"{AUTHOR_NAME} - CLIPTextEncode(ImageInterrogator)" + + +class MZ_ImageInterrogatorModelConfig_ManualSelect: + @classmethod + def INPUT_TYPES(s): + gguf_files = Utils.get_gguf_files() + return { + "required": { + "llama_cpp_model": (gguf_files,), + "mmproj_model": (["auto"] + gguf_files,), + }, + "optional": { + }, + } + + RETURN_TYPES = ("ImageInterrogatorModelConfig",) + RETURN_NAMES = ("image_interrogator_model",) + + FUNCTION = "create" + CATEGORY = f"{CATEGORY_NAME}/others" + + def create(self, **kwargs): + kwargs = kwargs.copy() + + llama_cpp_model = kwargs.get("llama_cpp_model", "") + if llama_cpp_model != "": + llama_cpp_model = os.path.join( + Utils.get_gguf_models_path(), llama_cpp_model) + + mmproj_model = kwargs.get("mmproj_model", "") + if mmproj_model != "": + mmproj_model = os.path.join( + Utils.get_gguf_models_path(), mmproj_model) + + return ({ + "type": "ManualSelect", + "model_path": llama_cpp_model, + "mmproj_model": mmproj_model, + },) + + +NODE_CLASS_MAPPINGS["MZ_ImageInterrogatorModelConfig_ManualSelect"] = MZ_ImageInterrogatorModelConfig_ManualSelect +NODE_DISPLAY_NAME_MAPPINGS[ + "MZ_ImageInterrogatorModelConfig_ManualSelect"] = f"{AUTHOR_NAME} - ModelConfigManualSelect(ImageInterrogator)" + + +class MZ_ImageInterrogatorModelConfig_DownloaderSelect: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model_name": ([ + "ggml_llava1_5-7b-q4_k_m", + "ggml_bakllava-1-q4_k_m", + "llava_v1_6_mistral_7b_q5_k_m", + ],), + "mmproj_model_name": ([ + "auto", + "ggml_llava1_5-7b-mmproj-f16", + "ggml_bakllava-1-mmproj-f16", + "llava_v1_6_mistral_7b_mmproj_f16", + ],), + }, + "optional": { + }, + } + + RETURN_TYPES = ("ImageInterrogatorModelConfig",) + RETURN_NAMES = ("image_interrogator_model",) + + FUNCTION = "create" + CATEGORY = f"{CATEGORY_NAME}/others" + + def create(self, **kwargs): + kwargs = kwargs.copy() + model_name = kwargs.get("model_name") + mmproj_model_name = kwargs.get("mmproj_model_name", "auto") + return ({ + "type": "DownloaderSelect", + "model_name": model_name, + "mmproj_model_name": mmproj_model_name, + },) + + +NODE_CLASS_MAPPINGS["MZ_ImageInterrogatorModelConfig_DownloaderSelect"] = MZ_ImageInterrogatorModelConfig_DownloaderSelect +NODE_DISPLAY_NAME_MAPPINGS[ + "MZ_ImageInterrogatorModelConfig_DownloaderSelect"] = f"{AUTHOR_NAME} - ModelConfigDownloaderSelect(ImageInterrogator)" + + from . import mz_gen_translate mz_gen_translate.gen_translate(NODE_DISPLAY_NAME_MAPPINGS, NODE_CLASS_MAPPINGS) diff --git a/mz_gen_translate.py b/mz_gen_translate.py index 4653914..157c265 100644 --- a/mz_gen_translate.py +++ b/mz_gen_translate.py @@ -8,10 +8,15 @@ COMFY_PATH = Path(folder_paths.__file__).parent ZH_Replace_Map = { + "mmproj_model_name": "mmproj模型名称", + "model_name": "模型名称", "llama_cpp_model": "llama.cpp模型", "mmproj_model": "mmproj模型", "resolution": "分辨率", "sd_format": "SD格式化", + "ImageInterrogator": "图像反推", + "image_interrogator_model": "图像反推模型", + "image_interrogator": "图像反推", "image": "图像", "download_source": "下载源", "prompt_version": "提示词版本", @@ -19,15 +24,17 @@ ZH_Replace_Map = { "keep_device": "模型常驻显存", "llama_cpp_options": "llama.cpp可选配置", "Options": "可选配置", + "LLamaCPPModelConfigManualSelect": "llama.cpp模型配置手动选择", "LLamaCPP": "llama.cpp", "CLIPTextEncode": "CLIP文本编码器", "clip": "CLIP", "conditioning": "条件", - "ImageInterrogator": "图像反推", "customize_instruct": "自定义指令", "CustomizeInstruct": "自定义指令", "deprecated": "已废弃", - "LLamaCPPModelConfigManualSelect": "llama.cpp模型配置手动选择", + "ModelConfigManualSelect": "手动模型选择器", + "ModelConfigDownloaderSelect": "预设模型下载器", + "captioner_config": "打标器配置", } diff --git a/mz_llama_core_nodes.py b/mz_llama_core_nodes.py index d3e1480..1a6a759 100644 --- a/mz_llama_core_nodes.py +++ b/mz_llama_core_nodes.py @@ -1,6 +1,7 @@ import importlib import json +import os from . import mz_prompt_utils from . import mz_llama_cpp from . import mz_prompts @@ -59,11 +60,21 @@ def llama_cpp_node_encode(args_dict): model_config = args_dict.get("llama_cpp_model", {}) mz_prompt_utils.Utils.print_log(f"model_config: {model_config}") - model_file = model_config.get("model_path", "auto") + select_model_type = model_config.get("type", "ManualSelect") + if select_model_type == "ManualSelect": + model_file = model_config.get("model_path", "auto") + if model_file == "auto": + model_file = mz_prompt_utils.Utils.get_auto_model_fullpath( + "Meta-Llama-3-8B-Instruct-Q4_K_M") + elif select_model_type == "DownloaderSelect": + model_name = model_config.get("model_name", "") + model_file = mz_prompt_utils.Utils.get_auto_model_fullpath( + model_name) + else: + raise Exception("Unknown select_model_type") mz_prompt_utils.Utils.print_log(f"model_file: {model_file}") - text = args_dict.get("text", "") style_presets = args_dict.get("style_presets", "") options = args_dict.get("llama_cpp_options", {}) @@ -118,54 +129,35 @@ def llama_cpp_node_encode(args_dict): mz_prompt_utils.Utils.print_log(f"question: {question}") if schema is not None: - response_json = mz_llama_cpp.llama_cpp_simple_interrogator_to_json( + response_text = mz_llama_cpp.llama_cpp_simple_interrogator_to_json( model_file=model_file, system=system_prompt, question=question, schema=schema, options=options, ) - mz_prompt_utils.Utils.print_log(f"response_json: {response_json}") - response = json.loads(response_json) - full_responses = [] + response_json = json.loads(response_text) + mz_prompt_utils.Utils.print_log( + f"response_json: {json.dumps(response_json, indent=2)}") - if response["description"] != "": - full_responses.append(f"({response['description']})") - if response["long_prompt"] != "": - full_responses.append(f"({response['long_prompt']})") - if response["main_color_word"] != "": - full_responses.append(f"({response['main_color_word']})") - if response["camera_angle_word"] != "": - full_responses.append(f"({response['camera_angle_word']})") + responses = [] + for key, value in response_json.items(): + if type(value) == list: + # 去除开头.和空格 + value = [v.strip().lstrip(".") for v in value] + # 去除空字符串 + value = [v for v in value if v != ""] + if len(value) > 0: + responses.append(f"({', '.join(value)})") - response["style_words"] = [ - x for x in response["style_words"] if x != ""] - if len(response["style_words"]) > 0: - full_responses.append( - f"({', '.join(response['style_words'])})") + else: + if value != "": + responses.append(f"({value})") - response["subject_words"] = [ - x for x in response["subject_words"] if x != ""] - if len(response["subject_words"]) > 0: - full_responses.append( - f"({', '.join(response['subject_words'])})") - - response["light_words"] = [ - x for x in response["light_words"] if x != ""] - if len(response["light_words"]) > 0: - full_responses.append( - f"({', '.join(response['light_words'])})") - - response["environment_words"] = [ - x for x in response["environment_words"] if x != ""] - if len(response["environment_words"]) > 0: - full_responses.append( - f"({', '.join(response['environment_words'])})") - - full_response = ", ".join(full_responses) + response = ", ".join(responses) else: - full_response = mz_llama_cpp.llama_cpp_simple_interrogator( + response = mz_llama_cpp.llama_cpp_simple_interrogator( model_file=model_file, system=system_prompt, question=question, @@ -173,42 +165,42 @@ def llama_cpp_node_encode(args_dict): ) start_str = customize_instruct.get("start_str", "") - if start_str != "" and full_response.find(start_str) != -1: - full_response_list = full_response.split(start_str) + if start_str != "" and response.find(start_str) != -1: + full_response_list = response.split(start_str) # 删除第一个元素 full_response_list.pop(0) - full_response = start_str.join(full_response_list) + response = start_str.join(full_response_list) end_str = customize_instruct.get("end_str", "") - if end_str != "" and full_response.find(end_str) != -1: - full_response_list = full_response.split(end_str) + if end_str != "" and response.find(end_str) != -1: + full_response_list = response.split(end_str) # 删除最后一个元素 full_response_list.pop() - full_response = end_str.join(full_response_list) + response = end_str.join(full_response_list) if keep_device is False: mz_llama_cpp.freed_gpu_memory(model_file=model_file) # 去除换行 - while full_response.find("\n") != -1: - full_response = full_response.replace("\n", " ") + while response.find("\n") != -1: + response = response.replace("\n", " ") # 句号换成逗号 - while full_response.find(".") != -1: - full_response = full_response.replace(".", ",") + while response.find(".") != -1: + response = response.replace(".", ",") # 去除多余逗号 - while full_response.find(",,") != -1: - full_response = full_response.replace(",,", ",") - while full_response.find(", ,") != -1: - full_response = full_response.replace(", ,", ",") + while response.find(",,") != -1: + response = response.replace(",,", ",") + while response.find(", ,") != -1: + response = response.replace(", ,", ",") - full_response = mz_prompt_utils.Utils.prompt_zh_to_en(full_response) + response = mz_prompt_utils.Utils.prompt_zh_to_en(response) style_presets_prompt_text = style_presets_prompt.get(style_presets, "") if style_presets_prompt_text != "": - full_response = f"{style_presets_prompt_text}, {full_response}" + response = f"{style_presets_prompt_text}, {response}" except Exception as e: mz_llama_cpp.freed_gpu_memory(model_file=model_file) @@ -218,6 +210,215 @@ def llama_cpp_node_encode(args_dict): clip = args_dict.get("clip", None) if clip is not None: conditionings = mz_prompt_utils.Utils.a1111_clip_text_encode( - clip, full_response, ) + clip, response, ) - return {"ui": {"string": [full_response,]}, "result": (full_response, conditionings)} + return {"ui": {"string": [mz_prompt_utils.Utils.to_debug_prompt(response),]}, "result": (response, conditionings)} + + +def image_interrogator_captioner(args_dict): + import PIL.Image as Image + captioner_config = args_dict.get("captioner_config", {}) + directory = captioner_config.get("directory", None) + force_update = captioner_config.get("force_update", False) + caption_suffix = captioner_config.get("caption_suffix", "") + retry_keyword = captioner_config.get("retry_keyword", "") + retry_keywords = retry_keyword.split(",") + + retry_keywords = [k.strip() for k in retry_keywords] + retry_keywords = [k for k in retry_keywords if k != ""] + + pre_images = [] + for root, dirs, files in os.walk(directory): + for file in files: + if file.endswith(".jpg") or file.endswith(".jpeg") or file.endswith(".png"): + image_path = os.path.join(root, file) + base_file_path = os.path.splitext(image_path)[0] + caption_file = os.path.join( + root, base_file_path + caption_suffix) + if os.path.exists(caption_file) and force_update is False: + continue + + pre_images.append({ + "image_path": image_path, + "caption_path": caption_file + }) + + result = [] + + pb = mz_prompt_utils.Utils.progress_bar(len(pre_images)) + for i in range(len(pre_images)): + pre_image = pre_images[i] + image_path = pre_image["image_path"] + caption_file = pre_image["caption_path"] + + onec_args_dict = args_dict.copy() + del onec_args_dict["captioner_config"] + + pil_image = Image.open(image_path) + onec_args_dict["image"] = pil_image + + if i < len(pre_images) - 1: + onec_args_dict["keep_device"] = True + + pb.update( + i, + len(pre_images), + pil_image.copy(), + ) + + response = image_interrogator_node_encode(onec_args_dict) + response = response.get("result", ())[0] + response = response.strip() + is_retry = response == "" + for k in retry_keywords: + if response.find(k) != -1: + print(f"存在需要重试的关键词 ; Retry keyword found: {k}") + is_retry = True + break + + mz_prompt_utils.Utils.print_log( + "\n\nonec_args_dict: ", onec_args_dict) + if is_retry: + for retry_n in range(5): + print(f"Retry {retry_n+1}...") + onec_args_dict["seed"] = onec_args_dict["seed"] + 1 + response = image_interrogator_node_encode(onec_args_dict) + response = response.get("result", ())[0] + response = response.strip() + is_retry = response == "" + for k in retry_keywords: + if response.find(k) != -1: + print(f"存在需要重试的关键词 ; Retry keyword found: {k}") + is_retry = True + break + + if is_retry is False: + break + if is_retry: + print(f"重试失败,图片被跳过 ; Retry failed") + response = "" + + if response != "": + with open(caption_file, "w") as f: + f.write(response) + + result.append(response) + + # mz_prompt_webserver.show_toast_success( + # f"提示词保存成功(prompt saved successfully): {caption_file}", + # 1000, + # ) + + return result + + +image_interrogator_auto_mmproj_model = { + "7ac9c2f7b8d76cc7f3118cdf0953ebab7a7a9b12bad5dbe237219d2ab61765ea": "ggml_llava1_5-7b-mmproj-f16", + "c93de1376be9b6977cc94d252a3d165d6059e07b528de0fa762534d9599b27d6": "ggml_bakllava-1-mmproj-f16", + "b1d37fc65ecb80aa8f1ce185bf4d7605bc3c5cc5bcc77a160c3a1b0377631112": "llava_v1_6_mistral_7b_q5_k_m", +} + + +def image_interrogator_node_encode(args_dict): + importlib.reload(mz_prompts) + + captioner_config = args_dict.get("captioner_config", None) + if captioner_config is not None: + image_interrogator_captioner(args_dict) + raise Exception( + "图片批量反推任务已完成 ; Image batch reverse push task completed") + + model_config = args_dict.get("image_interrogator_model", {}) + llama_cpp_model = model_config.get("llama_cpp_model", "auto") + mmproj_model = model_config.get("mmproj_model", "auto") + + select_model_type = model_config.get("type", "ManualSelect") + if select_model_type == "ManualSelect": + llama_cpp_model = model_config.get("model_path", "auto") + if llama_cpp_model == "auto": + llama_cpp_model = mz_prompt_utils.Utils.get_auto_model_fullpath( + "ggml_llava1_5-7b-q4_k_m") + + if mmproj_model == "auto": + llama_cpp_model_sha256 = mz_prompt_utils.Utils.file_sha256( + llama_cpp_model) + mmproj_model_name = image_interrogator_auto_mmproj_model.get( + llama_cpp_model_sha256, None) + if mmproj_model_name is None: + mz_prompt_utils.Utils.print_log( + "llama_cpp_model_sha256: ", llama_cpp_model_sha256) + raise Exception( + "未能自动找到对应的mmproj文件 ; Failed to automatically find the corresponding mmproj file.") + mmproj_model = mz_prompt_utils.Utils.get_auto_model_fullpath( + mmproj_model_name) + + elif select_model_type == "DownloaderSelect": + model_name = model_config.get("model_name") + llama_cpp_model = mz_prompt_utils.Utils.get_auto_model_fullpath( + model_name) + + mmproj_model = model_config.get("mmproj_model_name", "auto") + if mmproj_model == "auto": + llama_cpp_model_sha256 = mz_prompt_utils.Utils.file_sha256( + llama_cpp_model) + + mz_prompt_utils.Utils.print_log( + "llama_cpp_model_sha256: ", llama_cpp_model_sha256) + + mmproj_model_name = image_interrogator_auto_mmproj_model.get( + llama_cpp_model_sha256, None) + if mmproj_model_name is None: + raise Exception( + "未能自动找到对应的mmproj文件 ; Failed to automatically find the corresponding mmproj file") + mmproj_model = mz_prompt_utils.Utils.get_auto_model_fullpath( + mmproj_model_name) + else: + raise Exception("Unknown select_model_type") + + image = args_dict.get("image", None) + image = mz_prompt_utils.Utils.tensor2pil(image) + + resolution = args_dict.get("resolution", 512) + keep_device = args_dict.get("keep_device", False) + seed = args_dict.get("seed", -1) + options = args_dict.get("llama_cpp_options", {}) + options["seed"] = seed + + image = mz_prompt_utils.Utils.resize_max(image, resolution, resolution) + + customize_instruct = args_dict.get("customize_instruct", None) + if customize_instruct is None: + system_prompt = mz_prompts.M_ImageCaptioner_System + question = mz_prompts.M_ImageCaptioner_Prompt + else: + system_prompt = customize_instruct.get("system", "") + question = customize_instruct.get("instruct", "") + + response = mz_llama_cpp.llava_cpp_simple_interrogator( + model_file=llama_cpp_model, + mmproj_file=mmproj_model, + image=image, + options=options, + system=system_prompt, + question=question, + ) + if response is not None: + response = response.strip() + if customize_instruct is None: + # 去除开头的In the image, + start_text = "In the image, " + if response.startswith(start_text): + response = response[len(start_text):] + + if keep_device is False: + mz_llama_cpp.freed_gpu_memory(model_file=llama_cpp_model) + + # return response + + conditionings = None + clip = args_dict.get("clip", None) + if clip is not None: + conditionings = mz_prompt_utils.Utils.a1111_clip_text_encode( + clip, response, ) + + return {"ui": {"string": [mz_prompt_utils.Utils.to_debug_prompt(response),]}, "result": (response, conditionings)} diff --git a/mz_llama_cpp.py b/mz_llama_cpp.py index 9bf406d..b2fde17 100644 --- a/mz_llama_cpp.py +++ b/mz_llama_cpp.py @@ -13,9 +13,13 @@ except ImportError: def check_llama_cpp_requirements(): - last_version = "0.2.63" + min_version = "0.2.63" + last_version = "0.2.76" try: from llama_cpp import Llama + import llama_cpp + if llama_cpp.__version__ < min_version: + raise ImportError("llama_cpp version is too low. (llama_cpp版本过低)") except ImportError: py_version = "" if sys.version_info.major == 3: @@ -65,9 +69,39 @@ def check_llama_cpp_requirements(): wheel_url = f"https://github.com/abetlen/llama-cpp-python/releases/download/v{last_version}-{cuda_version}/{wheel_name}" print(f"pip install {wheel_url}") - ret = subprocess.run([ - sys.executable, "-m", - "pip", "install", wheel_url], check=True) + modelscope_url = f"https://www.modelscope.cn/api/v1/models/wailovet/MinusZoneAIModels/repo?Revision=master&FilePath=llama-cpp-python-win%2F{cuda_version}%2F{wheel_name}" + if mz_prompt_utils.Utils.testDownloadSpeed(wheel_url): + ret = subprocess.run([ + sys.executable, "-m", + "pip", "install", wheel_url], check=True) + elif mz_prompt_utils.Utils.testDownloadSpeed(modelscope_url): + import tempfile + whl_download_file = os.path.join( + tempfile.gettempdir(), wheel_name) + mz_prompt_utils.Utils.download_file( + modelscope_url, whl_download_file) + print(f"pip install {whl_download_file}") + ret = subprocess.run([ + sys.executable, "-m", + "pip", "install", whl_download_file], check=True) + else: + + # 兜底方案 + modelscope_url = f"https://www.modelscope.cn/api/v1/models/wailovet/MinusZoneAIModels/repo?Revision=master&FilePath=llama-cpp-python-win%2Fcu121%2Fllama_cpp_python-0.2.76-cp310-cp310-win_amd64.whl" + if py_version == "310" and system_name == "win_amd64" and mz_prompt_utils.Utils.testDownloadSpeed(modelscope_url): + import tempfile + whl_download_file = os.path.join( + tempfile.gettempdir(), wheel_name) + mz_prompt_utils.Utils.download_file( + modelscope_url, whl_download_file) + print(f"pip install {whl_download_file}") + ret = subprocess.run([ + sys.executable, "-m", + "pip", "install", whl_download_file], check=True) + else: + ret = subprocess.run([ + sys.executable, "-m", + "pip", "install", wheel_url], check=True) if ret.returncode != 0: raise ValueError("Failed to install llama_cpp. (安装llama_cpp失败)") @@ -357,7 +391,8 @@ def llava_cpp_simple_interrogator( check_llama_cpp_requirements() from llama_cpp.llama_chat_format import Llava15ChatHandler if mmproj_file is not None: - mz_prompt_utils.Utils.print_log(f"llava_cpp_simple_interrogator mmproj_file: {mmproj_file}") + mz_prompt_utils.Utils.print_log( + f"llava_cpp_simple_interrogator mmproj_file: {mmproj_file}") chat_handler = Llava15ChatHandler(clip_model_path=mmproj_file) return llava_cpp_messages(model_file, chat_handler, [ diff --git a/mz_openaiapi.py b/mz_openaiapi.py index 3176d94..cd889cc 100644 --- a/mz_openaiapi.py +++ b/mz_openaiapi.py @@ -5,6 +5,7 @@ import subprocess from . import mz_prompt_utils from . import mz_llama_cpp +from . import mz_llama_core_nodes from . import mz_prompts @@ -63,16 +64,16 @@ def query_beautify_prompt_text(args_dict): schema = None if customize_instruct is None: - schema = mz_llama_cpp.get_schema_obj( + schema = mz_llama_core_nodes.get_schema_obj( keys_type={ - "description": mz_llama_cpp.get_schema_base_type("string"), - "long_prompt": mz_llama_cpp.get_schema_base_type("string"), - "main_color_word": mz_llama_cpp.get_schema_base_type("string"), - "camera_angle_word": mz_llama_cpp.get_schema_base_type("string"), - "style_words": mz_llama_cpp.get_schema_array("string"), - "subject_words": mz_llama_cpp.get_schema_array("string"), - "light_words": mz_llama_cpp.get_schema_array("string"), - "environment_words": mz_llama_cpp.get_schema_array("string"), + "description": mz_llama_core_nodes.get_schema_base_type("string"), + "long_prompt": mz_llama_core_nodes.get_schema_base_type("string"), + "main_color_word": mz_llama_core_nodes.get_schema_base_type("string"), + "camera_angle_word": mz_llama_core_nodes.get_schema_base_type("string"), + "style_words": mz_llama_core_nodes.get_schema_array("string"), + "subject_words": mz_llama_core_nodes.get_schema_array("string"), + "light_words": mz_llama_core_nodes.get_schema_array("string"), + "environment_words": mz_llama_core_nodes.get_schema_array("string"), }, required=[ "description", @@ -184,7 +185,7 @@ def query_beautify_prompt_text(args_dict): full_response = full_response.replace(",,", ",") while full_response.find(", ,") != -1: full_response = full_response.replace(", ,", ",") - style_presets_prompt_text = mz_llama_cpp.style_presets_prompt.get( + style_presets_prompt_text = mz_llama_core_nodes.style_presets_prompt.get( style_presets, "") if style_presets_prompt_text != "": full_response = f"{style_presets_prompt_text}, {full_response}" diff --git a/mz_prompt_utils.py b/mz_prompt_utils.py index 30db8d8..d3be6ce 100644 --- a/mz_prompt_utils.py +++ b/mz_prompt_utils.py @@ -494,8 +494,8 @@ class Utils: base_model_path, zone_path), exist_ok=True) Utils.download_file(url, save_path) - Utils.print_log( - f"File {save_path} => {os.path.getsize(save_path)} ") + # Utils.print_log( + # f"File {save_path} => {os.path.getsize(save_path)} ") # 获取大小 if os.path.getsize(save_path) == 0: @@ -617,7 +617,7 @@ class Utils: if t != "": result.append(Utils.split_en_to_zh(t)) return ", ".join(result) - + return Utils.en2zh(text) def to_debug_prompt(p): @@ -633,6 +633,256 @@ class Utils: {zh} """ + def get_gguf_files(): + gguf_dir = Utils.get_gguf_models_path() + if not os.path.exists(gguf_dir): + os.makedirs(gguf_dir) + gguf_files = [] + # walk gguf_dir + for root, dirs, files in os.walk(gguf_dir): + for file in files: + if file.endswith(".gguf"): + gguf_files.append( + os.path.relpath(os.path.join(root, file), gguf_dir)) + + return gguf_files + + def get_comfyui_models_path(): + return folder_paths.models_dir + + def download_model(model_info, only_get_path=False): + + url = model_info["url"] + output = model_info["output"] + save_path = os.path.abspath( + os.path.join(Utils.get_comfyui_models_path(), output)) + if not os.path.exists(save_path): + if only_get_path: + return None + save_path = Utils.download_file(url, save_path) + return save_path + + def file_hash(file_path, hash_method): + if not os.path.isfile(file_path): + return '' + h = hash_method() + with open(file_path, 'rb') as f: + while b := f.read(8192): + h.update(b) + return h.hexdigest() + + def file_sha256(file_path): + return Utils.file_hash(file_path, hashlib.sha256) + + def get_auto_model_fullpath(model_name): + fullpath = Utils.cache_get(f"get_auto_model_fullpath_{model_name}") + Utils.print_log(f"get_auto_model_fullpath_{model_name} => {fullpath}") + if fullpath is not None: + if os.path.exists(fullpath): + return fullpath + + find_paths = [] + target_sha256 = "" + file_path = "" + download_url = "" + for model in MODEL_ZOO: + if model["model"] == model_name: + find_paths = model["find_path"] + target_sha256 = model["SHA256"] + file_path = model["file_path"] + download_url = model["url"] + break + + if target_sha256 == "": + raise ValueError(f"Model {model_name} not found in MODEL_ZOO") + + if os.path.exists(file_path): + if Utils.file_sha256(file_path) != target_sha256: + print(f"Model {model_name} file hash not match...") + return file_path + + for find_path in find_paths: + find_fullpath = os.path.join( + Utils.get_comfyui_models_path(), find_path) + + if os.path.exists(find_fullpath): + for root, dirs, files in os.walk(find_fullpath): + for file in files: + if target_sha256 == Utils.file_sha256(os.path.join(root, file)): + Utils.cache_set(f"get_auto_model_fullpath_{model_name}", os.path.join(root, file)) + return os.path.join(root, file) + else: + Utils.print_log( + f"Model {os.path.join(root, file)} file hash not match, {target_sha256} != {Utils.file_sha256(os.path.join(root, file))}") + + result = Utils.download_model({"url": download_url, "output": file_path}) + Utils.cache_set(f"get_auto_model_fullpath_{model_name}", result) + return result + + + def testDownloadSpeed(url): + try: + print(f"Testing download speed for {url}") + start = time.time() + # 下载2M数据 + headers = {"Range": "bytes=0-2097151"} + _ = requests.get(url, headers=headers, timeout=5) + end = time.time() + print( + f"Download speed: {round(5.00 / (float(end) - float(start)) / 1024, 2)} KB/s") + return float(end) - float(start) < 4 + except Exception as e: + print(f"Test download speed failed: {e}") + return False + + +MODEL_ZOO = [ + { + "model": "Meta-Llama-3-8B-Instruct-Q4_K_M", + "find_path": [ + "gguf", + ], + "file_path": "gguf/Meta-Llama-3-8B-Instruct-Q4_K_M.gguf", + "url": "https://modelscope.cn/api/v1/models/LLM-Research/Meta-Llama-3-8B-Instruct-GGUF/repo?Revision=master&FilePath=Meta-Llama-3-8B-Instruct-Q4_K_M.gguf", + "SHA256": "57b26bac2df51111affec600077708de06133b8f49e697723672657c7cbe3b9c", + }, + { + "model": "Meta-Llama-3-8B-Q4_K_M", + "find_path": [ + "gguf", + ], + "file_path": "gguf/Meta-Llama-3-8B-Q4_K_M.gguf", + "url": "https://modelscope.cn/api/v1/models/LLM-Research/Meta-Llama-3-8B-GGUF/repo?Revision=master&FilePath=Meta-Llama-3-8B-Q4_K_M.gguf", + "SHA256": "b3413ff367d0c568d1ebb06faa6d1e44c3a7fc2ad8eea40b1685788ce4f90e3e", + }, + { + "model": "Phi-3-mini-4k-instruct-q4", + "find_path": [ + "gguf", + ], + "file_path": "gguf/Phi-3-mini-4k-instruct-q4.gguf", + "url": "https://www.modelscope.cn/api/v1/models/wailovet/MinusZoneAIModels/repo?Revision=master&FilePath=Phi-3-mini-4k-instruct-gguf%2FPhi-3-mini-4k-instruct-q4.gguf", + "SHA256": "1cd9a9df07350196623f93bf4829cf228959e07ad32f787b8fdd7f5956f5b9de", + }, + { + "model": "llama3-zh.Q4_K_M", + "find_path": [ + "gguf", + ], + "file_path": "gguf/llama3-zh.Q4_K_M.gguf", + "url": "https://modelscope.cn/api/v1/models/ModelM/Llama-3-8b-zh-gguf/repo?Revision=master&FilePath=llama3-zh.Q4_K_M.gguf", + "SHA256": "1b04ec22e4079af8064a8378d55d2cd79e43eff9faf4bbe8f341f1fd792a53cd", + }, + { + "model": "llama3_8b_instruct_dpo_zh-Q4_K_M", + "find_path": [ + "gguf", + ], + "file_path": "gguf/llama3_8b_instruct_dpo_zh-Q4_K_M.gguf", + "url": "https://modelscope.cn/api/v1/models/shareAI/llama-3-8b-Instruct-dpo-chinese-loftq-gguf/repo?Revision=master&FilePath=llama3_8b_instruct_dpo_zh-Q4_K_M.gguf", + "SHA256": "5231f5f119e1ef7db058211b8a140b530930a40b9b89c54db8455cc20ae3f699", + }, + { + "model": "qwen1_5-14b-chat-q4_k_m", + "find_path": [ + "gguf", + ], + "file_path": "gguf/qwen1_5-14b-chat-q4_k_m.gguf", + "url": "https://modelscope.cn/api/v1/models/qwen/Qwen1.5-14B-Chat-GGUF/repo?Revision=master&FilePath=qwen1_5-14b-chat-q4_k_m.gguf", + "SHA256": "46fbff2797c39c2d6aa555db0b0b4fe3f41b712a9b45266e438aa9a5047c0563", + }, + { + "model": "qwen1_5-7b-chat-q4_k_m", + "find_path": [ + "gguf", + ], + "file_path": "gguf/qwen1_5-7b-chat-q4_k_m.gguf", + "url": "https://modelscope.cn/api/v1/models/qwen/Qwen1.5-7B-Chat-GGUF/repo?Revision=master&FilePath=qwen1_5-7b-chat-q4_k_m.gguf", + "SHA256": "d7f132b1eff9ce35acf8e83ab96d2bc87eaedb68244e467bbc99e9f46a122a4c", + }, + { + "model": "qwen1_5-4b-chat-q4_k_m", + "find_path": [ + "gguf", + ], + "file_path": "gguf/qwen1_5-4b-chat-q4_k_m.gguf", + "url": "https://modelscope.cn/api/v1/models/qwen/Qwen1.5-4B-Chat-GGUF/repo?Revision=master&FilePath=qwen1_5-4b-chat-q4_k_m.gguf", + "SHA256": "426143ccd3241b9547c2b70c622b4f4ef3436ee07e44991bd69ad84b36cd9b9b", + }, + { + "model": "qwen1_5-1_8b-chat-q4_k_m", + "find_path": [ + "gguf", + ], + "file_path": "gguf/qwen1_5-1_8b-chat-q4_k_m.gguf", + "url": "https://modelscope.cn/api/v1/models/qwen/Qwen1.5-1.8B-Chat-GGUF/repo?Revision=master&FilePath=qwen1_5-1_8b-chat-q4_k_m.gguf", + "SHA256": "702e983c77883426806a2af75d34ab3e462e1b822f9dc23b49e02280c24b2b18", + }, + { + "model": "qwen1_5-0_5b-chat-q4_k_m", + "find_path": [ + "gguf", + ], + "file_path": "gguf/qwen1_5-0_5b-chat-q4_k_m.gguf", + "url": "https://modelscope.cn/api/v1/models/qwen/Qwen1.5-0.5B-Chat-GGUF/repo?Revision=master&FilePath=qwen1_5-0_5b-chat-q4_k_m.gguf", + "SHA256": "92916b71d32f5afea48fb7383e3b48c5b1c111f5a59f0b83c764ea1d07fe1a3a", + }, + { + "model": "ggml_llava1_5-7b-q4_k_m", + "find_path": [ + "gguf", + ], + "file_path": "gguf/ggml_llava1_5-7b-q4_k_m.gguf", + "url": "https://www.modelscope.cn/api/v1/models/wailovet/MinusZoneAIModels/repo?Revision=master&FilePath=ggml_llava-v1.5-7b%2Fggml-model-q4_k.gguf", + "SHA256": "7ac9c2f7b8d76cc7f3118cdf0953ebab7a7a9b12bad5dbe237219d2ab61765ea", + }, + { + "model": "ggml_llava1_5-7b-mmproj-f16", + "find_path": [ + "gguf", + ], + "file_path": "gguf/ggml_llava1_5-7b-mmproj-f16.gguf", + "url": "https://www.modelscope.cn/api/v1/models/wailovet/MinusZoneAIModels/repo?Revision=master&FilePath=ggml_llava-v1.5-7b%2Fmmproj-model-f16.gguf", + "SHA256": "b7c8ff0f58fca47d28ba92c4443adf8653f3349282cb8d9e6911f22d9b3814fe", + }, + { + "model": "ggml_bakllava-1-q4_k_m", + "find_path": [ + "gguf", + ], + "file_path": "gguf/ggml_bakllava-1-q4_k_m.gguf", + "url": "https://www.modelscope.cn/api/v1/models/wailovet/MinusZoneAIModels/repo?Revision=master&FilePath=ggml_bakllava-1%2Fggml-model-q5_k.gguf", + "SHA256": "c93de1376be9b6977cc94d252a3d165d6059e07b528de0fa762534d9599b27d6", + }, + { + "model": "ggml_bakllava-1-mmproj-f16", + "find_path": [ + "gguf", + ], + "file_path": "gguf/ggml_bakllava-1-mmproj-f16.gguf", + "url": "https://www.modelscope.cn/api/v1/models/wailovet/MinusZoneAIModels/repo?Revision=master&FilePath=ggml_bakllava-1%2Fmmproj-model-f16.gguf", + "SHA256": "2e467eba710002839e0966d5e329942bb836eabd4e787bc713b07eff1d8ea13b", + }, + { + "model": "llava_v1_6_mistral_7b_q5_k_m", + "find_path": [ + "gguf", + ], + "file_path": "gguf/llava_v1_6_mistral_7b_q5_k_m.gguf", + "url": "https://www.modelscope.cn/api/v1/models/wailovet/MinusZoneAIModels/repo?Revision=master&FilePath=llava-1.6-mistral-7b-gguf%2Fllava-v1.6-mistral-7b.Q5_K_M.gguf", + "SHA256": "b1d37fc65ecb80aa8f1ce185bf4d7605bc3c5cc5bcc77a160c3a1b0377631112", + }, + { + "model": "llava_v1_6_mistral_7b_mmproj_f16", + "find_path": [ + "gguf", + ], + "file_path": "gguf/llava_v1_6_mistral_7b_mmproj_f16.gguf", + "url": "https://www.modelscope.cn/api/v1/models/wailovet/MinusZoneAIModels/repo?Revision=master&FilePath=llava-1.6-mistral-7b-gguf%2Fmmproj-model-f16.gguf", + "SHA256": "00205ee8a0d7a381900cd031e43105f86aa0d8c07bf329851e85c71a26632d16", + }, + +] modelscope_models_map = { "llama3": { diff --git a/mz_prompts.py b/mz_prompts.py index f7e557f..c7a2328 100644 --- a/mz_prompts.py +++ b/mz_prompts.py @@ -26,7 +26,7 @@ As an AI image tagging expert, please provide precise tags for these images to e # 来自https://github.com/jiayev/GPT4V-Image-Captioner GPT4VImageCaptioner_Prompt = """ -Employ succinct keywords or phrases, steering clear of elaborate sentences and extraneous conjunctions. +Employ succinct keywords or phrases or sentence, steering clear of elaborate sentences and extraneous conjunctions. Prioritize the tags by relevance. Your tags should capture key elements such as the main subject, setting, artistic style, composition, image quality, color tone, filter, and camera specifications, and any other tags crucial for the image. When tagging photos of people, include specific details like gender, nationality, attire, actions, pose, expressions, accessories, makeup, composition type, age, etc. @@ -35,4 +35,15 @@ Recognize and tag any celebrities, well-known landmark or IPs if clearly feature Your tags should be accurate, non-duplicative, and within a 20-75 word count range. These tags will use for image re-creation, so the closer the resemblance to the original image, the better the tag quality. Tags should be comma-separated. -""" \ No newline at end of file +""" + + +M_ImageCaptioner_System = """ +Long prompt version should consist of 3 to 5 sentences. Long prompt version must sepcify the color, shape, texture or spatial relation of the included objects. DO NOT generate sentences that describe any atmosphere!!! +""" + +M_ImageCaptioner_Prompt = """ +Describe this image in detail please. +The language of reply is English only!!! +Starts with "In the image," +""" diff --git a/v1/init.py b/v1/init.py index 0864781..c9701f2 100644 --- a/v1/init.py +++ b/v1/init.py @@ -13,6 +13,7 @@ from . import mz_phi3 from .. import mz_llama_cpp from .. import mz_llama_core_nodes from . import mz_deprecated +from . import mz_llava AUTHOR_NAME = u"MinusZone" @@ -131,7 +132,7 @@ class MZ_Phi3CLIPTextEncode: def encode(self, **kwargs): kwargs = kwargs.copy() - + importlib.reload(mz_llama3) kwargs["llama_cpp_model"] = kwargs.get( @@ -179,7 +180,7 @@ class MZ_BaseLLamaCPPCLIPTextEncode: def encode(self, **kwargs): kwargs = kwargs.copy() - + kwargs["llama_cpp_model"] = kwargs.get( "llama_cpp_model", "").replace("[downloaded]", "") text = mz_deprecated.base_query_beautify_prompt_text(kwargs) @@ -194,3 +195,125 @@ class MZ_BaseLLamaCPPCLIPTextEncode: NODE_CLASS_MAPPINGS["MZ_BaseLLamaCPPCLIPTextEncode"] = MZ_BaseLLamaCPPCLIPTextEncode NODE_DISPLAY_NAME_MAPPINGS[ "MZ_BaseLLamaCPPCLIPTextEncode"] = f"{AUTHOR_NAME} - deprecated - CLIPTextEncode(BaseLLamaCPP)" + + +class MZ_LLavaImageInterrogator: + @classmethod + def INPUT_TYPES(s): + importlib.reload(mz_llava) + m_llava_models = mz_llava.LLava_models.copy() + for i in range(len(m_llava_models)): + if mz_llava.get_exist_model(m_llava_models[i]) is not None: + m_llava_models[i] += "[downloaded]" + + m_llava_mmproj_models = mz_llava.LLava_mmproj_models.copy() + for i in range(len(m_llava_mmproj_models)): + if mz_llava.get_exist_model(m_llava_mmproj_models[i]) is not None: + m_llava_mmproj_models[i] += "[downloaded]" + + return { + "required": { + "llama_cpp_model": (m_llava_models, {"default": m_llava_models[0]}), + "mmproj_model": (m_llava_mmproj_models, {"default": m_llava_mmproj_models[0]}), + "download_source": ( + [ + "none", + "modelscope", + "hf-mirror.com", + ], + {"default": "none"} + ), + "resolution": ("INT", {"default": 512, "min": 128, "max": 2048}), + "sd_format": (["none", "v1", ], {"default": "none"}), + "keep_device": ([False, True], {"default": False}), + "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), + }, + "optional": { + "image": ("IMAGE",), + "clip": ("CLIP", ), + "llama_cpp_options": ("LLamaCPPOptions", ), + "customize_instruct": ("CustomizeInstruct", ), + "captioner_config": ("ImageCaptionerConfig", ), + }, + } + RETURN_TYPES = ("STRING", "CONDITIONING",) + RETURN_NAMES = ("text", "conditioning",) + OUTPUT_NODE = True + FUNCTION = "interrogate" + CATEGORY = CATEGORY_NAME + + def interrogate(self, **kwargs): + kwargs = kwargs.copy() + + kwargs["llama_cpp_model"] = kwargs.get( + "llama_cpp_model", "").replace("[downloaded]", "") + kwargs["mmproj_model"] = kwargs.get( + "mmproj_model", "").replace("[downloaded]", "") + + if kwargs.get("image", None) is not None: + kwargs["image"] = Utils.tensor2pil(kwargs["image"]) + else: + kwargs["image"] = None + + text = mz_llava.image_interrogator(kwargs) + conditionings = None + clip = kwargs.get("clip", None) + if clip is not None: + conditionings = Utils.a1111_clip_text_encode(clip, text, ) + + return {"ui": {"string": [Utils.to_debug_prompt(text),]}, "result": (text, conditionings)} + + +NODE_CLASS_MAPPINGS["MZ_LLavaImageInterrogator"] = MZ_LLavaImageInterrogator +NODE_DISPLAY_NAME_MAPPINGS[ + "MZ_LLavaImageInterrogator"] = f"{AUTHOR_NAME} - deprecated - ImageInterrogator(LLava)" + + +class MZ_BaseLLavaImageInterrogator: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "llama_cpp_model": ("STRING", {"default": ""}), + "mmproj_model": ("STRING", {"default": ""}), + "resolution": ("INT", {"default": 512, "min": 128, "max": 2048}), + "sd_format": (["none", "v1"], {"default": "none"}), + "keep_device": ([False, True], {"default": False}), + "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), + }, + "optional": { + "image": ("IMAGE",), + "clip": ("CLIP", ), + "llama_cpp_options": ("LLamaCPPOptions", ), + "customize_instruct": ("CustomizeInstruct", ), + "captioner_config": ("ImageCaptionerConfig", ), + }, + } + RETURN_TYPES = ("STRING", "CONDITIONING",) + RETURN_NAMES = ("text", "conditioning",) + OUTPUT_NODE = True + FUNCTION = "interrogate" + CATEGORY = CATEGORY_NAME + + def interrogate(self, **kwargs): + kwargs = kwargs.copy() + + importlib.reload(mz_llava) + + if kwargs.get("image", None) is not None: + kwargs["image"] = Utils.tensor2pil(kwargs["image"]) + else: + kwargs["image"] = None + + text = mz_llava.base_image_interrogator(kwargs) + conditionings = None + clip = kwargs.get("clip", None) + if clip is not None: + conditionings = Utils.a1111_clip_text_encode(clip, text, ) + + return {"ui": {"string": [Utils.to_debug_prompt(text),]}, "result": (text, conditionings)} + + +NODE_CLASS_MAPPINGS["MZ_BaseLLavaImageInterrogator"] = MZ_BaseLLavaImageInterrogator +NODE_DISPLAY_NAME_MAPPINGS[ + "MZ_BaseLLavaImageInterrogator"] = f"{AUTHOR_NAME} - deprecated - ImageInterrogator(BaseLLava)" diff --git a/mz_llava.py b/v1/mz_llava.py similarity index 98% rename from mz_llava.py rename to v1/mz_llava.py index 550e4db..769c77c 100644 --- a/mz_llava.py +++ b/v1/mz_llava.py @@ -1,10 +1,10 @@ import json import os -from . import mz_prompt_utils -from . import mz_llama_cpp -from . import mz_prompts -from . import mz_llama_core_nodes -from . import mz_prompt_webserver +from .. import mz_prompt_utils +from .. import mz_llama_cpp +from .. import mz_prompts +from .. import mz_llama_core_nodes +from .. import mz_prompt_webserver import importlib diff --git a/web/prompt_mz.js b/web/prompt_mz.js index cb328d4..4ea3a28 100644 --- a/web/prompt_mz.js +++ b/web/prompt_mz.js @@ -90,6 +90,7 @@ const my_ui = { case "MZ_LLavaImageInterrogator": case "MZ_BaseLLavaImageInterrogator": case "MZ_LLamaCPPCLIPTextEncode": + case "MZ_ImageInterrogatorCLIPTextEncode": // Node Created const onNodeCreated = nodeType.prototype.onNodeCreated; nodeType.prototype.onNodeCreated = function () { @@ -136,7 +137,7 @@ const my_ui = { const onConfigure = nodeType.prototype.onConfigure; nodeType.prototype.onConfigure = function (w) { onConfigure?.apply(this, arguments); - + // outSet.call(this, a?.string); };