nodes/MinusZoneAI-ComfyUI-Prompt-MZ
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #14 from MinusZoneAI/dev

Browse Source 
Dev
This commit is contained in:
喵哩个咪
2024-06-02 14:04:16 +08:00
committed by GitHub
parent 9c53acd073 371b37c46b
commit 8a179f2532
10 changed files with 919 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files
__init__.py
+203 -170
View File
@@ -32,22 +32,34 @@ NODE_DISPLAY_NAME_MAPPINGS = {
from . import mz_llama_cpp
from . import mz_llava
def getCommonCLIPTextEncodeInput():
from . import mz_llama_core_nodes
style_presets = mz_llama_core_nodes.get_style_presets()
CommonCLIPTextEncodeInput = {
"required": {
"style_presets": (
style_presets, {"default": style_presets[1]}
),
"text": ("STRING", {"multiline": True, }),
"keep_device": ([False, True], {"default": False}),
"seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}),
},
"optional": {
"clip": ("CLIP", ),
"llama_cpp_options": ("LLamaCPPOptions", ),
"customize_instruct": ("CustomizeInstruct", ),
}
}
return CommonCLIPTextEncodeInput
class MZ_LLamaCPPModelConfig_ManualSelect:
@classmethod
def INPUT_TYPES(s):
gguf_dir = Utils.get_gguf_models_path()
if not os.path.exists(gguf_dir):
os.makedirs(gguf_dir)
gguf_files = []
# walk gguf_dir
for root, dirs, files in os.walk(gguf_dir):
for file in files:
if file.endswith(".gguf"):
gguf_files.append(
os.path.relpath(os.path.join(root, file), gguf_dir))
gguf_files = Utils.get_gguf_files()
return {
"required": {
"llama_cpp_model": (gguf_files,),
@@ -60,7 +72,7 @@ class MZ_LLamaCPPModelConfig_ManualSelect:
RETURN_NAMES = ("llama_cpp_model_config",)
FUNCTION = "create"
CATEGORY = CATEGORY_NAME
CATEGORY = f"{CATEGORY_NAME}/others"
def create(self, **kwargs):
kwargs = kwargs.copy()
@@ -78,7 +90,50 @@ class MZ_LLamaCPPModelConfig_ManualSelect:
NODE_CLASS_MAPPINGS["MZ_LLamaCPPModelConfig_ManualSelect"] = MZ_LLamaCPPModelConfig_ManualSelect
NODE_DISPLAY_NAME_MAPPINGS[
"MZ_LLamaCPPModelConfig_ManualSelect"] = f"{AUTHOR_NAME} - ModelConfigManualSelect"
"MZ_LLamaCPPModelConfig_ManualSelect"] = f"{AUTHOR_NAME} - ModelConfigManualSelect(LLamaCPP)"
class MZ_LLamaCPPModelConfig_DownloaderSelect:
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"model_name": ([
"Meta-Llama-3-8B-Instruct-Q4_K_M",
"Phi-3-mini-4k-instruct-q4",
"Meta-Llama-3-8B-Q4_K_M",
"llama3_8b_instruct_dpo_zh-Q4_K_M",
"llama3-zh.Q4_K_M",
"qwen1_5-14b-chat-q4_k_m",
"qwen1_5-7b-chat-q4_k_m",
"qwen1_5-4b-chat-q4_k_m",
"qwen1_5-1_8b-chat-q4_k_m",
"qwen1_5-0_5b-chat-q4_k_m",
],),
},
"optional": {
},
}
RETURN_TYPES = ("LLamaCPPModelConfig",)
RETURN_NAMES = ("llama_cpp_model_config",)
FUNCTION = "create"
CATEGORY = f"{CATEGORY_NAME}/others"
def create(self, **kwargs):
kwargs = kwargs.copy()
model_name = kwargs.get("model_name", "")
return ({
"type": "DownloaderSelect",
"model_name": model_name,
},)
NODE_CLASS_MAPPINGS["MZ_LLamaCPPModelConfig_DownloaderSelect"] = MZ_LLamaCPPModelConfig_DownloaderSelect
NODE_DISPLAY_NAME_MAPPINGS[
"MZ_LLamaCPPModelConfig_DownloaderSelect"] = f"{AUTHOR_NAME} - ModelConfigDownloaderSelect(LLamaCPP)"
class MZ_LLamaCPPCLIPTextEncode:
@@ -109,7 +164,7 @@ class MZ_LLamaCPPCLIPTextEncode:
CATEGORY = CATEGORY_NAME
DESCRIPTION = """
llama_cpp_model不设置时，将使用默认模型: Meta-Llama-3-8B-Instruct.Q4_K_M.gguf
llama_cpp_model不设置时，将使用默认模型: Meta-Llama-3-8B-Instruct-Q4_K_M.gguf
"""
def encode(self, **kwargs):
@@ -155,7 +210,7 @@ class MZ_LLamaCPPOptions:
RETURN_NAMES = ("llama_cpp_options",)
FUNCTION = "create"
CATEGORY = CATEGORY_NAME
CATEGORY = f"{CATEGORY_NAME}/others"
def create(self, **kwargs):
kwargs = kwargs.copy()
@@ -178,18 +233,18 @@ class MZ_CustomizeInstruct:
return {
"required": {
"system": ("STRING", {"multiline": True, "default": mz_prompts.Long_prompt}),
"instruct": ("STRING", {"multiline": True, "default": "Short: %text%"}),
"instruct": ("STRING", {"multiline": True, "default": ""}),
},
}
RETURN_TYPES = ("CustomizeInstruct",)
RETURN_NAMES = ("customize_instruct",)
FUNCTION = "create"
CATEGORY = CATEGORY_NAME
CATEGORY = f"{CATEGORY_NAME}/others"
def create(self, **kwargs):
kwargs = kwargs.copy()
return (kwargs,)
@@ -197,154 +252,6 @@ NODE_CLASS_MAPPINGS["MZ_CustomizeInstruct"] = MZ_CustomizeInstruct
NODE_DISPLAY_NAME_MAPPINGS["MZ_CustomizeInstruct"] = f"{AUTHOR_NAME} - CustomizeInstruct"
def getCommonCLIPTextEncodeInput():
from . import mz_llama_core_nodes
style_presets = mz_llama_core_nodes.get_style_presets()
CommonCLIPTextEncodeInput = {
"required": {
"prompt_version": (["v1"], {"default": "v1"}),
"style_presets": (
style_presets, {"default": style_presets[1]}
),
"text": ("STRING", {"multiline": True, }),
"keep_device": ([False, True], {"default": False}),
"seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}),
},
"optional": {
"clip": ("CLIP", ),
"llama_cpp_options": ("LLamaCPPOptions", ),
"customize_instruct": ("CustomizeInstruct", ),
# "customize_json_schema": ("STRING", ),
}
}
return CommonCLIPTextEncodeInput
class MZ_LLavaImageInterrogator:
@classmethod
def INPUT_TYPES(s):
importlib.reload(mz_llava)
m_llava_models = mz_llava.LLava_models.copy()
for i in range(len(m_llava_models)):
if mz_llava.get_exist_model(m_llava_models[i]) is not None:
m_llava_models[i] += "[downloaded]"
m_llava_mmproj_models = mz_llava.LLava_mmproj_models.copy()
for i in range(len(m_llava_mmproj_models)):
if mz_llava.get_exist_model(m_llava_mmproj_models[i]) is not None:
m_llava_mmproj_models[i] += "[downloaded]"
return {
"required": {
"llama_cpp_model": (m_llava_models, {"default": m_llava_models[0]}),
"mmproj_model": (m_llava_mmproj_models, {"default": m_llava_mmproj_models[0]}),
"download_source": (
[
"none",
"modelscope",
"hf-mirror.com",
],
{"default": "none"}
),
"resolution": ("INT", {"default": 512, "min": 128, "max": 2048}),
"sd_format": (["none", "v1", ], {"default": "none"}),
"keep_device": ([False, True], {"default": False}),
"seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}),
},
"optional": {
"image": ("IMAGE",),
"clip": ("CLIP", ),
"llama_cpp_options": ("LLamaCPPOptions", ),
"customize_instruct": ("CustomizeInstruct", ),
"captioner_config": ("ImageCaptionerConfig", ),
},
}
RETURN_TYPES = ("STRING", "CONDITIONING",)
RETURN_NAMES = ("text", "conditioning",)
OUTPUT_NODE = True
FUNCTION = "interrogate"
CATEGORY = CATEGORY_NAME
def interrogate(self, **kwargs):
kwargs = kwargs.copy()
importlib.reload(mz_llava)
kwargs["llama_cpp_model"] = kwargs.get(
"llama_cpp_model", "").replace("[downloaded]", "")
kwargs["mmproj_model"] = kwargs.get(
"mmproj_model", "").replace("[downloaded]", "")
if kwargs.get("image", None) is not None:
kwargs["image"] = Utils.tensor2pil(kwargs["image"])
else:
kwargs["image"] = None
text = mz_llava.image_interrogator(kwargs)
conditionings = None
clip = kwargs.get("clip", None)
if clip is not None:
conditionings = Utils.a1111_clip_text_encode(clip, text, )
return {"ui": {"string": [Utils.to_debug_prompt(text),]}, "result": (text, conditionings)}
NODE_CLASS_MAPPINGS["MZ_LLavaImageInterrogator"] = MZ_LLavaImageInterrogator
NODE_DISPLAY_NAME_MAPPINGS[
"MZ_LLavaImageInterrogator"] = f"{AUTHOR_NAME} - ImageInterrogator(LLava)"
class MZ_BaseLLavaImageInterrogator:
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"llama_cpp_model": ("STRING", {"default": ""}),
"mmproj_model": ("STRING", {"default": ""}),
"resolution": ("INT", {"default": 512, "min": 128, "max": 2048}),
"sd_format": (["none", "v1"], {"default": "none"}),
"keep_device": ([False, True], {"default": False}),
"seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}),
},
"optional": {
"image": ("IMAGE",),
"clip": ("CLIP", ),
"llama_cpp_options": ("LLamaCPPOptions", ),
"customize_instruct": ("CustomizeInstruct", ),
"captioner_config": ("ImageCaptionerConfig", ),
},
}
RETURN_TYPES = ("STRING", "CONDITIONING",)
RETURN_NAMES = ("text", "conditioning",)
OUTPUT_NODE = True
FUNCTION = "interrogate"
CATEGORY = CATEGORY_NAME
def interrogate(self, **kwargs):
kwargs = kwargs.copy()
importlib.reload(mz_llava)
if kwargs.get("image", None) is not None:
kwargs["image"] = Utils.tensor2pil(kwargs["image"])
else:
kwargs["image"] = None
text = mz_llava.base_image_interrogator(kwargs)
conditionings = None
clip = kwargs.get("clip", None)
if clip is not None:
conditionings = Utils.a1111_clip_text_encode(clip, text, )
return {"ui": {"string": [Utils.to_debug_prompt(text),]}, "result": (text, conditionings)}
NODE_CLASS_MAPPINGS["MZ_BaseLLavaImageInterrogator"] = MZ_BaseLLavaImageInterrogator
NODE_DISPLAY_NAME_MAPPINGS[
"MZ_BaseLLavaImageInterrogator"] = f"{AUTHOR_NAME} - ImageInterrogator(BaseLLava)"
class MZ_ImageCaptionerConfig:
@classmethod
def INPUT_TYPES(s):
@@ -364,11 +271,11 @@ class MZ_ImageCaptionerConfig:
RETURN_NAMES = ("captioner_config", )
FUNCTION = "interrogate_batch"
CATEGORY = CATEGORY_NAME
CATEGORY = f"{CATEGORY_NAME}/others"
def interrogate_batch(self, **kwargs):
kwargs = kwargs.copy()
return (kwargs, )
@@ -427,8 +334,8 @@ class MZ_OpenAIApiCLIPTextEncode:
def encode(self, **kwargs):
kwargs = kwargs.copy()
import mz_openaiapi
from . import mz_openaiapi
importlib.reload(mz_openaiapi)
if kwargs.get("api_key", "").endswith("******"):
@@ -463,6 +370,132 @@ NODE_CLASS_MAPPINGS["MZ_OpenAIApiCLIPTextEncode"] = MZ_OpenAIApiCLIPTextEncode
NODE_DISPLAY_NAME_MAPPINGS[
"MZ_OpenAIApiCLIPTextEncode"] = f"{AUTHOR_NAME} - CLIPTextEncode(OpenAIApi)"
class MZ_ImageInterrogatorCLIPTextEncode:
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"resolution": ("INT", {"default": 512, "min": 128, "max": 0xffffffffffffffff}),
"keep_device": ([False, True], {"default": False}),
"seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}),
},
"optional": {
"image_interrogator_model": ("ImageInterrogatorModelConfig", ),
"image": ("IMAGE",),
"clip": ("CLIP", ),
"llama_cpp_options": ("LLamaCPPOptions", ),
"customize_instruct": ("CustomizeInstruct", ),
"captioner_config": ("ImageCaptionerConfig", ),
},
}
RETURN_TYPES = ("STRING", "CONDITIONING",)
RETURN_NAMES = ("text", "conditioning",)
OUTPUT_NODE = True
FUNCTION = "encode"
CATEGORY = CATEGORY_NAME
def encode(self, **kwargs):
kwargs = kwargs.copy()
from . import mz_llama_core_nodes
importlib.reload(mz_llama_core_nodes)
return mz_llama_core_nodes.image_interrogator_node_encode(kwargs)
NODE_CLASS_MAPPINGS["MZ_ImageInterrogatorCLIPTextEncode"] = MZ_ImageInterrogatorCLIPTextEncode
NODE_DISPLAY_NAME_MAPPINGS[
"MZ_ImageInterrogatorCLIPTextEncode"] = f"{AUTHOR_NAME} - CLIPTextEncode(ImageInterrogator)"
class MZ_ImageInterrogatorModelConfig_ManualSelect:
@classmethod
def INPUT_TYPES(s):
gguf_files = Utils.get_gguf_files()
return {
"required": {
"llama_cpp_model": (gguf_files,),
"mmproj_model": (["auto"] + gguf_files,),
},
"optional": {
},
}
RETURN_TYPES = ("ImageInterrogatorModelConfig",)
RETURN_NAMES = ("image_interrogator_model",)
FUNCTION = "create"
CATEGORY = f"{CATEGORY_NAME}/others"
def create(self, **kwargs):
kwargs = kwargs.copy()
llama_cpp_model = kwargs.get("llama_cpp_model", "")
if llama_cpp_model != "":
llama_cpp_model = os.path.join(
Utils.get_gguf_models_path(), llama_cpp_model)
mmproj_model = kwargs.get("mmproj_model", "")
if mmproj_model != "":
mmproj_model = os.path.join(
Utils.get_gguf_models_path(), mmproj_model)
return ({
"type": "ManualSelect",
"model_path": llama_cpp_model,
"mmproj_model": mmproj_model,
},)
NODE_CLASS_MAPPINGS["MZ_ImageInterrogatorModelConfig_ManualSelect"] = MZ_ImageInterrogatorModelConfig_ManualSelect
NODE_DISPLAY_NAME_MAPPINGS[
"MZ_ImageInterrogatorModelConfig_ManualSelect"] = f"{AUTHOR_NAME} - ModelConfigManualSelect(ImageInterrogator)"
class MZ_ImageInterrogatorModelConfig_DownloaderSelect:
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"model_name": ([
"ggml_llava1_5-7b-q4_k_m",
"ggml_bakllava-1-q4_k_m",
"llava_v1_6_mistral_7b_q5_k_m",
],),
"mmproj_model_name": ([
"auto",
"ggml_llava1_5-7b-mmproj-f16",
"ggml_bakllava-1-mmproj-f16",
"llava_v1_6_mistral_7b_mmproj_f16",
],),
},
"optional": {
},
}
RETURN_TYPES = ("ImageInterrogatorModelConfig",)
RETURN_NAMES = ("image_interrogator_model",)
FUNCTION = "create"
CATEGORY = f"{CATEGORY_NAME}/others"
def create(self, **kwargs):
kwargs = kwargs.copy()
model_name = kwargs.get("model_name")
mmproj_model_name = kwargs.get("mmproj_model_name", "auto")
return ({
"type": "DownloaderSelect",
"model_name": model_name,
"mmproj_model_name": mmproj_model_name,
},)
NODE_CLASS_MAPPINGS["MZ_ImageInterrogatorModelConfig_DownloaderSelect"] = MZ_ImageInterrogatorModelConfig_DownloaderSelect
NODE_DISPLAY_NAME_MAPPINGS[
"MZ_ImageInterrogatorModelConfig_DownloaderSelect"] = f"{AUTHOR_NAME} - ModelConfigDownloaderSelect(ImageInterrogator)"
from . import mz_gen_translate
mz_gen_translate.gen_translate(NODE_DISPLAY_NAME_MAPPINGS, NODE_CLASS_MAPPINGS)
mz_gen_translate.py
+9 -2
View File
@@ -8,10 +8,15 @@ COMFY_PATH = Path(folder_paths.__file__).parent
ZH_Replace_Map = {
"mmproj_model_name": "mmproj模型名称",
"model_name": "模型名称",
"llama_cpp_model": "llama.cpp模型",
"mmproj_model": "mmproj模型",
"resolution": "分辨率",
"sd_format": "SD格式化",
"ImageInterrogator": "图像反推",
"image_interrogator_model": "图像反推模型",
"image_interrogator": "图像反推",
"image": "图像",
"download_source": "下载源",
"prompt_version": "提示词版本",
@@ -19,15 +24,17 @@ ZH_Replace_Map = {
"keep_device": "模型常驻显存",
"llama_cpp_options": "llama.cpp可选配置",
"Options": "可选配置",
"LLamaCPPModelConfigManualSelect": "llama.cpp模型配置手动选择",
"LLamaCPP": "llama.cpp",
"CLIPTextEncode": "CLIP文本编码器",
"clip": "CLIP",
"conditioning": "条件",
"ImageInterrogator": "图像反推",
"customize_instruct": "自定义指令",
"CustomizeInstruct": "自定义指令",
"deprecated": "已废弃",
"LLamaCPPModelConfigManualSelect": "llama.cpp模型配置手动选择",
"ModelConfigManualSelect": "手动模型选择器",
"ModelConfigDownloaderSelect": "预设模型下载器",
"captioner_config": "打标器配置",
}
mz_llama_core_nodes.py
+258 -57
View File
@@ -1,6 +1,7 @@
import importlib
import json
import os
from . import mz_prompt_utils
from . import mz_llama_cpp
from . import mz_prompts
@@ -59,11 +60,21 @@ def llama_cpp_node_encode(args_dict):
model_config = args_dict.get("llama_cpp_model", {})
mz_prompt_utils.Utils.print_log(f"model_config: {model_config}")
model_file = model_config.get("model_path", "auto")
select_model_type = model_config.get("type", "ManualSelect")
if select_model_type == "ManualSelect":
model_file = model_config.get("model_path", "auto")
if model_file == "auto":
model_file = mz_prompt_utils.Utils.get_auto_model_fullpath(
"Meta-Llama-3-8B-Instruct-Q4_K_M")
elif select_model_type == "DownloaderSelect":
model_name = model_config.get("model_name", "")
model_file = mz_prompt_utils.Utils.get_auto_model_fullpath(
model_name)
else:
raise Exception("Unknown select_model_type")
mz_prompt_utils.Utils.print_log(f"model_file: {model_file}")
text = args_dict.get("text", "")
style_presets = args_dict.get("style_presets", "")
options = args_dict.get("llama_cpp_options", {})
@@ -118,54 +129,35 @@ def llama_cpp_node_encode(args_dict):
mz_prompt_utils.Utils.print_log(f"question: {question}")
if schema is not None:
response_json = mz_llama_cpp.llama_cpp_simple_interrogator_to_json(
response_text = mz_llama_cpp.llama_cpp_simple_interrogator_to_json(
model_file=model_file,
system=system_prompt,
question=question,
schema=schema,
options=options,
)
mz_prompt_utils.Utils.print_log(f"response_json: {response_json}")
response = json.loads(response_json)
full_responses = []
response_json = json.loads(response_text)
mz_prompt_utils.Utils.print_log(
f"response_json: {json.dumps(response_json, indent=2)}")
if response["description"] != "":
full_responses.append(f"({response['description']})")
if response["long_prompt"] != "":
full_responses.append(f"({response['long_prompt']})")
if response["main_color_word"] != "":
full_responses.append(f"({response['main_color_word']})")
if response["camera_angle_word"] != "":
full_responses.append(f"({response['camera_angle_word']})")
responses = []
for key, value in response_json.items():
if type(value) == list:
# 去除开头.和空格
value = [v.strip().lstrip(".") for v in value]
# 去除空字符串
value = [v for v in value if v != ""]
if len(value) > 0:
responses.append(f"({', '.join(value)})")
response["style_words"] = [
x for x in response["style_words"] if x != ""]
if len(response["style_words"]) > 0:
full_responses.append(
f"({', '.join(response['style_words'])})")
else:
if value != "":
responses.append(f"({value})")
response["subject_words"] = [
x for x in response["subject_words"] if x != ""]
if len(response["subject_words"]) > 0:
full_responses.append(
f"({', '.join(response['subject_words'])})")
response["light_words"] = [
x for x in response["light_words"] if x != ""]
if len(response["light_words"]) > 0:
full_responses.append(
f"({', '.join(response['light_words'])})")
response["environment_words"] = [
x for x in response["environment_words"] if x != ""]
if len(response["environment_words"]) > 0:
full_responses.append(
f"({', '.join(response['environment_words'])})")
full_response = ", ".join(full_responses)
response = ", ".join(responses)
else:
full_response = mz_llama_cpp.llama_cpp_simple_interrogator(
response = mz_llama_cpp.llama_cpp_simple_interrogator(
model_file=model_file,
system=system_prompt,
question=question,
@@ -173,42 +165,42 @@ def llama_cpp_node_encode(args_dict):
)
start_str = customize_instruct.get("start_str", "")
if start_str != "" and full_response.find(start_str) != -1:
full_response_list = full_response.split(start_str)
if start_str != "" and response.find(start_str) != -1:
full_response_list = response.split(start_str)
# 删除第一个元素
full_response_list.pop(0)
full_response = start_str.join(full_response_list)
response = start_str.join(full_response_list)
end_str = customize_instruct.get("end_str", "")
if end_str != "" and full_response.find(end_str) != -1:
full_response_list = full_response.split(end_str)
if end_str != "" and response.find(end_str) != -1:
full_response_list = response.split(end_str)
# 删除最后一个元素
full_response_list.pop()
full_response = end_str.join(full_response_list)
response = end_str.join(full_response_list)
if keep_device is False:
mz_llama_cpp.freed_gpu_memory(model_file=model_file)
# 去除换行
while full_response.find("\n") != -1:
full_response = full_response.replace("\n", " ")
while response.find("\n") != -1:
response = response.replace("\n", " ")
# 句号换成逗号
while full_response.find(".") != -1:
full_response = full_response.replace(".", ",")
while response.find(".") != -1:
response = response.replace(".", ",")
# 去除多余逗号
while full_response.find(",,") != -1:
full_response = full_response.replace(",,", ",")
while full_response.find(", ,") != -1:
full_response = full_response.replace(", ,", ",")
while response.find(",,") != -1:
response = response.replace(",,", ",")
while response.find(", ,") != -1:
response = response.replace(", ,", ",")
full_response = mz_prompt_utils.Utils.prompt_zh_to_en(full_response)
response = mz_prompt_utils.Utils.prompt_zh_to_en(response)
style_presets_prompt_text = style_presets_prompt.get(style_presets, "")
if style_presets_prompt_text != "":
full_response = f"{style_presets_prompt_text}, {full_response}"
response = f"{style_presets_prompt_text}, {response}"
except Exception as e:
mz_llama_cpp.freed_gpu_memory(model_file=model_file)
@@ -218,6 +210,215 @@ def llama_cpp_node_encode(args_dict):
clip = args_dict.get("clip", None)
if clip is not None:
conditionings = mz_prompt_utils.Utils.a1111_clip_text_encode(
clip, full_response, )
clip, response, )
return {"ui": {"string": [full_response,]}, "result": (full_response, conditionings)}
return {"ui": {"string": [mz_prompt_utils.Utils.to_debug_prompt(response),]}, "result": (response, conditionings)}
def image_interrogator_captioner(args_dict):
import PIL.Image as Image
captioner_config = args_dict.get("captioner_config", {})
directory = captioner_config.get("directory", None)
force_update = captioner_config.get("force_update", False)
caption_suffix = captioner_config.get("caption_suffix", "")
retry_keyword = captioner_config.get("retry_keyword", "")
retry_keywords = retry_keyword.split(",")
retry_keywords = [k.strip() for k in retry_keywords]
retry_keywords = [k for k in retry_keywords if k != ""]
pre_images = []
for root, dirs, files in os.walk(directory):
for file in files:
if file.endswith(".jpg") or file.endswith(".jpeg") or file.endswith(".png"):
image_path = os.path.join(root, file)
base_file_path = os.path.splitext(image_path)[0]
caption_file = os.path.join(
root, base_file_path + caption_suffix)
if os.path.exists(caption_file) and force_update is False:
continue
pre_images.append({
"image_path": image_path,
"caption_path": caption_file
})
result = []
pb = mz_prompt_utils.Utils.progress_bar(len(pre_images))
for i in range(len(pre_images)):
pre_image = pre_images[i]
image_path = pre_image["image_path"]
caption_file = pre_image["caption_path"]
onec_args_dict = args_dict.copy()
del onec_args_dict["captioner_config"]
pil_image = Image.open(image_path)
onec_args_dict["image"] = pil_image
if i < len(pre_images) - 1:
onec_args_dict["keep_device"] = True
pb.update(
i,
len(pre_images),
pil_image.copy(),
)
response = image_interrogator_node_encode(onec_args_dict)
response = response.get("result", ())[0]
response = response.strip()
is_retry = response == ""
for k in retry_keywords:
if response.find(k) != -1:
print(f"存在需要重试的关键词 ; Retry keyword found: {k}")
is_retry = True
break
mz_prompt_utils.Utils.print_log(
"\n\nonec_args_dict: ", onec_args_dict)
if is_retry:
for retry_n in range(5):
print(f"Retry {retry_n+1}...")
onec_args_dict["seed"] = onec_args_dict["seed"] + 1
response = image_interrogator_node_encode(onec_args_dict)
response = response.get("result", ())[0]
response = response.strip()
is_retry = response == ""
for k in retry_keywords:
if response.find(k) != -1:
print(f"存在需要重试的关键词 ; Retry keyword found: {k}")
is_retry = True
break
if is_retry is False:
break
if is_retry:
print(f"重试失败,图片被跳过 ; Retry failed")
response = ""
if response != "":
with open(caption_file, "w") as f:
f.write(response)
result.append(response)
# mz_prompt_webserver.show_toast_success(
# f"提示词保存成功(prompt saved successfully): {caption_file}",
# 1000,
# )
return result
image_interrogator_auto_mmproj_model = {
"7ac9c2f7b8d76cc7f3118cdf0953ebab7a7a9b12bad5dbe237219d2ab61765ea": "ggml_llava1_5-7b-mmproj-f16",
"c93de1376be9b6977cc94d252a3d165d6059e07b528de0fa762534d9599b27d6": "ggml_bakllava-1-mmproj-f16",
"b1d37fc65ecb80aa8f1ce185bf4d7605bc3c5cc5bcc77a160c3a1b0377631112": "llava_v1_6_mistral_7b_q5_k_m",
}
def image_interrogator_node_encode(args_dict):
importlib.reload(mz_prompts)
captioner_config = args_dict.get("captioner_config", None)
if captioner_config is not None:
image_interrogator_captioner(args_dict)
raise Exception(
"图片批量反推任务已完成 ; Image batch reverse push task completed")
model_config = args_dict.get("image_interrogator_model", {})
llama_cpp_model = model_config.get("llama_cpp_model", "auto")
mmproj_model = model_config.get("mmproj_model", "auto")
select_model_type = model_config.get("type", "ManualSelect")
if select_model_type == "ManualSelect":
llama_cpp_model = model_config.get("model_path", "auto")
if llama_cpp_model == "auto":
llama_cpp_model = mz_prompt_utils.Utils.get_auto_model_fullpath(
"ggml_llava1_5-7b-q4_k_m")
if mmproj_model == "auto":
llama_cpp_model_sha256 = mz_prompt_utils.Utils.file_sha256(
llama_cpp_model)
mmproj_model_name = image_interrogator_auto_mmproj_model.get(
llama_cpp_model_sha256, None)
if mmproj_model_name is None:
mz_prompt_utils.Utils.print_log(
"llama_cpp_model_sha256: ", llama_cpp_model_sha256)
raise Exception(
"未能自动找到对应的mmproj文件 ; Failed to automatically find the corresponding mmproj file.")
mmproj_model = mz_prompt_utils.Utils.get_auto_model_fullpath(
mmproj_model_name)
elif select_model_type == "DownloaderSelect":
model_name = model_config.get("model_name")
llama_cpp_model = mz_prompt_utils.Utils.get_auto_model_fullpath(
model_name)
mmproj_model = model_config.get("mmproj_model_name", "auto")
if mmproj_model == "auto":
llama_cpp_model_sha256 = mz_prompt_utils.Utils.file_sha256(
llama_cpp_model)
mz_prompt_utils.Utils.print_log(
"llama_cpp_model_sha256: ", llama_cpp_model_sha256)
mmproj_model_name = image_interrogator_auto_mmproj_model.get(
llama_cpp_model_sha256, None)
if mmproj_model_name is None:
raise Exception(
"未能自动找到对应的mmproj文件 ; Failed to automatically find the corresponding mmproj file")
mmproj_model = mz_prompt_utils.Utils.get_auto_model_fullpath(
mmproj_model_name)
else:
raise Exception("Unknown select_model_type")
image = args_dict.get("image", None)
image = mz_prompt_utils.Utils.tensor2pil(image)
resolution = args_dict.get("resolution", 512)
keep_device = args_dict.get("keep_device", False)
seed = args_dict.get("seed", -1)
options = args_dict.get("llama_cpp_options", {})
options["seed"] = seed
image = mz_prompt_utils.Utils.resize_max(image, resolution, resolution)
customize_instruct = args_dict.get("customize_instruct", None)
if customize_instruct is None:
system_prompt = mz_prompts.M_ImageCaptioner_System
question = mz_prompts.M_ImageCaptioner_Prompt
else:
system_prompt = customize_instruct.get("system", "")
question = customize_instruct.get("instruct", "")
response = mz_llama_cpp.llava_cpp_simple_interrogator(
model_file=llama_cpp_model,
mmproj_file=mmproj_model,
image=image,
options=options,
system=system_prompt,
question=question,
)
if response is not None:
response = response.strip()
if customize_instruct is None:
# 去除开头的In the image,
start_text = "In the image, "
if response.startswith(start_text):
response = response[len(start_text):]
if keep_device is False:
mz_llama_cpp.freed_gpu_memory(model_file=llama_cpp_model)
# return response
conditionings = None
clip = args_dict.get("clip", None)
if clip is not None:
conditionings = mz_prompt_utils.Utils.a1111_clip_text_encode(
clip, response, )
return {"ui": {"string": [mz_prompt_utils.Utils.to_debug_prompt(response),]}, "result": (response, conditionings)}
mz_llama_cpp.py
+40 -5
View File
@@ -13,9 +13,13 @@ except ImportError:
def check_llama_cpp_requirements():
last_version = "0.2.63"
min_version = "0.2.63"
last_version = "0.2.76"
try:
from llama_cpp import Llama
import llama_cpp
if llama_cpp.__version__ < min_version:
raise ImportError("llama_cpp version is too low. (llama_cpp版本过低)")
except ImportError:
py_version = ""
if sys.version_info.major == 3:
@@ -65,9 +69,39 @@ def check_llama_cpp_requirements():
wheel_url = f"https://github.com/abetlen/llama-cpp-python/releases/download/v{last_version}-{cuda_version}/{wheel_name}"
print(f"pip install {wheel_url}")
ret = subprocess.run([
sys.executable, "-m",
"pip", "install", wheel_url], check=True)
modelscope_url = f"https://www.modelscope.cn/api/v1/models/wailovet/MinusZoneAIModels/repo?Revision=master&FilePath=llama-cpp-python-win%2F{cuda_version}%2F{wheel_name}"
if mz_prompt_utils.Utils.testDownloadSpeed(wheel_url):
ret = subprocess.run([
sys.executable, "-m",
"pip", "install", wheel_url], check=True)
elif mz_prompt_utils.Utils.testDownloadSpeed(modelscope_url):
import tempfile
whl_download_file = os.path.join(
tempfile.gettempdir(), wheel_name)
mz_prompt_utils.Utils.download_file(
modelscope_url, whl_download_file)
print(f"pip install {whl_download_file}")
ret = subprocess.run([
sys.executable, "-m",
"pip", "install", whl_download_file], check=True)
else:
# 兜底方案
modelscope_url = f"https://www.modelscope.cn/api/v1/models/wailovet/MinusZoneAIModels/repo?Revision=master&FilePath=llama-cpp-python-win%2Fcu121%2Fllama_cpp_python-0.2.76-cp310-cp310-win_amd64.whl"
if py_version == "310" and system_name == "win_amd64" and mz_prompt_utils.Utils.testDownloadSpeed(modelscope_url):
import tempfile
whl_download_file = os.path.join(
tempfile.gettempdir(), wheel_name)
mz_prompt_utils.Utils.download_file(
modelscope_url, whl_download_file)
print(f"pip install {whl_download_file}")
ret = subprocess.run([
sys.executable, "-m",
"pip", "install", whl_download_file], check=True)
else:
ret = subprocess.run([
sys.executable, "-m",
"pip", "install", wheel_url], check=True)
if ret.returncode != 0:
raise ValueError("Failed to install llama_cpp. (安装llama_cpp失败)")
@@ -357,7 +391,8 @@ def llava_cpp_simple_interrogator(
check_llama_cpp_requirements()
from llama_cpp.llama_chat_format import Llava15ChatHandler
if mmproj_file is not None:
mz_prompt_utils.Utils.print_log(f"llava_cpp_simple_interrogator mmproj_file: {mmproj_file}")
mz_prompt_utils.Utils.print_log(
f"llava_cpp_simple_interrogator mmproj_file: {mmproj_file}")
chat_handler = Llava15ChatHandler(clip_model_path=mmproj_file)
return llava_cpp_messages(model_file, chat_handler, [
mz_openaiapi.py
+11 -10
View File
@@ -5,6 +5,7 @@ import subprocess
from . import mz_prompt_utils
from . import mz_llama_cpp
from . import mz_llama_core_nodes
from . import mz_prompts
@@ -63,16 +64,16 @@ def query_beautify_prompt_text(args_dict):
schema = None
if customize_instruct is None:
schema = mz_llama_cpp.get_schema_obj(
schema = mz_llama_core_nodes.get_schema_obj(
keys_type={
"description": mz_llama_cpp.get_schema_base_type("string"),
"long_prompt": mz_llama_cpp.get_schema_base_type("string"),
"main_color_word": mz_llama_cpp.get_schema_base_type("string"),
"camera_angle_word": mz_llama_cpp.get_schema_base_type("string"),
"style_words": mz_llama_cpp.get_schema_array("string"),
"subject_words": mz_llama_cpp.get_schema_array("string"),
"light_words": mz_llama_cpp.get_schema_array("string"),
"environment_words": mz_llama_cpp.get_schema_array("string"),
"description": mz_llama_core_nodes.get_schema_base_type("string"),
"long_prompt": mz_llama_core_nodes.get_schema_base_type("string"),
"main_color_word": mz_llama_core_nodes.get_schema_base_type("string"),
"camera_angle_word": mz_llama_core_nodes.get_schema_base_type("string"),
"style_words": mz_llama_core_nodes.get_schema_array("string"),
"subject_words": mz_llama_core_nodes.get_schema_array("string"),
"light_words": mz_llama_core_nodes.get_schema_array("string"),
"environment_words": mz_llama_core_nodes.get_schema_array("string"),
},
required=[
"description",
@@ -184,7 +185,7 @@ def query_beautify_prompt_text(args_dict):
full_response = full_response.replace(",,", ",")
while full_response.find(", ,") != -1:
full_response = full_response.replace(", ,", ",")
style_presets_prompt_text = mz_llama_cpp.style_presets_prompt.get(
style_presets_prompt_text = mz_llama_core_nodes.style_presets_prompt.get(
style_presets, "")
if style_presets_prompt_text != "":
full_response = f"{style_presets_prompt_text}, {full_response}"
mz_prompt_utils.py
+253 -3
View File
@@ -494,8 +494,8 @@ class Utils:
base_model_path, zone_path), exist_ok=True)
Utils.download_file(url, save_path)
Utils.print_log(
f"File {save_path} => {os.path.getsize(save_path)} ")
# Utils.print_log(
# f"File {save_path} => {os.path.getsize(save_path)} ")
# 获取大小
if os.path.getsize(save_path) == 0:
@@ -617,7 +617,7 @@ class Utils:
if t != "":
result.append(Utils.split_en_to_zh(t))
return ", ".join(result)
return Utils.en2zh(text)
def to_debug_prompt(p):
@@ -633,6 +633,256 @@ class Utils:
{zh}
"""
def get_gguf_files():
gguf_dir = Utils.get_gguf_models_path()
if not os.path.exists(gguf_dir):
os.makedirs(gguf_dir)
gguf_files = []
# walk gguf_dir
for root, dirs, files in os.walk(gguf_dir):
for file in files:
if file.endswith(".gguf"):
gguf_files.append(
os.path.relpath(os.path.join(root, file), gguf_dir))
return gguf_files
def get_comfyui_models_path():
return folder_paths.models_dir
def download_model(model_info, only_get_path=False):
url = model_info["url"]
output = model_info["output"]
save_path = os.path.abspath(
os.path.join(Utils.get_comfyui_models_path(), output))
if not os.path.exists(save_path):
if only_get_path:
return None
save_path = Utils.download_file(url, save_path)
return save_path
def file_hash(file_path, hash_method):
if not os.path.isfile(file_path):
return ''
h = hash_method()
with open(file_path, 'rb') as f:
while b := f.read(8192):
h.update(b)
return h.hexdigest()
def file_sha256(file_path):
return Utils.file_hash(file_path, hashlib.sha256)
def get_auto_model_fullpath(model_name):
fullpath = Utils.cache_get(f"get_auto_model_fullpath_{model_name}")
Utils.print_log(f"get_auto_model_fullpath_{model_name} => {fullpath}")
if fullpath is not None:
if os.path.exists(fullpath):
return fullpath
find_paths = []
target_sha256 = ""
file_path = ""
download_url = ""
for model in MODEL_ZOO:
if model["model"] == model_name:
find_paths = model["find_path"]
target_sha256 = model["SHA256"]
file_path = model["file_path"]
download_url = model["url"]
break
if target_sha256 == "":
raise ValueError(f"Model {model_name} not found in MODEL_ZOO")
if os.path.exists(file_path):
if Utils.file_sha256(file_path) != target_sha256:
print(f"Model {model_name} file hash not match...")
return file_path
for find_path in find_paths:
find_fullpath = os.path.join(
Utils.get_comfyui_models_path(), find_path)
if os.path.exists(find_fullpath):
for root, dirs, files in os.walk(find_fullpath):
for file in files:
if target_sha256 == Utils.file_sha256(os.path.join(root, file)):
Utils.cache_set(f"get_auto_model_fullpath_{model_name}", os.path.join(root, file))
return os.path.join(root, file)
else:
Utils.print_log(
f"Model {os.path.join(root, file)} file hash not match, {target_sha256} != {Utils.file_sha256(os.path.join(root, file))}")
result = Utils.download_model({"url": download_url, "output": file_path})
Utils.cache_set(f"get_auto_model_fullpath_{model_name}", result)
return result
def testDownloadSpeed(url):
try:
print(f"Testing download speed for {url}")
start = time.time()
# 下载2M数据
headers = {"Range": "bytes=0-2097151"}
_ = requests.get(url, headers=headers, timeout=5)
end = time.time()
print(
f"Download speed: {round(5.00 / (float(end) - float(start)) / 1024, 2)} KB/s")
return float(end) - float(start) < 4
except Exception as e:
print(f"Test download speed failed: {e}")
return False
MODEL_ZOO = [
{
"model": "Meta-Llama-3-8B-Instruct-Q4_K_M",
"find_path": [
"gguf",
],
"file_path": "gguf/Meta-Llama-3-8B-Instruct-Q4_K_M.gguf",
"url": "https://modelscope.cn/api/v1/models/LLM-Research/Meta-Llama-3-8B-Instruct-GGUF/repo?Revision=master&FilePath=Meta-Llama-3-8B-Instruct-Q4_K_M.gguf",
"SHA256": "57b26bac2df51111affec600077708de06133b8f49e697723672657c7cbe3b9c",
},
{
"model": "Meta-Llama-3-8B-Q4_K_M",
"find_path": [
"gguf",
],
"file_path": "gguf/Meta-Llama-3-8B-Q4_K_M.gguf",
"url": "https://modelscope.cn/api/v1/models/LLM-Research/Meta-Llama-3-8B-GGUF/repo?Revision=master&FilePath=Meta-Llama-3-8B-Q4_K_M.gguf",
"SHA256": "b3413ff367d0c568d1ebb06faa6d1e44c3a7fc2ad8eea40b1685788ce4f90e3e",
},
{
"model": "Phi-3-mini-4k-instruct-q4",
"find_path": [
"gguf",
],
"file_path": "gguf/Phi-3-mini-4k-instruct-q4.gguf",
"url": "https://www.modelscope.cn/api/v1/models/wailovet/MinusZoneAIModels/repo?Revision=master&FilePath=Phi-3-mini-4k-instruct-gguf%2FPhi-3-mini-4k-instruct-q4.gguf",
"SHA256": "1cd9a9df07350196623f93bf4829cf228959e07ad32f787b8fdd7f5956f5b9de",
},
{
"model": "llama3-zh.Q4_K_M",
"find_path": [
"gguf",
],
"file_path": "gguf/llama3-zh.Q4_K_M.gguf",
"url": "https://modelscope.cn/api/v1/models/ModelM/Llama-3-8b-zh-gguf/repo?Revision=master&FilePath=llama3-zh.Q4_K_M.gguf",
"SHA256": "1b04ec22e4079af8064a8378d55d2cd79e43eff9faf4bbe8f341f1fd792a53cd",
},
{
"model": "llama3_8b_instruct_dpo_zh-Q4_K_M",
"find_path": [
"gguf",
],
"file_path": "gguf/llama3_8b_instruct_dpo_zh-Q4_K_M.gguf",
"url": "https://modelscope.cn/api/v1/models/shareAI/llama-3-8b-Instruct-dpo-chinese-loftq-gguf/repo?Revision=master&FilePath=llama3_8b_instruct_dpo_zh-Q4_K_M.gguf",
"SHA256": "5231f5f119e1ef7db058211b8a140b530930a40b9b89c54db8455cc20ae3f699",
},
{
"model": "qwen1_5-14b-chat-q4_k_m",
"find_path": [
"gguf",
],
"file_path": "gguf/qwen1_5-14b-chat-q4_k_m.gguf",
"url": "https://modelscope.cn/api/v1/models/qwen/Qwen1.5-14B-Chat-GGUF/repo?Revision=master&FilePath=qwen1_5-14b-chat-q4_k_m.gguf",
"SHA256": "46fbff2797c39c2d6aa555db0b0b4fe3f41b712a9b45266e438aa9a5047c0563",
},
{
"model": "qwen1_5-7b-chat-q4_k_m",
"find_path": [
"gguf",
],
"file_path": "gguf/qwen1_5-7b-chat-q4_k_m.gguf",
"url": "https://modelscope.cn/api/v1/models/qwen/Qwen1.5-7B-Chat-GGUF/repo?Revision=master&FilePath=qwen1_5-7b-chat-q4_k_m.gguf",
"SHA256": "d7f132b1eff9ce35acf8e83ab96d2bc87eaedb68244e467bbc99e9f46a122a4c",
},
{
"model": "qwen1_5-4b-chat-q4_k_m",
"find_path": [
"gguf",
],
"file_path": "gguf/qwen1_5-4b-chat-q4_k_m.gguf",
"url": "https://modelscope.cn/api/v1/models/qwen/Qwen1.5-4B-Chat-GGUF/repo?Revision=master&FilePath=qwen1_5-4b-chat-q4_k_m.gguf",
"SHA256": "426143ccd3241b9547c2b70c622b4f4ef3436ee07e44991bd69ad84b36cd9b9b",
},
{
"model": "qwen1_5-1_8b-chat-q4_k_m",
"find_path": [
"gguf",
],
"file_path": "gguf/qwen1_5-1_8b-chat-q4_k_m.gguf",
"url": "https://modelscope.cn/api/v1/models/qwen/Qwen1.5-1.8B-Chat-GGUF/repo?Revision=master&FilePath=qwen1_5-1_8b-chat-q4_k_m.gguf",
"SHA256": "702e983c77883426806a2af75d34ab3e462e1b822f9dc23b49e02280c24b2b18",
},
{
"model": "qwen1_5-0_5b-chat-q4_k_m",
"find_path": [
"gguf",
],
"file_path": "gguf/qwen1_5-0_5b-chat-q4_k_m.gguf",
"url": "https://modelscope.cn/api/v1/models/qwen/Qwen1.5-0.5B-Chat-GGUF/repo?Revision=master&FilePath=qwen1_5-0_5b-chat-q4_k_m.gguf",
"SHA256": "92916b71d32f5afea48fb7383e3b48c5b1c111f5a59f0b83c764ea1d07fe1a3a",
},
{
"model": "ggml_llava1_5-7b-q4_k_m",
"find_path": [
"gguf",
],
"file_path": "gguf/ggml_llava1_5-7b-q4_k_m.gguf",
"url": "https://www.modelscope.cn/api/v1/models/wailovet/MinusZoneAIModels/repo?Revision=master&FilePath=ggml_llava-v1.5-7b%2Fggml-model-q4_k.gguf",
"SHA256": "7ac9c2f7b8d76cc7f3118cdf0953ebab7a7a9b12bad5dbe237219d2ab61765ea",
},
{
"model": "ggml_llava1_5-7b-mmproj-f16",
"find_path": [
"gguf",
],
"file_path": "gguf/ggml_llava1_5-7b-mmproj-f16.gguf",
"url": "https://www.modelscope.cn/api/v1/models/wailovet/MinusZoneAIModels/repo?Revision=master&FilePath=ggml_llava-v1.5-7b%2Fmmproj-model-f16.gguf",
"SHA256": "b7c8ff0f58fca47d28ba92c4443adf8653f3349282cb8d9e6911f22d9b3814fe",
},
{
"model": "ggml_bakllava-1-q4_k_m",
"find_path": [
"gguf",
],
"file_path": "gguf/ggml_bakllava-1-q4_k_m.gguf",
"url": "https://www.modelscope.cn/api/v1/models/wailovet/MinusZoneAIModels/repo?Revision=master&FilePath=ggml_bakllava-1%2Fggml-model-q5_k.gguf",
"SHA256": "c93de1376be9b6977cc94d252a3d165d6059e07b528de0fa762534d9599b27d6",
},
{
"model": "ggml_bakllava-1-mmproj-f16",
"find_path": [
"gguf",
],
"file_path": "gguf/ggml_bakllava-1-mmproj-f16.gguf",
"url": "https://www.modelscope.cn/api/v1/models/wailovet/MinusZoneAIModels/repo?Revision=master&FilePath=ggml_bakllava-1%2Fmmproj-model-f16.gguf",
"SHA256": "2e467eba710002839e0966d5e329942bb836eabd4e787bc713b07eff1d8ea13b",
},
{
"model": "llava_v1_6_mistral_7b_q5_k_m",
"find_path": [
"gguf",
],
"file_path": "gguf/llava_v1_6_mistral_7b_q5_k_m.gguf",
"url": "https://www.modelscope.cn/api/v1/models/wailovet/MinusZoneAIModels/repo?Revision=master&FilePath=llava-1.6-mistral-7b-gguf%2Fllava-v1.6-mistral-7b.Q5_K_M.gguf",
"SHA256": "b1d37fc65ecb80aa8f1ce185bf4d7605bc3c5cc5bcc77a160c3a1b0377631112",
},
{
"model": "llava_v1_6_mistral_7b_mmproj_f16",
"find_path": [
"gguf",
],
"file_path": "gguf/llava_v1_6_mistral_7b_mmproj_f16.gguf",
"url": "https://www.modelscope.cn/api/v1/models/wailovet/MinusZoneAIModels/repo?Revision=master&FilePath=llava-1.6-mistral-7b-gguf%2Fmmproj-model-f16.gguf",
"SHA256": "00205ee8a0d7a381900cd031e43105f86aa0d8c07bf329851e85c71a26632d16",
},
]
modelscope_models_map = {
"llama3": {
mz_prompts.py
+13 -2
View File
@@ -26,7 +26,7 @@ As an AI image tagging expert, please provide precise tags for these images to e
# 来自https://github.com/jiayev/GPT4V-Image-Captioner
GPT4VImageCaptioner_Prompt = """
Employ succinct keywords or phrases, steering clear of elaborate sentences and extraneous conjunctions.
Employ succinct keywords or phrases or sentence, steering clear of elaborate sentences and extraneous conjunctions.
Prioritize the tags by relevance.
Your tags should capture key elements such as the main subject, setting, artistic style, composition, image quality, color tone, filter, and camera specifications, and any other tags crucial for the image.
When tagging photos of people, include specific details like gender, nationality, attire, actions, pose, expressions, accessories, makeup, composition type, age, etc.
@@ -35,4 +35,15 @@ Recognize and tag any celebrities, well-known landmark or IPs if clearly feature
Your tags should be accurate, non-duplicative, and within a 20-75 word count range.
These tags will use for image re-creation, so the closer the resemblance to the original image, the better the tag quality.
Tags should be comma-separated.
"""
"""
M_ImageCaptioner_System = """
Long prompt version should consist of 3 to 5 sentences. Long prompt version must sepcify the color, shape, texture or spatial relation of the included objects. DO NOT generate sentences that describe any atmosphere!!!
"""
M_ImageCaptioner_Prompt = """
Describe this image in detail please.
The language of reply is English only!!!
Starts with "In the image,"
"""
v1/init.py
+125 -2
View File
@@ -13,6 +13,7 @@ from . import mz_phi3
from .. import mz_llama_cpp
from .. import mz_llama_core_nodes
from . import mz_deprecated
from . import mz_llava
AUTHOR_NAME = u"MinusZone"
@@ -131,7 +132,7 @@ class MZ_Phi3CLIPTextEncode:
def encode(self, **kwargs):
kwargs = kwargs.copy()
importlib.reload(mz_llama3)
kwargs["llama_cpp_model"] = kwargs.get(
@@ -179,7 +180,7 @@ class MZ_BaseLLamaCPPCLIPTextEncode:
def encode(self, **kwargs):
kwargs = kwargs.copy()
kwargs["llama_cpp_model"] = kwargs.get(
"llama_cpp_model", "").replace("[downloaded]", "")
text = mz_deprecated.base_query_beautify_prompt_text(kwargs)
@@ -194,3 +195,125 @@ class MZ_BaseLLamaCPPCLIPTextEncode:
NODE_CLASS_MAPPINGS["MZ_BaseLLamaCPPCLIPTextEncode"] = MZ_BaseLLamaCPPCLIPTextEncode
NODE_DISPLAY_NAME_MAPPINGS[
"MZ_BaseLLamaCPPCLIPTextEncode"] = f"{AUTHOR_NAME} - deprecated - CLIPTextEncode(BaseLLamaCPP)"
class MZ_LLavaImageInterrogator:
@classmethod
def INPUT_TYPES(s):
importlib.reload(mz_llava)
m_llava_models = mz_llava.LLava_models.copy()
for i in range(len(m_llava_models)):
if mz_llava.get_exist_model(m_llava_models[i]) is not None:
m_llava_models[i] += "[downloaded]"
m_llava_mmproj_models = mz_llava.LLava_mmproj_models.copy()
for i in range(len(m_llava_mmproj_models)):
if mz_llava.get_exist_model(m_llava_mmproj_models[i]) is not None:
m_llava_mmproj_models[i] += "[downloaded]"
return {
"required": {
"llama_cpp_model": (m_llava_models, {"default": m_llava_models[0]}),
"mmproj_model": (m_llava_mmproj_models, {"default": m_llava_mmproj_models[0]}),
"download_source": (
[
"none",
"modelscope",
"hf-mirror.com",
],
{"default": "none"}
),
"resolution": ("INT", {"default": 512, "min": 128, "max": 2048}),
"sd_format": (["none", "v1", ], {"default": "none"}),
"keep_device": ([False, True], {"default": False}),
"seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}),
},
"optional": {
"image": ("IMAGE",),
"clip": ("CLIP", ),
"llama_cpp_options": ("LLamaCPPOptions", ),
"customize_instruct": ("CustomizeInstruct", ),
"captioner_config": ("ImageCaptionerConfig", ),
},
}
RETURN_TYPES = ("STRING", "CONDITIONING",)
RETURN_NAMES = ("text", "conditioning",)
OUTPUT_NODE = True
FUNCTION = "interrogate"
CATEGORY = CATEGORY_NAME
def interrogate(self, **kwargs):
kwargs = kwargs.copy()
kwargs["llama_cpp_model"] = kwargs.get(
"llama_cpp_model", "").replace("[downloaded]", "")
kwargs["mmproj_model"] = kwargs.get(
"mmproj_model", "").replace("[downloaded]", "")
if kwargs.get("image", None) is not None:
kwargs["image"] = Utils.tensor2pil(kwargs["image"])
else:
kwargs["image"] = None
text = mz_llava.image_interrogator(kwargs)
conditionings = None
clip = kwargs.get("clip", None)
if clip is not None:
conditionings = Utils.a1111_clip_text_encode(clip, text, )
return {"ui": {"string": [Utils.to_debug_prompt(text),]}, "result": (text, conditionings)}
NODE_CLASS_MAPPINGS["MZ_LLavaImageInterrogator"] = MZ_LLavaImageInterrogator
NODE_DISPLAY_NAME_MAPPINGS[
"MZ_LLavaImageInterrogator"] = f"{AUTHOR_NAME} - deprecated - ImageInterrogator(LLava)"
class MZ_BaseLLavaImageInterrogator:
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"llama_cpp_model": ("STRING", {"default": ""}),
"mmproj_model": ("STRING", {"default": ""}),
"resolution": ("INT", {"default": 512, "min": 128, "max": 2048}),
"sd_format": (["none", "v1"], {"default": "none"}),
"keep_device": ([False, True], {"default": False}),
"seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}),
},
"optional": {
"image": ("IMAGE",),
"clip": ("CLIP", ),
"llama_cpp_options": ("LLamaCPPOptions", ),
"customize_instruct": ("CustomizeInstruct", ),
"captioner_config": ("ImageCaptionerConfig", ),
},
}
RETURN_TYPES = ("STRING", "CONDITIONING",)
RETURN_NAMES = ("text", "conditioning",)
OUTPUT_NODE = True
FUNCTION = "interrogate"
CATEGORY = CATEGORY_NAME
def interrogate(self, **kwargs):
kwargs = kwargs.copy()
importlib.reload(mz_llava)
if kwargs.get("image", None) is not None:
kwargs["image"] = Utils.tensor2pil(kwargs["image"])
else:
kwargs["image"] = None
text = mz_llava.base_image_interrogator(kwargs)
conditionings = None
clip = kwargs.get("clip", None)
if clip is not None:
conditionings = Utils.a1111_clip_text_encode(clip, text, )
return {"ui": {"string": [Utils.to_debug_prompt(text),]}, "result": (text, conditionings)}
NODE_CLASS_MAPPINGS["MZ_BaseLLavaImageInterrogator"] = MZ_BaseLLavaImageInterrogator
NODE_DISPLAY_NAME_MAPPINGS[
"MZ_BaseLLavaImageInterrogator"] = f"{AUTHOR_NAME} - deprecated - ImageInterrogator(BaseLLava)"
mz_llava.py → v1/mz_llava.py
+5 -5
View File
@@ -1,10 +1,10 @@
import json
import os
from . import mz_prompt_utils
from . import mz_llama_cpp
from . import mz_prompts
from . import mz_llama_core_nodes
from . import mz_prompt_webserver
from .. import mz_prompt_utils
from .. import mz_llama_cpp
from .. import mz_prompts
from .. import mz_llama_core_nodes
from .. import mz_prompt_webserver
import importlib
web/prompt_mz.js
+2 -1
View File
@@ -90,6 +90,7 @@ const my_ui = {
case "MZ_LLavaImageInterrogator":
case "MZ_BaseLLavaImageInterrogator":
case "MZ_LLamaCPPCLIPTextEncode":
case "MZ_ImageInterrogatorCLIPTextEncode":
// Node Created
const onNodeCreated = nodeType.prototype.onNodeCreated;
nodeType.prototype.onNodeCreated = function () {
@@ -136,7 +137,7 @@ const my_ui = {
const onConfigure = nodeType.prototype.onConfigure;
nodeType.prototype.onConfigure = function (w) {
onConfigure?.apply(this, arguments);
// outSet.call(this, a?.string);
};