This commit is contained in:
wailovet
2024-06-02 14:03:48 +08:00
parent 0d5b6ef1f3
commit 371b37c46b
7 changed files with 246 additions and 94 deletions
+50 -5
View File
@@ -114,14 +114,13 @@ class MZ_LLamaCPPModelConfig_DownloaderSelect:
"optional": {
},
}
RETURN_TYPES = ("LLamaCPPModelConfig",)
RETURN_NAMES = ("llama_cpp_model_config",)
FUNCTION = "create"
CATEGORY = f"{CATEGORY_NAME}/others"
def create(self, **kwargs):
kwargs = kwargs.copy()
@@ -133,7 +132,9 @@ class MZ_LLamaCPPModelConfig_DownloaderSelect:
NODE_CLASS_MAPPINGS["MZ_LLamaCPPModelConfig_DownloaderSelect"] = MZ_LLamaCPPModelConfig_DownloaderSelect
NODE_DISPLAY_NAME_MAPPINGS["MZ_LLamaCPPModelConfig_DownloaderSelect"] = f"{AUTHOR_NAME} - ModelConfigDownloaderSelect(LLamaCPP)"
NODE_DISPLAY_NAME_MAPPINGS[
"MZ_LLamaCPPModelConfig_DownloaderSelect"] = f"{AUTHOR_NAME} - ModelConfigDownloaderSelect(LLamaCPP)"
class MZ_LLamaCPPCLIPTextEncode:
@classmethod
@@ -232,7 +233,7 @@ class MZ_CustomizeInstruct:
return {
"required": {
"system": ("STRING", {"multiline": True, "default": mz_prompts.Long_prompt}),
"instruct": ("STRING", {"multiline": True, "default": "Short: %text%"}),
"instruct": ("STRING", {"multiline": True, "default": ""}),
},
}
@@ -334,7 +335,7 @@ class MZ_OpenAIApiCLIPTextEncode:
def encode(self, **kwargs):
kwargs = kwargs.copy()
import mz_openaiapi
from . import mz_openaiapi
importlib.reload(mz_openaiapi)
if kwargs.get("api_key", "").endswith("******"):
@@ -451,6 +452,50 @@ NODE_CLASS_MAPPINGS["MZ_ImageInterrogatorModelConfig_ManualSelect"] = MZ_ImageIn
NODE_DISPLAY_NAME_MAPPINGS[
"MZ_ImageInterrogatorModelConfig_ManualSelect"] = f"{AUTHOR_NAME} - ModelConfigManualSelect(ImageInterrogator)"
class MZ_ImageInterrogatorModelConfig_DownloaderSelect:
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"model_name": ([
"ggml_llava1_5-7b-q4_k_m",
"ggml_bakllava-1-q4_k_m",
"llava_v1_6_mistral_7b_q5_k_m",
],),
"mmproj_model_name": ([
"auto",
"ggml_llava1_5-7b-mmproj-f16",
"ggml_bakllava-1-mmproj-f16",
"llava_v1_6_mistral_7b_mmproj_f16",
],),
},
"optional": {
},
}
RETURN_TYPES = ("ImageInterrogatorModelConfig",)
RETURN_NAMES = ("image_interrogator_model",)
FUNCTION = "create"
CATEGORY = f"{CATEGORY_NAME}/others"
def create(self, **kwargs):
kwargs = kwargs.copy()
model_name = kwargs.get("model_name")
mmproj_model_name = kwargs.get("mmproj_model_name", "auto")
return ({
"type": "DownloaderSelect",
"model_name": model_name,
"mmproj_model_name": mmproj_model_name,
},)
NODE_CLASS_MAPPINGS["MZ_ImageInterrogatorModelConfig_DownloaderSelect"] = MZ_ImageInterrogatorModelConfig_DownloaderSelect
NODE_DISPLAY_NAME_MAPPINGS[
"MZ_ImageInterrogatorModelConfig_DownloaderSelect"] = f"{AUTHOR_NAME} - ModelConfigDownloaderSelect(ImageInterrogator)"
from . import mz_gen_translate
mz_gen_translate.gen_translate(NODE_DISPLAY_NAME_MAPPINGS, NODE_CLASS_MAPPINGS)
+3 -1
View File
@@ -8,6 +8,8 @@ COMFY_PATH = Path(folder_paths.__file__).parent
ZH_Replace_Map = {
"mmproj_model_name": "mmproj模型名称",
"model_name": "模型名称",
"llama_cpp_model": "llama.cpp模型",
"mmproj_model": "mmproj模型",
"resolution": "分辨率",
@@ -22,6 +24,7 @@ ZH_Replace_Map = {
"keep_device": "模型常驻显存",
"llama_cpp_options": "llama.cpp可选配置",
"Options": "可选配置",
"LLamaCPPModelConfigManualSelect": "llama.cpp模型配置手动选择",
"LLamaCPP": "llama.cpp",
"CLIPTextEncode": "CLIP文本编码器",
"clip": "CLIP",
@@ -29,7 +32,6 @@ ZH_Replace_Map = {
"customize_instruct": "自定义指令",
"CustomizeInstruct": "自定义指令",
"deprecated": "已废弃",
"LLamaCPPModelConfigManualSelect": "llama.cpp模型配置手动选择",
"ModelConfigManualSelect": "手动模型选择器",
"ModelConfigDownloaderSelect": "预设模型下载器",
"captioner_config": "打标器配置",
+103 -70
View File
@@ -129,54 +129,35 @@ def llama_cpp_node_encode(args_dict):
mz_prompt_utils.Utils.print_log(f"question: {question}")
if schema is not None:
response_json = mz_llama_cpp.llama_cpp_simple_interrogator_to_json(
response_text = mz_llama_cpp.llama_cpp_simple_interrogator_to_json(
model_file=model_file,
system=system_prompt,
question=question,
schema=schema,
options=options,
)
mz_prompt_utils.Utils.print_log(f"response_json: {response_json}")
response = json.loads(response_json)
full_responses = []
response_json = json.loads(response_text)
mz_prompt_utils.Utils.print_log(
f"response_json: {json.dumps(response_json, indent=2)}")
if response["description"] != "":
full_responses.append(f"({response['description']})")
if response["long_prompt"] != "":
full_responses.append(f"({response['long_prompt']})")
if response["main_color_word"] != "":
full_responses.append(f"({response['main_color_word']})")
if response["camera_angle_word"] != "":
full_responses.append(f"({response['camera_angle_word']})")
responses = []
for key, value in response_json.items():
if type(value) == list:
# 去除开头.和空格
value = [v.strip().lstrip(".") for v in value]
# 去除空字符串
value = [v for v in value if v != ""]
if len(value) > 0:
responses.append(f"({', '.join(value)})")
response["style_words"] = [
x for x in response["style_words"] if x != ""]
if len(response["style_words"]) > 0:
full_responses.append(
f"({', '.join(response['style_words'])})")
else:
if value != "":
responses.append(f"({value})")
response["subject_words"] = [
x for x in response["subject_words"] if x != ""]
if len(response["subject_words"]) > 0:
full_responses.append(
f"({', '.join(response['subject_words'])})")
response["light_words"] = [
x for x in response["light_words"] if x != ""]
if len(response["light_words"]) > 0:
full_responses.append(
f"({', '.join(response['light_words'])})")
response["environment_words"] = [
x for x in response["environment_words"] if x != ""]
if len(response["environment_words"]) > 0:
full_responses.append(
f"({', '.join(response['environment_words'])})")
full_response = ", ".join(full_responses)
response = ", ".join(responses)
else:
full_response = mz_llama_cpp.llama_cpp_simple_interrogator(
response = mz_llama_cpp.llama_cpp_simple_interrogator(
model_file=model_file,
system=system_prompt,
question=question,
@@ -184,42 +165,42 @@ def llama_cpp_node_encode(args_dict):
)
start_str = customize_instruct.get("start_str", "")
if start_str != "" and full_response.find(start_str) != -1:
full_response_list = full_response.split(start_str)
if start_str != "" and response.find(start_str) != -1:
full_response_list = response.split(start_str)
# 删除第一个元素
full_response_list.pop(0)
full_response = start_str.join(full_response_list)
response = start_str.join(full_response_list)
end_str = customize_instruct.get("end_str", "")
if end_str != "" and full_response.find(end_str) != -1:
full_response_list = full_response.split(end_str)
if end_str != "" and response.find(end_str) != -1:
full_response_list = response.split(end_str)
# 删除最后一个元素
full_response_list.pop()
full_response = end_str.join(full_response_list)
response = end_str.join(full_response_list)
if keep_device is False:
mz_llama_cpp.freed_gpu_memory(model_file=model_file)
# 去除换行
while full_response.find("\n") != -1:
full_response = full_response.replace("\n", " ")
while response.find("\n") != -1:
response = response.replace("\n", " ")
# 句号换成逗号
while full_response.find(".") != -1:
full_response = full_response.replace(".", ",")
while response.find(".") != -1:
response = response.replace(".", ",")
# 去除多余逗号
while full_response.find(",,") != -1:
full_response = full_response.replace(",,", ",")
while full_response.find(", ,") != -1:
full_response = full_response.replace(", ,", ",")
while response.find(",,") != -1:
response = response.replace(",,", ",")
while response.find(", ,") != -1:
response = response.replace(", ,", ",")
full_response = mz_prompt_utils.Utils.prompt_zh_to_en(full_response)
response = mz_prompt_utils.Utils.prompt_zh_to_en(response)
style_presets_prompt_text = style_presets_prompt.get(style_presets, "")
if style_presets_prompt_text != "":
full_response = f"{style_presets_prompt_text}, {full_response}"
response = f"{style_presets_prompt_text}, {response}"
except Exception as e:
mz_llama_cpp.freed_gpu_memory(model_file=model_file)
@@ -229,9 +210,9 @@ def llama_cpp_node_encode(args_dict):
clip = args_dict.get("clip", None)
if clip is not None:
conditionings = mz_prompt_utils.Utils.a1111_clip_text_encode(
clip, full_response, )
clip, response, )
return {"ui": {"string": [full_response,]}, "result": (full_response, conditionings)}
return {"ui": {"string": [mz_prompt_utils.Utils.to_debug_prompt(response),]}, "result": (response, conditionings)}
def image_interrogator_captioner(args_dict):
@@ -331,16 +312,15 @@ def image_interrogator_captioner(args_dict):
return result
image_interrogator_auto_mmproj_model = {
"7ac9c2f7b8d76cc7f3118cdf0953ebab7a7a9b12bad5dbe237219d2ab61765ea": "ggml_llava1_5-7b-mmproj-f16",
"c93de1376be9b6977cc94d252a3d165d6059e07b528de0fa762534d9599b27d6": "ggml_bakllava-1-mmproj-f16",
"b1d37fc65ecb80aa8f1ce185bf4d7605bc3c5cc5bcc77a160c3a1b0377631112": "llava_v1_6_mistral_7b_q5_k_m",
}
def image_interrogator_node_encode(args_dict):
image_interrogator_model = args_dict.get("image_interrogator_model", {})
llama_cpp_model = image_interrogator_model.get("llama_cpp_model", "auto")
mmproj_model = image_interrogator_model.get("mmproj_model", "auto")
def image_interrogator_node_encode(args_dict):
importlib.reload(mz_prompts)
captioner_config = args_dict.get("captioner_config", None)
if captioner_config is not None:
@@ -348,7 +328,56 @@ def image_interrogator_node_encode(args_dict):
raise Exception(
"图片批量反推任务已完成 ; Image batch reverse push task completed")
model_config = args_dict.get("image_interrogator_model", {})
llama_cpp_model = model_config.get("llama_cpp_model", "auto")
mmproj_model = model_config.get("mmproj_model", "auto")
select_model_type = model_config.get("type", "ManualSelect")
if select_model_type == "ManualSelect":
llama_cpp_model = model_config.get("model_path", "auto")
if llama_cpp_model == "auto":
llama_cpp_model = mz_prompt_utils.Utils.get_auto_model_fullpath(
"ggml_llava1_5-7b-q4_k_m")
if mmproj_model == "auto":
llama_cpp_model_sha256 = mz_prompt_utils.Utils.file_sha256(
llama_cpp_model)
mmproj_model_name = image_interrogator_auto_mmproj_model.get(
llama_cpp_model_sha256, None)
if mmproj_model_name is None:
mz_prompt_utils.Utils.print_log(
"llama_cpp_model_sha256: ", llama_cpp_model_sha256)
raise Exception(
"未能自动找到对应的mmproj文件 ; Failed to automatically find the corresponding mmproj file.")
mmproj_model = mz_prompt_utils.Utils.get_auto_model_fullpath(
mmproj_model_name)
elif select_model_type == "DownloaderSelect":
model_name = model_config.get("model_name")
llama_cpp_model = mz_prompt_utils.Utils.get_auto_model_fullpath(
model_name)
mmproj_model = model_config.get("mmproj_model_name", "auto")
if mmproj_model == "auto":
llama_cpp_model_sha256 = mz_prompt_utils.Utils.file_sha256(
llama_cpp_model)
mz_prompt_utils.Utils.print_log(
"llama_cpp_model_sha256: ", llama_cpp_model_sha256)
mmproj_model_name = image_interrogator_auto_mmproj_model.get(
llama_cpp_model_sha256, None)
if mmproj_model_name is None:
raise Exception(
"未能自动找到对应的mmproj文件 ; Failed to automatically find the corresponding mmproj file")
mmproj_model = mz_prompt_utils.Utils.get_auto_model_fullpath(
mmproj_model_name)
else:
raise Exception("Unknown select_model_type")
image = args_dict.get("image", None)
image = mz_prompt_utils.Utils.tensor2pil(image)
resolution = args_dict.get("resolution", 512)
keep_device = args_dict.get("keep_device", False)
seed = args_dict.get("seed", -1)
@@ -359,23 +388,27 @@ def image_interrogator_node_encode(args_dict):
customize_instruct = args_dict.get("customize_instruct", None)
if customize_instruct is None:
system_text = mz_prompts.GPT4VImageCaptioner_System
question_text = mz_prompts.GPT4VImageCaptioner_Prompt
system_prompt = mz_prompts.M_ImageCaptioner_System
question = mz_prompts.M_ImageCaptioner_Prompt
else:
system_prompt = customize_instruct.get("system", "")
question = customize_instruct.get("instruct", "%text%")
system_prompt = system_prompt.replace("%text%", "")
question = question.replace("%text%", "")
question = customize_instruct.get("instruct", "")
response = mz_llama_cpp.llava_cpp_simple_interrogator(
model_file=llama_cpp_model,
mmproj_file=mmproj_model,
image=image,
options=options,
system=system_text,
question=question_text,
system=system_prompt,
question=question,
)
if response is not None:
response = response.strip()
if customize_instruct is None:
# 去除开头的In the image,
start_text = "In the image, "
if response.startswith(start_text):
response = response[len(start_text):]
if keep_device is False:
mz_llama_cpp.freed_gpu_memory(model_file=llama_cpp_model)
@@ -388,4 +421,4 @@ def image_interrogator_node_encode(args_dict):
conditionings = mz_prompt_utils.Utils.a1111_clip_text_encode(
clip, response, )
return {"ui": {"string": [response,]}, "result": (response, conditionings)}
return {"ui": {"string": [mz_prompt_utils.Utils.to_debug_prompt(response),]}, "result": (response, conditionings)}
+40 -5
View File
@@ -13,9 +13,13 @@ except ImportError:
def check_llama_cpp_requirements():
last_version = "0.2.63"
min_version = "0.2.63"
last_version = "0.2.76"
try:
from llama_cpp import Llama
import llama_cpp
if llama_cpp.__version__ < min_version:
raise ImportError("llama_cpp version is too low. (llama_cpp版本过低)")
except ImportError:
py_version = ""
if sys.version_info.major == 3:
@@ -65,9 +69,39 @@ def check_llama_cpp_requirements():
wheel_url = f"https://github.com/abetlen/llama-cpp-python/releases/download/v{last_version}-{cuda_version}/{wheel_name}"
print(f"pip install {wheel_url}")
ret = subprocess.run([
sys.executable, "-m",
"pip", "install", wheel_url], check=True)
modelscope_url = f"https://www.modelscope.cn/api/v1/models/wailovet/MinusZoneAIModels/repo?Revision=master&FilePath=llama-cpp-python-win%2F{cuda_version}%2F{wheel_name}"
if mz_prompt_utils.Utils.testDownloadSpeed(wheel_url):
ret = subprocess.run([
sys.executable, "-m",
"pip", "install", wheel_url], check=True)
elif mz_prompt_utils.Utils.testDownloadSpeed(modelscope_url):
import tempfile
whl_download_file = os.path.join(
tempfile.gettempdir(), wheel_name)
mz_prompt_utils.Utils.download_file(
modelscope_url, whl_download_file)
print(f"pip install {whl_download_file}")
ret = subprocess.run([
sys.executable, "-m",
"pip", "install", whl_download_file], check=True)
else:
# 兜底方案
modelscope_url = f"https://www.modelscope.cn/api/v1/models/wailovet/MinusZoneAIModels/repo?Revision=master&FilePath=llama-cpp-python-win%2Fcu121%2Fllama_cpp_python-0.2.76-cp310-cp310-win_amd64.whl"
if py_version == "310" and system_name == "win_amd64" and mz_prompt_utils.Utils.testDownloadSpeed(modelscope_url):
import tempfile
whl_download_file = os.path.join(
tempfile.gettempdir(), wheel_name)
mz_prompt_utils.Utils.download_file(
modelscope_url, whl_download_file)
print(f"pip install {whl_download_file}")
ret = subprocess.run([
sys.executable, "-m",
"pip", "install", whl_download_file], check=True)
else:
ret = subprocess.run([
sys.executable, "-m",
"pip", "install", wheel_url], check=True)
if ret.returncode != 0:
raise ValueError("Failed to install llama_cpp. (安装llama_cpp失败)")
@@ -357,7 +391,8 @@ def llava_cpp_simple_interrogator(
check_llama_cpp_requirements()
from llama_cpp.llama_chat_format import Llava15ChatHandler
if mmproj_file is not None:
mz_prompt_utils.Utils.print_log(f"llava_cpp_simple_interrogator mmproj_file: {mmproj_file}")
mz_prompt_utils.Utils.print_log(
f"llava_cpp_simple_interrogator mmproj_file: {mmproj_file}")
chat_handler = Llava15ChatHandler(clip_model_path=mmproj_file)
return llava_cpp_messages(model_file, chat_handler, [
+11 -10
View File
@@ -5,6 +5,7 @@ import subprocess
from . import mz_prompt_utils
from . import mz_llama_cpp
from . import mz_llama_core_nodes
from . import mz_prompts
@@ -63,16 +64,16 @@ def query_beautify_prompt_text(args_dict):
schema = None
if customize_instruct is None:
schema = mz_llama_cpp.get_schema_obj(
schema = mz_llama_core_nodes.get_schema_obj(
keys_type={
"description": mz_llama_cpp.get_schema_base_type("string"),
"long_prompt": mz_llama_cpp.get_schema_base_type("string"),
"main_color_word": mz_llama_cpp.get_schema_base_type("string"),
"camera_angle_word": mz_llama_cpp.get_schema_base_type("string"),
"style_words": mz_llama_cpp.get_schema_array("string"),
"subject_words": mz_llama_cpp.get_schema_array("string"),
"light_words": mz_llama_cpp.get_schema_array("string"),
"environment_words": mz_llama_cpp.get_schema_array("string"),
"description": mz_llama_core_nodes.get_schema_base_type("string"),
"long_prompt": mz_llama_core_nodes.get_schema_base_type("string"),
"main_color_word": mz_llama_core_nodes.get_schema_base_type("string"),
"camera_angle_word": mz_llama_core_nodes.get_schema_base_type("string"),
"style_words": mz_llama_core_nodes.get_schema_array("string"),
"subject_words": mz_llama_core_nodes.get_schema_array("string"),
"light_words": mz_llama_core_nodes.get_schema_array("string"),
"environment_words": mz_llama_core_nodes.get_schema_array("string"),
},
required=[
"description",
@@ -184,7 +185,7 @@ def query_beautify_prompt_text(args_dict):
full_response = full_response.replace(",,", ",")
while full_response.find(", ,") != -1:
full_response = full_response.replace(", ,", ",")
style_presets_prompt_text = mz_llama_cpp.style_presets_prompt.get(
style_presets_prompt_text = mz_llama_core_nodes.style_presets_prompt.get(
style_presets, "")
if style_presets_prompt_text != "":
full_response = f"{style_presets_prompt_text}, {full_response}"
+26 -1
View File
@@ -675,6 +675,12 @@ class Utils:
return Utils.file_hash(file_path, hashlib.sha256)
def get_auto_model_fullpath(model_name):
fullpath = Utils.cache_get(f"get_auto_model_fullpath_{model_name}")
Utils.print_log(f"get_auto_model_fullpath_{model_name} => {fullpath}")
if fullpath is not None:
if os.path.exists(fullpath):
return fullpath
find_paths = []
target_sha256 = ""
file_path = ""
@@ -703,12 +709,31 @@ class Utils:
for root, dirs, files in os.walk(find_fullpath):
for file in files:
if target_sha256 == Utils.file_sha256(os.path.join(root, file)):
Utils.cache_set(f"get_auto_model_fullpath_{model_name}", os.path.join(root, file))
return os.path.join(root, file)
else:
Utils.print_log(
f"Model {os.path.join(root, file)} file hash not match, {target_sha256} != {Utils.file_sha256(os.path.join(root, file))}")
return Utils.download_model({"url": download_url, "output": file_path})
result = Utils.download_model({"url": download_url, "output": file_path})
Utils.cache_set(f"get_auto_model_fullpath_{model_name}", result)
return result
def testDownloadSpeed(url):
try:
print(f"Testing download speed for {url}")
start = time.time()
# 下载2M数据
headers = {"Range": "bytes=0-2097151"}
_ = requests.get(url, headers=headers, timeout=5)
end = time.time()
print(
f"Download speed: {round(5.00 / (float(end) - float(start)) / 1024, 2)} KB/s")
return float(end) - float(start) < 4
except Exception as e:
print(f"Test download speed failed: {e}")
return False
MODEL_ZOO = [
+13 -2
View File
@@ -26,7 +26,7 @@ As an AI image tagging expert, please provide precise tags for these images to e
# 来自https://github.com/jiayev/GPT4V-Image-Captioner
GPT4VImageCaptioner_Prompt = """
Employ succinct keywords or phrases, steering clear of elaborate sentences and extraneous conjunctions.
Employ succinct keywords or phrases or sentence, steering clear of elaborate sentences and extraneous conjunctions.
Prioritize the tags by relevance.
Your tags should capture key elements such as the main subject, setting, artistic style, composition, image quality, color tone, filter, and camera specifications, and any other tags crucial for the image.
When tagging photos of people, include specific details like gender, nationality, attire, actions, pose, expressions, accessories, makeup, composition type, age, etc.
@@ -35,4 +35,15 @@ Recognize and tag any celebrities, well-known landmark or IPs if clearly feature
Your tags should be accurate, non-duplicative, and within a 20-75 word count range.
These tags will use for image re-creation, so the closer the resemblance to the original image, the better the tag quality.
Tags should be comma-separated.
"""
"""
M_ImageCaptioner_System = """
Long prompt version should consist of 3 to 5 sentences. Long prompt version must sepcify the color, shape, texture or spatial relation of the included objects. DO NOT generate sentences that describe any atmosphere!!!
"""
M_ImageCaptioner_Prompt = """
Describe this image in detail please.
The language of reply is English only!!!
Starts with "In the image,"
"""