diff --git a/README.md b/README.md index 793e7b2..863ee76 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,10 @@ https://github.com/shadowcz007/comfyui-mixlab-nodes/assets/12645064/e7e77f90-e43 > PromptImage & PromptSimplification,Assist in simplifying prompt words, comparing images and prompt word nodes. +> ChinesePrompt && PromptGenerate，中文prompt节点，直接用中文书写你的prompt + +![](./assets/ChinesePrompt_workflow.svg) + ### Layers > A new layer class node has been added, allowing you to separate the image into layers. After merging the images, you can input the controlnet for further processing. @@ -159,23 +163,16 @@ An improvement has been made to directly redirect to GitHub to search for missin ![node-not-found](./assets/node-not-found.png) -### Update -v0.8.0 🚀🚗🚚🏃‍ LaMaInpainting -- 新增 LaMaInpainting -- 优化color节点的输出 -- 修复高清显示屏上定位节点不准的情况 - -- Add LaMaInpainting -- Optimize the output of the color node -- Fix the issue of inaccurate positioning node on high-definition display screens - - ### Models [Download CLIPSeg](https://huggingface.co/CIDAS/clipseg-rd64-refined/tree/main), move to : models/clipseg [Download lama](https://github.com/enesmsahin/simple-lama-inpainting/releases/download/v0.1.0/big-lama.pt), move to : models/lama -[Download Salesforce\blip-image-captioning-base](https://huggingface.co/Salesforce/blip-image-captioning-base), move to : models/clip_interrogator/Salesforce/blip-image-captioning-base +[Download Salesforce/blip-image-captioning-base](https://huggingface.co/Salesforce/blip-image-captioning-base), move to : models/clip_interrogator/Salesforce/blip-image-captioning-base + +[Download succinctly/text2image-prompt-generator](https://huggingface.co/succinctly/text2image-prompt-generator/tree/main),move to:text_generator/text2image-prompt-generator + +[Download Helsinki-NLP/opus-mt-zh-en](https://huggingface.co/Helsinki-NLP/opus-mt-zh-en/tree/main),move to:prompt_generator/opus-mt-zh-en ## Installation @@ -216,9 +213,6 @@ pip3 install -r requirements.txt #### discussions: [discussions](https://github.com/shadowcz007/comfyui-mixlab-nodes/discussions) -### TODO: -- 音频播放节点：带可视化、支持多音轨、可配置音轨音量 -- vector https://github.com/GeorgLegato/stable-diffusion-webui-vectorstudio diff --git a/__init__.py b/__init__.py index e7d027b..6ab92f7 100644 --- a/__init__.py +++ b/__init__.py @@ -620,7 +620,8 @@ NODE_DISPLAY_NAME_MAPPINGS = { "DynamicDelayProcessor":"DynamicDelayByText ♾️Mixlab", "LaMaInpainting":"LaMaInpainting ♾️Mixlab", "PromptSlide":"PromptSlide ♾️Mixlab", - + "PromptGenerate_Mix":"PromptGenerate ♾️Mixlab", + "ChinesePrompt_Mix":"ChinesePrompt ♾️Mixlab", "GamePal":"GamePal ♾️Mixlab" } @@ -645,6 +646,17 @@ try: NODE_CLASS_MAPPINGS['ClipInterrogator']=ClipInterrogator except: print('ClipInterrogator.available',False) + +try: + from .nodes.TextGenerateNode import PromptGenerate,ChinesePrompt + print('PromptGenerate.available',PromptGenerate.available) + if PromptGenerate.available: + NODE_CLASS_MAPPINGS['PromptGenerate_Mix']=PromptGenerate + print('ChinesePrompt.available',ChinesePrompt.available) + if ChinesePrompt.available: + NODE_CLASS_MAPPINGS['ChinesePrompt_Mix']=ChinesePrompt +except: + print('TextGenerateNode.available',False) print('\033[93m -------------- \033[0m') \ No newline at end of file diff --git a/assets/ChinesePrompt_workflow.svg b/assets/ChinesePrompt_workflow.svg new file mode 100644 index 0000000..891648e --- /dev/null +++ b/assets/ChinesePrompt_workflow.svg @@ -0,0 +1 @@ +T: 0.00sI: 0N: 9 [9]V: 43FPS:312.50ShowTextForGPT ♾️Mixlab#4 comfyui-mixlab-nodestextSTRINGChinese dragons, red lamps, radiant radiances, warm, snowy days. Artistic elements, --ar 21:9 ChinesePrompt ♾️Mixlab#3 prompt中国龙，红灯笼，过年热闹的氛围灯光，暖色，雪天， seed913653control_after_generaterandomizePreview Image#11 🦊imagesVAE Decode#10 🦊samplesvaeIMAGEEmpty Latent Image#7 🦊LATENTwidth512height512batch_size4KSampler#5 🦊modelpositivenegativelatent_imageLATENTseed517333504418991control_after_generaterandomizesteps20cfg8.0sampler_nameeulerschedulernormaldenoise1.00CLIP Text Encode (Prompt)#8 🦊cliptextCONDITIONINGCLIP Text Encode (Prompt)#9 🦊clipCONDITIONINGtext, watermark Load Checkpoint#6 🦊MODELCLIPVAEckpt_namecardosAnime_v20.safetensors{"last_node_id":11,"last_link_id":12,"nodes":[{"id":4,"type":"ShowTextForGPT","pos":[888,396],"size":[335.62449456837567,127.43873884973459],"flags":{},"order":3,"mode":0,"inputs":[{"name":"text","type":"STRING","link":2,"widget":{"name":"text"}}],"outputs":[{"name":"STRING","type":"STRING","links":null,"shape":6}],"properties":{"Node name for S&R":"ShowTextForGPT"},"widgets_values":["Chinese dragons, red lamps, radiant radiances, warm, snowy days. Artistic elements, --ar 21:9"]},{"id":3,"type":"ChinesePrompt_Mix","pos":[441,403],"size":{"0":400,"1":200},"flags":{},"order":1,"mode":0,"outputs":[{"name":"prompt","type":"STRING","links":[2,12],"shape":6,"slot_index":0}],"properties":{"Node name for S&R":"ChinesePrompt_Mix"},"widgets_values":["中国龙，红灯笼，过年热闹的氛围灯光，暖色，雪天，",913653,"randomize"]},{"id":11,"type":"PreviewImage","pos":[1312,347],"size":[303.95421253551126,283.9511385830965],"flags":{},"order":8,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":11}],"properties":{"Node name for S&R":"PreviewImage"}},{"id":10,"type":"VAEDecode","pos":[1328,674],"size":{"0":210,"1":46},"flags":{"collapsed":false},"order":7,"mode":0,"inputs":[{"name":"samples","type":"LATENT","link":9},{"name":"vae","type":"VAE","link":10}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[11],"slot_index":0}],"properties":{"Node name for S&R":"VAEDecode"}},{"id":7,"type":"EmptyLatentImage","pos":[1300,1090],"size":{"0":315,"1":106},"flags":{},"order":0,"mode":0,"outputs":[{"name":"LATENT","type":"LATENT","links":[6],"slot_index":0}],"properties":{"Node name for S&R":"EmptyLatentImage"},"widgets_values":[512,512,4]},{"id":5,"type":"KSampler","pos":[1310,780],"size":{"0":315,"1":262},"flags":{},"order":6,"mode":0,"inputs":[{"name":"model","type":"MODEL","link":3},{"name":"positive","type":"CONDITIONING","link":4},{"name":"negative","type":"CONDITIONING","link":5},{"name":"latent_image","type":"LATENT","link":6}],"outputs":[{"name":"LATENT","type":"LATENT","links":[9],"slot_index":0}],"properties":{"Node name for S&R":"KSampler"},"widgets_values":[517333504418991,"randomize",20,8,"euler","normal",1]},{"id":8,"type":"CLIPTextEncode","pos":[817,766],"size":[422.84503173828125,164.31304931640625],"flags":{},"order":4,"mode":0,"inputs":[{"name":"clip","type":"CLIP","link":7},{"name":"text","type":"STRING","link":12,"widget":{"name":"text"}}],"outputs":[{"name":"CONDITIONING","type":"CONDITIONING","links":[4],"slot_index":0}],"properties":{"Node name for S&R":"CLIPTextEncode"},"widgets_values":["beautiful scenery nature glass bottle landscape, , purple galaxy bottle,"]},{"id":9,"type":"CLIPTextEncode","pos":[808,993],"size":{"0":425.27801513671875,"1":180.6060791015625},"flags":{},"order":5,"mode":0,"inputs":[{"name":"clip","type":"CLIP","link":8}],"outputs":[{"name":"CONDITIONING","type":"CONDITIONING","links":[5],"slot_index":0}],"properties":{"Node name for S&R":"CLIPTextEncode"},"widgets_values":["text, watermark"]},{"id":6,"type":"CheckpointLoaderSimple","pos":[437,782],"size":{"0":315,"1":98},"flags":{},"order":2,"mode":0,"outputs":[{"name":"MODEL","type":"MODEL","links":[3],"slot_index":0},{"name":"CLIP","type":"CLIP","links":[7,8],"slot_index":1},{"name":"VAE","type":"VAE","links":[10],"slot_index":2}],"properties":{"Node name for S&R":"CheckpointLoaderSimple"},"widgets_values":["cardosAnime_v20.safetensors"]}],"links":[[2,3,0,4,0,"STRING"],[3,6,0,5,0,"MODEL"],[4,8,0,5,1,"CONDITIONING"],[5,9,0,5,2,"CONDITIONING"],[6,7,0,5,3,"LATENT"],[7,6,1,8,0,"CLIP"],[8,6,1,9,0,"CLIP"],[9,5,0,10,0,"LATENT"],[10,6,2,10,1,"VAE"],[11,10,0,11,0,"IMAGE"],[12,3,0,8,1,"STRING"]],"groups":[{"title":"中文提示词","bounding":[431,329,860,284],"color":"#3f789e","font_size":24,"locked":false}],"config":{},"extra":{},"version":0.4} \ No newline at end of file diff --git a/data/extension-node-map.json b/data/extension-node-map.json index 93f57cb..5996e50 100644 --- a/data/extension-node-map.json +++ b/data/extension-node-map.json @@ -4761,6 +4761,8 @@ ], "https://github.com/shadowcz007/comfyui-mixlab-nodes": [ [ + "PromptGenerate_Mix", + "ChinesePrompt_Mix", "3DImage", "AppInfo", "IntNumber", diff --git a/nodes/ClipInterrogator.py b/nodes/ClipInterrogator.py index 2fe92cd..b753cee 100644 --- a/nodes/ClipInterrogator.py +++ b/nodes/ClipInterrogator.py @@ -68,11 +68,11 @@ def load_caption_model(model_path,config,t='blip-base'): caption_model_path=os.path.join(folder_paths.models_dir, "clip_interrogator/Salesforce/blip-image-captioning-base") if not os.path.exists(caption_model_path): print(f"## clip_interrogator_model not found: {caption_model_path}, pls download from https://huggingface.co/Salesforce/blip-image-captioning-base") + caption_model_path='Salesforce/blip-image-captioning-base' cache_path=os.path.join(folder_paths.models_dir, "clip_interrogator") - # Tensor to PIL def tensor2pil(image): return Image.fromarray(np.clip(255. * image.cpu().numpy().squeeze(), 0, 255).astype(np.uint8)) diff --git a/nodes/TextGenerateNode.py b/nodes/TextGenerateNode.py new file mode 100644 index 0000000..bd4b088 --- /dev/null +++ b/nodes/TextGenerateNode.py @@ -0,0 +1,215 @@ +from transformers import pipeline, set_seed,AutoTokenizer, AutoModelForSeq2SeqLM +import random +import re + +import os,sys +import folder_paths + +# from PIL import Image +# import importlib.util + +import comfy.utils +# import numpy as np +import torch +import random + +global _available +_available=True + + +text_generator_model_path=os.path.join(folder_paths.models_dir, "prompt_generator/text2image-prompt-generator") +if not os.path.exists(text_generator_model_path): + print(f"## text_generator_model not found: {text_generator_model_path}, pls download from https://huggingface.co/succinctly/text2image-prompt-generator/tree/main") + text_generator_model_path='succinctly/text2image-prompt-generator' + +zh_en_model_path=os.path.join(folder_paths.models_dir, "prompt_generator/opus-mt-zh-en") +if not os.path.exists(zh_en_model_path): + print(f"## zh_en_model not found: {zh_en_model_path}, pls download from https://huggingface.co/Helsinki-NLP/opus-mt-zh-en/tree/main") + zh_en_model_path='Helsinki-NLP/opus-mt-zh-en' + + + +def translate(zh_en_tokenizer,zh_en_model,texts): + with torch.no_grad(): + encoded = zh_en_tokenizer(texts, return_tensors="pt") + encoded.to(zh_en_model.device) + sequences = zh_en_model.generate(**encoded) + return zh_en_tokenizer.batch_decode(sequences, skip_special_tokens=True) + +# input = "青春不能回头，所以青春没有终点。 ——《火影忍者》" +# print(input, translate(input)) + + + +def text_generate(text_pipe,input,seed=None): + + if seed==None: + seed = random.randint(100, 1000000) + + set_seed(seed) + + for count in range(6): + sequences = text_pipe(input, max_length=random.randint(60, 90), num_return_sequences=8) + list = [] + for sequence in sequences: + line = sequence['generated_text'].strip() + if line != input and len(line) > (len(input) + 4) and line.endswith((":", "-", "—")) is False: + list.append(line) + + result = "\n".join(list) + result = re.sub('[^ ]+\.[^ ]+','', result) + result = result.replace("<", "").replace(">", "") + if result != "": + return result + if count == 5: + return result + +# input = "Youth can't turn back, so there's no end to youth." +# print(input, text_generate(input)) + + +class ChinesePrompt: + + global _available + available=_available + + @classmethod + def INPUT_TYPES(s): + return {"required": { + "text": ("STRING",{"multiline": True,"default": "", "dynamicPrompts": False}), + + }, + + "optional":{ + "seed":("INT", {"default": 100, "min": 100, "max": 1000000}), + }, + + } + + RETURN_TYPES = ("STRING",) + RETURN_NAMES = ("prompt",) + + FUNCTION = "run" + + CATEGORY = "♾️Mixlab/Prompt" + OUTPUT_NODE = True + INPUT_IS_LIST = True + OUTPUT_IS_LIST = (True,) + + global text_pipe,zh_en_model,zh_en_tokenizer + + text_pipe= None + zh_en_model=None + zh_en_tokenizer=None + + def run(self,text,seed): + global text_pipe,zh_en_model,zh_en_tokenizer + + seed=seed[0] + + # 进度条 + pbar = comfy.utils.ProgressBar(len(text)+1) + + if zh_en_model==None: + zh_en_model = AutoModelForSeq2SeqLM.from_pretrained(zh_en_model_path).eval() + zh_en_tokenizer = AutoTokenizer.from_pretrained(zh_en_model_path) + + zh_en_model.to("cuda" if torch.cuda.is_available() else "cpu") + # zh_en_tokenizer.to("cuda" if torch.cuda.is_available() else "cpu") + + text_pipe=pipeline('text-generation', model=text_generator_model_path,device="cuda" if torch.cuda.is_available() else "cpu") + + # text_pipe.model.to("cuda" if torch.cuda.is_available() else "cpu") + + prompt_result=[] + + # print('zh_en_model device',zh_en_model.device,text_pipe.model.device,torch.cuda.current_device() ) + en_text=translate(zh_en_tokenizer,zh_en_model,text) + zh_en_model.to('cpu') + # en_text.to("cuda" if torch.cuda.is_available() else "cpu") + + pbar.update(1) + for t in en_text: + prompt =text_generate(text_pipe,t,seed) + # 多条，还是单条 + lines = prompt.split("\n") + longest_line = max(lines, key=len) + # print(longest_line) + prompt_result.append(longest_line) + pbar.update(1) + + text_pipe.model.to('cpu') + + + + return { + "ui":{ + "prompt": prompt_result + }, + "result": (prompt_result,)} + + +class PromptGenerate: + + global _available + available=_available + + @classmethod + def INPUT_TYPES(s): + return {"required": { + "text": ("STRING",{"multiline": True,"default": "", "dynamicPrompts": False}), + }, + + "optional":{ + "multiple": (["off","on"],), + "seed":("INT", {"default": 100, "min": 100, "max": 1000000}), + }, + + } + + RETURN_TYPES = ("STRING",) + RETURN_NAMES = ("prompt",) + + FUNCTION = "run" + + CATEGORY = "♾️Mixlab/Prompt" + OUTPUT_NODE = True + INPUT_IS_LIST = True + OUTPUT_IS_LIST = (True,) + + global text_pipe + + text_pipe= None + # + + def run(self,text,multiple,seed): + global text_pipe + + seed=seed[0] + + multiple=multiple[0] + + # 进度条 + pbar = comfy.utils.ProgressBar(len(text)) + + text_pipe=pipeline('text-generation', model=text_generator_model_path,device="cuda" if torch.cuda.is_available() else "cpu") + + prompt_result=[] + + for t in text: + prompt =text_generate(text_pipe,t,seed) + prompt = prompt.split("\n") + if multiple=='off': + prompt = [max(prompt, key=len)] + + for p in prompt: + prompt_result.append(p) + pbar.update(1) + + text_pipe.model.to('cpu') + + return { + "ui":{ + "prompt": prompt_result + }, + "result": (prompt_result,)} diff --git a/web/javascript/checkVersion_mixlab.js b/web/javascript/checkVersion_mixlab.js index 3391a0b..954e217 100644 --- a/web/javascript/checkVersion_mixlab.js +++ b/web/javascript/checkVersion_mixlab.js @@ -3,7 +3,7 @@ import { app } from '../../../scripts/app.js' const repoOwner = 'shadowcz007' // 替换为仓库的所有者 const repoName = 'comfyui-mixlab-nodes' // 替换为仓库的名称 -const version = 'v0.11.4' +const version = 'v0.12.0' fetch(`https://api.github.com/repos/${repoOwner}/${repoName}/releases/latest`) .then(response => response.json())