diff --git a/README.md b/README.md index 8d85277..590153b 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,9 @@ https://github.com/shadowcz007/comfyui-mixlab-nodes/assets/12645064/e7e77f90-e43 ![randomPrompt](./assets/randomPrompt.png) +> ClipInterrogator + +[add clip-interrogator](https://github.com/pharmapsychotic/clip-interrogator) ### Layers > A new layer class node has been added, allowing you to separate the image into layers. After merging the images, you can input the controlnet for further processing. @@ -167,9 +170,7 @@ v0.8.0 🚀🚗🚚🏃‍ LaMaInpainting [Download lama](https://github.com/enesmsahin/simple-lama-inpainting/releases/download/v0.1.0/big-lama.pt), move to : models/lama - - +[Download Salesforce\blip-image-captioning-base](https://huggingface.co/Salesforce/blip-image-captioning-base), move to : models/clip_interrogator/Salesforce/blip-image-captioning-base ## Installation diff --git a/__init__.py b/__init__.py index 9957e5b..eac42d4 100644 --- a/__init__.py +++ b/__init__.py @@ -512,6 +512,7 @@ from .nodes.ChatGPT import ChatGPTNode,ShowTextForGPT,CharacterInText from .nodes.Audio import GamePal,SpeechRecognition,SpeechSynthesis from .nodes.Utils import AppInfo,IntNumber,FloatSlider,TextInput,ColorInput,FontInput,TextToNumber,DynamicDelayProcessor,LimitNumber,SwitchByIndex,GetImageSize_,MultiplicationNode from .nodes.Lama import LaMaInpainting +from .nodes.ClipInterrogator import ClipInterrogator # 要导出的所有节点及其名称的字典 # 注意：名称应全局唯一 @@ -519,6 +520,7 @@ NODE_CLASS_MAPPINGS = { "AppInfo":AppInfo, "RandomPrompt":RandomPrompt, "PromptSlide":PromptSlide, + "ClipInterrogator":ClipInterrogator, "NoiseImage":NoiseImage, "GradientImage":GradientImage, "TransparentImage":TransparentImage, diff --git a/data/extension-node-map.json b/data/extension-node-map.json index e7aab3a..7b594ec 100644 --- a/data/extension-node-map.json +++ b/data/extension-node-map.json @@ -4786,6 +4786,7 @@ "NewLayer", "RandomPrompt", "PromptSlide", + "ClipInterrogator", "ScreenShare", "ShowLayer", "ShowTextForGPT", diff --git a/nodes/ClipInterrogator.py b/nodes/ClipInterrogator.py new file mode 100644 index 0000000..1524984 --- /dev/null +++ b/nodes/ClipInterrogator.py @@ -0,0 +1,158 @@ +import os +import folder_paths + +from PIL import Image +import comfy.utils +import numpy as np +import json +import torch + +from transformers import AutoProcessor, BlipForConditionalGeneration + +from clip_interrogator import Config, Interrogator + +def load_caption_model(model_path,config,t='blip-base'): + dtype=torch.float16 if config.device == 'cuda' else torch.float32 + caption_model = BlipForConditionalGeneration.from_pretrained(model_path, torch_dtype=dtype) + + caption_processor = AutoProcessor.from_pretrained(model_path) + + caption_model.eval() + if not config.caption_offload: + caption_model = caption_model.to(config.device) + + return (caption_model,caption_processor) + + + +caption_model_path=os.path.join(folder_paths.models_dir, "clip_interrogator/Salesforce/blip-image-captioning-base") +if not os.path.exists(caption_model_path): + print(f"## clip_interrogator_model not found: {caption_model_path}, pls download from https://huggingface.co/Salesforce/blip-image-captioning-base") + +cache_path=os.path.join(folder_paths.models_dir, "clip_interrogator") + + + +# Tensor to PIL +def tensor2pil(image): + return Image.fromarray(np.clip(255. * image.cpu().numpy().squeeze(), 0, 255).astype(np.uint8)) + +# Convert PIL to Tensor +def pil2tensor(image): + return torch.from_numpy(np.array(image).astype(np.float32) / 255.0).unsqueeze(0) + + +def image_analysis(ci,image): + image = image.convert('RGB') + image_features = ci.image_to_features(image) + + top_mediums = ci.mediums.rank(image_features, 5) + top_artists = ci.artists.rank(image_features, 5) + top_movements = ci.movements.rank(image_features, 5) + top_trendings = ci.trendings.rank(image_features, 5) + top_flavors = ci.flavors.rank(image_features, 5) + + medium_ranks = {medium: sim for medium, sim in zip(top_mediums, ci.similarities(image_features, top_mediums))} + artist_ranks = {artist: sim for artist, sim in zip(top_artists, ci.similarities(image_features, top_artists))} + movement_ranks = {movement: sim for movement, sim in zip(top_movements, ci.similarities(image_features, top_movements))} + trending_ranks = {trending: sim for trending, sim in zip(top_trendings, ci.similarities(image_features, top_trendings))} + flavor_ranks = {flavor: sim for flavor, sim in zip(top_flavors, ci.similarities(image_features, top_flavors))} + + return medium_ranks, artist_ranks, movement_ranks, trending_ranks, flavor_ranks + +def image_to_prompt(ci,image, mode): + ci.config.chunk_size = 2048 if ci.config.clip_model_name == "ViT-L-14/openai" else 1024 + ci.config.flavor_intermediate_count = 2048 if ci.config.clip_model_name == "ViT-L-14/openai" else 1024 + image = image.convert('RGB') + if mode == 'best': + return ci.interrogate(image) + elif mode == 'classic': + return ci.interrogate_classic(image) + elif mode == 'fast': + return ci.interrogate_fast(image) + elif mode == 'negative': + return ci.interrogate_negative(image) + +# image = Image.open(image_path).convert('RGB') +# ci = Interrogator(Config(clip_model_name="ViT-L-14/openai")) +# print(ci.interrogate(image)) + + +class ClipInterrogator: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "image": ("IMAGE",), + "prompt_mode": (['fast','classic','best','negative'],), + "image_analysis": (["off","on"],), + }, + } + + RETURN_TYPES = ("STRING","STRING",) + RETURN_NAMES = ("prompt","analysis",) + + FUNCTION = "run" + + CATEGORY = "♾️Mixlab/prompt" + + INPUT_IS_LIST = True + OUTPUT_IS_LIST = (True,) + global ci + ci = None + def run(self,image,prompt_mode,image_analysis): + global ci + + prompt_mode=prompt_mode[0] + analysis=image_analysis[0] + + prompt_result=[] + analysis_result=[] + + # 进度条 + pbar = comfy.utils.ProgressBar(len(image)*(2 if analysis=='on' else 1)) + + if ci==None: + config=Config( + clip_model_name="ViT-L-14/openai", + device="cuda" if torch.cuda.is_available() else "cpu", + download_cache=True, + clip_model_path=cache_path, + cache_path=cache_path + ) + config.apply_low_vram_defaults() + + caption_model,caption_processor=load_caption_model(caption_model_path,config) + + config.caption_model= caption_model + config.caption_processor= caption_processor + + ci = Interrogator(config) + # else: + # simple_lama.model.to("cuda" if torch.cuda.is_available() else "cpu") + + for i in range(len(image)): + im=image[i] + + im=tensor2pil(im) + im=im.convert('RGB') + + if analysis=='on': + analysis_res=image_analysis(ci,im) + analysis_result.append(json.dumps(analysis_res)) + pbar.update(1) + + prompt=image_to_prompt(ci,im,prompt_mode) + pbar.update(1) + prompt_result.append(prompt) + + + # result.save("inpainted.png") + if ci.config.clip_offload and not ci.clip_offloaded: + ci.clip_model = ci.clip_model.to('cpu') + ci.clip_offloaded = True + + if ci.config.caption_offload and not ci.caption_offloaded: + ci.caption_model = ci.caption_model.to('cpu') + ci.caption_offloaded = True + + return {"ui":{"prompt": prompt_result,"analysis":analysis_result},"result": (prompt_result,analysis_result,)} \ No newline at end of file diff --git a/nodes/ImageNode.py b/nodes/ImageNode.py index 1a47952..76df646 100644 --- a/nodes/ImageNode.py +++ b/nodes/ImageNode.py @@ -953,20 +953,27 @@ class EnhanceImage: CATEGORY = "♾️Mixlab/image" - INPUT_IS_LIST = False + INPUT_IS_LIST = True - OUTPUT_IS_LIST = (False,) + OUTPUT_IS_LIST = (True,) # 运行的函数 def run(self,image,contrast): - # print('EnhanceImage',image.shape) - image=tensor2pil(image) - - image=enhance_depth_map(image,contrast) + # print('EnhanceImage',len(image),image[0].shape) + contrast=contrast[0] + res=[] + for ims in image: + for im in ims: - image=pil2tensor(image) + image=tensor2pil(im) + + image=enhance_depth_map(image,contrast) + + image=pil2tensor(image) + + res.append(image) - return (image,) + return (res,) diff --git a/nodes/Utils.py b/nodes/Utils.py index 5c9d978..2c52cde 100644 --- a/nodes/Utils.py +++ b/nodes/Utils.py @@ -34,13 +34,13 @@ def create_temp_file(image): ) = folder_paths.get_save_image_path('tmp', output_dir) - image=tensor2pil(image) + im=tensor2pil(image) image_file = f"{filename}_{counter:05}.png" image_path=os.path.join(full_output_folder, image_file) - image.save(image_path,compress_level=4) + im.save(image_path,compress_level=4) return [{ "filename": image_file, @@ -461,12 +461,24 @@ class AppInfo: CATEGORY = "♾️Mixlab" - INPUT_IS_LIST = False - OUTPUT_IS_LIST = (False,) + INPUT_IS_LIST = True + OUTPUT_IS_LIST = (True,) def run(self,name,image,input_ids,output_ids,description,version,share_prefix,link,category): + name=name[0] + im=image[0][0] + # image [img,] img[batch,w,h,a] 列表里面是batch， + + input_ids=input_ids[0] + output_ids=output_ids[0] + description=description[0] + version=version[0] + share_prefix=share_prefix[0] + link=link[0] + category=category[0] + #TODO batch 的方式需要处理 - im=create_temp_file(image) + im=create_temp_file(im) # id=get_json_hash([name,im,input_ids,output_ids,description,version]) diff --git a/requirements.txt b/requirements.txt index 3a06273..7cf54d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,5 @@ watchdog opencv-python-headless matplotlib openai -simple-lama-inpainting \ No newline at end of file +simple-lama-inpainting +clip-interrogator==0.6.0 \ No newline at end of file diff --git a/web/index.html b/web/index.html index 3f5c0b3..8c546fa 100644 --- a/web/index.html +++ b/web/index.html @@ -276,7 +276,7 @@ .show_text { font-size: 14px; - /* display: inline-block; */ + user-select: text; margin: 8px; padding: 32px; min-width: 200px; @@ -980,9 +980,20 @@ // Create an input field for the image name const textInput = document.createElement("textarea"); - // uploadImageInput.type = "text"; + // textInput.className=; textInput.value = data.inputs.text; + // uploadImageInput.type = "text"; + let json = localStorage.getItem(`t_${data.id}`) + try { + const { value, height } = JSON.parse(json); + textInput.value = value; + textInput.style.height=height; + } catch (error) { + + } + uploadContainer.appendChild(textInput); + // autoResize(textInput); function autoResize(textarea) { textarea.style.height = 'auto'; @@ -993,6 +1004,10 @@ // console.log(textInput.value) autoResize(textInput); window._appData.data[data.id].inputs.text = textInput.value; + + localStorage.setItem(`t_${data.id}`, JSON.stringify({ + value: textInput.value, + height: textInput.style.height })); }) // Append the upload container to the main container @@ -1167,13 +1182,17 @@ if (keywords && keywords[0]) { // label.innerHTML = `` // 有备选的关键词 + + let defaultValue = (targetId ? localStorage.getItem(`_slide_${targetId}`) : '') || keywords[0]; + let selectTag = createSelect(Array.from(keywords, (k, i) => { return { value: k, text: k, selected: i == 0 } - }), keywords[0]); + }), defaultValue); + selectTag.style = `background: none; color: black; max-width: 300px; border-bottom: 1px solid #acacac; @@ -1182,8 +1201,8 @@ selectTag.addEventListener('change', e => { e.preventDefault(); window._appData.data[targetId].inputs.prompt_keyword = selectTag.value; - // label.querySelector('.label').innerText = selectTag.value - // console.log(window._appData.data[targetId].inputs.prompt_keyword,selectTag.value) + + targetId ? localStorage.setItem(`_slide_${targetId}`, selectTag.value) : '' }) label.appendChild(selectTag); } @@ -1240,7 +1259,7 @@ // 设置默认值 selectElement.value = defaultValue; - console.log(defaultValue, options) + // console.log(defaultValue, options) return selectElement } @@ -1474,7 +1493,6 @@ update: async function (type = "image", val, id) { console.log(val, id) if (val && type == "image" && output.querySelector(`#output_${id} img`)) { - // if (output.querySelector(`#output_${id}`)) { let im = await createImage(val) @@ -1486,10 +1504,33 @@ a.setAttribute('target', "_blank"); a.setAttribute('href', val); - // } - // else { - // output.querySelector(`#output_${id}`).src = val; - // } + } + + if (val && type == "images" && output.querySelector(`#output_${id} img`)) { + let imgDiv = output.querySelector(`#output_${id}`) + imgDiv.style.display = 'none'; + + // 清空 + // Array.from(imgDiv.parentElement.querySelectorAll('.output_images'), im => im.remove()); + + for (const v of val) { + let im = await createImage(v); + + // 构建新的 + let a = document.createElement('a'); + a.className = `${imgDiv.id} output_images` + a.setAttribute('data-pswp-width', im.naturalWidth); + a.setAttribute('data-pswp-height', im.naturalHeight); + a.setAttribute('target', "_blank"); + a.setAttribute('href', v); + + let img = new Image(); + // img; + img.src = v; + a.appendChild(img) + // imgDiv.parentElement.appendChild(a); + imgDiv.parentElement.insertBefore(a, imgDiv.parentElement.firstChild); + } } @@ -1672,9 +1713,14 @@ const gifs = detail?.output?.gifs; if (images) { - // if (!images) return; - const src = `${get_url()}/view?filename=${encodeURIComponent(images[0].filename)}&type=${images[0].type}&subfolder=${encodeURIComponent(images[0].subfolder)}&t=${+new Date()}`; - show(src, detail.node, 'image'); + // if (!images) return; + + let url = get_url(); + + show(Array.from(images, img => { + return `${url}/view?filename=${encodeURIComponent(img.filename)}&type=${img.type}&subfolder=${encodeURIComponent(img.subfolder)}&t=${+new Date()}`; + }), detail.node, 'images'); + } else if (text) { ui.output.update("text", Array.isArray(text) ? text[0] : text, detail.node) } else if (gifs && gifs[0]) { @@ -1686,7 +1732,7 @@ } - try { + try { ui.status.update(`executed_#${window._appData.data[detail.node]?.class_type}`); ui.submitButton.reset() } catch (error) { diff --git a/web/javascript/checkVersion_mixlab.js b/web/javascript/checkVersion_mixlab.js index 600fba9..525c262 100644 --- a/web/javascript/checkVersion_mixlab.js +++ b/web/javascript/checkVersion_mixlab.js @@ -3,7 +3,7 @@ import { app } from '../../../scripts/app.js' const repoOwner = 'shadowcz007' // 替换为仓库的所有者 const repoName = 'comfyui-mixlab-nodes' // 替换为仓库的名称 -const version = 'v0.9.1' +const version = 'v0.10.0' fetch(`https://api.github.com/repos/${repoOwner}/${repoName}/releases/latest`) .then(response => response.json()) diff --git a/web/javascript/prompt_mixlab.js b/web/javascript/prompt_mixlab.js index e50ed5a..203aff6 100644 --- a/web/javascript/prompt_mixlab.js +++ b/web/javascript/prompt_mixlab.js @@ -132,8 +132,10 @@ app.registerExtension({ inp.click() inp.addEventListener('change', event => { // 获取选择的文件 - const file = event.target.files[0] + const file = event.target.files[0]; + this.title=file.name.split('.')[0]; + // console.log(file.name.split('.')[0]) // 创建文件读取器 const reader = new FileReader()