Merge pull request #106 from shadowcz007/v0.10-add-clip-interrogator

V0.10 add clip interrogator

README.md

@@ -71,6 +71,9 @@ https://github.com/shadowcz007/comfyui-mixlab-nodes/assets/12645064/e7e77f90-e43
 
 ![randomPrompt](./assets/randomPrompt.png)
 
+> ClipInterrogator
+
+[add clip-interrogator](https://github.com/pharmapsychotic/clip-interrogator)
 
 ### Layers
 > A new layer class node has been added, allowing you to separate the image into layers. After merging the images, you can input the controlnet for further processing.
@@ -167,9 +170,7 @@ v0.8.0 🚀🚗🚚🏃‍ LaMaInpainting
 
 [Download lama](https://github.com/enesmsahin/simple-lama-inpainting/releases/download/v0.1.0/big-lama.pt), move to : models/lama
 
-<!-- ### Workflow
-[Workflow](./workflow.md) -->
-
+[Download Salesforce\blip-image-captioning-base](https://huggingface.co/Salesforce/blip-image-captioning-base), move to : models/clip_interrogator/Salesforce/blip-image-captioning-base
 
 ## Installation
 

__init__.py

@@ -512,6 +512,7 @@ from .nodes.ChatGPT import ChatGPTNode,ShowTextForGPT,CharacterInText
 from .nodes.Audio import GamePal,SpeechRecognition,SpeechSynthesis
 from .nodes.Utils import AppInfo,IntNumber,FloatSlider,TextInput,ColorInput,FontInput,TextToNumber,DynamicDelayProcessor,LimitNumber,SwitchByIndex,GetImageSize_,MultiplicationNode
 from .nodes.Lama import LaMaInpainting
+from .nodes.ClipInterrogator import ClipInterrogator
 
 # 要导出的所有节点及其名称的字典
 # 注意：名称应全局唯一
@@ -519,6 +520,7 @@ NODE_CLASS_MAPPINGS = {
     "AppInfo":AppInfo,
     "RandomPrompt":RandomPrompt,
     "PromptSlide":PromptSlide,
+    "ClipInterrogator":ClipInterrogator,
     "NoiseImage":NoiseImage,
     "GradientImage":GradientImage,
     "TransparentImage":TransparentImage,

data/extension-node-map.json

@@ -4786,6 +4786,7 @@
             "NewLayer",
             "RandomPrompt",
             "PromptSlide",
+            "ClipInterrogator",
             "ScreenShare",
             "ShowLayer",
             "ShowTextForGPT",

nodes/ClipInterrogator.py

@@ -0,0 +1,158 @@
+import os
+import folder_paths
+
+from PIL import Image
+import comfy.utils
+import numpy as np
+import json
+import torch
+
+from transformers import AutoProcessor, BlipForConditionalGeneration
+
+from clip_interrogator import Config, Interrogator
+
+def load_caption_model(model_path,config,t='blip-base'):
+    dtype=torch.float16 if config.device == 'cuda' else torch.float32
+    caption_model = BlipForConditionalGeneration.from_pretrained(model_path, torch_dtype=dtype)
+    
+    caption_processor = AutoProcessor.from_pretrained(model_path)
+
+    caption_model.eval()
+    if not config.caption_offload:
+        caption_model = caption_model.to(config.device)
+    
+    return (caption_model,caption_processor)
+    
+
+
+caption_model_path=os.path.join(folder_paths.models_dir, "clip_interrogator/Salesforce/blip-image-captioning-base")
+if not os.path.exists(caption_model_path):
+    print(f"## clip_interrogator_model not found: {caption_model_path}, pls download from https://huggingface.co/Salesforce/blip-image-captioning-base")
+
+cache_path=os.path.join(folder_paths.models_dir, "clip_interrogator")
+
+
+
+# Tensor to PIL
+def tensor2pil(image):
+    return Image.fromarray(np.clip(255. * image.cpu().numpy().squeeze(), 0, 255).astype(np.uint8))
+
+# Convert PIL to Tensor
+def pil2tensor(image):
+    return torch.from_numpy(np.array(image).astype(np.float32) / 255.0).unsqueeze(0)
+
+
+def image_analysis(ci,image):
+    image = image.convert('RGB')
+    image_features = ci.image_to_features(image)
+
+    top_mediums = ci.mediums.rank(image_features, 5)
+    top_artists = ci.artists.rank(image_features, 5)
+    top_movements = ci.movements.rank(image_features, 5)
+    top_trendings = ci.trendings.rank(image_features, 5)
+    top_flavors = ci.flavors.rank(image_features, 5)
+
+    medium_ranks = {medium: sim for medium, sim in zip(top_mediums, ci.similarities(image_features, top_mediums))}
+    artist_ranks = {artist: sim for artist, sim in zip(top_artists, ci.similarities(image_features, top_artists))}
+    movement_ranks = {movement: sim for movement, sim in zip(top_movements, ci.similarities(image_features, top_movements))}
+    trending_ranks = {trending: sim for trending, sim in zip(top_trendings, ci.similarities(image_features, top_trendings))}
+    flavor_ranks = {flavor: sim for flavor, sim in zip(top_flavors, ci.similarities(image_features, top_flavors))}
+    
+    return medium_ranks, artist_ranks, movement_ranks, trending_ranks, flavor_ranks
+
+def image_to_prompt(ci,image, mode):
+    ci.config.chunk_size = 2048 if ci.config.clip_model_name == "ViT-L-14/openai" else 1024
+    ci.config.flavor_intermediate_count = 2048 if ci.config.clip_model_name == "ViT-L-14/openai" else 1024
+    image = image.convert('RGB')
+    if mode == 'best':
+        return ci.interrogate(image)
+    elif mode == 'classic':
+        return ci.interrogate_classic(image)
+    elif mode == 'fast':
+        return ci.interrogate_fast(image)
+    elif mode == 'negative':
+        return ci.interrogate_negative(image)
+
+# image = Image.open(image_path).convert('RGB')
+# ci = Interrogator(Config(clip_model_name="ViT-L-14/openai"))
+# print(ci.interrogate(image))
+
+
+class ClipInterrogator:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": {
+            "image": ("IMAGE",),
+            "prompt_mode": (['fast','classic','best','negative'],),
+            "image_analysis": (["off","on"],), 
+                             },
+                }
+    
+    RETURN_TYPES = ("STRING","STRING",)
+    RETURN_NAMES = ("prompt","analysis",)
+
+    FUNCTION = "run"
+
+    CATEGORY = "♾️Mixlab/prompt"
+
+    INPUT_IS_LIST = True
+    OUTPUT_IS_LIST = (True,)
+    global ci
+    ci = None
+    def run(self,image,prompt_mode,image_analysis):
+        global ci
+
+        prompt_mode=prompt_mode[0]
+        analysis=image_analysis[0]
+
+        prompt_result=[]
+        analysis_result=[]
+
+        # 进度条
+        pbar = comfy.utils.ProgressBar(len(image)*(2 if analysis=='on' else 1))
+        
+        if ci==None:
+            config=Config(
+                clip_model_name="ViT-L-14/openai",
+                device="cuda" if torch.cuda.is_available() else "cpu",
+                download_cache=True,
+                clip_model_path=cache_path,
+                cache_path=cache_path
+                )
+            config.apply_low_vram_defaults()
+
+            caption_model,caption_processor=load_caption_model(caption_model_path,config)
+
+            config.caption_model= caption_model
+            config.caption_processor= caption_processor
+
+            ci = Interrogator(config)
+        # else:
+        #     simple_lama.model.to("cuda" if torch.cuda.is_available() else "cpu")
+
+        for i in range(len(image)):
+            im=image[i]
+
+            im=tensor2pil(im)
+            im=im.convert('RGB')
+
+            if analysis=='on':
+                analysis_res=image_analysis(ci,im)
+                analysis_result.append(json.dumps(analysis_res))
+                pbar.update(1)
+
+            prompt=image_to_prompt(ci,im,prompt_mode)
+            pbar.update(1)
+            prompt_result.append(prompt)
+
+
+        # result.save("inpainted.png")
+        if ci.config.clip_offload and not ci.clip_offloaded:
+            ci.clip_model = ci.clip_model.to('cpu')
+            ci.clip_offloaded = True
+
+        if ci.config.caption_offload and not ci.caption_offloaded:
+            ci.caption_model = ci.caption_model.to('cpu')
+            ci.caption_offloaded = True
+
+        return {"ui":{"prompt": prompt_result,"analysis":analysis_result},"result": (prompt_result,analysis_result,)}

nodes/ImageNode.py

@@ -953,20 +953,27 @@ class EnhanceImage:
 
     CATEGORY = "♾️Mixlab/image"
 
-    INPUT_IS_LIST = False
+    INPUT_IS_LIST = True
 
-    OUTPUT_IS_LIST = (False,)
+    OUTPUT_IS_LIST = (True,)
   
     # 运行的函数
     def run(self,image,contrast):
-        # print('EnhanceImage',image.shape)
-        image=tensor2pil(image)
-     
-        image=enhance_depth_map(image,contrast)
+        # print('EnhanceImage',len(image),image[0].shape)
+        contrast=contrast[0]
+        res=[]
+        for ims in image:
+            for im in ims:
 
-        image=pil2tensor(image)
+                image=tensor2pil(im)
+            
+                image=enhance_depth_map(image,contrast)
+
+                image=pil2tensor(image)
+
+                res.append(image)
            
-        return (image,)
+        return (res,)
 
 
 

nodes/Utils.py

@@ -34,13 +34,13 @@ def create_temp_file(image):
         ) = folder_paths.get_save_image_path('tmp', output_dir)
 
     
-    image=tensor2pil(image)
+    im=tensor2pil(image)
  
     image_file = f"{filename}_{counter:05}.png"
      
     image_path=os.path.join(full_output_folder, image_file)
 
-    image.save(image_path,compress_level=4)
+    im.save(image_path,compress_level=4)
 
     return [{
                 "filename": image_file,
@@ -461,12 +461,24 @@ class AppInfo:
 
     CATEGORY = "♾️Mixlab"
 
-    INPUT_IS_LIST = False
-    OUTPUT_IS_LIST = (False,)
+    INPUT_IS_LIST = True
+    OUTPUT_IS_LIST = (True,)
 
     def run(self,name,image,input_ids,output_ids,description,version,share_prefix,link,category):
+        name=name[0]
+        im=image[0][0]
+        # image [img,] img[batch,w,h,a] 列表里面是batch，
+
+        input_ids=input_ids[0]
+        output_ids=output_ids[0]
+        description=description[0]
+        version=version[0]
+        share_prefix=share_prefix[0]
+        link=link[0]
+        category=category[0]
+
         #TODO batch 的方式需要处理
-        im=create_temp_file(image)
+        im=create_temp_file(im)
         
         # id=get_json_hash([name,im,input_ids,output_ids,description,version])
 

requirements.txt

@@ -4,4 +4,5 @@ watchdog
 opencv-python-headless
 matplotlib
 openai
-simple-lama-inpainting
+simple-lama-inpainting
+clip-interrogator==0.6.0

web/index.html

@@ -276,7 +276,7 @@
 
         .show_text {
             font-size: 14px;
-            /* display: inline-block; */
+            user-select: text;
             margin: 8px;
             padding: 32px;
             min-width: 200px;
@@ -980,9 +980,20 @@
 
                     // Create an input field for the image name
                     const textInput = document.createElement("textarea");
-                    // uploadImageInput.type = "text";
+                    // textInput.className=;
                     textInput.value = data.inputs.text;
+                    // uploadImageInput.type = "text";
+                    let json = localStorage.getItem(`t_${data.id}`)
+                    try {
+                        const { value, height } = JSON.parse(json);
+                        textInput.value = value;
+                        textInput.style.height=height;
+                    } catch (error) {
+
+                    }
+
                     uploadContainer.appendChild(textInput);
+                    // autoResize(textInput);
 
                     function autoResize(textarea) {
                         textarea.style.height = 'auto';
@@ -993,6 +1004,10 @@
                         // console.log(textInput.value)
                         autoResize(textInput);
                         window._appData.data[data.id].inputs.text = textInput.value;
+
+                        localStorage.setItem(`t_${data.id}`, JSON.stringify({
+                             value: textInput.value, 
+                             height: textInput.style.height }));
                     })
 
                     // Append the upload container to the main container
@@ -1167,13 +1182,17 @@
             if (keywords && keywords[0]) {
                 // label.innerHTML = ``
                 // 有备选的关键词
+
+                let defaultValue = (targetId ? localStorage.getItem(`_slide_${targetId}`) : '') || keywords[0];
+
                 let selectTag = createSelect(Array.from(keywords, (k, i) => {
                     return {
                         value: k,
                         text: k,
                         selected: i == 0
                     }
-                }), keywords[0]);
+                }), defaultValue);
+
                 selectTag.style = `background: none;
                                     color: black;    max-width: 300px;
                                     border-bottom: 1px solid #acacac;
@@ -1182,8 +1201,8 @@
                 selectTag.addEventListener('change', e => {
                     e.preventDefault();
                     window._appData.data[targetId].inputs.prompt_keyword = selectTag.value;
-                    // label.querySelector('.label').innerText = selectTag.value
-                    // console.log(window._appData.data[targetId].inputs.prompt_keyword,selectTag.value)
+
+                    targetId ? localStorage.setItem(`_slide_${targetId}`, selectTag.value) : ''
                 })
                 label.appendChild(selectTag);
             }
@@ -1240,7 +1259,7 @@
 
             // 设置默认值
             selectElement.value = defaultValue;
-            console.log(defaultValue, options)
+            // console.log(defaultValue, options)
             return selectElement
         }
 
@@ -1474,7 +1493,6 @@
                     update: async function (type = "image", val, id) {
                         console.log(val, id)
                         if (val && type == "image" && output.querySelector(`#output_${id} img`)) {
-                            // if (output.querySelector(`#output_${id}`)) {
 
                             let im = await createImage(val)
 
@@ -1486,10 +1504,33 @@
                             a.setAttribute('target', "_blank");
                             a.setAttribute('href', val);
 
-                            // }
-                            //  else {
-                            //     output.querySelector(`#output_${id}`).src = val;
-                            // }
+                        }
+
+                        if (val && type == "images" && output.querySelector(`#output_${id} img`)) {
+                            let imgDiv = output.querySelector(`#output_${id}`)
+                            imgDiv.style.display = 'none';
+
+                            // 清空
+                            // Array.from(imgDiv.parentElement.querySelectorAll('.output_images'), im => im.remove());
+
+                            for (const v of val) {
+                                let im = await createImage(v);
+
+                                // 构建新的
+                                let a = document.createElement('a');
+                                a.className = `${imgDiv.id} output_images`
+                                a.setAttribute('data-pswp-width', im.naturalWidth);
+                                a.setAttribute('data-pswp-height', im.naturalHeight);
+                                a.setAttribute('target', "_blank");
+                                a.setAttribute('href', v);
+
+                                let img = new Image();
+                                // img;
+                                img.src = v;
+                                a.appendChild(img)
+                                // imgDiv.parentElement.appendChild(a);
+                                imgDiv.parentElement.insertBefore(a, imgDiv.parentElement.firstChild);
+                            }
 
                         }
 
@@ -1672,9 +1713,14 @@
                 const gifs = detail?.output?.gifs;
 
                 if (images) {
-                    // if (!images) return; 
-                    const src = `${get_url()}/view?filename=${encodeURIComponent(images[0].filename)}&type=${images[0].type}&subfolder=${encodeURIComponent(images[0].subfolder)}&t=${+new Date()}`;
-                    show(src, detail.node, 'image');
+                    // if (!images) return;
+
+                    let url = get_url();
+
+                    show(Array.from(images, img => {
+                        return `${url}/view?filename=${encodeURIComponent(img.filename)}&type=${img.type}&subfolder=${encodeURIComponent(img.subfolder)}&t=${+new Date()}`;
+                    }), detail.node, 'images');
+
                 } else if (text) {
                     ui.output.update("text", Array.isArray(text) ? text[0] : text, detail.node)
                 } else if (gifs && gifs[0]) {
@@ -1686,7 +1732,7 @@
                 }
 
 
-                try { 
+                try {
                     ui.status.update(`executed_#${window._appData.data[detail.node]?.class_type}`);
                     ui.submitButton.reset()
                 } catch (error) {

web/javascript/checkVersion_mixlab.js

@@ -3,7 +3,7 @@ import { app } from '../../../scripts/app.js'
 const repoOwner = 'shadowcz007' // 替换为仓库的所有者
 const repoName = 'comfyui-mixlab-nodes' // 替换为仓库的名称
 
-const version = 'v0.9.1'
+const version = 'v0.10.0'
 
 fetch(`https://api.github.com/repos/${repoOwner}/${repoName}/releases/latest`)
   .then(response => response.json())

web/javascript/prompt_mixlab.js

@@ -132,8 +132,10 @@ app.registerExtension({
           inp.click()
           inp.addEventListener('change', event => {
             // 获取选择的文件
-            const file = event.target.files[0]
+            const file = event.target.files[0];
+            this.title=file.name.split('.')[0];
 
+            // console.log(file.name.split('.')[0])
             // 创建文件读取器
             const reader = new FileReader()