Merge pull request #111 from shadowcz007/v.11.0-PromptImage-node-图片和prompt匹配

V0.11.0 PromptImage & PromptSimplification

README.md

@@ -77,6 +77,9 @@ https://github.com/shadowcz007/comfyui-mixlab-nodes/assets/12645064/e7e77f90-e43
 
 [add clip-interrogator](https://github.com/pharmapsychotic/clip-interrogator)
 
+> PromptImage & PromptSimplification,Assist in simplifying prompt words, comparing images and prompt word nodes.
+
+
 ### Layers
 > A new layer class node has been added, allowing you to separate the image into layers. After merging the images, you can input the controlnet for further processing.
 

__init__.py

@@ -503,7 +503,7 @@ PromptServer.add_routes=new_add_routes
 
 
 # 导入节点
-from .nodes.PromptNode import RandomPrompt,PromptSlide,PromptSimplification
+from .nodes.PromptNode import RandomPrompt,PromptSlide,PromptSimplification,PromptImage
 from .nodes.ImageNode import NoiseImage,TransparentImage,GradientImage,LoadImagesFromPath,LoadImagesFromURL,ResizeImage,TextImage,SvgImage,Image3D,ShowLayer,NewLayer,MergeLayers,AreaToMask,SmoothMask,FeatheredMask,SplitLongMask,ImageCropByAlpha,EnhanceImage,FaceToMask
 from .nodes.Vae import VAELoader,VAEDecode
 from .nodes.ScreenShareNode import ScreenShareNode,FloatingVideo
@@ -521,6 +521,7 @@ NODE_CLASS_MAPPINGS = {
     "RandomPrompt":RandomPrompt,
     "PromptSlide":PromptSlide,
     "PromptSimplification":PromptSimplification,
+    "PromptImage":PromptImage,
     "ClipInterrogator":ClipInterrogator,
     "NoiseImage":NoiseImage,
     "GradientImage":GradientImage,

nodes/PromptNode.py

@@ -1,9 +1,11 @@
 import random
 import comfy.utils
-import json
+import os
+import numpy as np
 from urllib import request, parse
-
-
+import folder_paths
+from PIL import Image, ImageOps,ImageFilter,ImageEnhance,ImageDraw,ImageSequence, ImageFont
+from PIL.PngImagePlugin import PngInfo
 # def queue_prompt(prompt_workflow):
 #     p = {"prompt": prompt_workflow}
 #     data = json.dumps(p).encode('utf-8')
@@ -45,6 +47,9 @@ default_prompt1='''Swing
 default_prompt1="\n".join([p.strip() for p in default_prompt1.split('\n') if p.strip()!=''])
 
 
+def tensor2pil(image):
+    return Image.fromarray(np.clip(255. * image.cpu().numpy().squeeze(), 0, 255).astype(np.uint8))
+
 def addWeight(text, weight=1):
     if weight == 1:
         return text
@@ -78,6 +83,73 @@ def prompt_delete_words(sentence, new_words_length):
 # result = prompt_delete_words(sentence, new_words_length)
 # print(result)
 
+class PromptImage:
+    def __init__(self):
+        self.temp_dir = folder_paths.get_temp_directory()
+        self.type = "temp"
+        self.prefix_append = "PromptImage"
+        self.compress_level = 4
+
+    @classmethod
+    def INPUT_TYPES(s):
+        return {
+            "required": {
+                "prompts": ("STRING", 
+                         {
+                            "multiline": True, 
+                            "default": '',
+                            "dynamicPrompts": False
+                          }),
+
+                "images": ("IMAGE",{"default": None}), 
+                 "save_to_image": (["enable", "disable"],),
+                }
+            }
+    
+    RETURN_TYPES = ()
+   
+    OUTPUT_NODE = True
+
+    INPUT_IS_LIST = True
+
+    FUNCTION = "run"
+
+    CATEGORY = "♾️Mixlab/prompt"
+
+    # 运行的函数
+    def run(self,prompts,images,save_to_image):
+        filename_prefix="mixlab_"
+        filename_prefix += self.prefix_append
+        full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(
+            filename_prefix, self.temp_dir, images[0].shape[1], images[0].shape[0])
+        results = list()
+
+        save_to_image=save_to_image[0]=='enable'
+
+        for index in range(len(images)):
+            image=images[index]
+            img=tensor2pil(image)
+           
+            metadata = None
+            if save_to_image:
+                metadata = PngInfo()
+                prompt_text=prompts[index]
+                if prompt_text is not None:
+                    metadata.add_text("prompt_text", prompt_text)
+                
+            file = f"{filename}_{index}_{counter:05}_.png"
+            img.save(os.path.join(full_output_folder, file), pnginfo=metadata, compress_level=self.compress_level)
+            results.append({
+                "filename": file,
+                "subfolder": subfolder,
+                "type": self.type
+            })
+            counter += 1
+
+        return { "ui": { "_images": results,"prompts":prompts } }
+
+
+
 
 class PromptSimplification:
     @classmethod

web/index.html

@@ -323,6 +323,15 @@
             height: 56px !important;
             outline: 1px solid white;
         }
+
+
+        /* 给prompt image 节点使用 */
+        .prompt_image {
+            color: #2f2f2f;
+            padding: 0 10px;
+            font-size: 12px;
+            width: 200px;
+        }
     </style>
     <!-- <script src="../../../scripts/api.js" type="module"></script> -->
     <link href="/extensions/comfyui-mixlab-nodes/lib/photoswipe.min.css" rel="stylesheet">
@@ -603,7 +612,16 @@
                     div.innerText = Array.isArray(node.inputs.text) ? node.inputs.text[0] : node.inputs.text
                     output_card.appendChild(div);
                 };
-                if (["SaveImage", "PreviewImage"].includes(node.class_type)) {
+
+                if (node.class_type == "ClipInterrogator") {
+                    let div = document.createElement('div');
+                    div.className = "show_text";
+                    div.id = `output_${node.id}`;
+                    div.innerText = '#ClipInterrogator: …… '
+                    output_card.appendChild(div);
+                };
+
+                if (["SaveImage", "PreviewImage", "PromptImage"].includes(node.class_type)) {
 
                     let a = document.createElement('a');
                     a.id = `output_${node.id}`
@@ -1567,7 +1585,7 @@
 
                         }
 
-                        if (val && type == "images" && output.querySelector(`#output_${id} img`)) {
+                        if (val && (type == "images" || type == 'images_prompts') && output.querySelector(`#output_${id} img`)) {
                             let imgDiv = output.querySelector(`#output_${id}`)
                             imgDiv.style.display = 'none';
 
@@ -1575,7 +1593,16 @@
                             // Array.from(imgDiv.parentElement.querySelectorAll('.output_images'), im => im.remove());
 
                             for (const v of val) {
-                                let im = await createImage(v);
+
+                                let url = v, prompt = ''
+
+                                if (type == 'images_prompts') {
+                                    // 是个数组，多了对应的prompt
+                                    url = v[0];
+                                    prompt = v[1];
+                                }
+
+                                let im = await createImage(url);
 
                                 // 构建新的
                                 let a = document.createElement('a');
@@ -1583,12 +1610,22 @@
                                 a.setAttribute('data-pswp-width', im.naturalWidth);
                                 a.setAttribute('data-pswp-height', im.naturalHeight);
                                 a.setAttribute('target', "_blank");
-                                a.setAttribute('href', v);
+                                a.setAttribute('href', url);
+
 
                                 let img = new Image();
                                 // img;
-                                img.src = v;
-                                a.appendChild(img)
+                                img.src = url;
+                                a.appendChild(img);
+
+                                if (prompt) {
+                                    a.style.textDecoration = 'none';
+                                    let p = document.createElement('p')
+                                    p.className = 'prompt_image'
+                                    p.innerText = prompt;
+                                    a.appendChild(p)
+                                }
+
                                 // imgDiv.parentElement.appendChild(a);
                                 imgDiv.parentElement.insertBefore(a, imgDiv.parentElement.firstChild);
                             }
@@ -1747,10 +1784,10 @@
             };
 
             api.addEventListener("status", ({ detail }) => {
-                console.log("status", detail, detail.exec_info.queue_remaining);
+                console.log("status", detail, detail.exec_info?.queue_remaining);
                 try {
-                    ui.status.update(`queue#${detail.exec_info.queue_remaining}`);
-                    if (detail.exec_info.queue_remaining === 0) {
+                    ui.status.update(`queue#${detail.exec_info?.queue_remaining}`);
+                    if (detail.exec_info?.queue_remaining === 0) {
                         // 运行按钮重设
                         ui.submitButton.reset()
                         console.log('运行按钮重设')
@@ -1763,8 +1800,9 @@
 
             api.addEventListener("progress", ({ detail }) => {
                 console.log("progress", detail);
+                const class_type = window._appData.data[detail?.node]?.class_type || ''
                 try {
-                    ui.status.update(`${detail.value}/${detail.max}`);
+                    ui.status.update(`${parseFloat(100 * detail.value / detail.max).toFixed(1)}% ${class_type}`);
                     ui.submitButton.running()
                 } catch (error) {
 
@@ -1778,6 +1816,12 @@
                 const text = detail?.output?.text;
                 const gifs = detail?.output?.gifs;
 
+                const prompt = detail?.output?.prompt;
+                const analysis = detail?.output?.analysis;
+
+                const _images = detail?.output?._images;
+                const prompts = detail?.output?.prompts;
+
                 if (images) {
                     // if (!images) return;
 
@@ -1787,6 +1831,15 @@
                         return `${url}/view?filename=${encodeURIComponent(img.filename)}&type=${img.type}&subfolder=${encodeURIComponent(img.subfolder)}&t=${+new Date()}`;
                     }), detail.node, 'images');
 
+                } else if (_images && prompts) {
+                    let url = get_url();
+
+                    show(Array.from(_images, (img, i) => {
+                        return [`${url}/view?filename=${encodeURIComponent(img.filename)
+                            }&type=${img.type}&subfolder=${encodeURIComponent(img.subfolder)
+                            }&t=${+new Date()}`, prompts[i]];
+                    }), detail.node, 'images_prompts');
+
                 } else if (text) {
                     ui.output.update("text", Array.isArray(text) ? text.join('\n\n') : text, detail.node)
                 } else if (gifs && gifs[0]) {
@@ -1795,9 +1848,13 @@
                         }&&format=${gifs[0].format}&t=${+new Date()}`;
 
                     show(src, detail.node, gifs[0].format.match('video') ? 'video' : 'image');
+                } else if (prompt && analysis) {
+                    // #ClipInterrogator: …… 
+                    ui.output.update("text", `${prompt.join('\n\n')}\n${JSON.stringify(analysis, null, 2)}`, detail.node)
                 }
 
 
+
                 try {
                     ui.status.update(`executed_#${window._appData.data[detail.node]?.class_type}`);
                     ui.submitButton.reset()

web/javascript/checkVersion_mixlab.js

@@ -3,7 +3,7 @@ import { app } from '../../../scripts/app.js'
 const repoOwner = 'shadowcz007' // 替换为仓库的所有者
 const repoName = 'comfyui-mixlab-nodes' // 替换为仓库的名称
 
-const version = 'v0.10.0'
+const version = 'v0.11.0'
 
 fetch(`https://api.github.com/repos/${repoOwner}/${repoName}/releases/latest`)
   .then(response => response.json())

web/javascript/prompt_mixlab.js

@@ -35,6 +35,24 @@ function get_position_style (ctx, widget_width, y, node_height) {
     justifyContent: 'space-between'
   }
 }
+function createImage (url) {
+  let im = new Image()
+  return new Promise((res, rej) => {
+    im.onload = () => res(im)
+    im.src = url
+  })
+}
+
+async function fetchImage (url) {
+  try {
+    const response = await fetch(url)
+    const blob = await response.blob()
+     
+    return blob
+  } catch (error) {
+    console.error('出现错误:', error)
+  }
+}
 
 const getLocalData = key => {
   let data = {}
@@ -132,8 +150,8 @@ app.registerExtension({
           inp.click()
           inp.addEventListener('change', event => {
             // 获取选择的文件
-            const file = event.target.files[0];
-            this.title=file.name.split('.')[0];
+            const file = event.target.files[0]
+            this.title = file.name.split('.')[0]
 
             // console.log(file.name.split('.')[0])
             // 创建文件读取器
@@ -202,3 +220,147 @@ app.registerExtension({
     }
   }
 })
+
+app.registerExtension({
+  name: 'Mixlab.prompt.PromptImage',
+  _createResult: async (node, widget, message) => {
+    widget.div.innerHTML = ``
+
+    const width = node.size[0] * 0.5 - 12
+
+    let height_add = 0
+
+    for (let index = 0; index < message._images.length; index++) {
+      const img = message._images[index]
+      let url = api.apiURL(
+        `/view?filename=${encodeURIComponent(img.filename)}&type=${
+          img.type
+        }&subfolder=${
+          img.subfolder
+        }${app.getPreviewFormatParam()}${app.getRandParam()}`
+      )
+
+      let image = await createImage(url)
+
+      // 创建card
+      let div = document.createElement('div')
+      div.className = 'card'
+      div.draggable = true
+
+      div.ondragend = async event => {
+        console.log('拖动停止')
+        let url = div.querySelector('img').src
+
+        let blob = await fetchImage(url)
+
+        let imageNode = null
+        // No image node selected: add a new one
+        if (!imageNode) {
+          const newNode = LiteGraph.createNode('LoadImage')
+          newNode.pos = [...app.canvas.graph_mouse]
+          imageNode = app.graph.add(newNode)
+          app.graph.change()
+        }
+
+        
+        // const blob = item.getAsFile();
+        imageNode.pasteFile(blob)
+      }
+
+      div.setAttribute('data-scale', image.naturalHeight / image.naturalWidth)
+
+      let h = (image.naturalHeight * width) / image.naturalWidth
+      if (index % 2 === 0) height_add += h
+      div.style = `width: ${width}px;height:${h}px;position: relative;margin: 4px;`
+      div.innerHTML = `<img src="${url}" style='width: 100%'/>
+            <p style="position: absolute;
+            bottom: 0;
+            left: 0; 
+            background:#444444c2;
+            margin: 0;
+            font-size: 12px;
+            padding: 5px;
+            text-align: left;">${message.prompts[index]}</p>`
+      widget.div.appendChild(div)
+    }
+
+    node.size[1] = 98 + height_add
+  },
+  async beforeRegisterNodeDef (nodeType, nodeData, app) {
+    if (nodeType.comfyClass == 'PromptImage') {
+      const orig_nodeCreated = nodeType.prototype.onNodeCreated
+      nodeType.prototype.onNodeCreated = function () {
+        orig_nodeCreated?.apply(this, arguments)
+        console.log('#orig_nodeCreated', this)
+        const widget = {
+          type: 'div',
+          name: 'result',
+          draw (ctx, node, widget_width, y, widget_height) {
+            Object.assign(this.div.style, {
+              ...get_position_style(ctx, widget_width, y, node.size[1]),
+              flexWrap: 'wrap',
+              justifyContent: 'space-between',
+              // outline: '1px solid red',
+              paddingLeft: '0px',
+              width: widget_width + 'px'
+            })
+          }
+        }
+
+        widget.div = $el('div', {})
+
+        document.body.appendChild(widget.div)
+
+        this.addCustomWidget(widget)
+
+        const onRemoved = this.onRemoved
+        this.onRemoved = () => {
+          widget.div.remove()
+          return onRemoved?.()
+        }
+
+        const onResize = this.onResize
+        this.onResize = function () {
+          // 缩放发生
+          // console.log('##缩放发生', this.size)
+          let w = this.size[0] * 0.5 - 12
+          Array.from(widget.div.querySelectorAll('.card'), card => {
+            card.style.width = `${w}px`
+            card.style.height = `${
+              w * parseFloat(card.getAttribute('data-scale'))
+            }px`
+          })
+          return onResize?.apply(this, arguments)
+        }
+
+        // this.serialize_widgets = true //需要保存参数
+      }
+
+      const onExecuted = nodeType.prototype.onExecuted
+      nodeType.prototype.onExecuted = async function (message) {
+        onExecuted?.apply(this, arguments)
+        console.log('PromptImage', message.prompts, message._images)
+        // window._mixlab_app_json = message.json
+        try {
+          let widget = this.widgets.filter(w => w.name === 'result')[0]
+          widget.value = message
+
+          this._createResult(this, widget, message)
+        } catch (error) {}
+      }
+
+      this.serialize_widgets = true //需要保存参数
+    }
+  },
+  async loadedGraphNode (node, app) {
+    if (node.type === 'PromptImage') {
+      // await sleep(0)
+      let widget = node.widgets.filter(w => w.name === 'result')[0]
+      console.log('widget.value', widget.value)
+      let cards = widget.div.querySelectorAll('.card')
+      if (cards.length == 0) node.size = [280, 120]
+
+      this._createResult(node, widget, widget.value)
+    }
+  }
+})