v0.2.5.1

新增GPT节点

README.md

@@ -14,6 +14,24 @@ https://github.com/shadowcz007/comfyui-mixlab-nodes/assets/12645064/e7e77f90-e43
 
 !! Please use the address with HTTPS (https://127.0.0.1).
 
+
+### LoadImagesFromLocal
+> Monitor changes to images in a local folder, and trigger real-time execution of workflows, supporting common image formats, especially PSD format, in conjunction with Photoshop. Q: Translate into English
+
+![watch](./assets/4-loadfromlocal-watcher-workflow.svg)
+
+[workflow-4](./workflow/4-loadfromlocal-watcher-workflow.json)
+
+
+### GPT
+> ChatGPT、ChatGLM3 , Some code provided by rui.
+
+
+![gpt-workflow.svg](./assets/gpt-workflow.svg)
+
+[workflow-5](./workflow/5-gpt-workflow.json)
+
+
 ## Installation
 
 manually install, simply clone the repo into the custom_nodes directory with this command:
@@ -47,7 +65,7 @@ pip3 install -r requirements.txt
 
 ## Nodes
 
-![main](./assets/all.png)
+![main](./assets/all-workflow.svg)
 ![main2](./assets/detect-face-all.png)
 
 [workflow-1](./workflow/1-workflow.json)
@@ -62,11 +80,7 @@ pip3 install -r requirements.txt
 
 
 
-> LoadImagesFromLocal
 
-![watch](./assets/load-watch.png)
-
-[workflow-4](./workflow/4-loadfromlocal-watcher-workflow.json)
 
 > Consistency Decoder
 
@@ -84,6 +98,13 @@ Add edges to an image.
 ![FeatheredMask](./assets/FlVou_Y6kaGWYoEj1Tn0aTd4AjMI.jpg)
 
 
+
+### Improvement 
+An improvement has been made to directly redirect to GitHub to search for missing nodes when loading the graph.
+
+![node-not-found](./assets/node-not-found.png)
+
+
 ### Models
 [Download CLIPSeg](https://huggingface.co/CIDAS/clipseg-rd64-refined/tree/main), move to : model/clipseg
 

__init__.py

@@ -260,6 +260,7 @@ from .nodes.ImageNode import TransparentImage,LoadImagesFromPath,AreaToMask,Smoo
 from .nodes.Vae import VAELoader,VAEDecode
 from .nodes.ScreenShareNode import ScreenShareNode,FloatingVideo
 from .nodes.Clipseg import CLIPSeg,CombineMasks
+from .nodes.ChatGPT import ChatGPTNode,SessionHistory
 
 # 要导出的所有节点及其名称的字典
 # 注意：名称应全局唯一
@@ -279,17 +280,21 @@ NODE_CLASS_MAPPINGS = {
     "ScreenShare":ScreenShareNode,
     "FloatingVideo":FloatingVideo,
     "CLIPSeg_":CLIPSeg,
-    "CombineMasks_":CombineMasks
+    "CombineMasks_":CombineMasks,
+    "ChatGPT":ChatGPTNode,
+    "SessionHistory":SessionHistory
 }
 
 # 一个包含节点友好/可读的标题的字典
 NODE_DISPLAY_NAME_MAPPINGS = {
-    "RandomPrompt": "Random Prompt #Example Node",
+    "RandomPrompt": "Random Prompt #Mixlab",
     "SplitLongMask":"Splitting a long image into sections",
     "VAELoaderConsistencyDecoder":"Consistency Decoder Loader",
     "VAEDecodeConsistencyDecoder":"Consistency Decoder Decode",
     "ScreenShare":"ScreenShare #Mixlab",
-    "FloatingVideo":"FloatingVideo #Mixlab"
+    "FloatingVideo":"FloatingVideo #Mixlab",
+    "ChatGPT":"ChatGPT #Mixlab",
+    "SessionHistory":"SessionHistory #Mixlab"
 }
 
 # web ui的节点功能

nodes/ChatGPT.py

@@ -0,0 +1,160 @@
+import openai 
+import time
+import urllib.error
+import re,json
+
+#  判断是否是azure服务
+def is_azure_url(url):
+    pattern = r'.*\.azure\.com$'
+    if re.match(pattern, url):
+        return True
+    else:
+        return False
+
+def azure_client(key,url):
+    client = openai.AzureOpenAI(
+        api_key=key,
+    # https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#rest-api-versioning
+    api_version="2023-07-01-preview",
+    # https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource
+    azure_endpoint=url
+    )
+    return client
+
+def openai_client(key,url):
+    client = openai.OpenAI(
+    api_key=key,
+    base_url=url
+    )
+    return client
+
+
+
+def chat(client, model_name,messages ):
+
+        try_count = 0
+        while True:
+            try_count += 1
+            try:
+                response = client.chat.completions.create(
+                    model=model_name,
+                    messages=messages
+                )
+                break
+            except openai.AuthenticationError as ex:
+                raise ex
+            except (urllib.error.HTTPError, openai.OpenAIError) as ex:
+                if try_count >= 3:
+                    raise ex
+                time.sleep(5)
+                continue
+
+        finish_reason = response.choices[0].finish_reason
+        if finish_reason != "stop":
+            raise RuntimeError("API finished with unexpected reason: " + finish_reason)
+
+        content=""
+        try:
+            content=response.choices[0].message.content
+        except:
+            content=response.choices[0].delta['content']
+
+        return content
+
+
+class ChatGPTNode:
+    def __init__(self):
+        # self.__client = OpenAI()
+        self.session_history = []  # 用于存储会话历史的列表
+        self.seed=0
+        self.system_content="You are ChatGPT, a large language model trained by OpenAI. Answer as concisely as possible."
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "api_key":("KEY", {"default": "", "multiline": True}),
+                "api_url":("URL", {"default": "", "multiline": True}),
+                "prompt": ("STRING", {"default": "", "multiline": True}),
+                "system_content": ("STRING", 
+                                   {
+                                       "default": "You are ChatGPT, a large language model trained by OpenAI. Answer as concisely as possible.", 
+                                       "multiline": True
+                                       }),
+                "model": (["gpt-3.5-turbo", "gpt-3.5-turbo-16k-0613", "gpt-4-0613","gpt-4-1106-preview"], 
+                          {"default": "gpt-3.5-turbo"}),
+                "seed": ("INT", {"default": 0, "min": 0, "max": 10000, "step": 1}),
+            },
+        }
+
+    RETURN_TYPES = ("STRING","STRING",)
+    RETURN_NAMES = ("text","session_history",)
+    FUNCTION = "generate_contextual_text"
+    CATEGORY = "Mixlab/GPT"
+    INPUT_IS_LIST = False
+    OUTPUT_IS_LIST = (False,False,)
+
+    
+    def generate_contextual_text(self,
+                                 api_key,
+                                 api_url, 
+                                 prompt, 
+                                 system_content,
+                                   model, 
+                                   seed):
+        print(api_key,
+                                 api_url, 
+                                 prompt, 
+                                 system_content,
+                                   model, 
+                                   seed)
+        # 可以选择保留会话历史以维持上下文记忆
+        # 或者在此处清除会话历史 self.session_history.clear()
+        if seed!=self.seed:
+            self.seed=seed
+            self.session_history=[]
+        
+        # 把系统信息和初始信息添加到会话历史中
+        if system_content:
+            self.system_content=system_content
+            # self.session_history=[]
+            # self.session_history.append({"role": "system", "content": system_content})
+        
+        # 
+        if is_azure_url(api_url):
+            client=azure_client(api_key,api_url)
+        else:
+            client=openai_client(api_key,api_url)
+            print('openai url')
+
+        # 把用户的提示添加到会话历史中
+        # 调用API时传递整个会话历史
+        messages=[{"role": "system", "content": self.system_content}]+self.session_history+[{"role": "user", "content": prompt}]
+        response_content = chat(client,model,messages)
+        
+        self.session_history=self.session_history+[{"role": "user", "content": prompt}]+[{'role':'assistant',"content":response_content}]
+        
+        return (response_content,json.dumps(self.session_history, indent=4),)
+
+
+
+class SessionHistory:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {
+            "required": {
+                "session_history": ("STRING", {"forceInput": True}),
+            }
+        }
+
+    INPUT_IS_LIST = True
+    RETURN_TYPES = ("STRING",)
+    FUNCTION = "run"
+    OUTPUT_NODE = True
+    OUTPUT_IS_LIST = (True,)
+
+    CATEGORY = "Mixlab/GPT"
+
+    def run(self, session_history):
+        # print(session_history)
+        return {"ui": {"text": session_history}, "result": (session_history,)}

nodes/ImageNode.py

@@ -1,6 +1,6 @@
 import numpy as np
 import torch
-from PIL import Image, ImageOps,ImageFilter,ImageEnhance,ImageDraw
+from PIL import Image, ImageOps,ImageFilter,ImageEnhance,ImageDraw,ImageSequence
 from PIL.PngImagePlugin import PngInfo
 import base64,os
 from io import BytesIO
@@ -153,25 +153,48 @@ def get_not_transparent_area(image):
     return (x, y, w, h)
 
 
-
+# 读取不了分层
+def load_psd(image):
+    layers=[]
+    print('load_psd',image.format)
+    if image.format=='PSD':
+        layers = [frame.copy() for frame in ImageSequence.Iterator(image)]
+        print('#PSD',len(layers))
+    else:
+        image = ImageOps.exif_transpose(image) #校对方向
+    layers.append(image)
+    return layers
 
 
 def load_image(fp,white_bg=False):
-    i = Image.open(fp)
-    i = ImageOps.exif_transpose(i)
-    image = i.convert("RGB")
-    image = np.array(image).astype(np.float32) / 255.0
-    image = torch.from_numpy(image)[None,]
-    if 'A' in i.getbands():
-        mask = np.array(i.getchannel('A')).astype(np.float32) / 255.0
-        mask = 1. - torch.from_numpy(mask)
-        if white_bg==True:
-            nw = mask.unsqueeze(0).unsqueeze(-1).repeat(1, 1, 1, 3)
-            # 将mask的黑色部分对image进行白色处理
-            image[nw == 1] = 1.0
-    else:
-        mask = torch.zeros((64,64), dtype=torch.float32, device="cpu")
-    return (image,mask)
+    im = Image.open(fp)
+
+    # ims=load_psd(im)
+    im = ImageOps.exif_transpose(im) #校对方向
+    ims=[im]
+
+    images=[]
+ 
+    for i in ims:
+        image = i.convert("RGB")
+        image = np.array(image).astype(np.float32) / 255.0
+        image = torch.from_numpy(image)[None,]
+        if 'A' in i.getbands():
+            mask = np.array(i.getchannel('A')).astype(np.float32) / 255.0
+            mask = 1. - torch.from_numpy(mask)
+            if white_bg==True:
+                nw = mask.unsqueeze(0).unsqueeze(-1).repeat(1, 1, 1, 3)
+                # 将mask的黑色部分对image进行白色处理
+                image[nw == 1] = 1.0
+        else:
+            mask = torch.zeros((64,64), dtype=torch.float32, device="cpu")
+        
+        images.append({
+            "image":image,
+            "mask":mask
+        })
+        
+    return images
 
 
 # 获取图片s
@@ -183,23 +206,27 @@ def get_images_filepath(f,white_bg=False):
             for file in files:
                 file_path = os.path.join(root, file)
                 try:
-                    (im,mask)=load_image(file_path,white_bg)
-                    images.append({
-                        "image":im,
-                        "mask":mask,
-                        "file_path":file_path
-                    })
+                    imgs=load_image(file_path,white_bg)
+                    for img in imgs:
+                        images.append({
+                            "image":img['image'],
+                            "mask":img['mask'],
+                            "file_path":file_path,
+                            "psd":len(imgs)>1
+                        })
                 except:
                     print('非图片',file_path)
  
     elif os.path.isfile(f):
         try:
-            (im,mask)=load_image(f,white_bg)
-            images.append({
-                        "image":im,
-                        "mask":mask,
-                        "file_path":f
-                    })
+            imgs=load_image(f,white_bg)
+            for img in imgs:
+                images.append({
+                    "image":img['image'],
+                    "mask":img['mask'],
+                    "file_path":file_path,
+                    "psd":len(imgs)>1
+                })
         except:
             print('非图片',f)
     else:
@@ -613,14 +640,15 @@ class LoadImagesFromPath:
         global watcher_folder
         # print('###监听:',watcher_folder,watcher,file_path,result)
 
-        if watcher=='enable':
-            if watcher_folder==None:
-                watcher_folder = FolderWatcher(file_path)
-            
+        if watcher_folder==None:
+            watcher_folder = FolderWatcher(file_path)
+
+        watcher_folder.set_folder_path(file_path)
+        
+        if watcher=='enable': 
             # 在这里可以进行其他操作，监听会在后台持续
             watcher_folder.set_folder_path(file_path)
             watcher_folder.start()
-
         else:
             if watcher_folder!=None:
                 watcher_folder.stop()

nodes/ScreenShareNode.py

@@ -24,9 +24,42 @@ def base64_save(base64_data):
     return (image,mask)
 
 
+# # 把白色部分处理成黑色
+# def convert_to_bw(image):
+#     # 读取图片
+#     # image = Image.open(image_path)
+    
+#     # 获取图片的宽度和高度
+#     width, height = image.size
+    
+#     # 遍历图片的每个像素点
+#     for x in range(width):
+#         for y in range(height):
+#             # 获取当前像素点的RGB值
+#             r, g, b = image.getpixel((x, y))
+            
+#             # 判断当前像素点是否为白色
+#             if r == 255 and g == 255 and b == 255:
+#                 # 将白色部分处理成黑色
+#                 image.putpixel((x, y), (0, 0, 0))
+#             else:
+#                 # 将非白色部分处理成白色
+#                 image.putpixel((x, y), (255, 255, 255))
+    
+#     # 转换为黑白图
+#     mask = image.convert("L")
+    
+#     # # 保存处理后的图片
+#     # image.save("black_white_image.jpg")
+    
+#     # print("图片处理完成！")
+#     return mask
+
+
 def load_image(i,white_bg=False):
     # i = Image.open(fp)
     image = i.convert("RGB")
+
     image = np.array(image).astype(np.float32) / 255.0
     image = torch.from_numpy(image)[None,]
     if 'A' in i.getbands():
@@ -52,7 +85,7 @@ class ScreenShareNode:
             #   "seed": ("INT", {"default": 1, "min": 0, "max": 0xffffffffffffffff}),
                 } }
     
-    RETURN_TYPES = ('IMAGE','MASK','STRING')
+    RETURN_TYPES = ('IMAGE','STRING')
 
     FUNCTION = "run"
 
@@ -65,7 +98,7 @@ class ScreenShareNode:
     def run(self,image_base64,prompt):
         im,mask=base64_save(image_base64)
         # print('##########prompt',prompt)
-        return (im,mask,prompt)
+        return (im,prompt)
     
 
 class FloatingVideo:

nodes/Watcher.py

@@ -48,7 +48,7 @@ class FolderWatcher:
             config['folder_path']=folder_path
             save_to_json(config_json,config)
 
-        # self.observer = Observer()
+        self.observer = None
         self.event_handler = self._create_event_handler()
         self.status = "Not started"
         self.event_type='-'
@@ -97,11 +97,12 @@ class FolderWatcher:
         print('Listening')
 
     def stop(self):
-        self.observer.stop()
-        self.observer.join()
-        self.observer=None
-        self.status = "Stopped"
-        self.event_type='-'
+        if self.observer!=None:
+            self.observer.stop()
+            self.observer.join()
+            self.observer=None
+            self.status = "Stopped"
+            self.event_type='-'
         print('Stopped')
     
   

requirements.txt

@@ -2,4 +2,5 @@ numpy
 pyOpenSSL
 watchdog
 opencv-python-headless
-matplotlib
+matplotlib
+openai

web/javascript/checkVersion_mixlab.js

@@ -3,17 +3,16 @@ import { app } from '../../../scripts/app.js'
 const repoOwner = 'shadowcz007' // 替换为仓库的所有者
 const repoName = 'comfyui-mixlab-nodes' // 替换为仓库的名称
 
-const version='v0.2.2'
-
+const version = 'v0.2.5.1'
 
 fetch(`https://api.github.com/repos/${repoOwner}/${repoName}/releases/latest`)
   .then(response => response.json())
   .then(data => {
     const latestVersion = data.tag_name
-    console.log('Latest release version:', latestVersion )
-    
-    if(latestVersion!=version){
-
+    console.log('Latest release version:', latestVersion)
+    // if (latestVersion === localStorage.getItem('_mixlab_nodes_vesion')) return
+    if (latestVersion != version) {
+      // localStorage.setItem('_mixlab_nodes_vesion', latestVersion)
       app.ui.dialog.show(`<h4 style="font-size: 18px;">${repoName} <br>
       Latest release version: ${latestVersion}</h4>
       <p>Please proceed to the official repository to download the latest version.</p>
@@ -25,12 +24,12 @@ fetch(`https://api.github.com/repos/${repoOwner}/${repoName}/releases/latest`)
   href="https://github.com/shadowcz007/comfyui-mixlab-nodes/releases/">https://github.com/shadowcz007/comfyui-mixlab-nodes/releases</a>
       `)
 
-        // window.alert(
-        //     `Please proceed to the official repository to download the latest version.https://github.com/shadowcz007/comfyui-mixlab-nodes/releases/`
-        //   )
-        //   window.open(
-        //     'https://github.com/shadowcz007/comfyui-mixlab-nodes/releases/'
-        //   )
+      // window.alert(
+      //     `Please proceed to the official repository to download the latest version.https://github.com/shadowcz007/comfyui-mixlab-nodes/releases/`
+      //   )
+      //   window.open(
+      //     'https://github.com/shadowcz007/comfyui-mixlab-nodes/releases/'
+      //   )
     }
   })
   .catch(error => {

web/javascript/gpt_mixlab.js

@@ -0,0 +1,239 @@
+import { app } from '../../../scripts/app.js'
+// import { api } from '../../../scripts/api.js'
+import { ComfyWidgets } from '../../../scripts/widgets.js'
+import { $el } from '../../../scripts/ui.js'
+
+async function getConfig () {
+  let api_host = `${window.location.hostname}:${window.location.port}`
+  let api_base = ''
+  let url = `${window.location.protocol}//${api_host}${api_base}`
+
+  const res = await fetch(`${url}/mixlab`, {
+    method: 'POST'
+  })
+  return await res.json()
+}
+
+function get_position_style (ctx, widget_width, y, node_height) {
+  const MARGIN = 4 // the margin around the html element
+
+  /* Create a transform that deals with all the scrolling and zooming */
+  const elRect = ctx.canvas.getBoundingClientRect()
+  const transform = new DOMMatrix()
+    .scaleSelf(
+      elRect.width / ctx.canvas.width,
+      elRect.height / ctx.canvas.height
+    )
+    .multiplySelf(ctx.getTransform())
+    .translateSelf(MARGIN, MARGIN + y)
+
+  return {
+    transformOrigin: '0 0',
+    transform: transform,
+    left: `0`,
+    top: `0`,
+    cursor: 'pointer',
+    position: 'absolute',
+    maxWidth: `${widget_width - MARGIN * 2}px`,
+    // maxHeight: `${node_height - MARGIN * 2}px`, // we're assuming we have the whole height of the node
+    width: `${widget_width - MARGIN * 2}px`,
+    // height: `${node_height * 0.3 - MARGIN * 2}px`,
+    // background: '#EEEEEE',
+    display: 'flex',
+    flexDirection: 'column',
+    // alignItems: 'center',
+    justifyContent: 'space-around'
+  }
+}
+
+app.registerExtension({
+  name: 'Mixlab.GPT.ChatGPT',
+  async getCustomWidgets (app) {
+    return {
+      KEY (node, inputName, inputData, app) {
+        // console.log('node', inputName, inputData[0])
+        const widget = {
+          type: inputData[0], // the type, CHEESE
+          name: inputName, // the name, slice
+          size: [128, 24], // a default size
+          draw (ctx, node, width, y) {},
+          computeSize (...args) {
+            return [128, 24] // a method to compute the current size of the widget
+          },
+          async serializeValue (nodeId, widgetIndex) {
+            return localStorage.getItem('_mixlab_api_key') || ''
+          }
+        }
+        //  widget.something = something;          // maybe adds stuff to it
+        node.addCustomWidget(widget) // adds it to the node
+        return widget // and returns it.
+      },
+      URL (node, inputName, inputData, app) {
+        // console.log('node', inputName, inputData[0])
+        const widget = {
+          type: inputData[0], // the type, CHEESE
+          name: inputName, // the name, slice
+          size: [128, 24], // a default size
+          draw (ctx, node, width, y) {
+            // a method to draw the widget (ctx is a CanvasRenderingContext2D)
+          },
+          computeSize (...args) {
+            return [128, 24] // a method to compute the current size of the widget
+          },
+          async serializeValue (nodeId, widgetIndex) {
+            return localStorage.getItem('_mixlab_api_url') || ''
+          }
+        }
+        //  widget.something = something;          // maybe adds stuff to it
+        node.addCustomWidget(widget) // adds it to the node
+        return widget // and returns it.
+      }
+    }
+  },
+  async beforeRegisterNodeDef (nodeType, nodeData, app) {
+    // console.log(nodeType.comfyClass)
+    if (nodeType.comfyClass == 'ChatGPT') {
+      const orig_nodeCreated = nodeType.prototype.onNodeCreated
+      nodeType.prototype.onNodeCreated = function () {
+        orig_nodeCreated?.apply(this, arguments)
+
+        console.log('ChatGPT widtget', this.widgets)
+
+        const api_key = this.widgets.filter(w => w.name == 'api_key')[0]
+        const api_url = this.widgets.filter(w => w.name == 'api_url')[0]
+        console.log('api_key', api_key, api_url)
+
+        const widget = {
+          type: 'div',
+          name: 'chatgpt div',
+          draw (ctx, node, widget_width, y, widget_height) {
+            Object.assign(
+              this.div.style,
+              get_position_style(ctx, widget_width, api_key.y, node.size[1])
+            )
+          }
+        }
+
+        widget.div = $el('div', {})
+
+        document.body.appendChild(widget.div)
+
+        const inputKey = document.createElement('input'),
+          inputUrl = document.createElement('input')
+        inputKey.type = 'text'
+        inputUrl.type = 'text'
+
+        inputKey.placeholder = 'Key'
+        inputUrl.placeholder = 'URL'
+
+        inputKey.style = `margin:4px 48px;`
+        inputUrl.style = `margin:4px 48px`
+
+        inputUrl.value = localStorage.getItem('_mixlab_api_url') || ''
+        inputKey.value = localStorage.getItem('_mixlab_api_key') || ''
+
+        widget.div.appendChild(inputKey)
+        widget.div.appendChild(inputUrl)
+
+        inputKey.addEventListener('change', () => {
+          api_key.serializeValue = () => inputKey.value || ''
+          localStorage.setItem('_mixlab_api_key', inputKey.value)
+        })
+
+        inputUrl.addEventListener('change', () => {
+          api_url.serializeValue = () => inputUrl.value || ''
+          localStorage.setItem('_mixlab_api_url', inputUrl.value)
+        })
+
+        /*
+                Add the widget, make sure we clean up nicely, and we do not want to be serialized!
+                */
+        this.addCustomWidget(widget)
+
+        const onRemoved = this.onRemoved
+        this.onRemoved = () => {
+          inputUrl.remove()
+          inputKey.remove()
+          widget.div.remove()
+          return onRemoved?.()
+        }
+
+        this.serialize_widgets = false
+      }
+    }
+  }
+})
+
+app.registerExtension({
+  name: 'Mixlab.GPT.SessionHistory',
+  async beforeRegisterNodeDef (nodeType, nodeData, app) {
+    if (nodeData.name === 'SessionHistory') {
+      function populate (text) {
+        if (this.widgets) {
+          // const pos = this.widgets.findIndex((w) => w.name === "text");
+          // if (pos !== -1) {
+          // 	for (let i = pos; i < this.widgets.length; i++) {
+          // 		this.widgets[i].onRemove?.();
+          // 	}
+          // 	this.widgets.length = pos;
+          // }
+
+          for (let i = 0; i < this.widgets.length; i++) {
+            this.widgets[i].onRemove?.()
+          }
+          this.widgets.length = 0
+        }
+
+        console.log('SessionHistory', this.widgets, text)
+
+        for (const list of text) {
+          const w = ComfyWidgets['STRING'](
+            this,
+            'text',
+            ['STRING', { multiline: true }],
+            app
+          ).widget
+          w.inputEl.readOnly = true
+          w.inputEl.style.opacity = 0.6
+
+          let res = list
+
+          try {
+            res = JSON.stringify(JSON.parse(list), null, 2)
+          } catch (error) {
+            // console.log(list)
+          }
+
+          w.value = res
+        }
+
+        requestAnimationFrame(() => {
+          const sz = this.computeSize()
+          if (sz[0] < this.size[0]) {
+            sz[0] = this.size[0]
+          }
+          if (sz[1] < this.size[1]) {
+            sz[1] = this.size[1]
+          }
+          this.onResize?.(sz)
+          app.graph.setDirtyCanvas(true, false)
+        })
+      }
+
+      // When the node is executed we will be sent the input text, display this in the widget
+      const onExecuted = nodeType.prototype.onExecuted
+      nodeType.prototype.onExecuted = function (message) {
+        onExecuted?.apply(this, arguments)
+        populate.call(this, message.text)
+      }
+
+      const onConfigure = nodeType.prototype.onConfigure
+      nodeType.prototype.onConfigure = function () {
+        onConfigure?.apply(this, arguments)
+        if (this.widgets_values?.length) {
+          populate.call(this, this.widgets_values)
+        }
+      }
+    }
+  }
+})

web/javascript/main_mixlab.js

@@ -29,6 +29,36 @@ async function interrupt () {
   })
 }
 
+async function clipboardWriteImage (win, url) {
+  const canvas = document.createElement('canvas')
+  const ctx = canvas.getContext('2d')
+  // console.log(url)
+  const img = await createImage(url)
+  // console.log(img)
+  canvas.width = img.naturalWidth
+  canvas.height = img.naturalHeight
+
+  ctx.clearRect(0, 0, canvas.width, canvas.height)
+  ctx.drawImage(img, 0, 0)
+  // 将canvas转为blob
+  canvas.toBlob(async blob => {
+    const data = [
+      new ClipboardItem({
+        [blob.type]: blob
+      })
+    ]
+
+    win.navigator.clipboard
+      .write(data)
+      .then(() => {
+        console.log('Image copied to clipboard')
+      })
+      .catch(error => {
+        console.error('Failed to copy image to clipboard:', error)
+      })
+  })
+}
+
 async function uploadFile (file) {
   try {
     const body = new FormData()
@@ -801,6 +831,7 @@ app.registerExtension({
 
         widget.preview = $el('video', {
           controls: true,
+          draggable: true,
           style: {
             width: '100%'
           },
@@ -808,6 +839,28 @@ app.registerExtension({
             'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAwAAAAMCAYAAABWdVznAAAAAXNSR0IArs4c6QAAALZJREFUKFOFkLERwjAQBPdbgBkInECGaMLUQDsE0AkRVRAYWqAByxldPPOWHwnw4OBGye1p50UDSoA+W2ABLPN7i+C5dyC6R/uiAUXRQCs0bXoNIu4QPQzAxDKxHoALOrZcqtiyR/T6CXw7+3IGHhkYcy6BOR2izwT8LptG8rbMiCRAUb+CQ6WzQVb0SNOi5Z2/nX35DRyb/ENazhpWKoGwrpD6nICp5c2qogc4of+c7QcrhgF4Aa/aoAFHiL+RAAAAAElFTkSuQmCC'
         })
 
+        // })
+        // const dropTarget = document.getElementById('your-drop-target-id')
+
+        // dropTarget.addEventListener('dragover', event => {
+        //   event.preventDefault()
+        // })
+
+        // dropTarget.addEventListener('drop', event => {
+        //   event.preventDefault()
+
+        //   const imageUrl = event.dataTransfer.getData('text/plain')
+
+        //   navigator.clipboard
+        //     .writeText(imageUrl)
+        //     .then(() => {
+        //       console.log('Image URL copied to clipboard')
+        //     })
+        //     .catch(error => {
+        //       console.error('Failed to copy image URL to clipboard:', error)
+        //     })
+        // })
+
         widget.canvas = $el('canvas', {
           style: {
             display: 'none'
@@ -818,7 +871,6 @@ app.registerExtension({
           innerText: 'PictureInPicture',
           style: {
             display: 'pictureInPictureEnabled' in document ? 'block' : 'none',
-
             cursor: 'pointer',
             padding: '8px 0',
             fontWeight: '300',
@@ -831,6 +883,28 @@ app.registerExtension({
         widget.card.appendChild(widget.preview)
         widget.card.appendChild(widget.canvas)
 
+        widget.preview.addEventListener('click', event => {
+          const imageUrl = window._mixlab_screen_result || ''
+          // console.log(imageUrl)
+          try {
+            if (imageUrl) clipboardWriteImage(pipWindow, imageUrl)
+          } catch (error) {
+            console.log(error)
+            if (imageUrl) clipboardWriteImage(window, imageUrl)
+          }
+
+          // pipWindow.navigator.permissions
+          //   .query({ name: 'clipboard-write' })
+          //   .then(result => {
+          //     if (result.state === 'granted' || result.state === 'prompt') {
+          //       // 执行复制操作
+
+          //     } else {
+          //       console.error('Clipboard write permission denied')
+          //     }
+          //   })
+        })
+
         // widget.card.appendChild(widget.PictureInPicture)
 
         widget.PictureInPicture.addEventListener('click', async () => {
@@ -874,7 +948,17 @@ app.registerExtension({
           width: calc(100% - 24px);
           margin: 12px;`
 
-          // console.log(pipWindow.document)
+          let inputDiv = document.createElement('div')
+          // inputDiv.style = ``
+          let infoDiv = document.createElement('div')
+          infoDiv.style = `    width: 100%;
+          height: 16px;
+          color: white;
+          margin-bottom: 4px;
+          font-size: 12px;
+          text-shadow: 1px 1px gray;`
+          infoDiv.id = 'info'
+
           // Move the player to the Picture-in-Picture window.
           let input = document.createElement('textarea')
           input.style = `
@@ -928,8 +1012,14 @@ app.registerExtension({
             } else {
               input.style.display = 'none'
             }
+            try {
+              pipWindow.document.querySelector('#info').innerText = ''
+            } catch (error) {
+              console.log(error)
+            }
           })
 
+          // TODO 需要判断是否有screenshare节点，没有的话，不需要添加
           let pauseBtn = document.createElement('butotn')
           pauseBtn.innerText = '⏸'
           pauseBtn.style = `cursor: pointer;height: 24px;margin:4px;
@@ -950,6 +1040,13 @@ app.registerExtension({
               if (w) {
                 w.liveBtn.innerText = 'Live Run'
               }
+
+              try {
+                pipWindow.document.querySelector('#info').innerText =
+                  'Stop Live'
+              } catch (error) {
+                console.log(error)
+              }
             } else {
               pauseBtn.innerText = '⏸'
               let node = this.graph._nodes.filter(
@@ -961,6 +1058,12 @@ app.registerExtension({
                 window._mixlab_stopLive = await startLive(w.liveBtn)
                 console.log('window._mixlab_stopLive', window._mixlab_stopLive)
               }
+
+              try {
+                pipWindow.document.querySelector('#info').innerText = 'Live'
+              } catch (error) {
+                console.log(error)
+              }
             }
           })
 
@@ -972,19 +1075,69 @@ app.registerExtension({
             window._mixlab_screen_prompt =
               window._mixlab_screen_prompt_input || window._mixlab_screen_prompt
             document.querySelector('#queue-button').click()
+
+            try {
+              pipWindow.document.querySelector('#info').innerText =
+                'Update Prompt'
+            } catch (error) {
+              console.log(error)
+            }
+          })
+
+          widget.preview.addEventListener('click', event => {
+            const imageUrl = window._mixlab_screen_result || ''
+            // console.log(imageUrl)
+            try {
+              if (imageUrl) clipboardWriteImage(pipWindow, imageUrl)
+            } catch (error) {
+              console.log(error)
+              if (imageUrl) clipboardWriteImage(window, imageUrl)
+            }
+
+            try {
+              pipWindow.document.querySelector('#info').innerText =
+                'Image copied to clipboard'
+              setTimeout(
+                () =>
+                  (pipWindow.document.querySelector('#info').innerText = ''),
+                8000
+              )
+            } catch (error) {
+              console.log(error)
+            }
+            // pipWindow.navigator.permissions
+            //   .query({ name: 'clipboard-write' })
+            //   .then(result => {
+            //     if (result.state === 'granted' || result.state === 'prompt') {
+            //       // 执行复制操作
+
+            //     } else {
+            //       console.error('Clipboard write permission denied')
+            //     }
+            //   })
           })
 
           pipWindow.document.body.append(widget.preview)
           pipWindow.document.body.append(div)
+          // console.log(pipWindow)
 
           div.appendChild(btnDiv)
           btnDiv.appendChild(btn)
           btnDiv.appendChild(pauseBtn)
           btnDiv.appendChild(promptFinishBtn)
-          div.appendChild(input)
+
+          // 输入框
+          div.appendChild(inputDiv)
+          inputDiv.appendChild(infoDiv)
+          inputDiv.appendChild(input)
 
           input.addEventListener('input', () => {
             window._mixlab_screen_prompt_input = input.value
+            try {
+              pipWindow.document.querySelector('#info').innerText = ''
+            } catch (error) {
+              console.log(error)
+            }
           })
 
           input.addEventListener('keydown', handleKeyDown)
@@ -1000,6 +1153,13 @@ app.registerExtension({
                   window._mixlab_screen_prompt_input ||
                   window._mixlab_screen_prompt
                 document.querySelector('#queue-button').click()
+
+                try {
+                  pipWindow.document.querySelector('#info').innerText =
+                    'Update Prompt'
+                } catch (error) {
+                  console.log(error)
+                }
               }
             }
           }
@@ -1035,7 +1195,11 @@ app.registerExtension({
             const videoTrack = stream.getVideoTracks()[0]
 
             video.preview.srcObject = new MediaStream([videoTrack])
-            video.preview.play()
+            try {
+              video.preview.play()
+            } catch (error) {
+              console.log(error)
+            }
 
             // 检查浏览器是否支持画中画模式
             if ('pictureInPictureEnabled' in document) {
@@ -1068,7 +1232,7 @@ app.registerExtension({
           const context = canvas.getContext('2d')
 
           if (message?.images_) {
-            const base64 = message.images_[0]
+            window._mixlab_screen_result = `data:image/png;base64,${message.images_[0]}`
             const image = new Image()
             image.onload = function () {
               canvas.width = image.width
@@ -1076,7 +1240,7 @@ app.registerExtension({
               context.drawImage(image, 0, 0)
             }
             // console.log(`data:image/jpeg;base64,${base64}`)
-            image.src = `data:image/jpeg;base64,${base64}`
+            image.src = window._mixlab_screen_result
           }
           const onRemoved = this.onRemoved
           this.onRemoved = () => {
@@ -1457,7 +1621,7 @@ const node = {
           )[0]
           if (my_workflow?.data) {
             // app.loadGraphData(my_workflow.data)
-            localStorage.setItem('workflow',JSON.stringify(my_workflow.data));
+            localStorage.setItem('workflow', JSON.stringify(my_workflow.data))
           }
         })
     }

web/javascript/ui.js

@@ -1,39 +0,0 @@
-import { app } from '../../../scripts/app.js'
-
-const missingNodeGithub = missingNodeTypes => {
-  return Array.from(
-    new Set(missingNodeTypes)
-  ,n=>{
-    const url = `https://github.com/search?q=${n}&type=code`;
-
-   return `<li style="color: white;
-   background: black;
-   padding: 8px;
-   font-size: 12px;">${n}<a href="${url}" target="_blank"> 🔗</a></li>`;
-  })
-}
-
-app.showMissingNodesError = function (missingNodeTypes, hasAddedNodes = true) {
-//   console.log('###MIXLAB', missingNodeTypes, hasAddedNodes)
-  this.ui.dialog.show(
-    `When loading the graph, the following node types were not found: <ul>${ missingNodeGithub(missingNodeTypes)
-      .join('')}</ul>${
-      hasAddedNodes
-        ? 'Nodes that have failed to load will show as red on the graph.'
-        : ''
-    }`
-  )
-  this.logging.addEntry('Comfy.App', 'warn', {
-    MissingNodes: missingNodeTypes
-  })
-}
-
-// app.ui.dialog.show = function (html) {
-//   console.log('###MIXLAB', html)
-//   if (typeof html === 'string') {
-//     this.textElement.innerHTML = html
-//   } else {
-//     this.textElement.replaceChildren(html)
-//   }
-//   this.element.style.display = 'flex'
-// }

web/javascript/ui_mixlab.js

@@ -0,0 +1,88 @@
+import { app } from '../../../scripts/app.js'
+
+async function getCustomnodeMappings (mode = 'url') {
+  // mode = "local";
+  let api_host = `${window.location.hostname}:${window.location.port}`
+  let api_base = ''
+  let url = `${window.location.protocol}//${api_host}${api_base}`
+
+  const response = await fetch(`${url}/customnode/getmappings?mode=${mode}`)
+  const data = await response.json()
+
+  let nodes = {}
+  try {
+    for (let url in data) {
+      let n = data[url]
+      for (let node of n[0]) {
+        nodes[node] = { url, title: n[1].title_aux }
+      }
+    }
+  } catch (error) {}
+
+  return nodes
+}
+
+const missingNodeGithub = (missingNodeTypes, nodesMap) => {
+  let ts = {}
+
+  Array.from(new Set(missingNodeTypes), n => {
+    if (nodesMap[n]) {
+      let title = nodesMap[n].title
+      if (!ts[title]) {
+        ts[title] = {
+          title,
+          nodes: {},
+          url: nodesMap[n].url
+        }
+      }
+      ts[title].nodes[n] = 1
+    } else {
+      ts[n] = {
+        title: n,
+        nodes: {},
+        url: `https://github.com/search?q=${n}&type=code`
+      }
+      ts[n].nodes[n] = 1
+    }
+  })
+
+  return Array.from(Object.values(ts), n => {
+    const url = n.url
+    return `<li style="color: white;
+   background: black;
+   padding: 8px;
+   font-size: 12px;">${n.title}<a href="${url}" target="_blank"> 🔗</a></li>`
+  })
+}
+
+app.showMissingNodesError = async function (
+  missingNodeTypes,
+  hasAddedNodes = true
+) {
+  const nodesMap = await getCustomnodeMappings()
+  console.log('#nodesMap', nodesMap)
+  //   console.log('###MIXLAB', missingNodeTypes, hasAddedNodes)
+  this.ui.dialog.show(
+    `When loading the graph, the following node types were not found: <ul>${missingNodeGithub(
+      missingNodeTypes,
+      nodesMap
+    ).join('')}</ul>${
+      hasAddedNodes
+        ? 'Nodes that have failed to load will show as red on the graph.'
+        : ''
+    }`
+  )
+  this.logging.addEntry('Comfy.App', 'warn', {
+    MissingNodes: missingNodeTypes
+  })
+}
+
+// app.ui.dialog.show = function (html) {
+//   console.log('###MIXLAB', html)
+//   if (typeof html === 'string') {
+//     this.textElement.innerHTML = html
+//   } else {
+//     this.textElement.replaceChildren(html)
+//   }
+//   this.element.style.display = 'flex'
+// }

web/javascript/watcher_mixlab.js

@@ -14,7 +14,9 @@ async function getConfig () {
   return await res.json()
 }
 
-if(!window._mixlab_screen_prompt) window._mixlab_screen_prompt="beautiful scenery nature glass bottle landscape,under water"
+if (!window._mixlab_screen_prompt)
+  window._mixlab_screen_prompt =
+    'beautiful scenery nature glass bottle landscape,under water'
 
 function get_position_style (ctx, widget_width, y, node_height) {
   const MARGIN = 4 // the margin around the html element
@@ -59,7 +61,28 @@ app.registerExtension({
           name: inputName, // the name, slice
           size: [128, 24], // a default size
           draw (ctx, node, width, y) {
-            // a method to draw the widget (ctx is a CanvasRenderingContext2D)
+            // // 绘制文件图标的函数
+            // function drawFileIcon () {
+            //   // 清空画布
+            //   // ctx.clearRect(0, 0, canvas.width, canvas.height)
+
+            //   // 绘制文件外框
+            //   ctx.fillStyle = '#000'
+            //   ctx.fillRect(5, 5, 40, 40)
+
+            //   // 绘制文件夹图标
+            //   ctx.fillStyle = '#f00'
+            //   ctx.fillRect(10, 15, 30, 20)
+
+            //   // 绘制监听符号
+            //   ctx.beginPath()
+            //   ctx.arc(30, 35, 5, 0, 2 * Math.PI)
+            //   ctx.fillStyle = '#00f'
+            //   ctx.fill()
+            // }
+
+            // // 调用绘制函数
+            // drawFileIcon()
           },
           computeSize (...args) {
             return [128, 24] // a method to compute the current size of the widget
@@ -101,11 +124,7 @@ app.registerExtension({
       nodeType.prototype.onNodeCreated = function () {
         orig_nodeCreated?.apply(this, arguments)
 
-         
-        // console.log(
-        //   'watch widtget',
-        //   this.widgets.filter(w => w.name == 'watcher')[0]
-        // )
+        console.log('watch widtget', this.widgets)
 
         const watcher = this.widgets.filter(w => w.name == 'watcher')[0]
 
@@ -133,7 +152,6 @@ app.registerExtension({
           }
         }
 
-      
         // 上次路径填充
         getConfig().then(json => {
           let w = this.widgets.filter(w => w.name == 'file_path')[0]
@@ -142,7 +160,6 @@ app.registerExtension({
           }
           // console.log(json.event_type)
           window._mixlab_file_path_watcher = json.event_type
-          
         })
 
         /*

workflow/3-FloatVideo-workflow.json

@@ -1,6 +1,6 @@
 {
-  "last_node_id": 21,
-  "last_link_id": 45,
+  "last_node_id": 22,
+  "last_link_id": 48,
   "nodes": [
     {
       "id": 7,
@@ -14,7 +14,7 @@
         "1": 200
       },
       "flags": {},
-      "order": 7,
+      "order": 4,
       "mode": 0,
       "inputs": [
         {
@@ -89,12 +89,12 @@
         504,
         33
       ],
-      "size": [
-        400,
-        200
-      ],
+      "size": {
+        "0": 400,
+        "1": 200
+      },
       "flags": {},
-      "order": 6,
+      "order": 8,
       "mode": 0,
       "inputs": [
         {
@@ -105,7 +105,7 @@
         {
           "name": "text",
           "type": "STRING",
-          "link": 43,
+          "link": 48,
           "widget": {
             "name": "text"
           }
@@ -295,7 +295,7 @@
         "Node name for S&R": "KSampler"
       },
       "widgets_values": [
-        613900833686415,
+        1115769645491668,
         "randomize",
         4,
         1.6,
@@ -311,94 +311,24 @@
         338,
         786
       ],
-      "size": [
-        210,
-        246
-      ],
+      "size": {
+        "0": 210,
+        "1": 246
+      },
       "flags": {},
-      "order": 4,
+      "order": 6,
       "mode": 0,
       "inputs": [
         {
           "name": "images",
           "type": "IMAGE",
-          "link": 37
+          "link": 46
         }
       ],
       "properties": {
         "Node name for S&R": "PreviewImage"
       }
     },
-    {
-      "id": 19,
-      "type": "ScreenShare",
-      "pos": [
-        -55,
-        512
-      ],
-      "size": {
-        "0": 325.3117370605469,
-        "1": 459.7692565917969
-      },
-      "flags": {},
-      "order": 2,
-      "mode": 0,
-      "outputs": [
-        {
-          "name": "IMAGE",
-          "type": "IMAGE",
-          "links": [
-            37,
-            42
-          ],
-          "shape": 3,
-          "slot_index": 0
-        },
-        {
-          "name": "MASK",
-          "type": "MASK",
-          "links": null,
-          "shape": 3
-        },
-        {
-          "name": "STRING",
-          "type": "STRING",
-          "links": [
-            43
-          ],
-          "shape": 3,
-          "slot_index": 2
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "ScreenShare"
-      }
-    },
-    {
-      "id": 20,
-      "type": "FloatingVideo",
-      "pos": [
-        2041,
-        277
-      ],
-      "size": [
-        315,
-        106
-      ],
-      "flags": {},
-      "order": 12,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "images",
-          "type": "IMAGE",
-          "link": 41
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "FloatingVideo"
-      }
-    },
     {
       "id": 6,
       "type": "LoraLoader",
@@ -411,7 +341,7 @@
         "1": 126
       },
       "flags": {},
-      "order": 8,
+      "order": 5,
       "mode": 0,
       "inputs": [
         {
@@ -463,7 +393,7 @@
         "1": 98
       },
       "flags": {},
-      "order": 3,
+      "order": 2,
       "mode": 0,
       "outputs": [
         {
@@ -515,13 +445,13 @@
         "1": 58
       },
       "flags": {},
-      "order": 5,
+      "order": 7,
       "mode": 0,
       "inputs": [
         {
           "name": "image",
           "type": "IMAGE",
-          "link": 42
+          "link": 47
         }
       ],
       "outputs": [
@@ -541,6 +471,70 @@
       "widgets_values": [
         512
       ]
+    },
+    {
+      "id": 22,
+      "type": "ScreenShare",
+      "pos": [
+        -63,
+        483
+      ],
+      "size": {
+        "0": 315,
+        "1": 98
+      },
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "outputs": [
+        {
+          "name": "IMAGE",
+          "type": "IMAGE",
+          "links": [
+            46,
+            47
+          ],
+          "shape": 3,
+          "slot_index": 0
+        },
+        {
+          "name": "STRING",
+          "type": "STRING",
+          "links": [
+            48
+          ],
+          "shape": 3,
+          "slot_index": 1
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "ScreenShare"
+      }
+    },
+    {
+      "id": 20,
+      "type": "FloatingVideo",
+      "pos": [
+        1928,
+        295
+      ],
+      "size": [
+        315,
+        58
+      ],
+      "flags": {},
+      "order": 12,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "link": 41
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "FloatingVideo"
+      }
     }
   ],
   "links": [
@@ -744,14 +738,6 @@
       0,
       "LATENT"
     ],
-    [
-      37,
-      19,
-      0,
-      2,
-      0,
-      "IMAGE"
-    ],
     [
       38,
       16,
@@ -784,22 +770,6 @@
       0,
       "IMAGE"
     ],
-    [
-      42,
-      19,
-      0,
-      18,
-      0,
-      "IMAGE"
-    ],
-    [
-      43,
-      19,
-      2,
-      8,
-      1,
-      "STRING"
-    ],
     [
       44,
       6,
@@ -815,6 +785,30 @@
       6,
       1,
       "CLIP"
+    ],
+    [
+      46,
+      22,
+      0,
+      2,
+      0,
+      "IMAGE"
+    ],
+    [
+      47,
+      22,
+      0,
+      18,
+      0,
+      "IMAGE"
+    ],
+    [
+      48,
+      22,
+      1,
+      8,
+      1,
+      "STRING"
     ]
   ],
   "groups": [],