v0.29.0 支持把输出显示到comfyui背景（TouchDesigner 风格）

Appinfo supports outputting to the background, enhancing the performance features of ComfyUI.

.github/workflows/publish.yml

@@ -0,0 +1,21 @@
+name: Publish to Comfy registry
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - "pyproject.toml"
+
+jobs:
+  publish-node:
+    name: Publish Custom Node to registry
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Publish Custom Node
+        uses: Comfy-Org/publish-node-action@main
+        with:
+          ## Add your own personal access token to your Github Repository secrets and reference it here.
+          personal_access_token: ${{ secrets.REGISTRY_ACCESS_TOKEN }}

README.md

@@ -10,7 +10,10 @@ ChatGPT 节点支持 Local LLM（llama.cpp），Phi3、llama3 都可以直接一
 
 - 右键菜单支持 text-to-text，方便对 prompt 词补全
 
-强烈推荐：[Phi-3-mini-4k-instruct-GGUF](https://huggingface.co/lmstudio-community/Phi-3-mini-4k-instruct-GGUF/tree/main)，备选：[llama3_if_ai_sdpromptmkr_q2k](https://hf-mirror.com/impactframes/llama3_if_ai_sdpromptmkr_q2k/tree/main)
+强烈推荐：
+[Phi-3-mini-4k-instruct-function-calling-GGUF](https://huggingface.co/nold/Phi-3-mini-4k-instruct-function-calling-GGUF)
+
+[Phi-3-mini-4k-instruct-GGUF](https://huggingface.co/lmstudio-community/Phi-3-mini-4k-instruct-GGUF/tree/main)，备选：[llama3_if_ai_sdpromptmkr_q2k](https://hf-mirror.com/impactframes/llama3_if_ai_sdpromptmkr_q2k/tree/main)
 
 - 右键菜单支持 image-to-text，使用多模态模型，多模态使用 [llava-phi-3-mini-gguf](https://huggingface.co/xtuner/llava-phi-3-mini-gguf/tree/main)，注意需要把llava-phi-3-mini-mmproj-f16.gguf也下载
 
@@ -37,6 +40,7 @@ ChatGPT 节点支持 Local LLM（llama.cpp），Phi3、llama3 都可以直接一
 - 发布为 app 的 workflow，可以在右键里再次编辑了
 - web app 可以设置分类，在 comfyui 右键菜单可以编辑更新 web app
 - 支持动态提示
+- 支持把输出显示到comfyui背景（TouchDesigner 风格）
 
 ![](./assets/微信图片_20240421205440.png)
 

__init__.py

@@ -819,12 +819,12 @@ async def start_local_llm(data):
 async def my_hander_method(request):
     data =await request.json()
     # print(data)
+    if llama_port and llama_model and llama_chat_format:
+        return web.json_response({"port":llama_port,"model":llama_model,"chat_format":llama_chat_format} )
     try:
         result=await start_local_llm(data)
     except:
-        result={
-            {"port":None,"model":"","llama_cpp_error":True}
-        }
+        result= {"port":None,"model":"","llama_cpp_error":True}
         print('start_local_llm error')
 
     return web.json_response(result)

data/extension-node-map.json

@@ -5201,7 +5201,7 @@
             "title_aux": "ComfyUI Stable Video Diffusion"
         }
     ],
-    "https://github.com/thedyze/save-image-extended-comfyui": [
+    "https://github.com/audioscavenger/save-image-extended-comfyui": [
         [
             "SaveImageExtended"
         ],
@@ -5705,4 +5705,4 @@
             "title_aux": "SDXLCustomAspectRatio"
         }
     ]
-}
+}

nodes/RembgNode.py

@@ -467,10 +467,14 @@ class BriaRMBG(nn.Module):
 
 
 
+def get_U2NET_model_path():
+    try:
+        return folder_paths.get_folder_paths('rembg')[0]
+    except:
+        return os.path.join(folder_paths.models_dir, "rembg")
+    
 
-
-
-U2NET_HOME=os.path.join(folder_paths.models_dir, "rembg")
+U2NET_HOME=get_U2NET_model_path()
 os.environ["U2NET_HOME"] = U2NET_HOME
 
 global _available
@@ -528,7 +532,7 @@ except:
 
 
 def run_briarmbg(images=[]):
-    mroot=os.path.join(folder_paths.models_dir, "rembg")
+    mroot=U2NET_HOME
     m=os.path.join(mroot,'briarmbg.pth')
     if os.path.exists(m)==False:
         # 下载

nodes/TripoSR.py

@@ -5,13 +5,20 @@ from PIL import Image
 import numpy as np
 import torch
 
-from folder_paths import get_filename_list, get_full_path, get_save_image_path, get_output_directory,models_dir
+from folder_paths import get_folder_paths, get_full_path, get_save_image_path, get_output_directory,models_dir
 from comfy.model_management import get_torch_device
 from .tsr.system import TSR
 
 import comfy.utils
 
-triposr_model_path=path.join(models_dir,'triposr/model.ckpt')
+
+def get_triposr_model_path():
+    try:
+        return path.join(get_folder_paths('triposr')[0],'model.ckpt')
+    except:
+        return path.join(path.join(models_dir, "triposr"),'model.ckpt')
+    
+triposr_model_path=get_triposr_model_path()
 
 
 # Tensor to PIL

nodes/Utils.py

@@ -284,7 +284,7 @@ class FloatSlider:
                 }
     
     RETURN_TYPES = ("FLOAT",) 
-    RETURN_NAMES = ('weight(0-1)',)
+    RETURN_NAMES = ('FLOAT',)
     FUNCTION = "run"
 
     CATEGORY = "♾️Mixlab/Input"
@@ -297,9 +297,7 @@ class FloatSlider:
             number = min_value
         elif number > max_value:
             number = max_value
-        scaled_number = (number - min_value) / (max_value - min_value)
-        return (scaled_number,)
-    
+        return (number,)    
   
 class IntNumber:
     @classmethod

nodes/Video.py

@@ -87,7 +87,7 @@ def split_video(video_path, video_segment_frames, transition_frames, output_dir)
 
 folder_paths.folder_names_and_paths["video_formats"] = (
     [
-        os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "video_formats"),
+        os.path.join(os.path.dirname(os.path.abspath(__file__)), ".", "video_formats"),
     ],
     [".json"]
 )
@@ -433,6 +433,7 @@ class VideoCombine_Adv:
             ffmpeg_formats = ["video/"+x[:-5] for x in folder_paths.get_filename_list("video_formats")]
         else:
             ffmpeg_formats = []
+        # ffmpeg_formats =["video/"+x for x in  ['webm', 'mp4', 'mkv']]
         return {
             "required": {
                 "image_batch": ("IMAGE",),

nodes/video_formats/av1-webm.json

@@ -0,0 +1,10 @@
+{
+    "main_pass":
+    [
+        "-n", "-c:v", "libsvtav1",
+        "-pix_fmt", "yuv420p10le",
+        "-crf", "23"
+    ],
+     "extension": "webm",
+     "environment": {"SVT_LOG": "1"}
+}

nodes/video_formats/h264-mp4.json

@@ -0,0 +1,9 @@
+{
+    "main_pass":
+    [
+        "-n", "-c:v", "libx264",
+        "-pix_fmt", "yuv420p",
+        "-crf", "19"
+    ],
+     "extension": "mp4"
+}

nodes/video_formats/h265-mp4.json

@@ -0,0 +1,11 @@
+{
+    "main_pass":
+    [
+        "-n", "-c:v", "libx265",
+        "-pix_fmt", "yuv420p10le",
+        "-preset", "medium",
+        "-crf", "22",
+        "-x265-params", "log-level=quiet"
+    ],
+     "extension": "mp4"
+}

nodes/video_formats/webm.json

@@ -0,0 +1,9 @@
+{
+    "main_pass":
+    [
+        "-n",
+        "-pix_fmt", "yuv420p",
+        "-crf", "23"
+    ],
+     "extension": "webm"
+}

pyproject.toml

@@ -0,0 +1,15 @@
+[project]
+name = "comfyui-mixlab-nodes"
+description = "3D, ScreenShareNode & FloatingVideoNode, SpeechRecognition & SpeechSynthesis, GPT, LoadImagesFromLocal, Layers, Other Nodes, ..."
+version = "0.28.3"
+license = "MIT"
+dependencies = ["numpy", "pyOpenSSL", "watchdog", "opencv-python-headless", "matplotlib", "openai", "simple-lama-inpainting", "clip-interrogator==0.6.0", "transformers>=4.36.0", "lark-parser", "imageio-ffmpeg", "rembg[gpu]", "omegaconf==2.3.0", "Pillow>=9.5.0", "einops==0.7.0", "trimesh>=4.0.5", "huggingface-hub", "scikit-image"]
+
+[project.urls]
+Repository = "https://github.com/shadowcz007/comfyui-mixlab-nodes"
+#  Used by Comfy Registry https://comfyregistry.org
+
+[tool.comfy]
+PublisherId = "shadow"
+DisplayName = "comfyui-mixlab-nodes"
+Icon = ""

web/index.html

@@ -24,6 +24,7 @@
             margin-left: 5%;
             user-select: none;
             margin-top: 32px;
+            margin-bottom: 120px;
         }
 
         .apps {
@@ -65,9 +66,9 @@
             box-shadow: 0px 0px 10px 10px #fbe9f0
         }
 
-        .card:hover {
+        /* .card:hover {
             box-shadow: 0px 0px 10px 10px #e9fbfa
-        }
+        } */
 
         .card h5 {
             font-size: 14px;
@@ -145,15 +146,6 @@
             margin-top: 4px;
         }
 
-        .status {
-            background: black;
-            color: white;
-            display: flex;
-            width: fit-content;
-            padding: 4px;
-            font-size: 12px;
-
-        }
 
         .description {
             display: flex;
@@ -263,18 +255,44 @@
             margin-top: 12px;
         }
 
+        .run_div {
+            position: fixed;
+            bottom: 28px;
+            display: flex;
+            left: 144px;
+            z-index: 100;
+            /* align-items: center; */
+            /* justify-content: space-around; */
+            /* width: calc(50% - 100px); */
+            flex-direction: column;
+            background-color: #f7f7f7;
+            box-shadow: 3px 3px 8px #cacaca;
+            border-radius: 8px;
+        }
+
+        .status {
+            /* background: #e7e7e7; */
+            color: black;
+            display: flex;
+            width: 120px;
+            padding-left: 11px;
+            font-size: 12px;
+            border-radius: 8px;
+            justify-content: flex-start;
+            align-items: center;
+            height: 48px;
+        }
+
         .run_btn {
             background: black;
             color: white;
-            width: 88px;
-            height: 88px;
-            position: fixed;
-            bottom: 72px;
-            left: calc(50% - 44px);
-            border-radius: 100%;
+            /* width: calc(50% - 100px); */
+            min-width: 200px;
+            max-width: 460px;
+            height: 48px;
+            border-radius: 8px;
             cursor: pointer;
             border: 3px solid;
-            z-index: 100;
         }
 
         button:hover {
@@ -916,6 +934,7 @@
                     tempCanvas.width = dim.width;
                     tempCanvas.height = dim.height;
 
+                    //todo 获取Image更新后的数据(暂不支持image的修改)
 
                     for (const layer of Layers.get_layers()) {
                         if (layer.name !== 'Image_' + data.id) {
@@ -955,10 +974,15 @@
                     // // 获取读取的文件内容，即 Blob 对象
                     let hashId = await calculateImageHash(fileBlob)
 
-                    if (hashId == window._appData.data[data.id].hashId) return;
+                    if (hashId == window._appData.data[data.id].hashId) {
+                        document.body.querySelector('.app').style.display = 'flex'
+                        document.body.querySelector('#author').style.display = 'block'
+                        return;
+                    }
 
+                    //底图
                     const { url: imgurl } = await uploadImage(base64ToBlob(data.options.defaultImage))
-
+                    //mask
                     let { url, name } = await uploadMask(fileBlob, imgurl);
                     // 在这里可以对 Blob 对象进行进一步处理
                     // imageElement.src = url;
@@ -966,8 +990,6 @@
                     window._appData.data[data.id].hashId = hashId;
 
                     // console.log("上传的文件：", url, data.id, name);
-
-
                     //更新图片
                     const canvas = document.createElement("canvas");
                     canvas.width = dim.width;
@@ -975,7 +997,6 @@
 
                     const ctx = canvas.getContext('2d');
 
-
                     const defaultImage = await createImage(data.options.defaultImage)
                     ctx.drawImage(defaultImage, 0, 0, dim.width, dim.height);
 
@@ -1001,22 +1022,24 @@
             let layers1 = Layers.get_layers();
 
             //通过layer.link.src 判断是否图片更新
-            console.log(layers1.filter(l => l.name == 'Image_' + data.id)[0]?.link?.currentSrc !== image.src)
+            console.log(layers1.filter(l => l.name == 'Image_' + data.id)[0]?.link?.currentSrc !== data.options.defaultImage)
             if (layers1.filter(l => l.name == 'Image_' + data.id)[0]
-                && layers1.filter(l => l.name == 'Image_' + data.id)[0].link.src !== image.src) {
+                && layers1.filter(l => l.name == 'Image_' + data.id)[0].link.src !== data.options.defaultImage) {
                 //清空图层
                 removeAllLayer();
                 layers1 = [];
                 await sleep()
             }
-            console.log(layers1, layers1.length)
+
+            let im = await createImage(data.options.defaultImage)
+            // console.log(layers1, layers1.length)
             if (!layers1.filter(l => l.name == 'Image_' + data.id)[0]) {
-                addImage(0, 'Image_' + data.id, image)
+                addImage(0, 'Image_' + data.id, im)
             }
 
             if (isMask) {
                 if (!layers1.filter(l => l.name == 'Mask_' + data.id)[0]) {
-                    addMask(1, 'Mask_' + data.id, image)
+                    addMask(1, 'Mask_' + data.id, im)
                 }
             }
 
@@ -1210,7 +1233,7 @@
         }
 
 
-        function createOutputs(outputData, link) {
+        async function createOutputs(outputData, link) {
             const url = new URL(window.location.href);
             const params = new URLSearchParams(url.search);
             const innerApp = params.get("innerApp");
@@ -1331,20 +1354,21 @@
                     "TransparentImage"].includes(node.class_type)) {
 
                     const url = node.options?.defaultImage || window._appData?.icon || base64Df;
-                    let a = document.createElement('a');
+                    let a = document.createElement('div');
                     a.id = `output_${node.id}`
-                    a.setAttribute('data-pswp-width', "200");
-                    a.setAttribute('data-pswp-height', "200");
-                    a.setAttribute('target', "_blank");
-                    a.setAttribute('href', url);
-                    a.setAttribute('title', node.title);
 
-                    let img = new Image();
-                    // img;
-                    img.src = url;
+                    let img = await createImage(url)
+
                     a.appendChild(img)
+                    // a.setAttribute('data-pswp-width', img.naturalWidth);
+                    // a.setAttribute('data-pswp-height', img.naturalHeight);
+                    // a.setAttribute('target', "_blank");
+                    // a.setAttribute('href', url);
+                    // a.setAttribute('title', node.title);
+
                     output_card.appendChild(a);
                     isShowImageFn = true;
+
                 }
 
                 //3d 
@@ -2392,7 +2416,7 @@
             })
         }
 
-        function createUI(data, share = true) {
+        async function createUI(data, share = true) {
             // appData.input, appData.output, appData.seed, share, appData.link
             if (!data) return
             const { input: inputData, output: outputData, data: workflow, seed, seedTitle, link, name } = data;
@@ -2533,30 +2557,44 @@
             }
 
 
-            statusDiv.appendChild(status);
+            // statusDiv.appendChild(status);
             statusDiv.appendChild(seeds);
 
             // 创建输入框
             var input1 = createInputs(inputData)
 
-            var output = createOutputs(outputData, link)
+            var output = await createOutputs(outputData, link)
 
             // 创建提交按钮
+            let submitDiv = document.createElement('div');
+            submitDiv.appendChild(statusDiv);
+
+            let submitDivBtn = document.createElement('div');
+            submitDivBtn.style.display = 'flex'
+            submitDiv.appendChild(submitDivBtn);
+
+            submitDivBtn.appendChild(status);
+
+            submitDiv.className = 'run_div';
             var submitButton = document.createElement('button');
             submitButton.textContent = 'Create';
             submitButton.className = 'run_btn'
 
             // 将所有UI元素添加到页面中
-            leftDiv.appendChild(titleDiv);
-            leftDiv.appendChild(iconDes);
+            // leftDiv.appendChild(titleDiv);
+            // leftDiv.appendChild(iconDes);
             // leftDiv.appendChild(des);
-            leftDiv.appendChild(statusDiv);
+            // leftDiv.appendChild(statusDiv);
             leftDiv.appendChild(input1);
 
-            if (typeof (data.data) == 'object') mainDiv.appendChild(submitButton);
+            submitDivBtn.appendChild(submitButton);
+            if (typeof (data.data) == 'object') mainDiv.appendChild(submitDiv);
 
             rightDiv.appendChild(output);
 
+            // mainDiv.appendChild(titleDiv);
+            // mainDiv.appendChild(iconDes);
+
             mainDiv.appendChild(leftDetails);
             mainDiv.appendChild(rightDiv);
 
@@ -2904,7 +2942,7 @@
         async function createApp(appData, share = true) {
             // console.log(appData)
             // 使用示例：
-            var ui = createUI(appData, share);
+            var ui = await createUI(appData, share);
 
             // 更新标题
             ui.title.update(appData.name || 'Mixlab APP');
@@ -3044,7 +3082,8 @@
             const lightbox = new PhotoSwipeLightbox({
                 gallery: '.output_card',
                 children: 'a',
-                pswpModule: () => import('/extensions/comfyui-mixlab-nodes/lib/photoswipe.esm.min.js')
+                pswpModule: () => import('/extensions/comfyui-mixlab-nodes/lib/photoswipe.esm.min.js'),
+                // appendToEl: document.querySelector('#result')
             });
             lightbox.on('uiRegister', function () {
                 lightbox.pswp.ui.registerElement({
@@ -3089,8 +3128,8 @@
             if (appData.author) {
                 let div = document.body.querySelector('#author');
                 if (appData.author.link) div.href = appData.author.link
-                div.style = `z-index:20;display: flex;flex-direction: column;position: fixed;bottom: 12px;right: 24px;cursor: pointer;text-decoration: none;color: black;`
-                div.innerHTML = `<p style="font-size:12px">Author:</p>
+                div.style = `z-index:20;display: flex;flex-direction: column;position: fixed;bottom: 24px;left: 24px;cursor: pointer;text-decoration: none;color: black;`
+                div.innerHTML = `<p style="font-size:12px;margin: 8px 0;">Author:</p>
                 <div style="display: flex;"> <img style="width:32px;height:32px;border-radius: 100%;" 
                 src="${appData.author.avatar || base64Df}"/>
                 <p style="margin-left:8px;font-size:12px;font-weight:800">${appData.author.name || '-'}</p></div>`

web/javascript/app_mixlab.js

@@ -2,6 +2,8 @@ import { app } from '../../../scripts/app.js'
 import { $el } from '../../../scripts/ui.js'
 import { api } from '../../../scripts/api.js'
 
+import { td_bg } from './td_background.js'
+console.log('td_bg', td_bg)
 //本机安装的插件节点全集
 window._nodesAll = null
 
@@ -503,6 +505,22 @@ app.registerExtension({
           }
         })
 
+        //td bg
+        const tdBG = document.createElement('button')
+        tdBG.innerText = 'TDBG'
+        tdBG.style = style
+        tdBG.style.marginLeft = '12px'
+
+        tdBG.addEventListener('click', () => {
+          td_bg.toggle();
+          if(td_bg.running){
+            tdBG.style.background = 'yellow'
+          }else{
+            tdBG.style.background = 'transparent'
+          }
+
+        })
+
         // author
         let author = document.createElement('div')
         // author.style=`display: flex`
@@ -659,6 +677,7 @@ app.registerExtension({
 
         btns.appendChild(btn)
         btns.appendChild(download)
+        btns.appendChild(tdBG)
 
         document.body.appendChild(widget.div)
         this.addCustomWidget(widget)
@@ -687,9 +706,8 @@ app.registerExtension({
           }
 
           const div = this.widgets.filter(w => w.div)[0].div
-          Array.from(
-            div.querySelectorAll('button'),
-            b => (b.style.background = 'yellow')
+          Array.from(div.querySelectorAll('button'), b =>
+            b.innerText != 'TDBG' ? (b.style.background = 'yellow') : ''
           )
         } catch (error) {}
       }

web/javascript/chat.js

@@ -2,7 +2,7 @@ async function* completion (url, messages, controller) {
   let data = {
     model: 'gpt-3.5-turbo-16k',
     messages,
-    temperature: 0.6,
+    temperature: 0.05,
     stream: true
   }
   // if (imageNode) {

web/javascript/checkVersion_mixlab.js

@@ -3,7 +3,7 @@ import { app } from '../../../scripts/app.js'
 const repoOwner = 'shadowcz007' // 替换为仓库的所有者
 const repoName = 'comfyui-mixlab-nodes' // 替换为仓库的名称
 
-const version = 'v0.28.1'
+const version = 'v0.29.0'
 
 fetch(`https://api.github.com/repos/${repoOwner}/${repoName}/releases/latest`)
   .then(response => response.json())

web/javascript/td_background.js

@@ -0,0 +1,323 @@
+// touchdesigner的背景效果，把appinfo的输出，选择一张图片作为背景
+
+window._bg_img = null
+
+/**
+ * draws the back canvas (the one containing the background and the connections)
+ * @method drawBackCanvas
+ **/
+LGraphCanvas.prototype.drawBackCanvas = function () {
+  var canvas = this.bgcanvas
+  if (
+    canvas.width != this.canvas.width ||
+    canvas.height != this.canvas.height
+  ) {
+    canvas.width = this.canvas.width
+    canvas.height = this.canvas.height
+  }
+
+  if (!this.bgctx) {
+    this.bgctx = this.bgcanvas.getContext('2d')
+  }
+  var ctx = this.bgctx
+  if (ctx.start) {
+    ctx.start()
+  }
+
+  var viewport = this.viewport || [0, 0, ctx.canvas.width, ctx.canvas.height]
+
+  //clear
+  if (this.clear_background) {
+    ctx.clearRect(viewport[0], viewport[1], viewport[2], viewport[3])
+  }
+
+  //show subgraph stack header
+  if (this._graph_stack && this._graph_stack.length) {
+    ctx.save()
+    var parent_graph = this._graph_stack[this._graph_stack.length - 1]
+    var subgraph_node = this.graph._subgraph_node
+    ctx.strokeStyle = subgraph_node.bgcolor
+    ctx.lineWidth = 10
+    ctx.strokeRect(1, 1, canvas.width - 2, canvas.height - 2)
+    ctx.lineWidth = 1
+    ctx.font = '40px Arial'
+    ctx.textAlign = 'center'
+    ctx.fillStyle = subgraph_node.bgcolor || '#AAA'
+    var title = ''
+    for (var i = 1; i < this._graph_stack.length; ++i) {
+      title += this._graph_stack[i]._subgraph_node.getTitle() + ' >> '
+    }
+    ctx.fillText(title + subgraph_node.getTitle(), canvas.width * 0.5, 40)
+    ctx.restore()
+  }
+
+  var bg_already_painted = false
+  if (this.onRenderBackground) {
+    bg_already_painted = this.onRenderBackground(canvas, ctx)
+  }
+
+  //reset in case of error
+  if (!this.viewport) {
+    ctx.restore()
+    ctx.setTransform(1, 0, 0, 1, 0, 0)
+  }
+  this.visible_links.length = 0
+
+  if (this.graph) {
+    //apply transformations
+    ctx.save()
+    this.ds.toCanvasContext(ctx)
+
+    //render BG
+    if (
+      this.ds.scale < 1 &&
+      !bg_already_painted &&
+      this.clear_background_color
+    ) {
+      ctx.fillStyle = this.clear_background_color
+      ctx.fillRect(
+        this.visible_area[0],
+        this.visible_area[1],
+        this.visible_area[2],
+        this.visible_area[3]
+      )
+    }
+
+    // 主要修改
+    if (this.background_image && this.ds.scale > 0.5 && !bg_already_painted) {
+      if (this.zoom_modify_alpha) {
+        //使得 alpha 越接近0时变化越缓慢。
+        let alpha = (1.0 - 0.5 / this.ds.scale) * this.editor_alpha
+        ctx.globalAlpha = Math.min(Math.max(0, Math.sqrt(alpha)), 1)
+        // console.log((1.0 - 0.5 / this.ds.scale) * this.editor_alpha)
+      } else {
+        ctx.globalAlpha = this.editor_alpha
+      }
+      ctx.imageSmoothingEnabled = ctx.imageSmoothingEnabled = false // ctx.mozImageSmoothingEnabled =
+      if (!this._bg_img || this._bg_img.name != this.background_image) {
+        this._bg_img = new Image()
+        this._bg_img.name = this.background_image
+        this._bg_img.src = this.background_image
+        var that = this
+        this._bg_img.onload = function () {
+          that.draw(true, true)
+        }
+      }
+
+      var pattern = null
+      if (this._pattern == null && this._bg_img.width > 0) {
+        pattern = ctx.createPattern(this._bg_img, 'repeat')
+        this._pattern_img = this._bg_img
+        this._pattern = pattern
+      } else {
+        pattern = this._pattern
+      }
+
+      if (pattern) {
+        ctx.fillStyle = pattern
+        ctx.fillRect(
+          this.visible_area[0],
+          this.visible_area[1],
+          this.visible_area[2],
+          this.visible_area[3]
+        )
+        ctx.fillStyle = 'transparent'
+      }
+
+      ctx.globalAlpha = 1.0
+      ctx.imageSmoothingEnabled = ctx.imageSmoothingEnabled = true //= ctx.mozImageSmoothingEnabled
+    }
+
+    //groups
+    if (this.graph._groups.length && !this.live_mode) {
+      this.drawGroups(canvas, ctx)
+    }
+
+    if (this.onDrawBackground) {
+      this.onDrawBackground(ctx, this.visible_area)
+    }
+    if (this.onBackgroundRender) {
+      //LEGACY
+      console.error(
+        'WARNING! onBackgroundRender deprecated, now is named onDrawBackground '
+      )
+      this.onBackgroundRender = null
+    }
+
+    //DEBUG: show clipping area
+    //ctx.fillStyle = "red";
+    //ctx.fillRect( this.visible_area[0] + 10, this.visible_area[1] + 10, this.visible_area[2] - 20, this.visible_area[3] - 20);
+
+    //bg
+    if (this.render_canvas_border) {
+      ctx.strokeStyle = '#235'
+      ctx.strokeRect(0, 0, canvas.width, canvas.height)
+    }
+
+    if (this.render_connections_shadows) {
+      ctx.shadowColor = '#000'
+      ctx.shadowOffsetX = 0
+      ctx.shadowOffsetY = 0
+      ctx.shadowBlur = 6
+    } else {
+      ctx.shadowColor = 'rgba(0,0,0,0)'
+    }
+
+    //draw connections
+    if (!this.live_mode) {
+      this.drawConnections(ctx)
+    }
+
+    ctx.shadowColor = 'rgba(0,0,0,0)'
+
+    //restore state
+    ctx.restore()
+  }
+
+  if (ctx.finish) {
+    ctx.finish()
+  }
+
+  this.dirty_bgcanvas = false
+  this.dirty_canvas = true //to force to repaint the front canvas with the bgcanvas
+}
+
+function imgToCanvasBase64 (img) {
+  const canvas = document.createElement('canvas')
+  const ctx = canvas.getContext('2d')
+  canvas.width = img.width
+  canvas.height = img.height
+  ctx.drawImage(img, 0, 0)
+  const base64 = canvas.toDataURL('image/png')
+
+  return base64
+}
+
+// 使用示例
+function convertImageToBase64 (img) {
+  //   const img = new Image()
+  //   img.src = 'path/to/your/image.jpg' // 替换为你的图片路径
+  // console.log('convertImageToBase64',img)
+  try {
+    const base64 = imgToCanvasBase64(img)
+    return base64
+  } catch (error) {
+    console.error(error)
+  }
+}
+
+function getInputsAndOutputs () {
+  const outputs =
+    `PreviewImage,SaveImage,TransparentImage,VHS_VideoCombine,VideoCombine_Adv,Image Save,SaveImageAndMetadata_`.split(
+      ','
+    )
+
+  let outputsId = []
+
+  for (let node of app.graph._nodes) {
+    if (outputs.includes(node.type)) {
+      outputsId.push(node.id)
+    }
+  }
+
+  return outputsId
+}
+
+function getRandomElement (arr) {
+  const randomIndex = Math.floor(Math.random() * arr.length)
+  return arr[randomIndex]
+}
+
+async function getBG () {
+  var outputs = []
+
+  for (let id of app.graph
+    .getNodeById(50)
+    .widgets.filter(w => w.name === 'output_ids')[0]
+    .value.split('\n')) {
+    if (getInputsAndOutputs().map(Number).includes(Number(id))) {
+      if (app.graph.getNodeById(id).imgs && app.graph.getNodeById(id).imgs[0]) {
+        let b = convertImageToBase64(app.graph.getNodeById(id).imgs[0])
+        // console.log(b)
+        outputs.push(b)
+      }
+    }
+  }
+
+  var BACKGROUND_IMAGE = getRandomElement(outputs),
+    CLEAR_BACKGROUND_COLOR = 'rgba(0,0,0,0.9)'
+
+  if (!window._bg_img) {
+    window._bg_img = app.canvas._bg_img.src
+  }
+  // let img=new Image();
+  // img.src=BACKGROUND_IMAGE;
+
+  //去掉透明度过度
+  // app.canvas.zoom_modify_alpha=false;
+  //整体透明度
+  app.canvas.editor_alpha = 1.1
+  //   app.canvas._pattern=ctx.createPattern(img, "no-repeat");
+  app.canvas.updateBackground(BACKGROUND_IMAGE, CLEAR_BACKGROUND_COLOR)
+  app.canvas.draw(true, true)
+}
+
+class BgRunner {
+  constructor () {
+    this.intervalId = null
+    this.running = false
+  }
+
+  // 要运行的方法
+  bg () {
+    console.log('方法bg正在运行')
+    getBG()
+  }
+
+  // 启动bg方法每秒运行一次
+  start () {
+    if (!this.running) {
+      this.intervalId = setInterval(() => this.bg(), 1500)
+      this.running = true
+    }
+  }
+
+  // 停止bg方法的运行
+  stop () {
+    if (this.running) {
+      clearInterval(this.intervalId)
+      this.intervalId = null
+      this.running = false
+
+      if (window._bg_img) {
+        var BACKGROUND_IMAGE = window._bg_img,
+          CLEAR_BACKGROUND_COLOR = 'rgba(0,0,0,1)'
+        app.canvas.editor_alpha = 1
+
+        app.canvas.updateBackground(BACKGROUND_IMAGE, CLEAR_BACKGROUND_COLOR)
+        app.canvas.draw(true, true)
+      }
+    }
+  }
+
+  // 切换start和stop
+  toggle () {
+    if (this.running) {
+      this.stop()
+    } else {
+      this.start()
+    }
+  }
+
+  // 获取运行状态
+  isRunning () {
+    return this.running
+  }
+}
+
+// 示例用法
+// const runner = new BgRunner();
+// runner.start();
+// setTimeout(() => runner.stop(), 5000);
+
+export const td_bg = new BgRunner()

web/javascript/ui_mixlab.js

@@ -26,7 +26,35 @@ function showTextByLanguage (key, json) {
 }
 
 //系统prompt
-const systemPrompt = `You are a prompt creator, your task is to create prompts for the user input request, the prompts are image descriptions that include keywords for (an adjective, type of image, framing/composition, subject, subject appearance/action, environment, lighting situation, details of the shoot/illustration, visuals aesthetics and artists), brake keywords by comas, provide high quality, non-verboose, coherent, brief, concise, and not superfluous prompts, the subject from the input request must be included verbatim on the prompt,the prompt is english`
+// const systemPrompt = `You are a prompt creator, your task is to create prompts for the user input request, the prompts are image descriptions that include keywords for (an adjective, type of image, framing/composition, subject, subject appearance/action, environment, lighting situation, details of the shoot/illustration, visuals aesthetics and artists), brake keywords by comas, provide high quality, non-verboose, coherent, brief, concise, and not superfluous prompts, the subject from the input request must be included verbatim on the prompt,the prompt is english`
+
+let tool ={
+  "name": "create_prompt",
+  "description": "Create a prompt with a given subject, content, and style based on user input for image descriptions.",
+  "parameter": {
+    "type": "object",
+    "properties": {
+      "subject": {
+        "type": "string",
+        "description": "The subject of the prompt, included verbatim from the input request.",
+        "required": true
+      },
+      "content": {
+        "type": "string",
+        "description": "The content of the prompt, primarily focusing on the scene and objects, including keywords for adjective, type of image, framing/composition, subject appearance/action, and environment.",
+        "required": true
+      },
+      "style": {
+        "type": "string",
+        "description": "The style of the prompt, including lighting situation, details of the shoot/illustration, visual aesthetics, and artists. Ensure it is high quality, non-verbose, coherent, brief, concise, and not superfluous.",
+        "required": true
+      }
+    }
+  }
+}
+
+const systemPrompt=`You are a helpful assistant with access to the following functions. Use them if required - ${JSON.stringify(tool,null,2)}`
+
 
 if (!localStorage.getItem('_mixlab_system_prompt')) {
   localStorage.setItem('_mixlab_system_prompt', systemPrompt)
@@ -135,7 +163,7 @@ async function createMenu () {
 
     // appsButton.onclick = () =>
     appsButton.onclick = async () => {
-      if (window._mixlab_llamacpp) {
+      if (window._mixlab_llamacpp&&window._mixlab_llamacpp.model&&window._mixlab_llamacpp.model.length>0) {
         //显示运行的模型
         createModelsModal([
           window._mixlab_llamacpp.url,
@@ -770,6 +798,22 @@ async function fetchReadmeContent (url) {
   }
 }
 
+async function startLLM (model) {
+  let res = await start_llama(model)
+  window._mixlab_llamacpp = res||{ model:[] }
+
+  localStorage.setItem('_mixlab_llama_select', res?.model||'')
+
+  if (document.body.querySelector('#mixlab_chatbot_by_llamacpp')&&window._mixlab_llamacpp?.url) {
+    document.body
+      .querySelector('#mixlab_chatbot_by_llamacpp')
+      .setAttribute('title', window._mixlab_llamacpp.url)
+  }
+  if (document.body.querySelector('#llm_status_btn')&&window._mixlab_llamacpp) {
+    document.body.querySelector('#llm_status_btn').innerText = window._mixlab_llamacpp.model
+  }
+}
+
 function createModelsModal (models) {
   var div =
     document.querySelector('#model-modal') || document.createElement('div')
@@ -825,7 +869,7 @@ function createModelsModal (models) {
       color: var(--descrip-text);
       font-size: 18px;
       display: flex;
-      align-items: center;
+      align-items: flex-start;
       flex: 1;
       overflow: hidden;
       text-decoration: none;
@@ -844,6 +888,27 @@ function createModelsModal (models) {
   })
   linkIcon.style.padding = '4px'
 
+  const statusIcon = document.createElement('small')
+  statusIcon.textContent = showTextByLanguage('Status', {
+    Status: 'OFF'
+  })
+  statusIcon.id = 'llm_status_btn'
+  statusIcon.style=`padding: 4px;
+  background-color: rgb(102, 255, 108);
+  color: black;
+  font-size: 12px;
+  margin-left: 12px;`
+  if (window._mixlab_llamacpp?.url) {
+    statusIcon.textContent = window._mixlab_llamacpp.model
+    statusIcon.style.backgroundColor = '#66ff6c'
+    statusIcon.style.color = 'black'
+  } else {
+  }
+  statusIcon.addEventListener('click', e => {
+    e.stopPropagation()
+    // startLLM()
+  })
+
   const n_gpu = document.createElement('input')
   n_gpu.type = 'number'
   n_gpu.setAttribute('min', -1)
@@ -878,20 +943,22 @@ function createModelsModal (models) {
   const title = document.createElement('p')
   title.innerText = 'Models'
   title.style = `font-size: 18px;
-  margin-right: 8px;`
+  margin-right: 8px;
+  margin-top: 0;`
 
   const left_d = document.createElement('div')
   left_d.style = `display: flex;
   justify-content: center;
-  align-items: center;
-  font-size: 12px;`
+  align-items: flex-start;
+  font-size: 12px;
+  flex-direction: column; `
   left_d.appendChild(title)
-
+  title.appendChild(statusIcon)
   left_d.appendChild(linkIcon)
-
+  left_d.appendChild(n_gpu_div)
   headTitleElement.appendChild(left_d)
 
-  headTitleElement.appendChild(n_gpu_div)
+  // headTitleElement.appendChild(n_gpu_div)
 
   //重启
   const reStart = document.createElement('small')
@@ -899,7 +966,11 @@ function createModelsModal (models) {
     restart: '重启'
   })
 
-  reStart.style.padding = '4px'
+  reStart.style=`padding: 8px;
+  font-size: 16px;
+  outline: 1px solid;
+  padding-top: 4px;
+  padding-bottom: 4px;`
 
   headTitleElement.appendChild(reStart)
 
@@ -958,29 +1029,22 @@ function createModelsModal (models) {
 
   modalContent.appendChild(input)
 
-  for (const m of models) {
-    let d = document.createElement('div')
-    d.innerText = m
-    d.className = `mix_tag`
+  if (!window._mixlab_llamacpp||(window._mixlab_llamacpp?.model?.length==0)) {
+    for (const m of models) {
+      let d = document.createElement('div')
+      d.innerText = `${showTextByLanguage('Run', {
+        Run: '运行'
+      })} ${m}`
+      d.className = `mix_tag`
 
-    if (!window._mixlab_llamacpp) {
       d.addEventListener('click', async e => {
         e.stopPropagation()
         div.remove()
-        let res = await start_llama(m)
-        window._mixlab_llamacpp = res
-
-        localStorage.setItem('_mixlab_llama_select', res.model)
-
-        if (document.body.querySelector('#mixlab_chatbot_by_llamacpp')) {
-          document.body
-            .querySelector('#mixlab_chatbot_by_llamacpp')
-            .setAttribute('title', window._mixlab_llamacpp.url)
-        }
+        startLLM(m)
       })
-    }
 
-    modalContent.appendChild(d)
+      modalContent.appendChild(d)
+    }
   }
   modal.appendChild(modalContent)
 
@@ -994,8 +1058,8 @@ function createModelsModal (models) {
   cursor: pointer;
   font-size: 12px;
   color: white;`
-  helpInfo.href="https://discord.gg/cXs9vZSqeK"
-  helpInfo.target="_blank"
+  helpInfo.href = 'https://discord.gg/cXs9vZSqeK'
+  helpInfo.target = '_blank'
   modal.appendChild(helpInfo)
 
   document.body.appendChild(div)
@@ -1349,6 +1413,8 @@ app.registerExtension({
           .querySelector('#mixlab_chatbot_by_llamacpp')
           .setAttribute('title', res.url)
       })
+    }else{
+      startLLM('')
     }
 
     LGraphCanvas.prototype.helpAboutNode = async function (node) {
@@ -1357,7 +1423,13 @@ app.registerExtension({
           ? nodesMap
           : await getCustomnodeMappings('url')
 
-      console.log('node & node map', node, nodesMap, nodesMap[node.type])
+      console.log(
+        '%c### node & node map',
+        'background: yellow; color: black',
+        node,
+        nodesMap,
+        nodesMap[node.type]
+      )
       let repo = nodesMap[node.type]
       if (repo) {
         let markdown = await fetchReadmeContent(repo.url)
@@ -1388,6 +1460,7 @@ app.registerExtension({
         let userInput = widget.value
         widget.value = widget.value.trim()
         widget.value += '\n'
+        let jsonStr="";
         try {
           await completion_(
             window._mixlab_llamacpp.url + '/v1/chat/completions',
@@ -1402,6 +1475,7 @@ app.registerExtension({
             t => {
               // console.log(t)
               widget.value += t
+              jsonStr+=t
             }
           )
         } catch (error) {
@@ -1427,13 +1501,27 @@ app.registerExtension({
                 t => {
                   // console.log(t)
                   widget.value += t
+                  jsonStr+=t
                 }
               )
             })
           }
         }
 
-        widget.value = widget.value.trim()
+        let json=null;
+
+        try {
+          json=JSON.parse(jsonStr.trim())
+        } catch (error) {
+          json=JSON.parse(jsonStr.trim()+"}")
+        }
+
+        if(json){
+          widget.value = [json.subject,json.content,json.style].join('\n')
+        }else{
+          widget.value = widget.value.trim()
+        }
+        
       }
     }