0.25.0 text-to-text for prompt

README.md

@@ -1,8 +1,7 @@
 > 适配了最新版comfyui的py3.11 ，torch 2.1.2+cu121
-
 > [Mixlab nodes discord](https://discord.gg/cXs9vZSqeK)
 
-#### 
+#### `相关插件推荐`
 [comfyui-sd-prompt-mixlab](https://github.com/shadowcz007/comfyui-sd-prompt-mixlab)
 
 [comfyui-Image-reward](https://github.com/shadowcz007/comfyui-Image-reward)
@@ -13,8 +12,8 @@
 
 <!-- [comfyui-CLIPSeg](https://github.com/shadowcz007/comfyui-CLIPSeg) -->
 
-#### 
-最新：ChatGPT节点支持 Local LLM（llama.cpp），Phi3、llama3 都可以直接一个节点运行了。
+##### `最新`：
+ChatGPT节点支持 Local LLM（llama.cpp），Phi3、llama3 都可以直接一个节点运行了。
 
 Model download,move to :```models/llamafile/```
 
@@ -23,7 +22,9 @@ Model download,move to :```models/llamafile/```
 备选：[llama3_if_ai_sdpromptmkr_q2k](https://hf-mirror.com/impactframes/llama3_if_ai_sdpromptmkr_q2k/tree/main)
 
 
-
+> 右键菜单支持 text-to-text，方便对prompt词补全
+![](./assets/prompt_ai_setup.png)
+![](./assets/prompt-ai.png)
 
 
 ## 🚀🚗🚚🏃 Workflow-to-APP 
@@ -107,6 +108,14 @@ Model download,move to :```models/llamafile/```
 备选：[llama3_if_ai_sdpromptmkr_q2k](https://hf-mirror.com/impactframes/llama3_if_ai_sdpromptmkr_q2k/tree/main)
 
 
+> 如果碰到安装失败，可以尝试手动安装
+```
+../../../python_embeded/python.exe -s -m pip install llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu121
+
+../../../python_embeded/python.exe -s -m pip install llama-cpp-python[server]
+
+```
+
 
 ## Prompt
 > PromptSlide

__init__.py

@@ -11,6 +11,10 @@ import logging
 from comfy.cli_args import args
 python = sys.executable
 
+llama_port=None
+llama_model=""
+
+from .nodes.ChatGPT import get_llama_models,get_llama_model_path
 
 from server import PromptServer
 
@@ -43,7 +47,7 @@ def is_installed(package, package_overwrite=None):
             print(f"Couldn't install\nCommand: {command}\nError code: {result.returncode}")
     else:
         print(package+'## OK')
-        
+     
 try:
     import OpenSSL
 except ImportError:
@@ -437,19 +441,26 @@ async def new_start(self, address, port, verbose=True, call_on_start=None):
 
         ssl_context = None
         scheme = "http"
-        # 跟着本体修改
-        if args.tls_keyfile and args.tls_certfile:
+        try:
+            # 跟着本体修改
+            if args.tls_keyfile and args.tls_certfile:
                 scheme = "https"
                 ssl_context = ssl.SSLContext(protocol=ssl.PROTOCOL_TLS_SERVER, verify_mode=ssl.CERT_NONE)
                 ssl_context.load_cert_chain(certfile=args.tls_certfile,
-                                keyfile=args.tls_keyfile)
-        else:
-            # 如果没传，则自动创建
+                                    keyfile=args.tls_keyfile)
+            else:
+                # 如果没传，则自动创建
+                import ssl
+                crt, key = create_for_https()
+                ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
+                ssl_context.load_cert_chain(crt, key)
+        except:
             import ssl
             crt, key = create_for_https()
             ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
             ssl_context.load_cert_chain(crt, key)
 
+
         success = False
         for i in range(11):  # 尝试最多11次
             if await check_port_available(address, http_port + 1 + i):
@@ -480,10 +491,14 @@ async def new_start(self, address, port, verbose=True, call_on_start=None):
             # print("\033[93mTo see the GUI go to: https://{}:{}\033[0m".format(address, https_port))
 
         if call_on_start is not None:
-            if scheme=='https':
-                call_on_start(scheme,address, https_port)
-            else:
-                call_on_start(scheme,address, http_port)
+            try:
+                if scheme=='https':
+                    call_on_start(scheme,address, https_port)
+                else:
+                    call_on_start(scheme,address, http_port)
+            except:
+                call_on_start(address,http_port)
+            
 
     except Exception as e:
         print(f"Error starting the server: {e}")
@@ -494,8 +509,6 @@ async def new_start(self, address, port, verbose=True, call_on_start=None):
         # webbrowser.open(f"https://{address}")
         # webbrowser.open(f"http://{address}:{port}")
 
-
-
 PromptServer.start=new_start
 
 # 创建路由表
@@ -590,13 +603,19 @@ async def nodes_map_hander(request):
 async def get_checkpoints(request):
     data = await request.json()
     t="checkpoints"
+    names=[]
     try:
         t=data['type']
+        names = folder_paths.get_filename_list(t)
     except Exception as e:
         print('/mixlab/folder_paths',False,e)
-    
-    names = folder_paths.get_filename_list(t)
 
+    try:
+        if data['type']=='llamafile':
+            names=get_llama_models()
+    except:
+        print("llamafile none")
+    
     return web.json_response({"names":names,"types":list(folder_paths.folder_names_and_paths.keys())})
 
 
@@ -617,15 +636,87 @@ async def post_prompt_result(request):
     
     return web.json_response({"result":res})
 
-# 扩展api接口
-# from server import PromptServer
-# from aiohttp import web
 
-# @routes.post('/ws_image')
-# async def my_hander_method(request):
-#     post = await request.post()
-#     x = post.get("something")
-#     return web.json_response({})
+async def start_local_llm(data):
+    global llama_port,llama_model
+    if llama_port and llama_model:
+        return {"port":llama_port,"model":llama_model}
+
+    import threading
+    import uvicorn
+    from llama_cpp.server.app import create_app
+    from llama_cpp.server.settings import (
+                Settings,
+                ServerSettings,
+                ModelSettings,
+                ConfigFileSettings,
+            )
+    
+    if not "model" in data and "model_path" in data:
+        data['model']= os.path.basename(data["model_path"])
+        model=data["model_path"]
+
+    elif "model" in data:
+        model=get_llama_model_path(data['model'])
+    
+
+    address="127.0.0.1"
+    port=9090
+    success = False
+    for i in range(11):  # 尝试最多11次
+        if await check_port_available(address, port + i):
+            port = port + i
+            success = True
+            break
+
+    if success == False:
+        return {"port":None,"model":""}
+    
+    
+    server_settings=ServerSettings(host=address,port=port)
+
+    app = create_app(
+                server_settings=server_settings,
+                model_settings=[ModelSettings(model=model,n_gpu_layers=9999,n_ctx=4098)],
+            )
+
+    def run_uvicorn():
+        uvicorn.run(
+                app,
+                host=os.getenv("HOST", server_settings.host),
+                port=int(os.getenv("PORT", server_settings.port)),
+                ssl_keyfile=server_settings.ssl_keyfile,
+                ssl_certfile=server_settings.ssl_certfile,
+            )
+
+    # 创建一个子线程
+    thread = threading.Thread(target=run_uvicorn)
+
+    # 启动子线程
+    thread.start()
+
+    llama_port=port
+    llama_model=data['model']
+
+    return  {"port":llama_port,"model":llama_model}
+
+# llam服务的开启
+@routes.post('/mixlab/start_llama')
+async def my_hander_method(request):
+    data =await request.json()
+    # print(data)
+    result=await start_local_llm(data)
+
+    return web.json_response(result)
+
+# parser.add_argument("--llm-model", type=str, help="Path to Model file (gguf). Run the Local LLM by MixlabNodes")
+# if args.llm_model and os.path.exists(args.llm_model):
+#     start_local_llm({
+#         "model_path":args.llm_model
+#     })
+#     print("Local LLM Start")
+
+
 
 
 # 导入节点

install.bat

@@ -10,6 +10,12 @@ if exist "%python_exec%" (
     for /f "delims=" %%i in (%requirements_txt%) do (
         %python_exec% -s -m pip install "%%i" -i https://pypi.tuna.tsinghua.edu.cn/simple
     )
+
+    %python_exec% -s -m pip install llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu121
+
+    %python_exec% -s -m pip install llama-cpp-python[server]
+    
+
 ) else (
     echo Installing with system Python
     for /f "delims=" %%i in (%requirements_txt%) do (

nodes/ChatGPT.py

@@ -91,10 +91,16 @@ def ZhipuAI_client(key):
 def phi_sort(lst):
     return sorted(lst, key=lambda x: x.lower().count('phi'), reverse=True)
 
+def get_llama_path():
+    try:
+        return folder_paths.get_folder_paths('llamafile')[0]
+    except:
+        return os.path.join(folder_paths.models_dir, "llamafile")
+
 def get_llama_models():
     res=[]
 
-    model_path=os.path.join(folder_paths.models_dir, "llamafile")
+    model_path=get_llama_path()
     if os.path.exists(model_path):
         files = os.listdir(model_path)
         for file in files:
@@ -106,7 +112,7 @@ def get_llama_models():
 llama_modes_list=get_llama_models()
 
 def get_llama_model_path(file_name):
-    model_path=os.path.join(folder_paths.models_dir, "llamafile")
+    model_path=get_llama_path()
     mp=os.path.join(model_path,file_name)
     return mp
 
@@ -129,6 +135,12 @@ def llama_cpp_client(file_name):
             if result.returncode == 0:
                 print("#install success")
                 from llama_cpp import Llama
+
+                subprocess.run([sys.executable, '-s', '-m', 'pip', 
+                                     'install', 
+                                     'llama-cpp-python[server]'
+                                     ], capture_output=True, text=True)
+
             else:
                 print("#install error")
             
@@ -137,14 +149,15 @@ def llama_cpp_client(file_name):
     except:
         print("#install llama-cpp-python error")
 
-    mp=get_llama_model_path(file_name)
-    # file_name=get_llama_models()[0]
-    # model_path=os.path.join(folder_paths.models_dir, "llamafile")
-    # mp=os.path.join(model_path,file_name)
+    if file_name:
+        mp=get_llama_model_path(file_name)
+        # file_name=get_llama_models()[0]
+        # model_path=os.path.join(folder_paths.models_dir, "llamafile")
+        # mp=os.path.join(model_path,file_name)
 
-    llm = Llama(model_path=mp, chat_format="chatml")
+        llm = Llama(model_path=mp, chat_format="chatml",n_gpu_layers=-1,n_ctx=512)
 
-    return llm
+        return llm
 
     
 

nodes/ImageNode.py

@@ -29,23 +29,122 @@ def opencv_to_pil(image):
     return pil_image
 
 
-def composite_images(foreground, background, mask,is_multiply_blend=False):
+def composite_images(foreground, background, mask,is_multiply_blend=False,position="overall"):
     width,height=foreground.size
  
     bg_image=background
+
+    bwidth,bheight=bg_image.size
+
     # 按z-index排序
-    layer = {
-            "x":0,
-            "y":0,
-            "width":width,
-            "height":height,
-            "z_index":88,
-            "scale_option":'overall',
-            "image":foreground,
-            "mask":mask
-    }
+    if position=="overall":
+        layer = {
+                "x":0,
+                "y":0,
+                "width":bwidth,
+                "height":bheight,
+                "z_index":88,
+                "scale_option":'overall',
+                "image":foreground,
+                "mask":mask
+        }
+
+    elif position=='center_bottom':
+        
+        scale = int(bwidth*0.25) / width
+        new_height = int(height * scale)
+
+        layer = {
+                "x":int(bwidth*0.75*0.5),
+                "y":bheight-new_height-24,
+                "width":int(bwidth*0.25),
+                "height":int(bheight*0.25),
+                "z_index":88,
+                "scale_option":'width',
+                "image":foreground,
+                "mask":mask
+        }
+
+    elif position=='right_bottom':
+        
+        scale = int(bwidth*0.25) / width
+        new_height = int(height * scale)
+
+        layer = {
+                "x":bwidth-int(bwidth*0.25)-24,
+                "y":bheight-new_height-24,
+                "width":int(bwidth*0.25),
+                "height":int(bheight*0.25),
+                "z_index":88,
+                "scale_option":'width',
+                "image":foreground,
+                "mask":mask
+        }
+
+
+    elif position=='center_top':
+        
+        scale = int(bwidth*0.25) / width
+        new_height = int(height * scale)
+
+        layer = {
+                "x":int( bwidth*0.75*0.5),
+                "y":24,
+                "width":int(bwidth*0.25),
+                "height":int(bheight*0.25),
+                "z_index":88,
+                "scale_option":'width',
+                "image":foreground,
+                "mask":mask
+        }
+
+    elif position=='right_top':
+        
+        scale = int(bwidth*0.25) / width
+        new_height = int(height * scale)
+
+        layer = {
+                "x":bwidth-int(bwidth*0.25)-24,
+                "y":24,
+                "width":int(bwidth*0.25),
+                "height":int(bheight*0.25),
+                "z_index":88,
+                "scale_option":'width',
+                "image":foreground,
+                "mask":mask
+        }
+    elif position=='left_top':
+        
+        scale = int(bwidth*0.25) / width
+        new_height = int(height * scale)
+
+        layer = {
+                "x":24,
+                "y":24,
+                "width":int(bwidth*0.25),
+                "height":int(bheight*0.25),
+                "z_index":88,
+                "scale_option":'width',
+                "image":foreground,
+                "mask":mask
+        }
+    elif position=='left_bottom':
+        
+        scale = int(bwidth*0.25) / width
+        new_height = int(height * scale)
+
+        layer = {
+                "x":24,
+                "y":bheight-new_height-24,
+                "width":int(bwidth*0.25),
+                "height":int(bheight*0.25),
+                "z_index":88,
+                "scale_option":'width',
+                "image":foreground,
+                "mask":mask
+        }
                 
-    width, height = bg_image.size
+    # width, height = bg_image.size
 
     layer_image=layer['image']
     layer_mask=layer['mask']
@@ -729,6 +828,27 @@ def merge_images(bg_image, layer_image, mask, x, y, width, height, scale_option,
         # 整体缩放
         layer_image = layer_image.resize((width, height))
 
+    elif scale_option == "longest":
+        original_width, original_height = layer_image.size
+        if original_width > original_height:
+            new_width=width
+            scale = width / original_width
+            new_height = int(original_height * scale)
+            x=0
+            y=int((height-new_height)*0.5)
+        else:
+            new_height=height
+            scale = height / original_height
+            new_width = int(original_height * scale)
+            x=int((width-new_width)*0.5)
+            y=0
+    # elif side == "shortest":
+    #         if width < height:
+    #              
+    #         else:
+    #              
+    
+
     # 调整mask的大小
     nw, nh = layer_image.size
     mask = mask.resize((nw, nh))
@@ -752,8 +872,9 @@ def merge_images(bg_image, layer_image, mask, x, y, width, height, scale_option,
         bg_image=multiply_blend(bg_image_white,bg_image)
         bg_image=bg_image.convert("RGBA")
     else:
-        transparent_img = Image.new("RGBA",layer_image.size, (0, 0, 0, 0))
+        transparent_img = Image.new("RGBA",layer_image.size, (255, 255, 255, 0))
         transparent_img.paste(layer_image,(0, 0), mask)
+        # transparent_img.save('test.png')
         bg_image.paste(transparent_img, (x, y), transparent_img)
 
 
@@ -803,6 +924,31 @@ def resize_image(layer_image, scale_option, width, height,color="white"):
         resized_image = resized_image.convert("RGB")
         resized_image=resize_2(resized_image)
         return resized_image
+    elif scale_option == "longest":
+    #暂时不用，  
+        if original_width > original_height:
+            new_width=width
+            scale = width / original_width
+            new_height = int(original_height * scale)
+            x=0
+            y=int((new_height-height)*0.5)
+            resized_image = Image.new("RGB", (new_width, new_height), color=color)
+            resized_image.paste(layer_image.resize((new_width, new_height)), (x,y))
+            resized_image = resized_image.convert("RGB")
+            resized_image=resize_2(resized_image)
+            return resized_image
+        else:
+            new_height=height
+            scale = height / original_height
+            new_width = int(original_height * scale)
+            x=int((new_width-width)*0.5)
+            y=0
+            resized_image = Image.new("RGB", (new_width, new_height), color=color)
+            resized_image.paste(layer_image.resize((new_width, new_height)), (x,y))
+            resized_image = resized_image.convert("RGB")
+            resized_image=resize_2(resized_image)
+            return resized_image
+
     
     layer_image=resize_2(layer_image)
     return layer_image
@@ -1704,14 +1850,14 @@ class CompositeImages:
     def INPUT_TYPES(s):
         return {
             "required": { 
-                    "foreground": ("IMAGE",),
+                    "foreground": (any_type,),
                     "mask":("MASK",),
                     "background": ("IMAGE",),
                     },
              "optional":{
                            
                   "is_multiply_blend":  ("BOOLEAN", {"default": False}),
-                            
+                  "position":  (['overall',"center_bottom","center_top","right_bottom","left_bottom","right_top","left_top"],),    
                     }
                 }
     
@@ -1724,11 +1870,11 @@ class CompositeImages:
 
     # OUTPUT_IS_LIST = (True,)
 
-    def run(self, foreground,mask,background,is_multiply_blend):
+    def run(self, foreground,mask,background,is_multiply_blend,position):
         foreground= tensor2pil(foreground)
         mask= tensor2pil(mask)
         background= tensor2pil(background)
-        res=composite_images(foreground,background,mask,is_multiply_blend)
+        res=composite_images(foreground,background,mask,is_multiply_blend,position)
         
         return (pil2tensor(res),)
 

web/index.html

@@ -444,10 +444,9 @@
             border-radius: 4px;
         }
 
-        summary{
+        summary {
             user-select: none;
         }
-        
     </style>
     <!-- <script src="../../../scripts/api.js" type="module"></script> -->
     <link href="/extensions/comfyui-mixlab-nodes/lib/photoswipe.min.css" rel="stylesheet">
@@ -953,6 +952,10 @@
 
 
         function createOutputs(outputData, link) {
+            const url = new URL(window.location.href);
+            const params = new URLSearchParams(url.search);
+            const innerApp = params.get("innerApp");
+
             const container = document.createElement('div');
             container.className = "output";
 
@@ -961,11 +964,11 @@
 
             const copyHTML = document.createElement('button');
             copyHTML.innerText = 'copy as html'
-            action.appendChild(copyHTML)
+            if (!innerApp) action.appendChild(copyHTML)
 
             const copyImage = document.createElement('button');
             copyImage.innerText = 'copy image'
-            action.appendChild(copyImage)
+            if (!innerApp) action.appendChild(copyImage)
             copyImage.style.marginLeft = '18px';
 
             let isURL = false;
@@ -1067,18 +1070,19 @@
                     "Image Save",
                     "SaveImageAndMetadata_",
                     "TransparentImage"].includes(node.class_type)) {
-                    console.log('output#image', node)
+
+                    const url = node.options?.defaultImage || window._appData?.icon || base64Df;
                     let a = document.createElement('a');
                     a.id = `output_${node.id}`
                     a.setAttribute('data-pswp-width', "200");
                     a.setAttribute('data-pswp-height', "200");
                     a.setAttribute('target', "_blank");
-                    a.setAttribute('href', base64Df);
+                    a.setAttribute('href', url);
                     a.setAttribute('title', node.title);
 
                     let img = new Image();
                     // img;
-                    img.src = window._appData?.icon || base64Df;
+                    img.src = url;
                     a.appendChild(img)
                     output_card.appendChild(a);
                     isShowImageFn = true;
@@ -2272,7 +2276,7 @@
             // leftDiv.appendChild(des);
             leftDiv.appendChild(statusDiv);
             leftDiv.appendChild(input1);
-            
+
             if (typeof (data.data) == 'object') mainDiv.appendChild(submitButton);
 
             rightDiv.appendChild(output);
@@ -2630,7 +2634,7 @@
             ui.title.update(appData.name || 'Mixlab APP');
 
             // 更新应用图标
-            ui.icon.update(appData.icon || base64Df);
+            ui.icon.update(appData.icon || appData.output[0]?.options?.defaultImage || base64Df);
 
             ui.des.update(appData.description || '-');
 

web/javascript/app_mixlab.js

@@ -207,9 +207,27 @@ async function extractInputAndOutputData (
         // input.push()
       }
       if (outputIds.includes(id)) {
+        let options = {}
+        //输出的默认图
+        if (
+          node.type === 'SaveImageAndMetadata_' &&
+          app.graph.getNodeById(id).imgs
+        ) {
+          // SaveImageAndMetadata_的默认图，转为base64
+          let imgurl = app.graph.getNodeById(id).imgs[0].src
+
+          options.defaultImage = await drawImageToCanvas(imgurl, 512)
+          console.log('#SaveImageAndMetadata_的默认图', options)
+        }
+
         // let node = app.graph.getNodeById(id)
         // output.push()
-        output[outputIds.indexOf(id)] = { ...data[id], title: node.title, id }
+        output[outputIds.indexOf(id)] = {
+          ...data[id],
+          title: node.title,
+          id,
+          options
+        }
       }
 
       if (

web/javascript/chat.js

@@ -0,0 +1,108 @@
+async function* completion (url, messages, controller) {
+  let data = {
+    model: 'gpt-3.5-turbo-16k',
+    messages,
+    temperature: 0.6,
+    stream: true
+  }
+  // if (imageNode) {
+  //   data = { ...data, image_data: [imageNode] }
+  // }
+
+  // let controller = new AbortController()
+
+  let response = await fetch(url, {
+    method: 'POST',
+    body: JSON.stringify(data),
+    headers: {
+      Connection: 'keep-alive',
+      'Content-Type': 'application/json',
+      Accept: 'text/event-stream'
+    },
+    signal: controller.signal
+  })
+
+  const reader = response.body.getReader()
+  const decoder = new TextDecoder()
+
+  let content = ''
+  let leftover = '' // Buffer for partially read lines
+
+  try {
+    let cont = true
+    while (cont) {
+      let result = await reader.read()
+      if (result.done) {
+        break
+      }
+
+     
+      // Add any leftover data to the current chunk of data
+      const text = leftover + decoder.decode(result.value)
+
+      // Check if the last character is a line break
+      const endsWithLineBreak = text.endsWith('\n')
+
+      // Split the text into lines
+      let lines = text.split('\n')
+
+      // If the text doesn't end with a line break, then the last line is incomplete
+      // Store it in leftover to be added to the next chunk of data
+      if (!endsWithLineBreak) {
+        leftover = lines.pop()
+      } else {
+        leftover = '' // Reset leftover if we have a line break at the end
+      }
+
+      // Parse all sse events and add them to result
+      const regex = /^(\S+):\s(.*)$/gm
+      for (const line of lines) {
+        const match = regex.exec(line)
+        if (match) {
+          result[match[1]] = match[2]
+          // since we know this is llama.cpp, let's just decode the json in data
+          if (result.data) {
+            result.data = JSON.parse(result.data)
+            // console.log('#result.data',result.data)
+            
+            content += result.data.choices[0].delta?.content||''
+
+            // yield
+            yield result
+
+            // if we got a stop token from server, we will break here
+            if (result.data.choices[0].finish_reason=="stop") {
+              if (result.data.generation_settings) {
+                // generation_settings = result.data.generation_settings;
+              }
+              cont = false
+              break
+            }
+          }
+        }
+      }
+    }
+  } catch (e) {
+    console.error('llama error: ', e)
+    throw e
+  } finally {
+    controller.abort()
+  }
+
+  return content
+  // return (await response.json()).content
+}
+
+export async function completion_ (url, messages, controller, callback) {
+  let request = await completion(url, messages, controller)
+  for await (const chunk of request) {
+    
+    let content=chunk.data.choices[0].delta.content||""
+    if(chunk.data.choices[0].role=="assistant"){
+      //开始
+      content=""
+    }
+
+    if (callback) callback(content)
+  }
+}

web/javascript/checkVersion_mixlab.js

@@ -3,7 +3,7 @@ import { app } from '../../../scripts/app.js'
 const repoOwner = 'shadowcz007' // 替换为仓库的所有者
 const repoName = 'comfyui-mixlab-nodes' // 替换为仓库的名称
 
-const version = 'v0.24.0'
+const version = 'v0.25.0'
 
 fetch(`https://api.github.com/repos/${repoOwner}/${repoName}/releases/latest`)
   .then(response => response.json())

web/javascript/ui_mixlab.js

@@ -9,6 +9,89 @@ import {
 
 import { smart_init, addSmartMenu } from './smart_connect.js'
 
+import { completion_ } from './chat.js'
+
+//系统prompt
+const systemPrompt = `You are a prompt creator, your task is to create prompts for the user input request, the prompts are image descriptions that include keywords for (an adjective, type of image, framing/composition, subject, subject appearance/action, environment, lighting situation, details of the shoot/illustration, visuals aesthetics and artists), brake keywords by comas, provide high quality, non-verboose, coherent, brief, concise, and not superfluous prompts, the subject from the input request must be included verbatim on the prompt,the prompt is english`
+
+if (!localStorage.getItem('_mixlab_system_prompt')) {
+  localStorage.setItem('_mixlab_system_prompt', systemPrompt)
+}
+
+// 获取llama 模型
+async function get_llamafile_models () {
+  try {
+    const response = await fetch('/mixlab/folder_paths', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json'
+      },
+      body: JSON.stringify({
+        type: 'llamafile'
+      })
+    })
+
+    const data = await response.json()
+    console.log(data)
+    return data.names
+  } catch (error) {
+    console.error(error)
+  }
+}
+// 运行llama
+async function start_llama (model = 'Phi-3-mini-4k-instruct-Q5_K_S.gguf') {
+  try {
+    const response = await fetch('/mixlab/start_llama', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json'
+      },
+      body: JSON.stringify({
+        model
+      })
+    })
+
+    const data = await response.json()
+    return { url: `http://127.0.0.1:${data.port}`, model: data.model }
+  } catch (error) {
+    console.error(error)
+  }
+}
+
+// 菜单入口
+async function createMenu () {
+  const menu = document.querySelector('.comfy-menu')
+  const separator = document.createElement('div')
+  separator.style = `margin: 20px 0px;
+  width: 100%;
+  height: 1px;
+  background: var(--border-color);
+  `
+  menu.append(separator)
+
+  if (!menu.querySelector('#mixlab_chatbot_by_llamacpp')) {
+    const appsButton = document.createElement('button')
+    appsButton.id = 'mixlab_chatbot_by_llamacpp'
+    appsButton.textContent = '♾️Mixlab'
+
+    // appsButton.onclick = () =>
+    appsButton.onclick = async () => {
+      if (window._mixlab_llamacpp) {
+        //显示运行的模型
+        createModelsModal([
+          window._mixlab_llamacpp.url,
+          window._mixlab_llamacpp.model
+        ])
+      } else {
+        let ms = await get_llamafile_models()
+        ms = ms.filter(m => !m.match('-mmproj-'))
+        if (ms.length > 0) createModelsModal(ms)
+      }
+    }
+    menu.append(appsButton)
+  }
+}
+
 let isScriptLoaded = {}
 
 function loadExternalScript (url) {
@@ -299,8 +382,9 @@ async function get_my_app (filename = null, category = '') {
     data = []
 
     for (const res of result.data) {
-      let { app, workflow } = res.data;
-      if (app?.filename) data.push({
+      let { app, workflow } = res.data
+      if (app?.filename)
+        data.push({
           ...app,
           data: workflow,
           date: res.date
@@ -344,6 +428,33 @@ injectCSS(`::-webkit-scrollbar {
   width: 2px;
 }
 
+#mixlab_chatbot_by_llamacpp{
+  font-size:14px
+}
+
+#mixlab_chatbot_by_llamacpp::before {
+  content: attr(title);
+  position: absolute;
+  margin-top: 24px;
+  font-size: 10px;
+}
+
+.mix_tag{
+  padding:8px;cursor: pointer;font-size: 14px;
+    color: var(--input-text);
+    background-color: var(--comfy-input-bg);
+    border-radius: 8px;
+    border-color: var(--border-color);
+    border-style: solid;
+    margin-top: 2px;
+    margin-bottom: 14px;
+}
+
+.mix_tag:hover{
+  background-color: #101c19;
+  color: aquamarine;
+}
+
 @keyframes loading_mixlab {
   0% {
     background-color: green;
@@ -586,13 +697,160 @@ async function fetchReadmeContent (url) {
   }
 }
 
+function createModelsModal (models) {
+  var div =
+    document.querySelector('#model-modal') || document.createElement('div')
+  div.id = 'model-modal'
+  div.innerHTML = ''
+  div.style.cssText = `
+    width: 100%;
+    z-index: 9990;
+    height: 100vh;
+    display: flex;
+    color: var(--descrip-text);
+    position: fixed;
+    top: 0;
+    left: 0;
+    background: #000000a8;
+    `
+
+  var modal = document.createElement('div')
+
+  div.addEventListener('click', e => {
+    e.stopPropagation()
+    div.remove()
+  })
+
+  div.appendChild(modal)
+  modal.classList.add('modal-body')
+  // Set modal styles
+  modal.style.cssText = `
+  color: var(--descrip-text);
+    background-color: var(--comfy-menu-bg);
+    position: fixed;
+    overflow:hidden;
+    top: 50%;
+    left: 50%;
+    transform: translate(-50%, -50%);
+    z-index: 9999;
+    border-radius: 4px;
+    box-shadow: 4px 4px 14px rgba(255,255,255,0.2);
+  `
+
+  // Create modal header
+  const headerElement = document.createElement('div')
+  headerElement.classList.add('modal-header')
+  headerElement.style.cssText = `
+      display: flex;
+      padding: 20px 24px 8px 24px;
+      justify-content: space-between;
+    `
+
+  const headTitleElement = document.createElement('a')
+  headTitleElement.classList.add('header-title')
+  headTitleElement.style.cssText = `
+      color: var(--descrip-text);
+      font-size: 18px;
+      display: flex;
+      align-items: center;
+      flex: 1;
+      overflow: hidden;
+      text-decoration: none;
+      font-weight: bold;
+      justify-content: space-between;
+      padding: 20px;
+      cursor: pointer;
+      user-select: none;
+    `
+
+  headTitleElement.textContent = 'Models'
+  // headTitleElement.href = 'https://github.com/shadowcz007/comfyui-mixlab-nodes'
+  // headTitleElement.target = '_blank'
+  const linkIcon = document.createElement('small')
+  linkIcon.textContent = '自动开启'
+  linkIcon.style.padding = '4px'
+
+  headTitleElement.appendChild(linkIcon)
+  headerElement.appendChild(headTitleElement)
+  if (localStorage.getItem('_mixlab_auto_llama_open')) {
+    linkIcon.style.backgroundColor = '#66ff6c'
+    linkIcon.style.color = 'black'
+  }
+  linkIcon.addEventListener('click', e => {
+    e.stopPropagation()
+    if (localStorage.getItem('_mixlab_auto_llama_open')) {
+      localStorage.setItem('_mixlab_auto_llama_open', '')
+      linkIcon.style.backgroundColor = ''
+      linkIcon.style.color = 'var(--descrip-text)'
+    } else {
+      localStorage.setItem('_mixlab_auto_llama_open', 'true')
+      linkIcon.style.backgroundColor = '#66ff6c'
+      linkIcon.style.color = 'black'
+    }
+  })
+
+  modal.appendChild(headTitleElement)
+
+  // Create modal content area
+  var modalContent = document.createElement('div')
+  modalContent.classList.add('modal-content')
+
+  var input = document.createElement('textarea')
+  input.className = 'comfy-multiline-input'
+  input.style = `    height: 260px;
+  width: 480px;
+  font-size: 16px;
+  padding: 18px;`
+  input.value = localStorage.getItem('_mixlab_system_prompt')
+
+  input.addEventListener('change', e => {
+    e.stopPropagation()
+    localStorage.setItem('_mixlab_system_prompt', input.value)
+  })
+
+  input.addEventListener('click', e => {
+    e.stopPropagation()
+  })
+
+  modalContent.appendChild(input)
+
+  for (const m of models) {
+    let d = document.createElement('div')
+    d.innerText = m
+    d.className = `mix_tag`
+
+    if (!window._mixlab_llamacpp) {
+      d.addEventListener('click', async e => {
+        e.stopPropagation()
+        div.remove()
+        let res = await start_llama(m)
+        window._mixlab_llamacpp = res
+
+        localStorage.setItem('_mixlab_llama_select', res.model)
+
+        if (document.body.querySelector('#mixlab_chatbot_by_llamacpp')) {
+          document.body
+            .querySelector('#mixlab_chatbot_by_llamacpp')
+            .setAttribute('title', window._mixlab_llamacpp.url)
+        }
+      })
+    }
+
+    modalContent.appendChild(d)
+  }
+  modal.appendChild(modalContent)
+
+  document.body.appendChild(div)
+}
+
 function createModal (url, markdown, title) {
   // Create modal element
   var div =
     document.querySelector('#mix-modal') || document.createElement('div')
   div.id = 'mix-modal'
   div.innerHTML = ''
-  div.style.cssText = `width: 100%;
+  div.style.cssText = `
+    width: 100%;
     z-index: 9990;
     height: 100vh;
     display: flex;
@@ -898,6 +1156,17 @@ function drawBadge (node, orig, restArgs) {
 app.registerExtension({
   name: 'Comfy.Mixlab.ui',
   init () {
+    //是否要自动加载模型
+    if (localStorage.getItem('_mixlab_auto_llama_open')) {
+      let model = localStorage.getItem('_mixlab_llama_select')
+      start_llama(model).then(res => {
+        window._mixlab_llamacpp = res
+        document.body
+          .querySelector('#mixlab_chatbot_by_llamacpp')
+          .setAttribute('title', res.url)
+      })
+    }
+
     LGraphCanvas.prototype.helpAboutNode = async function (node) {
       nodesMap =
         nodesMap && Object.keys(nodesMap).length > 0
@@ -922,75 +1191,64 @@ app.registerExtension({
 
     smart_init()
 
-    const getNodeMenuOptions = LGraphCanvas.prototype.getNodeMenuOptions // store the existing method
-    LGraphCanvas.prototype.getNodeMenuOptions = function (node) {
-      // replace it
-      const options = getNodeMenuOptions.apply(this, arguments) // start by calling the stored one
-      node.setDirtyCanvas(true, true) // force a redraw of (foreground, background)
-      console.log('getNodeMenuOptions', node.type == 'CLIPTextEncode')
+    LGraphCanvas.prototype.text2text = async function (node) {
+      // console.log(node)
+      let widget = node.widgets.filter(
+        w => w.name === 'text' && typeof w.value == 'string'
+      )[0]
+      if (widget) {
+        let controller = new AbortController()
+        let ends = []
+        let userInput = widget.value
+        widget.value = widget.value.trim()
+        widget.value += '\n'
+        try {
+          await completion_(
+            window._mixlab_llamacpp.url + '/v1/chat/completions',
+            [
+              {
+                role: 'system',
+                content: localStorage.getItem('_mixlab_system_prompt')
+              },
+              { role: 'user', content: userInput }
+            ],
+            controller,
+            t => {
+              // console.log(t)
+              widget.value += t
+            }
+          )
+        } catch (error) {
+          //是否要自动加载模型
+          if (localStorage.getItem('_mixlab_auto_llama_open')) {
+            let model = localStorage.getItem('_mixlab_llama_select')
+            start_llama(model).then(async res => {
+              window._mixlab_llamacpp = res
+              document.body
+                .querySelector('#mixlab_chatbot_by_llamacpp')
+                .setAttribute('title', res.url)
 
-      let opts = [
-        {
-          content: 'Help ♾️Mixlab', // with a name
-          callback: () => {
-            LGraphCanvas.prototype.helpAboutNode(node)
-          } // and the callback
-        },
-        {
-          content: 'Fix node v2', // with a name
-          callback: () => {
-            LGraphCanvas.prototype.fixTheNode(node)
+              await completion_(
+                window._mixlab_llamacpp.url + '/v1/chat/completions',
+                [
+                  {
+                    role: 'system',
+                    content: localStorage.getItem('_mixlab_system_prompt')
+                  },
+                  { role: 'user', content: userInput }
+                ],
+                controller,
+                t => {
+                  // console.log(t)
+                  widget.value += t
+                }
+              )
+            })
           }
         }
-      ]
 
-      if (node.widgets) {
-        // let text_widget = node.widgets.filter(
-        //   w => w.name === 'text' && typeof w.value == 'string'
-        // )
-        // if (text_widget && text_widget.length == 1) {
-        //   opts.push({
-        //     content: 'Text-to-Text ♾️Mixlab', // with a name
-        //     callback: () => {
-        //       LGraphCanvas.prototype.text2text(node)
-        //     } // and the callback
-        //   })
-        // }
+        widget.value = widget.value.trim()
       }
-
-      opts = addSmartMenu(opts, node)
-
-      // if (node.type == 'CLIPTextEncode') {
-      //   // 则出现 randomPrompt
-      //   // CLIPTextEncode 的widget ，name== 'text'
-      //   let node_widget_name = 'text'
-      //   const widget = node.widgets.filter(w => w.name === node_widget_name)[0]
-
-      //  let mixlab_nodes_smart_connect= [{node_type:'CLIPTextEncode',
-      //   node_widget_name:'text',
-      //   inputNodeName:'RandomPrompt',
-      //   inputNode_output_type:'STRING'}]
-
-      //   if (widget) {
-      //     opts = [
-      //       {
-      //         content: 'RandomPrompt',
-      //         callback: () => {
-      //           LGraphCanvas.prototype._createNodeForInput(
-      //             node, //当前node
-      //             widget,//当前node里需要自动连线的widget
-      //             'RandomPrompt',//作为input的node type
-      //             'STRING'// 作为input的node的outputs的type. the input slot type of the target node
-      //           )
-      //         }
-      //       },
-      //       null,
-      //       ...opts
-      //     ]
-      //   }
-      // }
-
-      return [...opts, null, ...options] // and return the options
     }
 
     const getGroupMenuOptions = LGraphCanvas.prototype.getGroupMenuOptions // store the existing method
@@ -1139,6 +1397,56 @@ app.registerExtension({
       this.setDirty(true, true)
     }
 
+    const getNodeMenuOptions=LGraphCanvas.prototype.getNodeMenuOptions;
+    LGraphCanvas.prototype.getNodeMenuOptions = function (node) {
+      // replace it
+      const options = getNodeMenuOptions.apply(this, arguments) // start by calling the stored one
+      node.setDirtyCanvas(true, true) // force a redraw of (foreground, background)
+
+      let opts = [
+        {
+          content: 'Help ♾️Mixlab', // with a name
+          callback: () => {
+            LGraphCanvas.prototype.helpAboutNode(node)
+          } // and the callback
+        },
+        {
+          content: 'Fix node v2', // with a name
+          callback: () => {
+            LGraphCanvas.prototype.fixTheNode(node)
+          }
+        }
+      ]
+
+      if (node.widgets) {
+        let text_widget = node.widgets.filter(
+          w => w.name === 'text' && typeof w.value == 'string'
+        )
+
+        let text_input = node.inputs?.filter(
+          inp => inp.name == 'text' && inp.type == 'STRING'
+        )
+
+        if (
+          text_input&&
+          text_input.length == 0 &&
+          text_widget &&
+          text_widget.length == 1 &&
+          window._mixlab_llamacpp &&
+          node.type != 'ShowTextForGPT'
+        ) {
+          opts.push({
+            content: 'Text-to-Text ♾️Mixlab', // with a name
+            callback: () => {
+              LGraphCanvas.prototype.text2text(node)
+            } // and the callback
+          })
+        }
+      }
+
+      return [...opts, null, ...options] // and return the options
+    }
+
     // 支持app模式的json
     const loadAppJson = async data => {
       let workflow
@@ -1173,8 +1481,6 @@ app.registerExtension({
         event.preventDefault()
         event.stopPropagation()
 
-       
-
         // Dragging from Chrome->Firefox there is a file but its a bmp, so ignore that
         if (
           event.dataTransfer.files.length &&
@@ -1182,13 +1488,14 @@ app.registerExtension({
         ) {
           const reader = new FileReader()
           reader.onload = async () => {
-       
             loadAppJson(reader.result)
           }
           reader.readAsText(event.dataTransfer.files[0])
         }
       })
     }
+
+    createMenu()
   },
   setup () {
     setTimeout(async () => {
@@ -1198,7 +1505,7 @@ app.registerExtension({
       const apps = await get_my_app()
       if (!apps) return
 
-      console.log('apps',apps)
+      console.log('apps', apps)
 
       let apps_map = { 0: [] }