text-to-text

__init__.py

@@ -11,6 +11,10 @@ import logging
 from comfy.cli_args import args
 python = sys.executable
 
+
+llama_port=None
+llama_model=""
+
 from .nodes.ChatGPT import get_llama_models,get_llama_model_path
 
 from server import PromptServer
@@ -624,11 +628,13 @@ async def post_prompt_result(request):
 
 
 # llam服务的开启
-
-
 @routes.post('/mixlab/start_llama')
 async def my_hander_method(request):
     data = await request.json()
+    global llama_port,llama_model
+
+    if llama_port and llama_model:
+        return web.json_response({"port":llama_port,"model":llama_model})
 
     import threading
     import uvicorn
@@ -640,12 +646,25 @@ async def my_hander_method(request):
                 ConfigFileSettings,
             )
     model=get_llama_model_path(data['model'])
+
+    address="127.0.0.1"
     port=9090
-    server_settings=ServerSettings(host="127.0.0.1",port=port)
+    success = False
+    for i in range(11):  # 尝试最多11次
+        if await check_port_available(address, port + i):
+            port = port + i
+            success = True
+            break
+
+    if success == False:
+        return web.json_response({"port":None,"model":""})
+    
+    
+    server_settings=ServerSettings(host=address,port=port)
 
     app = create_app(
                 server_settings=server_settings,
-                model_settings=[ModelSettings(model=model)],
+                model_settings=[ModelSettings(model=model,n_gpu_layers=9999,n_ctx=4098)],
             )
 
     def run_uvicorn():
@@ -663,7 +682,10 @@ async def my_hander_method(request):
     # 启动子线程
     thread.start()
 
-    return web.json_response({"port":port,"model":model})
+    llama_port=port
+    llama_model=data['model']
+
+    return web.json_response({"port":llama_port,"model":llama_model})
 
 
 

web/javascript/chat.js

@@ -0,0 +1,108 @@
+async function* completion (url, messages, controller) {
+  let data = {
+    model: 'gpt-3.5-turbo-16k',
+    messages,
+    temperature: 0.6,
+    stream: true
+  }
+  // if (imageNode) {
+  //   data = { ...data, image_data: [imageNode] }
+  // }
+
+  // let controller = new AbortController()
+
+  let response = await fetch(url, {
+    method: 'POST',
+    body: JSON.stringify(data),
+    headers: {
+      Connection: 'keep-alive',
+      'Content-Type': 'application/json',
+      Accept: 'text/event-stream'
+    },
+    signal: controller.signal
+  })
+
+  const reader = response.body.getReader()
+  const decoder = new TextDecoder()
+
+  let content = ''
+  let leftover = '' // Buffer for partially read lines
+
+  try {
+    let cont = true
+    while (cont) {
+      let result = await reader.read()
+      if (result.done) {
+        break
+      }
+
+     
+      // Add any leftover data to the current chunk of data
+      const text = leftover + decoder.decode(result.value)
+
+      // Check if the last character is a line break
+      const endsWithLineBreak = text.endsWith('\n')
+
+      // Split the text into lines
+      let lines = text.split('\n')
+
+      // If the text doesn't end with a line break, then the last line is incomplete
+      // Store it in leftover to be added to the next chunk of data
+      if (!endsWithLineBreak) {
+        leftover = lines.pop()
+      } else {
+        leftover = '' // Reset leftover if we have a line break at the end
+      }
+
+      // Parse all sse events and add them to result
+      const regex = /^(\S+):\s(.*)$/gm
+      for (const line of lines) {
+        const match = regex.exec(line)
+        if (match) {
+          result[match[1]] = match[2]
+          // since we know this is llama.cpp, let's just decode the json in data
+          if (result.data) {
+            result.data = JSON.parse(result.data)
+            // console.log('#result.data',result.data)
+            
+            content += result.data.choices[0].delta?.content||''
+
+            // yield
+            yield result
+
+            // if we got a stop token from server, we will break here
+            if (result.data.choices[0].finish_reason=="stop") {
+              if (result.data.generation_settings) {
+                // generation_settings = result.data.generation_settings;
+              }
+              cont = false
+              break
+            }
+          }
+        }
+      }
+    }
+  } catch (e) {
+    console.error('llama error: ', e)
+    throw e
+  } finally {
+    controller.abort()
+  }
+
+  return content
+  // return (await response.json()).content
+}
+
+export async function completion_ (url, messages, controller, callback) {
+  let request = await completion(url, messages, controller)
+  for await (const chunk of request) {
+    
+    let content=chunk.data.choices[0].delta.content||""
+    if(chunk.data.choices[0].role=="assistant"){
+      //开始
+      content=""
+    }
+
+    if (callback) callback(content)
+  }
+}

web/javascript/ui_mixlab.js

@@ -9,6 +9,15 @@ import {
 
 import { smart_init, addSmartMenu } from './smart_connect.js'
 
+import { completion_ } from './chat.js'
+
+//系统prompt
+const systemPrompt = `You are a prompt creator, your task is to create prompts for the user input request, the prompts are image descriptions that include keywords for (an adjective, type of image, framing/composition, subject, subject appearance/action, environment, lighting situation, details of the shoot/illustration, visuals aesthetics and artists), brake keywords by comas, provide high quality, non-verboose, coherent, brief, concise, and not superfluous prompts, the subject from the input request must be included verbatim on the prompt,the prompt is english`
+
+if (!localStorage.getItem('_mixlab_system_prompt')) {
+  localStorage.setItem('_mixlab_system_prompt', systemPrompt)
+}
+
 // 获取llama 模型
 async function get_llamafile_models () {
   try {
@@ -43,13 +52,45 @@ async function start_llama (model = 'Phi-3-mini-4k-instruct-Q5_K_S.gguf') {
     })
 
     const data = await response.json()
-    return `http://127.0.0.1:${data.port}`
+    return { url: `http://127.0.0.1:${data.port}`, model: data.model }
   } catch (error) {
     console.error(error)
   }
 }
 
+// 菜单入口
+async function createMenu () {
+  const menu = document.querySelector('.comfy-menu')
+  const separator = document.createElement('div')
+  separator.style = `margin: 20px 0px;
+  width: 100%;
+  height: 1px;
+  background: var(--border-color);
+  `
+  menu.append(separator)
 
+  if (!menu.querySelector('#mixlab_chatbot_by_llamacpp')) {
+    const appsButton = document.createElement('button')
+    appsButton.id = 'mixlab_chatbot_by_llamacpp'
+    appsButton.textContent = 'llamacpp♾️Mixlab'
+
+    // appsButton.onclick = () =>
+    appsButton.onclick = async () => {
+      if (window._mixlab_llamacpp) {
+        //显示运行的模型
+        createModelsModal([
+          window._mixlab_llamacpp.url,
+          window._mixlab_llamacpp.model
+        ])
+      } else {
+        let ms = await get_llamafile_models()
+        ms = ms.filter(m => !m.match('-mmproj-'))
+        if (ms.length > 0) createModelsModal(ms)
+      }
+    }
+    menu.append(appsButton)
+  }
+}
 
 let isScriptLoaded = {}
 
@@ -387,6 +428,33 @@ injectCSS(`::-webkit-scrollbar {
   width: 2px;
 }
 
+#mixlab_chatbot_by_llamacpp{
+  font-size:14px
+}
+
+#mixlab_chatbot_by_llamacpp::before {
+  content: attr(title);
+  position: absolute;
+  margin-top: 24px;
+  font-size: 10px;
+}
+
+.mix_tag{
+  padding:8px;cursor: pointer;font-size: 14px;
+    color: var(--input-text);
+    background-color: var(--comfy-input-bg);
+    border-radius: 8px;
+    border-color: var(--border-color);
+    border-style: solid;
+    margin-top: 2px;
+    margin-bottom: 14px;
+}
+
+.mix_tag:hover{
+  background-color: #101c19;
+  color: aquamarine;
+}
+
 @keyframes loading_mixlab {
   0% {
     background-color: green;
@@ -629,13 +697,160 @@ async function fetchReadmeContent (url) {
   }
 }
 
+function createModelsModal (models) {
+  var div =
+    document.querySelector('#model-modal') || document.createElement('div')
+  div.id = 'model-modal'
+  div.innerHTML = ''
+  div.style.cssText = `
+    width: 100%;
+    z-index: 9990;
+    height: 100vh;
+    display: flex;
+    color: var(--descrip-text);
+    position: fixed;
+    top: 0;
+    left: 0;
+    background: #000000a8;
+    `
+
+  var modal = document.createElement('div')
+
+  div.addEventListener('click', e => {
+    e.stopPropagation()
+    div.remove()
+  })
+
+  div.appendChild(modal)
+  modal.classList.add('modal-body')
+  // Set modal styles
+  modal.style.cssText = `
+  color: var(--descrip-text);
+    background-color: var(--comfy-menu-bg);
+    position: fixed;
+    overflow:hidden;
+    top: 50%;
+    left: 50%;
+    transform: translate(-50%, -50%);
+    z-index: 9999;
+    border-radius: 4px;
+    box-shadow: 4px 4px 14px rgba(255,255,255,0.2);
+  `
+
+  // Create modal header
+  const headerElement = document.createElement('div')
+  headerElement.classList.add('modal-header')
+  headerElement.style.cssText = `
+      display: flex;
+      padding: 20px 24px 8px 24px;
+      justify-content: space-between;
+    `
+
+  const headTitleElement = document.createElement('a')
+  headTitleElement.classList.add('header-title')
+  headTitleElement.style.cssText = `
+      color: var(--descrip-text);
+      font-size: 18px;
+      display: flex;
+      align-items: center;
+      flex: 1;
+      overflow: hidden;
+      text-decoration: none;
+      font-weight: bold;
+      justify-content: space-between;
+      padding: 20px;
+      cursor: pointer;
+      user-select: none;
+    `
+
+  headTitleElement.textContent = 'Models'
+  // headTitleElement.href = 'https://github.com/shadowcz007/comfyui-mixlab-nodes'
+  // headTitleElement.target = '_blank'
+  const linkIcon = document.createElement('small')
+  linkIcon.textContent = '自动开启'
+  linkIcon.style.padding = '4px'
+
+  headTitleElement.appendChild(linkIcon)
+  headerElement.appendChild(headTitleElement)
+  if (localStorage.getItem('_mixlab_auto_llama_open')) {
+    linkIcon.style.backgroundColor = '#66ff6c'
+    linkIcon.style.color = 'black'
+  }
+  linkIcon.addEventListener('click', e => {
+    e.stopPropagation()
+    if (localStorage.getItem('_mixlab_auto_llama_open')) {
+      localStorage.setItem('_mixlab_auto_llama_open', '')
+      linkIcon.style.backgroundColor = ''
+      linkIcon.style.color = 'var(--descrip-text)'
+    } else {
+      localStorage.setItem('_mixlab_auto_llama_open', 'true')
+      linkIcon.style.backgroundColor = '#66ff6c'
+      linkIcon.style.color = 'black'
+    }
+  })
+
+  modal.appendChild(headTitleElement)
+
+  // Create modal content area
+  var modalContent = document.createElement('div')
+  modalContent.classList.add('modal-content')
+
+  var input = document.createElement('textarea')
+  input.className = 'comfy-multiline-input'
+  input.style = `    height: 260px;
+  width: 480px;
+  font-size: 16px;
+  padding: 18px;`
+  input.value = localStorage.getItem('_mixlab_system_prompt')
+
+  input.addEventListener('change', e => {
+    e.stopPropagation()
+    localStorage.setItem('_mixlab_system_prompt', input.value)
+  })
+
+  input.addEventListener('click', e => {
+    e.stopPropagation()
+  })
+
+  modalContent.appendChild(input)
+
+  for (const m of models) {
+    let d = document.createElement('div')
+    d.innerText = m
+    d.className = `mix_tag`
+
+    if (!window._mixlab_llamacpp) {
+      d.addEventListener('click', async e => {
+        e.stopPropagation()
+        div.remove()
+        let res = await start_llama(m)
+        window._mixlab_llamacpp = res
+
+        localStorage.setItem('_mixlab_llama_select', res.model)
+
+        if (document.body.querySelector('#mixlab_chatbot_by_llamacpp')) {
+          document.body
+            .querySelector('#mixlab_chatbot_by_llamacpp')
+            .setAttribute('title', window._mixlab_llamacpp.url)
+        }
+      })
+    }
+
+    modalContent.appendChild(d)
+  }
+  modal.appendChild(modalContent)
+
+  document.body.appendChild(div)
+}
+
 function createModal (url, markdown, title) {
   // Create modal element
   var div =
     document.querySelector('#mix-modal') || document.createElement('div')
   div.id = 'mix-modal'
   div.innerHTML = ''
-  div.style.cssText = `width: 100%;
+  div.style.cssText = `
+    width: 100%;
     z-index: 9990;
     height: 100vh;
     display: flex;
@@ -941,6 +1156,17 @@ function drawBadge (node, orig, restArgs) {
 app.registerExtension({
   name: 'Comfy.Mixlab.ui',
   init () {
+    //是否要自动加载模型
+    if (localStorage.getItem('_mixlab_auto_llama_open')) {
+      let model = localStorage.getItem('_mixlab_llama_select')
+      start_llama(model).then(res => {
+        window._mixlab_llamacpp = res
+        document.body
+          .querySelector('#mixlab_chatbot_by_llamacpp')
+          .setAttribute('title', res.url)
+      })
+    }
+
     LGraphCanvas.prototype.helpAboutNode = async function (node) {
       nodesMap =
         nodesMap && Object.keys(nodesMap).length > 0
@@ -965,12 +1191,57 @@ app.registerExtension({
 
     smart_init()
 
+    LGraphCanvas.prototype.text2text = async function (node) {
+      // console.log(node)
+      let widget = node.widgets.filter(
+        w => w.name === 'text' && typeof w.value == 'string'
+      )[0]
+      if (widget) {
+        let controller = new AbortController()
+        let ends = []
+        let userInput = widget.value
+        widget.value = widget.value.trim()
+        widget.value += '\n'
+        await completion_(
+          window._mixlab_llamacpp.url + '/v1/chat/completions',
+          [
+            {
+              role: 'system',
+              content: localStorage.getItem('_mixlab_system_prompt')
+            },
+            { role: 'user', content: userInput }
+          ],
+          controller,
+          t => {
+            // console.log(t)
+            widget.value += t
+          }
+        )
+        widget.value = widget.value.trim()
+        // await chat(
+        //   userInput,
+        //   await getSelectImageNode(),
+        //   t => {
+        //     widget.value += t
+        //     //有回车则终止
+        //     t = t.replace(/\n/g, '<br>')
+        //     ends.push(t.trim())
+        //     if (hasRepeatingPhrases(ends.join(' '))) t = '<br>'
+        //     if (t.trim() == '<br>') {
+        //       controller.abort()
+        //     }
+        //   },
+        //   controller
+        // )
+      }
+    }
+
     const getNodeMenuOptions = LGraphCanvas.prototype.getNodeMenuOptions // store the existing method
     LGraphCanvas.prototype.getNodeMenuOptions = function (node) {
       // replace it
       const options = getNodeMenuOptions.apply(this, arguments) // start by calling the stored one
       node.setDirtyCanvas(true, true) // force a redraw of (foreground, background)
-      console.log('getNodeMenuOptions', node.type == 'CLIPTextEncode')
+      console.log('#getNodeMenuOptions', node.type)
 
       let opts = [
         {
@@ -988,19 +1259,39 @@ app.registerExtension({
       ]
 
       if (node.widgets) {
-        // let text_widget = node.widgets.filter(
-        //   w => w.name === 'text' && typeof w.value == 'string'
-        // )
-        // if (text_widget && text_widget.length == 1) {
-        //   opts.push({
-        //     content: 'Text-to-Text ♾️Mixlab', // with a name
-        //     callback: () => {
-        //       LGraphCanvas.prototype.text2text(node)
-        //     } // and the callback
-        //   })
-        // }
+        let text_widget = node.widgets.filter(
+          w => w.name === 'text' && typeof w.value == 'string'
+        )
+
+        let text_input = node.inputs.filter(
+          inp => inp.name == 'text' && inp.type == 'STRING'
+        )
+
+        if (
+          text_input.length == 0 &&
+          text_widget &&
+          text_widget.length == 1 &&
+          window._mixlab_llamacpp &&
+          node.type != 'ShowTextForGPT'
+        ) {
+          opts.push({
+            content: 'Text-to-Text ♾️Mixlab', // with a name
+            callback: () => {
+              LGraphCanvas.prototype.text2text(node)
+            } // and the callback
+          })
+        }
       }
 
+      // if (node.imgs && node.imgs.length > 0) {
+      //   opts.push({
+      //     content: 'Image-to-Text ♾️Mixlab', // with a name
+      //     callback: () => {
+      //       LGraphCanvas.prototype.text2text(node)
+      //     } // and the callback
+      //   })
+      // }
+
       opts = addSmartMenu(opts, node)
 
       // if (node.type == 'CLIPTextEncode') {
@@ -1182,6 +1473,56 @@ app.registerExtension({
       this.setDirty(true, true)
     }
 
+    LGraphCanvas.prototype.getNodeMenuOptions = function (node) {
+      // replace it
+      const options = getNodeMenuOptions.apply(this, arguments) // start by calling the stored one
+      node.setDirtyCanvas(true, true) // force a redraw of (foreground, background)
+
+      let opts = []
+
+      if (node.widgets) {
+        let text_widget = node.widgets.filter(
+          w => w.name === 'text' && typeof w.value == 'string'
+        )
+
+        if (text_widget && text_widget.length == 1) {
+          opts = [
+            {
+              content: 'Text-to-Text ♾️Mixlab', // with a name
+              callback: () => {
+                LGraphCanvas.prototype.text2text(node)
+              } // and the callback
+            }
+            // {
+            //   content: 'Fix node v2', // with a name
+            //   callback: () => {
+            //     LGraphCanvas.prototype.fixTheNode(node)
+            //   }
+            // }
+          ]
+        }
+      }
+
+      if (node.imgs && node.imgs.length > 0) {
+        opts = [
+          {
+            content: 'Image-to-Text ♾️Mixlab', // with a name
+            callback: () => {
+              LGraphCanvas.prototype.text2text(node)
+            } // and the callback
+          }
+          // {
+          //   content: 'Fix node v2', // with a name
+          //   callback: () => {
+          //     LGraphCanvas.prototype.fixTheNode(node)
+          //   }
+          // }
+        ]
+      }
+
+      return [...opts, null, ...options] // and return the options
+    }
+
     // 支持app模式的json
     const loadAppJson = async data => {
       let workflow
@@ -1229,6 +1570,8 @@ app.registerExtension({
         }
       })
     }
+
+    createMenu()
   },
   setup () {
     setTimeout(async () => {