system prompt

Update extension-node-map.json

.github/workflows/publish.yml

@@ -0,0 +1,21 @@
+name: Publish to Comfy registry
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - "pyproject.toml"
+
+jobs:
+  publish-node:
+    name: Publish Custom Node to registry
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Publish Custom Node
+        uses: Comfy-Org/publish-node-action@main
+        with:
+          ## Add your own personal access token to your Github Repository secrets and reference it here.
+          personal_access_token: ${{ secrets.REGISTRY_ACCESS_TOKEN }}

README.md

@@ -10,7 +10,10 @@ ChatGPT 节点支持 Local LLM（llama.cpp），Phi3、llama3 都可以直接一
 
 - 右键菜单支持 text-to-text，方便对 prompt 词补全
 
-强烈推荐：[Phi-3-mini-4k-instruct-GGUF](https://huggingface.co/lmstudio-community/Phi-3-mini-4k-instruct-GGUF/tree/main)，备选：[llama3_if_ai_sdpromptmkr_q2k](https://hf-mirror.com/impactframes/llama3_if_ai_sdpromptmkr_q2k/tree/main)
+强烈推荐：
+[Phi-3-mini-4k-instruct-function-calling-GGUF](https://huggingface.co/nold/Phi-3-mini-4k-instruct-function-calling-GGUF)
+
+[Phi-3-mini-4k-instruct-GGUF](https://huggingface.co/lmstudio-community/Phi-3-mini-4k-instruct-GGUF/tree/main)，备选：[llama3_if_ai_sdpromptmkr_q2k](https://hf-mirror.com/impactframes/llama3_if_ai_sdpromptmkr_q2k/tree/main)
 
 - 右键菜单支持 image-to-text，使用多模态模型，多模态使用 [llava-phi-3-mini-gguf](https://huggingface.co/xtuner/llava-phi-3-mini-gguf/tree/main)，注意需要把llava-phi-3-mini-mmproj-f16.gguf也下载
 

__init__.py

@@ -1,4 +1,3 @@
-#
 import os
 import subprocess
 import importlib.util
@@ -8,14 +7,18 @@ import hashlib
 import datetime
 import folder_paths
 import logging
+import base64,io,re
+from PIL import Image
 from comfy.cli_args import args
 python = sys.executable
 
+# print("sys.path", sys.path)
+
 #修复 sys.stdout.isatty()  object has no attribute 'isatty'
 try:
     sys.stdout.isatty()
 except:
-    print('#fix sys.stdout.isatty')
+    # print('#fix sys.stdout.isatty')
     sys.stdout.isatty = lambda: False
 
 llama_port=None
@@ -30,6 +33,8 @@ except:
     print("##nodes.ChatGPT ImportError")
 
 
+from .nodes.RembgNode import get_rembg_models,U2NET_HOME,run_briarmbg,run_rembg
+
 from server import PromptServer
 
 try:
@@ -98,6 +103,26 @@ install_openai()
 current_path = os.path.abspath(os.path.dirname(__file__))
 
 
+def remove_base64_prefix(base64_str):
+  """
+  去除 base64 字符串中的 data:image/*;base64, 前缀
+
+  Args:
+    base64_str: base64 编码的字符串
+
+  Returns:
+    去除前缀后的 base64 字符串
+  """
+
+  # 使用正则表达式匹配常见的前缀
+  pattern = r'^data:image\/(.*);base64,(.+)$'
+  match = re.match(pattern, base64_str)
+  if match:
+    # 如果匹配到常见的前缀，则去除前缀并返回
+    return match.group(2)
+  else:
+    # 如果不匹配到常见的前缀，则直接返回
+    return base64_str
 
 def calculate_md5(string):
     encoded_string = string.encode()
@@ -551,7 +576,7 @@ async def mixlab_app_handler(request):
             return web.Response(text=html_data, content_type='text/html')
     else:
         return web.Response(text="HTML file not found", status=404)
-    
+
 
 @routes.post('/mixlab/workflow')
 async def mixlab_workflow_hander(request):
@@ -629,10 +654,53 @@ async def get_checkpoints(request):
             names=get_llama_models()
     except:
         print("llamafile none")
+
+    try:
+        if data['type']=='rembg':
+            names=get_rembg_models(U2NET_HOME)
+    except:
+        print("rembg none")
     
     return web.json_response({"names":names,"types":list(folder_paths.folder_names_and_paths.keys())})
 
 
+@routes.post('/mixlab/rembg')
+async def rembg_hander(request):
+    data = await request.json()
+    model=data['model']
+    result={}
+
+    data_base64=remove_base64_prefix(data['base64'])
+    image_data = base64.b64decode(data_base64)
+    
+    # 创建一个BytesIO对象
+    image_stream = io.BytesIO(image_data)
+
+    # 使用PIL Image模块读取图像
+    image = Image.open(image_stream)
+
+    if model=='briarmbg':
+        _,rgba_images,_=run_briarmbg([image])
+    else:
+        _,rgba_images,_=run_rembg(model,[image])
+
+    with io.BytesIO() as buf:
+        rgba_images[0].save(buf, format='PNG')
+        img_bytes = buf.getvalue()
+    img_base64 = base64.b64encode(img_bytes).decode('utf-8')
+    
+    try: 
+        result={
+            'data':img_base64,
+            'model':model,
+            'status':'success',
+            }
+    except Exception as e:
+            print(e)
+
+    return web.json_response(result)
+
+
 @routes.post("/mixlab/prompt_result")
 async def post_prompt_result(request):
     data = await request.json()
@@ -651,6 +719,44 @@ async def post_prompt_result(request):
     return web.json_response({"result":res})
 
 
+
+def start_local_live_thread(data):
+    import asyncio
+    from VoiceStreamAI.server import Server
+    from VoiceStreamAI.asr.asr_factory import ASRFactory
+    from VoiceStreamAI.vad.vad_factory import VADFactory
+
+    model="large-v3"
+    if "model" in data:
+        model=data['model']
+
+    vad_pipeline = VADFactory.create_vad_pipeline("pyannote")
+    #device 
+    asr_pipeline = ASRFactory.create_asr_pipeline("faster_whisper", **{"model_size":model})
+
+    port=8765
+    if 'port' in data:
+        port=data['port']
+
+    llm_port=9000
+    if 'llm_port' in data:
+        llm_port=data['llm_port']
+
+    server = Server(vad_pipeline, 
+                    asr_pipeline, 
+                    host="127.0.0.1", 
+                    port=port, 
+                    sampling_rate=16000, 
+                    samples_width=2,
+                    llm_port=llm_port
+                    )
+
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    loop.run_until_complete(server.start())
+    loop.run_forever()
+
+
 async def start_local_llm(data):
     global llama_port,llama_model,llama_chat_format
     if llama_port and llama_model and llama_chat_format:
@@ -680,6 +786,8 @@ async def start_local_llm(data):
 
 
     chat_format="chatml"
+    if "model" in data and "function-calling" in data['model']:
+        chat_format="functionary-v2"
 
     model_alias=os.path.basename(model)
 
@@ -711,12 +819,14 @@ async def start_local_llm(data):
     
     server_settings=ServerSettings(host=address,port=port)
 
+    name, ext = os.path.splitext(os.path.basename(model))
+    print('#model',name)
     app = create_app(
                 server_settings=server_settings,
                 model_settings=[
                     ModelSettings(
                     model=model,
-                    model_alias=os.path.basename(model),
+                    model_alias=name,
                     n_gpu_layers=n_gpu_layers,
                     n_ctx=4098,
                     chat_format=chat_format,
@@ -750,16 +860,52 @@ async def start_local_llm(data):
 async def my_hander_method(request):
     data =await request.json()
     # print(data)
+    if llama_port and llama_model and llama_chat_format:
+        return web.json_response({"port":llama_port,"model":llama_model,"chat_format":llama_chat_format} )
     try:
         result=await start_local_llm(data)
     except:
-        result={
-            {"port":None,"model":"","llama_cpp_error":True}
-        }
+        result= {"port":None,"model":"","llama_cpp_error":True}
         print('start_local_llm error')
 
     return web.json_response(result)
 
+
+@routes.post('/mixlab/start_live')
+async def mixlab_live_start_handler(request):
+    import threading
+    llm=await start_local_llm({
+        "model":"Phi-3-mini-4k-instruct-Q5_K_S.gguf",
+        "n_gpu_layers":2
+    })
+
+    # {"port":llama_port,"model":llama_model,"chat_format":llama_chat_format}
+
+    os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com' #hf_hub_download 里的下载地址修改
+    os.environ['PYANNOTE_AUTH_TOKEN'] = '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'
+
+    # Create and start the thread
+    data = {
+        "llm_port":llm['port'],
+        "port":8725,
+        "model":"large-v3"
+    }  # Replace with your actual data if needed
+    thread = threading.Thread(target=start_local_live_thread, args=(data,))
+    thread.start()
+
+    return web.json_response(data)
+
+
+@routes.get('/mixlab/live')
+async def mixlab_live_handler(request):
+    html_file = os.path.join(current_path, "web/live.html")
+    if os.path.exists(html_file):
+        with open(html_file, 'r', encoding='utf-8', errors='ignore') as f:
+            html_data = f.read()
+            return web.Response(text=html_data, content_type='text/html')
+    else:
+        return web.Response(text="HTML file not found", status=404)
+
 # 重启服务
 @routes.post('/mixlab/re_start')
 def re_start(request):

data/extension-node-map.json

@@ -5201,7 +5201,7 @@
             "title_aux": "ComfyUI Stable Video Diffusion"
         }
     ],
-    "https://github.com/thedyze/save-image-extended-comfyui": [
+    "https://github.com/audioscavenger/save-image-extended-comfyui": [
         [
             "SaveImageExtended"
         ],
@@ -5705,4 +5705,4 @@
             "title_aux": "SDXLCustomAspectRatio"
         }
     ]
-}
+}

nodes/RembgNode.py

@@ -467,15 +467,37 @@ class BriaRMBG(nn.Module):
 
 
 
+def get_U2NET_model_path():
+    try:
+        return folder_paths.get_folder_paths('rembg')[0]
+    except:
+        return os.path.join(folder_paths.models_dir, "rembg")
+    
 
-
-
-U2NET_HOME=os.path.join(folder_paths.models_dir, "rembg")
+U2NET_HOME=get_U2NET_model_path()
 os.environ["U2NET_HOME"] = U2NET_HOME
 
 global _available
 _available=False
 
+
+def get_rembg_models(path):
+    """从目录中获取文件并提取文件名
+    Args:
+        path: 目录路径
+    Returns:
+        文件名列表
+    """
+    filenames = []
+    for root, _, files in os.walk(path):
+        for filename in files:
+        # 过滤隐藏文件
+            if not filename.startswith('.'):
+                name, ext = os.path.splitext(os.path.basename(filename))
+                filenames.append(name)
+    return filenames
+
+
 def is_installed(package):
     try:
         spec = importlib.util.find_spec(package)
@@ -509,8 +531,8 @@ except:
     _available=False
 
 
-def briarmbg_run(images=[]):
-    mroot=os.path.join(folder_paths.models_dir, "rembg")
+def run_briarmbg(images=[]):
+    mroot=U2NET_HOME
     m=os.path.join(mroot,'briarmbg.pth')
     if os.path.exists(m)==False:
         # 下载
@@ -573,14 +595,15 @@ def briarmbg_run(images=[]):
     return (masks,rgba_images,rgb_images)   
 
 
-def run_bg(model_name= "unet",images=[]):
+def run_rembg(model_name= "unet",images=[],callback=None):
     # model_name = "unet" # "isnet-general-use"
+    # print('#run_rembg',model_name)
     rembg_session = new_session(model_name)
     masks=[]
     rgba_images=[]
     rgb_images=[]
     # 进度条
-    pbar = comfy.utils.ProgressBar(len(images) )
+    pbar=callback 
     for img in images:
         # use the post_process_mask argument to post process the mask to get better results.
         mask = remove(img, session=rembg_session,only_mask=True,post_process_mask=True)
@@ -620,8 +643,9 @@ def run_bg(model_name= "unet",images=[]):
             rgb_image = Image.new("RGB", image_rgba.size, (0, 0, 0))
             rgb_image.paste(image_rgba, mask=image_rgba.split()[3])
             rgb_images.append(rgb_image)
-            
-        pbar.update(1)
+
+        if pbar:
+            pbar.update(1)
     return (masks,rgba_images,rgb_images)
 
 
@@ -643,17 +667,7 @@ class RembgNode_:
     def INPUT_TYPES(s):
         return {"required": {
             "image": ("IMAGE",),
-            "model_name": ([
-                            "briarmbg",
-                            "u2net",
-                            "u2netp",
-                            "u2net_human_seg",
-                            "u2net_cloth_seg",
-                            "silueta",
-                            "isnet-general-use",
-                            "isnet-anime",
-                            
-                            ],),
+            "model_name": (get_rembg_models(U2NET_HOME),),
             
                              },
                 }
@@ -681,9 +695,9 @@ class RembgNode_:
                 images.append(im)
 
         if model_name=='briarmbg':
-            masks,rgba_images,rgb_images=briarmbg_run(images)
+            masks,rgba_images,rgb_images=run_briarmbg(images)
         else:
-            masks,rgba_images,rgb_images=run_bg(model_name,images)
+            masks,rgba_images,rgb_images=run_rembg(model_name,images, comfy.utils.ProgressBar(len(images) ))
 
         masks=[pil2tensor(m) for m in masks]
 

nodes/TripoSR.py

@@ -5,13 +5,20 @@ from PIL import Image
 import numpy as np
 import torch
 
-from folder_paths import get_filename_list, get_full_path, get_save_image_path, get_output_directory,models_dir
+from folder_paths import get_folder_paths, get_full_path, get_save_image_path, get_output_directory,models_dir
 from comfy.model_management import get_torch_device
 from .tsr.system import TSR
 
 import comfy.utils
 
-triposr_model_path=path.join(models_dir,'triposr/model.ckpt')
+
+def get_triposr_model_path():
+    try:
+        return path.join(get_folder_paths('triposr')[0],'model.ckpt')
+    except:
+        return path.join(path.join(models_dir, "triposr"),'model.ckpt')
+    
+triposr_model_path=get_triposr_model_path()
 
 
 # Tensor to PIL

nodes/Utils.py

@@ -284,7 +284,7 @@ class FloatSlider:
                 }
     
     RETURN_TYPES = ("FLOAT",) 
-    RETURN_NAMES = ('weight(0-1)',)
+    RETURN_NAMES = ('FLOAT',)
     FUNCTION = "run"
 
     CATEGORY = "♾️Mixlab/Input"
@@ -297,9 +297,7 @@ class FloatSlider:
             number = min_value
         elif number > max_value:
             number = max_value
-        scaled_number = (number - min_value) / (max_value - min_value)
-        return (scaled_number,)
-    
+        return (number,)    
   
 class IntNumber:
     @classmethod

nodes/Video.py

@@ -87,7 +87,7 @@ def split_video(video_path, video_segment_frames, transition_frames, output_dir)
 
 folder_paths.folder_names_and_paths["video_formats"] = (
     [
-        os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "video_formats"),
+        os.path.join(os.path.dirname(os.path.abspath(__file__)), ".", "video_formats"),
     ],
     [".json"]
 )
@@ -433,6 +433,7 @@ class VideoCombine_Adv:
             ffmpeg_formats = ["video/"+x[:-5] for x in folder_paths.get_filename_list("video_formats")]
         else:
             ffmpeg_formats = []
+        # ffmpeg_formats =["video/"+x for x in  ['webm', 'mp4', 'mkv']]
         return {
             "required": {
                 "image_batch": ("IMAGE",),

nodes/VoiceStreamAI/asr/asr_factory.py

@@ -0,0 +1,12 @@
+from VoiceStreamAI.asr.whisper_asr import WhisperASR
+from VoiceStreamAI.asr.faster_whisper_asr import FasterWhisperASR
+
+class ASRFactory:
+    @staticmethod
+    def create_asr_pipeline(type, **kwargs):
+        if type == "whisper":
+            return WhisperASR(**kwargs)
+        if type == "faster_whisper":
+            return FasterWhisperASR(**kwargs)
+        else:
+            raise ValueError(f"Unknown ASR pipeline type: {type}")

nodes/VoiceStreamAI/asr/asr_interface.py

@@ -0,0 +1,9 @@
+class ASRInterface:
+    async def transcribe(self, client):
+        """
+        Transcribe the given audio data.
+
+        :param client: The client object with all the member variables including the buffer
+        :return: The transcription structure, see for example the faster_whisper_asr.py file.
+        """
+        raise NotImplementedError("This method should be implemented by subclasses.")

nodes/VoiceStreamAI/asr/faster_whisper_asr.py

@@ -0,0 +1,142 @@
+import os
+from faster_whisper import WhisperModel
+
+from VoiceStreamAI.asr.asr_interface import ASRInterface
+from VoiceStreamAI.audio_utils import save_audio_to_file
+
+import folder_paths
+
+language_codes = {
+    "afrikaans": "af",
+    "amharic": "am",
+    "arabic": "ar",
+    "assamese": "as",
+    "azerbaijani": "az",
+    "bashkir": "ba",
+    "belarusian": "be",
+    "bulgarian": "bg",
+    "bengali": "bn",
+    "tibetan": "bo",
+    "breton": "br",
+    "bosnian": "bs",
+    "catalan": "ca",
+    "czech": "cs",
+    "welsh": "cy",
+    "danish": "da",
+    "german": "de",
+    "greek": "el",
+    "english": "en",
+    "spanish": "es",
+    "estonian": "et",
+    "basque": "eu",
+    "persian": "fa",
+    "finnish": "fi",
+    "faroese": "fo",
+    "french": "fr",
+    "galician": "gl",
+    "gujarati": "gu",
+    "hausa": "ha",
+    "hawaiian": "haw",
+    "hebrew": "he",
+    "hindi": "hi",
+    "croatian": "hr",
+    "haitian": "ht",
+    "hungarian": "hu",
+    "armenian": "hy",
+    "indonesian": "id",
+    "icelandic": "is",
+    "italian": "it",
+    "japanese": "ja",
+    "javanese": "jw",
+    "georgian": "ka",
+    "kazakh": "kk",
+    "khmer": "km",
+    "kannada": "kn",
+    "korean": "ko",
+    "latin": "la",
+    "luxembourgish": "lb",
+    "lingala": "ln",
+    "lao": "lo",
+    "lithuanian": "lt",
+    "latvian": "lv",
+    "malagasy": "mg",
+    "maori": "mi",
+    "macedonian": "mk",
+    "malayalam": "ml",
+    "mongolian": "mn",
+    "marathi": "mr",
+    "malay": "ms",
+    "maltese": "mt",
+    "burmese": "my",
+    "nepali": "ne",
+    "dutch": "nl",
+    "norwegian nynorsk": "nn",
+    "norwegian": "no",
+    "occitan": "oc",
+    "punjabi": "pa",
+    "polish": "pl",
+    "pashto": "ps",
+    "portuguese": "pt",
+    "romanian": "ro",
+    "russian": "ru",
+    "sanskrit": "sa",
+    "sindhi": "sd",
+    "sinhalese": "si",
+    "slovak": "sk",
+    "slovenian": "sl",
+    "shona": "sn",
+    "somali": "so",
+    "albanian": "sq",
+    "serbian": "sr",
+    "sundanese": "su",
+    "swedish": "sv",
+    "swahili": "sw",
+    "tamil": "ta",
+    "telugu": "te",
+    "tajik": "tg",
+    "thai": "th",
+    "turkmen": "tk",
+    "tagalog": "tl",
+    "turkish": "tr",
+    "tatar": "tt",
+    "ukrainian": "uk",
+    "urdu": "ur",
+    "uzbek": "uz",
+    "vietnamese": "vi",
+    "yiddish": "yi",
+    "yoruba": "yo",
+    "chinese": "zh",
+    "cantonese": "yue",
+}
+
+
+class FasterWhisperASR(ASRInterface):
+    def __init__(self, **kwargs):
+        model_size = kwargs.get('model_size', "large-v3")
+        device = kwargs.get('device', "cuda")
+        model_root = os.path.join(folder_paths.models_dir, "whisper") 
+        # Run on GPU with FP16
+        self.asr_pipeline = WhisperModel(model_size, device=device, compute_type="float16",download_root=model_root)
+
+    async def transcribe(self, client):
+        file_path = await save_audio_to_file(client.scratch_buffer, client.get_file_name())
+
+        language = None if client.config['language'] is None else language_codes.get(client.config['language'].lower())
+        segments, info = self.asr_pipeline.transcribe(file_path, word_timestamps=True, language=language)
+
+        segments = list(segments)  # The transcription will actually run here.
+        os.remove(file_path)
+
+        flattened_words = [word for segment in segments for word in segment.words]
+
+        to_return = {
+        "language": info.language,
+        "language_probability": info.language_probability,
+        "text": ' '.join([s.text.strip() for s in segments]),
+        "words":
+            [
+                {"word": w.word, "start": w.start, "end": w.end, "probability":w.probability} for w in flattened_words
+            ]
+        }
+        return to_return
+

nodes/VoiceStreamAI/asr/whisper_asr.py

@@ -0,0 +1,27 @@
+from transformers import pipeline
+from VoiceStreamAI.asr.asr_interface import ASRInterface
+from VoiceStreamAI.audio_utils import save_audio_to_file
+import os
+
+class WhisperASR(ASRInterface):
+    def __init__(self, **kwargs):
+        model_name = kwargs.get('model_name', "openai/whisper-large-v3")
+        self.asr_pipeline = pipeline("automatic-speech-recognition", model=model_name)
+
+    async def transcribe(self, client):
+        file_path = await save_audio_to_file(client.scratch_buffer, client.get_file_name())
+        
+        if client.config['language'] is not None:
+            to_return = self.asr_pipeline(file_path, generate_kwargs={"language": client.config['language']})['text']
+        else:
+            to_return = self.asr_pipeline(file_path)['text']
+
+        os.remove(file_path)
+
+        to_return = {
+            "language": "UNSUPPORTED_BY_HUGGINGFACE_WHISPER",
+            "language_probability": None,
+            "text": to_return.strip(),
+            "words": "UNSUPPORTED_BY_HUGGINGFACE_WHISPER"
+        }
+        return to_return

nodes/VoiceStreamAI/audio_utils.py

@@ -0,0 +1,26 @@
+import wave
+import os
+
+async def save_audio_to_file(audio_data, file_name, audio_dir="audio_files", audio_format="wav"):
+    """
+    Saves the audio data to a file.
+
+    :param client_id: Unique identifier for the client.
+    :param audio_data: The audio data to save.
+    :param file_counters: Dictionary to keep track of file counts for each client.
+    :param audio_dir: Directory where audio files will be saved.
+    :param audio_format: Format of the audio file.
+    :return: Path to the saved audio file.
+    """
+
+    os.makedirs(audio_dir, exist_ok=True)
+    
+    file_path = os.path.join(audio_dir, file_name)
+
+    with wave.open(file_path, 'wb') as wav_file:
+        wav_file.setnchannels(1)  # Assuming mono audio
+        wav_file.setsampwidth(2)
+        wav_file.setframerate(16000)
+        wav_file.writeframes(audio_data)
+
+    return file_path

nodes/VoiceStreamAI/buffering_strategy/buffering_strategies.py

@@ -0,0 +1,142 @@
+import os
+import asyncio
+import json
+import time
+
+from VoiceStreamAI.buffering_strategy.buffering_strategy_interface import BufferingStrategyInterface
+from openai import OpenAI
+
+class SilenceAtEndOfChunk(BufferingStrategyInterface):
+    """
+    A buffering strategy that processes audio at the end of each chunk with silence detection.
+
+    This class is responsible for handling audio chunks, detecting silence at the end of each chunk,
+    and initiating the transcription process for the chunk.
+
+    Attributes:
+        client (Client): The client instance associated with this buffering strategy.
+        chunk_length_seconds (float): Length of each audio chunk in seconds.
+        chunk_offset_seconds (float): Offset time in seconds to be considered for processing audio chunks.
+    """
+
+    def __init__(self, client, **kwargs):
+        """
+        Initialize the SilenceAtEndOfChunk buffering strategy.
+
+        Args:
+            client (Client): The client instance associated with this buffering strategy.
+            **kwargs: Additional keyword arguments, including 'chunk_length_seconds' and 'chunk_offset_seconds'.
+        """
+        self.client = client
+
+        self.chunk_length_seconds = os.environ.get('BUFFERING_CHUNK_LENGTH_SECONDS')
+        if not self.chunk_length_seconds:
+            self.chunk_length_seconds = kwargs.get('chunk_length_seconds')
+        self.chunk_length_seconds = float(self.chunk_length_seconds)
+
+        self.chunk_offset_seconds = os.environ.get('BUFFERING_CHUNK_OFFSET_SECONDS')
+        if not self.chunk_offset_seconds:
+            self.chunk_offset_seconds = kwargs.get('chunk_offset_seconds')
+        self.chunk_offset_seconds = float(self.chunk_offset_seconds)
+
+        self.error_if_not_realtime = os.environ.get('ERROR_IF_NOT_REALTIME')
+        if not self.error_if_not_realtime:
+            self.error_if_not_realtime = kwargs.get('error_if_not_realtime', False)
+        
+        self.processing_flag = False
+
+        self.messages=[]
+
+    def process_audio(self, websocket, vad_pipeline, asr_pipeline,llm_port):
+        """
+        Process audio chunks by checking their length and scheduling asynchronous processing.
+
+        This method checks if the length of the audio buffer exceeds the chunk length and, if so,
+        it schedules asynchronous processing of the audio.
+
+        Args:
+            websocket (Websocket): The WebSocket connection for sending transcriptions.
+            vad_pipeline: The voice activity detection pipeline.
+            asr_pipeline: The automatic speech recognition pipeline.
+        """
+        chunk_length_in_bytes = self.chunk_length_seconds * self.client.sampling_rate * self.client.samples_width
+        if len(self.client.buffer) > chunk_length_in_bytes:
+            if self.processing_flag:
+                exit("Error in realtime processing: tried processing a new chunk while the previous one was still being processed")
+
+            self.client.scratch_buffer += self.client.buffer
+            self.client.buffer.clear()
+            self.processing_flag = True
+            # Schedule the processing in a separate task
+            asyncio.create_task(self.process_audio_async(websocket, vad_pipeline, asr_pipeline,llm_port))
+    
+    async def process_audio_async(self, websocket, vad_pipeline, asr_pipeline,llm_port):
+        """
+        Asynchronously process audio for activity detection and transcription.
+
+        This method performs heavy processing, including voice activity detection and transcription of
+        the audio data. It sends the transcription results through the WebSocket connection.
+
+        Args:
+            websocket (Websocket): The WebSocket connection for sending transcriptions.
+            vad_pipeline: The voice activity detection pipeline.
+            asr_pipeline: The automatic speech recognition pipeline.
+        """   
+        start = time.time()
+        vad_results = await vad_pipeline.detect_activity(self.client)
+
+        if len(vad_results) == 0:
+            self.client.scratch_buffer.clear()
+            self.client.buffer.clear()
+            self.processing_flag = False
+            return
+
+        last_segment_should_end_before = ((len(self.client.scratch_buffer) / (self.client.sampling_rate * self.client.samples_width)) - self.chunk_offset_seconds)
+        if vad_results[-1]['end'] < last_segment_should_end_before:
+            transcription = await asr_pipeline.transcribe(self.client)
+            if transcription['text'] != '':
+                end = time.time()
+                transcription['processing_time'] = end - start
+
+                transcription['status']="chat_start"
+
+                json_transcription = json.dumps(transcription)
+
+                await websocket.send(json_transcription)
+
+                # Point to the local server
+                client = OpenAI(base_url=f"http://localhost:{llm_port}/v1", api_key="lm-studio")
+
+               
+                messages=[
+                    {"role": "system", "content": "You are a friendly and engaging AI designed to interact with users in a conversational manner. Your personality is that of a sophisticated and polite young professional who is both a designer and a programmer. You are well-mannered, articulate, and possess a good sense of humor. Your goal is to provide helpful and insightful responses while maintaining a pleasant and enjoyable conversation. Be sure to use your knowledge in design and programming to enrich the dialogue and offer relevant advice or information when appropriate. Always be respectful and considerate of the user's feelings and perspectives. Additionally, you are fluent in both English and Chinese, and can seamlessly switch between the two languages to best assist users."}, 
+                ]+self.messages[-10:0]+[{"role": "user", "content":transcription['text']}]
+                # print('#messages',messages)
+                
+                completion = client.chat.completions.create(
+                    model="model-identifier",
+                    messages=messages,
+                    temperature=0.7,
+                    )
+
+                transcription['asistant'] = completion.choices[0].message.content
+
+                transcription['status']="chat_end"
+
+                json_transcription = json.dumps(transcription) 
+
+                self.messages.append({
+                    "role": "user", 
+                    "content":transcription['text']})
+                
+                self.messages.append({
+                    "role": "asistant", 
+                    "content": transcription['asistant']
+                                      })
+                # print('#messages',completion.choices[0].message.content)
+
+                await websocket.send(json_transcription)
+            self.client.scratch_buffer.clear()
+            self.client.increment_file_counter()
+        
+        self.processing_flag = False

nodes/VoiceStreamAI/buffering_strategy/buffering_strategy_factory.py

@@ -0,0 +1,41 @@
+from VoiceStreamAI.buffering_strategy.buffering_strategies import SilenceAtEndOfChunk
+
+class BufferingStrategyFactory:
+    """
+    A factory class for creating instances of different buffering strategies.
+
+    This factory provides a centralized way to instantiate various buffering strategies
+    based on the type specified. It abstracts the creation logic, making it easier to
+    manage and extend with new buffering strategy types.
+
+    Methods:
+        create_buffering_strategy: Creates and returns an instance of a specified buffering strategy.
+    """
+
+    @staticmethod
+    def create_buffering_strategy(type, client, **kwargs):
+        """
+        Creates an instance of a buffering strategy based on the specified type.
+
+        This method acts as a factory for creating buffering strategy objects. It returns
+        an instance of the strategy corresponding to the given type. If the type is not
+        recognized, it raises a ValueError.
+
+        Args:
+            type (str): The type of buffering strategy to create. Currently supports 'silence_at_end_of_chunk'.
+            client (Client): The client instance to be associated with the buffering strategy.
+            **kwargs: Additional keyword arguments specific to the buffering strategy being created.
+
+        Returns:
+            An instance of the specified buffering strategy.
+
+        Raises:
+            ValueError: If the specified type is not recognized or supported.
+
+        Example:
+            strategy = BufferingStrategyFactory.create_buffering_strategy("silence_at_end_of_chunk", client)
+        """
+        if type == "silence_at_end_of_chunk":
+            return SilenceAtEndOfChunk(client, **kwargs)
+        else:
+            raise ValueError(f"Unknown buffering strategy type: {type}")

nodes/VoiceStreamAI/buffering_strategy/buffering_strategy_interface.py

@@ -0,0 +1,31 @@
+class BufferingStrategyInterface:
+    """
+    An interface class for buffering strategies in audio processing systems.
+
+    This class defines the structure for buffering strategies used in handling
+    and processing audio data. It serves as a template for creating custom buffering
+    strategies that fit specific requirements of an audio processing pipeline.
+
+    Subclasses should implement the methods defined in this interface to ensure
+    consistency and compatibility with the system's audio processing framework.
+
+    Methods:
+        process_audio: Process audio data. This method should be implemented by subclasses.
+    """
+
+    def process_audio(self, websocket, vad_pipeline, asr_pipeline):
+        """
+        Process audio data using the given WebSocket connection, VAD pipeline, and ASR pipeline.
+
+        This method is intended to be overridden in subclasses to provide specific logic
+        for handling and processing audio data in different buffering strategies.
+
+        Args:
+            websocket (Websocket): The WebSocket connection for communication with clients.
+            vad_pipeline: The Voice Activity Detection (VAD) pipeline used for detecting speech in the audio.
+            asr_pipeline: The Automatic Speech Recognition (ASR) pipeline used for transcribing speech in the audio.
+
+        Raises:
+            NotImplementedError: If the method is not implemented in the subclass.
+        """
+        raise NotImplementedError("This method should be implemented by subclasses.")

nodes/VoiceStreamAI/client.py

@@ -0,0 +1,54 @@
+from VoiceStreamAI.buffering_strategy.buffering_strategy_factory import BufferingStrategyFactory
+
+class Client:
+    """
+    Represents a client connected to the VoiceStreamAI server.
+
+    This class maintains the state for each connected client, including their
+    unique identifier, audio buffer, configuration, and a counter for processed audio files.
+
+    Attributes:
+        client_id (str): A unique identifier for the client.
+        buffer (bytearray): A buffer to store incoming audio data.
+        config (dict): Configuration settings for the client, like chunk length and offset.
+        file_counter (int): Counter for the number of audio files processed.
+        total_samples (int): Total number of audio samples received from this client.
+        sampling_rate (int): The sampling rate of the audio data in Hz.
+        samples_width (int): The width of each audio sample in bits.
+    """
+    def __init__(self, client_id, sampling_rate, samples_width):
+        self.client_id = client_id
+        self.buffer = bytearray()
+        self.scratch_buffer = bytearray()
+        self.config = {"language": None,
+                       "processing_strategy": "silence_at_end_of_chunk", 
+                       "processing_args": {
+                           "chunk_length_seconds": 5, 
+                           "chunk_offset_seconds": 0.1
+                           }
+                       }
+        self.file_counter = 0
+        self.total_samples = 0
+        self.sampling_rate = sampling_rate
+        self.samples_width = samples_width
+        self.buffering_strategy = BufferingStrategyFactory.create_buffering_strategy(self.config['processing_strategy'], self, **self.config['processing_args'])
+
+    def update_config(self, config_data):
+        self.config.update(config_data)
+        self.buffering_strategy = BufferingStrategyFactory.create_buffering_strategy(self.config['processing_strategy'], self, **self.config['processing_args'])
+
+    def append_audio_data(self, audio_data):
+        self.buffer.extend(audio_data)
+        self.total_samples += len(audio_data) / self.samples_width
+
+    def clear_buffer(self):
+        self.buffer.clear()
+
+    def increment_file_counter(self):
+        self.file_counter += 1
+
+    def get_file_name(self):
+        return f"{self.client_id}_{self.file_counter}.wav"
+    
+    def process_audio(self, websocket, vad_pipeline, asr_pipeline,llm_port):
+        self.buffering_strategy.process_audio(websocket, vad_pipeline, asr_pipeline,llm_port)

nodes/VoiceStreamAI/main.py

@@ -0,0 +1,54 @@
+import argparse
+import asyncio
+import json
+
+import os
+import sys
+from pathlib import Path
+
+
+# 获取当前文件的绝对路径
+current_file_path = os.path.abspath(__file__)
+
+# 获取当前文件的目录
+current_directory = os.path.dirname(current_file_path)
+sys.path.append(str(Path(current_directory).parent))
+# print("sys.path", current_directory)
+
+
+from VoiceStreamAI.server import Server
+from VoiceStreamAI.asr.asr_factory import ASRFactory
+from VoiceStreamAI.vad.vad_factory import VADFactory
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="VoiceStreamAI Server: Real-time audio transcription using self-hosted Whisper and WebSocket")
+    parser.add_argument("--vad-type", type=str, default="pyannote", help="Type of VAD pipeline to use (e.g., 'pyannote')")
+    parser.add_argument("--vad-args", type=str, default='{"auth_token": "huggingface_token"}', help="JSON string of additional arguments for VAD pipeline")
+    parser.add_argument("--asr-type", type=str, default="faster_whisper", help="Type of ASR pipeline to use (e.g., 'whisper')")
+    parser.add_argument("--asr-args", type=str, default='{"model_size": "large-v3"}', help="JSON string of additional arguments for ASR pipeline")
+    parser.add_argument("--host", type=str, default="127.0.0.1", help="Host for the WebSocket server")
+    parser.add_argument("--port", type=int, default=8765, help="Port for the WebSocket server")
+    parser.add_argument("--certfile", type=str, default=None, help="The path to the SSL certificate (cert file) if using secure websockets")
+    parser.add_argument("--keyfile", type=str, default=None, help="The path to the SSL key file if using secure websockets")
+    return parser.parse_args()
+
+def main():
+    args = parse_args()
+
+    try:
+        vad_args = json.loads(args.vad_args)
+        asr_args = json.loads(args.asr_args)
+    except json.JSONDecodeError as e:
+        print(f"Error parsing JSON arguments: {e}")
+        return
+
+    vad_pipeline = VADFactory.create_vad_pipeline(args.vad_type, **vad_args)
+    asr_pipeline = ASRFactory.create_asr_pipeline(args.asr_type, **asr_args)
+
+    server = Server(vad_pipeline, asr_pipeline, host=args.host, port=args.port, sampling_rate=16000, samples_width=2, certfile=args.certfile, keyfile=args.keyfile)
+
+    asyncio.get_event_loop().run_until_complete(server.start())
+    asyncio.get_event_loop().run_forever()
+
+if __name__ == "__main__":
+    main()

nodes/VoiceStreamAI/requirements.txt

@@ -0,0 +1,7 @@
+websockets
+speechbrain
+pyannote-audio
+asyncio
+sentence-transformers
+transformers
+faster-whisper

nodes/VoiceStreamAI/server.py

@@ -0,0 +1,88 @@
+import websockets
+import uuid
+import json
+import asyncio
+import ssl
+
+from VoiceStreamAI.audio_utils import save_audio_to_file
+from VoiceStreamAI.client import Client
+
+class Server:
+    """
+    Represents the WebSocket server for handling real-time audio transcription.
+
+    This class manages WebSocket connections, processes incoming audio data,
+    and interacts with VAD and ASR pipelines for voice activity detection and
+    speech recognition.
+
+    Attributes:
+        vad_pipeline: An instance of a voice activity detection pipeline.
+        asr_pipeline: An instance of an automatic speech recognition pipeline.
+        host (str): Host address of the server.
+        port (int): Port on which the server listens.
+        sampling_rate (int): The sampling rate of audio data in Hz.
+        samples_width (int): The width of each audio sample in bits.
+        connected_clients (dict): A dictionary mapping client IDs to Client objects.
+    """
+    def __init__(self, vad_pipeline, asr_pipeline, host='localhost', port=8765, sampling_rate=16000, samples_width=2, certfile = None, keyfile = None,llm_port=9000):
+        self.vad_pipeline = vad_pipeline
+        self.asr_pipeline = asr_pipeline
+        self.host = host
+        self.port = port
+        self.sampling_rate = sampling_rate
+        self.samples_width = samples_width
+        self.certfile = certfile
+        self.keyfile = keyfile
+        self.connected_clients = {}
+
+        self.llm_port=llm_port
+
+    async def handle_audio(self, client, websocket):
+        while True:
+            message = await websocket.recv()
+
+            if isinstance(message, bytes):
+                client.append_audio_data(message)
+            elif isinstance(message, str):
+                config = json.loads(message)
+                if config.get('type') == 'config':
+                    client.update_config(config['data'])
+                    continue
+            else:
+                print(f"Unexpected message type from {client.client_id}")
+
+            # this is synchronous, any async operation is in BufferingStrategy
+            client.process_audio(websocket, self.vad_pipeline, self.asr_pipeline,self.llm_port)
+
+
+    async def handle_websocket(self, websocket, path):
+        client_id = str(uuid.uuid4())
+        client = Client(client_id, self.sampling_rate, self.samples_width)
+        self.connected_clients[client_id] = client
+
+        print(f"Client {client_id} connected")
+
+        try:
+            await self.handle_audio(client, websocket)
+        except websockets.ConnectionClosed as e:
+            print(f"Connection with {client_id} closed: {e}")
+        finally:
+            del self.connected_clients[client_id]
+
+    def start(self):
+        if self.certfile:
+            # Create an SSL context to enforce encrypted connections
+            ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+            
+            # Load your server's certificate and private key
+            # Replace 'your_cert_path.pem' and 'your_key_path.pem' with the actual paths to your files
+            ssl_context.load_cert_chain(certfile=self.certfile, keyfile=self.keyfile)
+
+            print(f"WebSocket server ready to accept secure connections on {self.host}:{self.port}")
+            
+            # Pass the SSL context to the serve function along with the host and port
+            # Ensure the secure flag is set to True if using a secure WebSocket protocol (wss://)
+            return websockets.serve(self.handle_websocket, self.host, self.port, ssl=ssl_context)
+        else:
+            print(f"WebSocket server ready to accept secure connections on {self.host}:{self.port}")
+            return websockets.serve(self.handle_websocket, self.host, self.port)

nodes/VoiceStreamAI/vad/pyannote_vad.py

@@ -0,0 +1,50 @@
+from os import remove
+import os
+
+from pyannote.core import Segment
+from pyannote.audio import Model
+from pyannote.audio.pipelines import VoiceActivityDetection
+
+from VoiceStreamAI.vad.vad_interface import VADInterface
+from VoiceStreamAI.audio_utils import save_audio_to_file
+
+
+class PyannoteVAD(VADInterface):
+    """
+    Pyannote-based implementation of the VADInterface.
+    """
+
+    def __init__(self, **kwargs):
+        """
+        Initializes Pyannote's VAD pipeline.
+
+        Args:
+            model_name (str): The model name for Pyannote.
+            auth_token (str, optional): Authentication token for Hugging Face.
+        """
+        
+        model_name = kwargs.get('model_name', "pyannote/segmentation")
+
+        auth_token = os.environ.get('PYANNOTE_AUTH_TOKEN')
+        if not auth_token:
+            auth_token = kwargs.get('auth_token')
+        
+        if auth_token is None:
+            raise ValueError("Missing required env var in PYANNOTE_AUTH_TOKEN or argument in --vad-args: 'auth_token'")
+        
+        pyannote_args = kwargs.get('pyannote_args', {"onset": 0.5, "offset": 0.5, "min_duration_on": 0.3, "min_duration_off": 0.3})
+        self.model = Model.from_pretrained(model_name, use_auth_token=auth_token)
+        self.vad_pipeline = VoiceActivityDetection(segmentation=self.model)
+        self.vad_pipeline.instantiate(pyannote_args)
+
+    async def detect_activity(self, client):
+        audio_file_path = await save_audio_to_file(client.scratch_buffer, client.get_file_name())
+        vad_results = self.vad_pipeline(audio_file_path)
+        remove(audio_file_path)
+        vad_segments = []
+        if len(vad_results) > 0:
+            vad_segments = [
+                {"start": segment.start, "end": segment.end, "confidence": 1.0}
+                for segment in vad_results.itersegments()
+            ]
+        return vad_segments

nodes/VoiceStreamAI/vad/vad_factory.py

@@ -0,0 +1,23 @@
+from VoiceStreamAI.vad.pyannote_vad import PyannoteVAD
+
+class VADFactory:
+    """
+    Factory for creating instances of VAD systems.
+    """
+
+    @staticmethod
+    def create_vad_pipeline(type, **kwargs):
+        """
+        Creates a VAD pipeline based on the specified type.
+
+        Args:
+            type (str): The type of VAD pipeline to create (e.g., 'pyannote').
+            kwargs: Additional arguments for the VAD pipeline creation.
+
+        Returns:
+            VADInterface: An instance of a class that implements VADInterface.
+        """
+        if type == "pyannote":
+            return PyannoteVAD(**kwargs)
+        else:
+            raise ValueError(f"Unknown VAD pipeline type: {type}")

nodes/VoiceStreamAI/vad/vad_interface.py

@@ -0,0 +1,16 @@
+class VADInterface:
+    """
+    Interface for voice activity detection (VAD) systems.
+    """
+
+    async def detect_activity(self, client):
+        """
+        Detects voice activity in the given audio data.
+
+        Args:
+            client (src.Client): The client to detect on
+
+        Returns:
+            List: VAD result, a list of objects containing "start", "end", "confidence"
+        """
+        raise NotImplementedError("This method should be implemented by subclasses.")

nodes/video_formats/av1-webm.json

@@ -0,0 +1,10 @@
+{
+    "main_pass":
+    [
+        "-n", "-c:v", "libsvtav1",
+        "-pix_fmt", "yuv420p10le",
+        "-crf", "23"
+    ],
+     "extension": "webm",
+     "environment": {"SVT_LOG": "1"}
+}

nodes/video_formats/h264-mp4.json

@@ -0,0 +1,9 @@
+{
+    "main_pass":
+    [
+        "-n", "-c:v", "libx264",
+        "-pix_fmt", "yuv420p",
+        "-crf", "19"
+    ],
+     "extension": "mp4"
+}

nodes/video_formats/h265-mp4.json

@@ -0,0 +1,11 @@
+{
+    "main_pass":
+    [
+        "-n", "-c:v", "libx265",
+        "-pix_fmt", "yuv420p10le",
+        "-preset", "medium",
+        "-crf", "22",
+        "-x265-params", "log-level=quiet"
+    ],
+     "extension": "mp4"
+}

nodes/video_formats/webm.json

@@ -0,0 +1,9 @@
+{
+    "main_pass":
+    [
+        "-n",
+        "-pix_fmt", "yuv420p",
+        "-crf", "23"
+    ],
+     "extension": "webm"
+}

pyproject.toml

@@ -0,0 +1,15 @@
+[project]
+name = "comfyui-mixlab-nodes"
+description = "3D, ScreenShareNode & FloatingVideoNode, SpeechRecognition & SpeechSynthesis, GPT, LoadImagesFromLocal, Layers, Other Nodes, ..."
+version = "0.28.3"
+license = "MIT"
+dependencies = ["numpy", "pyOpenSSL", "watchdog", "opencv-python-headless", "matplotlib", "openai", "simple-lama-inpainting", "clip-interrogator==0.6.0", "transformers>=4.36.0", "lark-parser", "imageio-ffmpeg", "rembg[gpu]", "omegaconf==2.3.0", "Pillow>=9.5.0", "einops==0.7.0", "trimesh>=4.0.5", "huggingface-hub", "scikit-image"]
+
+[project.urls]
+Repository = "https://github.com/shadowcz007/comfyui-mixlab-nodes"
+#  Used by Comfy Registry https://comfyregistry.org
+
+[tool.comfy]
+PublisherId = "shadow"
+DisplayName = "comfyui-mixlab-nodes"
+Icon = ""

web/index.html

@@ -24,6 +24,7 @@
             margin-left: 5%;
             user-select: none;
             margin-top: 32px;
+            margin-bottom: 120px;
         }
 
         .apps {
@@ -65,9 +66,9 @@
             box-shadow: 0px 0px 10px 10px #fbe9f0
         }
 
-        .card:hover {
+        /* .card:hover {
             box-shadow: 0px 0px 10px 10px #e9fbfa
-        }
+        } */
 
         .card h5 {
             font-size: 14px;
@@ -145,15 +146,6 @@
             margin-top: 4px;
         }
 
-        .status {
-            background: black;
-            color: white;
-            display: flex;
-            width: fit-content;
-            padding: 4px;
-            font-size: 12px;
-
-        }
 
         .description {
             display: flex;
@@ -247,7 +239,7 @@
 
         .card textarea {
             width: 100%;
-            height:'fit-content';
+            height: 'fit-content';
             /* min-width: 300px; */
             margin-top: 12px;
             resize: none;
@@ -263,18 +255,44 @@
             margin-top: 12px;
         }
 
+        .run_div {
+            position: fixed;
+            bottom: 28px;
+            display: flex;
+            left: 144px;
+            z-index: 100;
+            /* align-items: center; */
+            /* justify-content: space-around; */
+            /* width: calc(50% - 100px); */
+            flex-direction: column;
+            background-color: #f7f7f7;
+            box-shadow: 3px 3px 8px #cacaca;
+            border-radius: 8px;
+        }
+
+        .status {
+            /* background: #e7e7e7; */
+            color: black;
+            display: flex;
+            width: 120px;
+            padding-left: 11px;
+            font-size: 12px;
+            border-radius: 8px;
+            justify-content: flex-start;
+            align-items: center;
+            height: 48px;
+        }
+
         .run_btn {
             background: black;
             color: white;
-            width: 88px;
-            height: 88px;
-            position: fixed;
-            bottom: 72px;
-            left: calc(50% - 44px);
-            border-radius: 100%;
+            /* width: calc(50% - 100px); */
+            min-width: 200px;
+            max-width: 460px;
+            height: 48px;
+            border-radius: 8px;
             cursor: pointer;
             border: 3px solid;
-            z-index: 100;
         }
 
         button:hover {
@@ -452,7 +470,7 @@
     <link href="/extensions/comfyui-mixlab-nodes/lib/photoswipe.min.css" rel="stylesheet">
     <link href="/extensions/comfyui-mixlab-nodes/lib/classic.min.css" rel="stylesheet">
     <script src="/extensions/comfyui-mixlab-nodes/lib/pickr.min.js"></script>
-    <script src="/extensions/comfyui-mixlab-nodes/lib/filerobot-image-editor.min.js"></script>
+    <!-- <script src="/extensions/comfyui-mixlab-nodes/lib/filerobot-image-editor.min.js"></script> -->
     <script type="module" src="/extensions/comfyui-mixlab-nodes/lib/model-viewer.min.js"></script>
     <link rel="stylesheet" href="/extensions/comfyui-mixlab-nodes/lib/login.css">
 </head>
@@ -474,15 +492,14 @@
             <a class="link" href="https://www.mixcomfy.com" target="_blank">ComfyUI中文爱好者社区推荐</a>
         </div>
 
-        <a target="_blank" id="login_btn" href="https://www.mixcomfy.com/blog/" style="text-decoration: none;
+        <a id="login_btn" target="_blank" href="https://discord.gg/xbP2GZF6gn" style="text-decoration: none;
         color: black;font-size:12px">
             <svg height="32" aria-hidden="true" viewBox="0 0 16 16" version="1.1" width="32" data-view-component="true"
                 class="octicon octicon-mark-github v-align-middle color-fg-default">
                 <path
                     d="M8 0c4.42 0 8 3.58 8 8a8.013 8.013 0 0 1-5.45 7.59c-.4.08-.55-.17-.55-.38 0-.27.01-1.13.01-2.2 0-.75-.25-1.23-.54-1.48 1.78-.2 3.65-.88 3.65-3.95 0-.88-.31-1.59-.82-2.15.08-.2.36-1.02-.08-2.12 0 0-.67-.22-2.2.82-.64-.18-1.32-.27-2-.27-.68 0-1.36.09-2 .27-1.53-1.03-2.2-.82-2.2-.82-.44 1.1-.16 1.92-.08 2.12-.51.56-.82 1.28-.82 2.15 0 3.06 1.86 3.75 3.64 3.95-.23.2-.44.55-.51 1.07-.46.21-1.61.55-2.33-.66-.15-.24-.6-.83-1.23-.82-.67.01-.27.38.01.53.34.19.73.9.82 1.13.16.45.68 1.31 2.69.94 0 .67.01 1.3.01 1.49 0 .21-.15.45-.55.38A7.995 7.995 0 0 1 0 8c0-4.42 3.58-8 8-8Z">
                 </path>
-            </svg> Community</a>
-
+            </svg> HELP/帮助</a>
 
     </div>
     <a id="author"></a>
@@ -680,11 +697,89 @@
         }
 
 
+        // 获取 rembg 模型
+        async function get_rembg_models() {
+            try {
+                const response = await fetch(`${get_url()}/mixlab/folder_paths`, {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json'
+                    },
+                    body: JSON.stringify({
+                        type: 'rembg'
+                    })
+                })
+
+                const data = await response.json()
+                // console.log(data)
+                return data.names
+            } catch (error) {
+                console.error(error)
+            }
+        }
+
+        //自动抠图
+        async function run_rembg(model, base64) {
+            try {
+                const response = await fetch(`${get_url()}/mixlab/rembg`, {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json'
+                    },
+                    body: JSON.stringify({
+                        model,
+                        base64
+                    })
+                })
+
+                const data = await response.json()
+                // console.log(data)
+                return data.data
+            } catch (error) {
+                console.error(error)
+            }
+        }
+
+        function convertImageToBlackBasedOnAlpha(image) {
+            const canvas = document.createElement('canvas');
+            const ctx = canvas.getContext('2d');
+
+            // Draw the image onto the canvas
+            canvas.width = image.width;
+            canvas.height = image.height;
+            ctx.drawImage(image, 0, 0);
+
+            // Get the image data from the canvas
+            const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
+            const pixels = imageData.data;
+
+            // Modify the RGB values based on the alpha channel
+            for (let i = 0; i < pixels.length; i += 4) {
+                const alpha = pixels[i + 3];
+                if (alpha !== 0) {
+                    // Set non-transparent pixels to black
+                    pixels[i] = 0; // Red
+                    pixels[i + 1] = 0; // Green
+                    pixels[i + 2] = 0; // Blue
+                }
+            }
+
+            // Put the modified image data back onto the canvas
+            ctx.putImageData(imageData, 0, 0);
+
+            // Convert the modified canvas to base64 data URL
+            const base64ImageData = canvas.toDataURL('image/png'); // Replace 'png' with your desired image format
+
+            return base64ImageData;
+        }
+
+
+
         // 图像编辑
         async function editImage(image, data) {
             //判断mask是否有输出
             let isMask = data.options.hasMask;
-            console.log(data)
+
             //app
             document.body.querySelector('.app').style.display = 'none'
             document.body.querySelector('#author').style.display = 'none'
@@ -693,21 +788,139 @@
             editor.style.display = 'block';
             const iframe = editor.querySelector('iframe');
 
+            const sleep = (t = 1000) => {
+                return new Promise((res, rej) => {
+                    setTimeout(() => {
+                        res(true)
+                    }, t)
+                })
+            }
+
+            //清空 图层
+            const removeAllLayer = () => {
+                let Layers = iframe.contentWindow.Layers;
+                //清空
+                Layers.reset_layers()
+                Layers.refresh_gui()
+            }
+
+            // 复原
+            const resetLayer = () => {
+                let Layers = iframe.contentWindow.Layers;
+                for (const layer of Layers.get_layers()) {
+                    layer.visible = true;
+                }
+                Layers.refresh_gui()
+            }
+            //取image
+            const getImageBase64FromLayer = () => {
+                let Layers = iframe.contentWindow.Layers;
+                let tempCanvas = document.createElement("canvas");
+                let tempCtx = tempCanvas.getContext("2d");
+                let dim = Layers.get_dimensions();
+                tempCanvas.width = dim.width;
+                tempCanvas.height = dim.height;
+                for (const layer of Layers.get_layers()) {
+                    if (layer.name === 'Image_' + data.id) {
+                        layer.visible = true;
+                    } else {
+                        layer.visible = false;
+                    }
+                }
+                Layers.refresh_gui()
+
+                Layers.convert_layers_to_canvas(tempCtx);
+                return tempCanvas.toDataURL()
+            }
+
+            //add mask
+            const addMask = (id, name, image) => {
+                let Layers = iframe.contentWindow.Layers;
+                var new_mask_layer = {
+                    id,
+                    name,
+                    type: 'brush',
+                    data: [],
+                    render_function: ['brush', 'render'],
+                    width: image.naturalWidth || image.width,
+                    height: image.naturalHeight || image.height,
+
+                };
+                Layers.insert(new_mask_layer);
+            }
+
+            //add image
+            const addImage = (id, name, image) => {
+                let Layers = iframe.contentWindow.Layers;
+                var new_mask_layer = {
+                    id,
+                    name,
+                    type: 'image',
+                    data: image,
+                    width: image.naturalWidth || image.width,
+                    height: image.naturalHeight || image.height,
+                    width_original: image.naturalWidth || image.width,
+                    height_original: image.naturalHeight || image.height,
+                };
+                Layers.insert(new_mask_layer);
+            }
+
             //默认的画笔size设置大
             // let inputSize = (iframe.contentDocument.getElementById('size')).querySelector('input');
             // inputSize.value=50;
             // (iframe.contentDocument.getElementById('size')).querySelector('.increase_number').click()
 
+            //自动抠图
+            let autoMaskSelect = iframe.contentDocument.getElementById('automask_image_mixlab');
+            const select = iframe.contentDocument.getElementById('automask_models_mixlab');
+            if (select.children.length === 0) {
+                let rembgModels = await get_rembg_models()
+                // 遍历模型列表并创建选项
+                for (const model of rembgModels) {
+                    const option = document.createElement('option');
+                    option.value = model;
+                    option.textContent = model;
+                    select.appendChild(option);
+                }
+            }
 
+            let autoMaskBtn = iframe.contentDocument.getElementById('automask_image_mixlab');
+            if (!autoMaskBtn.getAttribute('init')) autoMaskBtn.addEventListener('click', async e => {
+                //api请求
+                let base64 = getImageBase64FromLayer()
+                resetLayer()
+
+                let res = await run_rembg(select.value, base64)
+                const match = res.match(/^data:image\/(\w+);base64,/);
+                if (!match) {
+                    res = 'data:image/png;base64,' + res
+                }
+
+                let image = await createImage(res)
+
+                let mb = convertImageToBlackBasedOnAlpha(image)
+                let mask = await createImage(mb)
+                let id = Layers.auto_increment;
+                addImage(id, 'Mask_' + data.id + id, mask)
+            })
+
+            autoMaskBtn.setAttribute('init', 1)
             let cancelImageBtn = iframe.contentDocument.getElementById('cancel_image_mixlab');
-            cancelImageBtn.addEventListener('click', e => {
+            if (!cancelImageBtn.getAttribute('init')) cancelImageBtn.addEventListener('click', e => {
                 editor.style.display = 'none';
                 document.body.querySelector('.app').style.display = 'flex'
                 document.body.querySelector('#author').style.display = 'block'
             })
+
+            cancelImageBtn.setAttribute('init', 1)
             // 获取 id 为 "mix" 的 button 元素
             let saveImageBtn = iframe.contentDocument.getElementById('save_image_mixlab');
-            saveImageBtn.addEventListener('click', async e => {
+            saveImageBtn.style = `width: 98px;
+            height: 36px;
+            margin: 0 12px;
+            background-color: var(--background-color-active);
+            color: var(--text-color-active);`
+            if (!saveImageBtn.getAttribute('init')) saveImageBtn.addEventListener('click', async e => {
                 //保存，并更新图片
                 e.preventDefault();
 
@@ -721,6 +934,7 @@
                     tempCanvas.width = dim.width;
                     tempCanvas.height = dim.height;
 
+                    //todo 获取Image更新后的数据(暂不支持image的修改)
 
                     for (const layer of Layers.get_layers()) {
                         if (layer.name !== 'Image_' + data.id) {
@@ -734,13 +948,7 @@
                     Layers.convert_layers_to_canvas(tempCtx);
 
                     // 复原
-                    for (const layer of Layers.get_layers()) {
-                        layer.visible = true;
-                    }
-                    Layers.refresh_gui()
-
-
-
+                    resetLayer()
 
                     // 获取图像数据
                     const imageData = tempCtx.getImageData(0, 0, dim.width, dim.height);
@@ -766,10 +974,15 @@
                     // // 获取读取的文件内容，即 Blob 对象
                     let hashId = await calculateImageHash(fileBlob)
 
-                    if (hashId == window._appData.data[data.id].hashId) return;
+                    if (hashId == window._appData.data[data.id].hashId) {
+                        document.body.querySelector('.app').style.display = 'flex'
+                        document.body.querySelector('#author').style.display = 'block'
+                        return;
+                    }
 
+                    //底图
                     const { url: imgurl } = await uploadImage(base64ToBlob(data.options.defaultImage))
-
+                    //mask
                     let { url, name } = await uploadMask(fileBlob, imgurl);
                     // 在这里可以对 Blob 对象进行进一步处理
                     // imageElement.src = url;
@@ -777,31 +990,22 @@
                     window._appData.data[data.id].hashId = hashId;
 
                     // console.log("上传的文件：", url, data.id, name);
-
-
                     //更新图片
-
                     const canvas = document.createElement("canvas");
                     canvas.width = dim.width;
                     canvas.height = dim.height;
 
                     const ctx = canvas.getContext('2d');
 
-                    const defaultImage = new Image();
-                    defaultImage.src = data.options.defaultImage;
-                    defaultImage.onload = function () {
-                        ctx.drawImage(defaultImage, 0, 0, dim.width, dim.height);
+                    const defaultImage = await createImage(data.options.defaultImage)
+                    ctx.drawImage(defaultImage, 0, 0, dim.width, dim.height);
 
-                        // 绘制base64图片
-                        const base64Image = base64
-                        const base64ImageObj = new Image();
-                        base64ImageObj.onload = function () {
-                            ctx.globalCompositeOperation = 'destination-in';
-                            ctx.drawImage(base64ImageObj, 0, 0, dim.width, dim.height);
-                            image.src = canvas.toDataURL();
-                        };
-                        base64ImageObj.src = base64;
-                    };
+                    // 绘制base64图片
+                    // const base64Image = base64
+                    const base64ImageObj = await createImage(base64)
+                    ctx.globalCompositeOperation = 'destination-in';
+                    ctx.drawImage(base64ImageObj, 0, 0, dim.width, dim.height);
+                    image.src = canvas.toDataURL();
 
                 }
 
@@ -809,46 +1013,34 @@
                 document.body.querySelector('#author').style.display = 'block'
             })
 
+            saveImageBtn.setAttribute('init', 1)
 
             var Layers = iframe.contentWindow.Layers;
             // console.log(Layers)
 
             //判断是否已经存在
-            let layers1 = Layers.get_layers()
+            let layers1 = Layers.get_layers();
 
-            if (!layers1.filter(l => l.name == 'Image_' + data.id)[0]) {
-                var new_layer = {
-                    id: (Layers.get_layers()).length,
-                    name: 'Image_' + data.id,
-                    type: 'image',
-                    data: image,
-                    width: image.naturalWidth || image.width,
-                    height: image.naturalHeight || image.height,
-                    width_original: image.naturalWidth || image.width,
-                    height_original: image.naturalHeight || image.height,
-                };
-
-                Layers.insert(new_layer);
+            //通过layer.link.src 判断是否图片更新
+            console.log(layers1.filter(l => l.name == 'Image_' + data.id)[0]?.link?.currentSrc !== data.options.defaultImage)
+            if (layers1.filter(l => l.name == 'Image_' + data.id)[0]
+                && layers1.filter(l => l.name == 'Image_' + data.id)[0].link.src !== data.options.defaultImage) {
+                //清空图层
+                removeAllLayer();
+                layers1 = [];
+                await sleep()
             }
 
-
+            let im = await createImage(data.options.defaultImage)
+            // console.log(layers1, layers1.length)
+            if (!layers1.filter(l => l.name == 'Image_' + data.id)[0]) {
+                addImage(0, 'Image_' + data.id, im)
+            }
 
             if (isMask) {
                 if (!layers1.filter(l => l.name == 'Mask_' + data.id)[0]) {
-                    var new_mask_layer = {
-                        id: (Layers.get_layers()).length,
-                        name: 'Mask_' + data.id,
-                        type: 'brush',
-                        data: [],
-                        render_function: ['brush', 'render'],
-                        width: image.naturalWidth || image.width,
-                        height: image.naturalHeight || image.height,
-
-                    };
-
-                    Layers.insert(new_mask_layer);
+                    addMask(1, 'Mask_' + data.id, im)
                 }
-
             }
 
         }
@@ -1041,7 +1233,7 @@
         }
 
 
-        function createOutputs(outputData, link) {
+        async function createOutputs(outputData, link) {
             const url = new URL(window.location.href);
             const params = new URLSearchParams(url.search);
             const innerApp = params.get("innerApp");
@@ -1162,20 +1354,21 @@
                     "TransparentImage"].includes(node.class_type)) {
 
                     const url = node.options?.defaultImage || window._appData?.icon || base64Df;
-                    let a = document.createElement('a');
+                    let a = document.createElement('div');
                     a.id = `output_${node.id}`
-                    a.setAttribute('data-pswp-width', "200");
-                    a.setAttribute('data-pswp-height', "200");
-                    a.setAttribute('target', "_blank");
-                    a.setAttribute('href', url);
-                    a.setAttribute('title', node.title);
 
-                    let img = new Image();
-                    // img;
-                    img.src = url;
+                    let img = await createImage(url)
+
                     a.appendChild(img)
+                    // a.setAttribute('data-pswp-width', img.naturalWidth);
+                    // a.setAttribute('data-pswp-height', img.naturalHeight);
+                    // a.setAttribute('target', "_blank");
+                    // a.setAttribute('href', url);
+                    // a.setAttribute('title', node.title);
+
                     output_card.appendChild(a);
                     isShowImageFn = true;
+
                 }
 
                 //3d 
@@ -1297,8 +1490,9 @@
 
                             if (hashId == window._appData.data[data.id].hashId) return
 
+                            let base64 = await blobToBase64(fileBlob)
+
                             if (data.class_type === 'LoadImagesToBatch') {
-                                let base64 = await blobToBase64(fileBlob)
                                 createBase64ImageForLoadImageToBatch(imageElement, data.id, base64)
                             } else {
                                 let { url, name } = await uploadImage(fileBlob);
@@ -1307,6 +1501,14 @@
                                 window._appData.data[data.id].inputs.image = name;
                                 window._appData.data[data.id].hashId = hashId;
                                 console.log("上传的文件：", url, data.id, name);
+
+                                //更换option里的default image
+                                window._appData.input = Array.from(window._appData.input, inp => {
+                                    if (inp.id === data.id) {
+                                        inp.options.defaultImage = base64;
+                                    }
+                                    return inp
+                                })
                             }
                         }
                     }
@@ -1608,9 +1810,9 @@
                                 //上传，返回url
                                 let { url, name } = await uploadImage(fileBlob, '.' + file.type.split('/')[1])
 
+                                let base64 = await parseImageToBase64(url);
+
                                 if (data.class_type === 'ImagesPrompt_') {
-                                    // 
-                                    let base64 = await parseImageToBase64(url);
                                     uploadContainer.querySelector('.images_prompt_main').src = base64
                                     window._appData.data[data.id].inputs.image_base64 = base64;
                                 } else {
@@ -1620,10 +1822,17 @@
                                     // 在这里可以对 Blob 对象进行进一步处理
                                     imageElement.src = url;
 
-
                                     if (isVideoUpload) {
                                         window._appData.data[data.id].inputs.video = name;
                                     } else {
+                                        //更换option里的default image
+                                        window._appData.input = Array.from(window._appData.input, inp => {
+                                            if (inp.id === data.id) {
+                                                inp.options.defaultImage = base64;
+                                            }
+                                            return inp
+                                        })
+
                                         window._appData.data[data.id].inputs.image = name;
                                     }
 
@@ -2207,7 +2416,7 @@
             })
         }
 
-        function createUI(data, share = true) {
+        async function createUI(data, share = true) {
             // appData.input, appData.output, appData.seed, share, appData.link
             if (!data) return
             const { input: inputData, output: outputData, data: workflow, seed, seedTitle, link, name } = data;
@@ -2348,30 +2557,44 @@
             }
 
 
-            statusDiv.appendChild(status);
+            // statusDiv.appendChild(status);
             statusDiv.appendChild(seeds);
 
             // 创建输入框
             var input1 = createInputs(inputData)
 
-            var output = createOutputs(outputData, link)
+            var output = await createOutputs(outputData, link)
 
             // 创建提交按钮
+            let submitDiv = document.createElement('div');
+            submitDiv.appendChild(statusDiv);
+
+            let submitDivBtn = document.createElement('div');
+            submitDivBtn.style.display = 'flex'
+            submitDiv.appendChild(submitDivBtn);
+
+            submitDivBtn.appendChild(status);
+
+            submitDiv.className = 'run_div';
             var submitButton = document.createElement('button');
             submitButton.textContent = 'Create';
             submitButton.className = 'run_btn'
 
             // 将所有UI元素添加到页面中
-            leftDiv.appendChild(titleDiv);
-            leftDiv.appendChild(iconDes);
+            // leftDiv.appendChild(titleDiv);
+            // leftDiv.appendChild(iconDes);
             // leftDiv.appendChild(des);
-            leftDiv.appendChild(statusDiv);
+            // leftDiv.appendChild(statusDiv);
             leftDiv.appendChild(input1);
 
-            if (typeof (data.data) == 'object') mainDiv.appendChild(submitButton);
+            submitDivBtn.appendChild(submitButton);
+            if (typeof (data.data) == 'object') mainDiv.appendChild(submitDiv);
 
             rightDiv.appendChild(output);
 
+            // mainDiv.appendChild(titleDiv);
+            // mainDiv.appendChild(iconDes);
+
             mainDiv.appendChild(leftDetails);
             mainDiv.appendChild(rightDiv);
 
@@ -2719,7 +2942,7 @@
         async function createApp(appData, share = true) {
             // console.log(appData)
             // 使用示例：
-            var ui = createUI(appData, share);
+            var ui = await createUI(appData, share);
 
             // 更新标题
             ui.title.update(appData.name || 'Mixlab APP');
@@ -2859,7 +3082,8 @@
             const lightbox = new PhotoSwipeLightbox({
                 gallery: '.output_card',
                 children: 'a',
-                pswpModule: () => import('/extensions/comfyui-mixlab-nodes/lib/photoswipe.esm.min.js')
+                pswpModule: () => import('/extensions/comfyui-mixlab-nodes/lib/photoswipe.esm.min.js'),
+                // appendToEl: document.querySelector('#result')
             });
             lightbox.on('uiRegister', function () {
                 lightbox.pswp.ui.registerElement({
@@ -2904,8 +3128,8 @@
             if (appData.author) {
                 let div = document.body.querySelector('#author');
                 if (appData.author.link) div.href = appData.author.link
-                div.style = `z-index:20;display: flex;flex-direction: column;position: fixed;bottom: 12px;right: 24px;cursor: pointer;text-decoration: none;color: black;`
-                div.innerHTML = `<p style="font-size:12px">Author:</p>
+                div.style = `z-index:20;display: flex;flex-direction: column;position: fixed;bottom: 24px;left: 24px;cursor: pointer;text-decoration: none;color: black;`
+                div.innerHTML = `<p style="font-size:12px;margin: 8px 0;">Author:</p>
                 <div style="display: flex;"> <img style="width:32px;height:32px;border-radius: 100%;" 
                 src="${appData.author.avatar || base64Df}"/>
                 <p style="margin-left:8px;font-size:12px;font-weight:800">${appData.author.name || '-'}</p></div>`

web/javascript/chat.js

@@ -2,7 +2,7 @@ async function* completion (url, messages, controller) {
   let data = {
     model: 'gpt-3.5-turbo-16k',
     messages,
-    temperature: 0.6,
+    temperature: 0.05,
     stream: true
   }
   // if (imageNode) {

web/javascript/checkVersion_mixlab.js

@@ -3,7 +3,7 @@ import { app } from '../../../scripts/app.js'
 const repoOwner = 'shadowcz007' // 替换为仓库的所有者
 const repoName = 'comfyui-mixlab-nodes' // 替换为仓库的名称
 
-const version = 'v0.27.0'
+const version = 'v0.28.3'
 
 fetch(`https://api.github.com/repos/${repoOwner}/${repoName}/releases/latest`)
   .then(response => response.json())

web/javascript/ui_mixlab.js

@@ -26,7 +26,35 @@ function showTextByLanguage (key, json) {
 }
 
 //系统prompt
-const systemPrompt = `You are a prompt creator, your task is to create prompts for the user input request, the prompts are image descriptions that include keywords for (an adjective, type of image, framing/composition, subject, subject appearance/action, environment, lighting situation, details of the shoot/illustration, visuals aesthetics and artists), brake keywords by comas, provide high quality, non-verboose, coherent, brief, concise, and not superfluous prompts, the subject from the input request must be included verbatim on the prompt,the prompt is english`
+// const systemPrompt = `You are a prompt creator, your task is to create prompts for the user input request, the prompts are image descriptions that include keywords for (an adjective, type of image, framing/composition, subject, subject appearance/action, environment, lighting situation, details of the shoot/illustration, visuals aesthetics and artists), brake keywords by comas, provide high quality, non-verboose, coherent, brief, concise, and not superfluous prompts, the subject from the input request must be included verbatim on the prompt,the prompt is english`
+
+let tool ={
+  "name": "create_prompt",
+  "description": "Create a prompt with a given subject, content, and style based on user input for image descriptions.",
+  "parameter": {
+    "type": "object",
+    "properties": {
+      "subject": {
+        "type": "string",
+        "description": "The subject of the prompt, included verbatim from the input request.",
+        "required": true
+      },
+      "content": {
+        "type": "string",
+        "description": "The content of the prompt, primarily focusing on the scene and objects, including keywords for adjective, type of image, framing/composition, subject appearance/action, and environment.",
+        "required": true
+      },
+      "style": {
+        "type": "string",
+        "description": "The style of the prompt, including lighting situation, details of the shoot/illustration, visual aesthetics, and artists. Ensure it is high quality, non-verbose, coherent, brief, concise, and not superfluous.",
+        "required": true
+      }
+    }
+  }
+}
+
+const systemPrompt=`You are a helpful assistant with access to the following functions. Use them if required - ${JSON.stringify(tool,null,2)}`
+
 
 if (!localStorage.getItem('_mixlab_system_prompt')) {
   localStorage.setItem('_mixlab_system_prompt', systemPrompt)
@@ -46,7 +74,7 @@ async function get_llamafile_models () {
     })
 
     const data = await response.json()
-    console.log(data)
+    // console.log(data)
     return data.names
   } catch (error) {
     console.error(error)
@@ -539,10 +567,6 @@ injectCSS(`::-webkit-scrollbar {
   border-left: 2px solid var(--input-text); 
 }
  
-.litegraph{
-  background: var(--bg-color)!important;
-}
-
 `)
 
 async function getCustomnodeMappings (mode = 'url') {
@@ -774,6 +798,22 @@ async function fetchReadmeContent (url) {
   }
 }
 
+async function startLLM (model) {
+  let res = await start_llama(model)
+  window._mixlab_llamacpp = res
+
+  localStorage.setItem('_mixlab_llama_select', res.model)
+
+  if (document.body.querySelector('#mixlab_chatbot_by_llamacpp')&&window._mixlab_llamacpp.url) {
+    document.body
+      .querySelector('#mixlab_chatbot_by_llamacpp')
+      .setAttribute('title', window._mixlab_llamacpp.url)
+  }
+  if (document.body.querySelector('#llm_status_btn')&&window._mixlab_llamacpp) {
+    document.body.querySelector('#llm_status_btn').innerText = window._mixlab_llamacpp.model
+  }
+}
+
 function createModelsModal (models) {
   var div =
     document.querySelector('#model-modal') || document.createElement('div')
@@ -829,7 +869,7 @@ function createModelsModal (models) {
       color: var(--descrip-text);
       font-size: 18px;
       display: flex;
-      align-items: center;
+      align-items: flex-start;
       flex: 1;
       overflow: hidden;
       text-decoration: none;
@@ -848,6 +888,27 @@ function createModelsModal (models) {
   })
   linkIcon.style.padding = '4px'
 
+  const statusIcon = document.createElement('small')
+  statusIcon.textContent = showTextByLanguage('Status', {
+    Status: 'OFF'
+  })
+  statusIcon.id = 'llm_status_btn'
+  statusIcon.style=`padding: 4px;
+  background-color: rgb(102, 255, 108);
+  color: black;
+  font-size: 12px;
+  margin-left: 12px;`
+  if (window._mixlab_llamacpp?.url) {
+    statusIcon.textContent = window._mixlab_llamacpp.model
+    statusIcon.style.backgroundColor = '#66ff6c'
+    statusIcon.style.color = 'black'
+  } else {
+  }
+  statusIcon.addEventListener('click', e => {
+    e.stopPropagation()
+    // startLLM()
+  })
+
   const n_gpu = document.createElement('input')
   n_gpu.type = 'number'
   n_gpu.setAttribute('min', -1)
@@ -882,20 +943,22 @@ function createModelsModal (models) {
   const title = document.createElement('p')
   title.innerText = 'Models'
   title.style = `font-size: 18px;
-  margin-right: 8px;`
+  margin-right: 8px;
+  margin-top: 0;`
 
   const left_d = document.createElement('div')
   left_d.style = `display: flex;
   justify-content: center;
-  align-items: center;
-  font-size: 12px;`
+  align-items: flex-start;
+  font-size: 12px;
+  flex-direction: column; `
   left_d.appendChild(title)
-
+  title.appendChild(statusIcon)
   left_d.appendChild(linkIcon)
-
+  left_d.appendChild(n_gpu_div)
   headTitleElement.appendChild(left_d)
 
-  headTitleElement.appendChild(n_gpu_div)
+  // headTitleElement.appendChild(n_gpu_div)
 
   //重启
   const reStart = document.createElement('small')
@@ -903,7 +966,11 @@ function createModelsModal (models) {
     restart: '重启'
   })
 
-  reStart.style.padding = '4px'
+  reStart.style=`padding: 8px;
+  font-size: 16px;
+  outline: 1px solid;
+  padding-top: 4px;
+  padding-bottom: 4px;`
 
   headTitleElement.appendChild(reStart)
 
@@ -962,29 +1029,22 @@ function createModelsModal (models) {
 
   modalContent.appendChild(input)
 
-  for (const m of models) {
-    let d = document.createElement('div')
-    d.innerText = m
-    d.className = `mix_tag`
+  if (!window._mixlab_llamacpp) {
+    for (const m of models) {
+      let d = document.createElement('div')
+      d.innerText = `${showTextByLanguage('Run', {
+        Run: '运行'
+      })} ${m}`
+      d.className = `mix_tag`
 
-    if (!window._mixlab_llamacpp) {
       d.addEventListener('click', async e => {
         e.stopPropagation()
         div.remove()
-        let res = await start_llama(m)
-        window._mixlab_llamacpp = res
-
-        localStorage.setItem('_mixlab_llama_select', res.model)
-
-        if (document.body.querySelector('#mixlab_chatbot_by_llamacpp')) {
-          document.body
-            .querySelector('#mixlab_chatbot_by_llamacpp')
-            .setAttribute('title', window._mixlab_llamacpp.url)
-        }
+        startLLM(m)
       })
-    }
 
-    modalContent.appendChild(d)
+      modalContent.appendChild(d)
+    }
   }
   modal.appendChild(modalContent)
 
@@ -998,8 +1058,8 @@ function createModelsModal (models) {
   cursor: pointer;
   font-size: 12px;
   color: white;`
-  helpInfo.href="https://discord.gg/cXs9vZSqeK"
-  helpInfo.target="_blank"
+  helpInfo.href = 'https://discord.gg/cXs9vZSqeK'
+  helpInfo.target = '_blank'
   modal.appendChild(helpInfo)
 
   document.body.appendChild(div)
@@ -1353,6 +1413,8 @@ app.registerExtension({
           .querySelector('#mixlab_chatbot_by_llamacpp')
           .setAttribute('title', res.url)
       })
+    }else{
+      startLLM('')
     }
 
     LGraphCanvas.prototype.helpAboutNode = async function (node) {
@@ -1361,7 +1423,13 @@ app.registerExtension({
           ? nodesMap
           : await getCustomnodeMappings('url')
 
-      console.log('node & node map', node, nodesMap, nodesMap[node.type])
+      console.log(
+        '%c### node & node map',
+        'background: yellow; color: black',
+        node,
+        nodesMap,
+        nodesMap[node.type]
+      )
       let repo = nodesMap[node.type]
       if (repo) {
         let markdown = await fetchReadmeContent(repo.url)
@@ -1392,6 +1460,7 @@ app.registerExtension({
         let userInput = widget.value
         widget.value = widget.value.trim()
         widget.value += '\n'
+        let jsonStr="";
         try {
           await completion_(
             window._mixlab_llamacpp.url + '/v1/chat/completions',
@@ -1406,6 +1475,7 @@ app.registerExtension({
             t => {
               // console.log(t)
               widget.value += t
+              jsonStr+=t
             }
           )
         } catch (error) {
@@ -1431,13 +1501,27 @@ app.registerExtension({
                 t => {
                   // console.log(t)
                   widget.value += t
+                  jsonStr+=t
                 }
               )
             })
           }
         }
 
-        widget.value = widget.value.trim()
+        let json=null;
+
+        try {
+          json=JSON.parse(jsonStr.trim())
+        } catch (error) {
+          json=JSON.parse(jsonStr.trim()+"}")
+        }
+
+        if(json){
+          widget.value = [json.subject,json.content,json.style].join('\n')
+        }else{
+          widget.value = widget.value.trim()
+        }
+        
       }
     }
 

web/lib/miniPaint-4.14.2/.babelrc

@@ -1,8 +0,0 @@
-{
-	"presets": ["@babel/preset-env"],
-	"plugins": [
-		["@babel/plugin-transform-runtime", {
-			"regenerator": true
-		}]
-	]
-}

web/lib/miniPaint-4.14.2/.gitignore

@@ -1,17 +0,0 @@
-# OS generated files #
-.DS_Store
-.DS_Store?
-._*
-.Trashes
-ehthumbs.db
-Thumbs.db
-nbproject/
-.idea/
-.git/
-.vscode
-/.project
-*.log
-
-/node_modules/
-*.js.ignore
-

web/lib/miniPaint-4.14.2/MIT-LICENSE.txt

@@ -1,21 +0,0 @@
-Copyright (c) ViliusL
-https://github.com/viliusle
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

web/lib/miniPaint-4.14.2/README.md

@@ -1,63 +0,0 @@
-# miniPaint
-
-Online image editor lets you create, edit images using HTML5 technologies.
-No need to buy, download, install or have obsolete flash. No ads.
-Key features: layers, filters, HTML5, open source, Photoshop alternative.
-
-miniPaint operates directly in the browser. You can create images, paste from the clipboard (ctrl+v) 
-or upload from the computer (using menu or drag & drop). Nothing will be sent to any server. Everything stays in your 
-browser. 
-
-## URL:
-**https://viliusle.github.io/miniPaint/**
-
-## Preview:
-![miniPaint](https://raw.githubusercontent.com/viliusle/miniPaint/master/images/preview.gif)
-(generated using miniPaint)
-
-**Change log:** [/miniPaint/releases](https://github.com/viliusle/miniPaint/releases)
-
-## Browser Support
-- Chrome
-- Firefox
-- Opera
-- Edge
-- Safari
-
-## Features
-
-- **Files**: open images, directories, URL, data URL, drag and drop, save (PNG, JPG, BMP, WEBP, animated GIF, TIFF, JSON 
-(layers data), print.
-- **Edit**: Undo, cut, copy, paste, selection, paste from clipboard.
-- **Image**: information, EXIF, trim, zoom, resize (Hermite resample, default resize), rotate, flip, 
-color corrections (brightness, contrast, hue, saturation, luminance), auto adjust colors, grid, histogram, negative.
-- **Layers**: multiple layers system, differences, merge, flatten, Transparency support.
-- **Effects**: Black and White, Blur (box, Gaussian, stack, zoom), Bulge/Pinch, Denoise, Desaturate, Dither, Dot Screen, 
-Edge, Emboss, Enrich, Gamma, Grains, GrayScale, Heatmap, JPG Compression, Mosaic, Oil, Sepia, Sharpen, Solarize, 
-Tilt Shift, Vignette, Vibrance, Vintage, Blueprint, Night Vision, Pencil, also Instagram Filters: 1977, Aden, Clarendon, 
-Gingham, Inkwell, Lo-fi, Toaster, Valencia, X-Pro II.
-- **Tools**: pencil, brush, magic wand, erase, fill, color picker, letters, crop, blur, sharpen, desaturate, clone,
- borders, sprites, key-points, color zoom, replace color, restore alpha, content fill.
-- **Help**: keyboard shortcuts, translations.
-
-## Embed
-To embed this app in the other page, use this HTML code:
-
-    <iframe style="width:100%; height:1000px;" id="miniPaint" src="https://viliusle.github.io/miniPaint/" allow="camera"></iframe>
-
-## Build instructions
-See [Wiki > Build instructions](https://github.com/viliusle/miniPaint/wiki/Build-instructions)
-
-## Wiki
-See [Wiki](https://github.com/viliusle/miniPaint/wiki)
-
-## Contributors
-<a align="center" href="https://github.com/viliusle/miniPaint/graphs/contributors">
-  <img src="https://contrib.rocks/image?repo=viliusle/miniPaint" />
-</a>
-
-## License
-MIT License
-
-## Support
-Please use the GitHub issues for support, features, issues or use mail www.viliusl@gmail.com for contacts.

web/lib/miniPaint-4.14.2/SECURITY.md

@@ -1,15 +0,0 @@
-# Security Policy
-
-## Supported Versions
-
-Use this section to tell people about which versions of your project are
-currently being supported with security updates.
-
-| Version | Supported          |
-| ------- | ------------------ |
-| latest   | :white_check_mark: |
-| < latest   | :x:                |
-
-## Reporting a Vulnerability
-
-Please send details to www.viliusl@gmail.com

web/lib/miniPaint-4.14.2/index.html

@@ -34,7 +34,7 @@
 		content="miniPaint is free online image editor using HTML5. Edit, adjust your images, add effects online in your browser, without installing anything..." />
 	<meta property="og:site_name" content="miniPaint" />
 
-	<script src="dist/bundle.js"></script>
+	<script src="dist/bundle.ejs"></script>
 </head>
 
 <body>
@@ -45,6 +45,15 @@
 		<div class="submenu">
 			<!-- <a class="logo" href="#">miniPaint</a> -->
 			<div class="block attributes" id="action_attributes"></div>
+			
+			<select id="automask_models_mixlab"></select>
+
+			<button id="automask_image_mixlab" type="button" style="width: 98px;
+			height: 36px;margin-right: 18px;
+			color: white;">
+				RemoveBg
+			</button>
+
 			<button id="cancel_image_mixlab" type="button" style="width: 98px;
 			height: 36px;
 			color: white;">
@@ -55,6 +64,13 @@
 			color: white;">
 				Save
 			</button>
+
+			<a  
+			target="_blank" href="https://discord.gg/xbP2GZF6gn" 
+			style="width: 98px;
+			color: white;
+			text-decoration: none;"> Help/帮助 </a>
+
 			<button class="undo_button" id="undo_button" type="button">
 				<span class="sr_only">Undo</span>
 			</button>
@@ -82,7 +98,7 @@
 		</div>
 
 		<div class="sidebar_right">
-			<div class="preview block">
+			<div class="preview block" style="display: none;">
 				<h2 class="trn toggle" data-target="toggle_preview">Preview</h2>
 				<div id="toggle_preview"></div>
 			</div>
@@ -92,7 +108,7 @@
 				<div class="content" id="toggle_colors"></div>
 			</div>
 
-			<div class="block" id="info_base">
+			<div class="block" id="info_base" style="display: none;">
 				<h2 class="trn toggle toggle-full" data-target="toggle_info">Information</h2>
 				<div class="content" id="toggle_info"></div>
 			</div>

web/lib/miniPaint-4.14.2/manifest-disabled.json

@@ -1,41 +0,0 @@
-{
-	"name": "miniPaint",
-	"short_name": "miniPaint",
-	"start_url": "/",
-	"display": "standalone",
-	"orientation": "landscape",
-	"background_color": "#666d6f",
-	"description": "miniPaint is free online image editor using HTML5.",
-	"icons": [
-		{
-			"src": "images/manifest/48x48.png",
-			"sizes": "48x48",
-			"type": "image/png"
-		},
-		{
-			"src": "images/manifest/72x72.png",
-			"sizes": "72x72",
-			"type": "image/png"
-		},
-		{
-			"src": "images/manifest/96x96.png",
-			"sizes": "96x96",
-			"type": "image/png"
-		},
-		{
-			"src": "images/manifest/144x144.png",
-			"sizes": "144x144",
-			"type": "image/png"
-		},
-		{
-			"src": "images/manifest/168x168.png",
-			"sizes": "168x168",
-			"type": "image/png"
-		},
-		{
-			"src": "images/manifest/192x192.png",
-			"sizes": "192x192",
-			"type": "image/png"
-		}
-	]
-}

web/lib/miniPaint-4.14.2/package.json

@@ -1,51 +0,0 @@
-{
-  "name": "miniPaint",
-  "version": "4.14.2",
-  "author": "Vilius L.",
-  "description": "Online graphics editing tool lets create, edit images using HTML5 technologies.",
-  "keywords": [
-    "canvas",
-    "drawing",
-    "paint",
-    "layers",
-    "effects"
-  ],
-  "scripts": {
-    "server": "webpack serve --mode development --env development --open",
-    "dev": "webpack --mode development",
-    "build": "webpack --mode production"
-  },
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/viliusle/miniPaint"
-  },
-  "homepage": "https://github.com/viliusle/miniPaint",
-  "license": "MIT",
-  "devDependencies": {
-    "@babel/core": "^7.14.5",
-    "@babel/plugin-transform-runtime": "^7.14.5",
-    "@babel/preset-env": "^7.14.5",
-    "babel-loader": "^8.2.2",
-    "css-loader": "^5.2.6",
-    "source-map-loader": "^3.0.0",
-    "style-loader": "^2.0.0",
-    "webpack": "^5.76.0",
-    "webpack-cli": "^4.7.2",
-    "webpack-dev-server": "^4.3.1"
-  },
-  "dependencies": {
-    "@babel/runtime": "^7.14.5",
-    "alertifyjs": "^1.13.1",
-    "blueimp-canvas-to-blob": "^3.28.0",
-    "exif-js": "^2.3.0",
-    "file-saver": "^2.0.5",
-    "fuzzysort": "^1.1.4",
-    "gif.js.optimized": "^1.0.1",
-    "hermite-resize": "git+https://github.com/viliusle/Hermite-resize.git",
-    "jquery": "^3.5.1",
-    "pica": "^7.0.0",
-    "semver-compare": "^1.0.0",
-    "uuid": "^8.3.2",
-    "webfontloader": "^1.6.28"
-  }
-}

web/lib/miniPaint-4.14.2/service-worker.js

@@ -1,68 +0,0 @@
-
-//IMPORTANT - this file is not used !!!
-
-
-// use a cacheName for cache versioning
-var cacheName = 'v1:static';
-
-// during the install phase you usually want to cache static assets
-self.addEventListener('install', function(e) {
-	// once the SW is installed, go ahead and fetch the resources to make this work offline
-	e.waitUntil(
-		caches.open(cacheName).then(function(cache) {
-			return cache.addAll([
-				'./',
-				'./dist/bundle.js',
-				'./images/favicon.png',
-				'./images/logo.svg',
-				'./images/logo-colors.png',
-				'./images/icons/animation.svg',
-				'./images/icons/blur.svg',
-				'./images/icons/bold.svg',
-				'./images/icons/brush.svg',
-				'./images/icons/bulge_pinch.svg',
-				'./images/icons/clone.svg',
-				'./images/icons/crop.svg',
-				'./images/icons/delete.svg',
-				'./images/icons/desaturate.svg',
-				'./images/icons/erase.svg',
-				'./images/icons/external.png',
-				'./images/icons/fill.svg',
-				'./images/icons/gradient.png',
-				'./images/icons/grid.png',
-				'./images/icons/italic.svg',
-				'./images/icons/magic_erase.svg',
-				'./images/icons/media.svg',
-				'./images/icons/menu.svg',
-				'./images/icons/pencil.svg',
-				'./images/icons/pick_color.svg',
-				'./images/icons/refresh.svg',
-				'./images/icons/select.svg',
-				'./images/icons/selection.svg',
-				'./images/icons/shape.svg',
-				'./images/icons/sharpen.svg',
-				'./images/icons/strikethrough.svg',
-				'./images/icons/text.svg',
-				'./images/icons/underline.svg',
-				'./images/icons/view.svg'
-			]).then(function() {
-				self.skipWaiting();
-			});
-		})
-	);
-});
-
-// when the browser fetches a url
-self.addEventListener('fetch', function(event) {
-	// either respond with the cached object or go ahead and fetch the actual url
-	event.respondWith(
-		caches.match(event.request).then(function(response) {
-			if (response) {
-				// retrieve from cache
-				return response;
-			}
-			// fetch as normal
-			return fetch(event.request);
-		})
-	);
-});

web/lib/miniPaint-4.14.2/webpack.config.js

@@ -1,56 +0,0 @@
-var webpack = require('webpack');
-var path = require('path');
-
-module.exports = {
-	entry: [
-		'./src/js/main.js',
-	],
-	output: {
-		path: path.resolve(__dirname, 'dist'),
-		filename: 'bundle.js',
-		publicPath: '/dist/'
-	},
-	resolve: {
-		extensions: ['.js', '.css'],
-		alias: {
-			Utilities: path.resolve(__dirname, './../node_modules/')
-		}
-	},
-	module: {
-		rules: [
-			{
-				test: /\.css$/,
-				use: [
-					'style-loader',
-					{
-						loader: 'css-loader',
-						options: {url: false}
-					}
-				]
-			},
-			{
-				test: /\.js$/,
-				exclude: /(node_modules|bower_components)/,
-				use: ['babel-loader']
-			},
-		]
-	},
-	plugins: [
-		new webpack.ProvidePlugin({
-            $: "jquery",
-            jQuery: "jquery",
-            "window.jQuery": "jquery"
-		}),
-		new webpack.DefinePlugin({
-			VERSION: JSON.stringify(require("./package.json").version)
-		}),
-	],
-	devtool: "cheap-module-source-map",
-	devServer: {
-		// host: '0.0.0.0',
-		//contentBase: "./",
-		static: {
-			directory: path.resolve(__dirname, "./"),
-		},
-	}
-};

web/live.html

@@ -0,0 +1,525 @@
+<!DOCTYPE html>
+<!-- 
+  VoiceStreamAI Client Interface
+  Real-time audio transcription using self-hosted Whisper and WebSocket
+
+  Contributor:
+  - Alessandro Saccoia - alessandro.saccoia@gmail.com
+-->
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8">
+    <title>Audio Stream to WebSocket Server</title>
+    <style>
+        body {
+            font-family: Arial, sans-serif;
+            margin: 0;
+            padding: 0;
+            background: #f4f4f4;
+            text-align: center;
+        }
+
+        h1 {
+            color: #333;
+        }
+
+        .controls {
+            margin: 20px auto;
+            padding: 10px;
+            width: 80%;
+            display: flex;
+            justify-content: space-around;
+            align-items: center;
+        }
+
+        .control-group {
+            display: flex;
+            flex-direction: column;
+            align-items: center;
+        }
+
+        .controls input,
+        .controls button,
+        .controls select {
+            padding: 8px;
+            margin: 5px;
+            border: 1px solid #ddd;
+            border-radius: 5px;
+            font-size: 0.9em;
+        }
+
+        #transcription {
+            margin: 20px auto;
+            border: 1px solid #ddd;
+            padding: 10px;
+            width: 80%;
+            height: 150px;
+            overflow-y: auto;
+            background: white;
+        }
+
+        .label {
+            font-size: 0.9em;
+            color: #555;
+            margin-bottom: 5px;
+        }
+
+        button {
+            cursor: pointer;
+        }
+
+        .buffering-strategy-panel {
+            margin-top: 10px;
+        }
+
+        /* ... existing styles ... */
+        .hidden {
+            display: none;
+        }
+    </style>
+
+    <style>
+        body {
+            margin: 0;
+            padding: 0;
+            /* background-color: #333;
+            color:white */
+        }
+
+        #mic_container {
+            display: flex;
+            width: 100%;
+            align-items: center;
+            justify-content: center;
+        }
+
+        #mic {
+            border: 1px solid #ddd;
+            border-radius: 4px;
+            margin-top: 1rem;
+
+            width: 300px
+        }
+
+        #asistant {
+            width: 300px;
+            padding: 12px;
+            font-size: 12px;
+            overflow-y: scroll;
+            height: 300px;
+        }
+    </style>
+</head>
+
+<body>
+
+    <div id="mic_container">
+        <div id="mic"></div>
+        <div id="asistant"></div>
+    </div>
+
+    <h1>VAD + Whisper + </h1>
+
+    <button id="init_server">Server</button>
+
+    <div class="controls">
+        <div class="control-group">
+            <label class="label" for="websocketAddress">WebSocket Address:</label>
+            <input type="text" id="websocketAddress" value="ws://localhost:8725">
+        </div>
+        <div class="control-group">
+            <label class="label" for="bufferingStrategySelect" onchange="toggleBufferingStrategyPanel()">Buffering
+                Strategy:</label>
+            <select id="bufferingStrategySelect">
+                <option value="silence_at_end_of_chunk" selected>Silence at End of Chunk</option>
+            </select>
+        </div>
+        <div class="silence_at_end_of_chunk_options_panel">
+            <div class="control-group">
+                <label class="label" for="chunk_length_seconds">Chunk Length (s):</label>
+                <input type="number" id="chunk_length_seconds" value="3" min="1">
+            </div>
+            <div class="control-group">
+                <label class="label" for="chunk_offset_seconds">Silence at the End of Chunk (s):</label>
+                <input type="number" id="chunk_offset_seconds" value="0.1" min="0">
+            </div>
+        </div>
+        <div class="control-group">
+            <label class="label" for="languageSelect">Language:</label>
+            <select id="languageSelect">
+                <option value="multilingual">Multilingual</option>
+                <option value="english">English</option>
+                <option value="italian">Italian</option>
+                <option value="spanish">Spanish</option>
+                <option value="french">French</option>
+                <option value="german">German</option>
+                <option value="chinese">Chinese</option>
+                <option value="arabic">Arabic</option>
+                <option value="portuguese">Portuguese</option>
+                <option value="russian">Russian</option>
+                <option value="japanese">Japanese</option>
+                <option value="dutch">Dutch</option>
+                <option value="korean">Korean</option>
+                <option value="hindi">Hindi</option>
+                <option value="turkish">Turkish</option>
+                <option value="swedish">Swedish</option>
+                <option value="norwegian">Norwegian</option>
+                <option value="danish">Danish</option>
+                <option value="polish">Polish</option>
+                <option value="finnish">Finnish</option>
+                <option value="thai">Thai</option>
+                <option value="czech">Czech</option>
+                <option value="hungarian">Hungarian</option>
+                <option value="greek">Greek</option>
+            </select>
+        </div>
+        <button id="connectButton">Connect</button>
+    </div>
+    <button id="startButton" disabled>Start Streaming</button>
+    <button id="stopButton" disabled>Stop Streaming</button>
+    <div id="transcription"></div>
+    <br />
+    <div>WebSocket: <span id="webSocketStatus">Not Connected</span></div>
+    <div>Detected Language: <span id="detected_language">Undefined</span></div>
+    <div>Last Processing Time: <span id="processing_time">Undefined</span></div>
+
+    <script type="module">
+
+        // Record plugin
+
+        import WaveSurfer from 'https://cdn.jsdelivr.net/npm/wavesurfer.js@7/dist/wavesurfer.esm.js'
+        import RecordPlugin from 'https://cdn.jsdelivr.net/npm/wavesurfer.js/dist/plugins/record.esm.js'
+
+        // 可视化
+        let wavesurfer, record;
+
+        // ws服务
+        let websocket;
+        let context;
+        let processor;
+        let globalStream;
+
+        const websocket_uri = 'ws://localhost:8765';
+        const bufferSize = 4096;
+        let isRecording = false;
+        let chunk_length_seconds, chunk_offset_seconds, language;
+
+        // Record button
+        const startButton = document.getElementById('startButton'),
+            stopButton = document.getElementById('stopButton'),
+            connectButton = document.getElementById('connectButton'),
+            initServerButton = document.getElementById('init_server')
+
+        const createWaveSurfer = () => {
+            // Create an instance of WaveSurfer
+            if (wavesurfer) {
+                wavesurfer.destroy()
+            }
+            wavesurfer = WaveSurfer.create({
+                container: '#mic',
+                waveColor: 'rgb(200, 0, 200)',
+                progressColor: 'rgb(100, 0, 100)',
+                renderFunction: (channels, ctx) => {
+                    const { width, height } = ctx.canvas
+                    // console.log(width, height)
+                    const scale = channels[0].length / width
+                    const step = 20
+
+                    ctx.translate(0, height / 2)
+                    ctx.strokeStyle = ctx.fillStyle
+                    ctx.beginPath()
+
+                    for (let i = 0; i < width; i += step * 2) {
+                        const index = Math.floor(i * scale)
+                        const value = Math.abs(channels[0][index])
+                        let x = i
+                        let y = value * height * 1.2
+
+                        ctx.moveTo(x, 0)
+                        ctx.lineTo(x, y)
+                        ctx.arc(x + step / 2, y, step / 2, Math.PI, 0, true)
+                        ctx.lineTo(x + step, 0)
+
+                        x = x + step
+                        y = -y
+                        ctx.moveTo(x, 0)
+                        ctx.lineTo(x, y)
+                        ctx.arc(x + step / 2, y, step / 2, Math.PI, 0, false)
+                        ctx.lineTo(x + step, 0)
+                    }
+
+                    ctx.stroke()
+                    ctx.closePath()
+                },
+                // // Set a bar width
+                // barWidth: 20,
+                // // Optionally, specify the spacing between bars
+                // barGap: 4,
+                // // And the bar radius
+                // barRadius: 2,
+            })
+
+            // Initialize the Record plugin
+            record = wavesurfer.registerPlugin(RecordPlugin.create({ scrollingWaveform: false, renderRecordedAudio: false }))
+            // Render recorded audio
+            // recButton.textContent = 'Record'
+
+        }
+
+        startButton.addEventListener('click', e => {
+            record.startRecording()
+
+            if (isRecording) return;
+            isRecording = true;
+
+            const AudioContext = window.AudioContext || window.webkitAudioContext;
+            context = new AudioContext();
+
+            navigator.mediaDevices.getUserMedia({ audio: true }).then(stream => {
+                globalStream = stream;
+                const input = context.createMediaStreamSource(stream);
+                processor = context.createScriptProcessor(bufferSize, 1, 1);
+                processor.onaudioprocess = e => processAudio(e);
+                input.connect(processor);
+                processor.connect(context.destination);
+
+                sendAudioConfig();
+            }).catch(error => console.error('Error accessing microphone', error));
+
+            // Disable start button and enable stop button
+            startButton.disabled = true;
+            stopButton.disabled = false;
+
+        });
+
+        stopButton.addEventListener('click', e => {
+            if (!isRecording) return;
+            isRecording = false;
+
+            if (globalStream) {
+                globalStream.getTracks().forEach(track => track.stop());
+            }
+            if (processor) {
+                processor.disconnect();
+                processor = null;
+            }
+            if (context) {
+                context.close().then(() => context = null);
+            }
+            startButton.disabled = false;
+            stopButton.disabled = true;
+
+            // 可视化部分
+            if (record.isRecording() || record.isPaused()) {
+                record.stopRecording()
+                return
+            }
+        })
+
+        connectButton.addEventListener('click', e => {
+            initWebSocket()
+        })
+
+        initServerButton.addEventListener('click', async e => {
+            const response = await fetch('/mixlab/start_live', {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json'
+                },
+                body: JSON.stringify({
+                    port: 8323,
+                    model: ''
+                })
+            })
+            console.log(await response.json())
+
+        })
+
+        // Initialize on page load
+        window.onload = () => {
+            initWebSocket()
+            createWaveSurfer();
+        };
+
+        function initWebSocket() {
+            const websocketAddress = document.getElementById('websocketAddress').value;
+            chunk_length_seconds = document.getElementById('chunk_length_seconds').value;
+            chunk_offset_seconds = document.getElementById('chunk_offset_seconds').value;
+            const selectedLanguage = document.getElementById('languageSelect').value;
+            language = selectedLanguage !== 'multilingual' ? selectedLanguage : null;
+
+            if (!websocketAddress) {
+                console.log("WebSocket address is required.");
+                return;
+            }
+
+            if (websocket) websocket.close()
+
+            websocket = new WebSocket(websocketAddress);
+            websocket.onopen = () => {
+                console.log("WebSocket connection established");
+                document.getElementById("webSocketStatus").textContent = 'Connected';
+                startButton.disabled = false;
+            };
+            websocket.onclose = event => {
+                console.log("WebSocket connection closed", event);
+                document.getElementById("webSocketStatus").textContent = 'Not Connected';
+                stopButton.click()
+                startButton.disabled = true;
+                stopButton.disabled = true;
+
+                // setTimeout(()=>initWebSocket(),1000)
+            };
+            websocket.onmessage = event => {
+                console.log("Message from server:", event.data);
+                const transcript_data = JSON.parse(event.data);
+
+                if (transcript_data.status === 'chat_start') {
+                    updateTranscription(transcript_data);
+                    stopButton.click();
+                } else if (transcript_data.status === 'chat_end') {
+                    let asistant = decodeURIComponent(transcript_data.asistant)
+                    document.getElementById('asistant').innerText = asistant;
+                    startButton.click();
+                }
+
+            };
+            websocket.onerror = () => {
+                // setTimeout(()=>initWebSocket(),1000)
+            }
+        }
+
+        function updateTranscription(transcript_data) {
+            const transcriptionDiv = document.getElementById('transcription');
+            const languageDiv = document.getElementById('detected_language');
+
+            if (transcript_data['words'] && transcript_data['words'].length > 0) {
+                // Append words with color based on their probability
+                transcript_data['words'].forEach(wordData => {
+                    const span = document.createElement('span');
+                    const probability = wordData['probability'];
+                    span.textContent = wordData['word'] + ' ';
+
+                    // Set the color based on the probability
+                    if (probability > 0.9) {
+                        span.style.color = 'green';
+                    } else if (probability > 0.6) {
+                        span.style.color = 'orange';
+                    } else {
+                        span.style.color = 'red';
+                    }
+
+                    transcriptionDiv.appendChild(span);
+                });
+
+                // Add a new line at the end
+                transcriptionDiv.appendChild(document.createElement('br'));
+            } else {
+                // Fallback to plain text
+                transcriptionDiv.textContent += transcript_data['text'] + '\n';
+            }
+
+            // Update the language information
+            if (transcript_data['language'] && transcript_data['language_probability']) {
+                languageDiv.textContent = transcript_data['language'] + ' (' + transcript_data['language_probability'].toFixed(2) + ')';
+            }
+
+            // Update the processing time, if available
+            const processingTimeDiv = document.getElementById('processing_time');
+            if (transcript_data['processing_time']) {
+                processingTimeDiv.textContent = 'Processing time: ' + transcript_data['processing_time'].toFixed(2) + ' seconds';
+            }
+        }
+
+
+
+
+        function sendAudioConfig() {
+            let selectedStrategy = document.getElementById('bufferingStrategySelect').value;
+            let processingArgs = {};
+
+            if (selectedStrategy === 'silence_at_end_of_chunk') {
+                processingArgs = {
+                    chunk_length_seconds: parseFloat(document.getElementById('chunk_length_seconds').value),
+                    chunk_offset_seconds: parseFloat(document.getElementById('chunk_offset_seconds').value)
+                };
+            }
+
+            const audioConfig = {
+                type: 'config',
+                data: {
+                    sampleRate: context.sampleRate,
+                    bufferSize: bufferSize,
+                    channels: 1, // Assuming mono channel
+                    language: language,
+                    processing_strategy: selectedStrategy,
+                    processing_args: processingArgs
+                }
+            };
+
+            websocket.send(JSON.stringify(audioConfig));
+        }
+
+        function downsampleBuffer(buffer, inputSampleRate, outputSampleRate) {
+            if (inputSampleRate === outputSampleRate) {
+                return buffer;
+            }
+            var sampleRateRatio = inputSampleRate / outputSampleRate;
+            var newLength = Math.round(buffer.length / sampleRateRatio);
+            var result = new Float32Array(newLength);
+            var offsetResult = 0;
+            var offsetBuffer = 0;
+            while (offsetResult < result.length) {
+                var nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
+                var accum = 0, count = 0;
+                for (var i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) {
+                    accum += buffer[i];
+                    count++;
+                }
+                result[offsetResult] = accum / count;
+                offsetResult++;
+                offsetBuffer = nextOffsetBuffer;
+            }
+            return result;
+        }
+
+        function processAudio(e) {
+            const inputSampleRate = context.sampleRate;
+            const outputSampleRate = 16000; // Target sample rate
+
+            const left = e.inputBuffer.getChannelData(0);
+            const downsampledBuffer = downsampleBuffer(left, inputSampleRate, outputSampleRate);
+            const audioData = convertFloat32ToInt16(downsampledBuffer);
+
+            if (websocket && websocket.readyState === WebSocket.OPEN) {
+                websocket.send(audioData);
+            }
+        }
+
+        function convertFloat32ToInt16(buffer) {
+            let l = buffer.length;
+            const buf = new Int16Array(l);
+            while (l--) {
+                buf[l] = Math.min(1, buffer[l]) * 0x7FFF;
+            }
+            return buf.buffer;
+        }
+
+
+        function toggleBufferingStrategyPanel() {
+            var selectedStrategy = document.getElementById('bufferingStrategySelect').value;
+            if (selectedStrategy === 'silence_at_end_of_chunk') {
+                var panel = document.getElementById('silence_at_end_of_chunk_options_panel');
+                panel.classList.remove('hidden');
+            } else {
+                var panel = document.getElementById('silence_at_end_of_chunk_options_panel');
+                panel.classList.add('hidden');
+            }
+        }
+
+
+    </script>
+</body>
+
+</html>