0.30.2

Appinfo supports outputting to the background, enhancing the performance features of ComfyUI.

.github/workflows/publish.yml

@@ -0,0 +1,21 @@
+name: Publish to Comfy registry
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - "pyproject.toml"
+
+jobs:
+  publish-node:
+    name: Publish Custom Node to registry
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Publish Custom Node
+        uses: Comfy-Org/publish-node-action@main
+        with:
+          ## Add your own personal access token to your Github Repository secrets and reference it here.
+          personal_access_token: ${{ secrets.REGISTRY_ACCESS_TOKEN }}

CNAME

@@ -0,0 +1 @@
+mixlabnodes.com

README.md

@@ -6,11 +6,17 @@
 
 ##### `最新`：
 
-ChatGPT 节点支持 Local LLM（llama.cpp），Phi3、llama3 都可以直接一个节点运行了。模型下载后，放置到 `models/llamafile/`
+- 增加 Edit Mask，方便在生成的时候手动绘制 mask [workflow](./workflow/edit-mask-workflow.json)
+
+
+- ChatGPT 节点支持 Local LLM（llama.cpp），Phi3、llama3 都可以直接一个节点运行了。模型下载后，放置到 `models/llamafile/`
 
 - 右键菜单支持 text-to-text，方便对 prompt 词补全
 
-强烈推荐：[Phi-3-mini-4k-instruct-GGUF](https://huggingface.co/lmstudio-community/Phi-3-mini-4k-instruct-GGUF/tree/main)，备选：[llama3_if_ai_sdpromptmkr_q2k](https://hf-mirror.com/impactframes/llama3_if_ai_sdpromptmkr_q2k/tree/main)
+强烈推荐：
+[Phi-3-mini-4k-instruct-function-calling-GGUF](https://huggingface.co/nold/Phi-3-mini-4k-instruct-function-calling-GGUF)
+
+[Phi-3-mini-4k-instruct-GGUF](https://huggingface.co/lmstudio-community/Phi-3-mini-4k-instruct-GGUF/tree/main)，备选：[llama3_if_ai_sdpromptmkr_q2k](https://hf-mirror.com/impactframes/llama3_if_ai_sdpromptmkr_q2k/tree/main)
 
 - 右键菜单支持 image-to-text，使用多模态模型，多模态使用 [llava-phi-3-mini-gguf](https://huggingface.co/xtuner/llava-phi-3-mini-gguf/tree/main)，注意需要把llava-phi-3-mini-mmproj-f16.gguf也下载
 
@@ -20,7 +26,11 @@ ChatGPT 节点支持 Local LLM（llama.cpp），Phi3、llama3 都可以直接一
 
 #### `相关插件推荐`
 
-<!-- [comfyui-sd-prompt-mixlab](https://github.com/shadowcz007/comfyui-sd-prompt-mixlab) -->
+[comfyui-liveportrait](https://github.com/shadowcz007/comfyui-liveportrait)
+
+[Comfyui-ChatTTS](https://github.com/shadowcz007/Comfyui-ChatTTS)
+
+[comfyui-sound-lab](https://github.com/shadowcz007/comfyui-sound-lab)
 
 [comfyui-Image-reward](https://github.com/shadowcz007/comfyui-Image-reward)
 
@@ -37,6 +47,7 @@ ChatGPT 节点支持 Local LLM（llama.cpp），Phi3、llama3 都可以直接一
 - 发布为 app 的 workflow，可以在右键里再次编辑了
 - web app 可以设置分类，在 comfyui 右键菜单可以编辑更新 web app
 - 支持动态提示
+- 支持把输出显示到comfyui背景（TouchDesigner 风格）
 
 ![](./assets/微信图片_20240421205440.png)
 
@@ -158,6 +169,9 @@ pip install llama-cpp-python \
 
 > A new layer class node has been added, allowing you to separate the image into layers. After merging the images, you can input the controlnet for further processing.
 
+> The composite images node overlays a foreground image onto a background image at specified positions and scales, with optional blending modes and masking capabilities. position : 'overall',"center_center","left_bottom","center_bottom","right_bottom","left_top","center_top","right_top"
+
+
 ![layers](./assets/layers-workflow.svg)
 
 ![poster](./assets/poster-workflow.svg)
@@ -189,6 +203,12 @@ pip install llama-cpp-python \
 
 > Conveniently load images from a fixed address on the internet to ensure that default images in the workflow can be executed.
 
+#### TextImage
+
+> [下载字体](https://drxie.github.io/OSFCC/)放到 ```custom_nodes/comfyui-mixlab-nodes/assets/fonts```
+
+
+
 ### Style
 
 > Apply VisualStyle Prompting , Modified from [ComfyUI_VisualStylePrompting](https://github.com/ExponentialML/ComfyUI_VisualStylePrompting)

__init__.py

@@ -8,6 +8,8 @@ import hashlib
 import datetime
 import folder_paths
 import logging
+import base64,io,re
+from PIL import Image
 from comfy.cli_args import args
 python = sys.executable
 
@@ -30,6 +32,8 @@ except:
     print("##nodes.ChatGPT ImportError")
 
 
+from .nodes.RembgNode import get_rembg_models,U2NET_HOME,run_briarmbg,run_rembg
+
 from server import PromptServer
 
 try:
@@ -98,6 +102,26 @@ install_openai()
 current_path = os.path.abspath(os.path.dirname(__file__))
 
 
+def remove_base64_prefix(base64_str):
+  """
+  去除 base64 字符串中的 data:image/*;base64, 前缀
+
+  Args:
+    base64_str: base64 编码的字符串
+
+  Returns:
+    去除前缀后的 base64 字符串
+  """
+
+  # 使用正则表达式匹配常见的前缀
+  pattern = r'^data:image\/(.*);base64,(.+)$'
+  match = re.match(pattern, base64_str)
+  if match:
+    # 如果匹配到常见的前缀，则去除前缀并返回
+    return match.group(2)
+  else:
+    # 如果不匹配到常见的前缀，则直接返回
+    return base64_str
 
 def calculate_md5(string):
     encoded_string = string.encode()
@@ -629,10 +653,53 @@ async def get_checkpoints(request):
             names=get_llama_models()
     except:
         print("llamafile none")
+
+    try:
+        if data['type']=='rembg':
+            names=get_rembg_models(U2NET_HOME)
+    except:
+        print("rembg none")
     
     return web.json_response({"names":names,"types":list(folder_paths.folder_names_and_paths.keys())})
 
 
+@routes.post('/mixlab/rembg')
+async def rembg_hander(request):
+    data = await request.json()
+    model=data['model']
+    result={}
+
+    data_base64=remove_base64_prefix(data['base64'])
+    image_data = base64.b64decode(data_base64)
+    
+    # 创建一个BytesIO对象
+    image_stream = io.BytesIO(image_data)
+
+    # 使用PIL Image模块读取图像
+    image = Image.open(image_stream)
+
+    if model=='briarmbg':
+        _,rgba_images,_=run_briarmbg([image])
+    else:
+        _,rgba_images,_=run_rembg(model,[image])
+
+    with io.BytesIO() as buf:
+        rgba_images[0].save(buf, format='PNG')
+        img_bytes = buf.getvalue()
+    img_base64 = base64.b64encode(img_bytes).decode('utf-8')
+    
+    try: 
+        result={
+            'data':img_base64,
+            'model':model,
+            'status':'success',
+            }
+    except Exception as e:
+            print(e)
+
+    return web.json_response(result)
+
+
 @routes.post("/mixlab/prompt_result")
 async def post_prompt_result(request):
     data = await request.json()
@@ -711,12 +778,14 @@ async def start_local_llm(data):
     
     server_settings=ServerSettings(host=address,port=port)
 
+    name, ext = os.path.splitext(os.path.basename(model))
+    print('#model',name)
     app = create_app(
                 server_settings=server_settings,
                 model_settings=[
                     ModelSettings(
                     model=model,
-                    model_alias=os.path.basename(model),
+                    model_alias=name,
                     n_gpu_layers=n_gpu_layers,
                     n_ctx=4098,
                     chat_format=chat_format,
@@ -750,12 +819,12 @@ async def start_local_llm(data):
 async def my_hander_method(request):
     data =await request.json()
     # print(data)
+    if llama_port and llama_model and llama_chat_format:
+        return web.json_response({"port":llama_port,"model":llama_model,"chat_format":llama_chat_format} )
     try:
         result=await start_local_llm(data)
     except:
-        result={
-            {"port":None,"model":"","llama_cpp_error":True}
-        }
+        result= {"port":None,"model":"","llama_cpp_error":True}
         print('start_local_llm error')
 
     return web.json_response(result)
@@ -773,20 +842,16 @@ def re_start(request):
 
 # 导入节点
 from .nodes.PromptNode import GLIGENTextBoxApply_Advanced,EmbeddingPrompt,RandomPrompt,PromptSlide,PromptSimplification,PromptImage,JoinWithDelimiter
-from .nodes.ImageNode import ComparingTwoFrames,LoadImages_,CompositeImages,GridDisplayAndSave,GridInput,ImagesPrompt,SaveImageAndMetadata,SaveImageToLocal,SplitImage,GridOutput,GetImageSize_,MirroredImage,ImageColorTransfer,NoiseImage,TransparentImage,GradientImage,LoadImagesFromPath,LoadImagesFromURL,ResizeImage,TextImage,SvgImage,Image3D,ShowLayer,NewLayer,MergeLayers,CenterImage,AreaToMask,SmoothMask,SplitLongMask,ImageCropByAlpha,EnhanceImage,FaceToMask
+from .nodes.ImageNode import ImageListToBatch_,ComparingTwoFrames,LoadImages_,CompositeImages,GridDisplayAndSave,GridInput,ImagesPrompt,SaveImageAndMetadata,SaveImageToLocal,SplitImage,GridOutput,GetImageSize_,MirroredImage,ImageColorTransfer,NoiseImage,TransparentImage,GradientImage,LoadImagesFromPath,LoadImagesFromURL,ResizeImage,TextImage,SvgImage,Image3D,ShowLayer,NewLayer,MergeLayers,CenterImage,AreaToMask,SmoothMask,SplitLongMask,ImageCropByAlpha,EnhanceImage,FaceToMask
 # from .nodes.Vae import VAELoader,VAEDecode
 from .nodes.ScreenShareNode import ScreenShareNode,FloatingVideo
 
-from .nodes.ChatGPT import ChatGPTNode,ShowTextForGPT,CharacterInText,TextSplitByDelimiter
-from .nodes.Audio import GamePal,SpeechRecognition,SpeechSynthesis
+from .nodes.Audio import AudioPlayNode,SpeechRecognition,SpeechSynthesis
 from .nodes.Utils import IncrementingListNode,ListSplit,CreateLoraNames,CreateSampler_names,CreateCkptNames,CreateSeedNode,TESTNODE_,TESTNODE_TOKEN,AppInfo,IntNumber,FloatSlider,TextInput,ColorInput,FontInput,TextToNumber,DynamicDelayProcessor,LimitNumber,SwitchByIndex,MultiplicationNode
 from .nodes.Mask import PreviewMask_,MaskListReplace,MaskListMerge,OutlineMask,FeatheredMask
 
 from .nodes.Style import ApplyVisualStylePrompting,StyleAlignedReferenceSampler,StyleAlignedBatchAlign,StyleAlignedSampleReferenceLatents
 
-from .nodes.Video import VideoCombine_Adv,LoadVideoAndSegment,ImageListReplace,VAEEncodeForInpaint_Frames
-
-from .nodes.TripoSR import LoadTripoSRModel,TripoSRSampler,SaveTripoSRMesh
 
 
 # 要导出的所有节点及其名称的字典
@@ -817,6 +882,7 @@ NODE_CLASS_MAPPINGS = {
     "ImageColorTransfer":ImageColorTransfer,
     "ShowLayer":ShowLayer,
     "NewLayer":NewLayer,
+    "ImageListToBatch_":ImageListToBatch_,
     "CompositeImages_":CompositeImages,
     "SplitImage":SplitImage,
     "CenterImage":CenterImage,
@@ -838,10 +904,7 @@ NODE_CLASS_MAPPINGS = {
     # "VAEDecodeConsistencyDecoder":VAEDecode,
     "ScreenShare":ScreenShareNode,
     "FloatingVideo":FloatingVideo,
-    "ChatGPTOpenAI":ChatGPTNode,
-    "ShowTextForGPT":ShowTextForGPT,
-    "CharacterInText":CharacterInText,
-    "TextSplitByDelimiter":TextSplitByDelimiter,
+   
     "SpeechRecognition":SpeechRecognition,
     "SpeechSynthesis":SpeechSynthesis,
     "Color":ColorInput,
@@ -865,24 +928,21 @@ NODE_CLASS_MAPPINGS = {
     "ApplyVisualStylePrompting_":ApplyVisualStylePrompting,
     "StyleAlignedReferenceSampler_": StyleAlignedReferenceSampler,
     "StyleAlignedSampleReferenceLatents_": StyleAlignedSampleReferenceLatents,
-    "StyleAlignedBatchAlign_": StyleAlignedBatchAlign,
-    "LoadVideoAndSegment_":LoadVideoAndSegment,
-    "VideoCombine_Adv":VideoCombine_Adv,
+    "StyleAlignedBatchAlign_": StyleAlignedBatchAlign, 
     "ListSplit_":ListSplit,
-    "MaskListReplace_":MaskListReplace,
-    "ImageListReplace_":ImageListReplace,
-    "VAEEncodeForInpaint_Frames":VAEEncodeForInpaint_Frames,
+    "MaskListReplace_":MaskListReplace, 
     "IncrementingListNode_":IncrementingListNode,
     "PreviewMask_":PreviewMask_,
-     "LoadTripoSRModel_": LoadTripoSRModel,
-    "TripoSRSampler_": TripoSRSampler,
-    "SaveTripoSRMesh": SaveTripoSRMesh
-    # "GamePal":GamePal
+    "AudioPlay":AudioPlayNode
 }
 
 # 一个包含节点友好/可读的标题的字典
 NODE_DISPLAY_NAME_MAPPINGS = {
     "AppInfo":"App Info ♾️MixlabApp",
+    "ScreenShare":"Screen Share ♾️Mixlab",
+    "FloatingVideo":"Floating Video ♾️Mixlab",
+    "TextImage":"Text Image ♾️Mixlab",
+    
     "Color":"Color Input ♾️MixlabApp",
     "TextInput_":"Text Input ♾️MixlabApp",
     "FloatSlider":"Float Slider Input ♾️MixlabApp",
@@ -896,14 +956,13 @@ NODE_DISPLAY_NAME_MAPPINGS = {
     "SplitLongMask":"Splitting a long image into sections",
     "VAELoaderConsistencyDecoder":"Consistency Decoder Loader",
     "VAEDecodeConsistencyDecoder":"Consistency Decoder Decode",
-    "ScreenShare":"Screen Share ♾️Mixlab",
-    "FloatingVideo":"FloatingVideo ♾️Mixlab",
-    "ChatGPTOpenAI":"ChatGPT & Local LLM ♾️Mixlab",
-    "ShowTextForGPT":"Show Text ♾️MixlabApp",
+    
+    
     "MergeLayers":"Merge Layers ♾️Mixlab",
     "SpeechSynthesis":"SpeechSynthesis ♾️Mixlab",
     "SpeechRecognition":"SpeechRecognition ♾️Mixlab",
     "3DImage":"3DImage ♾️Mixlab",
+    "ImageListToBatch_":"Image List To Batch",
     "CompositeImages_":"Composite Images ♾️Mixlab",
     "DynamicDelayProcessor":"DynamicDelayByText ♾️Mixlab",
     "LaMaInpainting":"LaMaInpainting ♾️Mixlab",
@@ -929,13 +988,12 @@ NODE_DISPLAY_NAME_MAPPINGS = {
     "GridInput":"Grid Input ♾️Mixlab",
     "GridOutput":"Grid Output ♾️Mixlab",
     "GetImageSize_":"Get Image Size ♾️Mixlab",
-    "VAEEncodeForInpaint_Frames":"VAE Encode For Inpaint Frames ♾️Mixlab",
     "IncrementingListNode_":"Create Incrementing Number List ♾️Mixlab",
     "LoadImagesToBatch":"Load Images(base64) ♾️Mixlab",
     "PreviewMask_":"Preview Mask",
-    "LoadTripoSRModel_": "Load TripoSR Model",
-    "TripoSRSampler_": "TripoSR Sampler",
-    "SaveTripoSRMesh": "Save TripoSR Mesh"
+    "AudioPlay":"Audio Play ♾️Mixlab",
+
+     "MultiplicationNode":"Math Operation ♾️Mixlab",
 }
 
 # web ui的节点功能
@@ -946,6 +1004,41 @@ logging.info('--------------')
 logging.info('\033[91m ### Mixlab Nodes: \033[93mLoaded')
 # print('\033[91m ### Mixlab Nodes: \033[93mLoaded')
 
+try:
+    from .nodes.ChatGPT import ChatGPTNode,ShowTextForGPT,CharacterInText,TextSplitByDelimiter
+    logging.info('ChatGPT.available True')
+
+    NODE_CLASS_MAPPINGS_V = {
+       "ChatGPTOpenAI":ChatGPTNode,
+        "ShowTextForGPT":ShowTextForGPT,
+        "CharacterInText":CharacterInText,
+        "TextSplitByDelimiter":TextSplitByDelimiter,
+    }
+
+    # 一个包含节点友好/可读的标题的字典
+    NODE_DISPLAY_NAME_MAPPINGS_V = {
+        "ChatGPTOpenAI":"ChatGPT & Local LLM ♾️Mixlab",
+        "ShowTextForGPT":"Show Text ♾️MixlabApp",
+        "CharacterInText":"Character In Text",
+        "TextSplitByDelimiter":"Text Split By Delimiter",
+    }
+
+
+    NODE_CLASS_MAPPINGS.update(NODE_CLASS_MAPPINGS_V)
+    NODE_DISPLAY_NAME_MAPPINGS.update(NODE_DISPLAY_NAME_MAPPINGS_V)
+
+except Exception as e:
+    logging.info('ChatGPT.available False')
+
+
+try:
+    from .nodes.edit_mask import EditMask
+    logging.info('edit_mask.available True')
+    NODE_CLASS_MAPPINGS['EditMask']=EditMask
+    NODE_DISPLAY_NAME_MAPPINGS['EditMask']="Edit Mask ♾️Mixlab"
+except Exception as e:
+    logging.info('edit_mask.available False')
+
 try:
     from .nodes.Lama import LaMaInpainting
     logging.info('LaMaInpainting.available {}'.format(LaMaInpainting.available))
@@ -981,4 +1074,61 @@ try:
 except Exception as e:
     logging.info('RembgNode_.available False' )
 
+
+try:
+    from .nodes.Video import GenerateFramesByCount,scenesNode_,CombineAudioVideo,VideoCombine_Adv,LoadVideoAndSegment,ImageListReplace,VAEEncodeForInpaint_Frames,LoadAndCombinedAudio_
+    
+    NODE_CLASS_MAPPINGS_V = {
+        "VAEEncodeForInpaint_Frames":VAEEncodeForInpaint_Frames,
+        "ImageListReplace_":ImageListReplace,
+        "LoadVideoAndSegment_":LoadVideoAndSegment,
+        "VideoCombine_Adv":VideoCombine_Adv,
+        "LoadAndCombinedAudio_":LoadAndCombinedAudio_,
+        "CombineAudioVideo":CombineAudioVideo,
+        "ScenesNode_":scenesNode_,
+        "GenerateFramesByCount":GenerateFramesByCount 
+    }
+
+    # 一个包含节点友好/可读的标题的字典
+    NODE_DISPLAY_NAME_MAPPINGS_V = {
+        "VAEEncodeForInpaint_Frames":"VAE Encode For Inpaint Frames ♾️Mixlab",
+        "ImageListReplace_":"Image List Replace",
+        "LoadVideoAndSegment_":"Load Video And Segment",
+        "VideoCombine_Adv":"Video Combine",
+        "LoadAndCombinedAudio_":"Load And Combined Audio",
+        "CombineAudioVideo":"Combine Audio Video",
+        "ScenesNode_":"Scenes Node", 
+        "GenerateFramesByCount":"Generate Frames By Count"
+    }
+
+
+    NODE_CLASS_MAPPINGS.update(NODE_CLASS_MAPPINGS_V)
+    NODE_DISPLAY_NAME_MAPPINGS.update(NODE_DISPLAY_NAME_MAPPINGS_V)
+
+except:
+    logging.info('Video.available False')
+
+
+try:
+    from .nodes.TripoSR import LoadTripoSRModel,TripoSRSampler,SaveTripoSRMesh
+    logging.info('TripoSR.available')
+
+    NODE_CLASS_MAPPINGS['LoadTripoSRModel_']=LoadTripoSRModel
+    NODE_DISPLAY_NAME_MAPPINGS["LoadTripoSRModel_"]= "Load TripoSR Model"
+    
+    NODE_CLASS_MAPPINGS['TripoSRSampler_']=TripoSRSampler
+    NODE_DISPLAY_NAME_MAPPINGS["TripoSRSampler_"]= "TripoSR Sampler"
+
+    NODE_CLASS_MAPPINGS['SaveTripoSRMesh']=SaveTripoSRMesh
+    NODE_DISPLAY_NAME_MAPPINGS["SaveTripoSRMesh"]= "Save TripoSR Mesh"
+    
+
+except Exception as e:
+    logging.info('TripoSR.available False' )
+
+
+
+
+
+
 logging.info('\033[93m -------------- \033[0m')

nodes/Audio.py

@@ -1,6 +1,7 @@
 
-
-
+import os
+import folder_paths
+import torchaudio
 
 class SpeechRecognition:
     @classmethod
@@ -55,46 +56,60 @@ class SpeechSynthesis:
         return {"ui": {"text": text}, "result": (text,)}
     
 
-#
-class GamePal:
+
+class AudioPlayNode:
+    def __init__(self):
+        self.output_dir = folder_paths.get_temp_directory()
+        self.type = "temp"
+        self.prefix_append = ""
+        self.compress_level = 4
     @classmethod
     def INPUT_TYPES(s):
-        return {
-            "required": {
-                "input_text": ("STRING",{"multiline": True,"default": ""}),
-            },
-            "optional": {
-                 
-                "input_num": ("INT",{
-                                "default":100, 
-                                "min": -1, #Minimum value
-                                "max": 0xffffffffffffffff, #Maximum value
-                                "step": 1, #Slider's step
-                                "display": "slider" # Cosmetic only: display as "number" or "slider"
-                                }), 
-                "python_code": ("STRING",{"multiline": True,"default": "result= 1 if 'Mixlab' in input_text else 0"}),
-            }
-        }
-
-    INPUT_IS_LIST = False
-    RETURN_TYPES = ("INT",)
+        return {"required": {
+            "audio": ("AUDIO",),
+              }, 
+                }
+    
+    RETURN_TYPES = ()
+  
     FUNCTION = "run"
-    OUTPUT_NODE = True
-    OUTPUT_IS_LIST = (False,)
 
     CATEGORY = "♾️Mixlab/Audio"
 
-    def run(self, input_text,input_num,python_code):
-        exec(python_code)
-        res=None
-        try:
-            # 可能会引发异常的代码
-            res=result
-        except:
-            # 处理异常的代码
-            print('')
+    INPUT_IS_LIST = False
+    OUTPUT_IS_LIST = ()
 
-        print(res)
+    OUTPUT_NODE = True
+  
+    def run(self,audio):
 
-        # print(session_history)
-        return {"ui": {"text": [input_text],"num":[input_num]}, "result": (res,)}
+        # 判断是否是 Tensor 类型
+        is_tensor = not isinstance(audio, dict)
+        # print('#判断是否是 Tensor 类型',is_tensor,audio)
+        if not is_tensor and 'waveform' in audio and 'sample_rate' in audio:
+            # {'waveform': tensor([], size=(1, 1, 0)), 'sample_rate': 44100}
+            is_tensor=True
+
+        if is_tensor:
+            filename_prefix=""
+            # 保存
+            filename_prefix += self.prefix_append
+            full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, self.output_dir)
+            results = list()
+            
+            filename_with_batch_num = filename.replace("%batch_num%", str(1))
+            file = f"{filename_with_batch_num}_{counter:05}_.wav"
+            
+            torchaudio.save(os.path.join(full_output_folder, file), audio['waveform'].squeeze(0), audio["sample_rate"])
+            results.append({
+                    "filename": file,
+                    "subfolder": subfolder,
+                    "type": self.type
+                })
+            
+        else:
+            results=[audio]
+                
+
+        # print(audio)
+        return {"ui": {"audio":results}}

nodes/ChatGPT.py

@@ -215,15 +215,28 @@ class ChatGPTNode:
 
     @classmethod
     def INPUT_TYPES(cls):
-        model_list=llama_modes_list+[
+        model_list=llama_modes_list+[ 
                     "gpt-3.5-turbo",
-                    "gpt-3.5-turbo-0125",
-                    "gpt-35-turbo",
-                    "gpt-3.5-turbo-16k", 
-                    "gpt-3.5-turbo-16k-0613", 
-                    "gpt-4-0613",
-                    "gpt-4-1106-preview",
-                    "glm-4"
+            "gpt-3.5-turbo-16k",
+            "gpt-4o",
+            "gpt-4o-2024-05-13",
+            "gpt-4",
+            "gpt-4-0314",
+            "gpt-4-0613",
+            "gpt-3.5-turbo-0301",
+            "gpt-3.5-turbo-0613",
+            "gpt-3.5-turbo-16k-0613",
+            "qwen-turbo",
+            "qwen-plus",
+            "qwen-long",
+            "qwen-max",
+            "qwen-max-longcontext",
+            "glm-4",
+            "glm-3-turbo",
+            "moonshot-v1-8k",
+            "moonshot-v1-32k",
+            "moonshot-v1-128k",
+            "deepseek-chat" 
                     ]
         return {
             "required": {

nodes/ImageNode.py

@@ -1,6 +1,7 @@
 import numpy as np
 import requests
 import torch
+import torchvision.transforms.v2 as T
 # from PIL import Image, ImageDraw
 from PIL import Image, ImageOps,ImageFilter,ImageEnhance,ImageDraw,ImageSequence, ImageFont
 from PIL.PngImagePlugin import PngInfo
@@ -14,8 +15,8 @@ import cv2
 import string
 import math,glob
 from .Watcher import FolderWatcher
-import hashlib
 
+from itertools import product
 
 
 # 将PIL图片转换为OpenCV格式
@@ -28,142 +29,102 @@ def opencv_to_pil(image):
     pil_image = Image.fromarray(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
     return pil_image
 
+# 列出目录下面的所有文件
+def get_files_with_extension(directory, extension):
+    file_list = []
+    for root, dirs, files in os.walk(directory):
+        for file in files:
+            if file.endswith(extension):
+                file = os.path.splitext(file)[0]
+                file_path = os.path.join(root, file)
+                file_name = os.path.relpath(file_path, directory)
+                file_list.append(file_name)
+    return file_list
 
-def composite_images(foreground, background, mask,is_multiply_blend=False,position="overall"):
-    width,height=foreground.size
- 
-    bg_image=background
+def composite_images(foreground, background, mask, is_multiply_blend=False, position="overall", scale=0.25):
+    width, height = foreground.size
+    bg_image = background
+    bwidth, bheight = bg_image.size
 
-    bwidth,bheight=bg_image.size
+    scale=max(scale,1/bwidth)
+    scale=max(scale,1/bheight)
 
-    # 按z-index排序
-    if position=="overall":
+    def determine_scale_option(width, height):
+        return 'height' if height > width else 'width'
+
+    if position == "overall":
         layer = {
-                "x":0,
-                "y":0,
-                "width":bwidth,
-                "height":bheight,
-                "z_index":88,
-                "scale_option":'overall',
-                "image":foreground,
-                "mask":mask
+            "x": 0,
+            "y": 0,
+            "width": bwidth,
+            "height": bheight,
+            "z_index": 88,
+            "scale_option": 'overall',
+            "image": foreground,
+            "mask": mask
         }
+    else:
+        scale_option = determine_scale_option(width, height)
+        if scale_option == 'height':
+            scale = int(bheight * scale) / height
+        else:
+            scale = int(bwidth * scale) / width
 
-    elif position=='center_bottom':
-        
-        scale = int(bwidth*0.25) / width
+        new_width = int(width * scale)
         new_height = int(height * scale)
 
-        layer = {
-                "x":int(bwidth*0.75*0.5),
-                "y":bheight-new_height-24,
-                "width":int(bwidth*0.25),
-                "height":int(bheight*0.25),
-                "z_index":88,
-                "scale_option":'width',
-                "image":foreground,
-                "mask":mask
-        }
-
-    elif position=='right_bottom':
-        
-        scale = int(bwidth*0.25) / width
-        new_height = int(height * scale)
+        if position == 'center_bottom':
+            x_position = int((bwidth - new_width) * 0.5)
+            y_position = bheight - new_height - 24
+        elif position == 'right_bottom':
+            x_position = bwidth - new_width - 24
+            y_position = bheight - new_height - 24
+        elif position == 'center_top':
+            x_position = int((bwidth - new_width) * 0.5)
+            y_position = 24
+        elif position == 'right_top':
+            x_position = bwidth - new_width - 24
+            y_position = 24
+        elif position == 'left_top':
+            x_position = 24
+            y_position = 24
+        elif position == 'left_bottom':
+            x_position = 24
+            y_position = bheight - new_height - 24
+        elif position == 'center_center':
+            x_position = int((bwidth - new_width) * 0.5)
+            y_position = int((bheight - new_height) * 0.5)
 
         layer = {
-                "x":bwidth-int(bwidth*0.25)-24,
-                "y":bheight-new_height-24,
-                "width":int(bwidth*0.25),
-                "height":int(bheight*0.25),
-                "z_index":88,
-                "scale_option":'width',
-                "image":foreground,
-                "mask":mask
+            "x": x_position,
+            "y": y_position,
+            "width": new_width,
+            "height": new_height,
+            "z_index": 88,
+            "scale_option": scale_option,
+            "image": foreground,
+            "mask": mask
         }
 
+    layer_image = layer['image']
+    layer_mask = layer['mask']
 
-    elif position=='center_top':
-        
-        scale = int(bwidth*0.25) / width
-        new_height = int(height * scale)
+    bg_image = merge_images(bg_image,
+                            layer_image,
+                            layer_mask,
+                            layer['x'],
+                            layer['y'],
+                            layer['width'],
+                            layer['height'],
+                            layer['scale_option'],
+                            is_multiply_blend)
 
-        layer = {
-                "x":int( bwidth*0.75*0.5),
-                "y":24,
-                "width":int(bwidth*0.25),
-                "height":int(bheight*0.25),
-                "z_index":88,
-                "scale_option":'width',
-                "image":foreground,
-                "mask":mask
-        }
+    bg_image = bg_image.convert('RGB')
 
-    elif position=='right_top':
-        
-        scale = int(bwidth*0.25) / width
-        new_height = int(height * scale)
-
-        layer = {
-                "x":bwidth-int(bwidth*0.25)-24,
-                "y":24,
-                "width":int(bwidth*0.25),
-                "height":int(bheight*0.25),
-                "z_index":88,
-                "scale_option":'width',
-                "image":foreground,
-                "mask":mask
-        }
-    elif position=='left_top':
-        
-        scale = int(bwidth*0.25) / width
-        new_height = int(height * scale)
-
-        layer = {
-                "x":24,
-                "y":24,
-                "width":int(bwidth*0.25),
-                "height":int(bheight*0.25),
-                "z_index":88,
-                "scale_option":'width',
-                "image":foreground,
-                "mask":mask
-        }
-    elif position=='left_bottom':
-        
-        scale = int(bwidth*0.25) / width
-        new_height = int(height * scale)
-
-        layer = {
-                "x":24,
-                "y":bheight-new_height-24,
-                "width":int(bwidth*0.25),
-                "height":int(bheight*0.25),
-                "z_index":88,
-                "scale_option":'width',
-                "image":foreground,
-                "mask":mask
-        }
-                
-    # width, height = bg_image.size
-
-    layer_image=layer['image']
-    layer_mask=layer['mask']
-    
-    bg_image=merge_images(bg_image,
-                        layer_image,
-                        layer_mask,
-                        layer['x'],
-                        layer['y'],
-                        layer['width'],
-                        layer['height'],
-                        layer['scale_option'],
-                        is_multiply_blend )
-                   
-    bg_image=bg_image.convert('RGB')
-    
     return bg_image
 
 
+
 def count_files_in_directory(directory):
     file_count = 0
     for _, _, files in os.walk(directory):
@@ -200,7 +161,8 @@ class AnyType(str):
 any_type = AnyType("*")
 
 
-FONT_PATH= os.path.abspath(os.path.join(os.path.dirname(__file__),'../assets/王汉宗颜楷体繁.ttf'))
+FONT_PATH= os.path.abspath(os.path.join(os.path.dirname(__file__),'../assets/fonts'))
+
 
 MAX_RESOLUTION=8192
 
@@ -802,85 +764,78 @@ def multiply_blend(image1, image2):
 
 # cv2.imwrite('result.jpg', result)
 
+# 使用gpt4o优化代码
+# 为了消除图像合并时出现的灰色描边，可以使用以下方法：
+# 调整透明度：确保透明像素不会引入不需要的颜色。
+# 预处理图像：在缩放图像之前，可以先将图像的边缘进行预处理，例如扩展边缘颜色，减少抗锯齿带来的过渡效果。
 
-def merge_images(bg_image, layer_image, mask, x, y, width, height, scale_option,is_multiply_blend=False):
+def merge_images(bg_image, layer_image, mask, x, y, width, height, scale_option, is_multiply_blend=False):
     # 打开底图
     bg_image = bg_image.convert("RGBA")
 
     # 打开图层
     layer_image = layer_image.convert("RGBA")
-    # layer_image = layer_image.resize((width, height))
-
+   
     # 根据缩放选项调整图像大小
     if scale_option == "height":
         # 按照高度比例缩放
         original_width, original_height = layer_image.size
         scale = height / original_height
         new_width = int(original_width * scale)
-        layer_image = layer_image.resize((new_width, height))
+        layer_image = layer_image.resize((new_width, height), Image.NEAREST)
     elif scale_option == "width":
         # 按照宽度比例缩放
         original_width, original_height = layer_image.size
         scale = width / original_width
         new_height = int(original_height * scale)
-        layer_image = layer_image.resize((width, new_height))
+        layer_image = layer_image.resize((width, new_height), Image.NEAREST)
     elif scale_option == "overall":
         # 整体缩放
-        layer_image = layer_image.resize((width, height))
-
+        layer_image = layer_image.resize((width, height), Image.NEAREST)
     elif scale_option == "longest":
         original_width, original_height = layer_image.size
         if original_width > original_height:
-            new_width=width
+            new_width = width
             scale = width / original_width
             new_height = int(original_height * scale)
-            x=0
-            y=int((height-new_height)*0.5)
+            x = 0
+            y = int((height - new_height) * 0.5)
         else:
-            new_height=height
+            new_height = height
             scale = height / original_height
             new_width = int(original_height * scale)
-            x=int((width-new_width)*0.5)
-            y=0
-    # elif side == "shortest":
-    #         if width < height:
-    #              
-    #         else:
-    #              
-    
+            x = int((width - new_width) * 0.5)
+            y = 0
 
     # 调整mask的大小
     nw, nh = layer_image.size
-    mask = mask.resize((nw, nh))
+    mask = mask.resize((nw, nh), Image.NEAREST)
 
-    # # 分离出a通道
-    # r, g, b, alpha = layer_image.split()
-    # alpha = ImageOps.invert(alpha)
-    # # 创建一个新的RGB图像
-    # new_rgb_image = Image.new("RGB", layer_image.size)
-    # # 将透明通道粘贴到新的RGB图像上
-    # new_rgb_image.paste(layer_image, (0, 0), mask=alpha)
-   
-    # new_rgb_image.paste(layer_image, (x, y), mask=mask)
-    # mask=new_rgb_image.convert('L')
-    # mask = ImageOps.invert(mask)
+    # 预处理图像边缘以减少灰色描边
+    layer_image = layer_image.filter(ImageFilter.SMOOTH)
 
     if is_multiply_blend:
-        bg_image_white=Image.new("RGB", bg_image.size,(255, 255, 255))
+        bg_image_white = Image.new("RGB", bg_image.size, (255, 255, 255))
 
         bg_image_white.paste(layer_image, (x, y), mask=mask)
-        bg_image=multiply_blend(bg_image_white,bg_image)
-        bg_image=bg_image.convert("RGBA")
+        bg_image = multiply_blend(bg_image_white, bg_image)
+        bg_image = bg_image.convert("RGBA")
     else:
-        transparent_img = Image.new("RGBA",layer_image.size, (255, 255, 255, 0))
-        transparent_img.paste(layer_image,(0, 0), mask)
-        # transparent_img.save('test.png')
-        bg_image.paste(transparent_img, (x, y), transparent_img)
+        transparent_img = Image.new("RGBA", layer_image.size, (255, 255, 255, 0))
+        # 调整透明度处理
+        for i in range(transparent_img.size[0]):
+            for j in range(transparent_img.size[1]):
+                r, g, b, a = transparent_img.getpixel((i, j))
+                if a > 0:
+                    transparent_img.putpixel((i, j), (r, g, b, 255))
 
+        transparent_img.paste(layer_image, (0, 0), mask)
+        bg_image.paste(transparent_img, (x, y), transparent_img)
 
     # 输出合成后的图片
     return bg_image
 
+#MixCopilot
 
 def resize_2(img):
     # 检查图像的高度是否是2的倍数，如果不是，则调整高度
@@ -954,53 +909,13 @@ def resize_image(layer_image, scale_option, width, height,color="white"):
     return layer_image
 
 
-
-
-
-
-# def generate_text_image(text_list, font_path, font_size, text_color, vertical=True, spacing=0):
-#     # Load Chinese font
-#     font = ImageFont.truetype(font_path, font_size)
-
-#     # Calculate image size based on the number of characters and orientation
-#     if vertical:
-#         width = font_size + 100
-#         height = font_size * len(text_list) + (len(text_list) - 1) * spacing + 100
-#     else:
-#         width = font_size * len(text_list) + (len(text_list) - 1) * spacing + 100
-#         height = font_size + 100
-
-#     # Create a blank image
-#     image = Image.new('RGBA', (width, height), (255, 255, 255,0))
-#     draw = ImageDraw.Draw(image)
-
-#     # Draw text
-#     if vertical:
-#         for i, char in enumerate(text_list):
-#             char_position = (50, 50 + i * font_size)
-#             draw.text(char_position, char, font=font, fill=text_color)
-#     else:
-#         for i, char in enumerate(text_list):
-#             char_position = (50 + i * (font_size + spacing), 50)
-#             draw.text(char_position, char, font=font, fill=text_color)
-
-#     # Save the image
-#     # image.save(output_image_path)
-
-#     # 分离alpha通道
-#     alpha_channel = image.split()[3]
-
-#     # 创建一个只有alpha通道的新图像
-#     alpha_image = Image.new('L', image.size)
-#     alpha_image.putdata(alpha_channel.getdata())
-
-#     image=image.convert('RGB')
-
-#     return (image,alpha_image)
-def generate_text_image(text, font_path, font_size, text_color, vertical=True, stroke=False, stroke_color=(0, 0, 0), stroke_width=1, spacing=0):
+def generate_text_image(text, font_path, font_size, text_color, vertical=True, stroke=False, stroke_color=(0, 0, 0), stroke_width=1, spacing=0, padding=4):
     # Split text into lines based on line breaks
     lines = text.split("\n")
 
+    # Load font
+    font = ImageFont.truetype(font_path, font_size)
+
     # 1. Determine layout direction
     if vertical:
         layout = "vertical"
@@ -1009,49 +924,46 @@ def generate_text_image(text, font_path, font_size, text_color, vertical=True, s
 
     # 2. Calculate absolute coordinates for each character
     char_coordinates = []
-    if layout == "vertical":
-        x = 0
-        y = 0
-        for i in range(len(lines)):
-            line = lines[i]
-            for char in line:
-                char_coordinates.append((x, y))
-                y += font_size + spacing
-            x += font_size + spacing
-            y = 0
-    else:
-        x = 0
-        y = 0
-        for line in lines:
-            for char in line:
-                char_coordinates.append((x, y))
-                x += font_size + spacing
-            y += font_size + spacing
-            x = 0
+    x, y = padding, padding
+    max_width, max_height = 0, 0
 
-    # 3. Calculate image width and height
     if layout == "vertical":
-        width = (len(lines) * (font_size + spacing)) - spacing
-        height = ((len(max(lines, key=len)) + 1) * (font_size + spacing)) + spacing
+        for line in lines:
+            max_char_width = max(font.getsize(char)[0] for char in line)
+            for char in line:
+                char_width, char_height = font.getsize(char)
+                char_coordinates.append((x, y))
+                y += char_height + spacing
+                max_height = max(max_height, y + padding)
+            x += max_char_width + spacing
+            y = padding
+        max_width = x
     else:
-        width = (len(max(lines, key=len)) * (font_size + spacing)) - spacing
-        height = ((len(lines) - 1) * (font_size + spacing)) + font_size
+        for line in lines:
+            line_width, line_height = font.getsize(line)
+            for char in line:
+                char_width, char_height = font.getsize(char)
+                char_coordinates.append((x, y))
+                x += char_width + spacing
+                max_width = max(max_width, x + padding)
+            y += line_height + spacing
+            x = padding
+        max_height = y
+
+    # 3. Create image with calculated width and height
+    image = Image.new('RGBA', (max_width, max_height), (255, 255, 255, 0))
+    draw = ImageDraw.Draw(image)
 
     # 4. Draw each character on the image
-    image = Image.new('RGBA', (width, height), (255, 255, 255, 0))
-    draw = ImageDraw.Draw(image)
-    font = ImageFont.truetype(font_path, font_size)
-
     index = 0
-    for i, line in enumerate(lines):
-        for j, char in enumerate(line):
+    for line in lines:
+        for char in line:
             x, y = char_coordinates[index]
-            
             if stroke:
-                draw.text((x-stroke_width, y), char, font=font, fill=stroke_color)
-                draw.text((x+stroke_width, y), char, font=font, fill=stroke_color)
-                draw.text((x, y-stroke_width), char, font=font, fill=stroke_color)
-                draw.text((x, y+stroke_width), char, font=font, fill=stroke_color)
+                draw.text((x-stroke_width, y), char, font=font, fill=text_color)
+                draw.text((x+stroke_width, y), char, font=font, fill=text_color)
+                draw.text((x, y-stroke_width), char, font=font, fill=text_color)
+                draw.text((x, y+stroke_width), char, font=font, fill=text_color)
             
             draw.text((x, y), char, font=font, fill=text_color)
             index += 1
@@ -1578,7 +1490,7 @@ class ImageCropByAlpha:
 
 
 
-
+# get_files_with_extension(FONT_PATH,'.ttf')
 
 class TextImage:
     @classmethod
@@ -1586,7 +1498,7 @@ class TextImage:
         return {"required": { 
             
                     "text": ("STRING",{"multiline": True,"default": "龍馬精神迎新歲","dynamicPrompts": False}),
-                    "font_path": ("STRING",{"multiline": False,"default": FONT_PATH,"dynamicPrompts": False}),
+                    "font": (get_files_with_extension(FONT_PATH,'.ttf'),),#后缀为 ttf
                     "font_size": ("INT",{
                                 "default":100, 
                                 "min": 100, #Minimum value
@@ -1601,6 +1513,13 @@ class TextImage:
                                 "step": 1, #Slider's step
                                 "display": "number" # Cosmetic only: display as "number" or "slider"
                                 }), 
+                    "padding": ("INT",{
+                                "default":8, 
+                                "min": 0, #Minimum value
+                                "max": 200, #Maximum value
+                                "step": 1, #Slider's step
+                                "display": "number" # Cosmetic only: display as "number" or "slider"
+                                }), 
                     "text_color":("STRING",{"multiline": False,"default": "#000000","dynamicPrompts": False}),
                     "vertical":("BOOLEAN", {"default": True},),
                     "stroke":("BOOLEAN", {"default": False},),
@@ -1608,7 +1527,7 @@ class TextImage:
                 }
     
     RETURN_TYPES = ("IMAGE","MASK",)
-    # RETURN_NAMES = ("WIDTH","HEIGHT","X","Y",)
+    RETURN_NAMES = ("image","mask",)
 
     FUNCTION = "run"
 
@@ -1617,11 +1536,14 @@ class TextImage:
     INPUT_IS_LIST = False
     OUTPUT_IS_LIST = (False,False,)
 
-    def run(self,text,font_path,font_size,spacing,text_color,vertical,stroke):
+    def run(self,text,font,font_size,spacing,padding,text_color,vertical,stroke):
         
-        # text_list=list(text)
+        font_path=os.path.join(FONT_PATH,font+'.ttf')
+
+        if text=="":
+            text=" "
         # stroke=False, stroke_color=(0, 0, 0), stroke_width=1, spacing=0
-        img,mask=generate_text_image(text,font_path,font_size,text_color,vertical,stroke,(0, 0, 0),1,spacing)
+        img,mask=generate_text_image(text,font_path,font_size,text_color,vertical,stroke,(0, 0, 0),1,spacing,padding)
         
         img=pil2tensor(img)
         mask=pil2tensor(mask)
@@ -1854,10 +1776,16 @@ class CompositeImages:
                     "mask":("MASK",),
                     "background": ("IMAGE",),
                     },
-             "optional":{
-                           
+             "optional":{ 
                   "is_multiply_blend":  ("BOOLEAN", {"default": False}),
-                  "position":  (['overall',"center_bottom","center_top","right_bottom","left_bottom","right_top","left_top"],),    
+                  "position":  (['overall',"center_center","left_bottom","center_bottom","right_bottom","left_top","center_top","right_top"],),
+                   "scale": ("FLOAT",{
+                                "default":0.35, 
+                                "min": 0.01, #Minimum value
+                                "max": 1, #Maximum value
+                                "step": 0.01, #Slider's step
+                                "display": "number" # Cosmetic only: display as "number" or "slider"
+                            }), 
                     }
                 }
     
@@ -1870,15 +1798,33 @@ class CompositeImages:
 
     # OUTPUT_IS_LIST = (True,)
 
-    def run(self, foreground,mask,background,is_multiply_blend,position):
-        foreground= tensor2pil(foreground)
-        mask= tensor2pil(mask)
-        background= tensor2pil(background)
-        res=composite_images(foreground,background,mask,is_multiply_blend,position)
+    # def run(self, foreground,mask,background,is_multiply_blend,position,scale):
+    #     foreground= tensor2pil(foreground)
+    #     mask= tensor2pil(mask)
+    #     background= tensor2pil(background)
+    #     res=composite_images(foreground,background,mask,is_multiply_blend,position,scale)
         
-        return (pil2tensor(res),)
+    #     return (pil2tensor(res),)
 
+    def run(self, foreground,mask,background, is_multiply_blend, position, scale):
+        results = []
+        
+        f1=[]
+        for fg, mask in zip(foreground, mask ):
+            f1.append([fg,mask])
 
+        
+        for f, bg in product(f1, background):
+            [fg,mask]=f
+            fg_pil = tensor2pil(fg)
+            mask_pil = tensor2pil(mask)
+            bg_pil = tensor2pil(bg)
+            res = composite_images(fg_pil, bg_pil, mask_pil, is_multiply_blend, position, scale)
+            results.append(pil2tensor(res))
+        
+        output_image = torch.cat(results, dim=0)
+
+        return (output_image,)
 
 
 class EmptyLayer:
@@ -3207,3 +3153,52 @@ class SaveImageToLocal:
             counter += 1
 
         return ()
+
+
+class ImageBatchToList_:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": {"image_batch": ("IMAGE",), }}
+
+    RETURN_TYPES = ("IMAGE",)
+    RETURN_NAMES = ("image_list",)
+    OUTPUT_IS_LIST = (True,)
+    FUNCTION = "run"
+
+    CATEGORY = "♾️Mixlab/Image"
+
+    def run(self, image_batch):
+        images = [image_batch[i:i + 1, ...] for i in range(image_batch.shape[0])]
+        return (images, )
+
+class ImageListToBatch_:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {
+            "required": {
+                "images": ("IMAGE",),
+            }
+        }
+
+    RETURN_TYPES = ("IMAGE",)
+    FUNCTION = "run"
+    INPUT_IS_LIST = True
+    CATEGORY = "♾️Mixlab/Image"
+
+    def run(self, images):
+        shape = images[0].shape[1:3]
+        out = []
+
+        for i in range(len(images)):
+            img = images[i].permute([0,3,1,2])
+            if images[i].shape[1:3] != shape:
+                transforms = T.Compose([
+                    T.CenterCrop(min(img.shape[2], img.shape[3])),
+                    T.Resize((shape[0], shape[1]), interpolation=T.InterpolationMode.BICUBIC),
+                ])
+                img = transforms(img)
+            out.append(img.permute([0,2,3,1]))
+
+        out = torch.cat(out, dim=0)
+
+        return (out,)

nodes/Lama.py

@@ -85,8 +85,6 @@ class LaMaInpainting:
             "image": ("IMAGE",),
             "mask": ("MASK",),
                              },
-
-            
                 }
     
     RETURN_TYPES = ("IMAGE",)

nodes/RembgNode.py

@@ -467,15 +467,37 @@ class BriaRMBG(nn.Module):
 
 
 
+def get_U2NET_model_path():
+    try:
+        return folder_paths.get_folder_paths('rembg')[0]
+    except:
+        return os.path.join(folder_paths.models_dir, "rembg")
+    
 
-
-
-U2NET_HOME=os.path.join(folder_paths.models_dir, "rembg")
+U2NET_HOME=get_U2NET_model_path()
 os.environ["U2NET_HOME"] = U2NET_HOME
 
 global _available
 _available=False
 
+
+def get_rembg_models(path):
+    """从目录中获取文件并提取文件名
+    Args:
+        path: 目录路径
+    Returns:
+        文件名列表
+    """
+    filenames = []
+    for root, _, files in os.walk(path):
+        for filename in files:
+        # 过滤隐藏文件
+            if not filename.startswith('.'):
+                name, ext = os.path.splitext(os.path.basename(filename))
+                filenames.append(name)
+    return filenames
+
+
 def is_installed(package):
     try:
         spec = importlib.util.find_spec(package)
@@ -509,8 +531,8 @@ except:
     _available=False
 
 
-def briarmbg_run(images=[]):
-    mroot=os.path.join(folder_paths.models_dir, "rembg")
+def run_briarmbg(images=[]):
+    mroot=U2NET_HOME
     m=os.path.join(mroot,'briarmbg.pth')
     if os.path.exists(m)==False:
         # 下载
@@ -573,14 +595,15 @@ def briarmbg_run(images=[]):
     return (masks,rgba_images,rgb_images)   
 
 
-def run_bg(model_name= "unet",images=[]):
+def run_rembg(model_name= "unet",images=[],callback=None):
     # model_name = "unet" # "isnet-general-use"
+    # print('#run_rembg',model_name)
     rembg_session = new_session(model_name)
     masks=[]
     rgba_images=[]
     rgb_images=[]
     # 进度条
-    pbar = comfy.utils.ProgressBar(len(images) )
+    pbar=callback 
     for img in images:
         # use the post_process_mask argument to post process the mask to get better results.
         mask = remove(img, session=rembg_session,only_mask=True,post_process_mask=True)
@@ -620,8 +643,9 @@ def run_bg(model_name= "unet",images=[]):
             rgb_image = Image.new("RGB", image_rgba.size, (0, 0, 0))
             rgb_image.paste(image_rgba, mask=image_rgba.split()[3])
             rgb_images.append(rgb_image)
-            
-        pbar.update(1)
+
+        if pbar:
+            pbar.update(1)
     return (masks,rgba_images,rgb_images)
 
 
@@ -643,17 +667,7 @@ class RembgNode_:
     def INPUT_TYPES(s):
         return {"required": {
             "image": ("IMAGE",),
-            "model_name": ([
-                            "briarmbg",
-                            "u2net",
-                            "u2netp",
-                            "u2net_human_seg",
-                            "u2net_cloth_seg",
-                            "silueta",
-                            "isnet-general-use",
-                            "isnet-anime",
-                            
-                            ],),
+            "model_name": (get_rembg_models(U2NET_HOME),),
             
                              },
                 }
@@ -681,9 +695,9 @@ class RembgNode_:
                 images.append(im)
 
         if model_name=='briarmbg':
-            masks,rgba_images,rgb_images=briarmbg_run(images)
+            masks,rgba_images,rgb_images=run_briarmbg(images)
         else:
-            masks,rgba_images,rgb_images=run_bg(model_name,images)
+            masks,rgba_images,rgb_images=run_rembg(model_name,images, comfy.utils.ProgressBar(len(images) ))
 
         masks=[pil2tensor(m) for m in masks]
 

nodes/ScreenShareNode.py

@@ -90,7 +90,7 @@ class ScreenShareNode:
                 } }
     
     RETURN_TYPES = ('IMAGE','STRING','FLOAT',"INT")
-    RETURN_NAMES = ("IMAGE","PROMPT","FLOAT","INT")
+    RETURN_NAMES = ("current frame (image)","prompt","denoise (float)","seed (int)")
     FUNCTION = "run"
 
     CATEGORY = "♾️Mixlab/Screen"
@@ -109,7 +109,7 @@ class FloatingVideo:
     @classmethod
     def INPUT_TYPES(s):
         return { "required":{
-            "images": ("IMAGE",)
+            "image": ("IMAGE",)
         }, }
     
     # RETURN_TYPES = ('IMAGE','MASK')
@@ -124,16 +124,16 @@ class FloatingVideo:
     # OUTPUT_IS_LIST = (False,False,)
   
     # 运行的函数
-    def run(self,images):
+    def run(self,image):
        
         results = list()
     
-        for image in images:
-            image=tensor2pil(image)
+        for im in image:
+            im=tensor2pil(im)
             # image_base64 = base64.b64encode(image.tobytes())
 
             buffered = BytesIO()
-            image.save(buffered, format="JPEG")
+            im.save(buffered, format="JPEG")
             image_base64 = base64.b64encode(buffered.getvalue()).decode("utf-8")
 
             results.append(image_base64)

nodes/TripoSR.py

@@ -5,13 +5,20 @@ from PIL import Image
 import numpy as np
 import torch
 
-from folder_paths import get_filename_list, get_full_path, get_save_image_path, get_output_directory,models_dir
+from folder_paths import get_folder_paths, get_full_path, get_save_image_path, get_output_directory,models_dir
 from comfy.model_management import get_torch_device
 from .tsr.system import TSR
 
 import comfy.utils
 
-triposr_model_path=path.join(models_dir,'triposr/model.ckpt')
+
+def get_triposr_model_path():
+    try:
+        return path.join(get_folder_paths('triposr')[0],'model.ckpt')
+    except:
+        return path.join(path.join(models_dir, "triposr"),'model.ckpt')
+    
+triposr_model_path=get_triposr_model_path()
 
 
 # Tensor to PIL

nodes/Utils.py

@@ -284,7 +284,7 @@ class FloatSlider:
                 }
     
     RETURN_TYPES = ("FLOAT",) 
-    RETURN_NAMES = ('weight(0-1)',)
+    RETURN_NAMES = ('FLOAT',)
     FUNCTION = "run"
 
     CATEGORY = "♾️Mixlab/Input"
@@ -297,9 +297,7 @@ class FloatSlider:
             number = min_value
         elif number > max_value:
             number = max_value
-        scaled_number = (number - min_value) / (max_value - min_value)
-        return (scaled_number,)
-    
+        return (number,)    
   
 class IntNumber:
     @classmethod
@@ -568,7 +566,7 @@ class AppInfo:
                              },
 
                 "optional":{
-                    "IMAGE": ("IMAGE",),
+                    "image": ("IMAGE",),
                     "description":("STRING",{"multiline": True,"default": "","dynamicPrompts": False}),
                     "version":("INT", {
                         "default": 1, 
@@ -596,12 +594,12 @@ class AppInfo:
     INPUT_IS_LIST = True
     # OUTPUT_IS_LIST = (True,)
 
-    def run(self,name,input_ids,output_ids,IMAGE,description,version,share_prefix,link,category,auto_save):
+    def run(self,name,input_ids,output_ids,image,description,version,share_prefix,link,category,auto_save):
         name=name[0]
         
         im=None
-        if IMAGE:
-            im=IMAGE[0][0]
+        if image:
+            im=image[0][0]
             #TODO batch 的方式需要处理
             im=create_temp_file(im)
         # image [img,] img[batch,w,h,a] 列表里面是batch，

nodes/Video.py

@@ -17,9 +17,128 @@ import folder_paths
 from comfy.k_diffusion.utils import FolderOfImages
 from comfy.utils import common_upscale
 
+import torchaudio
+import base64
+
+import mimetypes
 
 
 
+def get_frames(frame_count, frames, revert=False):
+    if not revert:
+        if frame_count <= len(frames):
+            return frames[:frame_count]
+        else:
+            return [frames[i % len(frames)] for i in range(frame_count)]
+    else:
+        extended_frames = frames + frames[-2:0:-1]  # 正向加反向中间部分
+        if frame_count <= len(extended_frames):
+            return extended_frames[:frame_count]
+        else:
+            return [extended_frames[i % len(extended_frames)] for i in range(frame_count)]
+
+# # 示例用法
+# frames = ["frame1", "frame2", "frame3"]
+# frame_count = 2
+
+# result = get_frames(frame_count, frames, revert=False)
+# print(result)  # 输出: ['frame1', 'frame2', 'frame3', 'frame1', 'frame2', 'frame3', 'frame1']
+
+# result = get_frames(frame_count, frames, revert=True)
+# print(result)  # 输出: ['frame1', 'frame2', 'frame3', 'frame2', 'frame1', 'frame2', 'frame3']
+
+
+
+
+def get_mime_type(file_path):
+    # 获取文件的 MIME 类型
+    mime_type, _ = mimetypes.guess_type(file_path)
+    
+    # 如果无法猜测类型，返回默认类型
+    if mime_type is None:
+        return 'application/octet-stream'
+    
+    return mime_type
+# import subprocess
+# from imageio_ffmpeg import get_ffmpeg_exe
+
+
+def save_audio_base64s_to_file(base64_audios, output_folder, file_name):
+    # Ensure the output folder exists
+    if not os.path.exists(output_folder):
+        os.makedirs(output_folder)
+
+    decoded_audios=[]
+    for a in base64_audios:
+    
+        # If the base64 string contains a header, remove it
+        if ',' in a:
+            a = a.split(',')[1]
+        
+        # 解码 base64 数据
+        a=base64.b64decode(a)
+        decoded_audios.append(a)
+
+    # 拼接音频数据
+    combined_audio = b''.join(decoded_audios)
+    
+    # Create the full file path
+    file_path = os.path.join(output_folder, file_name)
+    
+    # Write the decoded audio to the file
+    with open(file_path, 'wb') as audio_file:
+        audio_file.write(combined_audio)
+    
+    return file_path
+
+# Example usage
+# base64_audio = "data:audio/wav;base64,UklGRiQAAABXQVZFZm10IBAAAAABAAEAIlYAAESsAAACABAAZGF0YQAAAAA="
+# output_folder = "audio_files"
+# file_name = "output.wav"
+
+# file_path = save_audio_base64_to_file(base64_audio, output_folder, file_name)
+# print(f"Audio saved to: {file_path}")
+
+# 写一个python文件，用来 判断文件夹内命名为 所有chat_tts开头的文件数量（chat_tts_00001），并输出新的编号
+def get_new_counter(full_output_folder, filename_prefix):
+    # 获取目录中的所有文件
+    files = os.listdir(full_output_folder)
+    
+    # 过滤出以 filename_prefix 开头并且后续部分为数字的文件
+    filtered_files = []
+    for f in files:
+        if f.startswith(filename_prefix):
+            # 去掉文件名中的前缀和后缀，只保留中间的数字部分
+            base_name = f[len(filename_prefix)+1:]
+            number_part = base_name.split('.')[0]  # 假设文件名中只有一个点，即扩展名
+            if number_part.isdigit():
+                filtered_files.append(int(number_part))
+
+    if not filtered_files:
+        return 1
+
+    # 获取最大的编号
+    max_number = max(filtered_files)
+    
+    # 新的编号
+    return max_number + 1
+
+def crop_audio(input_file, start_time, duration):
+    # Load the audio file
+    audio_tensor, sample_rate = torchaudio.load(input_file)
+    
+    # Convert start_time and duration from seconds to sample indices
+    start_sample = int(start_time * sample_rate)
+    end_sample = start_sample + int(duration * sample_rate)
+    
+    # Perform the slicing
+    cropped_audio_tensor = audio_tensor[:, start_sample:end_sample]
+    
+    # Save the cropped audio to a new file
+    torchaudio.save(input_file, cropped_audio_tensor, sample_rate)
+    
+    return input_file
+
 def generate_folder_name(directory,video_path):
     # Get the directory and filename from the video path
     _, filename = os.path.split(video_path)
@@ -60,6 +179,9 @@ def split_video(video_path, video_segment_frames, transition_frames, output_dir)
         
         # 打印当前片段的起始帧和结束帧
         print(f"Segment {i+1}: Start Frame {start_frame}, End Frame {end_frame}")
+
+        if end_frame<start_frame:
+            break
         
         # 保存当前片段为一个视频文件
         segment_video_path = f"{output_dir}/segment_{i+1}.avi"
@@ -68,6 +190,7 @@ def split_video(video_path, video_segment_frames, transition_frames, output_dir)
         segment_video = cv2.VideoWriter(segment_video_path, fourcc, fps, (int(video_capture.get(cv2.CAP_PROP_FRAME_WIDTH)),
                                                               int(video_capture.get(cv2.CAP_PROP_FRAME_HEIGHT))))
         
+
         for frame_num in range(start_frame, end_frame):
             ret, frame = video_capture.read()
             if ret:
@@ -87,7 +210,7 @@ def split_video(video_path, video_segment_frames, transition_frames, output_dir)
 
 folder_paths.folder_names_and_paths["video_formats"] = (
     [
-        os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "video_formats"),
+        os.path.join(os.path.dirname(os.path.abspath(__file__)), ".", "video_formats"),
     ],
     [".json"]
 )
@@ -101,6 +224,25 @@ if ffmpeg_path is None:
     except:
         print("ffmpeg could not be found. Outputs that require it have been disabled")
 
+
+def combine_audio_video(audio_path, video_path, output_path):
+   
+    command = [
+        ffmpeg_path,
+        '-i', video_path,
+        '-i', audio_path,
+        '-c:v', 'copy',
+        '-c:a', 'aac',
+        '-shortest',
+        output_path
+    ]
+    
+    subprocess.run(command, check=True)
+    return output_path
+
+
+
+
 # Tensor to PIL
 def tensor2pil(image):
     return Image.fromarray(np.clip(255. * image.cpu().numpy().squeeze(), 0, 255).astype(np.uint8))
@@ -262,7 +404,7 @@ class LoadVideoAndSegment:
                     files.append(f)
         return {"required": {
                     "video": (sorted(files), {"video_upload": True}),
-                     "video_segment_frames": ("INT", {"default": 10, "min": 1, "step": 1}),
+                     "video_segment_frames": ("INT", {"default": 10, "min": -1, "step": 1}),
                      "transition_frames": ("INT", {"default": 0, "min": 0, "step": 1}), 
                      },}
 
@@ -332,63 +474,6 @@ class LoadVideoAndSegment:
         
         video_path = folder_paths.get_annotated_filepath(video)
         
-        # check if video is a gif - will need to use cv fallback to read frames
-        # use cv fallback if ffmpeg not installed or gif
-        # if ffmpeg_path is None:
-        #     return self.load_video_cv_fallback(video, frame_load_cap, skip_first_frames)
-        # otherwise, continue with ffmpeg
-        
-        # args_dummy = [ffmpeg_path, "-i", video_path, "-f", "null", "-"]
-        # try:
-        #     with subprocess.Popen(args_dummy, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) as proc:
-        #         for line in proc.stderr.readlines():
-        #             match = re.search(", ([1-9]|\\d{2,})x(\\d+)",line.decode('utf-8'))
-        #             if match is not None:
-        #                 size = [int(match.group(1)), int(match.group(2))]
-        #                 break
-        # except Exception as e:
-        #     print(f"Retrying with opencv due to ffmpeg error: {e}")
-        #     return self.load_video_cv_fallback(video, frame_load_cap, skip_first_frames)
-        # args_all_frames = [ffmpeg_path, "-i", video_path, "-v", "error",
-        #                      "-pix_fmt", "rgb24"]
-
-        # vfilters = []
-       
-        # if skip_first_frames > 0:
-        #     vfilters.append(f"select=gt(n\\,{skip_first_frames-1})")
-        # if frame_load_cap > 0:
-        #     vfilters.append(f"select=gt({frame_load_cap}\\,n)")
-        # #manually calculate aspect ratio to ensure reads remain aligned
-        
-        # if len(vfilters) > 0:
-        #     args_all_frames += ["-vf", ",".join(vfilters)]
-
-        # args_all_frames += ["-f", "rawvideo", "-"]
-        # images = []
-        # try:
-        #     with subprocess.Popen(args_all_frames, stdout=subprocess.PIPE) as proc:
-        #         #Manually buffer enough bytes for an image
-        #         bpi = size[0]*size[1]*3
-        #         current_bytes = bytearray(bpi)
-        #         current_offset=0
-        #         while True:
-        #             bytes_read = proc.stdout.read(bpi - current_offset)
-        #             if bytes_read is None:#sleep to wait for more data
-        #                 time.sleep(.2)
-        #                 continue
-        #             if len(bytes_read) == 0:#EOF
-        #                 break
-        #             current_bytes[current_offset:len(bytes_read)] = bytes_read
-        #             current_offset+=len(bytes_read)
-        #             if current_offset == bpi:
-        #                 images.append(np.array(current_bytes, dtype=np.float32).reshape(size[1], size[0], 3) / 255.0)
-        #                 current_offset = 0
-        # except Exception as e:
-        #     print(f"Retrying with opencv due to ffmpeg error: {e}")
-        #     return self.load_video_cv_fallback(video, frame_load_cap, skip_first_frames)
-
-        # imgs=split_list(images,video_segment_frames,transition_frames)
-
         # temp path 
         tp=folder_paths.get_temp_directory()
         basename = os.path.basename(video_path)  # 获取文件名
@@ -396,15 +481,22 @@ class LoadVideoAndSegment:
 
         folder_path = create_folder(tp,name_without_extension)
         
- 
-        # 导出的数据
-        scenes_video,total_frames,fps=split_video(video_path,video_segment_frames,
-                                 transition_frames,folder_path)
+        if video_segment_frames==-1:
+            # 不切割视频
+            scenes_video=[video_path]
+            # 读取视频文件
+            video_capture = cv2.VideoCapture(video_path)
+            
+            # 获取视频的总帧数和帧率
+            total_frames = int(video_capture.get(cv2.CAP_PROP_FRAME_COUNT))
+            fps = video_capture.get(cv2.CAP_PROP_FPS)
+
+        else:
+            # 导出的数据
+            scenes_video,total_frames,fps=split_video(video_path,video_segment_frames,
+                                    transition_frames,folder_path)
       
 
-        # imgs=[torch.from_numpy(np.stack(im)) for im in imgs]
-
-        # images = torch.from_numpy(np.stack(images))
 
         return (scenes_video,len(scenes_video), total_frames,fps,)
 
@@ -422,7 +514,85 @@ class LoadVideoAndSegment:
             return "Invalid image file: {}".format(video)
 
         return True
-    
+
+
+
+class LoadAndCombinedAudio_:
+    @classmethod
+    def INPUT_TYPES(s):
+       
+        return {"required": {
+                    "audios": ("AUDIOBASE64",),
+                    "start_time": ("FLOAT" , {"default": 0, "min": 0, "max": 10000000, "step": 0.01}),
+                    "duration": ("FLOAT" , {"default": 10, "min": -1, "max": 10000000, "step": 0.01}),
+                     },
+                }
+
+    CATEGORY = "♾️Mixlab/Audio"
+
+    RETURN_TYPES = ("STRING","AUDIO",)
+    RETURN_NAMES = ("audio_file_path","audio",)
+    FUNCTION = "run"
+
+    def run(self,audios, start_time, duration):
+        output_dir = folder_paths.get_output_directory()
+        counter=get_new_counter(output_dir,'audio_')
+
+        audio_file_name = f"audio_{counter:05}.wav"
+
+        audio_file=save_audio_base64s_to_file(audios['base64'],output_dir,audio_file_name)
+        # duration == -1 则不裁切
+        if duration > -1:
+            crop_audio(audio_file, start_time, duration)
+
+        return (audio_file, {
+                "filename": audio_file_name,
+                "subfolder": "",
+                "type": "output",
+                "audio_path":audio_file
+                } ,)
+
+class CombineAudioVideo:
+    @classmethod
+    def INPUT_TYPES(s):
+       
+        return {"required": {
+                     "video_file_path": ("STRING",  {"forceInput": True}),
+                     "audio_file_path": ("STRING",  {"forceInput": True}), 
+                     },
+                }
+
+    CATEGORY = "♾️Mixlab/Video"
+
+    OUTPUT_NODE = True
+    FUNCTION = "run" 
+    RETURN_TYPES = ()
+    RETURN_NAMES = ()
+
+    def run(self,video_file_path, audio_file_path):
+
+        output_dir = folder_paths.get_output_directory()
+
+        counter=get_new_counter(output_dir,'video_final_')
+        
+        # 获取文件名和扩展名
+        base, ext = os.path.splitext(video_file_path)
+
+        v_file = f"video_final_{counter:05}{ext}"
+        
+        v_file_path=os.path.join(output_dir, v_file)
+
+        combine_audio_video(audio_file_path,video_file_path,v_file_path)
+
+        previews = [
+            {
+                "filename": v_file,
+                "subfolder": "",
+                "type": "output",
+                "format": get_mime_type(v_file),
+            }
+        ]
+        return {"ui": {"gifs": previews}}
 
 # The code is based on ComfyUI-VideoHelperSuite modification.
 class VideoCombine_Adv:
@@ -433,6 +603,7 @@ class VideoCombine_Adv:
             ffmpeg_formats = ["video/"+x[:-5] for x in folder_paths.get_filename_list("video_formats")]
         else:
             ffmpeg_formats = []
+        # ffmpeg_formats =["video/"+x for x in  ['webm', 'mp4', 'mkv']]
         return {
             "required": {
                 "image_batch": ("IMAGE",),
@@ -453,7 +624,8 @@ class VideoCombine_Adv:
             },
         }
 
-    RETURN_TYPES = ()
+    RETURN_TYPES = ("SCENE_VIDEO",)
+    RETURN_NAMES = ("scenes_video",)
     OUTPUT_NODE = True
     CATEGORY = "♾️Mixlab/Video"
     FUNCTION = "run"
@@ -622,7 +794,7 @@ class VideoCombine_Adv:
                 "format": format,
             }
         ]
-        return {"ui": {"gifs": previews}}
+        return {"ui": {"gifs": previews},"result":(file_path,)}
 
 
 class VAEEncodeForInpaint_Frames:
@@ -689,4 +861,113 @@ class VAEEncodeForInpaint_Frames:
             result.append({"samples":t, "noise_mask": (mask_erosion[:,:,:x,:y].round())})
 
 
-        return (result, )
+        return (result, )
+    
+
+
+class GenerateFramesByCount:
+    @classmethod
+    def INPUT_TYPES(cls):
+        
+        return {"required": {
+                    "frames": ('IMAGE',),
+                    "frame_count": ("INT", {"default": 72, "min": 1, "step": 1}), 
+                    "revert" :("BOOLEAN", {"default": True},),
+                     },}
+
+    RETURN_TYPES = ('IMAGE',)
+    RETURN_NAMES = ("frames",)
+
+    FUNCTION = "r"
+    CATEGORY = "♾️Mixlab/Video"
+    # INPUT_IS_LIST = True
+
+    def r(self, frames, frame_count, revert):
+        
+        image_list = [frames[i:i + 1, ...] for i in range(frames.shape[0])]
+
+        image_list=get_frames(frame_count,image_list,revert)
+
+        images = torch.cat(image_list, dim=0)
+
+        return (images,)
+
+
+class scenesNode_:
+    @classmethod
+    def INPUT_TYPES(cls):
+        
+        return {"required": {
+                    "scenes_video": ('SCENE_VIDEO',),
+                    "index": ("INT", {"default": 0, "min": 0, "step": 1}),
+                     
+                     },}
+
+    RETURN_TYPES = ('IMAGE','INT',)
+    RETURN_NAMES = ("frames","count",)
+    # OUTPUT_IS_LIST = (False,)
+
+    FUNCTION = "run"
+    CATEGORY = "♾️Mixlab/Video"
+    INPUT_IS_LIST = True
+
+    def load_video_cv_fallback(self, video, frame_load_cap, skip_first_frames):
+        # print('#video',video)
+        try:
+            video_cap = cv2.VideoCapture(video)
+            if not video_cap.isOpened():
+                raise ValueError(f"{video} could not be loaded with cv fallback.")
+            # set video_cap to look at start_index frame
+            images = []
+            total_frame_count = 0
+            frames_added = 0
+            base_frame_time = 1/video_cap.get(cv2.CAP_PROP_FPS)
+            
+            target_frame_time = base_frame_time
+
+            time_offset=0.0
+            while video_cap.isOpened():
+                if time_offset < target_frame_time:
+                    is_returned, frame = video_cap.read()
+                    # if didn't return frame, video has ended
+                    if not is_returned:
+                        break
+                    time_offset += base_frame_time
+                if time_offset < target_frame_time:
+                    continue
+                time_offset -= target_frame_time
+                # if not at start_index, skip doing anything with frame
+                total_frame_count += 1
+                if total_frame_count <= skip_first_frames:
+                    continue
+                # TODO: do whatever operations need to happen, like force_size, etc
+
+                # opencv loads images in BGR format (yuck), so need to convert to RGB for ComfyUI use
+                # follow up: can videos ever have an alpha channel?
+                frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+                # convert frame to comfyui's expected format (taken from comfy's load image code)
+                image = Image.fromarray(frame)
+                image = ImageOps.exif_transpose(image)
+                image = np.array(image, dtype=np.float32) / 255.0
+                image = torch.from_numpy(image)[None,]
+                images.append(image)
+                frames_added += 1
+                # if cap exists and we've reached it, stop processing frames
+                if frame_load_cap > 0 and frames_added >= frame_load_cap:
+                    break
+        finally:
+            video_cap.release()
+        
+        images = torch.cat(images, dim=0)
+       
+        return (images, frames_added,)
+
+    def run(self, scenes_video,index):
+        print('#scenes_video',index,scenes_video)
+        index=index[0]
+        if len(scenes_video) > index:
+            vp=scenes_video[index]
+
+            return self.load_video_cv_fallback(vp,0,0)
+        
+        return ([], 0,)

nodes/edit_mask.py

@@ -0,0 +1,172 @@
+import torch
+from PIL import Image, ImageOps, ImageSequence, ImageFile
+from PIL.PngImagePlugin import PngInfo
+
+import numpy as np
+import os
+import folder_paths 
+import node_helpers
+import hashlib
+
+# Tensor to PIL
+def tensor2pil(image):
+    return Image.fromarray(np.clip(255. * image.cpu().numpy().squeeze(), 0, 255).astype(np.uint8))
+
+# tensor 取hash值
+def tensor_to_hash(tensor):
+    # 将 Tensor 转换为 NumPy 数组
+    np_array = tensor.cpu().numpy()
+    
+    # 将 NumPy 数组转换为字节数据
+    byte_data = np_array.tobytes()
+    
+    # 计算哈希值
+    hash_value = hashlib.md5(byte_data).hexdigest()
+    
+    return hash_value
+
+
+def create_temp_file(image):
+    output_dir = folder_paths.get_temp_directory()
+
+    (
+            full_output_folder,
+            filename,
+            counter,
+            subfolder,
+            _,
+        ) = folder_paths.get_save_image_path('material', output_dir)
+
+    
+    image=tensor2pil(image)
+ 
+    image_file = f"{filename}_{counter:05}.png"
+     
+    image_path=os.path.join(full_output_folder, image_file)
+
+    image.save(image_path,compress_level=4)
+
+    return (image_path,[{
+    "filename": image_file,
+    "subfolder": subfolder,
+    "type": "temp"
+    }])
+
+
+# image - tensor - 文件路径
+# loadImage的方法（ 文件路径 - image-mask ）
+class EditMask:
+
+    def __init__(self):
+        self.image_id = None
+
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required":
+                    {"image": ("IMAGE",), # 表示一个张量
+                     
+                     },
+
+                    "optional":{
+                            "image_update": ("IMAGE_FILE",)
+                        },
+                   
+                }
+
+    CATEGORY = "♾️Mixlab/Mask"
+
+    RETURN_TYPES = ("IMAGE", "MASK")
+    RETURN_NAMES = ("image", "mask")
+
+    FUNCTION = "edit"
+
+    OUTPUT_NODE = True
+
+    def edit(self, image,image_update=None):
+
+        # 根据image输入来判断是否是新的图片
+        if self.image_id==None:
+            self.image_id=tensor_to_hash(image)
+            image_update=None
+        else:
+            image_id=tensor_to_hash(image)
+            if image_id!=self.image_id:
+                image_update=None
+                self.image_id=image_id
+
+
+        image_path=None
+        # print('#image_update',self.image_id,image_update)
+        if image_update==None:
+            print('--')
+        else:
+            if 'images' in image_update:
+                images=image_update['images']
+                filename=images[0]['filename']
+                subfolder=images[0]['subfolder']
+                type=images[0]['type']
+                name, base_dir=folder_paths.annotated_filepath(filename)
+                if type.endswith("output"):
+                    base_dir = folder_paths.get_output_directory() 
+                elif type.endswith("input"):
+                    base_dir = folder_paths.get_input_directory() 
+                elif type.endswith("temp"):
+                    base_dir = folder_paths.get_temp_directory() 
+                #base_dir = folder_paths.get_input_directory()  
+                # print(base_dir,subfolder, name)
+                image_path = os.path.join(base_dir,subfolder, name)
+        
+        if image_path==None:
+            image_path,images=create_temp_file(image)
+
+        print('#image_path',os.path.exists(image_path),image_path)
+        # image_path = folder_paths.get_annotated_filepath(image) #文件名
+        
+        if not os.path.exists(image_path):
+            image_path,images=create_temp_file(image)
+
+
+        img = node_helpers.pillow(Image.open, image_path)
+        
+        output_images = []
+        output_masks = []
+        w, h = None, None
+
+        excluded_formats = ['MPO']
+        
+        for i in ImageSequence.Iterator(img):
+            i = node_helpers.pillow(ImageOps.exif_transpose, i)
+
+            if i.mode == 'I':
+                i = i.point(lambda i: i * (1 / 255))
+            image = i.convert("RGB")
+
+            if len(output_images) == 0:
+                w = image.size[0]
+                h = image.size[1]
+            
+            if image.size[0] != w or image.size[1] != h:
+                continue
+            
+            image = np.array(image).astype(np.float32) / 255.0
+            image = torch.from_numpy(image)[None,]
+            if 'A' in i.getbands():
+                mask = np.array(i.getchannel('A')).astype(np.float32) / 255.0
+                mask = 1. - torch.from_numpy(mask)
+            else:
+                # 尺寸不对，需要按照image来
+                mask = torch.zeros((h, w), dtype=torch.float32, device="cpu")
+
+            output_images.append(image)
+            output_masks.append(mask.unsqueeze(0))
+
+        if len(output_images) > 1 and img.format not in excluded_formats:
+            output_image = torch.cat(output_images, dim=0)
+            output_mask = torch.cat(output_masks, dim=0)
+        else:
+            output_image = output_images[0]
+            output_mask = output_masks[0]
+
+        return {"ui":{"images": images},"result": (output_image, output_mask)}
+
+        # return (output_image, output_mask)

nodes/video_formats/av1-webm.json

@@ -0,0 +1,10 @@
+{
+    "main_pass":
+    [
+        "-n", "-c:v", "libsvtav1",
+        "-pix_fmt", "yuv420p10le",
+        "-crf", "23"
+    ],
+     "extension": "webm",
+     "environment": {"SVT_LOG": "1"}
+}

nodes/video_formats/h264-mp4.json

@@ -0,0 +1,9 @@
+{
+    "main_pass":
+    [
+        "-n", "-c:v", "libx264",
+        "-pix_fmt", "yuv420p",
+        "-crf", "19"
+    ],
+     "extension": "mp4"
+}

nodes/video_formats/h265-mp4.json

@@ -0,0 +1,11 @@
+{
+    "main_pass":
+    [
+        "-n", "-c:v", "libx265",
+        "-pix_fmt", "yuv420p10le",
+        "-preset", "medium",
+        "-crf", "22",
+        "-x265-params", "log-level=quiet"
+    ],
+     "extension": "mp4"
+}

nodes/video_formats/webm.json

@@ -0,0 +1,9 @@
+{
+    "main_pass":
+    [
+        "-n",
+        "-pix_fmt", "yuv420p",
+        "-crf", "23"
+    ],
+     "extension": "webm"
+}

pyproject.toml

@@ -0,0 +1,15 @@
+[project]
+name = "comfyui-mixlab-nodes"
+description = "3D, ScreenShareNode & FloatingVideoNode, SpeechRecognition & SpeechSynthesis, GPT, LoadImagesFromLocal, Layers, Other Nodes, ..."
+version = "0.30.2"
+license = "MIT"
+dependencies = ["numpy", "pyOpenSSL", "watchdog", "opencv-python-headless", "matplotlib", "openai", "simple-lama-inpainting", "clip-interrogator==0.6.0", "transformers>=4.36.0", "lark-parser", "imageio-ffmpeg", "rembg[gpu]", "omegaconf==2.3.0", "Pillow>=9.5.0", "einops==0.7.0", "trimesh>=4.0.5", "huggingface-hub", "scikit-image"]
+
+[project.urls]
+Repository = "https://github.com/shadowcz007/comfyui-mixlab-nodes"
+#  Used by Comfy Registry https://comfyregistry.org
+
+[tool.comfy]
+PublisherId = "shadow"
+DisplayName = "comfyui-mixlab-nodes"
+Icon = ""

requirements.txt

@@ -15,4 +15,6 @@ Pillow>=9.5.0
 einops==0.7.0
 trimesh>=4.0.5
 huggingface-hub
-scikit-image
+scikit-image
+torchaudio
+soundfile>=0.12.1

web/index.html

@@ -24,6 +24,7 @@
             margin-left: 5%;
             user-select: none;
             margin-top: 32px;
+            margin-bottom: 120px;
         }
 
         .apps {
@@ -65,9 +66,9 @@
             box-shadow: 0px 0px 10px 10px #fbe9f0
         }
 
-        .card:hover {
+        /* .card:hover {
             box-shadow: 0px 0px 10px 10px #e9fbfa
-        }
+        } */
 
         .card h5 {
             font-size: 14px;
@@ -145,15 +146,6 @@
             margin-top: 4px;
         }
 
-        .status {
-            background: black;
-            color: white;
-            display: flex;
-            width: fit-content;
-            padding: 4px;
-            font-size: 12px;
-
-        }
 
         .description {
             display: flex;
@@ -247,7 +239,7 @@
 
         .card textarea {
             width: 100%;
-            height:'fit-content';
+            height: 'fit-content';
             /* min-width: 300px; */
             margin-top: 12px;
             resize: none;
@@ -263,18 +255,44 @@
             margin-top: 12px;
         }
 
+        .run_div {
+            position: fixed;
+            bottom: 28px;
+            display: flex;
+            left: 144px;
+            z-index: 100;
+            /* align-items: center; */
+            /* justify-content: space-around; */
+            /* width: calc(50% - 100px); */
+            flex-direction: column;
+            background-color: #f7f7f7;
+            box-shadow: 3px 3px 8px #cacaca;
+            border-radius: 8px;
+        }
+
+        .status {
+            /* background: #e7e7e7; */
+            color: black;
+            display: flex;
+            width: 120px;
+            padding-left: 11px;
+            font-size: 12px;
+            border-radius: 8px;
+            justify-content: flex-start;
+            align-items: center;
+            height: 48px;
+        }
+
         .run_btn {
             background: black;
             color: white;
-            width: 88px;
-            height: 88px;
-            position: fixed;
-            bottom: 72px;
-            left: calc(50% - 44px);
-            border-radius: 100%;
+            /* width: calc(50% - 100px); */
+            min-width: 200px;
+            max-width: 460px;
+            height: 48px;
+            border-radius: 8px;
             cursor: pointer;
             border: 3px solid;
-            z-index: 100;
         }
 
         button:hover {
@@ -474,15 +492,14 @@
             <a class="link" href="https://www.mixcomfy.com" target="_blank">ComfyUI中文爱好者社区推荐</a>
         </div>
 
-        <a target="_blank" id="login_btn" href="https://www.mixcomfy.com/blog/" style="text-decoration: none;
+        <a id="login_btn" target="_blank" href="https://discord.gg/xbP2GZF6gn" style="text-decoration: none;
         color: black;font-size:12px">
             <svg height="32" aria-hidden="true" viewBox="0 0 16 16" version="1.1" width="32" data-view-component="true"
                 class="octicon octicon-mark-github v-align-middle color-fg-default">
                 <path
                     d="M8 0c4.42 0 8 3.58 8 8a8.013 8.013 0 0 1-5.45 7.59c-.4.08-.55-.17-.55-.38 0-.27.01-1.13.01-2.2 0-.75-.25-1.23-.54-1.48 1.78-.2 3.65-.88 3.65-3.95 0-.88-.31-1.59-.82-2.15.08-.2.36-1.02-.08-2.12 0 0-.67-.22-2.2.82-.64-.18-1.32-.27-2-.27-.68 0-1.36.09-2 .27-1.53-1.03-2.2-.82-2.2-.82-.44 1.1-.16 1.92-.08 2.12-.51.56-.82 1.28-.82 2.15 0 3.06 1.86 3.75 3.64 3.95-.23.2-.44.55-.51 1.07-.46.21-1.61.55-2.33-.66-.15-.24-.6-.83-1.23-.82-.67.01-.27.38.01.53.34.19.73.9.82 1.13.16.45.68 1.31 2.69.94 0 .67.01 1.3.01 1.49 0 .21-.15.45-.55.38A7.995 7.995 0 0 1 0 8c0-4.42 3.58-8 8-8Z">
                 </path>
-            </svg> Community</a>
-
+            </svg> HELP/帮助</a>
 
     </div>
     <a id="author"></a>
@@ -680,11 +697,89 @@
         }
 
 
+        // 获取 rembg 模型
+        async function get_rembg_models() {
+            try {
+                const response = await fetch(`${get_url()}/mixlab/folder_paths`, {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json'
+                    },
+                    body: JSON.stringify({
+                        type: 'rembg'
+                    })
+                })
+
+                const data = await response.json()
+                // console.log(data)
+                return data.names
+            } catch (error) {
+                console.error(error)
+            }
+        }
+
+        //自动抠图
+        async function run_rembg(model, base64) {
+            try {
+                const response = await fetch(`${get_url()}/mixlab/rembg`, {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json'
+                    },
+                    body: JSON.stringify({
+                        model,
+                        base64
+                    })
+                })
+
+                const data = await response.json()
+                // console.log(data)
+                return data.data
+            } catch (error) {
+                console.error(error)
+            }
+        }
+
+        function convertImageToBlackBasedOnAlpha(image) {
+            const canvas = document.createElement('canvas');
+            const ctx = canvas.getContext('2d');
+
+            // Draw the image onto the canvas
+            canvas.width = image.width;
+            canvas.height = image.height;
+            ctx.drawImage(image, 0, 0);
+
+            // Get the image data from the canvas
+            const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
+            const pixels = imageData.data;
+
+            // Modify the RGB values based on the alpha channel
+            for (let i = 0; i < pixels.length; i += 4) {
+                const alpha = pixels[i + 3];
+                if (alpha !== 0) {
+                    // Set non-transparent pixels to black
+                    pixels[i] = 0; // Red
+                    pixels[i + 1] = 0; // Green
+                    pixels[i + 2] = 0; // Blue
+                }
+            }
+
+            // Put the modified image data back onto the canvas
+            ctx.putImageData(imageData, 0, 0);
+
+            // Convert the modified canvas to base64 data URL
+            const base64ImageData = canvas.toDataURL('image/png'); // Replace 'png' with your desired image format
+
+            return base64ImageData;
+        }
+
+
+
         // 图像编辑
         async function editImage(image, data) {
             //判断mask是否有输出
             let isMask = data.options.hasMask;
-            console.log(data)
+
             //app
             document.body.querySelector('.app').style.display = 'none'
             document.body.querySelector('#author').style.display = 'none'
@@ -693,21 +788,139 @@
             editor.style.display = 'block';
             const iframe = editor.querySelector('iframe');
 
+            const sleep = (t = 1000) => {
+                return new Promise((res, rej) => {
+                    setTimeout(() => {
+                        res(true)
+                    }, t)
+                })
+            }
+
+            //清空 图层
+            const removeAllLayer = () => {
+                let Layers = iframe.contentWindow.Layers;
+                //清空
+                Layers.reset_layers()
+                Layers.refresh_gui()
+            }
+
+            // 复原
+            const resetLayer = () => {
+                let Layers = iframe.contentWindow.Layers;
+                for (const layer of Layers.get_layers()) {
+                    layer.visible = true;
+                }
+                Layers.refresh_gui()
+            }
+            //取image
+            const getImageBase64FromLayer = () => {
+                let Layers = iframe.contentWindow.Layers;
+                let tempCanvas = document.createElement("canvas");
+                let tempCtx = tempCanvas.getContext("2d");
+                let dim = Layers.get_dimensions();
+                tempCanvas.width = dim.width;
+                tempCanvas.height = dim.height;
+                for (const layer of Layers.get_layers()) {
+                    if (layer.name === 'Image_' + data.id) {
+                        layer.visible = true;
+                    } else {
+                        layer.visible = false;
+                    }
+                }
+                Layers.refresh_gui()
+
+                Layers.convert_layers_to_canvas(tempCtx);
+                return tempCanvas.toDataURL()
+            }
+
+            //add mask
+            const addMask = (id, name, image) => {
+                let Layers = iframe.contentWindow.Layers;
+                var new_mask_layer = {
+                    id,
+                    name,
+                    type: 'brush',
+                    data: [],
+                    render_function: ['brush', 'render'],
+                    width: image.naturalWidth || image.width,
+                    height: image.naturalHeight || image.height,
+
+                };
+                Layers.insert(new_mask_layer);
+            }
+
+            //add image
+            const addImage = (id, name, image) => {
+                let Layers = iframe.contentWindow.Layers;
+                var new_mask_layer = {
+                    id,
+                    name,
+                    type: 'image',
+                    data: image,
+                    width: image.naturalWidth || image.width,
+                    height: image.naturalHeight || image.height,
+                    width_original: image.naturalWidth || image.width,
+                    height_original: image.naturalHeight || image.height,
+                };
+                Layers.insert(new_mask_layer);
+            }
+
             //默认的画笔size设置大
             // let inputSize = (iframe.contentDocument.getElementById('size')).querySelector('input');
             // inputSize.value=50;
             // (iframe.contentDocument.getElementById('size')).querySelector('.increase_number').click()
 
+            //自动抠图
+            let autoMaskSelect = iframe.contentDocument.getElementById('automask_image_mixlab');
+            const select = iframe.contentDocument.getElementById('automask_models_mixlab');
+            if (select.children.length === 0) {
+                let rembgModels = await get_rembg_models()
+                // 遍历模型列表并创建选项
+                for (const model of rembgModels) {
+                    const option = document.createElement('option');
+                    option.value = model;
+                    option.textContent = model;
+                    select.appendChild(option);
+                }
+            }
 
+            let autoMaskBtn = iframe.contentDocument.getElementById('automask_image_mixlab');
+            if (!autoMaskBtn.getAttribute('init')) autoMaskBtn.addEventListener('click', async e => {
+                //api请求
+                let base64 = getImageBase64FromLayer()
+                resetLayer()
+
+                let res = await run_rembg(select.value, base64)
+                const match = res.match(/^data:image\/(\w+);base64,/);
+                if (!match) {
+                    res = 'data:image/png;base64,' + res
+                }
+
+                let image = await createImage(res)
+
+                let mb = convertImageToBlackBasedOnAlpha(image)
+                let mask = await createImage(mb)
+                let id = Layers.auto_increment;
+                addImage(id, 'Mask_' + data.id + id, mask)
+            })
+
+            autoMaskBtn.setAttribute('init', 1)
             let cancelImageBtn = iframe.contentDocument.getElementById('cancel_image_mixlab');
-            cancelImageBtn.addEventListener('click', e => {
+            if (!cancelImageBtn.getAttribute('init')) cancelImageBtn.addEventListener('click', e => {
                 editor.style.display = 'none';
                 document.body.querySelector('.app').style.display = 'flex'
                 document.body.querySelector('#author').style.display = 'block'
             })
+
+            cancelImageBtn.setAttribute('init', 1)
             // 获取 id 为 "mix" 的 button 元素
             let saveImageBtn = iframe.contentDocument.getElementById('save_image_mixlab');
-            saveImageBtn.addEventListener('click', async e => {
+            saveImageBtn.style = `width: 98px;
+            height: 36px;
+            margin: 0 12px;
+            background-color: var(--background-color-active);
+            color: var(--text-color-active);`
+            if (!saveImageBtn.getAttribute('init')) saveImageBtn.addEventListener('click', async e => {
                 //保存，并更新图片
                 e.preventDefault();
 
@@ -721,6 +934,7 @@
                     tempCanvas.width = dim.width;
                     tempCanvas.height = dim.height;
 
+                    //todo 获取Image更新后的数据(暂不支持image的修改)
 
                     for (const layer of Layers.get_layers()) {
                         if (layer.name !== 'Image_' + data.id) {
@@ -734,13 +948,7 @@
                     Layers.convert_layers_to_canvas(tempCtx);
 
                     // 复原
-                    for (const layer of Layers.get_layers()) {
-                        layer.visible = true;
-                    }
-                    Layers.refresh_gui()
-
-
-
+                    resetLayer()
 
                     // 获取图像数据
                     const imageData = tempCtx.getImageData(0, 0, dim.width, dim.height);
@@ -766,10 +974,15 @@
                     // // 获取读取的文件内容，即 Blob 对象
                     let hashId = await calculateImageHash(fileBlob)
 
-                    if (hashId == window._appData.data[data.id].hashId) return;
+                    if (hashId == window._appData.data[data.id].hashId) {
+                        document.body.querySelector('.app').style.display = 'flex'
+                        document.body.querySelector('#author').style.display = 'block'
+                        return;
+                    }
 
+                    //底图
                     const { url: imgurl } = await uploadImage(base64ToBlob(data.options.defaultImage))
-
+                    //mask
                     let { url, name } = await uploadMask(fileBlob, imgurl);
                     // 在这里可以对 Blob 对象进行进一步处理
                     // imageElement.src = url;
@@ -777,31 +990,22 @@
                     window._appData.data[data.id].hashId = hashId;
 
                     // console.log("上传的文件：", url, data.id, name);
-
-
                     //更新图片
-
                     const canvas = document.createElement("canvas");
                     canvas.width = dim.width;
                     canvas.height = dim.height;
 
                     const ctx = canvas.getContext('2d');
 
-                    const defaultImage = new Image();
-                    defaultImage.src = data.options.defaultImage;
-                    defaultImage.onload = function () {
-                        ctx.drawImage(defaultImage, 0, 0, dim.width, dim.height);
+                    const defaultImage = await createImage(data.options.defaultImage)
+                    ctx.drawImage(defaultImage, 0, 0, dim.width, dim.height);
 
-                        // 绘制base64图片
-                        const base64Image = base64
-                        const base64ImageObj = new Image();
-                        base64ImageObj.onload = function () {
-                            ctx.globalCompositeOperation = 'destination-in';
-                            ctx.drawImage(base64ImageObj, 0, 0, dim.width, dim.height);
-                            image.src = canvas.toDataURL();
-                        };
-                        base64ImageObj.src = base64;
-                    };
+                    // 绘制base64图片
+                    // const base64Image = base64
+                    const base64ImageObj = await createImage(base64)
+                    ctx.globalCompositeOperation = 'destination-in';
+                    ctx.drawImage(base64ImageObj, 0, 0, dim.width, dim.height);
+                    image.src = canvas.toDataURL();
 
                 }
 
@@ -809,46 +1013,34 @@
                 document.body.querySelector('#author').style.display = 'block'
             })
 
+            saveImageBtn.setAttribute('init', 1)
 
             var Layers = iframe.contentWindow.Layers;
             // console.log(Layers)
 
             //判断是否已经存在
-            let layers1 = Layers.get_layers()
+            let layers1 = Layers.get_layers();
 
-            if (!layers1.filter(l => l.name == 'Image_' + data.id)[0]) {
-                var new_layer = {
-                    id: (Layers.get_layers()).length,
-                    name: 'Image_' + data.id,
-                    type: 'image',
-                    data: image,
-                    width: image.naturalWidth || image.width,
-                    height: image.naturalHeight || image.height,
-                    width_original: image.naturalWidth || image.width,
-                    height_original: image.naturalHeight || image.height,
-                };
-
-                Layers.insert(new_layer);
+            //通过layer.link.src 判断是否图片更新
+            console.log(layers1.filter(l => l.name == 'Image_' + data.id)[0]?.link?.currentSrc !== data.options.defaultImage)
+            if (layers1.filter(l => l.name == 'Image_' + data.id)[0]
+                && layers1.filter(l => l.name == 'Image_' + data.id)[0].link.src !== data.options.defaultImage) {
+                //清空图层
+                removeAllLayer();
+                layers1 = [];
+                await sleep()
             }
 
-
+            let im = await createImage(data.options.defaultImage)
+            // console.log(layers1, layers1.length)
+            if (!layers1.filter(l => l.name == 'Image_' + data.id)[0]) {
+                addImage(0, 'Image_' + data.id, im)
+            }
 
             if (isMask) {
                 if (!layers1.filter(l => l.name == 'Mask_' + data.id)[0]) {
-                    var new_mask_layer = {
-                        id: (Layers.get_layers()).length,
-                        name: 'Mask_' + data.id,
-                        type: 'brush',
-                        data: [],
-                        render_function: ['brush', 'render'],
-                        width: image.naturalWidth || image.width,
-                        height: image.naturalHeight || image.height,
-
-                    };
-
-                    Layers.insert(new_mask_layer);
+                    addMask(1, 'Mask_' + data.id, im)
                 }
-
             }
 
         }
@@ -1041,7 +1233,7 @@
         }
 
 
-        function createOutputs(outputData, link) {
+        async function createOutputs(outputData, link) {
             const url = new URL(window.location.href);
             const params = new URLSearchParams(url.search);
             const innerApp = params.get("innerApp");
@@ -1162,20 +1354,21 @@
                     "TransparentImage"].includes(node.class_type)) {
 
                     const url = node.options?.defaultImage || window._appData?.icon || base64Df;
-                    let a = document.createElement('a');
+                    let a = document.createElement('div');
                     a.id = `output_${node.id}`
-                    a.setAttribute('data-pswp-width', "200");
-                    a.setAttribute('data-pswp-height', "200");
-                    a.setAttribute('target', "_blank");
-                    a.setAttribute('href', url);
-                    a.setAttribute('title', node.title);
 
-                    let img = new Image();
-                    // img;
-                    img.src = url;
+                    let img = await createImage(url)
+
                     a.appendChild(img)
+                    // a.setAttribute('data-pswp-width', img.naturalWidth);
+                    // a.setAttribute('data-pswp-height', img.naturalHeight);
+                    // a.setAttribute('target', "_blank");
+                    // a.setAttribute('href', url);
+                    // a.setAttribute('title', node.title);
+
                     output_card.appendChild(a);
                     isShowImageFn = true;
+
                 }
 
                 //3d 
@@ -1195,7 +1388,7 @@
                 }
 
                 // video ,gif
-                if (["VHS_VideoCombine", "VideoCombine_Adv"].includes(node.class_type)) {
+                if (["VHS_VideoCombine", "VideoCombine_Adv", "CombineAudioVideo"].includes(node.class_type)) {
 
                     let a = document.createElement('a');
                     a.id = `output_${node.id}`
@@ -1297,8 +1490,9 @@
 
                             if (hashId == window._appData.data[data.id].hashId) return
 
+                            let base64 = await blobToBase64(fileBlob)
+
                             if (data.class_type === 'LoadImagesToBatch') {
-                                let base64 = await blobToBase64(fileBlob)
                                 createBase64ImageForLoadImageToBatch(imageElement, data.id, base64)
                             } else {
                                 let { url, name } = await uploadImage(fileBlob);
@@ -1307,6 +1501,14 @@
                                 window._appData.data[data.id].inputs.image = name;
                                 window._appData.data[data.id].hashId = hashId;
                                 console.log("上传的文件：", url, data.id, name);
+
+                                //更换option里的default image
+                                window._appData.input = Array.from(window._appData.input, inp => {
+                                    if (inp.id === data.id) {
+                                        inp.options.defaultImage = base64;
+                                    }
+                                    return inp
+                                })
                             }
                         }
                     }
@@ -1443,6 +1645,42 @@
         // copyImagesToClipboard(htmlString);
 
 
+        // 上传音频转为base64
+        async function uploadAndConvertAudio(file) {
+            if (!file) {
+                alert('Please select a WAV file.')
+                return
+            }
+
+            if (file.type !== 'audio/wav') {
+                alert('Only WAV files are supported.')
+                return
+            }
+
+            try {
+                const base64Audio = await readFileAsDataURL(file)
+                return base64Audio
+            } catch (error) {
+                console.error('Error reading file:', error)
+                alert('Error reading file.')
+            }
+        }
+
+        function readFileAsDataURL(file) {
+            return new Promise((resolve, reject) => {
+                const reader = new FileReader()
+
+                reader.onload = function (event) {
+                    resolve(event.target.result)
+                }
+
+                reader.onerror = function (error) {
+                    reject(error)
+                }
+
+                reader.readAsDataURL(file)
+            })
+        }
 
         function createInputs(inputData) {
             // Assuming you have an HTML element with the id "container" to hold the UI
@@ -1608,9 +1846,9 @@
                                 //上传，返回url
                                 let { url, name } = await uploadImage(fileBlob, '.' + file.type.split('/')[1])
 
+                                let base64 = await parseImageToBase64(url);
+
                                 if (data.class_type === 'ImagesPrompt_') {
-                                    // 
-                                    let base64 = await parseImageToBase64(url);
                                     uploadContainer.querySelector('.images_prompt_main').src = base64
                                     window._appData.data[data.id].inputs.image_base64 = base64;
                                 } else {
@@ -1620,10 +1858,17 @@
                                     // 在这里可以对 Blob 对象进行进一步处理
                                     imageElement.src = url;
 
-
                                     if (isVideoUpload) {
                                         window._appData.data[data.id].inputs.video = name;
                                     } else {
+                                        //更换option里的default image
+                                        window._appData.input = Array.from(window._appData.input, inp => {
+                                            if (inp.id === data.id) {
+                                                inp.options.defaultImage = base64;
+                                            }
+                                            return inp
+                                        })
+
                                         window._appData.data[data.id].inputs.image = name;
                                     }
 
@@ -1882,7 +2127,53 @@
                     container.appendChild(d);
                 }
 
+                // 加载音频 base64
+                if (['LoadAndCombinedAudio_'].includes(data.class_type)) {
+                    let inpAudio = document.createElement('div');
 
+                    let inputAudio = document.createElement('button');
+
+                    inputAudio.innerText = data.title || 'Upload Audio'
+                    inputAudio.style = `cursor: pointer;
+                        font-weight: 300;
+                        margin: 2px; 
+                        color: var(--descrip-text);
+                        background-color: var(--comfy-input-bg);
+                        border-radius: 8px;
+                        border-color: var(--border-color);
+                        border-style: solid;height: 30px;min-width: 122px;
+                    `;
+
+                    let audioE = document.createElement('audio');
+                    audioE.setAttribute('controls', 'on')
+                    //暂时只支持一个
+                    audioE.src = data.inputs.audios.base64[0];
+
+                    inputAudio.addEventListener('click', e => {
+                        e.preventDefault()
+                        let inp = document.createElement('input')
+                        inp.type = 'file'
+                        inp.style.display = 'none'
+                        inp.addEventListener('change', async e => {
+                            e.preventDefault()
+                            const file = e.target.files[0]
+                            let base64 = await uploadAndConvertAudio(file)
+
+                            audioE.src = base64;
+
+                            window._appData.data[data.id].inputs.audios.base64 = [base64];
+                        })
+
+                        inp.click()
+                        inp.remove()
+                    })
+
+                    inpAudio.appendChild(inputAudio)
+                    inpAudio.appendChild(audioE)
+
+                    container.appendChild(inpAudio);
+
+                }
             });
             return container
         }
@@ -2207,7 +2498,7 @@
             })
         }
 
-        function createUI(data, share = true) {
+        async function createUI(data, share = true) {
             // appData.input, appData.output, appData.seed, share, appData.link
             if (!data) return
             const { input: inputData, output: outputData, data: workflow, seed, seedTitle, link, name } = data;
@@ -2348,30 +2639,44 @@
             }
 
 
-            statusDiv.appendChild(status);
+            // statusDiv.appendChild(status);
             statusDiv.appendChild(seeds);
 
             // 创建输入框
             var input1 = createInputs(inputData)
 
-            var output = createOutputs(outputData, link)
+            var output = await createOutputs(outputData, link)
 
             // 创建提交按钮
+            let submitDiv = document.createElement('div');
+            submitDiv.appendChild(statusDiv);
+
+            let submitDivBtn = document.createElement('div');
+            submitDivBtn.style.display = 'flex'
+            submitDiv.appendChild(submitDivBtn);
+
+            submitDivBtn.appendChild(status);
+
+            submitDiv.className = 'run_div';
             var submitButton = document.createElement('button');
             submitButton.textContent = 'Create';
             submitButton.className = 'run_btn'
 
             // 将所有UI元素添加到页面中
-            leftDiv.appendChild(titleDiv);
-            leftDiv.appendChild(iconDes);
+            // leftDiv.appendChild(titleDiv);
+            // leftDiv.appendChild(iconDes);
             // leftDiv.appendChild(des);
-            leftDiv.appendChild(statusDiv);
+            // leftDiv.appendChild(statusDiv);
             leftDiv.appendChild(input1);
 
-            if (typeof (data.data) == 'object') mainDiv.appendChild(submitButton);
+            submitDivBtn.appendChild(submitButton);
+            if (typeof (data.data) == 'object') mainDiv.appendChild(submitDiv);
 
             rightDiv.appendChild(output);
 
+            // mainDiv.appendChild(titleDiv);
+            // mainDiv.appendChild(iconDes);
+
             mainDiv.appendChild(leftDetails);
             mainDiv.appendChild(rightDiv);
 
@@ -2719,7 +3024,7 @@
         async function createApp(appData, share = true) {
             // console.log(appData)
             // 使用示例：
-            var ui = createUI(appData, share);
+            var ui = await createUI(appData, share);
 
             // 更新标题
             ui.title.update(appData.name || 'Mixlab APP');
@@ -2859,7 +3164,8 @@
             const lightbox = new PhotoSwipeLightbox({
                 gallery: '.output_card',
                 children: 'a',
-                pswpModule: () => import('/extensions/comfyui-mixlab-nodes/lib/photoswipe.esm.min.js')
+                pswpModule: () => import('/extensions/comfyui-mixlab-nodes/lib/photoswipe.esm.min.js'),
+                // appendToEl: document.querySelector('#result')
             });
             lightbox.on('uiRegister', function () {
                 lightbox.pswp.ui.registerElement({
@@ -2904,8 +3210,8 @@
             if (appData.author) {
                 let div = document.body.querySelector('#author');
                 if (appData.author.link) div.href = appData.author.link
-                div.style = `z-index:20;display: flex;flex-direction: column;position: fixed;bottom: 12px;right: 24px;cursor: pointer;text-decoration: none;color: black;`
-                div.innerHTML = `<p style="font-size:12px">Author:</p>
+                div.style = `z-index:20;display: flex;flex-direction: column;position: fixed;bottom: 24px;left: 24px;cursor: pointer;text-decoration: none;color: black;`
+                div.innerHTML = `<p style="font-size:12px;margin: 8px 0;">Author:</p>
                 <div style="display: flex;"> <img style="width:32px;height:32px;border-radius: 100%;" 
                 src="${appData.author.avatar || base64Df}"/>
                 <p style="margin-left:8px;font-size:12px;font-weight:800">${appData.author.name || '-'}</p></div>`
@@ -3154,7 +3460,7 @@
 
                 createAppList(window._apps);
 
-                if (window._apps.length > 0) await createHistoryList(category || '');
+                // if (window._apps.length > 0) await createHistoryList(category || '');
 
                 createApp(window._appData);
             }

web/javascript/app_mixlab.js

@@ -2,6 +2,8 @@ import { app } from '../../../scripts/app.js'
 import { $el } from '../../../scripts/ui.js'
 import { api } from '../../../scripts/api.js'
 
+import { td_bg } from './td_background.js'
+console.log('td_bg', td_bg)
 //本机安装的插件节点全集
 window._nodesAll = null
 
@@ -185,6 +187,25 @@ async function extractInputAndOutputData (
         if (node.type == 'Color') {
         }
 
+        // 语音输入的支持
+        if (node.type == 'LoadAndCombinedAudio_') {
+          // if (
+          //   data[id].widgets_values &&
+          //   data[id].widgets_values[0] &&
+          //   data[id].widgets_values[0].base64 &&
+          //   data[id].widgets_values[0].base64.length > 0
+          // ) {
+          //   options.defaultBase64 = data[id].widgets_values[0].base64
+          // }
+
+          input[inputIds.indexOf(id)] = {
+            ...data[id],
+            title: node.title,
+            id,
+            options
+          }
+        }
+
         if (node.type === 'LoadImage') {
           // loadImage的mask支持
           let output = node.outputs.filter(ot => ot.type == 'MASK')[0]
@@ -397,11 +418,11 @@ async function save (json, download = false, showInfo = true) {
 
 function getInputsAndOutputs () {
   const inputs =
-      `LoadImage LoadImagesToBatch ImagesPrompt_ VHS_LoadVideo CLIPTextEncode PromptSlide TextInput_ Color FloatSlider IntNumber CheckpointLoaderSimple LoraLoader`.split(
+      `LoadImage LoadImagesToBatch ImagesPrompt_ LoadAndCombinedAudio_ LoadVideoAndSegment_ VHS_LoadVideo CLIPTextEncode PromptSlide TextInput_ Color FloatSlider IntNumber CheckpointLoaderSimple LoraLoader`.split(
         ' '
       ),
     outputs =
-      `SaveTripoSRMesh,PreviewImage,SaveImage,TransparentImage,ShowTextForGPT,VHS_VideoCombine,VideoCombine_Adv,Image Save,SaveImageAndMetadata_,ClipInterrogator`.split(
+      `SaveTripoSRMesh,PreviewImage,SaveImage,TransparentImage,ShowTextForGPT,CombineAudioVideo,VHS_VideoCombine,VideoCombine_Adv,Image Save,SaveImageAndMetadata_,ClipInterrogator`.split(
         ','
       )
 
@@ -503,6 +524,21 @@ app.registerExtension({
           }
         })
 
+        //td bg
+        const tdBG = document.createElement('button')
+        tdBG.innerText = 'Canvas Mode'
+        tdBG.style = style
+        tdBG.style.marginLeft = '12px'
+
+        tdBG.addEventListener('click', () => {
+          td_bg.toggle()
+          if (td_bg.running) {
+            tdBG.style.background = 'yellow'
+          } else {
+            tdBG.style.background = 'transparent'
+          }
+        })
+
         // author
         let author = document.createElement('div')
         // author.style=`display: flex`
@@ -659,6 +695,7 @@ app.registerExtension({
 
         btns.appendChild(btn)
         btns.appendChild(download)
+        btns.appendChild(tdBG)
 
         document.body.appendChild(widget.div)
         this.addCustomWidget(widget)
@@ -672,6 +709,7 @@ app.registerExtension({
         this.serialize_widgets = true //需要保存参数
 
         window._mixlab_app_json = null
+ 
       }
 
       const onExecuted = nodeType.prototype.onExecuted
@@ -687,9 +725,8 @@ app.registerExtension({
           }
 
           const div = this.widgets.filter(w => w.div)[0].div
-          Array.from(
-            div.querySelectorAll('button'),
-            b => (b.style.background = 'yellow')
+          Array.from(div.querySelectorAll('button'), b =>
+            b.innerText != 'Canvas Mode' ? (b.style.background = 'yellow') : ''
           )
         } catch (error) {}
       }

web/javascript/audio_mixlab.js

@@ -396,3 +396,217 @@ app.registerExtension({
     }
   }
 })
+
+// 上传音频转为base64
+async function uploadAndConvertAudio (file) {
+  if (!file) {
+    alert('Please select a WAV file.')
+    return
+  }
+
+  if (file.type !== 'audio/wav') {
+    alert('Only WAV files are supported.')
+    return
+  }
+
+  try {
+    const base64Audio = await readFileAsDataURL(file)
+    return base64Audio
+  } catch (error) {
+    console.error('Error reading file:', error)
+    alert('Error reading file.')
+  }
+}
+
+function readFileAsDataURL (file) {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader()
+
+    reader.onload = function (event) {
+      resolve(event.target.result)
+    }
+
+    reader.onerror = function (error) {
+      reject(error)
+    }
+
+    reader.readAsDataURL(file)
+  })
+}
+
+const createInputAudioForBatch = (base64, widget) => {
+  // Create an audio element
+  let audio = document.createElement('audio')
+  audio.src = base64
+  audio.controls = true
+  audio.style = 'width: 120px; display: block'
+
+  // Create a delete button
+  let deleteButton = document.createElement('button')
+  deleteButton.textContent = 'Delete'
+
+  deleteButton.style = `cursor: pointer;
+        font-weight: 300;
+        margin: 2px; 
+        margin-left: 10px;
+        color: var(--descrip-text);
+        background-color: var(--comfy-input-bg);
+        border-radius: 8px;
+        border-color: var(--border-color);
+        border-style: solid;height: 30px;min-width: 122px;
+       `
+
+  // Create a container for the audio and delete button
+  let container = document.createElement('div')
+  container.appendChild(audio)
+  container.appendChild(deleteButton)
+  container.style = `display: flex;margin-top: 12px;`
+
+  // Add event listener for the delete button
+  deleteButton.addEventListener('click', e => {
+    let newValue = []
+    let items = widget.value?.base64 || []
+    for (const v of items) {
+      if (v != base64) newValue.push(v)
+    }
+    widget.value.base64 = newValue
+    container.remove()
+  })
+
+  return container
+}
+
+app.registerExtension({
+  name: 'Mixlab.Comfy.LoadAndCombinedAudio_',
+  async getCustomWidgets (app) {
+    return {
+      AUDIOBASE64 (node, inputName, inputData, app) {
+        // console.log('##node', node)
+        const widget = {
+          value: {
+            base64: []
+          }, // 不能[x,x,x]
+          type: inputData[0], // the type
+          name: inputName, // the name, slice
+          size: [128, 32], // a default size
+          draw (ctx, node, width, y) {},
+          computeSize (...args) {
+            return [128, 122] // a method to compute the current size of the widget
+          }
+          // serializeValue (nodeId, widgetIndex) {
+          //   return widget.value
+          // },
+        }
+        //  widget.something = something;          // maybe adds stuff to it
+        node.addCustomWidget(widget) // adds it to the node
+        return widget // and returns it.
+      }
+    }
+  },
+
+  async beforeRegisterNodeDef (nodeType, nodeData, app) {
+    if (nodeType.comfyClass == 'LoadAndCombinedAudio_') {
+      const orig_nodeCreated = nodeType.prototype.onNodeCreated
+
+      nodeType.prototype.onNodeCreated = function () {
+        orig_nodeCreated?.apply(this, arguments)
+
+        let audiosWidget = this.widgets.filter(w => w.name == 'audios')[0]
+
+        const widget = {
+          type: 'div',
+          name: 'audio_base64',
+          draw (ctx, node, widget_width, y, widget_height) {
+            Object.assign(
+              this.div.style,
+              get_position_style(ctx, widget_width, 44, node.size[1])
+            )
+          },
+          serialize: false
+        }
+
+        widget.div = $el('div', {})
+
+        document.body.appendChild(widget.div)
+
+        let audioPreview = document.createElement('div')
+        let audiosDiv = document.createElement('div') //显示图片
+        audiosDiv.className = 'audios_preview'
+        audiosDiv.style = `width: calc(100% - 14px);
+        display: flex;
+        flex-wrap: wrap;
+        padding: 7px;    justify-content: space-between;
+        align-items: center;`
+
+        const btn = document.createElement('button')
+        btn.innerText = 'Upload Audio'
+
+        btn.style = `cursor: pointer;
+        font-weight: 300;
+        margin: 2px; 
+        color: var(--descrip-text);
+        background-color: var(--comfy-input-bg);
+        border-radius: 8px;
+        border-color: var(--border-color);
+        border-style: solid;height: 30px;min-width: 122px;
+       `
+
+        btn.addEventListener('click', e => {
+          e.preventDefault()
+          let inputAudio = document.createElement('input')
+          inputAudio.type = 'file'
+          inputAudio.style.display = 'none'
+          inputAudio.addEventListener('change', async e => {
+            e.preventDefault()
+            const file = e.target.files[0]
+            let base64 = await uploadAndConvertAudio(file)
+            if (!audiosWidget.value) audiosWidget.value = { base64: [] }
+            audiosWidget.value.base64.push(base64)
+
+            let a = createInputAudioForBatch(base64, audiosWidget)
+            audiosDiv.appendChild(a)
+          })
+
+          inputAudio.click()
+          inputAudio.remove()
+        })
+
+        widget.div.appendChild(audioPreview)
+        audioPreview.appendChild(audiosDiv)
+        audioPreview.appendChild(btn)
+        // audioPreview.appendChild(inputAudio)
+
+        this.addCustomWidget(widget)
+
+        // document.addEventListener('wheel', handleMouseWheel)
+
+        const onRemoved = this.onRemoved
+        this.onRemoved = () => {
+          widget.div.remove()
+          try {
+            // document.removeEventListener('wheel', handleMouseWheel)
+          } catch (error) {
+            console.log(error)
+          }
+
+          return onRemoved?.()
+        }
+
+        this.serialize_widgets = true //需要保存参数
+      }
+    }
+  },
+  async loadedGraphNode (node, app) {
+    if (node.type === 'LoadAndCombinedAudio_') {
+      // await sleep(0)
+      let audiosWidget = node.widgets.filter(w => w.name === 'audios')[0]
+      let audioPreview = node.widgets.filter(w => w.name == 'audio_base64')[0]
+
+      let pre = audioPreview.div.querySelector('.audios_preview')
+      for (const d of audiosWidget.value?.base64 || []) {
+        let im = createInputAudioForBatch(d, audiosWidget)
+        pre.appendChild(im)
+      }
+    }
+  }
+})

web/javascript/chat.js

@@ -2,7 +2,7 @@ async function* completion (url, messages, controller) {
   let data = {
     model: 'gpt-3.5-turbo-16k',
     messages,
-    temperature: 0.6,
+    temperature: 0.05,
     stream: true
   }
   // if (imageNode) {

web/javascript/checkVersion_mixlab.js

@@ -3,7 +3,7 @@ import { app } from '../../../scripts/app.js'
 const repoOwner = 'shadowcz007' // 替换为仓库的所有者
 const repoName = 'comfyui-mixlab-nodes' // 替换为仓库的名称
 
-const version = 'v0.27.1'
+const version = 'v0.30.2'
 
 fetch(`https://api.github.com/repos/${repoOwner}/${repoName}/releases/latest`)
   .then(response => response.json())

web/javascript/gpt_mixlab.js

@@ -209,13 +209,16 @@ app.registerExtension({
         text = text.filter(t => t && t?.trim())
 
         if (this.widgets) {
+          // console.log('#ShowTextForGPT',this.widgets)
           // const pos = this.widgets.findIndex(w => w.name === 'text')
           for (let i = 0; i < this.widgets.length; i++) {
-            if (this.widgets[i].name == 'show_text') this.widgets[i].onRemove?.()
+            if (this.widgets[i].name == 'show_text')
+              this.widgets[i].onRemove?.()
+            console.log('#ShowTextForGPT', this.widgets[i])
           }
-          this.widgets.length = 1
+          this.widgets.length = 2
         }
-        // console.log('ShowTextForGPT',text)
+
         for (let list of text) {
           if (list) {
             // console.log('#####', list)
@@ -228,6 +231,8 @@ app.registerExtension({
             w.inputEl.readOnly = true
             w.inputEl.style.opacity = 0.6
 
+            // w.inputEl.style.display='none'
+
             try {
               if (typeof list != 'string') {
                 let data = JSON.parse(list)
@@ -280,5 +285,24 @@ app.registerExtension({
 
       this.serialize_widgets = true //需要保存参数
     }
+  },
+  async loadedGraphNode (node, app) {
+    if (node.type === 'ShowTextForGPT') {
+      let widget = node.widgets.filter(w => w.name == 'show_text')[0]
+
+      // if (widget.value) {
+      //   let [url, prompt] = widget.value
+
+      //   this[`wavesurfer_${node.id}`] = updateWaveWidgetValue(
+      //     node.widgets,
+      //     node.id,
+      //     url,
+      //     prompt,
+      //     this[`wavesurfer_${node.id}`]
+      //   )
+      // }
+
+      console.log('#loadedGraphNode', node)
+    }
   }
 })

web/javascript/main_mixlab.js

@@ -1267,7 +1267,7 @@ app.registerExtension({
         })
 
         widget.PictureInPicture = $el('button', {
-          innerText: 'PictureInPicture',
+          innerText: 'Picture In Picture',
           style: {
             display: 'pictureInPictureEnabled' in document ? 'block' : 'none',
             cursor: 'pointer',

web/javascript/sound-lab.js

@@ -0,0 +1,295 @@
+import { app } from '../../../scripts/app.js'
+import { api } from '../../../scripts/api.js'
+import { ComfyWidgets } from '../../../scripts/widgets.js'
+import { $el } from '../../../scripts/ui.js'
+
+import WaveSurfer from 'https://cdn.jsdelivr.net/npm/wavesurfer.js@7/dist/wavesurfer.esm.js'
+
+function get_position_style (ctx, widget_width, y, node_height) {
+  const MARGIN = 4 // the margin around the html element
+
+  /* Create a transform that deals with all the scrolling and zooming */
+  const elRect = ctx.canvas.getBoundingClientRect()
+  const transform = new DOMMatrix()
+    .scaleSelf(
+      elRect.width / ctx.canvas.width,
+      elRect.height / ctx.canvas.height
+    )
+    .multiplySelf(ctx.getTransform())
+    .translateSelf(MARGIN, MARGIN + y)
+
+  return {
+    transformOrigin: '0 0',
+    transform: transform,
+    left: `0`,
+    top: '0',
+    cursor: 'pointer',
+    position: 'absolute',
+    maxWidth: `${widget_width - MARGIN * 2}px`,
+    // maxHeight: `${node_height - MARGIN * 2}px`, // we're assuming we have the whole height of the node
+    width: `${widget_width - MARGIN * 2}px`,
+    // height: `${node_height * 0.3 - MARGIN * 2}px`,
+    // background: '#EEEEEE',
+    display: 'flex',
+    flexDirection: 'column',
+    // alignItems: 'center',
+    justifyContent: 'space-around'
+  }
+}
+
+//把文件转为url访问
+const parseUrl = data => {
+  let { filename, subfolder, type, prompt } = data
+  return {
+    url: api.apiURL(
+      `/view?filename=${encodeURIComponent(
+        filename
+      )}&type=${type}&subfolder=${subfolder}${app.getPreviewFormatParam()}${app.getRandParam()}`
+    ),
+    prompt
+  }
+}
+
+const createWaveSurfer = (wavesurfer, id,url) => {
+  // Create an instance of WaveSurfer
+  if (wavesurfer) {
+    wavesurfer.destroy()
+  }
+  wavesurfer = WaveSurfer.create({
+    container: '#' + id,
+    waveColor: 'rgb(200, 0, 200)',
+    progressColor: 'rgb(100, 0, 100)',
+    // Set a bar width
+    barWidth: 10,
+    // Optionally, specify the spacing between bars
+    barGap: 2,
+    // And the bar radius
+    barRadius: 6,
+    url
+  })
+
+  wavesurfer._auto = true
+
+  // 监听播放结束事件，重新开始播放以实现循环播放
+  wavesurfer.on('finish', function () {
+    // console.log(wavesurfer)
+    if (wavesurfer._auto) wavesurfer.play()
+  })
+
+  wavesurfer.on('interaction', () => {
+    wavesurfer._auto = false
+    if (!wavesurfer.isPlaying()) wavesurfer.play()
+  })
+
+  // 获取当前播放时间的峰值
+  wavesurfer.on('audioprocess', () => {
+    if (wavesurfer.isPlaying()&&wavesurfer.getDecodedData()) {
+      const channelData = wavesurfer.getDecodedData().getChannelData(0); 
+      const currentTime = wavesurfer.getCurrentTime()
+      // console.log(wavesurfer)
+      const sampleRate = wavesurfer.getDecodedData().sampleRate
+
+      // 定义要分析的时间窗口（例如1秒）
+      const windowSize = 1
+      const startSample = Math.floor(currentTime * sampleRate)
+      const endSample = Math.min(
+        startSample + windowSize * sampleRate,
+        channelData.length
+      )
+
+      let peak = 0
+      for (let i = startSample; i < endSample; i++) {
+        const value = Math.abs(channelData[i])
+        if (value > peak) {
+          peak = value
+        }
+      }
+      // console.log('Current Peak:', peak)
+    }
+  })
+
+  return wavesurfer
+}
+
+//更新gui
+function updateWaveWidgetValue (widgets, id, url, prompt, wavesurfer) {
+  let widget = widgets.filter(w => w.name == 'AudioPlay')[0]
+  // 手动更新widget值
+  widget.value = [url, prompt]
+
+  if (widget.div) {
+    widget.div.querySelector('.wave').id = `AudioPlay_${id}`
+  }
+
+  wavesurfer = createWaveSurfer(wavesurfer, `AudioPlay_${id}`,url)
+
+  wavesurfer.on('ready', duration => {
+    console.log('Audio duration: ' + duration + ' seconds')
+    if (widget.div) {
+      widget.div.setAttribute('data-url', url)
+      widget.div.querySelector('.link').setAttribute('href', url)
+      widget.div.querySelector(
+        '.info'
+      ).innerHTML = `<span style="font-size: 12px;
+      margin: 8px;">${duration.toFixed(
+        2
+      )} seconds</span> <br><span style="font-size: 14px;">${prompt||''}</span> <br>`
+    }
+  })
+  
+  
+  wavesurfer.load(url)
+  // console.log('updateWaveWidgetValue' ,url,wavesurfer)
+  return wavesurfer
+}
+
+app.registerExtension({
+  name: 'SoundLab.AudioPlay',
+  async beforeRegisterNodeDef (nodeType, nodeData, app) {
+    if (nodeType.comfyClass == 'AudioPlay') {
+      let that = this
+      // console.log('that', that)
+
+      const orig_nodeCreated = nodeType.prototype.onNodeCreated
+      nodeType.prototype.onNodeCreated = function () {
+        orig_nodeCreated?.apply(this, arguments)
+
+        const widget = {
+          type: 'div',
+          name: 'AudioPlay',
+          draw (ctx, node, widget_width, y, widget_height) {
+            Object.assign(
+              this.div.style,
+              get_position_style(ctx, widget_width, y, node.size[1])
+            )
+          }
+        }
+
+        // console.log('AudioPlay nodeData', this)
+        widget.div = $el('div', {})
+
+        document.body.appendChild(widget.div)
+
+        // wave
+        const waveDiv = document.createElement('div')
+        waveDiv.className = 'wave'
+        waveDiv.style.minHeight = '172px'
+        widget.div.appendChild(waveDiv)
+
+        //prompt 相关信息展示
+        const infoDiv = document.createElement('div')
+        infoDiv.className = 'info'
+        infoDiv.style.marginBottom = '20px'
+        widget.div.appendChild(infoDiv)
+
+        // 按钮的区域
+        let btns = document.createElement('div')
+        btns.className = 'btns'
+        btns.style = `display: flex;
+        width: 100%;
+        justify-content: space-between;`
+        widget.div.appendChild(btns)
+
+        //play button
+        const playBtn = document.createElement('a')
+        playBtn.innerText = 'Play/Pause'
+
+        playBtn.style = `
+        display: flex;
+        padding: 4px 15px;
+        background-color: var(--comfy-input-bg);
+        border-radius: 8px;
+        border-color: var(--border-color);
+        border-style: solid;
+        color: var(--descrip-text);
+        text-decoration: none;
+        border-radius: 5px;
+        transition: background-color 0.3s ease 0s;
+          `
+
+        playBtn.addEventListener('click', e => {
+          e.preventDefault()
+          if (that[`wavesurfer_${this.id}`]) {
+            that[`wavesurfer_${this.id}`]?.playPause()
+            that[`wavesurfer_${this.id}`]._auto = true
+          }
+        })
+        btns.appendChild(playBtn)
+
+        const urlLink = document.createElement('a')
+        urlLink.className = 'link'
+        urlLink.innerText = 'URL'
+        urlLink.setAttribute('target', '_blank')
+        urlLink.style = `display: flex;
+        padding: 4px 15px;
+        background-color: var(--comfy-input-bg);
+        border-radius: 8px;
+        border-color: var(--border-color);
+        border-style: solid;
+        color: var(--descrip-text);
+        text-decoration: none;
+        border-radius: 5px;
+        transition: background-color 0.3s ease 0s;`
+        // urlLink.style.minHeight = '200px'
+        btns.appendChild(urlLink)
+
+
+        //todo 导出视频 that[`wavesurfer_${this.id}`].renderer.exportImage('image/png',1,'dataURL')
+        // https://github.com/diffusion-studio/ffmpeg-js
+
+
+        this.addCustomWidget(widget)
+
+        const onRemoved = this.onRemoved
+        this.onRemoved = () => {
+          widget.div.remove()
+          return onRemoved?.()
+        }
+
+        this.size = [this.size[0], 280]
+        this.serialize_widgets = true //需保存widget的值
+      }
+
+      const onExecuted = nodeType.prototype.onExecuted
+      nodeType.prototype.onExecuted = function (message) {
+        onExecuted?.apply(this, arguments)
+        const audio = message.audio
+        console.log('#onExecuted', `AudioPlay_${this.id}`, message,audio)
+        try {
+          let { url, prompt } = parseUrl(audio[0])
+
+          that[`wavesurfer_${this.id}`] = updateWaveWidgetValue(
+            this.widgets,
+            this.id,
+            url,
+            prompt,
+            that[`wavesurfer_${this.id}`]
+          )
+
+          that[`wavesurfer_${this.id}`]?.playPause()
+        } catch (error) {
+          console.log(error)
+        }
+      }
+    }
+  },
+  async loadedGraphNode (node, app) {
+    if (node.type === 'AudioPlay') {
+      let widget = node.widgets.filter(w => w.name == 'AudioPlay')[0]
+
+      if (widget.value) {
+        let [url, prompt] = widget.value
+
+        this[`wavesurfer_${node.id}`] = updateWaveWidgetValue(
+          node.widgets,
+          node.id,
+          url,
+          prompt,
+          this[`wavesurfer_${node.id}`]
+        )
+      }
+
+      console.log('#loadedGraphNode', node)
+    }
+  }
+})

web/javascript/td_background.js

@@ -0,0 +1,323 @@
+// touchdesigner的背景效果，把appinfo的输出，选择一张图片作为背景
+
+window._bg_img = null
+
+/**
+ * draws the back canvas (the one containing the background and the connections)
+ * @method drawBackCanvas
+ **/
+LGraphCanvas.prototype.drawBackCanvas = function () {
+  var canvas = this.bgcanvas
+  if (
+    canvas.width != this.canvas.width ||
+    canvas.height != this.canvas.height
+  ) {
+    canvas.width = this.canvas.width
+    canvas.height = this.canvas.height
+  }
+
+  if (!this.bgctx) {
+    this.bgctx = this.bgcanvas.getContext('2d')
+  }
+  var ctx = this.bgctx
+  if (ctx.start) {
+    ctx.start()
+  }
+
+  var viewport = this.viewport || [0, 0, ctx.canvas.width, ctx.canvas.height]
+
+  //clear
+  if (this.clear_background) {
+    ctx.clearRect(viewport[0], viewport[1], viewport[2], viewport[3])
+  }
+
+  //show subgraph stack header
+  if (this._graph_stack && this._graph_stack.length) {
+    ctx.save()
+    var parent_graph = this._graph_stack[this._graph_stack.length - 1]
+    var subgraph_node = this.graph._subgraph_node
+    ctx.strokeStyle = subgraph_node.bgcolor
+    ctx.lineWidth = 10
+    ctx.strokeRect(1, 1, canvas.width - 2, canvas.height - 2)
+    ctx.lineWidth = 1
+    ctx.font = '40px Arial'
+    ctx.textAlign = 'center'
+    ctx.fillStyle = subgraph_node.bgcolor || '#AAA'
+    var title = ''
+    for (var i = 1; i < this._graph_stack.length; ++i) {
+      title += this._graph_stack[i]._subgraph_node.getTitle() + ' >> '
+    }
+    ctx.fillText(title + subgraph_node.getTitle(), canvas.width * 0.5, 40)
+    ctx.restore()
+  }
+
+  var bg_already_painted = false
+  if (this.onRenderBackground) {
+    bg_already_painted = this.onRenderBackground(canvas, ctx)
+  }
+
+  //reset in case of error
+  if (!this.viewport) {
+    ctx.restore()
+    ctx.setTransform(1, 0, 0, 1, 0, 0)
+  }
+  this.visible_links.length = 0
+
+  if (this.graph) {
+    //apply transformations
+    ctx.save()
+    this.ds.toCanvasContext(ctx)
+
+    //render BG
+    if (
+      this.ds.scale < 1 &&
+      !bg_already_painted &&
+      this.clear_background_color
+    ) {
+      ctx.fillStyle = this.clear_background_color
+      ctx.fillRect(
+        this.visible_area[0],
+        this.visible_area[1],
+        this.visible_area[2],
+        this.visible_area[3]
+      )
+    }
+
+    // 主要修改
+    if (this.background_image && this.ds.scale > 0.5 && !bg_already_painted) {
+      if (this.zoom_modify_alpha) {
+        //使得 alpha 越接近0时变化越缓慢。
+        let alpha = (1.0 - 0.5 / this.ds.scale) * this.editor_alpha
+        ctx.globalAlpha = Math.min(Math.max(0, Math.sqrt(alpha)), 1)
+        // console.log((1.0 - 0.5 / this.ds.scale) * this.editor_alpha)
+      } else {
+        ctx.globalAlpha = this.editor_alpha
+      }
+      ctx.imageSmoothingEnabled = ctx.imageSmoothingEnabled = false // ctx.mozImageSmoothingEnabled =
+      if (!this._bg_img || this._bg_img.name != this.background_image) {
+        this._bg_img = new Image()
+        this._bg_img.name = this.background_image
+        this._bg_img.src = this.background_image
+        var that = this
+        this._bg_img.onload = function () {
+          that.draw(true, true)
+        }
+      }
+
+      var pattern = null
+      if (this._pattern == null && this._bg_img.width > 0) {
+        pattern = ctx.createPattern(this._bg_img, 'repeat')
+        this._pattern_img = this._bg_img
+        this._pattern = pattern
+      } else {
+        pattern = this._pattern
+      }
+
+      if (pattern) {
+        ctx.fillStyle = pattern
+        ctx.fillRect(
+          this.visible_area[0],
+          this.visible_area[1],
+          this.visible_area[2],
+          this.visible_area[3]
+        )
+        ctx.fillStyle = 'transparent'
+      }
+
+      ctx.globalAlpha = 1.0
+      ctx.imageSmoothingEnabled = ctx.imageSmoothingEnabled = true //= ctx.mozImageSmoothingEnabled
+    }
+
+    //groups
+    if (this.graph._groups.length && !this.live_mode) {
+      this.drawGroups(canvas, ctx)
+    }
+
+    if (this.onDrawBackground) {
+      this.onDrawBackground(ctx, this.visible_area)
+    }
+    if (this.onBackgroundRender) {
+      //LEGACY
+      console.error(
+        'WARNING! onBackgroundRender deprecated, now is named onDrawBackground '
+      )
+      this.onBackgroundRender = null
+    }
+
+    //DEBUG: show clipping area
+    //ctx.fillStyle = "red";
+    //ctx.fillRect( this.visible_area[0] + 10, this.visible_area[1] + 10, this.visible_area[2] - 20, this.visible_area[3] - 20);
+
+    //bg
+    if (this.render_canvas_border) {
+      ctx.strokeStyle = '#235'
+      ctx.strokeRect(0, 0, canvas.width, canvas.height)
+    }
+
+    if (this.render_connections_shadows) {
+      ctx.shadowColor = '#000'
+      ctx.shadowOffsetX = 0
+      ctx.shadowOffsetY = 0
+      ctx.shadowBlur = 6
+    } else {
+      ctx.shadowColor = 'rgba(0,0,0,0)'
+    }
+
+    //draw connections
+    if (!this.live_mode) {
+      this.drawConnections(ctx)
+    }
+
+    ctx.shadowColor = 'rgba(0,0,0,0)'
+
+    //restore state
+    ctx.restore()
+  }
+
+  if (ctx.finish) {
+    ctx.finish()
+  }
+
+  this.dirty_bgcanvas = false
+  this.dirty_canvas = true //to force to repaint the front canvas with the bgcanvas
+}
+
+function imgToCanvasBase64 (img) {
+  const canvas = document.createElement('canvas')
+  const ctx = canvas.getContext('2d')
+  canvas.width = img.width
+  canvas.height = img.height
+  ctx.drawImage(img, 0, 0)
+  const base64 = canvas.toDataURL('image/png')
+
+  return base64
+}
+
+// 使用示例
+function convertImageToBase64 (img) {
+  //   const img = new Image()
+  //   img.src = 'path/to/your/image.jpg' // 替换为你的图片路径
+  // console.log('convertImageToBase64',img)
+  try {
+    const base64 = imgToCanvasBase64(img)
+    return base64
+  } catch (error) {
+    console.error(error)
+  }
+}
+
+function getInputsAndOutputs () {
+  const outputs =
+    `PreviewImage,SaveImage,TransparentImage,VHS_VideoCombine,VideoCombine_Adv,Image Save,SaveImageAndMetadata_`.split(
+      ','
+    )
+
+  let outputsId = []
+
+  for (let node of app.graph._nodes) {
+    if (outputs.includes(node.type)) {
+      outputsId.push(node.id)
+    }
+  }
+
+  return outputsId
+}
+
+function getRandomElement (arr) {
+  const randomIndex = Math.floor(Math.random() * arr.length)
+  return arr[randomIndex]
+}
+
+async function getBG () {
+  var outputs = []
+
+  for (let id of app.graph
+    .getNodeById(50)
+    .widgets.filter(w => w.name === 'output_ids')[0]
+    .value.split('\n')) {
+    if (getInputsAndOutputs().map(Number).includes(Number(id))) {
+      if (app.graph.getNodeById(id).imgs && app.graph.getNodeById(id).imgs[0]) {
+        let b = convertImageToBase64(app.graph.getNodeById(id).imgs[0])
+        // console.log(b)
+        outputs.push(b)
+      }
+    }
+  }
+
+  var BACKGROUND_IMAGE = getRandomElement(outputs),
+    CLEAR_BACKGROUND_COLOR = 'rgba(0,0,0,0.9)'
+
+  if (!window._bg_img) {
+    window._bg_img = app.canvas._bg_img.src
+  }
+  // let img=new Image();
+  // img.src=BACKGROUND_IMAGE;
+
+  //去掉透明度过度
+  // app.canvas.zoom_modify_alpha=false;
+  //整体透明度
+  app.canvas.editor_alpha = 1.1
+  //   app.canvas._pattern=ctx.createPattern(img, "no-repeat");
+  app.canvas.updateBackground(BACKGROUND_IMAGE, CLEAR_BACKGROUND_COLOR)
+  app.canvas.draw(true, true)
+}
+
+class BgRunner {
+  constructor () {
+    this.intervalId = null
+    this.running = false
+  }
+
+  // 要运行的方法
+  bg () {
+    console.log('方法bg正在运行')
+    getBG()
+  }
+
+  // 启动bg方法每秒运行一次
+  start () {
+    if (!this.running) {
+      this.intervalId = setInterval(() => this.bg(), 1500)
+      this.running = true
+    }
+  }
+
+  // 停止bg方法的运行
+  stop () {
+    if (this.running) {
+      clearInterval(this.intervalId)
+      this.intervalId = null
+      this.running = false
+
+      if (window._bg_img) {
+        var BACKGROUND_IMAGE = window._bg_img,
+          CLEAR_BACKGROUND_COLOR = 'rgba(0,0,0,1)'
+        app.canvas.editor_alpha = 1
+
+        app.canvas.updateBackground(BACKGROUND_IMAGE, CLEAR_BACKGROUND_COLOR)
+        app.canvas.draw(true, true)
+      }
+    }
+  }
+
+  // 切换start和stop
+  toggle () {
+    if (this.running) {
+      this.stop()
+    } else {
+      this.start()
+    }
+  }
+
+  // 获取运行状态
+  isRunning () {
+    return this.running
+  }
+}
+
+// 示例用法
+// const runner = new BgRunner();
+// runner.start();
+// setTimeout(() => runner.stop(), 5000);
+
+export const td_bg = new BgRunner()

web/javascript/ui_mixlab.js

@@ -26,7 +26,35 @@ function showTextByLanguage (key, json) {
 }
 
 //系统prompt
-const systemPrompt = `You are a prompt creator, your task is to create prompts for the user input request, the prompts are image descriptions that include keywords for (an adjective, type of image, framing/composition, subject, subject appearance/action, environment, lighting situation, details of the shoot/illustration, visuals aesthetics and artists), brake keywords by comas, provide high quality, non-verboose, coherent, brief, concise, and not superfluous prompts, the subject from the input request must be included verbatim on the prompt,the prompt is english`
+// const systemPrompt = `You are a prompt creator, your task is to create prompts for the user input request, the prompts are image descriptions that include keywords for (an adjective, type of image, framing/composition, subject, subject appearance/action, environment, lighting situation, details of the shoot/illustration, visuals aesthetics and artists), brake keywords by comas, provide high quality, non-verboose, coherent, brief, concise, and not superfluous prompts, the subject from the input request must be included verbatim on the prompt,the prompt is english`
+
+let tool ={
+  "name": "create_prompt",
+  "description": "Create a prompt with a given subject, content, and style based on user input for image descriptions.",
+  "parameter": {
+    "type": "object",
+    "properties": {
+      "subject": {
+        "type": "string",
+        "description": "The subject of the prompt, included verbatim from the input request.",
+        "required": true
+      },
+      "content": {
+        "type": "string",
+        "description": "The content of the prompt, primarily focusing on the scene and objects, including keywords for adjective, type of image, framing/composition, subject appearance/action, and environment.",
+        "required": true
+      },
+      "style": {
+        "type": "string",
+        "description": "The style of the prompt, including lighting situation, details of the shoot/illustration, visual aesthetics, and artists. Ensure it is high quality, non-verbose, coherent, brief, concise, and not superfluous.",
+        "required": true
+      }
+    }
+  }
+}
+
+const systemPrompt=`You are a helpful assistant with access to the following functions. Use them if required - ${JSON.stringify(tool,null,2)}`
+
 
 if (!localStorage.getItem('_mixlab_system_prompt')) {
   localStorage.setItem('_mixlab_system_prompt', systemPrompt)
@@ -46,7 +74,7 @@ async function get_llamafile_models () {
     })
 
     const data = await response.json()
-    console.log(data)
+    // console.log(data)
     return data.names
   } catch (error) {
     console.error(error)
@@ -135,7 +163,7 @@ async function createMenu () {
 
     // appsButton.onclick = () =>
     appsButton.onclick = async () => {
-      if (window._mixlab_llamacpp) {
+      if (window._mixlab_llamacpp&&window._mixlab_llamacpp.model&&window._mixlab_llamacpp.model.length>0) {
         //显示运行的模型
         createModelsModal([
           window._mixlab_llamacpp.url,
@@ -539,10 +567,6 @@ injectCSS(`::-webkit-scrollbar {
   border-left: 2px solid var(--input-text); 
 }
  
-.litegraph{
-  background: var(--bg-color)!important;
-}
-
 `)
 
 async function getCustomnodeMappings (mode = 'url') {
@@ -774,6 +798,22 @@ async function fetchReadmeContent (url) {
   }
 }
 
+async function startLLM (model) {
+  let res = await start_llama(model)
+  window._mixlab_llamacpp = res||{ model:[] }
+
+  localStorage.setItem('_mixlab_llama_select', res?.model||'')
+
+  if (document.body.querySelector('#mixlab_chatbot_by_llamacpp')&&window._mixlab_llamacpp?.url) {
+    document.body
+      .querySelector('#mixlab_chatbot_by_llamacpp')
+      .setAttribute('title', window._mixlab_llamacpp.url)
+  }
+  if (document.body.querySelector('#llm_status_btn')&&window._mixlab_llamacpp) {
+    document.body.querySelector('#llm_status_btn').innerText = window._mixlab_llamacpp.model
+  }
+}
+
 function createModelsModal (models) {
   var div =
     document.querySelector('#model-modal') || document.createElement('div')
@@ -829,7 +869,7 @@ function createModelsModal (models) {
       color: var(--descrip-text);
       font-size: 18px;
       display: flex;
-      align-items: center;
+      align-items: flex-start;
       flex: 1;
       overflow: hidden;
       text-decoration: none;
@@ -848,6 +888,27 @@ function createModelsModal (models) {
   })
   linkIcon.style.padding = '4px'
 
+  const statusIcon = document.createElement('small')
+  statusIcon.textContent = showTextByLanguage('Status', {
+    Status: 'OFF'
+  })
+  statusIcon.id = 'llm_status_btn'
+  statusIcon.style=`padding: 4px;
+  background-color: rgb(102, 255, 108);
+  color: black;
+  font-size: 12px;
+  margin-left: 12px;`
+  if (window._mixlab_llamacpp?.url) {
+    statusIcon.textContent = window._mixlab_llamacpp.model
+    statusIcon.style.backgroundColor = '#66ff6c'
+    statusIcon.style.color = 'black'
+  } else {
+  }
+  statusIcon.addEventListener('click', e => {
+    e.stopPropagation()
+    // startLLM()
+  })
+
   const n_gpu = document.createElement('input')
   n_gpu.type = 'number'
   n_gpu.setAttribute('min', -1)
@@ -882,20 +943,22 @@ function createModelsModal (models) {
   const title = document.createElement('p')
   title.innerText = 'Models'
   title.style = `font-size: 18px;
-  margin-right: 8px;`
+  margin-right: 8px;
+  margin-top: 0;`
 
   const left_d = document.createElement('div')
   left_d.style = `display: flex;
   justify-content: center;
-  align-items: center;
-  font-size: 12px;`
+  align-items: flex-start;
+  font-size: 12px;
+  flex-direction: column; `
   left_d.appendChild(title)
-
+  title.appendChild(statusIcon)
   left_d.appendChild(linkIcon)
-
+  left_d.appendChild(n_gpu_div)
   headTitleElement.appendChild(left_d)
 
-  headTitleElement.appendChild(n_gpu_div)
+  // headTitleElement.appendChild(n_gpu_div)
 
   //重启
   const reStart = document.createElement('small')
@@ -903,7 +966,11 @@ function createModelsModal (models) {
     restart: '重启'
   })
 
-  reStart.style.padding = '4px'
+  reStart.style=`padding: 8px;
+  font-size: 16px;
+  outline: 1px solid;
+  padding-top: 4px;
+  padding-bottom: 4px;`
 
   headTitleElement.appendChild(reStart)
 
@@ -962,29 +1029,22 @@ function createModelsModal (models) {
 
   modalContent.appendChild(input)
 
-  for (const m of models) {
-    let d = document.createElement('div')
-    d.innerText = m
-    d.className = `mix_tag`
+  if (!window._mixlab_llamacpp||(window._mixlab_llamacpp?.model?.length==0)) {
+    for (const m of models) {
+      let d = document.createElement('div')
+      d.innerText = `${showTextByLanguage('Run', {
+        Run: '运行'
+      })} ${m}`
+      d.className = `mix_tag`
 
-    if (!window._mixlab_llamacpp) {
       d.addEventListener('click', async e => {
         e.stopPropagation()
         div.remove()
-        let res = await start_llama(m)
-        window._mixlab_llamacpp = res
-
-        localStorage.setItem('_mixlab_llama_select', res.model)
-
-        if (document.body.querySelector('#mixlab_chatbot_by_llamacpp')) {
-          document.body
-            .querySelector('#mixlab_chatbot_by_llamacpp')
-            .setAttribute('title', window._mixlab_llamacpp.url)
-        }
+        startLLM(m)
       })
-    }
 
-    modalContent.appendChild(d)
+      modalContent.appendChild(d)
+    }
   }
   modal.appendChild(modalContent)
 
@@ -998,8 +1058,8 @@ function createModelsModal (models) {
   cursor: pointer;
   font-size: 12px;
   color: white;`
-  helpInfo.href="https://discord.gg/cXs9vZSqeK"
-  helpInfo.target="_blank"
+  helpInfo.href = 'https://discord.gg/cXs9vZSqeK'
+  helpInfo.target = '_blank'
   modal.appendChild(helpInfo)
 
   document.body.appendChild(div)
@@ -1353,6 +1413,8 @@ app.registerExtension({
           .querySelector('#mixlab_chatbot_by_llamacpp')
           .setAttribute('title', res.url)
       })
+    }else{
+      startLLM('')
     }
 
     LGraphCanvas.prototype.helpAboutNode = async function (node) {
@@ -1361,7 +1423,13 @@ app.registerExtension({
           ? nodesMap
           : await getCustomnodeMappings('url')
 
-      console.log('node & node map', node, nodesMap, nodesMap[node.type])
+      console.log(
+        '%c### node & node map',
+        'background: yellow; color: black',
+        node,
+        nodesMap,
+        nodesMap[node.type]
+      )
       let repo = nodesMap[node.type]
       if (repo) {
         let markdown = await fetchReadmeContent(repo.url)
@@ -1371,10 +1439,14 @@ app.registerExtension({
 
     LGraphCanvas.prototype.fixTheNode = function (node) {
       let new_node = LiteGraph.createNode(node.comfyClass)
-      new_node.pos = [node.pos[0], node.pos[1]]
-      app.canvas.graph.add(new_node, false)
-      copyNodeValues(node, new_node)
-      app.canvas.graph.remove(node)
+      console.log(node)
+      if(new_node){
+        new_node.pos = [node.pos[0], node.pos[1]]
+        app.canvas.graph.add(new_node, false)
+        copyNodeValues(node, new_node)
+        app.canvas.graph.remove(node)
+      }
+     
     }
 
     smart_init()
@@ -1392,6 +1464,7 @@ app.registerExtension({
         let userInput = widget.value
         widget.value = widget.value.trim()
         widget.value += '\n'
+        let jsonStr="";
         try {
           await completion_(
             window._mixlab_llamacpp.url + '/v1/chat/completions',
@@ -1406,6 +1479,7 @@ app.registerExtension({
             t => {
               // console.log(t)
               widget.value += t
+              jsonStr+=t
             }
           )
         } catch (error) {
@@ -1431,13 +1505,27 @@ app.registerExtension({
                 t => {
                   // console.log(t)
                   widget.value += t
+                  jsonStr+=t
                 }
               )
             })
           }
         }
 
-        widget.value = widget.value.trim()
+        let json=null;
+
+        try {
+          json=JSON.parse(jsonStr.trim())
+        } catch (error) {
+          json=JSON.parse(jsonStr.trim()+"}")
+        }
+
+        if(json){
+          widget.value = [json.subject,json.content,json.style].join('\n')
+        }else{
+          widget.value = widget.value.trim()
+        }
+        
       }
     }
 
@@ -1700,6 +1788,7 @@ app.registerExtension({
         {
           content: 'Help ♾️Mixlab', // with a name
           callback: () => {
+            // console.log('#data',node)
             LGraphCanvas.prototype.helpAboutNode(node)
           } // and the callback
         },

web/javascript/video_mixlab.js

@@ -6,8 +6,6 @@ import { $el } from '../../../scripts/ui.js'
 
 // The code is based on ComfyUI-VideoHelperSuite modification.
 
-
-
 function injectCSS (css) {
   // 检查页面中是否已经存在具有相同内容的style标签
   const existingStyle = document.querySelector('style')
@@ -240,15 +238,7 @@ app.registerExtension({
   }
 })
 
-
-function offsetDOMWidget(
-  widget,
-  ctx,
-  node,
-  widgetWidth,
-  widgetY,
-  height
-) {
+function offsetDOMWidget (widget, ctx, node, widgetWidth, widgetY, height) {
   const margin = 10
   const elRect = ctx.canvas.getBoundingClientRect()
   const transform = new DOMMatrix()
@@ -270,18 +260,18 @@ function offsetDOMWidget(
     position: 'absolute',
     background: !node.color ? '' : node.color,
     color: !node.color ? '' : 'white',
-    zIndex: 5, //app.graph._nodes.indexOf(node),
+    zIndex: 5 //app.graph._nodes.indexOf(node),
   })
 }
 
-export const hasWidgets = (node) => {
+export const hasWidgets = node => {
   if (!node.widgets || !node.widgets?.[Symbol.iterator]) {
     return false
   }
   return true
 }
 
-export const cleanupNode = (node) => {
+export const cleanupNode = node => {
   if (!hasWidgets(node)) {
     return
   }
@@ -298,43 +288,43 @@ export const cleanupNode = (node) => {
   }
 }
 
-const CreatePreviewElement = (name, val, format) => {
-  const [type] = format.split('/')  
+const createPreviewElement = (name, val, format) => {
+  const [type] = format.split('/')
   const w = {
-      name,
-      type,
-      value: val,
-      draw: function (ctx, node, widgetWidth, widgetY, height) {
-        const [cw, ch] = this.computeSize(widgetWidth)
-        offsetDOMWidget(this, ctx, node, widgetWidth, widgetY, ch)
-      },
-      computeSize: function (_) {
-        const ratio = this.inputRatio || 1
-        const width = Math.max(220, this.parent.size[0])
-        return [width, (width / ratio + 10)]
-      },
-      onRemoved: function () {
-        if (this.inputEl) {
-          this.inputEl.remove()
-        }
-      },
+    name,
+    type,
+    value: val,
+    draw: function (ctx, node, widgetWidth, widgetY, height) {
+      const [cw, ch] = this.computeSize(widgetWidth)
+      offsetDOMWidget(this, ctx, node, widgetWidth, widgetY, ch)
+    },
+    computeSize: function (_) {
+      const ratio = this.inputRatio || 1
+      const width = Math.max(220, this.parent.size[0])
+      return [width, width / ratio + 10]
+    },
+    onRemoved: function () {
+      if (this.inputEl) {
+        this.inputEl.remove()
+      }
     }
-
-    w.inputEl = document.createElement(type === 'video' ? 'video' : 'img')
-    w.inputEl.src = w.value
-    if (type === 'video') {
-      w.inputEl.setAttribute('type', 'video/webm');
-      w.inputEl.autoplay = true
-      w.inputEl.loop = true
-      w.inputEl.controls = false;
-    }
-    w.inputEl.onload = function () {
-      w.inputRatio = w.inputEl.naturalWidth / w.inputEl.naturalHeight
-    }
-    document.body.appendChild(w.inputEl)
-    return w
   }
 
+  w.inputEl = document.createElement(type === 'video' ? 'video' : 'img')
+  w.inputEl.src = w.value
+
+  if (type === 'video' || format.match('.mp4')) {
+    w.inputEl.setAttribute('type', 'video/webm')
+    w.inputEl.autoplay = true
+    w.inputEl.loop = true
+    w.inputEl.controls = true
+  }
+  w.inputEl.onload = function () {
+    w.inputRatio = w.inputEl.naturalWidth / w.inputEl.naturalHeight
+  }
+  document.body.appendChild(w.inputEl)
+  return w
+}
 
 app.registerExtension({
   name: 'Mixlab.Video.ImageListReplace',
@@ -469,7 +459,10 @@ app.registerExtension({
       }
     }
 
-    if (nodeData?.name == 'VideoCombine_Adv') {
+    if (
+      nodeData?.name == 'VideoCombine_Adv' ||
+      nodeData?.name == 'CombineAudioVideo'
+    ) {
       const onExecuted = nodeType.prototype.onExecuted
       nodeType.prototype.onExecuted = function (message) {
         const prefix = 'vhs_gif_preview_'
@@ -489,7 +482,7 @@ app.registerExtension({
                 '/view?' + new URLSearchParams(params).toString()
               )
               const w = this.addCustomWidget(
-                CreatePreviewElement(
+                createPreviewElement(
                   `${prefix}_${i}`,
                   previewUrl,
                   params.format || 'image/gif'

web/lib/miniPaint-4.14.2/index.html

@@ -45,6 +45,15 @@
 		<div class="submenu">
 			<!-- <a class="logo" href="#">miniPaint</a> -->
 			<div class="block attributes" id="action_attributes"></div>
+			
+			<select id="automask_models_mixlab"></select>
+
+			<button id="automask_image_mixlab" type="button" style="width: 98px;
+			height: 36px;margin-right: 18px;
+			color: white;">
+				RemoveBg
+			</button>
+
 			<button id="cancel_image_mixlab" type="button" style="width: 98px;
 			height: 36px;
 			color: white;">
@@ -55,6 +64,13 @@
 			color: white;">
 				Save
 			</button>
+
+			<a  
+			target="_blank" href="https://discord.gg/xbP2GZF6gn" 
+			style="width: 98px;
+			color: white;
+			text-decoration: none;"> Help/帮助 </a>
+
 			<button class="undo_button" id="undo_button" type="button">
 				<span class="sr_only">Undo</span>
 			</button>
@@ -82,7 +98,7 @@
 		</div>
 
 		<div class="sidebar_right">
-			<div class="preview block">
+			<div class="preview block" style="display: none;">
 				<h2 class="trn toggle" data-target="toggle_preview">Preview</h2>
 				<div id="toggle_preview"></div>
 			</div>
@@ -92,7 +108,7 @@
 				<div class="content" id="toggle_colors"></div>
 			</div>
 
-			<div class="block" id="info_base">
+			<div class="block" id="info_base" style="display: none;">
 				<h2 class="trn toggle toggle-full" data-target="toggle_info">Information</h2>
 				<div class="content" id="toggle_info"></div>
 			</div>