v0.30.0

Appinfo supports outputting to the background, enhancing the performance features of ComfyUI.

.github/workflows/publish.yml

@@ -0,0 +1,21 @@
+name: Publish to Comfy registry
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - "pyproject.toml"
+
+jobs:
+  publish-node:
+    name: Publish Custom Node to registry
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Publish Custom Node
+        uses: Comfy-Org/publish-node-action@main
+        with:
+          ## Add your own personal access token to your Github Repository secrets and reference it here.
+          personal_access_token: ${{ secrets.REGISTRY_ACCESS_TOKEN }}

README.md

@@ -6,11 +6,17 @@
 
 ##### `最新`：
 
-ChatGPT 节点支持 Local LLM（llama.cpp），Phi3、llama3 都可以直接一个节点运行了。模型下载后，放置到 `models/llamafile/`
+- 增加 Edit Mask，方便在生成的时候手动绘制 mask [workflow](./workflow/edit-mask-workflow.json)
+
+
+- ChatGPT 节点支持 Local LLM（llama.cpp），Phi3、llama3 都可以直接一个节点运行了。模型下载后，放置到 `models/llamafile/`
 
 - 右键菜单支持 text-to-text，方便对 prompt 词补全
 
-强烈推荐：[Phi-3-mini-4k-instruct-GGUF](https://huggingface.co/lmstudio-community/Phi-3-mini-4k-instruct-GGUF/tree/main)，备选：[llama3_if_ai_sdpromptmkr_q2k](https://hf-mirror.com/impactframes/llama3_if_ai_sdpromptmkr_q2k/tree/main)
+强烈推荐：
+[Phi-3-mini-4k-instruct-function-calling-GGUF](https://huggingface.co/nold/Phi-3-mini-4k-instruct-function-calling-GGUF)
+
+[Phi-3-mini-4k-instruct-GGUF](https://huggingface.co/lmstudio-community/Phi-3-mini-4k-instruct-GGUF/tree/main)，备选：[llama3_if_ai_sdpromptmkr_q2k](https://hf-mirror.com/impactframes/llama3_if_ai_sdpromptmkr_q2k/tree/main)
 
 - 右键菜单支持 image-to-text，使用多模态模型，多模态使用 [llava-phi-3-mini-gguf](https://huggingface.co/xtuner/llava-phi-3-mini-gguf/tree/main)，注意需要把llava-phi-3-mini-mmproj-f16.gguf也下载
 
@@ -20,7 +26,9 @@ ChatGPT 节点支持 Local LLM（llama.cpp），Phi3、llama3 都可以直接一
 
 #### `相关插件推荐`
 
-<!-- [comfyui-sd-prompt-mixlab](https://github.com/shadowcz007/comfyui-sd-prompt-mixlab) -->
+[Comfyui-ChatTTS](https://github.com/shadowcz007/Comfyui-ChatTTS)
+
+[comfyui-sound-lab](https://github.com/shadowcz007/comfyui-sound-lab)
 
 [comfyui-Image-reward](https://github.com/shadowcz007/comfyui-Image-reward)
 
@@ -37,6 +45,7 @@ ChatGPT 节点支持 Local LLM（llama.cpp），Phi3、llama3 都可以直接一
 - 发布为 app 的 workflow，可以在右键里再次编辑了
 - web app 可以设置分类，在 comfyui 右键菜单可以编辑更新 web app
 - 支持动态提示
+- 支持把输出显示到comfyui背景（TouchDesigner 风格）
 
 ![](./assets/微信图片_20240421205440.png)
 

__init__.py

@@ -778,12 +778,14 @@ async def start_local_llm(data):
     
     server_settings=ServerSettings(host=address,port=port)
 
+    name, ext = os.path.splitext(os.path.basename(model))
+    print('#model',name)
     app = create_app(
                 server_settings=server_settings,
                 model_settings=[
                     ModelSettings(
                     model=model,
-                    model_alias=os.path.basename(model),
+                    model_alias=name,
                     n_gpu_layers=n_gpu_layers,
                     n_ctx=4098,
                     chat_format=chat_format,
@@ -817,12 +819,12 @@ async def start_local_llm(data):
 async def my_hander_method(request):
     data =await request.json()
     # print(data)
+    if llama_port and llama_model and llama_chat_format:
+        return web.json_response({"port":llama_port,"model":llama_model,"chat_format":llama_chat_format} )
     try:
         result=await start_local_llm(data)
     except:
-        result={
-            {"port":None,"model":"","llama_cpp_error":True}
-        }
+        result= {"port":None,"model":"","llama_cpp_error":True}
         print('start_local_llm error')
 
     return web.json_response(result)
@@ -840,20 +842,16 @@ def re_start(request):
 
 # 导入节点
 from .nodes.PromptNode import GLIGENTextBoxApply_Advanced,EmbeddingPrompt,RandomPrompt,PromptSlide,PromptSimplification,PromptImage,JoinWithDelimiter
-from .nodes.ImageNode import ComparingTwoFrames,LoadImages_,CompositeImages,GridDisplayAndSave,GridInput,ImagesPrompt,SaveImageAndMetadata,SaveImageToLocal,SplitImage,GridOutput,GetImageSize_,MirroredImage,ImageColorTransfer,NoiseImage,TransparentImage,GradientImage,LoadImagesFromPath,LoadImagesFromURL,ResizeImage,TextImage,SvgImage,Image3D,ShowLayer,NewLayer,MergeLayers,CenterImage,AreaToMask,SmoothMask,SplitLongMask,ImageCropByAlpha,EnhanceImage,FaceToMask
+from .nodes.ImageNode import ImageListToBatch_,ComparingTwoFrames,LoadImages_,CompositeImages,GridDisplayAndSave,GridInput,ImagesPrompt,SaveImageAndMetadata,SaveImageToLocal,SplitImage,GridOutput,GetImageSize_,MirroredImage,ImageColorTransfer,NoiseImage,TransparentImage,GradientImage,LoadImagesFromPath,LoadImagesFromURL,ResizeImage,TextImage,SvgImage,Image3D,ShowLayer,NewLayer,MergeLayers,CenterImage,AreaToMask,SmoothMask,SplitLongMask,ImageCropByAlpha,EnhanceImage,FaceToMask
 # from .nodes.Vae import VAELoader,VAEDecode
 from .nodes.ScreenShareNode import ScreenShareNode,FloatingVideo
 
-from .nodes.ChatGPT import ChatGPTNode,ShowTextForGPT,CharacterInText,TextSplitByDelimiter
-from .nodes.Audio import GamePal,SpeechRecognition,SpeechSynthesis
+from .nodes.Audio import AudioPlayNode,SpeechRecognition,SpeechSynthesis
 from .nodes.Utils import IncrementingListNode,ListSplit,CreateLoraNames,CreateSampler_names,CreateCkptNames,CreateSeedNode,TESTNODE_,TESTNODE_TOKEN,AppInfo,IntNumber,FloatSlider,TextInput,ColorInput,FontInput,TextToNumber,DynamicDelayProcessor,LimitNumber,SwitchByIndex,MultiplicationNode
 from .nodes.Mask import PreviewMask_,MaskListReplace,MaskListMerge,OutlineMask,FeatheredMask
 
 from .nodes.Style import ApplyVisualStylePrompting,StyleAlignedReferenceSampler,StyleAlignedBatchAlign,StyleAlignedSampleReferenceLatents
 
-from .nodes.Video import VideoCombine_Adv,LoadVideoAndSegment,ImageListReplace,VAEEncodeForInpaint_Frames
-
-from .nodes.TripoSR import LoadTripoSRModel,TripoSRSampler,SaveTripoSRMesh
 
 
 # 要导出的所有节点及其名称的字典
@@ -884,6 +882,7 @@ NODE_CLASS_MAPPINGS = {
     "ImageColorTransfer":ImageColorTransfer,
     "ShowLayer":ShowLayer,
     "NewLayer":NewLayer,
+    "ImageListToBatch_":ImageListToBatch_,
     "CompositeImages_":CompositeImages,
     "SplitImage":SplitImage,
     "CenterImage":CenterImage,
@@ -905,10 +904,7 @@ NODE_CLASS_MAPPINGS = {
     # "VAEDecodeConsistencyDecoder":VAEDecode,
     "ScreenShare":ScreenShareNode,
     "FloatingVideo":FloatingVideo,
-    "ChatGPTOpenAI":ChatGPTNode,
-    "ShowTextForGPT":ShowTextForGPT,
-    "CharacterInText":CharacterInText,
-    "TextSplitByDelimiter":TextSplitByDelimiter,
+   
     "SpeechRecognition":SpeechRecognition,
     "SpeechSynthesis":SpeechSynthesis,
     "Color":ColorInput,
@@ -932,19 +928,12 @@ NODE_CLASS_MAPPINGS = {
     "ApplyVisualStylePrompting_":ApplyVisualStylePrompting,
     "StyleAlignedReferenceSampler_": StyleAlignedReferenceSampler,
     "StyleAlignedSampleReferenceLatents_": StyleAlignedSampleReferenceLatents,
-    "StyleAlignedBatchAlign_": StyleAlignedBatchAlign,
-    "LoadVideoAndSegment_":LoadVideoAndSegment,
-    "VideoCombine_Adv":VideoCombine_Adv,
+    "StyleAlignedBatchAlign_": StyleAlignedBatchAlign, 
     "ListSplit_":ListSplit,
-    "MaskListReplace_":MaskListReplace,
-    "ImageListReplace_":ImageListReplace,
-    "VAEEncodeForInpaint_Frames":VAEEncodeForInpaint_Frames,
+    "MaskListReplace_":MaskListReplace, 
     "IncrementingListNode_":IncrementingListNode,
     "PreviewMask_":PreviewMask_,
-     "LoadTripoSRModel_": LoadTripoSRModel,
-    "TripoSRSampler_": TripoSRSampler,
-    "SaveTripoSRMesh": SaveTripoSRMesh
-    # "GamePal":GamePal
+    "AudioPlay":AudioPlayNode
 }
 
 # 一个包含节点友好/可读的标题的字典
@@ -965,12 +954,12 @@ NODE_DISPLAY_NAME_MAPPINGS = {
     "VAEDecodeConsistencyDecoder":"Consistency Decoder Decode",
     "ScreenShare":"Screen Share ♾️Mixlab",
     "FloatingVideo":"FloatingVideo ♾️Mixlab",
-    "ChatGPTOpenAI":"ChatGPT & Local LLM ♾️Mixlab",
-    "ShowTextForGPT":"Show Text ♾️MixlabApp",
+    
     "MergeLayers":"Merge Layers ♾️Mixlab",
     "SpeechSynthesis":"SpeechSynthesis ♾️Mixlab",
     "SpeechRecognition":"SpeechRecognition ♾️Mixlab",
     "3DImage":"3DImage ♾️Mixlab",
+    "ImageListToBatch_":"Image List To Batch",
     "CompositeImages_":"Composite Images ♾️Mixlab",
     "DynamicDelayProcessor":"DynamicDelayByText ♾️Mixlab",
     "LaMaInpainting":"LaMaInpainting ♾️Mixlab",
@@ -996,13 +985,10 @@ NODE_DISPLAY_NAME_MAPPINGS = {
     "GridInput":"Grid Input ♾️Mixlab",
     "GridOutput":"Grid Output ♾️Mixlab",
     "GetImageSize_":"Get Image Size ♾️Mixlab",
-    "VAEEncodeForInpaint_Frames":"VAE Encode For Inpaint Frames ♾️Mixlab",
     "IncrementingListNode_":"Create Incrementing Number List ♾️Mixlab",
     "LoadImagesToBatch":"Load Images(base64) ♾️Mixlab",
     "PreviewMask_":"Preview Mask",
-    "LoadTripoSRModel_": "Load TripoSR Model",
-    "TripoSRSampler_": "TripoSR Sampler",
-    "SaveTripoSRMesh": "Save TripoSR Mesh"
+    "AudioPlay":"Audio Play ♾️Mixlab"
 }
 
 # web ui的节点功能
@@ -1013,6 +999,41 @@ logging.info('--------------')
 logging.info('\033[91m ### Mixlab Nodes: \033[93mLoaded')
 # print('\033[91m ### Mixlab Nodes: \033[93mLoaded')
 
+try:
+    from .nodes.ChatGPT import ChatGPTNode,ShowTextForGPT,CharacterInText,TextSplitByDelimiter
+    logging.info('ChatGPT.available True')
+
+    NODE_CLASS_MAPPINGS_V = {
+       "ChatGPTOpenAI":ChatGPTNode,
+        "ShowTextForGPT":ShowTextForGPT,
+        "CharacterInText":CharacterInText,
+        "TextSplitByDelimiter":TextSplitByDelimiter,
+    }
+
+    # 一个包含节点友好/可读的标题的字典
+    NODE_DISPLAY_NAME_MAPPINGS_V = {
+        "ChatGPTOpenAI":"ChatGPT & Local LLM ♾️Mixlab",
+        "ShowTextForGPT":"Show Text ♾️MixlabApp",
+        "CharacterInText":"Character In Text",
+        "TextSplitByDelimiter":"Text Split By Delimiter",
+    }
+
+
+    NODE_CLASS_MAPPINGS.update(NODE_CLASS_MAPPINGS_V)
+    NODE_DISPLAY_NAME_MAPPINGS.update(NODE_DISPLAY_NAME_MAPPINGS_V)
+
+except Exception as e:
+    logging.info('ChatGPT.available False')
+
+
+try:
+    from .nodes.edit_mask import EditMask
+    logging.info('edit_mask.available True')
+    NODE_CLASS_MAPPINGS['EditMask']=EditMask
+    NODE_DISPLAY_NAME_MAPPINGS['EditMask']="Edit Mask ♾️Mixlab"
+except Exception as e:
+    logging.info('edit_mask.available False')
+
 try:
     from .nodes.Lama import LaMaInpainting
     logging.info('LaMaInpainting.available {}'.format(LaMaInpainting.available))
@@ -1048,4 +1069,61 @@ try:
 except Exception as e:
     logging.info('RembgNode_.available False' )
 
+
+try:
+    from .nodes.Video import GenerateFramesByCount,scenesNode_,CombineAudioVideo,VideoCombine_Adv,LoadVideoAndSegment,ImageListReplace,VAEEncodeForInpaint_Frames,LoadAndCombinedAudio_
+    
+    NODE_CLASS_MAPPINGS_V = {
+        "VAEEncodeForInpaint_Frames":VAEEncodeForInpaint_Frames,
+        "ImageListReplace_":ImageListReplace,
+        "LoadVideoAndSegment_":LoadVideoAndSegment,
+        "VideoCombine_Adv":VideoCombine_Adv,
+        "LoadAndCombinedAudio_":LoadAndCombinedAudio_,
+        "CombineAudioVideo":CombineAudioVideo,
+        "ScenesNode_":scenesNode_,
+        "GenerateFramesByCount":GenerateFramesByCount 
+    }
+
+    # 一个包含节点友好/可读的标题的字典
+    NODE_DISPLAY_NAME_MAPPINGS_V = {
+        "VAEEncodeForInpaint_Frames":"VAE Encode For Inpaint Frames ♾️Mixlab",
+        "ImageListReplace_":"Image List Replace",
+        "LoadVideoAndSegment_":"Load Video And Segment",
+        "VideoCombine_Adv":"Video Combine",
+        "LoadAndCombinedAudio_":"Load And Combined Audio",
+        "CombineAudioVideo":"Combine Audio Video",
+        "ScenesNode_":"Scenes Node", 
+        "GenerateFramesByCount":"Generate Frames By Count"
+    }
+
+
+    NODE_CLASS_MAPPINGS.update(NODE_CLASS_MAPPINGS_V)
+    NODE_DISPLAY_NAME_MAPPINGS.update(NODE_DISPLAY_NAME_MAPPINGS_V)
+
+except:
+    logging.info('Video.available False')
+
+
+try:
+    from .nodes.TripoSR import LoadTripoSRModel,TripoSRSampler,SaveTripoSRMesh
+    logging.info('TripoSR.available')
+
+    NODE_CLASS_MAPPINGS['LoadTripoSRModel_']=LoadTripoSRModel
+    NODE_DISPLAY_NAME_MAPPINGS["LoadTripoSRModel_"]= "Load TripoSR Model"
+    
+    NODE_CLASS_MAPPINGS['TripoSRSampler_']=TripoSRSampler
+    NODE_DISPLAY_NAME_MAPPINGS["TripoSRSampler_"]= "TripoSR Sampler"
+
+    NODE_CLASS_MAPPINGS['SaveTripoSRMesh']=SaveTripoSRMesh
+    NODE_DISPLAY_NAME_MAPPINGS["SaveTripoSRMesh"]= "Save TripoSR Mesh"
+    
+
+except Exception as e:
+    logging.info('TripoSR.available False' )
+
+
+
+
+
+
 logging.info('\033[93m -------------- \033[0m')

nodes/Audio.py

@@ -1,6 +1,7 @@
 
-
-
+import os
+import folder_paths
+import torchaudio
 
 class SpeechRecognition:
     @classmethod
@@ -55,46 +56,60 @@ class SpeechSynthesis:
         return {"ui": {"text": text}, "result": (text,)}
     
 
-#
-class GamePal:
+
+class AudioPlayNode:
+    def __init__(self):
+        self.output_dir = folder_paths.get_temp_directory()
+        self.type = "temp"
+        self.prefix_append = ""
+        self.compress_level = 4
     @classmethod
     def INPUT_TYPES(s):
-        return {
-            "required": {
-                "input_text": ("STRING",{"multiline": True,"default": ""}),
-            },
-            "optional": {
-                 
-                "input_num": ("INT",{
-                                "default":100, 
-                                "min": -1, #Minimum value
-                                "max": 0xffffffffffffffff, #Maximum value
-                                "step": 1, #Slider's step
-                                "display": "slider" # Cosmetic only: display as "number" or "slider"
-                                }), 
-                "python_code": ("STRING",{"multiline": True,"default": "result= 1 if 'Mixlab' in input_text else 0"}),
-            }
-        }
-
-    INPUT_IS_LIST = False
-    RETURN_TYPES = ("INT",)
+        return {"required": {
+            "audio": ("AUDIO",),
+              }, 
+                }
+    
+    RETURN_TYPES = ()
+  
     FUNCTION = "run"
-    OUTPUT_NODE = True
-    OUTPUT_IS_LIST = (False,)
 
     CATEGORY = "♾️Mixlab/Audio"
 
-    def run(self, input_text,input_num,python_code):
-        exec(python_code)
-        res=None
-        try:
-            # 可能会引发异常的代码
-            res=result
-        except:
-            # 处理异常的代码
-            print('')
+    INPUT_IS_LIST = False
+    OUTPUT_IS_LIST = ()
 
-        print(res)
+    OUTPUT_NODE = True
+  
+    def run(self,audio):
 
-        # print(session_history)
-        return {"ui": {"text": [input_text],"num":[input_num]}, "result": (res,)}
+        # 判断是否是 Tensor 类型
+        is_tensor = not isinstance(audio, dict)
+        # print('#判断是否是 Tensor 类型',is_tensor,audio)
+        if not is_tensor and 'waveform' in audio and 'sample_rate' in audio:
+            # {'waveform': tensor([], size=(1, 1, 0)), 'sample_rate': 44100}
+            is_tensor=True
+
+        if is_tensor:
+            filename_prefix=""
+            # 保存
+            filename_prefix += self.prefix_append
+            full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, self.output_dir)
+            results = list()
+            
+            filename_with_batch_num = filename.replace("%batch_num%", str(1))
+            file = f"{filename_with_batch_num}_{counter:05}_.wav"
+            
+            torchaudio.save(os.path.join(full_output_folder, file), audio['waveform'].squeeze(0), audio["sample_rate"])
+            results.append({
+                    "filename": file,
+                    "subfolder": subfolder,
+                    "type": self.type
+                })
+            
+        else:
+            results=[audio]
+                
+
+        # print(audio)
+        return {"ui": {"audio":results}}

nodes/ImageNode.py

@@ -1,6 +1,7 @@
 import numpy as np
 import requests
 import torch
+import torchvision.transforms.v2 as T
 # from PIL import Image, ImageDraw
 from PIL import Image, ImageOps,ImageFilter,ImageEnhance,ImageDraw,ImageSequence, ImageFont
 from PIL.PngImagePlugin import PngInfo
@@ -3207,3 +3208,52 @@ class SaveImageToLocal:
             counter += 1
 
         return ()
+
+
+class ImageBatchToList_:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": {"image_batch": ("IMAGE",), }}
+
+    RETURN_TYPES = ("IMAGE",)
+    RETURN_NAMES = ("image_list",)
+    OUTPUT_IS_LIST = (True,)
+    FUNCTION = "run"
+
+    CATEGORY = "♾️Mixlab/Image"
+
+    def run(self, image_batch):
+        images = [image_batch[i:i + 1, ...] for i in range(image_batch.shape[0])]
+        return (images, )
+
+class ImageListToBatch_:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {
+            "required": {
+                "images": ("IMAGE",),
+            }
+        }
+
+    RETURN_TYPES = ("IMAGE",)
+    FUNCTION = "run"
+    INPUT_IS_LIST = True
+    CATEGORY = "♾️Mixlab/Image"
+
+    def run(self, images):
+        shape = images[0].shape[1:3]
+        out = []
+
+        for i in range(len(images)):
+            img = images[i].permute([0,3,1,2])
+            if images[i].shape[1:3] != shape:
+                transforms = T.Compose([
+                    T.CenterCrop(min(img.shape[2], img.shape[3])),
+                    T.Resize((shape[0], shape[1]), interpolation=T.InterpolationMode.BICUBIC),
+                ])
+                img = transforms(img)
+            out.append(img.permute([0,2,3,1]))
+
+        out = torch.cat(out, dim=0)
+
+        return (out,)

nodes/RembgNode.py

@@ -467,10 +467,14 @@ class BriaRMBG(nn.Module):
 
 
 
+def get_U2NET_model_path():
+    try:
+        return folder_paths.get_folder_paths('rembg')[0]
+    except:
+        return os.path.join(folder_paths.models_dir, "rembg")
+    
 
-
-
-U2NET_HOME=os.path.join(folder_paths.models_dir, "rembg")
+U2NET_HOME=get_U2NET_model_path()
 os.environ["U2NET_HOME"] = U2NET_HOME
 
 global _available
@@ -528,7 +532,7 @@ except:
 
 
 def run_briarmbg(images=[]):
-    mroot=os.path.join(folder_paths.models_dir, "rembg")
+    mroot=U2NET_HOME
     m=os.path.join(mroot,'briarmbg.pth')
     if os.path.exists(m)==False:
         # 下载
@@ -591,14 +595,15 @@ def run_briarmbg(images=[]):
     return (masks,rgba_images,rgb_images)   
 
 
-def run_rembg(model_name= "unet",images=[]):
+def run_rembg(model_name= "unet",images=[],callback=None):
     # model_name = "unet" # "isnet-general-use"
+    # print('#run_rembg',model_name)
     rembg_session = new_session(model_name)
     masks=[]
     rgba_images=[]
     rgb_images=[]
     # 进度条
-    pbar = comfy.utils.ProgressBar(len(images) )
+    pbar=callback 
     for img in images:
         # use the post_process_mask argument to post process the mask to get better results.
         mask = remove(img, session=rembg_session,only_mask=True,post_process_mask=True)
@@ -638,8 +643,9 @@ def run_rembg(model_name= "unet",images=[]):
             rgb_image = Image.new("RGB", image_rgba.size, (0, 0, 0))
             rgb_image.paste(image_rgba, mask=image_rgba.split()[3])
             rgb_images.append(rgb_image)
-            
-        pbar.update(1)
+
+        if pbar:
+            pbar.update(1)
     return (masks,rgba_images,rgb_images)
 
 
@@ -691,7 +697,7 @@ class RembgNode_:
         if model_name=='briarmbg':
             masks,rgba_images,rgb_images=run_briarmbg(images)
         else:
-            masks,rgba_images,rgb_images=run_rembg(model_name,images)
+            masks,rgba_images,rgb_images=run_rembg(model_name,images, comfy.utils.ProgressBar(len(images) ))
 
         masks=[pil2tensor(m) for m in masks]
 

nodes/TripoSR.py

@@ -5,13 +5,20 @@ from PIL import Image
 import numpy as np
 import torch
 
-from folder_paths import get_filename_list, get_full_path, get_save_image_path, get_output_directory,models_dir
+from folder_paths import get_folder_paths, get_full_path, get_save_image_path, get_output_directory,models_dir
 from comfy.model_management import get_torch_device
 from .tsr.system import TSR
 
 import comfy.utils
 
-triposr_model_path=path.join(models_dir,'triposr/model.ckpt')
+
+def get_triposr_model_path():
+    try:
+        return path.join(get_folder_paths('triposr')[0],'model.ckpt')
+    except:
+        return path.join(path.join(models_dir, "triposr"),'model.ckpt')
+    
+triposr_model_path=get_triposr_model_path()
 
 
 # Tensor to PIL

nodes/Utils.py

@@ -284,7 +284,7 @@ class FloatSlider:
                 }
     
     RETURN_TYPES = ("FLOAT",) 
-    RETURN_NAMES = ('weight(0-1)',)
+    RETURN_NAMES = ('FLOAT',)
     FUNCTION = "run"
 
     CATEGORY = "♾️Mixlab/Input"
@@ -297,9 +297,7 @@ class FloatSlider:
             number = min_value
         elif number > max_value:
             number = max_value
-        scaled_number = (number - min_value) / (max_value - min_value)
-        return (scaled_number,)
-    
+        return (number,)    
   
 class IntNumber:
     @classmethod

nodes/Video.py

@@ -17,9 +17,128 @@ import folder_paths
 from comfy.k_diffusion.utils import FolderOfImages
 from comfy.utils import common_upscale
 
+import torchaudio
+import base64
+
+import mimetypes
 
 
 
+def get_frames(frame_count, frames, revert=False):
+    if not revert:
+        if frame_count <= len(frames):
+            return frames[:frame_count]
+        else:
+            return [frames[i % len(frames)] for i in range(frame_count)]
+    else:
+        extended_frames = frames + frames[-2:0:-1]  # 正向加反向中间部分
+        if frame_count <= len(extended_frames):
+            return extended_frames[:frame_count]
+        else:
+            return [extended_frames[i % len(extended_frames)] for i in range(frame_count)]
+
+# # 示例用法
+# frames = ["frame1", "frame2", "frame3"]
+# frame_count = 2
+
+# result = get_frames(frame_count, frames, revert=False)
+# print(result)  # 输出: ['frame1', 'frame2', 'frame3', 'frame1', 'frame2', 'frame3', 'frame1']
+
+# result = get_frames(frame_count, frames, revert=True)
+# print(result)  # 输出: ['frame1', 'frame2', 'frame3', 'frame2', 'frame1', 'frame2', 'frame3']
+
+
+
+
+def get_mime_type(file_path):
+    # 获取文件的 MIME 类型
+    mime_type, _ = mimetypes.guess_type(file_path)
+    
+    # 如果无法猜测类型，返回默认类型
+    if mime_type is None:
+        return 'application/octet-stream'
+    
+    return mime_type
+# import subprocess
+# from imageio_ffmpeg import get_ffmpeg_exe
+
+
+def save_audio_base64s_to_file(base64_audios, output_folder, file_name):
+    # Ensure the output folder exists
+    if not os.path.exists(output_folder):
+        os.makedirs(output_folder)
+
+    decoded_audios=[]
+    for a in base64_audios:
+    
+        # If the base64 string contains a header, remove it
+        if ',' in a:
+            a = a.split(',')[1]
+        
+        # 解码 base64 数据
+        a=base64.b64decode(a)
+        decoded_audios.append(a)
+
+    # 拼接音频数据
+    combined_audio = b''.join(decoded_audios)
+    
+    # Create the full file path
+    file_path = os.path.join(output_folder, file_name)
+    
+    # Write the decoded audio to the file
+    with open(file_path, 'wb') as audio_file:
+        audio_file.write(combined_audio)
+    
+    return file_path
+
+# Example usage
+# base64_audio = "data:audio/wav;base64,UklGRiQAAABXQVZFZm10IBAAAAABAAEAIlYAAESsAAACABAAZGF0YQAAAAA="
+# output_folder = "audio_files"
+# file_name = "output.wav"
+
+# file_path = save_audio_base64_to_file(base64_audio, output_folder, file_name)
+# print(f"Audio saved to: {file_path}")
+
+# 写一个python文件，用来 判断文件夹内命名为 所有chat_tts开头的文件数量（chat_tts_00001），并输出新的编号
+def get_new_counter(full_output_folder, filename_prefix):
+    # 获取目录中的所有文件
+    files = os.listdir(full_output_folder)
+    
+    # 过滤出以 filename_prefix 开头并且后续部分为数字的文件
+    filtered_files = []
+    for f in files:
+        if f.startswith(filename_prefix):
+            # 去掉文件名中的前缀和后缀，只保留中间的数字部分
+            base_name = f[len(filename_prefix)+1:]
+            number_part = base_name.split('.')[0]  # 假设文件名中只有一个点，即扩展名
+            if number_part.isdigit():
+                filtered_files.append(int(number_part))
+
+    if not filtered_files:
+        return 1
+
+    # 获取最大的编号
+    max_number = max(filtered_files)
+    
+    # 新的编号
+    return max_number + 1
+
+def crop_audio(input_file, start_time, duration):
+    # Load the audio file
+    audio_tensor, sample_rate = torchaudio.load(input_file)
+    
+    # Convert start_time and duration from seconds to sample indices
+    start_sample = int(start_time * sample_rate)
+    end_sample = start_sample + int(duration * sample_rate)
+    
+    # Perform the slicing
+    cropped_audio_tensor = audio_tensor[:, start_sample:end_sample]
+    
+    # Save the cropped audio to a new file
+    torchaudio.save(input_file, cropped_audio_tensor, sample_rate)
+    
+    return input_file
+
 def generate_folder_name(directory,video_path):
     # Get the directory and filename from the video path
     _, filename = os.path.split(video_path)
@@ -87,7 +206,7 @@ def split_video(video_path, video_segment_frames, transition_frames, output_dir)
 
 folder_paths.folder_names_and_paths["video_formats"] = (
     [
-        os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "video_formats"),
+        os.path.join(os.path.dirname(os.path.abspath(__file__)), ".", "video_formats"),
     ],
     [".json"]
 )
@@ -101,6 +220,25 @@ if ffmpeg_path is None:
     except:
         print("ffmpeg could not be found. Outputs that require it have been disabled")
 
+
+def combine_audio_video(audio_path, video_path, output_path):
+   
+    command = [
+        ffmpeg_path,
+        '-i', video_path,
+        '-i', audio_path,
+        '-c:v', 'copy',
+        '-c:a', 'aac',
+        '-shortest',
+        output_path
+    ]
+    
+    subprocess.run(command, check=True)
+    return output_path
+
+
+
+
 # Tensor to PIL
 def tensor2pil(image):
     return Image.fromarray(np.clip(255. * image.cpu().numpy().squeeze(), 0, 255).astype(np.uint8))
@@ -262,7 +400,7 @@ class LoadVideoAndSegment:
                     files.append(f)
         return {"required": {
                     "video": (sorted(files), {"video_upload": True}),
-                     "video_segment_frames": ("INT", {"default": 10, "min": 1, "step": 1}),
+                     "video_segment_frames": ("INT", {"default": 10, "min": -1, "step": 1}),
                      "transition_frames": ("INT", {"default": 0, "min": 0, "step": 1}), 
                      },}
 
@@ -332,63 +470,6 @@ class LoadVideoAndSegment:
         
         video_path = folder_paths.get_annotated_filepath(video)
         
-        # check if video is a gif - will need to use cv fallback to read frames
-        # use cv fallback if ffmpeg not installed or gif
-        # if ffmpeg_path is None:
-        #     return self.load_video_cv_fallback(video, frame_load_cap, skip_first_frames)
-        # otherwise, continue with ffmpeg
-        
-        # args_dummy = [ffmpeg_path, "-i", video_path, "-f", "null", "-"]
-        # try:
-        #     with subprocess.Popen(args_dummy, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) as proc:
-        #         for line in proc.stderr.readlines():
-        #             match = re.search(", ([1-9]|\\d{2,})x(\\d+)",line.decode('utf-8'))
-        #             if match is not None:
-        #                 size = [int(match.group(1)), int(match.group(2))]
-        #                 break
-        # except Exception as e:
-        #     print(f"Retrying with opencv due to ffmpeg error: {e}")
-        #     return self.load_video_cv_fallback(video, frame_load_cap, skip_first_frames)
-        # args_all_frames = [ffmpeg_path, "-i", video_path, "-v", "error",
-        #                      "-pix_fmt", "rgb24"]
-
-        # vfilters = []
-       
-        # if skip_first_frames > 0:
-        #     vfilters.append(f"select=gt(n\\,{skip_first_frames-1})")
-        # if frame_load_cap > 0:
-        #     vfilters.append(f"select=gt({frame_load_cap}\\,n)")
-        # #manually calculate aspect ratio to ensure reads remain aligned
-        
-        # if len(vfilters) > 0:
-        #     args_all_frames += ["-vf", ",".join(vfilters)]
-
-        # args_all_frames += ["-f", "rawvideo", "-"]
-        # images = []
-        # try:
-        #     with subprocess.Popen(args_all_frames, stdout=subprocess.PIPE) as proc:
-        #         #Manually buffer enough bytes for an image
-        #         bpi = size[0]*size[1]*3
-        #         current_bytes = bytearray(bpi)
-        #         current_offset=0
-        #         while True:
-        #             bytes_read = proc.stdout.read(bpi - current_offset)
-        #             if bytes_read is None:#sleep to wait for more data
-        #                 time.sleep(.2)
-        #                 continue
-        #             if len(bytes_read) == 0:#EOF
-        #                 break
-        #             current_bytes[current_offset:len(bytes_read)] = bytes_read
-        #             current_offset+=len(bytes_read)
-        #             if current_offset == bpi:
-        #                 images.append(np.array(current_bytes, dtype=np.float32).reshape(size[1], size[0], 3) / 255.0)
-        #                 current_offset = 0
-        # except Exception as e:
-        #     print(f"Retrying with opencv due to ffmpeg error: {e}")
-        #     return self.load_video_cv_fallback(video, frame_load_cap, skip_first_frames)
-
-        # imgs=split_list(images,video_segment_frames,transition_frames)
-
         # temp path 
         tp=folder_paths.get_temp_directory()
         basename = os.path.basename(video_path)  # 获取文件名
@@ -396,15 +477,22 @@ class LoadVideoAndSegment:
 
         folder_path = create_folder(tp,name_without_extension)
         
- 
-        # 导出的数据
-        scenes_video,total_frames,fps=split_video(video_path,video_segment_frames,
-                                 transition_frames,folder_path)
+        if video_segment_frames==-1:
+            # 不切割视频
+            scenes_video=[video_path]
+            # 读取视频文件
+            video_capture = cv2.VideoCapture(video_path)
+            
+            # 获取视频的总帧数和帧率
+            total_frames = int(video_capture.get(cv2.CAP_PROP_FRAME_COUNT))
+            fps = video_capture.get(cv2.CAP_PROP_FPS)
+
+        else:
+            # 导出的数据
+            scenes_video,total_frames,fps=split_video(video_path,video_segment_frames,
+                                    transition_frames,folder_path)
       
 
-        # imgs=[torch.from_numpy(np.stack(im)) for im in imgs]
-
-        # images = torch.from_numpy(np.stack(images))
 
         return (scenes_video,len(scenes_video), total_frames,fps,)
 
@@ -422,7 +510,80 @@ class LoadVideoAndSegment:
             return "Invalid image file: {}".format(video)
 
         return True
-    
+
+
+
+class LoadAndCombinedAudio_:
+    @classmethod
+    def INPUT_TYPES(s):
+       
+        return {"required": {
+                    "audios": ("AUDIOBASE64",),
+                    "start_time": ("FLOAT" , {"default": 0, "min": 0, "max": 10000000, "step": 0.01}),
+                    "duration": ("FLOAT" , {"default": 10, "min": -1, "max": 10000000, "step": 0.01}),
+                     },
+                }
+
+    CATEGORY = "♾️Mixlab/Audio"
+
+    RETURN_TYPES = ("STRING",)
+    RETURN_NAMES = ("audio_file_path",)
+    FUNCTION = "run"
+
+    def run(self,audios, start_time, duration):
+        output_dir = folder_paths.get_output_directory()
+        counter=get_new_counter(output_dir,'audio_')
+
+        audio_file = f"audio_{counter:05}.wav"
+
+        audio_file=save_audio_base64s_to_file(audios['base64'],output_dir,audio_file)
+        # duration == -1 则不裁切
+        if duration > -1:
+            crop_audio(audio_file, start_time, duration)
+
+        return (audio_file,)
+
+class CombineAudioVideo:
+    @classmethod
+    def INPUT_TYPES(s):
+       
+        return {"required": {
+                     "video_file_path": ("STRING",  {"forceInput": True}),
+                     "audio_file_path": ("STRING",  {"forceInput": True}), 
+                     },
+                }
+
+    CATEGORY = "♾️Mixlab/Video"
+
+    OUTPUT_NODE = True
+    FUNCTION = "run" 
+    RETURN_TYPES = ()
+    RETURN_NAMES = ()
+
+    def run(self,video_file_path, audio_file_path):
+
+        output_dir = folder_paths.get_output_directory()
+
+        counter=get_new_counter(output_dir,'video_final_')
+        
+        # 获取文件名和扩展名
+        base, ext = os.path.splitext(video_file_path)
+
+        v_file = f"video_final_{counter:05}{ext}"
+        
+        v_file_path=os.path.join(output_dir, v_file)
+
+        combine_audio_video(audio_file_path,video_file_path,v_file_path)
+
+        previews = [
+            {
+                "filename": v_file,
+                "subfolder": "",
+                "type": "output",
+                "format": get_mime_type(v_file),
+            }
+        ]
+        return {"ui": {"gifs": previews}}
 
 # The code is based on ComfyUI-VideoHelperSuite modification.
 class VideoCombine_Adv:
@@ -433,6 +594,7 @@ class VideoCombine_Adv:
             ffmpeg_formats = ["video/"+x[:-5] for x in folder_paths.get_filename_list("video_formats")]
         else:
             ffmpeg_formats = []
+        # ffmpeg_formats =["video/"+x for x in  ['webm', 'mp4', 'mkv']]
         return {
             "required": {
                 "image_batch": ("IMAGE",),
@@ -453,7 +615,8 @@ class VideoCombine_Adv:
             },
         }
 
-    RETURN_TYPES = ()
+    RETURN_TYPES = ("STRING",)
+    RETURN_NAMES = ("video_file_path",)
     OUTPUT_NODE = True
     CATEGORY = "♾️Mixlab/Video"
     FUNCTION = "run"
@@ -622,7 +785,7 @@ class VideoCombine_Adv:
                 "format": format,
             }
         ]
-        return {"ui": {"gifs": previews}}
+        return {"ui": {"gifs": previews},"result":(file_path,)}
 
 
 class VAEEncodeForInpaint_Frames:
@@ -689,4 +852,113 @@ class VAEEncodeForInpaint_Frames:
             result.append({"samples":t, "noise_mask": (mask_erosion[:,:,:x,:y].round())})
 
 
-        return (result, )
+        return (result, )
+    
+
+
+class GenerateFramesByCount:
+    @classmethod
+    def INPUT_TYPES(cls):
+        
+        return {"required": {
+                    "frames": ('IMAGE',),
+                    "frame_count": ("INT", {"default": 72, "min": 1, "step": 1}), 
+                    "revert" :("BOOLEAN", {"default": True},),
+                     },}
+
+    RETURN_TYPES = ('IMAGE',)
+    RETURN_NAMES = ("frames",)
+
+    FUNCTION = "r"
+    CATEGORY = "♾️Mixlab/Video"
+    # INPUT_IS_LIST = True
+
+    def r(self, frames, frame_count, revert):
+        
+        image_list = [frames[i:i + 1, ...] for i in range(frames.shape[0])]
+
+        image_list=get_frames(frame_count,image_list,revert)
+
+        images = torch.cat(image_list, dim=0)
+
+        return (images,)
+
+
+class scenesNode_:
+    @classmethod
+    def INPUT_TYPES(cls):
+        
+        return {"required": {
+                    "scenes_video": ('SCENE_VIDEO',),
+                     "index": ("INT", {"default": 0, "min": 0, "step": 1}),
+                     
+                     },}
+
+    RETURN_TYPES = ('IMAGE','INT',)
+    RETURN_NAMES = ("frames","count",)
+    # OUTPUT_IS_LIST = (False,)
+
+    FUNCTION = "run"
+    CATEGORY = "♾️Mixlab/Video"
+    INPUT_IS_LIST = True
+
+    def load_video_cv_fallback(self, video, frame_load_cap, skip_first_frames):
+        print('#video',video)
+        try:
+            video_cap = cv2.VideoCapture(video)
+            if not video_cap.isOpened():
+                raise ValueError(f"{video} could not be loaded with cv fallback.")
+            # set video_cap to look at start_index frame
+            images = []
+            total_frame_count = 0
+            frames_added = 0
+            base_frame_time = 1/video_cap.get(cv2.CAP_PROP_FPS)
+            
+            target_frame_time = base_frame_time
+
+            time_offset=0.0
+            while video_cap.isOpened():
+                if time_offset < target_frame_time:
+                    is_returned, frame = video_cap.read()
+                    # if didn't return frame, video has ended
+                    if not is_returned:
+                        break
+                    time_offset += base_frame_time
+                if time_offset < target_frame_time:
+                    continue
+                time_offset -= target_frame_time
+                # if not at start_index, skip doing anything with frame
+                total_frame_count += 1
+                if total_frame_count <= skip_first_frames:
+                    continue
+                # TODO: do whatever operations need to happen, like force_size, etc
+
+                # opencv loads images in BGR format (yuck), so need to convert to RGB for ComfyUI use
+                # follow up: can videos ever have an alpha channel?
+                frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+                # convert frame to comfyui's expected format (taken from comfy's load image code)
+                image = Image.fromarray(frame)
+                image = ImageOps.exif_transpose(image)
+                image = np.array(image, dtype=np.float32) / 255.0
+                image = torch.from_numpy(image)[None,]
+                images.append(image)
+                frames_added += 1
+                # if cap exists and we've reached it, stop processing frames
+                if frame_load_cap > 0 and frames_added >= frame_load_cap:
+                    break
+        finally:
+            video_cap.release()
+        
+        images = torch.cat(images, dim=0)
+       
+        return (images, frames_added,)
+
+    def run(self, scenes_video,index):
+        print('#scenes_video',index,scenes_video)
+        index=index[0]
+        if len(scenes_video) > index:
+            vp=scenes_video[index]
+
+            return self.load_video_cv_fallback(vp,0,0)
+        
+        return ([], 0,)

nodes/edit_mask.py

@@ -0,0 +1,172 @@
+import torch
+from PIL import Image, ImageOps, ImageSequence, ImageFile
+from PIL.PngImagePlugin import PngInfo
+
+import numpy as np
+import os
+import folder_paths 
+import node_helpers
+import hashlib
+
+# Tensor to PIL
+def tensor2pil(image):
+    return Image.fromarray(np.clip(255. * image.cpu().numpy().squeeze(), 0, 255).astype(np.uint8))
+
+# tensor 取hash值
+def tensor_to_hash(tensor):
+    # 将 Tensor 转换为 NumPy 数组
+    np_array = tensor.cpu().numpy()
+    
+    # 将 NumPy 数组转换为字节数据
+    byte_data = np_array.tobytes()
+    
+    # 计算哈希值
+    hash_value = hashlib.md5(byte_data).hexdigest()
+    
+    return hash_value
+
+
+def create_temp_file(image):
+    output_dir = folder_paths.get_temp_directory()
+
+    (
+            full_output_folder,
+            filename,
+            counter,
+            subfolder,
+            _,
+        ) = folder_paths.get_save_image_path('material', output_dir)
+
+    
+    image=tensor2pil(image)
+ 
+    image_file = f"{filename}_{counter:05}.png"
+     
+    image_path=os.path.join(full_output_folder, image_file)
+
+    image.save(image_path,compress_level=4)
+
+    return (image_path,[{
+    "filename": image_file,
+    "subfolder": subfolder,
+    "type": "temp"
+    }])
+
+
+# image - tensor - 文件路径
+# loadImage的方法（ 文件路径 - image-mask ）
+class EditMask:
+
+    def __init__(self):
+        self.image_id = None
+
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required":
+                    {"image": ("IMAGE",), # 表示一个张量
+                     
+                     },
+
+                    "optional":{
+                            "image_update": ("IMAGE_FILE",)
+                        },
+                   
+                }
+
+    CATEGORY = "♾️Mixlab/Mask"
+
+    RETURN_TYPES = ("IMAGE", "MASK")
+    RETURN_NAMES = ("image", "mask")
+
+    FUNCTION = "edit"
+
+    OUTPUT_NODE = True
+
+    def edit(self, image,image_update=None):
+
+        # 根据image输入来判断是否是新的图片
+        if self.image_id==None:
+            self.image_id=tensor_to_hash(image)
+            image_update=None
+        else:
+            image_id=tensor_to_hash(image)
+            if image_id!=self.image_id:
+                image_update=None
+                self.image_id=image_id
+
+
+        image_path=None
+        # print('#image_update',self.image_id,image_update)
+        if image_update==None:
+            print('--')
+        else:
+            if 'images' in image_update:
+                images=image_update['images']
+                filename=images[0]['filename']
+                subfolder=images[0]['subfolder']
+                type=images[0]['type']
+                name, base_dir=folder_paths.annotated_filepath(filename)
+                if type.endswith("output"):
+                    base_dir = folder_paths.get_output_directory() 
+                elif type.endswith("input"):
+                    base_dir = folder_paths.get_input_directory() 
+                elif type.endswith("temp"):
+                    base_dir = folder_paths.get_temp_directory() 
+                #base_dir = folder_paths.get_input_directory()  
+                # print(base_dir,subfolder, name)
+                image_path = os.path.join(base_dir,subfolder, name)
+        
+        if image_path==None:
+            image_path,images=create_temp_file(image)
+
+        print('#image_path',os.path.exists(image_path),image_path)
+        # image_path = folder_paths.get_annotated_filepath(image) #文件名
+        
+        if not os.path.exists(image_path):
+            image_path,images=create_temp_file(image)
+
+
+        img = node_helpers.pillow(Image.open, image_path)
+        
+        output_images = []
+        output_masks = []
+        w, h = None, None
+
+        excluded_formats = ['MPO']
+        
+        for i in ImageSequence.Iterator(img):
+            i = node_helpers.pillow(ImageOps.exif_transpose, i)
+
+            if i.mode == 'I':
+                i = i.point(lambda i: i * (1 / 255))
+            image = i.convert("RGB")
+
+            if len(output_images) == 0:
+                w = image.size[0]
+                h = image.size[1]
+            
+            if image.size[0] != w or image.size[1] != h:
+                continue
+            
+            image = np.array(image).astype(np.float32) / 255.0
+            image = torch.from_numpy(image)[None,]
+            if 'A' in i.getbands():
+                mask = np.array(i.getchannel('A')).astype(np.float32) / 255.0
+                mask = 1. - torch.from_numpy(mask)
+            else:
+                # 尺寸不对，需要按照image来
+                mask = torch.zeros((h, w), dtype=torch.float32, device="cpu")
+
+            output_images.append(image)
+            output_masks.append(mask.unsqueeze(0))
+
+        if len(output_images) > 1 and img.format not in excluded_formats:
+            output_image = torch.cat(output_images, dim=0)
+            output_mask = torch.cat(output_masks, dim=0)
+        else:
+            output_image = output_images[0]
+            output_mask = output_masks[0]
+
+        return {"ui":{"images": images},"result": (output_image, output_mask)}
+
+        # return (output_image, output_mask)

nodes/video_formats/av1-webm.json

@@ -0,0 +1,10 @@
+{
+    "main_pass":
+    [
+        "-n", "-c:v", "libsvtav1",
+        "-pix_fmt", "yuv420p10le",
+        "-crf", "23"
+    ],
+     "extension": "webm",
+     "environment": {"SVT_LOG": "1"}
+}

nodes/video_formats/h264-mp4.json

@@ -0,0 +1,9 @@
+{
+    "main_pass":
+    [
+        "-n", "-c:v", "libx264",
+        "-pix_fmt", "yuv420p",
+        "-crf", "19"
+    ],
+     "extension": "mp4"
+}

nodes/video_formats/h265-mp4.json

@@ -0,0 +1,11 @@
+{
+    "main_pass":
+    [
+        "-n", "-c:v", "libx265",
+        "-pix_fmt", "yuv420p10le",
+        "-preset", "medium",
+        "-crf", "22",
+        "-x265-params", "log-level=quiet"
+    ],
+     "extension": "mp4"
+}

nodes/video_formats/webm.json

@@ -0,0 +1,9 @@
+{
+    "main_pass":
+    [
+        "-n",
+        "-pix_fmt", "yuv420p",
+        "-crf", "23"
+    ],
+     "extension": "webm"
+}

pyproject.toml

@@ -0,0 +1,15 @@
+[project]
+name = "comfyui-mixlab-nodes"
+description = "3D, ScreenShareNode & FloatingVideoNode, SpeechRecognition & SpeechSynthesis, GPT, LoadImagesFromLocal, Layers, Other Nodes, ..."
+version = "0.28.3"
+license = "MIT"
+dependencies = ["numpy", "pyOpenSSL", "watchdog", "opencv-python-headless", "matplotlib", "openai", "simple-lama-inpainting", "clip-interrogator==0.6.0", "transformers>=4.36.0", "lark-parser", "imageio-ffmpeg", "rembg[gpu]", "omegaconf==2.3.0", "Pillow>=9.5.0", "einops==0.7.0", "trimesh>=4.0.5", "huggingface-hub", "scikit-image"]
+
+[project.urls]
+Repository = "https://github.com/shadowcz007/comfyui-mixlab-nodes"
+#  Used by Comfy Registry https://comfyregistry.org
+
+[tool.comfy]
+PublisherId = "shadow"
+DisplayName = "comfyui-mixlab-nodes"
+Icon = ""

requirements.txt

@@ -15,4 +15,6 @@ Pillow>=9.5.0
 einops==0.7.0
 trimesh>=4.0.5
 huggingface-hub
-scikit-image
+scikit-image
+torchaudio
+soundfile

web/index.html

@@ -24,6 +24,7 @@
             margin-left: 5%;
             user-select: none;
             margin-top: 32px;
+            margin-bottom: 120px;
         }
 
         .apps {
@@ -65,9 +66,9 @@
             box-shadow: 0px 0px 10px 10px #fbe9f0
         }
 
-        .card:hover {
+        /* .card:hover {
             box-shadow: 0px 0px 10px 10px #e9fbfa
-        }
+        } */
 
         .card h5 {
             font-size: 14px;
@@ -145,15 +146,6 @@
             margin-top: 4px;
         }
 
-        .status {
-            background: black;
-            color: white;
-            display: flex;
-            width: fit-content;
-            padding: 4px;
-            font-size: 12px;
-
-        }
 
         .description {
             display: flex;
@@ -263,18 +255,44 @@
             margin-top: 12px;
         }
 
+        .run_div {
+            position: fixed;
+            bottom: 28px;
+            display: flex;
+            left: 144px;
+            z-index: 100;
+            /* align-items: center; */
+            /* justify-content: space-around; */
+            /* width: calc(50% - 100px); */
+            flex-direction: column;
+            background-color: #f7f7f7;
+            box-shadow: 3px 3px 8px #cacaca;
+            border-radius: 8px;
+        }
+
+        .status {
+            /* background: #e7e7e7; */
+            color: black;
+            display: flex;
+            width: 120px;
+            padding-left: 11px;
+            font-size: 12px;
+            border-radius: 8px;
+            justify-content: flex-start;
+            align-items: center;
+            height: 48px;
+        }
+
         .run_btn {
             background: black;
             color: white;
-            width: 88px;
-            height: 88px;
-            position: fixed;
-            bottom: 72px;
-            left: calc(50% - 44px);
-            border-radius: 100%;
+            /* width: calc(50% - 100px); */
+            min-width: 200px;
+            max-width: 460px;
+            height: 48px;
+            border-radius: 8px;
             cursor: pointer;
             border: 3px solid;
-            z-index: 100;
         }
 
         button:hover {
@@ -474,8 +492,7 @@
             <a class="link" href="https://www.mixcomfy.com" target="_blank">ComfyUI中文爱好者社区推荐</a>
         </div>
 
-        <a id="login_btn" target="_blank" href="https://discord.gg/xbP2GZF6gn" 
-        style="text-decoration: none;
+        <a id="login_btn" target="_blank" href="https://discord.gg/xbP2GZF6gn" style="text-decoration: none;
         color: black;font-size:12px">
             <svg height="32" aria-hidden="true" viewBox="0 0 16 16" version="1.1" width="32" data-view-component="true"
                 class="octicon octicon-mark-github v-align-middle color-fg-default">
@@ -762,7 +779,6 @@
         async function editImage(image, data) {
             //判断mask是否有输出
             let isMask = data.options.hasMask;
-            console.log(data)
 
             //app
             document.body.querySelector('.app').style.display = 'none'
@@ -772,6 +788,22 @@
             editor.style.display = 'block';
             const iframe = editor.querySelector('iframe');
 
+            const sleep = (t = 1000) => {
+                return new Promise((res, rej) => {
+                    setTimeout(() => {
+                        res(true)
+                    }, t)
+                })
+            }
+
+            //清空 图层
+            const removeAllLayer = () => {
+                let Layers = iframe.contentWindow.Layers;
+                //清空
+                Layers.reset_layers()
+                Layers.refresh_gui()
+            }
+
             // 复原
             const resetLayer = () => {
                 let Layers = iframe.contentWindow.Layers;
@@ -841,17 +873,19 @@
             //自动抠图
             let autoMaskSelect = iframe.contentDocument.getElementById('automask_image_mixlab');
             const select = iframe.contentDocument.getElementById('automask_models_mixlab');
-            let rembgModels = await get_rembg_models()
-            // 遍历模型列表并创建选项
-            for (const model of rembgModels) {
-                const option = document.createElement('option');
-                option.value = model;
-                option.textContent = model;
-                select.appendChild(option);
+            if (select.children.length === 0) {
+                let rembgModels = await get_rembg_models()
+                // 遍历模型列表并创建选项
+                for (const model of rembgModels) {
+                    const option = document.createElement('option');
+                    option.value = model;
+                    option.textContent = model;
+                    select.appendChild(option);
+                }
             }
 
             let autoMaskBtn = iframe.contentDocument.getElementById('automask_image_mixlab');
-            autoMaskBtn.addEventListener('click', async e => {
+            if (!autoMaskBtn.getAttribute('init')) autoMaskBtn.addEventListener('click', async e => {
                 //api请求
                 let base64 = getImageBase64FromLayer()
                 resetLayer()
@@ -866,16 +900,19 @@
 
                 let mb = convertImageToBlackBasedOnAlpha(image)
                 let mask = await createImage(mb)
-                addImage((Layers.get_layers()).length, 'Mask_' + data.id, mask)
+                let id = Layers.auto_increment;
+                addImage(id, 'Mask_' + data.id + id, mask)
             })
 
-
+            autoMaskBtn.setAttribute('init', 1)
             let cancelImageBtn = iframe.contentDocument.getElementById('cancel_image_mixlab');
-            cancelImageBtn.addEventListener('click', e => {
+            if (!cancelImageBtn.getAttribute('init')) cancelImageBtn.addEventListener('click', e => {
                 editor.style.display = 'none';
                 document.body.querySelector('.app').style.display = 'flex'
                 document.body.querySelector('#author').style.display = 'block'
             })
+
+            cancelImageBtn.setAttribute('init', 1)
             // 获取 id 为 "mix" 的 button 元素
             let saveImageBtn = iframe.contentDocument.getElementById('save_image_mixlab');
             saveImageBtn.style = `width: 98px;
@@ -883,7 +920,7 @@
             margin: 0 12px;
             background-color: var(--background-color-active);
             color: var(--text-color-active);`
-            saveImageBtn.addEventListener('click', async e => {
+            if (!saveImageBtn.getAttribute('init')) saveImageBtn.addEventListener('click', async e => {
                 //保存，并更新图片
                 e.preventDefault();
 
@@ -897,6 +934,7 @@
                     tempCanvas.width = dim.width;
                     tempCanvas.height = dim.height;
 
+                    //todo 获取Image更新后的数据(暂不支持image的修改)
 
                     for (const layer of Layers.get_layers()) {
                         if (layer.name !== 'Image_' + data.id) {
@@ -936,10 +974,15 @@
                     // // 获取读取的文件内容，即 Blob 对象
                     let hashId = await calculateImageHash(fileBlob)
 
-                    if (hashId == window._appData.data[data.id].hashId) return;
+                    if (hashId == window._appData.data[data.id].hashId) {
+                        document.body.querySelector('.app').style.display = 'flex'
+                        document.body.querySelector('#author').style.display = 'block'
+                        return;
+                    }
 
+                    //底图
                     const { url: imgurl } = await uploadImage(base64ToBlob(data.options.defaultImage))
-
+                    //mask
                     let { url, name } = await uploadMask(fileBlob, imgurl);
                     // 在这里可以对 Blob 对象进行进一步处理
                     // imageElement.src = url;
@@ -947,8 +990,6 @@
                     window._appData.data[data.id].hashId = hashId;
 
                     // console.log("上传的文件：", url, data.id, name);
-
-
                     //更新图片
                     const canvas = document.createElement("canvas");
                     canvas.width = dim.width;
@@ -956,7 +997,6 @@
 
                     const ctx = canvas.getContext('2d');
 
-
                     const defaultImage = await createImage(data.options.defaultImage)
                     ctx.drawImage(defaultImage, 0, 0, dim.width, dim.height);
 
@@ -973,22 +1013,34 @@
                 document.body.querySelector('#author').style.display = 'block'
             })
 
+            saveImageBtn.setAttribute('init', 1)
 
             var Layers = iframe.contentWindow.Layers;
             // console.log(Layers)
 
             //判断是否已经存在
-            let layers1 = Layers.get_layers()
+            let layers1 = Layers.get_layers();
 
+            //通过layer.link.src 判断是否图片更新
+            console.log(layers1.filter(l => l.name == 'Image_' + data.id)[0]?.link?.currentSrc !== data.options.defaultImage)
+            if (layers1.filter(l => l.name == 'Image_' + data.id)[0]
+                && layers1.filter(l => l.name == 'Image_' + data.id)[0].link.src !== data.options.defaultImage) {
+                //清空图层
+                removeAllLayer();
+                layers1 = [];
+                await sleep()
+            }
+
+            let im = await createImage(data.options.defaultImage)
+            // console.log(layers1, layers1.length)
             if (!layers1.filter(l => l.name == 'Image_' + data.id)[0]) {
-                addImage((Layers.get_layers()).length, 'Image_' + data.id, image)
+                addImage(0, 'Image_' + data.id, im)
             }
 
             if (isMask) {
                 if (!layers1.filter(l => l.name == 'Mask_' + data.id)[0]) {
-                    addMask((Layers.get_layers()).length, 'Mask_' + data.id, image)
+                    addMask(1, 'Mask_' + data.id, im)
                 }
-
             }
 
         }
@@ -1181,7 +1233,7 @@
         }
 
 
-        function createOutputs(outputData, link) {
+        async function createOutputs(outputData, link) {
             const url = new URL(window.location.href);
             const params = new URLSearchParams(url.search);
             const innerApp = params.get("innerApp");
@@ -1302,20 +1354,21 @@
                     "TransparentImage"].includes(node.class_type)) {
 
                     const url = node.options?.defaultImage || window._appData?.icon || base64Df;
-                    let a = document.createElement('a');
+                    let a = document.createElement('div');
                     a.id = `output_${node.id}`
-                    a.setAttribute('data-pswp-width', "200");
-                    a.setAttribute('data-pswp-height', "200");
-                    a.setAttribute('target', "_blank");
-                    a.setAttribute('href', url);
-                    a.setAttribute('title', node.title);
 
-                    let img = new Image();
-                    // img;
-                    img.src = url;
+                    let img = await createImage(url)
+
                     a.appendChild(img)
+                    // a.setAttribute('data-pswp-width', img.naturalWidth);
+                    // a.setAttribute('data-pswp-height', img.naturalHeight);
+                    // a.setAttribute('target', "_blank");
+                    // a.setAttribute('href', url);
+                    // a.setAttribute('title', node.title);
+
                     output_card.appendChild(a);
                     isShowImageFn = true;
+
                 }
 
                 //3d 
@@ -1335,7 +1388,7 @@
                 }
 
                 // video ,gif
-                if (["VHS_VideoCombine", "VideoCombine_Adv"].includes(node.class_type)) {
+                if (["VHS_VideoCombine", "VideoCombine_Adv", "CombineAudioVideo"].includes(node.class_type)) {
 
                     let a = document.createElement('a');
                     a.id = `output_${node.id}`
@@ -1437,8 +1490,9 @@
 
                             if (hashId == window._appData.data[data.id].hashId) return
 
+                            let base64 = await blobToBase64(fileBlob)
+
                             if (data.class_type === 'LoadImagesToBatch') {
-                                let base64 = await blobToBase64(fileBlob)
                                 createBase64ImageForLoadImageToBatch(imageElement, data.id, base64)
                             } else {
                                 let { url, name } = await uploadImage(fileBlob);
@@ -1447,6 +1501,14 @@
                                 window._appData.data[data.id].inputs.image = name;
                                 window._appData.data[data.id].hashId = hashId;
                                 console.log("上传的文件：", url, data.id, name);
+
+                                //更换option里的default image
+                                window._appData.input = Array.from(window._appData.input, inp => {
+                                    if (inp.id === data.id) {
+                                        inp.options.defaultImage = base64;
+                                    }
+                                    return inp
+                                })
                             }
                         }
                     }
@@ -1583,6 +1645,42 @@
         // copyImagesToClipboard(htmlString);
 
 
+        // 上传音频转为base64
+        async function uploadAndConvertAudio(file) {
+            if (!file) {
+                alert('Please select a WAV file.')
+                return
+            }
+
+            if (file.type !== 'audio/wav') {
+                alert('Only WAV files are supported.')
+                return
+            }
+
+            try {
+                const base64Audio = await readFileAsDataURL(file)
+                return base64Audio
+            } catch (error) {
+                console.error('Error reading file:', error)
+                alert('Error reading file.')
+            }
+        }
+
+        function readFileAsDataURL(file) {
+            return new Promise((resolve, reject) => {
+                const reader = new FileReader()
+
+                reader.onload = function (event) {
+                    resolve(event.target.result)
+                }
+
+                reader.onerror = function (error) {
+                    reject(error)
+                }
+
+                reader.readAsDataURL(file)
+            })
+        }
 
         function createInputs(inputData) {
             // Assuming you have an HTML element with the id "container" to hold the UI
@@ -1748,9 +1846,9 @@
                                 //上传，返回url
                                 let { url, name } = await uploadImage(fileBlob, '.' + file.type.split('/')[1])
 
+                                let base64 = await parseImageToBase64(url);
+
                                 if (data.class_type === 'ImagesPrompt_') {
-                                    // 
-                                    let base64 = await parseImageToBase64(url);
                                     uploadContainer.querySelector('.images_prompt_main').src = base64
                                     window._appData.data[data.id].inputs.image_base64 = base64;
                                 } else {
@@ -1760,10 +1858,17 @@
                                     // 在这里可以对 Blob 对象进行进一步处理
                                     imageElement.src = url;
 
-
                                     if (isVideoUpload) {
                                         window._appData.data[data.id].inputs.video = name;
                                     } else {
+                                        //更换option里的default image
+                                        window._appData.input = Array.from(window._appData.input, inp => {
+                                            if (inp.id === data.id) {
+                                                inp.options.defaultImage = base64;
+                                            }
+                                            return inp
+                                        })
+
                                         window._appData.data[data.id].inputs.image = name;
                                     }
 
@@ -2022,7 +2127,53 @@
                     container.appendChild(d);
                 }
 
+                // 加载音频 base64
+                if (['LoadAndCombinedAudio_'].includes(data.class_type)) {
+                    let inpAudio = document.createElement('div');
 
+                    let inputAudio = document.createElement('button');
+
+                    inputAudio.innerText = data.title || 'Upload Audio'
+                    inputAudio.style = `cursor: pointer;
+                        font-weight: 300;
+                        margin: 2px; 
+                        color: var(--descrip-text);
+                        background-color: var(--comfy-input-bg);
+                        border-radius: 8px;
+                        border-color: var(--border-color);
+                        border-style: solid;height: 30px;min-width: 122px;
+                    `;
+
+                    let audioE = document.createElement('audio');
+                    audioE.setAttribute('controls', 'on')
+                    //暂时只支持一个
+                    audioE.src = data.inputs.audios.base64[0];
+
+                    inputAudio.addEventListener('click', e => {
+                        e.preventDefault()
+                        let inp = document.createElement('input')
+                        inp.type = 'file'
+                        inp.style.display = 'none'
+                        inp.addEventListener('change', async e => {
+                            e.preventDefault()
+                            const file = e.target.files[0]
+                            let base64 = await uploadAndConvertAudio(file)
+
+                            audioE.src = base64;
+
+                            window._appData.data[data.id].inputs.audios.base64 = [base64];
+                        })
+
+                        inp.click()
+                        inp.remove()
+                    })
+
+                    inpAudio.appendChild(inputAudio)
+                    inpAudio.appendChild(audioE)
+
+                    container.appendChild(inpAudio);
+
+                }
             });
             return container
         }
@@ -2347,7 +2498,7 @@
             })
         }
 
-        function createUI(data, share = true) {
+        async function createUI(data, share = true) {
             // appData.input, appData.output, appData.seed, share, appData.link
             if (!data) return
             const { input: inputData, output: outputData, data: workflow, seed, seedTitle, link, name } = data;
@@ -2488,30 +2639,44 @@
             }
 
 
-            statusDiv.appendChild(status);
+            // statusDiv.appendChild(status);
             statusDiv.appendChild(seeds);
 
             // 创建输入框
             var input1 = createInputs(inputData)
 
-            var output = createOutputs(outputData, link)
+            var output = await createOutputs(outputData, link)
 
             // 创建提交按钮
+            let submitDiv = document.createElement('div');
+            submitDiv.appendChild(statusDiv);
+
+            let submitDivBtn = document.createElement('div');
+            submitDivBtn.style.display = 'flex'
+            submitDiv.appendChild(submitDivBtn);
+
+            submitDivBtn.appendChild(status);
+
+            submitDiv.className = 'run_div';
             var submitButton = document.createElement('button');
             submitButton.textContent = 'Create';
             submitButton.className = 'run_btn'
 
             // 将所有UI元素添加到页面中
-            leftDiv.appendChild(titleDiv);
-            leftDiv.appendChild(iconDes);
+            // leftDiv.appendChild(titleDiv);
+            // leftDiv.appendChild(iconDes);
             // leftDiv.appendChild(des);
-            leftDiv.appendChild(statusDiv);
+            // leftDiv.appendChild(statusDiv);
             leftDiv.appendChild(input1);
 
-            if (typeof (data.data) == 'object') mainDiv.appendChild(submitButton);
+            submitDivBtn.appendChild(submitButton);
+            if (typeof (data.data) == 'object') mainDiv.appendChild(submitDiv);
 
             rightDiv.appendChild(output);
 
+            // mainDiv.appendChild(titleDiv);
+            // mainDiv.appendChild(iconDes);
+
             mainDiv.appendChild(leftDetails);
             mainDiv.appendChild(rightDiv);
 
@@ -2859,7 +3024,7 @@
         async function createApp(appData, share = true) {
             // console.log(appData)
             // 使用示例：
-            var ui = createUI(appData, share);
+            var ui = await createUI(appData, share);
 
             // 更新标题
             ui.title.update(appData.name || 'Mixlab APP');
@@ -2999,7 +3164,8 @@
             const lightbox = new PhotoSwipeLightbox({
                 gallery: '.output_card',
                 children: 'a',
-                pswpModule: () => import('/extensions/comfyui-mixlab-nodes/lib/photoswipe.esm.min.js')
+                pswpModule: () => import('/extensions/comfyui-mixlab-nodes/lib/photoswipe.esm.min.js'),
+                // appendToEl: document.querySelector('#result')
             });
             lightbox.on('uiRegister', function () {
                 lightbox.pswp.ui.registerElement({
@@ -3044,8 +3210,8 @@
             if (appData.author) {
                 let div = document.body.querySelector('#author');
                 if (appData.author.link) div.href = appData.author.link
-                div.style = `z-index:20;display: flex;flex-direction: column;position: fixed;bottom: 12px;right: 24px;cursor: pointer;text-decoration: none;color: black;`
-                div.innerHTML = `<p style="font-size:12px">Author:</p>
+                div.style = `z-index:20;display: flex;flex-direction: column;position: fixed;bottom: 24px;left: 24px;cursor: pointer;text-decoration: none;color: black;`
+                div.innerHTML = `<p style="font-size:12px;margin: 8px 0;">Author:</p>
                 <div style="display: flex;"> <img style="width:32px;height:32px;border-radius: 100%;" 
                 src="${appData.author.avatar || base64Df}"/>
                 <p style="margin-left:8px;font-size:12px;font-weight:800">${appData.author.name || '-'}</p></div>`

web/javascript/app_mixlab.js

@@ -2,6 +2,8 @@ import { app } from '../../../scripts/app.js'
 import { $el } from '../../../scripts/ui.js'
 import { api } from '../../../scripts/api.js'
 
+import { td_bg } from './td_background.js'
+console.log('td_bg', td_bg)
 //本机安装的插件节点全集
 window._nodesAll = null
 
@@ -185,6 +187,25 @@ async function extractInputAndOutputData (
         if (node.type == 'Color') {
         }
 
+        // 语音输入的支持
+        if (node.type == 'LoadAndCombinedAudio_') {
+          // if (
+          //   data[id].widgets_values &&
+          //   data[id].widgets_values[0] &&
+          //   data[id].widgets_values[0].base64 &&
+          //   data[id].widgets_values[0].base64.length > 0
+          // ) {
+          //   options.defaultBase64 = data[id].widgets_values[0].base64
+          // }
+
+          input[inputIds.indexOf(id)] = {
+            ...data[id],
+            title: node.title,
+            id,
+            options
+          }
+        }
+
         if (node.type === 'LoadImage') {
           // loadImage的mask支持
           let output = node.outputs.filter(ot => ot.type == 'MASK')[0]
@@ -397,11 +418,11 @@ async function save (json, download = false, showInfo = true) {
 
 function getInputsAndOutputs () {
   const inputs =
-      `LoadImage LoadImagesToBatch ImagesPrompt_ VHS_LoadVideo CLIPTextEncode PromptSlide TextInput_ Color FloatSlider IntNumber CheckpointLoaderSimple LoraLoader`.split(
+      `LoadImage LoadImagesToBatch ImagesPrompt_ LoadAndCombinedAudio_ VHS_LoadVideo CLIPTextEncode PromptSlide TextInput_ Color FloatSlider IntNumber CheckpointLoaderSimple LoraLoader`.split(
         ' '
       ),
     outputs =
-      `SaveTripoSRMesh,PreviewImage,SaveImage,TransparentImage,ShowTextForGPT,VHS_VideoCombine,VideoCombine_Adv,Image Save,SaveImageAndMetadata_,ClipInterrogator`.split(
+      `SaveTripoSRMesh,PreviewImage,SaveImage,TransparentImage,ShowTextForGPT,CombineAudioVideo,VHS_VideoCombine,VideoCombine_Adv,Image Save,SaveImageAndMetadata_,ClipInterrogator`.split(
         ','
       )
 
@@ -503,6 +524,21 @@ app.registerExtension({
           }
         })
 
+        //td bg
+        const tdBG = document.createElement('button')
+        tdBG.innerText = 'TDBG'
+        tdBG.style = style
+        tdBG.style.marginLeft = '12px'
+
+        tdBG.addEventListener('click', () => {
+          td_bg.toggle()
+          if (td_bg.running) {
+            tdBG.style.background = 'yellow'
+          } else {
+            tdBG.style.background = 'transparent'
+          }
+        })
+
         // author
         let author = document.createElement('div')
         // author.style=`display: flex`
@@ -659,6 +695,7 @@ app.registerExtension({
 
         btns.appendChild(btn)
         btns.appendChild(download)
+        btns.appendChild(tdBG)
 
         document.body.appendChild(widget.div)
         this.addCustomWidget(widget)
@@ -687,9 +724,8 @@ app.registerExtension({
           }
 
           const div = this.widgets.filter(w => w.div)[0].div
-          Array.from(
-            div.querySelectorAll('button'),
-            b => (b.style.background = 'yellow')
+          Array.from(div.querySelectorAll('button'), b =>
+            b.innerText != 'TDBG' ? (b.style.background = 'yellow') : ''
           )
         } catch (error) {}
       }

web/javascript/audio_mixlab.js

@@ -396,3 +396,207 @@ app.registerExtension({
     }
   }
 })
+
+// 上传音频转为base64
+async function uploadAndConvertAudio (file) {
+  if (!file) {
+    alert('Please select a WAV file.')
+    return
+  }
+
+  if (file.type !== 'audio/wav') {
+    alert('Only WAV files are supported.')
+    return
+  }
+
+  try {
+    const base64Audio = await readFileAsDataURL(file)
+    return base64Audio
+  } catch (error) {
+    console.error('Error reading file:', error)
+    alert('Error reading file.')
+  }
+}
+
+function readFileAsDataURL (file) {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader()
+
+    reader.onload = function (event) {
+      resolve(event.target.result)
+    }
+
+    reader.onerror = function (error) {
+      reject(error)
+    }
+
+    reader.readAsDataURL(file)
+  })
+}
+
+const createInputAudioForBatch = (base64, widget) => {
+  // Create an audio element
+  let audio = document.createElement('audio')
+  audio.src = base64
+  audio.controls = true
+  audio.style = 'width: 120px; display: block'
+
+  // Create a delete button
+  let deleteButton = document.createElement('button')
+  deleteButton.textContent = 'Delete'
+  deleteButton.style = 'margin-left: 10px;'
+
+  // Create a container for the audio and delete button
+  let container = document.createElement('div')
+  container.appendChild(audio)
+  container.appendChild(deleteButton)
+  container.style= `display: flex`
+
+  // Add event listener for the delete button
+  deleteButton.addEventListener('click', e => {
+    let newValue = []
+    let items = widget.value?.base64 || []
+    for (const v of items) {
+      if (v != base64) newValue.push(v)
+    }
+    widget.value.base64 = newValue
+    container.remove()
+  })
+
+  return container
+}
+
+app.registerExtension({
+  name: 'Mixlab.Comfy.LoadAndCombinedAudio_',
+  async getCustomWidgets (app) {
+    return {
+      AUDIOBASE64 (node, inputName, inputData, app) {
+        // console.log('##node', node)
+        const widget = {
+          value: {
+            base64: []
+          }, // 不能[x,x,x]
+          type: inputData[0], // the type
+          name: inputName, // the name, slice
+          size: [128, 32], // a default size
+          draw (ctx, node, width, y) {},
+          computeSize (...args) {
+            return [128, 122] // a method to compute the current size of the widget
+          }
+          // serializeValue (nodeId, widgetIndex) {
+          //   return widget.value
+          // },
+        }
+        //  widget.something = something;          // maybe adds stuff to it
+        node.addCustomWidget(widget) // adds it to the node
+        return widget // and returns it.
+      }
+    }
+  },
+
+  async beforeRegisterNodeDef (nodeType, nodeData, app) {
+    if (nodeType.comfyClass == 'LoadAndCombinedAudio_') {
+      const orig_nodeCreated = nodeType.prototype.onNodeCreated
+
+      nodeType.prototype.onNodeCreated = function () {
+        orig_nodeCreated?.apply(this, arguments)
+
+        let audiosWidget = this.widgets.filter(w => w.name == 'audios')[0]
+
+        const widget = {
+          type: 'div',
+          name: 'audio_base64',
+          draw (ctx, node, widget_width, y, widget_height) {
+            Object.assign(
+              this.div.style,
+              get_position_style(ctx, widget_width, 44, node.size[1])
+            )
+          },
+          serialize: false
+        }
+
+        widget.div = $el('div', {})
+
+        document.body.appendChild(widget.div)
+
+        let audioPreview = document.createElement('div')
+        let audiosDiv = document.createElement('div') //显示图片
+        audiosDiv.className = 'audios_preview'
+        audiosDiv.style = `width: calc(100% - 14px);
+        display: flex;
+        flex-wrap: wrap;
+        padding: 7px;    justify-content: space-between;
+        align-items: center;`
+
+        const btn = document.createElement('button')
+        btn.innerText = 'Upload Audio'
+
+        btn.style = `cursor: pointer;
+        font-weight: 300;
+        margin: 2px; 
+        color: var(--descrip-text);
+        background-color: var(--comfy-input-bg);
+        border-radius: 8px;
+        border-color: var(--border-color);
+        border-style: solid;height: 30px;min-width: 122px;
+       `
+
+        btn.addEventListener('click', e => {
+          e.preventDefault()
+          let inputAudio = document.createElement('input')
+          inputAudio.type = 'file'
+          inputAudio.style.display = 'none'
+          inputAudio.addEventListener('change', async e => {
+            e.preventDefault()
+            const file = e.target.files[0]
+            let base64 = await uploadAndConvertAudio(file)
+            if (!audiosWidget.value) audiosWidget.value = { base64: [] }
+            audiosWidget.value.base64.push(base64)
+
+            let a = createInputAudioForBatch(base64, audiosWidget)
+            audiosDiv.appendChild(a)
+          })
+
+          inputAudio.click()
+          inputAudio.remove()
+        })
+
+        widget.div.appendChild(audioPreview)
+        audioPreview.appendChild(audiosDiv)
+        audioPreview.appendChild(btn)
+        // audioPreview.appendChild(inputAudio)
+
+        this.addCustomWidget(widget)
+
+        // document.addEventListener('wheel', handleMouseWheel)
+
+        const onRemoved = this.onRemoved
+        this.onRemoved = () => { 
+          widget.div.remove()
+          try {
+            // document.removeEventListener('wheel', handleMouseWheel)
+          } catch (error) {
+            console.log(error)
+          }
+
+          return onRemoved?.()
+        }
+
+        this.serialize_widgets = true //需要保存参数
+      }
+    }
+  },
+  async loadedGraphNode (node, app) {
+    if (node.type === 'LoadAndCombinedAudio_') {
+      // await sleep(0)
+      let audiosWidget = node.widgets.filter(w => w.name === 'audios')[0]
+      let audioPreview = node.widgets.filter(w => w.name == 'audio_base64')[0]
+
+      let pre = audioPreview.div.querySelector('.audios_preview')
+      for (const d of audiosWidget.value?.base64 || []) {
+        let im = createInputAudioForBatch(d, audiosWidget)
+        pre.appendChild(im)
+      }
+    }
+  }
+})

web/javascript/chat.js

@@ -2,7 +2,7 @@ async function* completion (url, messages, controller) {
   let data = {
     model: 'gpt-3.5-turbo-16k',
     messages,
-    temperature: 0.6,
+    temperature: 0.05,
     stream: true
   }
   // if (imageNode) {

web/javascript/checkVersion_mixlab.js

@@ -3,7 +3,7 @@ import { app } from '../../../scripts/app.js'
 const repoOwner = 'shadowcz007' // 替换为仓库的所有者
 const repoName = 'comfyui-mixlab-nodes' // 替换为仓库的名称
 
-const version = 'v0.28.0'
+const version = 'v0.30.0'
 
 fetch(`https://api.github.com/repos/${repoOwner}/${repoName}/releases/latest`)
   .then(response => response.json())

web/javascript/sound-lab.js

@@ -0,0 +1,295 @@
+import { app } from '../../../scripts/app.js'
+import { api } from '../../../scripts/api.js'
+import { ComfyWidgets } from '../../../scripts/widgets.js'
+import { $el } from '../../../scripts/ui.js'
+
+import WaveSurfer from 'https://cdn.jsdelivr.net/npm/wavesurfer.js@7/dist/wavesurfer.esm.js'
+
+function get_position_style (ctx, widget_width, y, node_height) {
+  const MARGIN = 4 // the margin around the html element
+
+  /* Create a transform that deals with all the scrolling and zooming */
+  const elRect = ctx.canvas.getBoundingClientRect()
+  const transform = new DOMMatrix()
+    .scaleSelf(
+      elRect.width / ctx.canvas.width,
+      elRect.height / ctx.canvas.height
+    )
+    .multiplySelf(ctx.getTransform())
+    .translateSelf(MARGIN, MARGIN + y)
+
+  return {
+    transformOrigin: '0 0',
+    transform: transform,
+    left: `0`,
+    top: '0',
+    cursor: 'pointer',
+    position: 'absolute',
+    maxWidth: `${widget_width - MARGIN * 2}px`,
+    // maxHeight: `${node_height - MARGIN * 2}px`, // we're assuming we have the whole height of the node
+    width: `${widget_width - MARGIN * 2}px`,
+    // height: `${node_height * 0.3 - MARGIN * 2}px`,
+    // background: '#EEEEEE',
+    display: 'flex',
+    flexDirection: 'column',
+    // alignItems: 'center',
+    justifyContent: 'space-around'
+  }
+}
+
+//把文件转为url访问
+const parseUrl = data => {
+  let { filename, subfolder, type, prompt } = data
+  return {
+    url: api.apiURL(
+      `/view?filename=${encodeURIComponent(
+        filename
+      )}&type=${type}&subfolder=${subfolder}${app.getPreviewFormatParam()}${app.getRandParam()}`
+    ),
+    prompt
+  }
+}
+
+const createWaveSurfer = (wavesurfer, id,url) => {
+  // Create an instance of WaveSurfer
+  if (wavesurfer) {
+    wavesurfer.destroy()
+  }
+  wavesurfer = WaveSurfer.create({
+    container: '#' + id,
+    waveColor: 'rgb(200, 0, 200)',
+    progressColor: 'rgb(100, 0, 100)',
+    // Set a bar width
+    barWidth: 10,
+    // Optionally, specify the spacing between bars
+    barGap: 2,
+    // And the bar radius
+    barRadius: 6,
+    url
+  })
+
+  wavesurfer._auto = true
+
+  // 监听播放结束事件，重新开始播放以实现循环播放
+  wavesurfer.on('finish', function () {
+    // console.log(wavesurfer)
+    if (wavesurfer._auto) wavesurfer.play()
+  })
+
+  wavesurfer.on('interaction', () => {
+    wavesurfer._auto = false
+    if (!wavesurfer.isPlaying()) wavesurfer.play()
+  })
+
+  // 获取当前播放时间的峰值
+  wavesurfer.on('audioprocess', () => {
+    if (wavesurfer.isPlaying()&&wavesurfer.getDecodedData()) {
+      const channelData = wavesurfer.getDecodedData().getChannelData(0); 
+      const currentTime = wavesurfer.getCurrentTime()
+      // console.log(wavesurfer)
+      const sampleRate = wavesurfer.getDecodedData().sampleRate
+
+      // 定义要分析的时间窗口（例如1秒）
+      const windowSize = 1
+      const startSample = Math.floor(currentTime * sampleRate)
+      const endSample = Math.min(
+        startSample + windowSize * sampleRate,
+        channelData.length
+      )
+
+      let peak = 0
+      for (let i = startSample; i < endSample; i++) {
+        const value = Math.abs(channelData[i])
+        if (value > peak) {
+          peak = value
+        }
+      }
+      // console.log('Current Peak:', peak)
+    }
+  })
+
+  return wavesurfer
+}
+
+//更新gui
+function updateWaveWidgetValue (widgets, id, url, prompt, wavesurfer) {
+  let widget = widgets.filter(w => w.name == 'AudioPlay')[0]
+  // 手动更新widget值
+  widget.value = [url, prompt]
+
+  if (widget.div) {
+    widget.div.querySelector('.wave').id = `AudioPlay_${id}`
+  }
+
+  wavesurfer = createWaveSurfer(wavesurfer, `AudioPlay_${id}`,url)
+
+  wavesurfer.on('ready', duration => {
+    console.log('Audio duration: ' + duration + ' seconds')
+    if (widget.div) {
+      widget.div.setAttribute('data-url', url)
+      widget.div.querySelector('.link').setAttribute('href', url)
+      widget.div.querySelector(
+        '.info'
+      ).innerHTML = `<span style="font-size: 12px;
+      margin: 8px;">${duration.toFixed(
+        2
+      )} seconds</span> <br><span style="font-size: 14px;">${prompt||''}</span> <br>`
+    }
+  })
+  
+  
+  wavesurfer.load(url)
+  // console.log('updateWaveWidgetValue' ,url,wavesurfer)
+  return wavesurfer
+}
+
+app.registerExtension({
+  name: 'SoundLab.AudioPlay',
+  async beforeRegisterNodeDef (nodeType, nodeData, app) {
+    if (nodeType.comfyClass == 'AudioPlay') {
+      let that = this
+      // console.log('that', that)
+
+      const orig_nodeCreated = nodeType.prototype.onNodeCreated
+      nodeType.prototype.onNodeCreated = function () {
+        orig_nodeCreated?.apply(this, arguments)
+
+        const widget = {
+          type: 'div',
+          name: 'AudioPlay',
+          draw (ctx, node, widget_width, y, widget_height) {
+            Object.assign(
+              this.div.style,
+              get_position_style(ctx, widget_width, y, node.size[1])
+            )
+          }
+        }
+
+        // console.log('AudioPlay nodeData', this)
+        widget.div = $el('div', {})
+
+        document.body.appendChild(widget.div)
+
+        // wave
+        const waveDiv = document.createElement('div')
+        waveDiv.className = 'wave'
+        waveDiv.style.minHeight = '172px'
+        widget.div.appendChild(waveDiv)
+
+        //prompt 相关信息展示
+        const infoDiv = document.createElement('div')
+        infoDiv.className = 'info'
+        infoDiv.style.marginBottom = '20px'
+        widget.div.appendChild(infoDiv)
+
+        // 按钮的区域
+        let btns = document.createElement('div')
+        btns.className = 'btns'
+        btns.style = `display: flex;
+        width: 100%;
+        justify-content: space-between;`
+        widget.div.appendChild(btns)
+
+        //play button
+        const playBtn = document.createElement('a')
+        playBtn.innerText = 'Play/Pause'
+
+        playBtn.style = `
+        display: flex;
+        padding: 4px 15px;
+        background-color: var(--comfy-input-bg);
+        border-radius: 8px;
+        border-color: var(--border-color);
+        border-style: solid;
+        color: var(--descrip-text);
+        text-decoration: none;
+        border-radius: 5px;
+        transition: background-color 0.3s ease 0s;
+          `
+
+        playBtn.addEventListener('click', e => {
+          e.preventDefault()
+          if (that[`wavesurfer_${this.id}`]) {
+            that[`wavesurfer_${this.id}`]?.playPause()
+            that[`wavesurfer_${this.id}`]._auto = true
+          }
+        })
+        btns.appendChild(playBtn)
+
+        const urlLink = document.createElement('a')
+        urlLink.className = 'link'
+        urlLink.innerText = 'URL'
+        urlLink.setAttribute('target', '_blank')
+        urlLink.style = `display: flex;
+        padding: 4px 15px;
+        background-color: var(--comfy-input-bg);
+        border-radius: 8px;
+        border-color: var(--border-color);
+        border-style: solid;
+        color: var(--descrip-text);
+        text-decoration: none;
+        border-radius: 5px;
+        transition: background-color 0.3s ease 0s;`
+        // urlLink.style.minHeight = '200px'
+        btns.appendChild(urlLink)
+
+
+        //todo 导出视频 that[`wavesurfer_${this.id}`].renderer.exportImage('image/png',1,'dataURL')
+        // https://github.com/diffusion-studio/ffmpeg-js
+
+
+        this.addCustomWidget(widget)
+
+        const onRemoved = this.onRemoved
+        this.onRemoved = () => {
+          widget.div.remove()
+          return onRemoved?.()
+        }
+
+        this.size = [this.size[0], 280]
+        this.serialize_widgets = true //需保存widget的值
+      }
+
+      const onExecuted = nodeType.prototype.onExecuted
+      nodeType.prototype.onExecuted = function (message) {
+        onExecuted?.apply(this, arguments)
+        const audio = message.audio
+        console.log('#onExecuted', `AudioPlay_${this.id}`, message,audio)
+        try {
+          let { url, prompt } = parseUrl(audio[0])
+
+          that[`wavesurfer_${this.id}`] = updateWaveWidgetValue(
+            this.widgets,
+            this.id,
+            url,
+            prompt,
+            that[`wavesurfer_${this.id}`]
+          )
+
+          that[`wavesurfer_${this.id}`]?.playPause()
+        } catch (error) {
+          console.log(error)
+        }
+      }
+    }
+  },
+  async loadedGraphNode (node, app) {
+    if (node.type === 'AudioPlay') {
+      let widget = node.widgets.filter(w => w.name == 'AudioPlay')[0]
+
+      if (widget.value) {
+        let [url, prompt] = widget.value
+
+        this[`wavesurfer_${node.id}`] = updateWaveWidgetValue(
+          node.widgets,
+          node.id,
+          url,
+          prompt,
+          this[`wavesurfer_${node.id}`]
+        )
+      }
+
+      console.log('#loadedGraphNode', node)
+    }
+  }
+})

web/javascript/td_background.js

@@ -0,0 +1,323 @@
+// touchdesigner的背景效果，把appinfo的输出，选择一张图片作为背景
+
+window._bg_img = null
+
+/**
+ * draws the back canvas (the one containing the background and the connections)
+ * @method drawBackCanvas
+ **/
+LGraphCanvas.prototype.drawBackCanvas = function () {
+  var canvas = this.bgcanvas
+  if (
+    canvas.width != this.canvas.width ||
+    canvas.height != this.canvas.height
+  ) {
+    canvas.width = this.canvas.width
+    canvas.height = this.canvas.height
+  }
+
+  if (!this.bgctx) {
+    this.bgctx = this.bgcanvas.getContext('2d')
+  }
+  var ctx = this.bgctx
+  if (ctx.start) {
+    ctx.start()
+  }
+
+  var viewport = this.viewport || [0, 0, ctx.canvas.width, ctx.canvas.height]
+
+  //clear
+  if (this.clear_background) {
+    ctx.clearRect(viewport[0], viewport[1], viewport[2], viewport[3])
+  }
+
+  //show subgraph stack header
+  if (this._graph_stack && this._graph_stack.length) {
+    ctx.save()
+    var parent_graph = this._graph_stack[this._graph_stack.length - 1]
+    var subgraph_node = this.graph._subgraph_node
+    ctx.strokeStyle = subgraph_node.bgcolor
+    ctx.lineWidth = 10
+    ctx.strokeRect(1, 1, canvas.width - 2, canvas.height - 2)
+    ctx.lineWidth = 1
+    ctx.font = '40px Arial'
+    ctx.textAlign = 'center'
+    ctx.fillStyle = subgraph_node.bgcolor || '#AAA'
+    var title = ''
+    for (var i = 1; i < this._graph_stack.length; ++i) {
+      title += this._graph_stack[i]._subgraph_node.getTitle() + ' >> '
+    }
+    ctx.fillText(title + subgraph_node.getTitle(), canvas.width * 0.5, 40)
+    ctx.restore()
+  }
+
+  var bg_already_painted = false
+  if (this.onRenderBackground) {
+    bg_already_painted = this.onRenderBackground(canvas, ctx)
+  }
+
+  //reset in case of error
+  if (!this.viewport) {
+    ctx.restore()
+    ctx.setTransform(1, 0, 0, 1, 0, 0)
+  }
+  this.visible_links.length = 0
+
+  if (this.graph) {
+    //apply transformations
+    ctx.save()
+    this.ds.toCanvasContext(ctx)
+
+    //render BG
+    if (
+      this.ds.scale < 1 &&
+      !bg_already_painted &&
+      this.clear_background_color
+    ) {
+      ctx.fillStyle = this.clear_background_color
+      ctx.fillRect(
+        this.visible_area[0],
+        this.visible_area[1],
+        this.visible_area[2],
+        this.visible_area[3]
+      )
+    }
+
+    // 主要修改
+    if (this.background_image && this.ds.scale > 0.5 && !bg_already_painted) {
+      if (this.zoom_modify_alpha) {
+        //使得 alpha 越接近0时变化越缓慢。
+        let alpha = (1.0 - 0.5 / this.ds.scale) * this.editor_alpha
+        ctx.globalAlpha = Math.min(Math.max(0, Math.sqrt(alpha)), 1)
+        // console.log((1.0 - 0.5 / this.ds.scale) * this.editor_alpha)
+      } else {
+        ctx.globalAlpha = this.editor_alpha
+      }
+      ctx.imageSmoothingEnabled = ctx.imageSmoothingEnabled = false // ctx.mozImageSmoothingEnabled =
+      if (!this._bg_img || this._bg_img.name != this.background_image) {
+        this._bg_img = new Image()
+        this._bg_img.name = this.background_image
+        this._bg_img.src = this.background_image
+        var that = this
+        this._bg_img.onload = function () {
+          that.draw(true, true)
+        }
+      }
+
+      var pattern = null
+      if (this._pattern == null && this._bg_img.width > 0) {
+        pattern = ctx.createPattern(this._bg_img, 'repeat')
+        this._pattern_img = this._bg_img
+        this._pattern = pattern
+      } else {
+        pattern = this._pattern
+      }
+
+      if (pattern) {
+        ctx.fillStyle = pattern
+        ctx.fillRect(
+          this.visible_area[0],
+          this.visible_area[1],
+          this.visible_area[2],
+          this.visible_area[3]
+        )
+        ctx.fillStyle = 'transparent'
+      }
+
+      ctx.globalAlpha = 1.0
+      ctx.imageSmoothingEnabled = ctx.imageSmoothingEnabled = true //= ctx.mozImageSmoothingEnabled
+    }
+
+    //groups
+    if (this.graph._groups.length && !this.live_mode) {
+      this.drawGroups(canvas, ctx)
+    }
+
+    if (this.onDrawBackground) {
+      this.onDrawBackground(ctx, this.visible_area)
+    }
+    if (this.onBackgroundRender) {
+      //LEGACY
+      console.error(
+        'WARNING! onBackgroundRender deprecated, now is named onDrawBackground '
+      )
+      this.onBackgroundRender = null
+    }
+
+    //DEBUG: show clipping area
+    //ctx.fillStyle = "red";
+    //ctx.fillRect( this.visible_area[0] + 10, this.visible_area[1] + 10, this.visible_area[2] - 20, this.visible_area[3] - 20);
+
+    //bg
+    if (this.render_canvas_border) {
+      ctx.strokeStyle = '#235'
+      ctx.strokeRect(0, 0, canvas.width, canvas.height)
+    }
+
+    if (this.render_connections_shadows) {
+      ctx.shadowColor = '#000'
+      ctx.shadowOffsetX = 0
+      ctx.shadowOffsetY = 0
+      ctx.shadowBlur = 6
+    } else {
+      ctx.shadowColor = 'rgba(0,0,0,0)'
+    }
+
+    //draw connections
+    if (!this.live_mode) {
+      this.drawConnections(ctx)
+    }
+
+    ctx.shadowColor = 'rgba(0,0,0,0)'
+
+    //restore state
+    ctx.restore()
+  }
+
+  if (ctx.finish) {
+    ctx.finish()
+  }
+
+  this.dirty_bgcanvas = false
+  this.dirty_canvas = true //to force to repaint the front canvas with the bgcanvas
+}
+
+function imgToCanvasBase64 (img) {
+  const canvas = document.createElement('canvas')
+  const ctx = canvas.getContext('2d')
+  canvas.width = img.width
+  canvas.height = img.height
+  ctx.drawImage(img, 0, 0)
+  const base64 = canvas.toDataURL('image/png')
+
+  return base64
+}
+
+// 使用示例
+function convertImageToBase64 (img) {
+  //   const img = new Image()
+  //   img.src = 'path/to/your/image.jpg' // 替换为你的图片路径
+  // console.log('convertImageToBase64',img)
+  try {
+    const base64 = imgToCanvasBase64(img)
+    return base64
+  } catch (error) {
+    console.error(error)
+  }
+}
+
+function getInputsAndOutputs () {
+  const outputs =
+    `PreviewImage,SaveImage,TransparentImage,VHS_VideoCombine,VideoCombine_Adv,Image Save,SaveImageAndMetadata_`.split(
+      ','
+    )
+
+  let outputsId = []
+
+  for (let node of app.graph._nodes) {
+    if (outputs.includes(node.type)) {
+      outputsId.push(node.id)
+    }
+  }
+
+  return outputsId
+}
+
+function getRandomElement (arr) {
+  const randomIndex = Math.floor(Math.random() * arr.length)
+  return arr[randomIndex]
+}
+
+async function getBG () {
+  var outputs = []
+
+  for (let id of app.graph
+    .getNodeById(50)
+    .widgets.filter(w => w.name === 'output_ids')[0]
+    .value.split('\n')) {
+    if (getInputsAndOutputs().map(Number).includes(Number(id))) {
+      if (app.graph.getNodeById(id).imgs && app.graph.getNodeById(id).imgs[0]) {
+        let b = convertImageToBase64(app.graph.getNodeById(id).imgs[0])
+        // console.log(b)
+        outputs.push(b)
+      }
+    }
+  }
+
+  var BACKGROUND_IMAGE = getRandomElement(outputs),
+    CLEAR_BACKGROUND_COLOR = 'rgba(0,0,0,0.9)'
+
+  if (!window._bg_img) {
+    window._bg_img = app.canvas._bg_img.src
+  }
+  // let img=new Image();
+  // img.src=BACKGROUND_IMAGE;
+
+  //去掉透明度过度
+  // app.canvas.zoom_modify_alpha=false;
+  //整体透明度
+  app.canvas.editor_alpha = 1.1
+  //   app.canvas._pattern=ctx.createPattern(img, "no-repeat");
+  app.canvas.updateBackground(BACKGROUND_IMAGE, CLEAR_BACKGROUND_COLOR)
+  app.canvas.draw(true, true)
+}
+
+class BgRunner {
+  constructor () {
+    this.intervalId = null
+    this.running = false
+  }
+
+  // 要运行的方法
+  bg () {
+    console.log('方法bg正在运行')
+    getBG()
+  }
+
+  // 启动bg方法每秒运行一次
+  start () {
+    if (!this.running) {
+      this.intervalId = setInterval(() => this.bg(), 1500)
+      this.running = true
+    }
+  }
+
+  // 停止bg方法的运行
+  stop () {
+    if (this.running) {
+      clearInterval(this.intervalId)
+      this.intervalId = null
+      this.running = false
+
+      if (window._bg_img) {
+        var BACKGROUND_IMAGE = window._bg_img,
+          CLEAR_BACKGROUND_COLOR = 'rgba(0,0,0,1)'
+        app.canvas.editor_alpha = 1
+
+        app.canvas.updateBackground(BACKGROUND_IMAGE, CLEAR_BACKGROUND_COLOR)
+        app.canvas.draw(true, true)
+      }
+    }
+  }
+
+  // 切换start和stop
+  toggle () {
+    if (this.running) {
+      this.stop()
+    } else {
+      this.start()
+    }
+  }
+
+  // 获取运行状态
+  isRunning () {
+    return this.running
+  }
+}
+
+// 示例用法
+// const runner = new BgRunner();
+// runner.start();
+// setTimeout(() => runner.stop(), 5000);
+
+export const td_bg = new BgRunner()

web/javascript/ui_mixlab.js

@@ -26,7 +26,35 @@ function showTextByLanguage (key, json) {
 }
 
 //系统prompt
-const systemPrompt = `You are a prompt creator, your task is to create prompts for the user input request, the prompts are image descriptions that include keywords for (an adjective, type of image, framing/composition, subject, subject appearance/action, environment, lighting situation, details of the shoot/illustration, visuals aesthetics and artists), brake keywords by comas, provide high quality, non-verboose, coherent, brief, concise, and not superfluous prompts, the subject from the input request must be included verbatim on the prompt,the prompt is english`
+// const systemPrompt = `You are a prompt creator, your task is to create prompts for the user input request, the prompts are image descriptions that include keywords for (an adjective, type of image, framing/composition, subject, subject appearance/action, environment, lighting situation, details of the shoot/illustration, visuals aesthetics and artists), brake keywords by comas, provide high quality, non-verboose, coherent, brief, concise, and not superfluous prompts, the subject from the input request must be included verbatim on the prompt,the prompt is english`
+
+let tool ={
+  "name": "create_prompt",
+  "description": "Create a prompt with a given subject, content, and style based on user input for image descriptions.",
+  "parameter": {
+    "type": "object",
+    "properties": {
+      "subject": {
+        "type": "string",
+        "description": "The subject of the prompt, included verbatim from the input request.",
+        "required": true
+      },
+      "content": {
+        "type": "string",
+        "description": "The content of the prompt, primarily focusing on the scene and objects, including keywords for adjective, type of image, framing/composition, subject appearance/action, and environment.",
+        "required": true
+      },
+      "style": {
+        "type": "string",
+        "description": "The style of the prompt, including lighting situation, details of the shoot/illustration, visual aesthetics, and artists. Ensure it is high quality, non-verbose, coherent, brief, concise, and not superfluous.",
+        "required": true
+      }
+    }
+  }
+}
+
+const systemPrompt=`You are a helpful assistant with access to the following functions. Use them if required - ${JSON.stringify(tool,null,2)}`
+
 
 if (!localStorage.getItem('_mixlab_system_prompt')) {
   localStorage.setItem('_mixlab_system_prompt', systemPrompt)
@@ -135,7 +163,7 @@ async function createMenu () {
 
     // appsButton.onclick = () =>
     appsButton.onclick = async () => {
-      if (window._mixlab_llamacpp) {
+      if (window._mixlab_llamacpp&&window._mixlab_llamacpp.model&&window._mixlab_llamacpp.model.length>0) {
         //显示运行的模型
         createModelsModal([
           window._mixlab_llamacpp.url,
@@ -770,6 +798,22 @@ async function fetchReadmeContent (url) {
   }
 }
 
+async function startLLM (model) {
+  let res = await start_llama(model)
+  window._mixlab_llamacpp = res||{ model:[] }
+
+  localStorage.setItem('_mixlab_llama_select', res?.model||'')
+
+  if (document.body.querySelector('#mixlab_chatbot_by_llamacpp')&&window._mixlab_llamacpp?.url) {
+    document.body
+      .querySelector('#mixlab_chatbot_by_llamacpp')
+      .setAttribute('title', window._mixlab_llamacpp.url)
+  }
+  if (document.body.querySelector('#llm_status_btn')&&window._mixlab_llamacpp) {
+    document.body.querySelector('#llm_status_btn').innerText = window._mixlab_llamacpp.model
+  }
+}
+
 function createModelsModal (models) {
   var div =
     document.querySelector('#model-modal') || document.createElement('div')
@@ -825,7 +869,7 @@ function createModelsModal (models) {
       color: var(--descrip-text);
       font-size: 18px;
       display: flex;
-      align-items: center;
+      align-items: flex-start;
       flex: 1;
       overflow: hidden;
       text-decoration: none;
@@ -844,6 +888,27 @@ function createModelsModal (models) {
   })
   linkIcon.style.padding = '4px'
 
+  const statusIcon = document.createElement('small')
+  statusIcon.textContent = showTextByLanguage('Status', {
+    Status: 'OFF'
+  })
+  statusIcon.id = 'llm_status_btn'
+  statusIcon.style=`padding: 4px;
+  background-color: rgb(102, 255, 108);
+  color: black;
+  font-size: 12px;
+  margin-left: 12px;`
+  if (window._mixlab_llamacpp?.url) {
+    statusIcon.textContent = window._mixlab_llamacpp.model
+    statusIcon.style.backgroundColor = '#66ff6c'
+    statusIcon.style.color = 'black'
+  } else {
+  }
+  statusIcon.addEventListener('click', e => {
+    e.stopPropagation()
+    // startLLM()
+  })
+
   const n_gpu = document.createElement('input')
   n_gpu.type = 'number'
   n_gpu.setAttribute('min', -1)
@@ -878,20 +943,22 @@ function createModelsModal (models) {
   const title = document.createElement('p')
   title.innerText = 'Models'
   title.style = `font-size: 18px;
-  margin-right: 8px;`
+  margin-right: 8px;
+  margin-top: 0;`
 
   const left_d = document.createElement('div')
   left_d.style = `display: flex;
   justify-content: center;
-  align-items: center;
-  font-size: 12px;`
+  align-items: flex-start;
+  font-size: 12px;
+  flex-direction: column; `
   left_d.appendChild(title)
-
+  title.appendChild(statusIcon)
   left_d.appendChild(linkIcon)
-
+  left_d.appendChild(n_gpu_div)
   headTitleElement.appendChild(left_d)
 
-  headTitleElement.appendChild(n_gpu_div)
+  // headTitleElement.appendChild(n_gpu_div)
 
   //重启
   const reStart = document.createElement('small')
@@ -899,7 +966,11 @@ function createModelsModal (models) {
     restart: '重启'
   })
 
-  reStart.style.padding = '4px'
+  reStart.style=`padding: 8px;
+  font-size: 16px;
+  outline: 1px solid;
+  padding-top: 4px;
+  padding-bottom: 4px;`
 
   headTitleElement.appendChild(reStart)
 
@@ -958,29 +1029,22 @@ function createModelsModal (models) {
 
   modalContent.appendChild(input)
 
-  for (const m of models) {
-    let d = document.createElement('div')
-    d.innerText = m
-    d.className = `mix_tag`
+  if (!window._mixlab_llamacpp||(window._mixlab_llamacpp?.model?.length==0)) {
+    for (const m of models) {
+      let d = document.createElement('div')
+      d.innerText = `${showTextByLanguage('Run', {
+        Run: '运行'
+      })} ${m}`
+      d.className = `mix_tag`
 
-    if (!window._mixlab_llamacpp) {
       d.addEventListener('click', async e => {
         e.stopPropagation()
         div.remove()
-        let res = await start_llama(m)
-        window._mixlab_llamacpp = res
-
-        localStorage.setItem('_mixlab_llama_select', res.model)
-
-        if (document.body.querySelector('#mixlab_chatbot_by_llamacpp')) {
-          document.body
-            .querySelector('#mixlab_chatbot_by_llamacpp')
-            .setAttribute('title', window._mixlab_llamacpp.url)
-        }
+        startLLM(m)
       })
-    }
 
-    modalContent.appendChild(d)
+      modalContent.appendChild(d)
+    }
   }
   modal.appendChild(modalContent)
 
@@ -994,8 +1058,8 @@ function createModelsModal (models) {
   cursor: pointer;
   font-size: 12px;
   color: white;`
-  helpInfo.href="https://discord.gg/cXs9vZSqeK"
-  helpInfo.target="_blank"
+  helpInfo.href = 'https://discord.gg/cXs9vZSqeK'
+  helpInfo.target = '_blank'
   modal.appendChild(helpInfo)
 
   document.body.appendChild(div)
@@ -1349,6 +1413,8 @@ app.registerExtension({
           .querySelector('#mixlab_chatbot_by_llamacpp')
           .setAttribute('title', res.url)
       })
+    }else{
+      startLLM('')
     }
 
     LGraphCanvas.prototype.helpAboutNode = async function (node) {
@@ -1357,7 +1423,13 @@ app.registerExtension({
           ? nodesMap
           : await getCustomnodeMappings('url')
 
-      console.log('node & node map', node, nodesMap, nodesMap[node.type])
+      console.log(
+        '%c### node & node map',
+        'background: yellow; color: black',
+        node,
+        nodesMap,
+        nodesMap[node.type]
+      )
       let repo = nodesMap[node.type]
       if (repo) {
         let markdown = await fetchReadmeContent(repo.url)
@@ -1388,6 +1460,7 @@ app.registerExtension({
         let userInput = widget.value
         widget.value = widget.value.trim()
         widget.value += '\n'
+        let jsonStr="";
         try {
           await completion_(
             window._mixlab_llamacpp.url + '/v1/chat/completions',
@@ -1402,6 +1475,7 @@ app.registerExtension({
             t => {
               // console.log(t)
               widget.value += t
+              jsonStr+=t
             }
           )
         } catch (error) {
@@ -1427,13 +1501,27 @@ app.registerExtension({
                 t => {
                   // console.log(t)
                   widget.value += t
+                  jsonStr+=t
                 }
               )
             })
           }
         }
 
-        widget.value = widget.value.trim()
+        let json=null;
+
+        try {
+          json=JSON.parse(jsonStr.trim())
+        } catch (error) {
+          json=JSON.parse(jsonStr.trim()+"}")
+        }
+
+        if(json){
+          widget.value = [json.subject,json.content,json.style].join('\n')
+        }else{
+          widget.value = widget.value.trim()
+        }
+        
       }
     }
 
@@ -1696,6 +1784,7 @@ app.registerExtension({
         {
           content: 'Help ♾️Mixlab', // with a name
           callback: () => {
+            // console.log('#data',node)
             LGraphCanvas.prototype.helpAboutNode(node)
           } // and the callback
         },

web/javascript/video_mixlab.js

@@ -6,8 +6,6 @@ import { $el } from '../../../scripts/ui.js'
 
 // The code is based on ComfyUI-VideoHelperSuite modification.
 
-
-
 function injectCSS (css) {
   // 检查页面中是否已经存在具有相同内容的style标签
   const existingStyle = document.querySelector('style')
@@ -240,15 +238,7 @@ app.registerExtension({
   }
 })
 
-
-function offsetDOMWidget(
-  widget,
-  ctx,
-  node,
-  widgetWidth,
-  widgetY,
-  height
-) {
+function offsetDOMWidget (widget, ctx, node, widgetWidth, widgetY, height) {
   const margin = 10
   const elRect = ctx.canvas.getBoundingClientRect()
   const transform = new DOMMatrix()
@@ -270,18 +260,18 @@ function offsetDOMWidget(
     position: 'absolute',
     background: !node.color ? '' : node.color,
     color: !node.color ? '' : 'white',
-    zIndex: 5, //app.graph._nodes.indexOf(node),
+    zIndex: 5 //app.graph._nodes.indexOf(node),
   })
 }
 
-export const hasWidgets = (node) => {
+export const hasWidgets = node => {
   if (!node.widgets || !node.widgets?.[Symbol.iterator]) {
     return false
   }
   return true
 }
 
-export const cleanupNode = (node) => {
+export const cleanupNode = node => {
   if (!hasWidgets(node)) {
     return
   }
@@ -298,43 +288,43 @@ export const cleanupNode = (node) => {
   }
 }
 
-const CreatePreviewElement = (name, val, format) => {
-  const [type] = format.split('/')  
+const createPreviewElement = (name, val, format) => {
+  const [type] = format.split('/')
   const w = {
-      name,
-      type,
-      value: val,
-      draw: function (ctx, node, widgetWidth, widgetY, height) {
-        const [cw, ch] = this.computeSize(widgetWidth)
-        offsetDOMWidget(this, ctx, node, widgetWidth, widgetY, ch)
-      },
-      computeSize: function (_) {
-        const ratio = this.inputRatio || 1
-        const width = Math.max(220, this.parent.size[0])
-        return [width, (width / ratio + 10)]
-      },
-      onRemoved: function () {
-        if (this.inputEl) {
-          this.inputEl.remove()
-        }
-      },
+    name,
+    type,
+    value: val,
+    draw: function (ctx, node, widgetWidth, widgetY, height) {
+      const [cw, ch] = this.computeSize(widgetWidth)
+      offsetDOMWidget(this, ctx, node, widgetWidth, widgetY, ch)
+    },
+    computeSize: function (_) {
+      const ratio = this.inputRatio || 1
+      const width = Math.max(220, this.parent.size[0])
+      return [width, width / ratio + 10]
+    },
+    onRemoved: function () {
+      if (this.inputEl) {
+        this.inputEl.remove()
+      }
     }
-
-    w.inputEl = document.createElement(type === 'video' ? 'video' : 'img')
-    w.inputEl.src = w.value
-    if (type === 'video') {
-      w.inputEl.setAttribute('type', 'video/webm');
-      w.inputEl.autoplay = true
-      w.inputEl.loop = true
-      w.inputEl.controls = false;
-    }
-    w.inputEl.onload = function () {
-      w.inputRatio = w.inputEl.naturalWidth / w.inputEl.naturalHeight
-    }
-    document.body.appendChild(w.inputEl)
-    return w
   }
 
+  w.inputEl = document.createElement(type === 'video' ? 'video' : 'img')
+  w.inputEl.src = w.value
+
+  if (type === 'video' || format.match('.mp4')) {
+    w.inputEl.setAttribute('type', 'video/webm')
+    w.inputEl.autoplay = true
+    w.inputEl.loop = true
+    w.inputEl.controls = true
+  }
+  w.inputEl.onload = function () {
+    w.inputRatio = w.inputEl.naturalWidth / w.inputEl.naturalHeight
+  }
+  document.body.appendChild(w.inputEl)
+  return w
+}
 
 app.registerExtension({
   name: 'Mixlab.Video.ImageListReplace',
@@ -469,7 +459,10 @@ app.registerExtension({
       }
     }
 
-    if (nodeData?.name == 'VideoCombine_Adv') {
+    if (
+      nodeData?.name == 'VideoCombine_Adv' ||
+      nodeData?.name == 'CombineAudioVideo'
+    ) {
       const onExecuted = nodeType.prototype.onExecuted
       nodeType.prototype.onExecuted = function (message) {
         const prefix = 'vhs_gif_preview_'
@@ -489,7 +482,7 @@ app.registerExtension({
                 '/view?' + new URLSearchParams(params).toString()
               )
               const w = this.addCustomWidget(
-                CreatePreviewElement(
+                createPreviewElement(
                   `${prefix}_${i}`,
                   previewUrl,
                   params.format || 'image/gif'