Update avatar-agent-workflow.json

0.46.0

README.md

@@ -10,6 +10,16 @@ For business cooperation, please contact email 389570357@qq.com
 
 ##### `最新`：
 
+- AvatarGeneratorAgent [工作流示例](./workflow/avatar-agent-workflow.json)
+
+- 新增[fal.ai](https://fal.ai/dashboard)的视频生成:Kling、RunwayGen3、LumaDreamMachine，[工作流下载](./workflow/video-all-in-one-test-workflow.json)
+
+- 新增 SimulateDevDesignDiscussions，需要安装[swarm](https://github.com/openai/swarm)和[Comfyui-ChatTTS](https://github.com/shadowcz007/Comfyui-ChatTTS)，[工作流下载](./workflow/swarm制作的播客节点workflow.json)
+
+- 新增 SenseVoice
+
+- [新增JS-SDK，方便直接在前端项目中使用comfyui](https://github.com/shadowcz007/comfyui-js-sdk)
+
 - 新增API调用图像生成节点 TextToImage Siliconflow，可以直接调用Siliconflow提供的flux生成图像
 
 - [增加 Her 的DEMO页面，和数字人对话](https://github.com/shadowcz007/ComfyUI-Backend-MixlabNodes/blob/main/workflow/her_demo_workflow.json)

__init__.py

@@ -32,7 +32,7 @@ _URL_=None
 # except:
 #     print("##nodes.ChatGPT ImportError")
 
-from .nodes.ChatGPT import openai_client
+# from .nodes.ChatGPT import openai_client
 
 from .nodes.RembgNode import get_rembg_models,U2NET_HOME,run_briarmbg,run_rembg
 
@@ -1006,8 +1006,8 @@ from .nodes.ImageNode import DepthViewer_,ImageBatchToList_,ImageListToBatch_,Co
 # from .nodes.Vae import VAELoader,VAEDecode
 from .nodes.ScreenShareNode import ScreenShareNode,FloatingVideo
 
-from .nodes.Audio import AudioPlayNode,SpeechRecognition,SpeechSynthesis
-from .nodes.Utils import KeyInput,IncrementingListNode,ListSplit,CreateLoraNames,CreateSampler_names,CreateCkptNames,CreateSeedNode,TESTNODE_,TESTNODE_TOKEN,AppInfo,IntNumber,FloatSlider,TextInput,ColorInput,FontInput,TextToNumber,DynamicDelayProcessor,LimitNumber,SwitchByIndex,MultiplicationNode
+from .nodes.Audio import AudioPlayNode,SpeechRecognition,SpeechSynthesis,AnalyzeAudioNone
+from .nodes.Utils import CreateJsonNode,KeyInput,IncrementingListNode,ListSplit,CreateLoraNames,CreateSampler_names,CreateCkptNames,CreateSeedNode,TESTNODE_,TESTNODE_TOKEN,AppInfo,IntNumber,FloatSlider,TextInput,ColorInput,FontInput,TextToNumber,DynamicDelayProcessor,LimitNumber,SwitchByIndex,MultiplicationNode
 from .nodes.Mask import PreviewMask_,MaskListReplace,MaskListMerge,OutlineMask,FeatheredMask
 
 from .nodes.Style import ApplyVisualStylePrompting,StyleAlignedReferenceSampler,StyleAlignedBatchAlign,StyleAlignedSampleReferenceLatents
@@ -1045,6 +1045,7 @@ NODE_CLASS_MAPPINGS = {
     "SaveImageToLocal":SaveImageToLocal,
     "SaveImageAndMetadata_":SaveImageAndMetadata,
     "ComparingTwoFrames_":ComparingTwoFrames,
+    "CreateJsonNode":CreateJsonNode,
 
     # Image
     "MirroredImage":MirroredImage,
@@ -1102,6 +1103,7 @@ NODE_CLASS_MAPPINGS = {
     "SpeechRecognition":SpeechRecognition,
     "SpeechSynthesis":SpeechSynthesis,
     "AudioPlay":AudioPlayNode,
+    "AnalyzeAudio":AnalyzeAudioNone,
 
     # Text
     "TextToNumber":TextToNumber,
@@ -1219,12 +1221,14 @@ NODE_DISPLAY_NAME_MAPPINGS = {
     "SpeechSynthesis":"SpeechSynthesis ♾️Mixlab",
     "SpeechRecognition":"SpeechRecognition ♾️Mixlab",
     "AudioPlay":"Preview Audio ♾️Mixlab",
+    "AnalyzeAudio":"Analyze Audio ♾️Mixlab",
 
     # Utils
     "DynamicDelayProcessor":"DynamicDelayByText ♾️Mixlab",
     "MultiplicationNode":"Math Operation ♾️Mixlab",
     "ListSplit_":"Split List ♾️Mixlab",
     "SwitchByIndex":"List Switch By Index ♾️Mixlab",
+    "CreateJsonNode":"Create Json",
 
     # "GamePal":"GamePal ♾️Mixlab",
     # Experiment
@@ -1267,7 +1271,7 @@ try:
         "ShowTextForGPT":ShowTextForGPT,
         "CharacterInText":CharacterInText,
         "TextSplitByDelimiter":TextSplitByDelimiter,
-        "JsonRepair":JsonRepair
+        "JsonRepair":JsonRepair, 
     }
 
     # 一个包含节点友好/可读的标题的字典
@@ -1278,7 +1282,7 @@ try:
         "ShowTextForGPT":"Show Text ♾️MixlabApp",
         "CharacterInText":"Character In Text",
         "TextSplitByDelimiter":"Text Split By Delimiter",
-        "JsonRepair":"Json Repair"
+        "JsonRepair":"Json Repair", 
     }
 
 
@@ -1286,7 +1290,7 @@ try:
     NODE_DISPLAY_NAME_MAPPINGS.update(NODE_DISPLAY_NAME_MAPPINGS_V)
 
 except Exception as e:
-    logging.info('ChatGPT.available False')
+    logging.info('ChatGPT.available False',e)
 
 
 try:
@@ -1427,6 +1431,72 @@ try:
 except Exception as e:
     logging.info('FishSpeech.available False' )
 
+try:
+    from .nodes.SenseVoice import SenseVoiceNode
+    logging.info('SenseVoice.available')
+    NODE_CLASS_MAPPINGS['SenseVoiceNode']=SenseVoiceNode
+    NODE_DISPLAY_NAME_MAPPINGS["SenseVoiceNode"]= "Sense Voice ♾️Mixlab"
+
+except Exception as e:
+    logging.info('SenseVoice.available False' )  
+
+try:
+    from .nodes.Whisper import LoadWhisperModel,WhisperTranscribe
+    logging.info('Whisper.available')
+    NODE_CLASS_MAPPINGS['LoadWhisperModel_']=LoadWhisperModel
+    NODE_CLASS_MAPPINGS['WhisperTranscribe_']=WhisperTranscribe
+    NODE_DISPLAY_NAME_MAPPINGS["LoadWhisperModel_"]= "Load Whisper Model ♾️Mixlab"
+    NODE_DISPLAY_NAME_MAPPINGS["WhisperTranscribe_"]= "Whisper Transcribe ♾️Mixlab"
+
+except Exception as e:
+    logging.info('Whisper.available False' )  
+
+
+try:
+    from .nodes.FalVideo import VideoGenKlingNode,VideoGenLumaDreamMachineNode,VideoGenRunwayGen3Node,LoadVideoFromURL
+    logging.info('FalVideo.available')
+    # Update Node class mappings
+    NODE_CLASS_MAPPINGS['VideoGenKlingNode']=VideoGenKlingNode
+    NODE_CLASS_MAPPINGS['VideoGenRunwayGen3Node']=VideoGenRunwayGen3Node
+    NODE_CLASS_MAPPINGS['VideoGenLumaDreamMachineNode']=VideoGenLumaDreamMachineNode
+    NODE_CLASS_MAPPINGS['LoadVideoFromURL']=LoadVideoFromURL
+    
+    NODE_DISPLAY_NAME_MAPPINGS["VideoGenKlingNode"]=  "Kling Video Generation @fal"
+    NODE_DISPLAY_NAME_MAPPINGS["VideoGenRunwayGen3Node"]= "Runway Gen3 Image-to-Video @fal"
+    NODE_DISPLAY_NAME_MAPPINGS["VideoGenLumaDreamMachineNode"]= "Luma Dream Machine @fal"
+    NODE_DISPLAY_NAME_MAPPINGS["LoadVideoFromURL"]= "Load Video from URL"
+    
+
+except Exception as e:
+    logging.info('FalVideo.available False' )  
+
+
+try:
+    from .nodes.SocialProfile import NewSocialProfileNode,LoadSocialProfileNode
+    logging.info('SocialProfile.available')
+    # Update Node class mappings
+    NODE_CLASS_MAPPINGS['NewSocialProfileNode']=NewSocialProfileNode
+    NODE_CLASS_MAPPINGS['LoadSocialProfileNode']=LoadSocialProfileNode
+
+    NODE_DISPLAY_NAME_MAPPINGS["NewSocialProfileNode"]=  "New Social Profile ♾️Mixlab"
+    NODE_DISPLAY_NAME_MAPPINGS["LoadSocialProfileNode"]=  "Load Social Profile ♾️Mixlab"
+
+except Exception as e:
+    logging.info('SocialProfile.available False' ) 
+
+
+try:
+    from .nodes.Agent import AvatarGeneratorAgent,SimulateDevDesignDiscussions
+    logging.info('Agent.available')
+
+    NODE_CLASS_MAPPINGS['SimulateDevDesignDiscussions']=SimulateDevDesignDiscussions
+    NODE_CLASS_MAPPINGS['AvatarGeneratorAgent']=AvatarGeneratorAgent
+    
+    NODE_DISPLAY_NAME_MAPPINGS["SimulateDevDesignDiscussions"]= "SimulateDevDesignDiscussions ♾️Mixlab Podcast"
+    NODE_DISPLAY_NAME_MAPPINGS["AvatarGeneratorAgent"]=  "Avatar Generator Agent ♾️Mixlab"
+    
+except Exception as e:
+    logging.info('Agent.available False' ) 
 
 
 logging.info('\033[93m -------------- \033[0m')

nodes/Agent.py

@@ -0,0 +1,892 @@
+import openai 
+import time
+import urllib.error
+import re,json,os,string,random
+import folder_paths
+import hashlib
+import sys
+import importlib.util
+import subprocess
+
+import torch
+import numpy as np
+
+python = sys.executable
+
+# Convert PIL to Tensor
+def pil2tensor(image):
+    return torch.from_numpy(np.array(image).astype(np.float32) / 255.0).unsqueeze(0)
+
+def is_installed(package, package_overwrite=None,auto_install=True):
+    is_has=False
+    try:
+        spec = importlib.util.find_spec(package)
+        is_has=spec is not None
+    except ModuleNotFoundError:
+        pass
+
+    package = package_overwrite or package
+
+    if spec is None:
+        if auto_install==True:
+            print(f"Installing {package}...")
+            # 清华源 -i https://pypi.tuna.tsinghua.edu.cn/simple
+            command = f'"{python}" -m pip install {package}'
+    
+            result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, env=os.environ)
+
+            is_has=True
+
+            if result.returncode != 0:
+                print(f"Couldn't install\nCommand: {command}\nError code: {result.returncode}")
+                is_has=False
+    else:
+        print(package+'## OK')
+
+    return is_has
+  
+
+if is_installed('json_repair'):
+    from json_repair import repair_json
+
+# 从文本中提取json
+def extract_json_strings(text):
+    json_strings = []
+    brace_level = 0
+    json_str = ''
+    in_json = False
+    
+    for char in text:
+        if char == '{':
+            brace_level += 1
+            in_json = True
+        if in_json:
+            json_str += char
+        if char == '}':
+            brace_level -= 1
+        if in_json and brace_level == 0:
+            json_strings.append(json_str)
+            json_str = ''
+            in_json = False
+
+    return json_strings[0] if len(json_strings)>0 else "{}"
+
+# text2json
+def text_to_json(text):
+    text=extract_json_strings(text)
+    good_json_string = repair_json(text)
+    # 将 JSON 字符串解析为 Python 对象
+    data = json.loads(good_json_string)
+    return data
+
+
+def save_to_json(file_path, data):
+    try:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(data, f, ensure_ascii=False, indent=4)
+    except Exception as e:
+            print(e)
+
+def read_from_json(file_path):
+    data = None
+    try:
+        with open(file_path, 'r', encoding='utf-8') as f:  # 使用 'r' 模式打开文件
+            data = json.load(f)
+    except Exception as e:
+        print('#read_from_json', e)
+        return None
+    return data
+
+
+default_agent={
+                    "user_name":"shadow",
+                    "profile_id":"ML000",
+                    "skills":"Design Hacker,Programmer,Architect,Experience Designer".split(",")
+                }
+
+
+def create_default_file(agent_dir):
+    fp=os.path.join(agent_dir,'shadow.json')
+    if not os.path.exists(fp):
+        save_to_json(fp,default_agent)
+
+def get_social_profile_dir():
+    try:
+        return folder_paths.get_folder_paths('agent')[0]
+    except:
+        agent_dir=os.path.join(folder_paths.models_dir, "agent")
+        if not os.path.exists(agent_dir):
+            os.makedirs(agent_dir, exist_ok=True)
+            create_default_file(agent_dir)
+        return agent_dir
+
+def save_social_profile_config(agent_dir,file_name,data):
+     save_to_json(os.path.join(agent_dir,file_name),data)
+
+
+def get_unique_hash(string):
+    hash_object = hashlib.sha1(string.encode())
+    unique_hash = hash_object.hexdigest()
+    return unique_hash
+
+def generate_random_string(length):
+    letters = string.ascii_letters + string.digits
+    return ''.join(random.choice(letters) for _ in range(length))
+
+class AnyType(str):
+  """A special class that is always equal in not equal comparisons. Credit to pythongosssss"""
+
+  def __ne__(self, __value: object) -> bool:
+    return False
+
+any_type = AnyType("*")
+
+#  判断是否是azure服务
+def is_azure_url(url):
+    pattern = r'.*\.azure\.com$'
+    if re.match(pattern, url):
+        return True
+    else:
+        return False
+
+def azure_client(key,url):
+    client = openai.AzureOpenAI(
+        api_key=key,
+    # https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#rest-api-versioning
+    api_version="2023-07-01-preview",
+    # https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource
+    azure_endpoint=url
+    )
+    return client
+
+def openai_client(key,url):
+    client = openai.OpenAI(
+        api_key=key,
+        base_url=url
+    )
+    return client
+
+def ZhipuAI_client(key):
+    try:
+        if is_installed('zhipuai')==True:
+            from zhipuai import ZhipuAI
+    except:
+        print("#install zhipuai error")
+
+    client = ZhipuAI(
+        api_key=key, # 填写您的 APIKey
+    ) 
+    return client
+
+
+if is_installed('swarm','git+https://github.com/openai/swarm.git'):
+    from swarm import Swarm, Agent
+
+
+def chat(client, model_name,messages,max_tokens=4096,temperature=0.6 ):
+        print('#chat',model_name,messages)
+        try_count = 0
+        while True:
+            try_count += 1
+            try:
+                if hasattr(client, "chat"):
+                    response = client.chat.completions.create(
+                        model=model_name,
+                        messages=messages,
+                        max_tokens=max_tokens,
+                        temperature=temperature
+                    )
+                else:
+                    # 是llama的
+                    response = client.create_chat_completion_openai_v1(
+                        messages=messages,
+                        # response_format={
+                        #     "type": "json_object",
+                        # },
+                        # temperature=0.7,
+                    )
+
+                break
+            except openai.AuthenticationError as ex:
+                raise ex
+            except (urllib.error.HTTPError, openai.OpenAIError) as ex:
+                if try_count >= 3:
+                    raise ex
+                time.sleep(3)
+                continue
+        
+        # print(response.keys())
+        finish_reason = response.choices[0].finish_reason
+        if finish_reason != "stop":
+            raise RuntimeError("API finished with unexpected reason: " + finish_reason)
+
+        content=""
+        try:
+            content=response.choices[0].message.content
+        except:
+            content=response.choices[0].delta['content']
+
+        return content
+
+
+llm_apis=[
+            {
+                "value": "https://api.openai.com/v1",
+                "label": "openai"
+            },
+            {
+                "value": "https://openai.api2d.net/v1",
+                "label": "api2d"
+            },
+            # {
+            #     "value": "https://docs-test-001.openai.azure.com",
+            #     "label": "https://docs-test-001.openai.azure.com"
+            # },
+             
+            {
+                "value": "https://api.moonshot.cn/v1",
+                "label": "Kimi"
+            },
+            {
+                "value": "https://api.deepseek.com/v1",
+                "label": "DeepSeek-V2"
+            },
+            {
+                "value": "https://api.siliconflow.cn/v1",
+                "label": "SiliconCloud"
+            }]
+        
+llm_apis_dict = {api["label"]: api["value"] for api in llm_apis}
+
+model_list=[ 
+            "gpt-4o",
+            "gpt-4o-2024-05-13",
+            "gpt-4",
+            "gpt-4-0314",
+            "gpt-4-0613",
+            "qwen-turbo",
+            "qwen-plus",
+            "qwen-long",
+            "qwen-max",
+            "qwen-max-longcontext",
+            "glm-4",
+            "glm-3-turbo",
+            "moonshot-v1-8k",
+            "moonshot-v1-32k",
+            "moonshot-v1-128k",
+            "deepseek-chat",
+            "Qwen/Qwen2-7B-Instruct",
+            "THUDM/glm-4-9b-chat",
+            "01-ai/Yi-1.5-9B-Chat-16K"
+                    ]
+
+software_architect_agent = Agent(
+            name="Software Architect",
+            instructions='''用脱口秀的风格回答编程问题，简短且口语化。
+
+        输出格式
+        ====
+
+        *   答案格式：`程序员：xxxxxxxxx`
+
+        示例
+        ==
+
+        **输入：**  
+        如何优化代码性能？
+
+        **输出：**  
+        程序员：兄弟，先把那些循环里的debug信息删掉，CPU都快哭了。'''
+        )
+
+designer_agent = Agent(
+            name="Designer",
+            instructions='''回答问题时，请扮演一位具有多年空间设计和用户体验设计经验的设计师。你的回答应当天马行空，但又富有深度，带有苏格拉底的思考方式，并且使用脱口秀的风格。回答要简短且非常口语化。格式如下：
+
+        设计师：\[回答内容\]
+
+        Output Format
+        =============
+
+        *   回答应当使用“设计师：\[回答内容\]”的格式。
+        *   回答应当简短、口语化，富有创意和深度。
+
+        Examples
+        ========
+
+        **Example 1:**
+
+        主持人：你觉得未来的家会是什么样子？
+
+        设计师：未来的家？想象一下，房子会像变形金刚一样，随时变形满足你的需求。今天是健身房，明天是电影院，后天是游戏场。家不再是四面墙，而是一个随心所欲的魔法空间。
+
+        **Example 2:**
+
+        主持人：你怎么看待极简主义设计？
+
+        设计师：极简主义？就像吃寿司，去掉所有不必要的装饰，只留下最精华的部分。让空间呼吸，让心灵自由。
+
+        **Example 3:**
+
+        主持人：你觉得色彩在设计中有多重要？
+
+        设计师：色彩？哦，那可是设计的灵魂！就像人生中的调味料，一点红色让你激情澎湃，一点蓝色让你心如止水。色彩决定了空间的情绪基调。'''
+        )
+
+
+# 问题生成
+host_agent = Agent(
+            name="Host",
+            instructions='''
+        为播客的主持人生成4到5个问题，这些问题有些是针对设计师问的，有些是针对程序员问的。
+
+        *   主持人：你知道如何开发一款APP产品，从想法到上线吗？
+        *   主持人：站在设计师的角度，你怎么看？
+        *   主持人：不知道程序员又是怎么想的呢？
+        *   主持人：感谢大家的参与，今天收获蛮大的
+
+        Steps
+        =====
+
+        1.  确定问题的对象：设计师或程序员。
+        2.  根据对象设计相关的问题，确保问题的多样性和深度。
+        3.  整理问题，使其符合播客主持人的风格和语气。
+
+        Output Format
+        =============
+
+        问题列表，每个问题以“主持人：”开头，不要出现序号。
+
+        Examples
+        ========
+
+        *   主持人：作为一名设计师，你是如何开始一个新项目的？
+        *   主持人：程序员在开发过程中遇到的最大挑战是什么？
+        *   主持人：设计师在团队协作中扮演什么角色？
+        *   主持人：程序员如何确保代码的质量和稳定性？
+        *   主持人：感谢大家的参与，今天的讨论非常有意义。
+
+        Notes
+        =====
+
+        *   确保问题针对不同的角色（设计师和程序员）。
+        *   保持问题的多样性，涵盖从项目开始到完成的各个阶段。
+        *   确保问题能引导出深入的讨论和见解。
+        ''')
+
+
+# 以下为固定提示词的LLM节点示例
+class SimulateDevDesignDiscussions:
+    
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "subject": ("STRING", {"multiline": True,"dynamicPrompts": False}),
+                "model": ( model_list, 
+                    {"default": model_list[0]}),
+                "api_url":(list(llm_apis_dict.keys()), 
+                    {"default": list(llm_apis_dict.keys())[0]}),
+                "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff, "step": 1}),
+            },
+             "optional":{
+                    "api_key":("STRING", {"forceInput": True,}),
+                    "custom_model_name":("STRING", {"forceInput": True,}), #适合自定义model
+                     "custom_api_url":("STRING", {"forceInput": True,}), #适合自定义model
+                },
+             
+        }
+
+    RETURN_TYPES = ("STRING",)
+    RETURN_NAMES = ("text",)
+    FUNCTION = "generate_contextual_text"
+    CATEGORY = "♾️Mixlab/Agent"
+    INPUT_IS_LIST = False
+    OUTPUT_IS_LIST = (False,)
+
+    def generate_contextual_text(self,
+                                 subject, 
+                                 model, 
+                                api_url,
+                                seed,
+                                api_key=None,
+                                custom_model_name=None,
+                                custom_api_url=None,
+                                ):
+        
+        # 设置黄色文本的ANSI转义序列
+        YELLOW = "\033[33m"
+        # 重置文本颜色的ANSI转义序列
+        RESET = "\033[0m"
+
+        if custom_model_name!=None:
+            model=custom_model_name
+
+        api_url=llm_apis_dict[api_url] if api_url in llm_apis_dict else ""
+
+        if custom_api_url!=None:
+            api_url=custom_api_url
+
+        if api_key==None:
+            api_key="lm_studio"
+
+        print("api_key,api_url",api_key,api_url)
+        # 
+        if is_azure_url(api_url):
+            client=azure_client(api_key,api_url)
+        else:
+            # 根据用户选择的模型，设置相应的接口和模型名称
+            if model == "glm-4" :
+                client = ZhipuAI_client(api_key)  # 使用 Zhipuai 的接口
+                print('using Zhipuai interface')
+            else :
+                client = openai_client(api_key,api_url)  # 使用 ChatGPT  的接口
+               
+        # 以下为多智能体框架
+        client = Swarm(client=client)
+
+        # 定义两个代理：软件系统架构师和设计师
+        
+        # 定义一个函数，用于转移问题到designer_agent
+        def transfer_to_designer_agent():
+            return designer_agent
+
+        # 将转移函数添加到软件系统架构师和设计师的函数列表中
+        software_architect_agent.functions.append(transfer_to_designer_agent)
+
+        response = client.run(agent=host_agent, messages=[{
+            "role":"user",
+            "content":f"主题是‘{subject}’"
+        }],model_override=model)
+
+        content=response.messages[-1]["content"]
+        print(f"{YELLOW}{content}{RESET}")
+
+        texts=content.split("\n")
+
+        # texts='''
+        # 主持人：你知道如何开发一款APP产品，从想法到上线吗？
+        # 主持人：站在设计师的角度，你怎么看？
+        # 主持人：不知道程序员又是怎么想的呢？
+        # 主持人：感谢大家的参与，今天收获蛮大的
+        # '''.split("\n")
+
+        messages=[]
+
+        texts = [text.strip() for text in texts if text.strip()]
+
+        result=[]
+
+        for text in texts:
+            messages.append({
+                "role": "user",
+                "content": text
+            })
+
+            # 运行客户端，使用软件系统架构师作为初始代理
+            response = client.run(agent=software_architect_agent, messages=messages,model_override=model)
+
+            print(f"{text}")
+            result.append(text)
+
+            # 输出最后一个响应消息的内容
+            content=response.messages[-1]["content"]
+            print(f"{YELLOW}{content}{RESET}")
+            
+            result.append(content)
+
+            messages.append({
+                "role":"assistant",
+                "content":content
+            })
+
+        
+        return ("\n".join(result),)
+
+
+
+class AvatarGeneratorAgent:
+    
+    @classmethod
+    def INPUT_TYPES(cls):
+
+        return {
+            "required": {
+                "subject": ("STRING", {"multiline": True,"dynamicPrompts": False}),
+                "social_profile":("STRING", {"forceInput": True}),
+                "model": ( model_list, 
+                    {"default": model_list[0]}),
+                "api_url":(list(llm_apis_dict.keys()), 
+                    {"default": list(llm_apis_dict.keys())[0]}),
+                "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff, "step": 1}),
+            },
+             "optional":{
+                    "api_key":("STRING", {"forceInput": True,}),
+                    "custom_model_name":("STRING", {"forceInput": True,}), #适合自定义model
+                     "custom_api_url":("STRING", {"forceInput": True,}), #适合自定义model
+                },
+             
+        }
+
+    RETURN_TYPES = ("STRING","STRING","STRING","STRING",)
+    RETURN_NAMES = ("subject","result","task_desc","new_skill",)
+    FUNCTION = "run"
+    CATEGORY = "♾️Mixlab/Agent"
+
+    def run(self, subject,  social_profile,  model, 
+              api_url, seed, api_key=None, custom_model_name=None, custom_api_url=None):
+        
+        # 设置黄色文本的ANSI转义序列
+        YELLOW = "\033[33m"
+        # 重置文本颜色的ANSI转义序列
+        RESET = "\033[0m"
+
+        if custom_model_name!=None:
+            model=custom_model_name
+
+        api_url=llm_apis_dict[api_url] if api_url in llm_apis_dict else ""
+
+        if custom_api_url!=None:
+            api_url=custom_api_url
+
+        if api_key==None:
+            api_key="lm_studio"
+
+        print("api_key,api_url",api_key,api_url)
+        # 
+        if is_azure_url(api_url):
+            client=azure_client(api_key,api_url)
+        else:
+            # 根据用户选择的模型，设置相应的接口和模型名称
+            if model == "glm-4" :
+                client = ZhipuAI_client(api_key)  # 使用 Zhipuai 的接口
+                print('using Zhipuai interface')
+            else :
+                client = openai_client(api_key,api_url)  # 使用 ChatGPT  的接口
+               
+  
+        # 以下为多智能体框架
+        client = Swarm(client=client)
+
+        # 定义1个代理：生成器
+
+        # instructions
+        def generator_instructions(context_variables):
+            # user = context_variables.get("name", "User")
+            user=json.dumps(context_variables)
+            return f'''根据用户的背景信息{user},生成角色新的技能挑战，欢迎并根据提供的信息创意性地整合和演变技能挑战，包含任务控制在3个以内。'''+'''
+步骤
+==
+
+1.  **欢迎用户**
+
+    *   欢迎用户并提及其用户名。
+    
+2.  **生成技能挑战**
+
+    *   根据用户的技能，设计创意性的技能挑战。
+    *   每个挑战需要整合用户的多个技能。
+    *   挑战数量控制在3个以内。
+
+输出格式
+====
+
+*   输出格式为口语化的一个段落,不要有换行。
+
+示例1
+==
+
+**输入：**
+
+{
+  "user_name": "shadow",
+  "profile_id": "ML000",
+  "skills": ["Design Hacker", "Programmer", "Architect", "Experience Designer", "设计黑客"]
+}
+
+信息：Mothbox是一款低成本、高性能的昆虫监测设备，旨在帮助野外生物学家在丛林深处进行部署，同时也适合在家中研究生物多样性。所有物理设计、电子原理图、Pi脚本和昆虫识别的人工智能都是免费开源的，用户可以自行构建、分享和改进。Mothbox利用超高分辨率传感器和开源AI脚本进行昆虫监测，提供高度本地化的环境健康数据，特别适用于研究飞蛾和甲虫等高度多样化的昆虫。
+
+
+**输出：**  
+欢迎shadow！我们很高兴你对Mothbox感兴趣。根据你的技能，我们为你设计了以下创意性技能挑战：第一个挑战是**设计和构建一个模块化的Mothbox外壳**，利用你的建筑师和设计黑客技能，创造一个可以适应不同环境和需求的模块化外壳设计。挑战二是**开发一个用户友好的界面**，作为一名程序员和体验设计师，设计一个直观的用户界面，使用户可以轻松地配置和监测Mothbox。挑战三是**优化昆虫识别算法**：结合你的程序员和设计黑客技能，改进现有的开源AI脚本，提高昆虫识别的准确性和速度。我们期待看到你如何运用你的多重技能来挑战和改进Mothbox！
+
+
+示例2
+==
+
+**输入：**
+
+{
+  "user_name": "meadow",
+  "profile_id": "ML002",
+  "skills": ["体验设计师", "作家"]
+}
+
+信息：苹果发布新的iPad mini，AI功能来了
+
+
+**输出：**  
+欢迎meadow！我们很高兴你对苹果发布的新的iPad mini感兴趣。根据你的技能，我们为你设计了以下创意性技能挑战：第一个挑战是**设计一个AI驱动的用户体验测试平台**：利用你的体验设计师技能，创建一个平台，使用户可以通过iPad mini的AI功能进行用户体验测试，收集和分析数据，优化应用和服务。挑战二是**撰写一篇关于AI在移动设备上应用的深度文章**：作为一名作家，撰写一篇详细的文章，探讨AI功能如何改变了iPad mini的使用体验，并预测未来的发展趋势。挑战三是**开发一个互动的iPad mini用户指南**：结合你的体验设计师和作家技能，设计并编写一个互动的用户指南，使新用户可以轻松上手iPad mini，充分利用其AI功能。我们期待看到你如何运用你的多重技能来挑战和改进iPad mini的用户体验！
+'''.strip()
+
+        
+        def skills_instructions(context_variables):
+            new_skill = context_variables.get("new_skill", "")
+            user=json.dumps(context_variables)
+            return f'''根据用户信息{user}和新技能{new_skill}，判断是否需要更新用户的技能树。'''+'''
+如果需要更新，输出技能树中需要添加的新标签`new_tag`；如果不需要更新，则输出`new_tag`为空。
+
+*   用户信息 (`user`): 包括用户名、配置文件ID和现有技能。
+*   新技能 (`new_skill`): 需要判断的新技能。
+
+Steps
+=====
+
+1.  检查新技能是否已经存在于用户的技能列表中。
+2.  如果新技能存在，`new_tag`为空。
+3.  如果新技能不存在，`new_tag`为新技能。
+
+Output Format
+=============
+
+```
+{
+  "new_tag": "[new_skill or empty string]"
+}
+```
+
+Examples
+========
+
+**Example 1**
+
+*   **Input:**
+
+    ```
+    {
+      "user": {
+        "user_name": "shadow",
+        "profile_id": "ML000",
+        "skills": ["Design Hacker", "Programmer", "Architect", "Experience Designer"]
+      },
+      "new_skill": "设计黑客"
+    }
+    ```
+
+*   **Reasoning:**  
+    新技能"设计黑客"不在现有技能列表中，因此需要更新技能树。
+
+*   **Output:**
+
+    ```
+    {
+      "new_tag": "设计黑客"
+    }
+    ```
+
+**Example 2**
+
+*   **Input:**
+
+    ```
+    {
+      "user": {
+        "user_name": "shadow",
+        "profile_id": "ML000",
+        "skills": ["Design Hacker", "Programmer", "Architect", "Experience Designer"]
+      },
+      "new_skill": "Programmer"
+    }
+    ```
+
+*   **Reasoning:**  
+    新技能"Programmer"已经在现有技能列表中，因此不需要更新技能树。
+
+*   **Output:**
+
+    ```
+    {
+      "new_tag": ""
+    }
+    ```
+    '''
+
+        
+        # 用于更新用户信息
+        def update_account_details(context_variables: dict):
+            profile_id = context_variables.get("profile_id", None)
+            user_name = context_variables.get("user_name", None)
+            skills = context_variables.get("skills", None)
+            file_name=context_variables.get('file_name','agent.json')
+            if skills:
+                skills=",".join(skills)
+            print(f"Account Details: {user_name} {profile_id} {skills}")
+
+            # 
+            save_social_profile_config(get_social_profile_dir(),file_name,context_variables)
+            return "Success"
+
+        generator_agent = Agent(
+            name="generator_agent",
+            instructions=generator_instructions,
+            # functions=[update_account_details],
+        )
+
+        # 用于判断是否需要更新
+        skills_agent = Agent(
+            name="skills_agent",
+            instructions=skills_instructions,
+            functions=[update_account_details],
+        )
+
+        # dict
+        context_variables = json.loads(social_profile)
+
+        response = client.run(
+            messages=[{"role": "user", "content":subject}],
+            agent=generator_agent,
+            context_variables=context_variables,
+            model_override=model
+        )
+        result=response.messages[-1]["content"]
+        print(f"{YELLOW}{result}{RESET}")
+
+
+        # 答案生成
+
+        def answer_instructions(context_variables):
+            # user = context_variables.get("name", "User")
+            user=json.dumps(context_variables)
+
+            return '''根据提供的挑战任务，提炼任务的关键词，生成任务的图像文字描述。然后，结合用户信息'''+user+'''里的技能信息，构建一个技能树，给完成任务设计一个奖励，这个奖励是一个新的技能标签，为这颗技能树提供新的分支或者是加粗原有的技能。用JSON格式输出新的技能标签和图像的文字描述:
+
+Steps
+=====
+
+1.  提炼任务的关键词。
+2.  根据关键词生成图像文字描述。
+3.  结合用户的技能信息，构建技能树。
+4.  设计奖励的新技能标签。
+5.  用JSON格式输出新的技能标签和图像的文字描述。
+
+Output Format
+=============
+
+JSON格式，包含新的技能标签和图像文字描述。
+
+Examples
+========
+
+**Input:**
+
+```
+{
+  "user_name": "shadow",
+  "profile_id": "ML000",
+  "skills": ["Design Hacker", "Programmer", "Architect", "Experience Designer", "设计黑客"]
+}
+
+挑战任务:设计模块化Mothbox外壳，开发用户友好的数据分析平台，并将美食数据分析技术应用于昆虫数据解析，提升设备功能和表现。
+
+```
+
+**Output:**
+
+```
+{
+  "skill": "Environmental Data Analyst",
+  "image": "设计模块化Mothbox外壳，开发用户友好数据分析平台，结合美食数据分析技术应用于环保领域。"
+}
+```
+
+Notes
+=====
+
+*   关键词应包括：模块化外壳、数据分析平台、美食数据分析、环境健康、昆虫监测。
+*   奖励的新技能标签应与任务相关，并补充现有技能树。'''.strip()
+
+        answer_agent = Agent(
+                    name="Answer",
+                    instructions=answer_instructions)
+
+        response = client.run(
+            messages=[{"role": "user", "content": result}],
+            agent=answer_agent,
+            context_variables=context_variables,
+            model_override=model
+        )
+        content=response.messages[-1]["content"]
+
+        answer_json=text_to_json(content)
+
+        print(f"{YELLOW}{content}{RESET}") 
+
+        
+        task_desc=""
+        new_skill=""
+        if "skill" in answer_json and 'image' in answer_json:
+            context_variables['new_skill']=answer_json['skill']
+            new_skill=answer_json['skill']
+            task_desc=answer_json['image']
+            print(f"{YELLOW}{task_desc}{RESET}") 
+        
+
+        response = client.run(
+            messages=[{"role": "user", "content": "请判断是否需要更新我的数字分身信息"}],
+            agent=skills_agent,
+            context_variables=context_variables,
+            model_override=model
+        )
+
+        content=response.messages[-1]["content"]
+        print(f"{YELLOW}{content}{RESET}")
+
+        # 重新改写subject - > 提问
+        prompt='''用口语化的表达改写文本，开头使用“我有个新话题，是”。
+
+确保改写后的内容自然、流畅。
+
+Steps
+=====
+
+1.  阅读并理解原文本。
+2.  将文本改写成口语化的表达，注意语气和句式。
+3.  在改写后的文本开头加上“我有个新话题，是”。
+
+Output Format
+=============
+
+输出格式为段落，使用口语化的表达，开头为“我有个新话题，是”。
+
+Examples
+========
+
+**Input:**  
+今天的天气真好，阳光明媚，适合出去散步。
+
+**Output:**  
+我有个新话题，是今天的天气真好，阳光明媚，特别适合出去散步。
+
+**Input:**  
+读书可以增长知识，开阔视野。
+
+**Output:**  
+我有个新话题，是读书可以增长知识，还能开阔视野。'''
+
+        # 改写
+        subject_agent = Agent(
+            name="subject_agent",
+            instructions=prompt, 
+        )
+
+        response = client.run(
+            messages=[{"role": "user", "content":subject}],
+            agent=subject_agent,
+            model_override=model
+        )
+        # 改写
+        subject=response.messages[-1]["content"]
+        print(f"{YELLOW}{subject}{RESET}")
+
+        return (subject,result,task_desc,new_skill,)
+

nodes/Audio.py

@@ -3,6 +3,101 @@ import os
 import folder_paths
 import torchaudio
 
+class AnyType(str):
+  """A special class that is always equal in not equal comparisons. Credit to pythongosssss"""
+
+  def __ne__(self, __value: object) -> bool:
+    return False
+
+any_type = AnyType("*")
+
+
+def analyze_audio_data(audio_data):
+    total_duration = 0
+    total_gap_duration = 0
+    emotion_counts = {}
+    audio_types = set()
+    languages = set()
+
+    for i, entry in enumerate(audio_data):
+        # Calculate the duration of each audio segment
+        start_time = entry['start_time']
+        end_time = entry['end_time']
+        duration = end_time - start_time
+        total_duration += duration
+
+        # Count the emotions
+        if "emotion" in entry:
+            emotion = entry['emotion']
+            if emotion in emotion_counts:
+                emotion_counts[emotion] += 1
+            else:
+                emotion_counts[emotion] = 1
+
+        # Collect the audio types
+        if "audio_type" in entry:
+            audio_types.add(entry['audio_type'])
+
+        if "language" in entry:
+            languages.add(entry['language'])
+
+        # Calculate gap duration if not the last entry
+        if i < len(audio_data) - 1:
+            next_start_time = audio_data[i + 1]['start_time']
+            gap_duration = next_start_time - end_time
+            if gap_duration > 0:
+                total_gap_duration += gap_duration
+
+    # Get the most frequent emotion
+    if len(emotion_counts.keys())>0:
+        most_frequent_emotion = max(emotion_counts, key=emotion_counts.get)
+    else:
+        most_frequent_emotion=None
+
+    # Convert audio_types set to list for better readability
+    audio_types = list(audio_types)
+
+    languages=list(languages)
+
+    # Print the results
+    print(f"Total Effective Duration: {total_duration:.2f} seconds")
+    print(f"Total Gap Duration: {total_gap_duration:.2f} seconds")
+    print(f"Emotion Changes: {emotion_counts}")
+    print(f"Most Frequent Emotion: {most_frequent_emotion}")
+    print(f"Audio Types: {audio_types}")
+
+
+    return {
+        "total_duration": total_duration,
+        "total_gap_duration": total_gap_duration,
+        "emotion_changes": emotion_counts,
+        "most_frequent_emotion": most_frequent_emotion,
+        "audio_types": audio_types,
+        "languages":languages
+    }
+
+
+# 分析音频数据
+class AnalyzeAudioNone:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { 
+                    "json":(any_type,),}, 
+                }
+    
+    RETURN_TYPES = (any_type,)
+    RETURN_NAMES = ("result",)
+
+    FUNCTION = "run"
+
+    CATEGORY = "♾️Mixlab/Audio"
+
+    def run(self,json):
+        result=analyze_audio_data(json)
+        return (result,)
+
+
+
 class SpeechRecognition:
     @classmethod
     def INPUT_TYPES(s):

nodes/ChatGPT.py

@@ -19,30 +19,6 @@ python = sys.executable
 def pil2tensor(image):
     return torch.from_numpy(np.array(image).astype(np.float32) / 255.0).unsqueeze(0)
 
-
-# 从文本中提取json
-def extract_json_strings(text):
-    json_strings = []
-    brace_level = 0
-    json_str = ''
-    in_json = False
-    
-    for char in text:
-        if char == '{':
-            brace_level += 1
-            in_json = True
-        if in_json:
-            json_str += char
-        if char == '}':
-            brace_level -= 1
-        if in_json and brace_level == 0:
-            json_strings.append(json_str)
-            json_str = ''
-            in_json = False
-
-    return json_strings[0] if len(json_strings)>0 else "{}"
-
-
 def is_installed(package, package_overwrite=None,auto_install=True):
     is_has=False
     try:
@@ -71,6 +47,58 @@ def is_installed(package, package_overwrite=None,auto_install=True):
 
     return is_has
   
+if is_installed('json_repair'):
+    from json_repair import repair_json
+
+
+
+# 从文本中提取json
+def extract_json_strings(text):
+    json_strings = []
+    brace_level = 0
+    json_str = ''
+    in_json = False
+    
+    for char in text:
+        if char == '{':
+            brace_level += 1
+            in_json = True
+        if in_json:
+            json_str += char
+        if char == '}':
+            brace_level -= 1
+        if in_json and brace_level == 0:
+            json_strings.append(json_str)
+            json_str = ''
+            in_json = False
+
+    return json_strings[0] if len(json_strings)>0 else "{}"
+
+# text2json
+def text_to_json(text):
+    text=extract_json_strings(text)
+    good_json_string = repair_json(text)
+    # 将 JSON 字符串解析为 Python 对象
+    data = json.loads(good_json_string)
+    return data
+
+
+def save_to_json(file_path, data):
+    try:
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(data, f, ensure_ascii=False, indent=4)
+    except Exception as e:
+            print(e)
+
+def read_from_json(file_path):
+    data = None
+    try:
+        with open(file_path, 'r', encoding='utf-8') as f:  # 使用 'r' 模式打开文件
+            data = json.load(f)
+    except Exception as e:
+        print('#read_from_json', e)
+        return None
+    return data
 
 
 # def is_installed(package):
@@ -210,8 +238,6 @@ def get_llama_path():
 #         return llm
 
 
-if is_installed('json_repair'):
-    from json_repair import repair_json
 
 
 def chat(client, model_name,messages,max_tokens=4096,temperature=0.6 ):
@@ -460,7 +486,8 @@ class SiliconflowFreeNode:
     @classmethod
     def INPUT_TYPES(cls):
         model_list= [ 
-            "Qwen/Qwen2-7B-Instruct",
+            "Qwen/Qwen2.5-7B-Instruct",
+            "Qwen/Qwen2-7B-Instruct", 
             "THUDM/glm-4-9b-chat",
             "01-ai/Yi-1.5-9B-Chat-16K",
             "meta-llama/Meta-Llama-3.1-8B-Instruct"
@@ -785,9 +812,12 @@ class JsonRepair:
     def INPUT_TYPES(s):
         return {
             "required": {
-                 "json_string":("STRING", {"forceInput": True,}), 
-                 "key":("STRING", {"multiline": False,"dynamicPrompts": False,"default": ""}),
-            }
+                "json_string":("STRING", {"forceInput": True,}), 
+                "key":("STRING", {"multiline": False,"dynamicPrompts": False,"default": ""}),
+            },
+            "optional":{
+                "json_string2":("STRING", {"forceInput": True,})
+            },
         }
 
     INPUT_IS_LIST = False
@@ -797,10 +827,13 @@ class JsonRepair:
     # OUTPUT_NODE = True
     OUTPUT_IS_LIST = (False,False,)
 
-    CATEGORY = "♾️Mixlab/GPT"
+    CATEGORY = "♾️Mixlab/Utils"
 
-    def run(self, json_string,key=""):
+    def run(self, json_string,key="",json_string2=None):
 
+        if not isinstance(json_string, str):
+            json_string=json.dumps(json_string)
+    
         json_string=extract_json_strings(json_string)
         # print(json_string)
         good_json_string = repair_json(json_string)
@@ -808,6 +841,20 @@ class JsonRepair:
         # 将 JSON 字符串解析为 Python 对象
         data = json.loads(good_json_string)
 
+        if json_string2!=None:
+            if not isinstance(json_string2, str):
+                json_string2=json.dumps(json_string2)
+    
+            json_string2=extract_json_strings(json_string2)
+            # print(json_string)
+            good_json_string2 = repair_json(json_string2)
+
+            # 将 JSON 字符串解析为 Python 对象
+            data2 = json.loads(good_json_string2)
+
+            data={**data, **data2}
+
+
         v=""
         if key!="" and (key in data):
             v=data[key]
@@ -815,4 +862,4 @@ class JsonRepair:
         # 将 Python 对象转换回 JSON 字符串，确保中文字符不被转义
         json_str_with_chinese = json.dumps(data, ensure_ascii=False)
 
-        return (json_str_with_chinese,v,)
+        return (json_str_with_chinese,v,)

nodes/FalVideo.py

@@ -0,0 +1,332 @@
+# 修改自 https://github.com/gokayfem/ComfyUI-fal-API/blob/main/nodes/video_node.py
+# image-to-video all in one
+
+import os,sys
+import torch
+from PIL import Image
+import tempfile
+import numpy as np
+import requests
+import cv2
+import subprocess
+import importlib.util
+python = sys.executable
+
+def is_installed(package, package_overwrite=None,auto_install=True):
+    is_has=False
+    try:
+        spec = importlib.util.find_spec(package)
+        is_has=spec is not None
+    except ModuleNotFoundError:
+        pass
+
+    package = package_overwrite or package
+
+    if spec is None:
+        if auto_install==True:
+            print(f"Installing {package}...")
+            # 清华源 -i https://pypi.tuna.tsinghua.edu.cn/simple
+            command = f'"{python}" -m pip install {package}'
+    
+            result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, env=os.environ)
+
+            is_has=True
+
+            if result.returncode != 0:
+                print(f"Couldn't install\nCommand: {command}\nError code: {result.returncode}")
+                is_has=False
+    else:
+        print(package+'## OK')
+
+    return is_has
+
+
+try:
+    if is_installed('fal_client','fal-client')==True:
+        from fal_client import submit, upload_file
+except:
+    print("#install fal-client error")
+
+
+def upload_image(image):
+    try:
+        # Convert the image tensor to a numpy array
+        if isinstance(image, torch.Tensor):
+            image_np = image.cpu().numpy()
+        else:
+            image_np = np.array(image)
+
+        # Ensure the image is in the correct format (H, W, C)
+        if image_np.ndim == 4:
+            image_np = image_np.squeeze(0)  # Remove batch dimension if present
+        if image_np.ndim == 2:
+            image_np = np.stack([image_np] * 3, axis=-1)  # Convert grayscale to RGB
+        elif image_np.shape[0] == 3:
+            image_np = np.transpose(image_np, (1, 2, 0))  # Change from (C, H, W) to (H, W, C)
+
+        # Normalize the image data to 0-255 range
+        if image_np.dtype == np.float32 or image_np.dtype == np.float64:
+            image_np = (image_np * 255).astype(np.uint8)
+
+        # Convert to PIL Image
+        pil_image = Image.fromarray(image_np)
+
+        # Save the image to a temporary file
+        with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as temp_file:
+            pil_image.save(temp_file, format="PNG")
+            temp_file_path = temp_file.name
+
+        # Upload the temporary file
+        image_url = upload_file(temp_file_path)
+        return image_url
+    except Exception as e:
+        print(f"Error uploading image: {str(e)}")
+        return None
+    finally:
+        # Clean up the temporary file
+        if 'temp_file_path' in locals():
+            os.unlink(temp_file_path)
+
+
+class VideoGenKlingNode:
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "prompt": ("STRING", {"default": "", "multiline": True}),
+                "duration": (["5", "10"], {"default": "5"}),
+                "aspect_ratio": (["16:9", "9:16", "1:1"], {"default": "16:9"}),
+                "mode": (["standard", "pro"], {"default": "standard"}),
+                "fal_key":("STRING", {"forceInput": True,}),
+            },
+            "optional": {
+                "image": ("IMAGE",),
+            },
+        }
+
+    RETURN_TYPES = ("STRING",)
+    FUNCTION = "generate_video"
+    CATEGORY = "♾️Mixlab/Video"
+
+    def generate_video(self, prompt, duration, aspect_ratio,mode,fal_key, image=None):
+        arguments = {
+            "prompt": prompt,
+            "duration": duration,
+            "aspect_ratio": aspect_ratio,
+        }
+
+        os.environ["FAL_KEY"] = fal_key
+
+        api_url="fal-ai/kling-video/v1/"+mode
+
+        try:
+            if image is not None:
+                image_url = upload_image(image)
+                if image_url:
+                    arguments["image_url"] = image_url
+                    handler = submit(api_url+"/image-to-video", arguments=arguments)
+                else:
+                    return ("Error: Unable to upload image.",)
+            else:
+                handler = submit(api_url+"/text-to-video", arguments=arguments)
+
+            result = handler.get()
+            video_url = result["video"]["url"]
+            return (video_url,)
+        except Exception as e:
+            print(f"Error generating video: {str(e)}")
+            return ("Error: Unable to generate video.",)
+
+
+class VideoGenRunwayGen3Node:
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "prompt": ("STRING", {"default": "", "multiline": True}),
+                "image": ("IMAGE",),
+                "duration": (["5", "10"], {"default": "5"}),
+                "aspect_ratio": (["16:9", "9:16"], {"default": "16:9"}),
+                "fal_key":("STRING", {"forceInput": True,}),
+            },
+        }
+
+    RETURN_TYPES = ("STRING",)
+    FUNCTION = "generate_video"
+    CATEGORY = "♾️Mixlab/Video"
+
+    def generate_video(self, prompt, image, duration,aspect_ratio,fal_key):
+        os.environ["FAL_KEY"] = fal_key
+        try:
+            image_url = upload_image(image)
+            if not image_url:
+                return ("Error: Unable to upload image.",)
+
+            arguments = {
+                "prompt": prompt,
+                "image_url": image_url,
+                "duration": duration,
+                "ratio":aspect_ratio
+            }
+
+            handler = submit("fal-ai/runway-gen3/turbo/image-to-video", arguments=arguments)
+            result = handler.get()
+            video_url = result["video"]["url"]
+            return (video_url,)
+        except Exception as e:
+            print(f"Error generating video: {str(e)}")
+            return ("Error: Unable to generate video.",)
+
+class VideoGenLumaDreamMachineNode:
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "prompt": ("STRING", {"default": "", "multiline": True}), 
+                "aspect_ratio": (["16:9", "9:16", "4:3", "3:4", "21:9", "9:21"], {"default": "16:9"}),
+                "fal_key":("STRING", {"forceInput": True,}),
+            },
+            "optional": {
+                "image": ("IMAGE",),
+                "loop": ("BOOLEAN", {"default": True}),
+            }, 
+        }
+
+    RETURN_TYPES = ("STRING",)
+    FUNCTION = "generate_video"
+    CATEGORY = "♾️Mixlab/Video"
+
+    def generate_video(self, prompt, aspect_ratio,fal_key, image=None, loop=True):
+
+        os.environ["FAL_KEY"] = fal_key
+
+        arguments = {
+            "prompt": prompt,
+            "aspect_ratio": aspect_ratio,
+            "loop": loop,
+        }
+
+        try:
+            if image is not None:
+                image_url = upload_image(image)
+                if not image_url:
+                    return ("Error: Unable to upload image.",)
+                arguments["image_url"] = image_url
+                endpoint = "fal-ai/luma-dream-machine/image-to-video"
+            else:
+                endpoint = "fal-ai/luma-dream-machine"
+
+            handler = submit(endpoint, arguments=arguments)
+            result = handler.get()
+            video_url = result["video"]["url"]
+            return (video_url,)
+        except Exception as e:
+            print(f"Error generating video: {str(e)}")
+            return ("Error: Unable to generate video.",)
+
+class LoadVideoFromURL:
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "url": ("STRING", {"default": "https://example.com/video.mp4"}),
+                "force_rate": ("INT", {"default": 0, "min": 0, "max": 60, "step": 1}),
+                "force_size": (["Disabled", "Custom Height", "Custom Width", "Custom", "256x?", "?x256", "256x256", "512x?", "?x512", "512x512"],),
+                "custom_width": ("INT", {"default": 512, "min": 0, "max": 8192, "step": 8}),
+                "custom_height": ("INT", {"default": 512, "min": 0, "max": 8192, "step": 8}),
+                "frame_load_cap": ("INT", {"default": 0, "min": 0, "max": 1000000, "step": 1}),
+                "skip_first_frames": ("INT", {"default": 0, "min": 0, "max": 1000000, "step": 1}),
+                "select_every_nth": ("INT", {"default": 1, "min": 1, "max": 1000000, "step": 1}),
+            },
+        }
+
+    RETURN_TYPES = ("IMAGE", "INT", "VHS_VIDEOINFO")
+    RETURN_NAMES = ("frames", "frame_count", "video_info")
+    FUNCTION = "load_video_from_url"
+    CATEGORY = "♾️Mixlab/Video"
+
+    def load_video_from_url(self, url, force_rate, force_size, custom_width, custom_height, frame_load_cap, skip_first_frames, select_every_nth):
+        # Download the video to a temporary file
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") as temp_file:
+            response = requests.get(url, stream=True)
+            for chunk in response.iter_content(chunk_size=8192):
+                temp_file.write(chunk)
+            temp_file_path = temp_file.name
+
+        # Load the video using OpenCV
+        cap = cv2.VideoCapture(temp_file_path)
+        
+        # Get video properties
+        fps = cap.get(cv2.CAP_PROP_FPS)
+        total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+        width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+        height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+        duration = total_frames / fps
+
+        # Calculate target size
+        if force_size != "Disabled":
+            if force_size == "Custom Width":
+                new_height = int(height * (custom_width / width))
+                new_width = custom_width
+            elif force_size == "Custom Height":
+                new_width = int(width * (custom_height / height))
+                new_height = custom_height
+            elif force_size == "Custom":
+                new_width, new_height = custom_width, custom_height
+            else:
+                target_width, target_height = map(int, force_size.replace("?", "0").split("x"))
+                if target_width == 0:
+                    new_width = int(width * (target_height / height))
+                    new_height = target_height
+                else:
+                    new_height = int(height * (target_width / width))
+                    new_width = target_width
+        else:
+            new_width, new_height = width, height
+
+        frames = []
+        frame_count = 0
+
+        for i in range(total_frames):
+            ret, frame = cap.read()
+            if not ret:
+                break
+
+            if i < skip_first_frames:
+                continue
+
+            if (i - skip_first_frames) % select_every_nth != 0:
+                continue
+
+            if force_size != "Disabled":
+                frame = cv2.resize(frame, (new_width, new_height))
+
+            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            frame = torch.from_numpy(frame).float() / 255.0
+            frames.append(frame)
+
+            frame_count += 1
+
+            if frame_load_cap > 0 and frame_count >= frame_load_cap:
+                break
+
+        cap.release()
+        os.unlink(temp_file_path)
+
+        frames = torch.stack(frames)
+
+        video_info = {
+            "source_fps": fps,
+            "source_frame_count": total_frames,
+            "source_duration": duration,
+            "source_width": width,
+            "source_height": height,
+            "loaded_fps": fps if force_rate == 0 else force_rate,
+            "loaded_frame_count": frame_count,
+            "loaded_duration": frame_count / (fps if force_rate == 0 else force_rate),
+            "loaded_width": new_width,
+            "loaded_height": new_height,
+        }
+
+        return (frames, frame_count, video_info)
+

nodes/ImageNode.py

@@ -13,12 +13,26 @@ import json,io
 import comfy.utils
 from comfy.cli_args import args
 import cv2
-import string
+import string,re
 import math,glob
 from .Watcher import FolderWatcher
 
 from itertools import product
 
+
+# 文件名排序
+def sort_by_filename(items):
+    def extract_parts(filename):
+        # 使用正则表达式将文件名拆分为数字和非数字部分
+        parts = re.split(r'(\d+)', filename)
+        # 将数字部分转换为整数以便正确排序，同时保留非数字部分
+        parts = [int(part) if part.isdigit() else part for part in parts]
+        return parts
+
+    # 按照 file_name 的拆分部分进行排序
+    sorted_items = sorted(items, key=lambda x: extract_parts(x['file_name']))
+    return sorted_items
+
 # 将PIL图片转换为OpenCV格式
 def pil_to_opencv(image):
     open_cv_image = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
@@ -768,8 +782,7 @@ def areaToMask(x,y,w,h,image):
 #     return bg_image
 
 
-import cv2
-import numpy as np
+
 
 # ps的正片叠底
 # 可以基于https://www.cnblogs.com/jsxyhelu/p/16947810.html ，用gpt写python代码
@@ -1433,7 +1446,7 @@ class LoadImagesFromPath:
                             },
                 "optional":{
                     "white_bg": (["disable","enable"],),
-                    "newest_files": (["enable", "disable"],),
+                    "sort_by": (["file_name", "newest"],),#根据文件名来排序，还是按照最新创建时间
                     "index_variable":("INT", {
                         "default": 0, 
                         "min": -1, #Minimum value
@@ -1462,7 +1475,7 @@ class LoadImagesFromPath:
     watcher_folder=None
 
     # 运行的函数
-    def run(self,file_path,white_bg,newest_files,index_variable,watcher,result,prompt,seed=1):
+    def run(self,file_path,white_bg,sort_by,index_variable,watcher,result,prompt,seed=1):
         global watcher_folder
         # print('###监听:',watcher_folder,watcher,file_path,result)
 
@@ -1485,19 +1498,23 @@ class LoadImagesFromPath:
         # 当开启了监听，则取最新的，第一个文件
         if watcher=='enable':
             index_variable=0
-            newest_files='enable'
+            sort_by='newest'
 
         # 排序
-        sorted_files = sorted(images, key=lambda x: os.path.getmtime(x['file_path']), reverse=(newest_files=='enable'))
+        if sort_by=='newest':
+            sorted_files = sorted(images, key=lambda x: os.path.getmtime(x['file_path']), reverse=True)
+        elif sort_by=='file_name':
+            # 根据文件名排序
+            sorted_files = sort_by_filename(images)
 
         imgs=[]
         masks=[]
-        file_names=[]
+        file_paths=[]
 
         for im in sorted_files:
             imgs.append(im['image'])
             masks.append(im['mask'])
-            file_names.append(im['file_name'])
+            file_paths.append(im['file_path'])
         
         # print('index_variable',index_variable)
         
@@ -1505,13 +1522,13 @@ class LoadImagesFromPath:
             if index_variable!=-1:
                 imgs=[imgs[index_variable]] if index_variable < len(imgs) else None
                 masks=[masks[index_variable]] if index_variable < len(masks) else None
-                file_names=[file_names[index_variable]] if index_variable < len(file_names) else None
+                file_paths=[file_paths[index_variable]] if index_variable < len(file_paths) else None
         except Exception as e:
             print("发生了一个未知的错误：", str(e))
 
         # print('#prompt::::',prompt)
         # return  {"ui": {"seed": [1]}, "result":(imgs,masks,prompt,file_names,)}
-        return  (imgs,masks,prompt,file_names,)
+        return  (imgs,masks,prompt,file_paths,)
 
 
 # TODO 扩大选区的功能,重新输出mask
@@ -2873,14 +2890,14 @@ class ResizeImage:
                     "default": 512, 
                     "min": 1, #Minimum value
                     "max": 8192, #Maximum value
-                    "step": 8, #Slider's step
+                    "step": 1, #Slider's step
                     "display": "number" # Cosmetic only: display as "number" or "slider"
                 }),
                 "height": ("INT",{
                     "default": 512, 
                     "min": 1, #Minimum value
                     "max": 8192, #Maximum value
-                    "step": 8, #Slider's step
+                    "step": 1, #Slider's step
                     "display": "number" # Cosmetic only: display as "number" or "slider"
                 }),
                 "scale_option": (["width","height",'overall','center'],),

nodes/MiniCPMNode.py

@@ -1,4 +1,5 @@
 # Referenced some code：https://github.com/IuvenisSapiens/ComfyUI_MiniCPM-V-2_6-int4
+# https://github.com/CY-CHENYUE/ComfyUI-MiniCPM-Plus
 
 import os
 import torch
@@ -35,6 +36,7 @@ class MiniCPM_VQA_Simple:
                 "images": ("IMAGE",),
                 "text": ("STRING", {"default": "", "multiline": True}),   
                 "seed": ("INT", {"default": -1}),  # add seed parameter, default is -1
+                "extract_keywords":("BOOLEAN", {"default": False}),
                 "temperature": (
                     "FLOAT",
                     {
@@ -46,7 +48,9 @@ class MiniCPM_VQA_Simple:
             
         }
 
-    RETURN_TYPES = ("STRING",)
+    RETURN_TYPES = ("STRING","STRING",)
+    RETURN_NAMES = ("result","keywords",)
+
     FUNCTION = "inference"
     CATEGORY = "♾️Mixlab/Image"
 
@@ -55,6 +59,7 @@ class MiniCPM_VQA_Simple:
         images,
         text, 
         seed,  # add seed parameter, default is -1
+        extract_keywords,
         temperature,
         keep_model_loaded,
     ):
@@ -90,6 +95,7 @@ class MiniCPM_VQA_Simple:
                 torch_dtype=torch.bfloat16 if self.bf16_support else torch.float16,
             )
 
+
         with torch.no_grad():
             images = images.permute([0, 3, 1, 2])
             images = [ToPILImage()(img).convert("RGB") for img in images]
@@ -113,6 +119,30 @@ class MiniCPM_VQA_Simple:
                 # max_new_tokens=max_new_tokens,
                 **params,
             )
+
+            keyword_result=""
+
+            if extract_keywords:#extract_keywords
+                keyword_prompt = f"""Please extract keywords from the following text, including all occurrences of language (e.g. Chinese, English, etc.)：
+                [[[{result}]]]
+                Please list the keywords extracted, separated by commas. Make sure to include all important words, no matter what language. For English words, please keep the original case."""
+
+                keyword_msgs = [{'role': 'user', 'content': keyword_prompt}]
+                keyword_result = self.model.chat(
+                    image=None,
+                    msgs=keyword_msgs,
+                    tokenizer=self.tokenizer,
+                    sampling=True,
+                    # top_k=top_k,
+                    # top_p=top_p,
+                    temperature=temperature,
+                    # repetition_penalty=repetition_penalty,
+                    # max_new_tokens=max_new_tokens,
+                    **params,
+                )
+                print("keyword_result",keyword_result)
+
+               
             # offload model to GPU
             # self.model = self.model.to(torch.device("cpu"))
             # self.model.eval()
@@ -124,4 +154,4 @@ class MiniCPM_VQA_Simple:
                 torch.cuda.empty_cache()  # release GPU memory
                 torch.cuda.ipc_collect()
             # print(result)
-            return (result,)
+            return (result,keyword_result,)

nodes/PromptNode.py

@@ -187,7 +187,8 @@ class PromptImage:
                 }
             }
     
-    RETURN_TYPES = ()
+    RETURN_TYPES = ("STRING",)
+    RETURN_NAMES = ("json_str",)
    
     OUTPUT_NODE = True
 
@@ -202,12 +203,19 @@ class PromptImage:
         filename_prefix="mixlab_"
         filename_prefix += self.prefix_append
         full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(
-            filename_prefix, self.output_dir, images[0].shape[1], images[0].shape[0])
+            filename_prefix,self.output_dir, images[0].shape[1], images[0].shape[0])
         
+        full_output_folder=os.path.join(full_output_folder,'PromptImage')
+        subfolder='PromptImage'
+
         results = list()
 
         save_to_image=save_to_image[0]=='enable'
 
+        #保存到本地的json文件，记录图片和prompt的对应关系
+        output_images=[]
+        output_prompt=[]
+
         for index in range(len(images)):
             res=[]
             imgs=images[index]
@@ -215,24 +223,36 @@ class PromptImage:
             for image in imgs:
                 img=tensor2pil(image)
 
+                prompt_text=prompts[index]
+
                 metadata = None
                 if save_to_image:
                     metadata = PngInfo()
-                    prompt_text=prompts[index]
                     if prompt_text is not None:
                         metadata.add_text("prompt_text", prompt_text)
                     
                 file = f"{filename}_{index}_{counter:05}_.png"
-                img.save(os.path.join(full_output_folder, file), pnginfo=metadata, compress_level=self.compress_level)
+                fp=os.path.join(full_output_folder,file)
+                img.save(fp, pnginfo=metadata, compress_level=self.compress_level)
                 res.append({
                     "filename": file,
                     "subfolder": subfolder,
                     "type": self.type
                 })
+                output_images.append(fp)
+                output_prompt.append(prompt_text)
                 counter += 1
             results.append(res)
-        
-        return { "ui": { "_images": results,"prompts":prompts } }
+
+        # if save_to_image:
+        #     # 保存为本地文件
+        #     with open(os.path.join(full_output_folder,'PromptImage.json'), 'w') as file:
+        #         json.dump(output_dict, file, ensure_ascii=False, indent=4)
+
+        return { "ui": { "_images": results,"prompts":prompts },"result":(json.dumps({
+            "images":output_images,
+            "prompts":output_prompt
+        }),) }
 
 
 
@@ -664,7 +684,7 @@ class JoinWithDelimiter:
     def INPUT_TYPES(s):
         return {"required": {
                     "text_list": (any_type,),
-                    "delimiter":(["newline","comma","backslash","space"],),
+                    "delimiter":(["newline","comma","backslash","space","colon"],),
                              },
                 }
     
@@ -687,7 +707,13 @@ class JoinWithDelimiter:
             delimiter='\\'
         elif delimiter=='space':
             delimiter=' '
+        elif delimiter=='colon':
+            delimiter=":"
         t=''
         if isinstance(text_list, list):
             t=join_with_(text_list,delimiter)
+            # 如果只有一个元素，则末尾补充delimiter ，尤其适合制作播客的 name: 这种前缀
+            if len(text_list)==1:
+                t=t+delimiter
+
         return (t,)

nodes/SenseVoice.py

@@ -0,0 +1,226 @@
+# -*- coding:utf-8 -*-
+import logging
+import os
+import time
+
+from huggingface_hub import snapshot_download
+import torch,re
+from sensevoice.onnx.sense_voice_ort_session import SenseVoiceInferenceSession
+from sensevoice.utils.frontend import WavFrontend
+from sensevoice.utils.fsmn_vad import FSMNVad
+import comfy.utils
+import folder_paths
+
+languages = {"auto": 0, "zh": 3, "en": 4, "yue": 7, "ja": 11, "ko": 12, "nospeech": 13}
+
+# 设置环境变量
+os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'
+
+# 
+def get_model_path():
+    try:
+        return folder_paths.get_folder_paths('sense_voice')[0]
+    except:
+        return os.path.join(folder_paths.models_dir, "sense_voice")
+
+class AnyType(str):
+  """A special class that is always equal in not equal comparisons. Credit to pythongosssss"""
+
+  def __ne__(self, __value: object) -> bool:
+    return False
+
+any_type = AnyType("*")
+
+# 字幕
+def format_to_srt(channel_id, start_time_ms, end_time_ms, asr_result):
+    start_time = start_time_ms / 1000
+    end_time = end_time_ms / 1000
+    
+    def format_time(seconds):
+        hours = int(seconds // 3600)
+        minutes = int((seconds % 3600) // 60)
+        seconds = seconds % 60
+        milliseconds = int((seconds - int(seconds)) * 1000)
+        return f"{hours:02}:{minutes:02}:{int(seconds):02},{milliseconds:03}"
+    
+    start_time_str = format_time(start_time)
+    end_time_str = format_time(end_time)
+
+    pattern = r"<\|(.+?)\|><\|(.+?)\|><\|(.+?)\|><\|(.+?)\|>(.+)"
+    match = re.match(pattern,asr_result)
+    print('#format_to_srt',match,asr_result)
+    if match==None:
+        return None, None, None, None,None,start_time,end_time,None
+    lang, emotion, audio_type, itn, text = match.groups()
+     # 😊 表示高兴，😡 表示愤怒，😔 表示悲伤。对于音频事件，🎼 表示音乐，😀 表示笑声，👏 表示掌声
+    
+    srt_content = f"1\n{start_time_str} --> {end_time_str}\n{text}\n"
+    
+    logging.info(f"[Channel {channel_id}] [{start_time}s - {end_time}s] [{lang}] [{emotion}] [{audio_type}] [{itn}] {text}")
+
+    return lang, emotion, audio_type, itn,srt_content,start_time,end_time,text
+
+
+class SenseVoiceProcessor:
+    def __init__(self, download_model_path, device, num_threads, use_int8):
+
+        if not os.path.exists(download_model_path):
+            logging.info(
+                "Downloading model from huggingface hub from https://huggingface.co/lovemefan/SenseVoice-onnx"
+            )
+            logging.info(
+                "You can speed up with  `export HF_ENDPOINT=https://hf-mirror.com`"
+            )
+            snapshot_download(
+                repo_id="lovemefan/SenseVoice-onnx", local_dir=download_model_path
+            )
+
+        self.download_model_path = download_model_path
+        self.device = device
+        self.num_threads = num_threads
+        self.use_int8 = use_int8
+        self.front = WavFrontend(os.path.join(download_model_path, "am.mvn"))
+        self.model = SenseVoiceInferenceSession(
+            os.path.join(download_model_path, "embedding.npy"),
+            os.path.join(
+                download_model_path,
+                "sense-voice-encoder-int8.onnx"
+                if use_int8
+                else "sense-voice-encoder.onnx",
+            ),
+            os.path.join(download_model_path, "chn_jpn_yue_eng_ko_spectok.bpe.model"),
+            device,
+            num_threads,
+        )
+        self.vad = FSMNVad(download_model_path)
+
+    def process_audio(self, waveform, _sample_rate, language, use_itn):
+    
+        start = time.time()
+        pbar = comfy.utils.ProgressBar(waveform.shape[1])  # 进度条
+
+        results = []
+
+        for channel_id, channel_data in enumerate(waveform.T):
+            segments = self.vad.segments_offline(channel_data)
+            
+            for part in segments:
+                audio_feats = self.front.get_features(channel_data[part[0] * 16 : part[1] * 16])
+                asr_result = self.model(
+                    audio_feats[None, ...],
+                    language=languages[language],
+                    use_itn=use_itn,
+                )
+
+                lang, emotion, audio_type, itn,srt_content,start_time,end_time,text=format_to_srt(
+                    channel_id, 
+                    part[0] , 
+                    part[1], 
+                    asr_result)
+
+                if lang!=None:
+                    results.append({
+                        "language":lang, 
+                        "emotion":emotion,
+                        "audio_type":audio_type, 
+                        "itn":itn,
+                        "srt_content":srt_content,
+                        "start_time":start_time,
+                        "end_time":end_time,
+                        "text":text
+                    })
+
+            self.vad.vad.all_reset_detection()
+            pbar.update(1)  # 更新进度条
+
+        decoding_time = time.time() - start
+        logging.info(f"Decoder audio takes {decoding_time} seconds")
+        logging.info(f"The RTF is {decoding_time/(waveform.shape[1] * len(waveform) / _sample_rate)}.")
+        return results
+
+
+class SenseVoiceNode:
+
+    def __init__(self):
+        self.processor = None
+        self.download_model_path=get_model_path()
+        self.device="cpu"
+        self.num_threads = 4
+        self.use_int8 = True
+        self.language='auto'
+
+    @classmethod
+    def INPUT_TYPES(s):
+       
+        return {"required": {
+                    "audio": ("AUDIO", ), 
+                    "device": ( ['auto','cpu'], {"default": 'auto'}),
+                     "language": (list(languages.keys()), {"default": 'auto'}),# 不能直接写 languages.keys(),json.dumps会报错
+                     "num_threads":("INT",{
+                                "default":4, 
+                                "min": 1, #Minimum value
+                                "max": 32, #Maximum value
+                                "step": 1, #Slider's step
+                                "display": "number" # Cosmetic only: display as "number" or "slider"
+                                },),
+                     "use_int8":("BOOLEAN", {"default": True},),
+                     "use_itn":("BOOLEAN", {"default": True},),
+                     },
+                }
+
+    CATEGORY = "♾️Mixlab/Audio"
+
+    OUTPUT_NODE = True
+    FUNCTION = "run" 
+    
+    RETURN_TYPES = (any_type,"STRING","STRING","FLOAT",)
+    RETURN_NAMES = ("result","srt","text","total_seconds",)
+
+    def run(self,audio,device,language,num_threads,use_int8,use_itn ):
+ 
+        if device!=self.device:
+            self.device=device
+            self.processor=None
+        if language!=self.language:
+            self.language=language
+            self.processor=None
+        if num_threads!=self.num_threads:
+            self.num_threads=num_threads
+            self.processor=None
+        if use_int8!=self.use_int8:
+            self.use_int8=use_int8
+            self.processor=None
+        
+        if device=='auto' and torch.cuda.is_available():
+            self.device='cuda'
+
+        # num_threads=4
+        # use_int8=True
+
+        if self.processor==None:
+            self.processor = SenseVoiceProcessor(self.download_model_path, 
+                                                 self.device, 
+                                                 self.num_threads, 
+                                                 self.use_int8)
+
+        if 'waveform' in audio and 'sample_rate' in audio:
+            waveform = audio['waveform']
+            sample_rate = audio['sample_rate']
+            # print("Original shape:", waveform.shape)  # 打印原始形状
+            if waveform.ndim == 3 and waveform.shape[0] == 1:  # 检查是否为三维且 batch_size 为 1
+                waveform = waveform.squeeze(0)  # 移除 batch_size 维度
+            else:
+                raise ValueError("Unexpected waveform dimensions")
+
+            print("waveform.shape:", waveform.shape) 
+            total_length_seconds = waveform.shape[1] / sample_rate
+
+            waveform_numpy = waveform.numpy().transpose(1, 0)  # 转换为 (num_samples, num_channels)
+
+        results=self.processor.process_audio(waveform_numpy, sample_rate, language, use_itn)
+
+        srt_content="\n".join([s['srt_content'] for s in results])
+        text="\n".join([s['text'] for s in results])
+
+        return (results,srt_content,text,total_length_seconds,)
+ 

nodes/SocialProfile.py

@@ -0,0 +1,155 @@
+# 用于定义社交名片
+
+import json,os,re
+import folder_paths
+
+def save_to_json(file_path, data):
+    try:
+        print(f"Saving data to {file_path}")  # 调试信息
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(data, f, ensure_ascii=False, indent=4)
+        print("Data saved successfully.")  # 调试信息
+    except Exception as e:
+        print('#save_to_json', e)
+
+def read_from_json(file_path):
+    data = None
+    try:
+        with open(file_path, 'r', encoding='utf-8') as f:  # 使用 'r' 模式打开文件
+            data = json.load(f)
+    except Exception as e:
+        print('#read_from_json', e)
+        return None
+    return data
+
+
+default_agent={
+                    "user_name":"shadow",
+                    "profile_id":"ML000",
+                    "skills":"Design Hacker,Programmer,Architect,Experience Designer".split(",")
+                }
+
+def create_default_file(agent_dir):
+    fp=os.path.join(agent_dir,'shadow.json')
+    if not os.path.exists(fp):
+        save_to_json(fp,default_agent)
+
+def get_social_profile_dir():
+    try:
+        return folder_paths.get_folder_paths('agent')[0]
+    except:
+        agent_dir=os.path.join(folder_paths.models_dir, "agent")
+        if not os.path.exists(agent_dir):
+            os.makedirs(agent_dir, exist_ok=True)
+            create_default_file(agent_dir)
+        return agent_dir
+
+def list_all_json_files(directory_path):
+    json_files = []
+    
+    try:
+        for filename in os.listdir(directory_path):
+            if filename.endswith('.json'):
+                json_files.append(filename)
+                
+    except Exception as e:
+        print(f"An error occurred: {e}")
+    
+    return json_files
+
+
+def read_social_profiles():
+    agent_dir=get_social_profile_dir()
+    files=list_all_json_files(agent_dir)
+    if len(files)==0:
+        create_default_file(agent_dir)
+    files=list_all_json_files(agent_dir)
+    return files
+
+def save_social_profile_config(agent_dir,file_name,data):
+     save_to_json(os.path.join(agent_dir,file_name),data)
+
+
+
+agent_dir=get_social_profile_dir()
+# print('Watcher:',config_json)
+
+
+class NewSocialProfileNode:
+
+    @classmethod
+    def INPUT_TYPES(s):
+ 
+        user=default_agent
+
+        return {
+            "required": {
+                "user_name": ("STRING", {"forceInput": False, "default": user['user_name']}),
+                "profile_id": ("STRING", {"forceInput": False, "default": user['profile_id']}),
+                "skills": ("STRING", {"forceInput": False, "multiline": True, "default": ",".join(user['skills'])}),
+                "file_name": ("STRING", {"forceInput": False, "default": "agent"}),
+            },
+        }
+
+    # INPUT_IS_LIST = False
+    RETURN_TYPES = ("STRING",)
+    RETURN_NAMES = ("social_profile(json_string)",)
+    FUNCTION = "run"
+    # OUTPUT_IS_LIST = (False,)
+    CATEGORY = "♾️Mixlab/Agent"
+
+    def run(self, user_name, profile_id, skills,file_name):
+        
+        # 使用正则表达式匹配英文和中文逗号
+        skills_list = [skill.strip() for skill in re.split(r'[，,]', skills)]
+
+        # 创建社交名片的JSON对象
+        social_profile = {
+            "user_name": user_name,
+            "profile_id": profile_id,
+            "skills": skills_list,
+        }
+
+        # 将社交名片转换为JSON字符串
+        json_string = json.dumps(social_profile, ensure_ascii=False)
+
+        save_social_profile_config(agent_dir,file_name+'.json',social_profile)
+
+        # 返回JSON字符串和一个示例值
+        return (json_string,)
+
+
+class LoadSocialProfileNode:
+
+    @classmethod
+    def INPUT_TYPES(s):
+        files=read_social_profiles()
+        return {
+            "required": {
+                "file_name": ( files, 
+                    {"default": files[0]}),
+                "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff, "step": 1}),
+            },
+        }
+
+    # INPUT_IS_LIST = False
+    RETURN_TYPES = ("STRING",)
+    RETURN_NAMES = ("social_profile(json_string)",)
+    FUNCTION = "run"
+    # OUTPUT_IS_LIST = (False,)
+    CATEGORY = "♾️Mixlab/Agent"
+
+    def run(self,file_name,seed):
+        
+        social_profile=read_from_json(os.path.join(agent_dir,file_name))
+        # print("#read_from_json",os.path.join(agent_dir,file_name),social_profile)
+        if social_profile==None:
+            return ("",)
+
+        social_profile['file_name']=file_name
+
+        # 将社交名片转换为JSON字符串
+        json_string = json.dumps(social_profile, ensure_ascii=False)
+
+        # 返回JSON字符串和一个示例值
+        return (json_string,)

nodes/Utils.py

@@ -648,6 +648,49 @@ class AppInfo:
     
 
     
+class CreateJsonNode:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { 
+                    "key": ("STRING",{"multiline": False,"default": "data","dynamicPrompts": False}),
+                    "value":(any_type,),
+                    "save":("BOOLEAN", {"default": True},),
+                             }, 
+                "optional":{
+                    "json_str":("STRING", {"forceInput": True,}),
+                }
+
+                }
+    
+    RETURN_TYPES = ("STRING",)
+    RETURN_NAMES = ("json_str",)
+
+    FUNCTION = "run"
+
+    CATEGORY = "♾️Mixlab/Utils"
+
+    OUTPUT_NODE = True
+    INPUT_IS_LIST = False
+    # OUTPUT_IS_LIST = (True,)
+
+    def run(self,key,value,save,json_str=None):
+        data={}
+
+        data[key]=value
+
+        if json_str:
+            json_obj = json.loads(json_str)
+            data.update(json_obj)
+
+        if save:
+            # 保存为本地文件
+            with open(os.path.join(folder_paths.get_output_directory(),'data.json'), 'w') as file:
+                json.dump(data, file, ensure_ascii=False, indent=4)
+
+        return (json.dumps(data),)
+    
+
+    
 
 
 class SwitchByIndex:

nodes/Whisper.py

@@ -0,0 +1,173 @@
+import os,re
+import sys,time
+from pathlib import Path
+import torchaudio
+import hashlib
+import torch
+import folder_paths
+import comfy.utils
+
+from faster_whisper import WhisperModel
+
+class AnyType(str):
+  """A special class that is always equal in not equal comparisons. Credit to pythongosssss"""
+
+  def __ne__(self, __value: object) -> bool:
+    return False
+
+any_type = AnyType("*")
+
+def get_model_dir(m):
+    try:
+        return folder_paths.get_folder_paths(m)[0]
+    except:
+        return os.path.join(folder_paths.models_dir, m)
+
+
+
+whisper_model_path=get_model_dir('whisper')
+
+model_sizes=[
+                d for d in os.listdir(whisper_model_path) if os.path.isdir(
+                    os.path.join(whisper_model_path, d)
+                    ) and os.path.isfile(os.path.join(os.path.join(whisper_model_path, d), "config.json"))
+                    ]
+
+
+class LoadWhisperModel:
+    def __init__(self):
+        self.model = None
+        self.device="cuda" if torch.cuda.is_available() else "cpu"
+        self.model_size=model_sizes[0]
+        self.compute_type='float16'
+
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": {
+            "model_size": (model_sizes,),
+            "device": (["auto","cpu"],),
+            "compute_type": (["float16","int8_float16","int8"],),
+                             },
+                }
+    
+    RETURN_TYPES = ("WHISPER",)
+    RETURN_NAMES = ("whisper_model",)
+
+    FUNCTION = "run"
+
+    CATEGORY = "♾️Mixlab/Audio/Whisper"
+
+    INPUT_IS_LIST = False
+    OUTPUT_IS_LIST = (False,)
+
+    def run(self,model_size,device,compute_type):
+
+        if device=="auto" and self.device!='cuda':
+            self.device="cuda" if torch.cuda.is_available() else "cpu"
+            self.model=None
+
+        if device=='cpu' and self.device!='cpu':
+            self.device="cpu"
+            self.model=None
+        
+        if model_size!= self.model_size:
+            self.model_size=model_size
+            self.model=None
+
+        if compute_type!=self.compute_type:
+            self.compute_type=compute_type
+            self.model=None
+
+        if self.model==None:
+            self.model = WhisperModel(
+                os.path.join(whisper_model_path, self.model_size), 
+                device=self.device,
+                compute_type=self.compute_type
+                )
+
+        return (self.model,)
+    
+
+class WhisperTranscribe:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": {
+                                "whisper_model": ("WHISPER",),
+                                "audio": ("AUDIO",),
+                             },
+                }
+    
+    RETURN_TYPES = (any_type,"STRING","STRING","FLOAT",)
+    RETURN_NAMES = ("result","srt","text","total_seconds",)
+
+    FUNCTION = "run"
+
+    CATEGORY = "♾️Mixlab/Audio/Whisper"
+
+    INPUT_IS_LIST = False
+    # OUTPUT_IS_LIST = (False,False,False,)
+    
+    def run(self,whisper_model,audio):
+
+        if 'audio_path' in audio and (not 'waveform' in audio):
+            waveform, sample_rate = torchaudio.load(audio['audio_path'])
+            waveform=waveform.mean(0)
+            total_length_seconds = waveform.shape[0] / sample_rate
+            waveform=waveform.numpy()
+            
+        elif 'waveform' in audio and 'sample_rate' in audio:
+            print("Original shape:", audio["waveform"].shape, isinstance(audio["waveform"], torch.Tensor))  # 打印原始形状
+            waveform = audio["waveform"].squeeze(0)  # Remove the added batch dimension
+            sample_rate = audio["sample_rate"]
+
+            # if audio_sf != sampling_rate:
+            #     waveform = torchaudio.functional.resample(
+            #         waveform, orig_freq=audio_sf, new_freq=sampling_rate
+            #     )
+            
+            waveform=waveform.mean(0)
+
+            total_length_seconds = waveform.shape[0] / sample_rate
+
+            waveform=waveform.numpy() #whisper_model.transcribe 旧版不支持直接传tensor，先用numpy
+        
+        segments, info = whisper_model.transcribe(waveform, beam_size=5)
+
+        print("Detected language '%s' with probability %f" % (info.language, info.language_probability))
+
+        # Function to format time for SRT
+        def format_time(seconds):
+            millis = int((seconds - int(seconds)) * 1000)
+            hours, remainder = divmod(int(seconds), 3600)
+            minutes, seconds = divmod(remainder, 60)
+            return f"{hours:02}:{minutes:02}:{seconds:02},{millis:03}"
+
+        # Prepare SRT content as a string
+        results = []
+        for i, segment in enumerate(segments):
+            start_time = format_time(segment.start)
+            end_time = format_time(segment.end)
+            srt_content = f"{i + 1}\n"
+            srt_content += f"{start_time} --> {end_time}\n"
+
+            text=segment.text.strip()
+
+            srt_content += f"{text}\n\n"
+
+            start_time=segment.start
+            end_time=segment.end
+            
+
+            results.append({ 
+                    "srt_content":srt_content,
+                    "start_time":start_time,
+                    "end_time":end_time,
+                    "text":text,
+                    "language":[info.language]
+            })
+        
+        srt_content="\n".join([s['srt_content'] for s in results])
+        text="\n".join([s['text'] for s in results])
+
+        return (results,srt_content,text,total_length_seconds,)
+

nodes/scenedetectNode.py

@@ -132,6 +132,9 @@ def split_video_by_scenes(video_path, scenes, output_path, number_of_sample_fram
     width = int(video.get(cv2.CAP_PROP_FRAME_WIDTH))
     height = int(video.get(cv2.CAP_PROP_FRAME_HEIGHT))
     
+    # 视频的总帧数
+    total_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
+
     # Create a list to hold the paths of the scene videos
     scenes_video = []
     keyframes = []
@@ -194,7 +197,7 @@ def split_video_by_scenes(video_path, scenes, output_path, number_of_sample_fram
     # Release the video file
     video.release()
     
-    return scenes_video, keyframes
+    return scenes_video, keyframes,total_frames
 
 
 def get_files_with_extension(directory, extension):
@@ -287,9 +290,9 @@ class ScenedetectNode_:
                      "number_of_sample_frames": ("INT", {"default": 1, "min": 1, "step": 1}), # 抽取的帧数，默认是1帧，中间帧
                      },}
 
-    RETURN_TYPES = ("SCENE_VIDEO","SCENE_", "INT",)
-    RETURN_NAMES = ("scenes_video","scenes","scene_len",)
-    OUTPUT_IS_LIST = (False,False,False,)
+    RETURN_TYPES = ("SCENE_VIDEO","SCENE_", "INT","INT",)
+    RETURN_NAMES = ("scenes_video","scenes","scene_len","total_frames",)
+    # OUTPUT_IS_LIST = (False,False,False,)
 
     FUNCTION = "run"
     CATEGORY = "♾️Mixlab/Video"
@@ -310,9 +313,9 @@ class ScenedetectNode_:
         folder_path = create_folder(tp,name_without_extension)
         # print("New folder created:", folder_path)
 
-        vs_files,keyframes=split_video_by_scenes(video_path,scenes,folder_path,number_of_sample_frames)
+        vs_files,keyframes,total=split_video_by_scenes(video_path,scenes,folder_path,number_of_sample_frames)
         # print("New folder created:", vs_files)
 
-        return (vs_files,keyframes,len(scenes),)
+        return (vs_files,keyframes,len(scenes),total,)
     
 

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "comfyui-mixlab-nodes"
 description = "3D, ScreenShareNode & FloatingVideoNode, SpeechRecognition & SpeechSynthesis, GPT, LoadImagesFromLocal, Layers, Other Nodes, ..."
-version = "0.42.0"
+version = "0.46.0"
 license = "MIT"
 dependencies = ["numpy", "pyOpenSSL", "watchdog", "opencv-python-headless", "matplotlib", "openai", "simple-lama-inpainting", "clip-interrogator==0.6.0", "transformers>=4.36.0", "lark-parser", "imageio-ffmpeg", "rembg[gpu]", "omegaconf==2.3.0", "Pillow>=9.5.0", "einops==0.7.0", "trimesh>=4.0.5", "huggingface-hub", "scikit-image"]
 

requirements.txt

@@ -27,4 +27,11 @@ scenedetect[opencv-headless]
 hydra-core>=1.3.2
 loralib>=0.1.2
 natsort>=8.4.0
-# simple-lama-inpainting
+
+#simple-lama-inpainting
+
+git+https://github.com/shadowcz007/SenseVoice-python.git
+
+faster_whisper
+
+git+https://github.com/openai/swarm.git

web/javascript/checkVersion_mixlab.js

@@ -3,7 +3,7 @@ import { app } from '../../../scripts/app.js'
 const repoOwner = 'shadowcz007' // 替换为仓库的所有者
 const repoName = 'comfyui-mixlab-nodes' // 替换为仓库的名称
 
-const version = 'v0.42.0'
+const version = 'v0.46.0'
 
 fetch(`https://api.github.com/repos/${repoOwner}/${repoName}/releases/latest`)
   .then(response => response.json())

web/javascript/td_background.js

@@ -6,6 +6,10 @@ window._bg_img = null
  * draws the back canvas (the one containing the background and the connections)
  * @method drawBackCanvas
  **/
+
+// 判断是否是新版的，LGraphCanvas.prototype.drawBackCanvas.toString().match('window.devicePixelRatio')
+let scale=LGraphCanvas.prototype.drawBackCanvas.toString().match('window.devicePixelRatio')?window.devicePixelRatio:1;
+
 LGraphCanvas.prototype.drawBackCanvas = function () {
   var canvas = this.bgcanvas
   if (
@@ -60,7 +64,7 @@ LGraphCanvas.prototype.drawBackCanvas = function () {
   if (!this.viewport) {
     ctx.restore()
     // ctx.setTransform(1, 0, 0, 1, 0, 0)
-    ctx.setTransform(window.devicePixelRatio, 0, 0, window.devicePixelRatio, 0, 0)
+    ctx.setTransform(scale, 0, 0, scale, 0, 0)
   }
   this.visible_links.length = 0
 

web/javascript/ui_mixlab.js

@@ -544,7 +544,7 @@ async function getCustomnodeMappings () {
     const data = (await get_nodes_map()).data
     window._nodes_maps = data
   }
-  console.log('#getCustomnodeMappings', window._nodes_maps)
+  // console.log('#getCustomnodeMappings', window._nodes_maps)
   for (let url in window._nodes_maps) {
     let n = window._nodes_maps[url]
     for (let node of n[0]) {

web/javascript/visualization.js

@@ -41,6 +41,9 @@ class Visualizer {
       overflow: 'hidden'
     })
     this.iframe.src = '/mixlab/app/' + visualSrc + '.html'
+    // this.iframe.width="300";
+    // this.iframe.height="400";
+
     console.log('#Visualizer', container, this.iframe)
     container.appendChild(this.iframe)
   }
@@ -73,7 +76,7 @@ function createVisualizer (node, inputName, typeName, inputData, app) {
     draw: function (ctx, node, widgetWidth, widgetY, widgetHeight) {
       const margin = 10
       const top_offset = 5
-      const visible = app.canvas.ds.scale > 0.5 && this.type === typeName
+      const visible = app.canvas.ds.scale > 0.3 && this.type === typeName
       const w = widgetWidth - margin * 4
       const clientRectBound = ctx.canvas.getBoundingClientRect()
       const transform = new DOMMatrix()
@@ -85,12 +88,13 @@ function createVisualizer (node, inputName, typeName, inputData, app) {
         .translateSelf(margin, margin + widgetY)
 
       Object.assign(this.visualizer.style, {
-        left: `${transform.a * margin + transform.e + 40}px`,
+        left: `${transform.a * margin + transform.e + 0}px`,
         top: `${transform.d + transform.f + top_offset}px`,
         width: `${w * transform.a}px`,
-        height: `${
-          w * transform.d - widgetHeight - margin * 15 * transform.d
-        }px`,
+        height: `${(w * transform.a * 4) / 3 - margin * 5 * transform.d}px`,
+        // height: `${
+        //   w * transform.d - widgetHeight - margin * 15 * transform.d
+        // }px`,
         position: 'absolute',
         overflow: 'hidden',
         zIndex: app.graph._nodes.indexOf(node)
@@ -137,11 +141,11 @@ function createVisualizer (node, inputName, typeName, inputData, app) {
   // Make sure visualization iframe is always inside the node when resize the node
   node.onResize = function () {
     let [w, h] = this.size
-    if (w <= 600) w = 600
-    if (h <= 500) h = 500
+    if (w <= 300) w = 300
+    if (h <= 400) h = 400
 
-    if (w > 600) {
-      h = w - 100
+    if (w > 300) {
+      h = Math.round((w * 4) / 3)
     }
 
     this.size = [w, h]
@@ -181,14 +185,14 @@ function registerVisualizer (nodeType, nodeData, nodeClassName, typeName) {
         app
       ])
 
-      this.setSize([600, 500])
+      this.setSize([300, 400])
 
       return r
     }
 
     nodeType.prototype.onExecuted = async function (message) {
       // Check if reference image and depth map are available
-      console.log("#message",message)
+      console.log('#message', message)
       if (message.reference_image && message.depth_map) {
         const params = {}
         params.reference_image = message.reference_image[0]

webApp/custom.html

@@ -2026,8 +2026,8 @@
             var iframe = document.createElement('iframe')
             iframe.src = "https://mememagic-editor.vercel.app/"
             iframe.setAttribute('frameborder', '0')
-            iframe.setAttribute('width', '500')
-            iframe.setAttribute('height', '700')
+            iframe.setAttribute('width', '700')
+            iframe.setAttribute('height', '720')
             iframe.setAttribute('allow',"clipboard-read; clipboard-write")
            
 

webApp/index.html

@@ -223,6 +223,10 @@
             margin-top: 0;
         }
 
+        .image-with-grid img {
+            margin: 0 !important;
+        }
+
         /* .card:hover {
     box-shadow: 0px 0px 10px 10px #e9fbfa;
 } */

webApp/lib/threeVisualizer.js

@@ -237,8 +237,11 @@ const sleep = (t = 1000) => {
 // 方法：旋转摄像机并拍摄图片 // 每次旋转的角度增量，转换为弧度
 async function captureImages (
   totalFrames = 20,
-  angleIncrement = THREE.MathUtils.degToRad(1.5)
+  angleIncrement = 1.5,
+  scaleFactor = 1 // 添加放大倍数参数，默认为1
 ) {
+  angleIncrement = THREE.MathUtils.degToRad(angleIncrement)
+
   // 计算场景中所有物体的中心点
   const box = new THREE.Box3().setFromObject(scene)
   const center = new THREE.Vector3()
@@ -264,6 +267,21 @@ async function captureImages (
   const startAngle = initialAngle
   //  - (angleIncrement * totalFrames) / 2
 
+  // 保存原始尺寸
+  const originalWidth = renderer.domElement.width
+  const originalHeight = renderer.domElement.height
+
+  // 调整渲染器尺寸
+  renderer.setSize(
+    originalWidth * scaleFactor,
+    originalHeight * scaleFactor,
+    false
+  )
+
+  // 调整相机的视图矩阵（如果需要）
+  camera.aspect = (originalWidth * scaleFactor) / (originalHeight * scaleFactor)
+  camera.updateProjectionMatrix()
+
   for (let i = 0; i < totalFrames; i++) {
     const angle = startAngle + i * angleIncrement
 
@@ -284,6 +302,13 @@ async function captureImages (
     await new Promise(resolve => setTimeout(resolve, 500))
   }
 
+  // 恢复渲染器尺寸
+  renderer.setSize(originalWidth, originalHeight, false)
+
+  // 恢复相机的视图矩阵
+  camera.aspect = originalWidth / originalHeight
+  camera.updateProjectionMatrix()
+
   // 恢复相机到初始位置和朝向
   camera.position.copy(initialPosition)
   camera.lookAt(initialTarget)
@@ -294,7 +319,7 @@ async function captureImages (
 async function takeScreenshot () {
   // 更新相机的矩阵，以确保其世界矩阵是最新的
   camera.updateMatrixWorld()
-  const imgs = await captureImages()
+  const imgs = await captureImages(12,3,4)
 
   // 获取当前网页的 URL
   const currentUrl = window.location.href

workflow/swarm制作的播客节点workflow.json

@@ -0,0 +1,416 @@
+{
+  "last_node_id": 9,
+  "last_link_id": 8,
+  "nodes": [
+    {
+      "id": 3,
+      "type": "TextInput_",
+      "pos": [
+        137,
+        420
+      ],
+      "size": {
+        "0": 400,
+        "1": 200
+      },
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "outputs": [
+        {
+          "name": "STRING",
+          "type": "STRING",
+          "links": [
+            2
+          ],
+          "shape": 3,
+          "slot_index": 0
+        }
+      ],
+      "title": "使用 Azure OpenAI",
+      "properties": {
+        "Node name for S&R": "TextInput_"
+      },
+      "widgets_values": [
+        "https://mixcopilot.openai.azure.com"
+      ]
+    },
+    {
+      "id": 2,
+      "type": "KeyInput",
+      "pos": [
+        144,
+        257
+      ],
+      "size": {
+        "0": 315,
+        "1": 70
+      },
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "outputs": [
+        {
+          "name": "key",
+          "type": "STRING",
+          "links": [
+            1
+          ],
+          "shape": 3,
+          "slot_index": 0
+        }
+      ],
+      "title": "使用你自己的key",
+      "properties": {
+        "Node name for S&R": "KeyInput"
+      },
+      "widgets_values": [
+        null,
+        null
+      ]
+    },
+    {
+      "id": 6,
+      "type": "MultiPersonPodcast",
+      "pos": [
+        1099,
+        480
+      ],
+      "size": [
+        481.8963185574753,
+        268.61682945154007
+      ],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "speaker",
+          "type": "SPEAKER",
+          "link": 7,
+          "slot_index": 0
+        },
+        {
+          "name": "text",
+          "type": "STRING",
+          "link": 4,
+          "widget": {
+            "name": "text"
+          }
+        }
+      ],
+      "outputs": [
+        {
+          "name": "audio_list",
+          "type": "AUDIO",
+          "links": null,
+          "shape": 3
+        },
+        {
+          "name": "audio",
+          "type": "AUDIO",
+          "links": [
+            8
+          ],
+          "shape": 3,
+          "slot_index": 1
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "MultiPersonPodcast"
+      },
+      "widgets_values": [
+        "小明：大家好，欢迎收听本周的《AI新动态》。我是主持人小明，今天我们有两位嘉宾，分别是小李和小王。大家跟听众打个招呼吧！\n小李：大家好，我是小李，很高兴今天能和大家聊聊最新的AI动态。\n小王：大家好，我是小王，也很期待今天的讨论。",
+        0,
+        0,
+        0,
+        0,
+        false,
+        1
+      ]
+    },
+    {
+      "id": 7,
+      "type": "LoadSpeaker",
+      "pos": [
+        584,
+        567
+      ],
+      "size": {
+        "0": 315,
+        "1": 58
+      },
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "outputs": [
+        {
+          "name": "speaker",
+          "type": "SPEAKER",
+          "links": [
+            6
+          ],
+          "shape": 3,
+          "slot_index": 0
+        }
+      ],
+      "title": "opus",
+      "properties": {
+        "Node name for S&R": "LoadSpeaker"
+      },
+      "widgets_values": [
+        "opus_00001"
+      ]
+    },
+    {
+      "id": 1,
+      "type": "SimulateDevDesignDiscussions",
+      "pos": [
+        611,
+        201
+      ],
+      "size": [
+        391.9864763335838,
+        217.95792637114943
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "api_key",
+          "type": "STRING",
+          "link": 1,
+          "widget": {
+            "name": "api_key"
+          }
+        },
+        {
+          "name": "custom_model_name",
+          "type": "STRING",
+          "link": null,
+          "widget": {
+            "name": "custom_model_name"
+          }
+        },
+        {
+          "name": "custom_api_url",
+          "type": "STRING",
+          "link": 2,
+          "widget": {
+            "name": "custom_api_url"
+          }
+        }
+      ],
+      "outputs": [
+        {
+          "name": "text",
+          "type": "STRING",
+          "links": [
+            3,
+            4
+          ],
+          "shape": 3,
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SimulateDevDesignDiscussions"
+      },
+      "widgets_values": [
+        "数字艺术好看吗？",
+        "gpt-4o",
+        "openai",
+        "",
+        "",
+        ""
+      ]
+    },
+    {
+      "id": 8,
+      "type": "RenameSpeaker",
+      "pos": [
+        593,
+        681
+      ],
+      "size": {
+        "0": 315,
+        "1": 58
+      },
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "speaker",
+          "type": "SPEAKER",
+          "link": 6
+        }
+      ],
+      "outputs": [
+        {
+          "name": "speaker",
+          "type": "SPEAKER",
+          "links": [
+            7
+          ],
+          "shape": 3,
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "RenameSpeaker"
+      },
+      "widgets_values": [
+        "主持人"
+      ]
+    },
+    {
+      "id": 5,
+      "type": "ShowTextForGPT",
+      "pos": [
+        1071,
+        128
+      ],
+      "size": [
+        624.2005965936271,
+        279.47889630613906
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "text",
+          "type": "STRING",
+          "link": 3,
+          "widget": {
+            "name": "text"
+          }
+        },
+        {
+          "name": "output_dir",
+          "type": "STRING",
+          "link": null,
+          "widget": {
+            "name": "output_dir"
+          }
+        }
+      ],
+      "outputs": [
+        {
+          "name": "STRING",
+          "type": "STRING",
+          "links": null,
+          "shape": 6
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "ShowTextForGPT"
+      },
+      "widgets_values": [
+        "",
+        "",
+        "* 主持人：作为一名设计师，你如何定义“好看”的数字艺术？\n设计师：好看的数字艺术？就像你在沙漠中看到绿洲的那一刻，它能吸引你的眼球，抓住你的心，它能传达情感，让人产生共鸣。可能是颜色的对撞，也可能是形状的魔法，总之，它让你想多看几眼，还想收藏到你的精神博物馆里。\n* 主持人：程序员，你们在开发支持数字艺术的软件时，如何确保用户体验的直观性和美观性？\n程序员：哎呀，这可是门艺术活啊！这时候我们可不像写代码那样呆板，想象力飞起来。我们会尽量让界面简洁好用，不搞那些让人摸不着头脑的功能。动效啥的也要调校好，太多就变花里胡哨了，太少用户觉得干巴巴。最重要的是，多听设计师的，他们可是颜值担当啊！\n* 主持人：站在设计师的角度，你觉得技术如何影响了数字艺术的表现力？\n设计师：技术啊，那可是我们的魔法棒！有了高端的硬件和软件，我们可以在屏幕上玩出各种花样，大到宇宙，小到细胞，想象力在技术的加持下，才能飞得更高更远。不管是3D渲染，还是AR互动，技术就是让我们的创意从草图变成现实的桥梁，让我们画布上的每一个像素都能发光。\n* 主持人：不知道程序员又是怎么看待数字艺术的后台开发和前端展示关系的呢？\n程序员：后端和前端就像魔法师和舞台演员。后端是幕后默默挥舞魔法杖，搞定数据处理啊、服务器啥的，让那台机器运转得顺溜。前端呢，就是站在舞台中央光彩夺目，把数据和功能打包成美美的界面展示给用户。说白了，后端是灵魂，前端是颜值，两个缺一不可，配合得好才是真正的艺术！\n* 主持人：感谢大家的参与，今天关于数字艺术的讨论让我受益匪浅。\n程序员：不客气，代码和艺术的碰撞总是火花四射！\n\n设计师：没错，灵感和技术结合，才能创作出让人惊艳的作品。期待下次再聊！"
+      ]
+    },
+    {
+      "id": 9,
+      "type": "PreviewAudio",
+      "pos": [
+        1740,
+        453
+      ],
+      "size": {
+        "0": 315,
+        "1": 76
+      },
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "audio",
+          "type": "AUDIO",
+          "link": 8
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "PreviewAudio"
+      },
+      "widgets_values": [
+        null
+      ]
+    }
+  ],
+  "links": [
+    [
+      1,
+      2,
+      0,
+      1,
+      0,
+      "STRING"
+    ],
+    [
+      2,
+      3,
+      0,
+      1,
+      2,
+      "STRING"
+    ],
+    [
+      3,
+      1,
+      0,
+      5,
+      0,
+      "STRING"
+    ],
+    [
+      4,
+      1,
+      0,
+      6,
+      1,
+      "STRING"
+    ],
+    [
+      6,
+      7,
+      0,
+      8,
+      0,
+      "SPEAKER"
+    ],
+    [
+      7,
+      8,
+      0,
+      6,
+      0,
+      "SPEAKER"
+    ],
+    [
+      8,
+      6,
+      1,
+      9,
+      0,
+      "AUDIO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "ds": {
+      "scale": 1.3310000000000006,
+      "offset": [
+        -361.773434010461,
+        27.423855709687306
+      ]
+    }
+  },
+  "version": 0.4
+}

workflow/video-all-in-one-test-workflow.json

@@ -0,0 +1,775 @@
+{
+  "last_node_id": 18,
+  "last_link_id": 15,
+  "nodes": [
+    {
+      "id": 6,
+      "type": "LoadImage",
+      "pos": [
+        -32,
+        79
+      ],
+      "size": {
+        "0": 315,
+        "1": 314
+      },
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "outputs": [
+        {
+          "name": "IMAGE",
+          "type": "IMAGE",
+          "links": [
+            1,
+            2,
+            3
+          ],
+          "shape": 3,
+          "slot_index": 0
+        },
+        {
+          "name": "MASK",
+          "type": "MASK",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadImage"
+      },
+      "widgets_values": [
+        "1.jpg",
+        "image"
+      ]
+    },
+    {
+      "id": 7,
+      "type": "KeyInput",
+      "pos": [
+        -34,
+        585
+      ],
+      "size": {
+        "0": 315,
+        "1": 94
+      },
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "outputs": [
+        {
+          "name": "key",
+          "type": "STRING",
+          "links": [
+            4,
+            5,
+            6
+          ],
+          "shape": 3,
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "KeyInput"
+      },
+      "widgets_values": [
+        null,
+        null
+      ]
+    },
+    {
+      "id": 10,
+      "type": "LoadVideoFromURL",
+      "pos": [
+        1038,
+        21
+      ],
+      "size": [
+        315,
+        266
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "url",
+          "type": "STRING",
+          "link": 7,
+          "widget": {
+            "name": "url"
+          }
+        }
+      ],
+      "outputs": [
+        {
+          "name": "frames",
+          "type": "IMAGE",
+          "links": [
+            8
+          ],
+          "shape": 3,
+          "slot_index": 0
+        },
+        {
+          "name": "frame_count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        },
+        {
+          "name": "video_info",
+          "type": "VHS_VIDEOINFO",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideoFromURL"
+      },
+      "widgets_values": [
+        "https://example.com/video.mp4",
+        0,
+        "Disabled",
+        512,
+        512,
+        0,
+        0,
+        1
+      ]
+    },
+    {
+      "id": 13,
+      "type": "LoadVideoFromURL",
+      "pos": [
+        1032,
+        715
+      ],
+      "size": {
+        "0": 315,
+        "1": 266
+      },
+      "flags": {},
+      "order": 9,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "url",
+          "type": "STRING",
+          "link": 10,
+          "widget": {
+            "name": "url"
+          }
+        }
+      ],
+      "outputs": [
+        {
+          "name": "frames",
+          "type": "IMAGE",
+          "links": [
+            12
+          ],
+          "shape": 3,
+          "slot_index": 0
+        },
+        {
+          "name": "frame_count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        },
+        {
+          "name": "video_info",
+          "type": "VHS_VIDEOINFO",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideoFromURL"
+      },
+      "widgets_values": [
+        "https://example.com/video.mp4",
+        0,
+        "Disabled",
+        512,
+        512,
+        0,
+        0,
+        1
+      ]
+    },
+    {
+      "id": 11,
+      "type": "PreviewImage",
+      "pos": [
+        1436,
+        6
+      ],
+      "size": [
+        210,
+        246
+      ],
+      "flags": {},
+      "order": 11,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "link": 8
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "PreviewImage"
+      }
+    },
+    {
+      "id": 3,
+      "type": "VideoGenKlingNode",
+      "pos": [
+        527.4816383216086,
+        19.100152539671797
+      ],
+      "size": {
+        "0": 400,
+        "1": 200
+      },
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image",
+          "type": "IMAGE",
+          "link": 1
+        },
+        {
+          "name": "fal_key",
+          "type": "STRING",
+          "link": 4,
+          "widget": {
+            "name": "fal_key"
+          }
+        }
+      ],
+      "outputs": [
+        {
+          "name": "STRING",
+          "type": "STRING",
+          "links": [
+            7,
+            13
+          ],
+          "shape": 3,
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VideoGenKlingNode"
+      },
+      "widgets_values": [
+        "The man is shaking his head with a wry smile.\n\n",
+        "5",
+        "16:9",
+        "standard",
+        ""
+      ]
+    },
+    {
+      "id": 16,
+      "type": "ShowTextForGPT",
+      "pos": [
+        1702,
+        -85
+      ],
+      "size": [
+        400,
+        200
+      ],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "text",
+          "type": "STRING",
+          "link": 13,
+          "widget": {
+            "name": "text"
+          }
+        },
+        {
+          "name": "output_dir",
+          "type": "STRING",
+          "link": null,
+          "widget": {
+            "name": "output_dir"
+          }
+        }
+      ],
+      "outputs": [
+        {
+          "name": "STRING",
+          "type": "STRING",
+          "links": null,
+          "shape": 6
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "ShowTextForGPT"
+      },
+      "widgets_values": [
+        "",
+        "",
+        "https://v2.fal.media/files/0f9f44093c7442d5b0819616880bca65_output.mp4"
+      ]
+    },
+    {
+      "id": 4,
+      "type": "VideoGenRunwayGen3Node",
+      "pos": [
+        529,
+        297
+      ],
+      "size": {
+        "0": 400,
+        "1": 200
+      },
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image",
+          "type": "IMAGE",
+          "link": 2
+        },
+        {
+          "name": "fal_key",
+          "type": "STRING",
+          "link": 5,
+          "widget": {
+            "name": "fal_key"
+          }
+        }
+      ],
+      "outputs": [
+        {
+          "name": "STRING",
+          "type": "STRING",
+          "links": [
+            9,
+            14
+          ],
+          "shape": 3,
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VideoGenRunwayGen3Node"
+      },
+      "widgets_values": [
+        "The man is shaking his head with a wry smile.\n\n",
+        "5",
+        "16:9",
+        ""
+      ]
+    },
+    {
+      "id": 5,
+      "type": "VideoGenLumaDreamMachineNode",
+      "pos": [
+        534,
+        570
+      ],
+      "size": {
+        "0": 400,
+        "1": 200
+      },
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image",
+          "type": "IMAGE",
+          "link": 3
+        },
+        {
+          "name": "fal_key",
+          "type": "STRING",
+          "link": 6,
+          "widget": {
+            "name": "fal_key"
+          }
+        }
+      ],
+      "outputs": [
+        {
+          "name": "STRING",
+          "type": "STRING",
+          "links": [
+            10,
+            15
+          ],
+          "shape": 3,
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VideoGenLumaDreamMachineNode"
+      },
+      "widgets_values": [
+        "The man is shaking his head with a wry smile.\n\n",
+        "16:9",
+        "",
+        true
+      ]
+    },
+    {
+      "id": 18,
+      "type": "ShowTextForGPT",
+      "pos": [
+        1719,
+        665
+      ],
+      "size": [
+        400,
+        200
+      ],
+      "flags": {},
+      "order": 10,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "text",
+          "type": "STRING",
+          "link": 15,
+          "widget": {
+            "name": "text"
+          }
+        },
+        {
+          "name": "output_dir",
+          "type": "STRING",
+          "link": null,
+          "widget": {
+            "name": "output_dir"
+          }
+        }
+      ],
+      "outputs": [
+        {
+          "name": "STRING",
+          "type": "STRING",
+          "links": null,
+          "shape": 6
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "ShowTextForGPT"
+      },
+      "widgets_values": [
+        "",
+        "",
+        "https://v2.fal.media/files/a622a5aac002452ba0e75f7d8871389d_output.mp4"
+      ]
+    },
+    {
+      "id": 12,
+      "type": "LoadVideoFromURL",
+      "pos": [
+        1049,
+        349
+      ],
+      "size": {
+        "0": 315,
+        "1": 266
+      },
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "url",
+          "type": "STRING",
+          "link": 9,
+          "widget": {
+            "name": "url"
+          }
+        }
+      ],
+      "outputs": [
+        {
+          "name": "frames",
+          "type": "IMAGE",
+          "links": [
+            11
+          ],
+          "shape": 3,
+          "slot_index": 0
+        },
+        {
+          "name": "frame_count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        },
+        {
+          "name": "video_info",
+          "type": "VHS_VIDEOINFO",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideoFromURL"
+      },
+      "widgets_values": [
+        "https://example.com/video.mp4",
+        0,
+        "Disabled",
+        512,
+        512,
+        0,
+        0,
+        1
+      ]
+    },
+    {
+      "id": 17,
+      "type": "ShowTextForGPT",
+      "pos": [
+        1711,
+        318
+      ],
+      "size": [
+        400,
+        200
+      ],
+      "flags": {},
+      "order": 8,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "text",
+          "type": "STRING",
+          "link": 14,
+          "widget": {
+            "name": "text"
+          }
+        },
+        {
+          "name": "output_dir",
+          "type": "STRING",
+          "link": null,
+          "widget": {
+            "name": "output_dir"
+          }
+        }
+      ],
+      "outputs": [
+        {
+          "name": "STRING",
+          "type": "STRING",
+          "links": null,
+          "shape": 6
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "ShowTextForGPT"
+      },
+      "widgets_values": [
+        "",
+        "",
+        "https://v2.fal.media/files/755d51c8984a4445a852268a209b07e4_output.mp4"
+      ]
+    },
+    {
+      "id": 15,
+      "type": "PreviewImage",
+      "pos": [
+        1444,
+        650
+      ],
+      "size": [
+        210,
+        246
+      ],
+      "flags": {},
+      "order": 13,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "link": 12
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "PreviewImage"
+      }
+    },
+    {
+      "id": 14,
+      "type": "PreviewImage",
+      "pos": [
+        1416,
+        310
+      ],
+      "size": [
+        210,
+        246
+      ],
+      "flags": {},
+      "order": 12,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "link": 11
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "PreviewImage"
+      }
+    }
+  ],
+  "links": [
+    [
+      1,
+      6,
+      0,
+      3,
+      0,
+      "IMAGE"
+    ],
+    [
+      2,
+      6,
+      0,
+      4,
+      0,
+      "IMAGE"
+    ],
+    [
+      3,
+      6,
+      0,
+      5,
+      0,
+      "IMAGE"
+    ],
+    [
+      4,
+      7,
+      0,
+      3,
+      1,
+      "STRING"
+    ],
+    [
+      5,
+      7,
+      0,
+      4,
+      1,
+      "STRING"
+    ],
+    [
+      6,
+      7,
+      0,
+      5,
+      1,
+      "STRING"
+    ],
+    [
+      7,
+      3,
+      0,
+      10,
+      0,
+      "STRING"
+    ],
+    [
+      8,
+      10,
+      0,
+      11,
+      0,
+      "IMAGE"
+    ],
+    [
+      9,
+      4,
+      0,
+      12,
+      0,
+      "STRING"
+    ],
+    [
+      10,
+      5,
+      0,
+      13,
+      0,
+      "STRING"
+    ],
+    [
+      11,
+      12,
+      0,
+      14,
+      0,
+      "IMAGE"
+    ],
+    [
+      12,
+      13,
+      0,
+      15,
+      0,
+      "IMAGE"
+    ],
+    [
+      13,
+      3,
+      0,
+      16,
+      0,
+      "STRING"
+    ],
+    [
+      14,
+      4,
+      0,
+      17,
+      0,
+      "STRING"
+    ],
+    [
+      15,
+      5,
+      0,
+      18,
+      0,
+      "STRING"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "ds": {
+      "scale": 0.7247295000000012,
+      "offset": [
+        -59.526177135875514,
+        197.10624904779425
+      ]
+    }
+  },
+  "version": 0.4
+}