feat: support input audio

comfy-nodes/input_audio.py

@@ -0,0 +1,195 @@
+import folder_paths
+import node_helpers
+
+from PIL import Image, ImageOps, ImageSequence, ImageFile
+import numpy as np
+import torch
+import os
+import uuid
+import tqdm
+import torchaudio
+import hashlib
+from comfy_extras.nodes_audio import SaveAudio
+
+
+class LoadAudio:
+    @classmethod
+    def INPUT_TYPES(s):
+        input_dir = folder_paths.get_input_directory()
+        files = folder_paths.filter_files_content_types(
+            os.listdir(input_dir), ["audio", "video"])
+        return {"required": {"audio": (sorted(files), {"audio_upload": True})}}
+
+    CATEGORY = "audio"
+
+    RETURN_TYPES = ("AUDIO", )
+    FUNCTION = "load"
+
+    def load(self, audio):
+        audio_path = folder_paths.get_annotated_filepath(audio)
+        waveform, sample_rate = torchaudio.load(audio_path)
+        audio = {"waveform": waveform.unsqueeze(0), "sample_rate": sample_rate}
+        return (audio, )
+
+    @classmethod
+    def IS_CHANGED(s, audio):
+        image_path = folder_paths.get_annotated_filepath(audio)
+        m = hashlib.sha256()
+        with open(image_path, 'rb') as f:
+            m.update(f.read())
+        return m.digest().hex()
+
+    @classmethod
+    def VALIDATE_INPUTS(s, audio):
+        if not folder_paths.exists_annotated_filepath(audio):
+            return "Invalid audio file: {}".format(audio)
+        return True
+
+
+class ShellAgentPluginInputAudio:
+    @classmethod
+    def INPUT_TYPES(s):
+        input_dir = folder_paths.get_input_directory()
+        files = folder_paths.filter_files_content_types(
+            os.listdir(input_dir), ["audio", "video"])
+        return {
+            "required": {
+                "input_name": (
+                    "STRING",
+                    {"multiline": False, "default": "input_audio", "forceInput": False},
+                ),
+                "default_value": (
+                    sorted(files), {"audio_upload": True, "forceInput": False}
+                ),
+            },
+            "optional": {
+                "description": (
+                    "STRING",
+                    {"multiline": True, "default": "", "forceInput": False},
+                ),
+            }
+        }
+
+    RETURN_TYPES = ("AUDIO", )
+    FUNCTION = "load"
+
+    CATEGORY = "shellagent"
+
+    @classmethod
+    def validate(cls, **kwargs):
+        schema = {
+            "title": kwargs["input_name"],
+            "type": "string",
+            "default": kwargs["default_value"],
+            "description": kwargs.get("description", ""),
+            "url_type": "audio"
+        }
+        return schema
+
+    @classmethod
+    def VALIDATE_INPUTS(s, audio):
+        if not folder_paths.exists_annotated_filepath(audio):
+            return "Invalid audio file: {}".format(audio)
+        return True
+
+    @classmethod
+    def VALIDATE_INPUTS(s, input_name, default_value, description=""):
+        audio = default_value
+        if audio.startswith("http"):
+            return True
+
+        if not folder_paths.exists_annotated_filepath(audio):
+            return "Invalid audio file: {}".format(audio)
+        return True
+
+    def load(self, input_name, default_value=None, display_name=None, description=None):
+        input_dir = folder_paths.get_input_directory()
+        audio_path = default_value
+        try:
+            if audio_path.startswith('http'):
+                import requests
+                from io import BytesIO
+                print("Fetching audio from url: ", audio_path)
+                response = requests.get(audio_path)
+                response.raise_for_status()
+                audio_file = BytesIO(response.content)
+                waveform, sample_rate = torchaudio.load(audio_file)
+            else:
+                if not os.path.isfile(audio_path):  # abs path
+                    # local path
+                    audio_path = os.path.join(input_dir, audio_path)
+                waveform, sample_rate = torchaudio.load(audio_path)
+
+            audio = {"waveform": waveform.unsqueeze(
+                0), "sample_rate": sample_rate}
+            return (audio, )
+            # image = ImageOps.exif_transpose(image)
+            # image = image.convert("RGB")
+            # image = np.array(image).astype(np.float32) / 255.0
+            # image = torch.from_numpy(image)[None,]
+            # return [image]
+        except Exception as e:
+            raise e
+
+
+class ShellAgentSaveAudios(SaveAudio):
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": {"audio": ("AUDIO", ),
+                "output_name": ("STRING", {"multiline": False, "default": "output_audio"},),
+                             "filename_prefix": ("STRING", {"default": "audio/ComfyUI"})},
+                "hidden": {"prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO"},
+                }
+    # {
+    #         "required": {
+    #             "images": ("IMAGE", {"tooltip": "The audio to save."}),
+    #             "output_name": ("STRING", {"multiline": False, "default": "output_image"},),
+    #             "filename_prefix": ("STRING", {"default": "ComfyUI", "tooltip": "The prefix for the file to save. This may include formatting information such as %date:yyyy-MM-dd% or %Empty Latent Image.width% to include values from nodes."})
+    #         },
+    #         "hidden": {
+    #             "prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO"
+    #         },
+    #     }
+
+    CATEGORY = "shellagent"
+
+    @classmethod
+    def validate(cls, **kwargs):
+        schema = {
+            "title": kwargs["output_name"],
+            "type": "array",
+            "items": {
+                "type": "string",
+                "url_type": "audio",
+            }
+        }
+        return schema
+
+    def save_audio(self, audio, filename_prefix="ComfyUI", prompt=None, extra_pnginfo=None, **extra_kwargs):
+        results = super().save_audio(audio, filename_prefix, prompt, extra_pnginfo)
+        results["shellagent_kwargs"] = extra_kwargs
+        return results
+    
+    
+class ShellAgentSaveAudio(ShellAgentSaveAudios):
+    @classmethod
+    def validate(cls, **kwargs):
+        schema = {
+            "title": kwargs["output_name"],
+            "type": "string",
+            "url_type": "audio",
+        }
+        return schema
+
+
+NODE_CLASS_MAPPINGS = {
+    "ShellAgentPluginInputAudio": ShellAgentPluginInputAudio,
+    "ShellAgentPluginSaveAudios": ShellAgentSaveAudios,
+    "ShellAgentPluginSaveAudio": ShellAgentSaveAudio,
+}
+
+NODE_DISPLAY_NAME_MAPPINGS = {
+    "ShellAgentPluginInputAudio": "Input Audio (ShellAgent Plugin)",
+    "ShellAgentPluginSaveAudios": "Save Audios (ShellAgent Plugin)",
+    "ShellAgentPluginSaveAudio": "Save Audio (ShellAgent Plugin)",
+}

comfy-nodes/input_image.py

@@ -7,7 +7,33 @@ import torch
 import os
 import uuid
 import tqdm
+from io import BytesIO
+import PIL
+import cv2
+from pillow_heif import register_heif_opener
 
+register_heif_opener()
+
+def safe_open_image(image_bytes):
+    try:
+        image_pil = Image.open(BytesIO(image_bytes))
+    except PIL.UnidentifiedImageError as e:
+        print(e)
+        # Convert response content (bytes) to a NumPy array
+        image_array = np.frombuffer(image_bytes, np.uint8)
+        
+        # Decode the image from the NumPy array (OpenCV format: BGR)
+        image_cv = cv2.imdecode(image_array, cv2.IMREAD_COLOR)
+        
+        if image_cv is not None:
+            # Convert the BGR image to RGB
+            image_rgb = cv2.cvtColor(image_cv, cv2.COLOR_BGR2RGB)
+            
+            # Convert the RGB NumPy array to a PIL Image
+            image_pil = Image.fromarray(image_rgb)
+        else:
+            raise ValueError("The image cannot be identified by neither PIL nor OpenCV")
+    return image_pil
 
 class ShellAgentPluginInputImage:
     @classmethod
@@ -55,9 +81,16 @@ class ShellAgentPluginInputImage:
     @classmethod
     def VALIDATE_INPUTS(s, input_name, default_value, description=""):
         image = default_value
+        
         if image.startswith("http"):
             return True
         
+        if image == "":
+            return "Invalid image file: please check if the image is empty or invalid"
+        
+        if os.path.isfile(image):
+            return True
+        
         if not folder_paths.exists_annotated_filepath(image):
             return "Invalid image file: {}".format(image)
 
@@ -103,6 +136,7 @@ class ShellAgentPluginInputImage:
 
         return (output_image, output_mask)
 
+
     def run(self, input_name, default_value=None, display_name=None, description=None):
         input_dir = folder_paths.get_input_directory()
         image_path = default_value
@@ -112,7 +146,7 @@ class ShellAgentPluginInputImage:
                 from io import BytesIO
                 print("Fetching image from url: ", image_path)
                 response = requests.get(image_path)
-                image = Image.open(BytesIO(response.content))
+                image = safe_open_image(response.content)
             elif image_path.startswith('data:image/png;base64,') or image_path.startswith('data:image/jpeg;base64,') or image_path.startswith('data:image/jpg;base64,'):
                 import base64
                 from io import BytesIO
@@ -124,7 +158,7 @@ class ShellAgentPluginInputImage:
                 if not os.path.isfile(image_path): # abs path
                     # local path
                     image_path = os.path.join(input_dir, image_path)
-                image = Image.open(image_path).convert("RGB")
+                image = node_helpers.pillow(Image.open, image_path)
 
             return self.convert_image_mask(image)
             # image = ImageOps.exif_transpose(image)

custom_routes.py

@@ -31,9 +31,10 @@ import nodes
 import traceback
 import re
 import keyword
+import uuid
 
-from .dependency_checker import resolve_dependencies
-
+from .dependency_checker import resolve_dependencies, inspect_repo_version
+from folder_paths import base_path as BASE_PATH
 
 WORKFLOW_ROOT = "shellagent/comfy_workflow"
 
@@ -193,4 +194,34 @@ async def shellagent_export(request):
             "message_detail": str(traceback.format_exc()),
             "message": str(e),
         }
-    return web.json_response(return_dict, status=status)
+    return web.json_response(return_dict, status=status)
+
+
+@server.PromptServer.instance.routes.post("/shellagent/inspect_version") # data same as queue prompt, plus workflow_name
+async def shellagent_inspect_version(request):
+    data = await request.json()
+    comfyui_version = inspect_repo_version(BASE_PATH)
+    comfyui_shellagent_plugin_version = inspect_repo_version(os.path.dirname(__file__))
+    return_dict = {
+        "comfyui_version": comfyui_version,
+        "comfyui_shellagent_plugin_version": comfyui_shellagent_plugin_version,
+    }
+    return web.json_response(return_dict, status=200)
+
+
+@server.PromptServer.instance.routes.post("/shellagent/get_mac_addr") # data same as queue prompt, plus workflow_name
+async def shellagent_get_mac_addr(request):
+    data = await request.json()
+    return_dict = {
+        "mac_addr": uuid.getnode()
+    }
+    return web.json_response(return_dict, status=200)
+
+@server.PromptServer.instance.routes.post("/shellagent/check_exist") # check if the file or folder exist
+async def shellagent_check_exist(request):
+    data = await request.json()
+
+    return_dict = {
+        "exist": uuid.getnode() == data["mac_addr"] and os.path.exists(data["path"]) # really exist, instead of same name
+    }
+    return web.json_response(return_dict, status=200)

dependency_checker.py

@@ -22,7 +22,7 @@ node_deps_info = json.load(open(os.path.join(os.path.dirname(__file__), "node_de
 node_blacklist = json.load(open(os.path.join(os.path.dirname(__file__), "node_blacklist.json")))
 node_remote_skip_models = json.load(open(os.path.join(os.path.dirname(__file__), "node_remote.json")))
 
-model_suffix = [".ckpt", ".safetensors", ".bin", ".pth", ".pt", ".onnx", ".gguf", ".sft"]
+model_suffix = [".ckpt", ".safetensors", ".bin", ".pth", ".pt", ".onnx", ".gguf", ".sft", ".ttf"]
 extra_packages = ["transformers", "timm", "diffusers", "accelerate"]
 
 
@@ -283,7 +283,10 @@ def resolve_dependencies(prompt, custom_dependencies): # resolve custom nodes an
             print(f"failed to resolve repo info of {custom_node}")
         requirement_file = os.path.join(BASE_PATH, custom_node.replace(".", "/"), "requirements.txt")
         if os.path.isfile(requirement_file):
-            requirements_lines += open(requirement_file).readlines()
+            try:
+                requirements_lines += open(requirement_file).readlines()
+            except:
+                pass
     requirements_lines = list(set(requirements_lines))
     requirements_packages = [package_name for package_name, version_specifier in map(split_package_version, requirements_lines) if package_name is not None]
     package_names = set(requirements_packages + extra_packages)

node_deps_info.json

@@ -1,33 +1,45 @@
 {
-  "ComfyUI-Easy-Use": [
-    {
-      "name": "ComfyUI-Inspire-Pack",
-      "repo": "https://github.com/ltdrdata/ComfyUI-Inspire-Pack.git",
-      "commit": ""
-    },
-    {
-      "name": "ComfyUI-Advanced-ControlNet",
-      "repo": "https://github.com/Kosinkadink/ComfyUI-Advanced-ControlNet.git",
-      "commit": ""
-    },
-    {
-        "name": "ComfyUI_smZNodes",
-        "repo": "https://github.com/shiimizu/ComfyUI_smZNodes.git",
-        "commit": ""
-      }
-  ],
-  "efficiency-nodes-comfyui": [
-    {
-      "name": "comfyui_controlnet_aux",
-      "repo": "https://github.com/Fannovel16/comfyui_controlnet_aux.git",
-      "commit": ""
-    }
-  ],
-  "ComfyUI-Anyline": [
-    {
-      "name": "comfyui_controlnet_aux",
-      "repo": "https://github.com/Fannovel16/comfyui_controlnet_aux.git",
-      "commit": ""
-    }
-  ]
-}
+    "ComfyUI-Easy-Use": [
+        {
+            "name": "ComfyUI-Inspire-Pack",
+            "repo": "https://github.com/ltdrdata/ComfyUI-Inspire-Pack.git",
+            "commit": ""
+        },
+        {
+            "name": "ComfyUI-Advanced-ControlNet",
+            "repo": "https://github.com/Kosinkadink/ComfyUI-Advanced-ControlNet.git",
+            "commit": ""
+        },
+        {
+            "name": "ComfyUI_smZNodes",
+            "repo": "https://github.com/shiimizu/ComfyUI_smZNodes.git",
+            "commit": ""
+        },
+        {
+            "name": "ComfyUI_IPAdapter_plus",
+            "repo": "https://github.com/cubiq/ComfyUI_IPAdapter_plus.git",
+            "commit": ""
+        }
+    ],
+    "efficiency-nodes-comfyui": [
+        {
+            "name": "comfyui_controlnet_aux",
+            "repo": "https://github.com/Fannovel16/comfyui_controlnet_aux.git",
+            "commit": ""
+        }
+    ],
+    "ComfyUI-Anyline": [
+        {
+            "name": "comfyui_controlnet_aux",
+            "repo": "https://github.com/Fannovel16/comfyui_controlnet_aux.git",
+            "commit": ""
+        }
+    ],
+    "ComfyUI-Impact-Pack": [
+        {
+            "name": "ComfyUI-Impact-Subpack",
+            "repo": "https://github.com/ltdrdata/ComfyUI-Impact-Subpack.git",
+            "commit": ""
+        }
+    ]
+}

requirements.txt

@@ -3,4 +3,6 @@ pydantic
 opencv-python
 imageio-ffmpeg
 brotli
-# logfire
+pillow_heif
+easydict
+# logfire

web/shellagent.js

@@ -1,6 +1,9 @@
 import { app } from "../../scripts/app.js";
 import { api } from "../../scripts/api.js";
 
+var __defProp = Object.defineProperty;
+var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
+
 app.registerExtension({
   name: "Shellagent.extension",
   async setup() {
@@ -173,6 +176,18 @@ app.registerExtension({
       }
     }
 
+    if (nodeData.name === "ShellAgentPluginInputAudio") {
+      if (
+        nodeData?.input?.required?.default_value?.[1]?.audio_upload === true
+      ) {
+        nodeData.input.required.audioUI = ["AUDIO_UI"];
+        nodeData.input.required.upload = [
+          "SHELLAGENT_AUDIOUPLOAD",
+          { widget: "default_value" },
+        ];
+      }
+    }
+
     if (nodeData.name === "ShellAgentPluginInputVideo") {
       addUploadWidget(nodeType, nodeData, "default_value");
       chainCallback(nodeType.prototype, "onNodeCreated", function () {
@@ -354,23 +369,81 @@ app.registerExtension({
 
   afterConfigureGraph(missingNodeTypes, app) {
     function addIn(type, nodeId) {
+      if(LiteGraph.slot_types_default_in[type] == null) {
+        LiteGraph.slot_types_default_in[type] = []
+      }
       if (LiteGraph.slot_types_default_in[type].indexOf(nodeId) === -1) {
         LiteGraph.slot_types_default_in[type].unshift(nodeId)
       }
     }
 
     function addOut(type, nodeId) {
+      if(LiteGraph.slot_types_default_out[type] == null) {
+        LiteGraph.slot_types_default_out[type] = []
+      }
       if (LiteGraph.slot_types_default_out[type].indexOf(nodeId) === -1) {
         LiteGraph.slot_types_default_out[type].unshift(nodeId)
       }
     }
 
     addIn('IMAGE', 'ShellAgentPluginInputImage')
+    addIn('AUDIO', 'ShellAgentPluginInputAudio')
     addOut('IMAGE', 'ShellAgentPluginSaveImage')
     addOut('IMAGE', 'ShellAgentPluginSaveImages')
+    addOut('AUDIO', 'ShellAgentPluginSaveAudios')
+    addOut('AUDIO', 'ShellAgentPluginSaveAudio')
     addOut('STRING', 'ShellAgentPluginOutputInteger')
     addOut('STRING', 'ShellAgentPluginOutputFloat')
     addOut('STRING', 'ShellAgentPluginOutputText')
+  },
+  getCustomWidgets() {
+    return {
+      SHELLAGENT_AUDIOUPLOAD(node, inputName) {
+        const audioWidget = node.widgets.find(
+          (w) => w.name === "default_value"
+        );
+        const audioUIWidget = node.widgets.find(
+          (w) => w.name === "audioUI"
+        );
+        const onAudioWidgetUpdate = /* @__PURE__ */ __name(() => {
+          audioUIWidget.element.src = api.apiURL(
+            getResourceURL(...splitFilePath(audioWidget.value))
+          );
+        }, "onAudioWidgetUpdate");
+        if (audioWidget.value) {
+          onAudioWidgetUpdate();
+        }
+        audioWidget.callback = onAudioWidgetUpdate;
+        const onGraphConfigured = node.onGraphConfigured;
+        node.onGraphConfigured = function() {
+          onGraphConfigured?.apply(this, arguments);
+          if (audioWidget.value) {
+            onAudioWidgetUpdate();
+          }
+        };
+        const fileInput = document.createElement("input");
+        fileInput.type = "file";
+        fileInput.accept = "audio/*";
+        fileInput.style.display = "none";
+        fileInput.onchange = () => {
+          if (fileInput.files.length) {
+            uploadFileAudio(audioWidget, audioUIWidget, fileInput.files[0], true);
+          }
+        };
+        const uploadWidget = node.addWidget(
+          "button",
+          inputName,
+          /* value=*/
+          "",
+          () => {
+            fileInput.click();
+          },
+          { serialize: false }
+        );
+        uploadWidget.label = "choose file to upload";
+        return { widget: uploadWidget };
+      }
+    };
   }
 });
 
@@ -744,4 +817,55 @@ function addLoadVideoCommon(nodeType, nodeData) {
       }
     });
   });
+}
+
+function getResourceURL(subfolder, filename, type = "input") {
+  const params = [
+    "filename=" + encodeURIComponent(filename),
+    "type=" + type,
+    "subfolder=" + subfolder,
+    app.getRandParam().substring(1)
+  ].join("&");
+  return `/view?${params}`;
+}
+
+function splitFilePath(path) {
+  const folder_separator = path.lastIndexOf("/");
+  if (folder_separator === -1) {
+    return ["", path];
+  }
+  return [
+    path.substring(0, folder_separator),
+    path.substring(folder_separator + 1)
+  ];
+}
+
+async function uploadFileAudio(audioWidget, audioUIWidget, file2, updateNode, pasted = false) {
+  try {
+    const body = new FormData();
+    body.append("image", file2);
+    if (pasted) body.append("subfolder", "pasted");
+    const resp = await api.fetchApi("/upload/image", {
+      method: "POST",
+      body
+    });
+    if (resp.status === 200) {
+      const data = await resp.json();
+      let path = data.name;
+      if (data.subfolder) path = data.subfolder + "/" + path;
+      if (!audioWidget.options.values.includes(path)) {
+        audioWidget.options.values.push(path);
+      }
+      if (updateNode) {
+        audioUIWidget.element.src = api.apiURL(
+          getResourceURL(...splitFilePath(path))
+        );
+        audioWidget.value = path;
+      }
+    } else {
+      window.alert(resp.status + " - " + resp.statusText);
+    }
+  } catch (error) {
+    window.alert(error);
+  }
 }