From e2b5cda983d0aa2ad41e4822bdc2953b7940858d Mon Sep 17 00:00:00 2001 From: jueming Date: Wed, 16 Apr 2025 13:33:07 +0800 Subject: [PATCH] init code --- README.md | 104 ++++++++++++++++++- README_CN.md | 104 +++++++++++++++++++ __init__.py | 20 ++++ nodes/__init__.py | 1 + nodes/load_audio.py | 63 ++++++++++++ nodes/text_to_speech.py | 177 ++++++++++++++++++++++++++++++++ nodes/voice_cloning.py | 151 ++++++++++++++++++++++++++++ requirements.txt | 1 + web/index.js | 1 + web/jm_audio_upload.js | 217 ++++++++++++++++++++++++++++++++++++++++ 10 files changed, 838 insertions(+), 1 deletion(-) create mode 100644 README_CN.md create mode 100644 __init__.py create mode 100644 nodes/__init__.py create mode 100644 nodes/load_audio.py create mode 100644 nodes/text_to_speech.py create mode 100644 nodes/voice_cloning.py create mode 100644 requirements.txt create mode 100644 web/index.js create mode 100644 web/jm_audio_upload.js diff --git a/README.md b/README.md index 267cc42..d61614d 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,104 @@ # ComfyUI-JM-MiniMax-API -ComfyUI MiniMax Custom Nodes + +A collection of ComfyUI custom nodes that integrate with MiniMax API services. + +[中文文档](README_CN.md) + +## Features + +### Speech Nodes (JM-MiniMax-API/Speech) + +- **Text to Speech**: Convert text to natural-sounding speech using MiniMax's advanced text-to-speech API +- **Voice Cloning**: Clone voices from audio samples +- **Load Audio**: Load and preview audio files for voice cloning + +## Installation + +1. Clone this repository to your ComfyUI custom_nodes folder: +```bash +cd ComfyUI/custom_nodes +git clone https://github.com/yourusername/ComfyUI-JM-MiniMax-API.git +``` + +2. Install dependencies: +```bash +pip install -r requirements.txt +``` + +3. Restart ComfyUI + +## Usage + +### Load Audio Node + +This node allows you to load and preview audio files for voice cloning. + +#### Input Parameters: +- **audio_path**: Select an existing audio file from the input directory +- **Upload Button**: Click to upload a new audio file (supports .mp3, .wav, .m4a) + +#### Output: +- **audio_file**: Absolute path to the selected audio file + +### Voice Cloning Node + +This node uses MiniMax's voice cloning API to clone voices from audio samples. + +#### Input Parameters: +- **api_key**: MiniMax API key +- **group_id**: MiniMax group ID +- **audio_file**: Path to the audio file to clone (connect from Load Audio node) +- **voice_id**: ID for the cloned voice (at least 8 characters, start with a letter, include numbers) +- **need_noise_reduction**: Whether to apply noise reduction (true/false) +- **need_volume_normalization**: Whether to normalize volume (true/false) +- **preview_text**: Optional text to preview the cloned voice (max 300 characters) +- **model**: TTS model to use for preview +- **accuracy**: Cloning accuracy threshold (0.0-1.0) + +#### Output: +- **voice_id**: ID of the cloned voice (can be connected to TextToSpeech node) + +### Text to Speech Node + +This node uses MiniMax's API to convert text to speech. + +#### Input Parameters: +- **api_key**: MiniMax API key +- **group_id**: MiniMax group ID +- **text**: Text to convert (up to 5000 characters) +- **model**: TTS model selection (speech-02-hd, speech-02-turbo, etc.) +- **voice_id**: Voice selection (multiple options available) +- **custom_voice_id** (optional): Custom voice ID from voice cloning (overrides voice_id if provided) +- **speed**: Speech rate (0.5 to 2.0) +- **volume**: Volume (0.1 to 10.0) +- **pitch**: Pitch adjustment (-12 to 12) +- **emotion**: Emotion selection (happy, sad, etc.) +- **subtitle_enable**: Whether to generate subtitle file (true/false) +- **filename_prefix**: Prefix for output filenames +- **language_boost** (optional): Language enhancement + +#### Output: +- **audio_path**: Absolute path to the generated audio file +- **subtitle_path**: Absolute path to the generated subtitle file (if subtitles enabled) + +#### Output File Details: +1. Audio File (audio_path): + - Format: MP3 + - Filename format: prefix_YYYYMMDD-HHMMSS.mp3 + +2. Subtitle File (subtitle_path): + - Format: JSON + - Filename format: prefix_subtitle_YYYYMMDD-HHMMSS.json + - Contains sentence-level timestamps accurate to milliseconds + +## Workflow Example + +1. Use **Load Audio** node to upload or select an audio file +2. Connect the output to a **Voice Cloning** node to clone the voice +3. Connect the voice_id output to a **Text to Speech** node's custom_voice_id input +4. Enter text and configure other parameters in the Text to Speech node +5. Run the workflow to generate speech using the cloned voice + +## License + +MIT License diff --git a/README_CN.md b/README_CN.md new file mode 100644 index 0000000..185d715 --- /dev/null +++ b/README_CN.md @@ -0,0 +1,104 @@ +# ComfyUI-JM-MiniMax-API + +一个集成 MiniMax API 服务的 ComfyUI 自定义节点集合。 + +[English Document](README.md) + +## 功能 + +### 语音节点 (JM-MiniMax-API/Speech) + +- **文本转语音**: 使用 MiniMax 的高级文本转语音 API 将文本转换为自然的语音 +- **声音克隆**: 从音频样本中克隆声音 +- **加载音频**: 加载和预览用于声音克隆的音频文件 + +## 安装 + +1. 克隆此仓库到你的 ComfyUI custom_nodes 文件夹: +```bash +cd ComfyUI/custom_nodes +git clone https://github.com/yourusername/ComfyUI-JM-MiniMax-API.git +``` + +2. 安装依赖: +```bash +pip install -r requirements.txt +``` + +3. 重启 ComfyUI + +## 使用方法 + +### LoadAudio 节点 + +此节点允许您加载和预览用于声音克隆的音频文件。 + +#### 输入参数: +- **audio_path**: 从输入目录中选择现有的音频文件 +- **上传按钮**: 点击上传新的音频文件（支持 .mp3、.wav、.m4a） + +#### 输出: +- **audio_file**: 所选音频文件的绝对路径 + +### VoiceCloning 节点 + +此节点使用 MiniMax 的声音克隆 API 从音频样本中克隆声音。 + +#### 输入参数: +- **api_key**: MiniMax API 密钥 +- **group_id**: MiniMax 组 ID +- **audio_file**: 要克隆的音频文件路径（连接自 LoadAudio 节点） +- **voice_id**: 克隆声音的 ID（至少 8 个字符，以字母开头，包含数字） +- **need_noise_reduction**: 是否应用噪声降低（true/false） +- **need_volume_normalization**: 是否规范化音量（true/false） +- **preview_text**: 可选的预览克隆声音的文本（最多 300 个字符） +- **model**: 用于预览的 TTS 模型 +- **accuracy**: 克隆准确度阈值（0.0-1.0） + +#### 输出: +- **voice_id**: 克隆声音的 ID（可以连接到 TextToSpeech 节点） + +### TextToSpeech 节点 + +此节点使用 MiniMax 的 API 将文本转换为语音。 + +#### 输入参数: +- **api_key**: MiniMax API 密钥 +- **group_id**: MiniMax 组 ID +- **text**: 要转换的文本（最多 5000 个字符） +- **model**: TTS 模型选择（speech-02-hd、speech-02-turbo 等） +- **voice_id**: 声音选择（有多种选项可用） +- **custom_voice_id**（可选）: 来自声音克隆的自定义声音 ID（如果提供，将覆盖 voice_id） +- **speed**: 语速（0.5 到 2.0） +- **volume**: 音量（0.1 到 10.0） +- **pitch**: 音调调整（-12 到 12） +- **emotion**: 情感选择（happy、sad 等） +- **subtitle_enable**: 是否生成字幕文件（true/false） +- **filename_prefix**: 输出文件名前缀 +- **language_boost**（可选）: 语言增强 + +#### 输出: +- **audio_path**: 生成的音频文件的绝对路径 +- **subtitle_path**: 生成的字幕文件的绝对路径（如果启用了字幕） + +#### 输出文件说明: +1. 音频文件 (audio_path): + - 格式: MP3 + - 文件名格式: prefix_YYYYMMDD-HHMMSS.mp3 + +2. 字幕文件 (subtitle_path): + - 格式: JSON + - 文件名格式: prefix_subtitle_YYYYMMDD-HHMMSS.json + - 包含精确到毫秒的句子级别时间戳 + +## 工作流示例 + +1. 使用 **LoadAudio** 节点上传或选择音频文件 +2. 将输出连接到 **VoiceCloning** 节点以克隆声音 +3. 将 voice_id 输出连接到 **TextToSpeech** 节点的 custom_voice_id 输入 +4. 在 TextToSpeech 节点中输入文本并配置其他参数 +5. 运行工作流以使用克隆的声音生成语音 + +## 许可证 + +MIT License \ No newline at end of file diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e17ca7b --- /dev/null +++ b/__init__.py @@ -0,0 +1,20 @@ +from .nodes.text_to_speech import TextToSpeech +from .nodes.voice_cloning import VoiceCloning +from .nodes.load_audio import JM_LoadAudio + +NODE_CLASS_MAPPINGS = { + "JM-MiniMax-API/text-to-speech": TextToSpeech, + "JM-MiniMax-API/voice-cloning": VoiceCloning, + "JM-MiniMax-API/load-audio": JM_LoadAudio +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "JM-MiniMax-API/text-to-speech": "MiniMax Text to Speech", + "JM-MiniMax-API/voice-cloning": "MiniMax Voice Cloning", + "JM-MiniMax-API/load-audio": "Load Audio" +} + +# Tell ComfyUI where to find web extensions +WEB_DIRECTORY = "./web" + +__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"] \ No newline at end of file diff --git a/nodes/__init__.py b/nodes/__init__.py new file mode 100644 index 0000000..7a116a2 --- /dev/null +++ b/nodes/__init__.py @@ -0,0 +1 @@ +# This file is required to make Python treat the directory as a package \ No newline at end of file diff --git a/nodes/load_audio.py b/nodes/load_audio.py new file mode 100644 index 0000000..5320098 --- /dev/null +++ b/nodes/load_audio.py @@ -0,0 +1,63 @@ +import os +import folder_paths +import shutil + +class JM_LoadAudio: + """ + Load Audio node for ComfyUI + Supports selecting audio files and provides upload capability + """ + + @classmethod + def INPUT_TYPES(cls): + files = [] + input_dir = folder_paths.get_input_directory() + + if os.path.exists(input_dir): + for f in os.listdir(input_dir): + if os.path.isfile(os.path.join(input_dir, f)) and f.split('.')[-1].lower() in ["mp3", "wav", "m4a"]: + files.append(f) + + return { + "required": { + "audio_path": (files,), + }, + "optional": { + "upload": ("JMAUDIOUPLOAD",), # Special widget for audio upload + } + } + + RETURN_TYPES = ("STRING",) + RETURN_NAMES = ("audio_file",) + FUNCTION = "load_audio" + CATEGORY = "JM-MiniMax-API/Speech" + + def load_audio(self, audio_path, upload=None): + # Get the full path of the audio file + audio_file_path = os.path.join(folder_paths.get_input_directory(), audio_path) + + if not os.path.exists(audio_file_path): + raise FileNotFoundError(f"Audio file not found: {audio_file_path}") + + return (audio_file_path,) + + @classmethod + def IS_CHANGED(cls, audio_path, upload=None): + if not audio_path: + return float("nan") + + filepath = folder_paths.get_annotated_filepath(audio_path) + if not os.path.exists(filepath): + return float("nan") + + return os.path.getmtime(filepath) + + @classmethod + def VALIDATE_INPUTS(cls, audio_path, upload=None): + if not audio_path: + return "Please select an audio file" + + if not folder_paths.exists_annotated_filepath(audio_path): + return f"Invalid audio file: {audio_path}" + + return True \ No newline at end of file diff --git a/nodes/text_to_speech.py b/nodes/text_to_speech.py new file mode 100644 index 0000000..bf22e4d --- /dev/null +++ b/nodes/text_to_speech.py @@ -0,0 +1,177 @@ +import os +import json +import time +import binascii +import requests +import folder_paths + +class TextToSpeech: + """ + MiniMax Text to Speech node for ComfyUI + """ + def __init__(self): + self.api_base = "https://api.minimaxi.chat/v1/t2a_v2" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "api_key": ("STRING", {"multiline": False}), + "group_id": ("STRING", {"multiline": False}), + "text": ("STRING", {"multiline": True}), + "model": (["speech-02-hd", "speech-02-turbo", "speech-01-hd", "speech-01-turbo"], {"default": "speech-02-hd"}), + "voice_id": (["Wise_Woman", "Friendly_Person", "Inspirational_girl", "Deep_Voice_Man", + "Calm_Woman", "Casual_Guy", "Lively_Girl", "Patient_Man", "Young_Knight", + "Determined_Man", "Lovely_Girl", "Decent_Boy", "Imposing_Manner", + "Elegant_Man", "Abbess", "Sweet_Girl_2", "Exuberant_Girl"], {"default": "Wise_Woman"}), + "speed": ("FLOAT", {"default": 1.0, "min": 0.5, "max": 2.0, "step": 0.1}), + "volume": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 10.0, "step": 0.1}), + "pitch": ("INT", {"default": 0, "min": -12, "max": 12, "step": 1}), + "emotion": (["happy", "sad", "angry", "fearful", "disgusted", "surprised", "neutral"], {"default": "neutral"}), + "subtitle_enable": ("BOOLEAN", {"default": False}), + "filename_prefix": ("STRING", {"default": "tts_output", "multiline": False}), + }, + "optional": { + "custom_voice_id": ("STRING", { + "multiline": False, + "default": "", + "placeholder": "Custom voice ID from voice cloning" + }), + "language_boost": (["auto", "Chinese", "Chinese,Yue", "English", "Arabic", "Russian", "Spanish", + "French", "Portuguese", "German", "Turkish", "Dutch", "Ukrainian", "Vietnamese", + "Indonesian", "Japanese", "Italian", "Korean", "Thai", "Polish", "Romanian", + "Greek", "Czech", "Finnish", "Hindi"], {"default": "auto"}), + } + } + + RETURN_TYPES = ("STRING", "STRING") + RETURN_NAMES = ("audio_path", "subtitle_path") + FUNCTION = "generate_speech" + CATEGORY = "JM-MiniMax-API/Speech" + + def generate_speech(self, api_key, group_id, text, model, voice_id, speed, volume, pitch, emotion, subtitle_enable, filename_prefix, custom_voice_id="", language_boost="auto"): + if not api_key or not group_id: + raise ValueError("API Key and Group ID must be provided") + + url = f"{self.api_base}?GroupId={group_id}" + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + "accept": "application/json, text/plain, */*" + } + + # Use custom_voice_id if provided, otherwise use voice_id + selected_voice_id = custom_voice_id if custom_voice_id else voice_id + print(f"Using voice_id: {selected_voice_id} {'(custom)' if custom_voice_id else '(predefined)'}") + + payload = { + "model": model, + "text": text, + "voice_setting": { + "voice_id": selected_voice_id, + "speed": speed, + "vol": volume, + "pitch": pitch, + "emotion": emotion + }, + "language_boost": language_boost, + "audio_setting": { + "sample_rate": 32000, + "bitrate": 128000, + "format": "mp3", + "channel": 1 + }, + "subtitle_enable": subtitle_enable, + "output_format": "hex" + } + + try: + print(f"Sending request to {url}") + print(f"Headers: {json.dumps(headers, indent=2)}") + print(f"Payload: {json.dumps(payload, indent=2)}") + + response = requests.post(url, headers=headers, json=payload) + print(f"Response status code: {response.status_code}") + print(f"Response headers: {json.dumps(dict(response.headers), indent=2)}") + + try: + resp_data = response.json() + print(f"Response data: {json.dumps(resp_data, indent=2)}") + except json.JSONDecodeError: + print(f"Raw response content: {response.content}") + raise RuntimeError("Failed to decode JSON response") + + # Check for API error response + base_resp = resp_data.get("base_resp", {}) + status_code = base_resp.get("status_code") + status_msg = base_resp.get("status_msg", "Unknown error") + + if status_code is not None and status_code != 0: + raise RuntimeError(f"API Error {status_code}: {status_msg}") + + data = resp_data.get("data", {}) + if not data: + print(f"Full response: {json.dumps(resp_data, indent=2)}") + raise RuntimeError("No data returned from API") + + # Convert hex to binary + audio_hex = data.get("audio", "") + if not audio_hex: + raise RuntimeError("No audio data returned") + + print(f"Received audio hex data length: {len(audio_hex)}") + audio_data = binascii.unhexlify(audio_hex) + print(f"Decoded audio data length: {len(audio_data)}") + + # Create output directory + output_dir = folder_paths.get_output_directory() + os.makedirs(output_dir, exist_ok=True) + + # Generate timestamp for filenames + timestamp = time.strftime("%Y%m%d-%H%M%S") + + # Clean filename prefix (remove any invalid characters) + clean_prefix = "".join(c for c in filename_prefix if c.isalnum() or c in ('-', '_')) + if not clean_prefix: + clean_prefix = "tts_output" + + # Save audio file + audio_filename = f"{clean_prefix}_{timestamp}.mp3" + audio_filepath = os.path.join(output_dir, audio_filename) + with open(audio_filepath, "wb") as f: + f.write(audio_data) + print(f"Saved audio file to: {audio_filepath}") + + # Save subtitle file if available + subtitle_filepath = "" + if subtitle_enable and data.get("subtitle_file"): + subtitle_filename = f"{clean_prefix}_subtitle_{timestamp}.json" + subtitle_filepath = os.path.join(output_dir, subtitle_filename) + subtitle_response = requests.get(data["subtitle_file"]) + subtitle_response.raise_for_status() + with open(subtitle_filepath, "wb") as f: + f.write(subtitle_response.content) + print(f"Saved subtitle file to: {subtitle_filepath}") + + return ( + os.path.abspath(audio_filepath), + os.path.abspath(subtitle_filepath) if subtitle_filepath else "" + ) + + except requests.exceptions.RequestException as e: + print(f"Request error: {str(e)}") + if hasattr(e, 'response') and e.response is not None: + print(f"Error response status code: {e.response.status_code}") + print(f"Error response headers: {json.dumps(dict(e.response.headers), indent=2)}") + try: + error_data = e.response.json() + print(f"Error response data: {json.dumps(error_data, indent=2)}") + except: + print(f"Error response content: {e.response.content}") + raise RuntimeError(f"Error calling MiniMax API: {str(e)}") + except (ValueError, binascii.Error) as e: + print(f"Data processing error: {str(e)}") + raise RuntimeError(f"Error processing API response: {str(e)}") + except Exception as e: + print(f"Unexpected error: {str(e)}") + raise RuntimeError(f"Unexpected error: {str(e)}") \ No newline at end of file diff --git a/nodes/voice_cloning.py b/nodes/voice_cloning.py new file mode 100644 index 0000000..a6f7e04 --- /dev/null +++ b/nodes/voice_cloning.py @@ -0,0 +1,151 @@ +import os +import json +import requests +import folder_paths + +class VoiceCloning: + """ + MiniMax Voice Cloning node for ComfyUI + """ + def __init__(self): + self.base_url = "https://api.minimaxi.chat/v1" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "api_key": ("STRING", {"multiline": False}), + "group_id": ("STRING", {"multiline": False}), + "audio_file": ("STRING", {"multiline": False}), # Path to audio file + "voice_id": ("STRING", { + "multiline": False, + "default": "MiniMax001", + "placeholder": "At least 8 chars, start with letter, include numbers" + }), + "need_noise_reduction": ("BOOLEAN", {"default": False}), + "need_volume_normalization": ("BOOLEAN", {"default": False}), + "preview_text": ("STRING", { + "multiline": True, + "default": "", + "placeholder": "Optional: Enter text to preview the cloned voice (max 300 chars)" + }), + "model": (["speech-02-hd", "speech-02-turbo", "speech-01-hd", "speech-01-turbo"], { + "default": "speech-02-hd" + }), + "accuracy": ("FLOAT", { + "default": 0.7, + "min": 0.0, + "max": 1.0, + "step": 0.1 + }), + } + } + + RETURN_TYPES = ("STRING",) # Returns voice_id + RETURN_NAMES = ("voice_id",) + FUNCTION = "clone_voice" + CATEGORY = "JM-MiniMax-API/Speech" + + def clone_voice(self, api_key, group_id, audio_file, voice_id, need_noise_reduction, need_volume_normalization, + preview_text, model, accuracy): + if not api_key: + raise ValueError("API Key must be provided") + + if not group_id: + raise ValueError("Group ID must be provided") + + if not os.path.exists(audio_file): + raise ValueError(f"Audio file not found: {audio_file}") + + if len(voice_id) < 8 or not voice_id[0].isalpha() or not any(c.isdigit() for c in voice_id): + raise ValueError("Voice ID must be at least 8 characters, start with a letter, and include numbers") + + try: + # Step 1: Upload audio file + print(f"Uploading audio file: {audio_file}") + upload_url = f"{self.base_url}/files/upload?GroupId={group_id}" + + headers = { + 'authority': 'api.minimaxi.chat', + 'Authorization': f'Bearer {api_key}' + } + + data = { + 'purpose': 'voice_clone' + } + + files = { + 'file': open(audio_file, 'rb') + } + + upload_response = requests.post( + upload_url, + headers=headers, + data=data, + files=files + ) + upload_response.raise_for_status() + upload_data = upload_response.json() + + print(f"Upload response: {json.dumps(upload_data, indent=2)}") + + # Get file_id from the correct path in response + file_id = upload_data.get("file", {}).get("file_id") + if not file_id: + raise RuntimeError("No file_id returned from upload") + + # Step 2: Clone voice + print(f"Cloning voice with file_id: {file_id}") + clone_url = f"{self.base_url}/voice_clone?GroupId={group_id}" + + clone_payload = { + "file_id": file_id, + "voice_id": voice_id, + "need_noise_reduction": need_noise_reduction, + "need_volume_normalization": need_volume_normalization, + "accuracy": accuracy + } + + # Add optional preview text and model if preview text is provided + if preview_text.strip(): + if len(preview_text) > 300: + print("Warning: Preview text exceeds 300 characters, it will be truncated") + preview_text = preview_text[:300] + clone_payload["text"] = preview_text + clone_payload["model"] = model + + clone_headers = { + 'authorization': f'Bearer {api_key}', + 'content-type': 'application/json' + } + + clone_response = requests.post( + clone_url, + headers=clone_headers, + data=json.dumps(clone_payload) + ) + clone_response.raise_for_status() + clone_data = clone_response.json() + + print(f"Clone response: {json.dumps(clone_data, indent=2)}") + + if clone_data.get("base_resp", {}).get("status_code") != 0: + raise RuntimeError(f"Voice cloning failed: {clone_data.get('base_resp', {}).get('status_msg', 'Unknown error')}") + + if clone_data.get("input_sensitive", False): + print(f"Warning: Input audio triggered sensitivity check (type: {clone_data.get('input_sensitive_type', 'unknown')})") + + return (voice_id,) + + except requests.exceptions.RequestException as e: + print(f"API request failed: {str(e)}") + if hasattr(e, 'response') and e.response is not None: + try: + error_data = e.response.json() + print(f"Error response: {json.dumps(error_data, indent=2)}") + except: + print(f"Error response content: {e.response.content}") + raise RuntimeError(f"API request failed: {str(e)}") + except Exception as e: + print(f"Unexpected error: {str(e)}") + raise RuntimeError(f"Voice cloning failed: {str(e)}") \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..659c37c --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +requests>=2.31.0 \ No newline at end of file diff --git a/web/index.js b/web/index.js new file mode 100644 index 0000000..76e9340 --- /dev/null +++ b/web/index.js @@ -0,0 +1 @@ +import "./jm_audio_upload.js"; \ No newline at end of file diff --git a/web/jm_audio_upload.js b/web/jm_audio_upload.js new file mode 100644 index 0000000..35de8fd --- /dev/null +++ b/web/jm_audio_upload.js @@ -0,0 +1,217 @@ +import { app } from "../../scripts/app.js"; +import { api } from "../../scripts/api.js"; +import { ComfyWidgets } from "../../scripts/widgets.js"; + +// Function to preview the audio file +function previewAudio(node, file) { + // Remove existing preview widgets + while (node.widgets.length > 2) { + node.widgets.pop(); + } + + // Remove existing audio element if any + try { + var el = document.getElementById("JM_AudioPreview"); + if (el) el.remove(); + } catch (error) { + console.log(error); + } + + // Create audio preview element + var element = document.createElement("div"); + element.id = "JM_AudioPreview"; + const previewNode = node; + + // Add a DOM widget for the audio preview + var previewWidget = node.addDOMWidget("audiopreview", "preview", element, { + serialize: false, + hideOnZoom: false, + getValue() { + return element.value; + }, + setValue(v) { + element.value = v; + }, + }); + + // Compute size for the widget + previewWidget.computeSize = function (width) { + if (this.aspectRatio && !this.parentEl.hidden) { + let height = (previewNode.size[0] - 20) / this.aspectRatio + 10; + if (!(height > 0)) { + height = 0; + } + this.computedHeight = height + 10; + return [width, height]; + } + return [width, -4]; // No loaded src, widget should not display + }; + + // Set up widget properties + previewWidget.value = { hidden: false, paused: false, params: {} }; + previewWidget.parentEl = document.createElement("div"); + previewWidget.parentEl.className = "audio_preview"; + previewWidget.parentEl.style["width"] = "100%"; + element.appendChild(previewWidget.parentEl); + + // Create audio element + previewWidget.audioEl = document.createElement("audio"); + previewWidget.audioEl.controls = true; + previewWidget.audioEl.loop = false; + previewWidget.audioEl.muted = false; + previewWidget.audioEl.style["width"] = "100%"; + + // Listen for metadata loading + previewWidget.audioEl.addEventListener("loadedmetadata", () => { + previewWidget.aspectRatio = 4; // Default aspect ratio for audio controls + }); + + // Listen for errors + previewWidget.audioEl.addEventListener("error", () => { + previewWidget.parentEl.hidden = true; + console.error("Error loading audio preview"); + }); + + // Set parameters for the audio view + let params = { + filename: file, + type: "input", + }; + + // Set visibility and autoplay + previewWidget.parentEl.hidden = previewWidget.value.hidden; + previewWidget.audioEl.autoplay = !previewWidget.value.paused && !previewWidget.value.hidden; + + // Set the audio source + previewWidget.audioEl.src = api.apiURL("/view?" + new URLSearchParams(params)); + previewWidget.audioEl.hidden = false; + previewWidget.parentEl.appendChild(previewWidget.audioEl); +} + +// Function to handle audio upload +function audioUpload(node, inputName, inputData, app) { + // Find the audio path widget + const audioWidget = node.widgets.find((w) => w.name === "audio_path"); + let uploadWidget; + + // Get default value + var default_value = audioWidget.value; + + // Define getter/setter for audio_path + Object.defineProperty(audioWidget, "audio_path", { + set: function (value) { + this._real_value = value; + }, + get: function () { + let value = ""; + if (this._real_value) { + value = this._real_value; + } else { + return default_value; + } + + if (value.filename) { + let real_value = value; + value = ""; + if (real_value.subfolder) { + value = real_value.subfolder + "/"; + } + value += real_value.filename; + if (real_value.type && real_value.type !== "input") + value += ` [${real_value.type}]`; + } + return value; + }, + }); + + // Function to upload a file + async function uploadFile(file, updateNode, pasted = false) { + try { + // Wrap file in formdata so it includes filename + const body = new FormData(); + body.append("image", file); + if (pasted) body.append("subfolder", "pasted"); + + // Send the upload request + const resp = await api.fetchApi("/upload/image", { + method: "POST", + body, + }); + + if (resp.status === 200) { + const data = await resp.json(); + // Add the file to the dropdown list and update the widget value + let path = data.name; + if (data.subfolder) path = data.subfolder + "/" + path; + + if (!audioWidget.options.values.includes(path)) { + audioWidget.options.values.push(path); + } + + if (updateNode) { + audioWidget.value = path; + previewAudio(node, path); + } + } else { + alert(resp.status + " - " + resp.statusText); + } + } catch (error) { + alert(error); + } + } + + // Create file input element + const fileInput = document.createElement("input"); + Object.assign(fileInput, { + type: "file", + accept: "audio/mpeg, audio/wav, audio/x-m4a", + style: "display: none", + onchange: async () => { + if (fileInput.files.length) { + await uploadFile(fileInput.files[0], true); + } + }, + }); + document.body.append(fileInput); + + // Create the button widget for selecting files + uploadWidget = node.addWidget( + "button", + "Upload Audio", + "Audio", + () => { + fileInput.click(); + } + ); + + uploadWidget.serialize = false; + + // Preview the current audio file + previewAudio(node, audioWidget.value); + + // Override callback to update preview when selection changes + const cb = node.callback; + audioWidget.callback = function () { + previewAudio(node, audioWidget.value); + if (cb) { + return cb.apply(this, arguments); + } + }; + + return { widget: uploadWidget }; +} + +// Register our custom widget +ComfyWidgets.JMAUDIOUPLOAD = audioUpload; + +// Register extension +app.registerExtension({ + name: "JM.MiniMax.AudioUpload", + async beforeRegisterNodeDef(nodeType, nodeData, app) { + if (nodeData?.name == "JM_LoadAudio") { + // This will be processed by the ComfyWidgets + nodeData.input.optional = nodeData.input.optional || {}; + nodeData.input.optional.upload = ["JMAUDIOUPLOAD"]; + } + }, +}); \ No newline at end of file