diff --git a/.github/workflows/publish_action.yml b/.github/workflows/publish_action.yml new file mode 100644 index 0000000..ef9d8d9 --- /dev/null +++ b/.github/workflows/publish_action.yml @@ -0,0 +1,22 @@ +name: Publish to Comfy registry +on: + workflow_dispatch: + push: + branches: + - master + - main + paths: + - "pyproject.toml" + +jobs: + publish-node: + name: Publish Custom Node to registry + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Publish Custom Node + uses: Comfy-Org/publish-node-action@main + with: + ## Add your own personal access token to your Github Repository secrets and reference it here. + personal_access_token: ${{ secrets.REGISTRY_ACCESS_TOKEN }} diff --git a/AddSubtitlesToVideo.py b/AddSubtitlesToVideo.py new file mode 100644 index 0000000..d461e9f --- /dev/null +++ b/AddSubtitlesToVideo.py @@ -0,0 +1,152 @@ +import torch +import numpy as np +from PIL import Image, ImageDraw, ImageFont +import ast + + +class AddSubtitlesToTensor: + """ + ComfyUI 节点，用于将字幕添加到视频帧的张量中。 + """ + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "images": ("IMAGE",), + "subtitles": ("STRING", ), + "language": (["类中文字符", "English_word"], {"default": "类中文字符"}), + "max_words_per_line": ( + "INT", {"default": 12, "min": 1, "max": 30}), + "font_path": ( + "STRING", {"default": "simhei.ttf"}), + "font_size": ("INT", {"default": 30, "min": 1, "max": 200}), + "font_color": ("STRING", {"default": "black"}), + "subtitle_background_color": ( + "STRING", {"default": "#B8860B", "tooltip": "None: Do not add background color"}), + "y_offset": ( + "INT", {"default": 0, "min": -1000, "max": 1000, "step": 5}), + "x_offset": ( + "INT", {"default": 0, "min": -1000, "max": 1000, "step": 5}), + "width": ("INT", {"default": 1024}), + "height": ("INT", {"default": 1024}), + "fps": ("FLOAT", {"default": 20, "min": 1, "max": 240}), + } + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES = ("image",) + + FUNCTION = "add_subtitles" + + CATEGORY = "🎤MW/MW-Audio-Tools" + + def add_subtitles(self, + images, + subtitles, + language, + font_path, + font_size, + font_color, + y_offset, + x_offset, + width, + height, + max_words_per_line, + subtitle_background_color, + fps, + ): + + if subtitle_background_color == "None": + subtitle_background_color = None + + try: + subtitles = ast.literal_eval(subtitles) + if not isinstance(subtitles, list): + raise ValueError("Subtitles must be a list of dictionaries.") + for sub in subtitles: + if not isinstance(sub, dict): + raise ValueError("Each subtitle must be a dictionary.") + except Exception as e: + print(f"Error parsing subtitles: {e}") + + if not isinstance(images, torch.Tensor): + print("Images must be a torch tensor") + return (images,) + + print(f"images.shape:{images.shape}") + if len(images.shape) == 3: + images = images.unsqueeze(0) + images = torch.clamp(images, 0.0, 1.0) + font = ImageFont.truetype(font_path, font_size) + + # x_offset = - x_offset + y_offset = - y_offset + + num_frames = images.shape[0] + for i in range(num_frames): + frame_time = i / fps + + relevant_subtitles = [ + sub + for sub in subtitles + if sub["timestamp"][0] <= frame_time <= sub["timestamp"][1] + ] + + if relevant_subtitles: + frame = Image.fromarray((images[i].numpy() * 255).astype(np.uint8)) + draw = ImageDraw.Draw(frame) + + # 重置 y_offset + current_y_offset = y_offset + + for sub in relevant_subtitles: + text = sub["text"] + wrapped_text = self.wrap_text(text, language, max_words_per_line) + + bbox = draw.multiline_textbbox((0, 0), wrapped_text, font=font) + text_width = bbox[2] - bbox[0] + text_height = bbox[3] - bbox[1] + + # 计算居中后的坐标 + center_x = width / 2 + x_offset + center_y = height / 2 + current_y_offset + + # 计算左上角坐标 + x = center_x - text_width / 2 + y = center_y - text_height / 2 + + if subtitle_background_color: + bbox = draw.multiline_textbbox((x, y), wrapped_text, font=font) + bg_x0 = bbox[0] + bg_y0 = bbox[1] + bg_x1 = bbox[2] + bg_y1 = bbox[3] + draw.rectangle((bg_x0, bg_y0, bg_x1, bg_y1), fill=subtitle_background_color) + + draw.multiline_text((x, y), wrapped_text, font=font, fill=font_color) + + # 更新 y 偏移量，以便下一条字幕显示在下方 (如果需要显示多条字幕) + current_y_offset += text_height + 10 + + images[i] = torch.from_numpy(np.array(frame).astype(np.float32) / 255) + + return (images,) + def wrap_text(self, text, language, max_words_per_line): + words = text.split() + wrapped_lines = [] + current_line = [] + if language == "类中文字符": + for i in range(0, len(text), max_words_per_line): + wrapped_lines.append(text[i:i + max_words_per_line].strip()) + else: + for word in words: + if len(current_line) < max_words_per_line: + current_line.append(word) + else: + wrapped_lines.append(" ".join(current_line)) + current_line = [word] + + if current_line: + wrapped_lines.append(" ".join(current_line)) + + return "\n".join(wrapped_lines) diff --git a/MWAudioRecorderAT.py b/MWAudioRecorderAT.py new file mode 100644 index 0000000..94c4e4e --- /dev/null +++ b/MWAudioRecorderAT.py @@ -0,0 +1,130 @@ +import numpy as np +import torch +import time +import librosa +import sounddevice as sd +from scipy import ndimage +from comfy.utils import ProgressBar + +class AudioRecorderAT: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + # 触发控制 + "trigger": ("BOOLEAN", {"default": False}), + # 录音时长 + "record_sec": ("INT", { + "default": 5, + "min": 1, + "step": 1 # 整数秒递增 + }), + "sample_rate": (["16000", "44100", "48000"], { # 限定标准采样率 + "default": "48000" + }), + "n_fft": ("INT", { # 限定为2的幂次方 + "default": 2048, + "min": 512, + "max": 4096, + "step": 512 # 只能选择512,1024,1536,2048,...4096 + }), + "sensitivity": ("FLOAT", { # 灵敏度精确控制 + "default": 1.2, + "min": 0.1, + "max": 3.0, + "step": 0.1 # 0.1步进 + }), + "smooth": ("INT", { # 确保为奇数 + "default": 5, + "min": 1, + "max": 7, + "step": 2 # 生成1,3,5,7 + }), + "seed": ("INT", {"default": 0, "min": 0, "max": 0xFFFFFFFFFFFFFFFF}), + } + } + + RETURN_TYPES = ("AUDIO",) + RETURN_NAMES = ("audio",) + FUNCTION = "record_and_clean" + CATEGORY = "🎤MW/MW-Audio-Tools" + + def _stft(self, y, n_fft): + hop = n_fft // 4 + return librosa.stft(y, n_fft=n_fft, hop_length=hop, win_length=n_fft) + + def _istft(self, spec, n_fft): + hop = n_fft // 4 + return librosa.istft(spec, hop_length=hop, win_length=n_fft) + + def _calc_noise_profile(self, noise_clip, n_fft): + noise_spec = self._stft(noise_clip, n_fft) + return { + 'mean': np.mean(np.abs(noise_spec), axis=1, keepdims=True), + 'std': np.std(np.abs(noise_spec), axis=1, keepdims=True) + } + + def _spectral_gate(self, spec, noise_profile, sensitivity): + threshold = noise_profile['mean'] + sensitivity * noise_profile['std'] + return np.where(np.abs(spec) > threshold, spec, 0) + + def _smooth_mask(self, mask, kernel_size): + smoothed = ndimage.uniform_filter(mask, size=(kernel_size, kernel_size)) + return np.clip(smoothed * 1.2, 0, 1) # 增强边缘保留 + + def record_and_clean(self, trigger, record_sec, n_fft, sensitivity, smooth, sample_rate, seed): + if not trigger: + return (None,) + + sr = int(sample_rate) + final_audio = None + + try: + noise_clip = None + # 主录音 + # print(f"开始主录音 {record_sec}秒...") + main_rec = sd.rec(int(record_sec * sr), samplerate=sr, channels=1, dtype='float32') + pb = ProgressBar(record_sec) + for _ in range(record_sec * 2): + time.sleep(0.5) + pb.update(0.5) + sd.wait() + audio = main_rec.flatten() + + # 自动噪声检测 + if noise_clip is None: + # print("自动检测静默段作为噪声参考...") + energy = librosa.feature.rms(y=audio, frame_length=n_fft, hop_length=n_fft//4) + min_idx = np.argmin(energy) + start = min_idx * (n_fft//4) + noise_clip = audio[start:start + n_fft*2] + + # 降噪处理 + # print("进行频谱降噪...") + noise_profile = self._calc_noise_profile(noise_clip, n_fft) + spec = self._stft(audio, n_fft) + + # 多步骤处理 + mask = np.ones_like(spec) # 初始掩膜 + for _ in range(2): # 双重处理循环 + cleaned_spec = self._spectral_gate(spec, noise_profile, sensitivity) + mask = np.where(np.abs(cleaned_spec) > 0, 1, 0) + mask = self._smooth_mask(mask, smooth//2+1) + spec = spec * mask + + # 相位恢复重建 + processed = self._istft(spec * mask, n_fft) + + # 动态增益归一化 + peak = np.max(np.abs(processed)) + processed = processed * (0.99 / peak) if peak > 0 else processed + + # 格式转换 + waveform = torch.from_numpy(processed).float().unsqueeze(0).unsqueeze(0) + final_audio = {"waveform": waveform, "sample_rate": sr} + + except Exception as e: + print(f"Recording/processing failed: {str(e)}") + raise + + return (final_audio,) diff --git a/README-CN.md b/README-CN.md new file mode 100644 index 0000000..ccfee32 --- /dev/null +++ b/README-CN.md @@ -0,0 +1,54 @@ +[中文](README-CN.md)|[English](README.md) + +# 音频处理等相关的 ComfyUI 节点 + +音频是连接文本, 视频, 图像的桥梁, 没有音频或文字的视频是无味的. 这个项目目前包括以下几个主要节点: +- 视频自动添加字幕 +- 音频任意时间刻度裁剪 +- 音频音量, 速度, 音高, 回音处理等 +- 去除音频中无声部分 +- 录音 + +示例: + +1, 视频添加字幕: + +![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/2025-03-25_14-00-28.png) + +2, 结合 [ComfyUI_EraX-WoW-Turbo](https://github.com/billwuhao/ComfyUI_EraX-WoW-Turbo) 语音自动识别, 然后给视频添加字幕: + +![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/2025-03-25_13-33-54.png) + +3, 结合 [ComfyUI_EraX-WoW-Turbo](https://github.com/billwuhao/ComfyUI_EraX-WoW-Turbo), [ComfyUI_gemmax](https://github.com/billwuhao/ComfyUI_gemmax), [ComfyUI_SparkTTS](https://github.com/billwuhao/ComfyUI_SparkTTS), [ComfyUI-LatentSyncWrapper](https://github.com/ShmuelRonen/ComfyUI-LatentSyncWrapper) 自动识别语音, 自动翻译, 自动克隆声音, 自动对口型, 视频自动添加字幕(详解示例工作流 [workflow-examples](./workflow-examples)): + +![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/20250326001631.png) + +4, 任意时间刻度裁剪音频: + +![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/2025-03-25_13-14-52.png) + +5, 音频音量, 速度, 音高, 回音处理等: + +![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/2025-03-25_13-02-40.png) + +6, 去除音频中无声部分和录音: + +![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/2025-03-25_13-20-30.png) + +## 📣 更新 + +[2025-03-26]⚒️: 发布版本 v1.0.0. + +## 安装 + +安装 sox 并添加到系统 path. + +``` +cd ComfyUI/custom_nodes +git clone https://github.com/billwuhao/ComfyUI_AudioTools.git +cd ComfyUI_AudioTools +pip install -r requirements.txt + +# python_embeded +./python_embeded/python.exe -m pip install -r requirements.txt +``` diff --git a/README.md b/README.md index 914f423..32eb4ce 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,54 @@ -# ComfyUI_AudioTools -A ComfyUI node containing multiple audio processing tools. +[中文](README-CN.md)|[English](README.md) + +# Audio Processing Related ComfyUI Nodes + +Audio is the bridge connecting text, video, and images. Videos without audio or text are bland. This project currently includes the following main nodes: +- Automatically add subtitles to videos +- Arbitrary time scale audio cropping +- Audio volume, speed, pitch, echo processing, etc. +- Remove silent parts from audio +- Recording + +Examples: + +1, Add subtitles to video: + +![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/2025-03-25_14-00-28.png) + +2, Combine [ComfyUI_EraX-WoW-Turbo](https://github.com/billwuhao/ComfyUI_EraX-WoW-Turbo) for automatic speech recognition, and then add subtitles to the video: + +![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/2025-03-25_13-33-54.png) + +3, Combine [ComfyUI_EraX-WoW-Turbo](https://github.com/billwuhao/ComfyUI_EraX-WoW-Turbo), [ComfyUI_gemmax](https://github.com/billwuhao/ComfyUI_gemmax), [ComfyUI_SparkTTS](https://github.com/billwuhao/ComfyUI_SparkTTS), [ComfyUI-LatentSyncWrapper](https://github.com/ShmuelRonen/ComfyUI-LatentSyncWrapper) for automatic speech recognition, automatic translation, automatic voice cloning, automatic lip sync, automatic subtitle addition to video (detailed example workflow [workflow-examples](./workflow-examples)): + +![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/20250326001631.png) + +4, Arbitrary time scale cropping of audio: + +![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/2025-03-25_13-14-52.png) + +5, Audio volume, speed, pitch, echo processing, etc.: + +![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/2025-03-25_13-02-40.png) + +6, Remove silent parts from audio and recording: + +![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/2025-03-25_13-20-30.png) + +## 📣 Updates + +[2025-03-26]⚒️: Released version v1.0.0. + +## Installation + +Install sox and add it to the system path. + +``` +cd ComfyUI/custom_nodes +git clone https://github.com/billwuhao/ComfyUI_AudioTools.git +cd ComfyUI_AudioTools +pip install -r requirements.txt + +# python_embeded +./python_embeded/python.exe -m pip install -r requirements.txt +``` diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..5a87a89 --- /dev/null +++ b/__init__.py @@ -0,0 +1,3 @@ +from .audiotoolsnode import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS + +__all__ = ['NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS'] diff --git a/audiotoolsnode.py b/audiotoolsnode.py new file mode 100644 index 0000000..bee4493 --- /dev/null +++ b/audiotoolsnode.py @@ -0,0 +1,331 @@ +import sox +import tempfile +import os +from glob import glob +import torchaudio +import librosa +import torch +# import sounddevice as sd +# from scipy import ndimage + + +class AdjustAudio: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "audio":("AUDIO",), + "volume_factor": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 5.0, "step": 0.1}), + "pitch_shift": ("INT", {"default": 0, "min": -1200, "max": 1200, "step": 100}), + "speed_factor": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 5.0, "step": 0.1}), + "normalize": ("BOOLEAN", {"default": False}), + "add_echo": ("BOOLEAN", {"default": False}), + "gain_in": ("FLOAT", {"default": 0.8, "min": 0.1, "max": 1.0, "step": 0.1}), + "gain_out": ("FLOAT", {"default": 0.88, "min": 0.1, "max": 1.0, "step": 0.1}), + "delays": ("INT", {"default": 60, "min": 10, "max": 500, "step": 10}), + "decays": ("FLOAT", {"default": 0.4, "min": 0.1, "max": 1.0, "step": 0.1}), + }, + } + + RETURN_TYPES = ("AUDIO",) + RETURN_NAMES = ("audio",) + FUNCTION = "adjust_audio_properties" + CATEGORY = "🎤MW/MW-Audio-Tools" + + def adjust_audio_properties(self, audio, volume_factor, pitch_shift, speed_factor, normalize, add_echo, gain_in, gain_out, delays, decays): + """ + 调节音频的音量、音高和语速，可选规范化音量和添加回声效果。 + + 参数说明: + - audio (AUDIO): 输入音频。 + - volume_factor (float): 音量倍数。 + - pitch_shift (int): 音高偏移。 + - speed_factor (float): 语速倍数。 + - normalize (bool): 是否规范化音量到 -0.1dB。 + - add_echo (bool): 是否添加回声效果。 + - gain_in (float): 输入增益。 + - gain_out (float): 输出增益。 + - delays ([int]): 回声延迟时间。 + - decays ([float]): 回声衰减时间。 + + 返回: + - audio (AUDIO): 处理后的音频。 + """ + temp_files = [] + waveform = audio["waveform"].squeeze(0) + sample_rate = audio["sample_rate"] + + # 创建临时文件 + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_file: + audio_file_path = temp_file.name + + # 使用 torchaudio.save() 保存临时文件 + torchaudio.save(audio_file_path, waveform, sample_rate, format="wav", + bits_per_sample=16, encoding="PCM_S") + temp_files.append(audio_file_path) + + tfm = sox.Transformer() + + # 音量调整 + tfm.vol(volume_factor, gain_type='amplitude') + # 音高调整 + tfm.pitch(pitch_shift / 100.0) + # 语速调整 + tfm.tempo(speed_factor, audio_type='s') + # 可选规范化音量 + if normalize: + tfm.norm(-0.1) + # 可选添加回声效果 + if add_echo: + tfm.echo(gain_in=gain_in, gain_out=gain_out, delays=[delays], decays=[decays]) + # 使用临时文件保存中间结果 + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_file: + temp_output = temp_file.name + tfm.build(audio_file_path, temp_output) + temp_files.append(temp_output) + + audio, sr = torchaudio.load(temp_output) + + audio_tensor = {"waveform": audio.unsqueeze(0), "sample_rate": sr} + + # 清理所有临时文件 + for temp_file in temp_files: + if os.path.exists(temp_file): + os.remove(temp_file) + + + return (audio_tensor,) + + +class TrimAudio: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "audio":("AUDIO",), + "start_time": ("FLOAT", {"default": 0.00, "min": 0.00, "step": 0.01}), + "end_time": ("FLOAT", {"default": 0.00, "min": 0.00, "step": 0.01}), + }, + } + + RETURN_TYPES = ("AUDIO",) + RETURN_NAMES = ("audio",) + FUNCTION = "trim_audio" + CATEGORY = "🎤MW/MW-Audio-Tools" + + def trim_audio(self, audio, start_time, end_time): + """ + 截取音频的指定时间段。 + + 参数说明: + - audio (dict): 输入音频，包含 "waveform" 和 "sample_rate"。 + - start_time (float): 开始时间 (秒)。 + - end_time (float): 结束时间 (秒)。 + + 返回: + - tuple: (audio_tensor,) 处理后的音频字典，包含 "waveform" 和 "sample_rate"。 + """ + waveform = audio["waveform"].squeeze(0) + sample_rate = audio["sample_rate"] + + # 记录所有临时文件路径 + temp_files = [] + + # 创建并保存输入音频的临时文件 + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_input_file: + audio_file_path = temp_input_file.name + temp_files.append(audio_file_path) + + # 使用 torchaudio.save() 保存临时文件 + torchaudio.save(audio_file_path, waveform, sample_rate, format="wav", + bits_per_sample=16, encoding="PCM_S") + + tfm = sox.Transformer() + + # 设置截取时间 + tfm.trim(start_time, end_time) + + # 使用临时文件保存截取结果 + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_output_file: + temp_output = temp_output_file.name + temp_files.append(temp_output) + tfm.build(audio_file_path, temp_output) + + # 加载处理后的音频 + audio, sr = torchaudio.load(temp_output) + audio_tensor = {"waveform": audio.unsqueeze(0), "sample_rate": sr} + + # 清理所有临时文件 + for temp_file in temp_files: + if os.path.exists(temp_file): + os.remove(temp_file) + + return (audio_tensor,) + + +class RemoveSilence: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "audio": ("AUDIO",), + "min_silence_duration": ("FLOAT", {"default": 0.3, "min": 0.1, "max": 3.0, "step": 0.1}), + "silence_threshold": ("FLOAT", {"default": 0.5, "min": 0.1, "max": 100, "step": 0.1}), + "buffer_around_silence": ("BOOLEAN", {"default": False}), + }, + } + + RETURN_TYPES = ("AUDIO",) + RETURN_NAMES = ("audio",) + FUNCTION = "remove_silence" + CATEGORY = "🎤MW/MW-Audio-Tools" + + def remove_silence(self, audio, min_silence_duration, silence_threshold, buffer_around_silence=False): + temp_files = [] + waveform = audio["waveform"].squeeze(0) + sample_rate = audio["sample_rate"] + + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_input_file: + audio_file_path = temp_input_file.name + temp_files.append(audio_file_path) + torchaudio.save(audio_file_path, waveform, sample_rate, format="wav", + bits_per_sample=16, encoding="PCM_S") + + y, sr = librosa.load(audio_file_path, sr=sample_rate) + normalized_y = librosa.util.normalize(y) + torchaudio.save(audio_file_path, torch.from_numpy(normalized_y).unsqueeze(0), sample_rate, format="wav", + bits_per_sample=16, encoding="PCM_S") + + tfm_silence = sox.Transformer() + tfm_silence.silence(min_silence_duration=min_silence_duration, + silence_threshold=silence_threshold, + buffer_around_silence=buffer_around_silence) + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_base: + temp_output = temp_base.name + temp_files.append(temp_output) + tfm_silence.build(audio_file_path, temp_output, sample_rate_in=sample_rate) + + processed_audio, sr = torchaudio.load(temp_output) + audio_tensor = {"waveform": processed_audio.unsqueeze(0), "sample_rate": sr} + + for temp_file in temp_files: + if os.path.exists(temp_file): + os.remove(temp_file) + + return (audio_tensor,) + + +# class AudioDenoising: +# @classmethod +# def INPUT_TYPES(cls): +# return { +# "required": { +# # 触发控制 +# "audio": ("AUDIO",), +# "n_fft": ("INT", { # 限定为2的幂次方 +# "default": 2048, +# "min": 512, +# "max": 4096, +# "step": 512 # 只能选择512,1024,1536,2048,...4096 +# }), +# "sensitivity": ("FLOAT", { # 灵敏度精确控制 +# "default": 1.2, +# "min": 0.5, +# "max": 3.0, +# "step": 0.1 # 0.1步进 +# }), +# "smooth": ("INT", { # 确保为奇数 +# "default": 5, +# "min": 1, +# "max": 11, +# "step": 2 # 生成1,3,5,7,9,11 +# }), +# "seed": ("INT", {"default": 0, "min": 0, "max": 0xFFFFFFFFFFFFFFFF}), +# } +# } + +# RETURN_TYPES = ("AUDIO",) +# RETURN_NAMES = ("audio",) +# FUNCTION = "clean" +# CATEGORY = "🎤MW/MW-Audio-Tools" + +# def _stft(self, y, n_fft): +# hop = n_fft // 4 +# return librosa.stft(y, n_fft=n_fft, hop_length=hop, win_length=n_fft) + +# def _istft(self, spec, n_fft): +# hop = n_fft // 4 +# return librosa.istft(spec, hop_length=hop, win_length=n_fft) + +# def _calc_noise_profile(self, noise_clip, n_fft): +# noise_spec = self._stft(noise_clip, n_fft) +# return { +# 'mean': np.mean(np.abs(noise_spec), axis=1, keepdims=True), +# 'std': np.std(np.abs(noise_spec), axis=1, keepdims=True) +# } + +# def _spectral_gate(self, spec, noise_profile, sensitivity): +# threshold = noise_profile['mean'] + sensitivity * noise_profile['std'] +# return np.where(np.abs(spec) > threshold, spec, 0) + +# def _smooth_mask(self, mask, kernel_size): +# smoothed = ndimage.uniform_filter(mask, size=(kernel_size, kernel_size)) +# return np.clip(smoothed * 1.2, 0, 1) # 增强边缘保留 + +# def clean(self, audio, n_fft, sensitivity, smooth, seed): +# noise_clip = None + +# waveform = audio["waveform"].squeeze().numpy().astype(np.float32) +# sample_rate = audio["sample_rate"] + +# energy = librosa.feature.rms(y=waveform, frame_length=n_fft, hop_length=n_fft//4) +# min_idx = np.argmin(energy) +# start = min_idx * (n_fft//4) +# noise_clip = waveform[start:start + n_fft*2] + +# # 降噪处理 +# noise_profile = self._calc_noise_profile(noise_clip, n_fft) +# spec = self._stft(waveform, n_fft) + +# # 多步骤处理 +# mask = np.ones_like(spec) # 初始掩膜 +# for _ in range(2): # 双重处理循环 +# cleaned_spec = self._spectral_gate(spec, noise_profile, sensitivity) +# mask = np.where(np.abs(cleaned_spec) > 0, 1, 0) +# mask = self._smooth_mask(mask, smooth//2+1) +# spec = spec * mask + +# # 相位恢复重建 +# processed = self._istft(spec * mask, n_fft) + +# # 动态增益归一化 +# peak = np.max(np.abs(processed)) +# processed = processed * (0.99 / peak) if peak > 0 else processed + +# # 格式转换 +# waveform = torch.from_numpy(processed).float().unsqueeze(0).unsqueeze(0) +# final_audio = {"waveform": waveform, "sample_rate": sample_rate} + +# return (final_audio,) + + +from .AddSubtitlesToVideo import AddSubtitlesToTensor +from .MWAudioRecorderAT import AudioRecorderAT + +NODE_CLASS_MAPPINGS = { + "AdjustAudio": AdjustAudio, + "TrimAudio": TrimAudio, + "RemoveSilence": RemoveSilence, + # "AudioDenoising": AudioDenoising, + "AudioRecorderAT": AudioRecorderAT, + "AddSubtitlesToVideo": AddSubtitlesToTensor, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "AdjustAudio": "Adjust Audio", + "TrimAudio": "Trim Audio", + "RemoveSilence": "Remove Silence", + # "AudioDenoising": "Audio Denoising", + "AudioRecorderAT": "MW Audio Recorder", + "AddSubtitlesToVideo": "Add Subtitles To Video", +} \ No newline at end of file diff --git a/images/2025-03-25_13-02-40.png b/images/2025-03-25_13-02-40.png new file mode 100644 index 0000000..3d3d066 Binary files /dev/null and b/images/2025-03-25_13-02-40.png differ diff --git a/images/2025-03-25_13-14-52.png b/images/2025-03-25_13-14-52.png new file mode 100644 index 0000000..eee04af Binary files /dev/null and b/images/2025-03-25_13-14-52.png differ diff --git a/images/2025-03-25_13-20-30.png b/images/2025-03-25_13-20-30.png new file mode 100644 index 0000000..9928f76 Binary files /dev/null and b/images/2025-03-25_13-20-30.png differ diff --git a/images/2025-03-25_13-33-54.png b/images/2025-03-25_13-33-54.png new file mode 100644 index 0000000..b73e287 Binary files /dev/null and b/images/2025-03-25_13-33-54.png differ diff --git a/images/2025-03-25_14-00-28.png b/images/2025-03-25_14-00-28.png new file mode 100644 index 0000000..67bf781 Binary files /dev/null and b/images/2025-03-25_14-00-28.png differ diff --git a/images/20250326001631.png b/images/20250326001631.png new file mode 100644 index 0000000..494c7de Binary files /dev/null and b/images/20250326001631.png differ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..6759a7c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "audiotools" +description = "A ComfyUI node containing multiple audio processing tools." +version = "1.0.0" +license = {file = "LICENSE"} +dependencies = ["sox", "librosa"] + +[project.urls] +Repository = "https://github.com/billwuhao/ComfyUI_AudioTools" +# Used by Comfy Registry https://comfyregistry.org + +[tool.comfy] +PublisherId = "mw" +DisplayName = "MW-ComfyUI_AudioTools" +Icon = "" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9d9e8ce --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +sox +librosa \ No newline at end of file diff --git a/workflow-examples/Automatic speech recognition - Automatic translation - Automatic sound cloning - Automatic lip syncing - Automatic subtitle addition.json b/workflow-examples/Automatic speech recognition - Automatic translation - Automatic sound cloning - Automatic lip syncing - Automatic subtitle addition.json new file mode 100644 index 0000000..b7cd6f1 --- /dev/null +++ b/workflow-examples/Automatic speech recognition - Automatic translation - Automatic sound cloning - Automatic lip syncing - Automatic subtitle addition.json @@ -0,0 +1 @@ +{"id":"a22ec44f-b275-4c55-b45d-e60542719cbc","revision":0,"last_node_id":66,"last_link_id":112,"nodes":[{"id":41,"type":"VHS_VideoCombine","pos":[1102.1680908203125,637.4804077148438],"size":[373.38238525390625,976.2353515625],"flags":{},"order":11,"mode":0,"inputs":[{"localized_name":"images","name":"images","type":"IMAGE","link":109},{"localized_name":"audio","name":"audio","shape":7,"type":"AUDIO","link":73},{"localized_name":"meta_batch","name":"meta_batch","shape":7,"type":"VHS_BatchManager","link":null},{"localized_name":"vae","name":"vae","shape":7,"type":"VAE","link":null}],"outputs":[{"localized_name":"Filenames","name":"Filenames","type":"VHS_FILENAMES","links":null}],"properties":{"cnr_id":"comfyui-videohelpersuite","ver":"5e61bcf218fe3bb7c899bbd584bbc99a9d05fb42","Node name for S&R":"VHS_VideoCombine"},"widgets_values":{"frame_rate":25,"loop_count":0,"filename_prefix":"latentsync","format":"video/h264-mp4","pix_fmt":"yuv420p","crf":19,"save_metadata":true,"trim_to_audio":false,"pingpong":false,"save_output":true,"videopreview":{"hidden":false,"paused":false,"params":{"filename":"latentsync_00008-audio.mp4","subfolder":"","type":"output","format":"video/h264-mp4","frame_rate":25,"workflow":"latentsync_00008.png","fullpath":"D:\\AIGC\\APP\\ComfyUI_v1\\ComfyUI\\output\\latentsync_00008-audio.mp4"},"muted":false}}},{"id":56,"type":"VHS_VideoInfoLoaded","pos":[-46.07161331176758,1159.599609375],"size":[242.013671875,106],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"video_info","name":"video_info","type":"VHS_VIDEOINFO","link":74}],"outputs":[{"localized_name":"fps🟦","name":"fps🟦","type":"FLOAT","links":[105]},{"localized_name":"frame_count🟦","name":"frame_count🟦","type":"INT","links":null},{"localized_name":"duration🟦","name":"duration🟦","type":"FLOAT","links":null},{"localized_name":"width🟦","name":"width🟦","type":"INT","links":[106]},{"localized_name":"height🟦","name":"height🟦","type":"INT","links":[107]}],"properties":{"cnr_id":"comfyui-videohelpersuite","ver":"5e61bcf218fe3bb7c899bbd584bbc99a9d05fb42","Node name for S&R":"VHS_VideoInfoLoaded"},"widgets_values":{}},{"id":61,"type":"EraXWoWRUN","pos":[-511.2992248535156,656.0599975585938],"size":[315,130],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"AUDIO","link":86}],"outputs":[{"localized_name":"text","name":"text","type":"STRING","links":[88]}],"properties":{"cnr_id":"erax-wow-turbo-mw","ver":"d26a2536761f231b70847bab8adacc73fb547baf","Node name for S&R":"EraXWoWRUN"},"widgets_values":["english",500,1,true]},{"id":62,"type":"GemmaxRun","pos":[-516.772705078125,874.990966796875],"size":[315,178],"flags":{},"order":3,"mode":0,"inputs":[{"name":"text","type":"STRING","widget":{"name":"text"},"link":88}],"outputs":[{"localized_name":"translations","name":"translations","type":"STRING","links":[92]}],"properties":{"cnr_id":"gemmax-mw","ver":"abe9bc37cb13e78027e7ec4c25fa0ea40e3b17cf","Node name for S&R":"GemmaxRun"},"widgets_values":["GemmaX2-28-2B-v0.1","English","中文","",500,true]},{"id":63,"type":"easy showAnything","pos":[-521.3506469726562,1144.076171875],"size":[324.938720703125,331.13482666015625],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"anything","name":"anything","shape":7,"type":"*","link":92}],"outputs":[{"localized_name":"output","name":"output","type":"*","links":[93]}],"properties":{"cnr_id":"comfyui-easy-use","ver":"3e84b8cd77719341adc71b08b5789bdb07b1a543","Node name for S&R":"easy showAnything"},"widgets_values":["里约热内卢著名的巴西狂欢节期间，街道上充满了桑巴节的节奏。"]},{"id":55,"type":"VideoLengthAdjuster","pos":[737.54541015625,644.2877197265625],"size":[315,126],"flags":{},"order":6,"mode":0,"inputs":[{"localized_name":"images","name":"images","type":"IMAGE","link":98},{"localized_name":"audio","name":"audio","type":"AUDIO","link":85}],"outputs":[{"localized_name":"images","name":"images","type":"IMAGE","slot_index":0,"links":[97]},{"localized_name":"audio","name":"audio","type":"AUDIO","slot_index":1,"links":[71]}],"properties":{"cnr_id":"ComfyUI-LatentSyncWrapper","ver":"26448c3071a43b9a54e6b03af91ee49939a32002","Node name for S&R":"VideoLengthAdjuster"},"widgets_values":["loop_to_audio",25,0.5]},{"id":40,"type":"VHS_LoadVideo","pos":[-912.7369995117188,653.9110107421875],"size":[345.1598815917969,902.06201171875],"flags":{},"order":0,"mode":0,"inputs":[{"localized_name":"meta_batch","name":"meta_batch","shape":7,"type":"VHS_BatchManager","link":null},{"localized_name":"vae","name":"vae","shape":7,"type":"VAE","link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","slot_index":0,"links":[98]},{"localized_name":"frame_count","name":"frame_count","type":"INT","links":null},{"localized_name":"audio","name":"audio","type":"AUDIO","links":[84,86]},{"localized_name":"video_info","name":"video_info","type":"VHS_VIDEOINFO","slot_index":3,"links":[74]}],"properties":{"cnr_id":"comfyui-videohelpersuite","ver":"5e61bcf218fe3bb7c899bbd584bbc99a9d05fb42","Node name for S&R":"VHS_LoadVideo"},"widgets_values":{"video":"latentsync_00002-audio.mp4","force_rate":25,"custom_width":0,"custom_height":768,"frame_load_cap":0,"skip_first_frames":0,"select_every_nth":1,"format":"AnimateDiff","choose video to upload":"image","videopreview":{"hidden":false,"paused":false,"params":{"force_rate":25,"frame_load_cap":0,"skip_first_frames":0,"select_every_nth":1,"filename":"latentsync_00002-audio.mp4","type":"input","format":"video/mp4"},"muted":false}}},{"id":60,"type":"SparkTTSClone","pos":[-107.64582061767578,662.9016723632812],"size":[400,358],"flags":{},"order":5,"mode":0,"inputs":[{"localized_name":"custom_clone_audio","name":"custom_clone_audio","shape":7,"type":"AUDIO","link":84},{"name":"text","type":"STRING","widget":{"name":"text"},"link":93}],"outputs":[{"localized_name":"audio","name":"audio","type":"AUDIO","links":[85,112]}],"properties":{"cnr_id":"sparktts_mw","ver":"d21d37fdbf4a0f1863aba5ecf1fe9f2ffb95f2d0","Node name for S&R":"SparkTTSClone"},"widgets_values":["","婷婷",0.8,50,0.95,3000,true,true,60491468633989,"fixed",""]},{"id":65,"type":"AddSubtitlesToVideo","pos":[757.4048461914062,1082.7010498046875],"size":[315,298],"flags":{},"order":10,"mode":0,"inputs":[{"localized_name":"images","name":"images","type":"IMAGE","link":104},{"name":"fps","type":"FLOAT","widget":{"name":"fps"},"link":105},{"name":"width","type":"INT","widget":{"name":"width"},"link":106},{"name":"height","type":"INT","widget":{"name":"height"},"link":107},{"name":"subtitles","type":"STRING","widget":{"name":"subtitles"},"link":108}],"outputs":[{"localized_name":"image","name":"image","type":"IMAGE","links":[109]}],"properties":{"aux_id":"billwuhao/ComfyUI_AudioTools","ver":"20b803b16905dab1ffab722b62be0ddc8603a9b7","Node name for S&R":"AddSubtitlesToVideo"},"widgets_values":["",8,"simhei.ttf",20,"black","#B8860B",-260,0,1024,1024,20]},{"id":66,"type":"WhisperTurboRun","pos":[353.8878173828125,1100.900146484375],"size":[315,178],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"AUDIO","link":112}],"outputs":[{"localized_name":"text","name":"text","type":"STRING","links":[111]}],"properties":{"cnr_id":"erax-wow-turbo-mw","ver":"895f239fd9dea069df7f5313da2a18ca9bb381ce","Node name for S&R":"WhisperTurboRun"},"widgets_values":[16,-0.1,0.03,true,true,true]},{"id":54,"type":"LatentSyncNode","pos":[752.3057250976562,848.6878662109375],"size":[315,150],"flags":{},"order":8,"mode":0,"inputs":[{"localized_name":"images","name":"images","type":"IMAGE","link":97},{"localized_name":"audio","name":"audio","type":"AUDIO","link":71}],"outputs":[{"localized_name":"images","name":"images","type":"IMAGE","slot_index":0,"links":[104]},{"localized_name":"audio","name":"audio","type":"AUDIO","slot_index":1,"links":[73]}],"properties":{"cnr_id":"ComfyUI-LatentSyncWrapper","ver":"26448c3071a43b9a54e6b03af91ee49939a32002","Node name for S&R":"LatentSyncNode"},"widgets_values":[1838,"fixed",1.5,20]},{"id":59,"type":"easy showAnything","pos":[355.803955078125,673.4379272460938],"size":[324.938720703125,331.13482666015625],"flags":{},"order":9,"mode":0,"inputs":[{"localized_name":"anything","name":"anything","shape":7,"type":"*","link":111}],"outputs":[{"localized_name":"output","name":"output","type":"*","links":[108]}],"properties":{"cnr_id":"comfyui-easy-use","ver":"3e84b8cd77719341adc71b08b5789bdb07b1a543","Node name for S&R":"easy showAnything"},"widgets_values":["[{'timestamp': [0.0, 3.9], 'text': '里约热内卢著名的巴西狂欢节期间街道'}, {'timestamp': [3.9, 5.88], 'text': '上充满了桑巴结的节奏'}]"]}],"links":[[71,55,1,54,1,"AUDIO"],[73,54,1,41,1,"AUDIO"],[74,40,3,56,0,"VHS_VIDEOINFO"],[84,40,2,60,0,"AUDIO"],[85,60,0,55,1,"AUDIO"],[86,40,2,61,0,"AUDIO"],[88,61,0,62,0,"STRING"],[92,62,0,63,0,"*"],[93,63,0,60,1,"STRING"],[97,55,0,54,0,"IMAGE"],[98,40,0,55,0,"IMAGE"],[104,54,0,65,0,"IMAGE"],[105,56,0,65,1,"FLOAT"],[106,56,3,65,2,"INT"],[107,56,4,65,3,"INT"],[108,59,0,65,4,"STRING"],[109,65,0,41,0,"IMAGE"],[111,66,0,59,0,"*"],[112,60,0,66,0,"AUDIO"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.6209213230591553,"offset":[1102.0982684104126,-476.2699235306927]},"workspace_info":{"id":"pdbYnk386LvXhcCuOca6c","saveLock":false,"cloudID":null,"coverMediaPath":null},"VHS_latentpreview":false,"VHS_latentpreviewrate":0,"node_versions":{"ComfyUI-VideoHelperSuite":"4c7858ddd5126f7293dc3c9f6e0fc4c263cde079","comfy-core":"0.3.26"},"VHS_MetadataImage":true,"VHS_KeepIntermediate":true,"ue_links":[]},"version":0.4} \ No newline at end of file