From b217dcee89a6a03dc87f8b17da7a3d4e4f847184 Mon Sep 17 00:00:00 2001 From: Hawk Lee Date: Wed, 31 Dec 2025 00:23:40 +0800 Subject: [PATCH] feat: integrate CosyVoice 3.0 with full capabilities (Instruct, Zero-Shot, Hybrid, SFT, Cross-Lingual) and update documentation --- README.md | 28 ++++++++-- aiia_cosyvoice_nodes.py | 121 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 140 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index ad55965..128c1d3 100644 --- a/README.md +++ b/README.md @@ -308,7 +308,23 @@ git clone https://github.com/havvk/ComfyUI_AIIA.git - `chunk_size`: 目标切片大小（默认 25 秒）。 - `overlap_size`: 重叠大小，用于平滑衔接。 -#### 3.5 智能音频增强 (Audio AI Enhance) (推荐) +#### 3.5 CosyVoice 3.0 TTS (AIIA) (文本转语音/生成) + +- **用途**: CosyVoice 3.0 全能文本转语音节点，支持纯文字描述生成音色。 +- **🔥 五种核心生成模式**: + 1. **描述生成 (Instruct)**: **文字 + 描述**。无需音频输入，直接根据文字描述（如“磁性的中年男声”）生成全新音色。 + 2. **音色克隆 (Zero-Shot)**: **文字 + 3秒音频**。模仿参考音频的音色。 + 3. **混合控制 (Hybrid)**: **音频 + 描述**。在克隆音色的基础上，通过描述词改变情感或表现力。 + 4. **跨语言 (Cross-Lingual)**: 让中文音频开口说地道的英文、日文等 9 种语言。 + 5. **固定 ID (SFT)**: 通过 `spk_id` 指定固定官方音色（如 pure_1, joy_1）。 +- **参数说明**: + - `tts_text`: 说的内容。 + - `instruct_text`: 对声音风格、情感、口音的文字描述。 + - `spk_id`: 传统的固定音色 ID（选填）。 + - `reference_audio`: 外部参考音频（选填，用于克隆/混合/跨语言）。 + - `seed`: 随机种子。固定种子可以“锁定”通过描述生成的那种随机音色。 + +#### 3.6 智能音频增强 (Audio AI Enhance) (推荐) - **用途**: 专为 CosyVoice 等生成式语音设计的增强器。它基于 `resemble-enhance` 强大的 **Conditional Flow Matching (CFM)** 模型，能同时完成后处理降噪和超分辨率（Bandwidth Extension）。 - **核心能力**: 将任意低采样率（如 22kHz, 16kHz）的输入音频，重构为 **44.1kHz** 的高保真音频。 @@ -353,7 +369,7 @@ git clone https://github.com/havvk/ComfyUI_AIIA.git - **依赖**: 首次运行会自动安装 `resemble-enhance`。 -#### 3.6 智能音频降噪 (VoiceFixer) (Legacy) +#### 3.7 智能音频降噪 (VoiceFixer) (Legacy) - **用途**: 用于修复**严重受损**的音频（如老电影、严重削波或极强背景噪）。 - **注意**: 对于 CosyVoice 生成的较干净语音，**不推荐**使用此节点，因为它通过重构方式修复，容易在干净语音上引入伪影（Spectral Stripes）。请优先使用 `Audio AI Enhance`。 @@ -396,7 +412,7 @@ git clone https://github.com/havvk/ComfyUI_AIIA.git hf download Diogodiogod/VoiceFixer-model.ckpt-1490000_trimed.pt model.ckpt-1490000_trimed.pt --local-dir synthesis_module/44100 ``` -#### 3.7 Audio Post-Process (Resample/Fade/Norm) +#### 3.8 Audio Post-Process (Resample/Fade/Norm) - **用途**: 音频后期处理“母带”节点。 - **功能**: @@ -407,7 +423,7 @@ git clone https://github.com/havvk/ComfyUI_AIIA.git - **Normalize**: 将音量标准化至 -1dB，确保输出响度饱满且不破音。 - **场景**: 建议串联在 `Voice Conversion` 节点之后使用。 -#### 3.8 Audio Splice Analyzer (AIIA Debug) +#### 3.9 Audio Splice Analyzer (AIIA Debug) - **用途**: 可视化验证音频拼接质量。 - **功能**: @@ -416,7 +432,7 @@ git clone https://github.com/havvk/ComfyUI_AIIA.git - 帮助用户直观地检查拼接点是否位于静音区，以及是否有明显的频谱断裂。 - **依赖**: 需要 `matplotlib` 库。如果未安装，节点会生成一张提示错误的图片，不会导致工作流崩溃。 -#### 3.9 音频信息查看 (Audio Info & Metadata) +#### 3.10 音频信息查看 (Audio Info & Metadata) - **用途**: 实时查看音频流的元数据，确保你的采样率符合预期。 - **输入**: `AUDIO` 张量。 @@ -426,7 +442,7 @@ git clone https://github.com/havvk/ComfyUI_AIIA.git - `duration`: 时长秒数 (FLOAT)。 - `channels`: 通道数 (INT)。 -#### 3.10 Microsoft VibeVoice (Beta) +#### 3.11 Microsoft VibeVoice (Beta) - **用途**: 微软最新的 TTS/音色克隆模型，支持 1.5B 和 7B 两种规格。 - **当前状态**: ✅ 可用（已通过测试） diff --git a/aiia_cosyvoice_nodes.py b/aiia_cosyvoice_nodes.py index 7ec3e6b..b196e32 100644 --- a/aiia_cosyvoice_nodes.py +++ b/aiia_cosyvoice_nodes.py @@ -107,7 +107,7 @@ class AIIA_CosyVoice_ModelLoader: "CosyVoice-ttsfrd" ],), "use_fp16": ("BOOLEAN", {"default": True}), - "use_rl_model": ("BOOLEAN", {"default": False, "tooltip": "Use llm.rl.pt (Reinforcement Learning optimized) if available. Only for V3/V2 models."}), + "use_rl_model": ("BOOLEAN", {"default": True, "tooltip": "Use llm.rl.pt (Reinforcement Learning optimized) if available. Only for V3/V2 models."}), } } @@ -148,6 +148,12 @@ class AIIA_CosyVoice_ModelLoader: model_dir = os.path.join(cosyvoice_path, local_name) + # --- Override: Check specific server path for Fun-CosyVoice3 --- + server_v3_path = "/app/ComfyUI/models/cosyvoice/Fun-CosyVoice3-0.5B-2512" + if local_name == "Fun-CosyVoice3-0.5B-2512" and os.path.exists(server_v3_path): + model_dir = server_v3_path + print(f"[AIIA] Using server cache for CosyVoice3: {model_dir}") + # Validation logic for V1/V2 vs V3 yaml_path = os.path.join(model_dir, "cosyvoice.yaml") yaml3_path = os.path.join(model_dir, "cosyvoice3.yaml") @@ -469,11 +475,120 @@ class AIIA_CosyVoice_VoiceConversion: finally: if os.path.exists(source_path): os.unlink(source_path) +class AIIA_CosyVoice_TTS: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "model": ("COSYVOICE_MODEL",), + "tts_text": ("STRING", {"multiline": True, "default": "你好，这是 CosyVoice 3.0 的全能模式测试。"}), + "instruct_text": ("STRING", {"multiline": True, "default": "一个沉稳、磁性的成熟男性声音，语法标准，情感饱满。"}), + "spk_id": ("STRING", {"default": "pure_1", "tooltip": "V1/V2 时代的固定 ID，如 pure_1, joy_1 等。如果提供参考音频且非 SFT 模型，此项可能被忽略。"}), + "speed": ("FLOAT", {"default": 1.0, "min": 0.5, "max": 2.0, "step": 0.1}), + "seed": ("INT", {"default": 42, "min": -1, "max": 2147483647}), + }, + "optional": { + "reference_audio": ("AUDIO",), + } + } + + RETURN_TYPES = ("AUDIO",) + RETURN_NAMES = ("audio",) + FUNCTION = "generate" + CATEGORY = "AIIA/Synthesis" + + def generate(self, model, tts_text, instruct_text, spk_id, speed, seed, reference_audio=None): + cosyvoice_model = model["model"] + sample_rate = cosyvoice_model.sample_rate + + if seed >= 0: + torch.manual_seed(seed) + if torch.cuda.is_available(): torch.cuda.manual_seed_all(seed) + + final_waveform = None + + try: + # Detect Model Type for specific logic + is_v3 = "CosyVoice3" in type(cosyvoice_model).__name__ + is_v2 = "CosyVoice2" in type(cosyvoice_model).__name__ + + # 1. Hybrid / Cross-Lingual / Zero-Shot (Reference Audio provided) + if reference_audio is not None: + ref_wav = reference_audio["waveform"] + if reference_audio["sample_rate"] != sample_rate: + import torchaudio + ref_wav = torchaudio.transforms.Resample(reference_audio["sample_rate"], sample_rate)(ref_wav) + + if ref_wav.abs().max() > 1.0: ref_wav = ref_wav / ref_wav.abs().max() + + with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp_ref: + ref_path = tmp_ref.name + ref_np = ref_wav.squeeze().cpu().numpy() + if ref_np.ndim == 2: ref_np = ref_np.T + sf.write(ref_path, ref_np, sample_rate, subtype='FLOAT') + + try: + if is_v3 or is_v2: + # V3/V2: Support Instruct + Audio (Hybrid Mode) + print(f"[AIIA] CosyVoice V3/V2: Hybrid/Zero-Shot Mode. Instruct: {instruct_text[:20]}...") + output = cosyvoice_model.inference_instruct2( + tts_text=tts_text, + instruct_text=instruct_text, + prompt_wav=ref_path, + zero_shot_spk_id=spk_id, # Optional placeholder + stream=False, + speed=speed + ) + else: + # V1: Source-Prompt based inference + print("[AIIA] CosyVoice V1: Zero-Shot / Cross-Lingual Mode") + # Some versions use inference_vc or inference_zero_shot + if hasattr(cosyvoice_model, 'inference_zero_shot'): + output = cosyvoice_model.inference_zero_shot(tts_text, ref_path, stream=False, speed=speed) + else: + output = cosyvoice_model.inference_vc(source_wav=None, prompt_wav=ref_path, stream=False, speed=speed) + + all_speech = [chunk['tts_speech'] for chunk in output] + final_waveform = torch.cat(all_speech, dim=-1) + finally: + if os.path.exists(ref_path): os.unlink(ref_path) + + # 2. Instruct / SFT / Random (No Reference Audio) + else: + if is_v3 or is_v2: + print(f"[AIIA] CosyVoice V3/V2: Instruct Generation. Description: {instruct_text[:20]}...") + # For pure generation in V3, prompt_wav can be None + output = cosyvoice_model.inference_instruct2( + tts_text=tts_text, + instruct_text=instruct_text, + prompt_wav=None, + zero_shot_spk_id=spk_id, + stream=False, + speed=speed + ) + else: + # V1 SFT / Instruct + print(f"[AIIA] CosyVoice V1: SFT/Instruct. Speaker ID: {spk_id}") + if "SFT" in type(cosyvoice_model).__name__ or spk_id: + output = cosyvoice_model.inference_sft(tts_text, spk_id, stream=False, speed=speed) + else: + output = cosyvoice_model.inference_instruct(tts_text, spk_id, instruct_text, stream=False, speed=speed) + + all_speech = [chunk['tts_speech'] for chunk in output] + final_waveform = torch.cat(all_speech, dim=-1) + + except Exception as e: + raise RuntimeError(f"CosyVoice generation failed: {e}") + + return ({"waveform": final_waveform.unsqueeze(0).cpu(), "sample_rate": sample_rate},) + NODE_CLASS_MAPPINGS = { "AIIA_CosyVoice_ModelLoader": AIIA_CosyVoice_ModelLoader, - "AIIA_CosyVoice_VoiceConversion": AIIA_CosyVoice_VoiceConversion + "AIIA_CosyVoice_VoiceConversion": AIIA_CosyVoice_VoiceConversion, + "AIIA_CosyVoice_TTS": AIIA_CosyVoice_TTS } NODE_DISPLAY_NAME_MAPPINGS = { "AIIA_CosyVoice_ModelLoader": "CosyVoice Model Loader (AIIA)", - "AIIA_CosyVoice_VoiceConversion": "Voice Conversion (AIIA Unlimited)" + "AIIA_CosyVoice_VoiceConversion": "Voice Conversion (AIIA Unlimited)", + "AIIA_CosyVoice_TTS": "CosyVoice 3.0 TTS (AIIA)" }