From f73661f6559b7da7211e31021f8be326bc56bba2 Mon Sep 17 00:00:00 2001 From: Hawk Lee Date: Fri, 20 Feb 2026 01:17:02 +0800 Subject: [PATCH] feat: release version 1.15.0 with IndexTTS-2 inline emotion tags and fixes --- README.md | 87 +++- aiia_indextts_nodes.py | 444 +++++++++++++----- aiia_podcast_stitcher.py | 9 +- .../gpt/transformers_generation_utils.py | 57 ++- .../indextts/gpt/transformers_gpt2.py | 61 ++- libs/index-tts/indextts/infer_v2.py | 53 ++- pyproject.toml | 2 +- test_strict_indextts.py | 32 +- 8 files changed, 566 insertions(+), 179 deletions(-) diff --git a/README.md b/README.md index 351e73c..7d6f0b0 100755 --- a/README.md +++ b/README.md @@ -1107,24 +1107,85 @@ Qwen3-TTS 最强大的特性之一是其**自然语言指令驱动**的能力。 #### 3.14 IndexTTS-2 (New! 🔥) - **用途**: Bilibili 开源的 **零样本语音克隆 + 情感控制** TTS 模型。支持**音色与情感解耦**，调整情感时不会导致音色漂移。 + - **🔥 核心特性**: - **音色/情感解耦 (Voice-Emotion Decoupling)**: 独立控制音色和情感，改变情感不影响声音音色。这解决了其他模型在调整情感时音色漂移的顽疾。 - **8 维情感向量控制**: 提供 `happy`, `angry`, `sad`, `afraid`, `disgusted`, `melancholic`, `surprised`, `calm` 八个情感滑块，精细控制输出语音的情感表达。 + - **🆕 内联情感标签 (Inline Emotion Tags)**: 支持在文本中嵌入 `[Happy]`、`[Sad]` 等标签，**逐句控制不同情感**。节点自动按标签分段生成，再以余弦淡入淡出 + 静音间隔无缝拼接。 - **情感音频参考**: 支持通过额外的 `emotion_audio` 输入捕获参考音频中的情感特征，并通过 `emo_alpha` 控制混合强度。 + - **QwenEmotion 自动推断**: 启用 `use_emo_text` 后，内置 Qwen 情感模型自动分析文本语义，推断最合适的情感向量。 - **零样本音色克隆**: 输入 `reference_audio` 即可克隆声音。 + - **节点**: - - **IndexTTS-2 Loader**: 加载 IndexTTS-2 模型。 - - `use_fp16`: 半精度推理（降低 VRAM，加速推理）。 - - `use_cuda_kernel`: 使用 BigVGAN 自定义 CUDA 核心（仅 NVIDIA GPU）。 - - `model_dir` (可选): 自定义模型目录，留空则使用默认路径 `ComfyUI/models/indextts2/`。 - - **IndexTTS-2 TTS**: 执行语音合成。 - - `text`: 待合成的文本（支持中英文混合）。 - - `reference_audio`: 音色参考音频。 - - `emotion_audio` (可选): 独立的情感参考音频。 - - 8 个情感滑块 (`happy` ~ `calm`)：直接控制情感向量。 - - `emo_alpha`: 情感混合强度 (0=无情感, 1=完全情感)。 - - `use_random`: 随机采样（开启后降低克隆保真度）。 - - `seed`: 随机种子，用于可复现生成。 + + **IndexTTS-2 Loader** — 加载 IndexTTS-2 模型 + + | 参数 | 默认 | 说明 | + |------|------|------| + | `use_fp16` | `True` | 半精度推理，降低 VRAM 占用 | + | `use_cuda_kernel` | `True` | BigVGAN CUDA 核心加速（仅 NVIDIA GPU，推理更快） | + | `model_dir` | 空 | 自定义模型路径，留空使用 `ComfyUI/models/indextts2/` | + + **IndexTTS-2 TTS** — 执行语音合成 + + | 参数 | 默认 | 说明 | + |------|------|------| + | `text` | — | 待合成文本，支持中英文混合及 **内联情感标签** | + | `voice_preset` | `Female_HQ` | 内置音色预设（在无 `reference_audio` 时使用） | + | `reference_audio` | — | 音色参考音频（零样本克隆，优先于 preset） | + | `emotion_audio` | — | 独立的情感参考音频 | + | `emo_alpha` | `1.0` | 情感混合强度 (0=无情感, 1=100%情感) | + | `happy` ~ `calm` | `0.0` | 8 个情感滑块，直接控制向量 | + | `use_emo_text` | `False` | 启用后使用 Qwen 模型自动推断情感 | + | `emo_text` | 空 | 自定义情感提示文本（配合 `use_emo_text`） | + | `interval_silence` | `200` | 长文本分段间静音时长 (ms) | + | `max_text_tokens_per_segment` | `120` | 每段最大 token 数（控制切分粒度） | + | `use_random` | `False` | 随机采样（降低克隆保真度） | + | `seed` | `0` | 随机种子 (-1=随机) | + +- **🆕 内联情感标签 (Inline Emotion Tags)**: + + 在文本中嵌入 `[标签名]` 即可逐句控制情感，节点会自动按标签拆分，每段独立生成后拼接。 + + **使用示例**： + ```text + [happy] 今天天气真好，阳光明媚！ + [sad] 但是我养的小猫走丢了，我好难过。 + [calm] 不过我相信它一定会自己找到回家的路。 + ``` + → 生成 3 段音频，分别带 happy / sad / calm 情感，最终余弦淡入淡出无缝拼接为一段完整音频。 + + > [!IMPORTANT] + > **控制情感强度**: 内联标签默认映射到全强度的情感向量 (1.0)。如果发现音色失真或情感过于夸张（如嘶吼、破音），请尝试**降低** `emo_alpha` 参数（推荐值 **0.6 ~ 0.8**）。较低的 alpha 值能更好地保留原始音色特征，同时赋予适度的情感色彩。 + + **支持的 26 种标签**： + + | 分类 | 标签 | + |------|------| + | 积极情感 | `happy`, `excited`, `enthusiastic`, `proud`, `romantic`, `innocent` | + | 消极情感 | `sad`, `angry`, `afraid`, `fearful`, `disgusted`, `disappointed`, `anxious`, `nervous` | + | 复合情感 | `sarcastic`, `nostalgic`, `confused`, `mysterious`, `gossip` | + | 中性情感 | `calm`, `neutral`, `gentle`, `serious`, `lazy`, `melancholic`, `surprised` | + + 每个标签自动映射到 IndexTTS-2 的 8 维情感向量。**未知标签**（不在上述列表中的）自动回退至 QwenEmotion 模型推断。 + + > [!TIP] + > 内联标签通常由 **AIIA Emotion Annotator** 节点自动注入到 Splitter 的输出中，无需手写。也可以手动添加来精细控制。 + +- **🔧 情感控制优先级**: + + 当多种情感来源同时存在时，按以下优先级生效： + + | 优先级 | 来源 | 说明 | + |:------:|------|------| + | 1 (最高) | 内联标签 `[Tag]` | 逐句独立控制，覆盖所有其他设置 | + | 2 | 滑块向量 | 8 个滑块全局生效（无标签时） | + | 3 | `use_emo_text` | Qwen 自动推断（无标签无滑块时） | + | 4 (最低) | `emotion_audio` | 从参考音频提取情感 | + + > [!NOTE] + > 如果文本含标签，则标签内的段落使用标签情感；**无标签的段落**继续使用滑块或其他全局设置。 + - **✅ 依赖版本兼容性 (Dependency Compatibility)**: AIIA 已内置深度兼容层，**完美支持 `transformers` 4.57+**。 @@ -1192,7 +1253,7 @@ Qwen3-TTS 最强大的特性之一是其**自然语言指令驱动**的能力。 - **追求“听起来最像真人” (音质+音色)**: 选 **VoxCPM 1.5**。它的 Tokenizer-free 架构带来了质的飞跃。 - **追求“方言/多语言/稳定性”**: 选 **CosyVoice 3.0**。目前依然是生产环境最稳的选择。 - **追求“多样化音色设计/最新 Qwen 生态/长语音流畅度”**: 选 **Qwen3-TTS**。其 VoiceDesign 功能能让你用描述语“捏”出从未听过的声音。 -- **追求“情感精细控制/音色情感解耦”**: 选 **IndexTTS-2**。独立的 8 维情感控制和情感音频参考是其独有优势。 +- **追求“情感精细控制/音色情感解耦”**: 选 **IndexTTS-2**。独立的 8 维情感控制、内联情感标签（逐句不同情感）和情感音频参考是其独有优势。 - **要做“长篇广播剧/播客”**: 选 **VibeVoice**。它的长窗口上下文优势依然不可替代。 ### 4. 播客与对话生成 (Podcast & Dialogue Generation) diff --git a/aiia_indextts_nodes.py b/aiia_indextts_nodes.py index 52d3eea..a0a0979 100644 --- a/aiia_indextts_nodes.py +++ b/aiia_indextts_nodes.py @@ -5,12 +5,17 @@ Loader + TTS nodes for Bilibili's IndexTTS-2 zero-shot voice cloning model. import os import sys +import re +import contextlib +import tempfile +import warnings + import torch import torchaudio -import tempfile import numpy as np -import contextlib -import shutil + +from pathlib import Path +from typing import Optional, List, Tuple # --- Preload kaldifst to prevent hang when used with NeMo Diarization --- # IndexTTS-2 seems to put the process (OpenMP/MKL/dlopen lock) in a state @@ -60,8 +65,7 @@ def _ensure_indextts(): _install_missing_deps() - # Patch transformers compatibility before any indextts import - # _patch_transformers_compat() -> Now using context manager in load/infer + _INDEXTTS_READY = True @@ -73,89 +77,7 @@ def _ensure_indextts(): _SENTINEL = object() # unique marker for "attribute didn't exist before" -@contextlib.contextmanager -def _transformers_patches(): - """ - Context manager: temporarily add shims for attributes removed in newer - transformers versions. On exit every added attribute is deleted again so - that downstream imports (NeMo, etc.) see an unmodified transformers. - IndexTTS's own modules are safe because they cache references at import - time via ``from X import Y`` — those bindings survive the delattr. - """ - - from transformers import cache_utils - from transformers.generation import candidate_generator as cg - from transformers.generation import configuration_utils as cu - import transformers.modeling_utils as mu - from transformers import GenerationConfig - - # ---- build the list of (module, attr_name, shim_value) --------------- - class _QCC: - def __init__(self, **kw): pass - - def _crop(model, past_key_values, max_length): - return past_key_values - - class _SeqSummary(torch.nn.Module): - def __init__(self, config): super().__init__() - def forward(self, hidden_states, **kw): return hidden_states[:, -1] - - def _chunking(forward_fn, chunk_size, *tensors, **kw): - return forward_fn(*tensors, **kw) - - patches = [ - (cache_utils, "QuantizedCacheConfig", _QCC), - (cg, "_crop_past_key_values", _crop), - (cu, "NEED_SETUP_CACHE_CLASSES_MAPPING", {}), - (cu, "QUANT_BACKEND_CLASSES_MAPPING", {}), - (mu, "SequenceSummary", _SeqSummary), - (mu, "apply_chunking_to_forward", _chunking), - ] - - # GenerationConfig.forced_decoder_ids (class-level attribute) - gc_had = hasattr(GenerationConfig, "forced_decoder_ids") - gc_old = getattr(GenerationConfig, "forced_decoder_ids", _SENTINEL) - - # ---- apply (only if the attribute is absent) ------------------------- - originals = [] # (module, attr_name, old_value_or_SENTINEL) - count = 0 - for mod, attr, shim in patches: - old = getattr(mod, attr, _SENTINEL) - originals.append((mod, attr, old)) - if old is _SENTINEL: - setattr(mod, attr, shim) - count += 1 - - if not gc_had: - setattr(GenerationConfig, "forced_decoder_ids", None) - count += 1 - - if count > 0: - print(f"[AIIA] Applied {count} transformers compatibility patches.") - - try: - yield - finally: - # ---- revert: remove everything we added ------------------------- - reverted = 0 - for mod, attr, old in originals: - if old is _SENTINEL: - # we added it → delete it - if hasattr(mod, attr): - delattr(mod, attr) - reverted += 1 - # else: attribute existed before us, leave it alone - - if not gc_had: - try: - delattr(GenerationConfig, "forced_decoder_ids") - reverted += 1 - except AttributeError: - pass - - if reverted > 0: - print(f"[AIIA] Reverted {reverted} transformers compatibility patches.") @@ -280,7 +202,7 @@ class AIIA_IndexTTS2_Loader: return { "required": { "use_fp16": ("BOOLEAN", {"default": True, "tooltip": "Use half-precision for lower VRAM and faster inference."}), - "use_cuda_kernel": ("BOOLEAN", {"default": True, "tooltip": "Use BigVGAN custom CUDA kernel (faster, CUDA only)."}), + "use_cuda_kernel": ("BOOLEAN", {"default": True, "tooltip": "Use BigVGAN custom CUDA kernel for faster vocoder inference (NVIDIA GPU only)."}), }, "optional": { "model_dir": ("STRING", { @@ -338,19 +260,15 @@ class AIIA_IndexTTS2_Loader: print(f"[AIIA] Loading IndexTTS-2 from {model_dir} (fp16={use_fp16}, cuda_kernel={use_cuda_kernel})") - # Temporarily apply transformers compatibility patches for the IndexTTS - # import chain. They are reverted on exit so other modules (NeMo etc.) - # see an unmodified transformers. - with _transformers_patches(): - from indextts.infer_v2 import IndexTTS2 - with _patch_indextts_loading(model_dir): - tts = IndexTTS2( - cfg_path=cfg_path, - model_dir=model_dir, - use_fp16=use_fp16, - use_cuda_kernel=use_cuda_kernel, - use_deepspeed=False, - ) + from indextts.infer_v2 import IndexTTS2 + with _patch_indextts_loading(model_dir): + tts = IndexTTS2( + cfg_path=cfg_path, + model_dir=model_dir, + use_fp16=use_fp16, + use_cuda_kernel=use_cuda_kernel, + use_deepspeed=False, + ) _INDEXTTS_MODEL_CACHE[cache_key] = tts print("[AIIA] IndexTTS-2 loaded successfully.") @@ -358,11 +276,159 @@ class AIIA_IndexTTS2_Loader: return (tts,) +# ============================================================================ +# EMOTION TAG MAPPING +# ============================================================================ + +# Emotion tag → IndexTTS-2 8-dim vector [happy, angry, sad, afraid, disgusted, melancholic, surprised, calm] +_EMOTION_TAG_TO_VECTOR = { + "happy": [1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + "excited": [0.8, 0.0, 0.0, 0.0, 0.0, 0.0, 0.3, 0.0], + "enthusiastic": [0.9, 0.0, 0.0, 0.0, 0.0, 0.0, 0.2, 0.0], + "proud": [0.6, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.3], + "angry": [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + "sad": [0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0], + "disappointed": [0.0, 0.0, 0.7, 0.0, 0.0, 0.3, 0.0, 0.0], + "nostalgic": [0.0, 0.0, 0.4, 0.0, 0.0, 0.5, 0.0, 0.1], + "afraid": [0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0], + "fearful": [0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0], + "anxious": [0.0, 0.0, 0.0, 0.6, 0.0, 0.0, 0.2, 0.0], + "nervous": [0.0, 0.0, 0.0, 0.5, 0.0, 0.0, 0.3, 0.0], + "disgusted": [0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0], + "sarcastic": [0.0, 0.3, 0.0, 0.0, 0.4, 0.0, 0.0, 0.0], + "melancholic": [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0], + "surprised": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0], + "confused": [0.0, 0.0, 0.0, 0.2, 0.0, 0.0, 0.6, 0.0], + "mysterious": [0.0, 0.0, 0.0, 0.2, 0.0, 0.3, 0.0, 0.3], + "calm": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0], + "gentle": [0.2, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.7], + "neutral": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.5], + "serious": [0.0, 0.2, 0.0, 0.0, 0.0, 0.0, 0.0, 0.5], + "romantic": [0.4, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4], + "lazy": [0.0, 0.0, 0.0, 0.0, 0.0, 0.3, 0.0, 0.5], + "gossip": [0.3, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4, 0.0], + "innocent": [0.3, 0.0, 0.0, 0.0, 0.0, 0.0, 0.3, 0.3], +} + + # ============================================================================ # TTS NODE # ============================================================================ class AIIA_IndexTTS2_TTS: + # Match emotion tags injected by Splitter/Annotator, e.g. [Happy], [Calm] + _EMOTION_TAG_RE = re.compile(r'^\[(\w[\w\s]*?)\]\s*', re.MULTILINE) + + @staticmethod + def _split_by_emotion_tags(text: str) -> List[Tuple[Optional[str], str]]: + """ + Split text by lines. + - Lines starting with `[Tag]` get that emotion. + - Lines without a tag get None (No Emotion). I.e. NO Inheritance. + - Consecutive lines with the same emotion are merged. + + Input: + [Happy] Line 1 + Line 2 (No tag -> None) + [Happy] Line 3 + [Happy] Line 4 + + Output: + [("Happy", "Line 1"), (None, "Line 2"), ("Happy", "Line 3\\nLine 4")] + """ + # Regex to match leading tag: [Happy] ... + tag_pattern = re.compile(r'^\s*\[([\w\s]+?)\]\s*(.*)$') + + lines = text.split('\n') + raw_segments = [] + + for line in lines: + m = tag_pattern.match(line) + if m: + tag = m.group(1) + content = m.group(2) + raw_segments.append((tag, content)) + else: + raw_segments.append((None, line)) + + if not raw_segments: + return [(None, text)] + + # Merge consecutive segments with the same emotion (case-insensitive) + merged = [raw_segments[0]] + for tag, txt in raw_segments[1:]: + prev_tag, prev_txt = merged[-1] + + same_emo = False + if prev_tag is None and tag is None: + same_emo = True + elif prev_tag is not None and tag is not None: + if prev_tag.strip().lower() == tag.strip().lower(): + same_emo = True + + if same_emo: + merged[-1] = (prev_tag, prev_txt + "\n" + txt) + else: + merged.append((tag, txt)) + + # Filter out purely empty segments if they have a tag (avoid generating silence for just tags) + # But keep None segments (for pauses) + final_segments = [] + for tag, txt in merged: + if txt.strip(): + final_segments.append((tag, txt)) + + if not final_segments: + return [(None, text)] + + return final_segments + + @staticmethod + def _crossfade_segments(waveforms: list, sample_rate: int, crossfade_ms: int = 50, silence_ms: int = 100) -> torch.Tensor: + """ + Join waveform segments with cosine fade-out / silence gap / fade-in. + Each waveform is (C, N). + """ + if not waveforms: + return torch.zeros(1, 0) + if len(waveforms) == 1: + return waveforms[0] + + xfade_samples = int(sample_rate * crossfade_ms / 1000) + silence_samples = int(sample_rate * silence_ms / 1000) + + result = waveforms[0] + for i in range(1, len(waveforms)): + curr = waveforms[i] + + # Fade out the tail of previous segment + fo_len = min(xfade_samples, result.shape[-1]) + if fo_len >= 2: + t_fo = torch.linspace(0, np.pi, fo_len, device=result.device) + fade_out = 0.5 * (1.0 + torch.cos(t_fo)) # 1 → 0 + result = torch.cat([ + result[..., :-fo_len], + result[..., -fo_len:] * fade_out, + ], dim=-1) + + # Insert silence gap + channels = result.shape[0] if result.dim() >= 2 else 1 + silence = torch.zeros(channels, silence_samples, device=result.device) + + # Fade in the head of next segment + fi_len = min(xfade_samples, curr.shape[-1]) + if fi_len >= 2: + t_fi = torch.linspace(0, np.pi, fi_len, device=curr.device) + fade_in = 1.0 - 0.5 * (1.0 + torch.cos(t_fi)) # 0 → 1 + curr = torch.cat([ + curr[..., :fi_len] * fade_in, + curr[..., fi_len:], + ], dim=-1) + + result = torch.cat([result, silence, curr], dim=-1) + + return result + @classmethod def INPUT_TYPES(cls): return { @@ -386,6 +452,22 @@ class AIIA_IndexTTS2_TTS: "melancholic": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.05}), "surprised": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.05}), "calm": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.05}), + "use_emo_text": ("BOOLEAN", { + "default": False, + "tooltip": "Auto-detect emotion from text using built-in Qwen emotion model. Overrides emotion sliders." + }), + "emo_text": ("STRING", { + "default": "", + "tooltip": "Custom emotion text prompt (used with use_emo_text). Leave empty to use main text." + }), + "interval_silence": ("INT", { + "default": 200, "min": 0, "max": 2000, "step": 50, + "tooltip": "Silence duration (ms) inserted between text segments for long text." + }), + "max_text_tokens_per_segment": ("INT", { + "default": 120, "min": 30, "max": 500, "step": 10, + "tooltip": "Max tokens per text segment. Lower = more segments, higher = longer per-segment generation." + }), "use_random": ("BOOLEAN", { "default": False, "tooltip": "Enable random sampling (reduces voice cloning fidelity)." @@ -444,12 +526,42 @@ class AIIA_IndexTTS2_TTS: tmp.close() return tmp.name + def _infer_single_segment(self, tts, text, ref_path, out_path, + emo_path=None, emo_alpha=1.0, emo_vector=None, + use_emo_text=False, emo_text=None, + interval_silence=200, max_text_tokens_per_segment=120, + use_random=False): + """Generate a single segment and return (waveform, sample_rate).""" + with torch.no_grad(): + tts.infer( + spk_audio_prompt=ref_path, + text=text, + output_path=out_path, + emo_audio_prompt=emo_path, + emo_alpha=emo_alpha, + emo_vector=emo_vector, + use_emo_text=use_emo_text, + emo_text=emo_text if (emo_text and emo_text.strip()) else None, + interval_silence=interval_silence, + max_text_tokens_per_segment=max_text_tokens_per_segment, + use_random=use_random, + verbose=True, + ) + + if not os.path.exists(out_path) or os.path.getsize(out_path) == 0: + return None, None + + wav, sr = torchaudio.load(out_path) + return wav, sr + def generate(self, indextts_model, text, voice_preset="Female_HQ", reference_audio=None, emotion_audio=None, emo_alpha=1.0, happy=0.0, angry=0.0, sad=0.0, afraid=0.0, disgusted=0.0, melancholic=0.0, surprised=0.0, calm=0.0, + use_emo_text=False, emo_text="", + interval_silence=200, max_text_tokens_per_segment=120, use_random=False, seed=0): _ensure_indextts() @@ -483,11 +595,20 @@ class AIIA_IndexTTS2_TTS: if emotion_audio is not None: emo_path = self._audio_to_wav_path(emotion_audio, prefix="indextts_emo_") - # --- Emotion vector (if any slider > 0) --- - emo_vector = [happy, angry, sad, afraid, disgusted, melancholic, surprised, calm] - has_emo_vector = any(v > 0.001 for v in emo_vector) - if not has_emo_vector: - emo_vector = None + # --- Emotion vector from sliders (if any slider > 0) --- + slider_emo_vector = [happy, angry, sad, afraid, disgusted, melancholic, surprised, calm] + has_slider_emo = any(v > 0.001 for v in slider_emo_vector) + if not has_slider_emo: + slider_emo_vector = None + + # --- Detect emotion tags in text --- + emotion_segments = self._split_by_emotion_tags(text) + has_multi_emotion = len(emotion_segments) > 1 or (len(emotion_segments) == 1 and emotion_segments[0][0] is not None) + + if has_multi_emotion: + print(f"[AIIA IndexTTS-2] Detected {len(emotion_segments)} emotion-tagged segment(s):") + for i, (emo, seg) in enumerate(emotion_segments): + print(f" Segment {i}: [{emo}] {seg[:50]}...") # --- Output temp file --- out_fd, out_path = tempfile.mkstemp(suffix=".wav", prefix="indextts_out_") @@ -496,37 +617,100 @@ class AIIA_IndexTTS2_TTS: # --- Generate --- try: print(f"[AIIA IndexTTS-2] Generating: text='{text[:50]}...', ref={ref_path}") - if emo_vector: - print(f" Emotion vector: {emo_vector}, alpha={emo_alpha}") - if emo_path: - print(f" Emotion audio: {emo_path}") - # Temporarily re-apply patches for inference, then revert. - with _transformers_patches(), torch.no_grad(): - tts.infer( - spk_audio_prompt=ref_path, - text=text, - output_path=out_path, - emo_audio_prompt=emo_path, + if has_multi_emotion: + # ===== Per-segment emotion generation ===== + all_waveforms = [] + final_sr = 22050 # will be overwritten by actual sr + + for seg_idx, (seg_emotion, seg_text) in enumerate(emotion_segments): + # Determine emotion vector for this segment + seg_emo_vector = None + seg_use_emo_text = False + seg_emo_text = None + seg_emo_path = emo_path # default: use global emotion audio + + if seg_emotion: + tag_lower = seg_emotion.lower().strip() + if tag_lower in _EMOTION_TAG_TO_VECTOR: + # Known tag → use mapped vector + seg_emo_vector = list(_EMOTION_TAG_TO_VECTOR[tag_lower]) + seg_emo_path = None # tag overrides emotion audio + print(f" [{seg_emotion}] → mapped vector: {seg_emo_vector}") + else: + # Unknown tag → fallback to QwenEmotion inference + seg_use_emo_text = True + seg_emo_text = seg_emotion + seg_emo_path = None + print(f" [{seg_emotion}] → unknown tag, falling back to QwenEmotion inference") + else: + # No tag on this segment → use slider vector or global settings + seg_emo_vector = slider_emo_vector + seg_use_emo_text = use_emo_text + seg_emo_text = emo_text + + print(f" Generating segment {seg_idx+1}/{len(emotion_segments)}: '{seg_text[:40]}...'") + + wav, sr = self._infer_single_segment( + tts, seg_text, ref_path, out_path, + emo_path=seg_emo_path, + emo_alpha=emo_alpha, + emo_vector=seg_emo_vector, + use_emo_text=seg_use_emo_text, + emo_text=seg_emo_text, + interval_silence=interval_silence, + max_text_tokens_per_segment=max_text_tokens_per_segment, + use_random=use_random, + ) + + if wav is not None: + all_waveforms.append(wav) + final_sr = sr + else: + print(f" WARNING: Segment {seg_idx+1} produced no output, skipping.") + + if not all_waveforms: + print("[AIIA IndexTTS-2] WARNING: All segments produced no output. Returning silence.") + silence = torch.zeros(1, 1, 22050) + return ({"waveform": silence, "sample_rate": 22050},) + + # Crossfade join all segments + final_wav = self._crossfade_segments(all_waveforms, final_sr) + final_wav = final_wav.unsqueeze(0) # (C, N) → (1, C, N) + + total_duration = final_wav.shape[-1] / final_sr + print(f"[AIIA IndexTTS-2] Generated {len(all_waveforms)} segments, total {total_duration:.2f}s at {final_sr}Hz") + + return ({"waveform": final_wav, "sample_rate": final_sr},) + + else: + # ===== Single segment (original path) ===== + if slider_emo_vector: + print(f" Emotion vector: {slider_emo_vector}, alpha={emo_alpha}") + if emo_path: + print(f" Emotion audio: {emo_path}") + + wav, sr = self._infer_single_segment( + tts, text, ref_path, out_path, + emo_path=emo_path, emo_alpha=emo_alpha, - emo_vector=emo_vector, + emo_vector=slider_emo_vector, + use_emo_text=use_emo_text, + emo_text=emo_text, + interval_silence=interval_silence, + max_text_tokens_per_segment=max_text_tokens_per_segment, use_random=use_random, - verbose=True, ) - # --- Read output wav → ComfyUI AUDIO format --- - if not os.path.exists(out_path) or os.path.getsize(out_path) == 0: - print("[AIIA IndexTTS-2] WARNING: Generation produced no output. Returning silence.") - silence = torch.zeros(1, 1, 22050) # 1 second silence - return ({"waveform": silence, "sample_rate": 22050},) + if wav is None: + print("[AIIA IndexTTS-2] WARNING: Generation produced no output. Returning silence.") + silence = torch.zeros(1, 1, 22050) + return ({"waveform": silence, "sample_rate": 22050},) - wav, sr = torchaudio.load(out_path) - # wav shape: (channels, samples) → (1, channels, samples) for ComfyUI batch dim - wav = wav.unsqueeze(0) + wav = wav.unsqueeze(0) # (C, N) → (1, C, N) + print(f"[AIIA IndexTTS-2] Generated {wav.shape[-1] / sr:.2f}s audio at {sr}Hz") - print(f"[AIIA IndexTTS-2] Generated {wav.shape[-1] / sr:.2f}s audio at {sr}Hz") - - return ({"waveform": wav, "sample_rate": sr},) + return ({"waveform": wav, "sample_rate": sr},) finally: # Cleanup temp files diff --git a/aiia_podcast_stitcher.py b/aiia_podcast_stitcher.py index 6969117..5937907 100755 --- a/aiia_podcast_stitcher.py +++ b/aiia_podcast_stitcher.py @@ -676,10 +676,15 @@ class AIIA_Podcast_Stitcher: duration_A = len(wav_A) / sr_A duration_B = len(wav_B) / sr_B - # 使用统一采样率 + # 使用统一采样率：如果 A 和 B 采样率不同，将 B 重采样到 A 的采样率 sr = sr_A if sr_A != sr_B: - print(f"{log} 警告: sr_A={sr_A} != sr_B={sr_B}, 使用 sr_A") + print(f"{log} 采样率不一致: sr_A={sr_A} != sr_B={sr_B}, 将 B 重采样到 {sr_A}Hz") + import torchaudio + wav_B_tensor = torch.from_numpy(wav_B).unsqueeze(0) # (1, samples) + wav_B_tensor = torchaudio.transforms.Resample(sr_B, sr_A)(wav_B_tensor) + wav_B = wav_B_tensor.squeeze(0).numpy() + sr_B = sr_A # Forced Alignment 模式：对每个说话人做字级强制对齐 fa_results_A = None diff --git a/libs/index-tts/indextts/gpt/transformers_generation_utils.py b/libs/index-tts/indextts/gpt/transformers_generation_utils.py index 4e71b0b..a15792f 100755 --- a/libs/index-tts/indextts/gpt/transformers_generation_utils.py +++ b/libs/index-tts/indextts/gpt/transformers_generation_utils.py @@ -30,9 +30,16 @@ from transformers.cache_utils import ( DynamicCache, EncoderDecoderCache, OffloadedCache, - QuantizedCacheConfig, StaticCache, ) +# Patch for transformers > 4.36 +try: + from transformers.cache_utils import QuantizedCacheConfig +except ImportError: + class QuantizedCacheConfig: + def __init__(self, **kwargs): + pass + from transformers.configuration_utils import PretrainedConfig from transformers.integrations.deepspeed import is_deepspeed_zero3_enabled from transformers.integrations.fsdp import is_fsdp_managed_module @@ -50,21 +57,45 @@ from transformers.utils import ( ) from transformers.generation.beam_constraints import DisjunctiveConstraint, PhrasalConstraint from transformers.generation.beam_search import BeamScorer, BeamSearchScorer, ConstrainedBeamSearchScorer -from transformers.generation.candidate_generator import ( - AssistedCandidateGenerator, - AssistedCandidateGeneratorDifferentTokenizers, - CandidateGenerator, - PromptLookupCandidateGenerator, - _crop_past_key_values, - _prepare_attention_mask, - _prepare_token_type_ids, -) + +# Patch for transformers > 4.49 +try: + from transformers.generation.candidate_generator import ( + AssistedCandidateGenerator, + AssistedCandidateGeneratorDifferentTokenizers, + CandidateGenerator, + PromptLookupCandidateGenerator, + _crop_past_key_values, + _prepare_attention_mask, + _prepare_token_type_ids, + ) +except ImportError: + from transformers.generation.candidate_generator import ( + AssistedCandidateGenerator, + AssistedCandidateGeneratorDifferentTokenizers, + CandidateGenerator, + PromptLookupCandidateGenerator, + _prepare_attention_mask, + _prepare_token_type_ids, + ) + def _crop_past_key_values(model, past_key_values, max_length): + return past_key_values + from transformers.generation.configuration_utils import ( - NEED_SETUP_CACHE_CLASSES_MAPPING, - QUANT_BACKEND_CLASSES_MAPPING, GenerationConfig, GenerationMode, ) + +# Patch for transformers mappings +try: + from transformers.generation.configuration_utils import ( + NEED_SETUP_CACHE_CLASSES_MAPPING, + QUANT_BACKEND_CLASSES_MAPPING, + ) +except ImportError: + NEED_SETUP_CACHE_CLASSES_MAPPING = {} + QUANT_BACKEND_CLASSES_MAPPING = {} + from transformers.generation.logits_process import ( EncoderNoRepeatNGramLogitsProcessor, EncoderRepetitionPenaltyLogitsProcessor, @@ -1002,7 +1033,7 @@ class GenerationMixin: device=device, ) ) - if generation_config.forced_decoder_ids is not None: + if getattr(generation_config, "forced_decoder_ids", None) is not None: # TODO (sanchit): move this exception to GenerationConfig.validate() when TF & FLAX are aligned with PT raise ValueError( "You have explicitly specified `forced_decoder_ids`. Please remove the `forced_decoder_ids` argument " diff --git a/libs/index-tts/indextts/gpt/transformers_gpt2.py b/libs/index-tts/indextts/gpt/transformers_gpt2.py index ab7fa96..4a126d4 100755 --- a/libs/index-tts/indextts/gpt/transformers_gpt2.py +++ b/libs/index-tts/indextts/gpt/transformers_gpt2.py @@ -32,7 +32,66 @@ import transformers from indextts.gpt.transformers_generation_utils import GenerationMixin from indextts.gpt.transformers_modeling_utils import PreTrainedModel -from transformers.modeling_utils import SequenceSummary + +# SequenceSummary locally implemented as it was removed from transformers > 4.40 +class SequenceSummary(nn.Module): + def __init__(self, config): + super().__init__() + + self.summary_type = getattr(config, "summary_type", "last") + if self.summary_type == "cls_index": + self.summary = nn.Identity() + elif self.summary_type == "first": + self.summary = nn.Identity() + elif self.summary_type == "last": + self.summary = nn.Identity() + elif self.summary_type == "mean": + self.summary = nn.Identity() + else: + self.summary = nn.Identity() # Fallback + + if getattr(config, "summary_use_proj", True): + self.summary_proj = nn.Linear(config.hidden_size, config.hidden_size) + else: + self.summary_proj = nn.Identity() + + if getattr(config, "summary_activation", None) == "tanh": + self.activation = nn.Tanh() + else: + self.activation = nn.Identity() + + self.last_dropout = nn.Dropout(getattr(config, "summary_last_dropout", 0.0)) + + def forward(self, hidden_states: torch.FloatTensor, cls_index: Optional[torch.LongTensor] = None) -> torch.FloatTensor: + if self.summary_type == "last": + output = hidden_states[:, -1, :] + elif self.summary_type == "first": + output = hidden_states[:, 0, :] + elif self.summary_type == "mean": + output = hidden_states.mean(dim=1) + elif self.summary_type == "cls_index": + if cls_index is None: + output = hidden_states[:, -1, :] + else: + # cls_index: [batch_size] or [batch_size, num_choices] (if flattened) + if cls_index.dim() > 1: + cls_index = cls_index.flatten() + + batch_size = hidden_states.shape[0] + if cls_index.shape[0] != batch_size: + # Mismatch, fallback to last + output = hidden_states[:, -1, :] + else: + output = hidden_states[torch.arange(batch_size, device=hidden_states.device), cls_index] + else: + output = hidden_states[:, -1, :] + + output = self.summary_proj(output) + output = self.activation(output) + output = self.last_dropout(output) + + return output + from transformers.modeling_attn_mask_utils import _prepare_4d_attention_mask_for_sdpa, _prepare_4d_causal_attention_mask_for_sdpa from transformers.modeling_outputs import ( diff --git a/libs/index-tts/indextts/infer_v2.py b/libs/index-tts/indextts/infer_v2.py index 6c22600..463e4d7 100644 --- a/libs/index-tts/indextts/infer_v2.py +++ b/libs/index-tts/indextts/infer_v2.py @@ -35,6 +35,25 @@ from transformers import SeamlessM4TFeatureExtractor import random import torch.nn.functional as F +# --- Patch for transformers > 4.40 --- +# apply_chunking_to_forward was removed +import transformers.modeling_utils +if not hasattr(transformers.modeling_utils, "apply_chunking_to_forward"): + def _chunking(forward_fn, chunk_size, *tensors, **kw): + return forward_fn(*tensors, **kw) + transformers.modeling_utils.apply_chunking_to_forward = _chunking + +# GenerationConfig cleanup for newer transformers +from transformers import GenerationConfig +# Remove deprecated attributes if they exist to prevent warnings/errors +if hasattr(GenerationConfig, "forced_decoder_ids"): + pass # valid +# But sometimes we might need to remove it from kwargs if passed inappropriately? +# The error "forced_decoder_ids is deprecated" usually comes from passing it to __init__ +# We can't easily patch the class __init__, but we can patch where it's used if we find it. +# For now, the apply_chunking_to_forward is the critical missing piece for compilation/startup. +# ------------------------------------- + class IndexTTS2: def __init__( self, cfg_path="checkpoints/config.yaml", model_dir="checkpoints", use_fp16=False, device=None, @@ -164,9 +183,19 @@ class IndexTTS2: self.bigvgan = self.bigvgan.to(self.device) self.bigvgan.remove_weight_norm() self.bigvgan.eval() - print(">> bigvgan weights restored from:", bigvgan_name) + try: + v_cfg = transformers.AutoConfig.from_pretrained(bigvgan_name, trust_remote_code=True) + if hasattr(v_cfg, "sampling_rate"): + self.sampling_rate = v_cfg.sampling_rate + else: + self.sampling_rate = 22050 + except Exception as e: + print(f">> Failed to load BigVGAN config for sampling_rate: {e}, using default 22050") + self.sampling_rate = 22050 + print(f"Loaded BigVGAN sampling_rate: {self.sampling_rate}") self.bpe_path = os.path.join(self.model_dir, self.cfg.dataset["bpe_model"]) + self.normalizer = TextNormalizer(enable_glossary=True) self.normalizer.load() print(">> TextNormalizer loaded") @@ -325,7 +354,10 @@ class IndexTTS2: def _load_and_cut_audio(self,audio_path,max_audio_length_seconds,verbose=False,sr=None): if not sr: - audio, sr = librosa.load(audio_path) + # Use native sampling rate! Do not let librosa default to 22050 implicitly. + audio, sr = librosa.load(audio_path, sr=None) + if verbose: + print(f">> Loaded audio {audio_path} at native SR: {sr}Hz") else: audio, _ = librosa.load(audio_path,sr=sr) audio = torch.tensor(audio).unsqueeze(0) @@ -524,7 +556,7 @@ class IndexTTS2: num_beams = generation_kwargs.pop("num_beams", 3) repetition_penalty = generation_kwargs.pop("repetition_penalty", 10.0) max_mel_tokens = generation_kwargs.pop("max_mel_tokens", 1500) - sampling_rate = 22050 + sampling_rate = self.sampling_rate wavs = [] gpt_gen_time = 0 @@ -686,7 +718,15 @@ class IndexTTS2: print(f">> RTF: {(end_time - start_time) / wav_length:.4f}") # save audio - wav = wav.cpu() # to cpu + wav = wav.cpu().float() # ensure cpu float + + # Normalize to prevent clipping (fixes distortion) + # User requested Float32 to avoid precision loss. + max_val = wav.abs().max() + if max_val > 0.99: + wav = wav / max_val * 0.99 + + # Use simple float saving for max quality if output_path: # 直接保存音频到指定路径中 if os.path.isfile(output_path): @@ -694,7 +734,8 @@ class IndexTTS2: print(">> remove old wav file:", output_path) if os.path.dirname(output_path) != "": os.makedirs(os.path.dirname(output_path), exist_ok=True) - torchaudio.save(output_path, wav.float(), sampling_rate) + + torchaudio.save(output_path, wav, sampling_rate) print(">> wav file saved to:", output_path) if stream_return: return None @@ -703,7 +744,7 @@ class IndexTTS2: if stream_return: return None # 返回以符合Gradio的格式要求 - wav_data = (wav * 32767).clamp(-32767, 32767).type(torch.int16) + wav_data = (wav * 32767).to(torch.int16) wav_data = wav_data.numpy().T yield (sampling_rate, wav_data) diff --git a/pyproject.toml b/pyproject.toml index 3319e30..7d85842 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "aiia" description = "Advanced AI Audio/Video toolkit for ComfyUI. Features Multi-Role Podcast/Dialogue Generation, High-Fidelity Voice Cloning (CosyVoice/VibeVoice), TTS, Media Management, and efficient Video tools." -version = "1.14.3" +version = "1.15.0" license = {file = "LICENSE"} readme = "README.md" authors = [ diff --git a/test_strict_indextts.py b/test_strict_indextts.py index 636ba93..121bc7b 100644 --- a/test_strict_indextts.py +++ b/test_strict_indextts.py @@ -164,24 +164,30 @@ for use_fp16 in [True, False]: print("No wavs generated via stream!") continue + # The model output in stream_return is now raw float (normalized internally in infer_v2 if we used that path, + # but here we get the raw tensor from generator. + full_wav = torch.cat(wavs, dim=1) if len(wavs) > 1 else wavs[0] - # Save as float (normalized) - # The model output seems to be already scaled to 32767 in `infer_v2.py` line 664? - # wav = torch.clamp(32767 * wav, ...) - # Wait, `infer_v2.py` does that inside the loop. - # So `wavs` collected here are ALREADY multiplied by 32767. + # infer_v2.py normalization happens inside the `if output_path` block or yielded. + # Wait, if we use `stream_return`, `infer_v2` yields: + # 1. `wav.cpu().float()` (raw chunk) + # 2. `(sampling_rate, wav_data_numpy)` (int16 version for Gradio) - # So we should divide by 32768.0 to get back to float [-1, 1] - full_wav_float = full_wav.float() / 32768.0 + # We collected raw chunks. They are NOT normalized if they come from the loop `yield wav.cpu()`. + # We need to normalize them manually here to MATCH what infer_v2 does when saving file. + + full_wav_float = full_wav.float() + max_val = full_wav_float.abs().max() + if max_val > 0.99: + full_wav_float = full_wav_float / max_val * 0.99 import torchaudio - torchaudio.save(output_path.replace(".wav", "_norm.wav"), full_wav_float, 24000) - print(f"Saved normalized float wav: {output_path.replace('.wav', '_norm.wav')}") - - # Also save as int16 as original - torchaudio.save(output_path, full_wav.to(torch.int16), 24000) - print(f"Saved int16 wav: {output_path}") + # Save as float32 (User requested format) + torchaudio.save(output_path, full_wav_float, 22050) # We assume 22050 for test or explicit SR? + # Ideally we should use the model's SR, but we can't easily access tts.sampling_rate here without more code. + # Let's assume 22050 as that is what we expect BigVGAN to be. + print(f"Saved float32 wav: {output_path}") except Exception as e: print(f"Inference failed: {e}")