Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 diodiogod
|
3a817a2526 | ||
|
diodiogod
|
78d3e867ef | ||
|
diodiogod
|
db3a888b77 | ||
|
diodiogod
|
1f5882ba44 | ||
|
diodiogod
|
e471cdbaa6 | ||
|
diodiogod
|
cb6dfe3a11 |
@@ -5,6 +5,17 @@ All notable changes to this project will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [5.1.0] - 2026-06-15
|
||||
|
||||
### Added
|
||||
|
||||
- Add Dots TTS engine with multilingual cloning support
|
||||
- Add official Dots TTS engine for text and SRT generation
|
||||
- Refine language behavior, Portuguese metadata, and Dots engine documentation
|
||||
|
||||
### Changed
|
||||
|
||||
- Improve model unload, caching, and generation progress feedback
|
||||
## [5.0.0] - 2026-06-14
|
||||
|
||||
### Added
|
||||
|
||||
@@ -42,6 +42,7 @@ The project code is MIT. Model weights carry their own licenses:
|
||||
Granite ASR Apache-2.0 Yes
|
||||
Step Audio EditX Apache-2.0 (verify before commercial use) Conditional
|
||||
Echo-TTS CC-BY-NC-SA-4.0 No
|
||||
Dots TTS Apache-2.0 Yes
|
||||
MOSS-TTS Apache-2.0 Yes
|
||||
RVC MIT (framework); community models vary Varies
|
||||
──────────────── ──────────────────────────────────────────────────────── ──────────────
|
||||
|
||||
+5
-5
@@ -96,11 +96,11 @@
|
||||
|
||||
## Node Implementations
|
||||
|
||||
### Unified Interface
|
||||
- `nodes/unified/tts_text_node.py` - Universal TTS text node
|
||||
- `nodes/unified/tts_srt_node.py` - Universal SRT node
|
||||
- `nodes/unified/voice_changer_node.py` - Universal voice conversion
|
||||
- `nodes/unified/asr_transcribe_node.py` - Universal ASR node
|
||||
### Unified Interface
|
||||
- `nodes/unified/tts_text_node.py` - Universal TTS text node
|
||||
- `nodes/unified/tts_srt_node.py` - Universal SRT node
|
||||
- `nodes/unified/voice_changer_node.py` - Universal voice conversion
|
||||
- `nodes/unified/asr_transcribe_node.py` - Universal ASR node
|
||||
|
||||
### Shared / Special Nodes
|
||||
- `nodes/shared/character_voices_node.py` - Character voice management (NARRATOR_VOICE output)
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
[![Dynamic TOML Badge][version-shield]][version-url]
|
||||
[](https://ko-fi.com/diogogo)
|
||||
|
||||
# TTS Audio Suite v5.0.0
|
||||
# TTS Audio Suite v5.1.0
|
||||
|
||||
[](https://ko-fi.com/diogogo)
|
||||
|
||||
@@ -23,7 +23,7 @@ Subtitle workflows are still a core focus: the suite can transcribe to SRT, rebu
|
||||
|
||||
<!-- ENGINE_COMPARISON_START -->
|
||||
|
||||
## Quick Engine Comparison — 14 Engines
|
||||
## Quick Engine Comparison — 15 Engines
|
||||
|
||||
| Engine | Languages | Size | Key Features |
|
||||
|--------|-----------|------|--------------|
|
||||
@@ -39,6 +39,7 @@ Subtitle workflows are still a core focus: the suite can transcribe to SRT, rebu
|
||||
| **Granite ASR** | 🇺🇸🇩🇪🇪🇸🇫🇷🇯🇵🇵🇹 | ~4.6GB | ASR (Automatic Speech Recognition), Custom timestamps/SRT via reused Qwen forced aligner |
|
||||
| **Step Audio EditX** | 🇺🇸🇨🇳🇯🇵🇰🇷 | ~7GB | Second Pass Speech Editing Node: 14 emotions, 32 speaking styles |
|
||||
| **Echo-TTS** | 🇺🇸 | ~5.3GB + ~1.8GB | Diffusion-based (~30s best), Force Speaker KV (speaker drift control) |
|
||||
| **Dots TTS** | 🇺🇸🇨🇳🇩🇪🇪🇸🇫🇷🇮🇹 +13 | ~6GB | Official auto language detect / language control, SOAR and MeanFlow distilled variants |
|
||||
| **MOSS-TTS** | 🇺🇸🇨🇳🇩🇪🇪🇸🇫🇷🇮🇹 +10 | ~8.5GB tokenizer + ~6.1GB/17GB/18GB model | 20-language generation, Long-form generation (TTSD/Delay) |
|
||||
| **RVC** | 🌐 Any | 100-300MB | Real-time VC, Integrated training workflow |
|
||||
|
||||
@@ -1432,6 +1433,7 @@ For offline/manual setup:
|
||||
| MOSS-TTS | `ComfyUI/models/TTS/moss_tts/` | ✅ | Local/Delay/TTSD models plus shared MOSS-Audio-Tokenizer codec |
|
||||
| Granite ASR | `ComfyUI/models/TTS/granite_asr/` | ✅ | Main Granite model; optional Qwen forced aligner reused lazily for timestamps/SRT |
|
||||
| Echo-TTS | `ComfyUI/models/TTS/echo-tts-base/` | ✅ | ~7.1GB total (base + dac); CC-BY-NC-SA |
|
||||
| Dots TTS | `ComfyUI/models/TTS/dots_tts/` | ✅ | Official base / soar / mf checkpoints with tokenizer, vocoder, speaker encoder |
|
||||
|
||||
*Generated from [tts_audio_suite_engines.yaml](docs/Dev%20reports/tts_audio_suite_engines.yaml).*
|
||||
|
||||
|
||||
@@ -1142,6 +1142,84 @@ engines:
|
||||
vram_efficient: { supported: "partial", notes: "(~7GB total)" }
|
||||
speed_performance: { supported: true, notes: "Fast (diffusion, realtime-capable)" }
|
||||
|
||||
- id: dots-tts
|
||||
name: Dots TTS
|
||||
models: "dots.tts-base, dots.tts-soar, dots.tts-mf"
|
||||
size: "~6GB"
|
||||
license: "Apache-2.0"
|
||||
commercial: true
|
||||
|
||||
capabilities:
|
||||
tts: true
|
||||
srt: true
|
||||
vc: false
|
||||
asr: false
|
||||
training: false
|
||||
|
||||
special_features:
|
||||
- "Official auto language detect / language control"
|
||||
- "SOAR and MeanFlow distilled variants"
|
||||
|
||||
model_sources:
|
||||
- component: "dots.tts-base"
|
||||
source_name: "rednote-hilab/dots.tts-base"
|
||||
source_url: "https://huggingface.co/rednote-hilab/dots.tts-base"
|
||||
size: "~6GB"
|
||||
auto_download: true
|
||||
notes: "Official base checkpoint with tokenizer, vocoder, speaker encoder, and latent stats"
|
||||
- component: "dots.tts-soar"
|
||||
source_name: "rednote-hilab/dots.tts-soar"
|
||||
source_url: "https://huggingface.co/rednote-hilab/dots.tts-soar"
|
||||
size: "~6GB"
|
||||
auto_download: true
|
||||
notes: "Official SOAR checkpoint for higher-quality zero-shot cloning"
|
||||
- component: "dots.tts-mf"
|
||||
source_name: "rednote-hilab/dots.tts-mf"
|
||||
source_url: "https://huggingface.co/rednote-hilab/dots.tts-mf"
|
||||
size: "~6GB"
|
||||
auto_download: true
|
||||
notes: "Official MeanFlow-distilled checkpoint for faster inference"
|
||||
|
||||
languages:
|
||||
en: { supported: true, flag: "🇺🇸", notes: "" }
|
||||
zh: { supported: true, flag: "🇨🇳", notes: "(Mandarin; official also exposes YUE separately outside this matrix)" }
|
||||
de: { supported: true, flag: "🇩🇪", notes: "" }
|
||||
es: { supported: true, flag: "🇪🇸", notes: "" }
|
||||
fr: { supported: true, flag: "🇫🇷", notes: "" }
|
||||
it: { supported: true, flag: "🇮🇹", notes: "" }
|
||||
ja: { supported: true, flag: "🇯🇵", notes: "" }
|
||||
ko: { supported: true, flag: "🇰🇷", notes: "" }
|
||||
ru: { supported: true, flag: "🇷🇺", notes: "" }
|
||||
pt: { supported: true, flag: "🇵🇹", notes: "(Official PT tag is generic; upstream does not expose separate PT-BR/PT-PT tags and it may lean more European Portuguese than Brazilian Portuguese)" }
|
||||
pl: { supported: true, flag: "🇵🇱", notes: "" }
|
||||
hi: { supported: true, flag: "🇮🇳", notes: "" }
|
||||
ar: { supported: true, flag: "🇦🇪", notes: "" }
|
||||
tr: { supported: true, flag: "🇹🇷", notes: "" }
|
||||
th: { supported: true, flag: "🇹🇭", notes: "" }
|
||||
no: { supported: false, flag: "🇳🇴", notes: "" }
|
||||
vi: { supported: true, flag: "🇻🇳", notes: "" }
|
||||
hy: { supported: false, flag: "🇦🇲", notes: "" }
|
||||
ka: { supported: false, flag: "🇬🇪", notes: "" }
|
||||
da: { supported: false, flag: "🇩🇰", notes: "" }
|
||||
fi: { supported: true, flag: "🇫🇮", notes: "" }
|
||||
el: { supported: true, flag: "🇬🇷", notes: "" }
|
||||
he: { supported: false, flag: "🇮🇱", notes: "" }
|
||||
ms: { supported: false, flag: "🇲🇾", notes: "" }
|
||||
nl: { supported: true, flag: "🇳🇱", notes: "" }
|
||||
sv: { supported: false, flag: "🇸🇪", notes: "" }
|
||||
sw: { supported: false, flag: "🇰🇪", notes: "" }
|
||||
|
||||
features:
|
||||
voice_cloning: { supported: true, notes: "" }
|
||||
native_multi_speaker: { supported: false, notes: "" }
|
||||
voice_conversion: { supported: false, notes: "" }
|
||||
asr_transcribe: { supported: false, notes: "" }
|
||||
emotion_control: { supported: false, notes: "" }
|
||||
native_long_form: { supported: false, notes: "(use suite chunking)" }
|
||||
community_finetunes: { supported: false, notes: "" }
|
||||
vram_efficient: { supported: "partial", notes: "(main env works; 2B-class model is not lightweight)" }
|
||||
speed_performance: { supported: "partial", notes: "Moderate; mf variant is faster" }
|
||||
|
||||
- id: moss-tts
|
||||
name: MOSS-TTS
|
||||
models: "Local 1.7B, Delay 8B, TTSD 8B"
|
||||
@@ -1508,6 +1586,10 @@ readme_model_download_table:
|
||||
primary_model_path: "ComfyUI/models/TTS/echo-tts-base/"
|
||||
auto_download: "✅"
|
||||
notes: "~7.1GB total (base + dac); CC-BY-NC-SA"
|
||||
- engine: "Dots TTS"
|
||||
primary_model_path: "ComfyUI/models/TTS/dots_tts/"
|
||||
auto_download: "✅"
|
||||
notes: "Official base / soar / mf checkpoints with tokenizer, vocoder, speaker encoder"
|
||||
|
||||
model_layouts_markdown: |
|
||||
# Model Folder Layouts
|
||||
@@ -1826,3 +1908,37 @@ model_layouts_markdown: |
|
||||
|
||||
- Both components are required and auto-downloaded on first use.
|
||||
- License: CC-BY-NC-SA (non-commercial).
|
||||
|
||||
## Dots TTS
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/dots_tts/
|
||||
├── dots.tts-base/
|
||||
├── dots.tts-soar/
|
||||
└── dots.tts-mf/
|
||||
```
|
||||
|
||||
Typical checkpoint contents:
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/dots_tts/dots.tts-soar/
|
||||
├── added_tokens.json
|
||||
├── chat_template.jinja
|
||||
├── config.json
|
||||
├── latent_stats.pt
|
||||
├── llm_config.json
|
||||
├── merges.txt
|
||||
├── model.safetensors
|
||||
├── speaker_encoder.safetensors
|
||||
├── special_tokens_map.json
|
||||
├── tokenizer.json
|
||||
├── tokenizer_config.json
|
||||
├── vocab.json
|
||||
└── vocoder.safetensors
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- Downloads into a dedicated `dots_tts/` folder.
|
||||
- Native sample rate is 48kHz.
|
||||
- Main-environment support works on Transformers 5; on Windows, `normalize_text` falls back to no-op if `WeTextProcessing` is unavailable.
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
| **Granite ASR** | Main | granite-4.0-1b-speech | ~4.6GB | ❌ | ✅ | ❌ | ✅ | ❌ | Apache-2.0 | ASR (Automatic Speech Recognition), Custom timestamps/SRT via reused Qwen forced aligner, Speech translation (experimental), Optional forced aligner auto-routed through shared legacy T4 runtime | 6 |
|
||||
| **Step Audio EditX** | Main | 3B LLM + CosyVoice | ~7GB | ✅ | ✅ | ❌ | ❌ | ❌ | Apache-2.0 (verify before commercial use) | Second Pass Speech Editing Node: 14 emotions, 32 speaking styles, Paralinguistic effects | 4 |
|
||||
| **Echo-TTS** | Main | echo-tts-base + fish-s1-dac-min | ~5.3GB + ~1.8GB | ✅ | ✅ | ❌ | ❌ | ❌ | CC-BY-NC-SA-4.0 | Diffusion-based (~30s best), Force Speaker KV (speaker drift control) | 1 |
|
||||
| **Dots TTS** | Main | dots.tts-base, dots.tts-soar, dots.tts-mf | ~6GB | ✅ | ✅ | ❌ | ❌ | ❌ | Apache-2.0 | Official auto language detect / language control, SOAR and MeanFlow distilled variants | 19 |
|
||||
| **MOSS-TTS** | Main | Local 1.7B, Delay 8B, TTSD 8B | ~8.5GB tokenizer + ~6.1GB/17GB/18GB model | ✅ | ✅ | ❌ | ❌ | ✅ | Apache-2.0 | 20-language generation, Long-form generation (TTSD/Delay), Native 1-5 speaker TTSD dialogue, Duration token hint, Local/Delay/TTSD variants, Initial integrated LoRA training workflow (Delay 8B) | 16 |
|
||||
| **RVC** | Main | Community .pth | 100-300MB | ❌ | ❌ | ✅ | ❌ | ✅ | MIT (framework); community models vary | Real-time VC, Integrated training workflow, Pitch shift (±14), 6 HuBERT models, Language-independent | Any |
|
||||
|
||||
|
||||
+14
-14
@@ -2,17 +2,17 @@
|
||||
|
||||
## Feature Comparison Matrix
|
||||
|
||||
| Feature | F5-TTS | ChatterBox | ChatterBox 23L | VibeVoice | Higgs Audio 2 | Higgs Audio v3 | IndexTTS-2 | CosyVoice3 | Qwen3-TTS | Granite ASR | Step Audio EditX | Echo-TTS | MOSS-TTS | RVC |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| **TTS** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ |
|
||||
| **SRT** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ |
|
||||
| **Voice Conversion** | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| **ASR (Transcribe)** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| **Training** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ |
|
||||
| **Voice Cloning** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (Base model) | ❌ | ✅ | ✅ | ✅ | ⚠️ (needs training) |
|
||||
| **Native Multi-Speaker** | ❌ | ❌ | ❌ | ✅ (Base only, Kugel uses fallback) | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (TTSD v1.0; 1-5 speakers) | ❌ |
|
||||
| **Emotion Control** | ❌ | ❌ | ⚠️ (v2 tags - doesn't work) | ❌ | ⚠️ (via prompt) | ✅ (native inline tags) | ✅ (8 emotions) | ⚠️ (via instruct) | ⚠️ (via instruct) | ❌ | ✅ (14 emotions) | ❌ | ❌ | ❌ |
|
||||
| **Native Long-form** | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (TTSD/Delay long-form; use chunk orchestration for very long inputs) | N/A |
|
||||
| **Community Finetunes** | ✅ | ✅ | ✅ | ✅ KugelAudio, Hindi | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (LoRA adapter inference supported; initial integrated LoRA training support added for MOSS-TTS Delay 8B) | ✅ |
|
||||
| **VRAM Efficient** | ✅ | ✅ | ✅ | ⚠️ (5-18GB) | ⚠️ (9GB) | ⚠️ (~8-10GB) | ⚠️ (9-12GB) | ✅ (5.4GB) | ✅ (3-6GB) | ✅ (~4.6GB) | ⚠️ (7GB) | ⚠️ (~7GB total) | ⚠️ (Local 1.7B smaller; tokenizer is large) | ✅ |
|
||||
| **Speed/Performance** | ✅ Very Fast | ✅ Fast | ✅ Fast | ⚠️ | ⚠️ | ⚠️ CUDA recommended | ⚠️ | ✅ Fast | ⚠️ | ⚠️ Moderate | ⚠️ | ✅ Fast (diffusion, realtime-capable) | ✅ Fast with CUDA/FlashAttention | ✅ Fast |
|
||||
| Feature | F5-TTS | ChatterBox | ChatterBox 23L | VibeVoice | Higgs Audio 2 | Higgs Audio v3 | IndexTTS-2 | CosyVoice3 | Qwen3-TTS | Granite ASR | Step Audio EditX | Echo-TTS | Dots TTS | MOSS-TTS | RVC |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| **TTS** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ |
|
||||
| **SRT** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ |
|
||||
| **Voice Conversion** | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| **ASR (Transcribe)** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||
| **Training** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ |
|
||||
| **Voice Cloning** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (Base model) | ❌ | ✅ | ✅ | ✅ | ✅ | ⚠️ (needs training) |
|
||||
| **Native Multi-Speaker** | ❌ | ❌ | ❌ | ✅ (Base only, Kugel uses fallback) | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (TTSD v1.0; 1-5 speakers) | ❌ |
|
||||
| **Emotion Control** | ❌ | ❌ | ⚠️ (v2 tags - doesn't work) | ❌ | ⚠️ (via prompt) | ✅ (native inline tags) | ✅ (8 emotions) | ⚠️ (via instruct) | ⚠️ (via instruct) | ❌ | ✅ (14 emotions) | ❌ | ❌ | ❌ | ❌ |
|
||||
| **Native Long-form** | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (TTSD/Delay long-form; use chunk orchestration for very long inputs) | N/A |
|
||||
| **Community Finetunes** | ✅ | ✅ | ✅ | ✅ KugelAudio, Hindi | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (LoRA adapter inference supported; initial integrated LoRA training support added for MOSS-TTS Delay 8B) | ✅ |
|
||||
| **VRAM Efficient** | ✅ | ✅ | ✅ | ⚠️ (5-18GB) | ⚠️ (9GB) | ⚠️ (~8-10GB) | ⚠️ (9-12GB) | ✅ (5.4GB) | ✅ (3-6GB) | ✅ (~4.6GB) | ⚠️ (7GB) | ⚠️ (~7GB total) | ⚠️ (main env works; 2B-class model is not lightweight) | ⚠️ (Local 1.7B smaller; tokenizer is large) | ✅ |
|
||||
| **Speed/Performance** | ✅ Very Fast | ✅ Fast | ✅ Fast | ⚠️ | ⚠️ | ⚠️ CUDA recommended | ⚠️ | ✅ Fast | ⚠️ | ⚠️ Moderate | ⚠️ | ✅ Fast (diffusion, realtime-capable) | ⚠️ Moderate; mf variant is faster | ✅ Fast with CUDA/FlashAttention | ✅ Fast |
|
||||
+104
-104
@@ -2,110 +2,110 @@
|
||||
|
||||
## Language Support by Engine
|
||||
|
||||
| Language | Code | F5-TTS | ChatterBox | ChatterBox 23L | VibeVoice | Higgs Audio 2 | Higgs Audio v3 | IndexTTS-2 | CosyVoice3 | Qwen3-TTS | Granite ASR | Step Audio EditX | Echo-TTS | MOSS-TTS | RVC |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| 🇺🇸 **English** | EN | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| 🇨🇳 **Chinese** | ZH | ❌ | ❌ | ✅ | ✅ | ✅ (Mandarin) | ✅ | ✅ | ✅ + 18 dialects | ✅ | ❌ | ✅ (Mandarin + Sichuanese, Cantonese) | ❌ | ✅ | ✅ |
|
||||
| 🇩🇪 **German** | DE | ✅ | ✅ (×3) | ✅ | ✅ (Kugel) | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ |
|
||||
| 🇪🇸 **Spanish** | ES | ✅ | ❌ | ✅ | ✅ (Kugel) | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ |
|
||||
| 🇫🇷 **French** | FR | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ |
|
||||
| 🇮🇹 **Italian** | IT | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ |
|
||||
| 🇯🇵 **Japanese** | JA | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ✅ ? | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ |
|
||||
| 🇰🇷 **Korean** | KO | ❌ | ✅ | ✅ | ✅ (Kugel) | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ |
|
||||
| 🇷🇺 **Russian** | RU | ❌ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ |
|
||||
| 🇧🇷 **Portuguese** | PT | ✅ (BR) | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ (EU/BR*) | ✅ | ❌ | ❌ | ✅ | ✅ |
|
||||
| 🇵🇱 **Polish** | PL | ✅ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ |
|
||||
| 🇮🇳 **Hindi** | HI | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇪🇬 **Arabic** | AR | ❌ | ❌ | ✅ (Egyptian) | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ |
|
||||
| 🇹🇷 **Turkish** | TR | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ |
|
||||
| 🇹🇭 **Thai** | TH | ✅ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇳🇴 **Norwegian** | NO | ❌ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇻🇳 **Vietnamese** | VI | ❌ | ❌ | ✅ (Viterbox) | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇦🇲 **Armenian** | HY | ❌ | ✅ | ❌ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇬🇪 **Georgian** | KA | ❌ | ✅ | ❌ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇩🇰 **Danish** | DA | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ |
|
||||
| 🇫🇮 **Finnish** | FI | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇬🇷 **Greek** | EL | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ |
|
||||
| 🇮🇱 **Hebrew** | HE | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇲🇾 **Malay** | MS | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇳🇱 **Dutch** | NL | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇸🇪 **Swedish** | SV | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ |
|
||||
| 🇰🇪 **Swahili** | SW | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇿🇦 **Afrikaans** | AF | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇦🇱 **Albanian** | SQ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇳 **Assamese** | AS | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇪🇸 **Asturian** | AST | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇦🇿 **Azerbaijani** | AZ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇷🇺 **Bashkir** | BA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇪🇸 **Basque** | EU | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇧🇾 **Belarusian** | BE | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇧🇩 **Bengali** | BN | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇧🇦 **Bosnian** | BS | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇧🇬 **Bulgarian** | BG | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇪🇸 **Catalan** | CA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇵🇭 **Cebuano** | CEB | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇶 **Central Kurdish** | CKB | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇭🇷 **Croatian** | HR | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇨🇿 **Czech** | CS | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇷🇺 **Eastern Mari** | MHR | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🌍 **Esperanto** | EO | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇪🇪 **Estonian** | ET | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇪🇸 **Galician** | GL | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇳 **Gujarati** | GU | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇭🇹 **Haitian Creole** | HT | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇳🇬 **Hausa** | HA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇭🇺 **Hungarian** | HU | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇩 **Indonesian** | ID | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇩 **Javanese** | JV | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇳 **Kannada** | KN | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇰🇿 **Kazakh** | KK | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇷🇼 **Kinyarwanda** | RW | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇰🇬 **Kyrgyz** | KY | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇱🇻 **Latvian** | LV | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇨🇩 **Lingala** | LN | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇱🇹 **Lithuanian** | LT | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇰🇪 **Luo** | LUO | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇲🇰 **Macedonian** | MK | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇳 **Malayalam** | ML | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇲🇹 **Maltese** | MT | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇳🇿 **Māori** | MI | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇳 **Marathi** | MR | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇲🇳 **Mongolian** | MN | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇳🇵 **Nepali** | NE | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇫🇷 **Occitan** | OC | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇷 **Persian** | FA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇷🇴 **Romanian** | RO | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇿🇦 **Sepedi** | NSO | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇷🇸 **Serbian** | SR | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇿🇼 **Shona** | SN | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇸🇰 **Slovak** | SK | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇸🇮 **Slovene** | SL | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇵🇭 **Tagalog** | TL | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇹🇯 **Tajik** | TG | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇳 **Tamil** | TA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇳 **Telugu** | TE | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇺🇦 **Ukrainian** | UK | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇵🇰 **Urdu** | UR | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇨🇳 **Uyghur** | UG | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇺🇿 **Uzbek** | UZ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇿🇦 **Xhosa** | XH | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇿🇦 **Zulu** | ZU | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇲🇼 **Chichewa/Nyanja** | NY | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇳 **Eastern Punjabi** | PA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇺🇬 **Ganda** | LG | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇸 **Icelandic** | IS | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇪 **Irish** | GA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇩🇿 **Kabyle** | KAB | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇨🇻 **Kabuverdianu** | KEA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇰🇪 **Kamba** | KAM | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇻🇦 **Latin** | LA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇱🇺 **Luxembourgish** | LB | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇪🇹 **Oromo** | OM | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇦🇫 **Pashto** | PS | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇵🇰 **Sindhi** | SD | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇸🇴 **Somali** | SO | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇦🇴 **Umbundu** | UMB | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🏴 **Welsh** | CY | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| Language | Code | F5-TTS | ChatterBox | ChatterBox 23L | VibeVoice | Higgs Audio 2 | Higgs Audio v3 | IndexTTS-2 | CosyVoice3 | Qwen3-TTS | Granite ASR | Step Audio EditX | Echo-TTS | Dots TTS | MOSS-TTS | RVC |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| 🇺🇸 **English** | EN | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| 🇨🇳 **Chinese** | ZH | ❌ | ❌ | ✅ | ✅ | ✅ (Mandarin) | ✅ | ✅ | ✅ + 18 dialects | ✅ | ❌ | ✅ (Mandarin + Sichuanese, Cantonese) | ❌ | ✅ (Mandarin; official also exposes YUE separately outside this matrix) | ✅ | ✅ |
|
||||
| 🇩🇪 **German** | DE | ✅ | ✅ (×3) | ✅ | ✅ (Kugel) | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ |
|
||||
| 🇪🇸 **Spanish** | ES | ✅ | ❌ | ✅ | ✅ (Kugel) | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ |
|
||||
| 🇫🇷 **French** | FR | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ |
|
||||
| 🇮🇹 **Italian** | IT | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ |
|
||||
| 🇯🇵 **Japanese** | JA | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ✅ ? | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ |
|
||||
| 🇰🇷 **Korean** | KO | ❌ | ✅ | ✅ | ✅ (Kugel) | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ |
|
||||
| 🇷🇺 **Russian** | RU | ❌ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ |
|
||||
| 🇧🇷 **Portuguese** | PT | ✅ (BR) | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ (EU/BR*) | ✅ | ❌ | ❌ | ✅ (Official PT tag is generic; upstream does not expose separate PT-BR/PT-PT tags and it may lean more European Portuguese than Brazilian Portuguese) | ✅ | ✅ |
|
||||
| 🇵🇱 **Polish** | PL | ✅ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ |
|
||||
| 🇮🇳 **Hindi** | HI | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ |
|
||||
| 🇪🇬 **Arabic** | AR | ❌ | ❌ | ✅ (Egyptian) | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ |
|
||||
| 🇹🇷 **Turkish** | TR | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ |
|
||||
| 🇹🇭 **Thai** | TH | ✅ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ |
|
||||
| 🇳🇴 **Norwegian** | NO | ❌ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇻🇳 **Vietnamese** | VI | ❌ | ❌ | ✅ (Viterbox) | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ |
|
||||
| 🇦🇲 **Armenian** | HY | ❌ | ✅ | ❌ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇬🇪 **Georgian** | KA | ❌ | ✅ | ❌ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇩🇰 **Danish** | DA | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ |
|
||||
| 🇫🇮 **Finnish** | FI | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ |
|
||||
| 🇬🇷 **Greek** | EL | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ |
|
||||
| 🇮🇱 **Hebrew** | HE | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇲🇾 **Malay** | MS | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇳🇱 **Dutch** | NL | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ |
|
||||
| 🇸🇪 **Swedish** | SV | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ |
|
||||
| 🇰🇪 **Swahili** | SW | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇿🇦 **Afrikaans** | AF | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇦🇱 **Albanian** | SQ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇳 **Assamese** | AS | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇪🇸 **Asturian** | AST | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇦🇿 **Azerbaijani** | AZ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇷🇺 **Bashkir** | BA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇪🇸 **Basque** | EU | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇧🇾 **Belarusian** | BE | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇧🇩 **Bengali** | BN | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇧🇦 **Bosnian** | BS | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇧🇬 **Bulgarian** | BG | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇪🇸 **Catalan** | CA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇵🇭 **Cebuano** | CEB | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇶 **Central Kurdish** | CKB | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇭🇷 **Croatian** | HR | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇨🇿 **Czech** | CS | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇷🇺 **Eastern Mari** | MHR | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🌍 **Esperanto** | EO | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇪🇪 **Estonian** | ET | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇪🇸 **Galician** | GL | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇳 **Gujarati** | GU | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇭🇹 **Haitian Creole** | HT | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇳🇬 **Hausa** | HA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇭🇺 **Hungarian** | HU | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇩 **Indonesian** | ID | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇩 **Javanese** | JV | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇳 **Kannada** | KN | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇰🇿 **Kazakh** | KK | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇷🇼 **Kinyarwanda** | RW | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇰🇬 **Kyrgyz** | KY | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇱🇻 **Latvian** | LV | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇨🇩 **Lingala** | LN | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇱🇹 **Lithuanian** | LT | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇰🇪 **Luo** | LUO | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇲🇰 **Macedonian** | MK | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇳 **Malayalam** | ML | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇲🇹 **Maltese** | MT | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇳🇿 **Māori** | MI | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇳 **Marathi** | MR | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇲🇳 **Mongolian** | MN | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇳🇵 **Nepali** | NE | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇫🇷 **Occitan** | OC | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇷 **Persian** | FA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇷🇴 **Romanian** | RO | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇿🇦 **Sepedi** | NSO | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇷🇸 **Serbian** | SR | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇿🇼 **Shona** | SN | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇸🇰 **Slovak** | SK | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇸🇮 **Slovene** | SL | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇵🇭 **Tagalog** | TL | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇹🇯 **Tajik** | TG | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇳 **Tamil** | TA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇳 **Telugu** | TE | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇺🇦 **Ukrainian** | UK | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇵🇰 **Urdu** | UR | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇨🇳 **Uyghur** | UG | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇺🇿 **Uzbek** | UZ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇿🇦 **Xhosa** | XH | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇿🇦 **Zulu** | ZU | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇲🇼 **Chichewa/Nyanja** | NY | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇳 **Eastern Punjabi** | PA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇺🇬 **Ganda** | LG | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇸 **Icelandic** | IS | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇪 **Irish** | GA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇩🇿 **Kabyle** | KAB | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇨🇻 **Kabuverdianu** | KEA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇰🇪 **Kamba** | KAM | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇻🇦 **Latin** | LA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇱🇺 **Luxembourgish** | LB | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇪🇹 **Oromo** | OM | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇦🇫 **Pashto** | PS | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇵🇰 **Sindhi** | SD | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇸🇴 **Somali** | SO | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇦🇴 **Umbundu** | UMB | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🏴 **Welsh** | CY | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
|
||||
**Notes:**
|
||||
|
||||
|
||||
@@ -112,6 +112,14 @@ Use this as the canonical list of model repositories/links for offline setup.
|
||||
| echo-tts-base (model + PCA state) | [jordand/echo-tts-base](https://huggingface.co/jordand/echo-tts-base) | ~5.3GB | ✅ | pytorch_model.safetensors + pca_state.safetensors |
|
||||
| fish-s1-dac-min (audio codec) | [jordand/fish-s1-dac-min](https://huggingface.co/jordand/fish-s1-dac-min) | ~1.8GB | ✅ | pytorch_model.safetensors — audio codec required by Echo-TTS |
|
||||
|
||||
## Dots TTS
|
||||
|
||||
| Component | Source | Size | Auto-Download | Notes |
|
||||
|---|---|---|---|---|
|
||||
| dots.tts-base | [rednote-hilab/dots.tts-base](https://huggingface.co/rednote-hilab/dots.tts-base) | ~6GB | ✅ | Official base checkpoint with tokenizer, vocoder, speaker encoder, and latent stats |
|
||||
| dots.tts-soar | [rednote-hilab/dots.tts-soar](https://huggingface.co/rednote-hilab/dots.tts-soar) | ~6GB | ✅ | Official SOAR checkpoint for higher-quality zero-shot cloning |
|
||||
| dots.tts-mf | [rednote-hilab/dots.tts-mf](https://huggingface.co/rednote-hilab/dots.tts-mf) | ~6GB | ✅ | Official MeanFlow-distilled checkpoint for faster inference |
|
||||
|
||||
## MOSS-TTS
|
||||
|
||||
| Component | Source | Size | Auto-Download | Notes |
|
||||
|
||||
@@ -314,3 +314,37 @@ Notes:
|
||||
|
||||
- Both components are required and auto-downloaded on first use.
|
||||
- License: CC-BY-NC-SA (non-commercial).
|
||||
|
||||
## Dots TTS
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/dots_tts/
|
||||
├── dots.tts-base/
|
||||
├── dots.tts-soar/
|
||||
└── dots.tts-mf/
|
||||
```
|
||||
|
||||
Typical checkpoint contents:
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/dots_tts/dots.tts-soar/
|
||||
├── added_tokens.json
|
||||
├── chat_template.jinja
|
||||
├── config.json
|
||||
├── latent_stats.pt
|
||||
├── llm_config.json
|
||||
├── merges.txt
|
||||
├── model.safetensors
|
||||
├── speaker_encoder.safetensors
|
||||
├── special_tokens_map.json
|
||||
├── tokenizer.json
|
||||
├── tokenizer_config.json
|
||||
├── vocab.json
|
||||
└── vocoder.safetensors
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- Downloads into a dedicated `dots_tts/` folder.
|
||||
- Native sample rate is 48kHz.
|
||||
- Main-environment support works on Transformers 5; on Windows, `normalize_text` falls back to no-op if `WeTextProcessing` is unavailable.
|
||||
|
||||
@@ -43,6 +43,7 @@ Use Qwen3-TTS for:
|
||||
- SRT processor structure that reuses the main TTS processor for generation.
|
||||
- Interrupt handling.
|
||||
- Progress feedback.
|
||||
- Console observability parity, such as input-text echo and useful live generation progress.
|
||||
- Unified model loading.
|
||||
|
||||
Secondary reference:
|
||||
|
||||
@@ -46,17 +46,19 @@ Text features:
|
||||
- Did you wire parameter switching for real native parameters?
|
||||
- Did you avoid adding fake unsupported parameters?
|
||||
- Did you remove misleading controls like language selection if unsupported?
|
||||
- If the engine supports language control, did you normalize the user-facing language UI/logs honestly and map them back to official backend codes without breaking segment/alias language switching?
|
||||
- Did you reuse the suite chunking/chunk-combination controls instead of inventing duplicate engine-local chunk UI or chunk-silence parameters?
|
||||
|
||||
SRT features:
|
||||
- If this is a TTS engine, did you implement the SRT path?
|
||||
- Did you add interrupt checks in long SRT loops?
|
||||
- Does SRT use the shared timing/assembly systems correctly?
|
||||
- Does SRT use the shared timing/assembly/reporting systems correctly, using the real existing APIs instead of guessed helper names?
|
||||
- Did you avoid unnecessary timing-mode restrictions?
|
||||
- Does character switching work in SRT if text character tags are supported?
|
||||
|
||||
UX:
|
||||
- Did you add progress feedback for long generation?
|
||||
- Did you preserve useful console observability parity instead of making the engine run silently (for example input-text echo and live it/s/progress)?
|
||||
- Are error messages clear when required inputs are missing?
|
||||
- Are tooltips honest about limitations, license, VRAM, and expected text length?
|
||||
|
||||
@@ -75,7 +77,7 @@ Manual tests:
|
||||
- Did character tags work?
|
||||
- Did pause tags work?
|
||||
- Did parameter switching work?
|
||||
- Did Clear VRAM then regenerate work?
|
||||
- Did Clear VRAM then regenerate work, and did unload actually tear down runtime/cache state instead of only moving weights to CPU?
|
||||
- Did interrupt/cancel work in long generation?
|
||||
```
|
||||
|
||||
|
||||
@@ -61,6 +61,13 @@
|
||||
- **Pattern**: F5-TTS and ChatterBox don't have `__del__`, only IndexTTS and StepAudio did (wrong)
|
||||
- **Correct behavior**: Models stay in VRAM for reuse; only unload on explicit Clear VRAM button, engine switch, or parameter change
|
||||
|
||||
### Clear VRAM - Runtime Teardown vs CPU Offload
|
||||
- **Wrong**: Assuming every engine is safe to unload with only `.to('cpu')`
|
||||
- **Symptom**: Clear VRAM frees only part of VRAM; most memory disappears only after closing ComfyUI
|
||||
- **Cause**: Runtime-heavy engines can keep compiled graphs, prompt caches, workspaces, or runtime objects alive after CPU offload
|
||||
- **Correct**: Add an engine-specific unload handler that clears internal caches, drops runtime/model references, then runs `gc.collect()`, `torch.cuda.empty_cache()`, and `torch.cuda.ipc_collect()` when available
|
||||
- **Prevention**: If the engine uses warmup, `torch.compile`, streaming caches, or a custom runtime object, explicitly test Clear VRAM and reload instead of assuming generic offload parity
|
||||
|
||||
### Generation Info Reports - Modular Pattern
|
||||
- **CRITICAL - Consistency required**: ALL engines must use `ChunkCombiner` + `ChunkTimingHelper` for generation reports
|
||||
- **Correct pattern** (from ChatterBox 23-Lang, F5-TTS):
|
||||
@@ -74,10 +81,27 @@
|
||||
### Segment Parameter Registration
|
||||
- **Forgot to add**: New engines to `PARAMETER_ENGINES` in `utils/text/segment_parameters.py` for `[seed:X]` tag support
|
||||
|
||||
### Language UI Normalization
|
||||
- **Wrong**: Exposing raw backend language codes in the engine node just because the official runtime accepts them
|
||||
- **Symptom**: New engine UI looks inconsistent with the suite, logs show backend codes instead of readable names, and character-tag / alias languages do not map cleanly to node config values
|
||||
- **Correct**: Keep user-facing language controls normalized to honest display names when the suite already does that, then map them back to the official backend codes in the adapter/runtime layer
|
||||
- **Prevention**: Reuse or add a small engine-local language helper so node UI, processor logs, segment language overrides, and adapter normalization all agree on the same mapping
|
||||
|
||||
### SRT Processor Import Pattern
|
||||
- **Wrong**: Using `from nodes.xxx import` - fails with `'nodes' is not a package`
|
||||
- **Correct**: Use `importlib.util.spec_from_file_location()` for cross-module imports in dynamically loaded processors
|
||||
|
||||
### SRT Shared API Guessing
|
||||
- **Wrong**: Inventing shared helper names like `AudioAssemblyEngine.assemble(...)`, `TimingReportGenerator`, or `AdjustedSRTGenerator`
|
||||
- **Correct**: Open a working SRT processor first and match the exact API that already exists in `utils/timing/assembly.py`, `utils/timing/engine.py`, and `utils/timing/reporting.py`
|
||||
- **Real pattern**:
|
||||
- `stretch_to_fit` → `TimedAudioAssembler.assemble_timed_audio(...)`
|
||||
- `pad_with_silence` → `AudioAssemblyEngine(...).assemble_with_overlaps(...)`
|
||||
- `concatenate` → `TimingEngine.calculate_concatenation_adjustments(...)` + `assemble_concatenation(...)`
|
||||
- `smart_natural` → `TimingEngine.calculate_smart_timing_adjustments(...)` + `assemble_smart_natural(...)`
|
||||
- reporting / adjusted SRT → `SRTReportGenerator().generate_timing_report(...)` and `generate_adjusted_srt_string(...)`
|
||||
- **Prevention**: Never assume shared helper names. Grep existing SRT processors and copy the real call pattern exactly.
|
||||
|
||||
### ComfyUI Audio Dict Format
|
||||
- **Wrong**: Passing `audio_tensor` directly to functions expecting raw tensor
|
||||
- **Correct**: Extract waveform with `audio_tensor.get('waveform')` - ComfyUI audio is `{"waveform": tensor, "sample_rate": int}`
|
||||
|
||||
@@ -40,6 +40,15 @@ except Exception as e:
|
||||
def __init__(self, *args, **kwargs):
|
||||
raise ImportError(f"Echo-TTS adapter not available: {e}")
|
||||
|
||||
try:
|
||||
from .dots_tts_adapter import DotsTTSEngineAdapter
|
||||
DOTS_TTS_ADAPTER_AVAILABLE = True
|
||||
except Exception as e:
|
||||
DOTS_TTS_ADAPTER_AVAILABLE = False
|
||||
class DotsTTSEngineAdapter:
|
||||
def __init__(self, *args, **kwargs):
|
||||
raise ImportError(f"Dots TTS adapter not available: {e}")
|
||||
|
||||
try:
|
||||
from .moss_tts_adapter import MossTTSEngineAdapter
|
||||
MOSS_TTS_ADAPTER_AVAILABLE = True
|
||||
@@ -60,7 +69,9 @@ except Exception as e:
|
||||
|
||||
__all__ = [
|
||||
'ChatterBoxEngineAdapter', 'F5TTSEngineAdapter', 'CosyVoiceAdapter', 'EchoTTSEngineAdapter',
|
||||
'DotsTTSEngineAdapter',
|
||||
'MossTTSEngineAdapter', 'HiggsAudioV3EngineAdapter',
|
||||
'CHATTERBOX_ADAPTER_AVAILABLE', 'F5TTS_ADAPTER_AVAILABLE', 'COSYVOICE_ADAPTER_AVAILABLE',
|
||||
'ECHO_TTS_ADAPTER_AVAILABLE', 'MOSS_TTS_ADAPTER_AVAILABLE', 'HIGGS_AUDIO_V3_ADAPTER_AVAILABLE'
|
||||
'ECHO_TTS_ADAPTER_AVAILABLE', 'DOTS_TTS_ADAPTER_AVAILABLE',
|
||||
'MOSS_TTS_ADAPTER_AVAILABLE', 'HIGGS_AUDIO_V3_ADAPTER_AVAILABLE'
|
||||
]
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
"""
|
||||
Dots TTS engine adapter.
|
||||
|
||||
Translates TTS Audio Suite processor calls into the official dots.tts runtime.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
current_dir = os.path.dirname(__file__)
|
||||
engines_dir = os.path.dirname(current_dir)
|
||||
project_root = os.path.dirname(engines_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
from utils.audio.audio_hash import generate_stable_audio_component
|
||||
from utils.audio.cache import get_audio_cache
|
||||
from utils.audio.processing import AudioProcessingUtils
|
||||
from utils.models.factory_config import ModelLoadConfig
|
||||
from utils.models.language_mapper import resolve_language_alias
|
||||
from engines.dots_tts.languages import normalize_dots_language
|
||||
|
||||
|
||||
class DotsTTSEngineAdapter:
|
||||
"""Adapter for official Dots TTS inference."""
|
||||
|
||||
SAMPLE_RATE = 48000
|
||||
|
||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||
self.config = config.copy() if config else {}
|
||||
self.audio_cache = get_audio_cache()
|
||||
self._last_config: Optional[ModelLoadConfig] = None
|
||||
self._load_signature = None
|
||||
|
||||
def update_config(self, new_config: Dict[str, Any]):
|
||||
self.config = new_config.copy() if new_config else {}
|
||||
|
||||
def _build_load_signature(self) -> Tuple[Any, ...]:
|
||||
return (
|
||||
self.config.get("model_variant", "dots.tts-soar"),
|
||||
self.config.get("device", "auto"),
|
||||
self.config.get("precision", "auto"),
|
||||
bool(self.config.get("optimize", False)),
|
||||
int(self.config.get("max_generate_length", 500)),
|
||||
)
|
||||
|
||||
def load_model(
|
||||
self,
|
||||
model_variant: str,
|
||||
device: str = "auto",
|
||||
precision: str = "auto",
|
||||
optimize: bool = False,
|
||||
max_generate_length: int = 500,
|
||||
):
|
||||
from utils.models.unified_model_interface import unified_model_interface
|
||||
|
||||
config = ModelLoadConfig(
|
||||
engine_name="dots_tts",
|
||||
model_type="tts",
|
||||
model_name=model_variant,
|
||||
model_path=model_variant,
|
||||
device=device,
|
||||
additional_params={
|
||||
"precision": precision,
|
||||
"optimize": bool(optimize),
|
||||
"max_generate_length": int(max_generate_length),
|
||||
},
|
||||
)
|
||||
self._last_config = config
|
||||
unified_model_interface.load_model(config)
|
||||
|
||||
def _ensure_model_loaded(self):
|
||||
signature = self._build_load_signature()
|
||||
if signature == self._load_signature and self._last_config is not None:
|
||||
return
|
||||
|
||||
self.load_model(
|
||||
model_variant=self.config.get("model_variant", "dots.tts-soar"),
|
||||
device=self.config.get("device", "auto"),
|
||||
precision=self.config.get("precision", "auto"),
|
||||
optimize=self.config.get("optimize", False),
|
||||
max_generate_length=self.config.get("max_generate_length", 500),
|
||||
)
|
||||
self._load_signature = signature
|
||||
|
||||
def _get_engine(self):
|
||||
if self._last_config is None:
|
||||
self._ensure_model_loaded()
|
||||
from utils.models.unified_model_interface import unified_model_interface
|
||||
|
||||
return unified_model_interface.load_model(self._last_config)
|
||||
|
||||
def _extract_voice_reference(self, voice_ref: Optional[Dict[str, Any]]) -> Tuple[Optional[str], str, str]:
|
||||
if not voice_ref or not isinstance(voice_ref, dict):
|
||||
return None, "", "default_voice"
|
||||
|
||||
prompt_text = (
|
||||
voice_ref.get("reference_text")
|
||||
or voice_ref.get("prompt_text")
|
||||
or voice_ref.get("text")
|
||||
or ""
|
||||
).strip()
|
||||
|
||||
ref_audio = (
|
||||
voice_ref.get("prompt_audio_path")
|
||||
or voice_ref.get("audio_path")
|
||||
or voice_ref.get("audio")
|
||||
or voice_ref.get("waveform")
|
||||
)
|
||||
|
||||
if ref_audio is None:
|
||||
return None, prompt_text, "default_voice"
|
||||
|
||||
if isinstance(ref_audio, str):
|
||||
return ref_audio, prompt_text, generate_stable_audio_component(audio_file_path=ref_audio)
|
||||
|
||||
if isinstance(ref_audio, dict) and "waveform" in ref_audio:
|
||||
temp_path = AudioProcessingUtils.save_audio_to_temp_file(
|
||||
ref_audio["waveform"],
|
||||
ref_audio.get("sample_rate", self.SAMPLE_RATE),
|
||||
)
|
||||
return temp_path, prompt_text, generate_stable_audio_component(reference_audio=ref_audio)
|
||||
|
||||
if torch.is_tensor(ref_audio):
|
||||
sample_rate = int(voice_ref.get("sample_rate", self.SAMPLE_RATE))
|
||||
audio_dict = {"waveform": ref_audio, "sample_rate": sample_rate}
|
||||
temp_path = AudioProcessingUtils.save_audio_to_temp_file(ref_audio, sample_rate)
|
||||
return temp_path, prompt_text, generate_stable_audio_component(reference_audio=audio_dict)
|
||||
|
||||
raise TypeError(f"Unsupported Dots TTS voice reference type: {type(ref_audio)}")
|
||||
|
||||
@staticmethod
|
||||
def _normalize_language(language: Optional[str]) -> Optional[str]:
|
||||
if language is None:
|
||||
return None
|
||||
normalized = str(language).strip()
|
||||
if not normalized:
|
||||
return None
|
||||
normalized_language = normalize_dots_language(normalized)
|
||||
if normalized_language is not None:
|
||||
return normalized_language
|
||||
resolved = resolve_language_alias(normalized)
|
||||
return normalize_dots_language(resolved)
|
||||
|
||||
def generate_single(
|
||||
self,
|
||||
text: str,
|
||||
voice_ref: Optional[Dict[str, Any]],
|
||||
seed: int = 0,
|
||||
enable_audio_cache: bool = True,
|
||||
character_name: Optional[str] = None,
|
||||
) -> torch.Tensor:
|
||||
stripped = (text or "").strip()
|
||||
if not stripped:
|
||||
return torch.zeros(1, 0, dtype=torch.float32)
|
||||
|
||||
self._ensure_model_loaded()
|
||||
engine = self._get_engine()
|
||||
|
||||
prompt_audio_path, prompt_text, audio_component = self._extract_voice_reference(voice_ref)
|
||||
language = self._normalize_language(self.config.get("language", "auto"))
|
||||
model_variant = self.config.get("model_variant", "dots.tts-soar")
|
||||
template_name = self.config.get("template_name", "tts")
|
||||
num_steps = int(self.config.get("num_steps", 10))
|
||||
guidance_scale = float(self.config.get("guidance_scale", 1.2))
|
||||
speaker_scale = float(self.config.get("speaker_scale", 1.5))
|
||||
normalize_text = bool(self.config.get("normalize_text", False))
|
||||
precision = self.config.get("precision", "auto")
|
||||
optimize = bool(self.config.get("optimize", False))
|
||||
max_generate_length = int(self.config.get("max_generate_length", 500))
|
||||
|
||||
cache_key = None
|
||||
if enable_audio_cache:
|
||||
cache_key = self.audio_cache.generate_cache_key(
|
||||
"dots_tts",
|
||||
text=stripped,
|
||||
audio_component=audio_component,
|
||||
prompt_text=prompt_text,
|
||||
model_variant=model_variant,
|
||||
language=language or "none",
|
||||
template_name=template_name,
|
||||
num_steps=num_steps,
|
||||
guidance_scale=guidance_scale,
|
||||
speaker_scale=speaker_scale,
|
||||
normalize_text=normalize_text,
|
||||
max_generate_length=max_generate_length,
|
||||
precision=precision,
|
||||
optimize=optimize,
|
||||
seed=int(seed or 0),
|
||||
device=self.config.get("device", "auto"),
|
||||
character=character_name or "narrator",
|
||||
)
|
||||
cached_audio = self.audio_cache.get_cached_audio(cache_key)
|
||||
if cached_audio:
|
||||
print(f"💾 Using cached Dots TTS audio for '{character_name or 'narrator'}': '{stripped[:30]}...'")
|
||||
return cached_audio[0]
|
||||
|
||||
result = engine.generate(
|
||||
text=stripped,
|
||||
prompt_audio_path=prompt_audio_path,
|
||||
prompt_text=prompt_text or None,
|
||||
language=language,
|
||||
num_steps=num_steps,
|
||||
guidance_scale=guidance_scale,
|
||||
speaker_scale=speaker_scale,
|
||||
normalize_text=normalize_text,
|
||||
template_name=template_name,
|
||||
seed=int(seed or 0),
|
||||
)
|
||||
|
||||
audio_tensor = result["audio"]
|
||||
if not isinstance(audio_tensor, torch.Tensor):
|
||||
audio_tensor = torch.tensor(audio_tensor, dtype=torch.float32)
|
||||
audio_tensor = audio_tensor.detach().float().cpu()
|
||||
if audio_tensor.dim() == 1:
|
||||
audio_tensor = audio_tensor.unsqueeze(0)
|
||||
|
||||
if enable_audio_cache and cache_key:
|
||||
duration = self.audio_cache._calculate_duration(audio_tensor, "dots_tts")
|
||||
self.audio_cache.cache_audio(cache_key, audio_tensor, duration)
|
||||
|
||||
return audio_tensor
|
||||
@@ -0,0 +1,6 @@
|
||||
"""Dots TTS engine integration."""
|
||||
|
||||
from .dots_tts_engine import DotsTTSEngine
|
||||
from .dots_tts_downloader import DotsTTSDownloader
|
||||
|
||||
__all__ = ["DotsTTSEngine", "DotsTTSDownloader"]
|
||||
@@ -0,0 +1,160 @@
|
||||
"""
|
||||
Dots TTS model downloader.
|
||||
|
||||
Downloads official Rednote dots.tts checkpoints into
|
||||
ComfyUI/models/TTS/dots_tts/ instead of hidden Hugging Face cache folders.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
from typing import Dict, Optional
|
||||
|
||||
import folder_paths
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
from utils.models.extra_paths import get_all_tts_model_paths, get_preferred_download_path
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _suppress_hf_http_logs():
|
||||
# Keep HF download output readable by muting per-request chatter during snapshot downloads.
|
||||
logger_names = ("httpx", "httpcore", "huggingface_hub")
|
||||
original_levels = {}
|
||||
try:
|
||||
for name in logger_names:
|
||||
logger = logging.getLogger(name)
|
||||
original_levels[name] = logger.level
|
||||
logger.setLevel(logging.WARNING)
|
||||
yield
|
||||
finally:
|
||||
for name, level in original_levels.items():
|
||||
logging.getLogger(name).setLevel(level)
|
||||
|
||||
|
||||
class DotsTTSDownloader:
|
||||
"""Resolve and download official dots.tts model folders."""
|
||||
|
||||
MODELS: Dict[str, Dict[str, str]] = {
|
||||
"dots.tts-base": {
|
||||
"repo_id": "rednote-hilab/dots.tts-base",
|
||||
"description": "Official Dots TTS base checkpoint",
|
||||
},
|
||||
"dots.tts-soar": {
|
||||
"repo_id": "rednote-hilab/dots.tts-soar",
|
||||
"description": "Official Dots TTS SOAR checkpoint",
|
||||
},
|
||||
"dots.tts-mf": {
|
||||
"repo_id": "rednote-hilab/dots.tts-mf",
|
||||
"description": "Official Dots TTS MeanFlow checkpoint",
|
||||
},
|
||||
}
|
||||
|
||||
REQUIRED_FILES = [
|
||||
"added_tokens.json",
|
||||
"chat_template.jinja",
|
||||
"config.json",
|
||||
"latent_stats.pt",
|
||||
"llm_config.json",
|
||||
"merges.txt",
|
||||
"model.safetensors",
|
||||
"speaker_encoder.safetensors",
|
||||
"special_tokens_map.json",
|
||||
"tokenizer.json",
|
||||
"tokenizer_config.json",
|
||||
"vocab.json",
|
||||
"vocoder.safetensors",
|
||||
]
|
||||
|
||||
def __init__(self, base_path: Optional[str] = None):
|
||||
if base_path is None:
|
||||
try:
|
||||
self.base_path = get_preferred_download_path(model_type="TTS", engine_name="dots_tts")
|
||||
except Exception:
|
||||
self.base_path = os.path.join(folder_paths.models_dir, "TTS", "dots_tts")
|
||||
else:
|
||||
self.base_path = base_path
|
||||
os.makedirs(self.base_path, exist_ok=True)
|
||||
|
||||
def resolve_model_path(self, model_identifier: str) -> str:
|
||||
"""Resolve local: names, absolute paths, or known official model identifiers."""
|
||||
if not model_identifier:
|
||||
model_identifier = "dots.tts-soar"
|
||||
|
||||
if os.path.isabs(model_identifier) and os.path.exists(model_identifier):
|
||||
return model_identifier
|
||||
|
||||
if os.path.isdir(model_identifier):
|
||||
return model_identifier
|
||||
|
||||
if model_identifier.startswith("local:"):
|
||||
local_name = model_identifier[6:]
|
||||
for base_path in get_all_tts_model_paths("TTS"):
|
||||
for folder_name in ("dots_tts", "DOTS", ""):
|
||||
candidate = os.path.join(base_path, folder_name, local_name) if folder_name else os.path.join(base_path, local_name)
|
||||
if os.path.exists(candidate):
|
||||
print(f"📁 Using local Dots TTS model: {candidate}")
|
||||
return candidate
|
||||
raise FileNotFoundError(f"Local Dots TTS model not found: {local_name}")
|
||||
|
||||
if model_identifier in self.MODELS:
|
||||
return self.get_model_path(model_identifier)
|
||||
|
||||
raise FileNotFoundError(f"Unknown Dots TTS model identifier: {model_identifier}")
|
||||
|
||||
def get_model_path(self, model_name: str) -> str:
|
||||
model_dir = os.path.join(self.base_path, model_name)
|
||||
if not os.path.exists(model_dir) or not self._is_model_complete(model_dir):
|
||||
return self.download_model(model_name, force=os.path.exists(model_dir))
|
||||
return model_dir
|
||||
|
||||
def download_model(self, model_name: str, force: bool = False) -> str:
|
||||
if model_name not in self.MODELS:
|
||||
raise ValueError(f"Unknown Dots TTS model: {model_name}")
|
||||
|
||||
model_info = self.MODELS[model_name]
|
||||
repo_id = model_info["repo_id"]
|
||||
model_dir = os.path.join(self.base_path, model_name)
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print("📦 Dots TTS Model Download")
|
||||
print(f"{'=' * 60}")
|
||||
print(f"Model: {model_name}")
|
||||
print(f"Description: {model_info['description']}")
|
||||
print(f"Repository: {repo_id}")
|
||||
print(f"Target: {model_dir}")
|
||||
print(f"{'=' * 60}\n")
|
||||
|
||||
try:
|
||||
with _suppress_hf_http_logs():
|
||||
snapshot_download(
|
||||
repo_id=repo_id,
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
resume_download=True,
|
||||
force_download=force,
|
||||
)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to download {model_name} from {repo_id}: {e}") from e
|
||||
|
||||
if not self._is_model_complete(model_dir):
|
||||
raise RuntimeError(f"Downloaded Dots TTS model is incomplete: {model_dir}")
|
||||
|
||||
print(f"✅ Dots TTS model ready: {model_dir}")
|
||||
return model_dir
|
||||
|
||||
def _is_model_complete(self, model_dir: str) -> bool:
|
||||
if not os.path.isdir(model_dir):
|
||||
return False
|
||||
|
||||
missing = [
|
||||
rel_path
|
||||
for rel_path in self.REQUIRED_FILES
|
||||
if not os.path.exists(os.path.join(model_dir, rel_path))
|
||||
]
|
||||
if missing:
|
||||
print(f"❌ Dots TTS model incomplete. Missing {len(missing)} file(s):")
|
||||
for rel_path in missing:
|
||||
print(f" - {rel_path}")
|
||||
return False
|
||||
return True
|
||||
@@ -0,0 +1,446 @@
|
||||
"""
|
||||
Dots TTS engine wrapper.
|
||||
|
||||
Wraps the official dots.tts runtime with ComfyUI-friendly model lifecycle hooks.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import types
|
||||
import importlib
|
||||
import logging
|
||||
import time
|
||||
import warnings
|
||||
import gc
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import torch
|
||||
|
||||
current_dir = os.path.dirname(__file__)
|
||||
engines_dir = os.path.dirname(current_dir)
|
||||
project_root = os.path.dirname(engines_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
from engines.dots_tts.dots_tts_downloader import DotsTTSDownloader
|
||||
from engines.dots_tts.progress_callback import DotsTTSProgressTracker
|
||||
from utils.device import resolve_torch_device
|
||||
|
||||
|
||||
class _SuppressDotsModelTypeWarning(logging.Filter):
|
||||
"""Drop the upstream bogus model-type warning while keeping real ones."""
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
try:
|
||||
message = record.getMessage()
|
||||
except Exception:
|
||||
return True
|
||||
return "model of type `dots_tts` to instantiate a model of type" not in message
|
||||
|
||||
|
||||
class DotsTTSEngine:
|
||||
"""Thin wrapper around the official DotsTtsRuntime."""
|
||||
|
||||
SAMPLE_RATE = 48000
|
||||
_normalizer_warning_shown = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_name: str = "dots.tts-soar",
|
||||
device: str = "auto",
|
||||
precision: str = "auto",
|
||||
optimize: bool = False,
|
||||
max_generate_length: int = 500,
|
||||
model_dir: Optional[str] = None,
|
||||
):
|
||||
self.model_name = model_name
|
||||
self.device = resolve_torch_device(device)
|
||||
self.precision = self._resolve_precision(precision, self.device)
|
||||
self.optimize = bool(optimize)
|
||||
self.max_generate_length = int(max_generate_length)
|
||||
self.model_dir = model_dir or DotsTTSDownloader().resolve_model_path(model_name)
|
||||
self._runtime = None
|
||||
|
||||
@staticmethod
|
||||
def _resolve_precision(precision: str, device: str) -> str:
|
||||
normalized = str(precision or "auto").lower()
|
||||
if normalized in {"bfloat16", "float16", "float32"}:
|
||||
if device == "cpu" and normalized != "float32":
|
||||
return "float32"
|
||||
return normalized
|
||||
|
||||
if device == "cpu":
|
||||
return "float32"
|
||||
|
||||
if torch.cuda.is_available():
|
||||
major, _minor = torch.cuda.get_device_capability()
|
||||
if major >= 8:
|
||||
return "bfloat16"
|
||||
return "float16"
|
||||
|
||||
def _import_runtime(self):
|
||||
self._ensure_text_normalizer_fallback()
|
||||
importlib.invalidate_caches()
|
||||
|
||||
nodes_dir = os.path.join(project_root, "nodes")
|
||||
blocked_paths = {
|
||||
os.path.abspath(engines_dir),
|
||||
os.path.abspath(nodes_dir),
|
||||
}
|
||||
original_sys_path = list(sys.path)
|
||||
|
||||
# ComfyUI can place the suite's `engines/` directory directly on sys.path.
|
||||
# That makes our local `engines/dots_tts` package shadow the pip-installed
|
||||
# official `dots_tts` package we actually need here.
|
||||
sys.path[:] = [
|
||||
path for path in sys.path
|
||||
if os.path.abspath(path or os.getcwd()) not in blocked_paths
|
||||
]
|
||||
|
||||
stale_modules = []
|
||||
for module_name, module in list(sys.modules.items()):
|
||||
if not (module_name == "dots_tts" or module_name.startswith("dots_tts.")):
|
||||
continue
|
||||
module_file = getattr(module, "__file__", "") or ""
|
||||
if module_file and os.path.abspath(module_file).startswith(os.path.abspath(project_root)):
|
||||
stale_modules.append((module_name, module))
|
||||
del sys.modules[module_name]
|
||||
try:
|
||||
from dots_tts.runtime import DotsTtsRuntime
|
||||
except Exception as e:
|
||||
for module_name, module in stale_modules:
|
||||
sys.modules.setdefault(module_name, module)
|
||||
raise ImportError(
|
||||
"Failed to import the official `dots_tts` package in the active ComfyUI Python "
|
||||
"environment. This is usually either a missing install or an import-path "
|
||||
f"collision. Active Python: {sys.executable}"
|
||||
) from e
|
||||
finally:
|
||||
sys.path[:] = original_sys_path
|
||||
return DotsTtsRuntime
|
||||
|
||||
@staticmethod
|
||||
@contextmanager
|
||||
def _suppress_noisy_runtime_logs():
|
||||
"""Mute verbose Dots chatter while keeping actionable Transformers warnings."""
|
||||
dots_logger = None
|
||||
dots_logger_disabled = False
|
||||
transformers_filter = _SuppressDotsModelTypeWarning()
|
||||
transformer_loggers = []
|
||||
|
||||
try:
|
||||
from loguru import logger as dots_logger
|
||||
|
||||
dots_logger.disable("dots_tts")
|
||||
dots_logger_disabled = True
|
||||
|
||||
transformer_loggers = [
|
||||
logging.getLogger("transformers"),
|
||||
logging.getLogger("transformers.configuration_utils"),
|
||||
logging.getLogger("transformers.modeling_utils"),
|
||||
logging.getLogger("transformers.tokenization_utils_base"),
|
||||
]
|
||||
for logger in transformer_loggers:
|
||||
logger.addFilter(transformers_filter)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
with warnings.catch_warnings():
|
||||
warnings.filterwarnings(
|
||||
"ignore",
|
||||
message=r".*model of type `dots_tts` to instantiate a model of type.*",
|
||||
)
|
||||
yield
|
||||
finally:
|
||||
if dots_logger is not None and dots_logger_disabled:
|
||||
try:
|
||||
dots_logger.enable("dots_tts")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for logger in transformer_loggers:
|
||||
try:
|
||||
logger.removeFilter(transformers_filter)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
@contextmanager
|
||||
def _patch_tokenizer_regex_fix():
|
||||
"""Force the upstream Dots tokenizer load to opt into the Mistral regex fix."""
|
||||
try:
|
||||
from transformers import AutoTokenizer
|
||||
except Exception:
|
||||
yield
|
||||
return
|
||||
|
||||
original_from_pretrained = AutoTokenizer.from_pretrained
|
||||
|
||||
def patched_from_pretrained(*args, **kwargs):
|
||||
kwargs.setdefault("fix_mistral_regex", True)
|
||||
return original_from_pretrained(*args, **kwargs)
|
||||
|
||||
AutoTokenizer.from_pretrained = patched_from_pretrained
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
AutoTokenizer.from_pretrained = original_from_pretrained
|
||||
|
||||
@staticmethod
|
||||
def _ensure_text_normalizer_fallback():
|
||||
"""Provide a no-op WeTextProcessing fallback when tn is unavailable."""
|
||||
try:
|
||||
import tn # noqa: F401
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
class _NoOpNormalizer:
|
||||
def normalize(self, text: str) -> str:
|
||||
return text
|
||||
|
||||
tn_module = types.ModuleType("tn")
|
||||
chinese_module = types.ModuleType("tn.chinese")
|
||||
chinese_normalizer_module = types.ModuleType("tn.chinese.normalizer")
|
||||
english_module = types.ModuleType("tn.english")
|
||||
english_normalizer_module = types.ModuleType("tn.english.normalizer")
|
||||
|
||||
chinese_normalizer_module.Normalizer = _NoOpNormalizer
|
||||
english_normalizer_module.Normalizer = _NoOpNormalizer
|
||||
|
||||
chinese_module.normalizer = chinese_normalizer_module
|
||||
english_module.normalizer = english_normalizer_module
|
||||
tn_module.chinese = chinese_module
|
||||
tn_module.english = english_module
|
||||
|
||||
sys.modules.setdefault("tn", tn_module)
|
||||
sys.modules.setdefault("tn.chinese", chinese_module)
|
||||
sys.modules.setdefault("tn.chinese.normalizer", chinese_normalizer_module)
|
||||
sys.modules.setdefault("tn.english", english_module)
|
||||
sys.modules.setdefault("tn.english.normalizer", english_normalizer_module)
|
||||
if not DotsTTSEngine._normalizer_warning_shown:
|
||||
print("[Dots TTS] WeTextProcessing not available; normalize_text will use a no-op fallback")
|
||||
DotsTTSEngine._normalizer_warning_shown = True
|
||||
|
||||
def _ensure_runtime_loaded(self):
|
||||
if self._runtime is not None:
|
||||
return
|
||||
|
||||
DotsTtsRuntime = self._import_runtime()
|
||||
with self._patch_tokenizer_regex_fix(), self._suppress_noisy_runtime_logs():
|
||||
self._runtime = DotsTtsRuntime.from_pretrained(
|
||||
self.model_dir,
|
||||
precision=self.precision,
|
||||
optimize=self.optimize,
|
||||
max_generate_length=self.max_generate_length,
|
||||
)
|
||||
self._ensure_runtime_device()
|
||||
|
||||
def _current_runtime_device(self) -> Optional[str]:
|
||||
if self._runtime is None:
|
||||
return None
|
||||
try:
|
||||
first_param = next(self._runtime.model.parameters())
|
||||
return str(first_param.device)
|
||||
except Exception:
|
||||
return str(getattr(self._runtime, "device", self.device))
|
||||
|
||||
def _move_runtime_to(self, target_device: str):
|
||||
if self._runtime is None:
|
||||
return
|
||||
|
||||
device_obj = torch.device(target_device)
|
||||
runtime = self._runtime
|
||||
runtime.model = runtime.model.to(device_obj).eval()
|
||||
runtime.device = device_obj
|
||||
|
||||
def _ensure_runtime_device(self):
|
||||
if self._runtime is None:
|
||||
return
|
||||
target_device = resolve_torch_device(self.device)
|
||||
current_device = self._current_runtime_device()
|
||||
if self._devices_equivalent(current_device, target_device):
|
||||
return
|
||||
if current_device != target_device:
|
||||
print(f"🔄 Dots TTS: moving runtime from {current_device} to {target_device}")
|
||||
self._move_runtime_to(target_device)
|
||||
|
||||
@staticmethod
|
||||
def _devices_equivalent(current_device: Optional[str], target_device: Optional[str]) -> bool:
|
||||
if current_device == target_device:
|
||||
return True
|
||||
if not current_device or not target_device:
|
||||
return False
|
||||
current = str(current_device)
|
||||
target = str(target_device)
|
||||
if current.startswith("cuda") and target.startswith("cuda"):
|
||||
return current in {"cuda", "cuda:0"} and target in {"cuda", "cuda:0"}
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _clear_runtime_caches(runtime: Any) -> None:
|
||||
if runtime is None:
|
||||
return
|
||||
|
||||
model = getattr(runtime, "model", None)
|
||||
if model is not None:
|
||||
try:
|
||||
if hasattr(model, "set_optimize"):
|
||||
model.set_optimize(False)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for attr_name in (
|
||||
"_compiled_models",
|
||||
"_prompt_feature_cache",
|
||||
"_static_generate_workspaces",
|
||||
"_fm_decode_workspaces",
|
||||
):
|
||||
try:
|
||||
cache = getattr(model, attr_name, None)
|
||||
if hasattr(cache, "clear"):
|
||||
cache.clear()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for attr_name in (
|
||||
"_double_streaming_prompt_g_cond_cache",
|
||||
"_double_streaming_silence_audio_patch_cache",
|
||||
):
|
||||
try:
|
||||
cache = getattr(runtime, attr_name, None)
|
||||
if hasattr(cache, "clear"):
|
||||
cache.clear()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def unload_runtime(self):
|
||||
"""Fully tear down the Dots runtime so Clear VRAM actually releases it."""
|
||||
runtime = self._runtime
|
||||
if runtime is None:
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
if hasattr(torch.cuda, "ipc_collect"):
|
||||
try:
|
||||
torch.cuda.ipc_collect()
|
||||
except Exception:
|
||||
pass
|
||||
gc.collect()
|
||||
return
|
||||
|
||||
self._clear_runtime_caches(runtime)
|
||||
|
||||
model = getattr(runtime, "model", None)
|
||||
try:
|
||||
if model is not None and hasattr(model, "to"):
|
||||
model.to("cpu")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
runtime.model = None
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
self._runtime = None
|
||||
|
||||
try:
|
||||
import torch._dynamo as torch_dynamo
|
||||
|
||||
torch_dynamo.reset()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
del model
|
||||
del runtime
|
||||
gc.collect()
|
||||
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
if hasattr(torch.cuda, "ipc_collect"):
|
||||
try:
|
||||
torch.cuda.ipc_collect()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def to(self, device):
|
||||
"""ComfyUI unload/reload hook."""
|
||||
self.device = str(device) if isinstance(device, str) else str(torch.device(device))
|
||||
self._ensure_runtime_device()
|
||||
return self
|
||||
|
||||
def unload(self):
|
||||
"""Explicit unload hook used by engine-specific ComfyUI handlers."""
|
||||
self.unload_runtime()
|
||||
|
||||
def generate(
|
||||
self,
|
||||
text: str,
|
||||
prompt_audio_path: Optional[str] = None,
|
||||
prompt_text: Optional[str] = None,
|
||||
language: Optional[str] = None,
|
||||
num_steps: int = 10,
|
||||
guidance_scale: float = 1.2,
|
||||
speaker_scale: float = 1.5,
|
||||
normalize_text: bool = False,
|
||||
template_name: Optional[str] = "tts",
|
||||
seed: int = 42,
|
||||
) -> Dict[str, Any]:
|
||||
self._ensure_runtime_loaded()
|
||||
self._ensure_runtime_device()
|
||||
|
||||
seed_value = int(seed or 0)
|
||||
if seed_value > 0:
|
||||
torch.manual_seed(seed_value)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed_all(seed_value)
|
||||
|
||||
chunks = []
|
||||
tracker = DotsTTSProgressTracker(
|
||||
max_generate_length=self.max_generate_length,
|
||||
sample_rate=self.SAMPLE_RATE,
|
||||
)
|
||||
start_time = time.time()
|
||||
|
||||
with self._patch_tokenizer_regex_fix(), self._suppress_noisy_runtime_logs():
|
||||
stream = self._runtime.generate_stream(
|
||||
text=text,
|
||||
prompt_audio_path=prompt_audio_path,
|
||||
prompt_text=prompt_text,
|
||||
language=language,
|
||||
template_name=template_name,
|
||||
num_steps=int(num_steps),
|
||||
guidance_scale=float(guidance_scale),
|
||||
speaker_scale=float(speaker_scale),
|
||||
normalize_text=bool(normalize_text),
|
||||
)
|
||||
for chunk in stream:
|
||||
if not isinstance(chunk, torch.Tensor):
|
||||
chunk = torch.tensor(chunk, dtype=torch.float32)
|
||||
chunk = chunk.detach().float().cpu()
|
||||
if chunk.dim() > 1:
|
||||
chunk = chunk.squeeze()
|
||||
chunks.append(chunk)
|
||||
tracker.update(chunk.shape[-1] if chunk.numel() > 0 else 0)
|
||||
|
||||
tracker.end()
|
||||
|
||||
if chunks:
|
||||
audio = torch.cat(chunks, dim=-1)
|
||||
else:
|
||||
audio = torch.zeros(0, dtype=torch.float32)
|
||||
|
||||
time_used = time.time() - start_time
|
||||
duration_seconds = audio.shape[-1] / self.SAMPLE_RATE if audio.numel() > 0 else 0.0
|
||||
rtf = time_used / duration_seconds if duration_seconds > 0 else float("inf")
|
||||
|
||||
return {
|
||||
"fid": None,
|
||||
"audio": audio,
|
||||
"sample_rate": self.SAMPLE_RATE,
|
||||
"time_used": time_used,
|
||||
"rtf": rtf,
|
||||
"profiling": None,
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
"""
|
||||
Language helpers for Dots TTS.
|
||||
"""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
|
||||
DOTS_LANGUAGE_CODE_BY_NAME = {
|
||||
"arabic": "AR",
|
||||
"czech": "CS",
|
||||
"german": "DE",
|
||||
"greek": "EL",
|
||||
"english": "EN",
|
||||
"spanish": "ES",
|
||||
"finnish": "FI",
|
||||
"french": "FR",
|
||||
"hindi": "HI",
|
||||
"indonesian": "ID",
|
||||
"italian": "IT",
|
||||
"japanese": "JA",
|
||||
"korean": "KO",
|
||||
"dutch": "NL",
|
||||
"polish": "PL",
|
||||
"portuguese": "PT",
|
||||
"pt": "PT",
|
||||
"pt-br": "PT",
|
||||
"pt-pt": "PT",
|
||||
"po": "PT",
|
||||
"romanian": "RO",
|
||||
"russian": "RU",
|
||||
"thai": "TH",
|
||||
"turkish": "TR",
|
||||
"ukrainian": "UK",
|
||||
"uk": "UK",
|
||||
"vietnamese": "VI",
|
||||
"cantonese": "YUE",
|
||||
"yue": "YUE",
|
||||
"chinese": "ZH",
|
||||
"mandarin": "ZH",
|
||||
"zh": "ZH",
|
||||
"zh-cn": "ZH",
|
||||
"zh-tw": "ZH",
|
||||
}
|
||||
|
||||
DOTS_LANGUAGE_DISPLAY_BY_CODE = {
|
||||
"AR": "Arabic",
|
||||
"CS": "Czech",
|
||||
"DE": "German",
|
||||
"EL": "Greek",
|
||||
"EN": "English",
|
||||
"ES": "Spanish",
|
||||
"FI": "Finnish",
|
||||
"FR": "French",
|
||||
"HI": "Hindi",
|
||||
"ID": "Indonesian",
|
||||
"IT": "Italian",
|
||||
"JA": "Japanese",
|
||||
"KO": "Korean",
|
||||
"NL": "Dutch",
|
||||
"PL": "Polish",
|
||||
"PT": "Portuguese",
|
||||
"RO": "Romanian",
|
||||
"RU": "Russian",
|
||||
"TH": "Thai",
|
||||
"TR": "Turkish",
|
||||
"UK": "Ukrainian",
|
||||
"VI": "Vietnamese",
|
||||
"YUE": "Cantonese",
|
||||
"ZH": "Chinese",
|
||||
}
|
||||
|
||||
DOTS_LANGUAGE_OPTIONS = [
|
||||
"Auto",
|
||||
"None",
|
||||
"Arabic",
|
||||
"Czech",
|
||||
"German",
|
||||
"Greek",
|
||||
"English",
|
||||
"Spanish",
|
||||
"Finnish",
|
||||
"French",
|
||||
"Hindi",
|
||||
"Indonesian",
|
||||
"Italian",
|
||||
"Japanese",
|
||||
"Korean",
|
||||
"Dutch",
|
||||
"Polish",
|
||||
"Portuguese",
|
||||
"Romanian",
|
||||
"Russian",
|
||||
"Thai",
|
||||
"Turkish",
|
||||
"Ukrainian",
|
||||
"Vietnamese",
|
||||
"Cantonese",
|
||||
"Chinese",
|
||||
]
|
||||
|
||||
|
||||
def normalize_dots_language(language: Optional[str]) -> Optional[str]:
|
||||
if language is None:
|
||||
return None
|
||||
|
||||
normalized = str(language).strip()
|
||||
if not normalized:
|
||||
return None
|
||||
|
||||
lowered = normalized.lower()
|
||||
if lowered in {"auto", "auto detect", "auto_detect"}:
|
||||
return "auto_detect"
|
||||
if lowered in {"none", "off", "no tag", "no language tag"}:
|
||||
return None
|
||||
|
||||
mapped = DOTS_LANGUAGE_CODE_BY_NAME.get(lowered)
|
||||
if mapped:
|
||||
return mapped
|
||||
|
||||
upper_value = normalized.upper()
|
||||
if upper_value in DOTS_LANGUAGE_DISPLAY_BY_CODE:
|
||||
return upper_value
|
||||
|
||||
if lowered == "yue":
|
||||
return "YUE"
|
||||
|
||||
return upper_value
|
||||
|
||||
|
||||
def format_dots_language_display(language: Optional[str]) -> str:
|
||||
normalized = normalize_dots_language(language)
|
||||
if normalized == "auto_detect":
|
||||
return "Auto"
|
||||
if normalized is None:
|
||||
return "None"
|
||||
|
||||
return DOTS_LANGUAGE_DISPLAY_BY_CODE.get(normalized, str(language).strip() or "Auto")
|
||||
@@ -0,0 +1,66 @@
|
||||
"""
|
||||
Dots TTS progress callback.
|
||||
|
||||
Provides suite-owned console progress output during streaming generation.
|
||||
"""
|
||||
|
||||
import time
|
||||
|
||||
|
||||
class DotsTTSProgressTracker:
|
||||
"""Track Dots streaming progress with live it/s and RTF reporting."""
|
||||
|
||||
def __init__(self, max_generate_length: int, sample_rate: int):
|
||||
self.max_generate_length = max(1, int(max_generate_length or 1))
|
||||
self.sample_rate = max(1, int(sample_rate or 1))
|
||||
self.generated_chunks = 0
|
||||
self.emitted_samples = 0
|
||||
self.start_time = time.time()
|
||||
self.last_print_time = self.start_time
|
||||
self.last_print_chunks = 0
|
||||
|
||||
def update(self, chunk_samples: int) -> None:
|
||||
self.generated_chunks += 1
|
||||
self.emitted_samples += max(0, int(chunk_samples or 0))
|
||||
|
||||
current_time = time.time()
|
||||
should_print = (
|
||||
self.generated_chunks == 1 or
|
||||
current_time - self.last_print_time >= 0.5
|
||||
)
|
||||
if not should_print:
|
||||
return
|
||||
|
||||
chunk_delta = self.generated_chunks - self.last_print_chunks
|
||||
time_delta = current_time - self.last_print_time
|
||||
its = chunk_delta / time_delta if time_delta > 0 else 0.0
|
||||
elapsed = current_time - self.start_time
|
||||
audio_seconds = self.emitted_samples / self.sample_rate
|
||||
rtf = elapsed / audio_seconds if audio_seconds > 0 else 0.0
|
||||
|
||||
bar_width = 12
|
||||
display_chunks = min(self.generated_chunks, self.max_generate_length)
|
||||
filled = int(bar_width * display_chunks / self.max_generate_length)
|
||||
filled = min(filled, bar_width)
|
||||
progress_bar = f"[{'█' * filled}{'░' * (bar_width - filled)}] {display_chunks}/{self.max_generate_length}"
|
||||
|
||||
print(
|
||||
f"\r Progress: {progress_bar} | {its:.1f} it/s | {elapsed:.0f}s | "
|
||||
f"{audio_seconds:.1f}s audio | RTF {rtf:.2f} ",
|
||||
end="",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
self.last_print_time = current_time
|
||||
self.last_print_chunks = self.generated_chunks
|
||||
|
||||
def end(self) -> None:
|
||||
total_time = time.time() - self.start_time
|
||||
avg_its = self.generated_chunks / total_time if total_time > 0 else 0.0
|
||||
audio_seconds = self.emitted_samples / self.sample_rate
|
||||
rtf = total_time / audio_seconds if audio_seconds > 0 else 0.0
|
||||
print(
|
||||
f"\r Complete: {self.generated_chunks} chunks in {total_time:.1f}s "
|
||||
f"(avg {avg_its:.1f} it/s, {audio_seconds:.1f}s audio, RTF {rtf:.2f})"
|
||||
+ " " * 20
|
||||
)
|
||||
+29
-1
@@ -1130,7 +1130,34 @@ class TTSAudioInstaller:
|
||||
"Installing Echo-TTS from GitHub (--no-deps)",
|
||||
ignore_errors=True
|
||||
)
|
||||
|
||||
|
||||
def install_dots_tts(self):
|
||||
"""Install official Dots TTS with minimal dependency impact."""
|
||||
self.log("Installing Dots TTS engine", "INFO")
|
||||
|
||||
dependency_probes = [
|
||||
("lingua", ["install", "lingua-language-detector"], "Installing lingua-language-detector"),
|
||||
("langcodes", ["install", "langcodes"], "Installing langcodes"),
|
||||
("loguru", ["install", "loguru"], "Installing loguru"),
|
||||
("tn", ["install", "WeTextProcessing"], "Installing WeTextProcessing"),
|
||||
]
|
||||
|
||||
for module_name, pip_args, description in dependency_probes:
|
||||
if self.verify_python_import(module_name):
|
||||
self.log(f"{module_name} import already satisfied - skipping", "SUCCESS")
|
||||
continue
|
||||
self.run_pip_command(pip_args, description, ignore_errors=True)
|
||||
|
||||
if self.verify_python_import("dots_tts.runtime"):
|
||||
self.log("dots_tts.runtime already satisfied - skipping", "SUCCESS")
|
||||
return
|
||||
|
||||
self.run_pip_command(
|
||||
["install", "git+https://github.com/rednote-hilab/dots.tts.git", "--no-deps"],
|
||||
"Installing official dots.tts from GitHub (--no-deps)",
|
||||
ignore_errors=True
|
||||
)
|
||||
|
||||
def install_f5tts_multilingual_support(self):
|
||||
"""Install phonemization support for F5-TTS multilingual models (Polish, German, French, Spanish, etc.)"""
|
||||
self.log("Installing F5-TTS multilingual phonemization support", "INFO")
|
||||
@@ -1567,6 +1594,7 @@ def main():
|
||||
installer.install_onnxruntime_with_gpu_support() # Install ONNX with GPU acceleration if available
|
||||
installer.install_vibevoice() # Install VibeVoice with careful dependency management
|
||||
installer.install_echo_tts() # Install Echo-TTS with minimal dependency impact
|
||||
installer.install_dots_tts() # Install official Dots TTS in the main environment first
|
||||
installer.install_f5tts_multilingual_support() # Install phonemization for Polish/multilingual F5-TTS
|
||||
installer.install_indexts_text_processing() # Install IndexTTS-2 text normalization with fallback
|
||||
installer.install_russian_text_stresser_support() # Install lightweight Russian stress package for Official 23-Lang
|
||||
|
||||
@@ -11,7 +11,7 @@ except ImportError:
|
||||
pass
|
||||
|
||||
# Version and constants
|
||||
VERSION = "5.0.0"
|
||||
VERSION = "5.1.0"
|
||||
IS_DEV = False # Set to False for release builds
|
||||
VERSION_DISPLAY = f"v{VERSION}" + (" (dev)" if IS_DEV else "")
|
||||
SEPARATOR = "=" * 70
|
||||
@@ -161,6 +161,14 @@ except Exception as e:
|
||||
print(f"❌ Echo-TTS Engine failed: {e}")
|
||||
ECHO_TTS_ENGINE_AVAILABLE = False
|
||||
|
||||
try:
|
||||
dots_tts_engine_module = load_node_module("dots_tts_engine_node", "engines/dots_tts_engine_node.py")
|
||||
DotsTTSEngineNode = dots_tts_engine_module.DotsTTSEngineNode
|
||||
DOTS_TTS_ENGINE_AVAILABLE = True
|
||||
except Exception as e:
|
||||
print(f"❌ Dots TTS Engine failed: {e}")
|
||||
DOTS_TTS_ENGINE_AVAILABLE = False
|
||||
|
||||
try:
|
||||
chatterbox_official_23lang_engine_module = load_node_module("chatterbox_official_23lang_engine_node", "engines/chatterbox_official_23lang_engine_node.py")
|
||||
ChatterBoxOfficial23LangEngineNode = chatterbox_official_23lang_engine_module.ChatterBoxOfficial23LangEngineNode
|
||||
@@ -597,6 +605,10 @@ if ECHO_TTS_ENGINE_AVAILABLE:
|
||||
NODE_CLASS_MAPPINGS["EchoTTSEngineNode"] = EchoTTSEngineNode
|
||||
NODE_DISPLAY_NAME_MAPPINGS["EchoTTSEngineNode"] = "⚙️ Echo-TTS Engine"
|
||||
|
||||
if DOTS_TTS_ENGINE_AVAILABLE:
|
||||
NODE_CLASS_MAPPINGS["DotsTTSEngineNode"] = DotsTTSEngineNode
|
||||
NODE_DISPLAY_NAME_MAPPINGS["DotsTTSEngineNode"] = "⚙️ Dots TTS Engine"
|
||||
|
||||
if QWEN3_TTS_VOICE_DESIGNER_AVAILABLE:
|
||||
NODE_CLASS_MAPPINGS["Qwen3TTSVoiceDesignerNode"] = Qwen3TTSVoiceDesignerNode
|
||||
NODE_DISPLAY_NAME_MAPPINGS["Qwen3TTSVoiceDesignerNode"] = "🎨 Qwen3-TTS Voice Designer"
|
||||
|
||||
@@ -0,0 +1,343 @@
|
||||
"""
|
||||
Dots TTS processor.
|
||||
|
||||
Handles chunking, character switching, pause tags, and orchestration for Dots TTS.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from typing import Any, Dict, List, Tuple, Union
|
||||
|
||||
import torch
|
||||
|
||||
current_dir = os.path.dirname(__file__)
|
||||
nodes_dir = os.path.dirname(current_dir)
|
||||
project_root = os.path.dirname(nodes_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
from utils.audio.chunk_timing import ChunkTimingHelper
|
||||
from utils.audio.edit_post_processor import process_segments as apply_edit_post_processing
|
||||
from engines.dots_tts.languages import format_dots_language_display, normalize_dots_language
|
||||
from utils.text.character_parser import character_parser
|
||||
from utils.text.pause_processor import PauseTagProcessor
|
||||
from utils.text.segment_parameters import ParameterValidator, apply_segment_parameters
|
||||
from utils.text.step_audio_editx_special_tags import get_edit_tags_for_segment
|
||||
from utils.voice.discovery import get_available_characters, get_character_mapping, voice_discovery
|
||||
|
||||
|
||||
class DotsTTSProcessor:
|
||||
"""Internal processor for Dots TTS generation."""
|
||||
|
||||
SAMPLE_RATE = 48000
|
||||
|
||||
def __init__(self, adapter, engine_config: Dict[str, Any]):
|
||||
self.adapter = adapter
|
||||
self.config = engine_config.copy() if engine_config else {}
|
||||
|
||||
def update_config(self, new_config: Dict[str, Any]):
|
||||
self.config = new_config.copy() if new_config else {}
|
||||
self.adapter.update_config(self.config)
|
||||
|
||||
@staticmethod
|
||||
def _voice_log_note(voice_ref: Dict[str, Any]) -> str:
|
||||
if not isinstance(voice_ref, dict):
|
||||
return " [⚠️ No voice reference - will use default]"
|
||||
audio_path = voice_ref.get("audio_path") or voice_ref.get("audio")
|
||||
if not audio_path:
|
||||
return " [⚠️ No voice reference - will use default]"
|
||||
reference_text = voice_ref.get("reference_text")
|
||||
if reference_text:
|
||||
return f" [ref text: {len(reference_text)} chars]"
|
||||
return ""
|
||||
|
||||
def _log_generation_text(
|
||||
self,
|
||||
character_name: str,
|
||||
text_content: str,
|
||||
language: str,
|
||||
voice_ref: Dict[str, Any],
|
||||
chunk_count: int,
|
||||
parameter_log: str = "",
|
||||
show_text_content: bool = True,
|
||||
) -> None:
|
||||
voice_note = self._voice_log_note(voice_ref)
|
||||
print(f"🎭 Dots TTS - Generating for '{character_name}' (Language: {language}){voice_note}:")
|
||||
if parameter_log:
|
||||
print(f"🎛️ Dots TTS params: {parameter_log}")
|
||||
if show_text_content:
|
||||
print("=" * 60)
|
||||
print(text_content)
|
||||
print("=" * 60)
|
||||
if chunk_count > 1:
|
||||
print(
|
||||
f"📝 Chunking {character_name}'s text into {chunk_count} chunks "
|
||||
f"(Language: {language}){voice_note}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _format_parameter_log(filtered_params: Dict[str, Any], current_config: Dict[str, Any], current_seed: int) -> str:
|
||||
if not filtered_params:
|
||||
return ""
|
||||
|
||||
ordered_parts = []
|
||||
if "seed" in filtered_params:
|
||||
ordered_parts.append(f"seed={current_seed}")
|
||||
|
||||
for key in ("num_steps", "guidance_scale", "speaker_scale", "max_generate_length"):
|
||||
if key in filtered_params:
|
||||
ordered_parts.append(f"{key}={current_config.get(key)}")
|
||||
|
||||
for key in ("normalize_text", "language"):
|
||||
if key in filtered_params:
|
||||
value = current_config.get(key)
|
||||
if key == "language":
|
||||
value = format_dots_language_display(value)
|
||||
ordered_parts.append(f"{key}={value}")
|
||||
|
||||
remaining_keys = [
|
||||
key for key in filtered_params.keys()
|
||||
if key not in {"seed", "num_steps", "guidance_scale", "speaker_scale", "max_generate_length", "normalize_text", "language"}
|
||||
]
|
||||
for key in remaining_keys:
|
||||
ordered_parts.append(f"{key}={current_config.get(key, filtered_params[key])}")
|
||||
|
||||
return ", ".join(ordered_parts)
|
||||
|
||||
def _setup_character_parser(self, text: str):
|
||||
configured_language = str(self.config.get("language", "auto") or "auto").strip()
|
||||
normalized_language = normalize_dots_language(configured_language)
|
||||
if normalized_language is None:
|
||||
resolved_language = "en"
|
||||
elif normalized_language == "auto_detect":
|
||||
resolved_language = "en"
|
||||
else:
|
||||
resolved_language = normalized_language.lower()
|
||||
|
||||
character_parser.language_resolver.default_language = resolved_language
|
||||
character_parser.default_language = resolved_language
|
||||
|
||||
character_tags = re.findall(r"\[([^\]]+)\]", text or "")
|
||||
characters_from_tags = []
|
||||
for tag in character_tags:
|
||||
if not tag.startswith("pause:"):
|
||||
characters_from_tags.append(tag.split("|")[0].strip())
|
||||
|
||||
available_chars = get_available_characters()
|
||||
character_aliases = voice_discovery.get_character_aliases()
|
||||
|
||||
all_available = set()
|
||||
if available_chars:
|
||||
all_available.update(available_chars)
|
||||
for alias, target in character_aliases.items():
|
||||
all_available.add(alias.lower())
|
||||
all_available.add(target.lower())
|
||||
for char in characters_from_tags:
|
||||
all_available.add(char.lower())
|
||||
all_available.add("narrator")
|
||||
|
||||
character_parser.set_available_characters(list(all_available))
|
||||
|
||||
char_lang_defaults = voice_discovery.get_character_language_defaults()
|
||||
for char, lang in char_lang_defaults.items():
|
||||
character_parser.set_character_language_default(char, lang)
|
||||
|
||||
character_parser.reset_session_cache()
|
||||
|
||||
@staticmethod
|
||||
def _should_apply_segment_language(seg: Any, base_config: Dict[str, Any]) -> bool:
|
||||
segment_language = getattr(seg, "language", None)
|
||||
if not segment_language:
|
||||
return False
|
||||
|
||||
if getattr(seg, "explicit_language", False):
|
||||
return True
|
||||
|
||||
global_language = normalize_dots_language(base_config.get("language", "auto"))
|
||||
parser_default = normalize_dots_language(character_parser.default_language)
|
||||
segment_language_code = normalize_dots_language(segment_language)
|
||||
|
||||
# Character parser always resolves untagged text to its fallback default
|
||||
# language. For Dots we must not feed that fallback back into generation,
|
||||
# or Auto/None collapse into English cache/runtime behavior.
|
||||
if segment_language_code == parser_default:
|
||||
return False
|
||||
|
||||
return segment_language_code != global_language
|
||||
|
||||
def process_text(
|
||||
self,
|
||||
text: str,
|
||||
voice_mapping: Dict[str, Any],
|
||||
seed: int,
|
||||
enable_chunking: bool = True,
|
||||
max_chars_per_chunk: int = 400,
|
||||
chunk_combination_method: str = "auto",
|
||||
silence_between_chunks_ms: int = 100,
|
||||
enable_audio_cache: bool = True,
|
||||
apply_edit_postprocessing: bool = True,
|
||||
show_text_logging: bool = True,
|
||||
) -> List[Dict[str, Any]]:
|
||||
self._setup_character_parser(text)
|
||||
|
||||
narrator_info = voice_mapping.get("narrator", {})
|
||||
narrator_voice = narrator_info if isinstance(narrator_info, dict) else {"audio": narrator_info}
|
||||
|
||||
base_config = self.config.copy()
|
||||
segment_objects = character_parser.parse_text_segments(text)
|
||||
if not segment_objects:
|
||||
segment_objects = character_parser.parse_text_segments("narrator " + text)
|
||||
|
||||
characters = list({seg.character for seg in segment_objects if seg.character})
|
||||
character_mapping = get_character_mapping(characters, engine_type="audio_only")
|
||||
|
||||
segment_records: List[Dict[str, Any]] = []
|
||||
for seg in segment_objects:
|
||||
segment_text = (seg.text or "").strip()
|
||||
if not segment_text:
|
||||
continue
|
||||
|
||||
segment_params = seg.parameters if seg.parameters else {}
|
||||
current_config = base_config
|
||||
current_seed = seed
|
||||
filtered_params: Dict[str, Any] = {}
|
||||
if segment_params:
|
||||
filtered_params = ParameterValidator.filter_parameters_for_engine(segment_params, "dots_tts")
|
||||
current_config = apply_segment_parameters(base_config, segment_params, "dots_tts")
|
||||
if "seed" in current_config:
|
||||
current_seed = int(current_config.get("seed", seed))
|
||||
|
||||
if self._should_apply_segment_language(seg, base_config):
|
||||
current_config = current_config.copy()
|
||||
current_config["language"] = seg.language
|
||||
global_language = normalize_dots_language(base_config.get("language", "auto"))
|
||||
segment_language_code = normalize_dots_language(seg.language)
|
||||
if segment_language_code != global_language:
|
||||
print(
|
||||
" 🌍 Dots TTS language switched to: "
|
||||
f"{format_dots_language_display(seg.language)}"
|
||||
)
|
||||
|
||||
self.adapter.update_config(current_config)
|
||||
segment_language = format_dots_language_display(current_config.get("language", "Auto"))
|
||||
parameter_log = self._format_parameter_log(filtered_params, current_config, current_seed)
|
||||
|
||||
voice_ref = narrator_voice.copy()
|
||||
char_name = seg.character or "narrator"
|
||||
if char_name != "narrator" and char_name in voice_mapping:
|
||||
char_info = voice_mapping[char_name]
|
||||
if isinstance(char_info, dict):
|
||||
voice_ref = char_info.copy()
|
||||
elif char_name != "narrator":
|
||||
audio_path, ref_text = character_mapping.get(char_name, (None, None))
|
||||
if audio_path:
|
||||
voice_ref = {
|
||||
"audio_path": audio_path,
|
||||
"reference_text": ref_text or "",
|
||||
}
|
||||
|
||||
def _generate_chunks(text_content: str, edit_tags: list):
|
||||
if enable_chunking:
|
||||
from utils.text.chunking import ImprovedChatterBoxChunker
|
||||
|
||||
max_chars = ImprovedChatterBoxChunker.validate_chunking_params(max_chars_per_chunk)
|
||||
chunks = ImprovedChatterBoxChunker.split_into_chunks(text_content, max_chars=max_chars)
|
||||
else:
|
||||
chunks = [text_content]
|
||||
|
||||
self._log_generation_text(
|
||||
character_name=char_name,
|
||||
text_content=text_content,
|
||||
language=segment_language,
|
||||
voice_ref=voice_ref,
|
||||
chunk_count=len(chunks),
|
||||
parameter_log=parameter_log,
|
||||
show_text_content=show_text_logging,
|
||||
)
|
||||
|
||||
for chunk_idx, chunk in enumerate(chunks):
|
||||
audio = self.adapter.generate_single(
|
||||
text=chunk,
|
||||
voice_ref=voice_ref,
|
||||
seed=current_seed + chunk_idx,
|
||||
enable_audio_cache=enable_audio_cache,
|
||||
character_name=char_name,
|
||||
)
|
||||
if audio.dim() == 1:
|
||||
audio = audio.unsqueeze(0)
|
||||
segment_records.append(
|
||||
{
|
||||
"waveform": audio.cpu(),
|
||||
"sample_rate": self.SAMPLE_RATE,
|
||||
"text": chunk,
|
||||
"edit_tags": edit_tags if chunk_idx == 0 else [],
|
||||
}
|
||||
)
|
||||
|
||||
if PauseTagProcessor.has_pause_tags(segment_text):
|
||||
raw_pause_segments, _ = PauseTagProcessor.parse_pause_tags(segment_text)
|
||||
for frag_type, frag_content in raw_pause_segments:
|
||||
if frag_type == "text":
|
||||
frag_clean, frag_edit_tags = get_edit_tags_for_segment(frag_content)
|
||||
frag_clean = frag_clean.strip()
|
||||
if frag_clean:
|
||||
_generate_chunks(frag_clean, frag_edit_tags)
|
||||
elif frag_type == "pause":
|
||||
silence = PauseTagProcessor.create_silence_segment(
|
||||
frag_content,
|
||||
self.SAMPLE_RATE,
|
||||
torch.device("cpu"),
|
||||
torch.float32,
|
||||
)
|
||||
if silence.dim() == 1:
|
||||
silence = silence.unsqueeze(0)
|
||||
segment_records.append(
|
||||
{
|
||||
"waveform": silence,
|
||||
"sample_rate": self.SAMPLE_RATE,
|
||||
"text": f"[pause:{frag_content}s]",
|
||||
"edit_tags": [],
|
||||
}
|
||||
)
|
||||
else:
|
||||
clean_text, edit_tags = get_edit_tags_for_segment(segment_text)
|
||||
_generate_chunks(clean_text, edit_tags)
|
||||
|
||||
if apply_edit_postprocessing and segment_records and any(seg["edit_tags"] for seg in segment_records):
|
||||
segment_records = apply_edit_post_processing(segment_records, engine_config=base_config)
|
||||
|
||||
return segment_records
|
||||
|
||||
def combine_audio_segments(
|
||||
self,
|
||||
segments: List[Dict[str, Any]],
|
||||
method: str = "auto",
|
||||
silence_ms: int = 100,
|
||||
original_text: str = "",
|
||||
return_info: bool = False,
|
||||
) -> Union[torch.Tensor, Tuple[torch.Tensor, Dict[str, Any]]]:
|
||||
if not segments:
|
||||
empty = torch.zeros(0, dtype=torch.float32)
|
||||
if return_info:
|
||||
return empty, {}
|
||||
return empty
|
||||
|
||||
audio_segments = [seg["waveform"] for seg in segments]
|
||||
text_chunks = [seg.get("text", "") for seg in segments]
|
||||
text_length = len(" ".join(text_chunks))
|
||||
|
||||
combined_audio, chunk_info = ChunkTimingHelper.combine_audio_with_timing(
|
||||
audio_segments=audio_segments,
|
||||
combination_method=method,
|
||||
silence_ms=silence_ms,
|
||||
crossfade_duration=0.1,
|
||||
sample_rate=self.SAMPLE_RATE,
|
||||
text_length=text_length,
|
||||
original_text=original_text,
|
||||
text_chunks=text_chunks,
|
||||
)
|
||||
|
||||
if return_info:
|
||||
return combined_audio, chunk_info
|
||||
return combined_audio
|
||||
@@ -0,0 +1,418 @@
|
||||
"""
|
||||
Dots TTS SRT processor.
|
||||
|
||||
Handles full SRT subtitle processing for Dots TTS.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import comfy.model_management as model_management
|
||||
import torch
|
||||
|
||||
current_dir = os.path.dirname(__file__)
|
||||
nodes_dir = os.path.dirname(current_dir)
|
||||
project_root = os.path.dirname(nodes_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
from engines.adapters.dots_tts_adapter import DotsTTSEngineAdapter
|
||||
|
||||
|
||||
class DotsTTSSRTProcessor:
|
||||
"""Complete SRT processor for Dots TTS."""
|
||||
|
||||
SAMPLE_RATE = 48000
|
||||
|
||||
def __init__(self, node_instance, config: Dict[str, Any]):
|
||||
self.node_instance = node_instance
|
||||
self.config = config.copy() if config else {}
|
||||
self.adapter = DotsTTSEngineAdapter(self.config)
|
||||
self._tts_processor = None
|
||||
self.srt_available = False
|
||||
self.SRTParser = None
|
||||
self.SRTSubtitle = None
|
||||
self.SRTParseError = None
|
||||
self._load_srt_modules()
|
||||
|
||||
def _load_srt_modules(self):
|
||||
try:
|
||||
from utils.system.import_manager import import_manager
|
||||
|
||||
success, modules, msg = import_manager.import_srt_modules()
|
||||
if not success:
|
||||
print(f"⚠️ Dots TTS SRT: SRT module not available: {msg}")
|
||||
return
|
||||
|
||||
self.SRTParser = modules.get("SRTParser")
|
||||
self.SRTSubtitle = modules.get("SRTSubtitle")
|
||||
self.SRTParseError = modules.get("SRTParseError")
|
||||
if self.SRTParser is None:
|
||||
print("⚠️ Dots TTS SRT: SRT parser not available")
|
||||
return
|
||||
self.srt_available = True
|
||||
except Exception as e:
|
||||
print(f"⚠️ Dots TTS SRT: Failed to load SRT modules: {e}")
|
||||
|
||||
@property
|
||||
def processor(self):
|
||||
if self._tts_processor is None:
|
||||
processor_path = os.path.join(current_dir, "dots_tts_processor.py")
|
||||
spec = importlib.util.spec_from_file_location("dots_tts_processor_module", processor_path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
self._tts_processor = module.DotsTTSProcessor(self.adapter, self.config)
|
||||
return self._tts_processor
|
||||
|
||||
def update_config(self, new_config: Dict[str, Any]):
|
||||
self.config = new_config.copy() if new_config else {}
|
||||
self.adapter.update_config(self.config)
|
||||
if self._tts_processor is not None:
|
||||
self._tts_processor.update_config(self.config)
|
||||
|
||||
def process_srt_content(
|
||||
self,
|
||||
srt_content: str,
|
||||
voice_mapping: Dict[str, Any],
|
||||
seed: int,
|
||||
timing_mode: str,
|
||||
timing_params: Dict[str, Any],
|
||||
enable_audio_cache: bool = True,
|
||||
) -> Tuple[Dict[str, Any], str, str, str]:
|
||||
if not self.srt_available:
|
||||
raise ImportError("SRT support not available - missing required SRT parser")
|
||||
|
||||
self._check_interrupt()
|
||||
srt_parser = self.SRTParser()
|
||||
subtitles = srt_parser.parse_srt_content(srt_content, allow_overlaps=True)
|
||||
|
||||
from utils.timing.overlap_detection import SRTOverlapHandler
|
||||
|
||||
has_overlaps = SRTOverlapHandler.detect_overlaps(subtitles)
|
||||
current_timing_mode, mode_switched = SRTOverlapHandler.handle_smart_natural_fallback(
|
||||
timing_mode, has_overlaps, "Dots TTS SRT"
|
||||
)
|
||||
|
||||
print(f"🚀 Dots TTS SRT: Processing {len(subtitles)} subtitles in {current_timing_mode} mode")
|
||||
audio_segments, adjustments = self._process_all_subtitles(
|
||||
subtitles,
|
||||
voice_mapping,
|
||||
seed,
|
||||
enable_audio_cache=enable_audio_cache,
|
||||
)
|
||||
self._check_interrupt()
|
||||
|
||||
final_audio, final_adjustments, stretch_method = self._assemble_final_audio(
|
||||
audio_segments,
|
||||
subtitles,
|
||||
current_timing_mode,
|
||||
timing_params,
|
||||
adjustments,
|
||||
)
|
||||
if final_adjustments is not None:
|
||||
adjustments = final_adjustments
|
||||
|
||||
timing_report = self._generate_timing_report(
|
||||
subtitles,
|
||||
adjustments,
|
||||
current_timing_mode,
|
||||
has_overlaps,
|
||||
mode_switched,
|
||||
timing_mode if mode_switched else None,
|
||||
stretch_method,
|
||||
)
|
||||
adjusted_srt_string = self._generate_adjusted_srt_string(subtitles, adjustments, current_timing_mode)
|
||||
|
||||
total_duration = final_audio.shape[-1] / self.SAMPLE_RATE
|
||||
mode_info = current_timing_mode
|
||||
if mode_switched:
|
||||
mode_info = f"{current_timing_mode} (switched from {timing_mode} due to overlaps)"
|
||||
info = (
|
||||
f"Generated {total_duration:.1f}s Dots TTS SRT-timed audio from {len(subtitles)} subtitles "
|
||||
f"using {mode_info} mode"
|
||||
)
|
||||
|
||||
if final_audio.dim() == 1:
|
||||
final_audio = final_audio.unsqueeze(0).unsqueeze(0)
|
||||
elif final_audio.dim() == 2:
|
||||
final_audio = final_audio.unsqueeze(0)
|
||||
|
||||
audio_output = {"waveform": final_audio, "sample_rate": self.SAMPLE_RATE}
|
||||
return audio_output, info, timing_report, adjusted_srt_string
|
||||
|
||||
def _check_interrupt(
|
||||
self,
|
||||
subtitle_index: Optional[int] = None,
|
||||
total_subtitles: Optional[int] = None,
|
||||
character: Optional[str] = None,
|
||||
):
|
||||
if model_management.interrupt_processing:
|
||||
if subtitle_index is not None and total_subtitles is not None:
|
||||
if character:
|
||||
raise InterruptedError(
|
||||
f"Dots TTS SRT generation interrupted at subtitle "
|
||||
f"{subtitle_index + 1}/{total_subtitles}, character '{character}'"
|
||||
)
|
||||
raise InterruptedError(
|
||||
f"Dots TTS SRT generation interrupted at subtitle {subtitle_index + 1}/{total_subtitles}"
|
||||
)
|
||||
raise InterruptedError("Dots TTS SRT generation interrupted by user")
|
||||
|
||||
def _process_all_subtitles(
|
||||
self,
|
||||
subtitles: List,
|
||||
voice_mapping: Dict[str, Any],
|
||||
seed: int,
|
||||
enable_audio_cache: bool = True,
|
||||
) -> Tuple[List[torch.Tensor], List[Dict]]:
|
||||
audio_segments: List[torch.Tensor] = []
|
||||
adjustments: List[Dict[str, Any]] = []
|
||||
all_segments_for_editing: List[Dict[str, Any]] = []
|
||||
|
||||
subtitle_lengths = [max(1, len((sub.text or "").strip())) for sub in subtitles]
|
||||
for i, sub in enumerate(subtitles):
|
||||
self._check_interrupt(i, len(subtitles))
|
||||
|
||||
text = sub.text.strip()
|
||||
if not text:
|
||||
target_duration = sub.duration
|
||||
silence = torch.zeros(1, int(target_duration * self.SAMPLE_RATE))
|
||||
audio_segments.append(silence)
|
||||
adjustments.append(
|
||||
{
|
||||
"index": i,
|
||||
"segment_index": i,
|
||||
"sequence": sub.sequence,
|
||||
"natural_duration": target_duration,
|
||||
"target_start": sub.start_time,
|
||||
"target_end": sub.end_time,
|
||||
"target_duration": target_duration,
|
||||
"start_time": sub.start_time,
|
||||
"end_time": sub.end_time,
|
||||
"stretch_factor": 1.0,
|
||||
"needs_stretching": False,
|
||||
"stretch_type": "none",
|
||||
"adjustment": 0.0,
|
||||
"adjusted_start": sub.start_time,
|
||||
"adjusted_end": sub.end_time,
|
||||
"adjusted_duration": target_duration,
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
print(f"📖 Dots TTS SRT Subtitle {i + 1}/{len(subtitles)}")
|
||||
segment_records = self.processor.process_text(
|
||||
text=text,
|
||||
voice_mapping=voice_mapping,
|
||||
seed=seed + i,
|
||||
enable_chunking=False,
|
||||
enable_audio_cache=enable_audio_cache,
|
||||
apply_edit_postprocessing=False,
|
||||
)
|
||||
|
||||
subtitle_has_edit_tags = any(seg.get("edit_tags") for seg in segment_records)
|
||||
if subtitle_has_edit_tags:
|
||||
for seg_idx, seg in enumerate(segment_records):
|
||||
waveform = seg["waveform"]
|
||||
if waveform.dim() == 1:
|
||||
waveform = waveform.unsqueeze(0)
|
||||
elif waveform.dim() == 3:
|
||||
waveform = waveform.squeeze(0)
|
||||
all_segments_for_editing.append(
|
||||
{
|
||||
**seg,
|
||||
"waveform": waveform.cpu(),
|
||||
"subtitle_index": i,
|
||||
"segment_index": seg_idx,
|
||||
}
|
||||
)
|
||||
audio_segments.append(None)
|
||||
else:
|
||||
if len(segment_records) > 1:
|
||||
audio, _ = self.processor.combine_audio_segments(
|
||||
segments=segment_records,
|
||||
method="auto",
|
||||
silence_ms=0,
|
||||
original_text=text,
|
||||
return_info=True,
|
||||
)
|
||||
elif segment_records:
|
||||
audio = segment_records[0]["waveform"]
|
||||
else:
|
||||
audio = torch.zeros(1, 0)
|
||||
|
||||
if audio.dim() == 1:
|
||||
audio = audio.unsqueeze(0)
|
||||
elif audio.dim() == 3:
|
||||
audio = audio.squeeze(0)
|
||||
audio_segments.append(audio.cpu())
|
||||
|
||||
target_start = sub.start_time
|
||||
target_end = sub.end_time
|
||||
target_duration = target_end - target_start
|
||||
adjustments.append(
|
||||
{
|
||||
"index": i,
|
||||
"segment_index": i,
|
||||
"sequence": sub.sequence,
|
||||
"natural_duration": 0.0,
|
||||
"target_start": target_start,
|
||||
"target_end": target_end,
|
||||
"target_duration": target_duration,
|
||||
"start_time": target_start,
|
||||
"end_time": target_end,
|
||||
"stretch_factor": 1.0,
|
||||
"needs_stretching": False,
|
||||
"stretch_type": "none",
|
||||
"adjustment": 0.0,
|
||||
"adjusted_start": target_start,
|
||||
"adjusted_end": target_end,
|
||||
"adjusted_duration": 0.0,
|
||||
}
|
||||
)
|
||||
|
||||
if not subtitle_has_edit_tags:
|
||||
audio = audio_segments[i]
|
||||
natural_duration = audio.shape[-1] / self.SAMPLE_RATE
|
||||
stretch_factor = target_duration / natural_duration if natural_duration > 0 else 1.0
|
||||
adjustments[i].update(
|
||||
{
|
||||
"natural_duration": natural_duration,
|
||||
"stretch_factor": stretch_factor,
|
||||
"needs_stretching": abs(stretch_factor - 1.0) > 0.05,
|
||||
"stretch_type": "compress" if stretch_factor < 1.0 else "expand" if stretch_factor > 1.0 else "none",
|
||||
"adjustment": natural_duration - target_duration,
|
||||
"adjusted_duration": natural_duration,
|
||||
}
|
||||
)
|
||||
print(f"✅ Dots TTS Subtitle {i + 1}/{len(subtitles)}: {natural_duration:.2f}s (expected {target_duration:.2f}s)")
|
||||
else:
|
||||
print(f"✅ Dots TTS Subtitle {i + 1}/{len(subtitles)}: queued for batch edit processing")
|
||||
|
||||
if all_segments_for_editing:
|
||||
from utils.audio.edit_post_processor import process_segments as apply_edit_post_processing
|
||||
|
||||
processed_segments = apply_edit_post_processing(all_segments_for_editing, engine_config=self.config)
|
||||
grouped_segments: Dict[int, List[Dict[str, Any]]] = {}
|
||||
for seg in processed_segments:
|
||||
grouped_segments.setdefault(seg["subtitle_index"], []).append(seg)
|
||||
|
||||
for subtitle_index, segments in grouped_segments.items():
|
||||
segments = sorted(segments, key=lambda item: item.get("segment_index", 0))
|
||||
if len(segments) > 1:
|
||||
combined_audio, _ = self.processor.combine_audio_segments(
|
||||
segments=segments,
|
||||
method="auto",
|
||||
silence_ms=0,
|
||||
original_text=subtitles[subtitle_index].text,
|
||||
return_info=True,
|
||||
)
|
||||
else:
|
||||
combined_audio = segments[0]["waveform"]
|
||||
if combined_audio.dim() == 1:
|
||||
combined_audio = combined_audio.unsqueeze(0)
|
||||
elif combined_audio.dim() == 3:
|
||||
combined_audio = combined_audio.squeeze(0)
|
||||
combined_audio = combined_audio.cpu()
|
||||
audio_segments[subtitle_index] = combined_audio
|
||||
|
||||
target_duration = adjustments[subtitle_index]["target_duration"]
|
||||
natural_duration = combined_audio.shape[-1] / self.SAMPLE_RATE
|
||||
stretch_factor = target_duration / natural_duration if natural_duration > 0 else 1.0
|
||||
adjustments[subtitle_index].update(
|
||||
{
|
||||
"natural_duration": natural_duration,
|
||||
"stretch_factor": stretch_factor,
|
||||
"needs_stretching": abs(stretch_factor - 1.0) > 0.05,
|
||||
"stretch_type": "compress" if stretch_factor < 1.0 else "expand" if stretch_factor > 1.0 else "none",
|
||||
"adjustment": natural_duration - target_duration,
|
||||
"adjusted_duration": natural_duration,
|
||||
}
|
||||
)
|
||||
|
||||
return audio_segments, adjustments
|
||||
|
||||
def _assemble_final_audio(self, audio_segments, subtitles, timing_mode, timing_params, adjustments):
|
||||
if timing_mode == "stretch_to_fit":
|
||||
from engines.chatterbox.audio_timing import TimedAudioAssembler
|
||||
|
||||
assembler = TimedAudioAssembler(self.SAMPLE_RATE)
|
||||
target_timings = [(sub.start_time, sub.end_time) for sub in subtitles]
|
||||
fade_duration = timing_params.get("fade_for_StretchToFit", 0.01)
|
||||
final_audio, stretch_method_used = assembler.assemble_timed_audio(
|
||||
audio_segments,
|
||||
target_timings,
|
||||
fade_duration=fade_duration,
|
||||
)
|
||||
return final_audio, None, stretch_method_used
|
||||
|
||||
if timing_mode == "pad_with_silence":
|
||||
from utils.timing.assembly import AudioAssemblyEngine
|
||||
|
||||
assembler = AudioAssemblyEngine(self.SAMPLE_RATE)
|
||||
final_audio = assembler.assemble_with_overlaps(audio_segments, subtitles, torch.device("cpu"))
|
||||
return final_audio, None, None
|
||||
|
||||
if timing_mode == "concatenate":
|
||||
from utils.timing.assembly import AudioAssemblyEngine
|
||||
from utils.timing.engine import TimingEngine
|
||||
|
||||
timing_engine = TimingEngine(self.SAMPLE_RATE)
|
||||
assembler = AudioAssemblyEngine(self.SAMPLE_RATE)
|
||||
new_adjustments = timing_engine.calculate_concatenation_adjustments(audio_segments, subtitles)
|
||||
fade_duration = timing_params.get("fade_for_StretchToFit", 0.01)
|
||||
final_audio = assembler.assemble_concatenation(audio_segments, fade_duration)
|
||||
return final_audio, new_adjustments, None
|
||||
|
||||
from utils.timing.assembly import AudioAssemblyEngine
|
||||
from utils.timing.engine import TimingEngine
|
||||
|
||||
timing_engine = TimingEngine(self.SAMPLE_RATE)
|
||||
assembler = AudioAssemblyEngine(self.SAMPLE_RATE)
|
||||
smart_adjustments, processed_segments = timing_engine.calculate_smart_timing_adjustments(
|
||||
audio_segments,
|
||||
subtitles,
|
||||
timing_params.get("timing_tolerance", 2.0),
|
||||
timing_params.get("max_stretch_ratio", 1.0),
|
||||
timing_params.get("min_stretch_ratio", 0.5),
|
||||
torch.device("cpu"),
|
||||
)
|
||||
final_audio = assembler.assemble_smart_natural(
|
||||
audio_segments,
|
||||
processed_segments,
|
||||
smart_adjustments,
|
||||
subtitles,
|
||||
torch.device("cpu"),
|
||||
)
|
||||
return final_audio, smart_adjustments, None
|
||||
|
||||
def _generate_timing_report(
|
||||
self,
|
||||
subtitles,
|
||||
adjustments,
|
||||
current_timing_mode,
|
||||
has_overlaps=False,
|
||||
mode_switched=False,
|
||||
original_mode=None,
|
||||
stretch_method=None,
|
||||
) -> str:
|
||||
from utils.timing.reporting import SRTReportGenerator
|
||||
|
||||
reporter = SRTReportGenerator()
|
||||
return reporter.generate_timing_report(
|
||||
subtitles,
|
||||
adjustments,
|
||||
current_timing_mode,
|
||||
has_overlaps,
|
||||
mode_switched,
|
||||
original_mode,
|
||||
stretch_method,
|
||||
)
|
||||
|
||||
def _generate_adjusted_srt_string(self, subtitles, adjustments, timing_mode: str) -> str:
|
||||
from utils.timing.reporting import SRTReportGenerator
|
||||
|
||||
reporter = SRTReportGenerator()
|
||||
return reporter.generate_adjusted_srt_string(subtitles, adjustments, timing_mode)
|
||||
@@ -0,0 +1,161 @@
|
||||
"""
|
||||
Dots TTS engine configuration node.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
|
||||
current_dir = os.path.dirname(__file__)
|
||||
nodes_dir = os.path.dirname(current_dir)
|
||||
project_root = os.path.dirname(nodes_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
base_node_path = os.path.join(nodes_dir, "base", "base_node.py")
|
||||
base_spec = importlib.util.spec_from_file_location("base_node_module", base_node_path)
|
||||
base_module = importlib.util.module_from_spec(base_spec)
|
||||
sys.modules["base_node_module"] = base_module
|
||||
base_spec.loader.exec_module(base_module)
|
||||
BaseTTSNode = base_module.BaseTTSNode
|
||||
|
||||
from engines.dots_tts.languages import DOTS_LANGUAGE_OPTIONS
|
||||
|
||||
|
||||
class DotsTTSEngineNode(BaseTTSNode):
|
||||
"""Dots TTS engine configuration node."""
|
||||
|
||||
MODEL_VARIANTS = [
|
||||
"dots.tts-soar",
|
||||
"dots.tts-mf",
|
||||
"dots.tts-base",
|
||||
]
|
||||
LANGUAGE_OPTIONS = DOTS_LANGUAGE_OPTIONS
|
||||
TEMPLATE_MODE_OPTIONS = [
|
||||
"TTS",
|
||||
"Instruction TTS",
|
||||
]
|
||||
TEMPLATE_NAME_BY_MODE = {
|
||||
"TTS": "tts",
|
||||
"Instruction TTS": "instruction_tts",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def NAME(cls):
|
||||
return "⚙️ Dots TTS Engine"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"model_variant": (cls.MODEL_VARIANTS, {
|
||||
"default": "dots.tts-soar",
|
||||
"tooltip": "Official Dots checkpoint.\n• dots.tts-soar: best zero-shot voice cloning\n• dots.tts-mf: fastest distilled MeanFlow checkpoint; use 4 steps\n• dots.tts-base: pretrained baseline"
|
||||
}),
|
||||
"device": (["auto", "cuda", "cpu"], {
|
||||
"default": "auto",
|
||||
"tooltip": "Device to run Dots TTS on.\n• auto: use the best available device\n• cuda: NVIDIA GPU\n• cpu: CPU-only (very slow for 2B model)"
|
||||
}),
|
||||
"language": (cls.LANGUAGE_OPTIONS, {
|
||||
"default": "Auto",
|
||||
"tooltip": "Official Dots language tag.\n• Auto: let Dots detect language from text\n• None: disable explicit language tagging\n• Full language names: force the model-side language tag\n\nVoice-cloning note: reference/narrator audio works best when the generated language matches the language spoken in the reference. Cross-language cloning can degrade quality, accent, and speaker similarity."
|
||||
}),
|
||||
"num_steps": ("INT", {
|
||||
"default": 10,
|
||||
"min": 1,
|
||||
"max": 64,
|
||||
"step": 1,
|
||||
"tooltip": "Flow-matching sampling steps.\nOfficial recommendation: 10-32 for base/soar, 4 for mf."
|
||||
}),
|
||||
"guidance_scale": ("FLOAT", {
|
||||
"default": 1.2,
|
||||
"min": 0.0,
|
||||
"max": 5.0,
|
||||
"step": 0.1,
|
||||
"tooltip": "Official CFG guidance scale.\nHigher values can increase energy and instability."
|
||||
}),
|
||||
"speaker_scale": ("FLOAT", {
|
||||
"default": 1.5,
|
||||
"min": 0.0,
|
||||
"max": 5.0,
|
||||
"step": 0.1,
|
||||
"tooltip": "Official speaker embedding scale for reference audio conditioning."
|
||||
}),
|
||||
"max_generate_length": ("INT", {
|
||||
"default": 500,
|
||||
"min": 32,
|
||||
"max": 1024,
|
||||
"step": 32,
|
||||
"tooltip": "Official maximum audio patch budget.\n500 is roughly 160 seconds of output budget."
|
||||
}),
|
||||
},
|
||||
"optional": {
|
||||
"template_mode": (cls.TEMPLATE_MODE_OPTIONS, {
|
||||
"default": "TTS",
|
||||
"tooltip": "Official non-standard Dots template mode from upstream.\n• TTS: standard Dots speech synthesis template\n• Instruction TTS: uses the same text field as normal TTS\nUpstream does not clearly document what behavior difference this mode is meant to produce.\nIt may yield different results, or little noticeable difference, versus standard TTS. Needs testing."
|
||||
}),
|
||||
"precision": (["auto", "bfloat16", "float16", "float32"], {
|
||||
"default": "auto",
|
||||
"tooltip": "Runtime precision.\n• auto: bfloat16 on newer CUDA GPUs, else float16, cpu -> float32"
|
||||
}),
|
||||
"normalize_text": ("BOOLEAN", {
|
||||
"default": False,
|
||||
"tooltip": "Use Dots native text normalization before inference."
|
||||
}),
|
||||
"optimize": ("BOOLEAN", {
|
||||
"default": False,
|
||||
"tooltip": "Enable official Dots runtime optimization / warmup.\nFirst load is slower; steady-state inference is faster."
|
||||
}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("TTS_ENGINE",)
|
||||
RETURN_NAMES = ("TTS_engine",)
|
||||
FUNCTION = "create_engine_config"
|
||||
CATEGORY = "TTS Audio Suite/⚙️ Engines"
|
||||
|
||||
def create_engine_config(
|
||||
self,
|
||||
model_variant: str,
|
||||
device: str,
|
||||
language: str,
|
||||
num_steps: int,
|
||||
guidance_scale: float,
|
||||
speaker_scale: float,
|
||||
max_generate_length: int,
|
||||
template_mode: str = "TTS",
|
||||
precision: str = "auto",
|
||||
normalize_text: bool = False,
|
||||
optimize: bool = False,
|
||||
) -> tuple:
|
||||
template_name = self.TEMPLATE_NAME_BY_MODE.get(template_mode, "tts")
|
||||
config = {
|
||||
"engine_type": "dots_tts",
|
||||
"model_variant": model_variant,
|
||||
"device": device,
|
||||
"precision": precision,
|
||||
"language": language,
|
||||
"num_steps": int(num_steps),
|
||||
"guidance_scale": float(guidance_scale),
|
||||
"speaker_scale": float(speaker_scale),
|
||||
"max_generate_length": int(max_generate_length),
|
||||
"normalize_text": bool(normalize_text),
|
||||
"optimize": bool(optimize),
|
||||
"template_name": template_name,
|
||||
}
|
||||
|
||||
print(f"⚙️ Dots TTS: Configured on {device}")
|
||||
print(f" Model: {model_variant} | Language: {language} | Mode: {template_mode} | Precision: {precision}")
|
||||
print(
|
||||
f" Settings: steps={num_steps}, guidance_scale={guidance_scale}, "
|
||||
f"speaker_scale={speaker_scale}, max_generate_length={max_generate_length}"
|
||||
)
|
||||
if normalize_text or optimize:
|
||||
print(f" Advanced: normalize_text={normalize_text}, optimize={optimize}")
|
||||
|
||||
engine_data = {
|
||||
"engine_type": "dots_tts",
|
||||
"config": config,
|
||||
"capabilities": ["tts"],
|
||||
}
|
||||
return (engine_data,)
|
||||
@@ -0,0 +1,206 @@
|
||||
"""
|
||||
Unified Sound Effects Node - engine-agnostic text-to-sound generation.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from typing import Any, Dict, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
# Add project root directory to path for imports
|
||||
current_dir = os.path.dirname(__file__)
|
||||
nodes_dir = os.path.dirname(current_dir)
|
||||
project_root = os.path.dirname(nodes_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
# Load base_node module directly
|
||||
base_node_path = os.path.join(nodes_dir, "base", "base_node.py")
|
||||
base_spec = importlib.util.spec_from_file_location("base_node_module", base_node_path)
|
||||
base_module = importlib.util.module_from_spec(base_spec)
|
||||
sys.modules["base_node_module"] = base_module
|
||||
base_spec.loader.exec_module(base_module)
|
||||
|
||||
BaseTTSNode = base_module.BaseTTSNode
|
||||
|
||||
from engines.adapters import DotsTTSEngineAdapter
|
||||
|
||||
|
||||
class UnifiedSoundEffectsNode(BaseTTSNode):
|
||||
"""Unified text-to-sound node for engines that support non-speech audio generation."""
|
||||
|
||||
ENGINE_ADAPTERS = {
|
||||
"dots_tts": DotsTTSEngineAdapter,
|
||||
}
|
||||
|
||||
TEMPLATE_OVERRIDES = {
|
||||
"dots_tts": "text_to_audio",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def NAME(cls):
|
||||
return "🎧 Sound Effects"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"engine": ("TTS_ENGINE", {
|
||||
"tooltip": "Engine configuration for a backend that supports sound-effects generation. Currently Dots text_to_audio is supported."
|
||||
}),
|
||||
"description": ("STRING", {
|
||||
"multiline": True,
|
||||
"default": "Heavy rain on a metal rooftop with distant thunder and occasional wind gusts.",
|
||||
"tooltip": "Describe the non-speech audio you want. This node is for sound-effects / scene audio generation, not spoken TTS."
|
||||
}),
|
||||
"seed": ("INT", {
|
||||
"default": 1, "min": 0, "max": 2**32 - 1,
|
||||
"tooltip": "Seed for reproducible generation. Set 0 for random output."
|
||||
}),
|
||||
},
|
||||
"optional": {
|
||||
"enable_audio_cache": ("BOOLEAN", {
|
||||
"default": True,
|
||||
"tooltip": "Cache generated sound effects in memory so identical reruns are instant."
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("AUDIO", "STRING")
|
||||
RETURN_NAMES = ("audio", "generation_info")
|
||||
FUNCTION = "generate_sound"
|
||||
CATEGORY = "TTS Audio Suite/🎧 Sound Effects"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._cached_adapters = {}
|
||||
|
||||
def _validate_engine(self, engine: Dict[str, Any]) -> Dict[str, Any]:
|
||||
if not isinstance(engine, dict):
|
||||
raise ValueError("Engine input must be a configuration dict")
|
||||
|
||||
capabilities = engine.get("capabilities", [])
|
||||
if capabilities and "sound_effects" not in capabilities:
|
||||
raise ValueError("Engine does not support Sound Effects")
|
||||
|
||||
config = engine.get("config", engine)
|
||||
engine_type = engine.get("engine_type") or config.get("engine_type")
|
||||
if engine_type not in self.ENGINE_ADAPTERS:
|
||||
raise ValueError(
|
||||
f"Engine '{engine_type}' is not wired for Sound Effects yet. "
|
||||
"This family exists now, but only Dots text_to_audio is implemented."
|
||||
)
|
||||
return config.copy()
|
||||
|
||||
def _get_adapter(self, engine_type: str, config: Dict[str, Any]):
|
||||
stable_params = {
|
||||
"engine_type": engine_type,
|
||||
"model_variant": config.get("model_variant"),
|
||||
"device": config.get("device"),
|
||||
"precision": config.get("precision"),
|
||||
"optimize": config.get("optimize"),
|
||||
"max_generate_length": config.get("max_generate_length"),
|
||||
"template_name": config.get("template_name"),
|
||||
}
|
||||
cache_key = hashlib.md5(str(sorted(stable_params.items())).encode()).hexdigest()[:12]
|
||||
adapter = self._cached_adapters.get(cache_key)
|
||||
if adapter is None:
|
||||
adapter_cls = self.ENGINE_ADAPTERS[engine_type]
|
||||
adapter = adapter_cls(config=config)
|
||||
self._cached_adapters[cache_key] = adapter
|
||||
else:
|
||||
adapter.update_config(config)
|
||||
return adapter
|
||||
|
||||
def _generate_with_dots(
|
||||
self,
|
||||
config: Dict[str, Any],
|
||||
description: str,
|
||||
seed: int,
|
||||
enable_audio_cache: bool,
|
||||
):
|
||||
config["template_name"] = self.TEMPLATE_OVERRIDES["dots_tts"]
|
||||
adapter = self._get_adapter("dots_tts", config)
|
||||
return adapter.generate_single(
|
||||
text=description,
|
||||
voice_ref=None,
|
||||
seed=seed,
|
||||
enable_audio_cache=enable_audio_cache,
|
||||
character_name="sound_effects",
|
||||
)
|
||||
|
||||
def generate_sound(
|
||||
self,
|
||||
engine: Dict[str, Any],
|
||||
description: str,
|
||||
seed: int,
|
||||
enable_audio_cache: bool = True,
|
||||
) -> Tuple[Dict[str, Any], str]:
|
||||
config = self._validate_engine(engine)
|
||||
engine_type = config.get("engine_type", engine.get("engine_type", "unknown"))
|
||||
clean_description = (description or "").strip()
|
||||
if not clean_description:
|
||||
raise ValueError("Sound Effects description cannot be empty")
|
||||
|
||||
print(f"🎧 Sound Effects: Starting {engine_type} generation")
|
||||
print("============================================================")
|
||||
print(clean_description)
|
||||
print("============================================================")
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
if engine_type == "dots_tts":
|
||||
audio_tensor = self._generate_with_dots(
|
||||
config=config,
|
||||
description=clean_description,
|
||||
seed=int(seed or 0),
|
||||
enable_audio_cache=bool(enable_audio_cache),
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported Sound Effects engine: {engine_type}")
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
sample_rate = 48000 if engine_type == "dots_tts" else 44100
|
||||
audio_duration = 0.0
|
||||
if hasattr(audio_tensor, "shape") and audio_tensor.shape[-1] > 0:
|
||||
audio_duration = float(audio_tensor.shape[-1]) / float(sample_rate)
|
||||
|
||||
print(
|
||||
f"✅ Sound Effects: {engine_type} generation complete "
|
||||
f"({audio_duration:.2f}s audio in {elapsed:.1f}s)"
|
||||
)
|
||||
|
||||
info = (
|
||||
f"Sound Effects generated successfully\n"
|
||||
f"Engine: {engine_type}\n"
|
||||
f"Mode: {self.TEMPLATE_OVERRIDES.get(engine_type, 'custom')}\n"
|
||||
f"Duration: {audio_duration:.2f}s\n"
|
||||
f"Elapsed: {elapsed:.2f}s\n"
|
||||
f"Seed: {int(seed or 0)}"
|
||||
)
|
||||
|
||||
if not isinstance(audio_tensor, torch.Tensor):
|
||||
audio_tensor = torch.tensor(audio_tensor, dtype=torch.float32)
|
||||
audio_tensor = audio_tensor.detach().float().cpu()
|
||||
if audio_tensor.dim() == 1:
|
||||
audio_tensor = audio_tensor.unsqueeze(0).unsqueeze(0)
|
||||
elif audio_tensor.dim() == 2:
|
||||
audio_tensor = audio_tensor.unsqueeze(0)
|
||||
|
||||
return ({
|
||||
"waveform": audio_tensor,
|
||||
"sample_rate": sample_rate,
|
||||
}, info)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"UnifiedSoundEffectsNode": UnifiedSoundEffectsNode
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"UnifiedSoundEffectsNode": "🎧 Sound Effects"
|
||||
}
|
||||
@@ -202,6 +202,12 @@ Hello! This is unified SRT TTS with character switching.
|
||||
stable_params['use_cuda_graphs'] = config.get('use_cuda_graphs', False)
|
||||
stable_params['compile_mode'] = config.get('compile_mode', 'default')
|
||||
|
||||
if engine_type == "dots_tts":
|
||||
stable_params['model_variant'] = config.get('model_variant', 'dots.tts-soar')
|
||||
stable_params['precision'] = config.get('precision', 'auto')
|
||||
stable_params['optimize'] = config.get('optimize', False)
|
||||
stable_params['max_generate_length'] = config.get('max_generate_length', 500)
|
||||
|
||||
if engine_type == "moss_tts":
|
||||
stable_params['model_variant'] = config.get('model_variant', 'MOSS-TTS-Local-Transformer')
|
||||
stable_params['multi_speaker_mode'] = config.get('multi_speaker_mode', 'Custom Character Switching')
|
||||
@@ -492,12 +498,57 @@ Hello! This is unified SRT TTS with character switching.
|
||||
'timestamp': time.time()
|
||||
}
|
||||
return engine_instance
|
||||
|
||||
elif engine_type == "vibevoice":
|
||||
# Import and create the VibeVoice SRT processor using absolute import
|
||||
vibevoice_srt_processor_path = os.path.join(nodes_dir, "vibevoice", "vibevoice_srt_processor.py")
|
||||
vibevoice_srt_spec = importlib.util.spec_from_file_location("vibevoice_srt_processor_module", vibevoice_srt_processor_path)
|
||||
vibevoice_srt_module = importlib.util.module_from_spec(vibevoice_srt_spec)
|
||||
|
||||
elif engine_type == "dots_tts":
|
||||
dots_tts_srt_processor_path = os.path.join(nodes_dir, "dots_tts", "dots_tts_srt_processor.py")
|
||||
dots_tts_srt_spec = importlib.util.spec_from_file_location("dots_tts_srt_processor_module", dots_tts_srt_processor_path)
|
||||
dots_tts_srt_module = importlib.util.module_from_spec(dots_tts_srt_spec)
|
||||
dots_tts_srt_spec.loader.exec_module(dots_tts_srt_module)
|
||||
|
||||
DotsTTSSRTProcessor = dots_tts_srt_module.DotsTTSSRTProcessor
|
||||
|
||||
class DotsTTSSRTWrapper:
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
self.processor = DotsTTSSRTProcessor(self, config)
|
||||
|
||||
def update_config(self, new_config):
|
||||
self.config = new_config.copy()
|
||||
self.processor.update_config(new_config)
|
||||
|
||||
def process_with_error_handling(self, func):
|
||||
try:
|
||||
return func()
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
||||
def format_audio_output(self, audio_tensor, sample_rate):
|
||||
if audio_tensor.is_cuda:
|
||||
audio_tensor = audio_tensor.cpu()
|
||||
if audio_tensor.dim() == 1:
|
||||
audio_tensor = audio_tensor.unsqueeze(0).unsqueeze(0)
|
||||
elif audio_tensor.dim() == 2:
|
||||
audio_tensor = audio_tensor.unsqueeze(0)
|
||||
return {"waveform": audio_tensor, "sample_rate": sample_rate}
|
||||
|
||||
def check_interrupt(self):
|
||||
if model_management.interrupt_processing:
|
||||
raise InterruptedError("Dots TTS SRT processing interrupted by user")
|
||||
|
||||
engine_instance = DotsTTSSRTWrapper(config)
|
||||
|
||||
import time
|
||||
self._cached_engine_instances[cache_key] = {
|
||||
'instance': engine_instance,
|
||||
'timestamp': time.time()
|
||||
}
|
||||
return engine_instance
|
||||
|
||||
elif engine_type == "vibevoice":
|
||||
# Import and create the VibeVoice SRT processor using absolute import
|
||||
vibevoice_srt_processor_path = os.path.join(nodes_dir, "vibevoice", "vibevoice_srt_processor.py")
|
||||
vibevoice_srt_spec = importlib.util.spec_from_file_location("vibevoice_srt_processor_module", vibevoice_srt_processor_path)
|
||||
vibevoice_srt_module = importlib.util.module_from_spec(vibevoice_srt_spec)
|
||||
vibevoice_srt_spec.loader.exec_module(vibevoice_srt_module)
|
||||
|
||||
VibeVoiceSRTProcessor = vibevoice_srt_module.VibeVoiceSRTProcessor
|
||||
@@ -1109,6 +1160,31 @@ Hello! This is unified SRT TTS with character switching.
|
||||
enable_audio_cache=enable_audio_cache
|
||||
)
|
||||
|
||||
elif engine_type == "dots_tts":
|
||||
timing_params = {
|
||||
'fade_for_StretchToFit': fade_for_StretchToFit,
|
||||
'max_stretch_ratio': max_stretch_ratio,
|
||||
'min_stretch_ratio': min_stretch_ratio,
|
||||
'timing_tolerance': timing_tolerance
|
||||
}
|
||||
|
||||
voice_mapping = {}
|
||||
if audio_tensor is not None or audio_path:
|
||||
voice_mapping['narrator'] = {
|
||||
'audio': audio_tensor,
|
||||
'audio_path': audio_path,
|
||||
'reference_text': reference_text if reference_text else "",
|
||||
}
|
||||
|
||||
result = engine_instance.processor.process_srt_content(
|
||||
srt_content=srt_content,
|
||||
voice_mapping=voice_mapping,
|
||||
seed=seed,
|
||||
timing_mode=timing_mode,
|
||||
timing_params=timing_params,
|
||||
enable_audio_cache=enable_audio_cache
|
||||
)
|
||||
|
||||
elif engine_type == "vibevoice":
|
||||
# Use the VibeVoice SRT processor from the wrapper instance
|
||||
voice_mapping = {"narrator": audio_tensor} if audio_tensor else {}
|
||||
|
||||
@@ -204,6 +204,12 @@ Back to the main narrator voice for the conclusion.""",
|
||||
stable_params['use_cuda_graphs'] = config.get('use_cuda_graphs', False)
|
||||
stable_params['compile_mode'] = config.get('compile_mode', 'default')
|
||||
|
||||
if engine_type == "dots_tts":
|
||||
stable_params['model_variant'] = config.get('model_variant', 'dots.tts-soar')
|
||||
stable_params['precision'] = config.get('precision', 'auto')
|
||||
stable_params['optimize'] = config.get('optimize', False)
|
||||
stable_params['max_generate_length'] = config.get('max_generate_length', 500)
|
||||
|
||||
# For MOSS-TTS, include model identity and load-time options.
|
||||
if engine_type == "moss_tts":
|
||||
stable_params['model_variant'] = config.get('model_variant', 'MOSS-TTS-Local-Transformer')
|
||||
@@ -515,6 +521,36 @@ Back to the main narrator voice for the conclusion.""",
|
||||
|
||||
return engine_instance
|
||||
|
||||
elif engine_type == "dots_tts":
|
||||
from engines.adapters.dots_tts_adapter import DotsTTSEngineAdapter
|
||||
dots_tts_processor_path = os.path.join(nodes_dir, "dots_tts", "dots_tts_processor.py")
|
||||
dots_tts_processor_spec = importlib.util.spec_from_file_location("dots_tts_processor_module", dots_tts_processor_path)
|
||||
dots_tts_processor_module = importlib.util.module_from_spec(dots_tts_processor_spec)
|
||||
dots_tts_processor_spec.loader.exec_module(dots_tts_processor_module)
|
||||
|
||||
DotsTTSProcessor = dots_tts_processor_module.DotsTTSProcessor
|
||||
|
||||
class DotsTTSWrapper:
|
||||
def __init__(self, cfg):
|
||||
self.config = cfg.copy()
|
||||
self.adapter = DotsTTSEngineAdapter(self.config)
|
||||
self.processor = DotsTTSProcessor(self.adapter, self.config)
|
||||
|
||||
def update_config(self, new_config):
|
||||
self.config = new_config.copy()
|
||||
self.adapter.update_config(new_config)
|
||||
self.processor.update_config(new_config)
|
||||
|
||||
engine_instance = DotsTTSWrapper(config)
|
||||
|
||||
import time
|
||||
self._cached_engine_instances[cache_key] = {
|
||||
'instance': engine_instance,
|
||||
'timestamp': time.time()
|
||||
}
|
||||
|
||||
return engine_instance
|
||||
|
||||
elif engine_type == "qwen3_tts":
|
||||
# Create Qwen3-TTS processor instance
|
||||
# Use global nodes_dir (already defined at module level)
|
||||
@@ -853,6 +889,8 @@ Back to the main narrator voice for the conclusion.""",
|
||||
lang_code = "local"
|
||||
elif language.lower() == "auto":
|
||||
lang_code = "auto"
|
||||
elif language.lower() == "none":
|
||||
lang_code = "none"
|
||||
else:
|
||||
# Standard language codes - take first 2 chars
|
||||
lang_code = language.lower()[:2] # en, fr, de, etc.
|
||||
@@ -1283,6 +1321,56 @@ Back to the main narrator voice for the conclusion.""",
|
||||
formatted_audio = AudioProcessingUtils.format_for_comfyui(combined_audio, 44100)
|
||||
result = (formatted_audio, generation_info)
|
||||
|
||||
elif engine_type == "dots_tts":
|
||||
import re
|
||||
from utils.audio.chunk_timing import ChunkTimingHelper
|
||||
|
||||
if not hasattr(engine_instance, "processor"):
|
||||
dots_tts_processor_path = os.path.join(nodes_dir, "dots_tts", "dots_tts_processor.py")
|
||||
dots_tts_processor_spec = importlib.util.spec_from_file_location("dots_tts_processor_module", dots_tts_processor_path)
|
||||
dots_tts_processor_module = importlib.util.module_from_spec(dots_tts_processor_spec)
|
||||
dots_tts_processor_spec.loader.exec_module(dots_tts_processor_module)
|
||||
DotsTTSProcessor = dots_tts_processor_module.DotsTTSProcessor
|
||||
engine_instance.processor = DotsTTSProcessor(engine_instance.adapter, config)
|
||||
engine_instance.processor.update_config(config)
|
||||
|
||||
voice_mapping = {}
|
||||
if audio_tensor is not None or audio_path:
|
||||
voice_mapping['narrator'] = {
|
||||
'audio': audio_tensor,
|
||||
'audio_path': audio_path,
|
||||
'reference_text': reference_text or '',
|
||||
}
|
||||
|
||||
segment_records = engine_instance.processor.process_text(
|
||||
text=text,
|
||||
voice_mapping=voice_mapping,
|
||||
seed=seed,
|
||||
enable_chunking=enable_chunking,
|
||||
max_chars_per_chunk=max_chars_per_chunk,
|
||||
chunk_combination_method=chunk_combination_method,
|
||||
silence_between_chunks_ms=silence_between_chunks_ms,
|
||||
enable_audio_cache=enable_audio_cache
|
||||
)
|
||||
|
||||
combined_audio, chunk_info = engine_instance.processor.combine_audio_segments(
|
||||
segments=segment_records,
|
||||
method=chunk_combination_method,
|
||||
silence_ms=silence_between_chunks_ms,
|
||||
original_text=text,
|
||||
return_info=True
|
||||
)
|
||||
|
||||
total_duration = combined_audio.shape[-1] / 48000.0 if combined_audio.numel() else 0.0
|
||||
clean_text = re.sub(r'\[.*?\]', '', text)
|
||||
text_length = len(clean_text)
|
||||
base_info = f"Generated {total_duration:.1f}s audio from {text_length} characters (Dots TTS, narrator: {char_display})"
|
||||
base_info += "\n🎭 Character switching and pause tags enabled"
|
||||
generation_info = ChunkTimingHelper.enhance_generation_info(f"✅ {base_info}", chunk_info)
|
||||
|
||||
formatted_audio = AudioProcessingUtils.format_for_comfyui(combined_audio, 48000)
|
||||
result = (formatted_audio, generation_info)
|
||||
|
||||
elif engine_type == "moss_tts":
|
||||
import re
|
||||
from utils.audio.chunk_timing import ChunkTimingHelper
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
[project]
|
||||
name = "tts_audio_suite"
|
||||
description = "TTS Audio Suite - Universal multi-engine TTS extension for ComfyUI with unified architecture supporting IndexTTS-2, ChatterBox, Chatterbox Multilingual TTS (Official 23-Lang), F5-TTS, Higgs Audio 2, VibeVoice, and RVC engines. It has character voice management, SRT subtitle TTS support, and audio processing capabilities."
|
||||
version = "5.0.0"
|
||||
version = "5.1.0"
|
||||
license = {file = "LICENSE"}
|
||||
|
||||
[project.urls]
|
||||
|
||||
+8
-5
@@ -27,12 +27,15 @@ sounddevice>=0.4.0
|
||||
jieba
|
||||
pypinyin
|
||||
unidecode
|
||||
phonemizer # IPA phonemization for multilingual TTS (requires espeak system dependency)
|
||||
omegaconf>=2.3.0
|
||||
phonemizer # IPA phonemization for multilingual TTS (requires espeak system dependency)
|
||||
omegaconf>=2.3.0
|
||||
transformers>=5.3.0,<6.0.0 # Main environment for Higgs Audio v3 / Step Audio EditX / MOSS-TTS. Legacy Transformers 4 engines use isolated runtimes.
|
||||
|
||||
# ML utilities (safe)
|
||||
accelerate
|
||||
loguru
|
||||
langcodes
|
||||
lingua-language-detector
|
||||
|
||||
# ML utilities (safe)
|
||||
accelerate
|
||||
datasets
|
||||
requests
|
||||
dacite
|
||||
|
||||
+42
-3
@@ -521,6 +521,42 @@ class EchoTTSCacheKeyGenerator(CacheKeyGenerator):
|
||||
return hashlib.md5(cache_string.encode()).hexdigest()
|
||||
|
||||
|
||||
class DotsTTSCacheKeyGenerator(CacheKeyGenerator):
|
||||
"""Cache key generator for Dots TTS engine."""
|
||||
|
||||
def generate_cache_key(self, **params) -> str:
|
||||
guidance_scale = params.get('guidance_scale', 1.2)
|
||||
speaker_scale = params.get('speaker_scale', 1.5)
|
||||
|
||||
if isinstance(guidance_scale, (int, float)):
|
||||
guidance_scale = round(float(guidance_scale), 3)
|
||||
if isinstance(speaker_scale, (int, float)):
|
||||
speaker_scale = round(float(speaker_scale), 3)
|
||||
|
||||
cache_data = {
|
||||
'text': params.get('text', ''),
|
||||
'audio_component': params.get('audio_component', ''),
|
||||
'prompt_text': params.get('prompt_text', ''),
|
||||
'model_variant': params.get('model_variant', 'dots.tts-soar'),
|
||||
'language': params.get('language', 'auto_detect'),
|
||||
'template_name': params.get('template_name', 'tts'),
|
||||
'num_steps': params.get('num_steps', 10),
|
||||
'guidance_scale': guidance_scale,
|
||||
'speaker_scale': speaker_scale,
|
||||
'normalize_text': params.get('normalize_text', False),
|
||||
'max_generate_length': params.get('max_generate_length', 500),
|
||||
'precision': params.get('precision', 'auto'),
|
||||
'optimize': params.get('optimize', False),
|
||||
'seed': params.get('seed', 0),
|
||||
'device': params.get('device', 'auto'),
|
||||
'character': params.get('character', 'narrator'),
|
||||
'engine': 'dots_tts',
|
||||
}
|
||||
|
||||
cache_string = str(sorted(cache_data.items()))
|
||||
return hashlib.md5(cache_string.encode()).hexdigest()
|
||||
|
||||
|
||||
class AudioCache:
|
||||
"""Unified audio cache manager for all TTS engines."""
|
||||
|
||||
@@ -536,6 +572,7 @@ class AudioCache:
|
||||
'index_tts': IndexTTSCacheKeyGenerator(),
|
||||
'cosyvoice': CosyVoiceCacheKeyGenerator(),
|
||||
'qwen3_tts': Qwen3TTSCacheKeyGenerator(),
|
||||
'dots_tts': DotsTTSCacheKeyGenerator(),
|
||||
'moss_tts': MossTTSCacheKeyGenerator(),
|
||||
'echo_tts': EchoTTSCacheKeyGenerator()
|
||||
}
|
||||
@@ -608,10 +645,12 @@ class AudioCache:
|
||||
num_samples = audio_tensor.numel()
|
||||
|
||||
# Use engine-specific sample rates
|
||||
if engine_type in ('f5tts', 'step_audio_editx', 'qwen3_tts', 'moss_tts', 'higgs_audio_v3'):
|
||||
if engine_type == 'dots_tts':
|
||||
sample_rate = 48000
|
||||
elif engine_type in ('f5tts', 'step_audio_editx', 'qwen3_tts', 'moss_tts', 'higgs_audio_v3'):
|
||||
sample_rate = 24000
|
||||
elif engine_type in ('index_tts', 'cosyvoice'):
|
||||
sample_rate = 22050
|
||||
elif engine_type in ('index_tts', 'cosyvoice'):
|
||||
sample_rate = 22050
|
||||
else:
|
||||
sample_rate = 44100
|
||||
return num_samples / sample_rate
|
||||
|
||||
@@ -131,7 +131,7 @@ class StepAudioEditXHandler(BaseEngineHandler):
|
||||
return 0
|
||||
|
||||
|
||||
class HiggsAudioHandler(BaseEngineHandler):
|
||||
class HiggsAudioHandler(BaseEngineHandler):
|
||||
"""Specialized handler for Higgs Audio engine with CUDA graphs"""
|
||||
|
||||
def model_unload(self, wrapper, memory_to_free=None, unpatch_weights=True) -> bool:
|
||||
@@ -141,18 +141,65 @@ class HiggsAudioHandler(BaseEngineHandler):
|
||||
"""
|
||||
result = super().model_unload(wrapper, memory_to_free, unpatch_weights)
|
||||
|
||||
if result:
|
||||
# Mark model as invalid to force fresh CUDA graph initialization
|
||||
wrapper._is_valid_for_reuse = False
|
||||
print(f"🔄 Marked Higgs Audio model for cache refresh (CUDA graphs will be recreated)")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# Engine registry
|
||||
if result:
|
||||
# Mark model as invalid to force fresh CUDA graph initialization
|
||||
wrapper._is_valid_for_reuse = False
|
||||
print(f"🔄 Marked Higgs Audio model for cache refresh (CUDA graphs will be recreated)")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
class DotsTTSHandler(BaseEngineHandler):
|
||||
"""Handler for Dots TTS, which needs explicit runtime teardown to free compile caches."""
|
||||
|
||||
def partially_unload(self, wrapper, device: str, memory_to_free: int) -> int:
|
||||
if not wrapper._is_loaded_on_gpu:
|
||||
print(f"⚠️ Skipping unload: model already marked as not on GPU")
|
||||
return 0
|
||||
|
||||
model = wrapper._model_ref() if wrapper._model_ref else None
|
||||
if model is None:
|
||||
print(f"⚠️ Model reference is None, cannot unload")
|
||||
return 0
|
||||
|
||||
try:
|
||||
if hasattr(model, "unload_runtime"):
|
||||
model.unload_runtime()
|
||||
print(
|
||||
f"🔄 Unloaded {wrapper.model_info.model_type} model "
|
||||
f"({wrapper.model_info.engine}) runtime and freed "
|
||||
f"{wrapper._memory_size // 1024 // 1024}MB"
|
||||
)
|
||||
elif hasattr(model, "to"):
|
||||
model.to(device)
|
||||
print(
|
||||
f"🔄 Moved {wrapper.model_info.model_type} model "
|
||||
f"({wrapper.model_info.engine}) to {device}, freed "
|
||||
f"{wrapper._memory_size // 1024 // 1024}MB"
|
||||
)
|
||||
|
||||
wrapper.current_device = device
|
||||
wrapper._is_loaded_on_gpu = False
|
||||
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
if hasattr(torch.cuda, "ipc_collect"):
|
||||
try:
|
||||
torch.cuda.ipc_collect()
|
||||
except Exception:
|
||||
pass
|
||||
gc.collect()
|
||||
return wrapper._memory_size
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error unloading {wrapper.model_info.engine} model: {e}")
|
||||
return 0
|
||||
|
||||
|
||||
# Engine registry
|
||||
_ENGINE_HANDLERS = {
|
||||
'chatterbox': BaseEngineHandler(),
|
||||
'chatterbox_official_23lang': BaseEngineHandler(),
|
||||
'dots_tts': DotsTTSHandler(),
|
||||
'f5tts': BaseEngineHandler(),
|
||||
'echo_tts': BaseEngineHandler(),
|
||||
'higgs_audio': HiggsAudioHandler(),
|
||||
|
||||
@@ -142,6 +142,13 @@ ENGINE_REGISTRY: Dict[str, EngineCapabilities] = {
|
||||
default_runtime_profile="vibevoice_transformers4_shared",
|
||||
),
|
||||
|
||||
"dots_tts": EngineCapabilities(
|
||||
supports_voice_conversion=False,
|
||||
multilingual_model_switching=False,
|
||||
can_corrupt_on_reload=False,
|
||||
fallback_languages=["English"],
|
||||
),
|
||||
|
||||
"qwen3_asr": EngineCapabilities(
|
||||
supports_voice_conversion=False,
|
||||
multilingual_model_switching=False,
|
||||
|
||||
@@ -1672,9 +1672,37 @@ def register_qwen3_tts_factory():
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to load Qwen3-TTS model: {e}")
|
||||
|
||||
unified_model_interface.register_model_factory("qwen3_tts", "tts", qwen3_tts_factory)
|
||||
|
||||
|
||||
unified_model_interface.register_model_factory("qwen3_tts", "tts", qwen3_tts_factory)
|
||||
|
||||
|
||||
def register_dots_tts_factory():
|
||||
"""Register Dots TTS model factory."""
|
||||
def dots_tts_factory(config: ModelLoadConfig):
|
||||
"""Factory for official Dots TTS models with ComfyUI integration."""
|
||||
model_name = config.model_name or "dots.tts-soar"
|
||||
device = config.device or "auto"
|
||||
additional_params = config.additional_params or {}
|
||||
precision = additional_params.get("precision", "auto")
|
||||
optimize = bool(additional_params.get("optimize", False))
|
||||
max_generate_length = int(additional_params.get("max_generate_length", 500))
|
||||
|
||||
from engines.dots_tts.dots_tts_engine import DotsTTSEngine
|
||||
|
||||
print(f"🔄 Loading Dots TTS model via unified interface: {model_name}")
|
||||
engine = DotsTTSEngine(
|
||||
model_name=model_name,
|
||||
device=device,
|
||||
precision=precision,
|
||||
optimize=optimize,
|
||||
max_generate_length=max_generate_length,
|
||||
)
|
||||
engine._ensure_runtime_loaded()
|
||||
print(f"✅ Dots TTS model '{model_name}' loaded successfully")
|
||||
return engine
|
||||
|
||||
unified_model_interface.register_model_factory("dots_tts", "tts", dots_tts_factory)
|
||||
|
||||
|
||||
def register_qwen3_asr_factory():
|
||||
"""Register Qwen3-ASR model factory"""
|
||||
def qwen3_asr_factory(config: ModelLoadConfig):
|
||||
@@ -1883,6 +1911,7 @@ def initialize_all_factories():
|
||||
register_cosyvoice_factory()
|
||||
register_moss_tts_factory()
|
||||
register_qwen3_tts_factory()
|
||||
register_dots_tts_factory()
|
||||
register_qwen3_asr_factory()
|
||||
register_qwen3_aligner_factory()
|
||||
register_granite_asr_factory()
|
||||
|
||||
@@ -22,6 +22,10 @@ PARAMETER_ALIASES = {
|
||||
'cfg_weight': 'cfg', # cfg_weight alias -> cfg (more universal)
|
||||
'cfgweight': 'cfg',
|
||||
'num_steps': 'num_steps',
|
||||
'guidance_scale': 'guidance_scale',
|
||||
'guidance': 'guidance_scale',
|
||||
'speaker_scale': 'speaker_scale',
|
||||
'speaker_guidance': 'speaker_scale',
|
||||
'cfg_text': 'cfg_scale_text',
|
||||
'cfg_scale_text': 'cfg_scale_text',
|
||||
'cfg_speaker': 'cfg_scale_speaker',
|
||||
@@ -55,6 +59,8 @@ PARAMETER_ALIASES = {
|
||||
'tokens': 'duration_tokens',
|
||||
'duration_tokens': 'duration_tokens',
|
||||
'max_new_tokens': 'max_new_tokens',
|
||||
'max_generate_length': 'max_generate_length',
|
||||
'max_audio_patches': 'max_generate_length',
|
||||
'n_vq': 'n_vq_for_inference',
|
||||
'n_vq_for_inference': 'n_vq_for_inference',
|
||||
'instruction': 'instruction',
|
||||
@@ -72,6 +78,7 @@ PARAMETER_ENGINES = {
|
||||
'seed': {
|
||||
'chatterbox', 'chatterbox_official_23lang', 'f5tts', 'higgs_audio',
|
||||
'higgs_audio_v3', 'vibevoice', 'index_tts', 'step_audio_editx', 'cosyvoice', 'qwen3_tts',
|
||||
'dots_tts',
|
||||
'echo_tts', 'moss_tts'
|
||||
},
|
||||
'temperature': {
|
||||
@@ -82,7 +89,13 @@ PARAMETER_ENGINES = {
|
||||
'f5tts', 'vibevoice', 'index_tts', 'chatterbox', 'chatterbox_official_23lang'
|
||||
},
|
||||
'num_steps': {
|
||||
'echo_tts'
|
||||
'echo_tts', 'dots_tts'
|
||||
},
|
||||
'guidance_scale': {
|
||||
'dots_tts'
|
||||
},
|
||||
'speaker_scale': {
|
||||
'dots_tts'
|
||||
},
|
||||
'cfg_scale_text': {
|
||||
'echo_tts'
|
||||
@@ -150,6 +163,9 @@ PARAMETER_ENGINES = {
|
||||
'max_new_tokens': {
|
||||
'higgs_audio_v3', 'moss_tts'
|
||||
},
|
||||
'max_generate_length': {
|
||||
'dots_tts'
|
||||
},
|
||||
'n_vq_for_inference': {
|
||||
'moss_tts'
|
||||
},
|
||||
@@ -179,6 +195,8 @@ PARAMETER_VALIDATION = {
|
||||
'temperature': (float, 0.1, 2.0, "Randomness/creativity control (lower=more deterministic)"),
|
||||
'cfg': (float, 0.0, 20.0, "Classifier-free guidance strength"),
|
||||
'num_steps': (int, 1, 200, "Number of inference steps"),
|
||||
'guidance_scale': (float, 0.0, 5.0, "Dots TTS classifier-free guidance scale"),
|
||||
'speaker_scale': (float, 0.0, 5.0, "Dots TTS speaker conditioning scale"),
|
||||
'cfg_scale_text': (float, 0.0, 20.0, "CFG scale for text guidance (Echo-TTS)"),
|
||||
'cfg_scale_speaker': (float, 0.0, 20.0, "CFG scale for speaker guidance (Echo-TTS)"),
|
||||
'cfg_min_t': (float, 0.0, 1.0, "CFG minimum t value (Echo-TTS)"),
|
||||
@@ -201,6 +219,7 @@ PARAMETER_VALIDATION = {
|
||||
'audio_repetition_penalty': (float, 0.5, 3.0, "MOSS-TTS audio repetition penalty"),
|
||||
'duration_tokens': (int, 0, 8192, "MOSS-TTS duration hint in audio tokens"),
|
||||
'max_new_tokens': (int, 64, 16384, "Maximum generated tokens"),
|
||||
'max_generate_length': (int, 32, 1024, "Dots TTS maximum audio patch budget"),
|
||||
'n_vq_for_inference': (int, 0, 32, "MOSS-TTS Local Transformer RVQ layers for inference"),
|
||||
'instruction': (str, None, None, "MOSS-TTS whole-segment instruction"),
|
||||
'quality': (str, None, None, "MOSS-TTS whole-segment quality hint"),
|
||||
@@ -217,6 +236,8 @@ PARAMETER_NODE_KEYS = {
|
||||
'temperature': 'temperature',
|
||||
'cfg': {'default': 'cfg_weight', 'f5tts': 'cfg_strength'}, # Engine-specific mapping
|
||||
'num_steps': 'num_steps',
|
||||
'guidance_scale': 'guidance_scale',
|
||||
'speaker_scale': 'speaker_scale',
|
||||
'cfg_scale_text': 'cfg_scale_text',
|
||||
'cfg_scale_speaker': 'cfg_scale_speaker',
|
||||
'cfg_min_t': 'cfg_min_t',
|
||||
@@ -239,6 +260,7 @@ PARAMETER_NODE_KEYS = {
|
||||
'audio_repetition_penalty': 'audio_repetition_penalty',
|
||||
'duration_tokens': 'duration_tokens',
|
||||
'max_new_tokens': 'max_new_tokens',
|
||||
'max_generate_length': 'max_generate_length',
|
||||
'n_vq_for_inference': 'n_vq_for_inference',
|
||||
'instruction': 'instruction',
|
||||
'quality': 'quality',
|
||||
|
||||
Reference in New Issue
Block a user