Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 diodiogod
|
73a282406e | ||
|
diodiogod
|
48608775e0 | ||
|
diodiogod
|
eaaacef869 |
@@ -5,6 +5,24 @@ All notable changes to this project will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [5.6.2] - 2026-07-30
|
||||
|
||||
### Changed
|
||||
|
||||
- Improve F5-TTS fallback so the standard PyTorch attention backend continues working
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fix F5-TTS failing to load with incomplete FlashAttention installations
|
||||
- Fix F5-TTS startup crashes when optional FlashAttention components are missing
|
||||
## [5.6.1] - 2026-07-30
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fix Fish Audio S2 installation in headless Linux environments
|
||||
- Fix missing optional audio libraries preventing Fish Audio S2 setup
|
||||
- Improve Linux and macOS dependency warnings so core TTS installation continues
|
||||
- Correct Fedora package installation guidance
|
||||
## [5.6.0] - 2026-07-25
|
||||
|
||||
### Added
|
||||
|
||||
@@ -44,9 +44,9 @@ The project code is MIT. Model weights carry their own licenses:
|
||||
Echo-TTS CC-BY-NC-SA-4.0 No
|
||||
Fish Audio S2 Pro Fish Audio Research License No
|
||||
Dots TTS Apache-2.0 Yes
|
||||
Audio8 TTS Apache-2.0 Yes
|
||||
DramaBox LTX-2 Community License Conditional
|
||||
OmniVoice Apache-2.0 Yes
|
||||
VoxCPM Apache-2.0 Yes
|
||||
MOSS-TTS Apache-2.0 Yes
|
||||
MOSS-SoundEffect v2 Apache-2.0 Yes
|
||||
RVC MIT (framework); community models vary Varies
|
||||
|
||||
+7
-8
@@ -17,7 +17,7 @@
|
||||
**Key architectural rules:**
|
||||
- Chunking happens in the **processor**, not the adapter (`generate_single()` on adapter = raw single call)
|
||||
- Runtime routing happens through `ModelLoadConfig.runtime_mode` + `runtime_profile`, not ad-hoc subprocess calls
|
||||
- Shared runtime workers are currently used for fragile engine families such as VibeVoice, Qwen3-TTS / ASR, Granite forced alignment, and Higgs Audio 2. Engines that support the modern stack run natively in the main Transformers 5 environment.
|
||||
- Shared runtime workers are currently used for fragile engine families such as VibeVoice, Qwen3-TTS / ASR, Granite forced alignment, Higgs Audio 2, and Audio8 TTS. Engines that support the modern stack run natively in the main Transformers 5 environment.
|
||||
- YAML (`docs/Dev reports/tts_audio_suite_engines.yaml`) is source of truth for engine doc tables → run `python3 scripts/generate_engine_tables.py --readme` to regenerate
|
||||
- Auxiliary YAML (`docs/Dev reports/tts_audio_suite_aux_models.yaml`) is source of truth for helper/post-process model docs → run `python3 scripts/generate_aux_model_docs.py`
|
||||
- All models download to `ComfyUI/models/TTS/<model-name>/`
|
||||
@@ -44,14 +44,14 @@
|
||||
| Echo-TTS | `echo_tts_adapter.py` | `nodes/echo_tts/echo_tts_processor.py` | `echo_tts_srt_processor.py` | `echo_tts_engine_node.py` |
|
||||
| Fish Audio S2 Pro | `fish_audio_s2_adapter.py` | `nodes/fish_audio_s2/fish_audio_s2_processor.py` | `fish_audio_s2_srt_processor.py` | `fish_audio_s2_engine_node.py` |
|
||||
| Dots TTS | `dots_tts_adapter.py` | `nodes/dots_tts/dots_tts_processor.py` | `dots_tts_srt_processor.py` | `dots_tts_engine_node.py` |
|
||||
| Audio8 TTS | `audio8_tts_adapter.py` | `nodes/audio8_tts/audio8_tts_processor.py` | `audio8_tts_srt_processor.py` | `audio8_tts_engine_node.py` |
|
||||
| DramaBox | `dramabox_adapter.py` | `nodes/dramabox/dramabox_processor.py` | `dramabox_srt_processor.py` | `dramabox_engine_node.py` |
|
||||
| OmniVoice | `omnivoice_adapter.py` | `nodes/omnivoice/omnivoice_processor.py` | `omnivoice_srt_processor.py` | `omnivoice_engine_node.py` |
|
||||
| VoxCPM | `voxcpm_adapter.py` | `nodes/voxcpm/voxcpm_processor.py` | `voxcpm_srt_processor.py` | `voxcpm_engine_node.py` |
|
||||
| MOSS-SoundEffect v2 | `moss_soundeffect_v2_adapter.py` | — | — | `moss_soundeffect_v2_engine_node.py` |
|
||||
| RVC | — | `engines/rvc/` | — | `rvc_engine_node.py` |
|
||||
|
||||
**Engine implementations live in:**
|
||||
- `engines/chatterbox/`, `engines/chatterbox_official_23lang/`, `engines/f5tts/`, `engines/higgs_audio/`, `engines/higgs_audio_v3/`, `engines/vibevoice_engine/`, `engines/step_audio_editx/`, `engines/cosyvoice/`, `engines/qwen3_tts/`, `engines/qwen3_asr/`, `engines/moss_tts/`, `engines/moss_soundeffect_v2/`, `engines/granite_asr/`, `engines/echo_tts/`, `engines/fish_audio_s2/`, `engines/dots_tts/`, `engines/dramabox/`, `engines/omnivoice/`, `engines/voxcpm/`, `engines/rvc/`
|
||||
- `engines/chatterbox/`, `engines/chatterbox_official_23lang/`, `engines/f5tts/`, `engines/higgs_audio/`, `engines/higgs_audio_v3/`, `engines/vibevoice_engine/`, `engines/step_audio_editx/`, `engines/cosyvoice/`, `engines/qwen3_tts/`, `engines/qwen3_asr/`, `engines/moss_tts/`, `engines/moss_soundeffect_v2/`, `engines/granite_asr/`, `engines/echo_tts/`, `engines/fish_audio_s2/`, `engines/dots_tts/`, `engines/audio8_tts/`, `engines/dramabox/`, `engines/omnivoice/`, `engines/rvc/`
|
||||
|
||||
## Documentation Files
|
||||
|
||||
@@ -111,10 +111,8 @@
|
||||
- `nodes/unified/asr_transcribe_node.py` - Universal ASR node
|
||||
|
||||
### Shared / Special Nodes
|
||||
- `nodes/unified/model_training_node.py` - Unified engine-routed training entry point
|
||||
- `nodes/training/voxcpm_dataset_prep_node.py` + `voxcpm_training_config_node.py` - VoxCPM2 official-format LoRA dataset/config workflow
|
||||
- `nodes/shared/character_voices_node.py` - Character voice management (NARRATOR_VOICE output)
|
||||
- `nodes/shared/unified_voice_designer_node.py` - Unified Qwen VoiceDesign, MOSS VoiceGenerator, reference-free OmniVoice design, and VoxCPM2 Voice Design
|
||||
- `nodes/shared/unified_voice_designer_node.py` - Unified Qwen VoiceDesign, MOSS VoiceGenerator, and reference-free OmniVoice design
|
||||
- `nodes/shared/save_character_voice_node.py` - Explicit output node for saving any NARRATOR_VOICE into the established voice library
|
||||
- `nodes/omnivoice/omnivoice_instruction_builder_node.py` - OmniVoice voice-design instruction helper with custom visual builder UI
|
||||
- `nodes/text/phoneme_text_normalizer_node.py` - Multilingual text preprocessing
|
||||
@@ -148,13 +146,14 @@
|
||||
- `launcher.py` - runtime bootstrap, venv creation, Windows toolchain env setup
|
||||
- `session.py`, `protocol.py` - JSONL worker transport and message protocol
|
||||
- `bootstrap.py` - shared runtime bootstrap helpers
|
||||
- `vibevoice_proxy.py`, `qwen3_tts_proxy.py`, `qwen3_asr_proxy.py`, `higgs_audio_proxy.py` - parent-process proxies
|
||||
- `workers/` - worker subprocess entrypoints for VibeVoice, Qwen3-TTS, Qwen3-ASR/aligner, Higgs Audio
|
||||
- `vibevoice_proxy.py`, `qwen3_tts_proxy.py`, `qwen3_asr_proxy.py`, `higgs_audio_proxy.py`, `audio8_tts_proxy.py` - parent-process proxies
|
||||
- `workers/` - worker subprocess entrypoints for VibeVoice, Qwen3-TTS, Qwen3-ASR/aligner, Higgs Audio, and Audio8 TTS
|
||||
- Current shared legacy T4 runtime profile is reused by:
|
||||
- VibeVoice / Kugel
|
||||
- Qwen3-TTS
|
||||
- Qwen3-ASR and Granite's optional Qwen forced aligner
|
||||
- Higgs Audio 2
|
||||
- Audio8 TTS
|
||||
|
||||
### Audio (`utils/audio/`)
|
||||
- `processing.py` - Tensor manipulation, normalization, format conversion
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
[![Dynamic TOML Badge][version-shield]][version-url]
|
||||
[](https://ko-fi.com/diogogo)
|
||||
|
||||
# TTS Audio Suite v5.6.0
|
||||
# TTS Audio Suite v5.6.2
|
||||
|
||||
[](https://ko-fi.com/diogogo)
|
||||
|
||||
@@ -41,9 +41,9 @@ Subtitle workflows are still a core focus: the suite can transcribe to SRT, rebu
|
||||
| **Echo-TTS** | 🇺🇸 | ~5.3GB + ~1.8GB | Diffusion-based (~30s best), Force Speaker KV (speaker drift control) |
|
||||
| **Fish Audio S2 Pro** | 🌐 80+ languages | ~10.3GB / ~8.0GB | Free-form sub-word emotion/prosody tags, Native multi-speaker and multi-turn dialogue with dynamic speaker references |
|
||||
| **Dots TTS** | 🇺🇸🇨🇳🇩🇪🇪🇸🇫🇷🇮🇹 +13 | ~6GB | Official auto language detect / language control, SOAR and MeanFlow distilled variants |
|
||||
| **Audio8 TTS** | 🌐 11 recommended languages | ~2.39 GiB | Reference-free TTS + zero-shot/cross-lingual cloning, 44.1kHz output with sampling or greedy decoding |
|
||||
| **DramaBox** | 🇺🇸 | ~16.4GB | Expressive scene prompting and stage directions, Native and SRT-aware duration targeting |
|
||||
| **OmniVoice** | 🌐 600+ languages | ~3.7GB | Inline non-verbal tags and pronunciation overrides, Reference-free voice design |
|
||||
| **VoxCPM** | 🌐 30 languages (V2); EN/ZH legacy | ~5.0GB / ~2.0GB / ~1.6GB | One architecture-gated engine for VoxCPM2, VoxCPM1.5, and legacy VoxCPM-0.5B, VoxCPM2 reference-only, controllable, and ultimate voice cloning |
|
||||
| **MOSS-TTS** | 🇺🇸🇨🇳🇩🇪🇪🇸🇫🇷🇮🇹 +18 | ~8.5GB tokenizer + ~6.1GB/17GB/18GB model | Reference-free voice design with MOSS-VoiceGenerator, Native 1-5 speaker TTSD dialogue |
|
||||
| **MOSS-SoundEffect v2** | 🇺🇸🇨🇳 | ~11.2GB | Durations up to 30 seconds, Native negative prompting, CFG, flow shift, and diffusion-step controls |
|
||||
| **RVC** | 🌐 Any | 100-300MB | Real-time VC, Integrated training workflow |
|
||||
@@ -166,7 +166,6 @@ Start with the **[New Engine Guide Hub](docs/New%20Engines%20Guides/README.md)**
|
||||
- [CosyVoice3 Multilingual Voice Cloning](#cosyvoice3-multilingual-voice-cloning)
|
||||
- [Qwen3-TTS - 4 Model Types with Text-to-Voice Design](#qwen3-tts---4-model-types-with-text-to-voice-design)
|
||||
- [OmniVoice + Visual Tag Builder](#omnivoice--visual-tag-builder)
|
||||
- [VoxCPM - V2, 1.5, and Legacy 0.5B](#voxcpm---v2-15-and-legacy-05b)
|
||||
- [Echo-TTS Voice Cloning](#echo-tts-voice-cloning)
|
||||
- [Phoneme Text Normalizer](#phoneme-text-normalizer)
|
||||
- [Multiline TTS Tag Editor and Per-Segment Parameter Switching](#multiline-tts-tag-editor-and-per-segment-parameter-switching)
|
||||
@@ -206,7 +205,7 @@ Start with the **[New Engine Guide Hub](docs/New%20Engines%20Guides/README.md)**
|
||||
## Features
|
||||
|
||||
- 🎤 **Multi-Engine TTS**
|
||||
- 🎨 **Voice Designer** → Create reusable voices with compatible Qwen3-TTS, MOSS, OmniVoice, and VoxCPM2 engines
|
||||
- 🎨 **Voice Designer** → Create reusable voices with compatible Qwen3-TTS, MOSS, and OmniVoice engines
|
||||
- 🌩️ **Sound Effects** → **[📖 Sound Effects Guide](docs/SOUND_EFFECTS_GUIDE.md)**
|
||||
- 🔄 **Voice Conversion**
|
||||
- ✏️ **ASR Transcription**
|
||||
@@ -937,7 +936,7 @@ Instruct: 用兴奋的语气说话。
|
||||
|
||||
**Voice Designer Node:**
|
||||
|
||||
The shared designer accepts Qwen3-TTS, MOSS-TTS, OmniVoice, or VoxCPM2 engine configurations and outputs the same `NARRATOR_VOICE` format. The voice-design instruction lives on **🎨 Voice Designer**; the engine keeps model and generation settings. Select Qwen VoiceDesign or MOSS VoiceGenerator in the engine's model dropdown, or set OmniVoice or VoxCPM2 to **Voice Design** mode. Incompatible models and modes stop with a direct correction message. OmniVoice's controlled tag vocabulary can still be assembled with **📐 Visual Tag Builder**. Connect the resulting `opt_narrator` to **💾 Save Character Voice** when persistence is wanted.
|
||||
The shared designer accepts Qwen3-TTS, MOSS-TTS, or OmniVoice engine configurations and outputs the same `NARRATOR_VOICE` format. The voice-design instruction lives on **🎨 Voice Designer**; the engine keeps model, language, and generation settings. Select Qwen VoiceDesign or MOSS VoiceGenerator in the engine's model dropdown, or set OmniVoice to **Voice Design** mode. The corresponding engine instruction stays visible but is disabled because it would be ignored. Incompatible modes stop with a direct correction message. OmniVoice's controlled tag vocabulary can still be assembled with **📐 Visual Tag Builder**. Connect the resulting `opt_narrator` to **💾 Save Character Voice** when persistence is wanted.
|
||||
|
||||
**💾 Save Character Voice** accepts only `opt_narrator`, keeping persistence separate from voice construction. For existing audio, use **🎭 Character Voices** with the audio and its exact transcription, then connect its `opt_narrator` output to Save Character Voice. The save node writes the established three-file format—`name.wav`, `name.reference.txt`, and metadata in `name.txt`—under `models/voices/`.
|
||||
|
||||
@@ -973,23 +972,6 @@ Use the built-in OmniVoice preset in **📐 Visual Tag Builder** for the canonic
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><h3>VoxCPM - V2, 1.5, and Legacy 0.5B</h3></summary>
|
||||
|
||||
One **⚙️ VoxCPM Engine** supports every official OpenBMB generation through the official runtime:
|
||||
|
||||
* **VoxCPM2 (default)**: 30 languages, 48kHz output, reference-only cloning, controllable cloning with a style description, exact-transcript “ultimate” cloning, and Voice Design
|
||||
* **VoxCPM1.5**: English/Chinese, 44.1kHz output, lower VRAM and faster inference than V2; cloning requires the exact reference transcript
|
||||
* **VoxCPM-0.5B (legacy)**: English/Chinese, 16kHz output, lowest memory requirement; retained for compatibility rather than recommended quality
|
||||
|
||||
The suite automatically maps the selected architecture to the correct official cloning mode. It supports TTS Text, TTS SRT, Character Voices, pause tags, per-segment seed/CFG/step overrides, and reusable VoxCPM2 voices through **🎨 Voice Designer**.
|
||||
|
||||
VoxCPM2 also supports the official LoRA workflow. Use **📦 VoxCPM2 Dataset Prep** with audio/transcript pairs, optionally connect **🎛️ VoxCPM2 LoRA Config**, then run **🎓 Model Training**. Finished adapters are stored under `models/TTS/voxcpm/loras/` and appear in the engine's LoRA selector after refreshing node definitions. Start with 1–5 steps, batch size 1, and gradient accumulation enabled to measure VRAM before a real run. Exact checkpoint resume and adapter warm-start are supported.
|
||||
|
||||
The optional upstream denoiser and bundled ASR helpers from community nodes are not installed by this engine.
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><h3>MOSS-TTS - Local/Delay/TTSD Engine Family</h3></summary>
|
||||
|
||||
@@ -1287,19 +1269,19 @@ This section provides a detailed guide for installing TTS Audio Suite, covering
|
||||
|
||||
* Python 3.12 or higher
|
||||
|
||||
* **System libraries** (Linux only):
|
||||
* **Optional system libraries** (Linux only):
|
||||
|
||||
```bash
|
||||
# Ubuntu/Debian - Required for audio processing
|
||||
# Ubuntu/Debian - Optional audio features
|
||||
sudo apt-get install portaudio19-dev libsamplerate0-dev
|
||||
|
||||
# Fedora/RHEL
|
||||
sudo dnf install portaudio-devel libsamplerate-devel
|
||||
```
|
||||
|
||||
> **📋 Why needed?** `libsamplerate0-dev` provides audio resampling libraries for packages like `resampy` and `soxr`. `portaudio19-dev` enables voice recording features.
|
||||
> **📋 Optional:** `libsamplerate0-dev` provides additional audio-resampling support. `portaudio19-dev` enables voice recording. Missing either package no longer blocks installation of the TTS engines.
|
||||
|
||||
* **macOS dependencies**:
|
||||
* **Optional macOS dependencies**:
|
||||
|
||||
```bash
|
||||
brew install portaudio
|
||||
@@ -1394,17 +1376,17 @@ If you have a direct installation with a virtual environment (venv), follow thes
|
||||
|
||||
### Troubleshooting Dependency Issues
|
||||
|
||||
#### System Dependencies (Linux)
|
||||
#### Optional System Dependencies (Linux)
|
||||
|
||||
**Our install script automatically detects missing system libraries** and will display helpful error messages like:
|
||||
**Our install script automatically detects missing optional system libraries** and will display feature warnings like:
|
||||
|
||||
```
|
||||
[!] Missing system dependencies detected!
|
||||
[!] Optional system dependencies are missing
|
||||
============================================================
|
||||
SYSTEM DEPENDENCIES REQUIRED
|
||||
OPTIONAL LINUX SYSTEM DEPENDENCIES
|
||||
============================================================
|
||||
• libsamplerate0-dev (for audio resampling)
|
||||
• portaudio19-dev (for voice recording)
|
||||
• libsamplerate0-dev (optional additional audio-resampling support)
|
||||
• portaudio19-dev (optional voice recording)
|
||||
|
||||
Please install with:
|
||||
# Ubuntu/Debian:
|
||||
@@ -1413,7 +1395,7 @@ sudo apt-get install libsamplerate0-dev portaudio19-dev
|
||||
# Fedora/RHEL:
|
||||
sudo dnf install libsamplerate-devel portaudio-devel
|
||||
============================================================
|
||||
Then run this install script again.
|
||||
Core TTS installation will continue; only the listed features may be unavailable.
|
||||
```
|
||||
|
||||
#### Python Environment Issues
|
||||
@@ -1546,10 +1528,10 @@ For offline/manual setup:
|
||||
| Granite ASR | `ComfyUI/models/TTS/granite_asr/` | ✅ | Granite ASR models; plus adds native diarization/timestamps, optional Qwen forced aligner reused lazily for timestamps/SRT fallback |
|
||||
| Echo-TTS | `ComfyUI/models/TTS/echo-tts-base/` | ✅ | ~7.1GB total (base + dac); CC-BY-NC-SA |
|
||||
| Dots TTS | `ComfyUI/models/TTS/dots_tts/` | ✅ | Official base / soar / mf checkpoints with tokenizer, vocoder, speaker encoder |
|
||||
| Audio8 TTS | `ComfyUI/models/TTS/audio8_tts/Audio8-TTS-Preview-0.6b/` | ✅ | ~2.39 GiB official 0.6B Preview checkpoint; shared Transformers 4.57.3 runtime required because main 5.10.2 collapses voice cloning |
|
||||
| DramaBox | `ComfyUI/models/TTS/dramabox/DramaBox/` | ✅ | ~16.4GB download; fast mode roughly 24GB VRAM; experimental FP8 peaks on RTX 4090: staged ~15.1GB allocated, sequential ~11.7GB allocated / ~12.4GB reserved; conditional LTX-2 Community License |
|
||||
| Fish Audio S2 Pro | `ComfyUI/models/TTS/fish_audio_s2_pro/` | ✅ | Official BF16 or optional community FP8 checkpoint; the official checkpoint can be quantized on load with BNB INT8/NF4; main T5 environment with process teardown for Clear VRAM; Fish Audio Research License |
|
||||
| OmniVoice | `ComfyUI/models/TTS/omnivoice/` | ✅ | Official OmniVoice model. Voice cloning in this suite requires explicit reference text. |
|
||||
| VoxCPM | `ComfyUI/models/TTS/voxcpm/` | ✅ | VoxCPM2 (recommended), VoxCPM1.5 (lower VRAM/faster), and VoxCPM-0.5B (legacy) |
|
||||
|
||||
*Generated from [tts_audio_suite_engines.yaml](docs/Dev%20reports/tts_audio_suite_engines.yaml).*
|
||||
|
||||
@@ -1581,7 +1563,7 @@ Your support helps maintain and improve this project for the entire community!
|
||||
| **Unified 🔄 Voice Changer** | Modern voice conversion with multiple engines | • RVC + ChatterBox VC<br>• Iterative refinement<br>• Real-time conversion | ✅ **Updated for v4.3** | [📁 JSON](example_workflows/Unified%20🔄%20Voice%20Changer%20-%20RVC%20X%20ChatterBox.json) |
|
||||
| **Unified ✏️ ASR Transcribe + SRT Builder** | Modular ASR + subtitle workflow | • Granite ASR + Qwen3 ASR examples<br>• Separate transcription and SRT building<br>• Works with the new Text to SRT Builder flow | ✅ **New in v4.23** | [📁 JSON](example_workflows/Unified%20✏️%20ASR%20Transcribe%20+%20SRT%20Builder.json) |
|
||||
| **Unified 🌩️ Sound Effects** | Text-to-sound generation with compatible engines | • MOSS-SoundEffect v1 and v2<br>• Per-segment parameters and pauses<br>• Long-duration chunking and audio cache | ✅ **New** | [📁 JSON](example_workflows/Unified%20🌩️%20Sound%20Effects.json) |
|
||||
| **Unified 🎨 Voice Designer** | Reference-free character voice creation | • Qwen3-TTS, MOSS-TTS, OmniVoice, and VoxCPM2<br>• Free-form descriptions or Visual Tag Builder<br>• Preview and save reusable character voices | ✅ **New** | [📁 JSON](example_workflows/Unified%20🎨%20Voice%20Designer.json) · [🖼️ Cover](example_workflows/Unified%20🎨%20Voice%20Designer.jpg) |
|
||||
| **Unified 🎨 Voice Designer** | Reference-free character voice creation | • Qwen3-TTS, MOSS-TTS, and OmniVoice<br>• Free-form descriptions or Visual Tag Builder<br>• Preview and save reusable character voices | ✅ **New** | [📁 JSON](example_workflows/Unified%20🎨%20Voice%20Designer.json) · [🖼️ Cover](example_workflows/Unified%20🎨%20Voice%20Designer.jpg) |
|
||||
|
||||
### Specific Workflows
|
||||
|
||||
|
||||
@@ -1339,6 +1339,77 @@ engines:
|
||||
speed_performance: { supported: "partial", notes: "Moderate; mf variant is faster" }
|
||||
reference_free_tts: { supported: true, notes: "(default speaker)" }
|
||||
|
||||
- id: audio8_tts
|
||||
name: Audio8 TTS
|
||||
models: "Audio8 TTS Preview 0.6B"
|
||||
size: "~2.39 GiB"
|
||||
license: "Apache-2.0"
|
||||
commercial: true
|
||||
language_summary_full: "11 recommended languages (including Cantonese)"
|
||||
language_summary_compact: "🌐 11 recommended languages"
|
||||
|
||||
capabilities:
|
||||
tts: true
|
||||
srt: true
|
||||
vc: false
|
||||
asr: false
|
||||
training: false
|
||||
|
||||
runtime_isolation:
|
||||
default_mode: "shared_runtime"
|
||||
main_environment: false
|
||||
shared_runtime: true
|
||||
dedicated_runtime: false
|
||||
runtime_profile: "vibevoice_transformers4_shared"
|
||||
notes: "Required for feature-complete Audio8 inference: main Transformers 5.10.2 collapsed the official voice-clone demo to 0.325s, while the shared Transformers 4.57.3 runtime generated 10.495s for the published 10.54s demo"
|
||||
|
||||
readme_key_features:
|
||||
- "Reference-free TTS + zero-shot/cross-lingual cloning"
|
||||
- "44.1kHz output with sampling or greedy decoding"
|
||||
|
||||
special_features:
|
||||
- "44.1kHz reference-free TTS and zero-shot/cross-lingual voice cloning"
|
||||
- "Exact reference transcript required when voice cloning"
|
||||
- "Greedy or sampled decoding with temperature, top-p, top-k, seed, and token-limit controls"
|
||||
- "Suite text chunking, character switching, and SRT timing/assembly"
|
||||
- "No explicit language selector; multilingual behavior is text-driven"
|
||||
- "No native emotion/style/speed controls, streaming, timestamps, dialogue/multi-speaker mode, or presets"
|
||||
- "Not compatible with Voice Designer: the official model has no instruction-conditioned voice-design mode"
|
||||
- "Unified TTS/SRT integration only; no special node or suite training integration"
|
||||
|
||||
model_sources:
|
||||
- component: "Audio8 TTS Preview 0.6B"
|
||||
source_name: "Audio8/Audio8-TTS-Preview-0.6b"
|
||||
source_url: "https://huggingface.co/Audio8/Audio8-TTS-Preview-0.6b"
|
||||
size: "~2.39 GiB"
|
||||
auto_download: true
|
||||
notes: "Official 0.6B Preview checkpoint with bundled 44.1kHz codec; suite uses the shared Transformers 4.57.3 runtime because voice cloning is not compatible with main Transformers 5.10.2"
|
||||
|
||||
languages:
|
||||
en: { supported: true, flag: "🇺🇸", notes: "" }
|
||||
zh: { supported: true, flag: "🇨🇳", notes: "(Chinese; upstream separately recommends Cantonese, folded into this matrix row)" }
|
||||
de: { supported: true, flag: "🇩🇪", notes: "" }
|
||||
es: { supported: true, flag: "🇪🇸", notes: "" }
|
||||
fr: { supported: true, flag: "🇫🇷", notes: "" }
|
||||
it: { supported: true, flag: "🇮🇹", notes: "" }
|
||||
ja: { supported: true, flag: "🇯🇵", notes: "" }
|
||||
ko: { supported: true, flag: "🇰🇷", notes: "" }
|
||||
pl: { supported: true, flag: "🇵🇱", notes: "" }
|
||||
nl: { supported: true, flag: "🇳🇱", notes: "" }
|
||||
|
||||
features:
|
||||
voice_cloning: { supported: true, notes: "Zero-shot and cross-lingual; reference audio must be paired with its exact transcript" }
|
||||
reference_transcript: { requirement: conditional }
|
||||
native_multi_speaker: { supported: false, notes: "No native dialogue or multi-speaker interface; suite character switching generates speakers as separate segments" }
|
||||
voice_conversion: { supported: false, notes: "No separate voice-conversion mode" }
|
||||
asr_transcribe: { supported: false, notes: "" }
|
||||
emotion_control: { supported: false, notes: "No native emotion, style, or speed controls" }
|
||||
native_long_form: { supported: false, notes: "(uses suite text chunking and SRT assembly)" }
|
||||
community_finetunes: { supported: false, notes: "No community model or upstream SFT/training integration in the suite" }
|
||||
vram_efficient: { supported: "partial", notes: "~1.75 GiB model-plus-codec parameter memory at fp16/bf16 or ~3.5 GiB at float32/CPU, before KV cache, activations, and framework overhead; shared runtime reuses the main environment's PyTorch" }
|
||||
speed_performance: { supported: "partial", notes: "Live shared-runtime validation: 6.36s reference-free audio in 39.3s including load; 9.99s cloned audio in 36.7s warm; exact cache reuse in 56ms" }
|
||||
reference_free_tts: { supported: true, notes: "Reference audio is optional" }
|
||||
|
||||
- id: dramabox
|
||||
name: DramaBox
|
||||
models: "DramaBox 3.3B"
|
||||
@@ -1493,92 +1564,6 @@ engines:
|
||||
speed_performance: { supported: true, notes: "Fast; upstream reports sub-realtime RTF" }
|
||||
reference_free_tts: { supported: true, notes: "(native default voice; instruct also works without narrator)" }
|
||||
|
||||
- id: voxcpm
|
||||
name: VoxCPM
|
||||
models: "VoxCPM2, VoxCPM1.5, VoxCPM-0.5B (legacy)"
|
||||
size: "~5.0GB / ~2.0GB / ~1.6GB"
|
||||
license: "Apache-2.0"
|
||||
commercial: true
|
||||
language_summary_compact: "🌐 30 languages (V2); EN/ZH legacy"
|
||||
language_summary_full: "30 (VoxCPM2); English/Chinese (1.5 and 0.5B)"
|
||||
|
||||
capabilities:
|
||||
tts: true
|
||||
srt: true
|
||||
vc: false
|
||||
asr: false
|
||||
voice_design: true
|
||||
training: false
|
||||
|
||||
special_features:
|
||||
- "One architecture-gated engine for VoxCPM2, VoxCPM1.5, and legacy VoxCPM-0.5B"
|
||||
- "VoxCPM2 reference-only, controllable, and ultimate voice cloning"
|
||||
- "VoxCPM2 reference-free Voice Design"
|
||||
- "30-language text-driven synthesis with no language tag"
|
||||
- "Per-model 48kHz, 44.1kHz, or 16kHz native output"
|
||||
|
||||
model_sources:
|
||||
- component: "VoxCPM2"
|
||||
source_name: "openbmb/VoxCPM2"
|
||||
source_url: "https://huggingface.co/openbmb/VoxCPM2"
|
||||
size: "~5.0GB"
|
||||
auto_download: true
|
||||
notes: "Recommended 2B model; 30 languages, Voice Design, controllable cloning, and 48kHz output"
|
||||
- component: "VoxCPM1.5"
|
||||
source_name: "openbmb/VoxCPM1.5"
|
||||
source_url: "https://huggingface.co/openbmb/VoxCPM1.5"
|
||||
size: "~2.0GB"
|
||||
auto_download: true
|
||||
notes: "Faster/lower-VRAM English and Chinese continuation-cloning model with 44.1kHz output"
|
||||
- component: "VoxCPM-0.5B"
|
||||
source_name: "openbmb/VoxCPM-0.5B"
|
||||
source_url: "https://huggingface.co/openbmb/VoxCPM-0.5B"
|
||||
size: "~1.6GB"
|
||||
auto_download: true
|
||||
notes: "Legacy lowest-memory English and Chinese checkpoint with 16kHz output"
|
||||
|
||||
languages:
|
||||
en: { supported: true, flag: "🇺🇸", notes: "All variants" }
|
||||
zh: { supported: true, flag: "🇨🇳", notes: "All variants; VoxCPM2 also supports nine documented Chinese dialects" }
|
||||
de: { supported: true, flag: "🇩🇪", notes: "VoxCPM2" }
|
||||
es: { supported: true, flag: "🇪🇸", notes: "VoxCPM2" }
|
||||
fr: { supported: true, flag: "🇫🇷", notes: "VoxCPM2" }
|
||||
it: { supported: true, flag: "🇮🇹", notes: "VoxCPM2" }
|
||||
ja: { supported: true, flag: "🇯🇵", notes: "VoxCPM2" }
|
||||
ko: { supported: true, flag: "🇰🇷", notes: "VoxCPM2" }
|
||||
ru: { supported: true, flag: "🇷🇺", notes: "VoxCPM2" }
|
||||
pt: { supported: true, flag: "🇵🇹", notes: "VoxCPM2" }
|
||||
pl: { supported: true, flag: "🇵🇱", notes: "VoxCPM2" }
|
||||
hi: { supported: true, flag: "🇮🇳", notes: "VoxCPM2" }
|
||||
ar: { supported: true, flag: "🇦🇪", notes: "VoxCPM2" }
|
||||
tr: { supported: true, flag: "🇹🇷", notes: "VoxCPM2" }
|
||||
th: { supported: true, flag: "🇹🇭", notes: "VoxCPM2" }
|
||||
no: { supported: true, flag: "🇳🇴", notes: "VoxCPM2" }
|
||||
vi: { supported: true, flag: "🇻🇳", notes: "VoxCPM2" }
|
||||
hy: { supported: false, flag: "🇦🇲", notes: "" }
|
||||
ka: { supported: false, flag: "🇬🇪", notes: "" }
|
||||
da: { supported: true, flag: "🇩🇰", notes: "VoxCPM2" }
|
||||
fi: { supported: true, flag: "🇫🇮", notes: "VoxCPM2" }
|
||||
el: { supported: true, flag: "🇬🇷", notes: "VoxCPM2" }
|
||||
he: { supported: true, flag: "🇮🇱", notes: "VoxCPM2" }
|
||||
ms: { supported: true, flag: "🇲🇾", notes: "VoxCPM2" }
|
||||
nl: { supported: true, flag: "🇳🇱", notes: "VoxCPM2" }
|
||||
sv: { supported: true, flag: "🇸🇪", notes: "VoxCPM2" }
|
||||
sw: { supported: true, flag: "🇰🇪", notes: "VoxCPM2" }
|
||||
|
||||
features:
|
||||
voice_cloning: { supported: true, notes: "VoxCPM2 supports reference-only and controllable cloning; all variants support continuation cloning with the exact transcript" }
|
||||
reference_transcript: { requirement: conditional }
|
||||
native_multi_speaker: { supported: false, notes: "Suite character switching generates speakers as separate segments" }
|
||||
voice_conversion: { supported: false, notes: "" }
|
||||
asr_transcribe: { supported: false, notes: "No bundled ASR; exact legacy reference transcripts must be supplied" }
|
||||
emotion_control: { supported: true, notes: "VoxCPM2 natural-language voice/style instruction" }
|
||||
native_long_form: { supported: false, notes: "Uses suite text chunking; official streaming API is not integrated" }
|
||||
community_finetunes: { supported: true, notes: "Complete local checkpoints plus official-format VoxCPM2 LoRA training and inference" }
|
||||
vram_efficient: { supported: "partial", notes: "Official estimates are roughly 8GB for V2, 6GB for 1.5, and 5GB for legacy 0.5B" }
|
||||
speed_performance: { supported: true, notes: "Official standard runtime reports about 0.30 RTF for V2 and 0.15 RTF for 1.5 on RTX 4090" }
|
||||
reference_free_tts: { supported: true, notes: "Plain reference-free TTS for all variants; VoxCPM2 also supports instructed Voice Design" }
|
||||
|
||||
- id: moss-tts
|
||||
name: MOSS-TTS
|
||||
models: "Local 1.7B, Delay 8B v1.5/1.0, VoiceGenerator 1.7B, SoundEffect 8B v1, TTSD 8B"
|
||||
@@ -1989,10 +1974,9 @@ table_notes:
|
||||
reference_transcript: >-
|
||||
Conditional means the transcript is required only for the specific mode:
|
||||
CosyVoice3 zero-shot, Qwen3-TTS full Base cloning, or MOSS-TTSD cloned-speaker
|
||||
dialogue. VoxCPM1.5 and VoxCPM-0.5B require it for cloning, while VoxCPM2
|
||||
can clone from reference audio alone and uses the transcript for ultimate
|
||||
cloning. Higgs Audio 2, Higgs Audio v3, and Dots TTS accept matching text
|
||||
when provided but do not require it.
|
||||
dialogue, and Audio8 TTS zero-shot/cross-lingual cloning. Higgs Audio 2,
|
||||
Higgs Audio v3, and Dots TTS accept matching text when provided but do not
|
||||
require it.
|
||||
language_support:
|
||||
- "**CosyVoice3 Chinese**: Includes 18+ dialects (Cantonese, Sichuan, Dongbei, Shanghai, etc.)"
|
||||
- "**Higgs Audio 2**: Trained on EN, ZH (Mandarin), KO, DE, ES (English majority) - 10M hours AudioVerse dataset"
|
||||
@@ -2000,7 +1984,7 @@ table_notes:
|
||||
- "**IndexTTS-2**: Trained on 55K+ hours - ZH, EN, JA primary."
|
||||
- "**MOSS-TTS**: Official list includes ZH, EN, DE, ES, FR, JA, IT, HU, KO, RU, FA, AR, PL, PT, CS, DA, SV, EL, TR; HU/FA/CS are not separate columns in this matrix."
|
||||
- "**OmniVoice**: Official model supports 600+ languages; this matrix only shows the suite's comparison subset."
|
||||
- "**VoxCPM**: VoxCPM2 supports 30 languages without language tags; VoxCPM1.5 and the legacy 0.5B model support English and Chinese."
|
||||
- "**Audio8 TTS**: The Preview recommends Cantonese, Chinese, Dutch, English, French, German, Italian, Japanese, Korean, Polish, and Spanish. It has no explicit language selector; language behavior is text-driven. Cantonese is folded into the Chinese row in this matrix."
|
||||
- "**RVC**: Language-agnostic voice conversion/post-processing; it is marked supported for every language row."
|
||||
- "Qwen3-TTS: Supports pt-BR only with instruction (voice design or harcoded voices). Base can't do instructions, so it will always ouput pt-PT."
|
||||
|
||||
@@ -2069,6 +2053,10 @@ readme_model_download_table:
|
||||
primary_model_path: "ComfyUI/models/TTS/dots_tts/"
|
||||
auto_download: "✅"
|
||||
notes: "Official base / soar / mf checkpoints with tokenizer, vocoder, speaker encoder"
|
||||
- engine: "Audio8 TTS"
|
||||
primary_model_path: "ComfyUI/models/TTS/audio8_tts/Audio8-TTS-Preview-0.6b/"
|
||||
auto_download: "✅"
|
||||
notes: "~2.39 GiB official 0.6B Preview checkpoint; shared Transformers 4.57.3 runtime required because main 5.10.2 collapses voice cloning"
|
||||
- engine: "DramaBox"
|
||||
primary_model_path: "ComfyUI/models/TTS/dramabox/DramaBox/"
|
||||
auto_download: "✅"
|
||||
@@ -2081,10 +2069,6 @@ readme_model_download_table:
|
||||
primary_model_path: "ComfyUI/models/TTS/omnivoice/"
|
||||
auto_download: "✅"
|
||||
notes: "Official OmniVoice model. Voice cloning in this suite requires explicit reference text."
|
||||
- engine: "VoxCPM"
|
||||
primary_model_path: "ComfyUI/models/TTS/voxcpm/"
|
||||
auto_download: "✅"
|
||||
notes: "VoxCPM2 (recommended), VoxCPM1.5 (lower VRAM/faster), and VoxCPM-0.5B (legacy)"
|
||||
|
||||
model_layouts_markdown: |
|
||||
# Model Folder Layouts
|
||||
@@ -2516,6 +2500,22 @@ model_layouts_markdown: |
|
||||
- Native sample rate is 48kHz.
|
||||
- Main-environment support works on Transformers 5; on Windows, `normalize_text` falls back to no-op if `WeTextProcessing` is unavailable.
|
||||
|
||||
## Audio8 TTS
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/audio8_tts/
|
||||
└── Audio8-TTS-Preview-0.6b/
|
||||
└── (13 required official runtime files, including codec weights)
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- The official `Audio8/Audio8-TTS-Preview-0.6b` checkpoint is about 2.39 GiB and includes the 44.1kHz neural codec.
|
||||
- The 0.6B model plus codec use about 1.75 GiB for parameters at fp16/bf16 or about 3.5 GiB at float32/CPU, before KV cache, activations, and framework overhead.
|
||||
- The suite uses the existing `vibevoice_transformers4_shared` runtime with Transformers 4.57.3. Main Transformers 5.10.2 is not compatible with Audio8 voice cloning: the official demo collapsed to 0.325s, while the shared runtime generated 10.495s for the published 10.54s demo.
|
||||
- Live shared-runtime validation produced 6.36s of reference-free audio in 39.3s including model load, 9.99s of cloned audio in 36.7s warm, and exact cache reuse in 56ms.
|
||||
- The engine has unified TTS and SRT support only: no Voice Designer support, special node, training integration, native streaming/timestamps/dialogue, or built-in presets.
|
||||
|
||||
## OmniVoice
|
||||
|
||||
```text
|
||||
@@ -2544,37 +2544,3 @@ model_layouts_markdown: |
|
||||
- Native sample rate is 24kHz.
|
||||
- The main OmniVoice model stays in the main Transformers 5 environment.
|
||||
- Voice cloning requires reference audio plus explicit reference text in this suite.
|
||||
|
||||
## VoxCPM
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/voxcpm/
|
||||
├── VoxCPM2/
|
||||
│ ├── config.json
|
||||
│ ├── model.safetensors
|
||||
│ ├── audiovae.pth
|
||||
│ ├── tokenization_voxcpm2.py
|
||||
│ └── tokenizer files...
|
||||
├── VoxCPM1.5/
|
||||
│ ├── config.json
|
||||
│ ├── model.safetensors
|
||||
│ ├── audiovae.pth
|
||||
│ └── tokenizer files...
|
||||
├── VoxCPM-0.5B/
|
||||
│ ├── config.json
|
||||
│ ├── pytorch_model.bin
|
||||
│ ├── audiovae.pth
|
||||
│ └── tokenizer files...
|
||||
└── loras/
|
||||
└── MyVoxCPM2LoRA/
|
||||
├── lora_config.json
|
||||
└── lora_weights.safetensors
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- All three official generations use one architecture-gated engine and the official `voxcpm` package.
|
||||
- VoxCPM2 is the default and outputs 48kHz audio; VoxCPM1.5 outputs 44.1kHz; VoxCPM-0.5B outputs 16kHz.
|
||||
- Complete `local:<folder>` checkpoints are detected in configured ComfyUI TTS model folders.
|
||||
- Models run in the main Transformers 5 environment. The optional upstream denoiser and ASR helpers are not part of this integration.
|
||||
- VoxCPM2 supports official-format LoRA training and inference under `models/TTS/voxcpm/loras/`; legacy-generation training remains disabled pending compatibility testing.
|
||||
|
||||
@@ -18,9 +18,9 @@
|
||||
| **Echo-TTS** | Main | echo-tts-base + fish-s1-dac-min | ~5.3GB + ~1.8GB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | CC-BY-NC-SA-4.0 | Diffusion-based (~30s best), Force Speaker KV (speaker drift control) | 1 |
|
||||
| **Fish Audio S2 Pro** | Main | S2 Pro 4B / FP8 | ~10.3GB / ~8.0GB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | Fish Audio Research License | Free-form sub-word emotion/prosody tags, Native multi-speaker and multi-turn dialogue with dynamic speaker references, Zero-shot voice cloning, Optional per-segment custom character switching, Configurable 4K-32K native context with reduced KV-cache VRAM, Optional community FP8 weight-only checkpoint with BF16 activations, Optional on-the-fly BitsAndBytes INT8/NF4 for the official checkpoint | 80+ languages |
|
||||
| **Dots TTS** | Main | dots.tts-base, dots.tts-soar, dots.tts-mf | ~6GB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | Apache-2.0 | Official auto language detect / language control, SOAR and MeanFlow distilled variants | 19 |
|
||||
| **Audio8 TTS** | Shared | Audio8 TTS Preview 0.6B | ~2.39 GiB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | Apache-2.0 | 44.1kHz reference-free TTS and zero-shot/cross-lingual voice cloning, Exact reference transcript required when voice cloning, Greedy or sampled decoding with temperature, top-p, top-k, seed, and token-limit controls, Suite text chunking, character switching, and SRT timing/assembly, No explicit language selector; multilingual behavior is text-driven, No native emotion/style/speed controls, streaming, timestamps, dialogue/multi-speaker mode, or presets, Not compatible with Voice Designer: the official model has no instruction-conditioned voice-design mode, Unified TTS/SRT integration only; no special node or suite training integration | 11 recommended languages (including Cantonese) |
|
||||
| **DramaBox** | Main | DramaBox 3.3B | ~16.4GB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | LTX-2 Community License | Expressive scene prompting and stage directions, Native and SRT-aware duration targeting, Official duration-aware long-form chunking with scene-prefix preservation, Optional 10-second zero-shot voice reference, CFG negative prompt with per-segment switching, Explicit generation/reference durations, CFG rescale control, and optional Perth watermark, Experimental staged and sequential strategies for lowering peak VRAM; official LTX FP8-cast storage, Optional official torch.compile path | 1 |
|
||||
| **OmniVoice** | Main | OmniVoice | ~3.7GB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | Apache-2.0 | Inline non-verbal tags and pronunciation overrides, Reference-free voice design, 600+ language support, Upstream long-form chunk orchestration | 600+ |
|
||||
| **VoxCPM** | Main | VoxCPM2, VoxCPM1.5, VoxCPM-0.5B (legacy) | ~5.0GB / ~2.0GB / ~1.6GB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | Apache-2.0 | One architecture-gated engine for VoxCPM2, VoxCPM1.5, and legacy VoxCPM-0.5B, VoxCPM2 reference-only, controllable, and ultimate voice cloning, VoxCPM2 reference-free Voice Design, 30-language text-driven synthesis with no language tag, Per-model 48kHz, 44.1kHz, or 16kHz native output | 30 (VoxCPM2); English/Chinese (1.5 and 0.5B) |
|
||||
| **MOSS-TTS** | Main | Local 1.7B, Delay 8B v1.5/1.0, VoiceGenerator 1.7B, SoundEffect 8B v1, TTSD 8B | ~8.5GB tokenizer + ~6.1GB/17GB/18GB model | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | Apache-2.0 | Reference-free voice design with MOSS-VoiceGenerator, Native 1-5 speaker TTSD dialogue, 31-language generation with MOSS-TTS-v1.5, Prompt-only sound-effect generation with MOSS-SoundEffect v1, Long-form generation (TTSD/Delay), Duration token hint, Local/Delay/TTSD variants, Initial integrated LoRA training workflow (Delay 8B) | 24 |
|
||||
| **MOSS-SoundEffect v2** | Main | MOSS-SoundEffect-v2.0 | ~11.2GB | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | Apache-2.0 | Durations up to 30 seconds, Native negative prompting, CFG, flow shift, and diffusion-step controls, Prompt-only text-to-sound generation, 48 kHz mono output, Seeded generation | 2 |
|
||||
| **RVC** | Main | Community .pth | 100-300MB | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | MIT (framework); community models vary | Real-time VC, Integrated training workflow, Pitch shift (±14), 6 HuBERT models, Language-independent | Any |
|
||||
|
||||
+10
-10
@@ -2,7 +2,7 @@
|
||||
|
||||
## Feature Comparison Matrix
|
||||
|
||||
| Feature | F5-TTS | ChatterBox | ChatterBox 23L | VibeVoice | Higgs Audio 2 | Higgs Audio v3 | IndexTTS-2 | CosyVoice3 | Qwen3-TTS | Granite ASR | Step Audio EditX | Echo-TTS | Fish Audio S2 Pro | Dots TTS | DramaBox | OmniVoice | VoxCPM | MOSS-TTS | MOSS-SoundEffect v2 | RVC |
|
||||
| Feature | F5-TTS | ChatterBox | ChatterBox 23L | VibeVoice | Higgs Audio 2 | Higgs Audio v3 | IndexTTS-2 | CosyVoice3 | Qwen3-TTS | Granite ASR | Step Audio EditX | Echo-TTS | Fish Audio S2 Pro | Dots TTS | Audio8 TTS | DramaBox | OmniVoice | MOSS-TTS | MOSS-SoundEffect v2 | RVC |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| **TTS** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| **SRT** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
@@ -10,14 +10,14 @@
|
||||
| **ASR (Transcribe)** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||
| **Sound Effects** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ |
|
||||
| **Training** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ |
|
||||
| **Voice Cloning** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (Base model) | ❌ | ✅ | ✅ | ✅ Reference audio plus exact transcript | ✅ | ✅ Optional reference audio; configurable 3-30 second window from the beginning of the source, default 10 seconds | ✅ | ✅ VoxCPM2 supports reference-only and controllable cloning; all variants support continuation cloning with the exact transcript | ✅ | ❌ | ⚠️ (needs training) |
|
||||
| **Reference Transcript†** | **Required** | Not used | Not used | Not used | Optional | Optional | Not used | Conditional | Conditional | N/A | **Required** | Not used | **Required** | Optional | Not used | **Required** | Conditional | Conditional | Not used | N/A |
|
||||
| **Voice Cloning** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (Base model) | ❌ | ✅ | ✅ | ✅ Reference audio plus exact transcript | ✅ | ✅ Zero-shot and cross-lingual; reference audio must be paired with its exact transcript | ✅ Optional reference audio; configurable 3-30 second window from the beginning of the source, default 10 seconds | ✅ | ✅ | ❌ | ⚠️ (needs training) |
|
||||
| **Reference Transcript†** | **Required** | Not used | Not used | Not used | Optional | Optional | Not used | Conditional | Conditional | N/A | **Required** | Not used | **Required** | Optional | Conditional | Not used | **Required** | Conditional | Not used | N/A |
|
||||
| **Native Multi-Speaker** | ❌ | ❌ | ❌ | ✅ (Base only, Kugel uses fallback) | ✅ | ❌ | ❌ | ❌ | ❌ | ⚠️ (Plus variant speaker attribution / diarization) | ❌ | ❌ | ✅ Native dialogue is default; optional custom mode generates each [Character] segment independently as local speaker 0 | ❌ | ❌ | ❌ | ❌ | ✅ (TTSD v1.0; 1-5 speakers) | ❌ | ❌ |
|
||||
| **Emotion Control** | ❌ | ❌ | ⚠️ (v2 tags - doesn't work) | ❌ | ⚠️ (via prompt) | ✅ (native inline tags) | ✅ (8 emotions) | ⚠️ (via instruct) | ⚠️ (via instruct) | ❌ | ✅ (14 emotions) | ❌ | ✅ Free-form inline natural-language tags | ❌ | ✅ Natural-language scene prompt and stage directions | ⚠️ (voice-design instruct + inline non-verbal tags) | ✅ VoxCPM2 natural-language voice/style instruction | ✅ (MOSS-VoiceGenerator instruction-conditioned voice design) | ❌ | ❌ |
|
||||
| **Native Long-form** | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ Configurable 4K-32K native context; suite text chunking is bypassed | ❌ | ✅ Official duration-aware quote-group chunking; ~37s target / 45s cap | ✅ (uses upstream audio_chunk_duration / audio_chunk_threshold orchestration; bypasses suite char-based chunk splitting) | ❌ | ✅ (TTSD/Delay long-form; use chunk orchestration for very long inputs) | ❌ | N/A |
|
||||
| **Community Finetunes** | ✅ | ✅ | ✅ | ✅ KugelAudio, Hindi | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ Complete local checkpoints plus official-format VoxCPM2 LoRA training and inference | ✅ (LoRA adapter inference supported; initial integrated LoRA training support added for MOSS-TTS Delay 8B) | ❌ | ✅ |
|
||||
| **VRAM Efficient** | ✅ | ✅ | ✅ | ⚠️ (5-18GB) | ⚠️ (9GB) | ⚠️ (~8-10GB) | ⚠️ (9-12GB) | ✅ (5.4GB) | ✅ (3-6GB) | ✅ (~4.6GB) | ⚠️ (7GB) | ⚠️ (~7GB total) | ⚠️ 8K context measured at ~15.2GB BF16, ~11.2GB FP8, ~11.2GB BNB INT8, or ~8.9GB BNB NF4; BF16 codec and activations; BNB is a load-time option for the official checkpoint | ⚠️ (main env works; 2B-class model is not lightweight) | ✅ Fast mode is ~24GB; experimental FP8 peaks on RTX 4090: staged ~15.1GB allocated, sequential ~11.7GB allocated / ~12.4GB reserved | ⚠️ (main env works; 2B-class model is not lightweight) | ⚠️ Official estimates are roughly 8GB for V2, 6GB for 1.5, and 5GB for legacy 0.5B | ⚠️ (Local 1.7B smaller; tokenizer is large) | ⚠️ (runs in the main ComfyUI environment but remains GPU-heavy) | ✅ |
|
||||
| **Speed/Performance** | ✅ Very Fast | ✅ Fast | ✅ Fast | ⚠️ | ⚠️ | ⚠️ CUDA recommended | ⚠️ | ✅ Fast | ⚠️ | ⚠️ Moderate | ⚠️ | ✅ Fast (diffusion, realtime-capable) | ✅ Main-environment subprocess with reliable teardown; local compile measurements: ~40 it/s BF16, ~11.8 it/s NF4 at ~8.9GB VRAM, and ~3.7 it/s INT8 at ~11.2GB VRAM; quality comparison pending | ⚠️ Moderate; mf variant is faster | ⚠️ Fast favors repeated generation; staged reloads temporary components; sequential also transfers the transformer; optional block-level torch.compile accelerates repeat denoising | ✅ Fast; upstream reports sub-realtime RTF | ✅ Official standard runtime reports about 0.30 RTF for V2 and 0.15 RTF for 1.5 on RTX 4090 | ✅ Fast with CUDA/FlashAttention | ⚠️ (100 diffusion steps by default) | ✅ Fast |
|
||||
| **No Narrator Required** | ❌ | ✅ (default speaker) | ✅ (default speaker) | ✅ (zero-shot / default speaker) | ✅ (basic TTS if no narrator/reference is provided) | ✅ (zero-shot) | ❌ | ✅ (cross-lingual or instruct mode) | ✅ (Base default voice or CustomVoice presets) | N/A | ❌ | ❌ | ✅ Reference audio is optional | ✅ (default speaker) | ✅ Voice reference is optional | ✅ (native default voice; instruct also works without narrator) | ✅ Plain reference-free TTS for all variants; VoxCPM2 also supports instructed Voice Design | ✅ (direct TTS and prompt-only generation) | ❌ | N/A |
|
||||
| **Emotion Control** | ❌ | ❌ | ⚠️ (v2 tags - doesn't work) | ❌ | ⚠️ (via prompt) | ✅ (native inline tags) | ✅ (8 emotions) | ⚠️ (via instruct) | ⚠️ (via instruct) | ❌ | ✅ (14 emotions) | ❌ | ✅ Free-form inline natural-language tags | ❌ | ❌ | ✅ Natural-language scene prompt and stage directions | ⚠️ (voice-design instruct + inline non-verbal tags) | ✅ (MOSS-VoiceGenerator instruction-conditioned voice design) | ❌ | ❌ |
|
||||
| **Native Long-form** | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ Configurable 4K-32K native context; suite text chunking is bypassed | ❌ | ❌ | ✅ Official duration-aware quote-group chunking; ~37s target / 45s cap | ✅ (uses upstream audio_chunk_duration / audio_chunk_threshold orchestration; bypasses suite char-based chunk splitting) | ✅ (TTSD/Delay long-form; use chunk orchestration for very long inputs) | ❌ | N/A |
|
||||
| **Community Finetunes** | ✅ | ✅ | ✅ | ✅ KugelAudio, Hindi | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (LoRA adapter inference supported; initial integrated LoRA training support added for MOSS-TTS Delay 8B) | ❌ | ✅ |
|
||||
| **VRAM Efficient** | ✅ | ✅ | ✅ | ⚠️ (5-18GB) | ⚠️ (9GB) | ⚠️ (~8-10GB) | ⚠️ (9-12GB) | ✅ (5.4GB) | ✅ (3-6GB) | ✅ (~4.6GB) | ⚠️ (7GB) | ⚠️ (~7GB total) | ⚠️ 8K context measured at ~15.2GB BF16, ~11.2GB FP8, ~11.2GB BNB INT8, or ~8.9GB BNB NF4; BF16 codec and activations; BNB is a load-time option for the official checkpoint | ⚠️ (main env works; 2B-class model is not lightweight) | ⚠️ ~1.75 GiB model-plus-codec parameter memory at fp16/bf16 or ~3.5 GiB at float32/CPU, before KV cache, activations, and framework overhead; shared runtime reuses the main environment's PyTorch | ✅ Fast mode is ~24GB; experimental FP8 peaks on RTX 4090: staged ~15.1GB allocated, sequential ~11.7GB allocated / ~12.4GB reserved | ⚠️ (main env works; 2B-class model is not lightweight) | ⚠️ (Local 1.7B smaller; tokenizer is large) | ⚠️ (runs in the main ComfyUI environment but remains GPU-heavy) | ✅ |
|
||||
| **Speed/Performance** | ✅ Very Fast | ✅ Fast | ✅ Fast | ⚠️ | ⚠️ | ⚠️ CUDA recommended | ⚠️ | ✅ Fast | ⚠️ | ⚠️ Moderate | ⚠️ | ✅ Fast (diffusion, realtime-capable) | ✅ Main-environment subprocess with reliable teardown; local compile measurements: ~40 it/s BF16, ~11.8 it/s NF4 at ~8.9GB VRAM, and ~3.7 it/s INT8 at ~11.2GB VRAM; quality comparison pending | ⚠️ Moderate; mf variant is faster | ⚠️ Live shared-runtime validation: 6.36s reference-free audio in 39.3s including load; 9.99s cloned audio in 36.7s warm; exact cache reuse in 56ms | ⚠️ Fast favors repeated generation; staged reloads temporary components; sequential also transfers the transformer; optional block-level torch.compile accelerates repeat denoising | ✅ Fast; upstream reports sub-realtime RTF | ✅ Fast with CUDA/FlashAttention | ⚠️ (100 diffusion steps by default) | ✅ Fast |
|
||||
| **No Narrator Required** | ❌ | ✅ (default speaker) | ✅ (default speaker) | ✅ (zero-shot / default speaker) | ✅ (basic TTS if no narrator/reference is provided) | ✅ (zero-shot) | ❌ | ✅ (cross-lingual or instruct mode) | ✅ (Base default voice or CustomVoice presets) | N/A | ❌ | ❌ | ✅ Reference audio is optional | ✅ (default speaker) | ✅ Reference audio is optional | ✅ Voice reference is optional | ✅ (native default voice; instruct also works without narrator) | ✅ (direct TTS and prompt-only generation) | ❌ | N/A |
|
||||
|
||||
† **Reference Transcript:** Conditional means the transcript is required only for the specific mode: CosyVoice3 zero-shot, Qwen3-TTS full Base cloning, or MOSS-TTSD cloned-speaker dialogue. VoxCPM1.5 and VoxCPM-0.5B require it for cloning, while VoxCPM2 can clone from reference audio alone and uses the transcript for ultimate cloning. Higgs Audio 2, Higgs Audio v3, and Dots TTS accept matching text when provided but do not require it.
|
||||
† **Reference Transcript:** Conditional means the transcript is required only for the specific mode: CosyVoice3 zero-shot, Qwen3-TTS full Base cloning, or MOSS-TTSD cloned-speaker dialogue, and Audio8 TTS zero-shot/cross-lingual cloning. Higgs Audio 2, Higgs Audio v3, and Dots TTS accept matching text when provided but do not require it.
|
||||
+29
-29
@@ -2,35 +2,35 @@
|
||||
|
||||
## Language Support by Engine
|
||||
|
||||
| Language | Code | F5-TTS | ChatterBox | ChatterBox 23L | VibeVoice | Higgs Audio 2 | Higgs Audio v3 | IndexTTS-2 | CosyVoice3 | Qwen3-TTS | Granite ASR | Step Audio EditX | Echo-TTS | Fish Audio S2 Pro | Dots TTS | DramaBox | OmniVoice | VoxCPM | MOSS-TTS | MOSS-SoundEffect v2 | RVC |
|
||||
| Language | Code | F5-TTS | ChatterBox | ChatterBox 23L | VibeVoice | Higgs Audio 2 | Higgs Audio v3 | IndexTTS-2 | CosyVoice3 | Qwen3-TTS | Granite ASR | Step Audio EditX | Echo-TTS | Fish Audio S2 Pro | Dots TTS | Audio8 TTS | DramaBox | OmniVoice | MOSS-TTS | MOSS-SoundEffect v2 | RVC |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| 🇺🇸 **English** | EN | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ Tier 1 | ✅ | ✅ Official model is English-only | ✅ | ✅ All variants | ✅ | ✅ Officially demonstrated prompt language | ✅ |
|
||||
| 🇨🇳 **Chinese** | ZH | ❌ | ❌ | ✅ | ✅ | ✅ (Mandarin) | ✅ | ✅ | ✅ + 18 dialects | ✅ | ❌ | ✅ (Mandarin + Sichuanese, Cantonese) | ❌ | ✅ Tier 1 | ✅ (Mandarin; official also exposes YUE separately outside this matrix) | ❌ | ✅ | ✅ All variants; VoxCPM2 also supports nine documented Chinese dialects | ✅ | ✅ Officially demonstrated prompt language | ✅ |
|
||||
| 🇩🇪 **German** | DE | ✅ | ✅ (×3) | ✅ | ✅ (Kugel) | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ | ❌ | ✅ | ✅ VoxCPM2 | ✅ | ❌ | ✅ |
|
||||
| 🇪🇸 **Spanish** | ES | ✅ | ❌ | ✅ | ✅ (Kugel) | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ | ❌ | ✅ | ✅ VoxCPM2 | ✅ | ❌ | ✅ |
|
||||
| 🇫🇷 **French** | FR | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ | ❌ | ✅ | ✅ VoxCPM2 | ✅ | ❌ | ✅ |
|
||||
| 🇮🇹 **Italian** | IT | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ VoxCPM2 | ✅ | ❌ | ✅ |
|
||||
| 🇯🇵 **Japanese** | JA | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ✅ ? | ✅ | ✅ | ✅ (4.1-2b supports Japanese; plus variant does not) | ✅ | ❌ | ✅ Tier 1 | ✅ | ❌ | ✅ | ✅ VoxCPM2 | ✅ | ❌ | ✅ |
|
||||
| 🇰🇷 **Korean** | KO | ❌ | ✅ | ✅ | ✅ (Kugel) | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ Tier 2 | ✅ | ❌ | ✅ | ✅ VoxCPM2 | ✅ | ❌ | ✅ |
|
||||
| 🇷🇺 **Russian** | RU | ❌ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ Tier 2 | ✅ | ❌ | ✅ | ✅ VoxCPM2 | ✅ | ❌ | ✅ |
|
||||
| 🇧🇷 **Portuguese** | PT | ✅ (BR) | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ (EU/BR*) | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ (Official PT tag is generic; upstream does not expose separate PT-BR/PT-PT tags and it may lean more European Portuguese than Brazilian Portuguese) | ❌ | ✅ (generic PT; official language space is much broader than this matrix) | ✅ VoxCPM2 | ✅ | ❌ | ✅ |
|
||||
| 🇵🇱 **Polish** | PL | ✅ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ VoxCPM2 | ✅ | ❌ | ✅ |
|
||||
| 🇮🇳 **Hindi** | HI | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ VoxCPM2 | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇪🇬 **Arabic** | AR | ❌ | ❌ | ✅ (Egyptian) | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ Tier 2 | ✅ | ❌ | ✅ | ✅ VoxCPM2 | ✅ | ❌ | ✅ |
|
||||
| 🇹🇷 **Turkish** | TR | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ VoxCPM2 | ✅ | ❌ | ✅ |
|
||||
| 🇹🇭 **Thai** | TH | ✅ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ VoxCPM2 | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇳🇴 **Norwegian** | NO | ❌ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ VoxCPM2 | ❌ | ❌ | ✅ |
|
||||
| 🇻🇳 **Vietnamese** | VI | ❌ | ❌ | ✅ (Viterbox) | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ VoxCPM2 | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇦🇲 **Armenian** | HY | ❌ | ✅ | ❌ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇬🇪 **Georgian** | KA | ❌ | ✅ | ❌ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇩🇰 **Danish** | DA | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ VoxCPM2 | ✅ | ❌ | ✅ |
|
||||
| 🇫🇮 **Finnish** | FI | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ VoxCPM2 | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇬🇷 **Greek** | EL | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ VoxCPM2 | ✅ | ❌ | ✅ |
|
||||
| 🇮🇱 **Hebrew** | HE | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ VoxCPM2 | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇲🇾 **Malay** | MS | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ VoxCPM2 | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇳🇱 **Dutch** | NL | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ VoxCPM2 | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇸🇪 **Swedish** | SV | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ VoxCPM2 | ✅ | ❌ | ✅ |
|
||||
| 🇰🇪 **Swahili** | SW | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ VoxCPM2 | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇺🇸 **English** | EN | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ Tier 1 | ✅ | ✅ | ✅ Official model is English-only | ✅ | ✅ | ✅ Officially demonstrated prompt language | ✅ |
|
||||
| 🇨🇳 **Chinese** | ZH | ❌ | ❌ | ✅ | ✅ | ✅ (Mandarin) | ✅ | ✅ | ✅ + 18 dialects | ✅ | ❌ | ✅ (Mandarin + Sichuanese, Cantonese) | ❌ | ✅ Tier 1 | ✅ (Mandarin; official also exposes YUE separately outside this matrix) | ✅ (Chinese; upstream separately recommends Cantonese, folded into this matrix row) | ❌ | ✅ | ✅ | ✅ Officially demonstrated prompt language | ✅ |
|
||||
| 🇩🇪 **German** | DE | ✅ | ✅ (×3) | ✅ | ✅ (Kugel) | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇪🇸 **Spanish** | ES | ✅ | ❌ | ✅ | ✅ (Kugel) | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇫🇷 **French** | FR | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇮🇹 **Italian** | IT | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇯🇵 **Japanese** | JA | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ✅ ? | ✅ | ✅ | ✅ (4.1-2b supports Japanese; plus variant does not) | ✅ | ❌ | ✅ Tier 1 | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇰🇷 **Korean** | KO | ❌ | ✅ | ✅ | ✅ (Kugel) | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ Tier 2 | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇷🇺 **Russian** | RU | ❌ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ Tier 2 | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇧🇷 **Portuguese** | PT | ✅ (BR) | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ (EU/BR*) | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ (Official PT tag is generic; upstream does not expose separate PT-BR/PT-PT tags and it may lean more European Portuguese than Brazilian Portuguese) | ❌ | ❌ | ✅ (generic PT; official language space is much broader than this matrix) | ✅ | ❌ | ✅ |
|
||||
| 🇵🇱 **Polish** | PL | ✅ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇮🇳 **Hindi** | HI | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇪🇬 **Arabic** | AR | ❌ | ❌ | ✅ (Egyptian) | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ Tier 2 | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇹🇷 **Turkish** | TR | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇹🇭 **Thai** | TH | ✅ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇳🇴 **Norwegian** | NO | ❌ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ |
|
||||
| 🇻🇳 **Vietnamese** | VI | ❌ | ❌ | ✅ (Viterbox) | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇦🇲 **Armenian** | HY | ❌ | ✅ | ❌ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ |
|
||||
| 🇬🇪 **Georgian** | KA | ❌ | ✅ | ❌ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ |
|
||||
| 🇩🇰 **Danish** | DA | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇫🇮 **Finnish** | FI | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇬🇷 **Greek** | EL | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇮🇱 **Hebrew** | HE | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇲🇾 **Malay** | MS | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇳🇱 **Dutch** | NL | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇸🇪 **Swedish** | SV | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇰🇪 **Swahili** | SW | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇿🇦 **Afrikaans** | AF | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇦🇱 **Albanian** | SQ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇳 **Assamese** | AS | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
@@ -115,6 +115,6 @@
|
||||
- **IndexTTS-2**: Trained on 55K+ hours - ZH, EN, JA primary.
|
||||
- **MOSS-TTS**: Official list includes ZH, EN, DE, ES, FR, JA, IT, HU, KO, RU, FA, AR, PL, PT, CS, DA, SV, EL, TR; HU/FA/CS are not separate columns in this matrix.
|
||||
- **OmniVoice**: Official model supports 600+ languages; this matrix only shows the suite's comparison subset.
|
||||
- **VoxCPM**: VoxCPM2 supports 30 languages without language tags; VoxCPM1.5 and the legacy 0.5B model support English and Chinese.
|
||||
- **Audio8 TTS**: The Preview recommends Cantonese, Chinese, Dutch, English, French, German, Italian, Japanese, Korean, Polish, and Spanish. It has no explicit language selector; language behavior is text-driven. Cantonese is folded into the Chinese row in this matrix.
|
||||
- **RVC**: Language-agnostic voice conversion/post-processing; it is marked supported for every language row.
|
||||
- Qwen3-TTS: Supports pt-BR only with instruction (voice design or harcoded voices). Base can't do instructions, so it will always ouput pt-PT.
|
||||
@@ -129,6 +129,12 @@ Use this as the canonical list of model repositories/links for offline setup.
|
||||
| dots.tts-soar | [rednote-hilab/dots.tts-soar](https://huggingface.co/rednote-hilab/dots.tts-soar) | ~6GB | ✅ | Official SOAR checkpoint for higher-quality zero-shot cloning |
|
||||
| dots.tts-mf | [rednote-hilab/dots.tts-mf](https://huggingface.co/rednote-hilab/dots.tts-mf) | ~6GB | ✅ | Official MeanFlow-distilled checkpoint for faster inference |
|
||||
|
||||
## Audio8 TTS
|
||||
|
||||
| Component | Source | Size | Auto-Download | Notes |
|
||||
|---|---|---|---|---|
|
||||
| Audio8 TTS Preview 0.6B | [Audio8/Audio8-TTS-Preview-0.6b](https://huggingface.co/Audio8/Audio8-TTS-Preview-0.6b) | ~2.39 GiB | ✅ | Official 0.6B Preview checkpoint with bundled 44.1kHz codec; suite uses the shared Transformers 4.57.3 runtime because voice cloning is not compatible with main Transformers 5.10.2 |
|
||||
|
||||
## DramaBox
|
||||
|
||||
| Component | Source | Size | Auto-Download | Notes |
|
||||
@@ -142,14 +148,6 @@ Use this as the canonical list of model repositories/links for offline setup.
|
||||
|---|---|---|---|---|
|
||||
| OmniVoice | [k2-fsa/OmniVoice](https://huggingface.co/k2-fsa/OmniVoice) | ~3.7GB | ✅ | Official main model including bundled audio_tokenizer assets |
|
||||
|
||||
## VoxCPM
|
||||
|
||||
| Component | Source | Size | Auto-Download | Notes |
|
||||
|---|---|---|---|---|
|
||||
| VoxCPM2 | [openbmb/VoxCPM2](https://huggingface.co/openbmb/VoxCPM2) | ~5.0GB | ✅ | Recommended 2B model; 30 languages, Voice Design, controllable cloning, and 48kHz output |
|
||||
| VoxCPM1.5 | [openbmb/VoxCPM1.5](https://huggingface.co/openbmb/VoxCPM1.5) | ~2.0GB | ✅ | Faster/lower-VRAM English and Chinese continuation-cloning model with 44.1kHz output |
|
||||
| VoxCPM-0.5B | [openbmb/VoxCPM-0.5B](https://huggingface.co/openbmb/VoxCPM-0.5B) | ~1.6GB | ✅ | Legacy lowest-memory English and Chinese checkpoint with 16kHz output |
|
||||
|
||||
## MOSS-TTS
|
||||
|
||||
| Component | Source | Size | Auto-Download | Notes |
|
||||
|
||||
+16
-34
@@ -427,6 +427,22 @@ Notes:
|
||||
- Native sample rate is 48kHz.
|
||||
- Main-environment support works on Transformers 5; on Windows, `normalize_text` falls back to no-op if `WeTextProcessing` is unavailable.
|
||||
|
||||
## Audio8 TTS
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/audio8_tts/
|
||||
└── Audio8-TTS-Preview-0.6b/
|
||||
└── (13 required official runtime files, including codec weights)
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- The official `Audio8/Audio8-TTS-Preview-0.6b` checkpoint is about 2.39 GiB and includes the 44.1kHz neural codec.
|
||||
- The 0.6B model plus codec use about 1.75 GiB for parameters at fp16/bf16 or about 3.5 GiB at float32/CPU, before KV cache, activations, and framework overhead.
|
||||
- The suite uses the existing `vibevoice_transformers4_shared` runtime with Transformers 4.57.3. Main Transformers 5.10.2 is not compatible with Audio8 voice cloning: the official demo collapsed to 0.325s, while the shared runtime generated 10.495s for the published 10.54s demo.
|
||||
- Live shared-runtime validation produced 6.36s of reference-free audio in 39.3s including model load, 9.99s of cloned audio in 36.7s warm, and exact cache reuse in 56ms.
|
||||
- The engine has unified TTS and SRT support only: no Voice Designer support, special node, training integration, native streaming/timestamps/dialogue, or built-in presets.
|
||||
|
||||
## OmniVoice
|
||||
|
||||
```text
|
||||
@@ -455,37 +471,3 @@ Notes:
|
||||
- Native sample rate is 24kHz.
|
||||
- The main OmniVoice model stays in the main Transformers 5 environment.
|
||||
- Voice cloning requires reference audio plus explicit reference text in this suite.
|
||||
|
||||
## VoxCPM
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/voxcpm/
|
||||
├── VoxCPM2/
|
||||
│ ├── config.json
|
||||
│ ├── model.safetensors
|
||||
│ ├── audiovae.pth
|
||||
│ ├── tokenization_voxcpm2.py
|
||||
│ └── tokenizer files...
|
||||
├── VoxCPM1.5/
|
||||
│ ├── config.json
|
||||
│ ├── model.safetensors
|
||||
│ ├── audiovae.pth
|
||||
│ └── tokenizer files...
|
||||
├── VoxCPM-0.5B/
|
||||
│ ├── config.json
|
||||
│ ├── pytorch_model.bin
|
||||
│ ├── audiovae.pth
|
||||
│ └── tokenizer files...
|
||||
└── loras/
|
||||
└── MyVoxCPM2LoRA/
|
||||
├── lora_config.json
|
||||
└── lora_weights.safetensors
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- All three official generations use one architecture-gated engine and the official `voxcpm` package.
|
||||
- VoxCPM2 is the default and outputs 48kHz audio; VoxCPM1.5 outputs 44.1kHz; VoxCPM-0.5B outputs 16kHz.
|
||||
- Complete `local:<folder>` checkpoints are detected in configured ComfyUI TTS model folders.
|
||||
- Models run in the main Transformers 5 environment. The optional upstream denoiser and ASR helpers are not part of this integration.
|
||||
- VoxCPM2 supports official-format LoRA training and inference under `models/TTS/voxcpm/loras/`; legacy-generation training remains disabled pending compatibility testing.
|
||||
|
||||
@@ -466,7 +466,7 @@ Also to test, requirements and dependencies need to be added.
|
||||
- [ ] Pause tags work with `[pause:1.5s]`
|
||||
- [ ] Caching works (same input = cached output)
|
||||
- [ ] Model auto-download works
|
||||
- [ ] VRAM management works (model unloads)
|
||||
- [ ] VRAM management works (unload, switch to another isolated engine, then switch back with cache disabled)
|
||||
- [ ] Different parameter combinations work
|
||||
- [ ] Engine prints a standard `Settings:` summary with the active generation/load parameters
|
||||
- [ ] **Interrupt handling works** - User can stop SRT generation and it stops within ~1 segment
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
### Import Errors
|
||||
- **Missing import**: Always add `import folder_paths` when using `folder_paths.get_temp_directory()` in node files
|
||||
- **Bundled code imports**: For complex bundled packages with internal cross-imports, add `sys.path.insert(0, impl_dir)` at top of main files instead of converting all imports
|
||||
- **Bundled code imports**: Never permanently prepend a bundled implementation directory to `sys.path` in the main ComfyUI process. Generic files such as `utils.py` can shadow suite packages and break later isolated workers. Prefer package-relative or vendor-namespaced imports; if upstream imports make path injection unavoidable, confine it to the isolated worker and load suite protocol modules before adding the vendor path.
|
||||
|
||||
### Audio Utility Functions
|
||||
- **Temp file creation**: Use `AudioProcessingUtils.save_audio_to_temp_file()` not `save_audio()` (doesn't exist)
|
||||
|
||||
@@ -49,6 +49,19 @@ except Exception as e:
|
||||
def __init__(self, *args, **kwargs):
|
||||
raise ImportError(f"Dots TTS adapter not available: {e}")
|
||||
|
||||
try:
|
||||
from .audio8_tts_adapter import Audio8TTSEngineAdapter
|
||||
AUDIO8_TTS_ADAPTER_AVAILABLE = True
|
||||
except Exception as exc:
|
||||
AUDIO8_TTS_ADAPTER_AVAILABLE = False
|
||||
_AUDIO8_TTS_ADAPTER_ERROR = str(exc)
|
||||
|
||||
class Audio8TTSEngineAdapter:
|
||||
def __init__(self, *args, **kwargs):
|
||||
raise ImportError(
|
||||
f"Audio8 TTS adapter not available: {_AUDIO8_TTS_ADAPTER_ERROR}"
|
||||
)
|
||||
|
||||
try:
|
||||
from .dramabox_adapter import DramaBoxEngineAdapter
|
||||
DRAMABOX_ADAPTER_AVAILABLE = True
|
||||
@@ -76,15 +89,6 @@ except Exception as e:
|
||||
def __init__(self, *args, **kwargs):
|
||||
raise ImportError(f"OmniVoice adapter not available: {e}")
|
||||
|
||||
try:
|
||||
from .voxcpm_adapter import VoxCPMEngineAdapter
|
||||
VOXCPM_ADAPTER_AVAILABLE = True
|
||||
except Exception as e:
|
||||
VOXCPM_ADAPTER_AVAILABLE = False
|
||||
class VoxCPMEngineAdapter:
|
||||
def __init__(self, *args, **kwargs):
|
||||
raise ImportError(f"VoxCPM adapter not available: {e}")
|
||||
|
||||
try:
|
||||
from .moss_tts_adapter import MossTTSEngineAdapter
|
||||
MOSS_TTS_ADAPTER_AVAILABLE = True
|
||||
@@ -105,13 +109,11 @@ except Exception as e:
|
||||
|
||||
__all__ = [
|
||||
'ChatterBoxEngineAdapter', 'F5TTSEngineAdapter', 'CosyVoiceAdapter', 'EchoTTSEngineAdapter',
|
||||
'DotsTTSEngineAdapter', 'DramaBoxEngineAdapter', 'OmniVoiceEngineAdapter',
|
||||
'VoxCPMEngineAdapter',
|
||||
'DotsTTSEngineAdapter', 'Audio8TTSEngineAdapter', 'DramaBoxEngineAdapter', 'OmniVoiceEngineAdapter',
|
||||
'MossTTSEngineAdapter', 'HiggsAudioV3EngineAdapter',
|
||||
'CHATTERBOX_ADAPTER_AVAILABLE', 'F5TTS_ADAPTER_AVAILABLE', 'COSYVOICE_ADAPTER_AVAILABLE',
|
||||
'ECHO_TTS_ADAPTER_AVAILABLE', 'DOTS_TTS_ADAPTER_AVAILABLE',
|
||||
'ECHO_TTS_ADAPTER_AVAILABLE', 'DOTS_TTS_ADAPTER_AVAILABLE', 'AUDIO8_TTS_ADAPTER_AVAILABLE',
|
||||
'DRAMABOX_ADAPTER_AVAILABLE', 'OMNIVOICE_ADAPTER_AVAILABLE',
|
||||
'VOXCPM_ADAPTER_AVAILABLE',
|
||||
'MOSS_TTS_ADAPTER_AVAILABLE', 'HIGGS_AUDIO_V3_ADAPTER_AVAILABLE',
|
||||
'MossSoundEffectV2Adapter'
|
||||
]
|
||||
|
||||
@@ -0,0 +1,277 @@
|
||||
"""Adapter between suite processors and the official Audio8 TTS engine."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from engines.audio8_tts.downloader import Audio8TTSDownloader
|
||||
from utils.audio.audio_hash import generate_stable_audio_component
|
||||
from utils.audio.cache import get_audio_cache
|
||||
from utils.device import resolve_torch_device
|
||||
from utils.models.factory_config import ModelLoadConfig, RUNTIME_MODE_SHARED
|
||||
from utils.voice.character_logging import resolved_character_label
|
||||
from utils.voice.reference import effective_voice_audio
|
||||
|
||||
|
||||
class Audio8TTSEngineAdapter:
|
||||
"""Translate unified TTS calls into Audio8's official inference API."""
|
||||
|
||||
SAMPLE_RATE = 44100
|
||||
|
||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||
self.config = dict(config or {})
|
||||
self.audio_cache = get_audio_cache()
|
||||
self.downloader = Audio8TTSDownloader()
|
||||
self._last_config: Optional[ModelLoadConfig] = None
|
||||
self._load_signature: Optional[Tuple[Any, ...]] = None
|
||||
|
||||
def update_config(self, new_config: Dict[str, Any]):
|
||||
self.config = dict(new_config or {})
|
||||
|
||||
def _model_selection(self) -> str:
|
||||
return str(
|
||||
self.config.get(
|
||||
"model_variant",
|
||||
self.config.get(
|
||||
"model_name",
|
||||
Audio8TTSDownloader.MODEL_NAME,
|
||||
),
|
||||
)
|
||||
or Audio8TTSDownloader.MODEL_NAME
|
||||
)
|
||||
|
||||
def _build_load_signature(self) -> Tuple[Any, ...]:
|
||||
return (
|
||||
self._model_selection(),
|
||||
resolve_torch_device(self.config.get("device", "auto")),
|
||||
self.config.get("dtype", "auto"),
|
||||
)
|
||||
|
||||
def load_model(
|
||||
self,
|
||||
model_variant: str,
|
||||
device: str = "auto",
|
||||
dtype: str = "auto",
|
||||
):
|
||||
"""Resolve the organized model path and load through the unified factory."""
|
||||
from utils.models.unified_model_interface import unified_model_interface
|
||||
|
||||
model_path = self.downloader.resolve_model_path(model_variant)
|
||||
config = ModelLoadConfig(
|
||||
engine_name="audio8_tts",
|
||||
model_type="tts",
|
||||
model_name=model_variant,
|
||||
model_path=model_path,
|
||||
device=device,
|
||||
additional_params={"dtype": dtype},
|
||||
runtime_mode=RUNTIME_MODE_SHARED,
|
||||
runtime_profile="vibevoice_transformers4_shared",
|
||||
)
|
||||
self._last_config = config
|
||||
return unified_model_interface.load_model(config)
|
||||
|
||||
def _ensure_model_loaded(self):
|
||||
signature = self._build_load_signature()
|
||||
if signature != self._load_signature or self._last_config is None:
|
||||
self.load_model(
|
||||
model_variant=self._model_selection(),
|
||||
device=self.config.get("device", "auto"),
|
||||
dtype=self.config.get("dtype", "auto"),
|
||||
)
|
||||
self._load_signature = signature
|
||||
|
||||
def _get_engine(self):
|
||||
if self._last_config is None:
|
||||
self._ensure_model_loaded()
|
||||
from utils.models.unified_model_interface import unified_model_interface
|
||||
|
||||
return unified_model_interface.load_model(self._last_config)
|
||||
|
||||
@staticmethod
|
||||
def _mono_waveform(waveform: Any) -> torch.Tensor:
|
||||
audio = torch.as_tensor(waveform, dtype=torch.float32).detach().cpu()
|
||||
if audio.ndim == 3:
|
||||
if audio.shape[0] != 1:
|
||||
raise ValueError(
|
||||
"Audio8 TTS reference audio must contain one batch item"
|
||||
)
|
||||
audio = audio[0]
|
||||
if audio.ndim == 2:
|
||||
audio = audio.mean(dim=0)
|
||||
if audio.ndim != 1 or audio.numel() == 0:
|
||||
raise ValueError(
|
||||
"Audio8 TTS reference audio must be non-empty mono or "
|
||||
"channels-first audio"
|
||||
)
|
||||
return audio.contiguous()
|
||||
|
||||
def _in_memory_reference(
|
||||
self,
|
||||
waveform: Any,
|
||||
sample_rate: Any,
|
||||
) -> Tuple[Dict[str, Any], str]:
|
||||
mono = self._mono_waveform(waveform)
|
||||
sample_rate = int(sample_rate)
|
||||
comfy_audio = {
|
||||
"waveform": mono.unsqueeze(0),
|
||||
"sample_rate": sample_rate,
|
||||
}
|
||||
processor_audio = {
|
||||
"array": mono,
|
||||
"sampling_rate": sample_rate,
|
||||
}
|
||||
return (
|
||||
processor_audio,
|
||||
generate_stable_audio_component(reference_audio=comfy_audio),
|
||||
)
|
||||
|
||||
def _extract_voice_reference(
|
||||
self,
|
||||
voice_ref: Optional[Dict[str, Any]],
|
||||
) -> Tuple[Any, str, str]:
|
||||
if not isinstance(voice_ref, dict):
|
||||
return None, "", "default_voice"
|
||||
|
||||
reference_text = str(
|
||||
voice_ref.get("reference_text")
|
||||
or voice_ref.get("prompt_text")
|
||||
or voice_ref.get("text")
|
||||
or ""
|
||||
).strip()
|
||||
reference_audio = effective_voice_audio(voice_ref)
|
||||
if reference_audio is None:
|
||||
return None, reference_text, "default_voice"
|
||||
|
||||
if isinstance(reference_audio, (str, os.PathLike)):
|
||||
path = os.fspath(reference_audio)
|
||||
component = generate_stable_audio_component(audio_file_path=path)
|
||||
return path, reference_text, component
|
||||
|
||||
if isinstance(reference_audio, dict):
|
||||
waveform = reference_audio.get(
|
||||
"waveform",
|
||||
reference_audio.get("array"),
|
||||
)
|
||||
sample_rate = reference_audio.get(
|
||||
"sample_rate",
|
||||
reference_audio.get("sampling_rate"),
|
||||
)
|
||||
if waveform is None or sample_rate is None:
|
||||
raise ValueError(
|
||||
"Audio8 TTS in-memory reference requires waveform and sample_rate"
|
||||
)
|
||||
normalized, component = self._in_memory_reference(
|
||||
waveform,
|
||||
sample_rate,
|
||||
)
|
||||
return normalized, reference_text, component
|
||||
|
||||
if isinstance(reference_audio, (tuple, list)) and len(reference_audio) == 2:
|
||||
normalized, component = self._in_memory_reference(
|
||||
reference_audio[0],
|
||||
reference_audio[1],
|
||||
)
|
||||
return normalized, reference_text, component
|
||||
|
||||
if torch.is_tensor(reference_audio):
|
||||
normalized, component = self._in_memory_reference(
|
||||
reference_audio,
|
||||
voice_ref.get("sample_rate", self.SAMPLE_RATE),
|
||||
)
|
||||
return normalized, reference_text, component
|
||||
|
||||
raise TypeError(
|
||||
f"Unsupported Audio8 TTS reference type: {type(reference_audio)}"
|
||||
)
|
||||
|
||||
def generate_single(
|
||||
self,
|
||||
text: str,
|
||||
voice_ref: Optional[Dict[str, Any]],
|
||||
seed: int = 0,
|
||||
enable_audio_cache: bool = True,
|
||||
character_name: Optional[str] = None,
|
||||
) -> torch.Tensor:
|
||||
"""Generate one raw utterance; chunking remains processor-owned."""
|
||||
text = str(text or "").strip()
|
||||
if not text:
|
||||
return torch.zeros(1, 0, dtype=torch.float32)
|
||||
|
||||
reference_audio, reference_text, audio_component = (
|
||||
self._extract_voice_reference(voice_ref)
|
||||
)
|
||||
if reference_audio is not None and not reference_text:
|
||||
raise ValueError(
|
||||
"Audio8 TTS voice cloning requires the exact transcript of "
|
||||
"the reference audio. Add reference text to Character Voices "
|
||||
"or the narrator voice."
|
||||
)
|
||||
|
||||
params = {
|
||||
"model_variant": self._model_selection(),
|
||||
"max_new_tokens": int(self.config.get("max_new_tokens", 1024)),
|
||||
"retry_max_new_tokens": int(self.config.get("retry_max_new_tokens", 2000)),
|
||||
"temperature": float(self.config.get("temperature", 0.8)),
|
||||
"top_p": float(self.config.get("top_p", 0.95)),
|
||||
"top_k": int(self.config.get("top_k", 50)),
|
||||
"do_sample": bool(self.config.get("do_sample", True)),
|
||||
"dtype": self.config.get("dtype", "auto"),
|
||||
"device": resolve_torch_device(self.config.get("device", "auto")),
|
||||
"seed": int(seed),
|
||||
}
|
||||
|
||||
cache_key = None
|
||||
if enable_audio_cache:
|
||||
cache_key = self.audio_cache.generate_cache_key(
|
||||
"audio8_tts",
|
||||
text=text,
|
||||
audio_component=audio_component,
|
||||
reference_text=reference_text,
|
||||
character=character_name or "narrator",
|
||||
**params,
|
||||
)
|
||||
cached = self.audio_cache.get_cached_audio(cache_key)
|
||||
if cached:
|
||||
display_name = resolved_character_label(
|
||||
character_name or "narrator",
|
||||
voice_ref,
|
||||
)
|
||||
print(
|
||||
"💾 Using cached Audio8 TTS audio for "
|
||||
f"'{display_name}': '{text[:30]}...'"
|
||||
)
|
||||
return cached[0]
|
||||
|
||||
self._ensure_model_loaded()
|
||||
audio = self._get_engine().generate(
|
||||
text=text,
|
||||
reference_audio=reference_audio,
|
||||
reference_text=reference_text or None,
|
||||
max_new_tokens=params["max_new_tokens"],
|
||||
retry_max_new_tokens=params["retry_max_new_tokens"],
|
||||
temperature=params["temperature"],
|
||||
top_p=params["top_p"],
|
||||
top_k=params["top_k"],
|
||||
do_sample=params["do_sample"],
|
||||
seed=params["seed"],
|
||||
)
|
||||
if not isinstance(audio, torch.Tensor):
|
||||
audio = torch.as_tensor(audio, dtype=torch.float32)
|
||||
audio = audio.detach().float().cpu()
|
||||
if audio.ndim == 1:
|
||||
audio = audio.unsqueeze(0)
|
||||
if audio.ndim != 2:
|
||||
raise RuntimeError(
|
||||
f"Audio8 TTS returned invalid audio shape: {tuple(audio.shape)}"
|
||||
)
|
||||
|
||||
if cache_key:
|
||||
self.audio_cache.cache_audio(
|
||||
cache_key,
|
||||
audio,
|
||||
audio.shape[-1] / self.SAMPLE_RATE,
|
||||
)
|
||||
return audio
|
||||
@@ -1,537 +0,0 @@
|
||||
"""Adapter between TTS Audio Suite generation and official VoxCPM inference."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import tempfile
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, Dict, Iterator, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from utils.audio.audio_hash import generate_stable_audio_component
|
||||
from utils.audio.cache import get_audio_cache
|
||||
from utils.audio.processing import AudioProcessingUtils
|
||||
from utils.models.factory_config import ModelLoadConfig
|
||||
from utils.voice.reference import effective_voice_audio
|
||||
|
||||
|
||||
class VoxCPMEngineAdapter:
|
||||
"""Translate suite voice/config/cache data into official VoxCPM calls."""
|
||||
|
||||
DEFAULT_MODEL = "VoxCPM2"
|
||||
SAMPLE_RATE = 48000
|
||||
SUPPORTED_ARCHITECTURES = frozenset({"voxcpm", "voxcpm2"})
|
||||
|
||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||
self.config = config.copy() if config else {}
|
||||
self.audio_cache = get_audio_cache()
|
||||
self._last_config: Optional[ModelLoadConfig] = None
|
||||
self._load_signature: Optional[Tuple[Any, ...]] = None
|
||||
self._model_spec: Optional[Dict[str, Any]] = None
|
||||
self._resolved_model_path: Optional[str] = None
|
||||
self._model_spec_selection: Optional[str] = None
|
||||
self.sample_rate = self.SAMPLE_RATE
|
||||
self.architecture = "voxcpm2"
|
||||
|
||||
def update_config(self, new_config: Dict[str, Any]):
|
||||
previous_selection = self._model_selection()
|
||||
previous_lora = self._lora_adapter()
|
||||
self.config = new_config.copy() if new_config else {}
|
||||
if (
|
||||
self._model_selection() != previous_selection
|
||||
or self._lora_adapter() != previous_lora
|
||||
):
|
||||
self._model_spec = None
|
||||
self._resolved_model_path = None
|
||||
self._model_spec_selection = None
|
||||
|
||||
def _model_selection(self) -> str:
|
||||
return str(
|
||||
self.config.get("model_variant")
|
||||
or self.config.get("model_name")
|
||||
or self.DEFAULT_MODEL
|
||||
)
|
||||
|
||||
def _lora_adapter(self) -> Optional[str]:
|
||||
value = str(self.config.get("lora_adapter") or "").strip()
|
||||
return os.path.abspath(os.path.expanduser(value)) if value else None
|
||||
|
||||
def _lora_signature(self) -> Optional[Tuple[Any, ...]]:
|
||||
path = self._lora_adapter()
|
||||
if not path:
|
||||
return None
|
||||
values = [path]
|
||||
for filename in ("lora_config.json", "lora_weights.safetensors"):
|
||||
target = os.path.join(path, filename)
|
||||
try:
|
||||
stat = os.stat(target)
|
||||
values.extend((stat.st_size, stat.st_mtime_ns))
|
||||
except OSError:
|
||||
values.extend((None, None))
|
||||
return tuple(values)
|
||||
|
||||
def _build_load_signature(self) -> Tuple[Any, ...]:
|
||||
return (
|
||||
self._model_selection(),
|
||||
self.config.get("device", "auto"),
|
||||
bool(self.config.get("optimize", False)),
|
||||
self.config.get("runtime_mode", "main_environment"),
|
||||
self.config.get("runtime_profile"),
|
||||
self._lora_signature(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _validate_spec(cls, spec: Dict[str, Any]) -> Dict[str, Any]:
|
||||
architecture = str(spec.get("architecture") or "").lower()
|
||||
if architecture not in cls.SUPPORTED_ARCHITECTURES:
|
||||
raise ValueError(
|
||||
f"Unsupported VoxCPM checkpoint architecture '{architecture or 'missing'}'. "
|
||||
f"Expected one of: {', '.join(sorted(cls.SUPPORTED_ARCHITECTURES))}."
|
||||
)
|
||||
sample_rate = int(spec.get("sample_rate") or 0)
|
||||
if sample_rate <= 0:
|
||||
raise ValueError("VoxCPM checkpoint spec has no valid output sample rate.")
|
||||
|
||||
validated = dict(spec)
|
||||
validated["architecture"] = architecture
|
||||
validated["sample_rate"] = sample_rate
|
||||
return validated
|
||||
|
||||
def _resolve_model(self, selection: Optional[str] = None) -> Tuple[str, Dict[str, Any]]:
|
||||
from engines.voxcpm.voxcpm_downloader import VoxCPMDownloader
|
||||
|
||||
chosen = str(selection or self._model_selection())
|
||||
downloader = VoxCPMDownloader()
|
||||
resolved_path = downloader.resolve_model_path(chosen)
|
||||
spec = self._validate_spec(
|
||||
downloader.get_model_spec(chosen, resolved_path=resolved_path)
|
||||
)
|
||||
return resolved_path, spec
|
||||
|
||||
def load_model(
|
||||
self,
|
||||
model_variant: Optional[str] = None,
|
||||
device: str = "auto",
|
||||
optimize: bool = False,
|
||||
runtime_mode: str = "main_environment",
|
||||
runtime_profile: Optional[str] = None,
|
||||
):
|
||||
"""Load one verified checkpoint through the unified model interface."""
|
||||
from utils.models.unified_model_interface import unified_model_interface
|
||||
|
||||
selection = str(model_variant or self._model_selection())
|
||||
self.config.update(
|
||||
{
|
||||
"model_variant": selection,
|
||||
"device": device,
|
||||
"optimize": bool(optimize),
|
||||
"runtime_mode": runtime_mode,
|
||||
"runtime_profile": runtime_profile,
|
||||
}
|
||||
)
|
||||
resolved_path, spec = self._resolve_model(selection)
|
||||
canonical = str(spec.get("canonical") or selection)
|
||||
|
||||
load_config = ModelLoadConfig(
|
||||
engine_name="voxcpm",
|
||||
model_type="tts",
|
||||
model_name=canonical,
|
||||
model_path=resolved_path,
|
||||
repo_id=spec.get("repo_id"),
|
||||
device=device,
|
||||
runtime_mode=runtime_mode,
|
||||
runtime_profile=runtime_profile,
|
||||
additional_params={
|
||||
"architecture": spec["architecture"],
|
||||
"sample_rate": spec["sample_rate"],
|
||||
"optimize": bool(optimize),
|
||||
"load_denoiser": False,
|
||||
"lora_adapter": self._lora_adapter(),
|
||||
"lora_signature": self._lora_signature(),
|
||||
},
|
||||
)
|
||||
self._last_config = load_config
|
||||
self._model_spec = spec
|
||||
self._resolved_model_path = resolved_path
|
||||
self._model_spec_selection = selection
|
||||
self.architecture = spec["architecture"]
|
||||
self.sample_rate = spec["sample_rate"]
|
||||
self.SAMPLE_RATE = self.sample_rate
|
||||
self._load_signature = (
|
||||
selection,
|
||||
device,
|
||||
bool(optimize),
|
||||
runtime_mode,
|
||||
runtime_profile,
|
||||
self._lora_signature(),
|
||||
)
|
||||
return unified_model_interface.load_model(load_config)
|
||||
|
||||
def _ensure_model_loaded(self):
|
||||
signature = self._build_load_signature()
|
||||
if signature == self._load_signature and self._last_config is not None:
|
||||
return
|
||||
self.load_model(
|
||||
model_variant=self._model_selection(),
|
||||
device=self.config.get("device", "auto"),
|
||||
optimize=bool(self.config.get("optimize", False)),
|
||||
runtime_mode=self.config.get("runtime_mode", "main_environment"),
|
||||
runtime_profile=self.config.get("runtime_profile"),
|
||||
)
|
||||
|
||||
def _get_engine(self):
|
||||
self._ensure_model_loaded()
|
||||
from utils.models.unified_model_interface import unified_model_interface
|
||||
|
||||
return unified_model_interface.load_model(self._last_config)
|
||||
|
||||
def get_model_spec(self) -> Dict[str, Any]:
|
||||
selection = self._model_selection()
|
||||
if (
|
||||
self._model_spec is None
|
||||
or self._resolved_model_path is None
|
||||
or self._model_spec_selection != selection
|
||||
):
|
||||
resolved_path, spec = self._resolve_model(selection)
|
||||
self._model_spec = spec
|
||||
self._resolved_model_path = resolved_path
|
||||
self._model_spec_selection = selection
|
||||
self.architecture = spec["architecture"]
|
||||
self.sample_rate = spec["sample_rate"]
|
||||
self.SAMPLE_RATE = self.sample_rate
|
||||
return dict(self._model_spec)
|
||||
|
||||
def get_sample_rate(self) -> int:
|
||||
return int(self.get_model_spec()["sample_rate"])
|
||||
|
||||
def get_architecture(self) -> str:
|
||||
return str(self.get_model_spec()["architecture"])
|
||||
|
||||
def supports_voice_cloning(self) -> bool:
|
||||
return True
|
||||
|
||||
def supports_reference_only(self) -> bool:
|
||||
return self.get_architecture() == "voxcpm2"
|
||||
|
||||
def supports_voice_instruction(self) -> bool:
|
||||
return self.get_architecture() == "voxcpm2"
|
||||
|
||||
def supports_voice_design(self) -> bool:
|
||||
return self.get_architecture() == "voxcpm2"
|
||||
|
||||
@staticmethod
|
||||
def _reference_text(voice_ref: Optional[Dict[str, Any]]) -> str:
|
||||
if not isinstance(voice_ref, dict):
|
||||
return ""
|
||||
return str(
|
||||
voice_ref.get("reference_text")
|
||||
or voice_ref.get("prompt_text")
|
||||
or voice_ref.get("text")
|
||||
or ""
|
||||
).strip()
|
||||
|
||||
def _extract_reference(
|
||||
self, voice_ref: Optional[Dict[str, Any]]
|
||||
) -> Tuple[Optional[torch.Tensor], Optional[int], str, str]:
|
||||
"""Return waveform, rate, transcript, and a content-based cache identity."""
|
||||
transcript = self._reference_text(voice_ref)
|
||||
audio = effective_voice_audio(voice_ref)
|
||||
if audio is None or (isinstance(audio, str) and audio.strip().lower() in {"", "none"}):
|
||||
return None, None, "", "default_voice"
|
||||
|
||||
if isinstance(audio, str):
|
||||
if not os.path.isfile(audio):
|
||||
raise FileNotFoundError(f"VoxCPM reference audio does not exist: {audio}")
|
||||
audio_component = generate_stable_audio_component(audio_file_path=audio)
|
||||
waveform, sample_rate = AudioProcessingUtils.safe_load_audio(audio)
|
||||
return waveform, int(sample_rate), transcript, audio_component
|
||||
|
||||
if isinstance(audio, dict) and "waveform" in audio:
|
||||
waveform = audio["waveform"]
|
||||
if not torch.is_tensor(waveform):
|
||||
raise TypeError(
|
||||
f"VoxCPM reference waveform must be a torch.Tensor, got {type(waveform)}"
|
||||
)
|
||||
sample_rate = int(audio.get("sample_rate") or self.get_sample_rate())
|
||||
return (
|
||||
waveform,
|
||||
sample_rate,
|
||||
transcript,
|
||||
generate_stable_audio_component(reference_audio=audio),
|
||||
)
|
||||
|
||||
if torch.is_tensor(audio):
|
||||
sample_rate = int(
|
||||
(voice_ref or {}).get("sample_rate") or self.get_sample_rate()
|
||||
)
|
||||
audio_dict = {"waveform": audio, "sample_rate": sample_rate}
|
||||
return (
|
||||
audio,
|
||||
sample_rate,
|
||||
transcript,
|
||||
generate_stable_audio_component(reference_audio=audio_dict),
|
||||
)
|
||||
|
||||
raise TypeError(f"Unsupported VoxCPM voice reference type: {type(audio)}")
|
||||
|
||||
@staticmethod
|
||||
@contextmanager
|
||||
def _temporary_reference_wav(
|
||||
waveform: Optional[torch.Tensor], sample_rate: Optional[int]
|
||||
) -> Iterator[Optional[str]]:
|
||||
"""Materialize one mono WAV and remove it even if saving/inference fails."""
|
||||
if waveform is None:
|
||||
yield None
|
||||
return
|
||||
|
||||
if not sample_rate or int(sample_rate) <= 0:
|
||||
raise ValueError("VoxCPM reference audio has no valid sample rate.")
|
||||
|
||||
temp_path = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_file:
|
||||
temp_path = temp_file.name
|
||||
|
||||
audio = waveform.detach().float().cpu()
|
||||
if audio.dim() == 3:
|
||||
if audio.shape[0] != 1:
|
||||
raise ValueError(
|
||||
"VoxCPM reference audio must contain exactly one batch item."
|
||||
)
|
||||
audio = audio.squeeze(0)
|
||||
if audio.dim() == 2:
|
||||
audio = audio.mean(dim=0)
|
||||
elif audio.dim() != 1:
|
||||
raise ValueError(
|
||||
f"Unsupported VoxCPM reference shape: {tuple(audio.shape)}"
|
||||
)
|
||||
|
||||
try:
|
||||
import soundfile as sf
|
||||
|
||||
sf.write(
|
||||
temp_path,
|
||||
audio.contiguous().numpy(),
|
||||
int(sample_rate),
|
||||
format="WAV",
|
||||
subtype="FLOAT",
|
||||
)
|
||||
except ImportError:
|
||||
# Keep the bridge usable in minimal environments without
|
||||
# soundfile/TorchCodec by writing a standard mono PCM16 WAV.
|
||||
import wave
|
||||
|
||||
pcm16 = (
|
||||
audio.clamp(-1.0, 1.0)
|
||||
.mul(32767.0)
|
||||
.round()
|
||||
.to(torch.int16)
|
||||
.contiguous()
|
||||
.numpy()
|
||||
)
|
||||
with wave.open(temp_path, "wb") as wav_file:
|
||||
wav_file.setnchannels(1)
|
||||
wav_file.setsampwidth(2)
|
||||
wav_file.setframerate(int(sample_rate))
|
||||
wav_file.writeframes(pcm16.tobytes())
|
||||
yield temp_path
|
||||
finally:
|
||||
if temp_path:
|
||||
try:
|
||||
os.unlink(temp_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def _sanitize_instruction(instruction: Any) -> str:
|
||||
cleaned = re.sub(r"[()（）]", "", str(instruction or ""))
|
||||
return re.sub(r"\s+", " ", cleaned).strip()
|
||||
|
||||
@staticmethod
|
||||
def _effective_seed(seed: Optional[int]) -> int:
|
||||
requested = int(seed or 0)
|
||||
return requested if requested != 0 else secrets.randbits(32)
|
||||
|
||||
def _generation_parameters(self, spec: Dict[str, Any]) -> Dict[str, Any]:
|
||||
min_len = int(self.config.get("min_len", 2))
|
||||
max_len = int(self.config.get("max_len", 4096))
|
||||
steps = int(self.config.get("inference_timesteps", 10))
|
||||
cfg_value = float(self.config.get("cfg_value", 2.0))
|
||||
retry_max = int(self.config.get("retry_badcase_max_times", 3))
|
||||
retry_ratio = float(self.config.get("retry_badcase_ratio_threshold", 6.0))
|
||||
|
||||
if min_len < 0 or max_len < min_len:
|
||||
raise ValueError("VoxCPM requires 0 <= min_len <= max_len.")
|
||||
if not 0.1 <= cfg_value <= 10.0:
|
||||
raise ValueError("VoxCPM cfg_value must be between 0.1 and 10.0.")
|
||||
if not 1 <= steps <= 100:
|
||||
raise ValueError(
|
||||
"VoxCPM inference_timesteps must be between 1 and 100."
|
||||
)
|
||||
if retry_max < 1 or retry_ratio <= 0:
|
||||
raise ValueError("VoxCPM bad-case retry settings must be positive.")
|
||||
if str(spec.get("canonical")) == "VoxCPM-0.5B":
|
||||
max_len = min(max_len, 4096)
|
||||
|
||||
return {
|
||||
"cfg_value": cfg_value,
|
||||
"inference_timesteps": steps,
|
||||
"min_len": min_len,
|
||||
"max_len": max_len,
|
||||
"normalize": bool(self.config.get("normalize_text", False)),
|
||||
"retry_badcase": bool(self.config.get("retry_badcase", True)),
|
||||
"retry_badcase_max_times": retry_max,
|
||||
"retry_badcase_ratio_threshold": retry_ratio,
|
||||
}
|
||||
|
||||
def generate_single(
|
||||
self,
|
||||
text: str,
|
||||
voice_ref: Optional[Dict[str, Any]],
|
||||
seed: int = 0,
|
||||
enable_audio_cache: bool = True,
|
||||
character_name: Optional[str] = None,
|
||||
) -> torch.Tensor:
|
||||
target_text = str(text or "").strip()
|
||||
if not target_text:
|
||||
return torch.zeros(1, 0, dtype=torch.float32)
|
||||
|
||||
spec = self.get_model_spec()
|
||||
params = self._generation_parameters(spec)
|
||||
waveform, reference_sample_rate, transcript, audio_component = (
|
||||
self._extract_reference(voice_ref)
|
||||
)
|
||||
instruction = self._sanitize_instruction(
|
||||
self.config.get("voice_instruction", "")
|
||||
)
|
||||
architecture = str(spec["architecture"])
|
||||
is_v2 = architecture == "voxcpm2"
|
||||
|
||||
if instruction and not is_v2:
|
||||
raise ValueError(
|
||||
f"{spec.get('canonical', self._model_selection())} does not support "
|
||||
"voice instructions. Use VoxCPM2 or clear voice_instruction."
|
||||
)
|
||||
if self.config.get("model_role") == "voice_design" and not is_v2:
|
||||
raise ValueError("Voice design is supported only by VoxCPM2 checkpoints.")
|
||||
if waveform is not None and not is_v2 and not transcript:
|
||||
raise ValueError(
|
||||
"Legacy VoxCPM voice cloning requires the exact transcript of the "
|
||||
"reference audio in reference_text."
|
||||
)
|
||||
|
||||
conditioned_text = f"({instruction}){target_text}" if instruction else target_text
|
||||
effective_seed = self._effective_seed(seed)
|
||||
model_variant = self._model_selection()
|
||||
|
||||
cache_key = None
|
||||
if enable_audio_cache:
|
||||
cache_key = self.audio_cache.generate_cache_key(
|
||||
"voxcpm",
|
||||
text=target_text,
|
||||
audio_component=audio_component,
|
||||
reference_text=transcript,
|
||||
model_variant=model_variant,
|
||||
architecture=architecture,
|
||||
voice_instruction=instruction,
|
||||
cfg_value=params["cfg_value"],
|
||||
inference_timesteps=params["inference_timesteps"],
|
||||
min_len=params["min_len"],
|
||||
max_len=params["max_len"],
|
||||
normalize_text=params["normalize"],
|
||||
retry_badcase=params["retry_badcase"],
|
||||
retry_badcase_max_times=params["retry_badcase_max_times"],
|
||||
retry_badcase_ratio_threshold=params[
|
||||
"retry_badcase_ratio_threshold"
|
||||
],
|
||||
seed=effective_seed,
|
||||
device=self.config.get("device", "auto"),
|
||||
optimize=bool(self.config.get("optimize", False)),
|
||||
runtime_mode=self.config.get("runtime_mode", "main_environment"),
|
||||
runtime_profile=self.config.get("runtime_profile"),
|
||||
model_path=self._resolved_model_path,
|
||||
lora_adapter=self._lora_signature(),
|
||||
load_denoiser=False,
|
||||
sample_rate=spec["sample_rate"],
|
||||
character=character_name or "narrator",
|
||||
)
|
||||
cached = self.audio_cache.get_cached_audio(cache_key)
|
||||
if cached:
|
||||
print(
|
||||
f"💾 Using cached VoxCPM audio for "
|
||||
f"'{character_name or 'narrator'}': '{target_text[:30]}...'"
|
||||
)
|
||||
return cached[0]
|
||||
|
||||
engine = self._get_engine()
|
||||
engine_architecture = str(engine.architecture).lower()
|
||||
engine_sample_rate = int(engine.sample_rate)
|
||||
if engine_architecture != architecture:
|
||||
raise RuntimeError(
|
||||
f"VoxCPM checkpoint architecture changed after load: spec={architecture}, "
|
||||
f"runtime={engine_architecture}."
|
||||
)
|
||||
if engine_sample_rate != int(spec["sample_rate"]):
|
||||
raise RuntimeError(
|
||||
f"VoxCPM sample-rate mismatch: spec={spec['sample_rate']}, "
|
||||
f"runtime={engine_sample_rate}."
|
||||
)
|
||||
|
||||
with self._temporary_reference_wav(
|
||||
waveform, reference_sample_rate
|
||||
) as reference_path:
|
||||
prompt_wav_path = None
|
||||
prompt_text = None
|
||||
reference_wav_path = None
|
||||
if reference_path:
|
||||
if is_v2:
|
||||
reference_wav_path = reference_path
|
||||
if transcript:
|
||||
# VoxCPM2 ultimate mode uses the same audio for both caches.
|
||||
prompt_wav_path = reference_path
|
||||
prompt_text = transcript
|
||||
else:
|
||||
prompt_wav_path = reference_path
|
||||
prompt_text = transcript
|
||||
|
||||
audio = engine.generate(
|
||||
text=conditioned_text,
|
||||
prompt_wav_path=prompt_wav_path,
|
||||
prompt_text=prompt_text,
|
||||
reference_wav_path=reference_wav_path,
|
||||
cfg_value=params["cfg_value"],
|
||||
inference_timesteps=params["inference_timesteps"],
|
||||
min_len=params["min_len"],
|
||||
max_len=params["max_len"],
|
||||
normalize=params["normalize"],
|
||||
retry_badcase=params["retry_badcase"],
|
||||
retry_badcase_max_times=params["retry_badcase_max_times"],
|
||||
retry_badcase_ratio_threshold=params[
|
||||
"retry_badcase_ratio_threshold"
|
||||
],
|
||||
seed=effective_seed,
|
||||
)
|
||||
|
||||
if not torch.is_tensor(audio):
|
||||
audio = torch.as_tensor(audio, dtype=torch.float32)
|
||||
audio = audio.detach().float().cpu()
|
||||
if audio.dim() == 1:
|
||||
audio = audio.unsqueeze(0)
|
||||
elif audio.dim() == 3 and audio.shape[0] == 1:
|
||||
audio = audio.squeeze(0)
|
||||
if audio.dim() != 2:
|
||||
raise ValueError(
|
||||
f"VoxCPM returned unsupported audio shape: {tuple(audio.shape)}"
|
||||
)
|
||||
|
||||
self.sample_rate = engine_sample_rate
|
||||
self.SAMPLE_RATE = engine_sample_rate
|
||||
if enable_audio_cache and cache_key:
|
||||
duration = self.audio_cache._calculate_duration(
|
||||
audio, "voxcpm", sample_rate=self.sample_rate
|
||||
)
|
||||
self.audio_cache.cache_audio(cache_key, audio, duration)
|
||||
return audio
|
||||
@@ -0,0 +1,6 @@
|
||||
"""Audio8 TTS engine integration."""
|
||||
|
||||
from .audio8_tts_engine import Audio8TTSEngine
|
||||
from .downloader import Audio8TTSDownloader
|
||||
|
||||
__all__ = ["Audio8TTSEngine", "Audio8TTSDownloader"]
|
||||
@@ -0,0 +1,375 @@
|
||||
"""Wrapper for the official Audio8 TTS remote-code model."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import random
|
||||
import warnings
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from engines.audio8_tts.downloader import Audio8TTSDownloader
|
||||
from engines.audio8_tts.progress import build_audio8_stopping_criteria
|
||||
from utils.device import resolve_torch_device
|
||||
|
||||
|
||||
class Audio8TTSEngine:
|
||||
"""ComfyUI-friendly wrapper around the official Audio8 model."""
|
||||
|
||||
SAMPLE_RATE = 44100
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_name: str = Audio8TTSDownloader.MODEL_NAME,
|
||||
device: str = "auto",
|
||||
dtype: str = "auto",
|
||||
model_dir: Optional[str] = None,
|
||||
):
|
||||
self.model_name = model_name
|
||||
self.device = resolve_torch_device(device)
|
||||
self.dtype_name = str(dtype or "auto").lower()
|
||||
self.dtype = self._resolve_dtype(self.dtype_name, self.device)
|
||||
self.model_dir = model_dir or Audio8TTSDownloader().resolve_model_path(
|
||||
model_name
|
||||
)
|
||||
self._processor = None
|
||||
self._model = None
|
||||
self._codec = None
|
||||
|
||||
@staticmethod
|
||||
def _resolve_dtype(dtype: str, device: str) -> torch.dtype:
|
||||
normalized = str(dtype or "auto").lower()
|
||||
valid = {
|
||||
"bfloat16": torch.bfloat16,
|
||||
"float16": torch.float16,
|
||||
"float32": torch.float32,
|
||||
}
|
||||
if normalized not in {"auto", *valid}:
|
||||
raise ValueError(
|
||||
"Audio8 TTS dtype must be auto, bfloat16, float16, or float32"
|
||||
)
|
||||
if str(device).startswith("cpu"):
|
||||
return torch.float32
|
||||
if normalized in valid:
|
||||
return valid[normalized]
|
||||
if str(device).startswith("cuda") and torch.cuda.is_available():
|
||||
major, _minor = torch.cuda.get_device_capability(torch.device(device))
|
||||
return torch.bfloat16 if major >= 8 else torch.float16
|
||||
if str(device).startswith("xpu"):
|
||||
return torch.bfloat16
|
||||
return torch.float16
|
||||
|
||||
def _ensure_model_loaded(self):
|
||||
"""Load the processor, model, and bundled codec strictly from local files."""
|
||||
if (
|
||||
self._processor is not None
|
||||
and self._model is not None
|
||||
and self._codec is not None
|
||||
):
|
||||
return self
|
||||
|
||||
if not os.path.isdir(self.model_dir):
|
||||
raise FileNotFoundError(
|
||||
f"Audio8 TTS local model directory does not exist: {self.model_dir}"
|
||||
)
|
||||
|
||||
try:
|
||||
from transformers import AutoModel, AutoProcessor
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
"Audio8 TTS requires the suite's shared Transformers 4 runtime"
|
||||
) from exc
|
||||
|
||||
print(
|
||||
f"📦 Loading Audio8 TTS from {self.model_dir} "
|
||||
f"on {self.device} ({self.dtype})"
|
||||
)
|
||||
with warnings.catch_warnings():
|
||||
warnings.filterwarnings(
|
||||
"ignore",
|
||||
message=(
|
||||
r"`torch\.nn\.utils\.weight_norm` is deprecated in favor of "
|
||||
r"`torch\.nn\.utils\.parametrizations\.weight_norm`\."
|
||||
),
|
||||
category=FutureWarning,
|
||||
)
|
||||
self._processor = AutoProcessor.from_pretrained(
|
||||
self.model_dir,
|
||||
trust_remote_code=True,
|
||||
local_files_only=True,
|
||||
)
|
||||
self._model = AutoModel.from_pretrained(
|
||||
self.model_dir,
|
||||
trust_remote_code=True,
|
||||
local_files_only=True,
|
||||
dtype=self.dtype,
|
||||
)
|
||||
self._model = self._model.eval().to(self.device)
|
||||
|
||||
# The official model stores the codec outside nn.Module registration.
|
||||
# Hold and move it explicitly so model offload/reload cannot strand it.
|
||||
self._codec = self._model.load_codec(
|
||||
device=self.device,
|
||||
dtype=self.dtype,
|
||||
)
|
||||
self._codec.eval()
|
||||
|
||||
sample_rate = int(
|
||||
getattr(self._model.config, "codec_sample_rate", self.SAMPLE_RATE)
|
||||
)
|
||||
if sample_rate != self.SAMPLE_RATE:
|
||||
raise RuntimeError(
|
||||
f"Audio8 TTS checkpoint reports unexpected sample rate "
|
||||
f"{sample_rate}; expected {self.SAMPLE_RATE}"
|
||||
)
|
||||
print("✅ Audio8 TTS model and codec loaded")
|
||||
return self
|
||||
|
||||
# Compatibility alias for callers using the shorter established name.
|
||||
_ensure_loaded = _ensure_model_loaded
|
||||
|
||||
def _clear_kv_caches(self) -> None:
|
||||
"""Drop every static slow/fast attention cache before device movement."""
|
||||
if self._model is None:
|
||||
return
|
||||
for collection_name in ("layers", "fast_layers"):
|
||||
for layer in getattr(self._model, collection_name, ()) or ():
|
||||
attention = getattr(layer, "attention", None)
|
||||
if attention is not None and hasattr(attention, "kv_cache"):
|
||||
attention.kv_cache = None
|
||||
|
||||
def to(self, device):
|
||||
"""Move all weights for ComfyUI Clear VRAM and reload handling."""
|
||||
target = resolve_torch_device(
|
||||
str(device) if not isinstance(device, str) else device
|
||||
)
|
||||
self._clear_kv_caches()
|
||||
self.device = target
|
||||
if self._model is not None:
|
||||
self._model = self._model.to(target).eval()
|
||||
if self._codec is not None:
|
||||
codec_dtype = torch.float32 if str(target).startswith("cpu") else self.dtype
|
||||
self._codec = self._codec.to(
|
||||
device=target,
|
||||
dtype=codec_dtype,
|
||||
).eval()
|
||||
return self
|
||||
|
||||
def parameters(self):
|
||||
"""Expose all weight tensors to ComfyUI memory accounting."""
|
||||
if self._model is not None:
|
||||
yield from self._model.parameters()
|
||||
if self._codec is not None:
|
||||
yield from self._codec.parameters()
|
||||
|
||||
@staticmethod
|
||||
def _mono_audio(waveform: Any) -> torch.Tensor:
|
||||
audio = torch.as_tensor(waveform, dtype=torch.float32).detach().cpu()
|
||||
if audio.ndim == 3:
|
||||
if audio.shape[0] != 1:
|
||||
raise ValueError(
|
||||
"Audio8 TTS reference audio must contain one batch item"
|
||||
)
|
||||
audio = audio[0]
|
||||
if audio.ndim == 2:
|
||||
audio = audio.mean(dim=0)
|
||||
if audio.ndim != 1 or audio.numel() == 0:
|
||||
raise ValueError(
|
||||
"Audio8 TTS reference audio must be non-empty mono or "
|
||||
"channels-first audio"
|
||||
)
|
||||
return audio.contiguous()
|
||||
|
||||
def _normalize_reference_audio(
|
||||
self,
|
||||
reference_audio: Any,
|
||||
) -> Tuple[Any, Optional[int]]:
|
||||
if isinstance(reference_audio, (str, os.PathLike)):
|
||||
return os.fspath(reference_audio), None
|
||||
if isinstance(reference_audio, dict):
|
||||
waveform = reference_audio.get(
|
||||
"waveform",
|
||||
reference_audio.get("array"),
|
||||
)
|
||||
sample_rate = reference_audio.get(
|
||||
"sample_rate",
|
||||
reference_audio.get("sampling_rate"),
|
||||
)
|
||||
if waveform is None or sample_rate is None:
|
||||
raise ValueError(
|
||||
"Audio8 TTS in-memory reference audio requires waveform "
|
||||
"and sample_rate"
|
||||
)
|
||||
return {
|
||||
"array": self._mono_audio(waveform),
|
||||
"sampling_rate": int(sample_rate),
|
||||
}, int(sample_rate)
|
||||
if isinstance(reference_audio, (tuple, list)) and len(reference_audio) == 2:
|
||||
waveform, sample_rate = reference_audio
|
||||
if not isinstance(sample_rate, (int, float)):
|
||||
raise ValueError(
|
||||
"Audio8 TTS reference tuple must be (waveform, sample_rate)"
|
||||
)
|
||||
return {
|
||||
"array": self._mono_audio(waveform),
|
||||
"sampling_rate": int(sample_rate),
|
||||
}, int(sample_rate)
|
||||
raise TypeError(
|
||||
f"Unsupported Audio8 TTS reference audio type: {type(reference_audio)}"
|
||||
)
|
||||
|
||||
def _prepare_inputs(
|
||||
self,
|
||||
text: str,
|
||||
reference_audio: Any,
|
||||
reference_text: Optional[str],
|
||||
) -> Dict[str, torch.Tensor]:
|
||||
processor_kwargs: Dict[str, Any] = {
|
||||
"text": [text],
|
||||
"return_tensors": "pt",
|
||||
}
|
||||
if reference_audio is not None:
|
||||
normalized_audio, sample_rate = self._normalize_reference_audio(
|
||||
reference_audio
|
||||
)
|
||||
processor_kwargs.update(
|
||||
reference_audio=[normalized_audio],
|
||||
reference_text=[reference_text],
|
||||
)
|
||||
if sample_rate is not None:
|
||||
processor_kwargs["sampling_rate"] = [sample_rate]
|
||||
|
||||
inputs = self._processor(**processor_kwargs)
|
||||
return {name: value.to(self.device) for name, value in inputs.items()}
|
||||
|
||||
def _make_generator(self, seed: int):
|
||||
seed = int(seed)
|
||||
random.seed(seed)
|
||||
torch.manual_seed(seed)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed_all(seed)
|
||||
try:
|
||||
return torch.Generator(device=torch.device(self.device)).manual_seed(seed)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _generate_once(
|
||||
self,
|
||||
inputs: Dict[str, torch.Tensor],
|
||||
*,
|
||||
max_new_tokens: int,
|
||||
temperature: float,
|
||||
top_p: float,
|
||||
top_k: int,
|
||||
do_sample: bool,
|
||||
generator,
|
||||
):
|
||||
criteria, tracker = build_audio8_stopping_criteria(max_new_tokens)
|
||||
try:
|
||||
output = self._model.generate(
|
||||
**inputs,
|
||||
max_new_tokens=max_new_tokens,
|
||||
temperature=temperature,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
do_sample=do_sample,
|
||||
generator=generator,
|
||||
stopping_criteria=criteria,
|
||||
return_dict_in_generate=True,
|
||||
)
|
||||
except BaseException:
|
||||
tracker.abort()
|
||||
raise
|
||||
tracker.close()
|
||||
return output
|
||||
|
||||
def generate(
|
||||
self,
|
||||
text: str,
|
||||
reference_audio: Any = None,
|
||||
reference_text: Optional[str] = None,
|
||||
*,
|
||||
max_new_tokens: int = 1024,
|
||||
retry_max_new_tokens: int = 2000,
|
||||
temperature: float = 0.8,
|
||||
top_p: float = 0.95,
|
||||
top_k: int = 50,
|
||||
do_sample: bool = True,
|
||||
seed: int = 42,
|
||||
) -> torch.Tensor:
|
||||
"""Generate one utterance as a CPU float tensor ``[channels, samples]``."""
|
||||
text = str(text or "").strip()
|
||||
if not text:
|
||||
return torch.zeros(1, 0, dtype=torch.float32)
|
||||
|
||||
reference_text = str(reference_text or "").strip()
|
||||
if reference_audio is not None and not reference_text:
|
||||
raise ValueError(
|
||||
"Audio8 TTS voice cloning requires the exact transcript of "
|
||||
"the reference audio"
|
||||
)
|
||||
if reference_audio is None and reference_text:
|
||||
raise ValueError(
|
||||
"Audio8 TTS reference_text requires matching reference audio"
|
||||
)
|
||||
|
||||
max_new_tokens = int(max_new_tokens)
|
||||
retry_max_new_tokens = int(retry_max_new_tokens)
|
||||
temperature = float(temperature)
|
||||
top_p = float(top_p)
|
||||
top_k = int(top_k)
|
||||
if max_new_tokens < 1:
|
||||
raise ValueError("Audio8 TTS max_new_tokens must be positive")
|
||||
if retry_max_new_tokens < max_new_tokens:
|
||||
raise ValueError(
|
||||
"Audio8 TTS retry_max_new_tokens must be >= max_new_tokens"
|
||||
)
|
||||
if temperature <= 0:
|
||||
raise ValueError("Audio8 TTS temperature must be positive")
|
||||
if not 0 < top_p <= 1:
|
||||
raise ValueError("Audio8 TTS top_p must be in (0, 1]")
|
||||
if top_k < 1:
|
||||
raise ValueError("Audio8 TTS top_k must be at least 1")
|
||||
|
||||
self._ensure_model_loaded()
|
||||
inputs = self._prepare_inputs(text, reference_audio, reference_text)
|
||||
generator = self._make_generator(seed)
|
||||
|
||||
output = self._generate_once(
|
||||
inputs,
|
||||
max_new_tokens=max_new_tokens,
|
||||
temperature=temperature,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
do_sample=bool(do_sample),
|
||||
generator=generator,
|
||||
)
|
||||
finished = bool(output.finished[0].item())
|
||||
|
||||
if not finished and retry_max_new_tokens > max_new_tokens:
|
||||
print(
|
||||
"⚠️ Audio8 TTS reached max_new_tokens without EOS; "
|
||||
f"retrying with {retry_max_new_tokens}"
|
||||
)
|
||||
output = self._generate_once(
|
||||
inputs,
|
||||
max_new_tokens=retry_max_new_tokens,
|
||||
temperature=temperature,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
do_sample=bool(do_sample),
|
||||
generator=generator,
|
||||
)
|
||||
finished = bool(output.finished[0].item())
|
||||
|
||||
if not finished:
|
||||
print(
|
||||
"⚠️ Audio8 TTS generation ended without EOS; returning the "
|
||||
"valid decoded frames"
|
||||
)
|
||||
|
||||
waveforms, waveform_lengths = self._model.decode_audio(output.codes)
|
||||
waveform_length = int(waveform_lengths[0].item())
|
||||
waveform = waveforms[0, :waveform_length].detach().float().cpu()
|
||||
return waveform.unsqueeze(0).contiguous()
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Organized model download and discovery for official Audio8 TTS."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import List, Optional
|
||||
|
||||
import folder_paths
|
||||
|
||||
from utils.downloads.unified_downloader import unified_downloader
|
||||
from utils.models.extra_paths import (
|
||||
get_all_tts_model_paths,
|
||||
get_preferred_download_path,
|
||||
)
|
||||
|
||||
|
||||
class Audio8TTSDownloader:
|
||||
"""Resolve Audio8 checkpoints without using the Hugging Face model cache."""
|
||||
|
||||
MODEL_NAME = "Audio8-TTS-Preview-0.6b"
|
||||
REPO_ID = "Audio8/Audio8-TTS-Preview-0.6b"
|
||||
REVISION = "1b17c91db5f4dccb6914aa4aa5cb0e56661a6c17"
|
||||
|
||||
# Complete runtime snapshot at the pinned revision. Non-runtime model-card
|
||||
# assets are deliberately excluded.
|
||||
REQUIRED_FILES = [
|
||||
"codec.pth",
|
||||
"config.json",
|
||||
"configuration_arktts.py",
|
||||
"generation_config.json",
|
||||
"model.safetensors",
|
||||
"modeling_arktts.py",
|
||||
"modeling_arktts_codec.py",
|
||||
"preprocessor_config.json",
|
||||
"processing_arktts.py",
|
||||
"processor_config.json",
|
||||
"special_tokens_map.json",
|
||||
"tokenizer.json",
|
||||
"tokenizer_config.json",
|
||||
]
|
||||
|
||||
def __init__(self, base_path: Optional[str] = None):
|
||||
if base_path is None:
|
||||
try:
|
||||
self.base_path = get_preferred_download_path(
|
||||
model_type="TTS",
|
||||
engine_name="audio8_tts",
|
||||
)
|
||||
except Exception:
|
||||
self.base_path = os.path.join(
|
||||
folder_paths.models_dir,
|
||||
"TTS",
|
||||
"audio8_tts",
|
||||
)
|
||||
else:
|
||||
self.base_path = os.path.abspath(os.fspath(base_path))
|
||||
os.makedirs(self.base_path, exist_ok=True)
|
||||
|
||||
def get_available_models(self) -> List[str]:
|
||||
"""Return the canonical model plus complete local installations."""
|
||||
models = [self.MODEL_NAME]
|
||||
canonical_path = os.path.normcase(
|
||||
os.path.abspath(os.path.join(self.base_path, self.MODEL_NAME))
|
||||
)
|
||||
for base_path in get_all_tts_model_paths("TTS"):
|
||||
for folder_name in ("audio8_tts", "Audio8_TTS", ""):
|
||||
root = (
|
||||
os.path.join(base_path, folder_name) if folder_name else base_path
|
||||
)
|
||||
if not os.path.isdir(root):
|
||||
continue
|
||||
for item in sorted(os.listdir(root)):
|
||||
candidate = os.path.join(root, item)
|
||||
if os.path.normcase(os.path.abspath(candidate)) == canonical_path:
|
||||
continue
|
||||
local_name = f"local:{item}"
|
||||
if local_name not in models and self._is_model_complete(candidate):
|
||||
models.append(local_name)
|
||||
return models
|
||||
|
||||
def resolve_model_path(
|
||||
self,
|
||||
model_identifier: str = MODEL_NAME,
|
||||
) -> str:
|
||||
"""Resolve an absolute path, ``local:`` name, or canonical model name."""
|
||||
model_identifier = str(model_identifier or self.MODEL_NAME).strip()
|
||||
|
||||
if os.path.isabs(model_identifier) or os.path.isdir(model_identifier):
|
||||
candidate = os.path.abspath(model_identifier)
|
||||
if self._is_model_complete(candidate, verbose=True):
|
||||
return candidate
|
||||
raise FileNotFoundError(
|
||||
f"Audio8 TTS model path is missing required files: {candidate}"
|
||||
)
|
||||
|
||||
if model_identifier.startswith("local:"):
|
||||
local_name = model_identifier[6:].strip()
|
||||
if not local_name:
|
||||
raise ValueError("Audio8 TTS local model name must not be empty")
|
||||
for base_path in get_all_tts_model_paths("TTS"):
|
||||
for folder_name in ("audio8_tts", "Audio8_TTS", ""):
|
||||
candidate = (
|
||||
os.path.join(base_path, folder_name, local_name)
|
||||
if folder_name
|
||||
else os.path.join(base_path, local_name)
|
||||
)
|
||||
if self._is_model_complete(candidate):
|
||||
print(f"📁 Using local Audio8 TTS model: {candidate}")
|
||||
return candidate
|
||||
raise FileNotFoundError(
|
||||
f"Local Audio8 TTS model not found or incomplete: {local_name}"
|
||||
)
|
||||
|
||||
if model_identifier != self.MODEL_NAME:
|
||||
raise ValueError(f"Unknown Audio8 TTS model: {model_identifier}")
|
||||
return self.get_model_path()
|
||||
|
||||
def get_model_path(self, model_name: str = MODEL_NAME) -> str:
|
||||
"""Return the organized canonical model path, downloading if needed."""
|
||||
if model_name != self.MODEL_NAME:
|
||||
raise ValueError(f"Unknown Audio8 TTS model: {model_name}")
|
||||
model_dir = os.path.join(self.base_path, self.MODEL_NAME)
|
||||
if not self._is_model_complete(model_dir):
|
||||
return self.download_model(model_dir)
|
||||
return model_dir
|
||||
|
||||
def download_model(
|
||||
self,
|
||||
model_dir: Optional[str] = None,
|
||||
force: bool = False,
|
||||
) -> str:
|
||||
"""Download the pinned official snapshot into the organized model folder."""
|
||||
model_dir = model_dir or os.path.join(self.base_path, self.MODEL_NAME)
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("📦 Audio8 TTS Model Download")
|
||||
print("=" * 60)
|
||||
print(f"Repository: {self.REPO_ID}")
|
||||
print(f"Revision: {self.REVISION}")
|
||||
print(f"Target: {model_dir}")
|
||||
print("=" * 60 + "\n")
|
||||
|
||||
unified_downloader.download_huggingface_snapshot(
|
||||
repo_id=self.REPO_ID,
|
||||
target_dir=model_dir,
|
||||
revision=self.REVISION,
|
||||
allow_patterns=self.REQUIRED_FILES,
|
||||
required_files=self.REQUIRED_FILES,
|
||||
force_download=bool(force),
|
||||
description=self.MODEL_NAME,
|
||||
)
|
||||
if not self._is_model_complete(model_dir, verbose=True):
|
||||
raise RuntimeError(
|
||||
f"Downloaded Audio8 TTS model is incomplete: {model_dir}"
|
||||
)
|
||||
print(f"✅ Audio8 TTS model ready: {model_dir}")
|
||||
return model_dir
|
||||
|
||||
def _is_model_complete(
|
||||
self,
|
||||
model_dir: str,
|
||||
*,
|
||||
verbose: bool = False,
|
||||
) -> bool:
|
||||
if not os.path.isdir(model_dir):
|
||||
return False
|
||||
missing = [
|
||||
rel_path
|
||||
for rel_path in self.REQUIRED_FILES
|
||||
if not os.path.isfile(os.path.join(model_dir, rel_path))
|
||||
or os.path.getsize(os.path.join(model_dir, rel_path)) <= 0
|
||||
]
|
||||
if missing and verbose:
|
||||
print(
|
||||
f"❌ Audio8 TTS model incomplete. Missing or empty "
|
||||
f"{len(missing)} file(s):"
|
||||
)
|
||||
for rel_path in missing:
|
||||
print(f" - {rel_path}")
|
||||
return not missing
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Native generation progress and interruption support for Audio8 TTS."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Any, Optional
|
||||
|
||||
import torch
|
||||
from transformers import StoppingCriteria, StoppingCriteriaList
|
||||
|
||||
|
||||
def _make_comfy_progress_bar(total_steps: int) -> Optional[Any]:
|
||||
try:
|
||||
from comfy.utils import ProgressBar
|
||||
|
||||
return ProgressBar(total_steps)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
class Audio8TTSStoppingCriteria(StoppingCriteria):
|
||||
"""Track native codec-frame generation and honor ComfyUI interruption."""
|
||||
|
||||
def __init__(self, max_steps: int, progress_bar: Optional[Any] = None):
|
||||
self.max_steps = max(1, int(max_steps))
|
||||
self.progress_bar = progress_bar
|
||||
self.steps = 0
|
||||
self.start_time = time.monotonic()
|
||||
self.last_print_time = self.start_time
|
||||
self.last_print_step = 0
|
||||
self.rendered = False
|
||||
|
||||
@staticmethod
|
||||
def _check_interrupted() -> None:
|
||||
try:
|
||||
import comfy.model_management as model_management
|
||||
except Exception:
|
||||
return
|
||||
|
||||
checker = getattr(
|
||||
model_management,
|
||||
"throw_exception_if_processing_interrupted",
|
||||
None,
|
||||
)
|
||||
if callable(checker):
|
||||
checker()
|
||||
elif getattr(model_management, "interrupt_processing", False):
|
||||
raise InterruptedError("Audio8 TTS generation interrupted by user")
|
||||
|
||||
def __call__(self, input_ids, scores, **kwargs):
|
||||
del scores, kwargs
|
||||
self._check_interrupted()
|
||||
self.steps = min(self.steps + 1, self.max_steps)
|
||||
if self.progress_bar is not None:
|
||||
try:
|
||||
self.progress_bar.update(1)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
now = time.monotonic()
|
||||
if (
|
||||
self.steps == 1
|
||||
or self.steps >= self.max_steps
|
||||
or now - self.last_print_time >= 0.5
|
||||
):
|
||||
delta_time = max(now - self.last_print_time, 1e-6)
|
||||
delta_steps = self.steps - self.last_print_step
|
||||
rate = delta_steps / delta_time
|
||||
elapsed = now - self.start_time
|
||||
width = 12
|
||||
filled = min(
|
||||
width,
|
||||
int(width * self.steps / self.max_steps),
|
||||
)
|
||||
bar = "█" * filled + "░" * (width - filled)
|
||||
print(
|
||||
f"\r Audio8: [{bar}] {self.steps}/{self.max_steps} | "
|
||||
f"{rate:.1f} frames/s | {elapsed:.1f}s ",
|
||||
end="",
|
||||
flush=True,
|
||||
)
|
||||
self.last_print_time = now
|
||||
self.last_print_step = self.steps
|
||||
self.rendered = True
|
||||
|
||||
# The official model handles EOS itself. Interruption raises above.
|
||||
batch_size = int(input_ids.shape[0]) if input_ids.ndim else 1
|
||||
return torch.zeros(
|
||||
batch_size,
|
||||
dtype=torch.bool,
|
||||
device=input_ids.device,
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
elapsed = max(time.monotonic() - self.start_time, 1e-6)
|
||||
average = self.steps / elapsed
|
||||
if self.rendered:
|
||||
print(
|
||||
f"\r Audio8 complete: {self.steps} frames in "
|
||||
f"{elapsed:.1f}s ({average:.1f} frames/s)" + " " * 20
|
||||
)
|
||||
|
||||
def abort(self) -> None:
|
||||
if self.rendered:
|
||||
print()
|
||||
|
||||
|
||||
def build_audio8_stopping_criteria(
|
||||
max_steps: int,
|
||||
) -> tuple[StoppingCriteriaList, Audio8TTSStoppingCriteria]:
|
||||
"""Create the native stopping-criteria list and its progress tracker."""
|
||||
tracker = Audio8TTSStoppingCriteria(
|
||||
max_steps=max_steps,
|
||||
progress_bar=_make_comfy_progress_bar(max(1, int(max_steps))),
|
||||
)
|
||||
return StoppingCriteriaList([tracker]), tracker
|
||||
@@ -434,9 +434,19 @@ class Attention(nn.Module):
|
||||
|
||||
# Attention processor
|
||||
|
||||
# TTS Audio Suite patch: FlashAttention is optional and F5 defaults to the
|
||||
# PyTorch backend. Treat partial/incompatible installations as unavailable
|
||||
# instead of crashing module import when submodules or compiled symbols are
|
||||
# missing.
|
||||
FLASH_ATTN_AVAILABLE = False
|
||||
if is_package_available("flash_attn"):
|
||||
from flash_attn.bert_padding import pad_input, unpad_input
|
||||
from flash_attn import flash_attn_varlen_func, flash_attn_func
|
||||
try:
|
||||
from flash_attn.bert_padding import pad_input, unpad_input
|
||||
from flash_attn import flash_attn_varlen_func, flash_attn_func
|
||||
|
||||
FLASH_ATTN_AVAILABLE = True
|
||||
except (ImportError, OSError, AttributeError):
|
||||
pass
|
||||
|
||||
|
||||
class AttnProcessor:
|
||||
@@ -447,7 +457,7 @@ class AttnProcessor:
|
||||
attn_mask_enabled: bool = True,
|
||||
):
|
||||
if attn_backend == "flash_attn":
|
||||
assert is_package_available("flash_attn"), "Please install flash-attn first."
|
||||
assert FLASH_ATTN_AVAILABLE, "Please install a compatible flash-attn build first."
|
||||
|
||||
self.pe_attn_head = pe_attn_head
|
||||
self.attn_backend = attn_backend
|
||||
|
||||
@@ -14,7 +14,6 @@ _HANDLERS: Dict[str, Type[BaseTrainingHandler]] = {}
|
||||
_HANDLER_MODULES = {
|
||||
"rvc": "engines.rvc.training.handler",
|
||||
"moss_tts": "engines.moss_tts.training.handler",
|
||||
"voxcpm": "engines.voxcpm.training.handler",
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
"""Official VoxCPM engine integration."""
|
||||
|
||||
from .voxcpm_downloader import VoxCPMDownloader
|
||||
from .voxcpm_engine import VoxCPMEngine
|
||||
|
||||
__all__ = ["VoxCPMDownloader", "VoxCPMEngine"]
|
||||
@@ -1,5 +0,0 @@
|
||||
"""VoxCPM2 LoRA training integration."""
|
||||
|
||||
from .handler import VoxCPMTrainingHandler
|
||||
|
||||
__all__ = ["VoxCPMTrainingHandler"]
|
||||
@@ -1,90 +0,0 @@
|
||||
"""Official-compatible VoxCPM LoRA checkpoint persistence.
|
||||
|
||||
Adapted from ``OpenBMB/VoxCPM/scripts/train_voxcpm_finetune.py`` (Apache-2.0).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
|
||||
|
||||
def load_checkpoint(model, optimizer, scheduler, save_dir: Path) -> int:
|
||||
latest = save_dir / "latest"
|
||||
if not latest.is_dir():
|
||||
return 0
|
||||
import torch
|
||||
from safetensors.torch import load_file
|
||||
|
||||
weights = latest / "lora_weights.safetensors"
|
||||
if not weights.is_file():
|
||||
raise FileNotFoundError(f"Resumable VoxCPM LoRA weights are missing: {weights}")
|
||||
unwrapped = model.module if hasattr(model, "module") else model
|
||||
unwrapped.load_state_dict(load_file(str(weights)), strict=False)
|
||||
|
||||
optimizer_path = latest / "optimizer.pth"
|
||||
scheduler_path = latest / "scheduler.pth"
|
||||
state_path = latest / "training_state.json"
|
||||
if optimizer_path.is_file():
|
||||
optimizer.load_state_dict(
|
||||
torch.load(optimizer_path, map_location="cpu", weights_only=True)
|
||||
)
|
||||
if scheduler_path.is_file():
|
||||
scheduler.load_state_dict(
|
||||
torch.load(scheduler_path, map_location="cpu", weights_only=True)
|
||||
)
|
||||
if not state_path.is_file():
|
||||
raise FileNotFoundError(f"VoxCPM training state is missing: {state_path}")
|
||||
with open(state_path, "r", encoding="utf-8") as handle:
|
||||
return int(json.load(handle).get("step", 0))
|
||||
|
||||
|
||||
def save_checkpoint(
|
||||
model,
|
||||
optimizer,
|
||||
scheduler,
|
||||
save_dir: Path,
|
||||
step: int,
|
||||
base_model_path: str,
|
||||
metadata: Dict[str, Any],
|
||||
) -> Path:
|
||||
"""Save the official VoxCPM LoRA checkpoint contract plus resume state."""
|
||||
import torch
|
||||
from safetensors.torch import save_file
|
||||
|
||||
# TTS Audio Suite adaptation of OpenBMB's official save_checkpoint:
|
||||
# retain tensor-only LoRA export while adding suite compatibility metadata.
|
||||
unwrapped = model.module if hasattr(model, "module") else model
|
||||
lora_config = unwrapped.lora_config
|
||||
folder = save_dir / f"step_{int(step):07d}"
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
state = {
|
||||
key: value.detach().cpu().contiguous()
|
||||
for key, value in unwrapped.state_dict().items()
|
||||
if "lora_" in key
|
||||
}
|
||||
save_file(state, str(folder / "lora_weights.safetensors"))
|
||||
config_payload = {
|
||||
"base_model": os.path.abspath(base_model_path),
|
||||
"lora_config": (
|
||||
lora_config.model_dump()
|
||||
if hasattr(lora_config, "model_dump")
|
||||
else vars(lora_config)
|
||||
),
|
||||
"tts_audio_suite": metadata,
|
||||
}
|
||||
with open(folder / "lora_config.json", "w", encoding="utf-8") as handle:
|
||||
json.dump(config_payload, handle, indent=2, ensure_ascii=False)
|
||||
torch.save(optimizer.state_dict(), folder / "optimizer.pth")
|
||||
torch.save(scheduler.state_dict(), folder / "scheduler.pth")
|
||||
with open(folder / "training_state.json", "w", encoding="utf-8") as handle:
|
||||
json.dump({"step": int(step)}, handle)
|
||||
|
||||
latest = save_dir / "latest"
|
||||
if latest.exists():
|
||||
shutil.rmtree(latest)
|
||||
shutil.copytree(folder, latest)
|
||||
return folder
|
||||
@@ -1,133 +0,0 @@
|
||||
"""Shared paths and validation for VoxCPM2 LoRA training."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import folder_paths
|
||||
|
||||
from engines.voxcpm.voxcpm_downloader import VoxCPMDownloader
|
||||
from utils.models.extra_paths import get_all_tts_model_paths
|
||||
|
||||
|
||||
def slugify(value: str) -> str:
|
||||
safe = "".join(
|
||||
char if char.isalnum() or char in ("-", "_") else "_"
|
||||
for char in str(value or "").strip()
|
||||
).strip("_")
|
||||
return safe or "voxcpm2_lora"
|
||||
|
||||
|
||||
def get_training_root() -> str:
|
||||
return os.path.join(
|
||||
folder_paths.get_output_directory(),
|
||||
"tts_audio_suite_training",
|
||||
"voxcpm",
|
||||
)
|
||||
|
||||
|
||||
def get_managed_lora_root() -> str:
|
||||
for base_path in get_all_tts_model_paths("TTS"):
|
||||
target = os.path.join(base_path, "voxcpm", "loras")
|
||||
os.makedirs(target, exist_ok=True)
|
||||
return target
|
||||
target = os.path.join(folder_paths.models_dir, "TTS", "voxcpm", "loras")
|
||||
os.makedirs(target, exist_ok=True)
|
||||
return target
|
||||
|
||||
|
||||
def discover_lora_adapters() -> List[str]:
|
||||
root = get_managed_lora_root()
|
||||
results = []
|
||||
for entry in sorted(os.scandir(root), key=lambda item: item.name.lower()):
|
||||
if entry.is_dir() and is_lora_adapter(entry.path):
|
||||
results.append(entry.path)
|
||||
return results
|
||||
|
||||
|
||||
def is_lora_adapter(path: str) -> bool:
|
||||
folder = os.path.abspath(os.path.expanduser(str(path or "").strip()))
|
||||
return (
|
||||
os.path.isdir(folder)
|
||||
and os.path.isfile(os.path.join(folder, "lora_config.json"))
|
||||
and os.path.isfile(os.path.join(folder, "lora_weights.safetensors"))
|
||||
)
|
||||
|
||||
|
||||
def read_lora_info(path: str) -> Dict[str, Any]:
|
||||
folder = os.path.abspath(os.path.expanduser(str(path or "").strip()))
|
||||
if not is_lora_adapter(folder):
|
||||
raise FileNotFoundError(
|
||||
"VoxCPM LoRA folder must contain lora_config.json and "
|
||||
f"lora_weights.safetensors: {folder}"
|
||||
)
|
||||
with open(os.path.join(folder, "lora_config.json"), "r", encoding="utf-8") as handle:
|
||||
data = json.load(handle)
|
||||
if not isinstance(data, dict) or not isinstance(data.get("lora_config"), dict):
|
||||
raise ValueError(f"Invalid VoxCPM LoRA configuration: {folder}")
|
||||
return data
|
||||
|
||||
|
||||
def resolve_voxcpm2_settings(config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
selection = str(
|
||||
config.get("model_variant") or config.get("model_name") or "VoxCPM2"
|
||||
)
|
||||
downloader = VoxCPMDownloader()
|
||||
model_path = downloader.resolve_model_path(selection)
|
||||
spec = downloader.get_model_spec(selection, resolved_path=model_path)
|
||||
if str(spec.get("architecture", "")).lower() != "voxcpm2":
|
||||
raise RuntimeError(
|
||||
"VoxCPM LoRA training currently supports VoxCPM2 only. "
|
||||
f"Selected checkpoint is {spec.get('canonical', selection)}."
|
||||
)
|
||||
return {
|
||||
"model_variant": str(spec.get("canonical") or selection),
|
||||
"model_path": os.path.abspath(model_path),
|
||||
"repo_id": str(spec.get("repo_id") or "openbmb/VoxCPM2"),
|
||||
"architecture": "voxcpm2",
|
||||
"sample_rate": int(spec.get("sample_rate") or 48000),
|
||||
"device": str(config.get("device") or "auto"),
|
||||
}
|
||||
|
||||
|
||||
def next_adapter_dir(name: str, overwrite: bool) -> str:
|
||||
target = os.path.join(get_managed_lora_root(), slugify(name))
|
||||
if overwrite or not os.path.exists(target):
|
||||
return target
|
||||
suffix = 2
|
||||
while os.path.exists(f"{target}_{suffix}"):
|
||||
suffix += 1
|
||||
return f"{target}_{suffix}"
|
||||
|
||||
|
||||
def resolve_continue_from(value: Any) -> str:
|
||||
if value in (None, ""):
|
||||
return ""
|
||||
if isinstance(value, str):
|
||||
path = value
|
||||
elif isinstance(value, dict):
|
||||
if str(value.get("engine_type", "")).lower() != "voxcpm":
|
||||
raise ValueError("continue_from artifacts must come from VoxCPM training")
|
||||
info = value.get("lora_adapter")
|
||||
path = (
|
||||
str(info.get("adapter_path", ""))
|
||||
if isinstance(info, dict)
|
||||
else str(value.get("model_path", ""))
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
"VoxCPM continue_from accepts a LoRA folder or VoxCPM TRAINING_ARTIFACTS"
|
||||
)
|
||||
path = os.path.abspath(os.path.expanduser(path))
|
||||
read_lora_info(path)
|
||||
return path
|
||||
|
||||
|
||||
def write_jsonl(records: List[Dict[str, Any]], path: str) -> None:
|
||||
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8") as handle:
|
||||
for record in records:
|
||||
handle.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
@@ -1,130 +0,0 @@
|
||||
"""Manifest validation and deterministic splitting for VoxCPM2 training."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from .common import get_training_root, slugify, write_jsonl
|
||||
|
||||
|
||||
def _load_manifest(path: str) -> List[Dict[str, Any]]:
|
||||
manifest = os.path.abspath(os.path.expanduser(str(path or "").strip()))
|
||||
if not os.path.isfile(manifest):
|
||||
raise FileNotFoundError(f"VoxCPM training manifest not found: {path}")
|
||||
base_dir = os.path.dirname(manifest)
|
||||
records = []
|
||||
with open(manifest, "r", encoding="utf-8") as handle:
|
||||
for line_number, line in enumerate(handle, 1):
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
raw = json.loads(line)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(
|
||||
f"Invalid JSON on line {line_number} of {manifest}: {exc}"
|
||||
) from exc
|
||||
audio = str(raw.get("audio") or "").strip()
|
||||
text = str(raw.get("text") or "").strip()
|
||||
if not audio or not text:
|
||||
raise ValueError(
|
||||
f"VoxCPM manifest line {line_number} requires non-empty audio and text"
|
||||
)
|
||||
audio_path = (
|
||||
audio if os.path.isabs(audio) else os.path.join(base_dir, audio)
|
||||
)
|
||||
audio_path = os.path.abspath(audio_path)
|
||||
if not os.path.isfile(audio_path):
|
||||
raise FileNotFoundError(
|
||||
f"VoxCPM audio on manifest line {line_number} was not found: {audio_path}"
|
||||
)
|
||||
record = {"audio": audio_path, "text": text}
|
||||
for key in ("duration", "dataset_id", "ref_audio", "ref_duration", "is_prompt"):
|
||||
if key in raw:
|
||||
record[key] = raw[key]
|
||||
if record.get("ref_audio"):
|
||||
ref_path = str(record["ref_audio"])
|
||||
if not os.path.isabs(ref_path):
|
||||
ref_path = os.path.join(base_dir, ref_path)
|
||||
ref_path = os.path.abspath(ref_path)
|
||||
if not os.path.isfile(ref_path):
|
||||
raise FileNotFoundError(
|
||||
f"VoxCPM ref_audio on line {line_number} was not found: {ref_path}"
|
||||
)
|
||||
record["ref_audio"] = ref_path
|
||||
records.append(record)
|
||||
if not records:
|
||||
raise ValueError(f"VoxCPM training manifest is empty: {manifest}")
|
||||
return records
|
||||
|
||||
|
||||
def prepare_voxcpm_dataset(
|
||||
shared_settings: Dict[str, Any],
|
||||
*,
|
||||
dataset_source: str,
|
||||
model_name: str,
|
||||
validation_source: str = "",
|
||||
validation_split: float = 0.05,
|
||||
split_seed: int = 42,
|
||||
reuse_existing: bool = True,
|
||||
**_: Any,
|
||||
) -> Dict[str, Any]:
|
||||
train_records = _load_manifest(dataset_source)
|
||||
val_records = _load_manifest(validation_source) if str(validation_source).strip() else []
|
||||
|
||||
if not val_records and float(validation_split) > 0:
|
||||
if len(train_records) < 2:
|
||||
raise ValueError(
|
||||
"Automatic VoxCPM validation splitting needs at least two clips. "
|
||||
"Set validation_split to 0 for a one-clip smoke test."
|
||||
)
|
||||
shuffled = list(train_records)
|
||||
random.Random(int(split_seed)).shuffle(shuffled)
|
||||
val_count = min(
|
||||
len(shuffled) - 1,
|
||||
max(1, int(round(len(shuffled) * min(float(validation_split), 0.5)))),
|
||||
)
|
||||
val_records, train_records = shuffled[:val_count], shuffled[val_count:]
|
||||
|
||||
fingerprint = hashlib.sha256()
|
||||
for record in train_records + val_records:
|
||||
stat = os.stat(record["audio"])
|
||||
fingerprint.update(
|
||||
f"{record['audio']}|{stat.st_size}|{stat.st_mtime_ns}|{record['text']}".encode()
|
||||
)
|
||||
dataset_dir = os.path.join(
|
||||
get_training_root(),
|
||||
"datasets",
|
||||
f"{slugify(model_name)}_{fingerprint.hexdigest()[:10]}",
|
||||
)
|
||||
train_path = os.path.join(dataset_dir, "train.jsonl")
|
||||
val_path = os.path.join(dataset_dir, "validation.jsonl")
|
||||
if not (
|
||||
reuse_existing
|
||||
and os.path.isfile(train_path)
|
||||
and (not val_records or os.path.isfile(val_path))
|
||||
):
|
||||
write_jsonl(train_records, train_path)
|
||||
if val_records:
|
||||
write_jsonl(val_records, val_path)
|
||||
|
||||
return {
|
||||
"type": "training_dataset",
|
||||
"engine_type": "voxcpm",
|
||||
"training_mode": "lora_adapter",
|
||||
"model_variant": shared_settings["model_variant"],
|
||||
"model_name": str(model_name or "VoxCPM2LoRA"),
|
||||
"dataset_dir": dataset_dir,
|
||||
"train_manifest": train_path,
|
||||
"val_manifest": val_path if val_records else "",
|
||||
"train_records": len(train_records),
|
||||
"val_records": len(val_records),
|
||||
"sample_rate": 16000,
|
||||
"output_sample_rate": shared_settings["sample_rate"],
|
||||
"source_summary": Path(dataset_source).name
|
||||
+ (f" + {Path(validation_source).name}" if validation_source else ""),
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
"""VoxCPM2 backend for the unified model-training node."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict
|
||||
|
||||
from engines.training.base_handler import BaseTrainingHandler
|
||||
from engines.training.registry import register_training_handler
|
||||
|
||||
from .common import resolve_voxcpm2_settings
|
||||
|
||||
|
||||
class VoxCPMTrainingHandler(BaseTrainingHandler):
|
||||
engine_type = "voxcpm"
|
||||
artifact_type = "lora_adapter"
|
||||
|
||||
def _settings(self, tts_engine: Any) -> Dict[str, Any]:
|
||||
return resolve_voxcpm2_settings(self.ensure_engine_type(tts_engine))
|
||||
|
||||
def build_default_training_config(self, tts_engine: Any) -> Dict[str, Any]:
|
||||
self._settings(tts_engine)
|
||||
return {
|
||||
"type": "training_config",
|
||||
"engine_type": "voxcpm",
|
||||
"training_mode": "lora_adapter",
|
||||
"max_train_steps": 1000,
|
||||
"batch_size": 1,
|
||||
"gradient_accumulation_steps": 8,
|
||||
"learning_rate": 1e-4,
|
||||
"weight_decay": 0.01,
|
||||
"warmup_steps": 100,
|
||||
"max_grad_norm": 1.0,
|
||||
"max_batch_tokens": 8192,
|
||||
"num_workers": 0,
|
||||
"cuda_device_index": -1,
|
||||
"save_steps": 500,
|
||||
"eval_steps": 500,
|
||||
"log_steps": 10,
|
||||
"lora_enable_lm": True,
|
||||
"lora_enable_dit": True,
|
||||
"lora_enable_proj": False,
|
||||
"lora_r": 32,
|
||||
"lora_alpha": 32,
|
||||
"lora_dropout": 0.0,
|
||||
}
|
||||
|
||||
def prepare_dataset(self, tts_engine: Any, **kwargs) -> Dict[str, Any]:
|
||||
from .dataset import prepare_voxcpm_dataset
|
||||
|
||||
return prepare_voxcpm_dataset(self._settings(tts_engine), **kwargs)
|
||||
|
||||
def train(
|
||||
self,
|
||||
tts_engine: Any,
|
||||
training_dataset: Dict[str, Any],
|
||||
training_config: Dict[str, Any],
|
||||
output_name: str = "",
|
||||
resume: bool = False,
|
||||
overwrite: bool = False,
|
||||
continue_from: Any = None,
|
||||
node_id: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
from .trainer import run_voxcpm_training_job
|
||||
|
||||
return run_voxcpm_training_job(
|
||||
shared_settings=self._settings(tts_engine),
|
||||
dataset_info=training_dataset,
|
||||
training_config=training_config,
|
||||
output_name=output_name,
|
||||
resume=resume,
|
||||
overwrite=overwrite,
|
||||
continue_from=continue_from,
|
||||
node_id=node_id,
|
||||
)
|
||||
|
||||
|
||||
register_training_handler("voxcpm", VoxCPMTrainingHandler)
|
||||
@@ -1,603 +0,0 @@
|
||||
"""In-process VoxCPM2 LoRA trainer based on OpenBMB's official launcher.
|
||||
|
||||
Adapted from ``OpenBMB/VoxCPM/scripts/train_voxcpm_finetune.py`` (Apache-2.0).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Tuple
|
||||
|
||||
from engines.training.progress_io import write_json_progress_file
|
||||
from engines.training.progress_registry import (
|
||||
finalize_training_job,
|
||||
register_training_job,
|
||||
update_training_job,
|
||||
)
|
||||
|
||||
from .common import (
|
||||
next_adapter_dir,
|
||||
read_lora_info,
|
||||
resolve_continue_from,
|
||||
slugify,
|
||||
)
|
||||
from .checkpoint import (
|
||||
load_checkpoint as _load_checkpoint,
|
||||
save_checkpoint as _save_checkpoint,
|
||||
)
|
||||
|
||||
|
||||
def _interrupted() -> bool:
|
||||
try:
|
||||
import comfy.model_management as model_management
|
||||
|
||||
return bool(model_management.processing_interrupted())
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _write_progress(path: str, *, status: str, phase: str, **updates: Any) -> None:
|
||||
payload: Dict[str, Any] = {}
|
||||
if os.path.isfile(path):
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
payload.update(json.load(handle))
|
||||
except Exception:
|
||||
payload = {}
|
||||
payload.update(updates)
|
||||
payload.update(
|
||||
{
|
||||
"status": status,
|
||||
"phase": phase,
|
||||
"updated_at": datetime.now().isoformat(),
|
||||
}
|
||||
)
|
||||
write_json_progress_file(path, payload, default=str)
|
||||
|
||||
|
||||
def _build_job_dirs(
|
||||
dataset_info: Dict[str, Any],
|
||||
output_name: str,
|
||||
*,
|
||||
resume: bool,
|
||||
overwrite: bool,
|
||||
) -> Tuple[str, str, str]:
|
||||
if resume and overwrite:
|
||||
raise ValueError("VoxCPM resume and overwrite cannot be enabled together")
|
||||
name = slugify(output_name or dataset_info.get("model_name") or "voxcpm2_lora")
|
||||
root = os.path.join(
|
||||
os.path.dirname(os.path.dirname(str(dataset_info["dataset_dir"]))),
|
||||
"jobs",
|
||||
)
|
||||
fingerprint = hashlib.sha256(
|
||||
f"{name}|{os.path.abspath(dataset_info['dataset_dir'])}".encode()
|
||||
).hexdigest()[:10]
|
||||
canonical = os.path.join(root, f"{name}_{fingerprint}")
|
||||
if resume:
|
||||
if not os.path.isdir(os.path.join(canonical, "checkpoints", "latest")):
|
||||
raise FileNotFoundError(
|
||||
f"No resumable VoxCPM checkpoint exists for '{name}': {canonical}"
|
||||
)
|
||||
job_dir = canonical
|
||||
elif overwrite:
|
||||
job_dir = canonical
|
||||
if os.path.isdir(job_dir):
|
||||
shutil.rmtree(job_dir)
|
||||
elif os.path.exists(canonical):
|
||||
job_dir = f"{canonical}_{int(time.time())}"
|
||||
else:
|
||||
job_dir = canonical
|
||||
os.makedirs(job_dir, exist_ok=True)
|
||||
return job_dir, next_adapter_dir(name, overwrite), name
|
||||
|
||||
|
||||
def _validate(model, loader, batch_processor, accelerator, lambdas) -> Dict[str, float]:
|
||||
if loader is None:
|
||||
return {}
|
||||
import torch
|
||||
|
||||
model.eval()
|
||||
values: Dict[str, list] = {}
|
||||
with torch.no_grad():
|
||||
for batch_index, batch in enumerate(loader):
|
||||
if batch_index >= 10:
|
||||
break
|
||||
processed = batch_processor(batch)
|
||||
with accelerator.autocast(dtype=torch.bfloat16):
|
||||
outputs = model(
|
||||
processed["text_tokens"],
|
||||
processed["text_mask"],
|
||||
processed["audio_feats"],
|
||||
processed["audio_mask"],
|
||||
processed["loss_mask"],
|
||||
processed["position_ids"],
|
||||
processed["labels"],
|
||||
progress=0.0,
|
||||
sample_generate=False,
|
||||
)
|
||||
for key, value in outputs.items():
|
||||
if key.startswith("loss/"):
|
||||
values.setdefault(key, []).append(
|
||||
float((value * float(lambdas.get(key, 1.0))).detach().item())
|
||||
)
|
||||
model.train()
|
||||
return {
|
||||
f"val/{key}": sum(items) / len(items)
|
||||
for key, items in values.items()
|
||||
if items
|
||||
}
|
||||
|
||||
|
||||
def run_voxcpm_training_job(
|
||||
shared_settings: Dict[str, Any],
|
||||
dataset_info: Dict[str, Any],
|
||||
training_config: Dict[str, Any],
|
||||
output_name: str = "",
|
||||
resume: bool = False,
|
||||
overwrite: bool = False,
|
||||
continue_from: Any = None,
|
||||
node_id: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
if dataset_info.get("model_variant") != shared_settings["model_variant"]:
|
||||
raise RuntimeError(
|
||||
"VoxCPM dataset/base mismatch. Prepare the dataset again with the selected engine."
|
||||
)
|
||||
if str(training_config.get("training_mode", "")).lower() != "lora_adapter":
|
||||
raise RuntimeError("VoxCPM training currently supports LoRA adapters only")
|
||||
if resume and continue_from not in (None, ""):
|
||||
raise ValueError("Use either exact resume or continue_from, not both")
|
||||
|
||||
import torch
|
||||
|
||||
if not torch.cuda.is_available():
|
||||
raise RuntimeError(
|
||||
"VoxCPM2 LoRA training requires CUDA. CPU/MPS training is not supported "
|
||||
"by the official BF16 training path."
|
||||
)
|
||||
try:
|
||||
from torch.optim import AdamW
|
||||
from transformers import get_cosine_schedule_with_warmup
|
||||
from safetensors.torch import load_file
|
||||
except ImportError as exc:
|
||||
raise RuntimeError(
|
||||
"VoxCPM training requires torch, transformers, datasets, and safetensors"
|
||||
) from exc
|
||||
|
||||
from engines.voxcpm.voxcpm_engine import VoxCPMEngine
|
||||
from utils.models.unified_model_interface import unified_model_interface
|
||||
|
||||
unified_model_interface.clear_engine_models("voxcpm")
|
||||
import gc
|
||||
|
||||
gc.collect()
|
||||
for device_index in range(torch.cuda.device_count()):
|
||||
with torch.cuda.device(device_index):
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
requested_device = int(training_config.get("cuda_device_index", -1))
|
||||
device_count = torch.cuda.device_count()
|
||||
if requested_device >= device_count:
|
||||
raise ValueError(
|
||||
f"VoxCPM cuda_device_index={requested_device} is invalid; "
|
||||
f"this process sees {device_count} CUDA device(s)"
|
||||
)
|
||||
if requested_device < 0:
|
||||
free_memory = []
|
||||
for index in range(device_count):
|
||||
with torch.cuda.device(index):
|
||||
free_bytes, _ = torch.cuda.mem_get_info()
|
||||
free_memory.append(int(free_bytes))
|
||||
requested_device = max(range(device_count), key=free_memory.__getitem__)
|
||||
torch.cuda.set_device(requested_device)
|
||||
if not torch.cuda.is_bf16_supported():
|
||||
raise RuntimeError("VoxCPM2 LoRA training requires a CUDA GPU with BF16 support")
|
||||
|
||||
# Import through the engine's collision-safe official-package loader.
|
||||
VoxCPMEngine(
|
||||
model_name=shared_settings["model_variant"],
|
||||
model_dir=shared_settings["model_path"],
|
||||
)._import_official_package()
|
||||
from voxcpm.model import VoxCPM2Model
|
||||
from voxcpm.model.voxcpm2 import LoRAConfig
|
||||
from voxcpm.training import (
|
||||
Accelerator,
|
||||
BatchProcessor,
|
||||
build_dataloader,
|
||||
load_audio_text_datasets,
|
||||
)
|
||||
|
||||
job_dir, final_adapter_dir, resolved_name = _build_job_dirs(
|
||||
dataset_info,
|
||||
output_name,
|
||||
resume=resume,
|
||||
overwrite=overwrite,
|
||||
)
|
||||
checkpoint_dir = Path(job_dir) / "checkpoints"
|
||||
progress_file = os.path.join(job_dir, "progress.json")
|
||||
continue_path = resolve_continue_from(continue_from)
|
||||
total_steps = int(training_config.get("max_train_steps", 1000))
|
||||
if total_steps < 1:
|
||||
raise ValueError("VoxCPM max_train_steps must be at least 1")
|
||||
|
||||
lora_values = {
|
||||
"enable_lm": bool(training_config.get("lora_enable_lm", True)),
|
||||
"enable_dit": bool(training_config.get("lora_enable_dit", True)),
|
||||
"enable_proj": bool(training_config.get("lora_enable_proj", False)),
|
||||
"r": int(training_config.get("lora_r", 32)),
|
||||
"alpha": int(training_config.get("lora_alpha", 32)),
|
||||
"dropout": float(training_config.get("lora_dropout", 0.0)),
|
||||
}
|
||||
resolved_config = {
|
||||
"dataset": dataset_info,
|
||||
"training_config": training_config,
|
||||
"shared_settings": shared_settings,
|
||||
"lora_config": lora_values,
|
||||
"continue_from": continue_path,
|
||||
}
|
||||
resolved_config_path = os.path.join(job_dir, "resolved_training_config.json")
|
||||
if resume and os.path.isfile(resolved_config_path):
|
||||
with open(resolved_config_path, "r", encoding="utf-8") as handle:
|
||||
previous_run = json.load(handle)
|
||||
previous_lora = previous_run.get("lora_config") or {}
|
||||
if any(previous_lora.get(key) != value for key, value in lora_values.items()):
|
||||
raise ValueError(
|
||||
"Exact VoxCPM resume requires the same LoRA module groups, rank, "
|
||||
"alpha, and dropout as the saved job"
|
||||
)
|
||||
previous_model_value = str(
|
||||
(previous_run.get("shared_settings") or {}).get("model_path") or ""
|
||||
).strip()
|
||||
previous_model = (
|
||||
os.path.abspath(previous_model_value) if previous_model_value else ""
|
||||
)
|
||||
if previous_model and previous_model != os.path.abspath(shared_settings["model_path"]):
|
||||
raise ValueError("Exact VoxCPM resume requires the same base model path")
|
||||
with open(resolved_config_path, "w", encoding="utf-8") as handle:
|
||||
json.dump(resolved_config, handle, indent=2, default=str)
|
||||
|
||||
register_training_job(
|
||||
node_id,
|
||||
engine_type="voxcpm",
|
||||
progress_file=progress_file,
|
||||
job_dir=job_dir,
|
||||
model_name=resolved_name,
|
||||
sample_rate="16k input / 48k output",
|
||||
total_epochs=0,
|
||||
)
|
||||
_write_progress(
|
||||
progress_file,
|
||||
status="starting",
|
||||
phase="loading",
|
||||
node_id=str(node_id or ""),
|
||||
step=0,
|
||||
total_steps=total_steps,
|
||||
summary=f"VoxCPM2 LoRA r={lora_values['r']}, alpha={lora_values['alpha']}",
|
||||
)
|
||||
|
||||
try:
|
||||
accelerator = Accelerator(amp=True)
|
||||
# The official single-process accelerator defaults to LOCAL_RANK=0.
|
||||
# TTS Audio Suite patch: respect the training node's selected CUDA GPU.
|
||||
accelerator.local_rank = requested_device
|
||||
accelerator.device_ctx = torch.cuda.device(requested_device)
|
||||
base_model = VoxCPM2Model.from_local(
|
||||
shared_settings["model_path"],
|
||||
optimize=False,
|
||||
training=True,
|
||||
lora_config=LoRAConfig(**lora_values),
|
||||
)
|
||||
if int(base_model.audio_vae.sample_rate) != 16000:
|
||||
raise RuntimeError(
|
||||
f"Unexpected VoxCPM2 AudioVAE input rate: {base_model.audio_vae.sample_rate}"
|
||||
)
|
||||
tokenizer = base_model.text_tokenizer
|
||||
train_ds, val_ds = load_audio_text_datasets(
|
||||
train_manifest=dataset_info["train_manifest"],
|
||||
val_manifest=dataset_info.get("val_manifest", ""),
|
||||
sample_rate=16000,
|
||||
)
|
||||
|
||||
def tokenize(batch):
|
||||
return {"text_ids": [tokenizer(text) for text in batch["text"]]}
|
||||
|
||||
train_ds = train_ds.map(tokenize, batched=True, remove_columns=["text"])
|
||||
if val_ds is not None:
|
||||
val_ds = val_ds.map(tokenize, batched=True, remove_columns=["text"])
|
||||
|
||||
max_batch_tokens = int(training_config.get("max_batch_tokens", 8192))
|
||||
batch_size = int(training_config.get("batch_size", 1))
|
||||
if max_batch_tokens > 0:
|
||||
from voxcpm.training.data import compute_sample_lengths
|
||||
|
||||
lengths = compute_sample_lengths(
|
||||
train_ds,
|
||||
audio_vae_fps=base_model.audio_vae.sample_rate
|
||||
/ base_model.audio_vae.hop_length,
|
||||
patch_size=base_model.config.patch_size,
|
||||
)
|
||||
max_sample = max_batch_tokens // max(batch_size, 1)
|
||||
train_ds = train_ds.select(
|
||||
[index for index, length in enumerate(lengths) if length <= max_sample]
|
||||
)
|
||||
if len(train_ds) < batch_size:
|
||||
raise RuntimeError(
|
||||
"No complete VoxCPM training batch remains. Reduce batch_size, "
|
||||
"increase max_batch_tokens, or shorten the clips."
|
||||
)
|
||||
|
||||
workers = int(training_config.get("num_workers", 0))
|
||||
train_loader = build_dataloader(
|
||||
train_ds,
|
||||
accelerator=accelerator,
|
||||
batch_size=batch_size,
|
||||
num_workers=workers,
|
||||
drop_last=True,
|
||||
)
|
||||
val_loader = (
|
||||
build_dataloader(
|
||||
val_ds,
|
||||
accelerator=accelerator,
|
||||
batch_size=batch_size,
|
||||
num_workers=workers,
|
||||
drop_last=False,
|
||||
)
|
||||
if val_ds is not None
|
||||
else None
|
||||
)
|
||||
dataset_count = (
|
||||
int(max(train_ds["dataset_id"])) + 1
|
||||
if "dataset_id" in train_ds.column_names
|
||||
else 1
|
||||
)
|
||||
batch_processor = BatchProcessor(
|
||||
config=base_model.config,
|
||||
audio_vae=base_model.audio_vae,
|
||||
dataset_cnt=dataset_count,
|
||||
device=accelerator.device,
|
||||
)
|
||||
del base_model.audio_vae
|
||||
|
||||
model = accelerator.prepare_model(base_model)
|
||||
unwrapped = accelerator.unwrap(model)
|
||||
if continue_path:
|
||||
previous = read_lora_info(continue_path)
|
||||
previous_config = previous["lora_config"]
|
||||
if any(
|
||||
previous_config.get(key) != value
|
||||
for key, value in lora_values.items()
|
||||
):
|
||||
raise ValueError(
|
||||
"continue_from LoRA settings differ from the new training configuration"
|
||||
)
|
||||
unwrapped.load_state_dict(
|
||||
load_file(os.path.join(continue_path, "lora_weights.safetensors")),
|
||||
strict=False,
|
||||
)
|
||||
model.train()
|
||||
optimizer = AdamW(
|
||||
(parameter for parameter in model.parameters() if parameter.requires_grad),
|
||||
lr=float(training_config.get("learning_rate", 1e-4)),
|
||||
weight_decay=float(training_config.get("weight_decay", 0.01)),
|
||||
)
|
||||
scheduler = get_cosine_schedule_with_warmup(
|
||||
optimizer,
|
||||
num_warmup_steps=int(training_config.get("warmup_steps", 100)),
|
||||
num_training_steps=total_steps,
|
||||
)
|
||||
start_step = _load_checkpoint(model, optimizer, scheduler, checkpoint_dir) if resume else 0
|
||||
if start_step >= total_steps:
|
||||
raise ValueError(
|
||||
f"VoxCPM resume checkpoint is already at step {start_step}; "
|
||||
f"set max_train_steps above {start_step}"
|
||||
)
|
||||
grad_accum = max(
|
||||
1, int(training_config.get("gradient_accumulation_steps", 8))
|
||||
)
|
||||
log_steps = max(1, int(training_config.get("log_steps", 10)))
|
||||
save_steps = int(training_config.get("save_steps", 500))
|
||||
eval_steps = int(training_config.get("eval_steps", 500))
|
||||
max_grad_norm = float(training_config.get("max_grad_norm", 1.0))
|
||||
lambdas = {"loss/diff": 1.0, "loss/stop": 1.0}
|
||||
train_iterator = iter(train_loader)
|
||||
recent_loss = []
|
||||
|
||||
def next_batch():
|
||||
nonlocal train_iterator
|
||||
try:
|
||||
return next(train_iterator)
|
||||
except StopIteration:
|
||||
train_iterator = iter(train_loader)
|
||||
return next(train_iterator)
|
||||
|
||||
for step in range(start_step, total_steps):
|
||||
if _interrupted():
|
||||
_save_checkpoint(
|
||||
model,
|
||||
optimizer,
|
||||
scheduler,
|
||||
checkpoint_dir,
|
||||
step,
|
||||
shared_settings["model_path"],
|
||||
shared_settings,
|
||||
)
|
||||
raise InterruptedError("VoxCPM training interrupted by user")
|
||||
optimizer.zero_grad(set_to_none=True)
|
||||
losses = {}
|
||||
for micro_step in range(grad_accum):
|
||||
processed = batch_processor(next_batch())
|
||||
sync = (
|
||||
contextlib.nullcontext()
|
||||
if micro_step == grad_accum - 1
|
||||
else accelerator.no_sync()
|
||||
)
|
||||
with sync, accelerator.autocast(dtype=torch.bfloat16):
|
||||
outputs = model(
|
||||
processed["text_tokens"],
|
||||
processed["text_mask"],
|
||||
processed["audio_feats"],
|
||||
processed["audio_mask"],
|
||||
processed["loss_mask"],
|
||||
processed["position_ids"],
|
||||
processed["labels"],
|
||||
progress=step / max(total_steps, 1),
|
||||
)
|
||||
total_loss = sum(
|
||||
value * lambdas.get(key, 1.0) / grad_accum
|
||||
for key, value in outputs.items()
|
||||
if key.startswith("loss/")
|
||||
)
|
||||
accelerator.backward(total_loss)
|
||||
losses = {
|
||||
key: float(value.detach().item())
|
||||
for key, value in outputs.items()
|
||||
if key.startswith("loss/")
|
||||
}
|
||||
|
||||
accelerator.scaler.unscale_(optimizer)
|
||||
grad_norm = torch.nn.utils.clip_grad_norm_(
|
||||
unwrapped.parameters(), max_grad_norm if max_grad_norm > 0 else 1e9
|
||||
)
|
||||
accelerator.step(optimizer)
|
||||
accelerator.update()
|
||||
scheduler.step()
|
||||
completed = step + 1
|
||||
|
||||
if completed % log_steps == 0 or completed == total_steps:
|
||||
metrics = {
|
||||
**losses,
|
||||
"lr": float(optimizer.param_groups[0]["lr"]),
|
||||
"grad_norm": float(grad_norm),
|
||||
}
|
||||
recent_loss = (recent_loss + [{"step": completed, **metrics}])[-100:]
|
||||
progress = completed / total_steps
|
||||
update_training_job(
|
||||
node_id,
|
||||
status="running",
|
||||
phase="training",
|
||||
current_step=completed,
|
||||
total_steps=total_steps,
|
||||
)
|
||||
_write_progress(
|
||||
progress_file,
|
||||
status="running",
|
||||
phase="training",
|
||||
step=completed,
|
||||
current_step=completed,
|
||||
total_steps=total_steps,
|
||||
completed_total_steps=completed,
|
||||
overall_progress=progress,
|
||||
current_metrics=metrics,
|
||||
recent_loss_trace=recent_loss,
|
||||
)
|
||||
|
||||
if val_loader is not None and eval_steps > 0 and (
|
||||
completed % eval_steps == 0 or completed == total_steps
|
||||
):
|
||||
validation = _validate(
|
||||
model, val_loader, batch_processor, accelerator, lambdas
|
||||
)
|
||||
_write_progress(
|
||||
progress_file,
|
||||
status="running",
|
||||
phase="validation",
|
||||
step=completed,
|
||||
total_steps=total_steps,
|
||||
current_metrics={**losses, **validation},
|
||||
overall_progress=completed / total_steps,
|
||||
)
|
||||
|
||||
if save_steps > 0 and (
|
||||
completed % save_steps == 0 or completed == total_steps
|
||||
):
|
||||
_save_checkpoint(
|
||||
model,
|
||||
optimizer,
|
||||
scheduler,
|
||||
checkpoint_dir,
|
||||
completed,
|
||||
shared_settings["model_path"],
|
||||
shared_settings,
|
||||
)
|
||||
|
||||
latest = checkpoint_dir / "latest"
|
||||
if not latest.is_dir():
|
||||
_save_checkpoint(
|
||||
model,
|
||||
optimizer,
|
||||
scheduler,
|
||||
checkpoint_dir,
|
||||
total_steps,
|
||||
shared_settings["model_path"],
|
||||
shared_settings,
|
||||
)
|
||||
if overwrite and os.path.isdir(final_adapter_dir):
|
||||
shutil.rmtree(final_adapter_dir)
|
||||
os.makedirs(final_adapter_dir, exist_ok=True)
|
||||
for filename in ("lora_weights.safetensors", "lora_config.json"):
|
||||
shutil.copy2(latest / filename, os.path.join(final_adapter_dir, filename))
|
||||
|
||||
summary = (
|
||||
f"VoxCPM2 LoRA complete: {resolved_name} | "
|
||||
f"r={lora_values['r']}, alpha={lora_values['alpha']} | "
|
||||
f"steps={total_steps} | cuda:{requested_device}"
|
||||
)
|
||||
artifacts = {
|
||||
"type": "training_artifacts",
|
||||
"engine_type": "voxcpm",
|
||||
"training_mode": "lora_adapter",
|
||||
"model_variant": shared_settings["model_variant"],
|
||||
"model_path": final_adapter_dir,
|
||||
"job_dir": job_dir,
|
||||
"summary": summary,
|
||||
"lora_adapter": {
|
||||
"type": "voxcpm_lora",
|
||||
"adapter_path": final_adapter_dir,
|
||||
"base_model_name_or_path": shared_settings["model_path"],
|
||||
},
|
||||
}
|
||||
finalize_training_job(
|
||||
node_id,
|
||||
status="completed",
|
||||
current_step=total_steps,
|
||||
total_steps=total_steps,
|
||||
artifacts=artifacts,
|
||||
)
|
||||
_write_progress(
|
||||
progress_file,
|
||||
status="completed",
|
||||
phase="done",
|
||||
step=total_steps,
|
||||
total_steps=total_steps,
|
||||
completed_total_steps=total_steps,
|
||||
overall_progress=1.0,
|
||||
output_adapter=final_adapter_dir,
|
||||
recent_loss_trace=recent_loss,
|
||||
summary=summary,
|
||||
)
|
||||
return artifacts
|
||||
except InterruptedError as exc:
|
||||
finalize_training_job(node_id, status="cancelled", error=str(exc))
|
||||
_write_progress(
|
||||
progress_file,
|
||||
status="cancelled",
|
||||
phase="cancelled",
|
||||
error=str(exc),
|
||||
summary="VoxCPM training cancelled; latest checkpoint was preserved",
|
||||
)
|
||||
raise
|
||||
except Exception as exc:
|
||||
finalize_training_job(node_id, status="error", error=str(exc))
|
||||
_write_progress(
|
||||
progress_file,
|
||||
status="error",
|
||||
phase="error",
|
||||
error=str(exc),
|
||||
)
|
||||
raise
|
||||
@@ -1,480 +0,0 @@
|
||||
"""Download and inspect official VoxCPM checkpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from typing import Any, Dict, List, Mapping, Optional
|
||||
|
||||
import folder_paths
|
||||
|
||||
from utils.downloads.unified_downloader import unified_downloader
|
||||
from utils.hf_download_logging import quiet_hf_download_logs
|
||||
from utils.models.extra_paths import (
|
||||
get_all_tts_model_paths,
|
||||
get_preferred_download_path,
|
||||
)
|
||||
|
||||
|
||||
_MODEL_SPECS: Dict[str, Dict[str, Any]] = {
|
||||
"VoxCPM2": {
|
||||
"repo_id": "openbmb/VoxCPM2",
|
||||
"folder_name": "VoxCPM2",
|
||||
"architecture": "voxcpm2",
|
||||
"sample_rate": 48000,
|
||||
"legacy": False,
|
||||
"required_files": (
|
||||
"audiovae.pth",
|
||||
"config.json",
|
||||
"model.safetensors",
|
||||
"special_tokens_map.json",
|
||||
"tokenization_voxcpm2.py",
|
||||
"tokenizer.json",
|
||||
"tokenizer_config.json",
|
||||
),
|
||||
},
|
||||
"VoxCPM1.5": {
|
||||
"repo_id": "openbmb/VoxCPM1.5",
|
||||
"folder_name": "VoxCPM1.5",
|
||||
"architecture": "voxcpm",
|
||||
"sample_rate": 44100,
|
||||
"legacy": False,
|
||||
"required_files": (
|
||||
"audiovae.pth",
|
||||
"config.json",
|
||||
"model.safetensors",
|
||||
"special_tokens_map.json",
|
||||
"tokenizer.json",
|
||||
"tokenizer_config.json",
|
||||
),
|
||||
},
|
||||
"VoxCPM-0.5B": {
|
||||
"repo_id": "openbmb/VoxCPM-0.5B",
|
||||
"folder_name": "VoxCPM-0.5B",
|
||||
"architecture": "voxcpm",
|
||||
"sample_rate": 16000,
|
||||
"legacy": True,
|
||||
"required_files": (
|
||||
"audiovae.pth",
|
||||
"config.json",
|
||||
"pytorch_model.bin",
|
||||
"special_tokens_map.json",
|
||||
"tokenizer.json",
|
||||
"tokenizer_config.json",
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class VoxCPMDownloader:
|
||||
"""Resolve official and local VoxCPM model folders."""
|
||||
|
||||
DEFAULT_MODEL = "VoxCPM2"
|
||||
MODELS = _MODEL_SPECS
|
||||
REQUIRED_FILES = {
|
||||
name: tuple(spec["required_files"])
|
||||
for name, spec in _MODEL_SPECS.items()
|
||||
}
|
||||
|
||||
_COMMON_LOCAL_FILES = (
|
||||
"config.json",
|
||||
"special_tokens_map.json",
|
||||
"tokenizer.json",
|
||||
"tokenizer_config.json",
|
||||
)
|
||||
_AUDIO_VAE_FILES = ("audiovae.safetensors", "audiovae.pth")
|
||||
_MODEL_WEIGHT_FILES = ("model.safetensors", "pytorch_model.bin")
|
||||
|
||||
def __init__(self, base_path: Optional[str] = None):
|
||||
if base_path is None:
|
||||
try:
|
||||
base_path = get_preferred_download_path(
|
||||
model_type="TTS",
|
||||
engine_name="voxcpm",
|
||||
)
|
||||
except Exception:
|
||||
base_path = os.path.join(
|
||||
folder_paths.models_dir,
|
||||
"TTS",
|
||||
"voxcpm",
|
||||
)
|
||||
self.base_path = os.path.abspath(base_path)
|
||||
os.makedirs(self.base_path, exist_ok=True)
|
||||
|
||||
@classmethod
|
||||
def _canonical_name(cls, selection: Optional[str]) -> Optional[str]:
|
||||
value = str(selection or cls.DEFAULT_MODEL).strip()
|
||||
for canonical, spec in cls.MODELS.items():
|
||||
if value == canonical or value == spec["repo_id"]:
|
||||
return canonical
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _file_is_present(path: str) -> bool:
|
||||
try:
|
||||
return os.path.isfile(path) and os.path.getsize(path) > 0
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def _read_local_metadata(cls, model_dir: str) -> Dict[str, Any]:
|
||||
model_dir = os.path.abspath(model_dir)
|
||||
config_path = os.path.join(model_dir, "config.json")
|
||||
if not cls._file_is_present(config_path):
|
||||
raise ValueError(f"VoxCPM config is missing or empty: {config_path}")
|
||||
|
||||
try:
|
||||
with open(config_path, "r", encoding="utf-8") as config_file:
|
||||
config = json.load(config_file)
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise ValueError(f"Invalid VoxCPM config: {config_path}") from exc
|
||||
|
||||
architecture = str(config.get("architecture", "")).lower()
|
||||
if architecture not in {"voxcpm", "voxcpm2"}:
|
||||
raise ValueError(
|
||||
f"Unsupported or missing VoxCPM architecture in {config_path}: "
|
||||
f"{architecture or '(missing)'}"
|
||||
)
|
||||
|
||||
audio_vae_config = config.get("audio_vae_config")
|
||||
if not isinstance(audio_vae_config, dict):
|
||||
audio_vae_config = {}
|
||||
|
||||
if architecture == "voxcpm2":
|
||||
# AudioVAEV2 uses 48 kHz when the optional embedded VAE config is
|
||||
# omitted, so this is deterministic rather than a heuristic.
|
||||
sample_rate = audio_vae_config.get("out_sample_rate", 48000)
|
||||
else:
|
||||
# The original 0.5B runtime uses AudioVAE's 16 kHz default when
|
||||
# audio_vae_config is absent.
|
||||
sample_rate = audio_vae_config.get("sample_rate", 16000)
|
||||
|
||||
try:
|
||||
sample_rate = int(sample_rate)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(
|
||||
f"Invalid VoxCPM output sample rate in {config_path}: {sample_rate}"
|
||||
) from exc
|
||||
if sample_rate <= 0:
|
||||
raise ValueError(
|
||||
f"Invalid VoxCPM output sample rate in {config_path}: {sample_rate}"
|
||||
)
|
||||
|
||||
if architecture == "voxcpm2" and sample_rate == 48000:
|
||||
canonical = "VoxCPM2"
|
||||
elif architecture == "voxcpm" and sample_rate == 44100:
|
||||
canonical = "VoxCPM1.5"
|
||||
elif architecture == "voxcpm" and sample_rate == 16000:
|
||||
canonical = "VoxCPM-0.5B"
|
||||
else:
|
||||
canonical = f"local:{os.path.basename(model_dir)}"
|
||||
|
||||
return {
|
||||
"canonical": canonical,
|
||||
"repo_id": None,
|
||||
"folder_name": os.path.basename(model_dir),
|
||||
"architecture": architecture,
|
||||
"sample_rate": sample_rate,
|
||||
"legacy": architecture == "voxcpm" and sample_rate == 16000,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def _missing_local_files(cls, model_dir: str) -> List[str]:
|
||||
missing = [
|
||||
rel_path
|
||||
for rel_path in cls._COMMON_LOCAL_FILES
|
||||
if not cls._file_is_present(os.path.join(model_dir, rel_path))
|
||||
]
|
||||
if not any(
|
||||
cls._file_is_present(os.path.join(model_dir, rel_path))
|
||||
for rel_path in cls._AUDIO_VAE_FILES
|
||||
):
|
||||
missing.append("audiovae.safetensors | audiovae.pth")
|
||||
if not any(
|
||||
cls._file_is_present(os.path.join(model_dir, rel_path))
|
||||
for rel_path in cls._MODEL_WEIGHT_FILES
|
||||
):
|
||||
missing.append("model.safetensors | pytorch_model.bin")
|
||||
return missing
|
||||
|
||||
def get_model_spec(
|
||||
self,
|
||||
selection: Optional[str],
|
||||
resolved_path: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Return architecture and output metadata for a model selection."""
|
||||
selection = str(selection or self.DEFAULT_MODEL).strip()
|
||||
canonical = self._canonical_name(selection)
|
||||
if canonical is not None:
|
||||
spec = dict(self.MODELS[canonical])
|
||||
spec["canonical"] = canonical
|
||||
spec["required_files"] = list(spec["required_files"])
|
||||
if resolved_path is not None:
|
||||
local_spec = self._read_local_metadata(resolved_path)
|
||||
if local_spec["architecture"] != spec["architecture"]:
|
||||
raise ValueError(
|
||||
f"{canonical} expects architecture {spec['architecture']}, "
|
||||
f"but {resolved_path} declares "
|
||||
f"{local_spec['architecture']}"
|
||||
)
|
||||
if local_spec["sample_rate"] != spec["sample_rate"]:
|
||||
raise ValueError(
|
||||
f"{canonical} expects {spec['sample_rate']} Hz output, "
|
||||
f"but {resolved_path} declares "
|
||||
f"{local_spec['sample_rate']} Hz"
|
||||
)
|
||||
return spec
|
||||
|
||||
model_dir = resolved_path
|
||||
if model_dir is None and selection.startswith("local:"):
|
||||
model_dir = self._find_local_path(selection[6:])
|
||||
if model_dir is None and os.path.isdir(selection):
|
||||
model_dir = selection
|
||||
if model_dir is None:
|
||||
raise ValueError(f"Unknown VoxCPM model selection: {selection}")
|
||||
|
||||
model_dir = os.path.abspath(model_dir)
|
||||
spec = self._read_local_metadata(model_dir)
|
||||
required_files = list(self._COMMON_LOCAL_FILES)
|
||||
required_files.extend(
|
||||
rel_path
|
||||
for rel_path in self._AUDIO_VAE_FILES
|
||||
if self._file_is_present(os.path.join(model_dir, rel_path))
|
||||
)
|
||||
required_files.extend(
|
||||
rel_path
|
||||
for rel_path in self._MODEL_WEIGHT_FILES
|
||||
if self._file_is_present(os.path.join(model_dir, rel_path))
|
||||
)
|
||||
spec["required_files"] = required_files
|
||||
return spec
|
||||
|
||||
def is_model_complete(
|
||||
self,
|
||||
model_dir: str,
|
||||
spec: Optional[Mapping[str, Any]] = None,
|
||||
) -> bool:
|
||||
"""Return whether a folder has every file needed by the official runtime."""
|
||||
if not os.path.isdir(model_dir):
|
||||
return False
|
||||
|
||||
try:
|
||||
local_spec = self._read_local_metadata(model_dir)
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
missing = self._missing_local_files(model_dir)
|
||||
if spec is not None:
|
||||
required_files = list(spec.get("required_files", ()))
|
||||
missing.extend(
|
||||
rel_path
|
||||
for rel_path in required_files
|
||||
if (
|
||||
rel_path not in missing
|
||||
and not self._file_is_present(
|
||||
os.path.join(model_dir, rel_path)
|
||||
)
|
||||
)
|
||||
)
|
||||
expected_architecture = spec.get("architecture")
|
||||
expected_sample_rate = spec.get("sample_rate")
|
||||
if (
|
||||
expected_architecture
|
||||
and local_spec["architecture"] != expected_architecture
|
||||
):
|
||||
missing.append(
|
||||
"config architecture "
|
||||
f"{expected_architecture} (found {local_spec['architecture']})"
|
||||
)
|
||||
if (
|
||||
expected_sample_rate
|
||||
and local_spec["sample_rate"] != int(expected_sample_rate)
|
||||
):
|
||||
missing.append(
|
||||
"config output sample rate "
|
||||
f"{expected_sample_rate} (found {local_spec['sample_rate']})"
|
||||
)
|
||||
if missing:
|
||||
print(
|
||||
f"❌ VoxCPM model incomplete or incompatible: {model_dir}"
|
||||
)
|
||||
for requirement in missing:
|
||||
print(f" - {requirement}")
|
||||
return False
|
||||
return True
|
||||
|
||||
def _search_roots(self) -> List[str]:
|
||||
roots = [self.base_path]
|
||||
try:
|
||||
configured_paths = get_all_tts_model_paths("TTS")
|
||||
except Exception:
|
||||
configured_paths = []
|
||||
|
||||
for base_path in configured_paths:
|
||||
normalized = os.path.abspath(base_path)
|
||||
if os.path.basename(normalized).lower() == "voxcpm":
|
||||
roots.append(normalized)
|
||||
else:
|
||||
roots.extend(
|
||||
(
|
||||
os.path.join(normalized, "voxcpm"),
|
||||
os.path.join(normalized, "VoxCPM"),
|
||||
)
|
||||
)
|
||||
|
||||
result: List[str] = []
|
||||
seen = set()
|
||||
for root in roots:
|
||||
key = os.path.normcase(os.path.normpath(root))
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
result.append(root)
|
||||
return result
|
||||
|
||||
def _find_local_path(self, local_name: str) -> str:
|
||||
local_name = str(local_name or "").strip()
|
||||
if not local_name:
|
||||
raise FileNotFoundError("A local VoxCPM model name is required")
|
||||
|
||||
if os.path.isabs(local_name) and os.path.isdir(local_name):
|
||||
if self.is_model_complete(local_name):
|
||||
return os.path.abspath(local_name)
|
||||
raise FileNotFoundError(
|
||||
f"Local VoxCPM model is incomplete: {local_name}"
|
||||
)
|
||||
|
||||
incomplete: List[str] = []
|
||||
for root in self._search_roots():
|
||||
candidate = os.path.join(root, local_name)
|
||||
if not os.path.isdir(candidate):
|
||||
continue
|
||||
if self.is_model_complete(candidate):
|
||||
return os.path.abspath(candidate)
|
||||
incomplete.append(candidate)
|
||||
|
||||
suffix = (
|
||||
f" Incomplete candidates: {incomplete}"
|
||||
if incomplete
|
||||
else ""
|
||||
)
|
||||
raise FileNotFoundError(
|
||||
f"Local VoxCPM model not found or incomplete: {local_name}.{suffix}"
|
||||
)
|
||||
|
||||
def get_available_models(self) -> List[str]:
|
||||
"""Return official choices plus complete local checkpoints."""
|
||||
available = list(self.MODELS)
|
||||
official_folders = {
|
||||
str(spec["folder_name"]).casefold()
|
||||
for spec in self.MODELS.values()
|
||||
}
|
||||
local_choices: List[str] = []
|
||||
for root in self._search_roots():
|
||||
if not os.path.isdir(root):
|
||||
continue
|
||||
try:
|
||||
entries = sorted(os.listdir(root))
|
||||
except OSError:
|
||||
continue
|
||||
for entry in entries:
|
||||
if entry.casefold() in official_folders:
|
||||
continue
|
||||
candidate = os.path.join(root, entry)
|
||||
choice = f"local:{entry}"
|
||||
if (
|
||||
choice not in local_choices
|
||||
and self.is_model_complete(candidate)
|
||||
):
|
||||
local_choices.append(choice)
|
||||
return local_choices + available
|
||||
|
||||
def get_model_path(self, model_name: str) -> str:
|
||||
"""Find or download an official model."""
|
||||
canonical = self._canonical_name(model_name)
|
||||
if canonical is None:
|
||||
raise ValueError(f"Unknown VoxCPM model: {model_name}")
|
||||
spec = self.get_model_spec(canonical)
|
||||
|
||||
for root in self._search_roots():
|
||||
candidate = os.path.join(root, spec["folder_name"])
|
||||
if self.is_model_complete(candidate, spec):
|
||||
return os.path.abspath(candidate)
|
||||
|
||||
return self.download_model(canonical)
|
||||
|
||||
def resolve_model_path(self, selection: Optional[str] = None) -> str:
|
||||
"""Resolve a canonical choice, exact repo ID, path, or ``local:`` name."""
|
||||
selection = str(selection or self.DEFAULT_MODEL).strip()
|
||||
canonical = self._canonical_name(selection)
|
||||
if canonical is not None:
|
||||
return self.get_model_path(canonical)
|
||||
|
||||
if selection.startswith("local:"):
|
||||
return self._find_local_path(selection[6:])
|
||||
|
||||
if os.path.isdir(selection):
|
||||
model_dir = os.path.abspath(selection)
|
||||
if self.is_model_complete(model_dir):
|
||||
return model_dir
|
||||
raise FileNotFoundError(
|
||||
f"VoxCPM model folder is incomplete: {model_dir}"
|
||||
)
|
||||
|
||||
raise ValueError(f"Unknown VoxCPM model selection: {selection}")
|
||||
|
||||
def download_model(self, model_name: str, force: bool = False) -> str:
|
||||
"""Download one official checkpoint into the organized TTS folder."""
|
||||
canonical = self._canonical_name(model_name)
|
||||
if canonical is None:
|
||||
raise ValueError(f"Unknown VoxCPM model: {model_name}")
|
||||
spec = self.get_model_spec(canonical)
|
||||
model_dir = os.path.join(self.base_path, spec["folder_name"])
|
||||
|
||||
if not force and self.is_model_complete(model_dir, spec):
|
||||
return os.path.abspath(model_dir)
|
||||
|
||||
files = []
|
||||
for rel_path in spec["required_files"]:
|
||||
files.append(
|
||||
{
|
||||
"remote": rel_path,
|
||||
"local": rel_path,
|
||||
"force_download": bool(
|
||||
force
|
||||
or not self._file_is_present(
|
||||
os.path.join(model_dir, rel_path)
|
||||
)
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("📦 VoxCPM Model Download")
|
||||
print("=" * 60)
|
||||
print(f"Model: {canonical}")
|
||||
print(f"Repository: {spec['repo_id']}")
|
||||
print(f"Target: {model_dir}")
|
||||
print("=" * 60 + "\n")
|
||||
|
||||
try:
|
||||
with quiet_hf_download_logs():
|
||||
result = unified_downloader.download_huggingface_model(
|
||||
repo_id=spec["repo_id"],
|
||||
model_name=spec["folder_name"],
|
||||
files=files,
|
||||
engine_type="voxcpm",
|
||||
force_download=force,
|
||||
target_dir=model_dir,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
f"Failed to download {canonical} from {spec['repo_id']}: {exc}"
|
||||
) from exc
|
||||
|
||||
if not result or not self.is_model_complete(model_dir, spec):
|
||||
raise RuntimeError(
|
||||
f"Downloaded VoxCPM model is incomplete: {model_dir}"
|
||||
)
|
||||
|
||||
print(f"✅ VoxCPM model ready: {model_dir}")
|
||||
return os.path.abspath(model_dir)
|
||||
@@ -1,390 +0,0 @@
|
||||
"""ComfyUI lifecycle wrapper for the official VoxCPM package."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import gc
|
||||
import importlib
|
||||
import importlib.metadata
|
||||
import inspect
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import sys
|
||||
from typing import Any, Iterator, Optional
|
||||
|
||||
from engines.voxcpm.voxcpm_downloader import VoxCPMDownloader
|
||||
|
||||
|
||||
_CURRENT_DIR = os.path.dirname(__file__)
|
||||
_ENGINES_DIR = os.path.dirname(_CURRENT_DIR)
|
||||
_PROJECT_ROOT = os.path.dirname(_ENGINES_DIR)
|
||||
|
||||
|
||||
class VoxCPMEngine:
|
||||
"""Thin, local-files-only wrapper around official ``voxcpm.VoxCPM``."""
|
||||
|
||||
REQUIRED_RUNTIME_VERSION = "2.0.3"
|
||||
SAMPLE_RATE = 48000
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_name: str = VoxCPMDownloader.DEFAULT_MODEL,
|
||||
device: str = "auto",
|
||||
optimize: bool = False,
|
||||
model_dir: Optional[str] = None,
|
||||
lora_adapter: Optional[str] = None,
|
||||
):
|
||||
self.model_name = str(model_name or VoxCPMDownloader.DEFAULT_MODEL)
|
||||
self.device = str(device or "auto")
|
||||
self.optimize = bool(optimize)
|
||||
self.downloader = VoxCPMDownloader()
|
||||
self.model_dir = os.path.abspath(model_dir) if model_dir else None
|
||||
self.lora_adapter = (
|
||||
os.path.abspath(os.path.expanduser(str(lora_adapter)))
|
||||
if str(lora_adapter or "").strip()
|
||||
else None
|
||||
)
|
||||
self._model_spec = self._inspect_model_spec(self.model_dir)
|
||||
self._runtime = None
|
||||
self._runtime_was_optimized = False
|
||||
|
||||
def _inspect_model_spec(
|
||||
self,
|
||||
resolved_path: Optional[str],
|
||||
):
|
||||
expected = self.downloader.get_model_spec(
|
||||
self.model_name,
|
||||
resolved_path=resolved_path,
|
||||
)
|
||||
if resolved_path is None:
|
||||
return expected
|
||||
|
||||
# A local finetune may use either supported weight serialization.
|
||||
# Preserve the expected generation metadata while validating against
|
||||
# the files the official runtime will actually open.
|
||||
local = self.downloader.get_model_spec(
|
||||
f"local:{os.path.basename(resolved_path)}",
|
||||
resolved_path=resolved_path,
|
||||
)
|
||||
local["canonical"] = expected["canonical"]
|
||||
local["repo_id"] = expected.get("repo_id")
|
||||
return local
|
||||
|
||||
@staticmethod
|
||||
def _version_tuple(version: str) -> tuple[int, ...]:
|
||||
values = re.findall(r"\d+", str(version))
|
||||
return tuple(int(value) for value in values[:3])
|
||||
|
||||
def _import_official_package(self):
|
||||
"""Import PyPI VoxCPM without this engine folder shadowing it."""
|
||||
try:
|
||||
installed_version = importlib.metadata.version("voxcpm")
|
||||
except importlib.metadata.PackageNotFoundError as exc:
|
||||
raise ImportError(
|
||||
"VoxCPM is not installed in the active ComfyUI Python "
|
||||
f"environment ({sys.executable}). Install voxcpm=="
|
||||
f"{self.REQUIRED_RUNTIME_VERSION}."
|
||||
) from exc
|
||||
|
||||
if self._version_tuple(installed_version) < self._version_tuple(
|
||||
self.REQUIRED_RUNTIME_VERSION
|
||||
):
|
||||
raise ImportError(
|
||||
f"VoxCPM {installed_version} is too old. Install voxcpm=="
|
||||
f"{self.REQUIRED_RUNTIME_VERSION} in {sys.executable}."
|
||||
)
|
||||
|
||||
importlib.invalidate_caches()
|
||||
nodes_dir = os.path.join(_PROJECT_ROOT, "nodes")
|
||||
blocked_paths = {
|
||||
os.path.abspath(_ENGINES_DIR),
|
||||
os.path.abspath(nodes_dir),
|
||||
}
|
||||
original_sys_path = list(sys.path)
|
||||
sys.path[:] = [
|
||||
path
|
||||
for path in sys.path
|
||||
if os.path.abspath(path or os.getcwd()) not in blocked_paths
|
||||
]
|
||||
|
||||
stale_modules = []
|
||||
for module_name, module in list(sys.modules.items()):
|
||||
if not (
|
||||
module_name == "voxcpm"
|
||||
or module_name.startswith("voxcpm.")
|
||||
):
|
||||
continue
|
||||
module_file = getattr(module, "__file__", "") or ""
|
||||
if module_file and os.path.abspath(module_file).startswith(
|
||||
os.path.abspath(_PROJECT_ROOT)
|
||||
):
|
||||
stale_modules.append((module_name, module))
|
||||
del sys.modules[module_name]
|
||||
|
||||
try:
|
||||
official_module = importlib.import_module("voxcpm")
|
||||
VoxCPM = official_module.VoxCPM
|
||||
except Exception as exc:
|
||||
for module_name, module in stale_modules:
|
||||
sys.modules.setdefault(module_name, module)
|
||||
raise ImportError(
|
||||
"Failed to import the official `voxcpm` package in the active "
|
||||
"ComfyUI Python environment. This is usually a missing "
|
||||
"dependency or an import-path collision. "
|
||||
f"Active Python: {sys.executable}"
|
||||
) from exc
|
||||
finally:
|
||||
sys.path[:] = original_sys_path
|
||||
|
||||
return VoxCPM
|
||||
|
||||
def _ensure_runtime_loaded(self) -> None:
|
||||
if self._runtime is not None:
|
||||
return
|
||||
|
||||
if self.model_dir is None:
|
||||
self.model_dir = self.downloader.resolve_model_path(self.model_name)
|
||||
self.model_dir = os.path.abspath(self.model_dir)
|
||||
self._model_spec = self._inspect_model_spec(self.model_dir)
|
||||
if not self.downloader.is_model_complete(
|
||||
self.model_dir,
|
||||
self._model_spec,
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"Refusing to load incomplete VoxCPM model: {self.model_dir}"
|
||||
)
|
||||
|
||||
if self.lora_adapter:
|
||||
if self.architecture != "voxcpm2":
|
||||
raise ValueError("VoxCPM LoRA adapters are currently supported only by VoxCPM2")
|
||||
config_path = os.path.join(self.lora_adapter, "lora_config.json")
|
||||
weights_path = os.path.join(
|
||||
self.lora_adapter, "lora_weights.safetensors"
|
||||
)
|
||||
if not os.path.isfile(config_path) or not os.path.isfile(weights_path):
|
||||
raise FileNotFoundError(
|
||||
"VoxCPM LoRA folder must contain lora_config.json and "
|
||||
f"lora_weights.safetensors: {self.lora_adapter}"
|
||||
)
|
||||
with open(config_path, "r", encoding="utf-8") as handle:
|
||||
lora_info = json.load(handle)
|
||||
suite_info = lora_info.get("tts_audio_suite") or {}
|
||||
adapter_architecture = str(
|
||||
suite_info.get("architecture") or "voxcpm2"
|
||||
).lower()
|
||||
if adapter_architecture != self.architecture:
|
||||
raise ValueError(
|
||||
f"VoxCPM LoRA architecture mismatch: adapter={adapter_architecture}, "
|
||||
f"base={self.architecture}"
|
||||
)
|
||||
adapter_variant = str(suite_info.get("model_variant") or "").strip()
|
||||
current_variant = str(self._model_spec.get("canonical") or "").strip()
|
||||
if adapter_variant and adapter_variant != current_variant:
|
||||
raise ValueError(
|
||||
f"VoxCPM LoRA base mismatch: adapter={adapter_variant}, "
|
||||
f"selected={current_variant}"
|
||||
)
|
||||
|
||||
from utils.device import resolve_torch_device
|
||||
|
||||
resolved_device = resolve_torch_device(self.device)
|
||||
effective_optimize = self.optimize and str(resolved_device).startswith(
|
||||
"cuda"
|
||||
)
|
||||
if self.optimize and not effective_optimize:
|
||||
print(
|
||||
"⚠️ VoxCPM torch.compile optimization is CUDA-only; "
|
||||
f"loading without it on {resolved_device}"
|
||||
)
|
||||
|
||||
VoxCPM = self._import_official_package()
|
||||
print(f"🔄 Loading {self._model_spec['canonical']} via official VoxCPM")
|
||||
print(f" Path: {self.model_dir}")
|
||||
print(
|
||||
f" Architecture: {self.architecture} | "
|
||||
f"Device: {resolved_device} | Optimize: {effective_optimize}"
|
||||
)
|
||||
|
||||
# Passing an already-validated absolute directory plus local_files_only
|
||||
# guarantees the official wrapper cannot create a second HF cache copy.
|
||||
runtime = VoxCPM.from_pretrained(
|
||||
hf_model_id=self.model_dir,
|
||||
load_denoiser=False,
|
||||
local_files_only=True,
|
||||
optimize=effective_optimize,
|
||||
device=resolved_device,
|
||||
lora_weights_path=self.lora_adapter,
|
||||
)
|
||||
self._runtime = runtime
|
||||
self._runtime_was_optimized = effective_optimize
|
||||
self.device = str(resolved_device)
|
||||
if self.lora_adapter:
|
||||
print(f" LoRA: {self.lora_adapter}")
|
||||
print("✅ VoxCPM runtime ready")
|
||||
|
||||
@property
|
||||
def architecture(self) -> str:
|
||||
return str(self._model_spec["architecture"])
|
||||
|
||||
@property
|
||||
def sample_rate(self) -> int:
|
||||
if self._runtime is not None:
|
||||
tts_model = getattr(self._runtime, "tts_model", None)
|
||||
runtime_rate = getattr(tts_model, "sample_rate", None)
|
||||
if runtime_rate is not None:
|
||||
return int(runtime_rate)
|
||||
return int(self._model_spec["sample_rate"])
|
||||
|
||||
@staticmethod
|
||||
def _seed_global_rngs(seed: int) -> None:
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
random.seed(seed)
|
||||
np.random.seed(seed % (2**32))
|
||||
torch.manual_seed(seed)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed_all(seed)
|
||||
|
||||
@staticmethod
|
||||
def _runtime_accepts_seed(runtime: Any) -> bool:
|
||||
# VoxCPM.generate itself is a *args/**kwargs forwarding method. Inspect
|
||||
# the concrete implementation so PyPI 2.0.3 is not sent an unsupported
|
||||
# seed keyword, while newer official releases can receive it directly.
|
||||
target = getattr(runtime, "_generate", None)
|
||||
if target is None:
|
||||
target = getattr(runtime, "generate")
|
||||
try:
|
||||
return "seed" in inspect.signature(target).parameters
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
|
||||
def generate(
|
||||
self,
|
||||
text: str,
|
||||
prompt_wav_path: Optional[str] = None,
|
||||
prompt_text: Optional[str] = None,
|
||||
reference_wav_path: Optional[str] = None,
|
||||
cfg_value: float = 2.0,
|
||||
inference_timesteps: int = 10,
|
||||
min_len: int = 2,
|
||||
max_len: int = 4096,
|
||||
normalize: bool = False,
|
||||
retry_badcase: bool = True,
|
||||
retry_badcase_max_times: int = 3,
|
||||
retry_badcase_ratio_threshold: float = 6.0,
|
||||
seed: Optional[int] = None,
|
||||
):
|
||||
"""Call the official non-streaming API and return mono ``[1, T]`` audio."""
|
||||
self._ensure_runtime_loaded()
|
||||
|
||||
kwargs = {
|
||||
"text": text,
|
||||
"prompt_wav_path": prompt_wav_path or None,
|
||||
"prompt_text": prompt_text if prompt_text is not None else None,
|
||||
"reference_wav_path": reference_wav_path or None,
|
||||
"cfg_value": float(cfg_value),
|
||||
"inference_timesteps": int(inference_timesteps),
|
||||
"min_len": int(min_len),
|
||||
"max_len": int(max_len),
|
||||
"normalize": bool(normalize),
|
||||
"denoise": False,
|
||||
"retry_badcase": bool(retry_badcase),
|
||||
"retry_badcase_max_times": int(retry_badcase_max_times),
|
||||
"retry_badcase_ratio_threshold": float(
|
||||
retry_badcase_ratio_threshold
|
||||
),
|
||||
}
|
||||
|
||||
if seed is not None:
|
||||
seed_value = int(seed)
|
||||
if self._runtime_accepts_seed(self._runtime):
|
||||
kwargs["seed"] = seed_value
|
||||
else:
|
||||
self._seed_global_rngs(seed_value)
|
||||
|
||||
waveform = self._runtime.generate(**kwargs)
|
||||
|
||||
import torch
|
||||
|
||||
if isinstance(waveform, torch.Tensor):
|
||||
audio = waveform.detach().float().cpu()
|
||||
else:
|
||||
audio = torch.as_tensor(waveform, dtype=torch.float32).cpu()
|
||||
return audio.reshape(1, -1)
|
||||
|
||||
def parameters(self) -> Iterator[Any]:
|
||||
"""Expose official model parameters for ComfyUI memory accounting."""
|
||||
if self._runtime is None:
|
||||
return
|
||||
tts_model = getattr(self._runtime, "tts_model", None)
|
||||
if tts_model is not None and hasattr(tts_model, "parameters"):
|
||||
yield from tts_model.parameters()
|
||||
|
||||
@staticmethod
|
||||
def _clear_model_caches(tts_model: Any) -> None:
|
||||
if tts_model is None:
|
||||
return
|
||||
for module_name in ("base_lm", "residual_lm"):
|
||||
module = getattr(tts_model, module_name, None)
|
||||
cache = getattr(module, "kv_cache", None)
|
||||
if hasattr(cache, "clear"):
|
||||
try:
|
||||
cache.clear()
|
||||
except Exception:
|
||||
pass
|
||||
if module is not None:
|
||||
try:
|
||||
module.kv_cache = None
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def unload_runtime(self) -> None:
|
||||
"""Destroy the official runtime instead of copying it into system RAM."""
|
||||
runtime = self._runtime
|
||||
self._runtime = None
|
||||
was_optimized = self._runtime_was_optimized
|
||||
self._runtime_was_optimized = False
|
||||
|
||||
if runtime is not None:
|
||||
tts_model = getattr(runtime, "tts_model", None)
|
||||
self._clear_model_caches(tts_model)
|
||||
for attr_name in ("tts_model", "text_normalizer", "denoiser"):
|
||||
try:
|
||||
setattr(runtime, attr_name, None)
|
||||
except Exception:
|
||||
pass
|
||||
del tts_model
|
||||
del runtime
|
||||
|
||||
torch = sys.modules.get("torch")
|
||||
if was_optimized and torch is not None:
|
||||
try:
|
||||
import torch._dynamo as torch_dynamo
|
||||
|
||||
torch_dynamo.reset()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
gc.collect()
|
||||
if torch is not None and torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
if hasattr(torch.cuda, "ipc_collect"):
|
||||
try:
|
||||
torch.cuda.ipc_collect()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def to(self, device):
|
||||
"""ComfyUI offload hook; the next generation reloads on demand."""
|
||||
target = str(device)
|
||||
if self._runtime is not None and (
|
||||
target.startswith("cpu") or target != self.device
|
||||
):
|
||||
self.unload_runtime()
|
||||
self.device = target
|
||||
return self
|
||||
|
||||
def unload(self) -> None:
|
||||
self.unload_runtime()
|
||||
+48
-40
@@ -29,7 +29,7 @@ class TTSAudioInstaller:
|
||||
|
||||
# Bump when the install procedure itself gains or changes dependencies.
|
||||
# This keeps normal ComfyUI patch updates on the fast path.
|
||||
INSTALL_STATE_VERSION = 3
|
||||
INSTALL_STATE_VERSION = 2
|
||||
|
||||
CORE_MODULE_CHECKS = (
|
||||
("torch", "PyTorch"),
|
||||
@@ -47,7 +47,6 @@ class TTSAudioInstaller:
|
||||
("VibeVoice", ("vibevoice", "av")),
|
||||
("Echo-TTS", ("echo_tts",)),
|
||||
("OmniVoice", ("omnivoice",)),
|
||||
("VoxCPM", ("voxcpm",)),
|
||||
("Dots TTS", ("dots_tts.runtime",)),
|
||||
("DramaBox", ("av", "einops", "yaml")),
|
||||
("Fish Audio S2", ("fish_speech.inference_engine",)),
|
||||
@@ -178,8 +177,8 @@ class TTSAudioInstaller:
|
||||
else:
|
||||
self.log("All requirements.txt dependencies already satisfied", "SUCCESS")
|
||||
|
||||
def check_system_dependencies(self):
|
||||
"""Check for required system libraries and provide helpful error messages"""
|
||||
def check_system_dependencies(self):
|
||||
"""Check optional system libraries and provide helpful feature warnings."""
|
||||
if self.is_windows:
|
||||
return True # Windows packages come pre-compiled
|
||||
|
||||
@@ -211,8 +210,8 @@ class TTSAudioInstaller:
|
||||
|
||||
return False
|
||||
|
||||
def check_macos_dependencies(self):
|
||||
"""Check for required system libraries on macOS"""
|
||||
def check_macos_dependencies(self):
|
||||
"""Check for optional system libraries on macOS."""
|
||||
self.log("Checking macOS system dependencies...", "INFO")
|
||||
missing_deps = []
|
||||
|
||||
@@ -231,12 +230,12 @@ class TTSAudioInstaller:
|
||||
pass
|
||||
|
||||
if missing_deps:
|
||||
self.log("Missing system dependencies detected!", "WARNING")
|
||||
print("\n" + "="*60)
|
||||
print("MACOS SYSTEM DEPENDENCIES REQUIRED")
|
||||
print("="*60)
|
||||
for dep, purpose in missing_deps:
|
||||
print(f"• {dep} (for {purpose})")
|
||||
self.log("Optional system dependencies are missing", "WARNING")
|
||||
print("\n" + "="*60)
|
||||
print("OPTIONAL MACOS SYSTEM DEPENDENCIES")
|
||||
print("="*60)
|
||||
for dep, purpose in missing_deps:
|
||||
print(f"• {dep} (for {purpose})")
|
||||
|
||||
print("\nPlease install with Homebrew:")
|
||||
deps_list = " ".join([dep for dep, _ in missing_deps])
|
||||
@@ -250,15 +249,19 @@ class TTSAudioInstaller:
|
||||
print("Check with: python -c \"import platform; print(platform.machine())\"")
|
||||
print("Should show: arm64 (not x86_64)")
|
||||
|
||||
print("="*60)
|
||||
print("Then run this install script again.\n")
|
||||
return False
|
||||
|
||||
self.log("macOS system dependencies check passed", "SUCCESS")
|
||||
return True
|
||||
|
||||
def check_linux_dependencies(self):
|
||||
"""Check for required system libraries on Linux"""
|
||||
print("="*60)
|
||||
print("Core TTS installation will continue; only the listed features may be unavailable.\n")
|
||||
self.add_install_warning(
|
||||
"Optional macOS audio libraries",
|
||||
"Install the listed Homebrew packages to enable every audio feature.",
|
||||
)
|
||||
|
||||
if not missing_deps:
|
||||
self.log("macOS system dependencies check passed", "SUCCESS")
|
||||
return True
|
||||
|
||||
def check_linux_dependencies(self):
|
||||
"""Check for optional system libraries on Linux."""
|
||||
self.log("Checking Linux system dependencies...", "INFO")
|
||||
missing_deps = []
|
||||
|
||||
@@ -280,26 +283,34 @@ class TTSAudioInstaller:
|
||||
pass
|
||||
|
||||
if missing_deps:
|
||||
self.log("Missing system dependencies detected!", "WARNING")
|
||||
print("\n" + "="*60)
|
||||
print("LINUX SYSTEM DEPENDENCIES REQUIRED")
|
||||
print("="*60)
|
||||
for dep, purpose in missing_deps:
|
||||
print(f"• {dep} (for {purpose})")
|
||||
self.log("Optional system dependencies are missing", "WARNING")
|
||||
print("\n" + "="*60)
|
||||
print("OPTIONAL LINUX SYSTEM DEPENDENCIES")
|
||||
print("="*60)
|
||||
for dep, purpose in missing_deps:
|
||||
print(f"• {dep} (for {purpose})")
|
||||
|
||||
print("\nPlease install with:")
|
||||
print("# Ubuntu/Debian:")
|
||||
deps_list = " ".join([dep for dep, _ in missing_deps])
|
||||
print(f"sudo apt-get install {deps_list}")
|
||||
print("\n# Fedora/RHEL:")
|
||||
fedora_deps = deps_list.replace('-dev', '-devel').replace('19', '')
|
||||
print(f"sudo dnf install {fedora_deps}")
|
||||
print("="*60)
|
||||
print("Then run this install script again.\n")
|
||||
return False
|
||||
|
||||
self.log("Linux system dependencies check passed", "SUCCESS")
|
||||
return True
|
||||
fedora_packages = {
|
||||
"libsamplerate0-dev": "libsamplerate-devel",
|
||||
"portaudio19-dev": "portaudio-devel",
|
||||
}
|
||||
fedora_deps = " ".join(fedora_packages.get(dep, dep) for dep, _ in missing_deps)
|
||||
print(f"sudo dnf install {fedora_deps}")
|
||||
print("="*60)
|
||||
print("Core TTS installation will continue; only the listed features may be unavailable.\n")
|
||||
self.add_install_warning(
|
||||
"Optional Linux audio libraries",
|
||||
"Install the listed system packages to enable every audio feature.",
|
||||
)
|
||||
|
||||
if not missing_deps:
|
||||
self.log("Linux system dependencies check passed", "SUCCESS")
|
||||
return True
|
||||
|
||||
def run_pip_command(self, args: List[str], description: str, ignore_errors: bool = False) -> bool:
|
||||
"""Execute pip command with error handling and Windows UTF-8 support"""
|
||||
@@ -1170,7 +1181,6 @@ class TTSAudioInstaller:
|
||||
"descript-audiotools", # Forces protobuf downgrade from 6.x to 3.19.x
|
||||
"cached-path", # Forces package downgrades
|
||||
"omnivoice", # Keep shared dependency resolution under our control
|
||||
"voxcpm==2.0.3", # Official runtime; its broad dependency set would replace suite packages
|
||||
"torchcrepe", # Conflicts via librosa dependency
|
||||
# NOTE: onnxruntime moved to install_onnxruntime_with_gpu_support() for smart GPU detection
|
||||
"opencv-python", # Forces numpy downgrade from 2.x to 1.26.x - dependencies pre-installed
|
||||
@@ -1909,10 +1919,8 @@ def main():
|
||||
else:
|
||||
installer.ensure_requirements_installed() # Ensure requirements.txt is installed first
|
||||
|
||||
# Check system dependencies (Linux only)
|
||||
if not installer.check_system_dependencies():
|
||||
installer.log("System dependency check failed - aborting installation", "ERROR")
|
||||
sys.exit(1)
|
||||
# Report optional system dependencies without blocking engine setup.
|
||||
installer.check_system_dependencies()
|
||||
|
||||
# Install in correct order to prevent conflicts
|
||||
installer.install_pytorch_with_cuda() # Install PyTorch first with proper CUDA detection
|
||||
|
||||
@@ -11,7 +11,7 @@ except ImportError:
|
||||
pass
|
||||
|
||||
# Version and constants
|
||||
VERSION = "5.6.0"
|
||||
VERSION = "5.6.2"
|
||||
IS_DEV = False # Set to False for release builds
|
||||
VERSION_DISPLAY = f"v{VERSION}" + (" (dev)" if IS_DEV else "")
|
||||
SEPARATOR = "=" * 70
|
||||
@@ -179,6 +179,14 @@ except Exception as e:
|
||||
print(f"❌ Dots TTS Engine failed: {e}")
|
||||
DOTS_TTS_ENGINE_AVAILABLE = False
|
||||
|
||||
try:
|
||||
audio8_tts_engine_module = load_node_module("audio8_tts_engine_node", "engines/audio8_tts_engine_node.py")
|
||||
Audio8TTSEngineNode = audio8_tts_engine_module.Audio8TTSEngineNode
|
||||
AUDIO8_TTS_ENGINE_AVAILABLE = True
|
||||
except Exception as e:
|
||||
print(f"❌ Audio8 TTS Engine failed: {e}")
|
||||
AUDIO8_TTS_ENGINE_AVAILABLE = False
|
||||
|
||||
try:
|
||||
dramabox_engine_module = load_node_module("dramabox_engine_node", "engines/dramabox_engine_node.py")
|
||||
DramaBoxEngineNode = dramabox_engine_module.DramaBoxEngineNode
|
||||
@@ -203,14 +211,6 @@ except Exception as e:
|
||||
print(f"❌ OmniVoice Engine failed: {e}")
|
||||
OMNIVOICE_ENGINE_AVAILABLE = False
|
||||
|
||||
try:
|
||||
voxcpm_engine_module = load_node_module("voxcpm_engine_node", "engines/voxcpm_engine_node.py")
|
||||
VoxCPMEngineNode = voxcpm_engine_module.VoxCPMEngineNode
|
||||
VOXCPM_ENGINE_AVAILABLE = True
|
||||
except Exception as e:
|
||||
print(f"❌ VoxCPM Engine failed: {e}")
|
||||
VOXCPM_ENGINE_AVAILABLE = False
|
||||
|
||||
try:
|
||||
omnivoice_instruction_builder_module = load_node_module("omnivoice_instruction_builder_node", "omnivoice/omnivoice_instruction_builder_node.py")
|
||||
OmniVoiceInstructionBuilderNode = omnivoice_instruction_builder_module.OmniVoiceInstructionBuilderNode
|
||||
@@ -508,26 +508,6 @@ except Exception as e:
|
||||
print(f"❌ MOSS Dataset Rows failed: {e}")
|
||||
MOSS_DATASET_ROWS_AVAILABLE = False
|
||||
|
||||
try:
|
||||
voxcpm_dataset_prep_module = load_node_module(
|
||||
"voxcpm_dataset_prep_node", "training/voxcpm_dataset_prep_node.py"
|
||||
)
|
||||
VoxCPMDatasetPrepNode = voxcpm_dataset_prep_module.VoxCPMDatasetPrepNode
|
||||
VOXCPM_DATASET_PREP_AVAILABLE = True
|
||||
except Exception as e:
|
||||
print(f"❌ VoxCPM Dataset Prep failed: {e}")
|
||||
VOXCPM_DATASET_PREP_AVAILABLE = False
|
||||
|
||||
try:
|
||||
voxcpm_training_config_module = load_node_module(
|
||||
"voxcpm_training_config_node", "training/voxcpm_training_config_node.py"
|
||||
)
|
||||
VoxCPMTrainingConfigNode = voxcpm_training_config_module.VoxCPMTrainingConfigNode
|
||||
VOXCPM_TRAINING_CONFIG_AVAILABLE = True
|
||||
except Exception as e:
|
||||
print(f"❌ VoxCPM Training Config failed: {e}")
|
||||
VOXCPM_TRAINING_CONFIG_AVAILABLE = False
|
||||
|
||||
try:
|
||||
phoneme_text_normalizer_module = load_node_module("phoneme_text_normalizer_node", "text/phoneme_text_normalizer_node.py")
|
||||
PhonemeTextNormalizer = phoneme_text_normalizer_module.PhonemeTextNormalizer
|
||||
@@ -698,6 +678,10 @@ if DOTS_TTS_ENGINE_AVAILABLE:
|
||||
NODE_CLASS_MAPPINGS["DotsTTSEngineNode"] = DotsTTSEngineNode
|
||||
NODE_DISPLAY_NAME_MAPPINGS["DotsTTSEngineNode"] = "⚙️ Dots TTS Engine"
|
||||
|
||||
if AUDIO8_TTS_ENGINE_AVAILABLE:
|
||||
NODE_CLASS_MAPPINGS["Audio8TTSEngineNode"] = Audio8TTSEngineNode
|
||||
NODE_DISPLAY_NAME_MAPPINGS["Audio8TTSEngineNode"] = "⚙️ Audio8 TTS Engine"
|
||||
|
||||
if DRAMABOX_ENGINE_AVAILABLE:
|
||||
NODE_CLASS_MAPPINGS["DramaBoxEngineNode"] = DramaBoxEngineNode
|
||||
NODE_DISPLAY_NAME_MAPPINGS["DramaBoxEngineNode"] = "⚙️ DramaBox Engine"
|
||||
@@ -710,10 +694,6 @@ if OMNIVOICE_ENGINE_AVAILABLE:
|
||||
NODE_CLASS_MAPPINGS["OmniVoiceEngineNode"] = OmniVoiceEngineNode
|
||||
NODE_DISPLAY_NAME_MAPPINGS["OmniVoiceEngineNode"] = "⚙️ OmniVoice Engine"
|
||||
|
||||
if VOXCPM_ENGINE_AVAILABLE:
|
||||
NODE_CLASS_MAPPINGS["VoxCPMEngineNode"] = VoxCPMEngineNode
|
||||
NODE_DISPLAY_NAME_MAPPINGS["VoxCPMEngineNode"] = "⚙️ VoxCPM Engine"
|
||||
|
||||
if OMNIVOICE_INSTRUCTION_BUILDER_AVAILABLE:
|
||||
NODE_CLASS_MAPPINGS["OmniVoiceInstructionBuilderNode"] = OmniVoiceInstructionBuilderNode
|
||||
NODE_DISPLAY_NAME_MAPPINGS["OmniVoiceInstructionBuilderNode"] = "📐 Visual Tag Builder"
|
||||
@@ -877,14 +857,6 @@ if MOSS_DATASET_ROWS_AVAILABLE:
|
||||
NODE_CLASS_MAPPINGS["MossDatasetRowsNode"] = MossDatasetRowsNode
|
||||
NODE_DISPLAY_NAME_MAPPINGS["MossDatasetRowsNode"] = "🧾 MOSS Dataset Rows"
|
||||
|
||||
if VOXCPM_DATASET_PREP_AVAILABLE:
|
||||
NODE_CLASS_MAPPINGS["VoxCPMDatasetPrepNode"] = VoxCPMDatasetPrepNode
|
||||
NODE_DISPLAY_NAME_MAPPINGS["VoxCPMDatasetPrepNode"] = "📦 VoxCPM2 Dataset Prep"
|
||||
|
||||
if VOXCPM_TRAINING_CONFIG_AVAILABLE:
|
||||
NODE_CLASS_MAPPINGS["VoxCPMTrainingConfigNode"] = VoxCPMTrainingConfigNode
|
||||
NODE_DISPLAY_NAME_MAPPINGS["VoxCPMTrainingConfigNode"] = "🎛️ VoxCPM2 LoRA Config"
|
||||
|
||||
# Register text processing nodes
|
||||
if PHONEME_TEXT_NORMALIZER_AVAILABLE:
|
||||
NODE_CLASS_MAPPINGS["PhonemeTextNormalizer"] = PhonemeTextNormalizer
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""Audio8 TTS text and SRT processors."""
|
||||
@@ -0,0 +1,403 @@
|
||||
"""Suite orchestration for Audio8 TTS text generation."""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from typing import Any, Dict, List, Tuple, Union
|
||||
|
||||
import comfy.model_management as model_management
|
||||
import torch
|
||||
|
||||
current_dir = os.path.dirname(__file__)
|
||||
nodes_dir = os.path.dirname(current_dir)
|
||||
project_root = os.path.dirname(nodes_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
from utils.audio.chunk_timing import ChunkTimingHelper
|
||||
from utils.audio.edit_post_processor import (
|
||||
process_segments as apply_edit_post_processing,
|
||||
)
|
||||
from utils.text.character_parser import character_parser
|
||||
from utils.text.pause_processor import PauseTagProcessor
|
||||
from utils.text.segment_parameters import ParameterValidator, apply_segment_parameters
|
||||
from utils.text.step_audio_editx_special_tags import get_edit_tags_for_segment
|
||||
from utils.voice.character_logging import (
|
||||
format_resolved_character_block,
|
||||
resolved_character_label,
|
||||
)
|
||||
from utils.voice.discovery import (
|
||||
get_available_characters,
|
||||
get_character_mapping,
|
||||
voice_discovery,
|
||||
)
|
||||
from utils.voice.reference import effective_voice_audio
|
||||
|
||||
|
||||
class Audio8TTSProcessor:
|
||||
"""Handle suite-level tags, voices, chunking, and Audio8 generation."""
|
||||
|
||||
SAMPLE_RATE = 44100
|
||||
|
||||
def __init__(self, adapter, engine_config: Dict[str, Any]):
|
||||
self.adapter = adapter
|
||||
self.config = engine_config.copy() if engine_config else {}
|
||||
|
||||
def update_config(self, new_config: Dict[str, Any]):
|
||||
self.config = new_config.copy() if new_config else {}
|
||||
self.adapter.update_config(self.config)
|
||||
|
||||
@staticmethod
|
||||
def _check_interrupt(context: str = "") -> None:
|
||||
if not model_management.interrupt_processing:
|
||||
return
|
||||
suffix = f" {context}" if context else ""
|
||||
raise InterruptedError(f"Audio8 TTS generation interrupted{suffix}")
|
||||
|
||||
def _setup_character_parser(self, text: str) -> None:
|
||||
character_tags = re.findall(r"\[([^\]]+)\]", text or "")
|
||||
characters_from_tags = [
|
||||
tag.split("|")[0].strip()
|
||||
for tag in character_tags
|
||||
if not tag.lower().startswith("pause:")
|
||||
]
|
||||
|
||||
all_available = set(get_available_characters() or [])
|
||||
for alias, target in voice_discovery.get_character_aliases().items():
|
||||
all_available.add(alias.lower())
|
||||
all_available.add(target.lower())
|
||||
all_available.update(
|
||||
character.lower() for character in characters_from_tags if character
|
||||
)
|
||||
all_available.add("narrator")
|
||||
|
||||
character_parser.set_available_characters(list(all_available))
|
||||
character_parser.reset_session_cache()
|
||||
|
||||
@staticmethod
|
||||
def _as_voice_reference(value: Any) -> Dict[str, Any]:
|
||||
if isinstance(value, dict):
|
||||
return value.copy()
|
||||
if value is None:
|
||||
return {}
|
||||
return {"audio": value}
|
||||
|
||||
def _resolve_voice(
|
||||
self,
|
||||
character: str,
|
||||
voice_mapping: Dict[str, Any],
|
||||
discovered_mapping: Dict[str, Tuple[Any, Any]],
|
||||
) -> Dict[str, Any]:
|
||||
if character in voice_mapping:
|
||||
mapped_voice = self._as_voice_reference(voice_mapping[character])
|
||||
if effective_voice_audio(mapped_voice) is not None or mapped_voice.get(
|
||||
"reference_text"
|
||||
):
|
||||
return mapped_voice
|
||||
|
||||
if character != "narrator":
|
||||
audio_path, reference_text = discovered_mapping.get(character, (None, None))
|
||||
if audio_path:
|
||||
return {
|
||||
"audio_path": audio_path,
|
||||
"reference_text": reference_text or "",
|
||||
}
|
||||
print(
|
||||
f"⚠️ Audio8 TTS: No voice found for '{character}'; "
|
||||
"using narrator/no-reference fallback"
|
||||
)
|
||||
|
||||
return self._as_voice_reference(voice_mapping.get("narrator"))
|
||||
|
||||
@staticmethod
|
||||
def _voice_log_note(voice_ref: Dict[str, Any]) -> str:
|
||||
reference_text = ""
|
||||
if isinstance(voice_ref, dict):
|
||||
reference_text = str(voice_ref.get("reference_text") or "").strip()
|
||||
has_audio = (
|
||||
isinstance(voice_ref, dict) and effective_voice_audio(voice_ref) is not None
|
||||
)
|
||||
|
||||
if not has_audio and not reference_text:
|
||||
return " [no reference voice]"
|
||||
if has_audio and reference_text:
|
||||
return f" [reference transcript: {len(reference_text)} chars]"
|
||||
if has_audio:
|
||||
return " [reference audio has no transcript; adapter will validate]"
|
||||
return " [reference transcript has no audio; adapter will validate]"
|
||||
|
||||
@staticmethod
|
||||
def _format_parameter_log(
|
||||
filtered_params: Dict[str, Any],
|
||||
current_config: Dict[str, Any],
|
||||
current_seed: int,
|
||||
) -> str:
|
||||
if not filtered_params:
|
||||
return ""
|
||||
|
||||
values = []
|
||||
for key in ("seed", "temperature", "top_p", "top_k", "max_new_tokens"):
|
||||
if key not in filtered_params:
|
||||
continue
|
||||
value = current_seed if key == "seed" else current_config.get(key)
|
||||
values.append(f"{key}={value}")
|
||||
return ", ".join(values)
|
||||
|
||||
@staticmethod
|
||||
def _validate_segment_config(
|
||||
config: Dict[str, Any],
|
||||
filtered_params: Dict[str, Any],
|
||||
) -> Dict[str, Any]:
|
||||
"""Apply Audio8-specific bounds and keep retry budgets consistent."""
|
||||
if "top_p" in filtered_params and float(config["top_p"]) <= 0:
|
||||
raise ValueError("Audio8 segment top_p must be greater than 0")
|
||||
|
||||
if "max_new_tokens" in filtered_params:
|
||||
max_new_tokens = int(config["max_new_tokens"])
|
||||
if max_new_tokens > 2048:
|
||||
raise ValueError("Audio8 segment max_new_tokens must be at most 2048")
|
||||
config["retry_max_new_tokens"] = max(
|
||||
max_new_tokens,
|
||||
int(config.get("retry_max_new_tokens", 2000)),
|
||||
)
|
||||
return config
|
||||
|
||||
def _log_generation(
|
||||
self,
|
||||
character: str,
|
||||
text: str,
|
||||
voice_ref: Dict[str, Any],
|
||||
config: Dict[str, Any],
|
||||
chunk_count: int,
|
||||
parameter_log: str,
|
||||
show_text_logging: bool,
|
||||
) -> None:
|
||||
display_name = resolved_character_label(character, voice_ref)
|
||||
print(
|
||||
f"🎭 Audio8 TTS - Generating for '{display_name}'"
|
||||
f"{self._voice_log_note(voice_ref)}"
|
||||
)
|
||||
print(
|
||||
" Settings: "
|
||||
f"mode={'Sampling' if config.get('do_sample', True) else 'Greedy'}, "
|
||||
f"temperature={config.get('temperature', 0.8)}, "
|
||||
f"top_p={config.get('top_p', 0.95)}, "
|
||||
f"top_k={config.get('top_k', 50)}, "
|
||||
f"max_new_tokens={config.get('max_new_tokens', 1024)}"
|
||||
)
|
||||
if parameter_log:
|
||||
print(f"🎛️ Audio8 TTS segment params: {parameter_log}")
|
||||
if show_text_logging:
|
||||
print(format_resolved_character_block(character, text, voice_ref))
|
||||
if chunk_count > 1:
|
||||
print(
|
||||
f"📝 Audio8 TTS: Chunking '{display_name}' into "
|
||||
f"{chunk_count} suite chunks"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_audio(audio: Any) -> torch.Tensor:
|
||||
if not isinstance(audio, torch.Tensor):
|
||||
audio = torch.as_tensor(audio, dtype=torch.float32)
|
||||
audio = audio.detach().to(device="cpu", dtype=torch.float32)
|
||||
if audio.dim() == 1:
|
||||
audio = audio.unsqueeze(0)
|
||||
elif audio.dim() == 3 and audio.shape[0] == 1:
|
||||
audio = audio.squeeze(0)
|
||||
if audio.dim() != 2:
|
||||
raise ValueError(
|
||||
"Audio8 adapter returned an invalid waveform shape; "
|
||||
"expected [channels, samples]"
|
||||
)
|
||||
return audio
|
||||
|
||||
def process_text(
|
||||
self,
|
||||
text: str,
|
||||
voice_mapping: Dict[str, Any],
|
||||
seed: int,
|
||||
enable_chunking: bool = True,
|
||||
max_chars_per_chunk: int = 400,
|
||||
chunk_combination_method: str = "auto",
|
||||
silence_between_chunks_ms: int = 100,
|
||||
enable_audio_cache: bool = True,
|
||||
apply_edit_postprocessing: bool = True,
|
||||
show_text_logging: bool = True,
|
||||
**_unused,
|
||||
) -> List[Dict[str, Any]]:
|
||||
del chunk_combination_method, silence_between_chunks_ms
|
||||
|
||||
self._check_interrupt()
|
||||
self._setup_character_parser(text)
|
||||
base_config = self.config.copy()
|
||||
segment_objects = character_parser.parse_text_segments(text)
|
||||
if not segment_objects:
|
||||
segment_objects = character_parser.parse_text_segments(
|
||||
"narrator " + (text or "")
|
||||
)
|
||||
|
||||
characters = list(
|
||||
{segment.character for segment in segment_objects if segment.character}
|
||||
)
|
||||
discovered_mapping = get_character_mapping(characters, engine_type="audio_only")
|
||||
segment_records: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
for segment_index, segment in enumerate(segment_objects):
|
||||
character = segment.character or "narrator"
|
||||
self._check_interrupt(
|
||||
f"before segment {segment_index + 1}/{len(segment_objects)} "
|
||||
f"for '{character}'"
|
||||
)
|
||||
segment_text = (segment.text or "").strip()
|
||||
if not segment_text:
|
||||
continue
|
||||
|
||||
segment_params = segment.parameters or {}
|
||||
filtered_params = ParameterValidator.filter_parameters_for_engine(
|
||||
segment_params, "audio8_tts"
|
||||
)
|
||||
current_config = (
|
||||
apply_segment_parameters(base_config, filtered_params, "audio8_tts")
|
||||
if filtered_params
|
||||
else base_config.copy()
|
||||
)
|
||||
current_config = self._validate_segment_config(
|
||||
current_config, filtered_params
|
||||
)
|
||||
current_seed = int(current_config.get("seed", seed))
|
||||
parameter_log = self._format_parameter_log(
|
||||
filtered_params, current_config, current_seed
|
||||
)
|
||||
self.adapter.update_config(current_config)
|
||||
|
||||
voice_ref = self._resolve_voice(
|
||||
character, voice_mapping, discovered_mapping
|
||||
)
|
||||
seed_offset = 0
|
||||
|
||||
def generate_chunks(text_content: str, edit_tags: list) -> None:
|
||||
nonlocal seed_offset
|
||||
clean_content = (text_content or "").strip()
|
||||
if not clean_content:
|
||||
return
|
||||
|
||||
if enable_chunking:
|
||||
from utils.text.chunking import ImprovedChatterBoxChunker
|
||||
|
||||
max_chars = ImprovedChatterBoxChunker.validate_chunking_params(
|
||||
max_chars_per_chunk
|
||||
)
|
||||
chunks = ImprovedChatterBoxChunker.split_into_chunks(
|
||||
clean_content, max_chars=max_chars
|
||||
)
|
||||
else:
|
||||
chunks = [clean_content]
|
||||
chunks = [chunk.strip() for chunk in chunks if chunk.strip()]
|
||||
|
||||
self._log_generation(
|
||||
character=character,
|
||||
text=clean_content,
|
||||
voice_ref=voice_ref,
|
||||
config=current_config,
|
||||
chunk_count=len(chunks),
|
||||
parameter_log=parameter_log,
|
||||
show_text_logging=show_text_logging,
|
||||
)
|
||||
|
||||
for chunk_index, chunk in enumerate(chunks):
|
||||
self._check_interrupt(
|
||||
f"before chunk {chunk_index + 1}/{len(chunks)} "
|
||||
f"for '{character}'"
|
||||
)
|
||||
chunk_seed = current_seed + seed_offset
|
||||
seed_offset += 1
|
||||
audio = self.adapter.generate_single(
|
||||
text=chunk,
|
||||
voice_ref=voice_ref,
|
||||
seed=chunk_seed,
|
||||
enable_audio_cache=enable_audio_cache,
|
||||
character_name=character,
|
||||
)
|
||||
segment_records.append(
|
||||
{
|
||||
"waveform": self._normalize_audio(audio),
|
||||
"sample_rate": self.SAMPLE_RATE,
|
||||
"text": chunk,
|
||||
"edit_tags": (edit_tags if chunk_index == 0 else []),
|
||||
}
|
||||
)
|
||||
|
||||
if PauseTagProcessor.has_pause_tags(segment_text):
|
||||
pause_segments, _ = PauseTagProcessor.parse_pause_tags(segment_text)
|
||||
for fragment_type, fragment_content in pause_segments:
|
||||
self._check_interrupt(
|
||||
f"while processing segment {segment_index + 1}"
|
||||
)
|
||||
if fragment_type == "text":
|
||||
clean_text, edit_tags = get_edit_tags_for_segment(
|
||||
fragment_content
|
||||
)
|
||||
generate_chunks(clean_text, edit_tags)
|
||||
elif fragment_type == "pause":
|
||||
silence = PauseTagProcessor.create_silence_segment(
|
||||
fragment_content,
|
||||
self.SAMPLE_RATE,
|
||||
torch.device("cpu"),
|
||||
torch.float32,
|
||||
)
|
||||
if silence.dim() == 1:
|
||||
silence = silence.unsqueeze(0)
|
||||
segment_records.append(
|
||||
{
|
||||
"waveform": silence.cpu(),
|
||||
"sample_rate": self.SAMPLE_RATE,
|
||||
"text": f"[pause:{fragment_content}s]",
|
||||
"edit_tags": [],
|
||||
}
|
||||
)
|
||||
else:
|
||||
clean_text, edit_tags = get_edit_tags_for_segment(segment_text)
|
||||
generate_chunks(clean_text, edit_tags)
|
||||
finally:
|
||||
self.adapter.update_config(base_config)
|
||||
|
||||
if (
|
||||
apply_edit_postprocessing
|
||||
and segment_records
|
||||
and any(record.get("edit_tags") for record in segment_records)
|
||||
):
|
||||
self._check_interrupt("before edit post-processing")
|
||||
segment_records = apply_edit_post_processing(
|
||||
segment_records, engine_config=base_config
|
||||
)
|
||||
for record in segment_records:
|
||||
record["waveform"] = self._normalize_audio(record["waveform"])
|
||||
|
||||
return segment_records
|
||||
|
||||
def combine_audio_segments(
|
||||
self,
|
||||
segments: List[Dict[str, Any]],
|
||||
method: str = "auto",
|
||||
silence_ms: int = 100,
|
||||
original_text: str = "",
|
||||
return_info: bool = False,
|
||||
) -> Union[torch.Tensor, Tuple[torch.Tensor, Dict[str, Any]]]:
|
||||
self._check_interrupt("before audio assembly")
|
||||
if not segments:
|
||||
empty = torch.zeros(0, dtype=torch.float32)
|
||||
return (empty, {}) if return_info else empty
|
||||
|
||||
text_chunks = [segment.get("text", "") for segment in segments]
|
||||
combined_audio, chunk_info = ChunkTimingHelper.combine_audio_with_timing(
|
||||
audio_segments=[segment["waveform"] for segment in segments],
|
||||
combination_method=method,
|
||||
silence_ms=silence_ms,
|
||||
crossfade_duration=0.1,
|
||||
sample_rate=self.SAMPLE_RATE,
|
||||
text_length=len(" ".join(text_chunks)),
|
||||
original_text=original_text,
|
||||
text_chunks=text_chunks,
|
||||
)
|
||||
return (combined_audio, chunk_info) if return_info else combined_audio
|
||||
@@ -0,0 +1,222 @@
|
||||
"""SRT orchestration for Audio8 TTS."""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
|
||||
import comfy.model_management as model_management
|
||||
import torch
|
||||
|
||||
from engines.adapters.audio8_tts_adapter import Audio8TTSEngineAdapter
|
||||
from utils.system.import_manager import import_manager
|
||||
from utils.timing.assembly import AudioAssemblyEngine
|
||||
from utils.timing.engine import TimingEngine
|
||||
from utils.timing.overlap_detection import SRTOverlapHandler
|
||||
from utils.timing.reporting import SRTReportGenerator
|
||||
|
||||
|
||||
class Audio8TTSSRTProcessor:
|
||||
"""Generate Audio8 speech per subtitle and apply shared timing modes."""
|
||||
|
||||
SAMPLE_RATE = 44100
|
||||
|
||||
def __init__(self, node_instance, config):
|
||||
self.node_instance = node_instance
|
||||
self.config = dict(config or {})
|
||||
self.adapter = Audio8TTSEngineAdapter(self.config)
|
||||
self._processor = None
|
||||
|
||||
success, modules, message = import_manager.import_srt_modules()
|
||||
if not success:
|
||||
raise ImportError(f"Audio8 TTS SRT unavailable: {message}")
|
||||
self.SRTParser = modules["SRTParser"]
|
||||
|
||||
@property
|
||||
def processor(self):
|
||||
if self._processor is None:
|
||||
processor_path = os.path.join(
|
||||
os.path.dirname(__file__), "audio8_tts_processor.py"
|
||||
)
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"audio8_tts_processor_module", processor_path
|
||||
)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
self._processor = module.Audio8TTSProcessor(self.adapter, self.config)
|
||||
return self._processor
|
||||
|
||||
def update_config(self, config):
|
||||
self.config = dict(config or {})
|
||||
self.adapter.update_config(self.config)
|
||||
if self._processor is not None:
|
||||
self._processor.update_config(self.config)
|
||||
|
||||
@staticmethod
|
||||
def _check_interrupt(index=None, total=None):
|
||||
if not model_management.interrupt_processing:
|
||||
return
|
||||
location = f" at subtitle {index + 1}/{total}" if index is not None else ""
|
||||
raise InterruptedError(f"Audio8 TTS SRT generation interrupted{location}")
|
||||
|
||||
def process_srt_content(
|
||||
self,
|
||||
srt_content,
|
||||
voice_mapping,
|
||||
seed,
|
||||
timing_mode,
|
||||
timing_params,
|
||||
enable_audio_cache=True,
|
||||
):
|
||||
self._check_interrupt()
|
||||
subtitles = self.SRTParser().parse_srt_content(srt_content, allow_overlaps=True)
|
||||
has_overlaps = SRTOverlapHandler.detect_overlaps(subtitles)
|
||||
active_mode, switched = SRTOverlapHandler.handle_smart_natural_fallback(
|
||||
timing_mode, has_overlaps, "Audio8 TTS SRT"
|
||||
)
|
||||
|
||||
audio_segments = []
|
||||
adjustments = []
|
||||
for index, subtitle in enumerate(subtitles):
|
||||
self._check_interrupt(index, len(subtitles))
|
||||
text = (subtitle.text or "").strip()
|
||||
if text:
|
||||
records = self.processor.process_text(
|
||||
text=text,
|
||||
voice_mapping=voice_mapping,
|
||||
seed=seed + index,
|
||||
enable_chunking=False,
|
||||
enable_audio_cache=enable_audio_cache,
|
||||
apply_edit_postprocessing=True,
|
||||
)
|
||||
audio, _ = self.processor.combine_audio_segments(
|
||||
records,
|
||||
method="auto",
|
||||
silence_ms=0,
|
||||
original_text=text,
|
||||
return_info=True,
|
||||
)
|
||||
if audio.dim() == 1:
|
||||
audio = audio.unsqueeze(0)
|
||||
else:
|
||||
audio = torch.zeros(1, int(subtitle.duration * self.SAMPLE_RATE))
|
||||
|
||||
audio = audio.detach().cpu().float()
|
||||
audio_segments.append(audio)
|
||||
natural_duration = audio.shape[-1] / self.SAMPLE_RATE
|
||||
target_duration = subtitle.duration
|
||||
stretch_factor = (
|
||||
target_duration / natural_duration if natural_duration else 1.0
|
||||
)
|
||||
adjustments.append(
|
||||
{
|
||||
"index": index,
|
||||
"segment_index": index,
|
||||
"sequence": subtitle.sequence,
|
||||
"natural_duration": natural_duration,
|
||||
"target_start": subtitle.start_time,
|
||||
"target_end": subtitle.end_time,
|
||||
"target_duration": target_duration,
|
||||
"start_time": subtitle.start_time,
|
||||
"end_time": subtitle.end_time,
|
||||
"stretch_factor": stretch_factor,
|
||||
"needs_stretching": (abs(stretch_factor - 1.0) > 0.05),
|
||||
"stretch_type": (
|
||||
"compress"
|
||||
if stretch_factor < 1.0
|
||||
else "expand"
|
||||
if stretch_factor > 1.0
|
||||
else "none"
|
||||
),
|
||||
"adjustment": natural_duration - target_duration,
|
||||
"adjusted_start": subtitle.start_time,
|
||||
"adjusted_end": subtitle.end_time,
|
||||
"adjusted_duration": natural_duration,
|
||||
}
|
||||
)
|
||||
|
||||
self._check_interrupt()
|
||||
final_audio, replacement, stretch_method = self._assemble(
|
||||
audio_segments, subtitles, active_mode, timing_params or {}
|
||||
)
|
||||
self._check_interrupt()
|
||||
if replacement is not None:
|
||||
adjustments = replacement
|
||||
|
||||
reporter = SRTReportGenerator()
|
||||
timing_report = reporter.generate_timing_report(
|
||||
subtitles,
|
||||
adjustments,
|
||||
active_mode,
|
||||
has_overlaps,
|
||||
switched,
|
||||
timing_mode if switched else None,
|
||||
stretch_method,
|
||||
)
|
||||
adjusted_srt = reporter.generate_adjusted_srt_string(
|
||||
subtitles, adjustments, active_mode
|
||||
)
|
||||
|
||||
final_audio = final_audio.detach().cpu().float()
|
||||
if final_audio.dim() == 1:
|
||||
final_audio = final_audio.unsqueeze(0).unsqueeze(0)
|
||||
elif final_audio.dim() == 2:
|
||||
final_audio = final_audio.unsqueeze(0)
|
||||
duration = final_audio.shape[-1] / self.SAMPLE_RATE
|
||||
generation_info = (
|
||||
f"Generated {duration:.1f}s Audio8 TTS SRT audio from "
|
||||
f"{len(subtitles)} subtitles using {active_mode} mode"
|
||||
)
|
||||
return (
|
||||
{"waveform": final_audio, "sample_rate": self.SAMPLE_RATE},
|
||||
generation_info,
|
||||
timing_report,
|
||||
adjusted_srt,
|
||||
)
|
||||
|
||||
def _assemble(self, audio_segments, subtitles, mode, params):
|
||||
if mode == "stretch_to_fit":
|
||||
from engines.chatterbox.audio_timing import TimedAudioAssembler
|
||||
|
||||
assembler = TimedAudioAssembler(self.SAMPLE_RATE)
|
||||
audio, method = assembler.assemble_timed_audio(
|
||||
audio_segments,
|
||||
[(subtitle.start_time, subtitle.end_time) for subtitle in subtitles],
|
||||
fade_duration=params.get("fade_for_StretchToFit", 0.01),
|
||||
)
|
||||
return audio, None, method
|
||||
|
||||
assembler = AudioAssemblyEngine(self.SAMPLE_RATE)
|
||||
if mode == "pad_with_silence":
|
||||
audio = assembler.assemble_with_overlaps(
|
||||
audio_segments, subtitles, torch.device("cpu")
|
||||
)
|
||||
return audio, None, None
|
||||
|
||||
timing = TimingEngine(self.SAMPLE_RATE)
|
||||
if mode == "concatenate":
|
||||
replacements = timing.calculate_concatenation_adjustments(
|
||||
audio_segments, subtitles
|
||||
)
|
||||
audio = assembler.assemble_concatenation(
|
||||
audio_segments,
|
||||
params.get("fade_for_StretchToFit", 0.01),
|
||||
)
|
||||
return audio, replacements, None
|
||||
|
||||
if mode != "smart_natural":
|
||||
raise ValueError(f"Unsupported Audio8 SRT timing mode: {mode}")
|
||||
replacements, processed = timing.calculate_smart_timing_adjustments(
|
||||
audio_segments,
|
||||
subtitles,
|
||||
params.get("timing_tolerance", 2.0),
|
||||
params.get("max_stretch_ratio", 1.0),
|
||||
params.get("min_stretch_ratio", 0.5),
|
||||
torch.device("cpu"),
|
||||
)
|
||||
audio = assembler.assemble_smart_natural(
|
||||
audio_segments,
|
||||
processed,
|
||||
replacements,
|
||||
subtitles,
|
||||
torch.device("cpu"),
|
||||
)
|
||||
return audio, replacements, None
|
||||
@@ -0,0 +1,230 @@
|
||||
"""Audio8 TTS Preview engine configuration node."""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
|
||||
current_dir = os.path.dirname(__file__)
|
||||
nodes_dir = os.path.dirname(current_dir)
|
||||
project_root = os.path.dirname(nodes_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
base_node_path = os.path.join(nodes_dir, "base", "base_node.py")
|
||||
base_spec = importlib.util.spec_from_file_location("base_node_module", base_node_path)
|
||||
base_module = importlib.util.module_from_spec(base_spec)
|
||||
sys.modules["base_node_module"] = base_module
|
||||
base_spec.loader.exec_module(base_module)
|
||||
BaseTTSNode = base_module.BaseTTSNode
|
||||
|
||||
|
||||
class Audio8TTSEngineNode(BaseTTSNode):
|
||||
"""Configure the official Audio8 TTS Preview checkpoint."""
|
||||
|
||||
DEFAULT_MODEL = "Audio8-TTS-Preview-0.6b"
|
||||
|
||||
@classmethod
|
||||
def NAME(cls):
|
||||
return "⚙️ Audio8 TTS Engine"
|
||||
|
||||
@classmethod
|
||||
def _get_model_options(cls):
|
||||
try:
|
||||
from engines.audio8_tts.downloader import Audio8TTSDownloader
|
||||
|
||||
options = Audio8TTSDownloader().get_available_models()
|
||||
return options or [cls.DEFAULT_MODEL]
|
||||
except Exception:
|
||||
return [cls.DEFAULT_MODEL]
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"model_variant": (
|
||||
cls._get_model_options(),
|
||||
{
|
||||
"default": cls.DEFAULT_MODEL,
|
||||
"tooltip": (
|
||||
"Official 0.6B Audio8 TTS Preview checkpoint. "
|
||||
"The canonical option downloads from "
|
||||
"Audio8/Audio8-TTS-Preview-0.6b; local: options are "
|
||||
"detected model folders.\n\n"
|
||||
"Preview scope: multilingual speech and zero-shot "
|
||||
"voice cloning, with 11 recommended languages. The "
|
||||
"model infers language from text and has no language "
|
||||
"selection control.\n\n"
|
||||
"Voice cloning requires reference audio and its exact "
|
||||
"matching transcript. Generation without a reference "
|
||||
"is supported. Audio8 has no instruction-conditioned "
|
||||
"voice-design mode. The suite runs it in the existing "
|
||||
"shared Transformers 4 runtime for correct cloning. "
|
||||
"Code and weights: Apache License 2.0."
|
||||
),
|
||||
},
|
||||
),
|
||||
"device": (
|
||||
["auto", "cuda", "cpu"],
|
||||
{
|
||||
"default": "auto",
|
||||
"tooltip": (
|
||||
"Execution device. CUDA is recommended. CPU inference "
|
||||
"is supported but slow and runs in float32."
|
||||
),
|
||||
},
|
||||
),
|
||||
"temperature": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.8,
|
||||
"min": 0.1,
|
||||
"max": 2.0,
|
||||
"step": 0.05,
|
||||
"tooltip": (
|
||||
"Official sampling temperature. Used only in Sampling "
|
||||
"mode; lower values are more conservative."
|
||||
),
|
||||
},
|
||||
),
|
||||
"top_p": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.95,
|
||||
"min": 0.01,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": (
|
||||
"Official nucleus-sampling cutoff. Used only in "
|
||||
"Sampling mode."
|
||||
),
|
||||
},
|
||||
),
|
||||
"top_k": (
|
||||
"INT",
|
||||
{
|
||||
"default": 50,
|
||||
"min": 1,
|
||||
"max": 100,
|
||||
"step": 1,
|
||||
"tooltip": (
|
||||
"Official top-k sampling limit. Used only in Sampling mode."
|
||||
),
|
||||
},
|
||||
),
|
||||
"max_new_tokens": (
|
||||
"INT",
|
||||
{
|
||||
"default": 1024,
|
||||
"min": 64,
|
||||
"max": 2048,
|
||||
"step": 64,
|
||||
"tooltip": (
|
||||
"Maximum acoustic frames for the first generation "
|
||||
"attempt. Audio8 emits about 21.5 frames per second. "
|
||||
"Larger budgets take more time and context memory."
|
||||
),
|
||||
},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
"retry_max_new_tokens": (
|
||||
"INT",
|
||||
{
|
||||
"default": 2048,
|
||||
"min": 64,
|
||||
"max": 2048,
|
||||
"step": 8,
|
||||
"tooltip": (
|
||||
"Second-attempt frame budget when the first attempt "
|
||||
"does not emit EOS. Must be at least max_new_tokens. "
|
||||
"Set it equal to max_new_tokens to disable the larger "
|
||||
"retry."
|
||||
),
|
||||
},
|
||||
),
|
||||
"dtype": (
|
||||
["auto", "bfloat16", "float16", "float32"],
|
||||
{
|
||||
"default": "auto",
|
||||
"tooltip": (
|
||||
"Model precision. Auto prefers bfloat16 on Ampere-or-"
|
||||
"newer CUDA GPUs, uses float16 on older CUDA GPUs, and "
|
||||
"float32 on CPU. CPU coerces reduced-precision choices "
|
||||
"to float32."
|
||||
),
|
||||
},
|
||||
),
|
||||
"sampling_mode": (
|
||||
["Sampling", "Greedy"],
|
||||
{
|
||||
"default": "Sampling",
|
||||
"tooltip": (
|
||||
"Official decoding mode. Sampling uses temperature, "
|
||||
"top_p, and top_k. Greedy is deterministic for a fixed "
|
||||
"prompt and ignores those sampling controls."
|
||||
),
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("TTS_ENGINE",)
|
||||
RETURN_NAMES = ("TTS_engine",)
|
||||
FUNCTION = "create_engine_config"
|
||||
CATEGORY = "TTS Audio Suite/⚙️ Engines"
|
||||
|
||||
def create_engine_config(
|
||||
self,
|
||||
model_variant: str,
|
||||
device: str,
|
||||
temperature: float,
|
||||
top_p: float,
|
||||
top_k: int,
|
||||
max_new_tokens: int,
|
||||
retry_max_new_tokens: int = 2048,
|
||||
dtype: str = "auto",
|
||||
sampling_mode: str = "Sampling",
|
||||
) -> tuple:
|
||||
if retry_max_new_tokens < max_new_tokens:
|
||||
raise ValueError(
|
||||
"Audio8 retry_max_new_tokens must be at least max_new_tokens"
|
||||
)
|
||||
|
||||
do_sample = sampling_mode == "Sampling"
|
||||
config = {
|
||||
"engine_type": "audio8_tts",
|
||||
"model_variant": model_variant,
|
||||
"device": device,
|
||||
"dtype": dtype,
|
||||
"max_new_tokens": int(max_new_tokens),
|
||||
"retry_max_new_tokens": int(retry_max_new_tokens),
|
||||
"temperature": float(temperature),
|
||||
"top_p": float(top_p),
|
||||
"top_k": int(top_k),
|
||||
"do_sample": do_sample,
|
||||
}
|
||||
|
||||
print(f"⚙️ Audio8 TTS Preview: {model_variant} on {device} ({dtype})")
|
||||
print(
|
||||
" Settings: "
|
||||
f"mode={sampling_mode}, temperature={temperature}, top_p={top_p}, "
|
||||
f"top_k={top_k}, max_new_tokens={max_new_tokens}, "
|
||||
f"retry_max_new_tokens={retry_max_new_tokens}"
|
||||
)
|
||||
print(
|
||||
" Voice cloning requires reference audio plus its exact matching "
|
||||
"transcript; no-reference generation is supported."
|
||||
)
|
||||
print(
|
||||
" Runtime: shared Transformers 4 profile (required for correct cloning)"
|
||||
)
|
||||
print(" Preview checkpoint | Apache License 2.0")
|
||||
|
||||
return (
|
||||
{
|
||||
"engine_type": "audio8_tts",
|
||||
"config": config,
|
||||
"adapter_class": "Audio8TTSEngineAdapter",
|
||||
"capabilities": ["tts"],
|
||||
},
|
||||
)
|
||||
@@ -1,245 +0,0 @@
|
||||
"""VoxCPM engine configuration node."""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
|
||||
|
||||
current_dir = os.path.dirname(__file__)
|
||||
nodes_dir = os.path.dirname(current_dir)
|
||||
project_root = os.path.dirname(nodes_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
base_node_path = os.path.join(nodes_dir, "base", "base_node.py")
|
||||
base_spec = importlib.util.spec_from_file_location("base_node_module", base_node_path)
|
||||
base_module = importlib.util.module_from_spec(base_spec)
|
||||
sys.modules["base_node_module"] = base_module
|
||||
base_spec.loader.exec_module(base_module)
|
||||
BaseTTSNode = base_module.BaseTTSNode
|
||||
|
||||
|
||||
class VoxCPMEngineNode(BaseTTSNode):
|
||||
"""Configure the official VoxCPM runtime for all released generations."""
|
||||
|
||||
DOWNLOADABLE_MODELS = ["VoxCPM2", "VoxCPM1.5", "VoxCPM-0.5B"]
|
||||
NO_LORA_OPTION = "None"
|
||||
|
||||
@classmethod
|
||||
def _get_lora_options(cls):
|
||||
try:
|
||||
from engines.voxcpm.training.common import discover_lora_adapters
|
||||
|
||||
return [cls.NO_LORA_OPTION] + discover_lora_adapters()
|
||||
except Exception:
|
||||
return [cls.NO_LORA_OPTION]
|
||||
|
||||
@classmethod
|
||||
def NAME(cls):
|
||||
return "⚙️ VoxCPM Engine"
|
||||
|
||||
@classmethod
|
||||
def _get_model_options(cls):
|
||||
try:
|
||||
from engines.voxcpm.voxcpm_downloader import VoxCPMDownloader
|
||||
|
||||
downloader = VoxCPMDownloader()
|
||||
if hasattr(downloader, "get_available_models"):
|
||||
return downloader.get_available_models()
|
||||
except Exception:
|
||||
pass
|
||||
return list(cls.DOWNLOADABLE_MODELS)
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"model_variant": (cls._get_model_options(), {
|
||||
"default": "VoxCPM2",
|
||||
"tooltip": (
|
||||
"Official OpenBMB checkpoint.\n"
|
||||
"• VoxCPM2: recommended; about 8 GB VRAM, 30 languages, "
|
||||
"reference-only cloning, controllable cloning, and Voice Design\n"
|
||||
"• VoxCPM1.5: about 6 GB VRAM, faster, English/Chinese cloning\n"
|
||||
"• VoxCPM-0.5B: legacy 16 kHz model; kept for lowest-memory compatibility\n"
|
||||
"Options prefixed with local: are complete checkpoints already on disk."
|
||||
),
|
||||
}),
|
||||
"device": (["auto", "cuda", "cpu", "mps"], {
|
||||
"default": "auto",
|
||||
"tooltip": (
|
||||
"Inference device. Auto selects the best available backend. "
|
||||
"CPU is very slow. The official runtime forces float32 on MPS "
|
||||
"because half precision degrades audio."
|
||||
),
|
||||
}),
|
||||
"cfg_value": ("FLOAT", {
|
||||
"default": 2.0,
|
||||
"min": 0.1,
|
||||
"max": 10.0,
|
||||
"step": 0.1,
|
||||
"tooltip": (
|
||||
"Official classifier-free guidance value. OpenBMB uses 2.0. "
|
||||
"Higher is not automatically better and can make speech less natural."
|
||||
),
|
||||
}),
|
||||
"inference_timesteps": ("INT", {
|
||||
"default": 10,
|
||||
"min": 1,
|
||||
"max": 100,
|
||||
"step": 1,
|
||||
"tooltip": (
|
||||
"Official diffusion sampling steps. 10 is the quality/speed default; "
|
||||
"more steps cost proportionally more time."
|
||||
),
|
||||
}),
|
||||
"max_len": ("INT", {
|
||||
"default": 4096,
|
||||
"min": 64,
|
||||
"max": 8192,
|
||||
"step": 64,
|
||||
"tooltip": (
|
||||
"Maximum generated audio-token length per suite chunk. VoxCPM2 and "
|
||||
"1.5 support up to 8192; the legacy 0.5B checkpoint supports 4096."
|
||||
),
|
||||
}),
|
||||
},
|
||||
"optional": {
|
||||
"local_lora_adapter": (cls._get_lora_options(), {
|
||||
"default": cls.NO_LORA_OPTION,
|
||||
"tooltip": (
|
||||
"Optional VoxCPM2 LoRA under models/TTS/voxcpm/loras. "
|
||||
"Legacy VoxCPM models do not accept these adapters."
|
||||
),
|
||||
}),
|
||||
"lora_adapter_override": ("STRING", {
|
||||
"default": "",
|
||||
"tooltip": "Optional absolute VoxCPM LoRA folder path. Overrides the dropdown.",
|
||||
}),
|
||||
"voice_instruction": ("STRING", {
|
||||
"default": "",
|
||||
"multiline": True,
|
||||
"tooltip": (
|
||||
"VoxCPM2 only. Describe age, timbre, accent, emotion, or delivery. "
|
||||
"Without reference audio it designs a voice; with reference audio it "
|
||||
"performs controllable cloning. Legacy models reject this input."
|
||||
),
|
||||
}),
|
||||
"normalize_text": ("BOOLEAN", {
|
||||
"default": False,
|
||||
"tooltip": "Run the official VoxCPM text normalizer before inference.",
|
||||
}),
|
||||
"retry_badcase": ("BOOLEAN", {
|
||||
"default": True,
|
||||
"tooltip": (
|
||||
"Retry generations whose audio-to-text length ratio suggests a bad case. "
|
||||
"This is an official VoxCPM safeguard and can make a render take longer."
|
||||
),
|
||||
}),
|
||||
"retry_badcase_max_times": ("INT", {
|
||||
"default": 3,
|
||||
"min": 1,
|
||||
"max": 10,
|
||||
"step": 1,
|
||||
"tooltip": "Maximum official bad-case generation attempts.",
|
||||
}),
|
||||
"retry_badcase_ratio_threshold": ("FLOAT", {
|
||||
"default": 6.0,
|
||||
"min": 1.0,
|
||||
"max": 20.0,
|
||||
"step": 0.5,
|
||||
"tooltip": "Official audio-to-text length-ratio threshold used for bad-case retries.",
|
||||
}),
|
||||
"optimize": ("BOOLEAN", {
|
||||
"default": False,
|
||||
"tooltip": (
|
||||
"Enable the official torch.compile optimization on CUDA. First load and "
|
||||
"generation become slower; leave disabled for CPU, MPS, and easier unloading."
|
||||
),
|
||||
}),
|
||||
"mode": (["Text to Speech", "Voice Design"], {
|
||||
"default": "Text to Speech",
|
||||
"tooltip": (
|
||||
"Text to Speech works with unified TTS Text/SRT. Voice Design is supported "
|
||||
"only by VoxCPM2 and works with the Voice Designer node. Duplicate this engine "
|
||||
"node if one workflow needs both roles."
|
||||
),
|
||||
}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("TTS_ENGINE",)
|
||||
RETURN_NAMES = ("TTS_engine",)
|
||||
FUNCTION = "create_engine_config"
|
||||
CATEGORY = "TTS Audio Suite/⚙️ Engines"
|
||||
|
||||
def create_engine_config(
|
||||
self,
|
||||
model_variant,
|
||||
device,
|
||||
cfg_value,
|
||||
inference_timesteps,
|
||||
max_len,
|
||||
local_lora_adapter="None",
|
||||
lora_adapter_override="",
|
||||
voice_instruction="",
|
||||
normalize_text=False,
|
||||
retry_badcase=True,
|
||||
retry_badcase_max_times=3,
|
||||
retry_badcase_ratio_threshold=6.0,
|
||||
optimize=False,
|
||||
mode="Text to Speech",
|
||||
):
|
||||
selected_lora = str(lora_adapter_override or "").strip()
|
||||
if not selected_lora and str(local_lora_adapter) != self.NO_LORA_OPTION:
|
||||
selected_lora = str(local_lora_adapter).strip()
|
||||
if selected_lora:
|
||||
selected_lora = os.path.abspath(os.path.expanduser(selected_lora))
|
||||
from engines.voxcpm.training.common import read_lora_info
|
||||
|
||||
read_lora_info(selected_lora)
|
||||
if model_variant in {"VoxCPM1.5", "VoxCPM-0.5B"}:
|
||||
raise ValueError("VoxCPM LoRA adapters currently require VoxCPM2")
|
||||
|
||||
if model_variant == "VoxCPM-0.5B" and int(max_len) > 4096:
|
||||
max_len = 4096
|
||||
|
||||
model_role = "voice_design" if mode == "Voice Design" else "tts"
|
||||
if model_role == "voice_design" and model_variant in {"VoxCPM1.5", "VoxCPM-0.5B"}:
|
||||
raise ValueError(
|
||||
f"{model_variant} cannot design voices. Select VoxCPM2 or a local VoxCPM2 checkpoint."
|
||||
)
|
||||
|
||||
config = {
|
||||
"engine_type": "voxcpm",
|
||||
"model_variant": str(model_variant),
|
||||
"model_name": str(model_variant),
|
||||
"model_role": model_role,
|
||||
"mode": str(mode),
|
||||
"device": str(device),
|
||||
"cfg_value": float(cfg_value),
|
||||
"inference_timesteps": int(inference_timesteps),
|
||||
"min_len": 2,
|
||||
"max_len": int(max_len),
|
||||
"voice_instruction": str(voice_instruction or "").strip(),
|
||||
"normalize_text": bool(normalize_text),
|
||||
"retry_badcase": bool(retry_badcase),
|
||||
"retry_badcase_max_times": int(retry_badcase_max_times),
|
||||
"retry_badcase_ratio_threshold": float(retry_badcase_ratio_threshold),
|
||||
"optimize": bool(optimize),
|
||||
"runtime_mode": "main_environment",
|
||||
"lora_adapter": selected_lora or None,
|
||||
}
|
||||
|
||||
print(
|
||||
f"⚙️ VoxCPM: {model_variant} on {device} | {mode} | "
|
||||
f"cfg={float(cfg_value):g}, steps={int(inference_timesteps)}, max_len={int(max_len)}"
|
||||
)
|
||||
if selected_lora:
|
||||
print(f" LoRA adapter: {selected_lora}")
|
||||
return ({
|
||||
"engine_type": "voxcpm",
|
||||
"config": config,
|
||||
"adapter_class": "VoxCPMEngineAdapter",
|
||||
"capabilities": [model_role],
|
||||
},)
|
||||
@@ -69,10 +69,11 @@ PRIORITY SYSTEM - When both .txt and .reference.txt exist:
|
||||
"default": "",
|
||||
"tooltip": """Create reference text on-the-fly for connected audio input.
|
||||
|
||||
ENGINE REQUIREMENTS:
|
||||
• F5-TTS: REQUIRES reference text (must match spoken audio exactly)
|
||||
• Higgs Audio 2: Optional but uses reference text if provided
|
||||
• ChatterBox/VibeVoice/IndexTTS: Don't use reference text
|
||||
ENGINE REQUIREMENTS:
|
||||
• Audio8 TTS: REQUIRES the exact spoken transcript for voice cloning
|
||||
• F5-TTS: REQUIRES reference text (must match spoken audio exactly)
|
||||
• Higgs Audio 2: Optional but uses reference text if provided
|
||||
• ChatterBox/VibeVoice/IndexTTS: Don't use reference text
|
||||
|
||||
Selecting a library voice loads its transcription here automatically. Edits are temporary workflow overrides and never modify the source .txt file."""
|
||||
}),
|
||||
|
||||
@@ -10,9 +10,9 @@ class UnifiedVoiceDesignerNode:
|
||||
"required": {
|
||||
"TTS_engine": ("TTS_ENGINE", {
|
||||
"tooltip": (
|
||||
"Configured Qwen3-TTS, MOSS-TTS, OmniVoice, or VoxCPM2 engine. This node owns the voice-design "
|
||||
"Configured Qwen3-TTS, MOSS-TTS, or OmniVoice engine. This node owns the voice-design "
|
||||
"instruction; the engine owns its model, language, and generation settings. Qwen and "
|
||||
"MOSS require their voice-design model, while OmniVoice and VoxCPM2 require Voice Design mode."
|
||||
"MOSS require their voice-design model, while OmniVoice requires Voice Design mode."
|
||||
)
|
||||
}),
|
||||
"reference_text": ("STRING", {
|
||||
|
||||
@@ -208,13 +208,20 @@ class StepAudioEditXAudioEditorNode:
|
||||
self._cached_settings = None
|
||||
|
||||
# Check if engine was deleted (MUST be after settings change check, before early return)
|
||||
engine_was_deleted = (self._engine is not None and
|
||||
hasattr(self._engine, '_tts_engine') and
|
||||
self._engine._tts_engine is None)
|
||||
|
||||
if engine_was_deleted:
|
||||
print("⚠️ Step Audio EditX engine was deleted, reloading...")
|
||||
self._engine = None
|
||||
engine_was_deleted = self._engine is not None and (
|
||||
(
|
||||
hasattr(self._engine, '_tts_engine')
|
||||
and self._engine._tts_engine is None
|
||||
)
|
||||
or (
|
||||
hasattr(self._engine, '_initialized')
|
||||
and not self._engine._initialized
|
||||
)
|
||||
)
|
||||
|
||||
if engine_was_deleted:
|
||||
print("⚠️ Step Audio EditX engine was unloaded, reloading...")
|
||||
self._engine = None
|
||||
elif self._engine is not None:
|
||||
# Using cached engine - ensure it's on the correct device
|
||||
from utils.device import resolve_torch_device
|
||||
|
||||
@@ -1,257 +0,0 @@
|
||||
"""Build and validate VoxCPM2 audio/text training datasets."""
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
import folder_paths
|
||||
import numpy as np
|
||||
from scipy.io import wavfile
|
||||
from scipy.signal import resample_poly
|
||||
|
||||
from engines.training.registry import get_training_handler
|
||||
|
||||
|
||||
current_dir = os.path.dirname(__file__)
|
||||
nodes_dir = os.path.dirname(current_dir)
|
||||
project_root = os.path.dirname(nodes_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
base_node_path = os.path.join(nodes_dir, "base", "base_node.py")
|
||||
base_spec = importlib.util.spec_from_file_location("base_node_module", base_node_path)
|
||||
base_module = importlib.util.module_from_spec(base_spec)
|
||||
sys.modules["base_node_module"] = base_module
|
||||
base_spec.loader.exec_module(base_module)
|
||||
BaseTTSNode = base_module.BaseTTSNode
|
||||
|
||||
|
||||
_AUDIO_KEY = re.compile(r"^opt_audio(\d+)$")
|
||||
|
||||
|
||||
class _DynamicAudioInputs(dict):
|
||||
def __contains__(self, key):
|
||||
return super().__contains__(key) or (
|
||||
isinstance(key, str) and _AUDIO_KEY.fullmatch(key) is not None
|
||||
)
|
||||
|
||||
def __getitem__(self, key):
|
||||
if super().__contains__(key):
|
||||
return super().__getitem__(key)
|
||||
if isinstance(key, str) and _AUDIO_KEY.fullmatch(key):
|
||||
return ("AUDIO", {"forceInput": True})
|
||||
raise KeyError(key)
|
||||
|
||||
def get(self, key, default=None):
|
||||
try:
|
||||
return self[key]
|
||||
except KeyError:
|
||||
return default
|
||||
|
||||
|
||||
def _slugify(value):
|
||||
safe = "".join(
|
||||
char if char.isalnum() or char in ("-", "_") else "_"
|
||||
for char in str(value or "").strip()
|
||||
).strip("_")
|
||||
return safe or "voxcpm2_dataset"
|
||||
|
||||
|
||||
def _iter_clips(waveform):
|
||||
if waveform.ndim == 1:
|
||||
yield waveform
|
||||
elif waveform.ndim == 2:
|
||||
yield waveform
|
||||
elif waveform.ndim == 3:
|
||||
yield from waveform
|
||||
else:
|
||||
raise ValueError(f"Unsupported VoxCPM training audio shape: {tuple(waveform.shape)}")
|
||||
|
||||
|
||||
def _stage_clip(audio, sample_rate, path):
|
||||
tensor = audio.detach().cpu().float()
|
||||
if tensor.ndim == 2:
|
||||
tensor = tensor.mean(dim=0)
|
||||
elif tensor.ndim != 1:
|
||||
raise ValueError(f"Expected one VoxCPM audio clip, got {tuple(tensor.shape)}")
|
||||
values = tensor.clamp(-1, 1).numpy()
|
||||
if int(sample_rate) != 16000:
|
||||
divisor = math.gcd(int(sample_rate), 16000)
|
||||
values = resample_poly(
|
||||
values,
|
||||
16000 // divisor,
|
||||
int(sample_rate) // divisor,
|
||||
)
|
||||
if len(values) < 800:
|
||||
raise ValueError("VoxCPM training clips must be at least 0.05 seconds")
|
||||
pcm16 = np.round(np.clip(values, -1, 1) * 32767).astype(np.int16)
|
||||
wavfile.write(path, 16000, pcm16)
|
||||
return len(pcm16) / 16000.0
|
||||
|
||||
|
||||
class VoxCPMDatasetPrepNode(BaseTTSNode):
|
||||
@classmethod
|
||||
def NAME(cls):
|
||||
return "📦 VoxCPM2 Dataset Prep"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
optional = _DynamicAudioInputs(
|
||||
{
|
||||
"dataset_source": ("STRING", {
|
||||
"default": "",
|
||||
"tooltip": (
|
||||
"Existing official-format JSONL manifest. Each line needs "
|
||||
'{"audio":"clip.wav","text":"transcript"}. Leave blank to use connected AUDIO inputs.'
|
||||
),
|
||||
}),
|
||||
"text_lines": ("STRING", {
|
||||
"default": "",
|
||||
"multiline": True,
|
||||
"tooltip": (
|
||||
"When AUDIO inputs are connected, provide exactly one transcript "
|
||||
"line per resulting clip, in input/batch order."
|
||||
),
|
||||
}),
|
||||
"validation_source": ("STRING", {
|
||||
"default": "",
|
||||
"tooltip": "Optional separate official-format validation JSONL.",
|
||||
}),
|
||||
"validation_split": ("FLOAT", {
|
||||
"default": 0.05,
|
||||
"min": 0.0,
|
||||
"max": 0.5,
|
||||
"step": 0.01,
|
||||
"tooltip": "Automatic holdout ratio. Set 0 to train without validation.",
|
||||
}),
|
||||
"split_seed": ("INT", {
|
||||
"default": 42,
|
||||
"min": 0,
|
||||
"max": 2**31 - 1,
|
||||
"step": 1,
|
||||
}),
|
||||
"reuse_existing": ("BOOLEAN", {
|
||||
"default": True,
|
||||
"tooltip": "Reuse matching validated manifests.",
|
||||
}),
|
||||
"opt_audio1": ("AUDIO", {
|
||||
"forceInput": True,
|
||||
"tooltip": "Optional training audio. More AUDIO sockets are added dynamically.",
|
||||
}),
|
||||
}
|
||||
)
|
||||
return {
|
||||
"required": {
|
||||
"TTS_engine": ("TTS_ENGINE", {
|
||||
"tooltip": "Connect a VoxCPM engine configured with VoxCPM2.",
|
||||
}),
|
||||
"model_name": ("STRING", {
|
||||
"default": "MyVoxCPM2LoRA",
|
||||
"tooltip": "Dataset and final adapter name.",
|
||||
}),
|
||||
},
|
||||
"optional": optional,
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("TRAINING_DATASET", "STRING")
|
||||
RETURN_NAMES = ("training_dataset", "dataset_info")
|
||||
FUNCTION = "prepare_dataset"
|
||||
CATEGORY = "TTS Audio Suite/🎓 Training"
|
||||
|
||||
def _collect_audio(self, opt_audio1=None, **kwargs):
|
||||
values = []
|
||||
if opt_audio1 is not None:
|
||||
values.append(("opt_audio1", opt_audio1))
|
||||
for key, value in kwargs.items():
|
||||
if value is not None and _AUDIO_KEY.fullmatch(str(key)):
|
||||
values.append((key, value))
|
||||
values.sort(key=lambda item: int(_AUDIO_KEY.fullmatch(item[0]).group(1)))
|
||||
return values
|
||||
|
||||
def _build_manifest(self, model_name, text_lines, audio_inputs):
|
||||
transcripts = str(text_lines or "").splitlines()
|
||||
normalized = []
|
||||
for input_name, value in audio_inputs:
|
||||
audio = self.normalize_audio_input(value, input_name=input_name)
|
||||
for clip in _iter_clips(audio["waveform"]):
|
||||
normalized.append((clip, int(audio["sample_rate"])))
|
||||
if len(transcripts) != len(normalized):
|
||||
raise ValueError(
|
||||
f"VoxCPM transcript count mismatch: {len(normalized)} audio clip(s), "
|
||||
f"{len(transcripts)} text line(s)."
|
||||
)
|
||||
if any(not line.strip() for line in transcripts):
|
||||
raise ValueError("VoxCPM training transcripts cannot be blank")
|
||||
|
||||
root = os.path.join(
|
||||
folder_paths.get_input_directory(),
|
||||
"tts_audio_suite_training",
|
||||
"voxcpm",
|
||||
_slugify(model_name),
|
||||
)
|
||||
os.makedirs(root, exist_ok=True)
|
||||
records = []
|
||||
for index, ((clip, sample_rate), text) in enumerate(
|
||||
zip(normalized, transcripts), 1
|
||||
):
|
||||
path = os.path.join(root, f"{index:05d}.wav")
|
||||
duration = _stage_clip(clip, sample_rate, path)
|
||||
records.append(
|
||||
{"audio": path, "text": text.strip(), "duration": duration}
|
||||
)
|
||||
manifest = os.path.join(root, "manifest.jsonl")
|
||||
with open(manifest, "w", encoding="utf-8") as handle:
|
||||
for record in records:
|
||||
handle.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
return manifest
|
||||
|
||||
def prepare_dataset(
|
||||
self,
|
||||
TTS_engine,
|
||||
model_name,
|
||||
dataset_source="",
|
||||
text_lines="",
|
||||
validation_source="",
|
||||
validation_split=0.05,
|
||||
split_seed=42,
|
||||
reuse_existing=True,
|
||||
opt_audio1=None,
|
||||
**kwargs,
|
||||
):
|
||||
audio_inputs = self._collect_audio(opt_audio1=opt_audio1, **kwargs)
|
||||
source = str(dataset_source or "").strip()
|
||||
if audio_inputs:
|
||||
if source:
|
||||
raise ValueError(
|
||||
"Use either dataset_source or connected AUDIO inputs, not both"
|
||||
)
|
||||
source = self._build_manifest(model_name, text_lines, audio_inputs)
|
||||
if not source:
|
||||
raise ValueError(
|
||||
"VoxCPM Dataset Prep needs dataset_source or at least one AUDIO input"
|
||||
)
|
||||
|
||||
handler = get_training_handler("voxcpm")
|
||||
dataset = handler.prepare_dataset(
|
||||
TTS_engine,
|
||||
dataset_source=source,
|
||||
model_name=model_name,
|
||||
validation_source=validation_source,
|
||||
validation_split=validation_split,
|
||||
split_seed=split_seed,
|
||||
reuse_existing=reuse_existing,
|
||||
)
|
||||
info = (
|
||||
f"VoxCPM2 dataset ready: {dataset['model_name']} | "
|
||||
f"train={dataset['train_records']} | val={dataset['val_records']} | "
|
||||
"AudioVAE input=16 kHz"
|
||||
)
|
||||
return dataset, info
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {"VoxCPMDatasetPrepNode": VoxCPMDatasetPrepNode}
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"VoxCPMDatasetPrepNode": "📦 VoxCPM2 Dataset Prep"
|
||||
}
|
||||
@@ -1,134 +0,0 @@
|
||||
"""Official VoxCPM2 LoRA training settings."""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
|
||||
|
||||
current_dir = os.path.dirname(__file__)
|
||||
nodes_dir = os.path.dirname(current_dir)
|
||||
project_root = os.path.dirname(nodes_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
base_node_path = os.path.join(nodes_dir, "base", "base_node.py")
|
||||
base_spec = importlib.util.spec_from_file_location("base_node_module", base_node_path)
|
||||
base_module = importlib.util.module_from_spec(base_spec)
|
||||
sys.modules["base_node_module"] = base_module
|
||||
base_spec.loader.exec_module(base_module)
|
||||
BaseTTSNode = base_module.BaseTTSNode
|
||||
|
||||
|
||||
class VoxCPMTrainingConfigNode(BaseTTSNode):
|
||||
@classmethod
|
||||
def NAME(cls):
|
||||
return "🎛️ VoxCPM2 LoRA Config"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"max_train_steps": ("INT", {
|
||||
"default": 1000, "min": 1, "max": 1000000, "step": 100,
|
||||
"tooltip": "Official trainer iteration count. Use 1-5 first for a VRAM smoke test.",
|
||||
}),
|
||||
"batch_size": ("INT", {
|
||||
"default": 1, "min": 1, "max": 32, "step": 1,
|
||||
"tooltip": "Per-device batch size. Keep 1 until measured on your GPU.",
|
||||
}),
|
||||
"gradient_accumulation_steps": ("INT", {
|
||||
"default": 8, "min": 1, "max": 256, "step": 1,
|
||||
}),
|
||||
"learning_rate": ("FLOAT", {
|
||||
"default": 0.0001, "min": 0.00000001, "max": 1.0, "step": 0.00000001,
|
||||
"tooltip": "Official VoxCPM LoRA default is 1e-4.",
|
||||
}),
|
||||
"lora_r": ("INT", {
|
||||
"default": 32, "min": 1, "max": 512, "step": 1,
|
||||
}),
|
||||
"lora_alpha": ("INT", {
|
||||
"default": 32, "min": 1, "max": 1024, "step": 1,
|
||||
}),
|
||||
},
|
||||
"optional": {
|
||||
"weight_decay": ("FLOAT", {
|
||||
"default": 0.01, "min": 0.0, "max": 1.0, "step": 0.001,
|
||||
}),
|
||||
"warmup_steps": ("INT", {
|
||||
"default": 100, "min": 0, "max": 100000, "step": 1,
|
||||
}),
|
||||
"max_grad_norm": ("FLOAT", {
|
||||
"default": 1.0, "min": 0.0, "max": 100.0, "step": 0.1,
|
||||
}),
|
||||
"max_batch_tokens": ("INT", {
|
||||
"default": 8192, "min": 0, "max": 131072, "step": 256,
|
||||
"tooltip": "Filters individual samples that would exceed this packed-token budget. 0 disables filtering.",
|
||||
}),
|
||||
"num_workers": ("INT", {
|
||||
"default": 0, "min": 0, "max": 16, "step": 1,
|
||||
"tooltip": "0 is the safest Windows setting.",
|
||||
}),
|
||||
"cuda_device_index": ("INT", {
|
||||
"default": -1, "min": -1, "max": 31, "step": 1,
|
||||
"tooltip": (
|
||||
"-1 selects the CUDA GPU with the most free VRAM. "
|
||||
"Use 0, 1, etc. to force a specific CUDA device."
|
||||
),
|
||||
}),
|
||||
"save_steps": ("INT", {
|
||||
"default": 500, "min": 0, "max": 100000, "step": 10,
|
||||
}),
|
||||
"eval_steps": ("INT", {
|
||||
"default": 500, "min": 0, "max": 100000, "step": 10,
|
||||
"tooltip": "0 disables validation passes.",
|
||||
}),
|
||||
"log_steps": ("INT", {
|
||||
"default": 10, "min": 1, "max": 10000, "step": 1,
|
||||
}),
|
||||
"lora_dropout": ("FLOAT", {
|
||||
"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01,
|
||||
}),
|
||||
"lora_enable_lm": ("BOOLEAN", {
|
||||
"default": True,
|
||||
"tooltip": "Official default: adapt the language-model blocks.",
|
||||
}),
|
||||
"lora_enable_dit": ("BOOLEAN", {
|
||||
"default": True,
|
||||
"tooltip": "Official default: adapt the diffusion-transformer blocks.",
|
||||
}),
|
||||
"lora_enable_proj": ("BOOLEAN", {
|
||||
"default": False,
|
||||
"tooltip": "Official default is off. Enable only for deliberate experiments.",
|
||||
}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("TRAINING_CONFIG", "STRING")
|
||||
RETURN_NAMES = ("training_config", "config_info")
|
||||
FUNCTION = "create_config"
|
||||
CATEGORY = "TTS Audio Suite/🎓 Training"
|
||||
|
||||
def create_config(self, **kwargs):
|
||||
config = {
|
||||
"type": "training_config",
|
||||
"engine_type": "voxcpm",
|
||||
"training_mode": "lora_adapter",
|
||||
**kwargs,
|
||||
}
|
||||
if not (
|
||||
config.get("lora_enable_lm")
|
||||
or config.get("lora_enable_dit")
|
||||
or config.get("lora_enable_proj")
|
||||
):
|
||||
raise ValueError("At least one VoxCPM LoRA module group must be enabled")
|
||||
info = (
|
||||
f"VoxCPM2 LoRA: steps={config['max_train_steps']} | "
|
||||
f"batch={config['batch_size']}x{config['gradient_accumulation_steps']} | "
|
||||
f"r={config['lora_r']}, alpha={config['lora_alpha']}"
|
||||
)
|
||||
return config, info
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {"VoxCPMTrainingConfigNode": VoxCPMTrainingConfigNode}
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"VoxCPMTrainingConfigNode": "🎛️ VoxCPM2 LoRA Config"
|
||||
}
|
||||
@@ -84,7 +84,7 @@ Hello! This is unified SRT TTS with character switching.
|
||||
}),
|
||||
"narrator_voice": (reference_files, {
|
||||
"default": "none",
|
||||
"tooltip": "Fallback narrator voice from voice folders. Used when opt_narrator is not connected. Select 'none' for engines that support direct TTS without voice cloning, such as MOSS."
|
||||
"tooltip": "Fallback narrator voice from voice folders. Used when opt_narrator is not connected. Select 'none' for engines that support direct TTS without voice cloning, such as MOSS or Audio8."
|
||||
}),
|
||||
"seed": ("INT", {
|
||||
"default": 1, "min": 0, "max": 2**32 - 1,
|
||||
@@ -223,6 +223,12 @@ Hello! This is unified SRT TTS with character switching.
|
||||
stable_params['optimize'] = config.get('optimize', False)
|
||||
stable_params['max_generate_length'] = config.get('max_generate_length', 500)
|
||||
|
||||
if engine_type == "audio8_tts":
|
||||
stable_params['model_variant'] = config.get(
|
||||
'model_variant', 'Audio8-TTS-Preview-0.6b'
|
||||
)
|
||||
stable_params['dtype'] = config.get('dtype', 'auto')
|
||||
|
||||
if engine_type == "dramabox":
|
||||
stable_params['model_name'] = config.get('model_name', 'DramaBox')
|
||||
stable_params['precision'] = config.get('precision', 'auto')
|
||||
@@ -242,11 +248,6 @@ Hello! This is unified SRT TTS with character switching.
|
||||
stable_params['model_variant'] = config.get('model_variant', 'OmniVoice')
|
||||
stable_params['dtype'] = config.get('dtype', 'auto')
|
||||
|
||||
if engine_type == "voxcpm":
|
||||
stable_params['model_variant'] = config.get('model_variant', 'VoxCPM2')
|
||||
stable_params['model_name'] = config.get('model_name', 'VoxCPM2')
|
||||
stable_params['optimize'] = config.get('optimize', False)
|
||||
|
||||
if engine_type == "moss_tts":
|
||||
stable_params['model_variant'] = config.get('model_variant', 'MOSS-TTS-Local-Transformer')
|
||||
stable_params['multi_speaker_mode'] = config.get('multi_speaker_mode', 'Custom Character Switching')
|
||||
@@ -583,6 +584,40 @@ Hello! This is unified SRT TTS with character switching.
|
||||
}
|
||||
return engine_instance
|
||||
|
||||
elif engine_type == "audio8_tts":
|
||||
processor_path = os.path.join(
|
||||
nodes_dir, "audio8_tts", "audio8_tts_srt_processor.py"
|
||||
)
|
||||
processor_spec = importlib.util.spec_from_file_location(
|
||||
"audio8_tts_srt_processor_module", processor_path
|
||||
)
|
||||
processor_module = importlib.util.module_from_spec(processor_spec)
|
||||
processor_spec.loader.exec_module(processor_module)
|
||||
Audio8TTSSRTProcessor = processor_module.Audio8TTSSRTProcessor
|
||||
|
||||
class Audio8TTSSRTWrapper:
|
||||
def __init__(self, cfg):
|
||||
self.config = cfg.copy()
|
||||
self.processor = Audio8TTSSRTProcessor(self, self.config)
|
||||
|
||||
def update_config(self, new_config):
|
||||
self.config = new_config.copy()
|
||||
self.processor.update_config(new_config)
|
||||
|
||||
def check_interrupt(self):
|
||||
if model_management.interrupt_processing:
|
||||
raise InterruptedError(
|
||||
"Audio8 TTS SRT processing interrupted by user"
|
||||
)
|
||||
|
||||
engine_instance = Audio8TTSSRTWrapper(config)
|
||||
import time
|
||||
self._cached_engine_instances[cache_key] = {
|
||||
'instance': engine_instance,
|
||||
'timestamp': time.time()
|
||||
}
|
||||
return engine_instance
|
||||
|
||||
elif engine_type == "dramabox":
|
||||
processor_path = os.path.join(
|
||||
nodes_dir, "dramabox", "dramabox_srt_processor.py"
|
||||
@@ -702,53 +737,6 @@ Hello! This is unified SRT TTS with character switching.
|
||||
}
|
||||
return engine_instance
|
||||
|
||||
elif engine_type == "voxcpm":
|
||||
processor_path = os.path.join(
|
||||
nodes_dir, "voxcpm", "voxcpm_srt_processor.py"
|
||||
)
|
||||
processor_spec = importlib.util.spec_from_file_location(
|
||||
"voxcpm_srt_processor_module", processor_path
|
||||
)
|
||||
processor_module = importlib.util.module_from_spec(processor_spec)
|
||||
processor_spec.loader.exec_module(processor_module)
|
||||
VoxCPMSRTProcessor = processor_module.VoxCPMSRTProcessor
|
||||
|
||||
class VoxCPMSRTWrapper:
|
||||
def __init__(self, cfg):
|
||||
self.config = cfg.copy()
|
||||
self.processor = VoxCPMSRTProcessor(self, self.config)
|
||||
|
||||
def update_config(self, new_config):
|
||||
self.config = new_config.copy()
|
||||
self.processor.update_config(new_config)
|
||||
|
||||
def process_with_error_handling(self, func):
|
||||
return func()
|
||||
|
||||
def format_audio_output(self, audio_tensor, sample_rate):
|
||||
if audio_tensor.is_cuda:
|
||||
audio_tensor = audio_tensor.cpu()
|
||||
if audio_tensor.dim() == 1:
|
||||
audio_tensor = audio_tensor.unsqueeze(0).unsqueeze(0)
|
||||
elif audio_tensor.dim() == 2:
|
||||
audio_tensor = audio_tensor.unsqueeze(0)
|
||||
return {
|
||||
"waveform": audio_tensor,
|
||||
"sample_rate": int(sample_rate),
|
||||
}
|
||||
|
||||
def check_interrupt(self):
|
||||
if model_management.interrupt_processing:
|
||||
raise InterruptedError("VoxCPM SRT processing interrupted by user")
|
||||
|
||||
engine_instance = VoxCPMSRTWrapper(config)
|
||||
import time
|
||||
self._cached_engine_instances[cache_key] = {
|
||||
'instance': engine_instance,
|
||||
'timestamp': time.time()
|
||||
}
|
||||
return engine_instance
|
||||
|
||||
elif engine_type == "vibevoice":
|
||||
# Import and create the VibeVoice SRT processor using absolute import
|
||||
vibevoice_srt_processor_path = os.path.join(nodes_dir, "vibevoice", "vibevoice_srt_processor.py")
|
||||
@@ -1039,7 +1027,13 @@ Hello! This is unified SRT TTS with character switching.
|
||||
raise ValueError(f"Unknown engine type: {engine_type}")
|
||||
|
||||
except Exception as e:
|
||||
if isinstance(e, InterruptedError):
|
||||
if isinstance(
|
||||
e,
|
||||
(
|
||||
InterruptedError,
|
||||
model_management.InterruptProcessingException,
|
||||
),
|
||||
):
|
||||
raise
|
||||
print(f"❌ Failed to create engine SRT node instance: {e}")
|
||||
return None
|
||||
@@ -1105,7 +1099,7 @@ Hello! This is unified SRT TTS with character switching.
|
||||
print(f"📺 TTS SRT: Using direct audio input ({character_name})")
|
||||
print(
|
||||
"⚠️ TTS SRT: Direct audio input has no reference text - "
|
||||
"F5-TTS and OmniVoice cloning will fail"
|
||||
"Audio8 TTS, F5-TTS, and OmniVoice cloning will fail"
|
||||
)
|
||||
return None, audio_tensor, reference_text, character_name
|
||||
|
||||
@@ -1132,7 +1126,13 @@ Hello! This is unified SRT TTS with character switching.
|
||||
return None, None, "", "narrator"
|
||||
|
||||
except Exception as e:
|
||||
if isinstance(e, InterruptedError):
|
||||
if isinstance(
|
||||
e,
|
||||
(
|
||||
InterruptedError,
|
||||
model_management.InterruptProcessingException,
|
||||
),
|
||||
):
|
||||
raise
|
||||
print(f"❌ Voice reference error: {e}")
|
||||
return None, None, "", "narrator"
|
||||
@@ -1227,16 +1227,12 @@ Hello! This is unified SRT TTS with character switching.
|
||||
"OmniVoice voice cloning requires reference text. "
|
||||
"Do not connect raw audio directly. Use Character Voices node or a narrator voice with a matching .reference.txt file."
|
||||
)
|
||||
if (
|
||||
engine_type == "voxcpm"
|
||||
and config.get("model_variant") in {"VoxCPM1.5", "VoxCPM-0.5B"}
|
||||
and (audio_tensor is not None or audio_path)
|
||||
and not reference_text.strip()
|
||||
):
|
||||
if engine_type == "audio8_tts" and (audio_tensor is not None or audio_path) and not reference_text.strip():
|
||||
raise ValueError(
|
||||
f"{config.get('model_variant')} voice cloning requires the exact reference transcript. "
|
||||
"Use Character Voices or a narrator voice with a matching .reference.txt file. "
|
||||
"VoxCPM2 supports reference-only cloning."
|
||||
"Audio8 TTS voice cloning requires reference text. "
|
||||
"Do not connect raw audio directly. Use Character Voices with "
|
||||
"the exact transcript, or a narrator voice with a matching "
|
||||
".reference.txt file."
|
||||
)
|
||||
|
||||
# Create proper engine SRT node instance to preserve ALL functionality
|
||||
@@ -1438,6 +1434,30 @@ Hello! This is unified SRT TTS with character switching.
|
||||
enable_audio_cache=enable_audio_cache
|
||||
)
|
||||
|
||||
elif engine_type == "audio8_tts":
|
||||
timing_params = {
|
||||
'fade_for_StretchToFit': fade_for_StretchToFit,
|
||||
'max_stretch_ratio': max_stretch_ratio,
|
||||
'min_stretch_ratio': min_stretch_ratio,
|
||||
'timing_tolerance': timing_tolerance,
|
||||
}
|
||||
voice_mapping = {}
|
||||
if audio_tensor is not None or audio_path:
|
||||
voice_mapping['narrator'] = {
|
||||
'audio': audio_tensor,
|
||||
'audio_path': audio_path,
|
||||
'reference_text': reference_text or '',
|
||||
'character_name': character_name or 'narrator',
|
||||
}
|
||||
result = engine_instance.processor.process_srt_content(
|
||||
srt_content=srt_content,
|
||||
voice_mapping=voice_mapping,
|
||||
seed=seed,
|
||||
timing_mode=timing_mode,
|
||||
timing_params=timing_params,
|
||||
enable_audio_cache=enable_audio_cache,
|
||||
)
|
||||
|
||||
elif engine_type == "dramabox":
|
||||
timing_params = {
|
||||
'fade_for_StretchToFit': fade_for_StretchToFit,
|
||||
@@ -1515,29 +1535,6 @@ Hello! This is unified SRT TTS with character switching.
|
||||
enable_audio_cache=enable_audio_cache
|
||||
)
|
||||
|
||||
elif engine_type == "voxcpm":
|
||||
timing_params = {
|
||||
'fade_for_StretchToFit': fade_for_StretchToFit,
|
||||
'max_stretch_ratio': max_stretch_ratio,
|
||||
'min_stretch_ratio': min_stretch_ratio,
|
||||
'timing_tolerance': timing_tolerance,
|
||||
}
|
||||
voice_mapping = {}
|
||||
if audio_tensor is not None or audio_path:
|
||||
voice_mapping['narrator'] = {
|
||||
'audio': audio_tensor,
|
||||
'audio_path': audio_path,
|
||||
'reference_text': reference_text or "",
|
||||
}
|
||||
result = engine_instance.processor.process_srt_content(
|
||||
srt_content=srt_content,
|
||||
voice_mapping=voice_mapping,
|
||||
seed=seed,
|
||||
timing_mode=timing_mode,
|
||||
timing_params=timing_params,
|
||||
enable_audio_cache=enable_audio_cache,
|
||||
)
|
||||
|
||||
elif engine_type == "vibevoice":
|
||||
# Use the VibeVoice SRT processor from the wrapper instance
|
||||
voice_mapping = {"narrator": audio_tensor} if audio_tensor else {}
|
||||
@@ -1774,9 +1771,16 @@ Hello! This is unified SRT TTS with character switching.
|
||||
"is a voice-design model and cannot be used with TTS SRT" in msg
|
||||
or "Pause tags are not compatible with force_speaker_kv" in msg
|
||||
or "MOSS-TTSD Native Multi-Speaker Dialogue does not support this SRT input" in msg
|
||||
or "Audio8 TTS voice cloning requires reference text" in msg
|
||||
):
|
||||
raise
|
||||
if isinstance(e, InterruptedError):
|
||||
if isinstance(
|
||||
e,
|
||||
(
|
||||
InterruptedError,
|
||||
model_management.InterruptProcessingException,
|
||||
),
|
||||
):
|
||||
raise
|
||||
error_msg = f"❌ TTS SRT generation failed: {e}"
|
||||
print(error_msg)
|
||||
|
||||
+143
-112
@@ -89,7 +89,7 @@ Back to the main narrator voice for the conclusion.""",
|
||||
}),
|
||||
"narrator_voice": (reference_files, {
|
||||
"default": "none",
|
||||
"tooltip": "Fallback narrator voice from voice folders. Used when opt_narrator is not connected. Select 'none' for engines that support direct TTS without voice cloning, such as MOSS."
|
||||
"tooltip": "Fallback narrator voice from voice folders. Used when opt_narrator is not connected. Select 'none' for engines that support direct TTS without voice cloning, such as MOSS or Audio8."
|
||||
}),
|
||||
"seed": ("INT", {
|
||||
"default": 1, "min": 0, "max": 2**32 - 1,
|
||||
@@ -218,6 +218,12 @@ Back to the main narrator voice for the conclusion.""",
|
||||
stable_params['optimize'] = config.get('optimize', False)
|
||||
stable_params['max_generate_length'] = config.get('max_generate_length', 500)
|
||||
|
||||
if engine_type == "audio8_tts":
|
||||
stable_params['model_variant'] = config.get(
|
||||
'model_variant', 'Audio8-TTS-Preview-0.6b'
|
||||
)
|
||||
stable_params['dtype'] = config.get('dtype', 'auto')
|
||||
|
||||
if engine_type == "dramabox":
|
||||
stable_params['model_name'] = config.get('model_name', 'DramaBox')
|
||||
stable_params['precision'] = config.get('precision', 'auto')
|
||||
@@ -237,11 +243,6 @@ Back to the main narrator voice for the conclusion.""",
|
||||
stable_params['model_variant'] = config.get('model_variant', 'OmniVoice')
|
||||
stable_params['dtype'] = config.get('dtype', 'auto')
|
||||
|
||||
if engine_type == "voxcpm":
|
||||
stable_params['model_variant'] = config.get('model_variant', 'VoxCPM2')
|
||||
stable_params['model_name'] = config.get('model_name', 'VoxCPM2')
|
||||
stable_params['optimize'] = config.get('optimize', False)
|
||||
|
||||
# For MOSS-TTS, include model identity and load-time options.
|
||||
if engine_type == "moss_tts":
|
||||
stable_params['model_variant'] = config.get('model_variant', 'MOSS-TTS-Local-Transformer')
|
||||
@@ -450,7 +451,13 @@ Back to the main narrator voice for the conclusion.""",
|
||||
voice_mapping[char] = {"waveform": waveform, "sample_rate": sample_rate}
|
||||
print(f"🎭 VibeVoice: Using character-specific voice for '{char}'")
|
||||
except Exception as e:
|
||||
if isinstance(e, InterruptedError):
|
||||
if isinstance(
|
||||
e,
|
||||
(
|
||||
InterruptedError,
|
||||
model_management.InterruptProcessingException,
|
||||
),
|
||||
):
|
||||
raise
|
||||
print(f"⚠️ Failed to load character audio for '{char}': {e}")
|
||||
voice_mapping[char] = char_audio # Fallback to main voice
|
||||
@@ -585,6 +592,38 @@ Back to the main narrator voice for the conclusion.""",
|
||||
|
||||
return engine_instance
|
||||
|
||||
elif engine_type == "audio8_tts":
|
||||
from engines.adapters.audio8_tts_adapter import Audio8TTSEngineAdapter
|
||||
|
||||
processor_path = os.path.join(
|
||||
nodes_dir, "audio8_tts", "audio8_tts_processor.py"
|
||||
)
|
||||
processor_spec = importlib.util.spec_from_file_location(
|
||||
"audio8_tts_processor_module", processor_path
|
||||
)
|
||||
processor_module = importlib.util.module_from_spec(processor_spec)
|
||||
processor_spec.loader.exec_module(processor_module)
|
||||
Audio8TTSProcessor = processor_module.Audio8TTSProcessor
|
||||
|
||||
class Audio8TTSWrapper:
|
||||
def __init__(self, cfg):
|
||||
self.config = cfg.copy()
|
||||
self.adapter = Audio8TTSEngineAdapter(self.config)
|
||||
self.processor = Audio8TTSProcessor(self.adapter, self.config)
|
||||
|
||||
def update_config(self, new_config):
|
||||
self.config = new_config.copy()
|
||||
self.adapter.update_config(new_config)
|
||||
self.processor.update_config(new_config)
|
||||
|
||||
engine_instance = Audio8TTSWrapper(config)
|
||||
import time
|
||||
self._cached_engine_instances[cache_key] = {
|
||||
'instance': engine_instance,
|
||||
'timestamp': time.time()
|
||||
}
|
||||
return engine_instance
|
||||
|
||||
elif engine_type == "dramabox":
|
||||
from engines.adapters.dramabox_adapter import DramaBoxEngineAdapter
|
||||
processor_path = os.path.join(nodes_dir, "dramabox", "dramabox_processor.py")
|
||||
@@ -675,41 +714,6 @@ Back to the main narrator voice for the conclusion.""",
|
||||
|
||||
return engine_instance
|
||||
|
||||
elif engine_type == "voxcpm":
|
||||
from engines.adapters.voxcpm_adapter import VoxCPMEngineAdapter
|
||||
|
||||
processor_path = os.path.join(nodes_dir, "voxcpm", "voxcpm_processor.py")
|
||||
processor_spec = importlib.util.spec_from_file_location(
|
||||
"voxcpm_processor_module", processor_path
|
||||
)
|
||||
processor_module = importlib.util.module_from_spec(processor_spec)
|
||||
processor_spec.loader.exec_module(processor_module)
|
||||
VoxCPMProcessor = processor_module.VoxCPMProcessor
|
||||
|
||||
class VoxCPMWrapper:
|
||||
def __init__(self, cfg):
|
||||
self.config = cfg.copy()
|
||||
self.adapter = VoxCPMEngineAdapter(self.config)
|
||||
self.processor = VoxCPMProcessor(self.adapter, self.config)
|
||||
|
||||
def update_config(self, new_config):
|
||||
self.config = new_config.copy()
|
||||
self.adapter.update_config(new_config)
|
||||
self.processor.update_config(new_config)
|
||||
|
||||
def check_interrupt(self):
|
||||
if model_management.interrupt_processing:
|
||||
raise InterruptedError("VoxCPM processing interrupted by user")
|
||||
|
||||
engine_instance = VoxCPMWrapper(config)
|
||||
|
||||
import time
|
||||
self._cached_engine_instances[cache_key] = {
|
||||
'instance': engine_instance,
|
||||
'timestamp': time.time()
|
||||
}
|
||||
return engine_instance
|
||||
|
||||
elif engine_type == "qwen3_tts":
|
||||
# Create Qwen3-TTS processor instance
|
||||
# Use global nodes_dir (already defined at module level)
|
||||
@@ -877,7 +881,13 @@ Back to the main narrator voice for the conclusion.""",
|
||||
raise ValueError(f"Unknown engine type: {engine_type}")
|
||||
|
||||
except Exception as e:
|
||||
if isinstance(e, InterruptedError):
|
||||
if isinstance(
|
||||
e,
|
||||
(
|
||||
InterruptedError,
|
||||
model_management.InterruptProcessingException,
|
||||
),
|
||||
):
|
||||
raise
|
||||
if "MOSS LoRA/base model mismatch" in str(e):
|
||||
raise
|
||||
@@ -946,7 +956,8 @@ Back to the main narrator voice for the conclusion.""",
|
||||
print(f"🎤 TTS Text: Using direct audio input ({character_name})")
|
||||
print(
|
||||
"⚠️ TTS Text: Direct audio input has no reference text - "
|
||||
"F5-TTS and OmniVoice cloning will fail, Qwen3-TTS will use x_vector_only mode (lower quality)"
|
||||
"Audio8 TTS, F5-TTS, and OmniVoice cloning will fail; "
|
||||
"Qwen3-TTS will use x_vector_only mode (lower quality)"
|
||||
)
|
||||
return None, audio_tensor, reference_text, character_name
|
||||
|
||||
@@ -992,7 +1003,13 @@ Back to the main narrator voice for the conclusion.""",
|
||||
return None, None, "", "narrator"
|
||||
|
||||
except Exception as e:
|
||||
if isinstance(e, InterruptedError):
|
||||
if isinstance(
|
||||
e,
|
||||
(
|
||||
InterruptedError,
|
||||
model_management.InterruptProcessingException,
|
||||
),
|
||||
):
|
||||
raise
|
||||
print(f"❌ Voice reference error: {e}")
|
||||
return None, None, "", "narrator"
|
||||
@@ -1099,16 +1116,12 @@ Back to the main narrator voice for the conclusion.""",
|
||||
"OmniVoice voice cloning requires reference text. "
|
||||
"Do not connect raw audio directly. Use Character Voices node or a narrator voice with a matching .reference.txt file."
|
||||
)
|
||||
if (
|
||||
engine_type == "voxcpm"
|
||||
and config.get("model_variant") in {"VoxCPM1.5", "VoxCPM-0.5B"}
|
||||
and (audio_tensor is not None or audio_path)
|
||||
and not reference_text.strip()
|
||||
):
|
||||
if engine_type == "audio8_tts" and (audio_tensor is not None or audio_path) and not reference_text.strip():
|
||||
raise ValueError(
|
||||
f"{config.get('model_variant')} voice cloning requires the exact reference transcript. "
|
||||
"Use Character Voices or a narrator voice with a matching .reference.txt file. "
|
||||
"VoxCPM2 supports reference-only cloning."
|
||||
"Audio8 TTS voice cloning requires reference text. "
|
||||
"Do not connect raw audio directly. Use Character Voices with "
|
||||
"the exact transcript, or a narrator voice with a matching "
|
||||
".reference.txt file."
|
||||
)
|
||||
|
||||
# Create proper engine node instance to preserve ALL functionality
|
||||
@@ -1571,6 +1584,59 @@ Back to the main narrator voice for the conclusion.""",
|
||||
formatted_audio = AudioProcessingUtils.format_for_comfyui(combined_audio, 48000)
|
||||
result = (formatted_audio, generation_info)
|
||||
|
||||
elif engine_type == "audio8_tts":
|
||||
import re
|
||||
from utils.audio.chunk_timing import ChunkTimingHelper
|
||||
|
||||
voice_mapping = {}
|
||||
if audio_tensor is not None or audio_path:
|
||||
voice_mapping['narrator'] = {
|
||||
'audio': audio_tensor,
|
||||
'audio_path': audio_path,
|
||||
'reference_text': reference_text or '',
|
||||
'character_name': character_name or 'narrator',
|
||||
}
|
||||
|
||||
segment_records = engine_instance.processor.process_text(
|
||||
text=text,
|
||||
voice_mapping=voice_mapping,
|
||||
seed=seed,
|
||||
enable_chunking=enable_chunking,
|
||||
max_chars_per_chunk=max_chars_per_chunk,
|
||||
chunk_combination_method=chunk_combination_method,
|
||||
silence_between_chunks_ms=silence_between_chunks_ms,
|
||||
enable_audio_cache=enable_audio_cache,
|
||||
)
|
||||
combined_audio, chunk_info = engine_instance.processor.combine_audio_segments(
|
||||
segments=segment_records,
|
||||
method=chunk_combination_method,
|
||||
silence_ms=silence_between_chunks_ms,
|
||||
original_text=text,
|
||||
return_info=True,
|
||||
)
|
||||
|
||||
total_duration = (
|
||||
combined_audio.shape[-1] / 44100.0
|
||||
if combined_audio.numel()
|
||||
else 0.0
|
||||
)
|
||||
clean_text = re.sub(r'\[.*?\]', '', text)
|
||||
base_info = (
|
||||
f"Generated {total_duration:.1f}s audio from {len(clean_text)} characters "
|
||||
f"(Audio8 TTS, narrator: {char_display})"
|
||||
)
|
||||
base_info += (
|
||||
"\n🎭 Reference-free speech, zero-shot cloning, character switching, "
|
||||
"and pause tags enabled"
|
||||
)
|
||||
generation_info = ChunkTimingHelper.enhance_generation_info(
|
||||
f"✅ {base_info}", chunk_info
|
||||
)
|
||||
result = (
|
||||
AudioProcessingUtils.format_for_comfyui(combined_audio, 44100),
|
||||
generation_info,
|
||||
)
|
||||
|
||||
elif engine_type == "dramabox":
|
||||
import re
|
||||
from utils.audio.chunk_timing import ChunkTimingHelper
|
||||
@@ -1728,61 +1794,6 @@ Back to the main narrator voice for the conclusion.""",
|
||||
formatted_audio = AudioProcessingUtils.format_for_comfyui(combined_audio, 24000)
|
||||
result = (formatted_audio, generation_info)
|
||||
|
||||
elif engine_type == "voxcpm":
|
||||
import re
|
||||
from utils.audio.chunk_timing import ChunkTimingHelper
|
||||
|
||||
voice_mapping = {}
|
||||
if audio_tensor is not None or audio_path:
|
||||
voice_mapping['narrator'] = {
|
||||
'audio': audio_tensor,
|
||||
'audio_path': audio_path,
|
||||
'reference_text': reference_text or '',
|
||||
}
|
||||
|
||||
segment_records = engine_instance.processor.process_text(
|
||||
text=text,
|
||||
voice_mapping=voice_mapping,
|
||||
seed=seed,
|
||||
enable_chunking=enable_chunking,
|
||||
max_chars_per_chunk=max_chars_per_chunk,
|
||||
chunk_combination_method=chunk_combination_method,
|
||||
silence_between_chunks_ms=silence_between_chunks_ms,
|
||||
enable_audio_cache=enable_audio_cache,
|
||||
)
|
||||
combined_audio, chunk_info = engine_instance.processor.combine_audio_segments(
|
||||
segments=segment_records,
|
||||
method=chunk_combination_method,
|
||||
silence_ms=silence_between_chunks_ms,
|
||||
original_text=text,
|
||||
return_info=True,
|
||||
)
|
||||
sample_rate = int(
|
||||
getattr(engine_instance.processor, "sample_rate", None)
|
||||
or getattr(engine_instance.adapter, "sample_rate", None)
|
||||
or getattr(engine_instance.adapter, "SAMPLE_RATE", 48000)
|
||||
)
|
||||
total_duration = (
|
||||
combined_audio.shape[-1] / sample_rate if combined_audio.numel() else 0.0
|
||||
)
|
||||
clean_text = re.sub(r'\[.*?\]', '', text)
|
||||
model_variant = config.get('model_variant', 'VoxCPM2')
|
||||
base_info = (
|
||||
f"Generated {total_duration:.1f}s audio from {len(clean_text)} characters "
|
||||
f"(VoxCPM {model_variant}, narrator: {char_display})"
|
||||
)
|
||||
base_info += (
|
||||
"\n🎭 Character switching, pause tags, segment parameters, "
|
||||
"and architecture-gated cloning enabled"
|
||||
)
|
||||
generation_info = ChunkTimingHelper.enhance_generation_info(
|
||||
f"✅ {base_info}", chunk_info
|
||||
)
|
||||
result = (
|
||||
AudioProcessingUtils.format_for_comfyui(combined_audio, sample_rate),
|
||||
generation_info,
|
||||
)
|
||||
|
||||
elif engine_type == "moss_tts":
|
||||
import re
|
||||
from utils.audio.chunk_timing import ChunkTimingHelper
|
||||
@@ -2006,7 +2017,13 @@ Back to the main narrator voice for the conclusion.""",
|
||||
}
|
||||
print(f"🎭 Qwen3-TTS: Using character-specific voice for '{character}' (ICL mode)")
|
||||
except Exception as e:
|
||||
if isinstance(e, InterruptedError):
|
||||
if isinstance(
|
||||
e,
|
||||
(
|
||||
InterruptedError,
|
||||
model_management.InterruptProcessingException,
|
||||
),
|
||||
):
|
||||
raise
|
||||
print(f"⚠️ Failed to load character audio for '{character}': {e}")
|
||||
# Fallback to narrator voice if available
|
||||
@@ -2051,7 +2068,13 @@ Back to the main narrator voice for the conclusion.""",
|
||||
print(f"⚠️⚠️ Qwen3-TTS: Character '{character}' has audio but NO reference text")
|
||||
print(f"⚠️⚠️ Using x_vector_only mode (speaker embedding only) - LOWER QUALITY than ICL mode")
|
||||
except Exception as e:
|
||||
if isinstance(e, InterruptedError):
|
||||
if isinstance(
|
||||
e,
|
||||
(
|
||||
InterruptedError,
|
||||
model_management.InterruptProcessingException,
|
||||
),
|
||||
):
|
||||
raise
|
||||
print(f"⚠️ Failed to load character audio for '{character}': {e}")
|
||||
# Fallback to narrator voice if available
|
||||
@@ -2188,9 +2211,17 @@ Back to the main narrator voice for the conclusion.""",
|
||||
raise
|
||||
if "MOSS LoRA/base model mismatch" in str(e):
|
||||
raise
|
||||
if "Audio8 TTS voice cloning requires reference text" in str(e):
|
||||
raise
|
||||
if engine_type == "index_tts":
|
||||
raise
|
||||
if isinstance(e, InterruptedError):
|
||||
if isinstance(
|
||||
e,
|
||||
(
|
||||
InterruptedError,
|
||||
model_management.InterruptProcessingException,
|
||||
),
|
||||
):
|
||||
raise
|
||||
error_msg = f"❌ TTS Text generation failed: {e}"
|
||||
print(error_msg)
|
||||
|
||||
@@ -1,417 +0,0 @@
|
||||
"""VoxCPM text orchestration for TTS Audio Suite."""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Dict, List, Tuple, Union
|
||||
|
||||
import comfy.model_management as model_management
|
||||
import torch
|
||||
|
||||
current_dir = os.path.dirname(__file__)
|
||||
nodes_dir = os.path.dirname(current_dir)
|
||||
project_root = os.path.dirname(nodes_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
from utils.audio.chunk_timing import ChunkTimingHelper
|
||||
from utils.audio.edit_post_processor import process_segments as apply_edit_post_processing
|
||||
from utils.models.language_mapper import resolve_language_alias
|
||||
from utils.text.character_parser import character_parser
|
||||
from utils.text.pause_processor import PauseTagProcessor
|
||||
from utils.text.segment_parameters import ParameterValidator, apply_segment_parameters
|
||||
from utils.text.step_audio_editx_special_tags import get_edit_tags_for_segment
|
||||
from utils.voice.character_logging import (
|
||||
format_resolved_character_block,
|
||||
resolved_character_label,
|
||||
)
|
||||
from utils.voice.discovery import (
|
||||
get_available_characters,
|
||||
get_character_mapping,
|
||||
voice_discovery,
|
||||
)
|
||||
from utils.voice.reference import effective_voice_audio
|
||||
|
||||
|
||||
class VoxCPMProcessor:
|
||||
"""Handle suite-level orchestration around one-call VoxCPM inference."""
|
||||
|
||||
ENGINE_TYPE = "voxcpm"
|
||||
|
||||
def __init__(self, adapter, engine_config: Dict[str, Any]):
|
||||
self.adapter = adapter
|
||||
self.config = dict(engine_config or {})
|
||||
|
||||
def update_config(self, new_config: Dict[str, Any]):
|
||||
self.config = dict(new_config or {})
|
||||
self.adapter.update_config(self.config)
|
||||
|
||||
@property
|
||||
def sample_rate(self) -> int:
|
||||
"""Return the selected model's native sample rate without hardcoding it."""
|
||||
for attribute_name in ("get_sample_rate", "sample_rate", "SAMPLE_RATE"):
|
||||
value = getattr(self.adapter, attribute_name, None)
|
||||
if callable(value):
|
||||
value = value()
|
||||
if value is not None:
|
||||
try:
|
||||
sample_rate = int(value)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if sample_rate > 0:
|
||||
return sample_rate
|
||||
|
||||
configured_rate = self.config.get("sample_rate")
|
||||
if configured_rate is not None:
|
||||
try:
|
||||
sample_rate = int(configured_rate)
|
||||
except (TypeError, ValueError):
|
||||
sample_rate = 0
|
||||
if sample_rate > 0:
|
||||
return sample_rate
|
||||
|
||||
raise RuntimeError(
|
||||
"VoxCPM adapter did not expose a valid model sample rate. "
|
||||
"Expected get_sample_rate(), sample_rate, or SAMPLE_RATE."
|
||||
)
|
||||
|
||||
@property
|
||||
def SAMPLE_RATE(self) -> int:
|
||||
"""Compatibility property for callers that use the suite's legacy name."""
|
||||
return self.sample_rate
|
||||
|
||||
def get_sample_rate(self) -> int:
|
||||
return self.sample_rate
|
||||
|
||||
@staticmethod
|
||||
def _check_interrupt(context: str):
|
||||
if model_management.interrupt_processing:
|
||||
raise InterruptedError(f"VoxCPM generation interrupted {context}")
|
||||
|
||||
def _setup_character_parser(self, text: str, voice_mapping: Dict[str, Any]):
|
||||
configured_language = str(self.config.get("language", "auto") or "auto").strip()
|
||||
parser_language = resolve_language_alias(configured_language)
|
||||
if parser_language in ("", "auto", "automatic", "none"):
|
||||
parser_language = "en"
|
||||
|
||||
character_parser.language_resolver.default_language = parser_language
|
||||
character_parser.default_language = parser_language
|
||||
|
||||
all_available = set(get_available_characters() or ())
|
||||
for alias, target in voice_discovery.get_character_aliases().items():
|
||||
all_available.add(alias.lower())
|
||||
all_available.add(target.lower())
|
||||
|
||||
for character_name in (voice_mapping or {}):
|
||||
if character_name:
|
||||
all_available.add(str(character_name).lower())
|
||||
|
||||
for tag in re.findall(r"\[([^\]]+)\]", text or ""):
|
||||
if tag.lower().startswith(("pause:", "wait:", "stop:")):
|
||||
continue
|
||||
all_available.add(tag.split("|", 1)[0].split(":")[-1].strip().lower())
|
||||
|
||||
all_available.add("narrator")
|
||||
character_parser.set_available_characters(list(all_available))
|
||||
|
||||
for character_name, language in voice_discovery.get_character_language_defaults().items():
|
||||
character_parser.set_character_language_default(character_name, language)
|
||||
|
||||
character_parser.reset_session_cache()
|
||||
|
||||
@staticmethod
|
||||
def _coerce_voice_reference(value: Any) -> Dict[str, Any]:
|
||||
if value is None:
|
||||
return {}
|
||||
if isinstance(value, Mapping):
|
||||
copied = dict(value)
|
||||
if "waveform" in copied and "sample_rate" in copied:
|
||||
return {"audio": copied}
|
||||
return copied
|
||||
if isinstance(value, (str, os.PathLike)):
|
||||
return {"audio_path": os.fspath(value)}
|
||||
return {"audio": value}
|
||||
|
||||
def _resolve_voice_reference(
|
||||
self,
|
||||
character_name: str,
|
||||
narrator_voice: Dict[str, Any],
|
||||
voice_mapping: Dict[str, Any],
|
||||
character_mapping: Dict[str, Tuple[Any, Any]],
|
||||
) -> Dict[str, Any]:
|
||||
if character_name != "narrator" and character_name in voice_mapping:
|
||||
return self._coerce_voice_reference(voice_mapping[character_name])
|
||||
|
||||
if character_name != "narrator":
|
||||
audio_path, reference_text = character_mapping.get(character_name, (None, None))
|
||||
if audio_path:
|
||||
return {
|
||||
"audio_path": audio_path,
|
||||
"reference_text": reference_text or "",
|
||||
}
|
||||
|
||||
return dict(narrator_voice)
|
||||
|
||||
@staticmethod
|
||||
def _voice_log_note(voice_ref: Dict[str, Any]) -> str:
|
||||
if not voice_ref or effective_voice_audio(voice_ref) is None:
|
||||
return " [no reference audio]"
|
||||
reference_text = (
|
||||
voice_ref.get("reference_text")
|
||||
or voice_ref.get("prompt_text")
|
||||
or voice_ref.get("text")
|
||||
or ""
|
||||
)
|
||||
if reference_text:
|
||||
return f" [reference transcript: {len(reference_text)} chars]"
|
||||
return ""
|
||||
|
||||
def _log_generation(
|
||||
self,
|
||||
character_name: str,
|
||||
text: str,
|
||||
voice_ref: Dict[str, Any],
|
||||
chunk_count: int,
|
||||
filtered_params: Dict[str, Any],
|
||||
show_text_logging: bool,
|
||||
):
|
||||
display_name = resolved_character_label(character_name, voice_ref)
|
||||
print(
|
||||
f"🎭 VoxCPM - Generating for '{display_name}'"
|
||||
f"{self._voice_log_note(voice_ref)}"
|
||||
)
|
||||
if filtered_params:
|
||||
parameter_text = ", ".join(
|
||||
f"{name}={value}" for name, value in sorted(filtered_params.items())
|
||||
)
|
||||
print(f"🎛️ VoxCPM segment parameters: {parameter_text}")
|
||||
if show_text_logging:
|
||||
print(format_resolved_character_block(character_name, text, voice_ref))
|
||||
if chunk_count > 1:
|
||||
print(f"📝 VoxCPM: Split '{display_name}' into {chunk_count} chunks")
|
||||
|
||||
@staticmethod
|
||||
def _normalize_generated_audio(audio: Any) -> torch.Tensor:
|
||||
if not isinstance(audio, torch.Tensor):
|
||||
audio = torch.as_tensor(audio, dtype=torch.float32)
|
||||
audio = audio.detach().float().cpu()
|
||||
if audio.dim() == 1:
|
||||
audio = audio.unsqueeze(0)
|
||||
elif audio.dim() == 3 and audio.shape[0] == 1:
|
||||
audio = audio.squeeze(0)
|
||||
if audio.dim() != 2:
|
||||
raise ValueError(
|
||||
f"VoxCPM adapter returned invalid audio shape {tuple(audio.shape)}; "
|
||||
"expected [samples] or [channels, samples]"
|
||||
)
|
||||
return audio
|
||||
|
||||
def process_text(
|
||||
self,
|
||||
text: str,
|
||||
voice_mapping: Dict[str, Any],
|
||||
seed: int,
|
||||
enable_chunking: bool = True,
|
||||
max_chars_per_chunk: int = 400,
|
||||
chunk_combination_method: str = "auto",
|
||||
silence_between_chunks_ms: int = 100,
|
||||
enable_audio_cache: bool = True,
|
||||
apply_edit_postprocessing: bool = True,
|
||||
show_text_logging: bool = True,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Generate record-form audio while preserving suite tag behavior."""
|
||||
del chunk_combination_method, silence_between_chunks_ms
|
||||
|
||||
voice_mapping = dict(voice_mapping or {})
|
||||
self._setup_character_parser(text, voice_mapping)
|
||||
|
||||
narrator_voice = self._coerce_voice_reference(voice_mapping.get("narrator"))
|
||||
base_config = dict(self.config)
|
||||
segment_objects = character_parser.parse_text_segments(
|
||||
text or "", engine_type=self.ENGINE_TYPE
|
||||
)
|
||||
|
||||
characters = list(
|
||||
{segment.character for segment in segment_objects if segment.character}
|
||||
)
|
||||
character_mapping = get_character_mapping(characters, engine_type="audio_only")
|
||||
segment_records: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
for segment_index, segment in enumerate(segment_objects):
|
||||
character_name = segment.character or "narrator"
|
||||
self._check_interrupt(
|
||||
f"before segment {segment_index + 1}/{len(segment_objects)} "
|
||||
f"('{character_name}')"
|
||||
)
|
||||
|
||||
segment_text = (segment.text or "").strip()
|
||||
if not segment_text:
|
||||
continue
|
||||
|
||||
raw_segment_params = dict(segment.parameters or {})
|
||||
filtered_params = ParameterValidator.filter_parameters_for_engine(
|
||||
raw_segment_params, self.ENGINE_TYPE
|
||||
)
|
||||
current_config = apply_segment_parameters(
|
||||
base_config, filtered_params, self.ENGINE_TYPE
|
||||
)
|
||||
current_seed = int(current_config.get("seed", seed))
|
||||
|
||||
if (
|
||||
getattr(segment, "explicit_language", False)
|
||||
and "language" in base_config
|
||||
and getattr(segment, "language", None)
|
||||
):
|
||||
current_config["language"] = segment.language
|
||||
|
||||
self.adapter.update_config(current_config)
|
||||
voice_ref = self._resolve_voice_reference(
|
||||
character_name,
|
||||
narrator_voice,
|
||||
voice_mapping,
|
||||
character_mapping,
|
||||
)
|
||||
|
||||
def generate_chunks(text_content: str, edit_tags: list):
|
||||
cleaned = (text_content or "").strip()
|
||||
if not cleaned:
|
||||
return
|
||||
|
||||
if enable_chunking:
|
||||
from utils.text.chunking import ImprovedChatterBoxChunker
|
||||
|
||||
maximum = ImprovedChatterBoxChunker.validate_chunking_params(
|
||||
max_chars_per_chunk
|
||||
)
|
||||
chunks = ImprovedChatterBoxChunker.split_into_chunks(
|
||||
cleaned, max_chars=maximum
|
||||
)
|
||||
else:
|
||||
chunks = [cleaned]
|
||||
|
||||
chunks = [chunk.strip() for chunk in chunks if chunk.strip()]
|
||||
self._log_generation(
|
||||
character_name,
|
||||
cleaned,
|
||||
voice_ref,
|
||||
len(chunks),
|
||||
filtered_params,
|
||||
show_text_logging,
|
||||
)
|
||||
|
||||
for chunk_index, chunk in enumerate(chunks):
|
||||
self._check_interrupt(
|
||||
f"before chunk {chunk_index + 1}/{len(chunks)} "
|
||||
f"for '{character_name}'"
|
||||
)
|
||||
audio = self.adapter.generate_single(
|
||||
text=chunk,
|
||||
voice_ref=voice_ref,
|
||||
seed=(
|
||||
0
|
||||
if current_seed == 0
|
||||
else current_seed + chunk_index
|
||||
),
|
||||
enable_audio_cache=enable_audio_cache,
|
||||
character_name=character_name,
|
||||
)
|
||||
audio = self._normalize_generated_audio(audio)
|
||||
segment_records.append(
|
||||
{
|
||||
"waveform": audio,
|
||||
"sample_rate": self.sample_rate,
|
||||
"text": chunk,
|
||||
"edit_tags": edit_tags if chunk_index == 0 else [],
|
||||
}
|
||||
)
|
||||
|
||||
if PauseTagProcessor.has_pause_tags(segment_text):
|
||||
pause_parts, _ = PauseTagProcessor.parse_pause_tags(segment_text)
|
||||
for part_index, (part_type, content) in enumerate(pause_parts):
|
||||
self._check_interrupt(
|
||||
f"before pause fragment {part_index + 1}/{len(pause_parts)} "
|
||||
f"for '{character_name}'"
|
||||
)
|
||||
if part_type == "text":
|
||||
clean_text, edit_tags = get_edit_tags_for_segment(content)
|
||||
generate_chunks(clean_text, edit_tags)
|
||||
elif part_type == "pause":
|
||||
sample_rate = self.sample_rate
|
||||
silence = PauseTagProcessor.create_silence_segment(
|
||||
content,
|
||||
sample_rate,
|
||||
torch.device("cpu"),
|
||||
torch.float32,
|
||||
)
|
||||
silence = self._normalize_generated_audio(silence)
|
||||
segment_records.append(
|
||||
{
|
||||
"waveform": silence,
|
||||
"sample_rate": sample_rate,
|
||||
"text": f"[pause:{content}s]",
|
||||
"edit_tags": [],
|
||||
}
|
||||
)
|
||||
else:
|
||||
clean_text, edit_tags = get_edit_tags_for_segment(segment_text)
|
||||
generate_chunks(clean_text, edit_tags)
|
||||
finally:
|
||||
# Segment-local overrides must never leak into the next invocation.
|
||||
self.adapter.update_config(base_config)
|
||||
|
||||
if (
|
||||
apply_edit_postprocessing
|
||||
and segment_records
|
||||
and any(record.get("edit_tags") for record in segment_records)
|
||||
):
|
||||
self._check_interrupt("before inline edit post-processing")
|
||||
segment_records = apply_edit_post_processing(
|
||||
segment_records, engine_config=base_config
|
||||
)
|
||||
self._check_interrupt("after inline edit post-processing")
|
||||
|
||||
return segment_records
|
||||
|
||||
def combine_audio_segments(
|
||||
self,
|
||||
segments: List[Dict[str, Any]],
|
||||
method: str = "auto",
|
||||
silence_ms: int = 100,
|
||||
original_text: str = "",
|
||||
return_info: bool = False,
|
||||
) -> Union[torch.Tensor, Tuple[torch.Tensor, Dict[str, Any]]]:
|
||||
if not segments:
|
||||
empty = torch.zeros(1, 0, dtype=torch.float32)
|
||||
return (empty, {}) if return_info else empty
|
||||
|
||||
sample_rates = {
|
||||
int(segment.get("sample_rate", 0))
|
||||
for segment in segments
|
||||
if segment.get("sample_rate") is not None
|
||||
}
|
||||
if len(sample_rates) != 1 or next(iter(sample_rates), 0) <= 0:
|
||||
raise ValueError(
|
||||
f"VoxCPM segments have inconsistent sample rates: {sorted(sample_rates)}"
|
||||
)
|
||||
sample_rate = next(iter(sample_rates))
|
||||
|
||||
audio_segments = [
|
||||
self._normalize_generated_audio(segment["waveform"]) for segment in segments
|
||||
]
|
||||
text_chunks = [segment.get("text", "") for segment in segments]
|
||||
combined_audio, chunk_info = ChunkTimingHelper.combine_audio_with_timing(
|
||||
audio_segments=audio_segments,
|
||||
combination_method=method,
|
||||
silence_ms=silence_ms,
|
||||
crossfade_duration=0.1,
|
||||
sample_rate=sample_rate,
|
||||
text_length=len(" ".join(text_chunks)),
|
||||
original_text=original_text,
|
||||
text_chunks=text_chunks,
|
||||
)
|
||||
|
||||
if return_info:
|
||||
return combined_audio, chunk_info or {}
|
||||
return combined_audio
|
||||
@@ -1,318 +0,0 @@
|
||||
"""VoxCPM SRT orchestration using the shared text processor."""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
import comfy.model_management as model_management
|
||||
import torch
|
||||
|
||||
current_dir = os.path.dirname(__file__)
|
||||
nodes_dir = os.path.dirname(current_dir)
|
||||
project_root = os.path.dirname(nodes_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
from engines.adapters.voxcpm_adapter import VoxCPMEngineAdapter
|
||||
from utils.system.import_manager import import_manager
|
||||
from utils.timing.assembly import AudioAssemblyEngine
|
||||
from utils.timing.engine import TimingEngine
|
||||
from utils.timing.overlap_detection import SRTOverlapHandler
|
||||
from utils.timing.reporting import SRTReportGenerator
|
||||
|
||||
|
||||
class VoxCPMSRTProcessor:
|
||||
"""Generate VoxCPM speech per subtitle, then apply suite timing modes."""
|
||||
|
||||
def __init__(self, node_instance, config: Dict[str, Any]):
|
||||
self.node_instance = node_instance
|
||||
self.config = dict(config or {})
|
||||
self.adapter = VoxCPMEngineAdapter(self.config)
|
||||
self._processor = None
|
||||
|
||||
success, modules, message = import_manager.import_srt_modules()
|
||||
if not success:
|
||||
raise ImportError(f"VoxCPM SRT unavailable: {message}")
|
||||
self.SRTParser = modules["SRTParser"]
|
||||
|
||||
@property
|
||||
def processor(self):
|
||||
if self._processor is None:
|
||||
processor_path = os.path.join(current_dir, "voxcpm_processor.py")
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"voxcpm_processor_module", processor_path
|
||||
)
|
||||
if spec is None or spec.loader is None:
|
||||
raise ImportError(f"Unable to load VoxCPM processor from {processor_path}")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
self._processor = module.VoxCPMProcessor(self.adapter, self.config)
|
||||
return self._processor
|
||||
|
||||
@property
|
||||
def sample_rate(self) -> int:
|
||||
return self.processor.sample_rate
|
||||
|
||||
@property
|
||||
def SAMPLE_RATE(self) -> int:
|
||||
return self.sample_rate
|
||||
|
||||
def get_sample_rate(self) -> int:
|
||||
return self.sample_rate
|
||||
|
||||
def update_config(self, new_config: Dict[str, Any]):
|
||||
self.config = dict(new_config or {})
|
||||
if self._processor is not None:
|
||||
self._processor.update_config(self.config)
|
||||
else:
|
||||
self.adapter.update_config(self.config)
|
||||
|
||||
@staticmethod
|
||||
def _check_interrupt(
|
||||
subtitle_index: Optional[int] = None,
|
||||
total_subtitles: Optional[int] = None,
|
||||
):
|
||||
if model_management.interrupt_processing:
|
||||
where = ""
|
||||
if subtitle_index is not None and total_subtitles is not None:
|
||||
where = f" before subtitle {subtitle_index + 1}/{total_subtitles}"
|
||||
raise InterruptedError(f"VoxCPM SRT generation interrupted{where}")
|
||||
|
||||
@staticmethod
|
||||
def _normalize_audio(audio: torch.Tensor) -> torch.Tensor:
|
||||
audio = audio.detach().float().cpu()
|
||||
if audio.dim() == 1:
|
||||
audio = audio.unsqueeze(0)
|
||||
elif audio.dim() == 3 and audio.shape[0] == 1:
|
||||
audio = audio.squeeze(0)
|
||||
if audio.dim() != 2:
|
||||
raise ValueError(
|
||||
f"VoxCPM SRT received invalid audio shape {tuple(audio.shape)}"
|
||||
)
|
||||
return audio
|
||||
|
||||
@staticmethod
|
||||
def _build_adjustment(index, subtitle, natural_duration: float) -> Dict[str, Any]:
|
||||
target_duration = subtitle.duration
|
||||
ratio = target_duration / natural_duration if natural_duration > 0 else 1.0
|
||||
return {
|
||||
"index": index,
|
||||
"segment_index": index,
|
||||
"sequence": subtitle.sequence,
|
||||
"natural_duration": natural_duration,
|
||||
"target_start": subtitle.start_time,
|
||||
"target_end": subtitle.end_time,
|
||||
"target_duration": target_duration,
|
||||
"start_time": subtitle.start_time,
|
||||
"end_time": subtitle.end_time,
|
||||
"stretch_factor": ratio,
|
||||
"needs_stretching": abs(ratio - 1.0) > 0.05,
|
||||
"stretch_type": (
|
||||
"compress" if ratio < 1.0 else "expand" if ratio > 1.0 else "none"
|
||||
),
|
||||
"adjustment": natural_duration - target_duration,
|
||||
"adjusted_start": subtitle.start_time,
|
||||
"adjusted_end": subtitle.end_time,
|
||||
"adjusted_duration": natural_duration,
|
||||
}
|
||||
|
||||
def process_srt_content(
|
||||
self,
|
||||
srt_content: str,
|
||||
voice_mapping: Dict[str, Any],
|
||||
seed: int,
|
||||
timing_mode: str,
|
||||
timing_params: Dict[str, Any],
|
||||
enable_audio_cache: bool = True,
|
||||
) -> Tuple[Dict[str, Any], str, str, str]:
|
||||
self._check_interrupt()
|
||||
subtitles = self.SRTParser().parse_srt_content(
|
||||
srt_content, allow_overlaps=True
|
||||
)
|
||||
|
||||
if not subtitles:
|
||||
sample_rate = self.sample_rate
|
||||
audio = torch.zeros(1, 1, 0, dtype=torch.float32)
|
||||
return (
|
||||
{"waveform": audio, "sample_rate": sample_rate},
|
||||
"Generated 0.0s VoxCPM SRT audio from 0 subtitles",
|
||||
"No subtitles to process.",
|
||||
srt_content or "",
|
||||
)
|
||||
|
||||
has_overlaps = SRTOverlapHandler.detect_overlaps(subtitles)
|
||||
active_mode, mode_switched = (
|
||||
SRTOverlapHandler.handle_smart_natural_fallback(
|
||||
timing_mode, has_overlaps, "VoxCPM SRT"
|
||||
)
|
||||
)
|
||||
|
||||
print(
|
||||
f"🚀 VoxCPM SRT: Processing {len(subtitles)} subtitles "
|
||||
f"in {active_mode} mode"
|
||||
)
|
||||
audio_segments = []
|
||||
adjustments = []
|
||||
active_sample_rate = None
|
||||
|
||||
for subtitle_index, subtitle in enumerate(subtitles):
|
||||
self._check_interrupt(subtitle_index, len(subtitles))
|
||||
subtitle_text = (subtitle.text or "").strip()
|
||||
|
||||
if subtitle_text:
|
||||
records = self.processor.process_text(
|
||||
text=subtitle_text,
|
||||
voice_mapping=voice_mapping,
|
||||
seed=0 if seed == 0 else seed + subtitle_index,
|
||||
enable_chunking=False,
|
||||
enable_audio_cache=enable_audio_cache,
|
||||
apply_edit_postprocessing=True,
|
||||
)
|
||||
subtitle_audio, _ = self.processor.combine_audio_segments(
|
||||
records,
|
||||
method="auto",
|
||||
silence_ms=0,
|
||||
original_text=subtitle_text,
|
||||
return_info=True,
|
||||
)
|
||||
subtitle_rate = (
|
||||
int(records[0]["sample_rate"]) if records else self.sample_rate
|
||||
)
|
||||
else:
|
||||
subtitle_rate = self.sample_rate
|
||||
subtitle_audio = torch.zeros(
|
||||
1,
|
||||
max(0, int(round(subtitle.duration * subtitle_rate))),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
self._check_interrupt(subtitle_index, len(subtitles))
|
||||
subtitle_audio = self._normalize_audio(subtitle_audio)
|
||||
|
||||
if active_sample_rate is None:
|
||||
active_sample_rate = subtitle_rate
|
||||
elif subtitle_rate != active_sample_rate:
|
||||
raise ValueError(
|
||||
"VoxCPM model sample rate changed during one SRT job: "
|
||||
f"{active_sample_rate} -> {subtitle_rate}"
|
||||
)
|
||||
|
||||
audio_segments.append(subtitle_audio)
|
||||
natural_duration = subtitle_audio.shape[-1] / active_sample_rate
|
||||
adjustments.append(
|
||||
self._build_adjustment(
|
||||
subtitle_index, subtitle, natural_duration
|
||||
)
|
||||
)
|
||||
|
||||
self._check_interrupt()
|
||||
final_audio, replacement_adjustments, stretch_method = self._assemble(
|
||||
audio_segments,
|
||||
subtitles,
|
||||
active_mode,
|
||||
timing_params or {},
|
||||
active_sample_rate,
|
||||
)
|
||||
if replacement_adjustments is not None:
|
||||
adjustments = replacement_adjustments
|
||||
|
||||
reporter = SRTReportGenerator()
|
||||
timing_report = reporter.generate_timing_report(
|
||||
subtitles,
|
||||
adjustments,
|
||||
active_mode,
|
||||
has_overlaps,
|
||||
mode_switched,
|
||||
timing_mode if mode_switched else None,
|
||||
stretch_method,
|
||||
)
|
||||
adjusted_srt = reporter.generate_adjusted_srt_string(
|
||||
subtitles, adjustments, active_mode
|
||||
)
|
||||
|
||||
final_audio = final_audio.detach().float().cpu()
|
||||
if final_audio.dim() == 1:
|
||||
final_audio = final_audio.unsqueeze(0).unsqueeze(0)
|
||||
elif final_audio.dim() == 2:
|
||||
final_audio = final_audio.unsqueeze(0)
|
||||
elif final_audio.dim() != 3:
|
||||
raise ValueError(
|
||||
f"VoxCPM SRT assembler returned invalid shape {tuple(final_audio.shape)}"
|
||||
)
|
||||
|
||||
duration = final_audio.shape[-1] / active_sample_rate
|
||||
mode_description = active_mode
|
||||
if mode_switched:
|
||||
mode_description += f" (switched from {timing_mode} due to overlaps)"
|
||||
info = (
|
||||
f"Generated {duration:.1f}s VoxCPM SRT audio from "
|
||||
f"{len(subtitles)} subtitles using {mode_description} mode"
|
||||
)
|
||||
return (
|
||||
{"waveform": final_audio, "sample_rate": active_sample_rate},
|
||||
info,
|
||||
timing_report,
|
||||
adjusted_srt,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _assemble(
|
||||
audio_segments,
|
||||
subtitles,
|
||||
timing_mode: str,
|
||||
timing_params: Dict[str, Any],
|
||||
sample_rate: int,
|
||||
):
|
||||
if timing_mode == "stretch_to_fit":
|
||||
from engines.chatterbox.audio_timing import TimedAudioAssembler
|
||||
|
||||
assembler = TimedAudioAssembler(sample_rate)
|
||||
audio, method = assembler.assemble_timed_audio(
|
||||
audio_segments,
|
||||
[
|
||||
(subtitle.start_time, subtitle.end_time)
|
||||
for subtitle in subtitles
|
||||
],
|
||||
fade_duration=timing_params.get("fade_for_StretchToFit", 0.01),
|
||||
)
|
||||
return audio, None, method
|
||||
|
||||
assembler = AudioAssemblyEngine(sample_rate)
|
||||
cpu_device = torch.device("cpu")
|
||||
if timing_mode == "pad_with_silence":
|
||||
audio = assembler.assemble_with_overlaps(
|
||||
audio_segments, subtitles, cpu_device
|
||||
)
|
||||
return audio, None, None
|
||||
|
||||
timing_engine = TimingEngine(sample_rate)
|
||||
if timing_mode == "concatenate":
|
||||
replacements = timing_engine.calculate_concatenation_adjustments(
|
||||
audio_segments, subtitles
|
||||
)
|
||||
audio = assembler.assemble_concatenation(
|
||||
audio_segments,
|
||||
timing_params.get("fade_for_StretchToFit", 0.01),
|
||||
)
|
||||
return audio, replacements, None
|
||||
|
||||
replacements, processed_segments = (
|
||||
timing_engine.calculate_smart_timing_adjustments(
|
||||
audio_segments,
|
||||
subtitles,
|
||||
timing_params.get("timing_tolerance", 2.0),
|
||||
timing_params.get("max_stretch_ratio", 1.0),
|
||||
timing_params.get("min_stretch_ratio", 0.5),
|
||||
cpu_device,
|
||||
)
|
||||
)
|
||||
audio = assembler.assemble_smart_natural(
|
||||
audio_segments,
|
||||
processed_segments,
|
||||
replacements,
|
||||
subtitles,
|
||||
cpu_device,
|
||||
)
|
||||
return audio, replacements, None
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
[project]
|
||||
name = "tts_audio_suite"
|
||||
description = "TTS Audio Suite - Universal multi-engine TTS extension for ComfyUI with unified architecture supporting IndexTTS-2, ChatterBox, Chatterbox Multilingual TTS (Official 23-Lang), F5-TTS, Higgs Audio 2, VibeVoice, and RVC engines. It has character voice management, SRT subtitle TTS support, and audio processing capabilities."
|
||||
version = "5.6.0"
|
||||
version = "5.6.2"
|
||||
license = {file = "LICENSE"}
|
||||
|
||||
[project.urls]
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
@@ -41,3 +44,40 @@ def test_build_env_preserves_valid_pythonhashseed(monkeypatch, value):
|
||||
env = IsolatedRuntimeLauncher().build_env(profile)
|
||||
|
||||
assert env["PYTHONHASHSEED"] == value
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_audio8_worker_prioritizes_project_root_over_shadowing_utils(tmp_path):
|
||||
shadow_dir = tmp_path / "step_audio_editx_impl"
|
||||
shadow_dir.mkdir()
|
||||
(shadow_dir / "utils.py").write_text(
|
||||
'raise RuntimeError("shadow utils imported")\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
worker_path = REPO_ROOT / "utils" / "runtimes" / "workers" / "audio8_tts_worker.py"
|
||||
env = os.environ.copy()
|
||||
env["PYTHONPATH"] = os.pathsep.join((str(shadow_dir), str(REPO_ROOT)))
|
||||
completed = subprocess.run(
|
||||
[sys.executable, str(worker_path)],
|
||||
input=json.dumps(
|
||||
{
|
||||
"action": "ping",
|
||||
"request_id": "audio8-path-priority-test",
|
||||
}
|
||||
)
|
||||
+ "\n",
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
env=env,
|
||||
cwd=str(REPO_ROOT),
|
||||
timeout=30,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
response = json.loads(completed.stdout.strip())
|
||||
assert response["ok"] is True
|
||||
assert response["result"] == {"pong": True}
|
||||
|
||||
+37
-53
@@ -577,6 +577,30 @@ class DotsTTSCacheKeyGenerator(CacheKeyGenerator):
|
||||
return hashlib.md5(cache_string.encode()).hexdigest()
|
||||
|
||||
|
||||
class Audio8TTSCacheKeyGenerator(CacheKeyGenerator):
|
||||
"""Cache key generator for official Audio8 TTS inference."""
|
||||
|
||||
def generate_cache_key(self, **params) -> str:
|
||||
cache_data = {
|
||||
'text': params.get('text', ''),
|
||||
'audio_component': params.get('audio_component', ''),
|
||||
'reference_text': params.get('reference_text', ''),
|
||||
'model_variant': params.get('model_variant', 'Audio8-TTS-Preview-0.6b'),
|
||||
'max_new_tokens': int(params.get('max_new_tokens', 1024)),
|
||||
'retry_max_new_tokens': int(params.get('retry_max_new_tokens', 2000)),
|
||||
'temperature': round(float(params.get('temperature', 0.8)), 4),
|
||||
'top_p': round(float(params.get('top_p', 0.95)), 4),
|
||||
'top_k': int(params.get('top_k', 50)),
|
||||
'do_sample': bool(params.get('do_sample', True)),
|
||||
'dtype': params.get('dtype', 'auto'),
|
||||
'device': params.get('device', 'auto'),
|
||||
'seed': int(params.get('seed', 42)),
|
||||
'character': params.get('character', 'narrator'),
|
||||
'engine': 'audio8_tts',
|
||||
}
|
||||
return hashlib.md5(str(sorted(cache_data.items())).encode()).hexdigest()
|
||||
|
||||
|
||||
class DramaBoxCacheKeyGenerator(CacheKeyGenerator):
|
||||
"""Cache key generator for official DramaBox inference."""
|
||||
|
||||
@@ -694,42 +718,6 @@ class OmniVoiceCacheKeyGenerator(CacheKeyGenerator):
|
||||
return hashlib.md5(cache_string.encode()).hexdigest()
|
||||
|
||||
|
||||
class VoxCPMCacheKeyGenerator(CacheKeyGenerator):
|
||||
"""Cache-key generator for all official VoxCPM checkpoint generations."""
|
||||
|
||||
def generate_cache_key(self, **params) -> str:
|
||||
cache_data = {
|
||||
'text': params.get('text', ''),
|
||||
'audio_component': params.get('audio_component', ''),
|
||||
'reference_text': params.get('reference_text', ''),
|
||||
'model_variant': params.get('model_variant', 'VoxCPM2'),
|
||||
'architecture': params.get('architecture', ''),
|
||||
'voice_instruction': params.get('voice_instruction', ''),
|
||||
'cfg_value': round(float(params.get('cfg_value', 2.0)), 3),
|
||||
'inference_timesteps': int(params.get('inference_timesteps', 10)),
|
||||
'min_len': int(params.get('min_len', 2)),
|
||||
'max_len': int(params.get('max_len', 4096)),
|
||||
'normalize_text': bool(params.get('normalize_text', False)),
|
||||
'retry_badcase': bool(params.get('retry_badcase', True)),
|
||||
'retry_badcase_max_times': int(params.get('retry_badcase_max_times', 3)),
|
||||
'retry_badcase_ratio_threshold': round(
|
||||
float(params.get('retry_badcase_ratio_threshold', 6.0)), 3
|
||||
),
|
||||
'seed': int(params.get('seed', 0)),
|
||||
'device': params.get('device', 'auto'),
|
||||
'optimize': bool(params.get('optimize', False)),
|
||||
'runtime_mode': params.get('runtime_mode', 'main_environment'),
|
||||
'runtime_profile': params.get('runtime_profile'),
|
||||
'model_path': params.get('model_path', ''),
|
||||
'load_denoiser': bool(params.get('load_denoiser', False)),
|
||||
'sample_rate': int(params.get('sample_rate', 48000)),
|
||||
'character': params.get('character', 'narrator'),
|
||||
'engine': 'voxcpm',
|
||||
}
|
||||
cache_string = str(sorted(cache_data.items()))
|
||||
return hashlib.md5(cache_string.encode()).hexdigest()
|
||||
|
||||
|
||||
class AudioCache:
|
||||
"""Unified audio cache manager for all TTS engines."""
|
||||
|
||||
@@ -746,10 +734,10 @@ class AudioCache:
|
||||
'cosyvoice': CosyVoiceCacheKeyGenerator(),
|
||||
'qwen3_tts': Qwen3TTSCacheKeyGenerator(),
|
||||
'dots_tts': DotsTTSCacheKeyGenerator(),
|
||||
'audio8_tts': Audio8TTSCacheKeyGenerator(),
|
||||
'dramabox': DramaBoxCacheKeyGenerator(),
|
||||
'fish_audio_s2': FishAudioS2CacheKeyGenerator(),
|
||||
'omnivoice': OmniVoiceCacheKeyGenerator(),
|
||||
'voxcpm': VoxCPMCacheKeyGenerator(),
|
||||
'moss_tts': MossTTSCacheKeyGenerator(),
|
||||
'moss_soundeffect_v2': MossSoundEffectV2CacheKeyGenerator(),
|
||||
'echo_tts': EchoTTSCacheKeyGenerator()
|
||||
@@ -813,12 +801,7 @@ class AudioCache:
|
||||
|
||||
return cache_fn
|
||||
|
||||
def _calculate_duration(
|
||||
self,
|
||||
audio_tensor: torch.Tensor,
|
||||
engine_type: str,
|
||||
sample_rate: Optional[int] = None,
|
||||
) -> float:
|
||||
def _calculate_duration(self, audio_tensor: torch.Tensor, engine_type: str) -> float:
|
||||
"""Calculate audio duration based on engine type."""
|
||||
if audio_tensor.dim() == 1:
|
||||
num_samples = audio_tensor.shape[0]
|
||||
@@ -828,16 +811,17 @@ class AudioCache:
|
||||
num_samples = audio_tensor.numel()
|
||||
|
||||
# Use engine-specific sample rates
|
||||
if sample_rate is None:
|
||||
if engine_type in ('dots_tts', 'dramabox', 'moss_soundeffect_v2', 'voxcpm'):
|
||||
sample_rate = 48000
|
||||
elif engine_type in ('f5tts', 'step_audio_editx', 'qwen3_tts', 'moss_tts', 'higgs_audio_v3', 'omnivoice'):
|
||||
sample_rate = 24000
|
||||
elif engine_type in ('index_tts', 'cosyvoice'):
|
||||
sample_rate = 22050
|
||||
else:
|
||||
sample_rate = 44100
|
||||
return num_samples / int(sample_rate)
|
||||
if engine_type in ('dots_tts', 'dramabox', 'moss_soundeffect_v2'):
|
||||
sample_rate = 48000
|
||||
elif engine_type in ('f5tts', 'step_audio_editx', 'qwen3_tts', 'moss_tts', 'higgs_audio_v3', 'omnivoice'):
|
||||
sample_rate = 24000
|
||||
elif engine_type in ('index_tts', 'cosyvoice'):
|
||||
sample_rate = 22050
|
||||
elif engine_type == 'audio8_tts':
|
||||
sample_rate = 44100
|
||||
else:
|
||||
sample_rate = 44100
|
||||
return num_samples / sample_rate
|
||||
|
||||
def clear_cache(self):
|
||||
"""Clear all cached audio."""
|
||||
|
||||
@@ -17,24 +17,21 @@ class ConfigSanitizer:
|
||||
|
||||
# Parameters that should be floats
|
||||
FLOAT_PARAMS = {
|
||||
'repetition_penalty', 'temperature', 'top_p', 'emotion_alpha',
|
||||
'length_penalty', 'speed', 'target_rms', 'cross_fade_duration',
|
||||
'cfg_weight', 'exaggeration', 'cfg_strength', 'min_p', 'fade_for_StretchToFit',
|
||||
'max_stretch_ratio', 'min_stretch_ratio', 'timing_tolerance',
|
||||
'cfg_value', 'retry_badcase_ratio_threshold'
|
||||
'repetition_penalty', 'temperature', 'top_p', 'emotion_alpha',
|
||||
'length_penalty', 'speed', 'target_rms', 'cross_fade_duration',
|
||||
'cfg_weight', 'exaggeration', 'cfg_strength', 'min_p', 'fade_for_StretchToFit',
|
||||
'max_stretch_ratio', 'min_stretch_ratio', 'timing_tolerance'
|
||||
}
|
||||
|
||||
# Parameters that should be strings
|
||||
STRING_PARAMS = {
|
||||
'device', 'language', 'model', 'model_version', 'model_path',
|
||||
'model_variant', 'model_name', 'model_role', 'mode', 'voice_instruction'
|
||||
'device', 'language', 'model', 'model_version', 'model_path'
|
||||
}
|
||||
|
||||
# Parameters that should be integers
|
||||
INT_PARAMS = {
|
||||
'top_k', 'num_beams', 'max_mel_tokens', 'interval_silence',
|
||||
'max_text_tokens_per_segment', 'seed', 'refinement_passes',
|
||||
'inference_timesteps', 'min_len', 'max_len', 'retry_badcase_max_times'
|
||||
'top_k', 'num_beams', 'max_mel_tokens', 'interval_silence',
|
||||
'max_text_tokens_per_segment', 'seed', 'refinement_passes'
|
||||
}
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -157,6 +157,15 @@ ENGINE_REGISTRY: Dict[str, EngineCapabilities] = {
|
||||
fallback_languages=["English"],
|
||||
),
|
||||
|
||||
"audio8_tts": EngineCapabilities(
|
||||
supports_voice_conversion=False,
|
||||
multilingual_model_switching=False,
|
||||
can_corrupt_on_reload=False,
|
||||
fallback_languages=["English"],
|
||||
supports_runtime_isolation=True,
|
||||
default_runtime_profile="vibevoice_transformers4_shared",
|
||||
),
|
||||
|
||||
"dramabox": EngineCapabilities(
|
||||
supports_voice_conversion=False,
|
||||
multilingual_model_switching=False,
|
||||
@@ -181,15 +190,6 @@ ENGINE_REGISTRY: Dict[str, EngineCapabilities] = {
|
||||
fallback_languages=["English"],
|
||||
),
|
||||
|
||||
"voxcpm": EngineCapabilities(
|
||||
supports_voice_conversion=False,
|
||||
multilingual_model_switching=False,
|
||||
can_corrupt_on_reload=False,
|
||||
fallback_languages=["English"],
|
||||
supports_training=True,
|
||||
training_modes=["lora_adapter"],
|
||||
),
|
||||
|
||||
"qwen3_asr": EngineCapabilities(
|
||||
supports_voice_conversion=False,
|
||||
multilingual_model_switching=False,
|
||||
|
||||
@@ -11,9 +11,14 @@ from typing import Any, Dict, Optional, Callable, Union
|
||||
from pathlib import Path
|
||||
|
||||
from utils.models.comfyui_model_wrapper import tts_model_manager, ModelInfo
|
||||
from utils.models.factory_config import ModelLoadConfig, runtime_uses_isolation
|
||||
from utils.models.factory_config import (
|
||||
ModelLoadConfig,
|
||||
RUNTIME_MODE_SHARED,
|
||||
runtime_uses_isolation,
|
||||
)
|
||||
from utils.models.engine_registry import get_default_runtime_profile
|
||||
from utils.runtimes import (
|
||||
build_audio8_tts_isolated_proxy,
|
||||
build_fish_audio_s2_proxy,
|
||||
build_higgs_audio_isolated_proxy,
|
||||
build_qwen3_asr_isolated_proxy,
|
||||
@@ -124,6 +129,10 @@ class UnifiedModelInterface:
|
||||
# Check PyTorch consistency on first model load
|
||||
self._check_pytorch_consistency()
|
||||
|
||||
if config.engine_name == "audio8_tts":
|
||||
config.runtime_mode = RUNTIME_MODE_SHARED
|
||||
config.runtime_profile = "vibevoice_transformers4_shared"
|
||||
|
||||
if runtime_uses_isolation(config.runtime_mode):
|
||||
return self._load_isolated_model(config, force_reload=force_reload)
|
||||
|
||||
@@ -148,7 +157,7 @@ class UnifiedModelInterface:
|
||||
# check if a DIFFERENT variant is already loaded and unload it to prevent device conflicts
|
||||
# Only applies to engines where model variants are mutually exclusive
|
||||
if config.engine_name in (
|
||||
"qwen3_tts", "moss_tts", "higgs_audio_v3", "dramabox", "voxcpm"
|
||||
"qwen3_tts", "moss_tts", "higgs_audio_v3", "dramabox"
|
||||
):
|
||||
# Check for any cached mutually-exclusive model variant for this engine
|
||||
cached_prefix = f"{config.engine_name}_tts_"
|
||||
@@ -219,6 +228,13 @@ class UnifiedModelInterface:
|
||||
|
||||
cached = self._isolated_model_cache.get(cache_key)
|
||||
if cached is not None:
|
||||
ensure_registration = getattr(
|
||||
cached,
|
||||
"_ensure_comfy_model_registration",
|
||||
None,
|
||||
)
|
||||
if callable(ensure_registration):
|
||||
ensure_registration()
|
||||
return cached
|
||||
|
||||
profile_name = config.runtime_profile or get_default_runtime_profile(config.engine_name or "")
|
||||
@@ -244,6 +260,11 @@ class UnifiedModelInterface:
|
||||
self._isolated_model_cache[cache_key] = proxy
|
||||
return proxy
|
||||
|
||||
if config.engine_name == "audio8_tts" and config.model_type == "tts":
|
||||
proxy = build_audio8_tts_isolated_proxy(config)
|
||||
self._isolated_model_cache[cache_key] = proxy
|
||||
return proxy
|
||||
|
||||
if config.engine_name == "qwen3_asr" and config.model_type in ("asr", "aligner"):
|
||||
proxy = build_qwen3_asr_isolated_proxy(config)
|
||||
self._isolated_model_cache[cache_key] = proxy
|
||||
@@ -1751,6 +1772,19 @@ def register_dots_tts_factory():
|
||||
unified_model_interface.register_model_factory("dots_tts", "tts", dots_tts_factory)
|
||||
|
||||
|
||||
def register_audio8_tts_factory():
|
||||
"""Register Audio8 TTS through its required Transformers 4 runtime."""
|
||||
def audio8_tts_factory(config: ModelLoadConfig):
|
||||
del config
|
||||
raise RuntimeError(
|
||||
"Audio8 TTS reached its embedded factory unexpectedly; all "
|
||||
"Audio8 loads must be routed through the shared Transformers 4 "
|
||||
"runtime"
|
||||
)
|
||||
|
||||
unified_model_interface.register_model_factory("audio8_tts", "tts", audio8_tts_factory)
|
||||
|
||||
|
||||
def register_dramabox_factory():
|
||||
"""Register official DramaBox in the main Transformers 5 environment."""
|
||||
def dramabox_factory(config: ModelLoadConfig):
|
||||
@@ -1807,42 +1841,6 @@ def register_omnivoice_factory():
|
||||
unified_model_interface.register_model_factory("omnivoice", "tts", omnivoice_factory)
|
||||
|
||||
|
||||
def register_voxcpm_factory():
|
||||
"""Register all official VoxCPM generations in the main environment."""
|
||||
|
||||
def voxcpm_factory(config: ModelLoadConfig):
|
||||
from engines.voxcpm.voxcpm_downloader import VoxCPMDownloader
|
||||
from engines.voxcpm.voxcpm_engine import VoxCPMEngine
|
||||
|
||||
downloader = VoxCPMDownloader()
|
||||
selection = (
|
||||
config.model_path
|
||||
or config.model_name
|
||||
or VoxCPMDownloader.DEFAULT_MODEL
|
||||
)
|
||||
resolved_path = downloader.resolve_model_path(selection)
|
||||
spec = downloader.get_model_spec(selection, resolved_path)
|
||||
model_name = spec["canonical"]
|
||||
additional_params = config.additional_params or {}
|
||||
|
||||
print(f"🔄 Loading VoxCPM via unified interface: {model_name}")
|
||||
engine = VoxCPMEngine(
|
||||
model_name=model_name,
|
||||
device=config.device or "auto",
|
||||
optimize=bool(additional_params.get("optimize", False)),
|
||||
model_dir=resolved_path,
|
||||
lora_adapter=additional_params.get("lora_adapter"),
|
||||
)
|
||||
engine._ensure_runtime_loaded()
|
||||
print(
|
||||
f"✅ VoxCPM '{model_name}' loaded successfully "
|
||||
f"({engine.architecture}, {engine.sample_rate} Hz)"
|
||||
)
|
||||
return engine
|
||||
|
||||
unified_model_interface.register_model_factory("voxcpm", "tts", voxcpm_factory)
|
||||
|
||||
|
||||
def register_qwen3_asr_factory():
|
||||
"""Register Qwen3-ASR model factory"""
|
||||
def qwen3_asr_factory(config: ModelLoadConfig):
|
||||
@@ -2054,9 +2052,9 @@ def initialize_all_factories():
|
||||
register_qwen3_tts_factory()
|
||||
register_fish_audio_s2_factory()
|
||||
register_dots_tts_factory()
|
||||
register_audio8_tts_factory()
|
||||
register_dramabox_factory()
|
||||
register_omnivoice_factory()
|
||||
register_voxcpm_factory()
|
||||
register_qwen3_asr_factory()
|
||||
register_qwen3_aligner_factory()
|
||||
register_granite_asr_factory()
|
||||
|
||||
@@ -6,6 +6,7 @@ separate Python environments without changing the existing ComfyUI-side UI.
|
||||
"""
|
||||
|
||||
from .bootstrap import ensure_runtime, resolve_runtime_dir, resolve_runtime_python
|
||||
from .audio8_tts_proxy import Audio8TTSIsolatedProxy, build_audio8_tts_isolated_proxy
|
||||
from .higgs_audio_proxy import HiggsAudioIsolatedProxy, build_higgs_audio_isolated_proxy
|
||||
from .fish_audio_s2_proxy import FishAudioS2Proxy, build_fish_audio_s2_proxy
|
||||
from .launcher import IsolatedRuntimeLauncher
|
||||
@@ -20,6 +21,8 @@ __all__ = [
|
||||
"ensure_runtime",
|
||||
"resolve_runtime_dir",
|
||||
"resolve_runtime_python",
|
||||
"Audio8TTSIsolatedProxy",
|
||||
"build_audio8_tts_isolated_proxy",
|
||||
"HiggsAudioIsolatedProxy",
|
||||
"IsolatedRuntimeLauncher",
|
||||
"RuntimeProfile",
|
||||
|
||||
@@ -0,0 +1,546 @@
|
||||
"""Audio8 TTS proxy for the shared official Transformers 4 runtime."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import uuid
|
||||
import weakref
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from utils.models.factory_config import ModelLoadConfig
|
||||
from .bootstrap import PROJECT_ROOT, ensure_runtime
|
||||
from .launcher import IsolatedRuntimeLauncher
|
||||
from .profiles import RuntimeProfile, get_runtime_profile
|
||||
from .protocol import RuntimeJobRequest
|
||||
from .session import JsonLineWorkerSession
|
||||
|
||||
|
||||
_INTERRUPT_EVENTS: set[threading.Event] = set()
|
||||
_INTERRUPT_EVENTS_LOCK = threading.Lock()
|
||||
_INTERRUPT_DISPATCH_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def _install_comfy_interrupt_dispatcher() -> None:
|
||||
"""Forward Comfy's server interrupt edge to active Audio8 requests."""
|
||||
server_module = sys.modules.get("server")
|
||||
comfy_nodes = getattr(server_module, "nodes", None)
|
||||
if comfy_nodes is None:
|
||||
comfy_nodes = sys.modules.get("nodes")
|
||||
if comfy_nodes is None:
|
||||
return
|
||||
|
||||
with _INTERRUPT_DISPATCH_LOCK:
|
||||
current_interrupt = getattr(comfy_nodes, "interrupt_processing", None)
|
||||
if not callable(current_interrupt):
|
||||
return
|
||||
if getattr(current_interrupt, "_tts_audio8_dispatcher", False):
|
||||
if (
|
||||
getattr(current_interrupt, "_tts_audio8_events", None)
|
||||
is _INTERRUPT_EVENTS
|
||||
):
|
||||
return
|
||||
current_interrupt = getattr(
|
||||
current_interrupt,
|
||||
"_tts_audio8_original_interrupt",
|
||||
current_interrupt,
|
||||
)
|
||||
|
||||
def dispatch_interrupt(value=True):
|
||||
# TTS Audio Suite patch: preserve Comfy's interrupt edge for
|
||||
# blocking Audio8 subprocess requests without replacing the
|
||||
# process-wide callback separately for every generation.
|
||||
result = current_interrupt(value)
|
||||
if value:
|
||||
with _INTERRUPT_EVENTS_LOCK:
|
||||
active_events = tuple(_INTERRUPT_EVENTS)
|
||||
for active_event in active_events:
|
||||
active_event.set()
|
||||
return result
|
||||
|
||||
dispatch_interrupt._tts_audio8_dispatcher = True
|
||||
dispatch_interrupt._tts_audio8_original_interrupt = current_interrupt
|
||||
dispatch_interrupt._tts_audio8_events = _INTERRUPT_EVENTS
|
||||
comfy_nodes.interrupt_processing = dispatch_interrupt
|
||||
|
||||
|
||||
def _register_interrupt_event(interrupt_event: threading.Event) -> None:
|
||||
_install_comfy_interrupt_dispatcher()
|
||||
with _INTERRUPT_EVENTS_LOCK:
|
||||
_INTERRUPT_EVENTS.add(interrupt_event)
|
||||
|
||||
|
||||
def _unregister_interrupt_event(interrupt_event: threading.Event) -> None:
|
||||
with _INTERRUPT_EVENTS_LOCK:
|
||||
_INTERRUPT_EVENTS.discard(interrupt_event)
|
||||
|
||||
|
||||
class Audio8TTSIsolatedProxy:
|
||||
"""Expose Audio8's engine API while inference runs under Transformers 4."""
|
||||
|
||||
SAMPLE_RATE = 44100
|
||||
|
||||
def __init__(self, config: ModelLoadConfig, profile: RuntimeProfile):
|
||||
self.config = config
|
||||
self.profile = profile
|
||||
self.model_name = config.model_name or "Audio8-TTS-Preview-0.6b"
|
||||
self.model_path = config.model_path
|
||||
self.device = str(config.device)
|
||||
self.current_device = self.device
|
||||
self.dtype_name = str((config.additional_params or {}).get("dtype", "auto"))
|
||||
self.dtype = self._resolve_dtype(self.dtype_name)
|
||||
self.load_device = self.current_loaded_device()
|
||||
self.offload_device = torch.device("cpu")
|
||||
self.currently_used = True
|
||||
self.model = self
|
||||
self.engine = self
|
||||
self.parent = None
|
||||
self.model_options = {}
|
||||
self.model_keys = set()
|
||||
estimate_gib = (
|
||||
3.5
|
||||
if self.device.startswith("cpu") or self.dtype == torch.float32
|
||||
else 1.75
|
||||
)
|
||||
self._estimated_memory_size = int(estimate_gib * 1024**3)
|
||||
self._comfy_loaded_model = None
|
||||
self._comfy_model_management = None
|
||||
self._initialized = False
|
||||
|
||||
launcher = IsolatedRuntimeLauncher(runtime_root=str(PROJECT_ROOT))
|
||||
python_path = ensure_runtime(profile)
|
||||
worker_script = PROJECT_ROOT / "utils/runtimes/workers/audio8_tts_worker.py"
|
||||
self._session = JsonLineWorkerSession(
|
||||
python_path=str(python_path),
|
||||
worker_script=str(worker_script),
|
||||
env=launcher.build_env(profile),
|
||||
)
|
||||
try:
|
||||
self._initialize_remote_engine()
|
||||
except Exception:
|
||||
self._session.close()
|
||||
raise
|
||||
self._register_with_comfy_model_management()
|
||||
|
||||
def _resolve_dtype(self, dtype_name: str) -> torch.dtype:
|
||||
normalized = str(dtype_name or "auto").lower()
|
||||
if not self.device.startswith("cuda"):
|
||||
return torch.float32
|
||||
if normalized == "float32":
|
||||
return torch.float32
|
||||
if normalized == "float16":
|
||||
return torch.float16
|
||||
if normalized == "bfloat16":
|
||||
return torch.bfloat16
|
||||
if torch.cuda.is_available():
|
||||
major, _minor = torch.cuda.get_device_capability(torch.device(self.device))
|
||||
return torch.bfloat16 if major >= 8 else torch.float16
|
||||
return torch.float16
|
||||
|
||||
def _request(self, action: str, payload: Dict[str, Any]):
|
||||
response = self._session.request(
|
||||
RuntimeJobRequest(
|
||||
engine_name="audio8_tts",
|
||||
action=action,
|
||||
model_name=self.model_name,
|
||||
device=self.device,
|
||||
runtime_profile=self.profile.name,
|
||||
payload=payload,
|
||||
request_id=str(uuid.uuid4()),
|
||||
)
|
||||
)
|
||||
if not response.ok:
|
||||
details = response.error or (
|
||||
f"Audio8 TTS isolated action '{action}' failed"
|
||||
)
|
||||
if response.logs:
|
||||
details = f"{details}\n" + "\n".join(response.logs)
|
||||
raise RuntimeError(details)
|
||||
return response
|
||||
|
||||
def _request_generation(self, payload: Dict[str, Any]):
|
||||
"""Run generation while the main process monitors Comfy interruption."""
|
||||
try:
|
||||
import comfy.model_management as model_management
|
||||
except Exception:
|
||||
return self._request("generate", payload)
|
||||
|
||||
checker = getattr(
|
||||
model_management,
|
||||
"throw_exception_if_processing_interrupted",
|
||||
None,
|
||||
)
|
||||
if not callable(checker):
|
||||
return self._request("generate", payload)
|
||||
|
||||
interrupt_event = threading.Event()
|
||||
_register_interrupt_event(interrupt_event)
|
||||
result: Dict[str, Any] = {}
|
||||
request_thread = None
|
||||
request_started = False
|
||||
|
||||
def run_request() -> None:
|
||||
try:
|
||||
result["response"] = self._request("generate", payload)
|
||||
except BaseException as exc:
|
||||
result["error"] = exc
|
||||
|
||||
try:
|
||||
checker()
|
||||
request_thread = threading.Thread(
|
||||
target=run_request,
|
||||
name="audio8-tts-worker-request",
|
||||
daemon=True,
|
||||
)
|
||||
request_thread.start()
|
||||
request_started = True
|
||||
|
||||
while request_thread.is_alive():
|
||||
request_thread.join(timeout=0.05)
|
||||
if interrupt_event.is_set():
|
||||
raise model_management.InterruptProcessingException()
|
||||
checker()
|
||||
except BaseException as exc:
|
||||
if request_started:
|
||||
self._initialized = False
|
||||
try:
|
||||
self._session.terminate_now()
|
||||
except Exception:
|
||||
pass
|
||||
request_thread.join(timeout=5)
|
||||
if isinstance(
|
||||
exc,
|
||||
model_management.InterruptProcessingException,
|
||||
):
|
||||
interrupt_setter = getattr(
|
||||
model_management,
|
||||
"interrupt_current_processing",
|
||||
None,
|
||||
)
|
||||
if callable(interrupt_setter):
|
||||
interrupt_setter(False)
|
||||
raise
|
||||
finally:
|
||||
_unregister_interrupt_event(interrupt_event)
|
||||
|
||||
if "error" in result:
|
||||
raise result["error"]
|
||||
return result["response"]
|
||||
|
||||
def _initialize_remote_engine(self) -> None:
|
||||
self._request(
|
||||
"initialize",
|
||||
{
|
||||
"model_path": self.model_path,
|
||||
"dtype": self.dtype_name,
|
||||
},
|
||||
)
|
||||
self._initialized = True
|
||||
print(f"✅ Audio8 TTS isolated runtime ready ({self.profile.name})")
|
||||
|
||||
def _ensure_remote_engine(self) -> None:
|
||||
process = getattr(self._session, "_process", None)
|
||||
if not self._initialized or process is None or process.poll() is not None:
|
||||
self._initialize_remote_engine()
|
||||
|
||||
@staticmethod
|
||||
def _serialize_reference_audio(
|
||||
reference_audio: Any,
|
||||
bundle_dir: Path,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
if reference_audio is None:
|
||||
return None
|
||||
if isinstance(reference_audio, (str, os.PathLike)):
|
||||
return {
|
||||
"kind": "audio_path",
|
||||
"audio_path": os.fspath(reference_audio),
|
||||
}
|
||||
|
||||
waveform = None
|
||||
sample_rate = None
|
||||
if isinstance(reference_audio, dict):
|
||||
waveform = reference_audio.get(
|
||||
"waveform",
|
||||
reference_audio.get("array"),
|
||||
)
|
||||
sample_rate = reference_audio.get(
|
||||
"sample_rate",
|
||||
reference_audio.get("sampling_rate"),
|
||||
)
|
||||
elif isinstance(reference_audio, (tuple, list)) and len(reference_audio) == 2:
|
||||
waveform, sample_rate = reference_audio
|
||||
|
||||
if waveform is None or sample_rate is None:
|
||||
raise TypeError(
|
||||
"Audio8 TTS isolated reference audio must be a path, "
|
||||
"audio dict, or (waveform, sample_rate) pair"
|
||||
)
|
||||
|
||||
tensor_path = bundle_dir / "reference_audio.pt"
|
||||
waveform_tensor = torch.as_tensor(waveform).detach().float().cpu()
|
||||
torch.save(
|
||||
{
|
||||
"waveform": waveform_tensor,
|
||||
"sample_rate": int(sample_rate),
|
||||
},
|
||||
tensor_path,
|
||||
)
|
||||
return {
|
||||
"kind": "tensor_path",
|
||||
"tensor_path": str(tensor_path),
|
||||
}
|
||||
|
||||
def generate(
|
||||
self,
|
||||
text: str,
|
||||
reference_audio: Any = None,
|
||||
reference_text: Optional[str] = None,
|
||||
*,
|
||||
max_new_tokens: int = 1024,
|
||||
retry_max_new_tokens: int = 2000,
|
||||
temperature: float = 0.8,
|
||||
top_p: float = 0.95,
|
||||
top_k: int = 50,
|
||||
do_sample: bool = True,
|
||||
seed: int = 42,
|
||||
) -> torch.Tensor:
|
||||
self._ensure_remote_engine()
|
||||
with tempfile.TemporaryDirectory(prefix="tts_audio8_iso_") as temp_dir:
|
||||
bundle_dir = Path(temp_dir)
|
||||
output_path = bundle_dir / "result.pt"
|
||||
serialized_reference = self._serialize_reference_audio(
|
||||
reference_audio,
|
||||
bundle_dir,
|
||||
)
|
||||
self._request_generation(
|
||||
{
|
||||
"text": text,
|
||||
"reference_audio": serialized_reference,
|
||||
"reference_text": reference_text,
|
||||
"max_new_tokens": int(max_new_tokens),
|
||||
"retry_max_new_tokens": int(retry_max_new_tokens),
|
||||
"temperature": float(temperature),
|
||||
"top_p": float(top_p),
|
||||
"top_k": int(top_k),
|
||||
"do_sample": bool(do_sample),
|
||||
"seed": int(seed),
|
||||
"output_path": str(output_path),
|
||||
}
|
||||
)
|
||||
if not output_path.exists():
|
||||
raise RuntimeError(
|
||||
"Audio8 TTS isolated worker returned no output payload"
|
||||
)
|
||||
result = torch.load(output_path, map_location="cpu")
|
||||
audio = torch.as_tensor(result["audio"]).detach().float().cpu()
|
||||
sample_rate = int(result["sample_rate"])
|
||||
if sample_rate != self.SAMPLE_RATE:
|
||||
raise RuntimeError(
|
||||
f"Audio8 TTS worker returned {sample_rate} Hz; "
|
||||
f"expected {self.SAMPLE_RATE} Hz"
|
||||
)
|
||||
if audio.ndim == 1:
|
||||
audio = audio.unsqueeze(0)
|
||||
if audio.ndim != 2 or audio.shape[0] != 1:
|
||||
raise RuntimeError(
|
||||
"Audio8 TTS worker returned invalid audio shape "
|
||||
f"{tuple(audio.shape)}"
|
||||
)
|
||||
return audio.contiguous()
|
||||
|
||||
def _register_with_comfy_model_management(self) -> None:
|
||||
self._ensure_comfy_model_registration()
|
||||
|
||||
def _ensure_comfy_model_registration(self) -> None:
|
||||
try:
|
||||
import comfy.model_management as model_management
|
||||
except Exception:
|
||||
return
|
||||
if not hasattr(model_management, "LoadedModel") or not hasattr(
|
||||
model_management,
|
||||
"current_loaded_models",
|
||||
):
|
||||
return
|
||||
for loaded_model in model_management.current_loaded_models:
|
||||
wrapper = getattr(loaded_model, "_tts_wrapper_ref", None)
|
||||
real_model = getattr(loaded_model, "real_model", None)
|
||||
resolved_real_model = (
|
||||
real_model()
|
||||
if isinstance(real_model, weakref.ReferenceType)
|
||||
else real_model
|
||||
)
|
||||
if wrapper is self or resolved_real_model is self:
|
||||
self._comfy_loaded_model = loaded_model
|
||||
self._comfy_model_management = model_management
|
||||
return
|
||||
try:
|
||||
loaded_model = model_management.LoadedModel(self)
|
||||
loaded_model.real_model = weakref.ref(self)
|
||||
finalizer = getattr(
|
||||
model_management,
|
||||
"cleanup_models",
|
||||
lambda: None,
|
||||
)
|
||||
loaded_model.model_finalizer = weakref.finalize(self, finalizer)
|
||||
loaded_model._tts_wrapper_ref = self
|
||||
model_management.current_loaded_models.insert(0, loaded_model)
|
||||
self._comfy_loaded_model = loaded_model
|
||||
self._comfy_model_management = model_management
|
||||
except Exception as exc:
|
||||
print(
|
||||
"⚠️ Failed to register isolated Audio8 TTS runtime with "
|
||||
f"ComfyUI model management: {exc}"
|
||||
)
|
||||
|
||||
def _unregister_from_comfy_model_management(self) -> None:
|
||||
model_management = self._comfy_model_management
|
||||
loaded_model = self._comfy_loaded_model
|
||||
if model_management is None or loaded_model is None:
|
||||
return
|
||||
try:
|
||||
if loaded_model in model_management.current_loaded_models:
|
||||
model_management.current_loaded_models.remove(loaded_model)
|
||||
except Exception as exc:
|
||||
print(
|
||||
"⚠️ Failed to remove isolated Audio8 TTS runtime from "
|
||||
f"ComfyUI tracking: {exc}"
|
||||
)
|
||||
finally:
|
||||
self._comfy_loaded_model = None
|
||||
self._comfy_model_management = None
|
||||
|
||||
def to(self, device):
|
||||
self.device = str(device)
|
||||
self.current_device = self.device
|
||||
self.load_device = self.current_loaded_device()
|
||||
return self
|
||||
|
||||
def eval(self):
|
||||
return self
|
||||
|
||||
def loaded_size(self) -> int:
|
||||
if self._initialized and self.device.startswith("cuda"):
|
||||
return self._estimated_memory_size
|
||||
return 0
|
||||
|
||||
def model_size(self) -> int:
|
||||
return self._estimated_memory_size
|
||||
|
||||
def model_memory(self) -> int:
|
||||
return self._estimated_memory_size
|
||||
|
||||
def get_ram_usage(self) -> int:
|
||||
return self._estimated_memory_size
|
||||
|
||||
def model_offloaded_memory(self) -> int:
|
||||
return (
|
||||
0
|
||||
if self._initialized and self.device.startswith("cuda")
|
||||
else self._estimated_memory_size
|
||||
)
|
||||
|
||||
def model_mmap_residency(self, free: bool = False) -> tuple[int, int]:
|
||||
del free
|
||||
return 0, self._estimated_memory_size
|
||||
|
||||
def pinned_memory_size(self) -> int:
|
||||
return 0
|
||||
|
||||
def lowvram_patch_counter(self) -> int:
|
||||
return 0
|
||||
|
||||
def model_dtype(self):
|
||||
return self.dtype
|
||||
|
||||
def model_patches_models(self):
|
||||
return ()
|
||||
|
||||
def model_patches_to(self, target) -> None:
|
||||
if isinstance(target, torch.dtype):
|
||||
self.dtype = target
|
||||
elif isinstance(target, torch.device):
|
||||
self.device = str(target)
|
||||
|
||||
def is_dynamic(self) -> bool:
|
||||
return False
|
||||
|
||||
def current_loaded_device(self) -> torch.device:
|
||||
if self.device.startswith("cuda") and torch.cuda.is_available():
|
||||
return torch.device(self.device)
|
||||
return torch.device("cpu")
|
||||
|
||||
def partially_load(
|
||||
self,
|
||||
device,
|
||||
extra_memory,
|
||||
force_patch_weights=False,
|
||||
) -> int:
|
||||
del extra_memory, force_patch_weights
|
||||
self.to(device)
|
||||
return 0
|
||||
|
||||
def partially_unload_ram(self, ram_to_unload: int) -> int:
|
||||
return self.partially_unload("cpu", ram_to_unload)
|
||||
|
||||
def partially_unload(self, device, memory_to_free) -> int:
|
||||
del device, memory_to_free
|
||||
freed = self.loaded_size()
|
||||
self.cleanup(unregister=False)
|
||||
return freed
|
||||
|
||||
def model_unload(
|
||||
self,
|
||||
memory_to_free=None,
|
||||
unpatch_weights=True,
|
||||
) -> bool:
|
||||
del memory_to_free, unpatch_weights
|
||||
self.cleanup(unregister=False)
|
||||
return True
|
||||
|
||||
def detach(self, unpatch_weights=True) -> None:
|
||||
del unpatch_weights
|
||||
self.cleanup(unregister=False)
|
||||
|
||||
def is_clone(self, other: Any) -> bool:
|
||||
return other is self
|
||||
|
||||
def cleanup(self, unregister: bool = True) -> None:
|
||||
if unregister:
|
||||
self._unregister_from_comfy_model_management()
|
||||
else:
|
||||
self._comfy_loaded_model = None
|
||||
self._comfy_model_management = None
|
||||
self._initialized = False
|
||||
if getattr(self, "_session", None) is not None:
|
||||
self._session.close()
|
||||
|
||||
close = cleanup
|
||||
|
||||
def __del__(self):
|
||||
try:
|
||||
self.cleanup()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def build_audio8_tts_isolated_proxy(
|
||||
config: ModelLoadConfig,
|
||||
) -> Audio8TTSIsolatedProxy:
|
||||
profile_name = config.runtime_profile or "vibevoice_transformers4_shared"
|
||||
profile = get_runtime_profile(profile_name)
|
||||
if profile is None:
|
||||
raise RuntimeError(
|
||||
f"Unknown isolated runtime profile '{profile_name}' for Audio8 TTS"
|
||||
)
|
||||
if profile.name != "vibevoice_transformers4_shared":
|
||||
raise RuntimeError(
|
||||
"Audio8 TTS requires the shared Transformers 4 runtime profile "
|
||||
"'vibevoice_transformers4_shared'"
|
||||
)
|
||||
config.runtime_profile = profile.name
|
||||
return Audio8TTSIsolatedProxy(config=config, profile=profile)
|
||||
@@ -28,7 +28,7 @@ _VIBEVOICE_T4_PACKAGES = [
|
||||
"numpy>=1.26.4,<2.3.0",
|
||||
"soundfile>=0.12.0",
|
||||
"omegaconf>=2.3.0",
|
||||
"transformers>=4.51.3,<=4.57.3",
|
||||
"transformers>=4.57.0,<=4.57.3",
|
||||
"kernels>=0.6.1,<=0.9",
|
||||
"accelerate",
|
||||
"requests",
|
||||
@@ -63,9 +63,12 @@ _QWEN3_T4_PACKAGES = [
|
||||
RUNTIME_PROFILES: Dict[str, RuntimeProfile] = {
|
||||
"vibevoice_transformers4_shared": RuntimeProfile(
|
||||
name="vibevoice_transformers4_shared",
|
||||
engine_names=["vibevoice", "step_audio_editx"],
|
||||
engine_names=["vibevoice", "step_audio_editx", "audio8_tts"],
|
||||
python_path_hint="runtimes/shared_legacy_t4/Scripts/python.exe",
|
||||
description="Shared legacy Transformers 4 runtime for VibeVoice/Kugel, Step Audio EditX, and similar engines.",
|
||||
description=(
|
||||
"Shared legacy Transformers 4 runtime for VibeVoice/Kugel, "
|
||||
"Step Audio EditX, Audio8 TTS, and similar engines."
|
||||
),
|
||||
inherit_base_site_packages=True,
|
||||
pip_packages=list(_VIBEVOICE_T4_PACKAGES),
|
||||
pip_packages_no_deps=[
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
"""
|
||||
Persistent JSON-line worker session for isolated runtimes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import threading
|
||||
@@ -118,6 +118,24 @@ class JsonLineWorkerSession:
|
||||
|
||||
self._process = None
|
||||
|
||||
def terminate_now(self) -> None:
|
||||
"""Stop a worker without waiting for the request lock."""
|
||||
process = self._process
|
||||
if process is None or process.poll() is not None:
|
||||
return
|
||||
try:
|
||||
process.terminate()
|
||||
except OSError:
|
||||
return
|
||||
try:
|
||||
process.wait(timeout=2)
|
||||
except subprocess.TimeoutExpired:
|
||||
try:
|
||||
process.kill()
|
||||
process.wait(timeout=2)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def __del__(self):
|
||||
try:
|
||||
self.close()
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
"""Persistent Audio8 TTS worker for the shared Transformers 4 runtime."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import torch
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
project_root_str = str(PROJECT_ROOT)
|
||||
sys.path = [
|
||||
entry for entry in sys.path if str(Path(entry).resolve()) != project_root_str
|
||||
]
|
||||
sys.path.insert(0, project_root_str)
|
||||
|
||||
from utils.runtimes.protocol import RuntimeJobResponse # noqa: E402
|
||||
|
||||
|
||||
def _emit(protocol_out, response: RuntimeJobResponse) -> None:
|
||||
protocol_out.write(json.dumps(response.to_dict(), ensure_ascii=True) + "\n")
|
||||
protocol_out.flush()
|
||||
|
||||
|
||||
def _load_reference_audio(
|
||||
payload: Optional[Dict[str, Any]],
|
||||
) -> Any:
|
||||
if payload is None:
|
||||
return None
|
||||
kind = payload.get("kind")
|
||||
if kind == "audio_path":
|
||||
audio_path = payload.get("audio_path")
|
||||
if not audio_path:
|
||||
raise RuntimeError("Audio8 TTS audio_path payload is missing its file path")
|
||||
return audio_path
|
||||
if kind == "tensor_path":
|
||||
tensor_path = payload.get("tensor_path")
|
||||
if not tensor_path:
|
||||
raise RuntimeError(
|
||||
"Audio8 TTS tensor_path payload is missing its file path"
|
||||
)
|
||||
tensor_payload = torch.load(tensor_path, map_location="cpu")
|
||||
return {
|
||||
"array": tensor_payload["waveform"],
|
||||
"sampling_rate": int(tensor_payload["sample_rate"]),
|
||||
}
|
||||
raise RuntimeError(f"Unsupported Audio8 TTS reference payload kind: {kind}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
protocol_out = sys.stdout
|
||||
sys.stdout = sys.stderr
|
||||
engine = None
|
||||
|
||||
for line in sys.stdin:
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
|
||||
request = None
|
||||
try:
|
||||
request = json.loads(stripped)
|
||||
action = request.get("action")
|
||||
payload = request.get("payload") or {}
|
||||
request_id = request.get("request_id")
|
||||
|
||||
if action == "shutdown":
|
||||
_emit(
|
||||
protocol_out,
|
||||
RuntimeJobResponse(
|
||||
ok=True,
|
||||
result={"shutdown": True},
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
break
|
||||
|
||||
if action == "ping":
|
||||
_emit(
|
||||
protocol_out,
|
||||
RuntimeJobResponse(
|
||||
ok=True,
|
||||
result={"pong": True},
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
if action == "initialize":
|
||||
import transformers
|
||||
from packaging.version import Version
|
||||
|
||||
transformers_version = Version(transformers.__version__)
|
||||
if not (Version("4.57.0") <= transformers_version < Version("5.0.0")):
|
||||
raise RuntimeError(
|
||||
"Audio8 TTS requires Transformers >=4.57.0,<5; "
|
||||
f"worker loaded Transformers {transformers.__version__}"
|
||||
)
|
||||
|
||||
from engines.audio8_tts.audio8_tts_engine import (
|
||||
Audio8TTSEngine,
|
||||
)
|
||||
|
||||
engine = Audio8TTSEngine(
|
||||
model_name=(request.get("model_name") or "Audio8-TTS-Preview-0.6b"),
|
||||
model_dir=payload.get("model_path"),
|
||||
device=request.get("device") or "auto",
|
||||
dtype=payload.get("dtype", "auto"),
|
||||
)
|
||||
engine._ensure_model_loaded()
|
||||
_emit(
|
||||
protocol_out,
|
||||
RuntimeJobResponse(
|
||||
ok=True,
|
||||
result={
|
||||
"model_name": engine.model_name,
|
||||
"sample_rate": engine.SAMPLE_RATE,
|
||||
"transformers_version": (transformers.__version__),
|
||||
},
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
if engine is None:
|
||||
raise RuntimeError(
|
||||
"Audio8 TTS worker received generation before initialization"
|
||||
)
|
||||
if action != "generate":
|
||||
raise RuntimeError(f"Unsupported Audio8 TTS worker action '{action}'")
|
||||
|
||||
reference_audio = _load_reference_audio(payload.get("reference_audio"))
|
||||
audio = engine.generate(
|
||||
text=payload["text"],
|
||||
reference_audio=reference_audio,
|
||||
reference_text=payload.get("reference_text"),
|
||||
max_new_tokens=int(payload.get("max_new_tokens", 1024)),
|
||||
retry_max_new_tokens=int(payload.get("retry_max_new_tokens", 2000)),
|
||||
temperature=float(payload.get("temperature", 0.8)),
|
||||
top_p=float(payload.get("top_p", 0.95)),
|
||||
top_k=int(payload.get("top_k", 50)),
|
||||
do_sample=bool(payload.get("do_sample", True)),
|
||||
seed=int(payload.get("seed", 42)),
|
||||
)
|
||||
audio = torch.as_tensor(audio).detach().float().cpu()
|
||||
if audio.ndim == 1:
|
||||
audio = audio.unsqueeze(0)
|
||||
if audio.ndim != 2 or audio.shape[0] != 1:
|
||||
raise RuntimeError(
|
||||
f"Audio8 TTS generated invalid audio shape {tuple(audio.shape)}"
|
||||
)
|
||||
|
||||
output_path = Path(payload["output_path"])
|
||||
torch.save(
|
||||
{
|
||||
"audio": audio.contiguous(),
|
||||
"sample_rate": engine.SAMPLE_RATE,
|
||||
},
|
||||
output_path,
|
||||
)
|
||||
_emit(
|
||||
protocol_out,
|
||||
RuntimeJobResponse(
|
||||
ok=True,
|
||||
result={
|
||||
"output_path": str(output_path),
|
||||
"sample_rate": engine.SAMPLE_RATE,
|
||||
},
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
request_id = (
|
||||
request.get("request_id") if isinstance(request, dict) else None
|
||||
)
|
||||
_emit(
|
||||
protocol_out,
|
||||
RuntimeJobResponse(
|
||||
ok=False,
|
||||
error=f"{exc}\n{traceback.format_exc()}",
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -115,16 +115,17 @@ PARAMETER_ENGINES = {
|
||||
'seed': {
|
||||
'chatterbox', 'chatterbox_official_23lang', 'f5tts', 'higgs_audio',
|
||||
'higgs_audio_v3', 'vibevoice', 'index_tts', 'step_audio_editx', 'cosyvoice', 'qwen3_tts',
|
||||
'dots_tts', 'fish_audio_s2', 'omnivoice', 'voxcpm',
|
||||
'dots_tts', 'audio8_tts', 'fish_audio_s2', 'omnivoice',
|
||||
'echo_tts', 'moss_tts', 'moss_soundeffect_v2', 'dramabox'
|
||||
},
|
||||
'temperature': {
|
||||
'chatterbox', 'chatterbox_official_23lang', 'f5tts', 'higgs_audio',
|
||||
'higgs_audio_v3', 'vibevoice', 'index_tts', 'step_audio_editx', 'qwen3_tts', 'moss_tts', 'fish_audio_s2'
|
||||
'higgs_audio_v3', 'vibevoice', 'index_tts', 'step_audio_editx', 'qwen3_tts',
|
||||
'moss_tts', 'fish_audio_s2', 'audio8_tts'
|
||||
},
|
||||
'cfg': {
|
||||
'f5tts', 'vibevoice', 'index_tts', 'chatterbox', 'chatterbox_official_23lang',
|
||||
'moss_soundeffect_v2', 'dramabox', 'voxcpm'
|
||||
'moss_soundeffect_v2', 'dramabox'
|
||||
},
|
||||
'stg_scale': {
|
||||
'dramabox'
|
||||
@@ -145,7 +146,7 @@ PARAMETER_ENGINES = {
|
||||
'dramabox'
|
||||
},
|
||||
'num_steps': {
|
||||
'echo_tts', 'dots_tts', 'omnivoice', 'voxcpm'
|
||||
'echo_tts', 'dots_tts', 'omnivoice'
|
||||
},
|
||||
'guidance_scale': {
|
||||
'dots_tts', 'omnivoice'
|
||||
@@ -214,10 +215,12 @@ PARAMETER_ENGINES = {
|
||||
'f5tts', 'cosyvoice', 'omnivoice'
|
||||
},
|
||||
'top_p': {
|
||||
'higgs_audio', 'higgs_audio_v3', 'vibevoice', 'index_tts', 'qwen3_tts', 'moss_tts', 'fish_audio_s2'
|
||||
'higgs_audio', 'higgs_audio_v3', 'vibevoice', 'index_tts', 'qwen3_tts',
|
||||
'moss_tts', 'fish_audio_s2', 'audio8_tts'
|
||||
},
|
||||
'top_k': {
|
||||
'higgs_audio', 'higgs_audio_v3', 'index_tts', 'qwen3_tts', 'moss_tts'
|
||||
'higgs_audio', 'higgs_audio_v3', 'index_tts', 'qwen3_tts', 'moss_tts',
|
||||
'audio8_tts'
|
||||
},
|
||||
'audio_temperature': {
|
||||
'moss_tts'
|
||||
@@ -238,7 +241,7 @@ PARAMETER_ENGINES = {
|
||||
'moss_tts'
|
||||
},
|
||||
'max_new_tokens': {
|
||||
'higgs_audio_v3', 'moss_tts', 'fish_audio_s2'
|
||||
'higgs_audio_v3', 'moss_tts', 'fish_audio_s2', 'audio8_tts'
|
||||
},
|
||||
'max_generate_length': {
|
||||
'dots_tts'
|
||||
@@ -259,7 +262,7 @@ PARAMETER_ENGINES = {
|
||||
'moss_tts'
|
||||
},
|
||||
'inference_steps': {
|
||||
'vibevoice', 'moss_soundeffect_v2', 'voxcpm'
|
||||
'vibevoice', 'moss_soundeffect_v2'
|
||||
},
|
||||
'sigma_shift': {
|
||||
'moss_soundeffect_v2'
|
||||
@@ -350,7 +353,6 @@ PARAMETER_NODE_KEYS = {
|
||||
'f5tts': 'cfg_strength',
|
||||
'moss_soundeffect_v2': 'cfg_scale',
|
||||
'dramabox': 'cfg_scale',
|
||||
'voxcpm': 'cfg_value',
|
||||
}, # Engine-specific mapping
|
||||
'stg_scale': 'stg_scale',
|
||||
'duration_multiplier': 'duration_multiplier',
|
||||
@@ -358,10 +360,7 @@ PARAMETER_NODE_KEYS = {
|
||||
'ref_duration': 'ref_duration',
|
||||
'rescale_scale': 'rescale_scale',
|
||||
'prompt_template': 'prompt_template',
|
||||
'num_steps': {
|
||||
'default': 'num_steps',
|
||||
'voxcpm': 'inference_timesteps',
|
||||
},
|
||||
'num_steps': 'num_steps',
|
||||
'guidance_scale': 'guidance_scale',
|
||||
'duration': 'duration',
|
||||
't_shift': 't_shift',
|
||||
@@ -399,10 +398,7 @@ PARAMETER_NODE_KEYS = {
|
||||
'quality': 'quality',
|
||||
'sound_event': 'sound_event',
|
||||
'ambient_sound': 'ambient_sound',
|
||||
'inference_steps': {
|
||||
'default': 'inference_steps',
|
||||
'voxcpm': 'inference_timesteps',
|
||||
},
|
||||
'inference_steps': 'inference_steps',
|
||||
'sigma_shift': 'sigma_shift',
|
||||
'negative_prompt': 'negative_prompt',
|
||||
'duration_seconds': 'duration_seconds',
|
||||
|
||||
@@ -202,58 +202,6 @@ class OmniVoiceDesignerProvider:
|
||||
return VoiceDesignResult(opt_narrator, audio, info)
|
||||
|
||||
|
||||
class VoxCPMVoiceDesignerProvider:
|
||||
def design(self, engine_data, voice_instruction, reference_text, seed):
|
||||
from engines.adapters.voxcpm_adapter import VoxCPMEngineAdapter
|
||||
|
||||
config = dict(_engine_config(engine_data))
|
||||
selected_model = str(
|
||||
config.get("model_variant") or config.get("model_name") or "VoxCPM2"
|
||||
)
|
||||
if config.get("model_role") != "voice_design":
|
||||
raise ValueError(
|
||||
"VoxCPM is configured for Text to Speech. Set mode to 'Voice Design' "
|
||||
"in the VoxCPM Engine, then run the workflow again."
|
||||
)
|
||||
if selected_model in {"VoxCPM1.5", "VoxCPM-0.5B"}:
|
||||
raise ValueError(
|
||||
f"{selected_model} cannot design voices. Select VoxCPM2 or a local "
|
||||
"checkpoint whose config architecture is voxcpm2."
|
||||
)
|
||||
|
||||
instruction = str(voice_instruction or "").strip()
|
||||
config["voice_instruction"] = instruction
|
||||
adapter = VoxCPMEngineAdapter(config)
|
||||
waveform = adapter.generate_single(
|
||||
reference_text,
|
||||
voice_ref=None,
|
||||
seed=seed,
|
||||
enable_audio_cache=True,
|
||||
character_name="voice_design",
|
||||
)
|
||||
sample_rate = int(
|
||||
getattr(adapter, "sample_rate", None)
|
||||
or getattr(adapter, "SAMPLE_RATE", 48000)
|
||||
)
|
||||
audio = _audio_output(waveform, sample_rate)
|
||||
opt_narrator = _narrator(
|
||||
audio,
|
||||
reference_text,
|
||||
instruction,
|
||||
"voxcpm",
|
||||
selected_model,
|
||||
"Auto",
|
||||
)
|
||||
_attach_generation_fingerprint(
|
||||
opt_narrator, "voxcpm", config, reference_text, seed
|
||||
)
|
||||
info = (
|
||||
"Voice designed with VoxCPM2\n"
|
||||
f"Instruction: {instruction}\nModel: {selected_model}"
|
||||
)
|
||||
return VoiceDesignResult(opt_narrator, audio, info)
|
||||
|
||||
|
||||
class MossVoiceDesignerProvider:
|
||||
MODEL_NAME = "MOSS-VoiceGenerator"
|
||||
|
||||
@@ -317,7 +265,6 @@ _PROVIDERS: Dict[str, VoiceDesignerProvider] = {
|
||||
"qwen3_tts": QwenVoiceDesignerProvider(),
|
||||
"moss_tts": MossVoiceDesignerProvider(),
|
||||
"omnivoice": OmniVoiceDesignerProvider(),
|
||||
"voxcpm": VoxCPMVoiceDesignerProvider(),
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ const ENGINE_PARAMETERS = {
|
||||
step_audio_editx: ["seed", "temperature"],
|
||||
cosyvoice3: ["seed", "speed"],
|
||||
omnivoice: ["seed", "num_steps", "guidance_scale", "duration", "t_shift", "layer_penalty_factor", "position_temperature", "class_temperature", "audio_chunk_duration", "audio_chunk_threshold", "speed"],
|
||||
voxcpm: ["seed", "cfg", "num_steps"],
|
||||
dramabox: ["seed", "cfg", "stg_scale", "duration_multiplier", "gen_duration", "ref_duration", "rescale_scale", "negative", "template"],
|
||||
};
|
||||
const ALL_PARAMETERS = new Set(Object.values(ENGINE_PARAMETERS).flat());
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
import { app } from "../../scripts/app.js";
|
||||
|
||||
function findWidget(node, name) {
|
||||
return node.widgets?.find((widget) => widget.name === name);
|
||||
}
|
||||
|
||||
function setWidgetEnabled(widget, enabled) {
|
||||
if (!widget) return;
|
||||
widget.disabled = !enabled;
|
||||
if (widget.element) {
|
||||
widget.element.style.opacity = enabled ? "1" : "0.55";
|
||||
widget.element.style.pointerEvents = enabled ? "auto" : "none";
|
||||
}
|
||||
}
|
||||
|
||||
function isVoxCPM2(value) {
|
||||
const model = String(value || "").replace(/^local:/, "");
|
||||
return model === "VoxCPM2" || !["VoxCPM1.5", "VoxCPM-0.5B"].includes(model);
|
||||
}
|
||||
|
||||
function refreshVoxCPMWidgets(node) {
|
||||
if (node.comfyClass !== "VoxCPMEngineNode") return;
|
||||
|
||||
const modelWidget = findWidget(node, "model_variant");
|
||||
const modeWidget = findWidget(node, "mode");
|
||||
const voiceInstruction = findWidget(node, "voice_instruction");
|
||||
const supportsVoiceDesign = isVoxCPM2(modelWidget?.value);
|
||||
|
||||
if (!supportsVoiceDesign && modeWidget?.value === "Voice Design") {
|
||||
modeWidget.value = "Text to Speech";
|
||||
}
|
||||
|
||||
setWidgetEnabled(modeWidget, supportsVoiceDesign);
|
||||
setWidgetEnabled(
|
||||
voiceInstruction,
|
||||
supportsVoiceDesign && modeWidget?.value !== "Voice Design",
|
||||
);
|
||||
node.graph?.setDirtyCanvas?.(true, true);
|
||||
}
|
||||
|
||||
function hookWidget(node, widget) {
|
||||
if (!widget || widget.__ttsVoxCPMHooked) return;
|
||||
widget.__ttsVoxCPMHooked = true;
|
||||
|
||||
let storedValue = widget.value;
|
||||
const descriptor = Object.getOwnPropertyDescriptor(widget, "value")
|
||||
|| Object.getOwnPropertyDescriptor(Object.getPrototypeOf(widget), "value")
|
||||
|| Object.getOwnPropertyDescriptor(widget.constructor?.prototype || {}, "value");
|
||||
|
||||
Object.defineProperty(widget, "value", {
|
||||
get() {
|
||||
return descriptor?.get ? descriptor.get.call(widget) : storedValue;
|
||||
},
|
||||
set(value) {
|
||||
if (descriptor?.set) descriptor.set.call(widget, value);
|
||||
else storedValue = value;
|
||||
refreshVoxCPMWidgets(node);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function setupNode(node) {
|
||||
if (node.comfyClass !== "VoxCPMEngineNode") return;
|
||||
hookWidget(node, findWidget(node, "model_variant"));
|
||||
hookWidget(node, findWidget(node, "mode"));
|
||||
refreshVoxCPMWidgets(node);
|
||||
}
|
||||
|
||||
app.registerExtension({
|
||||
name: "tts-audio-suite.voxcpm.widgets",
|
||||
nodeCreated(node) {
|
||||
if (node.comfyClass === "VoxCPMEngineNode") {
|
||||
setTimeout(() => setupNode(node), 0);
|
||||
}
|
||||
},
|
||||
loadedGraphNode(node) {
|
||||
if (node.comfyClass === "VoxCPMEngineNode") {
|
||||
setTimeout(() => setupNode(node), 0);
|
||||
}
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user