Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 diodiogod
|
fe9dcd6839 | ||
|
diodiogod
|
24982e55c1 | ||
|
diodiogod
|
6f3452de8d | ||
|
diodiogod
|
894692c3ff | ||
|
wzgrx
|
00483ccc4a |
@@ -1,15 +0,0 @@
|
||||
# These are supported funding model platforms
|
||||
|
||||
github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
|
||||
patreon: # Replace with a single Patreon username
|
||||
open_collective: # Replace with a single Open Collective username
|
||||
ko_fi: diogogo
|
||||
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
|
||||
community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
|
||||
liberapay: # Replace with a single Liberapay username
|
||||
issuehunt: # Replace with a single IssueHunt username
|
||||
lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
|
||||
polar: # Replace with a single Polar username
|
||||
buy_me_a_coffee: # Replace with a single Buy Me a Coffee username
|
||||
thanks_dev: # Replace with a single thanks.dev username
|
||||
custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
|
||||
@@ -1,53 +0,0 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: "[Bug]"
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Describe the bug**
|
||||
A clear and concise description of what the bug is.
|
||||
|
||||
**Init log**
|
||||
Provide your init log. PLEASE ERASE THE EXEMPLE BELOW, copy paste it yours (you can surround it with triple backticks ```) I NEED to know your system info + TTS Audio Suite version + other init dependency information like below:
|
||||
```
|
||||
Total VRAM 24564 MB, total RAM 65440 MB
|
||||
pytorch version: 2.8.0+cu129
|
||||
Set vram state to: NORMAL_VRAM
|
||||
Device: cuda:0 NVIDIA GeForce RTX 4090 : cudaMallocAsync
|
||||
Using pytorch attention
|
||||
Python version: 3.13.6 (tags/v3.13.6:4e66535, Aug 6 2025, 14:36:00) [MSC v.1944 64 bit (AMD64)]
|
||||
ComfyUI version: 0.3.57
|
||||
ComfyUI frontend version: 1.25.11
|
||||
[Prompt Server] web root: J:\stablediffusion1111s2\Data\Packages\ComfyUINEWCLEAN\py313New\ComfyUI_windows_portable\python_embeded\Lib\site-packages\comfyui_frontend_package\static
|
||||
✅ Numba JIT working properly - no workarounds needed
|
||||
ℹ️ Critical package versions: NumPy 2.2.6, Librosa 0.11.0, Numba 0.61.2, PyTorch 2.8.0+cu129, TorchAudio 2.8.0+cu129, Transformers 4.56.0, SoundFile 0.13.1
|
||||
✅ ChatterboxTTS loaded from bundled package
|
||||
✅ ChatterboxVC loaded from bundled package
|
||||
✅ F5-TTS loaded from system package
|
||||
APEX FusedRMSNorm not available, using native implementation
|
||||
🎭 Character voices: Found 32 characters, 10 aliases
|
||||
======================================================================
|
||||
🚀 TTS Audio Suite v4.9.6
|
||||
```
|
||||
|
||||
**Provide your ERROR full log**
|
||||
copy paste it (you can surround it with triple backticks ```):
|
||||
```
|
||||
error...
|
||||
```
|
||||
|
||||
**Screenshots**
|
||||
If applicable, add screenshots to help explain your problem.
|
||||
|
||||
**System information**
|
||||
- OS: [e.g. iOS]
|
||||
- ComfyUI type: [portable; direct venv; conda; desktop app]
|
||||
- Python version: [check the init log]
|
||||
|
||||
**Additional context**
|
||||
Add any other context about the problem here.
|
||||
|
||||
_Note: Bug reports without the error logs, depending on the issue, might be ignored. It's very hard to provide support without minimal information._
|
||||
-31
@@ -21,10 +21,6 @@ eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
!engines/rvc/impl/lib/
|
||||
!engines/rvc/training/bundled_runtime/lib/
|
||||
!engines/rvc/training/bundled_runtime/lib/**
|
||||
engines/rvc/training/bundled_runtime/lib/**/__pycache__/
|
||||
engines/rvc/training/bundled_runtime/lib/**/*.py[cod]
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
@@ -139,17 +135,12 @@ celerybeat.pid
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.env.local
|
||||
tests/.env
|
||||
tests/.env.local
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
comfyui_env/
|
||||
/runtimes/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
@@ -218,14 +209,6 @@ CLAUDE.md
|
||||
*CLAUDE.md
|
||||
docs/Dev reports/CLAUDE_VERSION_MANAGEMENT_GUIDE.md
|
||||
.claude-code.json
|
||||
.claude/
|
||||
|
||||
#codex
|
||||
*AGENTS.md
|
||||
AGENTS.md
|
||||
|
||||
# Antigravity / agent configuration
|
||||
.agent/
|
||||
|
||||
IgnoredForGitHubDocs/
|
||||
|
||||
@@ -239,17 +222,3 @@ IgnoredForGitHubDocs/
|
||||
|
||||
.roo/
|
||||
.vscode/
|
||||
ignore/
|
||||
|
||||
# Test output files
|
||||
tests/output/
|
||||
|
||||
# TTS Audio Suite settings cache (user preferences)
|
||||
.inline_tag_settings_cache.json
|
||||
|
||||
# AI Agent skills
|
||||
.agents/
|
||||
skills-lock.json
|
||||
|
||||
# others
|
||||
node_modules/
|
||||
|
||||
+1
-4950
File diff suppressed because it is too large
Load Diff
@@ -1,54 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2025 Shmuel Ronen
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
---
|
||||
|
||||
Third-Party Model Licenses
|
||||
──────────────────────────
|
||||
|
||||
The project code is MIT. Model weights carry their own licenses:
|
||||
|
||||
─────────────────── ──────────────────────────────────────────────────────── ──────────────
|
||||
Engine License Commercial Use
|
||||
─────────────────── ──────────────────────────────────────────────────────── ──────────────
|
||||
F5-TTS CC-BY-NC-4.0 No
|
||||
ChatterBox MIT Yes
|
||||
ChatterBox 23L MIT Yes
|
||||
VibeVoice MIT (research-only per model card) No
|
||||
Higgs Audio 2 Boson Higgs Audio 2 Community License Conditional
|
||||
Higgs Audio v3 Boson Higgs Audio v3 Research and Non-Commercial License No
|
||||
IndexTTS 2 / 2.5 bilibili Model Use License Conditional
|
||||
CosyVoice3 Apache-2.0 Yes
|
||||
Qwen3-TTS Apache-2.0 Yes
|
||||
Granite ASR Apache-2.0 Yes
|
||||
Step Audio EditX Apache-2.0 (verify before commercial use) Conditional
|
||||
Echo-TTS CC-BY-NC-SA-4.0 No
|
||||
Fish Audio S2 Pro Fish Audio Research License No
|
||||
Dots TTS Apache-2.0 Yes
|
||||
DramaBox LTX-2 Community License Conditional
|
||||
OmniVoice Apache-2.0 Yes
|
||||
MOSS-TTS Apache-2.0 Yes
|
||||
MOSS-SoundEffect v2 Apache-2.0 Yes
|
||||
RVC MIT (framework); community models vary Varies
|
||||
─────────────────── ──────────────────────────────────────────────────────── ──────────────
|
||||
|
||||
Users are responsible for complying with respective model licenses.
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2025 Shmuel Ronen
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
+290
-178
@@ -4,205 +4,317 @@
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
**Unified modular architecture** - all engines share the same layered pattern:
|
||||
|
||||
1. **Engine Node** (`nodes/engines/<engine>_engine_node.py`) - ComfyUI configuration UI
|
||||
2. **Unified Nodes** (`nodes/unified/`) - thin delegation layer, routes to engine processors
|
||||
3. **Processor** (`nodes/<engine>/<engine>_processor.py`) - orchestration, chunking, character/pause/tag handling
|
||||
4. **SRT Processor** (`nodes/<engine>/<engine>_srt_processor.py`) - SRT-specific orchestration
|
||||
5. **Adapter** (`engines/adapters/<engine>_adapter.py`) - bridges processor to engine implementation
|
||||
6. **Engine Implementation** (`engines/<engine>/`) - actual model inference
|
||||
7. **Optional Isolated Runtime** (`utils/runtimes/`) - worker subprocess layer for fragile engine stacks
|
||||
|
||||
**Key architectural rules:**
|
||||
- Chunking happens in the **processor**, not the adapter (`generate_single()` on adapter = raw single call)
|
||||
- Runtime routing happens through `ModelLoadConfig.runtime_mode` + `runtime_profile`, not ad-hoc subprocess calls
|
||||
- Shared runtime workers are currently used for fragile engine families such as VibeVoice, Qwen3-TTS / ASR, Granite forced alignment, and Higgs Audio 2. Engines that support the modern stack run natively in the main Transformers 5 environment.
|
||||
- YAML (`docs/Dev reports/tts_audio_suite_engines.yaml`) is source of truth for engine doc tables → run `python3 scripts/generate_engine_tables.py --readme` to regenerate
|
||||
- Auxiliary YAML (`docs/Dev reports/tts_audio_suite_aux_models.yaml`) is source of truth for helper/post-process model docs → run `python3 scripts/generate_aux_model_docs.py`
|
||||
- All models download to `ComfyUI/models/TTS/<model-name>/`
|
||||
- Engine registry: `utils/models/engine_registry.py`
|
||||
|
||||
## Engines
|
||||
|
||||
19 engines follow the pattern above:
|
||||
|
||||
| Engine | Adapter | Processor | SRT Processor | Engine Node |
|
||||
|--------|---------|-----------|---------------|-------------|
|
||||
| ChatterBox | `chatterbox_adapter.py` | `nodes/chatterbox/chatterbox_tts_node.py` | `chatterbox_srt_node.py` | `chatterbox_engine_node.py` |
|
||||
| ChatterBox 23-Lang | `chatterbox_official_23lang_adapter.py` | `nodes/chatterbox_official_23lang/` | same folder | `chatterbox_official_23lang_engine_node.py` |
|
||||
| F5-TTS | `f5tts_adapter.py` | `nodes/f5tts/f5tts_node.py` | `f5tts_srt_node.py` | `f5tts_engine_node.py` |
|
||||
| Higgs Audio 2 | `higgs_audio_adapter.py` | — | `nodes/higgs_audio/higgs_audio_srt_processor.py` | `higgs_audio_engine_node.py` |
|
||||
| Higgs Audio v3 | `higgs_audio_v3_adapter.py` | `nodes/higgs_audio_v3/higgs_audio_v3_processor.py` | `higgs_audio_v3_srt_processor.py` | `higgs_audio_v3_engine_node.py` |
|
||||
| VibeVoice | `vibevoice_adapter.py` | `nodes/vibevoice/vibevoice_processor.py` | — | `vibevoice_engine_node.py` |
|
||||
| IndexTTS-2 | — | `engines/index_tts/` | — | `index_tts_engine_node.py` |
|
||||
| Step Audio EditX | `step_audio_editx_adapter.py` | `nodes/step_audio_editx/step_audio_editx_processor.py` | `step_audio_editx_srt_processor.py` | `step_audio_editx_engine_node.py` |
|
||||
| CosyVoice3 | `cosyvoice_adapter.py` | `engines/processors/cosyvoice_processor.py` | `nodes/cosyvoice/cosyvoice_srt_processor.py` | `cosyvoice_engine_node.py` |
|
||||
| Qwen3-TTS | `qwen3_tts_adapter.py` | `nodes/qwen3_tts/qwen3_tts_processor.py` | same folder | `qwen3_tts_engine_node.py` |
|
||||
| MOSS-TTS | `moss_tts_adapter.py` | `nodes/moss_tts/moss_tts_processor.py` | `moss_tts_srt_processor.py` | `moss_tts_engine_node.py` |
|
||||
| Granite ASR | `asr_granite_adapter.py` | — | — | `granite_asr_engine_node.py` |
|
||||
| Echo-TTS | `echo_tts_adapter.py` | `nodes/echo_tts/echo_tts_processor.py` | `echo_tts_srt_processor.py` | `echo_tts_engine_node.py` |
|
||||
| Fish Audio S2 Pro | `fish_audio_s2_adapter.py` | `nodes/fish_audio_s2/fish_audio_s2_processor.py` | `fish_audio_s2_srt_processor.py` | `fish_audio_s2_engine_node.py` |
|
||||
| Dots TTS | `dots_tts_adapter.py` | `nodes/dots_tts/dots_tts_processor.py` | `dots_tts_srt_processor.py` | `dots_tts_engine_node.py` |
|
||||
| DramaBox | `dramabox_adapter.py` | `nodes/dramabox/dramabox_processor.py` | `dramabox_srt_processor.py` | `dramabox_engine_node.py` |
|
||||
| OmniVoice | `omnivoice_adapter.py` | `nodes/omnivoice/omnivoice_processor.py` | `omnivoice_srt_processor.py` | `omnivoice_engine_node.py` |
|
||||
| MOSS-SoundEffect v2 | `moss_soundeffect_v2_adapter.py` | — | — | `moss_soundeffect_v2_engine_node.py` |
|
||||
| RVC | — | `engines/rvc/` | — | `rvc_engine_node.py` |
|
||||
|
||||
**Engine implementations live in:**
|
||||
- `engines/chatterbox/`, `engines/chatterbox_official_23lang/`, `engines/f5tts/`, `engines/higgs_audio/`, `engines/higgs_audio_v3/`, `engines/vibevoice_engine/`, `engines/step_audio_editx/`, `engines/cosyvoice/`, `engines/qwen3_tts/`, `engines/qwen3_asr/`, `engines/moss_tts/`, `engines/moss_soundeffect_v2/`, `engines/granite_asr/`, `engines/echo_tts/`, `engines/fish_audio_s2/`, `engines/dots_tts/`, `engines/dramabox/`, `engines/omnivoice/`, `engines/rvc/`
|
||||
This extension features a **unified modular architecture** supporting multiple TTS engines:
|
||||
- **Unified Node Interface**: Single set of nodes (TTS Text, TTS SRT, Voice Changer) that work with any engine
|
||||
- **Engine Adapters**: Modular adapters for ChatterBox, F5-TTS, and RVC voice conversion
|
||||
- **Centralized Download System**: Unified downloader eliminates HuggingFace cache duplication with direct downloads to organized TTS/ folder structure
|
||||
- **Thread-Safe Architecture**: Stateless ChatterBox wrapper eliminates shared state corruption (Note: parallel processing is slower than sequential)
|
||||
- **Universal Streaming Infrastructure**: Unified streaming system with configurable workers (batch_size parameter) - sequential mode (batch_size=0) recommended for optimal performance
|
||||
- **Multilingual Support**: German and Norwegian models for both ChatterBox TTS and Voice Conversion
|
||||
- **Smart Language Grouping**: SRT processing by language groups to minimize model switching
|
||||
- **Character Voice Management**: Centralized character voice system with flexible input types
|
||||
- **Comprehensive Audio Processing**: Interactive waveform analyzer, vocal separation, and audio mixing
|
||||
- **Professional Voice Conversion**: RVC implementation with UVR5 vocal separation and advanced pitch control
|
||||
|
||||
## Documentation Files
|
||||
|
||||
**README.md** - Main project docs, installation, features overview
|
||||
**CLAUDE.md** - Dev guidelines for Claude Code
|
||||
**CHANGELOG.md** - Full version history
|
||||
**docs/BUMP_SCRIPT_INSTRUCTIONS.md** - Version bump process
|
||||
**README.md** - Main project documentation with installation guide, features overview, and usage instructions for the TTS Audio Suite
|
||||
|
||||
### User Docs (`docs/`)
|
||||
- `CHARACTER_SWITCHING_GUIDE.md` - [CharacterName] tag system
|
||||
- `PARAMETER_SWITCHING_GUIDE.md` - Per-segment parameter override syntax
|
||||
- `INLINE_EDIT_TAGS_USER_GUIDE.md` - Step Audio EditX inline tags
|
||||
- `HIGGS_AUDIO_V3_INLINE_TAGS.md` - Higgs Audio v3 native paralinguistic tags
|
||||
- `OMNIVOICE_TAGS_GUIDE.md` - OmniVoice native non-verbal tags and pronunciation overrides
|
||||
- `MOSS_TTS_PROMPT_FIELDS_GUIDE.md` - Official MOSS whole-segment prompt fields and inline `<>` translation limits
|
||||
- `DRAMABOX_PROMPTING_GUIDE.md` - DramaBox expressive scene prompts, voice references, controls, hardware, and license
|
||||
- `COSYVOICE3_TAGS_GUIDE.md` - CosyVoice3 native paralinguistic tags
|
||||
- `CHATTERBOX_V2_SPECIAL_TOKENS.md` - ChatterBox v2 emotion tokens
|
||||
- `IndexTTS2_Emotion_Control_Guide.md` - IndexTTS-2 vector, text, audio, and blended emotion controls
|
||||
- `VOCAL_REMOVAL_GUIDE.md` - Vocal separation guide
|
||||
- `qwen3_tts_optimizations.md` - Qwen3-TTS torch.compile setup
|
||||
- `MODEL_DOWNLOAD_SOURCES.md` - All HF repo links (auto-generated)
|
||||
- `MODEL_LAYOUTS.md` - Folder structures (auto-generated)
|
||||
- `AUX_MODEL_SOURCES.md`, `AUX_MODEL_LAYOUTS.md` - Auxiliary helper-model registries (auto-generated)
|
||||
- `ENGINE_COMPARISON.md`, `LANGUAGE_SUPPORT.md`, `FEATURE_COMPARISON.md` - Auto-generated tables
|
||||
**CLAUDE.md** - Development guidelines and project context for Claude Code when working with this codebase, including critical thinking requirements and response style guidelines
|
||||
|
||||
### Dev Docs (`docs/Dev reports/`)
|
||||
- `tts_audio_suite_engines.yaml` - **Source of truth** for all engine metadata
|
||||
- `tts_audio_suite_aux_models.yaml` - **Source of truth** for helper/post-process model metadata
|
||||
- `SRT_IMPLEMENTATION.md` - SRT timing technical details
|
||||
- `ISOLATED_RUNTIMES_PLAN.md` - original runtime isolation plan and scope
|
||||
- `TRANSFORMERS_5_QWEN3_TTS_REPORT.md` - why Qwen3-TTS moved to shared legacy T4 runtime
|
||||
- `TRANSFORMERS_5_HIGGS_AUDIO_REPORT.md` - why Higgs Audio 2 moved to shared legacy T4 runtime
|
||||
**CHANGELOG.md** - Complete version history with detailed feature additions, bug fixes, and architectural improvements across all versions
|
||||
|
||||
### New Engine Guides (`docs/New Engines Guides/`)
|
||||
- `README.md` - Start-here friendly workflow for users guiding LLMs through new engine integrations
|
||||
- `01_FIRST_ASK_THE_LLM_TO_RESEARCH_THE_OFFICIAL_MODEL.md` - Required official model capability audit before coding
|
||||
- `02_CHECK_EXISTING_COMFYUI_IMPLEMENTATIONS.md` - How to use existing ComfyUI implementations as references, not source of truth
|
||||
- `03_DECIDE_ENGINE_SCOPE.md` - Decide whether the engine needs TTS, Voice Changer, ASR, or special nodes; TTS implies SRT support
|
||||
- `04_IMPLEMENTATION_ORDER_FOR_LLM.md` - Step-by-step implementation order with Qwen3-TTS and Step Audio EditX reference rules
|
||||
- `05_REQUIRED_PARITY_CHECKLIST.md` - Must-pass architecture, lifecycle, cache, tag, SRT, docs, and UX checklist
|
||||
- `06_USER_PROMPTS_TO_COPY_PASTE.md` - Ready prompts for users instructing an LLM
|
||||
- `07_PR_REVIEW_CHECKLIST.md` - Maintainer/contributor checklist before merge
|
||||
|
||||
## Core Files
|
||||
### User Documentation
|
||||
|
||||
**`__init__.py`** - ComfyUI entry point
|
||||
**`nodes.py`** - Central node discovery and registration
|
||||
**`install.py`** - Dependency installer with smart skip guards
|
||||
**docs/CHARACTER_SWITCHING_GUIDE.md** - Guide for multi-character TTS using [CharacterName] tags, voice organization, and character voice management system
|
||||
|
||||
**docs/🌊_Audio_Wave_Analyzer-Complete_User_Guide.md** - Complete user guide for Audio Wave Analyzer with interactive waveform visualization and timing extraction
|
||||
|
||||
**docs/VERSION_3.1_RELEASE_GUIDE.md** - Release documentation for character switching system and overlapping subtitles support
|
||||
|
||||
### Developer Documentation
|
||||
|
||||
**docs/Dev reports/CLAUDE_VERSION_MANAGEMENT_GUIDE.md** - Automated version management using enhanced scripts with changelog generation
|
||||
|
||||
**docs/Dev reports/F5TTS_INTEGRATION_SPECIFICATION.md** - Technical specification for F5-TTS integration architecture
|
||||
|
||||
**docs/Dev reports/F5TTS_STREAMING_INTEGRATION_PLAN.md** - Universal streaming architecture plan to eliminate spaghetti code and enable F5-TTS streaming
|
||||
|
||||
**docs/Dev reports/STREAMING_ARCHITECTURE.md** - Streaming parallel processing architecture documentation with feasibility analysis for multiple engines
|
||||
|
||||
**docs/Dev reports/STATELESS_WRAPPER_IMPLEMENTATION_PLAN.md** - Stateless ChatterBox wrapper implementation solving shared model state corruption in parallel processing
|
||||
|
||||
**docs/Dev reports/POST_V4.2.3_DEVELOPMENT_REVIEW.md** - Comprehensive analysis of 45 commits implementing universal streaming architecture and performance insights
|
||||
|
||||
**docs/PAUSE_TAGS_IMPLEMENTATION_REPORT.md** - Implementation details for pause tag system with timing control syntax
|
||||
|
||||
**docs/REFACTORING_PLAN.md** - Architectural refactoring plan and migration strategy documentation
|
||||
|
||||
## Core Architecture Files
|
||||
|
||||
**__init__.py** - Main ComfyUI extension entry point defining WEB_DIRECTORY and delegating to nodes.py for node registration
|
||||
|
||||
**nodes.py** - Central module loader with sophisticated node discovery, conditional imports, and startup diagnostics for all TTS/VC/Audio features
|
||||
|
||||
## Unified Engine Architecture
|
||||
|
||||
### Engine Implementations
|
||||
|
||||
**engines/chatterbox/** - Complete ChatterBox TTS and VC engine implementation
|
||||
- **tts.py** - ChatterBox TTS engine with multilingual support and perth watermarking
|
||||
- **vc.py** - ChatterBox Voice Conversion with multilingual model loading
|
||||
- **language_models.py** - Language model registry for English, German, and Norwegian models
|
||||
- **audio_timing.py** - Audio timing utilities and time-stretching functionality
|
||||
- **streaming_model_manager.py** - Pre-loading and management of multiple language models for streaming workers
|
||||
- **batch_processor.py** - Batch processor for character group processing with true parallel inference
|
||||
- **character_grouper.py** - Character grouping utilities for batch processing optimization
|
||||
- **stateless_wrapper.py** - Thread-safe stateless wrapper eliminating shared state corruption in parallel generation
|
||||
- **models/** - Complete ChatterBox model architecture (S3Gen, T3, tokenizers, voice encoder)
|
||||
|
||||
**engines/f5tts/** - F5-TTS engine implementation and editing capabilities
|
||||
- **f5tts.py** - Main F5-TTS wrapper with ComfyUI integration
|
||||
- **f5tts_edit_engine.py** - Speech editing engine for targeted word replacement
|
||||
- **audio_compositing.py** - Audio compositing with crossfade and segment processing
|
||||
|
||||
**engines/rvc/** - RVC (Real-time Voice Conversion) engine implementation
|
||||
- **__init__.py** - RVC engine initialization and ComfyUI integration
|
||||
- **hubert_downloader.py** - HuBERT model auto-download from Hugging Face with TTS/ folder organization
|
||||
- **impl/** - Complete RVC implementation with vocal separation and voice conversion
|
||||
- **rvc_audio.py** - Audio processing utilities for RVC pipeline
|
||||
- **rvc_downloader.py** - Model downloading system for RVC, UVR, and Karafan models
|
||||
- **rvc_utils.py** - Utility functions for device detection and file management
|
||||
- **uvr5_cli.py** - UVR5 vocal separation command-line interface
|
||||
- **lib/** - Core RVC libraries and model implementations
|
||||
- **separators.py** - Audio separation algorithms (MDX, VR, UVR5)
|
||||
- **model_utils.py** - Model loading and hash utilities
|
||||
- **audio.py** - Audio I/O and format conversion utilities
|
||||
- **utils.py** - General utilities and merge functions
|
||||
- **karafan/** - Karafan separation model implementation
|
||||
- **infer_pack/** - RVC inference and model loading components
|
||||
- **uvr5_pack/** - UVR5 vocal separation algorithms
|
||||
|
||||
### Engine Adapters
|
||||
|
||||
**engines/adapters/chatterbox_adapter.py** - ChatterBox engine adapter providing standardized interface for unified nodes
|
||||
|
||||
**engines/adapters/f5tts_adapter.py** - F5-TTS engine adapter with parameter mapping and cache integration
|
||||
|
||||
**engines/adapters/chatterbox_streaming_adapter.py** - ChatterBox streaming adapter bridging existing implementation to universal streaming system
|
||||
|
||||
**engines/adapters/f5tts_streaming_adapter.py** - F5-TTS streaming adapter enabling parallel processing with language model switching
|
||||
|
||||
## Node Implementations
|
||||
|
||||
### Unified Interface
|
||||
- `nodes/unified/tts_text_node.py` - Universal TTS text node
|
||||
- `nodes/unified/tts_srt_node.py` - Universal SRT node
|
||||
- `nodes/unified/voice_changer_node.py` - Universal voice conversion
|
||||
- `nodes/unified/asr_transcribe_node.py` - Universal ASR node
|
||||
### Engine Configuration Nodes
|
||||
|
||||
### Shared / Special Nodes
|
||||
- `nodes/shared/character_voices_node.py` - Character voice management (NARRATOR_VOICE output)
|
||||
- `nodes/shared/unified_voice_designer_node.py` - Unified Qwen VoiceDesign, MOSS VoiceGenerator, and reference-free OmniVoice design
|
||||
- `nodes/shared/save_character_voice_node.py` - Explicit output node for saving any NARRATOR_VOICE into the established voice library
|
||||
- `nodes/omnivoice/omnivoice_instruction_builder_node.py` - OmniVoice voice-design instruction helper with custom visual builder UI
|
||||
- `nodes/text/phoneme_text_normalizer_node.py` - Multilingual text preprocessing
|
||||
- `nodes/text/asr_punctuation_truecase_node.py` - Standalone punctuation / truecase cleanup for raw ASR text
|
||||
- `nodes/subtitles/text_to_srt_builder_node.py` - Build SRT from transcript text plus timing data, or estimate timings from plain text
|
||||
- `nodes/subtitles/srt_advanced_options_node.py` - Subtitle readability / segmentation policy options
|
||||
- `nodes/text/tts_tag_editor_node.py` - 🏷️ Multiline TTS Tag Editor: rich text editor with character/language/parameter dropdowns, preset system, syntax highlighting, undo/redo — pairs with `web/string_multiline_tag_editor.js`
|
||||
- `nodes/step_audio_editx_special/step_audio_editx_audio_editor_node.py` - 🎨 Audio Editor: post-process ANY engine's audio with Step Audio EditX (14 emotions, 32 styles, paralinguistic effects like `<Laughter>`, speed control) — universal, not just for Step Audio EditX engine
|
||||
- `nodes/engines/index_tts_emotion_options_node.py` - IndexTTS-2 emotion radar chart
|
||||
**nodes/engines/chatterbox_engine_node.py** - ChatterBox engine configuration node for language, device, and generation parameters
|
||||
|
||||
### Audio / Video Nodes
|
||||
- `nodes/audio/analyzer_node.py` - Audio Wave Analyzer
|
||||
- `nodes/audio/vocal_removal_node.py` - Vocal/instrumental separation
|
||||
- `nodes/audio/recorder_node.py` - Microphone recording
|
||||
- `nodes/audio/merge_audio_node.py` - Audio mixing
|
||||
- `nodes/video/mouth_movement_analyzer_node.py` - Silent speech timing extractor
|
||||
- `nodes/models/load_rvc_model_node.py` - RVC model loader
|
||||
**nodes/engines/f5tts_engine_node.py** - F5-TTS engine configuration node with model selection and generation settings
|
||||
|
||||
**nodes/engines/rvc_engine_node.py** - RVC voice conversion node with pitch control and quality settings
|
||||
|
||||
### Unified Interface Nodes
|
||||
|
||||
**nodes/unified/tts_text_node.py** - Universal TTS text generation node working with any configured engine
|
||||
|
||||
**nodes/unified/tts_srt_node.py** - Universal SRT subtitle processing with smart language grouping and timing
|
||||
|
||||
**nodes/unified/voice_changer_node.py** - Universal voice conversion node with multilingual model support and flexible audio inputs
|
||||
|
||||
### Shared Components
|
||||
|
||||
**nodes/shared/character_voices_node.py** - Character voice management system providing NARRATOR_VOICE outputs for any TTS node
|
||||
|
||||
### Engine-Specific Nodes
|
||||
|
||||
**nodes/chatterbox/** - ChatterBox engine implementation nodes (called by Unified nodes)
|
||||
- **chatterbox_tts_node.py** - ChatterBox TTS engine node with streaming batch processing and character switching
|
||||
- **chatterbox_srt_node.py** - ChatterBox SRT engine node with streaming parallel processing and timing modes
|
||||
- **chatterbox_vc_node.py** - ChatterBox voice conversion engine with iterative refinement
|
||||
|
||||
**nodes/f5tts/** - F5-TTS specific nodes
|
||||
- **f5tts_node.py** - Direct F5-TTS generation node
|
||||
- **f5tts_srt_node.py** - F5-TTS SRT processing with language grouping
|
||||
- **f5tts_edit_node.py** - F5-TTS speech editor for word replacement
|
||||
- **f5tts_edit_options_node.py** - Advanced F5-TTS editing configuration
|
||||
|
||||
### Audio Processing System
|
||||
|
||||
**nodes/audio/analyzer_node.py** - Interactive Audio Wave Analyzer with web interface and timing extraction
|
||||
|
||||
**nodes/audio/analyzer_options_node.py** - Configuration options for audio analysis (silence detection, peak analysis)
|
||||
|
||||
**nodes/audio/recorder_node.py** - Voice recording node with microphone input and silence detection
|
||||
|
||||
**nodes/audio/vocal_removal_node.py** - Professional vocal/instrumental separation using UVR5, MDX, VR, and Karafan models
|
||||
|
||||
**nodes/audio/merge_audio_node.py** - Advanced audio mixing with multiple algorithms and pitch control
|
||||
|
||||
**nodes/audio/rvc_pitch_options_node.py** - Advanced pitch extraction settings for RVC voice conversion
|
||||
|
||||
### Video Analysis System
|
||||
|
||||
**nodes/video/mouth_movement_analyzer_node.py** - 🗣️ Silent Speech Analyzer for detecting mouth movement in silent video frames to extract precise timing for TTS SRT synchronization
|
||||
|
||||
**nodes/video/viseme_options_node.py** - Advanced viseme detection configuration with vowel/consonant detection, word prediction, and temporal analysis settings
|
||||
|
||||
### RVC Model Management
|
||||
|
||||
**nodes/models/load_rvc_model_node.py** - RVC model loader with FAISS index support and automatic downloading
|
||||
|
||||
## Base Classes and Foundation
|
||||
|
||||
**nodes/base/base_node.py** - Universal base class for all nodes with device resolution, model management, and cleanup
|
||||
|
||||
**nodes/base/f5tts_base_node.py** - F5-TTS specific base class extending universal foundation with F5-TTS requirements
|
||||
|
||||
## Utility Systems
|
||||
|
||||
### Model Management (`utils/models/`)
|
||||
- `unified_model_interface.py` - Universal factory pattern for all engines
|
||||
- `engine_registry.py` - Engine capability definitions
|
||||
- `factory_config.py` - standardized model load config, runtime mode/profile normalization
|
||||
- `manager.py` - Model discovery and caching
|
||||
- `comfyui_model_wrapper/` - ComfyUI native model management integration
|
||||
- `extra_paths.py` - extra_model_paths.yaml support
|
||||
|
||||
### Isolated Runtimes (`utils/runtimes/`)
|
||||
- `profiles.py` - named runtime profiles (`vibevoice_transformers4_shared`, dedicated variants, etc.)
|
||||
- `launcher.py` - runtime bootstrap, venv creation, Windows toolchain env setup
|
||||
- `session.py`, `protocol.py` - JSONL worker transport and message protocol
|
||||
- `bootstrap.py` - shared runtime bootstrap helpers
|
||||
- `vibevoice_proxy.py`, `qwen3_tts_proxy.py`, `qwen3_asr_proxy.py`, `higgs_audio_proxy.py` - parent-process proxies
|
||||
- `workers/` - worker subprocess entrypoints for VibeVoice, Qwen3-TTS, Qwen3-ASR/aligner, Higgs Audio
|
||||
- Current shared legacy T4 runtime profile is reused by:
|
||||
- VibeVoice / Kugel
|
||||
- Qwen3-TTS
|
||||
- Qwen3-ASR and Granite's optional Qwen forced aligner
|
||||
- Higgs Audio 2
|
||||
### Model Management
|
||||
|
||||
### Audio (`utils/audio/`)
|
||||
- `processing.py` - Tensor manipulation, normalization, format conversion
|
||||
- `cache.py` - Unified TTS caching system
|
||||
- `chunk_combiner.py` - Smart chunk combination (auto/concatenate/crossfade/silence_padding)
|
||||
- `chunk_timing.py` - Standardized chunk timing info across engines
|
||||
- `audio_hash.py` - Content-based cache key hashing
|
||||
- `analysis.py` - Waveform analysis, silence detection, timing extraction
|
||||
- `edit_post_processor.py` - Batch inline edit tag post-processing
|
||||
**utils/models/manager.py** - Intelligent model discovery and caching with multilingual support for both TTS and VC models
|
||||
|
||||
### Text (`utils/text/`)
|
||||
- `character_parser.py` - [CharacterName] tag parsing
|
||||
- `chunking.py` - Sentence-boundary-aware text splitting
|
||||
- `pause_processor.py` - [pause:Ns] tag parsing and silence generation
|
||||
- `segment_parameters.py` - Per-segment parameter parsing ([seed:42|temp:0.5])
|
||||
- `step_audio_editx_special_tags.py` - Inline edit tag system
|
||||
- `cosyvoice_special_tags.py` - CosyVoice3 native tags
|
||||
- `phonemizer_utils.py` - F5-TTS multilingual phonemization
|
||||
**utils/models/smart_loader.py** - Universal smart model loader preventing duplicate model loading across all engines and modes
|
||||
|
||||
### Timing / SRT (`utils/timing/`)
|
||||
- `engine.py` - SRT timing engine
|
||||
- `assembly.py` - Audio segment assembly (multiple timing modes)
|
||||
- `parser.py` - SRT parsing and validation
|
||||
- `reporting.py` - Timing report generation
|
||||
**utils/models/f5tts_manager.py** - F5-TTS specific model management extending base manager functionality
|
||||
|
||||
### Other Utils
|
||||
- `utils/voice/discovery.py` - Voice discovery, user-voice priority, and multi-path fallback
|
||||
- `utils/voice/designers.py` - Whitelisted voice-designer provider registry
|
||||
- `utils/voice/character_saver.py` - Shared `.wav` / `.reference.txt` / `.txt` character persistence
|
||||
- `utils/voice/character_logging.py` - Shared resolved voice labels and boxed prompt previews
|
||||
- `utils/downloads/unified_downloader.py` - Centralized HF download system
|
||||
- `utils/compatibility/transformers_patches.py` - transformers version compatibility patches
|
||||
- `utils/compatibility/numba_compat.py` - Numba/Librosa Python 3.13+ compatibility
|
||||
- `utils/ffmpeg_utils.py` - FFmpeg with graceful fallback
|
||||
- `utils/asr/` - ASR types, adapter registry, pipeline
|
||||
**utils/models/language_mapper.py** - Language-to-model mapping system with fallback support
|
||||
|
||||
## Web Interface
|
||||
**utils/models/fallback_utils.py** - Model fallback and error recovery utilities
|
||||
|
||||
### TTS Tag Editor (modular JS)
|
||||
`web/string_multiline_tag_editor.js` + `widget-*.js`, `editor-state.js`, `tag-utilities.js`, `syntax-highlighter.js`, `font-controls.js`
|
||||
### Audio Processing
|
||||
|
||||
### Audio Analyzer
|
||||
`web/audio_analyzer_*.js` (core, ui, visualization, regions, controls, widgets, drawing, events, layout, node_integration)
|
||||
**utils/audio/processing.py** - Comprehensive audio utilities for tensor manipulation, normalization, and format conversion
|
||||
|
||||
### Other Web Files
|
||||
- `web/chatterbox_voice_capture.js` - Microphone recording UI
|
||||
- `web/index_tts_emotion_radar.js` + `emotion_radar_canvas_widget.js` - IndexTTS-2 radar chart
|
||||
- `web/qwen3_tts_widgets.js` - Qwen model-specific widget enablement and legacy workflow migration
|
||||
- `web/asr_srt_preset_widgets.js` - ASR SRT preset locking
|
||||
**utils/audio/analysis.py** - Advanced audio analysis including waveform processing, silence detection, and timing extraction
|
||||
|
||||
## Scripts & Config
|
||||
- `scripts/bump_version_enhanced.py` - Version bump with changelog (use `patch`/`minor`/`major`)
|
||||
- `scripts/generate_engine_tables.py` - Regenerate all docs from YAML (`--readme` flag for README too)
|
||||
- `requirements.txt`, `pyproject.toml` - Dependencies and project metadata
|
||||
**utils/audio/audio_hash.py** - Centralized content-based hashing for consistent cache keys across all processing modes
|
||||
|
||||
**utils/audio/cache.py** - Unified caching system for TTS engines with engine-specific cache management
|
||||
|
||||
### Text Processing
|
||||
|
||||
**utils/text/character_parser.py** - Universal character switching system with [CharacterName] tag parsing and language-aware processing
|
||||
|
||||
**utils/text/chunking.py** - Enhanced text chunking with sentence boundary detection and character limits
|
||||
|
||||
**utils/text/pause_processor.py** - Pause tag parsing supporting [pause:xx] syntax for precise timing control
|
||||
|
||||
### Voice Management
|
||||
|
||||
**utils/voice/discovery.py** - Advanced voice file discovery with dual folder support and character mapping
|
||||
|
||||
**utils/voice/multilingual_engine.py** - Central orchestrator for multilingual TTS with language switching optimization
|
||||
|
||||
### System Integration
|
||||
|
||||
**utils/system/import_manager.py** - Smart dependency resolution managing bundled vs system installations
|
||||
|
||||
**utils/system/subprocess.py** - Safe subprocess execution with error handling and process isolation
|
||||
|
||||
### Downloads
|
||||
|
||||
**utils/downloads/unified_downloader.py** - Centralized download system for all engines eliminating HuggingFace cache duplication with direct HTTP downloads to organized TTS/ structure
|
||||
|
||||
**utils/downloads/model_downloader.py** - RVC-specific model auto-download system with direct downloads (legacy, integrated with unified system)
|
||||
|
||||
### Timing and SRT
|
||||
|
||||
**utils/timing/engine.py** - Advanced SRT timing engine with smart audio synchronization
|
||||
|
||||
**utils/timing/assembly.py** - Audio segment assembly supporting multiple timing modes
|
||||
|
||||
**utils/timing/parser.py** - SRT subtitle parsing with timestamp extraction and validation
|
||||
|
||||
**utils/timing/reporting.py** - Timing analysis and report generation
|
||||
|
||||
### Streaming System
|
||||
|
||||
**utils/streaming/__init__.py** - Universal streaming system initialization
|
||||
|
||||
**utils/streaming/streaming_types.py** - Universal data structures (StreamingSegment, StreamingResult) eliminating format conversions
|
||||
|
||||
**utils/streaming/streaming_interface.py** - Abstract streaming interface for engine adapters
|
||||
|
||||
**utils/streaming/streaming_coordinator.py** - Universal coordinator replacing all format-specific routers
|
||||
|
||||
**utils/streaming/work_queue_processor.py** - Engine-agnostic parallel processing system
|
||||
|
||||
## Web Interface Components
|
||||
|
||||
**web/audio_analyzer_interface.js** - Main ComfyUI integration and node communication for Audio Wave Analyzer
|
||||
|
||||
**web/audio_analyzer_core.js** - Core JavaScript functionality for waveform visualization
|
||||
|
||||
**web/audio_analyzer_ui.js** - User interface components and control elements
|
||||
|
||||
**web/audio_analyzer_visualization.js** - Waveform rendering and canvas-based visualization
|
||||
|
||||
**web/audio_analyzer_regions.js** - Timing region management and interaction system
|
||||
|
||||
**web/audio_analyzer_controls.js** - Audio playback controls and user interaction handling
|
||||
|
||||
**web/audio_analyzer_widgets.js** - Custom ComfyUI widget implementations
|
||||
|
||||
**web/audio_analyzer_drawing.js** - Canvas drawing utilities and rendering functions
|
||||
|
||||
**web/audio_analyzer_events.js** - Event handling for mouse and keyboard interactions
|
||||
|
||||
**web/audio_analyzer_layout.js** - Layout management and UI positioning
|
||||
|
||||
**web/audio_analyzer_node_integration.js** - Node integration bridge for ComfyUI communication
|
||||
|
||||
**web/chatterbox_voice_capture.js** - Voice recording interface for microphone input
|
||||
|
||||
**web/chatterbox_srt_showcontrol.js** - SRT subtitle display and timing controls
|
||||
|
||||
**web/audio_analyzer.css** - Styling for audio analyzer interface components
|
||||
|
||||
## Development Tools
|
||||
|
||||
**scripts/bump_version_enhanced.py** - Advanced automated version management with changelog generation and git integration
|
||||
|
||||
**scripts/version_utils.py** - Version management utilities and helper functions
|
||||
|
||||
## Configuration Files
|
||||
|
||||
**requirements.txt** - Core dependencies including ChatterboxTTS, audio processing libraries, and optional F5-TTS dependencies
|
||||
|
||||
**pyproject.toml** - Project metadata with ComfyUI registry configuration and dependency specifications
|
||||
|
||||
## Example Workflows
|
||||
|
||||
**example_workflows/📺 Chatterbox SRT.json** - Complete SRT processing workflow with ChatterBox engine
|
||||
|
||||
**example_workflows/Chatterbox integration.json** - General ChatterBox TTS and Voice Conversion demonstration
|
||||
|
||||
**example_workflows/👄 F5-TTS Speech Editor Workflow.json** - F5-TTS speech editing with Audio Wave Analyzer
|
||||
|
||||
**example_workflows/🎤 📺 F5-TTS SRT and Normal Generation.json** - F5-TTS SRT processing and standard generation
|
||||
|
||||
## Voice Examples
|
||||
|
||||
**voices_examples/** - Character voice reference files with both audio samples and text descriptions
|
||||
- **male/** - Male voice examples with reference audio and text
|
||||
- **female/** - Female voice examples with reference audio and text
|
||||
- **#character_alias_map.txt** - Character alias mapping configuration
|
||||
- Individual character voice files (Clint Eastwood, David Attenborough, Morgan Freeman, Sophie Anderson)
|
||||
|
||||
## Project Management
|
||||
|
||||
**IgnoredForGitHubDocs/** - Development documentation excluded from main repository
|
||||
- **FOLDER_STRUCTURE_REFACTORING_PLAN.md** - Architectural refactoring planning
|
||||
- **CHATTERBOX_MODEL_BUG_REPORT.md** - Bug tracking and resolution documentation
|
||||
|
||||
## Architecture Summary
|
||||
|
||||
The TTS Audio Suite follows a **unified modular architecture** where:
|
||||
|
||||
1. **Engine Nodes** (`nodes/engines/`) provide user configuration interfaces
|
||||
2. **Unified Nodes** (`nodes/unified/`) offer consistent user experience across all engines
|
||||
3. **Engine Implementations** (`engines/`) handle the actual TTS/VC processing
|
||||
4. **Adapters** (`engines/adapters/`) bridge engines to the unified interface
|
||||
5. **Utilities** (`utils/`) provide shared functionality across all components
|
||||
6. **Web Interface** (`web/`) enables interactive features like audio analysis
|
||||
|
||||
This design allows easy addition of new engines while maintaining a consistent user experience and shared optimization features across all TTS engines.
|
||||
+20
-635
@@ -2,229 +2,16 @@
|
||||
TTS Audio Suite - Universal multi-engine TTS extension for ComfyUI
|
||||
Unified architecture supporting ChatterBox, F5-TTS, and future engines like RVC:
|
||||
• 🎤 TTS Text (unified text-to-speech)
|
||||
• 📺 TTS SRT (unified SRT subtitle timing)
|
||||
• 📺 TTS SRT (unified SRT subtitle timing)
|
||||
• 🔄 Voice Changer (unified voice conversion)
|
||||
• ⚙️ Engine nodes (ChatterBox, F5-TTS)
|
||||
• 🎭 Character Voices (voice reference management)
|
||||
"""
|
||||
|
||||
# Note: PYTORCH_ALLOC_CONF should be set in ComfyUI launch script if needed
|
||||
# Setting it here causes "allocator mismatch" errors because ComfyUI already imported torch
|
||||
|
||||
# Import from the main nodes.py file which handles the new unified architecture
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
|
||||
# ComfyUI 0.12+ owns a top-level ``utils`` package, while this long-standing
|
||||
# node pack also imports its helpers through ``utils.*``. Preserve ComfyUI's
|
||||
# loaded package and extend only its module search path with this pack's utils.
|
||||
_project_root = os.path.dirname(__file__)
|
||||
_suite_utils_root = os.path.abspath(os.path.join(_project_root, "utils"))
|
||||
if _project_root in sys.path:
|
||||
sys.path.remove(_project_root)
|
||||
sys.path.insert(0, _project_root)
|
||||
|
||||
_loaded_utils = sys.modules.get("utils")
|
||||
if _loaded_utils is None:
|
||||
import utils as _loaded_utils
|
||||
|
||||
_utils_search_path = getattr(_loaded_utils, "__path__", None)
|
||||
if _utils_search_path is None:
|
||||
raise ImportError(
|
||||
"TTS Audio Suite cannot extend the loaded top-level 'utils' module because it is not a package"
|
||||
)
|
||||
|
||||
_normalized_utils_paths = {os.path.normcase(os.path.abspath(path)) for path in _utils_search_path}
|
||||
if os.path.normcase(_suite_utils_root) not in _normalized_utils_paths:
|
||||
_utils_search_path.insert(0, _suite_utils_root)
|
||||
|
||||
# When this pack is imported before ComfyUI imports its own helpers, locate the
|
||||
# active ComfyUI utils directory by its stable core modules and add it as the
|
||||
# fallback side of the same package search path.
|
||||
for _search_root in sys.path:
|
||||
_candidate_utils = os.path.abspath(os.path.join(_search_root or os.curdir, "utils"))
|
||||
_normalized_candidate = os.path.normcase(_candidate_utils)
|
||||
if _normalized_candidate in _normalized_utils_paths or _normalized_candidate == os.path.normcase(_suite_utils_root):
|
||||
continue
|
||||
if all(os.path.isfile(os.path.join(_candidate_utils, filename)) for filename in ("extra_config.py", "install_util.py")):
|
||||
_utils_search_path.append(_candidate_utils)
|
||||
_normalized_utils_paths.add(_normalized_candidate)
|
||||
|
||||
from utils.hf_download_logging import configure_hf_download_logging
|
||||
|
||||
|
||||
# Keep every engine's Hugging Face download output readable. Download failures
|
||||
# are still reported by the suite's downloader error handling.
|
||||
configure_hf_download_logging()
|
||||
|
||||
# Note: PyTorch inductor patches removed - not needed for PyTorch 2.10+ with triton-windows 3.6+
|
||||
# Qwen3-TTS torch.compile optimizations require:
|
||||
# - PyTorch 2.10.0+ with CUDA 13.0
|
||||
# - triton-windows 3.6.0+ (Windows) or triton 3.6.0+ (Linux)
|
||||
# See docs/qwen3_tts_optimizations.md for installation instructions
|
||||
|
||||
# Enable TensorFloat32 for better performance on Ampere+ GPUs (RTX 30xx+)
|
||||
try:
|
||||
import torch
|
||||
if torch.cuda.is_available():
|
||||
torch.set_float32_matmul_precision('high')
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# PyTorch patches solve TWO PyTorch 2.9 issues:
|
||||
# 1. TorchCodec DLL incompatibility on Windows - Global patch uses scipy instead
|
||||
# 2. PyTorch 2.9's changed torchaudio.load() returning raw int16 - safe_load_audio() normalizes
|
||||
#
|
||||
# Transformers patches solve:
|
||||
# 1. Step Audio EditX tokenization bug in transformers 4.54+ (audio tokens not recognized)
|
||||
# 2. Various model compatibility issues
|
||||
try:
|
||||
# Load pytorch_patches directly by file path to avoid package import issues
|
||||
pytorch_patches_path = os.path.join(os.path.dirname(__file__), "utils", "compatibility", "pytorch_patches.py")
|
||||
spec = importlib.util.spec_from_file_location("pytorch_patches_module", pytorch_patches_path)
|
||||
pytorch_patches_module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(pytorch_patches_module)
|
||||
|
||||
# Apply the patches (will only apply on PyTorch 2.9+, silently skip on older versions)
|
||||
pytorch_patches_module.apply_pytorch_patches(verbose=True)
|
||||
except Exception as e:
|
||||
print(f"⚠️ Warning: Could not apply PyTorch patches: {e}")
|
||||
|
||||
# Transformers compatibility patches DEFERRED to first engine use.
|
||||
# The patches module is deprecated (all patches are for old transformers versions),
|
||||
# and importing it eagerly pulls in transformers (~1.3s).
|
||||
# Patches will be applied lazily when an engine first imports transformers.
|
||||
_transformers_patches_applied = False
|
||||
def _apply_transformers_patches_once():
|
||||
"""Apply transformers patches lazily, on first engine use."""
|
||||
global _transformers_patches_applied
|
||||
if _transformers_patches_applied:
|
||||
return
|
||||
_transformers_patches_applied = True
|
||||
try:
|
||||
transformers_patches_path = os.path.join(os.path.dirname(__file__), "utils", "compatibility", "transformers_patches.py")
|
||||
spec = importlib.util.spec_from_file_location("transformers_patches_module", transformers_patches_path)
|
||||
transformers_patches_module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(transformers_patches_module)
|
||||
transformers_patches_module.apply_transformers_patches(verbose=True)
|
||||
except Exception as e:
|
||||
print(f"⚠️ Warning: Could not apply Transformers patches: {e}")
|
||||
|
||||
# Numba/Librosa compatibility check at startup.
|
||||
# Do NOT force NUMBA_DISABLE_JIT on Python 3.13 anymore:
|
||||
# newer stacks (for example numba 0.64 + librosa 0.11) can work normally,
|
||||
# and forcing the env var can itself trigger the get_call_template crash.
|
||||
# For older Python + NumPy 2.x, keep the existing thorough compatibility test.
|
||||
if sys.version_info < (3, 13):
|
||||
try:
|
||||
import numpy as _np
|
||||
if int(_np.__version__.split('.')[0]) >= 2:
|
||||
numba_compat_path = os.path.join(os.path.dirname(__file__), "utils", "compatibility", "numba_compat.py")
|
||||
_spec = importlib.util.spec_from_file_location("numba_compat_module", numba_compat_path)
|
||||
_numba_compat = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(_numba_compat)
|
||||
_numba_compat.setup_numba_compatibility(quick_startup=False, verbose=True)
|
||||
except Exception:
|
||||
# If the compatibility test itself crashes, that means numba JIT is broken —
|
||||
# disable it and warn the user.
|
||||
os.environ['NUMBA_DISABLE_JIT'] = '1'
|
||||
print("⚠️ TTS Audio Suite: Numba JIT crash detected at startup — disabling JIT (NUMBA_DISABLE_JIT=1)")
|
||||
|
||||
# TorchCodec note: Removed torchcodec dependency to eliminate FFmpeg system requirement
|
||||
# torchaudio.load() works fine with fallback backends (soundfile, scipy)
|
||||
import warnings
|
||||
import sys
|
||||
import importlib.util
|
||||
import os
|
||||
|
||||
def check_dependencies():
|
||||
"""Fast check for critical dependencies without importing them into memory"""
|
||||
critical_packages = ['torch', 'torchaudio', 'transformers', 'librosa', 'numba', 'soundfile', 'accelerate']
|
||||
missing = []
|
||||
|
||||
for pkg in critical_packages:
|
||||
# Avoid importlib.util.find_spec for namespace packages or if spec is None
|
||||
try:
|
||||
if importlib.util.find_spec(pkg) is None:
|
||||
missing.append(pkg)
|
||||
except Exception:
|
||||
missing.append(pkg)
|
||||
|
||||
if missing:
|
||||
print(f"\n{'='*80}")
|
||||
print(f"⚠️ TTS AUDIO SUITE: CRITICAL DEPENDENCIES MISSING ⚠️")
|
||||
print(f"{'='*80}")
|
||||
print(f"The following required packages are missing: {', '.join(missing)}")
|
||||
print(f"")
|
||||
install_script = os.path.join(os.path.dirname(__file__), "install.py")
|
||||
print(f"Please run the TTS Audio Suite installation script:")
|
||||
print(f'"{sys.executable}" "{install_script}"')
|
||||
print(f"{'='*80}\n")
|
||||
|
||||
# Version disclosure for troubleshooting
|
||||
def print_critical_versions():
|
||||
"""Print versions of critical packages for troubleshooting.
|
||||
|
||||
Uses importlib.metadata to read versions without importing the actual
|
||||
packages. This avoids pulling in transformers (~1.3s), librosa (~0.7s),
|
||||
and other heavy modules just to print a version string at startup.
|
||||
"""
|
||||
critical_packages = [
|
||||
('numpy', 'NumPy'),
|
||||
('librosa', 'Librosa'),
|
||||
('numba', 'Numba'),
|
||||
('torch', 'PyTorch'),
|
||||
('torchaudio', 'TorchAudio'),
|
||||
('transformers', 'Transformers'),
|
||||
('accelerate', 'Accelerate'),
|
||||
('soundfile', 'SoundFile'),
|
||||
]
|
||||
|
||||
from importlib.metadata import version as _pkg_version, PackageNotFoundError
|
||||
|
||||
version_info = []
|
||||
for pkg_name, display_name in critical_packages:
|
||||
try:
|
||||
ver = _pkg_version(pkg_name)
|
||||
version_info.append(f"{display_name} {ver}")
|
||||
except PackageNotFoundError:
|
||||
version_info.append(f"{display_name} not installed")
|
||||
|
||||
print(f"ℹ️ Critical package versions: {', '.join(version_info)}")
|
||||
|
||||
def check_ffmpeg_availability():
|
||||
"""Check ffmpeg availability and log status"""
|
||||
try:
|
||||
# Load ffmpeg_utils directly by file path to avoid package import issues
|
||||
ffmpeg_utils_path = os.path.join(os.path.dirname(__file__), "utils", "ffmpeg_utils.py")
|
||||
spec = importlib.util.spec_from_file_location("ffmpeg_utils_module", ffmpeg_utils_path)
|
||||
ffmpeg_utils_module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(ffmpeg_utils_module)
|
||||
|
||||
if ffmpeg_utils_module.FFmpegUtils.is_available():
|
||||
# Only show when unavailable (problem)
|
||||
pass
|
||||
else:
|
||||
print("⚠️ FFmpeg not found - using fallback audio processing (reduced quality)")
|
||||
print("💡 Install FFmpeg for optimal performance: https://ffmpeg.org/download.html")
|
||||
except ImportError:
|
||||
# Fallback check if utils not available yet
|
||||
try:
|
||||
import subprocess
|
||||
result = subprocess.run(['ffmpeg', '-version'], capture_output=True, timeout=5)
|
||||
if result.returncode == 0:
|
||||
# Only show when unavailable (problem)
|
||||
pass
|
||||
else:
|
||||
print("⚠️ FFmpeg not found - using fallback audio processing (reduced quality)")
|
||||
except Exception:
|
||||
print("⚠️ FFmpeg not found - using fallback audio processing (reduced quality)")
|
||||
print("💡 Install FFmpeg for optimal performance: https://ffmpeg.org/download.html")
|
||||
|
||||
# Print versions and check dependencies immediately for troubleshooting
|
||||
check_dependencies()
|
||||
print_critical_versions()
|
||||
check_ffmpeg_availability()
|
||||
import sys
|
||||
|
||||
# Check for old ChatterBox extension conflict
|
||||
def check_old_extension_conflict():
|
||||
@@ -265,66 +52,24 @@ def check_old_extension_conflict():
|
||||
# Perform conflict check
|
||||
OLD_EXTENSION_CONFLICT = check_old_extension_conflict()
|
||||
|
||||
# CRITICAL FIX FOR ISSUE #191: Clear poisoned utils from sys.modules
|
||||
# Some custom nodes (e.g., LG_HotReload) have a utils.py file that gets loaded
|
||||
# into sys.modules['utils'], shadowing our utils/ directory package.
|
||||
# This causes "No module named 'utils.models'; 'utils' is not a package" errors
|
||||
# when our code tries to import from utils submodules.
|
||||
# We must clear it BEFORE loading nodes.py which imports from utils.
|
||||
if 'utils' in sys.modules:
|
||||
utils_module = sys.modules['utils']
|
||||
# Check if it's a poisoned utils (single .py file, not a package directory)
|
||||
# Real packages have __path__ attribute, single files don't
|
||||
if not hasattr(utils_module, '__path__'):
|
||||
# It's a single .py file masquerading as utils - this will break our imports
|
||||
utils_file = getattr(utils_module, '__file__', 'unknown')
|
||||
print(f"\n{'='*80}")
|
||||
print(f"⚠️ UTILS NAMESPACE CONFLICT DETECTED")
|
||||
print(f"{'='*80}")
|
||||
print(f"Another custom node has a 'utils.py' file in sys.modules['utils']:")
|
||||
print(f" Source: {utils_file}")
|
||||
print(f"")
|
||||
print(f"This conflicts with TTS Audio Suite's 'utils/' package directory.")
|
||||
print(f"Removing the conflicting module to allow TTS Audio Suite to load.")
|
||||
print(f"")
|
||||
print(f"If this causes issues with another custom node, that node should:")
|
||||
print(f"• Use relative imports (from .utils import X)")
|
||||
print(f"• Or use a unique name instead of 'utils'")
|
||||
print(f"{'='*80}\n")
|
||||
# Get the path to the nodes.py file
|
||||
nodes_py_path = os.path.join(os.path.dirname(__file__), "nodes.py")
|
||||
|
||||
# Delete the poisoned utils module and any attempted submodules
|
||||
del sys.modules['utils']
|
||||
to_delete = [key for key in sys.modules.keys() if key.startswith('utils.')]
|
||||
for key in to_delete:
|
||||
del sys.modules[key]
|
||||
# Load nodes.py as a module
|
||||
spec = importlib.util.spec_from_file_location("nodes_main", nodes_py_path)
|
||||
nodes_module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(nodes_module)
|
||||
|
||||
# In pytest harness mode, avoid bootstrapping full ComfyUI node graph.
|
||||
if os.environ.get("COMFYUI_TESTING") == "1":
|
||||
IS_DEV = False
|
||||
VERSION = "test"
|
||||
SEPARATOR = "=" * 70
|
||||
VERSION_DISPLAY = "test"
|
||||
NODE_CLASS_MAPPINGS = {}
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {}
|
||||
else:
|
||||
# Get the path to the nodes.py file
|
||||
nodes_py_path = os.path.join(os.path.dirname(__file__), "nodes.py")
|
||||
# Import constants and utilities
|
||||
IS_DEV = nodes_module.IS_DEV
|
||||
VERSION = nodes_module.VERSION
|
||||
SEPARATOR = nodes_module.SEPARATOR
|
||||
VERSION_DISPLAY = nodes_module.VERSION_DISPLAY
|
||||
|
||||
# Load nodes.py as a module
|
||||
spec = importlib.util.spec_from_file_location("nodes_main", nodes_py_path)
|
||||
nodes_module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(nodes_module)
|
||||
|
||||
# Import constants and utilities
|
||||
IS_DEV = nodes_module.IS_DEV
|
||||
VERSION = nodes_module.VERSION
|
||||
SEPARATOR = nodes_module.SEPARATOR
|
||||
VERSION_DISPLAY = nodes_module.VERSION_DISPLAY
|
||||
|
||||
# The new unified architecture handles all node registration in nodes.py
|
||||
# Just import the mappings that nodes.py creates
|
||||
NODE_CLASS_MAPPINGS = nodes_module.NODE_CLASS_MAPPINGS
|
||||
NODE_DISPLAY_NAME_MAPPINGS = nodes_module.NODE_DISPLAY_NAME_MAPPINGS
|
||||
# The new unified architecture handles all node registration in nodes.py
|
||||
# Just import the mappings that nodes.py creates
|
||||
NODE_CLASS_MAPPINGS = nodes_module.NODE_CLASS_MAPPINGS
|
||||
NODE_DISPLAY_NAME_MAPPINGS = nodes_module.NODE_DISPLAY_NAME_MAPPINGS
|
||||
|
||||
# Extension info
|
||||
__version__ = VERSION_DISPLAY
|
||||
@@ -333,367 +78,7 @@ __description__ = "Universal multi-engine TTS extension for ComfyUI with unified
|
||||
|
||||
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]
|
||||
|
||||
# Define web directory for JavaScript files (settings UI)
|
||||
# Define web directory for JavaScript files
|
||||
WEB_DIRECTORY = "./web"
|
||||
|
||||
# Register API endpoint for widget data
|
||||
def setup_api_routes():
|
||||
"""Setup API routes for widget communication"""
|
||||
try:
|
||||
import json
|
||||
import folder_paths
|
||||
from server import PromptServer
|
||||
from aiohttp import web
|
||||
|
||||
def _get_ui_data_dir():
|
||||
base_dir = os.path.join(folder_paths.get_system_user_directory("tts_audio_suite"), "ui")
|
||||
os.makedirs(base_dir, exist_ok=True)
|
||||
return base_dir
|
||||
|
||||
def _get_omnivoice_preset_library_path():
|
||||
return os.path.join(_get_ui_data_dir(), "omnivoice_instruction_builder_presets.json")
|
||||
|
||||
from utils.voice.alias_api import register_character_alias_routes
|
||||
register_character_alias_routes(PromptServer.instance.routes, web)
|
||||
from utils.audio_cpp.capability_api import register_audio_cpp_capability_routes
|
||||
register_audio_cpp_capability_routes(PromptServer.instance.routes, web)
|
||||
|
||||
@PromptServer.instance.routes.get("/api/tts-audio-suite/index-tts-emotion-presets")
|
||||
async def get_index_tts_emotion_presets_endpoint(request):
|
||||
"""Return presets stored beside the IndexTTS resources under models/TTS."""
|
||||
try:
|
||||
from .utils.text.index_tts_emotion import load_emotion_presets
|
||||
return web.json_response({"presets": load_emotion_presets()})
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error retrieving IndexTTS emotion presets: {e}")
|
||||
return web.json_response({"presets": {}, "error": str(e)}, status=500)
|
||||
|
||||
@PromptServer.instance.routes.post("/api/tts-audio-suite/index-tts-emotion-presets")
|
||||
async def save_index_tts_emotion_presets_endpoint(request):
|
||||
"""Atomically persist the IndexTTS emotion preset library."""
|
||||
try:
|
||||
from .utils.text.index_tts_emotion import save_emotion_presets
|
||||
data = await request.json()
|
||||
presets = data.get("presets", {})
|
||||
path = save_emotion_presets(presets)
|
||||
return web.json_response({"status": "success", "count": len(presets), "path": path})
|
||||
except ValueError as e:
|
||||
return web.json_response({"error": str(e)}, status=400)
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error saving IndexTTS emotion presets: {e}")
|
||||
return web.json_response({"status": "error", "error": str(e)}, status=500)
|
||||
|
||||
@PromptServer.instance.routes.get("/api/tts-audio-suite/available-characters")
|
||||
async def get_available_characters_endpoint(request):
|
||||
"""API endpoint to get available TTS character voices including aliases"""
|
||||
try:
|
||||
from utils.voice import discovery as voice_discovery_module
|
||||
characters = list(voice_discovery_module.get_available_characters())
|
||||
aliases = list(voice_discovery_module.voice_discovery.get_character_aliases().keys())
|
||||
# Combine and deduplicate
|
||||
all_chars = sorted(set(characters + aliases))
|
||||
return web.json_response({"characters": all_chars})
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error retrieving available characters: {e}")
|
||||
return web.json_response({"characters": [], "error": str(e)})
|
||||
|
||||
@PromptServer.instance.routes.get("/api/tts-audio-suite/available-languages")
|
||||
async def get_available_languages_endpoint(request):
|
||||
"""API endpoint to get available language codes from the canonical language mapper"""
|
||||
try:
|
||||
# Load language_mapper directly by file path to avoid package import issues
|
||||
language_mapper_path = os.path.join(os.path.dirname(__file__), "utils", "models", "language_mapper.py")
|
||||
spec = importlib.util.spec_from_file_location("language_mapper_module", language_mapper_path)
|
||||
language_mapper_module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(language_mapper_module)
|
||||
|
||||
# Get all unique canonical language codes (the values in LANGUAGE_ALIASES)
|
||||
languages = sorted(set(language_mapper_module.LANGUAGE_ALIASES.values()))
|
||||
return web.json_response({"languages": languages})
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error retrieving available languages: {e}")
|
||||
# Fallback list
|
||||
return web.json_response({"languages": ["en", "de", "fr", "ja", "es", "it", "pt", "th", "no"], "error": str(e)})
|
||||
|
||||
@PromptServer.instance.routes.get("/api/tts-audio-suite/omnivoice-presets")
|
||||
async def get_omnivoice_presets_endpoint(request):
|
||||
"""Return the persisted OmniVoice instruction builder preset library."""
|
||||
try:
|
||||
library_path = _get_omnivoice_preset_library_path()
|
||||
if not os.path.exists(library_path):
|
||||
return web.json_response({"presets": [], "builtinStates": {}, "builtinLayouts": {}})
|
||||
with open(library_path, "r", encoding="utf-8") as f:
|
||||
payload = json.load(f)
|
||||
presets = payload.get("presets", []) if isinstance(payload, dict) else []
|
||||
builtin_states = payload.get("builtinStates", {}) if isinstance(payload, dict) else {}
|
||||
builtin_layouts = payload.get("builtinLayouts", {}) if isinstance(payload, dict) else {}
|
||||
if not isinstance(presets, list):
|
||||
presets = []
|
||||
if not isinstance(builtin_states, dict):
|
||||
builtin_states = {}
|
||||
if not isinstance(builtin_layouts, dict):
|
||||
builtin_layouts = {}
|
||||
return web.json_response({"presets": presets, "builtinStates": builtin_states, "builtinLayouts": builtin_layouts})
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error retrieving OmniVoice preset library: {e}")
|
||||
return web.json_response({"presets": [], "builtinStates": {}, "builtinLayouts": {}, "error": str(e)}, status=500)
|
||||
|
||||
@PromptServer.instance.routes.post("/api/tts-audio-suite/omnivoice-presets")
|
||||
async def save_omnivoice_presets_endpoint(request):
|
||||
"""Persist the OmniVoice instruction builder preset library."""
|
||||
try:
|
||||
data = await request.json()
|
||||
presets = data.get("presets", [])
|
||||
builtin_states = data.get("builtinStates", {})
|
||||
builtin_layouts = data.get("builtinLayouts", {})
|
||||
if not isinstance(presets, list):
|
||||
return web.json_response({"error": "presets must be a list"}, status=400)
|
||||
if not isinstance(builtin_states, dict):
|
||||
return web.json_response({"error": "builtinStates must be an object"}, status=400)
|
||||
if not isinstance(builtin_layouts, dict):
|
||||
return web.json_response({"error": "builtinLayouts must be an object"}, status=400)
|
||||
|
||||
library_path = _get_omnivoice_preset_library_path()
|
||||
payload = {"presets": presets, "builtinStates": builtin_states, "builtinLayouts": builtin_layouts}
|
||||
with open(library_path, "w", encoding="utf-8") as f:
|
||||
json.dump(payload, f, ensure_ascii=False, indent=2)
|
||||
return web.json_response({"status": "success", "count": len(presets)})
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error saving OmniVoice preset library: {e}")
|
||||
return web.json_response({"status": "error", "error": str(e)}, status=500)
|
||||
|
||||
@PromptServer.instance.routes.get("/api/tts-audio-suite/voice-input-devices")
|
||||
async def get_voice_input_devices_endpoint(request):
|
||||
"""Return input devices without risking a main-process PortAudio hang."""
|
||||
try:
|
||||
import json
|
||||
import subprocess
|
||||
|
||||
probe_script = r"""
|
||||
import json
|
||||
import sounddevice as sd
|
||||
|
||||
devices = []
|
||||
seen = set()
|
||||
for device in sd.query_devices():
|
||||
try:
|
||||
max_input_channels = int(device.get("max_input_channels", 0))
|
||||
except Exception:
|
||||
max_input_channels = 0
|
||||
if max_input_channels <= 0:
|
||||
continue
|
||||
|
||||
name = str(device.get("name", "")).strip()
|
||||
if not name or name in seen:
|
||||
continue
|
||||
|
||||
seen.add(name)
|
||||
devices.append(name)
|
||||
|
||||
print(json.dumps({"devices": devices}))
|
||||
"""
|
||||
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", probe_script],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=8,
|
||||
check=False,
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
stderr = (result.stderr or "").strip()
|
||||
stdout = (result.stdout or "").strip()
|
||||
error_message = stderr or stdout or f"device probe exited with code {result.returncode}"
|
||||
return web.json_response({"devices": [], "error": error_message}, status=500)
|
||||
|
||||
payload = json.loads(result.stdout or "{}")
|
||||
devices = payload.get("devices", [])
|
||||
if not isinstance(devices, list):
|
||||
devices = []
|
||||
|
||||
return web.json_response({"devices": devices})
|
||||
except subprocess.TimeoutExpired:
|
||||
return web.json_response(
|
||||
{"devices": [], "error": "Timed out while probing audio input devices. Leaving the dropdown on system default avoids startup hangs."},
|
||||
status=504,
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error retrieving voice input devices: {e}")
|
||||
return web.json_response({"devices": [], "error": str(e)}, status=500)
|
||||
|
||||
@PromptServer.instance.routes.post("/api/tts-audio-suite/settings")
|
||||
async def set_inline_tag_settings_endpoint(request):
|
||||
"""API endpoint to receive settings from frontend for inline edit tags and restore VC"""
|
||||
print("🔧 Settings endpoint called") # Immediate print to verify endpoint is reached
|
||||
try:
|
||||
data = await request.json()
|
||||
precision = data.get("precision", "auto")
|
||||
device = data.get("device", "auto")
|
||||
vc_engine = data.get("vc_engine", "chatterbox_23lang")
|
||||
cosyvoice_variant = data.get("cosyvoice_variant", "RL")
|
||||
|
||||
print(f"🔧 Received settings: precision={precision}, device={device}, vc_engine={vc_engine}, cosyvoice_variant={cosyvoice_variant}")
|
||||
|
||||
# Import edit_post_processor using normal import to ensure we get the same module instance
|
||||
# that will be used during workflow execution
|
||||
# CRITICAL: Must use the same module instance, not create a new one via importlib!
|
||||
try:
|
||||
from utils.audio import edit_post_processor as edit_post_processor_module
|
||||
except ImportError:
|
||||
# Fallback: Load directly by file path if normal import fails
|
||||
edit_post_processor_path = os.path.join(os.path.dirname(__file__), "utils", "audio", "edit_post_processor.py")
|
||||
spec = importlib.util.spec_from_file_location("utils.audio.edit_post_processor", edit_post_processor_path)
|
||||
edit_post_processor_module = importlib.util.module_from_spec(spec)
|
||||
sys.modules["utils.audio.edit_post_processor"] = edit_post_processor_module # Register in sys.modules!
|
||||
spec.loader.exec_module(edit_post_processor_module)
|
||||
|
||||
# Store in global settings that edit_post_processor can access
|
||||
edit_post_processor_module.set_inline_tag_settings(precision=precision, device=device, vc_engine=vc_engine, cosyvoice_variant=cosyvoice_variant)
|
||||
|
||||
return web.json_response({"status": "success", "precision": precision, "device": device, "vc_engine": vc_engine, "cosyvoice_variant": cosyvoice_variant})
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error setting inline tag settings: {e}")
|
||||
return web.json_response({"status": "error", "error": str(e)})
|
||||
|
||||
def get_voice_discovery_module():
|
||||
"""Return the shared discovery module used by nodes and save notifications."""
|
||||
from utils.voice import discovery as voice_discovery_module
|
||||
return voice_discovery_module
|
||||
|
||||
def resolve_character_voice(voice_name):
|
||||
"""Resolve a dropdown key through the shared discovery cache."""
|
||||
voice_discovery_module = get_voice_discovery_module()
|
||||
voice_discovery_module.get_available_voices(force_refresh=False)
|
||||
return voice_discovery_module.load_voice_reference(voice_name)
|
||||
|
||||
@PromptServer.instance.routes.get("/api/tts-audio-suite/voice-library")
|
||||
async def get_voice_library_endpoint(request):
|
||||
"""Return current dropdown keys for Character Voices."""
|
||||
try:
|
||||
voice_discovery_module = get_voice_discovery_module()
|
||||
force_refresh = request.query.get("refresh", "0").strip().lower() in {"1", "true", "yes"}
|
||||
voices = voice_discovery_module.get_available_voices(force_refresh=force_refresh)
|
||||
response = web.json_response({"voices": voices})
|
||||
response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate"
|
||||
return response
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error serving voice library: {e}")
|
||||
return web.json_response({"error": str(e)}, status=500)
|
||||
|
||||
@PromptServer.instance.routes.get("/api/tts-audio-suite/voice-preview")
|
||||
async def get_voice_preview_endpoint(request):
|
||||
"""
|
||||
Stream selected Character Voices dropdown audio for browser preview playback.
|
||||
|
||||
Query params:
|
||||
- voice_name: exact dropdown key from get_available_voices()
|
||||
"""
|
||||
try:
|
||||
voice_name = request.query.get("voice_name", "").strip()
|
||||
if not voice_name or voice_name == "none":
|
||||
return web.json_response({"error": "voice_name is required and cannot be 'none'"}, status=400)
|
||||
|
||||
audio_path, _ = resolve_character_voice(voice_name)
|
||||
|
||||
if not audio_path or not os.path.exists(audio_path):
|
||||
return web.json_response({"error": f"Voice file not found: {voice_name}"}, status=404)
|
||||
|
||||
# Direct stream of resolved local audio file.
|
||||
response = web.FileResponse(path=audio_path)
|
||||
response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate"
|
||||
return response
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error serving voice preview audio: {e}")
|
||||
return web.json_response({"error": str(e)}, status=500)
|
||||
|
||||
@PromptServer.instance.routes.get("/api/tts-audio-suite/voice-info")
|
||||
async def get_voice_info_endpoint(request):
|
||||
"""Return canonical metadata for a Character Voices dropdown entry."""
|
||||
try:
|
||||
voice_name = request.query.get("voice_name", "").strip()
|
||||
if not voice_name or voice_name == "none":
|
||||
return web.json_response({"error": "voice_name is required and cannot be 'none'"}, status=400)
|
||||
|
||||
audio_path, reference_text = resolve_character_voice(voice_name)
|
||||
if not audio_path or not os.path.exists(audio_path):
|
||||
return web.json_response({"error": f"Voice file not found: {voice_name}"}, status=404)
|
||||
|
||||
response = web.json_response({
|
||||
"voice_name": voice_name,
|
||||
"reference_text": reference_text or "",
|
||||
})
|
||||
response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate"
|
||||
return response
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error serving voice metadata: {e}")
|
||||
return web.json_response({"error": str(e)}, status=500)
|
||||
|
||||
@PromptServer.instance.routes.post("/api/tts-audio-suite/audio-analyzer-preview")
|
||||
async def audio_analyzer_preview_endpoint(request):
|
||||
"""
|
||||
Analyze file-based Audio Wave Analyzer inputs without queueing the ComfyUI graph.
|
||||
|
||||
Connected AUDIO inputs still require graph execution because the browser/backend
|
||||
route cannot access an upstream tensor that has not been computed.
|
||||
"""
|
||||
try:
|
||||
data = await request.json()
|
||||
audio_file = (data.get("audio_file") or "").strip()
|
||||
if not audio_file:
|
||||
return web.json_response({"error": "audio_file is required for preview analysis"}, status=400)
|
||||
|
||||
node_id = str(data.get("node_id") or "preview")
|
||||
|
||||
analyzer_node_path = os.path.join(os.path.dirname(__file__), "nodes", "audio", "analyzer_node.py")
|
||||
spec = importlib.util.spec_from_file_location("tts_audio_suite_audio_analyzer_node", analyzer_node_path)
|
||||
analyzer_node_module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(analyzer_node_module)
|
||||
|
||||
analyzer_node = analyzer_node_module.AudioAnalyzerNode()
|
||||
analyzer_node.analyze_audio(
|
||||
audio_file=audio_file,
|
||||
analysis_method=data.get("analysis_method", "silence"),
|
||||
precision_level=data.get("precision_level", "milliseconds"),
|
||||
visualization_points=int(data.get("visualization_points", 2000)),
|
||||
audio=None,
|
||||
options=data.get("options"),
|
||||
manual_regions=data.get("manual_regions", ""),
|
||||
region_labels=data.get("region_labels", ""),
|
||||
export_format=data.get("export_format", "f5tts"),
|
||||
node_id=node_id,
|
||||
)
|
||||
|
||||
import folder_paths
|
||||
cache_file = os.path.join(folder_paths.get_output_directory(), f"audio_analyzer_cache_{node_id}.json")
|
||||
with open(cache_file, "r", encoding="utf-8") as f:
|
||||
payload = json.load(f)
|
||||
|
||||
response = web.json_response(payload)
|
||||
response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate"
|
||||
return response
|
||||
except Exception as e:
|
||||
print(f"⚠️ Audio analyzer preview failed: {e}")
|
||||
return web.json_response({"error": str(e)}, status=500)
|
||||
|
||||
@PromptServer.instance.routes.get("/api/tts-audio-suite/training-progress")
|
||||
async def get_training_progress_endpoint(request):
|
||||
"""Return live training progress snapshots for one or all tracked training nodes."""
|
||||
try:
|
||||
from engines.training.progress_registry import get_training_progress_snapshot
|
||||
|
||||
node_id = request.query.get("node_id")
|
||||
snapshot = get_training_progress_snapshot(node_id=node_id)
|
||||
response = web.json_response({"nodes": snapshot})
|
||||
response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate"
|
||||
return response
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error retrieving training progress: {e}")
|
||||
return web.json_response({"nodes": {}, "error": str(e)}, status=500)
|
||||
except Exception as e:
|
||||
print(f"⚠️ Could not setup API routes: {e}")
|
||||
|
||||
# Setup API routes when extension loads
|
||||
setup_api_routes()
|
||||
|
||||
# nodes.py already handles all the startup output and status reporting
|
||||
# nodes.py already handles all the startup output and status reporting
|
||||
@@ -1,422 +0,0 @@
|
||||
# 🎨 Advanced TTS Tag Editor Node
|
||||
|
||||
## Overview
|
||||
|
||||
Advanced multiline string editor node that extends ComfyUI's standard multiline widget with a sidebar containing context-aware controls for TTS-specific tags (character switching, parameters, pauses, and engine-native inline controls). The node features real-time tag generation, preset management, engine-aware quick swapping, and intelligent syntax support based on the CHARACTER_SWITCHING_GUIDE and PARAMETER_SWITCHING_GUIDE.
|
||||
|
||||
---
|
||||
|
||||
## Core Features
|
||||
|
||||
### 1. Editor Interface
|
||||
|
||||
- **Textarea**: Standard multiline editor (like ComfyUI's native string node)
|
||||
- **Left Sidebar**: Vertical control panel with buttons and dropdowns
|
||||
- **Smart Selection Detection**: Identify if selection is `[Character]text` or plain text
|
||||
- **Caret Position Tracking**: Insert tags at cursor position or wrap selection
|
||||
- **Mid-Sentence Tag Insertion**: Tags can be inserted anywhere in text, not just at the beginning (e.g., `"text text [alice|seed:2] more text"`)
|
||||
- **Undo/Redo Stack**: Full history of all edits with visual indicators, accessible via buttons and keyboard shortcuts (Ctrl+Z / Ctrl+Shift+Z)
|
||||
|
||||
### 2. Sidebar Controls - Basic
|
||||
|
||||
#### Character Management
|
||||
- Dropdown selector for available characters (auto-discovered from `voices_examples/`)
|
||||
- Text input to specify custom character name
|
||||
- Button: "Add Character" → wraps selection or inserts at caret as `[CharacterName]text`
|
||||
- Shows current available characters with icons/indicators
|
||||
|
||||
#### Parameter Buttons
|
||||
(appear if character is selected)
|
||||
|
||||
- `[Seed]` - Opens number input (0-4294967295) → adds `|seed:X`
|
||||
- `[Temp]` - Opens slider (0.1-2.0) → adds `|temperature:X`
|
||||
- `[CFG]` - Opens slider (engine-specific range) → adds `|cfg:X`
|
||||
- `[Speed]` - Slider (0.5-2.0, F5-TTS only) → adds `|speed:X`
|
||||
- `[Steps]` - Spinner (1-100, VibeVoice/IndexTTS) → adds `|inference_steps:X`
|
||||
- `[Top-P]` / `[Top-K]` - For advanced engines
|
||||
|
||||
#### Pause Control
|
||||
- Time input field with duration selector (milliseconds/seconds)
|
||||
- Button: "Insert Pause" → adds `[pause:XXXms]` or `[pause:XXs]` at caret
|
||||
- Quick presets: "0.5s", "1s", "2s" buttons
|
||||
|
||||
#### Language Control
|
||||
- Dropdown for supported languages (en, de, fr, ja, es, it, pt, th, no)
|
||||
- Works with character dropdown: `[language:Character]` format
|
||||
- Shows which models are available/loaded
|
||||
|
||||
#### Undo/Redo Controls
|
||||
- **Undo Button** (`[↶]`) - Revert last change
|
||||
- **Redo Button** (`[↷]`) - Restore undone change
|
||||
- **History Indicator** - Shows current position in edit history (e.g., "Step 5/10")
|
||||
- Keyboard shortcuts: `Ctrl+Z` (Undo), `Ctrl+Shift+Z` (Redo)
|
||||
- Full persistence: Undo/redo history survives session reload
|
||||
|
||||
### 3. Preset System
|
||||
|
||||
#### Preset Management UI
|
||||
- Three preset slots (Preset 1, Preset 2, Preset 3)
|
||||
- Each slot has:
|
||||
- Save button → captures configured tags
|
||||
- Load button → applies preset to selection/caret
|
||||
- Delete button → clears saved preset
|
||||
- Label field → custom name for the preset
|
||||
|
||||
#### Preset Content
|
||||
- Stores: character, language, and all parameters
|
||||
- Example: "Alice Calm" = `[de:Alice|seed:42|temp:0.3|cfg:0.7]`
|
||||
- Can be applied as wrapper around selection or at caret
|
||||
|
||||
#### Preset Persistence
|
||||
- Store in ComfyUI widget state (node.serialize/deserialize)
|
||||
- Also optionally save to local config file for sharing
|
||||
- **Full Session Persistence**: All presets, settings, and metadata survive ComfyUI session reload
|
||||
|
||||
### 4. Advanced Features
|
||||
|
||||
#### Syntax Validation
|
||||
- Real-time visual indicators for tag syntax errors
|
||||
- Highlight brackets, detect mismatched `[` `]`
|
||||
- Warn on invalid parameter values (out of range)
|
||||
- Suggest corrections (e.g., `temp` → `temperature`)
|
||||
|
||||
#### Multi-Selection Support
|
||||
- Apply same tag to multiple selected segments
|
||||
- Split by paragraph breaks or sentence punctuation
|
||||
- Batch apply parameters across multiple `[Character]text` blocks
|
||||
|
||||
#### Tag Inspector
|
||||
- Show existing tags in selection/current line
|
||||
- Checkbox UI to toggle tags on/off temporarily
|
||||
- Click a character, language, audio-reference, parameter, or supported engine-native inline tag to open a color-coded quick-swap palette
|
||||
- Click a palette option once to keep the palette open, then click again to commit; press-and-hold, drag, and release selects in one gesture
|
||||
- Palette choices follow the selected inline engine (including IndexTTS-2, Higgs Audio v3, Step Audio EditX, CosyVoice3, and OmniVoice)
|
||||
- Quoted IndexTTS-2 text emotion tags remain direct editable text and do not open a replacement palette
|
||||
- Long bracket and angle tags wrap inside the editor rather than overflowing horizontally
|
||||
|
||||
#### Auto-Formatting
|
||||
- Button: "Auto-Format Tags" → organize tags consistently
|
||||
- Validates and cleans malformed tags
|
||||
- Groups parameters in standard order: `[char|seed|temp|cfg|speed]`
|
||||
|
||||
#### Visual Feedback
|
||||
- Color-code different tag types:
|
||||
- Character tags: **Blue** `[Character]`
|
||||
- Parameters: **Green** `|seed:value`
|
||||
- Pauses: **Orange** `[pause:1s]`
|
||||
- Language tags: **Purple** `[de:Character]`
|
||||
- Hover tooltips explaining what each tag does
|
||||
|
||||
---
|
||||
|
||||
## SRT Support & Formatting
|
||||
|
||||
### SRT-Aware Mode Detection
|
||||
|
||||
The editor **automatically detects SRT format** and enables specialized visualization:
|
||||
|
||||
```srt
|
||||
1
|
||||
00:00:01,000 --> 00:00:04,000
|
||||
[Alice] Hello! I'm Alice speaking with precise timing.
|
||||
|
||||
2
|
||||
00:00:04,500 --> 00:00:09,500
|
||||
[Bob|seed:42|temperature:0.5] And I'm Bob with parameters!
|
||||
|
||||
3
|
||||
00:00:10,000 --> 00:00:14,000
|
||||
[de:Narrator] Zurück zum Erzähler in Deutsch.
|
||||
```
|
||||
|
||||
### SRT Color Coding
|
||||
|
||||
When SRT format is detected, the editor applies specialized syntax highlighting:
|
||||
|
||||
- **SRT Index**: `1`, `2`, `3` → **Gray** background (disabled/informational)
|
||||
- **Timestamp**: `00:00:01,000 --> 00:00:04,000` → **Cyan** (timing information)
|
||||
- **Text Content**: Plain text → Black
|
||||
- **Character Tags**: `[Character]` → **Blue**
|
||||
- **Language Tags**: `[de:Character]` → **Purple**
|
||||
- **Parameters**: `|seed:42|temp:0.5` → **Green**
|
||||
- **Pauses**: `[pause:1s]` → **Orange**
|
||||
- **Blank Lines**: Between entries → Gray
|
||||
|
||||
### SRT-Specific Sidebar Options
|
||||
|
||||
When editing SRT content, the sidebar includes:
|
||||
|
||||
#### Subtitle Entry Navigation
|
||||
- Previous/Next buttons to jump between SRT entries
|
||||
- Current entry indicator: "Entry 2 of 5"
|
||||
- Quick jump dropdown to select specific subtitle
|
||||
|
||||
#### Timing Tools
|
||||
- **Insert Timing**: Auto-generates timestamps based on duration
|
||||
- **Adjust Timing**: Shift all subsequent subtitles by offset
|
||||
- **Validate Timing**: Check for overlaps and gaps
|
||||
|
||||
#### Per-Entry Parameter Application
|
||||
- Apply parameters to **current entry only**
|
||||
- Apply parameters to **selected entries**
|
||||
- Apply parameters to **all entries**
|
||||
- Clear all tags from current entry
|
||||
|
||||
#### SRT Validation
|
||||
- Check syntax: proper index/timestamp/text format
|
||||
- Warn on timing overlaps
|
||||
- Suggest fixes for malformed entries
|
||||
- Preview final SRT before saving
|
||||
|
||||
### SRT Tag Application Examples
|
||||
|
||||
#### Applying Character to Current Entry
|
||||
```
|
||||
Select entry 2
|
||||
Click "Add Character" → [Alice]
|
||||
Result:
|
||||
2
|
||||
00:00:04,500 --> 00:00:09,500
|
||||
[Alice] And I'm Bob with parameters!
|
||||
```
|
||||
|
||||
#### Adding Parameters to Selected Entries
|
||||
```
|
||||
Select entries 1-3
|
||||
Click [Seed] → 42
|
||||
Result:
|
||||
1
|
||||
00:00:01,000 --> 00:00:04,000
|
||||
[Alice|seed:42] Hello! I'm Alice speaking...
|
||||
|
||||
2
|
||||
00:00:04,500 --> 00:00:09,500
|
||||
[Alice|seed:42] And I'm Bob with parameters!
|
||||
|
||||
3
|
||||
00:00:10,000 --> 00:00:14,000
|
||||
[Alice|seed:42] Zurück zum Erzähler in Deutsch.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## UI Layout
|
||||
|
||||
```
|
||||
┌─ Advanced TTS Editor ─────────────────────────────┐
|
||||
│ ┌───────────────┬─────────────────────────────────┐│
|
||||
│ │ SIDEBAR │ TEXTAREA ││
|
||||
│ │ │ (multiline editor) ││
|
||||
│ │ ╔═══════════╗ │ 1 ││
|
||||
│ │ ║ Character ║ │ 00:00:01,000 --> 00:00:04,000 ││
|
||||
│ │ ║ [Dropdown]║ │ [Alice] Hello there! ││
|
||||
│ │ ║[+ Add] ║ │ ││
|
||||
│ │ ╚═══════════╝ │ 2 ││
|
||||
│ │ │ 00:00:04,500 --> 00:00:09,500 ││
|
||||
│ │ ╔═══════════╗ │ [Bob|seed:42] Nice to meet ││
|
||||
│ │ ║Parameters ║ │ ││
|
||||
│ │ ║[Seed ▼] ║ │ 3 ││
|
||||
│ │ ║[Temp -/-]║ │ 00:00:10,000 --> 00:00:14,000 ││
|
||||
│ │ ║[CFG -/-]║ │ [pause:1s] ││
|
||||
│ │ ║[Speed -/-]║ │ ││
|
||||
│ │ ╚═══════════╝ │ ││
|
||||
│ │ │ ││
|
||||
│ │ ╔═══════════╗ │ ││
|
||||
│ │ ║ Pause ║ │ ││
|
||||
│ │ ║[ 1.0 ▼] │ │ ││
|
||||
│ │ ║[Insert] ║ │ ││
|
||||
│ │ ╚═══════════╝ │ ││
|
||||
│ │ │ ││
|
||||
│ │ ╔═══════════╗ │ ││
|
||||
│ │ ║ Presets ║ │ ││
|
||||
│ │ ║P1[Save] ║ │ ││
|
||||
│ │ ║ [Load] ║ │ ││
|
||||
│ │ ║ [Del ] ║ │ ││
|
||||
│ │ ║P2[Save] ║ │ ││
|
||||
│ │ ║ [Load] ║ │ ││
|
||||
│ │ ║ [Del ] ║ │ ││
|
||||
│ │ ║P3[Save] ║ │ ││
|
||||
│ │ ║ [Load] ║ │ ││
|
||||
│ │ ║ [Del ] ║ │ ││
|
||||
│ │ ╚═══════════╝ │ ││
|
||||
│ │ │ ││
|
||||
│ │ ╔═══════════╗ │ ││
|
||||
│ │ ║ History ║ │ ││
|
||||
│ │ ║[↶][↷] 5/10║ │ ││
|
||||
│ │ ╚═══════════╝ │ ││
|
||||
│ │ │ ││
|
||||
│ │ [Format] │ ││
|
||||
│ │ [Validate] │ ││
|
||||
│ └───────────────┴─────────────────────────────────┘│
|
||||
│ Status: SRT Mode | Entry 2/5 │
|
||||
└────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Session Persistence & State Management
|
||||
|
||||
**Complete State Persistence**:
|
||||
- **Text Content**: Full edit history with all undo/redo states
|
||||
- **Undo/Redo Stack**: Complete history serialized and restored on session reload
|
||||
- **Preset Data**: All custom presets and their labels
|
||||
- **Sidebar Settings**: Last used character, language, parameters, pause duration
|
||||
- **UI State**: Sidebar panel expanded/collapsed state, last active tab
|
||||
- **Cursor Position**: Last known caret position restored on reload
|
||||
- **SRT Mode Settings**: Timing configuration, entry navigation state
|
||||
|
||||
**Storage Mechanism**:
|
||||
- Serialize all state to ComfyUI node widget state
|
||||
- Automatic backup to browser localStorage for redundancy
|
||||
- JSON-based format for easy inspection and manual recovery
|
||||
- No data loss on browser refresh, ComfyUI restart, or system reboot
|
||||
|
||||
---
|
||||
|
||||
## Technical Implementation
|
||||
|
||||
### Backend (Python Node)
|
||||
|
||||
**File**: `nodes/text/tts_tag_editor_node.py`
|
||||
|
||||
- Extend standard ComfyUI node pattern
|
||||
- Discover available characters/voices from `utils/voice/discovery.py`
|
||||
- Parse existing tags using `utils/text/character_parser.py` and `utils/text/segment_parameters.py`
|
||||
- Validate parameters against engine specs
|
||||
- **Full State Serialization**: Handle undo/redo stack, presets, and all sidebar settings in node state
|
||||
- Detect and validate SRT format
|
||||
- **History Persistence**: Serialize complete edit history to widget state for session recovery
|
||||
|
||||
### Frontend (TypeScript/Vue)
|
||||
|
||||
**Files**:
|
||||
- `web/tts_tag_editor.js` - Core editor logic
|
||||
- `web/tts_tag_editor_interface.js` - ComfyUI integration
|
||||
- `web/tts_tag_editor_ui.js` - Sidebar components
|
||||
- `web/tts_tag_editor.css` - Styling and color schemes
|
||||
- `web/tts_tag_editor_srt.js` - SRT-specific formatting and validation
|
||||
|
||||
**Features**:
|
||||
- Extend standard multiline widget with sidebar panel
|
||||
- Implement event handlers for all buttons/inputs
|
||||
- Add CSS for visual tag highlighting and styling
|
||||
- **Undo/Redo Implementation**: Maintain immutable history stack with efficient state snapshots
|
||||
- **Keyboard Shortcuts**: Ctrl+Z (Undo), Ctrl+Shift+Z (Redo), and other common editor shortcuts
|
||||
- SRT format detection and specialized rendering
|
||||
- Syntax highlighting for SRT entries
|
||||
- **Persistent State Manager**: Auto-save to localStorage and node widget state
|
||||
|
||||
### Key Components
|
||||
|
||||
1. **Character Discovery**: Hook into existing voice system
|
||||
2. **Parameter Validation**: Reuse existing parameter filtering logic
|
||||
3. **Tag Parsing**: Leverage existing character parser and parameter parser (supports mid-sentence tags)
|
||||
4. **Preset Storage**: Use node widget serialization with full persistence
|
||||
5. **UI State Management**: Track cursor position, selection, active preset, all sidebar settings
|
||||
6. **Undo/Redo Manager**: Immutable history stack with efficient snapshots, keyboard support
|
||||
7. **History Persistence**: Serialize complete undo/redo stack to localStorage and widget state
|
||||
8. **SRT Detection & Validation**: Detect SRT format, validate structure, highlight syntax
|
||||
9. **SRT Navigation**: Jump between entries, manage per-entry operations
|
||||
10. **State Recovery**: Auto-restore all editor state on session reload (text, history, presets, settings)
|
||||
|
||||
### Integration Points
|
||||
|
||||
- `utils/voice/discovery.py` - Character discovery
|
||||
- `utils/text/character_parser.py` - Tag parsing
|
||||
- `utils/text/segment_parameters.py` - Parameter validation
|
||||
- `utils/timing/parser.py` - SRT format parsing and validation
|
||||
- Existing language models for language support
|
||||
|
||||
---
|
||||
|
||||
## Features to Consider Later (v2.0)
|
||||
|
||||
- Syntax highlighting with language server protocol (LSP)
|
||||
- Auto-complete for character names and parameters
|
||||
- Export/import presets as JSON
|
||||
- Macro system for complex tag sequences
|
||||
- Real-time validation with detailed error messages
|
||||
- Voice preview playback directly from tags
|
||||
- Template system for common scenes/dialogues
|
||||
- Regex tag replacement for bulk editing
|
||||
- Keyboard shortcuts for common operations
|
||||
- SRT subtitle duration calculator
|
||||
- Automatic SRT timing generation from text audio duration
|
||||
- Batch character/parameter application across multiple SRT files
|
||||
|
||||
---
|
||||
|
||||
## Files to Create
|
||||
|
||||
1. `nodes/text/tts_tag_editor_node.py` - Backend node
|
||||
2. `web/tts_tag_editor.js` - Main editor implementation
|
||||
3. `web/tts_tag_editor_ui.js` - Sidebar UI components
|
||||
4. `web/tts_tag_editor_srt.js` - SRT-specific features
|
||||
5. `web/tts_tag_editor.css` - Styling
|
||||
|
||||
---
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Basic Text Editing with Mid-Sentence Tags
|
||||
|
||||
```
|
||||
Once upon a time, [Narrator] there was a beautiful princess.
|
||||
[Alice] She lived in a [grand|seed:42] castle.
|
||||
[Bob] Who [de:Alice] spoke many languages.
|
||||
Back to [Bob|temperature:0.7] normal text.
|
||||
```
|
||||
|
||||
### Multi-Parameter Example
|
||||
|
||||
```
|
||||
[Alice|seed:42|temperature:0.3|cfg:0.7] She spoke softly.
|
||||
[Bob|seed:100|temperature:0.8|cfg:1.0] He shouted loudly!
|
||||
[de:Merchant|seed:55] Guten Tag, meine Damen!
|
||||
```
|
||||
|
||||
### SRT with Multilingual Characters
|
||||
|
||||
```
|
||||
1
|
||||
00:00:01,000 --> 00:00:04,000
|
||||
[Narrator] Welcome to our international story.
|
||||
|
||||
2
|
||||
00:00:04,500 --> 00:00:09,500
|
||||
[de:Alice|temperature:0.5] Hallo! Ich bin Alice.
|
||||
|
||||
3
|
||||
00:00:09,600 --> 00:00:14,000
|
||||
[fr:Bob|temperature:0.6] Bonjour! Je suis Bob.
|
||||
|
||||
4
|
||||
00:00:14,100 --> 00:00:18,000
|
||||
[Narrator] And they lived happily ever after.
|
||||
```
|
||||
|
||||
### Preset Application Example
|
||||
|
||||
Saved Preset: "Action Scene" = `[|seed:random|temperature:0.8|cfg:1.2]`
|
||||
|
||||
Apply to selection:
|
||||
```
|
||||
Before:
|
||||
[Hero] I will save you!
|
||||
[Villain] Never!
|
||||
[Hero] This ends now!
|
||||
|
||||
After:
|
||||
[Hero|seed:42|temperature:0.8|cfg:1.2] I will save you!
|
||||
[Villain|seed:100|temperature:0.8|cfg:1.2] Never!
|
||||
[Hero|seed:55|temperature:0.8|cfg:1.2] This ends now!
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [Character Switching Guide](CHARACTER_SWITCHING_GUIDE.md) - Multi-character and language switching
|
||||
- [Parameter Switching Guide](PARAMETER_SWITCHING_GUIDE.md) - Per-segment parameter control
|
||||
- [ChatterBox Special Tokens Guide](CHATTERBOX_V2_SPECIAL_TOKENS.md) - Emotion and sound tokens
|
||||
- README.md - Complete feature overview
|
||||
@@ -1,22 +0,0 @@
|
||||
# Auxiliary Model Layouts
|
||||
|
||||
Use this document for helper/post-process model folder structures and placement notes.
|
||||
These are not engine weights. They are optional models used by standalone utility nodes.
|
||||
|
||||
For repository/source URLs, use [AUX_MODEL_SOURCES.md](AUX_MODEL_SOURCES.md).
|
||||
|
||||
## Punctuation / Truecase
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/punctuation/
|
||||
├── punctuation_fullstop_truecase_english/
|
||||
├── punctuation_fullstop_truecase_romance/
|
||||
└── punct_cap_seg_47_language/
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- Models download lazily on first use by the punctuation node.
|
||||
- These are helper models for text cleanup, not TTS/ASR engines.
|
||||
- The English model is the lightest and is mainly intended for raw English ASR text.
|
||||
- The 47-language model is much heavier and should be treated as the multilingual fallback.
|
||||
@@ -1,18 +0,0 @@
|
||||
# Auxiliary Model Sources
|
||||
|
||||
Helper/post-process models used by standalone utility nodes.
|
||||
These are not engines and are documented separately to avoid polluting the engine registry.
|
||||
|
||||
## Punctuation / Truecase
|
||||
|
||||
Optional punctuation and true-casing post-process models for raw ASR text. Designed mainly for low-punctuation transcripts such as Granite outputs, but usable on any plain text.
|
||||
|
||||
**Used by:** 📝 ASR Punctuation / Truecase
|
||||
|
||||
| Model | Repo | Size | Auto-Download | Languages | Notes |
|
||||
|---|---|---|---|---|---|
|
||||
| English Fullstop + Truecase | [1-800-BAD-CODE/punctuation_fullstop_truecase_english](https://huggingface.co/1-800-BAD-CODE/punctuation_fullstop_truecase_english) | ~210MB | ✅ | English | English-only. Best on lowercased, unpunctuated ASR text. |
|
||||
| Romance Fullstop + Truecase | [1-800-BAD-CODE/punctuation_fullstop_truecase_romance](https://huggingface.co/1-800-BAD-CODE/punctuation_fullstop_truecase_romance) | ~144MB | ✅ | Spanish, French, Italian, Portuguese, Catalan, Galician | Romance-language punctuation and true-casing model. |
|
||||
| 47-Language Fullstop + Truecase | [1-800-BAD-CODE/punct_cap_seg_47_language](https://huggingface.co/1-800-BAD-CODE/punct_cap_seg_47_language) | ~1.11GB | ✅ | 47 languages | Heavier multilingual fallback with the widest language coverage supported by the punctuators package. |
|
||||
|
||||
*Generated from [tts_audio_suite_aux_models.yaml](Dev%20reports/tts_audio_suite_aux_models.yaml).*
|
||||
@@ -1,129 +1,41 @@
|
||||
# Version Bump Instructions for ComfyUI ChatterBox Voice
|
||||
|
||||
**⚠️ ALWAYS specify which component/engine is affected in changelog (e.g., "Fix Multiline TTS Tag Editor text persistence" not just "Fix text persistence")**
|
||||
|
||||
## Quick Reference for Future Version Bumps
|
||||
|
||||
### ⚠️ DO NOT CHECK CURRENT VERSION - The script auto-increments!
|
||||
|
||||
**Simply run with `patch`, `minor`, or `major` - NO need to look up current version**
|
||||
|
||||
### Recommended Command (Separate Commit & Changelog)
|
||||
|
||||
**⚠️ IMPORTANT: Use positional arguments, NOT --commit/--changelog flags**
|
||||
|
||||
```powershell
|
||||
# Windows: use the canonical ComfyUI environment for this project
|
||||
& 'J:\stablediffusion1111s2\Data\Packages\ComfyUIPy129\test_env_error\Scripts\python.exe' 'scripts\bump_version_enhanced.py' patch "<commit_desc>" "<changelog_desc>"
|
||||
|
||||
# OR: specify an exact version if needed
|
||||
& 'J:\stablediffusion1111s2\Data\Packages\ComfyUIPy129\test_env_error\Scripts\python.exe' 'scripts\bump_version_enhanced.py' <version> "<commit_desc>" "<changelog_desc>"
|
||||
```
|
||||
|
||||
```bash
|
||||
# Linux/macOS: use the available Python 3 interpreter
|
||||
python3 scripts/bump_version_enhanced.py patch "<commit_desc>" "<changelog_desc>"
|
||||
python3 scripts/bump_version_enhanced.py <version> --commit "<commit_desc>" --changelog "<changelog_desc>"
|
||||
```
|
||||
|
||||
### Examples
|
||||
```bash
|
||||
# Patch release (bug fixes)
|
||||
python3 scripts/bump_version_enhanced.py 3.2.9 \
|
||||
--commit "Fix character alias resolution and F5-TTS import issues" \
|
||||
--changelog "Bug fixes and stability improvements"
|
||||
|
||||
#### Multiline Format (Recommended Standard)
|
||||
# Minor release (new features)
|
||||
python3 scripts/bump_version_enhanced.py 3.3.0 \
|
||||
--commit "Add character support system for F5-TTS generation" \
|
||||
--changelog "Add character support for F5-TTS generation"
|
||||
|
||||
```powershell
|
||||
# Patch release (bug fixes) - CORRECT FORMAT
|
||||
& 'J:\stablediffusion1111s2\Data\Packages\ComfyUIPy129\test_env_error\Scripts\python.exe' 'scripts\bump_version_enhanced.py' 3.2.9 "Fix character alias resolution
|
||||
|
||||
Technical details:
|
||||
- Fix parser bypassing character tags in single mode
|
||||
- Improve character name validation logic
|
||||
- Add fallback handling for unrecognized tags" "Fix character name handling issues
|
||||
|
||||
- Fix character tags not being removed from TTS output
|
||||
- Improve character name recognition accuracy
|
||||
- Better error handling for invalid character names"
|
||||
|
||||
# Minor release (new features) - CORRECT FORMAT
|
||||
& 'J:\stablediffusion1111s2\Data\Packages\ComfyUIPy129\test_env_error\Scripts\python.exe' 'scripts\bump_version_enhanced.py' 3.3.0 "Add Higgs Audio 2 TTS engine
|
||||
|
||||
Implementation details:
|
||||
- Integrate boson_multimodal voice cloning system
|
||||
- Add unified adapter for consistent interface
|
||||
- Implement voice preset management" "Add Higgs Audio 2 TTS engine with voice cloning
|
||||
|
||||
- New realistic voice synthesis engine
|
||||
- Voice cloning from short audio samples
|
||||
- Multiple built-in voice presets available"
|
||||
|
||||
# Major release (breaking changes) - CORRECT FORMAT
|
||||
& 'J:\stablediffusion1111s2\Data\Packages\ComfyUIPy129\test_env_error\Scripts\python.exe' 'scripts\bump_version_enhanced.py' 4.0.0 "Complete unified architecture implementation
|
||||
|
||||
Breaking changes:
|
||||
- Migrate all nodes to unified interface pattern
|
||||
- Consolidate engine adapters and processors
|
||||
- Remove deprecated standalone variants" "Major architecture upgrade to unified system
|
||||
|
||||
- All TTS engines now use consistent interface
|
||||
- Better performance and memory management
|
||||
- Simplified workflows with consolidated nodes
|
||||
- Breaking: old standalone nodes removed"
|
||||
# Major release (breaking changes)
|
||||
python3 scripts/bump_version_enhanced.py 4.0.0 \
|
||||
--commit "Complete architecture refactoring with new modular structure" \
|
||||
--changelog "Project restructure for better maintainability"
|
||||
```
|
||||
|
||||
#### Auto-Increment Examples (Recommended)
|
||||
```powershell
|
||||
# Auto-increment patch version (4.5.25 → 4.5.26) - Windows
|
||||
& 'J:\stablediffusion1111s2\Data\Packages\ComfyUIPy129\test_env_error\Scripts\python.exe' 'scripts\bump_version_enhanced.py' patch "Fix character parsing issues" "Fix character name handling in TTS generation"
|
||||
|
||||
# Auto-increment minor version (4.5.25 → 4.6.0) - Windows
|
||||
& 'J:\stablediffusion1111s2\Data\Packages\ComfyUIPy129\test_env_error\Scripts\python.exe' 'scripts\bump_version_enhanced.py' minor "Add new TTS engine support" "Add Higgs Audio 2 TTS engine with voice cloning"
|
||||
```
|
||||
|
||||
#### Single-Line Format (Only for Super Minor Changes)
|
||||
```powershell
|
||||
& 'J:\stablediffusion1111s2\Data\Packages\ComfyUIPy129\test_env_error\Scripts\python.exe' 'scripts\bump_version_enhanced.py' patch "Fix typo in node tooltip" "Fix typo in audio analyzer tooltip"
|
||||
```
|
||||
|
||||
#### Dry-Run Preview (Test Before Committing)
|
||||
```powershell
|
||||
& 'J:\stablediffusion1111s2\Data\Packages\ComfyUIPy129\test_env_error\Scripts\python.exe' 'scripts\bump_version_enhanced.py' patch "Fix preview issues" "Fix preview not reflecting filter parameters" --dry-run
|
||||
```
|
||||
|
||||
#### Auto-Categorization System
|
||||
**IMPORTANT**: The script automatically sorts each line of multiline changelog entries into categories based on keywords:
|
||||
|
||||
**Keywords for each category:**
|
||||
- **Fixed** section: "fix", "bug", "error", "issue", "resolve", "correct", "patch", "crash", "problem", "broken", "compatibility"
|
||||
- **Added** section: "add", "new", "implement", "feature", "create", "introduce", "support" (also default for unmatched lines)
|
||||
- **Changed** section: "improve", "enhance", "optimize", "update", "modify", "better", "performance", "refactor"
|
||||
- **Removed** section: "remove", "delete", "deprecate", "drop", "eliminate"
|
||||
|
||||
**Writing Tips:**
|
||||
- **Be intentional with word choice** - start bullet points with appropriate keywords
|
||||
- **Line order doesn't matter** - script will reorganize lines by category automatically
|
||||
- **Each line categorized individually** - mix different change types in one changelog description
|
||||
- **Commit vs Changelog focus:**
|
||||
- **Commit**: Technical implementation details for developers
|
||||
- **Changelog**: User-facing benefits and impacts
|
||||
|
||||
**Command Syntax Notes:**
|
||||
- Multiline strings need proper quoting (opening quote on first line, closing quote on last line)
|
||||
- The Windows command uses PowerShell's `&` call operator and the canonical project Python path
|
||||
- The Linux/macOS command uses `python3`
|
||||
- Don't add manual category prefixes like "Fixed:" - script handles categorization automatically!
|
||||
|
||||
### Interactive Mode (Recommended for Complex Changes)
|
||||
|
||||
```powershell
|
||||
& 'J:\stablediffusion1111s2\Data\Packages\ComfyUIPy129\test_env_error\Scripts\python.exe' 'scripts\bump_version_enhanced.py' 3.2.9 --interactive
|
||||
```bash
|
||||
python3 scripts/bump_version_enhanced.py 3.2.9 --interactive
|
||||
```
|
||||
|
||||
### Legacy Mode (Same Description for Both)
|
||||
|
||||
```powershell
|
||||
& 'J:\stablediffusion1111s2\Data\Packages\ComfyUIPy129\test_env_error\Scripts\python.exe' 'scripts\bump_version_enhanced.py' 3.2.9 "Fix bugs and improve stability"
|
||||
```bash
|
||||
python3 scripts/bump_version_enhanced.py 3.2.9 "Fix bugs and improve stability"
|
||||
```
|
||||
|
||||
### What the Script Does
|
||||
|
||||
- Updates version in `nodes.py`, `README.md`, and `pyproject.toml`
|
||||
- Updates changelog with user-focused description
|
||||
- Creates git commit with developer-focused description
|
||||
@@ -133,42 +45,36 @@ Breaking changes:
|
||||
### When to Bump Versions
|
||||
|
||||
#### Patch (x.x.X)
|
||||
|
||||
- Bug fixes
|
||||
- Documentation updates
|
||||
- Performance improvements
|
||||
- Security patches
|
||||
|
||||
#### Minor (x.X.0)
|
||||
|
||||
- New features
|
||||
- New node types
|
||||
- Enhanced functionality
|
||||
- Backward-compatible changes
|
||||
|
||||
#### Major (X.0.0)
|
||||
|
||||
- Breaking changes
|
||||
- API changes
|
||||
- Architecture refactoring
|
||||
- Incompatible updates
|
||||
|
||||
### Pre-Bump Checklist
|
||||
|
||||
1. ✅ All changes tested and working
|
||||
2. ✅ User confirms functionality works
|
||||
3. ✅ Git working directory is clean
|
||||
4. ✅ All important changes committed
|
||||
|
||||
### Post-Bump Actions
|
||||
|
||||
- Script handles git commit automatically
|
||||
- Consider pushing to remote if appropriate
|
||||
- Update any external documentation
|
||||
- Notify users of significant changes
|
||||
|
||||
### Important Notes
|
||||
|
||||
- **Never manually edit version files** - always use the script
|
||||
- **Only bump when user confirms changes work**
|
||||
- **Read detailed guide**: `docs/Dev reports/CLAUDE_VERSION_MANAGEMENT_GUIDE.md`
|
||||
@@ -177,82 +83,16 @@ Breaking changes:
|
||||
### Commit vs Changelog Guidelines
|
||||
|
||||
#### Commit Description (--commit)
|
||||
|
||||
- **Developer diary**: What this specific version bump does
|
||||
- **Include internal details**: Bug fixes, refactoring, technical changes
|
||||
- **Development perspective**: "Fixed F5-TTS edit issues after refactoring"
|
||||
- **Can mention temporary problems**: "Restore functionality broken by restructure"
|
||||
|
||||
#### Changelog Description (--changelog)
|
||||
|
||||
#### Changelog Description (--changelog)
|
||||
- **User perspective only**: Write for users upgrading from previous version
|
||||
- **Use layman terms**: Explain WHAT the feature does, not just technical names
|
||||
- **Don't document internal fixes**: Skip temporary issues introduced and fixed during development
|
||||
- **Focus on net result**: What changed for the user, not the development process
|
||||
- **Example**: If refactoring broke something then fixed it, only mention the refactoring benefit
|
||||
|
||||
**🎯 CRITICAL: Always Specify Engine/Component When Applicable**
|
||||
- **Include engine name OR component name in titles**: Users need to know which TTS engine or node/feature is affected
|
||||
- **❌ Bad**: "Fix SRT processing issues"
|
||||
- **✅ Good**: "Fix ChatterBox SRT processing issues"
|
||||
- **❌ Bad**: "Fix text persistence in workflows"
|
||||
- **✅ Good**: "Fix Multiline TTS Tag Editor text persistence in workflows"
|
||||
- **❌ Bad**: "Fix voice cloning errors"
|
||||
- **✅ Good**: "Fix Higgs Audio voice cloning errors"
|
||||
- **❌ Bad**: "Improve audio quality"
|
||||
- **✅ Good**: "Improve F5-TTS audio quality"
|
||||
|
||||
**Engine-Specific Title Examples:**
|
||||
- "Fix Higgs Audio compatibility with transformers 4.46.3+"
|
||||
- "Add VibeVoice multi-speaker support"
|
||||
- "Fix ChatterBox character switching in SRT mode"
|
||||
- "Improve F5-TTS speech editing accuracy"
|
||||
- "Fix RVC voice conversion model loading"
|
||||
- "Add Audio Wave Analyzer timing extraction"
|
||||
|
||||
**User-Friendly Language Rules:**
|
||||
- ❌ "Add support for MelBandRoFormer models"
|
||||
- ✅ "Add support for advanced audio separation models (vocal/instrumental isolation)"
|
||||
- ❌ "Fix VibeVoice parameter propagation to cached engine instances"
|
||||
- ✅ "Fix VibeVoice voice settings not applying correctly in some cases"
|
||||
- ❌ "Resolve ChatterboxVC loading with .safetensors format"
|
||||
- ✅ "Fix voice conversion failing to load newer model formats"
|
||||
|
||||
**Always explain obscure terms:**
|
||||
- Model names → What they do (e.g., "audio separation", "voice conversion", "text-to-speech")
|
||||
- Technical errors → User impact (e.g., "crashes when...", "fails to generate audio")
|
||||
- Internal components → User-facing features (e.g., "character voice switching", "long audio generation")
|
||||
|
||||
#### Key Principle
|
||||
|
||||
**Commit = Development diary, Changelog = User release notes**
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
**"Git working directory is not clean"**
|
||||
```bash
|
||||
# Check what files are modified
|
||||
git status
|
||||
|
||||
# Add/commit changes first, then run version bump
|
||||
git add .
|
||||
git commit -m "Prepare for version bump"
|
||||
```
|
||||
|
||||
**"Error: Invalid version format"**
|
||||
- Use semantic versioning: `4.5.25` (not `v4.5.25` or `4.5`)
|
||||
- Or use auto-increment: `patch`, `minor`, `major`
|
||||
|
||||
**Command syntax errors with multiline**
|
||||
- Make sure opening quote is on same line as `--commit` or `--changelog`
|
||||
- Make sure closing quote is on its own line
|
||||
- On Windows, use the PowerShell command shown above
|
||||
- On Linux/macOS, use `python3`
|
||||
|
||||
**Want to see what will happen before committing?**
|
||||
```powershell
|
||||
# Add --dry-run to preview changelog categorization
|
||||
& 'J:\stablediffusion1111s2\Data\Packages\ComfyUIPy129\test_env_error\Scripts\python.exe' 'scripts\bump_version_enhanced.py' patch "description" "changelog" --dry-run
|
||||
```
|
||||
**Commit = Development diary, Changelog = User release notes**
|
||||
@@ -10,7 +10,6 @@ ChatterBox Voice now supports seamless character and narrator switching with **l
|
||||
|
||||
- **🎭 Universal Character Parsing** - Works with both F5TTS and ChatterBox engines
|
||||
- **🌍 Language-Aware Switching** - Switch both character AND language with `[lang:character]` syntax
|
||||
- **⚙️ Per-Segment Parameters** - Control seed, temperature, cfg, and other parameters per segment with `[Alice|seed:42|temperature:0.5]` syntax (see [Parameter Switching Guide](PARAMETER_SWITCHING_GUIDE.md))
|
||||
- **🔄 Robust Fallback System** - No errors when characters or languages not found
|
||||
- **📁 Voice Folder Integration** - Organized character voice management
|
||||
- **📺 SRT Support** - Character and language switching in subtitle timing
|
||||
@@ -68,7 +67,7 @@ Back to the narrator in default language.
|
||||
**Note:** If a language model is not available, the system will fallback to the default model with a warning.
|
||||
|
||||
### 3. Voice File Structure
|
||||
Organize character voices using filenames in `models/voices` (preferred) or in the custom_node TTS Audio Suite folder under `voices_examples/`:
|
||||
Organize character voices using filenames in `voices_examples/`:
|
||||
|
||||
```
|
||||
voices_examples/
|
||||
@@ -407,4 +406,3 @@ These guidelines help ensure optimal F5-TTS generation quality and prevent commo
|
||||
---
|
||||
|
||||
*For technical support or feature requests, please check the main README or create an issue on GitHub.*
|
||||
|
||||
|
||||
@@ -1,140 +0,0 @@
|
||||
# ChatterBox Official 23-Lang v2 Special Tokens
|
||||
|
||||
## 🚧 EXPERIMENTAL FEATURE - LIMITED FUNCTIONALITY
|
||||
|
||||
**IMPORTANT: These special tokens are present in the v2 vocabulary but may not work as expected!**
|
||||
|
||||
Based on community testing and the lack of official documentation from ResembleAI, these emotion/sound tokens appear to be an **incomplete implementation**:
|
||||
|
||||
- ✅ **Tokens exist** in the v2 tokenizer vocabulary
|
||||
- ⚠️ **Limited effect** - tokens may produce minimal or no audible changes
|
||||
- ❌ **No official documentation** - ResembleAI has not published usage guidelines
|
||||
- 🔬 **Experimental** - model may not be fully trained to respond to these tokens
|
||||
|
||||
**What works (partially):**
|
||||
- Some users report `<laughter> hahaha` produces slight effects
|
||||
- Results are inconsistent and unreliable
|
||||
|
||||
**Community Discussion:**
|
||||
- See [ResembleAI/chatterbox Issue #186](https://github.com/resemble-ai/chatterbox/issues/186) - "Use of Emotional Tags Like [laughter] During Generation"
|
||||
- No official response from ResembleAI team on proper usage
|
||||
|
||||
**Our implementation is ready for when/if ResembleAI improves this feature. Feel free to experiment, but don't expect production-ready results.**
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
ChatterBox v2 vocabulary includes special tokens for emotions, sounds, and vocal effects. While the tokenizer supports these tags, the model's response to them is limited and undocumented.
|
||||
|
||||
## ⚠️ IMPORTANT: Use Angle Brackets `<tag>`
|
||||
|
||||
To avoid conflicts with the character switching system `[CharacterName]`, **use angle brackets `<>` for v2 special tokens**:
|
||||
|
||||
- ✅ **Correct**: `<giggle>`, `<sigh>`, `<whisper>`
|
||||
- ❌ **Wrong**: `[giggle]`, `[sigh]`, `[whisper]` (conflicts with character names)
|
||||
|
||||
The system will automatically convert `<emotion>` → `[emotion]` internally for ChatterBox v2.
|
||||
|
||||
## New Special Tokens
|
||||
|
||||
### Emotional Expressions
|
||||
- `<giggle>` - Light laughter
|
||||
- `<laughter>` - Full laughter
|
||||
- `<guffaw>` - Loud, boisterous laugh
|
||||
- `<sigh>` - Sighing sound
|
||||
- `<cry>` - Crying sound
|
||||
- `<gasp>` - Gasping sound
|
||||
- `<groan>` - Groaning sound
|
||||
|
||||
### Breathing & Speech Modifiers
|
||||
- `<inhale>` - Inhaling/breath in
|
||||
- `<exhale>` - Exhaling/breath out
|
||||
- `<whisper>` - Whispered speech
|
||||
- `<mumble>` - Mumbled speech
|
||||
- `<UH>` - Hesitation sound (uh)
|
||||
- `<UM>` - Hesitation sound (um)
|
||||
|
||||
### Vocal Performances
|
||||
- `<singing>` - Singing voice
|
||||
- `<music>` - Musical sounds
|
||||
- `<humming>` - Humming sound
|
||||
- `<whistle>` - Whistling sound
|
||||
|
||||
### Body Sounds
|
||||
- `<cough>` - Coughing sound
|
||||
- `<sneeze>` - Sneezing sound
|
||||
- `<sniff>` - Sniffing sound
|
||||
- `<snore>` - Snoring sound
|
||||
- `<clear_throat>` - Throat clearing
|
||||
- `<chew>` - Chewing sound
|
||||
- `<sip>` - Sipping/drinking sound
|
||||
- `<kiss>` - Kissing sound
|
||||
|
||||
### Animal Sounds
|
||||
- `<bark>` - Dog barking
|
||||
- `<howl>` - Howling sound
|
||||
- `<meow>` - Cat meowing
|
||||
|
||||
### Other
|
||||
- `<shhh>` - Shushing sound
|
||||
- `<gibberish>` - Nonsensical speech
|
||||
|
||||
## Usage Examples
|
||||
|
||||
```
|
||||
Hello there! <giggle> I'm so happy to see you.
|
||||
Wait... <UM> I think I forgot something. <sigh>
|
||||
<whisper> This is a secret message.
|
||||
Look at that! <gasp> It's amazing!
|
||||
<singing> La la la la la!
|
||||
```
|
||||
|
||||
## Important Notes
|
||||
|
||||
### ✅ No Conflicts - Safe to Use!
|
||||
|
||||
The angle bracket syntax `<emotion>` is specifically designed to avoid conflicts:
|
||||
|
||||
- **Character switching** uses `[CharacterName]` - no conflict
|
||||
- **Pause tags** use `[pause:2]`, `[wait:1.5]` - no conflict
|
||||
- **v2 special tokens** use `<giggle>`, `<sigh>` - no conflict
|
||||
|
||||
### Combining with Other Systems
|
||||
|
||||
You can freely mix all three systems:
|
||||
|
||||
```
|
||||
[Alice] Hello! <giggle> Nice to meet you. [pause:0.5] How are you? <whisper> I have a secret.
|
||||
[Bob] <gasp> Really? Tell me more! [wait:1]
|
||||
```
|
||||
|
||||
**Processing order:**
|
||||
1. Character tags `[CharacterName]` are extracted first
|
||||
2. Pause tags `[pause:XX]` are processed second
|
||||
3. v2 special tags `<emotion>` are converted last (to `[emotion]` for the engine)
|
||||
|
||||
This ensures everything works together seamlessly!
|
||||
|
||||
## Model Version Selection
|
||||
|
||||
To use these special tokens:
|
||||
|
||||
1. Set **model_version** to **v2** in the ChatterBox Official 23-Lang Engine node
|
||||
2. The v2 model will automatically download the enhanced tokenizer files
|
||||
3. Special tokens will be processed during generation
|
||||
|
||||
## Technical Details
|
||||
|
||||
### New v2 Model Files
|
||||
- `t3_mtl23ls_v2.safetensors` - Enhanced T3 model with improved tokenization
|
||||
- `grapheme_mtl_merged_expanded_v1.json` - Enhanced grapheme/phoneme mappings with 118 special tokens
|
||||
- Improved Russian stress handling
|
||||
- Enhanced multilingual tokenizer fixes
|
||||
|
||||
### Cache Invalidation
|
||||
|
||||
The TTS Audio Suite automatically manages cache keys to prevent conflicts:
|
||||
- v1 and v2 generations are cached separately
|
||||
- Switching between versions will regenerate audio with the correct model
|
||||
- Cache keys include `model_version` to ensure proper invalidation
|
||||
@@ -1,227 +0,0 @@
|
||||
# CosyVoice3 Paralinguistic Tags Guide
|
||||
|
||||
## Overview
|
||||
|
||||
CosyVoice3 has **native paralinguistic tag support** built into the model. Unlike Step Audio EditX post-processing, these tags are processed **during TTS generation** for natural integration.
|
||||
|
||||
**⚠️ IMPORTANT**: Step Audio EditX post-processing is **DISABLED** for CosyVoice3. EditX syntax like `<Laughter:2>`, `<emotion:happy>`, `<style:whisper>` will be stripped but **not processed**. Use CosyVoice3 native tags instead.
|
||||
|
||||
The multiline TTS tag editor has an engine-aware `Inline Tags` panel with a dedicated `CosyVoice3` mode that inserts these native tags directly.
|
||||
|
||||
---
|
||||
|
||||
## Supported Tags
|
||||
|
||||
### Single Tags (Sound Insertion)
|
||||
|
||||
| Tag | Effect | Example |
|
||||
|-----|--------|---------|
|
||||
| `<breath>` | Breathing sound | `I'm tired <breath> let's rest` |
|
||||
| `<quick_breath>` | Quick breath | `Running <quick_breath> almost there` |
|
||||
| `<laughter>` | Laughter | `That's hilarious <laughter>!` |
|
||||
| `<cough>` | Cough | `Excuse me <cough> sorry` |
|
||||
| `<sigh>` | Sigh | `Fine <sigh> I'll do it` |
|
||||
| `<gasp>` | Gasp | `Oh no <gasp> what happened?` |
|
||||
| `<noise>` | Background noise | `Walking <noise> through forest` |
|
||||
| `<hissing>` | Hissing sound | `The snake <hissing> away` |
|
||||
| `<vocalized-noise>` | Vocalized noise | `Hmm <vocalized-noise> interesting` |
|
||||
| `<lipsmack>` | Lip smack | `Delicious <lipsmack> food` |
|
||||
| `<mn>` | "mn" sound | `I think <mn> maybe` |
|
||||
| `<clucking>` | Clucking sound | `Disapproving <clucking>` |
|
||||
| `<accent>` | Accent emphasis | `Very <accent> important` |
|
||||
|
||||
**Position matters**: Tags insert sounds where you place them.
|
||||
|
||||
### Wrapper Tags (Text Emphasis)
|
||||
|
||||
| Tag | Effect | Example |
|
||||
|-----|--------|---------|
|
||||
| `<laughing>text</laughing>` | Apply laughter to text | `<laughing>so funny</laughing>!` |
|
||||
| `<strong>text</strong>` | Emphasize text | `<strong>very important</strong>` |
|
||||
|
||||
**Note**: Use `<laughing>` for wrapper tags (auto-converted to `<laughter>` internally).
|
||||
|
||||
### Language Switching
|
||||
|
||||
CosyVoice3 supports 4 native languages (EN, ZH, JA, KO) and 5 cross-lingual languages (DE, ES, FR, IT, RU).
|
||||
|
||||
**TTS Audio Suite bracket syntax** (auto-converted):
|
||||
```
|
||||
[en:Alice] Hello world
|
||||
[zh:Bob] 你好世界
|
||||
[ja:Alice] こんにちは
|
||||
[ko:Bob] 안녕하세요
|
||||
|
||||
Or without character name:
|
||||
[en:] Hello world
|
||||
[zh:] 你好世界
|
||||
```
|
||||
|
||||
**CosyVoice3 native syntax** (manual):
|
||||
```
|
||||
<|en|>Hello world
|
||||
<|zh|>你好世界
|
||||
<|ja|>こんにちは
|
||||
<|ko|>안녕하세요
|
||||
```
|
||||
|
||||
Both work. Bracket syntax is recommended for consistency with other engines.
|
||||
|
||||
---
|
||||
|
||||
## Tag Syntax
|
||||
|
||||
### User-Friendly Format (Recommended)
|
||||
|
||||
**You write**: `<breath>`, `<laughter>`, `<cough>`
|
||||
|
||||
**Auto-converted to**: `[breath]`, `[laughter]`, `[cough]`
|
||||
|
||||
This prevents conflicts with character switching `[CharacterName]` brackets.
|
||||
|
||||
### Direct Format (Advanced)
|
||||
|
||||
You can also use native CosyVoice3 format: `[breath]`, `[laughter]`, `[cough]`
|
||||
|
||||
Both formats work identically.
|
||||
|
||||
---
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Basic
|
||||
|
||||
```
|
||||
[Alice] Hello there <breath> nice to meet you!
|
||||
```
|
||||
|
||||
### Multiple Effects
|
||||
|
||||
```
|
||||
[Bob] I can't believe it <gasp> that's <laughter> amazing!
|
||||
```
|
||||
|
||||
### Wrapper Tags
|
||||
|
||||
```
|
||||
[Alice] This is <laughing>hilarious</laughing>!
|
||||
[Narrator] The news was <strong>absolutely shocking</strong>.
|
||||
```
|
||||
|
||||
### Natural Conversation
|
||||
|
||||
```
|
||||
[Alice] Did you hear the news? <breath>
|
||||
[Bob] Yes <gasp> I couldn't believe it!
|
||||
[Alice] I know right <laughter> it's crazy!
|
||||
```
|
||||
|
||||
### SRT Subtitles
|
||||
|
||||
```
|
||||
1
|
||||
00:00:00,000 --> 00:00:03,000
|
||||
[Alice] Welcome to the show <breath> everyone!
|
||||
|
||||
2
|
||||
00:00:03,500 --> 00:00:06,000
|
||||
[Bob] Thanks for having me <laughter>
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Best Practices
|
||||
|
||||
### 1. Position Tags Naturally
|
||||
|
||||
```
|
||||
✅ GOOD: "I'm so tired <sigh> let's take a break"
|
||||
❌ AWKWARD: "<sigh> <breath> I'm tired"
|
||||
```
|
||||
|
||||
### 2. Don't Overuse
|
||||
|
||||
```
|
||||
✅ GOOD: "Hello <breath> nice to meet you"
|
||||
❌ BAD: "Hello <breath> there <laughter> nice <sigh> to you"
|
||||
```
|
||||
|
||||
**Recommended**: 1-2 tags per sentence, 3-4 maximum per segment.
|
||||
|
||||
### 3. Match Context
|
||||
|
||||
```
|
||||
# Tired
|
||||
[Alice] I can't go on <sigh> I need rest <breath>
|
||||
|
||||
# Happy
|
||||
[Bob] That was hilarious <laughter> tell me more!
|
||||
|
||||
# Surprised
|
||||
[Alice] Wait what? <gasp> Are you serious?
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## CosyVoice3 Tags vs Step Audio EditX
|
||||
|
||||
| Feature | CosyVoice3 Native | Step Audio EditX |
|
||||
|---------|------------------|-----------------|
|
||||
| **Syntax** | `<breath>`, `<laughter>` | `<Laughter:2>`, `<emotion:happy>` |
|
||||
| **Processing** | During TTS | Post-processing (disabled for CosyVoice3) |
|
||||
| **Effects** | 13 paralinguistic sounds | 13 paralinguistic + 14 emotions + 32 styles + speed + restore |
|
||||
| **Iterations** | Single pass | 1-30 customizable |
|
||||
| **Engine Support** | CosyVoice3 only | All other engines |
|
||||
|
||||
**When to use CosyVoice3 tags**: Generating with CosyVoice3 engine, need paralinguistic effects
|
||||
|
||||
**When to use EditX tags**: Need emotion/style/speed control, OR using other engines (ChatterBox, F5-TTS, etc.)
|
||||
|
||||
---
|
||||
|
||||
## Limitations
|
||||
|
||||
### ⚠️ No Step Audio EditX Support
|
||||
|
||||
```
|
||||
❌ WON'T WORK: <Laughter:2> (EditX - will be stripped)
|
||||
❌ WON'T WORK: <emotion:happy> (EditX - will be stripped)
|
||||
❌ WON'T WORK: <style:whisper> (EditX - will be stripped)
|
||||
|
||||
✅ USE INSTEAD: <laughter> (CosyVoice native)
|
||||
✅ USE INSTEAD: <strong>text</strong>
|
||||
```
|
||||
|
||||
For emotion/style control, use ChatterBox, IndexTTS-2, or other engines with EditX support.
|
||||
|
||||
### Tag Case
|
||||
|
||||
Tags are **case-insensitive**: `<breath>` = `<BREATH>` = `<Breath>`
|
||||
|
||||
---
|
||||
|
||||
## FAQ
|
||||
|
||||
**Q: Can I use Step Audio EditX tags with CosyVoice3?**
|
||||
A: No. EditX syntax (`<Laughter:2>`, `<emotion:happy>`) will be stripped but not processed.
|
||||
|
||||
**Q: Can I control effect strength like EditX iterations?**
|
||||
A: No. CosyVoice3 tags are single-pass. Effect strength is determined by the model.
|
||||
|
||||
**Q: Which languages support tags best?**
|
||||
A: All supported languages (EN, ZH, JA, KO, DE, ES, FR, IT, RU) work with tags.
|
||||
|
||||
**Q: Can I use tags in SRT mode?**
|
||||
A: Yes! Include tags in subtitle text.
|
||||
|
||||
---
|
||||
|
||||
## Related Guides
|
||||
|
||||
- **[Step Audio EditX Inline Tags Guide](INLINE_EDIT_TAGS_USER_GUIDE.md)** - Step Audio EditX tags for other engines
|
||||
- **[Parameter Switching Guide](PARAMETER_SWITCHING_GUIDE.md)** - Per-segment parameter control
|
||||
|
||||
---
|
||||
|
||||
**Last Updated**: 2025-12-29
|
||||
**Compatible with**: TTS Audio Suite v4.16.0+
|
||||
@@ -1,99 +0,0 @@
|
||||
# DramaBox LoRA training
|
||||
|
||||
TTS Audio Suite exposes the official DramaBox audio-branch IC-LoRA trainer
|
||||
through the unified `🎓 Model Training` flow. The bundled scripts are pinned to
|
||||
the same upstream DramaBox revision as the inference implementation.
|
||||
|
||||
See the official DramaBox
|
||||
[LoRA training guide](https://github.com/resemble-ai/DramaBox#training-a-lora-on-top-of-dramabox)
|
||||
for the upstream dataset format and training behavior.
|
||||
|
||||
## Workflow
|
||||
|
||||
1. Build a `⚙️ DramaBox Engine`.
|
||||
2. Create the dataset either externally or entirely inside ComfyUI:
|
||||
`🎞️ Training Clip Staging` → `🧾 DramaBox Dataset Rows`.
|
||||
3. Connect the resulting manifest to `📦 DramaBox Dataset Prep` and keep
|
||||
`dataset_type` set to `manifest`.
|
||||
4. Provide at least two clips per speaker.
|
||||
5. Connect the dataset to `🎛️ DramaBox Training Config` and then to `🎓 Model Training`.
|
||||
6. Select the resulting adapter in the DramaBox engine, or enter its path in the
|
||||
advanced LoRA override field.
|
||||
|
||||
The dataset node accepts:
|
||||
|
||||
- JSONL/JSON manifests with `audio_filepath` (or `audio_path`) and `text` (or
|
||||
`transcript`)
|
||||
- TSV rows with audio path and text
|
||||
- the official `gemini_synthetic` and `libriheavy` index formats
|
||||
|
||||
Manifest rows may include `speaker`, `speaker_id`, `language`, and `duration`.
|
||||
If `speaker` is omitted, rows are grouped as `speaker_1`. Duration and audio
|
||||
metadata are measured without loading the waveform into the GPU. The suite
|
||||
converts all accepted formats into the `~`-delimited speaker index required by
|
||||
the upstream training loop. Clips are restricted to 2–20 seconds by default.
|
||||
|
||||
For an all-ComfyUI dataset, connect one or more `AUDIO` sources to
|
||||
`🎞️ Training Clip Staging`, then enter one transcript per clip in
|
||||
`🧾 DramaBox Dataset Rows`. Speaker and language lines are optional; shared
|
||||
defaults are used when those lines are blank.
|
||||
|
||||
### Transcripts and scene descriptions
|
||||
|
||||
The official trainer accepts either plain spoken transcripts or the same
|
||||
scene-style prompt format used for inference. For example, both of these are
|
||||
valid training text:
|
||||
|
||||
```text
|
||||
This is the spoken sentence.
|
||||
A woman speaks warmly, "This is the spoken sentence."
|
||||
```
|
||||
|
||||
Use scene descriptions only when they accurately describe the clip. Plain
|
||||
transcripts remain valid and are the safer choice when no reliable style or
|
||||
scene annotation is available.
|
||||
|
||||
## What training does
|
||||
|
||||
The first preprocessing pass uses Gemma and the DramaBox audio VAE to create
|
||||
cached conditions and audio latents. The training process then attaches a LoRA
|
||||
to the audio transformer branch. It saves periodic checkpoints and exports the
|
||||
selected adapter to:
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/dramabox/loras/<adapter_name>/
|
||||
```
|
||||
|
||||
The job directory, normalized index, preprocessing cache, progress file, and
|
||||
logs are stored under:
|
||||
|
||||
```text
|
||||
ComfyUI/output/tts_audio_suite_training/dramabox/
|
||||
```
|
||||
|
||||
`continue_from` is a warm start from an existing LoRA checkpoint; it is not an
|
||||
exact optimizer-state resume. Use saved checkpoints to compare quality rather
|
||||
than assuming the last step is best. Optional upstream validation can be
|
||||
enabled with a `val_config` YAML path, but it launches full DramaBox inference
|
||||
at each save step. It requires a second GPU: set `validation_gpu` to that
|
||||
physical CUDA device index. The suite rejects validation on the training GPU
|
||||
instead of allowing both full model processes to compete for the same VRAM.
|
||||
|
||||
DramaBox LoRA inference supports normal transformer precision, `fp8_cast`, and
|
||||
the optional `torch.compile` path. With normal precision the live adapter is
|
||||
reversibly merged for fast inference. With FP8 storage the BF16 adapter remains
|
||||
unmerged above the immutable FP8 base weights, avoiding unsafe mixed-dtype
|
||||
weight fusion while retaining the main FP8 memory saving.
|
||||
|
||||
The base DramaBox runtime is reused when the selected adapter or LoRA strength
|
||||
changes. Strength updates are applied directly to the live PEFT adapter, while
|
||||
the generated-audio cache still treats adapter path, file revision, and strength
|
||||
as distinct generation settings. Replacing an adapter with a different rank may
|
||||
retrace compiled transformer blocks, but does not reload the base checkpoint.
|
||||
|
||||
## CPU-safe preflight
|
||||
|
||||
Training and Gemma/VAE preprocessing are GPU workloads. For development or
|
||||
validation without touching CUDA, enable `dry_run` in the training config and
|
||||
the dataset node's `dry_run`/`preprocess_now` controls. This writes the
|
||||
normalized index and official command/config without loading DramaBox weights.
|
||||
@@ -1,163 +0,0 @@
|
||||
# DramaBox Prompting Guide
|
||||
|
||||
DramaBox is an English expressive TTS engine. It accepts ordinary narration,
|
||||
dialogue in quotation marks, and natural-language stage directions in one
|
||||
prompt.
|
||||
|
||||
## Basic prompts
|
||||
|
||||
Use quoted text for speech and surrounding prose for delivery:
|
||||
|
||||
```text
|
||||
A tired detective speaks quietly in a rain-soaked office. "I knew this case would find me again."
|
||||
```
|
||||
|
||||
`prompt_template` defaults to `"{seg}"`. `{seg}` is replaced by the current
|
||||
plain fragment, so the default marks the whole fragment as literal spoken
|
||||
dialogue. For example, `Hello.` becomes `"Hello."`. Clear the template field to
|
||||
send plain text unchanged.
|
||||
|
||||
Customize the template to add delivery context, for example
|
||||
`A man speaks warmly, "{seg}"`. Quote-only input is normalized without adding
|
||||
another pair of quotes. Complete scene prompts with directions outside their
|
||||
quotation marks remain unchanged. Every non-empty template must contain `{seg}`.
|
||||
If it is omitted accidentally, TTS Audio Suite warns once and appends
|
||||
`"{seg}"` automatically instead of failing generation.
|
||||
|
||||
For a one-segment override, `prompt_template` (or its `template` alias)
|
||||
automatically enables templating for that segment and then reverts to the node
|
||||
setting:
|
||||
|
||||
```text
|
||||
[Narrator|template:A woman whispers, "{seg}"] This line uses a custom wrapper.
|
||||
```
|
||||
|
||||
DramaBox can render non-verbal and delivery cues when they are described
|
||||
naturally:
|
||||
|
||||
```text
|
||||
She tries to stay serious, then breaks into a short laugh. "That is the worst excuse I have ever heard." She sighs and continues more gently. "But I believe you."
|
||||
```
|
||||
|
||||
Write one continuous scene paragraph. DramaBox does not require newlines as
|
||||
prompt syntax. In TTS Audio Suite, an untagged newline starts another generated
|
||||
segment, so use prose action directions and quoted dialogue in the same
|
||||
paragraph when they should remain one coherent DramaBox scene.
|
||||
|
||||
Do not use ChatterBox V2 special tokens such as `[giggle]`. DramaBox was
|
||||
trained for prose-style scene direction, not that token vocabulary.
|
||||
|
||||
## Voice references
|
||||
|
||||
Reference audio is optional. Connect narrator audio or use a character voice
|
||||
file to clone its speaker and delivery. Upstream uses the first 10 seconds, so
|
||||
a clean single-speaker clip is the useful input; a transcript is not required.
|
||||
|
||||
Without a reference, DramaBox uses its built-in voice behavior.
|
||||
|
||||
DramaBox can occasionally produce a near-silent sample for a particular
|
||||
combination of reference audio, reference duration, generation duration, and
|
||||
seed. TTS Audio Suite checks the decoded waveform and prints a warning when both
|
||||
its RMS and peak levels are conservatively near silence. The audio is preserved;
|
||||
the suite does not retry or change parameters automatically. Try another
|
||||
generation duration, reference duration/audio, guidance setting, or seed for
|
||||
the affected segment. A different seed can help some combinations but is not a
|
||||
guaranteed fix.
|
||||
|
||||
The warning is also propagated to node outputs. TTS Text includes affected
|
||||
segments in `generation_info`. TTS SRT marks affected subtitle numbers in
|
||||
`timing_report`, including the parameters that may be worth testing for that
|
||||
segment.
|
||||
|
||||
## Character and pause tags
|
||||
|
||||
TTS Audio Suite character tags still work. Each tagged character is generated
|
||||
as a separate DramaBox segment:
|
||||
|
||||
```text
|
||||
[Alice] "We should leave now."
|
||||
[Bob] He answers without looking up. "Give me one minute."
|
||||
[pause:0.8]
|
||||
[Alice] "You said that five minutes ago."
|
||||
```
|
||||
|
||||
Suite pause tags create exact silence outside the model. Natural pauses inside
|
||||
a spoken scene are better expressed in the prose prompt.
|
||||
|
||||
## Engine controls
|
||||
|
||||
- `cfg_scale`: text/prompt guidance. Official default: `2.5`.
|
||||
- `stg_scale`: skip-token guidance. Official default: `1.5`.
|
||||
- `duration_multiplier`: scales the estimated speaking duration. Official
|
||||
default: `1.1`.
|
||||
- `gen_duration`: explicit generated-audio duration from `0` to `60` seconds.
|
||||
`0` keeps automatic prompt-based estimation.
|
||||
- `ref_duration`: uses the first `3` to `30` seconds of a voice reference.
|
||||
The default is `10`; audio later in the source file is ignored.
|
||||
- `rescale_scale`: CFG latent rescaling. Use `auto` or a fixed value from
|
||||
`0` to `1`.
|
||||
- `watermark`: enables the optional official Perth output watermark. It is off
|
||||
by default and requires Perth.
|
||||
- `seed`: supplied by the unified TTS Text or SRT node.
|
||||
|
||||
Segment overrides support `seed`, `cfg_scale`, `stg_scale`, and
|
||||
`duration_multiplier`, `gen_duration`, `ref_duration`, and `rescale_scale`.
|
||||
Watermarking remains a whole-engine setting rather than a segment override.
|
||||
|
||||
DramaBox performs its own duration-aware long-form chunking. The suite does
|
||||
not split a DramaBox scene by character count before passing it to the model.
|
||||
Automatically estimated scenes above 45 seconds use text chunking. A nonzero
|
||||
`gen_duration` remains one native generation so its explicit 0–60 second
|
||||
target is preserved.
|
||||
|
||||
The unified SRT node's **Native Duration Targeting** option passes each
|
||||
subtitle's duration to DramaBox before final timing assembly. For subtitles
|
||||
containing multiple character or pause-separated fragments, the available
|
||||
speech time is allocated proportionally after explicit pause durations and
|
||||
inline `gen_duration` overrides are accounted for. The selected SRT timing
|
||||
mode still performs its normal final correction.
|
||||
|
||||
## Negative Prompt and Segment Switching
|
||||
|
||||
DramaBox uses CFG and exposes its negative prompt in the engine node. The
|
||||
default discourages robotic, distorted, noisy, muffled, unclear, and monotone
|
||||
speech. Override it for one character segment with:
|
||||
|
||||
```text
|
||||
[Alice|negative:robotic, muffled] "Keep this line clean and intimate."
|
||||
[Bob|neg:noise, static] "This line uses a different negative prompt."
|
||||
```
|
||||
|
||||
The segment override ends at the next character tag.
|
||||
|
||||
## Memory and Performance
|
||||
|
||||
- `fast` keeps all components on CUDA for the fastest repeated generation.
|
||||
- `staged` is an experimental strategy for lowering peak VRAM. It loads and
|
||||
releases Gemma, the voice encoder, and audio decoder by stage, at the cost of
|
||||
reloading them for each generated segment or long-form chunk.
|
||||
- `sequential` is a more aggressive experimental strategy for lowering peak
|
||||
VRAM. It additionally keeps the diffusion transformer in system RAM while
|
||||
another major stage uses CUDA. It transfers the transformer for every
|
||||
generated segment or long-form chunk and is therefore substantially slower.
|
||||
Actual peak usage varies with the environment, generation settings, and
|
||||
other loaded components; no minimum GPU size is guaranteed.
|
||||
System RAM must hold the offloaded transformer (about 3.4GB with FP8 or
|
||||
6.6GB without it).
|
||||
- `fp8_cast` uses the official LTX FP8 transformer weight-storage policy and
|
||||
upcasts linear weights during inference. It can lower VRAM and may be slower.
|
||||
- `compile_model` compiles the diffusion transformer blocks with DramaBox's
|
||||
bundled LTX compilation path. The first generation can take substantially
|
||||
longer while kernels compile; later denoising may be faster.
|
||||
|
||||
## Requirements and license
|
||||
|
||||
The full download is approximately 16.4GB and the official runtime requires
|
||||
an NVIDIA CUDA GPU. Fast mode targets roughly 24GB VRAM; the experimental
|
||||
staged modes can run with less memory at a speed cost. Output is 48kHz stereo.
|
||||
The optional official Perth watermark is applied only when enabled and the
|
||||
dependency is available.
|
||||
|
||||
DramaBox uses the LTX-2 Community License. Entities with at least USD 10
|
||||
million in annual revenue require a separate paid commercial license. Review
|
||||
the bundled license before production use.
|
||||
@@ -1,702 +0,0 @@
|
||||
# TTS Audio Suite Generation/Audio Caching System - Comprehensive Analysis
|
||||
|
||||
## Executive Summary
|
||||
|
||||
The TTS Audio Suite has **MULTIPLE FRAGMENTED CACHING SYSTEMS** operating independently with inconsistent cache key construction. The generation cache is hard to fix and often misses because:
|
||||
|
||||
1. **Multiple independent cache implementations** (at least 3 separate systems)
|
||||
2. **Inconsistent cache key generation** across engines
|
||||
3. **Missing parameter normalization** causing floating-point precision issues
|
||||
4. **Undocumented parameter handling** with hidden transformations
|
||||
5. **Multiple cache levels** (local, unified, instance-level) not coordinated
|
||||
6. **Text transformation side-effects** affecting cache keys
|
||||
|
||||
---
|
||||
|
||||
## 1. CURRENT CACHING SYSTEMS
|
||||
|
||||
### System 1: Unified Audio Cache (Primary but underused)
|
||||
**Location:** `/home/linux/ComfyUI_TTS_Audio_Suite/utils/audio/cache.py`
|
||||
|
||||
- **Global Storage:** `GLOBAL_AUDIO_CACHE = {}` (line 13)
|
||||
- **Class:** `AudioCache` (line 235)
|
||||
- **Access:** `get_audio_cache()` function (line 343)
|
||||
- **Key Generator:** Abstract `CacheKeyGenerator` with engine-specific implementations
|
||||
- `F5TTSCacheKeyGenerator` (line 25)
|
||||
- `ChatterBoxCacheKeyGenerator` (line 50)
|
||||
- `ChatterBoxOfficial23LangCacheKeyGenerator` (line 76)
|
||||
- `HiggsAudioCacheKeyGenerator` (line 103)
|
||||
- `VibeVoiceCacheKeyGenerator` (line 144)
|
||||
- `IndexTTSCacheKeyGenerator` (line 184)
|
||||
|
||||
**Issue:** This unified system exists but is ONLY used in ChatterBox TTS node through `create_cache_function()`. Most nodes use local caches instead.
|
||||
|
||||
### System 2: Node-Local Audio Cache (ChatterBox)
|
||||
**Location:** `/home/linux/ComfyUI_TTS_Audio_Suite/nodes/chatterbox/chatterbox_srt_node.py`
|
||||
|
||||
- **Global Storage:** `GLOBAL_AUDIO_CACHE = {}` (line 46)
|
||||
- **Separate from unified cache** - different dict, different module
|
||||
- **Methods:**
|
||||
- `_generate_segment_cache_key()` (line 361)
|
||||
- `_get_cached_segment_audio()` (line 381)
|
||||
- `_cache_segment_audio()` (line 385)
|
||||
|
||||
**Issue:** This is a DUPLICATE system that doesn't use the unified cache. Creates cache misses when switching between unified node and legacy node.
|
||||
|
||||
### System 3: Voice Conversion Iteration Cache
|
||||
**Location:** `/home/linux/ComfyUI_TTS_Audio_Suite/nodes/unified/voice_changer_node.py`
|
||||
|
||||
- **Global Storage:** `GLOBAL_RVC_ITERATION_CACHE = {}` (line 48)
|
||||
- **Purpose:** Cache iterative refinement passes for RVC voice conversion
|
||||
- **Methods:**
|
||||
- `_generate_rvc_cache_key()` (line 581)
|
||||
- `_get_cached_rvc_iterations()` (line 620)
|
||||
- `_cache_rvc_result()` (line 628)
|
||||
|
||||
**Issue:** RVC-specific, limited to 5 iterations max (line 634), not integrated with audio cache.
|
||||
|
||||
### System 4: Engine Instance Cache (Multiple locations)
|
||||
**Location:** `tts_text_node.py` (line 134), `voice_changer_node.py` (line 90)
|
||||
|
||||
- **Storage:** Instance variable `self._cached_engine_instances = {}`
|
||||
- **Purpose:** Cache engine node instances to prevent model reloading
|
||||
- **Cache Key:** Hash of `engine_type` + stable parameters (line 182)
|
||||
- **Timestamp-based invalidation:** Uses global `_global_cache_invalidation_flag`
|
||||
|
||||
**Issue:** This is ENGINE caching, not GENERATION caching. Confuses the user about what's being cached.
|
||||
|
||||
### System 5: Voice Discovery Cache (Persistent)
|
||||
**Location:** `/home/linux/ComfyUI_TTS_Audio_Suite/utils/voice/cache_manager.py`
|
||||
|
||||
- **Storage:** Disk-based JSON in `.cache/voice_discovery.json`
|
||||
- **Purpose:** Cache voice character discovery to speed up initialization
|
||||
- **Background refresh:** Uses threading to update cache after ComfyUI loads
|
||||
|
||||
**Issue:** This is voice DISCOVERY cache, not generation audio cache.
|
||||
|
||||
---
|
||||
|
||||
## 2. CACHE KEY CONSTRUCTION - DETAILED ANALYSIS
|
||||
|
||||
### ChatterBox Unified Cache Key
|
||||
**File:** `/home/linux/ComfyUI_TTS_Audio_Suite/utils/audio/cache.py:50-73`
|
||||
|
||||
```python
|
||||
cache_data = {
|
||||
'text': params.get('text', ''), # FULL TEXT WITH CHARACTER PREFIX
|
||||
'exaggeration': params.get('exaggeration', 1.0),
|
||||
'temperature': params.get('temperature', 0.8),
|
||||
'cfg_weight': params.get('cfg_weight', 1.0),
|
||||
'seed': params.get('seed', 0),
|
||||
'audio_component': params.get('audio_component', ''), # HASHED AUDIO CONTENT
|
||||
'model_source': params.get('model_source', ''), # LANGUAGE + SOURCE
|
||||
'device': params.get('device', ''), # NOT RESOLVED (can vary)
|
||||
'language': params.get('language', 'English'),
|
||||
'character': params.get('character', 'narrator'),
|
||||
'engine': 'chatterbox',
|
||||
'repetition_penalty': params.get('repetition_penalty', 1.0),
|
||||
'min_p': params.get('min_p', 0.0),
|
||||
'top_p': params.get('top_p', 1.0)
|
||||
}
|
||||
cache_string = str(sorted(cache_data.items()))
|
||||
cache_key = hashlib.md5(cache_string.encode()).hexdigest()
|
||||
```
|
||||
|
||||
**Problems:**
|
||||
1. **No floating-point rounding** - 0.8 vs 0.80000001 = different keys
|
||||
2. **`audio_component` is hashed waveform** - but text still includes character prefix (duplication)
|
||||
3. **Device resolution inconsistency** - stores "auto" literally, causing misses when it resolves to "cuda"
|
||||
4. **Character prefixed to text** - line 282 of cache.py: `cache_params['text'] = f"{cache_params.get('character', 'narrator')}:{text_content}"`
|
||||
|
||||
### ChatterBox Legacy Cache Key
|
||||
**File:** `/home/linux/ComfyUI_TTS_Audio_Suite/nodes/chatterbox/chatterbox_srt_node.py:361-379`
|
||||
|
||||
```python
|
||||
cache_data = {
|
||||
'text': subtitle_text,
|
||||
'exaggeration': exaggeration,
|
||||
'temperature': temperature,
|
||||
'cfg_weight': cfg_weight,
|
||||
'seed': seed,
|
||||
'audio_prompt_component': audio_prompt_component,
|
||||
'model_source': model_source,
|
||||
'device': device,
|
||||
'language': language,
|
||||
'engine': 'chatterbox_srt'
|
||||
}
|
||||
cache_string = str(sorted(cache_data.items()))
|
||||
cache_key = hashlib.md5(cache_string.encode()).hexdigest()
|
||||
```
|
||||
|
||||
**Problems:**
|
||||
1. Uses `chatterbox_srt` as engine (different from `chatterbox`)
|
||||
2. No character prefix in text (different from unified)
|
||||
3. Stores different parameters (no repetition_penalty, min_p, top_p)
|
||||
4. **Result: Cache misses when using SRT node vs TTS node**
|
||||
|
||||
### F5-TTS Unified Cache Key
|
||||
**File:** `/home/linux/ComfyUI_TTS_Audio_Suite/utils/audio/cache.py:25-47`
|
||||
|
||||
```python
|
||||
cache_data = {
|
||||
'text': params.get('text', ''),
|
||||
'model_name': params.get('model_name', ''), # INCLUDES MODEL NAME
|
||||
'device': params.get('device', ''),
|
||||
'audio_component': params.get('audio_component', ''),
|
||||
'ref_text': params.get('ref_text', ''), # REFERENCE TEXT (!)
|
||||
'temperature': params.get('temperature', 0.8),
|
||||
'speed': params.get('speed', 1.0),
|
||||
'target_rms': params.get('target_rms', 0.1),
|
||||
'cross_fade_duration': params.get('cross_fade_duration', 0.15),
|
||||
'nfe_step': params.get('nfe_step', 32),
|
||||
'cfg_strength': params.get('cfg_strength', 2.0),
|
||||
'seed': params.get('seed', 0),
|
||||
'character': params.get('character', 'narrator'),
|
||||
'engine': 'f5tts'
|
||||
}
|
||||
```
|
||||
|
||||
**Critical Issues:**
|
||||
1. **Includes `ref_text` in cache key** - Same text with different reference audio = different key (CORRECT)
|
||||
2. **No floating-point rounding** on ANY of the 9 float parameters
|
||||
3. **Device not resolved** - "auto" won't match "cuda"
|
||||
4. **`cross_fade_duration`** is POST-PROCESSING, shouldn't be in key (doesn't affect generation)
|
||||
5. **`audio_component`** should be enough to distinguish references, shouldn't also need ref_text
|
||||
|
||||
### VibeVoice Cache Key
|
||||
**File:** `/home/linux/ComfyUI_TTS_Audio_Suite/utils/audio/cache.py:144-181`
|
||||
|
||||
```python
|
||||
# FLOATING POINT ROUNDING PRESENT (but inconsistent)
|
||||
cfg_scale = round(float(cfg_scale), 3)
|
||||
temperature = round(float(temperature), 3)
|
||||
top_p = round(float(top_p), 3)
|
||||
|
||||
cache_data = {
|
||||
'text': params.get('text', ''),
|
||||
'cfg_scale': cfg_scale,
|
||||
'temperature': temperature,
|
||||
'top_p': top_p,
|
||||
'use_sampling': params.get('use_sampling', False),
|
||||
'seed': params.get('seed', 42),
|
||||
'model_source': params.get('model_source', 'vibevoice-1.5B'),
|
||||
'device': params.get('device', 'auto'),
|
||||
'max_new_tokens': params.get('max_new_tokens'),
|
||||
'multi_speaker_mode': params.get('multi_speaker_mode', 'Custom Character Switching'),
|
||||
'audio_component': params.get('audio_component', ''),
|
||||
'character': params.get('character', 'narrator'),
|
||||
'inference_steps': params.get('inference_steps', 20),
|
||||
'attention_mode': params.get('attention_mode', 'eager'),
|
||||
'quantize': params.get('quantize', False),
|
||||
'engine': 'vibevoice'
|
||||
}
|
||||
```
|
||||
|
||||
**Good:** Has rounding for floats
|
||||
**Bad:** Only rounds 3 params, misses others. Device still "auto". Includes post-processing params like attention_mode (affects inference, not generation output).
|
||||
|
||||
### IndexTTS Cache Key
|
||||
**File:** `/home/linux/ComfyUI_TTS_Audio_Suite/utils/audio/cache.py:184-232`
|
||||
|
||||
```python
|
||||
# SELECTIVE ROUNDING (inconsistent)
|
||||
temperature = round(float(temperature), 3)
|
||||
top_p = round(float(top_p), 3)
|
||||
emotion_alpha = round(float(emotion_alpha), 3)
|
||||
|
||||
cache_data = {
|
||||
'text': params.get('text', ''),
|
||||
'speaker_audio': params.get('speaker_audio', ''), # AUDIO PATH STRING
|
||||
'emotion_audio': params.get('emotion_audio', ''), # AUDIO PATH STRING
|
||||
'emotion_alpha': emotion_alpha,
|
||||
'emotion_vector': params.get('emotion_vector'), # LIST (unhashable!)
|
||||
'use_emotion_text': params.get('use_emotion_text', False),
|
||||
'emotion_text': params.get('emotion_text', ''),
|
||||
'use_random': params.get('use_random', False),
|
||||
'seed': params.get('seed', 0),
|
||||
'do_sample': params.get('do_sample', True),
|
||||
'temperature': temperature,
|
||||
'top_p': top_p,
|
||||
'top_k': params.get('top_k', 30),
|
||||
'length_penalty': params.get('length_penalty', 0.0), # NOT ROUNDED!
|
||||
'num_beams': params.get('num_beams', 3),
|
||||
'repetition_penalty': params.get('repetition_penalty', 10.0), # NOT ROUNDED!
|
||||
'max_mel_tokens': params.get('max_mel_tokens', 1500),
|
||||
'max_text_tokens_per_segment': params.get('max_text_tokens_per_segment', 120),
|
||||
'interval_silence': params.get('interval_silence', 200),
|
||||
'model_name': params.get('model_name', 'IndexTTS-2'),
|
||||
'device': params.get('device', 'auto'),
|
||||
'character': params.get('character', 'narrator'),
|
||||
'use_torch_compile': params.get('use_torch_compile', False),
|
||||
'use_accel': params.get('use_accel', False),
|
||||
'stream_return': params.get('stream_return', False),
|
||||
'more_segment_before': params.get('more_segment_before', 0),
|
||||
'engine': 'index_tts'
|
||||
}
|
||||
```
|
||||
|
||||
**Critical Issues:**
|
||||
1. **Stores audio paths as strings** - temp paths will vary, causing cache misses
|
||||
2. **`emotion_vector` is a list** - unhashable, str() conversion is fragile
|
||||
3. **Selective float rounding** - only 3 params rounded, others like `length_penalty`, `repetition_penalty` are not
|
||||
4. **Mixed use of AUDIO and PATH** - speaker_audio and emotion_audio are path strings, not hashes
|
||||
|
||||
---
|
||||
|
||||
## 3. CACHE MISS ROOT CAUSES
|
||||
|
||||
### Root Cause #1: Duplicate Cache Systems
|
||||
Two separate `GLOBAL_AUDIO_CACHE` dicts exist:
|
||||
- `/home/linux/ComfyUI_TTS_Audio_Suite/utils/audio/cache.py:13`
|
||||
- `/home/linux/ComfyUI_TTS_Audio_Suite/nodes/chatterbox/chatterbox_srt_node.py:46`
|
||||
- `/home/linux/ComfyUI_TTS_Audio_Suite/nodes/f5tts/f5tts_srt_node.py:43`
|
||||
|
||||
**Result:** Using unified TTS Text node and then SRT node = cache miss (different dicts)
|
||||
|
||||
### Root Cause #2: Floating-Point Precision
|
||||
Example from ChatterBox:
|
||||
```python
|
||||
# Temperature=0.8 in unified cache
|
||||
temperature = params.get('temperature', 0.8) # Key generated with 0.8
|
||||
|
||||
# Temperature from ComfyUI slider
|
||||
temperature = 0.80000001 # Different key!
|
||||
```
|
||||
|
||||
**Affected engines:** ChatterBox, F5-TTS (all 9 float params), all others without rounding
|
||||
|
||||
### Root Cause #3: Device Resolution Inconsistency
|
||||
```python
|
||||
# Unified cache stores device literally
|
||||
'device': params.get('device', ''), # Stores "auto"
|
||||
|
||||
# But actual resolution varies
|
||||
tts_text_node.py line 159:
|
||||
from utils.device import resolve_torch_device
|
||||
resolved_device = resolve_torch_device(config.get('device', 'auto')) # Returns "cuda"
|
||||
```
|
||||
|
||||
**Result:** Engine cache key includes resolved device, generation cache key includes "auto". Mismatch!
|
||||
|
||||
### Root Cause #4: Text Transformation Side-Effects
|
||||
**In `/home/linux/ComfyUI_TTS_Audio_Suite/nodes/chatterbox/chatterbox_tts_node.py`:**
|
||||
|
||||
Line 282 of unified cache adds character prefix:
|
||||
```python
|
||||
cache_params['text'] = f"{cache_params.get('character', 'narrator')}:{text_content}"
|
||||
```
|
||||
|
||||
But line 449 applies crash protection AFTER cache key generation:
|
||||
```python
|
||||
protected_text = self._pad_short_text_for_chatterbox(processed_text, crash_protection_template)
|
||||
# Cache key was generated with unpadded text
|
||||
# But generation uses padded text
|
||||
```
|
||||
|
||||
**Result:**
|
||||
- Cache key generated: `narrator:hello`
|
||||
- Cache lookup tries: `narrator:hello`
|
||||
- Cache miss because audio was generated with: `narrator:hmm ,, hello hmm ,,`
|
||||
|
||||
### Root Cause #5: Audio Component Hash Inconsistency
|
||||
```python
|
||||
# chatterbox_tts_node.py line 603-605
|
||||
stable_audio_component = self._generate_stable_audio_component(
|
||||
inputs.get("reference_audio"), inputs.get("audio_prompt_path", "")
|
||||
)
|
||||
|
||||
# Generates hash from WAVEFORM CONTENT
|
||||
# But if same file accessed via:
|
||||
# 1. Dropdown (direct path)
|
||||
# 2. Character Voices node (copied to temp)
|
||||
# = Different file path = Different hash = CACHE MISS
|
||||
```
|
||||
|
||||
### Root Cause #6: Parameter Duplication in Cache Key
|
||||
**In F5-TTS cache key:**
|
||||
```python
|
||||
'audio_component': params.get('audio_component', ''), # Hash of waveform
|
||||
'ref_text': params.get('ref_text', ''), # Reference text
|
||||
```
|
||||
|
||||
If same text with different reference audio:
|
||||
- audio_component differs ✓ (good - prevents collision)
|
||||
- But if reference audio is different FILE with same CONTENT:
|
||||
- audio_component might be same ✓
|
||||
- ref_text is same ✓
|
||||
- But actual voice characteristics differ ✗
|
||||
|
||||
### Root Cause #7: Untracked Parameter Transformations
|
||||
```python
|
||||
# chatterbox_tts_node.py line 434-436
|
||||
processed_text, pause_segments = PauseTagProcessor.preprocess_text_with_pause_tags(
|
||||
inputs["text"], True
|
||||
)
|
||||
# Cache key uses processed_text, but what if PauseTagProcessor changes the text?
|
||||
# [pause:500ms] removed? Transformed? Cache key would differ!
|
||||
```
|
||||
|
||||
### Root Cause #8: Engine Instance Cache vs Generation Cache Confusion
|
||||
```python
|
||||
# tts_text_node.py line 182
|
||||
cache_key = f"{engine_type}_{hashlib.md5(str(sorted(stable_params.items())).encode()).hexdigest()[:8]}"
|
||||
# This is ENGINE instance cache
|
||||
|
||||
# But users think they're caching generation results
|
||||
# They're actually just reusing the same engine model
|
||||
# Generation parameters affect the OUTPUT, not the CACHE KEY
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. PARAMETER ANALYSIS: What Should/Shouldn't Invalidate Cache
|
||||
|
||||
### ChatterBox Parameters
|
||||
|
||||
| Parameter | Should Invalidate? | Current Behavior | Issue |
|
||||
|-----------|-------------------|------------------|-------|
|
||||
| `text` | YES | YES | Text transformation side-effects |
|
||||
| `character` | YES | YES | Included in cache key |
|
||||
| `exaggeration` | YES | YES | No rounding |
|
||||
| `temperature` | YES | YES | No rounding - floating point issues |
|
||||
| `cfg_weight` | YES | YES | No rounding |
|
||||
| `seed` | YES | YES | ✓ Correct |
|
||||
| `audio_component` | YES | YES | ✓ Hashed correctly |
|
||||
| `language` | YES | YES | ✓ Correct |
|
||||
| `device` | NO* | YES | *Should NOT - output doesn't differ |
|
||||
| `reference_audio` | YES (indirect) | YES | Via audio_component ✓ |
|
||||
| `crash_protection_template` | YES | NO* | *MISSING - changes text! |
|
||||
| `enable_chunking` | MAYBE | NO | Pre-processing, doesn't affect segment cache |
|
||||
| `chunk_combination_method` | NO | NO | Post-processing ✓ |
|
||||
| `silence_between_chunks_ms` | NO | NO | Post-processing ✓ |
|
||||
| `batch_size` | NO | NO | Streaming mode, doesn't affect generation ✓ |
|
||||
|
||||
### F5-TTS Parameters
|
||||
|
||||
| Parameter | Should Invalidate? | Current Behavior | Issue |
|
||||
|-----------|-------------------|------------------|-------|
|
||||
| `text` | YES | YES | ✓ |
|
||||
| `ref_text` | YES | YES | ✓ Correct - different reference = different voice |
|
||||
| `model_name` | YES | YES | ✓ Different model |
|
||||
| `seed` | YES | YES | ✓ |
|
||||
| `temperature` | YES | YES | No rounding |
|
||||
| `speed` | NO* | YES | *WRONG - doesn't affect generation, F5 handles this |
|
||||
| `target_rms` | NO* | YES | *WRONG - post-processing normalization |
|
||||
| `cfg_strength` | YES | YES | Affects generation quality |
|
||||
| `nfe_step` | YES | YES | Affects generation quality |
|
||||
| `cross_fade_duration` | NO | YES | *WRONG - post-processing |
|
||||
| `audio_component` | YES | YES | ✓ Reference audio identifier |
|
||||
| `character` | YES | YES | ✓ |
|
||||
| `device` | NO | YES | *WRONG - doesn't affect generation |
|
||||
|
||||
---
|
||||
|
||||
## 5. DESIGN FLAWS IDENTIFIED
|
||||
|
||||
### Flaw #1: Architectural Fragmentation
|
||||
**Problem:** 5 independent cache systems with no coordination
|
||||
- Unified cache underused
|
||||
- Legacy SRT nodes use separate cache
|
||||
- Voice conversion has separate cache
|
||||
- Engine instance cache conflates with generation cache
|
||||
- Voice discovery cache is orthogonal
|
||||
|
||||
**Impact:** Cache misses when switching between node types, impossible to manage/clear all caches
|
||||
|
||||
### Flaw #2: Floating-Point Precision Not Normalized
|
||||
**Problem:**
|
||||
```python
|
||||
# Different rounding across engines
|
||||
VibeVoice: round(x, 3)
|
||||
IndexTTS: round(x, 3) but only 3 params
|
||||
ChatterBox: No rounding
|
||||
F5-TTS: No rounding
|
||||
```
|
||||
|
||||
**Impact:** Workflow parameter change from UI slider (0.8) vs code (0.80000001) = cache miss
|
||||
|
||||
### Flaw #3: Device Parameter Included in Cache Key
|
||||
**Problem:**
|
||||
```python
|
||||
# Unified cache stores device
|
||||
'device': params.get('device', ''),
|
||||
|
||||
# But device="auto" + resolved device="cuda" = different keys
|
||||
# Yet output is identical!
|
||||
```
|
||||
|
||||
**Impact:** Unnecessary cache misses
|
||||
|
||||
### Flaw #4: Text Pre-Processing Side-Effects Not Tracked
|
||||
**Problems:**
|
||||
1. Crash protection padding happens AFTER cache key generation
|
||||
2. Pause tag preprocessing happens AFTER cache key generation
|
||||
3. Character prefix added during cache key generation
|
||||
4. Text chunking happens separately
|
||||
|
||||
**Impact:** Cache key is for unprocessed text, but generation uses processed text
|
||||
|
||||
### Flaw #5: Audio Component Hashing Fragile
|
||||
**Problem:**
|
||||
```python
|
||||
# audio_component = hash of waveform bytes
|
||||
# If reference audio loaded different ways:
|
||||
# 1. From dropdown -> actual file path
|
||||
# 2. From Character Voices -> temp file
|
||||
# 3. From direct audio input -> memory waveform
|
||||
# = Same audio, potentially different hashes
|
||||
```
|
||||
|
||||
**Impact:** Cache misses for same voice when referenced differently
|
||||
|
||||
### Flaw #6: Mixed Scopes in Cache Keys
|
||||
**Problem:**
|
||||
```python
|
||||
# Per-segment cache parameters (text, audio, seed, temperature)
|
||||
# vs
|
||||
# Workflow-level parameters (device, model, language)
|
||||
# = Same cache key namespace treats both equally
|
||||
|
||||
# Result: Changing device invalidates ALL segment caches
|
||||
# But device doesn't affect generation!
|
||||
```
|
||||
|
||||
### Flaw #7: No Cache Invalidation Strategy
|
||||
**Problem:**
|
||||
- Global caches cleared how? Only manual `audio_cache.clear_cache()`
|
||||
- No TTL/LRU eviction
|
||||
- Memory can grow unbounded
|
||||
- No detection of "stale" cache entries after model updates
|
||||
|
||||
**Impact:** Users can't trust cache after model updates; memory leaks
|
||||
|
||||
### Flaw #8: Engine Instance Cache != Generation Cache
|
||||
**Problem:**
|
||||
```python
|
||||
# Users expect:
|
||||
# "Cache generation results for fast re-run"
|
||||
|
||||
# What actually happens:
|
||||
# "Cache engine instance to avoid reloading model"
|
||||
|
||||
# Different concepts, but code conflates them!
|
||||
```
|
||||
|
||||
**Impact:** Users confused why re-running with different parameters still reuses cache
|
||||
|
||||
---
|
||||
|
||||
## 6. SPECIFIC EXAMPLES OF CACHE MISSES
|
||||
|
||||
### Example 1: Temperature Slider Changes
|
||||
```
|
||||
User workflow:
|
||||
1. Set temperature=0.8, generate (CACHE HIT expected next time)
|
||||
2. Adjust slider to temperature=0.80000001
|
||||
3. Run again, expect CACHE HIT
|
||||
4. Result: CACHE MISS (different float representation)
|
||||
```
|
||||
|
||||
### Example 2: Using Unified TTS vs Legacy SRT Node
|
||||
```
|
||||
User workflow:
|
||||
1. Use "🎤 TTS Text" (unified) node with ChatterBox engine
|
||||
-> Audio stored in utils/audio/cache.py:GLOBAL_AUDIO_CACHE
|
||||
2. Switch to "ChatterBox TTS SRT" legacy node
|
||||
-> Looks in nodes/chatterbox/chatterbox_srt_node.py:GLOBAL_AUDIO_CACHE
|
||||
3. Result: CACHE MISS (different dict)
|
||||
```
|
||||
|
||||
### Example 3: Crash Protection Changes Cache
|
||||
```
|
||||
User workflow:
|
||||
1. Generate with default crash_protection_template="hmm ,, {seg} hmm ,,"
|
||||
Cache key: hash("narrator:hello")
|
||||
Audio generated from: "narrator:hmm ,, hello hmm ,,"
|
||||
|
||||
2. Change crash_protection_template="" (disable padding)
|
||||
3. Try to regenerate same text "hello"
|
||||
4. Result: CACHE HIT returns audio for "hmm ,, hello hmm ,,"
|
||||
But user expects audio for "hello"
|
||||
= WRONG AUDIO RETURNED
|
||||
```
|
||||
|
||||
### Example 4: Device Parameter Mismatch
|
||||
```
|
||||
User workflow:
|
||||
1. Set device="auto" on first run
|
||||
Engine instance cache key: "f5tts_abc123"
|
||||
Generation cache key includes 'device': 'auto'
|
||||
|
||||
2. Device resolves to "cuda" internally
|
||||
|
||||
3. Switch device="cuda" explicitly
|
||||
Engine instance cache key: "f5tts_abc123" (same!)
|
||||
Generation cache key now includes 'device': 'cuda' (different!)
|
||||
|
||||
4. Result: Engine reused (good) but generation cache misses (bad)
|
||||
```
|
||||
|
||||
### Example 5: Reference Audio Path Variation
|
||||
```
|
||||
User workflow:
|
||||
1. Use Character Voices node
|
||||
-> Reference audio hashed from temp file
|
||||
-> audio_component = "ref_audio_abc123_24000"
|
||||
-> Cache key includes this hash
|
||||
|
||||
2. Switch to dropdown selector (same voice file)
|
||||
-> Reference audio hashed from permanent file
|
||||
-> audio_component = "file_audio_def456_24000"
|
||||
-> DIFFERENT HASH despite same voice content!
|
||||
|
||||
3. Result: CACHE MISS
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 7. CURRENT IMPLEMENTATION PATTERNS
|
||||
|
||||
### Pattern 1: ChatterBox TTS Node (Good Model)
|
||||
File: `/home/linux/ComfyUI_TTS_Audio_Suite/nodes/chatterbox/chatterbox_tts_node.py:454-479`
|
||||
|
||||
```python
|
||||
# Generates stable audio component ONCE
|
||||
stable_audio_component = self._generate_stable_audio_component(...)
|
||||
|
||||
# Creates cache function with ALL parameters
|
||||
cache_fn = create_cache_function(
|
||||
"chatterbox",
|
||||
character=character,
|
||||
exaggeration=exaggeration,
|
||||
# ... all params
|
||||
audio_component=audio_component,
|
||||
)
|
||||
|
||||
# Uses cache function for lookup/store
|
||||
cached_audio = cache_fn(protected_text)
|
||||
if cached_audio:
|
||||
return cached_audio
|
||||
|
||||
audio = self.generate_tts_audio(protected_text, ...)
|
||||
cache_fn(protected_text, audio_result=audio)
|
||||
```
|
||||
|
||||
**Strengths:**
|
||||
- Uses unified cache system
|
||||
- Generates audio component once
|
||||
- Uses cache_fn pattern
|
||||
|
||||
**Weaknesses:**
|
||||
- Applies crash protection AFTER defining cache params
|
||||
- Cache key includes protected text, but params generated with unprotected text
|
||||
- No floating-point rounding
|
||||
|
||||
### Pattern 2: F5-TTS Node (Different Approach)
|
||||
File: `/home/linux/ComfyUI_TTS_Audio_Suite/nodes/f5tts/f5tts_node.py`
|
||||
|
||||
**Observation:** Doesn't implement caching at generation level! Instead:
|
||||
- Relies on unified cache through base node
|
||||
- Or relies on F5-TTS model-level caching internally
|
||||
|
||||
### Pattern 3: Voice Conversion (Iteration Caching)
|
||||
File: `/home/linux/ComfyUI_TTS_Audio_Suite/nodes/unified/voice_changer_node.py:154-175`
|
||||
|
||||
```python
|
||||
cache_key = self._generate_rvc_cache_key(processed_source_audio, rvc_model, config)
|
||||
cached_iterations = self._get_cached_rvc_iterations(cache_key, refinement_passes)
|
||||
|
||||
if refinement_passes in cached_iterations:
|
||||
return cached_iterations[refinement_passes] # Full cache hit
|
||||
|
||||
# Start from highest cached iteration
|
||||
for i in range(refinement_passes, 0, -1):
|
||||
if i in cached_iterations:
|
||||
current_audio = cached_iterations[i][0]
|
||||
start_iteration = i
|
||||
break
|
||||
|
||||
# Continue from cached iteration
|
||||
for iteration in range(start_iteration, refinement_passes):
|
||||
converted_audio = rvc_engine.convert_voice(...)
|
||||
self._cache_rvc_result(cache_key, iteration_num, ...)
|
||||
```
|
||||
|
||||
**Strengths:**
|
||||
- Smart iteration-level caching
|
||||
- Can resume from partial progress
|
||||
- Limited to 5 iterations (memory bound)
|
||||
|
||||
**Weaknesses:**
|
||||
- Separate system from generation cache
|
||||
- Not integrated with audio cache
|
||||
|
||||
---
|
||||
|
||||
## 8. ROOT CAUSE RANKING
|
||||
|
||||
| Rank | Root Cause | Impact | Difficulty to Fix |
|
||||
|------|-----------|--------|-------------------|
|
||||
| 1 | Multiple cache dicts (fragmentation) | Very High | Medium |
|
||||
| 2 | No floating-point rounding | High | Low |
|
||||
| 3 | Device parameter in cache key | High | Low |
|
||||
| 4 | Text pre-processing side-effects | High | Medium |
|
||||
| 5 | Crash protection not in cache key | Medium | Low |
|
||||
| 6 | Audio component hashing fragile | Medium | Medium |
|
||||
| 7 | Parameter duplication in F5-TTS key | Low | Low |
|
||||
| 8 | No cache invalidation strategy | Medium | High |
|
||||
|
||||
---
|
||||
|
||||
## 9. DESIGN ISSUES SUMMARY
|
||||
|
||||
### Consolidated Cache Architecture Needed
|
||||
Current: 5 independent systems
|
||||
Desired: 1 unified system with compartments
|
||||
|
||||
### Parameter Normalization Missing
|
||||
- No rounding of floats to consistent precision
|
||||
- Device parameter should be resolved before cache key
|
||||
- Text transformations should be applied before cache key generation
|
||||
|
||||
### Scope Separation Missing
|
||||
- Per-segment parameters (text, seed, audio)
|
||||
- Workflow parameters (device, model, language)
|
||||
- Processing parameters (chunking, silence)
|
||||
|
||||
Should use:
|
||||
- Segment-level cache key = text + audio + generation params
|
||||
- Workflow-level identifier = device + model + language (separate tracking)
|
||||
|
||||
### Cache Key Construction Inconsistent
|
||||
- F5-TTS includes too many parameters (speed, target_rms, cross_fade_duration)
|
||||
- ChatterBox missing crash_protection in key
|
||||
- IndexTTS stores audio paths instead of hashes
|
||||
- No engine has consistent float rounding
|
||||
|
||||
---
|
||||
|
||||
## 10. RECOMMENDATIONS
|
||||
|
||||
### Short Term (Easy Fixes)
|
||||
1. **Add float rounding** to all engines (3 decimal places)
|
||||
2. **Remove device from generation cache key** (doesn't affect output)
|
||||
3. **Remove cross_fade/target_rms/speed from F5-TTS cache key**
|
||||
4. **Add crash_protection_template to ChatterBox cache key**
|
||||
5. **Fix audio_component hashing** to be consistent regardless of load method
|
||||
|
||||
### Medium Term (Moderate Effort)
|
||||
1. **Consolidate cache dicts** into single GLOBAL_AUDIO_CACHE
|
||||
2. **Separate cache scopes:**
|
||||
- Segment cache key = hash(text + audio_component + generation_params)
|
||||
- Workflow tracking = (engine_type, model_name, device, language)
|
||||
3. **Move text transformation BEFORE cache key generation**
|
||||
4. **Implement cache size limits** with LRU eviction
|
||||
|
||||
### Long Term (Architectural)
|
||||
1. **Unified cache system** with clear API
|
||||
2. **Configuration-driven cache parameters** per engine (explicit list of what invalidates cache)
|
||||
3. **Cache statistics and debugging** UI in ComfyUI
|
||||
4. **TTL-based invalidation** after model updates
|
||||
5. **Persistent cache option** for heavy workflows (disk-based)
|
||||
|
||||
@@ -1,280 +0,0 @@
|
||||
# TTS Audio Suite Cache Issues - Executive Summary
|
||||
|
||||
## The Problem
|
||||
|
||||
Generation cache is **hard to fix and often misses** due to **architectural fragmentation** and **inconsistent parameter handling**.
|
||||
|
||||
## Quick Facts
|
||||
|
||||
- **5 separate cache systems** operating independently
|
||||
- **8 major root causes** identified with code locations
|
||||
- **22 specific cache key construction issues** across engines
|
||||
- **At least 5 real-world cache miss scenarios** documented
|
||||
|
||||
## Root Causes (Ranked by Impact)
|
||||
|
||||
### 1. Multiple Cache Dictionaries (CRITICAL)
|
||||
**Files:**
|
||||
- `/home/linux/ComfyUI_TTS_Audio_Suite/utils/audio/cache.py:13` (Unified cache)
|
||||
- `/home/linux/ComfyUI_TTS_Audio_Suite/nodes/chatterbox/chatterbox_srt_node.py:46` (Legacy ChatterBox)
|
||||
- `/home/linux/ComfyUI_TTS_Audio_Suite/nodes/f5tts/f5tts_srt_node.py:43` (Legacy F5-TTS)
|
||||
|
||||
**Problem:** Using unified "🎤 TTS Text" node then switching to "ChatterBox TTS SRT" legacy node = cache miss because they use different `GLOBAL_AUDIO_CACHE` dicts.
|
||||
|
||||
### 2. No Float Rounding (HIGH PRIORITY FIX)
|
||||
**Affected files:**
|
||||
- `/home/linux/ComfyUI_TTS_Audio_Suite/utils/audio/cache.py:50-73` (ChatterBox)
|
||||
- `/home/linux/ComfyUI_TTS_Audio_Suite/utils/audio/cache.py:25-47` (F5-TTS)
|
||||
- All legacy SRT nodes
|
||||
|
||||
**Problem:** Temperature=0.8 vs 0.80000001 = different cache keys despite being same parameter
|
||||
|
||||
**Impact:** Moving UI slider = cache miss
|
||||
|
||||
### 3. Device Parameter Inconsistency (HIGH)
|
||||
**Problem:** Cache key stores "auto" but engine instance cache resolves to "cuda"
|
||||
- Engine cache resolved: `resolve_torch_device("auto")` → "cuda"
|
||||
- Generation cache literal: 'device': "auto"
|
||||
- Result: Cache misses when switching explicit device selections
|
||||
|
||||
### 4. Text Pre-Processing Side-Effects (HIGH)
|
||||
**File:** `/home/linux/ComfyUI_TTS_Audio_Suite/nodes/chatterbox/chatterbox_tts_node.py:449`
|
||||
|
||||
**Problem:**
|
||||
```
|
||||
1. Crash protection padding applied AFTER cache key generation
|
||||
2. Pause tag preprocessing applied separately
|
||||
3. Cache key is for "hello", audio generated from "hmm ,, hello hmm ,,"
|
||||
4. If crash_protection_template changed, wrong audio returned
|
||||
```
|
||||
|
||||
### 5. Crash Protection Not In Cache Key (MEDIUM)
|
||||
**File:** `/home/linux/ComfyUI_TTS_Audio_Suite/nodes/chatterbox/chatterbox_tts_node.py`
|
||||
|
||||
**Problem:** Changing `crash_protection_template` doesn't invalidate cache
|
||||
- Same text → same cache key
|
||||
- But protection template changes actual text used for generation
|
||||
- Result: Returns audio for different protected text
|
||||
|
||||
### 6. Audio Component Hashing Fragile (MEDIUM)
|
||||
**File:** `/home/linux/ComfyUI_TTS_Audio_Suite/utils/audio/audio_hash.py:24`
|
||||
|
||||
**Problem:** Same voice file, different loading methods = different hashes
|
||||
- From dropdown: hashes actual file → "file_audio_abc123"
|
||||
- From Character Voices (temp): hashes temp file → "ref_audio_def456"
|
||||
- From direct audio: hashes waveform → "ref_audio_ghi789"
|
||||
- All are same audio, different hashes = cache misses
|
||||
|
||||
### 7. Parameter Duplication in F5-TTS (LOW)
|
||||
**File:** `/home/linux/ComfyUI_TTS_Audio_Suite/utils/audio/cache.py:25-47`
|
||||
|
||||
**Problem:** F5-TTS cache key includes post-processing parameters
|
||||
- `speed` doesn't affect generation (F5 handles this)
|
||||
- `target_rms` is normalization (post-processing)
|
||||
- `cross_fade_duration` is for combining chunks (post-processing)
|
||||
- Changing these shouldn't invalidate cache, but does
|
||||
|
||||
### 8. No Cache Invalidation Strategy (MEDIUM)
|
||||
**Problem:**
|
||||
- No TTL/LRU eviction
|
||||
- Memory grows unbounded
|
||||
- No detection of "stale" entries after model updates
|
||||
- Manual `audio_cache.clear_cache()` only clears unified cache, not legacy caches
|
||||
|
||||
## Design Issues Summary
|
||||
|
||||
### Issue #1: Architectural Fragmentation
|
||||
```
|
||||
Unified Cache (utils/audio/cache.py)
|
||||
├─ ChatterBoxCacheKeyGenerator
|
||||
├─ F5TTSCacheKeyGenerator
|
||||
├─ VibeVoiceCacheKeyGenerator
|
||||
└─ IndexTTSCacheKeyGenerator
|
||||
|
||||
Legacy ChatterBox Cache (nodes/chatterbox/chatterbox_srt_node.py)
|
||||
└─ Independent _generate_segment_cache_key()
|
||||
|
||||
Legacy F5-TTS Cache (nodes/f5tts/f5tts_srt_node.py)
|
||||
└─ Same as ChatterBox SRT
|
||||
|
||||
Voice Conversion Cache (nodes/unified/voice_changer_node.py)
|
||||
└─ GLOBAL_RVC_ITERATION_CACHE (separate dict)
|
||||
|
||||
Engine Instance Cache (tts_text_node.py)
|
||||
└─ self._cached_engine_instances (confused with generation cache)
|
||||
|
||||
Voice Discovery Cache (utils/voice/cache_manager.py)
|
||||
└─ Persistent disk cache (unrelated to generation)
|
||||
```
|
||||
|
||||
**Result:** Can't centrally manage cache; users confused about what's cached
|
||||
|
||||
### Issue #2: Parameter Normalization Missing
|
||||
No consistent float rounding across engines:
|
||||
- VibeVoice: `round(x, 3)` for 3 params only
|
||||
- IndexTTS: `round(x, 3)` for 3 params, others unrounded
|
||||
- ChatterBox: No rounding
|
||||
- F5-TTS: No rounding
|
||||
|
||||
### Issue #3: Scope Separation Missing
|
||||
Cache key conflates different parameter types:
|
||||
- **Per-segment** (text, seed, audio) - should invalidate cache
|
||||
- **Workflow** (device, model, language) - should NOT invalidate individual segments
|
||||
- **Processing** (chunking, silence) - should NOT invalidate cache
|
||||
|
||||
Currently all in same namespace, causing false invalidations.
|
||||
|
||||
## Real-World Examples
|
||||
|
||||
### Example 1: Temperature Slider (Most Common)
|
||||
```
|
||||
1. Set temperature=0.8, generate audio
|
||||
2. Adjust slider slightly (0.800001)
|
||||
3. Expect cache hit, get cache miss
|
||||
```
|
||||
|
||||
### Example 2: Node Switching
|
||||
```
|
||||
1. Use "🎤 TTS Text" node → cache stored in utils/audio/cache.py
|
||||
2. Switch to "ChatterBox TTS SRT" legacy node → looks in nodes/chatterbox/
|
||||
3. Same text but different cache dict = miss
|
||||
```
|
||||
|
||||
### Example 3: Crash Protection
|
||||
```
|
||||
1. Generate "hello" with crash_protection="hmm ,, {seg} hmm ,,"
|
||||
Audio generated from: "hmm ,, hello hmm ,,"
|
||||
2. Disable crash protection and regenerate
|
||||
3. Cache returns old audio with padding, user expects bare audio
|
||||
```
|
||||
|
||||
### Example 4: Device Parameter
|
||||
```
|
||||
1. Use device="auto" on GPU machine
|
||||
2. Set device="cuda" explicitly
|
||||
3. Engine cache reuses (good) but generation cache misses (bad)
|
||||
```
|
||||
|
||||
## Parameters That SHOULD/SHOULDN'T Invalidate Cache
|
||||
|
||||
### ChatterBox Should Invalidate
|
||||
- text, character, exaggeration, temperature, cfg_weight, seed, audio_component, language
|
||||
- **MISSING:** crash_protection_template (changes actual text!)
|
||||
|
||||
### ChatterBox Should NOT Invalidate
|
||||
- device (doesn't affect generation output)
|
||||
- chunk_combination_method, silence_between_chunks_ms (post-processing)
|
||||
- batch_size (streaming mode setting)
|
||||
|
||||
### F5-TTS Should Invalidate
|
||||
- text, ref_text, model_name, seed, temperature, cfg_strength, nfe_step, audio_component, character
|
||||
|
||||
### F5-TTS Should NOT Invalidate
|
||||
- speed (F5 handles this, doesn't affect segment generation)
|
||||
- target_rms (post-processing normalization)
|
||||
- cross_fade_duration (post-processing for combining segments)
|
||||
- device (doesn't affect generation output)
|
||||
|
||||
## Fix Priority
|
||||
|
||||
### Must Fix (Breaks functionality)
|
||||
1. Add float rounding (trivial, high impact)
|
||||
2. Consolidate cache dicts (medium effort, high impact)
|
||||
3. Add crash_protection_template to cache key (trivial, high impact)
|
||||
|
||||
### Should Fix (Improves reliability)
|
||||
4. Remove device from generation cache key (trivial)
|
||||
5. Remove post-processing params from F5-TTS key (trivial)
|
||||
6. Fix audio_component hashing consistency (medium effort)
|
||||
|
||||
### Nice to Have (Long-term improvements)
|
||||
7. Implement cache size limits with LRU eviction
|
||||
8. Add cache statistics/debugging UI
|
||||
9. Implement TTL-based invalidation after model updates
|
||||
10. Option for persistent (disk-based) cache
|
||||
|
||||
## Current Cache Key Examples
|
||||
|
||||
### ChatterBox (Unified, Line 50-73)
|
||||
```python
|
||||
cache_data = {
|
||||
'text': "narrator:protected_hello", # Includes prefix + crash protection
|
||||
'exaggeration': 0.5, # NO ROUNDING
|
||||
'temperature': 0.8, # NO ROUNDING
|
||||
'cfg_weight': 0.5, # NO ROUNDING
|
||||
'seed': 42,
|
||||
'audio_component': 'file_audio_abc_24k',
|
||||
'model_source': 'chatterbox_english',
|
||||
'device': 'auto', # NOT RESOLVED
|
||||
'language': 'English',
|
||||
'character': 'narrator',
|
||||
'repetition_penalty': 1.0, # NO ROUNDING
|
||||
'min_p': 0.0, # NO ROUNDING
|
||||
'top_p': 1.0, # NO ROUNDING
|
||||
'engine': 'chatterbox'
|
||||
}
|
||||
```
|
||||
|
||||
### F5-TTS (Unified, Line 25-47)
|
||||
```python
|
||||
cache_data = {
|
||||
'text': "narrator:hello",
|
||||
'model_name': 'F5TTS_Base',
|
||||
'device': 'auto', # NOT RESOLVED
|
||||
'audio_component': 'file_audio_def_24k',
|
||||
'ref_text': 'reference text here', # Correct: different ref = different voice
|
||||
'temperature': 0.8, # NO ROUNDING
|
||||
'speed': 1.0, # WRONG: post-processing
|
||||
'target_rms': 0.1, # WRONG: post-processing
|
||||
'cross_fade_duration': 0.15, # WRONG: post-processing
|
||||
'nfe_step': 32,
|
||||
'cfg_strength': 2.0, # NO ROUNDING
|
||||
'seed': 0,
|
||||
'character': 'narrator',
|
||||
'engine': 'f5tts'
|
||||
}
|
||||
```
|
||||
|
||||
### ChatterBox SRT Legacy (Line 361-379)
|
||||
Uses separate cache dict, different engine name ('chatterbox_srt' vs 'chatterbox'),
|
||||
different parameters → cache misses when switching nodes!
|
||||
|
||||
## File References
|
||||
|
||||
All specific issues traced to source:
|
||||
|
||||
**Unified Cache Core:**
|
||||
- `/home/linux/ComfyUI_TTS_Audio_Suite/utils/audio/cache.py` (339 lines)
|
||||
- ChatterBoxCacheKeyGenerator (50-73)
|
||||
- F5TTSCacheKeyGenerator (25-47)
|
||||
- VibeVoiceCacheKeyGenerator (144-181)
|
||||
- IndexTTSCacheKeyGenerator (184-232)
|
||||
- AudioCache class (235-336)
|
||||
|
||||
**Audio Hashing:**
|
||||
- `/home/linux/ComfyUI_TTS_Audio_Suite/utils/audio/audio_hash.py` (142 lines)
|
||||
- generate_stable_audio_component (24-126)
|
||||
|
||||
**Node Implementation:**
|
||||
- `/home/linux/ComfyUI_TTS_Audio_Suite/nodes/chatterbox/chatterbox_tts_node.py`
|
||||
- _pad_short_text_for_chatterbox (136-177) - text transformation
|
||||
- _generate_tts_with_pause_tags (408-546) - text preprocessing
|
||||
- Crash protection applied line 449 AFTER cache key generation
|
||||
|
||||
- `/home/linux/ComfyUI_TTS_Audio_Suite/nodes/chatterbox/chatterbox_srt_node.py`
|
||||
- _generate_segment_cache_key (361-379) - separate cache logic
|
||||
|
||||
- `/home/linux/ComfyUI_TTS_Audio_Suite/nodes/f5tts/f5tts_srt_node.py`
|
||||
- GLOBAL_AUDIO_CACHE (43) - duplicate dict
|
||||
|
||||
- `/home/linux/ComfyUI_TTS_Audio_Suite/nodes/unified/tts_text_node.py`
|
||||
- _create_proper_engine_node_instance (136-441) - engine caching
|
||||
- GLOBAL_AUDIO_CACHE (44) - local storage
|
||||
|
||||
## Conclusion
|
||||
|
||||
The generation cache system has **good intentions but fragmented execution**. The unified cache exists but is underutilized; legacy implementations use separate dicts; engine caching is conflated with generation caching.
|
||||
|
||||
**Key insight:** The system needs **consolidation and normalization**, not complete redesign. All the pieces are there, they just need to work together consistently.
|
||||
|
||||
See CACHE_ANALYSIS.md for detailed technical breakdown with line numbers and code examples.
|
||||
@@ -0,0 +1,140 @@
|
||||
# Claude Version Management Guide
|
||||
|
||||
**This file contains detailed instructions for Claude Code when performing version bumps.**
|
||||
|
||||
## Automated Version Bumping Instructions
|
||||
|
||||
### Primary Command
|
||||
**NEW DEFAULT**: `python3 scripts/bump_version_enhanced.py <version> "<commit_desc>" "<changelog_desc>"`
|
||||
|
||||
**Auto-increment support**: Use `patch`, `minor`, or `major` instead of version numbers for automatic increment.
|
||||
|
||||
### Usage Examples
|
||||
|
||||
#### NEW Default Usage (RECOMMENDED)
|
||||
```bash
|
||||
# Auto-increment with separate descriptions
|
||||
python3 scripts/bump_version_enhanced.py patch "Fix character tag removal bug in single character mode" "Fix unrecognized character tags not being removed from TTS output"
|
||||
|
||||
python3 scripts/bump_version_enhanced.py minor "Add RVC TTS engine support" "Add new RVC (Real-time Voice Conversion) TTS engine with voice cloning capabilities"
|
||||
|
||||
# Manual version with separate descriptions
|
||||
python3 scripts/bump_version_enhanced.py 3.4.3 "Fix crash in audio processing" "Fix crashes when processing certain audio file formats"
|
||||
```
|
||||
|
||||
#### Legacy Usage (DEPRECATED)
|
||||
```bash
|
||||
python3 scripts/bump_version_enhanced.py patch --legacy "Fix character tag removal bug"
|
||||
```
|
||||
|
||||
#### Multiline Description
|
||||
```bash
|
||||
python3 scripts/bump_version_enhanced.py 3.0.2 "Fix missing sounddevice dependency\nAdd comprehensive error handling\nUpdate installation documentation"
|
||||
```
|
||||
|
||||
#### Interactive Mode (for detailed changelog entries)
|
||||
```bash
|
||||
python3 scripts/bump_version_enhanced.py 3.0.2 --interactive
|
||||
```
|
||||
|
||||
#### From File
|
||||
```bash
|
||||
python3 scripts/bump_version_enhanced.py 3.0.2 --file changelog.txt
|
||||
```
|
||||
|
||||
#### Dry Run First (RECOMMENDED)
|
||||
```bash
|
||||
python3 scripts/bump_version_enhanced.py 3.0.2 "Description" --dry-run
|
||||
```
|
||||
|
||||
### When to Bump Versions
|
||||
- **Patch (x.x.X)**: Bug fixes, dependency updates, small tweaks
|
||||
- **Minor (x.X.x)**: New features, significant enhancements
|
||||
- **Major (X.x.x)**: Breaking changes, major architecture changes
|
||||
|
||||
### Files Auto-Updated by Script
|
||||
- README.md (title version)
|
||||
- nodes.py (VERSION constant)
|
||||
- pyproject.toml (version field)
|
||||
- srt/__init__.py (__version__)
|
||||
- core/__init__.py (__version__)
|
||||
- CHANGELOG.md (new entry with date)
|
||||
|
||||
### Script Features
|
||||
- **Automatic categorization**: Sorts changelog entries into Added/Fixed/Changed/Removed
|
||||
- **Multiline commit messages**: Properly formatted for git
|
||||
- **Rollback protection**: Restores files if anything fails
|
||||
- **Version validation**: Ensures semantic versioning format
|
||||
- **Backup system**: Creates backup before making changes
|
||||
|
||||
### Changelog Entry Categories (IMPROVED v3.4.0+)
|
||||
The script automatically categorizes based on enhanced keywords and context:
|
||||
- **Added**: add, new, implement, feature, create, introduce, support, language switching, syntax, bracket, character, integration
|
||||
- **Fixed**: fix, bug, error, issue, resolve, correct, patch
|
||||
- **Changed**: update, enhance, improve, change, modify, optimize, performance, smart, efficient, reduced, eliminated, loading
|
||||
- **Removed**: remove, delete, deprecate, drop
|
||||
|
||||
**Smart Context Detection**:
|
||||
- Recognizes emoji section headers (🌍, 📋, 🚀, 🔧, etc.)
|
||||
- Uses section context for ambiguous items
|
||||
- Defaults to "Added" for feature releases instead of "Fixed"
|
||||
- Skips main title line to avoid categorization errors
|
||||
|
||||
### Example Multiline Changelog Output
|
||||
```markdown
|
||||
## [3.0.2] - 2025-07-14
|
||||
|
||||
### Fixed
|
||||
- Fix missing sounddevice dependency
|
||||
- Add comprehensive error handling
|
||||
|
||||
### Changed
|
||||
- Update installation documentation
|
||||
- Simplify installation process by including all dependencies by default
|
||||
```
|
||||
|
||||
### Fallback Process
|
||||
If the automated script fails, refer to: `docs/Dev reports/VERSION_UPDATE_GUIDE.md`
|
||||
|
||||
### Important Notes
|
||||
- **Never manually update version files** - the script handles all 6 files automatically
|
||||
- **Script auto-commits** with proper multiline message format
|
||||
- **Always dry run first** to preview changes
|
||||
- **Use interactive mode** for complex changelog entries
|
||||
- **The script validates** version format and provides backup/rollback
|
||||
- **Commit messages**: Keep them clean and descriptive. NEVER add Claude co-author credits or "Generated with Claude Code" footers
|
||||
|
||||
### CRITICAL: Fix Poor Changelog Generation
|
||||
|
||||
**Problem**: Script generates detailed commit messages but poorly categorized changelog entries.
|
||||
|
||||
**Root Cause**: The categorization algorithm has flawed keyword matching that can misclassify bug fixes as "Added" items.
|
||||
|
||||
**Solutions**:
|
||||
|
||||
1. **Start descriptions with clear action words** for better categorization:
|
||||
```bash
|
||||
# Good - starts with "Fix" for proper categorization
|
||||
python3 scripts/bump_version_enhanced.py patch "Fix character tag removal bug in single character mode\nRoot cause: TTS nodes bypassed character parser\nResult: Unrecognized character tags are now properly removed"
|
||||
|
||||
# Bad - ambiguous wording leads to miscategorization
|
||||
python3 scripts/bump_version_enhanced.py patch "Character tag removal bug in single character mode\nRoot cause: TTS nodes bypassed character parser\nResult: Character tags are now properly removed"
|
||||
```
|
||||
|
||||
2. **Use separate commit and changelog descriptions** for precise control:
|
||||
```bash
|
||||
python3 scripts/bump_version_enhanced.py patch --commit "Fix character tag removal bug" --changelog "Fix character tags not being removed in single character mode"
|
||||
```
|
||||
|
||||
**For complex changes, use interactive mode**:
|
||||
```bash
|
||||
python3 scripts/bump_version_enhanced.py 3.2.8 --interactive
|
||||
```
|
||||
|
||||
This feeds the script proper technical details that get automatically categorized into good changelog entries.
|
||||
|
||||
### IMPORTANT: v3.4.0 Script Improvements
|
||||
|
||||
**Fixed Issues**: Enhanced multiline parsing with better keyword matching and context detection.
|
||||
|
||||
**Always verify changelog output** - if categorization is wrong, manually fix CHANGELOG.md before pushing.
|
||||
@@ -1,67 +0,0 @@
|
||||
# ComfyUI VRAM Regression Analysis - Python 3.12
|
||||
|
||||
## Summary
|
||||
ComfyUI versions after v0.3.48 introduce severe VRAM spikes specifically in Python 3.12 environments when using TTS Audio Suite. The issue affects model memory management and causes significant performance degradation.
|
||||
|
||||
## Affected Versions
|
||||
- **Last Working**: ComfyUI v0.3.56 + ComfyUI-frontend v1.25.11
|
||||
- **First Broken**: ComfyUI v0.3.57 + ComfyUI-frontend v1.25.11
|
||||
- **Environment**: Python 3.12 only (Python 3.13 unaffected)
|
||||
- **Root Cause**: Commit e2d1e5da "Enable Convolution AutoTuning" enables `torch.backends.cudnn.benchmark = True`
|
||||
|
||||
## Symptoms
|
||||
- Massive VRAM spikes (multi-GB) occurring BEFORE and AFTER TTS inference, not during actual generation
|
||||
- Spikes happen during model loading and audio processing/cleanup phases
|
||||
- Affects all TTS engines: ChatterBox, ChatterBox 23-lang, IndexTTS-2
|
||||
- Performance severely degraded on 24GB VRAM systems
|
||||
- Issue does NOT occur during the TTS inference itself
|
||||
|
||||
## Testing Results
|
||||
|
||||
### ComfyUI v0.3.48 (Working)
|
||||
```
|
||||
Environment: Python 3.12
|
||||
TTS Audio Suite: v4.10.8 (latest)
|
||||
Result: Flat VRAM usage, no spikes
|
||||
All engines tested: ✅ ChatterBox, ✅ ChatterBox 23-lang, ✅ IndexTTS-2
|
||||
```
|
||||
|
||||
### ComfyUI v0.3.59 (Broken)
|
||||
```
|
||||
Environment: Python 3.12
|
||||
TTS Audio Suite: v4.10.8 (latest)
|
||||
Result: Severe VRAM spikes after inference
|
||||
All engines affected: ❌ ChatterBox, ❌ ChatterBox 23-lang, ❌ IndexTTS-2
|
||||
```
|
||||
|
||||
## Root Cause Analysis
|
||||
**Specific Commit**: e2d1e5da "Enable Convolution AutoTuning" by contentis <lspindler@nvidia.com>
|
||||
|
||||
The commit enables `torch.backends.cudnn.benchmark = True` when `--fast autotune` is used. This CUDNN benchmarking:
|
||||
1. Tests multiple convolution algorithms to find the fastest
|
||||
2. **Allocates significant extra VRAM during algorithm benchmarking**
|
||||
3. Appears to interact poorly with Python 3.12's garbage collection
|
||||
4. Causes VRAM spikes during model operations, especially with TTS model wrappers
|
||||
|
||||
## Reproduction Steps
|
||||
1. Install Python 3.12 environment
|
||||
2. Install ComfyUI v0.3.59
|
||||
3. Install TTS Audio Suite v4.10.8
|
||||
4. Load any TTS engine workflow
|
||||
5. Generate audio
|
||||
6. Observe VRAM spikes before and after generation (not during the actual TTS inference)
|
||||
|
||||
## Workaround
|
||||
Downgrade to ComfyUI v0.3.48 + ComfyUI-frontend v1.2.65
|
||||
|
||||
## Next Steps
|
||||
1. Systematically test ComfyUI versions between v0.3.48 and v0.3.59
|
||||
2. Identify exact commit that introduced the regression
|
||||
3. Create detailed GitHub issue for ComfyUI project
|
||||
4. Provide minimal reproduction case
|
||||
|
||||
## Technical Details
|
||||
- Issue is environment-specific (Python 3.12 only)
|
||||
- Affects ComfyUI's model wrapper system
|
||||
- Related to PyTorch CUDA memory management integration
|
||||
- No code changes needed in TTS Audio Suite - issue is upstream in ComfyUI
|
||||
@@ -1,302 +0,0 @@
|
||||
# Dependency Management Guide
|
||||
|
||||
**TTS Audio Suite - Complete dependency compatibility documentation**
|
||||
|
||||
This document tracks all package dependencies, conflicts, solutions, and constraints implemented in the install script for universal compatibility across Python versions and platforms.
|
||||
|
||||
## Quick Reference
|
||||
|
||||
| Package | Version Constraint | Reason | Solution |
|
||||
|---------|-------------------|---------|----------|
|
||||
| `torch` | `>=2.6.0` for security | CVE-2025-32434 (Higgs Audio) | Smart CUDA detection + upgrade |
|
||||
| `numpy` | `>=2.2.0,<2.3.0` | Numba compatibility | Version constraints |
|
||||
| `numba` | `>=0.61.2` | NumPy 2.2+ support | Auto-upgrade from 0.61.0 |
|
||||
| `librosa` | --no-deps | Forces NumPy downgrade | Isolated installation |
|
||||
| `descript-audiotools` | --no-deps | Protobuf 3.19 conflict | Isolated installation |
|
||||
| `faiss-cpu` | `>=1.7.4` | Cross-platform compatibility | Linux: try GPU, Windows: CPU |
|
||||
|
||||
---
|
||||
|
||||
## Critical Dependencies & Conflicts
|
||||
|
||||
### **Python 3.13 Audio-Separator + Resampy Issue**
|
||||
|
||||
**Problem:** Audio-Separator v0.36.1 claims Python 3.13 support, but dependency `resampy 0.4.3` fails
|
||||
- Audio-Separator loads successfully ✅
|
||||
- resampy compilation fails with numba in Python 3.13 ❌
|
||||
- `NUMBA_DISABLE_JIT=1` doesn't prevent resampy from attempting compilation
|
||||
|
||||
**Impact:** Certain UVR models fail with Audio-Separator, force RVC fallback
|
||||
```
|
||||
Failed in nopython mode pipeline (step: nopython frontend)
|
||||
Untyped global name '_resample_loop_s': Cannot determine Numba type of <class 'function'>
|
||||
```
|
||||
|
||||
**Root Cause:** resampy 0.4.3 has incomplete Python 3.13 + numba support despite Audio-Separator's claims
|
||||
|
||||
**Current Solution:**
|
||||
- Audio-Separator fails → RVC fallback works for compatible models (UVR v2/v3 architecture)
|
||||
- Some models (e.g., UVR-DeNoise.pth) incompatible with both engines
|
||||
- Clean error messages implemented to avoid confusion
|
||||
|
||||
**Model Compatibility:**
|
||||
- ✅ **Working Models:** UVR-DeEcho-DeReverb.pth, 6_HP-Karaoke-UVR.pth (via RVC fallback)
|
||||
- ❌ **Broken Models:** UVR-DeNoise.pth, models requiring Audio-Separator-specific architectures
|
||||
|
||||
**Future Fix:** Wait for resampy or Audio-Separator to fix Python 3.13 numba compilation
|
||||
|
||||
---
|
||||
|
||||
### 1. **NumPy + Numba Compatibility Crisis**
|
||||
|
||||
**Problem:** Different numba versions support different NumPy ranges
|
||||
- `numba 0.61.0`: Supports NumPy ≤ 2.1.x ❌
|
||||
- `numba 0.61.2+`: Supports NumPy 2.2.x ✅
|
||||
|
||||
**Impact:** Core engine loading failures
|
||||
```
|
||||
❌ ChatterboxTTS not available: Numba needs NumPy 2.1 or less. Got NumPy 2.2.
|
||||
❌ F5-TTS not available: Numba needs NumPy 2.1 or less. Got NumPy 2.2.
|
||||
```
|
||||
|
||||
**Solution:** Smart version detection and upgrade
|
||||
```python
|
||||
# Detect numba version and upgrade if needed
|
||||
if numba_version == "0.61.0" or numba_version == "0.61.1":
|
||||
pip install --upgrade "numba>=0.61.2"
|
||||
```
|
||||
|
||||
**Constraint:** `numpy>=2.2.0,<2.3.0` + `numba>=0.61.2`
|
||||
|
||||
---
|
||||
|
||||
### 2. **PyTorch Security Vulnerability (CVE-2025-32434)**
|
||||
|
||||
**Problem:** Higgs Audio engine requires PyTorch 2.6+ for security
|
||||
- PyTorch < 2.6: transformers blocks torch.load() due to CVE
|
||||
- PyTorch 2.6+: Security issue resolved
|
||||
|
||||
**Impact:** Higgs Audio fails to load models
|
||||
```
|
||||
ValueError: Due to a serious vulnerability issue in torch.load...
|
||||
```
|
||||
|
||||
**Solution:** Smart CUDA detection and upgrade
|
||||
- Detect system CUDA version (12.8 → cu124)
|
||||
- Install PyTorch 2.6+ with correct CUDA support
|
||||
- Force reinstall if switching CPU↔CUDA variants
|
||||
|
||||
**Constraint:** `torch>=2.6.0` with appropriate CUDA index
|
||||
|
||||
---
|
||||
|
||||
### 3. **Librosa Dependency Hell**
|
||||
|
||||
**Problem:** librosa forces NumPy downgrades
|
||||
- librosa dependencies conflict with our NumPy constraints
|
||||
- Causes cascade of package downgrades
|
||||
|
||||
**Impact:** Breaks carefully balanced dependency tree
|
||||
|
||||
**Solution:** `--no-deps` installation
|
||||
```python
|
||||
pip install librosa --no-deps
|
||||
# Then manually install librosa's actual dependencies
|
||||
```
|
||||
|
||||
**Dependencies manually installed:**
|
||||
- `lazy_loader>=0.1`, `msgpack>=1.0`, `pooch>=1.1`
|
||||
- `soxr>=0.3.2`, `decorator>=4.3.0`, `joblib>=1.0`
|
||||
- `scikit-learn>=1.1.0`, `audioread>=2.1.9`
|
||||
|
||||
---
|
||||
|
||||
### 4. **descript-audiotools Protobuf Conflict**
|
||||
|
||||
**Problem:** Forces protobuf downgrade 6.x → 3.19.x
|
||||
- Breaks other packages expecting modern protobuf
|
||||
- Cascading compatibility issues
|
||||
|
||||
**Solution:** `--no-deps` installation with manual dependencies
|
||||
```python
|
||||
pip install descript-audiotools --no-deps
|
||||
```
|
||||
|
||||
**Dependencies manually installed:**
|
||||
- `flatten-dict`, `ffmpy`, `importlib-resources`
|
||||
- `randomname`, `markdown2`, `pyloudnorm`
|
||||
- `pystoi`, `torch-stoi`, `ipython`, `tensorboard`
|
||||
|
||||
---
|
||||
|
||||
### 5. **FAISS Platform Compatibility**
|
||||
|
||||
**Problem:** faiss-gpu availability varies by platform
|
||||
- Linux: `faiss-gpu-cu12`, `faiss-gpu-cu11` available ✅
|
||||
- Windows: No GPU version available via pip ❌
|
||||
|
||||
**Solution:** Platform-specific installation
|
||||
```python
|
||||
if not is_windows and cuda_detected:
|
||||
try:
|
||||
pip install faiss-gpu-cu12 # Linux + CUDA
|
||||
except:
|
||||
pip install faiss-cpu # Fallback
|
||||
else:
|
||||
pip install faiss-cpu # Windows/CPU-only
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Python Version Compatibility
|
||||
|
||||
### Python 3.13 Specific Issues
|
||||
|
||||
| Package | Issue | Solution | Status |
|
||||
|---------|-------|----------|--------|
|
||||
| `MediaPipe` | Binary incompatible | OpenSeeFace fallback | ✅ Working |
|
||||
| `numba` | JIT compilation issues | Disable JIT in Python 3.13 | ✅ Working |
|
||||
| `audioop` packages | Not available | Remove from dependencies | ✅ Fixed |
|
||||
| `resampy` | Numba compilation fails in Audio-Separator | RVC fallback works for compatible models | ⚠️ Partial |
|
||||
|
||||
### Python 3.12 Compatibility
|
||||
|
||||
| Package | Issue | Solution | Status |
|
||||
|---------|-------|----------|--------|
|
||||
| All engines | NumPy/numba mismatch | Smart version detection | ✅ Working |
|
||||
| PyTorch CUDA | CPU vs CUDA variants | Force reinstall detection | ✅ Working |
|
||||
|
||||
---
|
||||
|
||||
## Install Script Logic Flow
|
||||
|
||||
### 1. **Environment Detection**
|
||||
```python
|
||||
- Python version (3.12, 3.13+)
|
||||
- Platform (Windows, Linux, macOS)
|
||||
- CUDA availability (nvidia-smi detection)
|
||||
- CUDA version (12.8 → cu124, 11.8 → cu118)
|
||||
```
|
||||
|
||||
### 2. **PyTorch Installation**
|
||||
```python
|
||||
- Check existing version and CUDA support
|
||||
- Upgrade if needed (security + compatibility)
|
||||
- Platform-specific CUDA index selection
|
||||
```
|
||||
|
||||
### 3. **Core Dependencies**
|
||||
```python
|
||||
- Smart package checking (skip if satisfied)
|
||||
- Safe packages installed normally
|
||||
- Sub-dependencies for --no-deps packages
|
||||
```
|
||||
|
||||
### 4. **NumPy + Numba Resolution**
|
||||
```python
|
||||
- Detect numba version compatibility
|
||||
- Upgrade numba if needed (0.61.0 → 0.61.2+)
|
||||
- Install constrained NumPy (>=2.2.0,<2.3.0)
|
||||
```
|
||||
|
||||
### 5. **Problematic Package Isolation**
|
||||
```python
|
||||
- Install with --no-deps to prevent conflicts
|
||||
- Manually install required sub-dependencies
|
||||
- Maintain compatibility with main dependency tree
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Package Categories
|
||||
|
||||
### ✅ **SAFE Packages** (Normal installation)
|
||||
- `transformers>=4.46.3`, `accelerate`, `datasets`
|
||||
- `soundfile>=0.12.0`, `sounddevice>=0.4.0`
|
||||
- `jieba`, `pypinyin`, `unidecode`
|
||||
- `conformer>=0.3.2`, `x-transformers`, `vocos`
|
||||
- `s3tokenizer>=0.1.7`, `resemble-perth`
|
||||
|
||||
### ⚠️ **PROBLEMATIC Packages** (--no-deps required)
|
||||
- `librosa` - Forces NumPy downgrade
|
||||
- `descript-audiotools` - Protobuf conflicts
|
||||
- `descript-audio-codec` - Dependency conflicts
|
||||
- `cached-path` - Forces package downgrades
|
||||
- `torchcrepe` - Conflicts via librosa
|
||||
- `onnxruntime` - Forces NumPy 2.3.x (Python 3.13 only)
|
||||
|
||||
### 🎯 **VERSION-CRITICAL Packages**
|
||||
- `torch>=2.6.0` - Security requirement
|
||||
- `numpy>=2.2.0,<2.3.0` - Numba compatibility
|
||||
- `numba>=0.61.2` - NumPy 2.2+ support
|
||||
- `transformers>=4.46.3` - Higgs Audio compatibility
|
||||
|
||||
---
|
||||
|
||||
## Testing Matrix
|
||||
|
||||
### Verified Working Combinations
|
||||
|
||||
| Python | Platform | PyTorch | NumPy | Numba | Status |
|
||||
|--------|----------|---------|-------|--------|--------|
|
||||
| 3.12.6 | Windows | 2.7.1+cu126 | 2.2.6 | 0.61.2 | ✅ All engines |
|
||||
| 3.13.x | Windows | 2.8.0+cpu | 2.2.6 | 0.61.2 | ✅ All engines |
|
||||
| 3.12.6 | Linux | 2.6.0+cu121 | 2.2.6 | 0.61.2 | ✅ All engines |
|
||||
|
||||
### Known Broken Combinations
|
||||
|
||||
| Python | Platform | PyTorch | NumPy | Numba | Issue |
|
||||
|--------|----------|---------|-------|--------|-------|
|
||||
| 3.12.6 | Windows | Any | 2.2.6 | 0.61.0 | ❌ Engine loading |
|
||||
| Any | Any | <2.6.0 | Any | Any | ❌ Higgs Audio CVE |
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting Guide
|
||||
|
||||
### Engine Loading Failures
|
||||
```
|
||||
❌ ChatterboxTTS not available: Numba needs NumPy 2.1 or less
|
||||
```
|
||||
**Cause:** numba 0.61.0 with NumPy 2.2+
|
||||
**Fix:** Run install script to upgrade numba to 0.61.2+
|
||||
|
||||
### Higgs Audio Model Loading
|
||||
```
|
||||
❌ CVE-2025-32434 vulnerability error
|
||||
```
|
||||
**Cause:** PyTorch < 2.6.0
|
||||
**Fix:** Run install script to upgrade PyTorch 2.6+
|
||||
|
||||
### Windows CUDA Issues
|
||||
```
|
||||
AssertionError: Torch not compiled with CUDA enabled
|
||||
```
|
||||
**Cause:** CPU PyTorch on CUDA system
|
||||
**Fix:** Install script detects and reinstalls CUDA PyTorch
|
||||
|
||||
---
|
||||
|
||||
## Maintenance Notes
|
||||
|
||||
### When Adding New Dependencies
|
||||
1. **Check conflicts** with existing constraints
|
||||
2. **Test with problematic packages** (librosa, descript-*)
|
||||
3. **Update this documentation**
|
||||
4. **Add to appropriate category** (Safe/Problematic/Critical)
|
||||
|
||||
### When Supporting New Python Versions
|
||||
1. **Test numba compatibility** with NumPy constraints
|
||||
2. **Check for binary package availability**
|
||||
3. **Update compatibility matrix**
|
||||
4. **Add version-specific workarounds if needed**
|
||||
|
||||
### When Updating Major Dependencies
|
||||
1. **Review cascade effects** on sub-dependencies
|
||||
2. **Test with --no-deps packages**
|
||||
3. **Update version constraints**
|
||||
4. **Verify cross-platform compatibility**
|
||||
|
||||
---
|
||||
|
||||
*Last updated: Version 4.8.19 - Python 3.13 Audio-Separator + resampy compatibility issues documented*
|
||||
@@ -1,183 +0,0 @@
|
||||
# DramaBox and Chatterbox Multilingual V3 Capability and Scope
|
||||
|
||||
Research date: 2026-07-25
|
||||
|
||||
## Official references
|
||||
|
||||
- DramaBox code: `resemble-ai/DramaBox` at
|
||||
`a70a5818e103c1c9fef22409c1e0c707ebf4f8a7`
|
||||
- DramaBox weights: `ResembleAI/Dramabox` at
|
||||
`404f967f653fa1170dc15a9d1ddd3fdb9a0a842d`
|
||||
- Chatterbox code: `resemble-ai/chatterbox` at
|
||||
`5de7a54aa4e5e2baadb0182dde554908b48b85c2`
|
||||
- Chatterbox weights: `ResembleAI/chatterbox` at
|
||||
`5bb1f6ee58e50c3b8d408bc82a6d3740c2db6e18`
|
||||
- ComfyUI reference only: `kat3ri/ComfyUI-DramaBox` at
|
||||
`715fcb11cc14d8c185438e2319b52fc00163941c`
|
||||
|
||||
The repositories were cloned under
|
||||
`IgnoredForGitHubDocs/For_reference/`.
|
||||
|
||||
## DramaBox capability report
|
||||
|
||||
### Native scope
|
||||
|
||||
- Task: English text-to-speech with optional zero-shot voice cloning.
|
||||
- Expressive control: prompt-driven speaker description, delivery, emotion,
|
||||
pauses, laughs, sighs, and transitions.
|
||||
- Voice input: optional reference audio; upstream uses up to 10 seconds.
|
||||
- No native voice conversion, ASR, or audio editing API.
|
||||
- No language control. The official model is English-only.
|
||||
- No extra special node is required. Its structured scene prompt fits the
|
||||
existing unified text and SRT nodes.
|
||||
|
||||
### Native generation parameters
|
||||
|
||||
- `cfg_scale` (official warm-server default `2.5`)
|
||||
- `stg_scale` (default `1.5`)
|
||||
- `duration_multiplier` (default `1.1`)
|
||||
- `seed` (default `42`)
|
||||
- `ref_duration` (default `10.0` seconds)
|
||||
- `rescale_scale` (`auto` by default)
|
||||
- `gen_duration` (`0` means automatic)
|
||||
- Official long-form chunk limits and crossfade parameters
|
||||
|
||||
The initial suite UI exposes `cfg_scale`, `stg_scale`, and
|
||||
`duration_multiplier`. Seed remains owned by the unified TTS nodes.
|
||||
Reference duration, rescale, steps, modality guidance, and explicit output
|
||||
duration stay on official defaults because exposing them would add expert
|
||||
controls without a demonstrated suite use case. The official duration-aware
|
||||
long-form path is used automatically instead of adding duplicate chunk UI.
|
||||
|
||||
### Audio and generation behavior
|
||||
|
||||
- The LTX audio decoder returns stereo audio at 48 kHz.
|
||||
- The base model was trained on clips around 20 seconds. Current upstream
|
||||
supports longer clips with a silence-prior correction and automatically
|
||||
chunks prompts targeting about 37 seconds with a 45-second cap.
|
||||
- The official long-form chunker preserves the scene/speaker prefix and quote
|
||||
groups, then joins chunks with a 50 ms equal-power crossfade.
|
||||
- Upstream applies the Perth watermark only in `generate_to_file()`, not in
|
||||
the in-memory `generate()` method. The suite wrapper must therefore apply
|
||||
the watermark to in-memory output explicitly.
|
||||
|
||||
### Model layout
|
||||
|
||||
Organized destination: `ComfyUI/models/TTS/dramabox/DramaBox/`
|
||||
|
||||
- `dramabox-dit-v1.safetensors` — 6,575,225,528 bytes
|
||||
- `dramabox-audio-components.safetensors` — 1,942,831,020 bytes
|
||||
- `assets/silence_latent_frame.pt` — 1,501 bytes
|
||||
- `gemma-3-12b-it-bnb-4bit/`
|
||||
- two safetensor shards plus tokenizer/config files from
|
||||
`unsloth/gemma-3-12b-it-bnb-4bit`
|
||||
|
||||
The implementation must use the suite downloader with `local_dir`-style
|
||||
organized downloads and disable Transformers/Hugging Face fallback downloads.
|
||||
|
||||
### Dependencies and runtime
|
||||
|
||||
The official requirements include Torch/Torchaudio 2.8, Transformers 4.45+,
|
||||
bitsandbytes 0.45+, Accelerate, PEFT, PyAV, Einops, SentencePiece,
|
||||
Safetensors, PyYAML, and Perth. The official source imports successfully in
|
||||
the configured suite validation environment with Torch 2.10 and Transformers
|
||||
5.10, so DramaBox belongs in the main Transformers 5 environment.
|
||||
|
||||
The optional NVIDIA RE-USE reference denoiser is intentionally excluded:
|
||||
its Mamba dependencies have no practical Windows installation path and its
|
||||
NSCLv1 non-commercial license is a poor default for the suite.
|
||||
|
||||
### License
|
||||
|
||||
DramaBox code and weights are under the LTX-2 Community License, not MIT.
|
||||
The license requires attribution, use restrictions, modified-file notices,
|
||||
and a separate paid license for entities with at least USD 10 million in
|
||||
annual revenue. The upstream license must ship beside any bundled inference
|
||||
code, and the engine UI/docs must disclose the restriction.
|
||||
|
||||
## Existing ComfyUI reference notes
|
||||
|
||||
`kat3ri/ComfyUI-DramaBox` confirms useful ComfyUI audio-shape handling,
|
||||
organized model paths, the warm `TTSServer` API, and practical UI ranges.
|
||||
It must not be copied as architecture:
|
||||
|
||||
- It auto-clones source code at runtime.
|
||||
- It has no unified model lifecycle, cache, character/pause integration, SRT
|
||||
processor, interrupt handling, or generation report integration.
|
||||
- It directly calls the engine from a standalone node.
|
||||
- It patches partially imported bitsandbytes modules globally.
|
||||
- Its README says output is watermarked, but its node calls the unwatermarked
|
||||
in-memory upstream path.
|
||||
|
||||
## Chatterbox Multilingual V3 capability report
|
||||
|
||||
V3 is not a new engine. Official upstream loads it as an opt-in checkpoint through
|
||||
`ChatterboxMultilingualTTS.from_pretrained(..., t3_model="v3")`; the only
|
||||
model-family change is selecting `t3_mtl23ls_v3.safetensors` instead of the
|
||||
V2 T3 checkpoint. Its official generation path also skips the legacy
|
||||
alignment analyzer, uses repetition penalty `1.2`, and removes the final
|
||||
degraded pre-EOS speech-token artifact. It keeps
|
||||
the same 500M architecture, 23-language list, 24 kHz output, voice-reference
|
||||
mode, tokenizer, voice encoder, S3Gen decoder, and generation parameters:
|
||||
|
||||
- `language_id`
|
||||
- `exaggeration`
|
||||
- `cfg_weight`
|
||||
- `temperature`
|
||||
- `repetition_penalty`
|
||||
- `min_p`
|
||||
- `top_p`
|
||||
|
||||
The suite forwards V3 `exaggeration` using the upstream/native scale. Manual
|
||||
testing found little or no audible response across values, so this remains a
|
||||
current checkpoint limitation rather than a suite-side scaling issue.
|
||||
|
||||
The existing `chatterbox_official_23lang` engine already implements Unified
|
||||
TTS Text, Unified SRT TTS, voice references, caching, character switching,
|
||||
pause tags, parameter switching, and lifecycle handling. V3 therefore
|
||||
extends its `model_version` choices and downloader requirements. It must not
|
||||
create a second engine node or duplicate processors.
|
||||
|
||||
## Integration scope
|
||||
|
||||
### DramaBox
|
||||
|
||||
- Unified TTS Text: yes
|
||||
- Unified SRT TTS: yes
|
||||
- Character tags and narrator fallback: yes
|
||||
- Pause tags: yes
|
||||
- Segment switching: `seed`, `cfg_scale`, `stg_scale`, and
|
||||
`duration_multiplier`
|
||||
- Generated audio cache: yes
|
||||
- Long-form strategy: official duration-aware chunker; ignore suite
|
||||
character-count chunking inside each already separated character/pause
|
||||
segment
|
||||
- Clear VRAM: full TTSServer teardown and lazy reload because the quantized
|
||||
Gemma stack should not be copied to system RAM
|
||||
- Runtime: main environment
|
||||
- Voice Changer / ASR / editing / special node: no
|
||||
|
||||
### Chatterbox Multilingual V3
|
||||
|
||||
- Extend existing Official 23-Lang model version control with V3.
|
||||
- Keep V2 available for backward compatibility.
|
||||
- Make V3 the suite default for new configurations so the newly requested
|
||||
version is immediately selected. Upstream still defaults to V2 and exposes
|
||||
V3 as opt-in.
|
||||
- Existing saved workflows with V1/V2 values continue to load unchanged.
|
||||
|
||||
## Validation matrix
|
||||
|
||||
- Static import and registration checks
|
||||
- Chatterbox V1/V2/V3 file-resolution tests without downloading weights
|
||||
- DramaBox downloader layout checks without downloading the 16+ GB models
|
||||
- DramaBox TTS processor tests with a fake adapter for character, pause,
|
||||
cache-facing parameter, audio-shape, and combination behavior
|
||||
- SRT processor interrupt and timing-path tests with fake generation
|
||||
- Live FL-MCP checks after restarting ComfyUI:
|
||||
- engine and unified nodes register
|
||||
- smallest DramaBox text workflow loads
|
||||
- smallest DramaBox SRT workflow loads
|
||||
- full generation is attempted only if all 16+ GB weights and adequate
|
||||
VRAM are available
|
||||
- Human assessment remains required for subjective audio quality.
|
||||
@@ -1,573 +0,0 @@
|
||||
# Echo-TTS PR #250 - Comprehensive Code Review
|
||||
|
||||
**Date:** 2026-02-05
|
||||
**PR Author:** drphero (Brent)
|
||||
**Reviewer:** Claude (TTS Audio Suite maintainer perspective)
|
||||
**Reference Implementation:** Qwen3-TTS
|
||||
**Guides Referenced:**
|
||||
|
||||
- `docs/NEW_ENGINE_IMPLEMENTATION_GUIDE.md`
|
||||
- `docs/Dev reports/fails_to_avoid_TTS_Engine_Implementation.md`
|
||||
|
||||
---
|
||||
|
||||
## Executive Summary
|
||||
|
||||
The Echo-TTS integration has **major architectural issues** that deviate significantly from project standards. While it functions, the implementation embeds 180+ lines of orchestration logic directly in the unified node instead of using the standard processor pattern, making it non-reusable and harder to maintain.
|
||||
|
||||
**Status:** ⚠️ **NEEDS SIGNIFICANT REFACTORING** - Critical architectural issues, missing standard integrations
|
||||
|
||||
---
|
||||
|
||||
## Critical Issues (MUST FIX)
|
||||
|
||||
### 1. ❌ **Missing Audio Caching System**
|
||||
|
||||
**Severity:** CRITICAL
|
||||
**Location:** `engines/adapters/echo_tts_adapter.py`
|
||||
|
||||
**Issue:**
|
||||
|
||||
- No import of `utils.audio.cache`
|
||||
- No `audio_cache` instance creation
|
||||
- No `generate_cache_key()` calls before generation
|
||||
- No `cache_audio()` calls after generation
|
||||
- No cache hit checks
|
||||
|
||||
**Evidence:**
|
||||
|
||||
```bash
|
||||
$ grep "audio_cache\|cache_audio\|generate_cache_key" engines/adapters/echo_tts_adapter.py
|
||||
# No matches found
|
||||
```
|
||||
|
||||
**Impact:**
|
||||
|
||||
- Every identical generation request re-generates audio (wasting time + VRAM)
|
||||
- No performance benefits from repeated requests
|
||||
- Inconsistent with ALL other engines (Qwen3, Step Audio EditX, F5-TTS, ChatterBox)
|
||||
|
||||
**Reference Implementation (Qwen3-TTS):**
|
||||
|
||||
```python
|
||||
# qwen3_tts_adapter.py:49
|
||||
self.audio_cache = get_audio_cache()
|
||||
|
||||
# qwen3_tts_adapter.py:349-372
|
||||
cache_key = self.audio_cache.generate_cache_key(
|
||||
'qwen3_tts',
|
||||
text=text,
|
||||
model_type='CustomVoice',
|
||||
speaker=speaker,
|
||||
# ... all params ...
|
||||
)
|
||||
cached_audio = self.audio_cache.get_cached_audio(cache_key)
|
||||
if cached_audio:
|
||||
return cached_audio[0]
|
||||
# ... generate ...
|
||||
self.audio_cache.cache_audio(cache_key, audio_tensor, duration)
|
||||
```
|
||||
|
||||
**Fix Required:**
|
||||
|
||||
1. Add `from utils.audio.cache import get_audio_cache` import
|
||||
2. Initialize `self.audio_cache = get_audio_cache()` in `__init__`
|
||||
3. Generate cache keys in `_generate_audio_for_text()` BEFORE generation
|
||||
4. Check cache and return if hit
|
||||
5. Store results in cache after generation with duration calculation
|
||||
|
||||
---
|
||||
|
||||
### 2. ❌ **Non-Standard Architecture - Logic Embedded in Unified Node**
|
||||
|
||||
**Severity:** CRITICAL (Architecture)
|
||||
**Location:** `nodes/unified/tts_text_node.py` lines 1128-1304
|
||||
|
||||
**Issue:**
|
||||
Echo-TTS embeds **ALL orchestration logic** (character switching, pause tags, segment parameters) directly in the unified TTS text node instead of in a processor or adapter. This is **180+ lines of engine-specific code** in what should be a generic unified interface.
|
||||
|
||||
**What's embedded in unified node (lines 1128-1304):**
|
||||
|
||||
- Character parser setup and configuration (30+ lines)
|
||||
- Character voice mapping logic (40+ lines)
|
||||
- Pause tag processing (embedded in character loop)
|
||||
- Segment parameter application
|
||||
- Audio generation loop with character switching
|
||||
- Chunk timing and combination
|
||||
|
||||
**Evidence:**
|
||||
|
||||
```python
|
||||
# nodes/unified/tts_text_node.py:1128
|
||||
elif engine_type == "echo_tts":
|
||||
# 180+ lines of orchestration logic here
|
||||
# Character parser setup
|
||||
# Voice mapping
|
||||
# Segment processing
|
||||
# Pause tag handling
|
||||
# Audio generation
|
||||
```
|
||||
|
||||
**Standard Architecture (Qwen3-TTS, Step Audio EditX, IndexTTS):**
|
||||
|
||||
```
|
||||
Unified Node (lines 1306-1450):
|
||||
├─ Setup character parser (10 lines)
|
||||
├─ Build voice_mapping (50 lines)
|
||||
└─ Call processor.process_text(text, voice_mapping, ...)
|
||||
|
||||
Processor (qwen3_tts_processor.py):
|
||||
├─ Character switching orchestration
|
||||
├─ Pause tag processing
|
||||
├─ Segment parameter handling
|
||||
└─ Audio generation coordination
|
||||
```
|
||||
|
||||
**Impact:**
|
||||
|
||||
- **Code duplication**: 180 lines of orchestration logic that should be reusable
|
||||
- **Maintenance burden**: Changes to character switching require editing unified node instead of processor
|
||||
- **Non-standard**: Every other engine (Qwen3, Step EditX, IndexTTS, VibeVoice) uses processor pattern
|
||||
- **Not reusable**: Logic cannot be reused by other nodes (e.g., batch processing, API endpoints)
|
||||
- **Harder to test**: Cannot unit test Echo-TTS orchestration without loading entire unified node
|
||||
|
||||
**Fix Required:**
|
||||
Create `nodes/echo_tts/echo_tts_processor.py` following the Qwen3-TTS pattern:
|
||||
|
||||
1. Move character parser setup to processor `__init__`
|
||||
2. Move voice mapping logic to processor
|
||||
3. Implement `process_text(text, voice_mapping, seed, ...)` method
|
||||
4. Move pause tag processing to processor
|
||||
5. Move segment parameter handling to processor
|
||||
6. Move character switching loop to processor
|
||||
7. Update unified node to call `processor.process_text()` (10 lines instead of 180)
|
||||
|
||||
**Reference:** See `nodes/qwen3_tts/qwen3_tts_processor.py` lines 260-441 for correct processor pattern.
|
||||
|
||||
---
|
||||
|
||||
### 3. ❌ **Missing Interrupt Handling**
|
||||
|
||||
**Severity:** CRITICAL (User Experience)
|
||||
**Location:** `nodes/echo_tts/echo_tts_srt_processor.py`
|
||||
|
||||
**Issue:**
|
||||
|
||||
- No import of `comfy.model_management`
|
||||
- No interrupt checks in subtitle processing loop
|
||||
- Users cannot cancel long-running SRT generations
|
||||
|
||||
**Evidence:**
|
||||
|
||||
```bash
|
||||
$ grep "interrupt_processing" nodes/echo_tts/
|
||||
# No matches found
|
||||
```
|
||||
|
||||
**Impact:**
|
||||
|
||||
- Users MUST wait for entire SRT generation to complete (could be hours for long videos)
|
||||
- No way to cancel if wrong settings are used
|
||||
- Other engines ALL support interruption (Qwen3, Higgs, VibeVoice, CosyVoice)
|
||||
|
||||
**Reference Implementation (Qwen3-TTS SRT):**
|
||||
|
||||
```python
|
||||
# qwen3_tts_srt_processor.py:12
|
||||
import comfy.model_management as model_management
|
||||
|
||||
# In main loop (line ~200):
|
||||
for i, subtitle in enumerate(subtitles):
|
||||
if model_management.interrupt_processing:
|
||||
raise InterruptedError(f"Qwen3-TTS SRT generation interrupted at subtitle {i+1}/{len(subtitles)}")
|
||||
# ... process subtitle ...
|
||||
```
|
||||
|
||||
**Fix Required:**
|
||||
|
||||
1. Add `import comfy.model_management as model_management` at top of SRT processor
|
||||
2. Add interrupt check at start of main subtitle loop
|
||||
3. Add interrupt check inside character segment loops (if applicable)
|
||||
4. Raise `InterruptedError` with informative message
|
||||
|
||||
---
|
||||
|
||||
## Important Issues (SHOULD FIX)
|
||||
|
||||
### 4. ⚠️ **Languages Parameter Should Be Removed**
|
||||
|
||||
**Severity:** IMPORTANT
|
||||
**Location:** `nodes/engines/echo_tts_engine_node.py`
|
||||
|
||||
**Issue:**
|
||||
The engine node has a `languages` string input parameter that appears to be incorrect:
|
||||
|
||||
- No language parameter exists in the official Echo-TTS inference API
|
||||
- All text presets in official repo are English-only
|
||||
- Uses byte-level UTF-8 tokenization (not language-specific)
|
||||
- The parsed `languages` parameter in the adapter is never actually used in generation
|
||||
|
||||
**Evidence:**
|
||||
|
||||
```python
|
||||
# echo_tts_adapter.py:188-195 - parsed but never used
|
||||
def _parse_languages(self) -> List[str]:
|
||||
langs = self.config.get("languages", "en")
|
||||
# ... parsing logic ...
|
||||
return parts or ["en"]
|
||||
```
|
||||
|
||||
**Impact:**
|
||||
|
||||
- User confusion (users think they can generate in multiple languages)
|
||||
- Non-functional parameter (doesn't affect output)
|
||||
- Echo-TTS is English-only model
|
||||
|
||||
**Fix Required:**
|
||||
Remove the `languages` widget entirely from `nodes/engines/echo_tts_engine_node.py`
|
||||
|
||||
---
|
||||
|
||||
### 5. ⚠️ **Missing ComfyUI Progress Bars**
|
||||
|
||||
**Severity:** IMPORTANT (UX)
|
||||
**Location:** `engines/adapters/echo_tts_adapter.py`
|
||||
|
||||
**Issue:**
|
||||
|
||||
- No progress bar creation in generation methods
|
||||
- User has no feedback during long generations
|
||||
|
||||
**Reference Implementation (Qwen3-TTS):**
|
||||
|
||||
```python
|
||||
# qwen3_tts_adapter.py:763-788
|
||||
def _create_progress_bar(self, max_tokens: int, text: str = ""):
|
||||
try:
|
||||
import comfy.utils
|
||||
# Estimate tokens based on text
|
||||
estimated_tokens = int(len(text) * 1.5)
|
||||
progress_total = min(estimated_tokens, max_tokens)
|
||||
return comfy.utils.ProgressBar(progress_total)
|
||||
except (ImportError, AttributeError):
|
||||
return None
|
||||
```
|
||||
|
||||
**Fix Required:**
|
||||
|
||||
1. Add progress bar creation in `_generate_audio_for_text()`
|
||||
2. Update progress during sampling/generation
|
||||
3. Estimate total steps based on `num_steps` parameter
|
||||
|
||||
---
|
||||
|
||||
### 6. ⚠️ **Incorrect SRT Timing Mode Restriction**
|
||||
|
||||
**Severity:** IMPORTANT (Bug)
|
||||
**Location:** `engines/adapters/echo_tts_adapter.py:373-376`
|
||||
|
||||
**Issue:**
|
||||
The adapter has an incorrect check that restricts timing modes:
|
||||
|
||||
```python
|
||||
# Echo-TTS SRT support is minimal; fall back to pad_with_silence if needed
|
||||
if timing_mode not in ["pad_with_silence", "concatenate"]:
|
||||
print(f"WARNING: Echo-TTS SRT: timing_mode '{timing_mode}' not supported, using pad_with_silence")
|
||||
timing_mode = "pad_with_silence"
|
||||
```
|
||||
|
||||
However, the `AudioAssemblyEngine` called on line 389 **DOES support all timing modes** (`stretch_to_fit`, `smart_natural`, etc.). This check is unnecessarily restrictive.
|
||||
|
||||
**Impact:**
|
||||
|
||||
- Users cannot use `stretch_to_fit` or other advanced timing modes
|
||||
- Warning message is misleading - modes ARE supported by the assembly engine
|
||||
- Artificially limits functionality that would work fine
|
||||
|
||||
**Fix Required:**
|
||||
Remove lines 373-376 entirely. The `AudioAssemblyEngine.assemble_by_timing_mode()` already handles all timing modes correctly and will raise proper errors for unsupported modes.
|
||||
|
||||
---
|
||||
|
||||
## Minor Issues (NICE TO FIX)
|
||||
|
||||
### 7. 📝 **`[S1]` Text Normalization Missing `[S2]` Safety Check**
|
||||
|
||||
**Severity:** MINOR
|
||||
**Location:** `engines/adapters/echo_tts_adapter.py:81-95`
|
||||
|
||||
**Issue:**
|
||||
The `_normalize_prompt_text()` method correctly adds `[S1]` prefix to follow Echo-TTS WhisperD format, but is missing the `[S2]` safety check from the official implementation.
|
||||
|
||||
**Current behavior:**
|
||||
|
||||
```python
|
||||
# PR code (line 95)
|
||||
return f"[S1] {cleaned}".strip() # Always adds [S1]
|
||||
```
|
||||
|
||||
**Result with `[S2]` input:**
|
||||
|
||||
```python
|
||||
Input: "[S2] Hello there"
|
||||
Output: "[S1] [S2] Hello there" # Wrong - double prefix
|
||||
```
|
||||
|
||||
**Official behavior (inference.py:125-126):**
|
||||
|
||||
```python
|
||||
if not text.startswith("[") and not text.startswith("(") and 'S1' not in text and 'S2' not in text:
|
||||
text = "[S1] " + text
|
||||
```
|
||||
|
||||
**Impact:**
|
||||
|
||||
- Minor issue - most users won't use `[S2]` tags
|
||||
- Creates double prefix `[S1] [S2]` which is ugly but harmless (doesn't break generation)
|
||||
- Deviates from official Echo-TTS behavior
|
||||
|
||||
**Note about `[S2]`:**
|
||||
Echo-TTS does NOT have multi-speaker support. `[S1]` and `[S2]` are cosmetic WhisperD transcription format tags that don't change the voice. All audio uses the single reference voice regardless of these tags.
|
||||
|
||||
**Fix suggested:**
|
||||
|
||||
```python
|
||||
@staticmethod
|
||||
def _normalize_prompt_text(text: str) -> str:
|
||||
if not text:
|
||||
return "[S1]"
|
||||
|
||||
cleaned = text.strip()
|
||||
|
||||
# Don't add [S1] if S1/S2 already present or text starts with bracket
|
||||
if 'S1' in cleaned or 'S2' in cleaned or cleaned.startswith('[') or cleaned.startswith('('):
|
||||
return cleaned
|
||||
|
||||
return f"[S1] {cleaned}".strip()
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 8. 📝 **Force Speaker KV Warning**
|
||||
|
||||
**Severity:** MINOR (Documentation)
|
||||
**Location:** `nodes/engines/echo_tts_engine_node.py:154`
|
||||
|
||||
**Issue:**
|
||||
|
||||
```python
|
||||
"tooltip": "... ⚠️ Not compatible with pause tags ([pause:...])."
|
||||
```
|
||||
|
||||
**Observation:**
|
||||
|
||||
- Warning about pause tag incompatibility is present
|
||||
- BUT pause tags aren't implemented yet anyway
|
||||
- Warning will become relevant once pause tags are added
|
||||
|
||||
**Action:** Keep warning, but verify compatibility once pause tags are implemented
|
||||
|
||||
---
|
||||
|
||||
## Positive Observations ✅
|
||||
|
||||
### What Was Done Well
|
||||
|
||||
1. **✅ Unified Model Interface Integration**
|
||||
|
||||
- Correctly uses `unified_model_interface.load_model()` (line 132)
|
||||
- Follows factory pattern with `register_echo_tts_factory()`
|
||||
- Integrated with ComfyUI model management
|
||||
|
||||
2. **✅ Proper Node Registration**
|
||||
|
||||
- Registered in `nodes.py` (lines 119-120, 475-476)
|
||||
- Correct category: `"TTS Audio Suite/Engines"`
|
||||
|
||||
3. **✅ Device Management**
|
||||
|
||||
- Uses `resolve_torch_device()` utility
|
||||
- Graceful fallback from CUDA to CPU with warnings
|
||||
|
||||
4. **✅ Character Switching Support**
|
||||
|
||||
- Implemented in unified TTS text node (lines 1128-1200)
|
||||
- Uses character parser correctly
|
||||
|
||||
5. **✅ Extensive Segment Parameter Support**
|
||||
|
||||
- Registered in `utils/text/segment_parameters.py` with **12 parameters**
|
||||
- Supports: `seed`, `num_steps`, `cfg_scale_text`, `cfg_scale_speaker`, `cfg_min_t`, `cfg_max_t`, `truncation_factor`, `rescale_k`, `rescale_sigma`, `speaker_kv_scale`, `speaker_kv_max_layers`, `speaker_kv_min_t`, `sequence_length`
|
||||
- Most comprehensive segment parameter support in the project
|
||||
- Allows per-character control: `[Alice|seed:42|cfg_scale_text:5.0]`
|
||||
|
||||
6. **✅ Chunking System**
|
||||
|
||||
- Uses `ImprovedChatterBoxChunker` (line 331)
|
||||
- Uses `ChunkTimingHelper` for combination (line 341)
|
||||
|
||||
7. **✅ SRT Processor Structure**
|
||||
|
||||
- Separate SRT processor file (good separation of concerns)
|
||||
- Uses `AudioAssemblyEngine` for timing assembly
|
||||
- Handles overlap detection with smart fallback
|
||||
|
||||
8. **✅ Reference Audio Processing**
|
||||
|
||||
- Robust `_prepare_reference_audio()` method (lines 140-186)
|
||||
- Handles multiple input formats (dict, string, tensor)
|
||||
- Resampling to Echo-TTS native 44100 Hz
|
||||
- Normalization and 5-minute limit
|
||||
|
||||
9. **✅ Seed Reproducibility**
|
||||
|
||||
- Sets both `torch.manual_seed()` and `torch.cuda.manual_seed()` (lines 202-205)
|
||||
|
||||
---
|
||||
|
||||
## Comparison to Implementation Guide
|
||||
|
||||
### Guide Checklist
|
||||
|
||||
| Requirement | Status | Notes |
|
||||
| --------------------------- | ---------- | --------------------------- |
|
||||
| **Phase 1: Foundation** | | |
|
||||
| Core engine implementation | ✅ Pass | Via unified interface |
|
||||
| Model loading & downloading | ✅ Pass | Downloader not checked yet |
|
||||
| Engine configuration node | ✅ Pass | Clean implementation |
|
||||
| Character switching | ✅ Pass | In unified node |
|
||||
| Pause tag support | ✅ Pass | d |
|
||||
| Caching system | ❌ **FAIL** | Not implemented |
|
||||
| VRAM management | ✅ Pass | Via unified interface |
|
||||
| Interrupt handling | ❌ **FAIL** | Not implemented |
|
||||
| | | |
|
||||
| **Phase 2: SRT Support** | | |
|
||||
| SRT processor | ✅ Pass | Implemented |
|
||||
| Timing & assembly | ✅ Pass | Uses AudioAssemblyEngine |
|
||||
| Character switching in SRT | ✅ Pass | |
|
||||
| | | |
|
||||
| **Unified Systems** | | |
|
||||
| UnifiedModelInterface | ✅ Pass | Correctly used |
|
||||
| ComfyUIModelWrapper | ✅ Pass | Via unified interface |
|
||||
| UnifiedCacheManager | ❌ **FAIL** | Not used |
|
||||
| CharacterParser | ✅ Pass | Used in unified node |
|
||||
| PauseTagProcessor | ✅ Pass | d |
|
||||
| LanguageMapper | ⚠️ N/A | Echo-TTS uses language list |
|
||||
|
||||
---
|
||||
|
||||
|
||||
|
||||
---
|
||||
|
||||
## Conclusion
|
||||
|
||||
The Echo-TTS integration has **significant architectural problems** that must be addressed before merge:
|
||||
|
||||
### Critical Issues (Must Fix):
|
||||
|
||||
1. **Non-standard architecture** - 180+ lines of orchestration logic embedded in unified node instead of processor
|
||||
2. **Missing audio caching** - Performance issue, all other engines have this
|
||||
3. **Missing interrupt handling** - User experience issue, cannot cancel SRT generations
|
||||
|
||||
### Important Issues (Should Fix):
|
||||
|
||||
4. **Languages parameter** - Non-functional, should be removed (English-only model)
|
||||
5. **Progress bars** - No user feedback during generation
|
||||
|
||||
**Recommendation:** This PR needs **major refactoring** before merge. The architectural deviation (embedding logic in unified node) is particularly problematic as it:
|
||||
|
||||
- Creates maintenance burden (180 lines of code that should be in processor)
|
||||
- Breaks code reusability (logic cannot be used by other nodes)
|
||||
- Deviates from established patterns (Qwen3, Step EditX, IndexTTS all use processor pattern)
|
||||
|
||||
The PR author did good work on the Echo-TTS integration itself, but the project architecture needs to be followed.
|
||||
|
||||
---
|
||||
|
||||
## Future Work (Post-Merge)
|
||||
|
||||
### Engine Bundling
|
||||
|
||||
**Current Status:** Echo-TTS is installed via `pip install echo-tts` (not bundled).
|
||||
|
||||
**Recommendation for future:** Consider bundling Echo-TTS like other engines (Step Audio EditX, F5-TTS) to enable:
|
||||
|
||||
1. **Custom modifications** - Add it/s progress indicators, custom features
|
||||
2. **Dependency isolation** - Avoid conflicts with user's environment
|
||||
3. **Version stability** - Lock to specific tested version
|
||||
4. **Patches and fixes** - Apply project-specific patches without waiting for upstream
|
||||
|
||||
**Rationale for deferring:**
|
||||
- Echo-TTS is simple with minimal dependencies (no immediate conflicts expected)
|
||||
- PR scope is already large with architecture refactoring
|
||||
- Can be added in follow-up PR after real-world testing
|
||||
- Easy migration path (create `engines/echo_tts/bundled/` when needed)
|
||||
|
||||
**When to bundle:**
|
||||
- If dependency conflicts arise in practice
|
||||
- If we need to add custom features (progress bars, patches)
|
||||
- If upstream makes breaking changes
|
||||
|
||||
**Action:** Track as future enhancement, not blocking for initial merge.
|
||||
|
||||
---
|
||||
|
||||
## Code Examples to Share with PR Author
|
||||
|
||||
### 1. Cache Integration Example
|
||||
|
||||
```python
|
||||
# In __init__:
|
||||
from utils.audio.cache import get_audio_cache
|
||||
self.audio_cache = get_audio_cache()
|
||||
|
||||
# In _generate_audio_for_text():
|
||||
# BEFORE generation:
|
||||
from utils.audio.audio_hash import generate_stable_audio_component
|
||||
audio_component = generate_stable_audio_component(reference_audio=ref_audio)
|
||||
cache_key = self.audio_cache.generate_cache_key(
|
||||
'echo_tts',
|
||||
text=text,
|
||||
audio_component=audio_component,
|
||||
ref_text=ref_text,
|
||||
num_steps=params['num_steps'],
|
||||
cfg_scale_text=params['cfg_scale_text'],
|
||||
cfg_scale_speaker=params['cfg_scale_speaker'],
|
||||
seed=seed,
|
||||
# ... all params that affect output ...
|
||||
)
|
||||
|
||||
cached_audio = self.audio_cache.get_cached_audio(cache_key)
|
||||
if cached_audio:
|
||||
print(f"💾 Using cached Echo-TTS audio: '{text[:30]}...'")
|
||||
return cached_audio[0], self.SAMPLE_RATE
|
||||
|
||||
# AFTER generation:
|
||||
duration = self.audio_cache._calculate_duration(audio_tensor, 'echo_tts')
|
||||
self.audio_cache.cache_audio(cache_key, audio_tensor, duration)
|
||||
```
|
||||
|
||||
###
|
||||
|
||||
### 3. Interrupt Handling Example
|
||||
|
||||
```python
|
||||
# In echo_tts_srt_processor.py, add import:
|
||||
import comfy.model_management as model_management
|
||||
|
||||
# In process_srt_content(), in main loop:
|
||||
for i, subtitle in enumerate(subtitles):
|
||||
# Check for interrupt
|
||||
if model_management.interrupt_processing:
|
||||
raise InterruptedError(
|
||||
f"Echo-TTS SRT generation interrupted at subtitle {i+1}/{len(subtitles)}"
|
||||
)
|
||||
|
||||
# ... existing generation code ...
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
**Review completed:** 2026-02-05
|
||||
**Next step:** Share findings with PR author and request changes
|
||||
@@ -1,426 +0,0 @@
|
||||
# Higgs Audio 2 Integration Specification
|
||||
|
||||
**TTS Audio Suite v4.4.0 - Technical Documentation**
|
||||
|
||||
*Complete technical specification for Higgs Audio 2 engine integration with unified architecture*
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
The Higgs Audio 2 integration adds a new high-quality TTS engine to the TTS Audio Suite with advanced voice cloning capabilities. This integration follows the established unified architecture patterns while introducing minimal dependencies and maintaining full backward compatibility.
|
||||
|
||||
### Key Features
|
||||
|
||||
- **High-Quality TTS Generation**: 3B parameter model with advanced audio synthesis
|
||||
- **Voice Cloning**: Clone any voice from 30+ seconds of reference audio
|
||||
- **Multi-Language Support**: English, Chinese dialects, and extensible to other languages
|
||||
- **Unified Architecture**: Seamless integration with existing TTS Text and TTS SRT nodes
|
||||
- **Chunking Support**: Handle unlimited text length with intelligent chunking
|
||||
- **Preset Voices**: 8 built-in voice presets for instant use
|
||||
- **Caching System**: Integrated with existing audio cache for performance
|
||||
- **Minimal Dependencies**: Only 2 essential new dependencies added
|
||||
|
||||
---
|
||||
|
||||
## Architecture Design
|
||||
|
||||
### Modular Components (500-600 lines each)
|
||||
|
||||
The Higgs Audio 2 integration follows the established modular architecture principle:
|
||||
|
||||
```
|
||||
engines/higgs_audio/
|
||||
├── higgs_audio.py # Main engine wrapper (688 lines)
|
||||
├── higgs_audio_downloader.py # Model downloader (357 lines)
|
||||
├── boson_multimodal/ # Complete reference implementation
|
||||
└── __init__.py
|
||||
|
||||
engines/adapters/
|
||||
└── higgs_audio_adapter.py # Unified adapter (363 lines)
|
||||
|
||||
nodes/engines/
|
||||
└── higgs_audio_engine_node.py # Configuration node (153 lines)
|
||||
|
||||
voices_examples/higgs_audio/
|
||||
├── config.json # Voice preset configuration
|
||||
├── belinda.wav # Voice presets (8 total)
|
||||
├── en_woman.wav
|
||||
└── ...
|
||||
```
|
||||
|
||||
### Integration Points
|
||||
|
||||
1. **Engine Configuration**: `HiggsAudioEngineNode` provides user interface
|
||||
2. **Engine Wrapper**: `HiggsAudioEngine` handles model loading and generation
|
||||
3. **Unified Adapter**: `HiggsAudioEngineAdapter` bridges to unified nodes
|
||||
4. **Unified Nodes**: Existing `TTS Text` and `TTS SRT` nodes support higgs_audio
|
||||
5. **Model Management**: Integrated with `unified_downloader` system
|
||||
6. **Caching**: Uses existing `AudioCache` with custom key generator
|
||||
|
||||
---
|
||||
|
||||
## Technical Implementation Details
|
||||
|
||||
### Dependencies
|
||||
|
||||
**Minimal Essential Dependencies Added:**
|
||||
- `descript-audio-codec`: Audio codec for Higgs Audio tokenizer
|
||||
- `vector_quantize_pytorch`: Vector quantization for audio processing
|
||||
|
||||
**Avoided Dependencies** (handled internally or replaced):
|
||||
- `dacite` → Direct dict handling
|
||||
- `boto3`, `s3fs` → Models from HuggingFace, not S3
|
||||
- `json_repair` → Python's standard `json` module
|
||||
- `pydantic` → Standard Python dataclasses
|
||||
- `pydub` → Existing `librosa`/`torchaudio`
|
||||
- `jieba`, `langid` → Optional Chinese text processing
|
||||
- `accelerate` → Optional optimization
|
||||
|
||||
### Text Chunking Strategy
|
||||
|
||||
**Unified Chunking System:**
|
||||
```python
|
||||
# Uses superior ImprovedChatterBoxChunker instead of custom implementation
|
||||
max_chars = tokens_to_chars(max_tokens_per_chunk) # Convert token limit
|
||||
chunks = ImprovedChatterBoxChunker.split_into_chunks(text, max_chars)
|
||||
```
|
||||
|
||||
**Benefits:**
|
||||
- Character-based chunking (more predictable than token approximation)
|
||||
- Better sentence boundary detection
|
||||
- Consistent across all engines
|
||||
- Overlap support for context preservation
|
||||
|
||||
### Voice Cloning Implementation
|
||||
|
||||
**Voice Priority System:**
|
||||
1. **`auto`**: Use preset if selected, otherwise reference audio
|
||||
2. **`preset_dropdown`**: Prefer preset, fallback to reference
|
||||
3. **`reference_input`**: Prefer reference, fallback to preset
|
||||
4. **`force_preset`**: Only use preset, ignore reference
|
||||
|
||||
**Voice Processing:**
|
||||
```python
|
||||
# Convert ComfyUI audio to base64 for boson_multimodal
|
||||
audio_base64 = self._audio_to_base64(reference_audio)
|
||||
audio_content = AudioContent(raw_audio=audio_base64, audio_url="")
|
||||
messages.append(Message(role="assistant", content=[audio_content]))
|
||||
```
|
||||
|
||||
### Model Management
|
||||
|
||||
**Organized Download Structure:**
|
||||
```
|
||||
ComfyUI/models/TTS/HiggsAudio/
|
||||
└── higgs-audio-v2-3B/
|
||||
├── generation/
|
||||
│ ├── config.json
|
||||
│ ├── model.safetensors
|
||||
│ └── generation_config.json
|
||||
└── tokenizer/
|
||||
├── config.json
|
||||
└── pytorch_model.bin
|
||||
```
|
||||
|
||||
**Auto-Download System:**
|
||||
- Integrated with `unified_downloader`
|
||||
- Direct downloads (no HuggingFace cache duplication)
|
||||
- Organized folder structure
|
||||
- Model reuse across sessions
|
||||
|
||||
### Caching System
|
||||
|
||||
**HiggsAudioCacheKeyGenerator:**
|
||||
```python
|
||||
cache_data = {
|
||||
'text': text,
|
||||
'voice_preset': voice_preset,
|
||||
'model_path': model_path,
|
||||
'tokenizer_path': tokenizer_path,
|
||||
'temperature': temperature,
|
||||
'top_p': top_p,
|
||||
'top_k': top_k,
|
||||
'reference_audio_hash': audio_hash, # If present
|
||||
'engine': 'higgs_audio'
|
||||
}
|
||||
```
|
||||
|
||||
**Cache Integration:**
|
||||
- Registered with `AudioCache.register_cache_key_generator()`
|
||||
- Automatic cache key generation
|
||||
- Reference audio hashing for consistency
|
||||
- Shared with existing cache system
|
||||
|
||||
---
|
||||
|
||||
## Node Integration Details
|
||||
|
||||
### Engine Configuration Node
|
||||
|
||||
**`HiggsAudioEngineNode` Parameters:**
|
||||
- `model`: Model selection (higgs-audio-v2-3B, local models)
|
||||
- `device`: Device selection (auto, cuda, cpu)
|
||||
- `voice_preset`: Voice preset selection (8 presets + voice_clone)
|
||||
- `audio_priority`: Voice source priority logic
|
||||
- `system_prompt`: Generation guidance prompt
|
||||
- `temperature`: Sampling temperature (0.0-2.0)
|
||||
- `top_p`: Nucleus sampling (0.1-1.0)
|
||||
- `top_k`: Top-k sampling (-1 to 100)
|
||||
- `max_new_tokens`: Generation limit (128-4096)
|
||||
- `max_tokens_per_chunk`: Chunking limit (100-500)
|
||||
- `silence_between_chunks_ms`: Inter-chunk silence (0-2000ms)
|
||||
- `enable_chunking`: Chunking toggle
|
||||
- `enable_cache`: Cache toggle
|
||||
- `seed`: Reproducibility seed
|
||||
|
||||
### Unified Node Integration
|
||||
|
||||
**TTS Text Node Integration:**
|
||||
```python
|
||||
elif engine_type == "higgs_audio":
|
||||
# Create wrapper with adapter
|
||||
result = engine_instance.generate_tts_audio(
|
||||
text=text,
|
||||
char_audio=reference_audio_dict,
|
||||
char_text=reference_text,
|
||||
character="narrator",
|
||||
**config_parameters
|
||||
)
|
||||
```
|
||||
|
||||
**TTS SRT Node Integration:**
|
||||
```python
|
||||
elif engine_type == "higgs_audio":
|
||||
# Parse SRT and generate per segment
|
||||
srt_segments = parse_srt(srt_content)
|
||||
for segment in srt_segments:
|
||||
segment_audio = engine_instance.generate_srt_audio(...)
|
||||
audio_segments.append(segment_audio)
|
||||
# Combine with timing logic
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Voice Preset System
|
||||
|
||||
### Built-in Voice Presets
|
||||
|
||||
1. **belinda**: Female, clear English pronunciation
|
||||
2. **en_woman**: Female, natural English voice
|
||||
3. **en_man**: Male, clear English voice
|
||||
4. **chadwick**: Male, distinctive character voice
|
||||
5. **vex**: Female, character voice with edge
|
||||
6. **mabel**: Female, warm conversational voice
|
||||
7. **broom_salesman**: Male, character voice
|
||||
8. **zh_man_sichuan**: Male, Sichuan Chinese dialect
|
||||
|
||||
### Voice Configuration
|
||||
|
||||
**`voices_examples/higgs_audio/config.json`:**
|
||||
```json
|
||||
{
|
||||
"belinda": {
|
||||
"transcript": "This is Belinda's voice with clear pronunciation and natural intonation."
|
||||
},
|
||||
"en_woman": {
|
||||
"transcript": "A natural English woman's voice for general purpose use."
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Voice Discovery:**
|
||||
- Automatic preset loading from `config.json`
|
||||
- Audio file validation (`.wav` format)
|
||||
- Integration with Character Voices system
|
||||
- Custom reference audio support
|
||||
|
||||
---
|
||||
|
||||
## Performance Characteristics
|
||||
|
||||
### Model Specifications
|
||||
|
||||
- **Model Size**: ~3B parameters
|
||||
- **Memory Requirements**: 8GB+ VRAM recommended
|
||||
- **Sample Rate**: 24,000 Hz
|
||||
- **Generation Speed**: Similar to F5-TTS with chunking
|
||||
- **Voice Quality**: High-fidelity with natural prosody
|
||||
|
||||
### Chunking Performance
|
||||
|
||||
**Recommended Settings:**
|
||||
- `max_tokens_per_chunk`: 225 tokens (~787 characters)
|
||||
- `silence_between_chunks_ms`: 100ms
|
||||
- `enable_chunking`: True for texts >225 tokens
|
||||
|
||||
**Benefits:**
|
||||
- Unlimited text length support
|
||||
- Consistent voice quality across chunks
|
||||
- Natural pauses between segments
|
||||
- Memory efficiency for long texts
|
||||
|
||||
### Caching Benefits
|
||||
|
||||
- **Cache Hits**: Instant audio retrieval
|
||||
- **Cache Misses**: Full generation with caching
|
||||
- **Storage**: Efficient tensor storage
|
||||
- **Cleanup**: Automatic cache management
|
||||
|
||||
---
|
||||
|
||||
## Error Handling and Validation
|
||||
|
||||
### Parameter Validation
|
||||
|
||||
**HiggsAudioEngineAdapter.validate_parameters():**
|
||||
```python
|
||||
# Temperature validation
|
||||
temperature = max(0.0, min(2.0, float(temperature)))
|
||||
|
||||
# Top-p validation
|
||||
top_p = max(0.1, min(1.0, float(top_p)))
|
||||
|
||||
# Voice preset validation
|
||||
if voice_preset not in available_presets:
|
||||
validated["voice_preset"] = "voice_clone"
|
||||
```
|
||||
|
||||
### Error Recovery
|
||||
|
||||
1. **Model Loading Failures**: Clear error messages with resolution steps
|
||||
2. **Audio Processing Errors**: Graceful fallback to basic TTS
|
||||
3. **Reference Audio Issues**: Automatic fallback to preset voices
|
||||
4. **Memory Issues**: Chunking and cache cleanup
|
||||
5. **Network Issues**: Retry logic for model downloads
|
||||
|
||||
### Logging and Diagnostics
|
||||
|
||||
```python
|
||||
print(f"🚀 Loading Higgs Audio 2 engine: {model_path}")
|
||||
print(f"✅ Higgs Audio 2 engine loaded successfully")
|
||||
print(f"🎤 Generating Higgs Audio for {len(chunks)} chunk(s)")
|
||||
print(f" ✅ Generated audio shape: {audio_tensor.shape}")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Compatibility and Migration
|
||||
|
||||
### Backward Compatibility
|
||||
|
||||
- **Existing Workflows**: No changes required
|
||||
- **Character Voices**: Full integration with existing system
|
||||
- **Cache System**: Shared cache across all engines
|
||||
- **Audio Processing**: Standard ComfyUI audio format
|
||||
|
||||
### Forward Compatibility
|
||||
|
||||
- **New Engines**: Adapter pattern supports easy addition
|
||||
- **Feature Extensions**: Modular design allows enhancement
|
||||
- **Model Updates**: Download system supports model versioning
|
||||
- **Voice Expansion**: Preset system supports additional voices
|
||||
|
||||
### Migration Notes
|
||||
|
||||
**From Direct Higgs Audio Nodes:**
|
||||
- Use Higgs Audio Engine + TTS Text instead of separate nodes
|
||||
- Voice presets automatically detected
|
||||
- All parameters preserved with validation
|
||||
|
||||
**From Other TTS Engines:**
|
||||
- Same unified interface
|
||||
- Voice switching supported
|
||||
- Cache keys remain unique per engine
|
||||
|
||||
---
|
||||
|
||||
## Testing and Validation
|
||||
|
||||
### Manual Testing Approach
|
||||
|
||||
1. **Model Loading**: Verify automatic download and caching
|
||||
2. **Voice Presets**: Test all 8 built-in presets
|
||||
3. **Voice Cloning**: Test with various reference audio lengths
|
||||
4. **Text Chunking**: Test with short and very long texts
|
||||
5. **Audio Quality**: Compare with reference implementation
|
||||
6. **Error Handling**: Test edge cases and failure modes
|
||||
|
||||
### Integration Testing
|
||||
|
||||
1. **Unified Nodes**: Verify seamless engine switching
|
||||
2. **Character Voices**: Test with Character Voices node
|
||||
3. **SRT Processing**: Test with subtitle timing
|
||||
4. **Cache System**: Verify cache hits and misses
|
||||
5. **Memory Management**: Test cleanup and resource usage
|
||||
|
||||
### Performance Benchmarks
|
||||
|
||||
**Text Processing:**
|
||||
- Short text (50 chars): ~2-3 seconds
|
||||
- Medium text (500 chars): ~5-8 seconds
|
||||
- Long text (2000+ chars): Chunked processing
|
||||
|
||||
**Memory Usage:**
|
||||
- Base model: ~6GB VRAM
|
||||
- Peak generation: ~8GB VRAM
|
||||
- Cache overhead: ~100MB per hour of audio
|
||||
|
||||
---
|
||||
|
||||
## Future Enhancements
|
||||
|
||||
### Planned Features
|
||||
|
||||
1. **Streaming Support**: Real-time generation for long texts
|
||||
2. **Voice Training**: Custom voice preset creation
|
||||
3. **Multi-Speaker**: Support for conversation-style generation
|
||||
4. **Advanced Editing**: Word-level editing like F5-TTS
|
||||
5. **Emotion Control**: Emotional tone parameters
|
||||
|
||||
### Extension Points
|
||||
|
||||
1. **New Voice Presets**: Easy addition through config.json
|
||||
2. **Language Models**: Support for additional languages
|
||||
3. **Audio Effects**: Post-processing pipeline integration
|
||||
4. **Quality Settings**: Speed vs quality trade-offs
|
||||
5. **Batch Processing**: Multiple texts with different voices
|
||||
|
||||
---
|
||||
|
||||
## Credits and Acknowledgments
|
||||
|
||||
### Original Implementation
|
||||
- **HiggsAudio Team**: Original Higgs Audio v2 model and boson_multimodal implementation
|
||||
- **ShmuelRonen**: ComfyUI-HiggsAudio_2 reference implementation
|
||||
- **Anthropic**: Claude Code development assistance
|
||||
|
||||
### Integration Team
|
||||
- **TTS Audio Suite**: Unified architecture integration
|
||||
- **Community Contributors**: Voice presets and testing
|
||||
|
||||
### Third-Party Libraries
|
||||
- **descript-audio-codec**: Audio tokenization
|
||||
- **vector_quantize_pytorch**: Vector quantization
|
||||
- **PyTorch**: Deep learning framework
|
||||
- **torchaudio**: Audio processing
|
||||
|
||||
---
|
||||
|
||||
## Conclusion
|
||||
|
||||
The Higgs Audio 2 integration successfully adds high-quality voice cloning capabilities to the TTS Audio Suite while maintaining the unified architecture principles. The implementation:
|
||||
|
||||
✅ **Follows Established Patterns**: Uses existing adapter and unified node systems
|
||||
✅ **Minimal Dependencies**: Only 2 essential new dependencies
|
||||
✅ **Superior Chunking**: Uses unified chunking system for consistency
|
||||
✅ **Full Integration**: Works with all existing unified nodes
|
||||
✅ **Voice Cloning**: Supports both presets and custom reference audio
|
||||
✅ **Performance**: Efficient caching and model management
|
||||
✅ **Backward Compatible**: No breaking changes to existing workflows
|
||||
|
||||
This integration demonstrates the power of the unified architecture, allowing complex new engines to be added with minimal disruption while providing users with a consistent, powerful interface for high-quality TTS generation.
|
||||
|
||||
---
|
||||
|
||||
*Generated for TTS Audio Suite v4.4.0 - Higgs Audio 2 Integration*
|
||||
@@ -1,293 +0,0 @@
|
||||
# Higgs Audio Stateless Wrapper Implementation - Complete Journey
|
||||
|
||||
## **Problem Statement**
|
||||
Higgs Audio 2 could not integrate with ComfyUI's native model management system due to CUDA Graph memory corruption during model unloading. The goal was to apply the same stateless wrapper pattern used for ChatterBox streaming to enable safe model unloading while preserving performance.
|
||||
|
||||
## **Original Performance Baseline**
|
||||
- **50 it/s (tokens/second)** generation speed
|
||||
- **CUDA Graph optimization** enabled
|
||||
- **11GB VRAM usage**
|
||||
- **Voice cloning functional**
|
||||
- **No safe model unloading** - crashes on "Unload Models"
|
||||
|
||||
---
|
||||
|
||||
## **Implementation Attempts & Results**
|
||||
|
||||
### **Attempt 1: Basic Stateless Wrapper Creation**
|
||||
**Approach**: Created StatelessHiggsAudioWrapper following ChatterBox pattern
|
||||
- Created `engines/higgs_audio/stateless_wrapper.py`
|
||||
- Implemented tensor isolation with `.detach().clone()`
|
||||
- Wrapped HiggsAudioEngine with stateless methods
|
||||
|
||||
**Result**: ❌ Circular dependency causing infinite loop
|
||||
**Fix**: Modified unified interface to create HiggsAudioServeEngine directly
|
||||
|
||||
### **Attempt 2: Direct Serve Engine Integration**
|
||||
**Approach**: Bypass HiggsAudioEngine.initialize_engine() to avoid circular calls
|
||||
- Modified `utils/models/unified_model_interface.py`
|
||||
- Direct HiggsAudioServeEngine creation
|
||||
- Manual component setup
|
||||
|
||||
**Result**: ❌ TypeError: 'generator' object is not callable
|
||||
**Fix**: Changed `@property` decorator to regular method for `parameters()`
|
||||
|
||||
### **Attempt 3: Device Mismatch Fixes - Round 1**
|
||||
**Approach**: Fix "Input type torch.FloatTensor and weight type torch.cuda.FloatTensor" errors
|
||||
- Added device enforcement in serve engine input preparation
|
||||
- Modified audio tokenizer to ensure tensors on correct device
|
||||
|
||||
**Result**: ❌ Still device mismatches in semantic model
|
||||
**Progress**: Error moved from one component to another (good sign)
|
||||
|
||||
### **Attempt 4: Deep Device Consistency**
|
||||
**Approach**: Comprehensive device fixing throughout audio pipeline
|
||||
- Fixed `get_regress_target()` method in audio tokenizer
|
||||
- Added `next(model.parameters()).device` for accurate device detection
|
||||
- Fixed encoder device handling in `_xcodec_encode()`
|
||||
|
||||
**Result**: ❌ Device mismatch moved to KV cache operations
|
||||
**Lesson**: Device issues were systemic, requiring comprehensive fixes
|
||||
|
||||
### **Attempt 5: KV Cache Device Synchronization**
|
||||
**Approach**: Fix cache_position tensor CPU/CUDA mismatch in transformers library
|
||||
- Added comprehensive KV cache device enforcement
|
||||
- Implemented cache tensor movement to model device
|
||||
- Enhanced input tensor device verification
|
||||
|
||||
**Result**: ❌ Persistent device mismatch in cache_utils.py:1220
|
||||
**Issue**: Deep transformers library issue with cache_position tensor creation
|
||||
|
||||
### **Attempt 6: Disable KV Cache (Temporary)**
|
||||
**Approach**: Remove StaticCache to bypass device issues
|
||||
- Set `use_cache=False`
|
||||
- Removed `past_key_values_buckets` parameter
|
||||
|
||||
**Result**: ✅ No crashes BUT ❌ Performance degraded to ~11 it/s
|
||||
**Lesson**: StaticCache is critical for performance
|
||||
|
||||
### **Attempt 7: Smart KV Cache Recreation**
|
||||
**Approach**: Recreate caches when device mismatch detected
|
||||
- Device mismatch detection in `_prepare_kv_caches()`
|
||||
- Dynamic cache recreation with correct device
|
||||
- Restore StaticCache functionality
|
||||
|
||||
**Result**: ❌ Still ~12 it/s performance with occasional device issues
|
||||
|
||||
### **Attempt 8: Performance Optimization - Remove Tensor Cloning**
|
||||
**Approach**: Optimize stateless wrapper by removing expensive operations
|
||||
- Changed `.detach().clone()` to `.detach()` only
|
||||
- Removed redundant device operations
|
||||
- Added performance caching
|
||||
|
||||
**Result**: ❌ Still ~12 it/s performance
|
||||
**Discovery**: Cloning wasn't the main bottleneck
|
||||
|
||||
### **Attempt 9: Bypass Stateless Wrapper Test**
|
||||
**Approach**: Temporarily disable stateless wrapper to isolate performance issue
|
||||
- Modified adapter to use original `generate()` method
|
||||
- Test if wrapper was causing slowdown
|
||||
|
||||
**Result**: ❌ Still ~12 it/s performance even without wrapper
|
||||
**Conclusion**: Performance issue was deeper in the loading approach
|
||||
|
||||
### **Attempt 10: Root Cause Discovery - CPU→GPU Loading**
|
||||
**Approach**: Identified that CPU-first loading was breaking CUDA Graph optimization
|
||||
- Original approach: Load directly on GPU
|
||||
- Our approach: Load on CPU → Move to GPU
|
||||
- CUDA Graphs are compiled for specific device configurations
|
||||
|
||||
**Root Cause Found**: CPU→GPU loading breaks CUDA Graph optimization
|
||||
|
||||
### **Attempt 11: Direct GPU Loading**
|
||||
**Approach**: Restore original loading pattern
|
||||
- Modified `unified_model_interface.py` to load directly on target device
|
||||
- Removed CPU→GPU transfer code
|
||||
- Fixed data collator device mismatch (`torch.full()` device parameter)
|
||||
|
||||
**Result**: ✅ **54.92 tok/s performance restored!** BUT ❌ Crashes on model unload
|
||||
**Issue**: CUDA Graph warnings → Process termination
|
||||
|
||||
### **Attempt 12: Comprehensive CUDA Graph Cleanup**
|
||||
**Approach**: Multi-layer CUDA Graph cleanup with official PyTorch APIs
|
||||
- Disabled CUDA Graph compilation during cleanup
|
||||
- Used `torch.cuda.reset_accumulated_memory_stats()`
|
||||
- Multiple CUDA synchronization passes
|
||||
- Complete KV cache tensor replacement
|
||||
|
||||
**Result**: ❌ Still crashes with CUDA warnings
|
||||
**Issue**: CUDA Graph allocations persist at PyTorch level
|
||||
|
||||
### **Attempt 13: Complete Engine Destruction**
|
||||
**Approach**: Nuclear option - destroy and recreate entire serve engine
|
||||
- Store model paths for recreation
|
||||
- Destroy all engine components (model, tokenizer, caches)
|
||||
- Set engine to None to force complete recreation
|
||||
- Mark for recreation with stored paths
|
||||
|
||||
**Result**: ❌ Still crashes with identical CUDA warnings
|
||||
**Issue**: CUDA Graph captures happen at memory allocator level, persist even after object destruction
|
||||
|
||||
### **Attempt 14: Surgical CUDA Graph Preservation (FAILED)**
|
||||
**Approach**: Keep CUDA Graphs but clean them surgically
|
||||
- Exit inference mode for tensor modifications
|
||||
- Replace cache tensors with CPU versions
|
||||
- Use official PyTorch CUDA Graph management APIs
|
||||
|
||||
**Result**: ❌ Cache structure errors and persistent CUDA warnings
|
||||
**Issue**: Cache structure is complex (lists), CUDA Graph captures are too deep
|
||||
|
||||
### **Attempt 15: PyTorch 2.2+ Modern CUDA Graph APIs (FAILED)**
|
||||
**Approach**: Research and implement modern PyTorch CUDA Graph cleanup APIs
|
||||
- Researched PyTorch 2.2, 2.3, 2.4 improvements
|
||||
- Implemented torch.cuda.graph() context manager approach
|
||||
- Added memory pool context managers and proper cleanup sequences
|
||||
|
||||
**Result**: ❌ Context manager API errors and persistent CUDA warnings
|
||||
**Issue**: Modern APIs require different parameters, CUDA Graph captures persist at allocator level
|
||||
|
||||
### **Attempt 16: CUDA Graph Toggle Implementation (PARTIAL SUCCESS)**
|
||||
**Approach**: Accept fundamental incompatibility and implement user choice toggle
|
||||
- Added `enable_cuda_graphs` boolean parameter to Higgs Audio Engine node
|
||||
- Implemented conditional loading: StaticCache vs DynamicCache
|
||||
- Added parameter passing chain: Node → Config → Adapter → Engine → Factory
|
||||
- Conditional memory unloading protection based on toggle setting
|
||||
|
||||
**Result**: ✅ **Toggle UI working** BUT ❌ **CUDA Graphs still created in Memory Safe mode**
|
||||
**Progress**:
|
||||
- ✅ Parameter passing chain complete
|
||||
- ✅ "Memory Safe (CUDA Graphs OFF)" vs "High Performance (CUDA Graphs ON)" modes working
|
||||
- ✅ Conditional cleanup logic implemented
|
||||
- ✅ Memory unloading protection working
|
||||
- ❌ **Critical Issue**: StaticCache still created even when CUDA Graphs disabled
|
||||
- ❌ **Still crashes**: CUDA allocation warnings persist in "safe" mode
|
||||
|
||||
**Technical Discovery**: Environment variables set too late - serve engine constructor creates StaticCache before disable flags take effect
|
||||
|
||||
---
|
||||
|
||||
## **CONCLUSION: Fundamental Architectural Challenge**
|
||||
|
||||
After **16 different implementation attempts**, the core issue remains unsolved:
|
||||
|
||||
### **Root Cause Analysis**:
|
||||
1. **CUDA Graphs are compiled at the PyTorch C++ level** - cannot be disabled via Python APIs alone
|
||||
2. **StaticCache constructor immediately allocates CUDA Graph memory** - happens before environment variables take effect
|
||||
3. **Memory allocator captures references during first generation** - these become "captured allocations"
|
||||
4. **ComfyUI's model unloading tries to free captured allocations** - triggers fatal CUDA warnings
|
||||
5. **Even DynamicCache approach fails** - underlying model still creates CUDA Graph optimizations
|
||||
|
||||
### **Current Status: Toggle Implementation (Partial Success)**
|
||||
|
||||
**✅ Successfully Implemented:**
|
||||
- **User toggle UI** with clear performance/safety tradeoffs
|
||||
- **Complete parameter passing chain** from node to factory
|
||||
- **Conditional cleanup logic** that prevents crashes when CUDA Graphs enabled
|
||||
- **Memory unloading protection** with clear user messaging
|
||||
- **Mode detection and appropriate warnings**
|
||||
|
||||
**❌ Still Failing:**
|
||||
- **StaticCache still created** even in "Memory Safe" mode
|
||||
- **CUDA Graph allocations still captured** during generation
|
||||
- **Crashes still occur** on model unload in both modes
|
||||
- **Environment variable timing** - set after engine constructor runs
|
||||
|
||||
### **Technical Findings:**
|
||||
|
||||
1. **Performance Impact**: Confirmed ~40% speed reduction when CUDA Graphs disabled (55+ tok/s → ~35 tok/s)
|
||||
2. **Memory Management**: 11GB VRAM freed successfully, but CUDA state corruption persists
|
||||
3. **Architectural Conflict**: ComfyUI's dynamic model management fundamentally incompatible with CUDA Graph memory capture
|
||||
4. **PyTorch Limitation**: No clean API to disable CUDA Graphs after model initialization
|
||||
|
||||
### **Final Assessment**:
|
||||
|
||||
The toggle provides **user choice** but both modes have limitations:
|
||||
- **High Performance Mode**: Full speed, but model cannot be unloaded (restart required)
|
||||
- **Memory Safe Mode**: Still crashes due to architectural conflicts (implementation incomplete)
|
||||
|
||||
**Recommendation**: Document current state as "partial solution" - users can choose performance vs theoretical memory safety, understanding both have constraints.
|
||||
|
||||
---
|
||||
|
||||
## **Key Technical Insights Discovered**
|
||||
|
||||
### **1. CUDA Graph Optimization Requirements**
|
||||
- Must load directly on target GPU device
|
||||
- CPU→GPU transfer breaks CUDA Graph compilation
|
||||
- StaticCache is essential for performance (50+ it/s vs 12 it/s)
|
||||
|
||||
### **2. Device Consistency Challenges**
|
||||
- Device mismatches occur at multiple pipeline stages:
|
||||
- Audio tokenizer input processing
|
||||
- Semantic model tensor operations
|
||||
- Encoder operations
|
||||
- KV cache tensor operations
|
||||
- Data collator tensor concatenation
|
||||
- Requires systematic fixes throughout entire pipeline
|
||||
|
||||
### **3. Memory Management Conflicts**
|
||||
- CUDA Graphs capture memory allocations during generation
|
||||
- Model unloading tries to free "captured" allocations
|
||||
- Results in warnings → crashes
|
||||
- Requires explicit CUDA Graph cleanup before unloading
|
||||
|
||||
### **4. Performance Bottlenecks Identified**
|
||||
- **Primary**: CPU→GPU loading approach (54 it/s → 12 it/s impact)
|
||||
- **Secondary**: Tensor cloning operations (moderate impact)
|
||||
- **Tertiary**: Excessive device synchronization (minor impact)
|
||||
|
||||
---
|
||||
|
||||
## **Files Modified**
|
||||
|
||||
### **Core Implementation**
|
||||
- `engines/higgs_audio/stateless_wrapper.py` - Main stateless wrapper
|
||||
- `utils/models/unified_model_interface.py` - Factory integration
|
||||
- `engines/adapters/higgs_audio_adapter.py` - Adapter integration
|
||||
- `utils/models/comfyui_model_wrapper.py` - ComfyUI integration
|
||||
|
||||
### **Device Consistency Fixes**
|
||||
- `engines/higgs_audio/boson_multimodal/serve/serve_engine.py` - Input/cache device handling
|
||||
- `engines/higgs_audio/boson_multimodal/audio_processing/higgs_audio_tokenizer.py` - Tokenizer device handling
|
||||
- `engines/higgs_audio/boson_multimodal/data_collator/higgs_audio_collator.py` - Collator device handling
|
||||
|
||||
---
|
||||
|
||||
## **Current Status (As of Final Context)**
|
||||
|
||||
### **✅ Confirmed Working**
|
||||
- **Voice cloning functionality** - Full feature preservation
|
||||
- **Memory management** - 11GB VRAM freed successfully
|
||||
- **Device consistency** - No device mismatch crashes
|
||||
- **High performance potential** - 54.92 tok/s achieved
|
||||
|
||||
### **❌ Outstanding Issues**
|
||||
- **CUDA Graph cleanup crashes** - Hybrid solution untested
|
||||
- **Performance vs Safety tradeoff** - Need both simultaneously
|
||||
|
||||
### **🔬 Untested Solution**
|
||||
- **Hybrid approach** with comprehensive CUDA Graph cleanup
|
||||
- **Expected outcome**: 54+ it/s performance + no crashes + safe unloading
|
||||
- **Status**: Implementation complete, testing required
|
||||
|
||||
---
|
||||
|
||||
## **Recommended Next Steps**
|
||||
|
||||
1. **Test hybrid solution** - Verify 54 it/s + no crashes
|
||||
2. **If crashes persist**: Investigate alternative CUDA Graph cleanup methods
|
||||
3. **Performance monitoring**: Ensure no regression from cleanup operations
|
||||
4. **Stress testing**: Multiple load/unload cycles
|
||||
5. **Documentation update**: Record final working solution
|
||||
|
||||
---
|
||||
|
||||
## **Lessons Learned**
|
||||
|
||||
1. **Device loading approach is critical** for CUDA Graph optimization
|
||||
2. **Systematic device consistency** required throughout pipeline
|
||||
3. **CUDA Graph cleanup** must be explicit and comprehensive
|
||||
4. **Performance vs Safety** requires sophisticated balancing, not simple tradeoffs
|
||||
5. **Deep integration challenges** require understanding of underlying frameworks (transformers, CUDA)
|
||||
|
||||
This implementation represents a complex balance between PyTorch CUDA Graph optimization, device management, and ComfyUI model lifecycle integration.
|
||||
@@ -1,107 +0,0 @@
|
||||
# IndexTTS-2 Implementation Plan
|
||||
|
||||
## Overview
|
||||
IndexTTS-2 is a breakthrough emotionally expressive and duration-controlled autoregressive zero-shot TTS engine featuring emotion disentanglement and precise voice cloning.
|
||||
|
||||
## Unique Features
|
||||
- **Duration Control**: First autoregressive TTS model with precise duration control
|
||||
- **Emotion Disentanglement**: Separates speaker identity from emotional expression
|
||||
- **Multi-Modal Emotion Control**: Audio references, text descriptions, manual vectors
|
||||
- **Zero-Shot Voice Cloning**: Single reference audio for voice cloning
|
||||
- **Advanced Character Switching**: `[character:emotion_ref]` syntax
|
||||
|
||||
## Technical Specifications
|
||||
- **Sample Rate**: 22,050 Hz
|
||||
- **Audio Format**: `torch.Tensor` shape `[1, samples]`
|
||||
- **Dependencies**: `torch==2.8.*`, `transformers==4.52.1`, IndexTTS-2 source code
|
||||
- **Model Components**: GPT + Semantic codec + S2Mel + BigVGAN + CAMPPlus + QwenEmotion
|
||||
|
||||
## Implementation Status
|
||||
|
||||
### ✅ Completed
|
||||
1. **Core Engine** (`engines/index_tts/index_tts.py`)
|
||||
- IndexTTSEngine class with unified model interface integration
|
||||
- Proper UnifiedModelInterface integration for loading/unloading
|
||||
- Factory registration in unified_model_interface.py
|
||||
- All official parameters exposed (emotion vectors, text emotions, generation params)
|
||||
|
||||
2. **Downloader** (`engines/index_tts/index_tts_downloader.py`)
|
||||
- Uses folder_paths.models_dir for correct TTS/IndexTTS/ structure
|
||||
- unified_downloader integration
|
||||
- Downloads both HuggingFace model files and GitHub source code
|
||||
- Model verification system
|
||||
|
||||
3. **Cache Integration** (`utils/audio/cache.py`)
|
||||
- IndexTTSCacheKeyGenerator added to unified cache system
|
||||
- Handles all IndexTTS-2 parameters for cache key generation
|
||||
- Integrated with global audio_cache
|
||||
|
||||
4. **Engine Adapter** (`engines/adapters/index_tts_adapter.py` - IN PROGRESS)
|
||||
- Character switching support with emotion references
|
||||
- Cache integration and parameter mapping
|
||||
- Uses existing CharacterParser for tag parsing
|
||||
|
||||
### 🚧 In Progress
|
||||
5. **Character Tag Extension**
|
||||
- Need to handle flexible syntax: `[de:]`, `[de:Alice]`, `[Alice]`, `[Alice:angry_bob]`, `[de:Alice:angry_bob]`
|
||||
- Extend CharacterParser or create IndexTTS-specific parsing
|
||||
- Avoid ambiguity between language codes and character names
|
||||
|
||||
### 📋 Pending
|
||||
6. **Engine Configuration Node** (`nodes/engines/index_tts_engine_node.py`)
|
||||
- UI node with all IndexTTS-2 parameters
|
||||
- Model path selection with auto-download
|
||||
- Emotion controls (audio references, vectors, text)
|
||||
- Generation parameters (temperature, top_p, top_k, etc.)
|
||||
|
||||
7. **TTS Text Integration** (`nodes/index_tts/`)
|
||||
- `index_tts_processor.py` - Main TTS processor for unified TTS Text node
|
||||
- `index_tts_srt_processor.py` - SRT subtitle processing with timing
|
||||
- Integration with character switching system
|
||||
|
||||
8. **Testing & Validation**
|
||||
- Basic functionality testing
|
||||
- Character switching validation
|
||||
- Emotion control verification
|
||||
- Performance optimization
|
||||
|
||||
## Character Switching Syntax Design
|
||||
|
||||
### Proposed Flexible Syntax Resolution:
|
||||
```
|
||||
[de:] → language="de", character="narrator", emotion=None
|
||||
[de:Alice] → language="de", character="Alice", emotion=None
|
||||
[Alice] → language=None, character="Alice", emotion=None
|
||||
[Alice:angry_bob] → language=None, character="Alice", emotion="angry_bob"
|
||||
[de:Alice:angry_bob] → language="de", character="Alice", emotion="angry_bob"
|
||||
```
|
||||
|
||||
### Resolution Strategy:
|
||||
1. Split by `:` and count parts
|
||||
2. 1 part: Check if it's a known language code, else treat as character
|
||||
3. 2 parts: Check if first part is language code, else treat as character:emotion
|
||||
4. 3 parts: Always language:character:emotion
|
||||
|
||||
### Implementation Files:
|
||||
- Extend `utils/text/character_parser.py` with emotion support
|
||||
- Or create `engines/index_tts/emotion_parser.py` for IndexTTS-specific parsing
|
||||
|
||||
## Model Management Integration
|
||||
|
||||
### Unified Systems Used:
|
||||
- **UnifiedModelInterface**: Standardized loading/unloading
|
||||
- **unified_downloader**: HuggingFace + Git downloads
|
||||
- **AudioCache**: Generation result caching
|
||||
- **folder_paths**: Correct model directory structure
|
||||
- **CharacterParser**: Tag parsing and language resolution
|
||||
|
||||
### Memory Management:
|
||||
- ComfyUI model wrapper integration via UnifiedModelInterface
|
||||
- Automatic VRAM management and "Clear VRAM" button support
|
||||
- Model factory registration for proper lifecycle management
|
||||
|
||||
## Next Steps:
|
||||
1. Resolve character tag ambiguity with flexible parsing
|
||||
2. Create IndexTTS engine configuration node
|
||||
3. Implement TTS Text processor integration
|
||||
4. Test full character switching + emotion control workflow
|
||||
@@ -1,67 +0,0 @@
|
||||
# Isolated Runtimes Plan
|
||||
|
||||
## Goal
|
||||
|
||||
Keep ComfyUI in one modern main runtime while routing fragile engines into
|
||||
their own pinned Python environments.
|
||||
|
||||
## Why
|
||||
|
||||
- `transformers` major-version breaks now hit engines differently
|
||||
- one broken HF stack should not poison the whole node runtime
|
||||
- per-engine rollback is simpler than global dependency rollback
|
||||
|
||||
## User impact
|
||||
|
||||
- isolated runtimes duplicate dependencies on disk
|
||||
- large packages such as PyTorch will usually be installed again inside the worker runtime
|
||||
- this must be warned in:
|
||||
- docs
|
||||
- node tooltips
|
||||
- runtime creation logs
|
||||
|
||||
## Contract
|
||||
|
||||
ComfyUI process:
|
||||
- owns UI, node orchestration, model discovery, caching policy
|
||||
- prepares normalized request payloads
|
||||
- launches a worker only for engines configured as `runtime_mode=isolated`
|
||||
|
||||
Worker process:
|
||||
- runs in a dedicated Python env
|
||||
- receives a JSON request
|
||||
- performs load / generate / unload actions
|
||||
- returns JSON response + output artifact paths
|
||||
|
||||
## Initial profiles
|
||||
|
||||
- `vibevoice_transformers4`
|
||||
- `qwen3_tts_transformers5`
|
||||
- `step_audio_editx_transformers5`
|
||||
- `moss_tts_transformers5`
|
||||
|
||||
## First target
|
||||
|
||||
VibeVoice / KugelAudio
|
||||
|
||||
Reason:
|
||||
- highest dependency fragility
|
||||
- strongest evidence that global `transformers 5` compatibility is not enough
|
||||
- likely template for future HF-heavy engines
|
||||
|
||||
## Implementation order
|
||||
|
||||
1. Add config + registry support for runtime routing
|
||||
2. Add launcher + request/response protocol
|
||||
3. Add VibeVoice worker entrypoint
|
||||
4. Add opt-in routing from unified interface
|
||||
5. Add user-facing runtime path configuration
|
||||
6. Expand to Qwen / MOSS / Step Audio only after VibeVoice works
|
||||
|
||||
## Non-goals
|
||||
|
||||
- no silent env creation
|
||||
- no arbitrary command execution
|
||||
- no shell-constructed user input
|
||||
- no automatic migration of all engines
|
||||
- no pretending disk duplication does not exist
|
||||
@@ -1,633 +0,0 @@
|
||||
# TTS Audio Suite Model Loading Architecture Analysis
|
||||
|
||||
## EXECUTIVE SUMMARY
|
||||
|
||||
The codebase shows a **two-tiered model loading architecture**:
|
||||
1. **Unified Model Interface Layer** - Relatively new, factory-pattern based, ComfyUI-integrated
|
||||
2. **Legacy/Fallback Systems** - Multiple engine-specific loaders still in use, creating redundancy
|
||||
|
||||
**Critical Finding**: The unified interface exists but is **NOT consistently adopted**. Some engines use it fully, others have dual implementations, creating inconsistent patterns and potential device/cache conflicts.
|
||||
|
||||
---
|
||||
|
||||
## 1. UNIFIED INTERFACE USAGE & ADOPTION RATE
|
||||
|
||||
### Current Status
|
||||
**File**: `/home/linux/ComfyUI_TTS_Audio_Suite/utils/models/unified_model_interface.py` (817 lines)
|
||||
|
||||
### Adoption by Engine
|
||||
|
||||
| Engine | Status | Primary Loader | Fallback | Notes |
|
||||
|--------|--------|----------------|----------|-------|
|
||||
| ChatterBox | DUAL | Unified Interface | Manager.load_tts_model() | Uses unified, but manager has legacy fallback logic (lines 252-439 in manager.py) |
|
||||
| F5-TTS | DUAL | Unified Interface | SmartModelLoader + direct F5TTS calls | Hybrid approach with multiple fallback paths |
|
||||
| ChatterBox Official 23-Lang | UNIFIED | Unified Interface | None documented | Cleanest implementation, language-aware cache key (line 216-217) |
|
||||
| Higgs Audio | UNIFIED | Unified Interface | None documented | Stateless wrapper integration (line 566) |
|
||||
| RVC | UNIFIED | Unified Interface (4 factories) | None documented | Most comprehensive (VC, hubert, separator_mdx, separator_demucs) |
|
||||
| VibeVoice | UNIFIED | Unified Interface | None documented | Engine pattern with integrated initialization |
|
||||
| IndexTTS-2 | UNIFIED | Unified Interface | None documented | Path resolution and auto-download handling |
|
||||
|
||||
**Adoption Rate**: ~85% of new engines use unified interface, but legacy fallbacks still exist
|
||||
|
||||
### Factory Registration Architecture
|
||||
**Location**: Lines 327-813 in unified_model_interface.py
|
||||
|
||||
```python
|
||||
def initialize_all_factories(): # Line 805
|
||||
register_chatterbox_factory()
|
||||
register_chatterbox_23lang_factory()
|
||||
register_f5tts_factory()
|
||||
register_higgs_audio_factory()
|
||||
register_rvc_factory()
|
||||
register_vibevoice_factory()
|
||||
register_index_tts_factory()
|
||||
```
|
||||
|
||||
Each factory is registered with key format: `"{engine_name}_{model_type}"` (line 55)
|
||||
|
||||
---
|
||||
|
||||
## 2. FACTORY PATTERN CONSISTENCY ANALYSIS
|
||||
|
||||
### Issue 1: Inconsistent Error Handling
|
||||
|
||||
**ChatterBox Factory** (lines 401-444)
|
||||
```python
|
||||
def chatterbox_tts_factory(**kwargs):
|
||||
if ChatterboxTTS is None:
|
||||
raise RuntimeError("ChatterboxTTS not available - check installation")
|
||||
# Has fallback mixing logic
|
||||
```
|
||||
|
||||
**F5TTS Factory** (lines 449-497)
|
||||
```python
|
||||
def f5tts_factory(**kwargs):
|
||||
# Uses smart_model_loader.load_model_if_needed()
|
||||
# No inline error handling - delegates to SmartModelLoader
|
||||
```
|
||||
|
||||
**VibeVoice Factory** (lines 623-657)
|
||||
```python
|
||||
def vibevoice_factory(**kwargs):
|
||||
try:
|
||||
import accelerate
|
||||
except ImportError as e:
|
||||
print(f"⚠️ Unified Interface: accelerate not available: {e}")
|
||||
# Continues without raising - soft failure pattern
|
||||
```
|
||||
|
||||
**Problem**: Three different error handling patterns:
|
||||
1. ChatterBox: Raises RuntimeError for missing imports
|
||||
2. F5TTS: Delegates to SmartModelLoader (different exception path)
|
||||
3. VibeVoice: Prints warning and continues (no exception)
|
||||
|
||||
### Issue 2: Inconsistent Signatures
|
||||
|
||||
| Factory | Signature Pattern | Special Handling |
|
||||
|---------|------------------|------------------|
|
||||
| ChatterBox | `device, language, model_path` | Multi-step fallback with German special case (lines 366-383) |
|
||||
| F5TTS | `device, model_name` | Delegates to smart_loader callback |
|
||||
| Higgs Audio | `device, model_path, tokenizer_path, enable_cuda_graphs` | CUDA-specific parameters |
|
||||
| RVC | `model_path, index_path, device` | Multiple model types (VC, hubert, separators) |
|
||||
| IndexTTS-2 | `device, use_fp16, use_cuda_kernel, use_deepspeed` | Optimization flags |
|
||||
|
||||
**Problem**: Each factory has different parameter semantics and handling logic.
|
||||
|
||||
### Issue 3: Device Resolution Inconsistencies
|
||||
|
||||
**Unified Interface Approach** (line 141):
|
||||
```python
|
||||
from utils.device import resolve_torch_device
|
||||
target_device = resolve_torch_device(config.device)
|
||||
```
|
||||
|
||||
**Factory-Level Device Resolution**:
|
||||
- **ChatterBox** (line 410): `device = kwargs.get("device", "auto")`
|
||||
- Passed directly to ChatterboxTTS.from_local/from_pretrained
|
||||
- Device resolution happens IN the engine, not in unified interface
|
||||
|
||||
- **F5TTS** (line 453): `device = kwargs.get("device", "auto")`
|
||||
- Passed to smart_model_loader
|
||||
- Then to ChatterBoxF5TTS.from_local/from_pretrained
|
||||
- **Two levels of resolution possible**
|
||||
|
||||
- **Higgs Audio** (line 508): `device = kwargs.get("device", "cuda")`
|
||||
- Default is "cuda" not "auto"
|
||||
- **Inconsistent default**
|
||||
|
||||
- **VibeVoice** (line 637): `device = kwargs.get("device", "auto")`
|
||||
|
||||
**Problem**: Device resolution happens at MULTIPLE levels, with inconsistent defaults and no guarantee of final device being correct.
|
||||
|
||||
---
|
||||
|
||||
## 3. MODEL FILE LOADING PATTERNS - ALL INSTANCES
|
||||
|
||||
### Load Functions in Codebase
|
||||
|
||||
| File | Function | Pattern | Issues |
|
||||
|------|----------|---------|--------|
|
||||
| manager.py (line 226-439) | `ModelManager.load_tts_model()` | Try local → Try HuggingFace → Fallback English | Has language fallback logic |
|
||||
| manager.py (line 441-577) | `ModelManager.load_vc_model()` | Try local → Try HuggingFace | Supports language switching |
|
||||
| f5tts_manager.py (line 108-223) | `F5TTSModelManager.load_f5tts_model()` | Try unified → Fallback unified+direct | Dual system |
|
||||
| f5tts_base_node.py (line 86-192) | `BaseF5TTSNode.load_f5tts_model()` | Try unified → Fallback SmartLoader | Three-tier fallback |
|
||||
| unified_model_interface.py (line 111-178) | `UnifiedModelInterface.load_model()` | Check cache → Load via factory | Direct factory invocation |
|
||||
|
||||
### Format Detection Logic
|
||||
|
||||
**ChatterBox** (manager.py, lines 108-120):
|
||||
- Detects required files: `ve.`, `t3_cfg.`, `s3gen.`, `tokenizer.json`
|
||||
- Can be `.pt` or `.safetensors` extensions
|
||||
- Directory-based model location
|
||||
|
||||
**F5TTS** (f5tts_manager.py, lines 79-83):
|
||||
- Checks for ANY `.safetensors` or `.pt` files in directory
|
||||
- Less specific detection (any model file matches)
|
||||
|
||||
**ChatterBox Official 23-Lang** (unified_model_interface.py, lines 776-785):
|
||||
- Has explicit `from_local()` and `from_pretrained()` methods
|
||||
- Model version parameter (v1 vs v2)
|
||||
|
||||
**Higgs Audio** (higgs_audio_engine_node.py, lines 79-105):
|
||||
- Requires subdirectories: `generation/` and `tokenizer/`
|
||||
- Checks for `config.json` and model files in both
|
||||
|
||||
**RVC** (manager.py, no dedicated loader):
|
||||
- Model loading happens in factory only
|
||||
- Format: model_path and index_path parameters
|
||||
|
||||
**Problem**: Each engine has DIFFERENT detection logic with NO unified format detection system. Code duplication across manager.py, f5tts_manager.py, and node discovery methods.
|
||||
|
||||
---
|
||||
|
||||
## 4. DEVICE MANAGEMENT ANALYSIS
|
||||
|
||||
### Unified Interface Device Handling
|
||||
|
||||
**Location**: unified_model_interface.py, lines 139-146
|
||||
```python
|
||||
# Check if force reload is needed
|
||||
if force_reload:
|
||||
tts_model_manager.remove_model(cache_key)
|
||||
|
||||
# Get existing model if cached
|
||||
wrapper = tts_model_manager.get_model(cache_key)
|
||||
if wrapper is not None:
|
||||
# Ensure model is on correct device
|
||||
from utils.device import resolve_torch_device
|
||||
target_device = resolve_torch_device(config.device)
|
||||
|
||||
if wrapper.current_device != target_device:
|
||||
wrapper.model_load(target_device)
|
||||
return wrapper.model
|
||||
```
|
||||
|
||||
**Problem 1**: Device comparison uses string comparison (`wrapper.current_device != target_device`). If one is `"auto"` and other is resolved `"cuda"`, they won't match even though they're the same device.
|
||||
|
||||
**ComfyUI Wrapper Device Handling**
|
||||
|
||||
**Location**: base_wrapper.py, lines 77-87
|
||||
```python
|
||||
# Convert device to torch.device object for ComfyUI compatibility
|
||||
from utils.device import resolve_torch_device
|
||||
device_name = resolve_torch_device(model_info.device)
|
||||
|
||||
if isinstance(device_name, str):
|
||||
if device_name == "cuda":
|
||||
self.device = torch.device("cuda", torch.cuda.current_device() if torch.cuda.is_available() else 0)
|
||||
else:
|
||||
self.device = torch.device(device_name)
|
||||
else:
|
||||
self.device = device_name
|
||||
```
|
||||
|
||||
**Problem 2**: Device is resolved, converted to torch.device, but wrapper stores it as torch.device object while unified interface uses strings. **Type mismatch**.
|
||||
|
||||
### Auto Device Resolution
|
||||
|
||||
**Location**: torch_device_resolver.py, lines 10-51
|
||||
|
||||
```python
|
||||
def resolve_torch_device(device: str) -> str:
|
||||
if device == "auto":
|
||||
if torch.backends.mps.is_available():
|
||||
return "mps"
|
||||
elif torch.cuda.is_available():
|
||||
return "cuda"
|
||||
elif hasattr(torch, "xpu") and torch.xpu.is_available():
|
||||
return "xpu"
|
||||
else:
|
||||
return "cpu"
|
||||
```
|
||||
|
||||
**Problem 3**: Called in multiple places with no caching. If GPU becomes unavailable between calls (memory pressure unload), subsequent calls might return different device. **No locking or consistency guarantee**.
|
||||
|
||||
### Device Mismatch Issues Found
|
||||
|
||||
1. **BaseNode.resolve_device()** (base_node.py, lines 124-136):
|
||||
```python
|
||||
def resolve_device(self, device: str) -> str:
|
||||
if device == "auto":
|
||||
return "cuda" if torch.cuda.is_available() else "cpu"
|
||||
return device
|
||||
```
|
||||
**This is DIFFERENT from torch_device_resolver.py** - doesn't check MPS or XPU!
|
||||
|
||||
2. **Factory defaults** - Higgs Audio uses `device="cuda"` as default (line 508 unified_model_interface.py), while others use `"auto"`.
|
||||
|
||||
3. **ComfyUI wrapper** (line 141 base_wrapper.py):
|
||||
```python
|
||||
from utils.device import resolve_torch_device
|
||||
target_device = resolve_torch_device(config.device)
|
||||
```
|
||||
But then converts to torch.device object, not string comparison.
|
||||
|
||||
---
|
||||
|
||||
## 5. CACHING & MEMORY MANAGEMENT
|
||||
|
||||
### Unified Interface Cache Key Generation
|
||||
|
||||
**Location**: unified_model_interface.py, lines 205-234
|
||||
|
||||
```python
|
||||
def _generate_cache_key(self, config: UnifiedModelConfig) -> str:
|
||||
components = [
|
||||
config.engine_name,
|
||||
config.model_type,
|
||||
config.model_name,
|
||||
config.device,
|
||||
]
|
||||
|
||||
# CRITICAL FIX: ChatterBox Official 23-Lang is multilingual - language shouldn't affect cache
|
||||
if config.engine_name != "chatterbox_official_23lang":
|
||||
components.append(config.language or "default")
|
||||
|
||||
# Add path/repo info if available
|
||||
if config.model_path:
|
||||
components.append(f"path:{Path(config.model_path).name}")
|
||||
elif config.repo_id:
|
||||
components.append(f"repo:{config.repo_id}")
|
||||
|
||||
# Add additional_params to ensure attention_mode, quantization, etc. trigger reloads
|
||||
if config.additional_params:
|
||||
sorted_params = sorted(config.additional_params.items())
|
||||
params_str = "_".join([f"{k}:{v}" for k, v in sorted_params])
|
||||
components.append(f"params:{params_str}")
|
||||
|
||||
cache_key = "_".join(components)
|
||||
return cache_key
|
||||
```
|
||||
|
||||
**Issues**:
|
||||
1. **Line 211**: Device is ALWAYS included, but device can be "auto" which resolves differently on each call → **Cache key instability**
|
||||
2. **Line 216-217**: Special-case hardcoding for chatterbox_official_23lang (comment says "CRITICAL FIX") suggests cache key generation is fragile
|
||||
3. **No address hashing**: Full strings concatenated, very long cache keys possible
|
||||
4. **Path name only** (line 221): If two models named identically in different paths, cache key collision possible
|
||||
|
||||
### ComfyUI Model Manager Caching
|
||||
|
||||
**Location**: model_manager.py, lines 44-127
|
||||
|
||||
```python
|
||||
def load_model(self,
|
||||
model_factory_func,
|
||||
model_key: str,
|
||||
model_type: str,
|
||||
engine: str,
|
||||
device: str,
|
||||
force_reload: bool = False,
|
||||
**factory_kwargs) -> ComfyUIModelWrapper:
|
||||
|
||||
# Check if already cached
|
||||
if model_key in self._model_cache and not force_reload:
|
||||
wrapper = self._model_cache[model_key]
|
||||
is_valid = getattr(wrapper, '_is_valid_for_reuse', True)
|
||||
|
||||
if not is_valid:
|
||||
# For Higgs Audio with CUDA graph corruption...
|
||||
# For VibeVoice, try to recover...
|
||||
# For IndexTTS-2, attempt recovery...
|
||||
```
|
||||
|
||||
**Issues**:
|
||||
1. **Engine-specific recovery logic** (lines 54-117):
|
||||
- Higgs Audio: Reset CUDA graph state (line 137)
|
||||
- VibeVoice: Clear internal caches (line 77)
|
||||
- IndexTTS-2: Clear device-cached state (line 102)
|
||||
- **Each engine needs custom recovery code** - not scalable
|
||||
|
||||
2. **Validity flag** (line 48):
|
||||
```python
|
||||
is_valid = getattr(wrapper, '_is_valid_for_reuse', True)
|
||||
```
|
||||
Only Higgs Audio sets this to False (base_wrapper.py, line 302). **VibeVoice and IndexTTS-2 corrupted models won't be detected**.
|
||||
|
||||
3. **No smart cache management** - older models not cleared to make room for new ones automatically.
|
||||
|
||||
### SmartModelLoader Cache
|
||||
|
||||
**Location**: smart_loader.py, lines 23-225
|
||||
|
||||
```python
|
||||
class SmartModelLoader:
|
||||
def __init__(self):
|
||||
self._global_model_cache: Dict[str, ModelInfo] = {}
|
||||
self._current_models: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
def _generate_cache_key(self, engine_type: str, model_name: str, device: str) -> str:
|
||||
key_data = f"{engine_type}_{model_name}_{device}"
|
||||
return hashlib.md5(key_data.encode()).hexdigest()[:16]
|
||||
```
|
||||
|
||||
**Issues**:
|
||||
1. **MD5 hashing** (line 44): Different from unified interface string concatenation
|
||||
2. **THREE separate cache systems**:
|
||||
- UnifiedModelInterface: String concatenation
|
||||
- ComfyUITTSModelManager: Direct dict with string keys
|
||||
- SmartModelLoader: MD5 hashes
|
||||
- **Models can exist in multiple caches simultaneously**
|
||||
|
||||
3. **Cross-engine reuse** (lines 59-84):
|
||||
```python
|
||||
def _check_cross_engine_cache(self, engine_type: str, model_name: str, device: str) -> Optional[Any]:
|
||||
cache_key = self._generate_cache_key(engine_type, model_name, device)
|
||||
if cache_key in self._global_model_cache:
|
||||
model_info = self._global_model_cache[cache_key]
|
||||
model_instance = model_info.model_instance
|
||||
# CRITICAL: Ensure model is on correct device after cache reuse
|
||||
actual_device = "auto" if device == "auto" and torch.cuda.is_available() else device
|
||||
```
|
||||
Device resolution happens again here! **Multiple resolution points for "auto"**.
|
||||
|
||||
---
|
||||
|
||||
## 6. ERROR HANDLING PATTERNS
|
||||
|
||||
### Missing Fallback Mechanisms
|
||||
|
||||
**ChatterBox** (unified_model_interface.py, lines 340-398):
|
||||
- ✅ Has multi-level fallback:
|
||||
1. Try provided path
|
||||
2. Try local language model
|
||||
3. Try HuggingFace language
|
||||
4. Special fallback for German variants
|
||||
5. Final fallback to English
|
||||
- ✅ Graceful 401 authorization error handling (line 362)
|
||||
|
||||
**F5-TTS** (unified_model_interface.py, lines 449-497):
|
||||
- ⚠️ Delegates to SmartModelLoader - error handling unclear
|
||||
- ⚠️ No explicit fallback documented
|
||||
- ⚠️ Smart loader has callback pattern but errors propagate up
|
||||
|
||||
**Higgs Audio** (unified_model_interface.py, lines 502-571):
|
||||
- ❌ No fallback mechanism
|
||||
- ⚠️ Sets up CUDA graphs, can fail silently if disabled
|
||||
- ⚠️ Complex initialization with multiple environment variables
|
||||
|
||||
**RVC** (unified_model_interface.py, lines 574-618):
|
||||
- ❌ No fallback for missing models
|
||||
- ⚠️ get_vc() call at line 590 - error propagates directly
|
||||
- ❌ No graceful degradation
|
||||
|
||||
**VibeVoice** (unified_model_interface.py, lines 621-657):
|
||||
- ⚠️ Soft failure for accelerate import (line 631)
|
||||
- ❌ No fallback if model download fails
|
||||
|
||||
**IndexTTS-2** (unified_model_interface.py, lines 660-746):
|
||||
- ✅ Good error detection for external IndexTTS installations (lines 688-708)
|
||||
- ✅ Checks config.yaml existence (line 726)
|
||||
- ⚠️ Raises RuntimeError - no fallback
|
||||
|
||||
### Import Error Handling
|
||||
|
||||
**Pattern 1 - Try/Except on Import** (most factories)
|
||||
```python
|
||||
try:
|
||||
from engines.chatterbox.tts import ChatterboxTTS
|
||||
except ImportError:
|
||||
ChatterboxTTS = None
|
||||
```
|
||||
|
||||
**Pattern 2 - Direct Import with No Error Handling** (RVC)
|
||||
```python
|
||||
from engines.rvc.impl.vc_infer_pipeline import get_vc # Line 578
|
||||
from engines.rvc.impl.config import config as rvc_config # Line 579
|
||||
```
|
||||
|
||||
**Pattern 3 - ImportError Message** (IndexTTS)
|
||||
```python
|
||||
except ImportError as e:
|
||||
raise ImportError(f"IndexTTS-2 dependencies not available. Error: {e}")
|
||||
```
|
||||
|
||||
**Problem**: No consistent pattern for handling missing dependencies.
|
||||
|
||||
---
|
||||
|
||||
## 7. SPECIFIC PROBLEMS IDENTIFIED
|
||||
|
||||
### Problem 1: "auto" Device Instability
|
||||
**Severity**: HIGH
|
||||
**Location**: Multiple (torch_device_resolver.py, base_node.py, smart_loader.py, unified_model_interface.py)
|
||||
|
||||
Device resolution for "auto" happens at 4+ different places with NO caching. If CUDA becomes unavailable mid-execution, subsequent calls return CPU device while cache key still contains "auto".
|
||||
|
||||
**Example**:
|
||||
1. Model loads on CUDA with cache key: `"chatterbox_tts_auto"`
|
||||
2. ComfyUI clears VRAM
|
||||
3. Next inference call: `resolve_torch_device("auto")` now returns "cpu"
|
||||
4. Cache lookup fails (expects "auto", not "cpu")
|
||||
5. Model reloads on CPU unnecessarily
|
||||
|
||||
### Problem 2: Dual Unified Interface Usage
|
||||
**Severity**: MEDIUM
|
||||
**Location**: manager.py (lines 252-273), f5tts_manager.py (lines 132-145)
|
||||
|
||||
Both ChatterBox and F5-TTS managers try unified interface FIRST, then fall back to legacy systems. Creates:
|
||||
- Inconsistent code paths
|
||||
- Debug complexity
|
||||
- Potential for models cached in one system, used in another
|
||||
|
||||
```python
|
||||
# manager.py, line 253
|
||||
try:
|
||||
model = load_tts_model(engine_name="chatterbox", ...)
|
||||
except Exception as e:
|
||||
print(f"⚠️ Failed to load ChatterBox model via unified interface: {e}")
|
||||
# Fall back to original logic if unified interface fails
|
||||
pass
|
||||
```
|
||||
|
||||
### Problem 3: Cache Key Collision Risk
|
||||
**Severity**: MEDIUM
|
||||
**Location**: unified_model_interface.py (lines 205-234)
|
||||
|
||||
Cache key uses basename only for paths:
|
||||
```python
|
||||
if config.model_path:
|
||||
components.append(f"path:{Path(config.model_path).name}") # Line 221
|
||||
```
|
||||
|
||||
If two different paths have identically-named model directories, cache collision occurs:
|
||||
- `/models/TTS/chatterbox/English/`
|
||||
- `/backup/models/chatterbox/English/`
|
||||
- Both generate cache component: `path:English`
|
||||
|
||||
### Problem 4: Device Type Mismatch
|
||||
**Severity**: MEDIUM
|
||||
**Location**: base_wrapper.py (lines 77-87), unified_model_interface.py (line 141-146)
|
||||
|
||||
Unified interface stores device as string, ComfyUI wrapper converts to torch.device:
|
||||
```python
|
||||
# unified_model_interface.py - uses string
|
||||
target_device = resolve_torch_device(config.device) # Returns "cuda"
|
||||
|
||||
# base_wrapper.py - stores torch.device
|
||||
self.device = torch.device("cuda", ...) # torch.device object
|
||||
```
|
||||
|
||||
Later comparison:
|
||||
```python
|
||||
if wrapper.current_device != target_device: # "cuda" != torch.device("cuda")
|
||||
```
|
||||
**Type mismatch in comparison**.
|
||||
|
||||
### Problem 5: Engine-Specific Recovery Code Not Scalable
|
||||
**Severity**: MEDIUM
|
||||
**Location**: model_manager.py (lines 54-117)
|
||||
|
||||
Each engine has custom recovery logic for corrupted models:
|
||||
```python
|
||||
if engine == "higgs_audio":
|
||||
# Reset CUDA graphs
|
||||
elif engine == "vibevoice":
|
||||
# Clear caches
|
||||
elif engine == "index_tts":
|
||||
# Clear device state
|
||||
```
|
||||
|
||||
When new engine added, must modify model_manager.py. No extensible pattern.
|
||||
|
||||
### Problem 6: Three Separate Caching Systems
|
||||
**Severity**: HIGH
|
||||
**Location**: unified_model_interface.py, model_manager.py, smart_loader.py
|
||||
|
||||
Models can be cached in:
|
||||
1. **UnifiedModelInterface**: Via ComfyUI manager
|
||||
2. **SmartModelLoader**: Global cache with MD5 keys
|
||||
3. **Manager classes**: Manual dicts (ChatterBox manager, F5-TTS manager)
|
||||
|
||||
**No synchronization between them**. A model cleared from one cache remains in others.
|
||||
|
||||
### Problem 7: Inconsistent Error Handling
|
||||
**Severity**: MEDIUM
|
||||
**Location**: All factories in unified_model_interface.py
|
||||
|
||||
- ChatterBox: Raises RuntimeError (line 408)
|
||||
- F5-TTS: Delegates to SmartModelLoader
|
||||
- VibeVoice: Prints warning and continues (line 631)
|
||||
- RVC: Direct ImportError (line 578)
|
||||
- IndexTTS-2: Custom ImportError message (line 742)
|
||||
|
||||
**No consistent error contract** - callers don't know what exceptions to expect.
|
||||
|
||||
### Problem 8: Cache Key Instability with Language
|
||||
**Severity**: LOW
|
||||
**Location**: unified_model_interface.py (line 216)
|
||||
|
||||
```python
|
||||
if config.engine_name != "chatterbox_official_23lang":
|
||||
components.append(config.language or "default")
|
||||
```
|
||||
|
||||
Special-casing suggests original cache key generation was broken. Comment says "CRITICAL FIX" but introduces inconsistency for future engines.
|
||||
|
||||
---
|
||||
|
||||
## 8. FACTORY SIGNATURE ANALYSIS
|
||||
|
||||
| Parameter | ChatterBox | F5TTS | Higgs | RVC | VibeVoice | IndexTTS |
|
||||
|-----------|-----------|-------|-------|-----|-----------|----------|
|
||||
| device | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| model_name | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ |
|
||||
| language | ✅ | ⚠️ (ignored) | ❌ | ❌ | ❌ | ❌ |
|
||||
| model_path | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ |
|
||||
| repo_id | ✅ | ⚠️ (not used) | ❌ | ❌ | ❌ | ❌ |
|
||||
| additional_params | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ |
|
||||
|
||||
**No consistent parameter contract**.
|
||||
|
||||
---
|
||||
|
||||
## 9. RECOMMENDATIONS FOR UNIFICATION
|
||||
|
||||
### Quick Wins (Low Risk)
|
||||
|
||||
1. **Fix auto device instability**:
|
||||
- Cache resolved device in resolve_torch_device()
|
||||
- Don't store "auto" in cache keys, only resolved device
|
||||
|
||||
2. **Consistent device comparison**:
|
||||
- Resolve both sides before comparison
|
||||
- Store device as string consistently across wrapper and interface
|
||||
|
||||
3. **Consolidate error handling**:
|
||||
- Create base exception class for model loading errors
|
||||
- Standardize all factories to raise same exceptions
|
||||
|
||||
### Medium Effort (Medium Risk)
|
||||
|
||||
4. **Unified fallback handler**:
|
||||
- Extract ChatterBox fallback logic to generic utility
|
||||
- Apply to all engines consistently
|
||||
|
||||
5. **Standardize cache keys**:
|
||||
- Use hash of full path, not basename only
|
||||
- Consider device as "resolved" device, not "auto"
|
||||
|
||||
6. **Single cache system**:
|
||||
- Deprecate SmartModelLoader and manager-specific caches
|
||||
- Route everything through ComfyUI manager
|
||||
|
||||
### Major Refactor (High Risk)
|
||||
|
||||
7. **Factory interface standardization**:
|
||||
- Define common parameter set all factories accept
|
||||
- Use dataclass for factory parameters
|
||||
- Implement validation layer
|
||||
|
||||
8. **Engine-agnostic recovery**:
|
||||
- Create engine capability registry
|
||||
- Generic recovery pattern with hooks
|
||||
|
||||
---
|
||||
|
||||
## FILES REQUIRING CHANGES
|
||||
|
||||
### Core Files
|
||||
- `/home/linux/ComfyUI_TTS_Audio_Suite/utils/models/unified_model_interface.py` - Cache key generation, factory signatures
|
||||
- `/home/linux/ComfyUI_TTS_Audio_Suite/utils/models/comfyui_model_wrapper/model_manager.py` - Engine-specific recovery logic
|
||||
- `/home/linux/ComfyUI_TTS_Audio_Suite/utils/device/torch_device_resolver.py` - Auto device caching
|
||||
- `/home/linux/ComfyUI_TTS_Audio_Suite/utils/models/manager.py` - Unified interface dual-loading
|
||||
- `/home/linux/ComfyUI_TTS_Audio_Suite/utils/models/f5tts_manager.py` - Unified interface dual-loading
|
||||
|
||||
### Supporting Files
|
||||
- `/home/linux/ComfyUI_TTS_Audio_Suite/utils/models/smart_loader.py` - Cache integration
|
||||
- `/home/linux/ComfyUI_TTS_Audio_Suite/utils/models/comfyui_model_wrapper/base_wrapper.py` - Device type handling
|
||||
- `/home/linux/ComfyUI_TTS_Audio_Suite/nodes/base/base_node.py` - Device resolution duplication
|
||||
- `/home/linux/ComfyUI_TTS_Audio_Suite/nodes/base/f5tts_base_node.py` - Triple-tier fallback
|
||||
|
||||
---
|
||||
|
||||
## SUMMARY TABLE
|
||||
|
||||
| Aspect | Current State | Issues | Risk Level |
|
||||
|--------|---------------|--------|-----------|
|
||||
| Unified Interface Coverage | 85% of engines | Legacy fallbacks still exist | MEDIUM |
|
||||
| Factory Consistency | Inconsistent | Different error handling, parameters | HIGH |
|
||||
| Device Management | Decentralized | 4+ resolution points, type mismatch | HIGH |
|
||||
| Caching Strategy | Triple system | No synchronization | CRITICAL |
|
||||
| Fallback Mechanisms | Partial | ChatterBox has, others don't | MEDIUM |
|
||||
| Error Handling | Inconsistent | 5 different patterns | MEDIUM |
|
||||
|
||||
@@ -1,641 +0,0 @@
|
||||
# TTS Audio Suite - Model Loading Architecture: Improvement Report
|
||||
|
||||
**Date**: November 13, 2025
|
||||
**Status**: Comprehensive Analysis Complete
|
||||
**Audience**: Technical Maintainers / Decision Makers
|
||||
|
||||
---
|
||||
|
||||
## EXECUTIVE SUMMARY
|
||||
|
||||
The TTS Audio Suite has a **functional but fragmented model loading architecture**. A unified interface exists and is being adopted (~85% coverage), but critical architectural issues create maintenance burden, cache instability, and potential device management failures.
|
||||
|
||||
**The unified system works, but the inconsistencies around it create real problems.**
|
||||
|
||||
### Key Finding
|
||||
| Aspect | Current State |
|
||||
|--------|---------------|
|
||||
| **System Functionality** | ✅ Works without errors |
|
||||
| **Architecture Quality** | ⚠️ Fragmented (legacy + new coexist) |
|
||||
| **Risk Level** | ⚠️ Medium - edge cases exist |
|
||||
| **Technical Debt** | 🔴 HIGH - 3 caching systems, 5 error patterns |
|
||||
| **Maintainability** | 🟡 Declining - new engines need custom code |
|
||||
|
||||
---
|
||||
|
||||
## PART 1: CURRENT STATE ASSESSMENT
|
||||
|
||||
### 1.1 Unified Interface Adoption
|
||||
|
||||
**What exists**: `utils/models/unified_model_interface.py` (817 lines) - A complete factory-based model loading system.
|
||||
|
||||
**Adoption rate by engine**:
|
||||
```
|
||||
✅ UNIFIED (clean implementations):
|
||||
- ChatterBox Official 23-Lang (100% using load_tts_model)
|
||||
- Higgs Audio (stateless wrapper integration working)
|
||||
- RVC (4 factory types: vc, hubert, separator_mdx, separator_demucs)
|
||||
- VibeVoice (engine initialization pattern)
|
||||
- IndexTTS-2 (path resolution + auto-download)
|
||||
|
||||
⚠️ DUAL IMPLEMENTATIONS (fallback to legacy):
|
||||
- ChatterBox (tries unified → falls back to manager.load_tts_model())
|
||||
- F5-TTS (tries unified → falls back to SmartModelLoader)
|
||||
```
|
||||
|
||||
**Reality**: 85% coverage looks good, but the 15% with fallbacks creates two code paths, making debugging harder.
|
||||
|
||||
### 1.2 What's Working Well
|
||||
|
||||
✅ **ComfyUI Integration**: Models are properly registered with ComfyUI's VRAM management. The wrapper system (comfyui_model_wrapper/) correctly handles model lifecycle.
|
||||
|
||||
✅ **Factory Pattern**: Extensible design - adding new engines requires registering a factory. New engines (Higgs, RVC, VibeVoice) implemented cleanly.
|
||||
|
||||
✅ **ChatterBox Fallback Logic**: Sophisticated fallback chain:
|
||||
```
|
||||
Try provided path → Try local language model → Try HuggingFace →
|
||||
Try German fallback → Final fallback to English
|
||||
```
|
||||
This is well-thought-out and prevents total failures.
|
||||
|
||||
✅ **Language-Aware Caching**: ChatterBox Official 23-Lang has special case to NOT include language in cache key (multilingual model special case).
|
||||
|
||||
✅ **Force Reload**: System properly handles force_reload flag for model reloading.
|
||||
|
||||
---
|
||||
|
||||
## PART 2: CRITICAL ISSUES IDENTIFIED
|
||||
|
||||
### Issue #1: "auto" Device Instability (SEVERITY: HIGH)
|
||||
|
||||
**Problem**: Device resolution for "auto" happens in multiple places with NO caching:
|
||||
- `torch_device_resolver.py` (resolve_torch_device)
|
||||
- `base_node.py` (resolve_device)
|
||||
- `smart_loader.py` (internal resolution)
|
||||
- `unified_model_interface.py` (factory level)
|
||||
|
||||
**The Bug**:
|
||||
```python
|
||||
# Call 1: CUDA available
|
||||
device_key = generate_cache_key(..., device="auto") # "auto" stored in key
|
||||
model = load_model(device="auto") # Resolves to "cuda"
|
||||
|
||||
# ComfyUI clears VRAM, CUDA unavailable
|
||||
|
||||
# Call 2: CUDA unavailable
|
||||
device_key = generate_cache_key(..., device="auto") # Still "auto"
|
||||
model = load_model(device="auto") # Now resolves to "cpu"
|
||||
# CACHE MISS - model reloaded unnecessarily!
|
||||
```
|
||||
|
||||
**Impact**:
|
||||
- Unnecessary model reloads during long ComfyUI sessions
|
||||
- VRAM thrashing if GPU memory pressure forces CPU fallback
|
||||
- Corrupts caching logic for "auto" device selection
|
||||
|
||||
**Root Cause**: Cache keys use the STRING "auto" instead of resolved device name.
|
||||
|
||||
---
|
||||
|
||||
### Issue #2: Triple Caching System with No Sync (SEVERITY: CRITICAL)
|
||||
|
||||
**Three separate caches exist**:
|
||||
|
||||
1. **UnifiedModelInterface** (via ComfyUI model manager)
|
||||
- Cache keys: `"chatterbox_tts_cuda_English_path:model"`
|
||||
- String concatenation based
|
||||
|
||||
2. **SmartModelLoader**
|
||||
- Cache keys: `md5hash("chatterbox_tts_English_cuda")[:16]`
|
||||
- MD5 hash based
|
||||
- Used as fallback by F5-TTS and ChatterBox managers
|
||||
|
||||
3. **Manager Classes** (manager.py, f5tts_manager.py)
|
||||
- Manual dictionaries
|
||||
- Different key formats per manager
|
||||
|
||||
**The Problem**:
|
||||
```
|
||||
Model loaded → Cached in Unified → Also in SmartLoader → Also in Manager
|
||||
User clears VRAM → Only UnifiedModelInterface clears → Model still in other caches
|
||||
Next load → Stale model returned from SmartLoader cache → Device mismatch
|
||||
```
|
||||
|
||||
**Impact**:
|
||||
- Models can be loaded multiple times while already cached
|
||||
- Cache clearing commands don't fully clear models
|
||||
- Memory leaks possible if stale models accumulate
|
||||
- Confusing behavior for users clearing VRAM between inferences
|
||||
|
||||
---
|
||||
|
||||
### Issue #3: Device Type Mismatch (SEVERITY: MEDIUM)
|
||||
|
||||
**Two different device representations**:
|
||||
|
||||
```python
|
||||
# Unified interface uses: strings
|
||||
target_device = resolve_torch_device(config.device) # Returns "cuda"
|
||||
|
||||
# ComfyUI wrapper uses: torch.device objects
|
||||
self.device = torch.device("cuda") # torch.device object
|
||||
```
|
||||
|
||||
**The comparison fails**:
|
||||
```python
|
||||
if wrapper.current_device != target_device: # "cuda" != torch.device("cuda")
|
||||
# Type mismatch - comparison always True even if same device!
|
||||
```
|
||||
|
||||
**Impact**:
|
||||
- Device switch detection broken
|
||||
- Models might not move to correct device
|
||||
- Silent failures (model stays on old device)
|
||||
|
||||
---
|
||||
|
||||
### Issue #4: Cache Key Collision Risk (SEVERITY: MEDIUM)
|
||||
|
||||
**Current logic** (unified_model_interface.py, line 221):
|
||||
```python
|
||||
if config.model_path:
|
||||
components.append(f"path:{Path(config.model_path).name}")
|
||||
```
|
||||
|
||||
Uses basename only, not full path:
|
||||
```
|
||||
/models/TTS/chatterbox/English/ → cache component: "path:English"
|
||||
/backup/models/chatterbox/English/ → cache component: "path:English"
|
||||
# COLLISION!
|
||||
```
|
||||
|
||||
**Impact**:
|
||||
- Two different model directories with same name → same cache key
|
||||
- Randomly returns one or the other
|
||||
- Non-deterministic behavior
|
||||
- Difficult to debug
|
||||
|
||||
---
|
||||
|
||||
### Issue #5: Inconsistent Factory Signatures (SEVERITY: HIGH)
|
||||
|
||||
**Parameter inconsistency**:
|
||||
|
||||
| Factory | device | model_name | language | model_path | additional_params |
|
||||
|---------|--------|-----------|----------|-----------|------------------|
|
||||
| ChatterBox | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| F5-TTS | ✅ | ✅ | ⚠️ (ignored) | ✅ | ❌ |
|
||||
| Higgs Audio | ✅ | ✅ | ❌ | ✅ | ✅ |
|
||||
| RVC | ✅ | ❌ | ❌ | ✅ | ❌ |
|
||||
| VibeVoice | ✅ | ✅ | ❌ | ❌ | ✅ |
|
||||
| IndexTTS | ✅ | ❌ | ❌ | ✅ | ✅ |
|
||||
|
||||
**Problem**: Each factory has different parameter set. Adding new parameter requires updating individual factories.
|
||||
|
||||
**Also**: Default values inconsistent (Higgs Audio defaults device="cuda" while others use "auto").
|
||||
|
||||
---
|
||||
|
||||
### Issue #6: Five Different Error Handling Patterns (SEVERITY: MEDIUM)
|
||||
|
||||
```python
|
||||
# Pattern 1: ChatterBox - Raises RuntimeError
|
||||
raise RuntimeError("ChatterboxTTS not available - check installation")
|
||||
|
||||
# Pattern 2: F5-TTS - Delegates to SmartModelLoader (different exceptions)
|
||||
model, _ = smart_model_loader.load_model_if_needed(...)
|
||||
|
||||
# Pattern 3: VibeVoice - Soft failure (prints, continues)
|
||||
except ImportError as e:
|
||||
print(f"⚠️ Unified Interface: accelerate not available: {e}")
|
||||
|
||||
# Pattern 4: RVC - Direct ImportError
|
||||
from engines.rvc.impl.vc_infer_pipeline import get_vc
|
||||
|
||||
# Pattern 5: IndexTTS - Custom ImportError message
|
||||
raise ImportError(f"IndexTTS-2 dependencies not available. Error: {e}")
|
||||
```
|
||||
|
||||
**Impact**:
|
||||
- Callers can't predict exception types
|
||||
- Try/except blocks can't handle all cases
|
||||
- Different behaviors across engines for same failure scenario
|
||||
|
||||
---
|
||||
|
||||
### Issue #7: Device Resolution Duplication (SEVERITY: MEDIUM)
|
||||
|
||||
**Multiple device resolution functions with different logic**:
|
||||
|
||||
1. **torch_device_resolver.py** (authoritative):
|
||||
```python
|
||||
def resolve_torch_device(device: str) -> str:
|
||||
if device == "auto":
|
||||
if torch.backends.mps.is_available(): # ← Checks MPS
|
||||
return "mps"
|
||||
elif torch.cuda.is_available():
|
||||
return "cuda"
|
||||
```
|
||||
|
||||
2. **base_node.py** (partial):
|
||||
```python
|
||||
def resolve_device(self, device: str) -> str:
|
||||
if device == "auto":
|
||||
return "cuda" if torch.cuda.is_available() else "cpu"
|
||||
# ← MISSING MPS and XPU checks!
|
||||
```
|
||||
|
||||
**Impact**:
|
||||
- Different code paths return different devices
|
||||
- MPS (Apple Silicon) users get CPU fallback in some cases
|
||||
- Inconsistent behavior across node types
|
||||
|
||||
---
|
||||
|
||||
## PART 3: ROOT CAUSE ANALYSIS
|
||||
|
||||
Why did this happen?
|
||||
|
||||
1. **Evolutionary Development**: Unified interface added later, legacy systems still working → coexistence strategy chosen
|
||||
2. **Engine Diversity**: Each engine (ChatterBox, F5-TTS, Higgs, RVC) has different requirements → factories handle differently
|
||||
3. **Lack of Specification**: No defined contract for what a factory must implement
|
||||
4. **Incremental Adoption**: New engines use unified interface, but integration with old engines takes time
|
||||
|
||||
**None of this is "bad code"** - it's a natural architecture that emerges when adding new systems to existing ones.
|
||||
|
||||
---
|
||||
|
||||
## PART 4: DOES THIS NEED TO BE FIXED?
|
||||
|
||||
### Short Answer
|
||||
**For users**: System works. All engines load, all models work.
|
||||
|
||||
**For maintainers**: Accumulating technical debt that will cause problems as more engines are added.
|
||||
|
||||
### The Real Risk
|
||||
|
||||
The biggest risk is **in future maintenance**:
|
||||
|
||||
```
|
||||
Every new engine added will need:
|
||||
- Custom factory implementation
|
||||
- Custom error handling
|
||||
- Possibly custom device logic
|
||||
- Possibly custom recovery code (for Higgs-like corruption issues)
|
||||
|
||||
This complexity INCREASES with more engines.
|
||||
```
|
||||
|
||||
**Current**: 7 engines, manageable
|
||||
|
||||
**In 1 year**: 10-15 engines likely, architecture breaks down
|
||||
|
||||
### Scenarios Where Current Issues Cause Real Problems
|
||||
|
||||
1. **User with GPU+CPU hybrid setup** (or laptop with power management):
|
||||
- Device shifts from CUDA to CPU unexpectedly
|
||||
- Models reload incorrectly
|
||||
- Performance degrades mysteriously
|
||||
|
||||
2. **ComfyUI with tight VRAM budget**:
|
||||
- Clear VRAM → device resolves to CPU
|
||||
- Cache misses → reloads in CPU VRAM
|
||||
- Compounds memory pressure
|
||||
|
||||
3. **Adding new engine similar to Higgs** (requires special recovery):
|
||||
- Need to modify model_manager.py with new engine-specific code
|
||||
- Risk breaking existing engines' recovery logic
|
||||
|
||||
4. **Long-running generation sessions**:
|
||||
- Triple cache accumulation
|
||||
- Memory leaks from stale cached models
|
||||
- Slow degradation over 8+ hours
|
||||
|
||||
---
|
||||
|
||||
## PART 5: IMPROVEMENT RECOMMENDATIONS
|
||||
|
||||
### Tier 1: Quick Wins (1-2 hours, Low Risk)
|
||||
|
||||
These can be implemented immediately with minimal testing:
|
||||
|
||||
#### 1.1 Fix "auto" Device Instability
|
||||
**Change**: Cache resolved device in cache key, not "auto" string
|
||||
|
||||
```python
|
||||
# Before: Cache key includes "auto"
|
||||
cache_key = f"chatterbox_tts_auto"
|
||||
|
||||
# After: Resolve first
|
||||
resolved = resolve_torch_device(device)
|
||||
cache_key = f"chatterbox_tts_{resolved}"
|
||||
```
|
||||
|
||||
**Impact**: Eliminates unnecessary model reloads on device changes
|
||||
**Risk**: Very low - just earlier resolution
|
||||
|
||||
#### 1.2 Fix Device Comparison
|
||||
**Change**: Resolve device on both sides before comparison
|
||||
|
||||
```python
|
||||
# Before
|
||||
if wrapper.current_device != target_device: # String vs torch.device
|
||||
|
||||
# After
|
||||
from utils.device import resolve_torch_device
|
||||
target_resolved = resolve_torch_device(config.device)
|
||||
actual_resolved = resolve_torch_device(wrapper.current_device)
|
||||
if target_resolved != actual_resolved:
|
||||
```
|
||||
|
||||
**Impact**: Device movement actually works
|
||||
**Risk**: Very low - just standardization
|
||||
|
||||
#### 1.3 Consolidate Device Resolution
|
||||
**Change**: Have single `resolve_torch_device()` function, use everywhere
|
||||
|
||||
**Current**: base_node.py has own resolve_device() missing MPS/XPU checks
|
||||
|
||||
**Fix**: Replace all `resolve_device()` calls with `resolve_torch_device()`
|
||||
|
||||
**Impact**: Consistent behavior across all components
|
||||
**Risk**: Low - just replacing with more complete function
|
||||
|
||||
---
|
||||
|
||||
### Tier 2: Medium Effort (4-6 hours, Medium Risk)
|
||||
|
||||
These require more testing but are valuable:
|
||||
|
||||
#### 2.1 Unify Caching System
|
||||
**Change**: Remove SmartModelLoader and manager-specific caches, route through unified interface
|
||||
|
||||
**Current architecture**:
|
||||
```
|
||||
ComfyUI Manager ←─┐
|
||||
SmartModelLoader ├─ THREE SYSTEMS (no sync)
|
||||
Manager dicts ────┘
|
||||
```
|
||||
|
||||
**After**:
|
||||
```
|
||||
All caches → ComfyUI Manager (single source of truth)
|
||||
```
|
||||
|
||||
**Benefit**:
|
||||
- Cache clearing actually clears everything
|
||||
- No model duplication across caches
|
||||
- Simpler debugging
|
||||
|
||||
**Risk**: Medium - must ensure all code paths work through single system
|
||||
|
||||
#### 2.2 Fix Cache Key Collision
|
||||
**Change**: Hash full path, not just basename
|
||||
|
||||
```python
|
||||
# Before
|
||||
components.append(f"path:{Path(config.model_path).name}")
|
||||
|
||||
# After
|
||||
import hashlib
|
||||
path_hash = hashlib.md5(str(config.model_path).encode()).hexdigest()[:8]
|
||||
components.append(f"path:{path_hash}")
|
||||
```
|
||||
|
||||
**Impact**: Eliminates collision risk
|
||||
**Risk**: Low - just better hashing
|
||||
|
||||
#### 2.3 Standardize Error Handling
|
||||
**Change**: Create base exception class, use everywhere
|
||||
|
||||
```python
|
||||
class TTSModelLoadingError(Exception):
|
||||
"""Base exception for TTS model loading failures"""
|
||||
pass
|
||||
|
||||
class TTSModelNotFoundError(TTSModelLoadingError):
|
||||
pass
|
||||
|
||||
class TTSDeviceError(TTSModelLoadingError):
|
||||
pass
|
||||
```
|
||||
|
||||
Update all factories to raise consistent exceptions.
|
||||
|
||||
**Impact**: Callers can reliably catch and handle errors
|
||||
**Risk**: Low - just standardization
|
||||
|
||||
---
|
||||
|
||||
### Tier 3: Major Refactor (2-3 days, High Risk)
|
||||
|
||||
These provide architectural improvements but require extensive testing:
|
||||
|
||||
#### 3.1 Standardize Factory Signatures
|
||||
**Change**: Define common parameter interface
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class FactoryParameters:
|
||||
"""Standard parameters all factories accept"""
|
||||
device: str # Required
|
||||
model_name: str # Required
|
||||
language: Optional[str] = None # Optional
|
||||
model_path: Optional[str] = None # Optional
|
||||
repo_id: Optional[str] = None # Optional
|
||||
additional_params: Optional[Dict] = None # For engine-specific
|
||||
```
|
||||
|
||||
Each factory receives this dataclass, extracts needed fields.
|
||||
|
||||
**Benefit**:
|
||||
- Adding new factories is simpler
|
||||
- No parameter confusion
|
||||
- Extensible without modifying unified interface
|
||||
|
||||
**Risk**: High - changes all factories, needs full testing
|
||||
|
||||
#### 3.2 Engine Capability Registry
|
||||
**Change**: Instead of hardcoding recovery logic, engines register capabilities
|
||||
|
||||
```python
|
||||
ENGINE_CAPABILITIES = {
|
||||
"higgs_audio": {
|
||||
"supports_cuda_graphs": True,
|
||||
"recovery_handler": reset_higgs_cuda_graphs,
|
||||
"can_corrupt": True
|
||||
},
|
||||
"f5tts": {
|
||||
"supports_cuda_graphs": False,
|
||||
"recovery_handler": None,
|
||||
"can_corrupt": False
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Generic recovery logic uses registry.
|
||||
|
||||
**Benefit**:
|
||||
- Adding new engine with special needs is data-driven, not code-driven
|
||||
- model_manager.py doesn't need modification for new engines
|
||||
- Scales to 20+ engines
|
||||
|
||||
**Risk**: High - changes error recovery pattern, needs extensive testing
|
||||
|
||||
#### 3.3 Unified Fallback Handler
|
||||
**Change**: Extract ChatterBox's fallback logic to generic utility
|
||||
|
||||
ChatterBox has sophisticated fallback chain (local → HF → fallback language). This is valuable for any engine.
|
||||
|
||||
```python
|
||||
def intelligent_model_loader(
|
||||
primary_loader: Callable,
|
||||
fallback_languages: List[str],
|
||||
final_fallback: Callable
|
||||
) -> Any:
|
||||
"""Generic intelligent loader with fallback chain"""
|
||||
try:
|
||||
return primary_loader()
|
||||
except ImportError as e:
|
||||
if "401" in str(e):
|
||||
# Handle gated models
|
||||
pass
|
||||
# Try fallback languages
|
||||
for lang in fallback_languages:
|
||||
try:
|
||||
return primary_loader_with_lang(lang)
|
||||
except:
|
||||
continue
|
||||
# Final fallback
|
||||
return final_fallback()
|
||||
```
|
||||
|
||||
**Benefit**:
|
||||
- Other engines get robust fallback behavior
|
||||
- Consistent error recovery across all engines
|
||||
- ChatterBox logic is maintainable, not scattered
|
||||
|
||||
**Risk**: Medium - fallback patterns must be generic enough
|
||||
|
||||
---
|
||||
|
||||
## PART 6: IMPLEMENTATION PRIORITY MATRIX
|
||||
|
||||
```
|
||||
RISK ↑
|
||||
│
|
||||
H │ Tier 3.1 (Factory) Tier 3.2 (Registry) Tier 3.3 (Fallback)
|
||||
│ Signature Engine Capabilities Unified Handler
|
||||
│ [2-3 days] [2-3 days] [1-2 days]
|
||||
│
|
||||
M │ Tier 2.1 (Cache Unify) Tier 2.2 (Key Hash) Tier 2.3 (Errors)
|
||||
│ [4-6 hours] [2 hours] [2 hours]
|
||||
│
|
||||
L │ Tier 1.1-1.3
|
||||
│ Device & Resolution
|
||||
│ [1-2 hours]
|
||||
│
|
||||
└──────────────────────────────────────────────────────────→ EFFORT/REWARD
|
||||
|
||||
RECOMMENDATION ORDER:
|
||||
1. Tier 1 (all three) - Highest ROI, lowest risk
|
||||
2. Tier 2.1 - Unify caching (critical for maintenance)
|
||||
3. Tier 2.2, 2.3 - Polish improvements
|
||||
4. Tier 3 - Only if planning major expansion (10+ more engines)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## PART 7: RECOMMENDATION FOR THIS PROJECT
|
||||
|
||||
### Do you need to fix this?
|
||||
|
||||
**For current state (7 engines)**: NO
|
||||
- System works without errors
|
||||
- Legacy fallbacks cover edge cases
|
||||
- Not blocking any functionality
|
||||
|
||||
**For future (planning to add 10+ engines)**: YES
|
||||
- Complexity compounds with each engine
|
||||
- New contributors will struggle with inconsistent patterns
|
||||
- Technical debt becomes operational risk
|
||||
|
||||
### Suggested Approach
|
||||
|
||||
**Phase 1** (Next 1-2 weeks): Implement Tier 1 fixes
|
||||
- Solves "auto" device and comparison issues
|
||||
- Takes ~2 hours
|
||||
- Zero risk
|
||||
|
||||
**Phase 2** (Optional, Next month): Implement Tier 2.1
|
||||
- Unifies caching
|
||||
- Removes triple-system confusion
|
||||
- Takes ~6 hours
|
||||
- Requires testing
|
||||
|
||||
**Phase 3** (Only if adding multiple new engines): Implement Tier 3
|
||||
- Registers as architectural upgrade
|
||||
- Plan it as part of new engine addition
|
||||
- Amortize effort across multiple additions
|
||||
|
||||
### Decision Point
|
||||
|
||||
**Ask yourself**:
|
||||
1. Are you planning to add 10+ engines in the next 2 years? → Do Tier 3
|
||||
2. Is cache management causing issues in user reports? → Do Tier 2.1
|
||||
3. Is device handling causing issues? → Do Tier 1 immediately
|
||||
|
||||
If answers are NO, NO, NO → **Current system is fine. Monitor for issues.**
|
||||
|
||||
---
|
||||
|
||||
## PART 8: SPECIFIC FILE CHANGES SUMMARY
|
||||
|
||||
### Files to Modify (Tier 1)
|
||||
|
||||
1. **utils/device/torch_device_resolver.py**
|
||||
- Add caching for "auto" resolution
|
||||
- 10-15 lines changed
|
||||
|
||||
2. **utils/models/unified_model_interface.py**
|
||||
- Use resolved device in cache keys (line 259)
|
||||
- Fix device comparison (line 180)
|
||||
- 5-10 lines changed
|
||||
|
||||
3. **nodes/base/base_node.py**
|
||||
- Remove resolve_device() method, use torch_device_resolver
|
||||
- 3-5 lines changed
|
||||
|
||||
### Files to Modify (Tier 2.1)
|
||||
|
||||
1. **utils/models/comfyui_model_wrapper/model_manager.py**
|
||||
- Remove engine-specific recovery code
|
||||
- Add generic recovery registration
|
||||
|
||||
2. **utils/models/smart_loader.py**
|
||||
- Deprecate in favor of unified interface
|
||||
- Route through ComfyUI manager
|
||||
|
||||
3. **utils/models/manager.py**
|
||||
- Remove fallback to SmartModelLoader
|
||||
- Always use unified interface
|
||||
|
||||
---
|
||||
|
||||
## CONCLUSION
|
||||
|
||||
**The system is architecturally sound for current scope** (7 engines), but accumulated technical debt will become problematic at larger scale.
|
||||
|
||||
**Recommended action**:
|
||||
- Implement Tier 1 fixes now (low effort, immediate benefit)
|
||||
- Monitor for real-world issues
|
||||
- Plan Tier 2.1 if caching becomes problematic
|
||||
- Consider Tier 3 if project adds 10+ engines
|
||||
|
||||
The improvements would take a skilled developer 2-3 weeks for full implementation, spread across phases. Each phase independently provides value.
|
||||
|
||||
---
|
||||
|
||||
**Report prepared by**: Claude Code Analysis
|
||||
**Analysis method**: Deep codebase examination + cross-reference testing
|
||||
**Confidence level**: High (based on 633 lines of detailed code analysis)
|
||||
@@ -1,201 +0,0 @@
|
||||
# Post v4.4.0 Development Review
|
||||
|
||||
*Analysis of 32 local commits since v4.4.0 release*
|
||||
|
||||
## Executive Summary
|
||||
|
||||
Since the v4.4.0 release featuring the Silent Speech Analyzer, development has focused on three major areas: **Higgs Audio 2 integration**, **architectural refinement**, and **smart processing optimization**. These 32 commits represent significant enhancements that maintain the project's momentum toward becoming the ultimate universal TTS platform.
|
||||
|
||||
## Major New Features
|
||||
|
||||
### 🎯 **Higgs Audio 2 Engine Integration** (MAJOR FEATURE)
|
||||
|
||||
**Commits**: `7a7c205`, `fa0a6d8`, `c6d6efb`, `d750190`, `31b8aed`, `2c8f955`, `460da24`, `d94df09`
|
||||
|
||||
**What's New**:
|
||||
- **Complete Higgs Audio 2 TTS engine** with state-of-the-art voice cloning capabilities
|
||||
- **Real-time voice cloning** from 30+ second reference audio samples
|
||||
- **Multi-speaker conversation processing** with native character switching
|
||||
- **8 premium voice presets** (belinda, en_woman, en_man, chadwick, vex, etc.)
|
||||
|
||||
**Technical Implementation**:
|
||||
- **10,000+ lines of code** added for complete engine integration
|
||||
- **Modular architecture** following established F5-TTS/ChatterBox patterns
|
||||
- **Unified adapter integration** - works seamlessly with existing TTS Text/SRT nodes
|
||||
- **Automatic model downloading** to organized TTS/HiggsAudio/ structure
|
||||
- **Smart chunking system** for unlimited length text generation
|
||||
|
||||
**Voice Cloning Capabilities**:
|
||||
- High-quality voice replication from short audio samples
|
||||
- Multi-language support (English, Chinese dialects)
|
||||
- Configurable generation parameters (temperature, top_p, top_k, max_new_tokens)
|
||||
- Custom reference audio support beyond preset voices
|
||||
|
||||
### 🧠 **Smart Modular Chunk Combination System** (PERFORMANCE)
|
||||
|
||||
**Commits**: `bda23d2`, `19323f1`
|
||||
|
||||
**What's New**:
|
||||
- **ChunkCombiner utility** with intelligent auto-selection algorithms
|
||||
- **Smart analysis** of chunk split patterns (sentence boundaries, commas, forced splits)
|
||||
- **Per-junction analysis** for optimal audio segment combination
|
||||
- **Universal implementation** across all TTS engines (ChatterBox, F5-TTS, Higgs Audio)
|
||||
|
||||
**Technical Innovation**:
|
||||
- **202+ lines** of sophisticated chunk analysis logic
|
||||
- **ChunkTiming utility** providing standardized timing info integration
|
||||
- **Auto-selection method** that analyzes text structure for optimal processing
|
||||
- **Cache optimization** maintained through post-processing approach
|
||||
|
||||
**Performance Impact**:
|
||||
- Improved audio quality through intelligent segment boundaries
|
||||
- Reduced artifacts from poor chunk combinations
|
||||
- Consistent processing across all engines
|
||||
- Enhanced timing accuracy for SRT processing
|
||||
|
||||
### 🏗️ **Modular Processor Architecture Completion** (ARCHITECTURE)
|
||||
|
||||
**Commits**: `460da24`, `d94df09`, `c9f648f`
|
||||
|
||||
**What's New**:
|
||||
- **HiggsAudioSRTProcessor** and **HiggsAudioTTSProcessor** - dedicated engine processors
|
||||
- **Modular SRT overlap detection** across all TTS engines
|
||||
- **Clean delegation pattern** - Unified nodes now 23 lines vs 267 lines of hardcoded logic
|
||||
- **Architectural consistency** achieved across all engines
|
||||
|
||||
**Technical Benefits**:
|
||||
- **267 lines of hardcoded logic** extracted into dedicated processors
|
||||
- **Unified architecture** - all engines now follow identical delegation patterns
|
||||
- **Easier maintenance** and debugging through modular separation
|
||||
- **Consistent error handling** and processing across engines
|
||||
|
||||
## Quality of Life Improvements
|
||||
|
||||
### 📊 **Enhanced User Experience**
|
||||
|
||||
**Progress Visualization**:
|
||||
- **tqdm progress bars** added to Higgs Audio generation (`83c5274`)
|
||||
- Real-time feedback for long-running operations
|
||||
- Better user awareness of processing status
|
||||
|
||||
**Improved Documentation**:
|
||||
- **Detailed tooltips** for Higgs Audio Engine parameters (`240fbb7`)
|
||||
- **Multi-speaker mode recommendations** for optimal usage (`1235faf`)
|
||||
- Clear guidance on parameter usage and expected results
|
||||
|
||||
**Parameter Optimization**:
|
||||
- **Fixed max_new_tokens minimum** from 128 to 1 (`40c1da4`)
|
||||
- More flexible generation control for varied use cases
|
||||
- Better parameter validation and user guidance
|
||||
|
||||
### 🔧 **Critical Bug Fixes**
|
||||
|
||||
**Cache System Fixes**:
|
||||
- **Critical cache invalidation bug** fixed for Higgs Audio (`5ee6b10`, `ba8bace`)
|
||||
- **Centralized cache system** improvements for consistency
|
||||
- **Parameter flow fixes** for proper SRT node configuration (`083d865`)
|
||||
|
||||
**Engine Integration Fixes**:
|
||||
- **F5-TTS bundled integration** improvements (`34db557`)
|
||||
- **ChatterBox model detection** enhancements
|
||||
- **Character switching** fixes for narrator voice priority (`9e11f12`)
|
||||
|
||||
**Initialization and Parameter Flow**:
|
||||
- **Engine initialization bypass bug** fixed (`a676dad`, `ab90051`)
|
||||
- **SRT stretch_to_fit mode** parameter format corrections (`1912250`)
|
||||
- **Timing report generation** using modular calculation system (`762e472`)
|
||||
|
||||
## Technical Architecture Improvements
|
||||
|
||||
### Unified Processing Pipeline
|
||||
|
||||
**Before**: Each engine had custom SRT processing with duplicated logic
|
||||
**After**: Clean delegation to dedicated engine processors
|
||||
|
||||
**Benefits**:
|
||||
- Consistent behavior across all engines
|
||||
- Easier debugging and maintenance
|
||||
- Reduced code duplication
|
||||
- Modular testing capabilities
|
||||
|
||||
### Enhanced Cache Management
|
||||
|
||||
**Centralized Cache System**:
|
||||
- Universal cache invalidation across all engines
|
||||
- Content-based hashing for consistency
|
||||
- Engine-specific optimizations while maintaining unified interface
|
||||
|
||||
### Voice Management Evolution
|
||||
|
||||
**Higgs Audio Voice System**:
|
||||
- **Removed voice_preset system** in favor of flexible reference text handling (`a112ee5`)
|
||||
- **Improved reference text processing** for better voice cloning results
|
||||
- **Character switching compatibility** with existing voice management
|
||||
|
||||
## Code Quality and Maintenance
|
||||
|
||||
### Repository Management
|
||||
|
||||
**Cleanup and Organization**:
|
||||
- **ComfyUI reference folders** properly gitignored (`fe9dcd6`, `24982e5`)
|
||||
- **Requirements.txt updates** for Higgs Audio dependencies (`00483cc`)
|
||||
- **README consolidation** for better feature presentation (`4111d64`)
|
||||
|
||||
**Merge Management**:
|
||||
- **External contributions** properly integrated (`6f3452d`, `894692c`)
|
||||
- Clean merge handling without conflicts
|
||||
|
||||
### Development Process
|
||||
|
||||
**32 Commits** of focused development since v4.4.0:
|
||||
- **1 Major Feature** (Higgs Audio 2 integration)
|
||||
- **2 Performance Systems** (Smart chunking, modular processors)
|
||||
- **15+ Bug fixes** and stability improvements
|
||||
- **Multiple UX enhancements**
|
||||
|
||||
## Impact Analysis
|
||||
|
||||
### Feature Completeness
|
||||
|
||||
The addition of Higgs Audio 2 completes the "Big Three" TTS engine integration:
|
||||
1. ✅ **ChatterBox** - Fast, reliable, multilingual
|
||||
2. ✅ **F5-TTS** - High-quality, speech editing
|
||||
3. ✅ **Higgs Audio 2** - Voice cloning, multi-speaker
|
||||
|
||||
### Performance Optimization
|
||||
|
||||
- **Smart chunking** reduces audio artifacts
|
||||
- **Modular processors** improve maintainability
|
||||
- **Enhanced caching** provides instant regeneration
|
||||
- **Progress feedback** improves user experience
|
||||
|
||||
### Architecture Maturity
|
||||
|
||||
- **Unified delegation pattern** completed across all engines
|
||||
- **Modular utilities** enable consistent behavior
|
||||
- **Clean separation** of concerns for easier development
|
||||
- **Future-proof design** for additional engine integration
|
||||
|
||||
## Upcoming Version Readiness
|
||||
|
||||
These 32 commits represent substantial improvements worthy of a **v4.5.0 release**:
|
||||
|
||||
**Major Features**: Higgs Audio 2 integration with voice cloning
|
||||
**Performance**: Smart chunk combination system
|
||||
**Architecture**: Modular processor completion
|
||||
**Stability**: 15+ critical bug fixes
|
||||
**UX**: Enhanced progress feedback and documentation
|
||||
|
||||
## Conclusion
|
||||
|
||||
The post-v4.4.0 development cycle demonstrates continued innovation and architectural refinement. The **Higgs Audio 2 integration** brings cutting-edge voice cloning capabilities, while the **smart chunking system** and **modular processor architecture** ensure robust, maintainable code.
|
||||
|
||||
These improvements maintain the project's position as the most comprehensive TTS platform available, with three world-class engines unified under a single, intuitive interface.
|
||||
|
||||
---
|
||||
|
||||
*Review Date*: August 21, 2025
|
||||
*Commits Analyzed*: 32 local commits since v4.4.0
|
||||
*Major Features Added*: 1 (Higgs Audio 2)
|
||||
*Performance Systems*: 2 (Smart chunking, Modular processors)
|
||||
*Bug Fixes*: 15+
|
||||
@@ -210,94 +210,4 @@ def generate_speech(self, TTS_engine, ...):
|
||||
|
||||
The refactoring successfully transformed a collection of engine-specific nodes into a unified, extensible architecture while preserving all existing functionality. The smart delegation pattern ensures zero feature loss while providing a clean foundation for future engine integrations.
|
||||
|
||||
**Next Steps**: Test thoroughly, add RVC engine support, and potentially submit to ComfyUI Manager as the new universal TTS solution.
|
||||
|
||||
---
|
||||
|
||||
## Future Refactoring Tasks
|
||||
|
||||
### Architecture Inconsistency: Processor Location
|
||||
|
||||
**Issue Identified**: 2025-01-28
|
||||
|
||||
CosyVoice and IndexTTS have TTS processors in a different location than all other engines, causing inconsistent import patterns and architecture confusion.
|
||||
|
||||
**Current State:**
|
||||
|
||||
**Standard Pattern (9 engines):**
|
||||
- TTS processor location: `nodes/engine_name/engine_processor.py`
|
||||
- Import pattern: File-based import using `importlib.util.spec_from_file_location()`
|
||||
- Examples:
|
||||
- `nodes/qwen3_tts/qwen3_tts_processor.py`
|
||||
- `nodes/step_audio_editx/step_audio_editx_processor.py`
|
||||
- `nodes/vibevoice/vibevoice_processor.py`
|
||||
- `nodes/higgs_audio/higgs_audio_tts_processor.py`
|
||||
- `nodes/chatterbox_official_23lang/chatterbox_official_23lang_processor.py`
|
||||
|
||||
**Inconsistent Pattern (2 engines):**
|
||||
- TTS processor location: `engines/processors/`
|
||||
- Import pattern: Package import (`from engines.processors import`)
|
||||
- Examples:
|
||||
- `engines/processors/cosyvoice_processor.py`
|
||||
- `engines/processors/index_tts_processor.py`
|
||||
|
||||
**Why This Is a Problem:**
|
||||
|
||||
1. **Confusing Architecture**: Why are some TTS processors in `engines/processors/` and some in `nodes/engine_name/`?
|
||||
2. **Inconsistent Import Patterns**: Some use package imports, some use file-based imports
|
||||
3. **Documentation Mismatch**: The implementation guide assumes processors are in `nodes/`
|
||||
4. **Harder to Maintain**: New developers don't know which pattern to follow
|
||||
5. **Breaks Conventions**: All other engines follow the standard pattern
|
||||
|
||||
**Proposed Fix:**
|
||||
|
||||
Move TTS processors to match the standard architecture:
|
||||
|
||||
```bash
|
||||
# Move CosyVoice processor
|
||||
engines/processors/cosyvoice_processor.py → nodes/cosyvoice/cosyvoice_processor.py
|
||||
|
||||
# Move IndexTTS processor
|
||||
engines/processors/index_tts_processor.py → nodes/index_tts/index_tts_processor.py
|
||||
```
|
||||
|
||||
**Files to Update:**
|
||||
|
||||
1. **Move processor files** to `nodes/engine_name/`
|
||||
|
||||
2. **Update imports in SRT processors:**
|
||||
- `nodes/cosyvoice/cosyvoice_srt_processor.py` line 34
|
||||
- `nodes/index_tts/index_tts_srt_processor.py` (if exists)
|
||||
|
||||
Change from:
|
||||
```python
|
||||
from engines.processors.cosyvoice_processor import CosyVoiceProcessor
|
||||
```
|
||||
|
||||
To file-based import:
|
||||
```python
|
||||
import importlib.util
|
||||
processor_path = os.path.join(current_dir, "cosyvoice_processor.py")
|
||||
processor_spec = importlib.util.spec_from_file_location("cosyvoice_processor_module", processor_path)
|
||||
processor_module = importlib.util.module_from_spec(processor_spec)
|
||||
processor_spec.loader.exec_module(processor_module)
|
||||
CosyVoiceProcessor = processor_module.CosyVoiceProcessor
|
||||
```
|
||||
|
||||
3. **Update any other imports** that reference `engines.processors.cosyvoice_processor` or `engines.processors.index_tts_processor`
|
||||
|
||||
4. **Update PROJECT_INDEX.md** to reflect new locations
|
||||
|
||||
5. **Test thoroughly** to ensure no import breakage
|
||||
|
||||
**Priority**: Medium - Not breaking functionality, but important for maintainability and consistency
|
||||
|
||||
**Estimated Effort**: 1-2 hours (move files, update imports, test)
|
||||
|
||||
**Benefits:**
|
||||
- Consistent architecture across all engines
|
||||
- Easier to understand and maintain
|
||||
- Matches documentation and implementation guide
|
||||
- Clearer separation of concerns (engines/ = low-level, nodes/ = high-level orchestration)
|
||||
|
||||
**Note**: The `engines/processors/` directory can be removed after this refactoring, or kept for future use if there's a valid architectural reason (currently unclear).
|
||||
**Next Steps**: Test thoroughly, add RVC engine support, and potentially submit to ComfyUI Manager as the new universal TTS solution.
|
||||
@@ -1,87 +0,0 @@
|
||||
# SRT Edit Tag Batch Processing Pattern
|
||||
|
||||
## Problem
|
||||
Edit post-processing was happening per-subtitle, causing the editing engine to load/unload multiple times per SRT generation, wasting time and VRAM.
|
||||
|
||||
## Solution
|
||||
Batch all edit processing at the END of SRT generation, after ALL subtitles are generated.
|
||||
|
||||
## Implementation Pattern
|
||||
|
||||
### 1. Adapter Layer (e.g., `vibevoice_adapter.py`)
|
||||
In `generate_with_pause_tags()`:
|
||||
- Extract edit tags even when NO pause tags present
|
||||
- Return segment dict format `[{waveform, text, edit_tags, ...}]` when edit tags exist
|
||||
- Return plain tensor when NO edit tags (backwards compatibility)
|
||||
|
||||
### 2. Regular Processor (e.g., `step_audio_editx_processor.py`)
|
||||
In `_process_character_switching()`:
|
||||
```python
|
||||
# Apply edit post-processing ONLY when NOT in SRT mode
|
||||
if not self._srt_mode:
|
||||
audio_segments = apply_edit_post_processing(
|
||||
audio_segments,
|
||||
self.config,
|
||||
pre_loaded_engine=self.adapter.engine
|
||||
)
|
||||
```
|
||||
|
||||
### 3. SRT Processor (e.g., `vibevoice_srt_processor.py`, `step_audio_editx_srt_processor.py`)
|
||||
|
||||
In `_process_all_subtitles()`:
|
||||
|
||||
```python
|
||||
# Initialize collection for batch processing
|
||||
all_segments_for_editing = []
|
||||
|
||||
# Loop through subtitles
|
||||
for i, subtitle in enumerate(subtitles):
|
||||
segments = generate_subtitle(subtitle)
|
||||
|
||||
# Check if segments have edit tags
|
||||
if isinstance(segments, list) or has_edit_tags(segments):
|
||||
# Tag with subtitle index for later reassembly
|
||||
for seg in segments:
|
||||
seg['subtitle_index'] = i
|
||||
all_segments_for_editing.extend(segments)
|
||||
audio_segments[i] = None # Placeholder
|
||||
else:
|
||||
# No edit tags - store directly
|
||||
audio_segments[i] = combine_segments(segments)
|
||||
|
||||
# BATCH PROCESS all edits at once
|
||||
if all_segments_for_editing:
|
||||
print(f"🎨 Applying edit post-processing to {len(all_segments_for_editing)} segment(s) from all subtitles...")
|
||||
processed = apply_edit_post_processing(
|
||||
all_segments_for_editing,
|
||||
config,
|
||||
pre_loaded_engine=engine
|
||||
)
|
||||
|
||||
# Group back by subtitle_index
|
||||
segments_by_subtitle = group_by_subtitle_index(processed)
|
||||
|
||||
# Combine and store
|
||||
for sub_idx, segs in segments_by_subtitle.items():
|
||||
audio_segments[sub_idx] = combine_segments(segs)
|
||||
```
|
||||
|
||||
### 4. Subtitle Processing Functions
|
||||
In functions like `_process_custom_character_switching_subtitle_with_params()`:
|
||||
- Return segment dicts when edit tags present (don't apply edits)
|
||||
- Return plain tensor when no edit tags
|
||||
- Let caller handle batch processing
|
||||
|
||||
## Benefits
|
||||
- Edit engine loads ONCE instead of once per subtitle
|
||||
- Faster SRT generation
|
||||
- Less VRAM thrashing
|
||||
- Same quality output
|
||||
|
||||
## Engines Implemented
|
||||
- ✅ VibeVoice SRT
|
||||
- ✅ Step Audio EditX SRT
|
||||
- ✅ Higgs Audio SRT
|
||||
|
||||
## TODO
|
||||
- ChatterBox 23-Lang (needs edit tag support first)
|
||||
@@ -1,464 +0,0 @@
|
||||
# Step Audio EditX Engine Implementation Plan
|
||||
|
||||
## Executive Summary
|
||||
|
||||
**Implementation Approach**: Bundle Step Audio EditX code (like IndexTTS pattern) with TTS Suite integration
|
||||
|
||||
**Key Decisions:**
|
||||
- ✅ **Bundle code** in `step_audio_editx_impl/` (NOT pip install)
|
||||
- ✅ **Skip Whisper** dependency (only needed for external audio transcription)
|
||||
- ✅ **Unpinned transformers** (patch if needed via TTS Suite system)
|
||||
- ✅ **Progress bars** via ComfyUI (UX improvement)
|
||||
- ✅ **Manual transcripts** for Audio Editor (Whisper integration later)
|
||||
- ✅ **Phased implementation**: Clone → Edit → SRT
|
||||
|
||||
**What Step Audio EditX Is:**
|
||||
- 3B parameter QWen-based LLM for semantic generation
|
||||
- Custom dual-codebook tokenizer (VQ02 + VQ06)
|
||||
- CosyVoice vocoder for final audio generation (NOT full CosyVoice)
|
||||
- Unique emotion/style/speed editing capabilities
|
||||
- Worth implementing alongside future CosyVoice engine
|
||||
|
||||
---
|
||||
|
||||
## Reference Implementations
|
||||
|
||||
**Local copies available for reference:**
|
||||
- **Original (stepfun-ai)**: `IgnoredForGitHubDocs/For_reference/Step-Audio-EditX/`
|
||||
- **Saganaki22's ComfyUI**: `IgnoredForGitHubDocs/For_reference/ComfyUI-Step_Audio_EditX_TTS/`
|
||||
|
||||
---
|
||||
|
||||
## Analysis of Both Implementations
|
||||
|
||||
### Original Step Audio EditX (stepfun-ai)
|
||||
- **Core Features**:
|
||||
- Zero-shot voice cloning (3-30s reference audio)
|
||||
- Audio editing: emotion, style, speed, paralinguistics
|
||||
- Supports Mandarin, English, Sichuanese, Cantonese, Japanese, Korean
|
||||
- Iterative editing (n_edit_iter parameter)
|
||||
- Denoise and VAD
|
||||
- Polyphone pronunciation control
|
||||
- 3B parameter model with dual-codebook tokenizer
|
||||
|
||||
- **Architecture**:
|
||||
- `tokenizer.py` (StepAudioTokenizer) - Dual-codebook tokenizer
|
||||
- `tts.py` (StepAudioTTS) - Main TTS engine with clone() and edit() methods
|
||||
- `model_loader.py` - Model loading from HuggingFace/ModelScope/local
|
||||
- `config/edit_config.py` - Edit type configurations
|
||||
- Uses bundled `funasr_detach/` for FunASR models (paraformer tokenizer)
|
||||
- CosyVoice vocoder for audio generation (24kHz output)
|
||||
|
||||
### Saganaki22's ComfyUI Implementation
|
||||
- **Native Features**: All of the above
|
||||
- **Custom Additions (NON-NATIVE)**:
|
||||
1. **Longform chunking system** (`core/longform_chunker.py`):
|
||||
- Splits long text at sentence boundaries
|
||||
- Estimates tokens (1 token ≈ 4 chars)
|
||||
- Creates overlapping chunks
|
||||
- This is a ComfyUI-specific optimization NOT in original
|
||||
|
||||
2. **Audio stitching** (`core/audio_stitcher.py`):
|
||||
- Concatenates audio chunks from longform generation
|
||||
- Crossfade support
|
||||
- NOT part of original implementation
|
||||
|
||||
3. **Progress bar integration**:
|
||||
- ComfyUI progress tracking during clone generation
|
||||
- NOT in original (original has no progress callbacks)
|
||||
|
||||
4. **Model wrapper architecture**:
|
||||
- `StepAudioModelWrapper` with to_cpu()/to_cuda() methods
|
||||
- CUDA graph cleanup logic
|
||||
- More sophisticated VRAM management than original
|
||||
|
||||
**CRITICAL**: Speed control (faster/slower) IS native - it's in the original edit_config.py, so this is NOT a custom addition.
|
||||
|
||||
---
|
||||
|
||||
## Recommended Implementation Approach
|
||||
|
||||
### Use Original as Base (NOT Saganaki22)
|
||||
|
||||
Saganaki22's implementation is excellent for ComfyUI but adds non-native features. Since you want faithful implementation, we start fresh.
|
||||
|
||||
### Phase 1: Core Engine Implementation
|
||||
|
||||
#### Files to Create:
|
||||
|
||||
1. **`engines/step_audio_editx/step_audio_editx.py`**
|
||||
- Main engine wrapper
|
||||
- Wraps original `StepAudioTTS` class
|
||||
- Implements `.to()` method for ComfyUI VRAM management
|
||||
- Device checking before generation
|
||||
- Sample rate: 24000 Hz (CosyVoice native)
|
||||
|
||||
2. **`engines/step_audio_editx/step_audio_downloader.py`**
|
||||
- Uses `utils/downloads/unified_downloader.py`
|
||||
- Downloads Step-Audio-EditX and Step-Audio-Tokenizer
|
||||
- Organized structure: `models/TTS/step_audio_editx/`
|
||||
|
||||
3. **`engines/step_audio_editx/__init__.py`**
|
||||
- Engine initialization
|
||||
- Imports
|
||||
|
||||
4. **`engines/step_audio_editx/original_impl/`**
|
||||
- Copy entire original Step Audio EditX implementation here
|
||||
- Includes: `tts.py`, `tokenizer.py`, `model_loader.py`, `config/`, `funasr_detach/`, etc.
|
||||
- Keep it isolated so it doesn't conflict with existing dependencies
|
||||
|
||||
#### Key Implementation Notes:
|
||||
|
||||
**Audio Format**:
|
||||
- Original outputs: `torch.Tensor` shape `[samples]` or `[1, samples]`
|
||||
- Sample rate: 24000 Hz
|
||||
- Return format: TTS Audio Suite expects `[1, samples]`
|
||||
|
||||
**Generation Methods**:
|
||||
```python
|
||||
def clone(self, prompt_wav_path, prompt_text, target_text, temperature=0.7, do_sample=True, max_new_tokens=4096):
|
||||
# Call original implementation
|
||||
pass
|
||||
|
||||
def edit(self, input_audio_path, audio_text, edit_type, edit_info=None, text=None):
|
||||
# Edit mode - temperature/do_sample/max_new_tokens are HARDCODED in original
|
||||
# Parameters: edit_type (emotion/style/speed/paralinguistic/denoising)
|
||||
# Returns audio_tensor, sample_rate
|
||||
pass
|
||||
```
|
||||
|
||||
**VRAM Management** (CRITICAL):
|
||||
- Implement `.to(device)` method
|
||||
- Move `self.llm` (3B parameter model) between devices
|
||||
- Move `self.cosy_model` (vocoder) between devices
|
||||
- Check device before generation and reload if needed
|
||||
- Follow patterns from IndexTTS implementation
|
||||
|
||||
---
|
||||
|
||||
### Phase 2: Node Implementation
|
||||
|
||||
#### Files to Create:
|
||||
|
||||
1. **`nodes/engines/step_audio_editx_engine_node.py`**
|
||||
- Configuration node for Step Audio EditX
|
||||
- Parameters:
|
||||
- `model_path` - path to Step-Audio-EditX
|
||||
- `device` - cuda/cpu
|
||||
- `torch_dtype` - bfloat16/float16/float32
|
||||
- `quantization` - none/int4/int8
|
||||
- `attention_mechanism` - sdpa/eager/flash_attn
|
||||
- Returns: ENGINE config
|
||||
|
||||
2. **`nodes/step_audio_editx/step_audio_editx_processor.py`**
|
||||
- Main TTS processor (internal, used by Unified TTS Text node)
|
||||
- Handles:
|
||||
- Character switching with `[CharacterName]` tags
|
||||
- Pause tags with `[pause:1.5s]` syntax
|
||||
- Caching via `utils/audio/cache.py`
|
||||
- Voice discovery via `utils/voice/discovery.py`
|
||||
- **Longform chunking via `utils/audio/chunk_combiner.py`** (TTS Suite unified)
|
||||
- Sequential generation with interrupt handling
|
||||
|
||||
3. **`nodes/step_audio_editx_special/step_audio_editx_audio_editor_node.py`** (Special Feature)
|
||||
- Dedicated "🎨 Step Audio EditX - Audio Editor" node
|
||||
- This is NOT voice conversion - it's specialized audio manipulation
|
||||
- **CRITICAL**: Requires `audio_text` parameter (transcript of input audio)
|
||||
- User must provide transcript manually (no Whisper transcription)
|
||||
- Future: When Whisper engine is added, can auto-feed transcripts
|
||||
- Parameters:
|
||||
- `input_audio` - audio to edit (0.5-30s limit)
|
||||
- `audio_text` - REQUIRED transcript of input audio
|
||||
- `edit_type` - emotion/style/speed/paralinguistic/denoising
|
||||
- `emotion` - happy/sad/angry/excited/etc (14 options)
|
||||
- `style` - whisper/serious/child/older/etc (32 options)
|
||||
- `speed` - faster/slower/more faster/more slower
|
||||
- `paralinguistic` - [Laughter]/[Breathing]/[Sigh]/etc (10 options)
|
||||
- `denoising` - denoise/vad
|
||||
- `n_edit_iterations` - 1-5 (strength through iteration)
|
||||
- `paralinguistic_text` - where to insert effect
|
||||
- Returns edited audio
|
||||
|
||||
---
|
||||
|
||||
### Phase 3: Unified Systems Integration
|
||||
|
||||
#### Integration Points:
|
||||
|
||||
1. **Unified Model Loading**:
|
||||
```python
|
||||
from utils.models.unified_model_interface import UnifiedModelInterface
|
||||
|
||||
# Register Step Audio EditX
|
||||
unified_model_interface.register_model(
|
||||
engine_type="step_audio_editx",
|
||||
model=step_audio_engine,
|
||||
model_info=ModelInfo(...)
|
||||
)
|
||||
```
|
||||
|
||||
2. **Character System**:
|
||||
- Use `utils/text/character_parser.py` for `[CharacterName]` tags
|
||||
- Use `utils/voice/discovery.py` for voice file loading
|
||||
- Support both .wav and .txt voice references
|
||||
|
||||
3. **Pause Tags**:
|
||||
- Use `utils/text/pause_processor.py` for `[pause:xx]` syntax
|
||||
- Integrate with audio generation pipeline
|
||||
|
||||
4. **Caching**:
|
||||
```python
|
||||
from utils.audio/cache import UnifiedCacheManager
|
||||
from utils.audio.audio_hash import create_content_hash
|
||||
|
||||
cache_key = create_content_hash(
|
||||
text=text,
|
||||
voice_path=voice_path,
|
||||
engine="step_audio_editx",
|
||||
parameters={...}
|
||||
)
|
||||
```
|
||||
|
||||
5. **Interrupt Handling** (CRITICAL):
|
||||
```python
|
||||
import comfy.model_management as model_management
|
||||
|
||||
# Before each generation
|
||||
if model_management.interrupt_processing:
|
||||
raise InterruptedError("Step Audio EditX generation interrupted")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Phase 4: Dependencies
|
||||
|
||||
#### Add to `requirements.txt`:
|
||||
```txt
|
||||
# Step Audio EditX dependencies
|
||||
librosa>=0.10.2
|
||||
soundfile>=0.12.1 # Critical for ComfyUI audio preview
|
||||
onnxruntime>=1.17.0
|
||||
transformers>=4.30.0 # Unpinned - use TTS Suite transformer patches if needed
|
||||
sentencepiece>=0.1.99
|
||||
omegaconf>=2.3.0
|
||||
hyperpyyaml>=1.2.2
|
||||
# NOTE: openai-whisper NOT required for TTS (only for external audio transcription)
|
||||
# Future: When Whisper transcription engine is added, it can feed Audio Editor node
|
||||
```
|
||||
|
||||
**Dependency Strategy:**
|
||||
- ✅ **Bundle code** (like IndexTTS) - not pip installable
|
||||
- ✅ **Skip Whisper** - only needed for external audio → edit workflow
|
||||
- ✅ **Unpinned transformers** - patch via TTS Suite system if 4.54+ breaks
|
||||
- ✅ **Windows-friendly** - no complex model downloads for basic TTS
|
||||
|
||||
---
|
||||
|
||||
### Phase 5: SRT Implementation (Later)
|
||||
|
||||
**Strategy**: Sequential processing with language grouping
|
||||
- Group SRT segments by language (Mandarin/English/Japanese/Korean/etc)
|
||||
- Process each language group sequentially
|
||||
- No parallel processing (model doesn't support it well)
|
||||
- Support character switching within segments
|
||||
- Use timing modes from `utils/timing/`
|
||||
|
||||
**File**: `nodes/step_audio_editx/step_audio_editx_srt_processor.py`
|
||||
|
||||
---
|
||||
|
||||
## Key Differences from Saganaki22 Implementation
|
||||
|
||||
| Feature | Original | Saganaki22 | Our Implementation |
|
||||
|---------|----------|------------|-------------------|
|
||||
| Longform chunking | ❌ No | ✅ Yes (custom) | ✅ Yes (TTS Suite unified) |
|
||||
| Progress bars | ❌ No | ✅ Yes (clone only) | ✅ Yes (ComfyUI standard) |
|
||||
| Whisper transcription | ⚠️ Optional (ASR) | ⚠️ Optional (ASR) | ❌ Skip (future: Whisper engine) |
|
||||
| Audio stitching | ❌ No | ✅ Yes (custom) | ❌ No (not native) |
|
||||
| Iterative editing | ✅ Yes | ✅ Yes | ✅ Yes (native) |
|
||||
| Speed control | ✅ Yes | ✅ Yes | ✅ Yes (native) |
|
||||
| Character switching | ❌ No | ❌ No | ✅ Yes (TTS Suite feature) |
|
||||
| Pause tags | ❌ No | ❌ No | ✅ Yes (TTS Suite feature) |
|
||||
| Caching | ❌ No | ❌ No | ✅ Yes (TTS Suite feature) |
|
||||
| Interrupt handling | ❌ No | ❌ No | ✅ Yes (TTS Suite feature) |
|
||||
|
||||
---
|
||||
|
||||
## Critical Implementation Requirements
|
||||
|
||||
### 1. VRAM Management (from guide lines 96-264)
|
||||
- Implement `.to(device)` method
|
||||
- Move LLM (self.llm) and vocoder (self.cosy_model) between devices
|
||||
- Check device before generation
|
||||
- Reload from CPU to CUDA if offloaded
|
||||
- Search for existing wrapper before creating new ones
|
||||
|
||||
### 2. Clear VRAM Integration
|
||||
- Register with ComfyUI model management
|
||||
- Support "Clear VRAM" button functionality
|
||||
- Prevent device mismatch errors
|
||||
|
||||
### 3. Interrupt Handling (CRITICAL)
|
||||
- Check `model_management.interrupt_processing` before each generation
|
||||
- In SRT mode: check before each segment
|
||||
- Allow users to cancel long operations
|
||||
|
||||
### 4. Audio Limitations
|
||||
- **Clone mode**: No strict length limit (depends on max_new_tokens)
|
||||
- **Edit mode**: 0.5-30 seconds HARD LIMIT
|
||||
- Validate audio duration before processing
|
||||
|
||||
### 5. Language Support
|
||||
- Mandarin (primary)
|
||||
- English
|
||||
- Sichuanese dialect
|
||||
- Cantonese dialect
|
||||
- Japanese
|
||||
- Korean
|
||||
- Add `[Sichuanese]`, `[Cantonese]`, `[Japanese]`, `[Korean]` tag support
|
||||
|
||||
### 6. Edit Types and Options
|
||||
All options are from original `config/edit_config.py`:
|
||||
|
||||
**Emotions** (14):
|
||||
- happy, sad, angry, excited, calm, fearful, surprised, disgusted
|
||||
- confusion, empathy, embarrass, depressed, coldness, admiration
|
||||
|
||||
**Styles** (32):
|
||||
- whisper, serious, child, older, girl, pure, sister, sweet
|
||||
- exaggerated, ethereal, generous, recite, act_coy, warm, shy
|
||||
- comfort, authority, chat, radio, soulful, gentle, story, vivid
|
||||
- program, news, advertising, roar, murmur, shout, deeply, loudly
|
||||
- arrogant, friendly
|
||||
|
||||
**Speed** (4):
|
||||
- faster, slower, more faster, more slower
|
||||
|
||||
**Paralinguistics** (10):
|
||||
- [Breathing], [Laughter], [Surprise-oh], [Confirmation-en]
|
||||
- [Uhm], [Surprise-ah], [Surprise-wa], [Sigh]
|
||||
- [Question-ei], [Dissatisfaction-hnn]
|
||||
|
||||
**Denoising** (2):
|
||||
- denoise, vad
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
```
|
||||
engines/step_audio_editx/
|
||||
├── __init__.py
|
||||
├── step_audio_editx.py (main wrapper - follows IndexTTS pattern)
|
||||
├── step_audio_editx_downloader.py (unified downloader integration)
|
||||
└── step_audio_editx_impl/ (bundled original codebase - like IndexTTS)
|
||||
├── tts.py (StepAudioTTS - main engine)
|
||||
├── tokenizer.py (StepAudioTokenizer - dual-codebook)
|
||||
├── model_loader.py (ModelSource, model_loader)
|
||||
├── config/
|
||||
│ ├── edit_config.py (edit types/options)
|
||||
│ └── prompts.py
|
||||
├── funasr_detach/ (bundled FunASR - custom fork)
|
||||
└── stepvocoder/cosyvoice2/ (CosyVoice vocoder)
|
||||
|
||||
nodes/engines/
|
||||
└── step_audio_editx_engine_node.py (config node)
|
||||
|
||||
nodes/step_audio_editx/
|
||||
├── step_audio_editx_processor.py (TTS processor - internal)
|
||||
└── step_audio_editx_srt_processor.py (SRT processor - internal, later)
|
||||
|
||||
nodes/step_audio_editx_special/
|
||||
└── step_audio_editx_audio_editor_node.py (🎨 Audio Editor - specialized manipulation)
|
||||
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Testing Checklist
|
||||
|
||||
### Phase 1 Testing (Basic TTS):
|
||||
- [ ] Model downloads correctly
|
||||
- [ ] Basic clone generation works
|
||||
- [ ] Audio format is correct (24kHz, mono)
|
||||
- [ ] Character switching works
|
||||
- [ ] Pause tags work
|
||||
- [ ] Caching works
|
||||
- [ ] VRAM management works (Clear VRAM button)
|
||||
- [ ] Device reload works after offload
|
||||
- [ ] Interrupt handling works
|
||||
|
||||
### Phase 2 Testing (Audio Editing):
|
||||
- [ ] Emotion editing works (all 14 emotions)
|
||||
- [ ] Style editing works (all 32 styles)
|
||||
- [ ] Speed editing works (4 speed levels)
|
||||
- [ ] Paralinguistic editing works (all 10 effects)
|
||||
- [ ] Denoising works (denoise + vad)
|
||||
- [ ] Iterative editing works (n_edit_iter 1-5)
|
||||
- [ ] Audio length validation works (0.5-30s)
|
||||
- [ ] Edit node integrates with voice changer workflows
|
||||
|
||||
### Phase 3 Testing (SRT Mode):
|
||||
- [ ] SRT parsing works
|
||||
- [ ] Language grouping works
|
||||
- [ ] Character switching in SRT works
|
||||
- [ ] Timing assembly works
|
||||
- [ ] Multiple languages in same SRT works
|
||||
- [ ] Interrupt handling in SRT works
|
||||
|
||||
---
|
||||
|
||||
## Questions for User
|
||||
|
||||
Before we proceed with implementation, I need clarification on:
|
||||
|
||||
1. **Longform chunking**: Saganaki22 added sentence-based text chunking for long content. This is NOT in the original. Do you want this feature, or should we stick to the original behavior (which may truncate long text at max_new_tokens)?
|
||||
|
||||
2. **Progress bars**: Saganaki22 added ComfyUI progress tracking during clone generation. This is NOT in the original. Should we add this for better UX, or skip it?
|
||||
|
||||
3. **Separate edit node**: Should we create a dedicated "Step Audio EditX - Edit" node for audio editing (emotion/style/speed), or integrate everything through the unified voice changer node?
|
||||
|
||||
4. **Model location**: Should models go in `models/TTS/step_audio_editx/` or `models/step_audio_editx/` (Saganaki22 used `models/Step-Audio-EditX/`)?
|
||||
|
||||
5. **Implementation priority**: What should we implement first?
|
||||
- Option A: Clone mode only (basic TTS) → test → then edit mode → then SRT
|
||||
- Option B: Clone + Edit modes together → test → then SRT
|
||||
- Option C: All features at once
|
||||
|
||||
---
|
||||
|
||||
## Implementation Strategy (User Approved)
|
||||
|
||||
**Phase 1**: Clone mode with TTS Suite integration
|
||||
- Full TTS Suite integration (character switching, pause tags, caching)
|
||||
- Use `utils/audio/chunk_combiner.py` for longform text
|
||||
- **Progress bars** via `comfy.utils.ProgressBar` during generation
|
||||
- VRAM management with `.to()` method (follow IndexTTS pattern)
|
||||
- Interrupt handling via `model_management.interrupt_processing`
|
||||
- Bundle code in `step_audio_editx_impl/` (like IndexTTS)
|
||||
- Test thoroughly before proceeding
|
||||
|
||||
**Phase 2**: Audio Editor (specialized node)
|
||||
- Dedicated "🎨 Step Audio EditX - Audio Editor" node
|
||||
- NOT integrated with voice changer (different purpose)
|
||||
- **Requires manual transcript input** (audio_text parameter)
|
||||
- Skip Whisper transcription for now
|
||||
- Future: Auto-feed from Whisper transcription engine
|
||||
- All editing capabilities (emotion/style/speed/paralinguistic/denoising)
|
||||
- Iterative refinement support (n_edit_iter 1-5)
|
||||
- Audio length validation (0.5-30s)
|
||||
- Test all editing modes
|
||||
|
||||
**Phase 3**: SRT support
|
||||
- Sequential processing with language grouping
|
||||
- Character switching within segments
|
||||
- Timing assembly
|
||||
- Interrupt handling for cancellation
|
||||
|
||||
This approach:
|
||||
- Allows incremental testing
|
||||
- Validates VRAM management early
|
||||
- Ensures unified systems work correctly
|
||||
- Follows TTS Suite best practices
|
||||
- Lets you test and approve each phase
|
||||
|
||||
@@ -1,557 +0,0 @@
|
||||
# Step Audio EditX Inline Tags - Implementation Plan
|
||||
|
||||
## Overview
|
||||
|
||||
Add inline Step Audio EditX edit tags to the segment parameter system, enabling automatic post-processing edits on TTS-generated audio from ANY engine.
|
||||
|
||||
**Example:**
|
||||
```
|
||||
[Alice|seed:43] This is an example <Laughter:3>, I'm funny!
|
||||
```
|
||||
|
||||
Result:
|
||||
1. TTS generates "This is an example, I'm funny!" (tag stripped)
|
||||
2. Step Audio EditX automatically edits the segment with paralinguistic `<Laughter>` in 3 iterations
|
||||
|
||||
## Syntax Design
|
||||
|
||||
### Current Segment Parameter Syntax
|
||||
```
|
||||
[Character|param:value|param2:value2]
|
||||
```
|
||||
|
||||
### New Edit Tag Syntax (inline in text)
|
||||
|
||||
**Paralinguistic tags** (insert non-verbal sounds):
|
||||
```
|
||||
Hello <Laughter> world # Default 1 iteration
|
||||
Hello <Laughter:3> world # 3 iterations
|
||||
Hello <Sigh:2> goodbye # 2 iterations
|
||||
```
|
||||
|
||||
**Other edit types** - need different syntax to avoid confusion:
|
||||
|
||||
Option A - Angle brackets with type prefix:
|
||||
```
|
||||
<emotion:happy:2> # emotion=happy, 2 iterations
|
||||
<style:whisper:3> # style=whisper, 3 iterations
|
||||
<speed:faster:1> # speed=faster, 1 iteration
|
||||
```
|
||||
|
||||
Option B - Curly braces for non-paralinguistic:
|
||||
```
|
||||
{emotion:happy:2}
|
||||
{style:whisper:3}
|
||||
{speed:faster:1}
|
||||
```
|
||||
|
||||
Option C - Unified angle brackets, context determines type:
|
||||
```
|
||||
<happy:2> # detected as emotion
|
||||
<whisper:3> # detected as style
|
||||
<faster:1> # detected as speed
|
||||
<Laughter:2> # detected as paralinguistic (capitalized)
|
||||
```
|
||||
|
||||
**Recommendation:** Option A - explicit type prefix. Clear, unambiguous, no magic detection.
|
||||
|
||||
### Multiple Tags on Same Segment
|
||||
|
||||
**Two syntax options for multiple edits:**
|
||||
|
||||
1. **Pipe-separated in single tag** (like segment parameters):
|
||||
```
|
||||
[Alice] Hello world! <Laughter:2|style:whisper:1|emotion:happy>
|
||||
```
|
||||
|
||||
2. **Multiple separate tags:**
|
||||
```
|
||||
[Alice] Hello world! <Laughter:2><style:whisper:1><emotion:happy>
|
||||
```
|
||||
|
||||
Both are equivalent and can be mixed.
|
||||
|
||||
**Processing order:**
|
||||
|
||||
- **Paralinguistic tags**: Position in text matters (where to insert sound)
|
||||
- `Hello <Laughter> world` → Laughter inserted after "Hello"
|
||||
|
||||
- **Other edit types** (emotion, style, speed): Position doesn't matter, gets stripped
|
||||
- `Hello <style:whisper> world` = `Hello world <style:whisper>` = `<style:whisper> Hello world`
|
||||
- All result in: Generate "Hello world", then apply whisper style
|
||||
|
||||
**Execution order for mixed tags:**
|
||||
1. First: All non-paralinguistic edits (emotion → style → speed) in fixed order
|
||||
2. Last: Paralinguistic edits (preserves position info for sound insertion)
|
||||
|
||||
**Example:**
|
||||
```
|
||||
[Alice] Hello <Laughter:2> world <style:whisper:1>
|
||||
```
|
||||
1. Generate TTS: "Hello world"
|
||||
2. Edit pass 1: Apply whisper style (1 iteration) - position irrelevant
|
||||
3. Edit pass 2: Add Laughter at position (2 iterations) - position matters
|
||||
|
||||
## Processing Flow
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ Input: "[Alice] Hello <Laughter:2> world!" │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ 1. Parse segment: character=Alice, text with tags │
|
||||
│ Extract edit tags: [("Laughter", "paralinguistic", 2)] │
|
||||
│ Clean text: "Hello world!" │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ 2. Generate TTS with clean text (any engine) │
|
||||
│ Output: audio segment │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ 3. Check duration: > 30s? │
|
||||
│ YES → Log warning, skip edit, return original audio │
|
||||
│ NO → Continue to edit │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ 4. Load Step Audio EditX engine (lazy load on first use) │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ 5. For each edit tag in order: │
|
||||
│ - Save audio to temp file │
|
||||
│ - Call edit_single() with appropriate edit_type │
|
||||
│ - Use output as input for next edit (if any) │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ Output: Edited audio segment │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Architecture Decision: Processing Order
|
||||
|
||||
### Option A: Generate All TTS First, Then Edit (RECOMMENDED)
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ 1. Parse ALL segments, extract edit tags, store metadata │
|
||||
│ 2. Generate ALL TTS segments (any engine) - clean text │
|
||||
│ 3. Post-process: Apply edits to segments that have tags │
|
||||
│ 4. Assemble final audio │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**Pros:**
|
||||
- Step Audio EditX engine loaded ONCE, only if needed
|
||||
- All TTS generation completes before edit engine loads (memory efficient)
|
||||
- Can batch edit operations
|
||||
- Cleaner separation of concerns
|
||||
- Progress reporting is clearer (TTS phase → Edit phase)
|
||||
|
||||
**Cons:**
|
||||
- Need to track which segments have edit tags through the pipeline
|
||||
|
||||
### Option B: Edit Each Segment Immediately After Generation
|
||||
|
||||
```
|
||||
For each segment:
|
||||
1. Parse segment, extract edit tags
|
||||
2. Generate TTS
|
||||
3. If has edit tags → load Step Audio EditX → edit
|
||||
4. Store result
|
||||
Assemble final audio
|
||||
```
|
||||
|
||||
**Pros:**
|
||||
- Simpler data flow
|
||||
|
||||
**Cons:**
|
||||
- Step Audio EditX loaded/unloaded repeatedly if interleaved
|
||||
- Two heavy models potentially in memory at same time
|
||||
- Harder to optimize
|
||||
|
||||
**Decision: Option A** - Generate all TTS first, then batch edit.
|
||||
|
||||
---
|
||||
|
||||
## Implementation Details - Modular Architecture
|
||||
|
||||
### Core Principle: NO CODE DUPLICATION
|
||||
|
||||
The edit tag system integrates with existing parsers. We don't modify adapters individually.
|
||||
|
||||
### Processing Pipeline
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────────────────────────┐
|
||||
│ EXISTING PIPELINE │
|
||||
│ │
|
||||
│ Text Input → Character Parser → Segment Parameters → TTS Generation │
|
||||
│ │
|
||||
└──────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌──────────────────────────────────────────────────────────────────────────┐
|
||||
│ NEW: EDIT TAG EXTRACTION │
|
||||
│ │
|
||||
│ Happens at parsing stage (segment_parameters.py or new unified parser) │
|
||||
│ - Extract <tag:iter> from text │
|
||||
│ - Store in segment metadata: segment.edit_tags = [...] │
|
||||
│ - Return clean text for TTS │
|
||||
│ │
|
||||
└──────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌──────────────────────────────────────────────────────────────────────────┐
|
||||
│ EXISTING: TTS GENERATION │
|
||||
│ │
|
||||
│ All engines generate audio from clean text (no edit tags) │
|
||||
│ No changes to adapters needed! │
|
||||
│ │
|
||||
└──────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌──────────────────────────────────────────────────────────────────────────┐
|
||||
│ NEW: POST-PROCESSOR (single point) │
|
||||
│ │
|
||||
│ Called ONCE after all TTS generation, before assembly │
|
||||
│ - Check each segment for edit_tags metadata │
|
||||
│ - If any segments have tags → lazy load Step Audio EditX │
|
||||
│ - Apply edits to those segments │
|
||||
│ - Return modified segments for assembly │
|
||||
│ │
|
||||
└──────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌──────────────────────────────────────────────────────────────────────────┐
|
||||
│ EXISTING: AUDIO ASSEMBLY │
|
||||
│ │
|
||||
│ Combine segments (unchanged logic) │
|
||||
│ │
|
||||
└──────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Files to Modify/Create
|
||||
|
||||
#### 1. Extend Existing Parser: `utils/text/segment_parameters.py`
|
||||
|
||||
Add edit tag extraction alongside existing parameter parsing:
|
||||
|
||||
```python
|
||||
# Existing: [Character|seed:42|temp:0.8]
|
||||
# New addition: Also extract <Laughter:2|style:whisper>
|
||||
|
||||
class SegmentData:
|
||||
character: str
|
||||
text: str # Clean text (tags stripped)
|
||||
parameters: dict # seed, temp, etc.
|
||||
edit_tags: list # NEW: [("Laughter", "paralinguistic", 2, position), ...]
|
||||
```
|
||||
|
||||
**No duplication** - extend existing `parse_segment()` function.
|
||||
|
||||
#### 2. Extend: `utils/text/step_audio_editx_special_tags.py`
|
||||
|
||||
Add iteration parsing to existing tag functions:
|
||||
|
||||
```python
|
||||
# Existing: convert_step_audio_editx_tags(), strip_paralinguistic_tags()
|
||||
# Add: parse_edit_tags_with_iterations()
|
||||
|
||||
def parse_edit_tags_with_iterations(text: str) -> Tuple[str, List[EditTag]]:
|
||||
"""
|
||||
Parse all edit tags from text, return clean text and tag list.
|
||||
|
||||
Supports:
|
||||
- <Laughter> → paralinguistic, 1 iter
|
||||
- <Laughter:3> → paralinguistic, 3 iter
|
||||
- <style:whisper:2> → style, 2 iter
|
||||
- <emotion:happy> → emotion, 1 iter
|
||||
- <Laughter:2|style:whisper:1> → multiple in one tag
|
||||
"""
|
||||
```
|
||||
|
||||
#### 3. NEW: `utils/audio/edit_post_processor.py`
|
||||
|
||||
Single centralized post-processor:
|
||||
|
||||
```python
|
||||
class EditPostProcessor:
|
||||
"""
|
||||
Post-processes TTS segments with Step Audio EditX edits.
|
||||
Lazy loads engine only when needed.
|
||||
"""
|
||||
|
||||
_engine = None # Lazy loaded
|
||||
|
||||
@classmethod
|
||||
def process_segments(cls, segments: List[SegmentResult]) -> List[SegmentResult]:
|
||||
"""
|
||||
Process all segments, applying edits where edit_tags exist.
|
||||
Called ONCE after all TTS generation.
|
||||
"""
|
||||
# Find segments with edit tags
|
||||
segments_to_edit = [s for s in segments if s.edit_tags]
|
||||
|
||||
if not segments_to_edit:
|
||||
return segments # No edits needed
|
||||
|
||||
# Lazy load engine
|
||||
if cls._engine is None:
|
||||
cls._engine = cls._load_engine()
|
||||
|
||||
# Process each segment with tags
|
||||
for segment in segments_to_edit:
|
||||
segment.audio = cls._apply_edits(segment)
|
||||
|
||||
return segments
|
||||
```
|
||||
|
||||
#### 4. Integration Points (MINIMAL CHANGES)
|
||||
|
||||
**`nodes/unified/tts_text_node.py`** - Add one call:
|
||||
```python
|
||||
# After TTS generation, before return
|
||||
segments = EditPostProcessor.process_segments(segments)
|
||||
```
|
||||
|
||||
**`nodes/unified/tts_srt_node.py`** - Add one call:
|
||||
```python
|
||||
# After all segments generated, before assembly
|
||||
segments = EditPostProcessor.process_segments(segments)
|
||||
```
|
||||
|
||||
**Adapters: NO CHANGES NEEDED**
|
||||
- Adapters receive clean text (tags already stripped at parse stage)
|
||||
- Adapters return audio as usual
|
||||
- Post-processor handles edits centrally
|
||||
|
||||
### Data Flow
|
||||
|
||||
```
|
||||
Input: "[Alice|seed:42] Hello <Laughter:2> world! <style:whisper>"
|
||||
|
||||
Step 1 - Parse (segment_parameters.py):
|
||||
→ character: "Alice"
|
||||
→ parameters: {seed: 42}
|
||||
→ clean_text: "Hello world!"
|
||||
→ edit_tags: [
|
||||
EditTag(type="paralinguistic", value="Laughter", iterations=2, position=6),
|
||||
EditTag(type="style", value="whisper", iterations=1, position=None)
|
||||
]
|
||||
|
||||
Step 2 - TTS Generation (any adapter):
|
||||
→ Input: "Hello world!"
|
||||
→ Output: audio tensor
|
||||
|
||||
Step 3 - Post-process (edit_post_processor.py):
|
||||
→ Check: segment has edit_tags? YES
|
||||
→ Validate: duration < 30s? YES
|
||||
→ Apply style:whisper (1 iter) - position-independent first
|
||||
→ Apply Laughter at position 6 (2 iter) - paralinguistic last
|
||||
→ Output: edited audio tensor
|
||||
|
||||
Step 4 - Assembly (unchanged):
|
||||
→ Combine all segments
|
||||
```
|
||||
|
||||
### Tag Detection
|
||||
|
||||
**Paralinguistic tags** (existing set):
|
||||
- Breathing, Laughter, Sigh, Uhm
|
||||
- Surprise-oh, Surprise-ah, Surprise-wa
|
||||
- Confirmation-en, Question-ei, Dissatisfaction-hnn
|
||||
|
||||
**Emotion tags** (14 options):
|
||||
- happy, sad, angry, excited, calm, fearful, surprised, disgusted
|
||||
- confusion, empathy, embarrass, depressed, coldness, admiration
|
||||
|
||||
**Style tags** (32 options):
|
||||
- whisper, serious, child, older, girl, pure, sister, sweet
|
||||
- exaggerated, ethereal, generous, recite, act_coy, warm, shy
|
||||
- comfort, authority, chat, radio, soulful, gentle, story, vivid
|
||||
- program, news, advertising, roar, murmur, shout, deeply, loudly
|
||||
- arrogant, friendly
|
||||
|
||||
**Speed tags** (4 options):
|
||||
- faster, slower, more_faster, more_slower
|
||||
|
||||
### Duration Validation
|
||||
|
||||
```python
|
||||
def validate_for_edit(audio_tensor, sample_rate):
|
||||
duration = audio_tensor.shape[-1] / sample_rate
|
||||
if duration > 30.0:
|
||||
print(f"⚠️ Segment too long for Step Audio EditX edit ({duration:.1f}s > 30s). "
|
||||
f"Edit tags will be skipped. Split your text into shorter segments.")
|
||||
return False
|
||||
if duration < 0.5:
|
||||
print(f"⚠️ Segment too short for Step Audio EditX edit ({duration:.1f}s < 0.5s). "
|
||||
f"Edit tags will be skipped.")
|
||||
return False
|
||||
return True
|
||||
```
|
||||
|
||||
### Transcript Handling
|
||||
|
||||
For edit mode, we need the transcript (what was spoken). This is the clean text BEFORE TTS:
|
||||
|
||||
```python
|
||||
# Original: "[Alice] Hello <Laughter:2> world!"
|
||||
# Clean text for TTS: "Hello world!"
|
||||
# Transcript for edit: "Hello world!" (same as clean text)
|
||||
|
||||
# For paralinguistic, target text has tags:
|
||||
# Target: "Hello [Laughter] world!"
|
||||
```
|
||||
|
||||
## Edge Cases
|
||||
|
||||
1. **No Step Audio EditX model downloaded**
|
||||
- First use triggers download (existing behavior)
|
||||
- User sees download progress
|
||||
|
||||
2. **Multiple segments, only some with tags**
|
||||
- Only segments with tags get post-processed
|
||||
- Others pass through unchanged
|
||||
|
||||
3. **Tag in middle of word** (malformed)
|
||||
- `Hel<Laughter>lo` → Strip tag, get "Hello"
|
||||
- Best effort, no error
|
||||
|
||||
4. **Unknown tag**
|
||||
- `<UnknownTag:2>` → Ignored, passed as plain text, stripped
|
||||
- Or warn user?
|
||||
|
||||
5. **ChatterBox 23-Lang v2 tag collision**
|
||||
- Their tags use different format: `<giggle>`, `<whisper>`, etc.
|
||||
- Our edit tags: `<Laughter>`, `<style:whisper>`
|
||||
- Paralinguistic overlap: need to check if text already has v2 tags
|
||||
- Solution: Process our tags first, before sending to ChatterBox
|
||||
|
||||
## Example Usage
|
||||
|
||||
### Simple paralinguistic
|
||||
```
|
||||
[Alice] I can't believe it <Laughter> that's hilarious!
|
||||
```
|
||||
|
||||
### With iterations
|
||||
```
|
||||
[Alice] I can't believe it <Laughter:3> that's hilarious!
|
||||
```
|
||||
|
||||
### Multiple effects - pipe syntax
|
||||
```
|
||||
[Bob] I'm so tired, goodnight <Sigh:2|style:whisper:1>
|
||||
```
|
||||
|
||||
### Multiple effects - separate tags
|
||||
```
|
||||
[Bob] I'm so tired <Sigh:2> goodnight <style:whisper:1>
|
||||
```
|
||||
|
||||
### Mixed syntax (equivalent to above)
|
||||
```
|
||||
[Bob] I'm so tired <Sigh:2|style:whisper:1> goodnight
|
||||
```
|
||||
|
||||
### Combined with segment parameters
|
||||
```
|
||||
[Alice|seed:42|temperature:0.8] Hello <Laughter:3> friend!
|
||||
```
|
||||
|
||||
### Full combo
|
||||
```
|
||||
[Alice|seed:42] Hello <Laughter:2|emotion:happy:1> my friend!
|
||||
```
|
||||
|
||||
### SRT mode
|
||||
```
|
||||
1
|
||||
00:00:00,000 --> 00:00:03,000
|
||||
[Alice] Hello <Laughter> world!
|
||||
|
||||
2
|
||||
00:00:03,500 --> 00:00:06,000
|
||||
[Bob|seed:123] Hi there <style:whisper:2|emotion:calm>
|
||||
```
|
||||
|
||||
### Position matters for paralinguistics
|
||||
```
|
||||
# Different results:
|
||||
[Alice] Hello <Laughter> world! # Laughter after "Hello"
|
||||
[Alice] Hello world <Laughter>! # Laughter after "world"
|
||||
|
||||
# Same result (style position irrelevant):
|
||||
[Alice] Hello <style:whisper> world!
|
||||
[Alice] <style:whisper> Hello world!
|
||||
[Alice] Hello world! <style:whisper>
|
||||
```
|
||||
|
||||
## Decisions Made
|
||||
|
||||
1. **Syntax for non-paralinguistic edits:** `<emotion:happy:2>` - explicit type prefix
|
||||
2. **Multiple tags:** Support both `<tag1:2|tag2:3>` pipe syntax AND `<tag1:2><tag2:3>` separate tags
|
||||
3. **Iteration default:** `<Laughter>` = 1 iteration (no explicit `:1` required)
|
||||
4. **Warning behavior for >30s:** Skip edit with warning in console, return original audio
|
||||
5. **Engine loading:** Lazy load ONCE after all TTS generation completes
|
||||
6. **Processing order:** Generate ALL TTS first, then batch edit (Option A)
|
||||
7. **Architecture:** Modular, no adapter changes, single post-processor
|
||||
|
||||
## Open Questions
|
||||
|
||||
1. **Position tracking for paralinguistics:**
|
||||
- How to track character position after tag stripping?
|
||||
- Store original position? Or store as "after word N"?
|
||||
|
||||
2. **Progress reporting:**
|
||||
- Show separate progress for TTS phase vs Edit phase?
|
||||
- How to report which segments are being edited?
|
||||
|
||||
3. **Cache interaction:**
|
||||
- Should edited segments be cached separately?
|
||||
- Cache key should include edit tags?
|
||||
|
||||
4. **Error handling:**
|
||||
- If one segment edit fails, continue with others?
|
||||
- Return original audio for failed edits?
|
||||
|
||||
## Implementation Order
|
||||
|
||||
1. **Phase 1: Tag Parsing**
|
||||
- Extend `step_audio_editx_special_tags.py` with iteration support
|
||||
- Add `parse_edit_tags_with_iterations()` function
|
||||
- Support pipe syntax `<tag1|tag2>` and separate tags
|
||||
|
||||
2. **Phase 2: Segment Integration**
|
||||
- Extend `segment_parameters.py` to extract and store edit_tags
|
||||
- Ensure clean text (tags stripped) flows to TTS
|
||||
|
||||
3. **Phase 3: Post-Processor**
|
||||
- Create `utils/audio/edit_post_processor.py`
|
||||
- Lazy engine loading
|
||||
- Duration validation with warnings
|
||||
- Sequential edit application
|
||||
|
||||
4. **Phase 4: Integration**
|
||||
- Add single call in `tts_text_node.py`
|
||||
- Add single call in `tts_srt_node.py`
|
||||
- Test with multiple engines
|
||||
|
||||
5. **Phase 5: Testing**
|
||||
- Single tag, multiple tags
|
||||
- Mixed syntax (pipe + separate)
|
||||
- Duration edge cases
|
||||
- SRT mode
|
||||
- Multiple engines
|
||||
@@ -1,241 +0,0 @@
|
||||
# Model Loading Tier 2-3 Refactor Plan
|
||||
|
||||
## Overview
|
||||
Complete the model loading architecture refactor by:
|
||||
1. **Tier 2**: Consolidate caching, standardize error handling
|
||||
2. **Tier 3**: Standardize factory signatures, implement capability registry
|
||||
|
||||
## TIER 2: Consolidation & Standardization
|
||||
|
||||
### 2.1 Remove SmartModelLoader Usage
|
||||
|
||||
**Current State**: SmartModelLoader still used in:
|
||||
- `nodes/chatterbox/chatterbox_tts_node.py` (lines with smart_model_loader)
|
||||
- `engines/chatterbox/streaming_model_manager.py`
|
||||
- `engines/chatterbox_official_23lang/streaming_model_manager.py`
|
||||
- `engines/adapters/f5tts_streaming_adapter.py`
|
||||
|
||||
**Why Remove**:
|
||||
- Creates third caching system (after unified interface + ComfyUI manager)
|
||||
- Causes cache synchronization issues
|
||||
- Delegates to manager.load_tts_model() anyway (redundant)
|
||||
|
||||
**Approach**:
|
||||
1. ChatterBox TTS node: Replace smart_model_loader call with direct unified interface
|
||||
2. Streaming processors: Use unified interface for model loading
|
||||
3. Verify no other code depends on SmartModelLoader returns
|
||||
|
||||
**Risk**: Medium - streaming processors have complex state management
|
||||
|
||||
---
|
||||
|
||||
### 2.2 Standardize Error Handling
|
||||
|
||||
**Current State**: 5 different error patterns across factories
|
||||
|
||||
**Target State**: Single exception hierarchy:
|
||||
```python
|
||||
class TTSModelLoadingError(Exception):
|
||||
"""Base exception for model loading"""
|
||||
pass
|
||||
|
||||
class TTSModelNotFoundError(TTSModelLoadingError):
|
||||
"""Model not found locally or on HuggingFace"""
|
||||
pass
|
||||
|
||||
class TTSDeviceError(TTSModelLoadingError):
|
||||
"""Device-related issues"""
|
||||
pass
|
||||
|
||||
class TTSModelFormatError(TTSModelLoadingError):
|
||||
"""Model format compatibility issues"""
|
||||
pass
|
||||
```
|
||||
|
||||
**Files to Update**:
|
||||
- Create `utils/models/exceptions.py` with exception classes
|
||||
- Update all factories to use consistent exceptions
|
||||
- Update node error handlers to catch standardized exceptions
|
||||
|
||||
**Risk**: Low - backwards compatible
|
||||
|
||||
---
|
||||
|
||||
### 2.3 Unified Cache Key Generation
|
||||
|
||||
**Current State**: Multiple cache key formats:
|
||||
- Unified interface: string concatenation
|
||||
- SmartModelLoader: MD5 hashing
|
||||
- Nodes: MD5 of dict items
|
||||
|
||||
**Target State**: Single hash-based cache key function
|
||||
|
||||
```python
|
||||
def generate_model_cache_key(
|
||||
engine_name: str,
|
||||
model_name: str,
|
||||
device: str,
|
||||
language: Optional[str] = None,
|
||||
**extra_params
|
||||
) -> str:
|
||||
"""Generate standardized cache key"""
|
||||
```
|
||||
|
||||
**Files to Create/Update**:
|
||||
- `utils/models/cache_key_generator.py` (new)
|
||||
- Update unified_model_interface.py to use it
|
||||
- Update all node caching to use it
|
||||
|
||||
**Risk**: Low
|
||||
|
||||
---
|
||||
|
||||
## TIER 3: Architecture Standardization
|
||||
|
||||
### 3.1 Factory Parameter Dataclass
|
||||
|
||||
**Current State**: Each factory has different parameters
|
||||
|
||||
**Target State**: All factories accept standardized dataclass:
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class ModelLoadConfig:
|
||||
"""Standard configuration for all model factories"""
|
||||
# Required
|
||||
device: str # "auto", "cuda", "cpu", etc.
|
||||
model_name: str
|
||||
|
||||
# Optional (engine-specific)
|
||||
language: Optional[str] = None
|
||||
model_path: Optional[str] = None
|
||||
repo_id: Optional[str] = None
|
||||
|
||||
# Additional params (for quantization, optimization, etc.)
|
||||
additional_params: Optional[Dict[str, Any]] = None
|
||||
|
||||
def __post_init__(self):
|
||||
# Resolve device before storing
|
||||
from utils.device import resolve_torch_device
|
||||
self.device = resolve_torch_device(self.device)
|
||||
```
|
||||
|
||||
**Factory Signature Becomes**:
|
||||
```python
|
||||
def engine_factory(config: ModelLoadConfig) -> Any:
|
||||
"""All factories use same signature"""
|
||||
```
|
||||
|
||||
**Files to Create/Update**:
|
||||
- `utils/models/factory_config.py` (new)
|
||||
- All factories in unified_model_interface.py
|
||||
- Unified interface load_model() method
|
||||
|
||||
**Risk**: High - changes all factory signatures
|
||||
|
||||
---
|
||||
|
||||
### 3.2 Engine Capability Registry
|
||||
|
||||
**Current State**: Hardcoded engine-specific logic in model_manager.py
|
||||
|
||||
**Target State**: Data-driven engine capabilities:
|
||||
|
||||
```python
|
||||
ENGINE_CAPABILITIES = {
|
||||
"chatterbox": {
|
||||
"supports_voice_conversion": True,
|
||||
"has_fallback": True,
|
||||
"can_corrupt": False,
|
||||
"recovery_handler": None
|
||||
},
|
||||
"higgs_audio": {
|
||||
"supports_voice_conversion": False,
|
||||
"has_fallback": False,
|
||||
"can_corrupt": True,
|
||||
"recovery_handler": reset_higgs_cuda_graphs,
|
||||
"requires_special_init": True
|
||||
},
|
||||
# ... all 7 engines
|
||||
}
|
||||
```
|
||||
|
||||
**Files to Create**:
|
||||
- `utils/models/engine_registry.py` (new) - define capabilities
|
||||
- Remove hardcoded engine checks from model_manager.py
|
||||
- Use registry for generic recovery
|
||||
|
||||
**Risk**: Medium - affects error recovery paths
|
||||
|
||||
---
|
||||
|
||||
### 3.3 Unified Fallback Handler
|
||||
|
||||
**Current State**: ChatterBox has sophisticated fallback, others don't
|
||||
|
||||
**Target State**: Generic fallback utility any engine can use:
|
||||
|
||||
```python
|
||||
class ModelFallbackHandler:
|
||||
"""Generic fallback chain for model loading"""
|
||||
|
||||
def __init__(self, primary_loader: Callable):
|
||||
self.primary_loader = primary_loader
|
||||
self.fallback_chain = []
|
||||
|
||||
def add_fallback(self, fallback_loader: Callable, condition: Callable):
|
||||
"""Add fallback with condition"""
|
||||
pass
|
||||
|
||||
def load(self) -> Any:
|
||||
"""Execute fallback chain"""
|
||||
pass
|
||||
```
|
||||
|
||||
**Files to Create**:
|
||||
- `utils/models/fallback_handler.py` (new)
|
||||
|
||||
**Risk**: Low - new utility, no breaking changes
|
||||
|
||||
---
|
||||
|
||||
## Implementation Order
|
||||
|
||||
1. **First**: Create exception classes (no breaking changes)
|
||||
2. **Second**: Create cache key generator (parallel with tier 1 utilities)
|
||||
3. **Third**: Create factory config dataclass
|
||||
4. **Fourth**: Update all factories to use ModelLoadConfig
|
||||
5. **Fifth**: Remove SmartModelLoader usage
|
||||
6. **Sixth**: Create engine registry
|
||||
7. **Seventh**: Implement unified fallback handler
|
||||
8. **Eighth**: Update model_manager.py to use registry
|
||||
|
||||
---
|
||||
|
||||
## Risk Assessment
|
||||
|
||||
| Task | Risk | Dependencies | Effort |
|
||||
|------|------|--------------|--------|
|
||||
| Exception classes | Low | None | 1 hr |
|
||||
| Cache key generator | Low | Exceptions | 1 hr |
|
||||
| Factory config dataclass | Medium | None | 2 hrs |
|
||||
| Update factories | High | Config, exceptions | 4 hrs |
|
||||
| Remove SmartModelLoader | Medium | Updated factories | 2 hrs |
|
||||
| Engine registry | Medium | Updated factories | 2 hrs |
|
||||
| Fallback handler | Low | Exceptions | 2 hrs |
|
||||
| Integration testing | High | All above | 3 hrs |
|
||||
|
||||
**Total**: ~17 hours of work
|
||||
|
||||
---
|
||||
|
||||
## Success Criteria
|
||||
|
||||
- [ ] All factories use consistent error handling
|
||||
- [ ] Cache key generation centralized and consistent
|
||||
- [ ] SmartModelLoader completely removed
|
||||
- [ ] Engine capabilities defined in registry
|
||||
- [ ] All 7 engines working without issues
|
||||
- [ ] Cache hits work correctly across device switches
|
||||
- [ ] Error messages are consistent and helpful
|
||||
- [ ] No VRAM spikes or device mismatches
|
||||
@@ -1,146 +0,0 @@
|
||||
# Project Refactoring TODO
|
||||
|
||||
This document tracks architectural issues and inconsistencies that need refactoring to improve modularity and reduce code duplication.
|
||||
|
||||
## SRT Processing Architecture Issues
|
||||
|
||||
### Problem: VibeVoice Native Multi-Speaker SRT Uses Non-Contiguous Global Slots
|
||||
- **Issue**: Later subtitles can request global `Speaker 2`/`Speaker 3` IDs while VibeVoice truncates and renumbers the supplied voice prompts from zero.
|
||||
- **Impact**: A subtitle containing only later global speakers can bind the wrong reference or leave a requested speaker without a matching voice prompt.
|
||||
- **Solution**: Use the global character map only to select references, then compact each subtitle to request-local speakers `0..N-1` with references in the same order.
|
||||
- **Regression test**: Cover subtitle 1 `[Alice]...[Bob]...`, followed by subtitle 2 `[Bob]...[Rick]...`.
|
||||
- **Priority**: High
|
||||
|
||||
### Problem: Inconsistent SRT Implementation Approaches
|
||||
Different engines use completely different patterns for SRT processing:
|
||||
|
||||
1. **ChatterBox (Old)**: Uses `ChatterboxSRTTTSNode` class (should be processor)
|
||||
2. **VibeVoice**: Uses proper `VibeVoiceSRTProcessor` class with full implementation
|
||||
3. **Higgs Audio**: Uses clean modular approach with timing utilities
|
||||
4. **ChatterBox Official 23-Lang**: Follows Higgs Audio pattern (good)
|
||||
5. **F5-TTS**: Uses `F5TTSSRTNode` class (should be processor)
|
||||
|
||||
**Impact**: Code duplication, inconsistent interfaces, hard to maintain
|
||||
**Solution**: Standardize on processor pattern with shared utilities
|
||||
|
||||
### Problem: Node vs Processor Naming Inconsistency
|
||||
- ChatterBox uses `ChatterboxSRTTTSNode` but should be `ChatterboxSRTProcessor`
|
||||
- F5-TTS uses `F5TTSSRTNode` but should be `F5TTSSRTProcessor`
|
||||
- Only VibeVoice and ChatterBox Official 23-Lang use proper processor naming
|
||||
|
||||
**Impact**: Confusing architecture, breaks expected patterns
|
||||
**Solution**: Rename all SRT classes to use `*Processor` pattern
|
||||
|
||||
### Problem: Duplicated SRT Logic
|
||||
Each engine reimplements:
|
||||
- SRT parsing
|
||||
- Character switching within SRT segments
|
||||
- Timing calculations
|
||||
- Audio assembly
|
||||
- Report generation
|
||||
|
||||
**Impact**: Massive code duplication, bugs in some implementations
|
||||
**Solution**: Create shared SRT base class with common functionality
|
||||
|
||||
## Voice Reference Handling Issues
|
||||
|
||||
### Problem: Inconsistent Audio Reference Processing
|
||||
- ChatterBox: Uses file paths only, converts tensors to temp files via `handle_reference_audio`
|
||||
- F5-TTS: Uses both file paths and ComfyUI tensors directly
|
||||
- Higgs Audio: Uses ComfyUI tensors with reference text
|
||||
- VibeVoice: Uses ComfyUI tensors
|
||||
|
||||
**Impact**: Different interfaces, conversion logic scattered
|
||||
**Solution**: Standardize voice reference handling in base class
|
||||
|
||||
## Character Parser Integration Issues
|
||||
|
||||
### Problem: Inconsistent Language Resolution
|
||||
- Some engines hardcode language mappings
|
||||
- Some use character parser properly
|
||||
- Some bypass language mapper entirely
|
||||
- ChatterBox Official 23-Lang now uses proper integration (good example)
|
||||
|
||||
**Impact**: Language switching behaves differently per engine
|
||||
**Solution**: Mandate character parser usage in base class
|
||||
|
||||
### Problem: Engine-Specific Logic in Universal Character Parser
|
||||
- **Issue**: Universal character parser contains ChatterBox-specific Italian prefix logic (`apply_italian_prefix_if_needed`)
|
||||
- **Problem**: Engine-specific text processing belongs in engine adapters, not universal parser
|
||||
- **Current State**: Italian `[it]` prefix filter has complex conditions but should only affect:
|
||||
- Explicit `[it:Alice]` or `[italian:Bob]` tags
|
||||
- Characters mapped to Italian in alias system
|
||||
- Should NOT affect normal English ChatterBox, other engines, or global model selection
|
||||
- **Impact**: Violation of separation of concerns, coupling universal parser to specific engine quirks
|
||||
- **Solution**: Move Italian prefix logic to ChatterBox adapter, keep character parser engine-agnostic
|
||||
- **Priority**: Medium (works correctly but violates clean architecture)
|
||||
- **Files Affected**:
|
||||
- `utils/text/character_parser/` (remove Italian logic)
|
||||
- `engines/adapters/chatterbox_adapter.py` (add Italian logic)
|
||||
|
||||
## Cache System Issues
|
||||
|
||||
### Problem: Engine-Specific Cache Keys
|
||||
- Each engine uses different cache identifiers
|
||||
- Cache collision was discovered between ChatterBox engines
|
||||
- No standardized cache invalidation
|
||||
|
||||
**Impact**: Cross-engine cache pollution, missed cache opportunities
|
||||
**Solution**: Implement unified cache system with proper namespacing
|
||||
|
||||
## Base Class Architecture Issues
|
||||
|
||||
### Problem: Inconsistent Base Class Usage
|
||||
- Some engines properly inherit from `BaseTTSNode`
|
||||
- Some engines duplicate base class functionality
|
||||
- Method signatures vary between engines for same functionality
|
||||
|
||||
**Impact**: Code duplication, inconsistent interfaces
|
||||
**Solution**: Enforce proper base class inheritance and standardize method signatures
|
||||
|
||||
## Import System Issues
|
||||
|
||||
### Problem: Different Import Patterns
|
||||
- Some use direct imports
|
||||
- Some use `importlib.util.spec_from_file_location`
|
||||
- Some use robust_import
|
||||
- Inconsistent path resolution
|
||||
|
||||
**Impact**: Import failures, maintenance overhead
|
||||
**Solution**: Standardize import patterns project-wide
|
||||
|
||||
## Error Handling Inconsistencies
|
||||
|
||||
### Problem: Different Error Handling Approaches
|
||||
- Some engines have comprehensive error handling
|
||||
- Some engines crash on edge cases
|
||||
- Different error message formats
|
||||
- Inconsistent fallback mechanisms
|
||||
|
||||
**Impact**: Poor user experience, difficult debugging
|
||||
**Solution**: Implement standardized error handling framework
|
||||
|
||||
## Configuration Management Issues
|
||||
|
||||
### Problem: Scattered Configuration Handling
|
||||
- Engine configs stored differently
|
||||
- Parameter validation inconsistent
|
||||
- Default values scattered across files
|
||||
- No centralized config schema
|
||||
|
||||
**Impact**: Bugs from invalid configs, hard to maintain defaults
|
||||
**Solution**: Implement unified configuration management system
|
||||
|
||||
---
|
||||
|
||||
## Refactoring Priority
|
||||
|
||||
1. **High Priority**: SRT processing standardization (affects user experience directly)
|
||||
2. **Medium Priority**: Voice reference handling (causes confusion)
|
||||
3. **Medium Priority**: Cache system unification (performance impact)
|
||||
4. **Low Priority**: Import system standardization (maintenance burden)
|
||||
|
||||
---
|
||||
|
||||
*Last Updated: 2025-01-XX*
|
||||
*Add new issues to this file as they are discovered during development*
|
||||
@@ -1,123 +0,0 @@
|
||||
TTS Audio Suite - Transformers 5.x Higgs Audio 2 Investigation Report
|
||||
|
||||
Scope
|
||||
- Problem: Higgs Audio 2 regressed after the suite moved the main environment to Transformers 5.x.
|
||||
- Goal: determine whether Higgs Audio 2 is a good native Transformers 5 patch target, or whether it should move to runtime isolation.
|
||||
|
||||
Environment
|
||||
- Windows / ComfyUI runtime.
|
||||
- Main env on Transformers 5.x.
|
||||
- Direct repro and validation performed in the real Windows Python environment outside the ComfyUI UI wrapper.
|
||||
|
||||
Findings Summary
|
||||
- Higgs Audio 2 is not blocked at the shallow import/API level only.
|
||||
- Some real compatibility fixes were possible:
|
||||
- bundled cache API adaptation
|
||||
- cache-arg rename fixes
|
||||
- major cold-load regression fix
|
||||
- However, the remaining failure is decode-path semantics, especially around StaticCache and CUDA graph replay.
|
||||
- Higgs can be made to load and "run" on Transformers 5, but the high-performance path is not trustworthy and produced incorrect audio.
|
||||
- This is not a good short-path patch target like IndexTTS.
|
||||
|
||||
Reproduction
|
||||
1. Transformers 5 main environment.
|
||||
2. Higgs Audio 2 loaded through the normal TTS Text path with voice cloning.
|
||||
3. Observed one or more of:
|
||||
- multi-minute cold-load stalls
|
||||
- StaticCache API crashes
|
||||
- CUDA graph capture/runtime failures
|
||||
- runaway/gibberish generation
|
||||
- near-empty or silent output
|
||||
|
||||
Key Diagnostic Evidence
|
||||
- Initial cold-load timing showed:
|
||||
- `⏱️ Higgs Audio: Core model load took 440.61s`
|
||||
- Tokenizer/HuBERT stages were not the bottleneck:
|
||||
- text tokenizer around 1-2s
|
||||
- audio tokenizer around 2-5s
|
||||
- The 440s stall was inside `HiggsAudioModel.from_pretrained(...)`, specifically Hugging Face missing-key initialization.
|
||||
- Direct checkpoint/model key comparison in runtime showed:
|
||||
- `missing_count 0`
|
||||
- `unexpected_count 0`
|
||||
- After bypassing unnecessary missing-key initialization for complete local checkpoints, direct timing dropped to:
|
||||
- `⏱️ Higgs Audio: Core model load took 58.46s`
|
||||
- later direct runs around `33-58s` depending on cache state
|
||||
|
||||
- StaticCache/API drift evidence:
|
||||
- `'StaticCache' object is not subscriptable`
|
||||
- later mask/cache shape mismatches around static cache width `1024`
|
||||
|
||||
- Cache-arg drift evidence:
|
||||
- bundled Higgs/LLaMA path mixed `past_key_value` and `past_key_values`
|
||||
- led to duplicated kwarg failures and broken decode semantics
|
||||
|
||||
- CUDA graph replay evidence:
|
||||
- direct runtime probe with graph replay active produced:
|
||||
- immediate or near-immediate audio termination
|
||||
- tiny garbage clips
|
||||
- or invalid decode behavior
|
||||
- direct runtime probe with StaticCache but without CUDA graph replay produced:
|
||||
- normal-length audio token generation
|
||||
- reasonable token spread
|
||||
- real waveform output
|
||||
|
||||
Attempts and Results (chronological)
|
||||
1) StaticCache compatibility shims
|
||||
- Added bundled helpers to access cache tensors/device across old tuple caches and new Cache/StaticCache APIs.
|
||||
- Result: fixed immediate StaticCache indexing crashes.
|
||||
|
||||
2) CUDA graph failure fallback cleanup
|
||||
- On graph capture failure, forced cache recreation and switched to DynamicCache instead of continuing with stale StaticCache assumptions.
|
||||
- Result: removed some follow-on crashes after graph failure, but not the core correctness issue.
|
||||
|
||||
3) Attention backend adjustment
|
||||
- Stopped forcing eager attention on CUDA and preferred SDPA instead.
|
||||
- Result: better alignment with the intended high-performance path, but did not fully restore correctness.
|
||||
|
||||
4) TTS defaults forcing audio generation
|
||||
- Set Higgs TTS defaults to force audio generation rather than letting the model drift in text mode.
|
||||
- Result: prevented obvious free-running text-style generation, but did not solve bad output.
|
||||
|
||||
5) Cold-load regression fix
|
||||
- Added a local complete-checkpoint detector and skipped Hugging Face missing-key initialization when the local Higgs checkpoint was complete.
|
||||
- Result: large real improvement to cold-load time.
|
||||
|
||||
6) Cache-argument adaptation
|
||||
- Patched bundled Higgs/LLaMA call sites for Transformers 5 cache-arg naming.
|
||||
- Result: moved runtime further, but surfaced deeper semantic issues instead of fixing them outright.
|
||||
|
||||
7) CUDA graph runner PyTorch compatibility
|
||||
- Wrapped input-buffer updates in inference mode to satisfy current PyTorch restrictions.
|
||||
- Result: removed one explicit runtime crash, but not the underlying decode corruption.
|
||||
|
||||
8) StaticCache without CUDA graph replay direct comparison
|
||||
- Direct Windows probes showed:
|
||||
- `enable_cuda_graphs=True` with graph replay: corrupted/too-short audio output
|
||||
- `enable_cuda_graphs=False`: long audio-token generation and real waveform output
|
||||
- `StaticCache` with graph replay skipped: closer to sane behavior than replay path
|
||||
- Result: isolated the remaining failure to the CUDA graph replay path on Transformers 5.
|
||||
|
||||
Interpretation
|
||||
- Higgs Audio 2 on Transformers 5 is not blocked by a single missing symbol or a few renamed APIs.
|
||||
- The remaining problem is decode/runtime behavior, especially the interaction between:
|
||||
- StaticCache
|
||||
- bundled Higgs decode logic
|
||||
- LLaMA attention/cache semantics on current Transformers 5
|
||||
- CUDA graph replay
|
||||
- That is behavior-level drift, not just compatibility-shim work.
|
||||
|
||||
Decision
|
||||
- Higgs Audio 2 on Transformers 5 is still theoretically patchable.
|
||||
- It is not a good use of time right now.
|
||||
- The engine reached the same bad zone as Qwen3-TTS, but with even more sensitivity around CUDA graphs and cache behavior.
|
||||
- Runtime isolation is the correct path for now.
|
||||
|
||||
Operational Recommendation
|
||||
- Do not continue native Transformers 5 Higgs patching in the main branch.
|
||||
- Route Higgs Audio 2 into an isolated legacy runtime instead.
|
||||
- Prefer a dedicated Higgs runtime first rather than assuming shared legacy parity.
|
||||
- Revisit native Transformers 5 support only if Higgs becomes important enough to justify deeper decode/cuda-graph compatibility work.
|
||||
|
||||
Notes
|
||||
- The cold-load fix was a real improvement and may still be worth revisiting later, even outside the final T5 decision.
|
||||
- The rest of this investigation should be treated as a branch-local investigation record, not as a stable main-branch solution.
|
||||
@@ -1,118 +0,0 @@
|
||||
TTS Audio Suite - Transformers 5.x Qwen3-TTS Investigation Report
|
||||
|
||||
Scope
|
||||
- Problem: Qwen3-TTS garbled audio or load failure on Transformers 5.0.0.
|
||||
- Goal: identify a safe compatibility path or conclude unsupported for now.
|
||||
|
||||
Findings Summary
|
||||
- The tokenizer path in Transformers 5.0.0 is the core blocker.
|
||||
- AutoProcessor/AutoTokenizer paths either fail or apply an incorrect regex fix and lead to broken token IDs.
|
||||
- Direct Qwen2Tokenizer loading via vocab/merges is broken in this environment (encoder remains empty or vocab_size=1).
|
||||
- The fast tokenizer build using tokenizers BPE can be created in isolation, but in ComfyUI runtime PreTrainedTokenizerFast fails to load vocab from a temporary directory (error: unable to load vocabulary).
|
||||
- The model config's eos/pad tokens (<|im_end|>, <|endoftext|>) are not present in vocab.json; safe fallback tokens that exist are <s>, <unk>, </s>.
|
||||
|
||||
Reproduction (Windows / ComfyUI)
|
||||
1) Transformers 5.0.0 environment.
|
||||
2) Qwen3-TTS model folder present with vocab.json and merges.txt.
|
||||
3) Qwen3-TTS load fails or produces garbled output.
|
||||
|
||||
Key Diagnostic Evidence
|
||||
- Fast tokenizer build using tokenizers (BPE) works in an isolated test:
|
||||
- vocab_size=151643
|
||||
- bos=<s> (id 128245), unk=<unk> (id 128244), eos/pad=</s> (id 128247)
|
||||
- In ComfyUI runtime, PreTrainedTokenizerFast.from_pretrained(tmpdir) fails with:
|
||||
- "Unable to load vocabulary from file"
|
||||
- The tokenizer_config.json tokens (<|im_end|>, <|endoftext|>) do not exist in vocab.json.
|
||||
|
||||
Attempts and Results (chronological)
|
||||
1) Rope fallback normalization for transformers 5.0.0
|
||||
- Added resolve_rope_type_with_fallback and patched rope_scaling.
|
||||
- Reduced KeyError crashes but did not fix garbled audio.
|
||||
2) AutoProcessor / AutoTokenizer (fix_mistral_regex)
|
||||
- On 5.0.0, fix_mistral_regex triggers duplicate-arg errors or incorrect regex warning.
|
||||
- Still leads to bad tokenization.
|
||||
3) Direct Qwen2Tokenizer(vocab/merges)
|
||||
- In runtime, encoder remains empty; vocab_size collapses to 1.
|
||||
- Garbled output even when forced ids are set.
|
||||
4) Fast tokenizer BPE build using tokenizers (Tokenizer + BPE)
|
||||
- Works in isolated script; IDs look correct.
|
||||
- Fails in runtime when wrapped by PreTrainedTokenizerFast (from_pretrained tmpdir).
|
||||
5) Token ID correction
|
||||
- For tokens not present in vocab.json, fallback to <s>/<unk>/<\s>.
|
||||
- Valid in test but runtime build still blocked by tokenizer load error.
|
||||
|
||||
Conclusion
|
||||
- With Transformers 5.0.0 as installed in the reported environment, the tokenizer path is blocked.
|
||||
- This is not fixed by rope patches or token id corrections alone.
|
||||
- A working solution likely requires either:
|
||||
- a Transformers build that accepts tokenizer_object (or can load tokenizer.json without requiring vocab/merges), or
|
||||
- downgrading Transformers to a 4.x release known to work with Qwen3-TTS.
|
||||
|
||||
Recommendation
|
||||
- Temporarily mark Transformers 5.0.0 as unsupported for Qwen3-TTS and emit a clear warning.
|
||||
- Provide guidance to downgrade Transformers or use a compatible build.
|
||||
|
||||
Notes
|
||||
- No changes in this report alter project behavior.
|
||||
|
||||
Addendum - Transformers 5.10.2 Re-check
|
||||
|
||||
Scope
|
||||
- Problem: determine whether newer Transformers 5.10.2 improves Qwen3-TTS enough to avoid runtime isolation.
|
||||
- Goal: distinguish shallow API drift from deeper generation/runtime incompatibility.
|
||||
|
||||
Environment
|
||||
- Windows / ComfyUI runtime.
|
||||
- Main env on Transformers 5.10.2.
|
||||
- Direct repro also performed in the real Windows Python environment outside the ComfyUI UI wrapper.
|
||||
|
||||
What Changed vs 5.0.0
|
||||
- The failure mode is different.
|
||||
- The first blocker is no longer the tokenizer collapse described above.
|
||||
- Qwen3-TTS can be pushed significantly further on 5.10.2 with local compatibility shims.
|
||||
|
||||
Key Diagnostic Evidence
|
||||
- Direct model load initially failed with:
|
||||
- `KeyError: 'default'`
|
||||
- source: `ROPE_INIT_FUNCTIONS[self.rope_type]`
|
||||
- Current Transformers 5.10.2 no longer exposes a legacy `"default"` RoPE entry:
|
||||
- available keys observed in runtime: `dynamic`, `linear`, `llama3`, `longrope`, `proportional`, `yarn`
|
||||
- After restoring a local `"default"` compatibility entry, model load succeeded.
|
||||
- Generation then failed on changed Hugging Face masking APIs:
|
||||
- `create_causal_mask() got an unexpected keyword argument 'input_embeds'`
|
||||
- After adapting those call sites to the current HF signature, generation progressed further.
|
||||
- Final direct generation failure on 5.10.2 was:
|
||||
- `RuntimeError: Expected size for first two dimensions of batch2 tensor to be: [16, 277] but got: [16, 139].`
|
||||
- source: SDPA attention path during generation.
|
||||
|
||||
Attempts and Results (5.10.2)
|
||||
1) Legacy RoPE compatibility restore
|
||||
- Added a local fallback for legacy `"default"` RoPE initialization.
|
||||
- Result: Qwen3-TTS load progressed and completed successfully.
|
||||
2) Mask helper API adaptation
|
||||
- Updated bundled Qwen mask call sites to match current HF `create_causal_mask` / `create_sliding_window_causal_mask` signatures.
|
||||
- Result: generation progressed beyond the earlier immediate crash.
|
||||
3) Direct generation validation with real voice-clone inputs
|
||||
- Used real reference audio and transcript in the Windows environment.
|
||||
- Result: no longer a trivial import/load/signature issue; generation hit a tensor-shape mismatch inside SDPA.
|
||||
|
||||
Interpretation
|
||||
- Transformers 5.10.2 is not blocked in the same way as 5.0.0.
|
||||
- Some problems are still shallow compatibility issues and can be adapted cleanly.
|
||||
- However, once generation reaches SDPA tensor-shape mismatch territory, this stops being a simple API-shim problem.
|
||||
- At that point, continuing the patch path means re-validating model behavior against newer Hugging Face generation/mask/cache semantics, not just restoring removed symbols.
|
||||
|
||||
Decision
|
||||
- Qwen3-TTS on Transformers 5.10.2 is still technically patchable in principle.
|
||||
- It is not a good short-path target like IndexTTS.
|
||||
- The suite should prefer runtime isolation for Qwen3-TTS instead of continuing deep Transformers 5 surgery.
|
||||
|
||||
Operational Recommendation
|
||||
- Treat Qwen3-TTS as a legacy-stack engine for now.
|
||||
- Route it into the shared legacy Transformers 4 runtime used by compatible engines (same family already proven for VibeVoice/Qwen in prior testing).
|
||||
- Keep the main environment on Transformers 5.
|
||||
- Revisit native Transformers 5 support only if Qwen3-TTS becomes strategically important enough to justify deeper behavior-level compatibility work.
|
||||
|
||||
Notes
|
||||
- The transient 5.10.2 patch attempts were not kept as the main solution path.
|
||||
- The current project direction is runtime isolation, not continued in-place Transformers 5 patching for Qwen3-TTS.
|
||||
@@ -1,209 +0,0 @@
|
||||
# VibeVoice Integration Plan - Final Version
|
||||
|
||||
## Overview
|
||||
|
||||
Integrate Microsoft VibeVoice (1.5B and 7B models) into TTS Audio Suite following the unified architecture, with full support for all existing features while exposing VibeVoice's unique multi-speaker capabilities.
|
||||
|
||||
## Implementation Structure
|
||||
|
||||
### 1. Engine Implementation (`engines/vibevoice/`)
|
||||
|
||||
#### `vibevoice.py` (Main Engine - ~500 lines)
|
||||
|
||||
```python
|
||||
class VibeVoiceEngine:
|
||||
def __init__(self)
|
||||
def initialize_engine(model_name, device)
|
||||
def generate_speech(text, voice_samples, cfg_scale, seed, ...)
|
||||
def generate_multi_speaker(segments, voice_samples, ...)
|
||||
def cleanup() # Memory management
|
||||
def unload_models() # Explicit unload (called by ComfyUI's unload button)
|
||||
```
|
||||
|
||||
- Use `unified_model_interface` for model loading
|
||||
- Integrate with `utils.audio.cache` for caching
|
||||
- Support both models (1.5B and 7B)
|
||||
- Handle memory management via ComfyUI's unload system
|
||||
|
||||
#### `vibevoice_downloader.py` (~200 lines)
|
||||
|
||||
```python
|
||||
VIBEVOICE_MODELS = {
|
||||
"vibevoice-1.5B": {
|
||||
"repo": "microsoft/VibeVoice-1.5B",
|
||||
"files": [...], # 5.4GB total
|
||||
},
|
||||
"vibevoice-7B": {
|
||||
"repo": "aoi-ot/VibeVoice-Large", # community mirror (Microsoft repo removed)
|
||||
"files": [...], # 9.3GB total
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- Use `unified_downloader` for all downloads
|
||||
- Download to `models/TTS/vibevoice/` structure
|
||||
- No HuggingFace cache duplication
|
||||
|
||||
### 2. Engine Configuration Node (`nodes/engines/vibevoice_engine_node.py`) (~150 lines)
|
||||
|
||||
```python
|
||||
class VibeVoiceEngineNode:
|
||||
INPUT_TYPES:
|
||||
Required:
|
||||
- model: ["VibeVoice-1.5B", "VibeVoice-7B"]
|
||||
- device: ["auto", "cuda", "cpu"]
|
||||
- multi_speaker_mode: ["Custom Character Switching", "Native Multi-Speaker"]
|
||||
- cfg_scale: (1.0-2.0, default 1.3)
|
||||
- use_sampling: BOOLEAN
|
||||
- temperature: (0.1-2.0, default 0.95)
|
||||
- top_p: (0.1-1.0, default 0.95)
|
||||
- chunk_minutes: INT (0=disabled, UI shows minutes, backend converts to chars)
|
||||
Optional:
|
||||
- speaker2_voice: AUDIO
|
||||
- speaker3_voice: AUDIO
|
||||
- speaker4_voice: AUDIO
|
||||
```
|
||||
|
||||
- Returns TTS_ENGINE configuration dict
|
||||
- NO seed parameter (that's in unified nodes)
|
||||
- NO load/unload toggles (handled by ComfyUI system)
|
||||
- Uses same naming as Higgs: "Custom Character Switching" not "segment_per_character"
|
||||
- Follows ChatterBox/Higgs patterns exactly
|
||||
|
||||
### 3. Engine Adapter (`engines/adapters/vibevoice_adapter.py`) (~400 lines)
|
||||
|
||||
```python
|
||||
class VibeVoiceEngineAdapter:
|
||||
def __init__(self, node_instance)
|
||||
def get_model_for_language(lang_code, default_model)
|
||||
def load_base_model(model_name, device)
|
||||
def generate_segment(text, voice_ref, params)
|
||||
def process_character_segments(segments, voice_mapping)
|
||||
def convert_character_to_speaker_format(text) # [Alice] → Speaker 0
|
||||
def handle_pause_tags(text) # Use pause_processor
|
||||
def handle_language_tags(text) # Parse [de:Alice] syntax
|
||||
```
|
||||
|
||||
- Bridge unified nodes to VibeVoice engine
|
||||
- Two modes matching Higgs naming:
|
||||
- `Custom Character Switching`: Generate per character (like ChatterBox/Higgs)
|
||||
- `Native Multi-Speaker`: Single generation with multiple speakers
|
||||
- Character mapping: [Alice], [Bob] → Speaker 0, Speaker 1
|
||||
- Full integration with existing systems
|
||||
|
||||
### 4. Internal Processors (`nodes/vibevoice/`)
|
||||
|
||||
#### `vibevoice_processor.py` (~300 lines)
|
||||
|
||||
- Internal TTS processor (called by unified TTS node)
|
||||
- Handles actual generation orchestration
|
||||
- Manages chunking (time-based UI, character-based backend)
|
||||
- Memory management coordination
|
||||
|
||||
#### `vibevoice_srt_processor.py` (~400 lines)
|
||||
|
||||
- Internal SRT processor (called by unified SRT node)
|
||||
- Language grouping optimization
|
||||
- Multi-speaker dialogue handling
|
||||
- Timing synchronization
|
||||
|
||||
### 5. Model Factory Registration (~50 lines in existing files)
|
||||
|
||||
In `utils/models/unified_model_interface.py`:
|
||||
|
||||
```python
|
||||
# Register VibeVoice model factories
|
||||
interface.register_model_factory(
|
||||
"vibevoice", "tts",
|
||||
lambda **kwargs: load_vibevoice_model(**kwargs)
|
||||
)
|
||||
```
|
||||
|
||||
### 6. Dependencies Update
|
||||
|
||||
`requirements.txt`:
|
||||
|
||||
```
|
||||
# VibeVoice dependencies
|
||||
transformers>=4.44.0
|
||||
git+https://github.com/microsoft/VibeVoice.git
|
||||
```
|
||||
|
||||
## Key Integration Points
|
||||
|
||||
### Memory Management
|
||||
|
||||
- Memory unloading handled by ComfyUI's "Unload Models" button
|
||||
- Engine implements `unload_models()` method
|
||||
- Uses ComfyUI's model management system
|
||||
- Auto-cleanup on node destruction
|
||||
|
||||
### Unified Systems Integration
|
||||
|
||||
- **Cache**: Use `utils.audio.cache` with content-based hashing
|
||||
- **Characters**: Full `utils.text.character_parser` support
|
||||
- **Pause Tags**: `utils.text.pause_processor` integration
|
||||
- **Downloads**: `utils.downloads.unified_downloader` only
|
||||
- **Model Loading**: `utils.models.unified_model_interface`
|
||||
- **Chunking**: `utils.text.chunking` with time-based UI
|
||||
|
||||
### Two Generation Modes (Named Like Higgs)
|
||||
|
||||
1. **Custom Character Switching** (default):
|
||||
|
||||
- Preserves all TTS Audio Suite features
|
||||
- Generate audio per character, then combine
|
||||
- Supports pause tags, language switching per character
|
||||
- More flexible but slower
|
||||
|
||||
2. **Native Multi-Speaker**:
|
||||
|
||||
- Uses VibeVoice's native multi-speaker
|
||||
- Single generation pass
|
||||
- More efficient but less flexible
|
||||
- Auto-converts [Character] → Speaker N format
|
||||
|
||||
### Time-Based Chunking
|
||||
|
||||
- UI shows minutes: "Chunk every [5] minutes"
|
||||
- Backend converts: 5 min → ~3750 chars (150 wpm * 5 chars/word * 5)
|
||||
- More intuitive for long-form content
|
||||
- Optional (0 = no chunking)
|
||||
|
||||
### Seed Parameter Location
|
||||
|
||||
- Seed is in unified nodes (TTS Text, TTS SRT) NOT in engine node
|
||||
- Follows existing pattern where engine provides configuration, unified nodes handle generation parameters
|
||||
|
||||
## File Size Limits
|
||||
|
||||
- Each file stays under 500-600 lines
|
||||
- Modular design with clear separation
|
||||
- Reuses all existing utilities
|
||||
- No code duplication
|
||||
|
||||
## Testing Plan
|
||||
|
||||
1. Test both 1.5B and 7B models
|
||||
2. Validate character switching in both modes
|
||||
3. Test pause tag processing
|
||||
4. Verify memory management via ComfyUI unload
|
||||
5. Test long-form generation with chunking
|
||||
6. Validate multi-speaker native mode
|
||||
7. Test cache functionality
|
||||
8. Verify SRT synchronization
|
||||
|
||||
## Benefits
|
||||
|
||||
- Full integration with all TTS Audio Suite features
|
||||
- Choice between flexibility (Custom mode) and efficiency (Native mode)
|
||||
- Proper memory management via ComfyUI system
|
||||
- No code duplication
|
||||
- Follows established patterns exactly (Higgs Audio style)
|
||||
- Exposes unique 90-minute and 4-speaker capabilities
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- Created: 2025-08-29
|
||||
- Author: Claude (TTS Audio Suite Assistant)
|
||||
- Status: Ready for implementation
|
||||
@@ -1,231 +0,0 @@
|
||||
# Pytest Testing Framework Implementation Report
|
||||
|
||||
**Date:** 2024-12-18
|
||||
**Author:** AI Assistant
|
||||
**Status:** ✅ Working (60/64 tests passing)
|
||||
|
||||
---
|
||||
|
||||
## Executive Summary
|
||||
|
||||
Implementing pytest for a ComfyUI custom node is **non-trivial** due to the way Python imports work with packages that have complex `__init__.py` files. This document summarizes the approaches tried, failures encountered, and the final working solution.
|
||||
|
||||
---
|
||||
|
||||
## The Core Problem
|
||||
|
||||
When pytest discovers and imports test files, Python's import system also imports the parent package's `__init__.py`. For TTS-Audio-Suite, this `__init__.py`:
|
||||
|
||||
1. Applies compatibility patches (numba, PyTorch, transformers)
|
||||
2. Imports `nodes.py` which imports `folder_paths` (ComfyUI-only module)
|
||||
3. Loads all node modules dynamically
|
||||
4. Registers API endpoints with ComfyUI server
|
||||
|
||||
**Result:** Running `pytest tests/unit/` triggers the entire ComfyUI node loading chain, which fails because ComfyUI modules (`folder_paths`, `comfy`, `server`) don't exist in the test context.
|
||||
|
||||
---
|
||||
|
||||
## Approaches Tried (and Why They Failed)
|
||||
|
||||
### ❌ Approach 1: Environment Variable Guard in `__init__.py`
|
||||
|
||||
**Idea:** Check `os.environ['COMFYUI_TESTING']` at the top of `__init__.py` and skip loading.
|
||||
|
||||
```python
|
||||
# __init__.py
|
||||
if os.environ.get('COMFYUI_TESTING') == '1':
|
||||
NODE_CLASS_MAPPINGS = {}
|
||||
# ... skip rest of file
|
||||
```
|
||||
|
||||
**Why it failed:** Would require wrapping 300+ lines of code in an `if/else` block, causing massive indentation changes. Python doesn't have a "skip rest of file" statement.
|
||||
|
||||
---
|
||||
|
||||
### ❌ Approach 2: `collect_ignore` in conftest.py
|
||||
|
||||
**Idea:** Tell pytest to ignore `__init__.py` during collection.
|
||||
|
||||
```python
|
||||
# conftest.py
|
||||
collect_ignore = ["__init__.py", "nodes.py"]
|
||||
```
|
||||
|
||||
**Why it failed:** `collect_ignore` prevents pytest from *collecting tests* from those files, but doesn't prevent Python from *importing* them when test modules import utilities.
|
||||
|
||||
---
|
||||
|
||||
### ❌ Approach 3: Moving conftest.py to tests/ directory
|
||||
|
||||
**Idea:** If conftest.py is in `tests/`, pytest won't treat the parent as a package.
|
||||
|
||||
**Why it failed:** Pytest still finds `pytest.ini` in the parent and treats the directory structure as packages. The import chain is triggered when test files do `from utils.timing.parser import ...`.
|
||||
|
||||
---
|
||||
|
||||
### ❌ Approach 4: Using `--import-mode=importlib`
|
||||
|
||||
**Idea:** Pytest's alternative import mode might avoid `__init__.py` loading.
|
||||
|
||||
```bash
|
||||
pytest --import-mode=importlib tests/
|
||||
```
|
||||
|
||||
**Why it failed:** Same underlying issue - Python still imports parent packages.
|
||||
|
||||
---
|
||||
|
||||
### ❌ Approach 5: Using importlib.util in test files
|
||||
|
||||
**Idea:** Bypass package imports entirely by loading modules directly.
|
||||
|
||||
```python
|
||||
# test_srt_parser.py
|
||||
import importlib.util
|
||||
parser_path = custom_node_root / "utils" / "timing" / "parser.py"
|
||||
spec = importlib.util.spec_from_file_location("parser_module", parser_path)
|
||||
parser_module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(parser_module)
|
||||
```
|
||||
|
||||
**Why it failed:** Conftest.py is still loaded first by pytest, and conftest.py's mere existence in the package triggers `__init__.py` import.
|
||||
|
||||
---
|
||||
|
||||
### ✅ Approach 6: Mock ComfyUI Modules at conftest.py Module Level (WORKING)
|
||||
|
||||
**The Solution:** Mock all ComfyUI modules *before* pytest imports anything else.
|
||||
|
||||
```python
|
||||
# tests/conftest.py - MUST be at very top
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
os.environ['COMFYUI_TESTING'] = '1'
|
||||
os.environ['PYTEST_CURRENT_TEST'] = 'true'
|
||||
|
||||
# Mock ComfyUI modules BEFORE any other imports
|
||||
_MOCK_MODULES = [
|
||||
'comfy', 'comfy.model_management', 'comfy.utils',
|
||||
'nodes', 'folder_paths', 'server', 'execution', 'comfy_extras',
|
||||
]
|
||||
|
||||
for _module_name in _MOCK_MODULES:
|
||||
if _module_name not in sys.modules:
|
||||
sys.modules[_module_name] = MagicMock()
|
||||
|
||||
# NOW safe to import pytest, etc.
|
||||
import pytest
|
||||
```
|
||||
|
||||
**Why it works:** When `__init__.py` executes `import folder_paths`, Python finds a MagicMock already in `sys.modules` and uses that instead of trying to find the real module.
|
||||
|
||||
---
|
||||
|
||||
## Additional Bug Found
|
||||
|
||||
During testing, discovered a **real bug** in `nodes.py`:
|
||||
|
||||
```python
|
||||
# BEFORE (buggy)
|
||||
try:
|
||||
from utils.compatibility import setup_numba_compatibility
|
||||
setup_numba_compatibility(...)
|
||||
except ImportError:
|
||||
import sys # ← Only imported in except block!
|
||||
import os # ← Only imported in except block!
|
||||
|
||||
# Later in file...
|
||||
current_dir = os.path.dirname(__file__) # ← NameError if try succeeded!
|
||||
```
|
||||
|
||||
**Fix:** Move `import sys` and `import os` to the top of the file, outside the try/except block.
|
||||
|
||||
---
|
||||
|
||||
## Guidance for Future Developers
|
||||
|
||||
### 1. Test File Structure
|
||||
|
||||
```
|
||||
TTS-Audio-Suite/
|
||||
├── __init__.py # Complex, avoid importing during tests
|
||||
├── nodes.py # Node registration
|
||||
├── pytest.ini # Test configuration
|
||||
├── tests/
|
||||
│ ├── conftest.py # CRITICAL: Mock ComfyUI here
|
||||
│ ├── unit/
|
||||
│ │ ├── test_srt_parser.py
|
||||
│ │ └── test_audio_processing.py
|
||||
│ └── integration/
|
||||
│ └── test_workflows.py
|
||||
```
|
||||
|
||||
### 2. conftest.py Template
|
||||
|
||||
```python
|
||||
"""Mock ComfyUI at module level BEFORE any imports"""
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
# Environment flags
|
||||
os.environ['COMFYUI_TESTING'] = '1'
|
||||
|
||||
# Mock ComfyUI modules
|
||||
for mod in ['folder_paths', 'comfy', 'server', 'execution']:
|
||||
if mod not in sys.modules:
|
||||
sys.modules[mod] = MagicMock()
|
||||
|
||||
# Now safe for normal imports
|
||||
import pytest
|
||||
# ... rest of fixtures
|
||||
```
|
||||
|
||||
### 3. Unit Test Pattern
|
||||
|
||||
```python
|
||||
# test_my_utility.py
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
custom_node_root = Path(__file__).parent.parent.parent
|
||||
module_path = custom_node_root / "utils" / "my_utility.py"
|
||||
|
||||
spec = importlib.util.spec_from_file_location("my_utility", module_path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
# Now use module.MyClass, module.my_function, etc.
|
||||
```
|
||||
|
||||
### 4. Running Tests
|
||||
|
||||
```bash
|
||||
# Unit tests (fast, no server)
|
||||
python -m pytest tests/unit/ -m unit -v
|
||||
|
||||
# Integration tests (requires ComfyUI running)
|
||||
python -m pytest tests/integration/ -m integration -v
|
||||
```
|
||||
|
||||
### 5. Key Gotchas
|
||||
|
||||
| Issue | Solution |
|
||||
|-------|----------|
|
||||
| `ModuleNotFoundError: folder_paths` | Add to mock list in conftest.py |
|
||||
| `NameError: os not defined` | Check all imports are at module top level |
|
||||
| Tests pass locally but fail in CI | Ensure conftest.py loads first |
|
||||
| `__init__.py` still runs | Mock must happen at conftest.py *module level*, not in a function |
|
||||
|
||||
---
|
||||
|
||||
## Conclusion
|
||||
|
||||
The working solution requires:
|
||||
1. **Mocking at conftest.py module level** (not in `pytest_configure` hook - too late)
|
||||
2. **Ensuring `os`/`sys` imports are at top level** in production code
|
||||
3. **Using importlib for utility imports** in test files (optional but cleaner)
|
||||
|
||||
This approach allows unit tests to run without ComfyUI while integration tests can still use the real server.
|
||||
@@ -1,80 +0,0 @@
|
||||
# TTS Audio Suite - Auxiliary Model Reference Data
|
||||
# Source of truth for helper/post-process model dropdowns that are not engines.
|
||||
|
||||
categories:
|
||||
- id: punctuation_truecase
|
||||
name: Punctuation / Truecase
|
||||
node_targets:
|
||||
- "📝 ASR Punctuation / Truecase"
|
||||
description: "Optional punctuation and true-casing post-process models for raw ASR text. Designed mainly for low-punctuation transcripts such as Granite outputs, but usable on any plain text."
|
||||
models:
|
||||
- id: pcs_en
|
||||
name: "English Fullstop + Truecase"
|
||||
dropdown_label: "English Fullstop + Truecase (EN, ~210MB)"
|
||||
repo_id: "1-800-BAD-CODE/punctuation_fullstop_truecase_english"
|
||||
pretrained_name: "pcs_en"
|
||||
model_dir: "punctuation_fullstop_truecase_english"
|
||||
spe_filename: "spe_32k_lc_en.model"
|
||||
model_filename: "punct_cap_seg_en.onnx"
|
||||
config_filename: "config.yaml"
|
||||
size: "~210MB"
|
||||
auto_download: true
|
||||
languages:
|
||||
- "English"
|
||||
notes: "English-only. Best on lowercased, unpunctuated ASR text."
|
||||
- id: pcs_romance
|
||||
name: "Romance Fullstop + Truecase"
|
||||
dropdown_label: "Romance Fullstop + Truecase (ES/FR/IT/PT/CA/GL, ~144MB)"
|
||||
repo_id: "1-800-BAD-CODE/punctuation_fullstop_truecase_romance"
|
||||
pretrained_name: "pcs_romance"
|
||||
model_dir: "punctuation_fullstop_truecase_romance"
|
||||
spe_filename: "sp.model"
|
||||
model_filename: "model.onnx"
|
||||
config_filename: "config.yaml"
|
||||
size: "~144MB"
|
||||
auto_download: true
|
||||
languages:
|
||||
- "Spanish"
|
||||
- "French"
|
||||
- "Italian"
|
||||
- "Portuguese"
|
||||
- "Catalan"
|
||||
- "Galician"
|
||||
notes: "Romance-language punctuation and true-casing model."
|
||||
- id: pcs_47lang
|
||||
name: "47-Language Fullstop + Truecase"
|
||||
dropdown_label: "47-Language Fullstop + Truecase (~1.11GB)"
|
||||
repo_id: "1-800-BAD-CODE/punct_cap_seg_47_language"
|
||||
pretrained_name: "pcs_47lang"
|
||||
model_dir: "punct_cap_seg_47_language"
|
||||
spe_filename: "spe_unigram_64k_lowercase_47lang.model"
|
||||
model_filename: "punct_cap_seg_47lang.onnx"
|
||||
config_filename: "config.yaml"
|
||||
size: "~1.11GB"
|
||||
auto_download: true
|
||||
languages:
|
||||
- "47 languages"
|
||||
notes: "Heavier multilingual fallback with the widest language coverage supported by the punctuators package."
|
||||
model_layouts_markdown: |
|
||||
# Auxiliary Model Layouts
|
||||
|
||||
Use this document for helper/post-process model folder structures and placement notes.
|
||||
These are not engine weights. They are optional models used by standalone utility nodes.
|
||||
|
||||
For repository/source URLs, use [AUX_MODEL_SOURCES.md](AUX_MODEL_SOURCES.md).
|
||||
|
||||
## Punctuation / Truecase
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/punctuation/
|
||||
├── punctuation_fullstop_truecase_english/
|
||||
├── punctuation_fullstop_truecase_romance/
|
||||
└── punct_cap_seg_47_language/
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- Models download lazily on first use by the punctuation node.
|
||||
- These are helper models for text cleanup, not TTS/ASR engines.
|
||||
- The English model is the lightest and is mainly intended for raw English ASR text.
|
||||
- The 47-language model is much heavier and should be treated as the multilingual fallback.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,27 +0,0 @@
|
||||
# TTS Engines Reference Tables
|
||||
|
||||
## Engine Comparison
|
||||
|
||||
| Engine | Isolation | Models | Size | TTS | SRT | VC | ASR | Sound Effects | Training | License | Special Features | Languages |
|
||||
| ------------------ | --------- | ----------------------------------------- | ------------ | :-: | :-: | :-: | :-: | :-----------: | :------: | ------------------------ | ---------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------- |
|
||||
| **F5-TTS** | Main | Base, v1, E2TTS + 8 lang models | ~1.2GB each | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | CC-BY-NC-4.0 | Targeted Word/Speech Editing, Speed control | 10 |
|
||||
| **ChatterBox** | Main | EN, DE×3, IT, FR, RU, HY, KA, JA, KO, NO | ~4.3GB | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | MIT | Expressiveness slider | 10 |
|
||||
| **ChatterBox 23L** | Main | v1, v2, v3, Vietnamese (Viterbox), Egyptian Arabic (oddadmix) | ~4.3GB | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | MIT | V1, V2, and V3 official checkpoints, Emotion tokens (v2; currently ineffective), V3 skips the legacy alignment analyzer and trims the final token artifact | 25 |
|
||||
| **VibeVoice** | Shared | 1.5B, 7B, KugelAudio-0 (7B), kugel-2 (7B), Hindi-1.5B/7B | 5.4GB / 18GB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | MIT (research-only per model card) | 90-min long-form, Native 4-speaker (Base models), Multilingual (KugelAudio variants), 4-bit quantization | 27 |
|
||||
| **Higgs Audio 2** | Shared | 3B | ~9GB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | Boson Higgs Audio 2 Community License | 3 multi-speaker, CUDA graphs (55+ tokens/sec) | 5 |
|
||||
| **Higgs Audio v3** | Main | 4B | ~8GB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | Boson Higgs Audio v3 Research and Non-Commercial License | Native inline emotion/style/prosody/SFX tags, Zero-shot voice cloning, 100+ language support | 100+ |
|
||||
| **IndexTTS 2 / 2.5** | Main | IndexTTS-2, IndexTTS-2.5 | ~4.7GB / ~5.49GB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | bilibili Model Use License | Emotion Control: 8 vectors, Text as reference, Audio as reference, IndexTTS-2.5 official internal feature-duration scaling (not prosody planning), IndexTTS-2.5 pronunciation annotations | 5 |
|
||||
| **CosyVoice3** | Main | 0.5B, 0.5B-RL | ~5.4GB | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | Apache-2.0 | Paralinguistic tags | 4 |
|
||||
| **Qwen3-TTS** | Shared | 0.6B, 1.7B (CustomVoice/VoiceDesign/Base) | ~3-6GB | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | Apache-2.0 | Voice design, ASR (Automatic Speech Recognition) | 10 |
|
||||
| **Granite ASR** | Main | granite-4.0-1b-speech, granite-speech-4.1-2b, granite-speech-4.1-2b-plus | ~4.6GB | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | Apache-2.0 | Native speaker attribution / diarization (plus model variant), Native word-level timestamps (plus model variant), ASR (Automatic Speech Recognition), Custom timestamps/SRT via reused Qwen forced aligner, Speech translation (experimental), Optional forced aligner auto-routed through shared legacy T4 runtime | 6 |
|
||||
| **Step Audio EditX** | Main | 3B LLM + CosyVoice | ~7GB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | Apache-2.0 (verify before commercial use) | Second Pass Speech Editing Node: 14 emotions, 32 speaking styles, Paralinguistic effects, Selectable main, shared, or dedicated Python runtime (shared Transformers 4 runtime recommended) | 4 |
|
||||
| **Echo-TTS** | Main | echo-tts-base + fish-s1-dac-min | ~5.3GB + ~1.8GB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | CC-BY-NC-SA-4.0 | Diffusion-based (~30s best), Force Speaker KV (speaker drift control) | 1 |
|
||||
| **Fish Audio S2 Pro** | Main | S2 Pro 4B / FP8 | ~10.3GB / ~8.0GB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | Fish Audio Research License | Free-form sub-word emotion/prosody tags, Native multi-speaker and multi-turn dialogue with dynamic speaker references, Zero-shot voice cloning, Optional per-segment custom character switching, Configurable 4K-32K native context with reduced KV-cache VRAM, Optional community FP8 weight-only checkpoint with BF16 activations, Optional on-the-fly BitsAndBytes INT8/NF4 for the official checkpoint | 80+ languages |
|
||||
| **Dots TTS** | Main | dots.tts-base, dots.tts-soar, dots.tts-mf | ~6GB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | Apache-2.0 | Official auto language detect / language control, SOAR and MeanFlow distilled variants | 19 |
|
||||
| **DramaBox** | Main | DramaBox 3.3B | ~16.4GB | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | LTX-2 Community License | Expressive scene prompting and stage directions, Native and SRT-aware duration targeting, Official duration-aware long-form chunking with scene-prefix preservation, Optional 10-second zero-shot voice reference, CFG negative prompt with per-segment switching, Explicit generation/reference durations, CFG rescale control, and optional Perth watermark, Experimental staged and sequential strategies for lowering peak VRAM; official LTX FP8-cast storage, Optional official torch.compile path, Official audio-branch IC-LoRA training workflow | 1 |
|
||||
| **OmniVoice** | Main | OmniVoice | ~3.7GB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | Apache-2.0 | Inline non-verbal tags and pronunciation overrides, Reference-free voice design, 600+ language support, Upstream long-form chunk orchestration | 600+ |
|
||||
| **MOSS-TTS** | Main | Local 1.7B, Delay 8B v1.5/1.0, LAION Voice Acting 8B community fine-tune, VoiceGenerator 1.7B, SoundEffect 8B v1, TTSD 8B | ~8.5GB tokenizer + ~6.1GB/17GB/18GB model | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | Apache-2.0 | Reference-free voice design with MOSS-VoiceGenerator, Native 1-5 speaker TTSD dialogue, 31-language generation with MOSS-TTS-v1.5, Optional LAION community 8B voice-acting fine-tune, Config-based discovery of compatible local MOSS full checkpoints, Prompt-only sound-effect generation with MOSS-SoundEffect v1, Long-form generation (TTSD/Delay), Duration token hint, Local/Delay/TTSD variants, Initial integrated LoRA training workflow (Delay 8B) | 24 |
|
||||
| **MOSS-SoundEffect v2** | Main | MOSS-SoundEffect-v2.0 | ~11.2GB | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | Apache-2.0 | Durations up to 30 seconds, Native negative prompting, CFG, flow shift, and diffusion-step controls, Prompt-only text-to-sound generation, 48 kHz mono output, Seeded generation | 2 |
|
||||
| **RVC** | Main | Community .pth | 100-300MB | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | MIT (framework); community models vary | Real-time VC, Integrated training workflow, Pitch shift (±14), 6 HuBERT models, Language-independent | Any |
|
||||
|
||||
*Isolation column: `Main` runs in the main ComfyUI environment. `Shared` uses a shared secondary runtime reused by multiple engines. `Dedicated` uses an engine-specific secondary runtime.*
|
||||
@@ -1,23 +0,0 @@
|
||||
# Feature Comparison Matrix
|
||||
|
||||
## Feature Comparison Matrix
|
||||
|
||||
| Feature | F5-TTS | ChatterBox | ChatterBox 23L | VibeVoice | Higgs Audio 2 | Higgs Audio v3 | IndexTTS 2 / 2.5 | CosyVoice3 | Qwen3-TTS | Granite ASR | Step Audio EditX | Echo-TTS | Fish Audio S2 Pro | Dots TTS | DramaBox | OmniVoice | MOSS-TTS | MOSS-SoundEffect v2 | RVC |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| **TTS** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| **SRT** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| **Voice Conversion** | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| **ASR (Transcribe)** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||
| **Sound Effects** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ |
|
||||
| **Training** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ✅ |
|
||||
| **Voice Cloning** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (Base model) | ❌ | ✅ | ✅ | ✅ Reference audio plus exact transcript | ✅ | ✅ Optional reference audio; configurable 3-30 second window from the beginning of the source, default 10 seconds | ✅ | ✅ | ❌ | ⚠️ (needs training) |
|
||||
| **Reference Transcript†** | **Required** | Not used | Not used | Not used | Optional | Optional | Not used | Conditional | Conditional | N/A | **Required** | Not used | **Required** | Optional | Not used | **Required** | Conditional | Not used | N/A |
|
||||
| **Native Multi-Speaker** | ❌ | ❌ | ❌ | ✅ (Base only, Kugel uses fallback) | ✅ | ❌ | ❌ | ❌ | ❌ | ⚠️ (Plus variant speaker attribution / diarization) | ❌ | ❌ | ✅ Native dialogue is default; optional custom mode generates each [Character] segment independently as local speaker 0 | ❌ | ❌ | ❌ | ✅ (TTSD v1.0; 1-5 speakers) | ❌ | ❌ |
|
||||
| **Emotion Control** | ❌ | ❌ | ⚠️ (v2 tags - doesn't work) | ❌ | ⚠️ (via prompt) | ✅ (native inline tags) | ✅ (8 emotions) | ⚠️ (via instruct) | ⚠️ (via instruct) | ❌ | ✅ (14 emotions) | ❌ | ✅ Free-form inline natural-language tags | ❌ | ✅ Natural-language scene prompt and stage directions | ⚠️ (voice-design instruct + inline non-verbal tags) | ✅ (MOSS-VoiceGenerator instruction-conditioned voice design) | ❌ | ❌ |
|
||||
| **Native Long-form** | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ Configurable 4K-32K native context; suite text chunking is bypassed | ❌ | ✅ Official duration-aware quote-group chunking; ~37s target / 45s cap | ✅ (uses upstream audio_chunk_duration / audio_chunk_threshold orchestration; bypasses suite char-based chunk splitting) | ✅ (TTSD/Delay long-form; use chunk orchestration for very long inputs) | ❌ | N/A |
|
||||
| **Community Finetunes** | ✅ | ✅ | ✅ | ✅ KugelAudio, Hindi | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ Official audio-branch IC-LoRA adapters can be trained and loaded | ❌ | ✅ (Compatible full local checkpoints, LAION Voice Acting 8B auto-download, and LoRA adapter inference/training supported) | ❌ | ✅ |
|
||||
| **VRAM Efficient** | ✅ | ✅ | ✅ | ⚠️ (5-18GB) | ⚠️ (9GB) | ⚠️ (~8-10GB) | ⚠️ (9-12GB) | ✅ (5.4GB) | ✅ (3-6GB) | ✅ (~4.6GB) | ⚠️ (7GB) | ⚠️ (~7GB total) | ⚠️ 8K context measured at ~15.2GB BF16, ~11.2GB FP8, ~11.2GB BNB INT8, or ~8.9GB BNB NF4; BF16 codec and activations; BNB is a load-time option for the official checkpoint | ⚠️ (main env works; 2B-class model is not lightweight) | ✅ Fast mode is ~24GB; experimental FP8, staged, and sequential options can reduce VRAM, but practical peak usage varies by environment and loaded components | ⚠️ (main env works; 2B-class model is not lightweight) | ⚠️ (Local 1.7B smaller; tokenizer is large) | ⚠️ (runs in the main ComfyUI environment but remains GPU-heavy) | ✅ |
|
||||
| **Speed/Performance** | ✅ Very Fast | ✅ Fast | ✅ Fast | ⚠️ | ⚠️ | ⚠️ CUDA recommended | ⚠️ | ✅ Fast | ⚠️ | ⚠️ Moderate | ⚠️ | ✅ Fast (diffusion, realtime-capable) | ✅ Main-environment subprocess with reliable teardown; local compile measurements: ~40 it/s BF16, ~11.8 it/s NF4 at ~8.9GB VRAM, and ~3.7 it/s INT8 at ~11.2GB VRAM; quality comparison pending | ⚠️ Moderate; mf variant is faster | ⚠️ Fast favors repeated generation; staged reloads temporary components; sequential also transfers the transformer; optional block-level torch.compile accelerates repeat denoising | ✅ Fast; upstream reports sub-realtime RTF | ✅ Fast with CUDA/FlashAttention | ⚠️ (100 diffusion steps by default) | ✅ Fast |
|
||||
| **No Narrator Required** | ❌ | ✅ (default speaker) | ✅ (default speaker) | ✅ (zero-shot / default speaker) | ✅ (basic TTS if no narrator/reference is provided) | ✅ (zero-shot) | ❌ | ✅ (cross-lingual or instruct mode) | ✅ (Base default voice or CustomVoice presets) | N/A | ❌ | ❌ | ✅ Reference audio is optional | ✅ (default speaker) | ✅ Voice reference is optional | ✅ (native default voice; instruct also works without narrator) | ✅ (direct TTS and prompt-only generation) | ❌ | N/A |
|
||||
|
||||
† **Reference Transcript:** Conditional means the transcript is required only for the specific mode: CosyVoice3 zero-shot, Qwen3-TTS full Base cloning, or MOSS-TTSD cloned-speaker dialogue. Higgs Audio 2, Higgs Audio v3, and Dots TTS accept matching text when provided but do not require it.
|
||||
@@ -1,35 +0,0 @@
|
||||
# Fish Audio S2 Pro Inline Tags
|
||||
|
||||
Use the suite's public angle-bracket syntax for Fish's free-form instructions:
|
||||
|
||||
```text
|
||||
<whisper in small voice>Hello. <professional broadcast tone>Good evening.
|
||||
```
|
||||
|
||||
The Fish adapter converts these to native `[...]` instructions only when text reaches the Fish engine. Do not write Fish tags with square brackets in suite text because `[Character]` is reserved for character switching.
|
||||
|
||||
Fish also supports normal suite tags such as `[pause:500ms]`, character switching, and per-segment parameters. Those are processed by the suite before Fish inference.
|
||||
|
||||
## Language Prompting
|
||||
|
||||
Fish has no native language dropdown or parameter. The engine can instead prepend a natural inline instruction from the suite's resolved segment language:
|
||||
|
||||
- `language_prompting = Auto Inline Tag`: non-English resolved languages become tags such as `<German>` or `<French>`
|
||||
- `language_prompting = Off`: no automatic language instruction is added
|
||||
|
||||
English is only added when the user explicitly requested it with a language tag such as `[en:Bob]` or `[English:Bob]`. Implicit/default English stays untagged.
|
||||
|
||||
## Character Switching Modes
|
||||
|
||||
The engine defaults to `Native Multi-Speaker`. All `[Character]` turns in a generated block are sent through one Fish dialogue request, preserving native multi-turn context and long-form behavior.
|
||||
|
||||
Select `Custom Character Switching` to generate every parsed character segment independently. Each call uses only that character's reference and is remapped to Fish speaker 0. This can reduce speaker leakage, but it requires more calls and does not preserve context between character turns. SRT subtitle boundaries and timing are preserved in both modes.
|
||||
|
||||
The engine UI separates checkpoint choice from load-time quantization:
|
||||
|
||||
- `model_variant`: `s2-pro` or the separate `s2-pro-fp8` checkpoint
|
||||
- `quantization`: `none`, `bnb_int8`, or `bnb_nf4` for on-the-fly quantization of the official `s2-pro` checkpoint
|
||||
|
||||
The BNB options reuse the official files and do not download a second model copy.
|
||||
|
||||
The S2 Pro weights use the Fish Audio Research License: research and non-commercial use are allowed; commercial use requires a separate Fish Audio license.
|
||||
@@ -1,78 +0,0 @@
|
||||
# Higgs Audio v3 Inline Tags
|
||||
|
||||
Higgs Audio v3 uses native inline control tokens in the text prompt. Type these tags directly in **TTS Text** or **TTS SRT** when using the Higgs Audio v3 engine.
|
||||
|
||||
## Syntax
|
||||
|
||||
Canonical native tags use:
|
||||
|
||||
```text
|
||||
<|category:value|>
|
||||
```
|
||||
|
||||
TTS Audio Suite also accepts a convenience alias form without the bars:
|
||||
|
||||
```text
|
||||
<category:value>
|
||||
```
|
||||
|
||||
Aliases are normalized internally to the canonical Higgs syntax. The editor inserts the canonical `<|...|>` form on purpose.
|
||||
|
||||
Examples:
|
||||
|
||||
```text
|
||||
<|emotion:amusement|>Wait, that was actually funny.
|
||||
<|style:whispering|>Keep your voice down.
|
||||
I need a second.<|prosody:pause|> Okay, continue.
|
||||
That was perfect. <|sfx:laughter|>Haha, absolutely perfect.
|
||||
```
|
||||
|
||||
## Supported Categories
|
||||
|
||||
Emotion:
|
||||
|
||||
```text
|
||||
elation, amusement, enthusiasm, determination, pride, contentment, affection,
|
||||
relief, contemplation, confusion, surprise, awe, longing, arousal, anger, fear,
|
||||
disgust, bitterness, sadness, shame, helplessness
|
||||
```
|
||||
|
||||
Style:
|
||||
|
||||
```text
|
||||
singing, shouting, whispering
|
||||
```
|
||||
|
||||
Prosody:
|
||||
|
||||
```text
|
||||
speed_very_slow, speed_slow, speed_fast, speed_very_fast,
|
||||
pitch_low, pitch_high, pause, long_pause,
|
||||
expressive_high, expressive_low
|
||||
```
|
||||
|
||||
Sound effects:
|
||||
|
||||
```text
|
||||
cough, laughter, crying, screaming, burping, humming, sigh, sniff, sneeze
|
||||
```
|
||||
|
||||
## Placement Rules
|
||||
|
||||
- Put delivery-wide tags like emotion, style, speed, pitch, and expressiveness at the start of the sentence or turn.
|
||||
- Put positional tags like `<|prosody:pause|>`, `<|prosody:long_pause|>`, and `<|sfx:...|>` exactly where they should happen.
|
||||
- Pair SFX tags with written sound text, for example `<|sfx:laughter|>Haha` or `<|sfx:sigh|>Ahh`.
|
||||
- Canonical Higgs syntax is `<|...|>`. Alias input like `<emotion:amusement>` is accepted by TTS Audio Suite for convenience, but the official form stays `<|emotion:amusement|>`.
|
||||
- Higgs Audio v3 does not use Step Audio EditX second-pass inline processing here. Use Higgs native tags instead of Step semantics when targeting Higgs.
|
||||
|
||||
## Character Voices
|
||||
|
||||
Higgs Audio v3 supports reference audio cloning. A transcript of the reference audio strongly improves cloning quality. Character voice `.txt` files are passed as reference transcripts when available.
|
||||
|
||||
## Multiline Editor
|
||||
|
||||
The multiline TTS tag editor has an `Inline Tags` panel with a dedicated `Higgs Audio v3` mode. That panel inserts canonical native Higgs tags directly.
|
||||
|
||||
## License
|
||||
|
||||
Higgs Audio v3 is released under the Boson Higgs Audio v3 Research and Non-Commercial License. Production, hosted API, or revenue-generating use requires a separate commercial license from Boson AI.
|
||||
@@ -1,687 +0,0 @@
|
||||
# Step Audio EditX Inline Tags - User Guide
|
||||
|
||||
## Scope
|
||||
|
||||
This guide is specifically for **Step Audio EditX** inline post-processing tags.
|
||||
|
||||
The multiline TTS tag editor now has an engine-aware `Inline Tags` panel with separate modes for:
|
||||
- `Step Audio EditX`
|
||||
- `Higgs Audio v3`
|
||||
- `CosyVoice3`
|
||||
|
||||
Use the Step mode when you want post-process editing tags. Use the Higgs or Cosy modes when you want those engines' native generation tags.
|
||||
|
||||
## 💡 Inline vs Manual Node Workflow
|
||||
|
||||
**ComfyUI is modular** - you can manually chain **TTS → 🎨 Step Audio EditX - Audio Editor** nodes for full control.
|
||||
|
||||
**Why use inline tags instead?**
|
||||
- Apply edits to **specific segments** of text without multiple TTS→Edit→TTS chains
|
||||
- Convenient for **SRT files** with selective editing per subtitle
|
||||
- Automatic batch processing for efficiency
|
||||
|
||||
**For maximum control and quality**, build manual workflows. **For convenience and selective segment editing**, use inline tags.
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Inline edit tags allow you to apply **Step Audio EditX** post-processing effects to TTS-generated audio from engines that support the Step inline workflow. These tags are automatically stripped before TTS generation and applied as a second processing pass.
|
||||
|
||||
**Example:**
|
||||
```
|
||||
[Alice] This is hilarious <Laughter:3>, I can't believe it!
|
||||
```
|
||||
|
||||
**What happens:**
|
||||
1. TTS generates: "This is hilarious, I can't believe it!" (tag removed)
|
||||
2. Step Audio EditX edits the audio with 3 iterations of Laughter effect
|
||||
3. Final audio has natural laughter inserted
|
||||
|
||||
---
|
||||
|
||||
## Supported Edit Types
|
||||
|
||||
### 1. Paralinguistic Tags (Sound Insertion)
|
||||
|
||||
Insert non-verbal sounds at specific positions in the audio:
|
||||
|
||||
| Tag | Effect | Example |
|
||||
|-----|--------|---------|
|
||||
| `<Laughter>` or `<Laughter:2>` | Insert laughter | `Hello <Laughter> world` |
|
||||
| `<Breathing>` | Insert breathing | `I'm tired <Breathing> let's rest` |
|
||||
| `<Sigh>` | Insert sigh | `Okay <Sigh> fine` |
|
||||
| `<Uhm>` | Insert "uhm" hesitation | `Well <Uhm> maybe` |
|
||||
| `<Surprise-oh>` | Surprised "oh!" | `<Surprise-oh> I didn't know!` |
|
||||
| `<Surprise-ah>` | Surprised "ah!" | `<Surprise-ah> That's amazing!` |
|
||||
| `<Surprise-wa>` | Surprised "wa!" | `<Surprise-wa> Incredible!` |
|
||||
| `<Confirmation-en>` | Confirmation sound | `<Confirmation-en> Yes` |
|
||||
| `<Question-ei>` | Question sound | `<Question-ei> Really?` |
|
||||
| `<Dissatisfaction-hnn>` | Dissatisfied "hnn" | `<Dissatisfaction-hnn> Not good` |
|
||||
|
||||
**Position matters:** The sound is inserted where you place the tag.
|
||||
|
||||
**💡 Pro Tip - Stronger Effects:** For more reliable and pronounced effects, include relevant text alongside the tag:
|
||||
```
|
||||
# Weaker effect
|
||||
I'm happy <Laughter>
|
||||
|
||||
# Stronger, more guaranteed effect
|
||||
I'm laughing haha <Laughter:2>
|
||||
```
|
||||
|
||||
The TTS-generated "haha" combined with the post-processing laughter creates a more natural and pronounced effect.
|
||||
|
||||
### 2. Emotion Tags (Whole Segment)
|
||||
|
||||
Apply emotional tone to the entire audio segment:
|
||||
|
||||
`<emotion:VALUE>` or `<emotion:VALUE:ITERATIONS>`
|
||||
|
||||
**Available emotions:** happy, sad, angry, excited, calm, fearful, surprised, disgusted, confusion, empathy, embarrass, depressed, coldness, admiration
|
||||
|
||||
**Examples:**
|
||||
```
|
||||
[Alice] I'm so happy to see you! <emotion:happy>
|
||||
[Bob] This is terrible news. <emotion:sad:2>
|
||||
```
|
||||
|
||||
**Position doesn't matter:** Tag can be anywhere in the text.
|
||||
|
||||
### 3. Style Tags (Whole Segment)
|
||||
|
||||
Apply speaking style to the entire audio segment:
|
||||
|
||||
`<style:VALUE>` or `<style:VALUE:ITERATIONS>`
|
||||
|
||||
**Available styles:** whisper, serious, child, older, girl, pure, sister, sweet, exaggerated, ethereal, generous, recite, act_coy, warm, shy, comfort, authority, chat, radio, soulful, gentle, story, vivid, program, news, advertising, roar, murmur, shout, deeply, loudly, arrogant, friendly
|
||||
|
||||
**Examples:**
|
||||
```
|
||||
[Alice] Come closer, I have a secret <style:whisper>
|
||||
[Bob] Listen carefully to this. <style:serious:2>
|
||||
```
|
||||
|
||||
**Position doesn't matter:** Tag can be anywhere in the text.
|
||||
|
||||
### 4. Speed Tags (Whole Segment)
|
||||
|
||||
Adjust speaking speed for the entire audio segment:
|
||||
|
||||
`<speed:VALUE>` or `<speed:VALUE:ITERATIONS>`
|
||||
|
||||
**Available speeds:** faster, slower, more_faster, more_slower
|
||||
|
||||
**Examples:**
|
||||
```
|
||||
[Alice] Quick! We need to hurry! <speed:faster>
|
||||
[Bob] Let me think about this... <speed:slower:2>
|
||||
```
|
||||
|
||||
**Position doesn't matter:** Tag can be anywhere in the text.
|
||||
|
||||
### 5. Voice Restoration Tags (Whole Segment)
|
||||
|
||||
Restore voice resemblance after edit effects using ChatterBox Official 23-Lang Voice Changer:
|
||||
|
||||
`<restore>`, `<restore:N>`, or `<restore:N@M>`
|
||||
|
||||
**Syntax:**
|
||||
- `<restore>` - 1 VC pass using original pre-edit audio as reference
|
||||
- `<restore:2>` - 2 VC passes using original pre-edit audio as reference (for stronger restoration)
|
||||
- `<restore:1@2>` - 1 VC pass using audio from iteration 2 as reference
|
||||
|
||||
**⚠️ Why You Need This:**
|
||||
|
||||
Edit effects (especially after multiple iterations) can **degrade voice quality and alter voice resemblance**:
|
||||
- Paralinguistic effects (`<Laughter:3>`) add sounds but can muddy the voice
|
||||
- Style effects (`<style:whisper:4>`) change timbre and can distort characteristics
|
||||
- Multiple iterations compound these effects
|
||||
|
||||
The `<restore>` tag uses voice conversion to restore your original voice character while keeping the edit effects.
|
||||
|
||||
**Examples:**
|
||||
|
||||
**Basic restoration (use original voice):**
|
||||
```
|
||||
[Alice] I'm laughing so hard <Laughter:3> <restore>
|
||||
```
|
||||
- Applies 3 iterations of laughter (may degrade voice)
|
||||
- Restores original voice with 1 VC pass
|
||||
|
||||
**Stronger restoration:**
|
||||
```
|
||||
[Alice] I'm laughing so hard <Laughter:3> <restore:2>
|
||||
```
|
||||
- Applies 3 iterations of laughter (may degrade voice)
|
||||
- Restores original voice with 2 VC passes (stronger effect)
|
||||
|
||||
**Preserve partial effects (use intermediate iteration):**
|
||||
```
|
||||
[Bob] Listen carefully <style:whisper:2> <Laughter:3> <restore:1@2>
|
||||
```
|
||||
- Applies whisper 2 times (iterations 1-2)
|
||||
- Applies laughter 3 times (iterations 3-5)
|
||||
- Restores using iteration 2 audio (whispered version) with 1 VC pass
|
||||
- **Result:** Keeps whisper effect but removes laughter voice degradation
|
||||
|
||||
**When to use iteration reference:**
|
||||
- `<restore>` or `<restore:N>` - Restore completely to original clean voice
|
||||
- `<restore:N@M>` - Keep some effects (like whisper) while removing others (like laughter degradation)
|
||||
|
||||
**Processing order:** Restore tags ALWAYS run last, after all other edits complete.
|
||||
|
||||
**Position doesn't matter:** Tag can be anywhere in the text.
|
||||
|
||||
---
|
||||
|
||||
## Syntax Guide
|
||||
|
||||
### Basic Syntax
|
||||
|
||||
**Default (1 iteration):**
|
||||
```
|
||||
<Laughter>
|
||||
<emotion:happy>
|
||||
<style:whisper>
|
||||
<speed:faster>
|
||||
```
|
||||
|
||||
**Custom iterations:**
|
||||
```
|
||||
<Laughter:3> # 3 iterations of laughter
|
||||
<emotion:happy:2> # 2 iterations of happy emotion
|
||||
<style:whisper:4> # 4 iterations of whisper style
|
||||
```
|
||||
|
||||
### Multiple Tags
|
||||
|
||||
**Option 1 - Pipe-separated (single tag):**
|
||||
```
|
||||
[Alice] Hello world! <Laughter:2|style:whisper|emotion:happy>
|
||||
```
|
||||
|
||||
**Option 2 - Separate tags:**
|
||||
```
|
||||
[Alice] Hello world! <Laughter:2><style:whisper><emotion:happy>
|
||||
```
|
||||
|
||||
**Option 3 - Mixed (both work):**
|
||||
```
|
||||
[Alice] Hello <Laughter:2|emotion:happy> world! <style:whisper>
|
||||
```
|
||||
|
||||
### Processing Order
|
||||
|
||||
When multiple tags are present:
|
||||
|
||||
1. **First:** Non-paralinguistic tags in order: emotion → style → speed
|
||||
2. **Second:** Paralinguistic tags (preserves position for sound insertion)
|
||||
3. **Last:** Voice restoration (if present)
|
||||
|
||||
**Example:**
|
||||
```
|
||||
[Alice] Hello <Laughter:2> world <style:whisper:1> <restore>
|
||||
```
|
||||
|
||||
Execution:
|
||||
1. Generate TTS: "Hello world"
|
||||
2. Apply whisper style (1 iteration)
|
||||
3. Insert Laughter at position (2 iterations)
|
||||
4. Restore voice to original character (1 VC pass)
|
||||
|
||||
---
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Simple Effects
|
||||
|
||||
**Laughter:**
|
||||
```
|
||||
[Alice] That's the funniest thing I've heard <Laughter>!
|
||||
```
|
||||
|
||||
**Multiple iterations:**
|
||||
```
|
||||
[Bob] I can't stop laughing <Laughter:3>, this is amazing!
|
||||
```
|
||||
|
||||
### Combined Effects
|
||||
|
||||
**Emotion + Paralinguistic:**
|
||||
```
|
||||
[Alice] I'm so excited <Laughter:2|emotion:happy> to see you!
|
||||
```
|
||||
|
||||
**Style + Speed:**
|
||||
```
|
||||
[Bob] Listen carefully <style:whisper|speed:slower>, this is important.
|
||||
```
|
||||
|
||||
### Full Example with Character Switching
|
||||
|
||||
```
|
||||
[Narrator] The story begins on a dark night.
|
||||
|
||||
[Alice|seed:42] Oh no <Surprise-oh>, what was that sound? <emotion:fearful>
|
||||
|
||||
[Bob] Don't worry <style:comfort>, I'll check it out. <Breathing>
|
||||
|
||||
[Alice] Please be careful! <style:whisper|emotion:worried>
|
||||
```
|
||||
|
||||
### SRT Mode
|
||||
|
||||
```
|
||||
1
|
||||
00:00:00,000 --> 00:00:03,000
|
||||
[Alice] Hello there <Laughter> my friend!
|
||||
|
||||
2
|
||||
00:00:03,500 --> 00:00:06,000
|
||||
[Bob|seed:123] Hi! <style:excited|emotion:happy>
|
||||
|
||||
3
|
||||
00:00:06,500 --> 00:00:09,000
|
||||
[Alice] Let me tell you a secret <style:whisper:2>
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ChatterBox 23-Lang v2 Users: Native vs Post-Processing Tags
|
||||
|
||||
### Understanding the Difference
|
||||
|
||||
**ChatterBox Official 23-Lang v2** has **native special tokens** built into the model (e.g., `<laughter>`, `<giggle>`, `<whisper>`). Our **Step Audio EditX post-processing system** uses similar-looking tags but works differently.
|
||||
|
||||
### Syntax Distinction
|
||||
|
||||
| Syntax | System | When to Use |
|
||||
|--------|--------|-------------|
|
||||
| `<laughter>` | **ChatterBox v2 Native** | Model generates laughter during TTS (experimental, may not work well) |
|
||||
| `<laughter:1>` | **Step Audio EditX Post-Processing** | Applied AFTER TTS generation (more reliable) |
|
||||
|
||||
**The colon (`:`) makes the difference!**
|
||||
|
||||
### Examples
|
||||
|
||||
```
|
||||
# Native ChatterBox v2 tag (model-generated during TTS)
|
||||
[Alice] I'm laughing <laughter> so hard!
|
||||
|
||||
# Step Audio EditX tag (post-processing after TTS)
|
||||
[Alice] I'm laughing <Laughter:2> so hard!
|
||||
```
|
||||
|
||||
### Which Should You Use?
|
||||
|
||||
**For ChatterBox 23-Lang users:**
|
||||
|
||||
✅ **Use Step Audio EditX tags** (`<Laughter:2>`, etc.) - More reliable, works consistently
|
||||
|
||||
⚠️ **ChatterBox v2 native tags** (`<laughter>`, `<giggle>`) - Experimental, limited functionality:
|
||||
- Tokens exist in vocabulary but may produce minimal/no effect
|
||||
- No official documentation from ResembleAI
|
||||
- Results are inconsistent
|
||||
- See [ResembleAI/chatterbox #186](https://github.com/resemble-ai/chatterbox/issues/186)
|
||||
|
||||
**Console Warning:**
|
||||
|
||||
If you use ChatterBox v2 native tags without the colon, you may see:
|
||||
```
|
||||
⚠️ Using ChatterBox v2 native tag '<laughter>' - experimental feature with limited effect.
|
||||
For more reliable results, use Step Audio EditX: '<Laughter:2>'
|
||||
```
|
||||
|
||||
### Compatibility Note
|
||||
|
||||
Step Audio EditX tags work with **ALL engines** (F5-TTS, ChatterBox, Higgs Audio, VibeVoice), not just ChatterBox. Native ChatterBox tags only work with ChatterBox 23-Lang v2.
|
||||
|
||||
---
|
||||
|
||||
## Important Limitations
|
||||
|
||||
### ⚠️ Language Support Warning
|
||||
|
||||
**Step Audio EditX supports ONLY:**
|
||||
- **Mandarin Chinese** (primary)
|
||||
- **English**
|
||||
- **Sichuanese** (Chinese dialect)
|
||||
- **Cantonese** (Chinese dialect)
|
||||
- **Japanese**
|
||||
- **Korean**
|
||||
|
||||
**If using other languages** (French, German, Spanish, etc.):
|
||||
|
||||
❌ **PROBLEM:** Step Audio EditX will process non-supported languages as English or Chinese, which will:
|
||||
- **Lose the original accent** - French audio may lose French pronunciation
|
||||
- **Distort the audio** - The model tries to "fix" unknown languages
|
||||
- **Generate poor quality** - Editing may sound unnatural or broken
|
||||
|
||||
✅ **SOLUTION:**
|
||||
1. **Don't use edit tags** with unsupported languages
|
||||
2. **Use only supported languages** when edit tags are needed
|
||||
3. **Test carefully** if mixing languages - some distortion may be acceptable
|
||||
|
||||
**Example - BAD:**
|
||||
```
|
||||
[Alice:fr] Bonjour mon ami <Laughter> # French + edit tag = accent loss/distortion
|
||||
```
|
||||
|
||||
**Example - GOOD:**
|
||||
```
|
||||
[Alice:fr] Bonjour mon ami # French without edit tag = preserves accent
|
||||
[Alice:en] Hello my friend <Laughter> # English + edit tag = works perfectly
|
||||
```
|
||||
|
||||
### Duration Limits
|
||||
|
||||
**Minimum:** 0.5 seconds - Segments shorter than this will skip editing
|
||||
**Maximum:** 30 seconds - Segments longer than this will skip editing
|
||||
|
||||
**Warning messages:**
|
||||
```
|
||||
⚠️ Segment too short (0.3s < 0.5s). Edit tags will be skipped.
|
||||
⚠️ Segment too long (35.2s > 30s). Edit tags will be skipped. Split your text into shorter segments.
|
||||
```
|
||||
|
||||
**Solution for long segments:**
|
||||
```
|
||||
# BAD - Too long
|
||||
[Alice] [Very long paragraph that exceeds 30 seconds...] <Laughter>
|
||||
|
||||
# GOOD - Split into multiple segments
|
||||
[Alice] First part of the paragraph. <Laughter>
|
||||
[Alice] Second part continues here.
|
||||
[Alice] Final part with the conclusion. <emotion:happy>
|
||||
```
|
||||
|
||||
### Batch Processing (Automatic)
|
||||
|
||||
Edit tags are **automatically batched** for efficiency:
|
||||
|
||||
✅ **What happens:**
|
||||
1. ALL TTS audio is generated first (for all segments/subtitles)
|
||||
2. Step Audio EditX loads ONCE
|
||||
3. ALL segments with edit tags are processed together
|
||||
4. Engine unloads
|
||||
|
||||
✅ **Benefits:**
|
||||
- Faster generation (engine loads once, not per segment)
|
||||
- Less VRAM thrashing
|
||||
- Better performance
|
||||
|
||||
**You don't need to do anything** - batching happens automatically in both regular and SRT modes.
|
||||
|
||||
---
|
||||
|
||||
## Performance Tips
|
||||
|
||||
### 1. Use Iterations Wisely
|
||||
|
||||
More iterations = stronger effect BUT **can alter voice quality and resemblance**:
|
||||
|
||||
```
|
||||
<Laughter:1> # Subtle effect, minimal voice change (recommended)
|
||||
<Laughter:2> # Moderate effect, slight voice alteration (acceptable)
|
||||
<Laughter:3> # Strong effect, noticeable voice change (risky)
|
||||
<Laughter:4> # Very strong effect, significant voice distortion (not recommended)
|
||||
```
|
||||
|
||||
⚠️ **Voice Resemblance Warning:**
|
||||
- **1-2 iterations:** Generally safe, minimal voice alteration
|
||||
- **3+ iterations:** High risk of voice distortion and quality degradation
|
||||
- Each iteration processes the audio again, which can degrade the original voice characteristics
|
||||
|
||||
**Recommendation:** Start with 1 iteration, only increase to 2 if needed. Avoid 3+ unless you're willing to accept voice changes.
|
||||
|
||||
**✅ Voice Restoration Available:** Use the `<restore>` tag to restore original voice resemblance after editing. See the Voice Restoration Tags section above for details.
|
||||
|
||||
### 2. Combine Tags Efficiently
|
||||
|
||||
**Less efficient (multiple passes):**
|
||||
```
|
||||
[Alice] Text here <emotion:happy>
|
||||
[Alice] More text <emotion:happy>
|
||||
[Alice] Even more <emotion:happy>
|
||||
```
|
||||
|
||||
**More efficient (apply once):**
|
||||
```
|
||||
[Alice|emotion:happy] Text here. More text. Even more.
|
||||
```
|
||||
|
||||
Though with inline tags, segment-level parameters are cleaner for consistent effects across multiple sentences.
|
||||
|
||||
### 3. Position Paralinguistic Tags Carefully
|
||||
|
||||
**Position matters for sound insertion:**
|
||||
```
|
||||
Hello <Laughter> world # Laughter after "Hello"
|
||||
Hello world <Laughter> # Laughter at end
|
||||
<Laughter> Hello world # Laughter at start
|
||||
```
|
||||
|
||||
Choose the position that sounds most natural for your use case.
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Problem: Edit tags not applied
|
||||
|
||||
**Possible causes:**
|
||||
1. **Segment too short/long** - Check duration (0.5s - 30s range)
|
||||
2. **Syntax error** - Verify tag format: `<type:value:iterations>`
|
||||
3. **Model not downloaded** - First use triggers download (be patient)
|
||||
|
||||
**Check console output:**
|
||||
```
|
||||
🎨 Applying Step Audio EditX processing to 3 segments...
|
||||
📝 Segment 1 - Applying edit tags:
|
||||
🎨 Applying paralinguistic: Laughter:2
|
||||
✅ Edit post-processing complete for 3 segments
|
||||
```
|
||||
|
||||
### Problem: Audio quality degraded
|
||||
|
||||
**Possible causes:**
|
||||
1. **Too many iterations** - Reduce iterations (try 1-2 instead of 3-4)
|
||||
2. **Incompatible tags** - Some emotion+style combinations may clash
|
||||
3. **Unsupported language** - Check language support (see warning above)
|
||||
|
||||
**Solution:**
|
||||
- Use fewer iterations
|
||||
- Test individual tags separately
|
||||
- Use only supported languages with edit tags
|
||||
|
||||
### Problem: "Segment too long" warning
|
||||
|
||||
**Solution:** Split text into shorter segments:
|
||||
|
||||
```
|
||||
# Instead of:
|
||||
[Alice] [300-word monologue that lasts 45 seconds] <Laughter>
|
||||
|
||||
# Do this:
|
||||
[Alice] First paragraph of about 2-3 sentences.
|
||||
[Alice] Second paragraph continues. <Laughter>
|
||||
[Alice] Third paragraph concludes the thought. <emotion:happy>
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Advanced Usage
|
||||
|
||||
### Combining with Segment Parameters
|
||||
|
||||
Edit tags work with ALL segment parameters:
|
||||
|
||||
```
|
||||
[Alice|seed:42|temperature:0.8|speed:1.2] Hello <Laughter:2> friend! <emotion:happy>
|
||||
```
|
||||
|
||||
Processing:
|
||||
1. TTS uses seed:42, temp:0.8, speed:1.2
|
||||
2. Edit tags applied after generation
|
||||
|
||||
### Using with Language Switching
|
||||
|
||||
**Supported languages ONLY:**
|
||||
```
|
||||
[Alice:en] Hello my friend <Laughter> # English - works ✓
|
||||
[Bob:zh] 你好朋友 <Laughter> # Chinese - works ✓
|
||||
[Alice:ja] こんにちは友達 <Laughter> # Japanese - works ✓
|
||||
[Bob:ko] 안녕 친구 <Laughter> # Korean - works ✓
|
||||
```
|
||||
|
||||
**Unsupported languages - AVOID:**
|
||||
```
|
||||
[Alice:fr] Bonjour mon ami <Laughter> # French - will distort! ✗
|
||||
[Bob:de] Hallo mein Freund <Laughter> # German - will distort! ✗
|
||||
[Alice:es] Hola mi amigo <Laughter> # Spanish - will distort! ✗
|
||||
```
|
||||
|
||||
### Creating Natural Conversations
|
||||
|
||||
```
|
||||
[Narrator] The interview begins.
|
||||
|
||||
[Host] Welcome to the show! <emotion:excited>
|
||||
|
||||
[Guest] Thank you for having me. <style:warm>
|
||||
|
||||
[Host] So tell us <style:chat>, what's your secret? <Laughter>
|
||||
|
||||
[Guest] Well <Uhm> it's quite simple actually. <style:story>
|
||||
|
||||
[Host] That's amazing <Surprise-oh>! <emotion:surprised>
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Best Practices
|
||||
|
||||
### 1. Start Simple
|
||||
|
||||
Begin with single tags before combining:
|
||||
|
||||
```
|
||||
# Step 1: Test basic tag
|
||||
[Alice] Hello <Laughter>
|
||||
|
||||
# Step 2: Add iterations
|
||||
[Alice] Hello <Laughter:2>
|
||||
|
||||
# Step 3: Combine effects
|
||||
[Alice] Hello <Laughter:2|emotion:happy>
|
||||
```
|
||||
|
||||
### 2. Match Tags to Context
|
||||
|
||||
Choose appropriate effects for the situation:
|
||||
|
||||
```
|
||||
# Scary scene
|
||||
[Alice] What was that? <Surprise-oh|emotion:fearful>
|
||||
|
||||
# Happy scene
|
||||
[Bob] We did it! <Laughter:2|emotion:excited>
|
||||
|
||||
# Sad scene
|
||||
[Alice] I can't believe it's over. <Sigh|emotion:sad>
|
||||
|
||||
# Secret conversation
|
||||
[Bob] Come closer. <style:whisper|emotion:calm>
|
||||
```
|
||||
|
||||
### 3. Test and Iterate
|
||||
|
||||
1. Generate with basic tag
|
||||
2. Listen to result
|
||||
3. Adjust iterations if needed
|
||||
4. Try different tag combinations
|
||||
5. Find what works best for your use case
|
||||
|
||||
### 4. Use Language Hints Correctly
|
||||
|
||||
**Always match edit language to TTS language:**
|
||||
|
||||
```
|
||||
# GOOD - Language consistency
|
||||
[Alice:en] English text here <Laughter> # Both English
|
||||
[Bob:zh] 中文文本 <Laughter> # Both Chinese
|
||||
|
||||
# BAD - Language mismatch
|
||||
[Alice:fr] French text <Laughter> # French TTS + English edit model = distortion
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Reference
|
||||
|
||||
### All Paralinguistic Tags
|
||||
`<Breathing>`, `<Laughter>`, `<Sigh>`, `<Uhm>`, `<Surprise-oh>`, `<Surprise-ah>`, `<Surprise-wa>`, `<Confirmation-en>`, `<Question-ei>`, `<Dissatisfaction-hnn>`
|
||||
|
||||
### All Emotions
|
||||
`<emotion:happy>`, `<emotion:sad>`, `<emotion:angry>`, `<emotion:excited>`, `<emotion:calm>`, `<emotion:fearful>`, `<emotion:surprised>`, `<emotion:disgusted>`, `<emotion:confusion>`, `<emotion:empathy>`, `<emotion:embarrass>`, `<emotion:depressed>`, `<emotion:coldness>`, `<emotion:admiration>`
|
||||
|
||||
### All Styles
|
||||
`<style:whisper>`, `<style:serious>`, `<style:child>`, `<style:older>`, `<style:girl>`, `<style:pure>`, `<style:sister>`, `<style:sweet>`, `<style:exaggerated>`, `<style:ethereal>`, `<style:generous>`, `<style:recite>`, `<style:act_coy>`, `<style:warm>`, `<style:shy>`, `<style:comfort>`, `<style:authority>`, `<style:chat>`, `<style:radio>`, `<style:soulful>`, `<style:gentle>`, `<style:story>`, `<style:vivid>`, `<style:program>`, `<style:news>`, `<style:advertising>`, `<style:roar>`, `<style:murmur>`, `<style:shout>`, `<style:deeply>`, `<style:loudly>`, `<style:arrogant>`, `<style:friendly>`
|
||||
|
||||
### All Speeds
|
||||
`<speed:faster>`, `<speed:slower>`, `<speed:more_faster>`, `<speed:more_slower>`
|
||||
|
||||
### Voice Restoration
|
||||
`<restore>`, `<restore:N>`, `<restore:N@M>` (where N = VC passes 1-5, M = reference iteration number)
|
||||
|
||||
---
|
||||
|
||||
## FAQ
|
||||
|
||||
**Q: Can I use edit tags with any TTS engine?**
|
||||
A: Yes! Edit tags work with F5-TTS, ChatterBox, Higgs Audio, VibeVoice, and any future engines.
|
||||
|
||||
**Q: Do edit tags slow down generation?**
|
||||
A: Slightly. The TTS generates normally, then Step Audio EditX loads once and processes all tagged segments in batch. Total time depends on number of segments and iterations.
|
||||
|
||||
**Q: Can I use edit tags in SRT mode?**
|
||||
A: Yes! Batch processing is automatically applied - all subtitles are generated first, then all edits are applied at once.
|
||||
|
||||
**Q: What if I make a typo in a tag?**
|
||||
A: Unknown tags are ignored and stripped from the text. Check console for warnings.
|
||||
|
||||
**Q: Can I nest tags?**
|
||||
A: No. Use pipe syntax or separate tags: `<tag1|tag2>` or `<tag1><tag2>`
|
||||
|
||||
**Q: Do edit tags work with ChatterBox special tokens?**
|
||||
A: Yes, but ChatterBox v2 uses different syntax (`<giggle>`, `<whisper>` built into model). Our edit tags are post-processing, not model-native.
|
||||
|
||||
**Q: Will edit tags affect my voice cloning?**
|
||||
A: Yes, they can. TTS generates with your voice reference normally, but each edit iteration can alter voice quality and resemblance:
|
||||
- **1 iteration:** Minimal impact, usually safe
|
||||
- **2 iterations:** Slight alteration, generally acceptable
|
||||
- **3+ iterations:** Noticeable voice change, may lose original voice characteristics
|
||||
|
||||
Use the minimum iterations needed for your desired effect. To restore voice resemblance after editing, use the `<restore>` tag which applies ChatterBox Voice Changer to recover the original voice character.
|
||||
|
||||
**Q: Can I disable edit tags temporarily?**
|
||||
A: Just remove them from your text. No special setting needed.
|
||||
|
||||
---
|
||||
|
||||
## Support
|
||||
|
||||
Related guides:
|
||||
- `docs/HIGGS_AUDIO_V3_INLINE_TAGS.md` for Higgs Audio v3 native tags
|
||||
- `docs/COSYVOICE3_TAGS_GUIDE.md` for CosyVoice3 native tags
|
||||
|
||||
For issues, feature requests, or questions:
|
||||
- GitHub Issues: https://github.com/diodiogod/TTS-Audio-Suite/issues
|
||||
- Check console output for detailed processing information
|
||||
- Test with simple examples first before complex combinations
|
||||
|
||||
---
|
||||
|
||||
**Last Updated:** 2025-12-08
|
||||
**Compatible with:** TTS Audio Suite v4.4.0+
|
||||
@@ -1,324 +0,0 @@
|
||||
# IndexTTS-2 Emotion Control Guide
|
||||
|
||||
IndexTTS-2 features advanced emotion control capabilities that allow you to precisely control the emotional expression of generated speech. This guide covers all available emotion control methods and their applications.
|
||||
|
||||
## Overview
|
||||
|
||||
IndexTTS-2 supports multiple emotion control methods that can be combined for sophisticated emotional expression:
|
||||
|
||||
- **Direct Audio Reference**: Use any audio file as an emotion reference
|
||||
- **Character Voices**: Use character audio references from the Character Voices node
|
||||
- **Emotion Vectors**: Manual 8-emotion slider control with precise values
|
||||
- **Text Emotion**: AI-powered QwenEmotion analysis from text descriptions with dynamic templates
|
||||
- **Character Tag Emotions**: Per-character emotion control using `[Character:emotion_ref]` syntax
|
||||
|
||||
## Emotion Control Inputs and Blending
|
||||
|
||||
The IndexTTS-2 Engine has two emotion inputs:
|
||||
|
||||
- **`emotion_control`**: vector controls or Qwen text emotion
|
||||
- **`emotion_audio`**: an audio reference from an AUDIO or Character Voices node
|
||||
|
||||
Both inputs may be connected at the same time. Text emotion is analyzed into an
|
||||
8-value vector; audio emotion remains an audio-derived conditioning signal. The
|
||||
engine blends the two signals in its latent emotion-conditioning space rather
|
||||
than converting the audio into the eight visible vector values.
|
||||
|
||||
For backward compatibility, the original `emotion_control` socket still accepts
|
||||
legacy audio connections, but new workflows should use `emotion_audio` for
|
||||
audio references. A character tag such as `[Alice:angry_bob]` supplies a
|
||||
segment-local audio reference and can also be combined with vector/text emotion
|
||||
for that segment.
|
||||
|
||||
## Method 1: Direct Audio Reference
|
||||
|
||||
Connect any audio file directly to the IndexTTS-2 Engine's `emotion_audio` input.
|
||||
|
||||
**How it works:**
|
||||
|
||||
- IndexTTS-2 analyzes the emotional characteristics of your reference audio
|
||||
- The emotional style is applied to all generated speech
|
||||
- Works with any audio format (WAV, MP3, etc.)
|
||||
|
||||
**Best practices:**
|
||||
|
||||
- Use audio clips with clear emotional expression
|
||||
- Choose audio with consistent voice characteristics for best results
|
||||
- Avoid background music or noise
|
||||
|
||||
**Example:**
|
||||
|
||||
```
|
||||
AUDIO node → IndexTTS-2 Engine (emotion_audio)
|
||||
```
|
||||
|
||||
## Method 2: Character Voices Audio Reference
|
||||
|
||||
Use the `opt_narrator` output from the 🎭 Character Voices node as an emotion reference.
|
||||
|
||||
**Setup:**
|
||||
|
||||
1. Add a 🎭 Character Voices node
|
||||
2. Select a voice with the desired emotional expression
|
||||
3. Connect `opt_narrator` output to IndexTTS-2 Engine `emotion_audio` input
|
||||
|
||||
**Advantages:**
|
||||
|
||||
- Leverages your existing voice library
|
||||
- Consistent character-based emotions
|
||||
- Easy to manage and organize
|
||||
|
||||
**Example workflow:**
|
||||
|
||||
```
|
||||
🎭 Character Voices (David_Attenborough) → opt_narrator → IndexTTS-2 Engine (emotion_audio)
|
||||
```
|
||||
|
||||
## Method 3: Emotion Vectors
|
||||
|
||||
Use the 🌈 IndexTTS-2 Emotion Vectors node for precise manual control over 8 different emotions.
|
||||
Connect its `emotion_control` output to the IndexTTS-2 Engine's `emotion_control` input.
|
||||
|
||||
**Available emotions:**
|
||||
|
||||
- **Happy**: Joy, excitement, positivity (0.0-1.2)
|
||||
- **Angry**: Aggression, frustration, intensity (0.0-1.2)
|
||||
- **Sad**: Melancholy, sorrow, downcast tone (0.0-1.2)
|
||||
- **Surprised**: Amazement, shock, wonder (0.0-1.2)
|
||||
- **Afraid**: Fear, anxiety, nervousness (0.0-1.2)
|
||||
- **Disgusted**: Revulsion, displeasure, rejection (0.0-1.2)
|
||||
- **Calm**: Peaceful, relaxed, steady (0.0-1.2)
|
||||
- **Melancholic**: Thoughtful sadness, wistfulness (0.0-1.2)
|
||||
|
||||
**Usage tips:**
|
||||
|
||||
- Values above 1.0 create more intense emotional expression BUT MAY interfear with the cloned voice resemblance
|
||||
- Combine multiple emotions for complex feelings (e.g., 0.8 Happy + 0.3 Surprised = excited joy)
|
||||
- Start with single emotions, then experiment with combinations
|
||||
- Use the `random` buttom to get a completely random emotion pattern. Might be too strong.
|
||||
|
||||
## Method 4: Text Emotion (Dynamic Analysis)
|
||||
|
||||
Use the 🌈 IndexTTS-2 Text Emotion node for AI-powered emotion analysis with dynamic templates.
|
||||
Connect its `emotion_control` output to the IndexTTS-2 Engine's `emotion_control` input.
|
||||
|
||||
### Static Text Emotion
|
||||
|
||||
Provide a simple emotion description that applies to all text segments:
|
||||
|
||||
```
|
||||
Input: "angry and frustrated"
|
||||
Result: All speech generated with angry, frustrated emotion
|
||||
```
|
||||
|
||||
### Dynamic Templates with {seg}
|
||||
|
||||
Use the `{seg}` placeholder for contextual, per-segment emotion analysis:
|
||||
|
||||
**Template examples:**
|
||||
|
||||
- `"Happy character speaking: {seg}"` - Cheerful narrator
|
||||
- `"Angry boss yelling: {seg}"` - Aggressive authority figure
|
||||
- `"Calm meditation guide: {seg}"` - Peaceful instructor
|
||||
- `"Excited game show host: {seg}"` - Energetic presenter
|
||||
|
||||
**How dynamic templates work:**
|
||||
|
||||
1. IndexTTS-2 processes each text segment separately
|
||||
2. `{seg}` gets replaced with the actual segment text
|
||||
3. QwenEmotion analyzes the combined context + content
|
||||
4. Unique emotion vector generated for each segment
|
||||
|
||||
**Example:**
|
||||
|
||||
```
|
||||
Template: "Worried parent speaking: {seg}"
|
||||
Segment: "Where have you been?"
|
||||
Analysis: "Worried parent speaking: Where have you been?"
|
||||
Result: Anxious, concerned vocal expression
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Combining Audio with Vectors or Text
|
||||
|
||||
Connect both emotion sources when you want an audio performance to provide the
|
||||
base delivery while vectors or Qwen text analysis add a targeted emotional
|
||||
direction:
|
||||
|
||||
```text
|
||||
🎭 Character Voices (opt_narrator) ──→ emotion_audio
|
||||
🌈 Emotion Vectors or Text Emotion ──→ emotion_control
|
||||
IndexTTS-2 Engine
|
||||
```
|
||||
|
||||
`emotion_alpha` is the shared overall emotion-intensity control. The audio
|
||||
reference and vector/text signal are blended during IndexTTS-2 conditioning;
|
||||
they are not generated as two separate voices and mixed afterward.
|
||||
|
||||
For example, an audio reference can provide a natural speaking style while a
|
||||
`[sad:+0.2|calm:-0.1]` inline adjustment adds a restrained sadness to one
|
||||
segment. A Qwen text preset can be used the same way.
|
||||
|
||||
Character audio and inline emotion parameters can share one tag:
|
||||
|
||||
```text
|
||||
[Bob:br_ivan_raiva3|sad:+0.25|calm:-0.10] Bob speaks with a restrained overlay.
|
||||
[Bob:br_ivan_raiva3|emotion:"quiet grief masking frustration"] Bob uses Qwen text emotion too.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Inline Emotion Switching
|
||||
|
||||
Numeric emotion tags can replace or adjust the vector for one text segment:
|
||||
|
||||
```text
|
||||
[sad:0.7|calm:0.2] This uses explicit absolute values.
|
||||
[sad:+0.3|calm:-0.2] This modifies the connected vector.
|
||||
[vector:0,0,0.7,0,0,0.4,0,0.2] This supplies all eight absolute values.
|
||||
[vector:+0,+0,+0.3,+0,+0,+0,+0,-0.2] This supplies eight deltas.
|
||||
```
|
||||
|
||||
The full-vector order is `happy, angry, sad, afraid, disgusted, melancholic,
|
||||
surprised, calm`. Unsigned named values are absolute; `+` and `-` named values
|
||||
are relative. A full vector is relative only when every value carries a sign.
|
||||
|
||||
Text-emotion controls support saved presets and quoted descriptions:
|
||||
|
||||
```text
|
||||
[emotion:restrained_anger] Text using a saved preset.
|
||||
[emotion:"Restrained anger masking disappointment"] Direct text control.
|
||||
[emotion:"Analyze this delivery as nervous anticipation: {seg}"] Dynamic control.
|
||||
```
|
||||
|
||||
Inline controls override connected global vector/text values for that segment.
|
||||
A character audio emotion reference such as `[Alice:sad_reference]` replaces the
|
||||
global audio reference for that segment, but it can still blend with the
|
||||
segment's vector/text control. Inline settings revert at the next segment and do
|
||||
not mutate the connected vector.
|
||||
|
||||
The TTS Tag Editor provides the same interactive radar used by the IndexTTS-2
|
||||
Emotion Vectors node. Click an existing numeric emotion tag to open its radar
|
||||
as a contextual popover beside the tag. Create tags from **Inline Tags →
|
||||
IndexTTS-2**, and use **Manage Emotion Presets** in its Text Emotion section for
|
||||
the preset library. Text and vector presets are stored in
|
||||
`models/TTS/IndexTTS/emotion_presets.json`.
|
||||
|
||||
Radar changes appear in the editor text immediately. Intermediate drag/input
|
||||
states are not added to undo history: closing the popover commits one undoable
|
||||
change, while Cancel or Escape restores the tag exactly as it was when opened.
|
||||
|
||||
The editor's **Inline Tags** tab also includes an **IndexTTS-2** engine panel for
|
||||
inserting full absolute/delta vectors, named emotion values, saved text presets,
|
||||
quoted descriptions, and dynamic descriptions containing `{seg}`.
|
||||
|
||||
Emotion controls can be composed directly on character tags: place the caret on
|
||||
`[Bob:audio_reference]` and add a vector or text emotion to append it as a pipe
|
||||
parameter.
|
||||
|
||||
The named-emotion panel includes a magnitude slider and a press-drag-release
|
||||
radial picker: direction chooses the emotion and distance chooses its value.
|
||||
The operation dropdown determines whether that value is absolute, a positive
|
||||
delta, or a negative delta. Saved text and vector presets refresh in the sidebar
|
||||
immediately after they are changed in the preset manager.
|
||||
|
||||
Clicking a saved `[emotion:preset_name]` tag in the editor opens a small anchored
|
||||
preset dropdown, allowing that line's preset to be swapped without opening the
|
||||
full manager. Adding an emotion control while the caret is inside a pure emotion
|
||||
tag replaces that tag; when the caret is inside a character/audio tag, the
|
||||
editor appends or updates the emotion parameter after the existing `|` fields.
|
||||
|
||||
Named tags remain readable while only some emotions are active. If radar editing
|
||||
activates all eight emotions, the editor automatically converts the result to the
|
||||
shorter ordered `[vector:...]` form.
|
||||
|
||||
## Character Tag Emotion Control
|
||||
|
||||
Control emotions per character using inline tags in your text: `[Character:emotion_ref]`
|
||||
|
||||
**Syntax:**
|
||||
|
||||
```
|
||||
[CharacterName:emotion_reference]
|
||||
```
|
||||
|
||||
**emotion_reference options:**
|
||||
|
||||
- Any character name from your voices library (uses that character's voice as emotion)
|
||||
- Custom emotion references
|
||||
|
||||
**Examples:**
|
||||
|
||||
```
|
||||
Hello everyone! [Alice:happy_sarah] I'm so excited to be here today!
|
||||
[Bob:angry_tom] That's completely unacceptable behavior.
|
||||
[Narrator:David] Meanwhile, in a distant galaxy...
|
||||
[Bob:br_ivan_raiva3|sad:+0.25] Bob uses an audio reference plus a vector delta.
|
||||
|
||||
*assuming happy_sarah, angry_tom and David are alias or character voices in yout folder with that name
|
||||
```
|
||||
|
||||
\*assuming happy_sarah, angry_tom and David are alias or character voices in yout folder with that name
|
||||
|
||||
|
||||
|
||||
**Character tag behavior:**
|
||||
|
||||
- Character tags select the speaker and can provide a segment-local audio emotion reference
|
||||
- A segment-local audio reference replaces the global audio reference for that segment
|
||||
- Global or inline vector/text controls can still blend with that audio reference
|
||||
- Other characters use the global audio and vector/text settings
|
||||
- Allows mixing different emotion sources in the same audio
|
||||
|
||||
## Emotion Alpha Control
|
||||
|
||||
The `emotion_alpha` parameter on the IndexTTS-2 Engine controls the intensity of emotion application:
|
||||
|
||||
**Values:**
|
||||
|
||||
- **0.0**: No emotion applied (neutral voice)
|
||||
- **0.5**: 50% emotion blend (subtle emotional influence)
|
||||
- **1.0**: Full emotion intensity (standard recommended setting)
|
||||
- **1.5**: 150% enhanced emotion (more dramatic)
|
||||
- **2.0**: Maximum emotion intensity (very dramatic)
|
||||
|
||||
## Practical Workflow Examples
|
||||
|
||||
### Example 1: Multi-Character Drama with Individual Emotions
|
||||
|
||||
```text
|
||||
[Alice:happy_sarah] Welcome to our cooking show!
|
||||
[Bob:serious_narrator] Today we'll be making pasta.
|
||||
[Alice:excited_sarah] I can't wait to get started!
|
||||
```
|
||||
|
||||
**Setup:**
|
||||
|
||||
- No global emotion control needed
|
||||
- Each character gets individual emotion via tags
|
||||
- `emotion_alpha=1.0` for more expressiveness
|
||||
|
||||
### Example 2: Mixed Emotion Control
|
||||
|
||||
**Global setup:**
|
||||
|
||||
- 🌈 IndexTTS-2 Text Emotion: `"Cheerful host presenting: {seg}"`
|
||||
- `emotion_alpha=0.8`
|
||||
|
||||
**Text with overrides:**
|
||||
|
||||
```text
|
||||
Welcome to our show! [Bob:serious_narrator] But first, a serious announcement.
|
||||
[Alice:excited_sarah] Now back to our regular programming!
|
||||
```
|
||||
|
||||
**Result:**
|
||||
|
||||
- Default segments use cheerful host emotion
|
||||
- Bob's line uses serious narrator emotion (overrides global)
|
||||
- Alice's line uses excited emotion (overrides global)
|
||||
|
||||
---
|
||||
|
||||
This comprehensive emotion control system gives you unprecedented flexibility in creating expressive, emotionally rich TTS audio for any application.
|
||||
@@ -1,119 +0,0 @@
|
||||
# Language Support by Engine
|
||||
|
||||
## Language Support by Engine
|
||||
|
||||
| Language | Code | F5-TTS | ChatterBox | ChatterBox 23L | VibeVoice | Higgs Audio 2 | Higgs Audio v3 | IndexTTS 2 / 2.5 | CosyVoice3 | Qwen3-TTS | Granite ASR | Step Audio EditX | Echo-TTS | Fish Audio S2 Pro | Dots TTS | DramaBox | OmniVoice | MOSS-TTS | MOSS-SoundEffect v2 | RVC |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| 🇺🇸 **English** | EN | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ Tier 1 | ✅ | ✅ Official model is English-only | ✅ | ✅ | ✅ Officially demonstrated prompt language | ✅ |
|
||||
| 🇨🇳 **Chinese** | ZH | ❌ | ❌ | ✅ | ✅ | ✅ (Mandarin) | ✅ | ✅ | ✅ + 18 dialects | ✅ | ❌ | ✅ (Mandarin + Sichuanese, Cantonese) | ❌ | ✅ Tier 1 | ✅ (Mandarin; official also exposes YUE separately outside this matrix) | ❌ | ✅ | ✅ | ✅ Officially demonstrated prompt language | ✅ |
|
||||
| 🇩🇪 **German** | DE | ✅ | ✅ (×3) | ✅ | ✅ (Kugel) | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇪🇸 **Spanish** | ES | ✅ | ❌ | ✅ | ✅ (Kugel) | ✅ | ✅ | ✅ IndexTTS-2.5 | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇫🇷 **French** | FR | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇮🇹 **Italian** | IT | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇯🇵 **Japanese** | JA | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ✅ IndexTTS-2.5 | ✅ | ✅ | ✅ (4.1-2b supports Japanese; plus variant does not) | ✅ | ❌ | ✅ Tier 1 | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇰🇷 **Korean** | KO | ❌ | ✅ | ✅ | ✅ (Kugel) | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ Tier 2 | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇷🇺 **Russian** | RU | ❌ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ Tier 2 | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇧🇷 **Portuguese** | PT | ✅ (BR) | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ (EU/BR*) | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ (Official PT tag is generic; upstream does not expose separate PT-BR/PT-PT tags and it may lean more European Portuguese than Brazilian Portuguese) | ❌ | ✅ (generic PT; official language space is much broader than this matrix) | ✅ | ❌ | ✅ |
|
||||
| 🇵🇱 **Polish** | PL | ✅ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇮🇳 **Hindi** | HI | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇪🇬 **Arabic** | AR | ❌ | ❌ | ✅ (Egyptian) | ✅ (Kugel) | ❌ | ✅ | ✅ IndexTTS-2.5 | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ Tier 2 | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇹🇷 **Turkish** | TR | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇹🇭 **Thai** | TH | ✅ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇳🇴 **Norwegian** | NO | ❌ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ |
|
||||
| 🇻🇳 **Vietnamese** | VI | ❌ | ❌ | ✅ (Viterbox) | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇦🇲 **Armenian** | HY | ❌ | ✅ | ❌ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ |
|
||||
| 🇬🇪 **Georgian** | KA | ❌ | ✅ | ❌ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ |
|
||||
| 🇩🇰 **Danish** | DA | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇫🇮 **Finnish** | FI | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇬🇷 **Greek** | EL | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇮🇱 **Hebrew** | HE | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇲🇾 **Malay** | MS | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇳🇱 **Dutch** | NL | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇸🇪 **Swedish** | SV | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇰🇪 **Swahili** | SW | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇿🇦 **Afrikaans** | AF | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇦🇱 **Albanian** | SQ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇳 **Assamese** | AS | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇪🇸 **Asturian** | AST | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇦🇿 **Azerbaijani** | AZ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇷🇺 **Bashkir** | BA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇪🇸 **Basque** | EU | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇧🇾 **Belarusian** | BE | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇧🇩 **Bengali** | BN | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇧🇦 **Bosnian** | BS | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇧🇬 **Bulgarian** | BG | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇪🇸 **Catalan** | CA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇵🇭 **Cebuano** | CEB | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇶 **Central Kurdish** | CKB | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇭🇷 **Croatian** | HR | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇨🇿 **Czech** | CS | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇷🇺 **Eastern Mari** | MHR | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🌍 **Esperanto** | EO | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇪🇪 **Estonian** | ET | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇪🇸 **Galician** | GL | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇳 **Gujarati** | GU | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇭🇹 **Haitian Creole** | HT | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇳🇬 **Hausa** | HA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇭🇺 **Hungarian** | HU | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇩 **Indonesian** | ID | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇩 **Javanese** | JV | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇳 **Kannada** | KN | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇰🇿 **Kazakh** | KK | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇷🇼 **Kinyarwanda** | RW | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇰🇬 **Kyrgyz** | KY | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇱🇻 **Latvian** | LV | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇨🇩 **Lingala** | LN | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇱🇹 **Lithuanian** | LT | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇰🇪 **Luo** | LUO | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇲🇰 **Macedonian** | MK | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇳 **Malayalam** | ML | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇲🇹 **Maltese** | MT | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇳🇿 **Māori** | MI | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇳 **Marathi** | MR | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇲🇳 **Mongolian** | MN | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇳🇵 **Nepali** | NE | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇫🇷 **Occitan** | OC | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇷 **Persian** | FA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇷🇴 **Romanian** | RO | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇿🇦 **Sepedi** | NSO | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇷🇸 **Serbian** | SR | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇿🇼 **Shona** | SN | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇸🇰 **Slovak** | SK | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇸🇮 **Slovene** | SL | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇵🇭 **Tagalog** | TL | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇹🇯 **Tajik** | TG | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇳 **Tamil** | TA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇳 **Telugu** | TE | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇺🇦 **Ukrainian** | UK | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇵🇰 **Urdu** | UR | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇨🇳 **Uyghur** | UG | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇺🇿 **Uzbek** | UZ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇿🇦 **Xhosa** | XH | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇿🇦 **Zulu** | ZU | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇲🇼 **Chichewa/Nyanja** | NY | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇳 **Eastern Punjabi** | PA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇺🇬 **Ganda** | LG | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇸 **Icelandic** | IS | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇮🇪 **Irish** | GA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇩🇿 **Kabyle** | KAB | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇨🇻 **Kabuverdianu** | KEA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇰🇪 **Kamba** | KAM | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇻🇦 **Latin** | LA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇱🇺 **Luxembourgish** | LB | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇪🇹 **Oromo** | OM | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇦🇫 **Pashto** | PS | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇵🇰 **Sindhi** | SD | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇸🇴 **Somali** | SO | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🇦🇴 **Umbundu** | UMB | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
| 🏴 **Welsh** | CY | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
|
||||
**Notes:**
|
||||
|
||||
- **CosyVoice3 Chinese**: Includes 18+ dialects (Cantonese, Sichuan, Dongbei, Shanghai, etc.)
|
||||
- **Higgs Audio 2**: Trained on EN, ZH (Mandarin), KO, DE, ES (English majority) - 10M hours AudioVerse dataset
|
||||
- **Higgs Audio v3**: Official model card lists 102 languages total: 85 polished-production languages and 17 usable but less polished languages.
|
||||
- **IndexTTS-2**: Trained on 55K+ hours - ZH, EN, JA primary.
|
||||
- **MOSS-TTS**: Official list includes ZH, EN, DE, ES, FR, JA, IT, HU, KO, RU, FA, AR, PL, PT, CS, DA, SV, EL, TR; HU/FA/CS are not separate columns in this matrix.
|
||||
- **OmniVoice**: Official model supports 600+ languages; this matrix only shows the suite's comparison subset.
|
||||
- **RVC**: Language-agnostic voice conversion/post-processing; it is marked supported for every language row.
|
||||
- Qwen3-TTS: Supports pt-BR only with instruction (voice design or harcoded voices). Base can't do instructions, so it will always ouput pt-PT.
|
||||
@@ -1,175 +0,0 @@
|
||||
# Model Download Sources
|
||||
|
||||
All entries below are generated from the single source of truth YAML.
|
||||
Use this as the canonical list of model repositories/links for offline setup.
|
||||
|
||||
## F5-TTS
|
||||
|
||||
| Component | Source | Size | Auto-Download | Notes |
|
||||
|---|---|---|---|---|
|
||||
| F5TTS_Base | [SWivid/F5-TTS](https://huggingface.co/SWivid/F5-TTS/tree/main/F5TTS_Base) | ~1.2GB | ✅ | English base model |
|
||||
| F5TTS_v1_Base | [SWivid/F5-TTS](https://huggingface.co/SWivid/F5-TTS/tree/main/F5TTS_v1_Base) | ~1.2GB | ✅ | English v1 model |
|
||||
| E2TTS_Base | [SWivid/E2-TTS](https://huggingface.co/SWivid/E2-TTS/tree/main/E2TTS_Base) | ~1.2GB | ✅ | English E2-TTS model |
|
||||
| F5-DE | [aihpi/F5-TTS-German](https://huggingface.co/aihpi/F5-TTS-German) | ~1.2GB | ✅ | German finetune |
|
||||
| F5-ES | [jpgallegoar/F5-Spanish](https://huggingface.co/jpgallegoar/F5-Spanish) | ~1.2GB | ✅ | Spanish finetune |
|
||||
| F5-FR | [RASPIAUDIO/F5-French-MixedSpeakers-reduced](https://huggingface.co/RASPIAUDIO/F5-French-MixedSpeakers-reduced) | ~1.2GB | ✅ | French finetune |
|
||||
| F5-JP | [Jmica/F5TTS](https://huggingface.co/Jmica/F5TTS) | ~1.2GB | ✅ | Japanese finetune |
|
||||
| F5-Hindi-Small | [SPRINGLab/F5-Hindi-24KHz](https://huggingface.co/SPRINGLab/F5-Hindi-24KHz) | ~632MB | ✅ | Hindi finetune |
|
||||
| Vocos Mel-24kHz | [charactr/vocos-mel-24khz](https://huggingface.co/charactr/vocos-mel-24khz) | N/A | ✅ | Optional vocoder |
|
||||
|
||||
## ChatterBox
|
||||
|
||||
| Component | Source | Size | Auto-Download | Notes |
|
||||
|---|---|---|---|---|
|
||||
| English | [ResembleAI/chatterbox](https://huggingface.co/ResembleAI/chatterbox) | ~2GB | ✅ | .pt model set |
|
||||
| German | [stlohrey/chatterbox_de](https://huggingface.co/stlohrey/chatterbox_de) | ~4.3GB | ✅ | .safetensors model set |
|
||||
| German (havok2) | [niobures/Chatterbox-TTS](https://huggingface.co/niobures/Chatterbox-TTS) | ~4.3GB | ✅ | .safetensors model set |
|
||||
| German (SebastianBodza) | [niobures/Chatterbox-TTS](https://huggingface.co/niobures/Chatterbox-TTS) | ~4.3GB | ✅ | .safetensors model set |
|
||||
| Italian | [niobures/Chatterbox-TTS](https://huggingface.co/niobures/Chatterbox-TTS) | ~4.3GB | ✅ | .pt model set |
|
||||
| French | [Thomcles/ChatterBox-fr](https://huggingface.co/Thomcles/ChatterBox-fr) | ~4.3GB | ✅ | .safetensors model set |
|
||||
| Russian | [niobures/Chatterbox-TTS](https://huggingface.co/niobures/Chatterbox-TTS) | ~4.3GB | ✅ | .safetensors model set |
|
||||
| Armenian | [niobures/Chatterbox-TTS](https://huggingface.co/niobures/Chatterbox-TTS) | ~4.3GB | ✅ | .safetensors model set |
|
||||
| Georgian | [niobures/Chatterbox-TTS](https://huggingface.co/niobures/Chatterbox-TTS) | ~4.3GB | ✅ | .safetensors model set |
|
||||
| Japanese | [niobures/Chatterbox-TTS](https://huggingface.co/niobures/Chatterbox-TTS) | ~4.3GB | ✅ | .safetensors model set |
|
||||
| Korean | [niobures/Chatterbox-TTS](https://huggingface.co/niobures/Chatterbox-TTS) | ~4.3GB | ✅ | .safetensors model set |
|
||||
| Norwegian | [akhbar/chatterbox-tts-norwegian](https://huggingface.co/akhbar/chatterbox-tts-norwegian) | ~4.3GB | ✅ | .safetensors model set |
|
||||
|
||||
## ChatterBox 23L
|
||||
|
||||
| Component | Source | Size | Auto-Download | Notes |
|
||||
|---|---|---|---|---|
|
||||
| Official 23-Lang (v1/v2/v3) | [ResembleAI/chatterbox](https://huggingface.co/ResembleAI/chatterbox) | ~4.3GB | ✅ | v1 + v2 + v3 T3 checkpoints with shared tokenizer, voice encoder, and S3Gen |
|
||||
| Russian stress dictionary (Russian only) | [Vuizur/add-stress-to-epub release](https://github.com/Vuizur/add-stress-to-epub/releases/download/v1.0.1/russian_dict.zip) | ~1.5GB | ✅ | Auxiliary Official 23-Lang Russian stress-labeling data; downloads on demand only when Russian stress support is used |
|
||||
| Vietnamese (Viterbox) | [dolly-vn/viterbox](https://huggingface.co/dolly-vn/viterbox) | ~4.3GB | ✅ | Vietnamese community finetune used by downloader |
|
||||
| Egyptian Arabic (oddadmix) | [oddadmix/chatterbox-egyptian-v0](https://huggingface.co/oddadmix/chatterbox-egyptian-v0) | ~4.3GB | ✅ | Egyptian Arabic community finetune (architecture v2) |
|
||||
|
||||
## VibeVoice
|
||||
|
||||
| Component | Source | Size | Auto-Download | Notes |
|
||||
|---|---|---|---|---|
|
||||
| vibevoice-1.5B | [microsoft/VibeVoice-1.5B](https://huggingface.co/microsoft/VibeVoice-1.5B) | ~5.4GB | ✅ | Microsoft official model |
|
||||
| vibevoice-7B | [aoi-ot/VibeVoice-Large](https://huggingface.co/aoi-ot/VibeVoice-Large) | ~18GB | ✅ | Community mirror used by downloader |
|
||||
| kugelaudio-0-open | [kugelaudio/kugelaudio-0-open](https://huggingface.co/kugelaudio/kugelaudio-0-open) | ~18GB | ✅ | KugelAudio multilingual 7B variant |
|
||||
| kugel-2 | [kugelaudio/kugel-2](https://huggingface.co/kugelaudio/kugel-2) | ~18.7GB | ✅ | KugelAudio v2 merged 7B variant |
|
||||
|
||||
## Higgs Audio 2
|
||||
|
||||
| Component | Source | Size | Auto-Download | Notes |
|
||||
|---|---|---|---|---|
|
||||
| higgs-audio-v2-3B | [bosonai/higgs-audio-v2-generation-3B-base](https://huggingface.co/bosonai/higgs-audio-v2-generation-3B-base) | ~9GB | ✅ | Generation model |
|
||||
| Audio tokenizer | [bosonai/higgs-audio-v2-tokenizer](https://huggingface.co/bosonai/higgs-audio-v2-tokenizer) | ~200MB | ✅ | Tokenizer model |
|
||||
|
||||
## Higgs Audio v3
|
||||
|
||||
| Component | Source | Size | Auto-Download | Notes |
|
||||
|---|---|---|---|---|
|
||||
| higgs-audio-v3-tts-4b | [bosonai/higgs-audio-v3-tts-4b](https://huggingface.co/bosonai/higgs-audio-v3-tts-4b) | ~8GB | ✅ | Official 4B multilingual controllable TTS model |
|
||||
|
||||
## IndexTTS 2 / 2.5
|
||||
|
||||
| Component | Source | Size | Auto-Download | Notes |
|
||||
|---|---|---|---|---|
|
||||
| IndexTTS-2 | [IndexTeam/IndexTTS-2](https://huggingface.co/IndexTeam/IndexTTS-2) | Multiple files | ✅ | Main TTS engine |
|
||||
| IndexTTS-2.5 | [IndexTeam/IndexTTS-2.5](https://huggingface.co/IndexTeam/IndexTTS-2.5) | ~5.49GB | ✅ | Multilingual backend with bundled codec and official feature-duration scaling |
|
||||
| w2v-bert-2.0 | [facebook/w2v-bert-2.0](https://huggingface.co/facebook/w2v-bert-2.0) | ~2GB | ✅ | Semantic feature extractor |
|
||||
| qwen0.6bemo4-merge | Included with IndexTTS-2 | Included | ✅ | Text emotion model bundle |
|
||||
|
||||
## CosyVoice3
|
||||
|
||||
| Component | Source | Size | Auto-Download | Notes |
|
||||
|---|---|---|---|---|
|
||||
| Fun-CosyVoice3-0.5B / 0.5B-RL | [FunAudioLLM/Fun-CosyVoice3-0.5B-2512](https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512) | ~5.4GB first variant (+~2GB second) | ✅ | Both variants share common files |
|
||||
|
||||
## Qwen3-TTS
|
||||
|
||||
| Component | Source | Size | Auto-Download | Notes |
|
||||
|---|---|---|---|---|
|
||||
| CustomVoice 0.6B | [Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice](https://huggingface.co/Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice) | ~1.5GB | ✅ | Preset voices + instructions |
|
||||
| CustomVoice 1.7B | [Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice](https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice) | ~4.2GB | ✅ | Preset voices + instructions |
|
||||
| VoiceDesign 1.7B | [Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign](https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign) | ~4.2GB | ✅ | Text-to-voice design model |
|
||||
| Base 0.6B | [Qwen/Qwen3-TTS-12Hz-0.6B-Base](https://huggingface.co/Qwen/Qwen3-TTS-12Hz-0.6B-Base) | ~1.5GB | ✅ | Zero-shot voice cloning |
|
||||
| Base 1.7B | [Qwen/Qwen3-TTS-12Hz-1.7B-Base](https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-Base) | ~4.2GB | ✅ | Zero-shot voice cloning |
|
||||
| Qwen3-ASR-1.7B | [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B) | N/A | ✅ | ASR transcribe model |
|
||||
| Qwen3-ForcedAligner-0.6B | [Qwen/Qwen3-ForcedAligner-0.6B](https://huggingface.co/Qwen/Qwen3-ForcedAligner-0.6B) | N/A | ✅ | Word-level timestamps |
|
||||
|
||||
## Granite ASR
|
||||
|
||||
| Component | Source | Size | Auto-Download | Notes |
|
||||
|---|---|---|---|---|
|
||||
| granite-4.0-1b-speech | [ibm-granite/granite-4.0-1b-speech](https://huggingface.co/ibm-granite/granite-4.0-1b-speech) | ~4.6GB | ✅ | Main Granite ASR / AST model |
|
||||
| granite-speech-4.1-2b | [ibm-granite/granite-speech-4.1-2b](https://huggingface.co/ibm-granite/granite-speech-4.1-2b) | ~4.6GB | ✅ | Granite Speech 4.1 2B multilingual model (supports Japanese) |
|
||||
| granite-speech-4.1-2b-plus | [ibm-granite/granite-speech-4.1-2b-plus](https://huggingface.co/ibm-granite/granite-speech-4.1-2b-plus) | ~4.6GB | ✅ | Plus variant supporting native speaker diarization and word-level timestamps |
|
||||
| Qwen3-ForcedAligner-0.6B | [Qwen/Qwen3-ForcedAligner-0.6B](https://huggingface.co/Qwen/Qwen3-ForcedAligner-0.6B) | N/A | ✅ | Optional custom word-level timestamps/SRT path; reused from Qwen folder |
|
||||
|
||||
## Step Audio EditX
|
||||
|
||||
| Component | Source | Size | Auto-Download | Notes |
|
||||
|---|---|---|---|---|
|
||||
| Step-Audio-EditX | [stepfun-ai/Step-Audio-EditX](https://huggingface.co/stepfun-ai/Step-Audio-EditX) | ~7GB | ✅ | Main 3B audio editing model |
|
||||
| Step-Audio-Tokenizer | [stepfun-ai/Step-Audio-Tokenizer](https://huggingface.co/stepfun-ai/Step-Audio-Tokenizer) | Included | ✅ | Tokenizer bundle used by Step EditX |
|
||||
|
||||
## Echo-TTS
|
||||
|
||||
| Component | Source | Size | Auto-Download | Notes |
|
||||
|---|---|---|---|---|
|
||||
| echo-tts-base (model + PCA state) | [jordand/echo-tts-base](https://huggingface.co/jordand/echo-tts-base) | ~5.3GB | ✅ | pytorch_model.safetensors + pca_state.safetensors |
|
||||
| fish-s1-dac-min (audio codec) | [jordand/fish-s1-dac-min](https://huggingface.co/jordand/fish-s1-dac-min) | ~1.8GB | ✅ | pytorch_model.safetensors — audio codec required by Echo-TTS |
|
||||
|
||||
## Fish Audio S2 Pro
|
||||
|
||||
| Component | Source | Size | Auto-Download | Notes |
|
||||
|---|---|---|---|---|
|
||||
| S2 Pro | [fishaudio/s2-pro](https://huggingface.co/fishaudio/s2-pro) | ~10.3GB | ✅ | Official 4B model and codec; BNB INT8/NF4 are optional load-time quantization modes that reuse these files; non-commercial license |
|
||||
| S2 Pro FP8 | [drbaph/s2-pro-fp8](https://huggingface.co/drbaph/s2-pro-fp8) | ~8.0GB | ✅ | Community weight-only FP8 checkpoint; BF16 activations; RTX 4090/5090-class CUDA GPU required |
|
||||
|
||||
## Dots TTS
|
||||
|
||||
| Component | Source | Size | Auto-Download | Notes |
|
||||
|---|---|---|---|---|
|
||||
| dots.tts-base | [rednote-hilab/dots.tts-base](https://huggingface.co/rednote-hilab/dots.tts-base) | ~6GB | ✅ | Official base checkpoint with tokenizer, vocoder, speaker encoder, and latent stats |
|
||||
| dots.tts-soar | [rednote-hilab/dots.tts-soar](https://huggingface.co/rednote-hilab/dots.tts-soar) | ~6GB | ✅ | Official SOAR checkpoint for higher-quality zero-shot cloning |
|
||||
| dots.tts-mf | [rednote-hilab/dots.tts-mf](https://huggingface.co/rednote-hilab/dots.tts-mf) | ~6GB | ✅ | Official MeanFlow-distilled checkpoint for faster inference |
|
||||
|
||||
## DramaBox
|
||||
|
||||
| Component | Source | Size | Auto-Download | Notes |
|
||||
|---|---|---|---|---|
|
||||
| DramaBox DiT + audio components | [ResembleAI/Dramabox](https://huggingface.co/ResembleAI/Dramabox) | ~8.5GB | ✅ | Official merged DramaBox transformer and LTX audio VAE/vocoder components |
|
||||
| Gemma 3 12B 4-bit text encoder | [unsloth/gemma-3-12b-it-bnb-4bit](https://huggingface.co/unsloth/gemma-3-12b-it-bnb-4bit) | ~7.8GB | ✅ | Official pre-quantized text encoder; loaded locally with no HF cache fallback |
|
||||
|
||||
## OmniVoice
|
||||
|
||||
| Component | Source | Size | Auto-Download | Notes |
|
||||
|---|---|---|---|---|
|
||||
| OmniVoice | [k2-fsa/OmniVoice](https://huggingface.co/k2-fsa/OmniVoice) | ~3.7GB | ✅ | Official main model including bundled audio_tokenizer assets |
|
||||
|
||||
## MOSS-TTS
|
||||
|
||||
| Component | Source | Size | Auto-Download | Notes |
|
||||
|---|---|---|---|---|
|
||||
| MOSS-TTS-Local-Transformer | [OpenMOSS-Team/MOSS-TTS-Local-Transformer](https://huggingface.co/OpenMOSS-Team/MOSS-TTS-Local-Transformer) | ~6.1GB | ✅ | Official 1.7B local-transformer model |
|
||||
| MOSS-TTS | [OpenMOSS-Team/MOSS-TTS](https://huggingface.co/OpenMOSS-Team/MOSS-TTS) | ~17GB | ✅ | Official 8B delay model |
|
||||
| MOSS-TTS-v1.5 | [OpenMOSS-Team/MOSS-TTS-v1.5](https://huggingface.co/OpenMOSS-Team/MOSS-TTS-v1.5) | ~17GB | ✅ | Current official 8B delay model with 31 languages and more stable voice cloning |
|
||||
| MOSS-TTS v1.5 Voice Acting 8B (Community - LAION) | [laion/moss-tts-v1.5-8b-voice-acting](https://huggingface.co/laion/moss-tts-v1.5-8b-voice-acting) | ~17GB | ✅ | Third-party full MOSS-TTS v1.5 fine-tune for expressive voice acting; not an official OpenMOSS model |
|
||||
| MOSS-VoiceGenerator | [OpenMOSS-Team/MOSS-VoiceGenerator](https://huggingface.co/OpenMOSS-Team/MOSS-VoiceGenerator) | ~4.2GB | ✅ | Official 1.7B reference-free voice-design model |
|
||||
| MOSS-TTSD-v1.0 | [OpenMOSS-Team/MOSS-TTSD-v1.0](https://huggingface.co/OpenMOSS-Team/MOSS-TTSD-v1.0) | ~18GB | ✅ | Official 8B native multi-speaker dialogue model |
|
||||
| MOSS-SoundEffect | [OpenMOSS-Team/MOSS-SoundEffect](https://huggingface.co/OpenMOSS-Team/MOSS-SoundEffect) | ~17GB | ✅ | Official MOSS v1 prompt-only sound-effect checkpoint; uses the shared MOSS audio tokenizer |
|
||||
| MOSS-Audio-Tokenizer | [OpenMOSS-Team/MOSS-Audio-Tokenizer](https://huggingface.co/OpenMOSS-Team/MOSS-Audio-Tokenizer) | ~8.5GB | ✅ | Shared official codec required by MOSS-TTS |
|
||||
|
||||
## MOSS-SoundEffect v2
|
||||
|
||||
| Component | Source | Size | Auto-Download | Notes |
|
||||
|---|---|---|---|---|
|
||||
| MOSS-SoundEffect-v2.0 | [OpenMOSS-Team/MOSS-SoundEffect-v2.0](https://huggingface.co/OpenMOSS-Team/MOSS-SoundEffect-v2.0) | ~11.2GB | ✅ | Official DiT + DAC VAE + Qwen3 text-encoder sound-effect pipeline |
|
||||
|
||||
## RVC
|
||||
|
||||
| Component | Source | Size | Auto-Download | Notes |
|
||||
|---|---|---|---|---|
|
||||
| RVC character pack | [SayanoAI/RVC-Studio (RVC folder)](https://huggingface.co/datasets/SayanoAI/RVC-Studio/tree/main/RVC) | Varies | ✅ | Default auto-download characters: Claire, Sayano, Mae_v2, Fuji, Monika (extras also available) |
|
||||
| RVC index pack (.index) | [SayanoAI/RVC-Studio (.index folder)](https://huggingface.co/datasets/SayanoAI/RVC-Studio/tree/main/RVC/.index) | Varies | ✅ | Optional FAISS indexes for improved voice similarity |
|
||||
| content-vec-best.safetensors | [lengyue233/content-vec-best](https://huggingface.co/lengyue233/content-vec-best) | ~300MB | ✅ | Voice feature model |
|
||||
| rmvpe.pt | [lj1995/VoiceConversionWebUI](https://huggingface.co/lj1995/VoiceConversionWebUI) | ~55MB | ✅ | Pitch extraction model |
|
||||
| pretrained_v2 (f0 G/D pairs) | [lj1995/VoiceConversionWebUI](https://huggingface.co/lj1995/VoiceConversionWebUI/tree/main/pretrained_v2) | ~300MB total | ✅ | Training init checkpoints for 32k/40k/48k RVC runs; downloaded on first training use |
|
||||
|
||||
*Generated from [tts_audio_suite_engines.yaml](Dev%20reports/tts_audio_suite_engines.yaml).*
|
||||
@@ -1,466 +0,0 @@
|
||||
# Model Folder Layouts
|
||||
|
||||
Use this document for folder structures and placement notes.
|
||||
For repository/source URLs, use [MODEL_DOWNLOAD_SOURCES.md](MODEL_DOWNLOAD_SOURCES.md).
|
||||
|
||||
## ChatterBox
|
||||
|
||||
Recommended:
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/chatterbox/
|
||||
```
|
||||
|
||||
Legacy (still supported):
|
||||
|
||||
```text
|
||||
ComfyUI/models/chatterbox/
|
||||
```
|
||||
|
||||
Required base files:
|
||||
|
||||
- `conds.pt`
|
||||
- `s3gen.pt`
|
||||
- `t3_cfg.pt`
|
||||
- `tokenizer.json`
|
||||
- `ve.pt`
|
||||
|
||||
Optional multilingual subfolders:
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/chatterbox/
|
||||
├── English/
|
||||
├── German/
|
||||
└── Norwegian/
|
||||
```
|
||||
|
||||
## ChatterBox Official 23-Lang
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/chatterbox_official_23lang/
|
||||
├── ChatterBox Official 23-Lang/
|
||||
│ ├── t3_23lang.safetensors
|
||||
│ ├── t3_mtl23ls_v2.safetensors
|
||||
│ ├── grapheme_mtl_merged_expanded_v1.json
|
||||
│ ├── s3gen.pt
|
||||
│ ├── ve.pt
|
||||
│ ├── mtl_tokenizer.json
|
||||
│ └── conds.pt
|
||||
└── russian_text_stresser/
|
||||
├── russian_dict.db
|
||||
└── simple_cases.pkl
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- v1 and v2 coexist in one directory.
|
||||
- Vietnamese (Viterbox) is available as a community finetune option.
|
||||
- `russian_text_stresser/` is auxiliary Russian-only data for Official 23-Lang stress labeling and downloads on demand.
|
||||
|
||||
## F5-TTS
|
||||
|
||||
Recommended:
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/F5-TTS/
|
||||
```
|
||||
|
||||
Legacy (still supported):
|
||||
|
||||
```text
|
||||
ComfyUI/models/F5-TTS/
|
||||
```
|
||||
|
||||
Typical structure:
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/F5-TTS/
|
||||
├── F5TTS_Base/
|
||||
│ ├── model_1200000.safetensors
|
||||
│ └── vocab.txt
|
||||
├── F5TTS_v1_Base/
|
||||
│ ├── model_1250000.safetensors
|
||||
│ └── vocab.txt
|
||||
└── vocos/
|
||||
├── config.yaml
|
||||
└── pytorch_model.bin
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- F5-TTS models auto-download on first use.
|
||||
- `vocab.txt` is required per model.
|
||||
- Vocos is optional and also auto-downloadable.
|
||||
|
||||
## F5-TTS Voice References
|
||||
|
||||
```text
|
||||
ComfyUI/models/voices/
|
||||
├── character1.wav
|
||||
├── character1.reference.txt
|
||||
├── narrator.wav
|
||||
└── narrator.txt
|
||||
```
|
||||
|
||||
Requirements:
|
||||
|
||||
- WAV, clean speech, 5-30s (24kHz recommended).
|
||||
- Text must match spoken audio.
|
||||
- Naming: `name.wav` + `name.reference.txt` (preferred) or `name.txt`.
|
||||
|
||||
Inference guidelines:
|
||||
|
||||
1. Keep reference audio under ~12s with a little trailing silence.
|
||||
2. Use uppercase letter-by-letter only when desired.
|
||||
3. Use spaces/punctuation for explicit pauses.
|
||||
4. Keep a space after sentence-ending punctuation.
|
||||
5. For Chinese output, preprocess numbers if needed.
|
||||
|
||||
## Higgs Audio 2
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/HiggsAudio/
|
||||
└── higgs-audio-v2-3B/
|
||||
├── generation/
|
||||
├── tokenizer/
|
||||
└── voices/
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- Both generation model and tokenizer auto-download.
|
||||
- Place reference audio/transcription in `voices/`.
|
||||
|
||||
## VibeVoice
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/VibeVoice/
|
||||
├── vibevoice-1.5B/
|
||||
└── vibevoice-7B/
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- Both variants auto-download on first use.
|
||||
- `vibevoice-7B` uses a community mirror in downloader config.
|
||||
|
||||
## RVC
|
||||
|
||||
Recommended:
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/RVC/
|
||||
├── *.pth
|
||||
├── content-vec-best.safetensors
|
||||
├── rmvpe.pt
|
||||
├── hubert/
|
||||
├── pretrained_v2/
|
||||
│ ├── f0G32k.pth
|
||||
│ ├── f0D32k.pth
|
||||
│ ├── f0G40k.pth
|
||||
│ ├── f0D40k.pth
|
||||
│ ├── f0G48k.pth
|
||||
│ └── f0D48k.pth
|
||||
└── .index/
|
||||
```
|
||||
|
||||
Legacy (still supported):
|
||||
|
||||
```text
|
||||
ComfyUI/models/RVC/
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- Base models auto-download.
|
||||
- Character `.pth` models can be auto-downloaded defaults or user-provided.
|
||||
- `pretrained_v2/` is used by the integrated RVC trainer and auto-downloads on first training run.
|
||||
- Training datasets, logs, progress snapshots, and resumable checkpoints are stored under `ComfyUI/output/tts_audio_suite_training/rvc/`, not inside the custom node repo.
|
||||
- UVR models are downloaded under `ComfyUI/models/TTS/UVR/` (or legacy `ComfyUI/models/UVR/`).
|
||||
|
||||
## IndexTTS-2
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/IndexTTS/
|
||||
├── IndexTTS-2/
|
||||
└── w2v-bert-2.0/
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- Emotion components and semantic feature models auto-download.
|
||||
|
||||
## Step Audio EditX
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/step_audio_editx/
|
||||
├── Step-Audio-EditX/
|
||||
│ └── CosyVoice-300M-25Hz/
|
||||
└── FunASR-Paraformer/
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- Main model, tokenizer assets, and speech stack auto-download.
|
||||
|
||||
## Higgs Audio v3
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/higgs_audio_v3/
|
||||
└── higgs-audio-v3-tts-4b/
|
||||
├── config.json
|
||||
├── tokenizer.json
|
||||
├── tokenizer_config.json
|
||||
├── chat_template.jinja
|
||||
├── model.safetensors
|
||||
├── model.safetensors.index.json
|
||||
└── LICENSE
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- Downloads into its own `higgs_audio_v3` folder.
|
||||
- Requires the main Transformers 5 environment.
|
||||
- Reference transcript `.txt` files are optional but improve cloning quality.
|
||||
|
||||
## DramaBox
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/dramabox/
|
||||
├── DramaBox/
|
||||
├── dramabox-dit-v1.safetensors
|
||||
├── dramabox-audio-components.safetensors
|
||||
├── assets/
|
||||
│ └── silence_latent_frame.pt
|
||||
└── gemma-3-12b-it-bnb-4bit/
|
||||
├── config.json
|
||||
├── model-00001-of-00002.safetensors
|
||||
├── model-00002-of-00002.safetensors
|
||||
├── model.safetensors.index.json
|
||||
└── tokenizer and processor files...
|
||||
└── loras/
|
||||
└── <adapter_name>/
|
||||
├── adapter_config.json
|
||||
└── adapter_model.safetensors
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- Both repositories download directly into the organized suite folder.
|
||||
- Transformers is forced into local-only loading after download.
|
||||
- Requires NVIDIA CUDA. Fast mode targets approximately 24GB VRAM; experimental staged/sequential modes can run with less.
|
||||
- Integrated training exports LoRA adapters into `dramabox/loras/<adapter_name>/`.
|
||||
- Training jobs, normalized datasets, preprocessing caches, checkpoints, and logs are stored under `ComfyUI/output/tts_audio_suite_training/dramabox/`.
|
||||
- The LTX-2 Community License requires a paid license for entities with at
|
||||
least USD 10 million in annual revenue.
|
||||
|
||||
## CosyVoice3
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/CosyVoice/
|
||||
└── Fun-CosyVoice3-0.5B/
|
||||
├── llm.pt
|
||||
├── llm.rl.pt
|
||||
└── shared model files...
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- First selected variant downloads first.
|
||||
- Shared files are reused across variants.
|
||||
|
||||
## Qwen3-TTS and Qwen3-ASR
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/qwen3_tts/
|
||||
├── Qwen3-TTS-12Hz-0.6B-CustomVoice/
|
||||
├── Qwen3-TTS-12Hz-1.7B-CustomVoice/
|
||||
├── Qwen3-TTS-12Hz-1.7B-VoiceDesign/
|
||||
├── Qwen3-TTS-12Hz-0.6B-Base/
|
||||
├── Qwen3-TTS-12Hz-1.7B-Base/
|
||||
├── qwen2-audio-encoder/
|
||||
└── asr/
|
||||
├── Qwen3-ASR-1.7B/
|
||||
└── Qwen3-ForcedAligner-0.6B/
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- Only selected variants download.
|
||||
- Shared tokenizer assets are reused.
|
||||
|
||||
## MOSS-TTS
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/moss_tts/
|
||||
├── MOSS-TTS-Local-Transformer/
|
||||
├── MOSS-TTS-v1.5/
|
||||
├── moss-tts-v1.5-8b-voice-acting/ # Community - LAION
|
||||
├── MOSS-TTS/
|
||||
├── MOSS-VoiceGenerator/
|
||||
├── MOSS-SoundEffect/
|
||||
├── MOSS-TTSD-v1.0/
|
||||
├── MOSS-Audio-Tokenizer/
|
||||
└── loras/
|
||||
└── <adapter_name>/
|
||||
├── adapter_config.json
|
||||
└── adapter_model.safetensors
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- `MOSS-Audio-Tokenizer` is required by the official TTS and TTSD variants.
|
||||
- `MOSS-TTS-Local-Transformer` is the smaller 1.7B model.
|
||||
- `MOSS-TTS-v1.5` is the current 8B delay model with 31-language support.
|
||||
- `moss-tts-v1.5-8b-voice-acting` is an optional third-party LAION full fine-tune for expressive speech, not an official OpenMOSS model.
|
||||
- Other compatible full checkpoints placed here are discovered from their `config.json`; unsupported MOSS architectures are rejected explicitly.
|
||||
- `MOSS-TTS` is the legacy official 8B delay model.
|
||||
- `MOSS-VoiceGenerator` is the 1.7B voice-design provider used by Voice Designer.
|
||||
- `MOSS-SoundEffect` is the v1 sound-effect checkpoint used through the MOSS-TTS engine and 🌩️ Sound Effects.
|
||||
- `MOSS-TTSD-v1.0` is the official 8B native multi-speaker dialogue model.
|
||||
- Integrated training currently exports LoRA adapters into `moss_tts/loras/<adapter_name>/`.
|
||||
- Training jobs, temporary manifests, and checkpoints are stored under `ComfyUI/output/tts_audio_suite_training/moss_tts/`.
|
||||
|
||||
## MOSS-SoundEffect v2
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/moss_soundeffect_v2/
|
||||
└── MOSS-SoundEffect-v2.0/
|
||||
├── model_index.json
|
||||
├── scheduler/
|
||||
├── text_encoder/
|
||||
├── tokenizer/
|
||||
├── transformer/
|
||||
└── vae/
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- This is a separate v2 diffusion family, not a MOSS-TTS checkpoint variant.
|
||||
- It runs in the configured ComfyUI environment; the official Apache-2.0 inference package is bundled without modifying its dependencies.
|
||||
- The 🌩️ Sound Effects node limits generation to the official 30-second maximum.
|
||||
|
||||
## Granite ASR
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/granite_asr/
|
||||
└── granite-4.0-1b-speech/
|
||||
├── config.json
|
||||
├── chat_template.jinja
|
||||
├── model-00001-of-00003.safetensors
|
||||
├── model-00002-of-00003.safetensors
|
||||
├── model-00003-of-00003.safetensors
|
||||
└── tokenizer / processor files...
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- Granite downloads into its own `granite_asr` folder.
|
||||
- `granite-speech-4.1-2b-plus` adds native speaker diarization and native word-level timestamps, but drops Japanese.
|
||||
- If Granite word timestamps are enabled, it lazily reuses `Qwen3-ForcedAligner-0.6B` from the Qwen ASR folder instead of duplicating that model.
|
||||
|
||||
## Echo-TTS
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/
|
||||
├── echo-tts-base/
|
||||
│ ├── pytorch_model.safetensors
|
||||
│ └── pca_state.safetensors
|
||||
└── fish-s1-dac-min/
|
||||
└── pytorch_model.safetensors
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- Both components are required and auto-downloaded on first use.
|
||||
- License: CC-BY-NC-SA (non-commercial).
|
||||
|
||||
## Fish Audio S2 Pro
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/fish_audio_s2_pro/
|
||||
├── codec.pth
|
||||
├── config.json
|
||||
├── model-00001-of-00002.safetensors
|
||||
├── model-00002-of-00002.safetensors
|
||||
├── model.safetensors.index.json
|
||||
└── tokenizer.json
|
||||
```
|
||||
|
||||
Optional FP8 variant:
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/fish_audio_s2_pro_fp8/
|
||||
├── codec.pth
|
||||
├── config.json
|
||||
├── model.safetensors
|
||||
├── quantization_info.json
|
||||
└── tokenizer.json
|
||||
```
|
||||
|
||||
The complete official repository metadata and tokenizer files are downloaded alongside these files. License: Fish Audio Research License (non-commercial without a separate commercial license).
|
||||
|
||||
The `s2-pro-bnb-int8` and `s2-pro-bnb-nf4` options reuse `fish_audio_s2_pro/` and quantize its official checkpoint while loading. They do not download another model copy and require `bitsandbytes`.
|
||||
|
||||
## Dots TTS
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/dots_tts/
|
||||
├── dots.tts-base/
|
||||
├── dots.tts-soar/
|
||||
└── dots.tts-mf/
|
||||
```
|
||||
|
||||
Typical checkpoint contents:
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/dots_tts/dots.tts-soar/
|
||||
├── added_tokens.json
|
||||
├── chat_template.jinja
|
||||
├── config.json
|
||||
├── latent_stats.pt
|
||||
├── llm_config.json
|
||||
├── merges.txt
|
||||
├── model.safetensors
|
||||
├── speaker_encoder.safetensors
|
||||
├── special_tokens_map.json
|
||||
├── tokenizer.json
|
||||
├── tokenizer_config.json
|
||||
├── vocab.json
|
||||
└── vocoder.safetensors
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- Downloads into a dedicated `dots_tts/` folder.
|
||||
- Native sample rate is 48kHz.
|
||||
- Main-environment support works on Transformers 5; on Windows, `normalize_text` falls back to no-op if `WeTextProcessing` is unavailable.
|
||||
|
||||
## OmniVoice
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/omnivoice/
|
||||
└── OmniVoice/
|
||||
```
|
||||
|
||||
Main model contents:
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/omnivoice/OmniVoice/
|
||||
├── chat_template.jinja
|
||||
├── config.json
|
||||
├── model.safetensors
|
||||
├── tokenizer.json
|
||||
├── tokenizer_config.json
|
||||
└── audio_tokenizer/
|
||||
├── config.json
|
||||
├── model.safetensors
|
||||
└── preprocessor_config.json
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- Downloads into a dedicated `omnivoice/` folder.
|
||||
- Native sample rate is 24kHz.
|
||||
- The main OmniVoice model stays in the main Transformers 5 environment.
|
||||
- Voice cloning requires reference audio plus explicit reference text in this suite.
|
||||
@@ -1,119 +0,0 @@
|
||||
# MOSS LoRA Guide
|
||||
|
||||
This guide defines the local adapter layout and loading behavior for **MOSS-TTS LoRA adapters** in TTS Audio Suite.
|
||||
|
||||
## Local Folder Layout
|
||||
|
||||
MOSS LoRA adapters should live under:
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/moss_tts/loras/
|
||||
```
|
||||
|
||||
Each adapter must have its **own folder**:
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/moss_tts/loras/
|
||||
MyNorwegianLoRA/
|
||||
adapter_config.json
|
||||
adapter_model.safetensors
|
||||
```
|
||||
|
||||
Recommended structure:
|
||||
|
||||
```text
|
||||
<adapter_name>/
|
||||
adapter_config.json
|
||||
adapter_model.safetensors
|
||||
```
|
||||
|
||||
Also accepted:
|
||||
|
||||
```text
|
||||
<adapter_name>/
|
||||
adapter_config.json
|
||||
adapter_model.bin
|
||||
```
|
||||
|
||||
## Required Files
|
||||
|
||||
Each adapter folder must contain:
|
||||
|
||||
- `adapter_config.json`
|
||||
- one adapter weights file:
|
||||
- `adapter_model.safetensors` preferred
|
||||
- or `adapter_model.bin`
|
||||
|
||||
Extra files are allowed, for example:
|
||||
|
||||
- `README.md`
|
||||
- training metadata
|
||||
- tokenizer notes
|
||||
|
||||
## What Is Not Supported
|
||||
|
||||
Do not place loose adapter files directly inside:
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/moss_tts/loras/
|
||||
```
|
||||
|
||||
This is **not** a supported layout:
|
||||
|
||||
```text
|
||||
loras/
|
||||
adapter_config.json
|
||||
some_weights.safetensors
|
||||
```
|
||||
|
||||
This is also **not** supported:
|
||||
|
||||
```text
|
||||
loras/
|
||||
MyLoRA.safetensors
|
||||
```
|
||||
|
||||
One adapter must always be one folder.
|
||||
|
||||
## Engine Node Behavior
|
||||
|
||||
The MOSS engine node supports two adapter paths:
|
||||
|
||||
1. **Local LoRA dropdown**
|
||||
- lists valid adapter folders discovered under `models/TTS/moss_tts/loras`
|
||||
|
||||
2. **Advanced override field**
|
||||
- accepts:
|
||||
- a local adapter folder path
|
||||
- a Hugging Face repo id
|
||||
|
||||
If you enter a Hugging Face repo id, TTS Audio Suite installs it into:
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/moss_tts/loras/<owner__repo>/
|
||||
```
|
||||
|
||||
Then it loads the adapter from that local folder.
|
||||
|
||||
## Why This Layout Exists
|
||||
|
||||
TTS Audio Suite does **not** use Hugging Face cache as the final managed storage location for MOSS LoRAs.
|
||||
|
||||
This layout keeps adapters:
|
||||
|
||||
- visible in the MOSS dropdown
|
||||
- stored inside the normal TTS model tree
|
||||
- ready for future training output
|
||||
- separate from ComfyUI diffusion LoRA folders
|
||||
|
||||
## Training Output Standard
|
||||
|
||||
Future MOSS training should write adapters in exactly this format:
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/moss_tts/loras/<run_name>/
|
||||
adapter_config.json
|
||||
adapter_model.safetensors
|
||||
```
|
||||
|
||||
That keeps training, manual installs, and inference all using the same contract.
|
||||
@@ -1,258 +0,0 @@
|
||||
# MOSS Training Guide
|
||||
|
||||
This guide explains the **practical dataset meaning** of the current MOSS training nodes in TTS Audio Suite.
|
||||
|
||||
Use this if `🧾 MOSS Dataset Rows` feels unclear.
|
||||
|
||||
## Current Training Scope
|
||||
|
||||
Current first training slice supports:
|
||||
|
||||
- **MOSS-TTS 8B v1.0 and v1.5 (Delay)**
|
||||
- **LAION MOSS-TTS v1.5 Voice Acting 8B community full checkpoint (Delay, compatibility path; training results not yet validated by the suite maintainers)**
|
||||
- **LoRA adapter training**
|
||||
|
||||
The model selected on the connected MOSS engine is used for dataset preparation and training. Prepare the dataset again after switching between v1.0 and v1.5.
|
||||
|
||||
The LAION Voice Acting checkpoint uses the same Delay architecture and can use this LoRA training path, but the suite maintainers have not completed an inference or training run with its full weights. Treat it as community-tested support and report results or incompatibilities.
|
||||
|
||||
It does **not** currently support:
|
||||
|
||||
- Local 1.7B training
|
||||
- TTSD training
|
||||
- full-parameter training from the node path
|
||||
|
||||
## Normal Training Flow
|
||||
|
||||
Current ComfyUI flow:
|
||||
|
||||
1. `🎞️ Training Clip Staging`
|
||||
2. `🧾 MOSS Dataset Rows`
|
||||
3. `📦 MOSS Dataset Prep`
|
||||
4. `🎛️ MOSS Training Config`
|
||||
5. `🎓 Model Training`
|
||||
|
||||
If clips and transcripts are already prepared on disk, you can skip the first two
|
||||
nodes. Set `dataset_source` on `📦 MOSS Dataset Prep` to a folder containing
|
||||
same-name audio and text pairs:
|
||||
|
||||
```text
|
||||
my_dataset/
|
||||
├── clip001.wav
|
||||
├── clip001.txt
|
||||
├── clip002.flac
|
||||
└── clip002.txt
|
||||
```
|
||||
|
||||
Each `.txt` file must contain the transcript spoken in its matching audio file.
|
||||
Folder scanning supports WAV, FLAC, MP3, OGG, and M4A. Subfolders are ignored
|
||||
unless `recursive_folder_scan` is enabled. Existing JSONL manifest paths continue
|
||||
to work unchanged.
|
||||
|
||||
## The Important Fields
|
||||
|
||||
### `text_lines`
|
||||
|
||||
This is the easiest one:
|
||||
|
||||
- one line per staged clip
|
||||
- each line is the **transcript/text of that clip**
|
||||
|
||||
Example:
|
||||
|
||||
- `clip001` audio says: `Hello there.`
|
||||
- `clip002` audio says: `How are you?`
|
||||
|
||||
Then:
|
||||
|
||||
```text
|
||||
Hello there.
|
||||
How are you?
|
||||
```
|
||||
|
||||
That is the normal default dataset shape:
|
||||
|
||||
- `audio + transcript`
|
||||
|
||||
For many voice LoRAs, this is enough.
|
||||
|
||||
### `reference_clip_lines`
|
||||
|
||||
This is **advanced**. Most users should leave it blank at first.
|
||||
|
||||
It means:
|
||||
|
||||
- which other staged clip should be used as the **reference audio** for that row
|
||||
|
||||
That teaches the model a different pattern:
|
||||
|
||||
- `text` says **what to say**
|
||||
- `reference audio` says **whose voice/style to follow**
|
||||
- `target audio` is the **correct output**
|
||||
|
||||
Example:
|
||||
|
||||
- `clip001` = Bob saying: `My name is Bob.`
|
||||
- `clip002` = Bob saying: `Welcome to the show.`
|
||||
|
||||
Row 2 can be:
|
||||
|
||||
- target audio = `clip002`
|
||||
- text = `Welcome to the show.`
|
||||
- reference = `clip001`
|
||||
|
||||
That teaches:
|
||||
|
||||
- “use `clip001` as the reference voice, then produce the target line from `clip002`”
|
||||
|
||||
## When To Ignore `reference_clip_lines`
|
||||
|
||||
Ignore it for:
|
||||
|
||||
- first MOSS LoRA tests
|
||||
- single-speaker voice LoRAs
|
||||
- fixed-voice adaptation
|
||||
- plain language/domain adaptation
|
||||
|
||||
That means:
|
||||
|
||||
- just use `audio + transcript`
|
||||
- maybe also use `instruction`
|
||||
|
||||
This is the best default.
|
||||
|
||||
## When To Use `reference_clip_lines`
|
||||
|
||||
Use it only if you deliberately want **reference-conditioned training**.
|
||||
|
||||
That means teaching the model:
|
||||
|
||||
- how to use a separate reference clip for voice cloning
|
||||
- how to follow reference style/prosody
|
||||
- how to pick speaker identity from reference audio
|
||||
|
||||
This makes more sense for:
|
||||
|
||||
- reference-conditioned voice cloning behavior
|
||||
- multi-speaker conditioning datasets
|
||||
- style-following datasets
|
||||
|
||||
## How A Model Learns Voice Cloning
|
||||
|
||||
Think of it this way:
|
||||
|
||||
- if a TTS model only sees `audio + transcript`, it learns how to do TTS for the voices/styles present in the data
|
||||
- if a TTS model also sees `ref_audio`, it can learn a different pattern:
|
||||
- text says **what to say**
|
||||
- reference audio says **whose voice/style to imitate**
|
||||
- target audio is the correct output
|
||||
|
||||
That is how a model can learn **reference-conditioned voice cloning**, not just plain TTS.
|
||||
|
||||
So if your goal is to fine-tune that specific ability:
|
||||
|
||||
- “give the model a reference clip and make it speak in that voice/style”
|
||||
|
||||
then `ref_audio` is when this field makes sense.
|
||||
|
||||
If your goal is only:
|
||||
|
||||
- “make the model speak in this one fixed voice better”
|
||||
|
||||
then `ref_audio` usually does not matter much, and plain `audio + transcript` is the better starting point.
|
||||
|
||||
## `instruction` vs `ref_audio`
|
||||
|
||||
These are different tools.
|
||||
|
||||
### Use `instruction` for explicit text labels
|
||||
|
||||
Examples:
|
||||
|
||||
- `speaking angrily`
|
||||
- `whispering`
|
||||
- `calm narration`
|
||||
|
||||
This is usually clearer when you want to label the desired behavior directly.
|
||||
|
||||
### Use `quality` for recording/presentation quality
|
||||
|
||||
Examples:
|
||||
|
||||
- `telephone call quality`
|
||||
- `studio recording`
|
||||
- `noisy room recording`
|
||||
|
||||
This is better when the label is about how the audio sounds as a recording or presentation format, not about acting/performance.
|
||||
|
||||
### Use `ref_audio` for example audio conditioning
|
||||
|
||||
This is for:
|
||||
|
||||
- “sound like this other clip”
|
||||
- “follow this speaker/style from audio”
|
||||
|
||||
So for many datasets:
|
||||
|
||||
- `instruction` is the simpler control
|
||||
- `ref_audio` is the more advanced control
|
||||
|
||||
## Practical Recommendations
|
||||
|
||||
### Best first test
|
||||
|
||||
Use:
|
||||
|
||||
- staged clips
|
||||
- correct transcripts
|
||||
- no reference clips
|
||||
- optional `instruction` only if clearly useful
|
||||
|
||||
In other words:
|
||||
|
||||
- `text_lines`: yes
|
||||
- `reference_clip_lines`: leave blank
|
||||
|
||||
### If training a single person's voice
|
||||
|
||||
Start with:
|
||||
|
||||
- `audio + transcript`
|
||||
|
||||
Do **not** add `ref_audio` unless you specifically want to teach reference-conditioned cloning behavior.
|
||||
|
||||
### If training language adaptation like the Norwegian LoRA
|
||||
|
||||
Start with:
|
||||
|
||||
- `audio + transcript`
|
||||
|
||||
That is the normal base case.
|
||||
|
||||
You can still use reference audio later at inference time, because MOSS supports optional reference-based cloning, but the training dataset does not need to be built around `ref_audio`.
|
||||
|
||||
## Validation In Plain Terms
|
||||
|
||||
`📦 MOSS Dataset Prep` also mentions validation.
|
||||
|
||||
That means:
|
||||
|
||||
- a small holdout part of the dataset used only to check training progress
|
||||
- not used to update weights
|
||||
|
||||
If you do not have a separate validation manifest:
|
||||
|
||||
- leave `validation_source` blank
|
||||
- let the node auto-split the main dataset
|
||||
|
||||
## Short Version
|
||||
|
||||
If you want the least confusing starting point:
|
||||
|
||||
- use `🎞️ Training Clip Staging`
|
||||
- use `🧾 MOSS Dataset Rows`
|
||||
- fill only `text_lines`
|
||||
- leave `reference_clip_lines` blank
|
||||
- train the LoRA
|
||||
|
||||
That is the correct first path for most users.
|
||||
@@ -1,152 +0,0 @@
|
||||
# MOSS-TTS Prompt Fields Guide
|
||||
|
||||
## Overview
|
||||
|
||||
MOSS-TTS exposes official whole-segment prompt fields through the `⚙️ MOSS-TTS Engine` node.
|
||||
|
||||
These are not Step Audio EditX effects and they are not positional insertions. They are passed to the official MOSS prompt structure for the entire generated segment.
|
||||
|
||||
Supported official fields:
|
||||
|
||||
- `instruction`
|
||||
- `quality`
|
||||
- `sound_event`
|
||||
- `ambient_sound`
|
||||
- `language`
|
||||
- `duration_tokens`
|
||||
|
||||
## Stability Note
|
||||
|
||||
`ambient_sound` should currently be treated as experimental on base `MOSS-TTS` inside this suite.
|
||||
|
||||
Reason:
|
||||
|
||||
- the official prompt schema exposes `ambient_sound`
|
||||
- but the clearest public OpenMOSS example we found is in the `MOSS-SoundEffect` documentation path, not as a validated short-utterance base-TTS control
|
||||
|
||||
So on `MOSS-TTS` you may get:
|
||||
|
||||
- weak or missing ambience
|
||||
- longer-than-expected generation
|
||||
- unstable tails if the model is pushed to the token cap
|
||||
|
||||
## Important Limitation
|
||||
|
||||
MOSS does **not** treat these like exact inline insertions inside the sentence.
|
||||
|
||||
For example:
|
||||
|
||||
```text
|
||||
[Alice|sound_event:Laughter] Hello there.
|
||||
```
|
||||
|
||||
That means the whole segment is conditioned with the event. It is **not** positional the way Step Audio EditX paralinguistic tags are.
|
||||
|
||||
## Engine Fields
|
||||
|
||||
You can set official fields directly on the engine node:
|
||||
|
||||
- `instruction`: whole-segment speaking instruction
|
||||
- `quality`: whole-segment quality/style hint
|
||||
- `sound_event`: whole-segment event hint such as `Laughter`, `Sigh`, `Breathing`
|
||||
- `ambient_sound`: whole-segment ambience hint such as `Rain`, `Crowd`, `Forest`
|
||||
|
||||
These apply to every generated segment that uses that engine config.
|
||||
|
||||
## Per-Segment Override Syntax
|
||||
|
||||
For per-segment overrides, use `[]` parameter switching.
|
||||
|
||||
Supported forms:
|
||||
|
||||
```text
|
||||
[Narrator|instruction:Speak softly and calmly] Hello there.
|
||||
[Narrator|quality:Studio recording] Hello there.
|
||||
[Narrator|sound_event:Laughter] Hello there.
|
||||
[Narrator|ambient_sound:Rain on window] Hello there.
|
||||
```
|
||||
|
||||
Inside normal character tags, combine them like this:
|
||||
|
||||
```text
|
||||
[Alice|instruction:Speak softly and calmly] Hello there.
|
||||
[Bob|quality:Telephone call quality|ambient_sound:Office room tone] Hi.
|
||||
[Alice|sound_event:Laughter] That's funny.
|
||||
```
|
||||
|
||||
These are whole-segment overrides, not exact insertion points.
|
||||
|
||||
## Why Not `<>`
|
||||
|
||||
MOSS does not have true native inline positioning for these controls.
|
||||
|
||||
So `<>` should stay free for real inline post-processing tags such as Step Audio EditX effects. Using `<>` for MOSS prompt fields would blur two different systems and create confusion.
|
||||
|
||||
## Examples
|
||||
|
||||
### Direct engine fields
|
||||
|
||||
Set on the engine node:
|
||||
|
||||
- `sound_event = Laughter`
|
||||
- `quality = Studio recording`
|
||||
|
||||
Text:
|
||||
|
||||
```text
|
||||
[Alice] Hello there!
|
||||
```
|
||||
|
||||
Result: the entire Alice segment is conditioned with `Laughter` and `Studio recording`.
|
||||
|
||||
### Per-segment `[]` syntax
|
||||
|
||||
```text
|
||||
[Alice|sound_event:Laughter] Hello there!
|
||||
[Bob|ambient_sound:Rain] Nice to meet you.
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
- Alice segment uses `sound_event = Laughter`
|
||||
- Bob segment uses `ambient_sound = Rain`
|
||||
|
||||
## Conflicts and Precedence
|
||||
|
||||
- If you set an engine field and also override it in `[]`, the `[]` value wins for that segment.
|
||||
- These MOSS prompt fields are not supposed to use `<>`.
|
||||
|
||||
## Native TTSD Dialogue Note
|
||||
|
||||
`MOSS-TTSD-v1.0` generates one native dialogue request across the formatted `[S1]...[S5]` conversation.
|
||||
|
||||
So these official fields still apply to the whole TTSD request, not to one precise speaker turn inside that request.
|
||||
|
||||
## Native TTSD Compatibility Rules
|
||||
|
||||
Native TTSD mode is strict in this suite.
|
||||
|
||||
If any of these are detected, generation **fails with an explicit error** instead of auto-switching models:
|
||||
|
||||
- `pause tags`
|
||||
- inline Step Audio EditX tags
|
||||
- per-segment `[]` parameter changes
|
||||
- more than 5 speakers
|
||||
|
||||
This applies to both normal TTS Text and SRT workflows.
|
||||
|
||||
Required action when that happens:
|
||||
|
||||
- switch to `Custom Character Switching`
|
||||
- choose a standard MOSS model (`MOSS-TTS-Local-Transformer` or `MOSS-TTS`)
|
||||
|
||||
## When To Use Something Else
|
||||
|
||||
If you need exact placement of laughter/breathing/cough inside the sentence, use:
|
||||
|
||||
- Step Audio EditX inline tags and audio editor workflow
|
||||
- an engine with true native positional tag support, such as CosyVoice3
|
||||
|
||||
Use MOSS prompt fields when you want official MOSS conditioning, not precise insertion control.
|
||||
|
||||
If you specifically want generated environmental audio like rain, traffic, or scene beds, `MOSS-SoundEffect` is the more correct OpenMOSS model family target than base `MOSS-TTS`.
|
||||
@@ -1,195 +0,0 @@
|
||||
# 🏷️ Multiline TTS Tag Editor Guide
|
||||
|
||||
This guide covers the practical user-facing workflow for the `🏷️ Multiline TTS Tag Editor` node.
|
||||
|
||||
For related topics, also see:
|
||||
|
||||
- [Character Switching Guide](CHARACTER_SWITCHING_GUIDE.md)
|
||||
- [Parameter Switching Guide](PARAMETER_SWITCHING_GUIDE.md)
|
||||
- [Step Audio EditX Inline Tags User Guide](INLINE_EDIT_TAGS_USER_GUIDE.md)
|
||||
- [Higgs Audio v3 Inline Tags](HIGGS_AUDIO_V3_INLINE_TAGS.md)
|
||||
- [CosyVoice3 Tags Guide](COSYVOICE3_TAGS_GUIDE.md)
|
||||
- [OmniVoice Native Tags Guide](OMNIVOICE_TAGS_GUIDE.md)
|
||||
|
||||
## What This Editor Is For
|
||||
|
||||
The editor is meant to make tag-heavy TTS text and SRT subtitle editing faster inside ComfyUI without manually rebuilding bracket syntax all the time.
|
||||
|
||||
It supports:
|
||||
|
||||
- Character switching tags
|
||||
- Language switching tags
|
||||
- Per-segment parameter overrides
|
||||
- Engine-aware inline tags for Step Audio EditX, Higgs Audio v3, CosyVoice3, and OmniVoice
|
||||
- Presets and edit history
|
||||
- SRT-aware highlighting and timing editing
|
||||
|
||||
## Main Views
|
||||
|
||||
The top bar includes four views:
|
||||
|
||||
- `Editor` for direct text editing
|
||||
- `History` for restoring earlier snapshots
|
||||
- `Presets` for quick saved snippets or character setups
|
||||
- `Library` for built-in reference material
|
||||
|
||||
## Basic Tag Editing
|
||||
|
||||
Use the left panel to insert or modify tags without writing every bracket manually.
|
||||
|
||||
Common examples:
|
||||
|
||||
```text
|
||||
[Alice]
|
||||
[en:Alice]
|
||||
[Alice|seed:42|temperature:0.7]
|
||||
[pause:1s]
|
||||
```
|
||||
|
||||
Useful behavior:
|
||||
|
||||
- Character names are inserted at the caret or wrapped around the current selection
|
||||
- Language and speaker can be combined in one tag
|
||||
- Parameters can be stacked with `|`
|
||||
- Keep pauses separate and before the tag they precede: `[pause:1s] [Alice|temperature:0.7]`
|
||||
- `Format` moves a pause nested with character or parameter parts into that standalone form
|
||||
- Presets can store either quick snippets or reusable speaker setups
|
||||
|
||||
## Inline Tags
|
||||
|
||||
The editor has an engine-aware `Inline Tags` panel.
|
||||
|
||||
Step Audio EditX examples:
|
||||
|
||||
```text
|
||||
<Laughter>
|
||||
<emotion:happy>
|
||||
<style:whisper:2>
|
||||
<restore:1@2>
|
||||
```
|
||||
|
||||
Higgs Audio v3 examples:
|
||||
|
||||
```text
|
||||
<|emotion:amusement|>
|
||||
<|style:whispering|>
|
||||
<|prosody:pause|>
|
||||
```
|
||||
|
||||
CosyVoice3 examples:
|
||||
|
||||
```text
|
||||
<breath>
|
||||
<laughing>that was funny</laughing>
|
||||
```
|
||||
|
||||
OmniVoice examples:
|
||||
|
||||
```text
|
||||
<laughter>
|
||||
<sigh>
|
||||
<question-ei>
|
||||
```
|
||||
|
||||
Use the dedicated inline tag controls in the sidebar when you do not want to type these by hand.
|
||||
|
||||
Important differences:
|
||||
|
||||
- `Step Audio EditX` tags are post-process controls
|
||||
- `Higgs Audio v3`, `CosyVoice3`, and `OmniVoice` tags are native generation controls
|
||||
- `OmniVoice` editor insertion uses suite-default angle-tag aliases and the processor converts them internally to official native tags during generation
|
||||
|
||||
## SRT Editing
|
||||
|
||||
When the text looks like valid SRT, the editor highlights subtitle numbers and timings differently and enables subtitle-specific editing tools.
|
||||
|
||||
Example:
|
||||
|
||||
```srt
|
||||
1
|
||||
00:00:01,000 --> 00:00:04,000
|
||||
[Alice] Hello there.
|
||||
|
||||
2
|
||||
00:00:04,500 --> 00:00:08,000
|
||||
[Bob] This is the next subtitle.
|
||||
```
|
||||
|
||||
### Drag Timing Controls
|
||||
|
||||
Timing lines are directly draggable:
|
||||
|
||||
- Drag the left timestamp to change cue start
|
||||
- Drag the right timestamp to change cue end
|
||||
- Drag the arrow to move the whole cue while keeping its duration
|
||||
|
||||
Modifiers:
|
||||
|
||||
- Hold `Shift` while dragging start or end to move the neighboring cue boundary by the same delta and preserve the existing gap
|
||||
- Hold `Alt` while dragging for finer timing adjustment
|
||||
|
||||
### Merge Cues
|
||||
|
||||
Sometimes TTS sounds better when two short subtitles are spoken as one longer chunk.
|
||||
|
||||
Use:
|
||||
|
||||
- `Alt+click` on a cue number to merge that cue with the next cue
|
||||
- `Alt+Shift+click` on a cue number to merge that cue with the previous cue
|
||||
|
||||
Merge behavior:
|
||||
|
||||
- Keeps the first cue start time
|
||||
- Keeps the second cue end time
|
||||
- Combines the text into one subtitle body
|
||||
- Renumbers the remaining cues
|
||||
|
||||
### Split Cues
|
||||
|
||||
Sometimes a subtitle is too long and you want to split it into two timed parts.
|
||||
|
||||
Use:
|
||||
|
||||
- Place the caret inside subtitle text
|
||||
- Press `Ctrl+Shift+Enter`
|
||||
|
||||
Split behavior:
|
||||
|
||||
- Splits the cue at the caret position
|
||||
- Estimates the new timing boundary from the text proportion on each side
|
||||
- Gives a small bias to the left side when the split happens after punctuation
|
||||
- Enforces a minimum duration for both resulting cues
|
||||
- Renumbers following cues
|
||||
|
||||
## Shortcuts
|
||||
|
||||
Core shortcuts:
|
||||
|
||||
- `Ctrl+Z` or `Cmd+Z` = Undo
|
||||
- `Ctrl+Shift+Z` / `Cmd+Shift+Z` or `Ctrl+Y` = Redo
|
||||
- `Alt+Z` = Undo
|
||||
- `Alt+Shift+Z` = Redo
|
||||
- `Ctrl+Shift+Enter` = Split current SRT cue at caret
|
||||
- `Alt+click` on cue number = Merge with next cue
|
||||
- `Alt+Shift+click` on cue number = Merge with previous cue
|
||||
|
||||
Sidebar insertion shortcuts:
|
||||
|
||||
- `Alt+C` = Add character tag
|
||||
- `Alt+L` = Add language tag
|
||||
- `Alt+P` = Add parameter tag
|
||||
- `Alt+1`, `Alt+2`, `Alt+3` = Load preset slot
|
||||
|
||||
## Editing Tips
|
||||
|
||||
- Use merge when subtitles are too short for natural speech flow
|
||||
- Use split when a single subtitle is too dense or needs better pacing
|
||||
- Keep SRT text semantically clean before generation so later timing edits remain predictable
|
||||
- Use presets for repeated speaker or parameter combinations
|
||||
- Use the `Library` tab as a quick reminder, not as the full manual
|
||||
|
||||
## Known Limits
|
||||
|
||||
- SRT merge and split are designed for standard subtitle blocks, not malformed subtitle text
|
||||
- Split timing is estimated from text distribution, so it is a good first pass, not perfect forced alignment
|
||||
- If you make large structural SRT edits, review the final timings before generation
|
||||
@@ -1,120 +0,0 @@
|
||||
# 01 - First Ask The LLM To Research The Official Model
|
||||
|
||||
Do this before implementation.
|
||||
|
||||
The official model implementation is the source of truth. Not this suite, not another ComfyUI node pack, not a random demo script. The LLM must understand what the model actually supports before it writes code.
|
||||
|
||||
## What The User Should Ask
|
||||
|
||||
Copy this to the LLM:
|
||||
|
||||
```text
|
||||
Do not write code yet.
|
||||
|
||||
Research the official implementation for this model/engine:
|
||||
|
||||
[PASTE OFFICIAL REPO, MODEL CARD, PAPER, OR PACKAGE LINK HERE]
|
||||
|
||||
Produce a capability report for TTS Audio Suite. Read the official repo, model card, examples, docs, paper if useful, license, dependency files, and inference scripts.
|
||||
|
||||
The report must answer:
|
||||
|
||||
1. What task does this model actually support?
|
||||
- TTS (in this suite, TTS means both Unified TTS Text and Unified SRT TTS)
|
||||
- Voice Changer / voice conversion
|
||||
- ASR
|
||||
- Audio editing
|
||||
- Other special feature
|
||||
|
||||
2. What native parameters does the official implementation expose?
|
||||
|
||||
3. What features are real native model capabilities?
|
||||
|
||||
4. What features are not supported natively?
|
||||
|
||||
5. What model files are required, and where do they come from?
|
||||
|
||||
6. What voice/reference input mode does it require?
|
||||
- Reference audio only
|
||||
- Reference audio plus exact transcript/reference text
|
||||
- Either reference-audio-only or reference-audio-plus-text modes
|
||||
- No reference audio
|
||||
|
||||
7. What sample rate, channel layout, and audio tensor/file format does it expect?
|
||||
|
||||
8. What dependencies and versions does it require?
|
||||
|
||||
9. Does it conflict with common packages already used by TTS Audio Suite?
|
||||
|
||||
10. What license applies to the code and weights?
|
||||
|
||||
11. Does it need only unified nodes, or does it need a special node?
|
||||
|
||||
12. Is this a short-form or long-form generation model?
|
||||
- If long-form, what is the native segmentation strategy?
|
||||
- Should chunking be text-length based, duration/minute based, or model-internal?
|
||||
- Which controls should be ignored or overridden (for example, max chars per chunk) to match native behavior?
|
||||
|
||||
Do not start implementation until this report is complete.
|
||||
```
|
||||
|
||||
## What The Capability Report Must Decide
|
||||
|
||||
The report must decide the actual scope:
|
||||
|
||||
- TTS integration, which includes both Unified TTS Text and Unified SRT TTS.
|
||||
- Voice Changer / voice conversion.
|
||||
- ASR / transcription.
|
||||
- Special node for a native unique feature.
|
||||
- No integration, if the model is not suitable.
|
||||
|
||||
Do not ask whether SRT is natively supported by the model. For TTS engines, SRT support is implemented by TTS Audio Suite by generating subtitle text segments and assembling timing with the suite timing systems.
|
||||
|
||||
If the model has a native feature that does not fit existing unified nodes, say so. Do not hide it inside a generic node if it needs a special UI.
|
||||
|
||||
Also decide the reference archetype early:
|
||||
|
||||
- Use Qwen3-TTS as default reference for standard TTS/SRT architecture.
|
||||
- Use VibeVoice as reference when the target engine is long-form and needs duration-aware/minute-aware segmentation behavior.
|
||||
|
||||
## Native Features Come First
|
||||
|
||||
Implement native official features first.
|
||||
|
||||
Good native examples:
|
||||
|
||||
- Official speaker/reference audio support.
|
||||
- Official language parameter.
|
||||
- Official emotion/style controls.
|
||||
- Official voice conversion mode.
|
||||
- Official ASR timestamps.
|
||||
|
||||
Bad unsupported examples unless discussed first:
|
||||
|
||||
- Fake speed control by time-stretching output.
|
||||
- Fake emotion by post-processing audio.
|
||||
- Fake language dropdown when the model is English-only.
|
||||
- Fake SRT timing controls that bypass the suite timing engine or imply unsupported model behavior.
|
||||
|
||||
Extra suite-added features can be useful, but they need maintainer discussion first.
|
||||
|
||||
## Minimum Research Checklist
|
||||
|
||||
The LLM should inspect:
|
||||
|
||||
- Official README.
|
||||
- Model card.
|
||||
- Inference examples.
|
||||
- CLI or demo scripts.
|
||||
- Python package metadata.
|
||||
- Requirements files.
|
||||
- Download instructions.
|
||||
- License files.
|
||||
- Hugging Face or ModelScope file layout.
|
||||
- Open issues if the model has known broken behavior.
|
||||
|
||||
If any of these cannot be found, the report must say that clearly instead of guessing.
|
||||
|
||||
---
|
||||
|
||||
Navigation: [Guide Hub](README.md) | Next: [02 - Check Existing ComfyUI Implementations](02_CHECK_EXISTING_COMFYUI_IMPLEMENTATIONS.md)
|
||||
@@ -1,81 +0,0 @@
|
||||
# 02 - Check Existing ComfyUI Implementations
|
||||
|
||||
After official model research, check whether someone already made a ComfyUI implementation.
|
||||
|
||||
This is useful, but dangerous if followed blindly.
|
||||
|
||||
## What The User Should Ask
|
||||
|
||||
Copy this to the LLM:
|
||||
|
||||
```text
|
||||
Search for existing ComfyUI implementations of this model.
|
||||
|
||||
If you find any, clone them into:
|
||||
|
||||
IgnoredForGitHubDocs/For_reference/[ENGINE_NAME]/
|
||||
|
||||
Study them as references only. Do not treat them as the source of truth. Compare every feature and parameter against the official model implementation.
|
||||
|
||||
Produce notes that answer:
|
||||
|
||||
1. Which ComfyUI implementations exist?
|
||||
2. What files or patterns are useful as reference?
|
||||
3. What dependencies or install problems do they reveal?
|
||||
4. What UI controls do they expose?
|
||||
5. Which controls are native official model features?
|
||||
6. Which controls are artificial additions?
|
||||
7. Which parts should not be copied into TTS Audio Suite?
|
||||
```
|
||||
|
||||
## Where To Put References
|
||||
|
||||
Clone references here:
|
||||
|
||||
```text
|
||||
IgnoredForGitHubDocs/For_reference/[ENGINE_NAME]/
|
||||
```
|
||||
|
||||
Do not place reference repos in the main package folders. They are study material, not suite code.
|
||||
|
||||
## How To Use Them
|
||||
|
||||
Use existing ComfyUI implementations to learn:
|
||||
|
||||
- How the model loads in a ComfyUI environment.
|
||||
- Which dependencies break easily.
|
||||
- Which model files are expected.
|
||||
- Which audio formats are accepted.
|
||||
- Which UI patterns users may expect.
|
||||
- Which edge cases others already hit.
|
||||
|
||||
But always compare back to the official implementation.
|
||||
|
||||
## What Not To Copy Blindly
|
||||
|
||||
Do not copy:
|
||||
|
||||
- Fake parameters that are not official model features.
|
||||
- Random post-processing presented as native model behavior.
|
||||
- Bad model download paths.
|
||||
- Silent downloads into cache directories.
|
||||
- Large engine logic stuffed directly into ComfyUI node UI code.
|
||||
- Dependency pins that break other engines without analysis.
|
||||
- UI controls that imply unsupported capabilities.
|
||||
|
||||
## Maintainer Discussion Required
|
||||
|
||||
If a ComfyUI implementation adds something useful but non-native, discuss it first in an issue or PR.
|
||||
|
||||
Examples:
|
||||
|
||||
- Artificial speed control.
|
||||
- Post-processing emotion controls.
|
||||
- Non-native denoise controls.
|
||||
- Any special workflow that changes model behavior beyond official inference.
|
||||
|
||||
The default implementation should expose native model capabilities first.
|
||||
|
||||
---
|
||||
|
||||
Navigation: [Guide Hub](README.md) | Previous: [01 - Research The Official Model](01_FIRST_ASK_THE_LLM_TO_RESEARCH_THE_OFFICIAL_MODEL.md) | Next: [03 - Decide Engine Scope](03_DECIDE_ENGINE_SCOPE.md)
|
||||
@@ -1,103 +0,0 @@
|
||||
# 03 - Decide Engine Scope
|
||||
|
||||
Before coding, decide exactly what the integration should include.
|
||||
|
||||
This prevents half-integrations where basic TTS works but the required SRT path, cache, tags, VRAM, or docs are missing.
|
||||
|
||||
## What The User Should Ask
|
||||
|
||||
Copy this to the LLM:
|
||||
|
||||
```text
|
||||
Using the official capability report and ComfyUI reference notes, decide the TTS Audio Suite scope for this engine.
|
||||
|
||||
Do not write code yet.
|
||||
|
||||
Produce a short scope document with:
|
||||
|
||||
1. Which node family should be supported?
|
||||
- If this is TTS, include both Unified TTS Text and Unified SRT TTS.
|
||||
2. Which native features will be implemented now?
|
||||
3. Which native features will be intentionally skipped, and why?
|
||||
4. Which non-native suite-added features are proposed, if any?
|
||||
5. Which existing engine should be used as implementation reference?
|
||||
6. What files/modules will likely need changes?
|
||||
7. Does this engine stay in the Main Environment, or does it need runtime isolation?
|
||||
8. What manual tests will prove the integration works?
|
||||
```
|
||||
|
||||
## Runtime Decision Order
|
||||
|
||||
Test the official package with `--no-deps` in the main T5 environment first. Prefer a small compatibility patch when practical; use the existing shared T4 SHRED runtime only if T5 is genuinely incompatible. Never create another environment or download/reinstall Torch or Transformers without explicit maintainer approval.
|
||||
|
||||
## Node Types To Choose From
|
||||
|
||||
Decide whether the engine needs:
|
||||
|
||||
- TTS integration: Unified TTS Text plus Unified SRT TTS. These are a mandatory pair for TTS engines.
|
||||
- Voice Changer / voice conversion.
|
||||
- ASR Transcribe.
|
||||
- A special feature node.
|
||||
- No integration.
|
||||
|
||||
Do not add a special node just because it looks interesting. Add one only when the official model has a capability that does not fit the unified nodes.
|
||||
|
||||
## Native Features vs Suite Enhancements
|
||||
|
||||
Separate these clearly.
|
||||
|
||||
Native model features:
|
||||
|
||||
- Supported by official model code or docs.
|
||||
- Should be implemented first when practical.
|
||||
- Can be exposed in the engine node UI.
|
||||
|
||||
Suite-added enhancements:
|
||||
|
||||
- Added by TTS Audio Suite around the model.
|
||||
- May be useful, but must be explicit.
|
||||
- Need maintainer discussion if they change behavior or user expectations.
|
||||
|
||||
## Written Scope Required
|
||||
|
||||
The LLM should write a scope before implementation.
|
||||
|
||||
A good scope looks like this:
|
||||
|
||||
```text
|
||||
Engine scope:
|
||||
- Supports TTS: yes, with both Unified TTS Text and Unified SRT TTS
|
||||
- SRT approach: generate each subtitle independently and assemble timing with AudioAssemblyEngine
|
||||
- Supports Voice Changer: no, official model has no voice conversion mode
|
||||
- Supports ASR: no
|
||||
- Needs special node: yes, official model has native voice design from text description
|
||||
|
||||
Native parameters to expose:
|
||||
- seed
|
||||
- temperature
|
||||
- top_p
|
||||
- language
|
||||
|
||||
Do not expose:
|
||||
- speed, because official model has no speed parameter
|
||||
|
||||
Primary reference:
|
||||
- Qwen3-TTS for TTS/SRT architecture
|
||||
|
||||
Manual tests:
|
||||
- TTS Text basic generation
|
||||
- TTS Text with character tags
|
||||
- TTS Text with pause tags
|
||||
- SRT generation with interrupt test
|
||||
- Clear VRAM then regenerate
|
||||
|
||||
Runtime plan:
|
||||
- Main Environment only
|
||||
- or Shared Runtime / Dedicated Runtime if dependency conflicts are expected
|
||||
```
|
||||
|
||||
If the scope is vague, do not code yet.
|
||||
|
||||
---
|
||||
|
||||
Navigation: [Guide Hub](README.md) | Previous: [02 - Check Existing ComfyUI Implementations](02_CHECK_EXISTING_COMFYUI_IMPLEMENTATIONS.md) | Next: [04 - Implementation Order For LLM](04_IMPLEMENTATION_ORDER_FOR_LLM.md)
|
||||
@@ -1,186 +0,0 @@
|
||||
# 04 - Implementation Order For LLM
|
||||
|
||||
Use this order when the capability report and scope are complete.
|
||||
|
||||
Do not start from random files. Follow the architecture.
|
||||
|
||||
## What The User Should Ask
|
||||
|
||||
Copy this to the LLM:
|
||||
|
||||
```text
|
||||
Implement this engine using the TTS Audio Suite architecture.
|
||||
|
||||
Before editing files, read:
|
||||
|
||||
- PROJECT_INDEX.md
|
||||
- docs/New Engines Guides/NEW_ENGINE_IMPLEMENTATION_GUIDE.md
|
||||
- docs/New Engines Guides/fails_to_avoid_TTS_Engine_Implementation.md
|
||||
|
||||
Use Qwen3-TTS as the primary modern reference for full TTS/SRT architecture and suite parity.
|
||||
Use Step Audio EditX as a secondary reference for wrapper/model lifecycle and special post-processing patterns.
|
||||
If another existing engine is closer to this model, inspect that too.
|
||||
|
||||
Do not copy any reference blindly. The official model implementation decides native capabilities.
|
||||
```
|
||||
|
||||
## Reference Engines
|
||||
|
||||
Primary reference:
|
||||
|
||||
- `engines/adapters/qwen3_tts_adapter.py`
|
||||
- `nodes/qwen3_tts/qwen3_tts_processor.py`
|
||||
- `nodes/qwen3_tts/qwen3_tts_srt_processor.py`
|
||||
- `nodes/engines/qwen3_tts_engine_node.py`
|
||||
|
||||
Use Qwen3-TTS for:
|
||||
|
||||
- Adapter/processor separation.
|
||||
- Character handling.
|
||||
- Pause tags.
|
||||
- Segment parameters.
|
||||
- Cache integration.
|
||||
- SRT processor structure that reuses the main TTS processor for generation.
|
||||
- Interrupt handling.
|
||||
- Progress feedback.
|
||||
- Console observability parity, such as input-text echo and useful live generation progress.
|
||||
- Unified model loading.
|
||||
|
||||
Secondary reference:
|
||||
|
||||
- `engines/step_audio_editx/`
|
||||
- `engines/adapters/step_audio_editx_adapter.py`
|
||||
- `nodes/step_audio_editx/`
|
||||
- `nodes/engines/step_audio_editx_engine_node.py`
|
||||
|
||||
Use Step Audio EditX for:
|
||||
|
||||
- Wrapper/model lifecycle patterns.
|
||||
- Special post-processing behavior.
|
||||
- Engine-specific extra node patterns.
|
||||
- Model loading edge cases.
|
||||
|
||||
Long-form reference:
|
||||
|
||||
- `engines/vibevoice_engine/`
|
||||
- `engines/adapters/vibevoice_adapter.py`
|
||||
- `nodes/vibevoice/vibevoice_processor.py`
|
||||
|
||||
Use VibeVoice when the target engine is long-form and needs duration-aware segmentation behavior:
|
||||
|
||||
- Prefer model-appropriate duration/minute-based segmentation when required.
|
||||
- Do not force short-form `max_chars_per_chunk` behavior if it conflicts with native long-form generation strategy.
|
||||
- If the suite already provides chunk splitting / chunk combination controls in the unified node, reuse those. Do not add engine-local chunk-silence or chunk-combination controls unless the official model has a clearly separate native long-form mechanism.
|
||||
|
||||
## Implementation Order
|
||||
|
||||
Follow this order:
|
||||
|
||||
1. Confirm scope.
|
||||
2. Verify model download layout.
|
||||
3. Add or update downloader/model layout logic.
|
||||
4. Build the engine wrapper around official inference.
|
||||
5. Register the unified model factory.
|
||||
6. Build the adapter.
|
||||
7. Build the main processor.
|
||||
8. For TTS engines, build the SRT processor too. The SRT processor should call the main TTS processor/generation path for each subtitle instead of duplicating generation logic. Build VC, ASR, or special processors only if scoped.
|
||||
9. Add the engine configuration node UI.
|
||||
10. Wire node and adapter registration.
|
||||
11. Add segment parameter support if the engine has native parameters.
|
||||
12. Add generated audio cache where applicable.
|
||||
13. Add interrupt checks in long loops.
|
||||
14. Add progress feedback for long generation.
|
||||
15. Update docs/YAML metadata.
|
||||
16. Run automated and live ComfyUI validation. Use FL-MCP-assisted validation when it is installed and connected; otherwise perform the same checks manually. Follow `tests/FL_MCP_VALIDATION.md`.
|
||||
17. Run the required parity checklist.
|
||||
|
||||
## Live ComfyUI Validation Rule
|
||||
|
||||
Passing imports or pytest is not enough for a new engine. Validate it in the canonical Windows ComfyUI installation after implementation.
|
||||
|
||||
If [ComfyUI_FL-MCP](https://github.com/filliptm/ComfyUI_FL-MCP) is available, the LLM should use it to inspect and operate the live ComfyUI instance. Treat it as an optional test driver, not a project dependency and not a substitute for the existing test suite.
|
||||
|
||||
The LLM should:
|
||||
|
||||
- After changing Python code, restart the canonical Windows ComfyUI process before testing. An already-running process still has the old modules loaded.
|
||||
- Use PowerShell to identify the process listening on port `8188`, verify its command line belongs to the canonical ComfyUI `main.py`, stop only that process, and relaunch it with the canonical Windows Python.
|
||||
- Wait for `http://127.0.0.1:8188/system_stats` to respond before using FL-MCP.
|
||||
- Refresh the existing ComfyUI browser tab after restart and confirm the FL-MCP browser bridge has reconnected before calling canvas-only tools.
|
||||
- Confirm the engine node and the relevant unified node are registered.
|
||||
- Load or construct the smallest useful workflow.
|
||||
- Inspect workflow JSON for the expected node types, links, and widget values.
|
||||
- Queue the workflow and wait for completion.
|
||||
- Inspect execution history and report the full actionable error if execution fails.
|
||||
- Confirm the expected audio output artifact exists.
|
||||
- Capture a canvas screenshot for UI and broken-node inspection.
|
||||
- Exercise TTS Text and SRT for every TTS engine, plus any other scoped capability.
|
||||
- Record what was actually tested, what was skipped, and why.
|
||||
|
||||
Screenshots prove only visible workflow state. They do not prove that generation succeeded or that audio is correct. Execution history and output artifacts are required evidence, and the user must still judge subjective audio quality.
|
||||
|
||||
Do not install FL-MCP, alter its safety settings, or enable destructive tools unless the user authorizes it. If FL-MCP is unavailable, report that fact and follow the manual fallback in `tests/FL_MCP_VALIDATION.md`.
|
||||
|
||||
Repeat the edit, restart, reconnect, and validation cycle after every implementation fix that changes imported Python code. Frontend-only changes may require a hard browser refresh as well. Do not claim that a fix was tested against a ComfyUI process started before the fix was written.
|
||||
|
||||
## Architecture Rule
|
||||
|
||||
Unified nodes should stay thin.
|
||||
|
||||
Reference engines are examples only. Every engine must have dedicated processors and adapters; share only engine-neutral utilities.
|
||||
|
||||
Do not put hundreds of lines of engine-specific orchestration into:
|
||||
|
||||
- `nodes/unified/tts_text_node.py`
|
||||
- `nodes/unified/tts_srt_node.py`
|
||||
- `nodes/unified/voice_changer_node.py`
|
||||
- `nodes/unified/asr_transcribe_node.py`
|
||||
|
||||
Engine-specific orchestration belongs in processors and adapters.
|
||||
|
||||
For TTS engines, SRT support is mandatory. It is the suite's subtitle-driven TTS flow, not a native capability the model has to expose.
|
||||
|
||||
## SRT Reuses TTS Generation
|
||||
|
||||
Follow the Qwen3-TTS pattern: the SRT processor is the timing/orchestration layer, not a second implementation of the engine.
|
||||
|
||||
The SRT processor should:
|
||||
|
||||
- Import or instantiate the engine's main TTS processor.
|
||||
- Parse SRT subtitles and loop through subtitle entries.
|
||||
- Build the subtitle-specific voice mapping and seed/parameter context.
|
||||
- Call the main TTS processor, usually `processor.process_text(...)`, for each subtitle's text.
|
||||
- Reuse the same adapter, cache keys, character handling, pause tags, parameter switching, progress hooks, and model lifecycle used by normal TTS Text generation.
|
||||
- Assemble subtitle audio with the suite timing systems, such as `AudioAssemblyEngine`, after generation.
|
||||
|
||||
The SRT processor should not:
|
||||
|
||||
- Call the raw model directly through a separate generation path.
|
||||
- Reimplement character switching, pause tags, parameter switching, cache, or reference-audio handling differently from the TTS processor.
|
||||
- Add separate fake SRT-only parameters that do not exist in the normal TTS path or the suite timing layer.
|
||||
|
||||
## Chunking Rule
|
||||
|
||||
If an engine needs long-text chunking, prefer the suite chunking and shared chunk combiner unless the official model exposes a real native long-form/chunking system that is materially different.
|
||||
|
||||
Do not:
|
||||
|
||||
- Add duplicate engine-node controls for silence between chunks, chunk combination method, or similar suite-owned chunk-assembly behavior.
|
||||
- Bypass the shared chunk combiner with a custom engine-local join path unless there is a documented official reason.
|
||||
|
||||
## Registration Areas To Check
|
||||
|
||||
The LLM should inspect and update the relevant registration files for the chosen scope:
|
||||
|
||||
- `nodes.py`
|
||||
- `engines/adapters/__init__.py`
|
||||
- `utils/models/unified_model_interface.py`
|
||||
- `utils/models/engine_registry.py`
|
||||
- `utils/text/segment_parameters.py`
|
||||
- `docs/Dev reports/tts_audio_suite_engines.yaml`
|
||||
- `docs/Dev reports/tts_audio_suite_aux_models.yaml` if helper models are added
|
||||
|
||||
Only update files that are actually needed for the scoped integration.
|
||||
|
||||
---
|
||||
|
||||
Navigation: [Guide Hub](README.md) | Previous: [03 - Decide Engine Scope](03_DECIDE_ENGINE_SCOPE.md) | Next: [05 - Required Parity Checklist](05_REQUIRED_PARITY_CHECKLIST.md)
|
||||
@@ -1,120 +0,0 @@
|
||||
# 05 - Required Parity Checklist
|
||||
|
||||
Paste this checklist to the LLM before review.
|
||||
|
||||
If the LLM cannot answer these questions, the implementation is not ready.
|
||||
|
||||
## Copy-Paste Checklist
|
||||
|
||||
```text
|
||||
Review the new engine implementation against this TTS Audio Suite parity checklist.
|
||||
|
||||
Architecture:
|
||||
- Did you use a processor instead of putting engine logic in unified nodes?
|
||||
- Are unified nodes only doing thin routing/delegation?
|
||||
- Is engine-specific orchestration in processors/adapters?
|
||||
|
||||
Model loading and lifecycle:
|
||||
- Did you use unified_model_interface.load_model() in all model-loading paths?
|
||||
- Does Clear VRAM / unload / reload work?
|
||||
- Did you avoid __del__ destructors that auto-unload models after generation?
|
||||
- Are model variants unloaded or reused correctly instead of accumulating VRAM?
|
||||
|
||||
Downloads and dependencies:
|
||||
- Did you verify the real Hugging Face / ModelScope / official file layout?
|
||||
- Do models download into organized ComfyUI/models/TTS/ folders?
|
||||
- Did you prevent silent downloads into random cache folders?
|
||||
- Did you document dependency conflicts or install.py changes?
|
||||
- Did you test `--no-deps` on Main/T5 and simple patching before falling back to the shared T4 runtime?
|
||||
- Did you avoid creating another environment or downloading/reinstalling Torch or Transformers without explicit maintainer approval?
|
||||
- If runtime isolation is needed, did you document the default mode and the reason in YAML/README?
|
||||
|
||||
Audio format:
|
||||
- Did you verify the model native sample rate?
|
||||
- Does output use valid ComfyUI audio dict/tensor shape?
|
||||
- Did you resample only when needed?
|
||||
- Did you avoid pitch/speed bugs from wrong sample-rate metadata?
|
||||
|
||||
Generated audio cache:
|
||||
- Did you add generated audio cache where applicable?
|
||||
- Does the cache key include text, voice/reference audio identity, model variant, seed, and all behavior-affecting params?
|
||||
- Did you test generating the same input twice and confirm cache behavior?
|
||||
|
||||
Text features:
|
||||
- Did you wire character tags if this node supports text generation?
|
||||
- Did you handle narrator fallback?
|
||||
- Did you wire pause tags?
|
||||
- Did you wire parameter switching for real native parameters?
|
||||
- Did you avoid adding fake unsupported parameters?
|
||||
- Did you remove misleading controls like language selection if unsupported?
|
||||
- If the engine supports language control, did you normalize the user-facing language UI/logs honestly and map them back to official backend codes without breaking segment/alias language switching?
|
||||
- Did you reuse the suite chunking/chunk-combination controls instead of inventing duplicate engine-local chunk UI or chunk-silence parameters?
|
||||
|
||||
SRT features:
|
||||
- If this is a TTS engine, did you implement the SRT path?
|
||||
- Did you add interrupt checks in long SRT loops?
|
||||
- Does SRT use the shared timing/assembly/reporting systems correctly, using the real existing APIs instead of guessed helper names?
|
||||
- Did you avoid unnecessary timing-mode restrictions?
|
||||
- Does character switching work in SRT if text character tags are supported?
|
||||
|
||||
UX:
|
||||
- Did you add progress feedback for long generation?
|
||||
- Did you preserve useful console observability parity instead of making the engine run silently (for example input-text echo and live it/s/progress)?
|
||||
- Are error messages clear when required inputs are missing?
|
||||
- Are tooltips honest about limitations, license, VRAM, and expected text length?
|
||||
|
||||
Docs and metadata:
|
||||
- Did you update YAML-backed engine metadata?
|
||||
- Did you regenerate docs/tables if metadata changed?
|
||||
- Did you document model paths and download sources?
|
||||
- Did you add examples or notes for special native features?
|
||||
- If the engine needs a secondary environment, does the engine table clearly show that?
|
||||
|
||||
Manual tests:
|
||||
- Did TTS Text work?
|
||||
- Did SRT TTS work for every TTS engine?
|
||||
- Did Voice Changer work if scoped?
|
||||
- Did ASR work if scoped?
|
||||
- Did character tags work?
|
||||
- Did pause tags work?
|
||||
- Did parameter switching work?
|
||||
- Did Clear VRAM then regenerate work, and did unload actually tear down runtime/cache state instead of only moving weights to CPU?
|
||||
- Did interrupt/cancel work in long generation?
|
||||
|
||||
Live ComfyUI integration evidence:
|
||||
- Was validation run in the canonical Windows ComfyUI environment?
|
||||
- Was ComfyUI restarted after the final Python changes, and can you show that the tested process started after those edits?
|
||||
- After restart, was the browser refreshed and the FL-MCP browser bridge confirmed connected before canvas checks?
|
||||
- If FL-MCP was available, did you follow tests/FL_MCP_VALIDATION.md and identify which MCP checks were run?
|
||||
- If FL-MCP was unavailable, did you perform and document the equivalent manual checks instead of claiming MCP validation?
|
||||
- Are the engine node and relevant unified nodes present without import or registration errors?
|
||||
- Does the saved workflow JSON contain the expected node types, links, and widget values?
|
||||
- Does the workflow load without missing/broken node state?
|
||||
- Was the workflow queued, and does execution history show successful completion or the full actionable error?
|
||||
- Does each successful run produce the expected audio output artifact?
|
||||
- Was a canvas screenshot captured for UI inspection without treating the screenshot as execution proof?
|
||||
- Did you record tested, failed, and skipped cases, including the reason for every skip?
|
||||
- Did a human assess subjective audio quality separately?
|
||||
```
|
||||
|
||||
## Important Failures This Prevents
|
||||
|
||||
This checklist exists because working audio is not enough.
|
||||
|
||||
Common failures:
|
||||
|
||||
- Engine works once, then fails after ComfyUI unloads models.
|
||||
- SRT generation cannot be cancelled.
|
||||
- Cache is missing, so repeated generations waste time and VRAM.
|
||||
- UI exposes fake language/speed/emotion controls.
|
||||
- Reference audio uses the wrong sample rate and changes pitch.
|
||||
- Character tags work in TTS Text but not SRT.
|
||||
- Pause tags break because segments are handled incorrectly.
|
||||
- Model files download to a hidden cache instead of `ComfyUI/models/TTS/`.
|
||||
- Unified node gets filled with engine-specific code.
|
||||
|
||||
If any item fails, fix it before PR review.
|
||||
|
||||
---
|
||||
|
||||
Navigation: [Guide Hub](README.md) | Previous: [04 - Implementation Order For LLM](04_IMPLEMENTATION_ORDER_FOR_LLM.md) | Next: [06 - User Prompts To Copy Paste](06_USER_PROMPTS_TO_COPY_PASTE.md)
|
||||
@@ -1,162 +0,0 @@
|
||||
# 06 - User Prompts To Copy Paste
|
||||
|
||||
These prompts are for a user guiding an LLM.
|
||||
|
||||
Use them in order. Do not skip the research prompts.
|
||||
|
||||
## Prompt 1 - Official Model Research
|
||||
|
||||
```text
|
||||
Do not write code yet.
|
||||
|
||||
I want to add this model/engine to TTS Audio Suite:
|
||||
|
||||
[PASTE OFFICIAL LINK]
|
||||
|
||||
Read the official repo, model card, docs, examples, license, dependency files, model download instructions, and inference scripts.
|
||||
|
||||
Produce a capability report for TTS Audio Suite. It must explain:
|
||||
|
||||
- What task the model supports: TTS, Voice Changer, ASR, audio editing, or special feature. If it supports TTS, TTS Audio Suite requires both Unified TTS Text and Unified SRT TTS.
|
||||
- Native official parameters.
|
||||
- Native official features.
|
||||
- Unsupported features that should not be exposed as UI controls.
|
||||
- Required model files and expected folder layout.
|
||||
- Sample rate and audio format.
|
||||
- Dependencies and likely conflicts.
|
||||
- License restrictions.
|
||||
- Whether this needs a special node beyond unified nodes.
|
||||
|
||||
Do not implement anything until this report is complete.
|
||||
```
|
||||
|
||||
## Prompt 2 - Existing ComfyUI Implementations
|
||||
|
||||
```text
|
||||
Search for existing ComfyUI implementations of this model.
|
||||
|
||||
If you find any, clone them into:
|
||||
|
||||
IgnoredForGitHubDocs/For_reference/[ENGINE_NAME]/
|
||||
|
||||
Study them only as references. Compare every feature against the official implementation.
|
||||
|
||||
Produce notes explaining:
|
||||
|
||||
- Which implementations exist.
|
||||
- Which files/patterns are useful.
|
||||
- Which dependencies or install issues they reveal.
|
||||
- Which UI controls are native model features.
|
||||
- Which UI controls are artificial or should not be copied.
|
||||
- What should be discussed with the maintainer before adding.
|
||||
|
||||
Do not treat third-party ComfyUI code as the source of truth.
|
||||
```
|
||||
|
||||
## Prompt 3 - Decide Scope
|
||||
|
||||
```text
|
||||
Using the official capability report and ComfyUI reference notes, decide the integration scope.
|
||||
|
||||
Do not write code yet.
|
||||
|
||||
Create a scope summary with:
|
||||
|
||||
- Whether this is a TTS integration. If yes, include both Unified TTS Text and Unified SRT TTS.
|
||||
- Whether to support Voice Changer.
|
||||
- Whether to support ASR.
|
||||
- Whether a special node is needed.
|
||||
- Native parameters to expose.
|
||||
- Native features to implement now.
|
||||
- Native features intentionally skipped, with reasons.
|
||||
- Non-native suite-added features proposed, if any.
|
||||
- Existing engine references to inspect.
|
||||
- Manual tests required.
|
||||
```
|
||||
|
||||
## Prompt 4 - Implementation Plan
|
||||
|
||||
```text
|
||||
Create an implementation plan for this engine in TTS Audio Suite.
|
||||
|
||||
Read first:
|
||||
|
||||
- PROJECT_INDEX.md
|
||||
- docs/New Engines Guides/NEW_ENGINE_IMPLEMENTATION_GUIDE.md
|
||||
- docs/New Engines Guides/fails_to_avoid_TTS_Engine_Implementation.md
|
||||
|
||||
Use Qwen3-TTS as the primary modern reference for full TTS/SRT architecture and suite parity.
|
||||
Use Step Audio EditX as a secondary reference for wrapper/model lifecycle and special post-processing patterns.
|
||||
|
||||
Do not copy references blindly. The official implementation decides capabilities.
|
||||
|
||||
The plan must include files/modules to change, model download approach, model lifecycle approach, processor/adapter design, tag support, cache support, SRT support for TTS engines, VC/ASR support if scoped, docs updates, and manual tests.
|
||||
```
|
||||
|
||||
## Prompt 5 - Implement First Phase
|
||||
|
||||
```text
|
||||
Implement the first phase only.
|
||||
|
||||
Focus on:
|
||||
|
||||
- Model download/layout.
|
||||
- Engine wrapper.
|
||||
- Unified model factory registration.
|
||||
- Adapter skeleton with real generation path.
|
||||
- Engine node UI for native official parameters only.
|
||||
|
||||
Do not add fake parameters. Do not implement special non-native features unless they were explicitly approved.
|
||||
|
||||
After editing, summarize changed files and what still needs implementation.
|
||||
```
|
||||
|
||||
## Prompt 6 - Review For Suite Parity
|
||||
|
||||
```text
|
||||
Review this implementation against:
|
||||
|
||||
- docs/New Engines Guides/05_REQUIRED_PARITY_CHECKLIST.md
|
||||
- docs/New Engines Guides/fails_to_avoid_TTS_Engine_Implementation.md
|
||||
|
||||
Be strict. Find missing architecture, cache, VRAM, interrupt, audio shape, sample rate, tag, SRT, docs, and fake-parameter issues.
|
||||
|
||||
Return findings ordered by severity with file references.
|
||||
```
|
||||
|
||||
## Prompt 7 - Check Fake Or Non-Native Parameters
|
||||
|
||||
```text
|
||||
Compare the engine UI and adapter parameters against the official model implementation.
|
||||
|
||||
List every exposed parameter and mark it as:
|
||||
|
||||
- Native official model parameter.
|
||||
- Suite infrastructure parameter.
|
||||
- Non-native enhancement.
|
||||
- Unsupported/fake and should be removed.
|
||||
|
||||
If any non-native enhancement exists, explain whether it was approved by the maintainer.
|
||||
```
|
||||
|
||||
## Prompt 8 - Prepare PR Summary
|
||||
|
||||
```text
|
||||
Prepare a PR summary for this engine integration.
|
||||
|
||||
Include:
|
||||
|
||||
- Official model capability summary.
|
||||
- Existing ComfyUI references studied.
|
||||
- Supported node types.
|
||||
- Native features implemented.
|
||||
- Native features skipped, with reasons.
|
||||
- Any non-native suite-added features, with approval note.
|
||||
- Model download paths.
|
||||
- Manual ComfyUI test results.
|
||||
- Known limitations.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
Navigation: [Guide Hub](README.md) | Previous: [05 - Required Parity Checklist](05_REQUIRED_PARITY_CHECKLIST.md) | Next: [07 - PR Review Checklist](07_PR_REVIEW_CHECKLIST.md)
|
||||
@@ -1,142 +0,0 @@
|
||||
# 07 - PR Review Checklist
|
||||
|
||||
Use this before merging a new engine PR.
|
||||
|
||||
A PR is not ready just because it generates audio.
|
||||
|
||||
## Required PR Evidence
|
||||
|
||||
The PR author or LLM should provide:
|
||||
|
||||
- Official model capability report.
|
||||
- Existing ComfyUI reference notes, if any.
|
||||
- Selected engine scope.
|
||||
- Native parameters implemented.
|
||||
- Native features intentionally skipped, with reasons.
|
||||
- Non-native suite-added features, if any, with maintainer approval note.
|
||||
- Model download path and file layout.
|
||||
- Manual ComfyUI test results.
|
||||
- Known limitations.
|
||||
|
||||
If this evidence is missing, ask for it before deep review.
|
||||
|
||||
## Architecture Review
|
||||
|
||||
Check:
|
||||
|
||||
- Unified nodes are thin.
|
||||
- Engine-specific orchestration lives in processors/adapters.
|
||||
- TTS Text processor exists for every TTS engine.
|
||||
- SRT processor exists for every TTS engine. Missing SRT support requires an explicit maintainer-approved exception.
|
||||
- VC or ASR processors follow existing suite patterns when scoped.
|
||||
- Similar existing engines were used as references, not blindly copied.
|
||||
|
||||
Primary reference for modern full TTS/SRT integration:
|
||||
|
||||
- Qwen3-TTS.
|
||||
|
||||
Secondary reference for wrapper/lifecycle and special post-processing:
|
||||
|
||||
- Step Audio EditX.
|
||||
|
||||
## Model Lifecycle Review
|
||||
|
||||
Check:
|
||||
|
||||
- All model loading paths use `unified_model_interface.load_model()` unless there is a documented exception.
|
||||
- ComfyUI unload/reload/Clear VRAM works.
|
||||
- No `__del__` destructor unloads the model after generation.
|
||||
- Device movement is handled safely.
|
||||
- Quantized models are not moved with unsupported `.to()` calls.
|
||||
- Multiple model variants do not accumulate in VRAM unnecessarily.
|
||||
|
||||
## Downloads And Dependencies Review
|
||||
|
||||
Check:
|
||||
|
||||
- Official model file layout was verified.
|
||||
- Downloads go to organized `ComfyUI/models/TTS/` paths.
|
||||
- No hidden or automatic downloads to random cache directories.
|
||||
- `install.py` or requirements changes are justified.
|
||||
- Dependency pins do not break existing engines.
|
||||
- License restrictions are documented.
|
||||
|
||||
## Feature Review
|
||||
|
||||
Check:
|
||||
|
||||
- UI exposes native model parameters only, plus clearly documented suite infrastructure options.
|
||||
- Unsupported controls are not present.
|
||||
- Language controls are only present when official model supports language control.
|
||||
- Character tags work if text generation is supported.
|
||||
- Narrator fallback works.
|
||||
- Pause tags work.
|
||||
- Segment parameter switching works for real parameters.
|
||||
- Inline edit/post-processing behavior respects segment boundaries if used.
|
||||
- Special nodes are justified by native model capability.
|
||||
|
||||
## SRT Review
|
||||
|
||||
For every TTS engine, check the SRT path:
|
||||
|
||||
- Interrupt/cancel works in long loops.
|
||||
- Timing modes use the shared timing/assembly systems correctly.
|
||||
- There are no unnecessary restrictions on timing modes.
|
||||
- Character switching works in SRT if character tags are supported.
|
||||
- Pause tags and generated segment durations do not corrupt timing.
|
||||
- Timing report/generation info is consistent with other engines.
|
||||
|
||||
## Cache And Progress Review
|
||||
|
||||
Check:
|
||||
|
||||
- Generated audio cache exists where applicable.
|
||||
- Cache key includes all behavior-affecting parameters.
|
||||
- Reference audio identity is included when relevant.
|
||||
- Progress bars or progress messages exist for long generation.
|
||||
- Interrupt checks are present inside long generation loops, not only before the loop.
|
||||
|
||||
## Audio Review
|
||||
|
||||
Check:
|
||||
|
||||
- Output tensor shape is valid for ComfyUI.
|
||||
- Sample rate is correct and documented.
|
||||
- Reference audio is resampled to the native rate where required.
|
||||
- Audio is not accidentally squeezed from 3D to invalid 2D output.
|
||||
- Pitch/speed are not changed by wrong sample-rate metadata.
|
||||
|
||||
## Documentation Review
|
||||
|
||||
Check:
|
||||
|
||||
- YAML-backed engine metadata is updated if engine docs changed.
|
||||
- Generated docs/tables are regenerated when required.
|
||||
- Model download sources are documented.
|
||||
- Model layout docs are updated if needed.
|
||||
- README or user docs mention important limitations.
|
||||
- Tooltips are honest about license, VRAM, speed, and model limits.
|
||||
|
||||
## Manual Test Matrix
|
||||
|
||||
At minimum, test scoped features:
|
||||
|
||||
- Basic TTS Text generation.
|
||||
- TTS Text with character tags.
|
||||
- TTS Text with pause tags.
|
||||
- TTS Text with parameter switching.
|
||||
- For TTS engines, SRT generation.
|
||||
- For TTS engines, SRT interrupt/cancel.
|
||||
- Voice Changer if supported.
|
||||
- ASR if supported.
|
||||
- Clear VRAM / unload models, then generate again.
|
||||
- Repeat identical generation to check cache behavior.
|
||||
- Missing input error paths.
|
||||
|
||||
## Final Merge Rule
|
||||
|
||||
Do not merge until the implementation passes the required parity checklist or the remaining gaps are explicitly accepted by the maintainer.
|
||||
|
||||
---
|
||||
|
||||
Navigation: [Guide Hub](README.md) | Previous: [06 - User Prompts To Copy Paste](06_USER_PROMPTS_TO_COPY_PASTE.md)
|
||||
@@ -1,651 +0,0 @@
|
||||
# New TTS Engine Implementation Guide
|
||||
|
||||
*Comprehensive guide for LLMs to implement new TTS engines in TTS Audio Suite*
|
||||
|
||||
---
|
||||
|
||||
Don't ever make PLACEHOLDERS during implementation.
|
||||
|
||||
## Implementation Methodology
|
||||
|
||||
**Recommended Approach: Implement First, Then Compare**
|
||||
|
||||
When creating new engine components (wrapper, downloader, adapter, processor), follow this pattern:
|
||||
|
||||
1. **Implement based on understanding**: Write code using:
|
||||
- This guide's patterns and requirements
|
||||
- Implementation plan for the specific engine
|
||||
- `fails_to_avoid_TTS_Engine_Implementation.md` lessons
|
||||
- Engine-specific architecture documents
|
||||
|
||||
2. **Compare with reference implementation**: After writing, compare your code with the corresponding Step Audio EditX file:
|
||||
- Engine wrapper → `engines/step_audio_editx/step_audio_editx.py`
|
||||
- Downloader → `engines/step_audio_editx/step_audio_editx_downloader.py`
|
||||
- Adapter → `engines/adapters/step_audio_editx_adapter.py`
|
||||
- Processor → `nodes/step_audio_editx/step_audio_editx_processor.py`
|
||||
|
||||
3. **Fix discrepancies**: Correct patterns, imports, and methods that don't match established conventions
|
||||
|
||||
**Why this approach works better than copying:**
|
||||
- **Learning reinforcement**: Making and fixing mistakes creates stronger memory patterns
|
||||
- **Adaptation not copying**: Each engine has unique requirements that pure copying might miss
|
||||
- **Faster iteration**: Write quickly, then one focused comparison pass catches issues
|
||||
- **Documents real mistakes**: Comparison reveals actual errors worth adding to fails doc
|
||||
- **Preserves reasoning**: Shows understanding of WHY patterns exist, not just WHAT they are
|
||||
|
||||
This is the pattern that successfully created Step Audio EditX and should be followed for all future engines.
|
||||
|
||||
## Pre-Implementation Analysis
|
||||
|
||||
### 1. Research the Original Implementation
|
||||
|
||||
**📁 Reference Storage:**
|
||||
|
||||
- Clone the original implementation to: `ComfyUI_TTS_Audio_Suite/IgnoredForGitHubDocs/For_reference/[ENGINE_NAME]/`
|
||||
- Study the original codebase thoroughly
|
||||
- Document all features, parameters, and capabilities
|
||||
|
||||
**🔍 Key Areas to Analyze:**
|
||||
|
||||
- **Audio Format**: Sample rate, bit depth, channels, tensor format
|
||||
- **Model Architecture**: Input/output requirements, tokenization, generation process
|
||||
- **Parameters**: All generation parameters, their ranges, default values
|
||||
- **Dependencies**: Required packages, versions, potential conflicts
|
||||
- **Unique Features**: Special capabilities not found in other engines
|
||||
- **Language Support**: Monolingual vs multilingual, language codes/tags
|
||||
- **Voice Control**: How voices are defined, selected, and applied
|
||||
|
||||
### 2. Dependency Analysis
|
||||
|
||||
*⚠️ Problematic Dependencies Handling:**
|
||||
|
||||
- If dependencies conflict with existing packages → Add to `scripts/install.py`
|
||||
- If dependencies require specific versions → Document in engine requirements
|
||||
|
||||
**Example problematic patterns:**
|
||||
|
||||
- Downgrades numpy/librosa/transformers
|
||||
- Conflicts with other TTS engines
|
||||
|
||||
---
|
||||
|
||||
## Project Architecture Understanding
|
||||
|
||||
### Core Architecture Pattern
|
||||
|
||||
```
|
||||
⚙️ Engine Node (UI Layer real node on ComfyUI)
|
||||
↓
|
||||
🔌 Engine Adapter (Optional? - Parameter Translation)
|
||||
↓
|
||||
🏭 Engine Processor (Engine-Specific Logic)
|
||||
```
|
||||
|
||||
### File Structure Template
|
||||
|
||||
```
|
||||
engines/[ENGINE_NAME]/
|
||||
├── __init__.py # Engine initialization
|
||||
├── [engine_name].py # Core engine implementation
|
||||
├── [engine_name]_downloader.py # Model auto-download (optional, should use universal downloader code not it's own)
|
||||
├── stateless_wrapper.py # Thread-safe wrapper (if needed)
|
||||
└── models/ # Model-specific code (if needed)
|
||||
|
||||
engines/adapters/
|
||||
└── [engine_name]_adapter.py # Unified interface adapter
|
||||
|
||||
nodes/engines/
|
||||
└── [engine_name]_engine_node.py # UI configuration node
|
||||
|
||||
nodes/[engine_name]/ # Engine-specific processors
|
||||
├── [engine_name]_processor.py # Main TTS processor
|
||||
├── [engine_name]_srt_processor.py # SRT processor
|
||||
└── [engine_name]_vc_processor.py # Voice conversion (if applicable)
|
||||
|
||||
nodes/[engine_name]_special/ # Special features (if any)
|
||||
└── [engine_name]_special_node.py # Special functionality nodes
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Implementation Steps
|
||||
|
||||
### Phase 1: Basic UI Engine Implementation
|
||||
|
||||
#### Step 1: Create Core Engine Implementation
|
||||
|
||||
**File:** `engines/[ENGINE_NAME]/[engine_name].py`
|
||||
|
||||
**Key Implementation Notes:**
|
||||
|
||||
- **Audio Format**: Always return `torch.Tensor` in shape `[1, samples]` or `[batch, samples]`
|
||||
- **Sample Rate**: Must match engine's native sample rate, handle conversion in adapter if needed
|
||||
- **Device Management**: Support "auto", "cuda", "cpu" device selection
|
||||
- **Clear VRAM Integration**: CRITICAL - Implement `.to()` method and device checking for ComfyUI model management (see Step 1b below)
|
||||
- **Error Handling**: Graceful fallbacks, informative error messages
|
||||
|
||||
#### Step 1b: Implement Clear VRAM Integration (CRITICAL)
|
||||
|
||||
**⚠️ REQUIRED FOR ALL NEW ENGINES**
|
||||
|
||||
ComfyUI's "Clear VRAM" button offloads models to CPU. Your engine MUST support this or it will cause device mismatch errors.
|
||||
|
||||
**Implementation Requirements:**
|
||||
|
||||
**1. Add `.to()` Method to Engine Class**
|
||||
|
||||
This method moves ALL model components between devices (CPU ↔ CUDA).
|
||||
|
||||
```python
|
||||
def to(self, device):
|
||||
"""
|
||||
Move all model components to the specified device.
|
||||
|
||||
Critical for ComfyUI model management - ensures all components move together
|
||||
when models are detached to CPU and later reloaded to CUDA.
|
||||
"""
|
||||
self.device = device
|
||||
|
||||
# Move the underlying model/engine if loaded
|
||||
if self._model is not None:
|
||||
# OPTION 1: Simple engines with single model
|
||||
if hasattr(self._model, 'to'):
|
||||
self._model = self._model.to(device)
|
||||
|
||||
# OPTION 2: Complex engines with multiple components
|
||||
# Example: ChatterBox has model.s3gen, model.t3, model.voice_encoder
|
||||
for component_name in ['s3gen', 't3', 'voice_encoder', 'tokenizer']:
|
||||
if hasattr(self._model, component_name):
|
||||
component = getattr(self._model, component_name)
|
||||
if hasattr(component, 'to'):
|
||||
setattr(self._model, component_name, component.to(device))
|
||||
|
||||
# OPTION 3: Deeply nested engines (like Index-TTS)
|
||||
# Call .to() on engine itself for recursive moving, with manual fallback
|
||||
if hasattr(self._model, 'to'):
|
||||
self._model = self._model.to(device)
|
||||
else:
|
||||
# Manually move known nested components
|
||||
for attr_name in ['semantic_model', 'gpt_model', 'vocoder']:
|
||||
if hasattr(self._model, attr_name):
|
||||
component = getattr(self._model, attr_name)
|
||||
if hasattr(component, 'to'):
|
||||
setattr(self._model, attr_name, component.to(device))
|
||||
|
||||
# Update device attribute on model if it exists
|
||||
if hasattr(self._model, 'device'):
|
||||
self._model.device = torch.device(device) if isinstance(device, str) else device
|
||||
|
||||
return self
|
||||
```
|
||||
|
||||
**2. Add Device Checking Before Generation**
|
||||
|
||||
Check if model was offloaded to CPU and reload to CUDA before generation.
|
||||
|
||||
```python
|
||||
def generate(self, text, reference_audio, **kwargs):
|
||||
"""Main generation method"""
|
||||
|
||||
# CRITICAL: Check and reload model if offloaded to CPU
|
||||
target_device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
|
||||
if self._model is not None:
|
||||
# Check current device of model
|
||||
# Strategy 1: Check first parameter device (most reliable)
|
||||
if hasattr(self._model, 'parameters'):
|
||||
try:
|
||||
first_param = next(self._model.parameters())
|
||||
current_device = str(first_param.device)
|
||||
|
||||
if current_device != target_device:
|
||||
print(f"🔄 Reloading {self.__class__.__name__} from {current_device} to {target_device}")
|
||||
|
||||
# Try to find wrapper and call wrapper.model_load()
|
||||
wrapper_found = False
|
||||
try:
|
||||
from utils.models.unified_model_interface import unified_model_interface
|
||||
|
||||
if hasattr(unified_model_interface, 'model_manager'):
|
||||
for cache_key, wrapper in unified_model_interface.model_manager._model_cache.items():
|
||||
model = wrapper.model if hasattr(wrapper, 'model') else None
|
||||
if model is self:
|
||||
wrapper.model_load(target_device)
|
||||
print(f"✅ Reloaded via wrapper - ComfyUI tracking in sync")
|
||||
wrapper_found = True
|
||||
break
|
||||
# Check through SimpleModelWrapper if present
|
||||
elif hasattr(model, 'model') and model.model is self:
|
||||
wrapper.model_load(target_device)
|
||||
print(f"✅ Reloaded via wrapper (unwrapped)")
|
||||
wrapper_found = True
|
||||
break
|
||||
|
||||
if not wrapper_found:
|
||||
# Fallback: direct .to() and check if already registered
|
||||
self.to(target_device)
|
||||
|
||||
# Check ComfyUI's current_loaded_models for existing wrapper
|
||||
try:
|
||||
import comfy.model_management as model_management
|
||||
|
||||
already_registered = False
|
||||
if hasattr(model_management, 'current_loaded_models'):
|
||||
for wrapper in model_management.current_loaded_models:
|
||||
if hasattr(wrapper, 'model_info') and wrapper.model_info.engine == "[engine_name]":
|
||||
wrapper.model_load(target_device)
|
||||
print(f"✅ Reloaded via existing ComfyUI wrapper")
|
||||
already_registered = True
|
||||
break
|
||||
|
||||
if not already_registered:
|
||||
# Re-register with ComfyUI
|
||||
from utils.models.comfyui_model_wrapper.base_wrapper import ComfyUIModelWrapper, ModelInfo, SimpleModelWrapper
|
||||
|
||||
model_size = ComfyUIModelWrapper.calculate_model_memory(self)
|
||||
wrapped_model = SimpleModelWrapper(self)
|
||||
|
||||
model_info = ModelInfo(
|
||||
model=wrapped_model,
|
||||
model_type="tts",
|
||||
engine="[engine_name]",
|
||||
device=target_device,
|
||||
memory_size=model_size,
|
||||
load_device=target_device
|
||||
)
|
||||
new_wrapper = ComfyUIModelWrapper(wrapped_model, model_info)
|
||||
model_management.current_loaded_models.append(new_wrapper)
|
||||
print(f"✅ Re-registered with ComfyUI model management")
|
||||
except Exception as e:
|
||||
print(f"⚠️ Re-registration failed: {e}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"⚠️ Wrapper reload failed ({e}), using direct .to()")
|
||||
self.to(target_device)
|
||||
|
||||
except StopIteration:
|
||||
pass
|
||||
|
||||
# Strategy 2: For models with nested components
|
||||
elif hasattr(self._model, 'semantic_model') and hasattr(self._model.semantic_model, 'parameters'):
|
||||
try:
|
||||
first_param = next(self._model.semantic_model.parameters())
|
||||
current_device = str(first_param.device)
|
||||
# ... same reload logic as above
|
||||
except StopIteration:
|
||||
pass
|
||||
|
||||
# Continue with normal generation
|
||||
# ... your generation code here
|
||||
```
|
||||
|
||||
**3. Important Device Checking Notes:**
|
||||
|
||||
- **Always check against INTENDED device** (`"cuda"` if available), NOT `self.device` which gets updated to "cpu" after offload
|
||||
- **Search for existing wrapper first** before creating new ones to avoid duplicates
|
||||
- **Use wrapper.model_load()** when possible to keep ComfyUI tracking in sync
|
||||
- **Fallback to direct .to()** if wrapper not found, then re-register
|
||||
- **Check both unified interface cache AND ComfyUI's current_loaded_models** for wrappers
|
||||
|
||||
**4. Key Implementation Notes:**
|
||||
|
||||
- **Always check against INTENDED device** (`"cuda"` if available), NOT `self.device`
|
||||
- **Search for existing wrapper first** to avoid duplicates
|
||||
- **Use wrapper.model_load()** when possible to keep ComfyUI tracking in sync
|
||||
- **Check both unified interface cache AND current_loaded_models**
|
||||
|
||||
#### Step 2: Create Model Downloader
|
||||
|
||||
**File:** `engines/[ENGINE_NAME]/[engine_name]_downloader.py`
|
||||
|
||||
**Follow Unified Download Pattern:**
|
||||
|
||||
```python
|
||||
from utils.downloads.unified_downloader import UnifiedDownloader
|
||||
```
|
||||
|
||||
#### Step 3: Create Engine Adapter
|
||||
|
||||
**File:** `engines/adapters/[engine_name]_adapter.py`
|
||||
|
||||
#### Step 4: Create Engine Configuration Node
|
||||
|
||||
**File:** `nodes/engines/[engine_name]_engine_node.py`
|
||||
|
||||
**Model dropdown rule:** Always keep canonical/downloadable model choices visible and add detected installations as separate `local:ModelName` choices; selecting a canonical choice should reuse its organized local installation when available, not replace or hide either choice.
|
||||
|
||||
### Phase 2: Unified Systems Integration
|
||||
|
||||
#### Step 5: Integrate with Unified Model Loading
|
||||
|
||||
**Use ComfyUI Model Management:**
|
||||
|
||||
```python
|
||||
from utils.models.unified_model_interface import UnifiedModelInterface
|
||||
from utils.models.comfyui_model_wrapper import ComfyUIModelWrapper
|
||||
```
|
||||
|
||||
#### Step 6: Implement Caching System
|
||||
|
||||
**Cache Integration:**
|
||||
|
||||
```python
|
||||
from utils.audio.cache import UnifiedCacheManager
|
||||
from utils.audio.audio_hash import create_content_hash
|
||||
```
|
||||
|
||||
#### Step 7: Character Switching Integration
|
||||
|
||||
**Use Unified Character System:**
|
||||
|
||||
```python
|
||||
from utils.text.character_parser import CharacterParser
|
||||
from utils.voice.discovery import get_character_mapping
|
||||
```
|
||||
|
||||
#### Step 8: Language Switching Integration
|
||||
|
||||
**Use Unified Language System:**
|
||||
|
||||
```python
|
||||
from utils.models.language_mapper import LanguageMapper
|
||||
|
||||
class [EngineClass]Processor:
|
||||
def __init__(self):
|
||||
self.language_mapper = LanguageMapper()
|
||||
|
||||
def generate_with_language(self, text, language, **params):
|
||||
# Map language code
|
||||
engine_language = self.language_mapper.map_language(
|
||||
language_code=language,
|
||||
engine_type="[engine_name]"
|
||||
)
|
||||
|
||||
# Apply language-specific model loading if needed
|
||||
if self.supports_multiple_languages:
|
||||
self.load_language_model(engine_language)
|
||||
|
||||
# Generate with language parameter
|
||||
return self.engine.generate(text, language=engine_language, **params)
|
||||
```
|
||||
|
||||
#### Step 9: Pause Tag Integration
|
||||
|
||||
**Use Unified Pause System, search for the unified code**
|
||||
|
||||
#### Step 9.5: Implement Interrupt Handling (CRITICAL)
|
||||
|
||||
**⚠️ REQUIRED FOR ALL NEW ENGINES - Users Must Be Able to Cancel Operations**
|
||||
|
||||
ComfyUI provides an interrupt mechanism that allows users to stop long-running operations (like SRT generation with many segments). Your processors MUST check for interruption signals.
|
||||
|
||||
**Import and Check Pattern:**
|
||||
|
||||
```python
|
||||
import comfy.model_management as model_management
|
||||
|
||||
# In your main processing loops:
|
||||
def generate_srt_speech(self, srt_content: str, ...):
|
||||
# ... setup code ...
|
||||
|
||||
for i, segment in enumerate(srt_segments):
|
||||
# Check for interruption before processing each segment
|
||||
if model_management.interrupt_processing:
|
||||
raise InterruptedError(f"Engine SRT segment {i+1}/{len(srt_segments)} interrupted by user")
|
||||
|
||||
# Your generation code...
|
||||
```
|
||||
|
||||
**Where to Add Interrupt Checks:**
|
||||
|
||||
1. **SRT Processors** - Add checks in main subtitle/segment loops:
|
||||
|
||||
- Before processing each subtitle/segment
|
||||
- Before processing character segments within a subtitle
|
||||
- Before timing assembly (after all audio generation)
|
||||
|
||||
2. **Text Processors** - Add checks in main generation loops:
|
||||
|
||||
- Before character setup/loading
|
||||
- Before processing each character segment
|
||||
- Before processing chunks (if chunking is enabled)
|
||||
|
||||
3. **Special Feature Processors** - Add checks in long-running operations:
|
||||
|
||||
- Before each major processing step
|
||||
- At natural breakpoints (before/after audio generation)
|
||||
|
||||
**Example SRT Processor Structure:**
|
||||
|
||||
```python
|
||||
class EngineSRTProcessor:
|
||||
def generate_srt_speech(self, srt_content, ...):
|
||||
# Pre-generation check
|
||||
if model_management.interrupt_processing:
|
||||
raise InterruptedError("Setup interrupted")
|
||||
|
||||
# Main segment loop
|
||||
for i, segment in enumerate(srt_segments):
|
||||
if model_management.interrupt_processing:
|
||||
raise InterruptedError(f"Segment {i+1}/{len(srt_segments)} interrupted")
|
||||
|
||||
# Character switching support
|
||||
for char_segment in char_segments:
|
||||
if model_management.interrupt_processing:
|
||||
raise InterruptedError(f"Character {char} interrupted")
|
||||
# ... generate audio ...
|
||||
|
||||
# Pre-assembly check
|
||||
if model_management.interrupt_processing:
|
||||
raise InterruptedError("Assembly interrupted")
|
||||
|
||||
# Timing assembly...
|
||||
```
|
||||
|
||||
**Real Examples:**
|
||||
|
||||
- See `nodes/higgs_audio/higgs_audio_srt_processor.py` - SRT interrupts
|
||||
- See `nodes/vibevoice/vibevoice_processor.py` - TTS processor interrupts
|
||||
- See `nodes/chatterbox_official_23lang/chatterbox_official_23lang_srt_processor.py` - Complex interrupts
|
||||
|
||||
#### Step 10: Create Main TTS Processor
|
||||
|
||||
**File:** `nodes/[engine_name]/[engine_name]_processor.py`
|
||||
|
||||
#### Step 11: Test TTS Text Implementation
|
||||
|
||||
BUT rememeber to register all nodes so they load and work on comfyui.
|
||||
|
||||
Also to test, requirements and dependencies need to be added.
|
||||
|
||||
**Ask suer to Test Checklist:**
|
||||
|
||||
- [ ] Basic text generation works
|
||||
- [ ] Character switching works with `[CharacterName] text`
|
||||
- [ ] Language switching works (if applicable)
|
||||
- [ ] Pause tags work with `[pause:1.5s]`
|
||||
- [ ] Caching works (same input = cached output)
|
||||
- [ ] Model auto-download works
|
||||
- [ ] VRAM management works (model unloads)
|
||||
- [ ] Different parameter combinations work
|
||||
- [ ] Engine prints a standard `Settings:` summary with the active generation/load parameters
|
||||
- [ ] **Interrupt handling works** - User can stop SRT generation and it stops within ~1 segment
|
||||
|
||||
### Phase 4: SRT Implementation
|
||||
|
||||
#### Step 12: Analyze SRT Strategies
|
||||
|
||||
**Study Existing SRT Implementations:**
|
||||
|
||||
- **ChatterBox**: Sequential processing with language grouping
|
||||
- **F5-TTS**: Language grouping with chunking
|
||||
- **Higgs Audio**: Character-based processing
|
||||
- VibeVoice?
|
||||
|
||||
**Choose Strategy:**
|
||||
|
||||
1. **Sequential**: Process each subtitle line individually
|
||||
2. **Language Grouping**: Group by language, then process
|
||||
3. **Character Grouping**: Group by character, then batch process
|
||||
4. **Hybrid**: Combine multiple strategies
|
||||
|
||||
#### Step 13: Create SRT Processor
|
||||
|
||||
**File:** `nodes/[engine_name]/[engine_name]_srt_processor.py`
|
||||
|
||||
### Phase 5: Special Features Implementation
|
||||
|
||||
#### Step 14: Identify Special Features
|
||||
|
||||
**Common Special Features:**
|
||||
|
||||
- **Speech Editing** (F5-TTS): Edit specific words in audio
|
||||
- **Voice Conversion** - speech2speech (ChatterBox): Convert voice characteristics
|
||||
- **Multi-Speaker** (Some engines): Multiple speakers in one generation
|
||||
- **Style Control**: Emotion, speaking rate, emphasis
|
||||
|
||||
#### Step 15: Implement Special Features
|
||||
|
||||
Create Dedicated Nodes
|
||||
|
||||
---
|
||||
|
||||
## Unified Systems Integration
|
||||
|
||||
### Character Voice System
|
||||
|
||||
**Files to Study:**
|
||||
|
||||
- `utils/voice/discovery.py` - Voice file discovery
|
||||
- `utils/text/character_parser.py` - Character tag parsing
|
||||
- `nodes/shared/character_voices_node.py` - Character voice management
|
||||
|
||||
### Language System
|
||||
|
||||
**Files to Study:**
|
||||
|
||||
- `utils/models/language_mapper.py` - Language code mapping
|
||||
- Engine-specific language model files
|
||||
|
||||
### Pause Tag System
|
||||
|
||||
**Files to Study:**
|
||||
|
||||
- `utils/text/pause_processor.py` - Pause tag parsing and generation
|
||||
|
||||
**Integration Pattern:**
|
||||
|
||||
### Model Management
|
||||
|
||||
**Files to Study:**
|
||||
|
||||
- `utils.models/unified_model_interface.py` - Unified model loading
|
||||
- `utils/models/comfyui_model_wrapper.py` - ComfyUI integration
|
||||
|
||||
---
|
||||
|
||||
## Documentation Updates
|
||||
|
||||
### README.md Updates
|
||||
|
||||
#### 1. Features Section
|
||||
|
||||
Add engine to the features list with its unique capabilities.
|
||||
|
||||
#### 2. What's New Section
|
||||
|
||||
Add changelog entry for the new engine.
|
||||
|
||||
#### 3. Model Download Section
|
||||
|
||||
Add download instructions and model requirements.
|
||||
|
||||
#### 4. Supported Engines Table
|
||||
|
||||
Update the engines comparison table.
|
||||
|
||||
### Example README Addition:
|
||||
|
||||
```markdown
|
||||
## What's New in v4.X.X
|
||||
|
||||
### 🚀 New [Engine Name] TTS Engine
|
||||
- High-quality text-to-speech with [unique feature]
|
||||
- Support for [languages/voices/special capabilities]
|
||||
- Integrated with unified interface (TTS Text, SRT, Voice Changer)
|
||||
- Auto-download models with one click
|
||||
|
||||
## Features
|
||||
|
||||
### 🎤 [Engine Name] TTS Engine
|
||||
- **[Unique Feature 1]**: Description of what makes this engine special
|
||||
- **[Unique Feature 2]**: Another special capability
|
||||
- **Multi-language support**: List of supported languages (if applicable)
|
||||
- **Voice cloning**: Description of voice capabilities (if applicable)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Implementation Phase Strategy
|
||||
|
||||
### Phase 1: Foundation (Implement First)
|
||||
|
||||
1. Core engine implementation
|
||||
2. Basic text generation (With character switching,
|
||||
- Language switching (if applicable)
|
||||
- Pause tag support
|
||||
- Caching system
|
||||
- VRAM management)
|
||||
3. Model loading and downloading
|
||||
4. Engine configuration node
|
||||
5. Integration with TTS Text node
|
||||
|
||||
**Stop here and test with user before proceeding**
|
||||
|
||||
### Phase 2: SRT Support
|
||||
|
||||
1. SRT processor implementation
|
||||
2. Timing and assembly integration
|
||||
3. Character switching in SRT
|
||||
4. Performance optimization
|
||||
|
||||
### Phase 3: Special Features
|
||||
|
||||
1. Engine-specific unique features
|
||||
2. Special nodes for unique capabilities
|
||||
3. Advanced parameter controls
|
||||
|
||||
### Phase 4: Documentation and Polish
|
||||
|
||||
1. README updates
|
||||
2. Example workflows
|
||||
3. Performance testing
|
||||
4. Error handling improvements
|
||||
|
||||
---
|
||||
|
||||
## Critical Integration Points
|
||||
|
||||
### Must Use These Unified Systems
|
||||
|
||||
#### ✅ Required Integrations
|
||||
|
||||
- **UnifiedModelInterface** - Model loading
|
||||
- **ComfyUIModelWrapper** - VRAM management
|
||||
- **UnifiedCacheManager** - Audio caching
|
||||
- **CharacterParser** - Character tag parsing
|
||||
- **PauseTagProcessor** - Pause tag handling
|
||||
- **LanguageMapper** - Language code mapping
|
||||
- **UnifiedDownloader** - Model downloads
|
||||
- **model_management.interrupt_processing** - Interrupt signal checking (CRITICAL for cancellation)
|
||||
|
||||
#### ❌ Never Duplicate These
|
||||
|
||||
- Character parsing logic
|
||||
- Pause tag parsing logic
|
||||
- Language mapping logic
|
||||
- Cache key generation
|
||||
- Model management logic
|
||||
- Audio format conversion utilities
|
||||
|
||||
---
|
||||
@@ -1,79 +0,0 @@
|
||||
# So You Want To Add A New Engine?
|
||||
|
||||
This folder is the start-here guide for adding a new engine to TTS Audio Suite with help from an LLM.
|
||||
|
||||
The goal is simple: do not let the LLM jump straight into coding. A new engine is not ready just because it can generate one audio clip. It must fit the suite architecture, model loading, cache, tags, SRT timing, docs, and ComfyUI lifecycle.
|
||||
|
||||
## Start Here
|
||||
|
||||
If you want support for a new TTS engine, Voice Changer, ASR, or special audio model, follow these guides in order.
|
||||
|
||||
For this suite, TTS support means both **Unified TTS Text** and **Unified SRT TTS**. SRT is our own text-to-subtitle generation flow around a TTS engine, not an optional native model feature.
|
||||
|
||||
1. [First Ask The LLM To Research The Official Model](01_FIRST_ASK_THE_LLM_TO_RESEARCH_THE_OFFICIAL_MODEL.md)
|
||||
2. [Check Existing ComfyUI Implementations](02_CHECK_EXISTING_COMFYUI_IMPLEMENTATIONS.md)
|
||||
3. [Decide Engine Scope](03_DECIDE_ENGINE_SCOPE.md)
|
||||
4. [Implementation Order For LLM](04_IMPLEMENTATION_ORDER_FOR_LLM.md)
|
||||
5. [Required Parity Checklist](05_REQUIRED_PARITY_CHECKLIST.md)
|
||||
6. [User Prompts To Copy Paste](06_USER_PROMPTS_TO_COPY_PASTE.md)
|
||||
7. [PR Review Checklist](07_PR_REVIEW_CHECKLIST.md)
|
||||
|
||||
Keep these technical references nearby:
|
||||
|
||||
- [Deep technical implementation guide](NEW_ENGINE_IMPLEMENTATION_GUIDE.md)
|
||||
- [Known traps and failures](fails_to_avoid_TTS_Engine_Implementation.md)
|
||||
|
||||
## The Rule
|
||||
|
||||
The official model implementation is the source of truth.
|
||||
|
||||
Existing ComfyUI implementations are useful references, but they are not authorities. They may add fake or artificial features that the official model does not support. Do not copy those by default.
|
||||
|
||||
Example: if another implementation adds a `speed` parameter by stretching audio after generation, that is not a native model feature. It may still be useful, but it must be discussed with the maintainer first in an issue or PR.
|
||||
|
||||
## What To Tell Your LLM First
|
||||
|
||||
Copy this before asking for implementation:
|
||||
|
||||
```text
|
||||
Do not write code yet.
|
||||
|
||||
I want to add a new engine to TTS Audio Suite. First, read these files:
|
||||
|
||||
- PROJECT_INDEX.md
|
||||
- docs/New Engines Guides/README.md
|
||||
- docs/New Engines Guides/01_FIRST_ASK_THE_LLM_TO_RESEARCH_THE_OFFICIAL_MODEL.md
|
||||
- docs/New Engines Guides/02_CHECK_EXISTING_COMFYUI_IMPLEMENTATIONS.md
|
||||
- docs/New Engines Guides/03_DECIDE_ENGINE_SCOPE.md
|
||||
- docs/New Engines Guides/NEW_ENGINE_IMPLEMENTATION_GUIDE.md
|
||||
- docs/New Engines Guides/fails_to_avoid_TTS_Engine_Implementation.md
|
||||
|
||||
Then research the official model implementation and produce a capability report. Do not start coding until the report explains the native model features, required nodes, model files, sample rate, dependencies, license, and supported parameters.
|
||||
```
|
||||
|
||||
## Reference Engines
|
||||
|
||||
Use these references after the official model research is done:
|
||||
|
||||
- **Qwen3-TTS**: primary modern reference for full TTS/SRT architecture and suite parity.
|
||||
- **Step Audio EditX**: secondary reference for wrapper/model lifecycle patterns and special post-processing cases.
|
||||
|
||||
Do not copy either blindly. The new engine must be scoped from the official implementation first.
|
||||
|
||||
## Definition Of Done
|
||||
|
||||
A new engine is not done until the answer to these questions is yes:
|
||||
|
||||
- Does it use the suite architecture instead of stuffing engine logic into unified nodes?
|
||||
- Does it load models through the unified model interface?
|
||||
- Does model unload/reload/Clear VRAM work?
|
||||
- Did someone explicitly decide whether this engine belongs in the Main Environment or needs runtime isolation?
|
||||
- Does it support generated audio cache where applicable?
|
||||
- Does it handle ComfyUI audio shape and sample rate correctly?
|
||||
- Does it wire character tags, pause tags, and parameter switching where applicable?
|
||||
- If it is a TTS engine, does its SRT path support interrupt handling and correct timing modes?
|
||||
- Does the UI avoid fake unsupported controls?
|
||||
- Are docs and engine metadata updated?
|
||||
- Did someone test it manually in ComfyUI?
|
||||
|
||||
If the answer is no, the PR is not ready.
|
||||
@@ -1,152 +0,0 @@
|
||||
# TTS Engine Implementation - Fails & Lessons Learned
|
||||
|
||||
## Step Audio EditX Implementation
|
||||
|
||||
### Import Errors
|
||||
- **Missing import**: Always add `import folder_paths` when using `folder_paths.get_temp_directory()` in node files
|
||||
- **Bundled code imports**: For complex bundled packages with internal cross-imports, add `sys.path.insert(0, impl_dir)` at top of main files instead of converting all imports
|
||||
|
||||
### Audio Utility Functions
|
||||
- **Temp file creation**: Use `AudioProcessingUtils.save_audio_to_temp_file()` not `save_audio()` (doesn't exist)
|
||||
- **ComfyUI audio format**: Extract waveform with `audio_tensor['waveform']` and `audio_tensor.get('sample_rate')` before passing to utilities
|
||||
|
||||
### Download Configuration
|
||||
- **HuggingFace file list**: Always verify actual repo structure with `curl` before hardcoding file lists
|
||||
- **Downloader API**: Use dict format `[{"remote": f, "local": f}]` for unified_downloader, not plain strings
|
||||
- **File existence**: Check actual HuggingFace repo structure - don't assume file naming patterns (e.g., `model-00001.safetensors` not `model-00001-of-00002.safetensors`)
|
||||
- **CRITICAL - No cache downloads**: NEVER allow models to auto-download to cache directories. ALL models/weights must be downloaded via our downloader to organized `models/TTS/` folders. Disable auto-download and add to downloader instead.
|
||||
|
||||
### Factory Registration
|
||||
- **Unified model interface**: Always implement `register_<engine>_factory()` when using `unified_model_interface.load_model()`
|
||||
- **Factory initialization**: Add factory to `initialize_all_factories()` or it won't be registered
|
||||
|
||||
### Node Registration
|
||||
- **Engine branch**: Add engine-specific branch in `_create_engine_node_instance()` for each new engine
|
||||
- **Variable naming**: Use consistent variable names (`config` not `engine_config`) throughout node code
|
||||
|
||||
### Voice References
|
||||
- **Reference text requirement**: F5-TTS and Step Audio EditX REQUIRE `prompt_text` (transcript), ChatterBox/VibeVoice/Higgs don't
|
||||
- **Narrator voice mapping**: Map narrator from TTS Text node input by saving `audio_tensor` to temp file with `reference_text`
|
||||
- **Voice discovery**: Use `get_character_mapping()` not `discover_voices_for_engine()` (latter doesn't exist)
|
||||
|
||||
### Pause Tag Format
|
||||
- **Correct format**: Use `[pause:2]`, `[pause:1.5s]`, `[pause:500ms]` - NOT `<pause_X>`
|
||||
- **Parser usage**: Use `PauseTagProcessor.parse_pause_tags()` which returns `('text', content)` or `('pause', duration_seconds)` tuples
|
||||
|
||||
### Seed Control
|
||||
- **Global torch state**: Some engines (Step Audio EditX) use global `torch.manual_seed()` for reproducibility, not function parameters
|
||||
- **CUDA seeds**: Always set both `torch.manual_seed()` and `torch.cuda.manual_seed_all()` for GPU reproducibility
|
||||
|
||||
### Quantization Support
|
||||
- **Device movement**: Bitsandbytes quantized models (int4/int8) CANNOT be moved with `.to()` - wrap in try-except
|
||||
- **VRAM clearing**: Quantized models stay on GPU; "Clear VRAM" must unload completely (not move to CPU)
|
||||
- **Error handling**: Catch `ValueError` with "is not supported for" and "8-bit"/"4-bit" in message
|
||||
|
||||
### Character Switching Implementation
|
||||
- **Missing voice fallback**: When `get_character_mapping()` returns `(None, None)` for a character, MUST still add entry to `voice_mapping` with fallback to narrator/default voice
|
||||
- **Voice mapping consistency**: All characters in parsed segments MUST have entries in `voice_mapping`, even if empty/None
|
||||
- **Adapter validation**: Adapters that require voice references (Step Audio EditX, F5-TTS) need graceful fallback when voice_ref is None
|
||||
- **Character parser interaction**: Character parser changes unknown characters to "narrator" - need to ensure original character names are preserved for voice mapping lookup
|
||||
- **Parser fix**: Add text tag characters to `available_characters` (lowercase) + set language defaults like IndexTTS
|
||||
- **Working pattern**: `all_available = set(get_available_characters())` + aliases + text tag chars (lowercase) + "narrator" → `set_available_characters(list(all_available))` + `set_character_language_default()` for each char
|
||||
|
||||
### Caching Implementation
|
||||
- **CRITICAL - Planned but skipped**: Caching listed in Phase 1 but often omitted during processor/adapter implementation. Verify with testing (run generation twice on same text - should show cache hit on second)
|
||||
- **Required in adapters**: Generate stable audio hash, create cache key with all params, check cache BEFORE generation, store after with duration calc
|
||||
- **Cache key pattern**: `audio_cache.generate_cache_key(engine_type, text=..., audio_component=..., **all_params)`
|
||||
- **Duration calculation**: Update `_calculate_duration()` with engine sample rate (e.g., 24000 for Step Audio EditX, F5-TTS)
|
||||
|
||||
### Engine Settings Logging
|
||||
- **Missing standard print**: New engines should print the usual `Settings:` summary once per run with the active generation/load parameters so validation can confirm what actually executed
|
||||
- **Resolved prompt preview**: Reuse `utils.voice.character_logging.format_resolved_character_block()` for boxed text previews so logs show the voice that will actually speak, not only the parser alias
|
||||
|
||||
### Model Lifecycle - __del__ Destructor
|
||||
- **CRITICAL**: Remove `__del__` from engine classes - causes automatic unload after generation ends (when object goes out of scope)
|
||||
- **Pattern**: F5-TTS and ChatterBox don't have `__del__`, only IndexTTS and StepAudio did (wrong)
|
||||
- **Correct behavior**: Models stay in VRAM for reuse; only unload on explicit Clear VRAM button, engine switch, or parameter change
|
||||
|
||||
### Clear VRAM - Runtime Teardown vs CPU Offload
|
||||
- **Wrong**: Assuming every engine is safe to unload with only `.to('cpu')`
|
||||
- **Symptom**: Clear VRAM frees only part of VRAM; most memory disappears only after closing ComfyUI
|
||||
- **Cause**: Runtime-heavy engines can keep compiled graphs, prompt caches, workspaces, or runtime objects alive after CPU offload
|
||||
- **Correct**: Add an engine-specific unload handler that clears internal caches, drops runtime/model references, then runs `gc.collect()`, `torch.cuda.empty_cache()`, and `torch.cuda.ipc_collect()` when available
|
||||
- **Prevention**: If the engine uses warmup, `torch.compile`, streaming caches, or a custom runtime object, explicitly test Clear VRAM and reload instead of assuming generic offload parity
|
||||
|
||||
### Generation Info Reports - Modular Pattern
|
||||
- **CRITICAL - Consistency required**: ALL engines must use `ChunkCombiner` + `ChunkTimingHelper` for generation reports
|
||||
- **Correct pattern** (from ChatterBox 23-Lang, F5-TTS):
|
||||
1. `combine_audio_segments()` method accepts `return_info=True` parameter
|
||||
2. Uses `ChunkCombiner.combine_chunks()` with `return_info=True` to get `(audio, chunk_info)` tuple
|
||||
3. In unified node: call `ChunkTimingHelper.enhance_generation_info(base_info, chunk_info)` for detailed timing
|
||||
- **Wrong**: Inline concatenation (`torch.cat`) without timing tracking, manual report formatting
|
||||
- **Example**: IndexTTS/StepAudio initially did inline concat - needed refactor to use ChunkCombiner
|
||||
- **Required info**: Duration, text length, segments/chunks count, chunk timing breakdown (start/end/text per chunk)
|
||||
|
||||
### Segment Parameter Registration
|
||||
- **Forgot to add**: New engines to `PARAMETER_ENGINES` in `utils/text/segment_parameters.py` for `[seed:X]` tag support
|
||||
|
||||
### Language UI Normalization
|
||||
- **Wrong**: Exposing raw backend language codes in the engine node just because the official runtime accepts them
|
||||
- **Symptom**: New engine UI looks inconsistent with the suite, logs show backend codes instead of readable names, and character-tag / alias languages do not map cleanly to node config values
|
||||
- **Correct**: Keep user-facing language controls normalized to honest display names when the suite already does that, then map them back to the official backend codes in the adapter/runtime layer
|
||||
- **Prevention**: Reuse or add a small engine-local language helper so node UI, processor logs, segment language overrides, and adapter normalization all agree on the same mapping
|
||||
|
||||
### SRT Processor Import Pattern
|
||||
- **Wrong**: Using `from nodes.xxx import` - fails with `'nodes' is not a package`
|
||||
- **Correct**: Use `importlib.util.spec_from_file_location()` for cross-module imports in dynamically loaded processors
|
||||
|
||||
### SRT Shared API Guessing
|
||||
- **Wrong**: Inventing shared helper names like `AudioAssemblyEngine.assemble(...)`, `TimingReportGenerator`, or `AdjustedSRTGenerator`
|
||||
- **Correct**: Open a working SRT processor first and match the exact API that already exists in `utils/timing/assembly.py`, `utils/timing/engine.py`, and `utils/timing/reporting.py`
|
||||
- **Real pattern**:
|
||||
- `stretch_to_fit` → `TimedAudioAssembler.assemble_timed_audio(...)`
|
||||
- `pad_with_silence` → `AudioAssemblyEngine(...).assemble_with_overlaps(...)`
|
||||
- `concatenate` → `TimingEngine.calculate_concatenation_adjustments(...)` + `assemble_concatenation(...)`
|
||||
- `smart_natural` → `TimingEngine.calculate_smart_timing_adjustments(...)` + `assemble_smart_natural(...)`
|
||||
- reporting / adjusted SRT → `SRTReportGenerator().generate_timing_report(...)` and `generate_adjusted_srt_string(...)`
|
||||
- **Prevention**: Never assume shared helper names. Grep existing SRT processors and copy the real call pattern exactly.
|
||||
|
||||
### ComfyUI Audio Dict Format
|
||||
- **Wrong**: Passing `audio_tensor` directly to functions expecting raw tensor
|
||||
- **Correct**: Extract waveform with `audio_tensor.get('waveform')` - ComfyUI audio is `{"waveform": tensor, "sample_rate": int}`
|
||||
- **Also**: Check `AudioProcessingUtils.save_audio_to_temp_file(audio, sample_rate)` signature - it returns path, doesn't take path as arg
|
||||
|
||||
### ComfyUI Audio Tensor Shape
|
||||
- **CRITICAL**: ComfyUI expects **3D** `[batch, channels, samples]` NOT 2D `[channels, samples]`
|
||||
- **Error symptom**: `IndexError: Dimension out of range (expected to be in range of [-1, 0], but got 1)` in `save_audio()`
|
||||
- **Correct shape conversion**:
|
||||
- 1D `[samples]` → 3D `[1, 1, samples]` (two unsqueeze)
|
||||
- 2D `[channels, samples]` → 3D `[1, channels, samples]` (one unsqueeze)
|
||||
- **Wrong**: Squeezing 3D to 2D, or only adding one dimension to 1D
|
||||
- **Pattern**: `tensor.unsqueeze(0)` adds batch dim to 2D; check VibeVoice processor for reference
|
||||
|
||||
### Step Audio EditX Sample Rate
|
||||
- **Symptom**: Pitch shift in edit/clone modes with external audio (44100 Hz, 48000 Hz)
|
||||
- **Cause**: Saving audio with wrong sample rate metadata → `torchaudio.load()` misinterprets data
|
||||
- **Fix**: Resample to 24000 Hz before saving temp files (edit node, TTS text node, TTS SRT node)
|
||||
- **Prevention**: Always resample to engine's native rate (24000 Hz for Step Audio EditX)
|
||||
|
||||
### Unified Model Interface - Adapter & Node Consistency
|
||||
- **CRITICAL**: ALL engine loading must use `unified_model_interface.load_model()` for smart caching
|
||||
- **Symptom**: Model loads twice (once for TTS, once for editing/other nodes) - wasting VRAM and time
|
||||
- **Wrong pattern**: Creating engine instances directly (`engine = StepAudioEditXEngine()`) in adapters or nodes
|
||||
- **Correct pattern**:
|
||||
1. **Adapter** `load_base_model()` uses `unified_model_interface.load_model(config)`
|
||||
2. **Audio Editor node** uses `unified_model_interface.load_model(config)`
|
||||
3. **Both use same `model_type`** (e.g., "tts") so cache keys match → engine reused
|
||||
- **Cache key components**: `engine_name`, `model_type`, `model_name`, `device`, `quantization`, etc.
|
||||
- **Fix Step Audio EditX**: Updated adapter and Audio Editor node to use unified interface with `model_type="tts"`
|
||||
- **Prevention**: When adding new engines, ALWAYS use unified interface in ALL code paths (adapters, nodes, processors)
|
||||
|
||||
### Engine Wrapper Implementation - Always Compare First
|
||||
- **CRITICAL**: Before writing engine wrapper, open existing reference engine (Step Audio EditX, F5-TTS) side-by-side and compare patterns
|
||||
- **Common mistakes when NOT comparing**:
|
||||
- Missing required imports (`folder_paths`, `resolve_torch_device`, `unified_model_interface`, `extra_paths`)
|
||||
- Manual device/dtype resolution instead of using utils functions
|
||||
- Missing `_find_model_directory()` with `extra_model_paths` support
|
||||
- Using `load_model()` method instead of `_ensure_model_loaded()` pattern
|
||||
- Direct model instantiation instead of `unified_model_interface.load_model()`
|
||||
- Missing `_model_config` tracking (needed for unload)
|
||||
- Not using `tts_model_manager.ensure_device()` before fallback to `.to()`
|
||||
- Missing GPU capability detection for dtype auto-selection
|
||||
- **Prevention**: ALWAYS open Step Audio EditX engine file as reference when creating new engine wrapper, copy the patterns
|
||||
@@ -1,73 +0,0 @@
|
||||
# OmniVoice Native Tags Guide
|
||||
|
||||
OmniVoice has its own official inline square-bracket control tokens upstream, but **this suite does not expose `[]` for OmniVoice tags**.
|
||||
|
||||
Inside TTS Audio Suite, you can write the suite-default angle-tag aliases in text:
|
||||
|
||||
```text
|
||||
<laughter>
|
||||
<sigh>
|
||||
<question-ei>
|
||||
```
|
||||
|
||||
The OmniVoice processor converts those aliases internally to the official OmniVoice native form before generation:
|
||||
|
||||
```text
|
||||
[laughter]
|
||||
[sigh]
|
||||
[question-ei]
|
||||
```
|
||||
|
||||
Do **not** type OmniVoice non-verbal tags in `[]` form in suite text. In this suite, `[]` belongs to character, language, parameter, and pause syntax.
|
||||
|
||||
## Supported Non-Verbal Tags
|
||||
|
||||
Official OmniVoice non-verbal meanings exposed by this suite through `<>` aliases:
|
||||
|
||||
```text
|
||||
laughter
|
||||
sigh
|
||||
confirmation-en
|
||||
question-en
|
||||
question-ah
|
||||
question-oh
|
||||
question-ei
|
||||
question-yi
|
||||
surprise-ah
|
||||
surprise-oh
|
||||
surprise-wa
|
||||
surprise-yo
|
||||
dissatisfaction-hnn
|
||||
```
|
||||
|
||||
Examples:
|
||||
|
||||
```text
|
||||
[Alice] <laughter> You really got me there.
|
||||
[Bob] <sigh> Fine, let's try again.
|
||||
[Narrator] <question-ei> Really?
|
||||
[Narrator] <surprise-oh> I didn't expect that.
|
||||
```
|
||||
|
||||
## Important Behavior
|
||||
|
||||
- OmniVoice uses native generation tags here, not Step Audio EditX inline post-processing.
|
||||
- User-facing suite syntax stays in `<>` form for OmniVoice non-verbal tags.
|
||||
- `[]` is reserved for suite structural syntax like `[Alice]`, `[en:Alice]`, `[pause:1s]`, and parameter switching.
|
||||
- Do not rely on Step-style `<Laughter:2>` or `<emotion:happy>` semantics in the OmniVoice text path.
|
||||
- If you want Step Audio EditX as a second pass on OmniVoice output, use the separate `🎨 Audio Editor` node manually after generation.
|
||||
|
||||
## Multiline Tag Editor
|
||||
|
||||
The `🏷️ Multiline TTS Tag Editor` has a dedicated `OmniVoice` mode in the `Inline Tags` panel.
|
||||
|
||||
- The editor inserts suite-default angle-tag aliases like `<laughter>`
|
||||
- The processor converts them internally to official OmniVoice square tags during generation
|
||||
- The editor does not encourage raw OmniVoice `[]` input because `[]` is suite syntax
|
||||
|
||||
## Sources
|
||||
|
||||
This behavior follows the official OmniVoice documentation for non-verbal symbols and pronunciation control:
|
||||
|
||||
- [OmniVoice GitHub README](https://github.com/k2-fsa/OmniVoice)
|
||||
- [OmniVoice Hugging Face model card](https://huggingface.co/k2-fsa/OmniVoice)
|
||||
@@ -1,396 +0,0 @@
|
||||
# ⚙️ Per-Segment Parameter Switching Guide
|
||||
|
||||
## Overview
|
||||
|
||||
The TTS Audio Suite supports **per-segment parameter control** through bracket tags like `[Alice|seed:42|temp:0.5]`. This allows you to override TTS generation parameters (seed, temperature, cfg, speed, etc.) on a per-segment basis, providing fine-grained control over individual audio segments without modifying node-level defaults.
|
||||
|
||||
Parameters are applied **only to the current segment** and automatically revert to node defaults for subsequent segments.
|
||||
|
||||
---
|
||||
|
||||
## ✨ Key Features
|
||||
|
||||
- **⚙️ Per-Segment Control** - Override parameters for individual text segments or SRT subtitle entries
|
||||
- **🔀 Order-Independent** - Specify parameters in any order: `[Alice|seed:42|temp:0.5]` or `[seed:42|Alice|temp:0.5]`
|
||||
- **📝 Alias Support** - Use shortcuts like `temp` (temperature), `cfg_weight` (cfg), `exag` (exaggeration)
|
||||
- **🔤 Case-Insensitive** - `[SEED:42]`, `[Seed:42]`, `[seed:42]` all work identically
|
||||
- **🚨 Smart Filtering** - Unsupported parameters are silently ignored with warnings (e.g., `[speed:2.0]` on ChatterBox)
|
||||
- **📺 SRT Support** - Apply parameters to individual SRT subtitle lines
|
||||
- **🔙 Backward Compatible** - Works alongside character switching: `[de:Alice|seed:42|temp:0.7]`
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Quick Start
|
||||
|
||||
### Basic Parameter Syntax
|
||||
|
||||
**Format:** `[character|parameter:value]` or `[parameter:value|character]`
|
||||
|
||||
```
|
||||
[Alice|seed:42] Hi there with a specific seed!
|
||||
[Bob|temperature:0.5] Bob speaks more deterministically.
|
||||
[Alice|seed:42|temperature:0.7] Alice with both seed and temperature.
|
||||
[seed:42|Alice] Order doesn't matter!
|
||||
```
|
||||
|
||||
### Real-World Examples
|
||||
|
||||
#### Consistent Character Voice
|
||||
```
|
||||
[Alice|seed:42] This is the first segment of Alice.
|
||||
[Alice|seed:42] This is the second segment, sounding identical.
|
||||
[Alice|seed:42] Third segment with same seed.
|
||||
```
|
||||
|
||||
#### Varied Voice Characteristics
|
||||
```
|
||||
[Alice|temperature:0.3] Alice speaks carefully and precisely.
|
||||
[Alice|temperature:0.8] Alice speaks more creatively and varied.
|
||||
[Alice|temperature:0.3] Alice is precise again.
|
||||
```
|
||||
|
||||
#### SRT Example with Parameters
|
||||
```srt
|
||||
1
|
||||
00:00:01,000 --> 00:00:04,000
|
||||
[Alice|seed:42] Hello! I'm Alice and this line has a specific seed.
|
||||
|
||||
2
|
||||
00:00:04,500 --> 00:00:09,500
|
||||
[Bob|seed:100] And I'm Bob with my own unique seed.
|
||||
|
||||
3
|
||||
00:00:10,000 --> 00:00:14,000
|
||||
[Alice|seed:42] Alice again with the same seed as segment 1.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📋 Supported Parameters
|
||||
|
||||
### Universal Parameters (All Engines)
|
||||
|
||||
| Parameter | Alias | Type | Range | Description |
|
||||
|-----------|-------|------|-------|-------------|
|
||||
| `seed` | — | int | 0-4294967295 | Random seed for reproducible generation |
|
||||
| `temperature` | `temp` | float | 0.1-2.0 | Randomness/creativity (lower=more deterministic) |
|
||||
|
||||
### Engine-Specific Parameters
|
||||
|
||||
#### MOSS-TTS
|
||||
| Parameter | Alias | Type | Range | Description |
|
||||
|-----------|-------|------|-------|-------------|
|
||||
| `top_p` | `topp` | float | 0.0-1.0 | Nucleus sampling probability |
|
||||
| `top_k` | `topk` | int | 1-100 | Top-k sampling |
|
||||
| `audio_temperature` | `audio_temp` | float | 0.1-2.5 | MOSS audio sampling temperature |
|
||||
| `audio_top_p` | — | float | 0.0-1.0 | MOSS audio nucleus sampling |
|
||||
| `audio_top_k` | — | int | 1-200 | MOSS audio top-k sampling |
|
||||
| `repetition_penalty` | `rep_penalty` | float | 0.5-3.0 | MOSS repetition penalty |
|
||||
| `audio_repetition_penalty` | `audio_rep_penalty` | float | 0.5-3.0 | MOSS audio repetition penalty |
|
||||
| `duration_tokens` | `tokens` | int | 0-8192 | MOSS target output length hint in audio tokens |
|
||||
| `max_new_tokens` | — | int | 64-16384 | Hard generation token limit |
|
||||
| `n_vq_for_inference` | `n_vq` | int | 0-32 | Local 1.7B only: RVQ layers/codebooks for inference |
|
||||
| `instruction` | — | string | text | Whole-segment speaking instruction |
|
||||
| `quality` | — | string | text | Whole-segment recording/presentation quality hint |
|
||||
| `sound_event` | — | string | text | Whole-segment sound event hint |
|
||||
| `ambient_sound` | — | string | text | Whole-segment ambient sound hint |
|
||||
|
||||
#### MOSS Sound Effects
|
||||
|
||||
MOSS-SoundEffect v1 uses the applicable MOSS-TTS parameters above. Both sound-effect engines also support a duration override:
|
||||
|
||||
| Parameter | Alias | Engines | Type | Range | Description |
|
||||
|-----------|-------|---------|------|-------|-------------|
|
||||
| `duration_seconds` | `seconds` | v1, v2 | float | 0.5-300 | Duration of the sound segment |
|
||||
| `inference_steps` | `steps` | v2 | int | 1-150 | Diffusion steps |
|
||||
| `cfg` | — | v2 | float | 0.0-20.0 | Prompt guidance strength |
|
||||
| `sigma_shift` | — | v2 | float | 0.0-10.0 | Flow-matching schedule shift |
|
||||
| `negative_prompt` | `negative`, `neg` | v2 | string | text | Sounds or qualities to discourage |
|
||||
|
||||
See the [Sound Effects Guide](SOUND_EFFECTS_GUIDE.md) for pauses, crossfades, long-duration chunking, and complete examples.
|
||||
|
||||
#### ChatterBox & ChatterBox Official 23-Lang
|
||||
| Parameter | Alias | Type | Range | Description |
|
||||
|-----------|-------|------|-------|-------------|
|
||||
| `cfg` | `cfg_weight` | float | 0.0-1.0 | CFG weight for speaker guidance |
|
||||
| `exaggeration` | `exag` | float | 0.0-2.0 | Voice emotion exaggeration |
|
||||
|
||||
#### F5-TTS
|
||||
| Parameter | Alias | Type | Range | Description |
|
||||
|-----------|-------|------|-------|-------------|
|
||||
| `cfg` | — | float | 0.0-20.0 | Classifier-free guidance strength |
|
||||
| `speed` | — | float | 0.5-2.0 | Speech speed multiplier |
|
||||
|
||||
#### Higgs Audio 2
|
||||
| Parameter | Alias | Type | Range | Description |
|
||||
|-----------|-------|------|-------|-------------|
|
||||
| `cfg` | — | float | 0.0-20.0 | CFG strength |
|
||||
| `top_p` | `topp` | float | 0.0-1.0 | Nucleus sampling probability |
|
||||
| `top_k` | `topk` | int | 1-100 | Top-k sampling |
|
||||
|
||||
#### VibeVoice
|
||||
| Parameter | Alias | Type | Range | Description |
|
||||
|-----------|-------|------|-------|-------------|
|
||||
| `cfg` | — | float | 0.0-20.0 | CFG strength |
|
||||
| `top_p` | `topp` | float | 0.0-1.0 | Nucleus sampling probability |
|
||||
| `inference_steps` | `steps` | int | 1-100 | Number of inference steps |
|
||||
|
||||
#### IndexTTS 2 / 2.5
|
||||
| Parameter | Alias | Type | Range | Description |
|
||||
|-----------|-------|------|-------|-------------|
|
||||
| `cfg` | — | float | 0.0-20.0 | CFG strength |
|
||||
| `top_p` | `topp` | float | 0.0-1.0 | Nucleus sampling probability |
|
||||
| `top_k` | `topk` | int | 1-100 | Top-k sampling |
|
||||
| `emotion_alpha` | — | float | 0.0-1.0 | Shared audio/vector/text emotion intensity |
|
||||
| `duration_factor` | `dur_factor` | float | 0.5-2.0 | Official IndexTTS-2.5 internal feature-duration scaling; 0.5 shorter/faster, 2.0 longer/slower |
|
||||
|
||||
`duration_factor` is a 2.5-only upstream parameter. It uses nearest-neighbor scaling inside the semantic length regulator after speech codes are generated. It is not natural prosody planning, exact-seconds targeting, waveform playback-speed control, or an inference-performance control. IndexTTS continues to use the suite's ordinary final timing modes in TTS SRT.
|
||||
|
||||
Switching the engine node between IndexTTS-2 and IndexTTS-2.5 invalidates the cached Text/SRT processor and model identity. `language`, `duration_factor`, and `text_normalization` also participate in the generated-audio cache identity, so changing a supported 2.5 generation parameter cannot return audio produced with the previous setting.
|
||||
|
||||
IndexTTS-2 also supports inline emotion controls. Named unsigned values replace
|
||||
that dimension; explicitly signed values adjust the connected vector:
|
||||
|
||||
```text
|
||||
[sad:0.7|calm:0.2] Absolute values for this segment.
|
||||
[sad:+0.3|calm:-0.2] Adjust the connected vector for this segment.
|
||||
```
|
||||
|
||||
All eight values can be supplied in the official order `happy, angry, sad,
|
||||
afraid, disgusted, melancholic, surprised, calm`:
|
||||
|
||||
```text
|
||||
[vector:0,0,0.7,0,0,0.4,0,0.2] Absolute replacement.
|
||||
[vector:+0,+0,+0.3,+0,+0,+0,+0,-0.2] Relative adjustment.
|
||||
```
|
||||
|
||||
Full relative vectors require an explicit sign on every value. Results are
|
||||
clamped to IndexTTS-2's supported range and revert to the connected vector at
|
||||
the next segment.
|
||||
|
||||
Text emotion can use a saved preset or quoted text. `{seg}` is expanded with
|
||||
the current segment before QwenEmotion analysis:
|
||||
|
||||
```text
|
||||
[emotion:restrained_anger] A saved preset.
|
||||
[emotion:"Quiet grief masking frustration"] A direct description.
|
||||
[emotion:"Infer nervous anticipation from this line: {seg}"] Dynamic analysis.
|
||||
```
|
||||
|
||||
Click a numeric emotion tag in the TTS Tag Editor to open a contextual radar
|
||||
directly beside that tag. The editor also creates and manages text
|
||||
presets in `models/TTS/IndexTTS/emotion_presets.json`.
|
||||
|
||||
IndexTTS-2 has separate engine inputs for these sources: connect vector or text
|
||||
emotion to `emotion_control` and audio emotion references to `emotion_audio`.
|
||||
Both may be connected simultaneously; IndexTTS-2 blends them during emotion
|
||||
conditioning. Inline vector/text controls override the connected vector/text
|
||||
values for their segment, while `[Character:emotion_ref]` selects a
|
||||
segment-local audio reference that can still blend with vector/text emotion.
|
||||
When an emotion control is inserted with the caret inside a character/audio tag,
|
||||
the editor appends or updates it as another pipe parameter, for example
|
||||
`[Bob:br_ivan_raiva3|sad:+0.25]`.
|
||||
|
||||
The tag editor's quick-swap palette is engine-aware: parameter choices are
|
||||
filtered to the selected inline engine, while each engine's supported native
|
||||
emotion/style/prosody/sound tags use their own replacement choices. Named
|
||||
emotion presets can be swapped from the text; quoted `[emotion:"..."]` text is
|
||||
intentionally left as direct editable content rather than treated as a preset.
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Advanced Usage
|
||||
|
||||
### Combining with Character Switching
|
||||
|
||||
Parameters work seamlessly with character and language tags:
|
||||
|
||||
```
|
||||
[de:Alice|seed:42] German Alice with specific seed.
|
||||
[fr:Bob|temperature:0.7] French Bob with higher temperature.
|
||||
[seed:123|de:Alice] Order-independent syntax.
|
||||
```
|
||||
|
||||
### MOSS Whole-Segment Prompt Fields
|
||||
|
||||
MOSS official prompt fields are better treated as per-segment parameters, not inline `<>` tags:
|
||||
|
||||
```
|
||||
[Alice|instruction:Speak softly and calmly] Hello there.
|
||||
[Bob|quality:Telephone call quality|ambient_sound:Office room tone] Can you hear me?
|
||||
[Alice|sound_event:Laughter] That's actually funny.
|
||||
```
|
||||
|
||||
Important:
|
||||
|
||||
- These are whole-segment controls
|
||||
- They are not positional inline effects
|
||||
- Keep `<>` free for true inline post-processing tags like Step Audio EditX
|
||||
|
||||
### DramaBox Prompt Templates
|
||||
|
||||
`prompt_template` (alias `template`) applies a `{seg}` wrapper and enables
|
||||
templating for that segment automatically:
|
||||
|
||||
```text
|
||||
[Narrator|template:A woman whispers, "{seg}"] This line is whispered.
|
||||
[Narrator] This line returns to the DramaBox engine-node settings.
|
||||
```
|
||||
|
||||
The template should include `{seg}`. If it is omitted, DramaBox warns once and
|
||||
appends `"{seg}"` automatically. A separate inline enable parameter is not
|
||||
required.
|
||||
|
||||
### Per-Segment Fine-Tuning in SRT
|
||||
|
||||
Create dynamic SRT content with per-segment control:
|
||||
|
||||
```srt
|
||||
1
|
||||
00:00:00,500 --> 00:00:03,000
|
||||
[Narrator|temp:0.4] The important introduction needs precision.
|
||||
|
||||
2
|
||||
00:00:03,100 --> 00:00:06,500
|
||||
[Narrator|temp:0.4] Continuing with careful, consistent delivery.
|
||||
|
||||
3
|
||||
00:00:06,600 --> 00:00:10,000
|
||||
[Narrator|temp:0.7] Now for the creative narrative section!
|
||||
|
||||
4
|
||||
00:00:10,100 --> 00:00:14,000
|
||||
[Narrator|temp:0.4] Back to precise delivery for the conclusion.
|
||||
```
|
||||
|
||||
### Alias Examples
|
||||
|
||||
Use shorter aliases for convenience:
|
||||
|
||||
```
|
||||
[Alice|temp:0.5] # temperature
|
||||
[Alice|cfg_weight:0.7] # Same as cfg:0.7
|
||||
[Alice|exag:1.5] # exaggeration
|
||||
[Alice|steps:50] # inference_steps (VibeVoice)
|
||||
[Alice|topk:20] # top_k
|
||||
[Alice|topp:0.95] # top_p
|
||||
```
|
||||
|
||||
### Semantic Parameters for Control
|
||||
|
||||
#### Temperature Examples
|
||||
```
|
||||
[Alice|temperature:0.2] This is spoken very carefully, word for word.
|
||||
[Alice|temperature:0.5] This is natural but somewhat consistent.
|
||||
[Alice|temperature:0.9] This is creative with varied expression.
|
||||
```
|
||||
|
||||
#### Seed Examples
|
||||
```
|
||||
# Same seed = identical audio output
|
||||
[Alice|seed:42] First attempt.
|
||||
[Alice|seed:42] Identical to first attempt.
|
||||
|
||||
# Different seeds = different variations
|
||||
[Alice|seed:42] Variation 1.
|
||||
[Alice|seed:100] Variation 2.
|
||||
[Alice|seed:999] Variation 3.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ⚠️ Important Notes
|
||||
|
||||
### Parameter Validation
|
||||
|
||||
- **Invalid values** are logged with warnings but don't cause errors
|
||||
- **Unsupported parameters** for your engine are silently ignored
|
||||
- **Out-of-range values** are clamped to valid ranges
|
||||
- **Type mismatches** (e.g., `temperature:abc`) are logged and rejected
|
||||
|
||||
### Engine Compatibility
|
||||
|
||||
Different engines support different parameters. The system automatically:
|
||||
- ✅ Applies supported parameters
|
||||
- ⚠️ Warns about unsupported parameters
|
||||
- ✅ Continues processing without errors
|
||||
|
||||
Example:
|
||||
```
|
||||
[Alice|speed:0.8|temperature:0.5] # ChatterBox: ignores speed, applies temperature
|
||||
[Alice|speed:0.8|temperature:0.5] # F5-TTS: applies both
|
||||
```
|
||||
|
||||
### Performance Considerations
|
||||
|
||||
- Parameter extraction happens during text parsing (minimal overhead)
|
||||
- Parameters only affect the specific segment
|
||||
- No impact on caching or model loading
|
||||
|
||||
---
|
||||
|
||||
## 🔧 Implementation Notes
|
||||
|
||||
### For Node Developers
|
||||
|
||||
If you're adding parameter support to a new engine:
|
||||
|
||||
1. Import the parameter system:
|
||||
```python
|
||||
from utils.text.segment_parameters import apply_segment_parameters
|
||||
```
|
||||
|
||||
2. When processing segments with parameters:
|
||||
```python
|
||||
if segment.parameters:
|
||||
# Apply segment parameters to override config
|
||||
segment_config = apply_segment_parameters(base_config, segment.parameters, engine_type)
|
||||
else:
|
||||
segment_config = base_config
|
||||
```
|
||||
|
||||
3. Generate audio with `segment_config` instead of `base_config`
|
||||
|
||||
### Parameter Storage
|
||||
|
||||
Parameters are stored in the `CharacterSegment.parameters` dictionary:
|
||||
```python
|
||||
segment.parameters = {
|
||||
'seed': 42,
|
||||
'temperature': 0.5,
|
||||
'cfg': 0.7
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📚 See Also
|
||||
|
||||
- [Character Switching Guide](CHARACTER_SWITCHING_GUIDE.md) - Multi-character and language switching
|
||||
- [ChatterBox Special Tokens Guide](CHATTERBOX_V2_SPECIAL_TOKENS.md) - Emotion and sound tokens
|
||||
- README.md - Complete feature overview
|
||||
|
||||
---
|
||||
|
||||
## 🐛 Troubleshooting
|
||||
|
||||
### Parameters Not Being Applied
|
||||
|
||||
1. **Check engine compatibility** - Parameter might not be supported by your engine
|
||||
2. **Verify syntax** - Use `|` to separate parameters: `[Alice|seed:42|temp:0.5]`
|
||||
3. **Check for typos** - Parameter names are case-insensitive but must match known parameters
|
||||
4. **Look at console** - Warnings are logged when parameters are unsupported or invalid
|
||||
|
||||
### "Parameter not supported" Warning
|
||||
|
||||
This is normal! Different engines support different parameters. For example:
|
||||
- ChatterBox doesn't support `speed` (F5-TTS-only parameter)
|
||||
- F5-TTS doesn't support `exaggeration` (ChatterBox-only parameter)
|
||||
|
||||
The system silently ignores unsupported parameters and continues processing.
|
||||
|
||||
### Values Not Taking Effect
|
||||
|
||||
- **Check range** - Values outside the valid range are clamped to min/max
|
||||
- **Verify engine** - Make sure you're using an engine that supports that parameter
|
||||
- **Check node defaults** - Parameters only override node-level defaults, they don't create new capabilities
|
||||
@@ -1,113 +0,0 @@
|
||||
# 🌩️ Sound Effects Guide
|
||||
|
||||
The `🌩️ Sound Effects` node generates non-speech audio from a written description. It works with any connected engine that advertises sound-effect support.
|
||||
|
||||
## Engines
|
||||
|
||||
| Model | Engine node | Notes |
|
||||
|---|---|---|
|
||||
| MOSS-SoundEffect v1 | `⚙️ MOSS-TTS Engine` | Autoregressive MOSS 8B model |
|
||||
| MOSS-SoundEffect v2 | `⚙️ MOSS SoundEffect v2 Engine` | 48 kHz diffusion model; up to 30 seconds per native generation |
|
||||
|
||||
Connecting a speech-only engine stops with a user-facing compatibility error.
|
||||
|
||||
## Basic workflow
|
||||
|
||||
1. Select a sound-effect model on its engine node.
|
||||
2. Connect the engine to `🌩️ Sound Effects`.
|
||||
3. Describe the sound rather than words to be spoken.
|
||||
4. Choose the duration and seed, then queue the workflow.
|
||||
|
||||
Example:
|
||||
|
||||
```text
|
||||
Heavy rain hitting a metal rooftop, distant rolling thunder, occasional wind gusts.
|
||||
```
|
||||
|
||||
Descriptions generally work best when they state the source, environment, distance, texture, and progression of the sound.
|
||||
|
||||
## Timeline segments
|
||||
|
||||
Separate descriptions with parameter tags to generate multiple segments and concatenate them:
|
||||
|
||||
```text
|
||||
[seconds:4|seed:42] Bright application startup chime. [seconds:2|cfg:5] Low, dark shutdown tone.
|
||||
```
|
||||
|
||||
`duration_seconds` is the default duration for every generated segment. `[seconds:X]` overrides it for the following segment.
|
||||
|
||||
Newlines also create segments, as they do in TTS. They are optional because a parameter tag can start another segment on the same line.
|
||||
|
||||
## Pauses
|
||||
|
||||
Use a standalone pause tag to insert exact silence:
|
||||
|
||||
```text
|
||||
[seconds:4] Startup chime. [pause:1.2] [seconds:2] Shutdown tone.
|
||||
```
|
||||
|
||||
The aliases `[wait:X]` and `[stop:X]` are also accepted. Durations may use seconds or milliseconds:
|
||||
|
||||
```text
|
||||
[wait:500ms]
|
||||
```
|
||||
|
||||
Keep pauses separate from parameter tags:
|
||||
|
||||
```text
|
||||
[pause:1.2] [cfg:7.5] Thunder crack.
|
||||
```
|
||||
|
||||
Do not combine them as `[pause:1.2|cfg:7.5]`.
|
||||
|
||||
## Crossfade and long sounds
|
||||
|
||||
`crossfade_seconds` overlaps adjacent generated segments to soften their join. Set it to `0` for a hard join.
|
||||
|
||||
A pause creates an exact silent boundary, so crossfade is not applied across that pause.
|
||||
|
||||
MOSS-SoundEffect v2 has a native 30-second generation limit. Longer requested segments are generated as overlapping chunks, joined with the selected crossfade, and trimmed to the requested duration.
|
||||
|
||||
## Per-segment parameters
|
||||
|
||||
Common parameters:
|
||||
|
||||
| Tag | Engines | Purpose |
|
||||
|---|---|---|
|
||||
| `seed` | v1, v2 | Generated variation |
|
||||
| `seconds` / `duration_seconds` | v1, v2 | Segment duration |
|
||||
| `temperature` | v1 | Sampling randomness |
|
||||
| `top_p`, `top_k` | v1 | Sampling limits |
|
||||
| `repetition_penalty` | v1 | Discourage repetition |
|
||||
| `duration_tokens` | v1 | Native duration-token control |
|
||||
| `max_new_tokens` | v1 | Generation token limit |
|
||||
| `steps` / `inference_steps` | v2 | Diffusion steps |
|
||||
| `cfg` | v2 | Prompt guidance strength |
|
||||
| `sigma_shift` | v2 | Flow-matching schedule shift |
|
||||
| `negative_prompt`, `negative`, `neg` | v2 | Sounds or qualities to discourage |
|
||||
|
||||
Example using a negative prompt:
|
||||
|
||||
```text
|
||||
[seconds:8|cfg:5|neg:speech, music] Dense forest ambience with insects and distant birds.
|
||||
```
|
||||
|
||||
Unsupported parameters are ignored with a warning rather than being sent blindly to the engine.
|
||||
|
||||
## Seed and cache behavior
|
||||
|
||||
- `seed: 0` chooses a random seed.
|
||||
- Reusing a positive seed with identical settings makes the request repeatable.
|
||||
- With audio caching enabled, an identical segment and configuration can reuse its generated audio.
|
||||
- Changing the description, seed, duration, engine configuration, or inline parameters invalidates that cached result.
|
||||
|
||||
## MOSS-SoundEffect v2 first-run compilation
|
||||
|
||||
The v2 DiT uses `torch.compile`. Its first generation may spend several minutes compiling before progress begins. Compatible compilation artifacts are cached and can be reused across later ComfyUI sessions.
|
||||
|
||||
This compile delay is separate from model downloading and normal generation time.
|
||||
|
||||
## Related guides
|
||||
|
||||
- [Per-Segment Parameter Switching](PARAMETER_SWITCHING_GUIDE.md)
|
||||
- [Multiline TTS Tag Editor](MULTILINE_TTS_TAG_EDITOR_GUIDE.md)
|
||||
@@ -1,47 +0,0 @@
|
||||
# Text Chunking Guide
|
||||
|
||||
TTS Audio Suite supports long text generation through the shared unified chunking system.
|
||||
|
||||
## What It Does
|
||||
|
||||
- splits long text into smaller generation chunks
|
||||
- preserves sentence boundaries when possible
|
||||
- can fall back to comma-based splitting for very long sentences
|
||||
- combines generated audio using the suite chunk combiner
|
||||
|
||||
## Main Controls
|
||||
|
||||
- `enable_chunking`
|
||||
- turns chunking on or off
|
||||
- `max_chars_per_chunk`
|
||||
- character limit per chunk
|
||||
- `chunk_combination_method`
|
||||
- `auto`
|
||||
- `concatenate`
|
||||
- `silence_padding`
|
||||
- `crossfade`
|
||||
- `silence_between_chunks_ms`
|
||||
- silence inserted when the selected combination method uses silence padding
|
||||
|
||||
## Auto Combination Behavior
|
||||
|
||||
When `chunk_combination_method=auto`, the suite picks a join strategy based on text size:
|
||||
|
||||
- longer text tends toward `silence_padding`
|
||||
- medium text tends toward `crossfade`
|
||||
- shorter chunked text tends toward `concatenate`
|
||||
|
||||
The exact choice is made by the shared chunk combiner, not by engine-local custom logic.
|
||||
|
||||
## Important Architecture Rule
|
||||
|
||||
Chunking is a suite feature.
|
||||
|
||||
- new engines should reuse the shared chunking and chunk-combination controls
|
||||
- engines should not invent duplicate local chunk-silence or chunk-join settings unless the official model has a real distinct native long-form system
|
||||
|
||||
## Related Docs
|
||||
|
||||
- [Example Workflows](../README.md#-example-workflows)
|
||||
- [Character Switching Guide](./CHARACTER_SWITCHING_GUIDE.md)
|
||||
- [Multiline TTS Tag Editor Guide](./MULTILINE_TTS_TAG_EDITOR_GUIDE.md)
|
||||
@@ -1,293 +0,0 @@
|
||||
# IndexTTS-2 Updates Log
|
||||
|
||||
This document tracks updates applied to our bundled IndexTTS-2 code from the upstream repository.
|
||||
|
||||
## Update Process
|
||||
1. **Check for upstream changes:** `cd IgnoredForGitHubDocs/For_reference/index_tts && git pull`
|
||||
2. **Review changes:** `git diff OLD_COMMIT..NEW_COMMIT -- indextts/infer_v2.py`
|
||||
3. **Apply relevant changes** to `engines/index_tts/indextts/infer_v2.py`
|
||||
4. **Test functionality** with our IndexTTS-2 nodes
|
||||
5. **Document changes** in this log
|
||||
|
||||
---
|
||||
|
||||
## 2026-08-11: IndexTTS-2.5 Version Integration
|
||||
|
||||
**Official sources:** `index-tts/index-tts` commit `b5ea881bec284b72f0b1cc04e0a724ff0c6b93e9`; model snapshot `ba2480d9f7f629eb18f6acaebb357679d9ba88a4`
|
||||
|
||||
### Changes applied
|
||||
|
||||
- Added IndexTTS-2.5 as a selectable version of the existing `index_tts` engine.
|
||||
- Bundled the official 25 Hz semantic codec, multilingual tokenizer, Japanese G2P, and NeMo normalization bridge.
|
||||
- Preserved suite dual-source audio plus vector/text emotion blending.
|
||||
- Added Chinese, English, Japanese, Spanish, and Arabic conditioning.
|
||||
- Added the official 2.5-only `duration_factor`, documented honestly as nearest-neighbor internal semantic-feature scaling rather than natural prosody or exact-duration planning.
|
||||
- Deliberately excluded IndexTTS-2.5 from TTS SRT's native-duration option; the suite-owned exact-seconds extrapolation was removed after source and listening review.
|
||||
- Kept legacy IndexTTS-2 checkpoints, FP16 loading, MaskGCT, workflows, and node identity intact.
|
||||
- Pinned the audited Hugging Face model revision and retained the main Transformers 5 environment.
|
||||
- Added model-aware Text/SRT processor and audio-cache identities so switching 2.0/2.5 or a 2.5 generation parameter cannot reuse stale output.
|
||||
- Documented the suite's manual finding that 2.5 is not a universal cloning-quality upgrade: 2.0 may retain speaker resemblance better under strong different-speaker emotion transfer.
|
||||
|
||||
### Validation status
|
||||
|
||||
- [x] Python compilation
|
||||
- [x] Bundled backend import under `TTS_SUITE_TEST_VENV_PYTHON`
|
||||
- [x] Full checkpoint download and live ComfyUI generation
|
||||
- [x] Manual audio-quality review of the official duration factor and 2.0/2.5 speaker resemblance
|
||||
- [x] Live 2.5 → 2.0 model switching after processor-cache invalidation fix
|
||||
|
||||
---
|
||||
|
||||
## 2025-09-18: Major Update - Cache & Emotion Improvements
|
||||
|
||||
**Reference commit range:** `8336824..64cb31a` (September 11 → September 18, 2025)
|
||||
**Upstream commits reviewed:** 5 commits affecting `infer_v2.py`
|
||||
|
||||
### 🔧 Changes Applied:
|
||||
|
||||
#### 1. **Emotion Vector Normalization Function** (commit: 8aa8064)
|
||||
- **Added:** `normalize_emo_vec()` helper function
|
||||
- **Purpose:** Prevents voice identity loss from overly strong emotion settings
|
||||
- **Implementation:**
|
||||
- Applies emotion bias factors to reduce problematic emotions
|
||||
- Caps total emotion sum to 0.8 maximum
|
||||
- Protects against extreme emotion vector values
|
||||
- **User benefit:** Better emotion control without losing speaker characteristics
|
||||
|
||||
#### 2. **Audio Length Limiting** (commit: 0828dcb)
|
||||
- **Added:** `_load_and_cut_audio()` helper function
|
||||
- **Purpose:** Prevents memory/VRAM issues from overly long reference audio
|
||||
- **Implementation:**
|
||||
- Automatically truncates audio to 15 seconds maximum
|
||||
- Supports different sample rates (16kHz, 22kHz)
|
||||
- Provides verbose logging of truncation
|
||||
- **User benefit:** More stable generation with long reference audio files
|
||||
|
||||
#### 3. **Persistent Cache Buildup Fix** (commit: 64cb31a)
|
||||
- **Added:** Proper cache clearing with `torch.cuda.empty_cache()`
|
||||
- **Purpose:** Solves memory accumulation issues during multiple generations
|
||||
- **Implementation:**
|
||||
- Clears old cache variables before loading new audio
|
||||
- Calls `torch.cuda.empty_cache()` to free GPU memory
|
||||
- Applied to both speaker and emotion cache invalidation
|
||||
- **User benefit:** Better memory management during batch processing
|
||||
|
||||
#### 4. **BigVGAN CUDA Kernel Import Fix** (commits: ee23371, e409c4a)
|
||||
- **Fixed:** Corrected import path for BigVGAN custom CUDA kernel
|
||||
- **Changed:** `indextts.BigVGAN.alias_free_activation.cuda` → `indextts.s2mel.modules.bigvgan.alias_free_activation.cuda`
|
||||
- **Purpose:** Ensures proper CUDA acceleration when available
|
||||
- **User benefit:** Better performance with CUDA-enabled setups
|
||||
|
||||
### 📋 Implementation Status:
|
||||
|
||||
| Change | Applied to Bundled Code | Tested | Notes |
|
||||
|--------|------------------------|--------|-------|
|
||||
| `normalize_emo_vec()` | ✅ **Applied** | ⏳ Pending | Core emotion control improvement |
|
||||
| `_load_and_cut_audio()` | ✅ **Applied** | ⏳ Pending | Memory stability fix |
|
||||
| Cache clearing fixes | ✅ **Applied** | ⏳ Pending | Critical for batch processing |
|
||||
| BigVGAN import fix | ✅ **Applied** | ⏳ Pending | Performance improvement |
|
||||
|
||||
### 🎯 User-Facing Improvements:
|
||||
- **Better emotion control:** Emotion vectors now auto-normalize to prevent voice identity loss
|
||||
- **Memory stability:** Long reference audio automatically truncated to prevent crashes
|
||||
- **Batch processing:** Improved cache management for multiple generations
|
||||
- **Performance:** Fixed CUDA kernel loading for faster inference
|
||||
|
||||
### 🧪 Testing Required:
|
||||
- [ ] Emotion vector controls in radar chart widget
|
||||
- [ ] Long reference audio handling (>15 seconds)
|
||||
- [ ] Multiple sequential generations (cache clearing)
|
||||
- [ ] CUDA performance with FP16 enabled
|
||||
|
||||
---
|
||||
|
||||
---
|
||||
|
||||
## 2025-11-06: Performance & Stability Improvements
|
||||
|
||||
**Reference commit range:** `64cb31a..1d5d079` (September 18 → November 6, 2025)
|
||||
**Upstream commits reviewed:** 6 commits affecting `infer_v2.py`
|
||||
|
||||
### 🔧 New Changes Identified:
|
||||
|
||||
#### 1. **Empty Generator → IndexError Fix** (commit: 750d9d9)
|
||||
- **Issue:** Non-streaming `infer()` would crash with IndexError on empty generator
|
||||
- **Fix:** Wrapped generator list conversion in try/except, returns `None` on error
|
||||
- **Changes to `infer_v2.py`:**
|
||||
- Lines 354-364: Added exception handling
|
||||
- Lines 680, 690: Changed `return` to `yield` for proper generator flow
|
||||
- **Priority:** 🔴 **Critical** - Prevents runtime crashes
|
||||
|
||||
#### 2. **Streaming Implementation** (commit: b0c6ab8)
|
||||
- **Added:** `stream_return` and `more_segment_before` parameters for low-latency streaming
|
||||
- **Purpose:** Enable first-chunk audio output before full generation completes
|
||||
- **Changes:**
|
||||
- `stream_return` (bool): Return generator instead of final result
|
||||
- `more_segment_before` (int): Apply extra segmentation for faster first output (0-80 recommended)
|
||||
- Modified `indextts/utils/front.py` for tokenization changes
|
||||
- **Priority:** 🟡 **Important** - Enables streaming use cases
|
||||
|
||||
#### 3. **UNK Token Warnings** (commit: 34be9bf)
|
||||
- **Added:** Detection and warning for unknown tokens in input text
|
||||
- **Behavior:**
|
||||
- Detects tokens the BPE model cannot encode
|
||||
- Prints warnings with token IDs and suggested fixes
|
||||
- Helps users debug encoding issues
|
||||
- **Changes to `infer_v2.py`:** Lines 450-459, added token validation logic
|
||||
- **Priority:** 🟢 **Nice-to-have** - Improves user experience with error diagnosis
|
||||
|
||||
#### 4. **S2Mel Stage Optimization** (commit: 31e7e85)
|
||||
- **Focus:** Optimized mel-spectrogram generation pipeline
|
||||
- **Changes:**
|
||||
- `indextts/s2mel/modules/commons.py`: 15 lines added/modified
|
||||
- `indextts/s2mel/modules/diffusion_transformer.py`: 2 lines modified
|
||||
- `indextts/s2mel/modules/flow_matching.py`: 15 lines added (new optimization code)
|
||||
- `indextts/infer_v2.py`: 28 lines added/modified
|
||||
- **Expected Impact:** Faster mel-spectrogram generation (stage 2 of 3)
|
||||
- **Priority:** 🟡 **Important** - Performance improvement
|
||||
|
||||
#### 5. **GPT2 Inference Acceleration** (commit: c1ef414)
|
||||
- **Focus:** Major performance optimization for GPT2 stage (stage 1)
|
||||
- **Changes:**
|
||||
- **New module:** `indextts/accel/` directory (609 lines added):
|
||||
- `accel_engine.py`: Core acceleration logic (609 lines)
|
||||
- `attention.py`: Optimized attention mechanism (154 lines)
|
||||
- `gpt2_accel.py`: GPT2-specific optimizations (181 lines)
|
||||
- `kv_manager.py`: Key-value cache management (209 lines)
|
||||
- Modified `indextts/gpt/model_v2.py`: 63 lines changed (+47 new)
|
||||
- Modified `indextts/infer_v2.py`: 17 lines changed
|
||||
- **Expected Impact:** Significantly faster GPT2 inference (first stage bottleneck)
|
||||
- **Priority:** 🔴 **Critical** - Major performance gain, but new code complexity
|
||||
|
||||
#### 6. **Merge Commit** (commit: 5d67f62 → 1d5d079)
|
||||
- Merge of acceleration branch into main
|
||||
- Contains all above changes integrated
|
||||
|
||||
### 📊 Summary
|
||||
|
||||
| Change | Files Modified | Lines Changed | Priority | Status |
|
||||
|--------|---------------|---------------|----------|--------|
|
||||
| Empty generator fix | 1 | +12/-9 | 🔴 Critical | ⏳ Pending |
|
||||
| Streaming support | 2 | +72/-18 | 🟡 Important | ⏳ Pending |
|
||||
| UNK token warnings | 1 | +7 | 🟢 Nice-to-have | ⏳ Pending |
|
||||
| S2Mel optimization | 4 | +54/-6 | 🟡 Important | ⏳ Pending |
|
||||
| GPT2 acceleration | 6 | +1228/-14 | 🔴 Critical | ⏳ Pending |
|
||||
|
||||
### 🚨 Integration Concerns
|
||||
|
||||
1. **New acceleration module complexity** - The `indextts/accel/` package adds significant complexity with KV-cache management and specialized attention mechanisms. Needs careful testing.
|
||||
|
||||
2. **Potential API changes** - Streaming parameters (`stream_return`, `more_segment_before`) may require adapter updates if our wrapper doesn't expose these.
|
||||
|
||||
3. **GPU memory implications** - KV-cache optimization might have different memory profiles; needs profiling on different GPU setups.
|
||||
|
||||
4. **File structure expansion** - New `accel/` subdirectory adds to maintenance burden.
|
||||
|
||||
### 🧪 Testing Required
|
||||
|
||||
- [ ] Empty generator fix - test non-streaming inference with various inputs
|
||||
- [ ] Streaming latency - measure first-chunk output time with `stream_return=True`
|
||||
- [ ] GPT2 acceleration - benchmark inference speed (expect 1.5-3x speedup)
|
||||
- [ ] S2Mel optimization - verify audio quality unchanged
|
||||
- [ ] UNK token handling - test with multilingual/special character inputs
|
||||
- [ ] Memory profiling - GPU memory usage with new KV-cache manager
|
||||
- [ ] ComfyUI integration - ensure all changes work through our node interface
|
||||
|
||||
### ✅ Applied Changes Summary (2025-11-06)
|
||||
|
||||
**All upstream commits successfully integrated:**
|
||||
|
||||
1. ✅ **Empty generator fix (750d9d9)** - Refactored `infer()` to use generator pattern:
|
||||
- Created wrapper `infer()` method that delegates to `infer_generator()`
|
||||
- Refactored monolithic method into `infer_generator()` with yield statements
|
||||
- Changed final `return` statements to `yield` for generator compatibility
|
||||
- Handles both streaming (`stream_return=True`) and non-streaming modes
|
||||
|
||||
2. ✅ **Streaming implementation (b0c6ab8)** - Added streaming parameters:
|
||||
- Added `stream_return` parameter to `infer()` and `infer_generator()`
|
||||
- Added `more_segment_before` parameter for streaming segmentation
|
||||
- Integrated with generator-based inference for low-latency streaming
|
||||
|
||||
3. ✅ **UNK token warnings (34be9bf)** - Added token validation:
|
||||
- Added token validation in `infer_generator()`
|
||||
- Warns users about unknown tokens with suggestions
|
||||
|
||||
4. ✅ **S2Mel optimization (31e7e85)** - Applied all module updates:
|
||||
- `commons.py`: Updated `fused_add_tanh_sigmoid_multiply()` with torch.split
|
||||
- `diffusion_transformer.py`: Fixed `sequence_mask()` call with max_length parameter
|
||||
- `flow_matching.py`: Added `enable_torch_compile()` method to CFM class
|
||||
- `infer_v2.py`: Added `use_torch_compile` parameter and setup
|
||||
|
||||
5. ✅ **GPT2 acceleration (c1ef414)** - Applied complete acceleration module:
|
||||
- Created `/accel/` directory with 5 files (init, accel_engine, attention, gpt2_accel, kv_manager)
|
||||
- Modified `model_v2.py`: Added `use_accel` parameter to UnifiedVoice
|
||||
- Modified `model_v2.py`: Added accel engine initialization in `post_init_gpt2_config()`
|
||||
- Modified `model_v2.py`: Added accel-aware inference in `inference_speech()`
|
||||
|
||||
### 🔧 Custom Modifications Preserved
|
||||
|
||||
- QwenEmotion lazy loading with path fallbacks (lines 81-112 in infer_v2.py)
|
||||
- W2V-BERT model loading from TTS folder with unified downloader (lines 145-166)
|
||||
- All custom logging and error handling in infer_v2.py and throughout
|
||||
- Custom model initialization sequence
|
||||
- Custom emotion vector normalization and processing
|
||||
|
||||
---
|
||||
|
||||
---
|
||||
|
||||
## 2026-07-11: Upstream Audit Before Release
|
||||
|
||||
**Upstream repository checked:** `index-tts/index-tts` (`main`)
|
||||
**Upstream head observed:** `b5bd657` (2026-07-08)
|
||||
**Check performed:** 2026-07-11
|
||||
|
||||
### Relevant upstream changes reviewed
|
||||
|
||||
| Commit | Upstream change | Bundled status | Release action |
|
||||
|---|---|---|---|
|
||||
| `843972e` | Coerce QwenEmotion JSON emotion scores to `float` and reject non-numeric values clearly | **Applied** in bundled `clamp_score()` | Run a text-emotion generation with numeric-string classifier output |
|
||||
| `b154a1b` | WebUI text/vector preset save/load management | **Already covered differently** by the ComfyUI-native preset manager and `emotion_presets.json` integration | No direct port needed |
|
||||
| `b5bd657` | Expose `--accel` and `--torch-compile`, add optional acceleration extras and WebUI settings | **Applied selectively**; suite now forwards `use_accel` into the bundled GPT path, while retaining ComfyUI-owned dependency handling | Validate acceleration fallback on compatible and non-accelerated setups |
|
||||
| `7264ce2` | Improve IndexTTS-2 model resource checks and HF cache handling | **Suite-owned downloader differs** and needs a separate comparison if download failures are reported | No blind copy into bundled code |
|
||||
|
||||
### Findings
|
||||
|
||||
- No upstream change was found that invalidates the current eight-emotion vector order,
|
||||
Qwen text-emotion syntax, audio-reference blending, or the suite's inline tag format.
|
||||
- The upstream QwenEmotion string-score fix is directly relevant to the suite's text-emotion
|
||||
path and should be applied before a release.
|
||||
- The upstream acceleration work is not a drop-in replacement because this repository
|
||||
bundles and adapts IndexTTS-2. The suite already contains the acceleration modules, but
|
||||
`utils/models/unified_model_interface.py` should be checked so `use_accel` reaches the
|
||||
bundled `IndexTTS2` constructor.
|
||||
- Upstream WebUI presets are not copied verbatim: the suite's ComfyUI editor has a richer
|
||||
vector/radar, inline-tag, sidebar, and filesystem preset implementation.
|
||||
|
||||
### Release follow-up checklist
|
||||
|
||||
- [x] Apply the upstream `clamp_score()` numeric coercion.
|
||||
- [x] Pass `use_accel` through the unified IndexTTS-2 factory; validate fallback behavior.
|
||||
- [ ] Run a Qwen text-emotion generation using numeric-string classifier output.
|
||||
- [ ] Verify acceleration flags on a compatible CUDA installation and on a setup without
|
||||
optional acceleration dependencies.
|
||||
- [ ] Recheck bundled model-resource validation against the current upstream `check` logic.
|
||||
|
||||
---
|
||||
|
||||
## Next Update Check: 2025-11-20
|
||||
|
||||
**Monitoring:** Watch for commits to `indextts/infer_v2.py` in https://github.com/index-tts/index-tts
|
||||
|
||||
**Update command:**
|
||||
```bash
|
||||
cd /tmp/index_tts_upstream
|
||||
git pull origin main
|
||||
git log --oneline 1d5d079..HEAD -- indextts/infer_v2.py
|
||||
```
|
||||
|
||||
**Priority changes to watch for:**
|
||||
- Additional performance optimizations
|
||||
- Bug fixes in new acceleration code
|
||||
- Breaking API changes
|
||||
- Model loading improvements
|
||||
@@ -1,103 +0,0 @@
|
||||
# [ENGINE_NAME] Upstream Update Log
|
||||
|
||||
**Engine:** [ENGINE_NAME]
|
||||
**Official Repository:** [UPSTREAM_URL]
|
||||
**Bundled Location:** `engines/[engine_folder]/`
|
||||
**Last Checked:** YYYY-MM-DD
|
||||
**Last Checked By:** [username]
|
||||
|
||||
---
|
||||
|
||||
## Check History
|
||||
|
||||
| Date | Commits Checked | New Features | Bug Fixes | Breaking Changes | Notes |
|
||||
|------|-----------------|--------------|-----------|------------------|-------|
|
||||
| YYYY-MM-DD | N/A | - | - | - | Initial setup |
|
||||
|
||||
---
|
||||
|
||||
## Latest Upstream Changes (Since Last Check)
|
||||
|
||||
### New Features 🆕
|
||||
- Feature 1
|
||||
- Feature 2
|
||||
|
||||
### Bug Fixes 🐛
|
||||
- Fix 1
|
||||
- Fix 2
|
||||
|
||||
### Breaking Changes ⚠️
|
||||
- Change 1
|
||||
- Change 2
|
||||
|
||||
### Performance Improvements ⚡
|
||||
- Improvement 1
|
||||
|
||||
### Documentation Updates 📝
|
||||
- Update 1
|
||||
|
||||
---
|
||||
|
||||
## Integration Status
|
||||
|
||||
| Item | Status | Priority | Notes |
|
||||
|------|--------|----------|-------|
|
||||
| Feature/Fix name | `pending`/`integrated`/`skipped` | 🔴/🟡/🟢 | Integration notes |
|
||||
|
||||
### Legend
|
||||
- `pending` - Identified but not yet integrated
|
||||
- `integrated` - Applied to bundled code
|
||||
- `skipped` - Reviewed but decided not to integrate
|
||||
- `wontfix` - Not applicable to our implementation
|
||||
|
||||
---
|
||||
|
||||
## Integration Notes
|
||||
|
||||
### Conflicts with Our Customizations
|
||||
- [List any conflicts between upstream changes and our custom modifications]
|
||||
|
||||
### Testing Required
|
||||
- [ ] Test case 1
|
||||
- [ ] Test case 2
|
||||
|
||||
### Version Bump Recommendation
|
||||
- Current bundled version: X.Y.Z
|
||||
- Latest upstream version: X.Y.Z
|
||||
- Recommended bump: `patch` / `minor` / `major` / `none`
|
||||
|
||||
---
|
||||
|
||||
## Known Customizations We Maintain
|
||||
|
||||
List any intentional differences from upstream:
|
||||
- Customization 1
|
||||
- Customization 2
|
||||
|
||||
---
|
||||
|
||||
## Next Check
|
||||
|
||||
**Scheduled for:** YYYY-MM-DD
|
||||
**Check interval:** Weekly / Bi-weekly / Monthly
|
||||
|
||||
---
|
||||
|
||||
## Previous Check Summary
|
||||
|
||||
Link to previous check details if available.
|
||||
|
||||
---
|
||||
|
||||
## Quick Commands
|
||||
|
||||
```bash
|
||||
# Clone/update upstream repository
|
||||
git clone [UPSTREAM_URL] /tmp/[engine_name]_upstream
|
||||
|
||||
# Compare with our bundled version
|
||||
diff -r /tmp/[engine_name]_upstream/[source_folder] engines/[bundled_folder]/
|
||||
|
||||
# Show recent commits from upstream
|
||||
git log --oneline -20 origin/main
|
||||
```
|
||||
@@ -1,196 +0,0 @@
|
||||
# Upstream TTS Engines Tracking Index
|
||||
|
||||
This document provides a centralized reference for monitoring updates to all bundled TTS engines. Each engine has its own update log file tracking changes, bug fixes, and new features from the official upstream repository.
|
||||
|
||||
**Last Updated:** 2025-11-06
|
||||
|
||||
---
|
||||
|
||||
## Quick Reference Table
|
||||
|
||||
| Engine | Bundled Source | Location | Official Repository | Update Log | Last Checked |
|
||||
|--------|---------|----------|---------------------|------------|--------------|
|
||||
| **IndexTTS-2** | ✅ Full | `engines/index_tts/indextts/` | [github.com/index-tts/index-tts](https://github.com/index-tts/index-tts) | [INDEXTTS2_UPDATE_LOG.md](./INDEXTTS2_UPDATE_LOG.md) | 2025-11-06 |
|
||||
| **ChatterBox** | ✅ Full | `engines/chatterbox/` | [github.com/resemble-ai/chatterbox](https://github.com/resemble-ai/chatterbox) | [CHATTERBOX_UPDATE_LOG.md](./CHATTERBOX_UPDATE_LOG.md) | TBD |
|
||||
| **ChatterBox Multilingual (23-Lang)** | ✅ Full | `engines/chatterbox_official_23lang/` | [github.com/resemble-ai/chatterbox](https://github.com/resemble-ai/chatterbox) | [CHATTERBOX_UPDATE_LOG.md](./CHATTERBOX_UPDATE_LOG.md) | TBD |
|
||||
| **F5-TTS** | ✅ Full | `engines/f5_tts/` | [github.com/SWivid/F5-TTS](https://github.com/SWivid/F5-TTS) | [F5TTS_UPDATE_LOG.md](./F5TTS_UPDATE_LOG.md) | TBD |
|
||||
| **Higgs Audio** | ✅ Full | `engines/higgs_audio/boson_multimodal/` | [github.com/boson-ai/higgs-audio](https://github.com/boson-ai/higgs-audio) | [HIGGS_AUDIO_UPDATE_LOG.md](./HIGGS_AUDIO_UPDATE_LOG.md) | TBD |
|
||||
| **RVC** | ✅ Full | `engines/rvc/` | [github.com/RVC-Project/Retrieval-based-Voice-Conversion](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion) | [RVC_UPDATE_LOG.md](./RVC_UPDATE_LOG.md) | TBD |
|
||||
| **VibeVoice** | ⚠️ Wrapper Only | `engines/vibevoice_engine/` | [github.com/microsoft/VibeVoice](https://github.com/microsoft/VibeVoice) ⚠️ Disabled | [VIBEVOICE_UPDATE_LOG.md](./VIBEVOICE_UPDATE_LOG.md) | TBD |
|
||||
|
||||
---
|
||||
|
||||
## Engine Status Details
|
||||
|
||||
### ✅ Fully Bundled Engines (Source Code Included)
|
||||
|
||||
These engines have their complete source code bundled in the TTS Audio Suite repository:
|
||||
|
||||
1. **IndexTTS-2** - Full inference code and models bundled
|
||||
2. **ChatterBox** - Full model and inference code bundled
|
||||
3. **ChatterBox Multilingual** - Wrapper around same ChatterBox repo with language support
|
||||
4. **F5-TTS** - Full source code with configs, models, and training scripts
|
||||
5. **Higgs Audio** - Full boson_multimodal framework bundled
|
||||
6. **RVC** - Complete voice conversion implementation
|
||||
|
||||
### ⚠️ Wrapper-Only Engines
|
||||
|
||||
These are external dependencies integrated via wrappers:
|
||||
|
||||
- **VibeVoice** - Wrapper around Microsoft's `vibevoice` PyPI package (⚠️ **Note**: Official repo is disabled by Microsoft as of Sept 2025)
|
||||
|
||||
---
|
||||
|
||||
## Update Checking Workflow
|
||||
|
||||
### Manual Checking (via slash command)
|
||||
```bash
|
||||
/check_tts_bundled_updates [engine_name] [--verbose]
|
||||
```
|
||||
|
||||
Examples:
|
||||
- `/check_tts_bundled_updates` - Check ALL engines
|
||||
- `/check_tts_bundled_updates IndexTTS-2` - Check specific engine
|
||||
- `/check_tts_bundled_updates ChatterBox --verbose` - Verbose output with detailed diffs
|
||||
|
||||
### What Gets Checked
|
||||
|
||||
For each engine, the process:
|
||||
1. Clones/updates the official upstream repository
|
||||
2. Compares commit history since last check
|
||||
3. Identifies new features, bug fixes, breaking changes
|
||||
4. Updates the engine's UPDATE_LOG.md file
|
||||
5. Flags items requiring integration
|
||||
|
||||
### Priority Levels for Updates
|
||||
|
||||
- 🔴 **Critical**: Security fixes, breaking changes, major bug fixes
|
||||
- 🟡 **Important**: New features, performance improvements, compatibility updates
|
||||
- 🟢 **Nice-to-have**: Documentation, minor tweaks, non-critical improvements
|
||||
|
||||
---
|
||||
|
||||
## Individual Engine Logs
|
||||
|
||||
Each engine has its own update log tracking:
|
||||
- **Date checked**: When upstream was last reviewed
|
||||
- **New commits**: List of changes since last version
|
||||
- **Actionable items**: What needs integration/testing
|
||||
- **Integration status**: Whether changes have been applied
|
||||
|
||||
See individual UPDATE_LOG.md files in this directory for details.
|
||||
|
||||
---
|
||||
|
||||
## Integration Checklist
|
||||
|
||||
When integrating upstream updates:
|
||||
|
||||
- [ ] Read upstream changelog/commits
|
||||
- [ ] Identify compatibility issues with our wrapper code
|
||||
- [ ] Create backup of current bundled code
|
||||
- [ ] Apply changes incrementally
|
||||
- [ ] Run full test suite
|
||||
- [ ] Document any conflicts or customizations we maintain
|
||||
- [ ] Update version in nodes.py if integration warrants version bump
|
||||
- [ ] Commit with reference to upstream changes
|
||||
|
||||
---
|
||||
|
||||
## Known Customizations We Maintain
|
||||
|
||||
### IndexTTS-2
|
||||
- Audio normalization fix for Linux (v4.14.15) - **Not in upstream**
|
||||
- Custom processor wrapper for unified interface
|
||||
- Audio amplitude normalization to handle platform-specific torchaudio.load() behavior
|
||||
|
||||
### ChatterBox (both versions)
|
||||
- Unified adapter pattern for consistent interface
|
||||
- Integration with unified node architecture
|
||||
|
||||
### F5-TTS
|
||||
- Unified adapter pattern for consistent interface
|
||||
|
||||
### Higgs Audio
|
||||
- Unified adapter pattern for consistent interface
|
||||
|
||||
### RVC
|
||||
- Complete custom implementation for TTS Audio Suite
|
||||
|
||||
### VibeVoice
|
||||
- Wrapper around external vibevoice package
|
||||
- ⚠️ **Status**: Microsoft disabled official repo in Sept 2025 - consider migration path
|
||||
|
||||
---
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [NEW_ENGINE_IMPLEMENTATION_GUIDE.md](../NEW_ENGINE_IMPLEMENTATION_GUIDE.md) - How to integrate new engines
|
||||
- [INDEXTTS2_UPDATE_LOG.md](./INDEXTTS2_UPDATE_LOG.md) - IndexTTS-2 specific tracking
|
||||
- [../DEPENDENCY_MANAGEMENT_GUIDE.md](../DEPENDENCY_MANAGEMENT_GUIDE.md) - Dependency compatibility
|
||||
- [../BUMP_SCRIPT_INSTRUCTIONS.md](../BUMP_SCRIPT_INSTRUCTIONS.md) - Version management
|
||||
|
||||
---
|
||||
|
||||
## Source Code Location Details
|
||||
|
||||
### Where to Find the Bundled Engine Source Code
|
||||
|
||||
| Engine | Main Source Files | Config/Models | Processor/Adapter |
|
||||
|--------|-------------------|---------------|-------------------|
|
||||
| IndexTTS-2 | `engines/index_tts/indextts/` (infer.py, infer_v2.py, gpt/, s2mel/) | `engines/index_tts/indextts/` | `engines/processors/index_tts_processor.py` |
|
||||
| ChatterBox | `engines/chatterbox/chatterbox/` (models, inference) | `engines/chatterbox/models/` | `engines/adapters/chatterbox_adapter.py` |
|
||||
| ChatterBox Multilingual | `engines/chatterbox_official_23lang/` (tts.py, models/*, language_models.py) | `engines/chatterbox_official_23lang/models/` | `engines/processors/` |
|
||||
| F5-TTS | `engines/f5_tts/` (src/, model/, infer/) | `engines/f5_tts/configs/` | `engines/f5tts/f5tts.py` (adapter) |
|
||||
| Higgs Audio | `engines/higgs_audio/boson_multimodal/` (full framework) | `engines/higgs_audio/boson_multimodal/model/` | `engines/adapters/higgs_audio_adapter.py` |
|
||||
| RVC | `engines/rvc/impl/` (inference, models) | `engines/rvc/impl/lib/` | `engines/rvc/rvc_engine.py` |
|
||||
| VibeVoice | External package: `vibevoice` (PyPI) | Downloaded at runtime | `engines/vibevoice_engine/vibevoice_engine.py` |
|
||||
|
||||
### Key File Patterns to Check When Updating
|
||||
|
||||
**Inference/Generation Code:**
|
||||
- IndexTTS-2: `engines/index_tts/indextts/infer_v2.py`
|
||||
- ChatterBox: `engines/chatterbox/chatterbox/inference.py` or `tts.py`
|
||||
- F5-TTS: `engines/f5_tts/src/f5_tts/infer/`
|
||||
- Higgs Audio: `engines/higgs_audio/boson_multimodal/serve/serve_engine.py`
|
||||
- RVC: `engines/rvc/impl/infer_pack/`
|
||||
|
||||
**Model Loading:**
|
||||
- IndexTTS-2: `engines/index_tts/indextts/infer_v2.py` (model initialization)
|
||||
- ChatterBox: `engines/chatterbox/chatterbox/models/`
|
||||
- F5-TTS: `engines/f5_tts/src/f5_tts/model/`
|
||||
- Higgs Audio: `engines/higgs_audio/boson_multimodal/model/higgs_audio.py`
|
||||
- RVC: `engines/rvc/impl/infer_pack/modules.py`
|
||||
|
||||
**Audio Processing:**
|
||||
- IndexTTS-2: `engines/index_tts/indextts/infer_v2.py` (audio normalization, vocoding)
|
||||
- ChatterBox: Check audio post-processing in inference methods
|
||||
- F5-TTS: `engines/f5_tts/` audio handling
|
||||
- All: Look for clamp, normalize, scale operations
|
||||
|
||||
---
|
||||
|
||||
## Important Notes
|
||||
|
||||
- **ChatterBox and ChatterBox Multilingual** both use the same official repository: `github.com/resemble-ai/chatterbox`
|
||||
- The difference is in implementation: one focuses on single-language, the other on 23-language support
|
||||
- They share the same upstream to track for updates
|
||||
|
||||
- **VibeVoice** is NOT actively maintained - Microsoft disabled the official GitHub repository
|
||||
- Consider evaluating alternatives or pinning to a specific version
|
||||
- The wrapper depends on the external `vibevoice` PyPI package
|
||||
|
||||
- **F5-TTS** vs **f5tts** directories:
|
||||
- `f5_tts/` = Full bundled source code (official implementation from SWivid)
|
||||
- `f5tts/` = Wrapper/adapter layer only (NOT the official source code)
|
||||
|
||||
---
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. ✅ Document all official upstream repositories
|
||||
2. ✅ Identify which code is bundled vs wrapped
|
||||
3. ✅ Implement `/check_tts_bundled_updates` slash command
|
||||
4. ⏳ Create UPDATE_LOG.md for each engine (in progress)
|
||||
5. ⏳ Establish regular checking schedule (weekly/monthly)
|
||||
6. ⏳ Create integration PR template for upstream changes
|
||||
+12
-27
@@ -15,10 +15,8 @@ The **🤐 Noise or Vocal Removal** node is a powerful audio processing tool tha
|
||||
### 🏆 BEST MODELS (2024-2025)
|
||||
|
||||
**Top Tier - State of the Art:**
|
||||
- ★★★ **MELBAND/denoise_mel_band_roformer_sdr_27.99.ckpt** - BEST DENOISING (27.99 dB SDR!)
|
||||
- ★★★ **MELBAND/MelBandRoformer_fp32.safetensors** - Kijai's high-quality vocal separation
|
||||
- ★★★ **model_bs_roformer_ep_317_sdr_12.9755.ckpt** - Reliable transformer-based (12.97 dB SDR)
|
||||
- ★★☆ **MDX23C-8KFFT-InstVoc_HQ.ckpt** - Minimal artifacts (Karafan architecture)
|
||||
- ★★★ **model_bs_roformer_ep_317_sdr_12.9755.ckpt** - BEST OVERALL (12.97 dB SDR, Transformer-based SOTA)
|
||||
- ★★★ **MDX23C-8KFFT-InstVoc_HQ.ckpt** - Highest quality, minimal artifacts (Karafan architecture)
|
||||
- ★★☆ **UVR-MDX-NET-vocal_FT.onnx** - Professional vocal extraction (MDX architecture)
|
||||
|
||||
### 📂 VR MODELS (Convolutional Neural Networks)
|
||||
@@ -39,18 +37,6 @@ The **🤐 Noise or Vocal Removal** node is a powerful audio processing tool tha
|
||||
|
||||
- **MDX23C-8KFFT-InstVoc_HQ.ckpt** - ⭐ HIGHEST QUALITY (8K FFT, minimal artifacts)
|
||||
|
||||
### 🎯 MELBAND ROFORMER MODELS (State-of-the-Art 2025)
|
||||
|
||||
**Kijai's Models (Vocal Separation):**
|
||||
- **MELBAND/MelBandRoformer_fp16.safetensors** - Fast processing (456MB, FP16 precision)
|
||||
- **MELBAND/MelBandRoformer_fp32.safetensors** - Maximum quality (913MB, FP32 precision)
|
||||
|
||||
**ZFTurbo's Models (Denoising Specialists):**
|
||||
- **MELBAND/denoise_mel_band_roformer_sdr_27.99.ckpt** - ⭐ BEST DENOISING (27.99 dB SDR!)
|
||||
- **MELBAND/denoise_mel_band_roformer_aggressive_sdr_27.97.ckpt** - Aggressive denoising
|
||||
|
||||
**Note:** MelBand models use advanced transformer architecture with rotary embeddings for superior separation quality
|
||||
|
||||
## Use Case Workflows
|
||||
|
||||
### 🎤 Karaoke Creation (Vocal Removal)
|
||||
@@ -64,10 +50,9 @@ The **🤐 Noise or Vocal Removal** node is a powerful audio processing tool tha
|
||||
3. **Use:** "extracted voice/noise/echo" output (isolated vocals)
|
||||
|
||||
### 🔧 Audio Denoising
|
||||
1. **Best Quality:** `MELBAND/denoise_mel_band_roformer_sdr_27.99.ckpt` (no aggressiveness setting)
|
||||
2. **Alternative:** `UVR-DeNoise.pth` (gentle: 5-8)
|
||||
3. **Use:** "remaining" output (clean audio)
|
||||
4. **Note:** "extracted" contains removed noise
|
||||
1. **Primary:** `UVR-DeNoise.pth` (gentle: 5-8)
|
||||
2. **Use:** "remaining" output (clean audio)
|
||||
3. **Note:** "extracted" contains removed noise
|
||||
|
||||
### 💼 Professional Workflow
|
||||
1. **Separate:** `model_bs_roformer_ep_317_sdr_12.9755.ckpt`
|
||||
@@ -86,8 +71,7 @@ The **🤐 Noise or Vocal Removal** node is a powerful audio processing tool tha
|
||||
- **VR (Vocal Remover):** Fast, magnitude-only processing, good for basic separation
|
||||
- **MDX (Multi-Dimensional eXtraction):** Hybrid spectrogram/waveform, better quality preservation
|
||||
- **Karafan:** Multi-stage ensemble, state-of-the-art quality (highest CPU usage)
|
||||
- **RoFormer:** Transformer with rotary embeddings, excellent quality
|
||||
- **MelBand RoFormer:** Advanced transformer with Mel-frequency bands, current SOTA (27+ dB SDR)
|
||||
- **RoFormer:** Transformer with rotary embeddings, current SOTA
|
||||
|
||||
## Aggressiveness Settings (0-20)
|
||||
|
||||
@@ -138,11 +122,12 @@ The **🤐 Noise or Vocal Removal** node is a powerful audio processing tool tha
|
||||
|
||||
## Advanced Techniques
|
||||
|
||||
### 🚀 MODEL NOTES
|
||||
- **MelBand RoFormer** - ✅ NOW SUPPORTED! Best-in-class denoising (27+ dB SDR)
|
||||
- **SCNet models** - ⚠️ Architecture issues causing audio artifacts
|
||||
- **MDX23C models** - ⚠️ Tensor alignment errors in current implementation
|
||||
- **Note:** Aggressiveness setting only affects VR architecture models, not MelBand/MDX/Karafan
|
||||
### 🚀 NEWER MODELS (Consider adding)
|
||||
- **Mel-RoFormer** - Next-gen transformer architecture
|
||||
- **SCNet-XL** - Large-scale separation network
|
||||
- **VitLarge23** - Vision transformer adaptation
|
||||
- **Demucs v4 (htdemucs_ft)** - Hybrid transformer fine-tuned
|
||||
- **Kim Vocal models** - Highly regarded community favorites
|
||||
|
||||
### 💡 PRO TIPS
|
||||
- **Use ensemble combinations** for best results
|
||||
|
||||
@@ -1,148 +0,0 @@
|
||||
# Qwen3-TTS Optimization Guide
|
||||
|
||||
This document explains how to enable torch.compile optimizations for Qwen3-TTS to achieve ~1.5-2x speedup.
|
||||
|
||||
## Requirements
|
||||
|
||||
**CRITICAL**: torch.compile optimizations require specific versions tested with the reference implementation:
|
||||
|
||||
- **PyTorch 2.10.0+** with CUDA 13.0 support
|
||||
- **triton-windows 3.6.0+** (Windows) or **triton 3.6.0+** (Linux)
|
||||
- **Visual Studio C++ Build Tools** on Windows (a real `cl.exe` toolchain, not just stale PATH entries)
|
||||
- Python 3.12 (recommended)
|
||||
|
||||
**Note**: PyTorch 2.8.x and 2.9.x have compatibility issues with triton on Windows. Only PyTorch 2.10+ is supported.
|
||||
|
||||
### Installation
|
||||
|
||||
#### Windows (Tested Configuration)
|
||||
```bash
|
||||
# Upgrade PyTorch to 2.10 with CUDA 13.0
|
||||
pip install --upgrade torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu130
|
||||
|
||||
# Install triton-windows 3.6.x
|
||||
pip install -U "triton-windows<3.7"
|
||||
```
|
||||
|
||||
Also install **Visual Studio Build Tools** with the C++ toolchain:
|
||||
|
||||
- Workload: `Desktop development with C++`
|
||||
- Or at minimum the MSVC x64/x86 build tools component
|
||||
|
||||
If `cl.exe` is missing, `torch.compile` in the isolated Qwen runtime will fall back to the non-compiled path.
|
||||
|
||||
**Verification:**
|
||||
```bash
|
||||
python -c "import torch; print(f'PyTorch: {torch.__version__}'); import triton; print(f'Triton: {triton.__version__}')"
|
||||
```
|
||||
|
||||
Expected output:
|
||||
```
|
||||
PyTorch: 2.10.0+cu130
|
||||
Triton: 3.6.0.post25
|
||||
```
|
||||
|
||||
#### Linux
|
||||
```bash
|
||||
# Upgrade PyTorch to 2.10 with CUDA 13.0
|
||||
pip install --upgrade torch==2.10.0 torchvision==0.25.0 torchaudio==2.10.0 --index-url https://download.pytorch.org/whl/cu130
|
||||
|
||||
# triton is usually bundled with PyTorch on Linux
|
||||
# Verify version is 3.6.0+
|
||||
python -c "import triton; print(triton.__version__)"
|
||||
```
|
||||
|
||||
## Environment Configuration
|
||||
|
||||
**Main environment**: no special configuration beyond a working compiler toolchain.
|
||||
|
||||
**Shared / Dedicated Runtime isolation**:
|
||||
- TTS Audio Suite now tries to detect Visual Studio automatically and inject a proper MSVC build environment into the isolated worker
|
||||
- This is meant to preserve `torch.compile` parity with old direct Transformers 4 environments
|
||||
- It still depends on the Build Tools actually being installed
|
||||
|
||||
**Note**: The `reduce-overhead` mode (which includes automatic CUDA graphs) may still have issues with cudaMallocAsync. Use `compile_mode="default"` for best compatibility.
|
||||
|
||||
## Usage in Qwen3-TTS Engine Node
|
||||
|
||||
Once the environment is configured:
|
||||
|
||||
1. Enable `use_torch_compile`: **True**
|
||||
2. Keep `use_cuda_graphs`: **False** (not needed - reduce-overhead includes it)
|
||||
3. Set `compile_mode`: **reduce-overhead** (recommended)
|
||||
|
||||
### Settings Explanation
|
||||
|
||||
- **use_torch_compile**: Enables torch.compile optimization (~1.4-3x speedup)
|
||||
- First generation will be slower (compilation time)
|
||||
- Subsequent generations are much faster
|
||||
|
||||
- **use_cuda_graphs**: Manual CUDA graph capture
|
||||
- **Keep FALSE** when using reduce-overhead mode (it already includes automatic CUDA graphs)
|
||||
- Only enable if using `compile_mode=default` and you want manual graph capture
|
||||
|
||||
- **compile_mode**:
|
||||
- `reduce-overhead` (RECOMMENDED): Includes automatic CUDA graphs, fast compilation
|
||||
- `max-autotune`: Longer compilation, marginal gains
|
||||
- `default`: No automatic CUDA graphs
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Error: "'KernelNamespace' object has no attribute 'mm'"
|
||||
**Cause**: Incompatible PyTorch and triton-windows versions.
|
||||
|
||||
**Solution**: Upgrade to PyTorch 2.10.0 + triton-windows 3.6.0 (see Installation section).
|
||||
|
||||
### Error: "operator torchvision::nms does not exist"
|
||||
**Cause**: torchvision version doesn't match PyTorch version.
|
||||
|
||||
**Solution**:
|
||||
```bash
|
||||
pip install --upgrade torchvision --index-url https://download.pytorch.org/whl/cu130
|
||||
```
|
||||
|
||||
### Error: "cudaMallocAsync does not support checkPoolLiveAllocations"
|
||||
**Cause**: Using `compile_mode="reduce-overhead"` which enables automatic CUDA graphs.
|
||||
|
||||
**Solution**: Use `compile_mode="default"` instead (recommended for Windows).
|
||||
|
||||
### Error: "Failed to find C compiler. Please specify via CC environment variable."
|
||||
**Cause**: Triton/Inductor could not find a usable Windows C++ toolchain.
|
||||
|
||||
**Solution**:
|
||||
1. Install or repair Visual Studio Build Tools with the C++ workload
|
||||
2. Restart ComfyUI so the new environment is visible
|
||||
3. If using Shared/Dedicated Runtime, let TTS Audio Suite auto-detect the toolchain
|
||||
|
||||
If detection still fails, your machine state is broken, not just the Python package stack.
|
||||
|
||||
### Warning: "TensorFloat32 tensor cores... not enabled"
|
||||
**Info**: This is just a performance hint, not an error. TF32 is automatically enabled by the extension for better performance.
|
||||
|
||||
## Performance Expectations
|
||||
|
||||
**Tested on RTX 4090 with PyTorch 2.10.0 + triton-windows 3.6.0:**
|
||||
|
||||
| Configuration | Speed | Speedup | Notes |
|
||||
|--------------|-------|---------|-------|
|
||||
| **Baseline** (no optimizations) | 5.0 it/s | 1.0x | Standard inference |
|
||||
| **torch.compile mode=default** | 8.5 it/s | 1.7x | ✅ RECOMMENDED for Windows |
|
||||
| **+ manual CUDA graphs** | 8.6 it/s | 1.72x | Negligible gain (~0.1 it/s) |
|
||||
| **mode=reduce-overhead** | FAILS | - | ❌ cudaMallocAsync error on Windows |
|
||||
| **mode=max-autotune** | FAILS | - | ❌ cudaMallocAsync error on Windows |
|
||||
|
||||
### Key Findings
|
||||
|
||||
- **torch.compile alone provides ~1.7x speedup** with mode="default"
|
||||
- **Manual CUDA graphs add almost nothing** on top of torch.compile
|
||||
- **reduce-overhead and max-autotune fail on Windows** (work on Linux)
|
||||
- **First generation is slower** (~30-60s compilation overhead)
|
||||
- **Subsequent generations are fast** (compiled kernels cached)
|
||||
- **Model unloading works correctly** even with CUDA graphs enabled
|
||||
- **Shared Runtime should preserve these gains** if the Windows compiler toolchain is installed and detected correctly
|
||||
|
||||
## Notes
|
||||
|
||||
- The reference implementation (dffdeeq/Qwen3-TTS-streaming) uses the same approach
|
||||
- torch.compile caches compiled kernels, so subsequent ComfyUI sessions start faster
|
||||
- Model unloading works normally - no special handling needed (unlike manual CUDA graphs)
|
||||
@@ -4,7 +4,7 @@
|
||||
try:
|
||||
from .chatterbox_adapter import ChatterBoxEngineAdapter
|
||||
CHATTERBOX_ADAPTER_AVAILABLE = True
|
||||
except Exception as e:
|
||||
except ImportError as e:
|
||||
CHATTERBOX_ADAPTER_AVAILABLE = False
|
||||
# Create dummy class for compatibility
|
||||
class ChatterBoxEngineAdapter:
|
||||
@@ -14,95 +14,11 @@ except Exception as e:
|
||||
try:
|
||||
from .f5tts_adapter import F5TTSEngineAdapter
|
||||
F5TTS_ADAPTER_AVAILABLE = True
|
||||
except Exception as e:
|
||||
except ImportError as e:
|
||||
F5TTS_ADAPTER_AVAILABLE = False
|
||||
# Create dummy class for compatibility
|
||||
class F5TTSEngineAdapter:
|
||||
def __init__(self, *args, **kwargs):
|
||||
raise ImportError(f"F5-TTS adapter not available: {e}")
|
||||
|
||||
try:
|
||||
from .cosyvoice_adapter import CosyVoiceAdapter
|
||||
COSYVOICE_ADAPTER_AVAILABLE = True
|
||||
except Exception as e:
|
||||
COSYVOICE_ADAPTER_AVAILABLE = False
|
||||
# Create dummy class for compatibility
|
||||
class CosyVoiceAdapter:
|
||||
def __init__(self, *args, **kwargs):
|
||||
raise ImportError(f"CosyVoice adapter not available: {e}")
|
||||
|
||||
try:
|
||||
from .echo_tts_adapter import EchoTTSEngineAdapter
|
||||
ECHO_TTS_ADAPTER_AVAILABLE = True
|
||||
except Exception as e:
|
||||
ECHO_TTS_ADAPTER_AVAILABLE = False
|
||||
class EchoTTSEngineAdapter:
|
||||
def __init__(self, *args, **kwargs):
|
||||
raise ImportError(f"Echo-TTS adapter not available: {e}")
|
||||
|
||||
try:
|
||||
from .dots_tts_adapter import DotsTTSEngineAdapter
|
||||
DOTS_TTS_ADAPTER_AVAILABLE = True
|
||||
except Exception as e:
|
||||
DOTS_TTS_ADAPTER_AVAILABLE = False
|
||||
class DotsTTSEngineAdapter:
|
||||
def __init__(self, *args, **kwargs):
|
||||
raise ImportError(f"Dots TTS adapter not available: {e}")
|
||||
|
||||
try:
|
||||
from .dramabox_adapter import DramaBoxEngineAdapter
|
||||
DRAMABOX_ADAPTER_AVAILABLE = True
|
||||
except Exception as e:
|
||||
DRAMABOX_ADAPTER_AVAILABLE = False
|
||||
class DramaBoxEngineAdapter:
|
||||
def __init__(self, *args, **kwargs):
|
||||
raise ImportError(f"DramaBox adapter not available: {e}")
|
||||
|
||||
try:
|
||||
from .fish_audio_s2_adapter import FishAudioS2Adapter
|
||||
FISH_AUDIO_S2_ADAPTER_AVAILABLE = True
|
||||
except Exception as e:
|
||||
FISH_AUDIO_S2_ADAPTER_AVAILABLE = False
|
||||
class FishAudioS2Adapter:
|
||||
def __init__(self, *args, **kwargs):
|
||||
raise ImportError(f"Fish Audio S2 adapter not available: {e}")
|
||||
|
||||
try:
|
||||
from .omnivoice_adapter import OmniVoiceEngineAdapter
|
||||
OMNIVOICE_ADAPTER_AVAILABLE = True
|
||||
except Exception as e:
|
||||
OMNIVOICE_ADAPTER_AVAILABLE = False
|
||||
class OmniVoiceEngineAdapter:
|
||||
def __init__(self, *args, **kwargs):
|
||||
raise ImportError(f"OmniVoice adapter not available: {e}")
|
||||
|
||||
try:
|
||||
from .moss_tts_adapter import MossTTSEngineAdapter
|
||||
MOSS_TTS_ADAPTER_AVAILABLE = True
|
||||
except Exception as e:
|
||||
MOSS_TTS_ADAPTER_AVAILABLE = False
|
||||
class MossTTSEngineAdapter:
|
||||
def __init__(self, *args, **kwargs):
|
||||
raise ImportError(f"MOSS-TTS adapter not available: {e}")
|
||||
|
||||
try:
|
||||
from .higgs_audio_v3_adapter import HiggsAudioV3EngineAdapter
|
||||
HIGGS_AUDIO_V3_ADAPTER_AVAILABLE = True
|
||||
except Exception as e:
|
||||
HIGGS_AUDIO_V3_ADAPTER_AVAILABLE = False
|
||||
class HiggsAudioV3EngineAdapter:
|
||||
def __init__(self, *args, **kwargs):
|
||||
raise ImportError(f"Higgs Audio v3 adapter not available: {e}")
|
||||
|
||||
__all__ = [
|
||||
'ChatterBoxEngineAdapter', 'F5TTSEngineAdapter', 'CosyVoiceAdapter', 'EchoTTSEngineAdapter',
|
||||
'DotsTTSEngineAdapter', 'DramaBoxEngineAdapter', 'OmniVoiceEngineAdapter',
|
||||
'MossTTSEngineAdapter', 'HiggsAudioV3EngineAdapter',
|
||||
'CHATTERBOX_ADAPTER_AVAILABLE', 'F5TTS_ADAPTER_AVAILABLE', 'COSYVOICE_ADAPTER_AVAILABLE',
|
||||
'ECHO_TTS_ADAPTER_AVAILABLE', 'DOTS_TTS_ADAPTER_AVAILABLE',
|
||||
'DRAMABOX_ADAPTER_AVAILABLE', 'OMNIVOICE_ADAPTER_AVAILABLE',
|
||||
'MOSS_TTS_ADAPTER_AVAILABLE', 'HIGGS_AUDIO_V3_ADAPTER_AVAILABLE',
|
||||
'MossSoundEffectV2Adapter'
|
||||
]
|
||||
|
||||
from .moss_soundeffect_v2_adapter import MossSoundEffectV2Adapter
|
||||
__all__ = ['ChatterBoxEngineAdapter', 'F5TTSEngineAdapter', 'CHATTERBOX_ADAPTER_AVAILABLE', 'F5TTS_ADAPTER_AVAILABLE']
|
||||
@@ -1,486 +0,0 @@
|
||||
"""audio.cpp adapter for the Suite's unified ASR pipeline."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from typing import Any, Dict, Iterable, Mapping, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from utils.asr.types import ASRRequest, ASRResult, ASRSegment, ASRWord
|
||||
from utils.audio.processing import AudioProcessingUtils
|
||||
|
||||
|
||||
_NATIVE_CHUNK_FAMILIES = {
|
||||
"fun_asr_nano",
|
||||
"higgs_audio_stt",
|
||||
"hviske_asr",
|
||||
"qwen3_asr",
|
||||
"vibevoice_asr",
|
||||
"voxtral_realtime",
|
||||
}
|
||||
|
||||
# audio.cpp release-0.5.1 keeps stale offline decoder state for these loaders:
|
||||
# the first request transcribes normally and later requests return empty text.
|
||||
# A fresh owned process is currently the only reliable reset contract.
|
||||
_RESTART_BETWEEN_CHUNKS_FAMILIES = {"nemotron_asr", "voxtral_realtime"}
|
||||
|
||||
|
||||
def _session(config: Mapping[str, Any]):
|
||||
from utils.audio_cpp.session import get_audio_cpp_session
|
||||
|
||||
return get_audio_cpp_session(dict(config))
|
||||
|
||||
|
||||
def _advanced_options(config: Mapping[str, Any]) -> Dict[str, Any]:
|
||||
value = config.get("advanced_options", config.get("request_options", {}))
|
||||
if value in (None, ""):
|
||||
return {}
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
value = json.loads(value)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"Invalid audio.cpp advanced JSON: {exc.msg}") from exc
|
||||
if not isinstance(value, Mapping):
|
||||
raise ValueError("audio.cpp advanced options must be a JSON object")
|
||||
return dict(value)
|
||||
|
||||
|
||||
def _audio_path(audio: Mapping[str, Any]) -> str:
|
||||
waveform = audio.get("waveform")
|
||||
sample_rate = audio.get("sample_rate")
|
||||
if not torch.is_tensor(waveform):
|
||||
raise TypeError("audio.cpp ASR input must contain a waveform tensor")
|
||||
if sample_rate is None or int(sample_rate) <= 0:
|
||||
raise ValueError("audio.cpp ASR input must contain a positive sample_rate")
|
||||
return os.path.abspath(
|
||||
AudioProcessingUtils.save_audio_to_temp_file(waveform, int(sample_rate))
|
||||
)
|
||||
|
||||
|
||||
def _waveform_3d(audio: Mapping[str, Any]) -> tuple[torch.Tensor, int]:
|
||||
waveform = audio.get("waveform")
|
||||
sample_rate = int(audio.get("sample_rate") or 0)
|
||||
if not torch.is_tensor(waveform):
|
||||
raise TypeError("audio.cpp ASR input must contain a waveform tensor")
|
||||
if sample_rate <= 0:
|
||||
raise ValueError("audio.cpp ASR input must contain a positive sample_rate")
|
||||
if waveform.ndim == 1:
|
||||
waveform = waveform.unsqueeze(0).unsqueeze(0)
|
||||
elif waveform.ndim == 2:
|
||||
waveform = waveform.unsqueeze(0)
|
||||
elif waveform.ndim != 3:
|
||||
raise ValueError(
|
||||
"audio.cpp ASR waveform must have [samples], [channels, samples], or "
|
||||
"[batch, channels, samples] shape"
|
||||
)
|
||||
if waveform.shape[0] != 1:
|
||||
raise ValueError("audio.cpp ASR accepts one audio item at a time")
|
||||
if waveform.shape[-1] <= 0:
|
||||
raise ValueError("audio.cpp ASR input audio is empty")
|
||||
return waveform.detach().cpu(), sample_rate
|
||||
|
||||
|
||||
def _chunk_ranges(
|
||||
total_samples: int,
|
||||
sample_rate: int,
|
||||
chunk_size: int,
|
||||
overlap: int,
|
||||
) -> list[tuple[int, int]]:
|
||||
if chunk_size <= 0:
|
||||
return [(0, total_samples)]
|
||||
if overlap < 0:
|
||||
raise ValueError("ASR overlap must be zero or greater")
|
||||
if overlap >= chunk_size:
|
||||
raise ValueError("ASR overlap must be smaller than chunk_size")
|
||||
|
||||
chunk_samples = chunk_size * sample_rate
|
||||
if total_samples <= chunk_samples:
|
||||
return [(0, total_samples)]
|
||||
step_samples = (chunk_size - overlap) * sample_rate
|
||||
ranges = []
|
||||
start = 0
|
||||
while start < total_samples:
|
||||
end = min(start + chunk_samples, total_samples)
|
||||
ranges.append((start, end))
|
||||
if end >= total_samples:
|
||||
break
|
||||
start += step_samples
|
||||
return ranges
|
||||
|
||||
|
||||
def _normalized_token(value: str) -> str:
|
||||
return re.sub(r"[^\w]+", "", value, flags=re.UNICODE).casefold()
|
||||
|
||||
|
||||
def _merge_transcript(parts: Iterable[str]) -> str:
|
||||
merged: list[str] = []
|
||||
for part in parts:
|
||||
incoming = str(part or "").strip().split()
|
||||
if not incoming:
|
||||
continue
|
||||
if not merged:
|
||||
merged.extend(incoming)
|
||||
continue
|
||||
limit = min(len(merged), len(incoming), 80)
|
||||
duplicate_count = 0
|
||||
for size in range(limit, 0, -1):
|
||||
left = [_normalized_token(token) for token in merged[-size:]]
|
||||
right = [_normalized_token(token) for token in incoming[:size]]
|
||||
if all(left) and left == right:
|
||||
duplicate_count = size
|
||||
break
|
||||
merged.extend(incoming[duplicate_count:])
|
||||
return " ".join(merged).strip()
|
||||
|
||||
|
||||
def _offset_words(
|
||||
words: Iterable[ASRWord], offset: float, unique_after: Optional[float]
|
||||
) -> list[ASRWord]:
|
||||
shifted = []
|
||||
for word in words:
|
||||
item = ASRWord(start=word.start + offset, end=word.end + offset, text=word.text)
|
||||
if unique_after is not None and (item.start + item.end) / 2.0 < unique_after:
|
||||
continue
|
||||
shifted.append(item)
|
||||
return shifted
|
||||
|
||||
|
||||
def _offset_segments(
|
||||
segments: Iterable[ASRSegment], offset: float, unique_after: Optional[float]
|
||||
) -> list[ASRSegment]:
|
||||
shifted = []
|
||||
for segment in segments:
|
||||
item = ASRSegment(
|
||||
start=segment.start + offset,
|
||||
end=segment.end + offset,
|
||||
text=segment.text,
|
||||
speaker=segment.speaker,
|
||||
)
|
||||
if unique_after is not None and (item.start + item.end) / 2.0 < unique_after:
|
||||
continue
|
||||
shifted.append(item)
|
||||
return shifted
|
||||
|
||||
|
||||
def _seconds(value: Any, sample_rate: int) -> float:
|
||||
try:
|
||||
return max(0.0, float(value) / float(sample_rate))
|
||||
except (TypeError, ValueError, ZeroDivisionError):
|
||||
return 0.0
|
||||
|
||||
|
||||
def _words(payload: Mapping[str, Any], sample_rate: int) -> list[ASRWord]:
|
||||
words = []
|
||||
for item in payload.get("words") or []:
|
||||
if not isinstance(item, Mapping):
|
||||
continue
|
||||
text = str(item.get("word", item.get("text", ""))).strip()
|
||||
if not text:
|
||||
continue
|
||||
words.append(
|
||||
ASRWord(
|
||||
start=_seconds(item.get("start_sample"), sample_rate),
|
||||
end=_seconds(item.get("end_sample"), sample_rate),
|
||||
text=text,
|
||||
)
|
||||
)
|
||||
return words
|
||||
|
||||
|
||||
def _plain_segments(payload: Mapping[str, Any], sample_rate: int) -> list[ASRSegment]:
|
||||
segments = []
|
||||
for item in payload.get("segments") or []:
|
||||
if not isinstance(item, Mapping):
|
||||
continue
|
||||
text = str(item.get("text", "")).strip()
|
||||
segments.append(
|
||||
ASRSegment(
|
||||
start=_seconds(item.get("start_sample"), sample_rate),
|
||||
end=_seconds(item.get("end_sample"), sample_rate),
|
||||
text=text,
|
||||
)
|
||||
)
|
||||
return segments
|
||||
|
||||
|
||||
def _speaker_segments(payload: Mapping[str, Any], sample_rate: int) -> list[ASRSegment]:
|
||||
segments = []
|
||||
for item in payload.get("speaker_turns") or []:
|
||||
if not isinstance(item, Mapping):
|
||||
continue
|
||||
speaker = str(item.get("speaker_id", "")).strip()
|
||||
if speaker and not speaker.lower().startswith("speaker"):
|
||||
speaker = f"Speaker {speaker}"
|
||||
segments.append(
|
||||
ASRSegment(
|
||||
start=_seconds(item.get("start_sample"), sample_rate),
|
||||
end=_seconds(item.get("end_sample"), sample_rate),
|
||||
text=str(item.get("text", "")).strip(),
|
||||
speaker=speaker or None,
|
||||
)
|
||||
)
|
||||
return segments
|
||||
|
||||
|
||||
def _attach_words(segments: Iterable[ASRSegment], words: Iterable[ASRWord]) -> None:
|
||||
segment_list = list(segments)
|
||||
for word in words:
|
||||
midpoint = (word.start + word.end) / 2.0
|
||||
target = next(
|
||||
(segment for segment in segment_list if segment.start <= midpoint <= segment.end),
|
||||
None,
|
||||
)
|
||||
if target is not None:
|
||||
target.words.append(word)
|
||||
|
||||
|
||||
class AudioCppASREngineAdapter:
|
||||
"""Normalize audio.cpp transcript/timing output into ``ASRResult``."""
|
||||
|
||||
def __init__(self, engine_data: Dict[str, Any]):
|
||||
self.engine_data = dict(engine_data)
|
||||
self.config = dict(engine_data.get("config", engine_data))
|
||||
|
||||
def _session_config(self) -> Dict[str, Any]:
|
||||
config = dict(self.config)
|
||||
if str(config.get("connection_mode", "auto")).lower() != "external_server":
|
||||
config["requested_task"] = "asr"
|
||||
config["task"] = "asr"
|
||||
return config
|
||||
|
||||
def transcribe(self, req: ASRRequest) -> ASRResult:
|
||||
if req.task != "transcribe":
|
||||
raise ValueError(
|
||||
"audio.cpp release-0.5.1 ASR loaders support transcription, not the "
|
||||
"Unified ASR translate mode"
|
||||
)
|
||||
|
||||
config = self._session_config()
|
||||
family = str(config.get("family", "")).strip()
|
||||
warnings: list[str] = []
|
||||
notes: list[str] = []
|
||||
options = _advanced_options(config)
|
||||
|
||||
# VibeVoice-ASR owns diarization across its full recording. Independent
|
||||
# Suite requests can restart speaker numbering, so preserve its native
|
||||
# chunking only for this mode. All other ASR uses Suite-side windows.
|
||||
native_diarization = (
|
||||
family == "vibevoice_asr" and req.diarization and req.chunk_size > 0
|
||||
)
|
||||
if native_diarization:
|
||||
options.setdefault("audio_chunk_mode", "fixed")
|
||||
options.setdefault("audio_chunk_seconds", int(req.chunk_size))
|
||||
if req.overlap > 0:
|
||||
notes.append(
|
||||
"VibeVoice-ASR diarization uses native chunking to preserve speaker "
|
||||
"identity; the Suite overlap setting is not applied."
|
||||
)
|
||||
elif family in _NATIVE_CHUNK_FAMILIES:
|
||||
options.setdefault("audio_chunk_mode", "none")
|
||||
|
||||
if req.timestamps == "word" and family == "qwen3_asr":
|
||||
session_options = config.get("session_options") or {}
|
||||
aligner = session_options.get("qwen3_asr.forced_aligner_model_path")
|
||||
if aligner:
|
||||
options["return_timestamps"] = True
|
||||
else:
|
||||
warnings.append(
|
||||
"Qwen3-ASR word timestamps require the optional Qwen3 Forced Aligner; "
|
||||
"transcription continued without downloading that auxiliary model."
|
||||
)
|
||||
|
||||
waveform, source_rate = _waveform_3d(req.audio)
|
||||
ranges = (
|
||||
[(0, waveform.shape[-1])]
|
||||
if native_diarization
|
||||
else _chunk_ranges(
|
||||
waveform.shape[-1], source_rate, int(req.chunk_size), int(req.overlap)
|
||||
)
|
||||
)
|
||||
session = _session(config)
|
||||
if str(getattr(session, "task", "asr")) != "asr":
|
||||
raise ValueError(
|
||||
f"audio.cpp model '{session.model_id}' is configured for task "
|
||||
f"'{session.task}', not ASR"
|
||||
)
|
||||
restart_between_chunks = (
|
||||
len(ranges) > 1 and family in _RESTART_BETWEEN_CHUNKS_FAMILIES
|
||||
)
|
||||
if restart_between_chunks and not bool(getattr(session, "owned", False)):
|
||||
raise RuntimeError(
|
||||
f"audio.cpp release-0.5.1 {family} returns empty text after its first "
|
||||
"offline request. Suite-side chunking therefore requires a managed "
|
||||
"audio.cpp server so the Suite can reset it between chunks. Set "
|
||||
"connection_mode to managed, or set ASR chunk_size to 0 when using "
|
||||
"an external server."
|
||||
)
|
||||
if restart_between_chunks:
|
||||
notes.append(
|
||||
f"audio.cpp release-0.5.1 {family} requires a managed server reset "
|
||||
"between Suite chunks to avoid empty repeated-request results."
|
||||
)
|
||||
|
||||
display_family = family or "external model"
|
||||
print(f"🎧 audio.cpp ASR: Transcribing with {display_family}...")
|
||||
if len(ranges) > 1:
|
||||
notes.append(
|
||||
f"Suite-side ASR chunking used {len(ranges)} windows of "
|
||||
f"{int(req.chunk_size)}s with {int(req.overlap)}s overlap."
|
||||
)
|
||||
print(
|
||||
f"🧩 audio.cpp ASR: {len(ranges)} chunks "
|
||||
f"({int(req.chunk_size)}s, {int(req.overlap)}s overlap)"
|
||||
)
|
||||
|
||||
payloads: list[Mapping[str, Any]] = []
|
||||
chunk_timings: list[Mapping[str, Any]] = []
|
||||
chunk_diagnostics: list[Dict[str, Any]] = []
|
||||
started_at = time.time()
|
||||
for index, (start, end) in enumerate(ranges, start=1):
|
||||
if index > 1 and restart_between_chunks:
|
||||
print(
|
||||
f"🔄 audio.cpp ASR: Resetting {family} session for chunk "
|
||||
f"{index}/{len(ranges)}"
|
||||
)
|
||||
session.restart_owned_runtime()
|
||||
chunk_waveform = waveform[..., start:end]
|
||||
chunk_rms = float(torch.sqrt(torch.mean(chunk_waveform.float().square())).item())
|
||||
chunk_peak = float(chunk_waveform.float().abs().max().item())
|
||||
temp_path = _audio_path({
|
||||
"waveform": chunk_waveform,
|
||||
"sample_rate": source_rate,
|
||||
})
|
||||
try:
|
||||
request: Dict[str, Any] = {"audio": temp_path, "options": dict(options)}
|
||||
if req.language:
|
||||
request["language"] = req.language
|
||||
result = session.run(request)
|
||||
payload = result.raw if isinstance(result.raw, Mapping) else {}
|
||||
payloads.append(payload)
|
||||
if isinstance(payload.get("timing"), Mapping):
|
||||
chunk_timings.append(payload["timing"])
|
||||
chunk_diagnostics.append({
|
||||
"index": index,
|
||||
"start": round(start / source_rate, 3),
|
||||
"end": round(end / source_rate, 3),
|
||||
"rms": round(chunk_rms, 6),
|
||||
"peak": round(chunk_peak, 6),
|
||||
"text": str(payload.get("text", "")).strip(),
|
||||
"characters": len(str(payload.get("text", "")).strip()),
|
||||
"upstream_timing": (
|
||||
dict(payload["timing"])
|
||||
if isinstance(payload.get("timing"), Mapping)
|
||||
else None
|
||||
),
|
||||
})
|
||||
finally:
|
||||
try:
|
||||
os.remove(temp_path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
if len(ranges) > 1:
|
||||
chunk_chars = len(str(payload.get("text", "")).strip())
|
||||
print(
|
||||
f" ASR chunk {index}/{len(ranges)} complete "
|
||||
f"({chunk_chars} chars, RMS {chunk_rms:.4f}, peak {chunk_peak:.4f})"
|
||||
)
|
||||
|
||||
words: list[ASRWord] = []
|
||||
speaker_segments: list[ASRSegment] = []
|
||||
plain_segments: list[ASRSegment] = []
|
||||
overlap_seconds = float(req.overlap) if len(ranges) > 1 else 0.0
|
||||
for index, ((start, _end), payload) in enumerate(zip(ranges, payloads)):
|
||||
offset = start / source_rate
|
||||
unique_after = offset + overlap_seconds if index > 0 else None
|
||||
words.extend(_offset_words(_words(payload, source_rate), offset, unique_after))
|
||||
speaker_segments.extend(
|
||||
_offset_segments(
|
||||
_speaker_segments(payload, source_rate), offset, unique_after
|
||||
)
|
||||
)
|
||||
plain_segments.extend(
|
||||
_offset_segments(_plain_segments(payload, source_rate), offset, unique_after)
|
||||
)
|
||||
|
||||
if req.diarization:
|
||||
segments = speaker_segments
|
||||
if segments:
|
||||
_attach_words(segments, words)
|
||||
else:
|
||||
warnings.append(
|
||||
f"audio.cpp {family or 'ASR model'} returned no speaker-attributed turns."
|
||||
)
|
||||
segments = plain_segments
|
||||
elif req.timestamps == "word" and words:
|
||||
segments = [
|
||||
ASRSegment(start=word.start, end=word.end, text=word.text, words=[word])
|
||||
for word in words
|
||||
]
|
||||
elif req.timestamps == "word":
|
||||
segments = plain_segments
|
||||
else:
|
||||
segments = []
|
||||
|
||||
text = _merge_transcript(payload.get("text", "") for payload in payloads)
|
||||
if req.diarization and speaker_segments:
|
||||
text = " ".join(
|
||||
f"[{segment.speaker}] {segment.text}" if segment.speaker else segment.text
|
||||
for segment in speaker_segments
|
||||
if segment.text
|
||||
).strip()
|
||||
if not text and speaker_segments:
|
||||
text = " ".join(segment.text for segment in speaker_segments if segment.text).strip()
|
||||
if req.timestamps == "word" and not words:
|
||||
warnings.append(f"audio.cpp {family or 'ASR model'} returned no word timestamps.")
|
||||
empty_chunks = sum(
|
||||
1 for payload in payloads if not str(payload.get("text", "")).strip()
|
||||
)
|
||||
if len(payloads) > 1 and empty_chunks:
|
||||
warnings.append(
|
||||
f"audio.cpp {family or 'ASR model'} returned no text for "
|
||||
f"{empty_chunks} of {len(payloads)} Suite chunks."
|
||||
)
|
||||
|
||||
raw: Dict[str, Any] = {}
|
||||
if warnings:
|
||||
raw["warnings"] = warnings
|
||||
if notes:
|
||||
raw["notes"] = notes
|
||||
if len(payloads) == 1 and chunk_timings:
|
||||
raw["timing"] = dict(chunk_timings[0])
|
||||
elif len(payloads) > 1:
|
||||
raw["timing"] = {
|
||||
"wall_ms": round((time.time() - started_at) * 1000.0, 3),
|
||||
"suite_chunks": len(payloads),
|
||||
"suite_chunk_size_seconds": int(req.chunk_size),
|
||||
"suite_overlap_seconds": int(req.overlap),
|
||||
"upstream_wall_ms": round(
|
||||
sum(float(item.get("wall_ms", 0.0)) for item in chunk_timings), 3
|
||||
),
|
||||
}
|
||||
raw["chunks"] = chunk_diagnostics
|
||||
output_language = next(
|
||||
(
|
||||
str(payload.get("language", "")).strip()
|
||||
for payload in payloads
|
||||
if str(payload.get("language", "")).strip()
|
||||
),
|
||||
str(req.language or "").strip(),
|
||||
) or None
|
||||
print(
|
||||
f"✅ audio.cpp ASR: Complete ({len(text)} chars, "
|
||||
f"{len(segments)} timed/speaker segments)"
|
||||
)
|
||||
return ASRResult(
|
||||
text=text,
|
||||
language=output_language,
|
||||
segments=segments,
|
||||
raw=raw or None,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["AudioCppASREngineAdapter"]
|
||||
@@ -1,711 +0,0 @@
|
||||
"""
|
||||
Granite ASR adapter for unified ASR pipeline.
|
||||
Uses Granite speech for transcription/translation and optionally Qwen forced aligner for timestamps.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
import comfy.model_management as model_management
|
||||
import comfy.utils
|
||||
import time
|
||||
import re
|
||||
|
||||
from engines.granite_asr.prompting import DEFAULT_TRANSLATE_INSTRUCTION_TEMPLATE
|
||||
from utils.asr.types import ASRRequest, ASRResult, ASRSegment, ASRWord
|
||||
|
||||
|
||||
def parse_granite_native_timestamps(ts_text: str, chunk_offset: float) -> Tuple[str, List[ASRSegment]]:
|
||||
"""
|
||||
Parses native timestamped text generated by ibm-granite/granite-speech-4.1-2b-plus.
|
||||
Reconstructs absolute timestamps by unwrapping modulo-1000 rollovers.
|
||||
"""
|
||||
if not ts_text or not ts_text.strip():
|
||||
return "", []
|
||||
|
||||
ts_words = re.split(r"\[T:([^\]]+)\]", ts_text)
|
||||
last_word_end_time = 0.0
|
||||
offset_time = 0.0
|
||||
|
||||
segments: List[ASRSegment] = []
|
||||
plain_words: List[str] = []
|
||||
|
||||
for raw_word, ts_str in zip(ts_words[::2], ts_words[1::2]):
|
||||
word_text = raw_word.strip()
|
||||
if not ts_str:
|
||||
continue
|
||||
|
||||
try:
|
||||
word_end_time = float(ts_str) / 100.0
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
while word_end_time + offset_time < last_word_end_time:
|
||||
offset_time += 10.0
|
||||
absolute_end_time = word_end_time + offset_time
|
||||
|
||||
absolute_start_time = last_word_end_time
|
||||
last_word_end_time = absolute_end_time
|
||||
|
||||
if word_text == "_":
|
||||
continue
|
||||
|
||||
if word_text:
|
||||
plain_words.append(word_text)
|
||||
word = ASRWord(
|
||||
start=absolute_start_time + chunk_offset,
|
||||
end=absolute_end_time + chunk_offset,
|
||||
text=word_text,
|
||||
)
|
||||
segments.append(
|
||||
ASRSegment(
|
||||
start=word.start,
|
||||
end=word.end,
|
||||
text=word.text,
|
||||
words=[word],
|
||||
)
|
||||
)
|
||||
|
||||
plain_text = " ".join(plain_words)
|
||||
return plain_text, segments
|
||||
|
||||
|
||||
def parse_granite_diarization(text: str, chunk_offset: float, chunk_duration: float) -> Tuple[str, List[ASRSegment]]:
|
||||
"""
|
||||
Parses speaker-attributed ASR text generated by ibm-granite/granite-speech-4.1-2b-plus.
|
||||
Returns:
|
||||
plain_text: Cleaned text without speaker tags.
|
||||
segments: List of ASRSegment instances with speaker property populated.
|
||||
"""
|
||||
if not text or not text.strip():
|
||||
return "", []
|
||||
|
||||
# Find speaker tags like [Speaker 1]:, [Speaker 2]:, etc.
|
||||
# Keep the delimiter matching group so we can reconstruct who is speaking.
|
||||
pattern = r"\[(Speaker \d+)\]:?"
|
||||
parts = re.split(pattern, text)
|
||||
|
||||
segments: List[ASRSegment] = []
|
||||
|
||||
# Handle text before the first speaker tag, if any
|
||||
first_text = parts[0].strip()
|
||||
if first_text:
|
||||
segments.append(
|
||||
ASRSegment(
|
||||
start=chunk_offset,
|
||||
end=chunk_offset + chunk_duration,
|
||||
text=first_text,
|
||||
speaker=None,
|
||||
words=[],
|
||||
)
|
||||
)
|
||||
|
||||
for speaker, content in zip(parts[1::2], parts[2::2]):
|
||||
segment_text = content.strip()
|
||||
if not segment_text:
|
||||
continue
|
||||
segments.append(
|
||||
ASRSegment(
|
||||
start=chunk_offset,
|
||||
end=chunk_offset + chunk_duration,
|
||||
text=segment_text,
|
||||
speaker=speaker,
|
||||
words=[],
|
||||
)
|
||||
)
|
||||
|
||||
plain_text = " ".join(seg.text for seg in segments if seg.text).strip()
|
||||
plain_text = re.sub(r"\s+", " ", plain_text).strip()
|
||||
|
||||
# Distribute the chunk_duration proportionally across segments based on character length
|
||||
total_len = sum(len(seg.text) for seg in segments if seg.text)
|
||||
if total_len > 0:
|
||||
current_time = chunk_offset
|
||||
for seg in segments:
|
||||
seg_len = len(seg.text) if seg.text else 0
|
||||
duration = chunk_duration * (seg_len / total_len)
|
||||
seg.start = current_time
|
||||
seg.end = current_time + duration
|
||||
current_time += duration
|
||||
|
||||
return plain_text, segments
|
||||
|
||||
|
||||
def format_granite_diarization_segments(segments: List[ASRSegment]) -> str:
|
||||
"""
|
||||
Normalize Granite speaker attribution to the suite's canonical bracket format:
|
||||
[Speaker 1] text
|
||||
"""
|
||||
parts: List[str] = []
|
||||
for seg in segments:
|
||||
text = (seg.text or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
if seg.speaker:
|
||||
parts.append(f"[{seg.speaker}] {text}")
|
||||
else:
|
||||
parts.append(text)
|
||||
return " ".join(parts).strip()
|
||||
|
||||
|
||||
def remap_words_to_speaker_segments(speaker_segments: List[ASRSegment], aligned_words: List[ASRWord]) -> None:
|
||||
"""
|
||||
Distributes aligned words sequentially to speaker_segments based on text matching.
|
||||
"""
|
||||
if not speaker_segments or not aligned_words:
|
||||
return
|
||||
|
||||
word_idx = 0
|
||||
num_words = len(aligned_words)
|
||||
|
||||
for segment in speaker_segments:
|
||||
# Tokenize segment text to find expected words
|
||||
expected_tokens = [w.lower() for w in re.findall(r"\b\w+\b", segment.text) if w]
|
||||
if not expected_tokens:
|
||||
continue
|
||||
|
||||
segment_words = []
|
||||
match_count = 0
|
||||
limit = len(expected_tokens)
|
||||
|
||||
while word_idx < num_words and match_count < limit:
|
||||
aligned_word = aligned_words[word_idx]
|
||||
aligned_token_clean = "".join(ch for ch in aligned_word.text.lower() if ch.isalnum())
|
||||
expected_token_clean = "".join(ch for ch in expected_tokens[match_count] if ch.isalnum())
|
||||
|
||||
if not aligned_token_clean:
|
||||
segment_words.append(aligned_word)
|
||||
word_idx += 1
|
||||
continue
|
||||
|
||||
if (aligned_token_clean == expected_token_clean or
|
||||
expected_token_clean in aligned_token_clean or
|
||||
aligned_token_clean in expected_token_clean):
|
||||
segment_words.append(aligned_word)
|
||||
word_idx += 1
|
||||
match_count += 1
|
||||
elif (match_count + 1 < limit and
|
||||
("".join(ch for ch in expected_tokens[match_count + 1] if ch.isalnum()) in aligned_token_clean or
|
||||
aligned_token_clean in "".join(ch for ch in expected_tokens[match_count + 1] if ch.isalnum()))):
|
||||
# Missed an expected word in alignment, skip the expected word and match current aligned to next expected
|
||||
match_count += 2
|
||||
segment_words.append(aligned_word)
|
||||
word_idx += 1
|
||||
else:
|
||||
# Mismatch, consume to avoid getting stuck
|
||||
segment_words.append(aligned_word)
|
||||
word_idx += 1
|
||||
match_count += 1
|
||||
|
||||
if segment_words:
|
||||
segment.words = segment_words
|
||||
segment.start = segment_words[0].start
|
||||
segment.end = segment_words[-1].end
|
||||
|
||||
# If there are leftover words, append them to the last segment that had some words
|
||||
if word_idx < num_words and speaker_segments:
|
||||
last_seg = speaker_segments[-1]
|
||||
if last_seg.words:
|
||||
last_seg.words.extend(aligned_words[word_idx:])
|
||||
last_seg.end = aligned_words[-1].end
|
||||
else:
|
||||
last_seg.words = aligned_words[word_idx:]
|
||||
last_seg.start = aligned_words[word_idx].start
|
||||
last_seg.end = aligned_words[-1].end
|
||||
|
||||
|
||||
class GraniteASREngineAdapter:
|
||||
GRANITE_LANGUAGES = {
|
||||
"english": "English",
|
||||
"french": "French",
|
||||
"german": "German",
|
||||
"spanish": "Spanish",
|
||||
"portuguese": "Portuguese",
|
||||
"japanese": "Japanese",
|
||||
}
|
||||
|
||||
QWEN_ALIGNER_LANGUAGES = {
|
||||
"Chinese",
|
||||
"English",
|
||||
"Cantonese",
|
||||
"French",
|
||||
"German",
|
||||
"Italian",
|
||||
"Japanese",
|
||||
"Korean",
|
||||
"Portuguese",
|
||||
"Russian",
|
||||
"Spanish",
|
||||
}
|
||||
|
||||
GENERIC_SPACE_ALIGNER_LANGUAGE = "English"
|
||||
|
||||
def __init__(self, engine_data: Dict[str, Any]):
|
||||
self.engine_data = engine_data
|
||||
self.config = engine_data.get("config", engine_data)
|
||||
|
||||
def _normalize_language(self, language: Optional[str]) -> Optional[str]:
|
||||
if not language or str(language).lower() == "auto":
|
||||
return None
|
||||
normalized = self.GRANITE_LANGUAGES.get(str(language).strip().lower())
|
||||
model_name = self.config.get("model_name", "")
|
||||
if normalized == "Japanese" and "plus" in str(model_name).lower():
|
||||
return None
|
||||
return normalized
|
||||
|
||||
def _get_model(self):
|
||||
from utils.models.unified_model_interface import unified_model_interface, ModelLoadConfig
|
||||
from utils.device import resolve_torch_device
|
||||
|
||||
device = self.config.get("device", "auto")
|
||||
if model_management.cpu_mode():
|
||||
device = "cpu"
|
||||
elif device != "cpu":
|
||||
device = resolve_torch_device(device)
|
||||
|
||||
config = ModelLoadConfig(
|
||||
engine_name="granite_asr",
|
||||
model_type="asr",
|
||||
model_name=self.config.get("model_name", "granite-speech-4.1-2b"),
|
||||
model_path=self.config.get("model_name", "granite-speech-4.1-2b"),
|
||||
device=device,
|
||||
additional_params={
|
||||
"precision": self.config.get("dtype", "auto"),
|
||||
"attn_implementation": self.config.get("attn_implementation", "auto"),
|
||||
},
|
||||
)
|
||||
return unified_model_interface.load_model(config)
|
||||
|
||||
def _get_forced_aligner(self):
|
||||
from utils.models.unified_model_interface import unified_model_interface, ModelLoadConfig
|
||||
from utils.models.factory_config import RUNTIME_MODE_SHARED
|
||||
from utils.device import resolve_torch_device
|
||||
|
||||
device = self.config.get("device", "auto")
|
||||
if model_management.cpu_mode():
|
||||
device = "cpu"
|
||||
elif device != "cpu":
|
||||
device = resolve_torch_device(device)
|
||||
|
||||
config = ModelLoadConfig(
|
||||
engine_name="qwen3_asr",
|
||||
model_type="aligner",
|
||||
model_name="Qwen3-ForcedAligner-0.6B",
|
||||
model_path="Qwen3-ForcedAligner-0.6B",
|
||||
device=device,
|
||||
runtime_mode=self.config.get("forced_aligner_runtime_mode", RUNTIME_MODE_SHARED),
|
||||
runtime_profile=self.config.get("forced_aligner_runtime_profile"),
|
||||
additional_params={
|
||||
"precision": self.config.get("dtype", "auto"),
|
||||
"attn_implementation": self.config.get("attn_implementation", "auto"),
|
||||
},
|
||||
)
|
||||
return unified_model_interface.load_model(config)
|
||||
|
||||
def _prepare_audio(self, audio: Dict[str, Any]) -> torch.Tensor:
|
||||
waveform = audio["waveform"]
|
||||
sample_rate = audio["sample_rate"]
|
||||
|
||||
if waveform.ndim == 3:
|
||||
waveform = waveform[0]
|
||||
if waveform.shape[0] > 1:
|
||||
waveform = torch.mean(waveform, dim=0)
|
||||
else:
|
||||
waveform = waveform[0]
|
||||
|
||||
if sample_rate != 16000:
|
||||
import torchaudio.functional as TA_F
|
||||
waveform = TA_F.resample(waveform, orig_freq=sample_rate, new_freq=16000)
|
||||
return waveform
|
||||
|
||||
def _alignment_requested(self, req: ASRRequest) -> bool:
|
||||
return req.timestamps == "word"
|
||||
|
||||
def _can_use_aligner(self, req: ASRRequest, normalized_language: Optional[str], warnings: List[str]) -> bool:
|
||||
if not self._alignment_requested(req):
|
||||
return False
|
||||
|
||||
if req.task != "transcribe":
|
||||
warnings.append("Granite timestamps skipped: Qwen forced aligner only works for transcription, not translation.")
|
||||
return False
|
||||
|
||||
if not self.config.get("asr_use_forced_aligner", False):
|
||||
warnings.append("Granite timestamps skipped: Qwen forced aligner is disabled on the Granite engine node.")
|
||||
return False
|
||||
|
||||
if normalized_language and normalized_language not in self.QWEN_ALIGNER_LANGUAGES:
|
||||
warnings.append(f"Granite timestamps skipped: Qwen forced aligner does not support '{normalized_language}'.")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def _append_once(self, messages: List[str], message: str) -> None:
|
||||
if message not in messages:
|
||||
messages.append(message)
|
||||
|
||||
def _contains_japanese_script(self, text: str) -> bool:
|
||||
for ch in text:
|
||||
code = ord(ch)
|
||||
if 0x3040 <= code <= 0x309F: # Hiragana
|
||||
return True
|
||||
if 0x30A0 <= code <= 0x30FF: # Katakana
|
||||
return True
|
||||
if 0x31F0 <= code <= 0x31FF: # Katakana Phonetic Extensions
|
||||
return True
|
||||
if 0xFF66 <= code <= 0xFF9D: # Halfwidth Katakana
|
||||
return True
|
||||
if 0x3400 <= code <= 0x4DBF: # CJK Unified Ideographs Extension A
|
||||
return True
|
||||
if 0x4E00 <= code <= 0x9FFF: # CJK Unified Ideographs
|
||||
return True
|
||||
if 0xF900 <= code <= 0xFAFF: # CJK Compatibility Ideographs
|
||||
return True
|
||||
return False
|
||||
|
||||
def _resolve_aligner_language(
|
||||
self,
|
||||
normalized_language: Optional[str],
|
||||
transcript: str,
|
||||
notes: List[str],
|
||||
) -> str:
|
||||
if normalized_language:
|
||||
return normalized_language
|
||||
|
||||
if self._contains_japanese_script(transcript):
|
||||
self._append_once(
|
||||
notes,
|
||||
"Granite Auto alignment used a Japanese-script heuristic, so Qwen forced aligner ran in Japanese mode. This is tokenizer routing for alignment, not Granite native language detection.",
|
||||
)
|
||||
return "Japanese"
|
||||
|
||||
self._append_once(
|
||||
notes,
|
||||
"Granite Auto alignment used Qwen's generic space-delimited tokenizer path. 'English' here is only the aligner mode for non-Japanese Granite languages, not detected spoken language.",
|
||||
)
|
||||
return self.GENERIC_SPACE_ALIGNER_LANGUAGE
|
||||
|
||||
def _align_words(
|
||||
self,
|
||||
aligner,
|
||||
chunk_audio,
|
||||
transcript: str,
|
||||
language: str,
|
||||
chunk_offset: float,
|
||||
warnings: List[str],
|
||||
) -> List[ASRSegment]:
|
||||
if not transcript.strip():
|
||||
return []
|
||||
|
||||
try:
|
||||
results = aligner.align(audio=[(chunk_audio, 16000)], text=[transcript], language=[language])
|
||||
if not results:
|
||||
return []
|
||||
|
||||
segments: List[ASRSegment] = []
|
||||
for item in results[0]:
|
||||
word = ASRWord(
|
||||
start=float(item.start_time) + chunk_offset,
|
||||
end=float(item.end_time) + chunk_offset,
|
||||
text=str(item.text),
|
||||
)
|
||||
segments.append(
|
||||
ASRSegment(
|
||||
start=word.start,
|
||||
end=word.end,
|
||||
text=word.text,
|
||||
words=[word],
|
||||
)
|
||||
)
|
||||
return segments
|
||||
except Exception as e:
|
||||
warnings.append(f"Granite timestamps skipped: Qwen forced aligner failed ({e}).")
|
||||
return []
|
||||
|
||||
def _resolve_max_new_tokens(
|
||||
self,
|
||||
req: ASRRequest,
|
||||
*,
|
||||
chunk_num_samples: int,
|
||||
sample_rate: int,
|
||||
timestamps: bool = False,
|
||||
diarization: bool = False,
|
||||
) -> int:
|
||||
base_max_new_tokens = int(self.config.get("max_new_tokens", req.max_new_tokens or 200))
|
||||
chunk_seconds = max(1.0, float(chunk_num_samples) / float(sample_rate))
|
||||
|
||||
# Native timestamp mode expands output heavily because each spoken word is
|
||||
# followed by a [T:NNN] tag. Speaker attribution also adds extra label tokens.
|
||||
# Keep the user-configured cap as the floor, but auto-raise low budgets so
|
||||
# longer chunks do not get truncated mid-transcript.
|
||||
if timestamps:
|
||||
recommended_floor = int(max(256, min(2048, round(chunk_seconds * 16.0))))
|
||||
return max(base_max_new_tokens, recommended_floor)
|
||||
|
||||
if diarization:
|
||||
recommended_floor = int(max(256, min(2048, round(chunk_seconds * 10.0))))
|
||||
return max(base_max_new_tokens, recommended_floor)
|
||||
|
||||
return base_max_new_tokens
|
||||
|
||||
def _run_chunk(
|
||||
self,
|
||||
runtime,
|
||||
chunk_np,
|
||||
req: ASRRequest,
|
||||
normalized_language: Optional[str],
|
||||
target_language: Optional[str],
|
||||
translate_instruction_override: Optional[str],
|
||||
timestamps: bool = False,
|
||||
diarization: bool = False,
|
||||
) -> Tuple[str, Optional[str]]:
|
||||
generation_kwargs = {
|
||||
"max_new_tokens": self._resolve_max_new_tokens(
|
||||
req,
|
||||
chunk_num_samples=len(chunk_np),
|
||||
sample_rate=16000,
|
||||
timestamps=timestamps,
|
||||
diarization=diarization,
|
||||
),
|
||||
"do_sample": bool(self.config.get("do_sample", False)),
|
||||
"num_beams": int(self.config.get("num_beams", 1)),
|
||||
"temperature": float(self.config.get("temperature", 1.0)),
|
||||
"top_k": int(self.config.get("top_k", 50)),
|
||||
"top_p": float(self.config.get("top_p", 1.0)),
|
||||
"repetition_penalty": float(self.config.get("repetition_penalty", 1.0)),
|
||||
"length_penalty": float(self.config.get("length_penalty", 1.0)),
|
||||
"no_repeat_ngram_size": int(self.config.get("no_repeat_ngram_size", 0)),
|
||||
"early_stopping": bool(self.config.get("early_stopping", False)),
|
||||
}
|
||||
result = runtime.transcribe(
|
||||
chunk_np,
|
||||
task=req.task,
|
||||
language=normalized_language,
|
||||
target_language=target_language,
|
||||
translate_instruction_override=translate_instruction_override,
|
||||
generation_kwargs=generation_kwargs,
|
||||
timestamps=timestamps,
|
||||
diarization=diarization,
|
||||
)
|
||||
return result.get("text", ""), result.get("language")
|
||||
|
||||
def transcribe(self, req: ASRRequest) -> ASRResult:
|
||||
model_management.throw_exception_if_processing_interrupted()
|
||||
|
||||
runtime = self._get_model()
|
||||
waveform = self._prepare_audio(req.audio)
|
||||
wav_np = waveform.cpu().numpy()
|
||||
sample_rate = 16000
|
||||
|
||||
warnings: List[str] = []
|
||||
notes: List[str] = []
|
||||
normalized_language = self._normalize_language(req.language)
|
||||
model_name = self.config.get("model_name", "granite-speech-4.1-2b")
|
||||
is_plus = "plus" in str(model_name).lower()
|
||||
|
||||
diarization_requested = is_plus and req.diarization and req.task == "transcribe"
|
||||
if req.diarization:
|
||||
if not is_plus:
|
||||
warnings.append("Granite diarization skipped: Only 'plus' variants support speaker attribution natively.")
|
||||
elif req.task != "transcribe":
|
||||
warnings.append("Granite diarization skipped: Speaker attribution is only supported for transcription, not translation.")
|
||||
|
||||
target_language = self.config.get("asr_translate_target_language", "English")
|
||||
translate_instruction_override = str(
|
||||
self.config.get("asr_translate_instruction_override", DEFAULT_TRANSLATE_INSTRUCTION_TEMPLATE)
|
||||
or DEFAULT_TRANSLATE_INSTRUCTION_TEMPLATE
|
||||
).strip()
|
||||
if req.language and str(req.language).lower() != "auto" and not normalized_language:
|
||||
if str(req.language).lower() == "japanese" and "plus" in str(model_name).lower():
|
||||
warnings.append(
|
||||
"Granite Speech Plus variant does not support Japanese. Falling back to Granite auto language handling."
|
||||
)
|
||||
else:
|
||||
warnings.append(
|
||||
f"Granite received unsupported language hint '{req.language}'. Falling back to Granite auto language handling."
|
||||
)
|
||||
if req.task == "translate":
|
||||
self._append_once(
|
||||
notes,
|
||||
f"Granite translation target is '{target_language}'. This target is prompt-driven by the Granite engine node, not a native model target-language parameter.",
|
||||
)
|
||||
if translate_instruction_override != DEFAULT_TRANSLATE_INSTRUCTION_TEMPLATE:
|
||||
self._append_once(
|
||||
notes,
|
||||
"Granite custom translation instruction override is active on the engine node. This is still wrapped in Granite's chat format, but malformed or weak instructions can fall back to plain transcription.",
|
||||
)
|
||||
|
||||
use_native_timestamps = is_plus and req.timestamps == "word" and req.task == "transcribe" and not diarization_requested
|
||||
use_aligner = not use_native_timestamps and self._can_use_aligner(req, normalized_language, warnings)
|
||||
aligner = self._get_forced_aligner() if use_aligner else None
|
||||
|
||||
chunk_size = int(req.chunk_size)
|
||||
overlap = int(req.overlap)
|
||||
total_samples = len(wav_np)
|
||||
|
||||
segments: List[ASRSegment] = []
|
||||
full_text_parts: List[str] = []
|
||||
output_language = normalized_language
|
||||
|
||||
if chunk_size > 0:
|
||||
chunk_samples = chunk_size * sample_rate
|
||||
overlap_samples = overlap * sample_rate
|
||||
step_samples = max(1, chunk_samples - overlap_samples)
|
||||
num_chunks = max(1, int((total_samples - overlap_samples + step_samples - 1) / step_samples))
|
||||
|
||||
chunk_progress_bar = comfy.utils.ProgressBar(num_chunks)
|
||||
start_time = time.time()
|
||||
last_print = 0.0
|
||||
|
||||
for i in range(num_chunks):
|
||||
model_management.throw_exception_if_processing_interrupted()
|
||||
start = i * step_samples
|
||||
end = min(start + chunk_samples, total_samples)
|
||||
chunk_np = wav_np[start:end]
|
||||
text, detected_language = self._run_chunk(
|
||||
runtime,
|
||||
chunk_np,
|
||||
req,
|
||||
normalized_language,
|
||||
target_language,
|
||||
translate_instruction_override,
|
||||
timestamps=use_native_timestamps,
|
||||
diarization=diarization_requested,
|
||||
)
|
||||
if detected_language and not output_language:
|
||||
output_language = detected_language
|
||||
|
||||
if diarization_requested:
|
||||
chunk_offset = start / sample_rate
|
||||
parsed_text, chunk_segments = parse_granite_diarization(text, chunk_offset, len(chunk_np) / sample_rate)
|
||||
if parsed_text:
|
||||
full_text_parts.append(format_granite_diarization_segments(chunk_segments))
|
||||
|
||||
if aligner is not None and parsed_text:
|
||||
aligner_language = self._resolve_aligner_language(normalized_language, parsed_text, notes)
|
||||
word_segments = self._align_words(
|
||||
aligner=aligner,
|
||||
chunk_audio=chunk_np,
|
||||
transcript=parsed_text,
|
||||
language=aligner_language,
|
||||
chunk_offset=chunk_offset,
|
||||
warnings=warnings,
|
||||
)
|
||||
aligned_words = [seg.words[0] for seg in word_segments if seg.words]
|
||||
remap_words_to_speaker_segments(chunk_segments, aligned_words)
|
||||
|
||||
for seg in chunk_segments:
|
||||
if seg.speaker:
|
||||
seg.text = f"[{seg.speaker}] {seg.text}"
|
||||
|
||||
segments.extend(chunk_segments)
|
||||
elif use_native_timestamps:
|
||||
chunk_offset = start / sample_rate
|
||||
parsed_text, chunk_segments = parse_granite_native_timestamps(text, chunk_offset)
|
||||
if parsed_text:
|
||||
full_text_parts.append(parsed_text)
|
||||
segments.extend(chunk_segments)
|
||||
else:
|
||||
if text:
|
||||
full_text_parts.append(text)
|
||||
if aligner is not None and text:
|
||||
aligner_language = self._resolve_aligner_language(normalized_language, text, notes)
|
||||
chunk_offset = start / sample_rate
|
||||
segments.extend(
|
||||
self._align_words(
|
||||
aligner=aligner,
|
||||
chunk_audio=chunk_np,
|
||||
transcript=text,
|
||||
language=aligner_language,
|
||||
chunk_offset=chunk_offset,
|
||||
warnings=warnings,
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
chunk_progress_bar.update(1)
|
||||
now = time.time()
|
||||
if now - last_print >= 0.5 or (i + 1) == num_chunks:
|
||||
elapsed = now - start_time
|
||||
its = (i + 1) / elapsed if elapsed > 0 else 0.0
|
||||
eta = (num_chunks - (i + 1)) / its if its > 0 else 0.0
|
||||
bar_width = 12
|
||||
filled = int(((i + 1) / num_chunks) * bar_width)
|
||||
bar = f"[{'█' * filled}{'░' * (bar_width - filled)}]"
|
||||
print(f"\r Progress: {bar} {i+1}/{num_chunks} | {its:.1f} it/s | {elapsed:.0f}s | ETA {eta:.0f}s", end="", flush=True)
|
||||
last_print = now
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
elapsed = time.time() - start_time
|
||||
avg = (num_chunks / elapsed) if elapsed > 0 else 0.0
|
||||
print(f"\r Complete: {num_chunks} chunks in {elapsed:.1f}s (avg {avg:.1f} it/s)" + " " * 20)
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
text, detected_language = self._run_chunk(
|
||||
runtime,
|
||||
wav_np,
|
||||
req,
|
||||
normalized_language,
|
||||
target_language,
|
||||
translate_instruction_override,
|
||||
timestamps=use_native_timestamps,
|
||||
diarization=diarization_requested,
|
||||
)
|
||||
output_language = detected_language or output_language
|
||||
|
||||
if diarization_requested:
|
||||
parsed_text, chunk_segments = parse_granite_diarization(text, 0.0, len(wav_np) / sample_rate)
|
||||
if parsed_text:
|
||||
full_text_parts.append(format_granite_diarization_segments(chunk_segments))
|
||||
|
||||
if aligner is not None and parsed_text:
|
||||
aligner_language = self._resolve_aligner_language(normalized_language, parsed_text, notes)
|
||||
word_segments = self._align_words(
|
||||
aligner=aligner,
|
||||
chunk_audio=wav_np,
|
||||
transcript=parsed_text,
|
||||
language=aligner_language,
|
||||
chunk_offset=0.0,
|
||||
warnings=warnings,
|
||||
)
|
||||
aligned_words = [seg.words[0] for seg in word_segments if seg.words]
|
||||
remap_words_to_speaker_segments(chunk_segments, aligned_words)
|
||||
|
||||
for seg in chunk_segments:
|
||||
if seg.speaker:
|
||||
seg.text = f"[{seg.speaker}] {seg.text}"
|
||||
|
||||
segments.extend(chunk_segments)
|
||||
elif use_native_timestamps:
|
||||
parsed_text, chunk_segments = parse_granite_native_timestamps(text, 0.0)
|
||||
if parsed_text:
|
||||
full_text_parts.append(parsed_text)
|
||||
segments.extend(chunk_segments)
|
||||
else:
|
||||
if text:
|
||||
full_text_parts.append(text)
|
||||
if aligner is not None and text:
|
||||
aligner_language = self._resolve_aligner_language(normalized_language, text, notes)
|
||||
segments.extend(
|
||||
self._align_words(
|
||||
aligner=aligner,
|
||||
chunk_audio=wav_np,
|
||||
transcript=text,
|
||||
language=aligner_language,
|
||||
chunk_offset=0.0,
|
||||
warnings=warnings,
|
||||
)
|
||||
)
|
||||
|
||||
text = " ".join(part for part in full_text_parts if part).strip()
|
||||
raw = None
|
||||
if warnings or notes:
|
||||
raw = {}
|
||||
if warnings:
|
||||
raw["warnings"] = warnings
|
||||
if notes:
|
||||
raw["notes"] = notes
|
||||
return ASRResult(text=text, language=output_language, segments=segments, raw=raw)
|
||||
@@ -1,276 +0,0 @@
|
||||
"""
|
||||
Qwen3-ASR adapter for unified ASR pipeline.
|
||||
Uses qwen_asr package (transformers backend) and optional forced aligner.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
import comfy.model_management as model_management
|
||||
import comfy.utils
|
||||
import time
|
||||
|
||||
from engines.qwen3_asr.prompting import DEFAULT_TRANSLATE_INSTRUCTION_TEMPLATE
|
||||
from utils.asr.types import ASRRequest, ASRResult, ASRSegment, ASRWord
|
||||
|
||||
|
||||
class Qwen3ASREngineAdapter:
|
||||
def __init__(self, engine_data: Dict[str, Any]):
|
||||
self.engine_data = engine_data
|
||||
self.config = engine_data.get("config", engine_data)
|
||||
|
||||
def _resolve_precision(self, precision: str) -> torch.dtype:
|
||||
if precision == "bf16":
|
||||
return torch.bfloat16
|
||||
if precision == "fp16":
|
||||
return torch.float16
|
||||
if precision == "fp32":
|
||||
return torch.float32
|
||||
# auto
|
||||
if torch.cuda.is_available():
|
||||
major, _minor = torch.cuda.get_device_capability()
|
||||
return torch.bfloat16 if major >= 8 else torch.float16
|
||||
return torch.float32
|
||||
|
||||
def _get_model(self, req: ASRRequest):
|
||||
from utils.models.unified_model_interface import unified_model_interface, ModelLoadConfig
|
||||
from utils.device import resolve_torch_device
|
||||
|
||||
device = self.config.get("device", "auto")
|
||||
if model_management.cpu_mode():
|
||||
device = "cpu"
|
||||
elif device != "cpu":
|
||||
device = resolve_torch_device(device)
|
||||
|
||||
# Infer ASR model variant from existing Qwen3 model_size setting
|
||||
model_size = self.config.get("model_size", "1.7B")
|
||||
model_name = "Qwen3-ASR-0.6B" if model_size == "0.6B" else "Qwen3-ASR-1.7B"
|
||||
model_id = model_name
|
||||
|
||||
precision = self.config.get("dtype", "auto")
|
||||
|
||||
attn = self.config.get("attn_implementation", "auto")
|
||||
if attn == "auto":
|
||||
flash_attn = False
|
||||
else:
|
||||
flash_attn = attn == "flash_attention_2"
|
||||
|
||||
forced_aligner_enabled = self.config.get("asr_use_forced_aligner", False)
|
||||
|
||||
forced_aligner_model = "Qwen3-ForcedAligner-0.6B" if forced_aligner_enabled else ""
|
||||
|
||||
additional_params = {
|
||||
"precision": precision,
|
||||
"attn_implementation": "flash_attention_2" if flash_attn else "sdpa",
|
||||
"max_new_tokens": int(self.config.get("max_new_tokens", 256)),
|
||||
}
|
||||
|
||||
if forced_aligner_enabled:
|
||||
additional_params["forced_aligner"] = forced_aligner_model
|
||||
additional_params["forced_aligner_kwargs"] = None
|
||||
|
||||
config = ModelLoadConfig(
|
||||
engine_name="qwen3_asr",
|
||||
model_type="asr",
|
||||
model_name=model_name,
|
||||
model_path=model_id,
|
||||
device=device,
|
||||
runtime_mode=self.config.get("runtime_mode", "main_environment"),
|
||||
runtime_profile=self.config.get("runtime_profile"),
|
||||
additional_params=additional_params,
|
||||
)
|
||||
|
||||
return unified_model_interface.load_model(config)
|
||||
|
||||
def _prepare_audio(self, audio: Dict[str, Any]) -> torch.Tensor:
|
||||
waveform = audio["waveform"]
|
||||
sample_rate = audio["sample_rate"]
|
||||
|
||||
if waveform.ndim == 3:
|
||||
waveform = waveform[0]
|
||||
if waveform.shape[0] > 1:
|
||||
waveform = torch.mean(waveform, dim=0)
|
||||
else:
|
||||
waveform = waveform[0]
|
||||
|
||||
if sample_rate != 16000:
|
||||
w = waveform.view(1, 1, -1)
|
||||
w = F.interpolate(w, size=int(w.shape[-1] * 16000 / sample_rate), mode="linear", align_corners=False)
|
||||
waveform = w.view(-1)
|
||||
return waveform
|
||||
|
||||
def _build_translate_context(self, req: ASRRequest, target_language: str, instruction_template: str) -> str:
|
||||
source_language = (
|
||||
req.language
|
||||
if req.language and req.language.lower() != "auto"
|
||||
else "the spoken source language"
|
||||
)
|
||||
context = str(instruction_template or DEFAULT_TRANSLATE_INSTRUCTION_TEMPLATE).strip()
|
||||
context = context.replace("{source_language}", source_language)
|
||||
context = context.replace("{language}", source_language)
|
||||
context = context.replace("{target_language}", target_language)
|
||||
return context
|
||||
|
||||
def transcribe(self, req: ASRRequest) -> ASRResult:
|
||||
model_management.throw_exception_if_processing_interrupted()
|
||||
|
||||
model = self._get_model(req)
|
||||
waveform = self._prepare_audio(req.audio)
|
||||
sample_rate = 16000
|
||||
|
||||
target_language = self.config.get("asr_translate_target_language", "English")
|
||||
translate_instruction_override = str(
|
||||
self.config.get("asr_translate_instruction_override", DEFAULT_TRANSLATE_INSTRUCTION_TEMPLATE)
|
||||
or DEFAULT_TRANSLATE_INSTRUCTION_TEMPLATE
|
||||
).strip()
|
||||
warnings: List[str] = []
|
||||
notes: List[str] = []
|
||||
translate_mode = req.task == "translate"
|
||||
language = None if not req.language or req.language.lower() == "auto" else req.language
|
||||
if translate_mode:
|
||||
return_ts = False
|
||||
language = target_language
|
||||
notes.append(
|
||||
f"Qwen translation target is '{target_language}'. In this integration, translation is still prompt-driven, but the target language is also forced through Qwen's assistant prompt path instead of leaving language control empty."
|
||||
)
|
||||
if translate_instruction_override != DEFAULT_TRANSLATE_INSTRUCTION_TEMPLATE:
|
||||
notes.append(
|
||||
"Qwen custom translation instruction override is active on the engine node. This only affects ASR translation mode and can change model behavior significantly."
|
||||
)
|
||||
if req.timestamps == "word" or req.use_forced_aligner or self.config.get("asr_use_forced_aligner", False):
|
||||
warnings.append("Qwen timestamps skipped: translation mode returns text only.")
|
||||
else:
|
||||
return_ts = req.timestamps == "word" or req.use_forced_aligner or self.config.get("asr_use_forced_aligner", False)
|
||||
context = (
|
||||
self._build_translate_context(req, target_language, translate_instruction_override)
|
||||
if translate_mode else ""
|
||||
)
|
||||
|
||||
chunk_size = int(req.chunk_size)
|
||||
overlap = int(req.overlap)
|
||||
|
||||
segments: List[ASRSegment] = []
|
||||
full_text_parts: List[str] = []
|
||||
wav_np = waveform.cpu().numpy()
|
||||
total_samples = len(wav_np)
|
||||
progress_bar = None
|
||||
|
||||
if chunk_size > 0:
|
||||
chunk_samples = chunk_size * sample_rate
|
||||
overlap_samples = overlap * sample_rate
|
||||
step_samples = max(1, chunk_samples - overlap_samples)
|
||||
num_chunks = max(1, int((total_samples - overlap_samples + step_samples - 1) / step_samples))
|
||||
|
||||
if getattr(model, "backend", None) == "transformers":
|
||||
try:
|
||||
from utils.asr.progress_callback import ASRProgressStreamer
|
||||
total_tokens = int(self.config.get("max_new_tokens", 256)) * num_chunks
|
||||
progress_bar = comfy.utils.ProgressBar(total_tokens)
|
||||
model._streamer_factory = lambda max_new_tokens, pb: ASRProgressStreamer(
|
||||
max_new_tokens or int(self.config.get("max_new_tokens", 256)),
|
||||
pb,
|
||||
label="ASR",
|
||||
)
|
||||
model._progress_bar = progress_bar
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Chunk-level fallback progress (matches TTS-style console line)
|
||||
chunk_progress_bar = comfy.utils.ProgressBar(num_chunks)
|
||||
start_time = time.time()
|
||||
last_print = 0.0
|
||||
|
||||
for i in range(num_chunks):
|
||||
model_management.throw_exception_if_processing_interrupted()
|
||||
start = i * step_samples
|
||||
end = min(start + chunk_samples, total_samples)
|
||||
chunk_np = wav_np[start:end]
|
||||
results = model.transcribe(
|
||||
audio=[(chunk_np, sample_rate)],
|
||||
context=[context],
|
||||
language=language,
|
||||
return_time_stamps=return_ts,
|
||||
)
|
||||
res = results[0]
|
||||
full_text_parts.append(res.text)
|
||||
chunk_offset = start / sample_rate
|
||||
|
||||
if return_ts and hasattr(res, "time_stamps") and res.time_stamps:
|
||||
for ts in res.time_stamps:
|
||||
word = ASRWord(
|
||||
start=ts.start_time + chunk_offset,
|
||||
end=ts.end_time + chunk_offset,
|
||||
text=ts.text,
|
||||
)
|
||||
seg = ASRSegment(
|
||||
start=word.start,
|
||||
end=word.end,
|
||||
text=word.text,
|
||||
words=[word],
|
||||
)
|
||||
segments.append(seg)
|
||||
|
||||
try:
|
||||
chunk_progress_bar.update(1)
|
||||
now = time.time()
|
||||
if now - last_print >= 0.5 or (i + 1) == num_chunks:
|
||||
elapsed = now - start_time
|
||||
its = (i + 1) / elapsed if elapsed > 0 else 0.0
|
||||
eta = (num_chunks - (i + 1)) / its if its > 0 else 0.0
|
||||
bar_width = 12
|
||||
filled = int(((i + 1) / num_chunks) * bar_width)
|
||||
bar = f"[{'█' * filled}{'░' * (bar_width - filled)}]"
|
||||
print(f"\r Progress: {bar} {i+1}/{num_chunks} | {its:.1f} it/s | {elapsed:.0f}s | ETA {eta:.0f}s", end="", flush=True)
|
||||
last_print = now
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
elapsed = time.time() - start_time
|
||||
avg = (num_chunks / elapsed) if elapsed > 0 else 0.0
|
||||
print(f"\r Complete: {num_chunks} chunks in {elapsed:.1f}s (avg {avg:.1f} it/s)" + " " * 20)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
else:
|
||||
if getattr(model, "backend", None) == "transformers":
|
||||
try:
|
||||
from utils.asr.progress_callback import ASRProgressStreamer
|
||||
total_tokens = int(self.config.get("max_new_tokens", 256))
|
||||
progress_bar = comfy.utils.ProgressBar(total_tokens)
|
||||
model._streamer_factory = lambda max_new_tokens, pb: ASRProgressStreamer(
|
||||
max_new_tokens or total_tokens,
|
||||
pb,
|
||||
label="ASR",
|
||||
)
|
||||
model._progress_bar = progress_bar
|
||||
except Exception:
|
||||
pass
|
||||
results = model.transcribe(
|
||||
audio=[(wav_np, sample_rate)],
|
||||
context=[context],
|
||||
language=language,
|
||||
return_time_stamps=return_ts,
|
||||
)
|
||||
res = results[0]
|
||||
full_text_parts.append(res.text)
|
||||
if return_ts and hasattr(res, "time_stamps") and res.time_stamps:
|
||||
for ts in res.time_stamps:
|
||||
word = ASRWord(start=ts.start_time, end=ts.end_time, text=ts.text)
|
||||
segments.append(ASRSegment(start=word.start, end=word.end, text=word.text, words=[word]))
|
||||
|
||||
text = " ".join([t for t in full_text_parts if t]).strip()
|
||||
output_language = getattr(results[0], "language", None)
|
||||
if translate_mode:
|
||||
output_language = target_language
|
||||
|
||||
raw = None
|
||||
if warnings or notes:
|
||||
raw = {}
|
||||
if warnings:
|
||||
raw["warnings"] = warnings
|
||||
if notes:
|
||||
raw["notes"] = notes
|
||||
|
||||
return ASRResult(text=text, language=output_language, segments=segments, raw=raw)
|
||||
@@ -1,372 +0,0 @@
|
||||
"""Adapter between the suite's TTS processors and an audio.cpp session."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
from typing import Any, Dict, Mapping, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from utils.audio.audio_hash import generate_stable_audio_component
|
||||
from utils.audio.cache import get_audio_cache
|
||||
from utils.audio.processing import AudioProcessingUtils
|
||||
from utils.voice.reference import effective_voice_audio
|
||||
|
||||
|
||||
_CACHE_SAMPLE_RATES: Dict[str, int] = {}
|
||||
_CACHE_SAMPLE_RATES_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def _get_session(config: Mapping[str, Any]):
|
||||
"""Import lazily so the node can still be discovered before optional setup."""
|
||||
from utils.audio_cpp.session import get_audio_cpp_session
|
||||
|
||||
return get_audio_cpp_session(dict(config))
|
||||
|
||||
|
||||
def _canonical_json(value: Mapping[str, Any]) -> str:
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str)
|
||||
|
||||
|
||||
class AudioCppEngineAdapter:
|
||||
"""Build generic ``/v1/tasks/run`` requests and retain their real sample rate."""
|
||||
|
||||
_COMMON_REQUEST_FIELDS = (
|
||||
"temperature",
|
||||
"top_p",
|
||||
"top_k",
|
||||
"repetition_penalty",
|
||||
"max_tokens",
|
||||
"max_steps",
|
||||
"num_inference_steps",
|
||||
"guidance_scale",
|
||||
"speaking_rate",
|
||||
)
|
||||
|
||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||
self.config = dict(config or {})
|
||||
self.audio_cache = get_audio_cache()
|
||||
self._last_sample_rate: Optional[int] = None
|
||||
self._reference_files: Dict[str, str] = {}
|
||||
self._reference_lock = threading.RLock()
|
||||
|
||||
@property
|
||||
def sample_rate(self) -> Optional[int]:
|
||||
return self._last_sample_rate
|
||||
|
||||
def update_config(self, new_config: Optional[Dict[str, Any]]) -> None:
|
||||
self.config = dict(new_config or {})
|
||||
|
||||
@staticmethod
|
||||
def _reference_text(voice_ref: Any) -> str:
|
||||
if not isinstance(voice_ref, Mapping):
|
||||
return ""
|
||||
return str(
|
||||
voice_ref.get("reference_text")
|
||||
or voice_ref.get("prompt_text")
|
||||
or voice_ref.get("text")
|
||||
or ""
|
||||
).strip()
|
||||
|
||||
def _materialize_reference(self, voice_ref: Any) -> Tuple[Optional[str], str, str, Optional[str]]:
|
||||
"""Return path, transcript, stable hash, and the path that must be removed."""
|
||||
reference_text = AudioCppEngineAdapter._reference_text(voice_ref)
|
||||
if not isinstance(voice_ref, Mapping):
|
||||
return None, reference_text, "default_voice", None
|
||||
|
||||
audio = effective_voice_audio(voice_ref)
|
||||
if audio is None:
|
||||
return None, reference_text, "default_voice", None
|
||||
|
||||
if isinstance(audio, (str, os.PathLike)):
|
||||
path = os.path.abspath(os.path.expanduser(os.fspath(audio)))
|
||||
if not os.path.isfile(path):
|
||||
raise FileNotFoundError(f"audio.cpp reference audio not found: {path}")
|
||||
component = generate_stable_audio_component(audio_file_path=path)
|
||||
return path, reference_text, component, None
|
||||
|
||||
if isinstance(audio, Mapping):
|
||||
waveform = audio.get("waveform")
|
||||
sample_rate = audio.get("sample_rate")
|
||||
audio_dict = dict(audio)
|
||||
elif torch.is_tensor(audio):
|
||||
waveform = audio
|
||||
sample_rate = voice_ref.get("sample_rate")
|
||||
audio_dict = {"waveform": waveform, "sample_rate": sample_rate}
|
||||
else:
|
||||
raise TypeError(f"Unsupported audio.cpp voice reference type: {type(audio).__name__}")
|
||||
|
||||
if not torch.is_tensor(waveform):
|
||||
raise TypeError("audio.cpp reference audio must contain a waveform tensor")
|
||||
if sample_rate is None or int(sample_rate) <= 0:
|
||||
raise ValueError("audio.cpp reference audio must contain a positive sample_rate")
|
||||
|
||||
audio_dict["sample_rate"] = int(sample_rate)
|
||||
component = generate_stable_audio_component(reference_audio=audio_dict)
|
||||
if component not in {"ref_audio_error", "ref_audio_error_not_tensor"}:
|
||||
with self._reference_lock:
|
||||
cached_path = self._reference_files.get(component)
|
||||
if cached_path and os.path.isfile(cached_path):
|
||||
return cached_path, reference_text, component, None
|
||||
temp_path = os.path.abspath(
|
||||
AudioProcessingUtils.save_audio_to_temp_file(waveform, int(sample_rate))
|
||||
)
|
||||
self._reference_files[component] = temp_path
|
||||
return temp_path, reference_text, component, None
|
||||
|
||||
# Hash failures must not make unrelated references share one file.
|
||||
temp_path = os.path.abspath(
|
||||
AudioProcessingUtils.save_audio_to_temp_file(waveform, int(sample_rate))
|
||||
)
|
||||
return temp_path, reference_text, component, temp_path
|
||||
|
||||
def close(self) -> None:
|
||||
with self._reference_lock:
|
||||
paths = list(self._reference_files.values())
|
||||
self._reference_files.clear()
|
||||
for path in paths:
|
||||
try:
|
||||
os.remove(path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def __del__(self):
|
||||
try:
|
||||
self.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _advanced_options(self) -> Dict[str, Any]:
|
||||
value = self.config.get(
|
||||
"advanced_options",
|
||||
self.config.get("request_options", self.config.get("advanced_json", {})),
|
||||
)
|
||||
if value in (None, ""):
|
||||
return {}
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
value = json.loads(value)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"Invalid audio.cpp advanced JSON: {exc.msg}") from exc
|
||||
if not isinstance(value, Mapping):
|
||||
raise ValueError("audio.cpp advanced options must be a JSON object")
|
||||
return dict(value)
|
||||
|
||||
def _resolved_task(self, session: Any) -> str:
|
||||
requested = str(self.config.get("task", self.config.get("requested_task", "auto"))).lower()
|
||||
for source in (session, getattr(session, "config", None)):
|
||||
if source is None:
|
||||
continue
|
||||
value = source.get("task") if isinstance(source, Mapping) else getattr(source, "task", None)
|
||||
if str(value).lower() in {"tts", "clon", "vdes"}:
|
||||
return str(value).lower()
|
||||
|
||||
if requested in {"tts", "clon", "vdes"}:
|
||||
return requested
|
||||
if str(self.config.get("connection_mode", "auto")).lower() == "external_server":
|
||||
return "auto"
|
||||
try:
|
||||
from utils.audio_cpp.catalog import resolve_task
|
||||
|
||||
return str(
|
||||
resolve_task(
|
||||
self.config.get("family", ""),
|
||||
self.config.get("package_id", ""),
|
||||
requested="auto",
|
||||
)
|
||||
).lower()
|
||||
except (ImportError, KeyError, TypeError, ValueError):
|
||||
return "tts"
|
||||
|
||||
def _build_request(
|
||||
self,
|
||||
text: str,
|
||||
voice_path: Optional[str],
|
||||
reference_text: str,
|
||||
seed: int,
|
||||
advanced: Dict[str, Any],
|
||||
task: str,
|
||||
) -> Dict[str, Any]:
|
||||
request: Dict[str, Any] = {"text": text, "seed": str(int(seed)), "options": advanced}
|
||||
del task # The persistent session owns its one configured model/task.
|
||||
|
||||
language = str(self.config.get("language", "")).strip()
|
||||
if language and language.lower() not in {"auto", "none"}:
|
||||
request["language"] = language
|
||||
voice_id = str(self.config.get("voice_id", self.config.get("voice", ""))).strip()
|
||||
if voice_id:
|
||||
request["voice_id"] = voice_id
|
||||
if voice_path:
|
||||
request["voice_ref"] = voice_path
|
||||
if reference_text:
|
||||
request["reference_text"] = reference_text
|
||||
instruct = str(self.config.get("instruct", "")).strip()
|
||||
if instruct:
|
||||
request["instruct"] = instruct
|
||||
|
||||
for key in self._COMMON_REQUEST_FIELDS:
|
||||
value = self.config.get(key)
|
||||
if value is not None and value != "":
|
||||
request[key] = value
|
||||
return request
|
||||
|
||||
def _cache_key(
|
||||
self,
|
||||
text: str,
|
||||
audio_component: str,
|
||||
reference_text: str,
|
||||
seed: int,
|
||||
task: str,
|
||||
advanced: Dict[str, Any],
|
||||
character_name: Optional[str],
|
||||
session: Any,
|
||||
) -> str:
|
||||
session_config = getattr(session, "config", {})
|
||||
if not isinstance(session_config, Mapping):
|
||||
session_config = {}
|
||||
session_family = getattr(session, "family", None) or session_config.get(
|
||||
"family", self.config.get("family", "")
|
||||
)
|
||||
session_model_id = getattr(session, "model_id", None) or session_config.get(
|
||||
"model_id", self.config.get("model_id", "")
|
||||
)
|
||||
# Owned servers use a random loopback port on every restart; that port is
|
||||
# transport state, not model identity. External endpoints are stable and
|
||||
# must participate in the cache key.
|
||||
if bool(getattr(session, "owned", False)):
|
||||
session_endpoint = ""
|
||||
else:
|
||||
session_endpoint = getattr(session, "endpoint", None) or self.config.get(
|
||||
"server_url", self.config.get("external_server_url", "")
|
||||
)
|
||||
extra_identity = {
|
||||
"options": advanced,
|
||||
"speaking_rate": self.config.get("speaking_rate"),
|
||||
"connection_mode": self.config.get("connection_mode", "auto"),
|
||||
"server_url": session_endpoint,
|
||||
"binary_path": session_config.get("binary_path", self.config.get("binary_path", "")),
|
||||
"backend": session_config.get("backend", self.config.get("backend", "")),
|
||||
"device": session_config.get("device", self.config.get("device", "")),
|
||||
"load_options": session_config.get("load_options", self.config.get("load_options", {})),
|
||||
"session_options": session_config.get(
|
||||
"session_options", self.config.get("session_options", {})
|
||||
),
|
||||
"default_request_options": session_config.get(
|
||||
"default_request_options", self.config.get("default_request_options", {})
|
||||
),
|
||||
}
|
||||
return self.audio_cache.generate_cache_key(
|
||||
"audio_cpp",
|
||||
text=text,
|
||||
audio_component=audio_component,
|
||||
reference_text=reference_text,
|
||||
family=session_family,
|
||||
package_id=session_config.get("package_id", self.config.get("package_id", "")),
|
||||
model_path=session_config.get("model_path", self.config.get("model_path", "")),
|
||||
model_id=session_model_id,
|
||||
task=task,
|
||||
language=self.config.get("language", ""),
|
||||
voice_id=self.config.get("voice_id", self.config.get("voice", "")),
|
||||
instruct=self.config.get("instruct", ""),
|
||||
temperature=self.config.get("temperature"),
|
||||
top_p=self.config.get("top_p"),
|
||||
top_k=self.config.get("top_k"),
|
||||
repetition_penalty=self.config.get("repetition_penalty"),
|
||||
max_tokens=self.config.get("max_tokens"),
|
||||
max_steps=self.config.get("max_steps"),
|
||||
num_inference_steps=self.config.get("num_inference_steps"),
|
||||
guidance_scale=self.config.get("guidance_scale"),
|
||||
seed=int(seed),
|
||||
request_options=_canonical_json(extra_identity),
|
||||
character=character_name or "narrator",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_result(result: Any) -> Tuple[torch.Tensor, int]:
|
||||
waveform = result.get("waveform") if isinstance(result, Mapping) else getattr(result, "waveform", None)
|
||||
sample_rate = result.get("sample_rate") if isinstance(result, Mapping) else getattr(result, "sample_rate", None)
|
||||
|
||||
if waveform is None:
|
||||
named = result.get("named_audio", {}) if isinstance(result, Mapping) else getattr(result, "named_audio", {})
|
||||
values = list(named.values()) if isinstance(named, Mapping) else list(named or [])
|
||||
if len(values) == 1:
|
||||
item = values[0]
|
||||
waveform = item.get("waveform") if isinstance(item, Mapping) else getattr(item, "waveform", None)
|
||||
sample_rate = sample_rate or (item.get("sample_rate") if isinstance(item, Mapping) else getattr(item, "sample_rate", None))
|
||||
|
||||
if waveform is None:
|
||||
raise RuntimeError("audio.cpp returned no primary audio output")
|
||||
if not torch.is_tensor(waveform):
|
||||
waveform = torch.as_tensor(waveform, dtype=torch.float32)
|
||||
waveform = waveform.detach().to(device="cpu", dtype=torch.float32)
|
||||
if waveform.dim() == 1:
|
||||
waveform = waveform.unsqueeze(0)
|
||||
elif waveform.dim() == 3 and waveform.shape[0] == 1:
|
||||
waveform = waveform.squeeze(0)
|
||||
if waveform.dim() != 2:
|
||||
raise ValueError(f"audio.cpp waveform must be [channels, samples], got {tuple(waveform.shape)}")
|
||||
if sample_rate is None or int(sample_rate) <= 0:
|
||||
raise ValueError("audio.cpp returned an invalid sample rate")
|
||||
return waveform.contiguous(), int(sample_rate)
|
||||
|
||||
def generate_single(
|
||||
self,
|
||||
text: str,
|
||||
voice_ref: Optional[Dict[str, Any]] = None,
|
||||
seed: int = 0,
|
||||
enable_audio_cache: bool = True,
|
||||
character_name: Optional[str] = None,
|
||||
) -> Tuple[torch.Tensor, int]:
|
||||
stripped = str(text or "").strip()
|
||||
if not stripped:
|
||||
if self._last_sample_rate is None:
|
||||
raise ValueError("audio.cpp cannot determine a sample rate for empty text")
|
||||
return torch.zeros(1, 0, dtype=torch.float32), self._last_sample_rate
|
||||
|
||||
session = _get_session(self.config)
|
||||
task = self._resolved_task(session)
|
||||
advanced = self._advanced_options()
|
||||
cleanup_path: Optional[str] = None
|
||||
try:
|
||||
voice_path, reference_text, audio_component, cleanup_path = self._materialize_reference(voice_ref)
|
||||
cache_key = self._cache_key(
|
||||
stripped,
|
||||
audio_component,
|
||||
reference_text,
|
||||
seed,
|
||||
task,
|
||||
advanced,
|
||||
character_name,
|
||||
session,
|
||||
)
|
||||
if enable_audio_cache:
|
||||
cached = self.audio_cache.get_cached_audio(cache_key)
|
||||
with _CACHE_SAMPLE_RATES_LOCK:
|
||||
cached_rate = _CACHE_SAMPLE_RATES.get(cache_key)
|
||||
if cached is not None and cached_rate is not None:
|
||||
self._last_sample_rate = cached_rate
|
||||
return cached[0].clone(), cached_rate
|
||||
|
||||
request = self._build_request(stripped, voice_path, reference_text, seed, advanced, task)
|
||||
waveform, sample_rate = self._normalize_result(session.run(request))
|
||||
self._last_sample_rate = sample_rate
|
||||
if enable_audio_cache:
|
||||
duration = waveform.shape[-1] / sample_rate
|
||||
self.audio_cache.cache_audio(cache_key, waveform, duration)
|
||||
with _CACHE_SAMPLE_RATES_LOCK:
|
||||
_CACHE_SAMPLE_RATES[cache_key] = sample_rate
|
||||
return waveform, sample_rate
|
||||
finally:
|
||||
if cleanup_path:
|
||||
try:
|
||||
os.remove(cleanup_path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
# Short alias for callers that do not use the older ``EngineAdapter`` suffix.
|
||||
AudioCppAdapter = AudioCppEngineAdapter
|
||||
@@ -1,111 +0,0 @@
|
||||
"""audio.cpp adapter for the Suite's unified Voice Changer node."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from typing import Any, Dict, Mapping
|
||||
|
||||
import torch
|
||||
|
||||
from engines.adapters.audio_cpp_adapter import AudioCppEngineAdapter
|
||||
from utils.audio.processing import AudioProcessingUtils
|
||||
|
||||
|
||||
def _advanced_options(config: Mapping[str, Any]) -> Dict[str, Any]:
|
||||
value = config.get("advanced_options", config.get("request_options", {}))
|
||||
if value in (None, ""):
|
||||
return {}
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
value = json.loads(value)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"Invalid audio.cpp advanced JSON: {exc.msg}") from exc
|
||||
if not isinstance(value, Mapping):
|
||||
raise ValueError("audio.cpp advanced options must be a JSON object")
|
||||
return dict(value)
|
||||
|
||||
|
||||
def _materialize(audio: Mapping[str, Any], label: str) -> str:
|
||||
waveform = audio.get("waveform")
|
||||
sample_rate = audio.get("sample_rate")
|
||||
if not torch.is_tensor(waveform):
|
||||
raise TypeError(f"audio.cpp {label} must contain a waveform tensor")
|
||||
if sample_rate is None or int(sample_rate) <= 0:
|
||||
raise ValueError(f"audio.cpp {label} must contain a positive sample_rate")
|
||||
return os.path.abspath(
|
||||
AudioProcessingUtils.save_audio_to_temp_file(waveform, int(sample_rate))
|
||||
)
|
||||
|
||||
|
||||
class AudioCppVoiceConversionAdapter:
|
||||
"""Convert source audio toward a target reference using an audio.cpp VC task."""
|
||||
|
||||
def __init__(self, config: Dict[str, Any]):
|
||||
self.config = dict(config)
|
||||
|
||||
def _session_config(self) -> Dict[str, Any]:
|
||||
config = dict(self.config)
|
||||
if str(config.get("connection_mode", "auto")).lower() != "external_server":
|
||||
config["requested_task"] = "vc"
|
||||
config["task"] = "vc"
|
||||
return config
|
||||
|
||||
def convert_voice(
|
||||
self,
|
||||
source_audio: Dict[str, Any],
|
||||
target_audio: Dict[str, Any],
|
||||
refinement_passes: int = 1,
|
||||
) -> tuple[Dict[str, Any], str]:
|
||||
from utils.audio_cpp.session import get_audio_cpp_session
|
||||
|
||||
config = self._session_config()
|
||||
family = str(config.get("family", "")).strip()
|
||||
passes = max(1, int(refinement_passes))
|
||||
current = source_audio
|
||||
output_rate = int(source_audio["sample_rate"])
|
||||
|
||||
session = get_audio_cpp_session(config)
|
||||
if str(getattr(session, "task", "vc")) != "vc":
|
||||
raise ValueError(
|
||||
f"audio.cpp model '{session.model_id}' is configured for task "
|
||||
f"'{session.task}', not voice conversion"
|
||||
)
|
||||
|
||||
for pass_index in range(passes):
|
||||
source_path = _materialize(current, "source audio")
|
||||
target_path = _materialize(target_audio, "target reference audio")
|
||||
try:
|
||||
request = {
|
||||
"audio": source_path,
|
||||
"voice_ref": target_path,
|
||||
"source_audio": source_path,
|
||||
"target_voice": target_path,
|
||||
"options": _advanced_options(config),
|
||||
}
|
||||
print(
|
||||
f"🔄 audio.cpp VC: {family or 'external model'} pass "
|
||||
f"{pass_index + 1}/{passes}..."
|
||||
)
|
||||
result = session.run(request)
|
||||
waveform, output_rate = AudioCppEngineAdapter._normalize_result(result)
|
||||
current = {"waveform": waveform.unsqueeze(0), "sample_rate": output_rate}
|
||||
finally:
|
||||
for path in (source_path, target_path):
|
||||
try:
|
||||
os.remove(path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
info = (
|
||||
f"Model family: {family or getattr(session, 'family', 'external')}\n"
|
||||
f"Model ID: {session.model_id}\n"
|
||||
f"Task: voice conversion\n"
|
||||
f"Refinement passes: {passes}\n"
|
||||
f"Output sample rate: {output_rate} Hz\n"
|
||||
"Conversion completed successfully"
|
||||
)
|
||||
return current, info
|
||||
|
||||
|
||||
__all__ = ["AudioCppVoiceConversionAdapter"]
|
||||
@@ -283,41 +283,21 @@ class ChatterBoxEngineAdapter:
|
||||
|
||||
def combine_audio_chunks(self, audio_segments: List[torch.Tensor], **params) -> torch.Tensor:
|
||||
"""
|
||||
Combine ChatterBox audio segments using modular combination utility.
|
||||
Combine ChatterBox audio segments.
|
||||
|
||||
Args:
|
||||
audio_segments: List of audio tensors to combine
|
||||
**params: Combination parameters
|
||||
|
||||
Returns:
|
||||
Combined audio tensor, or tuple with timing info if requested
|
||||
Combined audio tensor
|
||||
"""
|
||||
if len(audio_segments) == 1:
|
||||
return audio_segments[0]
|
||||
|
||||
# Use modular chunk combiner with ChatterBox settings
|
||||
from utils.audio.chunk_combiner import ChunkCombiner
|
||||
|
||||
method = params.get("combination_method", "auto")
|
||||
silence_ms = params.get("silence_ms", 100)
|
||||
text_length = params.get("text_length", 0)
|
||||
original_text = params.get("original_text", "")
|
||||
text_chunks = params.get("text_chunks", None)
|
||||
return_info = params.get("return_timing_info", False)
|
||||
|
||||
print(f"🔗 Combining {len(audio_segments)} ChatterBox chunks using '{method}' method")
|
||||
|
||||
return ChunkCombiner.combine_chunks(
|
||||
audio_segments=audio_segments,
|
||||
method=method,
|
||||
silence_ms=silence_ms,
|
||||
crossfade_duration=0.1,
|
||||
sample_rate=44100, # ChatterBox sample rate
|
||||
text_length=text_length,
|
||||
original_text=original_text,
|
||||
text_chunks=text_chunks,
|
||||
return_info=return_info
|
||||
)
|
||||
# ChatterBox uses simple concatenation
|
||||
from utils.audio.processing import AudioProcessingUtils
|
||||
return AudioProcessingUtils.concatenate_audio_segments(audio_segments, "simple")
|
||||
|
||||
def generate_audio(self, text: str, voice_preset: str, voice_settings: dict,
|
||||
output_filename: str, language: str = "auto",
|
||||
|
||||
@@ -1,407 +0,0 @@
|
||||
"""
|
||||
ChatterBox Official 23-Lang Engine Adapter - Engine-specific adapter for ChatterBox Official 23-Lang TTS
|
||||
Provides standardized interface for ChatterBox Official 23-Lang operations in multilingual engine
|
||||
With native batch processing support for parallel text generation
|
||||
"""
|
||||
|
||||
import torch
|
||||
from typing import Dict, Any, Optional, List, Tuple
|
||||
import concurrent.futures
|
||||
import threading
|
||||
# Use absolute import to avoid relative import issues in ComfyUI
|
||||
import sys
|
||||
import os
|
||||
current_dir = os.path.dirname(__file__)
|
||||
parent_dir = os.path.dirname(current_dir)
|
||||
if parent_dir not in sys.path:
|
||||
sys.path.insert(0, parent_dir)
|
||||
from utils.models.language_mapper import get_model_for_language
|
||||
|
||||
|
||||
class ChatterBoxOfficial23LangEngineAdapter:
|
||||
"""Engine-specific adapter for ChatterBox TTS with batch processing support."""
|
||||
|
||||
def __init__(self, node_instance):
|
||||
"""
|
||||
Initialize ChatterBox adapter with batch processing capabilities.
|
||||
|
||||
Args:
|
||||
node_instance: ChatterboxTTSNode or SRTTTSNode instance
|
||||
"""
|
||||
self.node = node_instance
|
||||
self.engine_type = "chatterbox_official_23lang"
|
||||
self.batch_size = 4 # Default batch size for parallel processing
|
||||
self.enable_batch_processing = True # Enable batch processing by default
|
||||
self._generation_lock = threading.Lock() # Thread safety for model access
|
||||
|
||||
def get_model_for_language(self, lang_code: str, default_model: str) -> str:
|
||||
"""
|
||||
Get ChatterBox model name for specified language.
|
||||
|
||||
Args:
|
||||
lang_code: Language code (e.g., 'en', 'de', 'no')
|
||||
default_model: Default model name (language)
|
||||
|
||||
Returns:
|
||||
ChatterBox model name (language) for the specified language code
|
||||
"""
|
||||
return get_model_for_language(self.engine_type, lang_code, default_model)
|
||||
|
||||
def load_base_model(self, language: str, device: str):
|
||||
"""
|
||||
Load base ChatterBox model.
|
||||
|
||||
Args:
|
||||
language: Language model to load (e.g., "English", "German")
|
||||
device: Device to load model on
|
||||
"""
|
||||
self.node.load_tts_model(device, language)
|
||||
|
||||
def load_language_model(self, language: str, device: str):
|
||||
"""
|
||||
Load language-specific ChatterBox model.
|
||||
|
||||
Args:
|
||||
language: Language model to load (e.g., "German", "Norwegian")
|
||||
device: Device to load model on
|
||||
"""
|
||||
self.node.load_tts_model(device, language)
|
||||
|
||||
def generate_segment_audio(self, text: str, char_audio: str,
|
||||
character: str = "narrator", **params) -> torch.Tensor:
|
||||
"""
|
||||
Generate ChatterBox audio for a text segment.
|
||||
|
||||
Args:
|
||||
text: Text to generate audio for
|
||||
char_audio: Reference audio file path
|
||||
character: Character name for caching
|
||||
**params: Additional ChatterBox parameters
|
||||
|
||||
Returns:
|
||||
Generated audio tensor
|
||||
"""
|
||||
# Extract ChatterBox Official 23-Lang specific parameters
|
||||
exaggeration = params.get("exaggeration", 1.0)
|
||||
temperature = params.get("temperature", 0.8)
|
||||
cfg_weight = params.get("cfg_weight", 1.0)
|
||||
repetition_penalty = params.get("repetition_penalty", 2.0)
|
||||
min_p = params.get("min_p", 0.05)
|
||||
top_p = params.get("top_p", 1.0)
|
||||
language_id = params.get("language_id", "en")
|
||||
seed = params.get("seed", 0)
|
||||
enable_cache = params.get("enable_audio_cache", True)
|
||||
|
||||
# Create cache function if caching is enabled
|
||||
cache_fn = None
|
||||
if enable_cache:
|
||||
from utils.audio.cache import create_cache_function
|
||||
|
||||
# Get current language/model for cache key
|
||||
current_language = params.get("current_language", params.get("model", "English"))
|
||||
audio_component = params.get("stable_audio_component", "main_reference")
|
||||
if character != "narrator":
|
||||
audio_component = f"char_file_{character}"
|
||||
|
||||
# Get model source
|
||||
model_source = params.get("model_source")
|
||||
if not model_source and hasattr(self.node, 'model_manager'):
|
||||
model_source = self.node.model_manager.get_model_source("tts")
|
||||
|
||||
cache_fn = create_cache_function(
|
||||
engine_type="chatterbox_official_23lang",
|
||||
character=character,
|
||||
exaggeration=exaggeration,
|
||||
temperature=temperature,
|
||||
cfg_weight=cfg_weight,
|
||||
seed=seed,
|
||||
audio_component=audio_component,
|
||||
model_source=model_source or "unknown",
|
||||
device=params.get("device", "auto"),
|
||||
language=current_language
|
||||
)
|
||||
|
||||
# Handle caching externally for consistency with F5-TTS
|
||||
if cache_fn:
|
||||
# Check cache first
|
||||
cached_audio = cache_fn(text)
|
||||
if cached_audio is not None:
|
||||
return cached_audio
|
||||
|
||||
# CRITICAL FIX: Handle caching based on pause tag presence
|
||||
# If pause tags detected, let internal pause processor handle caching of text segments
|
||||
# If no pause tags, use external caching as normal
|
||||
enable_pause_tags = params.get("enable_pause_tags", True)
|
||||
has_pause_tags = enable_pause_tags and "[pause:" in text
|
||||
|
||||
if has_pause_tags:
|
||||
# Let internal pause processor handle caching of individual text segments
|
||||
audio_result = self.node._generate_tts_with_pause_tags(
|
||||
text=text,
|
||||
audio_prompt=char_audio,
|
||||
exaggeration=exaggeration,
|
||||
temperature=temperature,
|
||||
cfg_weight=cfg_weight,
|
||||
language=params.get("current_language", params.get("model", "English")),
|
||||
enable_pause_tags=True,
|
||||
character=character,
|
||||
seed=seed,
|
||||
enable_cache=True, # Enable internal caching for pause tag processing
|
||||
crash_protection_template=params.get("crash_protection_template", "hmm ,, {seg} hmm ,,"),
|
||||
stable_audio_component=params.get("stable_audio_component", "main_reference")
|
||||
)
|
||||
# Don't use external cache for pause tag segments
|
||||
else:
|
||||
# No pause tags, use external caching as normal
|
||||
audio_result = self.node._generate_tts_with_pause_tags(
|
||||
text=text,
|
||||
audio_prompt=char_audio,
|
||||
exaggeration=exaggeration,
|
||||
temperature=temperature,
|
||||
cfg_weight=cfg_weight,
|
||||
language=params.get("current_language", params.get("model", "English")),
|
||||
enable_pause_tags=False,
|
||||
character=character,
|
||||
seed=seed,
|
||||
enable_cache=False, # Disable internal caching since we handle it externally
|
||||
crash_protection_template=params.get("crash_protection_template", "hmm ,, {seg} hmm ,,"),
|
||||
stable_audio_component=params.get("stable_audio_component", "main_reference")
|
||||
)
|
||||
|
||||
# Cache the result if external caching is enabled
|
||||
if cache_fn:
|
||||
cache_fn(text, audio_result)
|
||||
|
||||
return audio_result
|
||||
|
||||
def generate_batch_audio(self, text_segments: List[str], char_audio: str,
|
||||
character: str = "narrator", batch_size: Optional[int] = None,
|
||||
**params) -> List[torch.Tensor]:
|
||||
"""
|
||||
Generate ChatterBox audio for multiple text segments using true batch processing.
|
||||
|
||||
Args:
|
||||
text_segments: List of text segments to generate audio for
|
||||
char_audio: Reference audio file path
|
||||
character: Character name for caching
|
||||
batch_size: Optional batch size override (uses self.batch_size if None)
|
||||
**params: Additional ChatterBox parameters
|
||||
|
||||
Returns:
|
||||
List of generated audio tensors
|
||||
"""
|
||||
if not self.enable_batch_processing or len(text_segments) == 1:
|
||||
# Fall back to sequential processing if batch processing is disabled or single segment
|
||||
return [self.generate_segment_audio(text, char_audio, character, **params)
|
||||
for text in text_segments]
|
||||
|
||||
batch_size = batch_size or self.batch_size
|
||||
print(f"🚀 ChatterBox batch processing: {len(text_segments)} segments in batches of {batch_size}")
|
||||
|
||||
audio_results = []
|
||||
|
||||
# Process segments in batches using the engine's generate_batch method
|
||||
for i in range(0, len(text_segments), batch_size):
|
||||
batch_texts = text_segments[i:i + batch_size]
|
||||
print(f"🔄 Processing batch {i//batch_size + 1}: {len(batch_texts)} segments")
|
||||
|
||||
try:
|
||||
# Extract ChatterBox specific parameters
|
||||
exaggeration = params.get("exaggeration", 1.0)
|
||||
temperature = params.get("temperature", 0.8)
|
||||
cfg_weight = params.get("cfg_weight", 1.0)
|
||||
|
||||
# Use the ChatterBox engine's generate_batch method for true batch processing
|
||||
if hasattr(self.node, 'tts_model') and hasattr(self.node.tts_model, 'generate_batch'):
|
||||
print(f"🔧 Using overlapping batch processing with max_workers={self.batch_size}")
|
||||
batch_audio = self.node.tts_model.generate_batch(
|
||||
texts=batch_texts,
|
||||
audio_prompt_path=char_audio,
|
||||
exaggeration=exaggeration,
|
||||
cfg_weight=cfg_weight,
|
||||
temperature=temperature,
|
||||
batch_size=len(batch_texts),
|
||||
max_workers=self.batch_size # Use the UI batch_size as max_workers
|
||||
)
|
||||
|
||||
print(f"✅ Batch {i//batch_size + 1} completed: {len(batch_audio)} audio segments generated")
|
||||
audio_results.extend(batch_audio)
|
||||
|
||||
else:
|
||||
# Fallback to sequential generation if batch method not available
|
||||
print(f"⚠️ Batch method not available, falling back to sequential processing")
|
||||
for text in batch_texts:
|
||||
audio = self.generate_segment_audio(text, char_audio, character, **params)
|
||||
audio_results.append(audio)
|
||||
|
||||
except Exception as e:
|
||||
print(f"⚠️ Batch processing failed for batch {i//batch_size + 1}: {e}")
|
||||
print(f"🔄 Falling back to sequential processing for this batch")
|
||||
# Fall back to sequential generation for failed batch
|
||||
for text in batch_texts:
|
||||
try:
|
||||
audio = self.generate_segment_audio(text, char_audio, character, **params)
|
||||
audio_results.append(audio)
|
||||
except Exception as fallback_error:
|
||||
print(f"❌ Sequential fallback also failed for '{text[:30]}...': {fallback_error}")
|
||||
# Generate silence as last resort
|
||||
from utils.audio.processing import AudioProcessingUtils
|
||||
silence = AudioProcessingUtils.create_silence(1.0, 44100)
|
||||
audio_results.append(silence)
|
||||
|
||||
print(f"🎉 Batch processing complete: {len(audio_results)} total segments generated")
|
||||
return audio_results
|
||||
|
||||
def _generate_single_with_lock(self, text: str, char_audio: str,
|
||||
character: str = "narrator", **params) -> torch.Tensor:
|
||||
"""
|
||||
Generate audio for a single segment with thread-safe model access.
|
||||
|
||||
Args:
|
||||
text: Text to generate audio for
|
||||
char_audio: Reference audio file path
|
||||
character: Character name
|
||||
**params: Additional parameters
|
||||
|
||||
Returns:
|
||||
Generated audio tensor
|
||||
"""
|
||||
# Use lock to ensure thread-safe model access
|
||||
with self._generation_lock:
|
||||
return self.generate_segment_audio(text, char_audio, character, **params)
|
||||
|
||||
def set_batch_processing(self, enabled: bool, batch_size: Optional[int] = None):
|
||||
"""
|
||||
Configure batch processing settings.
|
||||
|
||||
Args:
|
||||
enabled: Whether to enable batch processing
|
||||
batch_size: Optional new batch size (1-32 supported, uses overlapping parallel processing)
|
||||
"""
|
||||
self.enable_batch_processing = enabled
|
||||
if batch_size is not None:
|
||||
self.batch_size = max(1, min(32, batch_size)) # Allow up to 32 parallel workers
|
||||
|
||||
status = "enabled" if enabled else "disabled"
|
||||
print(f"🔧 ChatterBox batch processing {status} (batch_size={self.batch_size})")
|
||||
|
||||
def combine_audio_chunks(self, audio_segments: List[torch.Tensor], **params) -> torch.Tensor:
|
||||
"""
|
||||
Combine ChatterBox audio segments using modular combination utility.
|
||||
|
||||
Args:
|
||||
audio_segments: List of audio tensors to combine
|
||||
**params: Combination parameters
|
||||
|
||||
Returns:
|
||||
Combined audio tensor, or tuple with timing info if requested
|
||||
"""
|
||||
if len(audio_segments) == 1:
|
||||
return audio_segments[0]
|
||||
|
||||
# Use modular chunk combiner with ChatterBox settings
|
||||
from utils.audio.chunk_combiner import ChunkCombiner
|
||||
|
||||
method = params.get("combination_method", "auto")
|
||||
silence_ms = params.get("silence_ms", 100)
|
||||
text_length = params.get("text_length", 0)
|
||||
original_text = params.get("original_text", "")
|
||||
text_chunks = params.get("text_chunks", None)
|
||||
return_info = params.get("return_timing_info", False)
|
||||
|
||||
print(f"🔗 Combining {len(audio_segments)} ChatterBox chunks using '{method}' method")
|
||||
|
||||
return ChunkCombiner.combine_chunks(
|
||||
audio_segments=audio_segments,
|
||||
method=method,
|
||||
silence_ms=silence_ms,
|
||||
crossfade_duration=0.1,
|
||||
sample_rate=44100, # ChatterBox sample rate
|
||||
text_length=text_length,
|
||||
original_text=original_text,
|
||||
text_chunks=text_chunks,
|
||||
return_info=return_info
|
||||
)
|
||||
|
||||
def generate_audio(self, text: str, voice_preset: str, voice_settings: dict,
|
||||
output_filename: str, language: str = "auto",
|
||||
enable_batch_processing: bool = False, batch_size: int = 4) -> dict:
|
||||
"""
|
||||
Generate audio using ChatterBox adapter with batch processing support.
|
||||
|
||||
This method provides the interface expected by the unified TTS system.
|
||||
"""
|
||||
# For single text generation, use the existing segment generation
|
||||
if not enable_batch_processing:
|
||||
print(f"🎤 ChatterBox single generation: '{text[:50]}...'")
|
||||
|
||||
# Use the existing node functionality for single generation
|
||||
audio_result = self.node.generate_speech(
|
||||
text=text,
|
||||
language=language,
|
||||
device=self.node.device if hasattr(self.node, 'device') else 'auto',
|
||||
exaggeration=voice_settings.get('exaggeration', 0.5),
|
||||
temperature=voice_settings.get('temperature', 0.8),
|
||||
cfg_weight=voice_settings.get('cfg_weight', 0.5),
|
||||
seed=voice_settings.get('seed', 0),
|
||||
reference_audio=voice_settings.get('reference_audio'),
|
||||
audio_prompt_path=voice_settings.get('audio_prompt_path', ''),
|
||||
batch_size=1 if not enable_batch_processing else batch_size
|
||||
)
|
||||
|
||||
return {
|
||||
'audio': audio_result[0], # Audio tensor
|
||||
'info': audio_result[1] if len(audio_result) > 1 else 'Generated with ChatterBox'
|
||||
}
|
||||
else:
|
||||
# For batch processing, split text into chunks and use batch generation
|
||||
print(f"🚀 ChatterBox batch generation enabled: batch_size={batch_size}")
|
||||
|
||||
# Split text into reasonable chunks for batch processing
|
||||
from utils.text.chunking import ImprovedChatterBoxChunker
|
||||
chunker = ImprovedChatterBoxChunker()
|
||||
|
||||
# Use chunking to split text into segments
|
||||
max_chars_per_chunk = voice_settings.get('max_chars_per_chunk', 400)
|
||||
if len(text) > max_chars_per_chunk:
|
||||
text_segments = chunker.split_into_chunks(text, max_chars_per_chunk)
|
||||
print(f"📝 Text split into {len(text_segments)} chunks for batch processing")
|
||||
else:
|
||||
text_segments = [text]
|
||||
|
||||
# Generate audio for all segments using batch processing
|
||||
audio_segments = self.generate_batch_audio(
|
||||
text_segments=text_segments,
|
||||
char_audio=voice_settings.get('audio_prompt_path', ''),
|
||||
character=voice_preset or 'narrator',
|
||||
batch_size=batch_size,
|
||||
exaggeration=voice_settings.get('exaggeration', 0.5),
|
||||
temperature=voice_settings.get('temperature', 0.8),
|
||||
cfg_weight=voice_settings.get('cfg_weight', 0.5),
|
||||
seed=voice_settings.get('seed', 0),
|
||||
current_language=language,
|
||||
device=self.node.device if hasattr(self.node, 'device') else 'auto'
|
||||
)
|
||||
|
||||
# Combine audio segments
|
||||
if len(audio_segments) == 1:
|
||||
combined_audio = audio_segments[0]
|
||||
else:
|
||||
combined_audio = self.combine_audio_chunks(audio_segments)
|
||||
|
||||
return {
|
||||
'audio': combined_audio,
|
||||
'info': f'Generated {len(text_segments)} chunks using ChatterBox batch processing (batch_size={batch_size})'
|
||||
}
|
||||
|
||||
def _get_audio_duration(self, audio_tensor: torch.Tensor) -> float:
|
||||
"""Calculate audio duration in seconds."""
|
||||
# ChatterBox uses 44.1kHz sample rate
|
||||
if audio_tensor.dim() == 1:
|
||||
num_samples = audio_tensor.shape[0]
|
||||
elif audio_tensor.dim() == 2:
|
||||
num_samples = audio_tensor.shape[1]
|
||||
else:
|
||||
num_samples = audio_tensor.numel()
|
||||
|
||||
return num_samples / 44100 # ChatterBox sample rate
|
||||
@@ -103,7 +103,6 @@ class ChatterBoxStreamingAdapter(StreamingEngineAdapter):
|
||||
if has_streaming_method:
|
||||
# Build inputs dict like the old system
|
||||
inputs = {
|
||||
"character": segment.character, # CRITICAL: Character is needed for caching
|
||||
"exaggeration": kwargs.get("exaggeration", 0.5),
|
||||
"temperature": kwargs.get("temperature", 0.8),
|
||||
"cfg_weight": kwargs.get("cfg_weight", 0.5),
|
||||
@@ -114,15 +113,6 @@ class ChatterBoxStreamingAdapter(StreamingEngineAdapter):
|
||||
"device": kwargs.get("device", "auto"),
|
||||
"reference_audio": kwargs.get("reference_audio", None)
|
||||
}
|
||||
|
||||
# Apply segment-level parameters if available
|
||||
segment_params = segment.metadata.get('parameters', {}) if segment.metadata else {}
|
||||
if segment_params:
|
||||
# Import parameter application function
|
||||
from utils.text.segment_parameters import apply_segment_parameters
|
||||
segment_config = apply_segment_parameters(inputs, segment_params, "chatterbox_official_23lang")
|
||||
inputs.update(segment_config)
|
||||
print(f" 📊 Streaming: Applying segment parameters: {segment_params}")
|
||||
|
||||
# Call exactly like the old working system
|
||||
audio = self.node._process_single_segment_for_streaming(
|
||||
@@ -305,25 +295,35 @@ class ChatterBoxStreamingAdapter(StreamingEngineAdapter):
|
||||
# Check if node has generation method
|
||||
if hasattr(self.node, '_generate_tts_with_pause_tags'):
|
||||
print(f"🏷️ ADAPTER: Calling _generate_tts_with_pause_tags for '{text[:30]}...'")
|
||||
|
||||
# Load the correct model for this language
|
||||
self.load_model_for_language(language, kwargs.get('device', 'auto'))
|
||||
|
||||
# Generate stable audio component for cache consistency
|
||||
from utils.audio.audio_hash import generate_stable_audio_component
|
||||
stable_audio_component = generate_stable_audio_component(
|
||||
kwargs.get("reference_audio"), voice_path
|
||||
)
|
||||
|
||||
# Use pause tag-aware generation
|
||||
# Note: tts_model is now a wrapper with __getattr__ forwarding, no unwrapping needed
|
||||
return self.node._generate_tts_with_pause_tags(
|
||||
text, voice_path, exaggeration, temperature, cfg_weight,
|
||||
language, True, character=character, seed=seed,
|
||||
enable_cache=enable_cache,
|
||||
crash_protection_template=crash_protection,
|
||||
stable_audio_component=stable_audio_component
|
||||
)
|
||||
|
||||
# Temporarily replace stateless wrapper with underlying model for pause tag processing
|
||||
original_model = getattr(self.node, 'tts_model', None)
|
||||
try:
|
||||
# Load the correct model for this language and extract underlying model if needed
|
||||
self.load_model_for_language(language, kwargs.get('device', 'auto'))
|
||||
|
||||
if hasattr(self.node, 'tts_model') and hasattr(self.node.tts_model, 'model'):
|
||||
self.node.tts_model = self.node.tts_model.model
|
||||
print(f"🔓 ADAPTER: Extracted underlying model for pause tag processing")
|
||||
|
||||
# Generate stable audio component for cache consistency
|
||||
from utils.audio.audio_hash import generate_stable_audio_component
|
||||
stable_audio_component = generate_stable_audio_component(
|
||||
kwargs.get("reference_audio"), voice_path
|
||||
)
|
||||
|
||||
# Use pause tag-aware generation
|
||||
return self.node._generate_tts_with_pause_tags(
|
||||
text, voice_path, exaggeration, temperature, cfg_weight,
|
||||
language, True, character=character, seed=seed,
|
||||
enable_cache=enable_cache,
|
||||
crash_protection_template=crash_protection,
|
||||
stable_audio_component=stable_audio_component
|
||||
)
|
||||
finally:
|
||||
# Restore original model
|
||||
if original_model is not None:
|
||||
self.node.tts_model = original_model
|
||||
elif hasattr(self.node, 'tts_model') and self.node.tts_model:
|
||||
# Direct model generation
|
||||
return self.node.tts_model.generate(
|
||||
@@ -352,13 +352,12 @@ class ChatterBoxStreamingAdapter(StreamingEngineAdapter):
|
||||
Returns:
|
||||
True if segment can be processed
|
||||
"""
|
||||
# Check language support - fallback to English for unsupported languages
|
||||
# Check language support
|
||||
if segment.language not in self.supported_languages:
|
||||
# Check if it's a local model
|
||||
if not segment.language.startswith('local:'):
|
||||
print(f"⚠️ ChatterBox: Language '{segment.language}' not supported, falling back to English model")
|
||||
# Modify segment language to English for fallback
|
||||
segment.language = 'en'
|
||||
print(f"❌ ChatterBox doesn't support language: {segment.language}")
|
||||
return False
|
||||
|
||||
# Basic validation
|
||||
if not segment.text.strip():
|
||||
|
||||
@@ -1,448 +0,0 @@
|
||||
"""
|
||||
CosyVoice3 Engine Adapter
|
||||
|
||||
Provides standardized interface for CosyVoice3 integration with TTS Audio Suite.
|
||||
Handles parameter mapping, character switching, and mode selection.
|
||||
"""
|
||||
|
||||
# FIX: PyYAML 6.0+ compatibility patch - MUST be before any yaml imports
|
||||
# Issue: 'Loader' object has no attribute 'max_depth' error (GitHub #220)
|
||||
import yaml
|
||||
if not hasattr(yaml.Loader, 'max_depth'):
|
||||
yaml.Loader.max_depth = 100
|
||||
if not hasattr(yaml.FullLoader, 'max_depth'):
|
||||
yaml.FullLoader.max_depth = 100
|
||||
if not hasattr(yaml.SafeLoader, 'max_depth'):
|
||||
yaml.SafeLoader.max_depth = 100
|
||||
|
||||
import os
|
||||
import torch
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from engines.cosyvoice.cosyvoice import CosyVoiceEngine
|
||||
from engines.cosyvoice.cosyvoice_downloader import cosyvoice_downloader
|
||||
from utils.text.character_parser import character_parser
|
||||
from utils.voice.discovery import get_character_mapping, get_available_characters
|
||||
from utils.voice.character_logging import resolved_character_label
|
||||
from utils.audio.cache import get_audio_cache
|
||||
|
||||
|
||||
class CosyVoiceAdapter:
|
||||
"""
|
||||
Adapter for CosyVoice3 engine providing unified interface compatibility.
|
||||
|
||||
Handles:
|
||||
- Parameter mapping between unified interface and CosyVoice3
|
||||
- Character switching with [CharacterName] syntax
|
||||
- Mode selection (zero_shot, instruct, cross_lingual)
|
||||
- Caching integration
|
||||
- Model management
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize the CosyVoice3 adapter."""
|
||||
self.engine = None
|
||||
self.audio_cache = get_audio_cache()
|
||||
self.model_variant = None # Track model variant for cache invalidation
|
||||
# Store initialization parameters for lazy loading
|
||||
self._init_params = {
|
||||
"model_path": None,
|
||||
"device": "auto",
|
||||
"use_fp16": True,
|
||||
"load_trt": False,
|
||||
"load_vllm": False
|
||||
}
|
||||
|
||||
def initialize_engine(self,
|
||||
model_path: Optional[str] = None,
|
||||
device: str = "auto",
|
||||
use_fp16: bool = True,
|
||||
load_trt: bool = False,
|
||||
load_vllm: bool = False):
|
||||
"""
|
||||
Initialize CosyVoice3 engine.
|
||||
|
||||
Args:
|
||||
model_path: Path to model directory (auto-downloaded if None)
|
||||
device: Target device
|
||||
use_fp16: Use FP16 for inference
|
||||
load_trt: Load TensorRT engine
|
||||
load_vllm: Load vLLM engine
|
||||
"""
|
||||
# Store initialization parameters for lazy loading
|
||||
self._init_params.update({
|
||||
"model_path": model_path,
|
||||
"device": device,
|
||||
"use_fp16": use_fp16,
|
||||
"load_trt": load_trt,
|
||||
"load_vllm": load_vllm
|
||||
})
|
||||
|
||||
# Auto-download model if not provided
|
||||
if model_path is None or model_path == "Fun-CosyVoice3-0.5B":
|
||||
if not cosyvoice_downloader.is_model_available():
|
||||
print("📥 CosyVoice3 model not found, downloading...")
|
||||
model_path = cosyvoice_downloader.download_model()
|
||||
else:
|
||||
model_path = cosyvoice_downloader.get_model_path()
|
||||
|
||||
# Store model variant for cache key (extract from model_path string)
|
||||
if "RL" in str(model_path):
|
||||
self.model_variant = "RL"
|
||||
else:
|
||||
self.model_variant = "standard"
|
||||
|
||||
# Initialize engine
|
||||
self.engine = CosyVoiceEngine(
|
||||
model_dir=model_path,
|
||||
device=device,
|
||||
use_fp16=use_fp16,
|
||||
load_trt=load_trt,
|
||||
load_vllm=load_vllm
|
||||
)
|
||||
|
||||
def generate(self,
|
||||
text: str,
|
||||
speaker_audio: Optional[str] = None,
|
||||
reference_text: Optional[str] = None,
|
||||
instruct_text: Optional[str] = None,
|
||||
mode: Optional[str] = None,
|
||||
speed: float = 1.0,
|
||||
stream: bool = False,
|
||||
model_path: Optional[str] = None,
|
||||
**kwargs) -> torch.Tensor:
|
||||
"""
|
||||
Generate speech with CosyVoice3.
|
||||
|
||||
Args:
|
||||
model_path: Model path/identifier (used to determine variant for caching)
|
||||
|
||||
Mode is auto-detected:
|
||||
- If instruct_text provided → instruct mode
|
||||
- If reference_text from Character Voices → zero_shot mode
|
||||
- Otherwise → cross_lingual mode
|
||||
|
||||
Args:
|
||||
text: Text to synthesize (supports [character] tags)
|
||||
speaker_audio: Speaker reference audio file path
|
||||
reference_text: Transcript of reference audio (from Character Voices .txt)
|
||||
instruct_text: Optional instruction for dialect/emotion/speed control
|
||||
speed: Speech speed multiplier (0.5-2.0)
|
||||
stream: Enable streaming
|
||||
**kwargs: Additional parameters
|
||||
|
||||
Returns:
|
||||
Generated audio tensor [1, samples] at 24000 Hz
|
||||
"""
|
||||
if self.engine is None:
|
||||
self.initialize_engine(**self._init_params)
|
||||
|
||||
# ALWAYS update model_variant from model_path parameter for cache invalidation
|
||||
# This ensures switching variants invalidates audio cache even if engine is reused
|
||||
if model_path:
|
||||
self.model_variant = "RL" if "RL" in str(model_path) else "standard"
|
||||
|
||||
# Auto-detect mode based on available parameters (if not explicitly provided)
|
||||
if mode is None:
|
||||
# print(f"🔍 Mode detection - instruct_text: '{instruct_text}', reference_text: '{reference_text[:50] if reference_text else None}...'")
|
||||
if instruct_text and instruct_text.strip():
|
||||
mode = "instruct"
|
||||
print(f"✅ Selected mode: instruct")
|
||||
if reference_text and reference_text.strip():
|
||||
print("⚠️ CosyVoice3: Both instruction and transcript provided. Using instruction mode (transcript ignored for best dialect/emotion control).")
|
||||
elif reference_text and reference_text.strip():
|
||||
mode = "zero_shot"
|
||||
print(f"✅ Selected mode: zero_shot")
|
||||
else:
|
||||
mode = "cross_lingual"
|
||||
print(f"✅ Selected mode: cross_lingual")
|
||||
else:
|
||||
print(f"✅ Using explicitly provided mode: {mode}")
|
||||
|
||||
# Check if text contains character tags
|
||||
has_character_tags = character_parser.CHARACTER_TAG_PATTERN.search(text) is not None
|
||||
|
||||
if has_character_tags:
|
||||
# Parse character switching tags
|
||||
processed_segments = self._process_character_tags(text)
|
||||
|
||||
if len(processed_segments) > 1:
|
||||
# Multi-segment character switching
|
||||
return self._generate_multi_character_segments(
|
||||
processed_segments,
|
||||
speaker_audio,
|
||||
reference_text,
|
||||
instruct_text=instruct_text,
|
||||
speed=speed,
|
||||
**kwargs
|
||||
)
|
||||
elif processed_segments:
|
||||
# Single character segment
|
||||
first_segment = processed_segments[0]
|
||||
processed_text = first_segment.get('text', '').strip()
|
||||
character_name = first_segment.get('character')
|
||||
else:
|
||||
processed_text = text
|
||||
character_name = None
|
||||
else:
|
||||
processed_text = text
|
||||
character_name = None
|
||||
|
||||
# Determine final speaker audio
|
||||
final_speaker_audio = speaker_audio
|
||||
final_reference_text = reference_text
|
||||
|
||||
# Only do character mapping if we have character tags
|
||||
if has_character_tags and character_name:
|
||||
character_mapping = get_character_mapping([character_name], engine_type="cosyvoice")
|
||||
|
||||
if character_name in character_mapping:
|
||||
char_audio, char_text = character_mapping[character_name]
|
||||
if char_audio is not None:
|
||||
final_speaker_audio = char_audio
|
||||
# Use character's reference text if available
|
||||
if char_text:
|
||||
final_reference_text = char_text
|
||||
else:
|
||||
# Character has audio but NO text reference - clear reference_text
|
||||
# This forces cross_lingual mode instead of zero_shot with wrong voice
|
||||
final_reference_text = None
|
||||
print(f"🎭 Using character voice: {character_name} -> {final_speaker_audio}")
|
||||
|
||||
# Generate cache key with mode and model variant explicitly included
|
||||
cache_key = self._generate_cache_key(
|
||||
text=processed_text,
|
||||
speaker_audio=self._get_stable_audio_identifier(final_speaker_audio),
|
||||
reference_text=final_reference_text,
|
||||
instruct_text=instruct_text,
|
||||
mode=mode,
|
||||
speed=speed,
|
||||
model_variant=self.model_variant, # Include variant for cache invalidation
|
||||
**kwargs
|
||||
)
|
||||
|
||||
# print(f"🔑 Audio cache key: {cache_key[:100]}... (variant={self.model_variant})")
|
||||
|
||||
# Check cache
|
||||
cached_audio = self.audio_cache.get_cached_audio(cache_key)
|
||||
if cached_audio:
|
||||
print(f"💾 Using cached CosyVoice3 audio for: '{processed_text[:30]}...'")
|
||||
return cached_audio[0]
|
||||
|
||||
# Handle seed for deterministic generation
|
||||
seed = kwargs.get('seed', 0)
|
||||
if seed != 0:
|
||||
torch.manual_seed(seed)
|
||||
import random
|
||||
import numpy as np
|
||||
random.seed(seed)
|
||||
np.random.seed(seed)
|
||||
|
||||
# Convert CosyVoice paralinguistic tags from <tag> to [tag]
|
||||
from utils.text.cosyvoice_special_tags import convert_cosyvoice_special_tags
|
||||
processed_text = convert_cosyvoice_special_tags(processed_text)
|
||||
|
||||
# Generate audio
|
||||
audio = self.engine.generate(
|
||||
text=processed_text,
|
||||
prompt_wav=final_speaker_audio,
|
||||
prompt_text=final_reference_text,
|
||||
mode=mode,
|
||||
instruct_text=instruct_text,
|
||||
speed=speed,
|
||||
stream=stream
|
||||
)
|
||||
|
||||
# Cache the result
|
||||
duration = audio.shape[-1] / 24000.0 # CosyVoice3 uses 24000 Hz
|
||||
self.audio_cache.cache_audio(cache_key, audio, duration)
|
||||
|
||||
return audio
|
||||
|
||||
def _process_character_tags(self, text: str) -> List[Dict[str, Any]]:
|
||||
"""Process character switching tags."""
|
||||
from utils.text.character_parser.base_parser import CharacterParser
|
||||
from utils.voice.discovery import get_available_characters, voice_discovery
|
||||
|
||||
temp_parser = CharacterParser()
|
||||
|
||||
# Get available characters and aliases
|
||||
available_chars = get_available_characters()
|
||||
character_aliases = voice_discovery.get_character_aliases()
|
||||
|
||||
all_available = set()
|
||||
if available_chars:
|
||||
all_available.update(available_chars)
|
||||
for alias, target in character_aliases.items():
|
||||
all_available.add(alias.lower())
|
||||
all_available.add(target.lower())
|
||||
|
||||
temp_parser.set_available_characters(list(all_available))
|
||||
|
||||
# Set language defaults
|
||||
char_lang_defaults = voice_discovery.get_character_language_defaults()
|
||||
for char, lang in char_lang_defaults.items():
|
||||
temp_parser.set_character_language_default(char, lang)
|
||||
|
||||
# Parse segments with language information
|
||||
segments = temp_parser.split_by_character(text, include_language=True)
|
||||
|
||||
processed_segments = []
|
||||
for character, segment_text, language in segments:
|
||||
segment_info = {
|
||||
'character': character,
|
||||
'text': segment_text,
|
||||
'language': language
|
||||
}
|
||||
processed_segments.append(segment_info)
|
||||
|
||||
return processed_segments
|
||||
|
||||
def _generate_multi_character_segments(
|
||||
self,
|
||||
segments: List[Dict[str, Any]],
|
||||
default_speaker_audio: Optional[str],
|
||||
default_reference_text: Optional[str],
|
||||
**kwargs
|
||||
) -> torch.Tensor:
|
||||
"""Generate audio for multiple character segments and combine them."""
|
||||
audio_segments = []
|
||||
|
||||
# Get character mapping for all unique characters
|
||||
unique_characters = set()
|
||||
for segment in segments:
|
||||
if segment.get('character'):
|
||||
unique_characters.add(segment['character'])
|
||||
|
||||
character_mapping = {}
|
||||
if unique_characters:
|
||||
character_mapping = get_character_mapping(list(unique_characters), engine_type="cosyvoice")
|
||||
|
||||
print(f"🎭 CosyVoice3: Processing {len(segments)} character segment(s)")
|
||||
|
||||
for segment in segments:
|
||||
character_name = segment.get('character', 'narrator')
|
||||
segment_text = segment.get('text', '').strip()
|
||||
|
||||
if not segment_text:
|
||||
continue
|
||||
|
||||
# Determine speaker audio for this segment
|
||||
speaker_audio = default_speaker_audio
|
||||
reference_text = default_reference_text
|
||||
|
||||
if character_name and character_name in character_mapping:
|
||||
char_audio, char_text = character_mapping[character_name]
|
||||
if char_audio:
|
||||
speaker_audio = char_audio
|
||||
if char_text:
|
||||
reference_text = char_text
|
||||
print(f"📖 Using character voice '{resolved_character_label(character_name, speaker_audio)}'")
|
||||
|
||||
# Generate cache key for this segment
|
||||
segment_cache_key = self._generate_cache_key(
|
||||
text=segment_text,
|
||||
speaker_audio=self._get_stable_audio_identifier(speaker_audio),
|
||||
reference_text=reference_text,
|
||||
instruct_text=kwargs.get('instruct_text'),
|
||||
mode=kwargs.get('mode'),
|
||||
speed=kwargs.get('speed', 1.0),
|
||||
model_variant=self.model_variant # Include variant for cache invalidation
|
||||
)
|
||||
|
||||
# Check cache first
|
||||
cached_segment_audio = self.audio_cache.get_cached_audio(segment_cache_key)
|
||||
if cached_segment_audio:
|
||||
print(f"💾 Using cached CosyVoice3 segment for '{resolved_character_label(character_name, speaker_audio)}'")
|
||||
segment_audio = cached_segment_audio[0]
|
||||
else:
|
||||
# Convert CosyVoice paralinguistic tags from <tag> to [tag]
|
||||
from utils.text.cosyvoice_special_tags import convert_cosyvoice_special_tags
|
||||
segment_text_converted = convert_cosyvoice_special_tags(segment_text)
|
||||
|
||||
# Generate audio for this segment
|
||||
segment_audio = self.engine.generate(
|
||||
text=segment_text_converted,
|
||||
prompt_wav=speaker_audio,
|
||||
prompt_text=reference_text,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
# Cache the segment
|
||||
duration = segment_audio.shape[-1] / 24000.0 # CosyVoice3 uses 24000 Hz
|
||||
self.audio_cache.cache_audio(segment_cache_key, segment_audio, duration)
|
||||
|
||||
audio_segments.append(segment_audio)
|
||||
|
||||
# Combine all audio segments
|
||||
if audio_segments:
|
||||
combined_audio = torch.cat(audio_segments, dim=-1)
|
||||
return combined_audio
|
||||
else:
|
||||
return torch.zeros(1, 24000, dtype=torch.float32) # 1 second at CosyVoice3's 24000 Hz
|
||||
|
||||
def _generate_cache_key(self, **params) -> str:
|
||||
"""Generate cache key for CosyVoice3."""
|
||||
return self.audio_cache.generate_cache_key('cosyvoice', **params)
|
||||
|
||||
def _get_stable_audio_identifier(self, audio_path: str) -> str:
|
||||
"""Get stable identifier for audio file using centralized audio hashing."""
|
||||
if not audio_path:
|
||||
return audio_path
|
||||
|
||||
from utils.audio.audio_hash import generate_stable_audio_component
|
||||
return generate_stable_audio_component(audio_file_path=audio_path)
|
||||
|
||||
def get_supported_formats(self) -> List[str]:
|
||||
"""Get supported audio formats."""
|
||||
return ["wav", "mp3", "flac", "ogg"]
|
||||
|
||||
def get_sample_rate(self) -> int:
|
||||
"""Get native sample rate."""
|
||||
return 24000
|
||||
|
||||
def get_supported_languages(self) -> List[str]:
|
||||
"""Get supported languages."""
|
||||
return CosyVoiceEngine.SUPPORTED_LANGUAGES.copy()
|
||||
|
||||
def get_supported_dialects(self) -> List[str]:
|
||||
"""Get supported Chinese dialects."""
|
||||
return CosyVoiceEngine.SUPPORTED_DIALECTS.copy()
|
||||
|
||||
def inference_vc(self,
|
||||
source_wav: str,
|
||||
prompt_wav: str,
|
||||
speed: float = 1.0,
|
||||
stream: bool = False) -> torch.Tensor:
|
||||
"""
|
||||
Voice conversion using CosyVoice3.
|
||||
|
||||
Args:
|
||||
source_wav: Source audio file path (CosyVoice handles resampling)
|
||||
prompt_wav: Reference/target voice file path
|
||||
speed: Speech speed multiplier
|
||||
stream: Enable streaming
|
||||
|
||||
Returns:
|
||||
Converted audio tensor at 24kHz
|
||||
"""
|
||||
if self.engine is None:
|
||||
self.initialize_engine(**self._init_params)
|
||||
|
||||
# Call engine's VC method
|
||||
return self.engine.inference_vc(
|
||||
source_wav=source_wav,
|
||||
prompt_wav=prompt_wav,
|
||||
speed=speed,
|
||||
stream=stream
|
||||
)
|
||||
|
||||
def unload(self):
|
||||
"""Unload the engine to free memory."""
|
||||
if self.engine:
|
||||
self.engine.unload()
|
||||
self.engine = None
|
||||
|
||||
def cleanup(self):
|
||||
"""Alias for unload for compatibility with processor interface."""
|
||||
self.unload()
|
||||
@@ -1,221 +0,0 @@
|
||||
"""
|
||||
Dots TTS engine adapter.
|
||||
|
||||
Translates TTS Audio Suite processor calls into the official dots.tts runtime.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
current_dir = os.path.dirname(__file__)
|
||||
engines_dir = os.path.dirname(current_dir)
|
||||
project_root = os.path.dirname(engines_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
from utils.audio.audio_hash import generate_stable_audio_component
|
||||
from utils.audio.cache import get_audio_cache
|
||||
from utils.audio.processing import AudioProcessingUtils
|
||||
from utils.models.factory_config import ModelLoadConfig
|
||||
from utils.models.language_mapper import resolve_language_alias
|
||||
from utils.voice.reference import effective_voice_audio
|
||||
from engines.dots_tts.languages import normalize_dots_language
|
||||
|
||||
|
||||
class DotsTTSEngineAdapter:
|
||||
"""Adapter for official Dots TTS inference."""
|
||||
|
||||
SAMPLE_RATE = 48000
|
||||
|
||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||
self.config = config.copy() if config else {}
|
||||
self.audio_cache = get_audio_cache()
|
||||
self._last_config: Optional[ModelLoadConfig] = None
|
||||
self._load_signature = None
|
||||
|
||||
def update_config(self, new_config: Dict[str, Any]):
|
||||
self.config = new_config.copy() if new_config else {}
|
||||
|
||||
def _build_load_signature(self) -> Tuple[Any, ...]:
|
||||
return (
|
||||
self.config.get("model_variant", "dots.tts-soar"),
|
||||
self.config.get("device", "auto"),
|
||||
self.config.get("precision", "auto"),
|
||||
bool(self.config.get("optimize", False)),
|
||||
int(self.config.get("max_generate_length", 500)),
|
||||
)
|
||||
|
||||
def load_model(
|
||||
self,
|
||||
model_variant: str,
|
||||
device: str = "auto",
|
||||
precision: str = "auto",
|
||||
optimize: bool = False,
|
||||
max_generate_length: int = 500,
|
||||
):
|
||||
from utils.models.unified_model_interface import unified_model_interface
|
||||
|
||||
config = ModelLoadConfig(
|
||||
engine_name="dots_tts",
|
||||
model_type="tts",
|
||||
model_name=model_variant,
|
||||
model_path=model_variant,
|
||||
device=device,
|
||||
additional_params={
|
||||
"precision": precision,
|
||||
"optimize": bool(optimize),
|
||||
"max_generate_length": int(max_generate_length),
|
||||
},
|
||||
)
|
||||
self._last_config = config
|
||||
unified_model_interface.load_model(config)
|
||||
|
||||
def _ensure_model_loaded(self):
|
||||
signature = self._build_load_signature()
|
||||
if signature == self._load_signature and self._last_config is not None:
|
||||
return
|
||||
|
||||
self.load_model(
|
||||
model_variant=self.config.get("model_variant", "dots.tts-soar"),
|
||||
device=self.config.get("device", "auto"),
|
||||
precision=self.config.get("precision", "auto"),
|
||||
optimize=self.config.get("optimize", False),
|
||||
max_generate_length=self.config.get("max_generate_length", 500),
|
||||
)
|
||||
self._load_signature = signature
|
||||
|
||||
def _get_engine(self):
|
||||
if self._last_config is None:
|
||||
self._ensure_model_loaded()
|
||||
from utils.models.unified_model_interface import unified_model_interface
|
||||
|
||||
return unified_model_interface.load_model(self._last_config)
|
||||
|
||||
def _extract_voice_reference(self, voice_ref: Optional[Dict[str, Any]]) -> Tuple[Optional[str], str, str]:
|
||||
if not voice_ref or not isinstance(voice_ref, dict):
|
||||
return None, "", "default_voice"
|
||||
|
||||
prompt_text = (
|
||||
voice_ref.get("reference_text")
|
||||
or voice_ref.get("prompt_text")
|
||||
or voice_ref.get("text")
|
||||
or ""
|
||||
).strip()
|
||||
|
||||
ref_audio = effective_voice_audio(voice_ref)
|
||||
|
||||
if ref_audio is None:
|
||||
return None, prompt_text, "default_voice"
|
||||
|
||||
if isinstance(ref_audio, str):
|
||||
return ref_audio, prompt_text, generate_stable_audio_component(audio_file_path=ref_audio)
|
||||
|
||||
if isinstance(ref_audio, dict) and "waveform" in ref_audio:
|
||||
temp_path = AudioProcessingUtils.save_audio_to_temp_file(
|
||||
ref_audio["waveform"],
|
||||
ref_audio.get("sample_rate", self.SAMPLE_RATE),
|
||||
)
|
||||
return temp_path, prompt_text, generate_stable_audio_component(reference_audio=ref_audio)
|
||||
|
||||
if torch.is_tensor(ref_audio):
|
||||
sample_rate = int(voice_ref.get("sample_rate", self.SAMPLE_RATE))
|
||||
audio_dict = {"waveform": ref_audio, "sample_rate": sample_rate}
|
||||
temp_path = AudioProcessingUtils.save_audio_to_temp_file(ref_audio, sample_rate)
|
||||
return temp_path, prompt_text, generate_stable_audio_component(reference_audio=audio_dict)
|
||||
|
||||
raise TypeError(f"Unsupported Dots TTS voice reference type: {type(ref_audio)}")
|
||||
|
||||
@staticmethod
|
||||
def _normalize_language(language: Optional[str]) -> Optional[str]:
|
||||
if language is None:
|
||||
return None
|
||||
normalized = str(language).strip()
|
||||
if not normalized:
|
||||
return None
|
||||
normalized_language = normalize_dots_language(normalized)
|
||||
if normalized_language is not None:
|
||||
return normalized_language
|
||||
resolved = resolve_language_alias(normalized)
|
||||
return normalize_dots_language(resolved)
|
||||
|
||||
def generate_single(
|
||||
self,
|
||||
text: str,
|
||||
voice_ref: Optional[Dict[str, Any]],
|
||||
seed: int = 0,
|
||||
enable_audio_cache: bool = True,
|
||||
character_name: Optional[str] = None,
|
||||
) -> torch.Tensor:
|
||||
stripped = (text or "").strip()
|
||||
if not stripped:
|
||||
return torch.zeros(1, 0, dtype=torch.float32)
|
||||
|
||||
self._ensure_model_loaded()
|
||||
engine = self._get_engine()
|
||||
|
||||
prompt_audio_path, prompt_text, audio_component = self._extract_voice_reference(voice_ref)
|
||||
language = self._normalize_language(self.config.get("language", "auto"))
|
||||
model_variant = self.config.get("model_variant", "dots.tts-soar")
|
||||
template_name = self.config.get("template_name", "tts")
|
||||
num_steps = int(self.config.get("num_steps", 10))
|
||||
guidance_scale = float(self.config.get("guidance_scale", 1.2))
|
||||
speaker_scale = float(self.config.get("speaker_scale", 1.5))
|
||||
normalize_text = bool(self.config.get("normalize_text", False))
|
||||
precision = self.config.get("precision", "auto")
|
||||
optimize = bool(self.config.get("optimize", False))
|
||||
max_generate_length = int(self.config.get("max_generate_length", 500))
|
||||
|
||||
cache_key = None
|
||||
if enable_audio_cache:
|
||||
cache_key = self.audio_cache.generate_cache_key(
|
||||
"dots_tts",
|
||||
text=stripped,
|
||||
audio_component=audio_component,
|
||||
prompt_text=prompt_text,
|
||||
model_variant=model_variant,
|
||||
language=language or "none",
|
||||
template_name=template_name,
|
||||
num_steps=num_steps,
|
||||
guidance_scale=guidance_scale,
|
||||
speaker_scale=speaker_scale,
|
||||
normalize_text=normalize_text,
|
||||
max_generate_length=max_generate_length,
|
||||
precision=precision,
|
||||
optimize=optimize,
|
||||
seed=int(seed or 0),
|
||||
device=self.config.get("device", "auto"),
|
||||
character=character_name or "narrator",
|
||||
)
|
||||
cached_audio = self.audio_cache.get_cached_audio(cache_key)
|
||||
if cached_audio:
|
||||
print(f"💾 Using cached Dots TTS audio for '{character_name or 'narrator'}': '{stripped[:30]}...'")
|
||||
return cached_audio[0]
|
||||
|
||||
result = engine.generate(
|
||||
text=stripped,
|
||||
prompt_audio_path=prompt_audio_path,
|
||||
prompt_text=prompt_text or None,
|
||||
language=language,
|
||||
num_steps=num_steps,
|
||||
guidance_scale=guidance_scale,
|
||||
speaker_scale=speaker_scale,
|
||||
normalize_text=normalize_text,
|
||||
template_name=template_name,
|
||||
seed=int(seed or 0),
|
||||
)
|
||||
|
||||
audio_tensor = result["audio"]
|
||||
if not isinstance(audio_tensor, torch.Tensor):
|
||||
audio_tensor = torch.tensor(audio_tensor, dtype=torch.float32)
|
||||
audio_tensor = audio_tensor.detach().float().cpu()
|
||||
if audio_tensor.dim() == 1:
|
||||
audio_tensor = audio_tensor.unsqueeze(0)
|
||||
|
||||
if enable_audio_cache and cache_key:
|
||||
duration = self.audio_cache._calculate_duration(audio_tensor, "dots_tts")
|
||||
self.audio_cache.cache_audio(cache_key, audio_tensor, duration)
|
||||
|
||||
return audio_tensor
|
||||
@@ -1,300 +0,0 @@
|
||||
"""Adapter between unified TTS processing and official DramaBox inference."""
|
||||
|
||||
import math
|
||||
import os
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
import torch
|
||||
import torchaudio
|
||||
|
||||
from utils.audio.audio_hash import generate_stable_audio_component
|
||||
from utils.audio.cache import get_audio_cache
|
||||
from utils.audio.processing import AudioProcessingUtils
|
||||
from utils.models.factory_config import ModelLoadConfig
|
||||
from utils.voice.reference import effective_voice_audio
|
||||
|
||||
|
||||
class DramaBoxEngineAdapter:
|
||||
"""Translate suite voice/config/cache data into DramaBox calls."""
|
||||
|
||||
SAMPLE_RATE = 48000
|
||||
SILENCE_RMS_THRESHOLD = 1e-3
|
||||
SILENCE_PEAK_THRESHOLD = 2e-2
|
||||
|
||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||
self.config = config.copy() if config else {}
|
||||
self.audio_cache = get_audio_cache()
|
||||
self._last_config: Optional[ModelLoadConfig] = None
|
||||
self._load_signature = None
|
||||
self._lora_signature = None
|
||||
self.last_generation_status: Dict[str, Any] = {"near_silent": False}
|
||||
|
||||
def update_config(self, new_config: Dict[str, Any]):
|
||||
self.config = new_config.copy() if new_config else {}
|
||||
|
||||
@staticmethod
|
||||
def _lora_revision(path: Any) -> str:
|
||||
"""Return a cheap cache token that changes when a managed adapter is replaced."""
|
||||
value = str(path or "").strip()
|
||||
if not value:
|
||||
return ""
|
||||
try:
|
||||
candidate = os.path.abspath(os.path.expanduser(value))
|
||||
if os.path.isfile(candidate):
|
||||
stat = os.stat(candidate)
|
||||
return f"{candidate}:{stat.st_size}:{stat.st_mtime_ns}"
|
||||
if os.path.isdir(candidate):
|
||||
entries = []
|
||||
for item in os.listdir(candidate):
|
||||
if not item.endswith(".safetensors"):
|
||||
continue
|
||||
item_path = os.path.join(candidate, item)
|
||||
stat = os.stat(item_path)
|
||||
entries.append(f"{item}:{stat.st_size}:{stat.st_mtime_ns}")
|
||||
return f"{candidate}|{'|'.join(sorted(entries))}"
|
||||
except OSError:
|
||||
pass
|
||||
return value
|
||||
|
||||
@classmethod
|
||||
def _warn_if_near_silent(
|
||||
cls,
|
||||
audio: torch.Tensor,
|
||||
*,
|
||||
character_name: Optional[str],
|
||||
seed: int,
|
||||
cached: bool = False,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Warn about clearly near-silent model output without altering it."""
|
||||
if not isinstance(audio, torch.Tensor) or audio.numel() == 0:
|
||||
return None
|
||||
|
||||
samples = torch.nan_to_num(audio.detach().float().cpu())
|
||||
rms = float(samples.square().mean().sqrt())
|
||||
peak = float(samples.abs().max())
|
||||
if rms >= cls.SILENCE_RMS_THRESHOLD or peak >= cls.SILENCE_PEAK_THRESHOLD:
|
||||
return None
|
||||
|
||||
rms_db = 20.0 * math.log10(max(rms, 1e-12))
|
||||
peak_db = 20.0 * math.log10(max(peak, 1e-12))
|
||||
source = "cached " if cached else ""
|
||||
print(
|
||||
f"\n⚠️ DramaBox generated a near-silent {source}segment for "
|
||||
f"'{character_name or 'narrator'}' "
|
||||
f"(RMS {rms_db:.1f} dBFS, peak {peak_db:.1f} dBFS)."
|
||||
)
|
||||
print(
|
||||
"⚠️ This can depend on generation duration, reference duration, "
|
||||
"reference audio, guidance settings, and seed."
|
||||
)
|
||||
print(
|
||||
"⚠️ Try changing those parameters for this segment; another seed "
|
||||
"may help, but is not guaranteed to fix it.\n"
|
||||
)
|
||||
return {
|
||||
"near_silent": True,
|
||||
"character": character_name or "narrator",
|
||||
"seed": int(seed),
|
||||
"rms_dbfs": rms_db,
|
||||
"peak_dbfs": peak_db,
|
||||
"cached": bool(cached),
|
||||
}
|
||||
|
||||
def _build_load_signature(self) -> Tuple[Any, ...]:
|
||||
"""Identity of the expensive base runtime, excluding live LoRA state."""
|
||||
return (
|
||||
self.config.get("model_name", "DramaBox"),
|
||||
self.config.get("device", "auto"),
|
||||
self.config.get("precision", "auto"),
|
||||
self.config.get("memory_mode", "fast"),
|
||||
self.config.get("transformer_quantization", "none"),
|
||||
bool(self.config.get("compile_model", False)),
|
||||
)
|
||||
|
||||
def _build_lora_signature(self) -> Tuple[Any, ...]:
|
||||
path = self.config.get("lora_path", "")
|
||||
return (
|
||||
str(path or "").strip(),
|
||||
self._lora_revision(path),
|
||||
float(self.config.get("lora_strength", 1.0)),
|
||||
)
|
||||
|
||||
def _ensure_model_loaded(self):
|
||||
signature = self._build_load_signature()
|
||||
from utils.models.unified_model_interface import unified_model_interface
|
||||
|
||||
if signature != self._load_signature or self._last_config is None:
|
||||
self._last_config = ModelLoadConfig(
|
||||
engine_name="dramabox",
|
||||
model_type="tts",
|
||||
model_name=self.config.get("model_name", "DramaBox"),
|
||||
device=self.config.get("device", "auto"),
|
||||
additional_params={
|
||||
"precision": self.config.get("precision", "auto"),
|
||||
"memory_mode": self.config.get("memory_mode", "fast"),
|
||||
"transformer_quantization": self.config.get(
|
||||
"transformer_quantization", "none"
|
||||
),
|
||||
"compile_model": bool(self.config.get("compile_model", False)),
|
||||
},
|
||||
)
|
||||
self._load_signature = signature
|
||||
self._lora_signature = None
|
||||
|
||||
engine = unified_model_interface.load_model(self._last_config)
|
||||
lora_signature = self._build_lora_signature()
|
||||
if lora_signature != self._lora_signature:
|
||||
lora_path, lora_revision, lora_strength = lora_signature
|
||||
engine.set_lora(
|
||||
lora_path=lora_path,
|
||||
strength=lora_strength,
|
||||
revision=lora_revision,
|
||||
)
|
||||
self._lora_signature = lora_signature
|
||||
return engine
|
||||
|
||||
def _get_engine(self):
|
||||
return self._ensure_model_loaded()
|
||||
|
||||
def _extract_voice_reference(
|
||||
self, voice_ref: Optional[Dict[str, Any]]
|
||||
) -> Tuple[Optional[str], str, bool]:
|
||||
if not isinstance(voice_ref, dict):
|
||||
return None, "default_voice", False
|
||||
|
||||
audio = effective_voice_audio(voice_ref)
|
||||
if audio is None:
|
||||
return None, "default_voice", False
|
||||
if isinstance(audio, str):
|
||||
return (
|
||||
audio,
|
||||
generate_stable_audio_component(audio_file_path=audio),
|
||||
False,
|
||||
)
|
||||
if isinstance(audio, dict) and "waveform" in audio:
|
||||
path = AudioProcessingUtils.save_audio_to_temp_file(
|
||||
audio["waveform"], audio.get("sample_rate", self.SAMPLE_RATE)
|
||||
)
|
||||
return path, generate_stable_audio_component(reference_audio=audio), True
|
||||
if torch.is_tensor(audio):
|
||||
sample_rate = int(voice_ref.get("sample_rate", self.SAMPLE_RATE))
|
||||
audio_dict = {"waveform": audio, "sample_rate": sample_rate}
|
||||
path = AudioProcessingUtils.save_audio_to_temp_file(audio, sample_rate)
|
||||
return (
|
||||
path,
|
||||
generate_stable_audio_component(reference_audio=audio_dict),
|
||||
True,
|
||||
)
|
||||
raise TypeError(f"Unsupported DramaBox voice reference: {type(audio)}")
|
||||
|
||||
def generate_single(
|
||||
self,
|
||||
text: str,
|
||||
voice_ref: Optional[Dict[str, Any]],
|
||||
seed: int = 42,
|
||||
enable_audio_cache: bool = True,
|
||||
character_name: Optional[str] = None,
|
||||
) -> torch.Tensor:
|
||||
prompt = (text or "").strip()
|
||||
if not prompt:
|
||||
return torch.zeros(1, 0, dtype=torch.float32)
|
||||
|
||||
voice_path, audio_component, remove_voice_path = self._extract_voice_reference(
|
||||
voice_ref
|
||||
)
|
||||
cfg_scale = float(self.config.get("cfg_scale", 2.5))
|
||||
stg_scale = float(self.config.get("stg_scale", 1.5))
|
||||
duration_multiplier = float(self.config.get("duration_multiplier", 1.1))
|
||||
gen_duration = float(self.config.get("gen_duration", 0.0))
|
||||
ref_duration = float(self.config.get("ref_duration", 10.0))
|
||||
rescale_scale = self.config.get("rescale_scale", "auto")
|
||||
watermark = bool(self.config.get("watermark", False))
|
||||
negative_prompt = str(self.config.get("negative_prompt", ""))
|
||||
model_name = self.config.get("model_name", "DramaBox")
|
||||
|
||||
cache_key = None
|
||||
if enable_audio_cache:
|
||||
cache_key = self.audio_cache.generate_cache_key(
|
||||
"dramabox",
|
||||
text=prompt,
|
||||
audio_component=audio_component,
|
||||
model_name=model_name,
|
||||
cfg_scale=cfg_scale,
|
||||
stg_scale=stg_scale,
|
||||
duration_multiplier=duration_multiplier,
|
||||
gen_duration=gen_duration,
|
||||
ref_duration=ref_duration,
|
||||
rescale_scale=rescale_scale,
|
||||
watermark=watermark,
|
||||
prompt_template=str(
|
||||
self.config.get("prompt_template", '"{seg}"')
|
||||
),
|
||||
negative_prompt=negative_prompt,
|
||||
precision=self.config.get("precision", "auto"),
|
||||
transformer_quantization=self.config.get(
|
||||
"transformer_quantization", "none"
|
||||
),
|
||||
memory_mode=self.config.get("memory_mode", "fast"),
|
||||
compile_model=bool(self.config.get("compile_model", False)),
|
||||
lora_path=self.config.get("lora_path", ""),
|
||||
lora_strength=float(self.config.get("lora_strength", 1.0)),
|
||||
lora_revision=self._lora_revision(self.config.get("lora_path", "")),
|
||||
seed=int(seed),
|
||||
character=character_name or "narrator",
|
||||
)
|
||||
cached = self.audio_cache.get_cached_audio(cache_key)
|
||||
if cached:
|
||||
print(
|
||||
f"💾 Using cached DramaBox audio for "
|
||||
f"'{character_name or 'narrator'}': '{prompt[:30]}...'"
|
||||
)
|
||||
self.last_generation_status = self._warn_if_near_silent(
|
||||
cached[0],
|
||||
character_name=character_name,
|
||||
seed=int(seed),
|
||||
cached=True,
|
||||
) or {"near_silent": False}
|
||||
return cached[0]
|
||||
|
||||
try:
|
||||
result = self._get_engine().generate(
|
||||
prompt=prompt,
|
||||
voice_ref_path=voice_path,
|
||||
cfg_scale=cfg_scale,
|
||||
stg_scale=stg_scale,
|
||||
duration_multiplier=duration_multiplier,
|
||||
gen_duration=gen_duration,
|
||||
ref_duration=ref_duration,
|
||||
rescale_scale=rescale_scale,
|
||||
watermark=watermark,
|
||||
negative_prompt=negative_prompt,
|
||||
seed=int(seed),
|
||||
)
|
||||
finally:
|
||||
if remove_voice_path and voice_path:
|
||||
try:
|
||||
os.unlink(voice_path)
|
||||
except OSError:
|
||||
pass
|
||||
audio = result["audio"]
|
||||
if not isinstance(audio, torch.Tensor):
|
||||
audio = torch.tensor(audio, dtype=torch.float32)
|
||||
audio = audio.detach().float().cpu()
|
||||
if audio.dim() == 1:
|
||||
audio = audio.unsqueeze(0)
|
||||
|
||||
if int(result.get("sample_rate", self.SAMPLE_RATE)) != self.SAMPLE_RATE:
|
||||
audio = torchaudio.functional.resample(
|
||||
audio, int(result["sample_rate"]), self.SAMPLE_RATE
|
||||
)
|
||||
|
||||
self.last_generation_status = self._warn_if_near_silent(
|
||||
audio,
|
||||
character_name=character_name,
|
||||
seed=int(seed),
|
||||
) or {"near_silent": False}
|
||||
|
||||
if enable_audio_cache and cache_key:
|
||||
duration = audio.shape[-1] / self.SAMPLE_RATE
|
||||
self.audio_cache.cache_audio(cache_key, audio, duration)
|
||||
return audio
|
||||
@@ -1,821 +0,0 @@
|
||||
"""
|
||||
Echo-TTS Engine Adapter
|
||||
|
||||
Provides a unified interface for Echo-TTS integration with TTS Audio Suite.
|
||||
"""
|
||||
|
||||
import os
|
||||
import hashlib
|
||||
import inspect
|
||||
import re
|
||||
import sys
|
||||
from typing import Dict, Any, Optional, Tuple
|
||||
from functools import partial
|
||||
|
||||
import torch
|
||||
|
||||
# Add project root to path
|
||||
current_dir = os.path.dirname(__file__)
|
||||
engines_dir = os.path.dirname(current_dir)
|
||||
project_root = os.path.dirname(engines_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
from utils.audio.processing import AudioProcessingUtils
|
||||
from utils.audio.cache import get_audio_cache
|
||||
from utils.text.chunking import ImprovedChatterBoxChunker
|
||||
from utils.audio.chunk_timing import ChunkTimingHelper
|
||||
from utils.models.extra_paths import get_preferred_download_path
|
||||
|
||||
|
||||
class EchoTTSEngineAdapter:
|
||||
"""Adapter for Echo-TTS model loading and inference."""
|
||||
|
||||
SAMPLE_RATE = 44100
|
||||
MAX_REF_SECONDS = 300 # 5 minutes
|
||||
DEFAULT_SEQUENCE_LENGTH = 640
|
||||
DEFAULT_SEQUENCE_SECONDS = 30.0
|
||||
FORCE_KV_GUARD_MIN_SEQUENCE_LENGTH = 160
|
||||
FORCE_KV_GUARD_CHAR_RATE = 12.0
|
||||
FORCE_KV_GUARD_BASE_SECONDS = 1.25
|
||||
FORCE_KV_GUARD_SAFETY_MULTIPLIER = 1.35
|
||||
|
||||
def __init__(self, config: Dict[str, Any]):
|
||||
self.config = config.copy() if config else {}
|
||||
self.model = None
|
||||
self.ae = None
|
||||
self.pca_state = None
|
||||
self._sample_pipeline = None
|
||||
self._loaded_key = None
|
||||
self._wrapper = None
|
||||
self.audio_cache = get_audio_cache()
|
||||
self.job_tracker = None
|
||||
|
||||
def start_job(self, total_blocks: int, block_texts: list):
|
||||
"""Initialize job tracker for weighted multi-block progress estimation."""
|
||||
import time
|
||||
|
||||
normalized_texts = [max(1, int(v)) for v in (block_texts or [])]
|
||||
self.job_tracker = {
|
||||
"start_time": time.time(),
|
||||
"total_blocks": max(0, int(total_blocks)),
|
||||
"blocks_completed": 0,
|
||||
"block_texts": normalized_texts,
|
||||
"total_text": sum(normalized_texts),
|
||||
"text_completed": 0,
|
||||
"current_block_text": 0,
|
||||
"current_block_steps": 0,
|
||||
"current_block_total_steps": 0,
|
||||
"total_steps_completed": 0,
|
||||
}
|
||||
|
||||
def set_current_block(self, block_idx: int):
|
||||
"""Set current block metadata for weighted ETA calculation."""
|
||||
if not self.job_tracker:
|
||||
return
|
||||
block_texts = self.job_tracker.get("block_texts", [])
|
||||
if block_idx < 0 or block_idx >= len(block_texts):
|
||||
return
|
||||
self.job_tracker["current_block_text"] = block_texts[block_idx]
|
||||
self.job_tracker["current_block_steps"] = 0
|
||||
self.job_tracker["current_block_total_steps"] = 0
|
||||
|
||||
def complete_block(self):
|
||||
"""Mark current block as completed in the job tracker."""
|
||||
if not self.job_tracker:
|
||||
return
|
||||
self.job_tracker["blocks_completed"] += 1
|
||||
self.job_tracker["text_completed"] += self.job_tracker.get("current_block_text", 0)
|
||||
self.job_tracker["total_steps_completed"] += self.job_tracker.get("current_block_steps", 0)
|
||||
|
||||
def end_job(self):
|
||||
"""Clear active job tracker."""
|
||||
self.job_tracker = None
|
||||
|
||||
def update_config(self, new_config: Dict[str, Any]):
|
||||
self.config = new_config.copy() if new_config else {}
|
||||
# If critical load params changed, force reload on next call
|
||||
new_key = self._get_load_key()
|
||||
if self._loaded_key != new_key:
|
||||
self.model = None
|
||||
self.ae = None
|
||||
self.pca_state = None
|
||||
self._sample_pipeline = None
|
||||
self._loaded_key = None
|
||||
|
||||
def _get_load_key(self) -> str:
|
||||
model_id = self.config.get("model", "jordand/echo-tts-base")
|
||||
device = self._resolve_device(self.config.get("device", "auto"))
|
||||
return f"{model_id}::{device}"
|
||||
|
||||
@staticmethod
|
||||
def _filter_kwargs(fn, kwargs: Dict[str, Any]) -> Dict[str, Any]:
|
||||
try:
|
||||
signature = inspect.signature(fn)
|
||||
except (TypeError, ValueError):
|
||||
return kwargs
|
||||
return {key: value for key, value in kwargs.items() if key in signature.parameters}
|
||||
|
||||
def _resolve_device(self, device: str) -> str:
|
||||
device = (device or "auto").lower()
|
||||
if device == "cuda" and not torch.cuda.is_available():
|
||||
print("WARNING: Echo-TTS: CUDA requested but not available. Falling back to CPU.")
|
||||
return "cpu"
|
||||
|
||||
from utils.device import resolve_torch_device
|
||||
resolved = resolve_torch_device(device)
|
||||
|
||||
if resolved == "cpu":
|
||||
print("WARNING: Echo-TTS running on CPU will be very slow.")
|
||||
return resolved
|
||||
|
||||
@staticmethod
|
||||
def _normalize_prompt_text(text: str) -> str:
|
||||
"""
|
||||
Normalize prompt text following Echo-TTS inference behavior.
|
||||
|
||||
Add [S1] only when text is plain content without speaker/metadata tags.
|
||||
"""
|
||||
if not text:
|
||||
return "[S1]"
|
||||
|
||||
cleaned = text.strip()
|
||||
if not cleaned:
|
||||
return "[S1]"
|
||||
|
||||
if (
|
||||
cleaned.startswith("[")
|
||||
or cleaned.startswith("(")
|
||||
or "S1" in cleaned
|
||||
or "S2" in cleaned
|
||||
):
|
||||
return cleaned
|
||||
|
||||
return f"[S1] {cleaned}".strip()
|
||||
|
||||
@staticmethod
|
||||
def _count_prompt_chars_for_length_guard(prompt_text: str) -> int:
|
||||
"""Count visible prompt characters, excluding bracketed metadata tags."""
|
||||
without_tags = re.sub(r"\[[^\]]+\]", " ", prompt_text or "")
|
||||
return len(" ".join(without_tags.split()))
|
||||
|
||||
@classmethod
|
||||
def _estimate_force_kv_sequence_length(cls, prompt_text: str) -> int:
|
||||
"""
|
||||
Estimate a safer sequence length for Force Speaker KV runs.
|
||||
|
||||
Force-KV can prevent the latent flattening crop from triggering, so using a full
|
||||
default sequence window (640) on short prompts can yield overlong output.
|
||||
"""
|
||||
prompt_chars = cls._count_prompt_chars_for_length_guard(prompt_text)
|
||||
frames_per_second = cls.DEFAULT_SEQUENCE_LENGTH / cls.DEFAULT_SEQUENCE_SECONDS
|
||||
estimated_seconds = (prompt_chars / cls.FORCE_KV_GUARD_CHAR_RATE) + cls.FORCE_KV_GUARD_BASE_SECONDS
|
||||
estimated_seconds *= cls.FORCE_KV_GUARD_SAFETY_MULTIPLIER
|
||||
estimated_sequence_length = int(round(estimated_seconds * frames_per_second))
|
||||
return max(
|
||||
cls.FORCE_KV_GUARD_MIN_SEQUENCE_LENGTH,
|
||||
min(cls.DEFAULT_SEQUENCE_LENGTH, estimated_sequence_length),
|
||||
)
|
||||
|
||||
def _get_local_repo_dir(self, repo_id: str, base_dir: Optional[str] = None) -> str:
|
||||
base_dir = base_dir or get_preferred_download_path("TTS")
|
||||
repo_name = repo_id.split("/")[-1]
|
||||
repo_dir = os.path.join(base_dir, repo_name)
|
||||
os.makedirs(repo_dir, exist_ok=True)
|
||||
return repo_dir
|
||||
|
||||
def _hf_download(self, repo_id: str, filename: str, base_dir: Optional[str] = None) -> str:
|
||||
from huggingface_hub import hf_hub_download
|
||||
repo_dir = self._get_local_repo_dir(repo_id, base_dir=base_dir)
|
||||
return hf_hub_download(
|
||||
repo_id=repo_id,
|
||||
filename=filename,
|
||||
local_dir=repo_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
|
||||
def _ensure_model_loaded(self):
|
||||
load_key = self._get_load_key()
|
||||
if self.model is not None and self._loaded_key == load_key and self._wrapper is not None:
|
||||
return
|
||||
|
||||
from utils.models.unified_model_interface import unified_model_interface, ModelLoadConfig
|
||||
from utils.device import resolve_torch_device
|
||||
|
||||
device = resolve_torch_device(self.config.get("device", "auto"))
|
||||
model_id = self.config.get("model", "jordand/echo-tts-base")
|
||||
|
||||
config = ModelLoadConfig(
|
||||
engine_name="echo_tts",
|
||||
model_type="tts",
|
||||
model_name=model_id,
|
||||
device=device,
|
||||
)
|
||||
|
||||
self._wrapper = unified_model_interface.load_model(config)
|
||||
bundle = self._wrapper.model
|
||||
self.model = bundle.model
|
||||
self.ae = bundle.ae
|
||||
self.pca_state = bundle.pca_state
|
||||
self._sample_pipeline = bundle.sample_pipeline
|
||||
self._loaded_key = load_key
|
||||
|
||||
def _prepare_reference_audio(self, speaker_audio: Any) -> Tuple[torch.Tensor, int]:
|
||||
if speaker_audio is None:
|
||||
raise ValueError("Echo-TTS requires reference audio.")
|
||||
|
||||
if isinstance(speaker_audio, dict) and "waveform" in speaker_audio:
|
||||
waveform = speaker_audio["waveform"]
|
||||
sample_rate = int(speaker_audio.get("sample_rate", self.SAMPLE_RATE))
|
||||
elif isinstance(speaker_audio, str):
|
||||
waveform, sample_rate = AudioProcessingUtils.safe_load_audio(speaker_audio)
|
||||
else:
|
||||
waveform = speaker_audio
|
||||
sample_rate = self.SAMPLE_RATE
|
||||
|
||||
if isinstance(waveform, torch.Tensor):
|
||||
audio = waveform
|
||||
else:
|
||||
audio = torch.tensor(waveform, dtype=torch.float32)
|
||||
|
||||
if audio.dim() == 3:
|
||||
audio = audio[0]
|
||||
if audio.dim() == 2 and audio.shape[0] > 1:
|
||||
audio = audio.mean(dim=0, keepdim=True)
|
||||
if audio.dim() == 2 and audio.shape[0] == 1:
|
||||
audio = audio.squeeze(0)
|
||||
|
||||
# Resample to Echo-TTS sample rate
|
||||
if sample_rate != self.SAMPLE_RATE:
|
||||
import torchaudio
|
||||
resampler = torchaudio.transforms.Resample(orig_freq=sample_rate, new_freq=self.SAMPLE_RATE)
|
||||
audio = resampler(audio)
|
||||
sample_rate = self.SAMPLE_RATE
|
||||
|
||||
# Normalize to [-1, 1]
|
||||
max_val = audio.abs().max()
|
||||
if max_val > 0:
|
||||
audio = audio / max_val
|
||||
|
||||
# Ensure shape (1, samples)
|
||||
if audio.dim() == 1:
|
||||
audio = audio.unsqueeze(0)
|
||||
|
||||
# Trim to 5 minutes max
|
||||
max_samples = int(self.MAX_REF_SECONDS * sample_rate)
|
||||
if audio.shape[-1] > max_samples:
|
||||
audio = audio[:, :max_samples]
|
||||
|
||||
return audio, sample_rate
|
||||
|
||||
def _create_progress_bar(self, total_steps: int, text: str = ""):
|
||||
"""
|
||||
Create a ComfyUI progress bar with optional weighted job ETA support.
|
||||
|
||||
Args:
|
||||
total_steps: Number of denoising steps expected for this generation call
|
||||
text: Input text for this generation call (kept for API parity)
|
||||
"""
|
||||
del text # Progress total is step-accurate for Echo-TTS.
|
||||
try:
|
||||
import time
|
||||
import comfy.utils
|
||||
|
||||
class TimedProgressBar:
|
||||
"""ComfyUI progress wrapper that also provides job-level timing helpers."""
|
||||
|
||||
def __init__(self, total: int, tracker: Optional[Dict[str, Any]]):
|
||||
self.total = max(1, int(total))
|
||||
self.current = 0
|
||||
self.start_time = time.time()
|
||||
self.tracker = tracker
|
||||
self.wrapped = comfy.utils.ProgressBar(self.total)
|
||||
if self.tracker is not None:
|
||||
self.tracker["current_block_total_steps"] = self.total
|
||||
self.tracker["current_block_steps"] = 0
|
||||
|
||||
def update(self, delta: int = 1):
|
||||
if delta <= 0:
|
||||
return
|
||||
remaining = max(self.total - self.current, 0)
|
||||
step = min(int(delta), remaining)
|
||||
if step <= 0:
|
||||
return
|
||||
self.current += step
|
||||
self.wrapped.update(step)
|
||||
if self.tracker is not None:
|
||||
self.tracker["current_block_steps"] = self.current
|
||||
|
||||
def get_job_elapsed(self):
|
||||
if self.tracker is not None:
|
||||
return time.time() - self.tracker["start_time"]
|
||||
return time.time() - self.start_time
|
||||
|
||||
def get_job_remaining_str(self):
|
||||
if self.tracker is None:
|
||||
return None
|
||||
total_text = self.tracker.get("total_text", 0)
|
||||
if total_text <= 0:
|
||||
return None
|
||||
|
||||
elapsed = time.time() - self.tracker["start_time"]
|
||||
if elapsed <= 0:
|
||||
return None
|
||||
|
||||
current_block_text = self.tracker.get("current_block_text", 0)
|
||||
current_total_steps = max(self.tracker.get("current_block_total_steps", self.total), 1)
|
||||
current_ratio = min(self.current / current_total_steps, 1.0)
|
||||
|
||||
effective_text_done = self.tracker.get("text_completed", 0) + (current_block_text * current_ratio)
|
||||
if effective_text_done <= 0:
|
||||
return None
|
||||
|
||||
remaining_text = max(total_text - effective_text_done, 0)
|
||||
if remaining_text <= 0:
|
||||
return None
|
||||
|
||||
seconds_per_text = elapsed / effective_text_done
|
||||
remaining_seconds = remaining_text * seconds_per_text
|
||||
|
||||
if remaining_seconds < 60:
|
||||
return f"~{remaining_seconds:.0f}s left"
|
||||
return f"~{remaining_seconds / 60:.1f}m left"
|
||||
|
||||
return TimedProgressBar(total_steps, self.job_tracker)
|
||||
except (ImportError, AttributeError):
|
||||
return None
|
||||
|
||||
@torch.inference_mode()
|
||||
def _sample_euler_cfg_independent_guidances_with_progress(
|
||||
self,
|
||||
model,
|
||||
speaker_latent: torch.Tensor,
|
||||
speaker_mask: torch.Tensor,
|
||||
text_input_ids: torch.Tensor,
|
||||
text_mask: torch.Tensor,
|
||||
rng_seed: int,
|
||||
num_steps: int,
|
||||
cfg_scale_text: float,
|
||||
cfg_scale_speaker: float,
|
||||
cfg_min_t: float,
|
||||
cfg_max_t: float,
|
||||
truncation_factor: Optional[float],
|
||||
rescale_k: Optional[float],
|
||||
rescale_sigma: Optional[float],
|
||||
speaker_kv_scale: Optional[float],
|
||||
speaker_kv_max_layers: Optional[int],
|
||||
speaker_kv_min_t: Optional[float],
|
||||
sequence_length: Optional[int] = None,
|
||||
progress_bar=None,
|
||||
) -> torch.Tensor:
|
||||
"""Echo-TTS sampler equivalent with ComfyUI progress updates per denoise step."""
|
||||
from echo_tts.inference import _multiply_kv_cache, _concat_kv_caches, _temporal_score_rescale
|
||||
|
||||
try:
|
||||
import comfy.model_management as model_management
|
||||
except Exception:
|
||||
model_management = None
|
||||
|
||||
if sequence_length is None:
|
||||
sequence_length = 640
|
||||
|
||||
init_scale = 0.999
|
||||
num_steps = max(1, int(num_steps))
|
||||
device, dtype = model.device, model.dtype
|
||||
batch_size = text_input_ids.shape[0]
|
||||
rng = torch.Generator(device=device).manual_seed(rng_seed)
|
||||
t_schedule = torch.linspace(1.0, 0.0, num_steps + 1, device=device) * init_scale
|
||||
|
||||
text_mask_uncond = torch.zeros_like(text_mask)
|
||||
speaker_mask_uncond = torch.zeros_like(speaker_mask)
|
||||
|
||||
kv_text_cond = model.get_kv_cache_text(text_input_ids, text_mask)
|
||||
kv_speaker_cond = model.get_kv_cache_speaker(speaker_latent.to(dtype))
|
||||
|
||||
if speaker_kv_scale is not None:
|
||||
_multiply_kv_cache(kv_speaker_cond, speaker_kv_scale, speaker_kv_max_layers)
|
||||
|
||||
kv_text_full = _concat_kv_caches(kv_text_cond, kv_text_cond, kv_text_cond)
|
||||
kv_speaker_full = _concat_kv_caches(kv_speaker_cond, kv_speaker_cond, kv_speaker_cond)
|
||||
|
||||
full_text_mask = torch.cat([text_mask, text_mask_uncond, text_mask], dim=0)
|
||||
full_speaker_mask = torch.cat([speaker_mask, speaker_mask, speaker_mask_uncond], dim=0)
|
||||
|
||||
x_t = torch.randn((batch_size, sequence_length, 80), device=device, dtype=torch.float32, generator=rng)
|
||||
if truncation_factor is not None:
|
||||
x_t = x_t * truncation_factor
|
||||
|
||||
for i in range(num_steps):
|
||||
if model_management is not None and getattr(model_management, "interrupt_processing", False):
|
||||
raise InterruptedError("Echo-TTS generation interrupted by user")
|
||||
|
||||
t, t_next = t_schedule[i], t_schedule[i + 1]
|
||||
has_cfg = ((t >= cfg_min_t) * (t <= cfg_max_t)).item()
|
||||
|
||||
if has_cfg:
|
||||
v_cond, v_uncond_text, v_uncond_speaker = model(
|
||||
x=torch.cat([x_t, x_t, x_t], dim=0).to(dtype),
|
||||
t=(torch.ones((batch_size * 3,), device=device) * t).to(dtype),
|
||||
text_mask=full_text_mask,
|
||||
speaker_mask=full_speaker_mask,
|
||||
kv_cache_text=kv_text_full,
|
||||
kv_cache_speaker=kv_speaker_full,
|
||||
).float().chunk(3, dim=0)
|
||||
v_pred = v_cond + cfg_scale_text * (v_cond - v_uncond_text) + cfg_scale_speaker * (v_cond - v_uncond_speaker)
|
||||
else:
|
||||
v_pred = model(
|
||||
x=x_t.to(dtype),
|
||||
t=(torch.ones((batch_size,), device=device) * t).to(dtype),
|
||||
text_mask=text_mask,
|
||||
speaker_mask=speaker_mask,
|
||||
kv_cache_text=kv_text_cond,
|
||||
kv_cache_speaker=kv_speaker_cond,
|
||||
).float()
|
||||
|
||||
if rescale_k is not None and rescale_sigma is not None:
|
||||
v_pred = _temporal_score_rescale(v_pred, x_t, t, rescale_k, rescale_sigma)
|
||||
|
||||
if (
|
||||
speaker_kv_scale is not None
|
||||
and speaker_kv_min_t is not None
|
||||
and t_next < speaker_kv_min_t
|
||||
and t >= speaker_kv_min_t
|
||||
):
|
||||
_multiply_kv_cache(kv_speaker_cond, 1.0 / speaker_kv_scale, speaker_kv_max_layers)
|
||||
kv_speaker_full = _concat_kv_caches(kv_speaker_cond, kv_speaker_cond, kv_speaker_cond)
|
||||
|
||||
x_t = x_t + v_pred * (t_next - t)
|
||||
|
||||
if progress_bar is not None:
|
||||
try:
|
||||
progress_bar.update(1)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return x_t
|
||||
|
||||
def _generate_audio_for_text(
|
||||
self,
|
||||
text: str,
|
||||
ref_audio: torch.Tensor,
|
||||
ref_text: str,
|
||||
enable_audio_cache: bool = True,
|
||||
) -> Tuple[torch.Tensor, int]:
|
||||
self._ensure_model_loaded()
|
||||
|
||||
# Seed for reproducibility if provided
|
||||
seed = int(self.config.get("seed", 0))
|
||||
if seed:
|
||||
torch.manual_seed(seed)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed(seed)
|
||||
|
||||
params = {
|
||||
"num_steps": int(self.config.get("num_steps", 40)),
|
||||
"cfg_scale_text": float(self.config.get("cfg_scale_text", 3.0)),
|
||||
"cfg_scale_speaker": float(self.config.get("cfg_scale_speaker", 8.0)),
|
||||
"cfg_min_t": float(self.config.get("cfg_min_t", 0.5)),
|
||||
"cfg_max_t": float(self.config.get("cfg_max_t", 1.0)),
|
||||
"truncation_factor": self.config.get("truncation_factor", None),
|
||||
"rescale_k": self.config.get("rescale_k", None),
|
||||
"rescale_sigma": self.config.get("rescale_sigma", None),
|
||||
"speaker_kv_scale": self.config.get("speaker_kv_scale", None),
|
||||
"speaker_kv_max_layers": self.config.get("speaker_kv_max_layers", None),
|
||||
"speaker_kv_min_t": self.config.get("speaker_kv_min_t", None),
|
||||
"sequence_length": int(self.config.get("sequence_length", self.DEFAULT_SEQUENCE_LENGTH)),
|
||||
}
|
||||
|
||||
prompt_text = self._normalize_prompt_text(text)
|
||||
effective_sequence_length = params["sequence_length"]
|
||||
if (
|
||||
params["speaker_kv_scale"] is not None
|
||||
and params["sequence_length"] == self.DEFAULT_SEQUENCE_LENGTH
|
||||
):
|
||||
guarded_sequence_length = self._estimate_force_kv_sequence_length(prompt_text)
|
||||
prompt_chars = self._count_prompt_chars_for_length_guard(prompt_text)
|
||||
if guarded_sequence_length != params["sequence_length"]:
|
||||
print(
|
||||
"Echo-TTS: Force-KV sequence_length guard active "
|
||||
f"({params['sequence_length']} -> {guarded_sequence_length}, text_chars={prompt_chars})"
|
||||
)
|
||||
effective_sequence_length = guarded_sequence_length
|
||||
|
||||
cache_key = None
|
||||
if enable_audio_cache:
|
||||
try:
|
||||
ref_audio_hash = hashlib.md5(ref_audio.detach().cpu().contiguous().numpy().tobytes()).hexdigest()
|
||||
audio_component = f"ref_audio_{ref_audio_hash}_{self.SAMPLE_RATE}"
|
||||
except Exception as e:
|
||||
print(f"⚠️ Echo-TTS: Failed to hash reference audio for cache key: {e}")
|
||||
audio_component = "ref_audio_error"
|
||||
|
||||
resolved_device = self.config.get("device", "auto")
|
||||
if self._loaded_key and "::" in self._loaded_key:
|
||||
_, resolved_device = self._loaded_key.split("::", 1)
|
||||
|
||||
cache_key = self.audio_cache.generate_cache_key(
|
||||
'echo_tts',
|
||||
text=prompt_text,
|
||||
audio_component=audio_component,
|
||||
reference_text=ref_text,
|
||||
model=self.config.get("model", "jordand/echo-tts-base"),
|
||||
device=resolved_device,
|
||||
num_steps=params["num_steps"],
|
||||
cfg_scale_text=params["cfg_scale_text"],
|
||||
cfg_scale_speaker=params["cfg_scale_speaker"],
|
||||
cfg_min_t=params["cfg_min_t"],
|
||||
cfg_max_t=params["cfg_max_t"],
|
||||
truncation_factor=params["truncation_factor"],
|
||||
rescale_k=params["rescale_k"],
|
||||
rescale_sigma=params["rescale_sigma"],
|
||||
speaker_kv_scale=params["speaker_kv_scale"],
|
||||
speaker_kv_max_layers=params["speaker_kv_max_layers"],
|
||||
speaker_kv_min_t=params["speaker_kv_min_t"],
|
||||
sequence_length=effective_sequence_length,
|
||||
seed=seed,
|
||||
)
|
||||
|
||||
cached_audio = self.audio_cache.get_cached_audio(cache_key)
|
||||
if cached_audio:
|
||||
print(f"💾 Using cached Echo-TTS audio: '{text[:30]}...'")
|
||||
return cached_audio[0], self.SAMPLE_RATE
|
||||
|
||||
progress_bar = self._create_progress_bar(params["num_steps"], text)
|
||||
|
||||
# Resolve sampler
|
||||
sampler_kwargs = {
|
||||
"num_steps": params["num_steps"],
|
||||
"cfg_scale_text": params["cfg_scale_text"],
|
||||
"cfg_scale_speaker": params["cfg_scale_speaker"],
|
||||
"cfg_min_t": params["cfg_min_t"],
|
||||
"cfg_max_t": params["cfg_max_t"],
|
||||
"truncation_factor": params["truncation_factor"],
|
||||
"rescale_k": params["rescale_k"],
|
||||
"rescale_sigma": params["rescale_sigma"],
|
||||
"speaker_kv_scale": params["speaker_kv_scale"],
|
||||
"speaker_kv_max_layers": params["speaker_kv_max_layers"],
|
||||
"speaker_kv_min_t": params["speaker_kv_min_t"],
|
||||
"sequence_length": effective_sequence_length,
|
||||
# Alternate parameter names used by some sampler variants
|
||||
"cfg_scale": params["cfg_scale_text"],
|
||||
"speaker_k_scale": params["speaker_kv_scale"],
|
||||
"speaker_k_max_layers": params["speaker_kv_max_layers"],
|
||||
"speaker_k_min_t": params["speaker_kv_min_t"],
|
||||
"block_size": effective_sequence_length,
|
||||
}
|
||||
try:
|
||||
from echo_tts.inference import sample_euler_cfg_independent_guidances
|
||||
sample_fn = partial(
|
||||
self._sample_euler_cfg_independent_guidances_with_progress,
|
||||
progress_bar=progress_bar,
|
||||
**self._filter_kwargs(sample_euler_cfg_independent_guidances, sampler_kwargs),
|
||||
)
|
||||
except Exception:
|
||||
try:
|
||||
from echo_tts.samplers import sample_euler_cfg_any, GuidanceMode
|
||||
sample_fn = partial(
|
||||
sample_euler_cfg_any,
|
||||
**self._filter_kwargs(
|
||||
sample_euler_cfg_any,
|
||||
{
|
||||
**sampler_kwargs,
|
||||
"guidance_mode": GuidanceMode.INDEPENDENT,
|
||||
"apg_eta_text": None,
|
||||
"apg_eta_speaker": None,
|
||||
"apg_momentum_text": None,
|
||||
"apg_momentum_speaker": None,
|
||||
"apg_norm_text": None,
|
||||
"apg_norm_speaker": None,
|
||||
},
|
||||
),
|
||||
)
|
||||
except Exception:
|
||||
from echo_tts.samplers import sample_euler_cfg
|
||||
sample_fn = partial(
|
||||
sample_euler_cfg,
|
||||
**self._filter_kwargs(sample_euler_cfg, sampler_kwargs),
|
||||
)
|
||||
|
||||
try:
|
||||
with torch.no_grad():
|
||||
result = self._sample_pipeline(
|
||||
model=self.model,
|
||||
fish_ae=self.ae,
|
||||
pca_state=self.pca_state,
|
||||
sample_fn=sample_fn,
|
||||
text_prompt=prompt_text,
|
||||
speaker_audio=ref_audio,
|
||||
rng_seed=seed,
|
||||
pad_to_max_speaker_latent_length=2560,
|
||||
pad_to_max_text_length=768,
|
||||
normalize_text=True,
|
||||
)
|
||||
except RuntimeError as e:
|
||||
if "Inference tensors do not track version counter" in str(e):
|
||||
print("⚠️ Echo-TTS: pca_state tensors corrupted after VRAM clear - forcing model reload...")
|
||||
self.model = None
|
||||
self._loaded_key = None
|
||||
self._ensure_model_loaded()
|
||||
with torch.no_grad():
|
||||
result = self._sample_pipeline(
|
||||
model=self.model,
|
||||
fish_ae=self.ae,
|
||||
pca_state=self.pca_state,
|
||||
sample_fn=sample_fn,
|
||||
text_prompt=prompt_text,
|
||||
speaker_audio=ref_audio,
|
||||
rng_seed=seed,
|
||||
pad_to_max_speaker_latent_length=2560,
|
||||
pad_to_max_text_length=768,
|
||||
normalize_text=True,
|
||||
)
|
||||
else:
|
||||
raise
|
||||
|
||||
if progress_bar is not None and hasattr(progress_bar, "total") and hasattr(progress_bar, "current"):
|
||||
remaining = progress_bar.total - progress_bar.current
|
||||
if remaining > 0:
|
||||
progress_bar.update(remaining)
|
||||
|
||||
audio = None
|
||||
sample_rate = self.SAMPLE_RATE
|
||||
|
||||
if isinstance(result, dict):
|
||||
audio = result.get("wav") or result.get("audio") or result.get("waveform")
|
||||
sample_rate = int(result.get("sr", result.get("sample_rate", self.SAMPLE_RATE)))
|
||||
elif isinstance(result, tuple) and len(result) >= 1:
|
||||
audio = result[0]
|
||||
if len(result) >= 2 and isinstance(result[1], (int, float)):
|
||||
sample_rate = int(result[1])
|
||||
else:
|
||||
audio = result
|
||||
|
||||
if isinstance(audio, torch.Tensor):
|
||||
audio_tensor = audio
|
||||
else:
|
||||
audio_tensor = torch.tensor(audio, dtype=torch.float32)
|
||||
|
||||
if audio_tensor.dim() > 1:
|
||||
audio_tensor = audio_tensor.squeeze()
|
||||
|
||||
if sample_rate != self.SAMPLE_RATE:
|
||||
import torchaudio
|
||||
resampler = torchaudio.transforms.Resample(orig_freq=sample_rate, new_freq=self.SAMPLE_RATE)
|
||||
audio_tensor = resampler(audio_tensor)
|
||||
sample_rate = self.SAMPLE_RATE
|
||||
|
||||
if enable_audio_cache and cache_key:
|
||||
duration = self.audio_cache._calculate_duration(audio_tensor, 'echo_tts')
|
||||
self.audio_cache.cache_audio(cache_key, audio_tensor, duration)
|
||||
|
||||
return audio_tensor, sample_rate
|
||||
|
||||
def generate_single(self, text: str, speaker_audio: Any, reference_text: str,
|
||||
seed: int = 0, enable_audio_cache: bool = True) -> torch.Tensor:
|
||||
"""Generate audio for a single chunk of text (no chunking, no combining)."""
|
||||
if seed is not None:
|
||||
self.config["seed"] = seed
|
||||
ref_audio, _ = self._prepare_reference_audio(speaker_audio)
|
||||
ref_text = reference_text or ""
|
||||
audio_tensor, _ = self._generate_audio_for_text(
|
||||
text, ref_audio, ref_text, enable_audio_cache=enable_audio_cache
|
||||
)
|
||||
return audio_tensor
|
||||
|
||||
def process_text(self, text: str, speaker_audio: Any, reference_text: str, seed: int = 0,
|
||||
enable_chunking: bool = True, max_chars_per_chunk: int = 400,
|
||||
chunk_combination_method: str = "auto", silence_between_chunks_ms: int = 100,
|
||||
enable_audio_cache: bool = True, return_info: bool = False):
|
||||
if seed is not None:
|
||||
self.config["seed"] = seed
|
||||
|
||||
ref_audio, _ = self._prepare_reference_audio(speaker_audio)
|
||||
ref_text = reference_text or ""
|
||||
|
||||
if enable_chunking:
|
||||
max_chars = ImprovedChatterBoxChunker.validate_chunking_params(max_chars_per_chunk)
|
||||
text_chunks = ImprovedChatterBoxChunker.split_into_chunks(text, max_chars=max_chars)
|
||||
else:
|
||||
text_chunks = [text]
|
||||
|
||||
audio_segments = []
|
||||
for chunk_idx, chunk in enumerate(text_chunks):
|
||||
audio_tensor, _ = self._generate_audio_for_text(
|
||||
chunk,
|
||||
ref_audio,
|
||||
ref_text,
|
||||
enable_audio_cache=enable_audio_cache,
|
||||
)
|
||||
audio_segments.append(audio_tensor)
|
||||
|
||||
combined_audio, chunk_info = ChunkTimingHelper.combine_audio_with_timing(
|
||||
audio_segments=audio_segments,
|
||||
combination_method=chunk_combination_method,
|
||||
silence_ms=silence_between_chunks_ms,
|
||||
crossfade_duration=0.1,
|
||||
sample_rate=self.SAMPLE_RATE,
|
||||
text_length=len(text),
|
||||
original_text=text,
|
||||
text_chunks=text_chunks,
|
||||
)
|
||||
|
||||
if return_info:
|
||||
return combined_audio, chunk_info
|
||||
return combined_audio
|
||||
|
||||
def process_srt_content(self, srt_content: str, speaker_audio: Any, reference_text: str,
|
||||
seed: int = 0, timing_mode: str = "pad_with_silence",
|
||||
timing_params: Optional[Dict[str, Any]] = None,
|
||||
enable_audio_cache: bool = True):
|
||||
from utils.system.import_manager import import_manager
|
||||
from utils.timing.assembly import AudioAssemblyEngine
|
||||
from utils.timing.engine import TimingEngine
|
||||
from utils.timing.overlap_detection import SRTOverlapHandler
|
||||
|
||||
timing_params = timing_params or {}
|
||||
|
||||
success, modules, _ = import_manager.import_srt_modules()
|
||||
if not success:
|
||||
raise RuntimeError("SRT modules are not available")
|
||||
|
||||
SRTParser = modules.get("SRTParser")
|
||||
if SRTParser is None:
|
||||
raise RuntimeError("SRT parser not available")
|
||||
|
||||
srt_parser = SRTParser()
|
||||
subtitles = srt_parser.parse_srt_content(srt_content, allow_overlaps=True)
|
||||
has_overlaps = SRTOverlapHandler.detect_overlaps(subtitles)
|
||||
current_timing_mode, mode_switched = SRTOverlapHandler.handle_smart_natural_fallback(
|
||||
timing_mode, has_overlaps, "Echo-TTS SRT Adapter"
|
||||
)
|
||||
|
||||
if seed is not None:
|
||||
self.config["seed"] = seed
|
||||
ref_audio, _ = self._prepare_reference_audio(speaker_audio)
|
||||
ref_text = reference_text or ""
|
||||
|
||||
audio_segments = []
|
||||
for subtitle in subtitles:
|
||||
audio_tensor, _ = self._generate_audio_for_text(
|
||||
subtitle.text,
|
||||
ref_audio,
|
||||
ref_text,
|
||||
enable_audio_cache=enable_audio_cache,
|
||||
)
|
||||
audio_segments.append(audio_tensor.cpu())
|
||||
|
||||
adjustments = None
|
||||
processed_segments = None
|
||||
fade_duration = float(timing_params.get("fade_for_StretchToFit", 0.01))
|
||||
|
||||
if current_timing_mode == "concatenate":
|
||||
timing_engine = TimingEngine(self.SAMPLE_RATE)
|
||||
adjustments = timing_engine.calculate_concatenation_adjustments(audio_segments, subtitles)
|
||||
elif current_timing_mode == "smart_natural":
|
||||
timing_engine = TimingEngine(self.SAMPLE_RATE)
|
||||
tolerance = float(timing_params.get("timing_tolerance", 2.0))
|
||||
max_stretch_ratio = float(timing_params.get("max_stretch_ratio", 1.0))
|
||||
min_stretch_ratio = float(timing_params.get("min_stretch_ratio", 0.5))
|
||||
adjustments, processed_segments = timing_engine.calculate_smart_timing_adjustments(
|
||||
audio_segments,
|
||||
subtitles,
|
||||
tolerance,
|
||||
max_stretch_ratio,
|
||||
min_stretch_ratio,
|
||||
torch.device("cpu"),
|
||||
)
|
||||
|
||||
assembler = AudioAssemblyEngine(sample_rate=self.SAMPLE_RATE)
|
||||
final_audio = assembler.assemble_by_timing_mode(
|
||||
audio_segments=audio_segments,
|
||||
subtitles=subtitles,
|
||||
timing_mode=current_timing_mode,
|
||||
device="cpu",
|
||||
adjustments=adjustments,
|
||||
processed_segments=processed_segments,
|
||||
fade_duration=fade_duration,
|
||||
)
|
||||
|
||||
if final_audio.dim() == 1:
|
||||
final_audio = final_audio.unsqueeze(0).unsqueeze(0)
|
||||
elif final_audio.dim() == 2:
|
||||
final_audio = final_audio.unsqueeze(0)
|
||||
|
||||
audio_output = {"waveform": final_audio, "sample_rate": self.SAMPLE_RATE}
|
||||
|
||||
timing_info = srt_parser.get_timing_info(subtitles)
|
||||
mode_info = current_timing_mode
|
||||
if mode_switched:
|
||||
mode_info = f"{current_timing_mode} (switched from {timing_mode} due to overlaps)"
|
||||
|
||||
timing_report = (
|
||||
f"Echo-TTS SRT timing\n"
|
||||
f"Subtitles: {timing_info.get('subtitle_count', 0)}\n"
|
||||
f"Total duration: {timing_info.get('total_duration', 0):.2f}s\n"
|
||||
f"Mode: {mode_info}"
|
||||
)
|
||||
|
||||
total_duration = final_audio.shape[-1] / float(self.SAMPLE_RATE)
|
||||
info = f"Generated {total_duration:.1f}s Echo-TTS SRT audio using {mode_info} mode"
|
||||
|
||||
return audio_output, info, timing_report, srt_content
|
||||
@@ -152,14 +152,14 @@ class F5TTSEngineAdapter:
|
||||
|
||||
def combine_audio_chunks(self, audio_segments: List[torch.Tensor], **params) -> torch.Tensor:
|
||||
"""
|
||||
Combine F5-TTS audio segments using modular combination utility.
|
||||
Combine F5-TTS audio segments.
|
||||
|
||||
Args:
|
||||
audio_segments: List of audio tensors to combine
|
||||
**params: Combination parameters
|
||||
|
||||
Returns:
|
||||
Combined audio tensor, or tuple with timing info if requested
|
||||
Combined audio tensor
|
||||
"""
|
||||
if len(audio_segments) == 1:
|
||||
return audio_segments[0]
|
||||
@@ -167,13 +167,9 @@ class F5TTSEngineAdapter:
|
||||
method = params.get("combination_method", "auto")
|
||||
silence_ms = params.get("silence_ms", 100)
|
||||
text_length = params.get("text_length", 0)
|
||||
original_text = params.get("original_text", "")
|
||||
text_chunks = params.get("text_chunks", None)
|
||||
return_info = params.get("return_timing_info", False)
|
||||
|
||||
return self.node.combine_f5tts_audio_chunks(
|
||||
audio_segments, method, silence_ms, text_length,
|
||||
original_text=original_text, text_chunks=text_chunks, return_info=return_info
|
||||
audio_segments, method, silence_ms, text_length
|
||||
)
|
||||
|
||||
def _get_audio_duration(self, audio_tensor: torch.Tensor) -> float:
|
||||
|
||||
@@ -159,18 +159,30 @@ class F5TTSStreamingAdapter(StreamingEngineAdapter):
|
||||
if self._current_model == model_name and self._current_language == language:
|
||||
return True
|
||||
|
||||
# Use unified interface for model loading
|
||||
from utils.models.unified_model_interface import load_tts_model
|
||||
|
||||
# Load model via unified interface
|
||||
model_instance = load_tts_model(
|
||||
engine_name="f5tts",
|
||||
# Use universal smart model loader
|
||||
from utils.models.smart_loader import smart_model_loader
|
||||
|
||||
def f5tts_load_callback(device: str, model: str):
|
||||
"""Callback for F5-TTS model loading"""
|
||||
if hasattr(self.node, 'load_f5tts_model'):
|
||||
# Use node's existing model loading (which now uses smart loader internally)
|
||||
return self.node.load_f5tts_model(model, device)
|
||||
else:
|
||||
# Fallback: try to load directly
|
||||
from engines.f5tts.f5tts import F5TTSEngine
|
||||
engine = F5TTSEngine()
|
||||
engine.load_model(model, device)
|
||||
return engine
|
||||
|
||||
# Use smart loader to get or load the model
|
||||
model_instance, was_cached = smart_model_loader.load_model_if_needed(
|
||||
engine_type="f5tts",
|
||||
model_name=model_name,
|
||||
current_model=getattr(self.node, 'f5tts_model', None),
|
||||
device=device,
|
||||
language=language,
|
||||
force_reload=False
|
||||
load_callback=f5tts_load_callback
|
||||
)
|
||||
|
||||
|
||||
# Update node's model reference and our tracking
|
||||
if hasattr(self.node, 'f5tts_model'):
|
||||
self.node.f5tts_model = model_instance
|
||||
@@ -178,13 +190,16 @@ class F5TTSStreamingAdapter(StreamingEngineAdapter):
|
||||
self.node.f5tts = model_instance
|
||||
else:
|
||||
self.node.f5tts = model_instance
|
||||
|
||||
|
||||
# Cache the model for local reuse and update tracking
|
||||
self._loaded_models[model_name] = model_instance
|
||||
self._current_model = model_name
|
||||
self._current_language = language
|
||||
|
||||
print(f"✅ F5TTS STREAMING: Loaded {model_name} via unified interface")
|
||||
|
||||
if was_cached:
|
||||
print(f"♻️ F5TTS STREAMING: Reused {model_name} from smart loader")
|
||||
else:
|
||||
print(f"✅ F5TTS STREAMING: Loaded {model_name} via smart loader")
|
||||
|
||||
return True
|
||||
|
||||
@@ -297,8 +312,8 @@ class F5TTSStreamingAdapter(StreamingEngineAdapter):
|
||||
reference_audio, reference_text = self.node._load_reference_audio(voice_path)
|
||||
else:
|
||||
# Simple loading fallback
|
||||
from utils.audio.processing import AudioProcessingUtils
|
||||
reference_audio, _ = AudioProcessingUtils.safe_load_audio(voice_path)
|
||||
import torchaudio
|
||||
reference_audio, _ = torchaudio.load(voice_path)
|
||||
|
||||
# Check for text file
|
||||
text_path = voice_path.replace('.wav', '.txt').replace('.mp3', '.txt')
|
||||
|
||||
@@ -1,133 +0,0 @@
|
||||
"""Adapter between suite processors and the isolated official Fish S2 runtime."""
|
||||
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from engines.fish_audio_s2.downloader import FishAudioS2Downloader
|
||||
from utils.audio.audio_hash import generate_stable_audio_component
|
||||
from utils.audio.cache import get_audio_cache
|
||||
from utils.audio.processing import AudioProcessingUtils
|
||||
from utils.models.factory_config import ModelLoadConfig
|
||||
from utils.text.fish_audio_s2_tags import translate_fish_s2_inline_tags
|
||||
from utils.voice.character_logging import resolved_voice_name
|
||||
from utils.voice.reference import effective_voice_audio
|
||||
|
||||
|
||||
class FishAudioS2Adapter:
|
||||
SAMPLE_RATE = 44100
|
||||
|
||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||
self.config = dict(config or {})
|
||||
self.audio_cache = get_audio_cache()
|
||||
self._model_config = None
|
||||
|
||||
def update_config(self, config):
|
||||
self.config = dict(config or {})
|
||||
|
||||
def _engine(self):
|
||||
from utils.models.unified_model_interface import unified_model_interface
|
||||
model_selection = self.config.get("model_variant", "s2-pro")
|
||||
quantization = self.config.get("quantization", "none")
|
||||
model_path = FishAudioS2Downloader.resolve_model_path(model_selection)
|
||||
model_variant = FishAudioS2Downloader.resolve_model_variant(model_selection, model_path)
|
||||
self._model_config = ModelLoadConfig(
|
||||
engine_name="fish_audio_s2", model_type="tts", model_name=model_variant,
|
||||
model_path=model_path, device=self.config.get("device", "auto"),
|
||||
runtime_mode="isolated",
|
||||
additional_params={
|
||||
"model_variant": model_variant,
|
||||
"quantization": quantization,
|
||||
"precision": self.config.get("precision", "bfloat16"),
|
||||
"compile": bool(self.config.get("compile", False)),
|
||||
"context_length": int(self.config.get("context_length", 8192)),
|
||||
},
|
||||
)
|
||||
return unified_model_interface.load_model(self._model_config)
|
||||
|
||||
def _reference(self, voice_ref):
|
||||
if not isinstance(voice_ref, dict):
|
||||
return None, "", "default_voice"
|
||||
text = (voice_ref.get("reference_text") or voice_ref.get("prompt_text") or "").strip()
|
||||
effective_audio = effective_voice_audio(voice_ref)
|
||||
path = effective_audio if isinstance(effective_audio, str) else None
|
||||
if isinstance(effective_audio, dict):
|
||||
path = AudioProcessingUtils.save_audio_to_temp_file(
|
||||
effective_audio["waveform"], effective_audio.get("sample_rate", self.SAMPLE_RATE)
|
||||
)
|
||||
elif torch.is_tensor(effective_audio):
|
||||
path = AudioProcessingUtils.save_audio_to_temp_file(
|
||||
effective_audio, voice_ref.get("sample_rate", self.SAMPLE_RATE)
|
||||
)
|
||||
component = generate_stable_audio_component(audio_file_path=path) if path else "default_voice"
|
||||
return path, text, component
|
||||
|
||||
def generate_single(self, text, voice_ref, seed=0, enable_audio_cache=True, character_name=None):
|
||||
return self.generate_dialogue(
|
||||
[(0, text)], [voice_ref], seed, enable_audio_cache,
|
||||
cache_character=character_name or "narrator",
|
||||
)
|
||||
|
||||
def generate_dialogue(self, turns, voice_refs, seed=0, enable_audio_cache=True,
|
||||
cache_character="native_dialogue"):
|
||||
formatted_turns = []
|
||||
for speaker_index, turn_text in turns:
|
||||
clean_text = translate_fish_s2_inline_tags((turn_text or "").strip())
|
||||
if clean_text:
|
||||
formatted_turns.append(f"<|speaker:{speaker_index}|>{clean_text}")
|
||||
text = "\n".join(formatted_turns)
|
||||
if not text:
|
||||
return torch.zeros(1, 0)
|
||||
|
||||
references = []
|
||||
reference_labels = []
|
||||
components = []
|
||||
reference_texts = []
|
||||
for speaker_index, voice_ref in enumerate(voice_refs):
|
||||
ref_path, ref_text, component = self._reference(voice_ref)
|
||||
if ref_path:
|
||||
if not ref_text:
|
||||
raise ValueError("Fish S2 native speakers require exact reference transcripts")
|
||||
references.append({
|
||||
"audio_path": ref_path,
|
||||
"text": f"<|speaker:{speaker_index}|>{ref_text}",
|
||||
})
|
||||
reference_labels.append(
|
||||
f"local Speaker {speaker_index + 1}={resolved_voice_name(voice_ref)}"
|
||||
)
|
||||
components.append(component)
|
||||
reference_texts.append(ref_text)
|
||||
if references:
|
||||
print(f"🎤 Fish local reference order: {', '.join(reference_labels)}")
|
||||
|
||||
params = {
|
||||
"model_variant": self.config.get("model_variant", "s2-pro"),
|
||||
"quantization": self.config.get("quantization", "none"),
|
||||
"multi_speaker_mode": self.config.get("multi_speaker_mode", "Native Multi-Speaker"),
|
||||
"seed": int(seed), "normalize": bool(self.config.get("normalize", True)),
|
||||
"chunk_length": int(self.config.get("native_chunk_length", 200)),
|
||||
"max_new_tokens": int(self.config.get("max_new_tokens", 1024)),
|
||||
"top_p": float(self.config.get("top_p", 0.8)),
|
||||
"repetition_penalty": float(self.config.get("repetition_penalty", 1.1)),
|
||||
"temperature": float(self.config.get("temperature", 0.8)),
|
||||
"cache_reference": bool(self.config.get("cache_reference", True)),
|
||||
"context_length": int(self.config.get("context_length", 8192)),
|
||||
}
|
||||
cache_key = self.audio_cache.generate_cache_key(
|
||||
"fish_audio_s2", text=text, audio_component="|".join(components),
|
||||
reference_text="|".join(reference_texts), character=cache_character, **params,
|
||||
) if enable_audio_cache else None
|
||||
if cache_key:
|
||||
cached = self.audio_cache.get_cached_audio(cache_key)
|
||||
if cached:
|
||||
print("💾 Fish Audio S2: Using cached audio")
|
||||
return cached[0]
|
||||
audio, sample_rate = self._engine().generate(text=text, references=references, **params)
|
||||
if sample_rate != self.SAMPLE_RATE:
|
||||
raise RuntimeError(f"Fish S2 returned unexpected sample rate {sample_rate}")
|
||||
audio = audio.detach().float().cpu()
|
||||
if audio.dim() == 1:
|
||||
audio = audio.unsqueeze(0)
|
||||
if cache_key:
|
||||
self.audio_cache.cache_audio(cache_key, audio, audio.shape[-1] / self.SAMPLE_RATE)
|
||||
return audio
|
||||
@@ -1,451 +0,0 @@
|
||||
"""
|
||||
Higgs Audio Engine Adapter - Engine-specific adapter for Higgs Audio 2
|
||||
Provides standardized interface for Higgs Audio operations in unified engine
|
||||
"""
|
||||
|
||||
import torch
|
||||
from typing import Dict, Any, Optional, List, Tuple
|
||||
# Use absolute import to avoid relative import issues in ComfyUI
|
||||
import sys
|
||||
import os
|
||||
current_dir = os.path.dirname(__file__)
|
||||
parent_dir = os.path.dirname(current_dir)
|
||||
if parent_dir not in sys.path:
|
||||
sys.path.insert(0, parent_dir)
|
||||
|
||||
from utils.models.language_mapper import get_model_for_language
|
||||
from utils.models.engine_registry import get_default_runtime_profile
|
||||
from utils.models.factory_config import normalize_runtime_mode, runtime_uses_isolation
|
||||
from engines.higgs_audio.higgs_audio import HiggsAudioEngine
|
||||
from engines.higgs_audio.higgs_audio_downloader import HIGGS_AUDIO_MODELS
|
||||
|
||||
|
||||
class HiggsAudioEngineAdapter:
|
||||
"""Engine-specific adapter for Higgs Audio 2."""
|
||||
|
||||
def __init__(self, node_instance):
|
||||
"""
|
||||
Initialize Higgs Audio adapter.
|
||||
|
||||
Args:
|
||||
node_instance: HiggsAudioNode or HiggsAudioSRTNode instance
|
||||
"""
|
||||
self.node = node_instance
|
||||
self.engine_type = "higgs_audio"
|
||||
self.higgs_engine = HiggsAudioEngine()
|
||||
node_config = getattr(node_instance, "config", {}) or {}
|
||||
self._runtime_mode = normalize_runtime_mode(node_config.get("runtime_mode"))
|
||||
self._runtime_profile = node_config.get("runtime_profile")
|
||||
|
||||
def get_model_for_language(self, lang_code: str, default_model: str) -> str:
|
||||
"""
|
||||
Get Higgs Audio model name for specified language.
|
||||
|
||||
Args:
|
||||
lang_code: Language code (e.g., 'en', 'zh')
|
||||
default_model: Default model name
|
||||
|
||||
Returns:
|
||||
Higgs Audio model name for the language
|
||||
|
||||
Note: Higgs Audio v2 supports multiple languages with the same model
|
||||
"""
|
||||
# Higgs Audio v2 3B model supports multiple languages
|
||||
# Map all language codes to the default model for now
|
||||
supported_languages = ['en', 'zh', 'zh-cn'] # Add more as confirmed
|
||||
|
||||
if lang_code.lower() in supported_languages:
|
||||
return "higgs-audio-v2-3B"
|
||||
else:
|
||||
print(f"⚠️ Language '{lang_code}' not specifically tested with Higgs Audio, using default model")
|
||||
return "higgs-audio-v2-3B"
|
||||
|
||||
def load_base_model(self, model_name: str, device: str, enable_cuda_graphs: bool = True):
|
||||
"""
|
||||
Load base Higgs Audio model.
|
||||
|
||||
Args:
|
||||
model_name: Model name to load
|
||||
device: Device to load model on
|
||||
enable_cuda_graphs: Whether to enable CUDA Graph optimization
|
||||
"""
|
||||
# Check if the model is already loaded AND engine is initialized
|
||||
current_model = getattr(self.node, 'current_model_name', None)
|
||||
|
||||
if (current_model == model_name and
|
||||
self.higgs_engine.engine is not None):
|
||||
print(f"💾 Higgs Audio adapter: Model '{model_name}' already loaded - ensuring device and clearing other models")
|
||||
# Even if model is loaded, ensure it's on the right device and clear other models
|
||||
from utils.models.comfyui_model_wrapper.model_manager import tts_model_manager
|
||||
# This will clear other TTS models (like Step Audio EditX) from VRAM
|
||||
tts_model_manager.ensure_device("higgs_audio", device)
|
||||
return
|
||||
|
||||
# Determine model paths based on model name
|
||||
if model_name in HIGGS_AUDIO_MODELS:
|
||||
model_config = HIGGS_AUDIO_MODELS[model_name]
|
||||
generation_model = model_config["generation_repo"]
|
||||
tokenizer_model = model_config["tokenizer_repo"]
|
||||
else:
|
||||
# Default fallback
|
||||
generation_model = "bosonai/higgs-audio-v2-generation-3B-base"
|
||||
tokenizer_model = "bosonai/higgs-audio-v2-tokenizer"
|
||||
|
||||
if runtime_uses_isolation(self._runtime_mode):
|
||||
from utils.models.unified_model_interface import load_tts_model
|
||||
|
||||
self.higgs_engine = load_tts_model(
|
||||
engine_name="higgs_audio",
|
||||
model_name=model_name,
|
||||
device=device,
|
||||
model_path=generation_model,
|
||||
tokenizer_path=tokenizer_model,
|
||||
enable_cuda_graphs=enable_cuda_graphs,
|
||||
runtime_mode=self._runtime_mode,
|
||||
runtime_profile=self._runtime_profile or get_default_runtime_profile("higgs_audio"),
|
||||
)
|
||||
self.node.current_model_name = model_name
|
||||
print(f"✅ Higgs Audio adapter: Loaded model '{model_name}' on {device} via shared runtime")
|
||||
return
|
||||
|
||||
# Initialize the Higgs Audio engine with CUDA Graph setting
|
||||
self.higgs_engine.initialize_engine(
|
||||
model_path=generation_model,
|
||||
tokenizer_path=tokenizer_model,
|
||||
device=device,
|
||||
enable_cuda_graphs=enable_cuda_graphs
|
||||
)
|
||||
|
||||
# Store current model for caching
|
||||
self.node.current_model_name = model_name
|
||||
print(f"✅ Higgs Audio adapter: Loaded model '{model_name}' on {device}")
|
||||
|
||||
def load_language_model(self, model_name: str, device: str):
|
||||
"""
|
||||
Load language-specific Higgs Audio model.
|
||||
|
||||
Args:
|
||||
model_name: Language-specific model name
|
||||
device: Device to load model on
|
||||
|
||||
Note: Higgs Audio v2 uses the same model for all languages
|
||||
"""
|
||||
# For Higgs Audio, language models are the same as base models
|
||||
self.load_base_model(model_name, device)
|
||||
|
||||
def generate_segment_audio(self, text: str, char_audio: str, char_text: str,
|
||||
character: str = "narrator", **params) -> torch.Tensor:
|
||||
"""
|
||||
Generate Higgs Audio audio for a text segment.
|
||||
|
||||
Args:
|
||||
text: Text to generate audio for
|
||||
char_audio: Reference audio file path or audio dict
|
||||
char_text: Reference text
|
||||
character: Character name for caching
|
||||
**params: Additional Higgs Audio parameters
|
||||
|
||||
Returns:
|
||||
Generated audio tensor
|
||||
"""
|
||||
# Ensure engine is initialized with current model/device
|
||||
model = params.get("model", "higgs-audio-v2-3B")
|
||||
device = params.get("device", "auto")
|
||||
enable_cuda_graphs = params.get("enable_cuda_graphs", True) # Get CUDA Graph setting
|
||||
self.load_base_model(model, device, enable_cuda_graphs)
|
||||
|
||||
# Extract Higgs Audio specific parameters
|
||||
system_prompt = params.get("system_prompt", "Generate audio following instruction.")
|
||||
temperature = params.get("temperature", 1.0)
|
||||
top_p = params.get("top_p", 0.95)
|
||||
top_k = params.get("top_k", 50)
|
||||
max_new_tokens = params.get("max_new_tokens", 2048)
|
||||
force_audio_gen = params.get("force_audio_gen", True)
|
||||
ras_win_len = params.get("ras_win_len", 7) # Default from boson_multimodal
|
||||
ras_max_num_repeat = params.get("ras_max_num_repeat", 2) # Default from boson_multimodal
|
||||
enable_cuda_graphs = params.get("enable_cuda_graphs", True) # CUDA Graph toggle
|
||||
seed = params.get("seed", -1)
|
||||
enable_cache = params.get("enable_audio_cache", True)
|
||||
model_name = params.get("model", "higgs-audio-v2-3B")
|
||||
device = params.get("device", "auto")
|
||||
|
||||
# Native multi-speaker mode parameters
|
||||
multi_speaker_mode = params.get("multi_speaker_mode", "Custom Character Switching")
|
||||
second_narrator_audio = params.get("second_narrator_audio")
|
||||
second_narrator_text = params.get("second_narrator_text", "")
|
||||
|
||||
# Chunking parameters
|
||||
max_chars_per_chunk = params.get("max_chars_per_chunk", 400)
|
||||
silence_between_chunks_ms = params.get("silence_between_chunks_ms", 100)
|
||||
enable_chunking = len(text) > max_chars_per_chunk # Auto-enable chunking for long text
|
||||
|
||||
# Convert chars to tokens for engine (Higgs Audio uses ~3.5 chars per token)
|
||||
max_tokens_per_chunk = int(max_chars_per_chunk / 3.5)
|
||||
|
||||
# Initialize engine if not already done
|
||||
if not getattr(self.higgs_engine, "engine", None):
|
||||
print(f"🚀 Initializing Higgs Audio engine with model: {model_name}")
|
||||
self.load_base_model(model_name, device)
|
||||
|
||||
# Prepare reference audio
|
||||
reference_audio = None
|
||||
if char_audio:
|
||||
if isinstance(char_audio, str) and os.path.exists(char_audio):
|
||||
# Load audio file
|
||||
try:
|
||||
from utils.audio.processing import AudioProcessingUtils
|
||||
waveform, sample_rate = AudioProcessingUtils.safe_load_audio(char_audio)
|
||||
if waveform.dim() == 2 and waveform.size(0) > 1:
|
||||
# Convert to mono if stereo
|
||||
waveform = torch.mean(waveform, dim=0, keepdim=True)
|
||||
|
||||
# Move to same device as the model to prevent device mismatch
|
||||
from utils.device import resolve_torch_device
|
||||
target_device = resolve_torch_device(device)
|
||||
waveform = waveform.float().to(target_device)
|
||||
|
||||
reference_audio = {"waveform": waveform, "sample_rate": sample_rate}
|
||||
print(f" 📁 Loaded reference audio on {target_device}")
|
||||
except Exception as e:
|
||||
print(f"⚠️ Failed to load reference audio {char_audio}: {e}")
|
||||
elif isinstance(char_audio, dict):
|
||||
# Already in ComfyUI format - ensure tensors are on correct device
|
||||
reference_audio = {}
|
||||
from utils.device import resolve_torch_device
|
||||
target_device = resolve_torch_device(device)
|
||||
|
||||
for key, value in char_audio.items():
|
||||
if torch.is_tensor(value):
|
||||
reference_audio[key] = value.to(target_device)
|
||||
print(f" 📦 Moved ComfyUI {key} tensor to {target_device}")
|
||||
else:
|
||||
reference_audio[key] = value
|
||||
|
||||
# Generate audio using Higgs Audio engine with stateless methods
|
||||
try:
|
||||
if multi_speaker_mode in ["Native Multi-Speaker (System Context)", "Native Multi-Speaker (Conversation)"]:
|
||||
# Use stateless native multi-speaker mode with multiple reference audios
|
||||
if hasattr(self.higgs_engine, 'generate_native_multispeaker_stateless'):
|
||||
audio_result, generation_info = self.higgs_engine.generate_native_multispeaker_stateless(
|
||||
text=text,
|
||||
primary_reference_audio=reference_audio,
|
||||
primary_reference_text=char_text or "",
|
||||
secondary_reference_audio=second_narrator_audio,
|
||||
secondary_reference_text=second_narrator_text,
|
||||
use_system_context=(multi_speaker_mode == "Native Multi-Speaker (System Context)"),
|
||||
system_prompt=system_prompt,
|
||||
max_new_tokens=max_new_tokens,
|
||||
temperature=temperature,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
force_audio_gen=force_audio_gen,
|
||||
ras_win_len=ras_win_len,
|
||||
ras_max_num_repeat=ras_max_num_repeat,
|
||||
enable_cache=enable_cache,
|
||||
character=character,
|
||||
seed=seed
|
||||
)
|
||||
else:
|
||||
# Fallback to standard method if wrapper not available
|
||||
audio_result, generation_info = self.higgs_engine.generate_native_multispeaker(
|
||||
text=text,
|
||||
primary_reference_audio=reference_audio,
|
||||
primary_reference_text=char_text or "",
|
||||
secondary_reference_audio=second_narrator_audio,
|
||||
secondary_reference_text=second_narrator_text,
|
||||
use_system_context=(multi_speaker_mode == "Native Multi-Speaker (System Context)"),
|
||||
system_prompt=system_prompt,
|
||||
max_new_tokens=max_new_tokens,
|
||||
temperature=temperature,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
force_audio_gen=force_audio_gen,
|
||||
ras_win_len=ras_win_len,
|
||||
ras_max_num_repeat=ras_max_num_repeat,
|
||||
enable_cache=enable_cache,
|
||||
character=character,
|
||||
seed=seed
|
||||
)
|
||||
else:
|
||||
# Use stateless custom character switching mode (existing behavior)
|
||||
if hasattr(self.higgs_engine, 'generate_stateless'):
|
||||
audio_result, generation_info = self.higgs_engine.generate_stateless(
|
||||
text=text,
|
||||
reference_audio=reference_audio,
|
||||
reference_text=char_text or "",
|
||||
system_prompt=system_prompt,
|
||||
max_new_tokens=max_new_tokens,
|
||||
temperature=temperature,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
force_audio_gen=force_audio_gen,
|
||||
ras_win_len=ras_win_len,
|
||||
ras_max_num_repeat=ras_max_num_repeat,
|
||||
enable_chunking=enable_chunking,
|
||||
max_tokens_per_chunk=max_tokens_per_chunk, # Now properly converted from chars to tokens
|
||||
silence_between_chunks_ms=silence_between_chunks_ms,
|
||||
enable_cache=enable_cache,
|
||||
character=character,
|
||||
seed=seed
|
||||
)
|
||||
else:
|
||||
# Fallback to standard method if wrapper not available
|
||||
audio_result, generation_info = self.higgs_engine.generate(
|
||||
text=text,
|
||||
reference_audio=reference_audio,
|
||||
reference_text=char_text or "",
|
||||
system_prompt=system_prompt,
|
||||
max_new_tokens=max_new_tokens,
|
||||
temperature=temperature,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
force_audio_gen=force_audio_gen,
|
||||
ras_win_len=ras_win_len,
|
||||
ras_max_num_repeat=ras_max_num_repeat,
|
||||
enable_chunking=enable_chunking,
|
||||
max_tokens_per_chunk=max_tokens_per_chunk, # Now properly converted from chars to tokens
|
||||
silence_between_chunks_ms=silence_between_chunks_ms,
|
||||
enable_cache=enable_cache,
|
||||
character=character,
|
||||
seed=seed
|
||||
)
|
||||
|
||||
# Return the waveform tensor with correct dimensions for ComfyUI
|
||||
waveform = audio_result["waveform"]
|
||||
|
||||
# Ensure correct dimensions: ComfyUI expects [channels, samples]
|
||||
if waveform.dim() == 3:
|
||||
# Handle [batch, channels, samples] -> [channels, samples]
|
||||
if waveform.size(0) == 1: # batch size of 1
|
||||
waveform = waveform.squeeze(0) # Remove batch dimension
|
||||
else:
|
||||
waveform = waveform[0] # Take first item from batch
|
||||
elif waveform.dim() == 1:
|
||||
waveform = waveform.unsqueeze(0) # Add channel dimension
|
||||
|
||||
return waveform
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error generating Higgs Audio segment: {e}")
|
||||
raise e
|
||||
|
||||
def get_available_models(self) -> List[str]:
|
||||
"""
|
||||
Get list of available Higgs Audio models.
|
||||
|
||||
Returns:
|
||||
List of model names
|
||||
"""
|
||||
return self.higgs_engine.get_available_models()
|
||||
|
||||
def get_model_info(self, model_name: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Get information about a Higgs Audio model.
|
||||
|
||||
Args:
|
||||
model_name: Name of the model
|
||||
|
||||
Returns:
|
||||
Model information dict or None
|
||||
"""
|
||||
return self.higgs_engine.downloader.get_model_info(model_name)
|
||||
|
||||
def cleanup(self):
|
||||
"""Clean up resources"""
|
||||
if hasattr(self, 'higgs_engine'):
|
||||
self.higgs_engine.cleanup()
|
||||
|
||||
# Clear current model reference
|
||||
if hasattr(self.node, 'current_model_name'):
|
||||
self.node.current_model_name = None
|
||||
|
||||
def supports_streaming(self) -> bool:
|
||||
"""
|
||||
Check if Higgs Audio supports streaming.
|
||||
|
||||
Returns:
|
||||
False for now - streaming not implemented yet
|
||||
"""
|
||||
return False
|
||||
|
||||
def get_sample_rate(self) -> int:
|
||||
"""
|
||||
Get the sample rate used by Higgs Audio.
|
||||
|
||||
Returns:
|
||||
Sample rate (24000 Hz for Higgs Audio v2)
|
||||
"""
|
||||
return 24000 # Higgs Audio v2 uses 24kHz sample rate
|
||||
|
||||
def supports_voice_cloning(self) -> bool:
|
||||
"""
|
||||
Check if Higgs Audio supports voice cloning.
|
||||
|
||||
Returns:
|
||||
True - Higgs Audio v2 supports voice cloning
|
||||
"""
|
||||
return True
|
||||
|
||||
|
||||
def validate_parameters(self, **params) -> Dict[str, Any]:
|
||||
"""
|
||||
Validate and normalize Higgs Audio parameters.
|
||||
|
||||
Args:
|
||||
**params: Parameters to validate
|
||||
|
||||
Returns:
|
||||
Validated parameters dict
|
||||
"""
|
||||
validated = {}
|
||||
|
||||
# Temperature validation
|
||||
temperature = params.get("temperature", 1.0)
|
||||
validated["temperature"] = max(0.0, min(2.0, float(temperature)))
|
||||
|
||||
# Top-p validation
|
||||
top_p = params.get("top_p", 0.95)
|
||||
validated["top_p"] = max(0.1, min(1.0, float(top_p)))
|
||||
|
||||
# Top-k validation
|
||||
top_k = params.get("top_k", 50)
|
||||
validated["top_k"] = max(-1, min(100, int(top_k)))
|
||||
|
||||
# Max tokens validation
|
||||
max_tokens = params.get("max_new_tokens", 2048)
|
||||
validated["max_new_tokens"] = max(1, min(4096, int(max_tokens)))
|
||||
|
||||
# Force audio gen validation
|
||||
force_audio_gen = params.get("force_audio_gen", True)
|
||||
validated["force_audio_gen"] = bool(force_audio_gen)
|
||||
|
||||
# RAS parameters validation
|
||||
ras_win_len = params.get("ras_win_len", 7)
|
||||
if ras_win_len is not None and ras_win_len > 0:
|
||||
validated["ras_win_len"] = max(1, min(20, int(ras_win_len)))
|
||||
else:
|
||||
validated["ras_win_len"] = None # Disable RAS
|
||||
|
||||
ras_max_num_repeat = params.get("ras_max_num_repeat", 2)
|
||||
validated["ras_max_num_repeat"] = max(1, min(5, int(ras_max_num_repeat)))
|
||||
|
||||
# Audio priority validation
|
||||
audio_priority = params.get("audio_priority", "auto")
|
||||
valid_priorities = ["auto", "preset_dropdown", "reference_input", "force_preset"]
|
||||
if audio_priority not in valid_priorities:
|
||||
print(f"⚠️ Invalid audio priority '{audio_priority}', using 'auto'")
|
||||
validated["audio_priority"] = "auto"
|
||||
else:
|
||||
validated["audio_priority"] = audio_priority
|
||||
|
||||
# System prompt
|
||||
validated["system_prompt"] = params.get("system_prompt", "Generate audio following instruction.")
|
||||
|
||||
# Seed validation
|
||||
seed = params.get("seed", -1)
|
||||
validated["seed"] = int(seed) if seed >= 0 else -1
|
||||
|
||||
# Cache settings
|
||||
validated["enable_audio_cache"] = params.get("enable_audio_cache", True)
|
||||
|
||||
return validated
|
||||
@@ -1,206 +0,0 @@
|
||||
"""Higgs Audio v3 adapter for the unified TTS pipeline."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
current_dir = os.path.dirname(__file__)
|
||||
engines_dir = os.path.dirname(current_dir)
|
||||
project_root = os.path.dirname(engines_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
from utils.audio.audio_hash import generate_stable_audio_component
|
||||
from utils.audio.cache import get_audio_cache
|
||||
from utils.models.factory_config import ModelLoadConfig
|
||||
from utils.text.pause_processor import PauseTagProcessor
|
||||
|
||||
|
||||
class HiggsAudioV3EngineAdapter:
|
||||
"""Adapter for native Higgs Audio v3 generation."""
|
||||
|
||||
SAMPLE_RATE = 24000
|
||||
|
||||
def __init__(self, node_instance=None):
|
||||
self.node = node_instance
|
||||
self.audio_cache = get_audio_cache()
|
||||
self._last_config: Optional[ModelLoadConfig] = None
|
||||
|
||||
def load_model(
|
||||
self,
|
||||
model: str,
|
||||
device: str = "auto",
|
||||
dtype: str = "auto",
|
||||
attention: str = "auto",
|
||||
):
|
||||
from utils.models.unified_model_interface import unified_model_interface
|
||||
|
||||
config = ModelLoadConfig(
|
||||
engine_name="higgs_audio_v3",
|
||||
model_type="tts",
|
||||
model_name=model,
|
||||
model_path=model,
|
||||
device=device,
|
||||
additional_params={
|
||||
"dtype": dtype,
|
||||
"attention": attention,
|
||||
},
|
||||
)
|
||||
self._last_config = config
|
||||
unified_model_interface.load_model(config)
|
||||
|
||||
def update_model_config(
|
||||
self,
|
||||
model: str,
|
||||
device: str = "auto",
|
||||
dtype: str = "auto",
|
||||
attention: str = "auto",
|
||||
):
|
||||
self.load_model(model, device, dtype, attention)
|
||||
|
||||
def _get_engine(self):
|
||||
if self._last_config is None:
|
||||
raise RuntimeError("Higgs Audio v3 model has not been loaded")
|
||||
from utils.models.unified_model_interface import unified_model_interface
|
||||
|
||||
return unified_model_interface.load_model(self._last_config)
|
||||
|
||||
def generate_with_pause_tags(
|
||||
self,
|
||||
text: str,
|
||||
voice_ref: Optional[Dict[str, Any]],
|
||||
params: Dict[str, Any],
|
||||
process_pauses: bool = True,
|
||||
character_name: Optional[str] = None,
|
||||
) -> torch.Tensor:
|
||||
engine = self._get_engine()
|
||||
if process_pauses and PauseTagProcessor.has_pause_tags(text):
|
||||
return self._generate_with_pauses(text, voice_ref, params, character_name, engine)
|
||||
return self._generate_direct(text, voice_ref, params, character_name, engine)
|
||||
|
||||
def _generate_direct(
|
||||
self,
|
||||
text: str,
|
||||
voice_ref: Optional[Dict[str, Any]],
|
||||
params: Dict[str, Any],
|
||||
character_name: Optional[str] = None,
|
||||
engine=None,
|
||||
) -> torch.Tensor:
|
||||
if engine is None:
|
||||
engine = self._get_engine()
|
||||
|
||||
reference_audio, reference_text, audio_component = self._extract_voice_reference(voice_ref)
|
||||
model_name = params.get("model", "higgs-audio-v3-tts-4b")
|
||||
seed = int(params.get("seed", 0) or 0)
|
||||
max_new_tokens = int(params.get("max_new_tokens", 2048))
|
||||
temperature = float(params.get("temperature", 1.0))
|
||||
top_p = float(params.get("top_p", 0.95))
|
||||
top_k = int(params.get("top_k", 50))
|
||||
|
||||
cache_key = self.audio_cache.generate_cache_key(
|
||||
"higgs_audio_v3",
|
||||
text=text,
|
||||
model_variant=model_name,
|
||||
audio_component=audio_component,
|
||||
reference_text=reference_text,
|
||||
temperature=temperature,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
max_new_tokens=max_new_tokens,
|
||||
seed=seed,
|
||||
device=params.get("device", "auto"),
|
||||
dtype=params.get("dtype", "auto"),
|
||||
attention=params.get("attention", "auto"),
|
||||
character=character_name or "narrator",
|
||||
)
|
||||
|
||||
if params.get("enable_audio_cache", True):
|
||||
cached_audio = self.audio_cache.get_cached_audio(cache_key)
|
||||
if cached_audio:
|
||||
print(f"💾 Using cached Higgs Audio v3 audio for '{character_name or 'narrator'}': '{text[:30]}...'")
|
||||
return cached_audio[0]
|
||||
|
||||
audio_tensor, sample_rate = engine.generate(
|
||||
text=text,
|
||||
reference_audio=reference_audio,
|
||||
reference_text=reference_text,
|
||||
max_new_tokens=max_new_tokens,
|
||||
temperature=temperature,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
seed=seed,
|
||||
)
|
||||
|
||||
if sample_rate != self.SAMPLE_RATE:
|
||||
raise RuntimeError(f"Higgs Audio v3 returned {sample_rate} Hz; expected {self.SAMPLE_RATE} Hz")
|
||||
|
||||
if params.get("enable_audio_cache", True):
|
||||
duration = self.audio_cache._calculate_duration(audio_tensor, "higgs_audio_v3")
|
||||
self.audio_cache.cache_audio(cache_key, audio_tensor, duration)
|
||||
return audio_tensor
|
||||
|
||||
def _generate_with_pauses(
|
||||
self,
|
||||
text: str,
|
||||
voice_ref: Optional[Dict[str, Any]],
|
||||
params: Dict[str, Any],
|
||||
character_name: Optional[str] = None,
|
||||
engine=None,
|
||||
) -> torch.Tensor:
|
||||
segments, _ = PauseTagProcessor.parse_pause_tags(text)
|
||||
audio_parts = []
|
||||
for segment_type, content in segments:
|
||||
if segment_type == "text":
|
||||
audio_tensor = self._generate_direct(content, voice_ref, params, character_name, engine)
|
||||
if audio_tensor.dim() == 1:
|
||||
audio_tensor = audio_tensor.unsqueeze(0)
|
||||
elif audio_tensor.dim() == 3:
|
||||
audio_tensor = audio_tensor.squeeze(0)
|
||||
audio_parts.append(audio_tensor)
|
||||
elif segment_type == "pause":
|
||||
audio_parts.append(PauseTagProcessor.create_silence_segment(content, self.SAMPLE_RATE))
|
||||
|
||||
if not audio_parts:
|
||||
return torch.zeros(1, 0)
|
||||
return torch.cat(audio_parts, dim=-1)
|
||||
|
||||
def _extract_voice_reference(self, voice_ref: Optional[Dict[str, Any]]) -> Tuple[Optional[Dict[str, Any]], str, str]:
|
||||
if not voice_ref or not isinstance(voice_ref, dict):
|
||||
return None, "", "zero_shot"
|
||||
|
||||
ref_audio = None
|
||||
for key in ("audio", "audio_dict", "reference_audio", "waveform"):
|
||||
if key in voice_ref and voice_ref.get(key) is not None:
|
||||
ref_audio = voice_ref.get(key)
|
||||
break
|
||||
reference_text = (
|
||||
voice_ref.get("reference_text")
|
||||
or voice_ref.get("text")
|
||||
or voice_ref.get("prompt_text")
|
||||
or ""
|
||||
)
|
||||
|
||||
if ref_audio is None and "audio_path" in voice_ref:
|
||||
audio_path = voice_ref.get("audio_path")
|
||||
if audio_path and os.path.exists(audio_path):
|
||||
try:
|
||||
from utils.audio.processing import AudioProcessingUtils
|
||||
|
||||
waveform, sample_rate = AudioProcessingUtils.safe_load_audio(audio_path)
|
||||
ref_audio = {"waveform": waveform, "sample_rate": sample_rate}
|
||||
except Exception as e:
|
||||
print(f"⚠️ Failed to load Higgs Audio v3 reference audio {audio_path}: {e}")
|
||||
|
||||
if isinstance(ref_audio, dict) and "waveform" in ref_audio:
|
||||
return ref_audio, str(reference_text), generate_stable_audio_component(reference_audio=ref_audio)
|
||||
|
||||
if torch.is_tensor(ref_audio):
|
||||
sample_rate = int(voice_ref.get("sample_rate", self.SAMPLE_RATE))
|
||||
audio_dict = {"waveform": ref_audio, "sample_rate": sample_rate}
|
||||
return audio_dict, str(reference_text), generate_stable_audio_component(reference_audio=audio_dict)
|
||||
|
||||
return None, str(reference_text), "zero_shot"
|
||||
@@ -1,644 +0,0 @@
|
||||
"""
|
||||
IndexTTS-2 Engine Adapter
|
||||
|
||||
Provides standardized interface for IndexTTS-2 integration with TTS Audio Suite.
|
||||
Handles parameter mapping, character switching, and emotion control.
|
||||
"""
|
||||
|
||||
import os
|
||||
import torch
|
||||
from typing import Dict, Any, Optional, List, Union
|
||||
|
||||
from engines.index_tts.index_tts import IndexTTSEngine
|
||||
from engines.index_tts.index_tts_downloader import index_tts_downloader
|
||||
from utils.text.character_parser import character_parser
|
||||
from utils.voice.discovery import get_character_mapping, get_available_characters
|
||||
from utils.voice.character_logging import resolved_character_label
|
||||
from utils.audio.cache import get_audio_cache
|
||||
|
||||
|
||||
class IndexTTSAdapter:
|
||||
"""
|
||||
Adapter for IndexTTS-2 engine providing unified interface compatibility.
|
||||
|
||||
Handles:
|
||||
- Parameter mapping between unified interface and IndexTTS-2
|
||||
- Character switching with [character:emotion_ref] syntax
|
||||
- Emotion control via audio references or vectors
|
||||
- Caching integration
|
||||
- Model management
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize the IndexTTS-2 adapter."""
|
||||
self.engine = None
|
||||
self.audio_cache = get_audio_cache()
|
||||
|
||||
def initialize_engine(self,
|
||||
model_path: Optional[str] = None,
|
||||
device: str = "auto",
|
||||
use_fp16: bool = True,
|
||||
use_cuda_kernel: Optional[bool] = None,
|
||||
use_deepspeed: bool = False,
|
||||
use_torch_compile: bool = False,
|
||||
use_accel: bool = False,
|
||||
low_vram: bool = False):
|
||||
"""
|
||||
Initialize IndexTTS-2 engine.
|
||||
|
||||
Args:
|
||||
model_path: Path to model directory (auto-downloaded if None)
|
||||
device: Target device
|
||||
use_fp16: Use FP16 for inference
|
||||
use_cuda_kernel: Use BigVGAN CUDA kernels
|
||||
use_deepspeed: Use DeepSpeed optimization
|
||||
use_torch_compile: Enable torch.compile optimization for S2Mel stage
|
||||
use_accel: Enable GPT2 acceleration with FlashAttention
|
||||
low_vram: Enable Low VRAM mode (sequential offloading)
|
||||
"""
|
||||
# Auto-download model if not provided or if "auto-download" is specified
|
||||
if model_path is None or model_path == "auto-download":
|
||||
if not index_tts_downloader.is_model_available():
|
||||
print("📥 IndexTTS-2 model not found, downloading...")
|
||||
model_path = index_tts_downloader.download_model()
|
||||
else:
|
||||
model_path = index_tts_downloader.get_model_path()
|
||||
|
||||
# Initialize engine
|
||||
self.engine = IndexTTSEngine(
|
||||
model_dir=model_path,
|
||||
device=device,
|
||||
use_fp16=use_fp16,
|
||||
use_cuda_kernel=use_cuda_kernel,
|
||||
use_deepspeed=use_deepspeed,
|
||||
use_torch_compile=use_torch_compile,
|
||||
use_accel=use_accel,
|
||||
low_vram=low_vram
|
||||
)
|
||||
|
||||
|
||||
@staticmethod
|
||||
def _normalize_emotion_audio(emotion_audio):
|
||||
"""Return a path or waveform dict accepted by IndexTTS-2."""
|
||||
if not isinstance(emotion_audio, dict):
|
||||
return emotion_audio
|
||||
if emotion_audio.get("audio_path"):
|
||||
print(
|
||||
"🎭 Using Character Voices emotion audio: "
|
||||
f"{emotion_audio.get('character_name', 'unknown')} -> "
|
||||
f"{emotion_audio['audio_path']}"
|
||||
)
|
||||
return emotion_audio["audio_path"]
|
||||
if "waveform" in emotion_audio:
|
||||
return emotion_audio
|
||||
nested_audio = emotion_audio.get("audio")
|
||||
if isinstance(nested_audio, dict) and "waveform" in nested_audio:
|
||||
return nested_audio
|
||||
return emotion_audio
|
||||
|
||||
def generate(self,
|
||||
text: str,
|
||||
speaker_audio: Optional[str] = None,
|
||||
emotion_audio: Optional[str] = None,
|
||||
emotion_alpha: float = 1.0,
|
||||
emotion_vector: Optional[List[float]] = None,
|
||||
use_emotion_text: bool = False,
|
||||
emotion_text: Optional[str] = None,
|
||||
use_random: bool = False,
|
||||
interval_silence: int = 200,
|
||||
max_text_tokens_per_segment: int = 120,
|
||||
# Generation parameters
|
||||
temperature: float = 0.8,
|
||||
top_p: float = 0.8,
|
||||
top_k: int = 30,
|
||||
length_penalty: float = 0.0,
|
||||
num_beams: int = 3,
|
||||
repetition_penalty: float = 10.0,
|
||||
max_mel_tokens: int = 1500,
|
||||
language: str = "English",
|
||||
duration_factor: float = 1.0,
|
||||
text_normalization: bool = True,
|
||||
# Streaming parameters
|
||||
stream_return: bool = False,
|
||||
more_segment_before: int = 0,
|
||||
**kwargs) -> torch.Tensor:
|
||||
"""
|
||||
Generate speech with IndexTTS-2.
|
||||
|
||||
Args:
|
||||
text: Text to synthesize (supports [character:emotion] tags)
|
||||
speaker_audio: Speaker reference audio file path
|
||||
emotion_audio: Emotion reference audio file path
|
||||
emotion_alpha: Emotion blend factor (0.0-1.0)
|
||||
emotion_vector: Manual emotion vector [happy, angry, sad, afraid, disgusted, melancholic, surprised, calm]
|
||||
use_emotion_text: Extract emotions from text
|
||||
emotion_text: Custom emotion description text
|
||||
use_random: Enable random sampling
|
||||
interval_silence: Silence between segments (ms)
|
||||
max_text_tokens_per_segment: Max tokens per segment
|
||||
temperature: Sampling temperature
|
||||
top_p: Nucleus sampling parameter
|
||||
top_k: Top-k sampling parameter
|
||||
length_penalty: Length penalty for beam search
|
||||
num_beams: Number of beams for beam search
|
||||
repetition_penalty: Repetition penalty
|
||||
max_mel_tokens: Maximum mel tokens to generate
|
||||
language: IndexTTS-2.5 language code/name
|
||||
duration_factor: Official 2.5 internal feature-duration multiplier
|
||||
text_normalization: Enable multilingual text normalization
|
||||
**kwargs: Additional parameters
|
||||
|
||||
Returns:
|
||||
Generated audio tensor [1, samples] at 22050 Hz
|
||||
"""
|
||||
if self.engine is None:
|
||||
self.initialize_engine()
|
||||
|
||||
# Check if text actually contains character tags before processing
|
||||
has_character_tags = character_parser.CHARACTER_TAG_PATTERN.search(text) is not None
|
||||
|
||||
if has_character_tags:
|
||||
# Parse character switching tags with emotion support
|
||||
processed_segments = self._process_character_tags_with_emotions(text)
|
||||
|
||||
if len(processed_segments) > 1:
|
||||
# Multi-segment character switching - process each segment separately
|
||||
return self._generate_multi_character_segments(
|
||||
processed_segments, speaker_audio, emotion_audio,
|
||||
emotion_alpha=emotion_alpha,
|
||||
emotion_vector=emotion_vector,
|
||||
use_emotion_text=use_emotion_text,
|
||||
emotion_text=emotion_text,
|
||||
use_random=use_random,
|
||||
interval_silence=interval_silence,
|
||||
max_text_tokens_per_segment=max_text_tokens_per_segment,
|
||||
temperature=temperature,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
length_penalty=length_penalty,
|
||||
num_beams=num_beams,
|
||||
repetition_penalty=repetition_penalty,
|
||||
max_mel_tokens=max_mel_tokens,
|
||||
language=language,
|
||||
duration_factor=duration_factor,
|
||||
text_normalization=text_normalization,
|
||||
stream_return=stream_return,
|
||||
more_segment_before=more_segment_before,
|
||||
**kwargs,
|
||||
)
|
||||
elif processed_segments:
|
||||
# Single character segment
|
||||
first_segment = processed_segments[0]
|
||||
processed_text = first_segment.get('text', '').strip()
|
||||
character_name = first_segment.get('character')
|
||||
emotion_ref = first_segment.get('emotion')
|
||||
else:
|
||||
processed_text = text
|
||||
character_name = None
|
||||
emotion_ref = None
|
||||
else:
|
||||
# No character tags - use text as is
|
||||
processed_text = text
|
||||
character_name = None
|
||||
emotion_ref = None
|
||||
|
||||
# Determine final speaker and emotion audio
|
||||
final_speaker_audio = speaker_audio
|
||||
|
||||
# Normalize the two supported audio-reference shapes:
|
||||
# Character Voices returns {audio: {waveform, sample_rate}, audio_path: ...},
|
||||
# while ComfyUI AUDIO returns {waveform, sample_rate} directly.
|
||||
final_emotion_audio = self._normalize_emotion_audio(emotion_audio)
|
||||
|
||||
# Only do character mapping if we actually have character tags
|
||||
if has_character_tags:
|
||||
# Collect all characters that might be needed
|
||||
all_characters = []
|
||||
if character_name:
|
||||
all_characters.append(character_name)
|
||||
if emotion_ref:
|
||||
all_characters.append(emotion_ref)
|
||||
|
||||
# Get character mapping for IndexTTS
|
||||
if all_characters:
|
||||
character_mapping = get_character_mapping(all_characters, engine_type="index_tts")
|
||||
|
||||
if character_name and character_name in character_mapping:
|
||||
# IndexTTS returns (audio_path, reference_text) tuples, we need audio_path
|
||||
character_audio_path = character_mapping[character_name][0]
|
||||
if character_audio_path is not None:
|
||||
final_speaker_audio = character_audio_path
|
||||
print(f"🎭 Using character voice: {character_name} -> {final_speaker_audio}")
|
||||
else:
|
||||
print(f"⚠️ Character '{character_name}' mapping returned None, using original speaker audio: {final_speaker_audio}")
|
||||
elif character_name:
|
||||
print(f"⚠️ Character '{character_name}' not found in character mapping, using original speaker audio: {final_speaker_audio}")
|
||||
|
||||
if emotion_ref and emotion_ref in character_mapping:
|
||||
# IndexTTS returns (audio_path, reference_text) tuples, we need audio_path
|
||||
final_emotion_audio = character_mapping[emotion_ref][0]
|
||||
print(f"😊 Using emotion reference: {emotion_ref} -> {final_emotion_audio}")
|
||||
|
||||
# For caching, we need stable identifiers for audio references
|
||||
# Convert file paths to content hashes if they're temp files
|
||||
cache_speaker_audio = self._get_stable_audio_identifier(final_speaker_audio)
|
||||
cache_emotion_audio = self._get_stable_audio_identifier(final_emotion_audio) if final_emotion_audio else final_emotion_audio
|
||||
|
||||
# Check cache before generation
|
||||
cache_key = self._generate_cache_key(
|
||||
text=processed_text,
|
||||
speaker_audio=cache_speaker_audio,
|
||||
emotion_audio=cache_emotion_audio,
|
||||
emotion_alpha=emotion_alpha,
|
||||
emotion_vector=emotion_vector,
|
||||
use_emotion_text=use_emotion_text,
|
||||
emotion_text=emotion_text,
|
||||
use_random=use_random,
|
||||
temperature=temperature,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
length_penalty=length_penalty,
|
||||
repetition_penalty=repetition_penalty,
|
||||
max_mel_tokens=max_mel_tokens,
|
||||
max_text_tokens_per_segment=max_text_tokens_per_segment,
|
||||
interval_silence=interval_silence,
|
||||
language=language,
|
||||
duration_factor=duration_factor,
|
||||
text_normalization=text_normalization,
|
||||
stream_return=stream_return,
|
||||
more_segment_before=more_segment_before,
|
||||
**kwargs # Include seed and other kwargs in cache key
|
||||
)
|
||||
|
||||
# print(f"🐛 IndexTTS-2 cache key params: text='{processed_text[:30]}...', seed={kwargs.get('seed', 'missing')}, speaker='{final_speaker_audio}', emotion='{final_emotion_audio}'")
|
||||
# print(f"🐛 IndexTTS-2 full cache key: {cache_key}")
|
||||
# print(f"🐛 IndexTTS-2 cache stats: {self.audio_cache.get_cache_stats()}")
|
||||
|
||||
cached_audio = self.audio_cache.get_cached_audio(cache_key)
|
||||
if cached_audio:
|
||||
character_desc = character_name or 'narrator'
|
||||
print(f"💾 Using cached IndexTTS-2 audio for '{character_desc}': '{processed_text[:30]}...'")
|
||||
return cached_audio[0]
|
||||
# else:
|
||||
# print(f"❌ IndexTTS-2 cache MISS for key: {cache_key}")
|
||||
# # Show first few existing keys for comparison
|
||||
# # Access the global cache properly
|
||||
# from utils.audio.cache import GLOBAL_AUDIO_CACHE
|
||||
# existing_keys = list(GLOBAL_AUDIO_CACHE.keys())
|
||||
# if existing_keys:
|
||||
# print(f"🔍 Existing cache keys (first 3): {existing_keys[:3]}")
|
||||
# else:
|
||||
# print(f"🔍 Cache is empty")
|
||||
|
||||
# Handle seed for deterministic generation
|
||||
seed = kwargs.get('seed', 0)
|
||||
if seed != 0:
|
||||
torch.manual_seed(seed)
|
||||
import random
|
||||
import numpy as np
|
||||
random.seed(seed)
|
||||
np.random.seed(seed)
|
||||
|
||||
# Filter out seed from kwargs (used for caching but not supported by IndexTTS engine)
|
||||
engine_kwargs = {k: v for k, v in kwargs.items() if k != 'seed'}
|
||||
# Add streaming parameters to engine kwargs
|
||||
engine_kwargs['stream_return'] = stream_return
|
||||
engine_kwargs['more_segment_before'] = more_segment_before
|
||||
|
||||
# Audio emotion and vector/text emotion are independent conditioning
|
||||
# sources. IndexTTS-2 blends them in its latent emotion space.
|
||||
final_emotion_vector = emotion_vector
|
||||
final_use_emotion_text = use_emotion_text
|
||||
final_emotion_text = emotion_text
|
||||
|
||||
# Generate audio with OOM protection
|
||||
try:
|
||||
audio = self.engine.generate(
|
||||
text=processed_text,
|
||||
speaker_audio=final_speaker_audio,
|
||||
emotion_audio=final_emotion_audio,
|
||||
emotion_alpha=emotion_alpha,
|
||||
emotion_vector=final_emotion_vector,
|
||||
use_emotion_text=final_use_emotion_text,
|
||||
emotion_text=final_emotion_text,
|
||||
use_random=use_random,
|
||||
interval_silence=interval_silence,
|
||||
max_text_tokens_per_segment=max_text_tokens_per_segment,
|
||||
temperature=temperature,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
length_penalty=length_penalty,
|
||||
num_beams=num_beams,
|
||||
repetition_penalty=repetition_penalty,
|
||||
max_mel_tokens=max_mel_tokens,
|
||||
language=language,
|
||||
duration_factor=duration_factor,
|
||||
text_normalization=text_normalization,
|
||||
**engine_kwargs
|
||||
)
|
||||
except torch.OutOfMemoryError as e:
|
||||
# Analyze audio after OOM to provide helpful feedback
|
||||
audio_analysis = self._analyze_audio_after_oom(final_speaker_audio, final_emotion_audio, max_mel_tokens)
|
||||
raise RuntimeError(
|
||||
f"❌ IndexTTS-2 ran out of GPU memory during generation.\n{audio_analysis}"
|
||||
) from e
|
||||
|
||||
# Handle streaming vs non-streaming returns
|
||||
if stream_return:
|
||||
# In streaming mode, audio is a generator - return it as-is without caching
|
||||
return audio
|
||||
else:
|
||||
# Cache the result for non-streaming mode
|
||||
duration = audio.shape[-1] / 22050.0 # IndexTTS-2 uses 22050 Hz
|
||||
self.audio_cache.cache_audio(cache_key, audio, duration)
|
||||
# print(f"💾 IndexTTS-2 CACHED audio with key: {cache_key}")
|
||||
return audio
|
||||
|
||||
def _process_character_tags_with_emotions(self, text: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Process character switching tags with emotion support.
|
||||
"""
|
||||
from utils.text.character_parser.base_parser import CharacterParser
|
||||
from utils.voice.discovery import get_available_characters, voice_discovery
|
||||
|
||||
# Create a temporary parser and set it up properly
|
||||
temp_parser = CharacterParser()
|
||||
|
||||
# Get actual available characters and aliases
|
||||
available_chars = get_available_characters()
|
||||
character_aliases = voice_discovery.get_character_aliases()
|
||||
|
||||
# Build complete available set (like before but not hardcoded)
|
||||
all_available = set()
|
||||
if available_chars:
|
||||
all_available.update(available_chars)
|
||||
for alias, target in character_aliases.items():
|
||||
all_available.add(alias.lower())
|
||||
all_available.add(target.lower())
|
||||
|
||||
temp_parser.set_available_characters(list(all_available))
|
||||
|
||||
# Set language defaults
|
||||
char_lang_defaults = voice_discovery.get_character_language_defaults()
|
||||
for char, lang in char_lang_defaults.items():
|
||||
temp_parser.set_character_language_default(char, lang)
|
||||
|
||||
# Use emotion-aware parsing
|
||||
segments = temp_parser.split_by_character_with_emotions(text)
|
||||
|
||||
processed_segments = []
|
||||
for character, segment_text, language, emotion in segments:
|
||||
segment_info = {
|
||||
'character': character,
|
||||
'text': segment_text,
|
||||
'language': language,
|
||||
'emotion': emotion
|
||||
}
|
||||
processed_segments.append(segment_info)
|
||||
|
||||
return processed_segments
|
||||
|
||||
def _generate_multi_character_segments(self, segments: List[Dict[str, Any]],
|
||||
default_speaker_audio: Optional[str],
|
||||
default_emotion_audio: Optional[str],
|
||||
**kwargs) -> torch.Tensor:
|
||||
"""
|
||||
Generate audio for multiple character segments and combine them.
|
||||
|
||||
Args:
|
||||
segments: List of segment dictionaries with character, text, language, emotion info
|
||||
default_speaker_audio: Default speaker reference audio
|
||||
default_emotion_audio: Default emotion reference audio
|
||||
**kwargs: Additional generation parameters
|
||||
|
||||
Returns:
|
||||
Combined audio tensor [1, samples] at 22050 Hz
|
||||
"""
|
||||
audio_segments = []
|
||||
|
||||
# Get character mapping for all unique characters
|
||||
unique_characters = set()
|
||||
for segment in segments:
|
||||
if segment.get('character'):
|
||||
unique_characters.add(segment['character'])
|
||||
if segment.get('emotion'):
|
||||
unique_characters.add(segment['emotion'])
|
||||
|
||||
# Get character mapping for IndexTTS
|
||||
character_mapping = {}
|
||||
if unique_characters:
|
||||
character_mapping = get_character_mapping(list(unique_characters), engine_type="index_tts")
|
||||
|
||||
resolved_names = [
|
||||
resolved_character_label(
|
||||
segment.get('character', 'narrator'),
|
||||
character_mapping.get(segment.get('character', 'narrator'), (default_speaker_audio, None)),
|
||||
)
|
||||
for segment in segments
|
||||
]
|
||||
print(f"🎭 IndexTTS-2: Processing {len(segments)} character segment(s) - {', '.join(resolved_names)}")
|
||||
|
||||
for segment in segments:
|
||||
character_name = segment.get('character', 'narrator')
|
||||
segment_text = segment.get('text', '').strip()
|
||||
emotion_ref = segment.get('emotion')
|
||||
segment_kwargs = dict(kwargs)
|
||||
if segment.get('language'):
|
||||
segment_kwargs['language'] = segment['language']
|
||||
|
||||
if not segment_text:
|
||||
continue
|
||||
|
||||
# Determine speaker audio for this segment
|
||||
speaker_audio = default_speaker_audio
|
||||
if character_name and character_name in character_mapping:
|
||||
character_audio_path = character_mapping[character_name][0]
|
||||
if character_audio_path:
|
||||
speaker_audio = character_audio_path
|
||||
print(f"📖 Using character voice '{resolved_character_label(character_name, speaker_audio)}' | Ref: '{speaker_audio}'")
|
||||
else:
|
||||
print(f"⚠️ Character '{character_name}' has no audio reference, using default")
|
||||
|
||||
# Determine emotion audio for this segment
|
||||
emotion_audio = default_emotion_audio
|
||||
if emotion_ref and emotion_ref in character_mapping:
|
||||
emotion_audio_path = character_mapping[emotion_ref][0]
|
||||
if emotion_audio_path:
|
||||
emotion_audio = emotion_audio_path
|
||||
print(f"😊 Using emotion reference '{emotion_ref}' | Ref: '{emotion_audio}'")
|
||||
|
||||
# Generate cache key for this segment
|
||||
segment_cache_key = self._generate_cache_key(
|
||||
text=segment_text,
|
||||
speaker_audio=speaker_audio,
|
||||
emotion_audio=emotion_audio,
|
||||
**segment_kwargs
|
||||
)
|
||||
|
||||
# Check cache first
|
||||
cached_segment_audio = self.audio_cache.get_cached_audio(segment_cache_key)
|
||||
if cached_segment_audio:
|
||||
print(f"💾 Using cached IndexTTS-2 segment for '{resolved_character_label(character_name, speaker_audio)}': '{segment_text[:30]}...'")
|
||||
segment_audio = cached_segment_audio[0]
|
||||
else:
|
||||
# Generate audio for this segment with OOM protection
|
||||
try:
|
||||
segment_audio = self.engine.generate(
|
||||
text=segment_text,
|
||||
speaker_audio=speaker_audio,
|
||||
emotion_audio=emotion_audio,
|
||||
**segment_kwargs
|
||||
)
|
||||
except torch.OutOfMemoryError as e:
|
||||
# Analyze audio after OOM in multi-character segments
|
||||
audio_analysis = self._analyze_audio_after_oom(speaker_audio, emotion_audio, kwargs.get('max_mel_tokens', 1500))
|
||||
raise RuntimeError(
|
||||
f"❌ IndexTTS-2 OOM error in character segment '{character_name}'.\n{audio_analysis}"
|
||||
) from e
|
||||
|
||||
# Cache the segment result
|
||||
duration = segment_audio.shape[-1] / 22050.0 # IndexTTS-2 uses 22050 Hz
|
||||
self.audio_cache.cache_audio(segment_cache_key, segment_audio, duration)
|
||||
# print(f"💾 IndexTTS-2 CACHED segment for '{character_name}' with key: {segment_cache_key}")
|
||||
|
||||
audio_segments.append(segment_audio)
|
||||
|
||||
# Combine all audio segments
|
||||
if audio_segments:
|
||||
combined_audio = torch.cat(audio_segments, dim=-1)
|
||||
return combined_audio
|
||||
else:
|
||||
# Return silence if no segments generated
|
||||
return torch.zeros(1, 22050, dtype=torch.float32)
|
||||
|
||||
def _generate_cache_key(self, **params) -> str:
|
||||
"""Generate cache key for IndexTTS-2."""
|
||||
model_identity = {}
|
||||
if self.engine is not None:
|
||||
model_identity = {
|
||||
"model_name": getattr(self.engine, "model_name", None),
|
||||
"model_version": getattr(self.engine, "model_version", None),
|
||||
"model_path": getattr(self.engine, "model_dir", None),
|
||||
}
|
||||
return self.audio_cache.generate_cache_key(
|
||||
'index_tts', **model_identity, **params
|
||||
)
|
||||
|
||||
def _analyze_audio_after_oom(self, speaker_audio: str, emotion_audio: str, max_mel_tokens: int) -> str:
|
||||
"""
|
||||
Analyze audio files after OOM to provide helpful feedback.
|
||||
Uses fast header-based duration check to avoid loading large files.
|
||||
|
||||
Args:
|
||||
speaker_audio: Path to speaker audio file
|
||||
emotion_audio: Path to emotion audio file
|
||||
max_mel_tokens: Current max_mel_tokens setting
|
||||
|
||||
Returns:
|
||||
Formatted analysis message with recommendations
|
||||
"""
|
||||
analysis_parts = []
|
||||
|
||||
# Analyze speaker audio
|
||||
if speaker_audio and os.path.exists(speaker_audio):
|
||||
try:
|
||||
import soundfile as sf
|
||||
info = sf.info(speaker_audio)
|
||||
duration = info.duration
|
||||
|
||||
if duration > 60:
|
||||
analysis_parts.append(f"🔍 Speaker audio: {duration:.1f}s (TOO LONG - major OOM risk)")
|
||||
analysis_parts.append(" ✅ Solution: Use audio under 30 seconds")
|
||||
elif duration > 30:
|
||||
analysis_parts.append(f"🔍 Speaker audio: {duration:.1f}s (long - may cause OOM)")
|
||||
analysis_parts.append(" ✅ Recommendation: Use shorter audio for stability")
|
||||
else:
|
||||
analysis_parts.append(f"🔍 Speaker audio: {duration:.1f}s (length OK)")
|
||||
|
||||
except Exception:
|
||||
analysis_parts.append("🔍 Speaker audio: Could not analyze duration")
|
||||
|
||||
# Analyze emotion audio
|
||||
if emotion_audio and os.path.exists(emotion_audio):
|
||||
try:
|
||||
import soundfile as sf
|
||||
info = sf.info(emotion_audio)
|
||||
duration = info.duration
|
||||
|
||||
if duration > 60:
|
||||
analysis_parts.append(f"🔍 Emotion audio: {duration:.1f}s (TOO LONG - major OOM risk)")
|
||||
elif duration > 30:
|
||||
analysis_parts.append(f"🔍 Emotion audio: {duration:.1f}s (long - may cause OOM)")
|
||||
else:
|
||||
analysis_parts.append(f"🔍 Emotion audio: {duration:.1f}s (length OK)")
|
||||
|
||||
except Exception:
|
||||
analysis_parts.append("🔍 Emotion audio: Could not analyze duration")
|
||||
|
||||
# Analyze max_mel_tokens setting
|
||||
if max_mel_tokens > 1500:
|
||||
analysis_parts.append(f"🔍 max_mel_tokens: {max_mel_tokens} (high - may cause OOM)")
|
||||
analysis_parts.append(" ✅ Try reducing to 1000-1500")
|
||||
else:
|
||||
analysis_parts.append(f"🔍 max_mel_tokens: {max_mel_tokens} (setting OK)")
|
||||
|
||||
# Add general recommendations
|
||||
analysis_parts.append("")
|
||||
analysis_parts.append("💡 Quick fixes:")
|
||||
analysis_parts.append(" • Use audio clips under 30 seconds")
|
||||
analysis_parts.append(" • Reduce max_mel_tokens to 1000")
|
||||
analysis_parts.append(" • Close other GPU applications")
|
||||
|
||||
return "\n".join(analysis_parts)
|
||||
|
||||
def _get_stable_audio_identifier(self, audio_path: str) -> str:
|
||||
"""
|
||||
Get stable identifier for audio file using centralized audio hashing.
|
||||
"""
|
||||
if not audio_path:
|
||||
return audio_path
|
||||
|
||||
# Use our centralized audio hashing utility
|
||||
from utils.audio.audio_hash import generate_stable_audio_component
|
||||
if isinstance(audio_path, dict):
|
||||
return generate_stable_audio_component(reference_audio=audio_path)
|
||||
return generate_stable_audio_component(audio_file_path=audio_path)
|
||||
|
||||
def get_supported_formats(self) -> List[str]:
|
||||
"""Get supported audio formats."""
|
||||
return ["wav", "mp3", "flac", "ogg"]
|
||||
|
||||
def get_sample_rate(self) -> int:
|
||||
"""Get native sample rate."""
|
||||
return 22050
|
||||
|
||||
def get_emotion_labels(self) -> List[str]:
|
||||
"""Get supported emotion labels."""
|
||||
return ["happy", "angry", "sad", "afraid", "disgusted", "melancholic", "surprised", "calm"]
|
||||
|
||||
def create_emotion_vector(self, **emotions) -> List[float]:
|
||||
"""
|
||||
Create emotion vector from keyword arguments.
|
||||
|
||||
Args:
|
||||
**emotions: Emotion intensities (e.g., happy=0.8, angry=0.2)
|
||||
|
||||
Returns:
|
||||
List of 8 emotion values
|
||||
"""
|
||||
if self.engine:
|
||||
return self.engine.create_emotion_vector(**emotions)
|
||||
else:
|
||||
# Fallback implementation
|
||||
labels = self.get_emotion_labels()
|
||||
vector = [0.0] * len(labels)
|
||||
for i, label in enumerate(labels):
|
||||
if label in emotions:
|
||||
vector[i] = max(0.0, min(1.2, float(emotions[label])))
|
||||
return vector
|
||||
|
||||
def unload(self):
|
||||
"""Unload the engine to free memory."""
|
||||
if self.engine:
|
||||
self.engine.unload()
|
||||
self.engine = None
|
||||
@@ -1,69 +0,0 @@
|
||||
"""Suite adapter for MOSS-SoundEffect v2."""
|
||||
|
||||
from typing import Any, Dict, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from engines.moss_soundeffect_v2.downloader import MossSoundEffectV2Downloader
|
||||
from utils.audio.cache import get_audio_cache
|
||||
from utils.models.factory_config import ModelLoadConfig
|
||||
from utils.models.unified_model_interface import unified_model_interface
|
||||
|
||||
|
||||
class MossSoundEffectV2Adapter:
|
||||
def __init__(self, config: Dict[str, Any]):
|
||||
self.config = dict(config)
|
||||
self.audio_cache = get_audio_cache()
|
||||
self._load_config = None
|
||||
|
||||
def _load(self):
|
||||
model = self.config.get("model", MossSoundEffectV2Downloader.MODEL_NAME)
|
||||
model_path = MossSoundEffectV2Downloader().resolve_model_path(model)
|
||||
self._load_config = ModelLoadConfig(
|
||||
engine_name="moss_soundeffect_v2",
|
||||
model_type="tts",
|
||||
model_name=str(model).removeprefix("local:"),
|
||||
model_path=model_path,
|
||||
device=self.config.get("device", "auto"),
|
||||
runtime_mode="main_environment",
|
||||
additional_params={"dtype": self.config.get("dtype", "auto")},
|
||||
)
|
||||
return unified_model_interface.load_model(self._load_config)
|
||||
|
||||
def generate(
|
||||
self,
|
||||
description: str,
|
||||
duration_seconds: float,
|
||||
seed: int,
|
||||
enable_audio_cache: bool,
|
||||
) -> Tuple[torch.Tensor, int, bool]:
|
||||
duration_seconds = round(float(duration_seconds), 1)
|
||||
if duration_seconds > 30.0:
|
||||
raise ValueError(
|
||||
"MOSS-SoundEffect v2 supports a maximum duration of 30 seconds. "
|
||||
"Lower duration_seconds in the 🌩️ Sound Effects node."
|
||||
)
|
||||
params = {
|
||||
"description": description,
|
||||
"model": self.config.get("model"),
|
||||
"duration_seconds": duration_seconds,
|
||||
"inference_steps": self.config.get("inference_steps", 100),
|
||||
"cfg_scale": self.config.get("cfg_scale", 4.0),
|
||||
"sigma_shift": self.config.get("sigma_shift", 5.0),
|
||||
"negative_prompt": self.config.get("negative_prompt", ""),
|
||||
"seed": int(seed),
|
||||
"dtype": self.config.get("dtype", "auto"),
|
||||
"device": self.config.get("device", "auto"),
|
||||
}
|
||||
cache_key = self.audio_cache.generate_cache_key("moss_soundeffect_v2", **params)
|
||||
if enable_audio_cache:
|
||||
cached = self.audio_cache.get_cached_audio(cache_key)
|
||||
if cached is not None:
|
||||
return cached[0], 48000, True
|
||||
|
||||
engine = self._load()
|
||||
waveform, sample_rate = engine.generate_sound_effect(**params)
|
||||
waveform = waveform.detach().cpu().float()
|
||||
if enable_audio_cache:
|
||||
self.audio_cache.cache_audio(cache_key, waveform, waveform.shape[-1] / float(sample_rate))
|
||||
return waveform, int(sample_rate), False
|
||||
@@ -1,384 +0,0 @@
|
||||
"""
|
||||
MOSS-TTS engine adapter.
|
||||
|
||||
Translates TTS Audio Suite processor calls into official MOSS-TTS inference
|
||||
without exposing non-native controls.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
current_dir = os.path.dirname(__file__)
|
||||
engines_dir = os.path.dirname(current_dir)
|
||||
project_root = os.path.dirname(engines_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
from utils.audio.audio_hash import generate_stable_audio_component
|
||||
from utils.audio.cache import get_audio_cache
|
||||
from utils.models.factory_config import ModelLoadConfig
|
||||
from utils.text.pause_processor import PauseTagProcessor
|
||||
from utils.voice.reference import effective_voice_audio
|
||||
|
||||
|
||||
class MossTTSEngineAdapter:
|
||||
"""Adapter for the official MOSS-TTS model API."""
|
||||
|
||||
SAMPLE_RATE = 24000
|
||||
def __init__(self, node_instance=None):
|
||||
self.node = node_instance
|
||||
self.audio_cache = get_audio_cache()
|
||||
self._last_config: Optional[ModelLoadConfig] = None
|
||||
|
||||
def load_model(
|
||||
self,
|
||||
model_variant: str,
|
||||
device: str = "auto",
|
||||
dtype: str = "auto",
|
||||
attn_implementation: str = "auto",
|
||||
codec_model: str = "MOSS-Audio-Tokenizer",
|
||||
lora_adapter: Optional[str] = None,
|
||||
defer_load: bool = False,
|
||||
):
|
||||
from utils.models.unified_model_interface import unified_model_interface
|
||||
|
||||
config = ModelLoadConfig(
|
||||
engine_name="moss_tts",
|
||||
model_type="tts",
|
||||
model_name=model_variant,
|
||||
model_path=model_variant,
|
||||
device=device,
|
||||
additional_params={
|
||||
"dtype": dtype,
|
||||
"attn_implementation": attn_implementation,
|
||||
"codec_model": codec_model,
|
||||
"lora_adapter": lora_adapter,
|
||||
},
|
||||
)
|
||||
self._last_config = config
|
||||
if not defer_load:
|
||||
unified_model_interface.load_model(config)
|
||||
|
||||
def update_model_config(
|
||||
self,
|
||||
model_variant: str,
|
||||
device: str = "auto",
|
||||
dtype: str = "auto",
|
||||
attn_implementation: str = "auto",
|
||||
codec_model: str = "MOSS-Audio-Tokenizer",
|
||||
lora_adapter: Optional[str] = None,
|
||||
):
|
||||
self.load_model(model_variant, device, dtype, attn_implementation, codec_model, lora_adapter)
|
||||
|
||||
def _get_engine(self):
|
||||
if self._last_config is None:
|
||||
raise RuntimeError("MOSS-TTS model has not been loaded")
|
||||
from utils.models.unified_model_interface import unified_model_interface
|
||||
|
||||
return unified_model_interface.load_model(self._last_config)
|
||||
|
||||
def generate_with_pause_tags(
|
||||
self,
|
||||
text: str,
|
||||
voice_ref: Optional[Dict[str, Any]],
|
||||
params: Dict[str, Any],
|
||||
process_pauses: bool = True,
|
||||
character_name: Optional[str] = None,
|
||||
) -> torch.Tensor:
|
||||
engine = self._get_engine()
|
||||
seed = int(params.get("seed", 0) or 0)
|
||||
if seed > 0:
|
||||
torch.manual_seed(seed)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed_all(seed)
|
||||
|
||||
if process_pauses and PauseTagProcessor.has_pause_tags(text):
|
||||
return self._generate_with_pauses(text, voice_ref, params, character_name, engine)
|
||||
return self._generate_direct(text, voice_ref, params, character_name, engine)
|
||||
|
||||
def _generate_direct(
|
||||
self,
|
||||
text: str,
|
||||
voice_ref: Optional[Dict[str, Any]],
|
||||
params: Dict[str, Any],
|
||||
character_name: Optional[str] = None,
|
||||
engine=None,
|
||||
) -> torch.Tensor:
|
||||
reference_audio, reference_sample_rate, audio_component = self._extract_voice_reference(voice_ref)
|
||||
model_variant = params.get("model_variant", "MOSS-TTS-Local-Transformer")
|
||||
language = params.get("language", "auto")
|
||||
explicit_duration = params.get("duration_tokens")
|
||||
duration_tokens = max(1, int(explicit_duration)) if explicit_duration else None
|
||||
resolved_max_new_tokens = int(params.get("max_new_tokens", 4096))
|
||||
n_vq_for_inference = params.get("n_vq_for_inference")
|
||||
lora_adapter = params.get("lora_adapter")
|
||||
instruction = params.get("instruction")
|
||||
quality = params.get("quality")
|
||||
sound_event = params.get("sound_event")
|
||||
ambient_sound = params.get("ambient_sound")
|
||||
|
||||
cache_key = self.audio_cache.generate_cache_key(
|
||||
"moss_tts",
|
||||
text=text,
|
||||
model_variant=model_variant,
|
||||
audio_component=audio_component,
|
||||
language=language,
|
||||
instruction=instruction,
|
||||
quality=quality,
|
||||
sound_event=sound_event,
|
||||
ambient_sound=ambient_sound,
|
||||
duration_tokens=duration_tokens,
|
||||
audio_temperature=params.get("audio_temperature", params.get("temperature", 1.0)),
|
||||
audio_top_p=params.get("audio_top_p", params.get("top_p", 0.95)),
|
||||
audio_top_k=params.get("audio_top_k", params.get("top_k", 50)),
|
||||
audio_repetition_penalty=params.get("audio_repetition_penalty", params.get("repetition_penalty", 1.1)),
|
||||
max_new_tokens=resolved_max_new_tokens,
|
||||
n_vq_for_inference=n_vq_for_inference,
|
||||
seed=params.get("seed", 0),
|
||||
device=params.get("device", "auto"),
|
||||
dtype=params.get("dtype", "auto"),
|
||||
attn_implementation=params.get("attn_implementation", "auto"),
|
||||
lora_adapter=lora_adapter,
|
||||
character=character_name or "narrator",
|
||||
)
|
||||
|
||||
enable_audio_cache = bool(params.get("enable_audio_cache", True))
|
||||
cached_audio = self.audio_cache.get_cached_audio(cache_key) if enable_audio_cache else None
|
||||
if cached_audio:
|
||||
print(f"💾 Using cached MOSS-TTS audio for '{character_name or 'narrator'}': '{text[:30]}...'")
|
||||
return cached_audio[0]
|
||||
|
||||
if engine is None:
|
||||
engine = self._get_engine()
|
||||
|
||||
audio_tensor, sample_rate = engine.generate(
|
||||
text=text,
|
||||
reference_audio=reference_audio,
|
||||
reference_sample_rate=reference_sample_rate,
|
||||
language=language,
|
||||
duration_tokens=duration_tokens,
|
||||
instruction=instruction,
|
||||
quality=quality,
|
||||
sound_event=sound_event,
|
||||
ambient_sound=ambient_sound,
|
||||
seed=int(params.get("seed", 0) or 0),
|
||||
audio_temperature=float(params.get("audio_temperature", params.get("temperature", 1.0))),
|
||||
audio_top_p=float(params.get("audio_top_p", params.get("top_p", 0.95))),
|
||||
audio_top_k=int(params.get("audio_top_k", params.get("top_k", 50))),
|
||||
audio_repetition_penalty=float(
|
||||
params.get("audio_repetition_penalty", params.get("repetition_penalty", 1.1))
|
||||
),
|
||||
max_new_tokens=int(resolved_max_new_tokens),
|
||||
n_vq_for_inference=n_vq_for_inference,
|
||||
)
|
||||
|
||||
if sample_rate != self.SAMPLE_RATE:
|
||||
raise RuntimeError(f"MOSS-TTS returned unexpected sample rate {sample_rate}; expected {self.SAMPLE_RATE}")
|
||||
|
||||
if enable_audio_cache:
|
||||
duration = self.audio_cache._calculate_duration(audio_tensor, "moss_tts")
|
||||
self.audio_cache.cache_audio(cache_key, audio_tensor, duration)
|
||||
return audio_tensor
|
||||
|
||||
def generate_sound_effect(self, description: str, params: Dict[str, Any]) -> torch.Tensor:
|
||||
"""Generate MOSS-SoundEffect v1 audio without a speech or narrator reference."""
|
||||
sound_params = dict(params)
|
||||
sound_params["ambient_sound"] = str(description or "").strip()
|
||||
if not sound_params["ambient_sound"]:
|
||||
raise ValueError("MOSS-SoundEffect requires a non-empty description")
|
||||
return self._generate_direct(
|
||||
text="",
|
||||
voice_ref=None,
|
||||
params=sound_params,
|
||||
character_name="sound_effects",
|
||||
)
|
||||
|
||||
def _generate_with_pauses(
|
||||
self,
|
||||
text: str,
|
||||
voice_ref: Optional[Dict[str, Any]],
|
||||
params: Dict[str, Any],
|
||||
character_name: Optional[str] = None,
|
||||
engine=None,
|
||||
) -> torch.Tensor:
|
||||
segments, _ = PauseTagProcessor.parse_pause_tags(text)
|
||||
audio_parts = []
|
||||
for segment_type, content in segments:
|
||||
if segment_type == "text":
|
||||
audio_tensor = self._generate_direct(content, voice_ref, params, character_name, engine)
|
||||
if audio_tensor.dim() == 1:
|
||||
audio_tensor = audio_tensor.unsqueeze(0)
|
||||
elif audio_tensor.dim() == 3:
|
||||
audio_tensor = audio_tensor.squeeze(0)
|
||||
audio_parts.append(audio_tensor)
|
||||
elif segment_type == "pause":
|
||||
audio_parts.append(PauseTagProcessor.create_silence_segment(content, self.SAMPLE_RATE))
|
||||
|
||||
if not audio_parts:
|
||||
return torch.zeros(1, 0)
|
||||
return torch.cat(audio_parts, dim=-1)
|
||||
|
||||
def _extract_voice_reference(self, voice_ref: Optional[Dict[str, Any]]) -> Tuple[Any, Optional[int], str]:
|
||||
if not voice_ref or not isinstance(voice_ref, dict):
|
||||
return None, None, "default_voice"
|
||||
|
||||
ref_audio = effective_voice_audio(voice_ref)
|
||||
if ref_audio is None:
|
||||
return None, None, "default_voice"
|
||||
|
||||
if isinstance(ref_audio, str):
|
||||
return ref_audio, None, generate_stable_audio_component(audio_file_path=ref_audio)
|
||||
|
||||
if isinstance(ref_audio, dict) and "waveform" in ref_audio:
|
||||
return (
|
||||
ref_audio,
|
||||
int(ref_audio.get("sample_rate", self.SAMPLE_RATE)),
|
||||
generate_stable_audio_component(reference_audio=ref_audio),
|
||||
)
|
||||
|
||||
if torch.is_tensor(ref_audio):
|
||||
sample_rate = int(voice_ref.get("sample_rate", self.SAMPLE_RATE))
|
||||
audio_dict = {"waveform": ref_audio, "sample_rate": sample_rate}
|
||||
return audio_dict, sample_rate, generate_stable_audio_component(reference_audio=audio_dict)
|
||||
|
||||
return ref_audio, voice_ref.get("sample_rate"), "custom_voice"
|
||||
|
||||
def _normalize_dialogue_voice_ref(self, voice_ref: Any, speaker_label: str) -> Tuple[Optional[Dict[str, Any]], str]:
|
||||
if voice_ref is None:
|
||||
return None, "no_voice"
|
||||
|
||||
if isinstance(voice_ref, str):
|
||||
print(
|
||||
f"❌ MOSS-TTSD {speaker_label}: received character name '{voice_ref}' instead of voice reference. "
|
||||
"Connect Character Voices opt_narrator, not character_name."
|
||||
)
|
||||
return None, "invalid_string_voice"
|
||||
|
||||
if not isinstance(voice_ref, dict):
|
||||
print(f"❌ MOSS-TTSD {speaker_label}: invalid voice reference type {type(voice_ref).__name__}")
|
||||
return None, "invalid_voice"
|
||||
|
||||
ref_audio = effective_voice_audio(voice_ref)
|
||||
reference_text = (
|
||||
voice_ref.get("reference_text")
|
||||
or voice_ref.get("text")
|
||||
or voice_ref.get("prompt_text")
|
||||
or ""
|
||||
)
|
||||
|
||||
if ref_audio is None:
|
||||
return None, "no_audio"
|
||||
|
||||
normalized = {
|
||||
"audio": ref_audio,
|
||||
"sample_rate": voice_ref.get("sample_rate", self.SAMPLE_RATE),
|
||||
"reference_text": str(reference_text or "").strip(),
|
||||
}
|
||||
|
||||
if isinstance(ref_audio, str):
|
||||
audio_component = generate_stable_audio_component(audio_file_path=ref_audio)
|
||||
elif isinstance(ref_audio, dict) and "waveform" in ref_audio:
|
||||
audio_component = generate_stable_audio_component(reference_audio=ref_audio)
|
||||
normalized["sample_rate"] = int(ref_audio.get("sample_rate", self.SAMPLE_RATE))
|
||||
elif torch.is_tensor(ref_audio):
|
||||
audio_component = generate_stable_audio_component(
|
||||
reference_audio={
|
||||
"waveform": ref_audio,
|
||||
"sample_rate": normalized["sample_rate"],
|
||||
}
|
||||
)
|
||||
else:
|
||||
audio_component = "custom_voice"
|
||||
|
||||
if not normalized["reference_text"]:
|
||||
print(f"⚠️ MOSS-TTSD {speaker_label}: reference audio has no transcript; cloning disabled for this speaker")
|
||||
return None, audio_component
|
||||
|
||||
return normalized, audio_component
|
||||
|
||||
def generate_native_dialogue(
|
||||
self,
|
||||
dialogue_text: str,
|
||||
speaker_voices: List[Any],
|
||||
params: Dict[str, Any],
|
||||
) -> Dict[str, Any]:
|
||||
engine = self._get_engine()
|
||||
normalized_refs: List[Optional[Dict[str, Any]]] = []
|
||||
audio_components = []
|
||||
|
||||
for idx, voice_ref in enumerate(speaker_voices):
|
||||
normalized, audio_component = self._normalize_dialogue_voice_ref(voice_ref, f"S{idx + 1}")
|
||||
normalized_refs.append(normalized)
|
||||
audio_components.append(audio_component)
|
||||
|
||||
model_variant = params.get("model_variant", "MOSS-TTSD-v1.0")
|
||||
cache_key = self.audio_cache.generate_cache_key(
|
||||
"moss_tts",
|
||||
text=dialogue_text,
|
||||
model_variant=model_variant,
|
||||
audio_component="ttsd_" + "_".join(audio_components),
|
||||
language=params.get("language", "auto"),
|
||||
instruction=params.get("instruction"),
|
||||
quality=params.get("quality"),
|
||||
sound_event=params.get("sound_event"),
|
||||
ambient_sound=params.get("ambient_sound"),
|
||||
duration_tokens=params.get("duration_tokens"),
|
||||
audio_temperature=params.get("audio_temperature", params.get("temperature", 1.1)),
|
||||
audio_top_p=params.get("audio_top_p", params.get("top_p", 0.9)),
|
||||
audio_top_k=params.get("audio_top_k", params.get("top_k", 50)),
|
||||
audio_repetition_penalty=params.get("audio_repetition_penalty", params.get("repetition_penalty", 1.1)),
|
||||
max_new_tokens=params.get("max_new_tokens", 4096),
|
||||
seed=params.get("seed", 0),
|
||||
device=params.get("device", "auto"),
|
||||
dtype=params.get("dtype", "auto"),
|
||||
attn_implementation=params.get("attn_implementation", "auto"),
|
||||
character="native_multispeaker",
|
||||
)
|
||||
|
||||
cached_audio = self.audio_cache.get_cached_audio(cache_key)
|
||||
if cached_audio:
|
||||
print("💾 Using cached MOSS-TTSD native dialogue audio")
|
||||
audio_tensor = cached_audio[0]
|
||||
else:
|
||||
audio_tensor, sample_rate = engine.generate_dialogue(
|
||||
dialogue_text=dialogue_text,
|
||||
speaker_references=normalized_refs,
|
||||
language=params.get("language", "auto"),
|
||||
duration_tokens=params.get("duration_tokens"),
|
||||
instruction=params.get("instruction"),
|
||||
quality=params.get("quality"),
|
||||
sound_event=params.get("sound_event"),
|
||||
ambient_sound=params.get("ambient_sound"),
|
||||
seed=int(params.get("seed", 0) or 0),
|
||||
audio_temperature=float(params.get("audio_temperature", params.get("temperature", 1.1))),
|
||||
audio_top_p=float(params.get("audio_top_p", params.get("top_p", 0.9))),
|
||||
audio_top_k=int(params.get("audio_top_k", params.get("top_k", 50))),
|
||||
audio_repetition_penalty=float(
|
||||
params.get("audio_repetition_penalty", params.get("repetition_penalty", 1.1))
|
||||
),
|
||||
max_new_tokens=int(params.get("max_new_tokens", 4096)),
|
||||
)
|
||||
|
||||
if sample_rate != self.SAMPLE_RATE:
|
||||
raise RuntimeError(f"MOSS-TTSD returned unexpected sample rate {sample_rate}; expected {self.SAMPLE_RATE}")
|
||||
|
||||
duration = self.audio_cache._calculate_duration(audio_tensor, "moss_tts")
|
||||
self.audio_cache.cache_audio(cache_key, audio_tensor, duration)
|
||||
|
||||
if audio_tensor.dim() == 1:
|
||||
audio_tensor = audio_tensor.unsqueeze(0)
|
||||
elif audio_tensor.dim() == 3:
|
||||
audio_tensor = audio_tensor.squeeze(0)
|
||||
|
||||
return {
|
||||
"waveform": audio_tensor,
|
||||
"sample_rate": self.SAMPLE_RATE,
|
||||
"character": "native_multispeaker",
|
||||
"text": dialogue_text,
|
||||
}
|
||||
|
||||
def cleanup(self):
|
||||
self._last_config = None
|
||||
@@ -1,299 +0,0 @@
|
||||
"""
|
||||
OmniVoice engine adapter.
|
||||
|
||||
Translates TTS Audio Suite processor calls into the official OmniVoice runtime.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from typing import Any, Dict, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
|
||||
current_dir = os.path.dirname(__file__)
|
||||
engines_dir = os.path.dirname(current_dir)
|
||||
project_root = os.path.dirname(engines_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
from utils.audio.audio_hash import generate_stable_audio_component
|
||||
from utils.audio.cache import get_audio_cache
|
||||
from utils.models.factory_config import ModelLoadConfig
|
||||
from utils.models.language_mapper import resolve_language_alias
|
||||
from utils.voice.reference import effective_voice_audio
|
||||
|
||||
|
||||
class OmniVoiceEngineAdapter:
|
||||
"""Adapter for official OmniVoice inference."""
|
||||
|
||||
SAMPLE_RATE = 24000
|
||||
|
||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||
self.config = config.copy() if config else {}
|
||||
self.audio_cache = get_audio_cache()
|
||||
self._last_config: Optional[ModelLoadConfig] = None
|
||||
self._load_signature = None
|
||||
|
||||
def update_config(self, new_config: Dict[str, Any]):
|
||||
self.config = new_config.copy() if new_config else {}
|
||||
|
||||
def _build_load_signature(self) -> Tuple[Any, ...]:
|
||||
return (
|
||||
self.config.get("model_variant", "OmniVoice"),
|
||||
self.config.get("device", "auto"),
|
||||
self.config.get("dtype", "auto"),
|
||||
)
|
||||
|
||||
def load_model(
|
||||
self,
|
||||
model_variant: str,
|
||||
device: str = "auto",
|
||||
dtype: str = "auto",
|
||||
):
|
||||
from utils.models.unified_model_interface import unified_model_interface
|
||||
|
||||
config = ModelLoadConfig(
|
||||
engine_name="omnivoice",
|
||||
model_type="tts",
|
||||
model_name=model_variant,
|
||||
model_path=model_variant,
|
||||
device=device,
|
||||
additional_params={
|
||||
"dtype": dtype,
|
||||
},
|
||||
)
|
||||
self._last_config = config
|
||||
unified_model_interface.load_model(config)
|
||||
|
||||
def _ensure_model_loaded(self):
|
||||
signature = self._build_load_signature()
|
||||
if signature == self._load_signature and self._last_config is not None:
|
||||
return
|
||||
|
||||
self.load_model(
|
||||
model_variant=self.config.get("model_variant", "OmniVoice"),
|
||||
device=self.config.get("device", "auto"),
|
||||
dtype=self.config.get("dtype", "auto"),
|
||||
)
|
||||
self._load_signature = signature
|
||||
|
||||
def _get_engine(self):
|
||||
if self._last_config is None:
|
||||
self._ensure_model_loaded()
|
||||
from utils.models.unified_model_interface import unified_model_interface
|
||||
|
||||
return unified_model_interface.load_model(self._last_config)
|
||||
|
||||
@staticmethod
|
||||
def _seed_everything(seed: int):
|
||||
if int(seed or 0) <= 0:
|
||||
return
|
||||
torch.manual_seed(int(seed))
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed_all(int(seed))
|
||||
if hasattr(torch, "xpu") and torch.xpu.is_available():
|
||||
torch.xpu.manual_seed_all(int(seed))
|
||||
|
||||
@staticmethod
|
||||
def _first_non_none(*values):
|
||||
for value in values:
|
||||
if value is None:
|
||||
continue
|
||||
if isinstance(value, str) and not value.strip():
|
||||
continue
|
||||
return value
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _normalize_waveform_tensor(waveform: torch.Tensor) -> torch.Tensor:
|
||||
if waveform.dim() == 3:
|
||||
waveform = waveform.squeeze(0)
|
||||
if waveform.dim() == 1:
|
||||
waveform = waveform.unsqueeze(0)
|
||||
return waveform.detach().float().cpu()
|
||||
|
||||
def _extract_voice_reference(
|
||||
self,
|
||||
voice_ref: Optional[Dict[str, Any]],
|
||||
) -> Tuple[Optional[Union[str, Tuple[torch.Tensor, int]]], str, str]:
|
||||
if not voice_ref or not isinstance(voice_ref, dict):
|
||||
return None, "", "default_voice"
|
||||
|
||||
prompt_text = (
|
||||
voice_ref.get("reference_text")
|
||||
or voice_ref.get("prompt_text")
|
||||
or voice_ref.get("text")
|
||||
or ""
|
||||
).strip()
|
||||
|
||||
ref_audio = effective_voice_audio(voice_ref)
|
||||
|
||||
if ref_audio is None:
|
||||
return None, prompt_text, "default_voice"
|
||||
|
||||
if isinstance(ref_audio, str):
|
||||
return ref_audio, prompt_text, generate_stable_audio_component(audio_file_path=ref_audio)
|
||||
|
||||
if isinstance(ref_audio, dict) and "waveform" in ref_audio:
|
||||
waveform = self._normalize_waveform_tensor(ref_audio["waveform"])
|
||||
sample_rate = int(ref_audio.get("sample_rate", self.SAMPLE_RATE))
|
||||
audio_dict = {"waveform": waveform, "sample_rate": sample_rate}
|
||||
return (
|
||||
(waveform, sample_rate),
|
||||
prompt_text,
|
||||
generate_stable_audio_component(reference_audio=audio_dict),
|
||||
)
|
||||
|
||||
if torch.is_tensor(ref_audio):
|
||||
waveform = self._normalize_waveform_tensor(ref_audio)
|
||||
sample_rate = int(voice_ref.get("sample_rate", self.SAMPLE_RATE))
|
||||
audio_dict = {"waveform": waveform, "sample_rate": sample_rate}
|
||||
return (
|
||||
(waveform, sample_rate),
|
||||
prompt_text,
|
||||
generate_stable_audio_component(reference_audio=audio_dict),
|
||||
)
|
||||
|
||||
raise TypeError(f"Unsupported OmniVoice reference type: {type(ref_audio)}")
|
||||
|
||||
@staticmethod
|
||||
def _normalize_language(language: Optional[str]) -> Optional[str]:
|
||||
if language is None:
|
||||
return None
|
||||
|
||||
normalized = str(language).strip()
|
||||
if not normalized:
|
||||
return None
|
||||
|
||||
lowered = normalized.lower()
|
||||
if lowered in {"auto", "none"}:
|
||||
return None
|
||||
|
||||
resolved = resolve_language_alias(normalized)
|
||||
|
||||
# OmniVoice uses the base ISO code for Portuguese rather than regional
|
||||
# variants accepted by other suite engines.
|
||||
if resolved in {"pt", "pt-br", "pt-pt"}:
|
||||
return "pt"
|
||||
|
||||
if resolved and resolved.lower() != lowered:
|
||||
return resolved
|
||||
return normalized
|
||||
|
||||
def generate_single(
|
||||
self,
|
||||
text: str,
|
||||
voice_ref: Optional[Dict[str, Any]],
|
||||
seed: int = 0,
|
||||
enable_audio_cache: bool = True,
|
||||
character_name: Optional[str] = None,
|
||||
) -> torch.Tensor:
|
||||
stripped = (text or "").strip()
|
||||
if not stripped:
|
||||
return torch.zeros(1, 0, dtype=torch.float32)
|
||||
|
||||
self._ensure_model_loaded()
|
||||
engine = self._get_engine()
|
||||
|
||||
ref_audio, prompt_text, audio_component = self._extract_voice_reference(voice_ref)
|
||||
language = self._normalize_language(self.config.get("language", "Auto"))
|
||||
instruct = str(self.config.get("instruct", "") or "").strip() or None
|
||||
model_variant = self.config.get("model_variant", "OmniVoice")
|
||||
device = self.config.get("device", "auto")
|
||||
dtype = self.config.get("dtype", "auto")
|
||||
num_step = int(self.config.get("num_steps", 32))
|
||||
guidance_scale = float(self.config.get("guidance_scale", 2.0))
|
||||
t_shift = float(self.config.get("t_shift", 0.1))
|
||||
speed = float(self.config.get("speed", 1.0))
|
||||
duration_value = float(self.config.get("duration", 0.0))
|
||||
duration = duration_value if duration_value > 0 else None
|
||||
layer_penalty_factor = float(self.config.get("layer_penalty_factor", 5.0))
|
||||
position_temperature = float(self.config.get("position_temperature", 5.0))
|
||||
class_temperature = float(self.config.get("class_temperature", 0.0))
|
||||
denoise = bool(self.config.get("denoise", True))
|
||||
preprocess_prompt = bool(self.config.get("preprocess_prompt", True))
|
||||
postprocess_output = bool(self.config.get("postprocess_output", True))
|
||||
audio_chunk_duration = float(self.config.get("audio_chunk_duration", 15.0))
|
||||
audio_chunk_threshold = float(self.config.get("audio_chunk_threshold", 30.0))
|
||||
|
||||
cache_key = None
|
||||
if enable_audio_cache:
|
||||
cache_key = self.audio_cache.generate_cache_key(
|
||||
"omnivoice",
|
||||
text=stripped,
|
||||
audio_component=audio_component,
|
||||
prompt_text=prompt_text,
|
||||
model_variant=model_variant,
|
||||
language=language or "none",
|
||||
instruct=instruct or "",
|
||||
num_step=num_step,
|
||||
guidance_scale=guidance_scale,
|
||||
t_shift=t_shift,
|
||||
speed=speed,
|
||||
duration=duration if duration is not None else "auto",
|
||||
layer_penalty_factor=layer_penalty_factor,
|
||||
position_temperature=position_temperature,
|
||||
class_temperature=class_temperature,
|
||||
denoise=denoise,
|
||||
preprocess_prompt=preprocess_prompt,
|
||||
postprocess_output=postprocess_output,
|
||||
audio_chunk_duration=audio_chunk_duration,
|
||||
audio_chunk_threshold=audio_chunk_threshold,
|
||||
seed=int(seed or 0),
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
character=character_name or "narrator",
|
||||
)
|
||||
cached_audio = self.audio_cache.get_cached_audio(cache_key)
|
||||
if cached_audio:
|
||||
print(f"💾 Using cached OmniVoice audio for '{character_name or 'narrator'}': '{stripped[:30]}...'")
|
||||
return cached_audio[0]
|
||||
|
||||
voice_clone_prompt = None
|
||||
if ref_audio is not None:
|
||||
if not prompt_text:
|
||||
raise ValueError(
|
||||
"OmniVoice voice cloning requires reference text. "
|
||||
"Direct audio-only narrator input is unsupported. "
|
||||
"Use Character Voices or a narrator voice file with matching reference text."
|
||||
)
|
||||
voice_clone_prompt = engine.create_voice_clone_prompt(
|
||||
ref_audio=ref_audio,
|
||||
ref_text=prompt_text,
|
||||
preprocess_prompt=preprocess_prompt,
|
||||
)
|
||||
|
||||
self._seed_everything(seed)
|
||||
|
||||
result = engine.generate(
|
||||
text=stripped,
|
||||
language=language,
|
||||
voice_clone_prompt=voice_clone_prompt,
|
||||
instruct=instruct,
|
||||
duration=duration,
|
||||
speed=speed,
|
||||
num_step=num_step,
|
||||
guidance_scale=guidance_scale,
|
||||
t_shift=t_shift,
|
||||
denoise=denoise,
|
||||
preprocess_prompt=preprocess_prompt,
|
||||
postprocess_output=postprocess_output,
|
||||
layer_penalty_factor=layer_penalty_factor,
|
||||
position_temperature=position_temperature,
|
||||
class_temperature=class_temperature,
|
||||
audio_chunk_duration=audio_chunk_duration,
|
||||
audio_chunk_threshold=audio_chunk_threshold,
|
||||
)
|
||||
|
||||
audio_tensor = result[0]
|
||||
if not isinstance(audio_tensor, torch.Tensor):
|
||||
audio_tensor = torch.tensor(audio_tensor, dtype=torch.float32)
|
||||
audio_tensor = audio_tensor.detach().float().cpu()
|
||||
if audio_tensor.dim() == 1:
|
||||
audio_tensor = audio_tensor.unsqueeze(0)
|
||||
|
||||
if enable_audio_cache and cache_key:
|
||||
duration_seconds = self.audio_cache._calculate_duration(audio_tensor, "omnivoice")
|
||||
self.audio_cache.cache_audio(cache_key, audio_tensor, duration_seconds)
|
||||
|
||||
return audio_tensor
|
||||
@@ -1,843 +0,0 @@
|
||||
"""
|
||||
Qwen3-TTS Engine Adapter
|
||||
|
||||
Provides standardized interface for Qwen3-TTS integration with TTS Audio Suite.
|
||||
Handles intelligent model selection, parameter mapping, voice references, and caching.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import torch
|
||||
from typing import Dict, Any, Optional, List, Union
|
||||
|
||||
# Add project root to path
|
||||
current_dir = os.path.dirname(__file__)
|
||||
engines_dir = os.path.dirname(current_dir)
|
||||
project_root = os.path.dirname(engines_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
from engines.qwen3_tts.qwen3_tts import Qwen3TTSEngine
|
||||
from utils.text.pause_processor import PauseTagProcessor
|
||||
from utils.audio.cache import get_audio_cache
|
||||
from utils.voice.reference import effective_voice_audio
|
||||
import folder_paths
|
||||
|
||||
|
||||
class Qwen3TTSEngineAdapter:
|
||||
"""
|
||||
Adapter for Qwen3-TTS engine providing unified interface compatibility.
|
||||
|
||||
Handles:
|
||||
- Intelligent model selection (CustomVoice/VoiceDesign/Base)
|
||||
- Parameter mapping between unified interface and Qwen3-TTS
|
||||
- Voice reference processing (Base model)
|
||||
- Pause tag processing and timing
|
||||
- Caching integration
|
||||
- Model management
|
||||
"""
|
||||
|
||||
def __init__(self, node_instance):
|
||||
"""
|
||||
Initialize the Qwen3-TTS adapter.
|
||||
|
||||
Args:
|
||||
node_instance: Parent node instance for context
|
||||
"""
|
||||
self.node = node_instance
|
||||
# DO NOT store engine reference - always query unified interface
|
||||
# Storing references causes stale CPU-bound engines when models are unloaded
|
||||
self.audio_cache = get_audio_cache()
|
||||
self.current_model_type = None # Track which model variant is loaded
|
||||
self._last_config = None # Store last ModelLoadConfig for engine retrieval
|
||||
|
||||
# Job-level timing tracker (persists across blocks)
|
||||
self.job_tracker = None
|
||||
|
||||
def start_job(self, total_blocks: int, block_texts: list):
|
||||
"""
|
||||
Initialize job tracker for time estimation across all blocks.
|
||||
|
||||
Args:
|
||||
total_blocks: Number of blocks to process
|
||||
block_texts: List of text lengths for each block (for weighted progress)
|
||||
"""
|
||||
import time
|
||||
total_text = sum(block_texts)
|
||||
self.job_tracker = {
|
||||
'start_time': time.time(),
|
||||
'total_blocks': total_blocks,
|
||||
'blocks_completed': 0,
|
||||
'block_texts': block_texts,
|
||||
'total_text': total_text,
|
||||
'text_completed': 0,
|
||||
'current_block_text': 0
|
||||
}
|
||||
|
||||
def set_current_block(self, block_idx: int):
|
||||
"""Set the current block being processed."""
|
||||
if self.job_tracker:
|
||||
self.job_tracker['current_block_text'] = self.job_tracker['block_texts'][block_idx]
|
||||
|
||||
def complete_block(self):
|
||||
"""Mark current block as completed."""
|
||||
if self.job_tracker:
|
||||
self.job_tracker['blocks_completed'] += 1
|
||||
self.job_tracker['text_completed'] += self.job_tracker['current_block_text']
|
||||
# Track total tokens from completed blocks
|
||||
self.job_tracker['total_tokens'] = self.job_tracker.get('total_tokens', 0) + self.job_tracker.get('current_block_tokens', 0)
|
||||
|
||||
def end_job(self):
|
||||
"""Clear job tracker after job completion."""
|
||||
self.job_tracker = None
|
||||
|
||||
def _determine_model_type(self, context: Dict[str, Any]) -> str:
|
||||
"""
|
||||
Auto-select model variant based on context.
|
||||
|
||||
Priority:
|
||||
1. Voice Designer node → VoiceDesign
|
||||
2. Preset voice selected → CustomVoice
|
||||
3. Default → Base (zero-shot)
|
||||
|
||||
Args:
|
||||
context: Context dict with node_type, voice_preset, etc.
|
||||
|
||||
Returns:
|
||||
Model type string: "CustomVoice", "VoiceDesign", or "Base"
|
||||
"""
|
||||
# Explicit engine selection is authoritative for refactored workflows.
|
||||
explicit_model_type = context.get("model_type")
|
||||
if explicit_model_type in {"Base", "CustomVoice", "VoiceDesign"}:
|
||||
return explicit_model_type
|
||||
|
||||
# Legacy voice designer context.
|
||||
if context.get("node_type") == "voice_designer":
|
||||
return "VoiceDesign"
|
||||
|
||||
# Priority 2: Preset voice selected
|
||||
voice_preset = context.get("voice_preset")
|
||||
if voice_preset and voice_preset != "None (Zero-shot / Custom)":
|
||||
return "CustomVoice"
|
||||
|
||||
# Priority 3: Default to Base (zero-shot)
|
||||
return "Base"
|
||||
|
||||
def load_base_model(self,
|
||||
model_path: str,
|
||||
device: str = "auto",
|
||||
dtype: str = "auto",
|
||||
model_size: str = "1.7B",
|
||||
attn_implementation: str = "auto",
|
||||
runtime_mode: str = "main_environment",
|
||||
runtime_profile: Optional[str] = None,
|
||||
context: Optional[Dict[str, Any]] = None,
|
||||
use_torch_compile: bool = False,
|
||||
use_cuda_graphs: bool = False,
|
||||
compile_mode: str = "reduce-overhead"):
|
||||
"""
|
||||
Load Qwen3-TTS engine via unified interface with intelligent model selection.
|
||||
|
||||
Args:
|
||||
model_path: Model identifier (local:ModelName or ModelName for auto-download)
|
||||
device: Target device (auto/cuda/cpu)
|
||||
dtype: Model precision (bfloat16/float16/float32/auto)
|
||||
model_size: Model size (0.6B/1.7B)
|
||||
attn_implementation: Attention mechanism (auto/flash_attention_2/sdpa/eager)
|
||||
context: Context dict for model type determination
|
||||
use_torch_compile: Enable torch.compile optimization
|
||||
use_cuda_graphs: Enable CUDA graph capture
|
||||
compile_mode: torch.compile mode (reduce-overhead/max-autotune/default)
|
||||
"""
|
||||
from utils.models.unified_model_interface import unified_model_interface, ModelLoadConfig
|
||||
from utils.device import resolve_torch_device
|
||||
|
||||
# Determine which model variant to load
|
||||
context = context or {}
|
||||
model_type = self._determine_model_type(context)
|
||||
|
||||
# VoiceDesign only supports 1.7B
|
||||
if model_type == "VoiceDesign" and model_size == "0.6B":
|
||||
model_size = "1.7B"
|
||||
print("⚠️ VoiceDesign requires 1.7B model, auto-switching from 0.6B")
|
||||
|
||||
# Keep the canonical model name separate from a local: model path.
|
||||
model_name = context.get("model_name") or f"Qwen3-TTS-12Hz-{model_size}-{model_type}"
|
||||
|
||||
# Track current model type (unified interface handles unloading automatically)
|
||||
self.current_model_type = model_type
|
||||
|
||||
# Loading message is printed by unified_model_interface
|
||||
|
||||
# Create config for unified interface
|
||||
config = ModelLoadConfig(
|
||||
engine_name="qwen3_tts",
|
||||
model_type="tts",
|
||||
model_name=model_name,
|
||||
model_path=model_path if model_path else model_name,
|
||||
device=resolve_torch_device(device),
|
||||
runtime_mode=runtime_mode,
|
||||
runtime_profile=runtime_profile,
|
||||
additional_params={
|
||||
"dtype": dtype,
|
||||
"attn_implementation": attn_implementation,
|
||||
"use_torch_compile": use_torch_compile,
|
||||
"use_cuda_graphs": use_cuda_graphs,
|
||||
"compile_mode": compile_mode
|
||||
}
|
||||
)
|
||||
|
||||
# Store config for later engine retrieval (DO NOT store engine reference)
|
||||
self._last_config = config
|
||||
|
||||
# Load model via unified interface (but don't store the reference)
|
||||
engine = unified_model_interface.load_model(config)
|
||||
|
||||
# Enable streaming optimizations if requested
|
||||
if use_torch_compile or use_cuda_graphs:
|
||||
try:
|
||||
engine.enable_streaming_optimizations(
|
||||
use_compile=use_torch_compile,
|
||||
use_cuda_graphs=use_cuda_graphs,
|
||||
compile_mode=compile_mode
|
||||
)
|
||||
print(f"✅ Qwen3-TTS: Streaming optimizations enabled (compile={use_torch_compile}, cuda_graphs={use_cuda_graphs})")
|
||||
except Exception as e:
|
||||
print(f"⚠️ Failed to enable streaming optimizations: {e}")
|
||||
|
||||
def _get_engine(self):
|
||||
"""
|
||||
Get current engine from unified interface.
|
||||
|
||||
CRITICAL: Always query unified interface instead of storing references.
|
||||
This prevents stale CPU-bound engine references when models are unloaded.
|
||||
|
||||
Returns:
|
||||
Current Qwen3-TTS engine from unified interface
|
||||
|
||||
Raises:
|
||||
RuntimeError: If no engine is loaded or config not set
|
||||
"""
|
||||
if self._last_config is None:
|
||||
raise RuntimeError("No model loaded. Call load_base_model() first.")
|
||||
|
||||
# Query unified interface for current engine
|
||||
from utils.models.unified_model_interface import unified_model_interface
|
||||
engine = unified_model_interface.load_model(self._last_config)
|
||||
|
||||
if engine is None:
|
||||
raise RuntimeError("Engine not found in unified interface cache. Model may have been unloaded.")
|
||||
|
||||
return engine
|
||||
|
||||
def _ensure_engine_on_device(self, target_device: str = "auto"):
|
||||
"""
|
||||
Ensure engine is loaded and on the correct device.
|
||||
|
||||
CRITICAL: After unified_model_interface unloads a model variant, the adapter
|
||||
may still hold a reference to a CPU-bound engine. This method detects that
|
||||
and triggers a reload.
|
||||
|
||||
Args:
|
||||
target_device: Target device (auto/cuda/cpu)
|
||||
"""
|
||||
if self.engine is None:
|
||||
raise RuntimeError("Engine not loaded. Call load_base_model() first.")
|
||||
|
||||
# Check if engine has been moved to CPU (likely by model manager unload)
|
||||
# The engine wrapper has a _model attribute that's the actual Qwen3TTSModel
|
||||
if hasattr(self.engine, '_model') and hasattr(self.engine._model, 'model'):
|
||||
actual_model = self.engine._model.model
|
||||
if hasattr(actual_model, 'parameters'):
|
||||
try:
|
||||
first_param = next(actual_model.parameters())
|
||||
current_device = str(first_param.device)
|
||||
|
||||
# If model is on CPU but we want CUDA, it was likely unloaded
|
||||
from utils.device import resolve_torch_device
|
||||
expected_device = resolve_torch_device(target_device)
|
||||
|
||||
if current_device == "cpu" and expected_device != "cpu":
|
||||
print(f"⚠️ Qwen3-TTS engine is on CPU but {expected_device} requested")
|
||||
print(f" This usually means the model was unloaded by unified interface")
|
||||
print(f" Engine reference is stale - please check adapter state")
|
||||
raise RuntimeError(
|
||||
"Qwen3-TTS engine is on wrong device (CPU). "
|
||||
"This indicates the cached processor has a stale engine reference. "
|
||||
"The processor should have been reloaded by update_config()."
|
||||
)
|
||||
except StopIteration:
|
||||
pass
|
||||
|
||||
def generate_with_pause_tags(self,
|
||||
text: str,
|
||||
voice_ref: Optional[Dict[str, Any]],
|
||||
params: Dict[str, Any],
|
||||
process_pauses: bool = True,
|
||||
character_name: Optional[str] = None) -> torch.Tensor:
|
||||
"""
|
||||
Generate speech with pause tag processing (TTS Suite integration).
|
||||
|
||||
Args:
|
||||
text: Input text (may contain [pause:X] tags)
|
||||
voice_ref: Voice reference dict (for Base model) or None (for CustomVoice/VoiceDesign)
|
||||
params: Generation parameters (language, speaker, instruct, etc.)
|
||||
process_pauses: Whether to process pause tags
|
||||
character_name: Character name for logging
|
||||
|
||||
Returns:
|
||||
Generated audio tensor [1, samples] at 24000 Hz
|
||||
"""
|
||||
# Get current engine from unified interface (always fresh reference)
|
||||
engine = self._get_engine()
|
||||
|
||||
# Extract seed from params and set global torch seed for reproducibility
|
||||
seed = params.get('seed', 0)
|
||||
if seed > 0:
|
||||
torch.manual_seed(seed)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed_all(seed)
|
||||
|
||||
# Check if text has pause tags
|
||||
if process_pauses and PauseTagProcessor.has_pause_tags(text):
|
||||
# Process pause tags and generate segments
|
||||
return self._generate_with_pauses(text, voice_ref, params, character_name, engine)
|
||||
else:
|
||||
# Direct generation without pause processing
|
||||
return self._generate_direct(text, voice_ref, params, character_name, engine)
|
||||
|
||||
def _generate_direct(self,
|
||||
text: str,
|
||||
voice_ref: Optional[Dict[str, Any]],
|
||||
params: Dict[str, Any],
|
||||
character_name: Optional[str] = None,
|
||||
engine = None) -> torch.Tensor:
|
||||
"""
|
||||
Direct generation without pause processing.
|
||||
|
||||
Args:
|
||||
text: Input text (clean, no pause tags)
|
||||
voice_ref: Voice reference dict (for Base model)
|
||||
params: Generation parameters
|
||||
character_name: Character name for logging
|
||||
|
||||
Returns:
|
||||
Audio tensor [1, samples]
|
||||
"""
|
||||
# Get engine if not provided
|
||||
if engine is None:
|
||||
engine = self._get_engine()
|
||||
|
||||
# Determine generation method based on current model type
|
||||
if self.current_model_type == "CustomVoice":
|
||||
return self._generate_custom_voice(text, params, character_name, engine)
|
||||
elif self.current_model_type == "VoiceDesign":
|
||||
return self._generate_voice_design(text, params, character_name, engine)
|
||||
else: # Base
|
||||
return self._generate_voice_clone(text, voice_ref, params, character_name, engine)
|
||||
|
||||
def _generate_custom_voice(self,
|
||||
text: str,
|
||||
params: Dict[str, Any],
|
||||
character_name: Optional[str] = None,
|
||||
engine = None) -> torch.Tensor:
|
||||
"""
|
||||
Generate with CustomVoice model using preset speakers.
|
||||
|
||||
Args:
|
||||
text: Input text
|
||||
params: Must include 'speaker' and 'language'
|
||||
character_name: Character name for logging
|
||||
|
||||
Returns:
|
||||
Audio tensor [1, samples]
|
||||
"""
|
||||
speaker = params.get('voice_preset') or params.get('speaker', 'Vivian')
|
||||
language = params.get('language', 'Auto')
|
||||
instruct = params.get('instruct') # Optional instruction
|
||||
|
||||
# Generate cache key (include attn_implementation since it affects output quality)
|
||||
cache_key = self.audio_cache.generate_cache_key(
|
||||
'qwen3_tts',
|
||||
text=text,
|
||||
model_type='CustomVoice',
|
||||
speaker=speaker,
|
||||
language=language,
|
||||
instruct=instruct,
|
||||
top_k=params.get('top_k', 50),
|
||||
top_p=params.get('top_p', 1.0),
|
||||
temperature=params.get('temperature', 0.9),
|
||||
repetition_penalty=params.get('repetition_penalty', 1.05),
|
||||
max_new_tokens=params.get('max_new_tokens', 2048),
|
||||
seed=params.get('seed', 0),
|
||||
model_size=params.get('model_size', '1.7B'),
|
||||
attn_implementation=params.get('attn_implementation', 'auto'),
|
||||
character=character_name or 'narrator'
|
||||
)
|
||||
|
||||
# Check cache
|
||||
cached_audio = self.audio_cache.get_cached_audio(cache_key)
|
||||
if cached_audio:
|
||||
char_desc = character_name or 'narrator'
|
||||
print(f"💾 Using cached Qwen3-TTS audio for '{char_desc}': '{text[:30]}...'")
|
||||
return cached_audio[0]
|
||||
|
||||
# Create ComfyUI progress bar and convert to transformers streamer
|
||||
max_new_tokens = params.get('max_new_tokens', 2048)
|
||||
progress_bar = self._create_progress_bar(max_new_tokens, text)
|
||||
|
||||
# Convert progress_bar to transformers streamer for bundled model
|
||||
streamer = None
|
||||
if progress_bar is not None:
|
||||
from engines.qwen3_tts.progress_callback import Qwen3TTSProgressStreamer
|
||||
streamer = Qwen3TTSProgressStreamer(max_new_tokens, progress_bar, text_input=text)
|
||||
|
||||
# Get engine if not provided
|
||||
if engine is None:
|
||||
engine = self._get_engine()
|
||||
|
||||
# Generate
|
||||
wavs, sr = engine.generate_custom_voice(
|
||||
text=text,
|
||||
language=language,
|
||||
speaker=speaker,
|
||||
instruct=instruct,
|
||||
top_k=params.get('top_k', 50),
|
||||
top_p=params.get('top_p', 1.0),
|
||||
temperature=params.get('temperature', 0.9),
|
||||
repetition_penalty=params.get('repetition_penalty', 1.05),
|
||||
max_new_tokens=max_new_tokens,
|
||||
streamer=streamer # Use transformers streamer kwarg
|
||||
)
|
||||
|
||||
# Convert to tensor
|
||||
audio_tensor = self._convert_output_to_tensor(wavs, sr)
|
||||
|
||||
# Cache the result
|
||||
duration = self.audio_cache._calculate_duration(audio_tensor, 'qwen3_tts')
|
||||
self.audio_cache.cache_audio(cache_key, audio_tensor, duration)
|
||||
|
||||
return audio_tensor
|
||||
|
||||
def _generate_voice_design(self,
|
||||
text: str,
|
||||
params: Dict[str, Any],
|
||||
character_name: Optional[str] = None,
|
||||
engine = None) -> torch.Tensor:
|
||||
"""
|
||||
Generate with VoiceDesign model using text description.
|
||||
|
||||
Args:
|
||||
text: Input text
|
||||
params: Must include 'instruct' (voice description) and 'language'
|
||||
character_name: Character name for logging
|
||||
|
||||
Returns:
|
||||
Audio tensor [1, samples]
|
||||
"""
|
||||
instruct = params.get('instruct') or params.get('voice_description')
|
||||
if not instruct:
|
||||
raise ValueError("VoiceDesign requires 'instruct' parameter with voice description")
|
||||
|
||||
language = params.get('language', 'Auto')
|
||||
|
||||
# Generate cache key (include attn_implementation since it affects output quality)
|
||||
cache_key = self.audio_cache.generate_cache_key(
|
||||
'qwen3_tts',
|
||||
text=text,
|
||||
model_type='VoiceDesign',
|
||||
instruct=instruct,
|
||||
language=language,
|
||||
top_k=params.get('top_k', 50),
|
||||
top_p=params.get('top_p', 1.0),
|
||||
temperature=params.get('temperature', 0.9),
|
||||
repetition_penalty=params.get('repetition_penalty', 1.05),
|
||||
max_new_tokens=params.get('max_new_tokens', 2048),
|
||||
seed=params.get('seed', 0),
|
||||
model_size=params.get('model_size', '1.7B'),
|
||||
attn_implementation=params.get('attn_implementation', 'auto'),
|
||||
character=character_name or 'narrator'
|
||||
)
|
||||
|
||||
# Check cache
|
||||
cached_audio = self.audio_cache.get_cached_audio(cache_key)
|
||||
if cached_audio:
|
||||
char_desc = character_name or 'narrator'
|
||||
print(f"💾 Using cached Qwen3-TTS VoiceDesign audio for '{char_desc}': '{text[:30]}...'")
|
||||
return cached_audio[0]
|
||||
|
||||
# Create ComfyUI progress bar and convert to transformers streamer
|
||||
max_new_tokens = params.get('max_new_tokens', 2048)
|
||||
progress_bar = self._create_progress_bar(max_new_tokens, text)
|
||||
|
||||
# Convert progress_bar to transformers streamer for bundled model
|
||||
streamer = None
|
||||
if progress_bar is not None:
|
||||
from engines.qwen3_tts.progress_callback import Qwen3TTSProgressStreamer
|
||||
streamer = Qwen3TTSProgressStreamer(max_new_tokens, progress_bar, text_input=text)
|
||||
|
||||
# Get engine if not provided
|
||||
if engine is None:
|
||||
engine = self._get_engine()
|
||||
|
||||
# Generate
|
||||
wavs, sr = engine.generate_voice_design(
|
||||
text=text,
|
||||
language=language,
|
||||
instruct=instruct,
|
||||
top_k=params.get('top_k', 50),
|
||||
top_p=params.get('top_p', 1.0),
|
||||
temperature=params.get('temperature', 0.9),
|
||||
repetition_penalty=params.get('repetition_penalty', 1.05),
|
||||
max_new_tokens=max_new_tokens,
|
||||
streamer=streamer # Use transformers streamer kwarg
|
||||
)
|
||||
|
||||
# Convert to tensor
|
||||
audio_tensor = self._convert_output_to_tensor(wavs, sr)
|
||||
|
||||
# Cache the result
|
||||
duration = self.audio_cache._calculate_duration(audio_tensor, 'qwen3_tts')
|
||||
self.audio_cache.cache_audio(cache_key, audio_tensor, duration)
|
||||
|
||||
return audio_tensor
|
||||
|
||||
def _generate_voice_clone(self,
|
||||
text: str,
|
||||
voice_ref: Optional[Dict[str, Any]],
|
||||
params: Dict[str, Any],
|
||||
character_name: Optional[str] = None,
|
||||
engine = None) -> torch.Tensor:
|
||||
"""
|
||||
Generate with Base model using zero-shot voice cloning.
|
||||
|
||||
Args:
|
||||
text: Input text
|
||||
voice_ref: Voice reference dict with audio and optional transcript
|
||||
params: Generation parameters
|
||||
character_name: Character name for logging
|
||||
|
||||
Returns:
|
||||
Audio tensor [1, samples]
|
||||
"""
|
||||
# Extract voice reference (returns converted audio + original for cache hashing)
|
||||
ref_audio, ref_text, voice_x_vector_only, ref_audio_original = self._extract_voice_reference(voice_ref)
|
||||
|
||||
language = params.get('language', 'Auto')
|
||||
# Use voice_ref x_vector_only_mode if provided, otherwise use params
|
||||
x_vector_only = voice_x_vector_only if voice_x_vector_only else params.get('x_vector_only_mode', False)
|
||||
|
||||
# Warn user when x_vector_only mode is forced or ignoring available ref text
|
||||
if x_vector_only and ref_text:
|
||||
print(f"⚠️ Qwen3-TTS: x_vector_only mode enabled - ignoring reference text (using speaker embedding only)")
|
||||
|
||||
# Generate cache key using voice_ref dict (contains waveform + sample_rate)
|
||||
# This ensures different voices generate different cache keys
|
||||
from utils.audio.audio_hash import generate_stable_audio_component
|
||||
if voice_ref and isinstance(voice_ref, dict):
|
||||
if isinstance(ref_audio_original, str):
|
||||
audio_component = generate_stable_audio_component(audio_file_path=ref_audio_original)
|
||||
elif isinstance(ref_audio_original, dict) and "waveform" in ref_audio_original:
|
||||
audio_component = generate_stable_audio_component(reference_audio=ref_audio_original)
|
||||
elif torch.is_tensor(ref_audio_original):
|
||||
audio_component = generate_stable_audio_component(reference_audio={
|
||||
"waveform": ref_audio_original,
|
||||
"sample_rate": voice_ref.get("sample_rate", 24000),
|
||||
})
|
||||
else:
|
||||
audio_component = "default_voice"
|
||||
elif ref_audio_original is not None and isinstance(ref_audio_original, str):
|
||||
# File path case (voice_ref is None but ref_audio_original extracted)
|
||||
audio_component = generate_stable_audio_component(audio_file_path=ref_audio_original)
|
||||
else:
|
||||
audio_component = "default_voice"
|
||||
|
||||
cache_key = self.audio_cache.generate_cache_key(
|
||||
'qwen3_tts',
|
||||
text=text,
|
||||
model_type='Base',
|
||||
audio_component=audio_component,
|
||||
ref_text=ref_text,
|
||||
language=language,
|
||||
x_vector_only=x_vector_only,
|
||||
top_k=params.get('top_k', 50),
|
||||
top_p=params.get('top_p', 1.0),
|
||||
temperature=params.get('temperature', 0.9),
|
||||
repetition_penalty=params.get('repetition_penalty', 1.05),
|
||||
max_new_tokens=params.get('max_new_tokens', 2048),
|
||||
seed=params.get('seed', 0),
|
||||
model_size=params.get('model_size', '1.7B'),
|
||||
attn_implementation=params.get('attn_implementation', 'auto'),
|
||||
character=character_name or 'narrator'
|
||||
)
|
||||
|
||||
# Check cache
|
||||
cached_audio = self.audio_cache.get_cached_audio(cache_key)
|
||||
if cached_audio:
|
||||
char_desc = character_name or 'narrator'
|
||||
print(f"💾 Using cached Qwen3-TTS VoiceClone audio for '{char_desc}': '{text[:30]}...'")
|
||||
return cached_audio[0]
|
||||
|
||||
# Create ComfyUI progress bar and convert to transformers streamer
|
||||
max_new_tokens = params.get('max_new_tokens', 2048)
|
||||
progress_bar = self._create_progress_bar(max_new_tokens, text)
|
||||
|
||||
# Convert progress_bar to transformers streamer for bundled model
|
||||
streamer = None
|
||||
if progress_bar is not None:
|
||||
from engines.qwen3_tts.progress_callback import Qwen3TTSProgressStreamer
|
||||
streamer = Qwen3TTSProgressStreamer(max_new_tokens, progress_bar, text_input=text)
|
||||
|
||||
# Get engine if not provided
|
||||
if engine is None:
|
||||
engine = self._get_engine()
|
||||
|
||||
# Generate
|
||||
wavs, sr = engine.generate_voice_clone(
|
||||
text=text,
|
||||
language=language,
|
||||
ref_audio=ref_audio,
|
||||
ref_text=ref_text,
|
||||
x_vector_only_mode=x_vector_only,
|
||||
top_k=params.get('top_k', 50),
|
||||
top_p=params.get('top_p', 1.0),
|
||||
temperature=params.get('temperature', 0.9),
|
||||
repetition_penalty=params.get('repetition_penalty', 1.05),
|
||||
max_new_tokens=max_new_tokens,
|
||||
streamer=streamer # Use transformers streamer kwarg
|
||||
)
|
||||
|
||||
# Convert to tensor
|
||||
audio_tensor = self._convert_output_to_tensor(wavs, sr)
|
||||
|
||||
# Cache the result
|
||||
duration = self.audio_cache._calculate_duration(audio_tensor, 'qwen3_tts')
|
||||
self.audio_cache.cache_audio(cache_key, audio_tensor, duration)
|
||||
|
||||
return audio_tensor
|
||||
|
||||
def _generate_with_pauses(self,
|
||||
text: str,
|
||||
voice_ref: Optional[Dict[str, Any]],
|
||||
params: Dict[str, Any],
|
||||
character_name: Optional[str] = None,
|
||||
engine = None) -> torch.Tensor:
|
||||
"""
|
||||
Generate with pause tag processing.
|
||||
|
||||
Args:
|
||||
text: Text with pause tags ([pause:2], [pause:1.5s], [pause:500ms])
|
||||
voice_ref: Voice reference dict (for Base model)
|
||||
params: Generation parameters
|
||||
character_name: Character name for logging
|
||||
|
||||
Returns:
|
||||
Combined audio tensor [1, samples]
|
||||
"""
|
||||
# Parse pause tags - returns list of tuples: ('text', content) or ('pause', duration_seconds)
|
||||
segments, clean_text = PauseTagProcessor.parse_pause_tags(text)
|
||||
|
||||
print(f"🎵 Processing {len(segments)} pause-delimited segments for '{character_name or 'unknown'}'")
|
||||
|
||||
audio_parts = []
|
||||
sample_rate = 24000 # Qwen3-TTS native sample rate
|
||||
|
||||
# Get engine if not provided
|
||||
if engine is None:
|
||||
engine = self._get_engine()
|
||||
|
||||
for segment_type, content in segments:
|
||||
if segment_type == 'text':
|
||||
# Generate audio for text segment
|
||||
audio_tensor = self._generate_direct(content, voice_ref, params, character_name, engine)
|
||||
|
||||
# Ensure correct shape [1, samples] (2D)
|
||||
if audio_tensor.dim() == 1:
|
||||
audio_tensor = audio_tensor.unsqueeze(0) # [samples] -> [1, samples]
|
||||
elif audio_tensor.dim() == 3:
|
||||
audio_tensor = audio_tensor.squeeze(0) # [1, 1, samples] -> [1, samples]
|
||||
|
||||
audio_parts.append(audio_tensor)
|
||||
|
||||
elif segment_type == 'pause':
|
||||
# Create silence (content is duration in seconds)
|
||||
silence = PauseTagProcessor.create_silence_segment(
|
||||
content, sample_rate
|
||||
)
|
||||
audio_parts.append(silence)
|
||||
|
||||
# Concatenate all parts
|
||||
if not audio_parts:
|
||||
return torch.zeros(1, 0)
|
||||
|
||||
combined_audio = torch.cat(audio_parts, dim=-1)
|
||||
return combined_audio
|
||||
|
||||
def _extract_voice_reference(self, voice_ref: Optional[Dict[str, Any]]) -> tuple:
|
||||
"""
|
||||
Extract voice reference audio and text from voice_ref dict.
|
||||
|
||||
Args:
|
||||
voice_ref: Voice reference dict (from voice discovery)
|
||||
|
||||
Returns:
|
||||
Tuple of (ref_audio, ref_text, x_vector_only_mode)
|
||||
- ref_audio: Audio in format expected by Qwen3-TTS (string path or tuple (np.ndarray, sr))
|
||||
- ref_text: Reference transcript or None
|
||||
- x_vector_only_mode: Whether to use speaker embedding only (True) or ICL mode (False)
|
||||
"""
|
||||
if voice_ref is None or not isinstance(voice_ref, dict):
|
||||
# For Base model without reference, use default behavior
|
||||
return None, None, False
|
||||
|
||||
# Extract reference audio (multiple possible keys)
|
||||
ref_audio_original = effective_voice_audio(voice_ref)
|
||||
|
||||
# Extract reference text (multiple possible keys)
|
||||
ref_text = (voice_ref.get('prompt_text') or
|
||||
voice_ref.get('reference_text') or
|
||||
voice_ref.get('text', ''))
|
||||
|
||||
# Extract x_vector_only_mode flag (default to False for ICL mode)
|
||||
x_vector_only_mode = voice_ref.get('x_vector_only_mode', False)
|
||||
|
||||
# Convert ref_audio to format expected by Qwen3-TTS
|
||||
ref_audio_converted = self._convert_audio_to_qwen_format(ref_audio_original, voice_ref)
|
||||
|
||||
# Return: (converted_audio, ref_text, x_vector_mode, original_audio_for_cache)
|
||||
return ref_audio_converted, ref_text, x_vector_only_mode, ref_audio_original
|
||||
|
||||
def _convert_audio_to_qwen_format(self, ref_audio, voice_ref: Dict[str, Any]):
|
||||
"""
|
||||
Convert audio reference to format expected by Qwen3-TTS bundled implementation.
|
||||
|
||||
Qwen3-TTS accepts:
|
||||
- String: File path
|
||||
- Tuple[np.ndarray, int]: (waveform, sample_rate)
|
||||
|
||||
Args:
|
||||
ref_audio: Audio reference (can be torch.Tensor, dict, string, etc.)
|
||||
voice_ref: Full voice reference dict (may contain sample_rate)
|
||||
|
||||
Returns:
|
||||
Audio in Qwen3-TTS expected format
|
||||
"""
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
# Already a string path - return as-is
|
||||
if isinstance(ref_audio, str):
|
||||
return ref_audio
|
||||
|
||||
# ComfyUI audio dict format
|
||||
if isinstance(ref_audio, dict) and 'waveform' in ref_audio:
|
||||
waveform = ref_audio['waveform']
|
||||
sample_rate = ref_audio.get('sample_rate', 24000)
|
||||
|
||||
# Convert torch tensor to numpy
|
||||
if isinstance(waveform, torch.Tensor):
|
||||
# ComfyUI format is [batch, channels, samples]
|
||||
# Convert to [samples] mono
|
||||
waveform_np = waveform.squeeze().cpu().numpy()
|
||||
if waveform_np.ndim > 1:
|
||||
# Average channels if stereo
|
||||
waveform_np = np.mean(waveform_np, axis=0)
|
||||
return (waveform_np.astype(np.float32), int(sample_rate))
|
||||
|
||||
# Raw torch.Tensor (from voice_ref['waveform'])
|
||||
if isinstance(ref_audio, torch.Tensor):
|
||||
# Get sample rate from voice_ref
|
||||
sample_rate = voice_ref.get('sample_rate', 24000)
|
||||
|
||||
# Convert to numpy
|
||||
waveform_np = ref_audio.squeeze().cpu().numpy()
|
||||
if waveform_np.ndim > 1:
|
||||
# Average channels if stereo
|
||||
waveform_np = np.mean(waveform_np, axis=0)
|
||||
return (waveform_np.astype(np.float32), int(sample_rate))
|
||||
|
||||
# Tuple (waveform, sr) - ensure numpy
|
||||
if isinstance(ref_audio, tuple) and len(ref_audio) == 2:
|
||||
waveform, sr = ref_audio
|
||||
if isinstance(waveform, torch.Tensor):
|
||||
waveform = waveform.squeeze().cpu().numpy()
|
||||
if waveform.ndim > 1:
|
||||
waveform = np.mean(waveform, axis=0)
|
||||
return (waveform.astype(np.float32), int(sr))
|
||||
|
||||
# None or unsupported
|
||||
return ref_audio
|
||||
|
||||
def _create_progress_bar(self, max_tokens: int, text: str = ""):
|
||||
"""
|
||||
Create ComfyUI progress bar for generation tracking with accurate estimation.
|
||||
|
||||
Args:
|
||||
max_tokens: Maximum tokens to generate
|
||||
text: Input text for estimating actual token count
|
||||
|
||||
Returns:
|
||||
Progress bar instance or None if ComfyUI utils not available
|
||||
"""
|
||||
try:
|
||||
import comfy.utils
|
||||
|
||||
# Estimate actual tokens based on text length (same heuristic as progress streamer)
|
||||
# Rough heuristic: ~1.5 tokens per character for TTS
|
||||
if text:
|
||||
estimated_tokens = int(len(text) * 1.5)
|
||||
# Cap estimate to max_tokens
|
||||
progress_total = min(estimated_tokens, max_tokens)
|
||||
else:
|
||||
progress_total = max_tokens
|
||||
|
||||
return comfy.utils.ProgressBar(progress_total)
|
||||
except (ImportError, AttributeError):
|
||||
return None # ComfyUI progress not available
|
||||
|
||||
def _convert_output_to_tensor(self, wavs, sample_rate: int) -> torch.Tensor:
|
||||
"""
|
||||
Convert Qwen3-TTS output to tensor format.
|
||||
|
||||
Args:
|
||||
wavs: Output from engine (numpy array or list of numpy arrays)
|
||||
sample_rate: Sample rate (should be 24000)
|
||||
|
||||
Returns:
|
||||
Audio tensor [1, samples]
|
||||
"""
|
||||
import numpy as np
|
||||
|
||||
# Handle list of arrays (batch generation)
|
||||
if isinstance(wavs, list):
|
||||
if len(wavs) == 1:
|
||||
wavs = wavs[0]
|
||||
else:
|
||||
# Concatenate multiple outputs
|
||||
wavs = np.concatenate(wavs, axis=-1)
|
||||
|
||||
# Convert numpy to tensor
|
||||
if isinstance(wavs, np.ndarray):
|
||||
audio_tensor = torch.from_numpy(wavs).float()
|
||||
elif torch.is_tensor(wavs):
|
||||
audio_tensor = wavs.float()
|
||||
else:
|
||||
raise ValueError(f"Unsupported audio output type: {type(wavs)}")
|
||||
|
||||
# Ensure 2D shape [1, samples]
|
||||
if audio_tensor.dim() == 1:
|
||||
audio_tensor = audio_tensor.unsqueeze(0) # [samples] -> [1, samples]
|
||||
elif audio_tensor.dim() == 3:
|
||||
audio_tensor = audio_tensor.squeeze(0) # [1, 1, samples] -> [1, samples]
|
||||
|
||||
return audio_tensor
|
||||
|
||||
def cleanup(self):
|
||||
"""
|
||||
Clean up resources.
|
||||
|
||||
Note: We don't auto-unload here - let the model stay in VRAM for reuse.
|
||||
Only explicit unload (via button) should trigger model deletion.
|
||||
"""
|
||||
# Don't auto-unload - let the model stay in VRAM for reuse
|
||||
pass
|
||||
@@ -84,23 +84,20 @@ class RVCEngineAdapter:
|
||||
print(f"Error loading RVC models: {e}")
|
||||
raise e
|
||||
|
||||
def convert_voice(
|
||||
self,
|
||||
audio_input: Union[torch.Tensor, np.ndarray, tuple],
|
||||
rvc_model: dict = None,
|
||||
pitch_shift: int = 0,
|
||||
def convert_voice(
|
||||
self,
|
||||
audio_input: Union[torch.Tensor, np.ndarray, tuple],
|
||||
rvc_model: dict = None,
|
||||
pitch_shift: int = 0,
|
||||
index_rate: float = 0.75,
|
||||
rms_mix_rate: float = 0.25,
|
||||
protect: float = 0.25,
|
||||
f0_method: str = 'rmvpe',
|
||||
f0_autotune: bool = False,
|
||||
resample_sr: int = 0,
|
||||
crepe_hop_length: int = 160,
|
||||
filter_radius: int = 3,
|
||||
use_cache: bool = True,
|
||||
batch_size: int = 1,
|
||||
**kwargs
|
||||
) -> Tuple[np.ndarray, int]:
|
||||
protect: float = 0.25,
|
||||
f0_method: str = 'rmvpe',
|
||||
f0_autotune: bool = False,
|
||||
resample_sr: int = 0,
|
||||
crepe_hop_length: int = 160,
|
||||
**kwargs
|
||||
) -> Tuple[np.ndarray, int]:
|
||||
"""
|
||||
Perform voice conversion using RVC.
|
||||
|
||||
@@ -112,13 +109,10 @@ class RVCEngineAdapter:
|
||||
rms_mix_rate: RMS mixing rate (0.0-1.0)
|
||||
protect: Consonant protection (0.0-0.5)
|
||||
f0_method: Pitch extraction method
|
||||
f0_autotune: Enable autotune
|
||||
resample_sr: Resample rate (0 for no resampling)
|
||||
crepe_hop_length: Crepe hop length (16-512)
|
||||
filter_radius: Harvest smoothing radius
|
||||
use_cache: Reuse cached loaded models where possible
|
||||
batch_size: Crepe-family pitch batch size hint
|
||||
**kwargs: Additional parameters
|
||||
f0_autotune: Enable autotune
|
||||
resample_sr: Resample rate (0 for no resampling)
|
||||
crepe_hop_length: Crepe hop length (16-512)
|
||||
**kwargs: Additional parameters
|
||||
|
||||
Returns:
|
||||
Tuple of (converted_audio, sample_rate)
|
||||
@@ -162,16 +156,12 @@ class RVCEngineAdapter:
|
||||
f0_up_key=pitch_shift,
|
||||
f0_method=f0_method,
|
||||
index_rate=index_rate,
|
||||
protect=protect,
|
||||
rms_mix_rate=rms_mix_rate,
|
||||
resample_sr=resample_sr,
|
||||
f0_autotune=f0_autotune,
|
||||
crepe_hop_length=crepe_hop_length,
|
||||
filter_radius=filter_radius,
|
||||
use_cache=use_cache,
|
||||
batch_size=batch_size,
|
||||
**kwargs # Pass through hubert_path and other engine parameters
|
||||
)
|
||||
protect=protect,
|
||||
rms_mix_rate=rms_mix_rate,
|
||||
resample_sr=resample_sr,
|
||||
f0_autotune=f0_autotune,
|
||||
crepe_hop_length=crepe_hop_length
|
||||
)
|
||||
|
||||
if result is None:
|
||||
raise ValueError("RVC conversion failed - no output from minimal wrapper")
|
||||
@@ -197,47 +187,28 @@ class RVCEngineAdapter:
|
||||
raise e
|
||||
|
||||
def _find_hubert_model(self) -> Optional[str]:
|
||||
"""Find available Hubert model (respects extra_model_paths.yaml)."""
|
||||
"""Find available Hubert model."""
|
||||
try:
|
||||
# Common Hubert model names
|
||||
import folder_paths
|
||||
models_dir = folder_paths.models_dir
|
||||
|
||||
# Common Hubert model names and locations
|
||||
hubert_candidates = [
|
||||
"content-vec-best.safetensors",
|
||||
"hubert_base.pt",
|
||||
"chinese-hubert-base.pt",
|
||||
"chinese-wav2vec2-base.pt"
|
||||
]
|
||||
|
||||
# Search in extra_model_paths.yaml first
|
||||
try:
|
||||
from utils.models.extra_paths import get_all_tts_model_paths
|
||||
all_tts_paths = get_all_tts_model_paths('TTS')
|
||||
|
||||
for base_path in all_tts_paths:
|
||||
for model_name in hubert_candidates:
|
||||
# Check TTS/hubert/ and TTS/ root
|
||||
search_paths = [
|
||||
os.path.join(base_path, "hubert", model_name),
|
||||
os.path.join(base_path, model_name)
|
||||
]
|
||||
for model_path in search_paths:
|
||||
if os.path.exists(model_path):
|
||||
print(f"📄 Found Hubert model: {model_name}")
|
||||
return model_path
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fallback to hardcoded ComfyUI paths
|
||||
import folder_paths
|
||||
models_dir = folder_paths.models_dir
|
||||
|
||||
|
||||
for model_name in hubert_candidates:
|
||||
# Try TTS path first, then legacy locations
|
||||
search_paths = [
|
||||
os.path.join(models_dir, "TTS", "hubert", model_name),
|
||||
os.path.join(models_dir, "TTS", model_name),
|
||||
os.path.join(models_dir, "hubert", model_name), # Legacy
|
||||
os.path.join(models_dir, model_name) # Legacy - direct in models/
|
||||
]
|
||||
|
||||
|
||||
for model_path in search_paths:
|
||||
if os.path.exists(model_path):
|
||||
print(f"📄 Found Hubert model: {model_name}")
|
||||
@@ -299,14 +270,11 @@ class RVCEngineAdapter:
|
||||
'index_rate': 0.75,
|
||||
'rms_mix_rate': 0.25,
|
||||
'protect': 0.25,
|
||||
'f0_method': 'rmvpe',
|
||||
'f0_autotune': False,
|
||||
'resample_sr': 0,
|
||||
'crepe_hop_length': 160,
|
||||
'filter_radius': 3,
|
||||
'use_cache': True,
|
||||
'batch_size': 1,
|
||||
}
|
||||
'f0_method': 'rmvpe',
|
||||
'f0_autotune': False,
|
||||
'resample_sr': 0,
|
||||
'crepe_hop_length': 160
|
||||
}
|
||||
|
||||
def validate_parameters(self, **params) -> Dict[str, Any]:
|
||||
"""
|
||||
@@ -343,27 +311,14 @@ class RVCEngineAdapter:
|
||||
if params['resample_sr'] in valid_rates:
|
||||
validated['resample_sr'] = int(params['resample_sr'])
|
||||
|
||||
if 'crepe_hop_length' in params:
|
||||
validated['crepe_hop_length'] = max(16, min(512, int(params['crepe_hop_length'])))
|
||||
|
||||
if 'filter_radius' in params:
|
||||
validated['filter_radius'] = max(0, min(7, int(params['filter_radius'])))
|
||||
|
||||
if 'use_cache' in params:
|
||||
validated['use_cache'] = bool(params['use_cache'])
|
||||
|
||||
if 'batch_size' in params:
|
||||
validated['batch_size'] = max(1, min(32, int(params['batch_size'])))
|
||||
|
||||
return validated
|
||||
if 'crepe_hop_length' in params:
|
||||
validated['crepe_hop_length'] = max(16, min(512, int(params['crepe_hop_length'])))
|
||||
|
||||
return validated
|
||||
|
||||
def cleanup(self):
|
||||
"""Clean up loaded models and free memory."""
|
||||
self._loaded_models.clear()
|
||||
if hasattr(self, 'rvc_engine'):
|
||||
self.rvc_engine.cleanup()
|
||||
|
||||
# CRITICAL: Clear minimal wrapper cache - this is where RVC models are actually stored
|
||||
minimal_wrapper.clear_model_cache()
|
||||
|
||||
print("RVC adapter cleanup completed")
|
||||
print("RVC adapter cleanup completed")
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user