Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 diodiogod
|
73a282406e |
@@ -44,7 +44,7 @@ The project code is MIT. Model weights carry their own licenses:
|
||||
Echo-TTS CC-BY-NC-SA-4.0 No
|
||||
Fish Audio S2 Pro Fish Audio Research License No
|
||||
Dots TTS Apache-2.0 Yes
|
||||
TADA MIT code / Llama 3.2 weights Conditional
|
||||
Audio8 TTS Apache-2.0 Yes
|
||||
DramaBox LTX-2 Community License Conditional
|
||||
OmniVoice Apache-2.0 Yes
|
||||
MOSS-TTS Apache-2.0 Yes
|
||||
|
||||
+6
-7
@@ -17,7 +17,7 @@
|
||||
**Key architectural rules:**
|
||||
- Chunking happens in the **processor**, not the adapter (`generate_single()` on adapter = raw single call)
|
||||
- Runtime routing happens through `ModelLoadConfig.runtime_mode` + `runtime_profile`, not ad-hoc subprocess calls
|
||||
- Shared runtime workers are currently used for fragile engine families such as VibeVoice, Qwen3-TTS / ASR, Granite forced alignment, Higgs Audio 2, and TADA. Engines that support the modern stack run natively in the main Transformers 5 environment.
|
||||
- Shared runtime workers are currently used for fragile engine families such as VibeVoice, Qwen3-TTS / ASR, Granite forced alignment, Higgs Audio 2, and Audio8 TTS. Engines that support the modern stack run natively in the main Transformers 5 environment.
|
||||
- YAML (`docs/Dev reports/tts_audio_suite_engines.yaml`) is source of truth for engine doc tables → run `python3 scripts/generate_engine_tables.py --readme` to regenerate
|
||||
- Auxiliary YAML (`docs/Dev reports/tts_audio_suite_aux_models.yaml`) is source of truth for helper/post-process model docs → run `python3 scripts/generate_aux_model_docs.py`
|
||||
- All models download to `ComfyUI/models/TTS/<model-name>/`
|
||||
@@ -44,14 +44,14 @@
|
||||
| Echo-TTS | `echo_tts_adapter.py` | `nodes/echo_tts/echo_tts_processor.py` | `echo_tts_srt_processor.py` | `echo_tts_engine_node.py` |
|
||||
| Fish Audio S2 Pro | `fish_audio_s2_adapter.py` | `nodes/fish_audio_s2/fish_audio_s2_processor.py` | `fish_audio_s2_srt_processor.py` | `fish_audio_s2_engine_node.py` |
|
||||
| Dots TTS | `dots_tts_adapter.py` | `nodes/dots_tts/dots_tts_processor.py` | `dots_tts_srt_processor.py` | `dots_tts_engine_node.py` |
|
||||
| TADA | `tada_adapter.py` | `nodes/tada/tada_processor.py` | `tada_srt_processor.py` | `tada_engine_node.py` |
|
||||
| Audio8 TTS | `audio8_tts_adapter.py` | `nodes/audio8_tts/audio8_tts_processor.py` | `audio8_tts_srt_processor.py` | `audio8_tts_engine_node.py` |
|
||||
| DramaBox | `dramabox_adapter.py` | `nodes/dramabox/dramabox_processor.py` | `dramabox_srt_processor.py` | `dramabox_engine_node.py` |
|
||||
| OmniVoice | `omnivoice_adapter.py` | `nodes/omnivoice/omnivoice_processor.py` | `omnivoice_srt_processor.py` | `omnivoice_engine_node.py` |
|
||||
| MOSS-SoundEffect v2 | `moss_soundeffect_v2_adapter.py` | — | — | `moss_soundeffect_v2_engine_node.py` |
|
||||
| RVC | — | `engines/rvc/` | — | `rvc_engine_node.py` |
|
||||
|
||||
**Engine implementations live in:**
|
||||
- `engines/chatterbox/`, `engines/chatterbox_official_23lang/`, `engines/f5tts/`, `engines/higgs_audio/`, `engines/higgs_audio_v3/`, `engines/vibevoice_engine/`, `engines/step_audio_editx/`, `engines/cosyvoice/`, `engines/qwen3_tts/`, `engines/qwen3_asr/`, `engines/moss_tts/`, `engines/moss_soundeffect_v2/`, `engines/granite_asr/`, `engines/echo_tts/`, `engines/fish_audio_s2/`, `engines/dots_tts/`, `engines/tada/`, `engines/dramabox/`, `engines/omnivoice/`, `engines/rvc/`
|
||||
- `engines/chatterbox/`, `engines/chatterbox_official_23lang/`, `engines/f5tts/`, `engines/higgs_audio/`, `engines/higgs_audio_v3/`, `engines/vibevoice_engine/`, `engines/step_audio_editx/`, `engines/cosyvoice/`, `engines/qwen3_tts/`, `engines/qwen3_asr/`, `engines/moss_tts/`, `engines/moss_soundeffect_v2/`, `engines/granite_asr/`, `engines/echo_tts/`, `engines/fish_audio_s2/`, `engines/dots_tts/`, `engines/audio8_tts/`, `engines/dramabox/`, `engines/omnivoice/`, `engines/rvc/`
|
||||
|
||||
## Documentation Files
|
||||
|
||||
@@ -68,7 +68,6 @@
|
||||
- `OMNIVOICE_TAGS_GUIDE.md` - OmniVoice native non-verbal tags and pronunciation overrides
|
||||
- `MOSS_TTS_PROMPT_FIELDS_GUIDE.md` - Official MOSS whole-segment prompt fields and inline `<>` translation limits
|
||||
- `DRAMABOX_PROMPTING_GUIDE.md` - DramaBox expressive scene prompts, voice references, controls, hardware, and license
|
||||
- `TADA_ENGINE_GUIDE.md` - TADA models, gated tokenizer setup, exact-transcript voice cloning, native controls, limits, and licensing
|
||||
- `COSYVOICE3_TAGS_GUIDE.md` - CosyVoice3 native paralinguistic tags
|
||||
- `CHATTERBOX_V2_SPECIAL_TOKENS.md` - ChatterBox v2 emotion tokens
|
||||
- `IndexTTS2_Emotion_Control_Guide.md` - IndexTTS-2 vector, text, audio, and blended emotion controls
|
||||
@@ -147,14 +146,14 @@
|
||||
- `launcher.py` - runtime bootstrap, venv creation, Windows toolchain env setup
|
||||
- `session.py`, `protocol.py` - JSONL worker transport and message protocol
|
||||
- `bootstrap.py` - shared runtime bootstrap helpers
|
||||
- `vibevoice_proxy.py`, `qwen3_tts_proxy.py`, `qwen3_asr_proxy.py`, `higgs_audio_proxy.py`, `tada_proxy.py` - parent-process proxies
|
||||
- `workers/` - worker subprocess entrypoints for VibeVoice, Qwen3-TTS, Qwen3-ASR/aligner, Higgs Audio, and TADA
|
||||
- `vibevoice_proxy.py`, `qwen3_tts_proxy.py`, `qwen3_asr_proxy.py`, `higgs_audio_proxy.py`, `audio8_tts_proxy.py` - parent-process proxies
|
||||
- `workers/` - worker subprocess entrypoints for VibeVoice, Qwen3-TTS, Qwen3-ASR/aligner, Higgs Audio, and Audio8 TTS
|
||||
- Current shared legacy T4 runtime profile is reused by:
|
||||
- VibeVoice / Kugel
|
||||
- Qwen3-TTS
|
||||
- Qwen3-ASR and Granite's optional Qwen forced aligner
|
||||
- Higgs Audio 2
|
||||
- TADA
|
||||
- Audio8 TTS
|
||||
|
||||
### Audio (`utils/audio/`)
|
||||
- `processing.py` - Tensor manipulation, normalization, format conversion
|
||||
|
||||
@@ -41,7 +41,7 @@ Subtitle workflows are still a core focus: the suite can transcribe to SRT, rebu
|
||||
| **Echo-TTS** | 🇺🇸 | ~5.3GB + ~1.8GB | Diffusion-based (~30s best), Force Speaker KV (speaker drift control) |
|
||||
| **Fish Audio S2 Pro** | 🌐 80+ languages | ~10.3GB / ~8.0GB | Free-form sub-word emotion/prosody tags, Native multi-speaker and multi-turn dialogue with dynamic speaker references |
|
||||
| **Dots TTS** | 🇺🇸🇨🇳🇩🇪🇪🇸🇫🇷🇮🇹 +13 | ~6GB | Official auto language detect / language control, SOAR and MeanFlow distilled variants |
|
||||
| **TADA** | 🇺🇸🇨🇳🇩🇪🇪🇸🇫🇷🇮🇹 +4 | ~6.1GB / ~11GB with selected codec assets | Voice cloning, Reference-conditioned expression transfer |
|
||||
| **Audio8 TTS** | 🌐 11 recommended languages | ~2.39 GiB | Reference-free TTS + zero-shot/cross-lingual cloning, 44.1kHz output with sampling or greedy decoding |
|
||||
| **DramaBox** | 🇺🇸 | ~16.4GB | Expressive scene prompting and stage directions, Native and SRT-aware duration targeting |
|
||||
| **OmniVoice** | 🌐 600+ languages | ~3.7GB | Inline non-verbal tags and pronunciation overrides, Reference-free voice design |
|
||||
| **MOSS-TTS** | 🇺🇸🇨🇳🇩🇪🇪🇸🇫🇷🇮🇹 +18 | ~8.5GB tokenizer + ~6.1GB/17GB/18GB model | Reference-free voice design with MOSS-VoiceGenerator, Native 1-5 speaker TTSD dialogue |
|
||||
@@ -1528,7 +1528,7 @@ For offline/manual setup:
|
||||
| Granite ASR | `ComfyUI/models/TTS/granite_asr/` | ✅ | Granite ASR models; plus adds native diarization/timestamps, optional Qwen forced aligner reused lazily for timestamps/SRT fallback |
|
||||
| Echo-TTS | `ComfyUI/models/TTS/echo-tts-base/` | ✅ | ~7.1GB total (base + dac); CC-BY-NC-SA |
|
||||
| Dots TTS | `ComfyUI/models/TTS/dots_tts/` | ✅ | Official base / soar / mf checkpoints with tokenizer, vocoder, speaker encoder |
|
||||
| TADA | `ComfyUI/models/TTS/tada/` | ✅ | Selected 1B or 3B-ML model plus language-specific codec assets and an exact hash-verified ungated Llama tokenizer redistribution |
|
||||
| Audio8 TTS | `ComfyUI/models/TTS/audio8_tts/Audio8-TTS-Preview-0.6b/` | ✅ | ~2.39 GiB official 0.6B Preview checkpoint; shared Transformers 4.57.3 runtime required because main 5.10.2 collapses voice cloning |
|
||||
| DramaBox | `ComfyUI/models/TTS/dramabox/DramaBox/` | ✅ | ~16.4GB download; fast mode roughly 24GB VRAM; experimental FP8 peaks on RTX 4090: staged ~15.1GB allocated, sequential ~11.7GB allocated / ~12.4GB reserved; conditional LTX-2 Community License |
|
||||
| Fish Audio S2 Pro | `ComfyUI/models/TTS/fish_audio_s2_pro/` | ✅ | Official BF16 or optional community FP8 checkpoint; the official checkpoint can be quantized on load with BNB INT8/NF4; main T5 environment with process teardown for Clear VRAM; Fish Audio Research License |
|
||||
| OmniVoice | `ComfyUI/models/TTS/omnivoice/` | ✅ | Official OmniVoice model. Voice cloning in this suite requires explicit reference text. |
|
||||
|
||||
@@ -1339,12 +1339,14 @@ engines:
|
||||
speed_performance: { supported: "partial", notes: "Moderate; mf variant is faster" }
|
||||
reference_free_tts: { supported: true, notes: "(default speaker)" }
|
||||
|
||||
- id: tada
|
||||
name: TADA
|
||||
models: "TADA 1B (English), TADA 3B-ML"
|
||||
size: "~6.1GB / ~11GB with selected codec assets"
|
||||
license: "MIT code / Llama 3.2 weights"
|
||||
commercial: conditional
|
||||
- id: audio8_tts
|
||||
name: Audio8 TTS
|
||||
models: "Audio8 TTS Preview 0.6B"
|
||||
size: "~2.39 GiB"
|
||||
license: "Apache-2.0"
|
||||
commercial: true
|
||||
language_summary_full: "11 recommended languages (including Cantonese)"
|
||||
language_summary_compact: "🌐 11 recommended languages"
|
||||
|
||||
capabilities:
|
||||
tts: true
|
||||
@@ -1353,95 +1355,60 @@ engines:
|
||||
asr: false
|
||||
training: false
|
||||
|
||||
special_features:
|
||||
- "Zero-shot voice cloning"
|
||||
- "Reference-conditioned expression transfer"
|
||||
|
||||
readme_key_features:
|
||||
- "Voice cloning"
|
||||
- "Reference-conditioned expression transfer"
|
||||
|
||||
runtime_isolation:
|
||||
default_mode: "shared_runtime"
|
||||
supported_modes:
|
||||
- "shared_runtime"
|
||||
status: "implemented"
|
||||
main_environment: false
|
||||
shared_runtime: true
|
||||
dedicated_runtime: false
|
||||
runtime_profile: "vibevoice_transformers4_shared"
|
||||
notes: "The supported path uses the shared Transformers 4 runtime with hume-tada 0.1.9. TADA is incompatible with the suite's main Transformers 5 environment."
|
||||
notes: "Required for feature-complete Audio8 inference: main Transformers 5.10.2 collapsed the official voice-clone demo to 0.325s, while the shared Transformers 4.57.3 runtime generated 10.495s for the published 10.54s demo"
|
||||
|
||||
readme_key_features:
|
||||
- "Reference-free TTS + zero-shot/cross-lingual cloning"
|
||||
- "44.1kHz output with sampling or greedy decoding"
|
||||
|
||||
special_features:
|
||||
- "44.1kHz reference-free TTS and zero-shot/cross-lingual voice cloning"
|
||||
- "Exact reference transcript required when voice cloning"
|
||||
- "Greedy or sampled decoding with temperature, top-p, top-k, seed, and token-limit controls"
|
||||
- "Suite text chunking, character switching, and SRT timing/assembly"
|
||||
- "No explicit language selector; multilingual behavior is text-driven"
|
||||
- "No native emotion/style/speed controls, streaming, timestamps, dialogue/multi-speaker mode, or presets"
|
||||
- "Not compatible with Voice Designer: the official model has no instruction-conditioned voice-design mode"
|
||||
- "Unified TTS/SRT integration only; no special node or suite training integration"
|
||||
|
||||
model_sources:
|
||||
- component: "TADA 1B"
|
||||
source_name: "HumeAI/tada-1b"
|
||||
source_url: "https://huggingface.co/HumeAI/tada-1b"
|
||||
size: "~3.9GB"
|
||||
- component: "Audio8 TTS Preview 0.6B"
|
||||
source_name: "Audio8/Audio8-TTS-Preview-0.6b"
|
||||
source_url: "https://huggingface.co/Audio8/Audio8-TTS-Preview-0.6b"
|
||||
size: "~2.39 GiB"
|
||||
auto_download: true
|
||||
notes: "English-only model; downloaded only when selected"
|
||||
- component: "TADA 3B-ML"
|
||||
source_name: "HumeAI/tada-3b-ml"
|
||||
source_url: "https://huggingface.co/HumeAI/tada-3b-ml"
|
||||
size: "~8.9GB"
|
||||
auto_download: true
|
||||
notes: "Multilingual model; downloaded only when selected"
|
||||
- component: "TADA codec"
|
||||
source_name: "HumeAI/tada-codec"
|
||||
source_url: "https://huggingface.co/HumeAI/tada-codec"
|
||||
size: "~2.15GB per selected language"
|
||||
auto_download: true
|
||||
notes: "Downloads shared encoder/decoder plus only the aligner required by the selected language"
|
||||
- component: "Llama tokenizer"
|
||||
source_name: "onnx-community/Llama-3.2-1B"
|
||||
source_url: "https://huggingface.co/onnx-community/Llama-3.2-1B"
|
||||
size: "Tokenizer files only"
|
||||
auto_download: true
|
||||
notes: "Ungated redistribution; runtime files are verified against Meta's official Git blob hashes and remain under the Llama 3.2 Community License"
|
||||
- component: "Wav2Vec aligner configuration"
|
||||
source_name: "facebook/wav2vec2-large"
|
||||
source_url: "https://huggingface.co/facebook/wav2vec2-large"
|
||||
size: "Configuration file only"
|
||||
auto_download: true
|
||||
notes: "Downloads config.json only; no Wav2Vec model weights"
|
||||
notes: "Official 0.6B Preview checkpoint with bundled 44.1kHz codec; suite uses the shared Transformers 4.57.3 runtime because voice cloning is not compatible with main Transformers 5.10.2"
|
||||
|
||||
languages:
|
||||
en: { supported: true, flag: "🇺🇸", notes: "(1B and 3B-ML)" }
|
||||
zh: { supported: true, flag: "🇨🇳", notes: "(3B-ML; backend code `ch`)" }
|
||||
de: { supported: true, flag: "🇩🇪", notes: "(3B-ML)" }
|
||||
es: { supported: true, flag: "🇪🇸", notes: "(3B-ML)" }
|
||||
fr: { supported: true, flag: "🇫🇷", notes: "(3B-ML)" }
|
||||
it: { supported: true, flag: "🇮🇹", notes: "(3B-ML)" }
|
||||
ja: { supported: true, flag: "🇯🇵", notes: "(3B-ML)" }
|
||||
ko: { supported: false, flag: "🇰🇷", notes: "" }
|
||||
ru: { supported: false, flag: "🇷🇺", notes: "" }
|
||||
pt: { supported: true, flag: "🇵🇹", notes: "(3B-ML)" }
|
||||
pl: { supported: true, flag: "🇵🇱", notes: "(3B-ML)" }
|
||||
hi: { supported: false, flag: "🇮🇳", notes: "" }
|
||||
ar: { supported: true, flag: "🇦🇪", notes: "(3B-ML)" }
|
||||
tr: { supported: false, flag: "🇹🇷", notes: "" }
|
||||
th: { supported: false, flag: "🇹🇭", notes: "" }
|
||||
no: { supported: false, flag: "🇳🇴", notes: "" }
|
||||
vi: { supported: false, flag: "🇻🇳", notes: "" }
|
||||
hy: { supported: false, flag: "🇦🇲", notes: "" }
|
||||
ka: { supported: false, flag: "🇬🇪", notes: "" }
|
||||
da: { supported: false, flag: "🇩🇰", notes: "" }
|
||||
fi: { supported: false, flag: "🇫🇮", notes: "" }
|
||||
el: { supported: false, flag: "🇬🇷", notes: "" }
|
||||
he: { supported: false, flag: "🇮🇱", notes: "" }
|
||||
ms: { supported: false, flag: "🇲🇾", notes: "" }
|
||||
nl: { supported: false, flag: "🇳🇱", notes: "" }
|
||||
sv: { supported: false, flag: "🇸🇪", notes: "" }
|
||||
sw: { supported: false, flag: "🇰🇪", notes: "" }
|
||||
en: { supported: true, flag: "🇺🇸", notes: "" }
|
||||
zh: { supported: true, flag: "🇨🇳", notes: "(Chinese; upstream separately recommends Cantonese, folded into this matrix row)" }
|
||||
de: { supported: true, flag: "🇩🇪", notes: "" }
|
||||
es: { supported: true, flag: "🇪🇸", notes: "" }
|
||||
fr: { supported: true, flag: "🇫🇷", notes: "" }
|
||||
it: { supported: true, flag: "🇮🇹", notes: "" }
|
||||
ja: { supported: true, flag: "🇯🇵", notes: "" }
|
||||
ko: { supported: true, flag: "🇰🇷", notes: "" }
|
||||
pl: { supported: true, flag: "🇵🇱", notes: "" }
|
||||
nl: { supported: true, flag: "🇳🇱", notes: "" }
|
||||
|
||||
features:
|
||||
voice_cloning: { supported: true, notes: "Reference audio plus exact transcript" }
|
||||
reference_transcript: { requirement: required }
|
||||
native_multi_speaker: { supported: false, notes: "" }
|
||||
voice_conversion: { supported: false, notes: "" }
|
||||
voice_cloning: { supported: true, notes: "Zero-shot and cross-lingual; reference audio must be paired with its exact transcript" }
|
||||
reference_transcript: { requirement: conditional }
|
||||
native_multi_speaker: { supported: false, notes: "No native dialogue or multi-speaker interface; suite character switching generates speakers as separate segments" }
|
||||
voice_conversion: { supported: false, notes: "No separate voice-conversion mode" }
|
||||
asr_transcribe: { supported: false, notes: "" }
|
||||
emotion_control: { supported: false, notes: "No direct emotion control; expression may transfer from the reference audio" }
|
||||
native_long_form: { supported: false, notes: "Suite text chunking is used" }
|
||||
community_finetunes: { supported: false, notes: "No suite integration" }
|
||||
vram_efficient: { supported: "partial", notes: "1B is the lighter option; suite VRAM measurements are pending" }
|
||||
speed_performance: { supported: "partial", notes: "No suite benchmark; no streaming or batch inference" }
|
||||
reference_free_tts: { supported: false, notes: "This integration requires a reference voice and exact transcript" }
|
||||
emotion_control: { supported: false, notes: "No native emotion, style, or speed controls" }
|
||||
native_long_form: { supported: false, notes: "(uses suite text chunking and SRT assembly)" }
|
||||
community_finetunes: { supported: false, notes: "No community model or upstream SFT/training integration in the suite" }
|
||||
vram_efficient: { supported: "partial", notes: "~1.75 GiB model-plus-codec parameter memory at fp16/bf16 or ~3.5 GiB at float32/CPU, before KV cache, activations, and framework overhead; shared runtime reuses the main environment's PyTorch" }
|
||||
speed_performance: { supported: "partial", notes: "Live shared-runtime validation: 6.36s reference-free audio in 39.3s including load; 9.99s cloned audio in 36.7s warm; exact cache reuse in 56ms" }
|
||||
reference_free_tts: { supported: true, notes: "Reference audio is optional" }
|
||||
|
||||
- id: dramabox
|
||||
name: DramaBox
|
||||
@@ -2007,17 +1974,17 @@ table_notes:
|
||||
reference_transcript: >-
|
||||
Conditional means the transcript is required only for the specific mode:
|
||||
CosyVoice3 zero-shot, Qwen3-TTS full Base cloning, or MOSS-TTSD cloned-speaker
|
||||
dialogue. Higgs Audio 2, Higgs Audio v3, and Dots TTS accept matching text
|
||||
when provided but do not require it. TADA always requires an exact transcript
|
||||
for its reference audio in this integration.
|
||||
dialogue, and Audio8 TTS zero-shot/cross-lingual cloning. Higgs Audio 2,
|
||||
Higgs Audio v3, and Dots TTS accept matching text when provided but do not
|
||||
require it.
|
||||
language_support:
|
||||
- "**CosyVoice3 Chinese**: Includes 18+ dialects (Cantonese, Sichuan, Dongbei, Shanghai, etc.)"
|
||||
- "**Higgs Audio 2**: Trained on EN, ZH (Mandarin), KO, DE, ES (English majority) - 10M hours AudioVerse dataset"
|
||||
- "**Higgs Audio v3**: Official model card lists 102 languages total: 85 polished-production languages and 17 usable but less polished languages."
|
||||
- "**IndexTTS-2**: Trained on 55K+ hours - ZH, EN, JA primary."
|
||||
- "**MOSS-TTS**: Official list includes ZH, EN, DE, ES, FR, JA, IT, HU, KO, RU, FA, AR, PL, PT, CS, DA, SV, EL, TR; HU/FA/CS are not separate columns in this matrix."
|
||||
- "**TADA**: TADA 1B is English-only. TADA 3B-ML supports EN, AR, ZH, DE, ES, FR, IT, JA, PL, and PT; the upstream backend calls Chinese `ch`."
|
||||
- "**OmniVoice**: Official model supports 600+ languages; this matrix only shows the suite's comparison subset."
|
||||
- "**Audio8 TTS**: The Preview recommends Cantonese, Chinese, Dutch, English, French, German, Italian, Japanese, Korean, Polish, and Spanish. It has no explicit language selector; language behavior is text-driven. Cantonese is folded into the Chinese row in this matrix."
|
||||
- "**RVC**: Language-agnostic voice conversion/post-processing; it is marked supported for every language row."
|
||||
- "Qwen3-TTS: Supports pt-BR only with instruction (voice design or harcoded voices). Base can't do instructions, so it will always ouput pt-PT."
|
||||
|
||||
@@ -2086,10 +2053,10 @@ readme_model_download_table:
|
||||
primary_model_path: "ComfyUI/models/TTS/dots_tts/"
|
||||
auto_download: "✅"
|
||||
notes: "Official base / soar / mf checkpoints with tokenizer, vocoder, speaker encoder"
|
||||
- engine: "TADA"
|
||||
primary_model_path: "ComfyUI/models/TTS/tada/"
|
||||
- engine: "Audio8 TTS"
|
||||
primary_model_path: "ComfyUI/models/TTS/audio8_tts/Audio8-TTS-Preview-0.6b/"
|
||||
auto_download: "✅"
|
||||
notes: "Selected 1B or 3B-ML model plus language-specific codec assets and an exact hash-verified ungated Llama tokenizer redistribution"
|
||||
notes: "~2.39 GiB official 0.6B Preview checkpoint; shared Transformers 4.57.3 runtime required because main 5.10.2 collapses voice cloning"
|
||||
- engine: "DramaBox"
|
||||
primary_model_path: "ComfyUI/models/TTS/dramabox/DramaBox/"
|
||||
auto_download: "✅"
|
||||
@@ -2533,27 +2500,21 @@ model_layouts_markdown: |
|
||||
- Native sample rate is 48kHz.
|
||||
- Main-environment support works on Transformers 5; on Windows, `normalize_text` falls back to no-op if `WeTextProcessing` is unavailable.
|
||||
|
||||
## TADA
|
||||
## Audio8 TTS
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/tada/
|
||||
├── TADA-1B/ # downloaded only if selected
|
||||
├── TADA-3B-ML/ # downloaded only if selected
|
||||
├── tada-codec/
|
||||
│ ├── encoder/
|
||||
│ ├── decoder/
|
||||
│ ├── aligner[-<language>]/ # selected language only
|
||||
│ └── wav2vec2-large/
|
||||
└── llama-3.2-1b-tokenizer/ # tokenizer files only
|
||||
ComfyUI/models/TTS/audio8_tts/
|
||||
└── Audio8-TTS-Preview-0.6b/
|
||||
└── (13 required official runtime files, including codec weights)
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- TADA runs in the shared Transformers 4 runtime with `hume-tada==0.1.9`.
|
||||
- The selected 1B or 3B-ML model, shared codec components, and selected language aligner download on demand.
|
||||
- The Llama tokenizer downloads from an ungated redistribution and is verified against Meta's official Git blob hashes. It remains under the Llama 3.2 Community License.
|
||||
- Reference audio and its exact transcript are required by this integration.
|
||||
- Native output is mono 24kHz.
|
||||
- The official `Audio8/Audio8-TTS-Preview-0.6b` checkpoint is about 2.39 GiB and includes the 44.1kHz neural codec.
|
||||
- The 0.6B model plus codec use about 1.75 GiB for parameters at fp16/bf16 or about 3.5 GiB at float32/CPU, before KV cache, activations, and framework overhead.
|
||||
- The suite uses the existing `vibevoice_transformers4_shared` runtime with Transformers 4.57.3. Main Transformers 5.10.2 is not compatible with Audio8 voice cloning: the official demo collapsed to 0.325s, while the shared runtime generated 10.495s for the published 10.54s demo.
|
||||
- Live shared-runtime validation produced 6.36s of reference-free audio in 39.3s including model load, 9.99s of cloned audio in 36.7s warm, and exact cache reuse in 56ms.
|
||||
- The engine has unified TTS and SRT support only: no Voice Designer support, special node, training integration, native streaming/timestamps/dialogue, or built-in presets.
|
||||
|
||||
## OmniVoice
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
| **Echo-TTS** | Main | echo-tts-base + fish-s1-dac-min | ~5.3GB + ~1.8GB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | CC-BY-NC-SA-4.0 | Diffusion-based (~30s best), Force Speaker KV (speaker drift control) | 1 |
|
||||
| **Fish Audio S2 Pro** | Main | S2 Pro 4B / FP8 | ~10.3GB / ~8.0GB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | Fish Audio Research License | Free-form sub-word emotion/prosody tags, Native multi-speaker and multi-turn dialogue with dynamic speaker references, Zero-shot voice cloning, Optional per-segment custom character switching, Configurable 4K-32K native context with reduced KV-cache VRAM, Optional community FP8 weight-only checkpoint with BF16 activations, Optional on-the-fly BitsAndBytes INT8/NF4 for the official checkpoint | 80+ languages |
|
||||
| **Dots TTS** | Main | dots.tts-base, dots.tts-soar, dots.tts-mf | ~6GB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | Apache-2.0 | Official auto language detect / language control, SOAR and MeanFlow distilled variants | 19 |
|
||||
| **TADA** | Shared | TADA 1B (English), TADA 3B-ML | ~6.1GB / ~11GB with selected codec assets | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | MIT code / Llama 3.2 weights | Zero-shot voice cloning, Reference-conditioned expression transfer | 10 |
|
||||
| **Audio8 TTS** | Shared | Audio8 TTS Preview 0.6B | ~2.39 GiB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | Apache-2.0 | 44.1kHz reference-free TTS and zero-shot/cross-lingual voice cloning, Exact reference transcript required when voice cloning, Greedy or sampled decoding with temperature, top-p, top-k, seed, and token-limit controls, Suite text chunking, character switching, and SRT timing/assembly, No explicit language selector; multilingual behavior is text-driven, No native emotion/style/speed controls, streaming, timestamps, dialogue/multi-speaker mode, or presets, Not compatible with Voice Designer: the official model has no instruction-conditioned voice-design mode, Unified TTS/SRT integration only; no special node or suite training integration | 11 recommended languages (including Cantonese) |
|
||||
| **DramaBox** | Main | DramaBox 3.3B | ~16.4GB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | LTX-2 Community License | Expressive scene prompting and stage directions, Native and SRT-aware duration targeting, Official duration-aware long-form chunking with scene-prefix preservation, Optional 10-second zero-shot voice reference, CFG negative prompt with per-segment switching, Explicit generation/reference durations, CFG rescale control, and optional Perth watermark, Experimental staged and sequential strategies for lowering peak VRAM; official LTX FP8-cast storage, Optional official torch.compile path | 1 |
|
||||
| **OmniVoice** | Main | OmniVoice | ~3.7GB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | Apache-2.0 | Inline non-verbal tags and pronunciation overrides, Reference-free voice design, 600+ language support, Upstream long-form chunk orchestration | 600+ |
|
||||
| **MOSS-TTS** | Main | Local 1.7B, Delay 8B v1.5/1.0, VoiceGenerator 1.7B, SoundEffect 8B v1, TTSD 8B | ~8.5GB tokenizer + ~6.1GB/17GB/18GB model | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | Apache-2.0 | Reference-free voice design with MOSS-VoiceGenerator, Native 1-5 speaker TTSD dialogue, 31-language generation with MOSS-TTS-v1.5, Prompt-only sound-effect generation with MOSS-SoundEffect v1, Long-form generation (TTSD/Delay), Duration token hint, Local/Delay/TTSD variants, Initial integrated LoRA training workflow (Delay 8B) | 24 |
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
## Feature Comparison Matrix
|
||||
|
||||
| Feature | F5-TTS | ChatterBox | ChatterBox 23L | VibeVoice | Higgs Audio 2 | Higgs Audio v3 | IndexTTS-2 | CosyVoice3 | Qwen3-TTS | Granite ASR | Step Audio EditX | Echo-TTS | Fish Audio S2 Pro | Dots TTS | TADA | DramaBox | OmniVoice | MOSS-TTS | MOSS-SoundEffect v2 | RVC |
|
||||
| Feature | F5-TTS | ChatterBox | ChatterBox 23L | VibeVoice | Higgs Audio 2 | Higgs Audio v3 | IndexTTS-2 | CosyVoice3 | Qwen3-TTS | Granite ASR | Step Audio EditX | Echo-TTS | Fish Audio S2 Pro | Dots TTS | Audio8 TTS | DramaBox | OmniVoice | MOSS-TTS | MOSS-SoundEffect v2 | RVC |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| **TTS** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| **SRT** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
@@ -10,14 +10,14 @@
|
||||
| **ASR (Transcribe)** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||
| **Sound Effects** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ |
|
||||
| **Training** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ |
|
||||
| **Voice Cloning** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (Base model) | ❌ | ✅ | ✅ | ✅ Reference audio plus exact transcript | ✅ | ✅ Reference audio plus exact transcript | ✅ Optional reference audio; configurable 3-30 second window from the beginning of the source, default 10 seconds | ✅ | ✅ | ❌ | ⚠️ (needs training) |
|
||||
| **Reference Transcript†** | **Required** | Not used | Not used | Not used | Optional | Optional | Not used | Conditional | Conditional | N/A | **Required** | Not used | **Required** | Optional | **Required** | Not used | **Required** | Conditional | Not used | N/A |
|
||||
| **Voice Cloning** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (Base model) | ❌ | ✅ | ✅ | ✅ Reference audio plus exact transcript | ✅ | ✅ Zero-shot and cross-lingual; reference audio must be paired with its exact transcript | ✅ Optional reference audio; configurable 3-30 second window from the beginning of the source, default 10 seconds | ✅ | ✅ | ❌ | ⚠️ (needs training) |
|
||||
| **Reference Transcript†** | **Required** | Not used | Not used | Not used | Optional | Optional | Not used | Conditional | Conditional | N/A | **Required** | Not used | **Required** | Optional | Conditional | Not used | **Required** | Conditional | Not used | N/A |
|
||||
| **Native Multi-Speaker** | ❌ | ❌ | ❌ | ✅ (Base only, Kugel uses fallback) | ✅ | ❌ | ❌ | ❌ | ❌ | ⚠️ (Plus variant speaker attribution / diarization) | ❌ | ❌ | ✅ Native dialogue is default; optional custom mode generates each [Character] segment independently as local speaker 0 | ❌ | ❌ | ❌ | ❌ | ✅ (TTSD v1.0; 1-5 speakers) | ❌ | ❌ |
|
||||
| **Emotion Control** | ❌ | ❌ | ⚠️ (v2 tags - doesn't work) | ❌ | ⚠️ (via prompt) | ✅ (native inline tags) | ✅ (8 emotions) | ⚠️ (via instruct) | ⚠️ (via instruct) | ❌ | ✅ (14 emotions) | ❌ | ✅ Free-form inline natural-language tags | ❌ | ❌ | ✅ Natural-language scene prompt and stage directions | ⚠️ (voice-design instruct + inline non-verbal tags) | ✅ (MOSS-VoiceGenerator instruction-conditioned voice design) | ❌ | ❌ |
|
||||
| **Native Long-form** | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ Configurable 4K-32K native context; suite text chunking is bypassed | ❌ | ❌ | ✅ Official duration-aware quote-group chunking; ~37s target / 45s cap | ✅ (uses upstream audio_chunk_duration / audio_chunk_threshold orchestration; bypasses suite char-based chunk splitting) | ✅ (TTSD/Delay long-form; use chunk orchestration for very long inputs) | ❌ | N/A |
|
||||
| **Community Finetunes** | ✅ | ✅ | ✅ | ✅ KugelAudio, Hindi | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (LoRA adapter inference supported; initial integrated LoRA training support added for MOSS-TTS Delay 8B) | ❌ | ✅ |
|
||||
| **VRAM Efficient** | ✅ | ✅ | ✅ | ⚠️ (5-18GB) | ⚠️ (9GB) | ⚠️ (~8-10GB) | ⚠️ (9-12GB) | ✅ (5.4GB) | ✅ (3-6GB) | ✅ (~4.6GB) | ⚠️ (7GB) | ⚠️ (~7GB total) | ⚠️ 8K context measured at ~15.2GB BF16, ~11.2GB FP8, ~11.2GB BNB INT8, or ~8.9GB BNB NF4; BF16 codec and activations; BNB is a load-time option for the official checkpoint | ⚠️ (main env works; 2B-class model is not lightweight) | ⚠️ 1B is the lighter option; suite VRAM measurements are pending | ✅ Fast mode is ~24GB; experimental FP8 peaks on RTX 4090: staged ~15.1GB allocated, sequential ~11.7GB allocated / ~12.4GB reserved | ⚠️ (main env works; 2B-class model is not lightweight) | ⚠️ (Local 1.7B smaller; tokenizer is large) | ⚠️ (runs in the main ComfyUI environment but remains GPU-heavy) | ✅ |
|
||||
| **Speed/Performance** | ✅ Very Fast | ✅ Fast | ✅ Fast | ⚠️ | ⚠️ | ⚠️ CUDA recommended | ⚠️ | ✅ Fast | ⚠️ | ⚠️ Moderate | ⚠️ | ✅ Fast (diffusion, realtime-capable) | ✅ Main-environment subprocess with reliable teardown; local compile measurements: ~40 it/s BF16, ~11.8 it/s NF4 at ~8.9GB VRAM, and ~3.7 it/s INT8 at ~11.2GB VRAM; quality comparison pending | ⚠️ Moderate; mf variant is faster | ⚠️ No suite benchmark; no streaming or batch inference | ⚠️ Fast favors repeated generation; staged reloads temporary components; sequential also transfers the transformer; optional block-level torch.compile accelerates repeat denoising | ✅ Fast; upstream reports sub-realtime RTF | ✅ Fast with CUDA/FlashAttention | ⚠️ (100 diffusion steps by default) | ✅ Fast |
|
||||
| **No Narrator Required** | ❌ | ✅ (default speaker) | ✅ (default speaker) | ✅ (zero-shot / default speaker) | ✅ (basic TTS if no narrator/reference is provided) | ✅ (zero-shot) | ❌ | ✅ (cross-lingual or instruct mode) | ✅ (Base default voice or CustomVoice presets) | N/A | ❌ | ❌ | ✅ Reference audio is optional | ✅ (default speaker) | ❌ | ✅ Voice reference is optional | ✅ (native default voice; instruct also works without narrator) | ✅ (direct TTS and prompt-only generation) | ❌ | N/A |
|
||||
| **VRAM Efficient** | ✅ | ✅ | ✅ | ⚠️ (5-18GB) | ⚠️ (9GB) | ⚠️ (~8-10GB) | ⚠️ (9-12GB) | ✅ (5.4GB) | ✅ (3-6GB) | ✅ (~4.6GB) | ⚠️ (7GB) | ⚠️ (~7GB total) | ⚠️ 8K context measured at ~15.2GB BF16, ~11.2GB FP8, ~11.2GB BNB INT8, or ~8.9GB BNB NF4; BF16 codec and activations; BNB is a load-time option for the official checkpoint | ⚠️ (main env works; 2B-class model is not lightweight) | ⚠️ ~1.75 GiB model-plus-codec parameter memory at fp16/bf16 or ~3.5 GiB at float32/CPU, before KV cache, activations, and framework overhead; shared runtime reuses the main environment's PyTorch | ✅ Fast mode is ~24GB; experimental FP8 peaks on RTX 4090: staged ~15.1GB allocated, sequential ~11.7GB allocated / ~12.4GB reserved | ⚠️ (main env works; 2B-class model is not lightweight) | ⚠️ (Local 1.7B smaller; tokenizer is large) | ⚠️ (runs in the main ComfyUI environment but remains GPU-heavy) | ✅ |
|
||||
| **Speed/Performance** | ✅ Very Fast | ✅ Fast | ✅ Fast | ⚠️ | ⚠️ | ⚠️ CUDA recommended | ⚠️ | ✅ Fast | ⚠️ | ⚠️ Moderate | ⚠️ | ✅ Fast (diffusion, realtime-capable) | ✅ Main-environment subprocess with reliable teardown; local compile measurements: ~40 it/s BF16, ~11.8 it/s NF4 at ~8.9GB VRAM, and ~3.7 it/s INT8 at ~11.2GB VRAM; quality comparison pending | ⚠️ Moderate; mf variant is faster | ⚠️ Live shared-runtime validation: 6.36s reference-free audio in 39.3s including load; 9.99s cloned audio in 36.7s warm; exact cache reuse in 56ms | ⚠️ Fast favors repeated generation; staged reloads temporary components; sequential also transfers the transformer; optional block-level torch.compile accelerates repeat denoising | ✅ Fast; upstream reports sub-realtime RTF | ✅ Fast with CUDA/FlashAttention | ⚠️ (100 diffusion steps by default) | ✅ Fast |
|
||||
| **No Narrator Required** | ❌ | ✅ (default speaker) | ✅ (default speaker) | ✅ (zero-shot / default speaker) | ✅ (basic TTS if no narrator/reference is provided) | ✅ (zero-shot) | ❌ | ✅ (cross-lingual or instruct mode) | ✅ (Base default voice or CustomVoice presets) | N/A | ❌ | ❌ | ✅ Reference audio is optional | ✅ (default speaker) | ✅ Reference audio is optional | ✅ Voice reference is optional | ✅ (native default voice; instruct also works without narrator) | ✅ (direct TTS and prompt-only generation) | ❌ | N/A |
|
||||
|
||||
† **Reference Transcript:** Conditional means the transcript is required only for the specific mode: CosyVoice3 zero-shot, Qwen3-TTS full Base cloning, or MOSS-TTSD cloned-speaker dialogue. Higgs Audio 2, Higgs Audio v3, and Dots TTS accept matching text when provided but do not require it. TADA always requires an exact transcript for its reference audio in this integration.
|
||||
† **Reference Transcript:** Conditional means the transcript is required only for the specific mode: CosyVoice3 zero-shot, Qwen3-TTS full Base cloning, or MOSS-TTSD cloned-speaker dialogue, and Audio8 TTS zero-shot/cross-lingual cloning. Higgs Audio 2, Higgs Audio v3, and Dots TTS accept matching text when provided but do not require it.
|
||||
+14
-14
@@ -2,21 +2,21 @@
|
||||
|
||||
## Language Support by Engine
|
||||
|
||||
| Language | Code | F5-TTS | ChatterBox | ChatterBox 23L | VibeVoice | Higgs Audio 2 | Higgs Audio v3 | IndexTTS-2 | CosyVoice3 | Qwen3-TTS | Granite ASR | Step Audio EditX | Echo-TTS | Fish Audio S2 Pro | Dots TTS | TADA | DramaBox | OmniVoice | MOSS-TTS | MOSS-SoundEffect v2 | RVC |
|
||||
| Language | Code | F5-TTS | ChatterBox | ChatterBox 23L | VibeVoice | Higgs Audio 2 | Higgs Audio v3 | IndexTTS-2 | CosyVoice3 | Qwen3-TTS | Granite ASR | Step Audio EditX | Echo-TTS | Fish Audio S2 Pro | Dots TTS | Audio8 TTS | DramaBox | OmniVoice | MOSS-TTS | MOSS-SoundEffect v2 | RVC |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| 🇺🇸 **English** | EN | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ Tier 1 | ✅ | ✅ (1B and 3B-ML) | ✅ Official model is English-only | ✅ | ✅ | ✅ Officially demonstrated prompt language | ✅ |
|
||||
| 🇨🇳 **Chinese** | ZH | ❌ | ❌ | ✅ | ✅ | ✅ (Mandarin) | ✅ | ✅ | ✅ + 18 dialects | ✅ | ❌ | ✅ (Mandarin + Sichuanese, Cantonese) | ❌ | ✅ Tier 1 | ✅ (Mandarin; official also exposes YUE separately outside this matrix) | ✅ (3B-ML; backend code `ch`) | ❌ | ✅ | ✅ | ✅ Officially demonstrated prompt language | ✅ |
|
||||
| 🇩🇪 **German** | DE | ✅ | ✅ (×3) | ✅ | ✅ (Kugel) | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ | ✅ (3B-ML) | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇪🇸 **Spanish** | ES | ✅ | ❌ | ✅ | ✅ (Kugel) | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ | ✅ (3B-ML) | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇫🇷 **French** | FR | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ | ✅ (3B-ML) | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇮🇹 **Italian** | IT | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ (3B-ML) | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇯🇵 **Japanese** | JA | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ✅ ? | ✅ | ✅ | ✅ (4.1-2b supports Japanese; plus variant does not) | ✅ | ❌ | ✅ Tier 1 | ✅ | ✅ (3B-ML) | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇰🇷 **Korean** | KO | ❌ | ✅ | ✅ | ✅ (Kugel) | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ Tier 2 | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇺🇸 **English** | EN | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ Tier 1 | ✅ | ✅ | ✅ Official model is English-only | ✅ | ✅ | ✅ Officially demonstrated prompt language | ✅ |
|
||||
| 🇨🇳 **Chinese** | ZH | ❌ | ❌ | ✅ | ✅ | ✅ (Mandarin) | ✅ | ✅ | ✅ + 18 dialects | ✅ | ❌ | ✅ (Mandarin + Sichuanese, Cantonese) | ❌ | ✅ Tier 1 | ✅ (Mandarin; official also exposes YUE separately outside this matrix) | ✅ (Chinese; upstream separately recommends Cantonese, folded into this matrix row) | ❌ | ✅ | ✅ | ✅ Officially demonstrated prompt language | ✅ |
|
||||
| 🇩🇪 **German** | DE | ✅ | ✅ (×3) | ✅ | ✅ (Kugel) | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇪🇸 **Spanish** | ES | ✅ | ❌ | ✅ | ✅ (Kugel) | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇫🇷 **French** | FR | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇮🇹 **Italian** | IT | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇯🇵 **Japanese** | JA | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ✅ ? | ✅ | ✅ | ✅ (4.1-2b supports Japanese; plus variant does not) | ✅ | ❌ | ✅ Tier 1 | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇰🇷 **Korean** | KO | ❌ | ✅ | ✅ | ✅ (Kugel) | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ Tier 2 | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇷🇺 **Russian** | RU | ❌ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ Tier 2 | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇧🇷 **Portuguese** | PT | ✅ (BR) | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ (EU/BR*) | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ (Official PT tag is generic; upstream does not expose separate PT-BR/PT-PT tags and it may lean more European Portuguese than Brazilian Portuguese) | ✅ (3B-ML) | ❌ | ✅ (generic PT; official language space is much broader than this matrix) | ✅ | ❌ | ✅ |
|
||||
| 🇵🇱 **Polish** | PL | ✅ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ (3B-ML) | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇧🇷 **Portuguese** | PT | ✅ (BR) | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ (EU/BR*) | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ (Official PT tag is generic; upstream does not expose separate PT-BR/PT-PT tags and it may lean more European Portuguese than Brazilian Portuguese) | ❌ | ❌ | ✅ (generic PT; official language space is much broader than this matrix) | ✅ | ❌ | ✅ |
|
||||
| 🇵🇱 **Polish** | PL | ✅ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇮🇳 **Hindi** | HI | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇪🇬 **Arabic** | AR | ❌ | ❌ | ✅ (Egyptian) | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ Tier 2 | ✅ | ✅ (3B-ML) | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇪🇬 **Arabic** | AR | ❌ | ❌ | ✅ (Egyptian) | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ Tier 2 | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇹🇷 **Turkish** | TR | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇹🇭 **Thai** | TH | ✅ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇳🇴 **Norwegian** | NO | ❌ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ |
|
||||
@@ -28,7 +28,7 @@
|
||||
| 🇬🇷 **Greek** | EL | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇮🇱 **Hebrew** | HE | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇲🇾 **Malay** | MS | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇳🇱 **Dutch** | NL | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇳🇱 **Dutch** | NL | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇸🇪 **Swedish** | SV | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ |
|
||||
| 🇰🇪 **Swahili** | SW | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ |
|
||||
| 🇿🇦 **Afrikaans** | AF | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
||||
@@ -114,7 +114,7 @@
|
||||
- **Higgs Audio v3**: Official model card lists 102 languages total: 85 polished-production languages and 17 usable but less polished languages.
|
||||
- **IndexTTS-2**: Trained on 55K+ hours - ZH, EN, JA primary.
|
||||
- **MOSS-TTS**: Official list includes ZH, EN, DE, ES, FR, JA, IT, HU, KO, RU, FA, AR, PL, PT, CS, DA, SV, EL, TR; HU/FA/CS are not separate columns in this matrix.
|
||||
- **TADA**: TADA 1B is English-only. TADA 3B-ML supports EN, AR, ZH, DE, ES, FR, IT, JA, PL, and PT; the upstream backend calls Chinese `ch`.
|
||||
- **OmniVoice**: Official model supports 600+ languages; this matrix only shows the suite's comparison subset.
|
||||
- **Audio8 TTS**: The Preview recommends Cantonese, Chinese, Dutch, English, French, German, Italian, Japanese, Korean, Polish, and Spanish. It has no explicit language selector; language behavior is text-driven. Cantonese is folded into the Chinese row in this matrix.
|
||||
- **RVC**: Language-agnostic voice conversion/post-processing; it is marked supported for every language row.
|
||||
- Qwen3-TTS: Supports pt-BR only with instruction (voice design or harcoded voices). Base can't do instructions, so it will always ouput pt-PT.
|
||||
@@ -129,15 +129,11 @@ Use this as the canonical list of model repositories/links for offline setup.
|
||||
| dots.tts-soar | [rednote-hilab/dots.tts-soar](https://huggingface.co/rednote-hilab/dots.tts-soar) | ~6GB | ✅ | Official SOAR checkpoint for higher-quality zero-shot cloning |
|
||||
| dots.tts-mf | [rednote-hilab/dots.tts-mf](https://huggingface.co/rednote-hilab/dots.tts-mf) | ~6GB | ✅ | Official MeanFlow-distilled checkpoint for faster inference |
|
||||
|
||||
## TADA
|
||||
## Audio8 TTS
|
||||
|
||||
| Component | Source | Size | Auto-Download | Notes |
|
||||
|---|---|---|---|---|
|
||||
| TADA 1B | [HumeAI/tada-1b](https://huggingface.co/HumeAI/tada-1b) | ~3.9GB | ✅ | English-only model; downloaded only when selected |
|
||||
| TADA 3B-ML | [HumeAI/tada-3b-ml](https://huggingface.co/HumeAI/tada-3b-ml) | ~8.9GB | ✅ | Multilingual model; downloaded only when selected |
|
||||
| TADA codec | [HumeAI/tada-codec](https://huggingface.co/HumeAI/tada-codec) | ~2.15GB per selected language | ✅ | Downloads shared encoder/decoder plus only the aligner required by the selected language |
|
||||
| Llama tokenizer | [onnx-community/Llama-3.2-1B](https://huggingface.co/onnx-community/Llama-3.2-1B) | Tokenizer files only | ✅ | Ungated redistribution; runtime files are verified against Meta's official Git blob hashes and remain under the Llama 3.2 Community License |
|
||||
| Wav2Vec aligner configuration | [facebook/wav2vec2-large](https://huggingface.co/facebook/wav2vec2-large) | Configuration file only | ✅ | Downloads config.json only; no Wav2Vec model weights |
|
||||
| Audio8 TTS Preview 0.6B | [Audio8/Audio8-TTS-Preview-0.6b](https://huggingface.co/Audio8/Audio8-TTS-Preview-0.6b) | ~2.39 GiB | ✅ | Official 0.6B Preview checkpoint with bundled 44.1kHz codec; suite uses the shared Transformers 4.57.3 runtime because voice cloning is not compatible with main Transformers 5.10.2 |
|
||||
|
||||
## DramaBox
|
||||
|
||||
|
||||
+9
-15
@@ -427,27 +427,21 @@ Notes:
|
||||
- Native sample rate is 48kHz.
|
||||
- Main-environment support works on Transformers 5; on Windows, `normalize_text` falls back to no-op if `WeTextProcessing` is unavailable.
|
||||
|
||||
## TADA
|
||||
## Audio8 TTS
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/tada/
|
||||
├── TADA-1B/ # downloaded only if selected
|
||||
├── TADA-3B-ML/ # downloaded only if selected
|
||||
├── tada-codec/
|
||||
│ ├── encoder/
|
||||
│ ├── decoder/
|
||||
│ ├── aligner[-<language>]/ # selected language only
|
||||
│ └── wav2vec2-large/
|
||||
└── llama-3.2-1b-tokenizer/ # tokenizer files only
|
||||
ComfyUI/models/TTS/audio8_tts/
|
||||
└── Audio8-TTS-Preview-0.6b/
|
||||
└── (13 required official runtime files, including codec weights)
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- TADA runs in the shared Transformers 4 runtime with `hume-tada==0.1.9`.
|
||||
- The selected 1B or 3B-ML model, shared codec components, and selected language aligner download on demand.
|
||||
- The Llama tokenizer downloads from an ungated redistribution and is verified against Meta's official Git blob hashes. It remains under the Llama 3.2 Community License.
|
||||
- Reference audio and its exact transcript are required by this integration.
|
||||
- Native output is mono 24kHz.
|
||||
- The official `Audio8/Audio8-TTS-Preview-0.6b` checkpoint is about 2.39 GiB and includes the 44.1kHz neural codec.
|
||||
- The 0.6B model plus codec use about 1.75 GiB for parameters at fp16/bf16 or about 3.5 GiB at float32/CPU, before KV cache, activations, and framework overhead.
|
||||
- The suite uses the existing `vibevoice_transformers4_shared` runtime with Transformers 4.57.3. Main Transformers 5.10.2 is not compatible with Audio8 voice cloning: the official demo collapsed to 0.325s, while the shared runtime generated 10.495s for the published 10.54s demo.
|
||||
- Live shared-runtime validation produced 6.36s of reference-free audio in 39.3s including model load, 9.99s of cloned audio in 36.7s warm, and exact cache reuse in 56ms.
|
||||
- The engine has unified TTS and SRT support only: no Voice Designer support, special node, training integration, native streaming/timestamps/dialogue, or built-in presets.
|
||||
|
||||
## OmniVoice
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ Downloads and dependencies:
|
||||
- Did you document dependency conflicts or install.py changes?
|
||||
- Did you test `--no-deps` on Main/T5 and simple patching before falling back to the shared T4 runtime?
|
||||
- Did you avoid creating another environment or downloading/reinstalling Torch or Transformers without explicit maintainer approval?
|
||||
- If runtime isolation is recommended, does the engine expose the standard runtime selector, default to the recommended mode, route every choice honestly, and document why?
|
||||
- If runtime isolation is needed, did you document the default mode and the reason in YAML/README?
|
||||
|
||||
Audio format:
|
||||
- Did you verify the model native sample rate?
|
||||
@@ -42,7 +42,7 @@ Generated audio cache:
|
||||
|
||||
Text features:
|
||||
- Did you wire character tags if this node supports text generation?
|
||||
- Does a missing character use narrator fallback while an existing but invalid voice (for example, missing required transcript) raises a clear error?
|
||||
- Did you handle narrator fallback?
|
||||
- Did you wire pause tags?
|
||||
- Did you wire parameter switching for real native parameters?
|
||||
- Did you avoid adding fake unsupported parameters?
|
||||
|
||||
@@ -466,7 +466,7 @@ Also to test, requirements and dependencies need to be added.
|
||||
- [ ] Pause tags work with `[pause:1.5s]`
|
||||
- [ ] Caching works (same input = cached output)
|
||||
- [ ] Model auto-download works
|
||||
- [ ] VRAM management works (model unloads)
|
||||
- [ ] VRAM management works (unload, switch to another isolated engine, then switch back with cache disabled)
|
||||
- [ ] Different parameter combinations work
|
||||
- [ ] Engine prints a standard `Settings:` summary with the active generation/load parameters
|
||||
- [ ] **Interrupt handling works** - User can stop SRT generation and it stops within ~1 segment
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
### Import Errors
|
||||
- **Missing import**: Always add `import folder_paths` when using `folder_paths.get_temp_directory()` in node files
|
||||
- **Bundled code imports**: For complex bundled packages with internal cross-imports, add `sys.path.insert(0, impl_dir)` at top of main files instead of converting all imports
|
||||
- **Bundled code imports**: Never permanently prepend a bundled implementation directory to `sys.path` in the main ComfyUI process. Generic files such as `utils.py` can shadow suite packages and break later isolated workers. Prefer package-relative or vendor-namespaced imports; if upstream imports make path injection unavoidable, confine it to the isolated worker and load suite protocol modules before adding the vendor path.
|
||||
|
||||
### Audio Utility Functions
|
||||
- **Temp file creation**: Use `AudioProcessingUtils.save_audio_to_temp_file()` not `save_audio()` (doesn't exist)
|
||||
@@ -26,7 +26,6 @@
|
||||
|
||||
### Voice References
|
||||
- **Reference text requirement**: F5-TTS and Step Audio EditX REQUIRE `prompt_text` (transcript), ChatterBox/VibeVoice/Higgs don't
|
||||
- **Missing vs invalid character voice**: A character with no saved voice may fall back to narrator; a discovered/selected voice missing an engine-required transcript is invalid and MUST raise a clear error instead of silently using narrator
|
||||
- **Narrator voice mapping**: Map narrator from TTS Text node input by saving `audio_tensor` to temp file with `reference_text`
|
||||
- **Voice discovery**: Use `get_character_mapping()` not `discover_voices_for_engine()` (latter doesn't exist)
|
||||
|
||||
|
||||
@@ -1,122 +0,0 @@
|
||||
# TADA Engine Guide
|
||||
|
||||
TADA is Hume AI's zero-shot text-to-speech model. TTS Audio Suite exposes it through the unified **Text to Speech** and **SRT** nodes, including character switching, per-segment language and generation parameters, pause tags, caching, interruption, and Clear VRAM.
|
||||
|
||||
This integration deliberately describes TADA's expressive capability as **reference-conditioned expression transfer**. The model can carry voice, delivery, timing, and emotion present in the prompt audio into generated speech, but it does not provide a native emotion label, intensity slider, embedding, or proven text-instruction emotion control. It is not direct emotion control.
|
||||
|
||||
## Supported models and languages
|
||||
|
||||
| Model | Languages | Model weights | Intended use |
|
||||
|---|---|---:|---|
|
||||
| `HumeAI/tada-1b` | English | ~3.9GB | Lighter English-only option |
|
||||
| `HumeAI/tada-3b-ml` | English, Arabic, Chinese, German, Spanish, French, Italian, Japanese, Polish, Portuguese | ~8.9GB | Multilingual voice cloning |
|
||||
|
||||
The upstream language code for Chinese is `ch`; the suite presents it as Chinese/`zh` and maps it internally. TADA generates mono audio at 24kHz. Reference audio is resampled to 24kHz when needed.
|
||||
|
||||
## First-use requirements
|
||||
|
||||
TADA runs only in the suite's **shared Transformers 4 runtime**. The integration installs `hume-tada==0.1.9` there and does not modify the main ComfyUI Transformers 5 environment. TADA's current configuration code is incompatible with that main environment.
|
||||
|
||||
Before first use, select TADA in an **⚙️ TADA Engine** node and run a workflow. The suite downloads the selected TADA model, shared codec encoder/decoder, the aligner for the selected language, and tokenizer files into `ComfyUI/models/TTS/tada/`.
|
||||
|
||||
Downloads are selective. Choosing the 1B model does not download the 3B model, and changing language downloads only that language's codec aligner. A selected codec set is approximately 2.15GB in addition to the model checkpoint.
|
||||
|
||||
### Automatic download
|
||||
|
||||
No Hugging Face login is required for the normal automatic path. The suite downloads the three tokenizer files from the ungated `onnx-community/Llama-3.2-1B` redistribution, plus its `LICENSE.txt` and `USE_POLICY.md`. Those runtime files are byte-identical to Meta's official `meta-llama/Llama-3.2-1B` tokenizer and are checked against the official Git blob hashes before use.
|
||||
|
||||
The tokenizer remains governed by the Llama 3.2 Community License and Acceptable Use Policy. TADA's use of this component is **Built with Llama**. The full Llama model weights are not downloaded.
|
||||
|
||||
### Manual/offline installation
|
||||
|
||||
Users accustomed to managing ComfyUI models manually can copy the exact tokenizer and model files into the following layout. Only these files are required at runtime:
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/tada/
|
||||
├── TADA-1B/
|
||||
│ ├── config.json
|
||||
│ ├── generation_config.json
|
||||
│ └── model.safetensors
|
||||
├── TADA-3B-ML/
|
||||
│ ├── config.json
|
||||
│ ├── generation_config.json
|
||||
│ ├── model.safetensors.index.json
|
||||
│ ├── model-00001-of-00002.safetensors
|
||||
│ └── model-00002-of-00002.safetensors
|
||||
├── tada-codec/
|
||||
│ ├── encoder/config.json
|
||||
│ ├── encoder/model.safetensors
|
||||
│ ├── decoder/config.json
|
||||
│ ├── decoder/model.safetensors
|
||||
│ ├── aligner/config.json # English
|
||||
│ ├── aligner/model.safetensors
|
||||
│ ├── aligner-<language>/config.json # selected non-English language
|
||||
│ ├── aligner-<language>/model.safetensors
|
||||
│ └── wav2vec2-large/config.json
|
||||
└── llama-3.2-1b-tokenizer/
|
||||
├── tokenizer.json
|
||||
├── tokenizer_config.json
|
||||
└── special_tokens_map.json
|
||||
```
|
||||
|
||||
Install only the selected model directory. For non-English generation, `<language>` is one of `ar`, `ch`, `de`, `es`, `fr`, `it`, `ja`, `pl`, or `pt`. Existing complete files are detected and are not downloaded again. The full Llama model weights and Wav2Vec weights are not required.
|
||||
|
||||
## Reference voice requirements
|
||||
|
||||
Every generation in this integration requires:
|
||||
|
||||
- a clean reference speech clip; and
|
||||
- an exact transcript of the speech in that clip.
|
||||
|
||||
The transcript must match what is spoken, including word order. A mismatched transcript weakens alignment and cloning quality. TADA's official notebook can invoke an English Parakeet ASR model when text is omitted, but the suite does not do that: it would add a hidden ~4.25GB English-only dependency and would be unsuitable for multilingual references. Supply the exact text explicitly or save it beside a character voice as `.reference.txt`/`.txt`.
|
||||
|
||||
Expression comes from the reference recording. Use a prompt that actually contains the desired energy, pace, emotion, and speaking style. There is no emotion dropdown because adding one would be a fake control unsupported by the model API.
|
||||
|
||||
## Engine controls
|
||||
|
||||
Choose the model variant, device, language, and dtype on the engine node. `auto` is the default for device and dtype, and English is the default language. The engine node and unified TTS nodes expose these generation controls:
|
||||
|
||||
| Control | Suite default | Effect |
|
||||
|---|---:|---|
|
||||
| `seed` | unified-node value | Reproducible stochastic generation |
|
||||
| `acoustic_cfg_scale` | `1.6` | Acoustic classifier-free guidance strength |
|
||||
| `duration_cfg_scale` | `1.0` | Duration guidance strength |
|
||||
| `cfg_schedule` | `cosine` | Guidance schedule: `constant`, `linear`, or `cosine` |
|
||||
| `num_flow_matching_steps` | `10` | More ODE steps trade speed for refinement |
|
||||
| `noise_temperature` | `0.9` | Initial flow-matching noise scale |
|
||||
| `time_schedule` | `logsnr` | ODE schedule: `uniform`, `cosine`, or `logsnr` |
|
||||
| `negative_condition_source` | `negative_step_output` | Source used for negative CFG conditioning |
|
||||
| `speed_up_factor` | `0.0` (disabled) | Native two-pass duration scaling; values above 0 add a second generation pass, and values above 1 speak faster |
|
||||
| `num_transition_steps` | `5` | Prompt-to-generated-speech transition overlap |
|
||||
|
||||
Defaults follow the official implementation. Change one parameter at a time: CFG, noise, flow steps, and transition handling interact, and higher values are not automatically better. `speed_up_factor` is not a cheap playback-rate effect; it reruns generation with scaled predicted durations.
|
||||
|
||||
Per-segment tags can override supported parameters and language in unified Text or SRT prompts. The 1B model rejects non-English language selection instead of silently generating with the wrong model.
|
||||
|
||||
## Long text and SRT
|
||||
|
||||
TADA has no native long-form or streaming mode in this integration. The unified Text node uses the suite's sentence-aware chunking and combines the resulting 24kHz clips. The SRT processor generates each subtitle independently and uses the suite's timing modes and overlap fallback.
|
||||
|
||||
The official model was trained on short clips (up to roughly 30 seconds), so long uninterrupted generations can drift. Prefer normal chunking or subtitle-sized segments. Native batch inference, streaming, speech continuation, random-voice generation, text-sampling controls, multiple-candidate scoring, and speaker verification are not exposed in the first integration. Text sampling is irrelevant to normal suite TTS because the exact target text is supplied.
|
||||
|
||||
## Model layout
|
||||
|
||||
```text
|
||||
ComfyUI/models/TTS/tada/
|
||||
├── TADA-1B/ # downloaded only if selected
|
||||
├── TADA-3B-ML/ # downloaded only if selected
|
||||
├── tada-codec/
|
||||
│ ├── encoder/
|
||||
│ ├── decoder/
|
||||
│ ├── aligner[-<language>]/ # selected language only
|
||||
│ └── wav2vec2-large/ # aligner config only, not model weights
|
||||
└── llama-3.2-1b-tokenizer/ # tokenizer files only
|
||||
```
|
||||
|
||||
## Licensing and upstream sources
|
||||
|
||||
- Official implementation: [HumeAI/tada](https://github.com/HumeAI/tada) — MIT code license.
|
||||
- Model checkpoints: [HumeAI/tada-1b](https://huggingface.co/HumeAI/tada-1b), [HumeAI/tada-3b-ml](https://huggingface.co/HumeAI/tada-3b-ml), and [HumeAI/tada-codec](https://huggingface.co/HumeAI/tada-codec).
|
||||
- Model weights and tokenizer are subject to the Meta Llama 3.2 Community License. Commercial use is conditional on those terms. Applications distributing or presenting the model must follow the license's attribution requirements, including the required **Built with Llama** notice where applicable.
|
||||
|
||||
TTS Audio Suite does not replace or relax upstream license terms. Review them before redistributing models or generated products.
|
||||
@@ -49,6 +49,19 @@ except Exception as e:
|
||||
def __init__(self, *args, **kwargs):
|
||||
raise ImportError(f"Dots TTS adapter not available: {e}")
|
||||
|
||||
try:
|
||||
from .audio8_tts_adapter import Audio8TTSEngineAdapter
|
||||
AUDIO8_TTS_ADAPTER_AVAILABLE = True
|
||||
except Exception as exc:
|
||||
AUDIO8_TTS_ADAPTER_AVAILABLE = False
|
||||
_AUDIO8_TTS_ADAPTER_ERROR = str(exc)
|
||||
|
||||
class Audio8TTSEngineAdapter:
|
||||
def __init__(self, *args, **kwargs):
|
||||
raise ImportError(
|
||||
f"Audio8 TTS adapter not available: {_AUDIO8_TTS_ADAPTER_ERROR}"
|
||||
)
|
||||
|
||||
try:
|
||||
from .dramabox_adapter import DramaBoxEngineAdapter
|
||||
DRAMABOX_ADAPTER_AVAILABLE = True
|
||||
@@ -96,10 +109,10 @@ except Exception as e:
|
||||
|
||||
__all__ = [
|
||||
'ChatterBoxEngineAdapter', 'F5TTSEngineAdapter', 'CosyVoiceAdapter', 'EchoTTSEngineAdapter',
|
||||
'DotsTTSEngineAdapter', 'DramaBoxEngineAdapter', 'OmniVoiceEngineAdapter',
|
||||
'DotsTTSEngineAdapter', 'Audio8TTSEngineAdapter', 'DramaBoxEngineAdapter', 'OmniVoiceEngineAdapter',
|
||||
'MossTTSEngineAdapter', 'HiggsAudioV3EngineAdapter',
|
||||
'CHATTERBOX_ADAPTER_AVAILABLE', 'F5TTS_ADAPTER_AVAILABLE', 'COSYVOICE_ADAPTER_AVAILABLE',
|
||||
'ECHO_TTS_ADAPTER_AVAILABLE', 'DOTS_TTS_ADAPTER_AVAILABLE',
|
||||
'ECHO_TTS_ADAPTER_AVAILABLE', 'DOTS_TTS_ADAPTER_AVAILABLE', 'AUDIO8_TTS_ADAPTER_AVAILABLE',
|
||||
'DRAMABOX_ADAPTER_AVAILABLE', 'OMNIVOICE_ADAPTER_AVAILABLE',
|
||||
'MOSS_TTS_ADAPTER_AVAILABLE', 'HIGGS_AUDIO_V3_ADAPTER_AVAILABLE',
|
||||
'MossSoundEffectV2Adapter'
|
||||
|
||||
@@ -0,0 +1,277 @@
|
||||
"""Adapter between suite processors and the official Audio8 TTS engine."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from engines.audio8_tts.downloader import Audio8TTSDownloader
|
||||
from utils.audio.audio_hash import generate_stable_audio_component
|
||||
from utils.audio.cache import get_audio_cache
|
||||
from utils.device import resolve_torch_device
|
||||
from utils.models.factory_config import ModelLoadConfig, RUNTIME_MODE_SHARED
|
||||
from utils.voice.character_logging import resolved_character_label
|
||||
from utils.voice.reference import effective_voice_audio
|
||||
|
||||
|
||||
class Audio8TTSEngineAdapter:
|
||||
"""Translate unified TTS calls into Audio8's official inference API."""
|
||||
|
||||
SAMPLE_RATE = 44100
|
||||
|
||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||
self.config = dict(config or {})
|
||||
self.audio_cache = get_audio_cache()
|
||||
self.downloader = Audio8TTSDownloader()
|
||||
self._last_config: Optional[ModelLoadConfig] = None
|
||||
self._load_signature: Optional[Tuple[Any, ...]] = None
|
||||
|
||||
def update_config(self, new_config: Dict[str, Any]):
|
||||
self.config = dict(new_config or {})
|
||||
|
||||
def _model_selection(self) -> str:
|
||||
return str(
|
||||
self.config.get(
|
||||
"model_variant",
|
||||
self.config.get(
|
||||
"model_name",
|
||||
Audio8TTSDownloader.MODEL_NAME,
|
||||
),
|
||||
)
|
||||
or Audio8TTSDownloader.MODEL_NAME
|
||||
)
|
||||
|
||||
def _build_load_signature(self) -> Tuple[Any, ...]:
|
||||
return (
|
||||
self._model_selection(),
|
||||
resolve_torch_device(self.config.get("device", "auto")),
|
||||
self.config.get("dtype", "auto"),
|
||||
)
|
||||
|
||||
def load_model(
|
||||
self,
|
||||
model_variant: str,
|
||||
device: str = "auto",
|
||||
dtype: str = "auto",
|
||||
):
|
||||
"""Resolve the organized model path and load through the unified factory."""
|
||||
from utils.models.unified_model_interface import unified_model_interface
|
||||
|
||||
model_path = self.downloader.resolve_model_path(model_variant)
|
||||
config = ModelLoadConfig(
|
||||
engine_name="audio8_tts",
|
||||
model_type="tts",
|
||||
model_name=model_variant,
|
||||
model_path=model_path,
|
||||
device=device,
|
||||
additional_params={"dtype": dtype},
|
||||
runtime_mode=RUNTIME_MODE_SHARED,
|
||||
runtime_profile="vibevoice_transformers4_shared",
|
||||
)
|
||||
self._last_config = config
|
||||
return unified_model_interface.load_model(config)
|
||||
|
||||
def _ensure_model_loaded(self):
|
||||
signature = self._build_load_signature()
|
||||
if signature != self._load_signature or self._last_config is None:
|
||||
self.load_model(
|
||||
model_variant=self._model_selection(),
|
||||
device=self.config.get("device", "auto"),
|
||||
dtype=self.config.get("dtype", "auto"),
|
||||
)
|
||||
self._load_signature = signature
|
||||
|
||||
def _get_engine(self):
|
||||
if self._last_config is None:
|
||||
self._ensure_model_loaded()
|
||||
from utils.models.unified_model_interface import unified_model_interface
|
||||
|
||||
return unified_model_interface.load_model(self._last_config)
|
||||
|
||||
@staticmethod
|
||||
def _mono_waveform(waveform: Any) -> torch.Tensor:
|
||||
audio = torch.as_tensor(waveform, dtype=torch.float32).detach().cpu()
|
||||
if audio.ndim == 3:
|
||||
if audio.shape[0] != 1:
|
||||
raise ValueError(
|
||||
"Audio8 TTS reference audio must contain one batch item"
|
||||
)
|
||||
audio = audio[0]
|
||||
if audio.ndim == 2:
|
||||
audio = audio.mean(dim=0)
|
||||
if audio.ndim != 1 or audio.numel() == 0:
|
||||
raise ValueError(
|
||||
"Audio8 TTS reference audio must be non-empty mono or "
|
||||
"channels-first audio"
|
||||
)
|
||||
return audio.contiguous()
|
||||
|
||||
def _in_memory_reference(
|
||||
self,
|
||||
waveform: Any,
|
||||
sample_rate: Any,
|
||||
) -> Tuple[Dict[str, Any], str]:
|
||||
mono = self._mono_waveform(waveform)
|
||||
sample_rate = int(sample_rate)
|
||||
comfy_audio = {
|
||||
"waveform": mono.unsqueeze(0),
|
||||
"sample_rate": sample_rate,
|
||||
}
|
||||
processor_audio = {
|
||||
"array": mono,
|
||||
"sampling_rate": sample_rate,
|
||||
}
|
||||
return (
|
||||
processor_audio,
|
||||
generate_stable_audio_component(reference_audio=comfy_audio),
|
||||
)
|
||||
|
||||
def _extract_voice_reference(
|
||||
self,
|
||||
voice_ref: Optional[Dict[str, Any]],
|
||||
) -> Tuple[Any, str, str]:
|
||||
if not isinstance(voice_ref, dict):
|
||||
return None, "", "default_voice"
|
||||
|
||||
reference_text = str(
|
||||
voice_ref.get("reference_text")
|
||||
or voice_ref.get("prompt_text")
|
||||
or voice_ref.get("text")
|
||||
or ""
|
||||
).strip()
|
||||
reference_audio = effective_voice_audio(voice_ref)
|
||||
if reference_audio is None:
|
||||
return None, reference_text, "default_voice"
|
||||
|
||||
if isinstance(reference_audio, (str, os.PathLike)):
|
||||
path = os.fspath(reference_audio)
|
||||
component = generate_stable_audio_component(audio_file_path=path)
|
||||
return path, reference_text, component
|
||||
|
||||
if isinstance(reference_audio, dict):
|
||||
waveform = reference_audio.get(
|
||||
"waveform",
|
||||
reference_audio.get("array"),
|
||||
)
|
||||
sample_rate = reference_audio.get(
|
||||
"sample_rate",
|
||||
reference_audio.get("sampling_rate"),
|
||||
)
|
||||
if waveform is None or sample_rate is None:
|
||||
raise ValueError(
|
||||
"Audio8 TTS in-memory reference requires waveform and sample_rate"
|
||||
)
|
||||
normalized, component = self._in_memory_reference(
|
||||
waveform,
|
||||
sample_rate,
|
||||
)
|
||||
return normalized, reference_text, component
|
||||
|
||||
if isinstance(reference_audio, (tuple, list)) and len(reference_audio) == 2:
|
||||
normalized, component = self._in_memory_reference(
|
||||
reference_audio[0],
|
||||
reference_audio[1],
|
||||
)
|
||||
return normalized, reference_text, component
|
||||
|
||||
if torch.is_tensor(reference_audio):
|
||||
normalized, component = self._in_memory_reference(
|
||||
reference_audio,
|
||||
voice_ref.get("sample_rate", self.SAMPLE_RATE),
|
||||
)
|
||||
return normalized, reference_text, component
|
||||
|
||||
raise TypeError(
|
||||
f"Unsupported Audio8 TTS reference type: {type(reference_audio)}"
|
||||
)
|
||||
|
||||
def generate_single(
|
||||
self,
|
||||
text: str,
|
||||
voice_ref: Optional[Dict[str, Any]],
|
||||
seed: int = 0,
|
||||
enable_audio_cache: bool = True,
|
||||
character_name: Optional[str] = None,
|
||||
) -> torch.Tensor:
|
||||
"""Generate one raw utterance; chunking remains processor-owned."""
|
||||
text = str(text or "").strip()
|
||||
if not text:
|
||||
return torch.zeros(1, 0, dtype=torch.float32)
|
||||
|
||||
reference_audio, reference_text, audio_component = (
|
||||
self._extract_voice_reference(voice_ref)
|
||||
)
|
||||
if reference_audio is not None and not reference_text:
|
||||
raise ValueError(
|
||||
"Audio8 TTS voice cloning requires the exact transcript of "
|
||||
"the reference audio. Add reference text to Character Voices "
|
||||
"or the narrator voice."
|
||||
)
|
||||
|
||||
params = {
|
||||
"model_variant": self._model_selection(),
|
||||
"max_new_tokens": int(self.config.get("max_new_tokens", 1024)),
|
||||
"retry_max_new_tokens": int(self.config.get("retry_max_new_tokens", 2000)),
|
||||
"temperature": float(self.config.get("temperature", 0.8)),
|
||||
"top_p": float(self.config.get("top_p", 0.95)),
|
||||
"top_k": int(self.config.get("top_k", 50)),
|
||||
"do_sample": bool(self.config.get("do_sample", True)),
|
||||
"dtype": self.config.get("dtype", "auto"),
|
||||
"device": resolve_torch_device(self.config.get("device", "auto")),
|
||||
"seed": int(seed),
|
||||
}
|
||||
|
||||
cache_key = None
|
||||
if enable_audio_cache:
|
||||
cache_key = self.audio_cache.generate_cache_key(
|
||||
"audio8_tts",
|
||||
text=text,
|
||||
audio_component=audio_component,
|
||||
reference_text=reference_text,
|
||||
character=character_name or "narrator",
|
||||
**params,
|
||||
)
|
||||
cached = self.audio_cache.get_cached_audio(cache_key)
|
||||
if cached:
|
||||
display_name = resolved_character_label(
|
||||
character_name or "narrator",
|
||||
voice_ref,
|
||||
)
|
||||
print(
|
||||
"💾 Using cached Audio8 TTS audio for "
|
||||
f"'{display_name}': '{text[:30]}...'"
|
||||
)
|
||||
return cached[0]
|
||||
|
||||
self._ensure_model_loaded()
|
||||
audio = self._get_engine().generate(
|
||||
text=text,
|
||||
reference_audio=reference_audio,
|
||||
reference_text=reference_text or None,
|
||||
max_new_tokens=params["max_new_tokens"],
|
||||
retry_max_new_tokens=params["retry_max_new_tokens"],
|
||||
temperature=params["temperature"],
|
||||
top_p=params["top_p"],
|
||||
top_k=params["top_k"],
|
||||
do_sample=params["do_sample"],
|
||||
seed=params["seed"],
|
||||
)
|
||||
if not isinstance(audio, torch.Tensor):
|
||||
audio = torch.as_tensor(audio, dtype=torch.float32)
|
||||
audio = audio.detach().float().cpu()
|
||||
if audio.ndim == 1:
|
||||
audio = audio.unsqueeze(0)
|
||||
if audio.ndim != 2:
|
||||
raise RuntimeError(
|
||||
f"Audio8 TTS returned invalid audio shape: {tuple(audio.shape)}"
|
||||
)
|
||||
|
||||
if cache_key:
|
||||
self.audio_cache.cache_audio(
|
||||
cache_key,
|
||||
audio,
|
||||
audio.shape[-1] / self.SAMPLE_RATE,
|
||||
)
|
||||
return audio
|
||||
@@ -1,256 +0,0 @@
|
||||
"""Adapter between TTS Audio Suite processors and the isolated TADA runtime."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
current_dir = os.path.dirname(__file__)
|
||||
engines_dir = os.path.dirname(current_dir)
|
||||
project_root = os.path.dirname(engines_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
from engines.tada.languages import format_tada_language_display, validate_tada_model_language
|
||||
from engines.tada.tada_downloader import TadaDownloader
|
||||
from utils.audio.audio_hash import generate_stable_audio_component
|
||||
from utils.audio.cache import get_audio_cache
|
||||
from utils.device import resolve_torch_device
|
||||
from utils.models.factory_config import ModelLoadConfig, RUNTIME_MODE_SHARED
|
||||
from utils.voice.reference import effective_voice_audio
|
||||
|
||||
|
||||
class TadaEngineAdapter:
|
||||
"""Resolve assets, load the shared runtime proxy, and cache TADA waveforms."""
|
||||
|
||||
SAMPLE_RATE = 24000
|
||||
|
||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||
self.config = config.copy() if config else {}
|
||||
self.audio_cache = get_audio_cache()
|
||||
self.downloader = TadaDownloader()
|
||||
self._last_config: Optional[ModelLoadConfig] = None
|
||||
self._load_signature: Optional[Tuple[Any, ...]] = None
|
||||
self._ensured_assets: Dict[Tuple[str, str], Dict[str, str]] = {}
|
||||
|
||||
def update_config(self, new_config: Dict[str, Any]) -> None:
|
||||
self.config = new_config.copy() if new_config else {}
|
||||
|
||||
def _model_identifier(self) -> str:
|
||||
return str(
|
||||
self.config.get("model_variant")
|
||||
or self.config.get("model_path")
|
||||
or self.config.get("model_name")
|
||||
or "TADA-1B"
|
||||
)
|
||||
|
||||
def _ensure_assets(self, language: str) -> Dict[str, str]:
|
||||
identifier = self._model_identifier()
|
||||
display_language = format_tada_language_display(language)
|
||||
key = (identifier, display_language)
|
||||
assets = self._ensured_assets.get(key)
|
||||
if assets is None:
|
||||
assets = self.downloader.resolve_model_assets(identifier, display_language)
|
||||
self._ensured_assets[key] = assets
|
||||
return assets
|
||||
|
||||
def _build_load_signature(self, assets: Dict[str, str]) -> Tuple[Any, ...]:
|
||||
resolved_device = resolve_torch_device(self.config.get("device", "auto"))
|
||||
return (
|
||||
assets["model_name"],
|
||||
os.path.abspath(assets["model_path"]),
|
||||
os.path.abspath(assets["codec_path"]),
|
||||
os.path.abspath(assets["tokenizer_path"]),
|
||||
resolved_device,
|
||||
self.config.get("dtype", "auto"),
|
||||
self.config.get("attn_implementation", "sdpa"),
|
||||
bool(self.config.get("use_torch_compile", False)),
|
||||
int(self.config.get("prompt_cache_size", 8)),
|
||||
self.config.get("runtime_mode", RUNTIME_MODE_SHARED),
|
||||
self.config.get("runtime_profile", "vibevoice_transformers4_shared"),
|
||||
)
|
||||
|
||||
def _ensure_model_loaded(self, language: str) -> None:
|
||||
from utils.models.unified_model_interface import unified_model_interface
|
||||
|
||||
assets = self._ensure_assets(language)
|
||||
signature = self._build_load_signature(assets)
|
||||
if signature == self._load_signature and self._last_config is not None:
|
||||
return
|
||||
|
||||
resolved_device = resolve_torch_device(self.config.get("device", "auto"))
|
||||
load_config = ModelLoadConfig(
|
||||
engine_name="tada",
|
||||
model_type="tts",
|
||||
model_name=assets["model_name"],
|
||||
model_path=assets["model_path"],
|
||||
device=resolved_device,
|
||||
additional_params={
|
||||
"codec_path": assets["codec_path"],
|
||||
"tokenizer_path": assets["tokenizer_path"],
|
||||
"dtype": self.config.get("dtype", "auto"),
|
||||
"attn_implementation": self.config.get("attn_implementation", "sdpa"),
|
||||
"use_torch_compile": bool(self.config.get("use_torch_compile", False)),
|
||||
"prompt_cache_size": int(self.config.get("prompt_cache_size", 8)),
|
||||
},
|
||||
runtime_mode=self.config.get("runtime_mode", RUNTIME_MODE_SHARED),
|
||||
runtime_profile=self.config.get(
|
||||
"runtime_profile", "vibevoice_transformers4_shared"
|
||||
),
|
||||
)
|
||||
self._last_config = load_config
|
||||
unified_model_interface.load_model(load_config)
|
||||
self._load_signature = signature
|
||||
|
||||
def _get_engine(self):
|
||||
if self._last_config is None:
|
||||
raise RuntimeError("TADA model was requested before its assets were resolved")
|
||||
from utils.models.unified_model_interface import unified_model_interface
|
||||
|
||||
return unified_model_interface.load_model(self._last_config)
|
||||
|
||||
@staticmethod
|
||||
def _reference_text(voice_ref: Optional[Dict[str, Any]]) -> str:
|
||||
if not isinstance(voice_ref, dict):
|
||||
return ""
|
||||
return str(
|
||||
voice_ref.get("reference_text")
|
||||
or voice_ref.get("prompt_text")
|
||||
or voice_ref.get("text")
|
||||
or ""
|
||||
).strip()
|
||||
|
||||
@staticmethod
|
||||
def _audio_component(reference_audio: Any, voice_ref: Dict[str, Any]) -> str:
|
||||
if isinstance(reference_audio, str):
|
||||
return generate_stable_audio_component(audio_file_path=reference_audio)
|
||||
if isinstance(reference_audio, dict) and "waveform" in reference_audio:
|
||||
return generate_stable_audio_component(reference_audio=reference_audio)
|
||||
if torch.is_tensor(reference_audio):
|
||||
return generate_stable_audio_component(
|
||||
reference_audio={
|
||||
"waveform": reference_audio,
|
||||
"sample_rate": int(voice_ref.get("sample_rate", TadaEngineAdapter.SAMPLE_RATE)),
|
||||
}
|
||||
)
|
||||
if isinstance(reference_audio, (tuple, list)) and len(reference_audio) == 2:
|
||||
waveform, sample_rate = reference_audio
|
||||
if torch.is_tensor(waveform):
|
||||
return generate_stable_audio_component(
|
||||
reference_audio={"waveform": waveform, "sample_rate": int(sample_rate)}
|
||||
)
|
||||
raise TypeError(f"Unsupported TADA reference audio type: {type(reference_audio)}")
|
||||
|
||||
def generate_single(
|
||||
self,
|
||||
text: str,
|
||||
voice_ref: Optional[Dict[str, Any]],
|
||||
seed: int = 0,
|
||||
enable_audio_cache: bool = True,
|
||||
character_name: Optional[str] = None,
|
||||
) -> torch.Tensor:
|
||||
stripped = str(text or "").strip()
|
||||
if not stripped:
|
||||
return torch.zeros(1, 0, dtype=torch.float32)
|
||||
if not isinstance(voice_ref, dict):
|
||||
raise ValueError("TADA requires reference audio and its exact transcript")
|
||||
|
||||
reference_audio = effective_voice_audio(voice_ref)
|
||||
reference_text = self._reference_text(voice_ref)
|
||||
if reference_audio is None:
|
||||
raise ValueError("TADA requires reference audio; random-voice mode is not exposed")
|
||||
if not reference_text:
|
||||
raise ValueError(
|
||||
"TADA requires the exact reference transcript. Automatic Parakeet ASR is "
|
||||
"intentionally disabled to prevent a hidden 4.25 GB download."
|
||||
)
|
||||
|
||||
model_variant = self._model_identifier()
|
||||
language = format_tada_language_display(self.config.get("language", "English"))
|
||||
validate_tada_model_language(model_variant, language)
|
||||
speed_up_factor = float(self.config.get("speed_up_factor", 0.0) or 0.0)
|
||||
native_speed = speed_up_factor if speed_up_factor > 0 else None
|
||||
audio_component = self._audio_component(reference_audio, voice_ref)
|
||||
|
||||
generation_params = {
|
||||
"model_variant": model_variant,
|
||||
"decoder_source": "official_tada_codec_v1",
|
||||
"language": language,
|
||||
"reference_text": reference_text,
|
||||
"audio_component": audio_component,
|
||||
"seed": int(seed or 0),
|
||||
"acoustic_cfg_scale": float(self.config.get("acoustic_cfg_scale", 1.6)),
|
||||
"duration_cfg_scale": float(self.config.get("duration_cfg_scale", 1.0)),
|
||||
"noise_temperature": float(self.config.get("noise_temperature", 0.9)),
|
||||
"num_flow_matching_steps": int(self.config.get("num_flow_matching_steps", 10)),
|
||||
"cfg_schedule": self.config.get("cfg_schedule", "cosine"),
|
||||
"time_schedule": self.config.get("time_schedule", "logsnr"),
|
||||
"negative_condition_source": self.config.get(
|
||||
"negative_condition_source", "negative_step_output"
|
||||
),
|
||||
"speed_up_factor": native_speed,
|
||||
"num_transition_steps": int(self.config.get("num_transition_steps", 5)),
|
||||
"device": self.config.get("device", "auto"),
|
||||
"dtype": self.config.get("dtype", "auto"),
|
||||
"runtime_mode": self.config.get("runtime_mode", RUNTIME_MODE_SHARED),
|
||||
"runtime_profile": self.config.get(
|
||||
"runtime_profile", "vibevoice_transformers4_shared"
|
||||
),
|
||||
"character": character_name or "narrator",
|
||||
}
|
||||
|
||||
cache_key = None
|
||||
if enable_audio_cache:
|
||||
cache_key = self.audio_cache.generate_cache_key(
|
||||
"tada", text=stripped, **generation_params
|
||||
)
|
||||
cached = self.audio_cache.get_cached_audio(cache_key)
|
||||
if cached:
|
||||
print(
|
||||
f"💾 Using cached TADA audio for '{character_name or 'narrator'}': "
|
||||
f"'{stripped[:30]}...'"
|
||||
)
|
||||
return cached[0]
|
||||
|
||||
self._ensure_model_loaded(language)
|
||||
engine = self._get_engine()
|
||||
audio, sample_rate = engine.generate_speech(
|
||||
text=stripped,
|
||||
reference_audio=reference_audio,
|
||||
reference_text=reference_text,
|
||||
seed=generation_params["seed"],
|
||||
language=language,
|
||||
acoustic_cfg_scale=generation_params["acoustic_cfg_scale"],
|
||||
duration_cfg_scale=generation_params["duration_cfg_scale"],
|
||||
cfg_schedule=generation_params["cfg_schedule"],
|
||||
time_schedule=generation_params["time_schedule"],
|
||||
num_flow_matching_steps=generation_params["num_flow_matching_steps"],
|
||||
noise_temperature=generation_params["noise_temperature"],
|
||||
speed_up_factor=native_speed,
|
||||
num_transition_steps=generation_params["num_transition_steps"],
|
||||
negative_condition_source=generation_params["negative_condition_source"],
|
||||
)
|
||||
if int(sample_rate) != self.SAMPLE_RATE:
|
||||
raise RuntimeError(
|
||||
f"TADA returned {sample_rate} Hz audio; expected {self.SAMPLE_RATE} Hz"
|
||||
)
|
||||
if not isinstance(audio, torch.Tensor):
|
||||
audio = torch.as_tensor(audio, dtype=torch.float32)
|
||||
audio = audio.detach().cpu().float()
|
||||
if audio.dim() == 1:
|
||||
audio = audio.unsqueeze(0)
|
||||
elif audio.dim() != 2:
|
||||
raise RuntimeError(f"TADA returned an invalid audio shape: {tuple(audio.shape)}")
|
||||
if audio.shape[-1] == 0:
|
||||
raise RuntimeError("TADA returned empty audio")
|
||||
|
||||
if enable_audio_cache and cache_key:
|
||||
duration = self.audio_cache._calculate_duration(audio, "tada")
|
||||
self.audio_cache.cache_audio(cache_key, audio, duration)
|
||||
return audio
|
||||
|
||||
|
||||
__all__ = ["TadaEngineAdapter"]
|
||||
@@ -0,0 +1,6 @@
|
||||
"""Audio8 TTS engine integration."""
|
||||
|
||||
from .audio8_tts_engine import Audio8TTSEngine
|
||||
from .downloader import Audio8TTSDownloader
|
||||
|
||||
__all__ = ["Audio8TTSEngine", "Audio8TTSDownloader"]
|
||||
@@ -0,0 +1,375 @@
|
||||
"""Wrapper for the official Audio8 TTS remote-code model."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import random
|
||||
import warnings
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from engines.audio8_tts.downloader import Audio8TTSDownloader
|
||||
from engines.audio8_tts.progress import build_audio8_stopping_criteria
|
||||
from utils.device import resolve_torch_device
|
||||
|
||||
|
||||
class Audio8TTSEngine:
|
||||
"""ComfyUI-friendly wrapper around the official Audio8 model."""
|
||||
|
||||
SAMPLE_RATE = 44100
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_name: str = Audio8TTSDownloader.MODEL_NAME,
|
||||
device: str = "auto",
|
||||
dtype: str = "auto",
|
||||
model_dir: Optional[str] = None,
|
||||
):
|
||||
self.model_name = model_name
|
||||
self.device = resolve_torch_device(device)
|
||||
self.dtype_name = str(dtype or "auto").lower()
|
||||
self.dtype = self._resolve_dtype(self.dtype_name, self.device)
|
||||
self.model_dir = model_dir or Audio8TTSDownloader().resolve_model_path(
|
||||
model_name
|
||||
)
|
||||
self._processor = None
|
||||
self._model = None
|
||||
self._codec = None
|
||||
|
||||
@staticmethod
|
||||
def _resolve_dtype(dtype: str, device: str) -> torch.dtype:
|
||||
normalized = str(dtype or "auto").lower()
|
||||
valid = {
|
||||
"bfloat16": torch.bfloat16,
|
||||
"float16": torch.float16,
|
||||
"float32": torch.float32,
|
||||
}
|
||||
if normalized not in {"auto", *valid}:
|
||||
raise ValueError(
|
||||
"Audio8 TTS dtype must be auto, bfloat16, float16, or float32"
|
||||
)
|
||||
if str(device).startswith("cpu"):
|
||||
return torch.float32
|
||||
if normalized in valid:
|
||||
return valid[normalized]
|
||||
if str(device).startswith("cuda") and torch.cuda.is_available():
|
||||
major, _minor = torch.cuda.get_device_capability(torch.device(device))
|
||||
return torch.bfloat16 if major >= 8 else torch.float16
|
||||
if str(device).startswith("xpu"):
|
||||
return torch.bfloat16
|
||||
return torch.float16
|
||||
|
||||
def _ensure_model_loaded(self):
|
||||
"""Load the processor, model, and bundled codec strictly from local files."""
|
||||
if (
|
||||
self._processor is not None
|
||||
and self._model is not None
|
||||
and self._codec is not None
|
||||
):
|
||||
return self
|
||||
|
||||
if not os.path.isdir(self.model_dir):
|
||||
raise FileNotFoundError(
|
||||
f"Audio8 TTS local model directory does not exist: {self.model_dir}"
|
||||
)
|
||||
|
||||
try:
|
||||
from transformers import AutoModel, AutoProcessor
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
"Audio8 TTS requires the suite's shared Transformers 4 runtime"
|
||||
) from exc
|
||||
|
||||
print(
|
||||
f"📦 Loading Audio8 TTS from {self.model_dir} "
|
||||
f"on {self.device} ({self.dtype})"
|
||||
)
|
||||
with warnings.catch_warnings():
|
||||
warnings.filterwarnings(
|
||||
"ignore",
|
||||
message=(
|
||||
r"`torch\.nn\.utils\.weight_norm` is deprecated in favor of "
|
||||
r"`torch\.nn\.utils\.parametrizations\.weight_norm`\."
|
||||
),
|
||||
category=FutureWarning,
|
||||
)
|
||||
self._processor = AutoProcessor.from_pretrained(
|
||||
self.model_dir,
|
||||
trust_remote_code=True,
|
||||
local_files_only=True,
|
||||
)
|
||||
self._model = AutoModel.from_pretrained(
|
||||
self.model_dir,
|
||||
trust_remote_code=True,
|
||||
local_files_only=True,
|
||||
dtype=self.dtype,
|
||||
)
|
||||
self._model = self._model.eval().to(self.device)
|
||||
|
||||
# The official model stores the codec outside nn.Module registration.
|
||||
# Hold and move it explicitly so model offload/reload cannot strand it.
|
||||
self._codec = self._model.load_codec(
|
||||
device=self.device,
|
||||
dtype=self.dtype,
|
||||
)
|
||||
self._codec.eval()
|
||||
|
||||
sample_rate = int(
|
||||
getattr(self._model.config, "codec_sample_rate", self.SAMPLE_RATE)
|
||||
)
|
||||
if sample_rate != self.SAMPLE_RATE:
|
||||
raise RuntimeError(
|
||||
f"Audio8 TTS checkpoint reports unexpected sample rate "
|
||||
f"{sample_rate}; expected {self.SAMPLE_RATE}"
|
||||
)
|
||||
print("✅ Audio8 TTS model and codec loaded")
|
||||
return self
|
||||
|
||||
# Compatibility alias for callers using the shorter established name.
|
||||
_ensure_loaded = _ensure_model_loaded
|
||||
|
||||
def _clear_kv_caches(self) -> None:
|
||||
"""Drop every static slow/fast attention cache before device movement."""
|
||||
if self._model is None:
|
||||
return
|
||||
for collection_name in ("layers", "fast_layers"):
|
||||
for layer in getattr(self._model, collection_name, ()) or ():
|
||||
attention = getattr(layer, "attention", None)
|
||||
if attention is not None and hasattr(attention, "kv_cache"):
|
||||
attention.kv_cache = None
|
||||
|
||||
def to(self, device):
|
||||
"""Move all weights for ComfyUI Clear VRAM and reload handling."""
|
||||
target = resolve_torch_device(
|
||||
str(device) if not isinstance(device, str) else device
|
||||
)
|
||||
self._clear_kv_caches()
|
||||
self.device = target
|
||||
if self._model is not None:
|
||||
self._model = self._model.to(target).eval()
|
||||
if self._codec is not None:
|
||||
codec_dtype = torch.float32 if str(target).startswith("cpu") else self.dtype
|
||||
self._codec = self._codec.to(
|
||||
device=target,
|
||||
dtype=codec_dtype,
|
||||
).eval()
|
||||
return self
|
||||
|
||||
def parameters(self):
|
||||
"""Expose all weight tensors to ComfyUI memory accounting."""
|
||||
if self._model is not None:
|
||||
yield from self._model.parameters()
|
||||
if self._codec is not None:
|
||||
yield from self._codec.parameters()
|
||||
|
||||
@staticmethod
|
||||
def _mono_audio(waveform: Any) -> torch.Tensor:
|
||||
audio = torch.as_tensor(waveform, dtype=torch.float32).detach().cpu()
|
||||
if audio.ndim == 3:
|
||||
if audio.shape[0] != 1:
|
||||
raise ValueError(
|
||||
"Audio8 TTS reference audio must contain one batch item"
|
||||
)
|
||||
audio = audio[0]
|
||||
if audio.ndim == 2:
|
||||
audio = audio.mean(dim=0)
|
||||
if audio.ndim != 1 or audio.numel() == 0:
|
||||
raise ValueError(
|
||||
"Audio8 TTS reference audio must be non-empty mono or "
|
||||
"channels-first audio"
|
||||
)
|
||||
return audio.contiguous()
|
||||
|
||||
def _normalize_reference_audio(
|
||||
self,
|
||||
reference_audio: Any,
|
||||
) -> Tuple[Any, Optional[int]]:
|
||||
if isinstance(reference_audio, (str, os.PathLike)):
|
||||
return os.fspath(reference_audio), None
|
||||
if isinstance(reference_audio, dict):
|
||||
waveform = reference_audio.get(
|
||||
"waveform",
|
||||
reference_audio.get("array"),
|
||||
)
|
||||
sample_rate = reference_audio.get(
|
||||
"sample_rate",
|
||||
reference_audio.get("sampling_rate"),
|
||||
)
|
||||
if waveform is None or sample_rate is None:
|
||||
raise ValueError(
|
||||
"Audio8 TTS in-memory reference audio requires waveform "
|
||||
"and sample_rate"
|
||||
)
|
||||
return {
|
||||
"array": self._mono_audio(waveform),
|
||||
"sampling_rate": int(sample_rate),
|
||||
}, int(sample_rate)
|
||||
if isinstance(reference_audio, (tuple, list)) and len(reference_audio) == 2:
|
||||
waveform, sample_rate = reference_audio
|
||||
if not isinstance(sample_rate, (int, float)):
|
||||
raise ValueError(
|
||||
"Audio8 TTS reference tuple must be (waveform, sample_rate)"
|
||||
)
|
||||
return {
|
||||
"array": self._mono_audio(waveform),
|
||||
"sampling_rate": int(sample_rate),
|
||||
}, int(sample_rate)
|
||||
raise TypeError(
|
||||
f"Unsupported Audio8 TTS reference audio type: {type(reference_audio)}"
|
||||
)
|
||||
|
||||
def _prepare_inputs(
|
||||
self,
|
||||
text: str,
|
||||
reference_audio: Any,
|
||||
reference_text: Optional[str],
|
||||
) -> Dict[str, torch.Tensor]:
|
||||
processor_kwargs: Dict[str, Any] = {
|
||||
"text": [text],
|
||||
"return_tensors": "pt",
|
||||
}
|
||||
if reference_audio is not None:
|
||||
normalized_audio, sample_rate = self._normalize_reference_audio(
|
||||
reference_audio
|
||||
)
|
||||
processor_kwargs.update(
|
||||
reference_audio=[normalized_audio],
|
||||
reference_text=[reference_text],
|
||||
)
|
||||
if sample_rate is not None:
|
||||
processor_kwargs["sampling_rate"] = [sample_rate]
|
||||
|
||||
inputs = self._processor(**processor_kwargs)
|
||||
return {name: value.to(self.device) for name, value in inputs.items()}
|
||||
|
||||
def _make_generator(self, seed: int):
|
||||
seed = int(seed)
|
||||
random.seed(seed)
|
||||
torch.manual_seed(seed)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed_all(seed)
|
||||
try:
|
||||
return torch.Generator(device=torch.device(self.device)).manual_seed(seed)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _generate_once(
|
||||
self,
|
||||
inputs: Dict[str, torch.Tensor],
|
||||
*,
|
||||
max_new_tokens: int,
|
||||
temperature: float,
|
||||
top_p: float,
|
||||
top_k: int,
|
||||
do_sample: bool,
|
||||
generator,
|
||||
):
|
||||
criteria, tracker = build_audio8_stopping_criteria(max_new_tokens)
|
||||
try:
|
||||
output = self._model.generate(
|
||||
**inputs,
|
||||
max_new_tokens=max_new_tokens,
|
||||
temperature=temperature,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
do_sample=do_sample,
|
||||
generator=generator,
|
||||
stopping_criteria=criteria,
|
||||
return_dict_in_generate=True,
|
||||
)
|
||||
except BaseException:
|
||||
tracker.abort()
|
||||
raise
|
||||
tracker.close()
|
||||
return output
|
||||
|
||||
def generate(
|
||||
self,
|
||||
text: str,
|
||||
reference_audio: Any = None,
|
||||
reference_text: Optional[str] = None,
|
||||
*,
|
||||
max_new_tokens: int = 1024,
|
||||
retry_max_new_tokens: int = 2000,
|
||||
temperature: float = 0.8,
|
||||
top_p: float = 0.95,
|
||||
top_k: int = 50,
|
||||
do_sample: bool = True,
|
||||
seed: int = 42,
|
||||
) -> torch.Tensor:
|
||||
"""Generate one utterance as a CPU float tensor ``[channels, samples]``."""
|
||||
text = str(text or "").strip()
|
||||
if not text:
|
||||
return torch.zeros(1, 0, dtype=torch.float32)
|
||||
|
||||
reference_text = str(reference_text or "").strip()
|
||||
if reference_audio is not None and not reference_text:
|
||||
raise ValueError(
|
||||
"Audio8 TTS voice cloning requires the exact transcript of "
|
||||
"the reference audio"
|
||||
)
|
||||
if reference_audio is None and reference_text:
|
||||
raise ValueError(
|
||||
"Audio8 TTS reference_text requires matching reference audio"
|
||||
)
|
||||
|
||||
max_new_tokens = int(max_new_tokens)
|
||||
retry_max_new_tokens = int(retry_max_new_tokens)
|
||||
temperature = float(temperature)
|
||||
top_p = float(top_p)
|
||||
top_k = int(top_k)
|
||||
if max_new_tokens < 1:
|
||||
raise ValueError("Audio8 TTS max_new_tokens must be positive")
|
||||
if retry_max_new_tokens < max_new_tokens:
|
||||
raise ValueError(
|
||||
"Audio8 TTS retry_max_new_tokens must be >= max_new_tokens"
|
||||
)
|
||||
if temperature <= 0:
|
||||
raise ValueError("Audio8 TTS temperature must be positive")
|
||||
if not 0 < top_p <= 1:
|
||||
raise ValueError("Audio8 TTS top_p must be in (0, 1]")
|
||||
if top_k < 1:
|
||||
raise ValueError("Audio8 TTS top_k must be at least 1")
|
||||
|
||||
self._ensure_model_loaded()
|
||||
inputs = self._prepare_inputs(text, reference_audio, reference_text)
|
||||
generator = self._make_generator(seed)
|
||||
|
||||
output = self._generate_once(
|
||||
inputs,
|
||||
max_new_tokens=max_new_tokens,
|
||||
temperature=temperature,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
do_sample=bool(do_sample),
|
||||
generator=generator,
|
||||
)
|
||||
finished = bool(output.finished[0].item())
|
||||
|
||||
if not finished and retry_max_new_tokens > max_new_tokens:
|
||||
print(
|
||||
"⚠️ Audio8 TTS reached max_new_tokens without EOS; "
|
||||
f"retrying with {retry_max_new_tokens}"
|
||||
)
|
||||
output = self._generate_once(
|
||||
inputs,
|
||||
max_new_tokens=retry_max_new_tokens,
|
||||
temperature=temperature,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
do_sample=bool(do_sample),
|
||||
generator=generator,
|
||||
)
|
||||
finished = bool(output.finished[0].item())
|
||||
|
||||
if not finished:
|
||||
print(
|
||||
"⚠️ Audio8 TTS generation ended without EOS; returning the "
|
||||
"valid decoded frames"
|
||||
)
|
||||
|
||||
waveforms, waveform_lengths = self._model.decode_audio(output.codes)
|
||||
waveform_length = int(waveform_lengths[0].item())
|
||||
waveform = waveforms[0, :waveform_length].detach().float().cpu()
|
||||
return waveform.unsqueeze(0).contiguous()
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Organized model download and discovery for official Audio8 TTS."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import List, Optional
|
||||
|
||||
import folder_paths
|
||||
|
||||
from utils.downloads.unified_downloader import unified_downloader
|
||||
from utils.models.extra_paths import (
|
||||
get_all_tts_model_paths,
|
||||
get_preferred_download_path,
|
||||
)
|
||||
|
||||
|
||||
class Audio8TTSDownloader:
|
||||
"""Resolve Audio8 checkpoints without using the Hugging Face model cache."""
|
||||
|
||||
MODEL_NAME = "Audio8-TTS-Preview-0.6b"
|
||||
REPO_ID = "Audio8/Audio8-TTS-Preview-0.6b"
|
||||
REVISION = "1b17c91db5f4dccb6914aa4aa5cb0e56661a6c17"
|
||||
|
||||
# Complete runtime snapshot at the pinned revision. Non-runtime model-card
|
||||
# assets are deliberately excluded.
|
||||
REQUIRED_FILES = [
|
||||
"codec.pth",
|
||||
"config.json",
|
||||
"configuration_arktts.py",
|
||||
"generation_config.json",
|
||||
"model.safetensors",
|
||||
"modeling_arktts.py",
|
||||
"modeling_arktts_codec.py",
|
||||
"preprocessor_config.json",
|
||||
"processing_arktts.py",
|
||||
"processor_config.json",
|
||||
"special_tokens_map.json",
|
||||
"tokenizer.json",
|
||||
"tokenizer_config.json",
|
||||
]
|
||||
|
||||
def __init__(self, base_path: Optional[str] = None):
|
||||
if base_path is None:
|
||||
try:
|
||||
self.base_path = get_preferred_download_path(
|
||||
model_type="TTS",
|
||||
engine_name="audio8_tts",
|
||||
)
|
||||
except Exception:
|
||||
self.base_path = os.path.join(
|
||||
folder_paths.models_dir,
|
||||
"TTS",
|
||||
"audio8_tts",
|
||||
)
|
||||
else:
|
||||
self.base_path = os.path.abspath(os.fspath(base_path))
|
||||
os.makedirs(self.base_path, exist_ok=True)
|
||||
|
||||
def get_available_models(self) -> List[str]:
|
||||
"""Return the canonical model plus complete local installations."""
|
||||
models = [self.MODEL_NAME]
|
||||
canonical_path = os.path.normcase(
|
||||
os.path.abspath(os.path.join(self.base_path, self.MODEL_NAME))
|
||||
)
|
||||
for base_path in get_all_tts_model_paths("TTS"):
|
||||
for folder_name in ("audio8_tts", "Audio8_TTS", ""):
|
||||
root = (
|
||||
os.path.join(base_path, folder_name) if folder_name else base_path
|
||||
)
|
||||
if not os.path.isdir(root):
|
||||
continue
|
||||
for item in sorted(os.listdir(root)):
|
||||
candidate = os.path.join(root, item)
|
||||
if os.path.normcase(os.path.abspath(candidate)) == canonical_path:
|
||||
continue
|
||||
local_name = f"local:{item}"
|
||||
if local_name not in models and self._is_model_complete(candidate):
|
||||
models.append(local_name)
|
||||
return models
|
||||
|
||||
def resolve_model_path(
|
||||
self,
|
||||
model_identifier: str = MODEL_NAME,
|
||||
) -> str:
|
||||
"""Resolve an absolute path, ``local:`` name, or canonical model name."""
|
||||
model_identifier = str(model_identifier or self.MODEL_NAME).strip()
|
||||
|
||||
if os.path.isabs(model_identifier) or os.path.isdir(model_identifier):
|
||||
candidate = os.path.abspath(model_identifier)
|
||||
if self._is_model_complete(candidate, verbose=True):
|
||||
return candidate
|
||||
raise FileNotFoundError(
|
||||
f"Audio8 TTS model path is missing required files: {candidate}"
|
||||
)
|
||||
|
||||
if model_identifier.startswith("local:"):
|
||||
local_name = model_identifier[6:].strip()
|
||||
if not local_name:
|
||||
raise ValueError("Audio8 TTS local model name must not be empty")
|
||||
for base_path in get_all_tts_model_paths("TTS"):
|
||||
for folder_name in ("audio8_tts", "Audio8_TTS", ""):
|
||||
candidate = (
|
||||
os.path.join(base_path, folder_name, local_name)
|
||||
if folder_name
|
||||
else os.path.join(base_path, local_name)
|
||||
)
|
||||
if self._is_model_complete(candidate):
|
||||
print(f"📁 Using local Audio8 TTS model: {candidate}")
|
||||
return candidate
|
||||
raise FileNotFoundError(
|
||||
f"Local Audio8 TTS model not found or incomplete: {local_name}"
|
||||
)
|
||||
|
||||
if model_identifier != self.MODEL_NAME:
|
||||
raise ValueError(f"Unknown Audio8 TTS model: {model_identifier}")
|
||||
return self.get_model_path()
|
||||
|
||||
def get_model_path(self, model_name: str = MODEL_NAME) -> str:
|
||||
"""Return the organized canonical model path, downloading if needed."""
|
||||
if model_name != self.MODEL_NAME:
|
||||
raise ValueError(f"Unknown Audio8 TTS model: {model_name}")
|
||||
model_dir = os.path.join(self.base_path, self.MODEL_NAME)
|
||||
if not self._is_model_complete(model_dir):
|
||||
return self.download_model(model_dir)
|
||||
return model_dir
|
||||
|
||||
def download_model(
|
||||
self,
|
||||
model_dir: Optional[str] = None,
|
||||
force: bool = False,
|
||||
) -> str:
|
||||
"""Download the pinned official snapshot into the organized model folder."""
|
||||
model_dir = model_dir or os.path.join(self.base_path, self.MODEL_NAME)
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("📦 Audio8 TTS Model Download")
|
||||
print("=" * 60)
|
||||
print(f"Repository: {self.REPO_ID}")
|
||||
print(f"Revision: {self.REVISION}")
|
||||
print(f"Target: {model_dir}")
|
||||
print("=" * 60 + "\n")
|
||||
|
||||
unified_downloader.download_huggingface_snapshot(
|
||||
repo_id=self.REPO_ID,
|
||||
target_dir=model_dir,
|
||||
revision=self.REVISION,
|
||||
allow_patterns=self.REQUIRED_FILES,
|
||||
required_files=self.REQUIRED_FILES,
|
||||
force_download=bool(force),
|
||||
description=self.MODEL_NAME,
|
||||
)
|
||||
if not self._is_model_complete(model_dir, verbose=True):
|
||||
raise RuntimeError(
|
||||
f"Downloaded Audio8 TTS model is incomplete: {model_dir}"
|
||||
)
|
||||
print(f"✅ Audio8 TTS model ready: {model_dir}")
|
||||
return model_dir
|
||||
|
||||
def _is_model_complete(
|
||||
self,
|
||||
model_dir: str,
|
||||
*,
|
||||
verbose: bool = False,
|
||||
) -> bool:
|
||||
if not os.path.isdir(model_dir):
|
||||
return False
|
||||
missing = [
|
||||
rel_path
|
||||
for rel_path in self.REQUIRED_FILES
|
||||
if not os.path.isfile(os.path.join(model_dir, rel_path))
|
||||
or os.path.getsize(os.path.join(model_dir, rel_path)) <= 0
|
||||
]
|
||||
if missing and verbose:
|
||||
print(
|
||||
f"❌ Audio8 TTS model incomplete. Missing or empty "
|
||||
f"{len(missing)} file(s):"
|
||||
)
|
||||
for rel_path in missing:
|
||||
print(f" - {rel_path}")
|
||||
return not missing
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Native generation progress and interruption support for Audio8 TTS."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Any, Optional
|
||||
|
||||
import torch
|
||||
from transformers import StoppingCriteria, StoppingCriteriaList
|
||||
|
||||
|
||||
def _make_comfy_progress_bar(total_steps: int) -> Optional[Any]:
|
||||
try:
|
||||
from comfy.utils import ProgressBar
|
||||
|
||||
return ProgressBar(total_steps)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
class Audio8TTSStoppingCriteria(StoppingCriteria):
|
||||
"""Track native codec-frame generation and honor ComfyUI interruption."""
|
||||
|
||||
def __init__(self, max_steps: int, progress_bar: Optional[Any] = None):
|
||||
self.max_steps = max(1, int(max_steps))
|
||||
self.progress_bar = progress_bar
|
||||
self.steps = 0
|
||||
self.start_time = time.monotonic()
|
||||
self.last_print_time = self.start_time
|
||||
self.last_print_step = 0
|
||||
self.rendered = False
|
||||
|
||||
@staticmethod
|
||||
def _check_interrupted() -> None:
|
||||
try:
|
||||
import comfy.model_management as model_management
|
||||
except Exception:
|
||||
return
|
||||
|
||||
checker = getattr(
|
||||
model_management,
|
||||
"throw_exception_if_processing_interrupted",
|
||||
None,
|
||||
)
|
||||
if callable(checker):
|
||||
checker()
|
||||
elif getattr(model_management, "interrupt_processing", False):
|
||||
raise InterruptedError("Audio8 TTS generation interrupted by user")
|
||||
|
||||
def __call__(self, input_ids, scores, **kwargs):
|
||||
del scores, kwargs
|
||||
self._check_interrupted()
|
||||
self.steps = min(self.steps + 1, self.max_steps)
|
||||
if self.progress_bar is not None:
|
||||
try:
|
||||
self.progress_bar.update(1)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
now = time.monotonic()
|
||||
if (
|
||||
self.steps == 1
|
||||
or self.steps >= self.max_steps
|
||||
or now - self.last_print_time >= 0.5
|
||||
):
|
||||
delta_time = max(now - self.last_print_time, 1e-6)
|
||||
delta_steps = self.steps - self.last_print_step
|
||||
rate = delta_steps / delta_time
|
||||
elapsed = now - self.start_time
|
||||
width = 12
|
||||
filled = min(
|
||||
width,
|
||||
int(width * self.steps / self.max_steps),
|
||||
)
|
||||
bar = "█" * filled + "░" * (width - filled)
|
||||
print(
|
||||
f"\r Audio8: [{bar}] {self.steps}/{self.max_steps} | "
|
||||
f"{rate:.1f} frames/s | {elapsed:.1f}s ",
|
||||
end="",
|
||||
flush=True,
|
||||
)
|
||||
self.last_print_time = now
|
||||
self.last_print_step = self.steps
|
||||
self.rendered = True
|
||||
|
||||
# The official model handles EOS itself. Interruption raises above.
|
||||
batch_size = int(input_ids.shape[0]) if input_ids.ndim else 1
|
||||
return torch.zeros(
|
||||
batch_size,
|
||||
dtype=torch.bool,
|
||||
device=input_ids.device,
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
elapsed = max(time.monotonic() - self.start_time, 1e-6)
|
||||
average = self.steps / elapsed
|
||||
if self.rendered:
|
||||
print(
|
||||
f"\r Audio8 complete: {self.steps} frames in "
|
||||
f"{elapsed:.1f}s ({average:.1f} frames/s)" + " " * 20
|
||||
)
|
||||
|
||||
def abort(self) -> None:
|
||||
if self.rendered:
|
||||
print()
|
||||
|
||||
|
||||
def build_audio8_stopping_criteria(
|
||||
max_steps: int,
|
||||
) -> tuple[StoppingCriteriaList, Audio8TTSStoppingCriteria]:
|
||||
"""Create the native stopping-criteria list and its progress tracker."""
|
||||
tracker = Audio8TTSStoppingCriteria(
|
||||
max_steps=max_steps,
|
||||
progress_bar=_make_comfy_progress_bar(max(1, int(max_steps))),
|
||||
)
|
||||
return StoppingCriteriaList([tracker]), tracker
|
||||
@@ -1,32 +0,0 @@
|
||||
"""Hume TADA integration with lazy exports safe for the worker runtime."""
|
||||
|
||||
from .languages import (
|
||||
TADA_LANGUAGE_OPTIONS,
|
||||
format_tada_language_display,
|
||||
normalize_tada_language,
|
||||
validate_tada_model_language,
|
||||
)
|
||||
|
||||
|
||||
def __getattr__(name):
|
||||
if name in {"TadaDownloader", "tada_downloader"}:
|
||||
from .tada_downloader import TadaDownloader, tada_downloader
|
||||
|
||||
return {"TadaDownloader": TadaDownloader, "tada_downloader": tada_downloader}[name]
|
||||
if name in {"TADA_SAMPLE_RATE", "TadaEngine"}:
|
||||
from .tada_engine import TADA_SAMPLE_RATE, TadaEngine
|
||||
|
||||
return {"TADA_SAMPLE_RATE": TADA_SAMPLE_RATE, "TadaEngine": TadaEngine}[name]
|
||||
raise AttributeError(name)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"TADA_LANGUAGE_OPTIONS",
|
||||
"TADA_SAMPLE_RATE",
|
||||
"TadaDownloader",
|
||||
"TadaEngine",
|
||||
"format_tada_language_display",
|
||||
"normalize_tada_language",
|
||||
"tada_downloader",
|
||||
"validate_tada_model_language",
|
||||
]
|
||||
@@ -1,91 +0,0 @@
|
||||
"""Language metadata and validation for the official Hume TADA checkpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
|
||||
_LANGUAGE_DISPLAY_TO_CODE = {
|
||||
"English": None,
|
||||
"Arabic": "ar",
|
||||
"Chinese": "ch",
|
||||
"German": "de",
|
||||
"Spanish": "es",
|
||||
"French": "fr",
|
||||
"Italian": "it",
|
||||
"Japanese": "ja",
|
||||
"Polish": "pl",
|
||||
"Portuguese": "pt",
|
||||
}
|
||||
|
||||
TADA_LANGUAGE_OPTIONS = tuple(_LANGUAGE_DISPLAY_TO_CODE)
|
||||
|
||||
_LANGUAGE_ALIASES = {
|
||||
"": None,
|
||||
"auto": None,
|
||||
"default": None,
|
||||
"en": None,
|
||||
"eng": None,
|
||||
"english": None,
|
||||
"ar": "ar",
|
||||
"arabic": "ar",
|
||||
"ch": "ch",
|
||||
"zh": "ch",
|
||||
"zh-cn": "ch",
|
||||
"chinese": "ch",
|
||||
"de": "de",
|
||||
"german": "de",
|
||||
"es": "es",
|
||||
"spanish": "es",
|
||||
"fr": "fr",
|
||||
"french": "fr",
|
||||
"it": "it",
|
||||
"italian": "it",
|
||||
"ja": "ja",
|
||||
"jp": "ja",
|
||||
"japanese": "ja",
|
||||
"pl": "pl",
|
||||
"polish": "pl",
|
||||
"pt": "pt",
|
||||
"pt-br": "pt",
|
||||
"portuguese": "pt",
|
||||
}
|
||||
|
||||
|
||||
def normalize_tada_language(value: object) -> Optional[str]:
|
||||
"""Return the official aligner code, using ``None`` for English."""
|
||||
if value is None:
|
||||
return None
|
||||
normalized = str(value).strip().lower().replace("_", "-")
|
||||
if normalized not in _LANGUAGE_ALIASES:
|
||||
supported = ", ".join(TADA_LANGUAGE_OPTIONS)
|
||||
raise ValueError(f"Unsupported TADA language '{value}'. Supported languages: {supported}")
|
||||
return _LANGUAGE_ALIASES[normalized]
|
||||
|
||||
|
||||
def format_tada_language_display(value: object) -> str:
|
||||
"""Return the canonical UI label for a language value or backend code."""
|
||||
code = normalize_tada_language(value)
|
||||
for display, candidate in _LANGUAGE_DISPLAY_TO_CODE.items():
|
||||
if candidate == code:
|
||||
return display
|
||||
return "English"
|
||||
|
||||
|
||||
def validate_tada_model_language(model_name: str, language: object) -> Optional[str]:
|
||||
"""Validate checkpoint language support and return its backend aligner code."""
|
||||
code = normalize_tada_language(language)
|
||||
normalized_model = str(model_name or "").lower().replace("_", "-")
|
||||
if code is not None and ("tada-1b" in normalized_model or normalized_model == "1b"):
|
||||
raise ValueError(
|
||||
"TADA-1B is English-only. Select TADA-3B-ML for multilingual synthesis."
|
||||
)
|
||||
return code
|
||||
|
||||
|
||||
__all__ = [
|
||||
"TADA_LANGUAGE_OPTIONS",
|
||||
"format_tada_language_display",
|
||||
"normalize_tada_language",
|
||||
"validate_tada_model_language",
|
||||
]
|
||||
@@ -1,322 +0,0 @@
|
||||
"""Strict, organized downloader for the official Hume TADA assets."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import folder_paths
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
from utils.hf_download_logging import quiet_hf_download_logs
|
||||
from utils.models.extra_paths import get_all_tts_model_paths, get_preferred_download_path
|
||||
|
||||
from .languages import normalize_tada_language, validate_tada_model_language
|
||||
|
||||
|
||||
class TadaDownloader:
|
||||
"""Download only the files used by TTS Audio Suite's local-only TADA loader."""
|
||||
|
||||
MODELS = {
|
||||
"TADA-1B": {
|
||||
"repo_id": "HumeAI/tada-1b",
|
||||
"files": [
|
||||
"config.json",
|
||||
"generation_config.json",
|
||||
"model.safetensors",
|
||||
],
|
||||
},
|
||||
"TADA-3B-ML": {
|
||||
"repo_id": "HumeAI/tada-3b-ml",
|
||||
"files": [
|
||||
"config.json",
|
||||
"generation_config.json",
|
||||
"model.safetensors.index.json",
|
||||
"model-00001-of-00002.safetensors",
|
||||
"model-00002-of-00002.safetensors",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
# Ungated redistribution whose three tokenizer files are byte-identical to
|
||||
# Meta's official Llama 3.2 1B repository. Keep the official repo as the
|
||||
# provenance reference and verify Git blob hashes after every install.
|
||||
TOKENIZER_REPO = "onnx-community/Llama-3.2-1B"
|
||||
TOKENIZER_REFERENCE_REPO = "meta-llama/Llama-3.2-1B"
|
||||
TOKENIZER_DIRECTORY = "llama-3.2-1b-tokenizer"
|
||||
TOKENIZER_FILES = [
|
||||
"special_tokens_map.json",
|
||||
"tokenizer.json",
|
||||
"tokenizer_config.json",
|
||||
]
|
||||
TOKENIZER_DISTRIBUTION_FILES = TOKENIZER_FILES + ["LICENSE.txt", "USE_POLICY.md"]
|
||||
TOKENIZER_GIT_BLOB_SHA1 = {
|
||||
"special_tokens_map.json": "cfabacc2620186cd3dd4b1dde9a37e057208636e",
|
||||
"tokenizer.json": "5cc5f00a5b203e90a27a3bd60d1ec393b07971e8",
|
||||
"tokenizer_config.json": "cb9ec25536e44d86778b10509d3e5bdca459a5cf",
|
||||
}
|
||||
|
||||
CODEC_REPO = "HumeAI/tada-codec"
|
||||
CODEC_DIRECTORY = "tada-codec"
|
||||
CODEC_BASE_FILES = [
|
||||
"encoder/config.json",
|
||||
"encoder/model.safetensors",
|
||||
"decoder/config.json",
|
||||
"decoder/model.safetensors",
|
||||
]
|
||||
WAV2VEC_REPO = "facebook/wav2vec2-large"
|
||||
WAV2VEC_CONFIG_LOCAL = "wav2vec2-large/config.json"
|
||||
|
||||
_ALIASES = {
|
||||
"tada-1b": "TADA-1B",
|
||||
"1b": "TADA-1B",
|
||||
"humeai/tada-1b": "TADA-1B",
|
||||
"tada-3b-ml": "TADA-3B-ML",
|
||||
"tada-3b": "TADA-3B-ML",
|
||||
"3b": "TADA-3B-ML",
|
||||
"3b-ml": "TADA-3B-ML",
|
||||
"humeai/tada-3b-ml": "TADA-3B-ML",
|
||||
}
|
||||
|
||||
def __init__(self, base_path: Optional[str] = None):
|
||||
if base_path is None:
|
||||
try:
|
||||
base_path = get_preferred_download_path(model_type="TTS", engine_name="tada")
|
||||
except Exception:
|
||||
base_path = os.path.join(folder_paths.models_dir, "TTS", "tada")
|
||||
self.base_path = os.path.abspath(str(base_path))
|
||||
os.makedirs(self.base_path, exist_ok=True)
|
||||
|
||||
@classmethod
|
||||
def canonical_model_name(cls, model_identifier: str) -> Optional[str]:
|
||||
value = str(model_identifier or "TADA-1B").strip()
|
||||
if value in cls.MODELS:
|
||||
return value
|
||||
return cls._ALIASES.get(value.lower().replace("_", "-"))
|
||||
|
||||
@staticmethod
|
||||
def _files_ready(target_dir: str, files: List[str]) -> bool:
|
||||
return all(
|
||||
os.path.isfile(os.path.join(target_dir, name))
|
||||
and os.path.getsize(os.path.join(target_dir, name)) > 0
|
||||
for name in files
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _git_blob_sha1(path: str) -> str:
|
||||
size = os.path.getsize(path)
|
||||
digest = hashlib.sha1()
|
||||
digest.update(f"blob {size}\0".encode("ascii"))
|
||||
with open(path, "rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
@classmethod
|
||||
def _validate_tokenizer_files(cls, target_dir: str) -> None:
|
||||
mismatches = []
|
||||
for filename, expected in cls.TOKENIZER_GIT_BLOB_SHA1.items():
|
||||
path = os.path.join(target_dir, filename)
|
||||
actual = cls._git_blob_sha1(path)
|
||||
if actual != expected:
|
||||
mismatches.append(f"{filename} ({actual})")
|
||||
if mismatches:
|
||||
raise RuntimeError(
|
||||
"The installed TADA tokenizer does not match Meta's official Llama 3.2 1B "
|
||||
"tokenizer. Remove the tokenizer directory and retry. Mismatched files: "
|
||||
+ ", ".join(mismatches)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _download_files(
|
||||
repo_id: str,
|
||||
target_dir: str,
|
||||
files: List[str],
|
||||
label: str,
|
||||
force: bool,
|
||||
) -> str:
|
||||
if not force and TadaDownloader._files_ready(target_dir, files):
|
||||
return target_dir
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
print(f"\n{'=' * 60}")
|
||||
print("📦 TADA Asset Download")
|
||||
print(f"{'=' * 60}")
|
||||
print(f"Asset: {label}")
|
||||
print(f"Repository: {repo_id}")
|
||||
print(f"Target: {target_dir}")
|
||||
print(f"Files: {len(files)}")
|
||||
print(f"{'=' * 60}\n")
|
||||
try:
|
||||
with quiet_hf_download_logs():
|
||||
snapshot_download(
|
||||
repo_id=repo_id,
|
||||
local_dir=target_dir,
|
||||
allow_patterns=files,
|
||||
force_download=force,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f"Failed to download {label} from {repo_id}: {exc}") from exc
|
||||
|
||||
if not TadaDownloader._files_ready(target_dir, files):
|
||||
missing = [name for name in files if not os.path.isfile(os.path.join(target_dir, name))]
|
||||
raise RuntimeError(
|
||||
f"TADA asset download for {label} is incomplete. Missing files: {missing}"
|
||||
)
|
||||
print(f"✅ TADA asset ready: {target_dir}")
|
||||
return target_dir
|
||||
|
||||
def _search_roots(self) -> List[str]:
|
||||
roots = [self.base_path]
|
||||
try:
|
||||
roots.extend(os.path.join(path, "tada") for path in get_all_tts_model_paths("TTS"))
|
||||
except Exception:
|
||||
pass
|
||||
seen = set()
|
||||
return [path for path in roots if not (path in seen or seen.add(path))]
|
||||
|
||||
def _resolve_local_model(self, identifier: str) -> str:
|
||||
value = str(identifier).removeprefix("local:")
|
||||
if os.path.isdir(value):
|
||||
return os.path.abspath(value)
|
||||
for root in self._search_roots():
|
||||
candidate = os.path.join(root, value)
|
||||
if os.path.isdir(candidate):
|
||||
return candidate
|
||||
raise FileNotFoundError(
|
||||
f"Local TADA checkpoint '{value}' was not found under the configured TTS/tada paths."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def is_model_complete(cls, model_path: str) -> bool:
|
||||
if not os.path.isfile(os.path.join(model_path, "config.json")):
|
||||
return False
|
||||
try:
|
||||
return any(
|
||||
name.endswith(".safetensors")
|
||||
and os.path.getsize(os.path.join(model_path, name)) > 0
|
||||
for name in os.listdir(model_path)
|
||||
)
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
def get_available_models(self) -> List[str]:
|
||||
models = list(self.MODELS)
|
||||
local_names = set()
|
||||
for root in self._search_roots():
|
||||
if not os.path.isdir(root):
|
||||
continue
|
||||
try:
|
||||
entries = os.listdir(root)
|
||||
except OSError:
|
||||
continue
|
||||
for entry in entries:
|
||||
candidate = os.path.join(root, entry)
|
||||
if entry not in self.MODELS and self.is_model_complete(candidate):
|
||||
local_names.add(f"local:{entry}")
|
||||
return models + sorted(local_names)
|
||||
|
||||
def ensure_model(self, model_identifier: str, force: bool = False) -> tuple[str, str]:
|
||||
canonical = self.canonical_model_name(model_identifier)
|
||||
if canonical is None:
|
||||
model_path = self._resolve_local_model(model_identifier)
|
||||
if not self.is_model_complete(model_path):
|
||||
raise RuntimeError(f"Local TADA checkpoint is incomplete: {model_path}")
|
||||
return os.path.basename(os.path.normpath(model_path)), model_path
|
||||
|
||||
details = self.MODELS[canonical]
|
||||
target = os.path.join(self.base_path, canonical)
|
||||
try:
|
||||
self._download_files(
|
||||
details["repo_id"], target, details["files"], canonical, force
|
||||
)
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
f"Unable to download the public {canonical} checkpoint. Check the network, "
|
||||
f"or manually place the required checkpoint files in "
|
||||
f"ComfyUI/models/TTS/tada/{canonical}/, "
|
||||
f"then retry. Download error: {exc}"
|
||||
) from exc
|
||||
return canonical, target
|
||||
|
||||
def ensure_tokenizer(self, force: bool = False) -> str:
|
||||
target = os.path.join(self.base_path, self.TOKENIZER_DIRECTORY)
|
||||
if not force and self._files_ready(target, self.TOKENIZER_FILES):
|
||||
self._validate_tokenizer_files(target)
|
||||
return target
|
||||
try:
|
||||
result = self._download_files(
|
||||
self.TOKENIZER_REPO,
|
||||
target,
|
||||
self.TOKENIZER_DISTRIBUTION_FILES,
|
||||
"Llama-3.2-1B tokenizer",
|
||||
force,
|
||||
)
|
||||
self._validate_tokenizer_files(result)
|
||||
return result
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
"Unable to download the TADA tokenizer automatically from the exact ungated "
|
||||
f"redistribution https://huggingface.co/{self.TOKENIZER_REPO}.\n\n"
|
||||
"Manual alternative: copy these three exact Meta Llama 3.2 1B files:\n"
|
||||
"ComfyUI/models/TTS/tada/\n"
|
||||
"└── llama-3.2-1b-tokenizer/\n"
|
||||
" ├── tokenizer.json\n"
|
||||
" ├── tokenizer_config.json\n"
|
||||
" └── special_tokens_map.json\n\n"
|
||||
f"Download error: {exc}"
|
||||
) from exc
|
||||
|
||||
def ensure_codec_assets(
|
||||
self,
|
||||
language: object = None,
|
||||
force: bool = False,
|
||||
codec_path: Optional[str] = None,
|
||||
) -> str:
|
||||
code = normalize_tada_language(language)
|
||||
target = os.path.abspath(codec_path or os.path.join(self.base_path, self.CODEC_DIRECTORY))
|
||||
aligner_dir = "aligner" if code is None else f"aligner-{code}"
|
||||
codec_files = self.CODEC_BASE_FILES + [
|
||||
f"{aligner_dir}/config.json",
|
||||
f"{aligner_dir}/model.safetensors",
|
||||
]
|
||||
self._download_files(self.CODEC_REPO, target, codec_files, "TADA codec", force)
|
||||
|
||||
wav2vec_file = os.path.join(target, self.WAV2VEC_CONFIG_LOCAL)
|
||||
if force or not os.path.isfile(wav2vec_file) or os.path.getsize(wav2vec_file) == 0:
|
||||
self._download_files(
|
||||
self.WAV2VEC_REPO,
|
||||
os.path.dirname(wav2vec_file),
|
||||
["config.json"],
|
||||
"TADA Wav2Vec aligner configuration",
|
||||
force,
|
||||
)
|
||||
return target
|
||||
|
||||
def resolve_model_assets(
|
||||
self,
|
||||
model_identifier: str,
|
||||
language: object = None,
|
||||
force: bool = False,
|
||||
) -> Dict[str, str]:
|
||||
canonical = self.canonical_model_name(model_identifier)
|
||||
validation_name = canonical or os.path.basename(str(model_identifier))
|
||||
validate_tada_model_language(validation_name, language)
|
||||
# Validate the small tokenizer first so source/hash failures happen
|
||||
# before several gigabytes of TADA assets are downloaded.
|
||||
tokenizer_path = self.ensure_tokenizer(force=force)
|
||||
model_name, model_path = self.ensure_model(model_identifier, force=force)
|
||||
codec_path = self.ensure_codec_assets(language=language, force=force)
|
||||
return {
|
||||
"model_name": model_name,
|
||||
"model_path": model_path,
|
||||
"codec_path": codec_path,
|
||||
"tokenizer_path": tokenizer_path,
|
||||
}
|
||||
|
||||
|
||||
tada_downloader = TadaDownloader()
|
||||
|
||||
|
||||
__all__ = ["TadaDownloader", "tada_downloader"]
|
||||
@@ -1,567 +0,0 @@
|
||||
"""Local-only wrapper around the official ``hume-tada`` inference classes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import gc
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
from collections import OrderedDict
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import fields
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from .languages import normalize_tada_language, validate_tada_model_language
|
||||
|
||||
|
||||
TADA_SAMPLE_RATE = 24000
|
||||
|
||||
|
||||
class TadaEngine:
|
||||
"""Official TADA inference with every model dependency resolved locally."""
|
||||
|
||||
_CFG_SCHEDULES = {"constant", "linear", "cosine"}
|
||||
_TIME_SCHEDULES = {"uniform", "cosine", "logsnr"}
|
||||
_NEGATIVE_SOURCES = {"negative_step_output", "prompt", "zero"}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_name: str,
|
||||
model_path: str,
|
||||
codec_path: str,
|
||||
tokenizer_path: str,
|
||||
device: str = "auto",
|
||||
dtype: str = "auto",
|
||||
attn_implementation: str = "sdpa",
|
||||
use_torch_compile: bool = False,
|
||||
prompt_cache_size: int = 8,
|
||||
):
|
||||
self.model_name = str(model_name)
|
||||
self.model_path = os.path.abspath(str(model_path))
|
||||
self.codec_path = os.path.abspath(str(codec_path))
|
||||
self.tokenizer_path = os.path.abspath(str(tokenizer_path))
|
||||
self.device = self._resolve_device(device)
|
||||
self.dtype = self._resolve_dtype(dtype)
|
||||
self.attn_implementation = str(attn_implementation or "sdpa")
|
||||
self.use_torch_compile = bool(use_torch_compile)
|
||||
self.prompt_cache_size = max(0, int(prompt_cache_size))
|
||||
self._prompt_cache: OrderedDict[str, Dict[str, Any]] = OrderedDict()
|
||||
self.model = None
|
||||
|
||||
self._validate_local_assets()
|
||||
self._load_model()
|
||||
|
||||
@staticmethod
|
||||
def _resolve_device(device: object) -> torch.device:
|
||||
value = str(device or "auto")
|
||||
if value == "auto":
|
||||
value = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
if value.startswith("cuda") and not torch.cuda.is_available():
|
||||
raise RuntimeError("TADA was configured for CUDA, but CUDA is not available in its shared runtime.")
|
||||
return torch.device(value)
|
||||
|
||||
def _resolve_dtype(self, dtype: object) -> torch.dtype:
|
||||
value = str(dtype or "auto").lower()
|
||||
if value == "auto":
|
||||
if self.device.type == "cuda":
|
||||
return torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16
|
||||
return torch.float32
|
||||
mapping = {
|
||||
"bfloat16": torch.bfloat16,
|
||||
"bf16": torch.bfloat16,
|
||||
"float16": torch.float16,
|
||||
"fp16": torch.float16,
|
||||
"float32": torch.float32,
|
||||
"fp32": torch.float32,
|
||||
}
|
||||
if value not in mapping:
|
||||
raise ValueError(f"Unsupported TADA dtype '{dtype}'")
|
||||
if self.device.type == "cpu" and mapping[value] == torch.float16:
|
||||
return torch.float32
|
||||
return mapping[value]
|
||||
|
||||
@staticmethod
|
||||
def _require_files(root: str, relative_paths: Tuple[str, ...], label: str) -> None:
|
||||
missing = [name for name in relative_paths if not os.path.isfile(os.path.join(root, name))]
|
||||
if missing:
|
||||
raise RuntimeError(f"Local {label} assets are incomplete at {root}. Missing: {missing}")
|
||||
|
||||
def _validate_local_assets(self) -> None:
|
||||
model_files = ("config.json",)
|
||||
tokenizer_files = ("tokenizer.json", "tokenizer_config.json", "special_tokens_map.json")
|
||||
codec_files = (
|
||||
"encoder/config.json",
|
||||
"encoder/model.safetensors",
|
||||
"decoder/config.json",
|
||||
"decoder/model.safetensors",
|
||||
"wav2vec2-large/config.json",
|
||||
)
|
||||
self._require_files(self.model_path, model_files, "TADA checkpoint")
|
||||
if not any(Path(self.model_path).glob("*.safetensors")):
|
||||
raise RuntimeError(f"Local TADA checkpoint has no safetensors weights: {self.model_path}")
|
||||
self._require_files(self.tokenizer_path, tokenizer_files, "Llama tokenizer")
|
||||
self._require_files(self.codec_path, codec_files, "TADA codec")
|
||||
|
||||
def _load_model(self) -> None:
|
||||
try:
|
||||
import transformers
|
||||
from tada.modules.decoder import Decoder
|
||||
from tada.modules.tada import TadaForCausalLM
|
||||
from transformers import AutoTokenizer, LlamaForCausalLM
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
"TADA requires the shared Transformers 4 runtime with hume-tada installed. "
|
||||
"Select the Shared Runtime option and retry. "
|
||||
f"Import error: {exc}"
|
||||
) from exc
|
||||
|
||||
major = int(str(transformers.__version__).split(".", 1)[0])
|
||||
if major >= 5:
|
||||
raise RuntimeError(
|
||||
f"TADA is not compatible with Transformers {transformers.__version__} in the main runtime. "
|
||||
"Use the vibevoice_transformers4_shared runtime profile."
|
||||
)
|
||||
|
||||
load_kwargs = {
|
||||
"local_files_only": True,
|
||||
"low_cpu_mem_usage": True,
|
||||
"dtype": self.dtype,
|
||||
}
|
||||
if self.attn_implementation not in {"", "auto"}:
|
||||
load_kwargs["attn_implementation"] = self.attn_implementation
|
||||
|
||||
try:
|
||||
# Calling the parent classmethod directly bypasses TADA's override,
|
||||
# which otherwise downloads its decoder and tokenizer by repository ID.
|
||||
class _ExpectedBundledDecoderWarning(logging.Filter):
|
||||
def filter(self, record):
|
||||
message = record.getMessage()
|
||||
return not (
|
||||
"were not used when initializing TadaForCausalLM" in message
|
||||
and "_decoder." in message
|
||||
)
|
||||
|
||||
warning_filter = _ExpectedBundledDecoderWarning()
|
||||
model_logger = logging.getLogger("transformers.modeling_utils")
|
||||
model_logger.addFilter(warning_filter)
|
||||
try:
|
||||
model, loading_info = LlamaForCausalLM.from_pretrained.__func__(
|
||||
TadaForCausalLM,
|
||||
self.model_path,
|
||||
output_loading_info=True,
|
||||
**load_kwargs,
|
||||
)
|
||||
finally:
|
||||
model_logger.removeFilter(warning_filter)
|
||||
|
||||
unexpected = loading_info.get("unexpected_keys", [])
|
||||
non_decoder = [key for key in unexpected if not key.startswith("_decoder.")]
|
||||
if non_decoder:
|
||||
raise RuntimeError(
|
||||
"TADA checkpoint contains unexpected non-decoder weights: "
|
||||
f"{non_decoder[:10]}"
|
||||
)
|
||||
if not unexpected:
|
||||
raise RuntimeError(
|
||||
"TADA checkpoint did not expose the expected bundled decoder keys. "
|
||||
"Its layout may be incompatible with this hume-tada runtime."
|
||||
)
|
||||
# This matches hume-tada's official from_pretrained contract: the
|
||||
# checkpoint's bundled _decoder keys are intentionally ignored and
|
||||
# the released tada-codec decoder is attached instead.
|
||||
decoder = Decoder.from_pretrained(
|
||||
self.codec_path,
|
||||
subfolder="decoder",
|
||||
local_files_only=True,
|
||||
low_cpu_mem_usage=True,
|
||||
)
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
self.tokenizer_path,
|
||||
local_files_only=True,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f"Failed to load TADA entirely from local assets: {exc}") from exc
|
||||
|
||||
model._decoder = decoder
|
||||
model._tokenizer = tokenizer
|
||||
model.to(self.device)
|
||||
model.eval()
|
||||
if self.use_torch_compile:
|
||||
self._enable_torch_compile(model)
|
||||
self.model = model
|
||||
|
||||
def _enable_torch_compile(self, model) -> None:
|
||||
if self.device.type != "cuda":
|
||||
print("⚠️ TADA: torch.compile is only enabled on CUDA; using standard inference.")
|
||||
self.use_torch_compile = False
|
||||
return
|
||||
try:
|
||||
import triton # noqa: F401
|
||||
except Exception as exc:
|
||||
print(f"⚠️ TADA: torch.compile requires a working Triton install; using standard inference. ({exc})")
|
||||
self.use_torch_compile = False
|
||||
return
|
||||
|
||||
original_forward = model.prediction_head.forward
|
||||
try:
|
||||
compiled_forward = torch.compile(original_forward, mode="default")
|
||||
except Exception as exc:
|
||||
print(f"⚠️ TADA: could not prepare torch.compile; using standard inference. ({exc})")
|
||||
self.use_torch_compile = False
|
||||
return
|
||||
|
||||
failed = False
|
||||
|
||||
def compile_with_fallback(*args, **kwargs):
|
||||
nonlocal failed
|
||||
try:
|
||||
return compiled_forward(*args, **kwargs)
|
||||
except Exception as exc:
|
||||
if not failed:
|
||||
print(f"⚠️ TADA: compiled refinement failed; falling back to standard inference. ({exc})")
|
||||
failed = True
|
||||
model.prediction_head.forward = original_forward
|
||||
self.use_torch_compile = False
|
||||
return original_forward(*args, **kwargs)
|
||||
|
||||
model.prediction_head.forward = compile_with_fallback
|
||||
print("✅ TADA: torch.compile enabled for audio refinement (first generation will compile).")
|
||||
|
||||
def _required_aligner_subfolder(self, language: object) -> str:
|
||||
code = validate_tada_model_language(self.model_name, language)
|
||||
return "aligner" if code is None else f"aligner-{code}"
|
||||
|
||||
def _load_encoder(self, language: object):
|
||||
from tada.modules import aligner as aligner_module
|
||||
from tada.modules.aligner import Aligner, AlignerConfig
|
||||
from tada.modules.encoder import Encoder, EncoderConfig
|
||||
from transformers import PreTrainedModel, Wav2Vec2Config
|
||||
|
||||
aligner_subfolder = self._required_aligner_subfolder(language)
|
||||
self._require_files(
|
||||
self.codec_path,
|
||||
(
|
||||
f"{aligner_subfolder}/config.json",
|
||||
f"{aligner_subfolder}/model.safetensors",
|
||||
),
|
||||
f"TADA {aligner_subfolder}",
|
||||
)
|
||||
|
||||
wav2vec_config = Wav2Vec2Config.from_pretrained(
|
||||
os.path.join(self.codec_path, "wav2vec2-large"),
|
||||
local_files_only=True,
|
||||
)
|
||||
|
||||
# Aligner.__init__ normally resolves both names over the network. Supply
|
||||
# the already-loaded local tokenizer and explicit local Wav2Vec config.
|
||||
with patch.object(
|
||||
aligner_module.AutoTokenizer,
|
||||
"from_pretrained",
|
||||
return_value=self.model.tokenizer,
|
||||
), patch.object(
|
||||
aligner_module.AutoConfig,
|
||||
"from_pretrained",
|
||||
return_value=wav2vec_config,
|
||||
):
|
||||
aligner = Aligner.from_pretrained(
|
||||
self.codec_path,
|
||||
subfolder=aligner_subfolder,
|
||||
config=AlignerConfig(),
|
||||
local_files_only=True,
|
||||
low_cpu_mem_usage=True,
|
||||
)
|
||||
|
||||
# Encoder overrides from_pretrained and would construct another remotely
|
||||
# resolved aligner. Invoke the Transformers base loader and attach ours.
|
||||
encoder = PreTrainedModel.from_pretrained.__func__(
|
||||
Encoder,
|
||||
self.codec_path,
|
||||
subfolder="encoder",
|
||||
config=EncoderConfig(),
|
||||
local_files_only=True,
|
||||
low_cpu_mem_usage=True,
|
||||
)
|
||||
encoder._aligner = aligner
|
||||
encoder.to(self.device)
|
||||
encoder.eval()
|
||||
return encoder
|
||||
|
||||
@staticmethod
|
||||
def _normalize_reference(reference_audio: object) -> Tuple[torch.Tensor, int]:
|
||||
import torchaudio
|
||||
|
||||
sample_rate = TADA_SAMPLE_RATE
|
||||
audio = reference_audio
|
||||
|
||||
if isinstance(audio, (str, os.PathLike)):
|
||||
waveform, sample_rate = torchaudio.load(str(audio))
|
||||
audio = waveform
|
||||
elif isinstance(audio, (tuple, list)) and len(audio) == 2:
|
||||
audio, sample_rate = audio
|
||||
elif isinstance(audio, dict):
|
||||
if "audio" in audio and isinstance(audio["audio"], dict):
|
||||
audio = audio["audio"]
|
||||
if isinstance(audio, dict):
|
||||
path = audio.get("audio_path") or audio.get("path")
|
||||
if path:
|
||||
waveform, sample_rate = torchaudio.load(str(path))
|
||||
audio = waveform
|
||||
else:
|
||||
sample_rate = audio.get("sample_rate", audio.get("sampling_rate", sample_rate))
|
||||
audio = audio.get("waveform", audio.get("samples"))
|
||||
|
||||
if audio is None:
|
||||
raise ValueError("TADA requires reference audio; no waveform or audio path was provided.")
|
||||
waveform = audio.detach().cpu() if isinstance(audio, torch.Tensor) else torch.as_tensor(audio)
|
||||
waveform = waveform.float()
|
||||
if waveform.ndim == 0 or waveform.numel() == 0:
|
||||
raise ValueError("TADA reference audio is empty.")
|
||||
if waveform.ndim >= 3:
|
||||
waveform = waveform[0]
|
||||
if waveform.ndim == 2:
|
||||
waveform = waveform.mean(dim=0)
|
||||
elif waveform.ndim > 2:
|
||||
waveform = waveform.reshape(-1, waveform.shape[-1]).mean(dim=0)
|
||||
waveform = waveform.reshape(1, -1).contiguous()
|
||||
if not torch.isfinite(waveform).all():
|
||||
raise ValueError("TADA reference audio contains NaN or infinite values.")
|
||||
peak = waveform.abs().max()
|
||||
if float(peak) <= 0.0:
|
||||
raise ValueError("TADA reference audio is silent.")
|
||||
waveform = waveform / peak
|
||||
sample_rate = int(sample_rate)
|
||||
if sample_rate <= 0:
|
||||
raise ValueError(f"Invalid TADA reference sample rate: {sample_rate}")
|
||||
return waveform, sample_rate
|
||||
|
||||
@staticmethod
|
||||
def _prompt_state_to_device(state: Dict[str, Any], device: torch.device):
|
||||
from tada.modules.encoder import EncoderOutput
|
||||
|
||||
restored = {}
|
||||
for name, value in state.items():
|
||||
restored[name] = value.to(device) if isinstance(value, torch.Tensor) else value
|
||||
return EncoderOutput(**restored)
|
||||
|
||||
@staticmethod
|
||||
def _prompt_state_to_cpu(prompt: object) -> Dict[str, Any]:
|
||||
state = {}
|
||||
for field in fields(prompt):
|
||||
value = getattr(prompt, field.name)
|
||||
state[field.name] = value.detach().cpu() if isinstance(value, torch.Tensor) else value
|
||||
return state
|
||||
|
||||
@staticmethod
|
||||
def _prompt_cache_key(
|
||||
waveform: torch.Tensor,
|
||||
sample_rate: int,
|
||||
transcript: str,
|
||||
language: Optional[str],
|
||||
) -> str:
|
||||
digest = hashlib.sha256()
|
||||
digest.update(waveform.contiguous().numpy().tobytes())
|
||||
digest.update(str(sample_rate).encode("ascii"))
|
||||
digest.update(transcript.encode("utf-8"))
|
||||
digest.update(str(language).encode("ascii"))
|
||||
return digest.hexdigest()
|
||||
|
||||
def _get_prompt(
|
||||
self,
|
||||
reference_audio: object,
|
||||
reference_text: str,
|
||||
language: object,
|
||||
):
|
||||
waveform, sample_rate = self._normalize_reference(reference_audio)
|
||||
language_code = normalize_tada_language(language)
|
||||
key = self._prompt_cache_key(waveform, sample_rate, reference_text, language_code)
|
||||
cached = self._prompt_cache.pop(key, None)
|
||||
if cached is not None:
|
||||
self._prompt_cache[key] = cached
|
||||
return self._prompt_state_to_device(cached, self.device)
|
||||
|
||||
fork_devices = []
|
||||
if self.device.type == "cuda":
|
||||
fork_devices = [self.device.index if self.device.index is not None else torch.cuda.current_device()]
|
||||
with torch.random.fork_rng(devices=fork_devices):
|
||||
# Prompt sampling is intentionally stable and independent from the
|
||||
# user generation seed. A cold and warm prompt cache therefore
|
||||
# produce identical generation RNG state.
|
||||
torch.manual_seed(0)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed_all(0)
|
||||
encoder = self._load_encoder(language_code)
|
||||
try:
|
||||
prompt = encoder(
|
||||
waveform.to(self.device),
|
||||
text=[reference_text],
|
||||
sample_rate=sample_rate,
|
||||
)
|
||||
state = self._prompt_state_to_cpu(prompt)
|
||||
finally:
|
||||
del encoder
|
||||
gc.collect()
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
if self.prompt_cache_size > 0:
|
||||
self._prompt_cache[key] = state
|
||||
while len(self._prompt_cache) > self.prompt_cache_size:
|
||||
self._prompt_cache.popitem(last=False)
|
||||
return self._prompt_state_to_device(state, self.device)
|
||||
|
||||
@contextmanager
|
||||
def _generation_progress(
|
||||
self,
|
||||
text: str,
|
||||
num_flow_matching_steps: int,
|
||||
num_transition_steps: int,
|
||||
two_pass: bool,
|
||||
):
|
||||
"""Estimate total work from TADA's tokenizer and count real flow iterations."""
|
||||
from tqdm.auto import tqdm
|
||||
|
||||
original_compute_velocity = self.model._compute_velocity
|
||||
target_tokens = len(self.model.tokenizer.encode(text, add_special_tokens=False))
|
||||
# The official generator pre-fills the reference prompt, leaving target
|
||||
# text, transition, EOS, and a few structural positions to synthesize.
|
||||
estimated_generation_steps = max(
|
||||
1,
|
||||
target_tokens + int(num_transition_steps) + int(self.model.num_eos_tokens) + 4,
|
||||
)
|
||||
estimated_total = estimated_generation_steps * int(num_flow_matching_steps)
|
||||
if two_pass:
|
||||
estimated_total *= 2
|
||||
progress = tqdm(
|
||||
total=estimated_total,
|
||||
desc="TADA flow",
|
||||
unit="it",
|
||||
dynamic_ncols=True,
|
||||
mininterval=0.5,
|
||||
leave=True,
|
||||
)
|
||||
|
||||
def tracked_compute_velocity(*args, **kwargs):
|
||||
result = original_compute_velocity(*args, **kwargs)
|
||||
progress.update(1)
|
||||
return result
|
||||
|
||||
try:
|
||||
with patch.object(self.model, "_compute_velocity", tracked_compute_velocity):
|
||||
yield
|
||||
finally:
|
||||
# EOS can make the estimate short or long. Finish at the real count
|
||||
# instead of leaving a successful generation below/above 100%.
|
||||
progress.total = progress.n
|
||||
progress.close()
|
||||
|
||||
def generate_speech(
|
||||
self,
|
||||
text: str,
|
||||
reference_audio: object,
|
||||
reference_text: str,
|
||||
seed: int = 42,
|
||||
language: object = "English",
|
||||
acoustic_cfg_scale: float = 1.6,
|
||||
duration_cfg_scale: float = 1.0,
|
||||
cfg_schedule: str = "cosine",
|
||||
time_schedule: str = "logsnr",
|
||||
num_flow_matching_steps: int = 10,
|
||||
noise_temperature: float = 0.9,
|
||||
speed_up_factor: Optional[float] = None,
|
||||
num_transition_steps: int = 5,
|
||||
negative_condition_source: str = "negative_step_output",
|
||||
) -> Tuple[torch.Tensor, int]:
|
||||
if self.model is None:
|
||||
raise RuntimeError("TADA is not initialized.")
|
||||
if not str(text or "").strip():
|
||||
raise ValueError("TADA synthesis text cannot be empty.")
|
||||
if not str(reference_text or "").strip():
|
||||
raise ValueError(
|
||||
"TADA requires the exact transcript of the reference audio. "
|
||||
"Automatic transcription is intentionally disabled."
|
||||
)
|
||||
cfg_schedule = str(cfg_schedule).lower()
|
||||
time_schedule = str(time_schedule).lower()
|
||||
negative_condition_source = str(negative_condition_source).lower()
|
||||
if cfg_schedule not in self._CFG_SCHEDULES:
|
||||
raise ValueError(f"Invalid TADA CFG schedule: {cfg_schedule}")
|
||||
if time_schedule not in self._TIME_SCHEDULES:
|
||||
raise ValueError(f"Invalid TADA time schedule: {time_schedule}")
|
||||
if negative_condition_source not in self._NEGATIVE_SOURCES:
|
||||
raise ValueError(f"Invalid TADA negative condition source: {negative_condition_source}")
|
||||
if int(num_flow_matching_steps) < 1:
|
||||
raise ValueError("TADA flow matching steps must be at least 1.")
|
||||
if int(num_transition_steps) < 0:
|
||||
raise ValueError("TADA transition steps cannot be negative.")
|
||||
if speed_up_factor is not None and float(speed_up_factor) <= 0:
|
||||
raise ValueError("TADA speed_up_factor must be positive or None.")
|
||||
|
||||
from tada.modules.tada import InferenceOptions
|
||||
|
||||
seed = int(seed)
|
||||
torch.manual_seed(seed)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed_all(seed)
|
||||
|
||||
prompt = self._get_prompt(reference_audio, str(reference_text), language)
|
||||
options = InferenceOptions(
|
||||
acoustic_cfg_scale=float(acoustic_cfg_scale),
|
||||
duration_cfg_scale=float(duration_cfg_scale),
|
||||
cfg_schedule=cfg_schedule,
|
||||
noise_temperature=float(noise_temperature),
|
||||
num_flow_matching_steps=int(num_flow_matching_steps),
|
||||
time_schedule=time_schedule,
|
||||
speed_up_factor=None if speed_up_factor is None else float(speed_up_factor),
|
||||
negative_condition_source=negative_condition_source,
|
||||
)
|
||||
with self._generation_progress(
|
||||
text=str(text),
|
||||
num_flow_matching_steps=int(num_flow_matching_steps),
|
||||
num_transition_steps=int(num_transition_steps),
|
||||
two_pass=speed_up_factor is not None,
|
||||
):
|
||||
output = self.model.generate(
|
||||
prompt=prompt,
|
||||
text=str(text),
|
||||
num_transition_steps=int(num_transition_steps),
|
||||
inference_options=options,
|
||||
)
|
||||
audio = output.audio[0] if output.audio else None
|
||||
if audio is None:
|
||||
raise RuntimeError("TADA generation completed without a decodable audio result.")
|
||||
waveform = audio.detach().float().cpu() if isinstance(audio, torch.Tensor) else torch.as_tensor(audio).float()
|
||||
waveform = waveform.squeeze()
|
||||
if waveform.ndim > 1:
|
||||
waveform = waveform.reshape(-1, waveform.shape[-1]).mean(dim=0)
|
||||
waveform = waveform.contiguous()
|
||||
if waveform.numel() == 0 or not torch.isfinite(waveform).all():
|
||||
raise RuntimeError("TADA produced an empty or invalid waveform.")
|
||||
return waveform, TADA_SAMPLE_RATE
|
||||
|
||||
def to(self, device):
|
||||
self.device = self._resolve_device(device)
|
||||
if self.model is not None:
|
||||
self.model.to(self.device)
|
||||
return self
|
||||
|
||||
def eval(self):
|
||||
if self.model is not None:
|
||||
self.model.eval()
|
||||
return self
|
||||
|
||||
def cleanup(self) -> None:
|
||||
self._prompt_cache.clear()
|
||||
model = self.model
|
||||
self.model = None
|
||||
if model is not None:
|
||||
del model
|
||||
gc.collect()
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
|
||||
__all__ = ["TADA_SAMPLE_RATE", "TadaEngine"]
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 1015 KiB |
@@ -180,12 +180,12 @@ except Exception as e:
|
||||
DOTS_TTS_ENGINE_AVAILABLE = False
|
||||
|
||||
try:
|
||||
tada_engine_module = load_node_module("tada_engine_node", "engines/tada_engine_node.py")
|
||||
TadaEngineNode = tada_engine_module.TadaEngineNode
|
||||
TADA_ENGINE_AVAILABLE = True
|
||||
audio8_tts_engine_module = load_node_module("audio8_tts_engine_node", "engines/audio8_tts_engine_node.py")
|
||||
Audio8TTSEngineNode = audio8_tts_engine_module.Audio8TTSEngineNode
|
||||
AUDIO8_TTS_ENGINE_AVAILABLE = True
|
||||
except Exception as e:
|
||||
print(f"❌ TADA Engine failed: {e}")
|
||||
TADA_ENGINE_AVAILABLE = False
|
||||
print(f"❌ Audio8 TTS Engine failed: {e}")
|
||||
AUDIO8_TTS_ENGINE_AVAILABLE = False
|
||||
|
||||
try:
|
||||
dramabox_engine_module = load_node_module("dramabox_engine_node", "engines/dramabox_engine_node.py")
|
||||
@@ -678,9 +678,9 @@ if DOTS_TTS_ENGINE_AVAILABLE:
|
||||
NODE_CLASS_MAPPINGS["DotsTTSEngineNode"] = DotsTTSEngineNode
|
||||
NODE_DISPLAY_NAME_MAPPINGS["DotsTTSEngineNode"] = "⚙️ Dots TTS Engine"
|
||||
|
||||
if TADA_ENGINE_AVAILABLE:
|
||||
NODE_CLASS_MAPPINGS["TadaEngineNode"] = TadaEngineNode
|
||||
NODE_DISPLAY_NAME_MAPPINGS["TadaEngineNode"] = "⚙️ TADA Engine"
|
||||
if AUDIO8_TTS_ENGINE_AVAILABLE:
|
||||
NODE_CLASS_MAPPINGS["Audio8TTSEngineNode"] = Audio8TTSEngineNode
|
||||
NODE_DISPLAY_NAME_MAPPINGS["Audio8TTSEngineNode"] = "⚙️ Audio8 TTS Engine"
|
||||
|
||||
if DRAMABOX_ENGINE_AVAILABLE:
|
||||
NODE_CLASS_MAPPINGS["DramaBoxEngineNode"] = DramaBoxEngineNode
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""Audio8 TTS text and SRT processors."""
|
||||
@@ -0,0 +1,403 @@
|
||||
"""Suite orchestration for Audio8 TTS text generation."""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from typing import Any, Dict, List, Tuple, Union
|
||||
|
||||
import comfy.model_management as model_management
|
||||
import torch
|
||||
|
||||
current_dir = os.path.dirname(__file__)
|
||||
nodes_dir = os.path.dirname(current_dir)
|
||||
project_root = os.path.dirname(nodes_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
from utils.audio.chunk_timing import ChunkTimingHelper
|
||||
from utils.audio.edit_post_processor import (
|
||||
process_segments as apply_edit_post_processing,
|
||||
)
|
||||
from utils.text.character_parser import character_parser
|
||||
from utils.text.pause_processor import PauseTagProcessor
|
||||
from utils.text.segment_parameters import ParameterValidator, apply_segment_parameters
|
||||
from utils.text.step_audio_editx_special_tags import get_edit_tags_for_segment
|
||||
from utils.voice.character_logging import (
|
||||
format_resolved_character_block,
|
||||
resolved_character_label,
|
||||
)
|
||||
from utils.voice.discovery import (
|
||||
get_available_characters,
|
||||
get_character_mapping,
|
||||
voice_discovery,
|
||||
)
|
||||
from utils.voice.reference import effective_voice_audio
|
||||
|
||||
|
||||
class Audio8TTSProcessor:
|
||||
"""Handle suite-level tags, voices, chunking, and Audio8 generation."""
|
||||
|
||||
SAMPLE_RATE = 44100
|
||||
|
||||
def __init__(self, adapter, engine_config: Dict[str, Any]):
|
||||
self.adapter = adapter
|
||||
self.config = engine_config.copy() if engine_config else {}
|
||||
|
||||
def update_config(self, new_config: Dict[str, Any]):
|
||||
self.config = new_config.copy() if new_config else {}
|
||||
self.adapter.update_config(self.config)
|
||||
|
||||
@staticmethod
|
||||
def _check_interrupt(context: str = "") -> None:
|
||||
if not model_management.interrupt_processing:
|
||||
return
|
||||
suffix = f" {context}" if context else ""
|
||||
raise InterruptedError(f"Audio8 TTS generation interrupted{suffix}")
|
||||
|
||||
def _setup_character_parser(self, text: str) -> None:
|
||||
character_tags = re.findall(r"\[([^\]]+)\]", text or "")
|
||||
characters_from_tags = [
|
||||
tag.split("|")[0].strip()
|
||||
for tag in character_tags
|
||||
if not tag.lower().startswith("pause:")
|
||||
]
|
||||
|
||||
all_available = set(get_available_characters() or [])
|
||||
for alias, target in voice_discovery.get_character_aliases().items():
|
||||
all_available.add(alias.lower())
|
||||
all_available.add(target.lower())
|
||||
all_available.update(
|
||||
character.lower() for character in characters_from_tags if character
|
||||
)
|
||||
all_available.add("narrator")
|
||||
|
||||
character_parser.set_available_characters(list(all_available))
|
||||
character_parser.reset_session_cache()
|
||||
|
||||
@staticmethod
|
||||
def _as_voice_reference(value: Any) -> Dict[str, Any]:
|
||||
if isinstance(value, dict):
|
||||
return value.copy()
|
||||
if value is None:
|
||||
return {}
|
||||
return {"audio": value}
|
||||
|
||||
def _resolve_voice(
|
||||
self,
|
||||
character: str,
|
||||
voice_mapping: Dict[str, Any],
|
||||
discovered_mapping: Dict[str, Tuple[Any, Any]],
|
||||
) -> Dict[str, Any]:
|
||||
if character in voice_mapping:
|
||||
mapped_voice = self._as_voice_reference(voice_mapping[character])
|
||||
if effective_voice_audio(mapped_voice) is not None or mapped_voice.get(
|
||||
"reference_text"
|
||||
):
|
||||
return mapped_voice
|
||||
|
||||
if character != "narrator":
|
||||
audio_path, reference_text = discovered_mapping.get(character, (None, None))
|
||||
if audio_path:
|
||||
return {
|
||||
"audio_path": audio_path,
|
||||
"reference_text": reference_text or "",
|
||||
}
|
||||
print(
|
||||
f"⚠️ Audio8 TTS: No voice found for '{character}'; "
|
||||
"using narrator/no-reference fallback"
|
||||
)
|
||||
|
||||
return self._as_voice_reference(voice_mapping.get("narrator"))
|
||||
|
||||
@staticmethod
|
||||
def _voice_log_note(voice_ref: Dict[str, Any]) -> str:
|
||||
reference_text = ""
|
||||
if isinstance(voice_ref, dict):
|
||||
reference_text = str(voice_ref.get("reference_text") or "").strip()
|
||||
has_audio = (
|
||||
isinstance(voice_ref, dict) and effective_voice_audio(voice_ref) is not None
|
||||
)
|
||||
|
||||
if not has_audio and not reference_text:
|
||||
return " [no reference voice]"
|
||||
if has_audio and reference_text:
|
||||
return f" [reference transcript: {len(reference_text)} chars]"
|
||||
if has_audio:
|
||||
return " [reference audio has no transcript; adapter will validate]"
|
||||
return " [reference transcript has no audio; adapter will validate]"
|
||||
|
||||
@staticmethod
|
||||
def _format_parameter_log(
|
||||
filtered_params: Dict[str, Any],
|
||||
current_config: Dict[str, Any],
|
||||
current_seed: int,
|
||||
) -> str:
|
||||
if not filtered_params:
|
||||
return ""
|
||||
|
||||
values = []
|
||||
for key in ("seed", "temperature", "top_p", "top_k", "max_new_tokens"):
|
||||
if key not in filtered_params:
|
||||
continue
|
||||
value = current_seed if key == "seed" else current_config.get(key)
|
||||
values.append(f"{key}={value}")
|
||||
return ", ".join(values)
|
||||
|
||||
@staticmethod
|
||||
def _validate_segment_config(
|
||||
config: Dict[str, Any],
|
||||
filtered_params: Dict[str, Any],
|
||||
) -> Dict[str, Any]:
|
||||
"""Apply Audio8-specific bounds and keep retry budgets consistent."""
|
||||
if "top_p" in filtered_params and float(config["top_p"]) <= 0:
|
||||
raise ValueError("Audio8 segment top_p must be greater than 0")
|
||||
|
||||
if "max_new_tokens" in filtered_params:
|
||||
max_new_tokens = int(config["max_new_tokens"])
|
||||
if max_new_tokens > 2048:
|
||||
raise ValueError("Audio8 segment max_new_tokens must be at most 2048")
|
||||
config["retry_max_new_tokens"] = max(
|
||||
max_new_tokens,
|
||||
int(config.get("retry_max_new_tokens", 2000)),
|
||||
)
|
||||
return config
|
||||
|
||||
def _log_generation(
|
||||
self,
|
||||
character: str,
|
||||
text: str,
|
||||
voice_ref: Dict[str, Any],
|
||||
config: Dict[str, Any],
|
||||
chunk_count: int,
|
||||
parameter_log: str,
|
||||
show_text_logging: bool,
|
||||
) -> None:
|
||||
display_name = resolved_character_label(character, voice_ref)
|
||||
print(
|
||||
f"🎭 Audio8 TTS - Generating for '{display_name}'"
|
||||
f"{self._voice_log_note(voice_ref)}"
|
||||
)
|
||||
print(
|
||||
" Settings: "
|
||||
f"mode={'Sampling' if config.get('do_sample', True) else 'Greedy'}, "
|
||||
f"temperature={config.get('temperature', 0.8)}, "
|
||||
f"top_p={config.get('top_p', 0.95)}, "
|
||||
f"top_k={config.get('top_k', 50)}, "
|
||||
f"max_new_tokens={config.get('max_new_tokens', 1024)}"
|
||||
)
|
||||
if parameter_log:
|
||||
print(f"🎛️ Audio8 TTS segment params: {parameter_log}")
|
||||
if show_text_logging:
|
||||
print(format_resolved_character_block(character, text, voice_ref))
|
||||
if chunk_count > 1:
|
||||
print(
|
||||
f"📝 Audio8 TTS: Chunking '{display_name}' into "
|
||||
f"{chunk_count} suite chunks"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_audio(audio: Any) -> torch.Tensor:
|
||||
if not isinstance(audio, torch.Tensor):
|
||||
audio = torch.as_tensor(audio, dtype=torch.float32)
|
||||
audio = audio.detach().to(device="cpu", dtype=torch.float32)
|
||||
if audio.dim() == 1:
|
||||
audio = audio.unsqueeze(0)
|
||||
elif audio.dim() == 3 and audio.shape[0] == 1:
|
||||
audio = audio.squeeze(0)
|
||||
if audio.dim() != 2:
|
||||
raise ValueError(
|
||||
"Audio8 adapter returned an invalid waveform shape; "
|
||||
"expected [channels, samples]"
|
||||
)
|
||||
return audio
|
||||
|
||||
def process_text(
|
||||
self,
|
||||
text: str,
|
||||
voice_mapping: Dict[str, Any],
|
||||
seed: int,
|
||||
enable_chunking: bool = True,
|
||||
max_chars_per_chunk: int = 400,
|
||||
chunk_combination_method: str = "auto",
|
||||
silence_between_chunks_ms: int = 100,
|
||||
enable_audio_cache: bool = True,
|
||||
apply_edit_postprocessing: bool = True,
|
||||
show_text_logging: bool = True,
|
||||
**_unused,
|
||||
) -> List[Dict[str, Any]]:
|
||||
del chunk_combination_method, silence_between_chunks_ms
|
||||
|
||||
self._check_interrupt()
|
||||
self._setup_character_parser(text)
|
||||
base_config = self.config.copy()
|
||||
segment_objects = character_parser.parse_text_segments(text)
|
||||
if not segment_objects:
|
||||
segment_objects = character_parser.parse_text_segments(
|
||||
"narrator " + (text or "")
|
||||
)
|
||||
|
||||
characters = list(
|
||||
{segment.character for segment in segment_objects if segment.character}
|
||||
)
|
||||
discovered_mapping = get_character_mapping(characters, engine_type="audio_only")
|
||||
segment_records: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
for segment_index, segment in enumerate(segment_objects):
|
||||
character = segment.character or "narrator"
|
||||
self._check_interrupt(
|
||||
f"before segment {segment_index + 1}/{len(segment_objects)} "
|
||||
f"for '{character}'"
|
||||
)
|
||||
segment_text = (segment.text or "").strip()
|
||||
if not segment_text:
|
||||
continue
|
||||
|
||||
segment_params = segment.parameters or {}
|
||||
filtered_params = ParameterValidator.filter_parameters_for_engine(
|
||||
segment_params, "audio8_tts"
|
||||
)
|
||||
current_config = (
|
||||
apply_segment_parameters(base_config, filtered_params, "audio8_tts")
|
||||
if filtered_params
|
||||
else base_config.copy()
|
||||
)
|
||||
current_config = self._validate_segment_config(
|
||||
current_config, filtered_params
|
||||
)
|
||||
current_seed = int(current_config.get("seed", seed))
|
||||
parameter_log = self._format_parameter_log(
|
||||
filtered_params, current_config, current_seed
|
||||
)
|
||||
self.adapter.update_config(current_config)
|
||||
|
||||
voice_ref = self._resolve_voice(
|
||||
character, voice_mapping, discovered_mapping
|
||||
)
|
||||
seed_offset = 0
|
||||
|
||||
def generate_chunks(text_content: str, edit_tags: list) -> None:
|
||||
nonlocal seed_offset
|
||||
clean_content = (text_content or "").strip()
|
||||
if not clean_content:
|
||||
return
|
||||
|
||||
if enable_chunking:
|
||||
from utils.text.chunking import ImprovedChatterBoxChunker
|
||||
|
||||
max_chars = ImprovedChatterBoxChunker.validate_chunking_params(
|
||||
max_chars_per_chunk
|
||||
)
|
||||
chunks = ImprovedChatterBoxChunker.split_into_chunks(
|
||||
clean_content, max_chars=max_chars
|
||||
)
|
||||
else:
|
||||
chunks = [clean_content]
|
||||
chunks = [chunk.strip() for chunk in chunks if chunk.strip()]
|
||||
|
||||
self._log_generation(
|
||||
character=character,
|
||||
text=clean_content,
|
||||
voice_ref=voice_ref,
|
||||
config=current_config,
|
||||
chunk_count=len(chunks),
|
||||
parameter_log=parameter_log,
|
||||
show_text_logging=show_text_logging,
|
||||
)
|
||||
|
||||
for chunk_index, chunk in enumerate(chunks):
|
||||
self._check_interrupt(
|
||||
f"before chunk {chunk_index + 1}/{len(chunks)} "
|
||||
f"for '{character}'"
|
||||
)
|
||||
chunk_seed = current_seed + seed_offset
|
||||
seed_offset += 1
|
||||
audio = self.adapter.generate_single(
|
||||
text=chunk,
|
||||
voice_ref=voice_ref,
|
||||
seed=chunk_seed,
|
||||
enable_audio_cache=enable_audio_cache,
|
||||
character_name=character,
|
||||
)
|
||||
segment_records.append(
|
||||
{
|
||||
"waveform": self._normalize_audio(audio),
|
||||
"sample_rate": self.SAMPLE_RATE,
|
||||
"text": chunk,
|
||||
"edit_tags": (edit_tags if chunk_index == 0 else []),
|
||||
}
|
||||
)
|
||||
|
||||
if PauseTagProcessor.has_pause_tags(segment_text):
|
||||
pause_segments, _ = PauseTagProcessor.parse_pause_tags(segment_text)
|
||||
for fragment_type, fragment_content in pause_segments:
|
||||
self._check_interrupt(
|
||||
f"while processing segment {segment_index + 1}"
|
||||
)
|
||||
if fragment_type == "text":
|
||||
clean_text, edit_tags = get_edit_tags_for_segment(
|
||||
fragment_content
|
||||
)
|
||||
generate_chunks(clean_text, edit_tags)
|
||||
elif fragment_type == "pause":
|
||||
silence = PauseTagProcessor.create_silence_segment(
|
||||
fragment_content,
|
||||
self.SAMPLE_RATE,
|
||||
torch.device("cpu"),
|
||||
torch.float32,
|
||||
)
|
||||
if silence.dim() == 1:
|
||||
silence = silence.unsqueeze(0)
|
||||
segment_records.append(
|
||||
{
|
||||
"waveform": silence.cpu(),
|
||||
"sample_rate": self.SAMPLE_RATE,
|
||||
"text": f"[pause:{fragment_content}s]",
|
||||
"edit_tags": [],
|
||||
}
|
||||
)
|
||||
else:
|
||||
clean_text, edit_tags = get_edit_tags_for_segment(segment_text)
|
||||
generate_chunks(clean_text, edit_tags)
|
||||
finally:
|
||||
self.adapter.update_config(base_config)
|
||||
|
||||
if (
|
||||
apply_edit_postprocessing
|
||||
and segment_records
|
||||
and any(record.get("edit_tags") for record in segment_records)
|
||||
):
|
||||
self._check_interrupt("before edit post-processing")
|
||||
segment_records = apply_edit_post_processing(
|
||||
segment_records, engine_config=base_config
|
||||
)
|
||||
for record in segment_records:
|
||||
record["waveform"] = self._normalize_audio(record["waveform"])
|
||||
|
||||
return segment_records
|
||||
|
||||
def combine_audio_segments(
|
||||
self,
|
||||
segments: List[Dict[str, Any]],
|
||||
method: str = "auto",
|
||||
silence_ms: int = 100,
|
||||
original_text: str = "",
|
||||
return_info: bool = False,
|
||||
) -> Union[torch.Tensor, Tuple[torch.Tensor, Dict[str, Any]]]:
|
||||
self._check_interrupt("before audio assembly")
|
||||
if not segments:
|
||||
empty = torch.zeros(0, dtype=torch.float32)
|
||||
return (empty, {}) if return_info else empty
|
||||
|
||||
text_chunks = [segment.get("text", "") for segment in segments]
|
||||
combined_audio, chunk_info = ChunkTimingHelper.combine_audio_with_timing(
|
||||
audio_segments=[segment["waveform"] for segment in segments],
|
||||
combination_method=method,
|
||||
silence_ms=silence_ms,
|
||||
crossfade_duration=0.1,
|
||||
sample_rate=self.SAMPLE_RATE,
|
||||
text_length=len(" ".join(text_chunks)),
|
||||
original_text=original_text,
|
||||
text_chunks=text_chunks,
|
||||
)
|
||||
return (combined_audio, chunk_info) if return_info else combined_audio
|
||||
@@ -0,0 +1,222 @@
|
||||
"""SRT orchestration for Audio8 TTS."""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
|
||||
import comfy.model_management as model_management
|
||||
import torch
|
||||
|
||||
from engines.adapters.audio8_tts_adapter import Audio8TTSEngineAdapter
|
||||
from utils.system.import_manager import import_manager
|
||||
from utils.timing.assembly import AudioAssemblyEngine
|
||||
from utils.timing.engine import TimingEngine
|
||||
from utils.timing.overlap_detection import SRTOverlapHandler
|
||||
from utils.timing.reporting import SRTReportGenerator
|
||||
|
||||
|
||||
class Audio8TTSSRTProcessor:
|
||||
"""Generate Audio8 speech per subtitle and apply shared timing modes."""
|
||||
|
||||
SAMPLE_RATE = 44100
|
||||
|
||||
def __init__(self, node_instance, config):
|
||||
self.node_instance = node_instance
|
||||
self.config = dict(config or {})
|
||||
self.adapter = Audio8TTSEngineAdapter(self.config)
|
||||
self._processor = None
|
||||
|
||||
success, modules, message = import_manager.import_srt_modules()
|
||||
if not success:
|
||||
raise ImportError(f"Audio8 TTS SRT unavailable: {message}")
|
||||
self.SRTParser = modules["SRTParser"]
|
||||
|
||||
@property
|
||||
def processor(self):
|
||||
if self._processor is None:
|
||||
processor_path = os.path.join(
|
||||
os.path.dirname(__file__), "audio8_tts_processor.py"
|
||||
)
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"audio8_tts_processor_module", processor_path
|
||||
)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
self._processor = module.Audio8TTSProcessor(self.adapter, self.config)
|
||||
return self._processor
|
||||
|
||||
def update_config(self, config):
|
||||
self.config = dict(config or {})
|
||||
self.adapter.update_config(self.config)
|
||||
if self._processor is not None:
|
||||
self._processor.update_config(self.config)
|
||||
|
||||
@staticmethod
|
||||
def _check_interrupt(index=None, total=None):
|
||||
if not model_management.interrupt_processing:
|
||||
return
|
||||
location = f" at subtitle {index + 1}/{total}" if index is not None else ""
|
||||
raise InterruptedError(f"Audio8 TTS SRT generation interrupted{location}")
|
||||
|
||||
def process_srt_content(
|
||||
self,
|
||||
srt_content,
|
||||
voice_mapping,
|
||||
seed,
|
||||
timing_mode,
|
||||
timing_params,
|
||||
enable_audio_cache=True,
|
||||
):
|
||||
self._check_interrupt()
|
||||
subtitles = self.SRTParser().parse_srt_content(srt_content, allow_overlaps=True)
|
||||
has_overlaps = SRTOverlapHandler.detect_overlaps(subtitles)
|
||||
active_mode, switched = SRTOverlapHandler.handle_smart_natural_fallback(
|
||||
timing_mode, has_overlaps, "Audio8 TTS SRT"
|
||||
)
|
||||
|
||||
audio_segments = []
|
||||
adjustments = []
|
||||
for index, subtitle in enumerate(subtitles):
|
||||
self._check_interrupt(index, len(subtitles))
|
||||
text = (subtitle.text or "").strip()
|
||||
if text:
|
||||
records = self.processor.process_text(
|
||||
text=text,
|
||||
voice_mapping=voice_mapping,
|
||||
seed=seed + index,
|
||||
enable_chunking=False,
|
||||
enable_audio_cache=enable_audio_cache,
|
||||
apply_edit_postprocessing=True,
|
||||
)
|
||||
audio, _ = self.processor.combine_audio_segments(
|
||||
records,
|
||||
method="auto",
|
||||
silence_ms=0,
|
||||
original_text=text,
|
||||
return_info=True,
|
||||
)
|
||||
if audio.dim() == 1:
|
||||
audio = audio.unsqueeze(0)
|
||||
else:
|
||||
audio = torch.zeros(1, int(subtitle.duration * self.SAMPLE_RATE))
|
||||
|
||||
audio = audio.detach().cpu().float()
|
||||
audio_segments.append(audio)
|
||||
natural_duration = audio.shape[-1] / self.SAMPLE_RATE
|
||||
target_duration = subtitle.duration
|
||||
stretch_factor = (
|
||||
target_duration / natural_duration if natural_duration else 1.0
|
||||
)
|
||||
adjustments.append(
|
||||
{
|
||||
"index": index,
|
||||
"segment_index": index,
|
||||
"sequence": subtitle.sequence,
|
||||
"natural_duration": natural_duration,
|
||||
"target_start": subtitle.start_time,
|
||||
"target_end": subtitle.end_time,
|
||||
"target_duration": target_duration,
|
||||
"start_time": subtitle.start_time,
|
||||
"end_time": subtitle.end_time,
|
||||
"stretch_factor": stretch_factor,
|
||||
"needs_stretching": (abs(stretch_factor - 1.0) > 0.05),
|
||||
"stretch_type": (
|
||||
"compress"
|
||||
if stretch_factor < 1.0
|
||||
else "expand"
|
||||
if stretch_factor > 1.0
|
||||
else "none"
|
||||
),
|
||||
"adjustment": natural_duration - target_duration,
|
||||
"adjusted_start": subtitle.start_time,
|
||||
"adjusted_end": subtitle.end_time,
|
||||
"adjusted_duration": natural_duration,
|
||||
}
|
||||
)
|
||||
|
||||
self._check_interrupt()
|
||||
final_audio, replacement, stretch_method = self._assemble(
|
||||
audio_segments, subtitles, active_mode, timing_params or {}
|
||||
)
|
||||
self._check_interrupt()
|
||||
if replacement is not None:
|
||||
adjustments = replacement
|
||||
|
||||
reporter = SRTReportGenerator()
|
||||
timing_report = reporter.generate_timing_report(
|
||||
subtitles,
|
||||
adjustments,
|
||||
active_mode,
|
||||
has_overlaps,
|
||||
switched,
|
||||
timing_mode if switched else None,
|
||||
stretch_method,
|
||||
)
|
||||
adjusted_srt = reporter.generate_adjusted_srt_string(
|
||||
subtitles, adjustments, active_mode
|
||||
)
|
||||
|
||||
final_audio = final_audio.detach().cpu().float()
|
||||
if final_audio.dim() == 1:
|
||||
final_audio = final_audio.unsqueeze(0).unsqueeze(0)
|
||||
elif final_audio.dim() == 2:
|
||||
final_audio = final_audio.unsqueeze(0)
|
||||
duration = final_audio.shape[-1] / self.SAMPLE_RATE
|
||||
generation_info = (
|
||||
f"Generated {duration:.1f}s Audio8 TTS SRT audio from "
|
||||
f"{len(subtitles)} subtitles using {active_mode} mode"
|
||||
)
|
||||
return (
|
||||
{"waveform": final_audio, "sample_rate": self.SAMPLE_RATE},
|
||||
generation_info,
|
||||
timing_report,
|
||||
adjusted_srt,
|
||||
)
|
||||
|
||||
def _assemble(self, audio_segments, subtitles, mode, params):
|
||||
if mode == "stretch_to_fit":
|
||||
from engines.chatterbox.audio_timing import TimedAudioAssembler
|
||||
|
||||
assembler = TimedAudioAssembler(self.SAMPLE_RATE)
|
||||
audio, method = assembler.assemble_timed_audio(
|
||||
audio_segments,
|
||||
[(subtitle.start_time, subtitle.end_time) for subtitle in subtitles],
|
||||
fade_duration=params.get("fade_for_StretchToFit", 0.01),
|
||||
)
|
||||
return audio, None, method
|
||||
|
||||
assembler = AudioAssemblyEngine(self.SAMPLE_RATE)
|
||||
if mode == "pad_with_silence":
|
||||
audio = assembler.assemble_with_overlaps(
|
||||
audio_segments, subtitles, torch.device("cpu")
|
||||
)
|
||||
return audio, None, None
|
||||
|
||||
timing = TimingEngine(self.SAMPLE_RATE)
|
||||
if mode == "concatenate":
|
||||
replacements = timing.calculate_concatenation_adjustments(
|
||||
audio_segments, subtitles
|
||||
)
|
||||
audio = assembler.assemble_concatenation(
|
||||
audio_segments,
|
||||
params.get("fade_for_StretchToFit", 0.01),
|
||||
)
|
||||
return audio, replacements, None
|
||||
|
||||
if mode != "smart_natural":
|
||||
raise ValueError(f"Unsupported Audio8 SRT timing mode: {mode}")
|
||||
replacements, processed = timing.calculate_smart_timing_adjustments(
|
||||
audio_segments,
|
||||
subtitles,
|
||||
params.get("timing_tolerance", 2.0),
|
||||
params.get("max_stretch_ratio", 1.0),
|
||||
params.get("min_stretch_ratio", 0.5),
|
||||
torch.device("cpu"),
|
||||
)
|
||||
audio = assembler.assemble_smart_natural(
|
||||
audio_segments,
|
||||
processed,
|
||||
replacements,
|
||||
subtitles,
|
||||
torch.device("cpu"),
|
||||
)
|
||||
return audio, replacements, None
|
||||
@@ -0,0 +1,230 @@
|
||||
"""Audio8 TTS Preview engine configuration node."""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
|
||||
current_dir = os.path.dirname(__file__)
|
||||
nodes_dir = os.path.dirname(current_dir)
|
||||
project_root = os.path.dirname(nodes_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
base_node_path = os.path.join(nodes_dir, "base", "base_node.py")
|
||||
base_spec = importlib.util.spec_from_file_location("base_node_module", base_node_path)
|
||||
base_module = importlib.util.module_from_spec(base_spec)
|
||||
sys.modules["base_node_module"] = base_module
|
||||
base_spec.loader.exec_module(base_module)
|
||||
BaseTTSNode = base_module.BaseTTSNode
|
||||
|
||||
|
||||
class Audio8TTSEngineNode(BaseTTSNode):
|
||||
"""Configure the official Audio8 TTS Preview checkpoint."""
|
||||
|
||||
DEFAULT_MODEL = "Audio8-TTS-Preview-0.6b"
|
||||
|
||||
@classmethod
|
||||
def NAME(cls):
|
||||
return "⚙️ Audio8 TTS Engine"
|
||||
|
||||
@classmethod
|
||||
def _get_model_options(cls):
|
||||
try:
|
||||
from engines.audio8_tts.downloader import Audio8TTSDownloader
|
||||
|
||||
options = Audio8TTSDownloader().get_available_models()
|
||||
return options or [cls.DEFAULT_MODEL]
|
||||
except Exception:
|
||||
return [cls.DEFAULT_MODEL]
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"model_variant": (
|
||||
cls._get_model_options(),
|
||||
{
|
||||
"default": cls.DEFAULT_MODEL,
|
||||
"tooltip": (
|
||||
"Official 0.6B Audio8 TTS Preview checkpoint. "
|
||||
"The canonical option downloads from "
|
||||
"Audio8/Audio8-TTS-Preview-0.6b; local: options are "
|
||||
"detected model folders.\n\n"
|
||||
"Preview scope: multilingual speech and zero-shot "
|
||||
"voice cloning, with 11 recommended languages. The "
|
||||
"model infers language from text and has no language "
|
||||
"selection control.\n\n"
|
||||
"Voice cloning requires reference audio and its exact "
|
||||
"matching transcript. Generation without a reference "
|
||||
"is supported. Audio8 has no instruction-conditioned "
|
||||
"voice-design mode. The suite runs it in the existing "
|
||||
"shared Transformers 4 runtime for correct cloning. "
|
||||
"Code and weights: Apache License 2.0."
|
||||
),
|
||||
},
|
||||
),
|
||||
"device": (
|
||||
["auto", "cuda", "cpu"],
|
||||
{
|
||||
"default": "auto",
|
||||
"tooltip": (
|
||||
"Execution device. CUDA is recommended. CPU inference "
|
||||
"is supported but slow and runs in float32."
|
||||
),
|
||||
},
|
||||
),
|
||||
"temperature": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.8,
|
||||
"min": 0.1,
|
||||
"max": 2.0,
|
||||
"step": 0.05,
|
||||
"tooltip": (
|
||||
"Official sampling temperature. Used only in Sampling "
|
||||
"mode; lower values are more conservative."
|
||||
),
|
||||
},
|
||||
),
|
||||
"top_p": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.95,
|
||||
"min": 0.01,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": (
|
||||
"Official nucleus-sampling cutoff. Used only in "
|
||||
"Sampling mode."
|
||||
),
|
||||
},
|
||||
),
|
||||
"top_k": (
|
||||
"INT",
|
||||
{
|
||||
"default": 50,
|
||||
"min": 1,
|
||||
"max": 100,
|
||||
"step": 1,
|
||||
"tooltip": (
|
||||
"Official top-k sampling limit. Used only in Sampling mode."
|
||||
),
|
||||
},
|
||||
),
|
||||
"max_new_tokens": (
|
||||
"INT",
|
||||
{
|
||||
"default": 1024,
|
||||
"min": 64,
|
||||
"max": 2048,
|
||||
"step": 64,
|
||||
"tooltip": (
|
||||
"Maximum acoustic frames for the first generation "
|
||||
"attempt. Audio8 emits about 21.5 frames per second. "
|
||||
"Larger budgets take more time and context memory."
|
||||
),
|
||||
},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
"retry_max_new_tokens": (
|
||||
"INT",
|
||||
{
|
||||
"default": 2048,
|
||||
"min": 64,
|
||||
"max": 2048,
|
||||
"step": 8,
|
||||
"tooltip": (
|
||||
"Second-attempt frame budget when the first attempt "
|
||||
"does not emit EOS. Must be at least max_new_tokens. "
|
||||
"Set it equal to max_new_tokens to disable the larger "
|
||||
"retry."
|
||||
),
|
||||
},
|
||||
),
|
||||
"dtype": (
|
||||
["auto", "bfloat16", "float16", "float32"],
|
||||
{
|
||||
"default": "auto",
|
||||
"tooltip": (
|
||||
"Model precision. Auto prefers bfloat16 on Ampere-or-"
|
||||
"newer CUDA GPUs, uses float16 on older CUDA GPUs, and "
|
||||
"float32 on CPU. CPU coerces reduced-precision choices "
|
||||
"to float32."
|
||||
),
|
||||
},
|
||||
),
|
||||
"sampling_mode": (
|
||||
["Sampling", "Greedy"],
|
||||
{
|
||||
"default": "Sampling",
|
||||
"tooltip": (
|
||||
"Official decoding mode. Sampling uses temperature, "
|
||||
"top_p, and top_k. Greedy is deterministic for a fixed "
|
||||
"prompt and ignores those sampling controls."
|
||||
),
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("TTS_ENGINE",)
|
||||
RETURN_NAMES = ("TTS_engine",)
|
||||
FUNCTION = "create_engine_config"
|
||||
CATEGORY = "TTS Audio Suite/⚙️ Engines"
|
||||
|
||||
def create_engine_config(
|
||||
self,
|
||||
model_variant: str,
|
||||
device: str,
|
||||
temperature: float,
|
||||
top_p: float,
|
||||
top_k: int,
|
||||
max_new_tokens: int,
|
||||
retry_max_new_tokens: int = 2048,
|
||||
dtype: str = "auto",
|
||||
sampling_mode: str = "Sampling",
|
||||
) -> tuple:
|
||||
if retry_max_new_tokens < max_new_tokens:
|
||||
raise ValueError(
|
||||
"Audio8 retry_max_new_tokens must be at least max_new_tokens"
|
||||
)
|
||||
|
||||
do_sample = sampling_mode == "Sampling"
|
||||
config = {
|
||||
"engine_type": "audio8_tts",
|
||||
"model_variant": model_variant,
|
||||
"device": device,
|
||||
"dtype": dtype,
|
||||
"max_new_tokens": int(max_new_tokens),
|
||||
"retry_max_new_tokens": int(retry_max_new_tokens),
|
||||
"temperature": float(temperature),
|
||||
"top_p": float(top_p),
|
||||
"top_k": int(top_k),
|
||||
"do_sample": do_sample,
|
||||
}
|
||||
|
||||
print(f"⚙️ Audio8 TTS Preview: {model_variant} on {device} ({dtype})")
|
||||
print(
|
||||
" Settings: "
|
||||
f"mode={sampling_mode}, temperature={temperature}, top_p={top_p}, "
|
||||
f"top_k={top_k}, max_new_tokens={max_new_tokens}, "
|
||||
f"retry_max_new_tokens={retry_max_new_tokens}"
|
||||
)
|
||||
print(
|
||||
" Voice cloning requires reference audio plus its exact matching "
|
||||
"transcript; no-reference generation is supported."
|
||||
)
|
||||
print(
|
||||
" Runtime: shared Transformers 4 profile (required for correct cloning)"
|
||||
)
|
||||
print(" Preview checkpoint | Apache License 2.0")
|
||||
|
||||
return (
|
||||
{
|
||||
"engine_type": "audio8_tts",
|
||||
"config": config,
|
||||
"adapter_class": "Audio8TTSEngineAdapter",
|
||||
"capabilities": ["tts"],
|
||||
},
|
||||
)
|
||||
@@ -1,238 +0,0 @@
|
||||
"""TADA engine configuration node."""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
|
||||
current_dir = os.path.dirname(__file__)
|
||||
nodes_dir = os.path.dirname(current_dir)
|
||||
project_root = os.path.dirname(nodes_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
base_node_path = os.path.join(nodes_dir, "base", "base_node.py")
|
||||
base_spec = importlib.util.spec_from_file_location("base_node_module", base_node_path)
|
||||
base_module = importlib.util.module_from_spec(base_spec)
|
||||
sys.modules["base_node_module"] = base_module
|
||||
base_spec.loader.exec_module(base_module)
|
||||
BaseTTSNode = base_module.BaseTTSNode
|
||||
|
||||
from engines.tada.languages import TADA_LANGUAGE_OPTIONS, validate_tada_model_language
|
||||
from engines.tada.tada_downloader import TadaDownloader
|
||||
from utils.models.factory_config import (
|
||||
RUNTIME_MODE_SHARED,
|
||||
normalize_runtime_mode,
|
||||
)
|
||||
|
||||
|
||||
RUNTIME_MODE_MAIN_LABEL = "Main Environment"
|
||||
RUNTIME_MODE_SHARED_LABEL = "⚠️ Shared Runtime"
|
||||
|
||||
|
||||
class TadaEngineNode(BaseTTSNode):
|
||||
"""Configure official Hume TADA zero-shot voice cloning."""
|
||||
|
||||
RUNTIME_PROFILE = "vibevoice_transformers4_shared"
|
||||
|
||||
@classmethod
|
||||
def NAME(cls):
|
||||
return "⚙️ TADA Engine"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
try:
|
||||
model_options = TadaDownloader().get_available_models()
|
||||
except Exception:
|
||||
model_options = ["TADA-1B", "TADA-3B-ML"]
|
||||
|
||||
return {
|
||||
"required": {
|
||||
"model_variant": (model_options, {
|
||||
"default": "TADA-1B",
|
||||
"tooltip": "Choose the TADA model.\n"
|
||||
"• TADA-1B: faster, English only (about 3.9 GB).\n"
|
||||
"• TADA-3B-ML: multilingual (about 8.9 GB).\n"
|
||||
"Non-English characters automatically use 3B-ML. Missing files download on first use.",
|
||||
}),
|
||||
"device": (["auto", "cuda", "cpu"], {
|
||||
"default": "auto",
|
||||
"tooltip": "Choose where TADA runs.\n"
|
||||
"• Auto: recommended; uses an NVIDIA GPU when available.\n"
|
||||
"• CUDA: force GPU use.\n"
|
||||
"• CPU: works without a GPU, but is extremely slow.",
|
||||
}),
|
||||
"language": (list(TADA_LANGUAGE_OPTIONS), {
|
||||
"default": "English",
|
||||
"tooltip": "Language spoken in the reference clip and generated text.\n"
|
||||
"This improves pronunciation and voice alignment.\n"
|
||||
"Character language tags can change it automatically for each speaker.",
|
||||
}),
|
||||
"acoustic_cfg_scale": ("FLOAT", {
|
||||
"default": 1.6,
|
||||
"min": 0.0,
|
||||
"max": 5.0,
|
||||
"step": 0.05,
|
||||
"tooltip": "How strongly TADA follows the voice conditioning.\n"
|
||||
"1.6 is recommended. Higher can sound stronger but less natural.\n"
|
||||
"1.0 disables guidance and is faster.",
|
||||
}),
|
||||
"duration_cfg_scale": ("FLOAT", {
|
||||
"default": 1.0,
|
||||
"min": 0.0,
|
||||
"max": 5.0,
|
||||
"step": 0.05,
|
||||
"tooltip": "How strongly guidance affects pauses and sound duration.\n"
|
||||
"Keep 1.0 for natural automatic timing.\n"
|
||||
"Higher values can produce unusual pacing.",
|
||||
}),
|
||||
"num_flow_matching_steps": ("INT", {
|
||||
"default": 10,
|
||||
"min": 1,
|
||||
"max": 64,
|
||||
"step": 1,
|
||||
"tooltip": "Number of audio refinement steps.\n"
|
||||
"10 is recommended. Fewer is faster but may reduce quality.\n"
|
||||
"More is slower and may not improve the result.",
|
||||
}),
|
||||
"noise_temperature": ("FLOAT", {
|
||||
"default": 0.9,
|
||||
"min": 0.0,
|
||||
"max": 2.0,
|
||||
"step": 0.05,
|
||||
"tooltip": "Controls variation in the generated voice.\n"
|
||||
"0.9 is recommended. Lower is more predictable.\n"
|
||||
"Higher adds variation but may create unstable sounds.",
|
||||
}),
|
||||
},
|
||||
"optional": {
|
||||
"dtype": (["auto", "bfloat16", "float16", "float32"], {
|
||||
"default": "auto",
|
||||
"tooltip": "Controls model precision and memory use.\n"
|
||||
"Auto is recommended. Float16 and bfloat16 use less GPU memory.\n"
|
||||
"Float32 uses much more memory and is mainly for troubleshooting.",
|
||||
}),
|
||||
"cfg_schedule": (["cosine", "linear", "constant"], {
|
||||
"default": "cosine",
|
||||
"tooltip": "How guidance changes during audio refinement.\n"
|
||||
"• Cosine: smooth reduction; recommended.\n"
|
||||
"• Linear: even reduction.\n"
|
||||
"• Constant: full guidance throughout.",
|
||||
}),
|
||||
"time_schedule": (["logsnr", "cosine", "uniform"], {
|
||||
"default": "logsnr",
|
||||
"tooltip": "Controls where refinement steps are concentrated.\n"
|
||||
"• LogSNR: recommended.\n"
|
||||
"• Cosine: more work near the beginning and end.\n"
|
||||
"• Uniform: steps are evenly spaced.",
|
||||
}),
|
||||
"negative_condition_source": (["negative_step_output", "prompt", "zero"], {
|
||||
"default": "negative_step_output",
|
||||
"tooltip": "Baseline used by voice guidance.\n"
|
||||
"• negative_step_output: recommended; stronger but uses more compute.\n"
|
||||
"• prompt: uses the reference prompt.\n"
|
||||
"• zero: simplest and fastest baseline.",
|
||||
}),
|
||||
"speed_up_factor": ("FLOAT", {
|
||||
"default": 0.0,
|
||||
"min": 0.0,
|
||||
"max": 2.0,
|
||||
"step": 0.05,
|
||||
"tooltip": "Changes speaking speed using TADA's native method.\n"
|
||||
"0 disables it; values above 1 speak faster.\n"
|
||||
"Enabling it performs two passes and takes about twice as long.",
|
||||
}),
|
||||
"num_transition_steps": ("INT", {
|
||||
"default": 5,
|
||||
"min": 0,
|
||||
"max": 32,
|
||||
"step": 1,
|
||||
"tooltip": "Smooths the change from the reference clip to generated speech.\n"
|
||||
"5 is recommended. Lower is more abrupt.\n"
|
||||
"Higher is smoother but adds work.",
|
||||
}),
|
||||
"use_torch_compile": ("BOOLEAN", {
|
||||
"default": False,
|
||||
"tooltip": "Compile TADA's audio refinement stage for faster repeated generation.\n"
|
||||
"The first run is slower while compiling; later runs reuse a persistent cache.\n"
|
||||
"Uses extra disk space and may use slightly more GPU memory.",
|
||||
}),
|
||||
"runtime_mode": ([RUNTIME_MODE_SHARED_LABEL, RUNTIME_MODE_MAIN_LABEL], {
|
||||
"default": RUNTIME_MODE_SHARED_LABEL,
|
||||
"tooltip": "Choose the Python environment used by TADA.\n"
|
||||
"• Shared Runtime: recommended; uses the compatible Transformers 4 environment.\n"
|
||||
"• Main Environment: only works if ComfyUI already has compatible TADA dependencies.",
|
||||
}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("TTS_ENGINE",)
|
||||
RETURN_NAMES = ("TTS_engine",)
|
||||
FUNCTION = "create_engine_config"
|
||||
CATEGORY = "TTS Audio Suite/⚙️ Engines"
|
||||
|
||||
def create_engine_config(
|
||||
self,
|
||||
model_variant: str,
|
||||
device: str,
|
||||
language: str,
|
||||
acoustic_cfg_scale: float,
|
||||
duration_cfg_scale: float,
|
||||
num_flow_matching_steps: int,
|
||||
noise_temperature: float,
|
||||
dtype: str = "auto",
|
||||
cfg_schedule: str = "cosine",
|
||||
time_schedule: str = "logsnr",
|
||||
negative_condition_source: str = "negative_step_output",
|
||||
speed_up_factor: float = 0.0,
|
||||
num_transition_steps: int = 5,
|
||||
use_torch_compile: bool = False,
|
||||
runtime_mode: str = RUNTIME_MODE_SHARED_LABEL,
|
||||
) -> tuple:
|
||||
validate_tada_model_language(model_variant, language)
|
||||
runtime_mode = normalize_runtime_mode(runtime_mode)
|
||||
runtime_profile = self.RUNTIME_PROFILE if runtime_mode == RUNTIME_MODE_SHARED else None
|
||||
config = {
|
||||
"engine_type": "tada",
|
||||
"model_variant": model_variant,
|
||||
"model_name": model_variant,
|
||||
"model_path": model_variant,
|
||||
"device": device,
|
||||
"dtype": dtype,
|
||||
"use_torch_compile": bool(use_torch_compile),
|
||||
"attn_implementation": "sdpa",
|
||||
"language": language,
|
||||
"acoustic_cfg_scale": float(acoustic_cfg_scale),
|
||||
"duration_cfg_scale": float(duration_cfg_scale),
|
||||
"num_flow_matching_steps": int(num_flow_matching_steps),
|
||||
"noise_temperature": float(noise_temperature),
|
||||
"cfg_schedule": cfg_schedule,
|
||||
"time_schedule": time_schedule,
|
||||
"negative_condition_source": negative_condition_source,
|
||||
"speed_up_factor": float(speed_up_factor),
|
||||
"num_transition_steps": int(num_transition_steps),
|
||||
"runtime_mode": runtime_mode,
|
||||
"runtime_profile": runtime_profile,
|
||||
}
|
||||
|
||||
print(f"⚙️ TADA: Configured {model_variant} on {device} ({language})")
|
||||
runtime_label = "Shared Transformers-4" if runtime_mode == RUNTIME_MODE_SHARED else "Main Environment"
|
||||
print(
|
||||
f" Runtime: {runtime_label} | dtype={dtype} | "
|
||||
f"torch.compile={'on' if use_torch_compile else 'off'}"
|
||||
)
|
||||
print(
|
||||
" Settings: "
|
||||
f"acoustic_cfg={acoustic_cfg_scale}, duration_cfg={duration_cfg_scale}, "
|
||||
f"flow_steps={num_flow_matching_steps}, noise={noise_temperature}"
|
||||
)
|
||||
if speed_up_factor > 0:
|
||||
print(f" Native two-pass speed factor: {speed_up_factor}")
|
||||
|
||||
return ({
|
||||
"engine_type": "tada",
|
||||
"config": config,
|
||||
"capabilities": ["tts"],
|
||||
},)
|
||||
|
||||
|
||||
__all__ = ["TadaEngineNode"]
|
||||
@@ -69,10 +69,11 @@ PRIORITY SYSTEM - When both .txt and .reference.txt exist:
|
||||
"default": "",
|
||||
"tooltip": """Create reference text on-the-fly for connected audio input.
|
||||
|
||||
ENGINE REQUIREMENTS:
|
||||
• F5-TTS: REQUIRES reference text (must match spoken audio exactly)
|
||||
• Higgs Audio 2: Optional but uses reference text if provided
|
||||
• ChatterBox/VibeVoice/IndexTTS: Don't use reference text
|
||||
ENGINE REQUIREMENTS:
|
||||
• Audio8 TTS: REQUIRES the exact spoken transcript for voice cloning
|
||||
• F5-TTS: REQUIRES reference text (must match spoken audio exactly)
|
||||
• Higgs Audio 2: Optional but uses reference text if provided
|
||||
• ChatterBox/VibeVoice/IndexTTS: Don't use reference text
|
||||
|
||||
Selecting a library voice loads its transcription here automatically. Edits are temporary workflow overrides and never modify the source .txt file."""
|
||||
}),
|
||||
|
||||
@@ -208,13 +208,20 @@ class StepAudioEditXAudioEditorNode:
|
||||
self._cached_settings = None
|
||||
|
||||
# Check if engine was deleted (MUST be after settings change check, before early return)
|
||||
engine_was_deleted = (self._engine is not None and
|
||||
hasattr(self._engine, '_tts_engine') and
|
||||
self._engine._tts_engine is None)
|
||||
|
||||
if engine_was_deleted:
|
||||
print("⚠️ Step Audio EditX engine was deleted, reloading...")
|
||||
self._engine = None
|
||||
engine_was_deleted = self._engine is not None and (
|
||||
(
|
||||
hasattr(self._engine, '_tts_engine')
|
||||
and self._engine._tts_engine is None
|
||||
)
|
||||
or (
|
||||
hasattr(self._engine, '_initialized')
|
||||
and not self._engine._initialized
|
||||
)
|
||||
)
|
||||
|
||||
if engine_was_deleted:
|
||||
print("⚠️ Step Audio EditX engine was unloaded, reloading...")
|
||||
self._engine = None
|
||||
elif self._engine is not None:
|
||||
# Using cached engine - ensure it's on the correct device
|
||||
from utils.device import resolve_torch_device
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
"""TADA TTS processors."""
|
||||
|
||||
@@ -1,328 +0,0 @@
|
||||
"""TADA text processor for chunking, character voices, pauses, and tags."""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from typing import Any, Dict, List, Tuple, Union
|
||||
|
||||
import comfy.model_management as model_management
|
||||
import torch
|
||||
|
||||
current_dir = os.path.dirname(__file__)
|
||||
nodes_dir = os.path.dirname(current_dir)
|
||||
project_root = os.path.dirname(nodes_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
from engines.tada.languages import format_tada_language_display, normalize_tada_language
|
||||
from utils.audio.chunk_timing import ChunkTimingHelper
|
||||
from utils.audio.edit_post_processor import process_segments as apply_edit_post_processing
|
||||
from utils.text.character_parser import character_parser
|
||||
from utils.text.pause_processor import PauseTagProcessor
|
||||
from utils.text.segment_parameters import ParameterValidator, apply_segment_parameters
|
||||
from utils.text.step_audio_editx_special_tags import get_edit_tags_for_segment
|
||||
from utils.voice.character_logging import format_resolved_character_block, resolved_character_label
|
||||
from utils.voice.discovery import get_available_characters, get_character_mapping, voice_discovery
|
||||
from utils.voice.reference import effective_voice_audio
|
||||
|
||||
|
||||
class TadaProcessor:
|
||||
"""Orchestrate standard TTS Audio Suite text features around raw TADA calls."""
|
||||
|
||||
SAMPLE_RATE = 24000
|
||||
|
||||
def __init__(self, adapter, engine_config: Dict[str, Any]):
|
||||
self.adapter = adapter
|
||||
self.config = engine_config.copy() if engine_config else {}
|
||||
|
||||
def update_config(self, new_config: Dict[str, Any]):
|
||||
self.config = new_config.copy() if new_config else {}
|
||||
self.adapter.update_config(self.config)
|
||||
|
||||
@staticmethod
|
||||
def _check_interrupt(context: str = "generation") -> None:
|
||||
if model_management.interrupt_processing:
|
||||
raise InterruptedError(f"TADA {context} interrupted by user")
|
||||
|
||||
@staticmethod
|
||||
def _reference_text(voice_ref: Dict[str, Any]) -> str:
|
||||
if not isinstance(voice_ref, dict):
|
||||
return ""
|
||||
return str(
|
||||
voice_ref.get("reference_text")
|
||||
or voice_ref.get("prompt_text")
|
||||
or voice_ref.get("text")
|
||||
or ""
|
||||
).strip()
|
||||
|
||||
def _validate_voice(self, character_name: str, voice_ref: Dict[str, Any]) -> None:
|
||||
if not isinstance(voice_ref, dict) or effective_voice_audio(voice_ref) is None:
|
||||
raise ValueError(
|
||||
f"TADA requires reference audio for '{character_name}'. Connect a narrator voice "
|
||||
"or save a character voice with audio and its exact transcript."
|
||||
)
|
||||
if not self._reference_text(voice_ref):
|
||||
raise ValueError(
|
||||
f"TADA requires the exact reference transcript for '{character_name}'. "
|
||||
"Use Character Voices or add the matching .txt/.reference.txt file. "
|
||||
"Automatic Parakeet transcription is intentionally disabled to prevent a hidden large-model download."
|
||||
)
|
||||
|
||||
def _setup_character_parser(self, text: str) -> None:
|
||||
language_code = normalize_tada_language(self.config.get("language", "English")) or "en"
|
||||
character_parser.language_resolver.default_language = language_code
|
||||
character_parser.default_language = language_code
|
||||
|
||||
text_characters = []
|
||||
for tag in re.findall(r"\[([^\]]+)\]", text or ""):
|
||||
if not tag.lower().startswith(("pause:", "wait:", "stop:")):
|
||||
text_characters.append(tag.split("|")[0].split(":")[-1].strip())
|
||||
|
||||
available = set(get_available_characters() or [])
|
||||
for alias, target in voice_discovery.get_character_aliases().items():
|
||||
available.add(alias.lower())
|
||||
available.add(target.lower())
|
||||
available.update(name.lower() for name in text_characters if name)
|
||||
available.add("narrator")
|
||||
character_parser.set_available_characters(list(available))
|
||||
|
||||
for character, language in voice_discovery.get_character_language_defaults().items():
|
||||
character_parser.set_character_language_default(character, language)
|
||||
character_parser.reset_session_cache()
|
||||
|
||||
def _segment_config(self, segment: Any, base_config: Dict[str, Any], seed: int):
|
||||
params = segment.parameters or {}
|
||||
filtered = ParameterValidator.filter_parameters_for_engine(params, "tada") if params else {}
|
||||
current = apply_segment_parameters(base_config, params, "tada") if params else base_config.copy()
|
||||
current_seed = int(current.get("seed", seed))
|
||||
|
||||
segment_language = getattr(segment, "language", None)
|
||||
base_language = normalize_tada_language(base_config.get("language", "English"))
|
||||
segment_language_code = (
|
||||
normalize_tada_language(segment_language) if segment_language else base_language
|
||||
)
|
||||
should_switch_language = bool(
|
||||
segment_language
|
||||
and (
|
||||
getattr(segment, "explicit_language", False)
|
||||
or segment_language_code != base_language
|
||||
)
|
||||
)
|
||||
if should_switch_language:
|
||||
current["language"] = format_tada_language_display(segment_language)
|
||||
print(f" 🌍 TADA language switched to: {current['language']}")
|
||||
return current, current_seed, filtered
|
||||
|
||||
@staticmethod
|
||||
def _parameter_log(filtered: Dict[str, Any], config: Dict[str, Any], seed: int) -> str:
|
||||
if not filtered:
|
||||
return ""
|
||||
values = []
|
||||
if "seed" in filtered:
|
||||
values.append(f"seed={seed}")
|
||||
keys = (
|
||||
"acoustic_cfg_scale",
|
||||
"duration_cfg_scale",
|
||||
"num_flow_matching_steps",
|
||||
"noise_temperature",
|
||||
"speed_up_factor",
|
||||
)
|
||||
for key in keys:
|
||||
if key in config and (
|
||||
key in filtered
|
||||
or (key == "acoustic_cfg_scale" and "cfg" in filtered)
|
||||
or (key == "num_flow_matching_steps" and "num_steps" in filtered)
|
||||
or (key == "speed_up_factor" and "speed" in filtered)
|
||||
):
|
||||
values.append(f"{key}={config[key]}")
|
||||
return ", ".join(values)
|
||||
|
||||
def process_text(
|
||||
self,
|
||||
text: str,
|
||||
voice_mapping: Dict[str, Any],
|
||||
seed: int,
|
||||
enable_chunking: bool = True,
|
||||
max_chars_per_chunk: int = 400,
|
||||
chunk_combination_method: str = "auto",
|
||||
silence_between_chunks_ms: int = 100,
|
||||
enable_audio_cache: bool = True,
|
||||
apply_edit_postprocessing: bool = True,
|
||||
show_text_logging: bool = True,
|
||||
) -> List[Dict[str, Any]]:
|
||||
del chunk_combination_method, silence_between_chunks_ms
|
||||
self._check_interrupt("text setup")
|
||||
self._setup_character_parser(text)
|
||||
|
||||
narrator = voice_mapping.get("narrator", {})
|
||||
narrator_voice = narrator.copy() if isinstance(narrator, dict) else {"audio": narrator}
|
||||
base_config = self.config.copy()
|
||||
segments = character_parser.parse_text_segments(text)
|
||||
if not segments:
|
||||
segments = character_parser.parse_text_segments("narrator " + text)
|
||||
|
||||
execution_config = base_config.copy()
|
||||
selected_model = str(
|
||||
execution_config.get("model_variant")
|
||||
or execution_config.get("model_name")
|
||||
or "TADA-1B"
|
||||
)
|
||||
segment_languages = {
|
||||
format_tada_language_display(segment.language)
|
||||
for segment in segments
|
||||
if getattr(segment, "language", None)
|
||||
and normalize_tada_language(segment.language) is not None
|
||||
}
|
||||
if "tada-1b" in selected_model.lower().replace("_", "-") and segment_languages:
|
||||
execution_config.update(
|
||||
{
|
||||
"model_variant": "TADA-3B-ML",
|
||||
"model_name": "TADA-3B-ML",
|
||||
"model_path": "TADA-3B-ML",
|
||||
}
|
||||
)
|
||||
languages = ", ".join(sorted(segment_languages))
|
||||
print(f"🌍 Non-English character language detected ({languages})")
|
||||
print("🔄 Switching TADA-1B → TADA-3B-ML for this generation")
|
||||
|
||||
characters = list({segment.character for segment in segments if segment.character})
|
||||
discovered_voices = get_character_mapping(characters, engine_type="tada")
|
||||
records: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
for segment_index, segment in enumerate(segments):
|
||||
self._check_interrupt(f"text segment {segment_index + 1}/{len(segments)}")
|
||||
segment_text = (segment.text or "").strip()
|
||||
if not segment_text:
|
||||
continue
|
||||
|
||||
current_config, current_seed, filtered = self._segment_config(
|
||||
segment, execution_config, seed
|
||||
)
|
||||
self.adapter.update_config(current_config)
|
||||
character = segment.character or "narrator"
|
||||
voice_ref = narrator_voice.copy()
|
||||
|
||||
mapped_voice = voice_mapping.get(character)
|
||||
if character != "narrator" and isinstance(mapped_voice, dict):
|
||||
voice_ref = mapped_voice.copy()
|
||||
elif character != "narrator":
|
||||
audio_path, reference_text = discovered_voices.get(character, (None, None))
|
||||
if audio_path:
|
||||
voice_ref = {"audio_path": audio_path, "reference_text": reference_text or ""}
|
||||
else:
|
||||
discovered = voice_discovery.get_character_voice_info(
|
||||
character, engine_type="audio_only"
|
||||
)
|
||||
if discovered and discovered.get("audio_path") and not str(
|
||||
discovered.get("text_content") or ""
|
||||
).strip():
|
||||
raise ValueError(
|
||||
f"TADA found reference audio for '{character}', but it has no transcript. "
|
||||
"Add a matching .txt or .reference.txt file; TADA cannot use audio-only voices."
|
||||
)
|
||||
|
||||
self._validate_voice(character, voice_ref)
|
||||
|
||||
def generate_text(text_content: str, edit_tags: list) -> None:
|
||||
if enable_chunking:
|
||||
from utils.text.chunking import ImprovedChatterBoxChunker
|
||||
|
||||
max_chars = ImprovedChatterBoxChunker.validate_chunking_params(max_chars_per_chunk)
|
||||
chunks = ImprovedChatterBoxChunker.split_into_chunks(text_content, max_chars=max_chars)
|
||||
else:
|
||||
chunks = [text_content]
|
||||
|
||||
display_name = resolved_character_label(character, voice_ref)
|
||||
language = format_tada_language_display(current_config.get("language", "English"))
|
||||
print(f"🎭 TADA - Generating for '{display_name}' (Language: {language})")
|
||||
parameter_log = self._parameter_log(filtered, current_config, current_seed)
|
||||
if parameter_log:
|
||||
print(f"🎛️ TADA params: {parameter_log}")
|
||||
if show_text_logging:
|
||||
print(format_resolved_character_block(character, text_content, voice_ref))
|
||||
if len(chunks) > 1:
|
||||
print(f"📝 TADA chunking: {len(chunks)} chunks")
|
||||
|
||||
for chunk_index, chunk in enumerate(chunks):
|
||||
self._check_interrupt(
|
||||
f"chunk {chunk_index + 1}/{len(chunks)} for '{display_name}'"
|
||||
)
|
||||
audio = self.adapter.generate_single(
|
||||
text=chunk,
|
||||
voice_ref=voice_ref,
|
||||
seed=current_seed + chunk_index,
|
||||
enable_audio_cache=enable_audio_cache,
|
||||
character_name=character,
|
||||
)
|
||||
if audio.dim() == 1:
|
||||
audio = audio.unsqueeze(0)
|
||||
records.append(
|
||||
{
|
||||
"waveform": audio.detach().cpu().float(),
|
||||
"sample_rate": self.SAMPLE_RATE,
|
||||
"text": chunk,
|
||||
"edit_tags": edit_tags if chunk_index == 0 else [],
|
||||
}
|
||||
)
|
||||
|
||||
if PauseTagProcessor.has_pause_tags(segment_text):
|
||||
pause_segments, _ = PauseTagProcessor.parse_pause_tags(segment_text)
|
||||
for fragment_type, fragment in pause_segments:
|
||||
self._check_interrupt("pause-tag processing")
|
||||
if fragment_type == "text":
|
||||
clean_text, edit_tags = get_edit_tags_for_segment(fragment)
|
||||
if clean_text.strip():
|
||||
generate_text(clean_text.strip(), edit_tags)
|
||||
elif fragment_type == "pause":
|
||||
silence = PauseTagProcessor.create_silence_segment(
|
||||
fragment, self.SAMPLE_RATE, torch.device("cpu"), torch.float32
|
||||
)
|
||||
if silence.dim() == 1:
|
||||
silence = silence.unsqueeze(0)
|
||||
records.append(
|
||||
{
|
||||
"waveform": silence,
|
||||
"sample_rate": self.SAMPLE_RATE,
|
||||
"text": f"[pause:{fragment}s]",
|
||||
"edit_tags": [],
|
||||
}
|
||||
)
|
||||
else:
|
||||
clean_text, edit_tags = get_edit_tags_for_segment(segment_text)
|
||||
if clean_text.strip():
|
||||
generate_text(clean_text.strip(), edit_tags)
|
||||
finally:
|
||||
self.adapter.update_config(base_config)
|
||||
|
||||
if apply_edit_postprocessing and records and any(record["edit_tags"] for record in records):
|
||||
self._check_interrupt("edit post-processing")
|
||||
records = apply_edit_post_processing(records, engine_config=base_config)
|
||||
return records
|
||||
|
||||
def combine_audio_segments(
|
||||
self,
|
||||
segments: List[Dict[str, Any]],
|
||||
method: str = "auto",
|
||||
silence_ms: int = 100,
|
||||
original_text: str = "",
|
||||
return_info: bool = False,
|
||||
) -> Union[torch.Tensor, Tuple[torch.Tensor, Dict[str, Any]]]:
|
||||
if not segments:
|
||||
empty = torch.zeros(0, dtype=torch.float32)
|
||||
return (empty, {}) if return_info else empty
|
||||
|
||||
self._check_interrupt("audio assembly")
|
||||
waveforms = [segment["waveform"] for segment in segments]
|
||||
text_chunks = [segment.get("text", "") for segment in segments]
|
||||
combined, chunk_info = ChunkTimingHelper.combine_audio_with_timing(
|
||||
audio_segments=waveforms,
|
||||
combination_method=method,
|
||||
silence_ms=silence_ms,
|
||||
crossfade_duration=0.1,
|
||||
sample_rate=self.SAMPLE_RATE,
|
||||
text_length=len(" ".join(text_chunks)),
|
||||
original_text=original_text,
|
||||
text_chunks=text_chunks,
|
||||
)
|
||||
return (combined, chunk_info) if return_info else combined
|
||||
@@ -1,254 +0,0 @@
|
||||
"""SRT timing/orchestration for TADA using the normal TADA text processor."""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import comfy.model_management as model_management
|
||||
import torch
|
||||
|
||||
current_dir = os.path.dirname(__file__)
|
||||
nodes_dir = os.path.dirname(current_dir)
|
||||
project_root = os.path.dirname(nodes_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
from engines.adapters.tada_adapter import TadaEngineAdapter
|
||||
|
||||
|
||||
class TadaSRTProcessor:
|
||||
"""Generate each subtitle through :class:`TadaProcessor`, then assemble timing."""
|
||||
|
||||
SAMPLE_RATE = 24000
|
||||
|
||||
def __init__(self, node_instance, config: Dict[str, Any]):
|
||||
self.node_instance = node_instance
|
||||
self.config = config.copy() if config else {}
|
||||
self.adapter = TadaEngineAdapter(self.config)
|
||||
self._tts_processor = None
|
||||
self.srt_available = False
|
||||
self.SRTParser = None
|
||||
self._load_srt_modules()
|
||||
|
||||
def _load_srt_modules(self) -> None:
|
||||
try:
|
||||
from utils.system.import_manager import import_manager
|
||||
|
||||
success, modules, message = import_manager.import_srt_modules()
|
||||
if not success:
|
||||
print(f"⚠️ TADA SRT: SRT module not available: {message}")
|
||||
return
|
||||
self.SRTParser = modules.get("SRTParser")
|
||||
self.srt_available = self.SRTParser is not None
|
||||
except Exception as exc:
|
||||
print(f"⚠️ TADA SRT: Failed to load SRT modules: {exc}")
|
||||
|
||||
@property
|
||||
def processor(self):
|
||||
if self._tts_processor is None:
|
||||
processor_path = os.path.join(current_dir, "tada_processor.py")
|
||||
spec = importlib.util.spec_from_file_location("tada_processor_module", processor_path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
self._tts_processor = module.TadaProcessor(self.adapter, self.config)
|
||||
return self._tts_processor
|
||||
|
||||
def update_config(self, new_config: Dict[str, Any]) -> None:
|
||||
self.config = new_config.copy() if new_config else {}
|
||||
self.adapter.update_config(self.config)
|
||||
if self._tts_processor is not None:
|
||||
self._tts_processor.update_config(self.config)
|
||||
|
||||
@staticmethod
|
||||
def _check_interrupt(index: Optional[int] = None, total: Optional[int] = None) -> None:
|
||||
if model_management.interrupt_processing:
|
||||
location = f" at subtitle {index + 1}/{total}" if index is not None and total else ""
|
||||
raise InterruptedError(f"TADA SRT generation interrupted{location}")
|
||||
|
||||
def process_srt_content(
|
||||
self,
|
||||
srt_content: str,
|
||||
voice_mapping: Dict[str, Any],
|
||||
seed: int,
|
||||
timing_mode: str,
|
||||
timing_params: Dict[str, Any],
|
||||
enable_audio_cache: bool = True,
|
||||
) -> Tuple[Dict[str, Any], str, str, str]:
|
||||
if not self.srt_available:
|
||||
raise ImportError("TADA SRT support is unavailable because the shared SRT parser did not load")
|
||||
|
||||
self._check_interrupt()
|
||||
subtitles = self.SRTParser().parse_srt_content(srt_content, allow_overlaps=True)
|
||||
|
||||
from utils.timing.overlap_detection import SRTOverlapHandler
|
||||
|
||||
has_overlaps = SRTOverlapHandler.detect_overlaps(subtitles)
|
||||
current_mode, mode_switched = SRTOverlapHandler.handle_smart_natural_fallback(
|
||||
timing_mode, has_overlaps, "TADA SRT"
|
||||
)
|
||||
print(f"🚀 TADA SRT: Processing {len(subtitles)} subtitles in {current_mode} mode")
|
||||
|
||||
audio_segments, adjustments = self._generate_subtitles(
|
||||
subtitles, voice_mapping, seed, enable_audio_cache
|
||||
)
|
||||
self._check_interrupt()
|
||||
final_audio, final_adjustments, stretch_method = self._assemble(
|
||||
audio_segments, subtitles, current_mode, timing_params
|
||||
)
|
||||
if final_adjustments is not None:
|
||||
adjustments = final_adjustments
|
||||
|
||||
from utils.timing.reporting import SRTReportGenerator
|
||||
|
||||
reporter = SRTReportGenerator()
|
||||
timing_report = reporter.generate_timing_report(
|
||||
subtitles,
|
||||
adjustments,
|
||||
current_mode,
|
||||
has_overlaps,
|
||||
mode_switched,
|
||||
timing_mode if mode_switched else None,
|
||||
stretch_method,
|
||||
)
|
||||
adjusted_srt = reporter.generate_adjusted_srt_string(subtitles, adjustments, current_mode)
|
||||
|
||||
if final_audio.dim() == 1:
|
||||
final_audio = final_audio.unsqueeze(0).unsqueeze(0)
|
||||
elif final_audio.dim() == 2:
|
||||
final_audio = final_audio.unsqueeze(0)
|
||||
|
||||
duration = final_audio.shape[-1] / self.SAMPLE_RATE
|
||||
mode_info = current_mode
|
||||
if mode_switched:
|
||||
mode_info = f"{current_mode} (switched from {timing_mode} due to overlaps)"
|
||||
info = (
|
||||
f"Generated {duration:.1f}s TADA SRT-timed audio from {len(subtitles)} subtitles "
|
||||
f"using {mode_info} mode"
|
||||
)
|
||||
return {"waveform": final_audio.cpu().float(), "sample_rate": self.SAMPLE_RATE}, info, timing_report, adjusted_srt
|
||||
|
||||
def _generate_subtitles(
|
||||
self,
|
||||
subtitles: List,
|
||||
voice_mapping: Dict[str, Any],
|
||||
seed: int,
|
||||
enable_audio_cache: bool,
|
||||
) -> Tuple[List[torch.Tensor], List[Dict[str, Any]]]:
|
||||
audio_segments: List[torch.Tensor] = []
|
||||
adjustments: List[Dict[str, Any]] = []
|
||||
try:
|
||||
from comfy.utils import ProgressBar
|
||||
|
||||
progress = ProgressBar(len(subtitles))
|
||||
except Exception:
|
||||
progress = None
|
||||
|
||||
for index, subtitle in enumerate(subtitles):
|
||||
self._check_interrupt(index, len(subtitles))
|
||||
text = (subtitle.text or "").strip()
|
||||
target_duration = subtitle.end_time - subtitle.start_time
|
||||
|
||||
if text:
|
||||
print(f"📖 TADA SRT Subtitle {index + 1}/{len(subtitles)}")
|
||||
records = self.processor.process_text(
|
||||
text=text,
|
||||
voice_mapping=voice_mapping,
|
||||
seed=seed + index,
|
||||
enable_chunking=False,
|
||||
enable_audio_cache=enable_audio_cache,
|
||||
apply_edit_postprocessing=True,
|
||||
)
|
||||
if len(records) > 1:
|
||||
audio, _ = self.processor.combine_audio_segments(
|
||||
records, method="auto", silence_ms=0, original_text=text, return_info=True
|
||||
)
|
||||
elif records:
|
||||
audio = records[0]["waveform"]
|
||||
else:
|
||||
audio = torch.zeros(1, 0)
|
||||
else:
|
||||
audio = torch.zeros(1, max(0, int(target_duration * self.SAMPLE_RATE)))
|
||||
|
||||
if audio.dim() == 1:
|
||||
audio = audio.unsqueeze(0)
|
||||
elif audio.dim() == 3:
|
||||
audio = audio.squeeze(0)
|
||||
audio = audio.detach().cpu().float()
|
||||
audio_segments.append(audio)
|
||||
|
||||
natural_duration = audio.shape[-1] / self.SAMPLE_RATE
|
||||
stretch_factor = target_duration / natural_duration if natural_duration > 0 else 1.0
|
||||
adjustments.append(
|
||||
{
|
||||
"index": index,
|
||||
"segment_index": index,
|
||||
"sequence": subtitle.sequence,
|
||||
"natural_duration": natural_duration,
|
||||
"target_start": subtitle.start_time,
|
||||
"target_end": subtitle.end_time,
|
||||
"target_duration": target_duration,
|
||||
"start_time": subtitle.start_time,
|
||||
"end_time": subtitle.end_time,
|
||||
"stretch_factor": stretch_factor,
|
||||
"needs_stretching": abs(stretch_factor - 1.0) > 0.05,
|
||||
"stretch_type": (
|
||||
"compress" if stretch_factor < 1.0 else "expand" if stretch_factor > 1.0 else "none"
|
||||
),
|
||||
"adjustment": natural_duration - target_duration,
|
||||
"adjusted_start": subtitle.start_time,
|
||||
"adjusted_end": subtitle.end_time,
|
||||
"adjusted_duration": natural_duration,
|
||||
}
|
||||
)
|
||||
print(
|
||||
f"✅ TADA Subtitle {index + 1}/{len(subtitles)}: "
|
||||
f"{natural_duration:.2f}s (target {target_duration:.2f}s)"
|
||||
)
|
||||
if progress is not None:
|
||||
progress.update(1)
|
||||
|
||||
return audio_segments, adjustments
|
||||
|
||||
def _assemble(self, audio_segments, subtitles, timing_mode, timing_params):
|
||||
self._check_interrupt()
|
||||
if timing_mode == "stretch_to_fit":
|
||||
from engines.chatterbox.audio_timing import TimedAudioAssembler
|
||||
|
||||
assembler = TimedAudioAssembler(self.SAMPLE_RATE)
|
||||
target_timings = [(subtitle.start_time, subtitle.end_time) for subtitle in subtitles]
|
||||
final_audio, stretch_method = assembler.assemble_timed_audio(
|
||||
audio_segments,
|
||||
target_timings,
|
||||
fade_duration=timing_params.get("fade_for_StretchToFit", 0.01),
|
||||
)
|
||||
return final_audio, None, stretch_method
|
||||
|
||||
from utils.timing.assembly import AudioAssemblyEngine
|
||||
|
||||
assembler = AudioAssemblyEngine(self.SAMPLE_RATE)
|
||||
if timing_mode == "pad_with_silence":
|
||||
return assembler.assemble_with_overlaps(audio_segments, subtitles, torch.device("cpu")), None, None
|
||||
|
||||
from utils.timing.engine import TimingEngine
|
||||
|
||||
timing_engine = TimingEngine(self.SAMPLE_RATE)
|
||||
if timing_mode == "concatenate":
|
||||
adjustments = timing_engine.calculate_concatenation_adjustments(audio_segments, subtitles)
|
||||
audio = assembler.assemble_concatenation(
|
||||
audio_segments, timing_params.get("fade_for_StretchToFit", 0.01)
|
||||
)
|
||||
return audio, adjustments, None
|
||||
|
||||
adjustments, processed = timing_engine.calculate_smart_timing_adjustments(
|
||||
audio_segments,
|
||||
subtitles,
|
||||
timing_params.get("timing_tolerance", 2.0),
|
||||
timing_params.get("max_stretch_ratio", 1.0),
|
||||
timing_params.get("min_stretch_ratio", 0.5),
|
||||
torch.device("cpu"),
|
||||
)
|
||||
audio = assembler.assemble_smart_natural(
|
||||
audio_segments, processed, adjustments, subtitles, torch.device("cpu")
|
||||
)
|
||||
return audio, adjustments, None
|
||||
@@ -84,7 +84,7 @@ Hello! This is unified SRT TTS with character switching.
|
||||
}),
|
||||
"narrator_voice": (reference_files, {
|
||||
"default": "none",
|
||||
"tooltip": "Fallback narrator voice from voice folders. Used when opt_narrator is not connected. Select 'none' for engines that support direct TTS without voice cloning, such as MOSS."
|
||||
"tooltip": "Fallback narrator voice from voice folders. Used when opt_narrator is not connected. Select 'none' for engines that support direct TTS without voice cloning, such as MOSS or Audio8."
|
||||
}),
|
||||
"seed": ("INT", {
|
||||
"default": 1, "min": 0, "max": 2**32 - 1,
|
||||
@@ -223,12 +223,11 @@ Hello! This is unified SRT TTS with character switching.
|
||||
stable_params['optimize'] = config.get('optimize', False)
|
||||
stable_params['max_generate_length'] = config.get('max_generate_length', 500)
|
||||
|
||||
if engine_type == "tada":
|
||||
stable_params['model_variant'] = config.get('model_variant', 'TADA-1B')
|
||||
if engine_type == "audio8_tts":
|
||||
stable_params['model_variant'] = config.get(
|
||||
'model_variant', 'Audio8-TTS-Preview-0.6b'
|
||||
)
|
||||
stable_params['dtype'] = config.get('dtype', 'auto')
|
||||
stable_params['attn_implementation'] = config.get('attn_implementation', 'sdpa')
|
||||
stable_params['runtime_mode'] = config.get('runtime_mode')
|
||||
stable_params['runtime_profile'] = config.get('runtime_profile')
|
||||
|
||||
if engine_type == "dramabox":
|
||||
stable_params['model_name'] = config.get('model_name', 'DramaBox')
|
||||
@@ -585,19 +584,21 @@ Hello! This is unified SRT TTS with character switching.
|
||||
}
|
||||
return engine_instance
|
||||
|
||||
elif engine_type == "tada":
|
||||
processor_path = os.path.join(nodes_dir, "tada", "tada_srt_processor.py")
|
||||
elif engine_type == "audio8_tts":
|
||||
processor_path = os.path.join(
|
||||
nodes_dir, "audio8_tts", "audio8_tts_srt_processor.py"
|
||||
)
|
||||
processor_spec = importlib.util.spec_from_file_location(
|
||||
"tada_srt_processor_module", processor_path
|
||||
"audio8_tts_srt_processor_module", processor_path
|
||||
)
|
||||
processor_module = importlib.util.module_from_spec(processor_spec)
|
||||
processor_spec.loader.exec_module(processor_module)
|
||||
TadaSRTProcessor = processor_module.TadaSRTProcessor
|
||||
Audio8TTSSRTProcessor = processor_module.Audio8TTSSRTProcessor
|
||||
|
||||
class TadaSRTWrapper:
|
||||
class Audio8TTSSRTWrapper:
|
||||
def __init__(self, cfg):
|
||||
self.config = cfg.copy()
|
||||
self.processor = TadaSRTProcessor(self, self.config)
|
||||
self.processor = Audio8TTSSRTProcessor(self, self.config)
|
||||
|
||||
def update_config(self, new_config):
|
||||
self.config = new_config.copy()
|
||||
@@ -605,10 +606,11 @@ Hello! This is unified SRT TTS with character switching.
|
||||
|
||||
def check_interrupt(self):
|
||||
if model_management.interrupt_processing:
|
||||
raise InterruptedError("TADA SRT processing interrupted by user")
|
||||
|
||||
engine_instance = TadaSRTWrapper(config)
|
||||
raise InterruptedError(
|
||||
"Audio8 TTS SRT processing interrupted by user"
|
||||
)
|
||||
|
||||
engine_instance = Audio8TTSSRTWrapper(config)
|
||||
import time
|
||||
self._cached_engine_instances[cache_key] = {
|
||||
'instance': engine_instance,
|
||||
@@ -1025,7 +1027,13 @@ Hello! This is unified SRT TTS with character switching.
|
||||
raise ValueError(f"Unknown engine type: {engine_type}")
|
||||
|
||||
except Exception as e:
|
||||
if isinstance(e, InterruptedError):
|
||||
if isinstance(
|
||||
e,
|
||||
(
|
||||
InterruptedError,
|
||||
model_management.InterruptProcessingException,
|
||||
),
|
||||
):
|
||||
raise
|
||||
print(f"❌ Failed to create engine SRT node instance: {e}")
|
||||
return None
|
||||
@@ -1091,7 +1099,7 @@ Hello! This is unified SRT TTS with character switching.
|
||||
print(f"📺 TTS SRT: Using direct audio input ({character_name})")
|
||||
print(
|
||||
"⚠️ TTS SRT: Direct audio input has no reference text - "
|
||||
"F5-TTS and OmniVoice cloning will fail"
|
||||
"Audio8 TTS, F5-TTS, and OmniVoice cloning will fail"
|
||||
)
|
||||
return None, audio_tensor, reference_text, character_name
|
||||
|
||||
@@ -1118,7 +1126,13 @@ Hello! This is unified SRT TTS with character switching.
|
||||
return None, None, "", "narrator"
|
||||
|
||||
except Exception as e:
|
||||
if isinstance(e, InterruptedError):
|
||||
if isinstance(
|
||||
e,
|
||||
(
|
||||
InterruptedError,
|
||||
model_management.InterruptProcessingException,
|
||||
),
|
||||
):
|
||||
raise
|
||||
print(f"❌ Voice reference error: {e}")
|
||||
return None, None, "", "narrator"
|
||||
@@ -1147,11 +1161,10 @@ Hello! This is unified SRT TTS with character switching.
|
||||
batch_size: Number of parallel workers (0=sequential, 1+=streaming parallel)
|
||||
use_native_duration_targeting: Inject subtitle durations into supported engines during generation
|
||||
|
||||
Returns:
|
||||
Tuple of (audio_tensor, generation_info, timing_report, adjusted_srt)
|
||||
"""
|
||||
engine_type = None
|
||||
try:
|
||||
Returns:
|
||||
Tuple of (audio_tensor, generation_info, timing_report, adjusted_srt)
|
||||
"""
|
||||
try:
|
||||
# Apply Python 3.12 CUDNN compatibility fix before TTS generation
|
||||
from utils.comfyui_compatibility import ensure_python312_cudnn_fix
|
||||
ensure_python312_cudnn_fix()
|
||||
@@ -1214,10 +1227,12 @@ Hello! This is unified SRT TTS with character switching.
|
||||
"OmniVoice voice cloning requires reference text. "
|
||||
"Do not connect raw audio directly. Use Character Voices node or a narrator voice with a matching .reference.txt file."
|
||||
)
|
||||
if engine_type == "tada" and (audio_tensor is not None or audio_path) and not reference_text.strip():
|
||||
if engine_type == "audio8_tts" and (audio_tensor is not None or audio_path) and not reference_text.strip():
|
||||
raise ValueError(
|
||||
"TADA requires the exact reference transcript. Do not connect raw audio directly. "
|
||||
"Use Character Voices or a narrator voice with a matching .reference.txt file."
|
||||
"Audio8 TTS voice cloning requires reference text. "
|
||||
"Do not connect raw audio directly. Use Character Voices with "
|
||||
"the exact transcript, or a narrator voice with a matching "
|
||||
".reference.txt file."
|
||||
)
|
||||
|
||||
# Create proper engine SRT node instance to preserve ALL functionality
|
||||
@@ -1419,7 +1434,7 @@ Hello! This is unified SRT TTS with character switching.
|
||||
enable_audio_cache=enable_audio_cache
|
||||
)
|
||||
|
||||
elif engine_type == "tada":
|
||||
elif engine_type == "audio8_tts":
|
||||
timing_params = {
|
||||
'fade_for_StretchToFit': fade_for_StretchToFit,
|
||||
'max_stretch_ratio': max_stretch_ratio,
|
||||
@@ -1432,8 +1447,8 @@ Hello! This is unified SRT TTS with character switching.
|
||||
'audio': audio_tensor,
|
||||
'audio_path': audio_path,
|
||||
'reference_text': reference_text or '',
|
||||
'character_name': character_name or 'narrator',
|
||||
}
|
||||
|
||||
result = engine_instance.processor.process_srt_content(
|
||||
srt_content=srt_content,
|
||||
voice_mapping=voice_mapping,
|
||||
@@ -1756,11 +1771,16 @@ Hello! This is unified SRT TTS with character switching.
|
||||
"is a voice-design model and cannot be used with TTS SRT" in msg
|
||||
or "Pause tags are not compatible with force_speaker_kv" in msg
|
||||
or "MOSS-TTSD Native Multi-Speaker Dialogue does not support this SRT input" in msg
|
||||
or "Audio8 TTS voice cloning requires reference text" in msg
|
||||
):
|
||||
raise
|
||||
if engine_type == "tada":
|
||||
raise
|
||||
if isinstance(e, InterruptedError):
|
||||
if isinstance(
|
||||
e,
|
||||
(
|
||||
InterruptedError,
|
||||
model_management.InterruptProcessingException,
|
||||
),
|
||||
):
|
||||
raise
|
||||
error_msg = f"❌ TTS SRT generation failed: {e}"
|
||||
print(error_msg)
|
||||
|
||||
@@ -89,7 +89,7 @@ Back to the main narrator voice for the conclusion.""",
|
||||
}),
|
||||
"narrator_voice": (reference_files, {
|
||||
"default": "none",
|
||||
"tooltip": "Fallback narrator voice from voice folders. Used when opt_narrator is not connected. Select 'none' for engines that support direct TTS without voice cloning, such as MOSS."
|
||||
"tooltip": "Fallback narrator voice from voice folders. Used when opt_narrator is not connected. Select 'none' for engines that support direct TTS without voice cloning, such as MOSS or Audio8."
|
||||
}),
|
||||
"seed": ("INT", {
|
||||
"default": 1, "min": 0, "max": 2**32 - 1,
|
||||
@@ -218,12 +218,11 @@ Back to the main narrator voice for the conclusion.""",
|
||||
stable_params['optimize'] = config.get('optimize', False)
|
||||
stable_params['max_generate_length'] = config.get('max_generate_length', 500)
|
||||
|
||||
if engine_type == "tada":
|
||||
stable_params['model_variant'] = config.get('model_variant', 'TADA-1B')
|
||||
if engine_type == "audio8_tts":
|
||||
stable_params['model_variant'] = config.get(
|
||||
'model_variant', 'Audio8-TTS-Preview-0.6b'
|
||||
)
|
||||
stable_params['dtype'] = config.get('dtype', 'auto')
|
||||
stable_params['attn_implementation'] = config.get('attn_implementation', 'sdpa')
|
||||
stable_params['runtime_mode'] = config.get('runtime_mode')
|
||||
stable_params['runtime_profile'] = config.get('runtime_profile')
|
||||
|
||||
if engine_type == "dramabox":
|
||||
stable_params['model_name'] = config.get('model_name', 'DramaBox')
|
||||
@@ -452,7 +451,13 @@ Back to the main narrator voice for the conclusion.""",
|
||||
voice_mapping[char] = {"waveform": waveform, "sample_rate": sample_rate}
|
||||
print(f"🎭 VibeVoice: Using character-specific voice for '{char}'")
|
||||
except Exception as e:
|
||||
if isinstance(e, InterruptedError):
|
||||
if isinstance(
|
||||
e,
|
||||
(
|
||||
InterruptedError,
|
||||
model_management.InterruptProcessingException,
|
||||
),
|
||||
):
|
||||
raise
|
||||
print(f"⚠️ Failed to load character audio for '{char}': {e}")
|
||||
voice_mapping[char] = char_audio # Fallback to main voice
|
||||
@@ -587,30 +592,31 @@ Back to the main narrator voice for the conclusion.""",
|
||||
|
||||
return engine_instance
|
||||
|
||||
elif engine_type == "tada":
|
||||
from engines.adapters.tada_adapter import TadaEngineAdapter
|
||||
elif engine_type == "audio8_tts":
|
||||
from engines.adapters.audio8_tts_adapter import Audio8TTSEngineAdapter
|
||||
|
||||
processor_path = os.path.join(nodes_dir, "tada", "tada_processor.py")
|
||||
processor_path = os.path.join(
|
||||
nodes_dir, "audio8_tts", "audio8_tts_processor.py"
|
||||
)
|
||||
processor_spec = importlib.util.spec_from_file_location(
|
||||
"tada_processor_module", processor_path
|
||||
"audio8_tts_processor_module", processor_path
|
||||
)
|
||||
processor_module = importlib.util.module_from_spec(processor_spec)
|
||||
processor_spec.loader.exec_module(processor_module)
|
||||
TadaProcessor = processor_module.TadaProcessor
|
||||
Audio8TTSProcessor = processor_module.Audio8TTSProcessor
|
||||
|
||||
class TadaWrapper:
|
||||
class Audio8TTSWrapper:
|
||||
def __init__(self, cfg):
|
||||
self.config = cfg.copy()
|
||||
self.adapter = TadaEngineAdapter(self.config)
|
||||
self.processor = TadaProcessor(self.adapter, self.config)
|
||||
self.adapter = Audio8TTSEngineAdapter(self.config)
|
||||
self.processor = Audio8TTSProcessor(self.adapter, self.config)
|
||||
|
||||
def update_config(self, new_config):
|
||||
self.config = new_config.copy()
|
||||
self.adapter.update_config(new_config)
|
||||
self.processor.update_config(new_config)
|
||||
|
||||
engine_instance = TadaWrapper(config)
|
||||
|
||||
engine_instance = Audio8TTSWrapper(config)
|
||||
import time
|
||||
self._cached_engine_instances[cache_key] = {
|
||||
'instance': engine_instance,
|
||||
@@ -875,7 +881,13 @@ Back to the main narrator voice for the conclusion.""",
|
||||
raise ValueError(f"Unknown engine type: {engine_type}")
|
||||
|
||||
except Exception as e:
|
||||
if isinstance(e, InterruptedError):
|
||||
if isinstance(
|
||||
e,
|
||||
(
|
||||
InterruptedError,
|
||||
model_management.InterruptProcessingException,
|
||||
),
|
||||
):
|
||||
raise
|
||||
if "MOSS LoRA/base model mismatch" in str(e):
|
||||
raise
|
||||
@@ -944,7 +956,8 @@ Back to the main narrator voice for the conclusion.""",
|
||||
print(f"🎤 TTS Text: Using direct audio input ({character_name})")
|
||||
print(
|
||||
"⚠️ TTS Text: Direct audio input has no reference text - "
|
||||
"F5-TTS and OmniVoice cloning will fail, Qwen3-TTS will use x_vector_only mode (lower quality)"
|
||||
"Audio8 TTS, F5-TTS, and OmniVoice cloning will fail; "
|
||||
"Qwen3-TTS will use x_vector_only mode (lower quality)"
|
||||
)
|
||||
return None, audio_tensor, reference_text, character_name
|
||||
|
||||
@@ -990,7 +1003,13 @@ Back to the main narrator voice for the conclusion.""",
|
||||
return None, None, "", "narrator"
|
||||
|
||||
except Exception as e:
|
||||
if isinstance(e, InterruptedError):
|
||||
if isinstance(
|
||||
e,
|
||||
(
|
||||
InterruptedError,
|
||||
model_management.InterruptProcessingException,
|
||||
),
|
||||
):
|
||||
raise
|
||||
print(f"❌ Voice reference error: {e}")
|
||||
return None, None, "", "narrator"
|
||||
@@ -1097,10 +1116,12 @@ Back to the main narrator voice for the conclusion.""",
|
||||
"OmniVoice voice cloning requires reference text. "
|
||||
"Do not connect raw audio directly. Use Character Voices node or a narrator voice with a matching .reference.txt file."
|
||||
)
|
||||
if engine_type == "tada" and (audio_tensor is not None or audio_path) and not reference_text.strip():
|
||||
if engine_type == "audio8_tts" and (audio_tensor is not None or audio_path) and not reference_text.strip():
|
||||
raise ValueError(
|
||||
"TADA requires the exact reference transcript. Do not connect raw audio directly. "
|
||||
"Use Character Voices or a narrator voice with a matching .reference.txt file."
|
||||
"Audio8 TTS voice cloning requires reference text. "
|
||||
"Do not connect raw audio directly. Use Character Voices with "
|
||||
"the exact transcript, or a narrator voice with a matching "
|
||||
".reference.txt file."
|
||||
)
|
||||
|
||||
# Create proper engine node instance to preserve ALL functionality
|
||||
@@ -1563,7 +1584,7 @@ Back to the main narrator voice for the conclusion.""",
|
||||
formatted_audio = AudioProcessingUtils.format_for_comfyui(combined_audio, 48000)
|
||||
result = (formatted_audio, generation_info)
|
||||
|
||||
elif engine_type == "tada":
|
||||
elif engine_type == "audio8_tts":
|
||||
import re
|
||||
from utils.audio.chunk_timing import ChunkTimingHelper
|
||||
|
||||
@@ -1573,6 +1594,7 @@ Back to the main narrator voice for the conclusion.""",
|
||||
'audio': audio_tensor,
|
||||
'audio_path': audio_path,
|
||||
'reference_text': reference_text or '',
|
||||
'character_name': character_name or 'narrator',
|
||||
}
|
||||
|
||||
segment_records = engine_instance.processor.process_text(
|
||||
@@ -1593,18 +1615,27 @@ Back to the main narrator voice for the conclusion.""",
|
||||
return_info=True,
|
||||
)
|
||||
|
||||
total_duration = combined_audio.shape[-1] / 24000.0 if combined_audio.numel() else 0.0
|
||||
total_duration = (
|
||||
combined_audio.shape[-1] / 44100.0
|
||||
if combined_audio.numel()
|
||||
else 0.0
|
||||
)
|
||||
clean_text = re.sub(r'\[.*?\]', '', text)
|
||||
base_info = (
|
||||
f"Generated {total_duration:.1f}s audio from {len(clean_text)} characters "
|
||||
f"(TADA, narrator: {char_display})"
|
||||
f"(Audio8 TTS, narrator: {char_display})"
|
||||
)
|
||||
base_info += (
|
||||
"\n🎭 Reference-free speech, zero-shot cloning, character switching, "
|
||||
"and pause tags enabled"
|
||||
)
|
||||
base_info += "\n🎭 Character switching, expression transfer, and pause tags enabled"
|
||||
generation_info = ChunkTimingHelper.enhance_generation_info(
|
||||
f"✅ {base_info}", chunk_info
|
||||
)
|
||||
formatted_audio = AudioProcessingUtils.format_for_comfyui(combined_audio, 24000)
|
||||
result = (formatted_audio, generation_info)
|
||||
result = (
|
||||
AudioProcessingUtils.format_for_comfyui(combined_audio, 44100),
|
||||
generation_info,
|
||||
)
|
||||
|
||||
elif engine_type == "dramabox":
|
||||
import re
|
||||
@@ -1986,7 +2017,13 @@ Back to the main narrator voice for the conclusion.""",
|
||||
}
|
||||
print(f"🎭 Qwen3-TTS: Using character-specific voice for '{character}' (ICL mode)")
|
||||
except Exception as e:
|
||||
if isinstance(e, InterruptedError):
|
||||
if isinstance(
|
||||
e,
|
||||
(
|
||||
InterruptedError,
|
||||
model_management.InterruptProcessingException,
|
||||
),
|
||||
):
|
||||
raise
|
||||
print(f"⚠️ Failed to load character audio for '{character}': {e}")
|
||||
# Fallback to narrator voice if available
|
||||
@@ -2031,7 +2068,13 @@ Back to the main narrator voice for the conclusion.""",
|
||||
print(f"⚠️⚠️ Qwen3-TTS: Character '{character}' has audio but NO reference text")
|
||||
print(f"⚠️⚠️ Using x_vector_only mode (speaker embedding only) - LOWER QUALITY than ICL mode")
|
||||
except Exception as e:
|
||||
if isinstance(e, InterruptedError):
|
||||
if isinstance(
|
||||
e,
|
||||
(
|
||||
InterruptedError,
|
||||
model_management.InterruptProcessingException,
|
||||
),
|
||||
):
|
||||
raise
|
||||
print(f"⚠️ Failed to load character audio for '{character}': {e}")
|
||||
# Fallback to narrator voice if available
|
||||
@@ -2168,11 +2211,17 @@ Back to the main narrator voice for the conclusion.""",
|
||||
raise
|
||||
if "MOSS LoRA/base model mismatch" in str(e):
|
||||
raise
|
||||
if "Audio8 TTS voice cloning requires reference text" in str(e):
|
||||
raise
|
||||
if engine_type == "index_tts":
|
||||
raise
|
||||
if engine_type == "tada":
|
||||
raise
|
||||
if isinstance(e, InterruptedError):
|
||||
if isinstance(
|
||||
e,
|
||||
(
|
||||
InterruptedError,
|
||||
model_management.InterruptProcessingException,
|
||||
),
|
||||
):
|
||||
raise
|
||||
error_msg = f"❌ TTS Text generation failed: {e}"
|
||||
print(error_msg)
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
@@ -41,3 +44,40 @@ def test_build_env_preserves_valid_pythonhashseed(monkeypatch, value):
|
||||
env = IsolatedRuntimeLauncher().build_env(profile)
|
||||
|
||||
assert env["PYTHONHASHSEED"] == value
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_audio8_worker_prioritizes_project_root_over_shadowing_utils(tmp_path):
|
||||
shadow_dir = tmp_path / "step_audio_editx_impl"
|
||||
shadow_dir.mkdir()
|
||||
(shadow_dir / "utils.py").write_text(
|
||||
'raise RuntimeError("shadow utils imported")\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
worker_path = REPO_ROOT / "utils" / "runtimes" / "workers" / "audio8_tts_worker.py"
|
||||
env = os.environ.copy()
|
||||
env["PYTHONPATH"] = os.pathsep.join((str(shadow_dir), str(REPO_ROOT)))
|
||||
completed = subprocess.run(
|
||||
[sys.executable, str(worker_path)],
|
||||
input=json.dumps(
|
||||
{
|
||||
"action": "ping",
|
||||
"request_id": "audio8-path-priority-test",
|
||||
}
|
||||
)
|
||||
+ "\n",
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
env=env,
|
||||
cwd=str(REPO_ROOT),
|
||||
timeout=30,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
response = json.loads(completed.stdout.strip())
|
||||
assert response["ok"] is True
|
||||
assert response["result"] == {"pong": True}
|
||||
|
||||
+29
-64
@@ -413,67 +413,6 @@ class Qwen3TTSCacheKeyGenerator(CacheKeyGenerator):
|
||||
return hashlib.md5(cache_string.encode()).hexdigest()
|
||||
|
||||
|
||||
class TadaCacheKeyGenerator(CacheKeyGenerator):
|
||||
"""Cache key generator for the TADA engine."""
|
||||
|
||||
def generate_cache_key(self, **params) -> str:
|
||||
"""Include every TADA input that can change the generated waveform."""
|
||||
rounded_fields = {
|
||||
'acoustic_cfg_scale': params.get('acoustic_cfg_scale', 1.6),
|
||||
'duration_cfg_scale': params.get('duration_cfg_scale', 1.0),
|
||||
'noise_temperature': params.get('noise_temperature', 0.9),
|
||||
'speed_up_factor': params.get('speed_up_factor'),
|
||||
}
|
||||
for key, value in list(rounded_fields.items()):
|
||||
if isinstance(value, (int, float)):
|
||||
rounded_fields[key] = round(float(value), 3)
|
||||
|
||||
audio_component = (
|
||||
params.get('audio_component')
|
||||
or params.get('reference_audio_hash')
|
||||
or params.get('prompt_audio_hash')
|
||||
or ''
|
||||
)
|
||||
reference_text = (
|
||||
params.get('reference_text')
|
||||
or params.get('prompt_text')
|
||||
or params.get('ref_text')
|
||||
or ''
|
||||
)
|
||||
|
||||
cache_data = {
|
||||
'text': params.get('text', ''),
|
||||
'model_variant': params.get('model_variant', 'TADA-1B'),
|
||||
'language': params.get('language', 'English'),
|
||||
'audio_component': audio_component,
|
||||
'reference_text': reference_text,
|
||||
'seed': params.get('seed', 0),
|
||||
'acoustic_cfg_scale': rounded_fields['acoustic_cfg_scale'],
|
||||
'duration_cfg_scale': rounded_fields['duration_cfg_scale'],
|
||||
'noise_temperature': rounded_fields['noise_temperature'],
|
||||
'num_flow_matching_steps': params.get('num_flow_matching_steps', 10),
|
||||
'cfg_schedule': params.get('cfg_schedule', 'cosine'),
|
||||
'time_schedule': params.get('time_schedule', 'logsnr'),
|
||||
'negative_condition_source': params.get(
|
||||
'negative_condition_source', 'negative_step_output'
|
||||
),
|
||||
'speed_up_factor': rounded_fields['speed_up_factor'],
|
||||
'num_transition_steps': params.get('num_transition_steps', 5),
|
||||
'device': params.get('device', 'auto'),
|
||||
'dtype': params.get('dtype', 'auto'),
|
||||
'attn_implementation': params.get('attn_implementation', 'sdpa'),
|
||||
'runtime_mode': params.get('runtime_mode', params.get('runtime', 'shared_runtime')),
|
||||
'runtime_profile': params.get(
|
||||
'runtime_profile', 'vibevoice_transformers4_shared'
|
||||
),
|
||||
'character': params.get('character', 'narrator'),
|
||||
'engine': 'tada',
|
||||
}
|
||||
|
||||
cache_string = str(sorted(cache_data.items()))
|
||||
return hashlib.md5(cache_string.encode()).hexdigest()
|
||||
|
||||
|
||||
class MossTTSCacheKeyGenerator(CacheKeyGenerator):
|
||||
"""Cache key generator for MOSS-TTS engine."""
|
||||
|
||||
@@ -638,6 +577,30 @@ class DotsTTSCacheKeyGenerator(CacheKeyGenerator):
|
||||
return hashlib.md5(cache_string.encode()).hexdigest()
|
||||
|
||||
|
||||
class Audio8TTSCacheKeyGenerator(CacheKeyGenerator):
|
||||
"""Cache key generator for official Audio8 TTS inference."""
|
||||
|
||||
def generate_cache_key(self, **params) -> str:
|
||||
cache_data = {
|
||||
'text': params.get('text', ''),
|
||||
'audio_component': params.get('audio_component', ''),
|
||||
'reference_text': params.get('reference_text', ''),
|
||||
'model_variant': params.get('model_variant', 'Audio8-TTS-Preview-0.6b'),
|
||||
'max_new_tokens': int(params.get('max_new_tokens', 1024)),
|
||||
'retry_max_new_tokens': int(params.get('retry_max_new_tokens', 2000)),
|
||||
'temperature': round(float(params.get('temperature', 0.8)), 4),
|
||||
'top_p': round(float(params.get('top_p', 0.95)), 4),
|
||||
'top_k': int(params.get('top_k', 50)),
|
||||
'do_sample': bool(params.get('do_sample', True)),
|
||||
'dtype': params.get('dtype', 'auto'),
|
||||
'device': params.get('device', 'auto'),
|
||||
'seed': int(params.get('seed', 42)),
|
||||
'character': params.get('character', 'narrator'),
|
||||
'engine': 'audio8_tts',
|
||||
}
|
||||
return hashlib.md5(str(sorted(cache_data.items())).encode()).hexdigest()
|
||||
|
||||
|
||||
class DramaBoxCacheKeyGenerator(CacheKeyGenerator):
|
||||
"""Cache key generator for official DramaBox inference."""
|
||||
|
||||
@@ -770,8 +733,8 @@ class AudioCache:
|
||||
'index_tts': IndexTTSCacheKeyGenerator(),
|
||||
'cosyvoice': CosyVoiceCacheKeyGenerator(),
|
||||
'qwen3_tts': Qwen3TTSCacheKeyGenerator(),
|
||||
'tada': TadaCacheKeyGenerator(),
|
||||
'dots_tts': DotsTTSCacheKeyGenerator(),
|
||||
'audio8_tts': Audio8TTSCacheKeyGenerator(),
|
||||
'dramabox': DramaBoxCacheKeyGenerator(),
|
||||
'fish_audio_s2': FishAudioS2CacheKeyGenerator(),
|
||||
'omnivoice': OmniVoiceCacheKeyGenerator(),
|
||||
@@ -850,11 +813,13 @@ class AudioCache:
|
||||
# Use engine-specific sample rates
|
||||
if engine_type in ('dots_tts', 'dramabox', 'moss_soundeffect_v2'):
|
||||
sample_rate = 48000
|
||||
elif engine_type in ('f5tts', 'step_audio_editx', 'qwen3_tts', 'tada', 'moss_tts', 'higgs_audio_v3', 'omnivoice'):
|
||||
elif engine_type in ('f5tts', 'step_audio_editx', 'qwen3_tts', 'moss_tts', 'higgs_audio_v3', 'omnivoice'):
|
||||
sample_rate = 24000
|
||||
elif engine_type in ('index_tts', 'cosyvoice'):
|
||||
sample_rate = 22050
|
||||
else:
|
||||
elif engine_type == 'audio8_tts':
|
||||
sample_rate = 44100
|
||||
else:
|
||||
sample_rate = 44100
|
||||
return num_samples / sample_rate
|
||||
|
||||
|
||||
@@ -5,9 +5,7 @@ Centralized downloading for all models (F5-TTS, ChatterBox, RVC, etc.) without c
|
||||
|
||||
import os
|
||||
import requests
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Optional, Dict, Any, List
|
||||
from pathlib import Path
|
||||
import fnmatch
|
||||
@@ -36,28 +34,6 @@ class UnifiedDownloader:
|
||||
if not self.verify_ssl:
|
||||
print("⚠️ SSL certificate verification is DISABLED")
|
||||
print(" Set environment variable TTS_DISABLE_SSL_VERIFY=0 to re-enable")
|
||||
|
||||
@staticmethod
|
||||
def _resolve_cli_executable(command: str) -> str:
|
||||
"""Resolve console scripts installed beside the active Python executable."""
|
||||
discovered = shutil.which(command)
|
||||
if discovered:
|
||||
return discovered
|
||||
|
||||
suffix = ".exe" if os.name == "nt" else ""
|
||||
executable_name = command if command.endswith(suffix) else f"{command}{suffix}"
|
||||
python_dir = Path(sys.executable).resolve().parent
|
||||
candidates = (
|
||||
python_dir / executable_name,
|
||||
python_dir / "Scripts" / executable_name,
|
||||
python_dir.parent / "Scripts" / executable_name,
|
||||
python_dir / "bin" / executable_name,
|
||||
python_dir.parent / "bin" / executable_name,
|
||||
)
|
||||
for candidate in candidates:
|
||||
if candidate.is_file():
|
||||
return str(candidate)
|
||||
return command
|
||||
|
||||
def download_from_hf_cli(
|
||||
self,
|
||||
@@ -87,7 +63,6 @@ class UnifiedDownloader:
|
||||
|
||||
try:
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
print(f"📥 Downloading {filename} from {repo_id}")
|
||||
|
||||
if force_download and os.path.exists(target_path):
|
||||
try:
|
||||
@@ -99,10 +74,10 @@ class UnifiedDownloader:
|
||||
env = os.environ.copy()
|
||||
env['PYTHONIOENCODING'] = 'utf-8'
|
||||
|
||||
# Try modern 'hf download' first, then the legacy CLI. Capture their
|
||||
# noisy path/warning output and present consistent suite-level status.
|
||||
# Try modern 'hf download' first, fallback to legacy 'huggingface-cli download'
|
||||
# Show progress bar by not capturing output
|
||||
hf_command = [
|
||||
self._resolve_cli_executable("hf"), "download",
|
||||
"hf", "download",
|
||||
repo_id,
|
||||
filename,
|
||||
"--local-dir", target_dir
|
||||
@@ -115,20 +90,16 @@ class UnifiedDownloader:
|
||||
hf_command,
|
||||
check=False,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
capture_output=False # Allow progress bar to show
|
||||
)
|
||||
|
||||
# If new command succeeded, return
|
||||
if result.returncode == 0 and os.path.exists(target_path):
|
||||
print(f"✅ Downloaded: {target_path}")
|
||||
return True
|
||||
|
||||
# Try legacy command if new one failed
|
||||
legacy_command = [
|
||||
self._resolve_cli_executable("huggingface-cli"), "download",
|
||||
"huggingface-cli", "download",
|
||||
repo_id,
|
||||
filename,
|
||||
"--local-dir", target_dir
|
||||
@@ -141,14 +112,10 @@ class UnifiedDownloader:
|
||||
legacy_command,
|
||||
check=False,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
capture_output=False # Allow progress bar to show
|
||||
)
|
||||
|
||||
if result.returncode == 0 and os.path.exists(target_path):
|
||||
print(f"✅ Downloaded: {target_path}")
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
@@ -150,7 +150,14 @@ ENGINE_REGISTRY: Dict[str, EngineCapabilities] = {
|
||||
default_runtime_profile="vibevoice_transformers4_shared",
|
||||
),
|
||||
|
||||
"tada": EngineCapabilities(
|
||||
"dots_tts": EngineCapabilities(
|
||||
supports_voice_conversion=False,
|
||||
multilingual_model_switching=False,
|
||||
can_corrupt_on_reload=False,
|
||||
fallback_languages=["English"],
|
||||
),
|
||||
|
||||
"audio8_tts": EngineCapabilities(
|
||||
supports_voice_conversion=False,
|
||||
multilingual_model_switching=False,
|
||||
can_corrupt_on_reload=False,
|
||||
@@ -159,13 +166,6 @@ ENGINE_REGISTRY: Dict[str, EngineCapabilities] = {
|
||||
default_runtime_profile="vibevoice_transformers4_shared",
|
||||
),
|
||||
|
||||
"dots_tts": EngineCapabilities(
|
||||
supports_voice_conversion=False,
|
||||
multilingual_model_switching=False,
|
||||
can_corrupt_on_reload=False,
|
||||
fallback_languages=["English"],
|
||||
),
|
||||
|
||||
"dramabox": EngineCapabilities(
|
||||
supports_voice_conversion=False,
|
||||
multilingual_model_switching=False,
|
||||
|
||||
@@ -11,15 +11,19 @@ from typing import Any, Dict, Optional, Callable, Union
|
||||
from pathlib import Path
|
||||
|
||||
from utils.models.comfyui_model_wrapper import tts_model_manager, ModelInfo
|
||||
from utils.models.factory_config import ModelLoadConfig, runtime_uses_isolation
|
||||
from utils.models.factory_config import (
|
||||
ModelLoadConfig,
|
||||
RUNTIME_MODE_SHARED,
|
||||
runtime_uses_isolation,
|
||||
)
|
||||
from utils.models.engine_registry import get_default_runtime_profile
|
||||
from utils.runtimes import (
|
||||
build_audio8_tts_isolated_proxy,
|
||||
build_fish_audio_s2_proxy,
|
||||
build_higgs_audio_isolated_proxy,
|
||||
build_qwen3_asr_isolated_proxy,
|
||||
build_qwen3_tts_isolated_proxy,
|
||||
build_step_audio_editx_isolated_proxy,
|
||||
build_tada_isolated_proxy,
|
||||
build_vibevoice_isolated_proxy,
|
||||
)
|
||||
from utils.models.comfyui_model_wrapper.cache_utils import invalidate_all_caches
|
||||
@@ -125,6 +129,10 @@ class UnifiedModelInterface:
|
||||
# Check PyTorch consistency on first model load
|
||||
self._check_pytorch_consistency()
|
||||
|
||||
if config.engine_name == "audio8_tts":
|
||||
config.runtime_mode = RUNTIME_MODE_SHARED
|
||||
config.runtime_profile = "vibevoice_transformers4_shared"
|
||||
|
||||
if runtime_uses_isolation(config.runtime_mode):
|
||||
return self._load_isolated_model(config, force_reload=force_reload)
|
||||
|
||||
@@ -149,7 +157,7 @@ class UnifiedModelInterface:
|
||||
# check if a DIFFERENT variant is already loaded and unload it to prevent device conflicts
|
||||
# Only applies to engines where model variants are mutually exclusive
|
||||
if config.engine_name in (
|
||||
"qwen3_tts", "moss_tts", "higgs_audio_v3", "dramabox", "tada"
|
||||
"qwen3_tts", "moss_tts", "higgs_audio_v3", "dramabox"
|
||||
):
|
||||
# Check for any cached mutually-exclusive model variant for this engine
|
||||
cached_prefix = f"{config.engine_name}_tts_"
|
||||
@@ -220,6 +228,13 @@ class UnifiedModelInterface:
|
||||
|
||||
cached = self._isolated_model_cache.get(cache_key)
|
||||
if cached is not None:
|
||||
ensure_registration = getattr(
|
||||
cached,
|
||||
"_ensure_comfy_model_registration",
|
||||
None,
|
||||
)
|
||||
if callable(ensure_registration):
|
||||
ensure_registration()
|
||||
return cached
|
||||
|
||||
profile_name = config.runtime_profile or get_default_runtime_profile(config.engine_name or "")
|
||||
@@ -240,13 +255,13 @@ class UnifiedModelInterface:
|
||||
self._isolated_model_cache[cache_key] = proxy
|
||||
return proxy
|
||||
|
||||
if config.engine_name == "tada" and config.model_type == "tts":
|
||||
proxy = build_tada_isolated_proxy(config)
|
||||
if config.engine_name == "fish_audio_s2" and config.model_type == "tts":
|
||||
proxy = build_fish_audio_s2_proxy(config)
|
||||
self._isolated_model_cache[cache_key] = proxy
|
||||
return proxy
|
||||
|
||||
if config.engine_name == "fish_audio_s2" and config.model_type == "tts":
|
||||
proxy = build_fish_audio_s2_proxy(config)
|
||||
if config.engine_name == "audio8_tts" and config.model_type == "tts":
|
||||
proxy = build_audio8_tts_isolated_proxy(config)
|
||||
self._isolated_model_cache[cache_key] = proxy
|
||||
return proxy
|
||||
|
||||
@@ -1729,27 +1744,6 @@ def register_fish_audio_s2_factory():
|
||||
unified_model_interface.register_model_factory("fish_audio_s2", "tts", fish_audio_s2_factory)
|
||||
|
||||
|
||||
def register_tada_factory():
|
||||
"""Register TADA for explicit Main Environment routing."""
|
||||
def tada_factory(config: ModelLoadConfig):
|
||||
from engines.tada.tada_engine import TadaEngine
|
||||
|
||||
params = config.additional_params or {}
|
||||
return TadaEngine(
|
||||
model_name=config.model_name or "TADA-1B",
|
||||
model_path=config.model_path,
|
||||
codec_path=params.get("codec_path"),
|
||||
tokenizer_path=params.get("tokenizer_path"),
|
||||
device=config.device or "auto",
|
||||
dtype=params.get("dtype", "auto"),
|
||||
attn_implementation=params.get("attn_implementation", "sdpa"),
|
||||
use_torch_compile=bool(params.get("use_torch_compile", False)),
|
||||
prompt_cache_size=int(params.get("prompt_cache_size", 8)),
|
||||
)
|
||||
|
||||
unified_model_interface.register_model_factory("tada", "tts", tada_factory)
|
||||
|
||||
|
||||
def register_dots_tts_factory():
|
||||
"""Register Dots TTS model factory."""
|
||||
def dots_tts_factory(config: ModelLoadConfig):
|
||||
@@ -1778,6 +1772,19 @@ def register_dots_tts_factory():
|
||||
unified_model_interface.register_model_factory("dots_tts", "tts", dots_tts_factory)
|
||||
|
||||
|
||||
def register_audio8_tts_factory():
|
||||
"""Register Audio8 TTS through its required Transformers 4 runtime."""
|
||||
def audio8_tts_factory(config: ModelLoadConfig):
|
||||
del config
|
||||
raise RuntimeError(
|
||||
"Audio8 TTS reached its embedded factory unexpectedly; all "
|
||||
"Audio8 loads must be routed through the shared Transformers 4 "
|
||||
"runtime"
|
||||
)
|
||||
|
||||
unified_model_interface.register_model_factory("audio8_tts", "tts", audio8_tts_factory)
|
||||
|
||||
|
||||
def register_dramabox_factory():
|
||||
"""Register official DramaBox in the main Transformers 5 environment."""
|
||||
def dramabox_factory(config: ModelLoadConfig):
|
||||
@@ -2044,8 +2051,8 @@ def initialize_all_factories():
|
||||
register_moss_soundeffect_v2_factory()
|
||||
register_qwen3_tts_factory()
|
||||
register_fish_audio_s2_factory()
|
||||
register_tada_factory()
|
||||
register_dots_tts_factory()
|
||||
register_audio8_tts_factory()
|
||||
register_dramabox_factory()
|
||||
register_omnivoice_factory()
|
||||
register_qwen3_asr_factory()
|
||||
|
||||
@@ -6,6 +6,7 @@ separate Python environments without changing the existing ComfyUI-side UI.
|
||||
"""
|
||||
|
||||
from .bootstrap import ensure_runtime, resolve_runtime_dir, resolve_runtime_python
|
||||
from .audio8_tts_proxy import Audio8TTSIsolatedProxy, build_audio8_tts_isolated_proxy
|
||||
from .higgs_audio_proxy import HiggsAudioIsolatedProxy, build_higgs_audio_isolated_proxy
|
||||
from .fish_audio_s2_proxy import FishAudioS2Proxy, build_fish_audio_s2_proxy
|
||||
from .launcher import IsolatedRuntimeLauncher
|
||||
@@ -15,12 +16,13 @@ from .qwen3_asr_proxy import Qwen3ASRIsolatedProxy, build_qwen3_asr_isolated_pro
|
||||
from .qwen3_tts_proxy import Qwen3TTSIsolatedProxy, build_qwen3_tts_isolated_proxy
|
||||
from .vibevoice_proxy import VibeVoiceIsolatedProxy, build_vibevoice_isolated_proxy
|
||||
from .step_audio_editx_proxy import StepAudioEditXIsolatedProxy, build_step_audio_editx_isolated_proxy
|
||||
from .tada_proxy import TadaIsolatedProxy, build_tada_isolated_proxy
|
||||
|
||||
__all__ = [
|
||||
"ensure_runtime",
|
||||
"resolve_runtime_dir",
|
||||
"resolve_runtime_python",
|
||||
"Audio8TTSIsolatedProxy",
|
||||
"build_audio8_tts_isolated_proxy",
|
||||
"HiggsAudioIsolatedProxy",
|
||||
"IsolatedRuntimeLauncher",
|
||||
"RuntimeProfile",
|
||||
@@ -37,8 +39,6 @@ __all__ = [
|
||||
"build_vibevoice_isolated_proxy",
|
||||
"StepAudioEditXIsolatedProxy",
|
||||
"build_step_audio_editx_isolated_proxy",
|
||||
"TadaIsolatedProxy",
|
||||
"build_tada_isolated_proxy",
|
||||
"get_runtime_profile",
|
||||
"list_runtime_profiles",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,546 @@
|
||||
"""Audio8 TTS proxy for the shared official Transformers 4 runtime."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import uuid
|
||||
import weakref
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from utils.models.factory_config import ModelLoadConfig
|
||||
from .bootstrap import PROJECT_ROOT, ensure_runtime
|
||||
from .launcher import IsolatedRuntimeLauncher
|
||||
from .profiles import RuntimeProfile, get_runtime_profile
|
||||
from .protocol import RuntimeJobRequest
|
||||
from .session import JsonLineWorkerSession
|
||||
|
||||
|
||||
_INTERRUPT_EVENTS: set[threading.Event] = set()
|
||||
_INTERRUPT_EVENTS_LOCK = threading.Lock()
|
||||
_INTERRUPT_DISPATCH_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def _install_comfy_interrupt_dispatcher() -> None:
|
||||
"""Forward Comfy's server interrupt edge to active Audio8 requests."""
|
||||
server_module = sys.modules.get("server")
|
||||
comfy_nodes = getattr(server_module, "nodes", None)
|
||||
if comfy_nodes is None:
|
||||
comfy_nodes = sys.modules.get("nodes")
|
||||
if comfy_nodes is None:
|
||||
return
|
||||
|
||||
with _INTERRUPT_DISPATCH_LOCK:
|
||||
current_interrupt = getattr(comfy_nodes, "interrupt_processing", None)
|
||||
if not callable(current_interrupt):
|
||||
return
|
||||
if getattr(current_interrupt, "_tts_audio8_dispatcher", False):
|
||||
if (
|
||||
getattr(current_interrupt, "_tts_audio8_events", None)
|
||||
is _INTERRUPT_EVENTS
|
||||
):
|
||||
return
|
||||
current_interrupt = getattr(
|
||||
current_interrupt,
|
||||
"_tts_audio8_original_interrupt",
|
||||
current_interrupt,
|
||||
)
|
||||
|
||||
def dispatch_interrupt(value=True):
|
||||
# TTS Audio Suite patch: preserve Comfy's interrupt edge for
|
||||
# blocking Audio8 subprocess requests without replacing the
|
||||
# process-wide callback separately for every generation.
|
||||
result = current_interrupt(value)
|
||||
if value:
|
||||
with _INTERRUPT_EVENTS_LOCK:
|
||||
active_events = tuple(_INTERRUPT_EVENTS)
|
||||
for active_event in active_events:
|
||||
active_event.set()
|
||||
return result
|
||||
|
||||
dispatch_interrupt._tts_audio8_dispatcher = True
|
||||
dispatch_interrupt._tts_audio8_original_interrupt = current_interrupt
|
||||
dispatch_interrupt._tts_audio8_events = _INTERRUPT_EVENTS
|
||||
comfy_nodes.interrupt_processing = dispatch_interrupt
|
||||
|
||||
|
||||
def _register_interrupt_event(interrupt_event: threading.Event) -> None:
|
||||
_install_comfy_interrupt_dispatcher()
|
||||
with _INTERRUPT_EVENTS_LOCK:
|
||||
_INTERRUPT_EVENTS.add(interrupt_event)
|
||||
|
||||
|
||||
def _unregister_interrupt_event(interrupt_event: threading.Event) -> None:
|
||||
with _INTERRUPT_EVENTS_LOCK:
|
||||
_INTERRUPT_EVENTS.discard(interrupt_event)
|
||||
|
||||
|
||||
class Audio8TTSIsolatedProxy:
|
||||
"""Expose Audio8's engine API while inference runs under Transformers 4."""
|
||||
|
||||
SAMPLE_RATE = 44100
|
||||
|
||||
def __init__(self, config: ModelLoadConfig, profile: RuntimeProfile):
|
||||
self.config = config
|
||||
self.profile = profile
|
||||
self.model_name = config.model_name or "Audio8-TTS-Preview-0.6b"
|
||||
self.model_path = config.model_path
|
||||
self.device = str(config.device)
|
||||
self.current_device = self.device
|
||||
self.dtype_name = str((config.additional_params or {}).get("dtype", "auto"))
|
||||
self.dtype = self._resolve_dtype(self.dtype_name)
|
||||
self.load_device = self.current_loaded_device()
|
||||
self.offload_device = torch.device("cpu")
|
||||
self.currently_used = True
|
||||
self.model = self
|
||||
self.engine = self
|
||||
self.parent = None
|
||||
self.model_options = {}
|
||||
self.model_keys = set()
|
||||
estimate_gib = (
|
||||
3.5
|
||||
if self.device.startswith("cpu") or self.dtype == torch.float32
|
||||
else 1.75
|
||||
)
|
||||
self._estimated_memory_size = int(estimate_gib * 1024**3)
|
||||
self._comfy_loaded_model = None
|
||||
self._comfy_model_management = None
|
||||
self._initialized = False
|
||||
|
||||
launcher = IsolatedRuntimeLauncher(runtime_root=str(PROJECT_ROOT))
|
||||
python_path = ensure_runtime(profile)
|
||||
worker_script = PROJECT_ROOT / "utils/runtimes/workers/audio8_tts_worker.py"
|
||||
self._session = JsonLineWorkerSession(
|
||||
python_path=str(python_path),
|
||||
worker_script=str(worker_script),
|
||||
env=launcher.build_env(profile),
|
||||
)
|
||||
try:
|
||||
self._initialize_remote_engine()
|
||||
except Exception:
|
||||
self._session.close()
|
||||
raise
|
||||
self._register_with_comfy_model_management()
|
||||
|
||||
def _resolve_dtype(self, dtype_name: str) -> torch.dtype:
|
||||
normalized = str(dtype_name or "auto").lower()
|
||||
if not self.device.startswith("cuda"):
|
||||
return torch.float32
|
||||
if normalized == "float32":
|
||||
return torch.float32
|
||||
if normalized == "float16":
|
||||
return torch.float16
|
||||
if normalized == "bfloat16":
|
||||
return torch.bfloat16
|
||||
if torch.cuda.is_available():
|
||||
major, _minor = torch.cuda.get_device_capability(torch.device(self.device))
|
||||
return torch.bfloat16 if major >= 8 else torch.float16
|
||||
return torch.float16
|
||||
|
||||
def _request(self, action: str, payload: Dict[str, Any]):
|
||||
response = self._session.request(
|
||||
RuntimeJobRequest(
|
||||
engine_name="audio8_tts",
|
||||
action=action,
|
||||
model_name=self.model_name,
|
||||
device=self.device,
|
||||
runtime_profile=self.profile.name,
|
||||
payload=payload,
|
||||
request_id=str(uuid.uuid4()),
|
||||
)
|
||||
)
|
||||
if not response.ok:
|
||||
details = response.error or (
|
||||
f"Audio8 TTS isolated action '{action}' failed"
|
||||
)
|
||||
if response.logs:
|
||||
details = f"{details}\n" + "\n".join(response.logs)
|
||||
raise RuntimeError(details)
|
||||
return response
|
||||
|
||||
def _request_generation(self, payload: Dict[str, Any]):
|
||||
"""Run generation while the main process monitors Comfy interruption."""
|
||||
try:
|
||||
import comfy.model_management as model_management
|
||||
except Exception:
|
||||
return self._request("generate", payload)
|
||||
|
||||
checker = getattr(
|
||||
model_management,
|
||||
"throw_exception_if_processing_interrupted",
|
||||
None,
|
||||
)
|
||||
if not callable(checker):
|
||||
return self._request("generate", payload)
|
||||
|
||||
interrupt_event = threading.Event()
|
||||
_register_interrupt_event(interrupt_event)
|
||||
result: Dict[str, Any] = {}
|
||||
request_thread = None
|
||||
request_started = False
|
||||
|
||||
def run_request() -> None:
|
||||
try:
|
||||
result["response"] = self._request("generate", payload)
|
||||
except BaseException as exc:
|
||||
result["error"] = exc
|
||||
|
||||
try:
|
||||
checker()
|
||||
request_thread = threading.Thread(
|
||||
target=run_request,
|
||||
name="audio8-tts-worker-request",
|
||||
daemon=True,
|
||||
)
|
||||
request_thread.start()
|
||||
request_started = True
|
||||
|
||||
while request_thread.is_alive():
|
||||
request_thread.join(timeout=0.05)
|
||||
if interrupt_event.is_set():
|
||||
raise model_management.InterruptProcessingException()
|
||||
checker()
|
||||
except BaseException as exc:
|
||||
if request_started:
|
||||
self._initialized = False
|
||||
try:
|
||||
self._session.terminate_now()
|
||||
except Exception:
|
||||
pass
|
||||
request_thread.join(timeout=5)
|
||||
if isinstance(
|
||||
exc,
|
||||
model_management.InterruptProcessingException,
|
||||
):
|
||||
interrupt_setter = getattr(
|
||||
model_management,
|
||||
"interrupt_current_processing",
|
||||
None,
|
||||
)
|
||||
if callable(interrupt_setter):
|
||||
interrupt_setter(False)
|
||||
raise
|
||||
finally:
|
||||
_unregister_interrupt_event(interrupt_event)
|
||||
|
||||
if "error" in result:
|
||||
raise result["error"]
|
||||
return result["response"]
|
||||
|
||||
def _initialize_remote_engine(self) -> None:
|
||||
self._request(
|
||||
"initialize",
|
||||
{
|
||||
"model_path": self.model_path,
|
||||
"dtype": self.dtype_name,
|
||||
},
|
||||
)
|
||||
self._initialized = True
|
||||
print(f"✅ Audio8 TTS isolated runtime ready ({self.profile.name})")
|
||||
|
||||
def _ensure_remote_engine(self) -> None:
|
||||
process = getattr(self._session, "_process", None)
|
||||
if not self._initialized or process is None or process.poll() is not None:
|
||||
self._initialize_remote_engine()
|
||||
|
||||
@staticmethod
|
||||
def _serialize_reference_audio(
|
||||
reference_audio: Any,
|
||||
bundle_dir: Path,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
if reference_audio is None:
|
||||
return None
|
||||
if isinstance(reference_audio, (str, os.PathLike)):
|
||||
return {
|
||||
"kind": "audio_path",
|
||||
"audio_path": os.fspath(reference_audio),
|
||||
}
|
||||
|
||||
waveform = None
|
||||
sample_rate = None
|
||||
if isinstance(reference_audio, dict):
|
||||
waveform = reference_audio.get(
|
||||
"waveform",
|
||||
reference_audio.get("array"),
|
||||
)
|
||||
sample_rate = reference_audio.get(
|
||||
"sample_rate",
|
||||
reference_audio.get("sampling_rate"),
|
||||
)
|
||||
elif isinstance(reference_audio, (tuple, list)) and len(reference_audio) == 2:
|
||||
waveform, sample_rate = reference_audio
|
||||
|
||||
if waveform is None or sample_rate is None:
|
||||
raise TypeError(
|
||||
"Audio8 TTS isolated reference audio must be a path, "
|
||||
"audio dict, or (waveform, sample_rate) pair"
|
||||
)
|
||||
|
||||
tensor_path = bundle_dir / "reference_audio.pt"
|
||||
waveform_tensor = torch.as_tensor(waveform).detach().float().cpu()
|
||||
torch.save(
|
||||
{
|
||||
"waveform": waveform_tensor,
|
||||
"sample_rate": int(sample_rate),
|
||||
},
|
||||
tensor_path,
|
||||
)
|
||||
return {
|
||||
"kind": "tensor_path",
|
||||
"tensor_path": str(tensor_path),
|
||||
}
|
||||
|
||||
def generate(
|
||||
self,
|
||||
text: str,
|
||||
reference_audio: Any = None,
|
||||
reference_text: Optional[str] = None,
|
||||
*,
|
||||
max_new_tokens: int = 1024,
|
||||
retry_max_new_tokens: int = 2000,
|
||||
temperature: float = 0.8,
|
||||
top_p: float = 0.95,
|
||||
top_k: int = 50,
|
||||
do_sample: bool = True,
|
||||
seed: int = 42,
|
||||
) -> torch.Tensor:
|
||||
self._ensure_remote_engine()
|
||||
with tempfile.TemporaryDirectory(prefix="tts_audio8_iso_") as temp_dir:
|
||||
bundle_dir = Path(temp_dir)
|
||||
output_path = bundle_dir / "result.pt"
|
||||
serialized_reference = self._serialize_reference_audio(
|
||||
reference_audio,
|
||||
bundle_dir,
|
||||
)
|
||||
self._request_generation(
|
||||
{
|
||||
"text": text,
|
||||
"reference_audio": serialized_reference,
|
||||
"reference_text": reference_text,
|
||||
"max_new_tokens": int(max_new_tokens),
|
||||
"retry_max_new_tokens": int(retry_max_new_tokens),
|
||||
"temperature": float(temperature),
|
||||
"top_p": float(top_p),
|
||||
"top_k": int(top_k),
|
||||
"do_sample": bool(do_sample),
|
||||
"seed": int(seed),
|
||||
"output_path": str(output_path),
|
||||
}
|
||||
)
|
||||
if not output_path.exists():
|
||||
raise RuntimeError(
|
||||
"Audio8 TTS isolated worker returned no output payload"
|
||||
)
|
||||
result = torch.load(output_path, map_location="cpu")
|
||||
audio = torch.as_tensor(result["audio"]).detach().float().cpu()
|
||||
sample_rate = int(result["sample_rate"])
|
||||
if sample_rate != self.SAMPLE_RATE:
|
||||
raise RuntimeError(
|
||||
f"Audio8 TTS worker returned {sample_rate} Hz; "
|
||||
f"expected {self.SAMPLE_RATE} Hz"
|
||||
)
|
||||
if audio.ndim == 1:
|
||||
audio = audio.unsqueeze(0)
|
||||
if audio.ndim != 2 or audio.shape[0] != 1:
|
||||
raise RuntimeError(
|
||||
"Audio8 TTS worker returned invalid audio shape "
|
||||
f"{tuple(audio.shape)}"
|
||||
)
|
||||
return audio.contiguous()
|
||||
|
||||
def _register_with_comfy_model_management(self) -> None:
|
||||
self._ensure_comfy_model_registration()
|
||||
|
||||
def _ensure_comfy_model_registration(self) -> None:
|
||||
try:
|
||||
import comfy.model_management as model_management
|
||||
except Exception:
|
||||
return
|
||||
if not hasattr(model_management, "LoadedModel") or not hasattr(
|
||||
model_management,
|
||||
"current_loaded_models",
|
||||
):
|
||||
return
|
||||
for loaded_model in model_management.current_loaded_models:
|
||||
wrapper = getattr(loaded_model, "_tts_wrapper_ref", None)
|
||||
real_model = getattr(loaded_model, "real_model", None)
|
||||
resolved_real_model = (
|
||||
real_model()
|
||||
if isinstance(real_model, weakref.ReferenceType)
|
||||
else real_model
|
||||
)
|
||||
if wrapper is self or resolved_real_model is self:
|
||||
self._comfy_loaded_model = loaded_model
|
||||
self._comfy_model_management = model_management
|
||||
return
|
||||
try:
|
||||
loaded_model = model_management.LoadedModel(self)
|
||||
loaded_model.real_model = weakref.ref(self)
|
||||
finalizer = getattr(
|
||||
model_management,
|
||||
"cleanup_models",
|
||||
lambda: None,
|
||||
)
|
||||
loaded_model.model_finalizer = weakref.finalize(self, finalizer)
|
||||
loaded_model._tts_wrapper_ref = self
|
||||
model_management.current_loaded_models.insert(0, loaded_model)
|
||||
self._comfy_loaded_model = loaded_model
|
||||
self._comfy_model_management = model_management
|
||||
except Exception as exc:
|
||||
print(
|
||||
"⚠️ Failed to register isolated Audio8 TTS runtime with "
|
||||
f"ComfyUI model management: {exc}"
|
||||
)
|
||||
|
||||
def _unregister_from_comfy_model_management(self) -> None:
|
||||
model_management = self._comfy_model_management
|
||||
loaded_model = self._comfy_loaded_model
|
||||
if model_management is None or loaded_model is None:
|
||||
return
|
||||
try:
|
||||
if loaded_model in model_management.current_loaded_models:
|
||||
model_management.current_loaded_models.remove(loaded_model)
|
||||
except Exception as exc:
|
||||
print(
|
||||
"⚠️ Failed to remove isolated Audio8 TTS runtime from "
|
||||
f"ComfyUI tracking: {exc}"
|
||||
)
|
||||
finally:
|
||||
self._comfy_loaded_model = None
|
||||
self._comfy_model_management = None
|
||||
|
||||
def to(self, device):
|
||||
self.device = str(device)
|
||||
self.current_device = self.device
|
||||
self.load_device = self.current_loaded_device()
|
||||
return self
|
||||
|
||||
def eval(self):
|
||||
return self
|
||||
|
||||
def loaded_size(self) -> int:
|
||||
if self._initialized and self.device.startswith("cuda"):
|
||||
return self._estimated_memory_size
|
||||
return 0
|
||||
|
||||
def model_size(self) -> int:
|
||||
return self._estimated_memory_size
|
||||
|
||||
def model_memory(self) -> int:
|
||||
return self._estimated_memory_size
|
||||
|
||||
def get_ram_usage(self) -> int:
|
||||
return self._estimated_memory_size
|
||||
|
||||
def model_offloaded_memory(self) -> int:
|
||||
return (
|
||||
0
|
||||
if self._initialized and self.device.startswith("cuda")
|
||||
else self._estimated_memory_size
|
||||
)
|
||||
|
||||
def model_mmap_residency(self, free: bool = False) -> tuple[int, int]:
|
||||
del free
|
||||
return 0, self._estimated_memory_size
|
||||
|
||||
def pinned_memory_size(self) -> int:
|
||||
return 0
|
||||
|
||||
def lowvram_patch_counter(self) -> int:
|
||||
return 0
|
||||
|
||||
def model_dtype(self):
|
||||
return self.dtype
|
||||
|
||||
def model_patches_models(self):
|
||||
return ()
|
||||
|
||||
def model_patches_to(self, target) -> None:
|
||||
if isinstance(target, torch.dtype):
|
||||
self.dtype = target
|
||||
elif isinstance(target, torch.device):
|
||||
self.device = str(target)
|
||||
|
||||
def is_dynamic(self) -> bool:
|
||||
return False
|
||||
|
||||
def current_loaded_device(self) -> torch.device:
|
||||
if self.device.startswith("cuda") and torch.cuda.is_available():
|
||||
return torch.device(self.device)
|
||||
return torch.device("cpu")
|
||||
|
||||
def partially_load(
|
||||
self,
|
||||
device,
|
||||
extra_memory,
|
||||
force_patch_weights=False,
|
||||
) -> int:
|
||||
del extra_memory, force_patch_weights
|
||||
self.to(device)
|
||||
return 0
|
||||
|
||||
def partially_unload_ram(self, ram_to_unload: int) -> int:
|
||||
return self.partially_unload("cpu", ram_to_unload)
|
||||
|
||||
def partially_unload(self, device, memory_to_free) -> int:
|
||||
del device, memory_to_free
|
||||
freed = self.loaded_size()
|
||||
self.cleanup(unregister=False)
|
||||
return freed
|
||||
|
||||
def model_unload(
|
||||
self,
|
||||
memory_to_free=None,
|
||||
unpatch_weights=True,
|
||||
) -> bool:
|
||||
del memory_to_free, unpatch_weights
|
||||
self.cleanup(unregister=False)
|
||||
return True
|
||||
|
||||
def detach(self, unpatch_weights=True) -> None:
|
||||
del unpatch_weights
|
||||
self.cleanup(unregister=False)
|
||||
|
||||
def is_clone(self, other: Any) -> bool:
|
||||
return other is self
|
||||
|
||||
def cleanup(self, unregister: bool = True) -> None:
|
||||
if unregister:
|
||||
self._unregister_from_comfy_model_management()
|
||||
else:
|
||||
self._comfy_loaded_model = None
|
||||
self._comfy_model_management = None
|
||||
self._initialized = False
|
||||
if getattr(self, "_session", None) is not None:
|
||||
self._session.close()
|
||||
|
||||
close = cleanup
|
||||
|
||||
def __del__(self):
|
||||
try:
|
||||
self.cleanup()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def build_audio8_tts_isolated_proxy(
|
||||
config: ModelLoadConfig,
|
||||
) -> Audio8TTSIsolatedProxy:
|
||||
profile_name = config.runtime_profile or "vibevoice_transformers4_shared"
|
||||
profile = get_runtime_profile(profile_name)
|
||||
if profile is None:
|
||||
raise RuntimeError(
|
||||
f"Unknown isolated runtime profile '{profile_name}' for Audio8 TTS"
|
||||
)
|
||||
if profile.name != "vibevoice_transformers4_shared":
|
||||
raise RuntimeError(
|
||||
"Audio8 TTS requires the shared Transformers 4 runtime profile "
|
||||
"'vibevoice_transformers4_shared'"
|
||||
)
|
||||
config.runtime_profile = profile.name
|
||||
return Audio8TTSIsolatedProxy(config=config, profile=profile)
|
||||
@@ -28,8 +28,7 @@ _VIBEVOICE_T4_PACKAGES = [
|
||||
"numpy>=1.26.4,<2.3.0",
|
||||
"soundfile>=0.12.0",
|
||||
"omegaconf>=2.3.0",
|
||||
"transformers>=4.51.3,<=4.57.3",
|
||||
"huggingface-hub>=0.34,<1.0",
|
||||
"transformers>=4.57.0,<=4.57.3",
|
||||
"kernels>=0.6.1,<=0.9",
|
||||
"accelerate",
|
||||
"requests",
|
||||
@@ -64,14 +63,16 @@ _QWEN3_T4_PACKAGES = [
|
||||
RUNTIME_PROFILES: Dict[str, RuntimeProfile] = {
|
||||
"vibevoice_transformers4_shared": RuntimeProfile(
|
||||
name="vibevoice_transformers4_shared",
|
||||
engine_names=["vibevoice", "step_audio_editx", "tada"],
|
||||
engine_names=["vibevoice", "step_audio_editx", "audio8_tts"],
|
||||
python_path_hint="runtimes/shared_legacy_t4/Scripts/python.exe",
|
||||
description="Shared legacy Transformers 4 runtime for VibeVoice/Kugel, Step Audio EditX, TADA, and similar engines.",
|
||||
description=(
|
||||
"Shared legacy Transformers 4 runtime for VibeVoice/Kugel, "
|
||||
"Step Audio EditX, Audio8 TTS, and similar engines."
|
||||
),
|
||||
inherit_base_site_packages=True,
|
||||
pip_packages=list(_VIBEVOICE_T4_PACKAGES),
|
||||
pip_packages_no_deps=[
|
||||
"git+https://github.com/FushionHub/VibeVoice.git",
|
||||
"hume-tada==0.1.9",
|
||||
],
|
||||
),
|
||||
"vibevoice_transformers4_dedicated": RuntimeProfile(
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
"""
|
||||
Persistent JSON-line worker session for isolated runtimes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import threading
|
||||
@@ -118,6 +118,24 @@ class JsonLineWorkerSession:
|
||||
|
||||
self._process = None
|
||||
|
||||
def terminate_now(self) -> None:
|
||||
"""Stop a worker without waiting for the request lock."""
|
||||
process = self._process
|
||||
if process is None or process.poll() is not None:
|
||||
return
|
||||
try:
|
||||
process.terminate()
|
||||
except OSError:
|
||||
return
|
||||
try:
|
||||
process.wait(timeout=2)
|
||||
except subprocess.TimeoutExpired:
|
||||
try:
|
||||
process.kill()
|
||||
process.wait(timeout=2)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def __del__(self):
|
||||
try:
|
||||
self.close()
|
||||
|
||||
@@ -1,380 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
"""ComfyUI-side proxy for TADA in the shared Transformers 4 runtime."""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import uuid
|
||||
import weakref
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from engines.tada.languages import validate_tada_model_language
|
||||
from engines.tada.tada_downloader import TadaDownloader
|
||||
from utils.device import resolve_torch_device
|
||||
from .bootstrap import PROJECT_ROOT, ensure_runtime
|
||||
from .launcher import IsolatedRuntimeLauncher
|
||||
from .profiles import RuntimeProfile, get_runtime_profile
|
||||
from .protocol import RuntimeJobRequest
|
||||
from .session import JsonLineWorkerSession
|
||||
|
||||
|
||||
TADA_RUNTIME_PROFILE = "vibevoice_transformers4_shared"
|
||||
|
||||
|
||||
class TadaIsolatedProxy:
|
||||
"""Persistent TADA worker with explicit cleanup for ComfyUI's Clear VRAM action."""
|
||||
|
||||
def __init__(self, config, profile: RuntimeProfile):
|
||||
if profile.name != TADA_RUNTIME_PROFILE:
|
||||
raise RuntimeError(
|
||||
f"TADA only supports the '{TADA_RUNTIME_PROFILE}' shared runtime; got '{profile.name}'."
|
||||
)
|
||||
|
||||
self.config = config
|
||||
self.profile = profile
|
||||
self.current_model_name = str(config.model_name or "TADA-1B")
|
||||
self.device = resolve_torch_device(str(config.device or "auto"))
|
||||
self.current_device = self.device
|
||||
self.load_device = self.current_loaded_device()
|
||||
self.offload_device = torch.device("cpu")
|
||||
self.dtype = torch.bfloat16
|
||||
self.model = self
|
||||
self.parent = None
|
||||
self.currently_used = True
|
||||
self.model_options = {}
|
||||
self.model_keys = set()
|
||||
self.sample_rate = 24000
|
||||
|
||||
params = dict(config.additional_params or {})
|
||||
self.model_path = str(params.get("model_path") or config.model_path or "")
|
||||
self.codec_path = str(params.get("codec_path") or "")
|
||||
self.tokenizer_path = str(params.get("tokenizer_path") or "")
|
||||
self.load_dtype = str(params.get("dtype", "auto"))
|
||||
self.attn_implementation = str(params.get("attn_implementation", "sdpa"))
|
||||
self.use_torch_compile = bool(params.get("use_torch_compile", False))
|
||||
self.prompt_cache_size = int(params.get("prompt_cache_size", 8))
|
||||
|
||||
if not self.model_path or not self.codec_path or not self.tokenizer_path:
|
||||
assets = TadaDownloader().resolve_model_assets(self.current_model_name, language=None)
|
||||
self.current_model_name = assets["model_name"]
|
||||
self.model_path = assets["model_path"]
|
||||
self.codec_path = assets["codec_path"]
|
||||
self.tokenizer_path = assets["tokenizer_path"]
|
||||
|
||||
estimate_gb = 10 if "3b" in self.current_model_name.lower() else 6
|
||||
self._estimated_memory_size = estimate_gb * 1024**3
|
||||
self._initialized = False
|
||||
self._comfy_loaded_model = None
|
||||
self._comfy_model_management = None
|
||||
|
||||
launcher = IsolatedRuntimeLauncher(runtime_root=str(PROJECT_ROOT))
|
||||
worker_env = launcher.build_env(profile)
|
||||
worker_env["HF_HUB_OFFLINE"] = "1"
|
||||
worker_env["TRANSFORMERS_OFFLINE"] = "1"
|
||||
worker_env.setdefault("PYTHONUTF8", "1")
|
||||
self.compile_cache_dir = self._configure_compile_cache(worker_env)
|
||||
python_path = ensure_runtime(profile)
|
||||
self._session = JsonLineWorkerSession(
|
||||
python_path=str(python_path),
|
||||
worker_script=str(PROJECT_ROOT / "utils" / "runtimes" / "workers" / "tada_worker.py"),
|
||||
env=worker_env,
|
||||
)
|
||||
self._initialize_remote_engine()
|
||||
self._register_with_comfy_model_management()
|
||||
|
||||
def _configure_compile_cache(self, worker_env: Dict[str, str]) -> Optional[Path]:
|
||||
if not self.use_torch_compile:
|
||||
return None
|
||||
try:
|
||||
model_dir = Path(self.model_path).resolve()
|
||||
cache_root = model_dir.parent.parent / "compile_cache"
|
||||
inductor_dir = cache_root / "torchinductor"
|
||||
triton_dir = cache_root / "triton"
|
||||
inductor_dir.mkdir(parents=True, exist_ok=True)
|
||||
triton_dir.mkdir(parents=True, exist_ok=True)
|
||||
worker_env["TORCHINDUCTOR_FX_GRAPH_CACHE"] = "1"
|
||||
worker_env["TORCHINDUCTOR_CACHE_DIR"] = str(inductor_dir)
|
||||
worker_env["TRITON_CACHE_DIR"] = str(triton_dir)
|
||||
print(f" TADA compile cache: {cache_root}")
|
||||
return cache_root
|
||||
except Exception as exc:
|
||||
print(f"⚠️ TADA: failed to configure persistent compile cache: {exc}")
|
||||
return None
|
||||
|
||||
def _request(self, action: str, payload: Optional[Dict[str, Any]] = None):
|
||||
response = self._session.request(
|
||||
RuntimeJobRequest(
|
||||
engine_name="tada",
|
||||
action=action,
|
||||
model_name=self.current_model_name,
|
||||
device=self.device,
|
||||
runtime_profile=self.profile.name,
|
||||
payload=payload or {},
|
||||
request_id=str(uuid.uuid4()),
|
||||
)
|
||||
)
|
||||
if not response.ok:
|
||||
details = response.error or f"TADA {action} failed"
|
||||
if response.logs:
|
||||
details = f"{details}\n" + "\n".join(response.logs)
|
||||
raise RuntimeError(details)
|
||||
return response
|
||||
|
||||
def _initialize_remote_engine(self) -> None:
|
||||
self._request(
|
||||
"initialize",
|
||||
{
|
||||
"model_path": self.model_path,
|
||||
"codec_path": self.codec_path,
|
||||
"tokenizer_path": self.tokenizer_path,
|
||||
"dtype": self.load_dtype,
|
||||
"attn_implementation": self.attn_implementation,
|
||||
"use_torch_compile": self.use_torch_compile,
|
||||
"prompt_cache_size": self.prompt_cache_size,
|
||||
},
|
||||
)
|
||||
self._initialized = True
|
||||
|
||||
def _ensure_remote_engine(self) -> None:
|
||||
process = getattr(self._session, "_process", None)
|
||||
if not self._initialized or process is None or process.poll() is not None:
|
||||
self._initialize_remote_engine()
|
||||
if self._comfy_loaded_model is None:
|
||||
self._register_with_comfy_model_management()
|
||||
|
||||
def _register_with_comfy_model_management(self) -> None:
|
||||
try:
|
||||
import comfy.model_management as model_management
|
||||
except Exception:
|
||||
return
|
||||
if not hasattr(model_management, "LoadedModel") or not hasattr(model_management, "current_loaded_models"):
|
||||
return
|
||||
try:
|
||||
loaded_model = model_management.LoadedModel(self)
|
||||
loaded_model.real_model = weakref.ref(self)
|
||||
finalizer = model_management.cleanup_models if hasattr(model_management, "cleanup_models") else lambda: None
|
||||
loaded_model.model_finalizer = weakref.finalize(self, finalizer)
|
||||
loaded_model._tts_wrapper_ref = self
|
||||
model_management.current_loaded_models.insert(0, loaded_model)
|
||||
self._comfy_loaded_model = loaded_model
|
||||
self._comfy_model_management = model_management
|
||||
except Exception as exc:
|
||||
print(f"⚠️ Failed to register TADA runtime with ComfyUI model management: {exc}")
|
||||
|
||||
def _unregister_from_comfy_model_management(self) -> None:
|
||||
model_management = self._comfy_model_management
|
||||
loaded_model = self._comfy_loaded_model
|
||||
if model_management is None or loaded_model is None:
|
||||
return
|
||||
try:
|
||||
if loaded_model in model_management.current_loaded_models:
|
||||
model_management.current_loaded_models.remove(loaded_model)
|
||||
except Exception as exc:
|
||||
print(f"⚠️ Failed to remove TADA runtime from ComfyUI tracking: {exc}")
|
||||
finally:
|
||||
self._comfy_loaded_model = None
|
||||
self._comfy_model_management = None
|
||||
|
||||
@staticmethod
|
||||
def _serialize_reference(reference_audio: object, bundle_dir: Path) -> Dict[str, Any]:
|
||||
if isinstance(reference_audio, (str, os.PathLike)):
|
||||
path = os.path.abspath(str(reference_audio))
|
||||
if not os.path.isfile(path):
|
||||
raise FileNotFoundError(f"TADA reference audio file not found: {path}")
|
||||
return {"kind": "audio_path", "audio_path": path}
|
||||
|
||||
audio = reference_audio
|
||||
sample_rate = 24000
|
||||
if isinstance(audio, (tuple, list)) and len(audio) == 2:
|
||||
audio, sample_rate = audio
|
||||
elif isinstance(audio, dict):
|
||||
if "audio" in audio and isinstance(audio["audio"], dict):
|
||||
audio = audio["audio"]
|
||||
if isinstance(audio, dict):
|
||||
path = audio.get("audio_path") or audio.get("path")
|
||||
if path:
|
||||
path = os.path.abspath(str(path))
|
||||
if not os.path.isfile(path):
|
||||
raise FileNotFoundError(f"TADA reference audio file not found: {path}")
|
||||
return {"kind": "audio_path", "audio_path": path}
|
||||
sample_rate = audio.get("sample_rate", audio.get("sampling_rate", sample_rate))
|
||||
audio = audio.get("waveform", audio.get("samples"))
|
||||
|
||||
if audio is None:
|
||||
raise ValueError("TADA requires reference audio.")
|
||||
waveform = audio.detach().cpu() if isinstance(audio, torch.Tensor) else torch.as_tensor(audio)
|
||||
tensor_path = bundle_dir / "reference.pt"
|
||||
torch.save({"waveform": waveform, "sample_rate": int(sample_rate)}, tensor_path)
|
||||
return {"kind": "tensor_path", "tensor_path": str(tensor_path)}
|
||||
|
||||
def generate_speech(
|
||||
self,
|
||||
text: str,
|
||||
reference_audio: object,
|
||||
reference_text: str,
|
||||
seed: int = 42,
|
||||
language: object = "English",
|
||||
acoustic_cfg_scale: float = 1.6,
|
||||
duration_cfg_scale: float = 1.0,
|
||||
cfg_schedule: str = "cosine",
|
||||
time_schedule: str = "logsnr",
|
||||
num_flow_matching_steps: int = 10,
|
||||
noise_temperature: float = 0.9,
|
||||
speed_up_factor: Optional[float] = None,
|
||||
num_transition_steps: int = 5,
|
||||
negative_condition_source: str = "negative_step_output",
|
||||
) -> Tuple[torch.Tensor, int]:
|
||||
if not str(reference_text or "").strip():
|
||||
raise ValueError(
|
||||
"TADA requires the exact transcript of the reference audio; ASR fallback is disabled."
|
||||
)
|
||||
validate_tada_model_language(self.current_model_name, language)
|
||||
|
||||
# A segment may switch language without reloading the base model. Ensure
|
||||
# just its aligner exists before the worker enters enforced offline mode.
|
||||
codec_parent = str(Path(self.codec_path).resolve().parent)
|
||||
TadaDownloader(base_path=codec_parent).ensure_codec_assets(
|
||||
language=language,
|
||||
codec_path=self.codec_path,
|
||||
)
|
||||
self._ensure_remote_engine()
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="tts_tada_") as temp_dir:
|
||||
bundle_dir = Path(temp_dir)
|
||||
output_path = bundle_dir / "result.pt"
|
||||
serialized_reference = self._serialize_reference(reference_audio, bundle_dir)
|
||||
self._request(
|
||||
"generate",
|
||||
{
|
||||
"text": str(text),
|
||||
"reference_audio": serialized_reference,
|
||||
"reference_text": str(reference_text),
|
||||
"seed": int(seed),
|
||||
"language": language,
|
||||
"acoustic_cfg_scale": float(acoustic_cfg_scale),
|
||||
"duration_cfg_scale": float(duration_cfg_scale),
|
||||
"cfg_schedule": str(cfg_schedule),
|
||||
"time_schedule": str(time_schedule),
|
||||
"num_flow_matching_steps": int(num_flow_matching_steps),
|
||||
"noise_temperature": float(noise_temperature),
|
||||
"speed_up_factor": speed_up_factor,
|
||||
"num_transition_steps": int(num_transition_steps),
|
||||
"negative_condition_source": str(negative_condition_source),
|
||||
"output_path": str(output_path),
|
||||
},
|
||||
)
|
||||
if not output_path.is_file():
|
||||
raise RuntimeError("TADA worker returned without writing its audio result.")
|
||||
result = torch.load(output_path, map_location="cpu")
|
||||
audio = result.get("audio")
|
||||
if not isinstance(audio, torch.Tensor):
|
||||
audio = torch.as_tensor(audio)
|
||||
return audio.detach().cpu().float(), int(result.get("sample_rate", 24000))
|
||||
|
||||
def to(self, device):
|
||||
self.device = resolve_torch_device(str(device))
|
||||
self.current_device = self.device
|
||||
self.load_device = self.current_loaded_device()
|
||||
return self
|
||||
|
||||
def eval(self):
|
||||
return self
|
||||
|
||||
def model_size(self):
|
||||
return self._estimated_memory_size
|
||||
|
||||
def loaded_size(self):
|
||||
return self._estimated_memory_size if self._initialized else 0
|
||||
|
||||
def model_memory(self):
|
||||
return self._estimated_memory_size
|
||||
|
||||
def get_ram_usage(self):
|
||||
return self._estimated_memory_size
|
||||
|
||||
def model_offloaded_memory(self):
|
||||
return 0 if self._initialized else self._estimated_memory_size
|
||||
|
||||
def model_mmap_residency(self, free: bool = False):
|
||||
return 0, self._estimated_memory_size
|
||||
|
||||
def pinned_memory_size(self):
|
||||
return 0
|
||||
|
||||
def lowvram_patch_counter(self):
|
||||
return 0
|
||||
|
||||
def model_dtype(self):
|
||||
return self.dtype
|
||||
|
||||
def current_loaded_device(self):
|
||||
if str(self.device).startswith("cuda") and torch.cuda.is_available():
|
||||
return torch.device("cuda", torch.cuda.current_device())
|
||||
return torch.device("cpu")
|
||||
|
||||
def model_patches_models(self):
|
||||
return ()
|
||||
|
||||
def model_patches_to(self, target):
|
||||
return None
|
||||
|
||||
def is_dynamic(self):
|
||||
return False
|
||||
|
||||
def partially_load(self, device, extra_memory, force_patch_weights=False):
|
||||
return 0
|
||||
|
||||
def partially_unload_ram(self, ram_to_unload):
|
||||
return self.partially_unload("cpu", ram_to_unload)
|
||||
|
||||
def partially_unload(self, device, memory_to_free):
|
||||
self.cleanup(unregister=False)
|
||||
return self._estimated_memory_size
|
||||
|
||||
def model_unload(self, *args, **kwargs):
|
||||
self.cleanup(unregister=False)
|
||||
return True
|
||||
|
||||
def detach(self, unpatch_weights=True):
|
||||
self.cleanup(unregister=False)
|
||||
|
||||
def is_clone(self, other):
|
||||
return other is self
|
||||
|
||||
def cleanup(self, unregister: bool = True) -> None:
|
||||
if unregister:
|
||||
self._unregister_from_comfy_model_management()
|
||||
else:
|
||||
self._comfy_loaded_model = None
|
||||
self._comfy_model_management = None
|
||||
process = getattr(getattr(self, "_session", None), "_process", None)
|
||||
if self._initialized and process is not None and process.poll() is None:
|
||||
try:
|
||||
self._request("cleanup")
|
||||
except Exception:
|
||||
pass
|
||||
self._initialized = False
|
||||
session = getattr(self, "_session", None)
|
||||
if session is not None:
|
||||
session.close()
|
||||
|
||||
def close(self) -> None:
|
||||
self.cleanup()
|
||||
|
||||
|
||||
def build_tada_isolated_proxy(config):
|
||||
requested_profile = getattr(config, "runtime_profile", None) or TADA_RUNTIME_PROFILE
|
||||
if requested_profile != TADA_RUNTIME_PROFILE:
|
||||
raise RuntimeError(
|
||||
f"TADA only supports runtime profile '{TADA_RUNTIME_PROFILE}', not '{requested_profile}'."
|
||||
)
|
||||
profile = get_runtime_profile(TADA_RUNTIME_PROFILE)
|
||||
if profile is None:
|
||||
raise RuntimeError(f"TADA runtime profile '{TADA_RUNTIME_PROFILE}' is not configured.")
|
||||
return TadaIsolatedProxy(config, profile)
|
||||
|
||||
|
||||
__all__ = ["TADA_RUNTIME_PROFILE", "TadaIsolatedProxy", "build_tada_isolated_proxy"]
|
||||
@@ -0,0 +1,191 @@
|
||||
"""Persistent Audio8 TTS worker for the shared Transformers 4 runtime."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import torch
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
project_root_str = str(PROJECT_ROOT)
|
||||
sys.path = [
|
||||
entry for entry in sys.path if str(Path(entry).resolve()) != project_root_str
|
||||
]
|
||||
sys.path.insert(0, project_root_str)
|
||||
|
||||
from utils.runtimes.protocol import RuntimeJobResponse # noqa: E402
|
||||
|
||||
|
||||
def _emit(protocol_out, response: RuntimeJobResponse) -> None:
|
||||
protocol_out.write(json.dumps(response.to_dict(), ensure_ascii=True) + "\n")
|
||||
protocol_out.flush()
|
||||
|
||||
|
||||
def _load_reference_audio(
|
||||
payload: Optional[Dict[str, Any]],
|
||||
) -> Any:
|
||||
if payload is None:
|
||||
return None
|
||||
kind = payload.get("kind")
|
||||
if kind == "audio_path":
|
||||
audio_path = payload.get("audio_path")
|
||||
if not audio_path:
|
||||
raise RuntimeError("Audio8 TTS audio_path payload is missing its file path")
|
||||
return audio_path
|
||||
if kind == "tensor_path":
|
||||
tensor_path = payload.get("tensor_path")
|
||||
if not tensor_path:
|
||||
raise RuntimeError(
|
||||
"Audio8 TTS tensor_path payload is missing its file path"
|
||||
)
|
||||
tensor_payload = torch.load(tensor_path, map_location="cpu")
|
||||
return {
|
||||
"array": tensor_payload["waveform"],
|
||||
"sampling_rate": int(tensor_payload["sample_rate"]),
|
||||
}
|
||||
raise RuntimeError(f"Unsupported Audio8 TTS reference payload kind: {kind}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
protocol_out = sys.stdout
|
||||
sys.stdout = sys.stderr
|
||||
engine = None
|
||||
|
||||
for line in sys.stdin:
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
|
||||
request = None
|
||||
try:
|
||||
request = json.loads(stripped)
|
||||
action = request.get("action")
|
||||
payload = request.get("payload") or {}
|
||||
request_id = request.get("request_id")
|
||||
|
||||
if action == "shutdown":
|
||||
_emit(
|
||||
protocol_out,
|
||||
RuntimeJobResponse(
|
||||
ok=True,
|
||||
result={"shutdown": True},
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
break
|
||||
|
||||
if action == "ping":
|
||||
_emit(
|
||||
protocol_out,
|
||||
RuntimeJobResponse(
|
||||
ok=True,
|
||||
result={"pong": True},
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
if action == "initialize":
|
||||
import transformers
|
||||
from packaging.version import Version
|
||||
|
||||
transformers_version = Version(transformers.__version__)
|
||||
if not (Version("4.57.0") <= transformers_version < Version("5.0.0")):
|
||||
raise RuntimeError(
|
||||
"Audio8 TTS requires Transformers >=4.57.0,<5; "
|
||||
f"worker loaded Transformers {transformers.__version__}"
|
||||
)
|
||||
|
||||
from engines.audio8_tts.audio8_tts_engine import (
|
||||
Audio8TTSEngine,
|
||||
)
|
||||
|
||||
engine = Audio8TTSEngine(
|
||||
model_name=(request.get("model_name") or "Audio8-TTS-Preview-0.6b"),
|
||||
model_dir=payload.get("model_path"),
|
||||
device=request.get("device") or "auto",
|
||||
dtype=payload.get("dtype", "auto"),
|
||||
)
|
||||
engine._ensure_model_loaded()
|
||||
_emit(
|
||||
protocol_out,
|
||||
RuntimeJobResponse(
|
||||
ok=True,
|
||||
result={
|
||||
"model_name": engine.model_name,
|
||||
"sample_rate": engine.SAMPLE_RATE,
|
||||
"transformers_version": (transformers.__version__),
|
||||
},
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
if engine is None:
|
||||
raise RuntimeError(
|
||||
"Audio8 TTS worker received generation before initialization"
|
||||
)
|
||||
if action != "generate":
|
||||
raise RuntimeError(f"Unsupported Audio8 TTS worker action '{action}'")
|
||||
|
||||
reference_audio = _load_reference_audio(payload.get("reference_audio"))
|
||||
audio = engine.generate(
|
||||
text=payload["text"],
|
||||
reference_audio=reference_audio,
|
||||
reference_text=payload.get("reference_text"),
|
||||
max_new_tokens=int(payload.get("max_new_tokens", 1024)),
|
||||
retry_max_new_tokens=int(payload.get("retry_max_new_tokens", 2000)),
|
||||
temperature=float(payload.get("temperature", 0.8)),
|
||||
top_p=float(payload.get("top_p", 0.95)),
|
||||
top_k=int(payload.get("top_k", 50)),
|
||||
do_sample=bool(payload.get("do_sample", True)),
|
||||
seed=int(payload.get("seed", 42)),
|
||||
)
|
||||
audio = torch.as_tensor(audio).detach().float().cpu()
|
||||
if audio.ndim == 1:
|
||||
audio = audio.unsqueeze(0)
|
||||
if audio.ndim != 2 or audio.shape[0] != 1:
|
||||
raise RuntimeError(
|
||||
f"Audio8 TTS generated invalid audio shape {tuple(audio.shape)}"
|
||||
)
|
||||
|
||||
output_path = Path(payload["output_path"])
|
||||
torch.save(
|
||||
{
|
||||
"audio": audio.contiguous(),
|
||||
"sample_rate": engine.SAMPLE_RATE,
|
||||
},
|
||||
output_path,
|
||||
)
|
||||
_emit(
|
||||
protocol_out,
|
||||
RuntimeJobResponse(
|
||||
ok=True,
|
||||
result={
|
||||
"output_path": str(output_path),
|
||||
"sample_rate": engine.SAMPLE_RATE,
|
||||
},
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
request_id = (
|
||||
request.get("request_id") if isinstance(request, dict) else None
|
||||
)
|
||||
_emit(
|
||||
protocol_out,
|
||||
RuntimeJobResponse(
|
||||
ok=False,
|
||||
error=f"{exc}\n{traceback.format_exc()}",
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,170 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
"""JSON-line worker for TADA's shared Transformers 4 runtime."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
ENGINES_DIR = PROJECT_ROOT / "engines"
|
||||
|
||||
# ComfyUI may add the suite's engines directory directly to PYTHONPATH. In this
|
||||
# worker that would make engines/tada shadow the official hume-tada package.
|
||||
sys.path[:] = [
|
||||
path
|
||||
for path in sys.path
|
||||
if Path(path or os.getcwd()).resolve() != ENGINES_DIR.resolve()
|
||||
]
|
||||
if str(PROJECT_ROOT) in sys.path:
|
||||
sys.path.remove(str(PROJECT_ROOT))
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
os.environ.setdefault("HF_HUB_OFFLINE", "1")
|
||||
os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
|
||||
|
||||
from utils.runtimes.protocol import RuntimeJobResponse
|
||||
|
||||
|
||||
def _emit(stream, response: RuntimeJobResponse) -> None:
|
||||
stream.write(json.dumps(response.to_dict(), ensure_ascii=True) + "\n")
|
||||
stream.flush()
|
||||
|
||||
|
||||
def _load_reference(payload):
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("TADA worker received an invalid reference-audio payload.")
|
||||
kind = payload.get("kind")
|
||||
if kind == "audio_path":
|
||||
return payload.get("audio_path")
|
||||
if kind == "tensor_path":
|
||||
tensor_path = payload.get("tensor_path")
|
||||
if not tensor_path:
|
||||
raise ValueError("TADA tensor reference is missing tensor_path.")
|
||||
saved = torch.load(tensor_path, map_location="cpu")
|
||||
return saved["waveform"], int(saved.get("sample_rate", 24000))
|
||||
raise ValueError(f"Unsupported TADA reference-audio payload kind: {kind}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
protocol_out = sys.stdout
|
||||
sys.stdout = sys.stderr
|
||||
|
||||
from engines.tada.tada_engine import TadaEngine
|
||||
|
||||
engine = None
|
||||
for line in sys.stdin:
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
request = None
|
||||
try:
|
||||
request = json.loads(stripped)
|
||||
request_id = request.get("request_id")
|
||||
action = request.get("action")
|
||||
payload = request.get("payload") or {}
|
||||
|
||||
if action == "shutdown":
|
||||
if engine is not None:
|
||||
engine.cleanup()
|
||||
engine = None
|
||||
_emit(protocol_out, RuntimeJobResponse(ok=True, result={"shutdown": True}, request_id=request_id))
|
||||
break
|
||||
|
||||
if action == "ping":
|
||||
_emit(protocol_out, RuntimeJobResponse(ok=True, result={"pong": True}, request_id=request_id))
|
||||
continue
|
||||
|
||||
if action == "initialize":
|
||||
if request.get("runtime_profile") != "vibevoice_transformers4_shared":
|
||||
raise RuntimeError("TADA can only initialize in vibevoice_transformers4_shared.")
|
||||
if engine is not None:
|
||||
engine.cleanup()
|
||||
engine = TadaEngine(
|
||||
model_name=request.get("model_name") or "TADA-1B",
|
||||
model_path=payload["model_path"],
|
||||
codec_path=payload["codec_path"],
|
||||
tokenizer_path=payload["tokenizer_path"],
|
||||
device=request.get("device") or "auto",
|
||||
dtype=payload.get("dtype", "auto"),
|
||||
attn_implementation=payload.get("attn_implementation", "sdpa"),
|
||||
use_torch_compile=payload.get("use_torch_compile", False),
|
||||
prompt_cache_size=payload.get("prompt_cache_size", 8),
|
||||
)
|
||||
_emit(
|
||||
protocol_out,
|
||||
RuntimeJobResponse(
|
||||
ok=True,
|
||||
result={"model_name": request.get("model_name"), "sample_rate": 24000},
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
if action == "cleanup":
|
||||
if engine is not None:
|
||||
engine.cleanup()
|
||||
engine = None
|
||||
_emit(protocol_out, RuntimeJobResponse(ok=True, result={"cleaned": True}, request_id=request_id))
|
||||
continue
|
||||
|
||||
if engine is None:
|
||||
raise RuntimeError("TADA worker received a generation request before initialization.")
|
||||
|
||||
if action == "generate":
|
||||
audio, sample_rate = engine.generate_speech(
|
||||
text=payload["text"],
|
||||
reference_audio=_load_reference(payload.get("reference_audio")),
|
||||
reference_text=payload["reference_text"],
|
||||
seed=payload.get("seed", 42),
|
||||
language=payload.get("language", "English"),
|
||||
acoustic_cfg_scale=payload.get("acoustic_cfg_scale", 1.6),
|
||||
duration_cfg_scale=payload.get("duration_cfg_scale", 1.0),
|
||||
cfg_schedule=payload.get("cfg_schedule", "cosine"),
|
||||
time_schedule=payload.get("time_schedule", "logsnr"),
|
||||
num_flow_matching_steps=payload.get("num_flow_matching_steps", 10),
|
||||
noise_temperature=payload.get("noise_temperature", 0.9),
|
||||
speed_up_factor=payload.get("speed_up_factor"),
|
||||
num_transition_steps=payload.get("num_transition_steps", 5),
|
||||
negative_condition_source=payload.get(
|
||||
"negative_condition_source", "negative_step_output"
|
||||
),
|
||||
)
|
||||
output_path = payload["output_path"]
|
||||
torch.save(
|
||||
{"audio": audio.detach().cpu().float(), "sample_rate": int(sample_rate)},
|
||||
output_path,
|
||||
)
|
||||
_emit(
|
||||
protocol_out,
|
||||
RuntimeJobResponse(
|
||||
ok=True,
|
||||
result={"output_path": output_path, "sample_rate": int(sample_rate)},
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
raise RuntimeError(f"Unsupported TADA worker action '{action}'")
|
||||
|
||||
except Exception as exc:
|
||||
request_id = request.get("request_id") if isinstance(request, dict) else None
|
||||
_emit(
|
||||
protocol_out,
|
||||
RuntimeJobResponse(
|
||||
ok=False,
|
||||
error=f"{exc}\n{traceback.format_exc()}",
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -28,23 +28,6 @@ PARAMETER_ALIASES = {
|
||||
'cfg_weight': 'cfg', # cfg_weight alias -> cfg (more universal)
|
||||
'cfgweight': 'cfg',
|
||||
'cfg_scale': 'cfg',
|
||||
'acoustic_cfg': 'acoustic_cfg_scale',
|
||||
'acoustic_cfg_scale': 'acoustic_cfg_scale',
|
||||
'duration_cfg': 'duration_cfg_scale',
|
||||
'duration_cfg_scale': 'duration_cfg_scale',
|
||||
'noise_temp': 'noise_temperature',
|
||||
'noise_temperature': 'noise_temperature',
|
||||
'flow_steps': 'num_flow_matching_steps',
|
||||
'flow_matching_steps': 'num_flow_matching_steps',
|
||||
'num_flow_matching_steps': 'num_flow_matching_steps',
|
||||
'cfg_schedule': 'cfg_schedule',
|
||||
'time_schedule': 'time_schedule',
|
||||
'negative_condition': 'negative_condition_source',
|
||||
'negative_condition_source': 'negative_condition_source',
|
||||
'speed_up': 'speed_up_factor',
|
||||
'speed_up_factor': 'speed_up_factor',
|
||||
'transition_steps': 'num_transition_steps',
|
||||
'num_transition_steps': 'num_transition_steps',
|
||||
'stg': 'stg_scale',
|
||||
'stg_scale': 'stg_scale',
|
||||
'duration_multiplier': 'duration_multiplier',
|
||||
@@ -132,26 +115,18 @@ PARAMETER_ENGINES = {
|
||||
'seed': {
|
||||
'chatterbox', 'chatterbox_official_23lang', 'f5tts', 'higgs_audio',
|
||||
'higgs_audio_v3', 'vibevoice', 'index_tts', 'step_audio_editx', 'cosyvoice', 'qwen3_tts',
|
||||
'dots_tts', 'fish_audio_s2', 'omnivoice', 'tada',
|
||||
'dots_tts', 'audio8_tts', 'fish_audio_s2', 'omnivoice',
|
||||
'echo_tts', 'moss_tts', 'moss_soundeffect_v2', 'dramabox'
|
||||
},
|
||||
'temperature': {
|
||||
'chatterbox', 'chatterbox_official_23lang', 'f5tts', 'higgs_audio',
|
||||
'higgs_audio_v3', 'vibevoice', 'index_tts', 'step_audio_editx', 'qwen3_tts', 'moss_tts', 'fish_audio_s2'
|
||||
'higgs_audio_v3', 'vibevoice', 'index_tts', 'step_audio_editx', 'qwen3_tts',
|
||||
'moss_tts', 'fish_audio_s2', 'audio8_tts'
|
||||
},
|
||||
'cfg': {
|
||||
'f5tts', 'vibevoice', 'index_tts', 'chatterbox', 'chatterbox_official_23lang',
|
||||
'moss_soundeffect_v2', 'dramabox', 'tada'
|
||||
'moss_soundeffect_v2', 'dramabox'
|
||||
},
|
||||
'acoustic_cfg_scale': {'tada'},
|
||||
'duration_cfg_scale': {'tada'},
|
||||
'noise_temperature': {'tada'},
|
||||
'num_flow_matching_steps': {'tada'},
|
||||
'cfg_schedule': {'tada'},
|
||||
'time_schedule': {'tada'},
|
||||
'negative_condition_source': {'tada'},
|
||||
'speed_up_factor': {'tada'},
|
||||
'num_transition_steps': {'tada'},
|
||||
'stg_scale': {
|
||||
'dramabox'
|
||||
},
|
||||
@@ -171,7 +146,7 @@ PARAMETER_ENGINES = {
|
||||
'dramabox'
|
||||
},
|
||||
'num_steps': {
|
||||
'echo_tts', 'dots_tts', 'omnivoice', 'tada'
|
||||
'echo_tts', 'dots_tts', 'omnivoice'
|
||||
},
|
||||
'guidance_scale': {
|
||||
'dots_tts', 'omnivoice'
|
||||
@@ -237,13 +212,15 @@ PARAMETER_ENGINES = {
|
||||
'chatterbox', 'chatterbox_official_23lang'
|
||||
},
|
||||
'speed': {
|
||||
'f5tts', 'cosyvoice', 'omnivoice', 'tada'
|
||||
'f5tts', 'cosyvoice', 'omnivoice'
|
||||
},
|
||||
'top_p': {
|
||||
'higgs_audio', 'higgs_audio_v3', 'vibevoice', 'index_tts', 'qwen3_tts', 'moss_tts', 'fish_audio_s2'
|
||||
'higgs_audio', 'higgs_audio_v3', 'vibevoice', 'index_tts', 'qwen3_tts',
|
||||
'moss_tts', 'fish_audio_s2', 'audio8_tts'
|
||||
},
|
||||
'top_k': {
|
||||
'higgs_audio', 'higgs_audio_v3', 'index_tts', 'qwen3_tts', 'moss_tts'
|
||||
'higgs_audio', 'higgs_audio_v3', 'index_tts', 'qwen3_tts', 'moss_tts',
|
||||
'audio8_tts'
|
||||
},
|
||||
'audio_temperature': {
|
||||
'moss_tts'
|
||||
@@ -264,7 +241,7 @@ PARAMETER_ENGINES = {
|
||||
'moss_tts'
|
||||
},
|
||||
'max_new_tokens': {
|
||||
'higgs_audio_v3', 'moss_tts', 'fish_audio_s2'
|
||||
'higgs_audio_v3', 'moss_tts', 'fish_audio_s2', 'audio8_tts'
|
||||
},
|
||||
'max_generate_length': {
|
||||
'dots_tts'
|
||||
@@ -309,17 +286,6 @@ PARAMETER_VALIDATION = {
|
||||
'seed': (int, 0, 2**32 - 1, "Random seed for reproducible generation"),
|
||||
'temperature': (float, 0.1, 2.0, "Randomness/creativity control (lower=more deterministic)"),
|
||||
'cfg': (float, 0.0, 20.0, "Classifier-free guidance strength"),
|
||||
'acoustic_cfg_scale': (float, 0.0, 10.0, "TADA acoustic classifier-free guidance scale"),
|
||||
'duration_cfg_scale': (float, 0.0, 10.0, "TADA duration classifier-free guidance scale"),
|
||||
'noise_temperature': (float, 0.0, 3.0, "TADA acoustic noise temperature"),
|
||||
'num_flow_matching_steps': (int, 1, 100, "TADA flow-matching inference steps"),
|
||||
'cfg_schedule': (str, None, None, "TADA CFG schedule: constant, linear, or cosine"),
|
||||
'time_schedule': (str, None, None, "TADA time schedule: uniform, cosine, or logsnr"),
|
||||
'negative_condition_source': (
|
||||
str, None, None, "TADA negative condition: negative_step_output, prompt, or zero"
|
||||
),
|
||||
'speed_up_factor': (float, 0.0, 4.0, "TADA speech speed-up factor; 0 disables it"),
|
||||
'num_transition_steps': (int, 0, 64, "TADA prompt-to-generation transition steps"),
|
||||
'stg_scale': (float, 0.0, 5.0, "DramaBox skip-token guidance strength"),
|
||||
'duration_multiplier': (float, 0.5, 3.0, "DramaBox estimated-duration multiplier"),
|
||||
'gen_duration': (float, 0.0, 60.0, "DramaBox explicit output duration"),
|
||||
@@ -387,27 +353,14 @@ PARAMETER_NODE_KEYS = {
|
||||
'f5tts': 'cfg_strength',
|
||||
'moss_soundeffect_v2': 'cfg_scale',
|
||||
'dramabox': 'cfg_scale',
|
||||
'tada': 'acoustic_cfg_scale',
|
||||
}, # Engine-specific mapping
|
||||
'acoustic_cfg_scale': 'acoustic_cfg_scale',
|
||||
'duration_cfg_scale': 'duration_cfg_scale',
|
||||
'noise_temperature': 'noise_temperature',
|
||||
'num_flow_matching_steps': 'num_flow_matching_steps',
|
||||
'cfg_schedule': 'cfg_schedule',
|
||||
'time_schedule': 'time_schedule',
|
||||
'negative_condition_source': 'negative_condition_source',
|
||||
'speed_up_factor': 'speed_up_factor',
|
||||
'num_transition_steps': 'num_transition_steps',
|
||||
'stg_scale': 'stg_scale',
|
||||
'duration_multiplier': 'duration_multiplier',
|
||||
'gen_duration': 'gen_duration',
|
||||
'ref_duration': 'ref_duration',
|
||||
'rescale_scale': 'rescale_scale',
|
||||
'prompt_template': 'prompt_template',
|
||||
'num_steps': {
|
||||
'default': 'num_steps',
|
||||
'tada': 'num_flow_matching_steps',
|
||||
},
|
||||
'num_steps': 'num_steps',
|
||||
'guidance_scale': 'guidance_scale',
|
||||
'duration': 'duration',
|
||||
't_shift': 't_shift',
|
||||
@@ -429,10 +382,7 @@ PARAMETER_NODE_KEYS = {
|
||||
'speaker_kv_min_t': 'speaker_kv_min_t',
|
||||
'sequence_length': 'sequence_length',
|
||||
'exaggeration': 'exaggeration',
|
||||
'speed': {
|
||||
'default': 'speed',
|
||||
'tada': 'speed_up_factor',
|
||||
},
|
||||
'speed': 'speed',
|
||||
'top_p': 'top_p',
|
||||
'top_k': 'top_k',
|
||||
'audio_temperature': 'audio_temperature',
|
||||
@@ -531,17 +481,6 @@ class ParameterValidator:
|
||||
if not 0.0 <= converted <= 1.0:
|
||||
return False, "rescale_scale must be 'auto' or between 0 and 1", value
|
||||
|
||||
allowed_values = {
|
||||
'cfg_schedule': {'constant', 'linear', 'cosine'},
|
||||
'time_schedule': {'uniform', 'cosine', 'logsnr'},
|
||||
'negative_condition_source': {'negative_step_output', 'prompt', 'zero'},
|
||||
}
|
||||
if param_name in allowed_values:
|
||||
converted = str(converted).strip().lower()
|
||||
if converted not in allowed_values[param_name]:
|
||||
choices = ', '.join(sorted(allowed_values[param_name]))
|
||||
return False, f"{param_name} must be one of: {choices}", value
|
||||
|
||||
# Check bounds
|
||||
if min_val is not None and max_val is not None and (converted < min_val or converted > max_val):
|
||||
return False, f"{param_name} must be between {min_val} and {max_val}, got {converted}", value
|
||||
|
||||
@@ -570,8 +570,8 @@ class VoiceDiscovery:
|
||||
if engine_type in ("audio_only", "chatterbox", "moss_tts", "higgs_audio_v3"): # Keep backward compatibility
|
||||
return character_info
|
||||
|
||||
# For audio+text engines (F5-TTS, Step Audio EditX, CosyVoice, Qwen3-TTS, TADA), we need both audio and text
|
||||
if engine_type in ("audio_and_text", "f5tts", "step_audio_editx", "cosyvoice", "index_tts", "qwen3_tts", "tada"):
|
||||
# For audio+text engines (F5-TTS, Step Audio EditX, CosyVoice, Qwen3-TTS), we need both audio and text
|
||||
if engine_type in ("audio_and_text", "f5tts", "step_audio_editx", "cosyvoice", "index_tts", "qwen3_tts"):
|
||||
if character_info.get('text_content'):
|
||||
return character_info
|
||||
|
||||
|
||||
Reference in New Issue
Block a user