pentarab-ComfyUI-For-ChatterBox/node.py

"""
ComfyUI nodes for Chatterbox Multilingual TTS.
Provides voice cloning and text-to-speech synthesis with 23 language support.
"""

import os
import random
import tempfile

import numpy as np
import torch
import torchaudio

# Supported languages for the multilingual model
SUPPORTED_LANGUAGES = {
    "ar": "Arabic",
    "da": "Danish",
    "de": "German",
    "el": "Greek",
    "en": "English",
    "es": "Spanish",
    "fi": "Finnish",
    "fr": "French",
    "he": "Hebrew",
    "hi": "Hindi",
    "it": "Italian",
    "ja": "Japanese",
    "ko": "Korean",
    "ms": "Malay",
    "nl": "Dutch",
    "no": "Norwegian",
    "pl": "Polish",
    "pt": "Portuguese",
    "ru": "Russian",
    "sv": "Swedish",
    "sw": "Swahili",
    "tr": "Turkish",
    "zh": "Chinese",
}

# Global model cache
_MODEL_CACHE = {}


def get_device():
    """Get the appropriate device for inference."""
    if torch.cuda.is_available():
        return "cuda"
    elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
        return "mps"
    return "cpu"


def get_or_load_model(device=None):
    """Load and cache the Chatterbox model."""
    global _MODEL_CACHE

    if device is None:
        device = get_device()

    cache_key = f"chatterbox_mtl_{device}"

    if cache_key not in _MODEL_CACHE:
        from .chatterbox_handler import (
            load_chatterbox_multilingual_tts_model,
            DEFAULT_MTL_MODEL_PACK_NAME,
        )

        print(f"[Chatterbox] Loading multilingual model on {device}...")
        _MODEL_CACHE[cache_key] = load_chatterbox_multilingual_tts_model(
            DEFAULT_MTL_MODEL_PACK_NAME, device
        )
        print("[Chatterbox] Model loaded successfully.")

    return _MODEL_CACHE[cache_key]


def set_seed(seed: int, device: str):
    """Set random seed for reproducibility."""
    torch.manual_seed(seed)
    if device == "cuda":
        torch.cuda.manual_seed(seed)
        torch.cuda.manual_seed_all(seed)
    random.seed(seed)
    np.random.seed(seed)


class ChatterboxTTSNode:
    """
    Chatterbox Multilingual TTS Node.
    Generate speech from text with optional voice cloning from reference audio.
    Supports 23 languages including English, Korean, Japanese, Chinese, and more.
    """

    @classmethod
    def INPUT_TYPES(cls):
        # Get list of languages for dropdown
        language_list = [
            f"{code} ({name})" for code, name in sorted(SUPPORTED_LANGUAGES.items())
        ]

        return {
            "required": {
                "text": (
                    "STRING",
                    {
                        "multiline": True,
                        "default": "Hello, this is a test of the Chatterbox text to speech system.",
                        "tooltip": "Text to synthesize into speech (max 300 characters)",
                    },
                ),
                "language": (
                    language_list,
                    {
                        "default": "en (English)",
                        "tooltip": "Language for text-to-speech synthesis",
                    },
                ),
                "exaggeration": (
                    "FLOAT",
                    {
                        "default": 0.5,
                        "min": 0.25,
                        "max": 2.0,
                        "step": 0.05,
                        "tooltip": "Speech expressiveness (0.5=neutral, higher=more expressive, extreme values may be unstable)",
                    },
                ),
                "cfg_weight": (
                    "FLOAT",
                    {
                        "default": 0.5,
                        "min": 0.0,
                        "max": 1.0,
                        "step": 0.05,
                        "tooltip": "CFG/Pace weight. Set to 0 for language transfer to reduce accent from reference audio.",
                    },
                ),
                "temperature": (
                    "FLOAT",
                    {
                        "default": 0.8,
                        "min": 0.05,
                        "max": 5.0,
                        "step": 0.05,
                        "tooltip": "Randomness in generation (higher=more varied)",
                    },
                ),
                "seed": (
                    "INT",
                    {
                        "default": 0,
                        "min": 0,
                        "max": 0xFFFFFFFF,
                        "tooltip": "Random seed (0 for random generation)",
                    },
                ),
            },
            "optional": {
                "reference_audio": (
                    "AUDIO",
                    {
                        "tooltip": "Optional reference audio for voice cloning. If not provided, uses default voice.",
                    },
                ),
            },
        }

    RETURN_TYPES = ("AUDIO",)
    RETURN_NAMES = ("audio",)
    FUNCTION = "generate"
    CATEGORY = "audio/tts"
    DESCRIPTION = "Generate speech from text using Chatterbox Multilingual TTS. Supports 23 languages and optional voice cloning."

    def generate(
        self,
        text,
        language,
        exaggeration,
        cfg_weight,
        temperature,
        seed,
        reference_audio=None,
    ):
        # Extract language code from selection (e.g., "en (English)" -> "en")
        language_code = language.split(" ")[0]

        # Get device and model
        device = get_device()
        model = get_or_load_model(device)

        # Set seed for reproducibility
        if seed != 0:
            set_seed(seed, device)

        # Handle reference audio
        audio_prompt_path = None
        temp_file = None

        if reference_audio is not None:
            # ComfyUI AUDIO format: {"waveform": tensor, "sample_rate": int}
            waveform = reference_audio["waveform"]
            sample_rate = reference_audio["sample_rate"]

            # Save to temp file for the model
            temp_file = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
            audio_prompt_path = temp_file.name

            # Ensure waveform is in correct format [channels, samples]
            if waveform.dim() == 3:
                waveform = waveform.squeeze(0)  # Remove batch dimension

            torchaudio.save(audio_prompt_path, waveform.cpu(), sample_rate)

        try:
            # Truncate text to max length
            text = text[:300]

            print(f"[Chatterbox] Generating audio for: '{text[:50]}...'")
            print(
                f"[Chatterbox] Language: {language_code}, Exaggeration: {exaggeration}, CFG: {cfg_weight}"
            )

            if audio_prompt_path:
                print("[Chatterbox] Using reference audio for voice cloning")
            else:
                print("[Chatterbox] Using default voice")

            # Generate audio
            wav = model.generate(
                text,
                language_id=language_code,
                audio_prompt_path=audio_prompt_path,
                exaggeration=exaggeration,
                cfg_weight=cfg_weight,
                temperature=temperature,
            )

            print("[Chatterbox] Audio generation complete.")

            # Convert to ComfyUI AUDIO format
            # wav shape: [1, samples], model.sr is sample rate
            if wav.dim() == 1:
                wav = wav.unsqueeze(0)  # [samples] -> [1, samples]

            # Add batch dimension: [channels, samples] -> [batch, channels, samples]
            wav = wav.unsqueeze(0)

            return ({"waveform": wav, "sample_rate": model.sr},)

        finally:
            # Clean up temp file
            if temp_file is not None:
                try:
                    os.unlink(temp_file.name)
                except Exception:
                    pass


# Node registration for ComfyUI
NODE_CLASS_MAPPINGS = {
    "ChatterboxTTS": ChatterboxTTSNode,
}

NODE_DISPLAY_NAME_MAPPINGS = {
    "ChatterboxTTS": "Chatterbox TTS (Multilingual)",
}