diff --git a/README.md b/README.md index 18a0f42..b39e020 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ Unofficial [OpenVoice](https://github.com/myshell-ai/OpenVoice) support for [ComfyUI](https://github.com/comfyanonymous/ComfyUI). ## Important Updates +- **2024/05/14:** Added OpenVoice V2 support (needs extra installation) - **2024/04/12:** Added ComfyUI nodes and workflow examples ## Basic Workflow @@ -19,5 +20,12 @@ sudo apt install ffmpeg pip install -r requirements.txt ``` +## Extra Installation for V2 +``` +apt get install mecab libmecab-dev +pip install git+https://github.com/myshell-ai/MeloTTS.git +python -m unidic download +``` + ## Download Checkpoints All the models are downloaded automatically, no worries. \ No newline at end of file diff --git a/README.md.bak b/README.md.bak index 063f330..d1e29e1 100644 --- a/README.md.bak +++ b/README.md.bak @@ -8,6 +8,9 @@ ## Introduction + +### OpenVoice V1 + As we detailed in our [paper](https://arxiv.org/abs/2312.01479) and [website](https://research.myshell.ai/open-voice), the advantages of OpenVoice are three-fold: **1. Accurate Tone Color Cloning.** @@ -19,6 +22,19 @@ OpenVoice enables granular control over voice styles, such as emotion and accent **3. Zero-shot Cross-lingual Voice Cloning.** Neither of the language of the generated speech nor the language of the reference speech needs to be presented in the massive-speaker multi-lingual training dataset. +### OpenVoice V2 + +In April 2024, we released OpenVoice V2, which includes all features in V1 and has: + +**1. Better Audio Quality.** +OpenVoice V2 adopts a different training strategy that delivers better audio quality. + +**2. Native Multi-lingual Support.** +English, Spanish, French, Chinese, Japanese and Korean are natively supported in OpenVoice V2. + +**3. Free Commercial Use.** +Starting from April 2024, both V2 and V1 are released under MIT License. Free for commercial use. + [Video](https://github.com/myshell-ai/OpenVoice/assets/40556743/3cba936f-82bf-476c-9e52-09f0f417bb2f) OpenVoice has been powering the instant voice cloning capability of [myshell.ai](https://app.myshell.ai/explore) since May 2023. Until Nov 2023, the voice cloning model has been used tens of millions of times by users worldwide, and witnessed the explosive user growth on the platform. @@ -52,7 +68,7 @@ Join our [Discord community](https://discord.gg/myshell) and select the `Develop ``` ## License -This repository is licensed under a Creative Commons Attribution-NonCommercial 4.0 International License, which prohibits commercial usage. **This will be changed to a license that allows Free Commercial usage in the near future.** Stay tuned. For social responsibility and anti-misuse considerations, **MyShell reserves the ability to detect whether an audio is generated by OpenVoice**, no matter whether the watermark is added or not. +OpenVoice V1 and V2 are MIT Licensed. Free for both commercial and research use. ## Acknowledgements -This implementation is based on several excellent projects, [TTS](https://github.com/coqui-ai/TTS), [VITS](https://github.com/jaywalnut310/vits), and [VITS2](https://github.com/daniilrobnikov/vits2). Thanks for their awesome work! +This implementation is based on several excellent projects, [TTS](https://github.com/coqui-ai/TTS), [VITS](https://github.com/jaywalnut310/vits), and [VITS2](https://github.com/daniilrobnikov/vits2). Thanks for their awesome work! \ No newline at end of file diff --git a/nodes.py b/nodes.py index a65ccd3..2526fc7 100644 --- a/nodes.py +++ b/nodes.py @@ -4,6 +4,7 @@ import random import folder_paths import soundfile as sf +from melo.api import TTS from .openvoice import se_extractor from .openvoice.api import BaseSpeakerTTS, ToneColorConverter @@ -48,7 +49,7 @@ class OpenVoiceTTS: local_dir = os.path.join(folder_paths.models_dir, 'openovice') if not os.path.exists(local_dir) or not os.path.isdir(local_dir): from huggingface_hub import snapshot_download - snapshot_download(repo_id="Alignment-Lab-AI/OpenVoice", local_dir=local_dir, local_dir_use_symlinks=False) + snapshot_download(repo_id="myshell-ai/OpenVoice", local_dir=local_dir, local_dir_use_symlinks=False) mark = BaseSpeakerTTS.language_marks.get(lang.lower(), None) assert mark is not None, f"language {lang} is not supported" @@ -89,6 +90,73 @@ class OpenVoiceTTS: audio_samples, sample_rate =sf.read(save_path) return (list(audio_samples), sample_rate) +class OpenVoiceTTSV2: + @classmethod + def INPUT_TYPES(s): + audio_extensions = ["wav", "mp3", "flac"] + input_dir = folder_paths.get_input_directory() + files = [] + for f in os.listdir(input_dir): + if os.path.isfile(os.path.join(input_dir, f)): + file_parts = f.lower().split('.') + if len(file_parts) > 1 and (file_parts[-1] in audio_extensions): + files.append(f) + return { + "required": { + "text": ("STRING", {"default": '', "multiline": True}), + "lang": (["EN","EN_NEWEST","FR","JP","ES","ZH","KR"],), + "speed": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.1}), + "ref_voice": (sorted(files),), + }, + } + + CATEGORY = "OpenVoice" + + RETURN_TYPES = (any, "INT",) + RETURN_NAMES = ("AUDIO", "SAMPLE_RATE",) + FUNCTION = "inference" + + def inference(self, text, lang, speed, ref_voice): + local_dir = os.path.join(folder_paths.models_dir, 'openovice', 'checkpoints_v2') + if not os.path.exists(local_dir) or not os.path.isdir(local_dir): + from huggingface_hub import snapshot_download + snapshot_download(repo_id="myshell-ai/OpenVoiceV2", local_dir=local_dir, local_dir_use_symlinks=False) + + device="cuda:0" if torch.cuda.is_available() else "cpu" + model = TTS(language=lang, device=device) + speaker_key, speaker_id = list(model.hps.data.spk2id.items())[0] + + ckpt_base = os.path.join(local_dir, 'base_speakers') + ckpt_converter = os.path.join(local_dir, 'converter') + + tone_color_converter = ToneColorConverter(f'{ckpt_converter}/config.json', device=device) + tone_color_converter.load_ckpt(f'{ckpt_converter}/checkpoint.pth') + + source_se = torch.load(f'{ckpt_base}/ses/{speaker_key.lower()}.pth', map_location=device) + reference_speaker = os.path.join(folder_paths.get_input_directory(), ref_voice) + + temp_dir = folder_paths.get_temp_directory() + file_prefix = ''.join(random.choice("abcdefghijklmnopqrstupvxyz") for _ in range(5)) + target_se, audio_name = se_extractor.get_se(reference_speaker, tone_color_converter, target_dir=temp_dir, vad=False) + + save_path = f'{temp_dir}/{file_prefix}_output_v2.wav' + + # Run the base speaker tts + src_path = f'{temp_dir}/{file_prefix}_base_v2.wav' + model.tts_to_file(text, speaker_id, src_path, speed=speed) + + # Run the tone color converter + encode_message = "@MyShell" + tone_color_converter.convert( + audio_src_path=src_path, + src_se=source_se, + tgt_se=target_se, + output_path=save_path, + message=encode_message) + + audio_samples, sample_rate =sf.read(save_path) + return (list(audio_samples), sample_rate) + class OpenVoiceSTS: @classmethod def INPUT_TYPES(s): @@ -117,7 +185,7 @@ class OpenVoiceSTS: local_dir = os.path.join(folder_paths.models_dir, 'openovice') if not os.path.exists(local_dir) or not os.path.isdir(local_dir): from huggingface_hub import snapshot_download - snapshot_download(repo_id="Alignment-Lab-AI/OpenVoice", local_dir=local_dir, local_dir_use_symlinks=False) + snapshot_download(repo_id="myshell-ai/OpenVoice", local_dir=local_dir, local_dir_use_symlinks=False) ckpt_converter = os.path.join(local_dir, 'checkpoints/converter') device="cuda:0" if torch.cuda.is_available() else "cpu" @@ -150,11 +218,13 @@ class OpenVoiceSTS: NODE_CLASS_MAPPINGS = { - "D_OpenVoice_TTS": OpenVoiceTTS, - "D_OpenVoice_STS": OpenVoiceSTS, + "D_OpenVoice_TTS" : OpenVoiceTTS, + "D_OpenVoice_TTS_V2" : OpenVoiceTTSV2, + "D_OpenVoice_STS" : OpenVoiceSTS, } NODE_DISPLAY_NAME_MAPPINGS = { - "D_OpenVoice_TTS": "Open Voice TTS", - "D_OpenVoice_STS": "Open Voice STS", + "D_OpenVoice_TTS" : "Open Voice TTS", + "D_OpenVoice_TTS_V2" : "Open Voice TTS V2", + "D_OpenVoice_STS" : "Open Voice STS", } \ No newline at end of file