add v2 support

2024-05-14 11:56:35 +00:00
parent 67fa55aa02
commit cd2faa1f7d
3 changed files with 102 additions and 8 deletions
@@ -3,6 +3,7 @@
 Unofficial [OpenVoice](https://github.com/myshell-ai/OpenVoice) support for [ComfyUI](https://github.com/comfyanonymous/ComfyUI).

 ## Important Updates
+- **2024/05/14:** Added OpenVoice V2 support (needs extra installation)
 - **2024/04/12:** Added ComfyUI nodes and workflow examples

 ## Basic Workflow
@@ -19,5 +20,12 @@ sudo apt install ffmpeg
 pip install -r requirements.txt
 ```

+## Extra Installation for V2
+```
+apt get install mecab libmecab-dev
+pip install git+https://github.com/myshell-ai/MeloTTS.git
+python -m unidic download
+```
+
 ## Download Checkpoints
 All the models are downloaded automatically, no worries.
@@ -8,6 +8,9 @@
 </div>

 ## Introduction
+
+### OpenVoice V1
+
 As we detailed in our [paper](https://arxiv.org/abs/2312.01479) and [website](https://research.myshell.ai/open-voice), the advantages of OpenVoice are three-fold:

 **1. Accurate Tone Color Cloning.**
@@ -19,6 +22,19 @@ OpenVoice enables granular control over voice styles, such as emotion and accent
 **3. Zero-shot Cross-lingual Voice Cloning.**
 Neither of the language of the generated speech nor the language of the reference speech needs to be presented in the massive-speaker multi-lingual training dataset.

+### OpenVoice V2
+
+In April 2024, we released OpenVoice V2, which includes all features in V1 and has:
+
+**1. Better Audio Quality.**
+OpenVoice V2 adopts a different training strategy that delivers better audio quality.
+
+**2. Native Multi-lingual Support.**
+English, Spanish, French, Chinese, Japanese and Korean are natively supported in OpenVoice V2.
+
+**3. Free Commercial Use.**
+Starting from April 2024, both V2 and V1 are released under MIT License. Free for commercial use.
+
 [Video](https://github.com/myshell-ai/OpenVoice/assets/40556743/3cba936f-82bf-476c-9e52-09f0f417bb2f)

 OpenVoice has been powering the instant voice cloning capability of [myshell.ai](https://app.myshell.ai/explore) since May 2023. Until Nov 2023, the voice cloning model has been used tens of millions of times by users worldwide, and witnessed the explosive user growth on the platform.
@@ -52,7 +68,7 @@ Join our [Discord community](https://discord.gg/myshell) and select the `Develop
 ```

 ## License
-This repository is licensed under a Creative Commons Attribution-NonCommercial 4.0 International License, which prohibits commercial usage. **This will be changed to a license that allows Free Commercial usage in the near future.** Stay tuned. For social responsibility and anti-misuse considerations, **MyShell reserves the ability to detect whether an audio is generated by OpenVoice**, no matter whether the watermark is added or not.
+OpenVoice V1 and V2 are MIT Licensed. Free for both commercial and research use.

 ## Acknowledgements
-This implementation is based on several excellent projects, [TTS](https://github.com/coqui-ai/TTS), [VITS](https://github.com/jaywalnut310/vits), and [VITS2](https://github.com/daniilrobnikov/vits2). Thanks for their awesome work!
+This implementation is based on several excellent projects, [TTS](https://github.com/coqui-ai/TTS), [VITS](https://github.com/jaywalnut310/vits), and [VITS2](https://github.com/daniilrobnikov/vits2). Thanks for their awesome work!
@@ -4,6 +4,7 @@ import random
 import folder_paths
 import soundfile as sf

+from melo.api import TTS
 from .openvoice import se_extractor
 from .openvoice.api import BaseSpeakerTTS, ToneColorConverter

@@ -48,7 +49,7 @@ class OpenVoiceTTS:
        local_dir = os.path.join(folder_paths.models_dir, 'openovice')
        if not os.path.exists(local_dir) or not os.path.isdir(local_dir):
            from huggingface_hub import snapshot_download
-            snapshot_download(repo_id="Alignment-Lab-AI/OpenVoice", local_dir=local_dir, local_dir_use_symlinks=False)
+            snapshot_download(repo_id="myshell-ai/OpenVoice", local_dir=local_dir, local_dir_use_symlinks=False)
        
        mark = BaseSpeakerTTS.language_marks.get(lang.lower(), None)
        assert mark is not None, f"language {lang} is not supported"
@@ -89,6 +90,73 @@ class OpenVoiceTTS:
        audio_samples, sample_rate =sf.read(save_path)
        return (list(audio_samples), sample_rate)

+class OpenVoiceTTSV2:
+    @classmethod
+    def INPUT_TYPES(s):
+        audio_extensions = ["wav", "mp3", "flac"]
+        input_dir = folder_paths.get_input_directory()
+        files = []
+        for f in os.listdir(input_dir):
+            if os.path.isfile(os.path.join(input_dir, f)):
+                file_parts = f.lower().split('.')
+                if len(file_parts) > 1 and (file_parts[-1] in audio_extensions):
+                    files.append(f)
+        return {
+            "required": {
+                    "text": ("STRING", {"default": '', "multiline": True}),
+                    "lang": (["EN","EN_NEWEST","FR","JP","ES","ZH","KR"],),
+                    "speed": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.1}),
+                    "ref_voice": (sorted(files),),
+            },
+        }
+
+    CATEGORY = "OpenVoice"
+
+    RETURN_TYPES = (any, "INT",)
+    RETURN_NAMES = ("AUDIO", "SAMPLE_RATE",)
+    FUNCTION = "inference"
+
+    def inference(self, text, lang, speed, ref_voice):
+        local_dir = os.path.join(folder_paths.models_dir, 'openovice', 'checkpoints_v2')
+        if not os.path.exists(local_dir) or not os.path.isdir(local_dir):
+            from huggingface_hub import snapshot_download
+            snapshot_download(repo_id="myshell-ai/OpenVoiceV2", local_dir=local_dir, local_dir_use_symlinks=False)
+
+        device="cuda:0" if torch.cuda.is_available() else "cpu"
+        model = TTS(language=lang, device=device)
+        speaker_key, speaker_id = list(model.hps.data.spk2id.items())[0]
+
+        ckpt_base = os.path.join(local_dir, 'base_speakers')
+        ckpt_converter = os.path.join(local_dir, 'converter')
+
+        tone_color_converter = ToneColorConverter(f'{ckpt_converter}/config.json', device=device)
+        tone_color_converter.load_ckpt(f'{ckpt_converter}/checkpoint.pth')
+
+        source_se = torch.load(f'{ckpt_base}/ses/{speaker_key.lower()}.pth', map_location=device)
+        reference_speaker = os.path.join(folder_paths.get_input_directory(), ref_voice)
+        
+        temp_dir = folder_paths.get_temp_directory()
+        file_prefix = ''.join(random.choice("abcdefghijklmnopqrstupvxyz") for _ in range(5))
+        target_se, audio_name = se_extractor.get_se(reference_speaker, tone_color_converter, target_dir=temp_dir, vad=False)
+
+        save_path = f'{temp_dir}/{file_prefix}_output_v2.wav'
+
+        # Run the base speaker tts
+        src_path = f'{temp_dir}/{file_prefix}_base_v2.wav'
+        model.tts_to_file(text, speaker_id, src_path, speed=speed)
+
+        # Run the tone color converter
+        encode_message = "@MyShell"
+        tone_color_converter.convert(
+            audio_src_path=src_path,
+            src_se=source_se,
+            tgt_se=target_se,
+            output_path=save_path,
+            message=encode_message)
+
+        audio_samples, sample_rate =sf.read(save_path)
+        return (list(audio_samples), sample_rate)
+    
 class OpenVoiceSTS:
    @classmethod
    def INPUT_TYPES(s):
@@ -117,7 +185,7 @@ class OpenVoiceSTS:
        local_dir = os.path.join(folder_paths.models_dir, 'openovice')
        if not os.path.exists(local_dir) or not os.path.isdir(local_dir):
            from huggingface_hub import snapshot_download
-            snapshot_download(repo_id="Alignment-Lab-AI/OpenVoice", local_dir=local_dir, local_dir_use_symlinks=False)
+            snapshot_download(repo_id="myshell-ai/OpenVoice", local_dir=local_dir, local_dir_use_symlinks=False)

        ckpt_converter = os.path.join(local_dir, 'checkpoints/converter')
        device="cuda:0" if torch.cuda.is_available() else "cpu"
@@ -150,11 +218,13 @@ class OpenVoiceSTS:
    

 NODE_CLASS_MAPPINGS = {
-    "D_OpenVoice_TTS": OpenVoiceTTS,
-    "D_OpenVoice_STS": OpenVoiceSTS,
+    "D_OpenVoice_TTS"       : OpenVoiceTTS,
+    "D_OpenVoice_TTS_V2"    : OpenVoiceTTSV2,
+    "D_OpenVoice_STS"       : OpenVoiceSTS,
 }

 NODE_DISPLAY_NAME_MAPPINGS = {
-    "D_OpenVoice_TTS": "Open Voice TTS",
-    "D_OpenVoice_STS": "Open Voice STS",
+    "D_OpenVoice_TTS"       : "Open Voice TTS",
+    "D_OpenVoice_TTS_V2"    : "Open Voice TTS V2",
+    "D_OpenVoice_STS"       : "Open Voice STS",
 }