From 01bafea614374e6ec043b8f2cb7dbb81ef11a0c3 Mon Sep 17 00:00:00 2001 From: SpenserCai Date: Fri, 4 Oct 2024 20:15:17 +0800 Subject: [PATCH] update --- README.md | 6 ++- __init__.py | 16 ++++++ funaudio_utils/download_models.py | 12 ++++- nodes/cosyvoice_nodes.py | 81 +++++++++++++++++++++++++++---- 4 files changed, 103 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index bb7f84e..a6d6f4c 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ * @Date: 2024-10-04 11:28:28 * @version: * @LastEditors: SpenserCai - * @LastEditTime: 2024-10-04 18:25:48 + * @LastEditTime: 2024-10-04 20:13:48 * @Description: file content --> # ComfyUI-FunAudioLLM @@ -12,9 +12,11 @@ Comfyui custom node for [FunAudioLLM](https://funaudiollm.github.io/) include [C ## Features ### CosyVoice + - Support SFT,Zero-shot,Cross-lingual - Support CosyVoice-300M-25Hz in zero-shot and cross-lingual + - Support SFT's 25Hz(unoffical) -
- Save and load speaker model in zero-shot and cross-lingual + Save and load speaker model in zero-shot zh-CN
zh-CN
diff --git a/__init__.py b/__init__.py index 9002008..3d29db6 100644 --- a/__init__.py +++ b/__init__.py @@ -1,3 +1,11 @@ +''' +Author: SpenserCai +Date: 2024-10-04 12:14:22 +version: +LastEditors: SpenserCai +LastEditTime: 2024-10-04 20:14:52 +Description: file content +''' from .nodes.cosyvoice_nodes import * from .nodes.sensevoice_nodes import * @@ -6,6 +14,14 @@ NODE_CONFIG = { "class": CosyVoiceZeroShotNode, "name": "CosyVoice 3s极速克隆" }, + "CosyVoiceSFTNode": { + "class": CosyVoiceSFTNode, + "name": "CosyVoice 预训练音色" + }, + "CosyVoiceCrossLingualNode": { + "class": CosyVoiceCrossLingualNode, + "name": "CosyVoice 跨语言克隆" + }, "CosyVoiceSaveSpeakerModelNode": { "class": CosyVoiceSaveSpeakerModelNode, "name": "CosyVoice 保存说话人模型" diff --git a/funaudio_utils/download_models.py b/funaudio_utils/download_models.py index f7569b2..baeecbc 100644 --- a/funaudio_utils/download_models.py +++ b/funaudio_utils/download_models.py @@ -3,7 +3,7 @@ Author: SpenserCai Date: 2024-10-04 13:54:01 version: LastEditors: SpenserCai -LastEditTime: 2024-10-04 15:56:05 +LastEditTime: 2024-10-04 18:42:14 Description: file content ''' import modelscope @@ -24,5 +24,15 @@ def download_cosyvoice_300m(is_25hz=False): snapshot_download(model_id=model_id, local_dir=model_dir) return model_name, model_dir +def download_cosyvoice_300m_sft(is_25hz=False): + model_name = "CosyVoice-300M-SFT" + model_id = "iic/CosyVoice-300M-SFT" + if is_25hz: + model_name = "CosyVoice-300M-SFT-25Hz" + model_id = "MachineS/CosyVoice-300M-SFT-25Hz" + model_dir = os.path.join(base_cosyvoice_model_path, model_name) + snapshot_download(model_id=model_id, local_dir=model_dir) + return model_name, model_dir + def get_speaker_default_path(): return os.path.join(base_cosyvoice_model_path, "Speaker") \ No newline at end of file diff --git a/nodes/cosyvoice_nodes.py b/nodes/cosyvoice_nodes.py index 6c2f379..1b25590 100644 --- a/nodes/cosyvoice_nodes.py +++ b/nodes/cosyvoice_nodes.py @@ -3,7 +3,7 @@ Author: SpenserCai Date: 2024-10-04 12:13:28 version: LastEditors: SpenserCai -LastEditTime: 2024-10-04 17:25:20 +LastEditTime: 2024-10-04 20:13:31 Description: file content ''' import os @@ -12,8 +12,9 @@ import random import numpy as np import torch from funaudio_utils.pre import FunAudioLLMTool -from funaudio_utils.download_models import download_cosyvoice_300m, get_speaker_default_path +from funaudio_utils.download_models import download_cosyvoice_300m, get_speaker_default_path, download_cosyvoice_300m_sft from funaudio_utils.cosyvoice_plus import CosyVoicePlus +from cosyvoice.utils.common import set_all_random_seed fAudioTool = FunAudioLLMTool() @@ -21,12 +22,6 @@ CATEGORY_NAME = "FunAudioLLM - CosyVoice" folder_paths.add_model_folder_path("CosyVoice", os.path.join(folder_paths.models_dir, "CosyVoice")) -def set_all_random_seed(seed): - random.seed(seed) - np.random.seed(seed) - torch.manual_seed(seed) - torch.cuda.manual_seed_all(seed) - def return_audio(output,t0,spk_model): output_list = [] for out_dict in output: @@ -95,6 +90,74 @@ class CosyVoiceZeroShotNode: output = cosyvoice.inference_zero_shot_with_spkmodel(tts_text, speaker_model,False,speed) return return_audio(output,t0,speaker_model) +class CosyVoiceSFTNode: + sft_spk_list = ['中文女', '中文男', '日语男', '粤语女', '英文女', '英文男', '韩语女'] + + @classmethod + def INPUT_TYPES(s): + return { + "required":{ + "tts_text":("STRING",), + "speaker_name":(s.sft_spk_list,{ + "default":"中文女" + }), + "speed":("FLOAT",{ + "default": 1.0 + }), + "seed":("INT",{ + "default": 42 + }), + "use_25hz":("BOOLEAN",{ + "default": False + }), + } + } + + CATEGORY = CATEGORY_NAME + RETURN_TYPES = ("AUDIO",) + FUNCTION="generate" + + def generate(self, tts_text, speaker_name, speed, seed, use_25hz): + t0 = ttime() + _, model_dir = download_cosyvoice_300m_sft(use_25hz) + cosyvoice = CosyVoicePlus(model_dir) + set_all_random_seed(seed) + output = cosyvoice.inference_sft(tts_text, speaker_name, False, speed) + return return_audio(output,t0,None) + +class CosyVoiceCrossLingualNode: + @classmethod + def INPUT_TYPES(s): + return { + "required":{ + "tts_text":("STRING",), + "prompt_wav": ("AUDIO",), + "speed":("FLOAT",{ + "default": 1.0 + }), + "seed":("INT",{ + "default": 42 + }), + "use_25hz":("BOOLEAN",{ + "default": False + }), + } + } + + CATEGORY = CATEGORY_NAME + RETURN_TYPES = ("AUDIO",) + FUNCTION="generate" + + def generate(self, tts_text, prompt_wav, speed, seed, use_25hz): + t0 = ttime() + _, model_dir = download_cosyvoice_300m(use_25hz) + cosyvoice = CosyVoicePlus(model_dir) + speech = fAudioTool.audio_resample(prompt_wav["waveform"], prompt_wav["sample_rate"]) + prompt_speech_16k = fAudioTool.postprocess(speech) + set_all_random_seed(seed) + output = cosyvoice.inference_cross_lingual(tts_text, prompt_speech_16k, False, speed) + return return_audio(output,t0,None) + class CosyVoiceLoadSpeakerModelNode: @classmethod def INPUT_TYPES(s): @@ -112,7 +175,7 @@ class CosyVoiceLoadSpeakerModelNode: def generate(self, speaker_name, model_dir): # 加载模型 spk_model_path = os.path.join(model_dir, speaker_name + ".pt") - assert os.path.exists(spk_model_path), "模型文件不存在" + assert os.path.exists(spk_model_path), "Speaker model is not exist" spk_model = torch.load(os.path.join(model_dir, speaker_name + ".pt"),map_location=torch.device('cuda' if torch.cuda.is_available() else 'cpu')) return (spk_model,)