nodes/SpenserCai-ComfyUI-FunAudioLLM
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
SpenserCai
2024-10-04 20:15:17 +08:00
parent 7da957767a
commit 01bafea614
4 changed files with 103 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+4 -2
View File
@@ -3,7 +3,7 @@
* @Date: 2024-10-04 11:28:28
* @version:
* @LastEditors: SpenserCai
* @LastEditTime: 2024-10-04 18:25:48
* @LastEditTime: 2024-10-04 20:13:48
* @Description: file content
-->
# ComfyUI-FunAudioLLM
@@ -12,9 +12,11 @@ Comfyui custom node for [FunAudioLLM](https://funaudiollm.github.io/) include [C
## Features
### CosyVoice
- Support SFT,Zero-shot,Cross-lingual
- Support CosyVoice-300M-25Hz in zero-shot and cross-lingual
- Support SFT's 25Hz(unoffical)
- <details>
<summary>Save and load speaker model in zero-shot and cross-lingual</summary>
<summary>Save and load speaker model in zero-shot</summary>
<img src="./assets/SaveSpeakerModel.png" alt="zh-CN" /> <br>
<img src="./assets/LoadSpeakerModel.png" alt="zh-CN" />
</details>
__init__.py
+16
View File
@@ -1,3 +1,11 @@
'''
Author: SpenserCai
Date: 2024-10-04 12:14:22
version:
LastEditors: SpenserCai
LastEditTime: 2024-10-04 20:14:52
Description: file content
'''
from .nodes.cosyvoice_nodes import *
from .nodes.sensevoice_nodes import *
@@ -6,6 +14,14 @@ NODE_CONFIG = {
"class": CosyVoiceZeroShotNode,
"name": "CosyVoice 3s极速克隆"
},
"CosyVoiceSFTNode": {
"class": CosyVoiceSFTNode,
"name": "CosyVoice 预训练音色"
},
"CosyVoiceCrossLingualNode": {
"class": CosyVoiceCrossLingualNode,
"name": "CosyVoice 跨语言克隆"
},
"CosyVoiceSaveSpeakerModelNode": {
"class": CosyVoiceSaveSpeakerModelNode,
"name": "CosyVoice 保存说话人模型"
funaudio_utils/download_models.py
+11 -1
View File
@@ -3,7 +3,7 @@ Author: SpenserCai
Date: 2024-10-04 13:54:01
version:
LastEditors: SpenserCai
LastEditTime: 2024-10-04 15:56:05
LastEditTime: 2024-10-04 18:42:14
Description: file content
'''
import modelscope
@@ -24,5 +24,15 @@ def download_cosyvoice_300m(is_25hz=False):
snapshot_download(model_id=model_id, local_dir=model_dir)
return model_name, model_dir
def download_cosyvoice_300m_sft(is_25hz=False):
model_name = "CosyVoice-300M-SFT"
model_id = "iic/CosyVoice-300M-SFT"
if is_25hz:
model_name = "CosyVoice-300M-SFT-25Hz"
model_id = "MachineS/CosyVoice-300M-SFT-25Hz"
model_dir = os.path.join(base_cosyvoice_model_path, model_name)
snapshot_download(model_id=model_id, local_dir=model_dir)
return model_name, model_dir
def get_speaker_default_path():
return os.path.join(base_cosyvoice_model_path, "Speaker")
nodes/cosyvoice_nodes.py
+72 -9
View File
@@ -3,7 +3,7 @@ Author: SpenserCai
Date: 2024-10-04 12:13:28
version:
LastEditors: SpenserCai
LastEditTime: 2024-10-04 17:25:20
LastEditTime: 2024-10-04 20:13:31
Description: file content
'''
import os
@@ -12,8 +12,9 @@ import random
import numpy as np
import torch
from funaudio_utils.pre import FunAudioLLMTool
from funaudio_utils.download_models import download_cosyvoice_300m, get_speaker_default_path
from funaudio_utils.download_models import download_cosyvoice_300m, get_speaker_default_path, download_cosyvoice_300m_sft
from funaudio_utils.cosyvoice_plus import CosyVoicePlus
from cosyvoice.utils.common import set_all_random_seed
fAudioTool = FunAudioLLMTool()
@@ -21,12 +22,6 @@ CATEGORY_NAME = "FunAudioLLM - CosyVoice"
folder_paths.add_model_folder_path("CosyVoice", os.path.join(folder_paths.models_dir, "CosyVoice"))
def set_all_random_seed(seed):
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
def return_audio(output,t0,spk_model):
output_list = []
for out_dict in output:
@@ -95,6 +90,74 @@ class CosyVoiceZeroShotNode:
output = cosyvoice.inference_zero_shot_with_spkmodel(tts_text, speaker_model,False,speed)
return return_audio(output,t0,speaker_model)
class CosyVoiceSFTNode:
sft_spk_list = ['中文女', '中文男', '日语男', '粤语女', '英文女', '英文男', '韩语女']
@classmethod
def INPUT_TYPES(s):
return {
"required":{
"tts_text":("STRING",),
"speaker_name":(s.sft_spk_list,{
"default":"中文女"
}),
"speed":("FLOAT",{
"default": 1.0
}),
"seed":("INT",{
"default": 42
}),
"use_25hz":("BOOLEAN",{
"default": False
}),
}
}
CATEGORY = CATEGORY_NAME
RETURN_TYPES = ("AUDIO",)
FUNCTION="generate"
def generate(self, tts_text, speaker_name, speed, seed, use_25hz):
t0 = ttime()
_, model_dir = download_cosyvoice_300m_sft(use_25hz)
cosyvoice = CosyVoicePlus(model_dir)
set_all_random_seed(seed)
output = cosyvoice.inference_sft(tts_text, speaker_name, False, speed)
return return_audio(output,t0,None)
class CosyVoiceCrossLingualNode:
@classmethod
def INPUT_TYPES(s):
return {
"required":{
"tts_text":("STRING",),
"prompt_wav": ("AUDIO",),
"speed":("FLOAT",{
"default": 1.0
}),
"seed":("INT",{
"default": 42
}),
"use_25hz":("BOOLEAN",{
"default": False
}),
}
}
CATEGORY = CATEGORY_NAME
RETURN_TYPES = ("AUDIO",)
FUNCTION="generate"
def generate(self, tts_text, prompt_wav, speed, seed, use_25hz):
t0 = ttime()
_, model_dir = download_cosyvoice_300m(use_25hz)
cosyvoice = CosyVoicePlus(model_dir)
speech = fAudioTool.audio_resample(prompt_wav["waveform"], prompt_wav["sample_rate"])
prompt_speech_16k = fAudioTool.postprocess(speech)
set_all_random_seed(seed)
output = cosyvoice.inference_cross_lingual(tts_text, prompt_speech_16k, False, speed)
return return_audio(output,t0,None)
class CosyVoiceLoadSpeakerModelNode:
@classmethod
def INPUT_TYPES(s):
@@ -112,7 +175,7 @@ class CosyVoiceLoadSpeakerModelNode:
def generate(self, speaker_name, model_dir):
# 加载模型
spk_model_path = os.path.join(model_dir, speaker_name + ".pt")
assert os.path.exists(spk_model_path), "模型文件不存在"
assert os.path.exists(spk_model_path), "Speaker model is not exist"
spk_model = torch.load(os.path.join(model_dir, speaker_name + ".pt"),map_location=torch.device('cuda' if torch.cuda.is_available() else 'cpu'))
return (spk_model,)