diff --git a/custom_nodes/audio_nodes.py b/custom_nodes/audio_nodes.py index 44933c2..a5ad7f4 100644 --- a/custom_nodes/audio_nodes.py +++ b/custom_nodes/audio_nodes.py @@ -14,14 +14,11 @@ CATEGORY = "🌺RVC-Studio/audio" input_path = folder_paths.get_input_directory() temp_path = folder_paths.get_temp_directory() -def to_audio_dict(audio, sr): - #from https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite/blob/bf2a9402d0b2727c7170c43621d569f4d531015f/videohelpersuite/nodes.py#L706C22-L706C67 - - waveform = torch.from_numpy(audio).reshape((-1,audio.ndim)).transpose(0,1).unsqueeze(0) - print(f"{waveform.shape=} {waveform.ndim=} {audio.shape=} {audio.ndim=}") +def to_audio_dict(audio: "np.ndarray", sr: int): + audio = np.atleast_2d(audio) + waveform = torch.from_numpy(audio.reshape((-1,audio.shape[0]))).transpose(0,1).unsqueeze(0) return dict(waveform=waveform,sample_rate=sr) - class LoadAudio: @classmethod def INPUT_TYPES(cls): diff --git a/lib/audio.py b/lib/audio.py index 9172f53..959e875 100644 --- a/lib/audio.py +++ b/lib/audio.py @@ -50,21 +50,24 @@ class AudioProcessor: return get_hash(*values) def __call__(self, audio): - from .karafan.audio_utils import Normalize, Silent - + samples,sr = get_audio(audio) if self.threshold_silence: + from .karafan.audio_utils import Silent samples = np.atleast_2d([samples]) samples = Silent(samples, sample_rate=sr, threshold_dB=self.silence_threshold_db) samples = np.squeeze(samples, 0) - if self.dynamic_threshold: samples = self.dynamic_thresholding( + if self.dynamic_threshold: + samples = self.dynamic_thresholding( samples, multiplier=self.multiplier, sample_size=self.sample_size, method=self.fill_method, kernel_size=self.kernel_size) - if self.normalize: samples = Normalize(samples,threshold_dB=self.normalize_threshold_db) + if self.normalize: + from .karafan.audio_utils import Normalize + samples = Normalize(samples,threshold_dB=self.normalize_threshold_db) return samples, sr @@ -297,4 +300,8 @@ def autotune_f0(f0, threshold=0.): # Use the nearest value in array2 autotuned_f0.append(y) # Return the result as a numpy array - return np.array(autotuned_f0, dtype="float32") \ No newline at end of file + return np.array(autotuned_f0, dtype="float32") + +def hz_to_mel(hz: "np.ndarray"): + # 1127 * np.log(1 + f0_min / 700) + return 2595 * np.log10(1 + hz / 700) # from wikipedia \ No newline at end of file diff --git a/lib/infer_pack/onnx_inference.py b/lib/infer_pack/onnx_inference.py index 6517853..3509fbb 100644 --- a/lib/infer_pack/onnx_inference.py +++ b/lib/infer_pack/onnx_inference.py @@ -1,7 +1,8 @@ import onnxruntime import librosa import numpy as np -import soundfile + +from ..audio import hz_to_mel class ContentVec: @@ -79,6 +80,7 @@ class OnnxRVC: self.model = onnxruntime.InferenceSession(model_path, providers=providers) self.sampling_rate = sr self.hop_size = hop_size + self.f0_bins = 256 def forward(self, hubert, hubert_length, pitch, pitchf, ds, rnd): onnx_input = { @@ -102,8 +104,8 @@ class OnnxRVC: ): f0_min = 50 f0_max = 1100 - f0_mel_min = 1127 * np.log(1 + f0_min / 700) - f0_mel_max = 1127 * np.log(1 + f0_max / 700) + f0_mel_min = hz_to_mel(f0_min) + f0_mel_max = hz_to_mel(f0_max) f0_predictor = get_f0_predictor( f0_method, hop_length=self.hop_size, @@ -125,12 +127,9 @@ class OnnxRVC: pitchf = f0_predictor.compute_f0(wav, hubert_length) pitchf = pitchf * 2 ** (f0_up_key / 12) pitch = pitchf.copy() - f0_mel = 1127 * np.log(1 + pitch / 700) - f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - f0_mel_min) * 254 / ( - f0_mel_max - f0_mel_min - ) + 1 - f0_mel[f0_mel <= 1] = 1 - f0_mel[f0_mel > 255] = 255 + f0_mel = hz_to_mel(pitch) + f0_mel = (f0_mel - f0_mel_min)*(self.f0_bins-2)/(f0_mel_max - f0_mel_min) + 1 + f0_mel = np.clip(f0_mel, a_min=1, a_max=self.f0_bins-1) pitch = np.rint(f0_mel).astype(np.int64) pitchf = pitchf.reshape(1, len(pitchf)).astype(np.float32) diff --git a/lib/rmvpe.py b/lib/rmvpe.py index 2223cbb..9262146 100644 --- a/lib/rmvpe.py +++ b/lib/rmvpe.py @@ -654,7 +654,8 @@ class RMVPE: if self.is_half == True: hidden = hidden.astype("float32") f0 = self.decode(hidden, thred=thred) - f0[(f0 < f0_min) | (f0 > f0_max)] = 0 + # f0[(f0 < f0_min) | (f0 > f0_max)] = 0 + f0 = np.clip(f0, a_min=f0_min, a_max=f0_max) return f0 def to_local_average_cents(self, salience, thred=0.05): diff --git a/pitch_extraction.py b/pitch_extraction.py index 88768b5..05729c6 100644 --- a/pitch_extraction.py +++ b/pitch_extraction.py @@ -6,7 +6,7 @@ from scipy import signal import torch from .lib.rmvpe import RMVPE -from .lib.audio import autotune_f0, pad_audio +from .lib.audio import autotune_f0, pad_audio, hz_to_mel from .lib import BASE_MODELS_DIR from .lib.utils import gc_collect, get_merge_func, get_optimal_threads, get_optimal_torch_device @@ -20,14 +20,15 @@ class FeatureExtractor: config.is_half, ) - self.sr = 16000 # hubert输入采样率 - self.window = 160 # 每帧点数 - self.t_pad = self.sr * self.x_pad # 每条前后pad时间 + self.sr = 16000 # hubert sr + self.window = 160 + self.f0_bins = 256 + self.t_pad = self.sr * self.x_pad self.t_pad_tgt = tgt_sr * self.x_pad self.t_pad2 = self.t_pad * 2 - self.t_query = self.sr * self.x_query # 查询切点前后查询时间 - self.t_center = self.sr * self.x_center # 查询切点位置 - self.t_max = self.sr * self.x_max # 免查询时长阈值 + self.t_query = self.sr * self.x_query + self.t_center = self.sr * self.x_center + self.t_max = self.sr * self.x_max self.device = config.device self.onnx = onnx self.f0_method_dict = { @@ -193,7 +194,7 @@ class FeatureExtractor: return self.model_rmvpe.infer_from_audio(x, thred=0.03) - def get_pitch_dependant_rmvpe(self, x, f0_min=1, f0_max=40000, *args, **kwargs): + def get_pitch_dependant_rmvpe(self, x, f0_min=0, f0_max=40000, *args, **kwargs): if not hasattr(self,"model_rmvpe"): self.model_rmvpe = RMVPE(os.path.join(BASE_MODELS_DIR,f"rmvpe.{'onnx' if self.onnx else 'pt'}"), is_half=self.is_half, device=self.device, onnx=self.onnx) @@ -262,8 +263,8 @@ class FeatureExtractor: **kwargs ): time_step = self.window / self.sr * 1000 - f0_mel_min = 1127 * np.log(1 + f0_min / 700) - f0_mel_max = 1127 * np.log(1 + f0_max / 700) + f0_mel_min = hz_to_mel(f0_min) + f0_mel_max = hz_to_mel(f0_max) params = {'x': x, 'f0_up_key': f0_up_key, 'f0_min': f0_min, 'f0_max': f0_max, 'time_step': time_step, 'filter_radius': filter_radius, 'crepe_hop_length': crepe_hop_length, 'model': "full", 'onnx': rmvpe_onnx @@ -295,12 +296,9 @@ class FeatureExtractor: ] # f0bak = f0.copy() - f0_mel = 1127 * np.log(1 + f0 / 700) - f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - f0_mel_min) * 254 / ( - f0_mel_max - f0_mel_min - ) + 1 - f0_mel[f0_mel <= 1] = 1 - f0_mel[f0_mel > 255] = 255 + f0_mel = hz_to_mel(f0) + f0_mel = (f0_mel - f0_mel_min)*(self.f0_bins-2)/(f0_mel_max - f0_mel_min) + 1 + f0_mel = np.clip(f0_mel, a_min=1, a_max=self.f0_bins-1) f0_coarse = np.rint(f0_mel).astype(np.int16) return f0_coarse, f0 # 1-0 \ No newline at end of file diff --git a/preprocessing_utils.py b/preprocessing_utils.py index b57451e..48ba084 100644 --- a/preprocessing_utils.py +++ b/preprocessing_utils.py @@ -5,7 +5,7 @@ from .lib.slicer2 import Slicer import traceback from scipy.io import wavfile from .pitch_extraction import FeatureExtractor -from .lib.audio import load_input_audio, remix_audio, AudioProcessor +from .lib.audio import hz_to_mel, load_input_audio, remix_audio, AudioProcessor from .lib.utils import gc_collect from .config import config import torch @@ -14,7 +14,7 @@ class Preprocess: def __init__(self, sr, exp_dir, preprocessor: "AudioProcessor"=None, noparallel=True, period=3.0, overlap=.3, max_volume=.95): self.slicer = Slicer( sr=sr, - threshold=-42, + threshold=-50, min_length=1500, min_interval=400, hop_size=15, @@ -112,8 +112,8 @@ class FeatureInput(FeatureExtractor): self.f0_bin = 256 self.f0_max = 1100.0 self.f0_min = 50.0 - self.f0_mel_min = 1127 * np.log(1 + self.f0_min / 700) - self.f0_mel_max = 1127 * np.log(1 + self.f0_max / 700) + self.f0_mel_min = hz_to_mel(self.f0_min) + self.f0_mel_max = hz_to_mel(self.f0_max) self.model = model