added audioprocessor node + migrate get_audio + broken nodes fix
This commit is contained in:
SayanoAI
+65
-20
@@ -6,7 +6,7 @@ import yt_dlp
|
||||
import torch
|
||||
from .settings import MERGE_OPTIONS
|
||||
from .utils import MultipleTypeProxy, increment_filename_no_overwrite
|
||||
from ..lib.audio import MAX_INT16, SUPPORTED_AUDIO, audio_to_bytes, bytes_to_audio, load_input_audio, pad_audio, remix_audio, save_input_audio
|
||||
from ..lib.audio import MAX_INT16, SUPPORTED_AUDIO, AudioProcessor, audio_to_bytes, get_audio, load_input_audio, pad_audio, remix_audio, save_input_audio
|
||||
from ..lib.utils import get_filenames, get_hash, get_merge_func
|
||||
import folder_paths
|
||||
|
||||
@@ -14,12 +14,6 @@ CATEGORY = "🌺RVC-Studio/audio"
|
||||
input_path = folder_paths.get_input_directory()
|
||||
temp_path = folder_paths.get_temp_directory()
|
||||
|
||||
def get_audio(audio):
|
||||
if hasattr(audio,"__call__"): #VHS_AUDIO is a function
|
||||
return bytes_to_audio(audio())
|
||||
else: #comfyui AUDIO is a dict
|
||||
return audio["waveform"].squeeze(0).transpose(0,1).numpy(), audio["sample_rate"]
|
||||
|
||||
def to_audio_dict(audio, sr):
|
||||
#from https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite/blob/bf2a9402d0b2727c7170c43621d569f4d531015f/videohelpersuite/nodes.py#L706C22-L706C67
|
||||
|
||||
@@ -112,9 +106,6 @@ class DownloadAudio:
|
||||
return {"ui": {"preview": [{"filename": os.path.basename(audio_path), "type": "input", "subfolder": "audio", "widgetId": widgetId}]}, "result": (audio_name, lambda:audio_to_bytes(*input_audio),to_audio_dict(*input_audio))}
|
||||
|
||||
class MergeAudioNode:
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
@@ -163,8 +154,6 @@ class MergeAudioNode:
|
||||
return {"ui": {"preview": [{"filename": audio_name, "type": "temp", "subfolder": "preview", "widgetId": widgetId}]}, "result": (lambda: audio_to_bytes(*merged_audio),to_audio_dict(*merged_audio))}
|
||||
|
||||
class PreviewAudio:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
@@ -215,23 +204,78 @@ class PreviewAudio:
|
||||
"filename": audio_name, "type": "temp", "subfolder": "preview", "widgetId": widgetId, "autoplay": autoplay
|
||||
}]}, "result": (output_path, lambda:audio_to_bytes(*input_audio), to_audio_dict(*input_audio))}
|
||||
|
||||
|
||||
class ProcessAudioNode:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"normalize": ("BOOLEAN",{"default": True}),
|
||||
"threshold_silence": ("BOOLEAN",{"default": True}),
|
||||
"dynamic_threshold": ("BOOLEAN",{"default": True})
|
||||
},
|
||||
"optional": {
|
||||
"audio": (MultipleTypeProxy('AUDIO,VHS_AUDIO'),{"default": None}),
|
||||
"dynamic_threshold_sample_size": ("INT",{"default": 4000, "min": 160, "max": 48000, "step": 160}),
|
||||
"dynamic_threshold_multiplier": ("FLOAT",{"default": 2.}),
|
||||
"dynamic_threshold_fill_method": (["median","interpolation"],{"default": "median"}),
|
||||
"dynamic_threshold_kernel_size": ("INT",{"default": 5}),
|
||||
"silence_threshold_db": ("INT",{"default": -50, "min": -120, "max": 0}),
|
||||
"normalize_threshold_db": ("INT",{"default": -1, "min": -10, "max": 0})
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("AUDIO_PROCESSOR","VHS_AUDIO","AUDIO")
|
||||
RETURN_NAMES = ("audio_processor","vhs_audio","audio")
|
||||
|
||||
CATEGORY = CATEGORY
|
||||
|
||||
FUNCTION = "process_audio"
|
||||
|
||||
def process_audio(self, normalize, threshold_silence, dynamic_threshold, audio=None,
|
||||
dynamic_threshold_sample_size=16000,
|
||||
dynamic_threshold_multiplier=2.0,
|
||||
dynamic_threshold_fill_method="median",
|
||||
dynamic_threshold_kernel_size=5,
|
||||
silence_threshold_db=-50,
|
||||
normalize_threshold_db=-1):
|
||||
|
||||
audio_processor = AudioProcessor(
|
||||
normalize=normalize,
|
||||
threshold_silence=threshold_silence,
|
||||
dynamic_threshold=dynamic_threshold,
|
||||
sample_size=dynamic_threshold_sample_size,
|
||||
multiplier=dynamic_threshold_multiplier,
|
||||
fill_method=dynamic_threshold_fill_method,
|
||||
kernel_size=dynamic_threshold_kernel_size,
|
||||
silence_threshold_db=silence_threshold_db,
|
||||
normalize_threshold_db=normalize_threshold_db)
|
||||
|
||||
if audio is None:
|
||||
vhs_audio = comfy_audio = audio
|
||||
else:
|
||||
output_audio = audio_processor(audio)
|
||||
vhs_audio = lambda:audio_to_bytes(*output_audio)
|
||||
comfy_audio = to_audio_dict(*output_audio)
|
||||
|
||||
return ( audio_processor, vhs_audio, comfy_audio)
|
||||
|
||||
|
||||
class AudioBatchValueNode:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"audio": (MultipleTypeProxy('AUDIO,VHS_AUDIO'),),
|
||||
"num_segments": ('INT', {"min": 2, "max": 128, "step": 1, "forceInput": True}),
|
||||
"num_segments": ('INT', {"default": 2, "min": 2, "max": 256, "step": 1}),
|
||||
"output_min": ('FLOAT', {'default': 0., "min": -1000., "max": 1000., "step": .01}),
|
||||
"output_max": ('FLOAT', {'default': 1., "min": 0., "max": 1000., "step": .01}),
|
||||
"norm": (["scale","tanh","sigmoid"], {"default": "scale"}),
|
||||
},
|
||||
"optional": {
|
||||
"silence_threshold": ("INT", {"default": 1000, "min": 1, "max": MAX_INT16, "step": 1, "display": "slider"}),
|
||||
"duration_list": ("INT", {"default": None, "min": 1, "forceInput": True}),
|
||||
"silence_threshold": ("INT", {"default": 1000, "min": 1, "max": MAX_INT16, "step": 1}),
|
||||
"duration_list": ("INT", {"default": 0, "min": 0, "forceInput": True}),
|
||||
"print_output": ("BOOLEAN", {"default": False}),
|
||||
"inverse": ("BOOLEAN", {"default": False}),
|
||||
}
|
||||
@@ -248,8 +292,7 @@ class AudioBatchValueNode:
|
||||
def get_rms(audio): # root mean squared of audio segment
|
||||
return np.sqrt(np.nanmean(audio**2))
|
||||
|
||||
def get_frame_weights(self, audio, num_segments, output_min, output_max, norm,
|
||||
silence_threshold=1000, duration_list=None, print_output=False, inverse=False):
|
||||
def get_frame_weights(self, audio, num_segments, output_min, output_max, norm, silence_threshold=1000, duration_list=0, print_output=False, inverse=False):
|
||||
assert output_max>=output_min, f"{output_max=} must be greater or equal to {output_min=}!"
|
||||
|
||||
audio_data = get_audio(audio)
|
||||
@@ -277,7 +320,7 @@ class AudioBatchValueNode:
|
||||
print(f"{audio_rms.min()=} {audio_rms.max()=} {audio_rms.mean()=} {len(audio_rms)=}")
|
||||
print(f"{x_norm.min()=} {x_norm.max()=} {x_norm.mean()=} {len(x_norm)=}")
|
||||
|
||||
if duration_list is not None:
|
||||
if isinstance(duration_list,list):
|
||||
segments = np.cumsum(duration_list)
|
||||
x_norm = np.array_split(x_norm,segments)
|
||||
x_norm = map(list,x_norm)
|
||||
@@ -293,6 +336,7 @@ NODE_CLASS_MAPPINGS = {
|
||||
"MergeAudioNode": MergeAudioNode,
|
||||
"AudioBatchValueNode": AudioBatchValueNode,
|
||||
"DownloadAudio": DownloadAudio,
|
||||
"ProcessAudioNode": ProcessAudioNode
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
@@ -301,4 +345,5 @@ NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"RVC-Studio.PreviewAudio": "🌺Save Audio",
|
||||
"MergeAudioNode": "🌺Merge Audio",
|
||||
"AudioBatchValueNode": "🌺Audio RMS Batch Values",
|
||||
"ProcessAudioNode": "🌺Audio Processor"
|
||||
}
|
||||
@@ -6,10 +6,9 @@ import cv2
|
||||
import copy
|
||||
from tqdm import tqdm
|
||||
|
||||
from ..lib.audio import remix_audio
|
||||
from ..lib.audio import get_audio, remix_audio
|
||||
from ..lib import BASE_CACHE_DIR
|
||||
from ..lib.utils import gc_collect, get_hash, get_optimal_torch_device
|
||||
from .audio_nodes import get_audio
|
||||
from .utils import MultipleTypeProxy, model_downloader
|
||||
from ..lib.musetalk.models.unet import PositionalEncoding, UNet
|
||||
from ..lib.musetalk.models.vae import VAE
|
||||
|
||||
@@ -14,7 +14,6 @@ from ..training_cli import train_model
|
||||
|
||||
from ..preprocessing_utils import extract_features_trainset, preprocess_trainset
|
||||
|
||||
from .audio_nodes import get_audio
|
||||
from ..config import config
|
||||
|
||||
from ..lib.model_utils import load_hubert
|
||||
@@ -23,7 +22,7 @@ from .utils import MultipleTypeProxy, increment_filename_no_overwrite, model_dow
|
||||
from .settings import PITCH_EXTRACTION_OPTIONS, SR_MAP
|
||||
from .settings.downloader import PRETRAINED_MODELS, RVC_DOWNLOAD_LINK, RVC_INDEX, RVC_MODELS, download_file, extract_zip_without_structure
|
||||
|
||||
from ..lib.audio import SUPPORTED_AUDIO, audio_to_bytes, load_input_audio, save_input_audio
|
||||
from ..lib.audio import SUPPORTED_AUDIO, audio_to_bytes, load_input_audio, save_input_audio, get_audio
|
||||
|
||||
from ..vc_infer_pipeline import get_vc, vc_single
|
||||
import folder_paths
|
||||
@@ -237,6 +236,7 @@ class RVCProcessDatasetNode:
|
||||
"overlap": ("FLOAT",{"default": .3, "min": .1, "max": 1., "step": .1}),
|
||||
"max_volume": ("FLOAT",{"default": .99, "min": .1, "max": 1., "step": .01}),
|
||||
"mute_ratio": ("FLOAT",{"default": .0, "min": .0, "max": .5, "step": .01}),
|
||||
"audio_processor": ("AUDIO_PROCESSOR",)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -247,13 +247,13 @@ class RVCProcessDatasetNode:
|
||||
|
||||
CATEGORY = CATEGORY
|
||||
|
||||
def process(self, model_name: str, dataset: str, hubert_model, pitch_extraction_params={}, sr="40k", n_threads=1, period=3., overlap=.3, max_volume=1., mute_ratio=.0):
|
||||
def process(self, model_name: str, dataset: str, hubert_model, pitch_extraction_params={}, sr="40k", n_threads=1, period=3., overlap=.3, max_volume=1., mute_ratio=.0, audio_processor=None):
|
||||
|
||||
assert model_name, "Please provide a model name!"
|
||||
assert dataset, "Please upload a dataset!"
|
||||
|
||||
f0_method = pitch_extraction_params.get("f0_method", "")
|
||||
cached_params = [model_name, dataset, period, overlap, max_volume, mute_ratio, sr, f0_method]
|
||||
cached_params = [model_name, dataset, period, overlap, max_volume, mute_ratio, sr, f0_method, audio_processor]
|
||||
crepe_hop_length = pitch_extraction_params.get("crepe_hop_length",160)
|
||||
|
||||
if "crepe" in f0_method: cached_params.append(crepe_hop_length)
|
||||
@@ -271,9 +271,9 @@ class RVCProcessDatasetNode:
|
||||
files = extract_zip_without_structure(os.path.join(dataset_path,dataset),dataset_dir)
|
||||
assert len(files), "Failed to extract zip file..."
|
||||
|
||||
print(preprocess_trainset(dataset_dir,SR_MAP[sr],n_threads,model_log_dir,period,overlap,max_volume))
|
||||
assert preprocess_trainset(dataset_dir,SR_MAP[sr],n_threads,model_log_dir,audio_processor,period,overlap,max_volume), "Failed to preprocess audio..."
|
||||
|
||||
print(extract_features_trainset(hubert_model(), model_log_dir,n_p=n_threads,f0method=f0_method,device=device,if_f0=bool(f0_method),version="v2",crepe_hop_length=crepe_hop_length))
|
||||
assert extract_features_trainset(hubert_model(), model_log_dir,n_p=n_threads,f0method=f0_method,device=device,if_f0=bool(f0_method),version="v2",crepe_hop_length=crepe_hop_length), "Failed to extract features..."
|
||||
|
||||
gt_wavs_dir = os.path.join(model_log_dir,"0_gt_wavs")
|
||||
feature_dir = os.path.join(model_log_dir,"3_feature768")
|
||||
@@ -377,7 +377,7 @@ class RVCTrainModelNode:
|
||||
},
|
||||
"optional": dict(
|
||||
gpu=(DEVICES, {"default": DEVICES[0]}),
|
||||
batch_size=("INT",dict(default=4,min=1,max=64)),
|
||||
batch_size=("INT",dict(default=4,min=0,max=64,step=4)),
|
||||
total_epoch=("INT",dict(default=100,min=10,max=1000,step=10)),
|
||||
save_every_epoch=("INT",dict(default=0,min=0,max=100)),
|
||||
pretrained_G=(PRETRAINED_G,{"default": PRETRAINED_G[0]}),
|
||||
@@ -433,7 +433,7 @@ class RVCTrainModelNode:
|
||||
hparams.pretrainD = model_downloader(pretrained_D)
|
||||
hparams.version = "v2"
|
||||
hparams.gpus = gpu
|
||||
hparams.train.batch_size = batch_size
|
||||
hparams.train.batch_size = max(batch_size,1)
|
||||
hparams.sample_rate = sample_rate
|
||||
hparams.if_f0 = if_f0
|
||||
hparams.if_latest = if_save_latest
|
||||
@@ -443,7 +443,7 @@ class RVCTrainModelNode:
|
||||
hparams.save_best_model = save_best_model
|
||||
|
||||
file_index = self.train_index(model_dir, sample_rate, name) if train_index else None
|
||||
model_path = os.path.join(BASE_MODELS_DIR,"RVC",f"{name}.pth")
|
||||
model_path = os.path.join(BASE_MODELS_DIR,"RVC",f"{name}_{sample_rate}.pth")
|
||||
if os.path.isfile(model_path) and retrain: model_path = increment_filename_no_overwrite(model_path)
|
||||
hparams.model_path = model_path
|
||||
|
||||
|
||||
+5
-5
@@ -263,11 +263,11 @@ class BatchedTranscriptionEncoderNode:
|
||||
"use_tags": ('BOOLEAN', {'default': False}),
|
||||
"use_sentiment": ('BOOLEAN', {'default': False}),
|
||||
'language': (SUPPORTED_LANGUAGES,{"default": "en"}),
|
||||
"max_chunks": ('INT', {"min": 2, "default": None}),
|
||||
"max_words": ('INT', {'default': 16, "min": 0, "max": 32, "display": "slider"}),
|
||||
"max_chunks": ('INT', {"min": 0, "default": 0}),
|
||||
"max_words": ('INT', {'default': 16, "min": 0, "max": 32}),
|
||||
"frame_interpolation": ("INT", {"default": 0, "min": 0, "max": 120, "hidden": True}), # needs more testing
|
||||
"prefix": ("STRING", {"default": "masterpiece, best quality", "multiline": True, "forceInput": True}),
|
||||
"suffix": ("STRING", {"default": "", "multiline": True, "forceInput": True}),
|
||||
"prefix": ("STRING", {"default": "masterpiece, best quality", "multiline": True}),
|
||||
"suffix": ("STRING", {"default": "", "multiline": True}),
|
||||
"print_output": ('BOOLEAN', {'default': True}),
|
||||
"weights": ("FLOAT",{"default": 1., "step": .01}),
|
||||
"pad_frames": ("INT", {"default": 0})
|
||||
@@ -299,7 +299,7 @@ class BatchedTranscriptionEncoderNode:
|
||||
|
||||
def get_prompt(
|
||||
self, transcription, clip, language="en", loop=False, use_tags=False, use_sentiment=False,
|
||||
max_words=16,max_chunks=None, frame_interpolation=0, print_output=True, prefix="", suffix="",
|
||||
max_words=16,max_chunks=0, frame_interpolation=0, print_output=True, prefix="", suffix="",
|
||||
weights=1., pad_frames=0
|
||||
):
|
||||
|
||||
|
||||
+1
-3
@@ -1,10 +1,8 @@
|
||||
import os
|
||||
import audio_separator.separator as uvr
|
||||
|
||||
from .audio_nodes import get_audio
|
||||
|
||||
from .utils import MultipleTypeProxy
|
||||
from ..lib.audio import audio_to_bytes, save_input_audio, load_input_audio
|
||||
from ..lib.audio import audio_to_bytes, save_input_audio, load_input_audio, get_audio
|
||||
import folder_paths
|
||||
from ..lib.utils import get_filenames, get_hash, get_optimal_torch_device
|
||||
from ..lib import BASE_CACHE_DIR, BASE_MODELS_DIR, karafan
|
||||
|
||||
+89
-1
@@ -2,11 +2,13 @@ import base64
|
||||
import io
|
||||
import os
|
||||
import zlib
|
||||
from .utils import get_merge_func
|
||||
from .utils import get_hash, get_merge_func
|
||||
import numpy as np
|
||||
import librosa
|
||||
import soundfile as sf
|
||||
import ffmpeg
|
||||
from scipy.ndimage import uniform_filter1d, median_filter
|
||||
from scipy.interpolate import interp1d
|
||||
|
||||
MAX_INT16 = 32768
|
||||
SUPPORTED_AUDIO = ["mp3","flac","wav"] # ogg breaks soundfile
|
||||
@@ -26,6 +28,92 @@ AUTOTUNE_NOTES = np.array([
|
||||
2959.96, 3135.96, 3322.44, 3520.00, 3729.31, 3951.07
|
||||
])
|
||||
|
||||
class AudioProcessor:
|
||||
def __init__(self, normalize=True, threshold_silence=True, dynamic_threshold=True, sample_size=16000, multiplier=2.0, fill_method="median", kernel_size=5, silence_threshold_db=-50, normalize_threshold_db=-1):
|
||||
self.normalize=normalize
|
||||
self.threshold_silence=threshold_silence
|
||||
self.dynamic_threshold=dynamic_threshold
|
||||
self.sample_size=sample_size
|
||||
self.multiplier=multiplier
|
||||
self.fill_method=fill_method
|
||||
self.kernel_size=kernel_size
|
||||
self.silence_threshold_db=silence_threshold_db
|
||||
self.normalize_threshold_db=normalize_threshold_db
|
||||
|
||||
def __str__(self) -> str:
|
||||
values = [self.normalize, self.threshold_silence, self.dynamic_threshold]
|
||||
if self.normalize: values.append(self.normalize_threshold_db)
|
||||
if self.threshold_silence: values.append(self.silence_threshold_db)
|
||||
if self.dynamic_threshold: values.extend([self.sample_size, self.multiplier, self.fill_method, self.kernel_size])
|
||||
return get_hash(*values)
|
||||
|
||||
def __call__(self, audio):
|
||||
from .karafan.audio_utils import Normalize, Silent
|
||||
|
||||
samples,sr = get_audio(audio)
|
||||
|
||||
if self.threshold_silence:
|
||||
samples = np.atleast_2d([samples])
|
||||
samples = Silent(samples, sample_rate=sr, threshold_dB=self.silence_threshold_db)
|
||||
samples = np.squeeze(samples, 0)
|
||||
if self.dynamic_threshold: samples = self.dynamic_thresholding(
|
||||
samples,
|
||||
multiplier=self.multiplier,
|
||||
sample_size=self.sample_size,
|
||||
method=self.fill_method,
|
||||
kernel_size=self.kernel_size)
|
||||
if self.normalize: samples = Normalize(samples,threshold_dB=self.normalize_threshold_db)
|
||||
|
||||
return samples, sr
|
||||
|
||||
@staticmethod
|
||||
def dynamic_thresholding(samples, multiplier=2., sample_size=16000, method="median", kernel_size=5):
|
||||
# Calculate the local RMS (root mean square) to estimate the local amplitude
|
||||
local_rms = np.sqrt(uniform_filter1d(np.square(samples), size=int(sample_size)))
|
||||
|
||||
# Set dynamic threshold as a multiple of the local RMS
|
||||
dynamic_threshold = multiplier * local_rms
|
||||
|
||||
# Detect clicks where the absolute value exceeds the dynamic threshold
|
||||
clicks = np.abs(samples) > dynamic_threshold
|
||||
|
||||
# Replace clicks using the specified method
|
||||
cleaned_samples = AudioProcessor.replace_clicks(samples, clicks, method=method, kernel_size=kernel_size)
|
||||
return cleaned_samples
|
||||
|
||||
@staticmethod
|
||||
def replace_clicks(samples, clicks, method="median", kernel_size=5):
|
||||
if method == "median":
|
||||
# Replace detected clicks with the median value of surrounding samples
|
||||
cleaned_samples = samples.copy()
|
||||
cleaned_samples[clicks] = median_filter(samples, size=kernel_size)[clicks]
|
||||
elif method == "interpolation":
|
||||
# Replace detected clicks with interpolation
|
||||
cleaned_samples = samples.copy()
|
||||
|
||||
# Indices of non-clicks (valid points for interpolation)
|
||||
non_click_indices = np.where(~clicks)[0]
|
||||
|
||||
# Indices of clicks (points to be replaced)
|
||||
click_indices = np.where(clicks)[0]
|
||||
|
||||
# Interpolation function
|
||||
interp_func = interp1d(non_click_indices, cleaned_samples[non_click_indices], kind='linear', bounds_error=False, fill_value="extrapolate")
|
||||
|
||||
# Replace click samples with interpolated values
|
||||
cleaned_samples[click_indices] = interp_func(click_indices)
|
||||
else:
|
||||
raise ValueError("Method must be 'median' or 'interpolation'")
|
||||
|
||||
return cleaned_samples
|
||||
|
||||
def get_audio(audio):
|
||||
if hasattr(audio,"__call__"): #VHS_AUDIO is a function
|
||||
return bytes_to_audio(audio())
|
||||
elif isinstance(audio,dict): #comfyui AUDIO is a dict
|
||||
return audio["waveform"].squeeze(0).transpose(0,1).numpy(), audio["sample_rate"]
|
||||
else: return audio
|
||||
|
||||
def load_audio(file, sr, **kwargs):
|
||||
try:
|
||||
# https://github.com/openai/whisper/blob/main/whisper/audio.py#L26
|
||||
|
||||
+31
-49
@@ -124,7 +124,7 @@ def Silent(audio_in, sample_rate, threshold_dB = -50):
|
||||
|
||||
start = 0; end = 0
|
||||
audio = audio_in.copy()
|
||||
audio_length = audio_in.shape[1]
|
||||
audio_length = audio_in.shape[-1]
|
||||
|
||||
for i in range(0, audio_length, window_frame):
|
||||
|
||||
@@ -241,71 +241,53 @@ def Change_sample_rate(audio, way, current_cutoff, target_cutoff):
|
||||
|
||||
pitched_audio = librosa.resample(audio, orig_sr = current_cutoff * 2, target_sr = target_cutoff * 2, res_type = 'kaiser_best', axis=1)
|
||||
|
||||
# print(f"SRS input audio shape: {audio.shape}")
|
||||
# print(f"SRS output audio shape: {pitched_audio.shape}")
|
||||
# print (f"ratio : {ratio}")
|
||||
|
||||
return pitched_audio
|
||||
|
||||
# def Remove_High_freq_Noise(audio, threshold_freq):
|
||||
def Remove_High_freq_Noise(audio, threshold_freq):
|
||||
|
||||
# # Calculer la transformée de Fourier
|
||||
# stft = librosa.stft(audio)
|
||||
# Calculer la transformée de Fourier
|
||||
stft = librosa.stft(audio)
|
||||
|
||||
# # Calculer la somme des amplitudes pour chaque fréquence dans le spectre
|
||||
# amplitude_sum = np.sum(np.abs(stft), axis=0)
|
||||
# Calculer la somme des amplitudes pour chaque fréquence dans le spectre
|
||||
amplitude_sum = np.sum(np.abs(stft), axis=0)
|
||||
|
||||
# # Appliquer un masque pour supprimer les fréquences supérieures lorsque la somme des amplitudes est inférieure au seuil
|
||||
# stft[:, amplitude_sum > threshold_freq] = 0.0
|
||||
# Appliquer un masque pour supprimer les fréquences supérieures lorsque la somme des amplitudes est inférieure au seuil
|
||||
stft[:, amplitude_sum > threshold_freq] = 0.0
|
||||
|
||||
# # Reconstruire l'audio à partir du STFT modifié
|
||||
# audio_filtered = librosa.istft(stft)
|
||||
# Reconstruire l'audio à partir du STFT modifié
|
||||
audio_filtered = librosa.istft(stft)
|
||||
|
||||
# return audio_filtered
|
||||
return audio_filtered
|
||||
|
||||
# def Match_Freq_CutOFF(self, audio1, audio2, sample_rate):
|
||||
# # This option matches the Primary stem frequency cut-off to the Secondary stem frequency cut-off
|
||||
# # (if the Primary stem frequency cut-off is lower than the Secondary stem frequency cut-off)
|
||||
# Find the high cut-off frequency of the input audio
|
||||
def Find_Cut_OFF(audio, sample_rate, threshold=0.01):
|
||||
|
||||
# # Get the Primary stem frequency cut-off
|
||||
# freq_cut_off1 = Find_Cut_OFF(audio1, sample_rate)
|
||||
# freq_cut_off2 = Find_Cut_OFF(audio2, sample_rate)
|
||||
# Appliquer un filtre passe-bas pour réduire le bruit
|
||||
cutoff_frequency = sample_rate / 2.0 # Fréquence de Nyquist (la moitié du taux d'échantillonnage)
|
||||
|
||||
# # Match the Primary stem frequency cut-off to the Secondary stem frequency cut-off
|
||||
# if freq_cut_off1 < freq_cut_off2:
|
||||
# audio1 = Resize_Freq_CutOFF(audio1, freq_cut_off2, sample_rate)
|
||||
# Définir l'ordre du filtre passe-bas
|
||||
order = 6
|
||||
|
||||
# return audio1
|
||||
# Calculer les coefficients du filtre passe-bas
|
||||
b, a = signal.butter(order, cutoff_frequency - threshold, btype='low', analog=False, fs=sample_rate)
|
||||
|
||||
# # Find the high cut-off frequency of the input audio
|
||||
# def Find_Cut_OFF(audio, sample_rate, threshold=0.01):
|
||||
# Appliquer le filtre au signal audio
|
||||
filtered_audio = signal.lfilter(b, a, audio, axis=0)
|
||||
|
||||
# # Appliquer un filtre passe-bas pour réduire le bruit
|
||||
# cutoff_frequency = sample_rate / 2.0 # Fréquence de Nyquist (la moitié du taux d'échantillonnage)
|
||||
# Calculer la FFT du signal audio filtré
|
||||
fft_result = np.fft.fft(filtered_audio, axis=0)
|
||||
|
||||
# # Définir l'ordre du filtre passe-bas
|
||||
# order = 6
|
||||
# Calculer les magnitudes du spectre de fréquence
|
||||
magnitudes = np.abs(fft_result)
|
||||
|
||||
# # Calculer les coefficients du filtre passe-bas
|
||||
# b, a = signal.butter(order, cutoff_frequency - threshold, btype='low', analog=False, fs=sample_rate)
|
||||
# Calculer les fréquences correspondant aux bins de la FFT
|
||||
frequencies = np.fft.fftfreq(len(audio), 1.0 / sample_rate)
|
||||
|
||||
# # Appliquer le filtre au signal audio
|
||||
# filtered_audio = signal.lfilter(b, a, audio, axis=0)
|
||||
# Trouver la fréquence de coupure où la magnitude tombe en dessous du seuil
|
||||
cut_off_frequencies = frequencies[np.where(magnitudes > threshold)]
|
||||
|
||||
# # Calculer la FFT du signal audio filtré
|
||||
# fft_result = np.fft.fft(filtered_audio, axis=0)
|
||||
|
||||
# # Calculer les magnitudes du spectre de fréquence
|
||||
# magnitudes = np.abs(fft_result)
|
||||
|
||||
# # Calculer les fréquences correspondant aux bins de la FFT
|
||||
# frequencies = np.fft.fftfreq(len(audio), 1.0 / sample_rate)
|
||||
|
||||
# # Trouver la fréquence de coupure où la magnitude tombe en dessous du seuil
|
||||
# cut_off_frequencies = frequencies[np.where(magnitudes > threshold)]
|
||||
|
||||
# # Trouver la fréquence de coupure maximale parmi toutes les valeurs
|
||||
# return int(max(cut_off_frequencies))
|
||||
# Trouver la fréquence de coupure maximale parmi toutes les valeurs
|
||||
return int(max(cut_off_frequencies))
|
||||
|
||||
|
||||
|
||||
|
||||
+21
-14
@@ -5,13 +5,13 @@ from .lib.slicer2 import Slicer
|
||||
import traceback
|
||||
from scipy.io import wavfile
|
||||
from .pitch_extraction import FeatureExtractor
|
||||
from .lib.audio import load_input_audio, remix_audio
|
||||
from .lib.audio import load_input_audio, remix_audio, AudioProcessor
|
||||
from .lib.utils import gc_collect
|
||||
from .config import config
|
||||
import torch
|
||||
|
||||
class Preprocess:
|
||||
def __init__(self, sr, exp_dir, noparallel=True, period=3.0, overlap=.3, max_volume=.95):
|
||||
def __init__(self, sr, exp_dir, preprocessor: "AudioProcessor"=None, noparallel=True, period=3.0, overlap=.3, max_volume=.95):
|
||||
self.slicer = Slicer(
|
||||
sr=sr,
|
||||
threshold=-42,
|
||||
@@ -29,6 +29,7 @@ class Preprocess:
|
||||
self.gt_wavs_dir = os.path.join(exp_dir,"0_gt_wavs")
|
||||
self.wavs16k_dir = os.path.join(exp_dir,"1_16k_wavs")
|
||||
self.noparallel = noparallel
|
||||
self.preprocessor = preprocessor
|
||||
os.makedirs(self.exp_dir, exist_ok=True)
|
||||
os.makedirs(self.gt_wavs_dir, exist_ok=True)
|
||||
os.makedirs(self.wavs16k_dir, exist_ok=True)
|
||||
@@ -42,16 +43,19 @@ class Preprocess:
|
||||
# mutex.release()
|
||||
|
||||
def norm_write(self, tmp_audio, idx0, idx1):
|
||||
wavfile.write(os.path.join(self.gt_wavs_dir, f"{idx0}_{idx1}.wav"),self.sr,tmp_audio.astype(np.float32))
|
||||
remixed_audio = remix_audio((tmp_audio, self.sr), target_sr=16000, norm=True,max_volume=self.max_volume)
|
||||
wavfile.write(os.path.join(self.wavs16k_dir, f"{idx0}_{idx1}.wav"),16000,remixed_audio[0].astype(np.float32))
|
||||
if len(tmp_audio) > self.overlap*self.sr*2:
|
||||
wavfile.write(os.path.join(self.gt_wavs_dir, f"{idx0}_{idx1}.wav"),self.sr,tmp_audio.astype(np.float32))
|
||||
remixed_audio = remix_audio((tmp_audio, self.sr), target_sr=16000, max_volume=self.max_volume)
|
||||
wavfile.write(os.path.join(self.wavs16k_dir, f"{idx0}_{idx1}.wav"),16000,remixed_audio[0].astype(np.float32))
|
||||
else: print(f"skipped short audio clip: {idx0}_{idx1}.wav ({len(tmp_audio)=})")
|
||||
|
||||
def pipeline(self, path, idx0):
|
||||
try:
|
||||
audio,_ = load_input_audio(path, self.sr)
|
||||
input_audio = load_input_audio(path, self.sr)
|
||||
if self.preprocessor is not None: input_audio = self.preprocessor(input_audio)
|
||||
|
||||
idx1 = 0
|
||||
for audio in self.slicer.slice(audio):
|
||||
for audio in self.slicer.slice(input_audio[0]):
|
||||
i = 0
|
||||
while 1:
|
||||
start = int(self.sr * (self.per - self.overlap) * i)
|
||||
@@ -188,18 +192,20 @@ class FeatureInput(FeatureExtractor):
|
||||
except:
|
||||
self.printt("f0fail-%s-%s-%s" % (idx, inp_path, traceback.format_exc()))
|
||||
|
||||
def preprocess_trainset(inp_root, sr, n_p, exp_dir, period=3.0, overlap=.3, max_volume=1.):
|
||||
def preprocess_trainset(inp_root, sr, n_p, exp_dir, preprocessor=None, period=3.0, overlap=.3, max_volume=1.):
|
||||
try:
|
||||
pp = Preprocess(sr, exp_dir, period=period, overlap=overlap, max_volume=max_volume)
|
||||
pp = Preprocess(sr, exp_dir, preprocessor=preprocessor, period=period, overlap=overlap, max_volume=max_volume)
|
||||
pp.println("start preprocess")
|
||||
pp.println(sys.argv)
|
||||
pp.pipeline_mp_inp_dir(inp_root, n_p)
|
||||
pp.println("end preprocess")
|
||||
del pp
|
||||
gc_collect()
|
||||
return "Successfully preprocessed data"
|
||||
print("Successfully preprocessed data")
|
||||
return True
|
||||
except Exception as e:
|
||||
return f"Failed to preprocess data: {e}"
|
||||
print(f"Failed to preprocess data: {e}")
|
||||
return False
|
||||
|
||||
def extract_features_trainset(hubert_model,exp_dir,n_p,f0method,device,version,if_f0,crepe_hop_length):
|
||||
try:
|
||||
@@ -240,7 +246,8 @@ def extract_features_trainset(hubert_model,exp_dir,n_p,f0method,device,version,i
|
||||
p.join()
|
||||
except:
|
||||
featureInput.printt("f0_all_fail-%s" % (traceback.format_exc()))
|
||||
|
||||
return f"Successfully extracted features using {f0method}"
|
||||
print(f"Successfully extracted features using {f0method}")
|
||||
return True
|
||||
except Exception as e:
|
||||
return f"Failed to extract features: {e}"
|
||||
print(f"Failed to extract features: {e}")
|
||||
return False
|
||||
|
||||
Reference in New Issue
Block a user