diff --git a/custom_nodes/audio_nodes.py b/custom_nodes/audio_nodes.py index 3b413f9..e1c2a8f 100644 --- a/custom_nodes/audio_nodes.py +++ b/custom_nodes/audio_nodes.py @@ -6,7 +6,7 @@ import yt_dlp import torch from .settings import MERGE_OPTIONS from .utils import MultipleTypeProxy, increment_filename_no_overwrite -from ..lib.audio import MAX_INT16, SUPPORTED_AUDIO, audio_to_bytes, bytes_to_audio, load_input_audio, pad_audio, remix_audio, save_input_audio +from ..lib.audio import MAX_INT16, SUPPORTED_AUDIO, AudioProcessor, audio_to_bytes, get_audio, load_input_audio, pad_audio, remix_audio, save_input_audio from ..lib.utils import get_filenames, get_hash, get_merge_func import folder_paths @@ -14,12 +14,6 @@ CATEGORY = "🌺RVC-Studio/audio" input_path = folder_paths.get_input_directory() temp_path = folder_paths.get_temp_directory() -def get_audio(audio): - if hasattr(audio,"__call__"): #VHS_AUDIO is a function - return bytes_to_audio(audio()) - else: #comfyui AUDIO is a dict - return audio["waveform"].squeeze(0).transpose(0,1).numpy(), audio["sample_rate"] - def to_audio_dict(audio, sr): #from https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite/blob/bf2a9402d0b2727c7170c43621d569f4d531015f/videohelpersuite/nodes.py#L706C22-L706C67 @@ -112,9 +106,6 @@ class DownloadAudio: return {"ui": {"preview": [{"filename": os.path.basename(audio_path), "type": "input", "subfolder": "audio", "widgetId": widgetId}]}, "result": (audio_name, lambda:audio_to_bytes(*input_audio),to_audio_dict(*input_audio))} class MergeAudioNode: - - def __init__(self): - pass @classmethod def INPUT_TYPES(cls): @@ -163,8 +154,6 @@ class MergeAudioNode: return {"ui": {"preview": [{"filename": audio_name, "type": "temp", "subfolder": "preview", "widgetId": widgetId}]}, "result": (lambda: audio_to_bytes(*merged_audio),to_audio_dict(*merged_audio))} class PreviewAudio: - def __init__(self): - pass @classmethod def INPUT_TYPES(cls): @@ -215,23 +204,78 @@ class PreviewAudio: "filename": audio_name, "type": "temp", "subfolder": "preview", "widgetId": widgetId, "autoplay": autoplay }]}, "result": (output_path, lambda:audio_to_bytes(*input_audio), to_audio_dict(*input_audio))} + +class ProcessAudioNode: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "normalize": ("BOOLEAN",{"default": True}), + "threshold_silence": ("BOOLEAN",{"default": True}), + "dynamic_threshold": ("BOOLEAN",{"default": True}) + }, + "optional": { + "audio": (MultipleTypeProxy('AUDIO,VHS_AUDIO'),{"default": None}), + "dynamic_threshold_sample_size": ("INT",{"default": 4000, "min": 160, "max": 48000, "step": 160}), + "dynamic_threshold_multiplier": ("FLOAT",{"default": 2.}), + "dynamic_threshold_fill_method": (["median","interpolation"],{"default": "median"}), + "dynamic_threshold_kernel_size": ("INT",{"default": 5}), + "silence_threshold_db": ("INT",{"default": -50, "min": -120, "max": 0}), + "normalize_threshold_db": ("INT",{"default": -1, "min": -10, "max": 0}) + } + } + + RETURN_TYPES = ("AUDIO_PROCESSOR","VHS_AUDIO","AUDIO") + RETURN_NAMES = ("audio_processor","vhs_audio","audio") + + CATEGORY = CATEGORY + + FUNCTION = "process_audio" + + def process_audio(self, normalize, threshold_silence, dynamic_threshold, audio=None, + dynamic_threshold_sample_size=16000, + dynamic_threshold_multiplier=2.0, + dynamic_threshold_fill_method="median", + dynamic_threshold_kernel_size=5, + silence_threshold_db=-50, + normalize_threshold_db=-1): + + audio_processor = AudioProcessor( + normalize=normalize, + threshold_silence=threshold_silence, + dynamic_threshold=dynamic_threshold, + sample_size=dynamic_threshold_sample_size, + multiplier=dynamic_threshold_multiplier, + fill_method=dynamic_threshold_fill_method, + kernel_size=dynamic_threshold_kernel_size, + silence_threshold_db=silence_threshold_db, + normalize_threshold_db=normalize_threshold_db) + + if audio is None: + vhs_audio = comfy_audio = audio + else: + output_audio = audio_processor(audio) + vhs_audio = lambda:audio_to_bytes(*output_audio) + comfy_audio = to_audio_dict(*output_audio) + + return ( audio_processor, vhs_audio, comfy_audio) + + class AudioBatchValueNode: - def __init__(self): - pass @classmethod def INPUT_TYPES(cls): return { "required": { "audio": (MultipleTypeProxy('AUDIO,VHS_AUDIO'),), - "num_segments": ('INT', {"min": 2, "max": 128, "step": 1, "forceInput": True}), + "num_segments": ('INT', {"default": 2, "min": 2, "max": 256, "step": 1}), "output_min": ('FLOAT', {'default': 0., "min": -1000., "max": 1000., "step": .01}), "output_max": ('FLOAT', {'default': 1., "min": 0., "max": 1000., "step": .01}), "norm": (["scale","tanh","sigmoid"], {"default": "scale"}), }, "optional": { - "silence_threshold": ("INT", {"default": 1000, "min": 1, "max": MAX_INT16, "step": 1, "display": "slider"}), - "duration_list": ("INT", {"default": None, "min": 1, "forceInput": True}), + "silence_threshold": ("INT", {"default": 1000, "min": 1, "max": MAX_INT16, "step": 1}), + "duration_list": ("INT", {"default": 0, "min": 0, "forceInput": True}), "print_output": ("BOOLEAN", {"default": False}), "inverse": ("BOOLEAN", {"default": False}), } @@ -248,8 +292,7 @@ class AudioBatchValueNode: def get_rms(audio): # root mean squared of audio segment return np.sqrt(np.nanmean(audio**2)) - def get_frame_weights(self, audio, num_segments, output_min, output_max, norm, - silence_threshold=1000, duration_list=None, print_output=False, inverse=False): + def get_frame_weights(self, audio, num_segments, output_min, output_max, norm, silence_threshold=1000, duration_list=0, print_output=False, inverse=False): assert output_max>=output_min, f"{output_max=} must be greater or equal to {output_min=}!" audio_data = get_audio(audio) @@ -277,7 +320,7 @@ class AudioBatchValueNode: print(f"{audio_rms.min()=} {audio_rms.max()=} {audio_rms.mean()=} {len(audio_rms)=}") print(f"{x_norm.min()=} {x_norm.max()=} {x_norm.mean()=} {len(x_norm)=}") - if duration_list is not None: + if isinstance(duration_list,list): segments = np.cumsum(duration_list) x_norm = np.array_split(x_norm,segments) x_norm = map(list,x_norm) @@ -293,6 +336,7 @@ NODE_CLASS_MAPPINGS = { "MergeAudioNode": MergeAudioNode, "AudioBatchValueNode": AudioBatchValueNode, "DownloadAudio": DownloadAudio, + "ProcessAudioNode": ProcessAudioNode } NODE_DISPLAY_NAME_MAPPINGS = { @@ -301,4 +345,5 @@ NODE_DISPLAY_NAME_MAPPINGS = { "RVC-Studio.PreviewAudio": "🌺Save Audio", "MergeAudioNode": "🌺Merge Audio", "AudioBatchValueNode": "🌺Audio RMS Batch Values", + "ProcessAudioNode": "🌺Audio Processor" } \ No newline at end of file diff --git a/custom_nodes/musetalk_nodes.py b/custom_nodes/musetalk_nodes.py index 37252b4..fa6e88c 100644 --- a/custom_nodes/musetalk_nodes.py +++ b/custom_nodes/musetalk_nodes.py @@ -6,10 +6,9 @@ import cv2 import copy from tqdm import tqdm -from ..lib.audio import remix_audio +from ..lib.audio import get_audio, remix_audio from ..lib import BASE_CACHE_DIR from ..lib.utils import gc_collect, get_hash, get_optimal_torch_device -from .audio_nodes import get_audio from .utils import MultipleTypeProxy, model_downloader from ..lib.musetalk.models.unet import PositionalEncoding, UNet from ..lib.musetalk.models.vae import VAE diff --git a/custom_nodes/rvc_nodes.py b/custom_nodes/rvc_nodes.py index bbdf346..014e419 100644 --- a/custom_nodes/rvc_nodes.py +++ b/custom_nodes/rvc_nodes.py @@ -14,7 +14,6 @@ from ..training_cli import train_model from ..preprocessing_utils import extract_features_trainset, preprocess_trainset -from .audio_nodes import get_audio from ..config import config from ..lib.model_utils import load_hubert @@ -23,7 +22,7 @@ from .utils import MultipleTypeProxy, increment_filename_no_overwrite, model_dow from .settings import PITCH_EXTRACTION_OPTIONS, SR_MAP from .settings.downloader import PRETRAINED_MODELS, RVC_DOWNLOAD_LINK, RVC_INDEX, RVC_MODELS, download_file, extract_zip_without_structure -from ..lib.audio import SUPPORTED_AUDIO, audio_to_bytes, load_input_audio, save_input_audio +from ..lib.audio import SUPPORTED_AUDIO, audio_to_bytes, load_input_audio, save_input_audio, get_audio from ..vc_infer_pipeline import get_vc, vc_single import folder_paths @@ -237,6 +236,7 @@ class RVCProcessDatasetNode: "overlap": ("FLOAT",{"default": .3, "min": .1, "max": 1., "step": .1}), "max_volume": ("FLOAT",{"default": .99, "min": .1, "max": 1., "step": .01}), "mute_ratio": ("FLOAT",{"default": .0, "min": .0, "max": .5, "step": .01}), + "audio_processor": ("AUDIO_PROCESSOR",) } } @@ -247,13 +247,13 @@ class RVCProcessDatasetNode: CATEGORY = CATEGORY - def process(self, model_name: str, dataset: str, hubert_model, pitch_extraction_params={}, sr="40k", n_threads=1, period=3., overlap=.3, max_volume=1., mute_ratio=.0): + def process(self, model_name: str, dataset: str, hubert_model, pitch_extraction_params={}, sr="40k", n_threads=1, period=3., overlap=.3, max_volume=1., mute_ratio=.0, audio_processor=None): assert model_name, "Please provide a model name!" assert dataset, "Please upload a dataset!" f0_method = pitch_extraction_params.get("f0_method", "") - cached_params = [model_name, dataset, period, overlap, max_volume, mute_ratio, sr, f0_method] + cached_params = [model_name, dataset, period, overlap, max_volume, mute_ratio, sr, f0_method, audio_processor] crepe_hop_length = pitch_extraction_params.get("crepe_hop_length",160) if "crepe" in f0_method: cached_params.append(crepe_hop_length) @@ -271,9 +271,9 @@ class RVCProcessDatasetNode: files = extract_zip_without_structure(os.path.join(dataset_path,dataset),dataset_dir) assert len(files), "Failed to extract zip file..." - print(preprocess_trainset(dataset_dir,SR_MAP[sr],n_threads,model_log_dir,period,overlap,max_volume)) + assert preprocess_trainset(dataset_dir,SR_MAP[sr],n_threads,model_log_dir,audio_processor,period,overlap,max_volume), "Failed to preprocess audio..." - print(extract_features_trainset(hubert_model(), model_log_dir,n_p=n_threads,f0method=f0_method,device=device,if_f0=bool(f0_method),version="v2",crepe_hop_length=crepe_hop_length)) + assert extract_features_trainset(hubert_model(), model_log_dir,n_p=n_threads,f0method=f0_method,device=device,if_f0=bool(f0_method),version="v2",crepe_hop_length=crepe_hop_length), "Failed to extract features..." gt_wavs_dir = os.path.join(model_log_dir,"0_gt_wavs") feature_dir = os.path.join(model_log_dir,"3_feature768") @@ -377,7 +377,7 @@ class RVCTrainModelNode: }, "optional": dict( gpu=(DEVICES, {"default": DEVICES[0]}), - batch_size=("INT",dict(default=4,min=1,max=64)), + batch_size=("INT",dict(default=4,min=0,max=64,step=4)), total_epoch=("INT",dict(default=100,min=10,max=1000,step=10)), save_every_epoch=("INT",dict(default=0,min=0,max=100)), pretrained_G=(PRETRAINED_G,{"default": PRETRAINED_G[0]}), @@ -433,7 +433,7 @@ class RVCTrainModelNode: hparams.pretrainD = model_downloader(pretrained_D) hparams.version = "v2" hparams.gpus = gpu - hparams.train.batch_size = batch_size + hparams.train.batch_size = max(batch_size,1) hparams.sample_rate = sample_rate hparams.if_f0 = if_f0 hparams.if_latest = if_save_latest @@ -443,7 +443,7 @@ class RVCTrainModelNode: hparams.save_best_model = save_best_model file_index = self.train_index(model_dir, sample_rate, name) if train_index else None - model_path = os.path.join(BASE_MODELS_DIR,"RVC",f"{name}.pth") + model_path = os.path.join(BASE_MODELS_DIR,"RVC",f"{name}_{sample_rate}.pth") if os.path.isfile(model_path) and retrain: model_path = increment_filename_no_overwrite(model_path) hparams.model_path = model_path diff --git a/custom_nodes/stt.py b/custom_nodes/stt.py index 35e40c9..f58a4fa 100644 --- a/custom_nodes/stt.py +++ b/custom_nodes/stt.py @@ -263,11 +263,11 @@ class BatchedTranscriptionEncoderNode: "use_tags": ('BOOLEAN', {'default': False}), "use_sentiment": ('BOOLEAN', {'default': False}), 'language': (SUPPORTED_LANGUAGES,{"default": "en"}), - "max_chunks": ('INT', {"min": 2, "default": None}), - "max_words": ('INT', {'default': 16, "min": 0, "max": 32, "display": "slider"}), + "max_chunks": ('INT', {"min": 0, "default": 0}), + "max_words": ('INT', {'default': 16, "min": 0, "max": 32}), "frame_interpolation": ("INT", {"default": 0, "min": 0, "max": 120, "hidden": True}), # needs more testing - "prefix": ("STRING", {"default": "masterpiece, best quality", "multiline": True, "forceInput": True}), - "suffix": ("STRING", {"default": "", "multiline": True, "forceInput": True}), + "prefix": ("STRING", {"default": "masterpiece, best quality", "multiline": True}), + "suffix": ("STRING", {"default": "", "multiline": True}), "print_output": ('BOOLEAN', {'default': True}), "weights": ("FLOAT",{"default": 1., "step": .01}), "pad_frames": ("INT", {"default": 0}) @@ -299,7 +299,7 @@ class BatchedTranscriptionEncoderNode: def get_prompt( self, transcription, clip, language="en", loop=False, use_tags=False, use_sentiment=False, - max_words=16,max_chunks=None, frame_interpolation=0, print_output=True, prefix="", suffix="", + max_words=16,max_chunks=0, frame_interpolation=0, print_output=True, prefix="", suffix="", weights=1., pad_frames=0 ): diff --git a/custom_nodes/uvr.py b/custom_nodes/uvr.py index ba0b4aa..26e423f 100644 --- a/custom_nodes/uvr.py +++ b/custom_nodes/uvr.py @@ -1,10 +1,8 @@ import os import audio_separator.separator as uvr -from .audio_nodes import get_audio - from .utils import MultipleTypeProxy -from ..lib.audio import audio_to_bytes, save_input_audio, load_input_audio +from ..lib.audio import audio_to_bytes, save_input_audio, load_input_audio, get_audio import folder_paths from ..lib.utils import get_filenames, get_hash, get_optimal_torch_device from ..lib import BASE_CACHE_DIR, BASE_MODELS_DIR, karafan diff --git a/lib/audio.py b/lib/audio.py index 153dc01..f237ed3 100644 --- a/lib/audio.py +++ b/lib/audio.py @@ -2,11 +2,13 @@ import base64 import io import os import zlib -from .utils import get_merge_func +from .utils import get_hash, get_merge_func import numpy as np import librosa import soundfile as sf import ffmpeg +from scipy.ndimage import uniform_filter1d, median_filter +from scipy.interpolate import interp1d MAX_INT16 = 32768 SUPPORTED_AUDIO = ["mp3","flac","wav"] # ogg breaks soundfile @@ -26,6 +28,92 @@ AUTOTUNE_NOTES = np.array([ 2959.96, 3135.96, 3322.44, 3520.00, 3729.31, 3951.07 ]) +class AudioProcessor: + def __init__(self, normalize=True, threshold_silence=True, dynamic_threshold=True, sample_size=16000, multiplier=2.0, fill_method="median", kernel_size=5, silence_threshold_db=-50, normalize_threshold_db=-1): + self.normalize=normalize + self.threshold_silence=threshold_silence + self.dynamic_threshold=dynamic_threshold + self.sample_size=sample_size + self.multiplier=multiplier + self.fill_method=fill_method + self.kernel_size=kernel_size + self.silence_threshold_db=silence_threshold_db + self.normalize_threshold_db=normalize_threshold_db + + def __str__(self) -> str: + values = [self.normalize, self.threshold_silence, self.dynamic_threshold] + if self.normalize: values.append(self.normalize_threshold_db) + if self.threshold_silence: values.append(self.silence_threshold_db) + if self.dynamic_threshold: values.extend([self.sample_size, self.multiplier, self.fill_method, self.kernel_size]) + return get_hash(*values) + + def __call__(self, audio): + from .karafan.audio_utils import Normalize, Silent + + samples,sr = get_audio(audio) + + if self.threshold_silence: + samples = np.atleast_2d([samples]) + samples = Silent(samples, sample_rate=sr, threshold_dB=self.silence_threshold_db) + samples = np.squeeze(samples, 0) + if self.dynamic_threshold: samples = self.dynamic_thresholding( + samples, + multiplier=self.multiplier, + sample_size=self.sample_size, + method=self.fill_method, + kernel_size=self.kernel_size) + if self.normalize: samples = Normalize(samples,threshold_dB=self.normalize_threshold_db) + + return samples, sr + + @staticmethod + def dynamic_thresholding(samples, multiplier=2., sample_size=16000, method="median", kernel_size=5): + # Calculate the local RMS (root mean square) to estimate the local amplitude + local_rms = np.sqrt(uniform_filter1d(np.square(samples), size=int(sample_size))) + + # Set dynamic threshold as a multiple of the local RMS + dynamic_threshold = multiplier * local_rms + + # Detect clicks where the absolute value exceeds the dynamic threshold + clicks = np.abs(samples) > dynamic_threshold + + # Replace clicks using the specified method + cleaned_samples = AudioProcessor.replace_clicks(samples, clicks, method=method, kernel_size=kernel_size) + return cleaned_samples + + @staticmethod + def replace_clicks(samples, clicks, method="median", kernel_size=5): + if method == "median": + # Replace detected clicks with the median value of surrounding samples + cleaned_samples = samples.copy() + cleaned_samples[clicks] = median_filter(samples, size=kernel_size)[clicks] + elif method == "interpolation": + # Replace detected clicks with interpolation + cleaned_samples = samples.copy() + + # Indices of non-clicks (valid points for interpolation) + non_click_indices = np.where(~clicks)[0] + + # Indices of clicks (points to be replaced) + click_indices = np.where(clicks)[0] + + # Interpolation function + interp_func = interp1d(non_click_indices, cleaned_samples[non_click_indices], kind='linear', bounds_error=False, fill_value="extrapolate") + + # Replace click samples with interpolated values + cleaned_samples[click_indices] = interp_func(click_indices) + else: + raise ValueError("Method must be 'median' or 'interpolation'") + + return cleaned_samples + +def get_audio(audio): + if hasattr(audio,"__call__"): #VHS_AUDIO is a function + return bytes_to_audio(audio()) + elif isinstance(audio,dict): #comfyui AUDIO is a dict + return audio["waveform"].squeeze(0).transpose(0,1).numpy(), audio["sample_rate"] + else: return audio + def load_audio(file, sr, **kwargs): try: # https://github.com/openai/whisper/blob/main/whisper/audio.py#L26 diff --git a/lib/karafan/audio_utils.py b/lib/karafan/audio_utils.py index 8a4ac1b..90b98c5 100644 --- a/lib/karafan/audio_utils.py +++ b/lib/karafan/audio_utils.py @@ -124,7 +124,7 @@ def Silent(audio_in, sample_rate, threshold_dB = -50): start = 0; end = 0 audio = audio_in.copy() - audio_length = audio_in.shape[1] + audio_length = audio_in.shape[-1] for i in range(0, audio_length, window_frame): @@ -241,71 +241,53 @@ def Change_sample_rate(audio, way, current_cutoff, target_cutoff): pitched_audio = librosa.resample(audio, orig_sr = current_cutoff * 2, target_sr = target_cutoff * 2, res_type = 'kaiser_best', axis=1) - # print(f"SRS input audio shape: {audio.shape}") - # print(f"SRS output audio shape: {pitched_audio.shape}") - # print (f"ratio : {ratio}") - return pitched_audio -# def Remove_High_freq_Noise(audio, threshold_freq): +def Remove_High_freq_Noise(audio, threshold_freq): -# # Calculer la transformée de Fourier -# stft = librosa.stft(audio) + # Calculer la transformée de Fourier + stft = librosa.stft(audio) -# # Calculer la somme des amplitudes pour chaque fréquence dans le spectre -# amplitude_sum = np.sum(np.abs(stft), axis=0) + # Calculer la somme des amplitudes pour chaque fréquence dans le spectre + amplitude_sum = np.sum(np.abs(stft), axis=0) -# # Appliquer un masque pour supprimer les fréquences supérieures lorsque la somme des amplitudes est inférieure au seuil -# stft[:, amplitude_sum > threshold_freq] = 0.0 + # Appliquer un masque pour supprimer les fréquences supérieures lorsque la somme des amplitudes est inférieure au seuil + stft[:, amplitude_sum > threshold_freq] = 0.0 -# # Reconstruire l'audio à partir du STFT modifié - # audio_filtered = librosa.istft(stft) + # Reconstruire l'audio à partir du STFT modifié + audio_filtered = librosa.istft(stft) -# return audio_filtered + return audio_filtered -# def Match_Freq_CutOFF(self, audio1, audio2, sample_rate): -# # This option matches the Primary stem frequency cut-off to the Secondary stem frequency cut-off -# # (if the Primary stem frequency cut-off is lower than the Secondary stem frequency cut-off) +# Find the high cut-off frequency of the input audio +def Find_Cut_OFF(audio, sample_rate, threshold=0.01): -# # Get the Primary stem frequency cut-off -# freq_cut_off1 = Find_Cut_OFF(audio1, sample_rate) -# freq_cut_off2 = Find_Cut_OFF(audio2, sample_rate) + # Appliquer un filtre passe-bas pour réduire le bruit + cutoff_frequency = sample_rate / 2.0 # Fréquence de Nyquist (la moitié du taux d'échantillonnage) -# # Match the Primary stem frequency cut-off to the Secondary stem frequency cut-off -# if freq_cut_off1 < freq_cut_off2: -# audio1 = Resize_Freq_CutOFF(audio1, freq_cut_off2, sample_rate) + # Définir l'ordre du filtre passe-bas + order = 6 -# return audio1 + # Calculer les coefficients du filtre passe-bas + b, a = signal.butter(order, cutoff_frequency - threshold, btype='low', analog=False, fs=sample_rate) -# # Find the high cut-off frequency of the input audio -# def Find_Cut_OFF(audio, sample_rate, threshold=0.01): + # Appliquer le filtre au signal audio + filtered_audio = signal.lfilter(b, a, audio, axis=0) -# # Appliquer un filtre passe-bas pour réduire le bruit -# cutoff_frequency = sample_rate / 2.0 # Fréquence de Nyquist (la moitié du taux d'échantillonnage) + # Calculer la FFT du signal audio filtré + fft_result = np.fft.fft(filtered_audio, axis=0) -# # Définir l'ordre du filtre passe-bas -# order = 6 + # Calculer les magnitudes du spectre de fréquence + magnitudes = np.abs(fft_result) -# # Calculer les coefficients du filtre passe-bas -# b, a = signal.butter(order, cutoff_frequency - threshold, btype='low', analog=False, fs=sample_rate) + # Calculer les fréquences correspondant aux bins de la FFT + frequencies = np.fft.fftfreq(len(audio), 1.0 / sample_rate) -# # Appliquer le filtre au signal audio -# filtered_audio = signal.lfilter(b, a, audio, axis=0) + # Trouver la fréquence de coupure où la magnitude tombe en dessous du seuil + cut_off_frequencies = frequencies[np.where(magnitudes > threshold)] -# # Calculer la FFT du signal audio filtré -# fft_result = np.fft.fft(filtered_audio, axis=0) - -# # Calculer les magnitudes du spectre de fréquence -# magnitudes = np.abs(fft_result) - -# # Calculer les fréquences correspondant aux bins de la FFT -# frequencies = np.fft.fftfreq(len(audio), 1.0 / sample_rate) - -# # Trouver la fréquence de coupure où la magnitude tombe en dessous du seuil -# cut_off_frequencies = frequencies[np.where(magnitudes > threshold)] - -# # Trouver la fréquence de coupure maximale parmi toutes les valeurs -# return int(max(cut_off_frequencies)) + # Trouver la fréquence de coupure maximale parmi toutes les valeurs + return int(max(cut_off_frequencies)) diff --git a/preprocessing_utils.py b/preprocessing_utils.py index 0a35048..b57451e 100644 --- a/preprocessing_utils.py +++ b/preprocessing_utils.py @@ -5,13 +5,13 @@ from .lib.slicer2 import Slicer import traceback from scipy.io import wavfile from .pitch_extraction import FeatureExtractor -from .lib.audio import load_input_audio, remix_audio +from .lib.audio import load_input_audio, remix_audio, AudioProcessor from .lib.utils import gc_collect from .config import config import torch class Preprocess: - def __init__(self, sr, exp_dir, noparallel=True, period=3.0, overlap=.3, max_volume=.95): + def __init__(self, sr, exp_dir, preprocessor: "AudioProcessor"=None, noparallel=True, period=3.0, overlap=.3, max_volume=.95): self.slicer = Slicer( sr=sr, threshold=-42, @@ -29,6 +29,7 @@ class Preprocess: self.gt_wavs_dir = os.path.join(exp_dir,"0_gt_wavs") self.wavs16k_dir = os.path.join(exp_dir,"1_16k_wavs") self.noparallel = noparallel + self.preprocessor = preprocessor os.makedirs(self.exp_dir, exist_ok=True) os.makedirs(self.gt_wavs_dir, exist_ok=True) os.makedirs(self.wavs16k_dir, exist_ok=True) @@ -42,16 +43,19 @@ class Preprocess: # mutex.release() def norm_write(self, tmp_audio, idx0, idx1): - wavfile.write(os.path.join(self.gt_wavs_dir, f"{idx0}_{idx1}.wav"),self.sr,tmp_audio.astype(np.float32)) - remixed_audio = remix_audio((tmp_audio, self.sr), target_sr=16000, norm=True,max_volume=self.max_volume) - wavfile.write(os.path.join(self.wavs16k_dir, f"{idx0}_{idx1}.wav"),16000,remixed_audio[0].astype(np.float32)) + if len(tmp_audio) > self.overlap*self.sr*2: + wavfile.write(os.path.join(self.gt_wavs_dir, f"{idx0}_{idx1}.wav"),self.sr,tmp_audio.astype(np.float32)) + remixed_audio = remix_audio((tmp_audio, self.sr), target_sr=16000, max_volume=self.max_volume) + wavfile.write(os.path.join(self.wavs16k_dir, f"{idx0}_{idx1}.wav"),16000,remixed_audio[0].astype(np.float32)) + else: print(f"skipped short audio clip: {idx0}_{idx1}.wav ({len(tmp_audio)=})") def pipeline(self, path, idx0): try: - audio,_ = load_input_audio(path, self.sr) + input_audio = load_input_audio(path, self.sr) + if self.preprocessor is not None: input_audio = self.preprocessor(input_audio) idx1 = 0 - for audio in self.slicer.slice(audio): + for audio in self.slicer.slice(input_audio[0]): i = 0 while 1: start = int(self.sr * (self.per - self.overlap) * i) @@ -188,18 +192,20 @@ class FeatureInput(FeatureExtractor): except: self.printt("f0fail-%s-%s-%s" % (idx, inp_path, traceback.format_exc())) -def preprocess_trainset(inp_root, sr, n_p, exp_dir, period=3.0, overlap=.3, max_volume=1.): +def preprocess_trainset(inp_root, sr, n_p, exp_dir, preprocessor=None, period=3.0, overlap=.3, max_volume=1.): try: - pp = Preprocess(sr, exp_dir, period=period, overlap=overlap, max_volume=max_volume) + pp = Preprocess(sr, exp_dir, preprocessor=preprocessor, period=period, overlap=overlap, max_volume=max_volume) pp.println("start preprocess") pp.println(sys.argv) pp.pipeline_mp_inp_dir(inp_root, n_p) pp.println("end preprocess") del pp gc_collect() - return "Successfully preprocessed data" + print("Successfully preprocessed data") + return True except Exception as e: - return f"Failed to preprocess data: {e}" + print(f"Failed to preprocess data: {e}") + return False def extract_features_trainset(hubert_model,exp_dir,n_p,f0method,device,version,if_f0,crepe_hop_length): try: @@ -240,7 +246,8 @@ def extract_features_trainset(hubert_model,exp_dir,n_p,f0method,device,version,i p.join() except: featureInput.printt("f0_all_fail-%s" % (traceback.format_exc())) - - return f"Successfully extracted features using {f0method}" + print(f"Successfully extracted features using {f0method}") + return True except Exception as e: - return f"Failed to extract features: {e}" + print(f"Failed to extract features: {e}") + return False