281 lines
8.1 KiB
Python
281 lines
8.1 KiB
Python
# Copyright (c) 2023 Amphion.
|
|
#
|
|
# This source code is licensed under the MIT license found in the
|
|
# LICENSE file in the root directory of this source tree.
|
|
|
|
import torch
|
|
from librosa.filters import mel as librosa_mel_fn
|
|
|
|
|
|
def dynamic_range_compression_torch(x, C=1, clip_val=1e-5):
|
|
# Min value: ln(1e-5) = -11.5129
|
|
return torch.log(torch.clamp(x, min=clip_val) * C)
|
|
|
|
|
|
def spectral_normalize_torch(magnitudes):
|
|
output = dynamic_range_compression_torch(magnitudes)
|
|
return output
|
|
|
|
|
|
def extract_linear_features(y, cfg, center=False):
|
|
if torch.min(y) < -1.0:
|
|
print("min value is ", torch.min(y))
|
|
if torch.max(y) > 1.0:
|
|
print("max value is ", torch.max(y))
|
|
|
|
global hann_window
|
|
hann_window[str(y.device)] = torch.hann_window(cfg.win_size).to(y.device)
|
|
|
|
y = torch.nn.functional.pad(
|
|
y.unsqueeze(1),
|
|
(int((cfg.n_fft - cfg.hop_size) / 2), int((cfg.n_fft - cfg.hop_size) / 2)),
|
|
mode="reflect",
|
|
)
|
|
y = y.squeeze(1)
|
|
|
|
# complex tensor as default, then use view_as_real for future pytorch compatibility
|
|
spec = torch.stft(
|
|
y,
|
|
cfg.n_fft,
|
|
hop_length=cfg.hop_size,
|
|
win_length=cfg.win_size,
|
|
window=hann_window[str(y.device)],
|
|
center=center,
|
|
pad_mode="reflect",
|
|
normalized=False,
|
|
onesided=True,
|
|
return_complex=True,
|
|
)
|
|
spec = torch.view_as_real(spec)
|
|
spec = torch.sqrt(spec.pow(2).sum(-1) + (1e-9))
|
|
spec = torch.squeeze(spec, 0)
|
|
return spec
|
|
|
|
|
|
def mel_spectrogram_torch(y, cfg, center=False):
|
|
"""
|
|
TODO: to merge this funtion with the extract_mel_features below
|
|
"""
|
|
if torch.min(y) < -1.0:
|
|
print("min value is ", torch.min(y))
|
|
if torch.max(y) > 1.0:
|
|
print("max value is ", torch.max(y))
|
|
|
|
global mel_basis, hann_window
|
|
if cfg.fmax not in mel_basis:
|
|
mel = librosa_mel_fn(
|
|
sr=cfg.sample_rate,
|
|
n_fft=cfg.n_fft,
|
|
n_mels=cfg.n_mel,
|
|
fmin=cfg.fmin,
|
|
fmax=cfg.fmax,
|
|
)
|
|
mel_basis[str(cfg.fmax) + "_" + str(y.device)] = (
|
|
torch.from_numpy(mel).float().to(y.device)
|
|
)
|
|
hann_window[str(y.device)] = torch.hann_window(cfg.win_size).to(y.device)
|
|
|
|
y = torch.nn.functional.pad(
|
|
y.unsqueeze(1),
|
|
(int((cfg.n_fft - cfg.hop_size) / 2), int((cfg.n_fft - cfg.hop_size) / 2)),
|
|
mode="reflect",
|
|
)
|
|
y = y.squeeze(1)
|
|
|
|
spec = torch.stft(
|
|
y,
|
|
cfg.n_fft,
|
|
hop_length=cfg.hop_size,
|
|
win_length=cfg.win_size,
|
|
window=hann_window[str(y.device)],
|
|
center=center,
|
|
pad_mode="reflect",
|
|
normalized=False,
|
|
onesided=True,
|
|
return_complex=True,
|
|
)
|
|
|
|
spec = torch.view_as_real(spec)
|
|
spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6)
|
|
|
|
spec = torch.matmul(mel_basis[str(cfg.fmax) + "_" + str(y.device)], spec)
|
|
spec = spectral_normalize_torch(spec)
|
|
|
|
return spec
|
|
|
|
|
|
mel_basis = {}
|
|
hann_window = {}
|
|
|
|
|
|
def extract_mel_features(
|
|
y,
|
|
cfg,
|
|
center=False,
|
|
):
|
|
"""Extract mel features
|
|
|
|
Args:
|
|
y (tensor): audio data in tensor
|
|
cfg (dict): configuration in cfg.preprocess
|
|
center (bool, optional): In STFT, whether t-th frame is centered at time t*hop_length. Defaults to False.
|
|
|
|
Returns:
|
|
tensor: a tensor containing the mel feature calculated based on STFT result
|
|
"""
|
|
if torch.min(y) < -1.0:
|
|
print("min value is ", torch.min(y))
|
|
if torch.max(y) > 1.0:
|
|
print("max value is ", torch.max(y))
|
|
|
|
global mel_basis, hann_window
|
|
if cfg.fmax not in mel_basis:
|
|
mel = librosa_mel_fn(
|
|
sr=cfg.sample_rate,
|
|
n_fft=cfg.n_fft,
|
|
n_mels=cfg.n_mel,
|
|
fmin=cfg.fmin,
|
|
fmax=cfg.fmax,
|
|
)
|
|
mel_basis[str(cfg.fmax) + "_" + str(y.device)] = (
|
|
torch.from_numpy(mel).float().to(y.device)
|
|
)
|
|
hann_window[str(y.device)] = torch.hann_window(cfg.win_size).to(y.device)
|
|
|
|
y = torch.nn.functional.pad(
|
|
y.unsqueeze(1),
|
|
(int((cfg.n_fft - cfg.hop_size) / 2), int((cfg.n_fft - cfg.hop_size) / 2)),
|
|
mode="reflect",
|
|
)
|
|
y = y.squeeze(1)
|
|
|
|
# complex tensor as default, then use view_as_real for future pytorch compatibility
|
|
spec = torch.stft(
|
|
y,
|
|
cfg.n_fft,
|
|
hop_length=cfg.hop_size,
|
|
win_length=cfg.win_size,
|
|
window=hann_window[str(y.device)],
|
|
center=center,
|
|
pad_mode="reflect",
|
|
normalized=False,
|
|
onesided=True,
|
|
return_complex=True,
|
|
)
|
|
spec = torch.view_as_real(spec)
|
|
spec = torch.sqrt(spec.pow(2).sum(-1) + (1e-9))
|
|
|
|
spec = torch.matmul(mel_basis[str(cfg.fmax) + "_" + str(y.device)], spec)
|
|
spec = spectral_normalize_torch(spec)
|
|
return spec.squeeze(0)
|
|
|
|
|
|
def extract_mel_features_tts(
|
|
y,
|
|
cfg,
|
|
center=False,
|
|
taco=False,
|
|
_stft=None,
|
|
):
|
|
"""Extract mel features
|
|
|
|
Args:
|
|
y (tensor): audio data in tensor
|
|
cfg (dict): configuration in cfg.preprocess
|
|
center (bool, optional): In STFT, whether t-th frame is centered at time t*hop_length. Defaults to False.
|
|
taco: use tacotron mel
|
|
|
|
Returns:
|
|
tensor: a tensor containing the mel feature calculated based on STFT result
|
|
"""
|
|
if not taco:
|
|
if torch.min(y) < -1.0:
|
|
print("min value is ", torch.min(y))
|
|
if torch.max(y) > 1.0:
|
|
print("max value is ", torch.max(y))
|
|
|
|
global mel_basis, hann_window
|
|
if cfg.fmax not in mel_basis:
|
|
mel = librosa_mel_fn(
|
|
sr=cfg.sample_rate,
|
|
n_fft=cfg.n_fft,
|
|
n_mels=cfg.n_mel,
|
|
fmin=cfg.fmin,
|
|
fmax=cfg.fmax,
|
|
)
|
|
mel_basis[str(cfg.fmax) + "_" + str(y.device)] = (
|
|
torch.from_numpy(mel).float().to(y.device)
|
|
)
|
|
hann_window[str(y.device)] = torch.hann_window(cfg.win_size).to(y.device)
|
|
|
|
y = torch.nn.functional.pad(
|
|
y.unsqueeze(1),
|
|
(int((cfg.n_fft - cfg.hop_size) / 2), int((cfg.n_fft - cfg.hop_size) / 2)),
|
|
mode="reflect",
|
|
)
|
|
y = y.squeeze(1)
|
|
|
|
# complex tensor as default, then use view_as_real for future pytorch compatibility
|
|
spec = torch.stft(
|
|
y,
|
|
cfg.n_fft,
|
|
hop_length=cfg.hop_size,
|
|
win_length=cfg.win_size,
|
|
window=hann_window[str(y.device)],
|
|
center=center,
|
|
pad_mode="reflect",
|
|
normalized=False,
|
|
onesided=True,
|
|
return_complex=True,
|
|
)
|
|
spec = torch.view_as_real(spec)
|
|
spec = torch.sqrt(spec.pow(2).sum(-1) + (1e-9))
|
|
|
|
spec = torch.matmul(mel_basis[str(cfg.fmax) + "_" + str(y.device)], spec)
|
|
spec = spectral_normalize_torch(spec)
|
|
else:
|
|
audio = torch.clip(y, -1, 1)
|
|
audio = torch.autograd.Variable(audio, requires_grad=False)
|
|
spec, energy = _stft.mel_spectrogram(audio)
|
|
|
|
return spec.squeeze(0)
|
|
|
|
|
|
def amplitude_phase_spectrum(y, cfg):
|
|
hann_window = torch.hann_window(cfg.win_size).to(y.device)
|
|
|
|
y = torch.nn.functional.pad(
|
|
y.unsqueeze(1),
|
|
(int((cfg.n_fft - cfg.hop_size) / 2), int((cfg.n_fft - cfg.hop_size) / 2)),
|
|
mode="reflect",
|
|
)
|
|
y = y.squeeze(1)
|
|
|
|
stft_spec = torch.stft(
|
|
y,
|
|
cfg.n_fft,
|
|
hop_length=cfg.hop_size,
|
|
win_length=cfg.win_size,
|
|
window=hann_window,
|
|
center=False,
|
|
return_complex=True,
|
|
)
|
|
|
|
stft_spec = torch.view_as_real(stft_spec)
|
|
if stft_spec.size()[0] == 1:
|
|
stft_spec = stft_spec.squeeze(0)
|
|
|
|
if len(list(stft_spec.size())) == 4:
|
|
rea = stft_spec[:, :, :, 0] # [batch_size, n_fft//2+1, frames]
|
|
imag = stft_spec[:, :, :, 1] # [batch_size, n_fft//2+1, frames]
|
|
else:
|
|
rea = stft_spec[:, :, 0] # [n_fft//2+1, frames]
|
|
imag = stft_spec[:, :, 1] # [n_fft//2+1, frames]
|
|
|
|
log_amplitude = torch.log(
|
|
torch.abs(torch.sqrt(torch.pow(rea, 2) + torch.pow(imag, 2))) + 1e-5
|
|
) # [n_fft//2+1, frames]
|
|
phase = torch.atan2(imag, rea) # [n_fft//2+1, frames]
|
|
|
|
return log_amplitude, phase, rea, imag
|