# Reference: # https://github.com/bytedance/Make-An-Audio-2 from typing import Literal import torch import torch.nn as nn import numpy as np # following is from librosa def hz_to_mel(frequencies, *, htk = False): frequencies = np.asanyarray(frequencies) if htk: mels: np.ndarray = 2595.0 * np.log10(1.0 + frequencies / 700.0) return mels # Fill in the linear part f_min = 0.0 f_sp = 200.0 / 3 mels = (frequencies - f_min) / f_sp # Fill in the log-scale part min_log_hz = 1000.0 # beginning of log region (Hz) min_log_mel = (min_log_hz - f_min) / f_sp # same (Mels) logstep = np.log(6.4) / 27.0 # step size for log region if frequencies.ndim: # If we have array data, vectorize log_t = frequencies >= min_log_hz mels[log_t] = min_log_mel + np.log(frequencies[log_t] / min_log_hz) / logstep elif frequencies >= min_log_hz: # If we have scalar data, heck directly mels = min_log_mel + np.log(frequencies / min_log_hz) / logstep return mels def mel_to_hz(mels, *, htk = False): mels = np.asanyarray(mels) if htk: return 700.0 * (10.0 ** (mels / 2595.0) - 1.0) # Fill in the linear scale f_min = 0.0 f_sp = 200.0 / 3 freqs = f_min + f_sp * mels # And now the nonlinear scale min_log_hz = 1000.0 # beginning of log region (Hz) min_log_mel = (min_log_hz - f_min) / f_sp # same (Mels) logstep = np.log(6.4) / 27.0 # step size for log region if mels.ndim: # If we have vector data, vectorize log_t = mels >= min_log_mel freqs[log_t] = min_log_hz * np.exp(logstep * (mels[log_t] - min_log_mel)) elif mels >= min_log_mel: # If we have scalar data, check directly freqs = min_log_hz * np.exp(logstep * (mels - min_log_mel)) return freqs def mel_frequencies(n_mels = 128, *, fmin = 0.0, fmax = 11025.0, htk = False): min_mel = hz_to_mel(fmin, htk=htk) max_mel = hz_to_mel(fmax, htk=htk) mels = np.linspace(min_mel, max_mel, n_mels) hz: np.ndarray = mel_to_hz(mels, htk=htk) return hz def librosa_mel_fn( *, sr: float, n_fft: int, n_mels: int = 128, fmin: float = 0.0, fmax = None, htk = False, norm = "slaney", dtype = np.float32, ) -> np.ndarray: if fmax is None: fmax = float(sr) / 2 # Initialize the weights n_mels = int(n_mels) weights = np.zeros((n_mels, int(1 + n_fft // 2)), dtype=dtype) # Center freqs of each FFT bin fftfreqs = np.fft.rfftfreq(n=n_fft, d=1.0 / sr) # 'Center freqs' of mel bands - uniformly spaced between limits mel_f = mel_frequencies(n_mels + 2, fmin=fmin, fmax=fmax, htk=htk) fdiff = np.diff(mel_f) ramps = np.subtract.outer(mel_f, fftfreqs) for i in range(n_mels): # lower and upper slopes for all bins lower = -ramps[i] / fdiff[i] upper = ramps[i + 2] / fdiff[i + 1] # .. then intersect them with each other and zero weights[i] = np.maximum(0, np.minimum(lower, upper)) # Slaney-style mel is scaled to be approx constant energy per channel enorm = 2.0 / (mel_f[2 : n_mels + 2] - mel_f[:n_mels]) weights *= enorm[:, np.newaxis] return weights def dynamic_range_compression_torch(x, C=1, clip_val=1e-5, *, norm_fn): return norm_fn(torch.clamp(x, min=clip_val) * C) def spectral_normalize_torch(magnitudes, norm_fn): output = dynamic_range_compression_torch(magnitudes, norm_fn=norm_fn) return output class MelConverter(nn.Module): def __init__( self, *, sampling_rate: float, n_fft: int, num_mels: int, hop_size: int, win_size: int, fmin: float, fmax: float, norm_fn, ): super().__init__() self.sampling_rate = sampling_rate self.n_fft = n_fft self.num_mels = num_mels self.hop_size = hop_size self.win_size = win_size self.fmin = fmin self.fmax = fmax self.norm_fn = norm_fn mel = librosa_mel_fn(sr=self.sampling_rate, n_fft=self.n_fft, n_mels=self.num_mels, fmin=self.fmin, fmax=self.fmax) mel_basis = torch.from_numpy(mel).float() hann_window = torch.hann_window(self.win_size) self.register_buffer('mel_basis', mel_basis) self.register_buffer('hann_window', hann_window) @property def device(self): return self.mel_basis.device def forward(self, waveform: torch.Tensor, center: bool = False) -> torch.Tensor: waveform = waveform.clamp(min=-1., max=1.).to(self.device) waveform = torch.nn.functional.pad( waveform.unsqueeze(1), [int((self.n_fft - self.hop_size) / 2), int((self.n_fft - self.hop_size) / 2)], mode='reflect') waveform = waveform.squeeze(1) spec = torch.stft(waveform, self.n_fft, hop_length=self.hop_size, win_length=self.win_size, window=self.hann_window, center=center, pad_mode='reflect', normalized=False, onesided=True, return_complex=True) spec = torch.view_as_real(spec) spec = torch.sqrt(spec.pow(2).sum(-1) + (1e-9)).float() spec = torch.matmul(self.mel_basis, spec) spec = spectral_normalize_torch(spec, self.norm_fn) return spec def get_mel_converter(mode: Literal['16k', '44k']) -> MelConverter: if mode == '16k': return MelConverter(sampling_rate=16_000, n_fft=1024, num_mels=80, hop_size=256, win_size=1024, fmin=0, fmax=8_000, norm_fn=torch.log10) elif mode == '44k': return MelConverter(sampling_rate=44_100, n_fft=2048, num_mels=128, hop_size=512, win_size=2048, fmin=0, fmax=44100 / 2, norm_fn=torch.log) else: raise ValueError(f'Unknown mode: {mode}')