From 15533f3ab94d93fccc580882f47406dcb796a670 Mon Sep 17 00:00:00 2001
From: AIFSH <1509359472@qq.com>
Date: Tue, 15 Oct 2024 10:59:18 +0800
Subject: [PATCH] init commit
---
.gitignore | 1 +
README.md | 11 +-
__init__.py | 275 +
config_24k.json | 123 +
doc/base_workflow.json | 181 +
fireredtts/fireredtts.py | 163 +
fireredtts/modules/__init__.py | 42 +
fireredtts/modules/bigvgan/__init__.py | 6 +
fireredtts/modules/bigvgan/activations.py | 126 +
.../bigvgan/alias_free_cuda/__init__.py | 0
.../bigvgan/alias_free_cuda/activation1d.py | 75 +
.../alias_free_cuda/anti_alias_activation.cpp | 48 +
.../anti_alias_activation_cuda.cu | 314 +
.../modules/bigvgan/alias_free_cuda/compat.h | 31 +
.../modules/bigvgan/alias_free_cuda/load.py | 85 +
.../alias_free_cuda/test_activation.py | 64 +
.../test_activation_snake_beta.py | 64 +
.../bigvgan/alias_free_cuda/type_shim.h | 97 +
.../bigvgan/alias_free_torch/__init__.py | 5 +
.../modules/bigvgan/alias_free_torch/act.py | 29 +
.../bigvgan/alias_free_torch/filter.py | 98 +
.../bigvgan/alias_free_torch/resample.py | 57 +
fireredtts/modules/bigvgan/bigvgan.py | 399 +
fireredtts/modules/codec/speaker.py | 1052 +
fireredtts/modules/flow/__init__.py | 24 +
fireredtts/modules/flow/codebook.npy | Bin 0 -> 131200 bytes
fireredtts/modules/flow/codec_embedding.py | 31 +
fireredtts/modules/flow/conformer.py | 730 +
fireredtts/modules/flow/decoder.py | 396 +
fireredtts/modules/flow/flow_model.py | 89 +
fireredtts/modules/flow/mel_encoder.py | 170 +
fireredtts/modules/flow/mel_spectrogram.py | 132 +
fireredtts/modules/flow/transformer.py | 249 +
fireredtts/modules/flow/utils.py | 30 +
fireredtts/modules/gpt/__init__.py | 0
fireredtts/modules/gpt/gpt.py | 356 +
.../modules/text_normalizer/__init__.py | 0
.../modules/text_normalizer/normalize.py | 178 +
.../modules/text_normalizer/regex_common.py | 23 +
fireredtts/modules/text_normalizer/utils.py | 121 +
fireredtts/modules/tokenizer/__init__.py | 0
.../tokenizer/assets/multilingual.tiktoken | 57184 ++++++++++++++++
fireredtts/modules/tokenizer/tokenizer.py | 46 +
.../modules/tokenizer/whisper_tokenizer.py | 456 +
fireredtts/utils/__init__.py | 0
fireredtts/utils/utils.py | 37 +
requirements.txt | 12 +
47 files changed, 63609 insertions(+), 1 deletion(-)
create mode 100644 .gitignore
create mode 100644 __init__.py
create mode 100644 config_24k.json
create mode 100644 doc/base_workflow.json
create mode 100644 fireredtts/fireredtts.py
create mode 100644 fireredtts/modules/__init__.py
create mode 100644 fireredtts/modules/bigvgan/__init__.py
create mode 100644 fireredtts/modules/bigvgan/activations.py
create mode 100644 fireredtts/modules/bigvgan/alias_free_cuda/__init__.py
create mode 100644 fireredtts/modules/bigvgan/alias_free_cuda/activation1d.py
create mode 100644 fireredtts/modules/bigvgan/alias_free_cuda/anti_alias_activation.cpp
create mode 100644 fireredtts/modules/bigvgan/alias_free_cuda/anti_alias_activation_cuda.cu
create mode 100644 fireredtts/modules/bigvgan/alias_free_cuda/compat.h
create mode 100644 fireredtts/modules/bigvgan/alias_free_cuda/load.py
create mode 100644 fireredtts/modules/bigvgan/alias_free_cuda/test_activation.py
create mode 100644 fireredtts/modules/bigvgan/alias_free_cuda/test_activation_snake_beta.py
create mode 100644 fireredtts/modules/bigvgan/alias_free_cuda/type_shim.h
create mode 100644 fireredtts/modules/bigvgan/alias_free_torch/__init__.py
create mode 100644 fireredtts/modules/bigvgan/alias_free_torch/act.py
create mode 100644 fireredtts/modules/bigvgan/alias_free_torch/filter.py
create mode 100644 fireredtts/modules/bigvgan/alias_free_torch/resample.py
create mode 100644 fireredtts/modules/bigvgan/bigvgan.py
create mode 100644 fireredtts/modules/codec/speaker.py
create mode 100644 fireredtts/modules/flow/__init__.py
create mode 100644 fireredtts/modules/flow/codebook.npy
create mode 100644 fireredtts/modules/flow/codec_embedding.py
create mode 100644 fireredtts/modules/flow/conformer.py
create mode 100644 fireredtts/modules/flow/decoder.py
create mode 100644 fireredtts/modules/flow/flow_model.py
create mode 100644 fireredtts/modules/flow/mel_encoder.py
create mode 100644 fireredtts/modules/flow/mel_spectrogram.py
create mode 100644 fireredtts/modules/flow/transformer.py
create mode 100644 fireredtts/modules/flow/utils.py
create mode 100644 fireredtts/modules/gpt/__init__.py
create mode 100644 fireredtts/modules/gpt/gpt.py
create mode 100644 fireredtts/modules/text_normalizer/__init__.py
create mode 100644 fireredtts/modules/text_normalizer/normalize.py
create mode 100644 fireredtts/modules/text_normalizer/regex_common.py
create mode 100644 fireredtts/modules/text_normalizer/utils.py
create mode 100644 fireredtts/modules/tokenizer/__init__.py
create mode 100644 fireredtts/modules/tokenizer/assets/multilingual.tiktoken
create mode 100644 fireredtts/modules/tokenizer/tokenizer.py
create mode 100644 fireredtts/modules/tokenizer/whisper_tokenizer.py
create mode 100644 fireredtts/utils/__init__.py
create mode 100644 fireredtts/utils/utils.py
create mode 100644 requirements.txt
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..ed8ebf5
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+__pycache__
\ No newline at end of file
diff --git a/README.md b/README.md
index 35bae4c..06de30d 100644
--- a/README.md
+++ b/README.md
@@ -1,2 +1,11 @@
# FireRedTTS-ComfyUI
-a custom node for [FireRedTTS](https://github.com/FireRedTeam/FireRedTTS)
+a custom node for [FireRedTTS](https://github.com/FireRedTeam/FireRedTTS),you can find [workflow here](./doc/base_workflow.json)
+
+## Example
+|text|prompt_wav|out_wav|
+|--|--|--|
+|`《三体》是刘慈欣创作的长篇科幻小说系列，由《三体》《三体2：黑暗森林》《三体3：死神永生》组成，第一部于2006年5月起在《科幻世界》杂志上连载，第二部于2008年5月首次出版，第三部则于2010年11月出版。作品讲述了地球人类文明和三体文明的信息交流、生死搏杀及两个文明在宇宙中的兴衰历程。其第一部经过刘宇昆翻译后获得了第73届雨果奖最佳长篇小说奖，第三部英文版获得2017年轨迹奖最佳长篇科幻小说奖。2019年，列入“新中国70年70部长篇小说典藏”。2022年9月，《三体》入选2021十大年度国家IP。`| ||
+
+## Features
+- speed control
+- auto split text
diff --git a/__init__.py b/__init__.py
new file mode 100644
index 0000000..24d75ac
--- /dev/null
+++ b/__init__.py
@@ -0,0 +1,275 @@
+import os,sys
+import os.path as osp
+now_dir = osp.dirname(osp.abspath(__file__))
+sys.path.append(now_dir)
+tmp_dir = osp.join(now_dir, "tmp")
+import re
+import torch
+import ffmpeg
+import shutil
+import tempfile
+import torchaudio
+import folder_paths
+import numpy as np
+from tqdm import tqdm
+from comfy.utils import ProgressBar
+from pydub import AudioSegment
+from pydub.silence import split_on_silence
+from huggingface_hub import snapshot_download
+from fireredtts.fireredtts import FireRedTTS
+from zhon.hanzi import punctuation
+SPLIT_WORDS = [
+ "but", "however", "nevertheless", "yet", "still",
+ "therefore", "thus", "hence", "consequently",
+ "moreover", "furthermore", "additionally",
+ "meanwhile", "alternatively", "otherwise",
+ "namely", "specifically", "for example", "such as",
+ "in fact", "indeed", "notably",
+ "in contrast", "on the other hand", "conversely",
+ "in conclusion", "to summarize", "finally"
+]
+device = (
+ "cuda"
+ if torch.cuda.is_available()
+ else "mps" if torch.backends.mps.is_available() else "cpu"
+)
+aifsh_models = osp.join(folder_paths.models_dir,"AIFSH")
+fireredtss_dir = osp.join(aifsh_models,"FireRedTTS")
+
+def speed_change(input_audio, speed, sr):
+ # 检查输入数据类型和声道数
+ if input_audio.dtype != np.int16:
+ raise ValueError("输入音频数据类型必须为 np.int16")
+
+
+ # 转换为字节流
+ raw_audio = input_audio.astype(np.int16).tobytes()
+
+ # 设置 ffmpeg 输入流
+ input_stream = ffmpeg.input('pipe:', format='s16le', acodec='pcm_s16le', ar=str(sr), ac=1)
+
+ # 变速处理
+ output_stream = input_stream.filter('atempo', speed)
+
+ # 输出流到管道
+ out, _ = (
+ output_stream.output('pipe:', format='s16le', acodec='pcm_s16le')
+ .run(input=raw_audio, capture_stdout=True, capture_stderr=True)
+ )
+
+ # 将管道输出解码为 NumPy 数组
+ processed_audio = np.frombuffer(out, np.int16)
+
+ return processed_audio
+
+class FireRedTTSNode:
+ def __init__(self):
+ if not osp.exists(osp.join(fireredtss_dir,"fireredtts_gpt.pt")):
+ snapshot_download(repo_id="fireredteam/FireRedTTS",local_dir=fireredtss_dir)
+ @classmethod
+ def INPUT_TYPES(s):
+
+ return {
+ "required":{
+ "text":("TEXT",),
+ "prompt_wav":("AUDIO",),
+ "remove_slience":("BOOLEAN",{
+ "default": True
+ }),
+ "speed":("FLOAT",{
+ "default":1.0,
+ "min":0.5,
+ "max":2.0,
+ "step":0.05,
+ "round":0.001,
+ "display":"slider"
+ }),
+ "split_words":("STRING",{
+ "default":",".join(SPLIT_WORDS),
+ "multiline": True,
+ "dynamicPrompts": True,
+ "tooltip":"Enter custom words to split on, separated by commas. Leave blank to use default list.",
+ })
+ }
+ }
+
+ RETURN_TYPES = ("AUDIO",)
+ #RETURN_NAMES = ("image_output_name",)
+
+ FUNCTION = "gen_audio"
+
+ #OUTPUT_NODE = False
+
+ CATEGORY = "AIFSH_FireRedTSS"
+
+ def gen_audio(self,text,prompt_wav,remove_slience,speed,split_words):
+ os.makedirs(tmp_dir, exist_ok=True)
+ with tempfile.NamedTemporaryFile(delete=False, suffix=".wav",dir=tmp_dir) as f:
+ waveform = prompt_wav["waveform"].squeeze(0)
+ torchaudio.save(f.name,waveform,prompt_wav["sample_rate"])
+ if remove_slience:
+ combined = slice(f.name)
+ combined.export(f.name, format="wav")
+ prompt_wav = f.name
+ # Split the input text into batches
+ if len(text.encode('utf-8')) == len(text):
+ max_chars = 400-len(text.encode('utf-8'))
+ else:
+ max_chars = 300-len(text.encode('utf-8'))
+
+ if not split_words.strip():
+ custom_words = [word.strip() for word in split_words.split(',')]
+ global SPLIT_WORDS
+ SPLIT_WORDS = custom_words
+ gen_text_batches = split_text_into_batches(text, max_chars=max_chars)
+ comfy_par = ProgressBar(len(gen_text_batches))
+ tts = FireRedTTS(config_path=osp.join(now_dir,"config_24k.json"),
+ pretrained_path=fireredtss_dir,device=device)
+ rec_wavs_list = []
+ for i,i_text in enumerate(tqdm(gen_text_batches,total=len(gen_text_batches),desc="TTS ...")):
+ for dot in punctuation:
+ i_text = i_text.replace(dot,"")
+ print(f"sentence {i+1}, synthesize text:{i_text}")
+ rec_wavs = tts.synthesize(prompt_wav=prompt_wav,text=i_text)
+ rec_wavs = rec_wavs.detach().cpu().numpy()
+ rec_wavs_list.append(rec_wavs)
+ comfy_par.update(1)
+ res_np = np.concatenate(rec_wavs_list,axis=1)
+ if speed > 1.0 or speed < 1.0:
+ res_np = res_np * 32768
+ res_np = res_np.astype(np.int16)
+ res_np = speed_change(res_np,speed,sr=24000)
+ waveform = torch.from_numpy(res_np/32768).unsqueeze(0).unsqueeze(0)
+ else:
+ waveform = torch.from_numpy(res_np).unsqueeze(0)
+ print(waveform.shape)
+ res_audio = {
+ "waveform": waveform,
+ "sample_rate": 24000
+ }
+ shutil.rmtree(tmp_dir)
+ return (res_audio, )
+
+def split_text_into_batches(text, max_chars=200, split_words=SPLIT_WORDS):
+ if len(text.encode('utf-8')) <= max_chars:
+ return [text]
+ if text[-1] not in ['。', '.', '!', '！', '?', '？']:
+ text += '.'
+
+ sentences = re.split('([。.!?！？])', text)
+ sentences = [''.join(i) for i in zip(sentences[0::2], sentences[1::2])]
+
+ batches = []
+ current_batch = ""
+
+ def split_by_words(text):
+ words = text.split()
+ current_word_part = ""
+ word_batches = []
+ for word in words:
+ if len(current_word_part.encode('utf-8')) + len(word.encode('utf-8')) + 1 <= max_chars:
+ current_word_part += word + ' '
+ else:
+ if current_word_part:
+ # Try to find a suitable split word
+ for split_word in split_words:
+ split_index = current_word_part.rfind(' ' + split_word + ' ')
+ if split_index != -1:
+ word_batches.append(current_word_part[:split_index].strip())
+ current_word_part = current_word_part[split_index:].strip() + ' '
+ break
+ else:
+ # If no suitable split word found, just append the current part
+ word_batches.append(current_word_part.strip())
+ current_word_part = ""
+ current_word_part += word + ' '
+ if current_word_part:
+ word_batches.append(current_word_part.strip())
+ return word_batches
+
+ for sentence in sentences:
+ if len(current_batch.encode('utf-8')) + len(sentence.encode('utf-8')) <= max_chars:
+ current_batch += sentence
+ else:
+ # If adding this sentence would exceed the limit
+ if current_batch:
+ batches.append(current_batch)
+ current_batch = ""
+
+ # If the sentence itself is longer than max_chars, split it
+ if len(sentence.encode('utf-8')) > max_chars:
+ # First, try to split by colon
+ colon_parts = sentence.split(':')
+ if len(colon_parts) > 1:
+ for part in colon_parts:
+ if len(part.encode('utf-8')) <= max_chars:
+ batches.append(part)
+ else:
+ # If colon part is still too long, split by comma
+ comma_parts = re.split('[,，]', part)
+ if len(comma_parts) > 1:
+ current_comma_part = ""
+ for comma_part in comma_parts:
+ if len(current_comma_part.encode('utf-8')) + len(comma_part.encode('utf-8')) <= max_chars:
+ current_comma_part += comma_part + ','
+ else:
+ if current_comma_part:
+ batches.append(current_comma_part.rstrip(','))
+ current_comma_part = comma_part + ','
+ if current_comma_part:
+ batches.append(current_comma_part.rstrip(','))
+ else:
+ # If no comma, split by words
+ batches.extend(split_by_words(part))
+ else:
+ # If no colon, split by comma
+ comma_parts = re.split('[,，]', sentence)
+ if len(comma_parts) > 1:
+ current_comma_part = ""
+ for comma_part in comma_parts:
+ if len(current_comma_part.encode('utf-8')) + len(comma_part.encode('utf-8')) <= max_chars:
+ current_comma_part += comma_part + ','
+ else:
+ if current_comma_part:
+ batches.append(current_comma_part.rstrip(','))
+ current_comma_part = comma_part + ','
+ if current_comma_part:
+ batches.append(current_comma_part.rstrip(','))
+ else:
+ # If no comma, split by words
+ batches.extend(split_by_words(sentence))
+ else:
+ current_batch = sentence
+
+ if current_batch:
+ batches.append(current_batch)
+
+ return batches
+
+
+def slice(audio_path):
+ """_summary_
+
+ Args:
+ audio_path (_type_): audio path
+ """
+ try:
+ audio = AudioSegment.from_file(audio_path)
+ except:
+ print(audio_path)
+ return 0
+
+ segments = split_on_silence(
+ audio, min_silence_len=200, silence_thresh=-50, seek_step=100, keep_silence=100
+ )
+
+ print("---segments:\n", segments)
+
+ combined = segments[0]
+ for i in range(1, len(segments)):
+ combined += segments[i]
+ return combined
+
+NODE_CLASS_MAPPINGS = {
+ "FireRedTTSNode": FireRedTTSNode
+}
\ No newline at end of file
diff --git a/config_24k.json b/config_24k.json
new file mode 100644
index 0000000..1b81b5f
--- /dev/null
+++ b/config_24k.json
@@ -0,0 +1,123 @@
+{
+ "gpt": {
+ "gpt_max_audio_tokens": 630,
+ "gpt_max_text_tokens": 402,
+ "gpt_max_prompt_tokens": 70,
+ "gpt_layers": 30,
+ "gpt_n_model_channels": 1024,
+ "gpt_n_heads": 16,
+ "gpt_number_text_tokens": 57341,
+ "gpt_start_text_token": 57187,
+ "gpt_stop_text_token": 57184,
+ "gpt_num_audio_tokens": 16386,
+ "gpt_start_audio_token": 16384,
+ "gpt_stop_audio_token": 16385,
+ "gpt_code_stride_len": 640,
+ "duration_const": 102400,
+ "min_conditioning_length": 48000,
+ "max_conditioning_length": 128000,
+ "max_wav_length": 320000,
+ "max_text_length": 200
+ },
+ "flow": {
+ "output_size": 100,
+ "input_embedding": {
+ "out_channels": 512,
+ "codebook_path": "fireredtts/modules/flow/codebook.npy",
+ "freeze": true
+ },
+ "encoder": {
+ "input_size": 512,
+ "output_size": 512,
+ "attention_heads": 8,
+ "linear_units": 2048,
+ "num_blocks": 6,
+ "dropout_rate": 0.01,
+ "srcattention_start_index": 0,
+ "srcattention_end_index": 2,
+ "attention_dropout_rate": 0.01,
+ "positional_dropout_rate": 0.01,
+ "key_bias": true,
+ "normalize_before": true
+ },
+ "length_regulator": {
+ "channels": 512,
+ "num_blocks": 4
+ },
+ "mel_encoder": {
+ "in_channels": 100,
+ "out_channels": 512,
+ "hidden_channels": 384,
+ "reduction_rate": 4,
+ "n_layers": 2,
+ "n_blocks": 5,
+ "kernel_size": 3
+ },
+ "decoder": {
+ "t_scheduler": "cosine",
+ "inference_cfg_rate": 0.7,
+ "estimator": {
+ "in_channels": 200,
+ "out_channels": 100,
+ "channels": [
+ 256,
+ 256
+ ],
+ "dropout": 0,
+ "attention_head_dim": 64,
+ "n_blocks": 4,
+ "num_mid_blocks": 12,
+ "num_heads": 8,
+ "act_fn": "gelu"
+ }
+ }
+ },
+ "bigvgan": {
+ "num_mels": 100,
+ "upsample_initial_channel": 1536,
+ "upsample_rates": [
+ 5,
+ 3,
+ 2,
+ 2,
+ 2,
+ 2
+ ],
+ "upsample_kernel_sizes": [
+ 11,
+ 7,
+ 4,
+ 4,
+ 4,
+ 4
+ ],
+ "resblock_kernel_sizes": [
+ 3,
+ 7,
+ 11
+ ],
+ "resblock_dilation_sizes": [
+ [
+ 1,
+ 3,
+ 5
+ ],
+ [
+ 1,
+ 3,
+ 5
+ ],
+ [
+ 1,
+ 3,
+ 5
+ ]
+ ],
+ "resblock_type": "1",
+ "snake_logscale": true,
+ "activation": "snakebeta",
+ "use_tanh_at_final": false,
+ "use_bias_at_final": false,
+ "use_cuda_kernel": false
+ }
+}
\ No newline at end of file
diff --git a/doc/base_workflow.json b/doc/base_workflow.json
new file mode 100644
index 0000000..3ee2b40
--- /dev/null
+++ b/doc/base_workflow.json
@@ -0,0 +1,181 @@
+{
+ "last_node_id": 4,
+ "last_link_id": 3,
+ "nodes": [
+ {
+ "id": 3,
+ "type": "LoadAudio",
+ "pos": {
+ "0": 94,
+ "1": 383
+ },
+ "size": {
+ "0": 315,
+ "1": 124
+ },
+ "flags": {},
+ "order": 0,
+ "mode": 0,
+ "inputs": [],
+ "outputs": [
+ {
+ "name": "AUDIO",
+ "type": "AUDIO",
+ "links": [
+ 2
+ ]
+ }
+ ],
+ "properties": {
+ "Node name for S&R": "LoadAudio"
+ },
+ "widgets_values": [
+ "test_zh_1_ref_short.wav",
+ null,
+ ""
+ ]
+ },
+ {
+ "id": 2,
+ "type": "PromptTextNode",
+ "pos": {
+ "0": 26,
+ "1": 73
+ },
+ "size": {
+ "0": 400,
+ "1": 200
+ },
+ "flags": {},
+ "order": 1,
+ "mode": 0,
+ "inputs": [],
+ "outputs": [
+ {
+ "name": "TEXT",
+ "type": "TEXT",
+ "links": [
+ 1
+ ]
+ }
+ ],
+ "properties": {
+ "Node name for S&R": "PromptTextNode"
+ },
+ "widgets_values": [
+ "《三体》是刘慈欣创作的长篇科幻小说系列，由《三体》《三体2：黑暗森林》《三体3：死神永生》组成，第一部于2006年5月起在《科幻世界》杂志上连载，第二部于2008年5月首次出版，第三部则于2010年11月出版。\n作品讲述了地球人类文明和三体文明的信息交流、生死搏杀及两个文明在宇宙中的兴衰历程。其第一部经过刘宇昆翻译后获得了第73届雨果奖最佳长篇小说奖，第三部英文版获得2017年轨迹奖最佳长篇科幻小说奖。2019年，列入“新中国70年70部长篇小说典藏”。2022年9月，《三体》入选2021十大年度国家IP。"
+ ]
+ },
+ {
+ "id": 1,
+ "type": "FireRedTTSNode",
+ "pos": {
+ "0": 504,
+ "1": 86
+ },
+ "size": {
+ "0": 309.60003662109375,
+ "1": 210.80001831054688
+ },
+ "flags": {},
+ "order": 2,
+ "mode": 0,
+ "inputs": [
+ {
+ "name": "text",
+ "type": "TEXT",
+ "link": 1
+ },
+ {
+ "name": "prompt_wav",
+ "type": "AUDIO",
+ "link": 2
+ }
+ ],
+ "outputs": [
+ {
+ "name": "AUDIO",
+ "type": "AUDIO",
+ "links": [
+ 3
+ ],
+ "slot_index": 0
+ }
+ ],
+ "properties": {
+ "Node name for S&R": "FireRedTTSNode"
+ },
+ "widgets_values": [
+ true,
+ "but,however,nevertheless,yet,still,therefore,thus,hence,consequently,moreover,furthermore,additionally,meanwhile,alternatively,otherwise,namely,specifically,for example,such as,in fact,indeed,notably,in contrast,on the other hand,conversely,in conclusion,to summarize,finally"
+ ]
+ },
+ {
+ "id": 4,
+ "type": "PreviewAudio",
+ "pos": {
+ "0": 865,
+ "1": 93
+ },
+ "size": {
+ "0": 315,
+ "1": 76
+ },
+ "flags": {},
+ "order": 3,
+ "mode": 0,
+ "inputs": [
+ {
+ "name": "audio",
+ "type": "AUDIO",
+ "link": 3
+ }
+ ],
+ "outputs": [],
+ "properties": {
+ "Node name for S&R": "PreviewAudio"
+ },
+ "widgets_values": [
+ null
+ ]
+ }
+ ],
+ "links": [
+ [
+ 1,
+ 2,
+ 0,
+ 1,
+ 0,
+ "TEXT"
+ ],
+ [
+ 2,
+ 3,
+ 0,
+ 1,
+ 1,
+ "AUDIO"
+ ],
+ [
+ 3,
+ 1,
+ 0,
+ 4,
+ 0,
+ "AUDIO"
+ ]
+ ],
+ "groups": [],
+ "config": {},
+ "extra": {
+ "ds": {
+ "scale": 1,
+ "offset": [
+ 0,
+ 0
+ ]
+ }
+ },
+ "version": 0.4
+}
\ No newline at end of file
diff --git a/fireredtts/fireredtts.py b/fireredtts/fireredtts.py
new file mode 100644
index 0000000..876ef9e
--- /dev/null
+++ b/fireredtts/fireredtts.py
@@ -0,0 +1,163 @@
+import os
+import json
+import torch
+from fireredtts.modules.gpt.gpt import GPT
+from fireredtts.modules import Token2Wav, MelSpectrogramExtractor
+from fireredtts.modules.tokenizer.tokenizer import VoiceBpeTokenizer
+from fireredtts.modules.codec.speaker import SpeakerEmbedddingExtractor
+from fireredtts.utils.utils import load_audio
+
+import time
+
+
+class FireRedTTS:
+ def __init__(self, config_path, pretrained_path, device="cuda"):
+ self.device = device
+ self.config = json.load(open(config_path))
+ self.gpt_path = os.path.join(pretrained_path, "fireredtts_gpt.pt")
+ self.token2wav_path = os.path.join(pretrained_path, "fireredtts_token2wav.pt")
+ self.speaker_extractor_path = os.path.join(
+ pretrained_path, "fireredtts_speaker.bin"
+ )
+ assert os.path.exists(self.token2wav_path)
+ assert os.path.exists(self.gpt_path)
+ assert os.path.exists(self.speaker_extractor_path)
+
+ # tokenizer;
+ self.text_tokenizer = VoiceBpeTokenizer()
+
+ # speaker ectractor
+ self.speaker_extractor = SpeakerEmbedddingExtractor(
+ ckpt_path=self.speaker_extractor_path, device=device
+ )
+
+ # load gpt model
+ self.gpt = GPT(
+ start_text_token=self.config["gpt"]["gpt_start_text_token"],
+ stop_text_token=self.config["gpt"]["gpt_stop_text_token"],
+ layers=self.config["gpt"]["gpt_layers"],
+ model_dim=self.config["gpt"]["gpt_n_model_channels"],
+ heads=self.config["gpt"]["gpt_n_heads"],
+ max_text_tokens=self.config["gpt"]["gpt_max_text_tokens"],
+ max_mel_tokens=self.config["gpt"]["gpt_max_audio_tokens"],
+ max_prompt_tokens=self.config["gpt"]["gpt_max_prompt_tokens"],
+ code_stride_len=self.config["gpt"]["gpt_code_stride_len"],
+ number_text_tokens=self.config["gpt"]["gpt_number_text_tokens"],
+ num_audio_tokens=self.config["gpt"]["gpt_num_audio_tokens"],
+ start_audio_token=self.config["gpt"]["gpt_start_audio_token"],
+ stop_audio_token=self.config["gpt"]["gpt_stop_audio_token"],
+ )
+
+ sd = torch.load(self.gpt_path, map_location=device)["model"]
+ self.gpt.load_state_dict(sd, strict=True)
+ self.gpt = self.gpt.to(device=device)
+ self.gpt.eval()
+ self.gpt.init_gpt_for_inference(kv_cache=True)
+
+ # mel-spectrogram extractor
+ self.mel_extractor = MelSpectrogramExtractor()
+
+ # load token2wav model
+ self.token2wav = Token2Wav.init_from_config(self.config)
+ sd = torch.load(self.token2wav_path, map_location="cpu")
+ self.token2wav.load_state_dict(sd, strict=True)
+ self.token2wav.generator.remove_weight_norm()
+ self.token2wav.eval()
+ self.token2wav = self.token2wav.to(device)
+
+ def extract_spk_embeddings(self, prompt_wav):
+ _, _, audio_resampled = load_audio(audiopath=prompt_wav, sampling_rate=16000)
+ audio_len = torch.tensor(
+ data=[audio_resampled.shape[1]], dtype=torch.long, requires_grad=False
+ )
+
+ # speaker embeddings [1,512]
+ spk_embeddings = self.speaker_extractor(
+ audio_resampled.to(device="cuda")
+ ).unsqueeze(0)
+
+ return spk_embeddings
+
+ def do_gpt_inference(self, spk_gpt, text_tokens):
+ """_summary_
+
+ Args:
+ spk_gpt (_type_): speaker embeddidng in gpt
+ text_tokens (_type_): text tokens
+ """
+ with torch.no_grad():
+ gpt_codes = self.gpt.generate(
+ cond_latents=spk_gpt,
+ text_inputs=text_tokens,
+ input_tokens=None,
+ do_sample=True,
+ top_p=0.85,
+ top_k=30,
+ temperature=0.75,
+ num_return_sequences=9,
+ num_beams=1,
+ length_penalty=1.0,
+ repetition_penalty=2.0,
+ output_attentions=False,
+ )
+
+ seqs = []
+ EOS_TOKEN = self.config["gpt"]["gpt_stop_audio_token"]
+ for seq in gpt_codes:
+ index = (seq == EOS_TOKEN).nonzero(as_tuple=True)[0][0]
+ seq = seq[:index]
+ seqs.append(seq)
+
+ sorted_seqs = sorted(seqs, key=lambda i: len(i), reverse=False)
+ gpt_codes = sorted_seqs[2].unsqueeze(0) # [1, len]
+ # sorted_len = [len(l) for l in sorted_seqs]
+ # print("---sorted_len:", sorted_len)
+
+ return gpt_codes
+
+ def synthesize(self, prompt_wav, text, lang="auto"):
+ """_summary_
+
+ Args:
+ prompts_wav (_type_): prompts_wav path
+ text (_type_): text
+ lang (_type_): language of text
+ """
+ # Currently only supports Chinese and English
+ assert lang in ["zh", "en", "auto"]
+ assert os.path.exists(prompt_wav)
+
+ # text to tokens
+ text_tokens = self.text_tokenizer.encode(text=text, lang=lang)
+ text_tokens = torch.IntTensor(text_tokens).unsqueeze(0).to(self.device)
+ assert text_tokens.shape[-1] < 400
+
+ # extract speaker embedding
+ spk_embeddings = self.extract_spk_embeddings(prompt_wav=prompt_wav).unsqueeze(0)
+ with torch.no_grad():
+ spk_gpt = self.gpt.reference_embedding(spk_embeddings)
+
+ # gpt inference
+ gpt_start_time = time.time()
+ gpt_codes = self.do_gpt_inference(spk_gpt=spk_gpt, text_tokens=text_tokens)
+ gpt_end_time = time.time()
+ gpt_dur = gpt_end_time - gpt_start_time
+
+ # prompt mel-spectrogram compute
+ prompt_mel = (
+ self.mel_extractor(wav_path=prompt_wav).unsqueeze(0).to(self.device)
+ )
+ # convert token to waveform (b=1, t)
+ voc_start_time = time.time()
+ rec_wavs = self.token2wav.inference(gpt_codes, prompt_mel, n_timesteps=10)
+ voc_end_time = time.time()
+ voc_dur = voc_end_time - voc_start_time
+ all_dur = voc_end_time - gpt_start_time
+
+ # rtf compute
+ # audio_dur = rec_wavs.shape[-1] / 24000
+ # rtf_gpt = gpt_dur / audio_dur
+ # rtf_voc = voc_dur / audio_dur
+ # rtf_all = all_dur / audio_dur
+
+ return rec_wavs
diff --git a/fireredtts/modules/__init__.py b/fireredtts/modules/__init__.py
new file mode 100644
index 0000000..bfbf67d
--- /dev/null
+++ b/fireredtts/modules/__init__.py
@@ -0,0 +1,42 @@
+import json
+import torch
+import torch.nn as nn
+from fireredtts.modules.bigvgan import get_bigvgan_backend
+from fireredtts.modules.flow import get_flow_frontend, MelSpectrogramExtractor
+
+
+class Token2Wav(nn.Module):
+ def __init__(
+ self,
+ flow: nn.Module,
+ generator: nn.Module,
+ ):
+ super().__init__()
+ self.flow = flow
+ self.generator = generator
+
+ @torch.no_grad()
+ def inference(
+ self, tokens: torch.Tensor, prompt_mel: torch.Tensor, n_timesteps: int = 10
+ ) -> torch.Tensor:
+ token_len = torch.tensor([tokens.shape[1]], dtype=torch.long).to(tokens.device)
+ prompt_mel_len = torch.tensor([prompt_mel.shape[1]], dtype=torch.long).to(
+ prompt_mel.device
+ )
+ # flow
+ mel = self.flow.inference(
+ token=tokens,
+ token_len=token_len,
+ prompt_mel=prompt_mel,
+ prompt_mel_len=prompt_mel_len,
+ n_timesteps=n_timesteps,
+ )
+ # bigvgan
+ audio = self.generator(mel) # (b=1, 1, t)
+ return audio.squeeze(1)
+
+ @classmethod
+ def init_from_config(cls, config) -> "Token2Wav":
+ flow = get_flow_frontend(config["flow"])
+ bigvgan = get_bigvgan_backend(config["bigvgan"])
+ return cls(flow, bigvgan)
diff --git a/fireredtts/modules/bigvgan/__init__.py b/fireredtts/modules/bigvgan/__init__.py
new file mode 100644
index 0000000..88da4b2
--- /dev/null
+++ b/fireredtts/modules/bigvgan/__init__.py
@@ -0,0 +1,6 @@
+from fireredtts.modules.bigvgan.bigvgan import BigVGAN
+
+
+def get_bigvgan_backend(bigvgan_config):
+ generator = BigVGAN(**bigvgan_config)
+ return generator
diff --git a/fireredtts/modules/bigvgan/activations.py b/fireredtts/modules/bigvgan/activations.py
new file mode 100644
index 0000000..1e8478a
--- /dev/null
+++ b/fireredtts/modules/bigvgan/activations.py
@@ -0,0 +1,126 @@
+# Implementation adapted from https://github.com/EdwardDixon/snake under the MIT license.
+
+
+import torch
+from torch import nn, sin, pow
+from torch.nn import Parameter
+
+
+class Snake(nn.Module):
+ """
+ Implementation of a sine-based periodic activation function
+ Shape:
+ - Input: (B, C, T)
+ - Output: (B, C, T), same shape as the input
+ Parameters:
+ - alpha - trainable parameter
+ References:
+ - This activation function is from this paper by Liu Ziyin, Tilman Hartwig, Masahito Ueda:
+ https://arxiv.org/abs/2006.08195
+ Examples:
+ >>> a1 = snake(256)
+ >>> x = torch.randn(256)
+ >>> x = a1(x)
+ """
+
+ def __init__(
+ self, in_features, alpha=1.0, alpha_trainable=True, alpha_logscale=False
+ ):
+ """
+ Initialization.
+ INPUT:
+ - in_features: shape of the input
+ - alpha: trainable parameter
+ alpha is initialized to 1 by default, higher values = higher-frequency.
+ alpha will be trained along with the rest of your model.
+ """
+ super(Snake, self).__init__()
+ self.in_features = in_features
+
+ # initialize alpha
+ self.alpha_logscale = alpha_logscale
+ if self.alpha_logscale: # log scale alphas initialized to zeros
+ self.alpha = Parameter(torch.zeros(in_features) * alpha)
+ else: # linear scale alphas initialized to ones
+ self.alpha = Parameter(torch.ones(in_features) * alpha)
+
+ self.alpha.requires_grad = alpha_trainable
+
+ self.no_div_by_zero = 0.000000001
+
+ def forward(self, x):
+ """
+ Forward pass of the function.
+ Applies the function to the input elementwise.
+ Snake ∶= x + 1/a * sin^2 (xa)
+ """
+ alpha = self.alpha.unsqueeze(0).unsqueeze(-1) # line up with x to [B, C, T]
+ if self.alpha_logscale:
+ alpha = torch.exp(alpha)
+ x = x + (1.0 / (alpha + self.no_div_by_zero)) * pow(sin(x * alpha), 2)
+
+ return x
+
+
+class SnakeBeta(nn.Module):
+ """
+ A modified Snake function which uses separate parameters for the magnitude of the periodic components
+ Shape:
+ - Input: (B, C, T)
+ - Output: (B, C, T), same shape as the input
+ Parameters:
+ - alpha - trainable parameter that controls frequency
+ - beta - trainable parameter that controls magnitude
+ References:
+ - This activation function is a modified version based on this paper by Liu Ziyin, Tilman Hartwig, Masahito Ueda:
+ https://arxiv.org/abs/2006.08195
+ Examples:
+ >>> a1 = snakebeta(256)
+ >>> x = torch.randn(256)
+ >>> x = a1(x)
+ """
+
+ def __init__(
+ self, in_features, alpha=1.0, alpha_trainable=True, alpha_logscale=False
+ ):
+ """
+ Initialization.
+ INPUT:
+ - in_features: shape of the input
+ - alpha - trainable parameter that controls frequency
+ - beta - trainable parameter that controls magnitude
+ alpha is initialized to 1 by default, higher values = higher-frequency.
+ beta is initialized to 1 by default, higher values = higher-magnitude.
+ alpha will be trained along with the rest of your model.
+ """
+ super(SnakeBeta, self).__init__()
+ self.in_features = in_features
+
+ # initialize alpha
+ self.alpha_logscale = alpha_logscale
+ if self.alpha_logscale: # log scale alphas initialized to zeros
+ self.alpha = Parameter(torch.zeros(in_features) * alpha)
+ self.beta = Parameter(torch.zeros(in_features) * alpha)
+ else: # linear scale alphas initialized to ones
+ self.alpha = Parameter(torch.ones(in_features) * alpha)
+ self.beta = Parameter(torch.ones(in_features) * alpha)
+
+ self.alpha.requires_grad = alpha_trainable
+ self.beta.requires_grad = alpha_trainable
+
+ self.no_div_by_zero = 0.000000001
+
+ def forward(self, x):
+ """
+ Forward pass of the function.
+ Applies the function to the input elementwise.
+ SnakeBeta ∶= x + 1/b * sin^2 (xa)
+ """
+ alpha = self.alpha.unsqueeze(0).unsqueeze(-1) # line up with x to [B, C, T]
+ beta = self.beta.unsqueeze(0).unsqueeze(-1)
+ if self.alpha_logscale:
+ alpha = torch.exp(alpha)
+ beta = torch.exp(beta)
+ x = x + (1.0 / (beta + self.no_div_by_zero)) * pow(sin(x * alpha), 2)
+
+ return x
diff --git a/fireredtts/modules/bigvgan/alias_free_cuda/__init__.py b/fireredtts/modules/bigvgan/alias_free_cuda/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/fireredtts/modules/bigvgan/alias_free_cuda/activation1d.py b/fireredtts/modules/bigvgan/alias_free_cuda/activation1d.py
new file mode 100644
index 0000000..9c7c540
--- /dev/null
+++ b/fireredtts/modules/bigvgan/alias_free_cuda/activation1d.py
@@ -0,0 +1,75 @@
+# Copyright (c) 2024 NVIDIA CORPORATION.
+# Licensed under the MIT license.
+
+import torch
+import torch.nn as nn
+from token2wav.alias_free_torch.resample import UpSample1d, DownSample1d
+
+# load fused CUDA kernel: this enables importing anti_alias_activation_cuda
+from token2wav.alias_free_cuda import load
+
+load.load()
+
+
+class FusedAntiAliasActivation(torch.autograd.Function):
+ """
+ Assumes filter size 12, replication padding on upsampling, and logscale alpha/beta parameters as inputs
+ """
+
+ @staticmethod
+ def forward(ctx, inputs, ftr, alpha, beta):
+ import anti_alias_activation_cuda
+
+ activation_results = anti_alias_activation_cuda.forward(
+ inputs, ftr, alpha, beta
+ )
+ return activation_results
+
+ @staticmethod
+ def backward(ctx, output_grads):
+ # TODO: implement bwd pass
+ raise NotImplementedError
+ return output_grads, None, None
+
+
+class Activation1d(nn.Module):
+ def __init__(
+ self,
+ activation,
+ up_ratio: int = 2,
+ down_ratio: int = 2,
+ up_kernel_size: int = 12,
+ down_kernel_size: int = 12,
+ fused: bool = True,
+ ):
+ super().__init__()
+ self.up_ratio = up_ratio
+ self.down_ratio = down_ratio
+ self.act = activation
+ self.upsample = UpSample1d(up_ratio, up_kernel_size)
+ self.downsample = DownSample1d(down_ratio, down_kernel_size)
+
+ self.fused = fused # whether to use fused CUDA kernel or not
+
+ def forward(self, x):
+ if not self.fused:
+ x = self.upsample(x)
+ x = self.act(x)
+ x = self.downsample(x)
+ return x
+ else:
+ if self.act.__class__.__name__ == "Snake":
+ beta = self.act.alpha.data # snake uses same params for alpha and beta
+ else:
+ beta = (
+ self.act.beta.data
+ ) # snakebeta uses different params for alpha and beta
+ alpha = self.act.alpha.data
+ if (
+ not self.act.alpha_logscale
+ ): # exp baked into cuda kernel, cancel it out with a log
+ alpha = torch.log(alpha)
+ beta = torch.log(beta)
+ x = FusedAntiAliasActivation.apply(x, self.upsample.filter, alpha, beta)
+ x = self.downsample(x)
+ return x
diff --git a/fireredtts/modules/bigvgan/alias_free_cuda/anti_alias_activation.cpp b/fireredtts/modules/bigvgan/alias_free_cuda/anti_alias_activation.cpp
new file mode 100644
index 0000000..c68556e
--- /dev/null
+++ b/fireredtts/modules/bigvgan/alias_free_cuda/anti_alias_activation.cpp
@@ -0,0 +1,48 @@
+/* coding=utf-8
+ * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include
+#include
+#include
+
+namespace anti_alias_activation {
+
+torch::Tensor fwd_cuda(torch::Tensor const& input,
+ torch::Tensor const& filter,
+ torch::Tensor const& alpha,
+ torch::Tensor const& beta
+ );
+
+torch::Tensor fwd(torch::Tensor const& input,
+ torch::Tensor const& filter,
+ torch::Tensor const& alpha,
+ torch::Tensor const& beta
+ ) {
+ AT_ASSERTM(input.dim() == 3, "expected 3D tensor");
+ //AT_ASSERTM((input.scalar_type() == at::ScalarType::Half) ||
+ // (input.scalar_type() == at::ScalarType::BFloat16),
+ // "Only fp16 and bf16 are supported");
+
+ return fwd_cuda(input, filter, alpha, beta);
+}
+
+} // end namespace anti_alias_activation
+
+PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
+ m.def("forward",
+ &anti_alias_activation::fwd,
+ "Anti Alias Activation -- Forward.");
+}
diff --git a/fireredtts/modules/bigvgan/alias_free_cuda/anti_alias_activation_cuda.cu b/fireredtts/modules/bigvgan/alias_free_cuda/anti_alias_activation_cuda.cu
new file mode 100644
index 0000000..143ceea
--- /dev/null
+++ b/fireredtts/modules/bigvgan/alias_free_cuda/anti_alias_activation_cuda.cu
@@ -0,0 +1,314 @@
+/* coding=utf-8
+ * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include "type_shim.h"
+#include
+#include
+#include
+#include
+#include
+
+namespace {
+
+ /*
+template
+__device__ __inline__ void copy_vector(Datatype *dst, const Datatype *src);
+
+template <>
+__device__ __inline__ void copy_vector(c10::BFloat16 *dst, const c10::BFloat16 *src) { *dst = *src; }
+
+template <>
+__device__ __inline__ void copy_vector(c10::BFloat16 *dst, const c10::BFloat16 *src) { *((float2*) dst) = *((float2*) src); }
+
+template <>
+__device__ __inline__ void copy_vector(c10::Half *dst, const c10::Half *src) { *dst = *src; }
+
+template <>
+__device__ __inline__ void copy_vector(c10::Half *dst, const c10::Half *src) { *((float2*) dst) = *((float2*) src); }
+
+template <>
+__device__ __inline__ void copy_vector(uint8_t *dst, const uint8_t *src) { *dst = *src; }
+
+template <>
+__device__ __inline__ void copy_vector(uint8_t *dst, const uint8_t *src) {*((half2*) dst) = *((half2*) src); }
+
+int log2_ceil(int value) {
+ int log2_value = 0;
+ while ((1 << log2_value) < value) ++log2_value;
+ return log2_value;
+}
+
+template
+struct Add {
+ __device__ __forceinline__ T operator()(T a, T b) const {
+ return a + b;
+ }
+};
+
+template
+struct Max {
+ __device__ __forceinline__ T operator()(T a, T b) const {
+ return a < b ? b : a;
+ }
+};
+
+template
+__device__ __forceinline__ T WARP_SHFL_XOR_NATIVE(T value, int laneMask, int width = warpSize, unsigned int mask = 0xffffffff)
+{
+#if CUDA_VERSION >= 9000
+ return __shfl_xor_sync(mask, value, laneMask, width);
+#else
+ return __shfl_xor(value, laneMask, width);
+#endif
+}
+
+template class ReduceOp>
+__device__ __forceinline__ void warp_reduce(acc_t* sum) {
+ ReduceOp r;
+ #pragma unroll
+ for (int offset = WARP_SIZE / 2; offset > 0; offset /= 2) {
+ #pragma unroll
+ for (int i = 0; i < WARP_BATCH; ++i) {
+ acc_t b = WARP_SHFL_XOR_NATIVE(sum[i], offset, WARP_SIZE);
+ sum[i] = r(sum[i], b);
+ }
+ }
+}
+*/
+
+template
+__global__ void anti_alias_activation_forward(
+ output_t *dst,
+ const input_t *src,
+ const input_t *ftr,
+ const input_t *alpha,
+ const input_t *beta,
+ int batch_size,
+ int channels,
+ int seq_len)
+{
+ // WARP_SIZE and WARP_BATCH must match the return values batches_per_warp and
+ constexpr int ELEMENTS_PER_LDG_STG = 1; //(WARP_ITERATIONS < 4) ? 1 : 4;
+ constexpr int BUFFER_SIZE = 32;
+ constexpr int FILTER_SIZE = 12;
+ constexpr int HALF_FILTER_SIZE = 6;
+ constexpr int REPLICATION_PAD = 5; // 5 on each side
+
+ // blockDim/threadIdx = (128, 1, 1)
+ // gridDim/blockIdx = (seq_blocks, channels, batches)
+ int block_offset = (blockIdx.x * 128 * BUFFER_SIZE + seq_len * (blockIdx.y + gridDim.y * blockIdx.z));
+ int local_offset = threadIdx.x * BUFFER_SIZE;
+ int seq_offset = blockIdx.x * 128 * BUFFER_SIZE + local_offset;
+
+
+ //int intermediate_seq_len = seq_len * 2 - 1 + 4 * REPLICATION_PAD;
+ //int intermediate_block_offset = (blockIdx.x * 128 * BUFFER_SIZE * 2 + intermediate_seq_len * (blockIdx.y + gridDim.y * blockIdx.z));
+ //int intermediate_local_offset = threadIdx.x * BUFFER_SIZE * 2;
+
+ int output_seq_len = seq_len * 2 ; //
+ int output_block_offset = (blockIdx.x * 128 * BUFFER_SIZE * 2 + output_seq_len * (blockIdx.y + gridDim.y * blockIdx.z));
+ int output_local_offset = threadIdx.x * BUFFER_SIZE * 2;
+ int output_seq_offset = blockIdx.x * 128 * BUFFER_SIZE *2 + output_local_offset;
+ // get values needed for replication padding before moving pointer
+ const input_t *right_most_pntr = src + (seq_len * (blockIdx.y + gridDim.y * blockIdx.z));
+ input_t seq_left_most_value = right_most_pntr[0];
+ input_t seq_right_most_value = right_most_pntr[seq_len - 1];
+
+ src += block_offset + local_offset;
+ dst += output_block_offset + output_local_offset ;
+ alpha = alpha + blockIdx.y;
+ input_t alpha_val = expf(alpha[0]);
+ beta = beta + blockIdx.y;
+ input_t beta_val = expf(beta[0]);
+ // load data from global memory
+ input_t elements[2*FILTER_SIZE+2*BUFFER_SIZE] = {0};
+ input_t intermediates[2*FILTER_SIZE+2*BUFFER_SIZE] = {0};
+ //output_t output[2*BUFFER_SIZE];
+ input_t filter[FILTER_SIZE];
+ //input_t temp_data[ELEMENTS_PER_LDG_STG];
+ //uint8_t temp_mask[ELEMENTS_PER_LDG_STG];
+
+ #pragma unroll
+ for (int it = 0; it < FILTER_SIZE; it+=1) {
+ filter[it] = ftr[it];
+ }
+
+
+ #pragma unroll
+ for (int it = -HALF_FILTER_SIZE; it < BUFFER_SIZE + HALF_FILTER_SIZE ; it+=1) {
+ int element_index = seq_offset + it;
+ if ((element_index < 0) && (element_index >= -REPLICATION_PAD)) {
+ elements[2*(HALF_FILTER_SIZE+it)] = 2*seq_left_most_value;
+ }
+ if ((element_index >= seq_len) && (element_index < seq_len + REPLICATION_PAD)) {
+ elements[2*(HALF_FILTER_SIZE+it)] = 2*seq_right_most_value;
+ }
+ if ((element_index >= 0) && (element_index < seq_len)) {
+ elements[2*(HALF_FILTER_SIZE+it)] = 2*src[it];
+ }
+ }
+
+
+
+ // apply filter
+ #pragma unroll
+ for (int it = 0; it < (2 * BUFFER_SIZE + 2*FILTER_SIZE); it+=1) {
+ input_t acc = 0.0;
+
+ int element_index = output_seq_offset + it; // index for output
+ #pragma unroll
+ for (int f_idx = 0; f_idx < FILTER_SIZE; f_idx+=1){
+ if ((element_index + f_idx) >= 0){
+ acc += filter[f_idx] * elements[it+f_idx];
+ }
+ }
+ intermediates[it] = acc;
+ }
+
+ double no_div_by_zero = 0.000000001;
+ #pragma unroll
+ for (int it = 0; it < 12 + 2 * BUFFER_SIZE; it++) {
+ intermediates[it] += (1.0/(beta_val + no_div_by_zero)) * sinf(intermediates[it] * alpha_val) * sinf(intermediates[it] * alpha_val);
+ }
+
+
+ // now copy to output
+ #pragma unroll
+ for (int it = 0; it < 2*BUFFER_SIZE; it+=1){
+ int element_index = output_seq_offset + it;
+ if (element_index < output_seq_len) {
+ dst[it] = intermediates[it+6];
+ }
+ }
+
+
+
+ // for (int it = 0; it < BUFFER_SIZE; it+=ELEMENTS_PER_LDG_STG) {
+ // int element_index = seq_offset + it;
+ // if (element_index < seq_len) {
+ // dst[it] = output[it];
+ // }
+ // }
+
+
+ // // Upsample convolution
+ // for (int it = 0; it < 2 * BUFFER_SIZE + 12; it+=1) {
+ // input_t acc = 0.0;
+
+ // for (int f_idx = 0; f_idx < FILTER_SIZE; f_idx+=1){
+ // acc += filter[f_idx] * elements[it+f_idx];
+ // }
+ // intermediates[it] = acc;
+ // }
+
+ // // correct the corners of intermediates
+ // if (seq_offset == 0) {
+ // for (int it = 0; it < 6; it+=1)
+ // intermediates[it] = 0;
+ // }
+
+ // if (seq_offset + 32 >= seq_len) {
+ // int offset = seq_len % 32 == 0 ? 32 : seq_len % 32;
+
+ // for (int it = 0; it < 6; it++) {
+ // intermediates[6+2*offset+it] = 0;
+ // }
+ // }
+
+
+
+
+ // for (int it = 0; it < BUFFER_SIZE; it+=ELEMENTS_PER_LDG_STG) {
+ // int element_index = seq_offset + it;
+ // if (element_index < seq_len) {
+ // dst[it] = output[it];
+ // }
+ // }
+}
+
+template
+void dispatch_anti_alias_activation_forward(
+ output_t *dst,
+ const input_t *src,
+ const input_t *ftr,
+ const input_t *alpha,
+ const input_t *beta,
+ int batch_size,
+ int channels,
+ int seq_len)
+{
+ if (seq_len == 0) {
+ return;
+ } else {
+ // use 128 threads per block to maximimize gpu utilization
+ constexpr int threads_per_block = 128;
+ constexpr int seq_len_per_block = 4096;
+ int blocks_per_seq_len = (seq_len + seq_len_per_block - 1) / seq_len_per_block;
+ dim3 blocks(blocks_per_seq_len, channels, batch_size);
+ dim3 threads(threads_per_block, 1, 1);
+
+ anti_alias_activation_forward
+ <<>>(dst, src, ftr, alpha, beta, batch_size, channels, seq_len);
+ }
+}
+}
+
+namespace anti_alias_activation {
+
+ torch::Tensor fwd_cuda(torch::Tensor const& input, torch::Tensor const& filter, torch::Tensor const& alpha, torch::Tensor const& beta)
+{
+ // input is a 4d tensor with dimensions [batches, attn_heads, seq_len, seq_len]
+ const int batches = input.size(0);
+ const int channels = input.size(1);
+ const int seq_len = input.size(2);
+
+ // Output
+ auto act_options = input.options().requires_grad(false);
+ int output_seq_len = seq_len*2; // we'll be dilating between each element by interspersing with zeros
+
+ torch::Tensor anti_alias_activation_results =
+ torch::empty({batches, channels, output_seq_len}, act_options);
+
+ // Softmax Intermediate Result Ptr
+ void* input_ptr = static_cast(input.data_ptr());
+ void* filter_ptr = static_cast(filter.data_ptr());
+ void* alpha_ptr = static_cast(alpha.data_ptr());
+ void* beta_ptr = static_cast(beta.data_ptr());
+ void* anti_alias_activation_results_ptr = static_cast(anti_alias_activation_results.data_ptr());
+
+ DISPATCH_FLOAT_HALF_AND_BFLOAT(
+ input.scalar_type(),
+ "dispatch anti alias activation_forward",
+ dispatch_anti_alias_activation_forward(
+ reinterpret_cast(anti_alias_activation_results_ptr),
+ reinterpret_cast(input_ptr),
+ reinterpret_cast(filter_ptr),
+ reinterpret_cast(alpha_ptr),
+ reinterpret_cast(beta_ptr),
+ batches,
+ channels,
+ seq_len);
+ );
+ return anti_alias_activation_results;
+}
+}
diff --git a/fireredtts/modules/bigvgan/alias_free_cuda/compat.h b/fireredtts/modules/bigvgan/alias_free_cuda/compat.h
new file mode 100644
index 0000000..92e7eb7
--- /dev/null
+++ b/fireredtts/modules/bigvgan/alias_free_cuda/compat.h
@@ -0,0 +1,31 @@
+/* coding=utf-8
+ * Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*This code is copied fron NVIDIA apex:
+ * https://github.com/NVIDIA/apex
+ * with minor changes. */
+
+
+
+#ifndef TORCH_CHECK
+#define TORCH_CHECK AT_CHECK
+#endif
+
+#ifdef VERSION_GE_1_3
+#define DATA_PTR data_ptr
+#else
+#define DATA_PTR data
+#endif
diff --git a/fireredtts/modules/bigvgan/alias_free_cuda/load.py b/fireredtts/modules/bigvgan/alias_free_cuda/load.py
new file mode 100644
index 0000000..63a1248
--- /dev/null
+++ b/fireredtts/modules/bigvgan/alias_free_cuda/load.py
@@ -0,0 +1,85 @@
+# Copyright (c) 2024 NVIDIA CORPORATION.
+# Licensed under the MIT license.
+
+import os
+import pathlib
+import subprocess
+
+from torch.utils import cpp_extension
+
+# Setting this param to a list has a problem of generating different
+# compilation commands (with diferent order of architectures) and
+# leading to recompilation of fused kernels. Set it to empty string
+# to avoid recompilation and assign arch flags explicity in
+# extra_cuda_cflags below
+os.environ["TORCH_CUDA_ARCH_LIST"] = ""
+
+
+def load():
+ # Check if cuda 11 is installed for compute capability 8.0
+ cc_flag = []
+ _, bare_metal_major, _ = _get_cuda_bare_metal_version(cpp_extension.CUDA_HOME)
+ if int(bare_metal_major) >= 11:
+ cc_flag.append("-gencode")
+ cc_flag.append("arch=compute_80,code=sm_80")
+
+ # Build path
+ srcpath = pathlib.Path(__file__).parent.absolute()
+ buildpath = srcpath / "build"
+ _create_build_dir(buildpath)
+
+ # Helper function to build the kernels.
+ def _cpp_extention_load_helper(name, sources, extra_cuda_flags):
+ return cpp_extension.load(
+ name=name,
+ sources=sources,
+ build_directory=buildpath,
+ extra_cflags=[
+ "-O3",
+ ],
+ extra_cuda_cflags=[
+ "-O3",
+ "-gencode",
+ "arch=compute_70,code=sm_70",
+ "--use_fast_math",
+ ]
+ + extra_cuda_flags
+ + cc_flag,
+ verbose=True,
+ )
+
+ extra_cuda_flags = [
+ "-U__CUDA_NO_HALF_OPERATORS__",
+ "-U__CUDA_NO_HALF_CONVERSIONS__",
+ "--expt-relaxed-constexpr",
+ "--expt-extended-lambda",
+ ]
+
+ sources = [
+ srcpath / "anti_alias_activation.cpp",
+ srcpath / "anti_alias_activation_cuda.cu",
+ ]
+ anti_alias_activation_cuda = _cpp_extention_load_helper(
+ "anti_alias_activation_cuda", sources, extra_cuda_flags
+ )
+
+
+def _get_cuda_bare_metal_version(cuda_dir):
+ raw_output = subprocess.check_output(
+ [cuda_dir + "/bin/nvcc", "-V"], universal_newlines=True
+ )
+ output = raw_output.split()
+ release_idx = output.index("release") + 1
+ release = output[release_idx].split(".")
+ bare_metal_major = release[0]
+ bare_metal_minor = release[1][0]
+
+ return raw_output, bare_metal_major, bare_metal_minor
+
+
+def _create_build_dir(buildpath):
+ try:
+ os.mkdir(buildpath)
+ except OSError:
+ if not os.path.isdir(buildpath):
+ print(f"Creation of the build directory {buildpath} failed")
diff --git a/fireredtts/modules/bigvgan/alias_free_cuda/test_activation.py b/fireredtts/modules/bigvgan/alias_free_cuda/test_activation.py
new file mode 100644
index 0000000..eef8760
--- /dev/null
+++ b/fireredtts/modules/bigvgan/alias_free_cuda/test_activation.py
@@ -0,0 +1,64 @@
+# Copyright (c) 2024 NVIDIA CORPORATION.
+# Licensed under the MIT license.
+
+import math
+import torch
+import alias_free_cuda
+from alias_free_cuda import activation1d
+from activations import Snake, SnakeBeta
+
+
+def test_load_fused_kernels():
+ try:
+ import alias_free_cuda
+ import torch
+
+ print("[Success] load_fused_kernels")
+ except ImportError as e:
+ print("[Fail] load_fused_kernels")
+ raise e
+
+
+def test_anti_alias_activation():
+ data = torch.rand((10, 10, 50000), device="cuda")
+
+ # check activations.Snake cuda vs. torch
+ fused_anti_alias_activation = activation1d.Activation1d(
+ activation=Snake(10), fused=True
+ ).cuda()
+ fused_activation_output = fused_anti_alias_activation(data)
+
+ torch_anti_alias_activation = activation1d.Activation1d(
+ activation=Snake(10), fused=False
+ ).cuda()
+ torch_activation_output = torch_anti_alias_activation(data)
+
+ test_result = (fused_activation_output - torch_activation_output).abs()
+
+ while test_result.dim() != 1:
+ test_result = test_result.mean(dim=-1)
+
+ diff = test_result.mean(dim=-1)
+
+ if diff <= 1e-3:
+ print(
+ f"\n[Success] test_fused_anti_alias_activation"
+ f"\n > mean_difference={diff}"
+ f"\n > fused_values={fused_activation_output[-1][-1][-100:].tolist()}"
+ f"\n > torch_values={torch_activation_output[-1][-1][-100:].tolist()}"
+ )
+ else:
+ print(
+ f"\n[Fail] test_fused_anti_alias_activation"
+ f"\n > mean_difference={diff}, "
+ f"\n > fused_values={fused_activation_output[-1][-1][-30:].tolist()}, "
+ f"\n > torch_values={torch_activation_output[-1][-1][-30:].tolist()}"
+ )
+
+
+if __name__ == "__main__":
+ from alias_free_cuda import load
+
+ load.load()
+ test_load_fused_kernels()
+ test_anti_alias_activation()
diff --git a/fireredtts/modules/bigvgan/alias_free_cuda/test_activation_snake_beta.py b/fireredtts/modules/bigvgan/alias_free_cuda/test_activation_snake_beta.py
new file mode 100644
index 0000000..62153c4
--- /dev/null
+++ b/fireredtts/modules/bigvgan/alias_free_cuda/test_activation_snake_beta.py
@@ -0,0 +1,64 @@
+# Copyright (c) 2024 NVIDIA CORPORATION.
+# Licensed under the MIT license.
+
+import math
+import torch
+import alias_free_cuda
+from alias_free_cuda import activation1d
+from activations import Snake, SnakeBeta
+
+
+def test_load_fused_kernels():
+ try:
+ import alias_free_cuda
+ import torch
+
+ print("[Success] load_fused_kernels")
+ except ImportError as e:
+ print("[Fail] load_fused_kernels")
+ raise e
+
+
+def test_anti_alias_activation():
+ data = torch.rand((10, 10, 50000), device="cuda")
+
+ # check activations.Snake cuda vs. torch
+ fused_anti_alias_activation = activation1d.Activation1d(
+ activation=SnakeBeta(10), fused=True
+ ).cuda()
+ fused_activation_output = fused_anti_alias_activation(data)
+
+ torch_anti_alias_activation = activation1d.Activation1d(
+ activation=SnakeBeta(10), fused=False
+ ).cuda()
+ torch_activation_output = torch_anti_alias_activation(data)
+
+ test_result = (fused_activation_output - torch_activation_output).abs()
+
+ while test_result.dim() != 1:
+ test_result = test_result.mean(dim=-1)
+
+ diff = test_result.mean(dim=-1)
+
+ if diff <= 1e-3:
+ print(
+ f"\n[Success] test_fused_anti_alias_activation"
+ f"\n > mean_difference={diff}"
+ f"\n > fused_values={fused_activation_output[-1][-1][-100:].tolist()}"
+ f"\n > torch_values={torch_activation_output[-1][-1][-100:].tolist()}"
+ )
+ else:
+ print(
+ f"\n[Fail] test_fused_anti_alias_activation"
+ f"\n > mean_difference={diff}, "
+ f"\n > fused_values={fused_activation_output[-1][-1][-30:].tolist()}, "
+ f"\n > torch_values={torch_activation_output[-1][-1][-30:].tolist()}"
+ )
+
+
+if __name__ == "__main__":
+ from alias_free_cuda import load
+
+ load.load()
+ test_load_fused_kernels()
+ test_anti_alias_activation()
diff --git a/fireredtts/modules/bigvgan/alias_free_cuda/type_shim.h b/fireredtts/modules/bigvgan/alias_free_cuda/type_shim.h
new file mode 100644
index 0000000..d30a7e4
--- /dev/null
+++ b/fireredtts/modules/bigvgan/alias_free_cuda/type_shim.h
@@ -0,0 +1,97 @@
+/* coding=utf-8
+ * Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+#include
+#include "compat.h"
+
+
+#define DISPATCH_FLOAT_HALF_AND_BFLOAT(TYPE, NAME, ...) \
+ switch(TYPE) \
+ { \
+ case at::ScalarType::Float: \
+ { \
+ using scalar_t = float; \
+ __VA_ARGS__; \
+ break; \
+ } \
+ case at::ScalarType::Half: \
+ { \
+ using scalar_t = at::Half; \
+ __VA_ARGS__; \
+ break; \
+ } \
+ case at::ScalarType::BFloat16: \
+ { \
+ using scalar_t = at::BFloat16; \
+ __VA_ARGS__; \
+ break; \
+ } \
+ default: \
+ AT_ERROR(#NAME, " not implemented for '", toString(TYPE), "'"); \
+ }
+
+
+
+#define DISPATCH_FLOAT_HALF_AND_BFLOAT_INOUT_TYPES(TYPEIN, TYPEOUT, NAME, ...) \
+ switch(TYPEIN) \
+ { \
+ case at::ScalarType::Float: \
+ { \
+ using scalar_t_in = float; \
+ switch(TYPEOUT) \
+ { \
+ case at::ScalarType::Float: \
+ { \
+ using scalar_t_out = float; \
+ __VA_ARGS__; \
+ break; \
+ } \
+ case at::ScalarType::Half: \
+ { \
+ using scalar_t_out = at::Half; \
+ __VA_ARGS__; \
+ break; \
+ } \
+ case at::ScalarType::BFloat16: \
+ { \
+ using scalar_t_out = at::BFloat16; \
+ __VA_ARGS__; \
+ break; \
+ } \
+ default: \
+ AT_ERROR(#NAME, " not implemented for '", toString(TYPEOUT), "'"); \
+ } \
+ break; \
+ } \
+ case at::ScalarType::Half: \
+ { \
+ using scalar_t_in = at::Half; \
+ using scalar_t_out = at::Half; \
+ __VA_ARGS__; \
+ break; \
+ } \
+ case at::ScalarType::BFloat16: \
+ { \
+ using scalar_t_in = at::BFloat16; \
+ using scalar_t_out = at::BFloat16; \
+ __VA_ARGS__; \
+ break; \
+ } \
+ default: \
+ AT_ERROR(#NAME, " not implemented for '", toString(TYPEIN), "'"); \
+ }
+
diff --git a/fireredtts/modules/bigvgan/alias_free_torch/__init__.py b/fireredtts/modules/bigvgan/alias_free_torch/__init__.py
new file mode 100644
index 0000000..b3bccdb
--- /dev/null
+++ b/fireredtts/modules/bigvgan/alias_free_torch/__init__.py
@@ -0,0 +1,5 @@
+# Adapted from https://github.com/junjun3518/alias-free-torch under the Apache License 2.0
+
+from .filter import *
+from .resample import *
+from .act import *
diff --git a/fireredtts/modules/bigvgan/alias_free_torch/act.py b/fireredtts/modules/bigvgan/alias_free_torch/act.py
new file mode 100644
index 0000000..779d58d
--- /dev/null
+++ b/fireredtts/modules/bigvgan/alias_free_torch/act.py
@@ -0,0 +1,29 @@
+# Adapted from https://github.com/junjun3518/alias-free-torch under the Apache License 2.0
+
+import torch.nn as nn
+from .resample import UpSample1d, DownSample1d
+
+
+class Activation1d(nn.Module):
+ def __init__(
+ self,
+ activation,
+ up_ratio: int = 2,
+ down_ratio: int = 2,
+ up_kernel_size: int = 12,
+ down_kernel_size: int = 12,
+ ):
+ super().__init__()
+ self.up_ratio = up_ratio
+ self.down_ratio = down_ratio
+ self.act = activation
+ self.upsample = UpSample1d(up_ratio, up_kernel_size)
+ self.downsample = DownSample1d(down_ratio, down_kernel_size)
+
+ # x: [B,C,T]
+ def forward(self, x):
+ x = self.upsample(x)
+ x = self.act(x)
+ x = self.downsample(x)
+
+ return x
diff --git a/fireredtts/modules/bigvgan/alias_free_torch/filter.py b/fireredtts/modules/bigvgan/alias_free_torch/filter.py
new file mode 100644
index 0000000..394850e
--- /dev/null
+++ b/fireredtts/modules/bigvgan/alias_free_torch/filter.py
@@ -0,0 +1,98 @@
+# Adapted from https://github.com/junjun3518/alias-free-torch under the Apache License 2.0
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import math
+
+if "sinc" in dir(torch):
+ sinc = torch.sinc
+else:
+ # This code is adopted from adefossez's julius.core.sinc under the MIT License
+ # https://adefossez.github.io/julius/julius/core.html
+ # LICENSE is in incl_licenses directory.
+ def sinc(x: torch.Tensor):
+ """
+ Implementation of sinc, i.e. sin(pi * x) / (pi * x)
+ __Warning__: Different to julius.sinc, the input is multiplied by `pi`!
+ """
+ return torch.where(
+ x == 0,
+ torch.tensor(1.0, device=x.device, dtype=x.dtype),
+ torch.sin(math.pi * x) / math.pi / x,
+ )
+
+
+# This code is adopted from adefossez's julius.lowpass.LowPassFilters under the MIT License
+# https://adefossez.github.io/julius/julius/lowpass.html
+# LICENSE is in incl_licenses directory.
+def kaiser_sinc_filter1d(
+ cutoff, half_width, kernel_size
+): # return filter [1,1,kernel_size]
+ even = kernel_size % 2 == 0
+ half_size = kernel_size // 2
+
+ # For kaiser window
+ delta_f = 4 * half_width
+ A = 2.285 * (half_size - 1) * math.pi * delta_f + 7.95
+ if A > 50.0:
+ beta = 0.1102 * (A - 8.7)
+ elif A >= 21.0:
+ beta = 0.5842 * (A - 21) ** 0.4 + 0.07886 * (A - 21.0)
+ else:
+ beta = 0.0
+ window = torch.kaiser_window(kernel_size, beta=beta, periodic=False)
+
+ # ratio = 0.5/cutoff -> 2 * cutoff = 1 / ratio
+ if even:
+ time = torch.arange(-half_size, half_size) + 0.5
+ else:
+ time = torch.arange(kernel_size) - half_size
+ if cutoff == 0:
+ filter_ = torch.zeros_like(time)
+ else:
+ filter_ = 2 * cutoff * window * sinc(2 * cutoff * time)
+ # Normalize filter to have sum = 1, otherwise we will have a small leakage
+ # of the constant component in the input signal.
+ filter_ /= filter_.sum()
+ filter = filter_.view(1, 1, kernel_size)
+
+ return filter
+
+
+class LowPassFilter1d(nn.Module):
+ def __init__(
+ self,
+ cutoff=0.5,
+ half_width=0.6,
+ stride: int = 1,
+ padding: bool = True,
+ padding_mode: str = "replicate",
+ kernel_size: int = 12,
+ ):
+ # kernel_size should be even number for stylegan3 setup,
+ # in this implementation, odd number is also possible.
+ super().__init__()
+ if cutoff < -0.0:
+ raise ValueError("Minimum cutoff must be larger than zero.")
+ if cutoff > 0.5:
+ raise ValueError("A cutoff above 0.5 does not make sense.")
+ self.kernel_size = kernel_size
+ self.even = kernel_size % 2 == 0
+ self.pad_left = kernel_size // 2 - int(self.even)
+ self.pad_right = kernel_size // 2
+ self.stride = stride
+ self.padding = padding
+ self.padding_mode = padding_mode
+ filter = kaiser_sinc_filter1d(cutoff, half_width, kernel_size)
+ self.register_buffer("filter", filter)
+
+ # input [B, C, T]
+ def forward(self, x):
+ _, C, _ = x.shape
+
+ if self.padding:
+ x = F.pad(x, (self.pad_left, self.pad_right), mode=self.padding_mode)
+ out = F.conv1d(x, self.filter.expand(C, -1, -1), stride=self.stride, groups=C)
+
+ return out
diff --git a/fireredtts/modules/bigvgan/alias_free_torch/resample.py b/fireredtts/modules/bigvgan/alias_free_torch/resample.py
new file mode 100644
index 0000000..ee993b1
--- /dev/null
+++ b/fireredtts/modules/bigvgan/alias_free_torch/resample.py
@@ -0,0 +1,57 @@
+# Adapted from https://github.com/junjun3518/alias-free-torch under the Apache License 2.0
+
+import torch.nn as nn
+from torch.nn import functional as F
+from .filter import LowPassFilter1d
+from .filter import kaiser_sinc_filter1d
+
+
+class UpSample1d(nn.Module):
+ def __init__(self, ratio=2, kernel_size=None):
+ super().__init__()
+ self.ratio = ratio
+ self.kernel_size = (
+ int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size
+ )
+ self.stride = ratio
+ self.pad = self.kernel_size // ratio - 1
+ self.pad_left = self.pad * self.stride + (self.kernel_size - self.stride) // 2
+ self.pad_right = (
+ self.pad * self.stride + (self.kernel_size - self.stride + 1) // 2
+ )
+ filter = kaiser_sinc_filter1d(
+ cutoff=0.5 / ratio, half_width=0.6 / ratio, kernel_size=self.kernel_size
+ )
+ self.register_buffer("filter", filter)
+
+ # x: [B, C, T]
+ def forward(self, x):
+ _, C, _ = x.shape
+
+ x = F.pad(x, (self.pad, self.pad), mode="replicate")
+ x = self.ratio * F.conv_transpose1d(
+ x, self.filter.expand(C, -1, -1), stride=self.stride, groups=C
+ )
+ x = x[..., self.pad_left : -self.pad_right]
+
+ return x
+
+
+class DownSample1d(nn.Module):
+ def __init__(self, ratio=2, kernel_size=None):
+ super().__init__()
+ self.ratio = ratio
+ self.kernel_size = (
+ int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size
+ )
+ self.lowpass = LowPassFilter1d(
+ cutoff=0.5 / ratio,
+ half_width=0.6 / ratio,
+ stride=ratio,
+ kernel_size=self.kernel_size,
+ )
+
+ def forward(self, x):
+ xx = self.lowpass(x)
+
+ return xx
diff --git a/fireredtts/modules/bigvgan/bigvgan.py b/fireredtts/modules/bigvgan/bigvgan.py
new file mode 100644
index 0000000..3fb876d
--- /dev/null
+++ b/fireredtts/modules/bigvgan/bigvgan.py
@@ -0,0 +1,399 @@
+import typing as tp
+import torch
+import torch.nn as nn
+from torch.nn import Conv1d, ConvTranspose1d
+from torch.nn.utils import weight_norm, remove_weight_norm
+
+from fireredtts.modules.bigvgan.alias_free_torch import (
+ Activation1d as TorchActivation1d,
+)
+from fireredtts.modules.bigvgan.activations import Snake, SnakeBeta
+
+
+def init_weights(m, mean=0.0, std=0.01):
+ classname = m.__class__.__name__
+ if classname.find("Conv") != -1:
+ m.weight.data.normal_(mean, std)
+
+
+def get_padding(kernel_size, dilation=1):
+ return int((kernel_size * dilation - dilation) / 2)
+
+
+class AMPBlock1(torch.nn.Module):
+ def __init__(
+ self,
+ channels,
+ kernel_size=3,
+ dilation=(1, 3, 5),
+ activation=None,
+ snake_logscale=True,
+ use_cuda_kernel=False,
+ ):
+ super(AMPBlock1, self).__init__()
+
+ self.convs1 = nn.ModuleList(
+ [
+ weight_norm(
+ Conv1d(
+ channels,
+ channels,
+ kernel_size,
+ 1,
+ dilation=dilation[0],
+ padding=get_padding(kernel_size, dilation[0]),
+ )
+ ),
+ weight_norm(
+ Conv1d(
+ channels,
+ channels,
+ kernel_size,
+ 1,
+ dilation=dilation[1],
+ padding=get_padding(kernel_size, dilation[1]),
+ )
+ ),
+ weight_norm(
+ Conv1d(
+ channels,
+ channels,
+ kernel_size,
+ 1,
+ dilation=dilation[2],
+ padding=get_padding(kernel_size, dilation[2]),
+ )
+ ),
+ ]
+ )
+ self.convs1.apply(init_weights)
+
+ self.convs2 = nn.ModuleList(
+ [
+ weight_norm(
+ Conv1d(
+ channels,
+ channels,
+ kernel_size,
+ 1,
+ dilation=1,
+ padding=get_padding(kernel_size, 1),
+ )
+ ),
+ weight_norm(
+ Conv1d(
+ channels,
+ channels,
+ kernel_size,
+ 1,
+ dilation=1,
+ padding=get_padding(kernel_size, 1),
+ )
+ ),
+ weight_norm(
+ Conv1d(
+ channels,
+ channels,
+ kernel_size,
+ 1,
+ dilation=1,
+ padding=get_padding(kernel_size, 1),
+ )
+ ),
+ ]
+ )
+ self.convs2.apply(init_weights)
+
+ self.num_layers = len(self.convs1) + len(
+ self.convs2
+ ) # total number of conv layers
+
+ # select which Activation1d, lazy-load cuda version to ensure backward compatibility
+ if use_cuda_kernel:
+ from modules.bigvgan.alias_free_cuda.activation1d import (
+ Activation1d as CudaActivation1d,
+ )
+
+ Activation1d = CudaActivation1d
+ else:
+ Activation1d = TorchActivation1d
+
+ if (
+ activation == "snake"
+ ): # periodic nonlinearity with snake function and anti-aliasing
+ self.activations = nn.ModuleList(
+ [
+ Activation1d(
+ activation=Snake(channels, alpha_logscale=snake_logscale)
+ )
+ for _ in range(self.num_layers)
+ ]
+ )
+ elif (
+ activation == "snakebeta"
+ ): # periodic nonlinearity with snakebeta function and anti-aliasing
+ self.activations = nn.ModuleList(
+ [
+ Activation1d(
+ activation=SnakeBeta(channels, alpha_logscale=snake_logscale)
+ )
+ for _ in range(self.num_layers)
+ ]
+ )
+ else:
+ raise NotImplementedError(
+ "activation incorrectly specified. check the config file and look for 'activation'."
+ )
+
+ def forward(self, x):
+ acts1, acts2 = self.activations[::2], self.activations[1::2]
+ for c1, c2, a1, a2 in zip(self.convs1, self.convs2, acts1, acts2):
+ xt = a1(x)
+ xt = c1(xt)
+ xt = a2(xt)
+ xt = c2(xt)
+ x = xt + x
+
+ return x
+
+ def remove_weight_norm(self):
+ for l in self.convs1:
+ remove_weight_norm(l)
+ for l in self.convs2:
+ remove_weight_norm(l)
+
+
+class AMPBlock2(torch.nn.Module):
+ def __init__(
+ self,
+ channels,
+ kernel_size=3,
+ dilation=(1, 3),
+ activation=None,
+ snake_logscale=True,
+ use_cuda_kernel=False,
+ ):
+ super(AMPBlock2, self).__init__()
+
+ self.convs = nn.ModuleList(
+ [
+ weight_norm(
+ Conv1d(
+ channels,
+ channels,
+ kernel_size,
+ 1,
+ dilation=dilation[0],
+ padding=get_padding(kernel_size, dilation[0]),
+ )
+ ),
+ weight_norm(
+ Conv1d(
+ channels,
+ channels,
+ kernel_size,
+ 1,
+ dilation=dilation[1],
+ padding=get_padding(kernel_size, dilation[1]),
+ )
+ ),
+ ]
+ )
+ self.convs.apply(init_weights)
+
+ self.num_layers = len(self.convs) # total number of conv layers
+
+ # select which Activation1d, lazy-load cuda version to ensure backward compatibility
+ if use_cuda_kernel:
+ from modules.bigvgan.alias_free_cuda.activation1d import (
+ Activation1d as CudaActivation1d,
+ )
+
+ Activation1d = CudaActivation1d
+ else:
+ Activation1d = TorchActivation1d
+
+ if (
+ activation == "snake"
+ ): # periodic nonlinearity with snake function and anti-aliasing
+ self.activations = nn.ModuleList(
+ [
+ Activation1d(
+ activation=Snake(channels, alpha_logscale=snake_logscale)
+ )
+ for _ in range(self.num_layers)
+ ]
+ )
+ elif (
+ activation == "snakebeta"
+ ): # periodic nonlinearity with snakebeta function and anti-aliasing
+ self.activations = nn.ModuleList(
+ [
+ Activation1d(
+ activation=SnakeBeta(channels, alpha_logscale=snake_logscale)
+ )
+ for _ in range(self.num_layers)
+ ]
+ )
+ else:
+ raise NotImplementedError(
+ "activation incorrectly specified. check the config file and look for 'activation'."
+ )
+
+ def forward(self, x):
+ for c, a in zip(self.convs, self.activations):
+ xt = a(x)
+ xt = c(xt)
+ x = xt + x
+
+ return x
+
+ def remove_weight_norm(self):
+ for l in self.convs:
+ remove_weight_norm(l)
+
+
+class BigVGAN(torch.nn.Module):
+ # this is our main BigVGAN model. Applies anti-aliased periodic activation for resblocks.
+ def __init__(
+ self,
+ num_mels: int,
+ upsample_initial_channel: int,
+ resblock_kernel_sizes: tp.List[int],
+ resblock_dilation_sizes: tp.List[tp.List[int]],
+ upsample_rates: tp.List[int],
+ upsample_kernel_sizes: tp.List[int],
+ resblock_type: str = "1",
+ snake_logscale: bool = True,
+ activation: str = "snakebeta",
+ use_tanh_at_final: bool = False,
+ use_bias_at_final: bool = False,
+ use_cuda_kernel: bool = False,
+ ):
+ super(BigVGAN, self).__init__()
+
+ self.num_kernels = len(resblock_kernel_sizes)
+ self.num_upsamples = len(upsample_rates)
+
+ # pre conv
+ self.conv_pre = weight_norm(
+ Conv1d(num_mels, upsample_initial_channel, 7, 1, padding=3)
+ )
+
+ # define which AMPBlock to use. BigVGAN uses AMPBlock1 as default
+ resblock = AMPBlock1 if resblock_type == "1" else AMPBlock2
+
+ # transposed conv-based upsamplers. does not apply anti-aliasing
+ self.ups = nn.ModuleList()
+ for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
+ self.ups.append(
+ nn.ModuleList(
+ [
+ weight_norm(
+ ConvTranspose1d(
+ upsample_initial_channel // (2**i),
+ upsample_initial_channel // (2 ** (i + 1)),
+ k,
+ u,
+ padding=(k - u) // 2,
+ )
+ )
+ ]
+ )
+ )
+
+ # residual blocks using anti-aliased multi-periodicity composition modules (AMP)
+ self.resblocks = nn.ModuleList()
+ for i in range(len(self.ups)):
+ ch = upsample_initial_channel // (2 ** (i + 1))
+ for j, (k, d) in enumerate(
+ zip(resblock_kernel_sizes, resblock_dilation_sizes)
+ ):
+ self.resblocks.append(
+ resblock(
+ ch,
+ k,
+ d,
+ activation=activation,
+ snake_logscale=snake_logscale,
+ use_cuda_kernel=use_cuda_kernel,
+ )
+ )
+
+ # select which Activation1d, lazy-load cuda version to ensure backward compatibility
+ if use_cuda_kernel:
+ from modules.bigvgan.alias_free_cuda.activation1d import (
+ Activation1d as CudaActivation1d,
+ )
+
+ Activation1d = CudaActivation1d
+ else:
+ Activation1d = TorchActivation1d
+
+ # post conv
+ if (
+ activation == "snake"
+ ): # periodic nonlinearity with snake function and anti-aliasing
+ activation_post = Snake(ch, alpha_logscale=snake_logscale)
+ self.activation_post = Activation1d(activation=activation_post)
+ elif (
+ activation == "snakebeta"
+ ): # periodic nonlinearity with snakebeta function and anti-aliasing
+ activation_post = SnakeBeta(ch, alpha_logscale=snake_logscale)
+ self.activation_post = Activation1d(activation=activation_post)
+ else:
+ raise NotImplementedError(
+ "activation incorrectly specified. check the config file and look for 'activation'."
+ )
+
+ # whether to use bias for the final conv_post. Defaults to True for backward compatibility
+ self.use_bias_at_final = use_bias_at_final
+ self.conv_post = weight_norm(
+ Conv1d(ch, 1, 7, 1, padding=3, bias=self.use_bias_at_final)
+ )
+
+ # weight initialization
+ for i in range(len(self.ups)):
+ self.ups[i].apply(init_weights)
+ self.conv_post.apply(init_weights)
+
+ # final tanh activation. Defaults to True for backward compatibility
+ self.use_tanh_at_final = use_tanh_at_final
+
+ def forward(self, x):
+ # pre conv
+ x = self.conv_pre(x)
+
+ for i in range(self.num_upsamples):
+ # upsampling
+ for i_up in range(len(self.ups[i])):
+ x = self.ups[i][i_up](x)
+ # AMP blocks
+ xs = None
+ for j in range(self.num_kernels):
+ if xs is None:
+ xs = self.resblocks[i * self.num_kernels + j](x)
+ else:
+ xs += self.resblocks[i * self.num_kernels + j](x)
+ x = xs / self.num_kernels
+
+ # post conv
+ x = self.activation_post(x)
+ x = self.conv_post(x)
+ # final tanh activation
+ if self.use_tanh_at_final:
+ x = torch.tanh(x)
+ else:
+ x = torch.clamp(x, min=-1.0, max=1.0) # bound the output to [-1, 1]
+
+ return x
+
+ def remove_weight_norm(self):
+ print("Removing weight norm...")
+ for l in self.ups:
+ for l_i in l:
+ remove_weight_norm(l_i)
+ for l in self.resblocks:
+ l.remove_weight_norm()
+ remove_weight_norm(self.conv_pre)
+ remove_weight_norm(self.conv_post)
diff --git a/fireredtts/modules/codec/speaker.py b/fireredtts/modules/codec/speaker.py
new file mode 100644
index 0000000..d4614db
--- /dev/null
+++ b/fireredtts/modules/codec/speaker.py
@@ -0,0 +1,1052 @@
+from librosa.filters import mel as librosa_mel_fn
+from torch import nn
+from torch.nn import functional as F
+
+import math
+import numpy as np
+import torch
+import torchaudio
+
+
+def dynamic_range_compression_torch(x, C=1, clip_val=1e-5):
+ return torch.log(torch.clamp(x, min=clip_val) * C)
+
+
+def spectral_normalize_torch(magnitudes):
+ output = dynamic_range_compression_torch(magnitudes)
+ return output
+
+
+class TorchMelSpectrogram(nn.Module):
+
+ def __init__(
+ self,
+ filter_length=1024,
+ hop_length=160,
+ win_length=640,
+ n_mel_channels=80,
+ mel_fmin=0,
+ mel_fmax=8000,
+ sampling_rate=16000,
+ ):
+
+ super().__init__()
+ self.filter_length = filter_length
+ self.hop_length = hop_length
+ self.win_length = win_length
+ self.n_mel_channels = n_mel_channels
+ self.mel_fmin = mel_fmin
+ self.mel_fmax = mel_fmax
+ self.sampling_rate = sampling_rate
+
+ self.mel_basis = {}
+ self.hann_window = {}
+
+ def forward(self, inp, length=None):
+ if len(inp.shape) == 3:
+ inp = inp.squeeze(1) if inp.shape[1] == 1 else inp.squeeze(2)
+ assert len(inp.shape) == 2
+
+ y = inp
+ if len(list(self.mel_basis.keys())) == 0:
+ mel = librosa_mel_fn(
+ sr=self.sampling_rate,
+ n_fft=self.filter_length,
+ n_mels=self.n_mel_channels,
+ fmin=self.mel_fmin,
+ fmax=self.mel_fmax,
+ )
+ self.mel_basis[str(self.mel_fmax) + "_" + str(y.device)] = (
+ torch.from_numpy(mel).float().to(y.device)
+ )
+ self.hann_window[str(y.device)] = torch.hann_window(self.win_length).to(
+ y.device
+ )
+
+ y = torch.nn.functional.pad(
+ y.unsqueeze(1),
+ (
+ int((self.filter_length - self.hop_length) / 2),
+ int((self.filter_length - self.hop_length) / 2),
+ ),
+ mode="reflect",
+ )
+ y = y.squeeze(1)
+
+ # complex tensor as default, then use view_as_real for future pytorch compatibility
+ spec = torch.stft(
+ y,
+ self.filter_length,
+ hop_length=self.hop_length,
+ win_length=self.win_length,
+ window=self.hann_window[str(y.device)],
+ center=False,
+ pad_mode="reflect",
+ normalized=False,
+ onesided=True,
+ return_complex=True,
+ )
+ spec = torch.view_as_real(spec)
+ spec = torch.sqrt(spec.pow(2).sum(-1) + (1e-9))
+
+ spec = torch.matmul(
+ self.mel_basis[str(self.mel_fmax) + "_" + str(y.device)], spec
+ )
+ spec = spectral_normalize_torch(spec)
+
+ max_mel_length = math.ceil(y.shape[-1] / self.hop_length)
+ spec = spec[..., :max_mel_length].transpose(1, 2)
+
+ if length is None:
+ return spec
+ else:
+ spec_len = torch.ceil(length / self.hop_length).clamp(max=spec.shape[1])
+ return spec, spec_len
+
+
+def length_to_mask(length, max_len=None, dtype=None, device=None):
+ """Creates a binary mask for each sequence.
+
+ Reference: https://discuss.pytorch.org/t/how-to-generate-variable-length-mask/23397/3
+
+ Arguments
+ ---------
+ length : torch.LongTensor
+ Containing the length of each sequence in the batch. Must be 1D.
+ max_len : int
+ Max length for the mask, also the size of the second dimension.
+ dtype : torch.dtype, default: None
+ The dtype of the generated mask.
+ device: torch.device, default: None
+ The device to put the mask variable.
+
+ Returns
+ -------
+ mask : tensor
+ The binary mask.
+
+ Example
+ -------
+ >>> length=torch.Tensor([1,2,3])
+ >>> mask=length_to_mask(length)
+ >>> mask
+ tensor([[1., 0., 0.],
+ [1., 1., 0.],
+ [1., 1., 1.]])
+ """
+ assert len(length.shape) == 1
+
+ if max_len is None:
+ max_len = length.max().long().item() # using arange to generate mask
+ mask = torch.arange(max_len, device=length.device, dtype=length.dtype).expand(
+ len(length), max_len
+ ) < length.unsqueeze(1)
+
+ if dtype is None:
+ dtype = length.dtype
+
+ if device is None:
+ device = length.device
+
+ mask = torch.as_tensor(mask, dtype=dtype, device=device)
+ return mask
+
+
+def get_padding_elem(L_in: int, stride: int, kernel_size: int, dilation: int):
+ """This function computes the number of elements to add for zero-padding.
+
+ Arguments
+ ---------
+ L_in : int
+ stride: int
+ kernel_size : int
+ dilation : int
+ """
+ if stride > 1:
+ n_steps = math.ceil(((L_in - kernel_size * dilation) / stride) + 1)
+ L_out = stride * (n_steps - 1) + kernel_size * dilation
+ padding = [kernel_size // 2, kernel_size // 2]
+
+ else:
+ L_out = (L_in - dilation * (kernel_size - 1) - 1) // stride + 1
+
+ padding = [(L_in - L_out) // 2, (L_in - L_out) // 2]
+ return padding
+
+
+class Conv1d(nn.Module):
+ """This function implements 1d convolution.
+
+ Arguments
+ ---------
+ out_channels : int
+ It is the number of output channels.
+ kernel_size : int
+ Kernel size of the convolutional filters.
+ input_shape : tuple
+ The shape of the input. Alternatively use ``in_channels``.
+ in_channels : int
+ The number of input channels. Alternatively use ``input_shape``.
+ stride : int
+ Stride factor of the convolutional filters. When the stride factor > 1,
+ a decimation in time is performed.
+ dilation : int
+ Dilation factor of the convolutional filters.
+ padding : str
+ (same, valid, causal). If "valid", no padding is performed.
+ If "same" and stride is 1, output shape is the same as the input shape.
+ "causal" results in causal (dilated) convolutions.
+ padding_mode : str
+ This flag specifies the type of padding. See torch.nn documentation
+ for more information.
+ skip_transpose : bool
+ If False, uses batch x time x channel convention of speechbrain.
+ If True, uses batch x channel x time convention.
+
+ Example
+ -------
+ >>> inp_tensor = torch.rand([10, 40, 16])
+ >>> cnn_1d = Conv1d(
+ ... input_shape=inp_tensor.shape, out_channels=8, kernel_size=5
+ ... )
+ >>> out_tensor = cnn_1d(inp_tensor)
+ >>> out_tensor.shape
+ torch.Size([10, 40, 8])
+ """
+
+ def __init__(
+ self,
+ out_channels,
+ kernel_size,
+ input_shape=None,
+ in_channels=None,
+ stride=1,
+ dilation=1,
+ padding="same",
+ groups=1,
+ bias=True,
+ padding_mode="reflect",
+ skip_transpose=True,
+ ):
+ super().__init__()
+ self.kernel_size = kernel_size
+ self.stride = stride
+ self.dilation = dilation
+ self.padding = padding
+ self.padding_mode = padding_mode
+ self.unsqueeze = False
+ self.skip_transpose = skip_transpose
+
+ if input_shape is None and in_channels is None:
+ raise ValueError("Must provide one of input_shape or in_channels")
+
+ if in_channels is None:
+ in_channels = self._check_input_shape(input_shape)
+
+ self.conv = nn.Conv1d(
+ in_channels,
+ out_channels,
+ self.kernel_size,
+ stride=self.stride,
+ dilation=self.dilation,
+ padding=0,
+ groups=groups,
+ bias=bias,
+ )
+
+ def forward(self, x):
+ """Returns the output of the convolution.
+
+ Arguments
+ ---------
+ x : torch.Tensor (batch, time, channel)
+ input to convolve. 2d or 4d tensors are expected.
+ """
+
+ if not self.skip_transpose:
+ x = x.transpose(1, -1)
+
+ if self.unsqueeze:
+ x = x.unsqueeze(1)
+
+ if self.padding == "same":
+ x = self._manage_padding(x, self.kernel_size, self.dilation, self.stride)
+
+ elif self.padding == "causal":
+ num_pad = (self.kernel_size - 1) * self.dilation
+ x = F.pad(x, (num_pad, 0))
+
+ elif self.padding == "valid":
+ pass
+
+ else:
+ raise ValueError(
+ "Padding must be 'same', 'valid' or 'causal'. Got " + self.padding
+ )
+
+ wx = self.conv(x)
+
+ if self.unsqueeze:
+ wx = wx.squeeze(1)
+
+ if not self.skip_transpose:
+ wx = wx.transpose(1, -1)
+
+ return wx
+
+ def _manage_padding(
+ self,
+ x,
+ kernel_size: int,
+ dilation: int,
+ stride: int,
+ ):
+ """This function performs zero-padding on the time axis
+ such that their lengths is unchanged after the convolution.
+
+ Arguments
+ ---------
+ x : torch.Tensor
+ Input tensor.
+ kernel_size : int
+ Size of kernel.
+ dilation : int
+ Dilation used.
+ stride : int
+ Stride.
+ """
+
+ # Detecting input shape
+ L_in = x.shape[-1]
+
+ # Time padding
+ padding = get_padding_elem(L_in, stride, kernel_size, dilation)
+
+ # Applying padding
+ x = F.pad(x, padding, mode=self.padding_mode)
+
+ return x
+
+ def _check_input_shape(self, shape):
+ """Checks the input shape and returns the number of input channels."""
+
+ if len(shape) == 2:
+ self.unsqueeze = True
+ in_channels = 1
+ elif self.skip_transpose:
+ in_channels = shape[1]
+ elif len(shape) == 3:
+ in_channels = shape[2]
+ else:
+ raise ValueError("conv1d expects 2d, 3d inputs. Got " + str(len(shape)))
+
+ # Kernel size must be odd
+ if self.kernel_size % 2 == 0:
+ raise ValueError(
+ "The field kernel size must be an odd number. Got %s."
+ % (self.kernel_size)
+ )
+ return in_channels
+
+
+class Fp32BatchNorm(nn.Module):
+ def __init__(self, sync=True, *args, **kwargs):
+ super().__init__()
+
+ if (
+ not torch.distributed.is_initialized()
+ or torch.distributed.get_world_size() == 1
+ ):
+ sync = False
+
+ if sync:
+ self.bn = nn.SyncBatchNorm(*args, **kwargs)
+ else:
+ self.bn = nn.BatchNorm1d(*args, **kwargs)
+
+ self.sync = sync
+
+ def forward(self, input):
+ if self.bn.running_mean.dtype != torch.float:
+ if self.sync:
+ self.bn.running_mean = self.bn.running_mean.float()
+ self.bn.running_var = self.bn.running_var.float()
+ if self.bn.affine:
+ try:
+ self.bn.weight = self.bn.weight.float()
+ self.bn.bias = self.bn.bias.float()
+ except:
+ self.bn.float()
+ else:
+ self.bn.float()
+
+ output = self.bn(input.float())
+ return output.type_as(input)
+
+
+class BatchNorm1d(nn.Module):
+ """Applies 1d batch normalization to the input tensor.
+
+ Arguments
+ ---------
+ input_shape : tuple
+ The expected shape of the input. Alternatively, use ``input_size``.
+ input_size : int
+ The expected size of the input. Alternatively, use ``input_shape``.
+ eps : float
+ This value is added to std deviation estimation to improve the numerical
+ stability.
+ momentum : float
+ It is a value used for the running_mean and running_var computation.
+ affine : bool
+ When set to True, the affine parameters are learned.
+ track_running_stats : bool
+ When set to True, this module tracks the running mean and variance,
+ and when set to False, this module does not track such statistics.
+ combine_batch_time : bool
+ When true, it combines batch an time axis.
+
+
+ Example
+ -------
+ >>> input = torch.randn(100, 10)
+ >>> norm = BatchNorm1d(input_shape=input.shape)
+ >>> output = norm(input)
+ >>> output.shape
+ torch.Size([100, 10])
+ """
+
+ def __init__(
+ self,
+ input_shape=None,
+ input_size=None,
+ eps=1e-05,
+ momentum=0.1,
+ affine=True,
+ track_running_stats=True,
+ combine_batch_time=False,
+ skip_transpose=True,
+ enabled=True,
+ ):
+ super().__init__()
+ self.combine_batch_time = combine_batch_time
+ self.skip_transpose = skip_transpose
+
+ if input_size is None and skip_transpose:
+ input_size = input_shape[1]
+ elif input_size is None:
+ input_size = input_shape[-1]
+
+ if enabled:
+ self.norm = Fp32BatchNorm(
+ num_features=input_size,
+ eps=eps,
+ momentum=momentum,
+ affine=affine,
+ track_running_stats=track_running_stats,
+ )
+ else:
+ self.norm = nn.Identity()
+
+ def forward(self, x):
+ """Returns the normalized input tensor.
+
+ Arguments
+ ---------
+ x : torch.Tensor (batch, time, [channels])
+ input to normalize. 2d or 3d tensors are expected in input
+ 4d tensors can be used when combine_dims=True.
+ """
+ shape_or = x.shape
+ if self.combine_batch_time:
+ if x.ndim == 3:
+ x = x.reshape(shape_or[0] * shape_or[1], shape_or[2])
+ else:
+ x = x.reshape(shape_or[0] * shape_or[1], shape_or[3], shape_or[2])
+
+ elif not self.skip_transpose:
+ x = x.transpose(-1, 1)
+
+ x_n = self.norm(x)
+
+ if self.combine_batch_time:
+ x_n = x_n.reshape(shape_or)
+ elif not self.skip_transpose:
+ x_n = x_n.transpose(1, -1)
+
+ return x_n
+
+
+class Linear(torch.nn.Module):
+ """Computes a linear transformation y = wx + b.
+
+ Arguments
+ ---------
+ n_neurons : int
+ It is the number of output neurons (i.e, the dimensionality of the
+ output).
+ bias : bool
+ If True, the additive bias b is adopted.
+ combine_dims : bool
+ If True and the input is 4D, combine 3rd and 4th dimensions of input.
+
+ Example
+ -------
+ >>> inputs = torch.rand(10, 50, 40)
+ >>> lin_t = Linear(input_shape=(10, 50, 40), n_neurons=100)
+ >>> output = lin_t(inputs)
+ >>> output.shape
+ torch.Size([10, 50, 100])
+ """
+
+ def __init__(
+ self,
+ n_neurons,
+ input_shape=None,
+ input_size=None,
+ bias=True,
+ combine_dims=False,
+ ):
+ super().__init__()
+ self.combine_dims = combine_dims
+
+ if input_shape is None and input_size is None:
+ raise ValueError("Expected one of input_shape or input_size")
+
+ if input_size is None:
+ input_size = input_shape[-1]
+ if len(input_shape) == 4 and self.combine_dims:
+ input_size = input_shape[2] * input_shape[3]
+
+ # Weights are initialized following pytorch approach
+ self.w = nn.Linear(input_size, n_neurons, bias=bias)
+
+ def forward(self, x):
+ """Returns the linear transformation of input tensor.
+
+ Arguments
+ ---------
+ x : torch.Tensor
+ Input to transform linearly.
+ """
+ if x.ndim == 4 and self.combine_dims:
+ x = x.reshape(x.shape[0], x.shape[1], x.shape[2] * x.shape[3])
+
+ wx = self.w(x)
+
+ return wx
+
+
+class TDNNBlock(nn.Module):
+ """An implementation of TDNN.
+
+ Arguments
+ ----------
+ in_channels : int
+ Number of input channels.
+ out_channels : int
+ The number of output channels.
+ kernel_size : int
+ The kernel size of the TDNN blocks.
+ dilation : int
+ The dilation of the Res2Net block.
+ activation : torch class
+ A class for constructing the activation layers.
+
+ Example
+ -------
+ >>> inp_tensor = torch.rand([8, 120, 64]).transpose(1, 2)
+ >>> layer = TDNNBlock(64, 64, kernel_size=3, dilation=1)
+ >>> out_tensor = layer(inp_tensor).transpose(1, 2)
+ >>> out_tensor.shape
+ torch.Size([8, 120, 64])
+ """
+
+ def __init__(
+ self,
+ in_channels,
+ out_channels,
+ kernel_size,
+ dilation,
+ activation=nn.ReLU,
+ batch_norm=True,
+ ):
+ super(TDNNBlock, self).__init__()
+ self.conv = Conv1d(
+ in_channels=in_channels,
+ out_channels=out_channels,
+ kernel_size=kernel_size,
+ dilation=dilation,
+ )
+ self.activation = activation()
+ self.norm = BatchNorm1d(input_size=out_channels, enabled=batch_norm)
+
+ def forward(self, x):
+ return self.norm(self.activation(self.conv(x)))
+
+
+class Res2NetBlock(torch.nn.Module):
+ """An implementation of Res2NetBlock w/ dilation.
+
+ Arguments
+ ---------
+ in_channels : int
+ The number of channels expected in the input.
+ out_channels : int
+ The number of output channels.
+ scale : int
+ The scale of the Res2Net block.
+ kernel_size: int
+ The kernel size of the Res2Net block.
+ dilation : int
+ The dilation of the Res2Net block.
+
+ Example
+ -------
+ >>> inp_tensor = torch.rand([8, 120, 64]).transpose(1, 2)
+ >>> layer = Res2NetBlock(64, 64, scale=4, dilation=3)
+ >>> out_tensor = layer(inp_tensor).transpose(1, 2)
+ >>> out_tensor.shape
+ torch.Size([8, 120, 64])
+ """
+
+ def __init__(
+ self,
+ in_channels,
+ out_channels,
+ scale=8,
+ kernel_size=3,
+ dilation=1,
+ batch_norm=True,
+ ):
+ super(Res2NetBlock, self).__init__()
+ assert in_channels % scale == 0
+ assert out_channels % scale == 0
+
+ in_channel = in_channels // scale
+ hidden_channel = out_channels // scale
+
+ self.blocks = nn.ModuleList(
+ [
+ TDNNBlock(
+ in_channel,
+ hidden_channel,
+ kernel_size=kernel_size,
+ dilation=dilation,
+ batch_norm=batch_norm,
+ )
+ for i in range(scale - 1)
+ ]
+ )
+ self.scale = scale
+
+ def forward(self, x):
+ y = []
+ for i, x_i in enumerate(torch.chunk(x, self.scale, dim=1)):
+ if i == 0:
+ y_i = x_i
+ elif i == 1:
+ y_i = self.blocks[i - 1](x_i)
+ else:
+ y_i = self.blocks[i - 1](x_i + y_i)
+ y.append(y_i)
+ y = torch.cat(y, dim=1)
+ return y
+
+
+class SEBlock(nn.Module):
+ """An implementation of squeeze-and-excitation block.
+
+ Arguments
+ ---------
+ in_channels : int
+ The number of input channels.
+ se_channels : int
+ The number of output channels after squeeze.
+ out_channels : int
+ The number of output channels.
+
+ Example
+ -------
+ >>> inp_tensor = torch.rand([8, 120, 64]).transpose(1, 2)
+ >>> se_layer = SEBlock(64, 16, 64)
+ >>> lengths = torch.rand((8,))
+ >>> out_tensor = se_layer(inp_tensor, lengths).transpose(1, 2)
+ >>> out_tensor.shape
+ torch.Size([8, 120, 64])
+ """
+
+ def __init__(self, in_channels, se_channels, out_channels):
+ super(SEBlock, self).__init__()
+
+ self.conv1 = Conv1d(
+ in_channels=in_channels, out_channels=se_channels, kernel_size=1
+ )
+ self.relu = torch.nn.ReLU(inplace=True)
+ self.conv2 = Conv1d(
+ in_channels=se_channels, out_channels=out_channels, kernel_size=1
+ )
+ self.sigmoid = torch.nn.Sigmoid()
+
+ def forward(self, x, lengths=None):
+ L = x.shape[-1]
+ if lengths is not None:
+ mask = length_to_mask(lengths * L, max_len=L, device=x.device)
+ mask = mask.unsqueeze(1)
+ total = mask.sum(dim=2, keepdim=True)
+ s = (x * mask).sum(dim=2, keepdim=True) / total
+ else:
+ s = x.mean(dim=2, keepdim=True)
+
+ s = self.relu(self.conv1(s))
+ s = self.sigmoid(self.conv2(s))
+
+ return s * x
+
+
+class AttentiveStatisticsPooling(nn.Module):
+ """This class implements an attentive statistic pooling layer for each channel.
+ It returns the concatenated mean and std of the input tensor.
+
+ Arguments
+ ---------
+ channels: int
+ The number of input channels.
+ attention_channels: int
+ The number of attention channels.
+
+ Example
+ -------
+ >>> inp_tensor = torch.rand([8, 120, 64]).transpose(1, 2)
+ >>> asp_layer = AttentiveStatisticsPooling(64)
+ >>> lengths = torch.rand((8,))
+ >>> out_tensor = asp_layer(inp_tensor, lengths).transpose(1, 2)
+ >>> out_tensor.shape
+ torch.Size([8, 1, 128])
+ """
+
+ def __init__(
+ self, channels, attention_channels=128, global_context=True, batch_norm=True
+ ):
+ super().__init__()
+
+ self.eps = 1e-12
+ self.global_context = global_context
+ if global_context:
+ self.tdnn = TDNNBlock(
+ channels * 3, attention_channels, 1, 1, batch_norm=batch_norm
+ )
+ else:
+ self.tdnn = TDNNBlock(
+ channels, attention_channels, 1, 1, batch_norm, batch_norm
+ )
+ self.tanh = nn.Tanh()
+ self.conv = Conv1d(
+ in_channels=attention_channels, out_channels=channels, kernel_size=1
+ )
+
+ def forward(self, x, lengths=None):
+ """Calculates mean and std for a batch (input tensor).
+
+ Arguments
+ ---------
+ x : torch.Tensor
+ Tensor of shape [N, C, L].
+ """
+ L = x.shape[-1]
+
+ def _compute_statistics(x, m, dim=2, eps=self.eps):
+ mean = (m * x).sum(dim)
+ std = torch.sqrt((m * (x - mean.unsqueeze(dim)).pow(2)).sum(dim).clamp(eps))
+ return mean, std
+
+ if lengths is None:
+ lengths = torch.ones(x.shape[0], device=x.device)
+
+ # Make binary mask of shape [N, 1, L]
+ mask = length_to_mask(lengths * L, max_len=L, device=x.device)
+ mask = mask.unsqueeze(1)
+
+ # Expand the temporal context of the pooling layer by allowing the
+ # self-attention to look at global properties of the utterance.
+ if self.global_context:
+ # torch.std is unstable for backward computation
+ # https://github.com/pytorch/pytorch/issues/4320
+ total = mask.sum(dim=2, keepdim=True).float()
+ mean, std = _compute_statistics(x, mask / total)
+ mean = mean.unsqueeze(2).repeat(1, 1, L)
+ std = std.unsqueeze(2).repeat(1, 1, L)
+ attn = torch.cat([x, mean, std], dim=1)
+ else:
+ attn = x
+
+ # Apply layers
+ attn = self.conv(self.tanh(self.tdnn(attn)))
+
+ # Filter out zero-paddings
+ attn = attn.masked_fill(mask == 0, float("-inf"))
+
+ attn = F.softmax(attn, dim=2)
+ mean, std = _compute_statistics(x, attn)
+ # Append mean and std of the batch
+ pooled_stats = torch.cat((mean, std), dim=1)
+ pooled_stats = pooled_stats.unsqueeze(2)
+
+ return pooled_stats
+
+
+class SERes2NetBlock(nn.Module):
+ """An implementation of building block in ECAPA-TDNN, i.e.,
+ TDNN-Res2Net-TDNN-SEBlock.
+
+ Arguments
+ ----------
+ out_channels: int
+ The number of output channels.
+ res2net_scale: int
+ The scale of the Res2Net block.
+ kernel_size: int
+ The kernel size of the TDNN blocks.
+ dilation: int
+ The dilation of the Res2Net block.
+ activation : torch class
+ A class for constructing the activation layers.
+
+ Example
+ -------
+ >>> x = torch.rand(8, 120, 64).transpose(1, 2)
+ >>> conv = SERes2NetBlock(64, 64, res2net_scale=4)
+ >>> out = conv(x).transpose(1, 2)
+ >>> out.shape
+ torch.Size([8, 120, 64])
+ """
+
+ def __init__(
+ self,
+ in_channels,
+ out_channels,
+ res2net_scale=8,
+ se_channels=128,
+ kernel_size=1,
+ dilation=1,
+ activation=torch.nn.ReLU,
+ batch_norm=True,
+ ):
+ super().__init__()
+ self.out_channels = out_channels
+ self.tdnn1 = TDNNBlock(
+ in_channels,
+ out_channels,
+ kernel_size=1,
+ dilation=1,
+ activation=activation,
+ batch_norm=batch_norm,
+ )
+ self.res2net_block = Res2NetBlock(
+ out_channels,
+ out_channels,
+ res2net_scale,
+ kernel_size,
+ dilation,
+ batch_norm=batch_norm,
+ )
+ self.tdnn2 = TDNNBlock(
+ out_channels,
+ out_channels,
+ kernel_size=1,
+ dilation=1,
+ activation=activation,
+ batch_norm=batch_norm,
+ )
+ self.se_block = SEBlock(out_channels, se_channels, out_channels)
+
+ self.shortcut = None
+ if in_channels != out_channels:
+ self.shortcut = Conv1d(
+ in_channels=in_channels,
+ out_channels=out_channels,
+ kernel_size=1,
+ )
+
+ def forward(self, x, lengths=None):
+ residual = x
+ if self.shortcut:
+ residual = self.shortcut(x)
+
+ x = self.tdnn1(x)
+ x = self.res2net_block(x)
+ x = self.tdnn2(x)
+ x = self.se_block(x, lengths)
+
+ return x + residual
+
+
+class ECAPA_TDNN(torch.nn.Module):
+ """An implementation of the speaker embedding model in a paper.
+ "ECAPA-TDNN: Emphasized Channel Attention, Propagation and Aggregation in
+ TDNN Based Speaker Verification" (https://arxiv.org/abs/2005.07143).
+
+ Arguments
+ ---------
+ device : str
+ Device used, e.g., "cpu" or "cuda".
+ activation : torch class
+ A class for constructing the activation layers.
+ channels : list of ints
+ Output channels for TDNN/SERes2Net layer.
+ kernel_sizes : list of ints
+ List of kernel sizes for each layer.
+ dilations : list of ints
+ List of dilations for kernels in each layer.
+ lin_neurons : int
+ Number of neurons in linear layers.
+
+ Example
+ -------
+ >>> input_feats = torch.rand([5, 120, 80])
+ >>> compute_embedding = ECAPA_TDNN(80, lin_neurons=192)
+ >>> outputs = compute_embedding(input_feats)
+ >>> outputs.shape
+ torch.Size([5, 1, 192])
+ """
+
+ def __init__(
+ self,
+ input_size,
+ lin_neurons=192,
+ activation=torch.nn.ReLU,
+ channels=[512, 512, 512, 512, 1536],
+ kernel_sizes=[5, 3, 3, 3, 1],
+ dilations=[1, 2, 3, 4, 1],
+ attention_channels=128,
+ res2net_scale=8,
+ se_channels=128,
+ global_context=True,
+ batch_norm=True,
+ ):
+
+ super().__init__()
+ assert len(channels) == len(kernel_sizes)
+ assert len(channels) == len(dilations)
+ self.channels = channels
+ self.blocks = nn.ModuleList()
+
+ # The initial TDNN layer
+ self.blocks.append(
+ TDNNBlock(
+ input_size,
+ channels[0],
+ kernel_sizes[0],
+ dilations[0],
+ activation,
+ batch_norm=batch_norm,
+ )
+ )
+
+ # SE-Res2Net layers
+ for i in range(1, len(channels) - 1):
+ self.blocks.append(
+ SERes2NetBlock(
+ channels[i - 1],
+ channels[i],
+ res2net_scale=res2net_scale,
+ se_channels=se_channels,
+ kernel_size=kernel_sizes[i],
+ dilation=dilations[i],
+ activation=activation,
+ batch_norm=batch_norm,
+ )
+ )
+
+ # Multi-layer feature aggregation
+ self.mfa = TDNNBlock(
+ channels[-1],
+ channels[-1],
+ kernel_sizes[-1],
+ dilations[-1],
+ activation,
+ batch_norm=batch_norm,
+ )
+
+ # Attentive Statistical Pooling
+ self.asp = AttentiveStatisticsPooling(
+ channels[-1],
+ attention_channels=attention_channels,
+ global_context=global_context,
+ batch_norm=batch_norm,
+ )
+ self.asp_bn = BatchNorm1d(input_size=channels[-1] * 2, enabled=batch_norm)
+
+ # Final linear transformation
+ self.fc = Conv1d(
+ in_channels=channels[-1] * 2,
+ out_channels=lin_neurons,
+ kernel_size=1,
+ )
+
+ # @torch.cuda.amp.autocast(enabled=True, dtype=torch.float32)
+ def forward(self, x, lengths=None):
+ """Returns the embedding vector.
+
+ Arguments
+ ---------
+ x : torch.Tensor
+ Tensor of shape (batch, time, channel).
+ """
+ # Minimize transpose for efficiency
+ x = x.transpose(1, 2)
+
+ xl = []
+ for layer in self.blocks:
+ try:
+ x = layer(x, lengths=lengths)
+ except TypeError:
+ x = layer(x)
+ xl.append(x)
+
+ # Multi-layer feature aggregation
+ x = torch.cat(xl[1:], dim=1)
+ x = self.mfa(x)
+
+ # Attentive Statistical Pooling
+ x = self.asp(x, lengths=lengths)
+ x = self.asp_bn(x)
+
+ # Final linear transformation
+ x = self.fc(x)
+
+ x = x.squeeze(-1)
+ return x
+
+
+class SpeakerEmbedddingExtractor(object):
+
+ def __init__(self, ckpt_path, device="cuda"):
+ # NOTE: The sampling rate is 16000
+ self.mel_extractor = TorchMelSpectrogram()
+ self.mel_extractor.to(device)
+ model = ECAPA_TDNN(
+ 80,
+ 512,
+ channels=[512, 512, 512, 512, 1536],
+ kernel_sizes=[5, 3, 3, 3, 1],
+ dilations=[1, 2, 3, 4, 1],
+ attention_channels=128,
+ res2net_scale=4,
+ se_channels=128,
+ global_context=True,
+ batch_norm=True,
+ )
+ model.load_state_dict(torch.load(ckpt_path), strict=True)
+ model.eval()
+ self.model = model
+ self.model.to(device)
+
+ def __call__(self, wav):
+ # wav, sr = torchaudio.load(audio_path)
+ # assert sr == 16000, f"The sampling rate is not 16000"
+ # print(wav.shape)
+ mel = self.mel_extractor(wav.unsqueeze(0))
+ spk = self.model(mel)
+ spk = spk[0]
+ return spk
diff --git a/fireredtts/modules/flow/__init__.py b/fireredtts/modules/flow/__init__.py
new file mode 100644
index 0000000..221b727
--- /dev/null
+++ b/fireredtts/modules/flow/__init__.py
@@ -0,0 +1,24 @@
+from fireredtts.modules.flow.codec_embedding import HHGCodecEmbedding
+from fireredtts.modules.flow.conformer import ConformerDecoderV2
+from fireredtts.modules.flow.mel_encoder import MelReduceEncoder
+from fireredtts.modules.flow.decoder import ConditionalCFM, ConditionalDecoder
+from fireredtts.modules.flow.flow_model import InterpolateRegulator, CrossAttnFlowMatching
+from fireredtts.modules.flow.mel_spectrogram import MelSpectrogramExtractor
+
+
+def get_flow_frontend(flow_config):
+ flow = CrossAttnFlowMatching(
+ output_size=flow_config["output_size"],
+ input_embedding=HHGCodecEmbedding(**flow_config["input_embedding"]),
+ encoder=ConformerDecoderV2(**flow_config["encoder"]),
+ length_regulator=InterpolateRegulator(**flow_config["length_regulator"]),
+ mel_encoder=MelReduceEncoder(**flow_config["mel_encoder"]),
+ decoder=ConditionalCFM(
+ estimator=ConditionalDecoder(**flow_config["decoder"]["estimator"]),
+ t_scheduler=flow_config["decoder"]["t_scheduler"],
+ inference_cfg_rate=flow_config["decoder"]["inference_cfg_rate"]
+ )
+ )
+ return flow
+
+
diff --git a/fireredtts/modules/flow/codebook.npy b/fireredtts/modules/flow/codebook.npy
new file mode 100644
index 0000000000000000000000000000000000000000..072ae3fd249a628d54cd1fd60a86fe0cc7cc3993
GIT binary patch
literal 131200
zcmb4~`8QTy*v3tT%8<-tjtm(ZH1M2zKNYEDNToDLvr2HP`EX|AY7ZaMn8ehrP~Pd+mK+_vgBHip#=zZcD_)c8dj?tXaEbwZF+U
zJrnzN6HP4iOxA7n-|6qO#e1v&nzjEs?&!05$J)X19UFYMtsVRxJKjRiX8e@@dzo73
z1?&C)pJAK36>ILST1gV!1m2sjk(~c)nyRi0Hb=^7*tTK9Klgsa$*cRw@mL6!gua2j
zR>gQtL9QnBr5axneFy5IN~p1GDXH4|3tN{hVG5C7@Y>t)Y{AuLI_04#{OhIwTlC6t
zpab}=y_(IOmP`Q;qlFS7w`fQ6=$c0>ev;Wmec{#JujoPaJLYXzMwSWt
zh+DCn+qmWfIXm8@6)IwEtgAICZ0&@C>;^FKNu;ci@0p(pC)`vlg+q4iWgU@HbU~t*
zby#MSSbie@ZJ0m?pG(=oeAk+?_;EGY7FyIK5ekI-WqwBV8MejKiAkECV|QI<*7)Dc
zV&18$a4d8*xs6p}O1*BR^zs(0@jw>uUd47B)X|E!aW#Iv>)4{hfwb}T5nA}*Doy%+
z3O0Wq#u|gBvbq1|vz1z@v>-}Oc&uVX&8nSB?3GagC)=V%=`qqZ%H5r4_j@6=1Q`i)
zUrW=)?GZRiR)>sBOkk2tB%Rh^Od;tFtiP?!9*4WJt-h6P*v~OI>$@6<4z08+Jb;aJ
zwXT^dXHKuW)9Li$>8#1n1~WEl;F%-7Z2!#J_+(cBrP-KL_dF3BpEa3&ubTvwq|b&F
zRMGy=_3ZJ@Og7%Hn&cuk(W2c>Y+tVp{E|%KycftY_#w^SuD9k^Ryu;^DHXajqkz|#
z-~%mHjhvCl5N%ab&?v4G#M6q=$+nMQu+<7$^)z9x!+-QMHGo6&6o^oi#J7ee+>G^h
zr0`@0FFs%hjq9Xw^q&^kcjy5(zqcDznzixUM;EHkFM-4>Vr;^IC(HdW9F5$6gW%)|
zJR1EAV%t;UlCm5>-98Tf-H3+^Q=f51uPU>wWGkecJoHq5$!T5LMbo?v(l*m-&|2+;
z0imnmL)3roR9-`)CbH-15@TK7vw3n1)aXYuMTO%HD8wYZ^Z-wu;
z+Q~)Wb8tJhYu-n_OhvwTl>}KHJpqAoh9o^N2WAPEV)@AH{PNy;T;Q%*Ok6z}{+_)K
zXO}yWlK(XLa`FmqU}gt@!t6vp4|Z|KzlP(?_lLpuU=&^*r%l<4k+?==j+^3fRZvQXuOQt4>AF%-AOkLUa1Z)44~M*d@2D!g2z$L;(wl1??%!`|Ml
zaA=bwjJqp>sV?1k8*iab*?l^QO_usX
zj-^&RGb{UMN*mh>DjK?U)?gJX#WPQ3!dB
z=rLb~9XoajN608K#h5l&H>v=4W*E`rlS3)3(4X!G6ye$0m89P=iaU`{PLR3lK+TVubAg2HjjhiH2bmzGFMeUGt>I0AKREs7y9TEQH=;+gY~VDS`31
z@ytHy5eX+=gS5~CC|0zNg6x-4#99S*`PN8wENTqPzYt57#})}!dqz`5??LLiw16(S
z=m^uN#!^&AF1z>rC>_{pN1JXefN+Deln{G_N{aTeMtTTkN^Z2r@)wY`9cdd~ft@>&
znYZ*aNZPQSo!D1`sZ9hb&(7gx&uM^({xr$9kG+~btY-GoGI|izOq(hkQIedf?!X0h
zW-VMsTO2tq=@tAvZv^QVb?K5v4kSO*r!G+`ZOW*_;mxYd
zm&L=TsfJKSaqRjCpo}sHW-;##Y|V+Ji?@u)DC`T>_&-Ab)JEL#YCCPQj;Hvnc+vaA
zGL&(mh#d}+#Wi{AWW6{U|7#J!^SlqD5Qkb){MirJk9wiJw}31gEU8vmTc{zb!1#;N
z)V6mtTe^M`wv4>XP9=Pmwy8(Zv(%iqU0#ksB2D2vF)u3Jb%0)6
zyGp~X*D(I>Nm>`|&)x*clH?d2zax2>2yZ&RBLpNa#=+rMpp%Q+
zSnILT!XIPRXzDgYc1-Ap_JSJb7_UQ){b^|aM-%g(%Cea*@w9wuCzUAmP()4}r!u{X
zGsXpA|8_AGJUqnaOZU^qn*r>La*Uv5*F;pm>J4g1ax~?XDm(Xj0+TFuX9wn;p`^PX
z=--TXPHy;cdUM*2+Csg_!0`Z>_0&SJ_)!=?^%(mzB@}H8U$BAd2s+!81y7@Yi29n$
zansK$^dzqXcgic!$$L%A-rf&GH7*Oz)-|AeVLYUNUq`Xk1W6vL_{?Ji7>_x?rkQi>
z<~(hCA5S&@NDg@8TL(JimyGgl2T!pTU235P68_2wi3)g0wUxx=N#r-}>QN3p5DN3sXX#-ydP6!+?vbEaX&tVl_p|99^)_hQx=Zoy1hbZ%8*_J7av
z-qQ=wvVaqnJ$?%pXUWi$ruU%nrUC
z-RACTy733Kte~f9HJXzHMfmmd@a^GaLRpKY`X*{t`KLm%ysuQPA9w&bf?DhOu#bS$FCoJoV!q
zs4u?)Pm6R|pLhq2KWqpy-4t1m)&-m~Pz|f+hSKyIG2G4x$>7@e+`jHg2q&kw2pbAj
z3H&Ny<*jD2{$a!pILfmKcVD(X(1#}b3fP0a0`l#=3r$BNVEu&CoM5jFJGJm5c$B%~
zoJZ9V_iQCMZ>BGtpa@#VC5c?<_Nm(j>p~I`r6W4nN~}@!viF
zfW)3_7?M7m&IP=|@3f35YZtSUtbO=ABm?A5B!Z537hG623Y$zm@!CC#w5`L9dYpw^
zq*V%eoo4J#iWH`ZTzSh=*`WPuAKSEPJy;D-hKW8Alr!6gyJQjuTRY+*V!=tStxt>H
z9&!)E|8N5JxwUwHg*+L3Pr+55TKpFsKbSDgjxDemLnm5n*)!Me@Z@(B9Cxn8_Sk)x
zuD73ytj$H&kBR)Lxo=@X*i5+g+?F(-t%eNkH6Shu5Xp>GVk)Rg=U1M>ZOwbwmly=O
zzjv7PULl^Ucm|*|kIj2Hm3w>~M}z)FOZKVi
zEL__)of!{_VS!((@q$J+&ipFPBvkK{rnyng`u}FKDVaR#w~ygZ%y>&5*PF9xd*^U^
z|1}A?OOYu5bU&;coI83_C$sAjr|NE9tPb*mWf>K(%D1(c87K*DI
zV!24;k1%2PczT(16dS_cVPa=CtC%Xmx{GIn*qW^{NBs?Cg_*E*7=!Vbo}jexW^TUh
z4Knzvj5d>6=$)Dc)A?tK+lCz#b@j-@R`*txD=EVgBsgZ>e2rFGT!O2(26Gekaq8if
zd}7gL_IT7|EJ`VaaSJ-p%AuOB-(QJyx&3@zbX}F*6nD1%yAf?nAIUQ7U&4-oYSF`Z
zbGl%pOXepf>5G3U7C*mBK4F&kqZixutiXn$z1SBdzKEf(n3
ztW&#=wfA#KZ0#_X*{zQ5<4&UU-gzvl@*>VTb%9J@s8CsP7PDG*2WQlWfYlj4X008cEu=Gef2Eu?eeDP$+0Z4lz5FZHDLeZ7l@V<9DN)osMXBn-ff&upT8GE)#(b5
z&>IWAzy9FtMo9|Lw`B>dZ{k*xAk9y4TztYa$bK@5eq)1T=!(cGI4n*ri
z$TWByEXX~E1B;t^@%vf4-oY6#>v|f_(bHvebKjs=M-G*WR50T|uR*@xgF81Bfz0_@
ze5^Q$Ma|}6W1t@f*C^3FZYZwk?8B=6y3xr~hn(DN_*p_T5Y*bzL(|19@6Hj3dTPTQ
zy?SBqlc6LzuNy^+&Di4IwcLNzb*SyO5v9L)fVZXyOP(21ys;H-GtI#T{0#0+geo+B
z&fup@ZN~t)xA@<89h#c(3pel~{0Z1X+cN&*@ScYlG)Ba5*LL>3a}6d$OR;KMj+9&7
zc;NwG{QJ?E(q-B)DOsEib(dw&rvBo7n+zdwmp-OvAVELnH=tU_Rg`~S$dWY&d#v}y
ztZ$edWlTPUKn>xC!U9~7dU$Pssvgq9^n)3;h=U$lO+|;gDYvW
zgIXe+KfhcJUiNs3>YZ1EhL{m`eLKqj#?7b9yK1yceFvS_Jj!2wK8(`eS+L;rv)ICY
zf^>^RV)R8`ZoFJlFx?^w_;_m|xF_j2sE`aE97sujg0%E2%q8BHP{Vf5;0
zG#sD8tVnHqQ6K|nzw6VYhJzxTsrk@vTZM*}+9KF`2M>6ZpxG@sK&ozR-^mQ!1Jd{~qHiRqx?d4ue-{*UJ(z$a7F41={j*etG
z@LK!b+1Ny1784zpj(&zNTw`M+*oBXQ&e&2u?dv9vg{#2PY*|XN+Dk>gO3eK6W4>U>R*YV=l&P=u
zW;d!_NYB6(H@$b}M;=(k!W4}#RicCsFVBG~F>2(Xq{im{jl|-JR=grQ&pX>{vHfzB
zXkXbxR(9ku*r|@
zN|T2Kp|XQK-l{x|OCHuh=4cJDt(ip0gZoTVY&YZth+uZ_MDB>qJg}YpT$EOKihJ>3
zG;R#Pha)`2VQilmjQM$yQ?GA?F)@dEy!DxT)4L4U>!xw?Pc5id=9&F^opAPtKP2d|
zcLVJM9MpZP=2z?e;W}0)@%NvL5$Es%Ivu;fMyrzRiSCD#C9$FnJ=ySLc^X`pZ6xS$
zxdVD1hSJ-^iui8jMk*3{;q!N&c()T@xZr0AxUyK6mz?at`+O*)h1*WB*LD-Bb>Sj~COlCuq-W7;!eksn
z=~fHy$G?10<5Gdp#(yziQgw+vP+UWY`$v#k(?b~ZH;u~M%4wg1o>0tP$ddDkVr?#g
z>1I>f6y`}bx+)YZ+|LSY{8{DZax$7%hU#M~_=-p!w*C1bO4+9dP~k~#>{?-TusBO1^9iNf-s*I)7Qo|oXq5Y)UVn?0cuZ~C~XWo
zuzx>}kKA4({_r8Qd_aOhs)Bd;Gq84HGFV-|jMZ+2!sttqH1xtoTGliQGQT9yz-tlr
zLg^rQWjNB5Cyz1j1VWlnnR{iowt8o{)?iQ7M)&sar?K;@Fu+WYrb%bRc%vkCW{nY8
zn_M7;eQVhvk2b73m_(NsFQdnLL9Ev*j`Yt*!>;HvBzE;TU!#2q-5i_gwc9QDc0fdj
z{*{vNzFd*bAq^1tm9SU%4b+B=6uyo;%8mLkr-E`K_Bgs=}>DtFBJh)Mk^?K
z-#EGyZZ32jNN2zE@8HWX(;4L!(;?Xj%rJZ(rhOEWN4FKJyPN>)fk&Wn#+bcSbD$o8Pl%Vo
z{GK3oPz>17;N|33Gqpx4A(JUgYrrdyttj}`c|2Tyk9!p&1=h_xvze{Wa_y>+3%Jhu
zB#zP56%V+1y_I~*o+~&rQ$+i-lu$KdA~k3uYx)v_&t8NQ9`UX(&h
zuNB*~-518kJ>+wP?bwu<6WDO(GCD#Abq8{+NHvy1Z(peFnzNm5A72gz=d!_yT}7I7
z4kiVk!R)~`T$t2>aZ4Q7$FCdtz28oXmhZA4d!?0CN^{CR8kfWmkgZY#f5UXKlHLdq+x}!lk*^BUm7>mEuR}Z
zP|2&i)$mVkwAs{K34BZ_@!fXySijB^R=4Vtng1d1Qb@zF$OvBX+C`B^L^ame`J;P!
z1m34S%yMQtixq>N`cY_DHH1z7EM$E5UEI27Hurl%JvXCaI8#jB#ryr%hT*H{U~57R
zj3|+YTMD_TePArjw^_tJx~~KROH<6<77usFJOzFER^*Ex!OgFJT-}`sF#XwaK7GPa
zo({*tp2;@^vR*@>O<^Pc;=URqCP6eXfx->Y@xP@0Y+AdfzV+L+=5eK7#
zJ=mO@BbcTb#|yKS&}-c}xMLv=?aFaH=kSVm{oRCFg-gKX(P$7KHw7P^fsBDY?n-*Qh`~Aq%Iap5nh4sDt@!3tD8x
zVfm)z;Hi@ZRbSSlvCl`C|6P&|91n+vt`cZ7C>GuEXu*B9e(*V?m;0u<3eH?IW%hYL
zfT9+HT-q@9^Fsq{GUp-FdXPzq9iVR?jnH4Gbda~WGqa>CT=M>8kkWYy4l*|E*SEKL
zJ2MY`E86gC>|3b$-avPhG?`1$Ihe_Ifm&-S{n_LNrfKqYudEmTD=dUp$tHZuDRn0P
z`wuv`>5zGxE;$GMgptQr;4E4W5457Wzb9{_{_bHE9yo#$))>PVljWcmA`44ouXF7?
zsv&lq3t1LNv$z+^?6~$Arg2JAWa6sHf)`Jtn3DvBZ_H3)qAz>Y5(rKk$Iw^3c~F#7
zj1MCMVfean=-B)elMlz^_;o|!oK-XHPhU%6Pn2P;rxDckUgH+FuY*5_uW})#fi!&Q
zwaaxIM#50bShp#%l@nhUQgMM(xc80^s
zPdNABixB>KB|5ear)Qh=P)%N%t+5KgyQ(7~I4T{s$`4@&j~zq*hxg$7b!&X{lCgv1
zA96{jYk2-3cUzoO2S@`&*10C~^V_&X^QSPhJ^lf1n
zM3){V{YB9h+!X&PhjiUq=Af647>cygiTnW!h&pm68lj?vgg%c)m}x4S?$BFG#SF;
zS?%!ojU#Ie9?c%A+R?sa{kW5_#hW>cSmAWln$<(>nU4MtMo$|d->r>$BsnJGvVitp
z^J6z2*TD0$iS~Q%hp}(!OPNlo4$T_oKm&*8GAEh0c(6hQ6zwf&d2UU4FMeawc_UI8
zlglae#0bW|9?l)NwBXr$U=!RPk<7D9_9c|Dft6{@IXuBWdBuc54Y!aS+TKF4Q!lRh
zbC&&^7tD>F6-36BGoWWyI_6a`gi`4cTrNJG_OwLfda)j`jgb*jZ$1^K4}-3h7Wh0{
z5fh$|1D&ZM+&%9i(pvly^&;!}g^nTQxp6-8_gW8G@-ujYfA6@>W7o05P#^wuQ6DHa
zy3_kaDL5!6&6YY>!Z()7UH?9j-8A2U3odUV_Z>mp`^Wk$b$kWRphK{zLxHu;^<|31
z&#`;KSP;{E#b0+#V1mjhJ#)AaAS6s+4xH!>1>~!PViIEZ>4&&Nf__
z#6r?^mtYS)ZsNiOBb+uSlC`S$K~(ECtUq~zVu$502XS9Imi(D+xd!79>x1;JMVS)I3^9OTUF+chGt|RWXOew<9i*uomuoGg0{NZ!uGlKL7>K
z6>C=f?&3ZynT)dpk%MF{!P1TZGNAmdS{2bv?
z?{2J=yu~aPQYfp(hB9|vfg{BggJyO$spl&Q^MW+lx9U8~`E>`fk{!rp<73=0c`;X3
zxtdXX1S>m}O?#u#1dFTkFk`bU^Lm;@FO)|Le}+%0**GMb$SRobt~gER9=TMd|C22^
zR85W~WF>LBHLvuPxUsjyh40QCqc!WNf|zhPZAzL*-$I_Vb4%Qq)$?vVtQ%3YL%fas
z-l&Gfq8});Z9dn%vG`GT#B0o|a+e%dQAkZ@5Wu+qToy;6kiO&Z-vWf=)ytSwGp&rglF@
zyUG8NQFsJBSr@^MmR1htj%GnZ(s`P8;E<^9L>{_FS5lX)6kEHmfXcJ0$!@+Z+O4_*
z&XtiY>edH{+%GMB{^~g9@Jm?A?I>8Y>n2`SQYN)AMJ#qlDa0tgv)?g&0~4%^W<@Gn
zAmxAgO0^A@uV}P4IFJeI9(#@@DP)3b)xR0Krx
zouQ^I==>L)F5M5>8vb;PjRIQr35ujb1Un60@VAy-!`du)a(`#Y=1m<|<3DpU8)B-?
zD$_Ty2N$yGXigB5U+PO93OdZ7YYz-8My@P!3+wx7#x5_;#_eHV@V-0^bmyPL=s*ot
z5;JQsyYY~u>#lPPdvs~Cs2E1&Y@^WMb)cN{QLuDJC5)!+rl}=A*XQQsOhuuTr
z(XYwSmxDO@iV5yrDWFuThb-CJlG3ZKi4V{PmoassldbQ;KCc&-EH|Ku$KRsqqEPmz
z=N9Xfk|le^G-%Ci2I0SC(Al7ig@+FE<;}}T_#%+5k9)yO+!%D+*Y;XykJPK9CNumqev~txEdbS
zUqiR3HSAyBCm7;-#hz`Qj-&rONJ9T+KGiN38yiIk<Y&d9UTYv*v!|E%V-HBN!zL?hHIDB{He
zHsSoM58=eLIxaY61QSd#hRME~f|zdxtk~lce{=l>G>=nedc7mavm*qKiN52=j|(90
zdNlgD9poi<%R%0vzo2`58d&Xhf!VfG=zLBw=ld-k$Nn0IR+5|esD`W6C-W9zsz9Ed
zcCEz>y}hLLUp2qLVDbW471_OG=G2S;7rs=N1h^7`C?>UmY*7d_$uTgW@yC2tO#3f_Ae2dLn%Lc
zRX$e~zZd^)?8Mjmr0~fXNBr4REwIZ-<9Va4qKp%Ec=%csCRKh%hsif^ndQ~$A@^sK
zR#!cSCGa@(cpN)5QHm|v%VVs52edl3;JqcXqS?#}*G`e36_cG&>02vAMcx3X@|ehj%~hTLT5L!2wfS6Jy&feU{ef)eK4|oD
zg~eUN>5^U)v`;9<_5cTnJ28r7tB>N&kGcq-L#_*+^)G`To_+RB-YZ~@f)x3E+Qz!`
zy3ktk3EEk@Q^eB_`_gZx@NwV`kX~elx7M^mh}#@8zxsijCemi6A&l!4zsP|_JU>Mv
z1ir){f#WmQ!Um6*IQ-FCvI^5@VipDbGWm1p^3Mvae}{5XbISO0x7IQ5-pkxkFXxqi`G(R=IOY>(ywL
zXE3+rL=8xsXyfN$bQ+V~$_aG>`H%xsAbocl59~2Tq2p$G`XqM~6(sb9NS>GOCwzlTc!=
z-Y%?RRyJpqm&#^zsZw0{J+5L$H(#8dAZnT#LcU)-FmmJvJjg9)jr(MA*t~JbPtU`I
zP$LMOZ%bQ8sDf+sEUwYSR8%hzfv03OSiTiU`Q^@d;_N&sP5cNQ&(iU}r2xf$UciHT
zd62Z72mY4|x5m1h(^@wmDx5T&B!BM3+6Fb|V4RQRZj8g_^E!Cf8UqS!^?(7s8V>75
zQ|;Lo&hc`&CTH%fy_+nb4Ry71|%k^9Qs7
zV9%gGRg53`;idE8xnPhnIG=@%zsWSvlZl&09pgq_@Wt*wpP|C^3%&`<;Wmp+g=B{k
z=<)6mon4@hmq3!X{kIUx4`fof{}6UbwFO>Il4b4bw$#5N48-!pDfr7mXn2zd%_<4i
zl_Q?Rh~LGKd50Boe*$We2H;8y=eTb5m>pa51mrtVe#f5yt1qm
zG@Ax%Ve~5j_cj@O{w-!+Z)UUAug9})9RqIW#+^`7eH}+vci1l&%pKcb-O3z3wDLZh
ziKsJnGi+Kj7kvlwod%w>QK`ENcN(2TMcaB1>J~BK=rcG?v>Psdf5+)16S((%5V(dp
z@DY_mNk+vFBZ3w}b*l-TF;rv`@~ynB$#7mnH54|MO0p;OWVy$0rNJiGmhL|g2tLn<
zgs1Ly0HPdD7Y1|3*(o@rBAh=Ia036XyUV}V-HAn&ioEdoH*VYJSJ0z%9cRw8X3rmN
z;-;uIfmgL0I^C7#T^`%O+QpgpVC*q|(a37hT`dQJJ3nE$wiy>YN(@e54k|tsL0I=!
ze4!AIwqsvI+o}v$JhYPa*B=0rPxV|_@kXX(aEQ$;ngRFw;=yye2OSI6hS8>ZFyn0`
z9Ab&^u;VsJWGAxKnvu9n(Tr_3{06RyGuW1)ay6e`hQiIHrGq)Eezbg10R01*bid*u
zceDQ)>^bLwZ;!s@JYMGFCl5uMts>8o`9bFTU_ATfIOxR;YD!1vSPEK|$kIoyAX5`-
zHf`HXX#P0@tbVUyFJJ00o0XmXEa{;vRD}cM<|-5>>M+|cq4an7S6ZN-12Z;{qIKU(
z;af~TouByzynp40l$(yg*kem6WGrLapYqt|io2}s{B>@vrIN^+_d%)GA}pHjO3|Mm
zv+(abNd15&RnPvx1-(z<+Fiw1{p#m9H_d=jF1*L8$H^>U;c1pua}ou+HbKnMPIypQ
z0$V-?;q7r_F(J@~=x92bIkocx_pi{xRr6VUo((_pGxBu{(qOaeX_kB83M5)~!^Vho
zTDe*l{`?Mumru^a3*{89A=85`T=)_b@8)0<7~>XYZ1C!+09h#^&3h`b8WpmH5Zm
zxQ$~*oDS_t%fnXFQnuxC18n}f3d8rS!fKP>{QR@|-1Mqp0>Kd-e(`8K?!lR8QTgj@
z*mrj}H@B~wpFgsf-#aZ1cVF0wd8-=*^)-ia_%(4pw>}F6X1<(LqdYA$dp_6~3~=Wq
zl%fBP6`bzT5y|g-Dj50Q4!E}oa4_N-m$}j$Q`Yaq$4mcmcisgGEcFb*%%FyIR2s*n
z2tDv|tP+Ta0L1%A!^s_;{2ec2RJvg$@H+mO7hJq=-!R5nWGLV;s9`-CsvGh5GvdIZ
z82Meg-GanpP2BG7&$usAF9nriMlj{%N<3$-h+TdQ;j`gAZpXoLe5dnM^n)`Lj0swU
zS@N49Z3%;zwVB+2gd&vOyUlqRmx1}Vqr5>wB=<3{1>CH*;)dxN+?Qc%vG9isi`uY|
ze|J#|dnFX{*|HQtW^it`n7Tji$`0l#KjcH6Cl4isF);tNANMRfwMH@?6t%H6_okLq|S&xz#ir-c{$dBOG5FR(4u
zgf2W&#W?v?49kgwr(t{eHYWwN(F?{JpM%)#*^SeVSkQjWG*0Q;Oma*YrxI&PdSG{x
zIV}DJC9~z|vQUnVo+wMQkBi~d=81Uut2EJz5Y+ZR4i~mp0?eBOX6cQnQuvY8J<&Le1>--@~Z#>ui{2O}T&4seG
zH~2oV=#N9J2pa>-rnPl_hh~`N0_&^Q{YZ->rkg
zLNl_NUQB!Lm@&;uibNq~>`FrA|#+1#zJT-A+O_CeB!)YSq7
z`m4Y5Cp#|lnohf@e||q-@U{<)Y@OK`@4tNYBf?~xbNK9QH5VwDMsvlK;GRS=B$g#r
z%T|<&jyqMs535z^XqSynt}di@=MxMp3CBPBM~hsG0<@BiEnr7`6;#
zrW$jLI<@fJ?8|&fO*!BBvI4#S)8o=|Yry#g%aSnxd6e#(RK3?3|i(gGP;|l|&!Mh=<+_
zu6WzB;%YPA!Ob1s+*}G>(blY@z!$|vhQoc^3;1Sd2LH=V5+1yz!Q5v7u4=sw0mmen
z&Qc3X|2ZD!U$w&Bqg(Nu-XY968U?|Cj1;Z!mc@VD~;=zo<0d7EWPxo9cVe>w#Z
z4GeJQLHls^I5}*udW`Ek5bSq+$3M#~;af%+>SZ{i=|5}MxpoKlthJ91}L&%!#|%37A>a1uyRoRGl&lM;+~DEa*@IZ{NHF(sS%U
zcupN_GOV%R;yQ1xy`Hnx{LQ!YL~wP%3A~ku9$LHztTKCk1Jg7Q;yL3RqAiXWaoxa5
ze*R$IloZsIWnj
z6f|IT^X-l)rT&e^czvs0ks7{b{lx@fz<2%T(pfzDulN+Rb37%XsM+~M&|u*C?*
zbBrk!{|2AylFVw}J=!)UfWmx$3&?+lmA_orf_I1c_cPMr*^Z$sGfImk`wXFFQjNH{
z|2=Lf)nwCNexrSREZN8hmXiqg%K7;lm_5+I3(gbJ+TlZCe;Lu78m%1$;;Wcux3Gk
zU@fkE`4|qZ`oJ&#l{8qxrm&4Zli>E`GTbhkf}$x2oYkPV?riG<%gt|5+_(Bpp~m{4=g%86}}8qHp87BK6ycXaYlK3ioNM|@QT^OINvL(1E^
zzF`H-MG(P`RsO@QUB~d!ubIs4<_OX}YsvOISK#Zd6?CxA6faG%q*2E6VE$A$TINvA
zX^xOX*QgDwEBOmO&`4yjp2f2Ex7{p6-wvESKZ9tHD_D!m!@1~WOm!!l0xOWVmguK8&2TOXhyA~7kBiU_bDdC9RWD(m-DWErT1dbRr0nQYau=bebt2wW+JzQhpQyk2a!?eG4EaZ9?>&%)$8Mcy;=P(aK
z|D-VWm_Xe2gQNY&#M%8LhdIOY+3e4@Fy?wC60EKc;Dq8Llt>y1|IU2Dr%scYTiIuf
zZ6A$?%WksWWs+=V>?!c6G=u@wNN!X@B7EEs0H1yVS1SC-m3UsnQy(^A^~W#>Q8~`3
zH07bG)qL#j%7DZgW&FAyF~He
zjSGDc%Gquo5RH0m#jo){&PUsCfwFf;@n~y4mo)sApk{eB*1r3S68*{5b~~ovds82%
zYMYGL?wo)^|7h@SOT$ZEE%2%1CK#Q222aMkPJ!z__3Kn78>5Pn&$8_W3RDz1GL-(Z*e(R=+#=
zKIpY5e08G8adry7HBpmuPs!sKtDB-z;C^VxC_#tar*XyEnfD`^YRn;;De)hj-5CJ4
zt`A|`52mq}A4%|c_!P*wE5oWCn^9+fzsRW43MM!da37>*Gq-Q1xTL)rWp4YBX5njo
zzr-AJp3()sA_d%v)HL2#KcDaLyN)_*)}vHFEe7)2$zg39gtsl{5(@ipZBZ`EFSyA@
z{Fp_R^M7(4!5{e>pOabYloBjHYlMe+sX<@y6>mJaKRaTU(JrrWTK)Kv=+yFuaI>}r
z1v4F(UMa8<%Cl(NOE>0bd>DV8h~Z)_s(5Z}0!m)^3D*owz<1tNcrq)JMq5bZ(94oE
zX-o~C3A~0o|NOu?HD^K5#|N^<9l_SX{p?!140x^D&m48GV`R~OaQ~yrOZ3Tb?)R+e
z*3)PB)lCV_tS7UQ{rf}>kN$%cnVtAc&0&x)DNx{t+3;wO61|CV<@7#8vC?O05G^?j
z!e?*eo`tGXT|gs9A8Exun@3X7AulfLoif|mydAXq&%$0!X?X8`5LCvc@>}PO8LaU$
z*v584vKzMp`#(=6L9r%1$^Oa}JQTsN{P}dJ{WzAdbs|ey7rQW#H4e;4!sLaG*gbYB
zKW&Ku%HKZ^y3gyesKJMimoUNYUHACH0SgTAm8CA#5O7}f9gmJUiAU?N;)%Cm%+6^i
za=}1}9X0Uxz9#oS44wBQm46h+?U@lGM2I4zA`$L$KB6R*LMfxP)VGq6N<*?DJ0!EA
zWHn?K?sGmWg-9A2NJ~j6Nh+nj&mVB_5BGka&vVXszu&L(?CCd1m($=c6*`Ll#V-R>
zrz~bdu{iu)KZQ9EuZS}^?|I0TFdQ2`gy*wV>3-WDSmLsj9%=7|i7oxOa+Vb~l%1*X
zStLz|Qf{HAt2suil%T54JNDfi7g*O=&FOEPugN@?46C+caNRNx-?oz3{&gl%f2e}p
zmxq{;@^Do8J(1i`mPCsTWx8)*wwB3
z*h4pB&`(7LnhlkxgJ%IFFd)T-@%U_NF9o&SP}qF6hOLs+K#d>jH0N_Etn#s7gRGtK
zZodugtdEC|O=Xa)@PTpIf0-{hD9qOTUBjQ|>UgpH8R$4>;7n@;TzfehJ41%>>-o7%
zmcUV%T<$})erMwLAMKnDT+8?tAAoatJIJe(G2~voIEX#Af<2~(sQlYka7z9%xVVKw
zW5y=1{<{J`Rc4Ymf+ta5RGw75{RatQYe@KQg@%D%WAZlo6p09wppj?PVD5uE)L#1m
zb5-~*Oj|gN`+6$ji_+PifF^Iu_bYAnMFc3Um-J=2U5$ARperX6MY@G
zj)*0VGaAyHVY9;)GS5?$cpNwi&vku>sh%4nYukas+WJIDr-Tl+Olr8WOOYJl`k@W`
z{2)g4H68mWLEJB1Wxf0oLC$U!-Tvbr7$#Mc7vvq0-kQ&z6^O;Auo`~!jQLcn@IR{I
z^_I+*Eujkv$Fc6yQT_v2GdOWlkfdA?pel}&s22YVIvg({2ZN)ChwlUw2vG#}mm=iy
zuQqtYog^VTx0buRcU{w6Frb6fzvOQBY&?v
zxuf`rl@BK-|sO|(G|Va#~BBU82ABK;YxN5-rXZ%o1tY&e+x~f$!dG(tG;CN
zpdf~ZirSJAD?-yt?&If2zzeO}L_I2JQf`O^N|7$y|8^Sv^3KE4bZL6v?KX1aRU>h8
zTZ8n|Bx1Y9hjlVsOcMHXP_An(^DMs`*VOrw)yvmX->nIB+UE-TqI{Imr5@Ni{0LOf
zg+k5>rg!m->VIi)x$O&-R9IlG
z@(~Q*8%*n;E0KSV;&j(LaV+%}r(%K^z^>*d*jgN8|8h0?Cga1b_ZKUg;Clg$ubYw8
zi6@Zi8{u1<8;nAzE&Y4=I-LCS3VeQM(GpD|+WU1s^$Iv{bFe7~Gqh9i`3gREbj{#d
zes5wAZxSaPy#8Rj;VD+DP99&)T*92Vo4{*|SE7SDEnvoZkR!Q}nd|lgX9){v|
zE2;mDv+TgFBUm`@g>@F!&|b$L!|xmiH}{E96c@%`JLyR-PFV^~X{*RSw@maNpA6}*
zv)Pcr7N#Q2fEHZJM%j&FsO=#__+ydm>z%)twY@vg{i6|#Y!IQ|Z8w<0fs=T=?F8#3
zX3A+J8{yQEojktJe0E-l7~MDXG;9vO%dWe69S0Tj>U)ctL3+Un$axtB(<}Yza%nxI
z|4<8)ehjhapZx_H^>^rdQ6HP0NK!YOkJ$3B+h%WI0_0RX(m<1N)JRh%Q$^dc;sbY=
ze_PK;eYt{pQzl^X%m7^2rc7LMICEhxD40STHw^hR%sZ
zt4m#sh}kz7f*!KwFA#s5Ss*}csEkS$tzBe@-*k29=p9wMD$aw1b~%%PQ&}jLK9#=A
zcO~=Qx$|0hS7GP40(l#kMy_X?z%oS&e*+>1WifVj-GpS2N1{v@;j?jMACPGgO9-Bv^S4uYw
zEPYhpbYKho`Y(-&PK_t>f4!JFnf+`L>!L^f1sq^{iuzpVwy3LzN3odU*f6-fv(cNOu|6U8)yNzi=@d$2g6lB(`I#Hq9
zC$Pr!KN|mXIvsSL!7SO#Vr|nQA`lemJmC)nE~Krg($LAoBTr&IExX|(c9NZ)Z5^x})j)SMu2(`#f7Ppt=m%UxK~_1Naz
z-an{9t#6rd&21`{rsk8Y@z2p@*J=#PnZwA0
zKj(VcA{>}~g4)y+&XXY0?=`0@<1pB=u7UeIGEJ?sn#~5EFXv@DV}!k5#9sCixJqa0~=GN?BX)1afEm
zJ8y9C9GPCIMhi~G!G^24EZdO9-!2eLhg@Eu`l{nd-b9kjr~>ws>`V|`mx*p8|G_pH
zQ)*EX0hiAU&}_pyjK#@Vawq3CoNsGjD(bv=A0Ii>^;g7*L>`yTSfoU2*GOXbI$aWP
zkqZ7%36NqJ2#GR>z%M8Ye>6p-;*J0^R6d7#cEsc2uu0T-@e*3NHV(Q@{R4;R(NwGF
zCGOmEj^w;@q3&uch+xxg7}btOlWr;eTRj_XcLdYurW9g5o(%O`*ZHrmAEKUT7t#B!
zYf1l${W$*}=V#LrfywbY#9C?;|0K%V5~UFmt^NRC7c0_akv=;2Za**Z$p~I}Do5i7
zOvv|wgQV%L5_OS$j*B8^(RvMzH+3$8Ja;YTv^7FjEy*EU8b=rl2R@v-@rrquaT`Px
zt}r7`*?2bK18G~BMzjR8X(>)+-y3{lE~R~?fg{rzAnX8Hr9O*E|7S-pNwv|r%Pq*`
z{D&}f{VUph-A9Z}-l0TGG>Kv>NtAFcJ+~v4&NLw;>G>$_db9zDY+d2?hXkq>a*_PE
ziOcZkTHyVO>tN5aV|2QgASqsU1lRp>AX9tYXkfb|S<_nsUrNKl=X7fL7+D|8b>3INyzIj*xxuni;FuMV~zhXuoUQ!Su)V`-TbV{Z=pH46|$FU
z&}6KlR
z=pmmpV&2!zvn||)1xl}&1hF}6#g11tAKzxN%XO0YW<81ZeG$^^rucH?t@z4R%j{u;
z&mX~M`!-;n(N}h5(_%((-!L}H&BWtY1WO_)_#5qELrXcn$txdl{#eR%xm@N)r_W=h
z*`pZO?gW{WLSei{2R`dM;bQ+OxN-Cgs86|V^Fee4^De%hU3@+W3X;Y!S#%);^_Anu
zdqq}j-UptAK^|UyvjM9HcNNJ5O$2q)jOjNv$IZ-xALa~JF`HX>9$G->5eV9
zWz`yZ66*r-Aq?bxIL>a;OlNfM&0xppLVjvQD^D@C6m@PMXAY%Igylv?His_nMwN%V
zVdfPHQ2!*uCe+14%4W_d@+}d5jkSP7$wSuhx*IXI;%#U^MRt
zQ(bw09TPaj^5e|#=l6UjX=e=%u?p<9&@m=tnkT9lTk)39N@RL9gz?R^Y4~*{6WbH-
zGD`K;VExsCoySzLO^2MBjj6|By`wWTuXh=yRam0G+g5ybLx=6*O=dzbcJY6PE3ul}
zbTHzWE>rb>(#ai(jUIjKp;7A*(b{f4&3A_Z}fj*=AW@E
zqd)BouGtyEZnu(Rw%s^Yf5rAGJbbSUS>H~f)xUVWkhlha>P>_llS+Ud{{~Uvb9pB7
zZ!$&$=B&z|4A8o|3a;+UX2-qG)axjBF!QWeFr{n_Y+XDH=V$K1kAc_NADz;)dhbn?
zD7S#ap7YqR9n&F7vI1wm3V`F`wxF+*iY0lYYzKEgYcKi%okK^Mz&q)zZMq7&Y#d|1
zhUl<)d{Nfta1T`dBq+AP1)fbVgvGN%*ztx!J@TDV
z3cbhwE~>yTd9xb)UWwy}xm6saYn>OlQRQIqFIt4t9Cs$BO+?FLi67V-zm`zg2
zLZR?2P+l+*{1^H$yVPXBK`|G_rp4goPf@I|t{N|Nts!YP&!aP6-3C&7B_9&PV-~d5XO#gt-WtH
z$*Z;|vUWnYPdv@Zn;kyH!)`ymS>=cgLW2;&sL_)3K{Rctwyj-LBvI$o#~Q=su%-M0
z)>_`72QI#)X;yKhN07@lsuob^uo(K3W79}UO(TlYKJ;ke7vic)>0)zR&H+rL@R8pyF6r&>_H*#z77J96o}IjR^uX)X$^Pp3a7oF`{rID>=D
zb})$7Xpp{NOo9S3Ax-fS|D1&dons$PRq-?;-2-
zCK~?xH;T$PMacD5X(GPk7ZjF9)12H|Sgm=VUR%0@c0I4gG07y1t;psDecMJt=XqjX
z~I)eT3OPH|XDajNZ02fJV+vJuzup=mrRp_{kzitT=cflkaJ+PKJs2l^qVH&uy
z?G#+9JrBy&%3vQ71fKI&u_j5w*rE{#w%t@292&jvWAixuZhxo%FJ!+cWS5FT*>V}OV)GxiGF}?_e;RS|W-Zva
zY61P-(*)&P_CkDtB6vnlrpIeOvUluaak6$FbMnXr=&Lekn`OPhMAsQkc#mLBS2b$P
zVZh*5GFvLShf!LvoH1LWhJ$xHk&x>sX1fF=D|f*CjA@X)n}BJC6|=D04d0)3AnG@p
zLA1yg1oJoZESB7aQnzuu*Bc7`?;rAUb_9<_Hk*;_yJFpP1msWx3
zkxl&VpH0{~8{DCkiok6TVGN(u$rJJ7BH#L&Je-jPBfSyKdd*vm+`_rc@aCC#+I|go
zW%=U-ib1Fl
zFqwC2PXbITqD=1`6YM{wkLl^RdA)ivbn)Osm~L_nN8i3>vm{qAQ!kjnpY55P@45<%
z)~2(D8#a-6$0+hoW&y-{pM`ac)#&hVck6poO~Jb87L(*INM>(QBO~>`#7*xWdqXRT
zsA*~tXCHY|@SkGCkDrqCa<3|>3>Bsu=k$P3i54|Vuz{wrIV8(zBHQt+hrMOmiFf|%
zLDj2Tq#<3Q;S8tiwkzq9wJ(Cm2hWRCd+ABixN{vnp&rS7C&$&=cZt&7H}C5%VpQ-qB}EyZz)RaS`NTdl@QvM?F}t&mb1Tme!z3p
zCa&J|B);cTIfmO!Qf_>UdEjRWi5}l^@P`Lec*~Ji*u5klg67bd_ZHC~8#x*_p*JY33qY2ya#@--05PO6^
zzSqIN_b;MH(tuPc)iRPq0K>Sx{O*)e(mk~jieF455|d)-5+e^d+`1~i7V{53M;{AccG{mn9Ib7}mxrnt=Y%hCY5rl-m7zlgF5d(vqcn&eGAAK{Nmx;2
zNehk6K>Dx%$|=f_p>LZ>y66;elG31OFFUj5?mI!ka5`$)O()Od1?cnPR2)k@O%A8t
z1@#XM8voe_P6fwM>`e&dPu&T({tYk>qTL`v_!IN${Rh1LHjn9DPS|_gZ|v8)4XjkI
z@l~1~VCLk#U{oCfu`vtTvd=T{eLVSx)V&rR@
zE=o<6#KL`Hcu>5P`Sj3(?$#-Qx8sSh=CTR!mRV7cg{K*@xP|D(F~wZzSgc*;M
z!CQufw0dM|!>%|=M?qWXtn$*httcYg=9PWT->@>RG=?`psI>c*!q(DYGli;mmBZeqn
zhF0IbXs6x9xWt?QIgy7rSYpJEWEY^=>bd}Gr8u5Dk!q>%+tahJ$s#XBkeqO+xi!EUaT@R6C
zQ(@AhmCVXTdTb!a?@IaA#&Hjy)*q7+Cp!%s&~#-yBhxVn#-suvN->d*wLcD~5gI77
zvkw#^?I6L`8uO2|LI+oW-BY>>1w3J*lX{(ZG;kq%_;MJ^ERkn+S!&RM+#gK4@G%UD
zlfpYEH!`0$ykl&Yj*u03$LP({{Y17~6vTen(p`oA_{QM|&Q4oOC;AnWw>8Efb-^BG
ze&tbF?}K!`xC+^*Xo1~}(y7-9EA}flYe`teJu@l#wlcbZ$%6Zt_~}OgeadU2%PyJn
zqo3N-FkKzWKg%N)>1Rj|?ZOkkjOk=y0Y-7)6x_-8CxT_iNOW5{8FM*8uIv{iS1uJX
z2Z!$QQ>0$QXO4;6x@;QF5tOsNc_xWe46}4xv4-hTd(O`DdrNy7_X
zmfn!KLN$$5$$=9x#1LoGUDA9qt`$H7i@DkjjuGoAy;OX`Ihtx3OOEu&qFls>3Qxf=Hgt>2s+8#oV~yJ
zG)Sx<@M>BktgzWg&wd+4?csYU8q!LCHHc$uEK==-lj)VjOlV3_rW&$ONkRTc$bT?O
zO0$&-t5t!^B**cEga_fWJUDnQkG@Yy#eesYug)`Xg8rRYE@C;ib0Z__E7<
z`Pxd{XP${tu@0wB^VmoVqs0A#=
zt(BJK?xRk&Wx+1uDg6NN*y{1Ww0y-%(O8mx`viTq{R%yQyMu0CqX>ULMPSCSiKJ#z
z9!y>CLbhM)p#wYxQjqo+jJRi4!0ssQICKX+F80CL=p61DnMQiqFdDd9u)%nS37K^K
zDlSBA>N4vA&dC*poty1JYV&Tw9`PZ&uWDic>n4&J5=;Z_rJ(rTTXtaDQXDTUr$4TF
za2zCkx>BKnIG^#tlYvF(sNaIVC1wyTyoG+@de@V&yCGIv862)x0W)bb*%Y`KcWn&i
zKA%XBKQSSnzpbQC_4VLFUfB$DgD-h8M#X=(Cwm*osJT-n0$LWNKy@b
z=jjPG|E6FobN4Y_A9#b5+$f?>3)j(o3Tezo?Z;G0xRk1WXaEEG4%|?Do)`3?59UXO
zP&c~bm*e2t>({RRH?1NZ1;(DI$sa0>p2eYB6*UZDM4n$cEj!O
zl0-%;g)Eoci=S-jScRAy^!~SXWKy_5gX4=9ESVii*IgGU#`5o&nw*LF)lP?K&sl>X
zrO!|U_o=kxNjCYtSeiRHlk7UQmxyXKaJ*$VD!6bp$A!BNiTqb=
z)^{87H)9pKwJZ`GbSW6UXu>9|Pz*3k0*~K3I6Px7bf-@zK4tD?=g=$ubR%Qp689O>
zLS#tDw_X~*MS*&5-viZ09$}Z69#Pu+4EyRAz|LuJ;Eql+-2Ht4t8dq$TDBNfua+cr
zdq%0Du_5vE6DCHMlS$ZkHZ881L^`b;sP>*QNQo$c>XJ_DhCD}dx_B}X))JtFyEdWP
z`+DHtOhZWN1c6)aSoJ=Jdd6?WNB-*6?95TvoRx=8smYMF`X{qjp%}*MJ?SUuV`MPa
zpJa8Wkzf1cnGK!yp?Pc~?J5!mW5+?fFi=Rmj_ISsb8Wiu^fH+Lel-gIz5(UCD)3RS
z40W$;!F|!HMEr9v>4>tVFMCgd^oLX^+7kqd%UgJ}MRIwd7C;5FzT%{r9lQ*wFlzQ-
z3Hee&Z4y-m_*>K?;6~mOQgghNnfY=z{5;`9SH}r6VS4xBtYi_mtbNO51gVn7`IVUT
z`4(ml?!Z$&)0hsiTqO4r7>S@F;72j!`y*lcOH-aW{S0KD-JVajk%SJuJDaA&HYA4{v+Dp3EwEx1$2k(kck4Gu43fC(vtKc4eRU0DNW_3guh+xU3U
zvJj>jCsDomTGD3@Y!5-Hv_Q5_?nmG0zzig9*>6044UY7HujXz@g-HyTiTt#Zs
z9Y_|+G{DpeS#W9b4SvEOMe@~Y4juM-3>U0*c(-+3U}hHBGtS|ejPs&6XmG#Bc-t?yx?VG1Li_LSdscM0rH4~ACjm8exL3iaRa;WD2YO!NIG@bSlErrG-$^SWv(
zNF7t>xD`Ke=6PK__jWP~ojDgt(-33m>tJzt6w{Mk2W6ql
z$em&xyvl2Z?=t1^=HD++c4?rycL(4tmt$~1@;vU!9ET-~b?DQQW=5#sI5^ZsfpS3^
z`wjxoR6~YFCtU&+rDi&;rcUfOjl;}RV}fy8snqr{P)wLf(}PRFp*;siPp0r*ZO|ls
z8{a{oOdW>Rc`$>)s&FI26-MAo)x-E!cUrdsudwynv@6k7C{e;5sUVUhM
z`-K0DwoU~OPVxH&vLyuuQ3g6dbJvKhVz-yW0G?#B!y^r%IzFGqv
z!v<*koa3gBMwrxMklt=+JKt;
z_Z6$XKQlHv{-a_Mrks}8z}kokvuza#(B)%8y!^TucJDH1Gh9a%CY<2iewYL2?jMEx
zDc2eQvhVQpQ#t0$9cIIX3ov%&VmvLJgoYa7{Ha>5V39qQ91Inr)4BZ*?>-GOhBX4D
z($5W(yHwEZ@f^5cnu2XICH&5aQ7G4)0n?im=#nkojBDpKklh}OdbXpmbx@p&&Pv5T
zWlK_($#Q+fBRYM`do-$vU^fUv(}_N3>8C<3R`G@jjln$fdhvNi_>2-YTbhH*ZM0z6
zI|&n2Z%{uqd#Ij#i({>9r@LAS_3^Q^o#@;RnQ|NGb=HB5yK%M5;c_lZ9l~mV>_BnH
zCKNaUoF8hGA%)8cPeh&$pId5M*ON+5U=jWOwux%5ctn0Q*HN2AspP?-Na`#yof^8l
zu+H9mo;DeV)3RU7A&~PFC(v5fc=JOz)Z;|UBv?{2Mb-8OF9zwjHz{&(!qtMsr=aW*Ah!#aL+?7W5A`c?FTN5MPo!;S5a?oZYZ;U>^CLDNdERI&8-CB~;v<0riTD
zAlmLju7<{w_e&q+#zWsoPHX{nmkfuDN2%*dHr=VCZ@;WWE5;vWoh{NwE4BXp*a6#Y`0M*b-((EpmJW7XM}p!*}8-UxOj
zL()#-V4kh5Mrw47_cw?(?BKb1bo{a$!0b_W{d5Mnd*^r{HiI$Y`rat
zm(3$^f)A%LuRn*Xwl&N{R+&18r=nnI9=`pvl+3=9%dl@mX=&+YP<1-b%Ef$TE{C3F
zcaL4e7mbz7qk>qdXxswh2FKaRPiwJbWe~~v$j!F-SJ;XE1-QIOki6bKgAOjJgV`1T
zP^DcJds^o)gU-Lu%C{CfwU@99egSL1%R#$OJ|H}th_Pm6AaAWn&MsJtIcsz|EtO^N
z_uYgNO9{Tk7I!$I=5dAr)9UgN;w|N9
zdrB-Aede^>6bVob%DUzo^PZn;yC445i9zY>$eR9
zzOo!Ew6fmV<`2lB0Fkuyh6xi5iDzUwD*P7WKVA8Rbx&_#K0laDyXQ%uzQ-xl-FzOd
zi1JwvohkTpVg$n5dZzi`Ka6zx!z&ij2HT^WSYx5el(x==I8jxc-?0i?xBkHZIe!={
zFailrUXZVM#v&sZko?jUbw@!Zo9)q&Nv3U
zX6kV@P)vPq-ATSB&kz*S+`;SLAn)OpKlq9(v%56sQHg&KAeG+;CtgaC7VRR`Y!Ctd
zg9~Wh`j@AEayk5s(PwC!H`MG_gxGfcQXbPms(S+Ei>#&-nLRwO
zRYiDND-QBA%5kN86C@Y~gXK9>tT{E#d~}{dG!x87+t(2Ym?zEKt1ij7=+DBd!3)sc
zOCKjj%_i$h7NUR8U7NA}Zcu8;`M1B;!Zc+WB!3F=;j3I+w<;eZ+t!hzQ)}wwlU+Hl
zd>?O1%XhfDe2YzP_Fwij$M$N;+lIGB=>o}mbu6sYzW;@#GyyPPCm-8M-_0FLWegyKp8_H>)a0;C#DFs|mmO
z(d;Xq$qr3(+w>pz@a}U@GFv|fmne0Si_1J{^MN94>Tbnyl@PLKmS979rhLPlSowzP
z?&+}RS0w+{Su5i5VjT%c$|u`eH52k!s&?&{32`l+2^<+WTm@Vg|j$`BMF_jeuc~9J#geJdxs0q0Zx?wicx~ss7eR
zXxV$TK3}g8*Ch>78`FNQx?)d@)F;^H@jGxVri{%g{(&2oC&1{22-?r_$F&bO@{apc
zn7?yB`7bk;$Q`O`LP9|vl+*k
z#;~@XyJ!8Js8N3nie*1$5ATx1u|IuK{o^e9Z-`{BmRw{u3*3R5kj=B3dXxR@Jb;%s
zzhlTFPwJ}I1&=f3!;`=4TMP&_t%bg&_4rrw
zG;6!shKVvvW4*S^LjSZl)RyDefA81gs~3o+V!lwIu0cosiq`#&2J*AfNhhMTiBA!`b4DLnAH5*IL1k?xIvc^^oU^9i+GiP=sfzG2il-{97Mz%;>4__z(H71@NIv&M1
z)y{*-*RoJ};sZ9M<_I_?zGSMVSK?8XQX8kw7nym_E6~(3435ryf^j`d(NKIjq{Ml_
zZPgAuB-71%GjxG9(0RjJI7edJ;XkbIjS^Pk@D3)jDIO1=bY}g&FU9zcfG;@ayB`S9
z^{W*4YqrYKmb5h-ze9kQADTu?ha)lKiXdJ6HwOgI&Bk3*_rr=Ooot4%5NfHWvHRzA
z!m8+FaPnLsKU<_5`Rc1k*2{3b@za=ha=d_*(-bAdFO#bPJGxIEQW7b?w2-ajTPg{ery@HI-+nKP@rW7upDaSXR!;IU+8&K?=
z3gK0X%!H3ZEcVM1BL@@K>)$T0I#`F|7ap(^22&t&-Bz6aBNAm_da*Zs5?R-M=@`{E
z3a$%V;DBNUM8>O=D>~eaW`Pqs+iVJ|&YgmK2{BxM6vVre9>nvH+6~$}Mj5%%3@jf-
z_UgGIp3puOdb4sN6Pd9Q?tFIVpIUW<-_34gf`8ee3s
z%|f5qgN(3Y5ARv!W1hX1IWGF=!(0=ahV+a*E|5}zdevCo7V(w5^lRSCi}^?KJ#Ua7
z>RiHKaceeRH7g3McEkWR6sMm`rV_Sn85&xd($R%B=xo`F`)`z^8)}jDeJRYVczg8n
z^`M33AylG2lI^LB;P}rRE3&N@d0ItmrCu62SusLlqIJnf=R(+iB%0>#-%MxJdf?Q=
zW%T;ddGucT8nQ%nGZB(0VMX>j(v@fK5S=^6u-&eZF&X!RLtXcY+1e4-zf+$a>Jz8u
zSI%S3
ztQj=$NHVPDRl&krF7({PX5wxbL;EZ~vt_>v@we&}>K_wKYX%}nvwk}D|Bwu;M0U`V
z>VMJ3ri_$*XvW>a8O$kx;QFj>3z1hmml$$9+jj%q{GJlVO^P-RIS^QL0$^==gr`__h_@UjO(>k}&U)cxQTe(8G>GyUba(o?BPtPUw
zZPDyBD#u$={fXVNYdgx|Rpc-DigvEt9yo&({B|IW+2NGIGj&_S$O-o_?_B~wCgT@Y
zR;Tlp%f%S@Y@L@UUpJ{C7dFNY#+k%Uz4u?}+2V3nv`^dMAxc4W?Tb8K~|Q1&pb&D9pm@i-i2aYfn;xRbzd(7I$G9Tx9FRL|`mLVC_ud#0%g_+al
z%bDwQ?(pL~^BEs+J6K=ufY&+gc-emuKpk^ge~W5(fzxoGe>Qw3=ONEWll|fFiIGf*
z!`?Ic{9tKWdafag^?PTIooe&g0gZUBm;1^0DM}>2d=Zvx&1I{Wc#*Y6hl%~B82A~4
z5d1)lR<(??%O53x_5mkQvaNwvj~EhV9znXi!r83nTyLr`OEKm9YILB4v^LQ!Zp*(@s
z*Ic5z=RanRH|WtVA*oQ(sKPNUPJ&Hx3{$we3)an#p*0$@WF+huF-fl_>t^&ZMW&Sy
zc14VqAJAcrIq6cD#M|U!bsPSQ1o{rmp-)8^3*S9};m^Hj@OuhP`SBb}B3F~&6W);$
zK96cRL#2$LURlk(A;~JJ+Sf={h_>(h+NWRlqDS4Q+MLP
z$3B8c_W>ByWy6DpBXrKc`HW_82h162f=97mnV-iN5rd(7xV`-u`XsJHnW9Uq<%AM+
zI#JDJmOtUDkqFG`*-R)+(-9Y|u351tKcLd(Jv;CfvNmCwY&n`jO
z!79{dYd0%!X$I7oRO2GoDP#qg@65axg8MEe6Uh*6@5Sb38vFh_%7mL^`8QvxyC{)5
z&dFg#bEW9W{$nKH+mXF;^bc;|mx(rm@~|RoBT5HcrnVI+U>g03+w--UPM0#L6PD@N
z9$PmDW?TxSLG2bqubbPYRhLg*2I;V!AIfmf+YnUEu*C_HGLRpqN2V5Wb$P-p+hr&B
z(en7Cl;?DfI-3=fcR8hWr}RP6d2=mom?BBve<|QgaGB~|)oba?L1eUZ#pr(z&$7#|
z)Pg$a*Gk-4O&<2h+McPc!ox9yxOr)#*o4z$##R|>{$T<}D&4@?kRWpR-vt`|=plW$
z`vX1hr3dmyzOzU41&P5adGaCU0118elfEX7#9j0(G-#V)?vw;8pKQbOeKp+&^!
z_CgXPn@#ig$TZAZBub{Yy~oFktmvjqGigaQK)zoVw9K&~=hq%1C0$W?NaPin<`PG-
zaU&%DE9AHjUO0!}NT=(c!VwA1=WY3j+&>nJf>q1$n@R^pFAspzvjga~)uffW9;Ju(vapMlQA`;Y!K$O4DWZxUWNpZpe^xA}m#6&X7eOeX$Tg8vc{!8FH=HgApK@w!%nva%X{&x!=nxD|{$RuEmWB;0UTk@R=;
z!yOog=4oTpv0NV)ol_&p3tBl=M+Q8a;7-Nwon!tT7aux3J`ZI(`r`;rz=2c_CGZm0@MA&3A8~zUFqC0oD-QTKBYZHQLT-rkBxD&_f
zPtzeH=~3)Sm8Wdf*Z`Aq`z$%^aEPkR3FmI{z=rSyo9ljc!>9{^+;5$G4JQX2#}Z5Bl>RlVOZ}RDvO^-
z^R+i|9p~X$n)R7RF!CI;SAsN;$PwS5Q}q5r19JSc7H#d>NXnu+;JFL3E1oSN6L*-A
z2@O4HD_Vr#*UOQguf%AoO9D{i_d1F6DGvHas>#vnwUQM%X$
zrGZzel=5EkGG_^Sq3utccLngJcRq&d^|Pq#F$x@s2@lDgBc{Kj*{|h#G_j=^%-TG0
zXU#!K7Es{7ysl1FCD)=`Z$I=&em>=Yt!xCs^lZp6TO}Is=PrD?Ax2Bb7C|cs2e~$HCi3(P
zh>6L7g*psYtj>eQEj4Ur+#9y}Vh+p|Rw({pdGWckn`Aj1Y!GdIlg4Mck{nG{OF~CvUeRo
zTwWB#zjE0d_h?LV>&AQY?&1V11@gTykt8&
zq8IS-ll%BHwGHQ7*C5+E5*R#6qZcKrE3ehuI4Z*rni5g9jOXMY6NI+o>MJq7@D7NW`mZ#gv=4pb|THzvCn@8lA(u#3%b)1gx?--qZkmsu%WMWSI5K=h~lWCHX?@XW)-c(pi-m-
z_aj%|AII&oA|opeBb9_`$@{zxMJhsjOQ~p*mQnV~<|DF7QAnwzGT!%fj7md;649U{
z4H{IGeDC)U;0M<|_nh;3J|7R+4*a4Jib8&~q@&mX$JhVhzSrnjeyk}W*8+EN_5I6m
zh2nXx;7$d1e6|u^xFe6J)60k~dBR%j>(vLRaLBewkOLy0r?B9^hA4dzu_$?;)u1iNNBOPvUQlr*J!rXHYEw-2`4yoYEpH&k1Ei_sA6;i;%5{>
zNWlp>pIOej%p%Y=B_AeSjm7tT9wowV0`pxxi!{tf_~j!ZU=yZ+kK#w}ulG01FdHBP
z3rg6IH9sKht^%DEu#wjDS*=}NMpRl~nwjpJMCZP`3|}7#xI@9R)U5p%boYGcBtx%5
za@#A?>E47-OoE_fa0BbQ{vUJe+X)kg%J9F;KzRRa4jXy!3OqUel8jaB7I-QFo2AOH
zz1RHcrlvjgugu%Z>AHiK$|h&fF2=gNGw3}E4rDsomW2Nop$5gb~72`2kz
zppK2aU}wQ54Aj1ZZZ}Wig5{ZT5)lNRYa2jUr3_wL&Axe3C
zAv^Bmp?Yo;xjQILU-agY-{KlPf4~Ylt+qpJf3NWP-S1#^`4=wvS;0GPrczh_883ZT
zL7Y|>kxP#wNQ3YLd9SlhV4$nYrC9G4Oqmdbr}sGV84!J#`msy+JhBvv3i&>;y#lGo
ze8D-MAH}whl}4*qKe&gg#*lD!AD=n3z=ad%0xj9gMW|lElvVe+rv{05bW5n;2hy;FT}gvMUPWg6eR?qzW?WkSIFX?#G!~1DuT6JW?nTFIeuN2E7iI99gjzMsMMB
zO+$6~h04(El|3YB?@Sma;))xWHVA`DjqsLCAqjM_#y3jZ5WId0pMTKiRvr~X<+nNT
zVAnD(t62-Ke-wr8ncKON8htMM<1Nl?FoC?9QUD85Oo_)=BWyP-g%$kWYZ>p%@w?v!
zw$HnXUQRmb>P&{92h+JTE+g=v%^hyJZx_jicbw+X4a=u~ncNul@!0WOM5q)$6VDf^
zV5!L)!6B&=n1A9T2Rf&q&o}_HzUE=i?A?~XeiQtd;eskZ1SmBC!G
z+}VIUJQ9=5JrO3+QhvX~trmxEmF9Hg7hAgNyA*f@hQQx6!uF2RCHZ__T{CnqRGZXu
zm11qeU%o48=I>HmZU2NU+!FzR&;A4JwKsUSAHpsRRl4zPIM%OR2*%zqOmBXs#goeE
zAbMFE7u`*U*rf@=3(rQgL*{?Tz|&mz>Y^-tpQ=c84*Jqie|c6IYeN6FDX{}A(K_$mioQmJ%Rt$hRv&mSg>T#wM5htp{QvjC^eXb7Lq_a;K}V0po1
zm}I*bHkpi|dz7?knEYA#R9*%p{Jg2wC4IVa<90f`b4u05>GNREk0|Purp6MAk|1?v
zK5L&i4R-6O(u`#_ID=>HX@#_7{=i?{`f(YZxJauiNSe^iMH)0}IGv7IbdgO`6;f}_
z0QTnZF*^H3DQWMR2S116XtU;O`X%T%_hpkJ^=zpIfd&6Nu36JH>BpE^pH$VH*(Gqf
zSB-5LnFE)mHZfB}p6zL5jk4uxv`6eBYc-usx78I>IlHfP!qQQ2YDttZX3t(69Mg(l
zc;<|kyol9M%^6nJxs&z1+fD3}7sBUP-=W~~2NbC{XOABBp*t4Sqf1KZ!@A$tIQ<5x
z+^0v2PR$1IMX4yd-WY#3u42j)tCJ
z)uQCaSvIgL8mccQ!@!NFpr4{jYWN(|)2U>t-X9_0_
zGr+x75At9EeLq>78NH6hXG>qw*b8@H$6I~&NnREvez=cbQF}?kPd&0E&Jk>CtYM(5
z7Y=m%M&}a_WXX(A!dH8f;mk&PDq<~(O2Pe5df*DYsNDj4t$j(;q85xDU&F%rKKi!=
zTP%s%L2X|1{ISe>_!7u}ccNF&B@!|$E7%dmZ5LzV_+l{ZSj~#BJb>ct&t%bZI}F?J
z&ps(kMB!p1d^Jabs%Y_NERS>C7LRn0X)J=xOV!vC<-v;b9GA93qYj~Z6}
zfi-0d*{#ZA{IoY+Xt~RRdVjpcMVh#Cn>89?&)#+H*`($4uEzs`!`(#geeomCBG-so
z7dbG~xT!QwelNU!Pzb-xHL1bZag5~8gWQ=HkvqOXaP)u!hvCs?4LQ
zX98L07=6rN6H)2Fx#IW23h@6iOl;QdV7fp0!ED2B7#Jx^GR(pO|KC4ae?@3~>L$_X
z|4eXH1r|6Sfie>nd?_tphLQ>CoSzmGE+%vhYR(@1^fr%}s8*0`;%>dC1gs
zQqboHoOeHZ`rqW_v<@>_mpGxx@-;Z_oVsAUQIFuyx@NLuy9=GOxDF&wF2QMWj`W1d
z9@@J09CY^aj%t%Al#MvaS>7~&+C~9cDQ`;Mo@LP9l*6>>p&!ih%cJjRThsYJ1oYco
zjjDeQaZK`IGvxN^GR0b1VKw@Z~d9o1ki*-mum
zxdapZk)Z+Q1H{TZ~~XD_Xo
zSV!0IzePlUji+`!l)W!H&%aydeDCZz&EEKx4%{T8`+kC)!(q1jofypGnbl7l-ol67#YEI}AN5c^jgJ2vW)g*Y
zq+(qY_ypGBan}?mdtgXYCdSb#w^iBhXlW>2sYkYmq|>)4>D<2j@odLqS-L(xgM5#j
zNxRO3P_ZNHAnz)o|Mn;Nsy>ITt}$=>IpL
z6Up?Xr?14*uOSg2{fi^msnZoeTeE6y@%06aEayyT%VTVND(^2~kS&+zKzI!kd=GZ?I;!oaH
z|6~XkkDbb9jeLi-*TZRA%Lf{FWjRY$41>}@X=<+XoA?F1<3#65RCaj8ve}!$*~L=_
zG4feCzvdjKV|L5Hz|1GyebH#x5tTzHKa7UbnA`BJJerA}xldZww1Qh)JB+;ef&BG6
zLRZ(lLScXm6P9UVj-65
zz+Hlr)+8?a!#4=+Ur7&E1%k4aHf!UQ@N&dl_-|b*YNs31j>n_esz1&sdgnUb+cE&Y
zU!3T_*;TOb+!XwhxPbbN-$c#lq{F)5J0MYzfmYvLaK`jQ@anfRpY44CB_+G5J8q+5
z9bH7uX+Jf2rAk9~>(fMMkt#L+)od=Gh0jvr`Iu|1=>qjA?5#}&@!gtyNA4>~4~%3I
zFTUcrW54lC38hXVqpNa_Evcc*Ncwk*G&KsTVyY+S(QS@fn5iwFpA@+OCyf&Y9o+}%
znHz+*@!Tf+R6aj7Rhe#nSdJU_eTVHLN71Arm>u|c35TnuvL)A6LGAscIOXD8sHkuz
zVfz&5j-qTPIx3Zp7_*D2b>!32G=aDc$jP@kMu2=#927(2Kg1+sFz$2*{FI7{C{m{Hwu=M+9uwm>L-MQ
zPkxYUV*#!IeHqni446l%2)=or2)1?w=pix%?w$Dx&-r?BEKo=!>B!s|F2FW1jZ3g_cN?*#Vz`w|-WJ%?7ge#X%=-=P27g{)a>
zFN2Us_B=Hi53Mw1mc#p)REaE8yJKTj+c?oGOL;H#t!x6-9b>ANFP5VYw;geAZVJm7
z<;^PcWNG$nTROGFg(j>lWBQL=tfUv8U=o1|Z2mXi{Zim)^}1pu)5>n<9U?_czRH=I
zy*vp^dhW0Zebp>i>pP7Pe+Su}f$S9b8lH_?&-Cp|1xxpv(=^`mxN~0&?KKKxFV~kr
z$+{8j@vNuN9g@h>9ptRy+N`TytwqKw1z6!%U-nBF#hT~*<(U^@?04}RdhMKH)ws@J
zcrR$9s+%L(@{c)S8KuTF4zyr(R~xm^KS0gGblE-0=qhikR$6)~AHSIEunTwnkQ*NZ
z|2`OT(4uIyMvIZ7WnpaI*#P>o`w7bn*bNugOoy2&s~1
zd>V?`-IjH1yF@6iC|=8w^hJ5*(@FY!XB!NMI5LTUk+k(w1D=$(Wl4Ey%&sVu>i>Do
zWX0c*n_AbH`w=0z6(52L+NrGkg$8{+Ih?JFzQsyz1!H}p5*?~Hqh4*TP&euu-S}FG
zovMqZYQDX&Sak*~JvW)fcb_3j;_*4`D9FEJgn$K*M*{!x*{|P>->1x1NE>lJQJu
z&kD)|&7dUSfRmeTPxnNc(Y7}e*?aBHY>^kGS>A{zd>!FQk*
zrjwHLEQoFK5iA-U5*p=Bq+cpZ*bS9rn!Qejb}gc`sO=!_tNmy}lfF9P%V%{M$$Pb4&&>mF6
zk<-KAS8qP~??wizK6jf-bK1=_H4Is4=oDJ?n8M^=t-?FE>dBKXeac5EQMJAiE-V)QannJcm`gA=Et2~+I4IoG9gWE~JfF$r5#D+e
z%lF}r!}5R{GTTcN-=6ltROj15`w5;Hd#WBCn>=CZJ_~GZxQ;3(zJh1OR$3)M4A8nk
zes^S8>gSB4+qdbUOXvl`l;`f8bNgvj`!xYpNylLAj48|~X9m}7x)9_=YH`J7Y1*^=
z5nR5o31|Dgg{8}iIJZ+L;igd|_g>ZaQmdUvcd*kc}2Kb}t
z3hBIj4CJkbh}gwaLE!g&+>gvcqH%!tiT)@c_F;!{xsD9kxMwug&Nd|yP9udc9*pCj
z9L~g*K1X5ZnR>y|^*eCs2pM8|?;OlLA|`krpUUTtpKyD9
z?i)Yi?77w4LWe-uJ1`b5t&yUqtBz91rBU>(Z!)nT6HVp7fF6{zrh}y_RqF>Pu_M-h
z;7ykzYdAQKx_lkR#zXO-=B`gWzb3-LUEk4r#t5e6KY#`sw$S4%##bp@&ZiYVs&u`p
z5e>GfXU~55(&GgIY;q~3@2|&!`1zw`nR_&yF29mWb&O!|UH(JhSBu*9O*m$!I=vZJ
zhtpP_XLjdT;jJWNR#A2slD{Z2_O}bz_Zn~7#A!jLTq377
z!4FK2#n6|wGa>f;HK>dVV$?5|EOX+|*p@HA`Dj1UQrS+68}8wVv?)xk;vIKqZ4Ig6
zZlcg?Ht2njqxTo``})87?C0K_AeJQwGdq1~jjRAyriw7L>UyAuO*s0n7etr2P+O&K
zV82F+O?u~#8g4S+y7U)j9M_{|{A`R8Yb*}Qr577SX}@bAxX6cLl*e7@-@1=W6RpL#
zI%Ry|xCxhUet@rBbD?U)GeSR)LPIfQCLE@0@vQF_BJcPd&;p)yGVV04w0X(}dQ&=?
z%cq*9IdIbN|IRHw7qghKNw1fn$YNdgJf|1UY|ZKJEMs~leg;}kk!BfY(R6s3GG6$%
z1bUwBg24J%`h1)@`iyFYb8Elir4jS+{TWx-`9z96{UXBd4K1WMv*yr|t^3)tqalzt
zbq|gxQe{hJnsKY*HwcK#=kp*hIMb3Y+&3->q%NO_Wg13o?W2=)=X!b8_oG_y?QbN`
zn3RthK_;v<(3m#JIx&UEs~~ZlESq}a3L5!`(5Dl4R$rD8*)x%MFe~lhZhF;%L;eMF
z$LO1I@N)#C#)s*L88*c5!3s1zD?w|rc2sBu=b~bB9zCfRi21soh=@G@i4;%7C5`iN
z@1xV4a_>)}W8^4YS|X_5W_6V9p{~I~)QRyJKPTZZ$Uj=?Sd*CIIhuN^HS_wfOY+
zbS8RZDwEs0j(gWW9zFLO(lIMvl2p;fvz3lImf7}nuY7wNlGio@FuXTa<=pmVQCm
zcahAV-%GgGudma2EA@4?A)2HQa3e2ebA@
z(wnz$vfDp=t#+ndX7_&GW}D)|dH1w~)%7Q7tjDdL%9(~`OuRNCDGsKPcqDIwbRdc%gA;r6hP7H8m-%a1Z
zrb!c7tG^`f5ox54bB|Mr4Xatk;kYUVrv|DtLxNe&v11`iMzK*v!O(l+9r-Huj~@Tq
z1Tj|OOmFiwI#l(EZ90+$jg`-d$0g%Y%**%!>GLCirp3e+A
zrlPlm7lZxEJR>rR9-GO#ISzloRq=^*SpODoe=NsrGB2?24}+-4@Gz?`e^hC;<}&Mh
zy#cmqE#Ns4o7qxTL%J(t5!?Ch4tr%&g_G6osJ2r+ESQKOrEs54oxGA+e$J-JHq+>{
z75=Q=M8HUH99MsT95S)ppzJUa&3K;0Nb%A5W|T8(8dB79d`(JsPa@e@`93U>V&f-I
zWMkH@!~U{=oSL;etNDJN_a%noty*uY_;Ne#{Tje6eozC8rYo?Yk8hy()FpW7|GenU
z#rPyuiDvP8ur2NhoKC|RoLhdDT3GV);^apHcw2G;!*rL#OZlr$aUA-v7phZ9ePf4;Y%?xWG7i)80
zH;b}l_v?7nYM*dQu_Svvwp}oENsd